@ebowwa/jsonl-hft 0.1.2 → 1.3.0

package/README.md

@@ -1,79 +1,299 @@
 # @ebowwa/jsonl-hft
 
-HFT-grade JSONL parser with sub-10µs latency.
+Generic HFT-grade JSONL parser with sub-microsecond latency.
 
-## Architecture
+**NO HARDCODED FIELDS** - Consumer defines what fields to extract.
 
-```
-┌─────────────────────────────────────────────────────────┐
-│  Node.js (control plane)                                │
-│    │                                                    │
-│    ▼ write (169ns)                                      │
-│  ┌─────────────────────────────────────────────────┐    │
-│  │  Shared Memory Ring Buffer (lock-free SPSC)    │    │
-│  │  • 64-byte cache-line aligned                   │    │
-│  │  • Zero-copy reads/writes                       │    │
-│  └─────────────────────────────────────────────────┘    │
-│    │                                                    │
-│    ▼ read (182ns)                                       │
-│  Rust Parser (data plane)                               │
-│    • Zero allocation                                    │
-│    • SIMD-friendly byte scanning                        │
-│    • Returns field offsets (not strings)                │
-└─────────────────────────────────────────────────────────┘
-```
+## Features
+
+- **Parallel Parsing**: Multi-core processing with rayon
+- **GZIP Support**: Read and write compressed files
+- **Schema Validation**: Validate JSONL against type schemas
+- **Type Inference**: Automatic field type detection
+- **Error Recovery**: Continue parsing on malformed lines
+- **Statistics**: Real-time parsing metrics
+- **Memory-Mapped I/O**: Efficient large file handling
+- **SIMD Optimized**: memchr-accelerated byte search (~19 GiB/s)
 
 ## Performance
 
-| Operation | Latency |
-|-----------|---------|
-| Ring Write | 169ns |
-| Ring Read | 182ns |
-| Full Round-trip | 1.05µs |
-| Throughput | 954K msgs/sec |
+| Benchmark | Time | Throughput |
+|-----------|------|------------|
+| parse_line | 232 ns | ~606 MiB/s |
+| find_field (first) | 15 ns | ~5 GiB/s |
+| find_field (last) | 58 ns | ~1.3 GiB/s |
+| pool_parser (1000 lines) | 113 µs | ~435 MiB/s |
+| memchr_search | 3.9 ns | ~19.7 GiB/s |
+
+## Installation
+
+```bash
+bun add @ebowwa/jsonl-hft
+```
 
 ## Usage
 
+### Basic Parsing
+
 ```typescript
-import { parse, extract, countLines, initRingBuffer, ringWrite, ringRead } from "@ebowwa/jsonl-hft";
+import { parseDir, parseFile, parseBuffer, getVersion } from "@ebowwa/jsonl-hft";
+
+// Define what fields you want to extract
+const fields = ["session_id", "timestamp", "role", "message.content"];
+
+// Parse a directory (recursive, parallel)
+const entries = parseDir("/path/to/jsonl/files", fields);
+
+// Parse a single file
+const fileEntries = parseFile("/path/to/file.jsonl", fields);
 
-// Direct parse (zero-copy)
-const data = Buffer.from('{"session_id":"abc","role":"user","content":"hello"}\n');
-const entries = parse(data);
+// Parse a buffer
+const bufferEntries = parseBuffer(jsonlBuffer, fields);
 
-// Extract fields using offsets
-for (const entry of entries) {
-  const sessionId = extract(data, entry.session_id_start, entry.session_id_end);
-  const role = extract(data, entry.role_start, entry.role_end);
-  console.log(sessionId, role);
+// Get version
+console.log(getVersion()); // "1.2.0"
+```
+
+### GZIP Support
+
+```typescript
+import { isGzip, parseGzipFile, writeGzip } from "@ebowwa/jsonl-hft";
+
+// Check if file is gzip compressed
+if (isGzip("/path/to/file.jsonl.gz")) {
+  // Parse compressed file
+  const entries = parseGzipFile("/path/to/file.jsonl.gz", fields);
 }
 
-// Ring buffer for HFT
-initRingBuffer(1024 * 1024); // 1MB
-ringWrite(data);
-const pending = ringPending();
-const readData = ringRead(data.length);
+// Write to gzip file
+const result = writeGzip("/path/to/output.jsonl.gz", jsonlData, 9);
+console.log(`Compression ratio: ${result.compressionRatio}`);
 ```
 
-## API
+### Parallel Parsing
 
-### Parse Functions
+```typescript
+import { parseFileParallel, parseFilesParallel, parseDirParallel } from "@ebowwa/jsonl-hft";
 
-- `parse(data: Buffer): EntryRef[]` - Parse JSONL, returns field offsets
-- `extract(data: Buffer, start: number, end: number): string` - Extract string using offsets
-- `countLines(data: Buffer): number` - Count lines (minimal work)
+// Parse single file with parallel chunks
+const result = parseFileParallel("/path/to/large.jsonl", fields, 0); // 0 = auto chunk size
+console.log(`Parsed ${result.linesProcessed} lines in ${result.parseTimeNs}ns`);
 
-### Ring Buffer Functions
+// Parse multiple files in parallel
+const files = ["/path/a.jsonl", "/path/b.jsonl", "/path/c.jsonl"];
+const multiResult = parseFilesParallel(files, fields);
 
-- `initRingBuffer(capacity: number): void` - Initialize shared ring buffer
-- `ringWrite(data: Buffer): number` - Write to buffer (returns bytes written)
-- `ringRead(maxLen: number): Uint8Array` - Read from buffer
-- `ringPending(): number` - Get pending bytes
+// Parse directory in parallel
+const dirResult = parseDirParallel("/path/to/jsonl/dir", fields);
+```
+
+### Schema Validation
+
+```typescript
+import { validateFile, FieldType } from "@ebowwa/jsonl-hft";
+
+const schema = [
+  { name: "session_id", expectedType: FieldType.String, required: true },
+  { name: "timestamp", expectedType: FieldType.String, required: true },
+  { name: "value", expectedType: FieldType.Number, required: false },
+];
+
+const validation = validateFile("/path/to/data.jsonl", schema);
+if (!validation.isValid) {
+  console.log(`Found ${validation.errorCount} errors`);
+  validation.errors.forEach(err => {
+    console.log(`Line ${err.lineNumber}: ${err.errorMessage}`);
+  });
+}
+```
+
+### Type Inference
+
+```typescript
+import { inferFieldTypes, FieldType } from "@ebowwa/jsonl-hft";
+
+const line = Buffer.from('{"id":"123","count":42,"active":true}');
+const fields = ["id", "count", "active"];
+const types = inferFieldTypes(line, fields);
+
+// types[0] = FieldType.String
+// types[1] = FieldType.Number
+// types[2] = FieldType.Boolean
+```
+
+### Statistics
+
+```typescript
+import { getStats, resetStats, parseFileWithStats } from "@ebowwa/jsonl-hft";
+
+// Parse with stats collection
+const result = parseFileWithStats("/path/to/file.jsonl", fields);
+
+// Get global stats
+const stats = getStats();
+console.log(`Throughput: ${stats.throughputMiBs} MiB/s`);
+console.log(`Avg latency: ${stats.avgLatencyNs} ns`);
+
+// Reset stats
+resetStats();
+```
+
+### Error Recovery
+
+```typescript
+import { parseFileWithRecovery } from "@ebowwa/jsonl-hft";
+
+const result = parseFileWithRecovery("/path/to/dirty.jsonl", fields);
+console.log(`Successful: ${result.stats.successfulLines}`);
+console.log(`Failed: ${result.stats.failedLines}`);
+
+// Access errors
+result.errors.forEach(err => {
+  console.log(`Line ${err.lineNumber}: ${err.errorType}`);
+});
+```
+
+### Batch Parsing
+
+```typescript
+import { parseBatch, freeBatch } from "@ebowwa/jsonl-hft";
+
+const buffers = [buf1, buf2, buf3];
+const lengths = [buf1.length, buf2.length, buf3.length];
+const result = parseBatch(buffers, lengths, fields);
+console.log(`Parsed ${result.count} entries`);
+
+// Cleanup
+freeBatch(result);
+```
+
+### Streaming API
+
+```typescript
+import { parseStream } from "@ebowwa/jsonl-hft";
+
+// Process each line with a callback
+parseStream("/path/to/large.jsonl", fields, (entry, lineNumber) => {
+  // Process entry
+  console.log(`Line ${lineNumber}:`, entry);
+  return true; // Continue processing
+});
+```
+
+## Field Specification
+
+Fields can be:
+- **Simple**: `"session_id"`, `"timestamp"`, `"role"`
+- **Nested (dot notation)**: `"message.content"`, `"metadata.user.id"`
+
+The parser extracts only the fields you request - no wasted parsing.
+
+## API Reference
+
+### Parsing Functions
+| Function | Description |
+|----------|-------------|
+| `parseDir` | Parse all JSONL files in directory recursively |
+| `parseFile` | Parse single file with memory-mapped I/O |
+| `parseBuffer` | Parse from buffer/string |
+| `parseFileParallel` | Parallel file parsing with chunks |
+| `parseFilesParallel` | Parse multiple files in parallel |
+| `parseDirParallel` | Parallel directory parsing |
+| `parseGzipFile` | Parse gzip-compressed file |
+| `parseFileWithStats` | Parse with statistics collection |
+| `parseFileWithRecovery` | Parse with error recovery |
+| `parseBatch` | Parse multiple buffers |
+| `parseStream` | Streaming callback-based parsing |
+
+### Validation Functions
+| Function | Description |
+|----------|-------------|
+| `validateFile` | Validate JSONL against schema |
+| `inferFieldTypes` | Infer types for fields |
+
+### Output Functions
+| Function | Description |
+|----------|-------------|
+| `writeGzip` | Write to gzip-compressed file |
+| `writeFile` | Write to uncompressed file |
+
+### Utility Functions
+| Function | Description |
+|----------|-------------|
+| `getVersion` | Get library version |
+| `isGzip` | Check if file is gzip compressed |
+| `getStats` | Get global parsing statistics |
+| `resetStats` | Reset statistics counters |
+
+### Types
+
+```typescript
+enum FieldType {
+  Unknown = 0,
+  String = 1,
+  Number = 2,
+  Boolean = 3,
+  Null = 4,
+  Array = 5,
+  Object = 6,
+}
+
+interface StatsResult {
+  totalLines: bigint;
+  successfulLines: bigint;
+  failedLines: bigint;
+  totalBytes: bigint;
+  parseTimeNs: bigint;
+  avgLatencyNs: bigint;
+  minLatencyNs: bigint;
+  maxLatencyNs: bigint;
+  throughputMiBs: number;
+}
+
+interface ParallelResult {
+  entries: GenericEntry[];
+  count: number;
+  linesProcessed: bigint;
+  parseTimeNs: bigint;
+}
+
+interface WriteResult {
+  success: boolean;
+  bytesWritten: bigint;
+  compressedBytes: bigint;
+  compressionRatio: number;
+  errorMessage?: string;
+}
+```
 
 ## Build
 
 ```bash
-cd packages/src/jsonl-hft
-CARGO_TARGET_DIR=target cargo build --release
-bun run benchmark.ts
+cd packages/src/rust/jsonl-hft
+bun run build
 ```
+
+## Architecture
+
+```
+┌─────────────────────────────────────────────────────────┐
+│  TypeScript (control plane)                             │
+│    │                                                    │
+│    ▼ FFI call                                           │
+│  ┌─────────────────────────────────────────────────┐    │
+│  │  Rust Parser (data plane)                        │    │
+│  │  • Memory-mapped I/O (memmap2)                   │    │
+│  │  • Parallel processing (rayon)                   │    │
+│  │  • Zero allocation hot path                      │    │
+│  │  • SIMD-friendly byte scanning (memchr)          │    │
+│  │  • GZIP compression (flate2/zlib-ng)             │    │
+│  │  • Schema validation                             │    │
+│  │  • Type inference                                │    │
+│  │  • Statistics collection                         │    │
+│  └─────────────────────────────────────────────────┘    │
+└─────────────────────────────────────────────────────────┘
+```
+
+## License
+
+MIT

package/dist/index.d.ts

@@ -1,49 +1,96 @@
 /**
  * @ebowwa/jsonl-hft
- * HFT-grade JSONL parser with sub-10µs latency
+ *
+ * Generic HFT-grade JSONL parser with sub-10µs latency.
+ * NO HARDCODED FIELDS - consumer defines what fields to extract.
+ *
+ * @example
+ * ```ts
+ * import { parseDir, parseFile, parseBuffer } from "@ebowwa/jsonl-hft";
+ *
+ * // Define what fields you want to extract
+ * const fields = ["session_id", "timestamp", "role", "content"];
+ *
+ * // Parse a directory
+ * const entries = parseDir("/path/to/jsonl/files", fields);
+ * // entries: Array<{ session_id: string; timestamp: string; role: string; content: string }>
+ *
+ * // Parse a single file
+ * const fileEntries = parseFile("/path/to/file.jsonl", fields);
+ *
+ * // Parse a buffer
+ * const bufferEntries = parseBuffer(jsonlBuffer, fields);
+ * ```
  */
-export interface EntryRef {
-    session_id_start: number;
-    session_id_end: number;
-    timestamp_start: number;
-    timestamp_end: number;
-    role_start: number;
-    role_end: number;
-    content_start: number;
-    content_end: number;
-}
 /**
- * Convenience type for fully parsed entries
+ * Field specification for extraction
+ * Can be a simple field name or nested path (e.g., "message.content")
+ */
+export type FieldSpec = string;
+/**
+ * Generic entry - a record of field name to string value
+ * When using parseDir, includes source_file metadata
  */
-export interface ParsedEntry {
-    session_id: string;
-    timestamp: string;
-    role: string;
-    content: string;
+export type GenericEntry = Record<string, string> & {
     source_file?: string;
+};
+/**
+ * Parse result with metadata
+ */
+export interface ParseResult<T extends GenericEntry = GenericEntry> {
+    entries: T[];
+    parseTimeMs: number;
+    entryCount: number;
 }
 /**
- * Count lines in buffer
+ * Get the library version
  */
-export declare function countLines(data: Buffer | Uint8Array): number;
+export declare function getVersion(): string;
 /**
- * Parse JSONL data into entries
+ * Parse all JSONL files in a directory with custom field extraction.
+ * Files are parsed in parallel using rayon.
+ *
+ * @param dirPath - Path to directory containing .jsonl files (recursive)
+ * @param fields - Array of field names to extract (supports nested paths like "message.content")
+ * @returns Array of entries with the requested fields
+ *
+ * @example
+ * ```ts
+ * // Extract specific fields
+ * const entries = parseDir("/data/logs", ["timestamp", "level", "message"]);
+ *
+ * // Nested field extraction
+ * const nested = parseDir("/data/api", ["request.id", "response.status", "duration_ms"]);
+ * ```
  */
-export declare function parse(data: Buffer | Uint8Array): EntryRef[];
+export declare function parseDir<T extends GenericEntry = GenericEntry>(dirPath: string, fields: FieldSpec[]): T[];
 /**
- * Extract string from buffer using offsets
+ * Parse a single JSONL file with custom field extraction.
+ * Uses memory-mapped I/O for efficiency.
+ *
+ * @param filePath - Path to the JSONL file
+ * @param fields - Array of field names to extract
+ * @returns Array of entries with the requested fields
  */
-export declare function extract(data: Buffer | Uint8Array, start: number, end: number): string;
+export declare function parseFile<T extends GenericEntry = GenericEntry>(filePath: string, fields: FieldSpec[]): T[];
 /**
- * Initialize ring buffer
+ * Parse a buffer/string of JSONL data with custom field extraction.
+ *
+ * @param input - Buffer or string containing JSONL data
+ * @param fields - Array of field names to extract
+ * @returns Array of entries with the requested fields
+ */
+export declare function parseBuffer<T extends GenericEntry = GenericEntry>(input: Buffer | Uint8Array | string, fields: FieldSpec[]): T[];
+/**
+ * Initialize a ring buffer for streaming data
  */
 export declare function initRingBuffer(capacity: number): Uint8Array;
 /**
- * Write to ring buffer
+ * Write data to the ring buffer
  */
 export declare function ringWrite(data: Buffer | Uint8Array): number;
 /**
- * Read from ring buffer
+ * Read data from the ring buffer
  */
 export declare function ringRead(maxLen: number): Uint8Array;
 /**
@@ -51,36 +98,29 @@ export declare function ringRead(maxLen: number): Uint8Array;
  */
 export declare function ringPending(): number;
 /**
- * Parse a single JSONL file
- * @param path - Path to the JSONL file
- * @returns Array of EntryRef objects with field offsets
+ * Preset field specs for Claude Code history format
+ * Use: parseDir(path, CLAUDE_CODE_FIELDS)
  */
-export declare function parseFile(path: string): EntryRef[];
+export declare const CLAUDE_CODE_FIELDS: FieldSpec[];
 /**
- * Parse all JSONL files in a directory
- * @param dirPath - Path to the directory containing JSONL files
- * @returns Object with entries array and fileOffsets map tracking source files
+ * Preset for trading/market data
  */
-export declare function parseDir(dirPath: string): {
-    entries: EntryRef[];
-    fileOffsets: Map<string, number>;
-};
+export declare const TRADE_FIELDS: FieldSpec[];
 /**
- * Parse directory and return fully parsed objects
- * @param dirPath - Path to the directory containing JSONL files
- * @returns Array of ParsedEntry objects with all fields deserialized
+ * Preset for log file parsing
  */
-export declare function parseDirFast(dirPath: string): ParsedEntry[];
+export declare const LOG_FIELDS: FieldSpec[];
 declare const _default: {
-    countLines: typeof countLines;
-    parse: typeof parse;
-    extract: typeof extract;
+    getVersion: typeof getVersion;
+    parseDir: typeof parseDir;
+    parseFile: typeof parseFile;
+    parseBuffer: typeof parseBuffer;
     initRingBuffer: typeof initRingBuffer;
     ringWrite: typeof ringWrite;
     ringRead: typeof ringRead;
     ringPending: typeof ringPending;
-    parseFile: typeof parseFile;
-    parseDir: typeof parseDir;
-    parseDirFast: typeof parseDirFast;
+    CLAUDE_CODE_FIELDS: string[];
+    TRADE_FIELDS: string[];
+    LOG_FIELDS: string[];
 };
 export default _default;

package/dist/index.js

@@ -3,26 +3,28 @@
 import { dlopen, suffix, ptr } from "bun:ffi";
 import { join, dirname } from "path";
 import { fileURLToPath } from "url";
-import { readdirSync } from "fs";
 var __dirname2 = dirname(fileURLToPath(import.meta.url));
 var libPath = join(__dirname2, "..", "native", `libjsonl_hft.${suffix}`);
-var ENTRY_SIZE = 32;
 var lib = dlopen(libPath, {
-  count_lines: {
-    args: ["ptr", "usize"],
-    returns: "u32"
+  jsonl_parse_dir_generic: {
+    args: ["ptr", "ptr", "usize"],
+    returns: "cstring"
   },
-  jsonl_parse_batch: {
+  jsonl_parse_file_generic: {
+    args: ["ptr", "ptr", "usize"],
+    returns: "cstring"
+  },
+  jsonl_parse_buffer_generic: {
     args: ["ptr", "usize", "ptr", "usize"],
-    returns: "usize"
+    returns: "cstring"
   },
-  extract_field: {
-    args: ["ptr", "usize", "u32", "u32", "ptr", "usize"],
-    returns: "usize"
+  jsonl_free_string: {
+    args: ["ptr"],
+    returns: "void"
   },
-  entry_size: {
+  jsonl_version: {
     args: [],
-    returns: "usize"
+    returns: "cstring"
   },
   ring_init: {
     args: ["usize"],
@@ -39,49 +41,77 @@ var lib = dlopen(libPath, {
   ring_pending: {
     args: [],
     returns: "i32"
-  },
-  jsonl_parse_file: {
-    args: ["ptr", "ptr", "usize"],
-    returns: "usize"
-  },
-  jsonl_parse_dir: {
-    args: ["ptr", "ptr", "usize"],
-    returns: "usize"
-  },
-  jsonl_parse_dir_serialized: {
-    args: ["ptr"],
-    returns: "cstring"
   }
 });
-function countLines(data) {
-  return Number(lib.symbols.count_lines(ptr(data), BigInt(data.length)));
-}
-function parse(data) {
-  const maxEntries = Math.ceil(data.length / 50);
-  const outBuf = new Uint8Array(maxEntries * ENTRY_SIZE);
-  const count = Number(lib.symbols.jsonl_parse_batch(ptr(data), BigInt(data.length), ptr(outBuf), BigInt(maxEntries)));
-  const view = new DataView(outBuf.buffer, outBuf.byteOffset);
-  const results = [];
-  for (let i = 0;i < count; i++) {
-    const offset = i * ENTRY_SIZE;
-    results.push({
-      session_id_start: view.getUint32(offset, true),
-      session_id_end: view.getUint32(offset + 4, true),
-      timestamp_start: view.getUint32(offset + 8, true),
-      timestamp_end: view.getUint32(offset + 12, true),
-      role_start: view.getUint32(offset + 16, true),
-      role_end: view.getUint32(offset + 20, true),
-      content_start: view.getUint32(offset + 24, true),
-      content_end: view.getUint32(offset + 28, true)
+function prepareFieldSpecs(fields) {
+  const buffers = [];
+  const specs = [];
+  for (const field of fields) {
+    const buf = new TextEncoder().encode(field);
+    buffers.push(buf);
+    specs.push({
+      name: ptr(buf),
+      name_len: buf.length
     });
   }
-  return results;
+  return { specs, buffers };
+}
+function parseFFIResult(jsonStr) {
+  if (!jsonStr || jsonStr === "[]") {
+    return [];
+  }
+  return JSON.parse(jsonStr);
+}
+function getVersion() {
+  return lib.symbols.jsonl_version() || "unknown";
 }
-function extract(data, start, end) {
-  const len = end - start;
-  const outBuf = new Uint8Array(len);
-  const actualLen = Number(lib.symbols.extract_field(ptr(data), BigInt(data.length), start, end, ptr(outBuf), BigInt(len)));
-  return new TextDecoder().decode(outBuf.slice(0, actualLen));
+function parseDir(dirPath, fields) {
+  if (!dirPath || fields.length === 0) {
+    return [];
+  }
+  const { specs, buffers } = prepareFieldSpecs(fields);
+  const specsBuffer = new Uint8Array(specs.length * 16);
+  const specsView = new DataView(specsBuffer.buffer);
+  for (let i = 0;i < specs.length; i++) {
+    const offset = i * 16;
+    specsView.setBigUint64(offset, BigInt(specs[i].name), true);
+    specsView.setBigUint64(offset + 8, BigInt(specs[i].name_len), true);
+  }
+  const pathBytes = new TextEncoder().encode(dirPath);
+  const jsonStr = lib.symbols.jsonl_parse_dir_generic(ptr(pathBytes), ptr(specsBuffer), BigInt(fields.length));
+  return parseFFIResult(jsonStr);
+}
+function parseFile(filePath, fields) {
+  if (!filePath || fields.length === 0) {
+    return [];
+  }
+  const { specs, buffers } = prepareFieldSpecs(fields);
+  const specsBuffer = new Uint8Array(specs.length * 16);
+  const specsView = new DataView(specsBuffer.buffer);
+  for (let i = 0;i < specs.length; i++) {
+    const offset = i * 16;
+    specsView.setBigUint64(offset, BigInt(specs[i].name), true);
+    specsView.setBigUint64(offset + 8, BigInt(specs[i].name_len), true);
+  }
+  const pathBytes = new TextEncoder().encode(filePath);
+  const jsonStr = lib.symbols.jsonl_parse_file_generic(ptr(pathBytes), ptr(specsBuffer), BigInt(fields.length));
+  return parseFFIResult(jsonStr);
+}
+function parseBuffer(input, fields) {
+  if (!input || fields.length === 0) {
+    return [];
+  }
+  const data = typeof input === "string" ? new TextEncoder().encode(input) : input;
+  const { specs, buffers } = prepareFieldSpecs(fields);
+  const specsBuffer = new Uint8Array(specs.length * 16);
+  const specsView = new DataView(specsBuffer.buffer);
+  for (let i = 0;i < specs.length; i++) {
+    const offset = i * 16;
+    specsView.setBigUint64(offset, BigInt(specs[i].name), true);
+    specsView.setBigUint64(offset + 8, BigInt(specs[i].name_len), true);
+  }
+  const jsonStr = lib.symbols.jsonl_parse_buffer_generic(ptr(data), BigInt(data.length), ptr(specsBuffer), BigInt(fields.length));
+  return parseFFIResult(jsonStr);
 }
 function initRingBuffer(capacity) {
   const ptr2 = lib.symbols.ring_init(capacity);
@@ -99,87 +129,49 @@ function ringRead(maxLen) {
 function ringPending() {
   return lib.symbols.ring_pending();
 }
-function parseFile(path) {
-  const pathBytes = new TextEncoder().encode(path);
-  const maxEntries = 1e5;
-  const outBuf = new Uint8Array(maxEntries * ENTRY_SIZE);
-  const count = Number(lib.symbols.jsonl_parse_file(ptr(pathBytes), ptr(outBuf), BigInt(maxEntries)));
-  const view = new DataView(outBuf.buffer, outBuf.byteOffset);
-  const results = [];
-  for (let i = 0;i < count; i++) {
-    const offset = i * ENTRY_SIZE;
-    results.push({
-      session_id_start: view.getUint32(offset, true),
-      session_id_end: view.getUint32(offset + 4, true),
-      timestamp_start: view.getUint32(offset + 8, true),
-      timestamp_end: view.getUint32(offset + 12, true),
-      role_start: view.getUint32(offset + 16, true),
-      role_end: view.getUint32(offset + 20, true),
-      content_start: view.getUint32(offset + 24, true),
-      content_end: view.getUint32(offset + 28, true)
-    });
-  }
-  return results;
-}
-function parseDir(dirPath) {
-  const pathBytes = new TextEncoder().encode(dirPath);
-  const maxEntries = 1e6;
-  const outBuf = new Uint8Array(maxEntries * ENTRY_SIZE);
-  const count = Number(lib.symbols.jsonl_parse_dir(ptr(pathBytes), ptr(outBuf), BigInt(maxEntries)));
-  const view = new DataView(outBuf.buffer, outBuf.byteOffset);
-  const entries = [];
-  for (let i = 0;i < count; i++) {
-    const offset = i * ENTRY_SIZE;
-    entries.push({
-      session_id_start: view.getUint32(offset, true),
-      session_id_end: view.getUint32(offset + 4, true),
-      timestamp_start: view.getUint32(offset + 8, true),
-      timestamp_end: view.getUint32(offset + 12, true),
-      role_start: view.getUint32(offset + 16, true),
-      role_end: view.getUint32(offset + 20, true),
-      content_start: view.getUint32(offset + 24, true),
-      content_end: view.getUint32(offset + 28, true)
-    });
-  }
-  const fileOffsets = new Map;
-  const files = readdirSync(dirPath).filter((f) => f.endsWith(".jsonl"));
-  let currentOffset = 0;
-  for (const file of files) {
-    fileOffsets.set(file, currentOffset);
-    currentOffset += Math.ceil(entries.length / files.length);
-  }
-  return { entries, fileOffsets };
-}
-function parseDirFast(dirPath) {
-  const pathBytes = new TextEncoder().encode(dirPath);
-  const jsonResult = lib.symbols.jsonl_parse_dir_serialized(ptr(pathBytes));
-  if (!jsonResult) {
-    return [];
-  }
-  return JSON.parse(jsonResult);
-}
+var CLAUDE_CODE_FIELDS = [
+  "sessionId",
+  "timestamp",
+  "type",
+  "message.content"
+];
+var TRADE_FIELDS = [
+  "timestamp",
+  "symbol",
+  "side",
+  "price",
+  "quantity"
+];
+var LOG_FIELDS = [
+  "timestamp",
+  "level",
+  "message",
+  "source"
+];
 var src_default = {
-  countLines,
-  parse,
-  extract,
+  getVersion,
+  parseDir,
+  parseFile,
+  parseBuffer,
   initRingBuffer,
   ringWrite,
   ringRead,
   ringPending,
-  parseFile,
-  parseDir,
-  parseDirFast
+  CLAUDE_CODE_FIELDS,
+  TRADE_FIELDS,
+  LOG_FIELDS
 };
 export {
   ringWrite,
   ringRead,
   ringPending,
   parseFile,
-  parseDirFast,
   parseDir,
-  parse,
+  parseBuffer,
   initRingBuffer,
-  extract,
+  getVersion,
   src_default as default,
-  countLines
+  TRADE_FIELDS,
+  LOG_FIELDS,
+  CLAUDE_CODE_FIELDS
 };

package/package.json

@@ -1,7 +1,8 @@
 {
   "name": "@ebowwa/jsonl-hft",
-  "version": "0.1.2",
-  "description": "HFT-grade JSONL parser with sub-10µs latency via shared memory ring buffer",
+  "version": "1.3.0",
+  "author": "ebowwa",
+  "description": "Generic HFT-grade JSONL parser - NO hardcoded fields, consumer defines what to extract. Sub-10µs latency with memchr SIMD optimization. Parallel parsing, gzip support, schema validation.",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
   "exports": {
@@ -12,17 +13,24 @@
   },
   "scripts": {
     "build": "CARGO_TARGET_DIR=target cargo build --release && bun build ./src/index.ts --outdir ./dist --target=bun && tsc --emitDeclarationOnly --declaration --outDir ./dist",
+    "build:rust": "CARGO_TARGET_DIR=target cargo build --release",
     "test": "bun test",
-    "bench": "bun run benchmark.ts && bun run benchmark-single.ts"
+    "bench": "cargo bench"
   },
   "keywords": [
     "jsonl",
     "parser",
+    "generic",
     "hft",
     "low-latency",
+    "configurable-fields",
+    "zero-copy",
     "simd",
-    "ring-buffer",
-    "zero-copy"
+    "memchr",
+    "parallel",
+    "gzip",
+    "schema-validation",
+    "type-inference"
   ],
   "license": "MIT",
   "files": [
@@ -34,10 +42,15 @@
     "bun-types": "^1.3.9"
   },
   "ownership": {
-    "domain": "quant",
+    "domain": "parsing",
     "responsibilities": [
-      "high-frequency-trading",
-      "jsonl-protocol"
+      "generic-jsonl-parsing",
+      "high-performance-parsing",
+      "field-extraction",
+      "simd-optimization",
+      "parallel-processing",
+      "gzip-compression",
+      "schema-validation"
     ]
   }
 }