@ebowwa/jsonl-hft 0.1.1 → 1.0.0

package/README.md

@@ -1,79 +1,116 @@
 # @ebowwa/jsonl-hft
 
-HFT-grade JSONL parser with sub-10µs latency.
+Generic HFT-grade JSONL parser with sub-10us latency.
 
-## Architecture
-
-```
-┌─────────────────────────────────────────────────────────┐
-│  Node.js (control plane)                                │
-│    │                                                    │
-│    ▼ write (169ns)                                      │
-│  ┌─────────────────────────────────────────────────┐    │
-│  │  Shared Memory Ring Buffer (lock-free SPSC)    │    │
-│  │  • 64-byte cache-line aligned                   │    │
-│  │  • Zero-copy reads/writes                       │    │
-│  └─────────────────────────────────────────────────┘    │
-│    │                                                    │
-│    ▼ read (182ns)                                       │
-│  Rust Parser (data plane)                               │
-│    • Zero allocation                                    │
-│    • SIMD-friendly byte scanning                        │
-│    • Returns field offsets (not strings)                │
-└─────────────────────────────────────────────────────────┘
-```
+**NO HARDCODED FIELDS** - Consumer defines what fields to extract.
 
 ## Performance
 
-| Operation | Latency |
-|-----------|---------|
-| Ring Write | 169ns |
-| Ring Read | 182ns |
-| Full Round-trip | 1.05µs |
-| Throughput | 954K msgs/sec |
+| Metric | Value |
+|--------|-------|
+| Latency per entry | 6.21us |
+| Throughput | ~161K entries/sec |
+| vs old npm package | 1.13x faster |
+
+## Installation
+
+```bash
+bun add @ebowwa/jsonl-hft
+```
 
 ## Usage
 
 ```typescript
-import { parse, extract, countLines, initRingBuffer, ringWrite, ringRead } from "@ebowwa/jsonl-hft";
+import { parseDir, parseFile, parseBuffer, getVersion } from "@ebowwa/jsonl-hft";
 
-// Direct parse (zero-copy)
-const data = Buffer.from('{"session_id":"abc","role":"user","content":"hello"}\n');
-const entries = parse(data);
+// Define what fields you want to extract
+const fields = ["session_id", "timestamp", "role", "message.content"];
 
-// Extract fields using offsets
-for (const entry of entries) {
-  const sessionId = extract(data, entry.session_id_start, entry.session_id_end);
-  const role = extract(data, entry.role_start, entry.role_end);
-  console.log(sessionId, role);
-}
+// Parse a directory (recursive, parallel)
+const entries = parseDir("/path/to/jsonl/files", fields);
+// entries: Array<{ session_id: string; timestamp: string; role: string; "message.content": string }>
 
-// Ring buffer for HFT
-initRingBuffer(1024 * 1024); // 1MB
-ringWrite(data);
-const pending = ringPending();
-const readData = ringRead(data.length);
+// Parse a single file
+const fileEntries = parseFile("/path/to/file.jsonl", fields);
+
+// Parse a buffer
+const bufferEntries = parseBuffer(jsonlBuffer, fields);
+
+// Get version
+console.log(getVersion()); // "1.0.0"
 ```
 
+## Field Specification
+
+Fields can be:
+- Simple: `"session_id"`, `"timestamp"`, `"role"`
+- Nested (dot notation): `"message.content"`, `"metadata.user.id"`
+
+The parser extracts only the fields you request - no wasted parsing.
+
 ## API
 
-### Parse Functions
+### `parseDir<T>(dirPath: string, fields: FieldSpec[]): T[]`
+Parse all JSONL files in a directory recursively. Files are parsed in parallel using rayon.
+
+### `parseFile<T>(filePath: string, fields: FieldSpec[]): T[]`
+Parse a single JSONL file using memory-mapped I/O.
+
+### `parseBuffer<T>(input: Buffer | Uint8Array | string, fields: FieldSpec[]): T[]`
+Parse JSONL data from a buffer or string.
 
-- `parse(data: Buffer): EntryRef[]` - Parse JSONL, returns field offsets
-- `extract(data: Buffer, start: number, end: number): string` - Extract string using offsets
-- `countLines(data: Buffer): number` - Count lines (minimal work)
+### `getVersion(): string`
+Get the library version.
 
-### Ring Buffer Functions
+### Preset Field Arrays
+- `CLAUDE_CODE_FIELDS` - For Claude Code history format
+- `TRADE_FIELDS` - For trading/market data
+- `LOG_FIELDS` - For log file parsing
 
-- `initRingBuffer(capacity: number): void` - Initialize shared ring buffer
-- `ringWrite(data: Buffer): number` - Write to buffer (returns bytes written)
-- `ringRead(maxLen: number): Uint8Array` - Read from buffer
-- `ringPending(): number` - Get pending bytes
+## Ring Buffer (HFT Streaming)
+
+For high-frequency streaming scenarios:
+
+```typescript
+import { initRingBuffer, ringWrite, ringRead, ringPending } from "@ebowwa/jsonl-hft";
+
+// Initialize 1MB ring buffer
+initRingBuffer(1024 * 1024);
+
+// Write data
+ringWrite(data);
+
+// Check pending bytes
+const pending = ringPending();
+
+// Read data
+const readData = ringRead(maxLen);
+```
 
 ## Build
 
 ```bash
-cd packages/src/jsonl-hft
-CARGO_TARGET_DIR=target cargo build --release
-bun run benchmark.ts
+cd packages/src/rust/jsonl-hft
+bun run build
+```
+
+## Architecture
+
 ```
+┌─────────────────────────────────────────────────────────┐
+│  TypeScript (control plane)                             │
+│    │                                                    │
+│    ▼ FFI call                                           │
+│  ┌─────────────────────────────────────────────────┐    │
+│  │  Rust Parser (data plane)                        │    │
+│  │  • Memory-mapped I/O (memmap2)                   │    │
+│  │  • Parallel processing (rayon)                   │    │
+│  │  • Zero allocation hot path                      │    │
+│  │  • SIMD-friendly byte scanning (memchr)          │    │
+│  └─────────────────────────────────────────────────┘    │
+└─────────────────────────────────────────────────────────┘
+```
+
+## License
+
+MIT

package/dist/index.d.ts

@@ -1,49 +1,93 @@
 /**
  * @ebowwa/jsonl-hft
- * HFT-grade JSONL parser with sub-10µs latency
+ *
+ * Generic HFT-grade JSONL parser with sub-10µs latency.
+ * NO HARDCODED FIELDS - consumer defines what fields to extract.
+ *
+ * @example
+ * ```ts
+ * import { parseDir, parseFile, parseBuffer } from "@ebowwa/jsonl-hft";
+ *
+ * // Define what fields you want to extract
+ * const fields = ["session_id", "timestamp", "role", "content"];
+ *
+ * // Parse a directory
+ * const entries = parseDir("/path/to/jsonl/files", fields);
+ * // entries: Array<{ session_id: string; timestamp: string; role: string; content: string }>
+ *
+ * // Parse a single file
+ * const fileEntries = parseFile("/path/to/file.jsonl", fields);
+ *
+ * // Parse a buffer
+ * const bufferEntries = parseBuffer(jsonlBuffer, fields);
+ * ```
  */
-export interface EntryRef {
-    session_id_start: number;
-    session_id_end: number;
-    timestamp_start: number;
-    timestamp_end: number;
-    role_start: number;
-    role_end: number;
-    content_start: number;
-    content_end: number;
-}
 /**
- * Convenience type for fully parsed entries
+ * Field specification for extraction
+ * Can be a simple field name or nested path (e.g., "message.content")
+ */
+export type FieldSpec = string;
+/**
+ * Generic entry - a record of field name to string value
  */
-export interface ParsedEntry {
-    session_id: string;
-    timestamp: string;
-    role: string;
-    content: string;
-    source_file?: string;
+export type GenericEntry = Record<string, string>;
+/**
+ * Parse result with metadata
+ */
+export interface ParseResult<T extends GenericEntry = GenericEntry> {
+    entries: T[];
+    parseTimeMs: number;
+    entryCount: number;
 }
 /**
- * Count lines in buffer
+ * Get the library version
+ */
+export declare function getVersion(): string;
+/**
+ * Parse all JSONL files in a directory with custom field extraction.
+ * Files are parsed in parallel using rayon.
+ *
+ * @param dirPath - Path to directory containing .jsonl files (recursive)
+ * @param fields - Array of field names to extract (supports nested paths like "message.content")
+ * @returns Array of entries with the requested fields
+ *
+ * @example
+ * ```ts
+ * // Extract specific fields
+ * const entries = parseDir("/data/logs", ["timestamp", "level", "message"]);
+ *
+ * // Nested field extraction
+ * const nested = parseDir("/data/api", ["request.id", "response.status", "duration_ms"]);
+ * ```
  */
-export declare function countLines(data: Buffer | Uint8Array): number;
+export declare function parseDir<T extends GenericEntry = GenericEntry>(dirPath: string, fields: FieldSpec[]): T[];
 /**
- * Parse JSONL data into entries
+ * Parse a single JSONL file with custom field extraction.
+ * Uses memory-mapped I/O for efficiency.
+ *
+ * @param filePath - Path to the JSONL file
+ * @param fields - Array of field names to extract
+ * @returns Array of entries with the requested fields
  */
-export declare function parse(data: Buffer | Uint8Array): EntryRef[];
+export declare function parseFile<T extends GenericEntry = GenericEntry>(filePath: string, fields: FieldSpec[]): T[];
 /**
- * Extract string from buffer using offsets
+ * Parse a buffer/string of JSONL data with custom field extraction.
+ *
+ * @param input - Buffer or string containing JSONL data
+ * @param fields - Array of field names to extract
+ * @returns Array of entries with the requested fields
  */
-export declare function extract(data: Buffer | Uint8Array, start: number, end: number): string;
+export declare function parseBuffer<T extends GenericEntry = GenericEntry>(input: Buffer | Uint8Array | string, fields: FieldSpec[]): T[];
 /**
- * Initialize ring buffer
+ * Initialize a ring buffer for streaming data
  */
 export declare function initRingBuffer(capacity: number): Uint8Array;
 /**
- * Write to ring buffer
+ * Write data to the ring buffer
  */
 export declare function ringWrite(data: Buffer | Uint8Array): number;
 /**
- * Read from ring buffer
+ * Read data from the ring buffer
  */
 export declare function ringRead(maxLen: number): Uint8Array;
 /**
@@ -51,36 +95,29 @@ export declare function ringRead(maxLen: number): Uint8Array;
  */
 export declare function ringPending(): number;
 /**
- * Parse a single JSONL file
- * @param path - Path to the JSONL file
- * @returns Array of EntryRef objects with field offsets
+ * Preset field specs for Claude Code history format
+ * Use: parseDir(path, CLAUDE_CODE_FIELDS)
  */
-export declare function parseFile(path: string): EntryRef[];
+export declare const CLAUDE_CODE_FIELDS: FieldSpec[];
 /**
- * Parse all JSONL files in a directory
- * @param dirPath - Path to the directory containing JSONL files
- * @returns Object with entries array and fileOffsets map tracking source files
+ * Preset for trading/market data
  */
-export declare function parseDir(dirPath: string): {
-    entries: EntryRef[];
-    fileOffsets: Map<string, number>;
-};
+export declare const TRADE_FIELDS: FieldSpec[];
 /**
- * Parse directory and return fully parsed objects
- * @param dirPath - Path to the directory containing JSONL files
- * @returns Array of ParsedEntry objects with all fields deserialized
+ * Preset for log file parsing
  */
-export declare function parseDirFast(dirPath: string): ParsedEntry[];
+export declare const LOG_FIELDS: FieldSpec[];
 declare const _default: {
-    countLines: typeof countLines;
-    parse: typeof parse;
-    extract: typeof extract;
+    getVersion: typeof getVersion;
+    parseDir: typeof parseDir;
+    parseFile: typeof parseFile;
+    parseBuffer: typeof parseBuffer;
     initRingBuffer: typeof initRingBuffer;
     ringWrite: typeof ringWrite;
     ringRead: typeof ringRead;
     ringPending: typeof ringPending;
-    parseFile: typeof parseFile;
-    parseDir: typeof parseDir;
-    parseDirFast: typeof parseDirFast;
+    CLAUDE_CODE_FIELDS: string[];
+    TRADE_FIELDS: string[];
+    LOG_FIELDS: string[];
 };
 export default _default;

package/dist/index.js

@@ -3,26 +3,28 @@
 import { dlopen, suffix, ptr } from "bun:ffi";
 import { join, dirname } from "path";
 import { fileURLToPath } from "url";
-import { readdirSync } from "fs";
 var __dirname2 = dirname(fileURLToPath(import.meta.url));
 var libPath = join(__dirname2, "..", "native", `libjsonl_hft.${suffix}`);
-var ENTRY_SIZE = 32;
 var lib = dlopen(libPath, {
-  count_lines: {
-    args: ["ptr", "usize"],
-    returns: "u32"
+  jsonl_parse_dir_generic: {
+    args: ["ptr", "ptr", "usize"],
+    returns: "cstring"
   },
-  jsonl_parse_batch: {
+  jsonl_parse_file_generic: {
+    args: ["ptr", "ptr", "usize"],
+    returns: "cstring"
+  },
+  jsonl_parse_buffer_generic: {
     args: ["ptr", "usize", "ptr", "usize"],
-    returns: "usize"
+    returns: "cstring"
   },
-  extract_field: {
-    args: ["ptr", "usize", "u32", "u32", "ptr", "usize"],
-    returns: "usize"
+  jsonl_free_string: {
+    args: ["ptr"],
+    returns: "void"
   },
-  entry_size: {
+  jsonl_version: {
     args: [],
-    returns: "usize"
+    returns: "cstring"
   },
   ring_init: {
     args: ["usize"],
@@ -39,49 +41,77 @@ var lib = dlopen(libPath, {
   ring_pending: {
     args: [],
     returns: "i32"
-  },
-  jsonl_parse_file: {
-    args: ["ptr", "ptr", "usize"],
-    returns: "usize"
-  },
-  jsonl_parse_dir: {
-    args: ["ptr", "ptr", "usize"],
-    returns: "usize"
-  },
-  jsonl_parse_dir_serialized: {
-    args: ["ptr"],
-    returns: "cstring"
   }
 });
-function countLines(data) {
-  return Number(lib.symbols.count_lines(ptr(data), BigInt(data.length)));
-}
-function parse(data) {
-  const maxEntries = Math.ceil(data.length / 50);
-  const outBuf = new Uint8Array(maxEntries * ENTRY_SIZE);
-  const count = Number(lib.symbols.jsonl_parse_batch(ptr(data), BigInt(data.length), ptr(outBuf), BigInt(maxEntries)));
-  const view = new DataView(outBuf.buffer, outBuf.byteOffset);
-  const results = [];
-  for (let i = 0;i < count; i++) {
-    const offset = i * ENTRY_SIZE;
-    results.push({
-      session_id_start: view.getUint32(offset, true),
-      session_id_end: view.getUint32(offset + 4, true),
-      timestamp_start: view.getUint32(offset + 8, true),
-      timestamp_end: view.getUint32(offset + 12, true),
-      role_start: view.getUint32(offset + 16, true),
-      role_end: view.getUint32(offset + 20, true),
-      content_start: view.getUint32(offset + 24, true),
-      content_end: view.getUint32(offset + 28, true)
+function prepareFieldSpecs(fields) {
+  const buffers = [];
+  const specs = [];
+  for (const field of fields) {
+    const buf = new TextEncoder().encode(field);
+    buffers.push(buf);
+    specs.push({
+      name: ptr(buf),
+      name_len: buf.length
     });
   }
-  return results;
+  return { specs, buffers };
+}
+function parseFFIResult(jsonStr) {
+  if (!jsonStr || jsonStr === "[]") {
+    return [];
+  }
+  return JSON.parse(jsonStr);
+}
+function getVersion() {
+  return lib.symbols.jsonl_version() || "unknown";
 }
-function extract(data, start, end) {
-  const len = end - start;
-  const outBuf = new Uint8Array(len);
-  const actualLen = Number(lib.symbols.extract_field(ptr(data), BigInt(data.length), start, end, ptr(outBuf), BigInt(len)));
-  return new TextDecoder().decode(outBuf.slice(0, actualLen));
+function parseDir(dirPath, fields) {
+  if (!dirPath || fields.length === 0) {
+    return [];
+  }
+  const { specs, buffers } = prepareFieldSpecs(fields);
+  const specsBuffer = new Uint8Array(specs.length * 16);
+  const specsView = new DataView(specsBuffer.buffer);
+  for (let i = 0;i < specs.length; i++) {
+    const offset = i * 16;
+    specsView.setBigUint64(offset, BigInt(specs[i].name), true);
+    specsView.setBigUint64(offset + 8, BigInt(specs[i].name_len), true);
+  }
+  const pathBytes = new TextEncoder().encode(dirPath);
+  const jsonStr = lib.symbols.jsonl_parse_dir_generic(ptr(pathBytes), ptr(specsBuffer), BigInt(fields.length));
+  return parseFFIResult(jsonStr);
+}
+function parseFile(filePath, fields) {
+  if (!filePath || fields.length === 0) {
+    return [];
+  }
+  const { specs, buffers } = prepareFieldSpecs(fields);
+  const specsBuffer = new Uint8Array(specs.length * 16);
+  const specsView = new DataView(specsBuffer.buffer);
+  for (let i = 0;i < specs.length; i++) {
+    const offset = i * 16;
+    specsView.setBigUint64(offset, BigInt(specs[i].name), true);
+    specsView.setBigUint64(offset + 8, BigInt(specs[i].name_len), true);
+  }
+  const pathBytes = new TextEncoder().encode(filePath);
+  const jsonStr = lib.symbols.jsonl_parse_file_generic(ptr(pathBytes), ptr(specsBuffer), BigInt(fields.length));
+  return parseFFIResult(jsonStr);
+}
+function parseBuffer(input, fields) {
+  if (!input || fields.length === 0) {
+    return [];
+  }
+  const data = typeof input === "string" ? new TextEncoder().encode(input) : input;
+  const { specs, buffers } = prepareFieldSpecs(fields);
+  const specsBuffer = new Uint8Array(specs.length * 16);
+  const specsView = new DataView(specsBuffer.buffer);
+  for (let i = 0;i < specs.length; i++) {
+    const offset = i * 16;
+    specsView.setBigUint64(offset, BigInt(specs[i].name), true);
+    specsView.setBigUint64(offset + 8, BigInt(specs[i].name_len), true);
+  }
+  const jsonStr = lib.symbols.jsonl_parse_buffer_generic(ptr(data), BigInt(data.length), ptr(specsBuffer), BigInt(fields.length));
+  return parseFFIResult(jsonStr);
 }
 function initRingBuffer(capacity) {
   const ptr2 = lib.symbols.ring_init(capacity);
@@ -99,87 +129,49 @@ function ringRead(maxLen) {
 function ringPending() {
   return lib.symbols.ring_pending();
 }
-function parseFile(path) {
-  const pathBytes = new TextEncoder().encode(path);
-  const maxEntries = 1e5;
-  const outBuf = new Uint8Array(maxEntries * ENTRY_SIZE);
-  const count = Number(lib.symbols.jsonl_parse_file(ptr(pathBytes), ptr(outBuf), BigInt(maxEntries)));
-  const view = new DataView(outBuf.buffer, outBuf.byteOffset);
-  const results = [];
-  for (let i = 0;i < count; i++) {
-    const offset = i * ENTRY_SIZE;
-    results.push({
-      session_id_start: view.getUint32(offset, true),
-      session_id_end: view.getUint32(offset + 4, true),
-      timestamp_start: view.getUint32(offset + 8, true),
-      timestamp_end: view.getUint32(offset + 12, true),
-      role_start: view.getUint32(offset + 16, true),
-      role_end: view.getUint32(offset + 20, true),
-      content_start: view.getUint32(offset + 24, true),
-      content_end: view.getUint32(offset + 28, true)
-    });
-  }
-  return results;
-}
-function parseDir(dirPath) {
-  const pathBytes = new TextEncoder().encode(dirPath);
-  const maxEntries = 1e6;
-  const outBuf = new Uint8Array(maxEntries * ENTRY_SIZE);
-  const count = Number(lib.symbols.jsonl_parse_dir(ptr(pathBytes), ptr(outBuf), BigInt(maxEntries)));
-  const view = new DataView(outBuf.buffer, outBuf.byteOffset);
-  const entries = [];
-  for (let i = 0;i < count; i++) {
-    const offset = i * ENTRY_SIZE;
-    entries.push({
-      session_id_start: view.getUint32(offset, true),
-      session_id_end: view.getUint32(offset + 4, true),
-      timestamp_start: view.getUint32(offset + 8, true),
-      timestamp_end: view.getUint32(offset + 12, true),
-      role_start: view.getUint32(offset + 16, true),
-      role_end: view.getUint32(offset + 20, true),
-      content_start: view.getUint32(offset + 24, true),
-      content_end: view.getUint32(offset + 28, true)
-    });
-  }
-  const fileOffsets = new Map;
-  const files = readdirSync(dirPath).filter((f) => f.endsWith(".jsonl"));
-  let currentOffset = 0;
-  for (const file of files) {
-    fileOffsets.set(file, currentOffset);
-    currentOffset += Math.ceil(entries.length / files.length);
-  }
-  return { entries, fileOffsets };
-}
-function parseDirFast(dirPath) {
-  const pathBytes = new TextEncoder().encode(dirPath);
-  const jsonResult = lib.symbols.jsonl_parse_dir_serialized(ptr(pathBytes));
-  if (!jsonResult) {
-    return [];
-  }
-  return JSON.parse(jsonResult);
-}
+var CLAUDE_CODE_FIELDS = [
+  "sessionId",
+  "timestamp",
+  "type",
+  "message.content"
+];
+var TRADE_FIELDS = [
+  "timestamp",
+  "symbol",
+  "side",
+  "price",
+  "quantity"
+];
+var LOG_FIELDS = [
+  "timestamp",
+  "level",
+  "message",
+  "source"
+];
 var src_default = {
-  countLines,
-  parse,
-  extract,
+  getVersion,
+  parseDir,
+  parseFile,
+  parseBuffer,
   initRingBuffer,
   ringWrite,
   ringRead,
   ringPending,
-  parseFile,
-  parseDir,
-  parseDirFast
+  CLAUDE_CODE_FIELDS,
+  TRADE_FIELDS,
+  LOG_FIELDS
 };
 export {
   ringWrite,
   ringRead,
   ringPending,
   parseFile,
-  parseDirFast,
   parseDir,
-  parse,
+  parseBuffer,
   initRingBuffer,
-  extract,
+  getVersion,
   src_default as default,
-  countLines
+  TRADE_FIELDS,
+  LOG_FIELDS,
+  CLAUDE_CODE_FIELDS
 };

package/package.json

@@ -1,7 +1,8 @@
 {
   "name": "@ebowwa/jsonl-hft",
-  "version": "0.1.1",
-  "description": "HFT-grade JSONL parser with sub-10µs latency via shared memory ring buffer",
+  "version": "1.0.0",
+  "author": "ebowwa",
+  "description": "Generic HFT-grade JSONL parser - NO hardcoded fields, consumer defines what to extract. Sub-10µs latency.",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
   "exports": {
@@ -12,16 +13,17 @@
   },
   "scripts": {
     "build": "CARGO_TARGET_DIR=target cargo build --release && bun build ./src/index.ts --outdir ./dist --target=bun && tsc --emitDeclarationOnly --declaration --outDir ./dist",
+    "build:rust": "CARGO_TARGET_DIR=target cargo build --release",
     "test": "bun test",
-    "bench": "bun run benchmark.ts && bun run benchmark-single.ts"
+    "bench": "bun run benchmark.ts"
   },
   "keywords": [
     "jsonl",
     "parser",
+    "generic",
     "hft",
     "low-latency",
-    "simd",
-    "ring-buffer",
+    "configurable-fields",
     "zero-copy"
   ],
   "license": "MIT",
@@ -34,10 +36,11 @@
     "bun-types": "^1.3.9"
   },
   "ownership": {
-    "domain": "quant",
+    "domain": "parsing",
     "responsibilities": [
-      "high-frequency-trading",
-      "jsonl-protocol"
+      "generic-jsonl-parsing",
+      "high-performance-parsing",
+      "field-extraction"
     ]
   }
 }