@gobing-ai/ts-llm-jsonl-importer 0.4.5 → 0.4.7

package/README.md

@@ -56,6 +56,16 @@ result.skippedDuplicates;
 
 All modes preserve parse and validation issues in the returned `ImportResult`; malformed rows are not inserted.
 
+## Streaming
+
+The importer uses `FileSystem.readFileStream` when available (ADR-021), reading one line at a time
+for **O(line) memory usage** — enabling multi-MB or multi-GB LLM history files without buffering
+the entire file. Behavior is identical to the previous `readFile` + split approach (same `source_line`
+values, same `ImportResult`).
+
+When `readFileStream` is unavailable (e.g. Cloudflare Workers), the importer falls back to
+`readFile` + split transparently.
+
 ## Import Specific Files
 
 ```ts

package/dist/importer.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"importer.d.ts","sourceRoot":"","sources":["../src/importer.ts"],"names":[],"mappings":"AAMA,OAAO,KAAK,EAER,aAAa,EACb,YAAY,EAEZ,cAAc,EAGjB,MAAM,SAAS,CAAC;AAajB,0DAA0D;AAC1D,wBAAsB,wBAAwB,CAAC,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,CAOrF;AAED,+DAA+D;AAC/D,wBAAsB,cAAc,CAAC,MAAM,EAAE,cAAc,EAAE,OAAO,EAAE,aAAa,GAAG,OAAO,CAAC,YAAY,CAAC,CA6G1G"}
+{"version":3,"file":"importer.d.ts","sourceRoot":"","sources":["../src/importer.ts"],"names":[],"mappings":"AAMA,OAAO,KAAK,EAER,aAAa,EACb,YAAY,EAEZ,cAAc,EAGjB,MAAM,SAAS,CAAC;AAajB,0DAA0D;AAC1D,wBAAsB,wBAAwB,CAAC,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,CAOrF;AAED,+DAA+D;AAC/D,wBAAsB,cAAc,CAAC,MAAM,EAAE,cAAc,EAAE,OAAO,EAAE,aAAa,GAAG,OAAO,CAAC,YAAY,CAAC,CA4G1G"}

package/dist/importer.js

@@ -32,11 +32,11 @@ export async function runJsonlImport(source, options) {
     let checkpointUpdates = 0;
     for (const file of files) {
         const checkpoint = mode === 'incremental' ? await readCheckpoint(options.db, source, file) : 0;
-        const lines = (await fileSystem.readFile(file)).split(/\r?\n/);
-        for (let index = 0; index < lines.length; index += 1) {
-            const lineNumber = index + 1;
-            const line = lines[index]?.trim();
-            if (line === undefined || line.length === 0 || lineNumber <= checkpoint)
+        let lineNumber = 0;
+        for await (const rawLine of readLines(fileSystem, file)) {
+            lineNumber += 1;
+            const line = rawLine.trim();
+            if (line.length === 0 || lineNumber <= checkpoint)
                 continue;
             processedLines += 1;
             const raw = parseJsonLine(line, file, lineNumber, parseErrors);
@@ -209,7 +209,8 @@ async function ledgerExists(db, recordHash) {
 async function insertRecord(db, targetTable, recordHash, sourceFile, sourceLine, splitIndex, payload, now) {
     const table = targetTableFor(targetTable);
     await db.run(`INSERT INTO ${table} (record_hash, source_file, source_line, split_index, payload_json, imported_at)
-         VALUES (?, ?, ?, ?, ?, ?)`, recordHash, sourceFile, sourceLine, splitIndex, JSON.stringify(payload), timestamp(now));
+         VALUES (?, ?, ?, ?, ?, ?)
+         ON CONFLICT(record_hash) DO NOTHING`, recordHash, sourceFile, sourceLine, splitIndex, JSON.stringify(payload), timestamp(now));
 }
 async function insertLedger(db, recordHash, source, sourceFile, sourceLine, splitIndex, targetTable, now) {
     await db.run(`INSERT INTO history_import_ledger (record_hash, source, source_file, source_line, split_index, target_table, imported_at)
@@ -224,3 +225,21 @@ function targetTableFor(table) {
 function timestamp(now) {
     return (now?.() ?? new Date()).toISOString();
 }
+/**
+ * Yield lines from a file, streaming when the FileSystem supports it.
+ *
+ * WHY: large JSONL history files (100MB+) must not be loaded into memory all at
+ * once. When `fileSystem.readFileStream` is available, lines are streamed from
+ * disk in chunks. Otherwise, falls back to `readFile` + `split` — same behavior
+ * as before, preserving parity for stubs like CF Workers.
+ */
+async function* readLines(fileSystem, file) {
+    if (fileSystem.readFileStream) {
+        yield* fileSystem.readFileStream(file);
+        return;
+    }
+    const content = await fileSystem.readFile(file);
+    for (const line of content.split(/\r?\n/)) {
+        yield line;
+    }
+}

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@gobing-ai/ts-llm-jsonl-importer",
-    "version": "0.4.5",
+    "version": "0.4.7",
     "description": "@gobing-ai/ts-llm-jsonl-importer — Generic JSONL importer for LLM agent history files.",
     "keywords": [
         "typescript",
@@ -47,9 +47,9 @@
         "release": "echo 'Manual publish is disabled. Releases go through GitHub Actions via Trusted Publishing — push a tag: git tag @gobing-ai/ts-llm-jsonl-importer-v<version> && git push --tags' && exit 1"
     },
     "dependencies": {
-        "@gobing-ai/ts-db": "^0.4.5",
-        "@gobing-ai/ts-runtime": "^0.4.5",
-        "@gobing-ai/ts-utils": "^0.4.5",
+        "@gobing-ai/ts-db": "^0.4.7",
+        "@gobing-ai/ts-runtime": "^0.4.7",
+        "@gobing-ai/ts-utils": "^0.4.7",
         "zod": "^4.1.0"
     },
     "devDependencies": {

package/src/importer.ts

@@ -56,12 +56,11 @@ export async function runJsonlImport(source: LlmJsonlSource, options: ImportOpti
 
     for (const file of files) {
         const checkpoint = mode === 'incremental' ? await readCheckpoint(options.db, source, file) : 0;
-        const lines = (await fileSystem.readFile(file)).split(/\r?\n/);
-
-        for (let index = 0; index < lines.length; index += 1) {
-            const lineNumber = index + 1;
-            const line = lines[index]?.trim();
-            if (line === undefined || line.length === 0 || lineNumber <= checkpoint) continue;
+        let lineNumber = 0;
+        for await (const rawLine of readLines(fileSystem, file)) {
+            lineNumber += 1;
+            const line = rawLine.trim();
+            if (line.length === 0 || lineNumber <= checkpoint) continue;
             processedLines += 1;
 
             const raw = parseJsonLine(line, file, lineNumber, parseErrors);
@@ -298,7 +297,8 @@ async function insertRecord(
     const table = targetTableFor(targetTable);
     await db.run(
         `INSERT INTO ${table} (record_hash, source_file, source_line, split_index, payload_json, imported_at)
-         VALUES (?, ?, ?, ?, ?, ?)`,
+         VALUES (?, ?, ?, ?, ?, ?)
+         ON CONFLICT(record_hash) DO NOTHING`,
         recordHash,
         sourceFile,
         sourceLine,
@@ -341,3 +341,22 @@ function targetTableFor(table: string): string {
 function timestamp(now: ImportOptions['now']): string {
     return (now?.() ?? new Date()).toISOString();
 }
+
+/**
+ * Yield lines from a file, streaming when the FileSystem supports it.
+ *
+ * WHY: large JSONL history files (100MB+) must not be loaded into memory all at
+ * once. When `fileSystem.readFileStream` is available, lines are streamed from
+ * disk in chunks. Otherwise, falls back to `readFile` + `split` — same behavior
+ * as before, preserving parity for stubs like CF Workers.
+ */
+async function* readLines(fileSystem: FileSystem, file: string): AsyncGenerator<string> {
+    if (fileSystem.readFileStream) {
+        yield* fileSystem.readFileStream(file);
+        return;
+    }
+    const content = await fileSystem.readFile(file);
+    for (const line of content.split(/\r?\n/)) {
+        yield line;
+    }
+}