@gobing-ai/ts-llm-jsonl-importer 0.4.39 → 0.4.40

package/dist/importer.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"importer.d.ts","sourceRoot":"","sources":["../src/importer.ts"],"names":[],"mappings":"AA0BA,OAAO,KAAK,EAER,aAAa,EACb,YAAY,EAGZ,gBAAgB,EAEnB,MAAM,SAAS,CAAC;AAoFjB;;;;;;;GAOG;AACH,wBAAsB,cAAc,CAAC,MAAM,EAAE,MAAM,GAAG,gBAAgB,EAAE,OAAO,EAAE,aAAa,GAAG,OAAO,CAAC,YAAY,CAAC,CAkRrH"}
+{"version":3,"file":"importer.d.ts","sourceRoot":"","sources":["../src/importer.ts"],"names":[],"mappings":"AA0BA,OAAO,KAAK,EAER,aAAa,EACb,YAAY,EAGZ,gBAAgB,EAEnB,MAAM,SAAS,CAAC;AAoFjB;;;;;;;GAOG;AACH,wBAAsB,cAAc,CAAC,MAAM,EAAE,MAAM,GAAG,gBAAgB,EAAE,OAAO,EAAE,aAAa,GAAG,OAAO,CAAC,YAAY,CAAC,CAuRrH"}

package/dist/importer.js

@@ -64,6 +64,7 @@ export async function runJsonlImport(source, options) {
     let importedRecords = 0;
     let skippedDuplicates = 0;
     let unknownRecords = 0;
+    let skippedCorruptLines = 0;
     let checkpointUpdates = 0;
     // Full-mode desired set (R1, task 0504): every record hash the current source produces.
     // ReconcileFullImport diffs this against the persisted ledger to retire stale rows.
@@ -80,9 +81,12 @@ export async function runJsonlImport(source, options) {
             if (line.length === 0 || lineNumber <= checkpoint)
                 continue;
             processedLines += 1;
-            const raw = parseJsonLine(line, file, lineNumber, parseErrors);
-            if (raw === undefined)
+            const raw = parseJsonLine(line, file, lineNumber, parseErrors, definition.corruptLinePolicy);
+            if (raw === undefined) {
+                if (definition.corruptLinePolicy === 'skip')
+                    skippedCorruptLines += 1;
                 continue;
+            }
             const splitRecords = splitRawRecord(definition, raw, {
                 source: resolvedSource,
                 sourceFile: file,
@@ -244,6 +248,7 @@ export async function runJsonlImport(source, options) {
         processedLines,
         importedRecords,
         skippedDuplicates,
+        skippedCorruptLines,
         unknownRecords,
         parseErrors,
         validationErrors,
@@ -251,19 +256,28 @@ export async function runJsonlImport(source, options) {
         reconciliation,
     };
 }
-function parseJsonLine(line, sourceFile, sourceLine, parseErrors) {
+function parseJsonLine(line, sourceFile, sourceLine, parseErrors, corruptLinePolicy = 'error') {
+    let parsed;
     try {
-        const value = JSON.parse(line);
-        if (value === null || typeof value !== 'object' || Array.isArray(value)) {
-            parseErrors.push({ sourceFile, sourceLine, reason: 'JSONL row must be an object' });
-            return undefined;
-        }
-        return value;
+        parsed = JSON.parse(line);
     }
     catch (error) {
-        parseErrors.push({ sourceFile, sourceLine, reason: error instanceof Error ? error.message : 'Invalid JSON' });
+        if (corruptLinePolicy !== 'skip') {
+            parseErrors.push({
+                sourceFile,
+                sourceLine,
+                reason: error instanceof Error ? error.message : 'Invalid JSON',
+            });
+        }
+        return undefined;
+    }
+    if (parsed === null || typeof parsed !== 'object' || Array.isArray(parsed)) {
+        if (corruptLinePolicy !== 'skip') {
+            parseErrors.push({ sourceFile, sourceLine, reason: 'JSONL row must be an object' });
+        }
         return undefined;
     }
+    return parsed;
 }
 function splitRawRecord(definition, raw, context) {
     const targetTable = targetTableFor(definition.targetTable);

package/dist/opencode-importer.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"opencode-importer.d.ts","sourceRoot":"","sources":["../src/opencode-importer.ts"],"names":[],"mappings":"AAAA,OAAO,EAAmB,KAAK,SAAS,EAAkB,MAAM,kBAAkB,CAAC;AACnF,OAAO,EAGH,KAAK,UAAU,EAEf,KAAK,YAAY,EACpB,MAAM,uBAAuB,CAAC;AAgB/B,OAAO,KAAK,EAAe,UAAU,EAAE,YAAY,EAAgC,aAAa,EAAE,MAAM,SAAS,CAAC;AAYlH,8EAA8E;AAC9E,MAAM,WAAW,qBAAqB;IAClC,EAAE,EAAE,SAAS,CAAC;IACd,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,UAAU,CAAC,EAAE,UAAU,CAAC;IACxB,KAAK,CAAC,EAAE,YAAY,CAAC;IACrB,IAAI,CAAC,EAAE,UAAU,CAAC;IAClB,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,cAAc,CAAC,EAAE,SAAS,aAAa,EAAE,CAAC;IAC1C,GAAG,CAAC,EAAE,MAAM,IAAI,CAAC;CACpB;AAED,wFAAwF;AACxF,wBAAsB,iBAAiB,CAAC,OAAO,EAAE,qBAAqB,GAAG,OAAO,CAAC,YAAY,CAAC,CAoI7F"}
+{"version":3,"file":"opencode-importer.d.ts","sourceRoot":"","sources":["../src/opencode-importer.ts"],"names":[],"mappings":"AAAA,OAAO,EAAmB,KAAK,SAAS,EAAkB,MAAM,kBAAkB,CAAC;AACnF,OAAO,EAGH,KAAK,UAAU,EAEf,KAAK,YAAY,EACpB,MAAM,uBAAuB,CAAC;AAgB/B,OAAO,KAAK,EAAe,UAAU,EAAE,YAAY,EAAgC,aAAa,EAAE,MAAM,SAAS,CAAC;AAYlH,8EAA8E;AAC9E,MAAM,WAAW,qBAAqB;IAClC,EAAE,EAAE,SAAS,CAAC;IACd,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,UAAU,CAAC,EAAE,UAAU,CAAC;IACxB,KAAK,CAAC,EAAE,YAAY,CAAC;IACrB,IAAI,CAAC,EAAE,UAAU,CAAC;IAClB,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,cAAc,CAAC,EAAE,SAAS,aAAa,EAAE,CAAC;IAC1C,GAAG,CAAC,EAAE,MAAM,IAAI,CAAC;CACpB;AAED,wFAAwF;AACxF,wBAAsB,iBAAiB,CAAC,OAAO,EAAE,qBAAqB,GAAG,OAAO,CAAC,YAAY,CAAC,CAqI7F"}

package/dist/opencode-importer.js

@@ -124,6 +124,7 @@ export async function runOpenCodeImport(options) {
         processedLines,
         importedRecords,
         skippedDuplicates,
+        skippedCorruptLines: 0,
         unknownRecords: 0,
         parseErrors,
         validationErrors,
@@ -263,6 +264,7 @@ function emptyResult(mode) {
         processedLines: 0,
         importedRecords: 0,
         skippedDuplicates: 0,
+        skippedCorruptLines: 0,
         unknownRecords: 0,
         parseErrors: [],
         validationErrors: [],

package/dist/sources.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"sources.d.ts","sourceRoot":"","sources":["../src/sources.ts"],"names":[],"mappings":"AAyBA,OAAO,KAAK,EAA8B,cAAc,EAAE,gBAAgB,EAAoB,MAAM,SAAS,CAAC;AAsH9G;;;;;;;GAOG;AACH,eAAO,MAAM,gBAAgB,QAAsB,CAAC;AAEpD,8DAA8D;AAC9D,eAAO,MAAM,kBAAkB,EAAE,QAAQ,CAAC,MAAM,CAAC,cAAc,EAAE,gBAAgB,CAAC,CA2DjF,CAAC;AAEF;;;;;GAKG;AACH,wBAAgB,mBAAmB,CAAC,MAAM,EAAE,cAAc,GAAG,gBAAgB,CAM5E;AAED;;;;;;;;GAQG;AACH,wBAAgB,wBAAwB,CAAC,UAAU,EAAE,gBAAgB,GAAG,gBAAgB,CA+BvF;AAED;;;;GAIG;AACH,wBAAgB,uBAAuB,CAAC,KAAK,EAAE,MAAM,GAAG,gBAAgB,GAAG,gBAAgB,CAK1F"}
+{"version":3,"file":"sources.d.ts","sourceRoot":"","sources":["../src/sources.ts"],"names":[],"mappings":"AAyBA,OAAO,KAAK,EAA8B,cAAc,EAAE,gBAAgB,EAAoB,MAAM,SAAS,CAAC;AAsH9G;;;;;;;GAOG;AACH,eAAO,MAAM,gBAAgB,QAAsB,CAAC;AAEpD,8DAA8D;AAC9D,eAAO,MAAM,kBAAkB,EAAE,QAAQ,CAAC,MAAM,CAAC,cAAc,EAAE,gBAAgB,CAAC,CAiEjF,CAAC;AAEF;;;;;GAKG;AACH,wBAAgB,mBAAmB,CAAC,MAAM,EAAE,cAAc,GAAG,gBAAgB,CAM5E;AAED;;;;;;;;GAQG;AACH,wBAAgB,wBAAwB,CAAC,UAAU,EAAE,gBAAgB,GAAG,gBAAgB,CA+BvF;AAED;;;;GAIG;AACH,wBAAgB,uBAAuB,CAAC,KAAK,EAAE,MAAM,GAAG,gBAAgB,GAAG,gBAAgB,CAK1F"}

package/dist/sources.js

@@ -103,7 +103,13 @@ export const SOURCE_DEFINITIONS = {
     codex: customSourceDefinition('codex', 'Codex', ['.codex/sessions'], ['*.jsonl'], codexSplit, CODEX_FIELD_MAP, CODEX_SCHEMA),
     omp: customSourceDefinition('omp', 'OMP', ['.omp/agent/sessions'], ['*.jsonl'], ompSplit, OMP_FIELD_MAP, OMP_SCHEMA),
     grok: customSourceDefinition('grok', 'Grok', ['.grok/sessions'], ['*.jsonl'], grokSplit, GROK_FIELD_MAP, GROK_SCHEMA),
-    agy: customSourceDefinition('agy', 'Antigravity CLI', ['.gemini/antigravity-cli/brain'], ['*.jsonl'], agySplit, AGY_FIELD_MAP, AGY_SCHEMA),
+    agy: {
+        ...customSourceDefinition('agy', 'Antigravity CLI', ['.gemini/antigravity-cli/brain'], ['*.jsonl'], agySplit, AGY_FIELD_MAP, AGY_SCHEMA),
+        // agy's producer interleaves non-JSON fragments and torn tail writes into its
+        // brain jsonl (task 0623: 789/89,818 lines, all unrecoverable). Skipping keeps
+        // the 99%+ good records importable without degrading the whole source.
+        corruptLinePolicy: 'skip',
+    },
     gemini: customSourceDefinition('gemini', 'Gemini CLI', ['.gemini/tmp', '.config/gemini'], ['*.jsonl'], geminiSplit, GEMINI_FIELD_MAP, GEMINI_SCHEMA),
     opencode: sourceDefinition('opencode', 'OpenCode', ['.opencode', '.local/share/opencode'], ['*.jsonl']),
     antigravity: sourceDefinition('antigravity', 'Antigravity', ['.antigravity'], ['*.jsonl']),

package/dist/types.d.ts

@@ -41,6 +41,13 @@ export interface SourceDefinition<TRecord extends JsonObject = JsonObject> {
     readonly displayName: string;
     readonly defaultRoots: readonly string[];
     readonly filePatterns: readonly string[];
+    /**
+     * Policy for lines that fail JSON parsing. `'error'` (default) records a parse error and
+     * degrades the import; `'skip'` counts the line and moves on — for producers whose logs
+     * are known to interleave non-JSON fragments and torn tail writes (agy task 0623:
+     * 789/89,818 lines unparseable, all unrecoverable garbage).
+     */
+    readonly corruptLinePolicy?: 'error' | 'skip';
     readonly targetTable: string;
     readonly splitConfig: SplitConfig;
     readonly fieldMap: Readonly<Record<string, string>>;
@@ -100,6 +107,8 @@ export interface ImportResult {
     readonly importedRecords: number;
     readonly skippedDuplicates: number;
     readonly unknownRecords: number;
+    /** Lines dropped by `corruptLinePolicy: 'skip'` instead of degrading the import. */
+    readonly skippedCorruptLines: number;
     readonly parseErrors: readonly ImportIssue[];
     readonly validationErrors: readonly ImportIssue[];
     readonly checkpointUpdates: number;

package/dist/types.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAClD,OAAO,KAAK,EAAE,UAAU,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAC;AACtE,OAAO,KAAK,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAE7B,6DAA6D;AAC7D,MAAM,MAAM,cAAc,GACpB,IAAI,GACJ,QAAQ,GACR,OAAO,GACP,QAAQ,GACR,UAAU,GACV,aAAa,GACb,UAAU,GACV,KAAK,GACL,MAAM,GACN,KAAK,CAAC;AAEZ,mDAAmD;AACnD,MAAM,MAAM,UAAU,GAAG,MAAM,GAAG,aAAa,GAAG,YAAY,CAAC;AAE/D,gFAAgF;AAChF,MAAM,WAAW,UAAU;IACvB,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,MAAM,EAAE,UAAU,CAAC;CAC/B;AAED,kFAAkF;AAClF,MAAM,MAAM,WAAW,GACjB;IACI,QAAQ,CAAC,IAAI,EAAE,YAAY,CAAC;CAC/B,GACD;IACI,QAAQ,CAAC,IAAI,EAAE,aAAa,CAAC;IAC7B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,CAAC;CACjC,GACD;IACI,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAC;IACxB,QAAQ,CAAC,KAAK,EAAE,CAAC,GAAG,EAAE,UAAU,EAAE,OAAO,CAAC,EAAE,gBAAgB,KAAK,SAAS,CAAC,UAAU,GAAG,UAAU,CAAC,EAAE,CAAC;IACtG,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,CAAC;CACjC,CAAC;AAER,4DAA4D;AAC5D,MAAM,MAAM,UAAU,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;AAEjD,sDAAsD;AACtD,MAAM,MAAM,cAAc,GAAG,CAAC,KAAK,EAAE,OAAO,EAAE,GAAG,EAAE,UAAU,EAAE,OAAO,EAAE,gBAAgB,KAAK,OAAO,CAAC;AAErG,0DAA0D;AAC1D,MAAM,WAAW,gBAAgB;IAC7B,2FAA2F;IAC3F,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;CAC/B;AAED,qEAAqE;AACrE,MAAM,WAAW,gBAAgB,CAAC,OAAO,SAAS,UAAU,GAAG,UAAU;IACrE,iHAAiH;IACjH,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,YAAY,EAAE,SAAS,MAAM,EAAE,CAAC;IACzC,QAAQ,CAAC,YAAY,EAAE,SAAS,MAAM,EAAE,CAAC;IACzC,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,WAAW,EAAE,WAAW,CAAC;IAClC,QAAQ,CAAC,QAAQ,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;IACpD,QAAQ,CAAC,eAAe,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,cAAc,CAAC,CAAC,CAAC;IACnE,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;CACvC;AAED,6DAA6D;AAC7D,MAAM,WAAW,aAAa;IAC1B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;CAChC;AAED,oCAAoC;AACpC,MAAM,WAAW,aAAa;IAC1B,QAAQ,CAAC,EAAE,EAAE,SAAS,CAAC;IACvB,QAAQ,CAAC,UAAU,CAAC,EAAE,UAAU,CAAC;IACjC,QAAQ,CAAC,IAAI,CAAC,EAAE,UAAU,CAAC;IAC3B,QAAQ,CAAC,KAAK,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IACnC,QAAQ,CAAC,KAAK,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IACnC,QAAQ,CAAC,MAAM,CAAC,EAAE,OAAO,CAAC;IAC1B,QAAQ,CAAC,cAAc,CAAC,EAAE,SAAS,aAAa,EAAE,CAAC;IACnD,QAAQ,CAAC,GAAG,CAAC,EAAE,MAAM,IAAI,CAAC;IAC1B;;;;OAIG;IACH,QAAQ,CAAC,KAAK,CAAC,EAAE,YAAY,CAAC;CACjC;AAED,mFAAmF;AACnF,MAAM,WAAW,WAAW;IACxB,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;CAC3B;AAED;;;;;GAKG;AACH,MAAM,WAAW,gBAAgB;IAC7B,uGAAuG;IACvG,QAAQ,CAAC,eAAe,EAAE,MAAM,CAAC;IACjC,iDAAiD;IACjD,QAAQ,CAAC,eAAe,EAAE,MAAM,CAAC;IACjC,qEAAqE;IACrE,QAAQ,CAAC,mBAAmB,EAAE,MAAM,CAAC;CACxC;AAED,yDAAyD;AACzD,MAAM,WAAW,YAAY;IACzB,iGAAiG;IACjG,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,IAAI,EAAE,UAAU,CAAC;IAC1B,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC,QAAQ,CAAC,eAAe,EAAE,MAAM,CAAC;IACjC,QAAQ,CAAC,iBAAiB,EAAE,MAAM,CAAC;IACnC,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC,QAAQ,CAAC,WAAW,EAAE,SAAS,WAAW,EAAE,CAAC;IAC7C,QAAQ,CAAC,gBAAgB,EAAE,SAAS,WAAW,EAAE,CAAC;IAClD,QAAQ,CAAC,iBAAiB,EAAE,MAAM,CAAC;IACnC;;;OAGG;IACH,QAAQ,CAAC,cAAc,CAAC,EAAE,gBAAgB,CAAC;CAC9C"}
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAClD,OAAO,KAAK,EAAE,UAAU,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAC;AACtE,OAAO,KAAK,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAE7B,6DAA6D;AAC7D,MAAM,MAAM,cAAc,GACpB,IAAI,GACJ,QAAQ,GACR,OAAO,GACP,QAAQ,GACR,UAAU,GACV,aAAa,GACb,UAAU,GACV,KAAK,GACL,MAAM,GACN,KAAK,CAAC;AAEZ,mDAAmD;AACnD,MAAM,MAAM,UAAU,GAAG,MAAM,GAAG,aAAa,GAAG,YAAY,CAAC;AAE/D,gFAAgF;AAChF,MAAM,WAAW,UAAU;IACvB,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,MAAM,EAAE,UAAU,CAAC;CAC/B;AAED,kFAAkF;AAClF,MAAM,MAAM,WAAW,GACjB;IACI,QAAQ,CAAC,IAAI,EAAE,YAAY,CAAC;CAC/B,GACD;IACI,QAAQ,CAAC,IAAI,EAAE,aAAa,CAAC;IAC7B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,CAAC;CACjC,GACD;IACI,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAC;IACxB,QAAQ,CAAC,KAAK,EAAE,CAAC,GAAG,EAAE,UAAU,EAAE,OAAO,CAAC,EAAE,gBAAgB,KAAK,SAAS,CAAC,UAAU,GAAG,UAAU,CAAC,EAAE,CAAC;IACtG,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,CAAC;CACjC,CAAC;AAER,4DAA4D;AAC5D,MAAM,MAAM,UAAU,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;AAEjD,sDAAsD;AACtD,MAAM,MAAM,cAAc,GAAG,CAAC,KAAK,EAAE,OAAO,EAAE,GAAG,EAAE,UAAU,EAAE,OAAO,EAAE,gBAAgB,KAAK,OAAO,CAAC;AAErG,0DAA0D;AAC1D,MAAM,WAAW,gBAAgB;IAC7B,2FAA2F;IAC3F,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;CAC/B;AAED,qEAAqE;AACrE,MAAM,WAAW,gBAAgB,CAAC,OAAO,SAAS,UAAU,GAAG,UAAU;IACrE,iHAAiH;IACjH,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,YAAY,EAAE,SAAS,MAAM,EAAE,CAAC;IACzC,QAAQ,CAAC,YAAY,EAAE,SAAS,MAAM,EAAE,CAAC;IACzC;;;;;OAKG;IACH,QAAQ,CAAC,iBAAiB,CAAC,EAAE,OAAO,GAAG,MAAM,CAAC;IAC9C,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,WAAW,EAAE,WAAW,CAAC;IAClC,QAAQ,CAAC,QAAQ,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;IACpD,QAAQ,CAAC,eAAe,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,cAAc,CAAC,CAAC,CAAC;IACnE,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;CACvC;AAED,6DAA6D;AAC7D,MAAM,WAAW,aAAa;IAC1B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;CAChC;AAED,oCAAoC;AACpC,MAAM,WAAW,aAAa;IAC1B,QAAQ,CAAC,EAAE,EAAE,SAAS,CAAC;IACvB,QAAQ,CAAC,UAAU,CAAC,EAAE,UAAU,CAAC;IACjC,QAAQ,CAAC,IAAI,CAAC,EAAE,UAAU,CAAC;IAC3B,QAAQ,CAAC,KAAK,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IACnC,QAAQ,CAAC,KAAK,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IACnC,QAAQ,CAAC,MAAM,CAAC,EAAE,OAAO,CAAC;IAC1B,QAAQ,CAAC,cAAc,CAAC,EAAE,SAAS,aAAa,EAAE,CAAC;IACnD,QAAQ,CAAC,GAAG,CAAC,EAAE,MAAM,IAAI,CAAC;IAC1B;;;;OAIG;IACH,QAAQ,CAAC,KAAK,CAAC,EAAE,YAAY,CAAC;CACjC;AAED,mFAAmF;AACnF,MAAM,WAAW,WAAW;IACxB,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;CAC3B;AAED;;;;;GAKG;AACH,MAAM,WAAW,gBAAgB;IAC7B,uGAAuG;IACvG,QAAQ,CAAC,eAAe,EAAE,MAAM,CAAC;IACjC,iDAAiD;IACjD,QAAQ,CAAC,eAAe,EAAE,MAAM,CAAC;IACjC,qEAAqE;IACrE,QAAQ,CAAC,mBAAmB,EAAE,MAAM,CAAC;CACxC;AAED,yDAAyD;AACzD,MAAM,WAAW,YAAY;IACzB,iGAAiG;IACjG,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,IAAI,EAAE,UAAU,CAAC;IAC1B,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC,QAAQ,CAAC,eAAe,EAAE,MAAM,CAAC;IACjC,QAAQ,CAAC,iBAAiB,EAAE,MAAM,CAAC;IACnC,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC,oFAAoF;IACpF,QAAQ,CAAC,mBAAmB,EAAE,MAAM,CAAC;IACrC,QAAQ,CAAC,WAAW,EAAE,SAAS,WAAW,EAAE,CAAC;IAC7C,QAAQ,CAAC,gBAAgB,EAAE,SAAS,WAAW,EAAE,CAAC;IAClD,QAAQ,CAAC,iBAAiB,EAAE,MAAM,CAAC;IACnC;;;OAGG;IACH,QAAQ,CAAC,cAAc,CAAC,EAAE,gBAAgB,CAAC;CAC9C"}

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@gobing-ai/ts-llm-jsonl-importer",
-    "version": "0.4.39",
+    "version": "0.4.40",
     "description": "@gobing-ai/ts-llm-jsonl-importer — Generic JSONL importer for LLM agent history files.",
     "keywords": [
         "typescript",
@@ -47,9 +47,9 @@
         "release": "echo 'Manual publish is disabled. Releases go through GitHub Actions via Trusted Publishing — push a tag: git tag @gobing-ai/ts-llm-jsonl-importer-v<version> && git push --tags' && exit 1"
     },
     "dependencies": {
-        "@gobing-ai/ts-db": "^0.4.39",
-        "@gobing-ai/ts-runtime": "^0.4.39",
-        "@gobing-ai/ts-utils": "^0.4.39",
+        "@gobing-ai/ts-db": "^0.4.40",
+        "@gobing-ai/ts-runtime": "^0.4.40",
+        "@gobing-ai/ts-utils": "^0.4.40",
         "zod": "^4.1.0"
     },
     "devDependencies": {

package/src/importer.ts

@@ -143,6 +143,7 @@ export async function runJsonlImport(source: string | SourceDefinition, options:
     let importedRecords = 0;
     let skippedDuplicates = 0;
     let unknownRecords = 0;
+    let skippedCorruptLines = 0;
     let checkpointUpdates = 0;
     // Full-mode desired set (R1, task 0504): every record hash the current source produces.
     // ReconcileFullImport diffs this against the persisted ledger to retire stale rows.
@@ -161,8 +162,11 @@ export async function runJsonlImport(source: string | SourceDefinition, options:
             if (line.length === 0 || lineNumber <= checkpoint) continue;
             processedLines += 1;
 
-            const raw = parseJsonLine(line, file, lineNumber, parseErrors);
-            if (raw === undefined) continue;
+            const raw = parseJsonLine(line, file, lineNumber, parseErrors, definition.corruptLinePolicy);
+            if (raw === undefined) {
+                if (definition.corruptLinePolicy === 'skip') skippedCorruptLines += 1;
+                continue;
+            }
 
             const splitRecords = splitRawRecord(definition, raw, {
                 source: resolvedSource,
@@ -392,6 +396,7 @@ export async function runJsonlImport(source: string | SourceDefinition, options:
         processedLines,
         importedRecords,
         skippedDuplicates,
+        skippedCorruptLines,
         unknownRecords,
         parseErrors,
         validationErrors,
@@ -405,18 +410,28 @@ function parseJsonLine(
     sourceFile: string,
     sourceLine: number,
     parseErrors: ImportIssue[],
+    corruptLinePolicy: SourceDefinition['corruptLinePolicy'] = 'error',
 ): JsonObject | undefined {
+    let parsed: unknown;
     try {
-        const value = JSON.parse(line);
-        if (value === null || typeof value !== 'object' || Array.isArray(value)) {
+        parsed = JSON.parse(line);
+    } catch (error) {
+        if (corruptLinePolicy !== 'skip') {
+            parseErrors.push({
+                sourceFile,
+                sourceLine,
+                reason: error instanceof Error ? error.message : 'Invalid JSON',
+            });
+        }
+        return undefined;
+    }
+    if (parsed === null || typeof parsed !== 'object' || Array.isArray(parsed)) {
+        if (corruptLinePolicy !== 'skip') {
             parseErrors.push({ sourceFile, sourceLine, reason: 'JSONL row must be an object' });
-            return undefined;
         }
-        return value as JsonObject;
-    } catch (error) {
-        parseErrors.push({ sourceFile, sourceLine, reason: error instanceof Error ? error.message : 'Invalid JSON' });
         return undefined;
     }
+    return parsed as JsonObject;
 }
 
 function splitRawRecord(

package/src/opencode-importer.ts

@@ -172,6 +172,7 @@ export async function runOpenCodeImport(options: OpenCodeImportOptions): Promise
         processedLines,
         importedRecords,
         skippedDuplicates,
+        skippedCorruptLines: 0,
         unknownRecords: 0,
         parseErrors,
         validationErrors,
@@ -331,6 +332,7 @@ function emptyResult(mode: ImportMode): ImportResult {
         processedLines: 0,
         importedRecords: 0,
         skippedDuplicates: 0,
+        skippedCorruptLines: 0,
         unknownRecords: 0,
         parseErrors: [],
         validationErrors: [],

package/src/sources.ts

@@ -190,15 +190,21 @@ export const SOURCE_DEFINITIONS: Readonly<Record<LlmJsonlSource, SourceDefinitio
         GROK_FIELD_MAP,
         GROK_SCHEMA,
     ),
-    agy: customSourceDefinition(
-        'agy',
-        'Antigravity CLI',
-        ['.gemini/antigravity-cli/brain'],
-        ['*.jsonl'],
-        agySplit,
-        AGY_FIELD_MAP,
-        AGY_SCHEMA,
-    ),
+    agy: {
+        ...customSourceDefinition(
+            'agy',
+            'Antigravity CLI',
+            ['.gemini/antigravity-cli/brain'],
+            ['*.jsonl'],
+            agySplit,
+            AGY_FIELD_MAP,
+            AGY_SCHEMA,
+        ),
+        // agy's producer interleaves non-JSON fragments and torn tail writes into its
+        // brain jsonl (task 0623: 789/89,818 lines, all unrecoverable). Skipping keeps
+        // the 99%+ good records importable without degrading the whole source.
+        corruptLinePolicy: 'skip',
+    },
     gemini: customSourceDefinition(
         'gemini',
         'Gemini CLI',

package/src/types.ts

@@ -62,6 +62,13 @@ export interface SourceDefinition<TRecord extends JsonObject = JsonObject> {
     readonly displayName: string;
     readonly defaultRoots: readonly string[];
     readonly filePatterns: readonly string[];
+    /**
+     * Policy for lines that fail JSON parsing. `'error'` (default) records a parse error and
+     * degrades the import; `'skip'` counts the line and moves on — for producers whose logs
+     * are known to interleave non-JSON fragments and torn tail writes (agy task 0623:
+     * 789/89,818 lines unparseable, all unrecoverable garbage).
+     */
+    readonly corruptLinePolicy?: 'error' | 'skip';
     readonly targetTable: string;
     readonly splitConfig: SplitConfig;
     readonly fieldMap: Readonly<Record<string, string>>;
@@ -126,6 +133,8 @@ export interface ImportResult {
     readonly importedRecords: number;
     readonly skippedDuplicates: number;
     readonly unknownRecords: number;
+    /** Lines dropped by `corruptLinePolicy: 'skip'` instead of degrading the import. */
+    readonly skippedCorruptLines: number;
     readonly parseErrors: readonly ImportIssue[];
     readonly validationErrors: readonly ImportIssue[];
     readonly checkpointUpdates: number;