@gobing-ai/ts-llm-jsonl-importer 0.4.62 → 0.4.66
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -0
- package/dist/importer.d.ts.map +1 -1
- package/dist/importer.js +25 -2
- package/dist/index.d.ts +1 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1 -0
- package/dist/jsonl-importer-dao.d.ts.map +1 -1
- package/dist/jsonl-importer-dao.js +3 -1
- package/dist/mappers.d.ts +20 -0
- package/dist/mappers.d.ts.map +1 -1
- package/dist/mappers.js +90 -0
- package/dist/schema-sql.d.ts +1 -1
- package/dist/schema-sql.js +1 -1
- package/dist/sources.d.ts.map +1 -1
- package/dist/sources.js +8 -1
- package/dist/types.d.ts +1 -1
- package/dist/types.d.ts.map +1 -1
- package/dist/zstd.d.ts +12 -0
- package/dist/zstd.d.ts.map +1 -0
- package/dist/zstd.js +31 -0
- package/package.json +4 -4
- package/src/importer.ts +25 -1
- package/src/index.ts +1 -0
- package/src/jsonl-importer-dao.ts +3 -1
- package/src/mappers.ts +95 -0
- package/src/schema-sql.ts +1 -1
- package/src/sources.ts +18 -0
- package/src/types.ts +2 -1
- package/src/zstd.ts +33 -0
package/README.md
CHANGED
|
@@ -21,6 +21,8 @@ Generic JSONL import pipeline for AI-agent history-style files: discover files,
|
|
|
21
21
|
|
|
22
22
|
Built-in source keys are `claude`, `codex`, `gemini`, `pi`, `opencode`, `antigravity`, `openclaw`, `omp`, `grok`, and `agy`. The `agy` source scans `~/.gemini/antigravity-cli`, covering both `history.jsonl` and `brain/**` transcripts; conversation `.db` stores under `conversations/` are not imported.
|
|
23
23
|
|
|
24
|
+
The `deepseek` source scans `$DSH_HOME/sessions` (falling back to `~/.dsh/sessions`) for `session-<uuid>/` directories containing `session.v3.jsonl` or `session.v3.jsonl.zstd`. Compressed session logs are decompressed through the system `zstd` CLI — zstd must be on `PATH`, else the import fails with an actionable error naming zstd and the file. The session header line is imported as a `meta` row with the session `cwd`; `user/message` and `assistant/message` events map to user/assistant message rows (joined text content, epoch-ms `time` → ISO timestamp, `source.model` when present), while any other plugin-extensible event type (compaction, hooks, titles) is tolerated and skipped; torn crash tails are skipped (`corruptLinePolicy: 'skip'`).
|
|
25
|
+
|
|
24
26
|
## Installation
|
|
25
27
|
|
|
26
28
|
```bash
|
package/dist/importer.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"importer.d.ts","sourceRoot":"","sources":["../src/importer.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"importer.d.ts","sourceRoot":"","sources":["../src/importer.ts"],"names":[],"mappings":"AA8BA,OAAO,KAAK,EAER,aAAa,EACb,YAAY,EAGZ,gBAAgB,EAEnB,MAAM,SAAS,CAAC;AAqFjB;;;;;;;GAOG;AACH,wBAAsB,cAAc,CAAC,MAAM,EAAE,MAAM,GAAG,gBAAgB,EAAE,OAAO,EAAE,aAAa,GAAG,OAAO,CAAC,YAAY,CAAC,CA8ZrH"}
|
package/dist/importer.js
CHANGED
|
@@ -1,10 +1,11 @@
|
|
|
1
|
-
import { ambientRuntimePaths, createNodeFileSystem, resolvePath, walkDir, } from '@gobing-ai/ts-runtime';
|
|
1
|
+
import { ambientRuntimePaths, createNodeFileSystem, getProcessEnv, resolvePath, walkDir, } from '@gobing-ai/ts-runtime';
|
|
2
2
|
import { throwIfImportAborted } from './cancellation.js';
|
|
3
3
|
import { sha256 } from './hash.js';
|
|
4
4
|
import { applyHistoryImportSchema, checkpointUpsertOp, codexUsageAttributionUpdateOp, ensureTargetTable, ledgerExistingHashes, ledgerInsertOp, loadSourceCheckpoints, reconcileFullImport, recordInsertOp, resetCheckpoints, targetTableFor, toolCallDurationUpdateOp, toolCallResultBytesUpdateOp, } from './jsonl-importer-dao.js';
|
|
5
5
|
import { claudeToolResultTiming, ompToolResultTiming, timestampToEpochMs } from './mappers.js';
|
|
6
6
|
import { redactRecord } from './redaction.js';
|
|
7
7
|
import { resolveSourceDefinition } from './sources.js';
|
|
8
|
+
import { zstdDecompress } from './zstd.js';
|
|
8
9
|
const TOOL_CALL_KEY_SEP = '\u0000';
|
|
9
10
|
function toolCallKey(source, sessionId, callId) {
|
|
10
11
|
return `${source}${TOOL_CALL_KEY_SEP}${sessionId}${TOOL_CALL_KEY_SEP}${callId}`;
|
|
@@ -457,9 +458,13 @@ async function discoverFiles(definition, roots, files, fileSystem, paths) {
|
|
|
457
458
|
// Previously both kinds anchored to ambient cwd, so a cwd ≠ $HOME silently skipped every
|
|
458
459
|
// registry source because the home-relative paths never existed relative to cwd.
|
|
459
460
|
const ambient = paths ?? ambientRuntimePaths();
|
|
461
|
+
// deepseek (task 0067 R1): $DSH_HOME replaces the `~/.dsh` storage root when set;
|
|
462
|
+
// session-persistence-jsonl mounts its sessions under `<root>/sessions`.
|
|
463
|
+
const dshHome = definition.source === 'deepseek' ? nonEmptyString(getProcessEnv().DSH_HOME) : undefined;
|
|
464
|
+
const defaultRoots = dshHome !== undefined ? [`${dshHome}/sessions`] : (definition.defaultRoots ?? []);
|
|
460
465
|
const resolvedRoots = roots !== undefined
|
|
461
466
|
? roots.map((root) => resolvePath(ambient.cwd, root))
|
|
462
|
-
:
|
|
467
|
+
: defaultRoots.map((root) => resolvePath(ambient.home, root));
|
|
463
468
|
const found = new Set();
|
|
464
469
|
for (const root of resolvedRoots) {
|
|
465
470
|
if (!(await fileSystem.exists(root)))
|
|
@@ -509,6 +514,14 @@ function matchesPattern(path, patterns) {
|
|
|
509
514
|
return path.endsWith(pattern.replace(/^\*/, ''));
|
|
510
515
|
});
|
|
511
516
|
}
|
|
517
|
+
/** First non-empty string among values (task 0067: $DSH_HOME env override). */
|
|
518
|
+
function nonEmptyString(...values) {
|
|
519
|
+
for (const value of values) {
|
|
520
|
+
if (typeof value === 'string' && value.length > 0)
|
|
521
|
+
return value;
|
|
522
|
+
}
|
|
523
|
+
return undefined;
|
|
524
|
+
}
|
|
512
525
|
/**
|
|
513
526
|
* Stat a file for its identity, returning null on any failure (missing file, stat unsupported).
|
|
514
527
|
* 0675 R2: the short-circuit fails OPEN — any null falls through to the normal read path.
|
|
@@ -531,6 +544,16 @@ async function safeStat(fileSystem, file) {
|
|
|
531
544
|
* as before, preserving parity for stubs like CF Workers.
|
|
532
545
|
*/
|
|
533
546
|
async function* readLines(fileSystem, file) {
|
|
547
|
+
if (file.endsWith('.jsonl.zstd')) {
|
|
548
|
+
// task 0067: checksummed Zstandard session logs decompress through the
|
|
549
|
+
// system zstd CLI before line parsing; the same errors/checkpointing apply
|
|
550
|
+
// to the decompressed line stream.
|
|
551
|
+
const decompressed = await zstdDecompress(file);
|
|
552
|
+
for (const line of decompressed.split(/\r?\n/)) {
|
|
553
|
+
yield line;
|
|
554
|
+
}
|
|
555
|
+
return;
|
|
556
|
+
}
|
|
534
557
|
if (fileSystem.readFileStream) {
|
|
535
558
|
yield* fileSystem.readFileStream(file);
|
|
536
559
|
return;
|
package/dist/index.d.ts
CHANGED
|
@@ -10,4 +10,5 @@ export { HISTORY_IMPORT_SCHEMA_SQL, HISTORY_IMPORT_SCHEMA_VERSION } from './sche
|
|
|
10
10
|
export { getSourceDefinition, resolveSourceDefinition, SOURCE_DEFINITIONS, VALID_TABLE_NAME, validateSourceDefinition, } from './sources';
|
|
11
11
|
export { BOOKKEEPING_HISTORY_TABLES, IMPORTER_OWNED_TABLES } from './tables';
|
|
12
12
|
export type { FieldTransform, ImportIssue, ImportMode, ImportOptions, ImportResult, JsonObject, LlmJsonlSource, ReconcileSummary, RedactionRule, SkillCall, SkillCallSplitRecord, SourceDefinition, SplitConfig, SplitEntry, TransformContext, } from './types';
|
|
13
|
+
export { zstdDecompress } from './zstd';
|
|
13
14
|
//# sourceMappingURL=index.d.ts.map
|
package/dist/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EACH,2BAA2B,EAC3B,KAAK,8BAA8B,EACnC,uBAAuB,EACvB,wBAAwB,GAC3B,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EAAE,kBAAkB,EAAE,oBAAoB,EAAE,MAAM,UAAU,CAAC;AACpE,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AAC5C,OAAO,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AAC5C,OAAO,EAAE,wBAAwB,EAAE,wBAAwB,EAAE,oBAAoB,EAAE,MAAM,sBAAsB,CAAC;AAChH,OAAO,EACH,qBAAqB,EACrB,iBAAiB,EACjB,KAAK,iBAAiB,EACtB,uBAAuB,EACvB,cAAc,GACjB,MAAM,WAAW,CAAC;AACnB,OAAO,EAAE,KAAK,qBAAqB,EAAE,iBAAiB,EAAE,MAAM,qBAAqB,CAAC;AACpF,OAAO,EAAE,uBAAuB,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AACjF,OAAO,EAAE,yBAAyB,EAAE,6BAA6B,EAAE,MAAM,cAAc,CAAC;AACxF,OAAO,EACH,mBAAmB,EACnB,uBAAuB,EACvB,kBAAkB,EAClB,gBAAgB,EAChB,wBAAwB,GAC3B,MAAM,WAAW,CAAC;AACnB,OAAO,EAAE,0BAA0B,EAAE,qBAAqB,EAAE,MAAM,UAAU,CAAC;AAC7E,YAAY,EACR,cAAc,EACd,WAAW,EACX,UAAU,EACV,aAAa,EACb,YAAY,EACZ,UAAU,EACV,cAAc,EACd,gBAAgB,EAChB,aAAa,EACb,SAAS,EACT,oBAAoB,EACpB,gBAAgB,EAChB,WAAW,EACX,UAAU,EACV,gBAAgB,GACnB,MAAM,SAAS,CAAC"}
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EACH,2BAA2B,EAC3B,KAAK,8BAA8B,EACnC,uBAAuB,EACvB,wBAAwB,GAC3B,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EAAE,kBAAkB,EAAE,oBAAoB,EAAE,MAAM,UAAU,CAAC;AACpE,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AAC5C,OAAO,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AAC5C,OAAO,EAAE,wBAAwB,EAAE,wBAAwB,EAAE,oBAAoB,EAAE,MAAM,sBAAsB,CAAC;AAChH,OAAO,EACH,qBAAqB,EACrB,iBAAiB,EACjB,KAAK,iBAAiB,EACtB,uBAAuB,EACvB,cAAc,GACjB,MAAM,WAAW,CAAC;AACnB,OAAO,EAAE,KAAK,qBAAqB,EAAE,iBAAiB,EAAE,MAAM,qBAAqB,CAAC;AACpF,OAAO,EAAE,uBAAuB,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AACjF,OAAO,EAAE,yBAAyB,EAAE,6BAA6B,EAAE,MAAM,cAAc,CAAC;AACxF,OAAO,EACH,mBAAmB,EACnB,uBAAuB,EACvB,kBAAkB,EAClB,gBAAgB,EAChB,wBAAwB,GAC3B,MAAM,WAAW,CAAC;AACnB,OAAO,EAAE,0BAA0B,EAAE,qBAAqB,EAAE,MAAM,UAAU,CAAC;AAC7E,YAAY,EACR,cAAc,EACd,WAAW,EACX,UAAU,EACV,aAAa,EACb,YAAY,EACZ,UAAU,EACV,cAAc,EACd,gBAAgB,EAChB,aAAa,EACb,SAAS,EACT,oBAAoB,EACpB,gBAAgB,EAChB,WAAW,EACX,UAAU,EACV,gBAAgB,GACnB,MAAM,SAAS,CAAC;AACjB,OAAO,EAAE,cAAc,EAAE,MAAM,QAAQ,CAAC"}
|
package/dist/index.js
CHANGED
|
@@ -9,3 +9,4 @@ export { DEFAULT_REDACTION_RULES, redactRecord, redactValue } from './redaction.
|
|
|
9
9
|
export { HISTORY_IMPORT_SCHEMA_SQL, HISTORY_IMPORT_SCHEMA_VERSION } from './schema-sql.js';
|
|
10
10
|
export { getSourceDefinition, resolveSourceDefinition, SOURCE_DEFINITIONS, VALID_TABLE_NAME, validateSourceDefinition, } from './sources.js';
|
|
11
11
|
export { BOOKKEEPING_HISTORY_TABLES, IMPORTER_OWNED_TABLES } from './tables.js';
|
|
12
|
+
export { zstdDecompress } from './zstd.js';
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"jsonl-importer-dao.d.ts","sourceRoot":"","sources":["../src/jsonl-importer-dao.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAI7D,OAAO,KAAK,EAAE,aAAa,EAAE,UAAU,EAAE,gBAAgB,EAAE,gBAAgB,EAAE,MAAM,SAAS,CAAC;AAE7F,UAAU,aAAa;IACnB,QAAQ,CAAC,kBAAkB,EAAE,MAAM,CAAC;IACpC,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACrC,QAAQ,CAAC,eAAe,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CAC5C;AAED,gEAAgE;AAChE,MAAM,WAAW,gBAAgB;IAC7B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,QAAQ,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;CACnC;AAED;;;;GAIG;AACH,eAAO,MAAM,oBAAoB,yEAA0E,CAAC;
|
|
1
|
+
{"version":3,"file":"jsonl-importer-dao.d.ts","sourceRoot":"","sources":["../src/jsonl-importer-dao.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAI7D,OAAO,KAAK,EAAE,aAAa,EAAE,UAAU,EAAE,gBAAgB,EAAE,gBAAgB,EAAE,MAAM,SAAS,CAAC;AAE7F,UAAU,aAAa;IACnB,QAAQ,CAAC,kBAAkB,EAAE,MAAM,CAAC;IACpC,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACrC,QAAQ,CAAC,eAAe,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CAC5C;AAED,gEAAgE;AAChE,MAAM,WAAW,gBAAgB;IAC7B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,QAAQ,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;CACnC;AAED;;;;GAIG;AACH,eAAO,MAAM,oBAAoB,yEAA0E,CAAC;AAgF5G,iBAAS,cAAc,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAK7C;AAED,iBAAS,SAAS,CAAC,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,GAAG,MAAM,CAEpD;AAED,iBAAS,aAAa,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAS5C;AAED;;;;;;GAMG;AACH,wBAAsB,wBAAwB,CAAC,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,CAOrF;AAED,6FAA6F;AAC7F,iBAAe,iBAAiB,CAAC,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EAAE,WAAW,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAG5F;AAED;;;;;;;;;;GAUG;AACH,iBAAe,kBAAkB,CAAC,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EAAE,UAAU,EAAE,gBAAgB,GAAG,OAAO,CAAC,IAAI,CAAC,CAQtG;AAED,iBAAe,cAAc,CAAC,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAO1G;AAED;;;;;;;;GAQG;AACH,wBAAsB,qBAAqB,CACvC,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EACvB,MAAM,EAAE,MAAM,GACf,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,gBAAgB,CAAC,CAAC,CA0BxC;AAED,iBAAe,gBAAgB,CAAC,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,SAAS,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAIhH;AAED,iBAAe,eAAe,CAC1B,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EACvB,MAAM,EAAE,MAAM,EACd,UAAU,EAAE,MAAM,EAClB,IAAI,EAAE,MAAM,EACZ,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,GAC1B,OAAO,CAAC,IAAI,CAAC,CAYf;AAED,iBAAe,YAAY,CAAC,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EAAE,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAMzF;AAsBD;;;;;;;;;;GAUG;AACH,wBAAgB,mBAAmB,CAAC,OAAO,EAAE,UAAU,GAAG;IAAE,SAAS,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,CAI7F;AA6CD;;;GAGG;AACH,wBAAgB,cAAc,CAC1B,WAAW,EAAE,MAAM,EACnB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,OAAO,EAAE,UAAU,EACnB,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,GAC1B,SAAS,CA0CX;AAED,mDAAmD;AACnD,wBAAgB,cAAc,CAC1B,UAAU,EAAE,MAAM,EAClB,MAAM,EAAE,MAAM,EACd,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,WAAW,EAAE,MAAM,EACnB,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,GAC1B,SAAS,CAMX;AAED,0FAA0F;AAC1F,wBAAgB,kBAAkB,CAC9B,MAAM,EAAE,MAAM,EACd,UAAU,EAAE,MAAM,EAClB,IAAI,EAAE,MAAM,EACZ,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,EACzB,QAAQ,CAAC,EAAE;IAAE,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;IAAC,QAAQ,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI,CAAA;CAAE,GAC7E,SAAS,CAWX;AAED;;;;;;;;GAQG;AACH,iBAAS,wBAAwB,CAC7B,UAAU,EAAE,MAAM,EAClB,SAAS,EAAE,MAAM,GAAG,IAAI,EACxB,WAAW,EAAE,MAAM,GAAG,IAAI,EAC1B,UAAU,EAAE,MAAM,GAAG,IAAI,GAC1B,SAAS,CAOX;AAED;;;GAGG;AACH,iBAAS,2BAA2B,CAAC,UAAU,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,GAAG,SAAS,CAOvF;AAED;2FAC2F;AAC3F,wBAAgB,6BAA6B,CACzC,UAAU,EAAE,MAAM,EAClB,MAAM,EAAE;IAAE,QAAQ,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IAAC,QAAQ,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IAAC,QAAQ,CAAC,SAAS,EAAE,MAAM,GAAG,IAAI,CAAA;CAAE,GAC7G,SAAS,CAOX;AAED;;;GAGG;AACH,wBAAsB,oBAAoB,CAAC,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,SAAS,MAAM,EAAE,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAYnH;AAED,+DAA+D;AAC/D,MAAM,WAAW,0BAA0B;IACvC,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;CACnC;AAED;;GAEG;AACH,wBAAsB,+BAA+B,CAAC,EAAE,EAAE,SAAS,GAAG,OAAO,CAAC,0BAA0B,EAAE,CAAC,CAiB1G;AAED;;GAEG;AACH,wBAAsB,uBAAuB,CACzC,EAAE,EAAE,SAAS,EACb,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,cAAc,EAAE,MAAM,GACvB,OAAO,CAAC,IAAI,CAAC,CAOf;AAED,iBAAe,YAAY,CACvB,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EACvB,WAAW,EAAE,MAAM,EACnB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,OAAO,EAAE,UAAU,EACnB,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,GAC1B,OAAO,CAAC,IAAI,CAAC,CAIf;AAED,iBAAe,YAAY,CACvB,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EACvB,UAAU,EAAE,MAAM,EAClB,MAAM,EAAE,MAAM,EACd,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,WAAW,EAAE,MAAM,EACnB,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,GAC1B,OAAO,CAAC,IAAI,CAAC,CAYf;AAED;;;;;;;;;;;;;;;;;;;GAmBG;AACH,wBAAsB,wBAAwB,CAC1C,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EACvB,eAAe,EAAE,CAAC,UAAU,EAAE,MAAM,KAAK,MAAM,GAAG,IAAI,GAAG,SAAS,GACnE,OAAO,CAAC,IAAI,CAAC,CAWf;AA4GD;;;;;;;;;;;GAWG;AACH,iBAAe,mBAAmB,CAC9B,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EACvB,MAAM,EAAE,MAAM,EACd,aAAa,EAAE,WAAW,CAAC,MAAM,CAAC,EAClC,eAAe,EAAE,SAAS,MAAM,EAAE,EAClC,MAAM,EAAE,OAAO,GAChB,OAAO,CAAC,gBAAgB,CAAC,CA6C3B;AAID,4DAA4D;AAC5D,MAAM,WAAW,kBAAkB;IAC/B,EAAE,EAAE,MAAM,CAAC;IACX,UAAU,EAAE,MAAM,CAAC;IACnB,YAAY,EAAE,MAAM,CAAC;IACrB,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,MAAM,CAAC;CACrB;AAED,iEAAiE;AACjE,MAAM,WAAW,eAAe;IAC5B,EAAE,EAAE,MAAM,CAAC;IACX,UAAU,EAAE,MAAM,CAAC;IACnB,YAAY,EAAE,MAAM,CAAC;IACrB,IAAI,EAAE,MAAM,CAAC;CAChB;AAED,iDAAiD;AACjD,MAAM,WAAW,qBAAqB;IAClC,WAAW,EAAE,MAAM,CAAC;IACpB,YAAY,EAAE,MAAM,CAAC;CACxB;AAED,gEAAgE;AAChE,MAAM,WAAW,mBAAmB;IAChC,WAAW,EAAE,iBAAiB,GAAG,mBAAmB,GAAG,oBAAoB,CAAC;IAC5E,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,EAAE,UAAU,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;CACtB;AAED,4EAA4E;AAC5E,wBAAsB,oBAAoB,CACtC,EAAE,EAAE,SAAS,EACb,QAAQ,EAAE,MAAM,EAChB,MAAM,EAAE,MAAM,EACd,KAAK,EAAE,MAAM,GACd,OAAO,CAAC,kBAAkB,EAAE,CAAC,CAa/B;AAED,qEAAqE;AACrE,wBAAsB,iBAAiB,CAAC,EAAE,EAAE,SAAS,EAAE,UAAU,EAAE,SAAS,MAAM,EAAE,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC,CAShH;AAED,0EAA0E;AAC1E,wBAAsB,2BAA2B,CAAC,EAAE,EAAE,SAAS,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,qBAAqB,EAAE,CAAC,CAAC,CAY9G;AAED,2EAA2E;AAC3E,wBAAgB,wBAAwB,CAAC,OAAO,EAAE,SAAS,qBAAqB,EAAE,GAAG,SAAS,EAAE,CAW/F;AAED,6DAA6D;AAC7D,wBAAgB,yBAAyB,CAAC,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,SAAS,CAKvF;AAED,iFAAiF;AACjF,wBAAgB,2BAA2B,CACvC,OAAO,EAAE,SAAS,mBAAmB,EAAE,EACvC,eAAe,EAAE,SAAS,MAAM,EAAE,EAClC,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,GAC1B,SAAS,EAAE,CAiEb;AAED,YAAY,EAAE,aAAa,EAAE,CAAC;AAC9B,OAAO,EACH,aAAa,EACb,iBAAiB,EACjB,kBAAkB,EAClB,YAAY,EACZ,YAAY,EACZ,YAAY,EACZ,cAAc,EACd,mBAAmB,EACnB,gBAAgB,EAChB,cAAc,EACd,SAAS,EACT,wBAAwB,EACxB,2BAA2B,EAC3B,eAAe,GAClB,CAAC"}
|
|
@@ -80,7 +80,9 @@ const TYPED_TABLE_COLUMNS = {
|
|
|
80
80
|
],
|
|
81
81
|
};
|
|
82
82
|
/** Keys that a typed table mapper may produce that are not columns. */
|
|
83
|
-
|
|
83
|
+
// `split_index` and `source_record_id` (task 0067 R2) ride the normalized record into the
|
|
84
|
+
// record_hash but are not history_message columns — identity-relevant, never persisted.
|
|
85
|
+
const TYPED_IGNORED_KEYS = new Set(['_meta', 'split_index', 'source_record_id']);
|
|
84
86
|
function targetTableFor(table) {
|
|
85
87
|
if (!VALID_TABLE_NAME.test(table)) {
|
|
86
88
|
throw new HistoryImportError(`Invalid history ETL target table: ${table}`, { table });
|
package/dist/mappers.d.ts
CHANGED
|
@@ -92,6 +92,22 @@ export declare function codexSplit(raw: Record<string, unknown>, context?: Trans
|
|
|
92
92
|
export declare function agySplit(raw: Record<string, unknown>, context?: TransformContext): readonly SplitEntry[];
|
|
93
93
|
/** Map Gemini CLI message, tool, and session-state events into the forensic contract tables. */
|
|
94
94
|
export declare function geminiSplit(raw: Record<string, unknown>, context?: TransformContext): readonly SplitEntry[];
|
|
95
|
+
/**
|
|
96
|
+
* Map one DeepSeek dsh `session.v3.jsonl` line into forensic ETL entries (task 0067 R2).
|
|
97
|
+
*
|
|
98
|
+
* dsh layout: line 1 is the session header `{type:"session", version:3, id, createdAt, cwd, ...}`;
|
|
99
|
+
*
|
|
100
|
+
* any remaining lines are append-only events `{type, seq, time, data}` whose
|
|
101
|
+
* vocabulary is plugin-extensible. Only chat events (`user/message`,
|
|
102
|
+
* `assistant/message`) become history_message rows (content is an array of
|
|
103
|
+
* `{text}` blocks — the existing extractContentText joined-text rule); every
|
|
104
|
+
* unrecognized event type (compaction/*, hook/*, turn/*, session/title, …) is
|
|
105
|
+
* tolerated and skipped, and torn crash tails never reach the mapper because
|
|
106
|
+
* the source registers `corruptLinePolicy: 'skip'`. Session identity comes from
|
|
107
|
+
* the `session-<uuid>` dir path (stateless — message events carry no session id),
|
|
108
|
+
* mirroring the agy/codex path-identity precedent.
|
|
109
|
+
*/
|
|
110
|
+
export declare function dshSplit(raw: Record<string, unknown>, context?: TransformContext): readonly SplitEntry[];
|
|
95
111
|
/**
|
|
96
112
|
* Pure helper extracting canonical session identity from source transcript paths (task 0638 R4).
|
|
97
113
|
*
|
|
@@ -118,6 +134,8 @@ export declare const AGY_FIELD_MAP: Record<string, string>;
|
|
|
118
134
|
export declare const GROK_FIELD_MAP: Record<string, string>;
|
|
119
135
|
/** Identity field map for the Gemini mapper. */
|
|
120
136
|
export declare const GEMINI_FIELD_MAP: Record<string, string>;
|
|
137
|
+
/** Identity field map for the DeepSeek dsh mapper — message rows only, no tool-call rows. */
|
|
138
|
+
export declare const DSH_FIELD_MAP: Record<string, string>;
|
|
121
139
|
/** Passthrough Zod schema for the Claude mapper. */
|
|
122
140
|
export declare const CLAUDE_SCHEMA: z.ZodObject<{}, z.core.$loose>;
|
|
123
141
|
/** Passthrough Zod schema for the Pi mapper. */
|
|
@@ -132,4 +150,6 @@ export declare const AGY_SCHEMA: z.ZodObject<{}, z.core.$loose>;
|
|
|
132
150
|
export declare const GROK_SCHEMA: z.ZodObject<{}, z.core.$loose>;
|
|
133
151
|
/** Passthrough Zod schema for the Gemini mapper. */
|
|
134
152
|
export declare const GEMINI_SCHEMA: z.ZodObject<{}, z.core.$loose>;
|
|
153
|
+
/** Passthrough Zod schema for the DeepSeek dsh mapper. */
|
|
154
|
+
export declare const DSH_SCHEMA: z.ZodObject<{}, z.core.$loose>;
|
|
135
155
|
//# sourceMappingURL=mappers.d.ts.map
|
package/dist/mappers.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"mappers.d.ts","sourceRoot":"","sources":["../src/mappers.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,OAAO,KAAK,EAAE,UAAU,EAAE,oBAAoB,EAAE,UAAU,EAAE,gBAAgB,EAAE,MAAM,SAAS,CAAC;AAmF9F;;;;GAIG;AACH,wBAAgB,qBAAqB,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CASzD;AAED,4FAA4F;AAC5F,MAAM,WAAW,iBAAiB;IAC9B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;IACrB,4FAA4F;IAC5F,QAAQ,CAAC,iBAAiB,EAAE,MAAM,CAAC;CACtC;AAED;;;;;;;;;;GAUG;AACH,wBAAgB,iBAAiB,CAC7B,GAAG,EAAE,UAAU,EACf,OAAO,EAAE,gBAAgB,GAAG,SAAS,EACrC,QAAQ,EAAE,iBAAiB,GAC5B,SAAS,oBAAoB,EAAE,CAmBjC;AAED;;;;GAIG;AACH,wBAAgB,cAAc,CAAC,MAAM,EAAE,oBAAoB,GAAG,UAAU,CAIvE;AA+LD,kFAAkF;AAClF,wBAAgB,UAAU,CAAC,IAAI,EAAE,OAAO,GAAG,MAAM,CAGhD;AAwHD;;;;GAIG;AACH,wBAAgB,YAAY,CAAC,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,IAAI,EAAE,OAAO,GAAG,MAAM,GAAG,SAAS,CAwBjG;AAMD,yEAAyE;AACzE,wBAAgB,WAAW,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,OAAO,CAAC,EAAE,gBAAgB,GAAG,SAAS,UAAU,EAAE,CAuH3G;AAMD,gEAAgE;AAChE,wBAAgB,OAAO,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,OAAO,CAAC,EAAE,gBAAgB,GAAG,SAAS,UAAU,EAAE,CAwHvG;AAmBD,kEAAkE;AAClE,wBAAgB,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,OAAO,CAAC,EAAE,gBAAgB,GAAG,SAAS,UAAU,EAAE,CAoHxG;AAoBD,mFAAmF;AACnF,MAAM,WAAW,mBAAmB;IAChC,yEAAyE;IACzE,UAAU,EAAE,MAAM,CAAC;IACnB,wEAAwE;IACxE,UAAU,EAAE,MAAM,GAAG,SAAS,CAAC;IAC/B,yDAAyD;IACzD,WAAW,EAAE,MAAM,GAAG,SAAS,CAAC;CACnC;AAED,+FAA+F;AAC/F,wBAAgB,kBAAkB,CAAC,KAAK,EAAE,OAAO,GAAG,MAAM,GAAG,SAAS,CASrE;AAED;;;;;;;GAOG;AACH,wBAAgB,mBAAmB,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,mBAAmB,GAAG,IAAI,CAS5F;AAMD,wEAAwE;AACxE,MAAM,WAAW,sBAAsB;IACnC,uFAAuF;IACvF,UAAU,EAAE,MAAM,CAAC;IACnB,yDAAyD;IACzD,WAAW,EAAE,MAAM,GAAG,SAAS,CAAC;IAChC,wDAAwD;IACxD,UAAU,EAAE,MAAM,GAAG,SAAS,CAAC;IAC/B,2DAA2D;IAC3D,WAAW,EAAE,MAAM,CAAC;CACvB;AAED;;;;;;;;GAQG;AACH,wBAAgB,sBAAsB,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,sBAAsB,GAAG,IAAI,CA8BlG;AAMD,mEAAmE;AACnE,wBAAgB,UAAU,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,OAAO,CAAC,EAAE,gBAAgB,GAAG,SAAS,UAAU,EAAE,CAiM1G;AAMD,gFAAgF;AAChF,wBAAgB,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,OAAO,CAAC,EAAE,gBAAgB,GAAG,SAAS,UAAU,EAAE,CAwIxG;AAMD,gGAAgG;AAChG,wBAAgB,WAAW,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,OAAO,CAAC,EAAE,gBAAgB,GAAG,SAAS,UAAU,EAAE,CAuF3G;AAMD;;;;;;GAMG;AACH,wBAAgB,uBAAuB,CAAC,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,
|
|
1
|
+
{"version":3,"file":"mappers.d.ts","sourceRoot":"","sources":["../src/mappers.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,OAAO,KAAK,EAAE,UAAU,EAAE,oBAAoB,EAAE,UAAU,EAAE,gBAAgB,EAAE,MAAM,SAAS,CAAC;AAmF9F;;;;GAIG;AACH,wBAAgB,qBAAqB,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CASzD;AAED,4FAA4F;AAC5F,MAAM,WAAW,iBAAiB;IAC9B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;IACrB,4FAA4F;IAC5F,QAAQ,CAAC,iBAAiB,EAAE,MAAM,CAAC;CACtC;AAED;;;;;;;;;;GAUG;AACH,wBAAgB,iBAAiB,CAC7B,GAAG,EAAE,UAAU,EACf,OAAO,EAAE,gBAAgB,GAAG,SAAS,EACrC,QAAQ,EAAE,iBAAiB,GAC5B,SAAS,oBAAoB,EAAE,CAmBjC;AAED;;;;GAIG;AACH,wBAAgB,cAAc,CAAC,MAAM,EAAE,oBAAoB,GAAG,UAAU,CAIvE;AA+LD,kFAAkF;AAClF,wBAAgB,UAAU,CAAC,IAAI,EAAE,OAAO,GAAG,MAAM,CAGhD;AAwHD;;;;GAIG;AACH,wBAAgB,YAAY,CAAC,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,IAAI,EAAE,OAAO,GAAG,MAAM,GAAG,SAAS,CAwBjG;AAMD,yEAAyE;AACzE,wBAAgB,WAAW,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,OAAO,CAAC,EAAE,gBAAgB,GAAG,SAAS,UAAU,EAAE,CAuH3G;AAMD,gEAAgE;AAChE,wBAAgB,OAAO,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,OAAO,CAAC,EAAE,gBAAgB,GAAG,SAAS,UAAU,EAAE,CAwHvG;AAmBD,kEAAkE;AAClE,wBAAgB,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,OAAO,CAAC,EAAE,gBAAgB,GAAG,SAAS,UAAU,EAAE,CAoHxG;AAoBD,mFAAmF;AACnF,MAAM,WAAW,mBAAmB;IAChC,yEAAyE;IACzE,UAAU,EAAE,MAAM,CAAC;IACnB,wEAAwE;IACxE,UAAU,EAAE,MAAM,GAAG,SAAS,CAAC;IAC/B,yDAAyD;IACzD,WAAW,EAAE,MAAM,GAAG,SAAS,CAAC;CACnC;AAED,+FAA+F;AAC/F,wBAAgB,kBAAkB,CAAC,KAAK,EAAE,OAAO,GAAG,MAAM,GAAG,SAAS,CASrE;AAED;;;;;;;GAOG;AACH,wBAAgB,mBAAmB,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,mBAAmB,GAAG,IAAI,CAS5F;AAMD,wEAAwE;AACxE,MAAM,WAAW,sBAAsB;IACnC,uFAAuF;IACvF,UAAU,EAAE,MAAM,CAAC;IACnB,yDAAyD;IACzD,WAAW,EAAE,MAAM,GAAG,SAAS,CAAC;IAChC,wDAAwD;IACxD,UAAU,EAAE,MAAM,GAAG,SAAS,CAAC;IAC/B,2DAA2D;IAC3D,WAAW,EAAE,MAAM,CAAC;CACvB;AAED;;;;;;;;GAQG;AACH,wBAAgB,sBAAsB,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,sBAAsB,GAAG,IAAI,CA8BlG;AAMD,mEAAmE;AACnE,wBAAgB,UAAU,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,OAAO,CAAC,EAAE,gBAAgB,GAAG,SAAS,UAAU,EAAE,CAiM1G;AAMD,gFAAgF;AAChF,wBAAgB,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,OAAO,CAAC,EAAE,gBAAgB,GAAG,SAAS,UAAU,EAAE,CAwIxG;AAMD,gGAAgG;AAChG,wBAAgB,WAAW,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,OAAO,CAAC,EAAE,gBAAgB,GAAG,SAAS,UAAU,EAAE,CAuF3G;AAMD;;;;;;;;;;;;;;GAcG;AACH,wBAAgB,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,OAAO,CAAC,EAAE,gBAAgB,GAAG,SAAS,UAAU,EAAE,CAgExG;AAQD;;;;;;GAMG;AACH,wBAAgB,uBAAuB,CAAC,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,CAoB9F;AAiMD,kEAAkE;AAClE,wBAAgB,SAAS,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,OAAO,CAAC,EAAE,gBAAgB,GAAG,SAAS,UAAU,EAAE,CAkTzG;AAgFD,wGAAwG;AACxG,wBAAgB,gBAAgB,CAAC,GAAG,EAAE,UAAU,GAAG,MAAM,CAKxD;AAMD,gDAAgD;AAChD,eAAO,MAAM,gBAAgB,wBAE5B,CAAC;AACF,4CAA4C;AAC5C,eAAO,MAAM,YAAY,wBAA8F,CAAC;AACxH,6CAA6C;AAC7C,eAAO,MAAM,aAAa,wBAEzB,CAAC;AACF,+CAA+C;AAC/C,eAAO,MAAM,eAAe,wBAE3B,CAAC;AACF,6CAA6C;AAC7C,eAAO,MAAM,aAAa,wBAEzB,CAAC;AACF,8CAA8C;AAC9C,eAAO,MAAM,cAAc,wBAE1B,CAAC;AACF,gDAAgD;AAChD,eAAO,MAAM,gBAAgB,wBAE5B,CAAC;AACF,6FAA6F;AAC7F,eAAO,MAAM,aAAa,wBAAqE,CAAC;AAQhG,oDAAoD;AACpD,eAAO,MAAM,aAAa,gCAAoB,CAAC;AAC/C,gDAAgD;AAChD,eAAO,MAAM,SAAS,gCAAoB,CAAC;AAC3C,iDAAiD;AACjD,eAAO,MAAM,UAAU,gCAAoB,CAAC;AAC5C,mDAAmD;AACnD,eAAO,MAAM,YAAY,gCAAoB,CAAC;AAC9C,iDAAiD;AACjD,eAAO,MAAM,UAAU,gCAAoB,CAAC;AAC5C,kDAAkD;AAClD,eAAO,MAAM,WAAW,gCAAoB,CAAC;AAC7C,oDAAoD;AACpD,eAAO,MAAM,aAAa,gCAAoB,CAAC;AAC/C,0DAA0D;AAC1D,eAAO,MAAM,UAAU,gCAAoB,CAAC"}
|
package/dist/mappers.js
CHANGED
|
@@ -1325,8 +1325,90 @@ export function geminiSplit(raw, context) {
|
|
|
1325
1325
|
}
|
|
1326
1326
|
return entries;
|
|
1327
1327
|
}
|
|
1328
|
+
// ---------------------------------------------------------------------------
|
|
1329
|
+
// DeepSeek dsh mapper (task 0067)
|
|
1330
|
+
// ---------------------------------------------------------------------------
|
|
1331
|
+
/**
|
|
1332
|
+
* Map one DeepSeek dsh `session.v3.jsonl` line into forensic ETL entries (task 0067 R2).
|
|
1333
|
+
*
|
|
1334
|
+
* dsh layout: line 1 is the session header `{type:"session", version:3, id, createdAt, cwd, ...}`;
|
|
1335
|
+
*
|
|
1336
|
+
* any remaining lines are append-only events `{type, seq, time, data}` whose
|
|
1337
|
+
* vocabulary is plugin-extensible. Only chat events (`user/message`,
|
|
1338
|
+
* `assistant/message`) become history_message rows (content is an array of
|
|
1339
|
+
* `{text}` blocks — the existing extractContentText joined-text rule); every
|
|
1340
|
+
* unrecognized event type (compaction/*, hook/*, turn/*, session/title, …) is
|
|
1341
|
+
* tolerated and skipped, and torn crash tails never reach the mapper because
|
|
1342
|
+
* the source registers `corruptLinePolicy: 'skip'`. Session identity comes from
|
|
1343
|
+
* the `session-<uuid>` dir path (stateless — message events carry no session id),
|
|
1344
|
+
* mirroring the agy/codex path-identity precedent.
|
|
1345
|
+
*/
|
|
1346
|
+
export function dshSplit(raw, context) {
|
|
1347
|
+
const recordType = String(raw.type ?? '');
|
|
1348
|
+
// Session header: bookkeeping meta row carrying cwd/createdAt for the session.
|
|
1349
|
+
if (recordType === 'session') {
|
|
1350
|
+
const sessionId = s(raw.id) ?? sessionIdFromSourcePath('deepseek', context?.sourceFile ?? '') ?? 'unknown';
|
|
1351
|
+
return [
|
|
1352
|
+
{
|
|
1353
|
+
targetTable: 'history_message',
|
|
1354
|
+
record: {
|
|
1355
|
+
session_id: sessionId,
|
|
1356
|
+
seq: 0,
|
|
1357
|
+
role: 'meta',
|
|
1358
|
+
record_type: 'session',
|
|
1359
|
+
disposition: 'meta',
|
|
1360
|
+
ts: timestampOf(raw.createdAt),
|
|
1361
|
+
cwd: s(raw.cwd) ?? null,
|
|
1362
|
+
content_text: null,
|
|
1363
|
+
provenance: 'ambient',
|
|
1364
|
+
},
|
|
1365
|
+
},
|
|
1366
|
+
];
|
|
1367
|
+
}
|
|
1368
|
+
// Chat events only — unknown plugin-extensible types are dropped, not records.
|
|
1369
|
+
if (recordType !== 'user/message' && recordType !== 'assistant/message')
|
|
1370
|
+
return [];
|
|
1371
|
+
const data = o(raw.data);
|
|
1372
|
+
const msg = o(data.message);
|
|
1373
|
+
const seq = typeof raw.seq === 'number' ? raw.seq : (context?.sourceLine ?? 0);
|
|
1374
|
+
const sessionId = sessionIdFromSourcePath('deepseek', context?.sourceFile ?? '') ?? 'unknown';
|
|
1375
|
+
const role = recordType === 'user/message' ? 'user' : 'assistant';
|
|
1376
|
+
const source = o(data.source ?? msg.source);
|
|
1377
|
+
const model = s(source.model, o(data.model ?? msg.model).model);
|
|
1378
|
+
const usage = (data.usage ?? msg.usage);
|
|
1379
|
+
// Task 0067 R2: the message id is the record identity — it rides the normalized
|
|
1380
|
+
// record into the record_hash, so identity derives from `data.message.id` when
|
|
1381
|
+
// present and falls back to the importer's line-derived record hash when absent.
|
|
1382
|
+
const messageId = s(msg.id, data.id);
|
|
1383
|
+
return [
|
|
1384
|
+
{
|
|
1385
|
+
targetTable: 'history_message',
|
|
1386
|
+
record: {
|
|
1387
|
+
session_id: sessionId,
|
|
1388
|
+
seq,
|
|
1389
|
+
role,
|
|
1390
|
+
record_type: recordType,
|
|
1391
|
+
disposition: 'keep',
|
|
1392
|
+
...(messageId !== undefined ? { source_record_id: messageId } : {}),
|
|
1393
|
+
ts: timestampOf(raw.time, data.time, msg.time),
|
|
1394
|
+
duration_ms: null,
|
|
1395
|
+
model: model ?? null,
|
|
1396
|
+
input_tokens: numberOrNull(usage?.input_tokens ?? usage?.input) ?? null,
|
|
1397
|
+
output_tokens: numberOrNull(usage?.output_tokens ?? usage?.output) ?? null,
|
|
1398
|
+
cache_read_tokens: null,
|
|
1399
|
+
cache_write_tokens: null,
|
|
1400
|
+
cost_usd: null,
|
|
1401
|
+
content_text: extractContentText(data.content ?? msg.content) ?? s(data.text, msg.text) ?? null,
|
|
1402
|
+
cwd: null,
|
|
1403
|
+
provenance: 'ambient',
|
|
1404
|
+
},
|
|
1405
|
+
},
|
|
1406
|
+
];
|
|
1407
|
+
}
|
|
1328
1408
|
const UUID_PATTERN = '[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}';
|
|
1329
1409
|
const AGY_BRAIN_REGEX = new RegExp(`(?:^|\\/)brain\\/(${UUID_PATTERN})(?:\\/|$)`, 'i');
|
|
1410
|
+
/** DSH session-dir identity: `<normalized-abs-cwd>--/session-<uuid>/`. */
|
|
1411
|
+
const DSH_SESSION_REGEX = new RegExp(`(?:^|\\/)session-(${UUID_PATTERN})(?:\\/|$)`, 'i');
|
|
1330
1412
|
const CODEX_ROLLOUT_REGEX = new RegExp(`(?:^|-|_)(${UUID_PATTERN})$`, 'i');
|
|
1331
1413
|
/**
|
|
1332
1414
|
* Pure helper extracting canonical session identity from source transcript paths (task 0638 R4).
|
|
@@ -1342,6 +1424,10 @@ export function sessionIdFromSourcePath(source, sourceFile) {
|
|
|
1342
1424
|
const match = normalized.match(AGY_BRAIN_REGEX);
|
|
1343
1425
|
return match?.[1];
|
|
1344
1426
|
}
|
|
1427
|
+
if (sLower === 'deepseek') {
|
|
1428
|
+
const match = normalized.match(DSH_SESSION_REGEX);
|
|
1429
|
+
return match?.[1];
|
|
1430
|
+
}
|
|
1345
1431
|
if (sLower === 'codex') {
|
|
1346
1432
|
const basename = normalized.split('/').at(-1) ?? '';
|
|
1347
1433
|
if (basename.startsWith('rollout-') && basename.endsWith('.jsonl')) {
|
|
@@ -1904,6 +1990,8 @@ export const AGY_FIELD_MAP = identityFieldMap(MESSAGE_MAPPER_KEYS.concat(TOOL_CA
|
|
|
1904
1990
|
export const GROK_FIELD_MAP = identityFieldMap(MESSAGE_MAPPER_KEYS.concat(TOOL_CALL_MAPPER_KEYS, SKILL_CALL_MAPPER_KEYS));
|
|
1905
1991
|
/** Identity field map for the Gemini mapper. */
|
|
1906
1992
|
export const GEMINI_FIELD_MAP = identityFieldMap(MESSAGE_MAPPER_KEYS.concat(TOOL_CALL_MAPPER_KEYS, SKILL_CALL_MAPPER_KEYS));
|
|
1993
|
+
/** Identity field map for the DeepSeek dsh mapper — message rows only, no tool-call rows. */
|
|
1994
|
+
export const DSH_FIELD_MAP = identityFieldMap(MESSAGE_MAPPER_KEYS.concat(['source_record_id']));
|
|
1907
1995
|
// ---------------------------------------------------------------------------
|
|
1908
1996
|
// Zod schemas for the six source mappers
|
|
1909
1997
|
// ---------------------------------------------------------------------------
|
|
@@ -1922,3 +2010,5 @@ export const AGY_SCHEMA = passthroughSchema;
|
|
|
1922
2010
|
export const GROK_SCHEMA = passthroughSchema;
|
|
1923
2011
|
/** Passthrough Zod schema for the Gemini mapper. */
|
|
1924
2012
|
export const GEMINI_SCHEMA = passthroughSchema;
|
|
2013
|
+
/** Passthrough Zod schema for the DeepSeek dsh mapper. */
|
|
2014
|
+
export const DSH_SCHEMA = passthroughSchema;
|
package/dist/schema-sql.d.ts
CHANGED
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
* Guaranteed to match the package version; a bump-or-fail test enforces that
|
|
5
5
|
* any modification to HISTORY_IMPORT_SCHEMA_SQL must be accompanied by a version bump.
|
|
6
6
|
*/
|
|
7
|
-
export declare const HISTORY_IMPORT_SCHEMA_VERSION = "0.4.
|
|
7
|
+
export declare const HISTORY_IMPORT_SCHEMA_VERSION = "0.4.66";
|
|
8
8
|
/**
|
|
9
9
|
* DDL string that creates the history import checkpoint, ledger, and typed contract tables.
|
|
10
10
|
*
|
package/dist/schema-sql.js
CHANGED
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
* Guaranteed to match the package version; a bump-or-fail test enforces that
|
|
5
5
|
* any modification to HISTORY_IMPORT_SCHEMA_SQL must be accompanied by a version bump.
|
|
6
6
|
*/
|
|
7
|
-
export const HISTORY_IMPORT_SCHEMA_VERSION = '0.4.
|
|
7
|
+
export const HISTORY_IMPORT_SCHEMA_VERSION = '0.4.66';
|
|
8
8
|
/**
|
|
9
9
|
* DDL string that creates the history import checkpoint, ledger, and typed contract tables.
|
|
10
10
|
*
|
package/dist/sources.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"sources.d.ts","sourceRoot":"","sources":["../src/sources.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"sources.d.ts","sourceRoot":"","sources":["../src/sources.ts"],"names":[],"mappings":"AA4BA,OAAO,KAAK,EAA8B,cAAc,EAAE,gBAAgB,EAAoB,MAAM,SAAS,CAAC;AAsH9G;;;;;;;GAOG;AACH,eAAO,MAAM,gBAAgB,QAAsB,CAAC;AAEpD,8DAA8D;AAC9D,eAAO,MAAM,kBAAkB,EAAE,QAAQ,CAAC,MAAM,CAAC,cAAc,EAAE,gBAAgB,CAAC,CAgFjF,CAAC;AAEF;;;;;GAKG;AACH,wBAAgB,mBAAmB,CAAC,MAAM,EAAE,cAAc,GAAG,gBAAgB,CAM5E;AAED;;;;;;;;GAQG;AACH,wBAAgB,wBAAwB,CAAC,UAAU,EAAE,gBAAgB,GAAG,gBAAgB,CA+BvF;AAED;;;;GAIG;AACH,wBAAgB,uBAAuB,CAAC,KAAK,EAAE,MAAM,GAAG,gBAAgB,GAAG,gBAAgB,CAK1F"}
|
package/dist/sources.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { z } from 'zod';
|
|
2
2
|
import { HistoryImportError } from './errors.js';
|
|
3
|
-
import { AGY_FIELD_MAP, AGY_SCHEMA, agySplit, CLAUDE_FIELD_MAP, CLAUDE_SCHEMA, CODEX_FIELD_MAP, CODEX_SCHEMA, claudeSplit, codexSplit, GEMINI_FIELD_MAP, GEMINI_SCHEMA, GROK_FIELD_MAP, GROK_SCHEMA, geminiSplit, grokSplit, OMP_FIELD_MAP, OMP_SCHEMA, ompSplit, PI_FIELD_MAP, PI_SCHEMA, piSplit, } from './mappers.js';
|
|
3
|
+
import { AGY_FIELD_MAP, AGY_SCHEMA, agySplit, CLAUDE_FIELD_MAP, CLAUDE_SCHEMA, CODEX_FIELD_MAP, CODEX_SCHEMA, claudeSplit, codexSplit, DSH_FIELD_MAP, DSH_SCHEMA, dshSplit, GEMINI_FIELD_MAP, GEMINI_SCHEMA, GROK_FIELD_MAP, GROK_SCHEMA, geminiSplit, grokSplit, OMP_FIELD_MAP, OMP_SCHEMA, ompSplit, PI_FIELD_MAP, PI_SCHEMA, piSplit, } from './mappers.js';
|
|
4
4
|
const sourceRecordSchema = z
|
|
5
5
|
.object({
|
|
6
6
|
source_record_id: z.string().min(1),
|
|
@@ -111,6 +111,13 @@ export const SOURCE_DEFINITIONS = {
|
|
|
111
111
|
corruptLinePolicy: 'skip',
|
|
112
112
|
},
|
|
113
113
|
gemini: customSourceDefinition('gemini', 'Gemini CLI', ['.gemini/tmp', '.config/gemini'], ['*.jsonl'], geminiSplit, GEMINI_FIELD_MAP, GEMINI_SCHEMA),
|
|
114
|
+
deepseek: {
|
|
115
|
+
...customSourceDefinition('deepseek', 'DeepSeek dsh', ['.dsh/sessions'], ['session.v3.jsonl', 'session.v3.jsonl.zstd'], dshSplit, DSH_FIELD_MAP, DSH_SCHEMA),
|
|
116
|
+
// dsh crash recovery can tear the final append-only line; unknown plugin
|
|
117
|
+
// event types interleave freely (task 0067 R4). Skipping keeps every good
|
|
118
|
+
// message record importable — same rationale as agy/task 0623.
|
|
119
|
+
corruptLinePolicy: 'skip',
|
|
120
|
+
},
|
|
114
121
|
opencode: sourceDefinition('opencode', 'OpenCode', ['.opencode', '.local/share/opencode'], ['*.jsonl']),
|
|
115
122
|
antigravity: sourceDefinition('antigravity', 'Antigravity', ['.antigravity'], ['*.jsonl']),
|
|
116
123
|
openclaw: sourceDefinition('openclaw', 'OpenClaw', ['.openclaw'], ['*.jsonl']),
|
package/dist/types.d.ts
CHANGED
|
@@ -2,7 +2,7 @@ import type { DbAdapter } from '@gobing-ai/ts-db';
|
|
|
2
2
|
import type { FileSystem, RuntimePaths } from '@gobing-ai/ts-runtime';
|
|
3
3
|
import type { z } from 'zod';
|
|
4
4
|
/** Built-in source identifiers supported by the importer. */
|
|
5
|
-
export type LlmJsonlSource = 'pi' | 'claude' | 'codex' | 'gemini' | 'opencode' | 'antigravity' | 'openclaw' | 'omp' | 'grok' | 'agy';
|
|
5
|
+
export type LlmJsonlSource = 'pi' | 'claude' | 'codex' | 'gemini' | 'opencode' | 'antigravity' | 'openclaw' | 'omp' | 'grok' | 'agy' | 'deepseek';
|
|
6
6
|
/** Import mode controlling checkpoint behavior. */
|
|
7
7
|
export type ImportMode = 'full' | 'incremental' | 'force-file';
|
|
8
8
|
/** One record emitted by a custom split, optionally routed to its own table. */
|
package/dist/types.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAClD,OAAO,KAAK,EAAE,UAAU,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAC;AACtE,OAAO,KAAK,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAE7B,6DAA6D;AAC7D,MAAM,MAAM,cAAc,GACpB,IAAI,GACJ,QAAQ,GACR,OAAO,GACP,QAAQ,GACR,UAAU,GACV,aAAa,GACb,UAAU,GACV,KAAK,GACL,MAAM,GACN,KAAK,CAAC;
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAClD,OAAO,KAAK,EAAE,UAAU,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAC;AACtE,OAAO,KAAK,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAE7B,6DAA6D;AAC7D,MAAM,MAAM,cAAc,GACpB,IAAI,GACJ,QAAQ,GACR,OAAO,GACP,QAAQ,GACR,UAAU,GACV,aAAa,GACb,UAAU,GACV,KAAK,GACL,MAAM,GACN,KAAK,GACL,UAAU,CAAC;AAEjB,mDAAmD;AACnD,MAAM,MAAM,UAAU,GAAG,MAAM,GAAG,aAAa,GAAG,YAAY,CAAC;AAE/D,gFAAgF;AAChF,MAAM,WAAW,UAAU;IACvB,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,MAAM,EAAE,UAAU,CAAC;CAC/B;AAED,kFAAkF;AAClF,MAAM,MAAM,WAAW,GACjB;IACI,QAAQ,CAAC,IAAI,EAAE,YAAY,CAAC;CAC/B,GACD;IACI,QAAQ,CAAC,IAAI,EAAE,aAAa,CAAC;IAC7B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,CAAC;CACjC,GACD;IACI,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAC;IACxB,QAAQ,CAAC,KAAK,EAAE,CAAC,GAAG,EAAE,UAAU,EAAE,OAAO,CAAC,EAAE,gBAAgB,KAAK,SAAS,CAAC,UAAU,GAAG,UAAU,CAAC,EAAE,CAAC;IACtG,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,CAAC;CACjC,CAAC;AAER,4DAA4D;AAC5D,MAAM,MAAM,UAAU,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;AAEjD,sDAAsD;AACtD,MAAM,MAAM,cAAc,GAAG,CAAC,KAAK,EAAE,OAAO,EAAE,GAAG,EAAE,UAAU,EAAE,OAAO,EAAE,gBAAgB,KAAK,OAAO,CAAC;AAErG,0DAA0D;AAC1D,MAAM,WAAW,gBAAgB;IAC7B,2FAA2F;IAC3F,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;CAC/B;AAED,qEAAqE;AACrE,MAAM,WAAW,gBAAgB,CAAC,OAAO,SAAS,UAAU,GAAG,UAAU;IACrE,iHAAiH;IACjH,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,YAAY,EAAE,SAAS,MAAM,EAAE,CAAC;IACzC,QAAQ,CAAC,YAAY,EAAE,SAAS,MAAM,EAAE,CAAC;IACzC;;;;;OAKG;IACH,QAAQ,CAAC,iBAAiB,CAAC,EAAE,OAAO,GAAG,MAAM,CAAC;IAC9C,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,WAAW,EAAE,WAAW,CAAC;IAClC,QAAQ,CAAC,QAAQ,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;IACpD,QAAQ,CAAC,eAAe,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,cAAc,CAAC,CAAC,CAAC;IACnE,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;CACvC;AAED,6DAA6D;AAC7D,MAAM,WAAW,aAAa;IAC1B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;CAChC;AAED,oCAAoC;AACpC,MAAM,WAAW,aAAa;IAC1B,QAAQ,CAAC,EAAE,EAAE,SAAS,CAAC;IACvB,QAAQ,CAAC,UAAU,CAAC,EAAE,UAAU,CAAC;IACjC,QAAQ,CAAC,IAAI,CAAC,EAAE,UAAU,CAAC;IAC3B,QAAQ,CAAC,KAAK,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IACnC,QAAQ,CAAC,KAAK,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IACnC,QAAQ,CAAC,MAAM,CAAC,EAAE,OAAO,CAAC;IAC1B,QAAQ,CAAC,cAAc,CAAC,EAAE,SAAS,aAAa,EAAE,CAAC;IACnD,QAAQ,CAAC,GAAG,CAAC,EAAE,MAAM,IAAI,CAAC;IAC1B;;;;;;;;;OASG;IACH,QAAQ,CAAC,MAAM,CAAC,EAAE,WAAW,CAAC;IAC9B;;;;OAIG;IACH,QAAQ,CAAC,KAAK,CAAC,EAAE,YAAY,CAAC;CACjC;AAED,mFAAmF;AACnF,MAAM,WAAW,WAAW;IACxB,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;CAC3B;AAED;;;;;GAKG;AACH,MAAM,WAAW,gBAAgB;IAC7B,uGAAuG;IACvG,QAAQ,CAAC,eAAe,EAAE,MAAM,CAAC;IACjC,iDAAiD;IACjD,QAAQ,CAAC,eAAe,EAAE,MAAM,CAAC;IACjC,qEAAqE;IACrE,QAAQ,CAAC,mBAAmB,EAAE,MAAM,CAAC;CACxC;AAED,yDAAyD;AACzD,MAAM,WAAW,YAAY;IACzB,iGAAiG;IACjG,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,IAAI,EAAE,UAAU,CAAC;IAC1B,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC,QAAQ,CAAC,eAAe,EAAE,MAAM,CAAC;IACjC,QAAQ,CAAC,iBAAiB,EAAE,MAAM,CAAC;IACnC,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC,oFAAoF;IACpF,QAAQ,CAAC,mBAAmB,EAAE,MAAM,CAAC;IACrC,4FAA4F;IAC5F,QAAQ,CAAC,qBAAqB,EAAE,MAAM,CAAC;IACvC,QAAQ,CAAC,WAAW,EAAE,SAAS,WAAW,EAAE,CAAC;IAC7C,QAAQ,CAAC,gBAAgB,EAAE,SAAS,WAAW,EAAE,CAAC;IAClD,QAAQ,CAAC,iBAAiB,EAAE,MAAM,CAAC;IACnC;;;OAGG;IACH,QAAQ,CAAC,cAAc,CAAC,EAAE,gBAAgB,CAAC;CAC9C;AAED;;;;;;;GAOG;AACH;;;;;GAKG;AACH,MAAM,WAAW,oBAAoB;IACjC,QAAQ,CAAC,kBAAkB,CAAC,EAAE,MAAM,CAAC;IACrC,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,eAAe,EAAE,MAAM,GAAG,OAAO,CAAC;IAC3C,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACpC,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAClC,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACrC,QAAQ,CAAC,OAAO,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACjC,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAChC,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACpC,QAAQ,CAAC,YAAY,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACtC,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CACxC;AAED,qEAAqE;AACrE,MAAM,WAAW,SAAS;IACtB,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,eAAe,EAAE,MAAM,GAAG,OAAO,CAAC;IAC3C,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACpC,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAClC,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACrC,QAAQ,CAAC,OAAO,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACjC,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAChC,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACpC,QAAQ,CAAC,YAAY,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACtC,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACrC,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;CAChC"}
|
package/dist/zstd.d.ts
ADDED
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Decompress one checksummed Zstandard session log through the system `zstd`
|
|
3
|
+
* CLI (task 0067 R5 — sanctioned process seam, ADR-014). A missing executable
|
|
4
|
+
* or a decompression failure raises an actionable HistoryImportError naming
|
|
5
|
+
* zstd and the file — never a bare spawn failure.
|
|
6
|
+
*
|
|
7
|
+
* ponytail: buffered decompression; swap to `ProcessExecutor.runStreaming`
|
|
8
|
+
* (`zstd -dc <file>` stdout piping) if live sessions regularly exceed ~100 MB
|
|
9
|
+
* decompressed.
|
|
10
|
+
*/
|
|
11
|
+
export declare function zstdDecompress(file: string, env?: Record<string, string>): Promise<string>;
|
|
12
|
+
//# sourceMappingURL=zstd.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"zstd.d.ts","sourceRoot":"","sources":["../src/zstd.ts"],"names":[],"mappings":"AAGA;;;;;;;;;GASG;AACH,wBAAsB,cAAc,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,OAAO,CAAC,MAAM,CAAC,CAmBhG"}
|
package/dist/zstd.js
ADDED
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
import { NodeProcessExecutor } from '@gobing-ai/ts-runtime';
|
|
2
|
+
import { HistoryImportError } from './errors.js';
|
|
3
|
+
/**
|
|
4
|
+
* Decompress one checksummed Zstandard session log through the system `zstd`
|
|
5
|
+
* CLI (task 0067 R5 — sanctioned process seam, ADR-014). A missing executable
|
|
6
|
+
* or a decompression failure raises an actionable HistoryImportError naming
|
|
7
|
+
* zstd and the file — never a bare spawn failure.
|
|
8
|
+
*
|
|
9
|
+
* ponytail: buffered decompression; swap to `ProcessExecutor.runStreaming`
|
|
10
|
+
* (`zstd -dc <file>` stdout piping) if live sessions regularly exceed ~100 MB
|
|
11
|
+
* decompressed.
|
|
12
|
+
*/
|
|
13
|
+
export async function zstdDecompress(file, env) {
|
|
14
|
+
const result = await new NodeProcessExecutor().run({
|
|
15
|
+
command: 'zstd',
|
|
16
|
+
args: ['-dc', file],
|
|
17
|
+
env,
|
|
18
|
+
timeout: 120_000,
|
|
19
|
+
label: 'llm-jsonl-importer:zstd-decompression',
|
|
20
|
+
});
|
|
21
|
+
// execa resolves an unresolvable executable with a null exit code (no child
|
|
22
|
+
// ran) — that is the "no zstd on PATH" case; a numeric non-zero code is a
|
|
23
|
+
// real zstd failure (corrupt frame, missing file, bad checksum).
|
|
24
|
+
if (result.exitCode === null || result.exitCode !== 0) {
|
|
25
|
+
const reason = result.exitCode === null
|
|
26
|
+
? 'the zstd executable is required on PATH to import *.jsonl.zstd session logs'
|
|
27
|
+
: `zstd (exit ${result.exitCode}) reported: ${result.stderr.trim().slice(0, 500) || result.stdout.trim().slice(0, 200)}`;
|
|
28
|
+
throw new HistoryImportError(`zstd decompression failed for ${file}: ${reason}`, { file });
|
|
29
|
+
}
|
|
30
|
+
return result.stdout;
|
|
31
|
+
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@gobing-ai/ts-llm-jsonl-importer",
|
|
3
|
-
"version": "0.4.
|
|
3
|
+
"version": "0.4.66",
|
|
4
4
|
"description": "@gobing-ai/ts-llm-jsonl-importer — Generic JSONL importer for LLM agent history files.",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"typescript",
|
|
@@ -47,9 +47,9 @@
|
|
|
47
47
|
"release": "echo 'Manual publish is disabled. Releases go through GitHub Actions via Trusted Publishing — push a tag: git tag @gobing-ai/ts-llm-jsonl-importer-v<version> && git push --tags' && exit 1"
|
|
48
48
|
},
|
|
49
49
|
"dependencies": {
|
|
50
|
-
"@gobing-ai/ts-db": "^0.4.
|
|
51
|
-
"@gobing-ai/ts-runtime": "^0.4.
|
|
52
|
-
"@gobing-ai/ts-utils": "^0.4.
|
|
50
|
+
"@gobing-ai/ts-db": "^0.4.66",
|
|
51
|
+
"@gobing-ai/ts-runtime": "^0.4.66",
|
|
52
|
+
"@gobing-ai/ts-utils": "^0.4.66",
|
|
53
53
|
"zod": "^4.1.0"
|
|
54
54
|
},
|
|
55
55
|
"devDependencies": {
|
package/src/importer.ts
CHANGED
|
@@ -3,6 +3,7 @@ import {
|
|
|
3
3
|
ambientRuntimePaths,
|
|
4
4
|
createNodeFileSystem,
|
|
5
5
|
type FileSystem,
|
|
6
|
+
getProcessEnv,
|
|
6
7
|
type RuntimePaths,
|
|
7
8
|
resolvePath,
|
|
8
9
|
walkDir,
|
|
@@ -36,6 +37,7 @@ import type {
|
|
|
36
37
|
SourceDefinition,
|
|
37
38
|
TransformContext,
|
|
38
39
|
} from './types';
|
|
40
|
+
import { zstdDecompress } from './zstd';
|
|
39
41
|
|
|
40
42
|
interface SplitRecord {
|
|
41
43
|
readonly targetTable: string;
|
|
@@ -645,10 +647,14 @@ async function discoverFiles(
|
|
|
645
647
|
// Previously both kinds anchored to ambient cwd, so a cwd ≠ $HOME silently skipped every
|
|
646
648
|
// registry source because the home-relative paths never existed relative to cwd.
|
|
647
649
|
const ambient = paths ?? ambientRuntimePaths();
|
|
650
|
+
// deepseek (task 0067 R1): $DSH_HOME replaces the `~/.dsh` storage root when set;
|
|
651
|
+
// session-persistence-jsonl mounts its sessions under `<root>/sessions`.
|
|
652
|
+
const dshHome = definition.source === 'deepseek' ? nonEmptyString(getProcessEnv().DSH_HOME) : undefined;
|
|
653
|
+
const defaultRoots = dshHome !== undefined ? [`${dshHome}/sessions`] : (definition.defaultRoots ?? []);
|
|
648
654
|
const resolvedRoots =
|
|
649
655
|
roots !== undefined
|
|
650
656
|
? roots.map((root) => resolvePath(ambient.cwd, root))
|
|
651
|
-
:
|
|
657
|
+
: defaultRoots.map((root) => resolvePath(ambient.home, root));
|
|
652
658
|
const found = new Set<string>();
|
|
653
659
|
for (const root of resolvedRoots) {
|
|
654
660
|
if (!(await fileSystem.exists(root))) continue;
|
|
@@ -693,6 +699,14 @@ function matchesPattern(path: string, patterns: readonly string[]): boolean {
|
|
|
693
699
|
});
|
|
694
700
|
}
|
|
695
701
|
|
|
702
|
+
/** First non-empty string among values (task 0067: $DSH_HOME env override). */
|
|
703
|
+
function nonEmptyString(...values: readonly unknown[]): string | undefined {
|
|
704
|
+
for (const value of values) {
|
|
705
|
+
if (typeof value === 'string' && value.length > 0) return value;
|
|
706
|
+
}
|
|
707
|
+
return undefined;
|
|
708
|
+
}
|
|
709
|
+
|
|
696
710
|
/**
|
|
697
711
|
* Stat a file for its identity, returning null on any failure (missing file, stat unsupported).
|
|
698
712
|
* 0675 R2: the short-circuit fails OPEN — any null falls through to the normal read path.
|
|
@@ -715,6 +729,16 @@ async function safeStat(fileSystem: FileSystem, file: string): Promise<{ size: n
|
|
|
715
729
|
* as before, preserving parity for stubs like CF Workers.
|
|
716
730
|
*/
|
|
717
731
|
async function* readLines(fileSystem: FileSystem, file: string): AsyncGenerator<string> {
|
|
732
|
+
if (file.endsWith('.jsonl.zstd')) {
|
|
733
|
+
// task 0067: checksummed Zstandard session logs decompress through the
|
|
734
|
+
// system zstd CLI before line parsing; the same errors/checkpointing apply
|
|
735
|
+
// to the decompressed line stream.
|
|
736
|
+
const decompressed = await zstdDecompress(file);
|
|
737
|
+
for (const line of decompressed.split(/\r?\n/)) {
|
|
738
|
+
yield line;
|
|
739
|
+
}
|
|
740
|
+
return;
|
|
741
|
+
}
|
|
718
742
|
if (fileSystem.readFileStream) {
|
|
719
743
|
yield* fileSystem.readFileStream(file);
|
|
720
744
|
return;
|
package/src/index.ts
CHANGED
|
@@ -98,7 +98,9 @@ const TYPED_TABLE_COLUMNS: Readonly<Record<string, readonly string[]>> = {
|
|
|
98
98
|
};
|
|
99
99
|
|
|
100
100
|
/** Keys that a typed table mapper may produce that are not columns. */
|
|
101
|
-
|
|
101
|
+
// `split_index` and `source_record_id` (task 0067 R2) ride the normalized record into the
|
|
102
|
+
// record_hash but are not history_message columns — identity-relevant, never persisted.
|
|
103
|
+
const TYPED_IGNORED_KEYS = new Set<string>(['_meta', 'split_index', 'source_record_id']);
|
|
102
104
|
|
|
103
105
|
function targetTableFor(table: string): string {
|
|
104
106
|
if (!VALID_TABLE_NAME.test(table)) {
|
package/src/mappers.ts
CHANGED
|
@@ -1440,8 +1440,95 @@ export function geminiSplit(raw: Record<string, unknown>, context?: TransformCon
|
|
|
1440
1440
|
return entries;
|
|
1441
1441
|
}
|
|
1442
1442
|
|
|
1443
|
+
// ---------------------------------------------------------------------------
|
|
1444
|
+
// DeepSeek dsh mapper (task 0067)
|
|
1445
|
+
// ---------------------------------------------------------------------------
|
|
1446
|
+
|
|
1447
|
+
/**
|
|
1448
|
+
* Map one DeepSeek dsh `session.v3.jsonl` line into forensic ETL entries (task 0067 R2).
|
|
1449
|
+
*
|
|
1450
|
+
* dsh layout: line 1 is the session header `{type:"session", version:3, id, createdAt, cwd, ...}`;
|
|
1451
|
+
*
|
|
1452
|
+
* any remaining lines are append-only events `{type, seq, time, data}` whose
|
|
1453
|
+
* vocabulary is plugin-extensible. Only chat events (`user/message`,
|
|
1454
|
+
* `assistant/message`) become history_message rows (content is an array of
|
|
1455
|
+
* `{text}` blocks — the existing extractContentText joined-text rule); every
|
|
1456
|
+
* unrecognized event type (compaction/*, hook/*, turn/*, session/title, …) is
|
|
1457
|
+
* tolerated and skipped, and torn crash tails never reach the mapper because
|
|
1458
|
+
* the source registers `corruptLinePolicy: 'skip'`. Session identity comes from
|
|
1459
|
+
* the `session-<uuid>` dir path (stateless — message events carry no session id),
|
|
1460
|
+
* mirroring the agy/codex path-identity precedent.
|
|
1461
|
+
*/
|
|
1462
|
+
export function dshSplit(raw: Record<string, unknown>, context?: TransformContext): readonly SplitEntry[] {
|
|
1463
|
+
const recordType = String(raw.type ?? '');
|
|
1464
|
+
|
|
1465
|
+
// Session header: bookkeeping meta row carrying cwd/createdAt for the session.
|
|
1466
|
+
if (recordType === 'session') {
|
|
1467
|
+
const sessionId = s(raw.id) ?? sessionIdFromSourcePath('deepseek', context?.sourceFile ?? '') ?? 'unknown';
|
|
1468
|
+
return [
|
|
1469
|
+
{
|
|
1470
|
+
targetTable: 'history_message',
|
|
1471
|
+
record: {
|
|
1472
|
+
session_id: sessionId,
|
|
1473
|
+
seq: 0,
|
|
1474
|
+
role: 'meta',
|
|
1475
|
+
record_type: 'session',
|
|
1476
|
+
disposition: 'meta',
|
|
1477
|
+
ts: timestampOf(raw.createdAt),
|
|
1478
|
+
cwd: s(raw.cwd) ?? null,
|
|
1479
|
+
content_text: null,
|
|
1480
|
+
provenance: 'ambient',
|
|
1481
|
+
},
|
|
1482
|
+
},
|
|
1483
|
+
];
|
|
1484
|
+
}
|
|
1485
|
+
|
|
1486
|
+
// Chat events only — unknown plugin-extensible types are dropped, not records.
|
|
1487
|
+
if (recordType !== 'user/message' && recordType !== 'assistant/message') return [];
|
|
1488
|
+
|
|
1489
|
+
const data = o(raw.data);
|
|
1490
|
+
const msg = o(data.message);
|
|
1491
|
+
const seq = typeof raw.seq === 'number' ? raw.seq : (context?.sourceLine ?? 0);
|
|
1492
|
+
const sessionId = sessionIdFromSourcePath('deepseek', context?.sourceFile ?? '') ?? 'unknown';
|
|
1493
|
+
const role = recordType === 'user/message' ? 'user' : 'assistant';
|
|
1494
|
+
const source = o(data.source ?? msg.source);
|
|
1495
|
+
const model = s(source.model, o(data.model ?? msg.model).model);
|
|
1496
|
+
const usage = (data.usage ?? msg.usage) as Record<string, unknown> | undefined;
|
|
1497
|
+
// Task 0067 R2: the message id is the record identity — it rides the normalized
|
|
1498
|
+
// record into the record_hash, so identity derives from `data.message.id` when
|
|
1499
|
+
// present and falls back to the importer's line-derived record hash when absent.
|
|
1500
|
+
const messageId = s(msg.id, data.id);
|
|
1501
|
+
|
|
1502
|
+
return [
|
|
1503
|
+
{
|
|
1504
|
+
targetTable: 'history_message',
|
|
1505
|
+
record: {
|
|
1506
|
+
session_id: sessionId,
|
|
1507
|
+
seq,
|
|
1508
|
+
role,
|
|
1509
|
+
record_type: recordType,
|
|
1510
|
+
disposition: 'keep',
|
|
1511
|
+
...(messageId !== undefined ? { source_record_id: messageId } : {}),
|
|
1512
|
+
ts: timestampOf(raw.time, data.time, msg.time),
|
|
1513
|
+
duration_ms: null,
|
|
1514
|
+
model: model ?? null,
|
|
1515
|
+
input_tokens: numberOrNull(usage?.input_tokens ?? usage?.input) ?? null,
|
|
1516
|
+
output_tokens: numberOrNull(usage?.output_tokens ?? usage?.output) ?? null,
|
|
1517
|
+
cache_read_tokens: null,
|
|
1518
|
+
cache_write_tokens: null,
|
|
1519
|
+
cost_usd: null,
|
|
1520
|
+
content_text: extractContentText(data.content ?? msg.content) ?? s(data.text, msg.text) ?? null,
|
|
1521
|
+
cwd: null,
|
|
1522
|
+
provenance: 'ambient',
|
|
1523
|
+
},
|
|
1524
|
+
},
|
|
1525
|
+
];
|
|
1526
|
+
}
|
|
1527
|
+
|
|
1443
1528
|
const UUID_PATTERN = '[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}';
|
|
1444
1529
|
const AGY_BRAIN_REGEX = new RegExp(`(?:^|\\/)brain\\/(${UUID_PATTERN})(?:\\/|$)`, 'i');
|
|
1530
|
+
/** DSH session-dir identity: `<normalized-abs-cwd>--/session-<uuid>/`. */
|
|
1531
|
+
const DSH_SESSION_REGEX = new RegExp(`(?:^|\\/)session-(${UUID_PATTERN})(?:\\/|$)`, 'i');
|
|
1445
1532
|
const CODEX_ROLLOUT_REGEX = new RegExp(`(?:^|-|_)(${UUID_PATTERN})$`, 'i');
|
|
1446
1533
|
|
|
1447
1534
|
/**
|
|
@@ -1458,6 +1545,10 @@ export function sessionIdFromSourcePath(source: string, sourceFile: string): str
|
|
|
1458
1545
|
const match = normalized.match(AGY_BRAIN_REGEX);
|
|
1459
1546
|
return match?.[1];
|
|
1460
1547
|
}
|
|
1548
|
+
if (sLower === 'deepseek') {
|
|
1549
|
+
const match = normalized.match(DSH_SESSION_REGEX);
|
|
1550
|
+
return match?.[1];
|
|
1551
|
+
}
|
|
1461
1552
|
if (sLower === 'codex') {
|
|
1462
1553
|
const basename = normalized.split('/').at(-1) ?? '';
|
|
1463
1554
|
if (basename.startsWith('rollout-') && basename.endsWith('.jsonl')) {
|
|
@@ -2085,6 +2176,8 @@ export const GROK_FIELD_MAP = identityFieldMap(
|
|
|
2085
2176
|
export const GEMINI_FIELD_MAP = identityFieldMap(
|
|
2086
2177
|
MESSAGE_MAPPER_KEYS.concat(TOOL_CALL_MAPPER_KEYS, SKILL_CALL_MAPPER_KEYS),
|
|
2087
2178
|
);
|
|
2179
|
+
/** Identity field map for the DeepSeek dsh mapper — message rows only, no tool-call rows. */
|
|
2180
|
+
export const DSH_FIELD_MAP = identityFieldMap(MESSAGE_MAPPER_KEYS.concat(['source_record_id']));
|
|
2088
2181
|
|
|
2089
2182
|
// ---------------------------------------------------------------------------
|
|
2090
2183
|
// Zod schemas for the six source mappers
|
|
@@ -2106,3 +2199,5 @@ export const AGY_SCHEMA = passthroughSchema;
|
|
|
2106
2199
|
export const GROK_SCHEMA = passthroughSchema;
|
|
2107
2200
|
/** Passthrough Zod schema for the Gemini mapper. */
|
|
2108
2201
|
export const GEMINI_SCHEMA = passthroughSchema;
|
|
2202
|
+
/** Passthrough Zod schema for the DeepSeek dsh mapper. */
|
|
2203
|
+
export const DSH_SCHEMA = passthroughSchema;
|
package/src/schema-sql.ts
CHANGED
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
* Guaranteed to match the package version; a bump-or-fail test enforces that
|
|
5
5
|
* any modification to HISTORY_IMPORT_SCHEMA_SQL must be accompanied by a version bump.
|
|
6
6
|
*/
|
|
7
|
-
export const HISTORY_IMPORT_SCHEMA_VERSION = '0.4.
|
|
7
|
+
export const HISTORY_IMPORT_SCHEMA_VERSION = '0.4.66';
|
|
8
8
|
|
|
9
9
|
/**
|
|
10
10
|
* DDL string that creates the history import checkpoint, ledger, and typed contract tables.
|
package/src/sources.ts
CHANGED
|
@@ -10,6 +10,9 @@ import {
|
|
|
10
10
|
CODEX_SCHEMA,
|
|
11
11
|
claudeSplit,
|
|
12
12
|
codexSplit,
|
|
13
|
+
DSH_FIELD_MAP,
|
|
14
|
+
DSH_SCHEMA,
|
|
15
|
+
dshSplit,
|
|
13
16
|
GEMINI_FIELD_MAP,
|
|
14
17
|
GEMINI_SCHEMA,
|
|
15
18
|
GROK_FIELD_MAP,
|
|
@@ -214,6 +217,21 @@ export const SOURCE_DEFINITIONS: Readonly<Record<LlmJsonlSource, SourceDefinitio
|
|
|
214
217
|
GEMINI_FIELD_MAP,
|
|
215
218
|
GEMINI_SCHEMA,
|
|
216
219
|
),
|
|
220
|
+
deepseek: {
|
|
221
|
+
...customSourceDefinition(
|
|
222
|
+
'deepseek',
|
|
223
|
+
'DeepSeek dsh',
|
|
224
|
+
['.dsh/sessions'],
|
|
225
|
+
['session.v3.jsonl', 'session.v3.jsonl.zstd'],
|
|
226
|
+
dshSplit,
|
|
227
|
+
DSH_FIELD_MAP,
|
|
228
|
+
DSH_SCHEMA,
|
|
229
|
+
),
|
|
230
|
+
// dsh crash recovery can tear the final append-only line; unknown plugin
|
|
231
|
+
// event types interleave freely (task 0067 R4). Skipping keeps every good
|
|
232
|
+
// message record importable — same rationale as agy/task 0623.
|
|
233
|
+
corruptLinePolicy: 'skip',
|
|
234
|
+
},
|
|
217
235
|
opencode: sourceDefinition('opencode', 'OpenCode', ['.opencode', '.local/share/opencode'], ['*.jsonl']),
|
|
218
236
|
antigravity: sourceDefinition('antigravity', 'Antigravity', ['.antigravity'], ['*.jsonl']),
|
|
219
237
|
openclaw: sourceDefinition('openclaw', 'OpenClaw', ['.openclaw'], ['*.jsonl']),
|
package/src/types.ts
CHANGED
package/src/zstd.ts
ADDED
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
import { NodeProcessExecutor } from '@gobing-ai/ts-runtime';
|
|
2
|
+
import { HistoryImportError } from './errors';
|
|
3
|
+
|
|
4
|
+
/**
|
|
5
|
+
* Decompress one checksummed Zstandard session log through the system `zstd`
|
|
6
|
+
* CLI (task 0067 R5 — sanctioned process seam, ADR-014). A missing executable
|
|
7
|
+
* or a decompression failure raises an actionable HistoryImportError naming
|
|
8
|
+
* zstd and the file — never a bare spawn failure.
|
|
9
|
+
*
|
|
10
|
+
* ponytail: buffered decompression; swap to `ProcessExecutor.runStreaming`
|
|
11
|
+
* (`zstd -dc <file>` stdout piping) if live sessions regularly exceed ~100 MB
|
|
12
|
+
* decompressed.
|
|
13
|
+
*/
|
|
14
|
+
export async function zstdDecompress(file: string, env?: Record<string, string>): Promise<string> {
|
|
15
|
+
const result = await new NodeProcessExecutor().run({
|
|
16
|
+
command: 'zstd',
|
|
17
|
+
args: ['-dc', file],
|
|
18
|
+
env,
|
|
19
|
+
timeout: 120_000,
|
|
20
|
+
label: 'llm-jsonl-importer:zstd-decompression',
|
|
21
|
+
});
|
|
22
|
+
// execa resolves an unresolvable executable with a null exit code (no child
|
|
23
|
+
// ran) — that is the "no zstd on PATH" case; a numeric non-zero code is a
|
|
24
|
+
// real zstd failure (corrupt frame, missing file, bad checksum).
|
|
25
|
+
if (result.exitCode === null || result.exitCode !== 0) {
|
|
26
|
+
const reason =
|
|
27
|
+
result.exitCode === null
|
|
28
|
+
? 'the zstd executable is required on PATH to import *.jsonl.zstd session logs'
|
|
29
|
+
: `zstd (exit ${result.exitCode}) reported: ${result.stderr.trim().slice(0, 500) || result.stdout.trim().slice(0, 200)}`;
|
|
30
|
+
throw new HistoryImportError(`zstd decompression failed for ${file}: ${reason}`, { file });
|
|
31
|
+
}
|
|
32
|
+
return result.stdout;
|
|
33
|
+
}
|