@gobing-ai/ts-llm-jsonl-importer 0.4.55 → 0.4.56
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/jsonl-importer-dao.d.ts +17 -4
- package/dist/jsonl-importer-dao.d.ts.map +1 -1
- package/dist/jsonl-importer-dao.js +96 -1
- package/dist/schema-sql.d.ts +1 -1
- package/dist/schema-sql.d.ts.map +1 -1
- package/dist/schema-sql.js +8 -2
- package/package.json +4 -4
- package/src/jsonl-importer-dao.ts +95 -1
- package/src/schema-sql.ts +8 -2
|
@@ -57,10 +57,23 @@ declare function resetCheckpoints(db: ImportOptions['db'], source: string, files
|
|
|
57
57
|
declare function writeCheckpoint(db: ImportOptions['db'], source: string, sourceFile: string, line: number, now: ImportOptions['now']): Promise<void>;
|
|
58
58
|
declare function ledgerExists(db: ImportOptions['db'], recordHash: string): Promise<boolean>;
|
|
59
59
|
/**
|
|
60
|
-
*
|
|
61
|
-
*
|
|
62
|
-
*
|
|
63
|
-
*
|
|
60
|
+
* Tool identity for one `history_tool_call` row (0739).
|
|
61
|
+
*
|
|
62
|
+
* `effective` is extraction — recover the name when the agent recorded it inside a wrapper's
|
|
63
|
+
* arguments or only in the call id. `alias` is canonicalization — grouping the same logical tool
|
|
64
|
+
* across agents — and starts at identity, because the alias vocabulary is a curated decision
|
|
65
|
+
* that belongs to the consumer, not to the importer.
|
|
66
|
+
*
|
|
67
|
+
* Mirrors Spur migration 0034's backfill CASE so a row imported after the migration and a row
|
|
68
|
+
* backfilled by it resolve identically.
|
|
69
|
+
*/
|
|
70
|
+
export declare function resolveToolIdentity(payload: JsonObject): {
|
|
71
|
+
effective: string;
|
|
72
|
+
alias: string;
|
|
73
|
+
};
|
|
74
|
+
/**
|
|
75
|
+
* Build the insert op for one imported record, routing to the typed or JSON
|
|
76
|
+
* column path per target table.
|
|
64
77
|
*/
|
|
65
78
|
export declare function recordInsertOp(targetTable: string, recordHash: string, sourceFile: string, sourceLine: number, splitIndex: number, payload: JsonObject, now: ImportOptions['now']): DbBatchOp;
|
|
66
79
|
/** INSERT op for one ledger row (task 0060 F9). */
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"jsonl-importer-dao.d.ts","sourceRoot":"","sources":["../src/jsonl-importer-dao.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAI7D,OAAO,KAAK,EAAE,aAAa,EAAE,UAAU,EAAE,gBAAgB,EAAE,gBAAgB,EAAE,MAAM,SAAS,CAAC;AAE7F,UAAU,aAAa;IACnB,QAAQ,CAAC,kBAAkB,EAAE,MAAM,CAAC;IACpC,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACrC,QAAQ,CAAC,eAAe,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CAC5C;AAED,gEAAgE;AAChE,MAAM,WAAW,gBAAgB;IAC7B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,QAAQ,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;CACnC;AAED;;;;GAIG;AACH,eAAO,MAAM,oBAAoB,yEAA0E,CAAC;
|
|
1
|
+
{"version":3,"file":"jsonl-importer-dao.d.ts","sourceRoot":"","sources":["../src/jsonl-importer-dao.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAI7D,OAAO,KAAK,EAAE,aAAa,EAAE,UAAU,EAAE,gBAAgB,EAAE,gBAAgB,EAAE,MAAM,SAAS,CAAC;AAE7F,UAAU,aAAa;IACnB,QAAQ,CAAC,kBAAkB,EAAE,MAAM,CAAC;IACpC,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACrC,QAAQ,CAAC,eAAe,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CAC5C;AAED,gEAAgE;AAChE,MAAM,WAAW,gBAAgB;IAC7B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,QAAQ,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;CACnC;AAED;;;;GAIG;AACH,eAAO,MAAM,oBAAoB,yEAA0E,CAAC;AA8E5G,iBAAS,cAAc,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAK7C;AAED,iBAAS,SAAS,CAAC,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,GAAG,MAAM,CAEpD;AAED,iBAAS,aAAa,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAS5C;AAED;;;;;;GAMG;AACH,wBAAsB,wBAAwB,CAAC,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,CAOrF;AAED,6FAA6F;AAC7F,iBAAe,iBAAiB,CAAC,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EAAE,WAAW,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAG5F;AAED;;;;;;;;;;GAUG;AACH,iBAAe,kBAAkB,CAAC,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EAAE,UAAU,EAAE,gBAAgB,GAAG,OAAO,CAAC,IAAI,CAAC,CAQtG;AAED,iBAAe,cAAc,CAAC,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAO1G;AAED;;;;;;;;GAQG;AACH,wBAAsB,qBAAqB,CACvC,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EACvB,MAAM,EAAE,MAAM,GACf,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,gBAAgB,CAAC,CAAC,CA0BxC;AAED,iBAAe,gBAAgB,CAAC,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,SAAS,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAIhH;AAED,iBAAe,eAAe,CAC1B,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EACvB,MAAM,EAAE,MAAM,EACd,UAAU,EAAE,MAAM,EAClB,IAAI,EAAE,MAAM,EACZ,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,GAC1B,OAAO,CAAC,IAAI,CAAC,CAYf;AAED,iBAAe,YAAY,CAAC,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EAAE,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAMzF;AAsBD;;;;;;;;;;GAUG;AACH,wBAAgB,mBAAmB,CAAC,OAAO,EAAE,UAAU,GAAG;IAAE,SAAS,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,CAI7F;AA6CD;;;GAGG;AACH,wBAAgB,cAAc,CAC1B,WAAW,EAAE,MAAM,EACnB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,OAAO,EAAE,UAAU,EACnB,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,GAC1B,SAAS,CA0CX;AAED,mDAAmD;AACnD,wBAAgB,cAAc,CAC1B,UAAU,EAAE,MAAM,EAClB,MAAM,EAAE,MAAM,EACd,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,WAAW,EAAE,MAAM,EACnB,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,GAC1B,SAAS,CAMX;AAED,0FAA0F;AAC1F,wBAAgB,kBAAkB,CAC9B,MAAM,EAAE,MAAM,EACd,UAAU,EAAE,MAAM,EAClB,IAAI,EAAE,MAAM,EACZ,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,EACzB,QAAQ,CAAC,EAAE;IAAE,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;IAAC,QAAQ,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI,CAAA;CAAE,GAC7E,SAAS,CAWX;AAED;;;;;;;;GAQG;AACH,iBAAS,wBAAwB,CAC7B,UAAU,EAAE,MAAM,EAClB,SAAS,EAAE,MAAM,GAAG,IAAI,EACxB,WAAW,EAAE,MAAM,GAAG,IAAI,EAC1B,UAAU,EAAE,MAAM,GAAG,IAAI,GAC1B,SAAS,CAOX;AAED;;;GAGG;AACH,iBAAS,2BAA2B,CAAC,UAAU,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,GAAG,SAAS,CAOvF;AAED;2FAC2F;AAC3F,wBAAgB,6BAA6B,CACzC,UAAU,EAAE,MAAM,EAClB,MAAM,EAAE;IAAE,QAAQ,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IAAC,QAAQ,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IAAC,QAAQ,CAAC,SAAS,EAAE,MAAM,GAAG,IAAI,CAAA;CAAE,GAC7G,SAAS,CAOX;AAED;;;GAGG;AACH,wBAAsB,oBAAoB,CAAC,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,SAAS,MAAM,EAAE,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAYnH;AAED,+DAA+D;AAC/D,MAAM,WAAW,0BAA0B;IACvC,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;CACnC;AAED;;GAEG;AACH,wBAAsB,+BAA+B,CAAC,EAAE,EAAE,SAAS,GAAG,OAAO,CAAC,0BAA0B,EAAE,CAAC,CAiB1G;AAED;;GAEG;AACH,wBAAsB,uBAAuB,CACzC,EAAE,EAAE,SAAS,EACb,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,cAAc,EAAE,MAAM,GACvB,OAAO,CAAC,IAAI,CAAC,CAOf;AAED,iBAAe,YAAY,CACvB,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EACvB,WAAW,EAAE,MAAM,EACnB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,OAAO,EAAE,UAAU,EACnB,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,GAC1B,OAAO,CAAC,IAAI,CAAC,CAIf;AAED,iBAAe,YAAY,CACvB,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EACvB,UAAU,EAAE,MAAM,EAClB,MAAM,EAAE,MAAM,EACd,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,WAAW,EAAE,MAAM,EACnB,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,GAC1B,OAAO,CAAC,IAAI,CAAC,CAYf;AAED;;;;;;;;;;;;;;;;;;;GAmBG;AACH,wBAAsB,wBAAwB,CAC1C,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EACvB,eAAe,EAAE,CAAC,UAAU,EAAE,MAAM,KAAK,MAAM,GAAG,IAAI,GAAG,SAAS,GACnE,OAAO,CAAC,IAAI,CAAC,CAWf;AA4GD;;;;;;;;;;;GAWG;AACH,iBAAe,mBAAmB,CAC9B,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EACvB,MAAM,EAAE,MAAM,EACd,aAAa,EAAE,WAAW,CAAC,MAAM,CAAC,EAClC,eAAe,EAAE,SAAS,MAAM,EAAE,EAClC,MAAM,EAAE,OAAO,GAChB,OAAO,CAAC,gBAAgB,CAAC,CA6C3B;AAID,4DAA4D;AAC5D,MAAM,WAAW,kBAAkB;IAC/B,EAAE,EAAE,MAAM,CAAC;IACX,UAAU,EAAE,MAAM,CAAC;IACnB,YAAY,EAAE,MAAM,CAAC;IACrB,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,MAAM,CAAC;CACrB;AAED,iEAAiE;AACjE,MAAM,WAAW,eAAe;IAC5B,EAAE,EAAE,MAAM,CAAC;IACX,UAAU,EAAE,MAAM,CAAC;IACnB,YAAY,EAAE,MAAM,CAAC;IACrB,IAAI,EAAE,MAAM,CAAC;CAChB;AAED,iDAAiD;AACjD,MAAM,WAAW,qBAAqB;IAClC,WAAW,EAAE,MAAM,CAAC;IACpB,YAAY,EAAE,MAAM,CAAC;CACxB;AAED,gEAAgE;AAChE,MAAM,WAAW,mBAAmB;IAChC,WAAW,EAAE,iBAAiB,GAAG,mBAAmB,GAAG,oBAAoB,CAAC;IAC5E,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,EAAE,UAAU,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;CACtB;AAED,4EAA4E;AAC5E,wBAAsB,oBAAoB,CACtC,EAAE,EAAE,SAAS,EACb,QAAQ,EAAE,MAAM,EAChB,MAAM,EAAE,MAAM,EACd,KAAK,EAAE,MAAM,GACd,OAAO,CAAC,kBAAkB,EAAE,CAAC,CAa/B;AAED,qEAAqE;AACrE,wBAAsB,iBAAiB,CAAC,EAAE,EAAE,SAAS,EAAE,UAAU,EAAE,SAAS,MAAM,EAAE,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC,CAShH;AAED,0EAA0E;AAC1E,wBAAsB,2BAA2B,CAAC,EAAE,EAAE,SAAS,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,qBAAqB,EAAE,CAAC,CAAC,CAY9G;AAED,2EAA2E;AAC3E,wBAAgB,wBAAwB,CAAC,OAAO,EAAE,SAAS,qBAAqB,EAAE,GAAG,SAAS,EAAE,CAW/F;AAED,6DAA6D;AAC7D,wBAAgB,yBAAyB,CAAC,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,SAAS,CAKvF;AAED,iFAAiF;AACjF,wBAAgB,2BAA2B,CACvC,OAAO,EAAE,SAAS,mBAAmB,EAAE,EACvC,eAAe,EAAE,SAAS,MAAM,EAAE,EAClC,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,GAC1B,SAAS,EAAE,CAiEb;AAED,YAAY,EAAE,aAAa,EAAE,CAAC;AAC9B,OAAO,EACH,aAAa,EACb,iBAAiB,EACjB,kBAAkB,EAClB,YAAY,EACZ,YAAY,EACZ,YAAY,EACZ,cAAc,EACd,mBAAmB,EACnB,gBAAgB,EAChB,cAAc,EACd,SAAS,EACT,wBAAwB,EACxB,2BAA2B,EAC3B,eAAe,GAClB,CAAC"}
|
|
@@ -55,6 +55,8 @@ const TYPED_TABLE_COLUMNS = {
|
|
|
55
55
|
'result_bytes',
|
|
56
56
|
'error_text',
|
|
57
57
|
'imported_at',
|
|
58
|
+
'effective_tool_name',
|
|
59
|
+
'tool_name_alias',
|
|
58
60
|
],
|
|
59
61
|
history_skill_call: [
|
|
60
62
|
'record_hash',
|
|
@@ -195,6 +197,83 @@ async function ledgerExists(db, recordHash) {
|
|
|
195
197
|
* Typed tables validate unknown keys here; the caller must have ensured the
|
|
196
198
|
* target table exists (the import loop ensures accepted targets before batching).
|
|
197
199
|
*/
|
|
200
|
+
/** Wrapper-style `call_id` prefixes that name the tool the wrapper actually ran. */
|
|
201
|
+
const CALL_ID_TOOL_PREFIXES = [
|
|
202
|
+
['call_bash_', 'bash'],
|
|
203
|
+
['call_read_', 'read'],
|
|
204
|
+
['call_edit_', 'edit'],
|
|
205
|
+
['call_write_', 'write'],
|
|
206
|
+
['call_grep_', 'grep'],
|
|
207
|
+
['call_find_', 'find'],
|
|
208
|
+
['call_ls_', 'ls'],
|
|
209
|
+
];
|
|
210
|
+
/** Argument keys, in precedence order, that carry the tool name when `tool_name` is empty. */
|
|
211
|
+
const ARGS_TOOL_NAME_KEYS = ['tool', 'tool_name', 'toolName', 'name', 'command'];
|
|
212
|
+
/**
|
|
213
|
+
* Tool identity for one `history_tool_call` row (0739).
|
|
214
|
+
*
|
|
215
|
+
* `effective` is extraction — recover the name when the agent recorded it inside a wrapper's
|
|
216
|
+
* arguments or only in the call id. `alias` is canonicalization — grouping the same logical tool
|
|
217
|
+
* across agents — and starts at identity, because the alias vocabulary is a curated decision
|
|
218
|
+
* that belongs to the consumer, not to the importer.
|
|
219
|
+
*
|
|
220
|
+
* Mirrors Spur migration 0034's backfill CASE so a row imported after the migration and a row
|
|
221
|
+
* backfilled by it resolve identically.
|
|
222
|
+
*/
|
|
223
|
+
export function resolveToolIdentity(payload) {
|
|
224
|
+
const supplied = trimmedString(payload.effective_tool_name);
|
|
225
|
+
const effective = supplied !== undefined ? supplied : deriveEffectiveToolName(payload);
|
|
226
|
+
return { effective, alias: trimmedString(payload.tool_name_alias) ?? effective };
|
|
227
|
+
}
|
|
228
|
+
/** Returns the record with tool identity resolved, for insert paths that read columns off the payload. */
|
|
229
|
+
function withToolIdentity(record) {
|
|
230
|
+
const { effective, alias } = resolveToolIdentity(record);
|
|
231
|
+
return { ...record, effective_tool_name: effective, tool_name_alias: alias };
|
|
232
|
+
}
|
|
233
|
+
function trimmedString(value) {
|
|
234
|
+
if (typeof value !== 'string')
|
|
235
|
+
return undefined;
|
|
236
|
+
const trimmed = value.trim();
|
|
237
|
+
return trimmed === '' ? undefined : trimmed;
|
|
238
|
+
}
|
|
239
|
+
function deriveEffectiveToolName(payload) {
|
|
240
|
+
const recorded = trimmedString(payload.tool_name);
|
|
241
|
+
if (recorded !== undefined && recorded !== 'unknown')
|
|
242
|
+
return recorded;
|
|
243
|
+
const fromArgs = toolNameFromArgs(payload.args_raw);
|
|
244
|
+
if (fromArgs !== undefined)
|
|
245
|
+
return fromArgs;
|
|
246
|
+
const callId = typeof payload.call_id === 'string' ? payload.call_id : '';
|
|
247
|
+
for (const [prefix, tool] of CALL_ID_TOOL_PREFIXES) {
|
|
248
|
+
if (callId.startsWith(prefix))
|
|
249
|
+
return tool;
|
|
250
|
+
}
|
|
251
|
+
return 'unknown';
|
|
252
|
+
}
|
|
253
|
+
function toolNameFromArgs(argsRaw) {
|
|
254
|
+
if (typeof argsRaw !== 'string')
|
|
255
|
+
return undefined;
|
|
256
|
+
let parsed;
|
|
257
|
+
try {
|
|
258
|
+
parsed = JSON.parse(argsRaw);
|
|
259
|
+
}
|
|
260
|
+
catch {
|
|
261
|
+
// Non-JSON arguments are the `json_valid(args_raw)` false branch in SQL: fall through.
|
|
262
|
+
return undefined;
|
|
263
|
+
}
|
|
264
|
+
if (parsed === null || typeof parsed !== 'object' || Array.isArray(parsed))
|
|
265
|
+
return undefined;
|
|
266
|
+
for (const key of ARGS_TOOL_NAME_KEYS) {
|
|
267
|
+
const candidate = trimmedString(parsed[key]);
|
|
268
|
+
if (candidate !== undefined)
|
|
269
|
+
return candidate;
|
|
270
|
+
}
|
|
271
|
+
return undefined;
|
|
272
|
+
}
|
|
273
|
+
/**
|
|
274
|
+
* Build the insert op for one imported record, routing to the typed or JSON
|
|
275
|
+
* column path per target table.
|
|
276
|
+
*/
|
|
198
277
|
export function recordInsertOp(targetTable, recordHash, sourceFile, sourceLine, splitIndex, payload, now) {
|
|
199
278
|
const table = targetTableFor(targetTable);
|
|
200
279
|
const typedColumns = TYPED_TABLE_COLUMNS[table];
|
|
@@ -207,11 +286,20 @@ export function recordInsertOp(targetTable, recordHash, sourceFile, sourceLine,
|
|
|
207
286
|
}
|
|
208
287
|
// record_hash and imported_at come from the function parameters, not the payload.
|
|
209
288
|
const ts = timestamp(now);
|
|
289
|
+
// Tool identity is derived here rather than in each mapper: this is the single insert
|
|
290
|
+
// choke point for every typed table, so one derivation covers every source (0739 R1).
|
|
291
|
+
const toolIdentity = table === 'history_tool_call' ? resolveToolIdentity(payload) : undefined;
|
|
210
292
|
const values = typedColumns.map((col) => {
|
|
211
293
|
if (col === 'record_hash')
|
|
212
294
|
return recordHash;
|
|
213
295
|
if (col === 'imported_at')
|
|
214
296
|
return ts;
|
|
297
|
+
if (toolIdentity !== undefined) {
|
|
298
|
+
if (col === 'effective_tool_name')
|
|
299
|
+
return toolIdentity.effective;
|
|
300
|
+
if (col === 'tool_name_alias')
|
|
301
|
+
return toolIdentity.alias;
|
|
302
|
+
}
|
|
215
303
|
const v = payload[col];
|
|
216
304
|
return v === undefined ? null : v;
|
|
217
305
|
});
|
|
@@ -554,7 +642,14 @@ export function openCodeBulkWriteOperations(entries, checkpointFiles, now) {
|
|
|
554
642
|
FROM input
|
|
555
643
|
WHERE true
|
|
556
644
|
ON CONFLICT(record_hash) DO NOTHING`,
|
|
557
|
-
|
|
645
|
+
// Tool identity is folded into the payload rather than the SQL: this path
|
|
646
|
+
// json_extracts every column from payload_json, so a column the record lacks
|
|
647
|
+
// arrives NULL and violates the NOT NULL constraint (0739 R1).
|
|
648
|
+
params: chunk.flatMap((entry) => [
|
|
649
|
+
entry.recordHash,
|
|
650
|
+
JSON.stringify(table === 'history_tool_call' ? withToolIdentity(entry.record) : entry.record),
|
|
651
|
+
importedAt,
|
|
652
|
+
]),
|
|
558
653
|
});
|
|
559
654
|
}
|
|
560
655
|
}
|
package/dist/schema-sql.d.ts
CHANGED
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
* Guaranteed to match the package version; a bump-or-fail test enforces that
|
|
5
5
|
* any modification to HISTORY_IMPORT_SCHEMA_SQL must be accompanied by a version bump.
|
|
6
6
|
*/
|
|
7
|
-
export declare const HISTORY_IMPORT_SCHEMA_VERSION = "0.4.
|
|
7
|
+
export declare const HISTORY_IMPORT_SCHEMA_VERSION = "0.4.56";
|
|
8
8
|
/**
|
|
9
9
|
* DDL string that creates the history import checkpoint, ledger, and typed contract tables.
|
|
10
10
|
*
|
package/dist/schema-sql.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"schema-sql.d.ts","sourceRoot":"","sources":["../src/schema-sql.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AACH,eAAO,MAAM,6BAA6B,WAAW,CAAC;AAEtD;;;;;GAKG;AACH,eAAO,MAAM,yBAAyB,
|
|
1
|
+
{"version":3,"file":"schema-sql.d.ts","sourceRoot":"","sources":["../src/schema-sql.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AACH,eAAO,MAAM,6BAA6B,WAAW,CAAC;AAEtD;;;;;GAKG;AACH,eAAO,MAAM,yBAAyB,QAqH9B,CAAC"}
|
package/dist/schema-sql.js
CHANGED
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
* Guaranteed to match the package version; a bump-or-fail test enforces that
|
|
5
5
|
* any modification to HISTORY_IMPORT_SCHEMA_SQL must be accompanied by a version bump.
|
|
6
6
|
*/
|
|
7
|
-
export const HISTORY_IMPORT_SCHEMA_VERSION = '0.4.
|
|
7
|
+
export const HISTORY_IMPORT_SCHEMA_VERSION = '0.4.56';
|
|
8
8
|
/**
|
|
9
9
|
* DDL string that creates the history import checkpoint, ledger, and typed contract tables.
|
|
10
10
|
*
|
|
@@ -80,7 +80,13 @@ CREATE TABLE IF NOT EXISTS history_tool_call (
|
|
|
80
80
|
duration_ms INTEGER,
|
|
81
81
|
result_bytes INTEGER,
|
|
82
82
|
error_text TEXT,
|
|
83
|
-
imported_at TEXT NOT NULL
|
|
83
|
+
imported_at TEXT NOT NULL,
|
|
84
|
+
-- Tool identity (0739). Appended last so a database that gained these columns
|
|
85
|
+
-- via Spur's guarded ALTER converges on the same PRAGMA column order as a
|
|
86
|
+
-- database created fresh from this DDL. 'unknown' is the unresolved sentinel
|
|
87
|
+
-- every consumer already falls back from.
|
|
88
|
+
effective_tool_name TEXT NOT NULL DEFAULT 'unknown',
|
|
89
|
+
tool_name_alias TEXT NOT NULL DEFAULT 'unknown'
|
|
84
90
|
);
|
|
85
91
|
|
|
86
92
|
CREATE INDEX IF NOT EXISTS idx_history_message_session
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@gobing-ai/ts-llm-jsonl-importer",
|
|
3
|
-
"version": "0.4.
|
|
3
|
+
"version": "0.4.56",
|
|
4
4
|
"description": "@gobing-ai/ts-llm-jsonl-importer — Generic JSONL importer for LLM agent history files.",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"typescript",
|
|
@@ -47,9 +47,9 @@
|
|
|
47
47
|
"release": "echo 'Manual publish is disabled. Releases go through GitHub Actions via Trusted Publishing — push a tag: git tag @gobing-ai/ts-llm-jsonl-importer-v<version> && git push --tags' && exit 1"
|
|
48
48
|
},
|
|
49
49
|
"dependencies": {
|
|
50
|
-
"@gobing-ai/ts-db": "^0.4.
|
|
51
|
-
"@gobing-ai/ts-runtime": "^0.4.
|
|
52
|
-
"@gobing-ai/ts-utils": "^0.4.
|
|
50
|
+
"@gobing-ai/ts-db": "^0.4.56",
|
|
51
|
+
"@gobing-ai/ts-runtime": "^0.4.56",
|
|
52
|
+
"@gobing-ai/ts-utils": "^0.4.56",
|
|
53
53
|
"zod": "^4.1.0"
|
|
54
54
|
},
|
|
55
55
|
"devDependencies": {
|
|
@@ -72,6 +72,8 @@ const TYPED_TABLE_COLUMNS: Readonly<Record<string, readonly string[]>> = {
|
|
|
72
72
|
'result_bytes',
|
|
73
73
|
'error_text',
|
|
74
74
|
'imported_at',
|
|
75
|
+
'effective_tool_name',
|
|
76
|
+
'tool_name_alias',
|
|
75
77
|
],
|
|
76
78
|
history_skill_call: [
|
|
77
79
|
'record_hash',
|
|
@@ -252,6 +254,84 @@ async function ledgerExists(db: ImportOptions['db'], recordHash: string): Promis
|
|
|
252
254
|
* Typed tables validate unknown keys here; the caller must have ensured the
|
|
253
255
|
* target table exists (the import loop ensures accepted targets before batching).
|
|
254
256
|
*/
|
|
257
|
+
/** Wrapper-style `call_id` prefixes that name the tool the wrapper actually ran. */
|
|
258
|
+
const CALL_ID_TOOL_PREFIXES: readonly (readonly [string, string])[] = [
|
|
259
|
+
['call_bash_', 'bash'],
|
|
260
|
+
['call_read_', 'read'],
|
|
261
|
+
['call_edit_', 'edit'],
|
|
262
|
+
['call_write_', 'write'],
|
|
263
|
+
['call_grep_', 'grep'],
|
|
264
|
+
['call_find_', 'find'],
|
|
265
|
+
['call_ls_', 'ls'],
|
|
266
|
+
];
|
|
267
|
+
|
|
268
|
+
/** Argument keys, in precedence order, that carry the tool name when `tool_name` is empty. */
|
|
269
|
+
const ARGS_TOOL_NAME_KEYS = ['tool', 'tool_name', 'toolName', 'name', 'command'] as const;
|
|
270
|
+
|
|
271
|
+
/**
|
|
272
|
+
* Tool identity for one `history_tool_call` row (0739).
|
|
273
|
+
*
|
|
274
|
+
* `effective` is extraction — recover the name when the agent recorded it inside a wrapper's
|
|
275
|
+
* arguments or only in the call id. `alias` is canonicalization — grouping the same logical tool
|
|
276
|
+
* across agents — and starts at identity, because the alias vocabulary is a curated decision
|
|
277
|
+
* that belongs to the consumer, not to the importer.
|
|
278
|
+
*
|
|
279
|
+
* Mirrors Spur migration 0034's backfill CASE so a row imported after the migration and a row
|
|
280
|
+
* backfilled by it resolve identically.
|
|
281
|
+
*/
|
|
282
|
+
export function resolveToolIdentity(payload: JsonObject): { effective: string; alias: string } {
|
|
283
|
+
const supplied = trimmedString(payload.effective_tool_name);
|
|
284
|
+
const effective = supplied !== undefined ? supplied : deriveEffectiveToolName(payload);
|
|
285
|
+
return { effective, alias: trimmedString(payload.tool_name_alias) ?? effective };
|
|
286
|
+
}
|
|
287
|
+
|
|
288
|
+
/** Returns the record with tool identity resolved, for insert paths that read columns off the payload. */
|
|
289
|
+
function withToolIdentity(record: JsonObject): JsonObject {
|
|
290
|
+
const { effective, alias } = resolveToolIdentity(record);
|
|
291
|
+
return { ...record, effective_tool_name: effective, tool_name_alias: alias };
|
|
292
|
+
}
|
|
293
|
+
|
|
294
|
+
function trimmedString(value: unknown): string | undefined {
|
|
295
|
+
if (typeof value !== 'string') return undefined;
|
|
296
|
+
const trimmed = value.trim();
|
|
297
|
+
return trimmed === '' ? undefined : trimmed;
|
|
298
|
+
}
|
|
299
|
+
|
|
300
|
+
function deriveEffectiveToolName(payload: JsonObject): string {
|
|
301
|
+
const recorded = trimmedString(payload.tool_name);
|
|
302
|
+
if (recorded !== undefined && recorded !== 'unknown') return recorded;
|
|
303
|
+
|
|
304
|
+
const fromArgs = toolNameFromArgs(payload.args_raw);
|
|
305
|
+
if (fromArgs !== undefined) return fromArgs;
|
|
306
|
+
|
|
307
|
+
const callId = typeof payload.call_id === 'string' ? payload.call_id : '';
|
|
308
|
+
for (const [prefix, tool] of CALL_ID_TOOL_PREFIXES) {
|
|
309
|
+
if (callId.startsWith(prefix)) return tool;
|
|
310
|
+
}
|
|
311
|
+
return 'unknown';
|
|
312
|
+
}
|
|
313
|
+
|
|
314
|
+
function toolNameFromArgs(argsRaw: unknown): string | undefined {
|
|
315
|
+
if (typeof argsRaw !== 'string') return undefined;
|
|
316
|
+
let parsed: unknown;
|
|
317
|
+
try {
|
|
318
|
+
parsed = JSON.parse(argsRaw);
|
|
319
|
+
} catch {
|
|
320
|
+
// Non-JSON arguments are the `json_valid(args_raw)` false branch in SQL: fall through.
|
|
321
|
+
return undefined;
|
|
322
|
+
}
|
|
323
|
+
if (parsed === null || typeof parsed !== 'object' || Array.isArray(parsed)) return undefined;
|
|
324
|
+
for (const key of ARGS_TOOL_NAME_KEYS) {
|
|
325
|
+
const candidate = trimmedString((parsed as Record<string, unknown>)[key]);
|
|
326
|
+
if (candidate !== undefined) return candidate;
|
|
327
|
+
}
|
|
328
|
+
return undefined;
|
|
329
|
+
}
|
|
330
|
+
|
|
331
|
+
/**
|
|
332
|
+
* Build the insert op for one imported record, routing to the typed or JSON
|
|
333
|
+
* column path per target table.
|
|
334
|
+
*/
|
|
255
335
|
export function recordInsertOp(
|
|
256
336
|
targetTable: string,
|
|
257
337
|
recordHash: string,
|
|
@@ -275,9 +355,16 @@ export function recordInsertOp(
|
|
|
275
355
|
}
|
|
276
356
|
// record_hash and imported_at come from the function parameters, not the payload.
|
|
277
357
|
const ts = timestamp(now);
|
|
358
|
+
// Tool identity is derived here rather than in each mapper: this is the single insert
|
|
359
|
+
// choke point for every typed table, so one derivation covers every source (0739 R1).
|
|
360
|
+
const toolIdentity = table === 'history_tool_call' ? resolveToolIdentity(payload) : undefined;
|
|
278
361
|
const values = typedColumns.map((col) => {
|
|
279
362
|
if (col === 'record_hash') return recordHash;
|
|
280
363
|
if (col === 'imported_at') return ts;
|
|
364
|
+
if (toolIdentity !== undefined) {
|
|
365
|
+
if (col === 'effective_tool_name') return toolIdentity.effective;
|
|
366
|
+
if (col === 'tool_name_alias') return toolIdentity.alias;
|
|
367
|
+
}
|
|
281
368
|
const v = payload[col];
|
|
282
369
|
return v === undefined ? null : v;
|
|
283
370
|
});
|
|
@@ -826,7 +913,14 @@ export function openCodeBulkWriteOperations(
|
|
|
826
913
|
FROM input
|
|
827
914
|
WHERE true
|
|
828
915
|
ON CONFLICT(record_hash) DO NOTHING`,
|
|
829
|
-
|
|
916
|
+
// Tool identity is folded into the payload rather than the SQL: this path
|
|
917
|
+
// json_extracts every column from payload_json, so a column the record lacks
|
|
918
|
+
// arrives NULL and violates the NOT NULL constraint (0739 R1).
|
|
919
|
+
params: chunk.flatMap((entry) => [
|
|
920
|
+
entry.recordHash,
|
|
921
|
+
JSON.stringify(table === 'history_tool_call' ? withToolIdentity(entry.record) : entry.record),
|
|
922
|
+
importedAt,
|
|
923
|
+
]),
|
|
830
924
|
});
|
|
831
925
|
}
|
|
832
926
|
}
|
package/src/schema-sql.ts
CHANGED
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
* Guaranteed to match the package version; a bump-or-fail test enforces that
|
|
5
5
|
* any modification to HISTORY_IMPORT_SCHEMA_SQL must be accompanied by a version bump.
|
|
6
6
|
*/
|
|
7
|
-
export const HISTORY_IMPORT_SCHEMA_VERSION = '0.4.
|
|
7
|
+
export const HISTORY_IMPORT_SCHEMA_VERSION = '0.4.56';
|
|
8
8
|
|
|
9
9
|
/**
|
|
10
10
|
* DDL string that creates the history import checkpoint, ledger, and typed contract tables.
|
|
@@ -81,7 +81,13 @@ CREATE TABLE IF NOT EXISTS history_tool_call (
|
|
|
81
81
|
duration_ms INTEGER,
|
|
82
82
|
result_bytes INTEGER,
|
|
83
83
|
error_text TEXT,
|
|
84
|
-
imported_at TEXT NOT NULL
|
|
84
|
+
imported_at TEXT NOT NULL,
|
|
85
|
+
-- Tool identity (0739). Appended last so a database that gained these columns
|
|
86
|
+
-- via Spur's guarded ALTER converges on the same PRAGMA column order as a
|
|
87
|
+
-- database created fresh from this DDL. 'unknown' is the unresolved sentinel
|
|
88
|
+
-- every consumer already falls back from.
|
|
89
|
+
effective_tool_name TEXT NOT NULL DEFAULT 'unknown',
|
|
90
|
+
tool_name_alias TEXT NOT NULL DEFAULT 'unknown'
|
|
85
91
|
);
|
|
86
92
|
|
|
87
93
|
CREATE INDEX IF NOT EXISTS idx_history_message_session
|