@gobing-ai/ts-llm-jsonl-importer 0.4.54 → 0.4.56
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/assistant-duration.d.ts +41 -0
- package/dist/assistant-duration.d.ts.map +1 -0
- package/dist/assistant-duration.js +51 -0
- package/dist/index.d.ts +4 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +4 -2
- package/dist/jsonl-importer-dao.d.ts +36 -4
- package/dist/jsonl-importer-dao.d.ts.map +1 -1
- package/dist/jsonl-importer-dao.js +128 -1
- package/dist/schema-sql.d.ts +7 -0
- package/dist/schema-sql.d.ts.map +1 -1
- package/dist/schema-sql.js +15 -1
- package/dist/tables.d.ts +14 -0
- package/dist/tables.d.ts.map +1 -0
- package/dist/tables.js +19 -0
- package/dist/types.d.ts +1 -0
- package/dist/types.d.ts.map +1 -1
- package/package.json +4 -4
- package/src/assistant-duration.ts +64 -0
- package/src/index.ts +9 -2
- package/src/jsonl-importer-dao.ts +147 -1
- package/src/schema-sql.ts +16 -1
- package/src/tables.ts +21 -0
- package/src/types.ts +1 -0
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
import type { DbAdapter } from '@gobing-ai/ts-db';
|
|
2
|
+
/**
|
|
3
|
+
* ETL-derived assistant-step duration (repatriated upstream from Spur task 0747 / ADR-105).
|
|
4
|
+
*
|
|
5
|
+
* Four of six sources — `claude`, `pi`, `codex`, `agy` — write no per-step
|
|
6
|
+
* `duration_ms` at all, so bottleneck ranking and per-model latency were unusable for
|
|
7
|
+
* the two busiest of them and ~73% of the measured span could not be attributed to
|
|
8
|
+
* llm/tool/idle. The transcripts carry a per-record `ts`, so the delta from the
|
|
9
|
+
* preceding record in the same session is derivable here.
|
|
10
|
+
*
|
|
11
|
+
* **It is an approximation and is labelled as one.** The delta includes queue and
|
|
12
|
+
* network time, which is arguably what "assistant step duration" should mean for
|
|
13
|
+
* bottleneck ranking, but it is not the provider's own number. `duration_source` carries
|
|
14
|
+
* that distinction to every consumer: `NULL` is provider-reported, `'derived'` is this
|
|
15
|
+
* function's work. Nothing may present the two as the same measurement.
|
|
16
|
+
*/
|
|
17
|
+
/** `history_message.duration_source` values. `NULL` means provider-reported. */
|
|
18
|
+
export declare const DURATION_SOURCE_DERIVED = "derived";
|
|
19
|
+
/**
|
|
20
|
+
* Deltas above this are session gaps, not work: the operator walked away and came back.
|
|
21
|
+
* Attributing an overnight gap to an assistant step would corrupt the very ranking this
|
|
22
|
+
* derivation exists to make usable, so those rows stay unmeasured.
|
|
23
|
+
*/
|
|
24
|
+
export declare const DERIVED_DURATION_CEILING_MS: number;
|
|
25
|
+
/** What one derivation pass changed, and what it deliberately left alone. */
|
|
26
|
+
export interface DeriveAssistantDurationsResult {
|
|
27
|
+
/** Rows that gained a derived `duration_ms` on this pass. */
|
|
28
|
+
derived: number;
|
|
29
|
+
/** Candidate rows left unmeasured because the delta exceeded the ceiling. */
|
|
30
|
+
skippedOverCeiling: number;
|
|
31
|
+
}
|
|
32
|
+
/**
|
|
33
|
+
* Fill `duration_ms` for assistant rows that have none, from the timestamp delta to the
|
|
34
|
+
* preceding record in the same session.
|
|
35
|
+
*
|
|
36
|
+
* Idempotent and additive: a row whose `duration_ms` is already set is never touched, so
|
|
37
|
+
* a provider measurement always wins over a derived one, and re-running after a later
|
|
38
|
+
* import only fills rows the earlier pass could not reach.
|
|
39
|
+
*/
|
|
40
|
+
export declare function deriveAssistantDurations(db: DbAdapter): Promise<DeriveAssistantDurationsResult>;
|
|
41
|
+
//# sourceMappingURL=assistant-duration.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"assistant-duration.d.ts","sourceRoot":"","sources":["../src/assistant-duration.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAGlD;;;;;;;;;;;;;;GAcG;AAEH,gFAAgF;AAChF,eAAO,MAAM,uBAAuB,YAAY,CAAC;AAEjD;;;;GAIG;AACH,eAAO,MAAM,2BAA2B,QAAiB,CAAC;AAE1D,6EAA6E;AAC7E,MAAM,WAAW,8BAA8B;IAC3C,6DAA6D;IAC7D,OAAO,EAAE,MAAM,CAAC;IAChB,6EAA6E;IAC7E,kBAAkB,EAAE,MAAM,CAAC;CAC9B;AAED;;;;;;;GAOG;AACH,wBAAsB,wBAAwB,CAAC,EAAE,EAAE,SAAS,GAAG,OAAO,CAAC,8BAA8B,CAAC,CAkBrG"}
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
import { findAssistantDurationCandidates, updateAssistantDuration } from './jsonl-importer-dao.js';
|
|
2
|
+
/**
|
|
3
|
+
* ETL-derived assistant-step duration (repatriated upstream from Spur task 0747 / ADR-105).
|
|
4
|
+
*
|
|
5
|
+
* Four of six sources — `claude`, `pi`, `codex`, `agy` — write no per-step
|
|
6
|
+
* `duration_ms` at all, so bottleneck ranking and per-model latency were unusable for
|
|
7
|
+
* the two busiest of them and ~73% of the measured span could not be attributed to
|
|
8
|
+
* llm/tool/idle. The transcripts carry a per-record `ts`, so the delta from the
|
|
9
|
+
* preceding record in the same session is derivable here.
|
|
10
|
+
*
|
|
11
|
+
* **It is an approximation and is labelled as one.** The delta includes queue and
|
|
12
|
+
* network time, which is arguably what "assistant step duration" should mean for
|
|
13
|
+
* bottleneck ranking, but it is not the provider's own number. `duration_source` carries
|
|
14
|
+
* that distinction to every consumer: `NULL` is provider-reported, `'derived'` is this
|
|
15
|
+
* function's work. Nothing may present the two as the same measurement.
|
|
16
|
+
*/
|
|
17
|
+
/** `history_message.duration_source` values. `NULL` means provider-reported. */
|
|
18
|
+
export const DURATION_SOURCE_DERIVED = 'derived';
|
|
19
|
+
/**
|
|
20
|
+
* Deltas above this are session gaps, not work: the operator walked away and came back.
|
|
21
|
+
* Attributing an overnight gap to an assistant step would corrupt the very ranking this
|
|
22
|
+
* derivation exists to make usable, so those rows stay unmeasured.
|
|
23
|
+
*/
|
|
24
|
+
export const DERIVED_DURATION_CEILING_MS = 30 * 60 * 1000;
|
|
25
|
+
/**
|
|
26
|
+
* Fill `duration_ms` for assistant rows that have none, from the timestamp delta to the
|
|
27
|
+
* preceding record in the same session.
|
|
28
|
+
*
|
|
29
|
+
* Idempotent and additive: a row whose `duration_ms` is already set is never touched, so
|
|
30
|
+
* a provider measurement always wins over a derived one, and re-running after a later
|
|
31
|
+
* import only fills rows the earlier pass could not reach.
|
|
32
|
+
*/
|
|
33
|
+
export async function deriveAssistantDurations(db) {
|
|
34
|
+
const rows = await findAssistantDurationCandidates(db);
|
|
35
|
+
let derived = 0;
|
|
36
|
+
let skippedOverCeiling = 0;
|
|
37
|
+
for (const row of rows) {
|
|
38
|
+
// A non-positive delta means the records share a timestamp or arrived out of
|
|
39
|
+
// order; there is no duration to claim, so leave the row unmeasured rather than
|
|
40
|
+
// invent a zero (0680 R6: absent is not zero).
|
|
41
|
+
if (row.deltaMs === null || row.deltaMs <= 0)
|
|
42
|
+
continue;
|
|
43
|
+
if (row.deltaMs > DERIVED_DURATION_CEILING_MS) {
|
|
44
|
+
skippedOverCeiling += 1;
|
|
45
|
+
continue;
|
|
46
|
+
}
|
|
47
|
+
await updateAssistantDuration(db, row.recordHash, row.deltaMs, DURATION_SOURCE_DERIVED);
|
|
48
|
+
derived += 1;
|
|
49
|
+
}
|
|
50
|
+
return { derived, skippedOverCeiling };
|
|
51
|
+
}
|
package/dist/index.d.ts
CHANGED
|
@@ -1,11 +1,13 @@
|
|
|
1
|
+
export { DERIVED_DURATION_CEILING_MS, type DeriveAssistantDurationsResult, DURATION_SOURCE_DERIVED, deriveAssistantDurations, } from './assistant-duration';
|
|
1
2
|
export { HistoryImportError } from './errors';
|
|
2
3
|
export { sha256, stableJson } from './hash';
|
|
3
4
|
export { runJsonlImport } from './importer';
|
|
4
|
-
export { applyHistoryImportSchema, normalizeSourceFilePaths } from './jsonl-importer-dao';
|
|
5
|
+
export { applyHistoryImportSchema, normalizeSourceFilePaths, TYPED_HISTORY_TABLES } from './jsonl-importer-dao';
|
|
5
6
|
export { canonicalizeSkillName, extractSkillCalls, type SkillCallIdentity, sessionIdFromSourcePath, skillCallEntry, } from './mappers';
|
|
6
7
|
export { type OpenCodeImportOptions, runOpenCodeImport } from './opencode-importer';
|
|
7
8
|
export { DEFAULT_REDACTION_RULES, redactRecord, redactValue } from './redaction';
|
|
8
|
-
export { HISTORY_IMPORT_SCHEMA_SQL } from './schema-sql';
|
|
9
|
+
export { HISTORY_IMPORT_SCHEMA_SQL, HISTORY_IMPORT_SCHEMA_VERSION } from './schema-sql';
|
|
9
10
|
export { getSourceDefinition, resolveSourceDefinition, SOURCE_DEFINITIONS, VALID_TABLE_NAME, validateSourceDefinition, } from './sources';
|
|
11
|
+
export { BOOKKEEPING_HISTORY_TABLES, IMPORTER_OWNED_TABLES } from './tables';
|
|
10
12
|
export type { FieldTransform, ImportIssue, ImportMode, ImportOptions, ImportResult, JsonObject, LlmJsonlSource, ReconcileSummary, RedactionRule, SkillCall, SkillCallSplitRecord, SourceDefinition, SplitConfig, SplitEntry, TransformContext, } from './types';
|
|
11
13
|
//# sourceMappingURL=index.d.ts.map
|
package/dist/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,kBAAkB,EAAE,MAAM,UAAU,CAAC;AAC9C,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AAC5C,OAAO,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AAC5C,OAAO,EAAE,wBAAwB,EAAE,wBAAwB,EAAE,MAAM,sBAAsB,CAAC;
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EACH,2BAA2B,EAC3B,KAAK,8BAA8B,EACnC,uBAAuB,EACvB,wBAAwB,GAC3B,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EAAE,kBAAkB,EAAE,MAAM,UAAU,CAAC;AAC9C,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AAC5C,OAAO,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AAC5C,OAAO,EAAE,wBAAwB,EAAE,wBAAwB,EAAE,oBAAoB,EAAE,MAAM,sBAAsB,CAAC;AAChH,OAAO,EACH,qBAAqB,EACrB,iBAAiB,EACjB,KAAK,iBAAiB,EACtB,uBAAuB,EACvB,cAAc,GACjB,MAAM,WAAW,CAAC;AACnB,OAAO,EAAE,KAAK,qBAAqB,EAAE,iBAAiB,EAAE,MAAM,qBAAqB,CAAC;AACpF,OAAO,EAAE,uBAAuB,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AACjF,OAAO,EAAE,yBAAyB,EAAE,6BAA6B,EAAE,MAAM,cAAc,CAAC;AACxF,OAAO,EACH,mBAAmB,EACnB,uBAAuB,EACvB,kBAAkB,EAClB,gBAAgB,EAChB,wBAAwB,GAC3B,MAAM,WAAW,CAAC;AACnB,OAAO,EAAE,0BAA0B,EAAE,qBAAqB,EAAE,MAAM,UAAU,CAAC;AAC7E,YAAY,EACR,cAAc,EACd,WAAW,EACX,UAAU,EACV,aAAa,EACb,YAAY,EACZ,UAAU,EACV,cAAc,EACd,gBAAgB,EAChB,aAAa,EACb,SAAS,EACT,oBAAoB,EACpB,gBAAgB,EAChB,WAAW,EACX,UAAU,EACV,gBAAgB,GACnB,MAAM,SAAS,CAAC"}
|
package/dist/index.js
CHANGED
|
@@ -1,9 +1,11 @@
|
|
|
1
|
+
export { DERIVED_DURATION_CEILING_MS, DURATION_SOURCE_DERIVED, deriveAssistantDurations, } from './assistant-duration.js';
|
|
1
2
|
export { HistoryImportError } from './errors.js';
|
|
2
3
|
export { sha256, stableJson } from './hash.js';
|
|
3
4
|
export { runJsonlImport } from './importer.js';
|
|
4
|
-
export { applyHistoryImportSchema, normalizeSourceFilePaths } from './jsonl-importer-dao.js';
|
|
5
|
+
export { applyHistoryImportSchema, normalizeSourceFilePaths, TYPED_HISTORY_TABLES } from './jsonl-importer-dao.js';
|
|
5
6
|
export { canonicalizeSkillName, extractSkillCalls, sessionIdFromSourcePath, skillCallEntry, } from './mappers.js';
|
|
6
7
|
export { runOpenCodeImport } from './opencode-importer.js';
|
|
7
8
|
export { DEFAULT_REDACTION_RULES, redactRecord, redactValue } from './redaction.js';
|
|
8
|
-
export { HISTORY_IMPORT_SCHEMA_SQL } from './schema-sql.js';
|
|
9
|
+
export { HISTORY_IMPORT_SCHEMA_SQL, HISTORY_IMPORT_SCHEMA_VERSION } from './schema-sql.js';
|
|
9
10
|
export { getSourceDefinition, resolveSourceDefinition, SOURCE_DEFINITIONS, VALID_TABLE_NAME, validateSourceDefinition, } from './sources.js';
|
|
11
|
+
export { BOOKKEEPING_HISTORY_TABLES, IMPORTER_OWNED_TABLES } from './tables.js';
|
|
@@ -11,6 +11,12 @@ export interface SourceCheckpoint {
|
|
|
11
11
|
readonly size: number | null;
|
|
12
12
|
readonly mtimeMs: number | null;
|
|
13
13
|
}
|
|
14
|
+
/**
|
|
15
|
+
* Typed contract tables created and populated by the importer.
|
|
16
|
+
*
|
|
17
|
+
* Promoted to public surface per task 0749 (ADR-105).
|
|
18
|
+
*/
|
|
19
|
+
export declare const TYPED_HISTORY_TABLES: readonly ["history_message", "history_tool_call", "history_skill_call"];
|
|
14
20
|
declare function targetTableFor(table: string): string;
|
|
15
21
|
declare function timestamp(now: ImportOptions['now']): string;
|
|
16
22
|
declare function ETL_TABLE_DDL(table: string): string;
|
|
@@ -51,10 +57,23 @@ declare function resetCheckpoints(db: ImportOptions['db'], source: string, files
|
|
|
51
57
|
declare function writeCheckpoint(db: ImportOptions['db'], source: string, sourceFile: string, line: number, now: ImportOptions['now']): Promise<void>;
|
|
52
58
|
declare function ledgerExists(db: ImportOptions['db'], recordHash: string): Promise<boolean>;
|
|
53
59
|
/**
|
|
54
|
-
*
|
|
55
|
-
*
|
|
56
|
-
*
|
|
57
|
-
*
|
|
60
|
+
* Tool identity for one `history_tool_call` row (0739).
|
|
61
|
+
*
|
|
62
|
+
* `effective` is extraction — recover the name when the agent recorded it inside a wrapper's
|
|
63
|
+
* arguments or only in the call id. `alias` is canonicalization — grouping the same logical tool
|
|
64
|
+
* across agents — and starts at identity, because the alias vocabulary is a curated decision
|
|
65
|
+
* that belongs to the consumer, not to the importer.
|
|
66
|
+
*
|
|
67
|
+
* Mirrors Spur migration 0034's backfill CASE so a row imported after the migration and a row
|
|
68
|
+
* backfilled by it resolve identically.
|
|
69
|
+
*/
|
|
70
|
+
export declare function resolveToolIdentity(payload: JsonObject): {
|
|
71
|
+
effective: string;
|
|
72
|
+
alias: string;
|
|
73
|
+
};
|
|
74
|
+
/**
|
|
75
|
+
* Build the insert op for one imported record, routing to the typed or JSON
|
|
76
|
+
* column path per target table.
|
|
58
77
|
*/
|
|
59
78
|
export declare function recordInsertOp(targetTable: string, recordHash: string, sourceFile: string, sourceLine: number, splitIndex: number, payload: JsonObject, now: ImportOptions['now']): DbBatchOp;
|
|
60
79
|
/** INSERT op for one ledger row (task 0060 F9). */
|
|
@@ -91,6 +110,19 @@ export declare function codexUsageAttributionUpdateOp(recordHash: string, tokens
|
|
|
91
110
|
* most 200 per `IN (...)` (task 0060 F9 — replaces the per-record SELECT loop).
|
|
92
111
|
*/
|
|
93
112
|
export declare function ledgerExistingHashes(db: ImportOptions['db'], hashes: readonly string[]): Promise<Set<string>>;
|
|
113
|
+
/** Candidate assistant-message row for duration derivation. */
|
|
114
|
+
export interface AssistantDurationCandidate {
|
|
115
|
+
readonly recordHash: string;
|
|
116
|
+
readonly deltaMs: number | null;
|
|
117
|
+
}
|
|
118
|
+
/**
|
|
119
|
+
* Fetch candidate assistant rows lacking duration_ms along with timestamp delta to preceding record.
|
|
120
|
+
*/
|
|
121
|
+
export declare function findAssistantDurationCandidates(db: DbAdapter): Promise<AssistantDurationCandidate[]>;
|
|
122
|
+
/**
|
|
123
|
+
* Update duration_ms and duration_source for a previously unmeasured assistant row.
|
|
124
|
+
*/
|
|
125
|
+
export declare function updateAssistantDuration(db: DbAdapter, recordHash: string, durationMs: number, durationSource: string): Promise<void>;
|
|
94
126
|
declare function insertRecord(db: ImportOptions['db'], targetTable: string, recordHash: string, sourceFile: string, sourceLine: number, splitIndex: number, payload: JsonObject, now: ImportOptions['now']): Promise<void>;
|
|
95
127
|
declare function insertLedger(db: ImportOptions['db'], recordHash: string, source: string, sourceFile: string, sourceLine: number, splitIndex: number, targetTable: string, now: ImportOptions['now']): Promise<void>;
|
|
96
128
|
/**
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"jsonl-importer-dao.d.ts","sourceRoot":"","sources":["../src/jsonl-importer-dao.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAI7D,OAAO,KAAK,EAAE,aAAa,EAAE,UAAU,EAAE,gBAAgB,EAAE,gBAAgB,EAAE,MAAM,SAAS,CAAC;AAE7F,UAAU,aAAa;IACnB,QAAQ,CAAC,kBAAkB,EAAE,MAAM,CAAC;IACpC,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACrC,QAAQ,CAAC,eAAe,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CAC5C;AAED,gEAAgE;AAChE,MAAM,WAAW,gBAAgB;IAC7B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,QAAQ,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;CACnC;
|
|
1
|
+
{"version":3,"file":"jsonl-importer-dao.d.ts","sourceRoot":"","sources":["../src/jsonl-importer-dao.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAI7D,OAAO,KAAK,EAAE,aAAa,EAAE,UAAU,EAAE,gBAAgB,EAAE,gBAAgB,EAAE,MAAM,SAAS,CAAC;AAE7F,UAAU,aAAa;IACnB,QAAQ,CAAC,kBAAkB,EAAE,MAAM,CAAC;IACpC,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACrC,QAAQ,CAAC,eAAe,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CAC5C;AAED,gEAAgE;AAChE,MAAM,WAAW,gBAAgB;IAC7B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,QAAQ,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;CACnC;AAED;;;;GAIG;AACH,eAAO,MAAM,oBAAoB,yEAA0E,CAAC;AA8E5G,iBAAS,cAAc,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAK7C;AAED,iBAAS,SAAS,CAAC,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,GAAG,MAAM,CAEpD;AAED,iBAAS,aAAa,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAS5C;AAED;;;;;;GAMG;AACH,wBAAsB,wBAAwB,CAAC,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,CAOrF;AAED,6FAA6F;AAC7F,iBAAe,iBAAiB,CAAC,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EAAE,WAAW,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAG5F;AAED;;;;;;;;;;GAUG;AACH,iBAAe,kBAAkB,CAAC,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EAAE,UAAU,EAAE,gBAAgB,GAAG,OAAO,CAAC,IAAI,CAAC,CAQtG;AAED,iBAAe,cAAc,CAAC,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAO1G;AAED;;;;;;;;GAQG;AACH,wBAAsB,qBAAqB,CACvC,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EACvB,MAAM,EAAE,MAAM,GACf,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,gBAAgB,CAAC,CAAC,CA0BxC;AAED,iBAAe,gBAAgB,CAAC,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,SAAS,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAIhH;AAED,iBAAe,eAAe,CAC1B,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EACvB,MAAM,EAAE,MAAM,EACd,UAAU,EAAE,MAAM,EAClB,IAAI,EAAE,MAAM,EACZ,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,GAC1B,OAAO,CAAC,IAAI,CAAC,CAYf;AAED,iBAAe,YAAY,CAAC,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EAAE,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAMzF;AAsBD;;;;;;;;;;GAUG;AACH,wBAAgB,mBAAmB,CAAC,OAAO,EAAE,UAAU,GAAG;IAAE,SAAS,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,CAI7F;AA6CD;;;GAGG;AACH,wBAAgB,cAAc,CAC1B,WAAW,EAAE,MAAM,EACnB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,OAAO,EAAE,UAAU,EACnB,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,GAC1B,SAAS,CA0CX;AAED,mDAAmD;AACnD,wBAAgB,cAAc,CAC1B,UAAU,EAAE,MAAM,EAClB,MAAM,EAAE,MAAM,EACd,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,WAAW,EAAE,MAAM,EACnB,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,GAC1B,SAAS,CAMX;AAED,0FAA0F;AAC1F,wBAAgB,kBAAkB,CAC9B,MAAM,EAAE,MAAM,EACd,UAAU,EAAE,MAAM,EAClB,IAAI,EAAE,MAAM,EACZ,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,EACzB,QAAQ,CAAC,EAAE;IAAE,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;IAAC,QAAQ,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI,CAAA;CAAE,GAC7E,SAAS,CAWX;AAED;;;;;;;;GAQG;AACH,iBAAS,wBAAwB,CAC7B,UAAU,EAAE,MAAM,EAClB,SAAS,EAAE,MAAM,GAAG,IAAI,EACxB,WAAW,EAAE,MAAM,GAAG,IAAI,EAC1B,UAAU,EAAE,MAAM,GAAG,IAAI,GAC1B,SAAS,CAOX;AAED;;;GAGG;AACH,iBAAS,2BAA2B,CAAC,UAAU,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,GAAG,SAAS,CAOvF;AAED;2FAC2F;AAC3F,wBAAgB,6BAA6B,CACzC,UAAU,EAAE,MAAM,EAClB,MAAM,EAAE;IAAE,QAAQ,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IAAC,QAAQ,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IAAC,QAAQ,CAAC,SAAS,EAAE,MAAM,GAAG,IAAI,CAAA;CAAE,GAC7G,SAAS,CAOX;AAED;;;GAGG;AACH,wBAAsB,oBAAoB,CAAC,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,SAAS,MAAM,EAAE,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAYnH;AAED,+DAA+D;AAC/D,MAAM,WAAW,0BAA0B;IACvC,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;CACnC;AAED;;GAEG;AACH,wBAAsB,+BAA+B,CAAC,EAAE,EAAE,SAAS,GAAG,OAAO,CAAC,0BAA0B,EAAE,CAAC,CAiB1G;AAED;;GAEG;AACH,wBAAsB,uBAAuB,CACzC,EAAE,EAAE,SAAS,EACb,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,cAAc,EAAE,MAAM,GACvB,OAAO,CAAC,IAAI,CAAC,CAOf;AAED,iBAAe,YAAY,CACvB,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EACvB,WAAW,EAAE,MAAM,EACnB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,OAAO,EAAE,UAAU,EACnB,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,GAC1B,OAAO,CAAC,IAAI,CAAC,CAIf;AAED,iBAAe,YAAY,CACvB,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EACvB,UAAU,EAAE,MAAM,EAClB,MAAM,EAAE,MAAM,EACd,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,WAAW,EAAE,MAAM,EACnB,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,GAC1B,OAAO,CAAC,IAAI,CAAC,CAYf;AAED;;;;;;;;;;;;;;;;;;;GAmBG;AACH,wBAAsB,wBAAwB,CAC1C,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EACvB,eAAe,EAAE,CAAC,UAAU,EAAE,MAAM,KAAK,MAAM,GAAG,IAAI,GAAG,SAAS,GACnE,OAAO,CAAC,IAAI,CAAC,CAWf;AA4GD;;;;;;;;;;;GAWG;AACH,iBAAe,mBAAmB,CAC9B,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EACvB,MAAM,EAAE,MAAM,EACd,aAAa,EAAE,WAAW,CAAC,MAAM,CAAC,EAClC,eAAe,EAAE,SAAS,MAAM,EAAE,EAClC,MAAM,EAAE,OAAO,GAChB,OAAO,CAAC,gBAAgB,CAAC,CA6C3B;AAID,4DAA4D;AAC5D,MAAM,WAAW,kBAAkB;IAC/B,EAAE,EAAE,MAAM,CAAC;IACX,UAAU,EAAE,MAAM,CAAC;IACnB,YAAY,EAAE,MAAM,CAAC;IACrB,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,MAAM,CAAC;CACrB;AAED,iEAAiE;AACjE,MAAM,WAAW,eAAe;IAC5B,EAAE,EAAE,MAAM,CAAC;IACX,UAAU,EAAE,MAAM,CAAC;IACnB,YAAY,EAAE,MAAM,CAAC;IACrB,IAAI,EAAE,MAAM,CAAC;CAChB;AAED,iDAAiD;AACjD,MAAM,WAAW,qBAAqB;IAClC,WAAW,EAAE,MAAM,CAAC;IACpB,YAAY,EAAE,MAAM,CAAC;CACxB;AAED,gEAAgE;AAChE,MAAM,WAAW,mBAAmB;IAChC,WAAW,EAAE,iBAAiB,GAAG,mBAAmB,GAAG,oBAAoB,CAAC;IAC5E,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,EAAE,UAAU,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;CACtB;AAED,4EAA4E;AAC5E,wBAAsB,oBAAoB,CACtC,EAAE,EAAE,SAAS,EACb,QAAQ,EAAE,MAAM,EAChB,MAAM,EAAE,MAAM,EACd,KAAK,EAAE,MAAM,GACd,OAAO,CAAC,kBAAkB,EAAE,CAAC,CAa/B;AAED,qEAAqE;AACrE,wBAAsB,iBAAiB,CAAC,EAAE,EAAE,SAAS,EAAE,UAAU,EAAE,SAAS,MAAM,EAAE,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC,CAShH;AAED,0EAA0E;AAC1E,wBAAsB,2BAA2B,CAAC,EAAE,EAAE,SAAS,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,qBAAqB,EAAE,CAAC,CAAC,CAY9G;AAED,2EAA2E;AAC3E,wBAAgB,wBAAwB,CAAC,OAAO,EAAE,SAAS,qBAAqB,EAAE,GAAG,SAAS,EAAE,CAW/F;AAED,6DAA6D;AAC7D,wBAAgB,yBAAyB,CAAC,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,SAAS,CAKvF;AAED,iFAAiF;AACjF,wBAAgB,2BAA2B,CACvC,OAAO,EAAE,SAAS,mBAAmB,EAAE,EACvC,eAAe,EAAE,SAAS,MAAM,EAAE,EAClC,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,GAC1B,SAAS,EAAE,CAiEb;AAED,YAAY,EAAE,aAAa,EAAE,CAAC;AAC9B,OAAO,EACH,aAAa,EACb,iBAAiB,EACjB,kBAAkB,EAClB,YAAY,EACZ,YAAY,EACZ,YAAY,EACZ,cAAc,EACd,mBAAmB,EACnB,gBAAgB,EAChB,cAAc,EACd,SAAS,EACT,wBAAwB,EACxB,2BAA2B,EAC3B,eAAe,GAClB,CAAC"}
|
|
@@ -1,6 +1,12 @@
|
|
|
1
1
|
import { HistoryImportError } from './errors.js';
|
|
2
2
|
import { HISTORY_IMPORT_SCHEMA_SQL } from './schema-sql.js';
|
|
3
3
|
import { VALID_TABLE_NAME } from './sources.js';
|
|
4
|
+
/**
|
|
5
|
+
* Typed contract tables created and populated by the importer.
|
|
6
|
+
*
|
|
7
|
+
* Promoted to public surface per task 0749 (ADR-105).
|
|
8
|
+
*/
|
|
9
|
+
export const TYPED_HISTORY_TABLES = ['history_message', 'history_tool_call', 'history_skill_call'];
|
|
4
10
|
/** Column allowlist per typed contract table; order is the INSERT column order. */
|
|
5
11
|
const TYPED_TABLE_COLUMNS = {
|
|
6
12
|
history_message: [
|
|
@@ -49,6 +55,8 @@ const TYPED_TABLE_COLUMNS = {
|
|
|
49
55
|
'result_bytes',
|
|
50
56
|
'error_text',
|
|
51
57
|
'imported_at',
|
|
58
|
+
'effective_tool_name',
|
|
59
|
+
'tool_name_alias',
|
|
52
60
|
],
|
|
53
61
|
history_skill_call: [
|
|
54
62
|
'record_hash',
|
|
@@ -189,6 +197,83 @@ async function ledgerExists(db, recordHash) {
|
|
|
189
197
|
* Typed tables validate unknown keys here; the caller must have ensured the
|
|
190
198
|
* target table exists (the import loop ensures accepted targets before batching).
|
|
191
199
|
*/
|
|
200
|
+
/** Wrapper-style `call_id` prefixes that name the tool the wrapper actually ran. */
|
|
201
|
+
const CALL_ID_TOOL_PREFIXES = [
|
|
202
|
+
['call_bash_', 'bash'],
|
|
203
|
+
['call_read_', 'read'],
|
|
204
|
+
['call_edit_', 'edit'],
|
|
205
|
+
['call_write_', 'write'],
|
|
206
|
+
['call_grep_', 'grep'],
|
|
207
|
+
['call_find_', 'find'],
|
|
208
|
+
['call_ls_', 'ls'],
|
|
209
|
+
];
|
|
210
|
+
/** Argument keys, in precedence order, that carry the tool name when `tool_name` is empty. */
|
|
211
|
+
const ARGS_TOOL_NAME_KEYS = ['tool', 'tool_name', 'toolName', 'name', 'command'];
|
|
212
|
+
/**
|
|
213
|
+
* Tool identity for one `history_tool_call` row (0739).
|
|
214
|
+
*
|
|
215
|
+
* `effective` is extraction — recover the name when the agent recorded it inside a wrapper's
|
|
216
|
+
* arguments or only in the call id. `alias` is canonicalization — grouping the same logical tool
|
|
217
|
+
* across agents — and starts at identity, because the alias vocabulary is a curated decision
|
|
218
|
+
* that belongs to the consumer, not to the importer.
|
|
219
|
+
*
|
|
220
|
+
* Mirrors Spur migration 0034's backfill CASE so a row imported after the migration and a row
|
|
221
|
+
* backfilled by it resolve identically.
|
|
222
|
+
*/
|
|
223
|
+
export function resolveToolIdentity(payload) {
|
|
224
|
+
const supplied = trimmedString(payload.effective_tool_name);
|
|
225
|
+
const effective = supplied !== undefined ? supplied : deriveEffectiveToolName(payload);
|
|
226
|
+
return { effective, alias: trimmedString(payload.tool_name_alias) ?? effective };
|
|
227
|
+
}
|
|
228
|
+
/** Returns the record with tool identity resolved, for insert paths that read columns off the payload. */
|
|
229
|
+
function withToolIdentity(record) {
|
|
230
|
+
const { effective, alias } = resolveToolIdentity(record);
|
|
231
|
+
return { ...record, effective_tool_name: effective, tool_name_alias: alias };
|
|
232
|
+
}
|
|
233
|
+
function trimmedString(value) {
|
|
234
|
+
if (typeof value !== 'string')
|
|
235
|
+
return undefined;
|
|
236
|
+
const trimmed = value.trim();
|
|
237
|
+
return trimmed === '' ? undefined : trimmed;
|
|
238
|
+
}
|
|
239
|
+
function deriveEffectiveToolName(payload) {
|
|
240
|
+
const recorded = trimmedString(payload.tool_name);
|
|
241
|
+
if (recorded !== undefined && recorded !== 'unknown')
|
|
242
|
+
return recorded;
|
|
243
|
+
const fromArgs = toolNameFromArgs(payload.args_raw);
|
|
244
|
+
if (fromArgs !== undefined)
|
|
245
|
+
return fromArgs;
|
|
246
|
+
const callId = typeof payload.call_id === 'string' ? payload.call_id : '';
|
|
247
|
+
for (const [prefix, tool] of CALL_ID_TOOL_PREFIXES) {
|
|
248
|
+
if (callId.startsWith(prefix))
|
|
249
|
+
return tool;
|
|
250
|
+
}
|
|
251
|
+
return 'unknown';
|
|
252
|
+
}
|
|
253
|
+
function toolNameFromArgs(argsRaw) {
|
|
254
|
+
if (typeof argsRaw !== 'string')
|
|
255
|
+
return undefined;
|
|
256
|
+
let parsed;
|
|
257
|
+
try {
|
|
258
|
+
parsed = JSON.parse(argsRaw);
|
|
259
|
+
}
|
|
260
|
+
catch {
|
|
261
|
+
// Non-JSON arguments are the `json_valid(args_raw)` false branch in SQL: fall through.
|
|
262
|
+
return undefined;
|
|
263
|
+
}
|
|
264
|
+
if (parsed === null || typeof parsed !== 'object' || Array.isArray(parsed))
|
|
265
|
+
return undefined;
|
|
266
|
+
for (const key of ARGS_TOOL_NAME_KEYS) {
|
|
267
|
+
const candidate = trimmedString(parsed[key]);
|
|
268
|
+
if (candidate !== undefined)
|
|
269
|
+
return candidate;
|
|
270
|
+
}
|
|
271
|
+
return undefined;
|
|
272
|
+
}
|
|
273
|
+
/**
|
|
274
|
+
* Build the insert op for one imported record, routing to the typed or JSON
|
|
275
|
+
* column path per target table.
|
|
276
|
+
*/
|
|
192
277
|
export function recordInsertOp(targetTable, recordHash, sourceFile, sourceLine, splitIndex, payload, now) {
|
|
193
278
|
const table = targetTableFor(targetTable);
|
|
194
279
|
const typedColumns = TYPED_TABLE_COLUMNS[table];
|
|
@@ -201,11 +286,20 @@ export function recordInsertOp(targetTable, recordHash, sourceFile, sourceLine,
|
|
|
201
286
|
}
|
|
202
287
|
// record_hash and imported_at come from the function parameters, not the payload.
|
|
203
288
|
const ts = timestamp(now);
|
|
289
|
+
// Tool identity is derived here rather than in each mapper: this is the single insert
|
|
290
|
+
// choke point for every typed table, so one derivation covers every source (0739 R1).
|
|
291
|
+
const toolIdentity = table === 'history_tool_call' ? resolveToolIdentity(payload) : undefined;
|
|
204
292
|
const values = typedColumns.map((col) => {
|
|
205
293
|
if (col === 'record_hash')
|
|
206
294
|
return recordHash;
|
|
207
295
|
if (col === 'imported_at')
|
|
208
296
|
return ts;
|
|
297
|
+
if (toolIdentity !== undefined) {
|
|
298
|
+
if (col === 'effective_tool_name')
|
|
299
|
+
return toolIdentity.effective;
|
|
300
|
+
if (col === 'tool_name_alias')
|
|
301
|
+
return toolIdentity.alias;
|
|
302
|
+
}
|
|
209
303
|
const v = payload[col];
|
|
210
304
|
return v === undefined ? null : v;
|
|
211
305
|
});
|
|
@@ -299,6 +393,32 @@ export async function ledgerExistingHashes(db, hashes) {
|
|
|
299
393
|
}
|
|
300
394
|
return existing;
|
|
301
395
|
}
|
|
396
|
+
/**
|
|
397
|
+
* Fetch candidate assistant rows lacking duration_ms along with timestamp delta to preceding record.
|
|
398
|
+
*/
|
|
399
|
+
export async function findAssistantDurationCandidates(db) {
|
|
400
|
+
const candidateSql = `
|
|
401
|
+
WITH ordered AS (
|
|
402
|
+
SELECT record_hash,
|
|
403
|
+
role,
|
|
404
|
+
duration_ms,
|
|
405
|
+
ts,
|
|
406
|
+
LAG(ts) OVER (PARTITION BY source, session_id ORDER BY seq) AS prev_ts
|
|
407
|
+
FROM history_message
|
|
408
|
+
WHERE ts IS NOT NULL AND ts LIKE '____-__-__T%'
|
|
409
|
+
)
|
|
410
|
+
SELECT record_hash AS recordHash,
|
|
411
|
+
CAST(ROUND((unixepoch(ts, 'subsec') - unixepoch(prev_ts, 'subsec')) * 1000) AS INTEGER) AS deltaMs
|
|
412
|
+
FROM ordered
|
|
413
|
+
WHERE role = 'assistant' AND duration_ms IS NULL AND prev_ts IS NOT NULL`;
|
|
414
|
+
return db.queryAll(candidateSql);
|
|
415
|
+
}
|
|
416
|
+
/**
|
|
417
|
+
* Update duration_ms and duration_source for a previously unmeasured assistant row.
|
|
418
|
+
*/
|
|
419
|
+
export async function updateAssistantDuration(db, recordHash, durationMs, durationSource) {
|
|
420
|
+
await db.run('UPDATE history_message SET duration_ms = ?, duration_source = ? WHERE record_hash = ? AND duration_ms IS NULL', durationMs, durationSource, recordHash);
|
|
421
|
+
}
|
|
302
422
|
async function insertRecord(db, targetTable, recordHash, sourceFile, sourceLine, splitIndex, payload, now) {
|
|
303
423
|
const op = recordInsertOp(targetTable, recordHash, sourceFile, sourceLine, splitIndex, payload, now);
|
|
304
424
|
await ensureTargetTable(db, targetTable);
|
|
@@ -522,7 +642,14 @@ export function openCodeBulkWriteOperations(entries, checkpointFiles, now) {
|
|
|
522
642
|
FROM input
|
|
523
643
|
WHERE true
|
|
524
644
|
ON CONFLICT(record_hash) DO NOTHING`,
|
|
525
|
-
|
|
645
|
+
// Tool identity is folded into the payload rather than the SQL: this path
|
|
646
|
+
// json_extracts every column from payload_json, so a column the record lacks
|
|
647
|
+
// arrives NULL and violates the NOT NULL constraint (0739 R1).
|
|
648
|
+
params: chunk.flatMap((entry) => [
|
|
649
|
+
entry.recordHash,
|
|
650
|
+
JSON.stringify(table === 'history_tool_call' ? withToolIdentity(entry.record) : entry.record),
|
|
651
|
+
importedAt,
|
|
652
|
+
]),
|
|
526
653
|
});
|
|
527
654
|
}
|
|
528
655
|
}
|
package/dist/schema-sql.d.ts
CHANGED
|
@@ -1,3 +1,10 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Schema version of the history import DDL.
|
|
3
|
+
*
|
|
4
|
+
* Guaranteed to match the package version; a bump-or-fail test enforces that
|
|
5
|
+
* any modification to HISTORY_IMPORT_SCHEMA_SQL must be accompanied by a version bump.
|
|
6
|
+
*/
|
|
7
|
+
export declare const HISTORY_IMPORT_SCHEMA_VERSION = "0.4.56";
|
|
1
8
|
/**
|
|
2
9
|
* DDL string that creates the history import checkpoint, ledger, and typed contract tables.
|
|
3
10
|
*
|
package/dist/schema-sql.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"schema-sql.d.ts","sourceRoot":"","sources":["../src/schema-sql.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AACH,eAAO,MAAM,yBAAyB,
|
|
1
|
+
{"version":3,"file":"schema-sql.d.ts","sourceRoot":"","sources":["../src/schema-sql.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AACH,eAAO,MAAM,6BAA6B,WAAW,CAAC;AAEtD;;;;;GAKG;AACH,eAAO,MAAM,yBAAyB,QAqH9B,CAAC"}
|
package/dist/schema-sql.js
CHANGED
|
@@ -1,3 +1,10 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Schema version of the history import DDL.
|
|
3
|
+
*
|
|
4
|
+
* Guaranteed to match the package version; a bump-or-fail test enforces that
|
|
5
|
+
* any modification to HISTORY_IMPORT_SCHEMA_SQL must be accompanied by a version bump.
|
|
6
|
+
*/
|
|
7
|
+
export const HISTORY_IMPORT_SCHEMA_VERSION = '0.4.56';
|
|
1
8
|
/**
|
|
2
9
|
* DDL string that creates the history import checkpoint, ledger, and typed contract tables.
|
|
3
10
|
*
|
|
@@ -39,6 +46,7 @@ CREATE TABLE IF NOT EXISTS history_message (
|
|
|
39
46
|
disposition TEXT NOT NULL,
|
|
40
47
|
ts TEXT,
|
|
41
48
|
duration_ms INTEGER,
|
|
49
|
+
duration_source TEXT,
|
|
42
50
|
model TEXT,
|
|
43
51
|
input_tokens INTEGER,
|
|
44
52
|
output_tokens INTEGER,
|
|
@@ -72,7 +80,13 @@ CREATE TABLE IF NOT EXISTS history_tool_call (
|
|
|
72
80
|
duration_ms INTEGER,
|
|
73
81
|
result_bytes INTEGER,
|
|
74
82
|
error_text TEXT,
|
|
75
|
-
imported_at TEXT NOT NULL
|
|
83
|
+
imported_at TEXT NOT NULL,
|
|
84
|
+
-- Tool identity (0739). Appended last so a database that gained these columns
|
|
85
|
+
-- via Spur's guarded ALTER converges on the same PRAGMA column order as a
|
|
86
|
+
-- database created fresh from this DDL. 'unknown' is the unresolved sentinel
|
|
87
|
+
-- every consumer already falls back from.
|
|
88
|
+
effective_tool_name TEXT NOT NULL DEFAULT 'unknown',
|
|
89
|
+
tool_name_alias TEXT NOT NULL DEFAULT 'unknown'
|
|
76
90
|
);
|
|
77
91
|
|
|
78
92
|
CREATE INDEX IF NOT EXISTS idx_history_message_session
|
package/dist/tables.d.ts
ADDED
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
/** Bookkeeping tables owned and maintained by the importer. */
|
|
2
|
+
export declare const BOOKKEEPING_HISTORY_TABLES: readonly ["history_import_checkpoint", "history_import_ledger"];
|
|
3
|
+
/**
|
|
4
|
+
* Complete list of tables owned by the importer:
|
|
5
|
+
* - 3 typed contract tables (history_message, history_tool_call, history_skill_call)
|
|
6
|
+
* - 2 bookkeeping tables (history_import_checkpoint, history_import_ledger)
|
|
7
|
+
* - 10 per-source raw ETL landing tables (history_etl_<source>)
|
|
8
|
+
*
|
|
9
|
+
* Total: 15 tables.
|
|
10
|
+
* Sourced dynamically from static declarations so adding a source upstream automatically
|
|
11
|
+
* includes its landing table without downstream duplication.
|
|
12
|
+
*/
|
|
13
|
+
export declare const IMPORTER_OWNED_TABLES: readonly string[];
|
|
14
|
+
//# sourceMappingURL=tables.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"tables.d.ts","sourceRoot":"","sources":["../src/tables.ts"],"names":[],"mappings":"AAGA,+DAA+D;AAC/D,eAAO,MAAM,0BAA0B,iEAAkE,CAAC;AAE1G;;;;;;;;;GASG;AACH,eAAO,MAAM,qBAAqB,EAAE,SAAS,MAAM,EAIlD,CAAC"}
|
package/dist/tables.js
ADDED
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
import { TYPED_HISTORY_TABLES } from './jsonl-importer-dao.js';
|
|
2
|
+
import { SOURCE_DEFINITIONS } from './sources.js';
|
|
3
|
+
/** Bookkeeping tables owned and maintained by the importer. */
|
|
4
|
+
export const BOOKKEEPING_HISTORY_TABLES = ['history_import_checkpoint', 'history_import_ledger'];
|
|
5
|
+
/**
|
|
6
|
+
* Complete list of tables owned by the importer:
|
|
7
|
+
* - 3 typed contract tables (history_message, history_tool_call, history_skill_call)
|
|
8
|
+
* - 2 bookkeeping tables (history_import_checkpoint, history_import_ledger)
|
|
9
|
+
* - 10 per-source raw ETL landing tables (history_etl_<source>)
|
|
10
|
+
*
|
|
11
|
+
* Total: 15 tables.
|
|
12
|
+
* Sourced dynamically from static declarations so adding a source upstream automatically
|
|
13
|
+
* includes its landing table without downstream duplication.
|
|
14
|
+
*/
|
|
15
|
+
export const IMPORTER_OWNED_TABLES = [
|
|
16
|
+
...TYPED_HISTORY_TABLES,
|
|
17
|
+
...BOOKKEEPING_HISTORY_TABLES,
|
|
18
|
+
...Object.values(SOURCE_DEFINITIONS).map((def) => def.targetTable),
|
|
19
|
+
];
|
package/dist/types.d.ts
CHANGED
|
@@ -149,6 +149,7 @@ export interface SkillCallSplitRecord {
|
|
|
149
149
|
readonly completed_at?: string | null;
|
|
150
150
|
readonly duration_ms?: number | null;
|
|
151
151
|
}
|
|
152
|
+
/** Normalized skill-call record imported into history_skill_call. */
|
|
152
153
|
export interface SkillCall {
|
|
153
154
|
readonly record_hash: string;
|
|
154
155
|
readonly message_hash: string;
|
package/dist/types.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAClD,OAAO,KAAK,EAAE,UAAU,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAC;AACtE,OAAO,KAAK,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAE7B,6DAA6D;AAC7D,MAAM,MAAM,cAAc,GACpB,IAAI,GACJ,QAAQ,GACR,OAAO,GACP,QAAQ,GACR,UAAU,GACV,aAAa,GACb,UAAU,GACV,KAAK,GACL,MAAM,GACN,KAAK,CAAC;AAEZ,mDAAmD;AACnD,MAAM,MAAM,UAAU,GAAG,MAAM,GAAG,aAAa,GAAG,YAAY,CAAC;AAE/D,gFAAgF;AAChF,MAAM,WAAW,UAAU;IACvB,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,MAAM,EAAE,UAAU,CAAC;CAC/B;AAED,kFAAkF;AAClF,MAAM,MAAM,WAAW,GACjB;IACI,QAAQ,CAAC,IAAI,EAAE,YAAY,CAAC;CAC/B,GACD;IACI,QAAQ,CAAC,IAAI,EAAE,aAAa,CAAC;IAC7B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,CAAC;CACjC,GACD;IACI,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAC;IACxB,QAAQ,CAAC,KAAK,EAAE,CAAC,GAAG,EAAE,UAAU,EAAE,OAAO,CAAC,EAAE,gBAAgB,KAAK,SAAS,CAAC,UAAU,GAAG,UAAU,CAAC,EAAE,CAAC;IACtG,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,CAAC;CACjC,CAAC;AAER,4DAA4D;AAC5D,MAAM,MAAM,UAAU,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;AAEjD,sDAAsD;AACtD,MAAM,MAAM,cAAc,GAAG,CAAC,KAAK,EAAE,OAAO,EAAE,GAAG,EAAE,UAAU,EAAE,OAAO,EAAE,gBAAgB,KAAK,OAAO,CAAC;AAErG,0DAA0D;AAC1D,MAAM,WAAW,gBAAgB;IAC7B,2FAA2F;IAC3F,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;CAC/B;AAED,qEAAqE;AACrE,MAAM,WAAW,gBAAgB,CAAC,OAAO,SAAS,UAAU,GAAG,UAAU;IACrE,iHAAiH;IACjH,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,YAAY,EAAE,SAAS,MAAM,EAAE,CAAC;IACzC,QAAQ,CAAC,YAAY,EAAE,SAAS,MAAM,EAAE,CAAC;IACzC;;;;;OAKG;IACH,QAAQ,CAAC,iBAAiB,CAAC,EAAE,OAAO,GAAG,MAAM,CAAC;IAC9C,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,WAAW,EAAE,WAAW,CAAC;IAClC,QAAQ,CAAC,QAAQ,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;IACpD,QAAQ,CAAC,eAAe,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,cAAc,CAAC,CAAC,CAAC;IACnE,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;CACvC;AAED,6DAA6D;AAC7D,MAAM,WAAW,aAAa;IAC1B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;CAChC;AAED,oCAAoC;AACpC,MAAM,WAAW,aAAa;IAC1B,QAAQ,CAAC,EAAE,EAAE,SAAS,CAAC;IACvB,QAAQ,CAAC,UAAU,CAAC,EAAE,UAAU,CAAC;IACjC,QAAQ,CAAC,IAAI,CAAC,EAAE,UAAU,CAAC;IAC3B,QAAQ,CAAC,KAAK,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IACnC,QAAQ,CAAC,KAAK,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IACnC,QAAQ,CAAC,MAAM,CAAC,EAAE,OAAO,CAAC;IAC1B,QAAQ,CAAC,cAAc,CAAC,EAAE,SAAS,aAAa,EAAE,CAAC;IACnD,QAAQ,CAAC,GAAG,CAAC,EAAE,MAAM,IAAI,CAAC;IAC1B;;;;OAIG;IACH,QAAQ,CAAC,KAAK,CAAC,EAAE,YAAY,CAAC;CACjC;AAED,mFAAmF;AACnF,MAAM,WAAW,WAAW;IACxB,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;CAC3B;AAED;;;;;GAKG;AACH,MAAM,WAAW,gBAAgB;IAC7B,uGAAuG;IACvG,QAAQ,CAAC,eAAe,EAAE,MAAM,CAAC;IACjC,iDAAiD;IACjD,QAAQ,CAAC,eAAe,EAAE,MAAM,CAAC;IACjC,qEAAqE;IACrE,QAAQ,CAAC,mBAAmB,EAAE,MAAM,CAAC;CACxC;AAED,yDAAyD;AACzD,MAAM,WAAW,YAAY;IACzB,iGAAiG;IACjG,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,IAAI,EAAE,UAAU,CAAC;IAC1B,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC,QAAQ,CAAC,eAAe,EAAE,MAAM,CAAC;IACjC,QAAQ,CAAC,iBAAiB,EAAE,MAAM,CAAC;IACnC,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC,oFAAoF;IACpF,QAAQ,CAAC,mBAAmB,EAAE,MAAM,CAAC;IACrC,4FAA4F;IAC5F,QAAQ,CAAC,qBAAqB,EAAE,MAAM,CAAC;IACvC,QAAQ,CAAC,WAAW,EAAE,SAAS,WAAW,EAAE,CAAC;IAC7C,QAAQ,CAAC,gBAAgB,EAAE,SAAS,WAAW,EAAE,CAAC;IAClD,QAAQ,CAAC,iBAAiB,EAAE,MAAM,CAAC;IACnC;;;OAGG;IACH,QAAQ,CAAC,cAAc,CAAC,EAAE,gBAAgB,CAAC;CAC9C;AAED;;;;;;;GAOG;AACH;;;;;GAKG;AACH,MAAM,WAAW,oBAAoB;IACjC,QAAQ,CAAC,kBAAkB,CAAC,EAAE,MAAM,CAAC;IACrC,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,eAAe,EAAE,MAAM,GAAG,OAAO,CAAC;IAC3C,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACpC,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAClC,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACrC,QAAQ,CAAC,OAAO,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACjC,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAChC,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACpC,QAAQ,CAAC,YAAY,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACtC,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CACxC;AAED,MAAM,WAAW,SAAS;IACtB,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,eAAe,EAAE,MAAM,GAAG,OAAO,CAAC;IAC3C,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACpC,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAClC,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACrC,QAAQ,CAAC,OAAO,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACjC,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAChC,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACpC,QAAQ,CAAC,YAAY,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACtC,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACrC,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;CAChC"}
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAClD,OAAO,KAAK,EAAE,UAAU,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAC;AACtE,OAAO,KAAK,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAE7B,6DAA6D;AAC7D,MAAM,MAAM,cAAc,GACpB,IAAI,GACJ,QAAQ,GACR,OAAO,GACP,QAAQ,GACR,UAAU,GACV,aAAa,GACb,UAAU,GACV,KAAK,GACL,MAAM,GACN,KAAK,CAAC;AAEZ,mDAAmD;AACnD,MAAM,MAAM,UAAU,GAAG,MAAM,GAAG,aAAa,GAAG,YAAY,CAAC;AAE/D,gFAAgF;AAChF,MAAM,WAAW,UAAU;IACvB,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,MAAM,EAAE,UAAU,CAAC;CAC/B;AAED,kFAAkF;AAClF,MAAM,MAAM,WAAW,GACjB;IACI,QAAQ,CAAC,IAAI,EAAE,YAAY,CAAC;CAC/B,GACD;IACI,QAAQ,CAAC,IAAI,EAAE,aAAa,CAAC;IAC7B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,CAAC;CACjC,GACD;IACI,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAC;IACxB,QAAQ,CAAC,KAAK,EAAE,CAAC,GAAG,EAAE,UAAU,EAAE,OAAO,CAAC,EAAE,gBAAgB,KAAK,SAAS,CAAC,UAAU,GAAG,UAAU,CAAC,EAAE,CAAC;IACtG,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,CAAC;CACjC,CAAC;AAER,4DAA4D;AAC5D,MAAM,MAAM,UAAU,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;AAEjD,sDAAsD;AACtD,MAAM,MAAM,cAAc,GAAG,CAAC,KAAK,EAAE,OAAO,EAAE,GAAG,EAAE,UAAU,EAAE,OAAO,EAAE,gBAAgB,KAAK,OAAO,CAAC;AAErG,0DAA0D;AAC1D,MAAM,WAAW,gBAAgB;IAC7B,2FAA2F;IAC3F,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;CAC/B;AAED,qEAAqE;AACrE,MAAM,WAAW,gBAAgB,CAAC,OAAO,SAAS,UAAU,GAAG,UAAU;IACrE,iHAAiH;IACjH,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,YAAY,EAAE,SAAS,MAAM,EAAE,CAAC;IACzC,QAAQ,CAAC,YAAY,EAAE,SAAS,MAAM,EAAE,CAAC;IACzC;;;;;OAKG;IACH,QAAQ,CAAC,iBAAiB,CAAC,EAAE,OAAO,GAAG,MAAM,CAAC;IAC9C,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,WAAW,EAAE,WAAW,CAAC;IAClC,QAAQ,CAAC,QAAQ,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;IACpD,QAAQ,CAAC,eAAe,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,cAAc,CAAC,CAAC,CAAC;IACnE,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;CACvC;AAED,6DAA6D;AAC7D,MAAM,WAAW,aAAa;IAC1B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;CAChC;AAED,oCAAoC;AACpC,MAAM,WAAW,aAAa;IAC1B,QAAQ,CAAC,EAAE,EAAE,SAAS,CAAC;IACvB,QAAQ,CAAC,UAAU,CAAC,EAAE,UAAU,CAAC;IACjC,QAAQ,CAAC,IAAI,CAAC,EAAE,UAAU,CAAC;IAC3B,QAAQ,CAAC,KAAK,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IACnC,QAAQ,CAAC,KAAK,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IACnC,QAAQ,CAAC,MAAM,CAAC,EAAE,OAAO,CAAC;IAC1B,QAAQ,CAAC,cAAc,CAAC,EAAE,SAAS,aAAa,EAAE,CAAC;IACnD,QAAQ,CAAC,GAAG,CAAC,EAAE,MAAM,IAAI,CAAC;IAC1B;;;;OAIG;IACH,QAAQ,CAAC,KAAK,CAAC,EAAE,YAAY,CAAC;CACjC;AAED,mFAAmF;AACnF,MAAM,WAAW,WAAW;IACxB,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;CAC3B;AAED;;;;;GAKG;AACH,MAAM,WAAW,gBAAgB;IAC7B,uGAAuG;IACvG,QAAQ,CAAC,eAAe,EAAE,MAAM,CAAC;IACjC,iDAAiD;IACjD,QAAQ,CAAC,eAAe,EAAE,MAAM,CAAC;IACjC,qEAAqE;IACrE,QAAQ,CAAC,mBAAmB,EAAE,MAAM,CAAC;CACxC;AAED,yDAAyD;AACzD,MAAM,WAAW,YAAY;IACzB,iGAAiG;IACjG,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,IAAI,EAAE,UAAU,CAAC;IAC1B,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC,QAAQ,CAAC,eAAe,EAAE,MAAM,CAAC;IACjC,QAAQ,CAAC,iBAAiB,EAAE,MAAM,CAAC;IACnC,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC,oFAAoF;IACpF,QAAQ,CAAC,mBAAmB,EAAE,MAAM,CAAC;IACrC,4FAA4F;IAC5F,QAAQ,CAAC,qBAAqB,EAAE,MAAM,CAAC;IACvC,QAAQ,CAAC,WAAW,EAAE,SAAS,WAAW,EAAE,CAAC;IAC7C,QAAQ,CAAC,gBAAgB,EAAE,SAAS,WAAW,EAAE,CAAC;IAClD,QAAQ,CAAC,iBAAiB,EAAE,MAAM,CAAC;IACnC;;;OAGG;IACH,QAAQ,CAAC,cAAc,CAAC,EAAE,gBAAgB,CAAC;CAC9C;AAED;;;;;;;GAOG;AACH;;;;;GAKG;AACH,MAAM,WAAW,oBAAoB;IACjC,QAAQ,CAAC,kBAAkB,CAAC,EAAE,MAAM,CAAC;IACrC,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,eAAe,EAAE,MAAM,GAAG,OAAO,CAAC;IAC3C,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACpC,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAClC,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACrC,QAAQ,CAAC,OAAO,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACjC,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAChC,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACpC,QAAQ,CAAC,YAAY,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACtC,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CACxC;AAED,qEAAqE;AACrE,MAAM,WAAW,SAAS;IACtB,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,eAAe,EAAE,MAAM,GAAG,OAAO,CAAC;IAC3C,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACpC,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAClC,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACrC,QAAQ,CAAC,OAAO,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACjC,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAChC,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACpC,QAAQ,CAAC,YAAY,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACtC,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACrC,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;CAChC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@gobing-ai/ts-llm-jsonl-importer",
|
|
3
|
-
"version": "0.4.
|
|
3
|
+
"version": "0.4.56",
|
|
4
4
|
"description": "@gobing-ai/ts-llm-jsonl-importer — Generic JSONL importer for LLM agent history files.",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"typescript",
|
|
@@ -47,9 +47,9 @@
|
|
|
47
47
|
"release": "echo 'Manual publish is disabled. Releases go through GitHub Actions via Trusted Publishing — push a tag: git tag @gobing-ai/ts-llm-jsonl-importer-v<version> && git push --tags' && exit 1"
|
|
48
48
|
},
|
|
49
49
|
"dependencies": {
|
|
50
|
-
"@gobing-ai/ts-db": "^0.4.
|
|
51
|
-
"@gobing-ai/ts-runtime": "^0.4.
|
|
52
|
-
"@gobing-ai/ts-utils": "^0.4.
|
|
50
|
+
"@gobing-ai/ts-db": "^0.4.56",
|
|
51
|
+
"@gobing-ai/ts-runtime": "^0.4.56",
|
|
52
|
+
"@gobing-ai/ts-utils": "^0.4.56",
|
|
53
53
|
"zod": "^4.1.0"
|
|
54
54
|
},
|
|
55
55
|
"devDependencies": {
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
import type { DbAdapter } from '@gobing-ai/ts-db';
|
|
2
|
+
import { findAssistantDurationCandidates, updateAssistantDuration } from './jsonl-importer-dao';
|
|
3
|
+
|
|
4
|
+
/**
|
|
5
|
+
* ETL-derived assistant-step duration (repatriated upstream from Spur task 0747 / ADR-105).
|
|
6
|
+
*
|
|
7
|
+
* Four of six sources — `claude`, `pi`, `codex`, `agy` — write no per-step
|
|
8
|
+
* `duration_ms` at all, so bottleneck ranking and per-model latency were unusable for
|
|
9
|
+
* the two busiest of them and ~73% of the measured span could not be attributed to
|
|
10
|
+
* llm/tool/idle. The transcripts carry a per-record `ts`, so the delta from the
|
|
11
|
+
* preceding record in the same session is derivable here.
|
|
12
|
+
*
|
|
13
|
+
* **It is an approximation and is labelled as one.** The delta includes queue and
|
|
14
|
+
* network time, which is arguably what "assistant step duration" should mean for
|
|
15
|
+
* bottleneck ranking, but it is not the provider's own number. `duration_source` carries
|
|
16
|
+
* that distinction to every consumer: `NULL` is provider-reported, `'derived'` is this
|
|
17
|
+
* function's work. Nothing may present the two as the same measurement.
|
|
18
|
+
*/
|
|
19
|
+
|
|
20
|
+
/** `history_message.duration_source` values. `NULL` means provider-reported. */
|
|
21
|
+
export const DURATION_SOURCE_DERIVED = 'derived';
|
|
22
|
+
|
|
23
|
+
/**
|
|
24
|
+
* Deltas above this are session gaps, not work: the operator walked away and came back.
|
|
25
|
+
* Attributing an overnight gap to an assistant step would corrupt the very ranking this
|
|
26
|
+
* derivation exists to make usable, so those rows stay unmeasured.
|
|
27
|
+
*/
|
|
28
|
+
export const DERIVED_DURATION_CEILING_MS = 30 * 60 * 1000;
|
|
29
|
+
|
|
30
|
+
/** What one derivation pass changed, and what it deliberately left alone. */
|
|
31
|
+
export interface DeriveAssistantDurationsResult {
|
|
32
|
+
/** Rows that gained a derived `duration_ms` on this pass. */
|
|
33
|
+
derived: number;
|
|
34
|
+
/** Candidate rows left unmeasured because the delta exceeded the ceiling. */
|
|
35
|
+
skippedOverCeiling: number;
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
/**
|
|
39
|
+
* Fill `duration_ms` for assistant rows that have none, from the timestamp delta to the
|
|
40
|
+
* preceding record in the same session.
|
|
41
|
+
*
|
|
42
|
+
* Idempotent and additive: a row whose `duration_ms` is already set is never touched, so
|
|
43
|
+
* a provider measurement always wins over a derived one, and re-running after a later
|
|
44
|
+
* import only fills rows the earlier pass could not reach.
|
|
45
|
+
*/
|
|
46
|
+
export async function deriveAssistantDurations(db: DbAdapter): Promise<DeriveAssistantDurationsResult> {
|
|
47
|
+
const rows = await findAssistantDurationCandidates(db);
|
|
48
|
+
|
|
49
|
+
let derived = 0;
|
|
50
|
+
let skippedOverCeiling = 0;
|
|
51
|
+
for (const row of rows) {
|
|
52
|
+
// A non-positive delta means the records share a timestamp or arrived out of
|
|
53
|
+
// order; there is no duration to claim, so leave the row unmeasured rather than
|
|
54
|
+
// invent a zero (0680 R6: absent is not zero).
|
|
55
|
+
if (row.deltaMs === null || row.deltaMs <= 0) continue;
|
|
56
|
+
if (row.deltaMs > DERIVED_DURATION_CEILING_MS) {
|
|
57
|
+
skippedOverCeiling += 1;
|
|
58
|
+
continue;
|
|
59
|
+
}
|
|
60
|
+
await updateAssistantDuration(db, row.recordHash, row.deltaMs, DURATION_SOURCE_DERIVED);
|
|
61
|
+
derived += 1;
|
|
62
|
+
}
|
|
63
|
+
return { derived, skippedOverCeiling };
|
|
64
|
+
}
|
package/src/index.ts
CHANGED
|
@@ -1,7 +1,13 @@
|
|
|
1
|
+
export {
|
|
2
|
+
DERIVED_DURATION_CEILING_MS,
|
|
3
|
+
type DeriveAssistantDurationsResult,
|
|
4
|
+
DURATION_SOURCE_DERIVED,
|
|
5
|
+
deriveAssistantDurations,
|
|
6
|
+
} from './assistant-duration';
|
|
1
7
|
export { HistoryImportError } from './errors';
|
|
2
8
|
export { sha256, stableJson } from './hash';
|
|
3
9
|
export { runJsonlImport } from './importer';
|
|
4
|
-
export { applyHistoryImportSchema, normalizeSourceFilePaths } from './jsonl-importer-dao';
|
|
10
|
+
export { applyHistoryImportSchema, normalizeSourceFilePaths, TYPED_HISTORY_TABLES } from './jsonl-importer-dao';
|
|
5
11
|
export {
|
|
6
12
|
canonicalizeSkillName,
|
|
7
13
|
extractSkillCalls,
|
|
@@ -11,7 +17,7 @@ export {
|
|
|
11
17
|
} from './mappers';
|
|
12
18
|
export { type OpenCodeImportOptions, runOpenCodeImport } from './opencode-importer';
|
|
13
19
|
export { DEFAULT_REDACTION_RULES, redactRecord, redactValue } from './redaction';
|
|
14
|
-
export { HISTORY_IMPORT_SCHEMA_SQL } from './schema-sql';
|
|
20
|
+
export { HISTORY_IMPORT_SCHEMA_SQL, HISTORY_IMPORT_SCHEMA_VERSION } from './schema-sql';
|
|
15
21
|
export {
|
|
16
22
|
getSourceDefinition,
|
|
17
23
|
resolveSourceDefinition,
|
|
@@ -19,6 +25,7 @@ export {
|
|
|
19
25
|
VALID_TABLE_NAME,
|
|
20
26
|
validateSourceDefinition,
|
|
21
27
|
} from './sources';
|
|
28
|
+
export { BOOKKEEPING_HISTORY_TABLES, IMPORTER_OWNED_TABLES } from './tables';
|
|
22
29
|
export type {
|
|
23
30
|
FieldTransform,
|
|
24
31
|
ImportIssue,
|
|
@@ -17,6 +17,13 @@ export interface SourceCheckpoint {
|
|
|
17
17
|
readonly mtimeMs: number | null;
|
|
18
18
|
}
|
|
19
19
|
|
|
20
|
+
/**
|
|
21
|
+
* Typed contract tables created and populated by the importer.
|
|
22
|
+
*
|
|
23
|
+
* Promoted to public surface per task 0749 (ADR-105).
|
|
24
|
+
*/
|
|
25
|
+
export const TYPED_HISTORY_TABLES = ['history_message', 'history_tool_call', 'history_skill_call'] as const;
|
|
26
|
+
|
|
20
27
|
/** Column allowlist per typed contract table; order is the INSERT column order. */
|
|
21
28
|
const TYPED_TABLE_COLUMNS: Readonly<Record<string, readonly string[]>> = {
|
|
22
29
|
history_message: [
|
|
@@ -65,6 +72,8 @@ const TYPED_TABLE_COLUMNS: Readonly<Record<string, readonly string[]>> = {
|
|
|
65
72
|
'result_bytes',
|
|
66
73
|
'error_text',
|
|
67
74
|
'imported_at',
|
|
75
|
+
'effective_tool_name',
|
|
76
|
+
'tool_name_alias',
|
|
68
77
|
],
|
|
69
78
|
history_skill_call: [
|
|
70
79
|
'record_hash',
|
|
@@ -245,6 +254,84 @@ async function ledgerExists(db: ImportOptions['db'], recordHash: string): Promis
|
|
|
245
254
|
* Typed tables validate unknown keys here; the caller must have ensured the
|
|
246
255
|
* target table exists (the import loop ensures accepted targets before batching).
|
|
247
256
|
*/
|
|
257
|
+
/** Wrapper-style `call_id` prefixes that name the tool the wrapper actually ran. */
|
|
258
|
+
const CALL_ID_TOOL_PREFIXES: readonly (readonly [string, string])[] = [
|
|
259
|
+
['call_bash_', 'bash'],
|
|
260
|
+
['call_read_', 'read'],
|
|
261
|
+
['call_edit_', 'edit'],
|
|
262
|
+
['call_write_', 'write'],
|
|
263
|
+
['call_grep_', 'grep'],
|
|
264
|
+
['call_find_', 'find'],
|
|
265
|
+
['call_ls_', 'ls'],
|
|
266
|
+
];
|
|
267
|
+
|
|
268
|
+
/** Argument keys, in precedence order, that carry the tool name when `tool_name` is empty. */
|
|
269
|
+
const ARGS_TOOL_NAME_KEYS = ['tool', 'tool_name', 'toolName', 'name', 'command'] as const;
|
|
270
|
+
|
|
271
|
+
/**
|
|
272
|
+
* Tool identity for one `history_tool_call` row (0739).
|
|
273
|
+
*
|
|
274
|
+
* `effective` is extraction — recover the name when the agent recorded it inside a wrapper's
|
|
275
|
+
* arguments or only in the call id. `alias` is canonicalization — grouping the same logical tool
|
|
276
|
+
* across agents — and starts at identity, because the alias vocabulary is a curated decision
|
|
277
|
+
* that belongs to the consumer, not to the importer.
|
|
278
|
+
*
|
|
279
|
+
* Mirrors Spur migration 0034's backfill CASE so a row imported after the migration and a row
|
|
280
|
+
* backfilled by it resolve identically.
|
|
281
|
+
*/
|
|
282
|
+
export function resolveToolIdentity(payload: JsonObject): { effective: string; alias: string } {
|
|
283
|
+
const supplied = trimmedString(payload.effective_tool_name);
|
|
284
|
+
const effective = supplied !== undefined ? supplied : deriveEffectiveToolName(payload);
|
|
285
|
+
return { effective, alias: trimmedString(payload.tool_name_alias) ?? effective };
|
|
286
|
+
}
|
|
287
|
+
|
|
288
|
+
/** Returns the record with tool identity resolved, for insert paths that read columns off the payload. */
|
|
289
|
+
function withToolIdentity(record: JsonObject): JsonObject {
|
|
290
|
+
const { effective, alias } = resolveToolIdentity(record);
|
|
291
|
+
return { ...record, effective_tool_name: effective, tool_name_alias: alias };
|
|
292
|
+
}
|
|
293
|
+
|
|
294
|
+
function trimmedString(value: unknown): string | undefined {
|
|
295
|
+
if (typeof value !== 'string') return undefined;
|
|
296
|
+
const trimmed = value.trim();
|
|
297
|
+
return trimmed === '' ? undefined : trimmed;
|
|
298
|
+
}
|
|
299
|
+
|
|
300
|
+
function deriveEffectiveToolName(payload: JsonObject): string {
|
|
301
|
+
const recorded = trimmedString(payload.tool_name);
|
|
302
|
+
if (recorded !== undefined && recorded !== 'unknown') return recorded;
|
|
303
|
+
|
|
304
|
+
const fromArgs = toolNameFromArgs(payload.args_raw);
|
|
305
|
+
if (fromArgs !== undefined) return fromArgs;
|
|
306
|
+
|
|
307
|
+
const callId = typeof payload.call_id === 'string' ? payload.call_id : '';
|
|
308
|
+
for (const [prefix, tool] of CALL_ID_TOOL_PREFIXES) {
|
|
309
|
+
if (callId.startsWith(prefix)) return tool;
|
|
310
|
+
}
|
|
311
|
+
return 'unknown';
|
|
312
|
+
}
|
|
313
|
+
|
|
314
|
+
function toolNameFromArgs(argsRaw: unknown): string | undefined {
|
|
315
|
+
if (typeof argsRaw !== 'string') return undefined;
|
|
316
|
+
let parsed: unknown;
|
|
317
|
+
try {
|
|
318
|
+
parsed = JSON.parse(argsRaw);
|
|
319
|
+
} catch {
|
|
320
|
+
// Non-JSON arguments are the `json_valid(args_raw)` false branch in SQL: fall through.
|
|
321
|
+
return undefined;
|
|
322
|
+
}
|
|
323
|
+
if (parsed === null || typeof parsed !== 'object' || Array.isArray(parsed)) return undefined;
|
|
324
|
+
for (const key of ARGS_TOOL_NAME_KEYS) {
|
|
325
|
+
const candidate = trimmedString((parsed as Record<string, unknown>)[key]);
|
|
326
|
+
if (candidate !== undefined) return candidate;
|
|
327
|
+
}
|
|
328
|
+
return undefined;
|
|
329
|
+
}
|
|
330
|
+
|
|
331
|
+
/**
|
|
332
|
+
* Build the insert op for one imported record, routing to the typed or JSON
|
|
333
|
+
* column path per target table.
|
|
334
|
+
*/
|
|
248
335
|
export function recordInsertOp(
|
|
249
336
|
targetTable: string,
|
|
250
337
|
recordHash: string,
|
|
@@ -268,9 +355,16 @@ export function recordInsertOp(
|
|
|
268
355
|
}
|
|
269
356
|
// record_hash and imported_at come from the function parameters, not the payload.
|
|
270
357
|
const ts = timestamp(now);
|
|
358
|
+
// Tool identity is derived here rather than in each mapper: this is the single insert
|
|
359
|
+
// choke point for every typed table, so one derivation covers every source (0739 R1).
|
|
360
|
+
const toolIdentity = table === 'history_tool_call' ? resolveToolIdentity(payload) : undefined;
|
|
271
361
|
const values = typedColumns.map((col) => {
|
|
272
362
|
if (col === 'record_hash') return recordHash;
|
|
273
363
|
if (col === 'imported_at') return ts;
|
|
364
|
+
if (toolIdentity !== undefined) {
|
|
365
|
+
if (col === 'effective_tool_name') return toolIdentity.effective;
|
|
366
|
+
if (col === 'tool_name_alias') return toolIdentity.alias;
|
|
367
|
+
}
|
|
274
368
|
const v = payload[col];
|
|
275
369
|
return v === undefined ? null : v;
|
|
276
370
|
});
|
|
@@ -395,6 +489,51 @@ export async function ledgerExistingHashes(db: ImportOptions['db'], hashes: read
|
|
|
395
489
|
return existing;
|
|
396
490
|
}
|
|
397
491
|
|
|
492
|
+
/** Candidate assistant-message row for duration derivation. */
|
|
493
|
+
export interface AssistantDurationCandidate {
|
|
494
|
+
readonly recordHash: string;
|
|
495
|
+
readonly deltaMs: number | null;
|
|
496
|
+
}
|
|
497
|
+
|
|
498
|
+
/**
|
|
499
|
+
* Fetch candidate assistant rows lacking duration_ms along with timestamp delta to preceding record.
|
|
500
|
+
*/
|
|
501
|
+
export async function findAssistantDurationCandidates(db: DbAdapter): Promise<AssistantDurationCandidate[]> {
|
|
502
|
+
const candidateSql = `
|
|
503
|
+
WITH ordered AS (
|
|
504
|
+
SELECT record_hash,
|
|
505
|
+
role,
|
|
506
|
+
duration_ms,
|
|
507
|
+
ts,
|
|
508
|
+
LAG(ts) OVER (PARTITION BY source, session_id ORDER BY seq) AS prev_ts
|
|
509
|
+
FROM history_message
|
|
510
|
+
WHERE ts IS NOT NULL AND ts LIKE '____-__-__T%'
|
|
511
|
+
)
|
|
512
|
+
SELECT record_hash AS recordHash,
|
|
513
|
+
CAST(ROUND((unixepoch(ts, 'subsec') - unixepoch(prev_ts, 'subsec')) * 1000) AS INTEGER) AS deltaMs
|
|
514
|
+
FROM ordered
|
|
515
|
+
WHERE role = 'assistant' AND duration_ms IS NULL AND prev_ts IS NOT NULL`;
|
|
516
|
+
|
|
517
|
+
return db.queryAll<AssistantDurationCandidate>(candidateSql);
|
|
518
|
+
}
|
|
519
|
+
|
|
520
|
+
/**
|
|
521
|
+
* Update duration_ms and duration_source for a previously unmeasured assistant row.
|
|
522
|
+
*/
|
|
523
|
+
export async function updateAssistantDuration(
|
|
524
|
+
db: DbAdapter,
|
|
525
|
+
recordHash: string,
|
|
526
|
+
durationMs: number,
|
|
527
|
+
durationSource: string,
|
|
528
|
+
): Promise<void> {
|
|
529
|
+
await db.run(
|
|
530
|
+
'UPDATE history_message SET duration_ms = ?, duration_source = ? WHERE record_hash = ? AND duration_ms IS NULL',
|
|
531
|
+
durationMs,
|
|
532
|
+
durationSource,
|
|
533
|
+
recordHash,
|
|
534
|
+
);
|
|
535
|
+
}
|
|
536
|
+
|
|
398
537
|
async function insertRecord(
|
|
399
538
|
db: ImportOptions['db'],
|
|
400
539
|
targetTable: string,
|
|
@@ -774,7 +913,14 @@ export function openCodeBulkWriteOperations(
|
|
|
774
913
|
FROM input
|
|
775
914
|
WHERE true
|
|
776
915
|
ON CONFLICT(record_hash) DO NOTHING`,
|
|
777
|
-
|
|
916
|
+
// Tool identity is folded into the payload rather than the SQL: this path
|
|
917
|
+
// json_extracts every column from payload_json, so a column the record lacks
|
|
918
|
+
// arrives NULL and violates the NOT NULL constraint (0739 R1).
|
|
919
|
+
params: chunk.flatMap((entry) => [
|
|
920
|
+
entry.recordHash,
|
|
921
|
+
JSON.stringify(table === 'history_tool_call' ? withToolIdentity(entry.record) : entry.record),
|
|
922
|
+
importedAt,
|
|
923
|
+
]),
|
|
778
924
|
});
|
|
779
925
|
}
|
|
780
926
|
}
|
package/src/schema-sql.ts
CHANGED
|
@@ -1,3 +1,11 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Schema version of the history import DDL.
|
|
3
|
+
*
|
|
4
|
+
* Guaranteed to match the package version; a bump-or-fail test enforces that
|
|
5
|
+
* any modification to HISTORY_IMPORT_SCHEMA_SQL must be accompanied by a version bump.
|
|
6
|
+
*/
|
|
7
|
+
export const HISTORY_IMPORT_SCHEMA_VERSION = '0.4.56';
|
|
8
|
+
|
|
1
9
|
/**
|
|
2
10
|
* DDL string that creates the history import checkpoint, ledger, and typed contract tables.
|
|
3
11
|
*
|
|
@@ -39,6 +47,7 @@ CREATE TABLE IF NOT EXISTS history_message (
|
|
|
39
47
|
disposition TEXT NOT NULL,
|
|
40
48
|
ts TEXT,
|
|
41
49
|
duration_ms INTEGER,
|
|
50
|
+
duration_source TEXT,
|
|
42
51
|
model TEXT,
|
|
43
52
|
input_tokens INTEGER,
|
|
44
53
|
output_tokens INTEGER,
|
|
@@ -72,7 +81,13 @@ CREATE TABLE IF NOT EXISTS history_tool_call (
|
|
|
72
81
|
duration_ms INTEGER,
|
|
73
82
|
result_bytes INTEGER,
|
|
74
83
|
error_text TEXT,
|
|
75
|
-
imported_at TEXT NOT NULL
|
|
84
|
+
imported_at TEXT NOT NULL,
|
|
85
|
+
-- Tool identity (0739). Appended last so a database that gained these columns
|
|
86
|
+
-- via Spur's guarded ALTER converges on the same PRAGMA column order as a
|
|
87
|
+
-- database created fresh from this DDL. 'unknown' is the unresolved sentinel
|
|
88
|
+
-- every consumer already falls back from.
|
|
89
|
+
effective_tool_name TEXT NOT NULL DEFAULT 'unknown',
|
|
90
|
+
tool_name_alias TEXT NOT NULL DEFAULT 'unknown'
|
|
76
91
|
);
|
|
77
92
|
|
|
78
93
|
CREATE INDEX IF NOT EXISTS idx_history_message_session
|
package/src/tables.ts
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
import { TYPED_HISTORY_TABLES } from './jsonl-importer-dao';
|
|
2
|
+
import { SOURCE_DEFINITIONS } from './sources';
|
|
3
|
+
|
|
4
|
+
/** Bookkeeping tables owned and maintained by the importer. */
|
|
5
|
+
export const BOOKKEEPING_HISTORY_TABLES = ['history_import_checkpoint', 'history_import_ledger'] as const;
|
|
6
|
+
|
|
7
|
+
/**
|
|
8
|
+
* Complete list of tables owned by the importer:
|
|
9
|
+
* - 3 typed contract tables (history_message, history_tool_call, history_skill_call)
|
|
10
|
+
* - 2 bookkeeping tables (history_import_checkpoint, history_import_ledger)
|
|
11
|
+
* - 10 per-source raw ETL landing tables (history_etl_<source>)
|
|
12
|
+
*
|
|
13
|
+
* Total: 15 tables.
|
|
14
|
+
* Sourced dynamically from static declarations so adding a source upstream automatically
|
|
15
|
+
* includes its landing table without downstream duplication.
|
|
16
|
+
*/
|
|
17
|
+
export const IMPORTER_OWNED_TABLES: readonly string[] = [
|
|
18
|
+
...TYPED_HISTORY_TABLES,
|
|
19
|
+
...BOOKKEEPING_HISTORY_TABLES,
|
|
20
|
+
...Object.values(SOURCE_DEFINITIONS).map((def) => def.targetTable),
|
|
21
|
+
];
|
package/src/types.ts
CHANGED
|
@@ -177,6 +177,7 @@ export interface SkillCallSplitRecord {
|
|
|
177
177
|
readonly duration_ms?: number | null;
|
|
178
178
|
}
|
|
179
179
|
|
|
180
|
+
/** Normalized skill-call record imported into history_skill_call. */
|
|
180
181
|
export interface SkillCall {
|
|
181
182
|
readonly record_hash: string;
|
|
182
183
|
readonly message_hash: string;
|