@gobing-ai/ts-llm-jsonl-importer 0.4.54 → 0.4.55
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/assistant-duration.d.ts +41 -0
- package/dist/assistant-duration.d.ts.map +1 -0
- package/dist/assistant-duration.js +51 -0
- package/dist/index.d.ts +4 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +4 -2
- package/dist/jsonl-importer-dao.d.ts +19 -0
- package/dist/jsonl-importer-dao.d.ts.map +1 -1
- package/dist/jsonl-importer-dao.js +32 -0
- package/dist/schema-sql.d.ts +7 -0
- package/dist/schema-sql.d.ts.map +1 -1
- package/dist/schema-sql.js +8 -0
- package/dist/tables.d.ts +14 -0
- package/dist/tables.d.ts.map +1 -0
- package/dist/tables.js +19 -0
- package/dist/types.d.ts +1 -0
- package/dist/types.d.ts.map +1 -1
- package/package.json +4 -4
- package/src/assistant-duration.ts +64 -0
- package/src/index.ts +9 -2
- package/src/jsonl-importer-dao.ts +52 -0
- package/src/schema-sql.ts +9 -0
- package/src/tables.ts +21 -0
- package/src/types.ts +1 -0
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
import type { DbAdapter } from '@gobing-ai/ts-db';
|
|
2
|
+
/**
|
|
3
|
+
* ETL-derived assistant-step duration (repatriated upstream from Spur task 0747 / ADR-105).
|
|
4
|
+
*
|
|
5
|
+
* Four of six sources — `claude`, `pi`, `codex`, `agy` — write no per-step
|
|
6
|
+
* `duration_ms` at all, so bottleneck ranking and per-model latency were unusable for
|
|
7
|
+
* the two busiest of them and ~73% of the measured span could not be attributed to
|
|
8
|
+
* llm/tool/idle. The transcripts carry a per-record `ts`, so the delta from the
|
|
9
|
+
* preceding record in the same session is derivable here.
|
|
10
|
+
*
|
|
11
|
+
* **It is an approximation and is labelled as one.** The delta includes queue and
|
|
12
|
+
* network time, which is arguably what "assistant step duration" should mean for
|
|
13
|
+
* bottleneck ranking, but it is not the provider's own number. `duration_source` carries
|
|
14
|
+
* that distinction to every consumer: `NULL` is provider-reported, `'derived'` is this
|
|
15
|
+
* function's work. Nothing may present the two as the same measurement.
|
|
16
|
+
*/
|
|
17
|
+
/** `history_message.duration_source` values. `NULL` means provider-reported. */
|
|
18
|
+
export declare const DURATION_SOURCE_DERIVED = "derived";
|
|
19
|
+
/**
|
|
20
|
+
* Deltas above this are session gaps, not work: the operator walked away and came back.
|
|
21
|
+
* Attributing an overnight gap to an assistant step would corrupt the very ranking this
|
|
22
|
+
* derivation exists to make usable, so those rows stay unmeasured.
|
|
23
|
+
*/
|
|
24
|
+
export declare const DERIVED_DURATION_CEILING_MS: number;
|
|
25
|
+
/** What one derivation pass changed, and what it deliberately left alone. */
|
|
26
|
+
export interface DeriveAssistantDurationsResult {
|
|
27
|
+
/** Rows that gained a derived `duration_ms` on this pass. */
|
|
28
|
+
derived: number;
|
|
29
|
+
/** Candidate rows left unmeasured because the delta exceeded the ceiling. */
|
|
30
|
+
skippedOverCeiling: number;
|
|
31
|
+
}
|
|
32
|
+
/**
|
|
33
|
+
* Fill `duration_ms` for assistant rows that have none, from the timestamp delta to the
|
|
34
|
+
* preceding record in the same session.
|
|
35
|
+
*
|
|
36
|
+
* Idempotent and additive: a row whose `duration_ms` is already set is never touched, so
|
|
37
|
+
* a provider measurement always wins over a derived one, and re-running after a later
|
|
38
|
+
* import only fills rows the earlier pass could not reach.
|
|
39
|
+
*/
|
|
40
|
+
export declare function deriveAssistantDurations(db: DbAdapter): Promise<DeriveAssistantDurationsResult>;
|
|
41
|
+
//# sourceMappingURL=assistant-duration.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"assistant-duration.d.ts","sourceRoot":"","sources":["../src/assistant-duration.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAGlD;;;;;;;;;;;;;;GAcG;AAEH,gFAAgF;AAChF,eAAO,MAAM,uBAAuB,YAAY,CAAC;AAEjD;;;;GAIG;AACH,eAAO,MAAM,2BAA2B,QAAiB,CAAC;AAE1D,6EAA6E;AAC7E,MAAM,WAAW,8BAA8B;IAC3C,6DAA6D;IAC7D,OAAO,EAAE,MAAM,CAAC;IAChB,6EAA6E;IAC7E,kBAAkB,EAAE,MAAM,CAAC;CAC9B;AAED;;;;;;;GAOG;AACH,wBAAsB,wBAAwB,CAAC,EAAE,EAAE,SAAS,GAAG,OAAO,CAAC,8BAA8B,CAAC,CAkBrG"}
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
import { findAssistantDurationCandidates, updateAssistantDuration } from './jsonl-importer-dao.js';
|
|
2
|
+
/**
|
|
3
|
+
* ETL-derived assistant-step duration (repatriated upstream from Spur task 0747 / ADR-105).
|
|
4
|
+
*
|
|
5
|
+
* Four of six sources — `claude`, `pi`, `codex`, `agy` — write no per-step
|
|
6
|
+
* `duration_ms` at all, so bottleneck ranking and per-model latency were unusable for
|
|
7
|
+
* the two busiest of them and ~73% of the measured span could not be attributed to
|
|
8
|
+
* llm/tool/idle. The transcripts carry a per-record `ts`, so the delta from the
|
|
9
|
+
* preceding record in the same session is derivable here.
|
|
10
|
+
*
|
|
11
|
+
* **It is an approximation and is labelled as one.** The delta includes queue and
|
|
12
|
+
* network time, which is arguably what "assistant step duration" should mean for
|
|
13
|
+
* bottleneck ranking, but it is not the provider's own number. `duration_source` carries
|
|
14
|
+
* that distinction to every consumer: `NULL` is provider-reported, `'derived'` is this
|
|
15
|
+
* function's work. Nothing may present the two as the same measurement.
|
|
16
|
+
*/
|
|
17
|
+
/** `history_message.duration_source` values. `NULL` means provider-reported. */
|
|
18
|
+
export const DURATION_SOURCE_DERIVED = 'derived';
|
|
19
|
+
/**
|
|
20
|
+
* Deltas above this are session gaps, not work: the operator walked away and came back.
|
|
21
|
+
* Attributing an overnight gap to an assistant step would corrupt the very ranking this
|
|
22
|
+
* derivation exists to make usable, so those rows stay unmeasured.
|
|
23
|
+
*/
|
|
24
|
+
export const DERIVED_DURATION_CEILING_MS = 30 * 60 * 1000;
|
|
25
|
+
/**
|
|
26
|
+
* Fill `duration_ms` for assistant rows that have none, from the timestamp delta to the
|
|
27
|
+
* preceding record in the same session.
|
|
28
|
+
*
|
|
29
|
+
* Idempotent and additive: a row whose `duration_ms` is already set is never touched, so
|
|
30
|
+
* a provider measurement always wins over a derived one, and re-running after a later
|
|
31
|
+
* import only fills rows the earlier pass could not reach.
|
|
32
|
+
*/
|
|
33
|
+
export async function deriveAssistantDurations(db) {
|
|
34
|
+
const rows = await findAssistantDurationCandidates(db);
|
|
35
|
+
let derived = 0;
|
|
36
|
+
let skippedOverCeiling = 0;
|
|
37
|
+
for (const row of rows) {
|
|
38
|
+
// A non-positive delta means the records share a timestamp or arrived out of
|
|
39
|
+
// order; there is no duration to claim, so leave the row unmeasured rather than
|
|
40
|
+
// invent a zero (0680 R6: absent is not zero).
|
|
41
|
+
if (row.deltaMs === null || row.deltaMs <= 0)
|
|
42
|
+
continue;
|
|
43
|
+
if (row.deltaMs > DERIVED_DURATION_CEILING_MS) {
|
|
44
|
+
skippedOverCeiling += 1;
|
|
45
|
+
continue;
|
|
46
|
+
}
|
|
47
|
+
await updateAssistantDuration(db, row.recordHash, row.deltaMs, DURATION_SOURCE_DERIVED);
|
|
48
|
+
derived += 1;
|
|
49
|
+
}
|
|
50
|
+
return { derived, skippedOverCeiling };
|
|
51
|
+
}
|
package/dist/index.d.ts
CHANGED
|
@@ -1,11 +1,13 @@
|
|
|
1
|
+
export { DERIVED_DURATION_CEILING_MS, type DeriveAssistantDurationsResult, DURATION_SOURCE_DERIVED, deriveAssistantDurations, } from './assistant-duration';
|
|
1
2
|
export { HistoryImportError } from './errors';
|
|
2
3
|
export { sha256, stableJson } from './hash';
|
|
3
4
|
export { runJsonlImport } from './importer';
|
|
4
|
-
export { applyHistoryImportSchema, normalizeSourceFilePaths } from './jsonl-importer-dao';
|
|
5
|
+
export { applyHistoryImportSchema, normalizeSourceFilePaths, TYPED_HISTORY_TABLES } from './jsonl-importer-dao';
|
|
5
6
|
export { canonicalizeSkillName, extractSkillCalls, type SkillCallIdentity, sessionIdFromSourcePath, skillCallEntry, } from './mappers';
|
|
6
7
|
export { type OpenCodeImportOptions, runOpenCodeImport } from './opencode-importer';
|
|
7
8
|
export { DEFAULT_REDACTION_RULES, redactRecord, redactValue } from './redaction';
|
|
8
|
-
export { HISTORY_IMPORT_SCHEMA_SQL } from './schema-sql';
|
|
9
|
+
export { HISTORY_IMPORT_SCHEMA_SQL, HISTORY_IMPORT_SCHEMA_VERSION } from './schema-sql';
|
|
9
10
|
export { getSourceDefinition, resolveSourceDefinition, SOURCE_DEFINITIONS, VALID_TABLE_NAME, validateSourceDefinition, } from './sources';
|
|
11
|
+
export { BOOKKEEPING_HISTORY_TABLES, IMPORTER_OWNED_TABLES } from './tables';
|
|
10
12
|
export type { FieldTransform, ImportIssue, ImportMode, ImportOptions, ImportResult, JsonObject, LlmJsonlSource, ReconcileSummary, RedactionRule, SkillCall, SkillCallSplitRecord, SourceDefinition, SplitConfig, SplitEntry, TransformContext, } from './types';
|
|
11
13
|
//# sourceMappingURL=index.d.ts.map
|
package/dist/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,kBAAkB,EAAE,MAAM,UAAU,CAAC;AAC9C,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AAC5C,OAAO,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AAC5C,OAAO,EAAE,wBAAwB,EAAE,wBAAwB,EAAE,MAAM,sBAAsB,CAAC;
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EACH,2BAA2B,EAC3B,KAAK,8BAA8B,EACnC,uBAAuB,EACvB,wBAAwB,GAC3B,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EAAE,kBAAkB,EAAE,MAAM,UAAU,CAAC;AAC9C,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AAC5C,OAAO,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AAC5C,OAAO,EAAE,wBAAwB,EAAE,wBAAwB,EAAE,oBAAoB,EAAE,MAAM,sBAAsB,CAAC;AAChH,OAAO,EACH,qBAAqB,EACrB,iBAAiB,EACjB,KAAK,iBAAiB,EACtB,uBAAuB,EACvB,cAAc,GACjB,MAAM,WAAW,CAAC;AACnB,OAAO,EAAE,KAAK,qBAAqB,EAAE,iBAAiB,EAAE,MAAM,qBAAqB,CAAC;AACpF,OAAO,EAAE,uBAAuB,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AACjF,OAAO,EAAE,yBAAyB,EAAE,6BAA6B,EAAE,MAAM,cAAc,CAAC;AACxF,OAAO,EACH,mBAAmB,EACnB,uBAAuB,EACvB,kBAAkB,EAClB,gBAAgB,EAChB,wBAAwB,GAC3B,MAAM,WAAW,CAAC;AACnB,OAAO,EAAE,0BAA0B,EAAE,qBAAqB,EAAE,MAAM,UAAU,CAAC;AAC7E,YAAY,EACR,cAAc,EACd,WAAW,EACX,UAAU,EACV,aAAa,EACb,YAAY,EACZ,UAAU,EACV,cAAc,EACd,gBAAgB,EAChB,aAAa,EACb,SAAS,EACT,oBAAoB,EACpB,gBAAgB,EAChB,WAAW,EACX,UAAU,EACV,gBAAgB,GACnB,MAAM,SAAS,CAAC"}
|
package/dist/index.js
CHANGED
|
@@ -1,9 +1,11 @@
|
|
|
1
|
+
export { DERIVED_DURATION_CEILING_MS, DURATION_SOURCE_DERIVED, deriveAssistantDurations, } from './assistant-duration.js';
|
|
1
2
|
export { HistoryImportError } from './errors.js';
|
|
2
3
|
export { sha256, stableJson } from './hash.js';
|
|
3
4
|
export { runJsonlImport } from './importer.js';
|
|
4
|
-
export { applyHistoryImportSchema, normalizeSourceFilePaths } from './jsonl-importer-dao.js';
|
|
5
|
+
export { applyHistoryImportSchema, normalizeSourceFilePaths, TYPED_HISTORY_TABLES } from './jsonl-importer-dao.js';
|
|
5
6
|
export { canonicalizeSkillName, extractSkillCalls, sessionIdFromSourcePath, skillCallEntry, } from './mappers.js';
|
|
6
7
|
export { runOpenCodeImport } from './opencode-importer.js';
|
|
7
8
|
export { DEFAULT_REDACTION_RULES, redactRecord, redactValue } from './redaction.js';
|
|
8
|
-
export { HISTORY_IMPORT_SCHEMA_SQL } from './schema-sql.js';
|
|
9
|
+
export { HISTORY_IMPORT_SCHEMA_SQL, HISTORY_IMPORT_SCHEMA_VERSION } from './schema-sql.js';
|
|
9
10
|
export { getSourceDefinition, resolveSourceDefinition, SOURCE_DEFINITIONS, VALID_TABLE_NAME, validateSourceDefinition, } from './sources.js';
|
|
11
|
+
export { BOOKKEEPING_HISTORY_TABLES, IMPORTER_OWNED_TABLES } from './tables.js';
|
|
@@ -11,6 +11,12 @@ export interface SourceCheckpoint {
|
|
|
11
11
|
readonly size: number | null;
|
|
12
12
|
readonly mtimeMs: number | null;
|
|
13
13
|
}
|
|
14
|
+
/**
|
|
15
|
+
* Typed contract tables created and populated by the importer.
|
|
16
|
+
*
|
|
17
|
+
* Promoted to public surface per task 0749 (ADR-105).
|
|
18
|
+
*/
|
|
19
|
+
export declare const TYPED_HISTORY_TABLES: readonly ["history_message", "history_tool_call", "history_skill_call"];
|
|
14
20
|
declare function targetTableFor(table: string): string;
|
|
15
21
|
declare function timestamp(now: ImportOptions['now']): string;
|
|
16
22
|
declare function ETL_TABLE_DDL(table: string): string;
|
|
@@ -91,6 +97,19 @@ export declare function codexUsageAttributionUpdateOp(recordHash: string, tokens
|
|
|
91
97
|
* most 200 per `IN (...)` (task 0060 F9 — replaces the per-record SELECT loop).
|
|
92
98
|
*/
|
|
93
99
|
export declare function ledgerExistingHashes(db: ImportOptions['db'], hashes: readonly string[]): Promise<Set<string>>;
|
|
100
|
+
/** Candidate assistant-message row for duration derivation. */
|
|
101
|
+
export interface AssistantDurationCandidate {
|
|
102
|
+
readonly recordHash: string;
|
|
103
|
+
readonly deltaMs: number | null;
|
|
104
|
+
}
|
|
105
|
+
/**
|
|
106
|
+
* Fetch candidate assistant rows lacking duration_ms along with timestamp delta to preceding record.
|
|
107
|
+
*/
|
|
108
|
+
export declare function findAssistantDurationCandidates(db: DbAdapter): Promise<AssistantDurationCandidate[]>;
|
|
109
|
+
/**
|
|
110
|
+
* Update duration_ms and duration_source for a previously unmeasured assistant row.
|
|
111
|
+
*/
|
|
112
|
+
export declare function updateAssistantDuration(db: DbAdapter, recordHash: string, durationMs: number, durationSource: string): Promise<void>;
|
|
94
113
|
declare function insertRecord(db: ImportOptions['db'], targetTable: string, recordHash: string, sourceFile: string, sourceLine: number, splitIndex: number, payload: JsonObject, now: ImportOptions['now']): Promise<void>;
|
|
95
114
|
declare function insertLedger(db: ImportOptions['db'], recordHash: string, source: string, sourceFile: string, sourceLine: number, splitIndex: number, targetTable: string, now: ImportOptions['now']): Promise<void>;
|
|
96
115
|
/**
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"jsonl-importer-dao.d.ts","sourceRoot":"","sources":["../src/jsonl-importer-dao.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAI7D,OAAO,KAAK,EAAE,aAAa,EAAE,UAAU,EAAE,gBAAgB,EAAE,gBAAgB,EAAE,MAAM,SAAS,CAAC;AAE7F,UAAU,aAAa;IACnB,QAAQ,CAAC,kBAAkB,EAAE,MAAM,CAAC;IACpC,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACrC,QAAQ,CAAC,eAAe,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CAC5C;AAED,gEAAgE;AAChE,MAAM,WAAW,gBAAgB;IAC7B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,QAAQ,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;CACnC;
|
|
1
|
+
{"version":3,"file":"jsonl-importer-dao.d.ts","sourceRoot":"","sources":["../src/jsonl-importer-dao.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAI7D,OAAO,KAAK,EAAE,aAAa,EAAE,UAAU,EAAE,gBAAgB,EAAE,gBAAgB,EAAE,MAAM,SAAS,CAAC;AAE7F,UAAU,aAAa;IACnB,QAAQ,CAAC,kBAAkB,EAAE,MAAM,CAAC;IACpC,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACrC,QAAQ,CAAC,eAAe,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CAC5C;AAED,gEAAgE;AAChE,MAAM,WAAW,gBAAgB;IAC7B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,QAAQ,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;CACnC;AAED;;;;GAIG;AACH,eAAO,MAAM,oBAAoB,yEAA0E,CAAC;AA4E5G,iBAAS,cAAc,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAK7C;AAED,iBAAS,SAAS,CAAC,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,GAAG,MAAM,CAEpD;AAED,iBAAS,aAAa,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAS5C;AAED;;;;;;GAMG;AACH,wBAAsB,wBAAwB,CAAC,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,CAOrF;AAED,6FAA6F;AAC7F,iBAAe,iBAAiB,CAAC,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EAAE,WAAW,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAG5F;AAED;;;;;;;;;;GAUG;AACH,iBAAe,kBAAkB,CAAC,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EAAE,UAAU,EAAE,gBAAgB,GAAG,OAAO,CAAC,IAAI,CAAC,CAQtG;AAED,iBAAe,cAAc,CAAC,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAO1G;AAED;;;;;;;;GAQG;AACH,wBAAsB,qBAAqB,CACvC,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EACvB,MAAM,EAAE,MAAM,GACf,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,gBAAgB,CAAC,CAAC,CA0BxC;AAED,iBAAe,gBAAgB,CAAC,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,SAAS,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAIhH;AAED,iBAAe,eAAe,CAC1B,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EACvB,MAAM,EAAE,MAAM,EACd,UAAU,EAAE,MAAM,EAClB,IAAI,EAAE,MAAM,EACZ,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,GAC1B,OAAO,CAAC,IAAI,CAAC,CAYf;AAED,iBAAe,YAAY,CAAC,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EAAE,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAMzF;AAED;;;;;GAKG;AACH,wBAAgB,cAAc,CAC1B,WAAW,EAAE,MAAM,EACnB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,OAAO,EAAE,UAAU,EACnB,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,GAC1B,SAAS,CAmCX;AAED,mDAAmD;AACnD,wBAAgB,cAAc,CAC1B,UAAU,EAAE,MAAM,EAClB,MAAM,EAAE,MAAM,EACd,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,WAAW,EAAE,MAAM,EACnB,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,GAC1B,SAAS,CAMX;AAED,0FAA0F;AAC1F,wBAAgB,kBAAkB,CAC9B,MAAM,EAAE,MAAM,EACd,UAAU,EAAE,MAAM,EAClB,IAAI,EAAE,MAAM,EACZ,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,EACzB,QAAQ,CAAC,EAAE;IAAE,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;IAAC,QAAQ,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI,CAAA;CAAE,GAC7E,SAAS,CAWX;AAED;;;;;;;;GAQG;AACH,iBAAS,wBAAwB,CAC7B,UAAU,EAAE,MAAM,EAClB,SAAS,EAAE,MAAM,GAAG,IAAI,EACxB,WAAW,EAAE,MAAM,GAAG,IAAI,EAC1B,UAAU,EAAE,MAAM,GAAG,IAAI,GAC1B,SAAS,CAOX;AAED;;;GAGG;AACH,iBAAS,2BAA2B,CAAC,UAAU,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,GAAG,SAAS,CAOvF;AAED;2FAC2F;AAC3F,wBAAgB,6BAA6B,CACzC,UAAU,EAAE,MAAM,EAClB,MAAM,EAAE;IAAE,QAAQ,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IAAC,QAAQ,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IAAC,QAAQ,CAAC,SAAS,EAAE,MAAM,GAAG,IAAI,CAAA;CAAE,GAC7G,SAAS,CAOX;AAED;;;GAGG;AACH,wBAAsB,oBAAoB,CAAC,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,SAAS,MAAM,EAAE,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAYnH;AAED,+DAA+D;AAC/D,MAAM,WAAW,0BAA0B;IACvC,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;CACnC;AAED;;GAEG;AACH,wBAAsB,+BAA+B,CAAC,EAAE,EAAE,SAAS,GAAG,OAAO,CAAC,0BAA0B,EAAE,CAAC,CAiB1G;AAED;;GAEG;AACH,wBAAsB,uBAAuB,CACzC,EAAE,EAAE,SAAS,EACb,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,cAAc,EAAE,MAAM,GACvB,OAAO,CAAC,IAAI,CAAC,CAOf;AAED,iBAAe,YAAY,CACvB,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EACvB,WAAW,EAAE,MAAM,EACnB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,OAAO,EAAE,UAAU,EACnB,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,GAC1B,OAAO,CAAC,IAAI,CAAC,CAIf;AAED,iBAAe,YAAY,CACvB,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EACvB,UAAU,EAAE,MAAM,EAClB,MAAM,EAAE,MAAM,EACd,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,WAAW,EAAE,MAAM,EACnB,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,GAC1B,OAAO,CAAC,IAAI,CAAC,CAYf;AAED;;;;;;;;;;;;;;;;;;;GAmBG;AACH,wBAAsB,wBAAwB,CAC1C,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EACvB,eAAe,EAAE,CAAC,UAAU,EAAE,MAAM,KAAK,MAAM,GAAG,IAAI,GAAG,SAAS,GACnE,OAAO,CAAC,IAAI,CAAC,CAWf;AA4GD;;;;;;;;;;;GAWG;AACH,iBAAe,mBAAmB,CAC9B,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EACvB,MAAM,EAAE,MAAM,EACd,aAAa,EAAE,WAAW,CAAC,MAAM,CAAC,EAClC,eAAe,EAAE,SAAS,MAAM,EAAE,EAClC,MAAM,EAAE,OAAO,GAChB,OAAO,CAAC,gBAAgB,CAAC,CA6C3B;AAID,4DAA4D;AAC5D,MAAM,WAAW,kBAAkB;IAC/B,EAAE,EAAE,MAAM,CAAC;IACX,UAAU,EAAE,MAAM,CAAC;IACnB,YAAY,EAAE,MAAM,CAAC;IACrB,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,MAAM,CAAC;CACrB;AAED,iEAAiE;AACjE,MAAM,WAAW,eAAe;IAC5B,EAAE,EAAE,MAAM,CAAC;IACX,UAAU,EAAE,MAAM,CAAC;IACnB,YAAY,EAAE,MAAM,CAAC;IACrB,IAAI,EAAE,MAAM,CAAC;CAChB;AAED,iDAAiD;AACjD,MAAM,WAAW,qBAAqB;IAClC,WAAW,EAAE,MAAM,CAAC;IACpB,YAAY,EAAE,MAAM,CAAC;CACxB;AAED,gEAAgE;AAChE,MAAM,WAAW,mBAAmB;IAChC,WAAW,EAAE,iBAAiB,GAAG,mBAAmB,GAAG,oBAAoB,CAAC;IAC5E,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,EAAE,UAAU,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;CACtB;AAED,4EAA4E;AAC5E,wBAAsB,oBAAoB,CACtC,EAAE,EAAE,SAAS,EACb,QAAQ,EAAE,MAAM,EAChB,MAAM,EAAE,MAAM,EACd,KAAK,EAAE,MAAM,GACd,OAAO,CAAC,kBAAkB,EAAE,CAAC,CAa/B;AAED,qEAAqE;AACrE,wBAAsB,iBAAiB,CAAC,EAAE,EAAE,SAAS,EAAE,UAAU,EAAE,SAAS,MAAM,EAAE,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC,CAShH;AAED,0EAA0E;AAC1E,wBAAsB,2BAA2B,CAAC,EAAE,EAAE,SAAS,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,qBAAqB,EAAE,CAAC,CAAC,CAY9G;AAED,2EAA2E;AAC3E,wBAAgB,wBAAwB,CAAC,OAAO,EAAE,SAAS,qBAAqB,EAAE,GAAG,SAAS,EAAE,CAW/F;AAED,6DAA6D;AAC7D,wBAAgB,yBAAyB,CAAC,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,SAAS,CAKvF;AAED,iFAAiF;AACjF,wBAAgB,2BAA2B,CACvC,OAAO,EAAE,SAAS,mBAAmB,EAAE,EACvC,eAAe,EAAE,SAAS,MAAM,EAAE,EAClC,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,GAC1B,SAAS,EAAE,CA0Db;AAED,YAAY,EAAE,aAAa,EAAE,CAAC;AAC9B,OAAO,EACH,aAAa,EACb,iBAAiB,EACjB,kBAAkB,EAClB,YAAY,EACZ,YAAY,EACZ,YAAY,EACZ,cAAc,EACd,mBAAmB,EACnB,gBAAgB,EAChB,cAAc,EACd,SAAS,EACT,wBAAwB,EACxB,2BAA2B,EAC3B,eAAe,GAClB,CAAC"}
|
|
@@ -1,6 +1,12 @@
|
|
|
1
1
|
import { HistoryImportError } from './errors.js';
|
|
2
2
|
import { HISTORY_IMPORT_SCHEMA_SQL } from './schema-sql.js';
|
|
3
3
|
import { VALID_TABLE_NAME } from './sources.js';
|
|
4
|
+
/**
|
|
5
|
+
* Typed contract tables created and populated by the importer.
|
|
6
|
+
*
|
|
7
|
+
* Promoted to public surface per task 0749 (ADR-105).
|
|
8
|
+
*/
|
|
9
|
+
export const TYPED_HISTORY_TABLES = ['history_message', 'history_tool_call', 'history_skill_call'];
|
|
4
10
|
/** Column allowlist per typed contract table; order is the INSERT column order. */
|
|
5
11
|
const TYPED_TABLE_COLUMNS = {
|
|
6
12
|
history_message: [
|
|
@@ -299,6 +305,32 @@ export async function ledgerExistingHashes(db, hashes) {
|
|
|
299
305
|
}
|
|
300
306
|
return existing;
|
|
301
307
|
}
|
|
308
|
+
/**
|
|
309
|
+
* Fetch candidate assistant rows lacking duration_ms along with timestamp delta to preceding record.
|
|
310
|
+
*/
|
|
311
|
+
export async function findAssistantDurationCandidates(db) {
|
|
312
|
+
const candidateSql = `
|
|
313
|
+
WITH ordered AS (
|
|
314
|
+
SELECT record_hash,
|
|
315
|
+
role,
|
|
316
|
+
duration_ms,
|
|
317
|
+
ts,
|
|
318
|
+
LAG(ts) OVER (PARTITION BY source, session_id ORDER BY seq) AS prev_ts
|
|
319
|
+
FROM history_message
|
|
320
|
+
WHERE ts IS NOT NULL AND ts LIKE '____-__-__T%'
|
|
321
|
+
)
|
|
322
|
+
SELECT record_hash AS recordHash,
|
|
323
|
+
CAST(ROUND((unixepoch(ts, 'subsec') - unixepoch(prev_ts, 'subsec')) * 1000) AS INTEGER) AS deltaMs
|
|
324
|
+
FROM ordered
|
|
325
|
+
WHERE role = 'assistant' AND duration_ms IS NULL AND prev_ts IS NOT NULL`;
|
|
326
|
+
return db.queryAll(candidateSql);
|
|
327
|
+
}
|
|
328
|
+
/**
|
|
329
|
+
* Update duration_ms and duration_source for a previously unmeasured assistant row.
|
|
330
|
+
*/
|
|
331
|
+
export async function updateAssistantDuration(db, recordHash, durationMs, durationSource) {
|
|
332
|
+
await db.run('UPDATE history_message SET duration_ms = ?, duration_source = ? WHERE record_hash = ? AND duration_ms IS NULL', durationMs, durationSource, recordHash);
|
|
333
|
+
}
|
|
302
334
|
async function insertRecord(db, targetTable, recordHash, sourceFile, sourceLine, splitIndex, payload, now) {
|
|
303
335
|
const op = recordInsertOp(targetTable, recordHash, sourceFile, sourceLine, splitIndex, payload, now);
|
|
304
336
|
await ensureTargetTable(db, targetTable);
|
package/dist/schema-sql.d.ts
CHANGED
|
@@ -1,3 +1,10 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Schema version of the history import DDL.
|
|
3
|
+
*
|
|
4
|
+
* Guaranteed to match the package version; a bump-or-fail test enforces that
|
|
5
|
+
* any modification to HISTORY_IMPORT_SCHEMA_SQL must be accompanied by a version bump.
|
|
6
|
+
*/
|
|
7
|
+
export declare const HISTORY_IMPORT_SCHEMA_VERSION = "0.4.55";
|
|
1
8
|
/**
|
|
2
9
|
* DDL string that creates the history import checkpoint, ledger, and typed contract tables.
|
|
3
10
|
*
|
package/dist/schema-sql.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"schema-sql.d.ts","sourceRoot":"","sources":["../src/schema-sql.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AACH,eAAO,MAAM,yBAAyB,
|
|
1
|
+
{"version":3,"file":"schema-sql.d.ts","sourceRoot":"","sources":["../src/schema-sql.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AACH,eAAO,MAAM,6BAA6B,WAAW,CAAC;AAEtD;;;;;GAKG;AACH,eAAO,MAAM,yBAAyB,QA+G9B,CAAC"}
|
package/dist/schema-sql.js
CHANGED
|
@@ -1,3 +1,10 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Schema version of the history import DDL.
|
|
3
|
+
*
|
|
4
|
+
* Guaranteed to match the package version; a bump-or-fail test enforces that
|
|
5
|
+
* any modification to HISTORY_IMPORT_SCHEMA_SQL must be accompanied by a version bump.
|
|
6
|
+
*/
|
|
7
|
+
export const HISTORY_IMPORT_SCHEMA_VERSION = '0.4.55';
|
|
1
8
|
/**
|
|
2
9
|
* DDL string that creates the history import checkpoint, ledger, and typed contract tables.
|
|
3
10
|
*
|
|
@@ -39,6 +46,7 @@ CREATE TABLE IF NOT EXISTS history_message (
|
|
|
39
46
|
disposition TEXT NOT NULL,
|
|
40
47
|
ts TEXT,
|
|
41
48
|
duration_ms INTEGER,
|
|
49
|
+
duration_source TEXT,
|
|
42
50
|
model TEXT,
|
|
43
51
|
input_tokens INTEGER,
|
|
44
52
|
output_tokens INTEGER,
|
package/dist/tables.d.ts
ADDED
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
/** Bookkeeping tables owned and maintained by the importer. */
|
|
2
|
+
export declare const BOOKKEEPING_HISTORY_TABLES: readonly ["history_import_checkpoint", "history_import_ledger"];
|
|
3
|
+
/**
|
|
4
|
+
* Complete list of tables owned by the importer:
|
|
5
|
+
* - 3 typed contract tables (history_message, history_tool_call, history_skill_call)
|
|
6
|
+
* - 2 bookkeeping tables (history_import_checkpoint, history_import_ledger)
|
|
7
|
+
* - 10 per-source raw ETL landing tables (history_etl_<source>)
|
|
8
|
+
*
|
|
9
|
+
* Total: 15 tables.
|
|
10
|
+
* Sourced dynamically from static declarations so adding a source upstream automatically
|
|
11
|
+
* includes its landing table without downstream duplication.
|
|
12
|
+
*/
|
|
13
|
+
export declare const IMPORTER_OWNED_TABLES: readonly string[];
|
|
14
|
+
//# sourceMappingURL=tables.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"tables.d.ts","sourceRoot":"","sources":["../src/tables.ts"],"names":[],"mappings":"AAGA,+DAA+D;AAC/D,eAAO,MAAM,0BAA0B,iEAAkE,CAAC;AAE1G;;;;;;;;;GASG;AACH,eAAO,MAAM,qBAAqB,EAAE,SAAS,MAAM,EAIlD,CAAC"}
|
package/dist/tables.js
ADDED
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
import { TYPED_HISTORY_TABLES } from './jsonl-importer-dao.js';
|
|
2
|
+
import { SOURCE_DEFINITIONS } from './sources.js';
|
|
3
|
+
/** Bookkeeping tables owned and maintained by the importer. */
|
|
4
|
+
export const BOOKKEEPING_HISTORY_TABLES = ['history_import_checkpoint', 'history_import_ledger'];
|
|
5
|
+
/**
|
|
6
|
+
* Complete list of tables owned by the importer:
|
|
7
|
+
* - 3 typed contract tables (history_message, history_tool_call, history_skill_call)
|
|
8
|
+
* - 2 bookkeeping tables (history_import_checkpoint, history_import_ledger)
|
|
9
|
+
* - 10 per-source raw ETL landing tables (history_etl_<source>)
|
|
10
|
+
*
|
|
11
|
+
* Total: 15 tables.
|
|
12
|
+
* Sourced dynamically from static declarations so adding a source upstream automatically
|
|
13
|
+
* includes its landing table without downstream duplication.
|
|
14
|
+
*/
|
|
15
|
+
export const IMPORTER_OWNED_TABLES = [
|
|
16
|
+
...TYPED_HISTORY_TABLES,
|
|
17
|
+
...BOOKKEEPING_HISTORY_TABLES,
|
|
18
|
+
...Object.values(SOURCE_DEFINITIONS).map((def) => def.targetTable),
|
|
19
|
+
];
|
package/dist/types.d.ts
CHANGED
|
@@ -149,6 +149,7 @@ export interface SkillCallSplitRecord {
|
|
|
149
149
|
readonly completed_at?: string | null;
|
|
150
150
|
readonly duration_ms?: number | null;
|
|
151
151
|
}
|
|
152
|
+
/** Normalized skill-call record imported into history_skill_call. */
|
|
152
153
|
export interface SkillCall {
|
|
153
154
|
readonly record_hash: string;
|
|
154
155
|
readonly message_hash: string;
|
package/dist/types.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAClD,OAAO,KAAK,EAAE,UAAU,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAC;AACtE,OAAO,KAAK,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAE7B,6DAA6D;AAC7D,MAAM,MAAM,cAAc,GACpB,IAAI,GACJ,QAAQ,GACR,OAAO,GACP,QAAQ,GACR,UAAU,GACV,aAAa,GACb,UAAU,GACV,KAAK,GACL,MAAM,GACN,KAAK,CAAC;AAEZ,mDAAmD;AACnD,MAAM,MAAM,UAAU,GAAG,MAAM,GAAG,aAAa,GAAG,YAAY,CAAC;AAE/D,gFAAgF;AAChF,MAAM,WAAW,UAAU;IACvB,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,MAAM,EAAE,UAAU,CAAC;CAC/B;AAED,kFAAkF;AAClF,MAAM,MAAM,WAAW,GACjB;IACI,QAAQ,CAAC,IAAI,EAAE,YAAY,CAAC;CAC/B,GACD;IACI,QAAQ,CAAC,IAAI,EAAE,aAAa,CAAC;IAC7B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,CAAC;CACjC,GACD;IACI,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAC;IACxB,QAAQ,CAAC,KAAK,EAAE,CAAC,GAAG,EAAE,UAAU,EAAE,OAAO,CAAC,EAAE,gBAAgB,KAAK,SAAS,CAAC,UAAU,GAAG,UAAU,CAAC,EAAE,CAAC;IACtG,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,CAAC;CACjC,CAAC;AAER,4DAA4D;AAC5D,MAAM,MAAM,UAAU,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;AAEjD,sDAAsD;AACtD,MAAM,MAAM,cAAc,GAAG,CAAC,KAAK,EAAE,OAAO,EAAE,GAAG,EAAE,UAAU,EAAE,OAAO,EAAE,gBAAgB,KAAK,OAAO,CAAC;AAErG,0DAA0D;AAC1D,MAAM,WAAW,gBAAgB;IAC7B,2FAA2F;IAC3F,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;CAC/B;AAED,qEAAqE;AACrE,MAAM,WAAW,gBAAgB,CAAC,OAAO,SAAS,UAAU,GAAG,UAAU;IACrE,iHAAiH;IACjH,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,YAAY,EAAE,SAAS,MAAM,EAAE,CAAC;IACzC,QAAQ,CAAC,YAAY,EAAE,SAAS,MAAM,EAAE,CAAC;IACzC;;;;;OAKG;IACH,QAAQ,CAAC,iBAAiB,CAAC,EAAE,OAAO,GAAG,MAAM,CAAC;IAC9C,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,WAAW,EAAE,WAAW,CAAC;IAClC,QAAQ,CAAC,QAAQ,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;IACpD,QAAQ,CAAC,eAAe,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,cAAc,CAAC,CAAC,CAAC;IACnE,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;CACvC;AAED,6DAA6D;AAC7D,MAAM,WAAW,aAAa;IAC1B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;CAChC;AAED,oCAAoC;AACpC,MAAM,WAAW,aAAa;IAC1B,QAAQ,CAAC,EAAE,EAAE,SAAS,CAAC;IACvB,QAAQ,CAAC,UAAU,CAAC,EAAE,UAAU,CAAC;IACjC,QAAQ,CAAC,IAAI,CAAC,EAAE,UAAU,CAAC;IAC3B,QAAQ,CAAC,KAAK,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IACnC,QAAQ,CAAC,KAAK,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IACnC,QAAQ,CAAC,MAAM,CAAC,EAAE,OAAO,CAAC;IAC1B,QAAQ,CAAC,cAAc,CAAC,EAAE,SAAS,aAAa,EAAE,CAAC;IACnD,QAAQ,CAAC,GAAG,CAAC,EAAE,MAAM,IAAI,CAAC;IAC1B;;;;OAIG;IACH,QAAQ,CAAC,KAAK,CAAC,EAAE,YAAY,CAAC;CACjC;AAED,mFAAmF;AACnF,MAAM,WAAW,WAAW;IACxB,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;CAC3B;AAED;;;;;GAKG;AACH,MAAM,WAAW,gBAAgB;IAC7B,uGAAuG;IACvG,QAAQ,CAAC,eAAe,EAAE,MAAM,CAAC;IACjC,iDAAiD;IACjD,QAAQ,CAAC,eAAe,EAAE,MAAM,CAAC;IACjC,qEAAqE;IACrE,QAAQ,CAAC,mBAAmB,EAAE,MAAM,CAAC;CACxC;AAED,yDAAyD;AACzD,MAAM,WAAW,YAAY;IACzB,iGAAiG;IACjG,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,IAAI,EAAE,UAAU,CAAC;IAC1B,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC,QAAQ,CAAC,eAAe,EAAE,MAAM,CAAC;IACjC,QAAQ,CAAC,iBAAiB,EAAE,MAAM,CAAC;IACnC,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC,oFAAoF;IACpF,QAAQ,CAAC,mBAAmB,EAAE,MAAM,CAAC;IACrC,4FAA4F;IAC5F,QAAQ,CAAC,qBAAqB,EAAE,MAAM,CAAC;IACvC,QAAQ,CAAC,WAAW,EAAE,SAAS,WAAW,EAAE,CAAC;IAC7C,QAAQ,CAAC,gBAAgB,EAAE,SAAS,WAAW,EAAE,CAAC;IAClD,QAAQ,CAAC,iBAAiB,EAAE,MAAM,CAAC;IACnC;;;OAGG;IACH,QAAQ,CAAC,cAAc,CAAC,EAAE,gBAAgB,CAAC;CAC9C;AAED;;;;;;;GAOG;AACH;;;;;GAKG;AACH,MAAM,WAAW,oBAAoB;IACjC,QAAQ,CAAC,kBAAkB,CAAC,EAAE,MAAM,CAAC;IACrC,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,eAAe,EAAE,MAAM,GAAG,OAAO,CAAC;IAC3C,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACpC,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAClC,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACrC,QAAQ,CAAC,OAAO,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACjC,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAChC,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACpC,QAAQ,CAAC,YAAY,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACtC,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CACxC;AAED,MAAM,WAAW,SAAS;IACtB,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,eAAe,EAAE,MAAM,GAAG,OAAO,CAAC;IAC3C,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACpC,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAClC,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACrC,QAAQ,CAAC,OAAO,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACjC,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAChC,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACpC,QAAQ,CAAC,YAAY,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACtC,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACrC,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;CAChC"}
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAClD,OAAO,KAAK,EAAE,UAAU,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAC;AACtE,OAAO,KAAK,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAE7B,6DAA6D;AAC7D,MAAM,MAAM,cAAc,GACpB,IAAI,GACJ,QAAQ,GACR,OAAO,GACP,QAAQ,GACR,UAAU,GACV,aAAa,GACb,UAAU,GACV,KAAK,GACL,MAAM,GACN,KAAK,CAAC;AAEZ,mDAAmD;AACnD,MAAM,MAAM,UAAU,GAAG,MAAM,GAAG,aAAa,GAAG,YAAY,CAAC;AAE/D,gFAAgF;AAChF,MAAM,WAAW,UAAU;IACvB,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,MAAM,EAAE,UAAU,CAAC;CAC/B;AAED,kFAAkF;AAClF,MAAM,MAAM,WAAW,GACjB;IACI,QAAQ,CAAC,IAAI,EAAE,YAAY,CAAC;CAC/B,GACD;IACI,QAAQ,CAAC,IAAI,EAAE,aAAa,CAAC;IAC7B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,CAAC;CACjC,GACD;IACI,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAC;IACxB,QAAQ,CAAC,KAAK,EAAE,CAAC,GAAG,EAAE,UAAU,EAAE,OAAO,CAAC,EAAE,gBAAgB,KAAK,SAAS,CAAC,UAAU,GAAG,UAAU,CAAC,EAAE,CAAC;IACtG,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,CAAC;CACjC,CAAC;AAER,4DAA4D;AAC5D,MAAM,MAAM,UAAU,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;AAEjD,sDAAsD;AACtD,MAAM,MAAM,cAAc,GAAG,CAAC,KAAK,EAAE,OAAO,EAAE,GAAG,EAAE,UAAU,EAAE,OAAO,EAAE,gBAAgB,KAAK,OAAO,CAAC;AAErG,0DAA0D;AAC1D,MAAM,WAAW,gBAAgB;IAC7B,2FAA2F;IAC3F,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;CAC/B;AAED,qEAAqE;AACrE,MAAM,WAAW,gBAAgB,CAAC,OAAO,SAAS,UAAU,GAAG,UAAU;IACrE,iHAAiH;IACjH,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,YAAY,EAAE,SAAS,MAAM,EAAE,CAAC;IACzC,QAAQ,CAAC,YAAY,EAAE,SAAS,MAAM,EAAE,CAAC;IACzC;;;;;OAKG;IACH,QAAQ,CAAC,iBAAiB,CAAC,EAAE,OAAO,GAAG,MAAM,CAAC;IAC9C,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,WAAW,EAAE,WAAW,CAAC;IAClC,QAAQ,CAAC,QAAQ,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;IACpD,QAAQ,CAAC,eAAe,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,cAAc,CAAC,CAAC,CAAC;IACnE,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;CACvC;AAED,6DAA6D;AAC7D,MAAM,WAAW,aAAa;IAC1B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;CAChC;AAED,oCAAoC;AACpC,MAAM,WAAW,aAAa;IAC1B,QAAQ,CAAC,EAAE,EAAE,SAAS,CAAC;IACvB,QAAQ,CAAC,UAAU,CAAC,EAAE,UAAU,CAAC;IACjC,QAAQ,CAAC,IAAI,CAAC,EAAE,UAAU,CAAC;IAC3B,QAAQ,CAAC,KAAK,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IACnC,QAAQ,CAAC,KAAK,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IACnC,QAAQ,CAAC,MAAM,CAAC,EAAE,OAAO,CAAC;IAC1B,QAAQ,CAAC,cAAc,CAAC,EAAE,SAAS,aAAa,EAAE,CAAC;IACnD,QAAQ,CAAC,GAAG,CAAC,EAAE,MAAM,IAAI,CAAC;IAC1B;;;;OAIG;IACH,QAAQ,CAAC,KAAK,CAAC,EAAE,YAAY,CAAC;CACjC;AAED,mFAAmF;AACnF,MAAM,WAAW,WAAW;IACxB,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;CAC3B;AAED;;;;;GAKG;AACH,MAAM,WAAW,gBAAgB;IAC7B,uGAAuG;IACvG,QAAQ,CAAC,eAAe,EAAE,MAAM,CAAC;IACjC,iDAAiD;IACjD,QAAQ,CAAC,eAAe,EAAE,MAAM,CAAC;IACjC,qEAAqE;IACrE,QAAQ,CAAC,mBAAmB,EAAE,MAAM,CAAC;CACxC;AAED,yDAAyD;AACzD,MAAM,WAAW,YAAY;IACzB,iGAAiG;IACjG,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,IAAI,EAAE,UAAU,CAAC;IAC1B,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC,QAAQ,CAAC,eAAe,EAAE,MAAM,CAAC;IACjC,QAAQ,CAAC,iBAAiB,EAAE,MAAM,CAAC;IACnC,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC,oFAAoF;IACpF,QAAQ,CAAC,mBAAmB,EAAE,MAAM,CAAC;IACrC,4FAA4F;IAC5F,QAAQ,CAAC,qBAAqB,EAAE,MAAM,CAAC;IACvC,QAAQ,CAAC,WAAW,EAAE,SAAS,WAAW,EAAE,CAAC;IAC7C,QAAQ,CAAC,gBAAgB,EAAE,SAAS,WAAW,EAAE,CAAC;IAClD,QAAQ,CAAC,iBAAiB,EAAE,MAAM,CAAC;IACnC;;;OAGG;IACH,QAAQ,CAAC,cAAc,CAAC,EAAE,gBAAgB,CAAC;CAC9C;AAED;;;;;;;GAOG;AACH;;;;;GAKG;AACH,MAAM,WAAW,oBAAoB;IACjC,QAAQ,CAAC,kBAAkB,CAAC,EAAE,MAAM,CAAC;IACrC,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,eAAe,EAAE,MAAM,GAAG,OAAO,CAAC;IAC3C,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACpC,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAClC,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACrC,QAAQ,CAAC,OAAO,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACjC,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAChC,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACpC,QAAQ,CAAC,YAAY,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACtC,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CACxC;AAED,qEAAqE;AACrE,MAAM,WAAW,SAAS;IACtB,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,eAAe,EAAE,MAAM,GAAG,OAAO,CAAC;IAC3C,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACpC,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAClC,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACrC,QAAQ,CAAC,OAAO,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACjC,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAChC,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACpC,QAAQ,CAAC,YAAY,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACtC,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACrC,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;CAChC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@gobing-ai/ts-llm-jsonl-importer",
|
|
3
|
-
"version": "0.4.
|
|
3
|
+
"version": "0.4.55",
|
|
4
4
|
"description": "@gobing-ai/ts-llm-jsonl-importer — Generic JSONL importer for LLM agent history files.",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"typescript",
|
|
@@ -47,9 +47,9 @@
|
|
|
47
47
|
"release": "echo 'Manual publish is disabled. Releases go through GitHub Actions via Trusted Publishing — push a tag: git tag @gobing-ai/ts-llm-jsonl-importer-v<version> && git push --tags' && exit 1"
|
|
48
48
|
},
|
|
49
49
|
"dependencies": {
|
|
50
|
-
"@gobing-ai/ts-db": "^0.4.
|
|
51
|
-
"@gobing-ai/ts-runtime": "^0.4.
|
|
52
|
-
"@gobing-ai/ts-utils": "^0.4.
|
|
50
|
+
"@gobing-ai/ts-db": "^0.4.55",
|
|
51
|
+
"@gobing-ai/ts-runtime": "^0.4.55",
|
|
52
|
+
"@gobing-ai/ts-utils": "^0.4.55",
|
|
53
53
|
"zod": "^4.1.0"
|
|
54
54
|
},
|
|
55
55
|
"devDependencies": {
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
import type { DbAdapter } from '@gobing-ai/ts-db';
|
|
2
|
+
import { findAssistantDurationCandidates, updateAssistantDuration } from './jsonl-importer-dao';
|
|
3
|
+
|
|
4
|
+
/**
|
|
5
|
+
* ETL-derived assistant-step duration (repatriated upstream from Spur task 0747 / ADR-105).
|
|
6
|
+
*
|
|
7
|
+
* Four of six sources — `claude`, `pi`, `codex`, `agy` — write no per-step
|
|
8
|
+
* `duration_ms` at all, so bottleneck ranking and per-model latency were unusable for
|
|
9
|
+
* the two busiest of them and ~73% of the measured span could not be attributed to
|
|
10
|
+
* llm/tool/idle. The transcripts carry a per-record `ts`, so the delta from the
|
|
11
|
+
* preceding record in the same session is derivable here.
|
|
12
|
+
*
|
|
13
|
+
* **It is an approximation and is labelled as one.** The delta includes queue and
|
|
14
|
+
* network time, which is arguably what "assistant step duration" should mean for
|
|
15
|
+
* bottleneck ranking, but it is not the provider's own number. `duration_source` carries
|
|
16
|
+
* that distinction to every consumer: `NULL` is provider-reported, `'derived'` is this
|
|
17
|
+
* function's work. Nothing may present the two as the same measurement.
|
|
18
|
+
*/
|
|
19
|
+
|
|
20
|
+
/** `history_message.duration_source` values. `NULL` means provider-reported. */
|
|
21
|
+
export const DURATION_SOURCE_DERIVED = 'derived';
|
|
22
|
+
|
|
23
|
+
/**
|
|
24
|
+
* Deltas above this are session gaps, not work: the operator walked away and came back.
|
|
25
|
+
* Attributing an overnight gap to an assistant step would corrupt the very ranking this
|
|
26
|
+
* derivation exists to make usable, so those rows stay unmeasured.
|
|
27
|
+
*/
|
|
28
|
+
export const DERIVED_DURATION_CEILING_MS = 30 * 60 * 1000;
|
|
29
|
+
|
|
30
|
+
/** What one derivation pass changed, and what it deliberately left alone. */
|
|
31
|
+
export interface DeriveAssistantDurationsResult {
|
|
32
|
+
/** Rows that gained a derived `duration_ms` on this pass. */
|
|
33
|
+
derived: number;
|
|
34
|
+
/** Candidate rows left unmeasured because the delta exceeded the ceiling. */
|
|
35
|
+
skippedOverCeiling: number;
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
/**
|
|
39
|
+
* Fill `duration_ms` for assistant rows that have none, from the timestamp delta to the
|
|
40
|
+
* preceding record in the same session.
|
|
41
|
+
*
|
|
42
|
+
* Idempotent and additive: a row whose `duration_ms` is already set is never touched, so
|
|
43
|
+
* a provider measurement always wins over a derived one, and re-running after a later
|
|
44
|
+
* import only fills rows the earlier pass could not reach.
|
|
45
|
+
*/
|
|
46
|
+
export async function deriveAssistantDurations(db: DbAdapter): Promise<DeriveAssistantDurationsResult> {
|
|
47
|
+
const rows = await findAssistantDurationCandidates(db);
|
|
48
|
+
|
|
49
|
+
let derived = 0;
|
|
50
|
+
let skippedOverCeiling = 0;
|
|
51
|
+
for (const row of rows) {
|
|
52
|
+
// A non-positive delta means the records share a timestamp or arrived out of
|
|
53
|
+
// order; there is no duration to claim, so leave the row unmeasured rather than
|
|
54
|
+
// invent a zero (0680 R6: absent is not zero).
|
|
55
|
+
if (row.deltaMs === null || row.deltaMs <= 0) continue;
|
|
56
|
+
if (row.deltaMs > DERIVED_DURATION_CEILING_MS) {
|
|
57
|
+
skippedOverCeiling += 1;
|
|
58
|
+
continue;
|
|
59
|
+
}
|
|
60
|
+
await updateAssistantDuration(db, row.recordHash, row.deltaMs, DURATION_SOURCE_DERIVED);
|
|
61
|
+
derived += 1;
|
|
62
|
+
}
|
|
63
|
+
return { derived, skippedOverCeiling };
|
|
64
|
+
}
|
package/src/index.ts
CHANGED
|
@@ -1,7 +1,13 @@
|
|
|
1
|
+
export {
|
|
2
|
+
DERIVED_DURATION_CEILING_MS,
|
|
3
|
+
type DeriveAssistantDurationsResult,
|
|
4
|
+
DURATION_SOURCE_DERIVED,
|
|
5
|
+
deriveAssistantDurations,
|
|
6
|
+
} from './assistant-duration';
|
|
1
7
|
export { HistoryImportError } from './errors';
|
|
2
8
|
export { sha256, stableJson } from './hash';
|
|
3
9
|
export { runJsonlImport } from './importer';
|
|
4
|
-
export { applyHistoryImportSchema, normalizeSourceFilePaths } from './jsonl-importer-dao';
|
|
10
|
+
export { applyHistoryImportSchema, normalizeSourceFilePaths, TYPED_HISTORY_TABLES } from './jsonl-importer-dao';
|
|
5
11
|
export {
|
|
6
12
|
canonicalizeSkillName,
|
|
7
13
|
extractSkillCalls,
|
|
@@ -11,7 +17,7 @@ export {
|
|
|
11
17
|
} from './mappers';
|
|
12
18
|
export { type OpenCodeImportOptions, runOpenCodeImport } from './opencode-importer';
|
|
13
19
|
export { DEFAULT_REDACTION_RULES, redactRecord, redactValue } from './redaction';
|
|
14
|
-
export { HISTORY_IMPORT_SCHEMA_SQL } from './schema-sql';
|
|
20
|
+
export { HISTORY_IMPORT_SCHEMA_SQL, HISTORY_IMPORT_SCHEMA_VERSION } from './schema-sql';
|
|
15
21
|
export {
|
|
16
22
|
getSourceDefinition,
|
|
17
23
|
resolveSourceDefinition,
|
|
@@ -19,6 +25,7 @@ export {
|
|
|
19
25
|
VALID_TABLE_NAME,
|
|
20
26
|
validateSourceDefinition,
|
|
21
27
|
} from './sources';
|
|
28
|
+
export { BOOKKEEPING_HISTORY_TABLES, IMPORTER_OWNED_TABLES } from './tables';
|
|
22
29
|
export type {
|
|
23
30
|
FieldTransform,
|
|
24
31
|
ImportIssue,
|
|
@@ -17,6 +17,13 @@ export interface SourceCheckpoint {
|
|
|
17
17
|
readonly mtimeMs: number | null;
|
|
18
18
|
}
|
|
19
19
|
|
|
20
|
+
/**
|
|
21
|
+
* Typed contract tables created and populated by the importer.
|
|
22
|
+
*
|
|
23
|
+
* Promoted to public surface per task 0749 (ADR-105).
|
|
24
|
+
*/
|
|
25
|
+
export const TYPED_HISTORY_TABLES = ['history_message', 'history_tool_call', 'history_skill_call'] as const;
|
|
26
|
+
|
|
20
27
|
/** Column allowlist per typed contract table; order is the INSERT column order. */
|
|
21
28
|
const TYPED_TABLE_COLUMNS: Readonly<Record<string, readonly string[]>> = {
|
|
22
29
|
history_message: [
|
|
@@ -395,6 +402,51 @@ export async function ledgerExistingHashes(db: ImportOptions['db'], hashes: read
|
|
|
395
402
|
return existing;
|
|
396
403
|
}
|
|
397
404
|
|
|
405
|
+
/** Candidate assistant-message row for duration derivation. */
|
|
406
|
+
export interface AssistantDurationCandidate {
|
|
407
|
+
readonly recordHash: string;
|
|
408
|
+
readonly deltaMs: number | null;
|
|
409
|
+
}
|
|
410
|
+
|
|
411
|
+
/**
|
|
412
|
+
* Fetch candidate assistant rows lacking duration_ms along with timestamp delta to preceding record.
|
|
413
|
+
*/
|
|
414
|
+
export async function findAssistantDurationCandidates(db: DbAdapter): Promise<AssistantDurationCandidate[]> {
|
|
415
|
+
const candidateSql = `
|
|
416
|
+
WITH ordered AS (
|
|
417
|
+
SELECT record_hash,
|
|
418
|
+
role,
|
|
419
|
+
duration_ms,
|
|
420
|
+
ts,
|
|
421
|
+
LAG(ts) OVER (PARTITION BY source, session_id ORDER BY seq) AS prev_ts
|
|
422
|
+
FROM history_message
|
|
423
|
+
WHERE ts IS NOT NULL AND ts LIKE '____-__-__T%'
|
|
424
|
+
)
|
|
425
|
+
SELECT record_hash AS recordHash,
|
|
426
|
+
CAST(ROUND((unixepoch(ts, 'subsec') - unixepoch(prev_ts, 'subsec')) * 1000) AS INTEGER) AS deltaMs
|
|
427
|
+
FROM ordered
|
|
428
|
+
WHERE role = 'assistant' AND duration_ms IS NULL AND prev_ts IS NOT NULL`;
|
|
429
|
+
|
|
430
|
+
return db.queryAll<AssistantDurationCandidate>(candidateSql);
|
|
431
|
+
}
|
|
432
|
+
|
|
433
|
+
/**
|
|
434
|
+
* Update duration_ms and duration_source for a previously unmeasured assistant row.
|
|
435
|
+
*/
|
|
436
|
+
export async function updateAssistantDuration(
|
|
437
|
+
db: DbAdapter,
|
|
438
|
+
recordHash: string,
|
|
439
|
+
durationMs: number,
|
|
440
|
+
durationSource: string,
|
|
441
|
+
): Promise<void> {
|
|
442
|
+
await db.run(
|
|
443
|
+
'UPDATE history_message SET duration_ms = ?, duration_source = ? WHERE record_hash = ? AND duration_ms IS NULL',
|
|
444
|
+
durationMs,
|
|
445
|
+
durationSource,
|
|
446
|
+
recordHash,
|
|
447
|
+
);
|
|
448
|
+
}
|
|
449
|
+
|
|
398
450
|
async function insertRecord(
|
|
399
451
|
db: ImportOptions['db'],
|
|
400
452
|
targetTable: string,
|
package/src/schema-sql.ts
CHANGED
|
@@ -1,3 +1,11 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Schema version of the history import DDL.
|
|
3
|
+
*
|
|
4
|
+
* Guaranteed to match the package version; a bump-or-fail test enforces that
|
|
5
|
+
* any modification to HISTORY_IMPORT_SCHEMA_SQL must be accompanied by a version bump.
|
|
6
|
+
*/
|
|
7
|
+
export const HISTORY_IMPORT_SCHEMA_VERSION = '0.4.55';
|
|
8
|
+
|
|
1
9
|
/**
|
|
2
10
|
* DDL string that creates the history import checkpoint, ledger, and typed contract tables.
|
|
3
11
|
*
|
|
@@ -39,6 +47,7 @@ CREATE TABLE IF NOT EXISTS history_message (
|
|
|
39
47
|
disposition TEXT NOT NULL,
|
|
40
48
|
ts TEXT,
|
|
41
49
|
duration_ms INTEGER,
|
|
50
|
+
duration_source TEXT,
|
|
42
51
|
model TEXT,
|
|
43
52
|
input_tokens INTEGER,
|
|
44
53
|
output_tokens INTEGER,
|
package/src/tables.ts
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
import { TYPED_HISTORY_TABLES } from './jsonl-importer-dao';
|
|
2
|
+
import { SOURCE_DEFINITIONS } from './sources';
|
|
3
|
+
|
|
4
|
+
/** Bookkeeping tables owned and maintained by the importer. */
|
|
5
|
+
export const BOOKKEEPING_HISTORY_TABLES = ['history_import_checkpoint', 'history_import_ledger'] as const;
|
|
6
|
+
|
|
7
|
+
/**
|
|
8
|
+
* Complete list of tables owned by the importer:
|
|
9
|
+
* - 3 typed contract tables (history_message, history_tool_call, history_skill_call)
|
|
10
|
+
* - 2 bookkeeping tables (history_import_checkpoint, history_import_ledger)
|
|
11
|
+
* - 10 per-source raw ETL landing tables (history_etl_<source>)
|
|
12
|
+
*
|
|
13
|
+
* Total: 15 tables.
|
|
14
|
+
* Sourced dynamically from static declarations so adding a source upstream automatically
|
|
15
|
+
* includes its landing table without downstream duplication.
|
|
16
|
+
*/
|
|
17
|
+
export const IMPORTER_OWNED_TABLES: readonly string[] = [
|
|
18
|
+
...TYPED_HISTORY_TABLES,
|
|
19
|
+
...BOOKKEEPING_HISTORY_TABLES,
|
|
20
|
+
...Object.values(SOURCE_DEFINITIONS).map((def) => def.targetTable),
|
|
21
|
+
];
|
package/src/types.ts
CHANGED
|
@@ -177,6 +177,7 @@ export interface SkillCallSplitRecord {
|
|
|
177
177
|
readonly duration_ms?: number | null;
|
|
178
178
|
}
|
|
179
179
|
|
|
180
|
+
/** Normalized skill-call record imported into history_skill_call. */
|
|
180
181
|
export interface SkillCall {
|
|
181
182
|
readonly record_hash: string;
|
|
182
183
|
readonly message_hash: string;
|