@kaelio/ktx 0.7.0 → 0.9.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (142) hide show
  1. package/assets/python/{kaelio_ktx-0.7.0-py3-none-any.whl → kaelio_ktx-0.9.0-py3-none-any.whl} +0 -0
  2. package/assets/python/manifest.json +4 -4
  3. package/dist/.tsbuildinfo +1 -1
  4. package/dist/cli-program.js +7 -0
  5. package/dist/cli-runtime.js +50 -3
  6. package/dist/command-schemas.d.ts +1 -1
  7. package/dist/command-tree.js +5 -1
  8. package/dist/commands/completion-commands.d.ts +3 -0
  9. package/dist/commands/completion-commands.js +38 -0
  10. package/dist/commands/ingest-commands.js +0 -4
  11. package/dist/commands/knowledge-commands.js +15 -2
  12. package/dist/commands/setup-commands.js +3 -3
  13. package/dist/commands/sl-commands.js +19 -7
  14. package/dist/completion/complete-engine.d.ts +19 -0
  15. package/dist/completion/complete-engine.js +128 -0
  16. package/dist/completion/completion-scripts.d.ts +1 -0
  17. package/dist/completion/completion-scripts.js +36 -0
  18. package/dist/completion/dynamic-candidates.d.ts +6 -0
  19. package/dist/completion/dynamic-candidates.js +98 -0
  20. package/dist/connection-drivers.d.ts +3 -0
  21. package/dist/connection-drivers.js +17 -0
  22. package/dist/connection-recovery.d.ts +34 -0
  23. package/dist/connection-recovery.js +82 -0
  24. package/dist/connection.js +3 -1
  25. package/dist/context/ingest/adapters/historic-sql/bigquery-query-history-reader.js +71 -20
  26. package/dist/context/ingest/adapters/historic-sql/chunk-unified.js +2 -1
  27. package/dist/context/ingest/adapters/historic-sql/connection-dialect.d.ts +9 -0
  28. package/dist/context/ingest/adapters/historic-sql/connection-dialect.js +15 -4
  29. package/dist/context/ingest/adapters/historic-sql/pattern-inputs.js +8 -2
  30. package/dist/context/ingest/adapters/historic-sql/query-history-filter-picker.d.ts +29 -0
  31. package/dist/context/ingest/adapters/historic-sql/query-history-filter-picker.js +190 -0
  32. package/dist/context/ingest/adapters/historic-sql/scope-floor.d.ts +18 -0
  33. package/dist/context/ingest/adapters/historic-sql/scope-floor.js +229 -0
  34. package/dist/context/ingest/adapters/historic-sql/scope-membership.d.ts +8 -0
  35. package/dist/context/ingest/adapters/historic-sql/scope-membership.js +29 -0
  36. package/dist/context/ingest/adapters/historic-sql/snowflake-query-history-reader.js +68 -19
  37. package/dist/context/ingest/adapters/historic-sql/stage-unified.js +57 -50
  38. package/dist/context/ingest/adapters/historic-sql/types.d.ts +36 -3
  39. package/dist/context/ingest/adapters/historic-sql/types.js +14 -2
  40. package/dist/context/ingest/context-evidence/sqlite-context-evidence-store.d.ts +1 -1
  41. package/dist/context/ingest/ingest-bundle.runner.d.ts +8 -0
  42. package/dist/context/ingest/ingest-bundle.runner.js +72 -15
  43. package/dist/context/ingest/ingest-profile.d.ts +102 -0
  44. package/dist/context/ingest/ingest-profile.js +306 -0
  45. package/dist/context/ingest/isolated-diff/patch-integrator.js +75 -5
  46. package/dist/context/ingest/isolated-diff/work-unit-executor.js +25 -2
  47. package/dist/context/ingest/local-adapters.js +21 -4
  48. package/dist/context/ingest/local-bundle-runtime.js +4 -2
  49. package/dist/context/ingest/local-ingest.d.ts +1 -1
  50. package/dist/context/ingest/local-ingest.js +6 -4
  51. package/dist/context/ingest/memory-flow/events.js +2 -1
  52. package/dist/context/ingest/ports.d.ts +2 -0
  53. package/dist/context/ingest/reports.d.ts +3 -0
  54. package/dist/context/ingest/reports.js +10 -0
  55. package/dist/context/ingest/stages/stage-3-work-units.d.ts +3 -1
  56. package/dist/context/ingest/stages/stage-3-work-units.js +2 -0
  57. package/dist/context/ingest/stages/stage-4-reconciliation.d.ts +2 -1
  58. package/dist/context/ingest/stages/stage-4-reconciliation.js +1 -1
  59. package/dist/context/ingest/tools/tool-call-logger.d.ts +6 -0
  60. package/dist/context/ingest/tools/tool-call-logger.js +36 -1
  61. package/dist/context/llm/ai-sdk-runtime.js +32 -3
  62. package/dist/context/llm/claude-code-runtime.js +35 -2
  63. package/dist/context/llm/codex-exec-events.d.ts +20 -0
  64. package/dist/context/llm/codex-exec-events.js +155 -0
  65. package/dist/context/llm/codex-isolation.d.ts +3 -0
  66. package/dist/context/llm/codex-isolation.js +5 -0
  67. package/dist/context/llm/codex-mcp-runtime-server.d.ts +24 -0
  68. package/dist/context/llm/codex-mcp-runtime-server.js +51 -0
  69. package/dist/context/llm/codex-models.d.ts +2 -0
  70. package/dist/context/llm/codex-models.js +17 -0
  71. package/dist/context/llm/codex-runtime-config.d.ts +16 -0
  72. package/dist/context/llm/codex-runtime-config.js +19 -0
  73. package/dist/context/llm/codex-runtime.d.ts +37 -0
  74. package/dist/context/llm/codex-runtime.js +304 -0
  75. package/dist/context/llm/codex-sdk-runner.d.ts +21 -0
  76. package/dist/context/llm/codex-sdk-runner.js +63 -0
  77. package/dist/context/llm/local-config.d.ts +2 -0
  78. package/dist/context/llm/local-config.js +12 -1
  79. package/dist/context/llm/runtime-port.d.ts +25 -0
  80. package/dist/context/mcp/context-tools.d.ts +2 -1
  81. package/dist/context/mcp/context-tools.js +82 -15
  82. package/dist/context/mcp/server.js +4 -0
  83. package/dist/context/mcp/types.d.ts +15 -1
  84. package/dist/context/project/config.d.ts +3 -0
  85. package/dist/context/project/config.js +6 -2
  86. package/dist/context/project/driver-schemas.js +1 -1
  87. package/dist/context/search/discover.js +4 -3
  88. package/dist/context/sl/local-sl.d.ts +15 -0
  89. package/dist/context/sl/local-sl.js +30 -0
  90. package/dist/context/sql-analysis/http-sql-analysis-port.js +32 -2
  91. package/dist/context/sql-analysis/ports.d.ts +12 -2
  92. package/dist/context/tools/context-candidate-mark.tool.d.ts +2 -2
  93. package/dist/context/wiki/local-knowledge.d.ts +10 -0
  94. package/dist/context/wiki/local-knowledge.js +22 -0
  95. package/dist/context-build-view.d.ts +0 -3
  96. package/dist/context-build-view.js +5 -39
  97. package/dist/ingest.js +7 -10
  98. package/dist/io/buffered-command-io.d.ts +11 -0
  99. package/dist/io/buffered-command-io.js +28 -0
  100. package/dist/knowledge.d.ts +5 -0
  101. package/dist/knowledge.js +10 -1
  102. package/dist/llm/types.d.ts +1 -1
  103. package/dist/local-adapters.d.ts +10 -2
  104. package/dist/local-adapters.js +19 -3
  105. package/dist/next-steps.js +1 -2
  106. package/dist/progress-port-adapter.d.ts +6 -0
  107. package/dist/progress-port-adapter.js +18 -0
  108. package/dist/public-ingest-copy.js +1 -1
  109. package/dist/public-ingest.d.ts +20 -8
  110. package/dist/public-ingest.js +198 -61
  111. package/dist/scan.js +3 -1
  112. package/dist/setup-context.d.ts +2 -0
  113. package/dist/setup-context.js +138 -64
  114. package/dist/setup-databases.d.ts +17 -1
  115. package/dist/setup-databases.js +366 -326
  116. package/dist/setup-models.d.ts +10 -1
  117. package/dist/setup-models.js +90 -2
  118. package/dist/setup-ready-menu.d.ts +16 -2
  119. package/dist/setup-ready-menu.js +37 -5
  120. package/dist/setup-sources.js +141 -33
  121. package/dist/setup.js +24 -12
  122. package/dist/skills/analytics/SKILL.md +6 -1
  123. package/dist/sl.d.ts +6 -1
  124. package/dist/sl.js +32 -8
  125. package/dist/status-project.d.ts +11 -0
  126. package/dist/status-project.js +50 -1
  127. package/dist/telemetry/command-hook.d.ts +1 -0
  128. package/dist/telemetry/command-hook.js +3 -1
  129. package/dist/telemetry/emitter.js +1 -1
  130. package/dist/telemetry/events.d.ts +15 -9
  131. package/dist/telemetry/events.js +17 -5
  132. package/dist/telemetry/identity.d.ts +1 -2
  133. package/dist/telemetry/identity.js +13 -10
  134. package/dist/telemetry/index.d.ts +13 -1
  135. package/dist/telemetry/index.js +18 -3
  136. package/dist/telemetry/scrubber.d.ts +10 -0
  137. package/dist/telemetry/scrubber.js +20 -0
  138. package/package.json +20 -19
  139. package/dist/ingest-depth.d.ts +0 -8
  140. package/dist/ingest-depth.js +0 -56
  141. package/dist/setup-database-context-depth.d.ts +0 -23
  142. package/dist/setup-database-context-depth.js +0 -84
@@ -85,18 +85,88 @@ export async function integrateWorkUnitPatch(input) {
85
85
  });
86
86
  }
87
87
  catch (semanticError) {
88
- if (preApplyHead) {
89
- await input.integrationGit.resetHardTo(preApplyHead);
90
- }
88
+ const reason = errorMessage(semanticError);
91
89
  await input.trace.event('error', 'integration', 'patch_semantic_conflict_after_textual_resolution', {
92
90
  unitKey: input.unitKey,
93
91
  patchPath: input.patchPath,
94
92
  touchedPaths: textualResolution.changedPaths,
95
- reason: errorMessage(semanticError),
93
+ reason,
96
94
  });
95
+ // A textual conflict and a semantic-gate failure can co-occur: the resolver
96
+ // reconciles the text but can leave wiki sl_refs pointing at measures the
97
+ // merged source no longer defines. Recover via the same gate repair the
98
+ // clean-apply branch uses, instead of hard-failing the whole job.
99
+ if (input.repairGateFailure) {
100
+ const gateRepair = await input.repairGateFailure({
101
+ unitKey: input.unitKey,
102
+ patchPath: input.patchPath,
103
+ touchedPaths: textualResolution.changedPaths,
104
+ reason,
105
+ });
106
+ if (gateRepair.status !== 'failed') {
107
+ // The resolver wrote its merge to the worktree (unstaged); the repair
108
+ // edited a subset on top. Commit the union so neither is dropped.
109
+ const resolvedAndRepairedPaths = [
110
+ ...new Set([...textualResolution.changedPaths, ...gateRepair.changedPaths]),
111
+ ].sort();
112
+ try {
113
+ await traceTimed(input.trace, 'integration', 'semantic_gate_after_gate_repair', { unitKey: input.unitKey, touchedPaths: gateRepair.changedPaths }, async () => {
114
+ await input.validateAppliedTree(gateRepair.changedPaths);
115
+ });
116
+ const commit = await input.integrationGit.commitFiles(resolvedAndRepairedPaths, `ingest: resolve WorkUnit ${input.unitKey} conflict`, input.author.name, input.author.email);
117
+ if (commit.created) {
118
+ await input.trace.event('debug', 'integration', 'patch_accepted_after_textual_resolution', {
119
+ unitKey: input.unitKey,
120
+ commitSha: commit.commitHash,
121
+ touchedPaths: resolvedAndRepairedPaths,
122
+ attempts: textualResolution.attempts,
123
+ gateRepairAttempts: gateRepair.attempts,
124
+ });
125
+ return {
126
+ status: 'accepted',
127
+ commitSha: commit.commitHash,
128
+ touchedPaths: resolvedAndRepairedPaths,
129
+ textualResolution,
130
+ gateRepair,
131
+ };
132
+ }
133
+ }
134
+ catch (repairValidationError) {
135
+ if (preApplyHead) {
136
+ await input.integrationGit.resetHardTo(preApplyHead);
137
+ }
138
+ await input.trace.event('error', 'integration', 'patch_semantic_conflict_after_textual_resolution', {
139
+ unitKey: input.unitKey,
140
+ patchPath: input.patchPath,
141
+ touchedPaths: gateRepair.changedPaths,
142
+ reason: errorMessage(repairValidationError),
143
+ });
144
+ return {
145
+ status: 'semantic_conflict',
146
+ reason: errorMessage(repairValidationError),
147
+ touchedPaths: gateRepair.changedPaths,
148
+ textualResolution,
149
+ gateRepair,
150
+ };
151
+ }
152
+ }
153
+ if (preApplyHead) {
154
+ await input.integrationGit.resetHardTo(preApplyHead);
155
+ }
156
+ return {
157
+ status: 'semantic_conflict',
158
+ reason: gateRepair.status === 'failed' ? gateRepair.reason : reason,
159
+ touchedPaths: textualResolution.changedPaths,
160
+ textualResolution,
161
+ gateRepair,
162
+ };
163
+ }
164
+ if (preApplyHead) {
165
+ await input.integrationGit.resetHardTo(preApplyHead);
166
+ }
97
167
  return {
98
168
  status: 'semantic_conflict',
99
- reason: errorMessage(semanticError),
169
+ reason,
100
170
  touchedPaths: textualResolution.changedPaths,
101
171
  textualResolution,
102
172
  };
@@ -8,15 +8,37 @@ function patchFileName(unitIndex, unitKey) {
8
8
  export async function runIsolatedWorkUnit(input) {
9
9
  const sessionKey = `${input.trace.context.jobId}-${input.workUnit.unitKey}`;
10
10
  let cleanupOutcome = 'crash';
11
+ const createStartedAt = Date.now();
11
12
  const child = await input.sessionWorktreeService.create(sessionKey, input.ingestionBaseSha);
12
13
  await input.trace.event('debug', 'work_unit', 'work_unit_child_created', {
13
14
  unitKey: input.workUnit.unitKey,
14
15
  unitIndex: input.unitIndex,
15
16
  worktreePath: child.workdir,
16
17
  baseSha: input.ingestionBaseSha,
17
- });
18
+ }, undefined, Date.now() - createStartedAt);
18
19
  try {
20
+ const runStartedAt = Date.now();
19
21
  const outcome = await input.run(child);
22
+ await input.trace.event('debug', 'work_unit', 'work_unit_executed', {
23
+ unitKey: input.workUnit.unitKey,
24
+ unitIndex: input.unitIndex,
25
+ status: outcome.status,
26
+ ...(outcome.metrics
27
+ ? {
28
+ agentLoopMs: outcome.metrics.totalMs,
29
+ stepCount: outcome.metrics.stepCount,
30
+ ...(outcome.metrics.usage.inputTokens !== undefined
31
+ ? { inputTokens: outcome.metrics.usage.inputTokens }
32
+ : {}),
33
+ ...(outcome.metrics.usage.outputTokens !== undefined
34
+ ? { outputTokens: outcome.metrics.usage.outputTokens }
35
+ : {}),
36
+ ...(outcome.metrics.usage.totalTokens !== undefined
37
+ ? { totalTokens: outcome.metrics.usage.totalTokens }
38
+ : {}),
39
+ }
40
+ : {}),
41
+ }, undefined, Date.now() - runStartedAt);
20
42
  if (outcome.status !== 'success') {
21
43
  cleanupOutcome = 'success';
22
44
  await input.trace.event('error', 'work_unit', 'work_unit_failed_before_patch', {
@@ -51,11 +73,12 @@ export async function runIsolatedWorkUnit(input) {
51
73
  throw error;
52
74
  }
53
75
  finally {
76
+ const cleanupStartedAt = Date.now();
54
77
  await input.sessionWorktreeService.cleanup(child, cleanupOutcome);
55
78
  await input.trace.event('trace', 'work_unit', 'work_unit_child_cleanup', {
56
79
  unitKey: input.workUnit.unitKey,
57
80
  outcome: cleanupOutcome,
58
81
  worktreePath: child.workdir,
59
- });
82
+ }, undefined, Date.now() - cleanupStartedAt);
60
83
  }
61
84
  }
@@ -7,6 +7,7 @@ import { DbtSourceAdapter } from './adapters/dbt/dbt.adapter.js';
7
7
  import { FakeSourceAdapter } from './adapters/fake/fake.adapter.js';
8
8
  import { HistoricSqlSourceAdapter } from './adapters/historic-sql/historic-sql.adapter.js';
9
9
  import { PostgresPgssReader } from './adapters/historic-sql/postgres-pgss-reader.js';
10
+ import { resolveQueryHistoryScopeFloor } from './adapters/historic-sql/scope-floor.js';
10
11
  import { HISTORIC_SQL_SOURCE_KEY, historicSqlUnifiedPullConfigSchema, } from './adapters/historic-sql/types.js';
11
12
  import { createDaemonLiveDatabaseIntrospection, } from './adapters/live-database/daemon-introspection.js';
12
13
  import { LiveDatabaseSourceAdapter } from './adapters/live-database/live-database.adapter.js';
@@ -113,14 +114,30 @@ function queryHistoryRecord(connection) {
113
114
  const queryHistory = isRecord(context?.queryHistory) ? context.queryHistory : null;
114
115
  return queryHistory;
115
116
  }
116
- function queryHistoryPullConfig(connection) {
117
+ async function queryHistoryPullConfig(project, connectionId, connection) {
117
118
  const queryHistory = queryHistoryRecord(connection);
118
119
  if (queryHistory?.enabled !== true || !isRecord(connection))
119
120
  return null;
120
- const dialect = historicSqlDialectByDriver.get(String(connection.driver ?? '').toLowerCase());
121
+ const driver = String(connection.driver ?? '').toLowerCase();
122
+ const dialect = historicSqlDialectByDriver.get(driver);
121
123
  if (!dialect)
122
124
  return null;
123
- return { ...queryHistory, dialect };
125
+ const scopeFloor = await resolveQueryHistoryScopeFloor({
126
+ projectDir: project.projectDir,
127
+ connectionId,
128
+ driver,
129
+ connection,
130
+ storedQueryHistory: queryHistory,
131
+ });
132
+ const { enabled: _enabled, dialect: _dialect, enabledTables: _enabledTables, enabledSchemas: _enabledSchemas, scopeFloorWarnings: _scopeFloorWarnings, ...stored } = queryHistory;
133
+ return {
134
+ ...stored,
135
+ dialect,
136
+ ...(scopeFloor.enabledTables.length > 0 ? { enabledTables: scopeFloor.enabledTables } : {}),
137
+ ...(scopeFloor.enabledSchemas.length > 0 ? { enabledSchemas: scopeFloor.enabledSchemas } : {}),
138
+ ...(scopeFloor.modeledTableCatalog.length > 0 ? { modeledTableCatalog: scopeFloor.modeledTableCatalog } : {}),
139
+ ...(scopeFloor.warnings.length > 0 ? { scopeFloorWarnings: scopeFloor.warnings } : {}),
140
+ };
124
141
  }
125
142
  function stringField(value) {
126
143
  return typeof value === 'string' && value.trim().length > 0 ? value.trim() : null;
@@ -168,7 +185,7 @@ export async function localPullConfigForAdapter(project, adapter, connectionId,
168
185
  if (options.historicSqlPullConfigOverride) {
169
186
  return historicSqlUnifiedPullConfigSchema.parse(options.historicSqlPullConfigOverride);
170
187
  }
171
- const queryHistory = queryHistoryPullConfig(connection);
188
+ const queryHistory = await queryHistoryPullConfig(project, connectionId, connection);
172
189
  if (!queryHistory) {
173
190
  throw new Error(`Connection "${connectionId}" does not have context.queryHistory.enabled: true`);
174
191
  }
@@ -488,9 +488,10 @@ function nextLocalJobId() {
488
488
  }
489
489
  function localIngestLlmProviderGuardMessage(projectDir) {
490
490
  return [
491
- 'ktx ingest requires llm.provider.backend: anthropic, vertex, gateway, or claude-code, or an injected agentRunner.',
492
- 'Configure a local Claude Code session or API-backed LLM, then rerun ingest:',
491
+ 'ktx ingest requires llm.provider.backend: anthropic, vertex, gateway, claude-code, or codex, or an injected agentRunner.',
492
+ 'Configure a local Claude Code/Codex session or API-backed LLM, then rerun ingest:',
493
493
  ` ktx setup --project-dir ${projectDir} --llm-backend claude-code --no-input`,
494
+ ` ktx setup --project-dir ${projectDir} --llm-backend codex --llm-model gpt-5.5 --no-input`,
494
495
  ` ktx setup --project-dir ${projectDir} --llm-backend anthropic --anthropic-api-key-env ANTHROPIC_API_KEY --llm-model claude-sonnet-4-6 --no-input`,
495
496
  ].join('\n');
496
497
  }
@@ -574,6 +575,7 @@ export function createLocalBundleIngestRuntime(options) {
574
575
  workUnitMaxConcurrency: options.project.config.ingest.workUnits.maxConcurrency,
575
576
  workUnitStepBudget: options.project.config.ingest.workUnits.stepBudget,
576
577
  workUnitFailureMode: options.project.config.ingest.workUnits.failureMode,
578
+ profileIngest: options.project.config.ingest.profile,
577
579
  ingestTraceLevel: ingestTraceLevelFromEnv(),
578
580
  },
579
581
  skillsRegistry: new SkillsRegistryService({ skillsDir, logger }),
@@ -67,7 +67,7 @@ export interface LocalMetabaseFanoutProgress {
67
67
  metabaseDatabaseId: number;
68
68
  targetConnectionId: string;
69
69
  jobId: string;
70
- status: 'done' | 'failed';
70
+ status: 'done' | 'partial' | 'failed';
71
71
  }): void;
72
72
  }
73
73
  export interface RunLocalMetabaseIngestOptions extends Omit<RunLocalIngestOptions, 'adapter' | 'connectionId' | 'sourceDir' | 'jobId'> {
@@ -7,6 +7,7 @@ import { KtxYamlMetabaseSourceStateReader, LocalMetabaseDiscoveryCache } from '.
7
7
  import { localPullConfigForAdapter } from './local-adapters.js';
8
8
  import { createLocalBundleIngestRuntime } from './local-bundle-runtime.js';
9
9
  import { buildSyncId } from './raw-sources-paths.js';
10
+ import { ingestReportOutcome } from './reports.js';
10
11
  import { SqliteBundleIngestStore } from './sqlite-bundle-ingest-store.js';
11
12
  class LocalIngestPhase {
12
13
  async updateProgress() { }
@@ -117,11 +118,11 @@ export async function runLocalIngest(options) {
117
118
  return { result, report };
118
119
  }
119
120
  function metabaseFanoutStatus(children) {
120
- const succeeded = children.filter((child) => child.report.body.failedWorkUnits.length === 0).length;
121
- if (succeeded === children.length) {
121
+ const outcomes = children.map((child) => ingestReportOutcome(child.report));
122
+ if (outcomes.every((outcome) => outcome === 'done')) {
122
123
  return 'all_succeeded';
123
124
  }
124
- if (succeeded === 0) {
125
+ if (outcomes.every((outcome) => outcome === 'error')) {
125
126
  return 'all_failed';
126
127
  }
127
128
  return 'partial_failure';
@@ -266,12 +267,13 @@ export async function runLocalMetabaseIngest(options) {
266
267
  error,
267
268
  });
268
269
  }
270
+ const childOutcome = ingestReportOutcome(child.report);
269
271
  options.progress?.onMetabaseChildCompleted?.({
270
272
  metabaseConnectionId,
271
273
  metabaseDatabaseId: childPlan.metabaseDatabaseId,
272
274
  targetConnectionId,
273
275
  jobId: child.report.jobId,
274
- status: child.report.body.failedWorkUnits.length > 0 ? 'failed' : 'done',
276
+ status: childOutcome === 'error' ? 'failed' : childOutcome,
275
277
  });
276
278
  children.push({
277
279
  jobId: child.report.jobId,
@@ -1,3 +1,4 @@
1
+ import { ingestReportOutcome } from '../reports.js';
1
2
  function plannedWorkUnitFromLocal(workUnit) {
2
3
  return {
3
4
  unitKey: workUnit.unitKey,
@@ -39,7 +40,7 @@ function fullModeMetadata(input) {
39
40
  };
40
41
  }
41
42
  function reportStatus(report) {
42
- return report.body.failedWorkUnits.length > 0 ? 'error' : 'done';
43
+ return ingestReportOutcome(report) === 'error' ? 'error' : 'done';
43
44
  }
44
45
  function reportCreatedEvent(report) {
45
46
  return { type: 'report_created', runId: report.runId, reportPath: report.id };
@@ -111,6 +111,8 @@ interface IngestSettingsPort {
111
111
  workUnitMaxConcurrency?: number;
112
112
  workUnitStepBudget?: number;
113
113
  workUnitFailureMode?: 'abort' | 'continue';
114
+ /** Print a timing breakdown to stderr at the end of each run (config-driven; see also KTX_PROFILE_INGEST). `'json'` emits the raw structured profile. */
115
+ profileIngest?: boolean | 'json';
114
116
  ingestTraceLevel?: IngestTraceLevel;
115
117
  }
116
118
  interface IngestGitAuthor {
@@ -116,5 +116,8 @@ export interface IngestSavedMemoryCounts {
116
116
  slCount: number;
117
117
  }
118
118
  export declare function savedMemoryCountsForReport(report: IngestReportSnapshot): IngestSavedMemoryCounts;
119
+ /** @internal */
120
+ export type IngestReportOutcome = 'done' | 'partial' | 'error';
121
+ export declare function ingestReportOutcome(report: IngestReportSnapshot): IngestReportOutcome;
119
122
  export declare function buildStageIndexFromReportBody(jobId: string, connectionId: string, body: IngestReportBody): StageIndex;
120
123
  export {};
@@ -8,6 +8,16 @@ export function savedMemoryCountsForReport(report) {
8
8
  slCount: actions.filter((action) => action.target === 'sl').length,
9
9
  };
10
10
  }
11
+ export function ingestReportOutcome(report) {
12
+ if (report.body.status === 'failed') {
13
+ return 'error';
14
+ }
15
+ if (report.body.failedWorkUnits.length === 0) {
16
+ return 'done';
17
+ }
18
+ const { wikiCount, slCount } = savedMemoryCountsForReport(report);
19
+ return wikiCount + slCount > 0 ? 'partial' : 'error';
20
+ }
11
21
  export function buildStageIndexFromReportBody(jobId, connectionId, body) {
12
22
  return {
13
23
  jobId,
@@ -1,5 +1,5 @@
1
1
  import type { KtxModelRole } from '../../../llm/types.js';
2
- import type { AgentRunnerPort, KtxRuntimeToolSet } from '../../../context/llm/runtime-port.js';
2
+ import type { AgentRunnerPort, KtxRuntimeToolSet, RunLoopMetrics } from '../../../context/llm/runtime-port.js';
3
3
  import type { CaptureSession, MemoryAction } from '../../../context/memory/types.js';
4
4
  import { type TouchedSlSource } from '../../../context/tools/touched-sl-sources.js';
5
5
  import type { WorkUnit } from '../types.js';
@@ -44,6 +44,8 @@ export interface WorkUnitOutcome {
44
44
  patchPath?: string;
45
45
  patchTouchedPaths?: string[];
46
46
  childWorktreePath?: string;
47
+ /** Timing and token metrics for the work-unit agent loop, used for ingest profiling. */
48
+ metrics?: RunLoopMetrics;
47
49
  }
48
50
  export declare function executeWorkUnit(deps: WorkUnitExecutionDeps, wu: WorkUnit): Promise<WorkUnitOutcome>;
49
51
  export {};
@@ -72,6 +72,7 @@ export async function executeWorkUnit(deps, wu) {
72
72
  touchedSlSources: [],
73
73
  slDisallowed: wu.slDisallowed,
74
74
  slDisallowedReason: wu.slDisallowedReason,
75
+ ...(runResult.metrics ? { metrics: runResult.metrics } : {}),
75
76
  };
76
77
  };
77
78
  if (runResult.stopReason === 'error') {
@@ -104,5 +105,6 @@ export async function executeWorkUnit(deps, wu) {
104
105
  touchedSlSources: touched,
105
106
  slDisallowed: wu.slDisallowed,
106
107
  slDisallowedReason: wu.slDisallowedReason,
108
+ ...(runResult.metrics ? { metrics: runResult.metrics } : {}),
107
109
  };
108
110
  }
@@ -1,4 +1,4 @@
1
- import type { AgentRunnerPort, KtxRuntimeToolSet } from '../../../context/llm/runtime-port.js';
1
+ import type { AgentRunnerPort, KtxRuntimeToolSet, RunLoopMetrics } from '../../../context/llm/runtime-port.js';
2
2
  import type { KtxModelRole } from '../../../llm/types.js';
3
3
  import type { EvictionUnit } from '../types.js';
4
4
  import type { StageIndex } from './stage-index.types.js';
@@ -24,5 +24,6 @@ export interface ReconciliationOutcome {
24
24
  skipped: boolean;
25
25
  stopReason?: 'budget' | 'natural' | 'error';
26
26
  error?: Error;
27
+ metrics?: RunLoopMetrics;
27
28
  }
28
29
  export declare function runReconciliationStage4(ctx: ReconciliationContext): Promise<ReconciliationOutcome>;
@@ -13,5 +13,5 @@ export async function runReconciliationStage4(ctx) {
13
13
  telemetryTags: { operationName: 'ingest-bundle-reconcile', source: ctx.sourceKey, jobId: ctx.jobId },
14
14
  onStepFinish: ctx.onStepFinish,
15
15
  });
16
- return { skipped: false, stopReason: run.stopReason, error: run.error };
16
+ return { skipped: false, stopReason: run.stopReason, error: run.error, ...(run.metrics ? { metrics: run.metrics } : {}) };
17
17
  }
@@ -30,4 +30,10 @@ interface ToolCallLoggerOptions {
30
30
  * effectively single-writer and lines land in call order.
31
31
  */
32
32
  export declare function wrapToolsWithLogger<T extends KtxRuntimeToolSet>(tools: T, logFilePath: string, wuKey: string, options?: ToolCallLoggerOptions): T;
33
+ /**
34
+ * Await all in-flight tool-call log writes (best-effort, bounded by `timeoutMs`
35
+ * so it can never hang a caller). Lets readers such as the ingest profiler see
36
+ * complete transcripts despite the fire-and-forget append design.
37
+ */
38
+ export declare function flushToolCallLogs(timeoutMs?: number): Promise<void>;
33
39
  export {};
@@ -59,8 +59,12 @@ export function wrapToolsWithLogger(tools, logFilePath, wuKey, options = {}) {
59
59
  }
60
60
  return wrapped;
61
61
  }
62
+ // Fire-and-forget appends are intentional (the agent hot path must never block
63
+ // or fail on logging), but readers like the ingest profiler need to know when
64
+ // the writes have settled. Track in-flight appends so a consumer can flush.
65
+ const pendingWrites = new Set();
62
66
  function appendEntry(path, entry) {
63
- void (async () => {
67
+ const write = (async () => {
64
68
  try {
65
69
  await mkdir(dirname(path), { recursive: true });
66
70
  await appendFile(path, `${safeStringify(entry)}\n`, 'utf-8');
@@ -69,6 +73,37 @@ function appendEntry(path, entry) {
69
73
  // best-effort
70
74
  }
71
75
  })();
76
+ pendingWrites.add(write);
77
+ void write.finally(() => pendingWrites.delete(write));
78
+ }
79
+ /**
80
+ * Await all in-flight tool-call log writes (best-effort, bounded by `timeoutMs`
81
+ * so it can never hang a caller). Lets readers such as the ingest profiler see
82
+ * complete transcripts despite the fire-and-forget append design.
83
+ */
84
+ export async function flushToolCallLogs(timeoutMs = 5000) {
85
+ const pending = [...pendingWrites];
86
+ if (pending.length === 0) {
87
+ return;
88
+ }
89
+ const settled = Promise.allSettled(pending).then(() => undefined);
90
+ if (timeoutMs <= 0) {
91
+ await settled;
92
+ return;
93
+ }
94
+ let timer;
95
+ const timeout = new Promise((resolve) => {
96
+ timer = setTimeout(resolve, timeoutMs);
97
+ timer.unref?.();
98
+ });
99
+ try {
100
+ await Promise.race([settled, timeout]);
101
+ }
102
+ finally {
103
+ if (timer) {
104
+ clearTimeout(timer);
105
+ }
106
+ }
72
107
  }
73
108
  function safeStringify(v) {
74
109
  try {
@@ -3,6 +3,16 @@ import { generateText, Output, stepCountIs } from 'ai';
3
3
  import { noopLogger } from '../../context/core/config.js';
4
4
  import { summarizeKtxLlmDebugRequest } from './debug-request-recorder.js';
5
5
  import { createAiSdkToolSet } from './runtime-tools.js';
6
+ function toLlmTokenUsage(usage) {
7
+ if (!usage) {
8
+ return {};
9
+ }
10
+ return {
11
+ ...(usage.inputTokens !== undefined ? { inputTokens: usage.inputTokens } : {}),
12
+ ...(usage.outputTokens !== undefined ? { outputTokens: usage.outputTokens } : {}),
13
+ ...(usage.totalTokens !== undefined ? { totalTokens: usage.totalTokens } : {}),
14
+ };
15
+ }
6
16
  function hasTools(tools) {
7
17
  return Object.keys(tools).length > 0;
8
18
  }
@@ -26,6 +36,7 @@ export class AiSdkKtxLlmRuntime {
26
36
  model,
27
37
  });
28
38
  const split = splitKtxSystemMessages(built.messages);
39
+ const startedAt = Date.now();
29
40
  const result = await generateText({
30
41
  model,
31
42
  temperature: input.temperature ?? 0,
@@ -40,6 +51,7 @@ export class AiSdkKtxLlmRuntime {
40
51
  }
41
52
  : {}),
42
53
  });
54
+ input.onMetrics?.({ totalMs: Date.now() - startedAt, usage: toLlmTokenUsage(result.totalUsage ?? result.usage) });
43
55
  if (typeof result.text !== 'string') {
44
56
  throw new Error('KTX LLM text generation returned no text');
45
57
  }
@@ -55,6 +67,7 @@ export class AiSdkKtxLlmRuntime {
55
67
  model,
56
68
  });
57
69
  const split = splitKtxSystemMessages(built.messages);
70
+ const startedAt = Date.now();
58
71
  const result = await generateText({
59
72
  model,
60
73
  temperature: input.temperature ?? 0,
@@ -70,6 +83,7 @@ export class AiSdkKtxLlmRuntime {
70
83
  : {}),
71
84
  output: Output.object({ schema: input.schema }),
72
85
  });
86
+ input.onMetrics?.({ totalMs: Date.now() - startedAt, usage: toLlmTokenUsage(result.totalUsage ?? result.usage) });
73
87
  if (result.output == null) {
74
88
  throw new Error('KTX LLM object generation returned no output');
75
89
  }
@@ -77,6 +91,8 @@ export class AiSdkKtxLlmRuntime {
77
91
  }
78
92
  async runAgentLoop(params) {
79
93
  let stepIndex = 0;
94
+ const startedAt = Date.now();
95
+ const stepBoundariesMs = [];
80
96
  try {
81
97
  const model = this.deps.llmProvider.getModel(params.modelRole);
82
98
  const tools = createAiSdkToolSet(params.toolSet);
@@ -98,7 +114,7 @@ export class AiSdkKtxLlmRuntime {
98
114
  messages: built.messages,
99
115
  tools: built.tools,
100
116
  }));
101
- await generateText({
117
+ const result = await generateText({
102
118
  model,
103
119
  temperature: 0,
104
120
  stopWhen: stepCountIs(params.stepBudget),
@@ -111,6 +127,7 @@ export class AiSdkKtxLlmRuntime {
111
127
  tools: built.tools,
112
128
  onStepFinish: async () => {
113
129
  stepIndex += 1;
130
+ stepBoundariesMs.push(Date.now() - startedAt);
114
131
  if (!params.onStepFinish) {
115
132
  return;
116
133
  }
@@ -122,12 +139,24 @@ export class AiSdkKtxLlmRuntime {
122
139
  }
123
140
  },
124
141
  });
125
- return { stopReason: 'natural' };
142
+ return {
143
+ stopReason: 'natural',
144
+ metrics: {
145
+ totalMs: Date.now() - startedAt,
146
+ stepCount: stepIndex,
147
+ stepBoundariesMs,
148
+ usage: toLlmTokenUsage(result.totalUsage ?? result.usage),
149
+ },
150
+ };
126
151
  }
127
152
  catch (error) {
128
153
  const err = error instanceof Error ? error : new Error(String(error));
129
154
  this.logger.warn(`[agent-runner] loop failed: ${err.message}`);
130
- return { stopReason: 'error', error: err };
155
+ return {
156
+ stopReason: 'error',
157
+ error: err,
158
+ metrics: { totalMs: Date.now() - startedAt, stepCount: stepIndex, stepBoundariesMs, usage: {} },
159
+ };
131
160
  }
132
161
  }
133
162
  }
@@ -4,6 +4,19 @@ import { noopLogger } from '../../context/core/config.js';
4
4
  import { createKtxClaudeCodeEnv } from './claude-code-env.js';
5
5
  import { resolveClaudeCodeModel } from './claude-code-models.js';
6
6
  import { createClaudeSdkTools, mcpToolIds } from './runtime-tools.js';
7
+ function claudeTokenUsage(result) {
8
+ const usage = result.usage;
9
+ if (!usage) {
10
+ return {};
11
+ }
12
+ const { input_tokens: inputTokens, output_tokens: outputTokens } = usage;
13
+ const totalTokens = inputTokens !== undefined && outputTokens !== undefined ? inputTokens + outputTokens : undefined;
14
+ return {
15
+ ...(inputTokens !== undefined ? { inputTokens } : {}),
16
+ ...(outputTokens !== undefined ? { outputTokens } : {}),
17
+ ...(totalTokens !== undefined ? { totalTokens } : {}),
18
+ };
19
+ }
7
20
  const BUILTIN_TOOLS = [
8
21
  'Agent',
9
22
  'Task',
@@ -168,6 +181,7 @@ export class ClaudeCodeKtxLlmRuntime {
168
181
  maxTurns: 1,
169
182
  tools: input.tools,
170
183
  });
184
+ const startedAt = Date.now();
171
185
  const result = await collectResult({
172
186
  query: this.runQuery,
173
187
  prompt: [input.system, input.prompt].filter(Boolean).join('\n\n'),
@@ -175,6 +189,7 @@ export class ClaudeCodeKtxLlmRuntime {
175
189
  allowedToolIds: new Set(mcpToolIds(input.tools ?? {})),
176
190
  expectedMcpServerNames: expectedMcpServerNames(input.tools),
177
191
  });
192
+ input.onMetrics?.({ totalMs: Date.now() - startedAt, usage: claudeTokenUsage(result) });
178
193
  const error = resultError(result);
179
194
  if (error) {
180
195
  throw error;
@@ -200,6 +215,7 @@ export class ClaudeCodeKtxLlmRuntime {
200
215
  }),
201
216
  outputFormat: { type: 'json_schema', schema: jsonSchema(input.schema) },
202
217
  };
218
+ const startedAt = Date.now();
203
219
  const result = await collectResult({
204
220
  query: this.runQuery,
205
221
  prompt: [input.system, input.prompt].filter(Boolean).join('\n\n'),
@@ -207,6 +223,7 @@ export class ClaudeCodeKtxLlmRuntime {
207
223
  allowedToolIds: new Set([...mcpToolIds(input.tools ?? {}), STRUCTURED_OUTPUT_TOOL_NAME]),
208
224
  expectedMcpServerNames: expectedMcpServerNames(input.tools),
209
225
  });
226
+ input.onMetrics?.({ totalMs: Date.now() - startedAt, usage: claudeTokenUsage(result) });
210
227
  const error = resultError(result);
211
228
  if (error) {
212
229
  throw error;
@@ -218,6 +235,8 @@ export class ClaudeCodeKtxLlmRuntime {
218
235
  }
219
236
  async runAgentLoop(params) {
220
237
  let stepIndex = 0;
238
+ const startedAt = Date.now();
239
+ const stepBoundariesMs = [];
221
240
  try {
222
241
  const options = baseOptions({
223
242
  projectDir: this.deps.projectDir,
@@ -234,6 +253,7 @@ export class ClaudeCodeKtxLlmRuntime {
234
253
  expectedMcpServerNames: expectedMcpServerNames(params.toolSet),
235
254
  onAssistantTurn: async () => {
236
255
  stepIndex += 1;
256
+ stepBoundariesMs.push(Date.now() - startedAt);
237
257
  if (!params.onStepFinish) {
238
258
  return;
239
259
  }
@@ -247,11 +267,24 @@ export class ClaudeCodeKtxLlmRuntime {
247
267
  });
248
268
  const stopReason = mapClaudeCodeStopReason(result);
249
269
  const error = resultError(result);
250
- return { stopReason, ...(stopReason === 'error' && error ? { error } : {}) };
270
+ return {
271
+ stopReason,
272
+ ...(stopReason === 'error' && error ? { error } : {}),
273
+ metrics: {
274
+ totalMs: Date.now() - startedAt,
275
+ stepCount: stepIndex,
276
+ stepBoundariesMs,
277
+ usage: claudeTokenUsage(result),
278
+ },
279
+ };
251
280
  }
252
281
  catch (error) {
253
282
  const err = error instanceof Error ? error : new Error(String(error));
254
- return { stopReason: 'error', error: err };
283
+ return {
284
+ stopReason: 'error',
285
+ error: err,
286
+ metrics: { totalMs: Date.now() - startedAt, stepCount: stepIndex, stepBoundariesMs, usage: {} },
287
+ };
255
288
  }
256
289
  }
257
290
  }
@@ -0,0 +1,20 @@
1
+ import type { LlmTokenUsage, RunLoopStopReason } from './runtime-port.js';
2
+ export interface CodexExecEventSummary {
3
+ finalText: string;
4
+ stopReason: RunLoopStopReason;
5
+ usage: LlmTokenUsage;
6
+ stepCount: number;
7
+ stepBoundariesMs: number[];
8
+ toolCallCount: number;
9
+ toolFailures: string[];
10
+ error?: Error;
11
+ }
12
+ interface CodexEventParseOptions {
13
+ startedAt?: number;
14
+ now?: () => number;
15
+ }
16
+ export declare function isCompletedAgentStep(event: unknown): boolean;
17
+ /** @internal */
18
+ export declare function parseCodexExecEventLine(line: string): unknown;
19
+ export declare function summarizeCodexExecEvents(events: Iterable<unknown>, options?: CodexEventParseOptions): CodexExecEventSummary;
20
+ export {};