@kaelio/ktx 0.7.0 → 0.8.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (80) hide show
  1. package/assets/python/{kaelio_ktx-0.7.0-py3-none-any.whl → kaelio_ktx-0.8.0-py3-none-any.whl} +0 -0
  2. package/assets/python/manifest.json +4 -4
  3. package/dist/.tsbuildinfo +1 -1
  4. package/dist/cli-program.js +7 -0
  5. package/dist/command-schemas.d.ts +1 -1
  6. package/dist/command-tree.js +5 -1
  7. package/dist/commands/completion-commands.d.ts +3 -0
  8. package/dist/commands/completion-commands.js +38 -0
  9. package/dist/commands/ingest-commands.js +0 -4
  10. package/dist/commands/knowledge-commands.js +15 -2
  11. package/dist/commands/setup-commands.js +2 -2
  12. package/dist/commands/sl-commands.js +19 -7
  13. package/dist/completion/complete-engine.d.ts +19 -0
  14. package/dist/completion/complete-engine.js +128 -0
  15. package/dist/completion/completion-scripts.d.ts +1 -0
  16. package/dist/completion/completion-scripts.js +36 -0
  17. package/dist/completion/dynamic-candidates.d.ts +6 -0
  18. package/dist/completion/dynamic-candidates.js +98 -0
  19. package/dist/connection-drivers.d.ts +3 -0
  20. package/dist/connection-drivers.js +17 -0
  21. package/dist/context/ingest/ingest-bundle.runner.d.ts +8 -0
  22. package/dist/context/ingest/ingest-bundle.runner.js +72 -15
  23. package/dist/context/ingest/ingest-profile.d.ts +102 -0
  24. package/dist/context/ingest/ingest-profile.js +306 -0
  25. package/dist/context/ingest/isolated-diff/work-unit-executor.js +25 -2
  26. package/dist/context/ingest/local-bundle-runtime.js +1 -0
  27. package/dist/context/ingest/local-ingest.d.ts +1 -1
  28. package/dist/context/ingest/local-ingest.js +6 -4
  29. package/dist/context/ingest/memory-flow/events.js +2 -1
  30. package/dist/context/ingest/ports.d.ts +2 -0
  31. package/dist/context/ingest/reports.d.ts +3 -0
  32. package/dist/context/ingest/reports.js +10 -0
  33. package/dist/context/ingest/stages/stage-3-work-units.d.ts +3 -1
  34. package/dist/context/ingest/stages/stage-3-work-units.js +2 -0
  35. package/dist/context/ingest/stages/stage-4-reconciliation.d.ts +2 -1
  36. package/dist/context/ingest/stages/stage-4-reconciliation.js +1 -1
  37. package/dist/context/ingest/tools/tool-call-logger.d.ts +6 -0
  38. package/dist/context/ingest/tools/tool-call-logger.js +36 -1
  39. package/dist/context/llm/ai-sdk-runtime.js +32 -3
  40. package/dist/context/llm/claude-code-runtime.js +35 -2
  41. package/dist/context/llm/runtime-port.d.ts +25 -0
  42. package/dist/context/mcp/context-tools.d.ts +2 -1
  43. package/dist/context/mcp/context-tools.js +82 -15
  44. package/dist/context/mcp/server.js +4 -0
  45. package/dist/context/mcp/types.d.ts +15 -1
  46. package/dist/context/project/config.d.ts +1 -0
  47. package/dist/context/project/config.js +4 -0
  48. package/dist/context/project/driver-schemas.js +1 -1
  49. package/dist/context/search/discover.js +4 -3
  50. package/dist/context/sl/local-sl.d.ts +15 -0
  51. package/dist/context/sl/local-sl.js +30 -0
  52. package/dist/context/wiki/local-knowledge.d.ts +10 -0
  53. package/dist/context/wiki/local-knowledge.js +22 -0
  54. package/dist/context-build-view.d.ts +0 -3
  55. package/dist/context-build-view.js +1 -7
  56. package/dist/ingest.js +7 -10
  57. package/dist/knowledge.d.ts +5 -0
  58. package/dist/knowledge.js +10 -1
  59. package/dist/public-ingest-copy.js +1 -1
  60. package/dist/public-ingest.d.ts +0 -7
  61. package/dist/public-ingest.js +20 -34
  62. package/dist/setup-context.js +6 -38
  63. package/dist/setup-databases.js +13 -82
  64. package/dist/setup-sources.js +33 -5
  65. package/dist/setup.js +2 -2
  66. package/dist/skills/analytics/SKILL.md +6 -1
  67. package/dist/sl.d.ts +6 -1
  68. package/dist/sl.js +32 -8
  69. package/dist/telemetry/emitter.js +1 -1
  70. package/dist/telemetry/events.d.ts +4 -3
  71. package/dist/telemetry/events.js +7 -3
  72. package/dist/telemetry/identity.d.ts +1 -1
  73. package/dist/telemetry/identity.js +13 -10
  74. package/dist/telemetry/index.d.ts +1 -1
  75. package/dist/telemetry/index.js +5 -1
  76. package/package.json +22 -22
  77. package/dist/ingest-depth.d.ts +0 -8
  78. package/dist/ingest-depth.js +0 -56
  79. package/dist/setup-database-context-depth.d.ts +0 -23
  80. package/dist/setup-database-context-depth.js +0 -84
@@ -7,6 +7,7 @@ import { KtxYamlMetabaseSourceStateReader, LocalMetabaseDiscoveryCache } from '.
7
7
  import { localPullConfigForAdapter } from './local-adapters.js';
8
8
  import { createLocalBundleIngestRuntime } from './local-bundle-runtime.js';
9
9
  import { buildSyncId } from './raw-sources-paths.js';
10
+ import { ingestReportOutcome } from './reports.js';
10
11
  import { SqliteBundleIngestStore } from './sqlite-bundle-ingest-store.js';
11
12
  class LocalIngestPhase {
12
13
  async updateProgress() { }
@@ -117,11 +118,11 @@ export async function runLocalIngest(options) {
117
118
  return { result, report };
118
119
  }
119
120
  function metabaseFanoutStatus(children) {
120
- const succeeded = children.filter((child) => child.report.body.failedWorkUnits.length === 0).length;
121
- if (succeeded === children.length) {
121
+ const outcomes = children.map((child) => ingestReportOutcome(child.report));
122
+ if (outcomes.every((outcome) => outcome === 'done')) {
122
123
  return 'all_succeeded';
123
124
  }
124
- if (succeeded === 0) {
125
+ if (outcomes.every((outcome) => outcome === 'error')) {
125
126
  return 'all_failed';
126
127
  }
127
128
  return 'partial_failure';
@@ -266,12 +267,13 @@ export async function runLocalMetabaseIngest(options) {
266
267
  error,
267
268
  });
268
269
  }
270
+ const childOutcome = ingestReportOutcome(child.report);
269
271
  options.progress?.onMetabaseChildCompleted?.({
270
272
  metabaseConnectionId,
271
273
  metabaseDatabaseId: childPlan.metabaseDatabaseId,
272
274
  targetConnectionId,
273
275
  jobId: child.report.jobId,
274
- status: child.report.body.failedWorkUnits.length > 0 ? 'failed' : 'done',
276
+ status: childOutcome === 'error' ? 'failed' : childOutcome,
275
277
  });
276
278
  children.push({
277
279
  jobId: child.report.jobId,
@@ -1,3 +1,4 @@
1
+ import { ingestReportOutcome } from '../reports.js';
1
2
  function plannedWorkUnitFromLocal(workUnit) {
2
3
  return {
3
4
  unitKey: workUnit.unitKey,
@@ -39,7 +40,7 @@ function fullModeMetadata(input) {
39
40
  };
40
41
  }
41
42
  function reportStatus(report) {
42
- return report.body.failedWorkUnits.length > 0 ? 'error' : 'done';
43
+ return ingestReportOutcome(report) === 'error' ? 'error' : 'done';
43
44
  }
44
45
  function reportCreatedEvent(report) {
45
46
  return { type: 'report_created', runId: report.runId, reportPath: report.id };
@@ -111,6 +111,8 @@ interface IngestSettingsPort {
111
111
  workUnitMaxConcurrency?: number;
112
112
  workUnitStepBudget?: number;
113
113
  workUnitFailureMode?: 'abort' | 'continue';
114
+ /** Print a timing breakdown to stderr at the end of each run (config-driven; see also KTX_PROFILE_INGEST). `'json'` emits the raw structured profile. */
115
+ profileIngest?: boolean | 'json';
114
116
  ingestTraceLevel?: IngestTraceLevel;
115
117
  }
116
118
  interface IngestGitAuthor {
@@ -116,5 +116,8 @@ export interface IngestSavedMemoryCounts {
116
116
  slCount: number;
117
117
  }
118
118
  export declare function savedMemoryCountsForReport(report: IngestReportSnapshot): IngestSavedMemoryCounts;
119
+ /** @internal */
120
+ export type IngestReportOutcome = 'done' | 'partial' | 'error';
121
+ export declare function ingestReportOutcome(report: IngestReportSnapshot): IngestReportOutcome;
119
122
  export declare function buildStageIndexFromReportBody(jobId: string, connectionId: string, body: IngestReportBody): StageIndex;
120
123
  export {};
@@ -8,6 +8,16 @@ export function savedMemoryCountsForReport(report) {
8
8
  slCount: actions.filter((action) => action.target === 'sl').length,
9
9
  };
10
10
  }
11
+ export function ingestReportOutcome(report) {
12
+ if (report.body.status === 'failed') {
13
+ return 'error';
14
+ }
15
+ if (report.body.failedWorkUnits.length === 0) {
16
+ return 'done';
17
+ }
18
+ const { wikiCount, slCount } = savedMemoryCountsForReport(report);
19
+ return wikiCount + slCount > 0 ? 'partial' : 'error';
20
+ }
11
21
  export function buildStageIndexFromReportBody(jobId, connectionId, body) {
12
22
  return {
13
23
  jobId,
@@ -1,5 +1,5 @@
1
1
  import type { KtxModelRole } from '../../../llm/types.js';
2
- import type { AgentRunnerPort, KtxRuntimeToolSet } from '../../../context/llm/runtime-port.js';
2
+ import type { AgentRunnerPort, KtxRuntimeToolSet, RunLoopMetrics } from '../../../context/llm/runtime-port.js';
3
3
  import type { CaptureSession, MemoryAction } from '../../../context/memory/types.js';
4
4
  import { type TouchedSlSource } from '../../../context/tools/touched-sl-sources.js';
5
5
  import type { WorkUnit } from '../types.js';
@@ -44,6 +44,8 @@ export interface WorkUnitOutcome {
44
44
  patchPath?: string;
45
45
  patchTouchedPaths?: string[];
46
46
  childWorktreePath?: string;
47
+ /** Timing and token metrics for the work-unit agent loop, used for ingest profiling. */
48
+ metrics?: RunLoopMetrics;
47
49
  }
48
50
  export declare function executeWorkUnit(deps: WorkUnitExecutionDeps, wu: WorkUnit): Promise<WorkUnitOutcome>;
49
51
  export {};
@@ -72,6 +72,7 @@ export async function executeWorkUnit(deps, wu) {
72
72
  touchedSlSources: [],
73
73
  slDisallowed: wu.slDisallowed,
74
74
  slDisallowedReason: wu.slDisallowedReason,
75
+ ...(runResult.metrics ? { metrics: runResult.metrics } : {}),
75
76
  };
76
77
  };
77
78
  if (runResult.stopReason === 'error') {
@@ -104,5 +105,6 @@ export async function executeWorkUnit(deps, wu) {
104
105
  touchedSlSources: touched,
105
106
  slDisallowed: wu.slDisallowed,
106
107
  slDisallowedReason: wu.slDisallowedReason,
108
+ ...(runResult.metrics ? { metrics: runResult.metrics } : {}),
107
109
  };
108
110
  }
@@ -1,4 +1,4 @@
1
- import type { AgentRunnerPort, KtxRuntimeToolSet } from '../../../context/llm/runtime-port.js';
1
+ import type { AgentRunnerPort, KtxRuntimeToolSet, RunLoopMetrics } from '../../../context/llm/runtime-port.js';
2
2
  import type { KtxModelRole } from '../../../llm/types.js';
3
3
  import type { EvictionUnit } from '../types.js';
4
4
  import type { StageIndex } from './stage-index.types.js';
@@ -24,5 +24,6 @@ export interface ReconciliationOutcome {
24
24
  skipped: boolean;
25
25
  stopReason?: 'budget' | 'natural' | 'error';
26
26
  error?: Error;
27
+ metrics?: RunLoopMetrics;
27
28
  }
28
29
  export declare function runReconciliationStage4(ctx: ReconciliationContext): Promise<ReconciliationOutcome>;
@@ -13,5 +13,5 @@ export async function runReconciliationStage4(ctx) {
13
13
  telemetryTags: { operationName: 'ingest-bundle-reconcile', source: ctx.sourceKey, jobId: ctx.jobId },
14
14
  onStepFinish: ctx.onStepFinish,
15
15
  });
16
- return { skipped: false, stopReason: run.stopReason, error: run.error };
16
+ return { skipped: false, stopReason: run.stopReason, error: run.error, ...(run.metrics ? { metrics: run.metrics } : {}) };
17
17
  }
@@ -30,4 +30,10 @@ interface ToolCallLoggerOptions {
30
30
  * effectively single-writer and lines land in call order.
31
31
  */
32
32
  export declare function wrapToolsWithLogger<T extends KtxRuntimeToolSet>(tools: T, logFilePath: string, wuKey: string, options?: ToolCallLoggerOptions): T;
33
+ /**
34
+ * Await all in-flight tool-call log writes (best-effort, bounded by `timeoutMs`
35
+ * so it can never hang a caller). Lets readers such as the ingest profiler see
36
+ * complete transcripts despite the fire-and-forget append design.
37
+ */
38
+ export declare function flushToolCallLogs(timeoutMs?: number): Promise<void>;
33
39
  export {};
@@ -59,8 +59,12 @@ export function wrapToolsWithLogger(tools, logFilePath, wuKey, options = {}) {
59
59
  }
60
60
  return wrapped;
61
61
  }
62
+ // Fire-and-forget appends are intentional (the agent hot path must never block
63
+ // or fail on logging), but readers like the ingest profiler need to know when
64
+ // the writes have settled. Track in-flight appends so a consumer can flush.
65
+ const pendingWrites = new Set();
62
66
  function appendEntry(path, entry) {
63
- void (async () => {
67
+ const write = (async () => {
64
68
  try {
65
69
  await mkdir(dirname(path), { recursive: true });
66
70
  await appendFile(path, `${safeStringify(entry)}\n`, 'utf-8');
@@ -69,6 +73,37 @@ function appendEntry(path, entry) {
69
73
  // best-effort
70
74
  }
71
75
  })();
76
+ pendingWrites.add(write);
77
+ void write.finally(() => pendingWrites.delete(write));
78
+ }
79
+ /**
80
+ * Await all in-flight tool-call log writes (best-effort, bounded by `timeoutMs`
81
+ * so it can never hang a caller). Lets readers such as the ingest profiler see
82
+ * complete transcripts despite the fire-and-forget append design.
83
+ */
84
+ export async function flushToolCallLogs(timeoutMs = 5000) {
85
+ const pending = [...pendingWrites];
86
+ if (pending.length === 0) {
87
+ return;
88
+ }
89
+ const settled = Promise.allSettled(pending).then(() => undefined);
90
+ if (timeoutMs <= 0) {
91
+ await settled;
92
+ return;
93
+ }
94
+ let timer;
95
+ const timeout = new Promise((resolve) => {
96
+ timer = setTimeout(resolve, timeoutMs);
97
+ timer.unref?.();
98
+ });
99
+ try {
100
+ await Promise.race([settled, timeout]);
101
+ }
102
+ finally {
103
+ if (timer) {
104
+ clearTimeout(timer);
105
+ }
106
+ }
72
107
  }
73
108
  function safeStringify(v) {
74
109
  try {
@@ -3,6 +3,16 @@ import { generateText, Output, stepCountIs } from 'ai';
3
3
  import { noopLogger } from '../../context/core/config.js';
4
4
  import { summarizeKtxLlmDebugRequest } from './debug-request-recorder.js';
5
5
  import { createAiSdkToolSet } from './runtime-tools.js';
6
+ function toLlmTokenUsage(usage) {
7
+ if (!usage) {
8
+ return {};
9
+ }
10
+ return {
11
+ ...(usage.inputTokens !== undefined ? { inputTokens: usage.inputTokens } : {}),
12
+ ...(usage.outputTokens !== undefined ? { outputTokens: usage.outputTokens } : {}),
13
+ ...(usage.totalTokens !== undefined ? { totalTokens: usage.totalTokens } : {}),
14
+ };
15
+ }
6
16
  function hasTools(tools) {
7
17
  return Object.keys(tools).length > 0;
8
18
  }
@@ -26,6 +36,7 @@ export class AiSdkKtxLlmRuntime {
26
36
  model,
27
37
  });
28
38
  const split = splitKtxSystemMessages(built.messages);
39
+ const startedAt = Date.now();
29
40
  const result = await generateText({
30
41
  model,
31
42
  temperature: input.temperature ?? 0,
@@ -40,6 +51,7 @@ export class AiSdkKtxLlmRuntime {
40
51
  }
41
52
  : {}),
42
53
  });
54
+ input.onMetrics?.({ totalMs: Date.now() - startedAt, usage: toLlmTokenUsage(result.totalUsage ?? result.usage) });
43
55
  if (typeof result.text !== 'string') {
44
56
  throw new Error('KTX LLM text generation returned no text');
45
57
  }
@@ -55,6 +67,7 @@ export class AiSdkKtxLlmRuntime {
55
67
  model,
56
68
  });
57
69
  const split = splitKtxSystemMessages(built.messages);
70
+ const startedAt = Date.now();
58
71
  const result = await generateText({
59
72
  model,
60
73
  temperature: input.temperature ?? 0,
@@ -70,6 +83,7 @@ export class AiSdkKtxLlmRuntime {
70
83
  : {}),
71
84
  output: Output.object({ schema: input.schema }),
72
85
  });
86
+ input.onMetrics?.({ totalMs: Date.now() - startedAt, usage: toLlmTokenUsage(result.totalUsage ?? result.usage) });
73
87
  if (result.output == null) {
74
88
  throw new Error('KTX LLM object generation returned no output');
75
89
  }
@@ -77,6 +91,8 @@ export class AiSdkKtxLlmRuntime {
77
91
  }
78
92
  async runAgentLoop(params) {
79
93
  let stepIndex = 0;
94
+ const startedAt = Date.now();
95
+ const stepBoundariesMs = [];
80
96
  try {
81
97
  const model = this.deps.llmProvider.getModel(params.modelRole);
82
98
  const tools = createAiSdkToolSet(params.toolSet);
@@ -98,7 +114,7 @@ export class AiSdkKtxLlmRuntime {
98
114
  messages: built.messages,
99
115
  tools: built.tools,
100
116
  }));
101
- await generateText({
117
+ const result = await generateText({
102
118
  model,
103
119
  temperature: 0,
104
120
  stopWhen: stepCountIs(params.stepBudget),
@@ -111,6 +127,7 @@ export class AiSdkKtxLlmRuntime {
111
127
  tools: built.tools,
112
128
  onStepFinish: async () => {
113
129
  stepIndex += 1;
130
+ stepBoundariesMs.push(Date.now() - startedAt);
114
131
  if (!params.onStepFinish) {
115
132
  return;
116
133
  }
@@ -122,12 +139,24 @@ export class AiSdkKtxLlmRuntime {
122
139
  }
123
140
  },
124
141
  });
125
- return { stopReason: 'natural' };
142
+ return {
143
+ stopReason: 'natural',
144
+ metrics: {
145
+ totalMs: Date.now() - startedAt,
146
+ stepCount: stepIndex,
147
+ stepBoundariesMs,
148
+ usage: toLlmTokenUsage(result.totalUsage ?? result.usage),
149
+ },
150
+ };
126
151
  }
127
152
  catch (error) {
128
153
  const err = error instanceof Error ? error : new Error(String(error));
129
154
  this.logger.warn(`[agent-runner] loop failed: ${err.message}`);
130
- return { stopReason: 'error', error: err };
155
+ return {
156
+ stopReason: 'error',
157
+ error: err,
158
+ metrics: { totalMs: Date.now() - startedAt, stepCount: stepIndex, stepBoundariesMs, usage: {} },
159
+ };
131
160
  }
132
161
  }
133
162
  }
@@ -4,6 +4,19 @@ import { noopLogger } from '../../context/core/config.js';
4
4
  import { createKtxClaudeCodeEnv } from './claude-code-env.js';
5
5
  import { resolveClaudeCodeModel } from './claude-code-models.js';
6
6
  import { createClaudeSdkTools, mcpToolIds } from './runtime-tools.js';
7
+ function claudeTokenUsage(result) {
8
+ const usage = result.usage;
9
+ if (!usage) {
10
+ return {};
11
+ }
12
+ const { input_tokens: inputTokens, output_tokens: outputTokens } = usage;
13
+ const totalTokens = inputTokens !== undefined && outputTokens !== undefined ? inputTokens + outputTokens : undefined;
14
+ return {
15
+ ...(inputTokens !== undefined ? { inputTokens } : {}),
16
+ ...(outputTokens !== undefined ? { outputTokens } : {}),
17
+ ...(totalTokens !== undefined ? { totalTokens } : {}),
18
+ };
19
+ }
7
20
  const BUILTIN_TOOLS = [
8
21
  'Agent',
9
22
  'Task',
@@ -168,6 +181,7 @@ export class ClaudeCodeKtxLlmRuntime {
168
181
  maxTurns: 1,
169
182
  tools: input.tools,
170
183
  });
184
+ const startedAt = Date.now();
171
185
  const result = await collectResult({
172
186
  query: this.runQuery,
173
187
  prompt: [input.system, input.prompt].filter(Boolean).join('\n\n'),
@@ -175,6 +189,7 @@ export class ClaudeCodeKtxLlmRuntime {
175
189
  allowedToolIds: new Set(mcpToolIds(input.tools ?? {})),
176
190
  expectedMcpServerNames: expectedMcpServerNames(input.tools),
177
191
  });
192
+ input.onMetrics?.({ totalMs: Date.now() - startedAt, usage: claudeTokenUsage(result) });
178
193
  const error = resultError(result);
179
194
  if (error) {
180
195
  throw error;
@@ -200,6 +215,7 @@ export class ClaudeCodeKtxLlmRuntime {
200
215
  }),
201
216
  outputFormat: { type: 'json_schema', schema: jsonSchema(input.schema) },
202
217
  };
218
+ const startedAt = Date.now();
203
219
  const result = await collectResult({
204
220
  query: this.runQuery,
205
221
  prompt: [input.system, input.prompt].filter(Boolean).join('\n\n'),
@@ -207,6 +223,7 @@ export class ClaudeCodeKtxLlmRuntime {
207
223
  allowedToolIds: new Set([...mcpToolIds(input.tools ?? {}), STRUCTURED_OUTPUT_TOOL_NAME]),
208
224
  expectedMcpServerNames: expectedMcpServerNames(input.tools),
209
225
  });
226
+ input.onMetrics?.({ totalMs: Date.now() - startedAt, usage: claudeTokenUsage(result) });
210
227
  const error = resultError(result);
211
228
  if (error) {
212
229
  throw error;
@@ -218,6 +235,8 @@ export class ClaudeCodeKtxLlmRuntime {
218
235
  }
219
236
  async runAgentLoop(params) {
220
237
  let stepIndex = 0;
238
+ const startedAt = Date.now();
239
+ const stepBoundariesMs = [];
221
240
  try {
222
241
  const options = baseOptions({
223
242
  projectDir: this.deps.projectDir,
@@ -234,6 +253,7 @@ export class ClaudeCodeKtxLlmRuntime {
234
253
  expectedMcpServerNames: expectedMcpServerNames(params.toolSet),
235
254
  onAssistantTurn: async () => {
236
255
  stepIndex += 1;
256
+ stepBoundariesMs.push(Date.now() - startedAt);
237
257
  if (!params.onStepFinish) {
238
258
  return;
239
259
  }
@@ -247,11 +267,24 @@ export class ClaudeCodeKtxLlmRuntime {
247
267
  });
248
268
  const stopReason = mapClaudeCodeStopReason(result);
249
269
  const error = resultError(result);
250
- return { stopReason, ...(stopReason === 'error' && error ? { error } : {}) };
270
+ return {
271
+ stopReason,
272
+ ...(stopReason === 'error' && error ? { error } : {}),
273
+ metrics: {
274
+ totalMs: Date.now() - startedAt,
275
+ stepCount: stepIndex,
276
+ stepBoundariesMs,
277
+ usage: claudeTokenUsage(result),
278
+ },
279
+ };
251
280
  }
252
281
  catch (error) {
253
282
  const err = error instanceof Error ? error : new Error(String(error));
254
- return { stopReason: 'error', error: err };
283
+ return {
284
+ stopReason: 'error',
285
+ error: err,
286
+ metrics: { totalMs: Date.now() - startedAt, stepCount: stepIndex, stepBoundariesMs, usage: {} },
287
+ };
255
288
  }
256
289
  }
257
290
  }
@@ -17,6 +17,22 @@ export interface RunLoopStepInfo {
17
17
  stepIndex: number;
18
18
  stepBudget: number;
19
19
  }
20
+ export interface LlmTokenUsage {
21
+ inputTokens?: number;
22
+ outputTokens?: number;
23
+ totalTokens?: number;
24
+ }
25
+ /** Timing and token metrics for a multi-step agent loop, used for ingest profiling. */
26
+ export interface RunLoopMetrics {
27
+ /** Wall-clock time around the whole `generateText` call, in milliseconds. */
28
+ totalMs: number;
29
+ /** Aggregate token usage across all steps. */
30
+ usage: LlmTokenUsage;
31
+ /** Number of agent steps (model round-trips) that actually ran. */
32
+ stepCount: number;
33
+ /** Wall-clock offset (ms from loop start) at which each step finished. */
34
+ stepBoundariesMs: number[];
35
+ }
20
36
  export interface RunLoopParams {
21
37
  modelRole: KtxModelRole;
22
38
  systemPrompt: string;
@@ -29,6 +45,7 @@ export interface RunLoopParams {
29
45
  export interface RunLoopResult {
30
46
  stopReason: RunLoopStopReason;
31
47
  error?: Error;
48
+ metrics?: RunLoopMetrics;
32
49
  }
33
50
  export interface KtxGenerateTextInput {
34
51
  role: KtxModelRole;
@@ -36,6 +53,10 @@ export interface KtxGenerateTextInput {
36
53
  system?: string;
37
54
  tools?: KtxRuntimeToolSet;
38
55
  temperature?: number;
56
+ onMetrics?: (metrics: {
57
+ totalMs: number;
58
+ usage: LlmTokenUsage;
59
+ }) => void;
39
60
  }
40
61
  export interface KtxGenerateObjectInput<TOutput, TSchema extends z.ZodType<TOutput>> {
41
62
  role: KtxModelRole;
@@ -44,6 +65,10 @@ export interface KtxGenerateObjectInput<TOutput, TSchema extends z.ZodType<TOutp
44
65
  tools?: KtxRuntimeToolSet;
45
66
  temperature?: number;
46
67
  schema: TSchema;
68
+ onMetrics?: (metrics: {
69
+ totalMs: number;
70
+ usage: LlmTokenUsage;
71
+ }) => void;
47
72
  }
48
73
  export interface KtxLlmRuntimePort {
49
74
  generateText(input: KtxGenerateTextInput): Promise<string>;
@@ -1,11 +1,12 @@
1
1
  import type { KtxCliIo } from '../../cli-runtime.js';
2
- import type { KtxMcpContextPorts, KtxMcpServerLike, KtxMcpToolResult, KtxMcpUserContext, NonArrayObject } from './types.js';
2
+ import type { KtxMcpClientInfo, KtxMcpContextPorts, KtxMcpServerLike, KtxMcpToolResult, KtxMcpUserContext, NonArrayObject } from './types.js';
3
3
  export interface RegisterKtxContextToolsDeps {
4
4
  server: KtxMcpServerLike;
5
5
  ports: KtxMcpContextPorts;
6
6
  userContext: KtxMcpUserContext;
7
7
  projectDir?: string;
8
8
  io?: KtxCliIo;
9
+ getClientInfo?: () => KtxMcpClientInfo | undefined;
9
10
  }
10
11
  /** @internal */
11
12
  export declare function jsonToolResult<T extends NonArrayObject>(structuredContent: T): KtxMcpToolResult<T>;
@@ -30,7 +30,7 @@ const toolDescriptions = {
30
30
  entity_details: 'Read table and column metadata from the latest live-database scan snapshot. Example: entity_details({ connectionId: "warehouse", entities: [{ table: { catalog: null, db: "public", name: "orders" }, columns: ["id"] }] }).',
31
31
  dictionary_search: 'Search profile-sampled warehouse values to locate likely source columns for business values. Example: dictionary_search({ values: ["Acme Corp"], connectionId: "warehouse" }).',
32
32
  sl_read_source: 'Read a semantic-layer YAML source by connection id and source name. Example: sl_read_source({ connectionId: "warehouse", sourceName: "orders" }).',
33
- sl_query: 'Execute a semantic-layer query and return rows, headers, generated SQL, and plan details. Example: sl_query({ connectionId: "warehouse", measures: ["orders.order_count"], dimensions: [{ field: "orders.created_at", granularity: "month" }] }).',
33
+ sl_query: 'Execute a semantic-layer query and return headers, rows, and total row count, plus correctness notes (e.g. compile-only or fan-out) when relevant. The generated SQL and full query plan are omitted by default; request them with include: ["sql"] and/or include: ["plan"]. Example: sl_query({ connectionId: "warehouse", measures: ["orders.order_count"], dimensions: [{ field: "orders.created_at", granularity: "month" }], include: ["sql"] }).',
34
34
  sql_execution: 'Execute one parser-validated read-only SQL query against a configured KTX connection. Example: sql_execution({ connectionId: "warehouse", sql: "select count(*) from public.orders", maxRows: 100 }).',
35
35
  memory_ingest: 'Ingest free-form markdown knowledge into durable KTX memory. Use this for business rules, metric definitions, schema gotchas, recurring findings, or explicit user requests to remember something. Example: memory_ingest({ connectionId: "warehouse", content: "ARR is reported in cents in this warehouse." }).',
36
36
  memory_ingest_status: 'Read the current or final status for a memory ingest run. Example: memory_ingest_status({ runId: "memory-run-1" }).',
@@ -38,7 +38,7 @@ const toolDescriptions = {
38
38
  const connectionListSchema = z.object({});
39
39
  const knowledgeSearchSchema = z.object({
40
40
  query: z.string().min(1).describe('Natural-language wiki search query, e.g. "revenue recognition policy".'),
41
- limit: z.number().int().min(1).max(50).default(10).describe('Maximum wiki pages to return. Defaults to 10.'),
41
+ limit: z.number().int().min(1).max(50).default(10).describe('Maximum wiki pages to return.'),
42
42
  });
43
43
  const knowledgeReadSchema = z.object({
44
44
  key: z.string().min(1).describe('Wiki page key returned by wiki_search, e.g. "global/revenue".'),
@@ -67,10 +67,7 @@ const slQueryOrderBySchema = z.object({
67
67
  .string()
68
68
  .min(1)
69
69
  .describe('Field/measure/dimension id to order by, e.g. "orders.created_at", a dimension key like "mart_nrr_quarterly.quarter_label", or a measure alias.'),
70
- direction: z
71
- .enum(['asc', 'desc'])
72
- .default('asc')
73
- .describe('Sort direction: "asc" or "desc". Defaults to "asc".'),
70
+ direction: z.enum(['asc', 'desc']).default('asc').describe('Sort direction for this field.'),
74
71
  });
75
72
  const slQuerySchema = z.object({
76
73
  connectionId: connectionIdSchema
@@ -93,8 +90,12 @@ const slQuerySchema = z.object({
93
90
  .array(slQueryOrderBySchema)
94
91
  .default([])
95
92
  .describe('Sort clauses. Use {field, direction?} entries.'),
96
- limit: z.number().int().min(0).default(1000).describe('Maximum rows to return. Defaults to 1000.'),
97
- include_empty: z.boolean().default(true).describe('Whether to include empty dimension groups. Defaults to true.'),
93
+ limit: z.number().int().min(0).default(1000).describe('Maximum rows to return.'),
94
+ include_empty: z.boolean().default(true).describe('Whether to include empty dimension groups.'),
95
+ include: z
96
+ .array(z.enum(['plan', 'sql']))
97
+ .default([])
98
+ .describe('Extra detail to attach to the response: "sql" for the generated SQL, "plan" for the full query plan.'),
98
99
  });
99
100
  const entityDetailsTableRefSchema = z.object({
100
101
  catalog: z.string().nullable().describe('Catalog/project/database. Use null when not applicable.'),
@@ -134,12 +135,12 @@ const discoverDataSchema = z.object({
134
135
  .optional()
135
136
  .describe('Optional connection id. Pass it when user intent pins a specific warehouse.'),
136
137
  kinds: z.array(discoverDataKindSchema.describe('Reference kind to include.')).optional().describe('Optional kind filter.'),
137
- limit: z.number().int().min(1).max(50).default(15).optional().describe('Maximum refs to return. Defaults to 15.'),
138
+ limit: z.number().int().min(1).max(50).default(10).optional().describe('Maximum refs to return.'),
138
139
  });
139
140
  const sqlExecutionSchema = z.object({
140
141
  connectionId: connectionIdSchema.describe('Connection id to execute against. Required for raw SQL.'),
141
142
  sql: z.string().min(1).describe('Parser-validated read-only SQL, e.g. "select count(*) from public.orders".'),
142
- maxRows: z.number().int().min(1).max(10_000).default(1000).optional().describe('Maximum rows to return. Defaults to 1000.'),
143
+ maxRows: z.number().int().min(1).max(10_000).default(1000).optional().describe('Maximum rows to return.'),
143
144
  });
144
145
  const memoryIngestSchema = z.object({
145
146
  content: z
@@ -198,10 +199,14 @@ const slReadSourceOutputSchema = z.object({
198
199
  const slQueryOutputSchema = z.object({
199
200
  connectionId: z.string().optional(),
200
201
  dialect: z.string().optional(),
201
- sql: z.string(),
202
202
  headers: z.array(z.string()),
203
203
  rows: z.array(z.array(z.unknown())),
204
204
  totalRows: z.number(),
205
+ // Correctness signals hoisted out of `plan` so they survive default projection (e.g. compile-only
206
+ // status, fan-out warnings). Present only when there is something to report.
207
+ notes: z.array(z.string()).optional(),
208
+ // Opt-in detail, attached only when requested via the `include` input.
209
+ sql: z.string().optional(),
205
210
  plan: unknownRecordSchema.optional(),
206
211
  });
207
212
  const entityDetailsSnapshotOutputSchema = z.object({
@@ -321,11 +326,54 @@ const memoryIngestStatusOutputSchema = z.object({
321
326
  });
322
327
  /** @internal */
323
328
  export function jsonToolResult(structuredContent) {
329
+ // Compact (non-indented) JSON: this `content` text is the copy the model reads. Pretty-printing
330
+ // arrays-of-arrays (every `rows` payload) puts one scalar per line, inflating tabular results by
331
+ // a large constant factor. `structuredContent` carries the same data for structured-output clients.
324
332
  return {
325
- content: [{ type: 'text', text: JSON.stringify(structuredContent, null, 2) }],
333
+ content: [{ type: 'text', text: JSON.stringify(structuredContent) }],
326
334
  structuredContent,
327
335
  };
328
336
  }
337
+ /**
338
+ * Pull the correctness-critical signals out of a query plan so they survive even when the caller
339
+ * did not opt into the full `plan`. Returns an empty list when there is nothing to flag.
340
+ */
341
+ function slQueryNotes(plan) {
342
+ if (!plan) {
343
+ return [];
344
+ }
345
+ const notes = [];
346
+ const execution = plan.execution;
347
+ if (execution &&
348
+ typeof execution === 'object' &&
349
+ execution.mode === 'compile_only') {
350
+ const reason = execution.reason;
351
+ notes.push(typeof reason === 'string' ? reason : 'Compiled SQL only; no rows were executed.');
352
+ }
353
+ if (plan.has_fan_out === true) {
354
+ const description = typeof plan.fan_out_description === 'string' ? plan.fan_out_description.trim() : '';
355
+ notes.push(description.length > 0 ? description : 'Fan-out detected: measure totals may be inflated by joins.');
356
+ }
357
+ return notes;
358
+ }
359
+ /**
360
+ * Default sl_query response is the minimum the agent needs to read the result: connection, headers,
361
+ * rows, totals, plus any correctness notes. The generated `sql` and the full `plan` are attached only
362
+ * when explicitly requested via `include`, since both are large and echo information the caller already has.
363
+ */
364
+ function projectSlQueryResult(result, include) {
365
+ const notes = slQueryNotes(result.plan);
366
+ return {
367
+ ...(result.connectionId !== undefined ? { connectionId: result.connectionId } : {}),
368
+ ...(result.dialect !== undefined ? { dialect: result.dialect } : {}),
369
+ headers: result.headers,
370
+ rows: result.rows,
371
+ totalRows: result.totalRows,
372
+ ...(notes.length > 0 ? { notes } : {}),
373
+ ...(include.includes('sql') ? { sql: result.sql } : {}),
374
+ ...(include.includes('plan') && result.plan ? { plan: result.plan } : {}),
375
+ };
376
+ }
329
377
  function jsonErrorToolResult(text) {
330
378
  return {
331
379
  content: [{ type: 'text', text }],
@@ -367,6 +415,18 @@ function registerParsedTool(server, name, config, schema, handler) {
367
415
  }
368
416
  });
369
417
  }
418
+ /**
419
+ * Resolves the connected client's identity into the raw telemetry fields. The
420
+ * strings are client-controlled and untrusted, so they only ever land in the
421
+ * telemetry property bag — never in paths, logs, or error messages.
422
+ */
423
+ function clientTelemetryFields(getClientInfo) {
424
+ const client = getClientInfo?.();
425
+ return {
426
+ ...(client?.name ? { mcpClientName: client.name } : {}),
427
+ ...(client?.version ? { mcpClientVersion: client.version } : {}),
428
+ };
429
+ }
370
430
  function instrumentMcpServer(server, telemetry) {
371
431
  return {
372
432
  registerTool(name, config, handler) {
@@ -385,6 +445,7 @@ function instrumentMcpServer(server, telemetry) {
385
445
  outcome: isError ? 'error' : 'ok',
386
446
  durationMs: Math.max(0, performance.now() - startedAt),
387
447
  sampleRate: mcpTelemetrySampleRate(),
448
+ ...clientTelemetryFields(telemetry.getClientInfo),
388
449
  },
389
450
  });
390
451
  }
@@ -403,6 +464,7 @@ function instrumentMcpServer(server, telemetry) {
403
464
  ...(errorClass ? { errorClass } : {}),
404
465
  durationMs: Math.max(0, performance.now() - startedAt),
405
466
  sampleRate: mcpTelemetrySampleRate(),
467
+ ...clientTelemetryFields(telemetry.getClientInfo),
406
468
  },
407
469
  });
408
470
  }
@@ -414,7 +476,11 @@ function instrumentMcpServer(server, telemetry) {
414
476
  }
415
477
  export function registerKtxContextTools(deps) {
416
478
  const { ports, userContext } = deps;
417
- const server = instrumentMcpServer(deps.server, { projectDir: deps.projectDir, io: deps.io });
479
+ const server = instrumentMcpServer(deps.server, {
480
+ projectDir: deps.projectDir,
481
+ io: deps.io,
482
+ getClientInfo: deps.getClientInfo,
483
+ });
418
484
  if (ports.connections) {
419
485
  const connections = ports.connections;
420
486
  registerParsedTool(server, 'connection_list', {
@@ -471,7 +537,7 @@ export function registerKtxContextTools(deps) {
471
537
  annotations: toolAnnotations.sl_query,
472
538
  }, slQuerySchema, async (input, context) => {
473
539
  const onProgress = mcpProgressCallback(context);
474
- return jsonToolResult(await semanticLayer.query({
540
+ const result = await semanticLayer.query({
475
541
  connectionId: input.connectionId,
476
542
  query: {
477
543
  measures: input.measures,
@@ -482,7 +548,8 @@ export function registerKtxContextTools(deps) {
482
548
  limit: input.limit,
483
549
  include_empty: input.include_empty,
484
550
  },
485
- }, onProgress ? { onProgress } : undefined));
551
+ }, onProgress ? { onProgress } : undefined);
552
+ return jsonToolResult(projectSlQueryResult(result, input.include));
486
553
  });
487
554
  }
488
555
  if (ports.entityDetails) {