@kaelio/ktx 0.8.0 → 0.9.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (83) hide show
  1. package/assets/python/{kaelio_ktx-0.8.0-py3-none-any.whl → kaelio_ktx-0.9.0-py3-none-any.whl} +0 -0
  2. package/assets/python/manifest.json +4 -4
  3. package/dist/.tsbuildinfo +1 -1
  4. package/dist/cli-runtime.js +50 -3
  5. package/dist/commands/setup-commands.js +1 -1
  6. package/dist/connection-recovery.d.ts +34 -0
  7. package/dist/connection-recovery.js +82 -0
  8. package/dist/connection.js +3 -1
  9. package/dist/context/ingest/adapters/historic-sql/bigquery-query-history-reader.js +71 -20
  10. package/dist/context/ingest/adapters/historic-sql/chunk-unified.js +2 -1
  11. package/dist/context/ingest/adapters/historic-sql/connection-dialect.d.ts +9 -0
  12. package/dist/context/ingest/adapters/historic-sql/connection-dialect.js +15 -4
  13. package/dist/context/ingest/adapters/historic-sql/pattern-inputs.js +8 -2
  14. package/dist/context/ingest/adapters/historic-sql/query-history-filter-picker.d.ts +29 -0
  15. package/dist/context/ingest/adapters/historic-sql/query-history-filter-picker.js +190 -0
  16. package/dist/context/ingest/adapters/historic-sql/scope-floor.d.ts +18 -0
  17. package/dist/context/ingest/adapters/historic-sql/scope-floor.js +229 -0
  18. package/dist/context/ingest/adapters/historic-sql/scope-membership.d.ts +8 -0
  19. package/dist/context/ingest/adapters/historic-sql/scope-membership.js +29 -0
  20. package/dist/context/ingest/adapters/historic-sql/snowflake-query-history-reader.js +68 -19
  21. package/dist/context/ingest/adapters/historic-sql/stage-unified.js +57 -50
  22. package/dist/context/ingest/adapters/historic-sql/types.d.ts +36 -3
  23. package/dist/context/ingest/adapters/historic-sql/types.js +14 -2
  24. package/dist/context/ingest/context-evidence/sqlite-context-evidence-store.d.ts +1 -1
  25. package/dist/context/ingest/isolated-diff/patch-integrator.js +75 -5
  26. package/dist/context/ingest/local-adapters.js +21 -4
  27. package/dist/context/ingest/local-bundle-runtime.js +3 -2
  28. package/dist/context/llm/codex-exec-events.d.ts +20 -0
  29. package/dist/context/llm/codex-exec-events.js +155 -0
  30. package/dist/context/llm/codex-isolation.d.ts +3 -0
  31. package/dist/context/llm/codex-isolation.js +5 -0
  32. package/dist/context/llm/codex-mcp-runtime-server.d.ts +24 -0
  33. package/dist/context/llm/codex-mcp-runtime-server.js +51 -0
  34. package/dist/context/llm/codex-models.d.ts +2 -0
  35. package/dist/context/llm/codex-models.js +17 -0
  36. package/dist/context/llm/codex-runtime-config.d.ts +16 -0
  37. package/dist/context/llm/codex-runtime-config.js +19 -0
  38. package/dist/context/llm/codex-runtime.d.ts +37 -0
  39. package/dist/context/llm/codex-runtime.js +304 -0
  40. package/dist/context/llm/codex-sdk-runner.d.ts +21 -0
  41. package/dist/context/llm/codex-sdk-runner.js +63 -0
  42. package/dist/context/llm/local-config.d.ts +2 -0
  43. package/dist/context/llm/local-config.js +12 -1
  44. package/dist/context/project/config.d.ts +2 -0
  45. package/dist/context/project/config.js +2 -2
  46. package/dist/context/sql-analysis/http-sql-analysis-port.js +32 -2
  47. package/dist/context/sql-analysis/ports.d.ts +12 -2
  48. package/dist/context/tools/context-candidate-mark.tool.d.ts +2 -2
  49. package/dist/context-build-view.js +4 -32
  50. package/dist/io/buffered-command-io.d.ts +11 -0
  51. package/dist/io/buffered-command-io.js +28 -0
  52. package/dist/llm/types.d.ts +1 -1
  53. package/dist/local-adapters.d.ts +10 -2
  54. package/dist/local-adapters.js +19 -3
  55. package/dist/next-steps.js +1 -2
  56. package/dist/progress-port-adapter.d.ts +6 -0
  57. package/dist/progress-port-adapter.js +18 -0
  58. package/dist/public-ingest.d.ts +20 -1
  59. package/dist/public-ingest.js +178 -27
  60. package/dist/scan.js +3 -1
  61. package/dist/setup-context.d.ts +2 -0
  62. package/dist/setup-context.js +133 -27
  63. package/dist/setup-databases.d.ts +17 -1
  64. package/dist/setup-databases.js +358 -249
  65. package/dist/setup-models.d.ts +10 -1
  66. package/dist/setup-models.js +90 -2
  67. package/dist/setup-ready-menu.d.ts +16 -2
  68. package/dist/setup-ready-menu.js +37 -5
  69. package/dist/setup-sources.js +108 -28
  70. package/dist/setup.js +22 -10
  71. package/dist/status-project.d.ts +11 -0
  72. package/dist/status-project.js +50 -1
  73. package/dist/telemetry/command-hook.d.ts +1 -0
  74. package/dist/telemetry/command-hook.js +3 -1
  75. package/dist/telemetry/events.d.ts +11 -6
  76. package/dist/telemetry/events.js +10 -2
  77. package/dist/telemetry/identity.d.ts +0 -1
  78. package/dist/telemetry/identity.js +6 -6
  79. package/dist/telemetry/index.d.ts +12 -0
  80. package/dist/telemetry/index.js +13 -2
  81. package/dist/telemetry/scrubber.d.ts +10 -0
  82. package/dist/telemetry/scrubber.js +20 -0
  83. package/package.json +5 -4
@@ -0,0 +1,28 @@
1
+ /**
2
+ * Captures stdout/stderr from a command (e.g. `runKtxConnection`) into buffers
3
+ * instead of the terminal. Callers decide whether to flush the captured text to
4
+ * the user or discard it.
5
+ */
6
+ export function createBufferedCommandIo() {
7
+ let stdout = '';
8
+ let stderr = '';
9
+ return {
10
+ stdout: {
11
+ isTTY: false,
12
+ write(chunk) {
13
+ stdout += chunk;
14
+ },
15
+ },
16
+ stderr: {
17
+ write(chunk) {
18
+ stderr += chunk;
19
+ },
20
+ },
21
+ stdoutText() {
22
+ return stdout;
23
+ },
24
+ stderrText() {
25
+ return stderr;
26
+ },
27
+ };
28
+ }
@@ -1,7 +1,7 @@
1
1
  import type { LanguageModel, TelemetrySettings, ToolCallRepairFunction, ToolSet } from 'ai';
2
2
  export declare const KTX_MODEL_ROLES: readonly ["default", "triage", "candidateExtraction", "curator", "reconcile", "repair"];
3
3
  export type KtxModelRole = (typeof KTX_MODEL_ROLES)[number];
4
- type KtxLlmBackend = 'anthropic' | 'vertex' | 'gateway' | 'claude-code';
4
+ type KtxLlmBackend = 'anthropic' | 'vertex' | 'gateway' | 'claude-code' | 'codex';
5
5
  export type KtxPromptCacheTtl = '5m' | '1h';
6
6
  type KtxJsonValue = null | string | number | boolean | KtxJsonValue[] | {
7
7
  [key: string]: KtxJsonValue | undefined;
@@ -1,14 +1,22 @@
1
1
  import { type DefaultLocalIngestAdaptersOptions } from './context/ingest/local-adapters.js';
2
+ import type { HistoricSqlDialect, HistoricSqlReader } from './context/ingest/adapters/historic-sql/types.js';
2
3
  import type { SourceAdapter } from './context/ingest/types.js';
3
4
  import type { KtxLocalProject } from './context/project/project.js';
4
5
  import type { SqlAnalysisPort } from './context/sql-analysis/ports.js';
5
- import { type ManagedPythonCoreDaemonOptions } from './managed-python-http.js';
6
+ import { type ManagedPythonDaemonHttpOptions } from './managed-python-http.js';
6
7
  import type { KtxOperationalLogger } from './io/logger.js';
7
8
  export interface KtxCliLocalIngestAdaptersOptions extends DefaultLocalIngestAdaptersOptions {
8
9
  historicSqlConnectionId?: string;
9
10
  sqlAnalysis?: SqlAnalysisPort;
10
11
  sqlAnalysisUrl?: string;
11
- managedDaemon?: ManagedPythonCoreDaemonOptions;
12
+ managedDaemon?: ManagedPythonDaemonHttpOptions;
12
13
  logger?: KtxOperationalLogger;
13
14
  }
15
+ export interface KtxCliHistoricSqlRuntime {
16
+ dialect: HistoricSqlDialect;
17
+ sqlAnalysis: SqlAnalysisPort;
18
+ reader: HistoricSqlReader;
19
+ queryClient: unknown;
20
+ }
21
+ export declare function createKtxCliHistoricSqlRuntime(project: KtxLocalProject, connectionId: string, options?: KtxCliLocalIngestAdaptersOptions): KtxCliHistoricSqlRuntime | undefined;
14
22
  export declare function createKtxCliLocalIngestAdapters(project: KtxLocalProject, options?: KtxCliLocalIngestAdaptersOptions): SourceAdapter[];
@@ -12,7 +12,7 @@ import { isKtxSqliteConnectionConfig } from './connectors/sqlite/connector.js';
12
12
  import { createSqlServerLiveDatabaseIntrospection } from './connectors/sqlserver/live-database-introspection.js';
13
13
  import { isKtxSqlServerConnectionConfig } from './connectors/sqlserver/connector.js';
14
14
  import { BigQueryHistoricSqlQueryHistoryReader } from './context/ingest/adapters/historic-sql/bigquery-query-history-reader.js';
15
- import { queryHistoryDialectForConnection } from './context/ingest/adapters/historic-sql/connection-dialect.js';
15
+ import { historicSqlDialectForConnectionDriver } from './context/ingest/adapters/historic-sql/connection-dialect.js';
16
16
  import { createDaemonLiveDatabaseIntrospection } from './context/ingest/adapters/live-database/daemon-introspection.js';
17
17
  import { createDefaultLocalIngestAdapters } from './context/ingest/local-adapters.js';
18
18
  import { LiveDatabaseSourceAdapter } from './context/ingest/adapters/live-database/live-database.adapter.js';
@@ -224,7 +224,12 @@ function historicSqlOptionsForLocalRun(project, options) {
224
224
  return undefined;
225
225
  }
226
226
  const connection = project.config.connections[connectionId];
227
- const dialect = queryHistoryDialectForConnection(connection);
227
+ // historicSqlConnectionId is only set when query history was explicitly
228
+ // requested for this run (e.g. `--query-history`), so resolve the dialect from
229
+ // driver capability rather than the persisted context.queryHistory.enabled
230
+ // flag — otherwise the adapter is missing and findAdapter('historic-sql')
231
+ // throws even though the run asked for it.
232
+ const dialect = historicSqlDialectForConnectionDriver(connection);
228
233
  if (!dialect) {
229
234
  return undefined;
230
235
  }
@@ -234,6 +239,7 @@ function historicSqlOptionsForLocalRun(project, options) {
234
239
  if (dialect === 'postgres') {
235
240
  return {
236
241
  ...base,
242
+ dialect,
237
243
  reader: new PostgresPgssReader(),
238
244
  queryClient: createEphemeralPostgresHistoricSqlClient(project, connectionId),
239
245
  };
@@ -245,6 +251,7 @@ function historicSqlOptionsForLocalRun(project, options) {
245
251
  }
246
252
  return {
247
253
  ...base,
254
+ dialect,
248
255
  reader: new BigQueryHistoricSqlQueryHistoryReader({
249
256
  projectId: bigQueryProjectId(connection, process.env),
250
257
  region: bigQueryRegion(connection),
@@ -254,6 +261,7 @@ function historicSqlOptionsForLocalRun(project, options) {
254
261
  }
255
262
  return {
256
263
  ...base,
264
+ dialect,
257
265
  reader: new SnowflakeHistoricSqlQueryHistoryReader(),
258
266
  queryClient: {
259
267
  async executeQuery(query) {
@@ -264,8 +272,16 @@ function historicSqlOptionsForLocalRun(project, options) {
264
272
  },
265
273
  };
266
274
  }
275
+ export function createKtxCliHistoricSqlRuntime(project, connectionId, options = {}) {
276
+ return historicSqlOptionsForLocalRun(project, {
277
+ ...options,
278
+ historicSqlConnectionId: connectionId,
279
+ });
280
+ }
267
281
  export function createKtxCliLocalIngestAdapters(project, options = {}) {
268
- const historicSql = historicSqlOptionsForLocalRun(project, options);
282
+ const historicSql = options.historicSqlConnectionId
283
+ ? createKtxCliHistoricSqlRuntime(project, options.historicSqlConnectionId, options)
284
+ : undefined;
269
285
  const base = createDefaultLocalIngestAdapters(project, {
270
286
  ...options,
271
287
  databaseIntrospection: ktxCliDaemonDatabaseIntrospectionOptions(options),
@@ -53,8 +53,7 @@ export function formatSetupNextStepLines(state, indent = ' ') {
53
53
  }
54
54
  if (!state.contextReady) {
55
55
  return [
56
- `${indent}Build KTX context next.`,
57
- `${indent}Run ingest to build database schema context before context-source ingest.`,
56
+ `${indent}Setup is complete. The only step left is to build context for your agents.`,
58
57
  ...commandLines(KTX_CONTEXT_BUILD_COMMANDS, indent),
59
58
  ];
60
59
  }
@@ -0,0 +1,6 @@
1
+ import type { KtxProgressPort } from './context/scan/types.js';
2
+ import type { KtxIngestProgressUpdate } from './ingest.js';
3
+ export interface AggregateProgressState {
4
+ progress: number;
5
+ }
6
+ export declare function createAggregateProgressPort(onProgress: (update: KtxIngestProgressUpdate) => void, state?: AggregateProgressState, start?: number, weight?: number): KtxProgressPort;
@@ -0,0 +1,18 @@
1
+ export function createAggregateProgressPort(onProgress, state = { progress: 0 }, start = 0, weight = 1) {
2
+ return {
3
+ async update(value, message, options) {
4
+ const absoluteValue = start + Math.max(0, Math.min(1, value)) * weight;
5
+ state.progress = Math.max(state.progress, Math.min(1, absoluteValue));
6
+ if (!message)
7
+ return;
8
+ onProgress({
9
+ percent: Math.max(0, Math.min(100, Math.round(state.progress * 100))),
10
+ message,
11
+ ...(options?.transient !== undefined ? { transient: options.transient } : {}),
12
+ });
13
+ },
14
+ startPhase(phaseWeight) {
15
+ return createAggregateProgressPort(onProgress, state, state.progress, weight * phaseWeight);
16
+ },
17
+ };
18
+ }
@@ -5,6 +5,7 @@ import type { KtxIngestArgs, KtxIngestDeps, KtxIngestProgressUpdate } from './in
5
5
  import { type KtxManagedPythonInstallPolicy, type ManagedPythonCommandRuntime } from './managed-python-command.js';
6
6
  import type { KtxRuntimeFeature } from './managed-python-runtime.js';
7
7
  import type { KtxScanArgs, KtxScanDeps } from './scan.js';
8
+ import type { KtxTableRef } from './context/scan/types.js';
8
9
  type KtxPublicIngestStepName = 'database-schema' | 'query-history' | 'source-ingest' | 'memory-update';
9
10
  type KtxPublicIngestStepStatus = 'done' | 'skipped' | 'failed' | 'not-run';
10
11
  type KtxPublicIngestInputMode = 'auto' | 'disabled';
@@ -98,6 +99,17 @@ interface KtxPublicContextBuildArgs {
98
99
  cliVersion?: string;
99
100
  runtimeInstallPolicy?: KtxManagedPythonInstallPolicy;
100
101
  }
102
+ export declare function publicProgressMessage(message: string, target: KtxPublicIngestPlanTarget): string;
103
+ /** @internal */
104
+ export declare function queryHistoryPullConfig(input: {
105
+ stored: Record<string, unknown>;
106
+ dialect: HistoricSqlDialect;
107
+ windowDays?: number;
108
+ enabledTables?: KtxTableRef[];
109
+ enabledSchemas?: string[];
110
+ modeledTableCatalog?: KtxTableRef[];
111
+ scopeFloorWarnings?: string[];
112
+ }): Record<string, unknown>;
101
113
  export declare function buildPublicIngestPlan(project: KtxPublicIngestProject, args: {
102
114
  projectDir: string;
103
115
  targetConnectionId?: string;
@@ -108,8 +120,15 @@ export declare function buildPublicIngestPlan(project: KtxPublicIngestProject, a
108
120
  command: 'run';
109
121
  }>['mode'];
110
122
  }): KtxPublicIngestPlan;
123
+ /**
124
+ * Run one ingest target through its scan/ingest steps. The single per-target
125
+ * chokepoint reached by every entrypoint — standalone `ktx ingest` (plain/json
126
+ * and foreground) and `ktx setup` (via `runContextBuild`). The exported
127
+ * `executePublicIngestTarget` wraps this and emits the `ingest_completed`
128
+ * telemetry event exactly once, so every path is counted.
129
+ */
111
130
  export declare function executePublicIngestTarget(target: KtxPublicIngestPlanTarget, args: Extract<KtxPublicIngestArgs, {
112
131
  command: 'run';
113
- }>, io: KtxCliIo, deps: KtxPublicIngestDeps): Promise<KtxPublicIngestTargetResult>;
132
+ }>, io: KtxCliIo, deps: KtxPublicIngestDeps, project: KtxPublicIngestProject): Promise<KtxPublicIngestTargetResult>;
114
133
  export declare function runKtxPublicIngest(args: KtxPublicIngestArgs, io: KtxCliIo, deps?: KtxPublicIngestDeps): Promise<number>;
115
134
  export {};
@@ -1,12 +1,15 @@
1
1
  import { getKtxCliPackageInfo } from './cli-runtime.js';
2
2
  import { loadKtxProject } from './context/project/project.js';
3
3
  import { isDatabaseDriver, normalizeConnectionDriver } from './connection-drivers.js';
4
+ import { resolveQueryHistoryScopeFloor } from './context/ingest/adapters/historic-sql/scope-floor.js';
4
5
  import { ensureManagedPythonCommandRuntime, } from './managed-python-command.js';
5
- import { publicIngestOutputLine } from './public-ingest-copy.js';
6
+ import { publicDatabaseIngestMessage, publicIngestOutputLine, publicQueryHistoryMessage, } from './public-ingest-copy.js';
7
+ import { createAggregateProgressPort } from './progress-port-adapter.js';
6
8
  import { resolvePublicIngestRuntimeRequirements } from './runtime-requirements.js';
7
9
  import { profileMark } from './startup-profile.js';
8
10
  import { isDemoConnection } from './telemetry/demo-detect.js';
9
11
  import { emitProjectStackSnapshot, emitTelemetryEvent } from './telemetry/index.js';
12
+ import { formatErrorDetail } from './telemetry/scrubber.js';
10
13
  profileMark('module:public-ingest');
11
14
  const sourceAdapterByDriver = new Map([
12
15
  ['metabase', 'metabase'],
@@ -17,6 +20,16 @@ const sourceAdapterByDriver = new Map([
17
20
  ['dbt', 'dbt'],
18
21
  ['lookml', 'lookml'],
19
22
  ]);
23
+ export function publicProgressMessage(message, target) {
24
+ let current = message;
25
+ if (target.operation === 'database-ingest') {
26
+ current = publicDatabaseIngestMessage(current);
27
+ }
28
+ if (target.steps.includes('query-history')) {
29
+ current = publicQueryHistoryMessage(current, target.connectionId);
30
+ }
31
+ return current;
32
+ }
20
33
  const queryHistoryDialectByDriver = new Map([
21
34
  ['postgres', 'postgres'],
22
35
  ['bigquery', 'bigquery'],
@@ -99,20 +112,20 @@ function storedQueryHistory(connection) {
99
112
  function positiveInteger(value) {
100
113
  return typeof value === 'number' && Number.isInteger(value) && value > 0 ? value : undefined;
101
114
  }
102
- function enabledTablesForConnection(connection) {
103
- const raw = connection.enabled_tables;
104
- if (!Array.isArray(raw)) {
105
- return undefined;
106
- }
107
- const tables = raw.filter((value) => typeof value === 'string' && value.trim().length > 0);
108
- return tables.length > 0 ? tables : undefined;
109
- }
110
- function queryHistoryPullConfig(input) {
111
- const { enabled: _enabled, dialect: _dialect, ...storedConfig } = input.stored;
115
+ /** @internal */
116
+ export function queryHistoryPullConfig(input) {
117
+ const { enabled: _enabled, dialect: _dialect, enabledTables: _enabledTables, enabledSchemas: _enabledSchemas, scopeFloorWarnings: _scopeFloorWarnings, ...storedConfig } = input.stored;
112
118
  return {
113
119
  ...storedConfig,
114
120
  dialect: input.dialect,
115
- ...(input.enabledTables ? { enabledTables: input.enabledTables } : {}),
121
+ ...(input.enabledTables && input.enabledTables.length > 0 ? { enabledTables: input.enabledTables } : {}),
122
+ ...(input.enabledSchemas && input.enabledSchemas.length > 0 ? { enabledSchemas: input.enabledSchemas } : {}),
123
+ ...(input.modeledTableCatalog && input.modeledTableCatalog.length > 0
124
+ ? { modeledTableCatalog: input.modeledTableCatalog }
125
+ : {}),
126
+ ...(input.scopeFloorWarnings && input.scopeFloorWarnings.length > 0
127
+ ? { scopeFloorWarnings: input.scopeFloorWarnings }
128
+ : {}),
116
129
  ...(input.windowDays !== undefined ? { windowDays: input.windowDays } : {}),
117
130
  };
118
131
  }
@@ -157,7 +170,6 @@ function resolveDatabaseTargetOptions(input) {
157
170
  stored: storedQh,
158
171
  dialect,
159
172
  windowDays: queryHistory.windowDays,
160
- enabledTables: enabledTablesForConnection(input.connection),
161
173
  }),
162
174
  },
163
175
  steps: ['database-schema', 'query-history'],
@@ -168,6 +180,37 @@ function resolveDatabaseTargetOptions(input) {
168
180
  steps: ['database-schema'],
169
181
  };
170
182
  }
183
+ async function resolvedQueryHistoryPullConfigForTarget(target, project) {
184
+ if (target.operation !== 'database-ingest' || target.queryHistory?.enabled !== true || !target.queryHistory.dialect) {
185
+ return null;
186
+ }
187
+ const connection = project.config.connections[target.connectionId];
188
+ if (!connection) {
189
+ return (target.queryHistory.pullConfig ??
190
+ queryHistoryPullConfig({
191
+ stored: {},
192
+ dialect: target.queryHistory.dialect,
193
+ windowDays: target.queryHistory.windowDays,
194
+ }));
195
+ }
196
+ const stored = storedQueryHistory(connection);
197
+ const scopeFloor = await resolveQueryHistoryScopeFloor({
198
+ projectDir: project.projectDir,
199
+ connectionId: target.connectionId,
200
+ driver: target.driver,
201
+ connection: connection,
202
+ storedQueryHistory: stored,
203
+ });
204
+ return queryHistoryPullConfig({
205
+ stored,
206
+ dialect: target.queryHistory.dialect,
207
+ windowDays: target.queryHistory.windowDays,
208
+ enabledTables: scopeFloor.enabledTables,
209
+ enabledSchemas: scopeFloor.enabledSchemas,
210
+ modeledTableCatalog: scopeFloor.modeledTableCatalog,
211
+ scopeFloorWarnings: scopeFloor.warnings,
212
+ });
213
+ }
171
214
  function enrichmentReadinessGaps(config) {
172
215
  const gaps = [];
173
216
  if (config.llm.provider.backend === 'none' || !config.llm.models.default) {
@@ -348,6 +391,9 @@ function rowsBucket() {
348
391
  }
349
392
  async function emitIngestCompleted(input) {
350
393
  const failed = resultFailed(input.result);
394
+ const failureDetail = failed
395
+ ? formatErrorDetail(input.result.steps.find((step) => step.status === 'failed')?.detail)
396
+ : undefined;
351
397
  await emitTelemetryEvent({
352
398
  name: 'ingest_completed',
353
399
  projectDir: input.args.projectDir,
@@ -361,6 +407,7 @@ async function emitIngestCompleted(input) {
361
407
  rowsBucket: rowsBucket(),
362
408
  durationMs: Math.max(0, performance.now() - input.startedAt),
363
409
  outcome: failed ? 'error' : 'ok',
410
+ ...(failureDetail ? { errorDetail: failureDetail } : {}),
364
411
  },
365
412
  });
366
413
  }
@@ -427,6 +474,65 @@ function createCapturedPublicIngestIo() {
427
474
  },
428
475
  };
429
476
  }
477
+ function isCapturedPublicIngestIo(io) {
478
+ return typeof io.capturedOutput === 'function';
479
+ }
480
+ const PLAIN_PUBLIC_INGEST_PHASE_LABELS = {
481
+ 'database-schema': 'database schema',
482
+ 'query-history': 'query history',
483
+ 'source-ingest': 'source ingest',
484
+ };
485
+ function firstSummaryLine(summary) {
486
+ if (!summary)
487
+ return undefined;
488
+ return summary.split(/\r?\n/).find((line) => line.trim().length > 0)?.trim();
489
+ }
490
+ function plainPhaseHeader(options, phaseKey) {
491
+ const prefix = options.total > 1 ? `[${options.index + 1}/${options.total}] ` : '';
492
+ return `${prefix}${options.target.connectionId} · ${PLAIN_PUBLIC_INGEST_PHASE_LABELS[phaseKey]}`;
493
+ }
494
+ function plainPhaseEndLine(status, summary) {
495
+ const firstLine = firstSummaryLine(summary);
496
+ return firstLine ? ` ${status} · ${firstLine}` : ` ${status}`;
497
+ }
498
+ function createPlainPublicIngestProgress(io, options) {
499
+ let currentPhase = null;
500
+ const startedPhases = new Set();
501
+ const lastPercentByPhase = new Map();
502
+ const startPhase = (phaseKey) => {
503
+ currentPhase = phaseKey;
504
+ startedPhases.add(phaseKey);
505
+ lastPercentByPhase.set(phaseKey, -1);
506
+ io.stderr.write(`${plainPhaseHeader(options, phaseKey)}\n`);
507
+ };
508
+ const ensurePhaseStarted = (phaseKey) => {
509
+ if (!startedPhases.has(phaseKey)) {
510
+ startPhase(phaseKey);
511
+ return;
512
+ }
513
+ currentPhase = phaseKey;
514
+ };
515
+ const emitProgress = (update) => {
516
+ if (currentPhase === null)
517
+ return;
518
+ const rounded = Math.max(0, Math.min(100, Math.round(update.percent)));
519
+ const lastPercent = lastPercentByPhase.get(currentPhase) ?? -1;
520
+ if (rounded <= lastPercent)
521
+ return;
522
+ lastPercentByPhase.set(currentPhase, rounded);
523
+ io.stderr.write(` [${rounded}%] ${publicProgressMessage(update.message, options.target)}\n`);
524
+ };
525
+ return {
526
+ onPhaseStart: startPhase,
527
+ onPhaseEnd(phaseKey, status, summary) {
528
+ ensurePhaseStarted(phaseKey);
529
+ io.stderr.write(`${plainPhaseEndLine(status, summary)}\n`);
530
+ currentPhase = null;
531
+ },
532
+ scanProgress: createAggregateProgressPort(emitProgress),
533
+ ingestProgress: emitProgress,
534
+ };
535
+ }
430
536
  const INTERNAL_STATUS_LINE_RE = /^(Report|Run|Job|Status|Adapter|Connection|Sync|Diff|Tasks|Work units|Failed tasks|Saved memory|Provenance rows):\s*/;
431
537
  const ACTIONABLE_FAILURE_LINE_RE = /^(Missing bundled Python runtime manifest|KTX Python runtime is required|KTX daemon HTTP|Error:|Failed\b|Could not\b|Cannot\b)/;
432
538
  const RUNTIME_BACKED_RETRY_LINE_RE = /^Then retry the runtime-backed KTX command\.?$/;
@@ -457,7 +563,23 @@ function capturedFailureMessage(output) {
457
563
  .filter((line) => line.startsWith('In a source checkout, build the local runtime assets with:'));
458
564
  return [firstLine, ...followupLines].join('\n');
459
565
  }
460
- export async function executePublicIngestTarget(target, args, io, deps) {
566
+ /**
567
+ * Run one ingest target through its scan/ingest steps. The single per-target
568
+ * chokepoint reached by every entrypoint — standalone `ktx ingest` (plain/json
569
+ * and foreground) and `ktx setup` (via `runContextBuild`). The exported
570
+ * `executePublicIngestTarget` wraps this and emits the `ingest_completed`
571
+ * telemetry event exactly once, so every path is counted.
572
+ */
573
+ export async function executePublicIngestTarget(target, args, io, deps, project) {
574
+ const startedAt = performance.now();
575
+ const result = await runIngestTargetSteps(target, args, io, deps, project);
576
+ // `io` may be a capture buffer for the scan/ingest step output; the telemetry
577
+ // debug echo belongs on the real user-facing stream, which callers expose as
578
+ // `deps.runtimeIo` (falling back to `io` when the step io is already real).
579
+ await emitIngestCompleted({ args, project, target, result, startedAt, io: deps.runtimeIo ?? io });
580
+ return result;
581
+ }
582
+ async function runIngestTargetSteps(target, args, io, deps, project) {
461
583
  if (target.preflightFailure) {
462
584
  if (target.operation === 'database-ingest') {
463
585
  deps.onPhaseEnd?.('database-schema', 'failed', target.preflightFailure);
@@ -475,7 +597,7 @@ export async function executePublicIngestTarget(target, args, io, deps) {
475
597
  ? {
476
598
  ...step,
477
599
  status: 'failed',
478
- detail: target.preflightFailure,
600
+ detail: `${target.connectionId} failed: ${target.preflightFailure}`,
479
601
  }
480
602
  : step),
481
603
  };
@@ -493,7 +615,11 @@ export async function executePublicIngestTarget(target, args, io, deps) {
493
615
  ...(args.runtimeInstallPolicy ? { runtimeInstallPolicy: args.runtimeInstallPolicy } : {}),
494
616
  };
495
617
  const runScan = deps.runScan ?? runKtxScan;
496
- const capturedScanIo = deps.scanProgress ? null : createCapturedPublicIngestIo();
618
+ const capturedScanIo = deps.scanProgress
619
+ ? isCapturedPublicIngestIo(io)
620
+ ? io
621
+ : null
622
+ : createCapturedPublicIngestIo();
497
623
  const scanIo = capturedScanIo ?? io;
498
624
  const scanDeps = {
499
625
  ...(deps.scanProgress ? { progress: deps.scanProgress } : {}),
@@ -512,6 +638,10 @@ export async function executePublicIngestTarget(target, args, io, deps) {
512
638
  if (target.queryHistory?.enabled === true) {
513
639
  const { runKtxIngest } = await import('./ingest.js');
514
640
  const runIngest = deps.runIngest ?? runKtxIngest;
641
+ const historicSqlPullConfigOverride = (await resolvedQueryHistoryPullConfigForTarget(target, project)) ?? {
642
+ dialect: target.queryHistory.dialect,
643
+ ...(target.queryHistory.windowDays !== undefined ? { windowDays: target.queryHistory.windowDays } : {}),
644
+ };
515
645
  const ingestArgs = {
516
646
  command: 'run',
517
647
  projectDir: args.projectDir,
@@ -522,12 +652,14 @@ export async function executePublicIngestTarget(target, args, io, deps) {
522
652
  ...(args.cliVersion ? { cliVersion: args.cliVersion } : {}),
523
653
  ...(args.runtimeInstallPolicy ? { runtimeInstallPolicy: args.runtimeInstallPolicy } : {}),
524
654
  allowImplicitAdapter: true,
525
- historicSqlPullConfigOverride: target.queryHistory.pullConfig ?? {
526
- dialect: target.queryHistory.dialect,
527
- ...(target.queryHistory.windowDays !== undefined ? { windowDays: target.queryHistory.windowDays } : {}),
528
- },
655
+ historicSqlPullConfigOverride,
529
656
  };
530
- const capturedIngestIo = deps.ingestProgress ? null : createCapturedPublicIngestIo();
657
+ // Query history runs after the schema scan has already written its report
658
+ // into the shared target io, so it needs a phase-local capture. Reusing
659
+ // `io` here would let leftover scan text (e.g. "Mode: enriched") surface as
660
+ // the query-history failure detail. Only skip capture when progress is
661
+ // active and the caller manages its own buffer (io is not a capture).
662
+ const capturedIngestIo = deps.ingestProgress && !isCapturedPublicIngestIo(io) ? null : createCapturedPublicIngestIo();
531
663
  const ingestIo = capturedIngestIo ?? io;
532
664
  const ingestDeps = {
533
665
  ...(deps.ingestProgress ? { progress: deps.ingestProgress } : {}),
@@ -564,7 +696,11 @@ export async function executePublicIngestTarget(target, args, io, deps) {
564
696
  allowImplicitAdapter: true,
565
697
  };
566
698
  const runIngest = deps.runIngest ?? runKtxIngest;
567
- const capturedIngestIo = deps.ingestProgress ? null : createCapturedPublicIngestIo();
699
+ const capturedIngestIo = deps.ingestProgress
700
+ ? isCapturedPublicIngestIo(io)
701
+ ? io
702
+ : null
703
+ : createCapturedPublicIngestIo();
568
704
  const ingestIo = capturedIngestIo ?? io;
569
705
  const ingestDeps = {
570
706
  ...(deps.ingestProgress ? { progress: deps.ingestProgress } : {}),
@@ -628,11 +764,26 @@ export async function runKtxPublicIngest(args, io, deps = {}) {
628
764
  io.stderr.write(`Warning: ${warning}\n`);
629
765
  }
630
766
  }
631
- for (const target of plan.targets) {
632
- const startedAt = performance.now();
633
- const result = await executePublicIngestTarget(target, args, io, deps);
634
- results.push(result);
635
- await emitIngestCompleted({ args, project, target, result, startedAt, io });
767
+ for (const [index, target] of plan.targets.entries()) {
768
+ if (args.json) {
769
+ results.push(await executePublicIngestTarget(target, args, io, deps, project));
770
+ continue;
771
+ }
772
+ const capture = createCapturedPublicIngestIo();
773
+ const progress = createPlainPublicIngestProgress(io, {
774
+ target,
775
+ index,
776
+ total: plan.targets.length,
777
+ });
778
+ const targetDeps = {
779
+ ...deps,
780
+ scanProgress: progress.scanProgress,
781
+ ingestProgress: progress.ingestProgress,
782
+ onPhaseStart: progress.onPhaseStart,
783
+ onPhaseEnd: progress.onPhaseEnd,
784
+ runtimeIo: deps.runtimeIo ?? io,
785
+ };
786
+ results.push(await executePublicIngestTarget(target, args, capture, targetDeps, project));
636
787
  }
637
788
  if (args.json) {
638
789
  io.stdout.write(`${JSON.stringify({ plan, results }, null, 2)}\n`);
package/dist/scan.js CHANGED
@@ -6,7 +6,7 @@ import { createKtxCliLocalIngestAdapters } from './local-adapters.js';
6
6
  import { createKtxCliScanConnector } from './local-scan-connectors.js';
7
7
  import { profileMark } from './startup-profile.js';
8
8
  import { emitTelemetryEvent } from './telemetry/index.js';
9
- import { scrubErrorClass } from './telemetry/scrubber.js';
9
+ import { formatErrorDetail, scrubErrorClass } from './telemetry/scrubber.js';
10
10
  profileMark('module:scan');
11
11
  function shouldUseStyledOutput(io) {
12
12
  return io.stdout.isTTY === true && !process.env.NO_COLOR && process.env.TERM !== 'dumb' && !process.env.CI;
@@ -306,6 +306,7 @@ export async function runKtxScan(args, io = process, deps = {}) {
306
306
  }
307
307
  catch (error) {
308
308
  const errorClass = scrubErrorClass(error);
309
+ const errorDetail = formatErrorDetail(error);
309
310
  await emitTelemetryEvent({
310
311
  name: 'scan_completed',
311
312
  projectDir: args.projectDir,
@@ -319,6 +320,7 @@ export async function runKtxScan(args, io = process, deps = {}) {
319
320
  durationMs: Math.max(0, performance.now() - startedAt),
320
321
  outcome: 'error',
321
322
  ...(errorClass ? { errorClass } : {}),
323
+ ...(errorDetail ? { errorDetail } : {}),
322
324
  },
323
325
  });
324
326
  io.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`);
@@ -54,6 +54,7 @@ export type KtxSetupContextResult = {
54
54
  } | {
55
55
  status: 'failed';
56
56
  projectDir: string;
57
+ errorDetail?: string;
57
58
  };
58
59
  export interface KtxSetupContextStepArgs {
59
60
  projectDir: string;
@@ -77,6 +78,7 @@ export interface KtxSetupContextDeps {
77
78
  now?: () => Date;
78
79
  runContextBuild?: typeof runContextBuild;
79
80
  verifyContextReady?: (projectDir: string) => Promise<KtxSetupContextReadiness>;
81
+ testConnection?: (projectDir: string, connectionId: string, io: KtxCliIo) => Promise<number>;
80
82
  }
81
83
  /** @internal */
82
84
  export declare function contextBuildCommands(projectDir: string): KtxSetupContextCommands;