@kaelio/ktx 0.8.0 → 0.10.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (183) hide show
  1. package/assets/python/{kaelio_ktx-0.8.0-py3-none-any.whl → kaelio_ktx-0.10.0-py3-none-any.whl} +0 -0
  2. package/assets/python/manifest.json +4 -4
  3. package/dist/.tsbuildinfo +1 -1
  4. package/dist/clack.d.ts +6 -0
  5. package/dist/clack.js +17 -2
  6. package/dist/cli-program.d.ts +3 -0
  7. package/dist/cli-program.js +42 -2
  8. package/dist/cli-runtime.d.ts +3 -0
  9. package/dist/cli-runtime.js +94 -3
  10. package/dist/commands/setup-commands.js +3 -4
  11. package/dist/connection-recovery.d.ts +34 -0
  12. package/dist/connection-recovery.js +82 -0
  13. package/dist/connection.js +26 -2
  14. package/dist/connectors/bigquery/connector.d.ts +2 -5
  15. package/dist/connectors/bigquery/connector.js +2 -2
  16. package/dist/connectors/clickhouse/connector.d.ts +2 -5
  17. package/dist/connectors/clickhouse/connector.js +2 -2
  18. package/dist/connectors/mysql/connector.d.ts +7 -6
  19. package/dist/connectors/mysql/connector.js +25 -5
  20. package/dist/connectors/mysql/dialect.d.ts +1 -1
  21. package/dist/connectors/mysql/dialect.js +12 -2
  22. package/dist/connectors/postgres/connector.d.ts +2 -5
  23. package/dist/connectors/postgres/connector.js +2 -2
  24. package/dist/connectors/snowflake/connector.d.ts +2 -5
  25. package/dist/connectors/snowflake/connector.js +2 -2
  26. package/dist/connectors/sqlite/connector.d.ts +2 -5
  27. package/dist/connectors/sqlite/connector.js +2 -2
  28. package/dist/connectors/sqlserver/connector.d.ts +2 -5
  29. package/dist/connectors/sqlserver/connector.js +2 -2
  30. package/dist/context/connections/drivers.d.ts +0 -1
  31. package/dist/context/connections/drivers.js +0 -7
  32. package/dist/context/connections/query-executor.d.ts +2 -1
  33. package/dist/context/core/abort.d.ts +9 -0
  34. package/dist/context/core/abort.js +36 -0
  35. package/dist/context/ingest/adapters/historic-sql/bigquery-query-history-reader.js +71 -20
  36. package/dist/context/ingest/adapters/historic-sql/chunk-unified.js +2 -1
  37. package/dist/context/ingest/adapters/historic-sql/connection-dialect.d.ts +9 -0
  38. package/dist/context/ingest/adapters/historic-sql/connection-dialect.js +15 -4
  39. package/dist/context/ingest/adapters/historic-sql/pattern-inputs.js +8 -2
  40. package/dist/context/ingest/adapters/historic-sql/query-history-filter-picker.d.ts +30 -0
  41. package/dist/context/ingest/adapters/historic-sql/query-history-filter-picker.js +194 -0
  42. package/dist/context/ingest/adapters/historic-sql/scope-floor.d.ts +18 -0
  43. package/dist/context/ingest/adapters/historic-sql/scope-floor.js +229 -0
  44. package/dist/context/ingest/adapters/historic-sql/scope-membership.d.ts +8 -0
  45. package/dist/context/ingest/adapters/historic-sql/scope-membership.js +29 -0
  46. package/dist/context/ingest/adapters/historic-sql/snowflake-query-history-reader.js +68 -19
  47. package/dist/context/ingest/adapters/historic-sql/stage-unified.js +57 -50
  48. package/dist/context/ingest/adapters/historic-sql/types.d.ts +36 -3
  49. package/dist/context/ingest/adapters/historic-sql/types.js +14 -2
  50. package/dist/context/ingest/context-candidates/curator-pagination.service.d.ts +1 -5
  51. package/dist/context/ingest/context-candidates/curator-pagination.service.js +1 -3
  52. package/dist/context/ingest/context-evidence/sqlite-context-evidence-store.d.ts +1 -1
  53. package/dist/context/ingest/final-gate-repair.d.ts +1 -0
  54. package/dist/context/ingest/final-gate-repair.js +1 -0
  55. package/dist/context/ingest/ingest-bundle.runner.d.ts +3 -0
  56. package/dist/context/ingest/ingest-bundle.runner.js +127 -53
  57. package/dist/context/ingest/isolated-diff/patch-integrator.js +75 -5
  58. package/dist/context/ingest/isolated-diff/textual-conflict-resolver.d.ts +1 -0
  59. package/dist/context/ingest/isolated-diff/textual-conflict-resolver.js +1 -0
  60. package/dist/context/ingest/isolated-diff/work-unit-executor.d.ts +1 -0
  61. package/dist/context/ingest/local-adapters.js +21 -4
  62. package/dist/context/ingest/local-bundle-runtime.js +13 -5
  63. package/dist/context/ingest/local-ingest.d.ts +1 -0
  64. package/dist/context/ingest/local-ingest.js +13 -3
  65. package/dist/context/ingest/memory-flow/events.js +1 -1
  66. package/dist/context/ingest/memory-flow/schema.js +8 -3
  67. package/dist/context/ingest/memory-flow/types.d.ts +7 -3
  68. package/dist/context/ingest/ports.d.ts +3 -5
  69. package/dist/context/ingest/stages/stage-3-work-units.d.ts +1 -4
  70. package/dist/context/ingest/stages/stage-3-work-units.js +5 -1
  71. package/dist/context/ingest/stages/stage-4-reconciliation.d.ts +1 -4
  72. package/dist/context/ingest/stages/stage-4-reconciliation.js +1 -1
  73. package/dist/context/ingest/types.d.ts +1 -0
  74. package/dist/context/llm/ai-sdk-runtime.d.ts +3 -0
  75. package/dist/context/llm/ai-sdk-runtime.js +152 -16
  76. package/dist/context/llm/claude-code-runtime.d.ts +6 -4
  77. package/dist/context/llm/claude-code-runtime.js +127 -48
  78. package/dist/context/llm/codex-exec-events.d.ts +20 -0
  79. package/dist/context/llm/codex-exec-events.js +155 -0
  80. package/dist/context/llm/codex-isolation.d.ts +3 -0
  81. package/dist/context/llm/codex-isolation.js +5 -0
  82. package/dist/context/llm/codex-mcp-runtime-server.d.ts +24 -0
  83. package/dist/context/llm/codex-mcp-runtime-server.js +51 -0
  84. package/dist/context/llm/codex-models.d.ts +2 -0
  85. package/dist/context/llm/codex-models.js +17 -0
  86. package/dist/context/llm/codex-runtime-config.d.ts +16 -0
  87. package/dist/context/llm/codex-runtime-config.js +19 -0
  88. package/dist/context/llm/codex-runtime.d.ts +37 -0
  89. package/dist/context/llm/codex-runtime.js +347 -0
  90. package/dist/context/llm/codex-sdk-runner.d.ts +21 -0
  91. package/dist/context/llm/codex-sdk-runner.js +63 -0
  92. package/dist/context/llm/local-config.d.ts +16 -4
  93. package/dist/context/llm/local-config.js +18 -2
  94. package/dist/context/llm/rate-limit-governor.d.ts +103 -0
  95. package/dist/context/llm/rate-limit-governor.js +285 -0
  96. package/dist/context/llm/runtime-port.d.ts +3 -6
  97. package/dist/context/mcp/context-tools.js +43 -13
  98. package/dist/context/project/config.d.ts +14 -0
  99. package/dist/context/project/config.js +37 -2
  100. package/dist/context/scan/types.d.ts +15 -2
  101. package/dist/context/scan/types.js +12 -0
  102. package/dist/context/sl/description-normalization.js +4 -14
  103. package/dist/context/sql-analysis/http-sql-analysis-port.js +32 -2
  104. package/dist/context/sql-analysis/ports.d.ts +12 -2
  105. package/dist/context/tools/context-candidate-mark.tool.d.ts +2 -2
  106. package/dist/context-build-view.d.ts +13 -0
  107. package/dist/context-build-view.js +63 -32
  108. package/dist/demo-metrics.d.ts +0 -2
  109. package/dist/demo-metrics.js +1 -11
  110. package/dist/ingest.d.ts +1 -0
  111. package/dist/ingest.js +32 -3
  112. package/dist/io/buffered-command-io.d.ts +11 -0
  113. package/dist/io/buffered-command-io.js +28 -0
  114. package/dist/io/symbols.d.ts +2 -0
  115. package/dist/io/symbols.js +2 -0
  116. package/dist/llm/types.d.ts +1 -1
  117. package/dist/local-adapters.d.ts +10 -2
  118. package/dist/local-adapters.js +19 -3
  119. package/dist/memory-flow-hud.js +8 -16
  120. package/dist/next-steps.js +1 -2
  121. package/dist/progress-port-adapter.d.ts +6 -0
  122. package/dist/progress-port-adapter.js +18 -0
  123. package/dist/public-ingest.d.ts +20 -1
  124. package/dist/public-ingest.js +228 -42
  125. package/dist/reveal-password-prompt.d.ts +24 -0
  126. package/dist/reveal-password-prompt.js +78 -0
  127. package/dist/scan.js +21 -3
  128. package/dist/setup-context.d.ts +2 -0
  129. package/dist/setup-context.js +133 -27
  130. package/dist/setup-databases.d.ts +18 -1
  131. package/dist/setup-databases.js +378 -249
  132. package/dist/setup-demo-tour.js +1 -0
  133. package/dist/setup-embeddings.js +1 -1
  134. package/dist/setup-models.d.ts +11 -15
  135. package/dist/setup-models.js +140 -276
  136. package/dist/setup-prompts.js +3 -2
  137. package/dist/setup-ready-menu.d.ts +16 -2
  138. package/dist/setup-ready-menu.js +37 -5
  139. package/dist/setup-sources.js +115 -35
  140. package/dist/setup.d.ts +1 -1
  141. package/dist/setup.js +23 -11
  142. package/dist/sl.d.ts +2 -2
  143. package/dist/sl.js +20 -4
  144. package/dist/sql.js +18 -2
  145. package/dist/star-prompt/cache.d.ts +16 -0
  146. package/dist/star-prompt/cache.js +45 -0
  147. package/dist/star-prompt/star-count.d.ts +7 -0
  148. package/dist/star-prompt/star-count.js +66 -0
  149. package/dist/star-prompt/star-line.d.ts +12 -0
  150. package/dist/star-prompt/star-line.js +26 -0
  151. package/dist/status-project.d.ts +11 -0
  152. package/dist/status-project.js +50 -1
  153. package/dist/telemetry/command-hook.d.ts +1 -0
  154. package/dist/telemetry/command-hook.js +3 -1
  155. package/dist/telemetry/emitter.d.ts +10 -0
  156. package/dist/telemetry/emitter.js +31 -0
  157. package/dist/telemetry/events.d.ts +35 -6
  158. package/dist/telemetry/events.js +25 -2
  159. package/dist/telemetry/exception.d.ts +18 -0
  160. package/dist/telemetry/exception.js +162 -0
  161. package/dist/telemetry/identity.d.ts +0 -1
  162. package/dist/telemetry/identity.js +6 -6
  163. package/dist/telemetry/index.d.ts +15 -2
  164. package/dist/telemetry/index.js +15 -3
  165. package/dist/telemetry/redaction-secrets.d.ts +11 -0
  166. package/dist/telemetry/redaction-secrets.js +92 -0
  167. package/dist/telemetry/scrubber.d.ts +10 -0
  168. package/dist/telemetry/scrubber.js +20 -0
  169. package/dist/update-check/cache.d.ts +21 -0
  170. package/dist/update-check/cache.js +38 -0
  171. package/dist/update-check/channel.d.ts +15 -0
  172. package/dist/update-check/channel.js +30 -0
  173. package/dist/update-check/registry.d.ts +1 -0
  174. package/dist/update-check/registry.js +45 -0
  175. package/dist/update-check/update-check.d.ts +43 -0
  176. package/dist/update-check/update-check.js +116 -0
  177. package/package.json +12 -4
  178. package/dist/context/connections/local-query-executor.d.ts +0 -6
  179. package/dist/context/connections/local-query-executor.js +0 -39
  180. package/dist/context/connections/postgres-query-executor.d.ts +0 -25
  181. package/dist/context/connections/postgres-query-executor.js +0 -53
  182. package/dist/context/connections/sqlite-query-executor.d.ts +0 -4
  183. package/dist/context/connections/sqlite-query-executor.js +0 -74
@@ -97,27 +97,19 @@ function sourceDescription(input) {
97
97
  }
98
98
  function activeWorkUnits(input) {
99
99
  const finishedKeys = new Set();
100
- const unitMap = new Map();
101
100
  for (const e of input.events) {
102
- if (e.type === 'work_unit_started') {
103
- unitMap.set(e.unitKey, { stepIndex: 0, stepBudget: e.stepBudget });
104
- }
105
- if (e.type === 'work_unit_step') {
106
- const existing = unitMap.get(e.unitKey);
107
- if (existing) {
108
- existing.stepIndex = e.stepIndex;
109
- existing.stepBudget = e.stepBudget;
110
- }
111
- }
112
101
  if (e.type === 'work_unit_finished')
113
102
  finishedKeys.add(e.unitKey);
114
103
  }
115
- const result = [];
116
- for (const [unitKey, data] of unitMap) {
117
- if (!finishedKeys.has(unitKey))
118
- result.push({ unitKey, ...data });
104
+ const active = [];
105
+ const seen = new Set();
106
+ for (const e of input.events) {
107
+ if (e.type === 'work_unit_started' && !finishedKeys.has(e.unitKey) && !seen.has(e.unitKey)) {
108
+ seen.add(e.unitKey);
109
+ active.push(e.unitKey);
110
+ }
119
111
  }
120
- return result;
112
+ return active;
121
113
  }
122
114
  function queuedWorkUnits(input) {
123
115
  const startedKeys = new Set();
@@ -53,8 +53,7 @@ export function formatSetupNextStepLines(state, indent = ' ') {
53
53
  }
54
54
  if (!state.contextReady) {
55
55
  return [
56
- `${indent}Build KTX context next.`,
57
- `${indent}Run ingest to build database schema context before context-source ingest.`,
56
+ `${indent}Setup is complete. The only step left is to build context for your agents.`,
58
57
  ...commandLines(KTX_CONTEXT_BUILD_COMMANDS, indent),
59
58
  ];
60
59
  }
@@ -0,0 +1,6 @@
1
+ import type { KtxProgressPort } from './context/scan/types.js';
2
+ import type { KtxIngestProgressUpdate } from './ingest.js';
3
+ export interface AggregateProgressState {
4
+ progress: number;
5
+ }
6
+ export declare function createAggregateProgressPort(onProgress: (update: KtxIngestProgressUpdate) => void, state?: AggregateProgressState, start?: number, weight?: number): KtxProgressPort;
@@ -0,0 +1,18 @@
1
+ export function createAggregateProgressPort(onProgress, state = { progress: 0 }, start = 0, weight = 1) {
2
+ return {
3
+ async update(value, message, options) {
4
+ const absoluteValue = start + Math.max(0, Math.min(1, value)) * weight;
5
+ state.progress = Math.max(state.progress, Math.min(1, absoluteValue));
6
+ if (!message)
7
+ return;
8
+ onProgress({
9
+ percent: Math.max(0, Math.min(100, Math.round(state.progress * 100))),
10
+ message,
11
+ ...(options?.transient !== undefined ? { transient: options.transient } : {}),
12
+ });
13
+ },
14
+ startPhase(phaseWeight) {
15
+ return createAggregateProgressPort(onProgress, state, state.progress, weight * phaseWeight);
16
+ },
17
+ };
18
+ }
@@ -5,6 +5,7 @@ import type { KtxIngestArgs, KtxIngestDeps, KtxIngestProgressUpdate } from './in
5
5
  import { type KtxManagedPythonInstallPolicy, type ManagedPythonCommandRuntime } from './managed-python-command.js';
6
6
  import type { KtxRuntimeFeature } from './managed-python-runtime.js';
7
7
  import type { KtxScanArgs, KtxScanDeps } from './scan.js';
8
+ import type { KtxTableRef } from './context/scan/types.js';
8
9
  type KtxPublicIngestStepName = 'database-schema' | 'query-history' | 'source-ingest' | 'memory-update';
9
10
  type KtxPublicIngestStepStatus = 'done' | 'skipped' | 'failed' | 'not-run';
10
11
  type KtxPublicIngestInputMode = 'auto' | 'disabled';
@@ -98,6 +99,17 @@ interface KtxPublicContextBuildArgs {
98
99
  cliVersion?: string;
99
100
  runtimeInstallPolicy?: KtxManagedPythonInstallPolicy;
100
101
  }
102
+ export declare function publicProgressMessage(message: string, target: KtxPublicIngestPlanTarget): string;
103
+ /** @internal */
104
+ export declare function queryHistoryPullConfig(input: {
105
+ stored: Record<string, unknown>;
106
+ dialect: HistoricSqlDialect;
107
+ windowDays?: number;
108
+ enabledTables?: KtxTableRef[];
109
+ enabledSchemas?: string[];
110
+ modeledTableCatalog?: KtxTableRef[];
111
+ scopeFloorWarnings?: string[];
112
+ }): Record<string, unknown>;
101
113
  export declare function buildPublicIngestPlan(project: KtxPublicIngestProject, args: {
102
114
  projectDir: string;
103
115
  targetConnectionId?: string;
@@ -108,8 +120,15 @@ export declare function buildPublicIngestPlan(project: KtxPublicIngestProject, a
108
120
  command: 'run';
109
121
  }>['mode'];
110
122
  }): KtxPublicIngestPlan;
123
+ /**
124
+ * Run one ingest target through its scan/ingest steps. The single per-target
125
+ * chokepoint reached by every entrypoint — standalone `ktx ingest` (plain/json
126
+ * and foreground) and `ktx setup` (via `runContextBuild`). The exported
127
+ * `executePublicIngestTarget` wraps this and emits the `ingest_completed`
128
+ * telemetry event exactly once, so every path is counted.
129
+ */
111
130
  export declare function executePublicIngestTarget(target: KtxPublicIngestPlanTarget, args: Extract<KtxPublicIngestArgs, {
112
131
  command: 'run';
113
- }>, io: KtxCliIo, deps: KtxPublicIngestDeps): Promise<KtxPublicIngestTargetResult>;
132
+ }>, io: KtxCliIo, deps: KtxPublicIngestDeps, project: KtxPublicIngestProject): Promise<KtxPublicIngestTargetResult>;
114
133
  export declare function runKtxPublicIngest(args: KtxPublicIngestArgs, io: KtxCliIo, deps?: KtxPublicIngestDeps): Promise<number>;
115
134
  export {};
@@ -1,12 +1,16 @@
1
1
  import { getKtxCliPackageInfo } from './cli-runtime.js';
2
2
  import { loadKtxProject } from './context/project/project.js';
3
3
  import { isDatabaseDriver, normalizeConnectionDriver } from './connection-drivers.js';
4
+ import { resolveQueryHistoryScopeFloor } from './context/ingest/adapters/historic-sql/scope-floor.js';
4
5
  import { ensureManagedPythonCommandRuntime, } from './managed-python-command.js';
5
- import { publicIngestOutputLine } from './public-ingest-copy.js';
6
+ import { publicDatabaseIngestMessage, publicIngestOutputLine, publicQueryHistoryMessage, } from './public-ingest-copy.js';
7
+ import { createAggregateProgressPort } from './progress-port-adapter.js';
6
8
  import { resolvePublicIngestRuntimeRequirements } from './runtime-requirements.js';
7
9
  import { profileMark } from './startup-profile.js';
8
10
  import { isDemoConnection } from './telemetry/demo-detect.js';
9
- import { emitProjectStackSnapshot, emitTelemetryEvent } from './telemetry/index.js';
11
+ import { emitProjectStackSnapshot, emitTelemetryEvent, reportException } from './telemetry/index.js';
12
+ import { collectTelemetryRedactionSecrets } from './telemetry/redaction-secrets.js';
13
+ import { formatErrorDetail } from './telemetry/scrubber.js';
10
14
  profileMark('module:public-ingest');
11
15
  const sourceAdapterByDriver = new Map([
12
16
  ['metabase', 'metabase'],
@@ -17,6 +21,16 @@ const sourceAdapterByDriver = new Map([
17
21
  ['dbt', 'dbt'],
18
22
  ['lookml', 'lookml'],
19
23
  ]);
24
+ export function publicProgressMessage(message, target) {
25
+ let current = message;
26
+ if (target.operation === 'database-ingest') {
27
+ current = publicDatabaseIngestMessage(current);
28
+ }
29
+ if (target.steps.includes('query-history')) {
30
+ current = publicQueryHistoryMessage(current, target.connectionId);
31
+ }
32
+ return current;
33
+ }
20
34
  const queryHistoryDialectByDriver = new Map([
21
35
  ['postgres', 'postgres'],
22
36
  ['bigquery', 'bigquery'],
@@ -99,20 +113,20 @@ function storedQueryHistory(connection) {
99
113
  function positiveInteger(value) {
100
114
  return typeof value === 'number' && Number.isInteger(value) && value > 0 ? value : undefined;
101
115
  }
102
- function enabledTablesForConnection(connection) {
103
- const raw = connection.enabled_tables;
104
- if (!Array.isArray(raw)) {
105
- return undefined;
106
- }
107
- const tables = raw.filter((value) => typeof value === 'string' && value.trim().length > 0);
108
- return tables.length > 0 ? tables : undefined;
109
- }
110
- function queryHistoryPullConfig(input) {
111
- const { enabled: _enabled, dialect: _dialect, ...storedConfig } = input.stored;
116
+ /** @internal */
117
+ export function queryHistoryPullConfig(input) {
118
+ const { enabled: _enabled, dialect: _dialect, enabledTables: _enabledTables, enabledSchemas: _enabledSchemas, scopeFloorWarnings: _scopeFloorWarnings, ...storedConfig } = input.stored;
112
119
  return {
113
120
  ...storedConfig,
114
121
  dialect: input.dialect,
115
- ...(input.enabledTables ? { enabledTables: input.enabledTables } : {}),
122
+ ...(input.enabledTables && input.enabledTables.length > 0 ? { enabledTables: input.enabledTables } : {}),
123
+ ...(input.enabledSchemas && input.enabledSchemas.length > 0 ? { enabledSchemas: input.enabledSchemas } : {}),
124
+ ...(input.modeledTableCatalog && input.modeledTableCatalog.length > 0
125
+ ? { modeledTableCatalog: input.modeledTableCatalog }
126
+ : {}),
127
+ ...(input.scopeFloorWarnings && input.scopeFloorWarnings.length > 0
128
+ ? { scopeFloorWarnings: input.scopeFloorWarnings }
129
+ : {}),
116
130
  ...(input.windowDays !== undefined ? { windowDays: input.windowDays } : {}),
117
131
  };
118
132
  }
@@ -157,7 +171,6 @@ function resolveDatabaseTargetOptions(input) {
157
171
  stored: storedQh,
158
172
  dialect,
159
173
  windowDays: queryHistory.windowDays,
160
- enabledTables: enabledTablesForConnection(input.connection),
161
174
  }),
162
175
  },
163
176
  steps: ['database-schema', 'query-history'],
@@ -168,6 +181,37 @@ function resolveDatabaseTargetOptions(input) {
168
181
  steps: ['database-schema'],
169
182
  };
170
183
  }
184
+ async function resolvedQueryHistoryPullConfigForTarget(target, project) {
185
+ if (target.operation !== 'database-ingest' || target.queryHistory?.enabled !== true || !target.queryHistory.dialect) {
186
+ return null;
187
+ }
188
+ const connection = project.config.connections[target.connectionId];
189
+ if (!connection) {
190
+ return (target.queryHistory.pullConfig ??
191
+ queryHistoryPullConfig({
192
+ stored: {},
193
+ dialect: target.queryHistory.dialect,
194
+ windowDays: target.queryHistory.windowDays,
195
+ }));
196
+ }
197
+ const stored = storedQueryHistory(connection);
198
+ const scopeFloor = await resolveQueryHistoryScopeFloor({
199
+ projectDir: project.projectDir,
200
+ connectionId: target.connectionId,
201
+ driver: target.driver,
202
+ connection: connection,
203
+ storedQueryHistory: stored,
204
+ });
205
+ return queryHistoryPullConfig({
206
+ stored,
207
+ dialect: target.queryHistory.dialect,
208
+ windowDays: target.queryHistory.windowDays,
209
+ enabledTables: scopeFloor.enabledTables,
210
+ enabledSchemas: scopeFloor.enabledSchemas,
211
+ modeledTableCatalog: scopeFloor.modeledTableCatalog,
212
+ scopeFloorWarnings: scopeFloor.warnings,
213
+ });
214
+ }
171
215
  function enrichmentReadinessGaps(config) {
172
216
  const gaps = [];
173
217
  if (config.llm.provider.backend === 'none' || !config.llm.models.default) {
@@ -348,6 +392,9 @@ function rowsBucket() {
348
392
  }
349
393
  async function emitIngestCompleted(input) {
350
394
  const failed = resultFailed(input.result);
395
+ const failureDetail = failed
396
+ ? formatErrorDetail(input.result.steps.find((step) => step.status === 'failed')?.detail)
397
+ : undefined;
351
398
  await emitTelemetryEvent({
352
399
  name: 'ingest_completed',
353
400
  projectDir: input.args.projectDir,
@@ -361,6 +408,7 @@ async function emitIngestCompleted(input) {
361
408
  rowsBucket: rowsBucket(),
362
409
  durationMs: Math.max(0, performance.now() - input.startedAt),
363
410
  outcome: failed ? 'error' : 'ok',
411
+ ...(failureDetail ? { errorDetail: failureDetail } : {}),
364
412
  },
365
413
  });
366
414
  }
@@ -427,6 +475,65 @@ function createCapturedPublicIngestIo() {
427
475
  },
428
476
  };
429
477
  }
478
+ function isCapturedPublicIngestIo(io) {
479
+ return typeof io.capturedOutput === 'function';
480
+ }
481
+ const PLAIN_PUBLIC_INGEST_PHASE_LABELS = {
482
+ 'database-schema': 'database schema',
483
+ 'query-history': 'query history',
484
+ 'source-ingest': 'source ingest',
485
+ };
486
+ function firstSummaryLine(summary) {
487
+ if (!summary)
488
+ return undefined;
489
+ return summary.split(/\r?\n/).find((line) => line.trim().length > 0)?.trim();
490
+ }
491
+ function plainPhaseHeader(options, phaseKey) {
492
+ const prefix = options.total > 1 ? `[${options.index + 1}/${options.total}] ` : '';
493
+ return `${prefix}${options.target.connectionId} · ${PLAIN_PUBLIC_INGEST_PHASE_LABELS[phaseKey]}`;
494
+ }
495
+ function plainPhaseEndLine(status, summary) {
496
+ const firstLine = firstSummaryLine(summary);
497
+ return firstLine ? ` ${status} · ${firstLine}` : ` ${status}`;
498
+ }
499
+ function createPlainPublicIngestProgress(io, options) {
500
+ let currentPhase = null;
501
+ const startedPhases = new Set();
502
+ const lastPercentByPhase = new Map();
503
+ const startPhase = (phaseKey) => {
504
+ currentPhase = phaseKey;
505
+ startedPhases.add(phaseKey);
506
+ lastPercentByPhase.set(phaseKey, -1);
507
+ io.stderr.write(`${plainPhaseHeader(options, phaseKey)}\n`);
508
+ };
509
+ const ensurePhaseStarted = (phaseKey) => {
510
+ if (!startedPhases.has(phaseKey)) {
511
+ startPhase(phaseKey);
512
+ return;
513
+ }
514
+ currentPhase = phaseKey;
515
+ };
516
+ const emitProgress = (update) => {
517
+ if (currentPhase === null)
518
+ return;
519
+ const rounded = Math.max(0, Math.min(100, Math.round(update.percent)));
520
+ const lastPercent = lastPercentByPhase.get(currentPhase) ?? -1;
521
+ if (rounded <= lastPercent)
522
+ return;
523
+ lastPercentByPhase.set(currentPhase, rounded);
524
+ io.stderr.write(` [${rounded}%] ${publicProgressMessage(update.message, options.target)}\n`);
525
+ };
526
+ return {
527
+ onPhaseStart: startPhase,
528
+ onPhaseEnd(phaseKey, status, summary) {
529
+ ensurePhaseStarted(phaseKey);
530
+ io.stderr.write(`${plainPhaseEndLine(status, summary)}\n`);
531
+ currentPhase = null;
532
+ },
533
+ scanProgress: createAggregateProgressPort(emitProgress),
534
+ ingestProgress: emitProgress,
535
+ };
536
+ }
430
537
  const INTERNAL_STATUS_LINE_RE = /^(Report|Run|Job|Status|Adapter|Connection|Sync|Diff|Tasks|Work units|Failed tasks|Saved memory|Provenance rows):\s*/;
431
538
  const ACTIONABLE_FAILURE_LINE_RE = /^(Missing bundled Python runtime manifest|KTX Python runtime is required|KTX daemon HTTP|Error:|Failed\b|Could not\b|Cannot\b)/;
432
539
  const RUNTIME_BACKED_RETRY_LINE_RE = /^Then retry the runtime-backed KTX command\.?$/;
@@ -457,7 +564,23 @@ function capturedFailureMessage(output) {
457
564
  .filter((line) => line.startsWith('In a source checkout, build the local runtime assets with:'));
458
565
  return [firstLine, ...followupLines].join('\n');
459
566
  }
460
- export async function executePublicIngestTarget(target, args, io, deps) {
567
+ /**
568
+ * Run one ingest target through its scan/ingest steps. The single per-target
569
+ * chokepoint reached by every entrypoint — standalone `ktx ingest` (plain/json
570
+ * and foreground) and `ktx setup` (via `runContextBuild`). The exported
571
+ * `executePublicIngestTarget` wraps this and emits the `ingest_completed`
572
+ * telemetry event exactly once, so every path is counted.
573
+ */
574
+ export async function executePublicIngestTarget(target, args, io, deps, project) {
575
+ const startedAt = performance.now();
576
+ const result = await runIngestTargetSteps(target, args, io, deps, project);
577
+ // `io` may be a capture buffer for the scan/ingest step output; the telemetry
578
+ // debug echo belongs on the real user-facing stream, which callers expose as
579
+ // `deps.runtimeIo` (falling back to `io` when the step io is already real).
580
+ await emitIngestCompleted({ args, project, target, result, startedAt, io: deps.runtimeIo ?? io });
581
+ return result;
582
+ }
583
+ async function runIngestTargetSteps(target, args, io, deps, project) {
461
584
  if (target.preflightFailure) {
462
585
  if (target.operation === 'database-ingest') {
463
586
  deps.onPhaseEnd?.('database-schema', 'failed', target.preflightFailure);
@@ -475,7 +598,7 @@ export async function executePublicIngestTarget(target, args, io, deps) {
475
598
  ? {
476
599
  ...step,
477
600
  status: 'failed',
478
- detail: target.preflightFailure,
601
+ detail: `${target.connectionId} failed: ${target.preflightFailure}`,
479
602
  }
480
603
  : step),
481
604
  };
@@ -493,7 +616,11 @@ export async function executePublicIngestTarget(target, args, io, deps) {
493
616
  ...(args.runtimeInstallPolicy ? { runtimeInstallPolicy: args.runtimeInstallPolicy } : {}),
494
617
  };
495
618
  const runScan = deps.runScan ?? runKtxScan;
496
- const capturedScanIo = deps.scanProgress ? null : createCapturedPublicIngestIo();
619
+ const capturedScanIo = deps.scanProgress
620
+ ? isCapturedPublicIngestIo(io)
621
+ ? io
622
+ : null
623
+ : createCapturedPublicIngestIo();
497
624
  const scanIo = capturedScanIo ?? io;
498
625
  const scanDeps = {
499
626
  ...(deps.scanProgress ? { progress: deps.scanProgress } : {}),
@@ -512,6 +639,10 @@ export async function executePublicIngestTarget(target, args, io, deps) {
512
639
  if (target.queryHistory?.enabled === true) {
513
640
  const { runKtxIngest } = await import('./ingest.js');
514
641
  const runIngest = deps.runIngest ?? runKtxIngest;
642
+ const historicSqlPullConfigOverride = (await resolvedQueryHistoryPullConfigForTarget(target, project)) ?? {
643
+ dialect: target.queryHistory.dialect,
644
+ ...(target.queryHistory.windowDays !== undefined ? { windowDays: target.queryHistory.windowDays } : {}),
645
+ };
515
646
  const ingestArgs = {
516
647
  command: 'run',
517
648
  projectDir: args.projectDir,
@@ -522,12 +653,14 @@ export async function executePublicIngestTarget(target, args, io, deps) {
522
653
  ...(args.cliVersion ? { cliVersion: args.cliVersion } : {}),
523
654
  ...(args.runtimeInstallPolicy ? { runtimeInstallPolicy: args.runtimeInstallPolicy } : {}),
524
655
  allowImplicitAdapter: true,
525
- historicSqlPullConfigOverride: target.queryHistory.pullConfig ?? {
526
- dialect: target.queryHistory.dialect,
527
- ...(target.queryHistory.windowDays !== undefined ? { windowDays: target.queryHistory.windowDays } : {}),
528
- },
656
+ historicSqlPullConfigOverride,
529
657
  };
530
- const capturedIngestIo = deps.ingestProgress ? null : createCapturedPublicIngestIo();
658
+ // Query history runs after the schema scan has already written its report
659
+ // into the shared target io, so it needs a phase-local capture. Reusing
660
+ // `io` here would let leftover scan text (e.g. "Mode: enriched") surface as
661
+ // the query-history failure detail. Only skip capture when progress is
662
+ // active and the caller manages its own buffer (io is not a capture).
663
+ const capturedIngestIo = deps.ingestProgress && !isCapturedPublicIngestIo(io) ? null : createCapturedPublicIngestIo();
531
664
  const ingestIo = capturedIngestIo ?? io;
532
665
  const ingestDeps = {
533
666
  ...(deps.ingestProgress ? { progress: deps.ingestProgress } : {}),
@@ -564,7 +697,11 @@ export async function executePublicIngestTarget(target, args, io, deps) {
564
697
  allowImplicitAdapter: true,
565
698
  };
566
699
  const runIngest = deps.runIngest ?? runKtxIngest;
567
- const capturedIngestIo = deps.ingestProgress ? null : createCapturedPublicIngestIo();
700
+ const capturedIngestIo = deps.ingestProgress
701
+ ? isCapturedPublicIngestIo(io)
702
+ ? io
703
+ : null
704
+ : createCapturedPublicIngestIo();
568
705
  const ingestIo = capturedIngestIo ?? io;
569
706
  const ingestDeps = {
570
707
  ...(deps.ingestProgress ? { progress: deps.ingestProgress } : {}),
@@ -597,26 +734,60 @@ export async function runKtxPublicIngest(args, io, deps = {}) {
597
734
  });
598
735
  }
599
736
  catch (error) {
737
+ await reportException({
738
+ error,
739
+ context: { source: 'ingest runtime', handled: true, fatal: false },
740
+ projectDir: args.projectDir,
741
+ io,
742
+ redactionSecrets: await collectTelemetryRedactionSecrets({
743
+ project,
744
+ projectDir: args.projectDir,
745
+ connectionId: args.targetConnectionId,
746
+ includeLlm: true,
747
+ includeEmbeddings: true,
748
+ env: deps.env ?? process.env,
749
+ }),
750
+ });
600
751
  io.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`);
601
752
  return 1;
602
753
  }
603
754
  }
604
755
  const { runContextBuild } = await import('./context-build-view.js');
605
756
  const contextBuild = deps.runContextBuild ?? runContextBuild;
606
- const result = await contextBuild(project, {
607
- projectDir: args.projectDir,
608
- ...(args.targetConnectionId ? { targetConnectionId: args.targetConnectionId } : {}),
609
- all: args.all,
610
- entrypoint: 'ingest',
611
- inputMode: args.inputMode,
612
- ...(args.queryHistory ? { queryHistory: args.queryHistory } : {}),
613
- ...(args.queryHistoryWindowDays !== undefined ? { queryHistoryWindowDays: args.queryHistoryWindowDays } : {}),
614
- ...(args.scanMode ? { scanMode: args.scanMode } : {}),
615
- ...(args.detectRelationships !== undefined ? { detectRelationships: args.detectRelationships } : {}),
616
- ...(args.cliVersion ? { cliVersion: args.cliVersion } : {}),
617
- ...(args.runtimeInstallPolicy ? { runtimeInstallPolicy: args.runtimeInstallPolicy } : {}),
618
- }, io);
619
- return result.exitCode;
757
+ try {
758
+ const result = await contextBuild(project, {
759
+ projectDir: args.projectDir,
760
+ ...(args.targetConnectionId ? { targetConnectionId: args.targetConnectionId } : {}),
761
+ all: args.all,
762
+ entrypoint: 'ingest',
763
+ inputMode: args.inputMode,
764
+ ...(args.queryHistory ? { queryHistory: args.queryHistory } : {}),
765
+ ...(args.queryHistoryWindowDays !== undefined ? { queryHistoryWindowDays: args.queryHistoryWindowDays } : {}),
766
+ ...(args.scanMode ? { scanMode: args.scanMode } : {}),
767
+ ...(args.detectRelationships !== undefined ? { detectRelationships: args.detectRelationships } : {}),
768
+ ...(args.cliVersion ? { cliVersion: args.cliVersion } : {}),
769
+ ...(args.runtimeInstallPolicy ? { runtimeInstallPolicy: args.runtimeInstallPolicy } : {}),
770
+ }, io);
771
+ return result.exitCode;
772
+ }
773
+ catch (error) {
774
+ await reportException({
775
+ error,
776
+ context: { source: 'ingest context-build', handled: true, fatal: false },
777
+ projectDir: args.projectDir,
778
+ io,
779
+ redactionSecrets: await collectTelemetryRedactionSecrets({
780
+ project,
781
+ projectDir: args.projectDir,
782
+ connectionId: args.targetConnectionId,
783
+ includeLlm: true,
784
+ includeEmbeddings: true,
785
+ env: deps.env ?? process.env,
786
+ }),
787
+ });
788
+ io.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`);
789
+ return 1;
790
+ }
620
791
  }
621
792
  const plan = buildPublicIngestPlan(project, args);
622
793
  const results = [];
@@ -628,11 +799,26 @@ export async function runKtxPublicIngest(args, io, deps = {}) {
628
799
  io.stderr.write(`Warning: ${warning}\n`);
629
800
  }
630
801
  }
631
- for (const target of plan.targets) {
632
- const startedAt = performance.now();
633
- const result = await executePublicIngestTarget(target, args, io, deps);
634
- results.push(result);
635
- await emitIngestCompleted({ args, project, target, result, startedAt, io });
802
+ for (const [index, target] of plan.targets.entries()) {
803
+ if (args.json) {
804
+ results.push(await executePublicIngestTarget(target, args, io, deps, project));
805
+ continue;
806
+ }
807
+ const capture = createCapturedPublicIngestIo();
808
+ const progress = createPlainPublicIngestProgress(io, {
809
+ target,
810
+ index,
811
+ total: plan.targets.length,
812
+ });
813
+ const targetDeps = {
814
+ ...deps,
815
+ scanProgress: progress.scanProgress,
816
+ ingestProgress: progress.ingestProgress,
817
+ onPhaseStart: progress.onPhaseStart,
818
+ onPhaseEnd: progress.onPhaseEnd,
819
+ runtimeIo: deps.runtimeIo ?? io,
820
+ };
821
+ results.push(await executePublicIngestTarget(target, args, capture, targetDeps, project));
636
822
  }
637
823
  if (args.json) {
638
824
  io.stdout.write(`${JSON.stringify({ plan, results }, null, 2)}\n`);
@@ -0,0 +1,24 @@
1
+ import { type PasswordOptions } from '@clack/core';
2
+ /**
3
+ * Mask every character of `userInput` except the last `tail`, but only reveal the
4
+ * tail once the secret is long enough that the hidden portion still dominates
5
+ * (`length > tail * 2`). Short secrets stay fully masked so we never expose most
6
+ * of a small value. The returned string keeps the same code-unit length as the
7
+ * input so clack's cursor slicing in `userInputWithCursor` stays aligned.
8
+ *
9
+ * @internal
10
+ */
11
+ export declare function maskRevealingTail(userInput: string, maskChar: string, tail: number): string;
12
+ export interface RevealPasswordOptions {
13
+ message: string;
14
+ mask?: string;
15
+ tail?: number;
16
+ validate?: PasswordOptions['validate'];
17
+ signal?: AbortSignal;
18
+ }
19
+ /**
20
+ * Drop-in replacement for clack's `password()` that reveals the last few
21
+ * characters of the entered value while typing. Resolves to the raw value or the
22
+ * clack cancel symbol, matching `password()`'s contract.
23
+ */
24
+ export declare function revealPassword(options: RevealPasswordOptions): Promise<string | symbol>;
@@ -0,0 +1,78 @@
1
+ import { styleText } from 'node:util';
2
+ import { PasswordPrompt } from '@clack/core';
3
+ import { S_BAR, S_BAR_END, S_PASSWORD_MASK, settings, symbol } from '@clack/prompts';
4
+ // How many trailing characters of a pasted secret to leave visible so the user
5
+ // can confirm what landed (e.g. `••••••a1b2`). Kept small on purpose.
6
+ const REVEAL_TAIL_COUNT = 4;
7
+ /**
8
+ * Mask every character of `userInput` except the last `tail`, but only reveal the
9
+ * tail once the secret is long enough that the hidden portion still dominates
10
+ * (`length > tail * 2`). Short secrets stay fully masked so we never expose most
11
+ * of a small value. The returned string keeps the same code-unit length as the
12
+ * input so clack's cursor slicing in `userInputWithCursor` stays aligned.
13
+ *
14
+ * @internal
15
+ */
16
+ export function maskRevealingTail(userInput, maskChar, tail) {
17
+ const revealLength = userInput.length > tail * 2 ? tail : 0;
18
+ const hiddenLength = userInput.length - revealLength;
19
+ return maskChar.repeat(hiddenLength) + userInput.slice(hiddenLength);
20
+ }
21
+ class RevealTailPasswordPrompt extends PasswordPrompt {
22
+ #maskChar;
23
+ #tail;
24
+ constructor(options) {
25
+ super(options);
26
+ this.#maskChar = options.mask ?? S_PASSWORD_MASK;
27
+ this.#tail = options.tail;
28
+ }
29
+ get masked() {
30
+ return maskRevealingTail(this.userInput, this.#maskChar, this.#tail);
31
+ }
32
+ }
33
+ // Reproduces the @clack/prompts password frame (pinned to the installed version)
34
+ // so this prompt is visually identical to every other setup prompt; the only
35
+ // behavioral change is the tail-revealing `masked` getter above.
36
+ function renderPasswordFrame(prompt, message) {
37
+ const withGuide = settings.withGuide;
38
+ const title = `${withGuide ? `${styleText('gray', S_BAR)}\n` : ''}${symbol(prompt.state)} ${message}\n`;
39
+ const masked = prompt.masked;
40
+ switch (prompt.state) {
41
+ case 'error': {
42
+ const bar = withGuide ? `${styleText('yellow', S_BAR)} ` : '';
43
+ const end = withGuide ? `${styleText('yellow', S_BAR_END)} ` : '';
44
+ return `${title.trim()}\n${bar}${masked}\n${end}${styleText('yellow', prompt.error)}\n`;
45
+ }
46
+ case 'submit': {
47
+ const bar = withGuide ? `${styleText('gray', S_BAR)} ` : '';
48
+ return `${title}${bar}${masked ? styleText('dim', masked) : ''}`;
49
+ }
50
+ case 'cancel': {
51
+ const bar = withGuide ? `${styleText('gray', S_BAR)} ` : '';
52
+ const body = masked ? styleText(['strikethrough', 'dim'], masked) : '';
53
+ return `${title}${bar}${body}${masked && withGuide ? `\n${styleText('gray', S_BAR)}` : ''}`;
54
+ }
55
+ default: {
56
+ const bar = withGuide ? `${styleText('cyan', S_BAR)} ` : '';
57
+ const end = withGuide ? styleText('cyan', S_BAR_END) : '';
58
+ return `${title}${bar}${prompt.userInputWithCursor}\n${end}\n`;
59
+ }
60
+ }
61
+ }
62
+ /**
63
+ * Drop-in replacement for clack's `password()` that reveals the last few
64
+ * characters of the entered value while typing. Resolves to the raw value or the
65
+ * clack cancel symbol, matching `password()`'s contract.
66
+ */
67
+ export function revealPassword(options) {
68
+ const prompt = new RevealTailPasswordPrompt({
69
+ mask: options.mask ?? S_PASSWORD_MASK,
70
+ tail: options.tail ?? REVEAL_TAIL_COUNT,
71
+ validate: options.validate,
72
+ signal: options.signal,
73
+ render() {
74
+ return renderPasswordFrame(this, options.message);
75
+ },
76
+ });
77
+ return prompt.prompt();
78
+ }