@kaelio/ktx 0.7.0 → 0.8.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (80) hide show
  1. package/assets/python/{kaelio_ktx-0.7.0-py3-none-any.whl → kaelio_ktx-0.8.0-py3-none-any.whl} +0 -0
  2. package/assets/python/manifest.json +4 -4
  3. package/dist/.tsbuildinfo +1 -1
  4. package/dist/cli-program.js +7 -0
  5. package/dist/command-schemas.d.ts +1 -1
  6. package/dist/command-tree.js +5 -1
  7. package/dist/commands/completion-commands.d.ts +3 -0
  8. package/dist/commands/completion-commands.js +38 -0
  9. package/dist/commands/ingest-commands.js +0 -4
  10. package/dist/commands/knowledge-commands.js +15 -2
  11. package/dist/commands/setup-commands.js +2 -2
  12. package/dist/commands/sl-commands.js +19 -7
  13. package/dist/completion/complete-engine.d.ts +19 -0
  14. package/dist/completion/complete-engine.js +128 -0
  15. package/dist/completion/completion-scripts.d.ts +1 -0
  16. package/dist/completion/completion-scripts.js +36 -0
  17. package/dist/completion/dynamic-candidates.d.ts +6 -0
  18. package/dist/completion/dynamic-candidates.js +98 -0
  19. package/dist/connection-drivers.d.ts +3 -0
  20. package/dist/connection-drivers.js +17 -0
  21. package/dist/context/ingest/ingest-bundle.runner.d.ts +8 -0
  22. package/dist/context/ingest/ingest-bundle.runner.js +72 -15
  23. package/dist/context/ingest/ingest-profile.d.ts +102 -0
  24. package/dist/context/ingest/ingest-profile.js +306 -0
  25. package/dist/context/ingest/isolated-diff/work-unit-executor.js +25 -2
  26. package/dist/context/ingest/local-bundle-runtime.js +1 -0
  27. package/dist/context/ingest/local-ingest.d.ts +1 -1
  28. package/dist/context/ingest/local-ingest.js +6 -4
  29. package/dist/context/ingest/memory-flow/events.js +2 -1
  30. package/dist/context/ingest/ports.d.ts +2 -0
  31. package/dist/context/ingest/reports.d.ts +3 -0
  32. package/dist/context/ingest/reports.js +10 -0
  33. package/dist/context/ingest/stages/stage-3-work-units.d.ts +3 -1
  34. package/dist/context/ingest/stages/stage-3-work-units.js +2 -0
  35. package/dist/context/ingest/stages/stage-4-reconciliation.d.ts +2 -1
  36. package/dist/context/ingest/stages/stage-4-reconciliation.js +1 -1
  37. package/dist/context/ingest/tools/tool-call-logger.d.ts +6 -0
  38. package/dist/context/ingest/tools/tool-call-logger.js +36 -1
  39. package/dist/context/llm/ai-sdk-runtime.js +32 -3
  40. package/dist/context/llm/claude-code-runtime.js +35 -2
  41. package/dist/context/llm/runtime-port.d.ts +25 -0
  42. package/dist/context/mcp/context-tools.d.ts +2 -1
  43. package/dist/context/mcp/context-tools.js +82 -15
  44. package/dist/context/mcp/server.js +4 -0
  45. package/dist/context/mcp/types.d.ts +15 -1
  46. package/dist/context/project/config.d.ts +1 -0
  47. package/dist/context/project/config.js +4 -0
  48. package/dist/context/project/driver-schemas.js +1 -1
  49. package/dist/context/search/discover.js +4 -3
  50. package/dist/context/sl/local-sl.d.ts +15 -0
  51. package/dist/context/sl/local-sl.js +30 -0
  52. package/dist/context/wiki/local-knowledge.d.ts +10 -0
  53. package/dist/context/wiki/local-knowledge.js +22 -0
  54. package/dist/context-build-view.d.ts +0 -3
  55. package/dist/context-build-view.js +1 -7
  56. package/dist/ingest.js +7 -10
  57. package/dist/knowledge.d.ts +5 -0
  58. package/dist/knowledge.js +10 -1
  59. package/dist/public-ingest-copy.js +1 -1
  60. package/dist/public-ingest.d.ts +0 -7
  61. package/dist/public-ingest.js +20 -34
  62. package/dist/setup-context.js +6 -38
  63. package/dist/setup-databases.js +13 -82
  64. package/dist/setup-sources.js +33 -5
  65. package/dist/setup.js +2 -2
  66. package/dist/skills/analytics/SKILL.md +6 -1
  67. package/dist/sl.d.ts +6 -1
  68. package/dist/sl.js +32 -8
  69. package/dist/telemetry/emitter.js +1 -1
  70. package/dist/telemetry/events.d.ts +4 -3
  71. package/dist/telemetry/events.js +7 -3
  72. package/dist/telemetry/identity.d.ts +1 -1
  73. package/dist/telemetry/identity.js +13 -10
  74. package/dist/telemetry/index.d.ts +1 -1
  75. package/dist/telemetry/index.js +5 -1
  76. package/package.json +22 -22
  77. package/dist/ingest-depth.d.ts +0 -8
  78. package/dist/ingest-depth.js +0 -56
  79. package/dist/setup-database-context-depth.d.ts +0 -23
  80. package/dist/setup-database-context-depth.js +0 -84
@@ -6,8 +6,6 @@ import { markKtxSetupStateStepComplete, readKtxSetupState } from './context/proj
6
6
  import { serializeKtxProjectConfig } from './context/project/config.js';
7
7
  import { errorMessage, writePrefixedLines } from './clack.js';
8
8
  import { buildPublicIngestPlan } from './public-ingest.js';
9
- import { databaseContextDepth, } from './ingest-depth.js';
10
- import { ensureSetupDatabaseContextDepths } from './setup-database-context-depth.js';
11
9
  import { runContextBuild, } from './context-build-view.js';
12
10
  import { createKtxSetupPromptAdapter, } from './setup-prompts.js';
13
11
  const SETUP_CONTEXT_STATE_PATH = ['.ktx', 'setup', 'context-build.json'];
@@ -232,15 +230,6 @@ async function readLatestScanReport(projectDir, connectionId) {
232
230
  reports.sort((left, right) => left.sortKey.localeCompare(right.sortKey));
233
231
  return reports.at(-1)?.report ?? null;
234
232
  }
235
- function scanReportHasSchemaManifest(report, connectionId) {
236
- if (!isRecord(report)) {
237
- return false;
238
- }
239
- if (report.connectionId !== connectionId || report.dryRun === true) {
240
- return false;
241
- }
242
- return stringArrayValue(isRecord(report.artifactPaths) ? report.artifactPaths.manifestShards : undefined).length > 0;
243
- }
244
233
  function scanReportHasCompletedDeepEnrichment(report, connectionId, relationshipsRequired) {
245
234
  if (!isRecord(report)) {
246
235
  return false;
@@ -260,23 +249,13 @@ function scanReportHasCompletedDeepEnrichment(report, connectionId, relationship
260
249
  (!relationshipsRequired || completedStages.includes('relationships')) &&
261
250
  stringArrayValue(report.artifactPaths.manifestShards).length > 0);
262
251
  }
263
- function scanReportSatisfiesDepth(input) {
264
- if (input.depth === 'fast') {
265
- return scanReportHasSchemaManifest(input.report, input.connectionId);
266
- }
267
- return scanReportHasCompletedDeepEnrichment(input.report, input.connectionId, input.relationshipsRequired);
268
- }
269
252
  async function verifyPrimarySourceScans(project, connectionIds) {
270
253
  const details = [];
271
254
  const relationshipsRequired = project.config.scan.relationships.enabled;
272
255
  for (const connectionId of connectionIds) {
273
- const connection = project.config.connections[connectionId];
274
- const depth = connection ? (databaseContextDepth(connection) ?? 'fast') : 'fast';
275
256
  const report = await readLatestScanReport(project.projectDir, connectionId);
276
- if (!scanReportSatisfiesDepth({ report, connectionId, depth, relationshipsRequired })) {
277
- details.push(depth === 'fast'
278
- ? `${connectionId}: schema context has not completed.`
279
- : `${connectionId}: deep database context has not completed.`);
257
+ if (!scanReportHasCompletedDeepEnrichment(report, connectionId, relationshipsRequired)) {
258
+ details.push(`${connectionId}: database context has not completed.`);
280
259
  }
281
260
  }
282
261
  return { ready: details.length === 0, details };
@@ -331,7 +310,7 @@ function writeSkippedContext(projectDir, io) {
331
310
  io.stdout.write(`Build context:\n ktx setup --project-dir ${resolve(projectDir)}\n\n`);
332
311
  io.stdout.write(`Check status:\n ktx status --project-dir ${resolve(projectDir)}\n`);
333
312
  }
334
- function writeSuccess(project, readiness, targets, io) {
313
+ function writeSuccess(readiness, targets, io) {
335
314
  io.stdout.write('\nKTX context is ready for agents.\n\n');
336
315
  io.stdout.write('Databases:\n');
337
316
  if (targets.primarySourceConnectionIds.length === 0) {
@@ -339,9 +318,7 @@ function writeSuccess(project, readiness, targets, io) {
339
318
  }
340
319
  else {
341
320
  for (const connectionId of targets.primarySourceConnectionIds) {
342
- const connection = project.config.connections[connectionId];
343
- const depth = connection ? (databaseContextDepth(connection) ?? 'fast') : 'fast';
344
- io.stdout.write(` ${connectionId}: ${depth === 'deep' ? 'deep context complete' : 'schema context complete'}\n`);
321
+ io.stdout.write(` ${connectionId}: database context complete\n`);
345
322
  }
346
323
  }
347
324
  io.stdout.write('\nContext sources:\n');
@@ -466,7 +443,7 @@ async function runBuild(args, io, deps, project, targets) {
466
443
  failureReason: undefined,
467
444
  ...(lastSourceProgress ? { sourceProgress: lastSourceProgress } : {}),
468
445
  });
469
- writeSuccess(project, readiness, targets, io);
446
+ writeSuccess(readiness, targets, io);
470
447
  return { status: 'ready', projectDir: args.projectDir, runId };
471
448
  }
472
449
  async function completeExistingContext(args, io, deps, targets) {
@@ -496,17 +473,8 @@ async function completeExistingContext(args, io, deps, targets) {
496
473
  }
497
474
  export async function runKtxSetupContextStep(args, io, deps = {}) {
498
475
  try {
499
- let project = await loadKtxProject({ projectDir: args.projectDir });
476
+ const project = await loadKtxProject({ projectDir: args.projectDir });
500
477
  const prompts = deps.prompts ?? createPromptAdapter();
501
- const depthProject = await ensureSetupDatabaseContextDepths({
502
- project,
503
- args,
504
- prompts,
505
- });
506
- if (depthProject === 'back') {
507
- return { status: 'back', projectDir: args.projectDir };
508
- }
509
- project = depthProject;
510
478
  const existingState = await readKtxSetupContextState(args.projectDir);
511
479
  const completedSteps = (await readKtxSetupState(args.projectDir)).completed_steps;
512
480
  if (completedSteps.includes('context') && existingState.status === 'completed') {
@@ -14,13 +14,12 @@ import { runKtxConnection } from './connection.js';
14
14
  import { pickDatabaseScope as defaultPickDatabaseScope, } from './database-tree-picker.js';
15
15
  import { withMultiselectNavigation, withTextInputNavigation } from './prompt-navigation.js';
16
16
  import { runKtxScan } from './scan.js';
17
- import { applySetupDatabaseContextDepth } from './setup-database-context-depth.js';
18
17
  import { writeProjectLocalSecretReference } from './setup-secrets.js';
19
18
  import { isDemoConnection } from './telemetry/demo-detect.js';
20
19
  import { emitTelemetryEvent } from './telemetry/index.js';
21
20
  import { createKtxSetupPromptAdapter, createKtxSetupUiAdapter, } from './setup-prompts.js';
22
21
  const HISTORIC_SQL_WORK_UNIT_MAX_CONCURRENCY = 6;
23
- const KTX_QUICKSTART_URL = 'https://docs.kaelio.com/ktx/docs/getting-started/quickstart';
22
+ const KTX_DEMO_START_URL = 'https://www.kaelio.com/start';
24
23
  const execFileAsync = promisify(execFileCallback);
25
24
  const DRIVER_OPTIONS = [
26
25
  { value: 'postgres', label: 'PostgreSQL' },
@@ -1209,39 +1208,10 @@ async function applyHistoricSqlConfigToExistingConnection(input) {
1209
1208
  });
1210
1209
  if (withHistoricSql === 'back')
1211
1210
  return 'back';
1212
- const withContextDepth = await maybeApplyContextDepthConfig({
1213
- projectDir: input.projectDir,
1214
- connectionId: input.connectionId,
1215
- connection: withHistoricSql,
1216
- args: input.args,
1217
- prompts: input.prompts,
1218
- });
1219
- if (withContextDepth === 'back')
1220
- return 'back';
1221
1211
  await writeConnectionConfig({
1222
1212
  projectDir: input.projectDir,
1223
1213
  connectionId: input.connectionId,
1224
- connection: withContextDepth,
1225
- });
1226
- }
1227
- async function maybeApplyContextDepthConfig(input) {
1228
- const project = await loadKtxProject({ projectDir: input.projectDir });
1229
- return await applySetupDatabaseContextDepth({
1230
- project: {
1231
- ...project,
1232
- config: {
1233
- ...project.config,
1234
- connections: {
1235
- ...project.config.connections,
1236
- [input.connectionId]: input.connection,
1237
- },
1238
- },
1239
- },
1240
- connection: input.connection,
1241
- args: {
1242
- inputMode: input.args.inputMode === 'disabled' || input.args.databaseUrl ? 'disabled' : input.args.inputMode,
1243
- },
1244
- prompts: input.prompts,
1214
+ connection: withHistoricSql,
1245
1215
  });
1246
1216
  }
1247
1217
  async function validateAndScanConnection(input) {
@@ -1273,7 +1243,7 @@ async function validateAndScanConnection(input) {
1273
1243
  deps: input.deps,
1274
1244
  });
1275
1245
  writeSetupSection(input.io, `Building schema context for ${input.connectionId}`, [
1276
- 'Running fast database ingest…',
1246
+ 'Running database scan…',
1277
1247
  ]);
1278
1248
  let scanIo = createBufferedCommandIo();
1279
1249
  let scanCode = await scanConnection(input.projectDir, input.connectionId, scanIo);
@@ -1281,7 +1251,7 @@ async function validateAndScanConnection(input) {
1281
1251
  const nativeSqliteDetail = nativeSqliteAbiMismatchDetail(`${scanIo.stderrText()}\n${scanIo.stdoutText()}`);
1282
1252
  if (nativeSqliteDetail) {
1283
1253
  writePrefixedLines((chunk) => input.io.stderr.write(chunk), [
1284
- `Fast database ingest failed for ${input.connectionId}.`,
1254
+ `Database scan failed for ${input.connectionId}.`,
1285
1255
  'Native SQLite is built for a different Node.js ABI.',
1286
1256
  `Detail: ${nativeSqliteDetail}`,
1287
1257
  'Rebuilding Native SQLite with pnpm run native:rebuild…',
@@ -1289,7 +1259,7 @@ async function validateAndScanConnection(input) {
1289
1259
  const rebuildNativeSqlite = input.deps.rebuildNativeSqlite ?? defaultRebuildNativeSqlite;
1290
1260
  const rebuildCode = await rebuildNativeSqlite(input.io);
1291
1261
  if (rebuildCode === 0) {
1292
- writePrefixedLines((chunk) => input.io.stderr.write(chunk), 'Native SQLite rebuild complete. Retrying fast database ingest…');
1262
+ writePrefixedLines((chunk) => input.io.stderr.write(chunk), 'Native SQLite rebuild complete. Retrying database scan…');
1293
1263
  const retryScanIo = createBufferedCommandIo();
1294
1264
  scanCode = await scanConnection(input.projectDir, input.connectionId, retryScanIo);
1295
1265
  scanIo = retryScanIo;
@@ -1297,18 +1267,18 @@ async function validateAndScanConnection(input) {
1297
1267
  if (scanCode !== 0) {
1298
1268
  writePrefixedLines((chunk) => input.io.stderr.write(chunk), [
1299
1269
  rebuildCode === 0
1300
- ? `Fast database ingest still failed for ${input.connectionId} after rebuilding Native SQLite.`
1270
+ ? `Database scan still failed for ${input.connectionId} after rebuilding Native SQLite.`
1301
1271
  : `Native SQLite rebuild failed for ${input.connectionId}.`,
1302
1272
  'Fix: pnpm run native:rebuild',
1303
- `Retry: ktx ingest ${input.connectionId} --project-dir ${input.projectDir} --fast`,
1273
+ `Retry: ktx ingest ${input.connectionId} --project-dir ${input.projectDir}`,
1304
1274
  ].join('\n'));
1305
1275
  }
1306
1276
  }
1307
1277
  else {
1308
1278
  flushPrefixedBufferedCommandOutput(input.io, scanIo);
1309
1279
  writePrefixedLines((chunk) => input.io.stderr.write(chunk), [
1310
- `Fast database ingest failed for ${input.connectionId}.`,
1311
- `Debug command: ktx ingest ${input.connectionId} --project-dir ${input.projectDir} --fast --debug`,
1280
+ `Database scan failed for ${input.connectionId}.`,
1281
+ `Debug command: ktx ingest ${input.connectionId} --project-dir ${input.projectDir} --debug`,
1312
1282
  ].join('\n'));
1313
1283
  }
1314
1284
  if (scanCode !== 0) {
@@ -1334,7 +1304,7 @@ async function chooseDrivers(args, io, prompts, options) {
1334
1304
  return 'missing-input';
1335
1305
  }
1336
1306
  const initialValues = unique(options?.initialDrivers ?? []);
1337
- createKtxSetupUiAdapter().note(`Get demo credentials from the Quickstart: ${KTX_QUICKSTART_URL}`, '🎁 Need a warehouse to play with?', io);
1307
+ createKtxSetupUiAdapter().note(`Get demo credentials: ${KTX_DEMO_START_URL}`, '🎁 Need a warehouse to play with?', io);
1338
1308
  const choices = await prompts.multiselect({
1339
1309
  message: withMultiselectNavigation('Which databases should KTX connect to?'),
1340
1310
  options: [...DRIVER_OPTIONS],
@@ -1690,23 +1660,10 @@ export async function runKtxSetupDatabasesStep(args, io, deps = {}) {
1690
1660
  returnToDriverSelection = true;
1691
1661
  break;
1692
1662
  }
1693
- const withContextDepth = await maybeApplyContextDepthConfig({
1694
- projectDir: args.projectDir,
1695
- connectionId: connectionChoice.connectionId,
1696
- connection: withHistoricSql,
1697
- args,
1698
- prompts,
1699
- });
1700
- if (withContextDepth === 'back') {
1701
- if (!canReturnToDriverSelection)
1702
- return { status: 'back', projectDir: args.projectDir };
1703
- returnToDriverSelection = true;
1704
- break;
1705
- }
1706
1663
  await writeConnectionConfig({
1707
1664
  projectDir: args.projectDir,
1708
1665
  connectionId: connectionChoice.connectionId,
1709
- connection: withContextDepth,
1666
+ connection: withHistoricSql,
1710
1667
  io,
1711
1668
  });
1712
1669
  }
@@ -1719,23 +1676,10 @@ export async function runKtxSetupDatabasesStep(args, io, deps = {}) {
1719
1676
  returnToDriverSelection = true;
1720
1677
  break;
1721
1678
  }
1722
- const withContextDepth = await maybeApplyContextDepthConfig({
1723
- projectDir: args.projectDir,
1724
- connectionId: connectionChoice.connectionId,
1725
- connection: withHistoricSql,
1726
- args,
1727
- prompts,
1728
- });
1729
- if (withContextDepth === 'back') {
1730
- if (!canReturnToDriverSelection)
1731
- return { status: 'back', projectDir: args.projectDir };
1732
- returnToDriverSelection = true;
1733
- break;
1734
- }
1735
1679
  await writeConnectionConfig({
1736
1680
  projectDir: args.projectDir,
1737
1681
  connectionId: connectionChoice.connectionId,
1738
- connection: withContextDepth,
1682
+ connection: withHistoricSql,
1739
1683
  io,
1740
1684
  });
1741
1685
  }
@@ -1825,23 +1769,10 @@ export async function runKtxSetupDatabasesStep(args, io, deps = {}) {
1825
1769
  returnToDriverSelection = true;
1826
1770
  break;
1827
1771
  }
1828
- const withContextDepth = await maybeApplyContextDepthConfig({
1829
- projectDir: args.projectDir,
1830
- connectionId: connectionChoice.connectionId,
1831
- connection: withHistoricSql,
1832
- args,
1833
- prompts,
1834
- });
1835
- if (withContextDepth === 'back') {
1836
- if (!canReturnToDriverSelection)
1837
- return { status: 'back', projectDir: args.projectDir };
1838
- returnToDriverSelection = true;
1839
- break;
1840
- }
1841
1772
  await writeConnectionConfig({
1842
1773
  projectDir: args.projectDir,
1843
1774
  connectionId: connectionChoice.connectionId,
1844
- connection: withContextDepth,
1775
+ connection: withHistoricSql,
1845
1776
  io,
1846
1777
  });
1847
1778
  setupStatus = await validateAndScanConnection({
@@ -114,6 +114,31 @@ function credentialRef(value, label) {
114
114
  }
115
115
  return ref;
116
116
  }
117
+ // Each connector reads exactly one credential ref; the flag name mirrors the
118
+ // ktx.yaml field it writes (auth_token_ref / api_key_ref / client_secret_ref).
119
+ const SOURCE_CREDENTIAL_FLAG = {
120
+ dbt: { field: 'sourceAuthTokenRef', flag: '--source-auth-token-ref' },
121
+ metricflow: { field: 'sourceAuthTokenRef', flag: '--source-auth-token-ref' },
122
+ lookml: { field: 'sourceAuthTokenRef', flag: '--source-auth-token-ref' },
123
+ notion: { field: 'sourceAuthTokenRef', flag: '--source-auth-token-ref' },
124
+ metabase: { field: 'sourceApiKeyRef', flag: '--source-api-key-ref' },
125
+ looker: { field: 'sourceClientSecretRef', flag: '--source-client-secret-ref' },
126
+ };
127
+ const ALL_SOURCE_CREDENTIAL_FLAGS = [
128
+ { field: 'sourceAuthTokenRef', flag: '--source-auth-token-ref' },
129
+ { field: 'sourceApiKeyRef', flag: '--source-api-key-ref' },
130
+ { field: 'sourceClientSecretRef', flag: '--source-client-secret-ref' },
131
+ ];
132
+ // Reject a credential ref flag the chosen source does not read, so a wrong flag
133
+ // fails loudly instead of being silently dropped (KLO-724).
134
+ function assertSourceCredentialFlags(source, args) {
135
+ const allowed = SOURCE_CREDENTIAL_FLAG[source];
136
+ for (const { field, flag } of ALL_SOURCE_CREDENTIAL_FLAGS) {
137
+ if (args[field] && field !== allowed.field) {
138
+ throw new Error(`${flag} does not apply to --source ${source}; use ${allowed.flag}.`);
139
+ }
140
+ }
141
+ }
117
142
  async function chooseSourceCredentialRef(input) {
118
143
  while (true) {
119
144
  const choice = await input.prompts.select({
@@ -385,7 +410,7 @@ function buildNotionConnection(args) {
385
410
  }
386
411
  return {
387
412
  driver: 'notion',
388
- auth_token_ref: credentialRef(args.sourceApiKeyRef, 'Notion token ref'),
413
+ auth_token_ref: credentialRef(args.sourceAuthTokenRef, 'Notion token ref'),
389
414
  crawl_mode: crawlMode,
390
415
  ...(rootPageIds.length > 0 ? { root_page_ids: rootPageIds } : {}),
391
416
  root_database_ids: [],
@@ -1064,11 +1089,11 @@ async function promptForInteractiveSource(args, source, prompts, io, deps, defau
1064
1089
  label: 'Notion integration token',
1065
1090
  envName: 'NOTION_TOKEN',
1066
1091
  secretFileName: `${currentState.sourceConnectionId ?? 'notion-main'}-token`,
1067
- existingRef: currentState.sourceApiKeyRef,
1092
+ existingRef: currentState.sourceAuthTokenRef,
1068
1093
  });
1069
1094
  if (ref === 'back')
1070
1095
  return 'back';
1071
- currentState.sourceApiKeyRef = ref;
1096
+ currentState.sourceAuthTokenRef = ref;
1072
1097
  return 'next';
1073
1098
  },
1074
1099
  async (currentState) => {
@@ -1096,7 +1121,7 @@ async function promptForInteractiveSource(args, source, prompts, io, deps, defau
1096
1121
  connectionId,
1097
1122
  connection: {
1098
1123
  driver: 'notion',
1099
- auth_token_ref: credentialRef(currentState.sourceApiKeyRef, 'Notion token ref'),
1124
+ auth_token_ref: credentialRef(currentState.sourceAuthTokenRef, 'Notion token ref'),
1100
1125
  crawl_mode: 'selected_roots',
1101
1126
  root_page_ids: currentState.notionRootPageIds ?? [],
1102
1127
  root_database_ids: [],
@@ -1260,7 +1285,7 @@ function sourceArgsFromExistingConnection(input) {
1260
1285
  }
1261
1286
  return sourceArgs;
1262
1287
  }
1263
- sourceArgs.sourceApiKeyRef = stringField(input.connection.auth_token_ref);
1288
+ sourceArgs.sourceAuthTokenRef = stringField(input.connection.auth_token_ref);
1264
1289
  sourceArgs.notionCrawlMode =
1265
1290
  input.connection.crawl_mode === 'all_accessible' ? 'all_accessible' : 'selected_roots';
1266
1291
  if (Array.isArray(input.connection.root_page_ids)) {
@@ -1467,6 +1492,9 @@ export async function runKtxSetupSourcesStep(args, io, deps = {}) {
1467
1492
  io.stdout.write('│ Context source setup skipped.\n');
1468
1493
  return { status: 'skipped', projectDir: args.projectDir };
1469
1494
  }
1495
+ if (args.source) {
1496
+ assertSourceCredentialFlags(args.source, args);
1497
+ }
1470
1498
  const prompts = deps.prompts ?? createPromptAdapter();
1471
1499
  const project = await loadKtxProject({ projectDir: args.projectDir });
1472
1500
  if (!hasPrimarySource(project.config)) {
package/dist/setup.js CHANGED
@@ -1,7 +1,7 @@
1
1
  import { existsSync } from 'node:fs';
2
2
  import { basename, join, resolve } from 'node:path';
3
3
  import { getLatestLocalIngestStatus } from './context/ingest/local-ingest.js';
4
- import { savedMemoryCountsForReport } from './context/ingest/reports.js';
4
+ import { ingestReportOutcome, savedMemoryCountsForReport } from './context/ingest/reports.js';
5
5
  import { ktxLocalStateDbPath } from './context/project/local-state-db.js';
6
6
  import { loadKtxProject } from './context/project/project.js';
7
7
  import { readKtxSetupState } from './context/project/setup-config.js';
@@ -105,7 +105,7 @@ function sourceConnections(config) {
105
105
  .sort((left, right) => left.connectionId.localeCompare(right.connectionId));
106
106
  }
107
107
  function reportHasSavedContext(report) {
108
- if (report.body.failedWorkUnits.length > 0) {
108
+ if (ingestReportOutcome(report) === 'error') {
109
109
  return false;
110
110
  }
111
111
  const counts = savedMemoryCountsForReport(report);
@@ -28,7 +28,12 @@ You have access to KTX MCP tools for data discovery, semantic-layer analysis, ra
28
28
  - Read entity details before writing SQL against an unfamiliar table. Do not assume column names.
29
29
  - Treat `sql_execution` as read-only. Writes are rejected by the server.
30
30
  - Validate value mentions with `dictionary_search` instead of guessing case or spelling. Treat a `dictionary_search` miss as non-authoritative. The index is built from profile-sampled values, so a missing value may simply have been outside the sample. Follow up with `sql_execution` against the most plausible columns before concluding the value is absent.
31
- - When `connection_list` shows multiple connections, pass an explicit `connectionId` to every tool that takes one and where user intent pins a specific warehouse. Required: `entity_details`, `sl_read_source`, and `sql_execution`. Required when user intent is warehouse-specific, including wording like "in our warehouse" or "this warehouse": `memory_ingest`; without `connectionId`, the memory agent cannot update the semantic layer and the knowledge lands as wiki-only. Pass `connectionId` when intent pins a warehouse, otherwise omit for unscoped discovery: `sl_query`, `discover_data`, and `dictionary_search`. Never pass `connectionId` to `connection_list`, `wiki_search`, `wiki_read`, or `memory_ingest_status`. If intent is ambiguous for a required-or-scoped tool, ask the user which warehouse before calling.
31
+ - `connectionId` scoping when `connection_list` shows multiple connections:
32
+ - Always pass it: `entity_details`, `sl_read_source`, `sql_execution`.
33
+ - Pass it when intent pins a warehouse, otherwise omit for unscoped discovery: `sl_query`, `discover_data`, `dictionary_search`.
34
+ - `memory_ingest`: pass it for warehouse-specific knowledge (e.g. "in our warehouse"); without it the memory lands as wiki-only and cannot update the semantic layer.
35
+ - Never pass it: `connection_list`, `wiki_search`, `wiki_read`, `memory_ingest_status`.
36
+ - If scoping is required but intent is ambiguous, ask which warehouse before calling.
32
37
  - Show compact result tables for small outputs. For broad results, summarize the top findings and mention the applied limit.
33
38
  - Ask a concise clarification only when the metric, date range, entity, or grain is genuinely ambiguous and cannot be inferred from context.
34
39
  </rules>
package/dist/sl.d.ts CHANGED
@@ -23,10 +23,15 @@ export type KtxSlArgs = {
23
23
  output?: string;
24
24
  json?: boolean;
25
25
  cliVersion: string;
26
+ } | {
27
+ command: 'read';
28
+ projectDir: string;
29
+ connectionId?: string;
30
+ sourceName: string;
26
31
  } | {
27
32
  command: 'validate';
28
33
  projectDir: string;
29
- connectionId: string;
34
+ connectionId?: string;
30
35
  sourceName: string;
31
36
  } | {
32
37
  command: 'query';
package/dist/sl.js CHANGED
@@ -3,7 +3,7 @@ import { createDefaultLocalQueryExecutor } from './context/connections/local-que
3
3
  import { KtxIngestEmbeddingPortAdapter } from './context/llm/embedding-port.js';
4
4
  import { loadKtxProject } from './context/project/project.js';
5
5
  import { compileLocalSlQuery } from './context/sl/local-query.js';
6
- import { listLocalSlSources, readLocalSlSource, searchLocalSlSources as defaultSearchLocalSlSources, validateLocalSlSource } from './context/sl/local-sl.js';
6
+ import { listLocalSlSources, resolveLocalSlSource, searchLocalSlSources as defaultSearchLocalSlSources, validateLocalSlSource, } from './context/sl/local-sl.js';
7
7
  import { resolveProjectEmbeddingProvider, } from './embedding-resolution.js';
8
8
  import { createManagedPythonSemanticLayerComputePort, } from './managed-python-command.js';
9
9
  import { profileMark } from './startup-profile.js';
@@ -85,6 +85,9 @@ async function readSlQueryFile(path) {
85
85
  }
86
86
  return parsed;
87
87
  }
88
+ function ambiguousSourceMessage(sourceName, connectionIds) {
89
+ return `Source '${sourceName}' exists in multiple connections: ${connectionIds.join(', ')}. Re-run with --connection-id <id>.`;
90
+ }
88
91
  export async function runKtxSl(args, io = process, deps = {}) {
89
92
  const startedAt = performance.now();
90
93
  let queryForTelemetry;
@@ -132,17 +135,38 @@ export async function runKtxSl(args, io = process, deps = {}) {
132
135
  });
133
136
  return 0;
134
137
  }
138
+ if (args.command === 'read') {
139
+ const resolved = await resolveLocalSlSource(project, {
140
+ connectionId: args.connectionId,
141
+ sourceName: args.sourceName,
142
+ });
143
+ if (resolved.kind === 'not-found') {
144
+ throw new Error(args.connectionId !== undefined
145
+ ? `No semantic-layer source '${args.sourceName}' for connection '${args.connectionId}'`
146
+ : `No semantic-layer source '${args.sourceName}'`);
147
+ }
148
+ if (resolved.kind === 'ambiguous') {
149
+ throw new Error(ambiguousSourceMessage(args.sourceName, resolved.connectionIds));
150
+ }
151
+ io.stdout.write(resolved.source.yaml);
152
+ return 0;
153
+ }
135
154
  if (args.command === 'validate') {
136
- const source = await readLocalSlSource(project, {
155
+ const resolved = await resolveLocalSlSource(project, {
137
156
  connectionId: args.connectionId,
138
157
  sourceName: args.sourceName,
139
158
  });
140
- if (!source) {
141
- throw new Error(`Semantic-layer source "${args.connectionId}/${args.sourceName}" was not found`);
159
+ if (resolved.kind === 'not-found') {
160
+ throw new Error(args.connectionId !== undefined
161
+ ? `Semantic-layer source "${args.connectionId}/${args.sourceName}" was not found`
162
+ : `Semantic-layer source "${args.sourceName}" was not found`);
142
163
  }
143
- const result = await validateLocalSlSource(source.yaml, {
164
+ if (resolved.kind === 'ambiguous') {
165
+ throw new Error(ambiguousSourceMessage(args.sourceName, resolved.connectionIds));
166
+ }
167
+ const result = await validateLocalSlSource(resolved.source.yaml, {
144
168
  project,
145
- connectionId: args.connectionId,
169
+ connectionId: resolved.source.connectionId,
146
170
  sourceName: args.sourceName,
147
171
  });
148
172
  await emitTelemetryEvent({
@@ -150,7 +174,7 @@ export async function runKtxSl(args, io = process, deps = {}) {
150
174
  projectDir: args.projectDir,
151
175
  io,
152
176
  fields: {
153
- sourceCount: source ? 1 : 0,
177
+ sourceCount: 1,
154
178
  modelCount: 0,
155
179
  validationErrorCount: result.valid ? 0 : result.errors.length,
156
180
  outcome: result.valid ? 'ok' : 'error',
@@ -163,7 +187,7 @@ export async function runKtxSl(args, io = process, deps = {}) {
163
187
  }
164
188
  return 1;
165
189
  }
166
- io.stdout.write(`Valid semantic-layer source: ${args.connectionId}/${args.sourceName}\n`);
190
+ io.stdout.write(`Valid semantic-layer source: ${resolved.source.connectionId}/${args.sourceName}\n`);
167
191
  return 0;
168
192
  }
169
193
  if (args.command === 'query') {
@@ -17,7 +17,7 @@ async function getPostHogClient(projectApiKey, host) {
17
17
  return null;
18
18
  }
19
19
  clientPromise ??= import('posthog-node')
20
- .then(({ PostHog }) => new PostHog(projectApiKey, { host, flushAt: 1, flushInterval: 0 }))
20
+ .then(({ PostHog }) => new PostHog(projectApiKey, { host, flushAt: 1, flushInterval: 0, disableGeoip: false }))
21
21
  .catch(() => null);
22
22
  return await clientPromise;
23
23
  }
@@ -69,7 +69,6 @@ export declare const telemetryEventSchemas: {
69
69
  runtime: "runtime";
70
70
  agents: "agents";
71
71
  secrets: "secrets";
72
- "database-context-depth": "database-context-depth";
73
72
  "demo-tour": "demo-tour";
74
73
  }>;
75
74
  outcome: z.ZodEnum<{
@@ -299,7 +298,9 @@ export declare const telemetryEventSchemas: {
299
298
  }>;
300
299
  durationMs: z.ZodNumber;
301
300
  errorClass: z.ZodOptional<z.ZodString>;
302
- sampleRate: z.ZodLiteral<0.1>;
301
+ sampleRate: z.ZodLiteral<1>;
302
+ mcpClientName: z.ZodOptional<z.ZodString>;
303
+ mcpClientVersion: z.ZodOptional<z.ZodString>;
303
304
  }, z.core.$strict>;
304
305
  readonly daemon_started: z.ZodObject<{
305
306
  cliVersion: z.ZodString;
@@ -433,7 +434,7 @@ export declare const telemetryEventCatalog: readonly [{
433
434
  }, {
434
435
  readonly name: "mcp_request_completed";
435
436
  readonly description: "Emitted for sampled MCP tool requests.";
436
- readonly fields: readonly ["toolName", "outcome", "durationMs", "errorClass", "sampleRate"];
437
+ readonly fields: readonly ["toolName", "outcome", "durationMs", "errorClass", "sampleRate", "mcpClientName", "mcpClientVersion"];
437
438
  }, {
438
439
  readonly name: "daemon_started";
439
440
  readonly description: "Emitted when the long-lived ktx-daemon HTTP server starts.";
@@ -33,7 +33,6 @@ const setupStepSchema = telemetryCommonEnvelopeSchema
33
33
  'embeddings',
34
34
  'secrets',
35
35
  'databases',
36
- 'database-context-depth',
37
36
  'sources',
38
37
  'context',
39
38
  'agents',
@@ -141,7 +140,12 @@ const mcpRequestCompletedSchema = telemetryCommonEnvelopeSchema
141
140
  outcome: outcomeSchema,
142
141
  durationMs: z.number().nonnegative(),
143
142
  errorClass: z.string().optional(),
144
- sampleRate: z.literal(0.1),
143
+ sampleRate: z.literal(1),
144
+ // Raw, client-tool-controlled identity from the MCP initialize handshake
145
+ // (clientInfo.name/version). Optional: clients may omit clientInfo. Stored
146
+ // verbatim — normalize the free-form names at query time, not at write time.
147
+ mcpClientName: z.string().optional(),
148
+ mcpClientVersion: z.string().optional(),
145
149
  })
146
150
  .strict();
147
151
  const daemonStartedSchema = telemetryCommonEnvelopeSchema
@@ -304,7 +308,7 @@ export const telemetryEventCatalog = [
304
308
  {
305
309
  name: 'mcp_request_completed',
306
310
  description: 'Emitted for sampled MCP tool requests.',
307
- fields: ['toolName', 'outcome', 'durationMs', 'errorClass', 'sampleRate'],
311
+ fields: ['toolName', 'outcome', 'durationMs', 'errorClass', 'sampleRate', 'mcpClientName', 'mcpClientVersion'],
308
312
  },
309
313
  {
310
314
  name: 'daemon_started',
@@ -1,5 +1,5 @@
1
1
  /** @internal */
2
- export declare const TELEMETRY_NOTICE = "ktx collects anonymous usage data to improve the product. Opt out: set KTX_TELEMETRY_DISABLED=1.";
2
+ export declare const TELEMETRY_NOTICE = "ktx collects usage data to improve the product. Opt out: set KTX_TELEMETRY_DISABLED=1.";
3
3
  /** @internal */
4
4
  export interface TelemetryIdentityEnv {
5
5
  KTX_TELEMETRY_DISABLED?: string;
@@ -4,7 +4,7 @@ import { homedir } from 'node:os';
4
4
  import { dirname, join, resolve } from 'node:path';
5
5
  import { z } from 'zod';
6
6
  /** @internal */
7
- export const TELEMETRY_NOTICE = 'ktx collects anonymous usage data to improve the product. Opt out: set KTX_TELEMETRY_DISABLED=1.';
7
+ export const TELEMETRY_NOTICE = 'ktx collects usage data to improve the product. Opt out: set KTX_TELEMETRY_DISABLED=1.';
8
8
  const NOTICE_VERSION = 1;
9
9
  const telemetryFileSchema = z
10
10
  .object({
@@ -41,16 +41,13 @@ async function writeTelemetryFile(path, value) {
41
41
  export async function loadTelemetryIdentity(options) {
42
42
  const env = options.env ?? process.env;
43
43
  const path = telemetryPath(options.homeDir ?? homedir());
44
- if (envDisablesTelemetry(env) || options.stdoutIsTTY !== true) {
45
- const existing = await readTelemetryFile(path);
46
- return {
47
- installId: existing?.installId,
48
- enabled: false,
49
- createdFile: false,
50
- noticeShown: false,
51
- path,
52
- };
44
+ if (envDisablesTelemetry(env)) {
45
+ return { enabled: false, createdFile: false, noticeShown: false, path };
53
46
  }
47
+ // Honor an already-consented identity regardless of the current surface.
48
+ // Telemetry enablement follows the persisted decision and opt-out env vars,
49
+ // not whether this invocation happens to own a TTY — MCP servers always run
50
+ // headless (stdio stubs stdout; the HTTP server runs detached).
54
51
  const existing = await readTelemetryFile(path);
55
52
  if (existing) {
56
53
  return {
@@ -61,6 +58,12 @@ export async function loadTelemetryIdentity(options) {
61
58
  path,
62
59
  };
63
60
  }
61
+ // No identity yet. Minting one means showing the one-time opt-out notice, so
62
+ // first-run creation requires an interactive surface; a headless first run
63
+ // stays disabled and defers enablement until the next interactive run.
64
+ if (options.stdoutIsTTY !== true) {
65
+ return { enabled: false, createdFile: false, noticeShown: false, path };
66
+ }
64
67
  const timestamp = (options.now ?? (() => new Date()))().toISOString();
65
68
  const next = {
66
69
  installId: randomUUID(),
@@ -7,7 +7,7 @@ export type { CommandOutcome, CompletedCommandSpan };
7
7
  export declare function showTelemetryNoticeIfNeeded(io: KtxCliIo, packageInfo: KtxCliPackageInfo): Promise<void>;
8
8
  type TelemetryEventFields<Name extends TelemetryEventName> = Omit<TelemetryEventProperties<Name>, keyof TelemetryCommonEnvelope>;
9
9
  export declare function shouldEmitMcpTelemetry(): boolean;
10
- export declare function mcpTelemetrySampleRate(): 0.1;
10
+ export declare function mcpTelemetrySampleRate(): 1;
11
11
  export declare function emitTelemetryEvent<Name extends TelemetryEventName>(input: {
12
12
  name: Name;
13
13
  fields: TelemetryEventFields<Name>;
@@ -26,7 +26,11 @@ export async function showTelemetryNoticeIfNeeded(io, packageInfo) {
26
26
  });
27
27
  }
28
28
  const emittedProjectSnapshots = new Set();
29
- const MCP_SAMPLE_RATE = 0.1;
29
+ // MCP tool calls are captured at full rate while ktx is early-stage: at current
30
+ // install counts any sampling below 1.0 yields too few events to be useful, and
31
+ // the recorded sampleRate lets us dial this down (and reweight history) once
32
+ // per-session call volume justifies it.
33
+ const MCP_SAMPLE_RATE = 1;
30
34
  let mcpSampled;
31
35
  function telemetryDebugEnabled() {
32
36
  return process.env.KTX_TELEMETRY_DEBUG === '1';