@kaelio/ktx 0.7.0 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/assets/python/{kaelio_ktx-0.7.0-py3-none-any.whl → kaelio_ktx-0.8.0-py3-none-any.whl} +0 -0
- package/assets/python/manifest.json +4 -4
- package/dist/.tsbuildinfo +1 -1
- package/dist/cli-program.js +7 -0
- package/dist/command-schemas.d.ts +1 -1
- package/dist/command-tree.js +5 -1
- package/dist/commands/completion-commands.d.ts +3 -0
- package/dist/commands/completion-commands.js +38 -0
- package/dist/commands/ingest-commands.js +0 -4
- package/dist/commands/knowledge-commands.js +15 -2
- package/dist/commands/setup-commands.js +2 -2
- package/dist/commands/sl-commands.js +19 -7
- package/dist/completion/complete-engine.d.ts +19 -0
- package/dist/completion/complete-engine.js +128 -0
- package/dist/completion/completion-scripts.d.ts +1 -0
- package/dist/completion/completion-scripts.js +36 -0
- package/dist/completion/dynamic-candidates.d.ts +6 -0
- package/dist/completion/dynamic-candidates.js +98 -0
- package/dist/connection-drivers.d.ts +3 -0
- package/dist/connection-drivers.js +17 -0
- package/dist/context/ingest/ingest-bundle.runner.d.ts +8 -0
- package/dist/context/ingest/ingest-bundle.runner.js +72 -15
- package/dist/context/ingest/ingest-profile.d.ts +102 -0
- package/dist/context/ingest/ingest-profile.js +306 -0
- package/dist/context/ingest/isolated-diff/work-unit-executor.js +25 -2
- package/dist/context/ingest/local-bundle-runtime.js +1 -0
- package/dist/context/ingest/local-ingest.d.ts +1 -1
- package/dist/context/ingest/local-ingest.js +6 -4
- package/dist/context/ingest/memory-flow/events.js +2 -1
- package/dist/context/ingest/ports.d.ts +2 -0
- package/dist/context/ingest/reports.d.ts +3 -0
- package/dist/context/ingest/reports.js +10 -0
- package/dist/context/ingest/stages/stage-3-work-units.d.ts +3 -1
- package/dist/context/ingest/stages/stage-3-work-units.js +2 -0
- package/dist/context/ingest/stages/stage-4-reconciliation.d.ts +2 -1
- package/dist/context/ingest/stages/stage-4-reconciliation.js +1 -1
- package/dist/context/ingest/tools/tool-call-logger.d.ts +6 -0
- package/dist/context/ingest/tools/tool-call-logger.js +36 -1
- package/dist/context/llm/ai-sdk-runtime.js +32 -3
- package/dist/context/llm/claude-code-runtime.js +35 -2
- package/dist/context/llm/runtime-port.d.ts +25 -0
- package/dist/context/mcp/context-tools.d.ts +2 -1
- package/dist/context/mcp/context-tools.js +82 -15
- package/dist/context/mcp/server.js +4 -0
- package/dist/context/mcp/types.d.ts +15 -1
- package/dist/context/project/config.d.ts +1 -0
- package/dist/context/project/config.js +4 -0
- package/dist/context/project/driver-schemas.js +1 -1
- package/dist/context/search/discover.js +4 -3
- package/dist/context/sl/local-sl.d.ts +15 -0
- package/dist/context/sl/local-sl.js +30 -0
- package/dist/context/wiki/local-knowledge.d.ts +10 -0
- package/dist/context/wiki/local-knowledge.js +22 -0
- package/dist/context-build-view.d.ts +0 -3
- package/dist/context-build-view.js +1 -7
- package/dist/ingest.js +7 -10
- package/dist/knowledge.d.ts +5 -0
- package/dist/knowledge.js +10 -1
- package/dist/public-ingest-copy.js +1 -1
- package/dist/public-ingest.d.ts +0 -7
- package/dist/public-ingest.js +20 -34
- package/dist/setup-context.js +6 -38
- package/dist/setup-databases.js +13 -82
- package/dist/setup-sources.js +33 -5
- package/dist/setup.js +2 -2
- package/dist/skills/analytics/SKILL.md +6 -1
- package/dist/sl.d.ts +6 -1
- package/dist/sl.js +32 -8
- package/dist/telemetry/emitter.js +1 -1
- package/dist/telemetry/events.d.ts +4 -3
- package/dist/telemetry/events.js +7 -3
- package/dist/telemetry/identity.d.ts +1 -1
- package/dist/telemetry/identity.js +13 -10
- package/dist/telemetry/index.d.ts +1 -1
- package/dist/telemetry/index.js +5 -1
- package/package.json +22 -22
- package/dist/ingest-depth.d.ts +0 -8
- package/dist/ingest-depth.js +0 -56
- package/dist/setup-database-context-depth.d.ts +0 -23
- package/dist/setup-database-context-depth.js +0 -84
package/dist/setup-context.js
CHANGED
|
@@ -6,8 +6,6 @@ import { markKtxSetupStateStepComplete, readKtxSetupState } from './context/proj
|
|
|
6
6
|
import { serializeKtxProjectConfig } from './context/project/config.js';
|
|
7
7
|
import { errorMessage, writePrefixedLines } from './clack.js';
|
|
8
8
|
import { buildPublicIngestPlan } from './public-ingest.js';
|
|
9
|
-
import { databaseContextDepth, } from './ingest-depth.js';
|
|
10
|
-
import { ensureSetupDatabaseContextDepths } from './setup-database-context-depth.js';
|
|
11
9
|
import { runContextBuild, } from './context-build-view.js';
|
|
12
10
|
import { createKtxSetupPromptAdapter, } from './setup-prompts.js';
|
|
13
11
|
const SETUP_CONTEXT_STATE_PATH = ['.ktx', 'setup', 'context-build.json'];
|
|
@@ -232,15 +230,6 @@ async function readLatestScanReport(projectDir, connectionId) {
|
|
|
232
230
|
reports.sort((left, right) => left.sortKey.localeCompare(right.sortKey));
|
|
233
231
|
return reports.at(-1)?.report ?? null;
|
|
234
232
|
}
|
|
235
|
-
function scanReportHasSchemaManifest(report, connectionId) {
|
|
236
|
-
if (!isRecord(report)) {
|
|
237
|
-
return false;
|
|
238
|
-
}
|
|
239
|
-
if (report.connectionId !== connectionId || report.dryRun === true) {
|
|
240
|
-
return false;
|
|
241
|
-
}
|
|
242
|
-
return stringArrayValue(isRecord(report.artifactPaths) ? report.artifactPaths.manifestShards : undefined).length > 0;
|
|
243
|
-
}
|
|
244
233
|
function scanReportHasCompletedDeepEnrichment(report, connectionId, relationshipsRequired) {
|
|
245
234
|
if (!isRecord(report)) {
|
|
246
235
|
return false;
|
|
@@ -260,23 +249,13 @@ function scanReportHasCompletedDeepEnrichment(report, connectionId, relationship
|
|
|
260
249
|
(!relationshipsRequired || completedStages.includes('relationships')) &&
|
|
261
250
|
stringArrayValue(report.artifactPaths.manifestShards).length > 0);
|
|
262
251
|
}
|
|
263
|
-
function scanReportSatisfiesDepth(input) {
|
|
264
|
-
if (input.depth === 'fast') {
|
|
265
|
-
return scanReportHasSchemaManifest(input.report, input.connectionId);
|
|
266
|
-
}
|
|
267
|
-
return scanReportHasCompletedDeepEnrichment(input.report, input.connectionId, input.relationshipsRequired);
|
|
268
|
-
}
|
|
269
252
|
async function verifyPrimarySourceScans(project, connectionIds) {
|
|
270
253
|
const details = [];
|
|
271
254
|
const relationshipsRequired = project.config.scan.relationships.enabled;
|
|
272
255
|
for (const connectionId of connectionIds) {
|
|
273
|
-
const connection = project.config.connections[connectionId];
|
|
274
|
-
const depth = connection ? (databaseContextDepth(connection) ?? 'fast') : 'fast';
|
|
275
256
|
const report = await readLatestScanReport(project.projectDir, connectionId);
|
|
276
|
-
if (!
|
|
277
|
-
details.push(
|
|
278
|
-
? `${connectionId}: schema context has not completed.`
|
|
279
|
-
: `${connectionId}: deep database context has not completed.`);
|
|
257
|
+
if (!scanReportHasCompletedDeepEnrichment(report, connectionId, relationshipsRequired)) {
|
|
258
|
+
details.push(`${connectionId}: database context has not completed.`);
|
|
280
259
|
}
|
|
281
260
|
}
|
|
282
261
|
return { ready: details.length === 0, details };
|
|
@@ -331,7 +310,7 @@ function writeSkippedContext(projectDir, io) {
|
|
|
331
310
|
io.stdout.write(`Build context:\n ktx setup --project-dir ${resolve(projectDir)}\n\n`);
|
|
332
311
|
io.stdout.write(`Check status:\n ktx status --project-dir ${resolve(projectDir)}\n`);
|
|
333
312
|
}
|
|
334
|
-
function writeSuccess(
|
|
313
|
+
function writeSuccess(readiness, targets, io) {
|
|
335
314
|
io.stdout.write('\nKTX context is ready for agents.\n\n');
|
|
336
315
|
io.stdout.write('Databases:\n');
|
|
337
316
|
if (targets.primarySourceConnectionIds.length === 0) {
|
|
@@ -339,9 +318,7 @@ function writeSuccess(project, readiness, targets, io) {
|
|
|
339
318
|
}
|
|
340
319
|
else {
|
|
341
320
|
for (const connectionId of targets.primarySourceConnectionIds) {
|
|
342
|
-
|
|
343
|
-
const depth = connection ? (databaseContextDepth(connection) ?? 'fast') : 'fast';
|
|
344
|
-
io.stdout.write(` ${connectionId}: ${depth === 'deep' ? 'deep context complete' : 'schema context complete'}\n`);
|
|
321
|
+
io.stdout.write(` ${connectionId}: database context complete\n`);
|
|
345
322
|
}
|
|
346
323
|
}
|
|
347
324
|
io.stdout.write('\nContext sources:\n');
|
|
@@ -466,7 +443,7 @@ async function runBuild(args, io, deps, project, targets) {
|
|
|
466
443
|
failureReason: undefined,
|
|
467
444
|
...(lastSourceProgress ? { sourceProgress: lastSourceProgress } : {}),
|
|
468
445
|
});
|
|
469
|
-
writeSuccess(
|
|
446
|
+
writeSuccess(readiness, targets, io);
|
|
470
447
|
return { status: 'ready', projectDir: args.projectDir, runId };
|
|
471
448
|
}
|
|
472
449
|
async function completeExistingContext(args, io, deps, targets) {
|
|
@@ -496,17 +473,8 @@ async function completeExistingContext(args, io, deps, targets) {
|
|
|
496
473
|
}
|
|
497
474
|
export async function runKtxSetupContextStep(args, io, deps = {}) {
|
|
498
475
|
try {
|
|
499
|
-
|
|
476
|
+
const project = await loadKtxProject({ projectDir: args.projectDir });
|
|
500
477
|
const prompts = deps.prompts ?? createPromptAdapter();
|
|
501
|
-
const depthProject = await ensureSetupDatabaseContextDepths({
|
|
502
|
-
project,
|
|
503
|
-
args,
|
|
504
|
-
prompts,
|
|
505
|
-
});
|
|
506
|
-
if (depthProject === 'back') {
|
|
507
|
-
return { status: 'back', projectDir: args.projectDir };
|
|
508
|
-
}
|
|
509
|
-
project = depthProject;
|
|
510
478
|
const existingState = await readKtxSetupContextState(args.projectDir);
|
|
511
479
|
const completedSteps = (await readKtxSetupState(args.projectDir)).completed_steps;
|
|
512
480
|
if (completedSteps.includes('context') && existingState.status === 'completed') {
|
package/dist/setup-databases.js
CHANGED
|
@@ -14,13 +14,12 @@ import { runKtxConnection } from './connection.js';
|
|
|
14
14
|
import { pickDatabaseScope as defaultPickDatabaseScope, } from './database-tree-picker.js';
|
|
15
15
|
import { withMultiselectNavigation, withTextInputNavigation } from './prompt-navigation.js';
|
|
16
16
|
import { runKtxScan } from './scan.js';
|
|
17
|
-
import { applySetupDatabaseContextDepth } from './setup-database-context-depth.js';
|
|
18
17
|
import { writeProjectLocalSecretReference } from './setup-secrets.js';
|
|
19
18
|
import { isDemoConnection } from './telemetry/demo-detect.js';
|
|
20
19
|
import { emitTelemetryEvent } from './telemetry/index.js';
|
|
21
20
|
import { createKtxSetupPromptAdapter, createKtxSetupUiAdapter, } from './setup-prompts.js';
|
|
22
21
|
const HISTORIC_SQL_WORK_UNIT_MAX_CONCURRENCY = 6;
|
|
23
|
-
const
|
|
22
|
+
const KTX_DEMO_START_URL = 'https://www.kaelio.com/start';
|
|
24
23
|
const execFileAsync = promisify(execFileCallback);
|
|
25
24
|
const DRIVER_OPTIONS = [
|
|
26
25
|
{ value: 'postgres', label: 'PostgreSQL' },
|
|
@@ -1209,39 +1208,10 @@ async function applyHistoricSqlConfigToExistingConnection(input) {
|
|
|
1209
1208
|
});
|
|
1210
1209
|
if (withHistoricSql === 'back')
|
|
1211
1210
|
return 'back';
|
|
1212
|
-
const withContextDepth = await maybeApplyContextDepthConfig({
|
|
1213
|
-
projectDir: input.projectDir,
|
|
1214
|
-
connectionId: input.connectionId,
|
|
1215
|
-
connection: withHistoricSql,
|
|
1216
|
-
args: input.args,
|
|
1217
|
-
prompts: input.prompts,
|
|
1218
|
-
});
|
|
1219
|
-
if (withContextDepth === 'back')
|
|
1220
|
-
return 'back';
|
|
1221
1211
|
await writeConnectionConfig({
|
|
1222
1212
|
projectDir: input.projectDir,
|
|
1223
1213
|
connectionId: input.connectionId,
|
|
1224
|
-
connection:
|
|
1225
|
-
});
|
|
1226
|
-
}
|
|
1227
|
-
async function maybeApplyContextDepthConfig(input) {
|
|
1228
|
-
const project = await loadKtxProject({ projectDir: input.projectDir });
|
|
1229
|
-
return await applySetupDatabaseContextDepth({
|
|
1230
|
-
project: {
|
|
1231
|
-
...project,
|
|
1232
|
-
config: {
|
|
1233
|
-
...project.config,
|
|
1234
|
-
connections: {
|
|
1235
|
-
...project.config.connections,
|
|
1236
|
-
[input.connectionId]: input.connection,
|
|
1237
|
-
},
|
|
1238
|
-
},
|
|
1239
|
-
},
|
|
1240
|
-
connection: input.connection,
|
|
1241
|
-
args: {
|
|
1242
|
-
inputMode: input.args.inputMode === 'disabled' || input.args.databaseUrl ? 'disabled' : input.args.inputMode,
|
|
1243
|
-
},
|
|
1244
|
-
prompts: input.prompts,
|
|
1214
|
+
connection: withHistoricSql,
|
|
1245
1215
|
});
|
|
1246
1216
|
}
|
|
1247
1217
|
async function validateAndScanConnection(input) {
|
|
@@ -1273,7 +1243,7 @@ async function validateAndScanConnection(input) {
|
|
|
1273
1243
|
deps: input.deps,
|
|
1274
1244
|
});
|
|
1275
1245
|
writeSetupSection(input.io, `Building schema context for ${input.connectionId}`, [
|
|
1276
|
-
'Running
|
|
1246
|
+
'Running database scan…',
|
|
1277
1247
|
]);
|
|
1278
1248
|
let scanIo = createBufferedCommandIo();
|
|
1279
1249
|
let scanCode = await scanConnection(input.projectDir, input.connectionId, scanIo);
|
|
@@ -1281,7 +1251,7 @@ async function validateAndScanConnection(input) {
|
|
|
1281
1251
|
const nativeSqliteDetail = nativeSqliteAbiMismatchDetail(`${scanIo.stderrText()}\n${scanIo.stdoutText()}`);
|
|
1282
1252
|
if (nativeSqliteDetail) {
|
|
1283
1253
|
writePrefixedLines((chunk) => input.io.stderr.write(chunk), [
|
|
1284
|
-
`
|
|
1254
|
+
`Database scan failed for ${input.connectionId}.`,
|
|
1285
1255
|
'Native SQLite is built for a different Node.js ABI.',
|
|
1286
1256
|
`Detail: ${nativeSqliteDetail}`,
|
|
1287
1257
|
'Rebuilding Native SQLite with pnpm run native:rebuild…',
|
|
@@ -1289,7 +1259,7 @@ async function validateAndScanConnection(input) {
|
|
|
1289
1259
|
const rebuildNativeSqlite = input.deps.rebuildNativeSqlite ?? defaultRebuildNativeSqlite;
|
|
1290
1260
|
const rebuildCode = await rebuildNativeSqlite(input.io);
|
|
1291
1261
|
if (rebuildCode === 0) {
|
|
1292
|
-
writePrefixedLines((chunk) => input.io.stderr.write(chunk), 'Native SQLite rebuild complete. Retrying
|
|
1262
|
+
writePrefixedLines((chunk) => input.io.stderr.write(chunk), 'Native SQLite rebuild complete. Retrying database scan…');
|
|
1293
1263
|
const retryScanIo = createBufferedCommandIo();
|
|
1294
1264
|
scanCode = await scanConnection(input.projectDir, input.connectionId, retryScanIo);
|
|
1295
1265
|
scanIo = retryScanIo;
|
|
@@ -1297,18 +1267,18 @@ async function validateAndScanConnection(input) {
|
|
|
1297
1267
|
if (scanCode !== 0) {
|
|
1298
1268
|
writePrefixedLines((chunk) => input.io.stderr.write(chunk), [
|
|
1299
1269
|
rebuildCode === 0
|
|
1300
|
-
? `
|
|
1270
|
+
? `Database scan still failed for ${input.connectionId} after rebuilding Native SQLite.`
|
|
1301
1271
|
: `Native SQLite rebuild failed for ${input.connectionId}.`,
|
|
1302
1272
|
'Fix: pnpm run native:rebuild',
|
|
1303
|
-
`Retry: ktx ingest ${input.connectionId} --project-dir ${input.projectDir}
|
|
1273
|
+
`Retry: ktx ingest ${input.connectionId} --project-dir ${input.projectDir}`,
|
|
1304
1274
|
].join('\n'));
|
|
1305
1275
|
}
|
|
1306
1276
|
}
|
|
1307
1277
|
else {
|
|
1308
1278
|
flushPrefixedBufferedCommandOutput(input.io, scanIo);
|
|
1309
1279
|
writePrefixedLines((chunk) => input.io.stderr.write(chunk), [
|
|
1310
|
-
`
|
|
1311
|
-
`Debug command: ktx ingest ${input.connectionId} --project-dir ${input.projectDir} --
|
|
1280
|
+
`Database scan failed for ${input.connectionId}.`,
|
|
1281
|
+
`Debug command: ktx ingest ${input.connectionId} --project-dir ${input.projectDir} --debug`,
|
|
1312
1282
|
].join('\n'));
|
|
1313
1283
|
}
|
|
1314
1284
|
if (scanCode !== 0) {
|
|
@@ -1334,7 +1304,7 @@ async function chooseDrivers(args, io, prompts, options) {
|
|
|
1334
1304
|
return 'missing-input';
|
|
1335
1305
|
}
|
|
1336
1306
|
const initialValues = unique(options?.initialDrivers ?? []);
|
|
1337
|
-
createKtxSetupUiAdapter().note(`Get demo credentials
|
|
1307
|
+
createKtxSetupUiAdapter().note(`Get demo credentials: ${KTX_DEMO_START_URL}`, '🎁 Need a warehouse to play with?', io);
|
|
1338
1308
|
const choices = await prompts.multiselect({
|
|
1339
1309
|
message: withMultiselectNavigation('Which databases should KTX connect to?'),
|
|
1340
1310
|
options: [...DRIVER_OPTIONS],
|
|
@@ -1690,23 +1660,10 @@ export async function runKtxSetupDatabasesStep(args, io, deps = {}) {
|
|
|
1690
1660
|
returnToDriverSelection = true;
|
|
1691
1661
|
break;
|
|
1692
1662
|
}
|
|
1693
|
-
const withContextDepth = await maybeApplyContextDepthConfig({
|
|
1694
|
-
projectDir: args.projectDir,
|
|
1695
|
-
connectionId: connectionChoice.connectionId,
|
|
1696
|
-
connection: withHistoricSql,
|
|
1697
|
-
args,
|
|
1698
|
-
prompts,
|
|
1699
|
-
});
|
|
1700
|
-
if (withContextDepth === 'back') {
|
|
1701
|
-
if (!canReturnToDriverSelection)
|
|
1702
|
-
return { status: 'back', projectDir: args.projectDir };
|
|
1703
|
-
returnToDriverSelection = true;
|
|
1704
|
-
break;
|
|
1705
|
-
}
|
|
1706
1663
|
await writeConnectionConfig({
|
|
1707
1664
|
projectDir: args.projectDir,
|
|
1708
1665
|
connectionId: connectionChoice.connectionId,
|
|
1709
|
-
connection:
|
|
1666
|
+
connection: withHistoricSql,
|
|
1710
1667
|
io,
|
|
1711
1668
|
});
|
|
1712
1669
|
}
|
|
@@ -1719,23 +1676,10 @@ export async function runKtxSetupDatabasesStep(args, io, deps = {}) {
|
|
|
1719
1676
|
returnToDriverSelection = true;
|
|
1720
1677
|
break;
|
|
1721
1678
|
}
|
|
1722
|
-
const withContextDepth = await maybeApplyContextDepthConfig({
|
|
1723
|
-
projectDir: args.projectDir,
|
|
1724
|
-
connectionId: connectionChoice.connectionId,
|
|
1725
|
-
connection: withHistoricSql,
|
|
1726
|
-
args,
|
|
1727
|
-
prompts,
|
|
1728
|
-
});
|
|
1729
|
-
if (withContextDepth === 'back') {
|
|
1730
|
-
if (!canReturnToDriverSelection)
|
|
1731
|
-
return { status: 'back', projectDir: args.projectDir };
|
|
1732
|
-
returnToDriverSelection = true;
|
|
1733
|
-
break;
|
|
1734
|
-
}
|
|
1735
1679
|
await writeConnectionConfig({
|
|
1736
1680
|
projectDir: args.projectDir,
|
|
1737
1681
|
connectionId: connectionChoice.connectionId,
|
|
1738
|
-
connection:
|
|
1682
|
+
connection: withHistoricSql,
|
|
1739
1683
|
io,
|
|
1740
1684
|
});
|
|
1741
1685
|
}
|
|
@@ -1825,23 +1769,10 @@ export async function runKtxSetupDatabasesStep(args, io, deps = {}) {
|
|
|
1825
1769
|
returnToDriverSelection = true;
|
|
1826
1770
|
break;
|
|
1827
1771
|
}
|
|
1828
|
-
const withContextDepth = await maybeApplyContextDepthConfig({
|
|
1829
|
-
projectDir: args.projectDir,
|
|
1830
|
-
connectionId: connectionChoice.connectionId,
|
|
1831
|
-
connection: withHistoricSql,
|
|
1832
|
-
args,
|
|
1833
|
-
prompts,
|
|
1834
|
-
});
|
|
1835
|
-
if (withContextDepth === 'back') {
|
|
1836
|
-
if (!canReturnToDriverSelection)
|
|
1837
|
-
return { status: 'back', projectDir: args.projectDir };
|
|
1838
|
-
returnToDriverSelection = true;
|
|
1839
|
-
break;
|
|
1840
|
-
}
|
|
1841
1772
|
await writeConnectionConfig({
|
|
1842
1773
|
projectDir: args.projectDir,
|
|
1843
1774
|
connectionId: connectionChoice.connectionId,
|
|
1844
|
-
connection:
|
|
1775
|
+
connection: withHistoricSql,
|
|
1845
1776
|
io,
|
|
1846
1777
|
});
|
|
1847
1778
|
setupStatus = await validateAndScanConnection({
|
package/dist/setup-sources.js
CHANGED
|
@@ -114,6 +114,31 @@ function credentialRef(value, label) {
|
|
|
114
114
|
}
|
|
115
115
|
return ref;
|
|
116
116
|
}
|
|
117
|
+
// Each connector reads exactly one credential ref; the flag name mirrors the
|
|
118
|
+
// ktx.yaml field it writes (auth_token_ref / api_key_ref / client_secret_ref).
|
|
119
|
+
const SOURCE_CREDENTIAL_FLAG = {
|
|
120
|
+
dbt: { field: 'sourceAuthTokenRef', flag: '--source-auth-token-ref' },
|
|
121
|
+
metricflow: { field: 'sourceAuthTokenRef', flag: '--source-auth-token-ref' },
|
|
122
|
+
lookml: { field: 'sourceAuthTokenRef', flag: '--source-auth-token-ref' },
|
|
123
|
+
notion: { field: 'sourceAuthTokenRef', flag: '--source-auth-token-ref' },
|
|
124
|
+
metabase: { field: 'sourceApiKeyRef', flag: '--source-api-key-ref' },
|
|
125
|
+
looker: { field: 'sourceClientSecretRef', flag: '--source-client-secret-ref' },
|
|
126
|
+
};
|
|
127
|
+
const ALL_SOURCE_CREDENTIAL_FLAGS = [
|
|
128
|
+
{ field: 'sourceAuthTokenRef', flag: '--source-auth-token-ref' },
|
|
129
|
+
{ field: 'sourceApiKeyRef', flag: '--source-api-key-ref' },
|
|
130
|
+
{ field: 'sourceClientSecretRef', flag: '--source-client-secret-ref' },
|
|
131
|
+
];
|
|
132
|
+
// Reject a credential ref flag the chosen source does not read, so a wrong flag
|
|
133
|
+
// fails loudly instead of being silently dropped (KLO-724).
|
|
134
|
+
function assertSourceCredentialFlags(source, args) {
|
|
135
|
+
const allowed = SOURCE_CREDENTIAL_FLAG[source];
|
|
136
|
+
for (const { field, flag } of ALL_SOURCE_CREDENTIAL_FLAGS) {
|
|
137
|
+
if (args[field] && field !== allowed.field) {
|
|
138
|
+
throw new Error(`${flag} does not apply to --source ${source}; use ${allowed.flag}.`);
|
|
139
|
+
}
|
|
140
|
+
}
|
|
141
|
+
}
|
|
117
142
|
async function chooseSourceCredentialRef(input) {
|
|
118
143
|
while (true) {
|
|
119
144
|
const choice = await input.prompts.select({
|
|
@@ -385,7 +410,7 @@ function buildNotionConnection(args) {
|
|
|
385
410
|
}
|
|
386
411
|
return {
|
|
387
412
|
driver: 'notion',
|
|
388
|
-
auth_token_ref: credentialRef(args.
|
|
413
|
+
auth_token_ref: credentialRef(args.sourceAuthTokenRef, 'Notion token ref'),
|
|
389
414
|
crawl_mode: crawlMode,
|
|
390
415
|
...(rootPageIds.length > 0 ? { root_page_ids: rootPageIds } : {}),
|
|
391
416
|
root_database_ids: [],
|
|
@@ -1064,11 +1089,11 @@ async function promptForInteractiveSource(args, source, prompts, io, deps, defau
|
|
|
1064
1089
|
label: 'Notion integration token',
|
|
1065
1090
|
envName: 'NOTION_TOKEN',
|
|
1066
1091
|
secretFileName: `${currentState.sourceConnectionId ?? 'notion-main'}-token`,
|
|
1067
|
-
existingRef: currentState.
|
|
1092
|
+
existingRef: currentState.sourceAuthTokenRef,
|
|
1068
1093
|
});
|
|
1069
1094
|
if (ref === 'back')
|
|
1070
1095
|
return 'back';
|
|
1071
|
-
currentState.
|
|
1096
|
+
currentState.sourceAuthTokenRef = ref;
|
|
1072
1097
|
return 'next';
|
|
1073
1098
|
},
|
|
1074
1099
|
async (currentState) => {
|
|
@@ -1096,7 +1121,7 @@ async function promptForInteractiveSource(args, source, prompts, io, deps, defau
|
|
|
1096
1121
|
connectionId,
|
|
1097
1122
|
connection: {
|
|
1098
1123
|
driver: 'notion',
|
|
1099
|
-
auth_token_ref: credentialRef(currentState.
|
|
1124
|
+
auth_token_ref: credentialRef(currentState.sourceAuthTokenRef, 'Notion token ref'),
|
|
1100
1125
|
crawl_mode: 'selected_roots',
|
|
1101
1126
|
root_page_ids: currentState.notionRootPageIds ?? [],
|
|
1102
1127
|
root_database_ids: [],
|
|
@@ -1260,7 +1285,7 @@ function sourceArgsFromExistingConnection(input) {
|
|
|
1260
1285
|
}
|
|
1261
1286
|
return sourceArgs;
|
|
1262
1287
|
}
|
|
1263
|
-
sourceArgs.
|
|
1288
|
+
sourceArgs.sourceAuthTokenRef = stringField(input.connection.auth_token_ref);
|
|
1264
1289
|
sourceArgs.notionCrawlMode =
|
|
1265
1290
|
input.connection.crawl_mode === 'all_accessible' ? 'all_accessible' : 'selected_roots';
|
|
1266
1291
|
if (Array.isArray(input.connection.root_page_ids)) {
|
|
@@ -1467,6 +1492,9 @@ export async function runKtxSetupSourcesStep(args, io, deps = {}) {
|
|
|
1467
1492
|
io.stdout.write('│ Context source setup skipped.\n');
|
|
1468
1493
|
return { status: 'skipped', projectDir: args.projectDir };
|
|
1469
1494
|
}
|
|
1495
|
+
if (args.source) {
|
|
1496
|
+
assertSourceCredentialFlags(args.source, args);
|
|
1497
|
+
}
|
|
1470
1498
|
const prompts = deps.prompts ?? createPromptAdapter();
|
|
1471
1499
|
const project = await loadKtxProject({ projectDir: args.projectDir });
|
|
1472
1500
|
if (!hasPrimarySource(project.config)) {
|
package/dist/setup.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { existsSync } from 'node:fs';
|
|
2
2
|
import { basename, join, resolve } from 'node:path';
|
|
3
3
|
import { getLatestLocalIngestStatus } from './context/ingest/local-ingest.js';
|
|
4
|
-
import { savedMemoryCountsForReport } from './context/ingest/reports.js';
|
|
4
|
+
import { ingestReportOutcome, savedMemoryCountsForReport } from './context/ingest/reports.js';
|
|
5
5
|
import { ktxLocalStateDbPath } from './context/project/local-state-db.js';
|
|
6
6
|
import { loadKtxProject } from './context/project/project.js';
|
|
7
7
|
import { readKtxSetupState } from './context/project/setup-config.js';
|
|
@@ -105,7 +105,7 @@ function sourceConnections(config) {
|
|
|
105
105
|
.sort((left, right) => left.connectionId.localeCompare(right.connectionId));
|
|
106
106
|
}
|
|
107
107
|
function reportHasSavedContext(report) {
|
|
108
|
-
if (report
|
|
108
|
+
if (ingestReportOutcome(report) === 'error') {
|
|
109
109
|
return false;
|
|
110
110
|
}
|
|
111
111
|
const counts = savedMemoryCountsForReport(report);
|
|
@@ -28,7 +28,12 @@ You have access to KTX MCP tools for data discovery, semantic-layer analysis, ra
|
|
|
28
28
|
- Read entity details before writing SQL against an unfamiliar table. Do not assume column names.
|
|
29
29
|
- Treat `sql_execution` as read-only. Writes are rejected by the server.
|
|
30
30
|
- Validate value mentions with `dictionary_search` instead of guessing case or spelling. Treat a `dictionary_search` miss as non-authoritative. The index is built from profile-sampled values, so a missing value may simply have been outside the sample. Follow up with `sql_execution` against the most plausible columns before concluding the value is absent.
|
|
31
|
-
-
|
|
31
|
+
- `connectionId` scoping when `connection_list` shows multiple connections:
|
|
32
|
+
- Always pass it: `entity_details`, `sl_read_source`, `sql_execution`.
|
|
33
|
+
- Pass it when intent pins a warehouse, otherwise omit for unscoped discovery: `sl_query`, `discover_data`, `dictionary_search`.
|
|
34
|
+
- `memory_ingest`: pass it for warehouse-specific knowledge (e.g. "in our warehouse"); without it the memory lands as wiki-only and cannot update the semantic layer.
|
|
35
|
+
- Never pass it: `connection_list`, `wiki_search`, `wiki_read`, `memory_ingest_status`.
|
|
36
|
+
- If scoping is required but intent is ambiguous, ask which warehouse before calling.
|
|
32
37
|
- Show compact result tables for small outputs. For broad results, summarize the top findings and mention the applied limit.
|
|
33
38
|
- Ask a concise clarification only when the metric, date range, entity, or grain is genuinely ambiguous and cannot be inferred from context.
|
|
34
39
|
</rules>
|
package/dist/sl.d.ts
CHANGED
|
@@ -23,10 +23,15 @@ export type KtxSlArgs = {
|
|
|
23
23
|
output?: string;
|
|
24
24
|
json?: boolean;
|
|
25
25
|
cliVersion: string;
|
|
26
|
+
} | {
|
|
27
|
+
command: 'read';
|
|
28
|
+
projectDir: string;
|
|
29
|
+
connectionId?: string;
|
|
30
|
+
sourceName: string;
|
|
26
31
|
} | {
|
|
27
32
|
command: 'validate';
|
|
28
33
|
projectDir: string;
|
|
29
|
-
connectionId
|
|
34
|
+
connectionId?: string;
|
|
30
35
|
sourceName: string;
|
|
31
36
|
} | {
|
|
32
37
|
command: 'query';
|
package/dist/sl.js
CHANGED
|
@@ -3,7 +3,7 @@ import { createDefaultLocalQueryExecutor } from './context/connections/local-que
|
|
|
3
3
|
import { KtxIngestEmbeddingPortAdapter } from './context/llm/embedding-port.js';
|
|
4
4
|
import { loadKtxProject } from './context/project/project.js';
|
|
5
5
|
import { compileLocalSlQuery } from './context/sl/local-query.js';
|
|
6
|
-
import { listLocalSlSources,
|
|
6
|
+
import { listLocalSlSources, resolveLocalSlSource, searchLocalSlSources as defaultSearchLocalSlSources, validateLocalSlSource, } from './context/sl/local-sl.js';
|
|
7
7
|
import { resolveProjectEmbeddingProvider, } from './embedding-resolution.js';
|
|
8
8
|
import { createManagedPythonSemanticLayerComputePort, } from './managed-python-command.js';
|
|
9
9
|
import { profileMark } from './startup-profile.js';
|
|
@@ -85,6 +85,9 @@ async function readSlQueryFile(path) {
|
|
|
85
85
|
}
|
|
86
86
|
return parsed;
|
|
87
87
|
}
|
|
88
|
+
function ambiguousSourceMessage(sourceName, connectionIds) {
|
|
89
|
+
return `Source '${sourceName}' exists in multiple connections: ${connectionIds.join(', ')}. Re-run with --connection-id <id>.`;
|
|
90
|
+
}
|
|
88
91
|
export async function runKtxSl(args, io = process, deps = {}) {
|
|
89
92
|
const startedAt = performance.now();
|
|
90
93
|
let queryForTelemetry;
|
|
@@ -132,17 +135,38 @@ export async function runKtxSl(args, io = process, deps = {}) {
|
|
|
132
135
|
});
|
|
133
136
|
return 0;
|
|
134
137
|
}
|
|
138
|
+
if (args.command === 'read') {
|
|
139
|
+
const resolved = await resolveLocalSlSource(project, {
|
|
140
|
+
connectionId: args.connectionId,
|
|
141
|
+
sourceName: args.sourceName,
|
|
142
|
+
});
|
|
143
|
+
if (resolved.kind === 'not-found') {
|
|
144
|
+
throw new Error(args.connectionId !== undefined
|
|
145
|
+
? `No semantic-layer source '${args.sourceName}' for connection '${args.connectionId}'`
|
|
146
|
+
: `No semantic-layer source '${args.sourceName}'`);
|
|
147
|
+
}
|
|
148
|
+
if (resolved.kind === 'ambiguous') {
|
|
149
|
+
throw new Error(ambiguousSourceMessage(args.sourceName, resolved.connectionIds));
|
|
150
|
+
}
|
|
151
|
+
io.stdout.write(resolved.source.yaml);
|
|
152
|
+
return 0;
|
|
153
|
+
}
|
|
135
154
|
if (args.command === 'validate') {
|
|
136
|
-
const
|
|
155
|
+
const resolved = await resolveLocalSlSource(project, {
|
|
137
156
|
connectionId: args.connectionId,
|
|
138
157
|
sourceName: args.sourceName,
|
|
139
158
|
});
|
|
140
|
-
if (
|
|
141
|
-
throw new Error(
|
|
159
|
+
if (resolved.kind === 'not-found') {
|
|
160
|
+
throw new Error(args.connectionId !== undefined
|
|
161
|
+
? `Semantic-layer source "${args.connectionId}/${args.sourceName}" was not found`
|
|
162
|
+
: `Semantic-layer source "${args.sourceName}" was not found`);
|
|
142
163
|
}
|
|
143
|
-
|
|
164
|
+
if (resolved.kind === 'ambiguous') {
|
|
165
|
+
throw new Error(ambiguousSourceMessage(args.sourceName, resolved.connectionIds));
|
|
166
|
+
}
|
|
167
|
+
const result = await validateLocalSlSource(resolved.source.yaml, {
|
|
144
168
|
project,
|
|
145
|
-
connectionId:
|
|
169
|
+
connectionId: resolved.source.connectionId,
|
|
146
170
|
sourceName: args.sourceName,
|
|
147
171
|
});
|
|
148
172
|
await emitTelemetryEvent({
|
|
@@ -150,7 +174,7 @@ export async function runKtxSl(args, io = process, deps = {}) {
|
|
|
150
174
|
projectDir: args.projectDir,
|
|
151
175
|
io,
|
|
152
176
|
fields: {
|
|
153
|
-
sourceCount:
|
|
177
|
+
sourceCount: 1,
|
|
154
178
|
modelCount: 0,
|
|
155
179
|
validationErrorCount: result.valid ? 0 : result.errors.length,
|
|
156
180
|
outcome: result.valid ? 'ok' : 'error',
|
|
@@ -163,7 +187,7 @@ export async function runKtxSl(args, io = process, deps = {}) {
|
|
|
163
187
|
}
|
|
164
188
|
return 1;
|
|
165
189
|
}
|
|
166
|
-
io.stdout.write(`Valid semantic-layer source: ${
|
|
190
|
+
io.stdout.write(`Valid semantic-layer source: ${resolved.source.connectionId}/${args.sourceName}\n`);
|
|
167
191
|
return 0;
|
|
168
192
|
}
|
|
169
193
|
if (args.command === 'query') {
|
|
@@ -17,7 +17,7 @@ async function getPostHogClient(projectApiKey, host) {
|
|
|
17
17
|
return null;
|
|
18
18
|
}
|
|
19
19
|
clientPromise ??= import('posthog-node')
|
|
20
|
-
.then(({ PostHog }) => new PostHog(projectApiKey, { host, flushAt: 1, flushInterval: 0 }))
|
|
20
|
+
.then(({ PostHog }) => new PostHog(projectApiKey, { host, flushAt: 1, flushInterval: 0, disableGeoip: false }))
|
|
21
21
|
.catch(() => null);
|
|
22
22
|
return await clientPromise;
|
|
23
23
|
}
|
|
@@ -69,7 +69,6 @@ export declare const telemetryEventSchemas: {
|
|
|
69
69
|
runtime: "runtime";
|
|
70
70
|
agents: "agents";
|
|
71
71
|
secrets: "secrets";
|
|
72
|
-
"database-context-depth": "database-context-depth";
|
|
73
72
|
"demo-tour": "demo-tour";
|
|
74
73
|
}>;
|
|
75
74
|
outcome: z.ZodEnum<{
|
|
@@ -299,7 +298,9 @@ export declare const telemetryEventSchemas: {
|
|
|
299
298
|
}>;
|
|
300
299
|
durationMs: z.ZodNumber;
|
|
301
300
|
errorClass: z.ZodOptional<z.ZodString>;
|
|
302
|
-
sampleRate: z.ZodLiteral<
|
|
301
|
+
sampleRate: z.ZodLiteral<1>;
|
|
302
|
+
mcpClientName: z.ZodOptional<z.ZodString>;
|
|
303
|
+
mcpClientVersion: z.ZodOptional<z.ZodString>;
|
|
303
304
|
}, z.core.$strict>;
|
|
304
305
|
readonly daemon_started: z.ZodObject<{
|
|
305
306
|
cliVersion: z.ZodString;
|
|
@@ -433,7 +434,7 @@ export declare const telemetryEventCatalog: readonly [{
|
|
|
433
434
|
}, {
|
|
434
435
|
readonly name: "mcp_request_completed";
|
|
435
436
|
readonly description: "Emitted for sampled MCP tool requests.";
|
|
436
|
-
readonly fields: readonly ["toolName", "outcome", "durationMs", "errorClass", "sampleRate"];
|
|
437
|
+
readonly fields: readonly ["toolName", "outcome", "durationMs", "errorClass", "sampleRate", "mcpClientName", "mcpClientVersion"];
|
|
437
438
|
}, {
|
|
438
439
|
readonly name: "daemon_started";
|
|
439
440
|
readonly description: "Emitted when the long-lived ktx-daemon HTTP server starts.";
|
package/dist/telemetry/events.js
CHANGED
|
@@ -33,7 +33,6 @@ const setupStepSchema = telemetryCommonEnvelopeSchema
|
|
|
33
33
|
'embeddings',
|
|
34
34
|
'secrets',
|
|
35
35
|
'databases',
|
|
36
|
-
'database-context-depth',
|
|
37
36
|
'sources',
|
|
38
37
|
'context',
|
|
39
38
|
'agents',
|
|
@@ -141,7 +140,12 @@ const mcpRequestCompletedSchema = telemetryCommonEnvelopeSchema
|
|
|
141
140
|
outcome: outcomeSchema,
|
|
142
141
|
durationMs: z.number().nonnegative(),
|
|
143
142
|
errorClass: z.string().optional(),
|
|
144
|
-
sampleRate: z.literal(
|
|
143
|
+
sampleRate: z.literal(1),
|
|
144
|
+
// Raw, client-tool-controlled identity from the MCP initialize handshake
|
|
145
|
+
// (clientInfo.name/version). Optional: clients may omit clientInfo. Stored
|
|
146
|
+
// verbatim — normalize the free-form names at query time, not at write time.
|
|
147
|
+
mcpClientName: z.string().optional(),
|
|
148
|
+
mcpClientVersion: z.string().optional(),
|
|
145
149
|
})
|
|
146
150
|
.strict();
|
|
147
151
|
const daemonStartedSchema = telemetryCommonEnvelopeSchema
|
|
@@ -304,7 +308,7 @@ export const telemetryEventCatalog = [
|
|
|
304
308
|
{
|
|
305
309
|
name: 'mcp_request_completed',
|
|
306
310
|
description: 'Emitted for sampled MCP tool requests.',
|
|
307
|
-
fields: ['toolName', 'outcome', 'durationMs', 'errorClass', 'sampleRate'],
|
|
311
|
+
fields: ['toolName', 'outcome', 'durationMs', 'errorClass', 'sampleRate', 'mcpClientName', 'mcpClientVersion'],
|
|
308
312
|
},
|
|
309
313
|
{
|
|
310
314
|
name: 'daemon_started',
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
/** @internal */
|
|
2
|
-
export declare const TELEMETRY_NOTICE = "ktx collects
|
|
2
|
+
export declare const TELEMETRY_NOTICE = "ktx collects usage data to improve the product. Opt out: set KTX_TELEMETRY_DISABLED=1.";
|
|
3
3
|
/** @internal */
|
|
4
4
|
export interface TelemetryIdentityEnv {
|
|
5
5
|
KTX_TELEMETRY_DISABLED?: string;
|
|
@@ -4,7 +4,7 @@ import { homedir } from 'node:os';
|
|
|
4
4
|
import { dirname, join, resolve } from 'node:path';
|
|
5
5
|
import { z } from 'zod';
|
|
6
6
|
/** @internal */
|
|
7
|
-
export const TELEMETRY_NOTICE = 'ktx collects
|
|
7
|
+
export const TELEMETRY_NOTICE = 'ktx collects usage data to improve the product. Opt out: set KTX_TELEMETRY_DISABLED=1.';
|
|
8
8
|
const NOTICE_VERSION = 1;
|
|
9
9
|
const telemetryFileSchema = z
|
|
10
10
|
.object({
|
|
@@ -41,16 +41,13 @@ async function writeTelemetryFile(path, value) {
|
|
|
41
41
|
export async function loadTelemetryIdentity(options) {
|
|
42
42
|
const env = options.env ?? process.env;
|
|
43
43
|
const path = telemetryPath(options.homeDir ?? homedir());
|
|
44
|
-
if (envDisablesTelemetry(env)
|
|
45
|
-
|
|
46
|
-
return {
|
|
47
|
-
installId: existing?.installId,
|
|
48
|
-
enabled: false,
|
|
49
|
-
createdFile: false,
|
|
50
|
-
noticeShown: false,
|
|
51
|
-
path,
|
|
52
|
-
};
|
|
44
|
+
if (envDisablesTelemetry(env)) {
|
|
45
|
+
return { enabled: false, createdFile: false, noticeShown: false, path };
|
|
53
46
|
}
|
|
47
|
+
// Honor an already-consented identity regardless of the current surface.
|
|
48
|
+
// Telemetry enablement follows the persisted decision and opt-out env vars,
|
|
49
|
+
// not whether this invocation happens to own a TTY — MCP servers always run
|
|
50
|
+
// headless (stdio stubs stdout; the HTTP server runs detached).
|
|
54
51
|
const existing = await readTelemetryFile(path);
|
|
55
52
|
if (existing) {
|
|
56
53
|
return {
|
|
@@ -61,6 +58,12 @@ export async function loadTelemetryIdentity(options) {
|
|
|
61
58
|
path,
|
|
62
59
|
};
|
|
63
60
|
}
|
|
61
|
+
// No identity yet. Minting one means showing the one-time opt-out notice, so
|
|
62
|
+
// first-run creation requires an interactive surface; a headless first run
|
|
63
|
+
// stays disabled and defers enablement until the next interactive run.
|
|
64
|
+
if (options.stdoutIsTTY !== true) {
|
|
65
|
+
return { enabled: false, createdFile: false, noticeShown: false, path };
|
|
66
|
+
}
|
|
64
67
|
const timestamp = (options.now ?? (() => new Date()))().toISOString();
|
|
65
68
|
const next = {
|
|
66
69
|
installId: randomUUID(),
|
|
@@ -7,7 +7,7 @@ export type { CommandOutcome, CompletedCommandSpan };
|
|
|
7
7
|
export declare function showTelemetryNoticeIfNeeded(io: KtxCliIo, packageInfo: KtxCliPackageInfo): Promise<void>;
|
|
8
8
|
type TelemetryEventFields<Name extends TelemetryEventName> = Omit<TelemetryEventProperties<Name>, keyof TelemetryCommonEnvelope>;
|
|
9
9
|
export declare function shouldEmitMcpTelemetry(): boolean;
|
|
10
|
-
export declare function mcpTelemetrySampleRate():
|
|
10
|
+
export declare function mcpTelemetrySampleRate(): 1;
|
|
11
11
|
export declare function emitTelemetryEvent<Name extends TelemetryEventName>(input: {
|
|
12
12
|
name: Name;
|
|
13
13
|
fields: TelemetryEventFields<Name>;
|
package/dist/telemetry/index.js
CHANGED
|
@@ -26,7 +26,11 @@ export async function showTelemetryNoticeIfNeeded(io, packageInfo) {
|
|
|
26
26
|
});
|
|
27
27
|
}
|
|
28
28
|
const emittedProjectSnapshots = new Set();
|
|
29
|
-
|
|
29
|
+
// MCP tool calls are captured at full rate while ktx is early-stage: at current
|
|
30
|
+
// install counts any sampling below 1.0 yields too few events to be useful, and
|
|
31
|
+
// the recorded sampleRate lets us dial this down (and reweight history) once
|
|
32
|
+
// per-session call volume justifies it.
|
|
33
|
+
const MCP_SAMPLE_RATE = 1;
|
|
30
34
|
let mcpSampled;
|
|
31
35
|
function telemetryDebugEnabled() {
|
|
32
36
|
return process.env.KTX_TELEMETRY_DEBUG === '1';
|