@kaelio/ktx 0.6.0 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/assets/python/{kaelio_ktx-0.6.0-py3-none-any.whl → kaelio_ktx-0.8.0-py3-none-any.whl} +0 -0
- package/assets/python/manifest.json +4 -4
- package/dist/.tsbuildinfo +1 -1
- package/dist/cli-program.js +7 -0
- package/dist/command-schemas.d.ts +1 -1
- package/dist/command-tree.js +5 -1
- package/dist/commands/completion-commands.d.ts +3 -0
- package/dist/commands/completion-commands.js +38 -0
- package/dist/commands/ingest-commands.js +0 -4
- package/dist/commands/knowledge-commands.js +15 -2
- package/dist/commands/setup-commands.js +2 -2
- package/dist/commands/sl-commands.js +19 -7
- package/dist/completion/complete-engine.d.ts +19 -0
- package/dist/completion/complete-engine.js +128 -0
- package/dist/completion/completion-scripts.d.ts +1 -0
- package/dist/completion/completion-scripts.js +36 -0
- package/dist/completion/dynamic-candidates.d.ts +6 -0
- package/dist/completion/dynamic-candidates.js +98 -0
- package/dist/connection-drivers.d.ts +3 -0
- package/dist/connection-drivers.js +17 -0
- package/dist/context/ingest/ingest-bundle.runner.d.ts +8 -0
- package/dist/context/ingest/ingest-bundle.runner.js +72 -15
- package/dist/context/ingest/ingest-profile.d.ts +102 -0
- package/dist/context/ingest/ingest-profile.js +306 -0
- package/dist/context/ingest/isolated-diff/work-unit-executor.js +25 -2
- package/dist/context/ingest/local-bundle-runtime.js +1 -0
- package/dist/context/ingest/local-ingest.d.ts +1 -1
- package/dist/context/ingest/local-ingest.js +6 -4
- package/dist/context/ingest/memory-flow/events.js +2 -1
- package/dist/context/ingest/ports.d.ts +2 -0
- package/dist/context/ingest/reports.d.ts +3 -0
- package/dist/context/ingest/reports.js +10 -0
- package/dist/context/ingest/stages/stage-3-work-units.d.ts +3 -1
- package/dist/context/ingest/stages/stage-3-work-units.js +2 -0
- package/dist/context/ingest/stages/stage-4-reconciliation.d.ts +2 -1
- package/dist/context/ingest/stages/stage-4-reconciliation.js +1 -1
- package/dist/context/ingest/tools/tool-call-logger.d.ts +6 -0
- package/dist/context/ingest/tools/tool-call-logger.js +36 -1
- package/dist/context/llm/ai-sdk-runtime.js +32 -3
- package/dist/context/llm/claude-code-runtime.js +51 -3
- package/dist/context/llm/runtime-port.d.ts +25 -0
- package/dist/context/mcp/context-tools.d.ts +2 -1
- package/dist/context/mcp/context-tools.js +82 -15
- package/dist/context/mcp/server.js +4 -0
- package/dist/context/mcp/types.d.ts +15 -1
- package/dist/context/project/config.d.ts +1 -0
- package/dist/context/project/config.js +4 -0
- package/dist/context/project/driver-schemas.js +1 -1
- package/dist/context/search/discover.js +4 -3
- package/dist/context/sl/local-sl.d.ts +15 -0
- package/dist/context/sl/local-sl.js +30 -0
- package/dist/context/wiki/local-knowledge.d.ts +10 -0
- package/dist/context/wiki/local-knowledge.js +22 -0
- package/dist/context-build-view.d.ts +0 -3
- package/dist/context-build-view.js +1 -7
- package/dist/ingest.js +7 -10
- package/dist/knowledge.d.ts +5 -0
- package/dist/knowledge.js +10 -1
- package/dist/public-ingest-copy.js +1 -1
- package/dist/public-ingest.d.ts +0 -7
- package/dist/public-ingest.js +20 -34
- package/dist/setup-context.js +6 -38
- package/dist/setup-databases.js +13 -82
- package/dist/setup-project.d.ts +0 -8
- package/dist/setup-project.js +3 -27
- package/dist/setup-sources.js +33 -5
- package/dist/setup.js +3 -16
- package/dist/skills/analytics/SKILL.md +6 -1
- package/dist/sl.d.ts +6 -1
- package/dist/sl.js +32 -8
- package/dist/telemetry/emitter.js +1 -1
- package/dist/telemetry/events.d.ts +4 -3
- package/dist/telemetry/events.js +7 -3
- package/dist/telemetry/identity.d.ts +1 -1
- package/dist/telemetry/identity.js +13 -10
- package/dist/telemetry/index.d.ts +1 -1
- package/dist/telemetry/index.js +5 -1
- package/package.json +22 -22
- package/dist/ingest-depth.d.ts +0 -8
- package/dist/ingest-depth.js +0 -56
- package/dist/setup-database-context-depth.d.ts +0 -23
- package/dist/setup-database-context-depth.js +0 -84
|
@@ -30,7 +30,7 @@ const toolDescriptions = {
|
|
|
30
30
|
entity_details: 'Read table and column metadata from the latest live-database scan snapshot. Example: entity_details({ connectionId: "warehouse", entities: [{ table: { catalog: null, db: "public", name: "orders" }, columns: ["id"] }] }).',
|
|
31
31
|
dictionary_search: 'Search profile-sampled warehouse values to locate likely source columns for business values. Example: dictionary_search({ values: ["Acme Corp"], connectionId: "warehouse" }).',
|
|
32
32
|
sl_read_source: 'Read a semantic-layer YAML source by connection id and source name. Example: sl_read_source({ connectionId: "warehouse", sourceName: "orders" }).',
|
|
33
|
-
sl_query: 'Execute a semantic-layer query and return rows,
|
|
33
|
+
sl_query: 'Execute a semantic-layer query and return headers, rows, and total row count, plus correctness notes (e.g. compile-only or fan-out) when relevant. The generated SQL and full query plan are omitted by default; request them with include: ["sql"] and/or include: ["plan"]. Example: sl_query({ connectionId: "warehouse", measures: ["orders.order_count"], dimensions: [{ field: "orders.created_at", granularity: "month" }], include: ["sql"] }).',
|
|
34
34
|
sql_execution: 'Execute one parser-validated read-only SQL query against a configured KTX connection. Example: sql_execution({ connectionId: "warehouse", sql: "select count(*) from public.orders", maxRows: 100 }).',
|
|
35
35
|
memory_ingest: 'Ingest free-form markdown knowledge into durable KTX memory. Use this for business rules, metric definitions, schema gotchas, recurring findings, or explicit user requests to remember something. Example: memory_ingest({ connectionId: "warehouse", content: "ARR is reported in cents in this warehouse." }).',
|
|
36
36
|
memory_ingest_status: 'Read the current or final status for a memory ingest run. Example: memory_ingest_status({ runId: "memory-run-1" }).',
|
|
@@ -38,7 +38,7 @@ const toolDescriptions = {
|
|
|
38
38
|
const connectionListSchema = z.object({});
|
|
39
39
|
const knowledgeSearchSchema = z.object({
|
|
40
40
|
query: z.string().min(1).describe('Natural-language wiki search query, e.g. "revenue recognition policy".'),
|
|
41
|
-
limit: z.number().int().min(1).max(50).default(10).describe('Maximum wiki pages to return.
|
|
41
|
+
limit: z.number().int().min(1).max(50).default(10).describe('Maximum wiki pages to return.'),
|
|
42
42
|
});
|
|
43
43
|
const knowledgeReadSchema = z.object({
|
|
44
44
|
key: z.string().min(1).describe('Wiki page key returned by wiki_search, e.g. "global/revenue".'),
|
|
@@ -67,10 +67,7 @@ const slQueryOrderBySchema = z.object({
|
|
|
67
67
|
.string()
|
|
68
68
|
.min(1)
|
|
69
69
|
.describe('Field/measure/dimension id to order by, e.g. "orders.created_at", a dimension key like "mart_nrr_quarterly.quarter_label", or a measure alias.'),
|
|
70
|
-
direction: z
|
|
71
|
-
.enum(['asc', 'desc'])
|
|
72
|
-
.default('asc')
|
|
73
|
-
.describe('Sort direction: "asc" or "desc". Defaults to "asc".'),
|
|
70
|
+
direction: z.enum(['asc', 'desc']).default('asc').describe('Sort direction for this field.'),
|
|
74
71
|
});
|
|
75
72
|
const slQuerySchema = z.object({
|
|
76
73
|
connectionId: connectionIdSchema
|
|
@@ -93,8 +90,12 @@ const slQuerySchema = z.object({
|
|
|
93
90
|
.array(slQueryOrderBySchema)
|
|
94
91
|
.default([])
|
|
95
92
|
.describe('Sort clauses. Use {field, direction?} entries.'),
|
|
96
|
-
limit: z.number().int().min(0).default(1000).describe('Maximum rows to return.
|
|
97
|
-
include_empty: z.boolean().default(true).describe('Whether to include empty dimension groups.
|
|
93
|
+
limit: z.number().int().min(0).default(1000).describe('Maximum rows to return.'),
|
|
94
|
+
include_empty: z.boolean().default(true).describe('Whether to include empty dimension groups.'),
|
|
95
|
+
include: z
|
|
96
|
+
.array(z.enum(['plan', 'sql']))
|
|
97
|
+
.default([])
|
|
98
|
+
.describe('Extra detail to attach to the response: "sql" for the generated SQL, "plan" for the full query plan.'),
|
|
98
99
|
});
|
|
99
100
|
const entityDetailsTableRefSchema = z.object({
|
|
100
101
|
catalog: z.string().nullable().describe('Catalog/project/database. Use null when not applicable.'),
|
|
@@ -134,12 +135,12 @@ const discoverDataSchema = z.object({
|
|
|
134
135
|
.optional()
|
|
135
136
|
.describe('Optional connection id. Pass it when user intent pins a specific warehouse.'),
|
|
136
137
|
kinds: z.array(discoverDataKindSchema.describe('Reference kind to include.')).optional().describe('Optional kind filter.'),
|
|
137
|
-
limit: z.number().int().min(1).max(50).default(
|
|
138
|
+
limit: z.number().int().min(1).max(50).default(10).optional().describe('Maximum refs to return.'),
|
|
138
139
|
});
|
|
139
140
|
const sqlExecutionSchema = z.object({
|
|
140
141
|
connectionId: connectionIdSchema.describe('Connection id to execute against. Required for raw SQL.'),
|
|
141
142
|
sql: z.string().min(1).describe('Parser-validated read-only SQL, e.g. "select count(*) from public.orders".'),
|
|
142
|
-
maxRows: z.number().int().min(1).max(10_000).default(1000).optional().describe('Maximum rows to return.
|
|
143
|
+
maxRows: z.number().int().min(1).max(10_000).default(1000).optional().describe('Maximum rows to return.'),
|
|
143
144
|
});
|
|
144
145
|
const memoryIngestSchema = z.object({
|
|
145
146
|
content: z
|
|
@@ -198,10 +199,14 @@ const slReadSourceOutputSchema = z.object({
|
|
|
198
199
|
const slQueryOutputSchema = z.object({
|
|
199
200
|
connectionId: z.string().optional(),
|
|
200
201
|
dialect: z.string().optional(),
|
|
201
|
-
sql: z.string(),
|
|
202
202
|
headers: z.array(z.string()),
|
|
203
203
|
rows: z.array(z.array(z.unknown())),
|
|
204
204
|
totalRows: z.number(),
|
|
205
|
+
// Correctness signals hoisted out of `plan` so they survive default projection (e.g. compile-only
|
|
206
|
+
// status, fan-out warnings). Present only when there is something to report.
|
|
207
|
+
notes: z.array(z.string()).optional(),
|
|
208
|
+
// Opt-in detail, attached only when requested via the `include` input.
|
|
209
|
+
sql: z.string().optional(),
|
|
205
210
|
plan: unknownRecordSchema.optional(),
|
|
206
211
|
});
|
|
207
212
|
const entityDetailsSnapshotOutputSchema = z.object({
|
|
@@ -321,11 +326,54 @@ const memoryIngestStatusOutputSchema = z.object({
|
|
|
321
326
|
});
|
|
322
327
|
/** @internal */
|
|
323
328
|
export function jsonToolResult(structuredContent) {
|
|
329
|
+
// Compact (non-indented) JSON: this `content` text is the copy the model reads. Pretty-printing
|
|
330
|
+
// arrays-of-arrays (every `rows` payload) puts one scalar per line, inflating tabular results by
|
|
331
|
+
// a large constant factor. `structuredContent` carries the same data for structured-output clients.
|
|
324
332
|
return {
|
|
325
|
-
content: [{ type: 'text', text: JSON.stringify(structuredContent
|
|
333
|
+
content: [{ type: 'text', text: JSON.stringify(structuredContent) }],
|
|
326
334
|
structuredContent,
|
|
327
335
|
};
|
|
328
336
|
}
|
|
337
|
+
/**
|
|
338
|
+
* Pull the correctness-critical signals out of a query plan so they survive even when the caller
|
|
339
|
+
* did not opt into the full `plan`. Returns an empty list when there is nothing to flag.
|
|
340
|
+
*/
|
|
341
|
+
function slQueryNotes(plan) {
|
|
342
|
+
if (!plan) {
|
|
343
|
+
return [];
|
|
344
|
+
}
|
|
345
|
+
const notes = [];
|
|
346
|
+
const execution = plan.execution;
|
|
347
|
+
if (execution &&
|
|
348
|
+
typeof execution === 'object' &&
|
|
349
|
+
execution.mode === 'compile_only') {
|
|
350
|
+
const reason = execution.reason;
|
|
351
|
+
notes.push(typeof reason === 'string' ? reason : 'Compiled SQL only; no rows were executed.');
|
|
352
|
+
}
|
|
353
|
+
if (plan.has_fan_out === true) {
|
|
354
|
+
const description = typeof plan.fan_out_description === 'string' ? plan.fan_out_description.trim() : '';
|
|
355
|
+
notes.push(description.length > 0 ? description : 'Fan-out detected: measure totals may be inflated by joins.');
|
|
356
|
+
}
|
|
357
|
+
return notes;
|
|
358
|
+
}
|
|
359
|
+
/**
|
|
360
|
+
* Default sl_query response is the minimum the agent needs to read the result: connection, headers,
|
|
361
|
+
* rows, totals, plus any correctness notes. The generated `sql` and the full `plan` are attached only
|
|
362
|
+
* when explicitly requested via `include`, since both are large and echo information the caller already has.
|
|
363
|
+
*/
|
|
364
|
+
function projectSlQueryResult(result, include) {
|
|
365
|
+
const notes = slQueryNotes(result.plan);
|
|
366
|
+
return {
|
|
367
|
+
...(result.connectionId !== undefined ? { connectionId: result.connectionId } : {}),
|
|
368
|
+
...(result.dialect !== undefined ? { dialect: result.dialect } : {}),
|
|
369
|
+
headers: result.headers,
|
|
370
|
+
rows: result.rows,
|
|
371
|
+
totalRows: result.totalRows,
|
|
372
|
+
...(notes.length > 0 ? { notes } : {}),
|
|
373
|
+
...(include.includes('sql') ? { sql: result.sql } : {}),
|
|
374
|
+
...(include.includes('plan') && result.plan ? { plan: result.plan } : {}),
|
|
375
|
+
};
|
|
376
|
+
}
|
|
329
377
|
function jsonErrorToolResult(text) {
|
|
330
378
|
return {
|
|
331
379
|
content: [{ type: 'text', text }],
|
|
@@ -367,6 +415,18 @@ function registerParsedTool(server, name, config, schema, handler) {
|
|
|
367
415
|
}
|
|
368
416
|
});
|
|
369
417
|
}
|
|
418
|
+
/**
|
|
419
|
+
* Resolves the connected client's identity into the raw telemetry fields. The
|
|
420
|
+
* strings are client-controlled and untrusted, so they only ever land in the
|
|
421
|
+
* telemetry property bag — never in paths, logs, or error messages.
|
|
422
|
+
*/
|
|
423
|
+
function clientTelemetryFields(getClientInfo) {
|
|
424
|
+
const client = getClientInfo?.();
|
|
425
|
+
return {
|
|
426
|
+
...(client?.name ? { mcpClientName: client.name } : {}),
|
|
427
|
+
...(client?.version ? { mcpClientVersion: client.version } : {}),
|
|
428
|
+
};
|
|
429
|
+
}
|
|
370
430
|
function instrumentMcpServer(server, telemetry) {
|
|
371
431
|
return {
|
|
372
432
|
registerTool(name, config, handler) {
|
|
@@ -385,6 +445,7 @@ function instrumentMcpServer(server, telemetry) {
|
|
|
385
445
|
outcome: isError ? 'error' : 'ok',
|
|
386
446
|
durationMs: Math.max(0, performance.now() - startedAt),
|
|
387
447
|
sampleRate: mcpTelemetrySampleRate(),
|
|
448
|
+
...clientTelemetryFields(telemetry.getClientInfo),
|
|
388
449
|
},
|
|
389
450
|
});
|
|
390
451
|
}
|
|
@@ -403,6 +464,7 @@ function instrumentMcpServer(server, telemetry) {
|
|
|
403
464
|
...(errorClass ? { errorClass } : {}),
|
|
404
465
|
durationMs: Math.max(0, performance.now() - startedAt),
|
|
405
466
|
sampleRate: mcpTelemetrySampleRate(),
|
|
467
|
+
...clientTelemetryFields(telemetry.getClientInfo),
|
|
406
468
|
},
|
|
407
469
|
});
|
|
408
470
|
}
|
|
@@ -414,7 +476,11 @@ function instrumentMcpServer(server, telemetry) {
|
|
|
414
476
|
}
|
|
415
477
|
export function registerKtxContextTools(deps) {
|
|
416
478
|
const { ports, userContext } = deps;
|
|
417
|
-
const server = instrumentMcpServer(deps.server, {
|
|
479
|
+
const server = instrumentMcpServer(deps.server, {
|
|
480
|
+
projectDir: deps.projectDir,
|
|
481
|
+
io: deps.io,
|
|
482
|
+
getClientInfo: deps.getClientInfo,
|
|
483
|
+
});
|
|
418
484
|
if (ports.connections) {
|
|
419
485
|
const connections = ports.connections;
|
|
420
486
|
registerParsedTool(server, 'connection_list', {
|
|
@@ -471,7 +537,7 @@ export function registerKtxContextTools(deps) {
|
|
|
471
537
|
annotations: toolAnnotations.sl_query,
|
|
472
538
|
}, slQuerySchema, async (input, context) => {
|
|
473
539
|
const onProgress = mcpProgressCallback(context);
|
|
474
|
-
|
|
540
|
+
const result = await semanticLayer.query({
|
|
475
541
|
connectionId: input.connectionId,
|
|
476
542
|
query: {
|
|
477
543
|
measures: input.measures,
|
|
@@ -482,7 +548,8 @@ export function registerKtxContextTools(deps) {
|
|
|
482
548
|
limit: input.limit,
|
|
483
549
|
include_empty: input.include_empty,
|
|
484
550
|
},
|
|
485
|
-
}, onProgress ? { onProgress } : undefined)
|
|
551
|
+
}, onProgress ? { onProgress } : undefined);
|
|
552
|
+
return jsonToolResult(projectSlQueryResult(result, input.include));
|
|
486
553
|
});
|
|
487
554
|
}
|
|
488
555
|
if (ports.entityDetails) {
|
|
@@ -9,6 +9,7 @@ export function createKtxMcpServer(deps) {
|
|
|
9
9
|
userContext: deps.userContext,
|
|
10
10
|
projectDir: deps.projectDir,
|
|
11
11
|
io: deps.io,
|
|
12
|
+
getClientInfo: deps.getClientInfo,
|
|
12
13
|
});
|
|
13
14
|
}
|
|
14
15
|
return deps.server;
|
|
@@ -24,6 +25,9 @@ export function createDefaultKtxMcpServer(deps) {
|
|
|
24
25
|
contextTools: deps.contextTools,
|
|
25
26
|
projectDir: deps.projectDir,
|
|
26
27
|
io: deps.io,
|
|
28
|
+
// The SDK populates the client identity after the initialize handshake, so
|
|
29
|
+
// read it lazily at emit time rather than at registration (undefined here).
|
|
30
|
+
getClientInfo: () => server.server.getClientVersion(),
|
|
27
31
|
});
|
|
28
32
|
return server;
|
|
29
33
|
}
|
|
@@ -46,6 +46,15 @@ export interface MemoryIngestPort {
|
|
|
46
46
|
export interface KtxMcpUserContext {
|
|
47
47
|
userId: string;
|
|
48
48
|
}
|
|
49
|
+
/**
|
|
50
|
+
* Identity of the connected MCP client tool (e.g. Claude Desktop, Cursor),
|
|
51
|
+
* read from the initialize handshake. Untrusted, client-controlled strings —
|
|
52
|
+
* use only as telemetry properties, never to build paths or log lines.
|
|
53
|
+
*/
|
|
54
|
+
export interface KtxMcpClientInfo {
|
|
55
|
+
name: string;
|
|
56
|
+
version: string;
|
|
57
|
+
}
|
|
49
58
|
export interface KtxMcpServerLike {
|
|
50
59
|
registerTool(name: string, config: {
|
|
51
60
|
title?: string;
|
|
@@ -101,7 +110,10 @@ interface KtxSemanticLayerReadResponse {
|
|
|
101
110
|
sourceName: string;
|
|
102
111
|
yaml: string;
|
|
103
112
|
}
|
|
104
|
-
|
|
113
|
+
/** @internal */
|
|
114
|
+
export interface KtxSemanticLayerQueryResponse {
|
|
115
|
+
connectionId?: string;
|
|
116
|
+
dialect?: string;
|
|
105
117
|
sql: string;
|
|
106
118
|
headers: string[];
|
|
107
119
|
rows: unknown[][];
|
|
@@ -165,5 +177,7 @@ export interface KtxMcpServerDeps {
|
|
|
165
177
|
contextTools?: KtxMcpContextPorts;
|
|
166
178
|
projectDir?: string;
|
|
167
179
|
io?: KtxCliIo;
|
|
180
|
+
/** Reads the connected client's identity once the initialize handshake completes. */
|
|
181
|
+
getClientInfo?: () => KtxMcpClientInfo | undefined;
|
|
168
182
|
}
|
|
169
183
|
export {};
|
|
@@ -395,6 +395,7 @@ declare const ktxProjectConfigSchema: z.ZodObject<{
|
|
|
395
395
|
continue: "continue";
|
|
396
396
|
}>>;
|
|
397
397
|
}, z.core.$strict>>;
|
|
398
|
+
profile: z.ZodDefault<z.ZodUnion<readonly [z.ZodBoolean, z.ZodLiteral<"json">]>>;
|
|
398
399
|
}, z.core.$strict>>;
|
|
399
400
|
agent: z.ZodPrefault<z.ZodObject<{
|
|
400
401
|
run_research: z.ZodPrefault<z.ZodObject<{
|
|
@@ -96,6 +96,10 @@ const ingestSchema = z
|
|
|
96
96
|
.prefault({ backend: 'none' })
|
|
97
97
|
.describe('Embedding configuration used when ingest adapters need to embed documents.'),
|
|
98
98
|
workUnits: workUnitsSchema.prefault({}).describe('Concurrency and failure handling for ingest work units.'),
|
|
99
|
+
profile: z
|
|
100
|
+
.union([z.boolean(), z.literal('json')])
|
|
101
|
+
.default(false)
|
|
102
|
+
.describe('Print a timing breakdown to stderr at the end of each ingest run. `true` prints a human table; `"json"` prints the raw structured profile for coding agents; `false` disables it. Equivalent to the KTX_PROFILE_INGEST environment variable (`1`/`true`/`json`).'),
|
|
99
103
|
})
|
|
100
104
|
.describe('Ingest pipeline configuration: adapters, embeddings, and work-unit policy.');
|
|
101
105
|
const scanEnrichmentSchema = z
|
|
@@ -21,7 +21,7 @@ function warehouseConnectionSchema(driver) {
|
|
|
21
21
|
enabled_tables: z
|
|
22
22
|
.array(z.string().min(1))
|
|
23
23
|
.optional()
|
|
24
|
-
.describe('Optional allowlist of fully-qualified table names ("schema.table") to ingest. When set, live-database ingest discards any table whose schema-qualified name is not in this list. Useful for smoke-testing
|
|
24
|
+
.describe('Optional allowlist of fully-qualified table names ("schema.table") to ingest. When set, live-database ingest discards any table whose schema-qualified name is not in this list. Useful for smoke-testing ingest on a single table.'),
|
|
25
25
|
})
|
|
26
26
|
.describe(`${driver} warehouse connection. Additional driver-tunable fields (e.g. context.queryHistory) are accepted and passed through.`);
|
|
27
27
|
}
|
|
@@ -95,7 +95,7 @@ async function wikiCandidates(project, input, options, terms) {
|
|
|
95
95
|
query: input.query,
|
|
96
96
|
userId: options.userId,
|
|
97
97
|
embeddingService: options.embeddingService ?? null,
|
|
98
|
-
limit: Math.max(input.limit ??
|
|
98
|
+
limit: Math.max(input.limit ?? 10, 25),
|
|
99
99
|
});
|
|
100
100
|
const records = [];
|
|
101
101
|
for (const result of searchResults) {
|
|
@@ -300,7 +300,8 @@ function hydrate(fused, refsByKey) {
|
|
|
300
300
|
}
|
|
301
301
|
return {
|
|
302
302
|
...ref,
|
|
303
|
-
|
|
303
|
+
// 3 decimals is plenty for a relative-rank hint; 6 just spent bytes on noise.
|
|
304
|
+
score: maxScore > 0 ? Number((candidate.score / maxScore).toFixed(3)) : 0,
|
|
304
305
|
};
|
|
305
306
|
})
|
|
306
307
|
.filter((result) => result !== null);
|
|
@@ -308,7 +309,7 @@ function hydrate(fused, refsByKey) {
|
|
|
308
309
|
export function createKtxDiscoverDataService(project, options = {}) {
|
|
309
310
|
return {
|
|
310
311
|
async search(input) {
|
|
311
|
-
const limit = Math.max(1, Math.min(input.limit ??
|
|
312
|
+
const limit = Math.max(1, Math.min(input.limit ?? 10, 50));
|
|
312
313
|
const query = input.query.trim();
|
|
313
314
|
if (!query) {
|
|
314
315
|
return [];
|
|
@@ -28,6 +28,7 @@ export interface LocalSlSearchInput {
|
|
|
28
28
|
backend?: 'pglite-owner-prototype';
|
|
29
29
|
pglite?: PgliteSlSearchPrototypeOwnerOptions;
|
|
30
30
|
}
|
|
31
|
+
/** @internal */
|
|
31
32
|
export interface LocalSlSource extends LocalSlSourceSummary {
|
|
32
33
|
yaml: string;
|
|
33
34
|
}
|
|
@@ -38,6 +39,15 @@ export interface LocalSlValidationResult {
|
|
|
38
39
|
valid: boolean;
|
|
39
40
|
errors: string[];
|
|
40
41
|
}
|
|
42
|
+
export type ResolvedSlSource = {
|
|
43
|
+
kind: 'found';
|
|
44
|
+
source: LocalSlSource;
|
|
45
|
+
} | {
|
|
46
|
+
kind: 'not-found';
|
|
47
|
+
} | {
|
|
48
|
+
kind: 'ambiguous';
|
|
49
|
+
connectionIds: string[];
|
|
50
|
+
};
|
|
41
51
|
export declare function loadLocalSlSourceRecords(project: KtxLocalProject, input: {
|
|
42
52
|
connectionId: string;
|
|
43
53
|
}): Promise<LocalSlSourceRecord[]>;
|
|
@@ -52,10 +62,15 @@ export declare function writeLocalSlSource(project: KtxLocalProject, input: {
|
|
|
52
62
|
sourceName: string;
|
|
53
63
|
yaml: string;
|
|
54
64
|
}): Promise<KtxFileWriteResult>;
|
|
65
|
+
/** @internal */
|
|
55
66
|
export declare function readLocalSlSource(project: KtxLocalProject, input: {
|
|
56
67
|
connectionId: string;
|
|
57
68
|
sourceName: string;
|
|
58
69
|
}): Promise<LocalSlSource | null>;
|
|
70
|
+
export declare function resolveLocalSlSource(project: KtxLocalProject, input: {
|
|
71
|
+
sourceName: string;
|
|
72
|
+
connectionId?: string;
|
|
73
|
+
}): Promise<ResolvedSlSource>;
|
|
59
74
|
export declare function listLocalSlSources(project: KtxLocalProject, input?: {
|
|
60
75
|
connectionId?: string;
|
|
61
76
|
}): Promise<LocalSlSourceSummary[]>;
|
|
@@ -204,6 +204,7 @@ export async function writeLocalSlSource(project, input) {
|
|
|
204
204
|
const path = slPath(input.connectionId, input.sourceName);
|
|
205
205
|
return project.fileStore.writeFile(path, input.yaml.endsWith('\n') ? input.yaml : `${input.yaml}\n`, LOCAL_AUTHOR, LOCAL_AUTHOR_EMAIL, `Write semantic-layer source: ${input.connectionId}/${input.sourceName}`);
|
|
206
206
|
}
|
|
207
|
+
/** @internal */
|
|
207
208
|
export async function readLocalSlSource(project, input) {
|
|
208
209
|
const path = slPath(input.connectionId, input.sourceName);
|
|
209
210
|
try {
|
|
@@ -221,6 +222,35 @@ export async function readLocalSlSource(project, input) {
|
|
|
221
222
|
return record ? { ...record } : null;
|
|
222
223
|
}
|
|
223
224
|
}
|
|
225
|
+
export async function resolveLocalSlSource(project, input) {
|
|
226
|
+
if (input.connectionId !== undefined) {
|
|
227
|
+
const source = await readLocalSlSource(project, {
|
|
228
|
+
connectionId: input.connectionId,
|
|
229
|
+
sourceName: input.sourceName,
|
|
230
|
+
});
|
|
231
|
+
return source ? { kind: 'found', source } : { kind: 'not-found' };
|
|
232
|
+
}
|
|
233
|
+
const summaries = await listLocalSlSources(project, {});
|
|
234
|
+
const matches = summaries.filter((summary) => summary.name === input.sourceName);
|
|
235
|
+
if (matches.length === 0) {
|
|
236
|
+
return { kind: 'not-found' };
|
|
237
|
+
}
|
|
238
|
+
if (matches.length > 1) {
|
|
239
|
+
return {
|
|
240
|
+
kind: 'ambiguous',
|
|
241
|
+
connectionIds: [...new Set(matches.map((match) => match.connectionId))].sort(),
|
|
242
|
+
};
|
|
243
|
+
}
|
|
244
|
+
const match = matches[0];
|
|
245
|
+
if (match === undefined) {
|
|
246
|
+
return { kind: 'not-found' };
|
|
247
|
+
}
|
|
248
|
+
const source = await readLocalSlSource(project, {
|
|
249
|
+
connectionId: match.connectionId,
|
|
250
|
+
sourceName: input.sourceName,
|
|
251
|
+
});
|
|
252
|
+
return source ? { kind: 'found', source } : { kind: 'not-found' };
|
|
253
|
+
}
|
|
224
254
|
export async function listLocalSlSources(project, input = {}) {
|
|
225
255
|
if (input.connectionId) {
|
|
226
256
|
return (await loadLocalSlSourceRecords(project, { connectionId: input.connectionId })).map(({ source: _source, yaml: _yaml, ...summary }) => summary);
|
|
@@ -50,6 +50,16 @@ export declare function readLocalKnowledgePage(project: KtxLocalProject, input:
|
|
|
50
50
|
export declare function listLocalKnowledgePages(project: KtxLocalProject, input?: {
|
|
51
51
|
userId?: string;
|
|
52
52
|
}): Promise<LocalKnowledgeSummary[]>;
|
|
53
|
+
/**
|
|
54
|
+
* List wiki page keys without reading or parsing file contents.
|
|
55
|
+
*
|
|
56
|
+
* Keys are derived purely from file paths, so this stays cheap enough for
|
|
57
|
+
* shell tab-completion (unlike `listLocalKnowledgePages`, which reads every
|
|
58
|
+
* page to populate summaries).
|
|
59
|
+
*/
|
|
60
|
+
export declare function listLocalKnowledgePageKeys(project: KtxLocalProject, input?: {
|
|
61
|
+
userId?: string;
|
|
62
|
+
}): Promise<string[]>;
|
|
53
63
|
export declare function searchLocalKnowledgePages(project: KtxLocalProject, input: {
|
|
54
64
|
query: string;
|
|
55
65
|
userId?: string;
|
|
@@ -118,6 +118,28 @@ export async function listLocalKnowledgePages(project, input = {}) {
|
|
|
118
118
|
}
|
|
119
119
|
return pages.sort((left, right) => left.path.localeCompare(right.path));
|
|
120
120
|
}
|
|
121
|
+
/**
|
|
122
|
+
* List wiki page keys without reading or parsing file contents.
|
|
123
|
+
*
|
|
124
|
+
* Keys are derived purely from file paths, so this stays cheap enough for
|
|
125
|
+
* shell tab-completion (unlike `listLocalKnowledgePages`, which reads every
|
|
126
|
+
* page to populate summaries).
|
|
127
|
+
*/
|
|
128
|
+
export async function listLocalKnowledgePageKeys(project, input = {}) {
|
|
129
|
+
const userId = input.userId ?? 'local';
|
|
130
|
+
const keys = new Set();
|
|
131
|
+
for (const scope of ['GLOBAL', 'USER']) {
|
|
132
|
+
const root = scope === 'GLOBAL' ? 'wiki/global' : `wiki/user/${assertSafePathToken('user id', userId)}`;
|
|
133
|
+
const listed = await project.fileStore.listFiles(root);
|
|
134
|
+
for (const path of listed.files.filter((file) => file.endsWith('.md'))) {
|
|
135
|
+
const key = keyFromKnowledgePath(path, scope, userId);
|
|
136
|
+
if (key) {
|
|
137
|
+
keys.add(key);
|
|
138
|
+
}
|
|
139
|
+
}
|
|
140
|
+
}
|
|
141
|
+
return [...keys].sort();
|
|
142
|
+
}
|
|
121
143
|
function scorePage(page, terms) {
|
|
122
144
|
const haystack = buildKnowledgeSearchText(page.key, page.summary, page.content, page.tags).toLowerCase();
|
|
123
145
|
return terms.some((term) => haystack.includes(term)) ? 3 : 0;
|
|
@@ -39,9 +39,6 @@ export interface ContextBuildArgs {
|
|
|
39
39
|
targetConnectionId?: string;
|
|
40
40
|
all?: boolean;
|
|
41
41
|
entrypoint?: 'setup' | 'ingest';
|
|
42
|
-
depth?: Extract<KtxPublicIngestArgs, {
|
|
43
|
-
command: 'run';
|
|
44
|
-
}>['depth'];
|
|
45
42
|
queryHistory?: Extract<KtxPublicIngestArgs, {
|
|
46
43
|
command: 'run';
|
|
47
44
|
}>['queryHistory'];
|
|
@@ -241,12 +241,11 @@ function renderMessageGroup(label, messages, styled) {
|
|
|
241
241
|
function retryCommand(input) {
|
|
242
242
|
const projectPart = input.projectDir ? ` --project-dir ${input.projectDir}` : '';
|
|
243
243
|
if (input.entrypoint === 'ingest' && input.connectionId) {
|
|
244
|
-
const depthPart = input.depth ? ` --${input.depth}` : '';
|
|
245
244
|
const queryHistoryPart = input.queryHistory ? ' --query-history' : '';
|
|
246
245
|
const windowPart = input.queryHistory && input.queryHistoryWindowDays !== undefined
|
|
247
246
|
? ` --query-history-window-days ${input.queryHistoryWindowDays}`
|
|
248
247
|
: '';
|
|
249
|
-
return `ktx ingest ${input.connectionId}${projectPart}${
|
|
248
|
+
return `ktx ingest ${input.connectionId}${projectPart}${queryHistoryPart}${windowPart}`;
|
|
250
249
|
}
|
|
251
250
|
return input.projectDir ? `ktx setup --project-dir ${input.projectDir}` : 'ktx setup';
|
|
252
251
|
}
|
|
@@ -563,7 +562,6 @@ function appendRetryIfNeeded(input) {
|
|
|
563
562
|
projectDir: input.projectDir,
|
|
564
563
|
entrypoint: input.entrypoint,
|
|
565
564
|
connectionId: input.target.connectionId,
|
|
566
|
-
depth: input.target.databaseDepth,
|
|
567
565
|
queryHistory: input.target.queryHistory?.enabled === true,
|
|
568
566
|
queryHistoryWindowDays: input.target.queryHistory?.windowDays,
|
|
569
567
|
})}`;
|
|
@@ -578,7 +576,6 @@ function failureTextForTarget(input) {
|
|
|
578
576
|
projectDir: input.projectDir,
|
|
579
577
|
entrypoint: input.entrypoint,
|
|
580
578
|
connectionId: input.target.connectionId,
|
|
581
|
-
depth: input.target.databaseDepth,
|
|
582
579
|
queryHistory: input.target.queryHistory?.enabled === true,
|
|
583
580
|
queryHistoryWindowDays: input.target.queryHistory?.windowDays,
|
|
584
581
|
})}`,
|
|
@@ -593,7 +590,6 @@ function failureTextForTarget(input) {
|
|
|
593
590
|
projectDir: input.projectDir,
|
|
594
591
|
entrypoint: input.entrypoint,
|
|
595
592
|
connectionId: input.target.connectionId,
|
|
596
|
-
depth: input.target.databaseDepth,
|
|
597
593
|
queryHistory: input.target.queryHistory?.enabled === true,
|
|
598
594
|
queryHistoryWindowDays: input.target.queryHistory?.windowDays,
|
|
599
595
|
})}`,
|
|
@@ -659,7 +655,6 @@ export async function runContextBuild(project, args, io, deps = {}) {
|
|
|
659
655
|
projectDir: args.projectDir,
|
|
660
656
|
...(args.targetConnectionId ? { targetConnectionId: args.targetConnectionId } : {}),
|
|
661
657
|
all: args.all ?? true,
|
|
662
|
-
...(args.depth ? { depth: args.depth } : {}),
|
|
663
658
|
...(args.queryHistory ? { queryHistory: args.queryHistory } : {}),
|
|
664
659
|
...(args.queryHistoryWindowDays !== undefined ? { queryHistoryWindowDays: args.queryHistoryWindowDays } : {}),
|
|
665
660
|
...(args.scanMode ? { scanMode: args.scanMode } : {}),
|
|
@@ -721,7 +716,6 @@ export async function runContextBuild(project, args, io, deps = {}) {
|
|
|
721
716
|
all: args.all ?? true,
|
|
722
717
|
json: false,
|
|
723
718
|
inputMode: args.inputMode,
|
|
724
|
-
...(args.depth ? { depth: args.depth } : {}),
|
|
725
719
|
...(args.queryHistory ? { queryHistory: args.queryHistory } : {}),
|
|
726
720
|
...(args.queryHistoryWindowDays !== undefined ? { queryHistoryWindowDays: args.queryHistoryWindowDays } : {}),
|
|
727
721
|
...(args.scanMode ? { scanMode: args.scanMode } : {}),
|
package/dist/ingest.js
CHANGED
|
@@ -2,7 +2,7 @@ import { buildMemoryFlowViewModel } from './context/ingest/memory-flow/view-mode
|
|
|
2
2
|
import { createMemoryFlowLiveBuffer, sanitizeMemoryFlowError } from './context/ingest/memory-flow/live-buffer.js';
|
|
3
3
|
import { formatMemoryFlowFinalSummary } from './context/ingest/memory-flow/summary.js';
|
|
4
4
|
import { getLatestLocalIngestStatus, getLocalIngestStatus, runLocalIngest, runLocalMetabaseIngest } from './context/ingest/local-ingest.js';
|
|
5
|
-
import { savedMemoryCountsForReport } from './context/ingest/reports.js';
|
|
5
|
+
import { ingestReportOutcome, savedMemoryCountsForReport } from './context/ingest/reports.js';
|
|
6
6
|
import { ingestReportToMemoryFlowReplay } from './context/ingest/memory-flow/events.js';
|
|
7
7
|
import { renderMemoryFlowReplay } from './context/ingest/memory-flow/render.js';
|
|
8
8
|
import { loadKtxProject } from './context/project/project.js';
|
|
@@ -17,9 +17,6 @@ import { renderMemoryFlowTui, startLiveMemoryFlowTui, } from './memory-flow-tui.
|
|
|
17
17
|
import { resolveVizFallback, warnVizFallbackOnce } from './viz-fallback.js';
|
|
18
18
|
import { profileMark } from './startup-profile.js';
|
|
19
19
|
profileMark('module:ingest');
|
|
20
|
-
function reportStatus(report) {
|
|
21
|
-
return report.body.status === 'failed' || report.body.failedWorkUnits.length > 0 ? 'error' : 'done';
|
|
22
|
-
}
|
|
23
20
|
const REPORT_SOURCE_LABELS = new Map([
|
|
24
21
|
['live-database', 'Database schema'],
|
|
25
22
|
['historic-sql', 'Query history'],
|
|
@@ -106,7 +103,7 @@ function writeReportStatus(report, io) {
|
|
|
106
103
|
if (report.body.tracePath) {
|
|
107
104
|
io.stdout.write(`Trace: ${report.body.tracePath}\n`);
|
|
108
105
|
}
|
|
109
|
-
io.stdout.write(`Status: ${
|
|
106
|
+
io.stdout.write(`Status: ${ingestReportOutcome(report)}\n`);
|
|
110
107
|
io.stdout.write(`Source: ${reportSourceLabel(report.sourceKey)}\n`);
|
|
111
108
|
io.stdout.write(`Connection: ${report.connectionId}\n`);
|
|
112
109
|
io.stdout.write(`Sync: ${report.body.syncId}\n`);
|
|
@@ -138,7 +135,7 @@ function writeMetabaseFanoutStatus(result, io) {
|
|
|
138
135
|
}
|
|
139
136
|
io.stdout.write(`Saved memory: ${counts.wikiCount} wiki, ${counts.slCount} SL\n`);
|
|
140
137
|
for (const child of result.children) {
|
|
141
|
-
const status =
|
|
138
|
+
const status = ingestReportOutcome(child.report);
|
|
142
139
|
io.stdout.write(`- target=${child.targetConnectionId} database=${child.metabaseDatabaseId} status=${status} job=${child.jobId} report=${child.report.id}\n`);
|
|
143
140
|
}
|
|
144
141
|
}
|
|
@@ -425,7 +422,7 @@ function initialRunMemoryFlowInput(args, runId) {
|
|
|
425
422
|
};
|
|
426
423
|
}
|
|
427
424
|
function finalRunMemoryFlowInput(snapshot, report) {
|
|
428
|
-
const status =
|
|
425
|
+
const status = ingestReportOutcome(report) === 'error' ? 'error' : 'done';
|
|
429
426
|
return {
|
|
430
427
|
...snapshot,
|
|
431
428
|
runId: report.runId,
|
|
@@ -574,7 +571,7 @@ export async function runKtxIngest(args, io = process, deps = {}) {
|
|
|
574
571
|
finally {
|
|
575
572
|
plainProgress?.flush();
|
|
576
573
|
}
|
|
577
|
-
return result.status === '
|
|
574
|
+
return result.status === 'all_failed' ? 1 : 0;
|
|
578
575
|
}
|
|
579
576
|
const jobId = deps.jobIdFactory?.();
|
|
580
577
|
let liveTui = null;
|
|
@@ -636,7 +633,7 @@ export async function runKtxIngest(args, io = process, deps = {}) {
|
|
|
636
633
|
liveTui?.close();
|
|
637
634
|
liveTui = null;
|
|
638
635
|
io.stdout.write(formatMemoryFlowFinalSummary(latestMemoryFlowSnapshot));
|
|
639
|
-
return
|
|
636
|
+
return ingestReportOutcome(result.report) === 'error' ? 1 : 0;
|
|
640
637
|
}
|
|
641
638
|
plainProgress?.flush();
|
|
642
639
|
await writeReportRecord(result.report, runOutputMode, io, {
|
|
@@ -644,7 +641,7 @@ export async function runKtxIngest(args, io = process, deps = {}) {
|
|
|
644
641
|
renderStoredMemoryFlow: deps.renderStoredMemoryFlow,
|
|
645
642
|
env,
|
|
646
643
|
});
|
|
647
|
-
return
|
|
644
|
+
return ingestReportOutcome(result.report) === 'error' ? 1 : 0;
|
|
648
645
|
}
|
|
649
646
|
finally {
|
|
650
647
|
plainProgress?.flush();
|
package/dist/knowledge.d.ts
CHANGED
|
@@ -18,6 +18,11 @@ export type KtxKnowledgeArgs = {
|
|
|
18
18
|
limit?: number;
|
|
19
19
|
debug?: boolean;
|
|
20
20
|
cliVersion: string;
|
|
21
|
+
} | {
|
|
22
|
+
command: 'read';
|
|
23
|
+
projectDir: string;
|
|
24
|
+
key: string;
|
|
25
|
+
userId: string;
|
|
21
26
|
};
|
|
22
27
|
type KtxKnowledgeIo = import('./cli-runtime.js').KtxCliIo;
|
|
23
28
|
interface KtxKnowledgeDeps {
|
package/dist/knowledge.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { KtxIngestEmbeddingPortAdapter } from './context/llm/embedding-port.js';
|
|
2
2
|
import { loadKtxProject } from './context/project/project.js';
|
|
3
|
-
import { listLocalKnowledgePages, searchLocalKnowledgePages as defaultSearchLocalKnowledgePages } from './context/wiki/local-knowledge.js';
|
|
3
|
+
import { listLocalKnowledgePages, readLocalKnowledgePage, searchLocalKnowledgePages as defaultSearchLocalKnowledgePages, } from './context/wiki/local-knowledge.js';
|
|
4
4
|
import { resolveProjectEmbeddingProvider, } from './embedding-resolution.js';
|
|
5
5
|
import { resolveOutputMode } from './io/mode.js';
|
|
6
6
|
import { createRankBadgeFormatter, printList } from './io/print-list.js';
|
|
@@ -72,6 +72,15 @@ export async function runKtxKnowledge(args, io = process, deps = {}) {
|
|
|
72
72
|
});
|
|
73
73
|
return 0;
|
|
74
74
|
}
|
|
75
|
+
if (args.command === 'read') {
|
|
76
|
+
const page = await readLocalKnowledgePage(project, { key: args.key, userId: args.userId });
|
|
77
|
+
if (!page) {
|
|
78
|
+
throw new Error(`No wiki page found for key '${args.key}'`);
|
|
79
|
+
}
|
|
80
|
+
const raw = await project.fileStore.readFile(page.path);
|
|
81
|
+
io.stdout.write(raw.content);
|
|
82
|
+
return 0;
|
|
83
|
+
}
|
|
75
84
|
if (args.command === 'search') {
|
|
76
85
|
const embeddingService = await wikiSearchEmbeddingService(project, deps, { cliVersion: args.cliVersion }, io);
|
|
77
86
|
const search = deps.searchLocalKnowledgePages ?? defaultSearchLocalKnowledgePages;
|
|
@@ -11,7 +11,7 @@ const DATABASE_INGEST_REPLACEMENTS = [
|
|
|
11
11
|
'Database enrichment failed after schema context completed',
|
|
12
12
|
],
|
|
13
13
|
[/\bstructural scan\b/gi, 'schema context'],
|
|
14
|
-
[/\benriched scan\b/gi, '
|
|
14
|
+
[/\benriched scan\b/gi, 'database ingest'],
|
|
15
15
|
[/\bscan results\b/gi, 'database context'],
|
|
16
16
|
];
|
|
17
17
|
export function publicDatabaseIngestMessage(message) {
|