@kaelio/ktx 0.7.0 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/assets/python/{kaelio_ktx-0.7.0-py3-none-any.whl → kaelio_ktx-0.8.0-py3-none-any.whl} +0 -0
- package/assets/python/manifest.json +4 -4
- package/dist/.tsbuildinfo +1 -1
- package/dist/cli-program.js +7 -0
- package/dist/command-schemas.d.ts +1 -1
- package/dist/command-tree.js +5 -1
- package/dist/commands/completion-commands.d.ts +3 -0
- package/dist/commands/completion-commands.js +38 -0
- package/dist/commands/ingest-commands.js +0 -4
- package/dist/commands/knowledge-commands.js +15 -2
- package/dist/commands/setup-commands.js +2 -2
- package/dist/commands/sl-commands.js +19 -7
- package/dist/completion/complete-engine.d.ts +19 -0
- package/dist/completion/complete-engine.js +128 -0
- package/dist/completion/completion-scripts.d.ts +1 -0
- package/dist/completion/completion-scripts.js +36 -0
- package/dist/completion/dynamic-candidates.d.ts +6 -0
- package/dist/completion/dynamic-candidates.js +98 -0
- package/dist/connection-drivers.d.ts +3 -0
- package/dist/connection-drivers.js +17 -0
- package/dist/context/ingest/ingest-bundle.runner.d.ts +8 -0
- package/dist/context/ingest/ingest-bundle.runner.js +72 -15
- package/dist/context/ingest/ingest-profile.d.ts +102 -0
- package/dist/context/ingest/ingest-profile.js +306 -0
- package/dist/context/ingest/isolated-diff/work-unit-executor.js +25 -2
- package/dist/context/ingest/local-bundle-runtime.js +1 -0
- package/dist/context/ingest/local-ingest.d.ts +1 -1
- package/dist/context/ingest/local-ingest.js +6 -4
- package/dist/context/ingest/memory-flow/events.js +2 -1
- package/dist/context/ingest/ports.d.ts +2 -0
- package/dist/context/ingest/reports.d.ts +3 -0
- package/dist/context/ingest/reports.js +10 -0
- package/dist/context/ingest/stages/stage-3-work-units.d.ts +3 -1
- package/dist/context/ingest/stages/stage-3-work-units.js +2 -0
- package/dist/context/ingest/stages/stage-4-reconciliation.d.ts +2 -1
- package/dist/context/ingest/stages/stage-4-reconciliation.js +1 -1
- package/dist/context/ingest/tools/tool-call-logger.d.ts +6 -0
- package/dist/context/ingest/tools/tool-call-logger.js +36 -1
- package/dist/context/llm/ai-sdk-runtime.js +32 -3
- package/dist/context/llm/claude-code-runtime.js +35 -2
- package/dist/context/llm/runtime-port.d.ts +25 -0
- package/dist/context/mcp/context-tools.d.ts +2 -1
- package/dist/context/mcp/context-tools.js +82 -15
- package/dist/context/mcp/server.js +4 -0
- package/dist/context/mcp/types.d.ts +15 -1
- package/dist/context/project/config.d.ts +1 -0
- package/dist/context/project/config.js +4 -0
- package/dist/context/project/driver-schemas.js +1 -1
- package/dist/context/search/discover.js +4 -3
- package/dist/context/sl/local-sl.d.ts +15 -0
- package/dist/context/sl/local-sl.js +30 -0
- package/dist/context/wiki/local-knowledge.d.ts +10 -0
- package/dist/context/wiki/local-knowledge.js +22 -0
- package/dist/context-build-view.d.ts +0 -3
- package/dist/context-build-view.js +1 -7
- package/dist/ingest.js +7 -10
- package/dist/knowledge.d.ts +5 -0
- package/dist/knowledge.js +10 -1
- package/dist/public-ingest-copy.js +1 -1
- package/dist/public-ingest.d.ts +0 -7
- package/dist/public-ingest.js +20 -34
- package/dist/setup-context.js +6 -38
- package/dist/setup-databases.js +13 -82
- package/dist/setup-sources.js +33 -5
- package/dist/setup.js +2 -2
- package/dist/skills/analytics/SKILL.md +6 -1
- package/dist/sl.d.ts +6 -1
- package/dist/sl.js +32 -8
- package/dist/telemetry/emitter.js +1 -1
- package/dist/telemetry/events.d.ts +4 -3
- package/dist/telemetry/events.js +7 -3
- package/dist/telemetry/identity.d.ts +1 -1
- package/dist/telemetry/identity.js +13 -10
- package/dist/telemetry/index.d.ts +1 -1
- package/dist/telemetry/index.js +5 -1
- package/package.json +22 -22
- package/dist/ingest-depth.d.ts +0 -8
- package/dist/ingest-depth.js +0 -56
- package/dist/setup-database-context-depth.d.ts +0 -23
- package/dist/setup-database-context-depth.js +0 -84
|
@@ -9,6 +9,7 @@ export function createKtxMcpServer(deps) {
|
|
|
9
9
|
userContext: deps.userContext,
|
|
10
10
|
projectDir: deps.projectDir,
|
|
11
11
|
io: deps.io,
|
|
12
|
+
getClientInfo: deps.getClientInfo,
|
|
12
13
|
});
|
|
13
14
|
}
|
|
14
15
|
return deps.server;
|
|
@@ -24,6 +25,9 @@ export function createDefaultKtxMcpServer(deps) {
|
|
|
24
25
|
contextTools: deps.contextTools,
|
|
25
26
|
projectDir: deps.projectDir,
|
|
26
27
|
io: deps.io,
|
|
28
|
+
// The SDK populates the client identity after the initialize handshake, so
|
|
29
|
+
// read it lazily at emit time rather than at registration (undefined here).
|
|
30
|
+
getClientInfo: () => server.server.getClientVersion(),
|
|
27
31
|
});
|
|
28
32
|
return server;
|
|
29
33
|
}
|
|
@@ -46,6 +46,15 @@ export interface MemoryIngestPort {
|
|
|
46
46
|
export interface KtxMcpUserContext {
|
|
47
47
|
userId: string;
|
|
48
48
|
}
|
|
49
|
+
/**
|
|
50
|
+
* Identity of the connected MCP client tool (e.g. Claude Desktop, Cursor),
|
|
51
|
+
* read from the initialize handshake. Untrusted, client-controlled strings —
|
|
52
|
+
* use only as telemetry properties, never to build paths or log lines.
|
|
53
|
+
*/
|
|
54
|
+
export interface KtxMcpClientInfo {
|
|
55
|
+
name: string;
|
|
56
|
+
version: string;
|
|
57
|
+
}
|
|
49
58
|
export interface KtxMcpServerLike {
|
|
50
59
|
registerTool(name: string, config: {
|
|
51
60
|
title?: string;
|
|
@@ -101,7 +110,10 @@ interface KtxSemanticLayerReadResponse {
|
|
|
101
110
|
sourceName: string;
|
|
102
111
|
yaml: string;
|
|
103
112
|
}
|
|
104
|
-
|
|
113
|
+
/** @internal */
|
|
114
|
+
export interface KtxSemanticLayerQueryResponse {
|
|
115
|
+
connectionId?: string;
|
|
116
|
+
dialect?: string;
|
|
105
117
|
sql: string;
|
|
106
118
|
headers: string[];
|
|
107
119
|
rows: unknown[][];
|
|
@@ -165,5 +177,7 @@ export interface KtxMcpServerDeps {
|
|
|
165
177
|
contextTools?: KtxMcpContextPorts;
|
|
166
178
|
projectDir?: string;
|
|
167
179
|
io?: KtxCliIo;
|
|
180
|
+
/** Reads the connected client's identity once the initialize handshake completes. */
|
|
181
|
+
getClientInfo?: () => KtxMcpClientInfo | undefined;
|
|
168
182
|
}
|
|
169
183
|
export {};
|
|
@@ -395,6 +395,7 @@ declare const ktxProjectConfigSchema: z.ZodObject<{
|
|
|
395
395
|
continue: "continue";
|
|
396
396
|
}>>;
|
|
397
397
|
}, z.core.$strict>>;
|
|
398
|
+
profile: z.ZodDefault<z.ZodUnion<readonly [z.ZodBoolean, z.ZodLiteral<"json">]>>;
|
|
398
399
|
}, z.core.$strict>>;
|
|
399
400
|
agent: z.ZodPrefault<z.ZodObject<{
|
|
400
401
|
run_research: z.ZodPrefault<z.ZodObject<{
|
|
@@ -96,6 +96,10 @@ const ingestSchema = z
|
|
|
96
96
|
.prefault({ backend: 'none' })
|
|
97
97
|
.describe('Embedding configuration used when ingest adapters need to embed documents.'),
|
|
98
98
|
workUnits: workUnitsSchema.prefault({}).describe('Concurrency and failure handling for ingest work units.'),
|
|
99
|
+
profile: z
|
|
100
|
+
.union([z.boolean(), z.literal('json')])
|
|
101
|
+
.default(false)
|
|
102
|
+
.describe('Print a timing breakdown to stderr at the end of each ingest run. `true` prints a human table; `"json"` prints the raw structured profile for coding agents; `false` disables it. Equivalent to the KTX_PROFILE_INGEST environment variable (`1`/`true`/`json`).'),
|
|
99
103
|
})
|
|
100
104
|
.describe('Ingest pipeline configuration: adapters, embeddings, and work-unit policy.');
|
|
101
105
|
const scanEnrichmentSchema = z
|
|
@@ -21,7 +21,7 @@ function warehouseConnectionSchema(driver) {
|
|
|
21
21
|
enabled_tables: z
|
|
22
22
|
.array(z.string().min(1))
|
|
23
23
|
.optional()
|
|
24
|
-
.describe('Optional allowlist of fully-qualified table names ("schema.table") to ingest. When set, live-database ingest discards any table whose schema-qualified name is not in this list. Useful for smoke-testing
|
|
24
|
+
.describe('Optional allowlist of fully-qualified table names ("schema.table") to ingest. When set, live-database ingest discards any table whose schema-qualified name is not in this list. Useful for smoke-testing ingest on a single table.'),
|
|
25
25
|
})
|
|
26
26
|
.describe(`${driver} warehouse connection. Additional driver-tunable fields (e.g. context.queryHistory) are accepted and passed through.`);
|
|
27
27
|
}
|
|
@@ -95,7 +95,7 @@ async function wikiCandidates(project, input, options, terms) {
|
|
|
95
95
|
query: input.query,
|
|
96
96
|
userId: options.userId,
|
|
97
97
|
embeddingService: options.embeddingService ?? null,
|
|
98
|
-
limit: Math.max(input.limit ??
|
|
98
|
+
limit: Math.max(input.limit ?? 10, 25),
|
|
99
99
|
});
|
|
100
100
|
const records = [];
|
|
101
101
|
for (const result of searchResults) {
|
|
@@ -300,7 +300,8 @@ function hydrate(fused, refsByKey) {
|
|
|
300
300
|
}
|
|
301
301
|
return {
|
|
302
302
|
...ref,
|
|
303
|
-
|
|
303
|
+
// 3 decimals is plenty for a relative-rank hint; 6 just spent bytes on noise.
|
|
304
|
+
score: maxScore > 0 ? Number((candidate.score / maxScore).toFixed(3)) : 0,
|
|
304
305
|
};
|
|
305
306
|
})
|
|
306
307
|
.filter((result) => result !== null);
|
|
@@ -308,7 +309,7 @@ function hydrate(fused, refsByKey) {
|
|
|
308
309
|
export function createKtxDiscoverDataService(project, options = {}) {
|
|
309
310
|
return {
|
|
310
311
|
async search(input) {
|
|
311
|
-
const limit = Math.max(1, Math.min(input.limit ??
|
|
312
|
+
const limit = Math.max(1, Math.min(input.limit ?? 10, 50));
|
|
312
313
|
const query = input.query.trim();
|
|
313
314
|
if (!query) {
|
|
314
315
|
return [];
|
|
@@ -28,6 +28,7 @@ export interface LocalSlSearchInput {
|
|
|
28
28
|
backend?: 'pglite-owner-prototype';
|
|
29
29
|
pglite?: PgliteSlSearchPrototypeOwnerOptions;
|
|
30
30
|
}
|
|
31
|
+
/** @internal */
|
|
31
32
|
export interface LocalSlSource extends LocalSlSourceSummary {
|
|
32
33
|
yaml: string;
|
|
33
34
|
}
|
|
@@ -38,6 +39,15 @@ export interface LocalSlValidationResult {
|
|
|
38
39
|
valid: boolean;
|
|
39
40
|
errors: string[];
|
|
40
41
|
}
|
|
42
|
+
export type ResolvedSlSource = {
|
|
43
|
+
kind: 'found';
|
|
44
|
+
source: LocalSlSource;
|
|
45
|
+
} | {
|
|
46
|
+
kind: 'not-found';
|
|
47
|
+
} | {
|
|
48
|
+
kind: 'ambiguous';
|
|
49
|
+
connectionIds: string[];
|
|
50
|
+
};
|
|
41
51
|
export declare function loadLocalSlSourceRecords(project: KtxLocalProject, input: {
|
|
42
52
|
connectionId: string;
|
|
43
53
|
}): Promise<LocalSlSourceRecord[]>;
|
|
@@ -52,10 +62,15 @@ export declare function writeLocalSlSource(project: KtxLocalProject, input: {
|
|
|
52
62
|
sourceName: string;
|
|
53
63
|
yaml: string;
|
|
54
64
|
}): Promise<KtxFileWriteResult>;
|
|
65
|
+
/** @internal */
|
|
55
66
|
export declare function readLocalSlSource(project: KtxLocalProject, input: {
|
|
56
67
|
connectionId: string;
|
|
57
68
|
sourceName: string;
|
|
58
69
|
}): Promise<LocalSlSource | null>;
|
|
70
|
+
export declare function resolveLocalSlSource(project: KtxLocalProject, input: {
|
|
71
|
+
sourceName: string;
|
|
72
|
+
connectionId?: string;
|
|
73
|
+
}): Promise<ResolvedSlSource>;
|
|
59
74
|
export declare function listLocalSlSources(project: KtxLocalProject, input?: {
|
|
60
75
|
connectionId?: string;
|
|
61
76
|
}): Promise<LocalSlSourceSummary[]>;
|
|
@@ -204,6 +204,7 @@ export async function writeLocalSlSource(project, input) {
|
|
|
204
204
|
const path = slPath(input.connectionId, input.sourceName);
|
|
205
205
|
return project.fileStore.writeFile(path, input.yaml.endsWith('\n') ? input.yaml : `${input.yaml}\n`, LOCAL_AUTHOR, LOCAL_AUTHOR_EMAIL, `Write semantic-layer source: ${input.connectionId}/${input.sourceName}`);
|
|
206
206
|
}
|
|
207
|
+
/** @internal */
|
|
207
208
|
export async function readLocalSlSource(project, input) {
|
|
208
209
|
const path = slPath(input.connectionId, input.sourceName);
|
|
209
210
|
try {
|
|
@@ -221,6 +222,35 @@ export async function readLocalSlSource(project, input) {
|
|
|
221
222
|
return record ? { ...record } : null;
|
|
222
223
|
}
|
|
223
224
|
}
|
|
225
|
+
export async function resolveLocalSlSource(project, input) {
|
|
226
|
+
if (input.connectionId !== undefined) {
|
|
227
|
+
const source = await readLocalSlSource(project, {
|
|
228
|
+
connectionId: input.connectionId,
|
|
229
|
+
sourceName: input.sourceName,
|
|
230
|
+
});
|
|
231
|
+
return source ? { kind: 'found', source } : { kind: 'not-found' };
|
|
232
|
+
}
|
|
233
|
+
const summaries = await listLocalSlSources(project, {});
|
|
234
|
+
const matches = summaries.filter((summary) => summary.name === input.sourceName);
|
|
235
|
+
if (matches.length === 0) {
|
|
236
|
+
return { kind: 'not-found' };
|
|
237
|
+
}
|
|
238
|
+
if (matches.length > 1) {
|
|
239
|
+
return {
|
|
240
|
+
kind: 'ambiguous',
|
|
241
|
+
connectionIds: [...new Set(matches.map((match) => match.connectionId))].sort(),
|
|
242
|
+
};
|
|
243
|
+
}
|
|
244
|
+
const match = matches[0];
|
|
245
|
+
if (match === undefined) {
|
|
246
|
+
return { kind: 'not-found' };
|
|
247
|
+
}
|
|
248
|
+
const source = await readLocalSlSource(project, {
|
|
249
|
+
connectionId: match.connectionId,
|
|
250
|
+
sourceName: input.sourceName,
|
|
251
|
+
});
|
|
252
|
+
return source ? { kind: 'found', source } : { kind: 'not-found' };
|
|
253
|
+
}
|
|
224
254
|
export async function listLocalSlSources(project, input = {}) {
|
|
225
255
|
if (input.connectionId) {
|
|
226
256
|
return (await loadLocalSlSourceRecords(project, { connectionId: input.connectionId })).map(({ source: _source, yaml: _yaml, ...summary }) => summary);
|
|
@@ -50,6 +50,16 @@ export declare function readLocalKnowledgePage(project: KtxLocalProject, input:
|
|
|
50
50
|
export declare function listLocalKnowledgePages(project: KtxLocalProject, input?: {
|
|
51
51
|
userId?: string;
|
|
52
52
|
}): Promise<LocalKnowledgeSummary[]>;
|
|
53
|
+
/**
|
|
54
|
+
* List wiki page keys without reading or parsing file contents.
|
|
55
|
+
*
|
|
56
|
+
* Keys are derived purely from file paths, so this stays cheap enough for
|
|
57
|
+
* shell tab-completion (unlike `listLocalKnowledgePages`, which reads every
|
|
58
|
+
* page to populate summaries).
|
|
59
|
+
*/
|
|
60
|
+
export declare function listLocalKnowledgePageKeys(project: KtxLocalProject, input?: {
|
|
61
|
+
userId?: string;
|
|
62
|
+
}): Promise<string[]>;
|
|
53
63
|
export declare function searchLocalKnowledgePages(project: KtxLocalProject, input: {
|
|
54
64
|
query: string;
|
|
55
65
|
userId?: string;
|
|
@@ -118,6 +118,28 @@ export async function listLocalKnowledgePages(project, input = {}) {
|
|
|
118
118
|
}
|
|
119
119
|
return pages.sort((left, right) => left.path.localeCompare(right.path));
|
|
120
120
|
}
|
|
121
|
+
/**
|
|
122
|
+
* List wiki page keys without reading or parsing file contents.
|
|
123
|
+
*
|
|
124
|
+
* Keys are derived purely from file paths, so this stays cheap enough for
|
|
125
|
+
* shell tab-completion (unlike `listLocalKnowledgePages`, which reads every
|
|
126
|
+
* page to populate summaries).
|
|
127
|
+
*/
|
|
128
|
+
export async function listLocalKnowledgePageKeys(project, input = {}) {
|
|
129
|
+
const userId = input.userId ?? 'local';
|
|
130
|
+
const keys = new Set();
|
|
131
|
+
for (const scope of ['GLOBAL', 'USER']) {
|
|
132
|
+
const root = scope === 'GLOBAL' ? 'wiki/global' : `wiki/user/${assertSafePathToken('user id', userId)}`;
|
|
133
|
+
const listed = await project.fileStore.listFiles(root);
|
|
134
|
+
for (const path of listed.files.filter((file) => file.endsWith('.md'))) {
|
|
135
|
+
const key = keyFromKnowledgePath(path, scope, userId);
|
|
136
|
+
if (key) {
|
|
137
|
+
keys.add(key);
|
|
138
|
+
}
|
|
139
|
+
}
|
|
140
|
+
}
|
|
141
|
+
return [...keys].sort();
|
|
142
|
+
}
|
|
121
143
|
function scorePage(page, terms) {
|
|
122
144
|
const haystack = buildKnowledgeSearchText(page.key, page.summary, page.content, page.tags).toLowerCase();
|
|
123
145
|
return terms.some((term) => haystack.includes(term)) ? 3 : 0;
|
|
@@ -39,9 +39,6 @@ export interface ContextBuildArgs {
|
|
|
39
39
|
targetConnectionId?: string;
|
|
40
40
|
all?: boolean;
|
|
41
41
|
entrypoint?: 'setup' | 'ingest';
|
|
42
|
-
depth?: Extract<KtxPublicIngestArgs, {
|
|
43
|
-
command: 'run';
|
|
44
|
-
}>['depth'];
|
|
45
42
|
queryHistory?: Extract<KtxPublicIngestArgs, {
|
|
46
43
|
command: 'run';
|
|
47
44
|
}>['queryHistory'];
|
|
@@ -241,12 +241,11 @@ function renderMessageGroup(label, messages, styled) {
|
|
|
241
241
|
function retryCommand(input) {
|
|
242
242
|
const projectPart = input.projectDir ? ` --project-dir ${input.projectDir}` : '';
|
|
243
243
|
if (input.entrypoint === 'ingest' && input.connectionId) {
|
|
244
|
-
const depthPart = input.depth ? ` --${input.depth}` : '';
|
|
245
244
|
const queryHistoryPart = input.queryHistory ? ' --query-history' : '';
|
|
246
245
|
const windowPart = input.queryHistory && input.queryHistoryWindowDays !== undefined
|
|
247
246
|
? ` --query-history-window-days ${input.queryHistoryWindowDays}`
|
|
248
247
|
: '';
|
|
249
|
-
return `ktx ingest ${input.connectionId}${projectPart}${
|
|
248
|
+
return `ktx ingest ${input.connectionId}${projectPart}${queryHistoryPart}${windowPart}`;
|
|
250
249
|
}
|
|
251
250
|
return input.projectDir ? `ktx setup --project-dir ${input.projectDir}` : 'ktx setup';
|
|
252
251
|
}
|
|
@@ -563,7 +562,6 @@ function appendRetryIfNeeded(input) {
|
|
|
563
562
|
projectDir: input.projectDir,
|
|
564
563
|
entrypoint: input.entrypoint,
|
|
565
564
|
connectionId: input.target.connectionId,
|
|
566
|
-
depth: input.target.databaseDepth,
|
|
567
565
|
queryHistory: input.target.queryHistory?.enabled === true,
|
|
568
566
|
queryHistoryWindowDays: input.target.queryHistory?.windowDays,
|
|
569
567
|
})}`;
|
|
@@ -578,7 +576,6 @@ function failureTextForTarget(input) {
|
|
|
578
576
|
projectDir: input.projectDir,
|
|
579
577
|
entrypoint: input.entrypoint,
|
|
580
578
|
connectionId: input.target.connectionId,
|
|
581
|
-
depth: input.target.databaseDepth,
|
|
582
579
|
queryHistory: input.target.queryHistory?.enabled === true,
|
|
583
580
|
queryHistoryWindowDays: input.target.queryHistory?.windowDays,
|
|
584
581
|
})}`,
|
|
@@ -593,7 +590,6 @@ function failureTextForTarget(input) {
|
|
|
593
590
|
projectDir: input.projectDir,
|
|
594
591
|
entrypoint: input.entrypoint,
|
|
595
592
|
connectionId: input.target.connectionId,
|
|
596
|
-
depth: input.target.databaseDepth,
|
|
597
593
|
queryHistory: input.target.queryHistory?.enabled === true,
|
|
598
594
|
queryHistoryWindowDays: input.target.queryHistory?.windowDays,
|
|
599
595
|
})}`,
|
|
@@ -659,7 +655,6 @@ export async function runContextBuild(project, args, io, deps = {}) {
|
|
|
659
655
|
projectDir: args.projectDir,
|
|
660
656
|
...(args.targetConnectionId ? { targetConnectionId: args.targetConnectionId } : {}),
|
|
661
657
|
all: args.all ?? true,
|
|
662
|
-
...(args.depth ? { depth: args.depth } : {}),
|
|
663
658
|
...(args.queryHistory ? { queryHistory: args.queryHistory } : {}),
|
|
664
659
|
...(args.queryHistoryWindowDays !== undefined ? { queryHistoryWindowDays: args.queryHistoryWindowDays } : {}),
|
|
665
660
|
...(args.scanMode ? { scanMode: args.scanMode } : {}),
|
|
@@ -721,7 +716,6 @@ export async function runContextBuild(project, args, io, deps = {}) {
|
|
|
721
716
|
all: args.all ?? true,
|
|
722
717
|
json: false,
|
|
723
718
|
inputMode: args.inputMode,
|
|
724
|
-
...(args.depth ? { depth: args.depth } : {}),
|
|
725
719
|
...(args.queryHistory ? { queryHistory: args.queryHistory } : {}),
|
|
726
720
|
...(args.queryHistoryWindowDays !== undefined ? { queryHistoryWindowDays: args.queryHistoryWindowDays } : {}),
|
|
727
721
|
...(args.scanMode ? { scanMode: args.scanMode } : {}),
|
package/dist/ingest.js
CHANGED
|
@@ -2,7 +2,7 @@ import { buildMemoryFlowViewModel } from './context/ingest/memory-flow/view-mode
|
|
|
2
2
|
import { createMemoryFlowLiveBuffer, sanitizeMemoryFlowError } from './context/ingest/memory-flow/live-buffer.js';
|
|
3
3
|
import { formatMemoryFlowFinalSummary } from './context/ingest/memory-flow/summary.js';
|
|
4
4
|
import { getLatestLocalIngestStatus, getLocalIngestStatus, runLocalIngest, runLocalMetabaseIngest } from './context/ingest/local-ingest.js';
|
|
5
|
-
import { savedMemoryCountsForReport } from './context/ingest/reports.js';
|
|
5
|
+
import { ingestReportOutcome, savedMemoryCountsForReport } from './context/ingest/reports.js';
|
|
6
6
|
import { ingestReportToMemoryFlowReplay } from './context/ingest/memory-flow/events.js';
|
|
7
7
|
import { renderMemoryFlowReplay } from './context/ingest/memory-flow/render.js';
|
|
8
8
|
import { loadKtxProject } from './context/project/project.js';
|
|
@@ -17,9 +17,6 @@ import { renderMemoryFlowTui, startLiveMemoryFlowTui, } from './memory-flow-tui.
|
|
|
17
17
|
import { resolveVizFallback, warnVizFallbackOnce } from './viz-fallback.js';
|
|
18
18
|
import { profileMark } from './startup-profile.js';
|
|
19
19
|
profileMark('module:ingest');
|
|
20
|
-
function reportStatus(report) {
|
|
21
|
-
return report.body.status === 'failed' || report.body.failedWorkUnits.length > 0 ? 'error' : 'done';
|
|
22
|
-
}
|
|
23
20
|
const REPORT_SOURCE_LABELS = new Map([
|
|
24
21
|
['live-database', 'Database schema'],
|
|
25
22
|
['historic-sql', 'Query history'],
|
|
@@ -106,7 +103,7 @@ function writeReportStatus(report, io) {
|
|
|
106
103
|
if (report.body.tracePath) {
|
|
107
104
|
io.stdout.write(`Trace: ${report.body.tracePath}\n`);
|
|
108
105
|
}
|
|
109
|
-
io.stdout.write(`Status: ${
|
|
106
|
+
io.stdout.write(`Status: ${ingestReportOutcome(report)}\n`);
|
|
110
107
|
io.stdout.write(`Source: ${reportSourceLabel(report.sourceKey)}\n`);
|
|
111
108
|
io.stdout.write(`Connection: ${report.connectionId}\n`);
|
|
112
109
|
io.stdout.write(`Sync: ${report.body.syncId}\n`);
|
|
@@ -138,7 +135,7 @@ function writeMetabaseFanoutStatus(result, io) {
|
|
|
138
135
|
}
|
|
139
136
|
io.stdout.write(`Saved memory: ${counts.wikiCount} wiki, ${counts.slCount} SL\n`);
|
|
140
137
|
for (const child of result.children) {
|
|
141
|
-
const status =
|
|
138
|
+
const status = ingestReportOutcome(child.report);
|
|
142
139
|
io.stdout.write(`- target=${child.targetConnectionId} database=${child.metabaseDatabaseId} status=${status} job=${child.jobId} report=${child.report.id}\n`);
|
|
143
140
|
}
|
|
144
141
|
}
|
|
@@ -425,7 +422,7 @@ function initialRunMemoryFlowInput(args, runId) {
|
|
|
425
422
|
};
|
|
426
423
|
}
|
|
427
424
|
function finalRunMemoryFlowInput(snapshot, report) {
|
|
428
|
-
const status =
|
|
425
|
+
const status = ingestReportOutcome(report) === 'error' ? 'error' : 'done';
|
|
429
426
|
return {
|
|
430
427
|
...snapshot,
|
|
431
428
|
runId: report.runId,
|
|
@@ -574,7 +571,7 @@ export async function runKtxIngest(args, io = process, deps = {}) {
|
|
|
574
571
|
finally {
|
|
575
572
|
plainProgress?.flush();
|
|
576
573
|
}
|
|
577
|
-
return result.status === '
|
|
574
|
+
return result.status === 'all_failed' ? 1 : 0;
|
|
578
575
|
}
|
|
579
576
|
const jobId = deps.jobIdFactory?.();
|
|
580
577
|
let liveTui = null;
|
|
@@ -636,7 +633,7 @@ export async function runKtxIngest(args, io = process, deps = {}) {
|
|
|
636
633
|
liveTui?.close();
|
|
637
634
|
liveTui = null;
|
|
638
635
|
io.stdout.write(formatMemoryFlowFinalSummary(latestMemoryFlowSnapshot));
|
|
639
|
-
return
|
|
636
|
+
return ingestReportOutcome(result.report) === 'error' ? 1 : 0;
|
|
640
637
|
}
|
|
641
638
|
plainProgress?.flush();
|
|
642
639
|
await writeReportRecord(result.report, runOutputMode, io, {
|
|
@@ -644,7 +641,7 @@ export async function runKtxIngest(args, io = process, deps = {}) {
|
|
|
644
641
|
renderStoredMemoryFlow: deps.renderStoredMemoryFlow,
|
|
645
642
|
env,
|
|
646
643
|
});
|
|
647
|
-
return
|
|
644
|
+
return ingestReportOutcome(result.report) === 'error' ? 1 : 0;
|
|
648
645
|
}
|
|
649
646
|
finally {
|
|
650
647
|
plainProgress?.flush();
|
package/dist/knowledge.d.ts
CHANGED
|
@@ -18,6 +18,11 @@ export type KtxKnowledgeArgs = {
|
|
|
18
18
|
limit?: number;
|
|
19
19
|
debug?: boolean;
|
|
20
20
|
cliVersion: string;
|
|
21
|
+
} | {
|
|
22
|
+
command: 'read';
|
|
23
|
+
projectDir: string;
|
|
24
|
+
key: string;
|
|
25
|
+
userId: string;
|
|
21
26
|
};
|
|
22
27
|
type KtxKnowledgeIo = import('./cli-runtime.js').KtxCliIo;
|
|
23
28
|
interface KtxKnowledgeDeps {
|
package/dist/knowledge.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { KtxIngestEmbeddingPortAdapter } from './context/llm/embedding-port.js';
|
|
2
2
|
import { loadKtxProject } from './context/project/project.js';
|
|
3
|
-
import { listLocalKnowledgePages, searchLocalKnowledgePages as defaultSearchLocalKnowledgePages } from './context/wiki/local-knowledge.js';
|
|
3
|
+
import { listLocalKnowledgePages, readLocalKnowledgePage, searchLocalKnowledgePages as defaultSearchLocalKnowledgePages, } from './context/wiki/local-knowledge.js';
|
|
4
4
|
import { resolveProjectEmbeddingProvider, } from './embedding-resolution.js';
|
|
5
5
|
import { resolveOutputMode } from './io/mode.js';
|
|
6
6
|
import { createRankBadgeFormatter, printList } from './io/print-list.js';
|
|
@@ -72,6 +72,15 @@ export async function runKtxKnowledge(args, io = process, deps = {}) {
|
|
|
72
72
|
});
|
|
73
73
|
return 0;
|
|
74
74
|
}
|
|
75
|
+
if (args.command === 'read') {
|
|
76
|
+
const page = await readLocalKnowledgePage(project, { key: args.key, userId: args.userId });
|
|
77
|
+
if (!page) {
|
|
78
|
+
throw new Error(`No wiki page found for key '${args.key}'`);
|
|
79
|
+
}
|
|
80
|
+
const raw = await project.fileStore.readFile(page.path);
|
|
81
|
+
io.stdout.write(raw.content);
|
|
82
|
+
return 0;
|
|
83
|
+
}
|
|
75
84
|
if (args.command === 'search') {
|
|
76
85
|
const embeddingService = await wikiSearchEmbeddingService(project, deps, { cliVersion: args.cliVersion }, io);
|
|
77
86
|
const search = deps.searchLocalKnowledgePages ?? defaultSearchLocalKnowledgePages;
|
|
@@ -11,7 +11,7 @@ const DATABASE_INGEST_REPLACEMENTS = [
|
|
|
11
11
|
'Database enrichment failed after schema context completed',
|
|
12
12
|
],
|
|
13
13
|
[/\bstructural scan\b/gi, 'schema context'],
|
|
14
|
-
[/\benriched scan\b/gi, '
|
|
14
|
+
[/\benriched scan\b/gi, 'database ingest'],
|
|
15
15
|
[/\bscan results\b/gi, 'database context'],
|
|
16
16
|
];
|
|
17
17
|
export function publicDatabaseIngestMessage(message) {
|
package/dist/public-ingest.d.ts
CHANGED
|
@@ -2,14 +2,12 @@ import { type KtxLocalProject } from './context/project/project.js';
|
|
|
2
2
|
import type { KtxProgressPort } from './context/scan/types.js';
|
|
3
3
|
import type { KtxCliIo } from './index.js';
|
|
4
4
|
import type { KtxIngestArgs, KtxIngestDeps, KtxIngestProgressUpdate } from './ingest.js';
|
|
5
|
-
import { type KtxDatabaseContextDepth } from './ingest-depth.js';
|
|
6
5
|
import { type KtxManagedPythonInstallPolicy, type ManagedPythonCommandRuntime } from './managed-python-command.js';
|
|
7
6
|
import type { KtxRuntimeFeature } from './managed-python-runtime.js';
|
|
8
7
|
import type { KtxScanArgs, KtxScanDeps } from './scan.js';
|
|
9
8
|
type KtxPublicIngestStepName = 'database-schema' | 'query-history' | 'source-ingest' | 'memory-update';
|
|
10
9
|
type KtxPublicIngestStepStatus = 'done' | 'skipped' | 'failed' | 'not-run';
|
|
11
10
|
type KtxPublicIngestInputMode = 'auto' | 'disabled';
|
|
12
|
-
type KtxPublicIngestDepth = KtxDatabaseContextDepth;
|
|
13
11
|
type KtxPublicIngestQueryHistoryFlag = 'default' | 'enabled' | 'disabled';
|
|
14
12
|
type HistoricSqlDialect = 'postgres' | 'bigquery' | 'snowflake';
|
|
15
13
|
export type KtxPublicIngestArgs = {
|
|
@@ -19,7 +17,6 @@ export type KtxPublicIngestArgs = {
|
|
|
19
17
|
all: boolean;
|
|
20
18
|
json: boolean;
|
|
21
19
|
inputMode: KtxPublicIngestInputMode;
|
|
22
|
-
depth?: KtxPublicIngestDepth;
|
|
23
20
|
queryHistory?: KtxPublicIngestQueryHistoryFlag;
|
|
24
21
|
queryHistoryWindowDays?: number;
|
|
25
22
|
scanMode?: Extract<KtxScanArgs, {
|
|
@@ -37,7 +34,6 @@ export interface KtxPublicIngestPlanTarget {
|
|
|
37
34
|
sourceDir?: string;
|
|
38
35
|
debugCommand: string;
|
|
39
36
|
steps: KtxPublicIngestStepName[];
|
|
40
|
-
databaseDepth?: KtxPublicIngestDepth;
|
|
41
37
|
detectRelationships?: boolean;
|
|
42
38
|
preflightFailure?: string;
|
|
43
39
|
queryHistory?: {
|
|
@@ -46,7 +42,6 @@ export interface KtxPublicIngestPlanTarget {
|
|
|
46
42
|
windowDays?: number;
|
|
47
43
|
pullConfig?: Record<string, unknown>;
|
|
48
44
|
unsupported?: boolean;
|
|
49
|
-
skippedStoredByFast?: boolean;
|
|
50
45
|
};
|
|
51
46
|
}
|
|
52
47
|
export interface KtxPublicIngestPlan {
|
|
@@ -94,7 +89,6 @@ interface KtxPublicContextBuildArgs {
|
|
|
94
89
|
inputMode: 'auto' | 'disabled';
|
|
95
90
|
targetConnectionId?: string;
|
|
96
91
|
all?: boolean;
|
|
97
|
-
depth?: KtxPublicIngestDepth;
|
|
98
92
|
queryHistory?: KtxPublicIngestQueryHistoryFlag;
|
|
99
93
|
queryHistoryWindowDays?: number;
|
|
100
94
|
scanMode?: Extract<KtxScanArgs, {
|
|
@@ -108,7 +102,6 @@ export declare function buildPublicIngestPlan(project: KtxPublicIngestProject, a
|
|
|
108
102
|
projectDir: string;
|
|
109
103
|
targetConnectionId?: string;
|
|
110
104
|
all: boolean;
|
|
111
|
-
depth?: KtxPublicIngestDepth;
|
|
112
105
|
queryHistory?: KtxPublicIngestQueryHistoryFlag;
|
|
113
106
|
queryHistoryWindowDays?: number;
|
|
114
107
|
scanMode?: Extract<KtxScanArgs, {
|
package/dist/public-ingest.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { getKtxCliPackageInfo } from './cli-runtime.js';
|
|
2
2
|
import { loadKtxProject } from './context/project/project.js';
|
|
3
|
-
import {
|
|
3
|
+
import { isDatabaseDriver, normalizeConnectionDriver } from './connection-drivers.js';
|
|
4
4
|
import { ensureManagedPythonCommandRuntime, } from './managed-python-command.js';
|
|
5
5
|
import { publicIngestOutputLine } from './public-ingest-copy.js';
|
|
6
6
|
import { resolvePublicIngestRuntimeRequirements } from './runtime-requirements.js';
|
|
@@ -25,7 +25,6 @@ const queryHistoryDialectByDriver = new Map([
|
|
|
25
25
|
function createWarningAccumulator() {
|
|
26
26
|
return {
|
|
27
27
|
warnings: [],
|
|
28
|
-
ignoredDepthForSources: [],
|
|
29
28
|
ignoredQueryHistoryForSources: [],
|
|
30
29
|
unsupportedQueryHistoryForDatabases: [],
|
|
31
30
|
};
|
|
@@ -71,12 +70,6 @@ function finalizeWarnings(accumulator, args) {
|
|
|
71
70
|
...accumulator.warnings,
|
|
72
71
|
...unsupportedQueryHistoryWarnings(accumulator.unsupportedQueryHistoryForDatabases, args.all),
|
|
73
72
|
];
|
|
74
|
-
const depthOption = args.depth ? `--${args.depth}` : null;
|
|
75
|
-
if (depthOption) {
|
|
76
|
-
const warning = sourceIgnoredWarning(depthOption, accumulator.ignoredDepthForSources, args.all);
|
|
77
|
-
if (warning)
|
|
78
|
-
warnings.push(warning);
|
|
79
|
-
}
|
|
80
73
|
if (args.queryHistory === 'enabled' || args.queryHistoryWindowDays !== undefined) {
|
|
81
74
|
const warning = sourceIgnoredWarning('--query-history', accumulator.ignoredQueryHistoryForSources, args.all);
|
|
82
75
|
if (warning)
|
|
@@ -135,7 +128,6 @@ function resolveDatabaseTargetOptions(input) {
|
|
|
135
128
|
const windowOverrideRequested = input.args.queryHistoryWindowDays !== undefined;
|
|
136
129
|
const requestedQh = explicitQueryHistory === 'enabled' ||
|
|
137
130
|
(explicitQueryHistory !== 'disabled' && (windowOverrideRequested || storedEnabled));
|
|
138
|
-
let depth = input.args.depth ?? databaseContextDepth(input.connection) ?? 'fast';
|
|
139
131
|
const queryHistory = {
|
|
140
132
|
enabled: false,
|
|
141
133
|
...(input.args.queryHistoryWindowDays !== undefined
|
|
@@ -151,18 +143,12 @@ function resolveDatabaseTargetOptions(input) {
|
|
|
151
143
|
reason: explicitQueryHistory === 'enabled' || input.args.queryHistoryWindowDays !== undefined ? 'explicit' : 'stored',
|
|
152
144
|
});
|
|
153
145
|
return {
|
|
154
|
-
databaseDepth: depth,
|
|
155
146
|
queryHistory: { ...queryHistory, unsupported: true },
|
|
156
147
|
steps: ['database-schema'],
|
|
157
148
|
};
|
|
158
149
|
}
|
|
159
150
|
if (requestedQh && dialect) {
|
|
160
|
-
if (depth === 'fast') {
|
|
161
|
-
input.warnings.warnings.push(`--query-history requires deep ingest; running ${input.connectionId} with --deep.`);
|
|
162
|
-
}
|
|
163
|
-
depth = 'deep';
|
|
164
151
|
return {
|
|
165
|
-
databaseDepth: depth,
|
|
166
152
|
queryHistory: {
|
|
167
153
|
...queryHistory,
|
|
168
154
|
enabled: true,
|
|
@@ -177,28 +163,30 @@ function resolveDatabaseTargetOptions(input) {
|
|
|
177
163
|
steps: ['database-schema', 'query-history'],
|
|
178
164
|
};
|
|
179
165
|
}
|
|
180
|
-
if (input.args.depth === 'fast' && explicitQueryHistory !== 'enabled' && storedEnabled) {
|
|
181
|
-
input.warnings.warnings.push(`${input.connectionId} has query history enabled in ktx.yaml, but --fast skips query-history processing.`);
|
|
182
|
-
return {
|
|
183
|
-
databaseDepth: 'fast',
|
|
184
|
-
queryHistory: { ...queryHistory, skippedStoredByFast: true },
|
|
185
|
-
steps: ['database-schema'],
|
|
186
|
-
};
|
|
187
|
-
}
|
|
188
166
|
return {
|
|
189
|
-
databaseDepth: depth,
|
|
190
167
|
queryHistory,
|
|
191
168
|
steps: ['database-schema'],
|
|
192
169
|
};
|
|
193
170
|
}
|
|
171
|
+
function enrichmentReadinessGaps(config) {
|
|
172
|
+
const gaps = [];
|
|
173
|
+
if (config.llm.provider.backend === 'none' || !config.llm.models.default) {
|
|
174
|
+
gaps.push('model configuration');
|
|
175
|
+
}
|
|
176
|
+
if (config.scan.enrichment.mode !== 'llm') {
|
|
177
|
+
gaps.push('scan enrichment mode');
|
|
178
|
+
}
|
|
179
|
+
const embeddings = config.scan.enrichment.embeddings;
|
|
180
|
+
if (!embeddings || embeddings.backend === 'none' || !embeddings.model || embeddings.dimensions <= 0) {
|
|
181
|
+
gaps.push('scan embeddings');
|
|
182
|
+
}
|
|
183
|
+
return gaps;
|
|
184
|
+
}
|
|
194
185
|
function targetForConnection(connectionId, connection, projectConfig, args, warnings) {
|
|
195
186
|
const driver = normalizeConnectionDriver(connection);
|
|
196
187
|
const adapter = sourceAdapterByDriver.get(driver);
|
|
197
188
|
const sourceDir = sourceDirForConnection(connection);
|
|
198
189
|
if (adapter) {
|
|
199
|
-
if (args.depth) {
|
|
200
|
-
warnings.ignoredDepthForSources.push(connectionId);
|
|
201
|
-
}
|
|
202
190
|
if (args.queryHistory === 'enabled' || args.queryHistoryWindowDays !== undefined) {
|
|
203
191
|
warnings.ignoredQueryHistoryForSources.push(connectionId);
|
|
204
192
|
}
|
|
@@ -214,16 +202,16 @@ function targetForConnection(connectionId, connection, projectConfig, args, warn
|
|
|
214
202
|
}
|
|
215
203
|
if (isDatabaseDriver(driver)) {
|
|
216
204
|
const options = resolveDatabaseTargetOptions({ connectionId, driver, connection, args, warnings });
|
|
217
|
-
const gaps =
|
|
205
|
+
const gaps = enrichmentReadinessGaps(projectConfig);
|
|
218
206
|
return {
|
|
219
207
|
connectionId,
|
|
220
208
|
driver,
|
|
221
209
|
operation: 'database-ingest',
|
|
222
210
|
debugCommand: `ktx ingest ${connectionId} --debug`,
|
|
223
|
-
detectRelationships:
|
|
211
|
+
detectRelationships: projectConfig.scan.relationships.enabled,
|
|
224
212
|
...(gaps.length > 0
|
|
225
213
|
? {
|
|
226
|
-
preflightFailure: `${connectionId}
|
|
214
|
+
preflightFailure: `${connectionId} cannot be ingested: enrichment is not configured (${gaps.join(', ')}). Run ktx setup to configure a model and embeddings.`,
|
|
227
215
|
}
|
|
228
216
|
: {}),
|
|
229
217
|
...options,
|
|
@@ -281,12 +269,11 @@ function defaultSteps(target) {
|
|
|
281
269
|
}
|
|
282
270
|
function retryCommandForTarget(target, args) {
|
|
283
271
|
const projectPart = ` --project-dir ${args.projectDir}`;
|
|
284
|
-
const depthPart = target.databaseDepth ? ` --${target.databaseDepth}` : '';
|
|
285
272
|
const queryHistoryPart = target.queryHistory?.enabled === true ? ' --query-history' : '';
|
|
286
273
|
const windowPart = target.queryHistory?.enabled === true && target.queryHistory.windowDays !== undefined
|
|
287
274
|
? ` --query-history-window-days ${target.queryHistory.windowDays}`
|
|
288
275
|
: '';
|
|
289
|
-
return `ktx ingest ${target.connectionId}${projectPart}${
|
|
276
|
+
return `ktx ingest ${target.connectionId}${projectPart}${queryHistoryPart}${windowPart}`;
|
|
290
277
|
}
|
|
291
278
|
function trimTrailingPeriod(value) {
|
|
292
279
|
return value.endsWith('.') ? value.slice(0, -1) : value;
|
|
@@ -499,7 +486,7 @@ export async function executePublicIngestTarget(target, args, io, deps) {
|
|
|
499
486
|
command: 'run',
|
|
500
487
|
projectDir: args.projectDir,
|
|
501
488
|
connectionId: target.connectionId,
|
|
502
|
-
mode:
|
|
489
|
+
mode: 'enriched',
|
|
503
490
|
detectRelationships: target.detectRelationships === true,
|
|
504
491
|
dryRun: false,
|
|
505
492
|
...(args.cliVersion ? { cliVersion: args.cliVersion } : {}),
|
|
@@ -622,7 +609,6 @@ export async function runKtxPublicIngest(args, io, deps = {}) {
|
|
|
622
609
|
all: args.all,
|
|
623
610
|
entrypoint: 'ingest',
|
|
624
611
|
inputMode: args.inputMode,
|
|
625
|
-
...(args.depth ? { depth: args.depth } : {}),
|
|
626
612
|
...(args.queryHistory ? { queryHistory: args.queryHistory } : {}),
|
|
627
613
|
...(args.queryHistoryWindowDays !== undefined ? { queryHistoryWindowDays: args.queryHistoryWindowDays } : {}),
|
|
628
614
|
...(args.scanMode ? { scanMode: args.scanMode } : {}),
|