@kaelio/ktx 0.1.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/assets/python/{kaelio_ktx-0.1.0-py3-none-any.whl → kaelio_ktx-0.2.0-py3-none-any.whl} +0 -0
- package/assets/python/manifest.json +4 -4
- package/dist/admin-reindex.d.ts +15 -0
- package/dist/admin-reindex.js +168 -0
- package/dist/admin-reindex.test.js +116 -0
- package/dist/{dev.d.ts → admin.d.ts} +1 -1
- package/dist/{dev.js → admin.js} +14 -12
- package/dist/admin.test.d.ts +1 -0
- package/dist/{dev.test.js → admin.test.js} +36 -31
- package/dist/cli-program.js +7 -7
- package/dist/cli-program.test.js +1 -1
- package/dist/cli-runtime.d.ts +2 -0
- package/dist/commands/connection-commands.js +11 -10
- package/dist/commands/connection-selection.d.ts +11 -0
- package/dist/commands/connection-selection.js +9 -0
- package/dist/commands/ingest-commands.js +32 -26
- package/dist/commands/knowledge-commands.js +17 -28
- package/dist/commands/mcp-commands.js +17 -11
- package/dist/commands/setup-commands.js +14 -26
- package/dist/commands/sl-commands.js +27 -32
- package/dist/doctor.test.js +7 -8
- package/dist/example-smoke.test.js +3 -3
- package/dist/index.test.js +102 -70
- package/dist/ingest-depth.js +0 -1
- package/dist/ingest.test-utils.js +2 -2
- package/dist/ingest.test.js +4 -4
- package/dist/io/print-list.test.js +4 -4
- package/dist/knowledge.js +1 -1
- package/dist/managed-local-embeddings.d.ts +2 -0
- package/dist/managed-local-embeddings.js +2 -0
- package/dist/managed-local-embeddings.test.js +2 -0
- package/dist/managed-mcp-daemon.js +3 -2
- package/dist/managed-mcp-daemon.test.js +25 -0
- package/dist/managed-python-command.js +2 -2
- package/dist/managed-python-command.test.js +4 -3
- package/dist/managed-python-daemon.js +3 -2
- package/dist/managed-python-daemon.test.js +20 -0
- package/dist/managed-python-runtime.d.ts +5 -1
- package/dist/managed-python-runtime.js +50 -6
- package/dist/managed-python-runtime.test.js +53 -23
- package/dist/memory-flow-tui.test.js +2 -2
- package/dist/next-steps.d.ts +6 -6
- package/dist/next-steps.js +4 -4
- package/dist/next-steps.test.js +5 -5
- package/dist/print-command-tree.test.js +1 -1
- package/dist/proxy-env.d.ts +1 -0
- package/dist/proxy-env.js +23 -0
- package/dist/proxy-env.test.d.ts +1 -0
- package/dist/proxy-env.test.js +17 -0
- package/dist/public-ingest.js +3 -5
- package/dist/public-ingest.test.js +7 -3
- package/dist/runtime.test.js +2 -1
- package/dist/scan.test.js +2 -2
- package/dist/setup-agents.js +6 -4
- package/dist/setup-agents.test.js +35 -1
- package/dist/setup-embeddings.d.ts +1 -0
- package/dist/setup-embeddings.js +29 -7
- package/dist/setup-embeddings.test.js +49 -7
- package/dist/setup-models.d.ts +0 -1
- package/dist/setup-models.js +2 -3
- package/dist/setup-models.test.js +8 -10
- package/dist/setup-project.d.ts +9 -1
- package/dist/setup-project.js +52 -25
- package/dist/setup-project.test.js +8 -8
- package/dist/setup-runtime.test.js +4 -2
- package/dist/setup.d.ts +1 -2
- package/dist/setup.js +21 -5
- package/dist/setup.test.js +160 -43
- package/dist/sl.js +1 -1
- package/dist/sl.test.js +2 -1
- package/dist/standalone-smoke.test.js +8 -5
- package/dist/status-project.js +1 -10
- package/node_modules/@ktx/context/dist/index-sync/index.d.ts +2 -0
- package/node_modules/@ktx/context/dist/index-sync/index.js +1 -0
- package/node_modules/@ktx/context/dist/index-sync/reindex.d.ts +20 -0
- package/node_modules/@ktx/context/dist/index-sync/reindex.js +141 -0
- package/node_modules/@ktx/context/dist/index-sync/reindex.test.d.ts +1 -0
- package/node_modules/@ktx/context/dist/index-sync/reindex.test.js +139 -0
- package/node_modules/@ktx/context/dist/index-sync/types.d.ts +29 -0
- package/node_modules/@ktx/context/dist/index-sync/types.js +1 -0
- package/node_modules/@ktx/context/dist/index.d.ts +1 -0
- package/node_modules/@ktx/context/dist/index.js +1 -0
- package/node_modules/@ktx/context/dist/ingest/adapters/historic-sql/local-ingest-acceptance.test.js +1 -1
- package/node_modules/@ktx/context/dist/ingest/local-bundle-ingest.test.js +8 -8
- package/node_modules/@ktx/context/dist/ingest/local-bundle-runtime.js +4 -1
- package/node_modules/@ktx/context/dist/ingest/local-bundle-runtime.test.js +3 -3
- package/node_modules/@ktx/context/dist/ingest/local-embedding-provider.integration.test.js +9 -10
- package/node_modules/@ktx/context/dist/ingest/memory-flow/schema.d.ts +2 -2
- package/node_modules/@ktx/context/dist/ingest/report-snapshot.d.ts +2 -2
- package/node_modules/@ktx/context/dist/llm/local-config.js +2 -15
- package/node_modules/@ktx/context/dist/llm/local-config.test.js +3 -7
- package/node_modules/@ktx/context/dist/memory/local-memory.js +9 -3
- package/node_modules/@ktx/context/dist/project/config.d.ts +0 -5
- package/node_modules/@ktx/context/dist/project/config.js +5 -5
- package/node_modules/@ktx/context/dist/project/config.test.js +4 -7
- package/node_modules/@ktx/context/dist/scan/enrichment-state.test.js +4 -4
- package/node_modules/@ktx/context/dist/scan/index.d.ts +1 -1
- package/node_modules/@ktx/context/dist/scan/local-enrichment.d.ts +2 -6
- package/node_modules/@ktx/context/dist/scan/local-enrichment.js +31 -47
- package/node_modules/@ktx/context/dist/scan/local-enrichment.test.js +35 -18
- package/node_modules/@ktx/context/dist/scan/local-scan.test.js +2 -3
- package/node_modules/@ktx/context/dist/sl/ports.d.ts +3 -3
- package/node_modules/@ktx/context/dist/sl/sl-search.service.d.ts +3 -2
- package/node_modules/@ktx/context/dist/sl/sl-search.service.js +47 -45
- package/node_modules/@ktx/context/dist/sl/sl-search.service.test.js +61 -0
- package/node_modules/@ktx/context/dist/sl/sqlite-sl-sources-index.d.ts +4 -3
- package/node_modules/@ktx/context/dist/sl/sqlite-sl-sources-index.js +15 -5
- package/node_modules/@ktx/context/dist/sl/sqlite-sl-sources-index.test.js +24 -0
- package/node_modules/@ktx/context/dist/wiki/knowledge-wiki.service.d.ts +3 -2
- package/node_modules/@ktx/context/dist/wiki/knowledge-wiki.service.js +62 -51
- package/node_modules/@ktx/context/dist/wiki/knowledge-wiki.service.test.js +59 -3
- package/node_modules/@ktx/context/dist/wiki/ports.d.ts +3 -3
- package/node_modules/@ktx/context/dist/wiki/sqlite-knowledge-index.d.ts +33 -0
- package/node_modules/@ktx/context/dist/wiki/sqlite-knowledge-index.js +155 -2
- package/node_modules/@ktx/context/dist/wiki/sqlite-knowledge-index.test.js +26 -0
- package/node_modules/@ktx/context/package.json +5 -0
- package/node_modules/@ktx/llm/dist/embedding-provider.d.ts +0 -7
- package/node_modules/@ktx/llm/dist/embedding-provider.js +12 -138
- package/node_modules/@ktx/llm/dist/embedding-provider.test.js +10 -25
- package/node_modules/@ktx/llm/dist/types.d.ts +1 -1
- package/package.json +1 -1
- /package/dist/{dev.test.d.ts → admin-reindex.test.d.ts} +0 -0
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
import { type KtxLocalProject } from '../project/index.js';
|
|
2
|
+
import type { ReindexOptions, ReindexSummary } from './types.js';
|
|
3
|
+
type DiscoveredScope = {
|
|
4
|
+
kind: 'wiki';
|
|
5
|
+
scope: 'GLOBAL';
|
|
6
|
+
scopeId: null;
|
|
7
|
+
label: 'global';
|
|
8
|
+
} | {
|
|
9
|
+
kind: 'wiki';
|
|
10
|
+
scope: 'USER';
|
|
11
|
+
scopeId: string;
|
|
12
|
+
label: `user/${string}`;
|
|
13
|
+
} | {
|
|
14
|
+
kind: 'sl';
|
|
15
|
+
connectionId: string;
|
|
16
|
+
label: string;
|
|
17
|
+
};
|
|
18
|
+
export declare function discoverReindexScopes(project: KtxLocalProject): Promise<DiscoveredScope[]>;
|
|
19
|
+
export declare function reindexLocalIndexes(project: KtxLocalProject, options: ReindexOptions): Promise<ReindexSummary>;
|
|
20
|
+
export {};
|
|
@@ -0,0 +1,141 @@
|
|
|
1
|
+
import { readdir, stat } from 'node:fs/promises';
|
|
2
|
+
import { join, relative } from 'node:path';
|
|
3
|
+
import { ktxLocalStateDbPath } from '../project/index.js';
|
|
4
|
+
import { loadLocalSlSourceRecords, SlSearchService, SqliteSlSourcesIndex } from '../sl/index.js';
|
|
5
|
+
import { KnowledgeWikiService, SqliteKnowledgeIndex } from '../wiki/index.js';
|
|
6
|
+
const ZERO = {
|
|
7
|
+
scanned: 0,
|
|
8
|
+
updated: 0,
|
|
9
|
+
deleted: 0,
|
|
10
|
+
embeddingsRecomputed: 0,
|
|
11
|
+
embeddingsFailed: 0,
|
|
12
|
+
};
|
|
13
|
+
async function directoryExists(path) {
|
|
14
|
+
try {
|
|
15
|
+
return (await stat(path)).isDirectory();
|
|
16
|
+
}
|
|
17
|
+
catch {
|
|
18
|
+
return false;
|
|
19
|
+
}
|
|
20
|
+
}
|
|
21
|
+
async function childDirectories(path) {
|
|
22
|
+
try {
|
|
23
|
+
const entries = await readdir(path, { withFileTypes: true });
|
|
24
|
+
return entries
|
|
25
|
+
.filter((entry) => entry.isDirectory())
|
|
26
|
+
.map((entry) => entry.name)
|
|
27
|
+
.sort((left, right) => left.localeCompare(right));
|
|
28
|
+
}
|
|
29
|
+
catch (error) {
|
|
30
|
+
if (error.code === 'ENOENT') {
|
|
31
|
+
return [];
|
|
32
|
+
}
|
|
33
|
+
throw error;
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
export async function discoverReindexScopes(project) {
|
|
37
|
+
const scopes = [];
|
|
38
|
+
if (await directoryExists(join(project.projectDir, 'wiki/global'))) {
|
|
39
|
+
scopes.push({ kind: 'wiki', scope: 'GLOBAL', scopeId: null, label: 'global' });
|
|
40
|
+
}
|
|
41
|
+
for (const userId of await childDirectories(join(project.projectDir, 'wiki/user'))) {
|
|
42
|
+
scopes.push({ kind: 'wiki', scope: 'USER', scopeId: userId, label: `user/${userId}` });
|
|
43
|
+
}
|
|
44
|
+
for (const connectionId of await childDirectories(join(project.projectDir, 'semantic-layer'))) {
|
|
45
|
+
if (connectionId !== '_schema') {
|
|
46
|
+
scopes.push({ kind: 'sl', connectionId, label: connectionId });
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
return scopes;
|
|
50
|
+
}
|
|
51
|
+
function errorMessage(error) {
|
|
52
|
+
if (!(error instanceof Error)) {
|
|
53
|
+
return String(error);
|
|
54
|
+
}
|
|
55
|
+
return error.name && error.name !== 'Error' ? `${error.name}: ${error.message}` : error.message;
|
|
56
|
+
}
|
|
57
|
+
function addTotals(left, right) {
|
|
58
|
+
return {
|
|
59
|
+
scanned: left.scanned + right.scanned,
|
|
60
|
+
updated: left.updated + right.updated,
|
|
61
|
+
deleted: left.deleted + right.deleted,
|
|
62
|
+
embeddingsRecomputed: left.embeddingsRecomputed + right.embeddingsRecomputed,
|
|
63
|
+
embeddingsFailed: left.embeddingsFailed + right.embeddingsFailed,
|
|
64
|
+
};
|
|
65
|
+
}
|
|
66
|
+
function durationSince(startedAt) {
|
|
67
|
+
return Number((process.hrtime.bigint() - startedAt) / 1000000n);
|
|
68
|
+
}
|
|
69
|
+
function embeddingFailureError(work) {
|
|
70
|
+
if (work.embeddingsFailed === 0) {
|
|
71
|
+
return undefined;
|
|
72
|
+
}
|
|
73
|
+
return `${work.embeddingsFailed} embedding recomputation${work.embeddingsFailed === 1 ? '' : 's'} failed`;
|
|
74
|
+
}
|
|
75
|
+
export async function reindexLocalIndexes(project, options) {
|
|
76
|
+
const startedAt = process.hrtime.bigint();
|
|
77
|
+
const dbPath = ktxLocalStateDbPath(project);
|
|
78
|
+
const scopes = await discoverReindexScopes(project);
|
|
79
|
+
const wikiIndex = new SqliteKnowledgeIndex({ dbPath });
|
|
80
|
+
const slIndex = new SqliteSlSourcesIndex({ dbPath });
|
|
81
|
+
const wikiService = new KnowledgeWikiService(project.fileStore, options.embeddingService, wikiIndex, project.git);
|
|
82
|
+
const slService = new SlSearchService(options.embeddingService, slIndex);
|
|
83
|
+
const results = [];
|
|
84
|
+
for (const scope of scopes) {
|
|
85
|
+
const scopeStartedAt = process.hrtime.bigint();
|
|
86
|
+
try {
|
|
87
|
+
let work;
|
|
88
|
+
if (scope.kind === 'wiki') {
|
|
89
|
+
if (options.force) {
|
|
90
|
+
wikiIndex.clear(scope.scope, scope.scopeId);
|
|
91
|
+
}
|
|
92
|
+
work = await wikiService.syncIndex(scope.scope, scope.scopeId);
|
|
93
|
+
results.push({
|
|
94
|
+
kind: 'wiki',
|
|
95
|
+
label: scope.label,
|
|
96
|
+
scope: scope.scope === 'GLOBAL' ? 'global' : 'user',
|
|
97
|
+
scopeId: scope.scopeId,
|
|
98
|
+
...work,
|
|
99
|
+
...(options.force ? { deleted: 0 } : {}),
|
|
100
|
+
...(options.embeddingService && work.embeddingsFailed > 0 ? { error: embeddingFailureError(work) } : {}),
|
|
101
|
+
durationMs: durationSince(scopeStartedAt),
|
|
102
|
+
});
|
|
103
|
+
continue;
|
|
104
|
+
}
|
|
105
|
+
if (options.force) {
|
|
106
|
+
await slIndex.clear(scope.connectionId);
|
|
107
|
+
}
|
|
108
|
+
const records = await loadLocalSlSourceRecords(project, { connectionId: scope.connectionId });
|
|
109
|
+
work = await slService.indexSources(scope.connectionId, records.map((record) => record.source));
|
|
110
|
+
results.push({
|
|
111
|
+
kind: 'sl',
|
|
112
|
+
label: scope.label,
|
|
113
|
+
connectionId: scope.connectionId,
|
|
114
|
+
...work,
|
|
115
|
+
...(options.force ? { deleted: 0 } : {}),
|
|
116
|
+
...(options.embeddingService && work.embeddingsFailed > 0 ? { error: embeddingFailureError(work) } : {}),
|
|
117
|
+
durationMs: durationSince(scopeStartedAt),
|
|
118
|
+
});
|
|
119
|
+
}
|
|
120
|
+
catch (error) {
|
|
121
|
+
results.push({
|
|
122
|
+
kind: scope.kind,
|
|
123
|
+
label: scope.label,
|
|
124
|
+
...(scope.kind === 'wiki'
|
|
125
|
+
? { scope: scope.scope === 'GLOBAL' ? 'global' : 'user', scopeId: scope.scopeId }
|
|
126
|
+
: { connectionId: scope.connectionId }),
|
|
127
|
+
...ZERO,
|
|
128
|
+
durationMs: durationSince(scopeStartedAt),
|
|
129
|
+
error: errorMessage(error),
|
|
130
|
+
});
|
|
131
|
+
}
|
|
132
|
+
}
|
|
133
|
+
return {
|
|
134
|
+
scopes: results,
|
|
135
|
+
totals: results.reduce(addTotals, ZERO),
|
|
136
|
+
dbPath: relative(project.projectDir, dbPath) || dbPath,
|
|
137
|
+
force: options.force,
|
|
138
|
+
embeddingsAvailable: options.embeddingService !== null,
|
|
139
|
+
durationMs: durationSince(startedAt),
|
|
140
|
+
};
|
|
141
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,139 @@
|
|
|
1
|
+
import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises';
|
|
2
|
+
import { tmpdir } from 'node:os';
|
|
3
|
+
import { join } from 'node:path';
|
|
4
|
+
import { afterEach, beforeEach, describe, expect, it } from 'vitest';
|
|
5
|
+
import { initKtxProject, loadKtxProject } from '../project/index.js';
|
|
6
|
+
import { SqliteKnowledgeIndex } from '../wiki/sqlite-knowledge-index.js';
|
|
7
|
+
import { reindexLocalIndexes } from './reindex.js';
|
|
8
|
+
class FakeEmbeddingPort {
|
|
9
|
+
maxBatchSize = 8;
|
|
10
|
+
async computeEmbedding(text) {
|
|
11
|
+
return [text.length, 1];
|
|
12
|
+
}
|
|
13
|
+
async computeEmbeddingsBulk(texts) {
|
|
14
|
+
return texts.map((text) => [text.length, 1]);
|
|
15
|
+
}
|
|
16
|
+
}
|
|
17
|
+
async function createProject(tempDir) {
|
|
18
|
+
await initKtxProject({ projectDir: tempDir, force: true });
|
|
19
|
+
return loadKtxProject({ projectDir: tempDir });
|
|
20
|
+
}
|
|
21
|
+
describe('reindexLocalIndexes', () => {
|
|
22
|
+
let tempDir;
|
|
23
|
+
beforeEach(async () => {
|
|
24
|
+
tempDir = await mkdtemp(join(tmpdir(), 'ktx-reindex-'));
|
|
25
|
+
});
|
|
26
|
+
afterEach(async () => {
|
|
27
|
+
await rm(tempDir, { recursive: true, force: true });
|
|
28
|
+
});
|
|
29
|
+
it('returns an empty summary when no wiki or semantic-layer directories exist', async () => {
|
|
30
|
+
const project = await createProject(tempDir);
|
|
31
|
+
await rm(join(project.projectDir, 'wiki'), { recursive: true, force: true });
|
|
32
|
+
await rm(join(project.projectDir, 'semantic-layer'), { recursive: true, force: true });
|
|
33
|
+
await expect(reindexLocalIndexes(project, { force: false, embeddingService: null })).resolves.toMatchObject({
|
|
34
|
+
scopes: [],
|
|
35
|
+
totals: { scanned: 0, updated: 0, deleted: 0, embeddingsRecomputed: 0, embeddingsFailed: 0 },
|
|
36
|
+
force: false,
|
|
37
|
+
embeddingsAvailable: false,
|
|
38
|
+
});
|
|
39
|
+
});
|
|
40
|
+
it('discovers empty directories as zero-row scopes', async () => {
|
|
41
|
+
const project = await createProject(tempDir);
|
|
42
|
+
await mkdir(join(project.projectDir, 'wiki/user/local'), { recursive: true });
|
|
43
|
+
await mkdir(join(project.projectDir, 'semantic-layer/warehouse'), { recursive: true });
|
|
44
|
+
const summary = await reindexLocalIndexes(project, { force: false, embeddingService: null });
|
|
45
|
+
expect(summary.scopes.map((scope) => scope.label)).toEqual(['global', 'user/local', 'warehouse']);
|
|
46
|
+
expect(summary.totals.scanned).toBe(0);
|
|
47
|
+
});
|
|
48
|
+
it('indexes mixed wiki and SL sources and reports totals', async () => {
|
|
49
|
+
const project = await createProject(tempDir);
|
|
50
|
+
await writeFile(join(project.projectDir, 'wiki/global/revenue.md'), '---\nsummary: Revenue\nusage_mode: auto\n---\n\nPaid orders.\n', 'utf-8');
|
|
51
|
+
await mkdir(join(project.projectDir, 'semantic-layer/warehouse'), { recursive: true });
|
|
52
|
+
await writeFile(join(project.projectDir, 'semantic-layer/warehouse/orders.yaml'), 'name: orders\ntable: public.orders\ngrain: [id]\ncolumns:\n - name: id\n type: number\njoins: []\nmeasures: []\n', 'utf-8');
|
|
53
|
+
const summary = await reindexLocalIndexes(project, {
|
|
54
|
+
force: false,
|
|
55
|
+
embeddingService: new FakeEmbeddingPort(),
|
|
56
|
+
});
|
|
57
|
+
expect(summary.scopes).toHaveLength(2);
|
|
58
|
+
expect(summary.totals).toMatchObject({ scanned: 2, updated: 2, deleted: 0, embeddingsRecomputed: 2 });
|
|
59
|
+
expect(summary.embeddingsAvailable).toBe(true);
|
|
60
|
+
});
|
|
61
|
+
it('does not report unchanged lexical-only rows as updated on repeated runs', async () => {
|
|
62
|
+
const project = await createProject(tempDir);
|
|
63
|
+
await writeFile(join(project.projectDir, 'wiki/global/revenue.md'), '---\nsummary: Revenue\nusage_mode: auto\n---\n\nPaid orders.\n', 'utf-8');
|
|
64
|
+
await mkdir(join(project.projectDir, 'semantic-layer/warehouse'), { recursive: true });
|
|
65
|
+
await writeFile(join(project.projectDir, 'semantic-layer/warehouse/orders.yaml'), 'name: orders\ntable: public.orders\ngrain: [id]\ncolumns:\n - name: id\n type: number\njoins: []\nmeasures: []\n', 'utf-8');
|
|
66
|
+
const first = await reindexLocalIndexes(project, { force: false, embeddingService: null });
|
|
67
|
+
expect(first.totals).toMatchObject({
|
|
68
|
+
scanned: 2,
|
|
69
|
+
updated: 2,
|
|
70
|
+
deleted: 0,
|
|
71
|
+
embeddingsRecomputed: 0,
|
|
72
|
+
embeddingsFailed: 0,
|
|
73
|
+
});
|
|
74
|
+
const second = await reindexLocalIndexes(project, { force: false, embeddingService: null });
|
|
75
|
+
expect(second.totals).toMatchObject({
|
|
76
|
+
scanned: 2,
|
|
77
|
+
updated: 0,
|
|
78
|
+
deleted: 0,
|
|
79
|
+
embeddingsRecomputed: 0,
|
|
80
|
+
embeddingsFailed: 0,
|
|
81
|
+
});
|
|
82
|
+
expect(second.scopes.map((scope) => [scope.label, scope.updated])).toEqual([
|
|
83
|
+
['global', 0],
|
|
84
|
+
['warehouse', 0],
|
|
85
|
+
]);
|
|
86
|
+
});
|
|
87
|
+
it('force clears stale rows before rebuilding each discovered scope', async () => {
|
|
88
|
+
const project = await createProject(tempDir);
|
|
89
|
+
const wikiIndex = new SqliteKnowledgeIndex({ dbPath: join(project.projectDir, '.ktx/db.sqlite') });
|
|
90
|
+
wikiIndex.sync([
|
|
91
|
+
{
|
|
92
|
+
path: 'wiki/global/stale.md',
|
|
93
|
+
key: 'stale',
|
|
94
|
+
scope: 'GLOBAL',
|
|
95
|
+
scopeId: null,
|
|
96
|
+
summary: 'Stale',
|
|
97
|
+
content: 'Stale content',
|
|
98
|
+
tags: [],
|
|
99
|
+
embedding: [1, 0],
|
|
100
|
+
},
|
|
101
|
+
]);
|
|
102
|
+
await writeFile(join(project.projectDir, 'wiki/global/revenue.md'), '---\nsummary: Revenue\nusage_mode: auto\n---\n\nPaid orders.\n', 'utf-8');
|
|
103
|
+
const summary = await reindexLocalIndexes(project, {
|
|
104
|
+
force: true,
|
|
105
|
+
embeddingService: new FakeEmbeddingPort(),
|
|
106
|
+
});
|
|
107
|
+
expect(summary.force).toBe(true);
|
|
108
|
+
expect(summary.totals).toMatchObject({ scanned: 1, updated: 1, deleted: 0 });
|
|
109
|
+
expect(wikiIndex.search('Stale', 10)).toEqual([]);
|
|
110
|
+
});
|
|
111
|
+
it('captures a per-scope error and continues other scopes', async () => {
|
|
112
|
+
const project = await createProject(tempDir);
|
|
113
|
+
await writeFile(join(project.projectDir, 'wiki/global/revenue.md'), '---\nsummary: Revenue\nusage_mode: auto\n---\n\nPaid orders.\n', 'utf-8');
|
|
114
|
+
await mkdir(join(project.projectDir, 'semantic-layer/warehouse'), { recursive: true });
|
|
115
|
+
await writeFile(join(project.projectDir, 'semantic-layer/warehouse/broken.yaml'), 'not: [valid', 'utf-8');
|
|
116
|
+
const summary = await reindexLocalIndexes(project, { force: false, embeddingService: null });
|
|
117
|
+
expect(summary.scopes.find((scope) => scope.label === 'global')?.error).toBeUndefined();
|
|
118
|
+
expect(summary.scopes.find((scope) => scope.label === 'warehouse')?.error).toContain('YAML');
|
|
119
|
+
});
|
|
120
|
+
it('marks a scope errored when configured embeddings fail', async () => {
|
|
121
|
+
const project = await createProject(tempDir);
|
|
122
|
+
await writeFile(join(project.projectDir, 'wiki/global/revenue.md'), '---\nsummary: Revenue\nusage_mode: auto\n---\n\nPaid orders.\n', 'utf-8');
|
|
123
|
+
const embeddingService = {
|
|
124
|
+
maxBatchSize: 8,
|
|
125
|
+
async computeEmbedding() {
|
|
126
|
+
throw new Error('embedding provider unavailable');
|
|
127
|
+
},
|
|
128
|
+
async computeEmbeddingsBulk() {
|
|
129
|
+
throw new Error('embedding provider unavailable');
|
|
130
|
+
},
|
|
131
|
+
};
|
|
132
|
+
const summary = await reindexLocalIndexes(project, { force: false, embeddingService });
|
|
133
|
+
expect(summary.scopes[0]).toMatchObject({
|
|
134
|
+
label: 'global',
|
|
135
|
+
embeddingsFailed: 1,
|
|
136
|
+
error: '1 embedding recomputation failed',
|
|
137
|
+
});
|
|
138
|
+
});
|
|
139
|
+
});
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
import type { KtxEmbeddingPort } from '../core/index.js';
|
|
2
|
+
export interface ReindexOptions {
|
|
3
|
+
force: boolean;
|
|
4
|
+
embeddingService: KtxEmbeddingPort | null;
|
|
5
|
+
}
|
|
6
|
+
export interface ReindexWorkResult {
|
|
7
|
+
scanned: number;
|
|
8
|
+
updated: number;
|
|
9
|
+
deleted: number;
|
|
10
|
+
embeddingsRecomputed: number;
|
|
11
|
+
embeddingsFailed: number;
|
|
12
|
+
}
|
|
13
|
+
export interface ReindexScopeResult extends ReindexWorkResult {
|
|
14
|
+
kind: 'wiki' | 'sl';
|
|
15
|
+
label: string;
|
|
16
|
+
scope?: 'global' | 'user';
|
|
17
|
+
scopeId?: string | null;
|
|
18
|
+
connectionId?: string;
|
|
19
|
+
durationMs: number;
|
|
20
|
+
error?: string;
|
|
21
|
+
}
|
|
22
|
+
export interface ReindexSummary {
|
|
23
|
+
scopes: ReindexScopeResult[];
|
|
24
|
+
totals: ReindexWorkResult;
|
|
25
|
+
dbPath: string;
|
|
26
|
+
force: boolean;
|
|
27
|
+
embeddingsAvailable: boolean;
|
|
28
|
+
durationMs: number;
|
|
29
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -7,6 +7,7 @@ export * from './agent/index.js';
|
|
|
7
7
|
export * from './core/index.js';
|
|
8
8
|
export * from './daemon/index.js';
|
|
9
9
|
export * from './ingest/index.js';
|
|
10
|
+
export * from './index-sync/index.js';
|
|
10
11
|
export * from './llm/index.js';
|
|
11
12
|
export type { CaptureSession, CaptureSignals, MemoryAgentInput, MemoryAgentResult, MemoryAgentServiceDeps, MemoryAgentSettings, MemoryAgentSourceType, MemoryCommitMessagePort, MemoryConnectionPort, MemoryFileStorePort, MemoryKnowledgeSlRefsPort, MemoryLockPort, MemorySlSourceReconcilerPort, MemoryTelemetryPort, MemoryToolSetLike, MemoryToolsetFactoryPort, } from './memory/index.js';
|
|
12
13
|
export * from './project/index.js';
|
|
@@ -6,6 +6,7 @@ export * from './agent/index.js';
|
|
|
6
6
|
export * from './core/index.js';
|
|
7
7
|
export * from './daemon/index.js';
|
|
8
8
|
export * from './ingest/index.js';
|
|
9
|
+
export * from './index-sync/index.js';
|
|
9
10
|
export * from './llm/index.js';
|
|
10
11
|
export * from './project/index.js';
|
|
11
12
|
export * from './prompts/index.js';
|
|
@@ -262,7 +262,7 @@ describe('canonical local ingest', () => {
|
|
|
262
262
|
' adapters:',
|
|
263
263
|
' - fake',
|
|
264
264
|
' embeddings:',
|
|
265
|
-
' backend:
|
|
265
|
+
' backend: none',
|
|
266
266
|
'',
|
|
267
267
|
].join('\n'), 'utf-8');
|
|
268
268
|
project = await loadKtxProject({ projectDir });
|
|
@@ -348,9 +348,9 @@ describe('canonical local ingest', () => {
|
|
|
348
348
|
expect(result.result.failedWorkUnits).toEqual([]);
|
|
349
349
|
const db = new Database(join(project.projectDir, '.ktx', 'db.sqlite'), { readonly: true });
|
|
350
350
|
try {
|
|
351
|
-
expect(db
|
|
352
|
-
|
|
353
|
-
|
|
351
|
+
expect(db
|
|
352
|
+
.prepare('SELECT key, summary, embedding_json IS NOT NULL AS has_embedding FROM knowledge_pages ORDER BY key')
|
|
353
|
+
.all()).toEqual([{ key: 'orders_context', summary: 'Orders source context', has_embedding: 0 }]);
|
|
354
354
|
}
|
|
355
355
|
finally {
|
|
356
356
|
db.close();
|
|
@@ -437,7 +437,7 @@ describe('canonical local ingest', () => {
|
|
|
437
437
|
' adapters:',
|
|
438
438
|
' - historic-sql',
|
|
439
439
|
' embeddings:',
|
|
440
|
-
' backend:
|
|
440
|
+
' backend: none',
|
|
441
441
|
'storage:',
|
|
442
442
|
' state: sqlite',
|
|
443
443
|
' search: sqlite-fts5',
|
|
@@ -496,7 +496,7 @@ describe('canonical local ingest', () => {
|
|
|
496
496
|
' adapters:',
|
|
497
497
|
' - metabase',
|
|
498
498
|
' embeddings:',
|
|
499
|
-
' backend:
|
|
499
|
+
' backend: none',
|
|
500
500
|
'',
|
|
501
501
|
].join('\n'), 'utf-8');
|
|
502
502
|
const metabaseProject = await loadKtxProject({ projectDir });
|
|
@@ -556,7 +556,7 @@ describe('canonical local ingest', () => {
|
|
|
556
556
|
' adapters:',
|
|
557
557
|
' - metricflow',
|
|
558
558
|
' embeddings:',
|
|
559
|
-
' backend:
|
|
559
|
+
' backend: none',
|
|
560
560
|
'storage:',
|
|
561
561
|
' state: sqlite',
|
|
562
562
|
' search: sqlite-fts5',
|
|
@@ -656,7 +656,7 @@ describe('canonical local ingest', () => {
|
|
|
656
656
|
' adapters:',
|
|
657
657
|
' - looker',
|
|
658
658
|
' embeddings:',
|
|
659
|
-
' backend:
|
|
659
|
+
' backend: none',
|
|
660
660
|
'storage:',
|
|
661
661
|
' state: sqlite',
|
|
662
662
|
' search: sqlite-fts5',
|
|
@@ -256,12 +256,15 @@ class LocalKnowledgeIndex {
|
|
|
256
256
|
}
|
|
257
257
|
async deleteStale() {
|
|
258
258
|
await this.syncAllPagesFromDisk();
|
|
259
|
+
return 0;
|
|
259
260
|
}
|
|
260
261
|
async deleteByScope() {
|
|
261
262
|
await this.syncAllPagesFromDisk();
|
|
263
|
+
return 0;
|
|
262
264
|
}
|
|
263
265
|
async deleteByKey() {
|
|
264
266
|
await this.syncAllPagesFromDisk();
|
|
267
|
+
return 0;
|
|
265
268
|
}
|
|
266
269
|
async findPageByKey(scope, scopeId, pageKey) {
|
|
267
270
|
const path = scope === 'GLOBAL' ? `wiki/global/${pageKey}.md` : `wiki/user/${scopeId}/${pageKey}.md`;
|
|
@@ -461,7 +464,7 @@ function localIngestLlmProviderGuardMessage(projectDir) {
|
|
|
461
464
|
'ktx ingest requires llm.provider.backend: anthropic, vertex, gateway, or claude-code, or an injected agentRunner.',
|
|
462
465
|
'Configure a local Claude Code session or API-backed LLM, then rerun ingest:',
|
|
463
466
|
` ktx setup --project-dir ${projectDir} --llm-backend claude-code --no-input`,
|
|
464
|
-
` ktx setup --project-dir ${projectDir} --llm-backend anthropic --anthropic-api-key-env ANTHROPIC_API_KEY --
|
|
467
|
+
` ktx setup --project-dir ${projectDir} --llm-backend anthropic --anthropic-api-key-env ANTHROPIC_API_KEY --llm-model claude-sonnet-4-6 --no-input`,
|
|
465
468
|
].join('\n');
|
|
466
469
|
}
|
|
467
470
|
function resolveAgentRunner(options) {
|
|
@@ -23,7 +23,7 @@ describe('createLocalBundleIngestRuntime', () => {
|
|
|
23
23
|
' adapters:',
|
|
24
24
|
' - fake',
|
|
25
25
|
' embeddings:',
|
|
26
|
-
' backend:
|
|
26
|
+
' backend: none',
|
|
27
27
|
'',
|
|
28
28
|
].join('\n'), 'utf-8');
|
|
29
29
|
project = await loadKtxProject({ projectDir });
|
|
@@ -39,7 +39,7 @@ describe('createLocalBundleIngestRuntime', () => {
|
|
|
39
39
|
'ktx ingest requires llm.provider.backend: anthropic, vertex, gateway, or claude-code, or an injected agentRunner.',
|
|
40
40
|
'Configure a local Claude Code session or API-backed LLM, then rerun ingest:',
|
|
41
41
|
` ktx setup --project-dir ${project.projectDir} --llm-backend claude-code --no-input`,
|
|
42
|
-
` ktx setup --project-dir ${project.projectDir} --llm-backend anthropic --anthropic-api-key-env ANTHROPIC_API_KEY --
|
|
42
|
+
` ktx setup --project-dir ${project.projectDir} --llm-backend anthropic --anthropic-api-key-env ANTHROPIC_API_KEY --llm-model claude-sonnet-4-6 --no-input`,
|
|
43
43
|
].join('\n'));
|
|
44
44
|
});
|
|
45
45
|
it('uses a runtime-backed agent runner when claude-code is configured', () => {
|
|
@@ -211,7 +211,7 @@ describe('createLocalBundleIngestRuntime', () => {
|
|
|
211
211
|
' adapters:',
|
|
212
212
|
' - fake',
|
|
213
213
|
' embeddings:',
|
|
214
|
-
' backend:
|
|
214
|
+
' backend: none',
|
|
215
215
|
'',
|
|
216
216
|
].join('\n'), 'utf-8');
|
|
217
217
|
project = await loadKtxProject({ projectDir: project.projectDir });
|
|
@@ -2,7 +2,6 @@ import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises';
|
|
|
2
2
|
import { tmpdir } from 'node:os';
|
|
3
3
|
import { join } from 'node:path';
|
|
4
4
|
import { afterEach, beforeEach, describe, expect, it } from 'vitest';
|
|
5
|
-
import { createLocalKtxEmbeddingProviderFromConfig, KtxIngestEmbeddingPortAdapter } from '../llm/index.js';
|
|
6
5
|
import { CandidateDedupService } from './context-candidates/candidate-dedup.service.js';
|
|
7
6
|
import { ContextEvidenceIndexService } from './context-evidence/context-evidence-index.service.js';
|
|
8
7
|
import { SqliteContextEvidenceStore } from './context-evidence/sqlite-context-evidence-store.js';
|
|
@@ -31,15 +30,15 @@ describe('local ingest embedding providers with SQLite ingest stores', () => {
|
|
|
31
30
|
await rm(tempDir, { recursive: true, force: true });
|
|
32
31
|
});
|
|
33
32
|
function embeddings() {
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
33
|
+
return {
|
|
34
|
+
maxBatchSize: 4,
|
|
35
|
+
async computeEmbedding() {
|
|
36
|
+
return [1, 0, 0];
|
|
37
|
+
},
|
|
38
|
+
async computeEmbeddingsBulk(texts) {
|
|
39
|
+
return texts.map(() => [1, 0, 0]);
|
|
40
|
+
},
|
|
41
|
+
};
|
|
43
42
|
}
|
|
44
43
|
it('indexes and searches context evidence using a package-owned local embedding provider', async () => {
|
|
45
44
|
const store = new SqliteContextEvidenceStore({ dbPath });
|
|
@@ -143,8 +143,8 @@ export declare const memoryFlowActionDetailSchema: z.ZodObject<{
|
|
|
143
143
|
}>;
|
|
144
144
|
action: z.ZodEnum<{
|
|
145
145
|
created: "created";
|
|
146
|
-
removed: "removed";
|
|
147
146
|
updated: "updated";
|
|
147
|
+
removed: "removed";
|
|
148
148
|
}>;
|
|
149
149
|
key: z.ZodString;
|
|
150
150
|
summary: z.ZodString;
|
|
@@ -163,8 +163,8 @@ export declare const memoryFlowDetailSectionsSchema: z.ZodObject<{
|
|
|
163
163
|
}>;
|
|
164
164
|
action: z.ZodEnum<{
|
|
165
165
|
created: "created";
|
|
166
|
-
removed: "removed";
|
|
167
166
|
updated: "updated";
|
|
167
|
+
removed: "removed";
|
|
168
168
|
}>;
|
|
169
169
|
key: z.ZodString;
|
|
170
170
|
summary: z.ZodString;
|
|
@@ -103,8 +103,8 @@ export declare const ingestReportSnapshotSchema: z.ZodObject<{
|
|
|
103
103
|
}>;
|
|
104
104
|
type: z.ZodEnum<{
|
|
105
105
|
created: "created";
|
|
106
|
-
removed: "removed";
|
|
107
106
|
updated: "updated";
|
|
107
|
+
removed: "removed";
|
|
108
108
|
}>;
|
|
109
109
|
key: z.ZodString;
|
|
110
110
|
detail: z.ZodString;
|
|
@@ -129,8 +129,8 @@ export declare const ingestReportSnapshotSchema: z.ZodObject<{
|
|
|
129
129
|
}>;
|
|
130
130
|
type: z.ZodEnum<{
|
|
131
131
|
created: "created";
|
|
132
|
-
removed: "removed";
|
|
133
132
|
updated: "updated";
|
|
133
|
+
removed: "removed";
|
|
134
134
|
}>;
|
|
135
135
|
key: z.ZodString;
|
|
136
136
|
detail: z.ZodString;
|
|
@@ -130,26 +130,13 @@ export function resolveLocalKtxEmbeddingConfig(config, env) {
|
|
|
130
130
|
}
|
|
131
131
|
return {
|
|
132
132
|
backend: config.backend,
|
|
133
|
-
model: config.model ?? '
|
|
133
|
+
model: config.model ?? 'text-embedding-3-small',
|
|
134
134
|
dimensions: config.dimensions,
|
|
135
135
|
openai,
|
|
136
136
|
batchSize: config.batchSize,
|
|
137
137
|
};
|
|
138
138
|
}
|
|
139
|
-
|
|
140
|
-
backend: config.backend,
|
|
141
|
-
model: config.model ?? 'deterministic',
|
|
142
|
-
dimensions: config.dimensions,
|
|
143
|
-
...(config.sentenceTransformers
|
|
144
|
-
? {
|
|
145
|
-
sentenceTransformers: {
|
|
146
|
-
baseURL: config.sentenceTransformers.base_url,
|
|
147
|
-
pathPrefix: config.sentenceTransformers.pathPrefix,
|
|
148
|
-
},
|
|
149
|
-
}
|
|
150
|
-
: {}),
|
|
151
|
-
batchSize: config.batchSize,
|
|
152
|
-
};
|
|
139
|
+
throw new Error(`Unsupported KTX embedding backend: ${String(config.backend)}`);
|
|
153
140
|
}
|
|
154
141
|
export function createLocalKtxEmbeddingProviderFromConfig(config, deps = {}) {
|
|
155
142
|
const resolved = resolveLocalKtxEmbeddingConfig(config, deps.env ?? process.env);
|
|
@@ -173,15 +173,11 @@ describe('local KTX embedding config', () => {
|
|
|
173
173
|
batchSize: undefined,
|
|
174
174
|
});
|
|
175
175
|
});
|
|
176
|
-
it('
|
|
176
|
+
it('returns null for the default disabled project embedding config', () => {
|
|
177
177
|
const createKtxEmbeddingProvider = vi.fn(() => ({}));
|
|
178
178
|
const provider = createLocalKtxEmbeddingProviderFromConfig(buildDefaultKtxProjectConfig().ingest.embeddings, { createKtxEmbeddingProvider });
|
|
179
|
-
expect(provider).
|
|
180
|
-
expect(createKtxEmbeddingProvider).
|
|
181
|
-
backend: 'deterministic',
|
|
182
|
-
model: 'deterministic',
|
|
183
|
-
dimensions: 8,
|
|
184
|
-
}));
|
|
179
|
+
expect(provider).toBeNull();
|
|
180
|
+
expect(createKtxEmbeddingProvider).not.toHaveBeenCalled();
|
|
185
181
|
});
|
|
186
182
|
it('returns null when embeddings are disabled', () => {
|
|
187
183
|
expect(createLocalKtxEmbeddingProviderFromConfig({ backend: 'none', dimensions: 8 })).toBeNull();
|
|
@@ -128,9 +128,15 @@ class LocalKnowledgeIndex {
|
|
|
128
128
|
async getExistingSearchTexts() {
|
|
129
129
|
return new Map();
|
|
130
130
|
}
|
|
131
|
-
async deleteStale() {
|
|
132
|
-
|
|
133
|
-
|
|
131
|
+
async deleteStale() {
|
|
132
|
+
return 0;
|
|
133
|
+
}
|
|
134
|
+
async deleteByScope() {
|
|
135
|
+
return 0;
|
|
136
|
+
}
|
|
137
|
+
async deleteByKey() {
|
|
138
|
+
return 0;
|
|
139
|
+
}
|
|
134
140
|
async findPageByKey(scope, scopeId, pageKey) {
|
|
135
141
|
const path = this.pagePath(scope, scopeId, pageKey);
|
|
136
142
|
try {
|