@kaelio/ktx 0.1.1 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/assets/python/{kaelio_ktx-0.1.1-py3-none-any.whl → kaelio_ktx-0.2.0-py3-none-any.whl} +0 -0
- package/assets/python/manifest.json +4 -4
- package/dist/admin-reindex.d.ts +15 -0
- package/dist/admin-reindex.js +168 -0
- package/dist/admin-reindex.test.js +116 -0
- package/dist/{dev.d.ts → admin.d.ts} +1 -1
- package/dist/{dev.js → admin.js} +14 -12
- package/dist/admin.test.d.ts +1 -0
- package/dist/{dev.test.js → admin.test.js} +36 -31
- package/dist/cli-program.js +7 -7
- package/dist/cli-program.test.js +1 -1
- package/dist/cli-runtime.d.ts +2 -0
- package/dist/commands/connection-commands.js +11 -10
- package/dist/commands/connection-selection.d.ts +11 -0
- package/dist/commands/connection-selection.js +9 -0
- package/dist/commands/ingest-commands.js +32 -26
- package/dist/commands/knowledge-commands.js +17 -28
- package/dist/commands/mcp-commands.js +17 -11
- package/dist/commands/sl-commands.js +27 -32
- package/dist/doctor.test.js +4 -4
- package/dist/example-smoke.test.js +3 -3
- package/dist/index.test.js +76 -60
- package/dist/io/print-list.test.js +4 -4
- package/dist/knowledge.js +1 -1
- package/dist/managed-python-command.js +2 -2
- package/dist/managed-python-command.test.js +3 -3
- package/dist/managed-python-runtime.d.ts +1 -1
- package/dist/managed-python-runtime.js +3 -3
- package/dist/managed-python-runtime.test.js +2 -2
- package/dist/memory-flow-tui.test.js +2 -2
- package/dist/next-steps.d.ts +6 -6
- package/dist/next-steps.js +4 -4
- package/dist/next-steps.test.js +5 -5
- package/dist/print-command-tree.test.js +1 -1
- package/dist/public-ingest.js +3 -5
- package/dist/public-ingest.test.js +7 -3
- package/dist/runtime.test.js +1 -1
- package/dist/scan.test.js +2 -2
- package/dist/setup-agents.js +3 -3
- package/dist/setup-agents.test.js +1 -1
- package/dist/setup-embeddings.js +1 -1
- package/dist/setup-embeddings.test.js +3 -3
- package/dist/setup-runtime.test.js +2 -2
- package/dist/sl.js +1 -1
- package/dist/standalone-smoke.test.js +6 -2
- package/node_modules/@ktx/context/dist/index-sync/index.d.ts +2 -0
- package/node_modules/@ktx/context/dist/index-sync/index.js +1 -0
- package/node_modules/@ktx/context/dist/index-sync/reindex.d.ts +20 -0
- package/node_modules/@ktx/context/dist/index-sync/reindex.js +141 -0
- package/node_modules/@ktx/context/dist/index-sync/reindex.test.d.ts +1 -0
- package/node_modules/@ktx/context/dist/index-sync/reindex.test.js +139 -0
- package/node_modules/@ktx/context/dist/index-sync/types.d.ts +29 -0
- package/node_modules/@ktx/context/dist/index-sync/types.js +1 -0
- package/node_modules/@ktx/context/dist/index.d.ts +1 -0
- package/node_modules/@ktx/context/dist/index.js +1 -0
- package/node_modules/@ktx/context/dist/ingest/local-bundle-runtime.js +3 -0
- package/node_modules/@ktx/context/dist/ingest/memory-flow/schema.d.ts +2 -2
- package/node_modules/@ktx/context/dist/ingest/report-snapshot.d.ts +2 -2
- package/node_modules/@ktx/context/dist/memory/local-memory.js +9 -3
- package/node_modules/@ktx/context/dist/sl/ports.d.ts +3 -3
- package/node_modules/@ktx/context/dist/sl/sl-search.service.d.ts +3 -2
- package/node_modules/@ktx/context/dist/sl/sl-search.service.js +47 -45
- package/node_modules/@ktx/context/dist/sl/sl-search.service.test.js +61 -0
- package/node_modules/@ktx/context/dist/sl/sqlite-sl-sources-index.d.ts +4 -3
- package/node_modules/@ktx/context/dist/sl/sqlite-sl-sources-index.js +15 -5
- package/node_modules/@ktx/context/dist/sl/sqlite-sl-sources-index.test.js +24 -0
- package/node_modules/@ktx/context/dist/wiki/knowledge-wiki.service.d.ts +3 -2
- package/node_modules/@ktx/context/dist/wiki/knowledge-wiki.service.js +62 -51
- package/node_modules/@ktx/context/dist/wiki/knowledge-wiki.service.test.js +59 -3
- package/node_modules/@ktx/context/dist/wiki/ports.d.ts +3 -3
- package/node_modules/@ktx/context/dist/wiki/sqlite-knowledge-index.d.ts +33 -0
- package/node_modules/@ktx/context/dist/wiki/sqlite-knowledge-index.js +155 -2
- package/node_modules/@ktx/context/dist/wiki/sqlite-knowledge-index.test.js +26 -0
- package/node_modules/@ktx/context/package.json +5 -0
- package/package.json +1 -1
- /package/dist/{dev.test.d.ts → admin-reindex.test.d.ts} +0 -0
|
@@ -0,0 +1,139 @@
|
|
|
1
|
+
import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises';
|
|
2
|
+
import { tmpdir } from 'node:os';
|
|
3
|
+
import { join } from 'node:path';
|
|
4
|
+
import { afterEach, beforeEach, describe, expect, it } from 'vitest';
|
|
5
|
+
import { initKtxProject, loadKtxProject } from '../project/index.js';
|
|
6
|
+
import { SqliteKnowledgeIndex } from '../wiki/sqlite-knowledge-index.js';
|
|
7
|
+
import { reindexLocalIndexes } from './reindex.js';
|
|
8
|
+
class FakeEmbeddingPort {
|
|
9
|
+
maxBatchSize = 8;
|
|
10
|
+
async computeEmbedding(text) {
|
|
11
|
+
return [text.length, 1];
|
|
12
|
+
}
|
|
13
|
+
async computeEmbeddingsBulk(texts) {
|
|
14
|
+
return texts.map((text) => [text.length, 1]);
|
|
15
|
+
}
|
|
16
|
+
}
|
|
17
|
+
async function createProject(tempDir) {
|
|
18
|
+
await initKtxProject({ projectDir: tempDir, force: true });
|
|
19
|
+
return loadKtxProject({ projectDir: tempDir });
|
|
20
|
+
}
|
|
21
|
+
describe('reindexLocalIndexes', () => {
|
|
22
|
+
let tempDir;
|
|
23
|
+
beforeEach(async () => {
|
|
24
|
+
tempDir = await mkdtemp(join(tmpdir(), 'ktx-reindex-'));
|
|
25
|
+
});
|
|
26
|
+
afterEach(async () => {
|
|
27
|
+
await rm(tempDir, { recursive: true, force: true });
|
|
28
|
+
});
|
|
29
|
+
it('returns an empty summary when no wiki or semantic-layer directories exist', async () => {
|
|
30
|
+
const project = await createProject(tempDir);
|
|
31
|
+
await rm(join(project.projectDir, 'wiki'), { recursive: true, force: true });
|
|
32
|
+
await rm(join(project.projectDir, 'semantic-layer'), { recursive: true, force: true });
|
|
33
|
+
await expect(reindexLocalIndexes(project, { force: false, embeddingService: null })).resolves.toMatchObject({
|
|
34
|
+
scopes: [],
|
|
35
|
+
totals: { scanned: 0, updated: 0, deleted: 0, embeddingsRecomputed: 0, embeddingsFailed: 0 },
|
|
36
|
+
force: false,
|
|
37
|
+
embeddingsAvailable: false,
|
|
38
|
+
});
|
|
39
|
+
});
|
|
40
|
+
it('discovers empty directories as zero-row scopes', async () => {
|
|
41
|
+
const project = await createProject(tempDir);
|
|
42
|
+
await mkdir(join(project.projectDir, 'wiki/user/local'), { recursive: true });
|
|
43
|
+
await mkdir(join(project.projectDir, 'semantic-layer/warehouse'), { recursive: true });
|
|
44
|
+
const summary = await reindexLocalIndexes(project, { force: false, embeddingService: null });
|
|
45
|
+
expect(summary.scopes.map((scope) => scope.label)).toEqual(['global', 'user/local', 'warehouse']);
|
|
46
|
+
expect(summary.totals.scanned).toBe(0);
|
|
47
|
+
});
|
|
48
|
+
it('indexes mixed wiki and SL sources and reports totals', async () => {
|
|
49
|
+
const project = await createProject(tempDir);
|
|
50
|
+
await writeFile(join(project.projectDir, 'wiki/global/revenue.md'), '---\nsummary: Revenue\nusage_mode: auto\n---\n\nPaid orders.\n', 'utf-8');
|
|
51
|
+
await mkdir(join(project.projectDir, 'semantic-layer/warehouse'), { recursive: true });
|
|
52
|
+
await writeFile(join(project.projectDir, 'semantic-layer/warehouse/orders.yaml'), 'name: orders\ntable: public.orders\ngrain: [id]\ncolumns:\n - name: id\n type: number\njoins: []\nmeasures: []\n', 'utf-8');
|
|
53
|
+
const summary = await reindexLocalIndexes(project, {
|
|
54
|
+
force: false,
|
|
55
|
+
embeddingService: new FakeEmbeddingPort(),
|
|
56
|
+
});
|
|
57
|
+
expect(summary.scopes).toHaveLength(2);
|
|
58
|
+
expect(summary.totals).toMatchObject({ scanned: 2, updated: 2, deleted: 0, embeddingsRecomputed: 2 });
|
|
59
|
+
expect(summary.embeddingsAvailable).toBe(true);
|
|
60
|
+
});
|
|
61
|
+
it('does not report unchanged lexical-only rows as updated on repeated runs', async () => {
|
|
62
|
+
const project = await createProject(tempDir);
|
|
63
|
+
await writeFile(join(project.projectDir, 'wiki/global/revenue.md'), '---\nsummary: Revenue\nusage_mode: auto\n---\n\nPaid orders.\n', 'utf-8');
|
|
64
|
+
await mkdir(join(project.projectDir, 'semantic-layer/warehouse'), { recursive: true });
|
|
65
|
+
await writeFile(join(project.projectDir, 'semantic-layer/warehouse/orders.yaml'), 'name: orders\ntable: public.orders\ngrain: [id]\ncolumns:\n - name: id\n type: number\njoins: []\nmeasures: []\n', 'utf-8');
|
|
66
|
+
const first = await reindexLocalIndexes(project, { force: false, embeddingService: null });
|
|
67
|
+
expect(first.totals).toMatchObject({
|
|
68
|
+
scanned: 2,
|
|
69
|
+
updated: 2,
|
|
70
|
+
deleted: 0,
|
|
71
|
+
embeddingsRecomputed: 0,
|
|
72
|
+
embeddingsFailed: 0,
|
|
73
|
+
});
|
|
74
|
+
const second = await reindexLocalIndexes(project, { force: false, embeddingService: null });
|
|
75
|
+
expect(second.totals).toMatchObject({
|
|
76
|
+
scanned: 2,
|
|
77
|
+
updated: 0,
|
|
78
|
+
deleted: 0,
|
|
79
|
+
embeddingsRecomputed: 0,
|
|
80
|
+
embeddingsFailed: 0,
|
|
81
|
+
});
|
|
82
|
+
expect(second.scopes.map((scope) => [scope.label, scope.updated])).toEqual([
|
|
83
|
+
['global', 0],
|
|
84
|
+
['warehouse', 0],
|
|
85
|
+
]);
|
|
86
|
+
});
|
|
87
|
+
it('force clears stale rows before rebuilding each discovered scope', async () => {
|
|
88
|
+
const project = await createProject(tempDir);
|
|
89
|
+
const wikiIndex = new SqliteKnowledgeIndex({ dbPath: join(project.projectDir, '.ktx/db.sqlite') });
|
|
90
|
+
wikiIndex.sync([
|
|
91
|
+
{
|
|
92
|
+
path: 'wiki/global/stale.md',
|
|
93
|
+
key: 'stale',
|
|
94
|
+
scope: 'GLOBAL',
|
|
95
|
+
scopeId: null,
|
|
96
|
+
summary: 'Stale',
|
|
97
|
+
content: 'Stale content',
|
|
98
|
+
tags: [],
|
|
99
|
+
embedding: [1, 0],
|
|
100
|
+
},
|
|
101
|
+
]);
|
|
102
|
+
await writeFile(join(project.projectDir, 'wiki/global/revenue.md'), '---\nsummary: Revenue\nusage_mode: auto\n---\n\nPaid orders.\n', 'utf-8');
|
|
103
|
+
const summary = await reindexLocalIndexes(project, {
|
|
104
|
+
force: true,
|
|
105
|
+
embeddingService: new FakeEmbeddingPort(),
|
|
106
|
+
});
|
|
107
|
+
expect(summary.force).toBe(true);
|
|
108
|
+
expect(summary.totals).toMatchObject({ scanned: 1, updated: 1, deleted: 0 });
|
|
109
|
+
expect(wikiIndex.search('Stale', 10)).toEqual([]);
|
|
110
|
+
});
|
|
111
|
+
it('captures a per-scope error and continues other scopes', async () => {
|
|
112
|
+
const project = await createProject(tempDir);
|
|
113
|
+
await writeFile(join(project.projectDir, 'wiki/global/revenue.md'), '---\nsummary: Revenue\nusage_mode: auto\n---\n\nPaid orders.\n', 'utf-8');
|
|
114
|
+
await mkdir(join(project.projectDir, 'semantic-layer/warehouse'), { recursive: true });
|
|
115
|
+
await writeFile(join(project.projectDir, 'semantic-layer/warehouse/broken.yaml'), 'not: [valid', 'utf-8');
|
|
116
|
+
const summary = await reindexLocalIndexes(project, { force: false, embeddingService: null });
|
|
117
|
+
expect(summary.scopes.find((scope) => scope.label === 'global')?.error).toBeUndefined();
|
|
118
|
+
expect(summary.scopes.find((scope) => scope.label === 'warehouse')?.error).toContain('YAML');
|
|
119
|
+
});
|
|
120
|
+
it('marks a scope errored when configured embeddings fail', async () => {
|
|
121
|
+
const project = await createProject(tempDir);
|
|
122
|
+
await writeFile(join(project.projectDir, 'wiki/global/revenue.md'), '---\nsummary: Revenue\nusage_mode: auto\n---\n\nPaid orders.\n', 'utf-8');
|
|
123
|
+
const embeddingService = {
|
|
124
|
+
maxBatchSize: 8,
|
|
125
|
+
async computeEmbedding() {
|
|
126
|
+
throw new Error('embedding provider unavailable');
|
|
127
|
+
},
|
|
128
|
+
async computeEmbeddingsBulk() {
|
|
129
|
+
throw new Error('embedding provider unavailable');
|
|
130
|
+
},
|
|
131
|
+
};
|
|
132
|
+
const summary = await reindexLocalIndexes(project, { force: false, embeddingService });
|
|
133
|
+
expect(summary.scopes[0]).toMatchObject({
|
|
134
|
+
label: 'global',
|
|
135
|
+
embeddingsFailed: 1,
|
|
136
|
+
error: '1 embedding recomputation failed',
|
|
137
|
+
});
|
|
138
|
+
});
|
|
139
|
+
});
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
import type { KtxEmbeddingPort } from '../core/index.js';
|
|
2
|
+
export interface ReindexOptions {
|
|
3
|
+
force: boolean;
|
|
4
|
+
embeddingService: KtxEmbeddingPort | null;
|
|
5
|
+
}
|
|
6
|
+
export interface ReindexWorkResult {
|
|
7
|
+
scanned: number;
|
|
8
|
+
updated: number;
|
|
9
|
+
deleted: number;
|
|
10
|
+
embeddingsRecomputed: number;
|
|
11
|
+
embeddingsFailed: number;
|
|
12
|
+
}
|
|
13
|
+
export interface ReindexScopeResult extends ReindexWorkResult {
|
|
14
|
+
kind: 'wiki' | 'sl';
|
|
15
|
+
label: string;
|
|
16
|
+
scope?: 'global' | 'user';
|
|
17
|
+
scopeId?: string | null;
|
|
18
|
+
connectionId?: string;
|
|
19
|
+
durationMs: number;
|
|
20
|
+
error?: string;
|
|
21
|
+
}
|
|
22
|
+
export interface ReindexSummary {
|
|
23
|
+
scopes: ReindexScopeResult[];
|
|
24
|
+
totals: ReindexWorkResult;
|
|
25
|
+
dbPath: string;
|
|
26
|
+
force: boolean;
|
|
27
|
+
embeddingsAvailable: boolean;
|
|
28
|
+
durationMs: number;
|
|
29
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -7,6 +7,7 @@ export * from './agent/index.js';
|
|
|
7
7
|
export * from './core/index.js';
|
|
8
8
|
export * from './daemon/index.js';
|
|
9
9
|
export * from './ingest/index.js';
|
|
10
|
+
export * from './index-sync/index.js';
|
|
10
11
|
export * from './llm/index.js';
|
|
11
12
|
export type { CaptureSession, CaptureSignals, MemoryAgentInput, MemoryAgentResult, MemoryAgentServiceDeps, MemoryAgentSettings, MemoryAgentSourceType, MemoryCommitMessagePort, MemoryConnectionPort, MemoryFileStorePort, MemoryKnowledgeSlRefsPort, MemoryLockPort, MemorySlSourceReconcilerPort, MemoryTelemetryPort, MemoryToolSetLike, MemoryToolsetFactoryPort, } from './memory/index.js';
|
|
12
13
|
export * from './project/index.js';
|
|
@@ -6,6 +6,7 @@ export * from './agent/index.js';
|
|
|
6
6
|
export * from './core/index.js';
|
|
7
7
|
export * from './daemon/index.js';
|
|
8
8
|
export * from './ingest/index.js';
|
|
9
|
+
export * from './index-sync/index.js';
|
|
9
10
|
export * from './llm/index.js';
|
|
10
11
|
export * from './project/index.js';
|
|
11
12
|
export * from './prompts/index.js';
|
|
@@ -256,12 +256,15 @@ class LocalKnowledgeIndex {
|
|
|
256
256
|
}
|
|
257
257
|
async deleteStale() {
|
|
258
258
|
await this.syncAllPagesFromDisk();
|
|
259
|
+
return 0;
|
|
259
260
|
}
|
|
260
261
|
async deleteByScope() {
|
|
261
262
|
await this.syncAllPagesFromDisk();
|
|
263
|
+
return 0;
|
|
262
264
|
}
|
|
263
265
|
async deleteByKey() {
|
|
264
266
|
await this.syncAllPagesFromDisk();
|
|
267
|
+
return 0;
|
|
265
268
|
}
|
|
266
269
|
async findPageByKey(scope, scopeId, pageKey) {
|
|
267
270
|
const path = scope === 'GLOBAL' ? `wiki/global/${pageKey}.md` : `wiki/user/${scopeId}/${pageKey}.md`;
|
|
@@ -143,8 +143,8 @@ export declare const memoryFlowActionDetailSchema: z.ZodObject<{
|
|
|
143
143
|
}>;
|
|
144
144
|
action: z.ZodEnum<{
|
|
145
145
|
created: "created";
|
|
146
|
-
removed: "removed";
|
|
147
146
|
updated: "updated";
|
|
147
|
+
removed: "removed";
|
|
148
148
|
}>;
|
|
149
149
|
key: z.ZodString;
|
|
150
150
|
summary: z.ZodString;
|
|
@@ -163,8 +163,8 @@ export declare const memoryFlowDetailSectionsSchema: z.ZodObject<{
|
|
|
163
163
|
}>;
|
|
164
164
|
action: z.ZodEnum<{
|
|
165
165
|
created: "created";
|
|
166
|
-
removed: "removed";
|
|
167
166
|
updated: "updated";
|
|
167
|
+
removed: "removed";
|
|
168
168
|
}>;
|
|
169
169
|
key: z.ZodString;
|
|
170
170
|
summary: z.ZodString;
|
|
@@ -103,8 +103,8 @@ export declare const ingestReportSnapshotSchema: z.ZodObject<{
|
|
|
103
103
|
}>;
|
|
104
104
|
type: z.ZodEnum<{
|
|
105
105
|
created: "created";
|
|
106
|
-
removed: "removed";
|
|
107
106
|
updated: "updated";
|
|
107
|
+
removed: "removed";
|
|
108
108
|
}>;
|
|
109
109
|
key: z.ZodString;
|
|
110
110
|
detail: z.ZodString;
|
|
@@ -129,8 +129,8 @@ export declare const ingestReportSnapshotSchema: z.ZodObject<{
|
|
|
129
129
|
}>;
|
|
130
130
|
type: z.ZodEnum<{
|
|
131
131
|
created: "created";
|
|
132
|
-
removed: "removed";
|
|
133
132
|
updated: "updated";
|
|
133
|
+
removed: "removed";
|
|
134
134
|
}>;
|
|
135
135
|
key: z.ZodString;
|
|
136
136
|
detail: z.ZodString;
|
|
@@ -128,9 +128,15 @@ class LocalKnowledgeIndex {
|
|
|
128
128
|
async getExistingSearchTexts() {
|
|
129
129
|
return new Map();
|
|
130
130
|
}
|
|
131
|
-
async deleteStale() {
|
|
132
|
-
|
|
133
|
-
|
|
131
|
+
async deleteStale() {
|
|
132
|
+
return 0;
|
|
133
|
+
}
|
|
134
|
+
async deleteByScope() {
|
|
135
|
+
return 0;
|
|
136
|
+
}
|
|
137
|
+
async deleteByKey() {
|
|
138
|
+
return 0;
|
|
139
|
+
}
|
|
134
140
|
async findPageByKey(scope, scopeId, pageKey) {
|
|
135
141
|
const path = this.pagePath(scope, scopeId, pageKey);
|
|
136
142
|
try {
|
|
@@ -50,9 +50,9 @@ export interface SlSourcesIndexPort {
|
|
|
50
50
|
searchText: string;
|
|
51
51
|
hasEmbedding: boolean;
|
|
52
52
|
}>>;
|
|
53
|
-
deleteStale(connectionId: string, keepNames: string[]): Promise<
|
|
54
|
-
deleteByConnection(connectionId: string): Promise<
|
|
55
|
-
deleteByConnectionAndName(connectionId: string, sourceName: string): Promise<
|
|
53
|
+
deleteStale(connectionId: string, keepNames: string[]): Promise<number>;
|
|
54
|
+
deleteByConnection(connectionId: string): Promise<number>;
|
|
55
|
+
deleteByConnectionAndName(connectionId: string, sourceName: string): Promise<number>;
|
|
56
56
|
search(connectionId: string, queryEmbedding: number[] | null, queryText: string, limit: number, minRrfScore?: number): Promise<Array<{
|
|
57
57
|
sourceName: string;
|
|
58
58
|
rrfScore: number;
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import type { KtxEmbeddingPort, KtxLogger } from '../core/index.js';
|
|
2
|
+
import type { ReindexWorkResult } from '../index-sync/types.js';
|
|
2
3
|
import type { SlSourcesIndexPort } from './ports.js';
|
|
3
4
|
import type { SemanticLayerSource } from './types.js';
|
|
4
5
|
export declare function buildSemanticLayerSourceSearchText(source: SemanticLayerSource, priority?: string[]): string;
|
|
@@ -6,8 +7,8 @@ export declare class SlSearchService {
|
|
|
6
7
|
private readonly embeddingService;
|
|
7
8
|
private readonly slSourcesRepository;
|
|
8
9
|
private readonly logger;
|
|
9
|
-
constructor(embeddingService: KtxEmbeddingPort, slSourcesRepository: SlSourcesIndexPort, logger?: KtxLogger);
|
|
10
|
-
indexSources(connectionId: string, sources: SemanticLayerSource[]): Promise<
|
|
10
|
+
constructor(embeddingService: KtxEmbeddingPort | null, slSourcesRepository: SlSourcesIndexPort, logger?: KtxLogger);
|
|
11
|
+
indexSources(connectionId: string, sources: SemanticLayerSource[]): Promise<ReindexWorkResult>;
|
|
11
12
|
search(connectionId: string, query: string, limit?: number, minRrfScore?: number): Promise<Array<{
|
|
12
13
|
sourceName: string;
|
|
13
14
|
score: number;
|
|
@@ -84,64 +84,66 @@ export class SlSearchService {
|
|
|
84
84
|
this.logger = logger;
|
|
85
85
|
}
|
|
86
86
|
async indexSources(connectionId, sources) {
|
|
87
|
+
const existing = await this.slSourcesRepository.getExistingSearchTexts(connectionId);
|
|
87
88
|
if (sources.length === 0) {
|
|
88
|
-
await this.slSourcesRepository.deleteByConnection(connectionId);
|
|
89
|
-
return;
|
|
89
|
+
const deleted = await this.slSourcesRepository.deleteByConnection(connectionId);
|
|
90
|
+
return { scanned: 0, updated: 0, deleted, embeddingsRecomputed: 0, embeddingsFailed: 0 };
|
|
90
91
|
}
|
|
91
|
-
// Detect which sources actually changed by comparing search_text
|
|
92
|
-
const existing = await this.slSourcesRepository.getExistingSearchTexts(connectionId);
|
|
93
92
|
const searchTexts = sources.map((s) => this.buildSearchText(s));
|
|
93
|
+
const embeddingService = this.embeddingService;
|
|
94
94
|
const changedIndices = [];
|
|
95
|
-
for (let i = 0; i < sources.length; i
|
|
96
|
-
const
|
|
97
|
-
if (!
|
|
95
|
+
for (let i = 0; i < sources.length; i += 1) {
|
|
96
|
+
const previous = existing.get(sources[i].name);
|
|
97
|
+
if (!previous ||
|
|
98
|
+
previous.searchText !== searchTexts[i] ||
|
|
99
|
+
(embeddingService !== null && !previous.hasEmbedding)) {
|
|
98
100
|
changedIndices.push(i);
|
|
99
101
|
}
|
|
100
102
|
}
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
103
|
+
let changedEmbeddings = changedIndices.map(() => null);
|
|
104
|
+
let embeddingsRecomputed = 0;
|
|
105
|
+
let embeddingsFailed = 0;
|
|
106
|
+
if (embeddingService && changedIndices.length > 0) {
|
|
107
|
+
try {
|
|
108
|
+
const changedTexts = changedIndices.map((index) => searchTexts[index]);
|
|
109
|
+
const allEmbeddings = [];
|
|
110
|
+
for (let i = 0; i < changedTexts.length; i += embeddingService.maxBatchSize) {
|
|
111
|
+
const batch = changedTexts.slice(i, i + embeddingService.maxBatchSize);
|
|
112
|
+
allEmbeddings.push(...(await embeddingService.computeEmbeddingsBulk(batch)));
|
|
113
|
+
}
|
|
114
|
+
changedEmbeddings = allEmbeddings;
|
|
115
|
+
embeddingsRecomputed = allEmbeddings.length;
|
|
116
|
+
}
|
|
117
|
+
catch (error) {
|
|
118
|
+
this.logger.warn(`Failed to compute SL source embeddings: ${error instanceof Error ? error.message : String(error)}`);
|
|
119
|
+
embeddingsFailed = changedIndices.length;
|
|
118
120
|
}
|
|
119
|
-
changedEmbeddings = allEmbeddings;
|
|
120
|
-
}
|
|
121
|
-
catch (error) {
|
|
122
|
-
this.logger.warn(`Failed to compute SL source embeddings: ${error instanceof Error ? error.message : String(error)}`);
|
|
123
|
-
changedEmbeddings = changedIndices.map(() => null);
|
|
124
121
|
}
|
|
125
|
-
const rows = changedIndices.map((
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
};
|
|
131
|
-
});
|
|
122
|
+
const rows = changedIndices.map((sourceIndex, embeddingIndex) => ({
|
|
123
|
+
sourceName: sources[sourceIndex].name,
|
|
124
|
+
searchText: searchTexts[sourceIndex],
|
|
125
|
+
embedding: changedEmbeddings[embeddingIndex] ?? null,
|
|
126
|
+
}));
|
|
132
127
|
await this.slSourcesRepository.upsertSources(connectionId, rows);
|
|
133
|
-
|
|
134
|
-
const
|
|
135
|
-
|
|
136
|
-
|
|
128
|
+
const keepNames = sources.map((source) => source.name);
|
|
129
|
+
const deleted = await this.slSourcesRepository.deleteStale(connectionId, keepNames);
|
|
130
|
+
return {
|
|
131
|
+
scanned: sources.length,
|
|
132
|
+
updated: changedIndices.length,
|
|
133
|
+
deleted,
|
|
134
|
+
embeddingsRecomputed,
|
|
135
|
+
embeddingsFailed,
|
|
136
|
+
};
|
|
137
137
|
}
|
|
138
138
|
async search(connectionId, query, limit = 15, minRrfScore = 0) {
|
|
139
139
|
let queryEmbedding = null;
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
140
|
+
if (this.embeddingService) {
|
|
141
|
+
try {
|
|
142
|
+
queryEmbedding = await this.embeddingService.computeEmbedding(query);
|
|
143
|
+
}
|
|
144
|
+
catch (error) {
|
|
145
|
+
this.logger.warn(`Failed to compute query embedding, falling back to FTS + trigram: ${error instanceof Error ? error.message : String(error)}`);
|
|
146
|
+
}
|
|
145
147
|
}
|
|
146
148
|
const results = await this.slSourcesRepository.search(connectionId, queryEmbedding, query, limit, minRrfScore);
|
|
147
149
|
return results.map((result) => ({
|
|
@@ -192,4 +192,65 @@ describe('SlSearchService', () => {
|
|
|
192
192
|
},
|
|
193
193
|
]);
|
|
194
194
|
});
|
|
195
|
+
it('indexSources reports stats and supports lexical-only indexing', async () => {
|
|
196
|
+
const repository = {
|
|
197
|
+
upsertSources: vi.fn().mockResolvedValue(undefined),
|
|
198
|
+
getExistingSearchTexts: vi.fn().mockResolvedValue(new Map([
|
|
199
|
+
['old_source', { searchText: 'old source', hasEmbedding: true }],
|
|
200
|
+
])),
|
|
201
|
+
deleteStale: vi.fn().mockResolvedValue(1),
|
|
202
|
+
deleteByConnection: vi.fn().mockResolvedValue(0),
|
|
203
|
+
deleteByConnectionAndName: vi.fn(),
|
|
204
|
+
search: vi.fn(),
|
|
205
|
+
};
|
|
206
|
+
const service = new SlSearchService(null, repository);
|
|
207
|
+
const source = {
|
|
208
|
+
name: 'orders',
|
|
209
|
+
table: 'public.orders',
|
|
210
|
+
grain: ['id'],
|
|
211
|
+
columns: [{ name: 'id', type: 'number' }],
|
|
212
|
+
joins: [],
|
|
213
|
+
measures: [],
|
|
214
|
+
};
|
|
215
|
+
await expect(service.indexSources('warehouse', [source])).resolves.toEqual({
|
|
216
|
+
scanned: 1,
|
|
217
|
+
updated: 1,
|
|
218
|
+
deleted: 1,
|
|
219
|
+
embeddingsRecomputed: 0,
|
|
220
|
+
embeddingsFailed: 0,
|
|
221
|
+
});
|
|
222
|
+
expect(repository.upsertSources).toHaveBeenCalledWith('warehouse', [
|
|
223
|
+
expect.objectContaining({ sourceName: 'orders', embedding: null }),
|
|
224
|
+
]);
|
|
225
|
+
});
|
|
226
|
+
it('does not update unchanged lexical-only SL rows on repeated sync', async () => {
|
|
227
|
+
const repository = {
|
|
228
|
+
upsertSources: vi.fn().mockResolvedValue(undefined),
|
|
229
|
+
getExistingSearchTexts: vi.fn().mockResolvedValue(new Map([
|
|
230
|
+
['orders', { searchText: 'orders. table: public.orders. id (number)', hasEmbedding: false }],
|
|
231
|
+
])),
|
|
232
|
+
deleteStale: vi.fn().mockResolvedValue(0),
|
|
233
|
+
deleteByConnection: vi.fn().mockResolvedValue(0),
|
|
234
|
+
deleteByConnectionAndName: vi.fn(),
|
|
235
|
+
search: vi.fn(),
|
|
236
|
+
};
|
|
237
|
+
const service = new SlSearchService(null, repository);
|
|
238
|
+
const source = {
|
|
239
|
+
name: 'orders',
|
|
240
|
+
table: 'public.orders',
|
|
241
|
+
grain: ['id'],
|
|
242
|
+
columns: [{ name: 'id', type: 'number' }],
|
|
243
|
+
joins: [],
|
|
244
|
+
measures: [],
|
|
245
|
+
};
|
|
246
|
+
await expect(service.indexSources('warehouse', [source])).resolves.toEqual({
|
|
247
|
+
scanned: 1,
|
|
248
|
+
updated: 0,
|
|
249
|
+
deleted: 0,
|
|
250
|
+
embeddingsRecomputed: 0,
|
|
251
|
+
embeddingsFailed: 0,
|
|
252
|
+
});
|
|
253
|
+
expect(repository.upsertSources).toHaveBeenCalledWith('warehouse', []);
|
|
254
|
+
expect(repository.deleteStale).toHaveBeenCalledWith('warehouse', ['orders']);
|
|
255
|
+
});
|
|
195
256
|
});
|
|
@@ -28,9 +28,10 @@ export declare class SqliteSlSourcesIndex implements SlSourcesIndexPort {
|
|
|
28
28
|
searchText: string;
|
|
29
29
|
hasEmbedding: boolean;
|
|
30
30
|
}>>;
|
|
31
|
-
deleteStale(connectionId: string, keepNames: string[]): Promise<
|
|
32
|
-
deleteByConnection(connectionId: string): Promise<
|
|
33
|
-
|
|
31
|
+
deleteStale(connectionId: string, keepNames: string[]): Promise<number>;
|
|
32
|
+
deleteByConnection(connectionId: string): Promise<number>;
|
|
33
|
+
clear(connectionId: string): Promise<number>;
|
|
34
|
+
deleteByConnectionAndName(connectionId: string, sourceName: string): Promise<number>;
|
|
34
35
|
replaceDictionaryEntries(connectionId: string, entries: SlDictionaryEntry[]): Promise<void>;
|
|
35
36
|
searchLexicalCandidates(input: {
|
|
36
37
|
connectionIds?: readonly string[];
|
|
@@ -144,8 +144,7 @@ export class SqliteSlSourcesIndex {
|
|
|
144
144
|
}
|
|
145
145
|
async deleteStale(connectionId, keepNames) {
|
|
146
146
|
if (keepNames.length === 0) {
|
|
147
|
-
|
|
148
|
-
return;
|
|
147
|
+
return this.deleteByConnection(connectionId);
|
|
149
148
|
}
|
|
150
149
|
const placeholders = keepNames.map(() => '?').join(', ');
|
|
151
150
|
const stale = this.db
|
|
@@ -173,16 +172,26 @@ export class SqliteSlSourcesIndex {
|
|
|
173
172
|
}
|
|
174
173
|
});
|
|
175
174
|
remove(stale.map((row) => row.source_name));
|
|
175
|
+
return stale.length;
|
|
176
176
|
}
|
|
177
177
|
async deleteByConnection(connectionId) {
|
|
178
|
+
return this.clear(connectionId);
|
|
179
|
+
}
|
|
180
|
+
async clear(connectionId) {
|
|
181
|
+
const rows = this.db
|
|
182
|
+
.prepare('SELECT source_name FROM local_sl_sources WHERE connection_id = ?')
|
|
183
|
+
.all(connectionId);
|
|
178
184
|
const remove = this.db.transaction(() => {
|
|
179
185
|
this.db.prepare('DELETE FROM local_sl_sources_fts WHERE connection_id = ?').run(connectionId);
|
|
180
186
|
this.db.prepare('DELETE FROM local_sl_sources WHERE connection_id = ?').run(connectionId);
|
|
187
|
+
this.db.prepare('DELETE FROM local_sl_dictionary_values_fts WHERE connection_id = ?').run(connectionId);
|
|
188
|
+
this.db.prepare('DELETE FROM local_sl_dictionary_values WHERE connection_id = ?').run(connectionId);
|
|
181
189
|
});
|
|
182
190
|
remove();
|
|
191
|
+
return rows.length;
|
|
183
192
|
}
|
|
184
193
|
async deleteByConnectionAndName(connectionId, sourceName) {
|
|
185
|
-
this.deleteByConnectionAndNameSync(connectionId, sourceName);
|
|
194
|
+
return this.deleteByConnectionAndNameSync(connectionId, sourceName);
|
|
186
195
|
}
|
|
187
196
|
async replaceDictionaryEntries(connectionId, entries) {
|
|
188
197
|
const remove = this.db.transaction(() => {
|
|
@@ -406,14 +415,15 @@ export class SqliteSlSourcesIndex {
|
|
|
406
415
|
AND source_name = ?
|
|
407
416
|
`)
|
|
408
417
|
.run(connectionId, sourceName);
|
|
409
|
-
this.db
|
|
418
|
+
const result = this.db
|
|
410
419
|
.prepare(`
|
|
411
420
|
DELETE FROM local_sl_sources
|
|
412
421
|
WHERE connection_id = ?
|
|
413
422
|
AND source_name = ?
|
|
414
423
|
`)
|
|
415
424
|
.run(connectionId, sourceName);
|
|
425
|
+
return Number(result.changes);
|
|
416
426
|
});
|
|
417
|
-
remove();
|
|
427
|
+
return remove();
|
|
418
428
|
}
|
|
419
429
|
}
|
|
@@ -86,6 +86,30 @@ describe('SqliteSlSourcesIndex', () => {
|
|
|
86
86
|
await index.deleteByConnection('finance');
|
|
87
87
|
expect(await index.search('finance', null, 'revenue', 10)).toEqual([]);
|
|
88
88
|
});
|
|
89
|
+
it('clear removes sources and dictionary rows for one connection only', async () => {
|
|
90
|
+
const index = new SqliteSlSourcesIndex({ dbPath });
|
|
91
|
+
await index.upsertSources('warehouse', [
|
|
92
|
+
{ sourceName: 'orders', searchText: 'orders revenue paid', embedding: null },
|
|
93
|
+
]);
|
|
94
|
+
await index.upsertSources('finance', [
|
|
95
|
+
{ sourceName: 'invoices', searchText: 'invoices revenue paid', embedding: null },
|
|
96
|
+
]);
|
|
97
|
+
await index.replaceDictionaryEntries('warehouse', [
|
|
98
|
+
{ connectionId: 'warehouse', sourceName: 'orders', columnName: 'status', value: 'paid', cardinality: 1 },
|
|
99
|
+
]);
|
|
100
|
+
await index.replaceDictionaryEntries('finance', [
|
|
101
|
+
{ connectionId: 'finance', sourceName: 'invoices', columnName: 'status', value: 'paid', cardinality: 1 },
|
|
102
|
+
]);
|
|
103
|
+
await expect(index.clear('warehouse')).resolves.toBe(1);
|
|
104
|
+
expect(await index.search('warehouse', null, 'revenue', 10)).toEqual([]);
|
|
105
|
+
expect(await index.search('finance', null, 'revenue', 10)).toEqual([
|
|
106
|
+
expect.objectContaining({ sourceName: 'invoices' }),
|
|
107
|
+
]);
|
|
108
|
+
await expect(index.searchDictionaryCandidates({ connectionIds: ['warehouse'], queryText: 'paid', limit: 10 }))
|
|
109
|
+
.resolves.toEqual([]);
|
|
110
|
+
await expect(index.searchDictionaryCandidates({ connectionIds: ['finance'], queryText: 'paid', limit: 10 }))
|
|
111
|
+
.resolves.toEqual([expect.objectContaining({ connectionId: 'finance', sourceName: 'invoices' })]);
|
|
112
|
+
});
|
|
89
113
|
it('returns lane candidates with stable connection-scoped IDs', async () => {
|
|
90
114
|
const index = new SqliteSlSourcesIndex({ dbPath });
|
|
91
115
|
await index.upsertSources('warehouse', [
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import type { KtxEmbeddingPort, KtxFileStorePort, KtxLogger } from '../core/index.js';
|
|
2
|
+
import type { ReindexWorkResult } from '../index-sync/types.js';
|
|
2
3
|
import type { KnowledgeGitDiffPort, KnowledgeIndexPort } from './ports.js';
|
|
3
4
|
import type { WikiFrontmatter, WikiPage, WikiPageWithScope } from './types.js';
|
|
4
5
|
export type { WikiFrontmatter };
|
|
@@ -9,7 +10,7 @@ export declare class KnowledgeWikiService {
|
|
|
9
10
|
private readonly gitService;
|
|
10
11
|
private readonly logger;
|
|
11
12
|
private isWorktreeScoped;
|
|
12
|
-
constructor(configService: KtxFileStorePort, embeddingService: KtxEmbeddingPort, pagesRepository: KnowledgeIndexPort, gitService: KnowledgeGitDiffPort, logger?: KtxLogger);
|
|
13
|
+
constructor(configService: KtxFileStorePort, embeddingService: KtxEmbeddingPort | null, pagesRepository: KnowledgeIndexPort, gitService: KnowledgeGitDiffPort, logger?: KtxLogger);
|
|
13
14
|
/**
|
|
14
15
|
* Return a clone of this service whose disk writes go through a worktree-scoped
|
|
15
16
|
* ConfigService AND whose DB-index writes are no-ops. Used by memory-agent
|
|
@@ -56,7 +57,7 @@ export declare class KnowledgeWikiService {
|
|
|
56
57
|
* Full sync: load all pages from disk for a scope, reindex changed pages, clean stale entries.
|
|
57
58
|
* Mirrors SlSearchService.indexSources() pattern.
|
|
58
59
|
*/
|
|
59
|
-
syncIndex(scope: string, scopeId?: string | null): Promise<
|
|
60
|
+
syncIndex(scope: string, scopeId?: string | null): Promise<ReindexWorkResult>;
|
|
60
61
|
/**
|
|
61
62
|
* Delete a page from the DB index (after file deletion).
|
|
62
63
|
*/
|