@askdkc/kiokuko 0.2.7 → 0.2.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.ja.md +17 -22
- package/README.ko.md +16 -22
- package/README.md +23 -23
- package/README.zh-CN.md +14 -20
- package/dist/akinator/agent-task.d.ts +2 -0
- package/dist/akinator/agent-task.d.ts.map +1 -1
- package/dist/akinator/agent-task.js +62 -26
- package/dist/akinator/agent-task.js.map +1 -1
- package/dist/akinator/domain.d.ts.map +1 -1
- package/dist/akinator/domain.js +36 -19
- package/dist/akinator/domain.js.map +1 -1
- package/dist/cli.d.ts.map +1 -1
- package/dist/cli.js +8 -3
- package/dist/cli.js.map +1 -1
- package/dist/commands/doctor.d.ts.map +1 -1
- package/dist/commands/doctor.js +6 -3
- package/dist/commands/doctor.js.map +1 -1
- package/dist/commands/embeddings.d.ts +9 -0
- package/dist/commands/embeddings.d.ts.map +1 -1
- package/dist/commands/embeddings.js +58 -1
- package/dist/commands/embeddings.js.map +1 -1
- package/dist/commands/use.d.ts.map +1 -1
- package/dist/commands/use.js +4 -2
- package/dist/commands/use.js.map +1 -1
- package/dist/config/paths.d.ts +5 -0
- package/dist/config/paths.d.ts.map +1 -1
- package/dist/config/paths.js +34 -0
- package/dist/config/paths.js.map +1 -1
- package/dist/embedding/backend.d.ts +1 -1
- package/dist/embedding/backend.d.ts.map +1 -1
- package/dist/embedding/backend.js +20 -4
- package/dist/embedding/backend.js.map +1 -1
- package/dist/embedding/config.d.ts +3 -0
- package/dist/embedding/config.d.ts.map +1 -1
- package/dist/embedding/config.js +17 -0
- package/dist/embedding/config.js.map +1 -1
- package/dist/embedding/diagnostics.d.ts +1 -1
- package/dist/embedding/diagnostics.d.ts.map +1 -1
- package/dist/embedding/diagnostics.js +20 -5
- package/dist/embedding/diagnostics.js.map +1 -1
- package/dist/embedding/document.d.ts +5 -0
- package/dist/embedding/document.d.ts.map +1 -1
- package/dist/embedding/document.js +20 -0
- package/dist/embedding/document.js.map +1 -1
- package/dist/embedding/local-model-loader.d.ts +24 -0
- package/dist/embedding/local-model-loader.d.ts.map +1 -0
- package/dist/embedding/local-model-loader.js +64 -0
- package/dist/embedding/local-model-loader.js.map +1 -0
- package/dist/embedding/local-transformers-provider.d.ts +17 -0
- package/dist/embedding/local-transformers-provider.d.ts.map +1 -0
- package/dist/embedding/local-transformers-provider.js +54 -0
- package/dist/embedding/local-transformers-provider.js.map +1 -0
- package/dist/embedding/model-download.d.ts +32 -0
- package/dist/embedding/model-download.d.ts.map +1 -0
- package/dist/embedding/model-download.js +41 -0
- package/dist/embedding/model-download.js.map +1 -0
- package/dist/embedding/model-installation.d.ts +17 -0
- package/dist/embedding/model-installation.d.ts.map +1 -0
- package/dist/embedding/model-installation.js +71 -0
- package/dist/embedding/model-installation.js.map +1 -0
- package/dist/embedding/model-manifest.d.ts +18 -0
- package/dist/embedding/model-manifest.d.ts.map +1 -0
- package/dist/embedding/model-manifest.js +86 -0
- package/dist/embedding/model-manifest.js.map +1 -0
- package/dist/embedding/presets/local-small.d.ts +8 -0
- package/dist/embedding/presets/local-small.d.ts.map +1 -0
- package/dist/embedding/presets/local-small.js +67 -0
- package/dist/embedding/presets/local-small.js.map +1 -0
- package/dist/embedding/presets/manifest.d.ts +27 -0
- package/dist/embedding/presets/manifest.d.ts.map +1 -0
- package/dist/embedding/presets/manifest.js +5 -0
- package/dist/embedding/presets/manifest.js.map +1 -0
- package/dist/embedding/profile.d.ts +6 -2
- package/dist/embedding/profile.d.ts.map +1 -1
- package/dist/embedding/profile.js +36 -0
- package/dist/embedding/profile.js.map +1 -1
- package/dist/embedding/query-cache.d.ts +6 -0
- package/dist/embedding/query-cache.d.ts.map +1 -1
- package/dist/embedding/query-cache.js +21 -0
- package/dist/embedding/query-cache.js.map +1 -1
- package/dist/embedding/runtime.d.ts +2 -1
- package/dist/embedding/runtime.d.ts.map +1 -1
- package/dist/embedding/runtime.js +39 -5
- package/dist/embedding/runtime.js.map +1 -1
- package/dist/embedding/settings.d.ts +6 -0
- package/dist/embedding/settings.d.ts.map +1 -0
- package/dist/embedding/settings.js +69 -0
- package/dist/embedding/settings.js.map +1 -0
- package/dist/embedding/setup-lock.d.ts +7 -0
- package/dist/embedding/setup-lock.d.ts.map +1 -0
- package/dist/embedding/setup-lock.js +35 -0
- package/dist/embedding/setup-lock.js.map +1 -0
- package/dist/embedding/setup-service.d.ts +53 -0
- package/dist/embedding/setup-service.d.ts.map +1 -0
- package/dist/embedding/setup-service.js +184 -0
- package/dist/embedding/setup-service.js.map +1 -0
- package/dist/embedding/store.d.ts +3 -1
- package/dist/embedding/store.d.ts.map +1 -1
- package/dist/embedding/store.js +124 -11
- package/dist/embedding/store.js.map +1 -1
- package/dist/embedding/types.d.ts +31 -2
- package/dist/embedding/types.d.ts.map +1 -1
- package/dist/enno-oduno/directives.js +1 -1
- package/dist/enno-oduno/directives.js.map +1 -1
- package/dist/enno-oduno/instructions.d.ts +1 -1
- package/dist/enno-oduno/instructions.d.ts.map +1 -1
- package/dist/enno-oduno/instructions.js +1 -1
- package/dist/enno-oduno/instructions.js.map +1 -1
- package/dist/enno-oduno/schemas.d.ts +9 -0
- package/dist/enno-oduno/schemas.d.ts.map +1 -1
- package/dist/enno-oduno/schemas.js +11 -0
- package/dist/enno-oduno/schemas.js.map +1 -1
- package/dist/enno-oduno/service.d.ts +7 -1
- package/dist/enno-oduno/service.d.ts.map +1 -1
- package/dist/enno-oduno/service.js +52 -1
- package/dist/enno-oduno/service.js.map +1 -1
- package/dist/enno-oduno/validation-errors.d.ts +2 -1
- package/dist/enno-oduno/validation-errors.d.ts.map +1 -1
- package/dist/enno-oduno/validation-errors.js +1 -0
- package/dist/enno-oduno/validation-errors.js.map +1 -1
- package/dist/enno-oduno/verifier.d.ts.map +1 -1
- package/dist/enno-oduno/verifier.js +27 -2
- package/dist/enno-oduno/verifier.js.map +1 -1
- package/dist/mcp/bounded-stdio-transport.d.ts +3 -10
- package/dist/mcp/bounded-stdio-transport.d.ts.map +1 -1
- package/dist/mcp/bounded-stdio-transport.js +2 -89
- package/dist/mcp/bounded-stdio-transport.js.map +1 -1
- package/dist/mcp/request-deadline.d.ts +51 -0
- package/dist/mcp/request-deadline.d.ts.map +1 -0
- package/dist/mcp/request-deadline.js +155 -0
- package/dist/mcp/request-deadline.js.map +1 -0
- package/dist/mcp/runtime-owner.d.ts.map +1 -1
- package/dist/mcp/runtime-owner.js +2 -3
- package/dist/mcp/runtime-owner.js.map +1 -1
- package/dist/mcp/server.d.ts +2 -0
- package/dist/mcp/server.d.ts.map +1 -1
- package/dist/mcp/server.js +58 -23
- package/dist/mcp/server.js.map +1 -1
- package/dist/memory/scoped-memory.d.ts +1 -1
- package/dist/memory/scoped-memory.d.ts.map +1 -1
- package/dist/memory/scoped-memory.js +5 -1
- package/dist/memory/scoped-memory.js.map +1 -1
- package/dist/server/http.d.ts +1 -1
- package/dist/server/http.d.ts.map +1 -1
- package/dist/server/http.js +6 -3
- package/dist/server/http.js.map +1 -1
- package/dist/skills/discovery-service.d.ts.map +1 -1
- package/dist/skills/discovery-service.js +10 -8
- package/dist/skills/discovery-service.js.map +1 -1
- package/dist/skills/find.d.ts +1 -0
- package/dist/skills/find.d.ts.map +1 -1
- package/dist/skills/find.js +1 -0
- package/dist/skills/find.js.map +1 -1
- package/dist/skills/materialization-authority.d.ts +1 -1
- package/dist/skills/materialization-authority.d.ts.map +1 -1
- package/dist/skills/materialization-authority.js +2 -2
- package/dist/skills/materialization-authority.js.map +1 -1
- package/dist/skills/providers/skills-sh-compat.d.ts.map +1 -1
- package/dist/skills/providers/skills-sh-compat.js +4 -2
- package/dist/skills/providers/skills-sh-compat.js.map +1 -1
- package/dist/skills/providers/skills-sh-v1.d.ts +2 -2
- package/dist/skills/providers/skills-sh-v1.d.ts.map +1 -1
- package/dist/skills/providers/skills-sh-v1.js +22 -4
- package/dist/skills/providers/skills-sh-v1.js.map +1 -1
- package/dist/skills/source/github-fetcher.d.ts.map +1 -1
- package/dist/skills/source/github-fetcher.js +8 -4
- package/dist/skills/source/github-fetcher.js.map +1 -1
- package/dist/skills/types.d.ts +3 -2
- package/dist/skills/types.d.ts.map +1 -1
- package/docs/agent-file.md +72 -0
- package/docs/agent-gateway.md +104 -0
- package/docs/architecture.md +12 -0
- package/docs/cli-contract.md +35 -0
- package/docs/client-compatibility.md +298 -0
- package/docs/database.md +11 -0
- package/docs/execution-ledger.md +89 -0
- package/docs/release-notes-0.1.18.md +103 -0
- package/docs/retrieval-evaluation.md +10 -0
- package/docs/security.md +11 -0
- package/docs/sqlite-driver-adr.md +8 -0
- package/migrations/022_embedding_setup_v2.sql +236 -0
- package/migrations/down/022_embedding_setup_v2.sql +133 -0
- package/package.json +24 -3
- package/skills/kiokuko-enno-oduno/SKILL.md +44 -0
- package/templates/AGENTS.md +1 -1
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
import { KiokukoError } from '../errors.js';
|
|
2
|
+
const DISABLED_CONFIG = Object.freeze({
|
|
3
|
+
mode: 'off',
|
|
4
|
+
provider: 'openai-compatible',
|
|
5
|
+
allowRemote: false,
|
|
6
|
+
vectorBackend: 'auto',
|
|
7
|
+
timeoutMs: 30_000,
|
|
8
|
+
batchSize: 16,
|
|
9
|
+
});
|
|
10
|
+
function invalid(message) {
|
|
11
|
+
throw new KiokukoError('INTEGRITY_ERROR', message);
|
|
12
|
+
}
|
|
13
|
+
function tableExists(database) {
|
|
14
|
+
return Boolean(database.prepare(`
|
|
15
|
+
SELECT 1 AS present FROM sqlite_schema WHERE type = 'table' AND name = 'embedding_settings'
|
|
16
|
+
`).get());
|
|
17
|
+
}
|
|
18
|
+
function vectorBackend(value) {
|
|
19
|
+
if (value !== 'auto' && value !== 'javascript' && value !== 'sqlite-vec')
|
|
20
|
+
invalid('Persisted embedding vector backend is invalid');
|
|
21
|
+
return value;
|
|
22
|
+
}
|
|
23
|
+
function integer(value, minimum, maximum, field) {
|
|
24
|
+
if (typeof value !== 'number' || !Number.isSafeInteger(value) || value < minimum || value > maximum) {
|
|
25
|
+
invalid(`Persisted embedding ${field} is invalid`);
|
|
26
|
+
}
|
|
27
|
+
return value;
|
|
28
|
+
}
|
|
29
|
+
/** Read durable embedding settings without consulting embedding environment variables. */
|
|
30
|
+
export function readPersistedEmbeddingSettings(database) {
|
|
31
|
+
if (!tableExists(database))
|
|
32
|
+
return DISABLED_CONFIG;
|
|
33
|
+
const row = database.prepare(`
|
|
34
|
+
SELECT singleton, mode, provider_kind, vector_backend, batch_size, timeout_ms, setup_state
|
|
35
|
+
FROM embedding_settings
|
|
36
|
+
`).get();
|
|
37
|
+
if (row === undefined || row.singleton !== 1)
|
|
38
|
+
invalid('Persisted embedding settings must contain exactly one singleton row');
|
|
39
|
+
const backend = vectorBackend(row.vector_backend);
|
|
40
|
+
const batchSize = integer(row.batch_size, 1, 64, 'batch size');
|
|
41
|
+
const timeoutMs = integer(row.timeout_ms, 100, 120_000, 'timeout');
|
|
42
|
+
if (row.mode !== 'off' && row.mode !== 'optional' && row.mode !== 'required')
|
|
43
|
+
invalid('Persisted embedding mode is invalid');
|
|
44
|
+
if (row.setup_state !== 'disabled' && row.setup_state !== 'requires_setup' && row.setup_state !== 'installing'
|
|
45
|
+
&& row.setup_state !== 'ready' && row.setup_state !== 'degraded')
|
|
46
|
+
invalid('Persisted embedding setup state is invalid');
|
|
47
|
+
// v1 remote profiles intentionally cannot be reconstructed and local v2
|
|
48
|
+
// loading is supplied by the local provider work unit. Until a verified
|
|
49
|
+
// provider is available, fail closed to lexical retrieval.
|
|
50
|
+
if (row.mode === 'off' || row.provider_kind !== 'local-transformers' || row.setup_state !== 'ready') {
|
|
51
|
+
return Object.freeze({ ...DISABLED_CONFIG, vectorBackend: backend, batchSize, timeoutMs });
|
|
52
|
+
}
|
|
53
|
+
return Object.freeze({
|
|
54
|
+
mode: row.mode,
|
|
55
|
+
provider: 'openai-compatible',
|
|
56
|
+
baseUrl: 'http://127.0.0.1/kiokuko-local',
|
|
57
|
+
model: 'intfloat/multilingual-e5-small',
|
|
58
|
+
dimensions: 384,
|
|
59
|
+
distanceCeiling: 0.8,
|
|
60
|
+
allowRemote: false,
|
|
61
|
+
vectorBackend: backend,
|
|
62
|
+
batchSize,
|
|
63
|
+
timeoutMs,
|
|
64
|
+
});
|
|
65
|
+
}
|
|
66
|
+
export function defaultEmbeddingConfig() {
|
|
67
|
+
return DISABLED_CONFIG;
|
|
68
|
+
}
|
|
69
|
+
//# sourceMappingURL=settings.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"settings.js","sourceRoot":"","sources":["../../src/embedding/settings.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAI5C,MAAM,eAAe,GAAoB,MAAM,CAAC,MAAM,CAAC;IACrD,IAAI,EAAE,KAAK;IACX,QAAQ,EAAE,mBAAmB;IAC7B,WAAW,EAAE,KAAK;IAClB,aAAa,EAAE,MAAM;IACrB,SAAS,EAAE,MAAM;IACjB,SAAS,EAAE,EAAE;CACd,CAAC,CAAC;AAYH,SAAS,OAAO,CAAC,OAAe;IAC9B,MAAM,IAAI,YAAY,CAAC,iBAAiB,EAAE,OAAO,CAAC,CAAC;AACrD,CAAC;AAED,SAAS,WAAW,CAAC,QAAwB;IAC3C,OAAO,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC;;GAE/B,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC;AACZ,CAAC;AAED,SAAS,aAAa,CAAC,KAAc;IACnC,IAAI,KAAK,KAAK,MAAM,IAAI,KAAK,KAAK,YAAY,IAAI,KAAK,KAAK,YAAY;QAAE,OAAO,CAAC,+CAA+C,CAAC,CAAC;IACnI,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,OAAO,CAAC,KAAc,EAAE,OAAe,EAAE,OAAe,EAAE,KAAa;IAC9E,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,KAAK,CAAC,IAAI,KAAK,GAAG,OAAO,IAAI,KAAK,GAAG,OAAO,EAAE,CAAC;QACpG,OAAO,CAAC,uBAAuB,KAAK,aAAa,CAAC,CAAC;IACrD,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,0FAA0F;AAC1F,MAAM,UAAU,8BAA8B,CAAC,QAAwB;IACrE,IAAI,CAAC,WAAW,CAAC,QAAQ,CAAC;QAAE,OAAO,eAAe,CAAC;IACnD,MAAM,GAAG,GAAG,QAAQ,CAAC,OAAO,CAAC;;;GAG5B,CAAC,CAAC,GAAG,EAAe,CAAC;IACtB,IAAI,GAAG,KAAK,SAAS,IAAI,GAAG,CAAC,SAAS,KAAK,CAAC;QAAE,OAAO,CAAC,qEAAqE,CAAC,CAAC;IAC7H,MAAM,OAAO,GAAG,aAAa,CAAC,GAAG,CAAC,cAAc,CAAC,CAAC;IAClD,MAAM,SAAS,GAAG,OAAO,CAAC,GAAG,CAAC,UAAU,EAAE,CAAC,EAAE,EAAE,EAAE,YAAY,CAAC,CAAC;IAC/D,MAAM,SAAS,GAAG,OAAO,CAAC,GAAG,CAAC,UAAU,EAAE,GAAG,EAAE,OAAO,EAAE,SAAS,CAAC,CAAC;IACnE,IAAI,GAAG,CAAC,IAAI,KAAK,KAAK,IAAI,GAAG,CAAC,IAAI,KAAK,UAAU,IAAI,GAAG,CAAC,IAAI,KAAK,UAAU;QAAE,OAAO,CAAC,qCAAqC,CAAC,CAAC;IAC7H,IAAI,GAAG,CAAC,WAAW,KAAK,UAAU,IAAI,GAAG,CAAC,WAAW,KAAK,gBAAgB,IAAI,GAAG,CAAC,WAAW,KAAK,YAAY;WACzG,GAAG,CAAC,WAAW,KAAK,OAAO,IAAI,GAAG,CAAC,WAAW,KAAK,UAAU;QAAE,OAAO,CAAC,4CAA4C,CAAC,CAAC;IAE1H,wEAAwE;IACxE,wEAAwE;IACxE,2DAA2D;IAC3D,IAAI,GAAG,CAAC,IAAI,KAAK,KAAK,IAAI,GAAG,CAAC,aAAa,KAAK,oBAAoB,IAAI,GAAG,CAAC,WAAW,KAAK,OAAO,EAAE,CAAC;QACpG,OAAO,MAAM,CAAC,MAAM,CAAC,EAAE,GAAG,eAAe,EAAE,aAAa,EAAE,OAAO,EAAE,SAAS,EAAE,SAAS,EAAE,CAAC,CAAC;IAC7F,CAAC;IACD,OAAO,MAAM,CAAC,MAAM,CAAC;QACnB,IAAI,EAAE,GAAG,CAAC,IAA+B;QACzC,QAAQ,EAAE,mBAAmB;QAC7B,OAAO,EAAE,gCAAgC;QACzC,KAAK,EAAE,gCAAgC;QACvC,UAAU,EAAE,GAAG;QACf,eAAe,EAAE,GAAG;QACpB,WAAW,EAAE,KAAK;QAClB,aAAa,EAAE,OAAO;QACtB,SAAS;QACT,SAAS;KACV,CAAC,CAAC;AACL,CAAC;AAED,MAAM,UAAU,sBAAsB;IACpC,OAAO,eAAe,CAAC;AACzB,CAAC"}
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
import { type PathEnvironment } from '../config/paths.js';
|
|
2
|
+
export interface EmbeddingSetupLock {
|
|
3
|
+
readonly path: string;
|
|
4
|
+
release(): Promise<void>;
|
|
5
|
+
}
|
|
6
|
+
export declare function acquireEmbeddingSetupLock(options?: PathEnvironment): Promise<EmbeddingSetupLock>;
|
|
7
|
+
//# sourceMappingURL=setup-lock.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"setup-lock.d.ts","sourceRoot":"","sources":["../../src/embedding/setup-lock.ts"],"names":[],"mappings":"AAEA,OAAO,EAA0D,KAAK,eAAe,EAAE,MAAM,oBAAoB,CAAC;AAGlH,MAAM,WAAW,kBAAkB;IACjC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CAC1B;AAED,wBAAsB,yBAAyB,CAAC,OAAO,GAAE,eAAoB,GAAG,OAAO,CAAC,kBAAkB,CAAC,CAwB1G"}
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
import { randomUUID } from 'node:crypto';
|
|
2
|
+
import { mkdir, open, unlink } from 'node:fs/promises';
|
|
3
|
+
import { getEmbeddingModelsDirectory, getEmbeddingSetupLockPath } from '../config/paths.js';
|
|
4
|
+
import { KiokukoError } from '../errors.js';
|
|
5
|
+
export async function acquireEmbeddingSetupLock(options = {}) {
|
|
6
|
+
await mkdir(getEmbeddingModelsDirectory(options), { recursive: true, mode: 0o700 });
|
|
7
|
+
const lockPath = getEmbeddingSetupLockPath(options);
|
|
8
|
+
let handle;
|
|
9
|
+
try {
|
|
10
|
+
handle = await open(lockPath, 'wx', 0o600);
|
|
11
|
+
await handle.writeFile(`${randomUUID()}\n`, 'utf8');
|
|
12
|
+
}
|
|
13
|
+
catch (error) {
|
|
14
|
+
try {
|
|
15
|
+
await handle?.close();
|
|
16
|
+
}
|
|
17
|
+
catch { /* preserve the lock conflict */ }
|
|
18
|
+
if (error instanceof Error && 'code' in error && error.code === 'EEXIST') {
|
|
19
|
+
throw new KiokukoError('CONFLICT', 'Another embedding setup is already in progress');
|
|
20
|
+
}
|
|
21
|
+
throw error;
|
|
22
|
+
}
|
|
23
|
+
let released = false;
|
|
24
|
+
return Object.freeze({
|
|
25
|
+
path: lockPath,
|
|
26
|
+
release: async () => {
|
|
27
|
+
if (released)
|
|
28
|
+
return;
|
|
29
|
+
released = true;
|
|
30
|
+
await handle.close();
|
|
31
|
+
await unlink(lockPath);
|
|
32
|
+
},
|
|
33
|
+
});
|
|
34
|
+
}
|
|
35
|
+
//# sourceMappingURL=setup-lock.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"setup-lock.js","sourceRoot":"","sources":["../../src/embedding/setup-lock.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AACvD,OAAO,EAAE,2BAA2B,EAAE,yBAAyB,EAAwB,MAAM,oBAAoB,CAAC;AAClH,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAO5C,MAAM,CAAC,KAAK,UAAU,yBAAyB,CAAC,UAA2B,EAAE;IAC3E,MAAM,KAAK,CAAC,2BAA2B,CAAC,OAAO,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;IACpF,MAAM,QAAQ,GAAG,yBAAyB,CAAC,OAAO,CAAC,CAAC;IACpD,IAAI,MAAM,CAAC;IACX,IAAI,CAAC;QACH,MAAM,GAAG,MAAM,IAAI,CAAC,QAAQ,EAAE,IAAI,EAAE,KAAK,CAAC,CAAC;QAC3C,MAAM,MAAM,CAAC,SAAS,CAAC,GAAG,UAAU,EAAE,IAAI,EAAE,MAAM,CAAC,CAAC;IACtD,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,CAAC;YAAC,MAAM,MAAM,EAAE,KAAK,EAAE,CAAC;QAAC,CAAC;QAAC,MAAM,CAAC,CAAC,gCAAgC,CAAC,CAAC;QACzE,IAAI,KAAK,YAAY,KAAK,IAAI,MAAM,IAAI,KAAK,IAAK,KAA+B,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;YACpG,MAAM,IAAI,YAAY,CAAC,UAAU,EAAE,gDAAgD,CAAC,CAAC;QACvF,CAAC;QACD,MAAM,KAAK,CAAC;IACd,CAAC;IACD,IAAI,QAAQ,GAAG,KAAK,CAAC;IACrB,OAAO,MAAM,CAAC,MAAM,CAAC;QACnB,IAAI,EAAE,QAAQ;QACd,OAAO,EAAE,KAAK,IAAI,EAAE;YAClB,IAAI,QAAQ;gBAAE,OAAO;YACrB,QAAQ,GAAG,IAAI,CAAC;YAChB,MAAM,MAAM,CAAC,KAAK,EAAE,CAAC;YACrB,MAAM,MAAM,CAAC,QAAQ,CAAC,CAAC;QACzB,CAAC;KACF,CAAC,CAAC;AACL,CAAC"}
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
import { type InstalledModel } from './model-installation.js';
|
|
2
|
+
import { LOCAL_SMALL_PRESET } from './presets/local-small.js';
|
|
3
|
+
import type { ModelDownloader, ModelDownloadProgress } from './model-download.js';
|
|
4
|
+
import type { SqliteDatabase } from '../db/adapter.js';
|
|
5
|
+
import type { EmbeddingProvider } from './types.js';
|
|
6
|
+
import type { PathEnvironment } from '../config/paths.js';
|
|
7
|
+
export interface EmbeddingSetupInput {
|
|
8
|
+
readonly presetId: string;
|
|
9
|
+
readonly confirmed: boolean;
|
|
10
|
+
readonly dryRun: boolean;
|
|
11
|
+
readonly offline: boolean;
|
|
12
|
+
readonly replace: boolean;
|
|
13
|
+
}
|
|
14
|
+
export interface EmbeddingSetupResult {
|
|
15
|
+
readonly presetId: string;
|
|
16
|
+
readonly model: {
|
|
17
|
+
readonly repository: string;
|
|
18
|
+
readonly revision: string;
|
|
19
|
+
readonly installation: 'installed' | 'reused';
|
|
20
|
+
readonly bytes: number;
|
|
21
|
+
};
|
|
22
|
+
readonly migration: {
|
|
23
|
+
readonly fromVersion: number;
|
|
24
|
+
readonly toVersion: number;
|
|
25
|
+
readonly backupPath: string | null;
|
|
26
|
+
readonly applied: readonly number[];
|
|
27
|
+
};
|
|
28
|
+
readonly profile: {
|
|
29
|
+
readonly profileId: string;
|
|
30
|
+
readonly generation: number;
|
|
31
|
+
readonly activated: boolean;
|
|
32
|
+
};
|
|
33
|
+
readonly embeddings: {
|
|
34
|
+
readonly eligible: number;
|
|
35
|
+
readonly completed: number;
|
|
36
|
+
readonly failed: number;
|
|
37
|
+
readonly blocked: number;
|
|
38
|
+
readonly remaining: number;
|
|
39
|
+
};
|
|
40
|
+
readonly backend: string;
|
|
41
|
+
readonly semanticEnabled: boolean;
|
|
42
|
+
readonly restartRequired: boolean;
|
|
43
|
+
}
|
|
44
|
+
export interface EmbeddingSetupOptions extends PathEnvironment {
|
|
45
|
+
readonly downloader?: ModelDownloader;
|
|
46
|
+
readonly installer?: (preset: typeof LOCAL_SMALL_PRESET, options: EmbeddingSetupOptions) => Promise<InstalledModel>;
|
|
47
|
+
readonly provider?: EmbeddingProvider;
|
|
48
|
+
readonly backendId?: string;
|
|
49
|
+
readonly now?: () => string;
|
|
50
|
+
readonly onProgress?: (progress: ModelDownloadProgress) => void;
|
|
51
|
+
}
|
|
52
|
+
export declare function runEmbeddingSetup(database: SqliteDatabase, input: EmbeddingSetupInput, options?: EmbeddingSetupOptions): Promise<EmbeddingSetupResult>;
|
|
53
|
+
//# sourceMappingURL=setup-service.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"setup-service.d.ts","sourceRoot":"","sources":["../../src/embedding/setup-service.ts"],"names":[],"mappings":"AAQA,OAAO,EAAyB,KAAK,cAAc,EAAE,MAAM,yBAAyB,CAAC;AACrF,OAAO,EAAE,kBAAkB,EAAE,MAAM,0BAA0B,CAAC;AAG9D,OAAO,KAAK,EAAE,eAAe,EAAE,qBAAqB,EAAE,MAAM,qBAAqB,CAAC;AAClF,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,kBAAkB,CAAC;AACvD,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,YAAY,CAAC;AACpD,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAC;AAE1D,MAAM,WAAW,mBAAmB;IAClC,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,SAAS,EAAE,OAAO,CAAC;IAC5B,QAAQ,CAAC,MAAM,EAAE,OAAO,CAAC;IACzB,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC;IAC1B,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC;CAC3B;AAED,MAAM,WAAW,oBAAoB;IACnC,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,KAAK,EAAE;QAAE,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,YAAY,EAAE,WAAW,GAAG,QAAQ,CAAC;QAAC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC;IAClJ,QAAQ,CAAC,SAAS,EAAE;QAAE,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;QAAC,QAAQ,CAAC,OAAO,EAAE,SAAS,MAAM,EAAE,CAAA;KAAE,CAAC;IAC1J,QAAQ,CAAC,OAAO,EAAE;QAAE,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,SAAS,EAAE,OAAO,CAAA;KAAE,CAAC;IAC3G,QAAQ,CAAC,UAAU,EAAE;QAAE,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAA;KAAE,CAAC;IAC9J,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,eAAe,EAAE,OAAO,CAAC;IAClC,QAAQ,CAAC,eAAe,EAAE,OAAO,CAAC;CACnC;AAED,MAAM,WAAW,qBAAsB,SAAQ,eAAe;IAC5D,QAAQ,CAAC,UAAU,CAAC,EAAE,eAAe,CAAC;IACtC,QAAQ,CAAC,SAAS,CAAC,EAAE,CAAC,MAAM,EAAE,OAAO,kBAAkB,EAAE,OAAO,EAAE,qBAAqB,KAAK,OAAO,CAAC,cAAc,CAAC,CAAC;IACpH,QAAQ,CAAC,QAAQ,CAAC,EAAE,iBAAiB,CAAC;IACtC,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,GAAG,CAAC,EAAE,MAAM,MAAM,CAAC;IAC5B,QAAQ,CAAC,UAAU,CAAC,EAAE,CAAC,QAAQ,EAAE,qBAAqB,KAAK,IAAI,CAAC;CACjE;AAgGD,wBAAsB,iBAAiB,CACrC,QAAQ,EAAE,cAAc,EACxB,KAAK,EAAE,mBAAmB,EAC1B,OAAO,GAAE,qBAA0B,GAClC,OAAO,CAAC,oBAAoB,CAAC,CA0E/B"}
|
|
@@ -0,0 +1,184 @@
|
|
|
1
|
+
import { createHash, randomUUID } from 'node:crypto';
|
|
2
|
+
import { lstat } from 'node:fs/promises';
|
|
3
|
+
import { withImmediateTransaction } from '../db/transaction.js';
|
|
4
|
+
import { KiokukoError } from '../errors.js';
|
|
5
|
+
import { readEntry } from '../memory/entries.js';
|
|
6
|
+
import { buildCanonicalEmbeddingDocument, renderEmbeddingProviderInput } from './document.js';
|
|
7
|
+
import { claimEmbeddingJobs, failEmbeddingJob, finalizeEmbeddingJob, listEmbeddingJobs } from './jobs.js';
|
|
8
|
+
import { acquireEmbeddingSetupLock } from './setup-lock.js';
|
|
9
|
+
import { installEmbeddingModel } from './model-installation.js';
|
|
10
|
+
import { LOCAL_SMALL_PRESET } from './presets/local-small.js';
|
|
11
|
+
import { createLocalEmbeddingProfile } from './profile.js';
|
|
12
|
+
import { activateLocalEmbeddingProfileInTransaction } from './store.js';
|
|
13
|
+
function now(options) {
|
|
14
|
+
return options.now?.() ?? new Date().toISOString();
|
|
15
|
+
}
|
|
16
|
+
function setupFailureCode(error) {
|
|
17
|
+
if (error instanceof KiokukoError && error.code === 'SERVICE_UNAVAILABLE')
|
|
18
|
+
return 'provider_unavailable';
|
|
19
|
+
if (error instanceof KiokukoError && error.code === 'VALIDATION_ERROR')
|
|
20
|
+
return 'dimension_mismatch';
|
|
21
|
+
return 'provider_unavailable';
|
|
22
|
+
}
|
|
23
|
+
async function installationExists(directory) {
|
|
24
|
+
try {
|
|
25
|
+
return (await lstat(directory)).isDirectory();
|
|
26
|
+
}
|
|
27
|
+
catch (error) {
|
|
28
|
+
if (error instanceof Error && 'code' in error && error.code === 'ENOENT')
|
|
29
|
+
return false;
|
|
30
|
+
throw error;
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
function installationId(presetId, revision, manifestHash) {
|
|
34
|
+
return createHash('sha256').update(`${presetId}\u0000${revision}\u0000${manifestHash}`, 'utf8').digest('hex');
|
|
35
|
+
}
|
|
36
|
+
function activateSettings(database, input) {
|
|
37
|
+
database.prepare(`
|
|
38
|
+
UPDATE embedding_settings
|
|
39
|
+
SET mode = 'optional', provider_kind = 'local-transformers', preset_id = ?,
|
|
40
|
+
model_installation_id = ?, vector_backend = 'auto', batch_size = 16,
|
|
41
|
+
timeout_ms = 30000, legacy_profile_id = NULL, setup_state = ?, updated_at = ?
|
|
42
|
+
WHERE singleton = 1
|
|
43
|
+
`).run(LOCAL_SMALL_PRESET.id, input.installationId, input.setupState, input.now);
|
|
44
|
+
}
|
|
45
|
+
async function drainSetupJobs(database, provider, profileId, generation, options) {
|
|
46
|
+
let completed = 0;
|
|
47
|
+
let failed = 0;
|
|
48
|
+
let blocked = 0;
|
|
49
|
+
for (let round = 0; round < 3; round += 1) {
|
|
50
|
+
const jobs = claimEmbeddingJobs(database, { maxJobs: 64, now: now(options) });
|
|
51
|
+
if (jobs.length === 0)
|
|
52
|
+
break;
|
|
53
|
+
for (const job of jobs) {
|
|
54
|
+
try {
|
|
55
|
+
const entry = readEntry(database, { workspace: job.workspace, entryId: job.entryId });
|
|
56
|
+
const canonical = buildCanonicalEmbeddingDocument({
|
|
57
|
+
kind: entry.kind,
|
|
58
|
+
title: entry.title,
|
|
59
|
+
summary: entry.summary,
|
|
60
|
+
body: entry.body,
|
|
61
|
+
tags: entry.tags,
|
|
62
|
+
scope: entry.scope,
|
|
63
|
+
});
|
|
64
|
+
const providerInput = renderEmbeddingProviderInput(canonical.text);
|
|
65
|
+
const vectors = await provider.embed([providerInput]);
|
|
66
|
+
if (vectors.length !== 1)
|
|
67
|
+
throw new KiokukoError('VALIDATION_ERROR', 'Local provider returned an invalid batch');
|
|
68
|
+
const documentHash = createHash('sha256').update(providerInput, 'utf8').digest('hex');
|
|
69
|
+
finalizeEmbeddingJob(database, {
|
|
70
|
+
entryId: job.entryId,
|
|
71
|
+
profileId,
|
|
72
|
+
generation,
|
|
73
|
+
leaseId: job.leaseId,
|
|
74
|
+
revision: job.revision,
|
|
75
|
+
contentHash: job.contentHash,
|
|
76
|
+
documentHash,
|
|
77
|
+
vector: vectors[0],
|
|
78
|
+
now: now(options),
|
|
79
|
+
});
|
|
80
|
+
completed += 1;
|
|
81
|
+
}
|
|
82
|
+
catch (error) {
|
|
83
|
+
const code = setupFailureCode(error);
|
|
84
|
+
if (code === 'dimension_mismatch')
|
|
85
|
+
blocked += 1;
|
|
86
|
+
else
|
|
87
|
+
failed += 1;
|
|
88
|
+
failEmbeddingJob(database, {
|
|
89
|
+
entryId: job.entryId,
|
|
90
|
+
profileId,
|
|
91
|
+
generation,
|
|
92
|
+
leaseId: job.leaseId,
|
|
93
|
+
errorCode: code,
|
|
94
|
+
availableAt: now(options),
|
|
95
|
+
now: now(options),
|
|
96
|
+
permanent: code === 'dimension_mismatch',
|
|
97
|
+
});
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
const remaining = listEmbeddingJobs(database, { profileId }).filter((job) => job.state !== 'leased').length;
|
|
102
|
+
return { completed, failed, blocked, remaining };
|
|
103
|
+
}
|
|
104
|
+
export async function runEmbeddingSetup(database, input, options = {}) {
|
|
105
|
+
if (input.presetId !== LOCAL_SMALL_PRESET.id)
|
|
106
|
+
throw new KiokukoError('VALIDATION_ERROR', 'Only the local-small embedding preset is supported');
|
|
107
|
+
if (typeof input.confirmed !== 'boolean' || typeof input.dryRun !== 'boolean' || typeof input.offline !== 'boolean' || typeof input.replace !== 'boolean') {
|
|
108
|
+
throw new KiokukoError('VALIDATION_ERROR', 'Embedding setup options are invalid');
|
|
109
|
+
}
|
|
110
|
+
const eligible = Number(database.prepare('SELECT COUNT(*) AS count FROM entries').get()?.count ?? 0);
|
|
111
|
+
const fromVersion = Number(database.prepare('SELECT MAX(version) AS version FROM schema_migrations').get()?.version ?? 0);
|
|
112
|
+
if (input.dryRun) {
|
|
113
|
+
return {
|
|
114
|
+
presetId: LOCAL_SMALL_PRESET.id,
|
|
115
|
+
model: { repository: LOCAL_SMALL_PRESET.artifactRepository, revision: LOCAL_SMALL_PRESET.revision, installation: 'reused', bytes: LOCAL_SMALL_PRESET.files.reduce((sum, file) => sum + file.size, 0) },
|
|
116
|
+
migration: { fromVersion, toVersion: fromVersion, backupPath: null, applied: [] },
|
|
117
|
+
profile: { profileId: createLocalEmbeddingProfile(LOCAL_SMALL_PRESET).profileId, generation: 0, activated: false },
|
|
118
|
+
embeddings: { eligible, completed: 0, failed: 0, blocked: 0, remaining: eligible },
|
|
119
|
+
backend: options.backendId ?? 'javascript',
|
|
120
|
+
semanticEnabled: false,
|
|
121
|
+
restartRequired: false,
|
|
122
|
+
};
|
|
123
|
+
}
|
|
124
|
+
if (!input.confirmed)
|
|
125
|
+
throw new KiokukoError('USAGE_ERROR', 'Embedding setup requires explicit confirmation');
|
|
126
|
+
const lock = await acquireEmbeddingSetupLock(options);
|
|
127
|
+
try {
|
|
128
|
+
const finalDirectory = (await import('../config/paths.js')).getEmbeddingPresetDirectory(LOCAL_SMALL_PRESET.id, LOCAL_SMALL_PRESET.revision, options);
|
|
129
|
+
if (input.offline && !(await installationExists(finalDirectory)))
|
|
130
|
+
throw new KiokukoError('SERVICE_UNAVAILABLE', 'The pinned local embedding model is not installed for offline setup');
|
|
131
|
+
const installed = await (options.installer ?? installEmbeddingModel)(LOCAL_SMALL_PRESET, {
|
|
132
|
+
...options,
|
|
133
|
+
...(input.offline ? { downloader: { download: async () => { throw new KiokukoError('SERVICE_UNAVAILABLE', 'Offline setup cannot download the model'); } } } : {}),
|
|
134
|
+
});
|
|
135
|
+
const profile = createLocalEmbeddingProfile(LOCAL_SMALL_PRESET);
|
|
136
|
+
const setupNow = now(options);
|
|
137
|
+
const setupId = randomUUID();
|
|
138
|
+
const installation = installationId(LOCAL_SMALL_PRESET.id, LOCAL_SMALL_PRESET.revision, installed.manifestHash);
|
|
139
|
+
const activation = withImmediateTransaction(database, () => {
|
|
140
|
+
database.prepare(`
|
|
141
|
+
INSERT INTO embedding_model_installations (
|
|
142
|
+
installation_id, preset_id, repository_id, revision, artifact_manifest_hash,
|
|
143
|
+
relative_path, state, total_bytes, installed_at, verified_at, last_checked_at
|
|
144
|
+
) VALUES (?, ?, ?, ?, ?, ?, 'verified', ?, ?, ?, ?)
|
|
145
|
+
ON CONFLICT(installation_id) DO UPDATE SET state = 'verified', verified_at = excluded.verified_at, last_checked_at = excluded.last_checked_at
|
|
146
|
+
`).run(installation, LOCAL_SMALL_PRESET.id, LOCAL_SMALL_PRESET.artifactRepository, LOCAL_SMALL_PRESET.revision, installed.manifestHash, installed.relativePath, installed.totalBytes, setupNow, setupNow, setupNow);
|
|
147
|
+
database.prepare(`
|
|
148
|
+
INSERT INTO embedding_setup_runs (setup_id, preset_id, phase, initial_entry_count, processed_entry_count, started_at, updated_at)
|
|
149
|
+
VALUES (?, ?, 'activating', ?, 0, ?, ?)
|
|
150
|
+
`).run(setupId, LOCAL_SMALL_PRESET.id, eligible, setupNow, setupNow);
|
|
151
|
+
const result = activateLocalEmbeddingProfileInTransaction(database, profile, { replace: input.replace, now: setupNow });
|
|
152
|
+
activateSettings(database, { installationId: installation, setupState: 'installing', now: setupNow });
|
|
153
|
+
database.prepare('UPDATE embedding_setup_runs SET active_profile_id = ?, model_installation_id = ?, phase = \'embedding\', updated_at = ? WHERE setup_id = ?').run(profile.profileId, installation, setupNow, setupId);
|
|
154
|
+
return result;
|
|
155
|
+
});
|
|
156
|
+
const provider = options.provider ?? new (await import('./local-transformers-provider.js')).LocalTransformersEmbeddingProvider({ profile: profile.identity, modelDirectory: installed.directory });
|
|
157
|
+
let drain;
|
|
158
|
+
try {
|
|
159
|
+
drain = await drainSetupJobs(database, provider, profile.profileId, activation.generation, options);
|
|
160
|
+
}
|
|
161
|
+
finally {
|
|
162
|
+
await provider.close?.();
|
|
163
|
+
}
|
|
164
|
+
const finalState = drain.remaining === 0 ? 'ready' : 'degraded';
|
|
165
|
+
withImmediateTransaction(database, () => {
|
|
166
|
+
activateSettings(database, { installationId: installation, setupState: finalState, now: now(options) });
|
|
167
|
+
database.prepare('UPDATE embedding_setup_runs SET phase = ?, processed_entry_count = ?, updated_at = ?, completed_at = ? WHERE setup_id = ?').run(finalState === 'ready' ? 'completed' : 'failed', drain.completed, now(options), now(options), setupId);
|
|
168
|
+
});
|
|
169
|
+
return {
|
|
170
|
+
presetId: LOCAL_SMALL_PRESET.id,
|
|
171
|
+
model: { repository: LOCAL_SMALL_PRESET.artifactRepository, revision: LOCAL_SMALL_PRESET.revision, installation: installed.installation, bytes: installed.totalBytes },
|
|
172
|
+
migration: { fromVersion, toVersion: fromVersion, backupPath: null, applied: [] },
|
|
173
|
+
profile: { profileId: profile.profileId, generation: activation.generation, activated: activation.activated },
|
|
174
|
+
embeddings: { eligible, completed: drain.completed, failed: drain.failed, blocked: drain.blocked, remaining: drain.remaining },
|
|
175
|
+
backend: options.backendId ?? 'javascript',
|
|
176
|
+
semanticEnabled: true,
|
|
177
|
+
restartRequired: true,
|
|
178
|
+
};
|
|
179
|
+
}
|
|
180
|
+
finally {
|
|
181
|
+
await lock.release();
|
|
182
|
+
}
|
|
183
|
+
}
|
|
184
|
+
//# sourceMappingURL=setup-service.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"setup-service.js","sourceRoot":"","sources":["../../src/embedding/setup-service.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACrD,OAAO,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AACzC,OAAO,EAAE,wBAAwB,EAAE,MAAM,sBAAsB,CAAC;AAChE,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAC5C,OAAO,EAAE,SAAS,EAAE,MAAM,sBAAsB,CAAC;AACjD,OAAO,EAAE,+BAA+B,EAAE,4BAA4B,EAAE,MAAM,eAAe,CAAC;AAC9F,OAAO,EAAE,kBAAkB,EAAE,gBAAgB,EAAE,oBAAoB,EAAE,iBAAiB,EAAE,MAAM,WAAW,CAAC;AAC1G,OAAO,EAAE,yBAAyB,EAAE,MAAM,iBAAiB,CAAC;AAC5D,OAAO,EAAE,qBAAqB,EAAuB,MAAM,yBAAyB,CAAC;AACrF,OAAO,EAAE,kBAAkB,EAAE,MAAM,0BAA0B,CAAC;AAC9D,OAAO,EAAE,2BAA2B,EAAE,MAAM,cAAc,CAAC;AAC3D,OAAO,EAAE,0CAA0C,EAAE,MAAM,YAAY,CAAC;AAkCxE,SAAS,GAAG,CAAC,OAA8B;IACzC,OAAO,OAAO,CAAC,GAAG,EAAE,EAAE,IAAI,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;AACrD,CAAC;AAED,SAAS,gBAAgB,CAAC,KAAc;IACtC,IAAI,KAAK,YAAY,YAAY,IAAI,KAAK,CAAC,IAAI,KAAK,qBAAqB;QAAE,OAAO,sBAAsB,CAAC;IACzG,IAAI,KAAK,YAAY,YAAY,IAAI,KAAK,CAAC,IAAI,KAAK,kBAAkB;QAAE,OAAO,oBAAoB,CAAC;IACpG,OAAO,sBAAsB,CAAC;AAChC,CAAC;AAED,KAAK,UAAU,kBAAkB,CAAC,SAAiB;IACjD,IAAI,CAAC;QACH,OAAO,CAAC,MAAM,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC;IAChD,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,KAAK,YAAY,KAAK,IAAI,MAAM,IAAI,KAAK,IAAK,KAA+B,CAAC,IAAI,KAAK,QAAQ;YAAE,OAAO,KAAK,CAAC;QAClH,MAAM,KAAK,CAAC;IACd,CAAC;AACH,CAAC;AAED,SAAS,cAAc,CAAC,QAAgB,EAAE,QAAgB,EAAE,YAAoB;IAC9E,OAAO,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,GAAG,QAAQ,SAAS,QAAQ,SAAS,YAAY,EAAE,EAAE,MAAM,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;AAChH,CAAC;AAED,SAAS,gBAAgB,CAAC,QAAwB,EAAE,KAA+F;IACjJ,QAAQ,CAAC,OAAO,CAAC;;;;;;GAMhB,CAAC,CAAC,GAAG,CAAC,kBAAkB,CAAC,EAAE,EAAE,KAAK,CAAC,cAAc,EAAE,KAAK,CAAC,UAAU,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC;AACnF,CAAC;AAED,KAAK,UAAU,cAAc,CAC3B,QAAwB,EACxB,QAA2B,EAC3B,SAAiB,EACjB,UAAkB,EAClB,OAA8B;IAE9B,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,IAAI,OAAO,GAAG,CAAC,CAAC;IAChB,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,CAAC,EAAE,KAAK,IAAI,CAAC,EAAE,CAAC;QAC1C,MAAM,IAAI,GAAG,kBAAkB,CAAC,QAAQ,EAAE,EAAE,OAAO,EAAE,EAAE,EAAE,GAAG,EAAE,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;QAC9E,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;YAAE,MAAM;QAC7B,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;YACvB,IAAI,CAAC;gBACH,MAAM,KAAK,GAAG,SAAS,CAAC,QAAQ,EAAE,EAAE,SAAS,EAAE,GAAG,CAAC,SAAS,EAAE,OAAO,EAAE,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC;gBACtF,MAAM,SAAS,GAAG,+BAA+B,CAAC;oBAChD,IAAI,EAAE,KAAK,CAAC,IAAI;oBAChB,KAAK,EAAE,KAAK,CAAC,KAAK;oBAClB,OAAO,EAAE,KAAK,CAAC,OAAO;oBACtB,IAAI,EAAE,KAAK,CAAC,IAAI;oBAChB,IAAI,EAAE,KAAK,CAAC,IAAI;oBAChB,KAAK,EAAE,KAAK,CAAC,KAAK;iBACnB,CAAC,CAAC;gBACH,MAAM,aAAa,GAAG,4BAA4B,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;gBACnE,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,KAAK,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC;gBACtD,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;oBAAE,MAAM,IAAI,YAAY,CAAC,kBAAkB,EAAE,0CAA0C,CAAC,CAAC;gBACjH,MAAM,YAAY,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,aAAa,EAAE,MAAM,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;gBACtF,oBAAoB,CAAC,QAAQ,EAAE;oBAC7B,OAAO,EAAE,GAAG,CAAC,OAAO;oBACpB,SAAS;oBACT,UAAU;oBACV,OAAO,EAAE,GAAG,CAAC,OAAO;oBACpB,QAAQ,EAAE,GAAG,CAAC,QAAQ;oBACtB,WAAW,EAAE,GAAG,CAAC,WAAW;oBAC5B,YAAY;oBACZ,MAAM,EAAE,OAAO,CAAC,CAAC,CAAE;oBACnB,GAAG,EAAE,GAAG,CAAC,OAAO,CAAC;iBAClB,CAAC,CAAC;gBACH,SAAS,IAAI,CAAC,CAAC;YACjB,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,MAAM,IAAI,GAAG,gBAAgB,CAAC,KAAK,CAAC,CAAC;gBACrC,IAAI,IAAI,KAAK,oBAAoB;oBAAE,OAAO,IAAI,CAAC,CAAC;;oBAC3C,MAAM,IAAI,CAAC,CAAC;gBACjB,gBAAgB,CAAC,QAAQ,EAAE;oBACzB,OAAO,EAAE,GAAG,CAAC,OAAO;oBACpB,SAAS;oBACT,UAAU;oBACV,OAAO,EAAE,GAAG,CAAC,OAAO;oBACpB,SAAS,EAAE,IAAI;oBACf,WAAW,EAAE,GAAG,CAAC,OAAO,CAAC;oBACzB,GAAG,EAAE,GAAG,CAAC,OAAO,CAAC;oBACjB,SAAS,EAAE,IAAI,KAAK,oBAAoB;iBACzC,CAAC,CAAC;YACL,CAAC;QACH,CAAC;IACH,CAAC;IACD,MAAM,SAAS,GAAG,iBAAiB,CAAC,QAAQ,EAAE,EAAE,SAAS,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,KAAK,KAAK,QAAQ,CAAC,CAAC,MAAM,CAAC;IAC5G,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE,SAAS,EAAE,CAAC;AACnD,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACrC,QAAwB,EACxB,KAA0B,EAC1B,UAAiC,EAAE;IAEnC,IAAI,KAAK,CAAC,QAAQ,KAAK,kBAAkB,CAAC,EAAE;QAAE,MAAM,IAAI,YAAY,CAAC,kBAAkB,EAAE,oDAAoD,CAAC,CAAC;IAC/I,IAAI,OAAO,KAAK,CAAC,SAAS,KAAK,SAAS,IAAI,OAAO,KAAK,CAAC,MAAM,KAAK,SAAS,IAAI,OAAO,KAAK,CAAC,OAAO,KAAK,SAAS,IAAI,OAAO,KAAK,CAAC,OAAO,KAAK,SAAS,EAAE,CAAC;QAC1J,MAAM,IAAI,YAAY,CAAC,kBAAkB,EAAE,qCAAqC,CAAC,CAAC;IACpF,CAAC;IACD,MAAM,QAAQ,GAAG,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,uCAAuC,CAAC,CAAC,GAAG,EAAqB,EAAE,KAAK,IAAI,CAAC,CAAC,CAAC;IACxH,MAAM,WAAW,GAAG,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,uDAAuD,CAAC,CAAC,GAAG,EAAuB,EAAE,OAAO,IAAI,CAAC,CAAC,CAAC;IAC/I,IAAI,KAAK,CAAC,MAAM,EAAE,CAAC;QACjB,OAAO;YACL,QAAQ,EAAE,kBAAkB,CAAC,EAAE;YAC/B,KAAK,EAAE,EAAE,UAAU,EAAE,kBAAkB,CAAC,kBAAkB,EAAE,QAAQ,EAAE,kBAAkB,CAAC,QAAQ,EAAE,YAAY,EAAE,QAAQ,EAAE,KAAK,EAAE,kBAAkB,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,CAAC,GAAG,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC,EAAE;YACtM,SAAS,EAAE,EAAE,WAAW,EAAE,SAAS,EAAE,WAAW,EAAE,UAAU,EAAE,IAAI,EAAE,OAAO,EAAE,EAAE,EAAE;YACjF,OAAO,EAAE,EAAE,SAAS,EAAE,2BAA2B,CAAC,kBAAkB,CAAC,CAAC,SAAS,EAAE,UAAU,EAAE,CAAC,EAAE,SAAS,EAAE,KAAK,EAAE;YAClH,UAAU,EAAE,EAAE,QAAQ,EAAE,SAAS,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,SAAS,EAAE,QAAQ,EAAE;YAClF,OAAO,EAAE,OAAO,CAAC,SAAS,IAAI,YAAY;YAC1C,eAAe,EAAE,KAAK;YACtB,eAAe,EAAE,KAAK;SACvB,CAAC;IACJ,CAAC;IACD,IAAI,CAAC,KAAK,CAAC,SAAS;QAAE,MAAM,IAAI,YAAY,CAAC,aAAa,EAAE,gDAAgD,CAAC,CAAC;IAC9G,MAAM,IAAI,GAAG,MAAM,yBAAyB,CAAC,OAAO,CAAC,CAAC;IACtD,IAAI,CAAC;QACH,MAAM,cAAc,GAAG,CAAC,MAAM,MAAM,CAAC,oBAAoB,CAAC,CAAC,CAAC,2BAA2B,CAAC,kBAAkB,CAAC,EAAE,EAAE,kBAAkB,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;QACrJ,IAAI,KAAK,CAAC,OAAO,IAAI,CAAC,CAAC,MAAM,kBAAkB,CAAC,cAAc,CAAC,CAAC;YAAE,MAAM,IAAI,YAAY,CAAC,qBAAqB,EAAE,qEAAqE,CAAC,CAAC;QACvL,MAAM,SAAS,GAAG,MAAM,CAAC,OAAO,CAAC,SAAS,IAAI,qBAAqB,CAAC,CAAC,kBAAkB,EAAE;YACvF,GAAG,OAAO;YACV,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,EAAE,QAAQ,EAAE,KAAK,IAAI,EAAE,GAAG,MAAM,IAAI,YAAY,CAAC,qBAAqB,EAAE,yCAAyC,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SAClK,CAAC,CAAC;QACH,MAAM,OAAO,GAAG,2BAA2B,CAAC,kBAAkB,CAAC,CAAC;QAChE,MAAM,QAAQ,GAAG,GAAG,CAAC,OAAO,CAAC,CAAC;QAC9B,MAAM,OAAO,GAAG,UAAU,EAAE,CAAC;QAC7B,MAAM,YAAY,GAAG,cAAc,CAAC,kBAAkB,CAAC,EAAE,EAAE,kBAAkB,CAAC,QAAQ,EAAE,SAAS,CAAC,YAAY,CAAC,CAAC;QAChH,MAAM,UAAU,GAAG,wBAAwB,CAAC,QAAQ,EAAE,GAAG,EAAE;YACzD,QAAQ,CAAC,OAAO,CAAC;;;;;;OAMhB,CAAC,CAAC,GAAG,CAAC,YAAY,EAAE,kBAAkB,CAAC,EAAE,EAAE,kBAAkB,CAAC,kBAAkB,EAAE,kBAAkB,CAAC,QAAQ,EAAE,SAAS,CAAC,YAAY,EAAE,SAAS,CAAC,YAAY,EAAE,SAAS,CAAC,UAAU,EAAE,QAAQ,EAAE,QAAQ,EAAE,QAAQ,CAAC,CAAC;YACpN,QAAQ,CAAC,OAAO,CAAC;;;OAGhB,CAAC,CAAC,GAAG,CAAC,OAAO,EAAE,kBAAkB,CAAC,EAAE,EAAE,QAAQ,EAAE,QAAQ,EAAE,QAAQ,CAAC,CAAC;YACrE,MAAM,MAAM,GAAG,0CAA0C,CAAC,QAAQ,EAAE,OAAO,EAAE,EAAE,OAAO,EAAE,KAAK,CAAC,OAAO,EAAE,GAAG,EAAE,QAAQ,EAAE,CAAC,CAAC;YACxH,gBAAgB,CAAC,QAAQ,EAAE,EAAE,cAAc,EAAE,YAAY,EAAE,UAAU,EAAE,YAAY,EAAE,GAAG,EAAE,QAAQ,EAAE,CAAC,CAAC;YACtG,QAAQ,CAAC,OAAO,CAAC,4IAA4I,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,SAAS,EAAE,YAAY,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAC;YACvN,OAAO,MAAM,CAAC;QAChB,CAAC,CAAC,CAAC;QACH,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,IAAI,IAAI,CAAC,MAAM,MAAM,CAAC,kCAAkC,CAAC,CAAC,CAAC,kCAAkC,CAAC,EAAE,OAAO,EAAE,OAAO,CAAC,QAAQ,EAAE,cAAc,EAAE,SAAS,CAAC,SAAS,EAAE,CAAC,CAAC;QACnM,IAAI,KAAK,CAAC;QACV,IAAI,CAAC;YACH,KAAK,GAAG,MAAM,cAAc,CAAC,QAAQ,EAAE,QAAQ,EAAE,OAAO,CAAC,SAAS,EAAE,UAAU,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;QACtG,CAAC;gBAAS,CAAC;YACT,MAAO,QAA4C,CAAC,KAAK,EAAE,EAAE,CAAC;QAChE,CAAC;QACD,MAAM,UAAU,GAAG,KAAK,CAAC,SAAS,KAAK,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,UAAU,CAAC;QAChE,wBAAwB,CAAC,QAAQ,EAAE,GAAG,EAAE;YACtC,gBAAgB,CAAC,QAAQ,EAAE,EAAE,cAAc,EAAE,YAAY,EAAE,UAAU,EAAE,UAAU,EAAE,GAAG,EAAE,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;YACxG,QAAQ,CAAC,OAAO,CAAC,2HAA2H,CAAC,CAAC,GAAG,CAAC,UAAU,KAAK,OAAO,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,QAAQ,EAAE,KAAK,CAAC,SAAS,EAAE,GAAG,CAAC,OAAO,CAAC,EAAE,GAAG,CAAC,OAAO,CAAC,EAAE,OAAO,CAAC,CAAC;QAC3P,CAAC,CAAC,CAAC;QACH,OAAO;YACL,QAAQ,EAAE,kBAAkB,CAAC,EAAE;YAC/B,KAAK,EAAE,EAAE,UAAU,EAAE,kBAAkB,CAAC,kBAAkB,EAAE,QAAQ,EAAE,kBAAkB,CAAC,QAAQ,EAAE,YAAY,EAAE,SAAS,CAAC,YAAY,EAAE,KAAK,EAAE,SAAS,CAAC,UAAU,EAAE;YACtK,SAAS,EAAE,EAAE,WAAW,EAAE,SAAS,EAAE,WAAW,EAAE,UAAU,EAAE,IAAI,EAAE,OAAO,EAAE,EAAE,EAAE;YACjF,OAAO,EAAE,EAAE,SAAS,EAAE,OAAO,CAAC,SAAS,EAAE,UAAU,EAAE,UAAU,CAAC,UAAU,EAAE,SAAS,EAAE,UAAU,CAAC,SAAS,EAAE;YAC7G,UAAU,EAAE,EAAE,QAAQ,EAAE,SAAS,EAAE,KAAK,CAAC,SAAS,EAAE,MAAM,EAAE,KAAK,CAAC,MAAM,EAAE,OAAO,EAAE,KAAK,CAAC,OAAO,EAAE,SAAS,EAAE,KAAK,CAAC,SAAS,EAAE;YAC9H,OAAO,EAAE,OAAO,CAAC,SAAS,IAAI,YAAY;YAC1C,eAAe,EAAE,IAAI;YACrB,eAAe,EAAE,IAAI;SACtB,CAAC;IACJ,CAAC;YAAS,CAAC;QACT,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC;IACvB,CAAC;AACH,CAAC"}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import type { SqliteDatabase } from '../db/adapter.js';
|
|
2
|
-
import type { EmbeddingProfile } from './types.js';
|
|
2
|
+
import type { EmbeddingProfile, LocalEmbeddingProfile } from './types.js';
|
|
3
3
|
export interface EmbeddingRuntimeState {
|
|
4
4
|
readonly activeProfileId: string | null;
|
|
5
5
|
readonly generation: number;
|
|
@@ -55,6 +55,8 @@ export declare function enqueueCurrentEntryEmbeddingInTransaction(database: Sqli
|
|
|
55
55
|
export declare function enqueueAllCurrentEntryEmbeddingsInTransaction(database: SqliteDatabase, requestedNow?: string, requestedWorkspace?: string): number;
|
|
56
56
|
export declare function activateEmbeddingProfileInTransaction(database: SqliteDatabase, profile: EmbeddingProfile, options: ActivateEmbeddingProfileOptions): EmbeddingProfileActivation;
|
|
57
57
|
export declare function activateEmbeddingProfile(database: SqliteDatabase, profile: EmbeddingProfile, options: ActivateEmbeddingProfileOptions): EmbeddingProfileActivation;
|
|
58
|
+
export declare function activateLocalEmbeddingProfileInTransaction(database: SqliteDatabase, profile: LocalEmbeddingProfile, options: ActivateEmbeddingProfileOptions): EmbeddingProfileActivation;
|
|
59
|
+
export declare function activateLocalEmbeddingProfile(database: SqliteDatabase, profile: LocalEmbeddingProfile, options: ActivateEmbeddingProfileOptions): EmbeddingProfileActivation;
|
|
58
60
|
export declare function upsertEntryEmbeddingInTransaction(database: SqliteDatabase, input: EntryEmbeddingInput): void;
|
|
59
61
|
export declare function upsertEntryEmbedding(database: SqliteDatabase, input: EntryEmbeddingInput): void;
|
|
60
62
|
export declare function readEntryEmbedding(database: SqliteDatabase, input: {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"store.d.ts","sourceRoot":"","sources":["../../src/embedding/store.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAa,MAAM,kBAAkB,CAAC;AAOlE,OAAO,KAAK,EACV,gBAAgB,
|
|
1
|
+
{"version":3,"file":"store.d.ts","sourceRoot":"","sources":["../../src/embedding/store.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAa,MAAM,kBAAkB,CAAC;AAOlE,OAAO,KAAK,EACV,gBAAgB,EAEhB,qBAAqB,EAEtB,MAAM,YAAY,CAAC;AAWpB,MAAM,WAAW,qBAAqB;IACpC,QAAQ,CAAC,eAAe,EAAE,MAAM,GAAG,IAAI,CAAC;IACxC,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;CACrC;AAED,MAAM,WAAW,sBAAsB;IACrC,QAAQ,CAAC,OAAO,EAAE,gBAAgB,CAAC;IACnC,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;CAC9B;AAED,MAAM,WAAW,+BAA+B;IAC9C,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC;IAC1B,QAAQ,CAAC,GAAG,CAAC,EAAE,MAAM,CAAC;CACvB;AAED,MAAM,WAAW,0BAA0B;IACzC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,SAAS,EAAE,OAAO,CAAC;IAC5B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;CAC3B;AAED,MAAM,WAAW,6BAA6B;IAC5C,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;CACtB;AAED,MAAM,WAAW,mBAAmB;IAClC,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,MAAM,EAAE,YAAY,GAAG,SAAS,MAAM,EAAE,CAAC;IAClD,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;CAC5B;AAED,MAAM,WAAW,oBAAoB;IACnC,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,MAAM,EAAE,YAAY,CAAC;IAC9B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;CAC5B;AAgRD,wBAAgB,oBAAoB,CAAC,QAAQ,EAAE,cAAc,EAAE,SAAS,EAAE,MAAM,GAAG,gBAAgB,GAAG,SAAS,CAI9G;AAED,wBAAgB,yBAAyB,CAAC,QAAQ,EAAE,cAAc,GAAG,qBAAqB,CAsBzF;AAED,wBAAgB,0BAA0B,CAAC,QAAQ,EAAE,cAAc,GAAG,sBAAsB,GAAG,IAAI,CAMlG;AAiDD,qGAAqG;AACrG,wBAAgB,yCAAyC,CACvD,QAAQ,EAAE,cAAc,EACxB,KAAK,EAAE,6BAA6B,GACnC,IAAI,CASN;AAED,2EAA2E;AAC3E,wBAAgB,6CAA6C,CAC3D,QAAQ,EAAE,cAAc,EACxB,YAAY,CAAC,EAAE,MAAM,EACrB,kBAAkB,CAAC,EAAE,MAAM,GAC1B,MAAM,CAqBR;AAED,wBAAgB,qCAAqC,CACnD,QAAQ,EAAE,cAAc,EACxB,OAAO,EAAE,gBAAgB,EACzB,OAAO,EAAE,+BAA+B,GACvC,0BAA0B,CAkD5B;AAED,wBAAgB,wBAAwB,CACtC,QAAQ,EAAE,cAAc,EACxB,OAAO,EAAE,gBAAgB,EACzB,OAAO,EAAE,+BAA+B,GACvC,0BAA0B,CAE5B;AAED,wBAAgB,0CAA0C,CACxD,QAAQ,EAAE,cAAc,EACxB,OAAO,EAAE,qBAAqB,EAC9B,OAAO,EAAE,+BAA+B,GACvC,0BAA0B,CA8C5B;AAED,wBAAgB,6BAA6B,CAC3C,QAAQ,EAAE,cAAc,EACxB,OAAO,EAAE,qBAAqB,EAC9B,OAAO,EAAE,+BAA+B,GACvC,0BAA0B,CAE5B;AAgBD,wBAAgB,iCAAiC,CAAC,QAAQ,EAAE,cAAc,EAAE,KAAK,EAAE,mBAAmB,GAAG,IAAI,CAwC5G;AAED,wBAAgB,oBAAoB,CAClC,QAAQ,EAAE,cAAc,EACxB,KAAK,EAAE,mBAAmB,GACzB,IAAI,CAEN;AAED,wBAAgB,kBAAkB,CAChC,QAAQ,EAAE,cAAc,EACxB,KAAK,EAAE;IAAE,OAAO,EAAE,MAAM,CAAC;IAAC,SAAS,EAAE,MAAM,CAAA;CAAE,GAC5C,oBAAoB,GAAG,SAAS,CAWlC"}
|
package/dist/embedding/store.js
CHANGED
|
@@ -54,6 +54,32 @@ function entryIdValue(value, label) {
|
|
|
54
54
|
return value;
|
|
55
55
|
}
|
|
56
56
|
function requireProfileIdentity(identity) {
|
|
57
|
+
if (identity.schemaVersion === 2) {
|
|
58
|
+
if (identity.providerKind !== 'local-transformers'
|
|
59
|
+
|| identity.presetId !== 'local-small'
|
|
60
|
+
|| identity.sourceModel !== 'intfloat/multilingual-e5-small'
|
|
61
|
+
|| identity.artifactRepository !== 'Xenova/multilingual-e5-small'
|
|
62
|
+
|| !/^[0-9a-f]{40}$/u.test(identity.modelRevision)
|
|
63
|
+
|| !HASH_PATTERN.test(identity.artifactManifestHash)
|
|
64
|
+
|| identity.inferenceEngine !== 'transformers-js'
|
|
65
|
+
|| identity.inferenceEngineVersion.length === 0
|
|
66
|
+
|| identity.dtype !== 'q8'
|
|
67
|
+
|| identity.pooling !== 'mean'
|
|
68
|
+
|| identity.normalize !== true
|
|
69
|
+
|| identity.maximumTokens !== 512
|
|
70
|
+
|| identity.dimensions !== 384
|
|
71
|
+
|| identity.distanceMetric !== 'cosine'
|
|
72
|
+
|| identity.documentTemplateVersion !== 2
|
|
73
|
+
|| identity.queryTemplateVersion !== 2
|
|
74
|
+
|| identity.inputContract !== 'e5-query-passage-v1'
|
|
75
|
+
|| identity.queryPrefix !== 'query: '
|
|
76
|
+
|| identity.documentPrefix !== 'passage: '
|
|
77
|
+
|| !Number.isFinite(identity.distanceCeiling)
|
|
78
|
+
|| identity.distanceCeiling <= 0 || identity.distanceCeiling >= 2) {
|
|
79
|
+
invalid('Embedding local profile identity is invalid');
|
|
80
|
+
}
|
|
81
|
+
return;
|
|
82
|
+
}
|
|
57
83
|
if (identity.schemaVersion !== 1
|
|
58
84
|
|| identity.providerKind !== 'openai-compatible'
|
|
59
85
|
|| !HASH_PATTERN.test(identity.endpointFingerprint)
|
|
@@ -82,6 +108,51 @@ function requireProfile(profile) {
|
|
|
82
108
|
}
|
|
83
109
|
}
|
|
84
110
|
function profileIdentityFromRow(row) {
|
|
111
|
+
if (row.schema_version === 2) {
|
|
112
|
+
if (typeof row.profile_id !== 'string' || !HASH_PATTERN.test(row.profile_id)
|
|
113
|
+
|| row.provider_kind !== 'local-transformers' || row.endpoint_fingerprint !== null
|
|
114
|
+
|| row.preset_id !== 'local-small' || row.source_model !== 'intfloat/multilingual-e5-small'
|
|
115
|
+
|| row.artifact_repository !== 'Xenova/multilingual-e5-small'
|
|
116
|
+
|| typeof row.model_revision !== 'string' || !/^[0-9a-f]{40}$/u.test(row.model_revision)
|
|
117
|
+
|| typeof row.artifact_manifest_hash !== 'string' || !HASH_PATTERN.test(row.artifact_manifest_hash)
|
|
118
|
+
|| row.inference_engine !== 'transformers-js'
|
|
119
|
+
|| typeof row.inference_engine_version !== 'string' || row.inference_engine_version.length === 0
|
|
120
|
+
|| row.dtype !== 'q8' || row.pooling !== 'mean' || row.normalize !== 1
|
|
121
|
+
|| row.maximum_tokens !== 512 || row.dimensions !== 384 || row.distance_metric !== 'cosine'
|
|
122
|
+
|| row.document_template_version !== 2 || row.query_template_version !== 2
|
|
123
|
+
|| row.input_contract !== 'e5-query-passage-v1' || row.query_prefix !== 'query: '
|
|
124
|
+
|| row.document_prefix !== 'passage: '
|
|
125
|
+
|| typeof row.distance_ceiling !== 'number' || !Number.isFinite(row.distance_ceiling)
|
|
126
|
+
|| row.distance_ceiling <= 0 || row.distance_ceiling >= 2)
|
|
127
|
+
integrity('Stored local embedding profile is invalid');
|
|
128
|
+
const identity = {
|
|
129
|
+
schemaVersion: 2,
|
|
130
|
+
providerKind: 'local-transformers',
|
|
131
|
+
presetId: 'local-small',
|
|
132
|
+
sourceModel: 'intfloat/multilingual-e5-small',
|
|
133
|
+
artifactRepository: 'Xenova/multilingual-e5-small',
|
|
134
|
+
modelRevision: row.model_revision,
|
|
135
|
+
artifactManifestHash: row.artifact_manifest_hash,
|
|
136
|
+
inferenceEngine: 'transformers-js',
|
|
137
|
+
inferenceEngineVersion: row.inference_engine_version,
|
|
138
|
+
dtype: 'q8',
|
|
139
|
+
pooling: 'mean',
|
|
140
|
+
normalize: true,
|
|
141
|
+
maximumTokens: 512,
|
|
142
|
+
dimensions: 384,
|
|
143
|
+
distanceMetric: 'cosine',
|
|
144
|
+
distanceCeiling: row.distance_ceiling,
|
|
145
|
+
inputContract: 'e5-query-passage-v1',
|
|
146
|
+
documentTemplateVersion: 2,
|
|
147
|
+
queryTemplateVersion: 2,
|
|
148
|
+
queryPrefix: 'query: ',
|
|
149
|
+
documentPrefix: 'passage: ',
|
|
150
|
+
};
|
|
151
|
+
if (embeddingProfileId(identity) !== row.profile_id)
|
|
152
|
+
integrity('Stored local embedding profile ID does not match its identity');
|
|
153
|
+
storedTimestamp(row.created_at, 'embedding profile created_at');
|
|
154
|
+
return Object.freeze({ profileId: row.profile_id, identity: Object.freeze(identity) });
|
|
155
|
+
}
|
|
85
156
|
if (typeof row.profile_id !== 'string' || !HASH_PATTERN.test(row.profile_id)
|
|
86
157
|
|| row.provider_kind !== 'openai-compatible'
|
|
87
158
|
|| typeof row.endpoint_fingerprint !== 'string' || !HASH_PATTERN.test(row.endpoint_fingerprint)
|
|
@@ -111,15 +182,7 @@ function profileIdentityFromRow(row) {
|
|
|
111
182
|
return Object.freeze({ profileId: row.profile_id, identity: Object.freeze(identity) });
|
|
112
183
|
}
|
|
113
184
|
function profileIdentitiesEqual(left, right) {
|
|
114
|
-
return left.schemaVersion === right.schemaVersion
|
|
115
|
-
&& left.providerKind === right.providerKind
|
|
116
|
-
&& left.endpointFingerprint === right.endpointFingerprint
|
|
117
|
-
&& left.model === right.model
|
|
118
|
-
&& left.dimensions === right.dimensions
|
|
119
|
-
&& left.distanceMetric === right.distanceMetric
|
|
120
|
-
&& left.documentTemplateVersion === right.documentTemplateVersion
|
|
121
|
-
&& left.queryTemplateVersion === right.queryTemplateVersion
|
|
122
|
-
&& left.distanceCeiling === right.distanceCeiling;
|
|
185
|
+
return left.schemaVersion === right.schemaVersion && embeddingProfileId(left) === embeddingProfileId(right);
|
|
123
186
|
}
|
|
124
187
|
function schemaTables(database) {
|
|
125
188
|
return new Set(database.prepare(`
|
|
@@ -147,9 +210,12 @@ function embeddingSchemaInstalled(database) {
|
|
|
147
210
|
}
|
|
148
211
|
function readProfileRow(database, profileId) {
|
|
149
212
|
const row = database.prepare(`
|
|
150
|
-
SELECT profile_id, provider_kind, endpoint_fingerprint, model, dimensions,
|
|
213
|
+
SELECT profile_id, schema_version, provider_kind, endpoint_fingerprint, model, dimensions,
|
|
151
214
|
distance_metric, distance_ceiling, document_template_version,
|
|
152
|
-
query_template_version,
|
|
215
|
+
query_template_version, preset_id, source_model, artifact_repository,
|
|
216
|
+
model_revision, artifact_manifest_hash, inference_engine,
|
|
217
|
+
inference_engine_version, dtype, pooling, normalize, maximum_tokens,
|
|
218
|
+
input_contract, query_prefix, document_prefix, created_at
|
|
153
219
|
FROM embedding_profiles
|
|
154
220
|
WHERE profile_id = ?
|
|
155
221
|
`).get(profileId);
|
|
@@ -273,6 +339,8 @@ export function enqueueAllCurrentEntryEmbeddingsInTransaction(database, requeste
|
|
|
273
339
|
}
|
|
274
340
|
export function activateEmbeddingProfileInTransaction(database, profile, options) {
|
|
275
341
|
requireProfile(profile);
|
|
342
|
+
if (profile.identity.schemaVersion !== 1)
|
|
343
|
+
invalid('Legacy embedding activation requires a v1 profile identity');
|
|
276
344
|
if (typeof options.replace !== 'boolean')
|
|
277
345
|
invalid('replace must be a boolean');
|
|
278
346
|
const now = validateTimestamp(options.now ?? new Date().toISOString(), 'now');
|
|
@@ -313,6 +381,51 @@ export function activateEmbeddingProfileInTransaction(database, profile, options
|
|
|
313
381
|
export function activateEmbeddingProfile(database, profile, options) {
|
|
314
382
|
return withImmediateTransaction(database, () => activateEmbeddingProfileInTransaction(database, profile, options));
|
|
315
383
|
}
|
|
384
|
+
export function activateLocalEmbeddingProfileInTransaction(database, profile, options) {
|
|
385
|
+
requireProfile(profile);
|
|
386
|
+
if (profile.identity.schemaVersion !== 2)
|
|
387
|
+
invalid('Local embedding activation requires a v2 profile identity');
|
|
388
|
+
if (typeof options.replace !== 'boolean')
|
|
389
|
+
invalid('replace must be a boolean');
|
|
390
|
+
const now = validateTimestamp(options.now ?? new Date().toISOString(), 'now');
|
|
391
|
+
requireEmbeddingSchema(database);
|
|
392
|
+
const existing = readProfileRow(database, profile.profileId);
|
|
393
|
+
if (existing === undefined) {
|
|
394
|
+
database.prepare(`
|
|
395
|
+
INSERT INTO embedding_profiles (
|
|
396
|
+
profile_id, schema_version, provider_kind, endpoint_fingerprint, model,
|
|
397
|
+
dimensions, distance_metric, distance_ceiling, document_template_version,
|
|
398
|
+
query_template_version, preset_id, source_model, artifact_repository,
|
|
399
|
+
model_revision, artifact_manifest_hash, inference_engine,
|
|
400
|
+
inference_engine_version, dtype, pooling, normalize, maximum_tokens,
|
|
401
|
+
input_contract, query_prefix, document_prefix, created_at
|
|
402
|
+
) VALUES (?, 2, 'local-transformers', NULL, ?, 384, 'cosine', ?, 2, 2,
|
|
403
|
+
?, ?, ?, ?, ?, 'transformers-js', ?, 'q8', 'mean', 1, 512,
|
|
404
|
+
'e5-query-passage-v1', 'query: ', 'passage: ', ?)
|
|
405
|
+
`).run(profile.profileId, profile.identity.sourceModel, profile.identity.distanceCeiling, profile.identity.presetId, profile.identity.sourceModel, profile.identity.artifactRepository, profile.identity.modelRevision, profile.identity.artifactManifestHash, profile.identity.inferenceEngineVersion, now);
|
|
406
|
+
}
|
|
407
|
+
else if (!profileIdentitiesEqual(existing.identity, profile.identity)) {
|
|
408
|
+
conflict('Embedding profile ID already refers to different profile fields');
|
|
409
|
+
}
|
|
410
|
+
const runtime = readEmbeddingRuntimeState(database);
|
|
411
|
+
if (runtime.activeProfileId === profile.profileId)
|
|
412
|
+
return { profileId: profile.profileId, generation: runtime.generation, activated: false, enqueued: 0 };
|
|
413
|
+
if (runtime.activeProfileId !== null && !options.replace)
|
|
414
|
+
conflict('An active embedding profile already exists; use replace to switch it');
|
|
415
|
+
const generation = runtime.activeProfileId === null ? runtime.generation : runtime.generation + 1;
|
|
416
|
+
database.prepare(`
|
|
417
|
+
UPDATE embedding_runtime SET active_profile_id = ?, generation = ?, activated_at = ?
|
|
418
|
+
WHERE singleton = 1 AND generation = ?
|
|
419
|
+
`).run(profile.profileId, generation, now, runtime.generation);
|
|
420
|
+
const updated = readEmbeddingRuntimeState(database);
|
|
421
|
+
if (updated.activeProfileId !== profile.profileId || updated.generation !== generation || updated.activatedAt !== now)
|
|
422
|
+
conflict('Embedding runtime changed while activating a local profile');
|
|
423
|
+
const enqueued = enqueueAllCurrentEntryEmbeddingsInTransaction(database, now);
|
|
424
|
+
return { profileId: profile.profileId, generation, activated: true, enqueued };
|
|
425
|
+
}
|
|
426
|
+
export function activateLocalEmbeddingProfile(database, profile, options) {
|
|
427
|
+
return withImmediateTransaction(database, () => activateLocalEmbeddingProfileInTransaction(database, profile, options));
|
|
428
|
+
}
|
|
316
429
|
function storedEmbedding(row) {
|
|
317
430
|
const entryId = entryIdValue(row.entry_id, 'entryId');
|
|
318
431
|
const profileId = hashValue(row.profile_id, 'profileId');
|