@wei840222/qmd 2026.8.23
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +1373 -0
- package/LICENSE +45 -0
- package/README.md +1439 -0
- package/THIRD_PARTY_NOTICES.md +31 -0
- package/bin/qmd +192 -0
- package/dist/ast.d.ts +65 -0
- package/dist/ast.js +334 -0
- package/dist/bench/bench.d.ts +35 -0
- package/dist/bench/bench.js +338 -0
- package/dist/bench/cjk-baseline.d.ts +36 -0
- package/dist/bench/cjk-baseline.js +111 -0
- package/dist/bench/fixture.d.ts +2 -0
- package/dist/bench/fixture.js +84 -0
- package/dist/bench/score.d.ts +38 -0
- package/dist/bench/score.js +107 -0
- package/dist/bench/types.d.ts +110 -0
- package/dist/bench/types.js +8 -0
- package/dist/cli/build-info.json +4 -0
- package/dist/cli/embed-lock.d.ts +24 -0
- package/dist/cli/embed-lock.js +94 -0
- package/dist/cli/embedding-owner.d.ts +10 -0
- package/dist/cli/embedding-owner.js +20 -0
- package/dist/cli/formatter.d.ts +120 -0
- package/dist/cli/formatter.js +355 -0
- package/dist/cli/mcp-pid.d.ts +25 -0
- package/dist/cli/mcp-pid.js +86 -0
- package/dist/cli/qmd.d.ts +72 -0
- package/dist/cli/qmd.js +4806 -0
- package/dist/cli/version.d.ts +42 -0
- package/dist/cli/version.js +80 -0
- package/dist/collections.d.ts +200 -0
- package/dist/collections.js +433 -0
- package/dist/db.d.ts +65 -0
- package/dist/db.js +143 -0
- package/dist/diagnostics.d.ts +62 -0
- package/dist/diagnostics.js +260 -0
- package/dist/embedding/config.d.ts +52 -0
- package/dist/embedding/config.js +229 -0
- package/dist/embedding/identity.d.ts +58 -0
- package/dist/embedding/identity.js +321 -0
- package/dist/embedding/local-identity.d.ts +1 -0
- package/dist/embedding/local-identity.js +15 -0
- package/dist/embedding/local.d.ts +34 -0
- package/dist/embedding/local.js +290 -0
- package/dist/embedding/openai.d.ts +79 -0
- package/dist/embedding/openai.js +477 -0
- package/dist/embedding/owner.d.ts +13 -0
- package/dist/embedding/owner.js +36 -0
- package/dist/embedding/provider.d.ts +68 -0
- package/dist/embedding/provider.js +16 -0
- package/dist/embedding/remote-chunking.d.ts +22 -0
- package/dist/embedding/remote-chunking.js +83 -0
- package/dist/embedding/remote-embedding.d.ts +15 -0
- package/dist/embedding/remote-embedding.js +77 -0
- package/dist/hybrid-llm.d.ts +18 -0
- package/dist/hybrid-llm.js +53 -0
- package/dist/index.d.ts +244 -0
- package/dist/index.js +418 -0
- package/dist/llm.d.ts +566 -0
- package/dist/llm.js +1847 -0
- package/dist/maintenance.d.ts +33 -0
- package/dist/maintenance.js +52 -0
- package/dist/mcp/origin-guard.d.ts +67 -0
- package/dist/mcp/origin-guard.js +137 -0
- package/dist/mcp/server.d.ts +116 -0
- package/dist/mcp/server.js +919 -0
- package/dist/paths.d.ts +1 -0
- package/dist/paths.js +4 -0
- package/dist/remote-llm.d.ts +52 -0
- package/dist/remote-llm.js +464 -0
- package/dist/search/cjk-analyzer.d.ts +33 -0
- package/dist/search/cjk-analyzer.js +158 -0
- package/dist/search/cjk-index.d.ts +104 -0
- package/dist/search/cjk-index.js +1031 -0
- package/dist/search/jieba-loader.d.ts +23 -0
- package/dist/search/jieba-loader.js +79 -0
- package/dist/search/query-expansion.d.ts +23 -0
- package/dist/search/query-expansion.js +43 -0
- package/dist/search/zh-dict.txt +624013 -0
- package/dist/store.d.ts +1218 -0
- package/dist/store.js +6076 -0
- package/dist/trust.d.ts +152 -0
- package/dist/trust.js +249 -0
- package/package.json +139 -0
- package/scripts/build.mjs +83 -0
- package/scripts/check-package-grammars.mjs +29 -0
- package/scripts/package-smoke.mjs +205 -0
- package/scripts/sync-zh-dict.mjs +187 -0
- package/scripts/test-all.mjs +45 -0
- package/skills/qmd/SKILL.md +324 -0
- package/skills/qmd/references/mcp-setup.md +119 -0
- package/skills/release/SKILL.md +141 -0
- package/skills/release/scripts/install-hooks.sh +38 -0
- package/skills/release/scripts/release-context.sh +129 -0
|
@@ -0,0 +1,321 @@
|
|
|
1
|
+
import { createHash } from "node:crypto";
|
|
2
|
+
export class EmbeddingIdentityStateError extends Error {
|
|
3
|
+
code;
|
|
4
|
+
constructor(code, message) {
|
|
5
|
+
super(message);
|
|
6
|
+
this.name = "EmbeddingIdentityStateError";
|
|
7
|
+
this.code = code;
|
|
8
|
+
}
|
|
9
|
+
}
|
|
10
|
+
function hasVerifiableStoredIdentity(row) {
|
|
11
|
+
if (!/^[0-9a-f]{64}$/.test(row.fingerprint))
|
|
12
|
+
return false;
|
|
13
|
+
if (createHash("sha256").update(row.canonical_material).digest("hex") !== row.fingerprint) {
|
|
14
|
+
return false;
|
|
15
|
+
}
|
|
16
|
+
try {
|
|
17
|
+
const material = JSON.parse(row.canonical_material);
|
|
18
|
+
return typeof material === "object" && material !== null && !Array.isArray(material);
|
|
19
|
+
}
|
|
20
|
+
catch {
|
|
21
|
+
return false;
|
|
22
|
+
}
|
|
23
|
+
}
|
|
24
|
+
export function createEmbeddingIdentity(input) {
|
|
25
|
+
if (input.providerId.trim() === "" || input.model.trim() === "" || input.canonicalMaterial === "") {
|
|
26
|
+
throw new EmbeddingIdentityStateError("INVALID_IDENTITY", "Embedding identity fields must not be empty.");
|
|
27
|
+
}
|
|
28
|
+
if (!Number.isInteger(input.dimension) || input.dimension < 1) {
|
|
29
|
+
throw new EmbeddingIdentityStateError("INVALID_IDENTITY", "Embedding identity dimension must be positive.");
|
|
30
|
+
}
|
|
31
|
+
const fingerprint = createHash("sha256").update(input.canonicalMaterial).digest("hex");
|
|
32
|
+
return Object.freeze({ ...input, fingerprint });
|
|
33
|
+
}
|
|
34
|
+
export function ensureEmbeddingIdentitySchema(db) {
|
|
35
|
+
db.exec(`
|
|
36
|
+
CREATE TABLE IF NOT EXISTS embedding_index_state (
|
|
37
|
+
singleton INTEGER PRIMARY KEY CHECK (singleton = 1),
|
|
38
|
+
fingerprint TEXT NOT NULL,
|
|
39
|
+
provider_id TEXT NOT NULL,
|
|
40
|
+
model TEXT NOT NULL,
|
|
41
|
+
dimension INTEGER NOT NULL CHECK (dimension > 0),
|
|
42
|
+
remote INTEGER NOT NULL CHECK (remote IN (0, 1)),
|
|
43
|
+
canonical_material TEXT NOT NULL,
|
|
44
|
+
status TEXT NOT NULL CHECK (status IN ('building', 'ready', 'incompatible')),
|
|
45
|
+
generation INTEGER NOT NULL CHECK (generation > 0),
|
|
46
|
+
lease_owner TEXT,
|
|
47
|
+
lease_expires_at INTEGER,
|
|
48
|
+
updated_at INTEGER NOT NULL
|
|
49
|
+
)
|
|
50
|
+
`);
|
|
51
|
+
const schema = db.prepare(`
|
|
52
|
+
SELECT sql FROM sqlite_master WHERE type = 'table' AND name = 'embedding_index_state'
|
|
53
|
+
`).get();
|
|
54
|
+
if (schema && !schema.sql.includes("'incompatible'")) {
|
|
55
|
+
db.transaction(() => {
|
|
56
|
+
db.exec(`ALTER TABLE embedding_index_state RENAME TO embedding_index_state_legacy`);
|
|
57
|
+
db.exec(`
|
|
58
|
+
CREATE TABLE embedding_index_state (
|
|
59
|
+
singleton INTEGER PRIMARY KEY CHECK (singleton = 1),
|
|
60
|
+
fingerprint TEXT NOT NULL,
|
|
61
|
+
provider_id TEXT NOT NULL,
|
|
62
|
+
model TEXT NOT NULL,
|
|
63
|
+
dimension INTEGER NOT NULL CHECK (dimension > 0),
|
|
64
|
+
remote INTEGER NOT NULL CHECK (remote IN (0, 1)),
|
|
65
|
+
canonical_material TEXT NOT NULL,
|
|
66
|
+
status TEXT NOT NULL CHECK (status IN ('building', 'ready', 'incompatible')),
|
|
67
|
+
generation INTEGER NOT NULL CHECK (generation > 0),
|
|
68
|
+
lease_owner TEXT,
|
|
69
|
+
lease_expires_at INTEGER,
|
|
70
|
+
updated_at INTEGER NOT NULL
|
|
71
|
+
)
|
|
72
|
+
`);
|
|
73
|
+
db.exec(`
|
|
74
|
+
INSERT INTO embedding_index_state(
|
|
75
|
+
singleton, fingerprint, provider_id, model, dimension, remote,
|
|
76
|
+
canonical_material, status, generation, lease_owner, lease_expires_at, updated_at
|
|
77
|
+
)
|
|
78
|
+
SELECT singleton, fingerprint, provider_id, model, dimension, remote,
|
|
79
|
+
canonical_material, status, generation, lease_owner, lease_expires_at, updated_at
|
|
80
|
+
FROM embedding_index_state_legacy
|
|
81
|
+
`);
|
|
82
|
+
db.exec(`DROP TABLE embedding_index_state_legacy`);
|
|
83
|
+
})();
|
|
84
|
+
}
|
|
85
|
+
const row = readState(db);
|
|
86
|
+
if (row && row.status !== "incompatible" && !hasVerifiableStoredIdentity(row)) {
|
|
87
|
+
db.prepare(`
|
|
88
|
+
UPDATE embedding_index_state
|
|
89
|
+
SET status = 'incompatible', lease_owner = NULL, lease_expires_at = NULL,
|
|
90
|
+
updated_at = ?
|
|
91
|
+
WHERE singleton = 1
|
|
92
|
+
`).run(Date.now());
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
function readState(db) {
|
|
96
|
+
return db.prepare(`
|
|
97
|
+
SELECT fingerprint, provider_id, model, dimension, remote, canonical_material,
|
|
98
|
+
status, generation, lease_owner, lease_expires_at
|
|
99
|
+
FROM embedding_index_state
|
|
100
|
+
WHERE singleton = 1
|
|
101
|
+
`).get();
|
|
102
|
+
}
|
|
103
|
+
function rowIdentity(row) {
|
|
104
|
+
return Object.freeze({
|
|
105
|
+
fingerprint: row.fingerprint,
|
|
106
|
+
providerId: row.provider_id,
|
|
107
|
+
model: row.model,
|
|
108
|
+
dimension: row.dimension,
|
|
109
|
+
remote: row.remote === 1,
|
|
110
|
+
canonicalMaterial: row.canonical_material,
|
|
111
|
+
});
|
|
112
|
+
}
|
|
113
|
+
export function readStoredEmbeddingIdentity(db) {
|
|
114
|
+
ensureEmbeddingIdentitySchema(db);
|
|
115
|
+
const row = readState(db);
|
|
116
|
+
return row ? rowIdentity(row) : undefined;
|
|
117
|
+
}
|
|
118
|
+
function embeddingRowsExist(db) {
|
|
119
|
+
const row = db.prepare("SELECT EXISTS(SELECT 1 FROM content_vectors LIMIT 1) AS present").get();
|
|
120
|
+
const vecTable = db.prepare(`
|
|
121
|
+
SELECT EXISTS(
|
|
122
|
+
SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'vectors_vec'
|
|
123
|
+
) AS present
|
|
124
|
+
`).get();
|
|
125
|
+
return row.present === 1 || vecTable.present === 1;
|
|
126
|
+
}
|
|
127
|
+
export function inspectEmbeddingIndexState(db, expected, now = Date.now()) {
|
|
128
|
+
ensureEmbeddingIdentitySchema(db);
|
|
129
|
+
const row = readState(db);
|
|
130
|
+
if (!row) {
|
|
131
|
+
return { status: embeddingRowsExist(db) ? "mismatch" : "empty" };
|
|
132
|
+
}
|
|
133
|
+
const identity = rowIdentity(row);
|
|
134
|
+
if (row.status === "incompatible") {
|
|
135
|
+
return { status: "incompatible", identity, generation: row.generation };
|
|
136
|
+
}
|
|
137
|
+
if (row.fingerprint !== expected.fingerprint) {
|
|
138
|
+
return { status: "mismatch", identity, generation: row.generation };
|
|
139
|
+
}
|
|
140
|
+
if (row.status === "ready") {
|
|
141
|
+
return { status: "ready", identity, generation: row.generation };
|
|
142
|
+
}
|
|
143
|
+
if (row.lease_owner && row.lease_expires_at !== null && row.lease_expires_at > now) {
|
|
144
|
+
return {
|
|
145
|
+
status: "building",
|
|
146
|
+
identity,
|
|
147
|
+
ownerId: row.lease_owner,
|
|
148
|
+
generation: row.generation,
|
|
149
|
+
leaseExpiresAt: row.lease_expires_at,
|
|
150
|
+
};
|
|
151
|
+
}
|
|
152
|
+
return { status: "partial", identity, generation: row.generation };
|
|
153
|
+
}
|
|
154
|
+
function validateLeaseOptions(options) {
|
|
155
|
+
if (options.ownerId.trim() === "" || !Number.isFinite(options.now)
|
|
156
|
+
|| !Number.isFinite(options.leaseMs) || options.leaseMs <= 0) {
|
|
157
|
+
throw new EmbeddingIdentityStateError("INVALID_LEASE", "Embedding build lease options are invalid.");
|
|
158
|
+
}
|
|
159
|
+
}
|
|
160
|
+
function clearEmbeddingData(db) {
|
|
161
|
+
db.prepare("DELETE FROM content_vectors").run();
|
|
162
|
+
db.exec("DROP TABLE IF EXISTS vectors_vec");
|
|
163
|
+
}
|
|
164
|
+
function ensureEmbeddingVectorTable(db, dimension) {
|
|
165
|
+
const table = db.prepare(`
|
|
166
|
+
SELECT sql FROM sqlite_master WHERE type = 'table' AND name = 'vectors_vec'
|
|
167
|
+
`).get();
|
|
168
|
+
if (table) {
|
|
169
|
+
const storedDimension = table.sql.match(/float\[(\d+)\]/)?.[1];
|
|
170
|
+
const hasHashSeq = table.sql.includes("hash_seq");
|
|
171
|
+
const hasCollection = table.sql.includes("collection");
|
|
172
|
+
const hasCosine = table.sql.includes("distance_metric=cosine");
|
|
173
|
+
if (storedDimension === String(dimension)
|
|
174
|
+
&& hasHashSeq
|
|
175
|
+
&& hasCollection
|
|
176
|
+
&& hasCosine) {
|
|
177
|
+
return;
|
|
178
|
+
}
|
|
179
|
+
if (storedDimension === String(dimension) && hasHashSeq && !hasCollection) {
|
|
180
|
+
// Auto-migrate legacy table without collection column
|
|
181
|
+
try {
|
|
182
|
+
db.exec("CREATE TEMP TABLE _qmd_migrate_vecs (hash_seq TEXT, embedding BLOB)");
|
|
183
|
+
db.exec("INSERT INTO _qmd_migrate_vecs SELECT hash_seq, embedding FROM vectors_vec");
|
|
184
|
+
db.exec("DROP TABLE vectors_vec");
|
|
185
|
+
db.exec(`CREATE VIRTUAL TABLE vectors_vec USING vec0(hash_seq TEXT PRIMARY KEY, collection TEXT, embedding float[${dimension}] distance_metric=cosine)`);
|
|
186
|
+
db.exec(`
|
|
187
|
+
INSERT INTO vectors_vec (hash_seq, collection, embedding)
|
|
188
|
+
SELECT
|
|
189
|
+
t.hash_seq,
|
|
190
|
+
COALESCE((
|
|
191
|
+
SELECT d.collection
|
|
192
|
+
FROM content_vectors cv
|
|
193
|
+
JOIN documents d ON d.hash = cv.hash AND d.active = 1
|
|
194
|
+
WHERE (cv.hash || '_' || cv.seq) = t.hash_seq
|
|
195
|
+
LIMIT 1
|
|
196
|
+
), ''),
|
|
197
|
+
t.embedding
|
|
198
|
+
FROM _qmd_migrate_vecs t
|
|
199
|
+
`);
|
|
200
|
+
db.exec("DROP TABLE IF EXISTS _qmd_migrate_vecs");
|
|
201
|
+
return;
|
|
202
|
+
}
|
|
203
|
+
catch {
|
|
204
|
+
db.exec("DROP TABLE IF EXISTS _qmd_migrate_vecs");
|
|
205
|
+
db.exec("DROP TABLE IF EXISTS vectors_vec");
|
|
206
|
+
}
|
|
207
|
+
}
|
|
208
|
+
else {
|
|
209
|
+
throw new EmbeddingIdentityStateError("IDENTITY_MISMATCH", "Stored vector table schema does not match the active embedding identity.");
|
|
210
|
+
}
|
|
211
|
+
}
|
|
212
|
+
db.exec(`
|
|
213
|
+
CREATE VIRTUAL TABLE vectors_vec USING vec0(
|
|
214
|
+
hash_seq TEXT PRIMARY KEY,
|
|
215
|
+
collection TEXT,
|
|
216
|
+
embedding float[${dimension}] distance_metric=cosine
|
|
217
|
+
)
|
|
218
|
+
`);
|
|
219
|
+
}
|
|
220
|
+
export function beginEmbeddingBuild(db, identity, options) {
|
|
221
|
+
validateLeaseOptions(options);
|
|
222
|
+
ensureEmbeddingIdentitySchema(db);
|
|
223
|
+
db.exec("BEGIN IMMEDIATE");
|
|
224
|
+
try {
|
|
225
|
+
options.beforeMutation?.();
|
|
226
|
+
let row = readState(db);
|
|
227
|
+
const hasEmbeddingData = embeddingRowsExist(db);
|
|
228
|
+
const mismatch = row
|
|
229
|
+
? row.status === "incompatible" || row.fingerprint !== identity.fingerprint
|
|
230
|
+
: hasEmbeddingData;
|
|
231
|
+
if (mismatch && options.requireForceForIdentityChange && !options.forceRebuild) {
|
|
232
|
+
throw new EmbeddingIdentityStateError("IDENTITY_MISMATCH", "Remote embedding identity changes require both force and destructive rebuild authorization.");
|
|
233
|
+
}
|
|
234
|
+
if (mismatch || options.forceRebuild) {
|
|
235
|
+
if (!options.allowDestructiveRebuild) {
|
|
236
|
+
throw new EmbeddingIdentityStateError("IDENTITY_MISMATCH", "Stored embedding identity does not match the active provider.");
|
|
237
|
+
}
|
|
238
|
+
clearEmbeddingData(db);
|
|
239
|
+
db.prepare("DELETE FROM embedding_index_state WHERE singleton = 1").run();
|
|
240
|
+
row = undefined;
|
|
241
|
+
}
|
|
242
|
+
// Vector table creation is part of publication state, not a follow-up.
|
|
243
|
+
// If this fails, any destructive reset and the new lease roll back together.
|
|
244
|
+
ensureEmbeddingVectorTable(db, identity.dimension);
|
|
245
|
+
options.afterVectorTablePrepared?.();
|
|
246
|
+
if (row?.lease_owner && row.lease_expires_at !== null
|
|
247
|
+
&& row.lease_expires_at > options.now && row.lease_owner !== options.ownerId) {
|
|
248
|
+
throw new EmbeddingIdentityStateError("LEASE_BUSY", `Embedding build lease is owned by ${row.lease_owner}.`);
|
|
249
|
+
}
|
|
250
|
+
const mode = row ? "resume" : "rebuild";
|
|
251
|
+
const preservePublishedReady = row?.status === "ready" && !mismatch && !options.forceRebuild;
|
|
252
|
+
const generation = (row?.generation ?? 0) + 1;
|
|
253
|
+
const leaseExpiresAt = options.now + options.leaseMs;
|
|
254
|
+
db.prepare(`
|
|
255
|
+
INSERT INTO embedding_index_state(
|
|
256
|
+
singleton, fingerprint, provider_id, model, dimension, remote,
|
|
257
|
+
canonical_material, status, generation, lease_owner, lease_expires_at, updated_at
|
|
258
|
+
) VALUES (1, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
259
|
+
ON CONFLICT(singleton) DO UPDATE SET
|
|
260
|
+
fingerprint = excluded.fingerprint,
|
|
261
|
+
provider_id = excluded.provider_id,
|
|
262
|
+
model = excluded.model,
|
|
263
|
+
dimension = excluded.dimension,
|
|
264
|
+
remote = excluded.remote,
|
|
265
|
+
canonical_material = excluded.canonical_material,
|
|
266
|
+
status = excluded.status,
|
|
267
|
+
generation = excluded.generation,
|
|
268
|
+
lease_owner = excluded.lease_owner,
|
|
269
|
+
lease_expires_at = excluded.lease_expires_at,
|
|
270
|
+
updated_at = excluded.updated_at
|
|
271
|
+
`).run(identity.fingerprint, identity.providerId, identity.model, identity.dimension, identity.remote ? 1 : 0, identity.canonicalMaterial, preservePublishedReady ? "ready" : "building", generation, options.ownerId, leaseExpiresAt, options.now);
|
|
272
|
+
db.exec("COMMIT");
|
|
273
|
+
return Object.freeze({
|
|
274
|
+
fingerprint: identity.fingerprint,
|
|
275
|
+
ownerId: options.ownerId,
|
|
276
|
+
generation,
|
|
277
|
+
leaseExpiresAt,
|
|
278
|
+
mode,
|
|
279
|
+
});
|
|
280
|
+
}
|
|
281
|
+
catch (error) {
|
|
282
|
+
db.exec("ROLLBACK");
|
|
283
|
+
throw error;
|
|
284
|
+
}
|
|
285
|
+
}
|
|
286
|
+
function updateOwnedLease(db, lease, sql, params) {
|
|
287
|
+
const result = db.prepare(sql).run(...params, lease.fingerprint, lease.ownerId, lease.generation);
|
|
288
|
+
if (result.changes !== 1) {
|
|
289
|
+
throw new EmbeddingIdentityStateError("LEASE_LOST", "Embedding build lease is no longer owned by this operation.");
|
|
290
|
+
}
|
|
291
|
+
}
|
|
292
|
+
export function renewEmbeddingBuildLease(db, lease, now, leaseMs) {
|
|
293
|
+
if (!Number.isFinite(leaseMs) || leaseMs <= 0) {
|
|
294
|
+
throw new EmbeddingIdentityStateError("INVALID_LEASE", "Embedding lease duration must be positive.");
|
|
295
|
+
}
|
|
296
|
+
const leaseExpiresAt = now + leaseMs;
|
|
297
|
+
updateOwnedLease(db, lease, `
|
|
298
|
+
UPDATE embedding_index_state
|
|
299
|
+
SET lease_expires_at = ?, updated_at = ?
|
|
300
|
+
WHERE singleton = 1 AND lease_expires_at > ?
|
|
301
|
+
AND fingerprint = ? AND lease_owner = ? AND generation = ?
|
|
302
|
+
`, [leaseExpiresAt, now, now]);
|
|
303
|
+
return Object.freeze({ ...lease, leaseExpiresAt });
|
|
304
|
+
}
|
|
305
|
+
export function completeEmbeddingBuild(db, lease, now = Date.now()) {
|
|
306
|
+
updateOwnedLease(db, lease, `
|
|
307
|
+
UPDATE embedding_index_state
|
|
308
|
+
SET status = 'ready', lease_owner = NULL, lease_expires_at = NULL, updated_at = ?
|
|
309
|
+
WHERE singleton = 1 AND lease_expires_at > ?
|
|
310
|
+
AND fingerprint = ? AND lease_owner = ? AND generation = ?
|
|
311
|
+
`, [now, now]);
|
|
312
|
+
}
|
|
313
|
+
export function abandonEmbeddingBuild(db, lease, now = Date.now()) {
|
|
314
|
+
updateOwnedLease(db, lease, `
|
|
315
|
+
UPDATE embedding_index_state
|
|
316
|
+
SET status = CASE WHEN status = 'ready' THEN 'ready' ELSE 'building' END,
|
|
317
|
+
lease_owner = NULL, lease_expires_at = NULL, updated_at = ?
|
|
318
|
+
WHERE singleton = 1 AND lease_expires_at > ?
|
|
319
|
+
AND fingerprint = ? AND lease_owner = ? AND generation = ?
|
|
320
|
+
`, [now, now]);
|
|
321
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export declare function canonicalLocalEmbeddingIdentityMaterial(model: string, dimension: number): string;
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
import { formatDocForEmbedding, formatQueryForEmbedding } from "../llm.js";
|
|
2
|
+
import { EmbeddingProviderError } from "./provider.js";
|
|
3
|
+
export function canonicalLocalEmbeddingIdentityMaterial(model, dimension) {
|
|
4
|
+
if (!Number.isInteger(dimension) || dimension < 1) {
|
|
5
|
+
throw new EmbeddingProviderError("DIMENSION_MISMATCH", "Embedding identity dimension must be a positive integer.");
|
|
6
|
+
}
|
|
7
|
+
return JSON.stringify({
|
|
8
|
+
provider: "local-llama-cpp",
|
|
9
|
+
model,
|
|
10
|
+
dimension,
|
|
11
|
+
remote: false,
|
|
12
|
+
query_format: formatQueryForEmbedding("__qmd_query_identity__", model),
|
|
13
|
+
document_format: formatDocForEmbedding("__qmd_document_identity__", "__qmd_document_identity_title__", model),
|
|
14
|
+
});
|
|
15
|
+
}
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
import { type ILLMSession, type LlamaCpp } from "../llm.js";
|
|
2
|
+
import { type EmbeddingOperationOptions, type EmbeddingProvider, type EmbeddingProviderOwner, type EmbeddingVector } from "./provider.js";
|
|
3
|
+
export interface LocalEmbeddingProviderOptions {
|
|
4
|
+
model?: string;
|
|
5
|
+
dimension?: number;
|
|
6
|
+
}
|
|
7
|
+
export declare class LocalEmbeddingProvider implements EmbeddingProvider {
|
|
8
|
+
readonly providerId = "local-llama-cpp";
|
|
9
|
+
readonly remote = false;
|
|
10
|
+
readonly model: string;
|
|
11
|
+
private readonly session;
|
|
12
|
+
canonicalIdentityMaterialForDimension(dimension: number): string;
|
|
13
|
+
private readonly closeController;
|
|
14
|
+
private currentDimension;
|
|
15
|
+
private closed;
|
|
16
|
+
constructor(session: ILLMSession, options?: LocalEmbeddingProviderOptions);
|
|
17
|
+
get dimension(): number | null;
|
|
18
|
+
canonicalIdentityMaterial(): string;
|
|
19
|
+
formatQuery(query: string): string;
|
|
20
|
+
formatDocument(text: string, title?: string): string;
|
|
21
|
+
embed(text: string, options: EmbeddingOperationOptions): Promise<EmbeddingVector>;
|
|
22
|
+
embedBatch(texts: string[], options: EmbeddingOperationOptions): Promise<EmbeddingVector[]>;
|
|
23
|
+
close(): Promise<void>;
|
|
24
|
+
private validateResult;
|
|
25
|
+
private runOperation;
|
|
26
|
+
private waitForOperation;
|
|
27
|
+
}
|
|
28
|
+
export declare class LocalEmbeddingProviderOwner implements EmbeddingProviderOwner {
|
|
29
|
+
readonly provider: EmbeddingProvider;
|
|
30
|
+
private readonly llm;
|
|
31
|
+
private closePromise;
|
|
32
|
+
constructor(llm: LlamaCpp, options?: LocalEmbeddingProviderOptions);
|
|
33
|
+
close(): Promise<void>;
|
|
34
|
+
}
|
|
@@ -0,0 +1,290 @@
|
|
|
1
|
+
import { formatDocForEmbedding, formatQueryForEmbedding, waitForLLMSessionsToDrain, withLLMSessionForLlm, } from "../llm.js";
|
|
2
|
+
import { EmbeddingProviderError, } from "./provider.js";
|
|
3
|
+
import { canonicalLocalEmbeddingIdentityMaterial } from "./local-identity.js";
|
|
4
|
+
const MAX_TIMER_DELAY_MS = 2_147_483_647;
|
|
5
|
+
export class LocalEmbeddingProvider {
|
|
6
|
+
providerId = "local-llama-cpp";
|
|
7
|
+
remote = false;
|
|
8
|
+
model;
|
|
9
|
+
session;
|
|
10
|
+
canonicalIdentityMaterialForDimension(dimension) {
|
|
11
|
+
return canonicalLocalEmbeddingIdentityMaterial(this.model, dimension);
|
|
12
|
+
}
|
|
13
|
+
closeController = new AbortController();
|
|
14
|
+
currentDimension;
|
|
15
|
+
closed = false;
|
|
16
|
+
constructor(session, options = {}) {
|
|
17
|
+
this.session = session;
|
|
18
|
+
const requestedModel = options.model ?? session.embeddingModel;
|
|
19
|
+
if (requestedModel !== session.embeddingModel) {
|
|
20
|
+
throw new EmbeddingProviderError("MODEL_MISMATCH", `Configured embedding model ${requestedModel} does not match session model ${session.embeddingModel}.`);
|
|
21
|
+
}
|
|
22
|
+
this.model = session.embeddingModel;
|
|
23
|
+
if (options.dimension !== undefined && (!Number.isInteger(options.dimension) || options.dimension < 1)) {
|
|
24
|
+
throw new EmbeddingProviderError("DIMENSION_MISMATCH", "Configured embedding dimension must be a positive integer.");
|
|
25
|
+
}
|
|
26
|
+
this.currentDimension = options.dimension ?? null;
|
|
27
|
+
}
|
|
28
|
+
get dimension() {
|
|
29
|
+
return this.currentDimension;
|
|
30
|
+
}
|
|
31
|
+
canonicalIdentityMaterial() {
|
|
32
|
+
if (this.dimension === null) {
|
|
33
|
+
throw new EmbeddingProviderError("DIMENSION_UNKNOWN", "Embedding provider identity is unavailable until its vector dimension is known.");
|
|
34
|
+
}
|
|
35
|
+
return JSON.stringify({
|
|
36
|
+
provider: this.providerId,
|
|
37
|
+
model: this.model,
|
|
38
|
+
dimension: this.dimension,
|
|
39
|
+
remote: this.remote,
|
|
40
|
+
query_format: this.formatQuery("__qmd_query_identity__"),
|
|
41
|
+
document_format: this.formatDocument("__qmd_document_identity__", "__qmd_document_identity_title__"),
|
|
42
|
+
});
|
|
43
|
+
}
|
|
44
|
+
formatQuery(query) {
|
|
45
|
+
return formatQueryForEmbedding(query, this.model);
|
|
46
|
+
}
|
|
47
|
+
formatDocument(text, title) {
|
|
48
|
+
return formatDocForEmbedding(text, title, this.model);
|
|
49
|
+
}
|
|
50
|
+
async embed(text, options) {
|
|
51
|
+
const result = await this.runOperation("embed", options, () => this.session.embed(text, {
|
|
52
|
+
model: this.model,
|
|
53
|
+
isQuery: options.kind === "query",
|
|
54
|
+
}));
|
|
55
|
+
const vector = this.validateResult(result, "embed", undefined, this.currentDimension);
|
|
56
|
+
if (this.currentDimension === null)
|
|
57
|
+
this.currentDimension = vector.dimension;
|
|
58
|
+
return vector;
|
|
59
|
+
}
|
|
60
|
+
async embedBatch(texts, options) {
|
|
61
|
+
const results = await this.runOperation("embedBatch", options, () => texts.length === 0
|
|
62
|
+
? Promise.resolve([])
|
|
63
|
+
: this.session.embedBatch(texts, {
|
|
64
|
+
model: this.model,
|
|
65
|
+
isQuery: options.kind === "query",
|
|
66
|
+
}));
|
|
67
|
+
if (results.length !== texts.length) {
|
|
68
|
+
throw new EmbeddingProviderError("BATCH_CARDINALITY_MISMATCH", `Embedding batch returned ${results.length} results for ${texts.length} inputs.`, { operation: "embedBatch" });
|
|
69
|
+
}
|
|
70
|
+
let batchDimension = this.currentDimension;
|
|
71
|
+
const vectors = results.map((result, index) => {
|
|
72
|
+
const vector = this.validateResult(result, "embedBatch", index, batchDimension);
|
|
73
|
+
batchDimension ??= vector.dimension;
|
|
74
|
+
return vector;
|
|
75
|
+
});
|
|
76
|
+
if (this.currentDimension === null)
|
|
77
|
+
this.currentDimension = batchDimension;
|
|
78
|
+
return vectors;
|
|
79
|
+
}
|
|
80
|
+
async close() {
|
|
81
|
+
if (this.closed)
|
|
82
|
+
return;
|
|
83
|
+
this.closed = true;
|
|
84
|
+
this.closeController.abort();
|
|
85
|
+
}
|
|
86
|
+
validateResult(result, operation, index, expectedDimension) {
|
|
87
|
+
if (result === null) {
|
|
88
|
+
throw new EmbeddingProviderError("MISSING_EMBEDDING", index === undefined
|
|
89
|
+
? "Embedding provider returned no vector."
|
|
90
|
+
: `Embedding provider returned no vector for batch item ${index}.`, { operation, index });
|
|
91
|
+
}
|
|
92
|
+
if (result.model !== this.model) {
|
|
93
|
+
throw new EmbeddingProviderError("MODEL_MISMATCH", `Embedding result model ${result.model} does not match provider model ${this.model}.`, { operation, index });
|
|
94
|
+
}
|
|
95
|
+
const vector = result.embedding;
|
|
96
|
+
if (!Array.isArray(vector) || vector.length === 0) {
|
|
97
|
+
throw new EmbeddingProviderError("EMPTY_VECTOR", "Embedding provider returned an empty vector.", { operation, index });
|
|
98
|
+
}
|
|
99
|
+
if (!vector.every(value => typeof value === "number" && Number.isFinite(value))) {
|
|
100
|
+
throw new EmbeddingProviderError("NON_FINITE_VECTOR", "Embedding provider returned a vector containing a non-finite value.", { operation, index });
|
|
101
|
+
}
|
|
102
|
+
if (expectedDimension !== null && vector.length !== expectedDimension) {
|
|
103
|
+
throw new EmbeddingProviderError("DIMENSION_MISMATCH", `Embedding vector dimension ${vector.length} does not match provider dimension ${expectedDimension}.`, { operation, index });
|
|
104
|
+
}
|
|
105
|
+
return {
|
|
106
|
+
vector: [...vector],
|
|
107
|
+
model: result.model,
|
|
108
|
+
dimension: vector.length,
|
|
109
|
+
};
|
|
110
|
+
}
|
|
111
|
+
async runOperation(operation, options, invoke) {
|
|
112
|
+
if (this.closed) {
|
|
113
|
+
throw new EmbeddingProviderError("PROVIDER_CLOSED", "Embedding provider is closed.", { operation });
|
|
114
|
+
}
|
|
115
|
+
const signals = [...new Set([options.signal, this.session.signal, this.closeController.signal]
|
|
116
|
+
.filter((signal) => signal !== undefined))];
|
|
117
|
+
if (signals.some(signal => signal.aborted)) {
|
|
118
|
+
throw new EmbeddingProviderError("OPERATION_ABORTED", "Embedding operation was aborted.", { operation });
|
|
119
|
+
}
|
|
120
|
+
if (options.deadline !== undefined) {
|
|
121
|
+
if (!Number.isFinite(options.deadline) || options.deadline <= Date.now()) {
|
|
122
|
+
throw new EmbeddingProviderError("DEADLINE_EXCEEDED", "Embedding operation deadline was exceeded.", { operation });
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
try {
|
|
126
|
+
return await this.waitForOperation(operation, signals, options.deadline, invoke);
|
|
127
|
+
}
|
|
128
|
+
catch (error) {
|
|
129
|
+
if (error instanceof EmbeddingProviderError)
|
|
130
|
+
throw error;
|
|
131
|
+
throw new EmbeddingProviderError("PROVIDER_FAILURE", "Embedding provider operation failed.", { operation });
|
|
132
|
+
}
|
|
133
|
+
}
|
|
134
|
+
waitForOperation(operation, signals, deadline, invoke) {
|
|
135
|
+
if (signals.length === 0 && deadline === undefined) {
|
|
136
|
+
return Promise.resolve().then(invoke);
|
|
137
|
+
}
|
|
138
|
+
return new Promise((resolve, reject) => {
|
|
139
|
+
let settled = false;
|
|
140
|
+
let deadlineTimer;
|
|
141
|
+
const cleanup = () => {
|
|
142
|
+
if (deadlineTimer !== undefined)
|
|
143
|
+
clearTimeout(deadlineTimer);
|
|
144
|
+
for (const signal of signals)
|
|
145
|
+
signal.removeEventListener("abort", onAbort);
|
|
146
|
+
};
|
|
147
|
+
const finish = (callback) => {
|
|
148
|
+
if (settled)
|
|
149
|
+
return;
|
|
150
|
+
settled = true;
|
|
151
|
+
cleanup();
|
|
152
|
+
callback();
|
|
153
|
+
};
|
|
154
|
+
const onAbort = () => finish(() => reject(new EmbeddingProviderError("OPERATION_ABORTED", "Embedding operation was aborted.", { operation })));
|
|
155
|
+
const scheduleDeadline = () => {
|
|
156
|
+
if (deadline === undefined || settled)
|
|
157
|
+
return;
|
|
158
|
+
const remaining = deadline - Date.now();
|
|
159
|
+
if (remaining <= 0) {
|
|
160
|
+
finish(() => reject(new EmbeddingProviderError("DEADLINE_EXCEEDED", "Embedding operation deadline was exceeded.", { operation })));
|
|
161
|
+
return;
|
|
162
|
+
}
|
|
163
|
+
deadlineTimer = setTimeout(scheduleDeadline, Math.min(remaining, MAX_TIMER_DELAY_MS));
|
|
164
|
+
deadlineTimer.unref?.();
|
|
165
|
+
};
|
|
166
|
+
for (const signal of signals)
|
|
167
|
+
signal.addEventListener("abort", onAbort, { once: true });
|
|
168
|
+
scheduleDeadline();
|
|
169
|
+
void Promise.resolve().then(() => {
|
|
170
|
+
if (settled)
|
|
171
|
+
return;
|
|
172
|
+
if (deadline !== undefined && deadline <= Date.now()) {
|
|
173
|
+
finish(() => reject(new EmbeddingProviderError("DEADLINE_EXCEEDED", "Embedding operation deadline was exceeded.", { operation })));
|
|
174
|
+
return;
|
|
175
|
+
}
|
|
176
|
+
try {
|
|
177
|
+
void invoke().then(value => finish(() => resolve(value)), error => finish(() => reject(error)));
|
|
178
|
+
}
|
|
179
|
+
catch (error) {
|
|
180
|
+
finish(() => reject(error));
|
|
181
|
+
}
|
|
182
|
+
});
|
|
183
|
+
});
|
|
184
|
+
}
|
|
185
|
+
}
|
|
186
|
+
class ScopedLocalEmbeddingProvider {
|
|
187
|
+
providerId = "local-llama-cpp";
|
|
188
|
+
remote = false;
|
|
189
|
+
model;
|
|
190
|
+
llm;
|
|
191
|
+
canonicalIdentityMaterialForDimension(dimension) {
|
|
192
|
+
return canonicalLocalEmbeddingIdentityMaterial(this.model, dimension);
|
|
193
|
+
}
|
|
194
|
+
closeController = new AbortController();
|
|
195
|
+
currentDimension;
|
|
196
|
+
closed = false;
|
|
197
|
+
constructor(llm, options = {}) {
|
|
198
|
+
this.llm = llm;
|
|
199
|
+
this.model = options.model ?? llm.embedModelName;
|
|
200
|
+
this.currentDimension = options.dimension ?? null;
|
|
201
|
+
}
|
|
202
|
+
get dimension() {
|
|
203
|
+
return this.currentDimension;
|
|
204
|
+
}
|
|
205
|
+
canonicalIdentityMaterial() {
|
|
206
|
+
if (this.dimension === null) {
|
|
207
|
+
throw new EmbeddingProviderError("DIMENSION_UNKNOWN", "Embedding provider identity is unavailable until its vector dimension is known.");
|
|
208
|
+
}
|
|
209
|
+
return JSON.stringify({
|
|
210
|
+
provider: this.providerId,
|
|
211
|
+
model: this.model,
|
|
212
|
+
dimension: this.dimension,
|
|
213
|
+
remote: this.remote,
|
|
214
|
+
query_format: this.formatQuery("__qmd_query_identity__"),
|
|
215
|
+
document_format: this.formatDocument("__qmd_document_identity__", "__qmd_document_identity_title__"),
|
|
216
|
+
});
|
|
217
|
+
}
|
|
218
|
+
formatQuery(query) {
|
|
219
|
+
return formatQueryForEmbedding(query, this.model);
|
|
220
|
+
}
|
|
221
|
+
formatDocument(text, title) {
|
|
222
|
+
return formatDocForEmbedding(text, title, this.model);
|
|
223
|
+
}
|
|
224
|
+
embed(text, options) {
|
|
225
|
+
return this.runScoped(options, (provider, scopedOptions) => (provider.embed(text, scopedOptions)));
|
|
226
|
+
}
|
|
227
|
+
embedBatch(texts, options) {
|
|
228
|
+
return this.runScoped(options, (provider, scopedOptions) => (provider.embedBatch(texts, scopedOptions)));
|
|
229
|
+
}
|
|
230
|
+
async close() {
|
|
231
|
+
if (this.closed)
|
|
232
|
+
return;
|
|
233
|
+
this.closed = true;
|
|
234
|
+
this.closeController.abort();
|
|
235
|
+
}
|
|
236
|
+
async runScoped(options, invoke) {
|
|
237
|
+
if (this.closed) {
|
|
238
|
+
throw new EmbeddingProviderError("PROVIDER_CLOSED", "Embedding provider is closed.");
|
|
239
|
+
}
|
|
240
|
+
const signal = options.signal
|
|
241
|
+
? AbortSignal.any([options.signal, this.closeController.signal])
|
|
242
|
+
: this.closeController.signal;
|
|
243
|
+
const maxDuration = options.deadline === undefined
|
|
244
|
+
? undefined
|
|
245
|
+
: Math.max(1, options.deadline - Date.now());
|
|
246
|
+
return withLLMSessionForLlm(this.llm, async (session) => {
|
|
247
|
+
const provider = new LocalEmbeddingProvider(session, {
|
|
248
|
+
model: this.model,
|
|
249
|
+
dimension: this.currentDimension ?? undefined,
|
|
250
|
+
});
|
|
251
|
+
try {
|
|
252
|
+
const result = await invoke(provider, { ...options, signal });
|
|
253
|
+
const vectors = Array.isArray(result) ? result : [result];
|
|
254
|
+
for (const vector of vectors)
|
|
255
|
+
this.captureDimension(vector.dimension);
|
|
256
|
+
return result;
|
|
257
|
+
}
|
|
258
|
+
finally {
|
|
259
|
+
await provider.close();
|
|
260
|
+
}
|
|
261
|
+
}, {
|
|
262
|
+
maxDuration,
|
|
263
|
+
});
|
|
264
|
+
}
|
|
265
|
+
captureDimension(dimension) {
|
|
266
|
+
if (this.currentDimension !== null && this.currentDimension !== dimension) {
|
|
267
|
+
throw new EmbeddingProviderError("DIMENSION_MISMATCH", `Embedding dimension changed from ${this.currentDimension} to ${dimension}.`);
|
|
268
|
+
}
|
|
269
|
+
this.currentDimension = dimension;
|
|
270
|
+
}
|
|
271
|
+
}
|
|
272
|
+
export class LocalEmbeddingProviderOwner {
|
|
273
|
+
provider;
|
|
274
|
+
llm;
|
|
275
|
+
closePromise;
|
|
276
|
+
constructor(llm, options = {}) {
|
|
277
|
+
this.llm = llm;
|
|
278
|
+
this.provider = new ScopedLocalEmbeddingProvider(llm, options);
|
|
279
|
+
}
|
|
280
|
+
close() {
|
|
281
|
+
if (!this.closePromise) {
|
|
282
|
+
this.closePromise = (async () => {
|
|
283
|
+
await this.provider.close();
|
|
284
|
+
await waitForLLMSessionsToDrain(this.llm);
|
|
285
|
+
await this.llm.dispose();
|
|
286
|
+
})();
|
|
287
|
+
}
|
|
288
|
+
return this.closePromise;
|
|
289
|
+
}
|
|
290
|
+
}
|