@wei840222/qmd 2026.8.23
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +1373 -0
- package/LICENSE +45 -0
- package/README.md +1439 -0
- package/THIRD_PARTY_NOTICES.md +31 -0
- package/bin/qmd +192 -0
- package/dist/ast.d.ts +65 -0
- package/dist/ast.js +334 -0
- package/dist/bench/bench.d.ts +35 -0
- package/dist/bench/bench.js +338 -0
- package/dist/bench/cjk-baseline.d.ts +36 -0
- package/dist/bench/cjk-baseline.js +111 -0
- package/dist/bench/fixture.d.ts +2 -0
- package/dist/bench/fixture.js +84 -0
- package/dist/bench/score.d.ts +38 -0
- package/dist/bench/score.js +107 -0
- package/dist/bench/types.d.ts +110 -0
- package/dist/bench/types.js +8 -0
- package/dist/cli/build-info.json +4 -0
- package/dist/cli/embed-lock.d.ts +24 -0
- package/dist/cli/embed-lock.js +94 -0
- package/dist/cli/embedding-owner.d.ts +10 -0
- package/dist/cli/embedding-owner.js +20 -0
- package/dist/cli/formatter.d.ts +120 -0
- package/dist/cli/formatter.js +355 -0
- package/dist/cli/mcp-pid.d.ts +25 -0
- package/dist/cli/mcp-pid.js +86 -0
- package/dist/cli/qmd.d.ts +72 -0
- package/dist/cli/qmd.js +4806 -0
- package/dist/cli/version.d.ts +42 -0
- package/dist/cli/version.js +80 -0
- package/dist/collections.d.ts +200 -0
- package/dist/collections.js +433 -0
- package/dist/db.d.ts +65 -0
- package/dist/db.js +143 -0
- package/dist/diagnostics.d.ts +62 -0
- package/dist/diagnostics.js +260 -0
- package/dist/embedding/config.d.ts +52 -0
- package/dist/embedding/config.js +229 -0
- package/dist/embedding/identity.d.ts +58 -0
- package/dist/embedding/identity.js +321 -0
- package/dist/embedding/local-identity.d.ts +1 -0
- package/dist/embedding/local-identity.js +15 -0
- package/dist/embedding/local.d.ts +34 -0
- package/dist/embedding/local.js +290 -0
- package/dist/embedding/openai.d.ts +79 -0
- package/dist/embedding/openai.js +477 -0
- package/dist/embedding/owner.d.ts +13 -0
- package/dist/embedding/owner.js +36 -0
- package/dist/embedding/provider.d.ts +68 -0
- package/dist/embedding/provider.js +16 -0
- package/dist/embedding/remote-chunking.d.ts +22 -0
- package/dist/embedding/remote-chunking.js +83 -0
- package/dist/embedding/remote-embedding.d.ts +15 -0
- package/dist/embedding/remote-embedding.js +77 -0
- package/dist/hybrid-llm.d.ts +18 -0
- package/dist/hybrid-llm.js +53 -0
- package/dist/index.d.ts +244 -0
- package/dist/index.js +418 -0
- package/dist/llm.d.ts +566 -0
- package/dist/llm.js +1847 -0
- package/dist/maintenance.d.ts +33 -0
- package/dist/maintenance.js +52 -0
- package/dist/mcp/origin-guard.d.ts +67 -0
- package/dist/mcp/origin-guard.js +137 -0
- package/dist/mcp/server.d.ts +116 -0
- package/dist/mcp/server.js +919 -0
- package/dist/paths.d.ts +1 -0
- package/dist/paths.js +4 -0
- package/dist/remote-llm.d.ts +52 -0
- package/dist/remote-llm.js +464 -0
- package/dist/search/cjk-analyzer.d.ts +33 -0
- package/dist/search/cjk-analyzer.js +158 -0
- package/dist/search/cjk-index.d.ts +104 -0
- package/dist/search/cjk-index.js +1031 -0
- package/dist/search/jieba-loader.d.ts +23 -0
- package/dist/search/jieba-loader.js +79 -0
- package/dist/search/query-expansion.d.ts +23 -0
- package/dist/search/query-expansion.js +43 -0
- package/dist/search/zh-dict.txt +624013 -0
- package/dist/store.d.ts +1218 -0
- package/dist/store.js +6076 -0
- package/dist/trust.d.ts +152 -0
- package/dist/trust.js +249 -0
- package/package.json +139 -0
- package/scripts/build.mjs +83 -0
- package/scripts/check-package-grammars.mjs +29 -0
- package/scripts/package-smoke.mjs +205 -0
- package/scripts/sync-zh-dict.mjs +187 -0
- package/scripts/test-all.mjs +45 -0
- package/skills/qmd/SKILL.md +324 -0
- package/skills/qmd/references/mcp-setup.md +119 -0
- package/skills/release/SKILL.md +141 -0
- package/skills/release/scripts/install-hooks.sh +38 -0
- package/skills/release/scripts/release-context.sh +129 -0
|
@@ -0,0 +1,260 @@
|
|
|
1
|
+
import { createHash } from "node:crypto";
|
|
2
|
+
import { createEmbeddingIdentity } from "./embedding/identity.js";
|
|
3
|
+
import { canonicalLocalEmbeddingIdentityMaterial } from "./embedding/local-identity.js";
|
|
4
|
+
import { canonicalOpenAIEmbeddingIdentityMaterial } from "./embedding/openai.js";
|
|
5
|
+
import { getCjkAnalyzerFingerprint } from "./search/cjk-index.js";
|
|
6
|
+
import { canonicalEmbeddingBuildMaterial } from "./store.js";
|
|
7
|
+
function tableExists(db, name) {
|
|
8
|
+
return db.prepare(`
|
|
9
|
+
SELECT 1 FROM sqlite_master WHERE type IN ('table', 'view') AND name = ?
|
|
10
|
+
`).get(name) != null;
|
|
11
|
+
}
|
|
12
|
+
function tableColumns(db, name) {
|
|
13
|
+
if (!tableExists(db, name))
|
|
14
|
+
return new Set();
|
|
15
|
+
return new Set(db.prepare(`PRAGMA table_info("${name}")`).all()
|
|
16
|
+
.map(column => column.name));
|
|
17
|
+
}
|
|
18
|
+
function countRows(db, table) {
|
|
19
|
+
if (!tableExists(db, table))
|
|
20
|
+
return null;
|
|
21
|
+
try {
|
|
22
|
+
const row = db.prepare(`SELECT COUNT(*) AS count FROM "${table}"`).get();
|
|
23
|
+
return Number(row.count);
|
|
24
|
+
}
|
|
25
|
+
catch (error) {
|
|
26
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
27
|
+
if (table === "vectors_vec" && /no such module:\s*vec0/i.test(message))
|
|
28
|
+
return null;
|
|
29
|
+
if (table.startsWith("documents_fts") && /no such column/i.test(message))
|
|
30
|
+
return null;
|
|
31
|
+
throw error;
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
function activeDocumentCount(db) {
|
|
35
|
+
if (!tableExists(db, "documents"))
|
|
36
|
+
return 0;
|
|
37
|
+
const row = db.prepare(`SELECT COUNT(*) AS count FROM documents WHERE active = 1`).get();
|
|
38
|
+
return Number(row.count);
|
|
39
|
+
}
|
|
40
|
+
function shortFingerprint(value) {
|
|
41
|
+
return value == null ? null : value.slice(0, 12);
|
|
42
|
+
}
|
|
43
|
+
function resolvedIdentity(provider, configured, stored) {
|
|
44
|
+
const providerId = provider?.providerId ?? configured?.id;
|
|
45
|
+
const model = provider?.model ?? configured?.model;
|
|
46
|
+
const remote = provider?.remote ?? configured?.remote;
|
|
47
|
+
if (providerId == null || model == null || remote == null)
|
|
48
|
+
return undefined;
|
|
49
|
+
const storedMatches = stored?.provider_id === providerId && stored.model === model;
|
|
50
|
+
const dimension = provider?.dimension ?? configured?.dimension
|
|
51
|
+
?? (storedMatches ? Number(stored.dimension) : null);
|
|
52
|
+
if (dimension == null)
|
|
53
|
+
return undefined;
|
|
54
|
+
try {
|
|
55
|
+
const providerMaterial = provider
|
|
56
|
+
? provider.canonicalIdentityMaterialForDimension?.(dimension)
|
|
57
|
+
?? provider.canonicalIdentityMaterial()
|
|
58
|
+
: providerId === "local-llama-cpp"
|
|
59
|
+
? canonicalLocalEmbeddingIdentityMaterial(model, dimension)
|
|
60
|
+
: providerId === "openai"
|
|
61
|
+
? canonicalOpenAIEmbeddingIdentityMaterial(model, dimension)
|
|
62
|
+
: undefined;
|
|
63
|
+
if (providerMaterial == null)
|
|
64
|
+
return undefined;
|
|
65
|
+
const identities = ["regex", "auto"].map(chunkStrategy => createEmbeddingIdentity({
|
|
66
|
+
providerId,
|
|
67
|
+
model,
|
|
68
|
+
dimension,
|
|
69
|
+
remote,
|
|
70
|
+
canonicalMaterial: canonicalEmbeddingBuildMaterial(providerMaterial, chunkStrategy),
|
|
71
|
+
}));
|
|
72
|
+
return identities.find(identity => identity.fingerprint === stored?.fingerprint)
|
|
73
|
+
?? identities[0];
|
|
74
|
+
}
|
|
75
|
+
catch {
|
|
76
|
+
return undefined;
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
function inspectEmbeddingDiagnostics(db, fallbackModel, provider, keyConfigured, configured) {
|
|
80
|
+
const stateColumns = tableColumns(db, "embedding_index_state");
|
|
81
|
+
const stored = stateColumns.has("fingerprint")
|
|
82
|
+
? db.prepare(`
|
|
83
|
+
SELECT fingerprint, provider_id, model, dimension, remote, status,
|
|
84
|
+
generation, lease_expires_at
|
|
85
|
+
FROM embedding_index_state WHERE singleton = 1
|
|
86
|
+
`).get()
|
|
87
|
+
: undefined;
|
|
88
|
+
const expected = resolvedIdentity(provider, configured, stored);
|
|
89
|
+
const model = provider?.model ?? configured?.model ?? stored?.model ?? fallbackModel;
|
|
90
|
+
const fingerprint = expected?.fingerprint
|
|
91
|
+
?? (stored != null
|
|
92
|
+
&& stored.provider_id === provider?.providerId
|
|
93
|
+
&& stored.model === model
|
|
94
|
+
? stored.fingerprint
|
|
95
|
+
: null);
|
|
96
|
+
const activeDocs = activeDocumentCount(db);
|
|
97
|
+
const metadataColumns = tableColumns(db, "content_vectors");
|
|
98
|
+
const hasMetadata = metadataColumns.has("hash") && metadataColumns.has("seq");
|
|
99
|
+
const hasFingerprint = metadataColumns.has("embed_fingerprint");
|
|
100
|
+
const hasLayouts = metadataColumns.has("total_chunks");
|
|
101
|
+
const hasVectors = tableExists(db, "vectors_vec");
|
|
102
|
+
const vectorChunkCount = countRows(db, "vectors_vec");
|
|
103
|
+
const vectorsReadable = vectorChunkCount != null;
|
|
104
|
+
const hasEmbeddingRows = (countRows(db, "content_vectors") ?? 0) > 0
|
|
105
|
+
|| (vectorChunkCount ?? 0) > 0;
|
|
106
|
+
let metadataOnly = 0;
|
|
107
|
+
let vectorOnly = 0;
|
|
108
|
+
let incompleteLayouts = 0;
|
|
109
|
+
let pendingDocuments = activeDocs;
|
|
110
|
+
if (hasMetadata) {
|
|
111
|
+
if (vectorsReadable) {
|
|
112
|
+
const cvRows = db.prepare(`SELECT hash, seq FROM content_vectors`).all();
|
|
113
|
+
const vvRows = db.prepare(`SELECT hash_seq FROM vectors_vec`).all();
|
|
114
|
+
const vvKeys = new Set(vvRows.map(r => r.hash_seq));
|
|
115
|
+
const cvKeys = new Set(cvRows.map(r => `${r.hash}_${r.seq}`));
|
|
116
|
+
for (const cv of cvRows) {
|
|
117
|
+
if (!vvKeys.has(`${cv.hash}_${cv.seq}`))
|
|
118
|
+
metadataOnly++;
|
|
119
|
+
}
|
|
120
|
+
for (const vv of vvRows) {
|
|
121
|
+
if (!cvKeys.has(vv.hash_seq))
|
|
122
|
+
vectorOnly++;
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
if (hasLayouts) {
|
|
126
|
+
incompleteLayouts = Number(db.prepare(`
|
|
127
|
+
SELECT COUNT(*) AS count FROM (
|
|
128
|
+
SELECT hash, model
|
|
129
|
+
FROM content_vectors
|
|
130
|
+
GROUP BY hash, model, embed_fingerprint
|
|
131
|
+
HAVING COUNT(*) <> MAX(total_chunks)
|
|
132
|
+
)
|
|
133
|
+
`).get().count);
|
|
134
|
+
}
|
|
135
|
+
if (fingerprint != null && hasFingerprint && hasLayouts) {
|
|
136
|
+
pendingDocuments = Number(db.prepare(`
|
|
137
|
+
SELECT COUNT(*) AS count
|
|
138
|
+
FROM documents d
|
|
139
|
+
LEFT JOIN (
|
|
140
|
+
SELECT cv.hash
|
|
141
|
+
FROM content_vectors cv
|
|
142
|
+
WHERE cv.model = ? AND cv.embed_fingerprint = ?
|
|
143
|
+
GROUP BY cv.hash
|
|
144
|
+
HAVING COUNT(*) = MAX(cv.total_chunks)
|
|
145
|
+
) complete ON complete.hash = d.hash
|
|
146
|
+
WHERE d.active = 1 AND complete.hash IS NULL
|
|
147
|
+
`).get(model, fingerprint).count);
|
|
148
|
+
}
|
|
149
|
+
}
|
|
150
|
+
const compatible = stored != null && fingerprint != null && stored.fingerprint === fingerprint;
|
|
151
|
+
let buildState;
|
|
152
|
+
if (!tableExists(db, "embedding_index_state")) {
|
|
153
|
+
buildState = hasEmbeddingRows ? "incompatible" : "missing";
|
|
154
|
+
}
|
|
155
|
+
else if (!stored) {
|
|
156
|
+
buildState = hasEmbeddingRows ? "incompatible" : "empty";
|
|
157
|
+
}
|
|
158
|
+
else if (!compatible) {
|
|
159
|
+
buildState = "incompatible";
|
|
160
|
+
}
|
|
161
|
+
else if (stored.status === "building") {
|
|
162
|
+
buildState = stored.lease_expires_at != null && stored.lease_expires_at > Date.now()
|
|
163
|
+
? "building"
|
|
164
|
+
: "partial";
|
|
165
|
+
}
|
|
166
|
+
else if (metadataOnly > 0 || vectorOnly > 0 || incompleteLayouts > 0 || pendingDocuments > 0) {
|
|
167
|
+
buildState = "partial";
|
|
168
|
+
}
|
|
169
|
+
else {
|
|
170
|
+
buildState = "ready";
|
|
171
|
+
}
|
|
172
|
+
const effectiveRemote = provider?.remote ?? configured?.remote ?? false;
|
|
173
|
+
return {
|
|
174
|
+
provider: {
|
|
175
|
+
id: provider?.providerId ?? configured?.id ?? stored?.provider_id ?? null,
|
|
176
|
+
remote: provider?.remote ?? configured?.remote ?? stored?.remote === 1,
|
|
177
|
+
model,
|
|
178
|
+
dimension: provider?.dimension ?? configured?.dimension ?? (stored ? Number(stored.dimension) : null),
|
|
179
|
+
keyConfigured: (provider?.remote ?? configured?.remote) === true && keyConfigured,
|
|
180
|
+
},
|
|
181
|
+
identity: {
|
|
182
|
+
shortFingerprint: shortFingerprint(fingerprint),
|
|
183
|
+
fullFingerprint: fingerprint,
|
|
184
|
+
storedShortFingerprint: shortFingerprint(stored?.fingerprint ?? null),
|
|
185
|
+
storedFullFingerprint: stored?.fingerprint ?? null,
|
|
186
|
+
compatible,
|
|
187
|
+
},
|
|
188
|
+
build: {
|
|
189
|
+
state: buildState,
|
|
190
|
+
generation: stored ? Number(stored.generation) : null,
|
|
191
|
+
leaseExpiresAt: stored?.lease_expires_at == null ? null : Number(stored.lease_expires_at),
|
|
192
|
+
},
|
|
193
|
+
chunks: { pendingDocuments, metadataOnly, vectorOnly, incompleteLayouts },
|
|
194
|
+
repairCommand: buildState === "ready" || buildState === "empty" ? null : effectiveRemote
|
|
195
|
+
? "qmd embed --force"
|
|
196
|
+
: "qmd embed --force",
|
|
197
|
+
};
|
|
198
|
+
}
|
|
199
|
+
function channelReadiness(rowCount, activeDocs, state, fingerprintMatches, wordChannel) {
|
|
200
|
+
if (rowCount == null)
|
|
201
|
+
return "missing";
|
|
202
|
+
if (wordChannel && state?.word_capability === "unavailable")
|
|
203
|
+
return "incompatible";
|
|
204
|
+
if (wordChannel && (!fingerprintMatches || state?.status === "dirty"))
|
|
205
|
+
return "stale";
|
|
206
|
+
if (rowCount !== activeDocs)
|
|
207
|
+
return "partial";
|
|
208
|
+
return "ready";
|
|
209
|
+
}
|
|
210
|
+
function inspectLexicalDiagnostics(db) {
|
|
211
|
+
const activeDocs = activeDocumentCount(db);
|
|
212
|
+
const currentFingerprint = getCjkAnalyzerFingerprint();
|
|
213
|
+
const stateColumns = tableColumns(db, "cjk_index_state");
|
|
214
|
+
const state = stateColumns.has("analyzer_fingerprint")
|
|
215
|
+
? db.prepare(`
|
|
216
|
+
SELECT status, analyzer_fingerprint, word_capability, diagnostic_code,
|
|
217
|
+
dirty_since_mutation_seq
|
|
218
|
+
FROM cjk_index_state WHERE singleton = 1
|
|
219
|
+
`).get()
|
|
220
|
+
: undefined;
|
|
221
|
+
const fingerprintMatches = activeDocs === 0
|
|
222
|
+
|| state?.analyzer_fingerprint === currentFingerprint;
|
|
223
|
+
const channels = {
|
|
224
|
+
char: channelReadiness(countRows(db, "documents_fts"), activeDocs, state, true, false),
|
|
225
|
+
word: channelReadiness(countRows(db, "documents_fts_words"), activeDocs, state, fingerprintMatches, true),
|
|
226
|
+
bigram: channelReadiness(countRows(db, "documents_fts_bigrams"), activeDocs, state, fingerprintMatches, true),
|
|
227
|
+
};
|
|
228
|
+
const rebuildReason = state?.status === "dirty"
|
|
229
|
+
? state.diagnostic_code ?? "dirty"
|
|
230
|
+
: activeDocs === 0
|
|
231
|
+
? null
|
|
232
|
+
: !state
|
|
233
|
+
? "schema-missing"
|
|
234
|
+
: !fingerprintMatches
|
|
235
|
+
? "analyzer-fingerprint-mismatch"
|
|
236
|
+
: Object.values(channels).some(channel => channel !== "ready")
|
|
237
|
+
? "channel-readiness-incomplete"
|
|
238
|
+
: null;
|
|
239
|
+
return {
|
|
240
|
+
jiebaCapability: state?.word_capability ?? "unknown",
|
|
241
|
+
analyzerFingerprint: currentFingerprint,
|
|
242
|
+
storedAnalyzerFingerprint: state?.analyzer_fingerprint ?? null,
|
|
243
|
+
state: state?.status ?? "missing",
|
|
244
|
+
channels,
|
|
245
|
+
dirtySinceMutationSeq: state?.dirty_since_mutation_seq == null
|
|
246
|
+
? null
|
|
247
|
+
: Number(state.dirty_since_mutation_seq),
|
|
248
|
+
rebuildReason,
|
|
249
|
+
repairCommand: rebuildReason == null ? null : "qmd update",
|
|
250
|
+
};
|
|
251
|
+
}
|
|
252
|
+
export function inspectIndexDiagnostics(db, options) {
|
|
253
|
+
return {
|
|
254
|
+
embedding: inspectEmbeddingDiagnostics(db, options.fallbackModel, options.provider, options.keyConfigured === true, options.configuredProvider),
|
|
255
|
+
lexical: inspectLexicalDiagnostics(db),
|
|
256
|
+
};
|
|
257
|
+
}
|
|
258
|
+
export function diagnosticsSnapshotHash(diagnostics) {
|
|
259
|
+
return createHash("sha256").update(JSON.stringify(diagnostics)).digest("hex");
|
|
260
|
+
}
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
import type { Database } from "../db.js";
|
|
2
|
+
export declare const OPENAI_EMBEDDING_MODEL: "text-embedding-3-small";
|
|
3
|
+
export declare const OPENAI_EMBEDDING_DIMENSION: 1536;
|
|
4
|
+
export declare const EMBEDDING_CONFIG_DB_KEY: "embedding_config";
|
|
5
|
+
export declare const DEFAULT_OPENAI_BASE_URL: "https://api.openai.com/v1";
|
|
6
|
+
/** Supported OpenAI embedding models and their native dimensions. */
|
|
7
|
+
export declare const OPENAI_EMBEDDING_MODELS: ReadonlyMap<string, number>;
|
|
8
|
+
export type OpenAIEmbeddingModel = "text-embedding-3-small" | "text-embedding-3-large";
|
|
9
|
+
export type EmbeddingProviderName = "local" | "openai";
|
|
10
|
+
export type EmbeddingConfig = {
|
|
11
|
+
provider: "local";
|
|
12
|
+
model?: string;
|
|
13
|
+
dimension?: number;
|
|
14
|
+
} | {
|
|
15
|
+
provider: "openai";
|
|
16
|
+
model?: OpenAIEmbeddingModel;
|
|
17
|
+
dimension?: number;
|
|
18
|
+
baseUrl?: string;
|
|
19
|
+
};
|
|
20
|
+
export type CanonicalEmbeddingConfig = Readonly<{
|
|
21
|
+
provider: "local";
|
|
22
|
+
model: string;
|
|
23
|
+
dimension: number | null;
|
|
24
|
+
} | {
|
|
25
|
+
provider: "openai";
|
|
26
|
+
model: OpenAIEmbeddingModel;
|
|
27
|
+
dimension: number;
|
|
28
|
+
baseUrl: string;
|
|
29
|
+
}>;
|
|
30
|
+
export type EmbeddingConfigSource = "embedding-block" | "legacy-models" | "database" | "local-default";
|
|
31
|
+
export interface ResolvedEmbeddingConfig {
|
|
32
|
+
readonly canonical: CanonicalEmbeddingConfig;
|
|
33
|
+
readonly source: EmbeddingConfigSource;
|
|
34
|
+
readonly credentialAvailable: boolean;
|
|
35
|
+
/** Remote transport is configured; consent and purpose guards still apply per request. */
|
|
36
|
+
readonly remoteRequestsEnabled: boolean;
|
|
37
|
+
}
|
|
38
|
+
export declare class EmbeddingConfigError extends Error {
|
|
39
|
+
constructor(message: string);
|
|
40
|
+
}
|
|
41
|
+
export interface ResolveEmbeddingConfigOptions {
|
|
42
|
+
/** Per-store SDK inline/configPath or CLI YAML snapshot. */
|
|
43
|
+
config?: unknown;
|
|
44
|
+
/** Canonical, non-secret configuration restored from SQLite. */
|
|
45
|
+
dbConfig?: unknown;
|
|
46
|
+
defaultLocalModel: string;
|
|
47
|
+
env?: Readonly<Record<string, string | undefined>>;
|
|
48
|
+
}
|
|
49
|
+
export declare function resolveEmbeddingConfig(options: ResolveEmbeddingConfigOptions): ResolvedEmbeddingConfig;
|
|
50
|
+
export declare function resolveEmbeddingModelOverride(resolved: ResolvedEmbeddingConfig, override: string | undefined): string;
|
|
51
|
+
export declare function readCanonicalEmbeddingConfig(db: Database): CanonicalEmbeddingConfig | undefined;
|
|
52
|
+
export declare function writeCanonicalEmbeddingConfig(db: Database, config: CanonicalEmbeddingConfig): void;
|
|
@@ -0,0 +1,229 @@
|
|
|
1
|
+
export const OPENAI_EMBEDDING_MODEL = "text-embedding-3-small";
|
|
2
|
+
export const OPENAI_EMBEDDING_DIMENSION = 1536;
|
|
3
|
+
export const EMBEDDING_CONFIG_DB_KEY = "embedding_config";
|
|
4
|
+
export const DEFAULT_OPENAI_BASE_URL = "https://api.openai.com/v1";
|
|
5
|
+
/** Supported OpenAI embedding models and their native dimensions. */
|
|
6
|
+
export const OPENAI_EMBEDDING_MODELS = new Map([
|
|
7
|
+
["text-embedding-3-small", 1536],
|
|
8
|
+
["text-embedding-3-large", 3072],
|
|
9
|
+
]);
|
|
10
|
+
export class EmbeddingConfigError extends Error {
|
|
11
|
+
constructor(message) {
|
|
12
|
+
super(message);
|
|
13
|
+
this.name = "EmbeddingConfigError";
|
|
14
|
+
}
|
|
15
|
+
}
|
|
16
|
+
function isRecord(value) {
|
|
17
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
18
|
+
}
|
|
19
|
+
function hasOwn(value, key) {
|
|
20
|
+
return Object.prototype.hasOwnProperty.call(value, key);
|
|
21
|
+
}
|
|
22
|
+
function requireRecord(value, label) {
|
|
23
|
+
if (!isRecord(value)) {
|
|
24
|
+
throw new EmbeddingConfigError(`${label} must be an object.`);
|
|
25
|
+
}
|
|
26
|
+
return value;
|
|
27
|
+
}
|
|
28
|
+
function requireModel(value, label) {
|
|
29
|
+
if (typeof value !== "string" || value.trim() === "") {
|
|
30
|
+
throw new EmbeddingConfigError(`${label} must be a non-empty string.`);
|
|
31
|
+
}
|
|
32
|
+
return value;
|
|
33
|
+
}
|
|
34
|
+
function parseDimension(value, label) {
|
|
35
|
+
if (!Number.isInteger(value) || value < 1) {
|
|
36
|
+
throw new EmbeddingConfigError(`${label} must be a positive integer.`);
|
|
37
|
+
}
|
|
38
|
+
return value;
|
|
39
|
+
}
|
|
40
|
+
function requireBaseUrl(value, label) {
|
|
41
|
+
if (typeof value !== "string" || value.trim() === "") {
|
|
42
|
+
throw new EmbeddingConfigError(`${label} must be a non-empty string.`);
|
|
43
|
+
}
|
|
44
|
+
return value.trim().replace(/\/+$/, "");
|
|
45
|
+
}
|
|
46
|
+
function assertKnownKeys(value, label) {
|
|
47
|
+
const allowed = new Set(["provider", "model", "dimension", "baseUrl"]);
|
|
48
|
+
const unknown = Object.keys(value).filter(key => !allowed.has(key));
|
|
49
|
+
if (unknown.length > 0) {
|
|
50
|
+
throw new EmbeddingConfigError(`${label} contains unsupported field ${unknown[0]}.`);
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
function parseEmbeddingBlock(input, defaultLocalModel, label, requireCanonicalValues) {
|
|
54
|
+
const value = requireRecord(input, label);
|
|
55
|
+
assertKnownKeys(value, label);
|
|
56
|
+
const rawProvider = hasOwn(value, "provider") ? value.provider : undefined;
|
|
57
|
+
const rawBaseUrl = hasOwn(value, "baseUrl") ? value.baseUrl : undefined;
|
|
58
|
+
// Infer provider: "openai" if provider is omitted but baseUrl is present
|
|
59
|
+
const provider = rawProvider ?? (rawBaseUrl ? "openai" : "local");
|
|
60
|
+
if (provider === "local") {
|
|
61
|
+
const model = hasOwn(value, "model")
|
|
62
|
+
? requireModel(value.model, `${label}.model`)
|
|
63
|
+
: requireCanonicalValues
|
|
64
|
+
? requireModel(undefined, `${label}.model`)
|
|
65
|
+
: defaultLocalModel;
|
|
66
|
+
const dimension = hasOwn(value, "dimension")
|
|
67
|
+
? value.dimension === null && requireCanonicalValues
|
|
68
|
+
? null
|
|
69
|
+
: parseDimension(value.dimension, `${label}.dimension`)
|
|
70
|
+
: requireCanonicalValues
|
|
71
|
+
? null
|
|
72
|
+
: null;
|
|
73
|
+
return Object.freeze({ provider: "local", model, dimension });
|
|
74
|
+
}
|
|
75
|
+
if (provider === "openai") {
|
|
76
|
+
const model = hasOwn(value, "model")
|
|
77
|
+
? requireModel(value.model, `${label}.model`)
|
|
78
|
+
: OPENAI_EMBEDDING_MODEL;
|
|
79
|
+
const expectedDimension = OPENAI_EMBEDDING_MODELS.get(model);
|
|
80
|
+
if (expectedDimension === undefined) {
|
|
81
|
+
const supported = Array.from(OPENAI_EMBEDDING_MODELS.keys()).join(", ");
|
|
82
|
+
throw new EmbeddingConfigError(`${label}.model must be one of: ${supported}. Got: ${model}`);
|
|
83
|
+
}
|
|
84
|
+
const dimension = hasOwn(value, "dimension")
|
|
85
|
+
? parseDimension(value.dimension, `${label}.dimension`)
|
|
86
|
+
: expectedDimension;
|
|
87
|
+
if (dimension !== expectedDimension) {
|
|
88
|
+
throw new EmbeddingConfigError(`${label}.dimension must be ${expectedDimension} for model ${model}.`);
|
|
89
|
+
}
|
|
90
|
+
const baseUrl = rawBaseUrl
|
|
91
|
+
? requireBaseUrl(rawBaseUrl, `${label}.baseUrl`)
|
|
92
|
+
: DEFAULT_OPENAI_BASE_URL;
|
|
93
|
+
return Object.freeze({
|
|
94
|
+
provider: "openai",
|
|
95
|
+
model: model,
|
|
96
|
+
dimension: expectedDimension,
|
|
97
|
+
baseUrl,
|
|
98
|
+
});
|
|
99
|
+
}
|
|
100
|
+
throw new EmbeddingConfigError(`${label}.provider must be local or openai.`);
|
|
101
|
+
}
|
|
102
|
+
function resolveSource(options) {
|
|
103
|
+
const defaultLocalModel = requireModel(options.defaultLocalModel, "default local embedding model");
|
|
104
|
+
if (options.config !== undefined) {
|
|
105
|
+
const config = requireRecord(options.config, "embedding config source");
|
|
106
|
+
if (hasOwn(config, "embedding")) {
|
|
107
|
+
const canonical = parseEmbeddingBlock(config.embedding, defaultLocalModel, "embedding config", false);
|
|
108
|
+
if (canonical.provider === "openai") {
|
|
109
|
+
throw new EmbeddingConfigError("embedding config no longer selects remote embeddings; configure models.embed_api_url and models.embed_api_model instead.");
|
|
110
|
+
}
|
|
111
|
+
return {
|
|
112
|
+
canonical,
|
|
113
|
+
source: "embedding-block",
|
|
114
|
+
};
|
|
115
|
+
}
|
|
116
|
+
if (hasOwn(config, "models")) {
|
|
117
|
+
const models = requireRecord(config.models, "models");
|
|
118
|
+
const hasEmbed = hasOwn(models, "embed");
|
|
119
|
+
const embedModel = hasEmbed ? requireModel(models.embed, "models.embed") : undefined;
|
|
120
|
+
if (embedModel === "openai" || embedModel?.startsWith("openai:")) {
|
|
121
|
+
throw new EmbeddingConfigError("models.embed no longer accepts OpenAI shorthand; configure models.embed_api_url (or models.embed_url/models.embed_base_url) and models.embed_api_model instead.");
|
|
122
|
+
}
|
|
123
|
+
const rawEmbedBaseUrl = models.embed_url ?? models.embed_base_url ?? models.embed_api_url;
|
|
124
|
+
const hasEmbedBaseUrl = rawEmbedBaseUrl !== undefined && String(rawEmbedBaseUrl).trim() !== "";
|
|
125
|
+
const customDimension = hasOwn(models, "embed_dimension")
|
|
126
|
+
? parseDimension(models.embed_dimension, "models.embed_dimension")
|
|
127
|
+
: undefined;
|
|
128
|
+
const embedApiModel = hasOwn(models, "embed_api_model")
|
|
129
|
+
? requireModel(models.embed_api_model, "models.embed_api_model")
|
|
130
|
+
: undefined;
|
|
131
|
+
if (hasEmbedBaseUrl && embedApiModel !== undefined) {
|
|
132
|
+
return {
|
|
133
|
+
canonical: parseEmbeddingBlock({
|
|
134
|
+
provider: "openai",
|
|
135
|
+
model: embedApiModel,
|
|
136
|
+
...(customDimension === undefined ? {} : { dimension: customDimension }),
|
|
137
|
+
baseUrl: requireBaseUrl(rawEmbedBaseUrl, "models.embed_api_url"),
|
|
138
|
+
}, defaultLocalModel, "models", false),
|
|
139
|
+
source: "legacy-models",
|
|
140
|
+
};
|
|
141
|
+
}
|
|
142
|
+
if (hasEmbed || hasEmbedBaseUrl || embedApiModel !== undefined || customDimension !== undefined) {
|
|
143
|
+
return {
|
|
144
|
+
canonical: Object.freeze({
|
|
145
|
+
provider: "local",
|
|
146
|
+
model: embedModel ?? defaultLocalModel,
|
|
147
|
+
dimension: customDimension ?? null,
|
|
148
|
+
}),
|
|
149
|
+
source: "legacy-models",
|
|
150
|
+
};
|
|
151
|
+
}
|
|
152
|
+
}
|
|
153
|
+
}
|
|
154
|
+
if (options.dbConfig !== undefined) {
|
|
155
|
+
return {
|
|
156
|
+
canonical: parseEmbeddingBlock(options.dbConfig, defaultLocalModel, "database embedding config", true),
|
|
157
|
+
source: "database",
|
|
158
|
+
};
|
|
159
|
+
}
|
|
160
|
+
return {
|
|
161
|
+
canonical: Object.freeze({
|
|
162
|
+
provider: "local",
|
|
163
|
+
model: defaultLocalModel,
|
|
164
|
+
dimension: null,
|
|
165
|
+
}),
|
|
166
|
+
source: "local-default",
|
|
167
|
+
};
|
|
168
|
+
}
|
|
169
|
+
export function resolveEmbeddingConfig(options) {
|
|
170
|
+
const resolved = resolveSource(options);
|
|
171
|
+
const env = options.env ?? process.env;
|
|
172
|
+
const hasApiKey = typeof env.OPENAI_API_KEY === "string" && env.OPENAI_API_KEY.trim() !== "";
|
|
173
|
+
// For non-default endpoints (self-hosted / proxy), API key is optional
|
|
174
|
+
const isNonDefaultEndpoint = resolved.canonical.provider === "openai"
|
|
175
|
+
&& resolved.canonical.baseUrl !== DEFAULT_OPENAI_BASE_URL;
|
|
176
|
+
const credentialAvailable = resolved.canonical.provider === "local"
|
|
177
|
+
|| hasApiKey
|
|
178
|
+
|| isNonDefaultEndpoint;
|
|
179
|
+
return Object.freeze({
|
|
180
|
+
canonical: resolved.canonical,
|
|
181
|
+
source: resolved.source,
|
|
182
|
+
credentialAvailable,
|
|
183
|
+
remoteRequestsEnabled: resolved.canonical.provider === "openai" && credentialAvailable,
|
|
184
|
+
});
|
|
185
|
+
}
|
|
186
|
+
export function resolveEmbeddingModelOverride(resolved, override) {
|
|
187
|
+
if (override === undefined)
|
|
188
|
+
return resolved.canonical.model;
|
|
189
|
+
const model = requireModel(override, "embedding model override");
|
|
190
|
+
if (resolved.canonical.provider === "openai" && model !== resolved.canonical.model) {
|
|
191
|
+
throw new EmbeddingConfigError(`OpenAI embedding model override must be ${resolved.canonical.model}.`);
|
|
192
|
+
}
|
|
193
|
+
return model;
|
|
194
|
+
}
|
|
195
|
+
export function readCanonicalEmbeddingConfig(db) {
|
|
196
|
+
try {
|
|
197
|
+
const row = db.prepare("SELECT value FROM store_config WHERE key = ?").get(EMBEDDING_CONFIG_DB_KEY);
|
|
198
|
+
if (row?.value === undefined)
|
|
199
|
+
return undefined;
|
|
200
|
+
if (typeof row.value !== "string") {
|
|
201
|
+
throw new EmbeddingConfigError("database embedding config must be stored as JSON text.");
|
|
202
|
+
}
|
|
203
|
+
let parsed;
|
|
204
|
+
try {
|
|
205
|
+
parsed = JSON.parse(row.value);
|
|
206
|
+
}
|
|
207
|
+
catch {
|
|
208
|
+
throw new EmbeddingConfigError("database embedding config contains invalid JSON.");
|
|
209
|
+
}
|
|
210
|
+
return parseEmbeddingBlock(parsed, "unused-for-canonical-config", "database embedding config", true);
|
|
211
|
+
}
|
|
212
|
+
catch (err) {
|
|
213
|
+
if (err instanceof EmbeddingConfigError)
|
|
214
|
+
throw err;
|
|
215
|
+
return undefined;
|
|
216
|
+
}
|
|
217
|
+
}
|
|
218
|
+
export function writeCanonicalEmbeddingConfig(db, config) {
|
|
219
|
+
const canonical = parseEmbeddingBlock(config, "unused-for-canonical-config", "canonical embedding config", true);
|
|
220
|
+
const newValue = JSON.stringify(canonical);
|
|
221
|
+
const existing = db.prepare(`SELECT value FROM store_config WHERE key = ?`).get(EMBEDDING_CONFIG_DB_KEY);
|
|
222
|
+
if (existing?.value === newValue)
|
|
223
|
+
return;
|
|
224
|
+
db.prepare(`
|
|
225
|
+
INSERT INTO store_config(key, value)
|
|
226
|
+
VALUES (?, ?)
|
|
227
|
+
ON CONFLICT(key) DO UPDATE SET value = excluded.value
|
|
228
|
+
`).run(EMBEDDING_CONFIG_DB_KEY, newValue);
|
|
229
|
+
}
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
import type { Database } from "../db.js";
|
|
2
|
+
export type EmbeddingBuildMode = "rebuild" | "resume";
|
|
3
|
+
export type EmbeddingIndexStateStatus = "empty" | "building" | "partial" | "ready" | "incompatible" | "mismatch";
|
|
4
|
+
export interface EmbeddingIdentity {
|
|
5
|
+
readonly fingerprint: string;
|
|
6
|
+
readonly providerId: string;
|
|
7
|
+
readonly model: string;
|
|
8
|
+
readonly dimension: number;
|
|
9
|
+
readonly remote: boolean;
|
|
10
|
+
readonly canonicalMaterial: string;
|
|
11
|
+
}
|
|
12
|
+
export interface EmbeddingBuildLease {
|
|
13
|
+
readonly fingerprint: string;
|
|
14
|
+
readonly ownerId: string;
|
|
15
|
+
readonly generation: number;
|
|
16
|
+
readonly leaseExpiresAt: number;
|
|
17
|
+
readonly mode: EmbeddingBuildMode;
|
|
18
|
+
}
|
|
19
|
+
export interface EmbeddingIndexState {
|
|
20
|
+
readonly status: EmbeddingIndexStateStatus;
|
|
21
|
+
readonly identity?: EmbeddingIdentity;
|
|
22
|
+
readonly ownerId?: string;
|
|
23
|
+
readonly generation?: number;
|
|
24
|
+
readonly leaseExpiresAt?: number;
|
|
25
|
+
}
|
|
26
|
+
export type EmbeddingIdentityStateErrorCode = "IDENTITY_MISMATCH" | "LEASE_BUSY" | "LEASE_LOST" | "INVALID_IDENTITY" | "INVALID_LEASE";
|
|
27
|
+
export declare class EmbeddingIdentityStateError extends Error {
|
|
28
|
+
readonly code: EmbeddingIdentityStateErrorCode;
|
|
29
|
+
constructor(code: EmbeddingIdentityStateErrorCode, message: string);
|
|
30
|
+
}
|
|
31
|
+
export interface BeginEmbeddingBuildOptions {
|
|
32
|
+
ownerId: string;
|
|
33
|
+
now: number;
|
|
34
|
+
leaseMs: number;
|
|
35
|
+
allowDestructiveRebuild: boolean;
|
|
36
|
+
/** Force a same-identity global reset inside this transaction. */
|
|
37
|
+
forceRebuild?: boolean;
|
|
38
|
+
/** Remote identity changes require force independently from destructive permission. */
|
|
39
|
+
requireForceForIdentityChange?: boolean;
|
|
40
|
+
/** Runs under BEGIN IMMEDIATE before any identity state or vector data mutation. */
|
|
41
|
+
beforeMutation?: () => void;
|
|
42
|
+
/** Fault-injection/observability hook after vector-table preparation, still under the writer transaction. */
|
|
43
|
+
afterVectorTablePrepared?: () => void;
|
|
44
|
+
}
|
|
45
|
+
export declare function createEmbeddingIdentity(input: {
|
|
46
|
+
providerId: string;
|
|
47
|
+
model: string;
|
|
48
|
+
dimension: number;
|
|
49
|
+
remote: boolean;
|
|
50
|
+
canonicalMaterial: string;
|
|
51
|
+
}): EmbeddingIdentity;
|
|
52
|
+
export declare function ensureEmbeddingIdentitySchema(db: Database): void;
|
|
53
|
+
export declare function readStoredEmbeddingIdentity(db: Database): EmbeddingIdentity | undefined;
|
|
54
|
+
export declare function inspectEmbeddingIndexState(db: Database, expected: EmbeddingIdentity, now?: number): EmbeddingIndexState;
|
|
55
|
+
export declare function beginEmbeddingBuild(db: Database, identity: EmbeddingIdentity, options: BeginEmbeddingBuildOptions): EmbeddingBuildLease;
|
|
56
|
+
export declare function renewEmbeddingBuildLease(db: Database, lease: EmbeddingBuildLease, now: number, leaseMs: number): EmbeddingBuildLease;
|
|
57
|
+
export declare function completeEmbeddingBuild(db: Database, lease: EmbeddingBuildLease, now?: number): void;
|
|
58
|
+
export declare function abandonEmbeddingBuild(db: Database, lease: EmbeddingBuildLease, now?: number): void;
|