@mfdaves/okf-mcp 0.9.1 → 0.12.0-rc.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +69 -0
- package/README.md +172 -17
- package/bin/okf-mcp.js +6 -1
- package/okf/okf-mcp/policies/authoring-safety.md +11 -1
- package/package.json +17 -5
- package/server.json +2 -2
- package/src/assets.js +165 -73
- package/src/authoring/candidates.js +391 -0
- package/src/authoring/git-preflight.js +225 -0
- package/src/authoring/git-publish.js +776 -0
- package/src/authoring/live.js +789 -0
- package/src/authoring/paths.js +112 -0
- package/src/authoring/policy-hook.js +221 -0
- package/src/authoring/proposal-policy.js +137 -0
- package/src/authoring/receipts.js +329 -0
- package/src/authoring/render.js +38 -0
- package/src/authoring/shared.js +75 -0
- package/src/authoring/staging.js +216 -0
- package/src/authoring.js +386 -287
- package/src/cache/files.js +147 -0
- package/src/cache/lock.js +201 -0
- package/src/cache/paths.js +197 -0
- package/src/cache/store.js +703 -0
- package/src/catalog-text.js +19 -0
- package/src/cli.js +184 -151
- package/src/computation.js +99 -61
- package/src/context/assemble.js +567 -0
- package/src/context/cli-options.js +88 -0
- package/src/context/score.js +234 -0
- package/src/context/sections.js +274 -0
- package/src/embed/interface.js +160 -0
- package/src/embed/openai.js +715 -0
- package/src/embed/prepare.js +182 -0
- package/src/embed/units.js +96 -0
- package/src/git-source.js +112 -52
- package/src/graph.js +95 -37
- package/src/http-server.js +36 -17
- package/src/index.js +15 -0
- package/src/indexer/assemble.js +566 -0
- package/src/indexer/build.js +290 -0
- package/src/indexer/cache.js +81 -0
- package/src/indexer/edges.js +430 -0
- package/src/indexer/index.js +44 -0
- package/src/indexer/paths.js +178 -0
- package/src/indexer/resolve.js +282 -0
- package/src/indexer/update.js +720 -0
- package/src/indexer.js +3 -1068
- package/src/live-authoring.js +4 -2329
- package/src/markdown.js +21 -16
- package/src/mcp/registry.js +328 -0
- package/src/mcp/resources.js +45 -0
- package/src/mcp/server.js +472 -0
- package/src/mcp/tools/catalog.js +244 -0
- package/src/mcp/tools/computation.js +83 -0
- package/src/mcp/tools/context.js +133 -0
- package/src/mcp/tools/graph.js +167 -0
- package/src/mcp/tools/live-authoring.js +316 -0
- package/src/mcp/tools/migration.js +86 -0
- package/src/mcp/tools/producers.js +79 -0
- package/src/mcp/tools/proposals.js +213 -0
- package/src/mcp/tools/provenance.js +87 -0
- package/src/mcp/tools/remote.js +79 -0
- package/src/mcp/tools/search.js +242 -0
- package/src/mcp-server.js +3 -1611
- package/src/migration.js +431 -253
- package/src/parser.js +245 -130
- package/src/plugins.js +71 -48
- package/src/policy/write-policy.js +713 -0
- package/src/producer-publisher.js +195 -77
- package/src/producers.js +265 -101
- package/src/project.js +277 -93
- package/src/remote.js +96 -61
- package/src/search/cli-options.js +107 -0
- package/src/search/confidence.js +61 -0
- package/src/search/dense.js +304 -0
- package/src/search/embedding-cli.js +57 -0
- package/src/search/embedding-config.js +217 -0
- package/src/search/fusion.js +249 -0
- package/src/search/hybrid.js +526 -0
- package/src/search/languages.js +200 -0
- package/src/search/lexical.js +695 -0
- package/src/search/query.js +620 -0
- package/src/search/stem/among.js +43 -0
- package/src/search/stem/base-stemmer.js +311 -0
- package/src/search/stem/en-regions.js +456 -0
- package/src/search/stem/en-suffixes.js +664 -0
- package/src/search/stem/en-tables.js +176 -0
- package/src/search/stem/en.js +238 -0
- package/src/search/stem/it-regions.js +459 -0
- package/src/search/stem/it-suffixes.js +471 -0
- package/src/search/stem/it-tables.js +257 -0
- package/src/search/stem/it.js +155 -0
- package/src/search-index.js +2 -81
- package/src/search.js +2 -252
- package/src/store.js +134 -81
- package/src/v02.js +301 -105
- package/src/validation.js +24 -20
|
@@ -0,0 +1,304 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
|
|
3
|
+
// Dense scoring for hybrid retrieval (docs/plan-1.0.md R-9, R-10): brute-force
|
|
4
|
+
// cosine similarity between a query vector and the cached vectors of the
|
|
5
|
+
// retrieval units that belong to the candidate concepts. No ANN (R-12): at
|
|
6
|
+
// 20k vectors of 512 floats a full scan is a few milliseconds.
|
|
7
|
+
//
|
|
8
|
+
// Units are { id, text, hash } (src/embed/units.js). The cache keys rows by
|
|
9
|
+
// content hash only, so the unit -> row mapping is recomputed from the index
|
|
10
|
+
// (R-9) and memoised per index object and unit function; a cache mutation
|
|
11
|
+
// (new rows, pruning, a new generation) remaps on the next query. A unit id
|
|
12
|
+
// is a concept URI or `uri#anchor` for a section; a concept scores with its
|
|
13
|
+
// best unit, which is reported so section-level callers can cite it.
|
|
14
|
+
|
|
15
|
+
const { EmbedderError } = require("../embed/interface");
|
|
16
|
+
const { conceptUnits, contextualConceptUnits } = require("../embed/units");
|
|
17
|
+
|
|
18
|
+
// `embedUnits` names: whole concepts (default, chosen on the dev split) or sections
|
|
19
|
+
// (src/context/sections.js, one unit per section anchored `uri#slug`, the lead as `uri`;
|
|
20
|
+
// better on synonym queries, worse on keyword ones in the WP3c measurement).
|
|
21
|
+
// WP3c-2 adds the contextual variants: sections and concepts after a bounded prefix naming the
|
|
22
|
+
// concept's title, type and path (src/embed/units.js `contextPrefix`), each with its own hashes.
|
|
23
|
+
const EMBED_UNITS = Object.freeze([
|
|
24
|
+
"concepts",
|
|
25
|
+
"sections",
|
|
26
|
+
"contextual-sections",
|
|
27
|
+
"contextual-concepts",
|
|
28
|
+
]);
|
|
29
|
+
const DEFAULT_EMBED_UNITS = "concepts";
|
|
30
|
+
|
|
31
|
+
// index -> Map(units function -> memo)
|
|
32
|
+
const UNIT_MEMOS = new WeakMap();
|
|
33
|
+
|
|
34
|
+
function conceptUriOf(index, unitId) {
|
|
35
|
+
if (index.byUri?.has(unitId)) return unitId;
|
|
36
|
+
const anchor = unitId.indexOf("#");
|
|
37
|
+
return anchor === -1 ? unitId : unitId.slice(0, anchor);
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
function validUnit(unit) {
|
|
41
|
+
return (
|
|
42
|
+
unit &&
|
|
43
|
+
typeof unit.id === "string" &&
|
|
44
|
+
unit.id &&
|
|
45
|
+
typeof unit.text === "string" &&
|
|
46
|
+
typeof unit.hash === "string" &&
|
|
47
|
+
/^[a-f0-9]{64}$/.test(unit.hash)
|
|
48
|
+
);
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
// The units of an index with their concept grouping; computed once per index
|
|
52
|
+
// object and units function (hashing every unit text is the expensive part).
|
|
53
|
+
function unitMemo(index, unitsOf) {
|
|
54
|
+
const fn = typeof unitsOf === "function" ? unitsOf : conceptUnits;
|
|
55
|
+
let perIndex = UNIT_MEMOS.get(index);
|
|
56
|
+
if (!perIndex) {
|
|
57
|
+
perIndex = new Map();
|
|
58
|
+
UNIT_MEMOS.set(index, perIndex);
|
|
59
|
+
}
|
|
60
|
+
let memo = perIndex.get(fn);
|
|
61
|
+
if (!memo) {
|
|
62
|
+
const raw = fn(index);
|
|
63
|
+
if (!Array.isArray(raw) || !raw.every(validUnit)) {
|
|
64
|
+
throw new EmbedderError(
|
|
65
|
+
"The units function must return { id, text, hash } entries.",
|
|
66
|
+
"invalid_units",
|
|
67
|
+
);
|
|
68
|
+
}
|
|
69
|
+
const units = raw.map((unit) => ({ id: unit.id, text: unit.text, hash: unit.hash }));
|
|
70
|
+
const byConcept = new Map();
|
|
71
|
+
units.forEach((unit, position) => {
|
|
72
|
+
const uri = conceptUriOf(index, unit.id);
|
|
73
|
+
if (!byConcept.has(uri)) byConcept.set(uri, []);
|
|
74
|
+
byConcept.get(uri).push(position);
|
|
75
|
+
});
|
|
76
|
+
memo = { units, byConcept, hashes: new Set(units.map((unit) => unit.hash)), rows: null };
|
|
77
|
+
perIndex.set(fn, memo);
|
|
78
|
+
}
|
|
79
|
+
return memo;
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
// Row offsets and norms for the memo's units in the given cache, recomputed
|
|
83
|
+
// whenever the cache's backing store, size or generation changed.
|
|
84
|
+
function rowMapping(memo, cache) {
|
|
85
|
+
const key = memo.rows;
|
|
86
|
+
if (
|
|
87
|
+
key &&
|
|
88
|
+
key.cache === cache &&
|
|
89
|
+
key.data === cache.data &&
|
|
90
|
+
key.size === cache.size &&
|
|
91
|
+
key.generation === cache.generation
|
|
92
|
+
) {
|
|
93
|
+
return key;
|
|
94
|
+
}
|
|
95
|
+
const dimension = cache.dimension;
|
|
96
|
+
const offsets = new Int32Array(memo.units.length).fill(-1);
|
|
97
|
+
const norms = new Float32Array(memo.units.length);
|
|
98
|
+
const data = cache.data;
|
|
99
|
+
let present = 0;
|
|
100
|
+
memo.units.forEach((unit, position) => {
|
|
101
|
+
const view = cache.get(unit.hash);
|
|
102
|
+
if (!view) return;
|
|
103
|
+
// The view is a subarray of cache.data: its offset is the row's offset.
|
|
104
|
+
const offset = view.byteOffset / Float32Array.BYTES_PER_ELEMENT - data.byteOffset / 4;
|
|
105
|
+
offsets[position] = offset;
|
|
106
|
+
let sum = 0;
|
|
107
|
+
for (let index = 0; index < dimension; index += 1) sum += view[index] * view[index];
|
|
108
|
+
norms[position] = Math.sqrt(sum);
|
|
109
|
+
present += 1;
|
|
110
|
+
});
|
|
111
|
+
memo.rows = {
|
|
112
|
+
cache,
|
|
113
|
+
data,
|
|
114
|
+
size: cache.size,
|
|
115
|
+
generation: cache.generation,
|
|
116
|
+
offsets,
|
|
117
|
+
norms,
|
|
118
|
+
present,
|
|
119
|
+
};
|
|
120
|
+
return memo.rows;
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
// How many of the index's units have a vector in the cache.
|
|
124
|
+
function denseCoverage(index, cache, units) {
|
|
125
|
+
const memo = unitMemo(index, units);
|
|
126
|
+
if (!cache?.size) return { units: memo.units.length, present: 0 };
|
|
127
|
+
return { units: memo.units.length, present: rowMapping(memo, cache).present };
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
// Resolves an `embedUnits` name (or a units function) to a units function.
|
|
131
|
+
function resolveUnits(value) {
|
|
132
|
+
if (typeof value === "function") return value;
|
|
133
|
+
const name = value === undefined || value === null ? DEFAULT_EMBED_UNITS : value;
|
|
134
|
+
if (!EMBED_UNITS.includes(name)) {
|
|
135
|
+
throw new EmbedderError(
|
|
136
|
+
`embedUnits must be one of ${EMBED_UNITS.join(", ")}.`,
|
|
137
|
+
"invalid_embed_units",
|
|
138
|
+
);
|
|
139
|
+
}
|
|
140
|
+
if (name === "concepts") return conceptUnits;
|
|
141
|
+
if (name === "contextual-concepts") return contextualConceptUnits;
|
|
142
|
+
let sections = null;
|
|
143
|
+
try {
|
|
144
|
+
const module = require("../context/sections");
|
|
145
|
+
sections = name === "contextual-sections" ? module.contextualSectionUnits : module.sectionUnits;
|
|
146
|
+
} catch {
|
|
147
|
+
sections = null;
|
|
148
|
+
}
|
|
149
|
+
if (typeof sections !== "function") {
|
|
150
|
+
throw new EmbedderError(
|
|
151
|
+
"Section units are not available in this build; use embedUnits: concepts.",
|
|
152
|
+
"embed_units_unavailable",
|
|
153
|
+
);
|
|
154
|
+
}
|
|
155
|
+
return sections;
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
function vectorNorm(vector) {
|
|
159
|
+
let sum = 0;
|
|
160
|
+
for (let index = 0; index < vector.length; index += 1) sum += vector[index] * vector[index];
|
|
161
|
+
return Math.sqrt(sum);
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
// Dot product of the query with one cached row; four accumulators let the
|
|
165
|
+
// engine overlap the multiplications (about 20% faster at 512 dimensions).
|
|
166
|
+
function dot(query, data, offset, dimension) {
|
|
167
|
+
let a = 0;
|
|
168
|
+
let b = 0;
|
|
169
|
+
let c = 0;
|
|
170
|
+
let d = 0;
|
|
171
|
+
let index = 0;
|
|
172
|
+
const end = dimension - (dimension % 4);
|
|
173
|
+
for (; index < end; index += 4) {
|
|
174
|
+
a += query[index] * data[offset + index];
|
|
175
|
+
b += query[index + 1] * data[offset + index + 1];
|
|
176
|
+
c += query[index + 2] * data[offset + index + 2];
|
|
177
|
+
d += query[index + 3] * data[offset + index + 3];
|
|
178
|
+
}
|
|
179
|
+
for (; index < dimension; index += 1) a += query[index] * data[offset + index];
|
|
180
|
+
return a + b + c + d;
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
// options:
|
|
184
|
+
// queryVector Float32Array (or number[]) of the cache's dimension
|
|
185
|
+
// cache an open vector cache (src/cache/store.js)
|
|
186
|
+
// units units function (default conceptUnits); sections later
|
|
187
|
+
// candidates concept documents to score (default: applyFilters(index, filters))
|
|
188
|
+
// filters search filters (types, tagsAny, bundle, ...), used without candidates
|
|
189
|
+
// limit keep the best `limit` concepts (default: all)
|
|
190
|
+
// Returns [{ id, score, unit }] sorted by cosine descending, ties by URI.
|
|
191
|
+
function denseSearch(index, options) {
|
|
192
|
+
const config = options || {};
|
|
193
|
+
const cache = config.cache;
|
|
194
|
+
const query = config.queryVector;
|
|
195
|
+
if (!cache || typeof cache.get !== "function" || !Number.isSafeInteger(cache.dimension)) {
|
|
196
|
+
throw new EmbedderError("Dense search requires an open vector cache.", "invalid_cache");
|
|
197
|
+
}
|
|
198
|
+
if (!query || typeof query.length !== "number" || query.length !== cache.dimension) {
|
|
199
|
+
throw new EmbedderError(
|
|
200
|
+
"The query vector does not match the cache dimension.",
|
|
201
|
+
"embedder_dimension_mismatch",
|
|
202
|
+
{ expected: cache.dimension, received: query ? query.length : null },
|
|
203
|
+
);
|
|
204
|
+
}
|
|
205
|
+
if (!cache.size) return [];
|
|
206
|
+
const memo = unitMemo(index, config.units);
|
|
207
|
+
const mapping = rowMapping(memo, cache);
|
|
208
|
+
if (!mapping.present) return [];
|
|
209
|
+
const queryNorm = vectorNorm(query);
|
|
210
|
+
if (!(queryNorm > 0)) return [];
|
|
211
|
+
const candidates = Array.isArray(config.candidates)
|
|
212
|
+
? config.candidates
|
|
213
|
+
: // Required lazily: query.js requires this module for hybrid ranking.
|
|
214
|
+
require("./query").applyFilters(index, config.filters || {});
|
|
215
|
+
const { data, offsets, norms } = mapping;
|
|
216
|
+
const dimension = cache.dimension;
|
|
217
|
+
// Per-candidate best score and unit go to scratch arrays kept on the memo,
|
|
218
|
+
// so a query allocates only its result objects (for the kept candidates).
|
|
219
|
+
if (!memo.scratch || memo.scratch.scores.length < candidates.length) {
|
|
220
|
+
memo.scratch = {
|
|
221
|
+
scores: new Float64Array(candidates.length),
|
|
222
|
+
units: new Int32Array(candidates.length),
|
|
223
|
+
};
|
|
224
|
+
}
|
|
225
|
+
const { scores, units } = memo.scratch;
|
|
226
|
+
let scored = 0;
|
|
227
|
+
for (let position = 0; position < candidates.length; position += 1) {
|
|
228
|
+
const unitPositions = memo.byConcept.get(candidates[position].uri);
|
|
229
|
+
let best = -Infinity;
|
|
230
|
+
let bestUnit = -1;
|
|
231
|
+
if (unitPositions) {
|
|
232
|
+
for (let next = 0; next < unitPositions.length; next += 1) {
|
|
233
|
+
const unit = unitPositions[next];
|
|
234
|
+
const offset = offsets[unit];
|
|
235
|
+
const norm = norms[unit];
|
|
236
|
+
if (offset < 0 || !(norm > 0)) continue;
|
|
237
|
+
const score = dot(query, data, offset, dimension) / (queryNorm * norm);
|
|
238
|
+
if (score > best) {
|
|
239
|
+
best = score;
|
|
240
|
+
bestUnit = unit;
|
|
241
|
+
}
|
|
242
|
+
}
|
|
243
|
+
}
|
|
244
|
+
scores[position] = best;
|
|
245
|
+
units[position] = bestUnit;
|
|
246
|
+
if (bestUnit !== -1) scored += 1;
|
|
247
|
+
}
|
|
248
|
+
const limit = Number.isSafeInteger(config.limit) && config.limit > 0 ? config.limit : scored;
|
|
249
|
+
const kept = selectTop(candidates, scores, units, Math.min(limit, scored));
|
|
250
|
+
return kept.map((position) => ({
|
|
251
|
+
id: candidates[position].uri,
|
|
252
|
+
score: scores[position],
|
|
253
|
+
unit: memo.units[units[position]].id,
|
|
254
|
+
}));
|
|
255
|
+
}
|
|
256
|
+
|
|
257
|
+
// Positions of the best `count` scored candidates, best first (ties by URI),
|
|
258
|
+
// through a bounded min-heap: O(n log count) instead of sorting all n.
|
|
259
|
+
function selectTop(candidates, scores, units, count) {
|
|
260
|
+
if (count <= 0) return [];
|
|
261
|
+
// a precedes b in the final order when its score is higher, or equal with a smaller URI.
|
|
262
|
+
const before = (a, b) =>
|
|
263
|
+
scores[a] > scores[b] || (scores[a] === scores[b] && candidates[a].uri < candidates[b].uri);
|
|
264
|
+
const heap = [];
|
|
265
|
+
const sift = (start) => {
|
|
266
|
+
let node = start;
|
|
267
|
+
for (;;) {
|
|
268
|
+
const left = node * 2 + 1;
|
|
269
|
+
const right = left + 1;
|
|
270
|
+
let worst = node;
|
|
271
|
+
if (left < heap.length && before(heap[worst], heap[left])) worst = left;
|
|
272
|
+
if (right < heap.length && before(heap[worst], heap[right])) worst = right;
|
|
273
|
+
if (worst === node) return;
|
|
274
|
+
[heap[node], heap[worst]] = [heap[worst], heap[node]];
|
|
275
|
+
node = worst;
|
|
276
|
+
}
|
|
277
|
+
};
|
|
278
|
+
for (let position = 0; position < candidates.length; position += 1) {
|
|
279
|
+
if (units[position] === -1) continue;
|
|
280
|
+
if (heap.length < count) {
|
|
281
|
+
heap.push(position);
|
|
282
|
+
let node = heap.length - 1;
|
|
283
|
+
while (node > 0) {
|
|
284
|
+
const parent = (node - 1) >> 1;
|
|
285
|
+
if (!before(heap[parent], heap[node])) break;
|
|
286
|
+
[heap[node], heap[parent]] = [heap[parent], heap[node]];
|
|
287
|
+
node = parent;
|
|
288
|
+
}
|
|
289
|
+
} else if (before(position, heap[0])) {
|
|
290
|
+
heap[0] = position;
|
|
291
|
+
sift(0);
|
|
292
|
+
}
|
|
293
|
+
}
|
|
294
|
+
return heap.sort((a, b) => (before(a, b) ? -1 : before(b, a) ? 1 : 0));
|
|
295
|
+
}
|
|
296
|
+
|
|
297
|
+
module.exports = {
|
|
298
|
+
DEFAULT_EMBED_UNITS,
|
|
299
|
+
EMBED_UNITS,
|
|
300
|
+
denseCoverage,
|
|
301
|
+
denseSearch,
|
|
302
|
+
resolveUnits,
|
|
303
|
+
unitMemo,
|
|
304
|
+
};
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
|
|
3
|
+
// Command-line flags for hybrid search in --root mode (WP3c), kept out of
|
|
4
|
+
// src/cli.js (Q-1 shrink-only cap): --embedder-config <file> names the
|
|
5
|
+
// trusted embedder settings (src/search/embedding-config.js) and --cache-dir
|
|
6
|
+
// <dir> overrides the vector cache location. They are taken out of argv
|
|
7
|
+
// before the main parser runs and handed to the stdio server untouched.
|
|
8
|
+
|
|
9
|
+
const EMBEDDING_FLAGS = Object.freeze({
|
|
10
|
+
"--embedder-config": "configPath",
|
|
11
|
+
"--cache-dir": "cacheDir",
|
|
12
|
+
});
|
|
13
|
+
|
|
14
|
+
const EMBEDDING_USAGE =
|
|
15
|
+
" --embedder-config <file> Hybrid search for --root mode: embedder, fusion, embedUnits (JSON or YAML); --cache-dir <dir> overrides the vector cache location.";
|
|
16
|
+
|
|
17
|
+
function usageError(message) {
|
|
18
|
+
const error = new Error(message);
|
|
19
|
+
error.name = "UsageError";
|
|
20
|
+
error.code = "OKF_USAGE_ERROR";
|
|
21
|
+
error.exitCode = 2;
|
|
22
|
+
return error;
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
// -> { argv: the remaining arguments, embedding: { configPath?, cacheDir? } }
|
|
26
|
+
function splitEmbeddingArgs(argv) {
|
|
27
|
+
const rest = [];
|
|
28
|
+
const embedding = {};
|
|
29
|
+
const list = Array.isArray(argv) ? argv : [];
|
|
30
|
+
for (let index = 0; index < list.length; index += 1) {
|
|
31
|
+
const arg = list[index];
|
|
32
|
+
const field = Object.hasOwn(EMBEDDING_FLAGS, arg) ? EMBEDDING_FLAGS[arg] : null;
|
|
33
|
+
if (!field) {
|
|
34
|
+
rest.push(arg);
|
|
35
|
+
continue;
|
|
36
|
+
}
|
|
37
|
+
const value = list[index + 1];
|
|
38
|
+
if (!value || String(value).startsWith("-")) {
|
|
39
|
+
throw usageError(`${arg} requires a file or directory path.`);
|
|
40
|
+
}
|
|
41
|
+
embedding[field] = value;
|
|
42
|
+
index += 1;
|
|
43
|
+
}
|
|
44
|
+
return { argv: rest, embedding };
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
// The stdio server option, present only when a flag was given.
|
|
48
|
+
function embeddingServerOptions(args) {
|
|
49
|
+
const embedding = args?.embedding || {};
|
|
50
|
+
return embedding.configPath || embedding.cacheDir ? { embedding } : {};
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
module.exports = {
|
|
54
|
+
EMBEDDING_USAGE,
|
|
55
|
+
embeddingServerOptions,
|
|
56
|
+
splitEmbeddingArgs,
|
|
57
|
+
};
|
|
@@ -0,0 +1,217 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
|
|
3
|
+
// Trusted configuration of hybrid search (docs/plan-1.0.md R-8a, R-9, R-10):
|
|
4
|
+
// `embedder`, `cacheDir`, `fusion.alpha` and `embedUnits`, read from
|
|
5
|
+
// okf.project.yaml or, for --root mode, from the file named by
|
|
6
|
+
// --embedder-config. Never from tool arguments (B-3, B-6). The embedder block
|
|
7
|
+
// is checked by constructing the client offline (no request, no key read
|
|
8
|
+
// from the real environment), so a bad value is reported at load time.
|
|
9
|
+
|
|
10
|
+
const fs = require("node:fs");
|
|
11
|
+
const path = require("node:path");
|
|
12
|
+
|
|
13
|
+
const { createOpenAiEmbedder } = require("../embed/openai");
|
|
14
|
+
const { parseFrontmatterYaml } = require("../parser");
|
|
15
|
+
const { DEFAULT_EMBED_UNITS, EMBED_UNITS } = require("./dense");
|
|
16
|
+
const { minSimilarityOption } = require("./confidence");
|
|
17
|
+
const {
|
|
18
|
+
FUSION_NORMALIZATIONS,
|
|
19
|
+
adaptiveWeights,
|
|
20
|
+
fusionAlphaSetting,
|
|
21
|
+
fusionNormalization,
|
|
22
|
+
} = require("./fusion");
|
|
23
|
+
|
|
24
|
+
const EMBEDDER_TYPES = Object.freeze(["openai"]);
|
|
25
|
+
const EMBEDDER_FIELDS = Object.freeze([
|
|
26
|
+
"type",
|
|
27
|
+
"baseUrl",
|
|
28
|
+
"model",
|
|
29
|
+
"apiKeyEnv",
|
|
30
|
+
"dimensions",
|
|
31
|
+
"batchSize",
|
|
32
|
+
"timeoutMs",
|
|
33
|
+
"maxRetries",
|
|
34
|
+
"concurrency",
|
|
35
|
+
"maxInputChars",
|
|
36
|
+
]);
|
|
37
|
+
const SETTINGS_FIELDS = Object.freeze([
|
|
38
|
+
"embedder",
|
|
39
|
+
"cacheDir",
|
|
40
|
+
"fusion",
|
|
41
|
+
"embedUnits",
|
|
42
|
+
"confidence",
|
|
43
|
+
]);
|
|
44
|
+
const FUSION_FIELDS = Object.freeze(["alpha", "normalization", "adaptive"]);
|
|
45
|
+
|
|
46
|
+
function isObject(value) {
|
|
47
|
+
return Boolean(value) && typeof value === "object" && !Array.isArray(value);
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
function checkEmbedder(raw, errors) {
|
|
51
|
+
if (!isObject(raw)) {
|
|
52
|
+
errors.push({
|
|
53
|
+
code: "invalid_embedder",
|
|
54
|
+
field: "embedder",
|
|
55
|
+
message: "embedder must be an object.",
|
|
56
|
+
});
|
|
57
|
+
return null;
|
|
58
|
+
}
|
|
59
|
+
let valid = true;
|
|
60
|
+
Object.keys(raw)
|
|
61
|
+
.filter((key) => !EMBEDDER_FIELDS.includes(key))
|
|
62
|
+
.forEach((key) => {
|
|
63
|
+
valid = false;
|
|
64
|
+
errors.push({
|
|
65
|
+
code: "unknown_embedder_field",
|
|
66
|
+
field: `embedder.${key}`,
|
|
67
|
+
message: `Unknown embedder field: ${key}`,
|
|
68
|
+
});
|
|
69
|
+
});
|
|
70
|
+
if (!EMBEDDER_TYPES.includes(raw.type)) {
|
|
71
|
+
errors.push({
|
|
72
|
+
code: "invalid_embedder",
|
|
73
|
+
field: "embedder.type",
|
|
74
|
+
message: `embedder.type must be one of ${EMBEDDER_TYPES.join(", ")}.`,
|
|
75
|
+
});
|
|
76
|
+
return null;
|
|
77
|
+
}
|
|
78
|
+
const options = Object.assign({}, raw);
|
|
79
|
+
delete options.type;
|
|
80
|
+
try {
|
|
81
|
+
const keyEnv = typeof options.apiKeyEnv === "string" ? options.apiKeyEnv : null;
|
|
82
|
+
createOpenAiEmbedder(
|
|
83
|
+
Object.assign({}, options, {
|
|
84
|
+
env: keyEnv ? { [keyEnv]: "offline-validation" } : {},
|
|
85
|
+
fetch: () => {
|
|
86
|
+
throw new Error("offline validation");
|
|
87
|
+
},
|
|
88
|
+
}),
|
|
89
|
+
);
|
|
90
|
+
} catch (error) {
|
|
91
|
+
const field = error?.details?.field;
|
|
92
|
+
errors.push({
|
|
93
|
+
code: "invalid_embedder",
|
|
94
|
+
field: typeof field === "string" ? `embedder.${field}` : "embedder",
|
|
95
|
+
message: error.message,
|
|
96
|
+
});
|
|
97
|
+
return null;
|
|
98
|
+
}
|
|
99
|
+
return valid ? Object.assign({}, raw) : null;
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
// Returns { embedder, cacheDir, fusion: { alpha }, embedUnits } or null when
|
|
103
|
+
// no embedder is configured; problems go to `errors` like other project
|
|
104
|
+
// config errors.
|
|
105
|
+
function normalizeEmbeddingConfig(config, errors) {
|
|
106
|
+
const source = isObject(config) ? config : {};
|
|
107
|
+
if (source.embedder === undefined || source.embedder === null) return null;
|
|
108
|
+
const embedder = checkEmbedder(source.embedder, errors);
|
|
109
|
+
let cacheDir = null;
|
|
110
|
+
if (source.cacheDir !== undefined && source.cacheDir !== null) {
|
|
111
|
+
if (typeof source.cacheDir !== "string" || !source.cacheDir.trim()) {
|
|
112
|
+
errors.push({
|
|
113
|
+
code: "invalid_cache_dir",
|
|
114
|
+
field: "cacheDir",
|
|
115
|
+
message: "cacheDir must be a non-empty path.",
|
|
116
|
+
});
|
|
117
|
+
} else {
|
|
118
|
+
cacheDir = source.cacheDir;
|
|
119
|
+
}
|
|
120
|
+
}
|
|
121
|
+
let alpha = fusionAlphaSetting(undefined);
|
|
122
|
+
let normalization = fusionNormalization(undefined);
|
|
123
|
+
let adaptive = null;
|
|
124
|
+
if (source.fusion !== undefined) {
|
|
125
|
+
const unknown = isObject(source.fusion)
|
|
126
|
+
? Object.keys(source.fusion).filter((key) => !FUSION_FIELDS.includes(key))
|
|
127
|
+
: [];
|
|
128
|
+
try {
|
|
129
|
+
if (!isObject(source.fusion) || unknown.length) throw new TypeError("fusion");
|
|
130
|
+
alpha = fusionAlphaSetting(source.fusion.alpha);
|
|
131
|
+
normalization = fusionNormalization(source.fusion.normalization);
|
|
132
|
+
// WP3c-2: the adaptive rule's weights, { low, mid, high } (defaults when unset).
|
|
133
|
+
if (source.fusion.adaptive !== undefined) adaptive = adaptiveWeights(source.fusion.adaptive);
|
|
134
|
+
} catch {
|
|
135
|
+
errors.push({
|
|
136
|
+
code: "invalid_fusion",
|
|
137
|
+
field: "fusion",
|
|
138
|
+
message: `fusion must be { alpha, normalization, adaptive } with alpha "adaptive" or a number between 0 and 1, normalization one of ${FUSION_NORMALIZATIONS.join(", ")} and adaptive { low, mid, high } between 0 and 1 with high >= mid.`,
|
|
139
|
+
});
|
|
140
|
+
}
|
|
141
|
+
}
|
|
142
|
+
let embedUnits = DEFAULT_EMBED_UNITS;
|
|
143
|
+
if (source.embedUnits !== undefined) {
|
|
144
|
+
if (!EMBED_UNITS.includes(source.embedUnits)) {
|
|
145
|
+
errors.push({
|
|
146
|
+
code: "invalid_embed_units",
|
|
147
|
+
field: "embedUnits",
|
|
148
|
+
message: `embedUnits must be one of ${EMBED_UNITS.join(", ")}.`,
|
|
149
|
+
});
|
|
150
|
+
} else {
|
|
151
|
+
embedUnits = source.embedUnits;
|
|
152
|
+
}
|
|
153
|
+
}
|
|
154
|
+
// WP3c-2: `confidence.minSimilarity`, the similarity under which a weakly matched top result
|
|
155
|
+
// is flagged lowConfidence (model-specific; src/search/confidence.js).
|
|
156
|
+
let confidence = null;
|
|
157
|
+
if (source.confidence !== undefined) {
|
|
158
|
+
try {
|
|
159
|
+
const unknown = isObject(source.confidence)
|
|
160
|
+
? Object.keys(source.confidence).filter((key) => key !== "minSimilarity")
|
|
161
|
+
: ["confidence"];
|
|
162
|
+
if (unknown.length) throw new TypeError("confidence");
|
|
163
|
+
confidence = { minSimilarity: minSimilarityOption(source.confidence.minSimilarity) };
|
|
164
|
+
} catch {
|
|
165
|
+
errors.push({
|
|
166
|
+
code: "invalid_confidence",
|
|
167
|
+
field: "confidence",
|
|
168
|
+
message: "confidence must be { minSimilarity } with minSimilarity between -1 and 1.",
|
|
169
|
+
});
|
|
170
|
+
}
|
|
171
|
+
}
|
|
172
|
+
if (!embedder) return null;
|
|
173
|
+
const fusion = { alpha, normalization };
|
|
174
|
+
if (adaptive) fusion.adaptive = Object.assign({}, adaptive);
|
|
175
|
+
const settings = { embedder, cacheDir, fusion, embedUnits };
|
|
176
|
+
if (confidence) settings.confidence = confidence;
|
|
177
|
+
return settings;
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
// The --embedder-config file (JSON, or YAML for any other extension): either
|
|
181
|
+
// the project keys ({ embedder, cacheDir, fusion, embedUnits }) or a bare
|
|
182
|
+
// embedder block ({ type: openai, ... }). Throws on any problem: this is an
|
|
183
|
+
// operator's file, read once at startup.
|
|
184
|
+
function loadEmbeddingConfigFile(file) {
|
|
185
|
+
const absolute = path.resolve(String(file));
|
|
186
|
+
const text = fs.readFileSync(absolute, "utf8");
|
|
187
|
+
const parsed = absolute.endsWith(".json") ? JSON.parse(text) : parseFrontmatterYaml(text);
|
|
188
|
+
const settings =
|
|
189
|
+
isObject(parsed) && parsed.embedder === undefined && parsed.type !== undefined
|
|
190
|
+
? { embedder: parsed }
|
|
191
|
+
: parsed;
|
|
192
|
+
const errors = [];
|
|
193
|
+
if (isObject(settings)) {
|
|
194
|
+
Object.keys(settings)
|
|
195
|
+
.filter((key) => !SETTINGS_FIELDS.includes(key))
|
|
196
|
+
.forEach((key) => {
|
|
197
|
+
errors.push({ code: "unknown_embedding_field", field: key });
|
|
198
|
+
});
|
|
199
|
+
}
|
|
200
|
+
const normalized = normalizeEmbeddingConfig(settings, errors);
|
|
201
|
+
if (errors.length || !normalized) {
|
|
202
|
+
const fields = errors.map((entry) => entry.field).filter(Boolean);
|
|
203
|
+
const error = new Error(
|
|
204
|
+
`Invalid embedder configuration${fields.length ? ` (${fields.join(", ")})` : ": an embedder block is required"}.`,
|
|
205
|
+
);
|
|
206
|
+
error.code = "invalid_embedder_config";
|
|
207
|
+
error.details = { errors: errors.map(({ code, field }) => ({ code, field })) };
|
|
208
|
+
throw error;
|
|
209
|
+
}
|
|
210
|
+
return normalized;
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
module.exports = {
|
|
214
|
+
EMBEDDER_FIELDS,
|
|
215
|
+
loadEmbeddingConfigFile,
|
|
216
|
+
normalizeEmbeddingConfig,
|
|
217
|
+
};
|