@mfdaves/okf-mcp 0.9.1 → 0.12.0-rc.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +69 -0
- package/README.md +172 -17
- package/bin/okf-mcp.js +6 -1
- package/okf/okf-mcp/policies/authoring-safety.md +11 -1
- package/package.json +17 -5
- package/server.json +2 -2
- package/src/assets.js +165 -73
- package/src/authoring/candidates.js +391 -0
- package/src/authoring/git-preflight.js +225 -0
- package/src/authoring/git-publish.js +776 -0
- package/src/authoring/live.js +789 -0
- package/src/authoring/paths.js +112 -0
- package/src/authoring/policy-hook.js +221 -0
- package/src/authoring/proposal-policy.js +137 -0
- package/src/authoring/receipts.js +329 -0
- package/src/authoring/render.js +38 -0
- package/src/authoring/shared.js +75 -0
- package/src/authoring/staging.js +216 -0
- package/src/authoring.js +386 -287
- package/src/cache/files.js +147 -0
- package/src/cache/lock.js +201 -0
- package/src/cache/paths.js +197 -0
- package/src/cache/store.js +703 -0
- package/src/catalog-text.js +19 -0
- package/src/cli.js +184 -151
- package/src/computation.js +99 -61
- package/src/context/assemble.js +567 -0
- package/src/context/cli-options.js +88 -0
- package/src/context/score.js +234 -0
- package/src/context/sections.js +274 -0
- package/src/embed/interface.js +160 -0
- package/src/embed/openai.js +715 -0
- package/src/embed/prepare.js +182 -0
- package/src/embed/units.js +96 -0
- package/src/git-source.js +112 -52
- package/src/graph.js +95 -37
- package/src/http-server.js +36 -17
- package/src/index.js +15 -0
- package/src/indexer/assemble.js +566 -0
- package/src/indexer/build.js +290 -0
- package/src/indexer/cache.js +81 -0
- package/src/indexer/edges.js +430 -0
- package/src/indexer/index.js +44 -0
- package/src/indexer/paths.js +178 -0
- package/src/indexer/resolve.js +282 -0
- package/src/indexer/update.js +720 -0
- package/src/indexer.js +3 -1068
- package/src/live-authoring.js +4 -2329
- package/src/markdown.js +21 -16
- package/src/mcp/registry.js +328 -0
- package/src/mcp/resources.js +45 -0
- package/src/mcp/server.js +472 -0
- package/src/mcp/tools/catalog.js +244 -0
- package/src/mcp/tools/computation.js +83 -0
- package/src/mcp/tools/context.js +133 -0
- package/src/mcp/tools/graph.js +167 -0
- package/src/mcp/tools/live-authoring.js +316 -0
- package/src/mcp/tools/migration.js +86 -0
- package/src/mcp/tools/producers.js +79 -0
- package/src/mcp/tools/proposals.js +213 -0
- package/src/mcp/tools/provenance.js +87 -0
- package/src/mcp/tools/remote.js +79 -0
- package/src/mcp/tools/search.js +242 -0
- package/src/mcp-server.js +3 -1611
- package/src/migration.js +431 -253
- package/src/parser.js +245 -130
- package/src/plugins.js +71 -48
- package/src/policy/write-policy.js +713 -0
- package/src/producer-publisher.js +195 -77
- package/src/producers.js +265 -101
- package/src/project.js +277 -93
- package/src/remote.js +96 -61
- package/src/search/cli-options.js +107 -0
- package/src/search/confidence.js +61 -0
- package/src/search/dense.js +304 -0
- package/src/search/embedding-cli.js +57 -0
- package/src/search/embedding-config.js +217 -0
- package/src/search/fusion.js +249 -0
- package/src/search/hybrid.js +526 -0
- package/src/search/languages.js +200 -0
- package/src/search/lexical.js +695 -0
- package/src/search/query.js +620 -0
- package/src/search/stem/among.js +43 -0
- package/src/search/stem/base-stemmer.js +311 -0
- package/src/search/stem/en-regions.js +456 -0
- package/src/search/stem/en-suffixes.js +664 -0
- package/src/search/stem/en-tables.js +176 -0
- package/src/search/stem/en.js +238 -0
- package/src/search/stem/it-regions.js +459 -0
- package/src/search/stem/it-suffixes.js +471 -0
- package/src/search/stem/it-tables.js +257 -0
- package/src/search/stem/it.js +155 -0
- package/src/search-index.js +2 -81
- package/src/search.js +2 -252
- package/src/store.js +134 -81
- package/src/v02.js +301 -105
- package/src/validation.js +24 -20
|
@@ -0,0 +1,234 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
|
|
3
|
+
// Per-query section scoring for context assembly (plan C-1, WP3a). Only the
|
|
4
|
+
// sections of the candidate concepts are scored, on the fly: no section
|
|
5
|
+
// index, no per-index corpus, nothing to rebuild after a write. The query is
|
|
6
|
+
// the 0.10 analysis (`analyzeQuery`: stop words and single letters dropped,
|
|
7
|
+
// plain terms folded and stemmed, quoted and identifier-like terms literal),
|
|
8
|
+
// and each section is read the way the lexical index reads a document: tokens
|
|
9
|
+
// split on spaces, punctuation and symbols, lower-cased and diacritic-folded,
|
|
10
|
+
// plus one stem per language of the concept's bundle. A plain term counts
|
|
11
|
+
// where any of its forms meets any form of a token; a literal term counts
|
|
12
|
+
// once per adjacent run of its pieces, or once when the whole compound is an
|
|
13
|
+
// exact token of the field. Never through a stem.
|
|
14
|
+
//
|
|
15
|
+
// Evidence comes in two tiers. Own text: the section's text (on the
|
|
16
|
+
// preamble, which stands for the concept, also its aliases and identifiers;
|
|
17
|
+
// elsewhere the heading as an exact token). Concept metadata: the type, tags,
|
|
18
|
+
// path and identifiers every section of the concept shares. The primary score
|
|
19
|
+
// is BM25 (k1 1.2, b 0.75) over own text only, with document frequency and
|
|
20
|
+
// average length taken from the candidate sections, so a term common to every
|
|
21
|
+
// candidate section weighs little; `coverage` and the matched terms are
|
|
22
|
+
// own-text only. Metadata adds a bonus of at most 20% of the own-text score,
|
|
23
|
+
// in proportion to the IDF weight of the terms only metadata matched. A
|
|
24
|
+
// section with no own-text match is kept (a runbook's "Steps" rarely repeats
|
|
25
|
+
// its title) but as tier 0, which the caller ranks below every own-text
|
|
26
|
+
// section of the same concept. The preamble's own score is multiplied by
|
|
27
|
+
// `preambleWeight`: it carries the concept title, description and aliases,
|
|
28
|
+
// which match most queries that selected the concept and would otherwise take
|
|
29
|
+
// rank 1 over the section that answers.
|
|
30
|
+
|
|
31
|
+
const { exactTokens } = require("../search/lexical");
|
|
32
|
+
const { foldDiacritics, stemForms } = require("../search/languages");
|
|
33
|
+
|
|
34
|
+
const K1 = 1.2;
|
|
35
|
+
const B = 0.75;
|
|
36
|
+
// Ceiling of the metadata bonus as a share of the own-text score.
|
|
37
|
+
const METADATA_BONUS = 0.2;
|
|
38
|
+
// The splitter of src/search/lexical.js: spaces, punctuation and symbols.
|
|
39
|
+
const TOKEN_SEPARATORS = /[\n\r\p{Z}\p{P}\p{S}]+/u;
|
|
40
|
+
|
|
41
|
+
const lowerTokens = (text) =>
|
|
42
|
+
String(text || "")
|
|
43
|
+
.split(TOKEN_SEPARATORS)
|
|
44
|
+
.filter(Boolean)
|
|
45
|
+
.map((token) => token.toLowerCase());
|
|
46
|
+
|
|
47
|
+
const customIdPath = (customId) =>
|
|
48
|
+
customId ? String(customId).replace(/^okf:\/\/[^/]+\//, "") : "";
|
|
49
|
+
|
|
50
|
+
// The values the lexical index treats as identifiers, for the preamble.
|
|
51
|
+
function exactSegments(doc) {
|
|
52
|
+
return [doc.conceptId, customIdPath(doc.customId), doc.path, doc.title]
|
|
53
|
+
.concat(doc.aliases || [])
|
|
54
|
+
.filter(Boolean)
|
|
55
|
+
.map((value) => String(value).replace(/\s+/g, " ").trim())
|
|
56
|
+
.filter(Boolean);
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
// One evidence field: folded tokens (for literal runs), their forms with
|
|
60
|
+
// stems (for plain terms) and the exact tokens (for whole literals).
|
|
61
|
+
function evidenceField(lower, identifiers, stemsOf) {
|
|
62
|
+
return {
|
|
63
|
+
folded: lower.map((token) => foldDiacritics(token)),
|
|
64
|
+
forms: lower.map(stemsOf),
|
|
65
|
+
exact: new Set(exactTokens(identifiers.filter(Boolean).join("\n"))),
|
|
66
|
+
};
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
// A section's own text and its concept's shared metadata. Stems are memoised
|
|
70
|
+
// per call and language set by `stemsOf`.
|
|
71
|
+
function sectionFields(doc, section, stemsOf) {
|
|
72
|
+
const preamble = section.heading === null;
|
|
73
|
+
const own = evidenceField(
|
|
74
|
+
lowerTokens(preamble ? [section.text].concat(doc.aliases || []).join("\n") : section.text),
|
|
75
|
+
preamble ? exactSegments(doc) : [section.heading],
|
|
76
|
+
stemsOf,
|
|
77
|
+
);
|
|
78
|
+
const metadata = evidenceField(
|
|
79
|
+
lowerTokens([doc.type, (doc.tags || []).join(" "), doc.path].join("\n")),
|
|
80
|
+
[doc.conceptId, customIdPath(doc.customId), doc.path],
|
|
81
|
+
stemsOf,
|
|
82
|
+
);
|
|
83
|
+
return { own, metadata };
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
function runCount(folded, pieces) {
|
|
87
|
+
if (!pieces.length || pieces.length > folded.length) {
|
|
88
|
+
return 0;
|
|
89
|
+
}
|
|
90
|
+
let count = 0;
|
|
91
|
+
for (let start = 0; start + pieces.length <= folded.length; start += 1) {
|
|
92
|
+
let match = true;
|
|
93
|
+
for (let offset = 0; offset < pieces.length && match; offset += 1) {
|
|
94
|
+
match = folded[start + offset] === pieces[offset];
|
|
95
|
+
}
|
|
96
|
+
if (match) {
|
|
97
|
+
count += 1;
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
return count;
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
// Term frequency of every analysed term in one field.
|
|
104
|
+
function termFrequencies(field, terms, leafOwners) {
|
|
105
|
+
const tf = new Array(terms.length).fill(0);
|
|
106
|
+
for (const forms of field.forms) {
|
|
107
|
+
const seen = new Set();
|
|
108
|
+
for (const leaf of forms) {
|
|
109
|
+
for (const owner of leafOwners.get(leaf) || []) {
|
|
110
|
+
if (!seen.has(owner)) {
|
|
111
|
+
seen.add(owner);
|
|
112
|
+
tf[owner] += 1;
|
|
113
|
+
}
|
|
114
|
+
}
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
terms.forEach((term, position) => {
|
|
118
|
+
if (term.kind !== "plain") {
|
|
119
|
+
const runs = runCount(field.folded, term.pieces);
|
|
120
|
+
tf[position] = field.exact.has(term.literal) ? Math.max(1, runs) : runs;
|
|
121
|
+
}
|
|
122
|
+
});
|
|
123
|
+
return tf;
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
function formsFor(languages, cache) {
|
|
127
|
+
return (lower) => {
|
|
128
|
+
let forms = cache.get(lower);
|
|
129
|
+
if (!forms) {
|
|
130
|
+
const folded = foldDiacritics(lower);
|
|
131
|
+
forms = [folded].concat(stemForms(lower, languages));
|
|
132
|
+
cache.set(lower, forms);
|
|
133
|
+
}
|
|
134
|
+
return forms;
|
|
135
|
+
};
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
function leafOwnersOf(terms) {
|
|
139
|
+
const owners = new Map();
|
|
140
|
+
terms.forEach((term, position) => {
|
|
141
|
+
if (term.kind !== "plain") {
|
|
142
|
+
return;
|
|
143
|
+
}
|
|
144
|
+
term.leaves.forEach((leaf) => {
|
|
145
|
+
if (!owners.has(leaf)) {
|
|
146
|
+
owners.set(leaf, []);
|
|
147
|
+
}
|
|
148
|
+
if (!owners.get(leaf).includes(position)) {
|
|
149
|
+
owners.get(leaf).push(position);
|
|
150
|
+
}
|
|
151
|
+
});
|
|
152
|
+
});
|
|
153
|
+
return owners;
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
// entries: [{ doc, section, ... }]; analysis: analyzeQuery's result;
|
|
157
|
+
// languagesFor(bundle): the stemming languages of a concept's bundle.
|
|
158
|
+
// Returns, for every entry with evidence in either tier,
|
|
159
|
+
// { entry, tier, score, metadataShare, coverage, positions }: tier 1 with
|
|
160
|
+
// `score` the weighted own-text score plus the metadata bonus; tier 0 with
|
|
161
|
+
// score 0 and `metadataShare` (the IDF share of the terms metadata matched),
|
|
162
|
+
// for the caller to place below its concept's tier-1 sections. `positions`
|
|
163
|
+
// index own-text terms.
|
|
164
|
+
function scoreSections(entries, analysis, options) {
|
|
165
|
+
const config = options || {};
|
|
166
|
+
const preambleWeight = config.preambleWeight === undefined ? 1 : config.preambleWeight;
|
|
167
|
+
const terms = analysis.tokens;
|
|
168
|
+
if (!terms.length || !entries.length) {
|
|
169
|
+
return [];
|
|
170
|
+
}
|
|
171
|
+
const leafOwners = leafOwnersOf(terms);
|
|
172
|
+
const formCaches = new Map();
|
|
173
|
+
const stemsFor = (bundle) => {
|
|
174
|
+
const languages = config.languagesFor ? config.languagesFor(bundle) : [];
|
|
175
|
+
const key = languages.join(",");
|
|
176
|
+
if (!formCaches.has(key)) {
|
|
177
|
+
formCaches.set(key, formsFor(languages, new Map()));
|
|
178
|
+
}
|
|
179
|
+
return formCaches.get(key);
|
|
180
|
+
};
|
|
181
|
+
const rows = entries.map((entry) => {
|
|
182
|
+
const fields = sectionFields(entry.doc, entry.section, stemsFor(String(entry.doc.bundle)));
|
|
183
|
+
return {
|
|
184
|
+
entry,
|
|
185
|
+
own: termFrequencies(fields.own, terms, leafOwners),
|
|
186
|
+
metadata: termFrequencies(fields.metadata, terms, leafOwners),
|
|
187
|
+
length: Math.max(1, fields.own.folded.length),
|
|
188
|
+
};
|
|
189
|
+
});
|
|
190
|
+
const count = rows.length;
|
|
191
|
+
const averageLength = rows.reduce((sum, row) => sum + row.length, 0) / count;
|
|
192
|
+
const idfOver = (matches) =>
|
|
193
|
+
terms.map((_, position) => {
|
|
194
|
+
const df = rows.filter((row) => matches(row, position)).length;
|
|
195
|
+
return Math.log(1 + (count - df + 0.5) / (df + 0.5));
|
|
196
|
+
});
|
|
197
|
+
const idf = idfOver((row, position) => row.own[position] > 0);
|
|
198
|
+
// Metadata weights: IDF over either tier, so a term only metadata carries counts.
|
|
199
|
+
const anyIdf = idfOver((row, position) => row.own[position] > 0 || row.metadata[position] > 0);
|
|
200
|
+
const anyTotal = anyIdf.reduce((sum, value) => sum + value, 0) || 1;
|
|
201
|
+
const scored = [];
|
|
202
|
+
rows.forEach((row) => {
|
|
203
|
+
const positions = [];
|
|
204
|
+
let own = 0;
|
|
205
|
+
let metadataWeight = 0;
|
|
206
|
+
row.own.forEach((frequency, position) => {
|
|
207
|
+
if (frequency) {
|
|
208
|
+
positions.push(position);
|
|
209
|
+
const norm = K1 * (1 - B + (B * row.length) / averageLength);
|
|
210
|
+
own += (idf[position] * (frequency * (K1 + 1))) / (frequency + norm);
|
|
211
|
+
} else if (row.metadata[position]) {
|
|
212
|
+
metadataWeight += anyIdf[position];
|
|
213
|
+
}
|
|
214
|
+
});
|
|
215
|
+
const metadataShare = metadataWeight / anyTotal;
|
|
216
|
+
if (!positions.length && !metadataShare) {
|
|
217
|
+
return;
|
|
218
|
+
}
|
|
219
|
+
if (row.entry.section.heading === null) {
|
|
220
|
+
own *= preambleWeight;
|
|
221
|
+
}
|
|
222
|
+
scored.push({
|
|
223
|
+
entry: row.entry,
|
|
224
|
+
tier: positions.length ? 1 : 0,
|
|
225
|
+
score: positions.length ? own * (1 + METADATA_BONUS * metadataShare) : 0,
|
|
226
|
+
metadataShare,
|
|
227
|
+
coverage: positions.length / terms.length,
|
|
228
|
+
positions,
|
|
229
|
+
});
|
|
230
|
+
});
|
|
231
|
+
return scored;
|
|
232
|
+
}
|
|
233
|
+
|
|
234
|
+
module.exports = { METADATA_BONUS, scoreSections };
|
|
@@ -0,0 +1,274 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
|
|
3
|
+
// Section-level retrieval units (plan R-7, WP3a). A concept is cut at its
|
|
4
|
+
// document-level CommonMark headings into flat, non-overlapping sections: the
|
|
5
|
+
// preamble (frontmatter title and description plus any body before the first
|
|
6
|
+
// heading) and one section per heading, running to the next heading of any
|
|
7
|
+
// level. Headings inside block quotes, list items or unclosed fences are
|
|
8
|
+
// content, not boundaries (`markdownStructure` keeps document children only),
|
|
9
|
+
// which matches how the computation extractor reads sections.
|
|
10
|
+
//
|
|
11
|
+
// Anchor scheme, an okf-mcp extension (B-2): the preamble is addressed by the
|
|
12
|
+
// concept URI alone, every other section by `uri#slug`. The heading's text is
|
|
13
|
+
// first normalised (line breaks of a setext or wrapped heading and any
|
|
14
|
+
// whitespace run become one space, trimmed), so whitespace-only edits never
|
|
15
|
+
// move an anchor; then GitHub's rule as github-slugger implements it applies:
|
|
16
|
+
// lower-cased, every character that is not a letter, a combining mark, a
|
|
17
|
+
// number, an underscore, a hyphen or a space removed, each space turned into
|
|
18
|
+
// a hyphen (hyphens left by removed punctuation are not collapsed, as on
|
|
19
|
+
// GitHub), and repeated slugs take `-1`, `-2`, ... in document order. On the
|
|
20
|
+
// fixture labels it agrees with test/fixtures/retrieval/lib/slug.js. OKF v0.2
|
|
21
|
+
// defines no section anchors: nothing in a bundle needs them, and a reader
|
|
22
|
+
// that drops the fragment still lands on the concept. An anchor depends only
|
|
23
|
+
// on the heading texts up to its own, so edits to any body, whitespace-only
|
|
24
|
+
// edits and edits to later sections leave it unchanged; renaming the heading
|
|
25
|
+
// or adding a same-text heading before it changes it.
|
|
26
|
+
//
|
|
27
|
+
// `text` is the section's plain text: the heading once, then paragraphs, list
|
|
28
|
+
// items, code blocks and tables flattened one block or row per line (a
|
|
29
|
+
// table's delimiter row dropped; code is kept verbatim apart from whitespace
|
|
30
|
+
// runs), links
|
|
31
|
+
// and images reduced to their text, inline HTML dropped, whitespace runs
|
|
32
|
+
// collapsed. `hash` is the SHA-256 of that text from `hashUnitText` in
|
|
33
|
+
// src/embed/units.js, so `sectionUnits` can replace the concept units in
|
|
34
|
+
// `prepareEmbeddings` (WP3c: `units: sectionUnits`) and identical sections
|
|
35
|
+
// share one cache row (R-9). Section units cover valid concepts only, the set
|
|
36
|
+
// search ranks (`index.concepts`); concept units also embed invalid ones.
|
|
37
|
+
|
|
38
|
+
const { contextPrefix, hashUnitText } = require("../embed/units");
|
|
39
|
+
const { markdownStructure, sectionNodes } = require("../markdown");
|
|
40
|
+
|
|
41
|
+
// Parsed document -> its sections. Documents are shared unchanged across
|
|
42
|
+
// functional index updates, so an unchanged concept keeps its sections.
|
|
43
|
+
const SECTIONS = new WeakMap();
|
|
44
|
+
// Slug of a heading whose text has no letter or digit ("###" alone, "!!!").
|
|
45
|
+
const EMPTY_SLUG = "section";
|
|
46
|
+
const HTML_TAG = /<[^>]*>/g;
|
|
47
|
+
// A GFM table delimiter row (`|---|:--:|`), kept out of the plain text.
|
|
48
|
+
const TABLE_DELIMITER_ROW = /^\|?\s*:?-+:?\s*(\|\s*:?-+:?\s*)*\|?$/;
|
|
49
|
+
const TABLE_LINE = /^\s*\|/;
|
|
50
|
+
|
|
51
|
+
// Whitespace-stable heading text: every whitespace run is one space.
|
|
52
|
+
const collapseWhitespace = (text) =>
|
|
53
|
+
String(text || "")
|
|
54
|
+
.replace(/\s+/g, " ")
|
|
55
|
+
.trim();
|
|
56
|
+
|
|
57
|
+
function headingSlug(heading) {
|
|
58
|
+
return collapseWhitespace(heading)
|
|
59
|
+
.toLowerCase()
|
|
60
|
+
.replace(/[^\p{L}\p{M}\p{N}_ -]/gu, "")
|
|
61
|
+
.replace(/ /g, "-");
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
// A heading's text with soft and hard line breaks read as spaces (the shared
|
|
65
|
+
// markdown helper drops them, which would glue a setext heading's lines).
|
|
66
|
+
function headingNodeText(node) {
|
|
67
|
+
let out = "";
|
|
68
|
+
const walker = node.walker();
|
|
69
|
+
for (let event = walker.next(); event; event = walker.next()) {
|
|
70
|
+
if (!event.entering) {
|
|
71
|
+
continue;
|
|
72
|
+
}
|
|
73
|
+
const type = event.node.type;
|
|
74
|
+
if (type === "text" || type === "code") {
|
|
75
|
+
out += event.node.literal || "";
|
|
76
|
+
} else if (type === "softbreak" || type === "linebreak") {
|
|
77
|
+
out += " ";
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
return collapseWhitespace(out);
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
// One line per block, no empty lines, whitespace runs collapsed.
|
|
84
|
+
function normalizeText(text) {
|
|
85
|
+
return String(text || "")
|
|
86
|
+
.split("\n")
|
|
87
|
+
.map((line) => line.replace(/\s+/g, " ").trim())
|
|
88
|
+
.filter(Boolean)
|
|
89
|
+
.join("\n");
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
function firstTextLiteral(node) {
|
|
93
|
+
const walker = node.walker();
|
|
94
|
+
for (let event = walker.next(); event; event = walker.next()) {
|
|
95
|
+
if (event.entering && event.node.type === "text") {
|
|
96
|
+
return event.node.literal || "";
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
return "";
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
// Plain text of one block subtree. Soft line breaks become spaces, except in
|
|
103
|
+
// a paragraph that is a table (commonmark.js has no table node), where each
|
|
104
|
+
// source line is a row. Leaf nodes get a single entering event.
|
|
105
|
+
function blockText(block) {
|
|
106
|
+
let out = "";
|
|
107
|
+
let tableRows = false;
|
|
108
|
+
let paragraphStart = 0;
|
|
109
|
+
const walker = block.walker();
|
|
110
|
+
for (let event = walker.next(); event; event = walker.next()) {
|
|
111
|
+
const node = event.node;
|
|
112
|
+
if (!event.entering) {
|
|
113
|
+
if (node.type === "paragraph" && tableRows) {
|
|
114
|
+
// Only a table's own delimiter row is dropped, never a code line.
|
|
115
|
+
const rows = out
|
|
116
|
+
.slice(paragraphStart)
|
|
117
|
+
.split("\n")
|
|
118
|
+
.filter((row) => !TABLE_DELIMITER_ROW.test(row.trim()));
|
|
119
|
+
out = out.slice(0, paragraphStart) + rows.join("\n");
|
|
120
|
+
tableRows = false;
|
|
121
|
+
}
|
|
122
|
+
if (node.type === "paragraph" || node.type === "heading" || node.type === "item") {
|
|
123
|
+
out += "\n";
|
|
124
|
+
}
|
|
125
|
+
continue;
|
|
126
|
+
}
|
|
127
|
+
switch (node.type) {
|
|
128
|
+
case "text":
|
|
129
|
+
case "code":
|
|
130
|
+
out += node.literal || "";
|
|
131
|
+
break;
|
|
132
|
+
case "softbreak":
|
|
133
|
+
out += tableRows ? "\n" : " ";
|
|
134
|
+
break;
|
|
135
|
+
case "linebreak":
|
|
136
|
+
out += "\n";
|
|
137
|
+
break;
|
|
138
|
+
case "code_block":
|
|
139
|
+
out += `${node.literal || ""}\n`;
|
|
140
|
+
break;
|
|
141
|
+
case "html_block":
|
|
142
|
+
out += `${String(node.literal || "").replace(HTML_TAG, " ")}\n`;
|
|
143
|
+
break;
|
|
144
|
+
case "paragraph":
|
|
145
|
+
tableRows = TABLE_LINE.test(firstTextLiteral(node));
|
|
146
|
+
paragraphStart = out.length;
|
|
147
|
+
break;
|
|
148
|
+
default:
|
|
149
|
+
break;
|
|
150
|
+
}
|
|
151
|
+
}
|
|
152
|
+
return out;
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
function blocksBefore(tree, stop) {
|
|
156
|
+
const nodes = [];
|
|
157
|
+
let current = tree.firstChild;
|
|
158
|
+
while (current && current !== stop) {
|
|
159
|
+
nodes.push(current);
|
|
160
|
+
current = current.next;
|
|
161
|
+
}
|
|
162
|
+
return nodes;
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
// GitHub numbering: the first occurrence keeps the slug, repeats take -1, -2,
|
|
166
|
+
// ... in document order; a heading that spells an already taken suffixed slug
|
|
167
|
+
// pushes the counter further so anchors stay unique within the document.
|
|
168
|
+
function uniqueSlugs(headings) {
|
|
169
|
+
const used = new Set();
|
|
170
|
+
const counters = new Map();
|
|
171
|
+
return headings.map((heading) => {
|
|
172
|
+
const base = headingSlug(headingNodeText(heading.node)) || EMPTY_SLUG;
|
|
173
|
+
let slug = base;
|
|
174
|
+
let count = counters.get(base) || 0;
|
|
175
|
+
while (used.has(slug)) {
|
|
176
|
+
count += 1;
|
|
177
|
+
slug = `${base}-${count}`;
|
|
178
|
+
}
|
|
179
|
+
counters.set(base, count);
|
|
180
|
+
used.add(slug);
|
|
181
|
+
return slug;
|
|
182
|
+
});
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
function makeSection(doc, fields, parts) {
|
|
186
|
+
const text = parts.map(normalizeText).filter(Boolean).join("\n");
|
|
187
|
+
return Object.freeze(
|
|
188
|
+
Object.assign({ uri: doc.uri }, fields, {
|
|
189
|
+
text,
|
|
190
|
+
charCount: text.length,
|
|
191
|
+
hash: hashUnitText(text),
|
|
192
|
+
}),
|
|
193
|
+
);
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
// Ordered sections of a parsed document: the preamble first (anchor = the
|
|
197
|
+
// concept URI), then one per document-level heading (anchor = uri#slug).
|
|
198
|
+
function conceptSections(doc) {
|
|
199
|
+
const cacheable = doc !== null && typeof doc === "object";
|
|
200
|
+
const cached = cacheable ? SECTIONS.get(doc) : undefined;
|
|
201
|
+
if (cached) {
|
|
202
|
+
return cached;
|
|
203
|
+
}
|
|
204
|
+
const structure = markdownStructure(doc.body);
|
|
205
|
+
const headings = structure.headings;
|
|
206
|
+
const slugs = uniqueSlugs(headings);
|
|
207
|
+
const preamble = blocksBefore(structure.tree, headings.length ? headings[0].node : null)
|
|
208
|
+
.map(blockText)
|
|
209
|
+
.join("\n");
|
|
210
|
+
const sections = [
|
|
211
|
+
makeSection(doc, { anchor: doc.uri, heading: null, level: 0, slug: null }, [
|
|
212
|
+
doc.title,
|
|
213
|
+
doc.description,
|
|
214
|
+
preamble,
|
|
215
|
+
]),
|
|
216
|
+
];
|
|
217
|
+
headings.forEach((heading, position) => {
|
|
218
|
+
const headingText = headingNodeText(heading.node);
|
|
219
|
+
const body = sectionNodes(heading, { stopAtAnyHeading: true }).map(blockText).join("\n");
|
|
220
|
+
sections.push(
|
|
221
|
+
makeSection(
|
|
222
|
+
doc,
|
|
223
|
+
{
|
|
224
|
+
anchor: `${doc.uri}#${slugs[position]}`,
|
|
225
|
+
heading: headingText,
|
|
226
|
+
level: heading.level,
|
|
227
|
+
slug: slugs[position],
|
|
228
|
+
},
|
|
229
|
+
[headingText, body],
|
|
230
|
+
),
|
|
231
|
+
);
|
|
232
|
+
});
|
|
233
|
+
Object.freeze(sections);
|
|
234
|
+
if (cacheable) {
|
|
235
|
+
SECTIONS.set(doc, sections);
|
|
236
|
+
}
|
|
237
|
+
return sections;
|
|
238
|
+
}
|
|
239
|
+
|
|
240
|
+
// Every section of every valid concept as an embedding unit, the shape of
|
|
241
|
+
// `conceptUnits` in src/embed/units.js: { id: anchor, text, hash }.
|
|
242
|
+
function sectionUnits(index) {
|
|
243
|
+
const concepts = index?.concepts || [];
|
|
244
|
+
return concepts.flatMap((doc) =>
|
|
245
|
+
conceptSections(doc).map((section) => ({
|
|
246
|
+
id: section.anchor,
|
|
247
|
+
text: section.text,
|
|
248
|
+
hash: section.hash,
|
|
249
|
+
})),
|
|
250
|
+
);
|
|
251
|
+
}
|
|
252
|
+
|
|
253
|
+
// Contextual section units (WP3c-2): each section's text after a bounded `title · type · path`
|
|
254
|
+
// prefix of its concept (src/embed/units.js `contextPrefix`, at most 160 characters), then the
|
|
255
|
+
// heading and the section text as `sectionUnits` has them. Anchors are unchanged; the hash is
|
|
256
|
+
// that of the prefixed text, so plain section units keep their hashes and a body edit re-embeds
|
|
257
|
+
// only its section (a title, type or path change re-embeds every section of the concept).
|
|
258
|
+
function contextualSectionUnits(index) {
|
|
259
|
+
const concepts = index?.concepts || [];
|
|
260
|
+
return concepts.flatMap((doc) => {
|
|
261
|
+
const prefix = contextPrefix([doc.title, doc.type, doc.path]);
|
|
262
|
+
return conceptSections(doc).map((section) => {
|
|
263
|
+
const text = prefix ? `${prefix}\n${section.text}` : section.text;
|
|
264
|
+
return { id: section.anchor, text, hash: hashUnitText(text) };
|
|
265
|
+
});
|
|
266
|
+
});
|
|
267
|
+
}
|
|
268
|
+
|
|
269
|
+
module.exports = {
|
|
270
|
+
conceptSections,
|
|
271
|
+
contextualSectionUnits,
|
|
272
|
+
headingSlug,
|
|
273
|
+
sectionUnits,
|
|
274
|
+
};
|
|
@@ -0,0 +1,160 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
|
|
3
|
+
// The embedder contract (docs/plan-1.0.md R-8). An embedder turns texts into
|
|
4
|
+
// fixed-size float vectors. Core ships one implementation (./openai.js); further
|
|
5
|
+
// implementations are separate packages loaded like producers and must pass
|
|
6
|
+
// validateEmbedder() before the host uses them. The contract is deliberately
|
|
7
|
+
// small: everything the cache and the receipts need, nothing vendor-specific.
|
|
8
|
+
//
|
|
9
|
+
// Required fields:
|
|
10
|
+
//
|
|
11
|
+
// id string stable identity of the model as seen by the cache;
|
|
12
|
+
// a different id rebuilds every stored vector. It
|
|
13
|
+
// must cover everything that changes what a stored
|
|
14
|
+
// vector means: for the OpenAI client
|
|
15
|
+
// "openai:<endpoint>:<model>:<dimension>:<maxInputChars>".
|
|
16
|
+
// dimension integer length of every vector returned by embed().
|
|
17
|
+
// languages string[] ISO 639-1 codes the model is known to cover; an
|
|
18
|
+
// empty array means "not declared" and disables
|
|
19
|
+
// language checks. Never used to pick a model.
|
|
20
|
+
// remote boolean true whenever text leaves the machine (O-2). Only a
|
|
21
|
+
// loopback endpoint or an in-process model is local.
|
|
22
|
+
// maxBatch integer largest number of texts one embed() call accepts
|
|
23
|
+
// (the host splits larger inputs at this size).
|
|
24
|
+
// maxInputChars integer per-text truncation point the implementation applies
|
|
25
|
+
// before encoding, so callers can report truncation.
|
|
26
|
+
// embed(texts, options?) async; texts: string[] -> Promise<Float32Array[]>,
|
|
27
|
+
// same length and order as texts; every vector has
|
|
28
|
+
// `dimension` entries, all finite. Must never log or
|
|
29
|
+
// echo texts. options.signal aborts the call;
|
|
30
|
+
// options.concurrency caps the requests in flight
|
|
31
|
+
// inside the call (never above the implementation's
|
|
32
|
+
// own bound). One rule keeps the request count
|
|
33
|
+
// predictable: a host that runs several calls at
|
|
34
|
+
// once passes concurrency: 1, so the requests in
|
|
35
|
+
// flight equal the host's bound, never the product.
|
|
36
|
+
// options.skipRejected: true lets an implementation
|
|
37
|
+
// return null in place of the vector of an input
|
|
38
|
+
// the provider refuses (after any recovery of its
|
|
39
|
+
// own); a host that passes it records the unit as
|
|
40
|
+
// skipped. Without it, a refused input fails the call.
|
|
41
|
+
//
|
|
42
|
+
// Optional members:
|
|
43
|
+
//
|
|
44
|
+
// concurrency integer parallel requests the implementation issues inside
|
|
45
|
+
// one embed() call (default 1); the host uses it to
|
|
46
|
+
// size the chunks it persists between calls.
|
|
47
|
+
// usage() sync -> { requests, retries, promptTokens, totalTokens }
|
|
48
|
+
// cumulative counters since construction; the host
|
|
49
|
+
// reports the difference per run.
|
|
50
|
+
// probe() async discovers `dimension` with one minimal request when
|
|
51
|
+
// the implementation cannot know it from configuration;
|
|
52
|
+
// a no-op once known. Callers must await it before
|
|
53
|
+
// opening a cache when `dimension` is null.
|
|
54
|
+
//
|
|
55
|
+
// Errors thrown by an implementation carry a short `code` and never contain
|
|
56
|
+
// input text, credentials, or URLs with credentials (B-6, S-2).
|
|
57
|
+
|
|
58
|
+
const EMBEDDER_INTERFACE_VERSION = 1;
|
|
59
|
+
|
|
60
|
+
const MAX_ID_LENGTH = 256;
|
|
61
|
+
const MAX_DIMENSION = 65536;
|
|
62
|
+
const MAX_LANGUAGE_LENGTH = 16;
|
|
63
|
+
|
|
64
|
+
class EmbedderError extends Error {
|
|
65
|
+
constructor(message, code, details) {
|
|
66
|
+
super(message);
|
|
67
|
+
this.name = "EmbedderError";
|
|
68
|
+
this.code = code;
|
|
69
|
+
if (details !== undefined) this.details = details;
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
function invalid(field, message) {
|
|
74
|
+
return new EmbedderError(`Embedder ${field} ${message}.`, "invalid_embedder", { field });
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
function isPositiveInteger(value, max) {
|
|
78
|
+
return Number.isSafeInteger(value) && value > 0 && value <= max;
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
function hasWhitespaceOrControl(value) {
|
|
82
|
+
for (const character of value) {
|
|
83
|
+
const code = character.codePointAt(0);
|
|
84
|
+
if (code <= 0x20 || code === 0x7f) return true;
|
|
85
|
+
}
|
|
86
|
+
return false;
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
// Checks the shape of a candidate implementation and returns it unchanged.
|
|
90
|
+
// Throws EmbedderError with code "invalid_embedder" and the offending field.
|
|
91
|
+
function validateEmbedder(candidate) {
|
|
92
|
+
if (!candidate || typeof candidate !== "object") {
|
|
93
|
+
throw new EmbedderError("Embedder must be an object.", "invalid_embedder", { field: null });
|
|
94
|
+
}
|
|
95
|
+
const { id, dimension, languages, remote, maxBatch, maxInputChars } = candidate;
|
|
96
|
+
if (
|
|
97
|
+
typeof id !== "string" ||
|
|
98
|
+
!id.trim() ||
|
|
99
|
+
id.length > MAX_ID_LENGTH ||
|
|
100
|
+
hasWhitespaceOrControl(id)
|
|
101
|
+
) {
|
|
102
|
+
throw invalid("id", "must be a non-empty string without whitespace or control characters");
|
|
103
|
+
}
|
|
104
|
+
if (!isPositiveInteger(dimension, MAX_DIMENSION)) {
|
|
105
|
+
throw invalid("dimension", `must be a positive integer up to ${MAX_DIMENSION}`);
|
|
106
|
+
}
|
|
107
|
+
if (
|
|
108
|
+
!Array.isArray(languages) ||
|
|
109
|
+
languages.some(
|
|
110
|
+
(code) =>
|
|
111
|
+
typeof code !== "string" ||
|
|
112
|
+
!code ||
|
|
113
|
+
code.length > MAX_LANGUAGE_LENGTH ||
|
|
114
|
+
!/^[A-Za-z*][A-Za-z0-9-]*$/.test(code),
|
|
115
|
+
)
|
|
116
|
+
) {
|
|
117
|
+
throw invalid("languages", "must be an array of language codes");
|
|
118
|
+
}
|
|
119
|
+
if (typeof remote !== "boolean") {
|
|
120
|
+
throw invalid("remote", "must be a boolean");
|
|
121
|
+
}
|
|
122
|
+
if (!isPositiveInteger(maxBatch, Number.MAX_SAFE_INTEGER)) {
|
|
123
|
+
throw invalid("maxBatch", "must be a positive integer");
|
|
124
|
+
}
|
|
125
|
+
if (!isPositiveInteger(maxInputChars, Number.MAX_SAFE_INTEGER)) {
|
|
126
|
+
throw invalid("maxInputChars", "must be a positive integer");
|
|
127
|
+
}
|
|
128
|
+
if (typeof candidate.embed !== "function") {
|
|
129
|
+
throw invalid("embed", "must be a function");
|
|
130
|
+
}
|
|
131
|
+
if (
|
|
132
|
+
candidate.concurrency !== undefined &&
|
|
133
|
+
!isPositiveInteger(candidate.concurrency, Number.MAX_SAFE_INTEGER)
|
|
134
|
+
) {
|
|
135
|
+
throw invalid("concurrency", "must be a positive integer when present");
|
|
136
|
+
}
|
|
137
|
+
for (const optional of ["usage", "probe"]) {
|
|
138
|
+
if (candidate[optional] !== undefined && typeof candidate[optional] !== "function") {
|
|
139
|
+
throw invalid(optional, "must be a function when present");
|
|
140
|
+
}
|
|
141
|
+
}
|
|
142
|
+
return candidate;
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
// The part of an embedder that belongs in receipts (O-2): enough to explain
|
|
146
|
+
// where text went and which cache it keys, nothing that could hold a secret.
|
|
147
|
+
function describeEmbedder(embedder) {
|
|
148
|
+
return {
|
|
149
|
+
id: embedder.id,
|
|
150
|
+
remote: embedder.remote === true,
|
|
151
|
+
dimension: embedder.dimension,
|
|
152
|
+
};
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
module.exports = {
|
|
156
|
+
EMBEDDER_INTERFACE_VERSION,
|
|
157
|
+
EmbedderError,
|
|
158
|
+
describeEmbedder,
|
|
159
|
+
validateEmbedder,
|
|
160
|
+
};
|