@mfdaves/okf-mcp 0.9.0 → 0.12.0-rc.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +69 -0
- package/README.md +176 -19
- package/bin/okf-mcp.js +6 -1
- package/okf/okf-mcp/policies/authoring-safety.md +11 -1
- package/package.json +17 -5
- package/server.json +2 -2
- package/src/assets.js +165 -73
- package/src/authoring/candidates.js +391 -0
- package/src/authoring/git-preflight.js +225 -0
- package/src/authoring/git-publish.js +776 -0
- package/src/authoring/live.js +789 -0
- package/src/authoring/paths.js +112 -0
- package/src/authoring/policy-hook.js +221 -0
- package/src/authoring/proposal-policy.js +137 -0
- package/src/authoring/receipts.js +329 -0
- package/src/authoring/render.js +38 -0
- package/src/authoring/shared.js +75 -0
- package/src/authoring/staging.js +216 -0
- package/src/authoring.js +386 -287
- package/src/cache/files.js +147 -0
- package/src/cache/lock.js +201 -0
- package/src/cache/paths.js +197 -0
- package/src/cache/store.js +703 -0
- package/src/catalog-text.js +19 -0
- package/src/cli.js +184 -151
- package/src/computation.js +99 -61
- package/src/context/assemble.js +567 -0
- package/src/context/cli-options.js +88 -0
- package/src/context/score.js +234 -0
- package/src/context/sections.js +274 -0
- package/src/embed/interface.js +160 -0
- package/src/embed/openai.js +715 -0
- package/src/embed/prepare.js +182 -0
- package/src/embed/units.js +96 -0
- package/src/git-source.js +112 -52
- package/src/graph.js +95 -37
- package/src/http-server.js +36 -17
- package/src/index.js +15 -0
- package/src/indexer/assemble.js +566 -0
- package/src/indexer/build.js +290 -0
- package/src/indexer/cache.js +81 -0
- package/src/indexer/edges.js +430 -0
- package/src/indexer/index.js +44 -0
- package/src/indexer/paths.js +178 -0
- package/src/indexer/resolve.js +282 -0
- package/src/indexer/update.js +720 -0
- package/src/indexer.js +3 -1068
- package/src/live-authoring.js +4 -2329
- package/src/markdown.js +21 -16
- package/src/mcp/registry.js +328 -0
- package/src/mcp/resources.js +45 -0
- package/src/mcp/server.js +472 -0
- package/src/mcp/tools/catalog.js +244 -0
- package/src/mcp/tools/computation.js +83 -0
- package/src/mcp/tools/context.js +133 -0
- package/src/mcp/tools/graph.js +167 -0
- package/src/mcp/tools/live-authoring.js +316 -0
- package/src/mcp/tools/migration.js +86 -0
- package/src/mcp/tools/producers.js +79 -0
- package/src/mcp/tools/proposals.js +213 -0
- package/src/mcp/tools/provenance.js +87 -0
- package/src/mcp/tools/remote.js +79 -0
- package/src/mcp/tools/search.js +242 -0
- package/src/mcp-server.js +3 -1611
- package/src/migration.js +431 -253
- package/src/parser.js +245 -130
- package/src/plugins.js +71 -48
- package/src/policy/write-policy.js +713 -0
- package/src/producer-publisher.js +195 -77
- package/src/producers.js +265 -101
- package/src/project.js +277 -93
- package/src/remote.js +96 -61
- package/src/search/cli-options.js +107 -0
- package/src/search/confidence.js +61 -0
- package/src/search/dense.js +304 -0
- package/src/search/embedding-cli.js +57 -0
- package/src/search/embedding-config.js +217 -0
- package/src/search/fusion.js +249 -0
- package/src/search/hybrid.js +526 -0
- package/src/search/languages.js +200 -0
- package/src/search/lexical.js +695 -0
- package/src/search/query.js +620 -0
- package/src/search/stem/among.js +43 -0
- package/src/search/stem/base-stemmer.js +311 -0
- package/src/search/stem/en-regions.js +456 -0
- package/src/search/stem/en-suffixes.js +664 -0
- package/src/search/stem/en-tables.js +176 -0
- package/src/search/stem/en.js +238 -0
- package/src/search/stem/it-regions.js +459 -0
- package/src/search/stem/it-suffixes.js +471 -0
- package/src/search/stem/it-tables.js +257 -0
- package/src/search/stem/it.js +155 -0
- package/src/search-index.js +2 -81
- package/src/search.js +2 -252
- package/src/store.js +134 -81
- package/src/v02.js +301 -105
- package/src/validation.js +24 -20
|
@@ -0,0 +1,720 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
|
|
3
|
+
const fs = require("node:fs");
|
|
4
|
+
const { isDeepStrictEqual } = require("node:util");
|
|
5
|
+
const { collectAssetReferences } = require("../assets");
|
|
6
|
+
const { normalizeSlashes, safeRelativePath } = require("../parser");
|
|
7
|
+
const { assetKeys, completeIndex, documentKeys, indexRecord } = require("./assemble");
|
|
8
|
+
const { attachProject, buildIndex, dayKey, missingRootError, parseEntry } = require("./build");
|
|
9
|
+
const {
|
|
10
|
+
bundleAllowsPath,
|
|
11
|
+
documentAbsolutePath,
|
|
12
|
+
documentOverrideKey,
|
|
13
|
+
fileStat,
|
|
14
|
+
sameStat,
|
|
15
|
+
walkMarkdown,
|
|
16
|
+
} = require("./paths");
|
|
17
|
+
|
|
18
|
+
// Above this share of changed documents an update rebuilds the index: the
|
|
19
|
+
// bookkeeping of an incremental update no longer pays for itself, and the
|
|
20
|
+
// producer and remote-bundle paths, which replace whole bundles, land here.
|
|
21
|
+
const DEFAULT_MAX_CHANGE_RATIO = 0.2;
|
|
22
|
+
|
|
23
|
+
// Derived views over an index, memoised per index object. An index is never
|
|
24
|
+
// mutated after it is built, so a WeakMap keyed by the index object is a sound
|
|
25
|
+
// cache: a rebuilt or updated index is a new object with a fresh entry, and
|
|
26
|
+
// the entry of an index that is garbage-collected goes with it.
|
|
27
|
+
const LINK_SETS = new WeakMap();
|
|
28
|
+
const EDGES_BY_SOURCE = new WeakMap();
|
|
29
|
+
|
|
30
|
+
// Outbound and inbound neighbour sets over the index's non-broken edges, keyed
|
|
31
|
+
// by concept URI. Filters that ask "linked to" or "linked from" read these.
|
|
32
|
+
function linkSets(index) {
|
|
33
|
+
let sets = LINK_SETS.get(index);
|
|
34
|
+
if (sets) {
|
|
35
|
+
return sets;
|
|
36
|
+
}
|
|
37
|
+
const outbound = new Map();
|
|
38
|
+
const inbound = new Map();
|
|
39
|
+
(index.edges || []).forEach((edge) => {
|
|
40
|
+
if (edge.broken) {
|
|
41
|
+
return;
|
|
42
|
+
}
|
|
43
|
+
if (!outbound.has(edge.source)) {
|
|
44
|
+
outbound.set(edge.source, new Set());
|
|
45
|
+
}
|
|
46
|
+
if (!inbound.has(edge.target)) {
|
|
47
|
+
inbound.set(edge.target, new Set());
|
|
48
|
+
}
|
|
49
|
+
outbound.get(edge.source).add(edge.target);
|
|
50
|
+
inbound.get(edge.target).add(edge.source);
|
|
51
|
+
});
|
|
52
|
+
sets = { outbound, inbound };
|
|
53
|
+
LINK_SETS.set(index, sets);
|
|
54
|
+
return sets;
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
// Every edge of the index grouped by its source URI, in index edge order.
|
|
58
|
+
function edgesBySource(index) {
|
|
59
|
+
let grouped = EDGES_BY_SOURCE.get(index);
|
|
60
|
+
if (grouped) {
|
|
61
|
+
return grouped;
|
|
62
|
+
}
|
|
63
|
+
grouped = new Map();
|
|
64
|
+
(index.edges || []).forEach((edge) => {
|
|
65
|
+
const list = grouped.get(edge.source);
|
|
66
|
+
if (list) {
|
|
67
|
+
list.push(edge);
|
|
68
|
+
} else {
|
|
69
|
+
grouped.set(edge.source, [edge]);
|
|
70
|
+
}
|
|
71
|
+
});
|
|
72
|
+
EDGES_BY_SOURCE.set(index, grouped);
|
|
73
|
+
return grouped;
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
// What the last build or update of this index did: { mode: "build" |
|
|
77
|
+
// "incremental" | "full", changed, total, ratio, derived, documents, ... }.
|
|
78
|
+
// Null for an object that did not come out of buildIndex or updateIndex.
|
|
79
|
+
function indexUpdateStats(index) {
|
|
80
|
+
const record = indexRecord(index);
|
|
81
|
+
return record ? Object.assign({}, record.stats) : null;
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
// Every input that shaped an index, for a host deciding whether an index
|
|
85
|
+
// built elsewhere can stand in for a build of its own: the bundles (id,
|
|
86
|
+
// root, remote, include and exclude patterns), the relation types and flags,
|
|
87
|
+
// the date (null when it floats with the calendar) and the languages set on
|
|
88
|
+
// the index. Null for an object that did not come out of buildIndex.
|
|
89
|
+
function indexInputs(index) {
|
|
90
|
+
const record = indexRecord(index);
|
|
91
|
+
if (!record) {
|
|
92
|
+
return null;
|
|
93
|
+
}
|
|
94
|
+
return {
|
|
95
|
+
bundles: record.bundles.map(({ bundle }) => ({
|
|
96
|
+
id: bundle.id,
|
|
97
|
+
root: bundle.root,
|
|
98
|
+
remote: Boolean(bundle.remote),
|
|
99
|
+
include: (bundle.include || []).map(String),
|
|
100
|
+
exclude: (bundle.exclude || []).map(String),
|
|
101
|
+
})),
|
|
102
|
+
relationTypes: Array.from(record.config.edgeConfig.allowedRelationTypes),
|
|
103
|
+
allowCustomRelationTypes: record.config.edgeConfig.allowCustomRelationTypes,
|
|
104
|
+
strictLinks: record.config.strictLinks,
|
|
105
|
+
asOf: record.config.floatingDate ? null : record.config.asOf,
|
|
106
|
+
languages: index.languages === undefined ? null : index.languages,
|
|
107
|
+
};
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
function normalizeRelativePath(value) {
|
|
111
|
+
const relativePath = normalizeSlashes(String(value || "").trim());
|
|
112
|
+
const parts = relativePath.split("/");
|
|
113
|
+
if (
|
|
114
|
+
!relativePath ||
|
|
115
|
+
relativePath.startsWith("/") ||
|
|
116
|
+
/^[A-Za-z]:\//.test(relativePath) ||
|
|
117
|
+
parts.some((part) => !part || part === "." || part === "..")
|
|
118
|
+
) {
|
|
119
|
+
throw new TypeError(`Index update paths must be safe bundle-relative paths: ${value}`);
|
|
120
|
+
}
|
|
121
|
+
if (!relativePath.toLowerCase().endsWith(".md")) {
|
|
122
|
+
throw new TypeError(`Index update paths must name Markdown documents: ${value}`);
|
|
123
|
+
}
|
|
124
|
+
return relativePath;
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
function isParsedDocument(value) {
|
|
128
|
+
return (
|
|
129
|
+
value &&
|
|
130
|
+
typeof value === "object" &&
|
|
131
|
+
typeof value.pathUri === "string" &&
|
|
132
|
+
typeof value.bundle === "string" &&
|
|
133
|
+
typeof value.path === "string" &&
|
|
134
|
+
typeof value.text === "string"
|
|
135
|
+
);
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
function targetFromUri(index, uri) {
|
|
139
|
+
const doc = index.byPathUri?.get(uri) || index.byUri?.get(uri) || null;
|
|
140
|
+
return doc ? { bundleId: doc.bundle, relativePath: doc.path } : null;
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
function targetFromEntry(entry, label) {
|
|
144
|
+
const bundleId = String(entry.bundleId || entry.bundle || "");
|
|
145
|
+
if (!bundleId) {
|
|
146
|
+
throw new TypeError(`Index update ${label} entries need a bundleId.`);
|
|
147
|
+
}
|
|
148
|
+
return { bundleId, relativePath: normalizeRelativePath(entry.relativePath || entry.path) };
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
// A deletion names a document by URI (canonical, alias or path form), by
|
|
152
|
+
// { bundleId, relativePath }, as a parsed document, or as a bare relative
|
|
153
|
+
// path when the index has exactly one local bundle. An unknown URI deletes
|
|
154
|
+
// nothing.
|
|
155
|
+
function deletionTarget(index, record, entry) {
|
|
156
|
+
if (typeof entry === "string") {
|
|
157
|
+
if (entry.startsWith("okf://")) {
|
|
158
|
+
return targetFromUri(index, entry);
|
|
159
|
+
}
|
|
160
|
+
const locals = record.bundles.filter((bundleRecord) => !bundleRecord.bundle.remote);
|
|
161
|
+
if (locals.length !== 1) {
|
|
162
|
+
throw new TypeError(
|
|
163
|
+
"A bare relative path deletion needs exactly one local bundle; pass { bundleId, relativePath }.",
|
|
164
|
+
);
|
|
165
|
+
}
|
|
166
|
+
return { bundleId: locals[0].bundle.id, relativePath: normalizeRelativePath(entry) };
|
|
167
|
+
}
|
|
168
|
+
if (entry && typeof entry === "object") {
|
|
169
|
+
if (isParsedDocument(entry)) {
|
|
170
|
+
return { bundleId: entry.bundle, relativePath: normalizeRelativePath(entry.path) };
|
|
171
|
+
}
|
|
172
|
+
if (typeof entry.uri === "string" && !entry.bundleId && !entry.relativePath) {
|
|
173
|
+
return targetFromUri(index, entry.uri);
|
|
174
|
+
}
|
|
175
|
+
return targetFromEntry(entry, "deletion");
|
|
176
|
+
}
|
|
177
|
+
throw new TypeError(
|
|
178
|
+
"Index update deletions must be URIs, paths, parsed documents, or { bundleId, relativePath }.",
|
|
179
|
+
);
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
// An upsert is a parsed document, { bundleId, relativePath, text }, or
|
|
183
|
+
// { bundleId, relativePath } to read the document back from the bundle root.
|
|
184
|
+
function upsertTarget(entry) {
|
|
185
|
+
if (isParsedDocument(entry)) {
|
|
186
|
+
return { bundleId: entry.bundle, relativePath: normalizeRelativePath(entry.path), doc: entry };
|
|
187
|
+
}
|
|
188
|
+
if (!entry || typeof entry !== "object") {
|
|
189
|
+
throw new TypeError(
|
|
190
|
+
"Index update upserts must be parsed documents or { bundleId, relativePath, text? }.",
|
|
191
|
+
);
|
|
192
|
+
}
|
|
193
|
+
const target = targetFromEntry(entry, "upsert");
|
|
194
|
+
target.text = entry.text === undefined ? undefined : String(entry.text);
|
|
195
|
+
return target;
|
|
196
|
+
}
|
|
197
|
+
|
|
198
|
+
function bundleRecordFor(record, bundleId) {
|
|
199
|
+
const found = record.bundles.find((bundleRecord) => bundleRecord.bundle.id === bundleId);
|
|
200
|
+
if (!found) {
|
|
201
|
+
throw new TypeError(`Unknown bundle in index update: ${bundleId}`);
|
|
202
|
+
}
|
|
203
|
+
if (found.bundle.remote) {
|
|
204
|
+
throw new TypeError(`Remote bundles cannot be updated in place: ${bundleId}`);
|
|
205
|
+
}
|
|
206
|
+
return found;
|
|
207
|
+
}
|
|
208
|
+
|
|
209
|
+
// Groups the requested changes per bundle. Deletions apply before upserts, so
|
|
210
|
+
// a path in both is an upsert; paths the bundle policy excludes are dropped,
|
|
211
|
+
// as a build would drop them; deletions of unknown paths count for nothing.
|
|
212
|
+
// `assetsChanged` says the asset files may differ from what the index saw,
|
|
213
|
+
// so the registry is rebuilt and the documents that depend on assets are
|
|
214
|
+
// re-derived.
|
|
215
|
+
function planChanges(index, record, changes) {
|
|
216
|
+
const input = changes || {};
|
|
217
|
+
const plan = { bundles: new Map(), changed: 0, assetsChanged: Boolean(input.assetsChanged) };
|
|
218
|
+
const touch = (bundleId) => {
|
|
219
|
+
let state = plan.bundles.get(bundleId);
|
|
220
|
+
if (!state) {
|
|
221
|
+
const bundleRecord = bundleRecordFor(record, bundleId);
|
|
222
|
+
state = {
|
|
223
|
+
bundleRecord,
|
|
224
|
+
known: new Set(bundleRecord.entries.map((entry) => entry.relativePath)),
|
|
225
|
+
deletions: new Set(),
|
|
226
|
+
upserts: new Map(),
|
|
227
|
+
};
|
|
228
|
+
plan.bundles.set(bundleId, state);
|
|
229
|
+
}
|
|
230
|
+
return state;
|
|
231
|
+
};
|
|
232
|
+
(Array.isArray(input.deletions) ? input.deletions : []).forEach((raw) => {
|
|
233
|
+
const target = deletionTarget(index, record, raw);
|
|
234
|
+
if (!target) {
|
|
235
|
+
return;
|
|
236
|
+
}
|
|
237
|
+
const state = touch(target.bundleId);
|
|
238
|
+
if (state.known.has(target.relativePath)) {
|
|
239
|
+
state.deletions.add(target.relativePath);
|
|
240
|
+
}
|
|
241
|
+
});
|
|
242
|
+
(Array.isArray(input.upserts) ? input.upserts : []).forEach((raw) => {
|
|
243
|
+
const target = upsertTarget(raw);
|
|
244
|
+
const state = touch(target.bundleId);
|
|
245
|
+
if (!bundleAllowsPath(state.bundleRecord.bundle, target.relativePath)) {
|
|
246
|
+
return;
|
|
247
|
+
}
|
|
248
|
+
state.deletions.delete(target.relativePath);
|
|
249
|
+
state.upserts.set(target.relativePath, target);
|
|
250
|
+
});
|
|
251
|
+
plan.bundles.forEach((state) => {
|
|
252
|
+
plan.changed += state.deletions.size + state.upserts.size;
|
|
253
|
+
});
|
|
254
|
+
return plan;
|
|
255
|
+
}
|
|
256
|
+
|
|
257
|
+
// A full build with the same inputs as the index, for the given day. What the
|
|
258
|
+
// index holds apart from the files (documents kept as text, documents removed
|
|
259
|
+
// while their files remain) and the requested changes are folded into the
|
|
260
|
+
// document overrides and deletions of this one build, so the rebuilt index
|
|
261
|
+
// describes the same state; the rebuilt record keeps the original options and
|
|
262
|
+
// carries the overlay state on its entries and in overlayDeletions, never as
|
|
263
|
+
// pins that would outlive the files.
|
|
264
|
+
function rebuildIndex(record, plan, stats, asOf) {
|
|
265
|
+
const overrides = new Map(record.config.documentOverrides);
|
|
266
|
+
const deletions = new Set(record.config.documentDeletions);
|
|
267
|
+
const overlayDeletions = new Set(record.overlayDeletions);
|
|
268
|
+
overlayDeletions.forEach((key) => {
|
|
269
|
+
deletions.add(key);
|
|
270
|
+
});
|
|
271
|
+
record.bundles.forEach((bundleRecord) => {
|
|
272
|
+
bundleRecord.entries.forEach((entry) => {
|
|
273
|
+
if (entry.source === "text") {
|
|
274
|
+
overrides.set(documentOverrideKey(bundleRecord.bundle.id, entry.relativePath), entry.text);
|
|
275
|
+
}
|
|
276
|
+
});
|
|
277
|
+
});
|
|
278
|
+
plan.bundles.forEach((state, bundleId) => {
|
|
279
|
+
state.deletions.forEach((relativePath) => {
|
|
280
|
+
const key = documentOverrideKey(bundleId, relativePath);
|
|
281
|
+
overrides.delete(key);
|
|
282
|
+
deletions.add(key);
|
|
283
|
+
if (fs.existsSync(documentAbsolutePath(state.bundleRecord.bundle, relativePath))) {
|
|
284
|
+
overlayDeletions.add(key);
|
|
285
|
+
} else {
|
|
286
|
+
overlayDeletions.delete(key);
|
|
287
|
+
}
|
|
288
|
+
});
|
|
289
|
+
state.upserts.forEach((upsert, relativePath) => {
|
|
290
|
+
const key = documentOverrideKey(bundleId, relativePath);
|
|
291
|
+
deletions.delete(key);
|
|
292
|
+
overlayDeletions.delete(key);
|
|
293
|
+
if (upsert.doc) {
|
|
294
|
+
overrides.set(key, upsert.doc.text);
|
|
295
|
+
} else if (upsert.text !== undefined) {
|
|
296
|
+
overrides.set(key, upsert.text);
|
|
297
|
+
} else {
|
|
298
|
+
overrides.delete(key);
|
|
299
|
+
}
|
|
300
|
+
});
|
|
301
|
+
});
|
|
302
|
+
const rebuilt = buildIndex(
|
|
303
|
+
record.bundleArgs,
|
|
304
|
+
Object.assign({}, record.options, {
|
|
305
|
+
documentOverrides: overrides,
|
|
306
|
+
documentDeletions: deletions,
|
|
307
|
+
asOf,
|
|
308
|
+
}),
|
|
309
|
+
);
|
|
310
|
+
if (record.project) {
|
|
311
|
+
attachProject(rebuilt, record.project);
|
|
312
|
+
}
|
|
313
|
+
const next = indexRecord(rebuilt);
|
|
314
|
+
next.options = record.options;
|
|
315
|
+
next.config = Object.assign({}, record.config, { asOf });
|
|
316
|
+
next.overlayDeletions = overlayDeletions;
|
|
317
|
+
next.stats = Object.assign(stats, { mode: "full" });
|
|
318
|
+
return rebuilt;
|
|
319
|
+
}
|
|
320
|
+
|
|
321
|
+
function insertSorted(entries, entry) {
|
|
322
|
+
let position = 0;
|
|
323
|
+
while (
|
|
324
|
+
position < entries.length &&
|
|
325
|
+
(entries[position] === null || !(entries[position].absolutePath > entry.absolutePath))
|
|
326
|
+
) {
|
|
327
|
+
position += 1;
|
|
328
|
+
}
|
|
329
|
+
entries.splice(position, 0, entry);
|
|
330
|
+
}
|
|
331
|
+
|
|
332
|
+
// Applies one bundle's deletions and upserts to a copy of its entries. A
|
|
333
|
+
// replaced document keeps its position; a new one is inserted where a walk
|
|
334
|
+
// would have listed it, after every replacement, because an insertion shifts
|
|
335
|
+
// the positions of the entries behind it. Returns the documents that left
|
|
336
|
+
// and entered.
|
|
337
|
+
function applyBundleChanges(bundleRecord, state, config, changes) {
|
|
338
|
+
const bundle = bundleRecord.bundle;
|
|
339
|
+
if (!fs.existsSync(bundle.root)) {
|
|
340
|
+
bundleRecord.entries.forEach((entry) => {
|
|
341
|
+
if (entry.doc) {
|
|
342
|
+
changes.removed.push({ relativePath: entry.relativePath, entry });
|
|
343
|
+
}
|
|
344
|
+
changes.restored.add(documentOverrideKey(bundle.id, entry.relativePath));
|
|
345
|
+
});
|
|
346
|
+
bundleRecord.entries = [];
|
|
347
|
+
bundleRecord.rootMissing = missingRootError(bundle);
|
|
348
|
+
return;
|
|
349
|
+
}
|
|
350
|
+
bundleRecord.rootMissing = null;
|
|
351
|
+
const entries = bundleRecord.entries;
|
|
352
|
+
const positions = new Map(entries.map((entry, position) => [entry.relativePath, position]));
|
|
353
|
+
state.deletions.forEach((relativePath) => {
|
|
354
|
+
const position = positions.get(relativePath);
|
|
355
|
+
const old = entries[position];
|
|
356
|
+
if (old.doc) {
|
|
357
|
+
changes.removed.push({ relativePath, entry: old });
|
|
358
|
+
}
|
|
359
|
+
entries[position] = null;
|
|
360
|
+
// A document removed while its file remains is an overlay deletion that
|
|
361
|
+
// a later full rebuild has to repeat; one whose file is gone is not.
|
|
362
|
+
const key = documentOverrideKey(bundle.id, relativePath);
|
|
363
|
+
if (fs.existsSync(documentAbsolutePath(bundle, relativePath))) {
|
|
364
|
+
changes.overlayDeleted.add(key);
|
|
365
|
+
} else {
|
|
366
|
+
changes.restored.add(key);
|
|
367
|
+
}
|
|
368
|
+
});
|
|
369
|
+
const inserted = [];
|
|
370
|
+
state.upserts.forEach((upsert, relativePath) => {
|
|
371
|
+
const absolutePath = documentAbsolutePath(bundle, relativePath);
|
|
372
|
+
let entry;
|
|
373
|
+
if (upsert.doc) {
|
|
374
|
+
entry = {
|
|
375
|
+
relativePath,
|
|
376
|
+
absolutePath,
|
|
377
|
+
doc: upsert.doc,
|
|
378
|
+
parseError: null,
|
|
379
|
+
stat: null,
|
|
380
|
+
source: "text",
|
|
381
|
+
text: upsert.doc.text,
|
|
382
|
+
derived: null,
|
|
383
|
+
};
|
|
384
|
+
} else {
|
|
385
|
+
entry = parseEntry(bundle, relativePath, absolutePath, upsert.text, config);
|
|
386
|
+
changes.owned.add(entry);
|
|
387
|
+
}
|
|
388
|
+
const position = positions.get(relativePath);
|
|
389
|
+
if (position !== undefined) {
|
|
390
|
+
const old = entries[position];
|
|
391
|
+
if (old?.doc) {
|
|
392
|
+
changes.removed.push({ relativePath, entry: old });
|
|
393
|
+
}
|
|
394
|
+
entries[position] = entry;
|
|
395
|
+
} else {
|
|
396
|
+
inserted.push(entry);
|
|
397
|
+
}
|
|
398
|
+
changes.added.push({ relativePath, entry });
|
|
399
|
+
changes.restored.add(documentOverrideKey(bundle.id, relativePath));
|
|
400
|
+
});
|
|
401
|
+
inserted.forEach((entry) => {
|
|
402
|
+
insertSorted(entries, entry);
|
|
403
|
+
});
|
|
404
|
+
bundleRecord.entries = entries.filter(Boolean);
|
|
405
|
+
}
|
|
406
|
+
|
|
407
|
+
function assetReferencesChanged(changes) {
|
|
408
|
+
const before = new Map();
|
|
409
|
+
changes.removed.forEach(({ entry }) => {
|
|
410
|
+
before.set(`${entry.doc.bundle}\u0000${entry.relativePath}`, entry.derived.assetReferences);
|
|
411
|
+
});
|
|
412
|
+
const after = new Map();
|
|
413
|
+
changes.added.forEach(({ entry }) => {
|
|
414
|
+
if (entry.doc) {
|
|
415
|
+
after.set(
|
|
416
|
+
`${entry.doc.bundle}\u0000${entry.relativePath}`,
|
|
417
|
+
collectAssetReferences(entry.doc),
|
|
418
|
+
);
|
|
419
|
+
}
|
|
420
|
+
});
|
|
421
|
+
const keys = new Set([...before.keys(), ...after.keys()]);
|
|
422
|
+
for (const key of keys) {
|
|
423
|
+
if (!isDeepStrictEqual(before.get(key) || [], after.get(key) || [])) {
|
|
424
|
+
return true;
|
|
425
|
+
}
|
|
426
|
+
}
|
|
427
|
+
return false;
|
|
428
|
+
}
|
|
429
|
+
|
|
430
|
+
function applyIncrementally(record, plan, stats) {
|
|
431
|
+
const changes = {
|
|
432
|
+
owned: new Set(),
|
|
433
|
+
removed: [],
|
|
434
|
+
added: [],
|
|
435
|
+
overlayDeleted: new Set(),
|
|
436
|
+
restored: new Set(),
|
|
437
|
+
};
|
|
438
|
+
const bundleRecords = record.bundles.map((bundleRecord) => {
|
|
439
|
+
const copy = Object.assign({}, bundleRecord, { entries: bundleRecord.entries.slice() });
|
|
440
|
+
const state = plan.bundles.get(bundleRecord.bundle.id);
|
|
441
|
+
if (state) {
|
|
442
|
+
applyBundleChanges(copy, state, record.config, changes);
|
|
443
|
+
}
|
|
444
|
+
return copy;
|
|
445
|
+
});
|
|
446
|
+
const dirty = new Set();
|
|
447
|
+
const changedUris = new Set();
|
|
448
|
+
changes.removed.forEach(({ entry }) => {
|
|
449
|
+
documentKeys(entry.doc).forEach((key) => {
|
|
450
|
+
dirty.add(key);
|
|
451
|
+
});
|
|
452
|
+
changedUris.add(entry.doc.uri);
|
|
453
|
+
});
|
|
454
|
+
changes.added.forEach(({ entry }) => {
|
|
455
|
+
if (entry.doc) {
|
|
456
|
+
documentKeys(entry.doc).forEach((key) => {
|
|
457
|
+
dirty.add(key);
|
|
458
|
+
});
|
|
459
|
+
changedUris.add(entry.doc.uri);
|
|
460
|
+
}
|
|
461
|
+
});
|
|
462
|
+
const registryChanged = plan.assetsChanged || assetReferencesChanged(changes);
|
|
463
|
+
if (registryChanged) {
|
|
464
|
+
assetKeys(record.assets.byAssetKey, record.assets.byAssetUri).forEach((key) => {
|
|
465
|
+
dirty.add(key);
|
|
466
|
+
});
|
|
467
|
+
}
|
|
468
|
+
const changedEntries = new Set(changes.added.map(({ entry }) => entry));
|
|
469
|
+
const previousDuplicates = record.registration.duplicateErrors.length > 0;
|
|
470
|
+
const overlayDeletions = new Set(record.overlayDeletions);
|
|
471
|
+
changes.restored.forEach((key) => {
|
|
472
|
+
overlayDeletions.delete(key);
|
|
473
|
+
});
|
|
474
|
+
changes.overlayDeleted.forEach((key) => {
|
|
475
|
+
overlayDeletions.add(key);
|
|
476
|
+
});
|
|
477
|
+
const next = {
|
|
478
|
+
bundleArgs: record.bundleArgs,
|
|
479
|
+
options: record.options,
|
|
480
|
+
config: record.config,
|
|
481
|
+
project: record.project,
|
|
482
|
+
duplicateBundleErrors: record.duplicateBundleErrors,
|
|
483
|
+
bundles: bundleRecords,
|
|
484
|
+
registration: null,
|
|
485
|
+
assets: null,
|
|
486
|
+
overlayDeletions,
|
|
487
|
+
stats,
|
|
488
|
+
};
|
|
489
|
+
const selectAffected = ({ registration, assetState, located }) => {
|
|
490
|
+
if (registryChanged) {
|
|
491
|
+
assetKeys(assetState.byAssetKey, assetState.byAssetUri).forEach((key) => {
|
|
492
|
+
dirty.add(key);
|
|
493
|
+
});
|
|
494
|
+
}
|
|
495
|
+
const pooled = previousDuplicates || registration.duplicateErrors.length > 0;
|
|
496
|
+
const affected = [];
|
|
497
|
+
const chosen = new Set();
|
|
498
|
+
// The entry object is shared with the input index; re-derive a copy.
|
|
499
|
+
const rederive = (item) => {
|
|
500
|
+
const copy = Object.assign({}, item.entry);
|
|
501
|
+
item.bundleRecord.entries[item.position] = copy;
|
|
502
|
+
item.entry = copy;
|
|
503
|
+
chosen.add(item);
|
|
504
|
+
affected.push(item);
|
|
505
|
+
};
|
|
506
|
+
located.forEach((item) => {
|
|
507
|
+
if (changedEntries.has(item.entry)) {
|
|
508
|
+
chosen.add(item);
|
|
509
|
+
affected.push(item);
|
|
510
|
+
return;
|
|
511
|
+
}
|
|
512
|
+
const derived = item.entry.derived;
|
|
513
|
+
let hit = pooled && changedUris.has(item.entry.doc.uri);
|
|
514
|
+
if (!hit) {
|
|
515
|
+
for (const key of derived.dependsOn) {
|
|
516
|
+
if (dirty.has(key)) {
|
|
517
|
+
hit = true;
|
|
518
|
+
break;
|
|
519
|
+
}
|
|
520
|
+
}
|
|
521
|
+
}
|
|
522
|
+
if (hit) {
|
|
523
|
+
rederive(item);
|
|
524
|
+
}
|
|
525
|
+
});
|
|
526
|
+
if (pooled) {
|
|
527
|
+
// Readiness pools the edges of every document sharing a canonical URI,
|
|
528
|
+
// so when one owner is re-derived the others are too.
|
|
529
|
+
const affectedUris = new Set(affected.map((item) => item.entry.doc.uri));
|
|
530
|
+
located.forEach((item) => {
|
|
531
|
+
if (!chosen.has(item) && affectedUris.has(item.entry.doc.uri)) {
|
|
532
|
+
rederive(item);
|
|
533
|
+
}
|
|
534
|
+
});
|
|
535
|
+
}
|
|
536
|
+
stats.dependents = affected.length - changes.added.filter(({ entry }) => entry.doc).length;
|
|
537
|
+
return affected;
|
|
538
|
+
};
|
|
539
|
+
stats.mode = "incremental";
|
|
540
|
+
stats.registry = registryChanged ? "rebuilt" : "carried";
|
|
541
|
+
return completeIndex(next, {
|
|
542
|
+
owned: changes.owned,
|
|
543
|
+
assets: registryChanged ? null : record.assets,
|
|
544
|
+
selectAffected,
|
|
545
|
+
pooledReadiness: previousDuplicates,
|
|
546
|
+
});
|
|
547
|
+
}
|
|
548
|
+
|
|
549
|
+
// Settings a host puts on the index object itself rather than through the
|
|
550
|
+
// project (root mode sets the stemming `languages` there) travel to the index
|
|
551
|
+
// an update produces; a value assembly already took from the project is kept.
|
|
552
|
+
function carryIndexSettings(from, to) {
|
|
553
|
+
if (to !== from && to.languages === undefined && from.languages !== undefined) {
|
|
554
|
+
to.languages = from.languages;
|
|
555
|
+
}
|
|
556
|
+
return to;
|
|
557
|
+
}
|
|
558
|
+
|
|
559
|
+
// Returns a new index with the given documents added, replaced or removed.
|
|
560
|
+
// The input index is never mutated; unchanged documents, edges and derived
|
|
561
|
+
// lists are shared with it. Only the changed documents and the documents
|
|
562
|
+
// whose edges resolved through a key the change dirtied are re-derived;
|
|
563
|
+
// lookup maps are re-registered in index order (a linear pass); the asset
|
|
564
|
+
// registry is carried unless a changed document's asset references changed
|
|
565
|
+
// or `changes.assetsChanged` says the asset files did; validation is
|
|
566
|
+
// recomputed in full. Above `maxChangeRatio` (changed documents over
|
|
567
|
+
// documents in the index, default 0.2) the index is rebuilt instead, and so
|
|
568
|
+
// is an index built without an asOf when `options.asOf` (default: now) falls
|
|
569
|
+
// on another UTC day than the one its documents were stamped with: one full
|
|
570
|
+
// build per day per process, which in the live service runs under the
|
|
571
|
+
// writer lock, so the first batch of a day pays a build's latency.
|
|
572
|
+
function updateIndex(index, changes, options) {
|
|
573
|
+
const record = indexRecord(index);
|
|
574
|
+
if (!record) {
|
|
575
|
+
throw new TypeError("updateIndex needs an index produced by buildIndex.");
|
|
576
|
+
}
|
|
577
|
+
const config = options || {};
|
|
578
|
+
const maxChangeRatio =
|
|
579
|
+
config.maxChangeRatio === undefined ? DEFAULT_MAX_CHANGE_RATIO : Number(config.maxChangeRatio);
|
|
580
|
+
if (!(maxChangeRatio >= 0)) {
|
|
581
|
+
throw new TypeError("maxChangeRatio must be a non-negative number.");
|
|
582
|
+
}
|
|
583
|
+
const plan = planChanges(index, record, changes);
|
|
584
|
+
const asOf = record.config.floatingDate ? dayKey(config.asOf) : record.config.asOf;
|
|
585
|
+
const dateChanged = asOf !== record.config.asOf;
|
|
586
|
+
if (plan.changed === 0 && !plan.assetsChanged && !dateChanged) {
|
|
587
|
+
return index;
|
|
588
|
+
}
|
|
589
|
+
const total = index.documents.length;
|
|
590
|
+
const ratio = plan.changed / Math.max(1, total);
|
|
591
|
+
const stats = {
|
|
592
|
+
mode: null,
|
|
593
|
+
changed: plan.changed,
|
|
594
|
+
total,
|
|
595
|
+
ratio,
|
|
596
|
+
maxChangeRatio,
|
|
597
|
+
asOf,
|
|
598
|
+
assets: plan.assetsChanged ? "changed" : "unchanged",
|
|
599
|
+
};
|
|
600
|
+
if (dateChanged || ratio > maxChangeRatio) {
|
|
601
|
+
stats.reason = dateChanged ? "date" : "ratio";
|
|
602
|
+
return carryIndexSettings(index, rebuildIndex(record, plan, stats, asOf));
|
|
603
|
+
}
|
|
604
|
+
return carryIndexSettings(index, applyIncrementally(record, plan, stats));
|
|
605
|
+
}
|
|
606
|
+
|
|
607
|
+
// Whether the file on disk is what the entry already describes. A disk-backed
|
|
608
|
+
// entry is compared by size and mtime (nanoseconds). An entry held as text is
|
|
609
|
+
// compared by content; when the file carries the same text the entry becomes
|
|
610
|
+
// disk-backed in place (a cache fill that changes nothing the entry
|
|
611
|
+
// describes), so later refreshes need only a stat. A file that vanished
|
|
612
|
+
// mid-walk counts as changed and the update records whatever it finds.
|
|
613
|
+
//
|
|
614
|
+
// Limit: a rewrite that keeps a file's size and mtime is invisible here, for
|
|
615
|
+
// documents and (in assetsChangedOnDisk) for assets alike. The write path
|
|
616
|
+
// covers the documents a batch targets by comparing content hashes and
|
|
617
|
+
// re-reading on a mismatch (live.js, updateCandidate); the dependencies of a
|
|
618
|
+
// batch are not re-read. Before the incremental index, the write path built
|
|
619
|
+
// from disk every time, so every file was read; now a file whose stat is
|
|
620
|
+
// unchanged is trusted.
|
|
621
|
+
function fileMatchesEntry(entry, filePath) {
|
|
622
|
+
const stat = fileStat(filePath);
|
|
623
|
+
if (!stat) {
|
|
624
|
+
return false;
|
|
625
|
+
}
|
|
626
|
+
if (entry.source === "disk") {
|
|
627
|
+
return sameStat(entry.stat, stat);
|
|
628
|
+
}
|
|
629
|
+
if (entry.source !== "text") {
|
|
630
|
+
return false;
|
|
631
|
+
}
|
|
632
|
+
let text;
|
|
633
|
+
try {
|
|
634
|
+
text = new TextDecoder("utf-8", { fatal: true }).decode(fs.readFileSync(filePath));
|
|
635
|
+
} catch {
|
|
636
|
+
return false;
|
|
637
|
+
}
|
|
638
|
+
if (text !== entry.text) {
|
|
639
|
+
return false;
|
|
640
|
+
}
|
|
641
|
+
entry.stat = stat;
|
|
642
|
+
entry.source = "disk";
|
|
643
|
+
return true;
|
|
644
|
+
}
|
|
645
|
+
|
|
646
|
+
// Whether any asset the documents reference was modified, removed or added
|
|
647
|
+
// since the index recorded it, judged by size and mtime of the watched paths.
|
|
648
|
+
function assetsChangedOnDisk(record) {
|
|
649
|
+
for (const [absolutePath, recorded] of record.assets.watch) {
|
|
650
|
+
if (!sameStat(recorded, fileStat(absolutePath))) {
|
|
651
|
+
return true;
|
|
652
|
+
}
|
|
653
|
+
}
|
|
654
|
+
return false;
|
|
655
|
+
}
|
|
656
|
+
|
|
657
|
+
// Brings an index up to date with its local bundle roots by comparing each
|
|
658
|
+
// Markdown file's and each referenced asset's size and mtime with what the
|
|
659
|
+
// index recorded, then applying the differences through updateIndex (which
|
|
660
|
+
// also notices a changed day; `options` are its options). Returns the same
|
|
661
|
+
// index object when nothing changed. Paths pinned by document overrides or
|
|
662
|
+
// deletions at build time are left as built.
|
|
663
|
+
function refreshIndex(index, options) {
|
|
664
|
+
const record = indexRecord(index);
|
|
665
|
+
if (!record) {
|
|
666
|
+
throw new TypeError("refreshIndex needs an index produced by buildIndex.");
|
|
667
|
+
}
|
|
668
|
+
const upserts = [];
|
|
669
|
+
const deletions = [];
|
|
670
|
+
record.bundles.forEach((bundleRecord) => {
|
|
671
|
+
const bundle = bundleRecord.bundle;
|
|
672
|
+
if (bundle.remote) {
|
|
673
|
+
return;
|
|
674
|
+
}
|
|
675
|
+
const pinned = (key) =>
|
|
676
|
+
record.config.documentOverrides.has(key) || record.config.documentDeletions.has(key);
|
|
677
|
+
const seen = new Set();
|
|
678
|
+
if (fs.existsSync(bundle.root)) {
|
|
679
|
+
const byPath = new Map(bundleRecord.entries.map((entry) => [entry.relativePath, entry]));
|
|
680
|
+
walkMarkdown(bundle.root).forEach((filePath) => {
|
|
681
|
+
const relativePath = safeRelativePath(bundle.root, filePath) || normalizeSlashes(filePath);
|
|
682
|
+
if (!bundleAllowsPath(bundle, relativePath)) {
|
|
683
|
+
return;
|
|
684
|
+
}
|
|
685
|
+
const key = documentOverrideKey(bundle.id, relativePath);
|
|
686
|
+
if (pinned(key)) {
|
|
687
|
+
return;
|
|
688
|
+
}
|
|
689
|
+
seen.add(relativePath);
|
|
690
|
+
const entry = byPath.get(relativePath);
|
|
691
|
+
if (!entry || !fileMatchesEntry(entry, filePath)) {
|
|
692
|
+
upserts.push({ bundleId: bundle.id, relativePath });
|
|
693
|
+
}
|
|
694
|
+
});
|
|
695
|
+
}
|
|
696
|
+
bundleRecord.entries.forEach((entry) => {
|
|
697
|
+
if (
|
|
698
|
+
!seen.has(entry.relativePath) &&
|
|
699
|
+
!pinned(documentOverrideKey(bundle.id, entry.relativePath))
|
|
700
|
+
) {
|
|
701
|
+
deletions.push({ bundleId: bundle.id, relativePath: entry.relativePath });
|
|
702
|
+
}
|
|
703
|
+
});
|
|
704
|
+
});
|
|
705
|
+
return updateIndex(
|
|
706
|
+
index,
|
|
707
|
+
{ upserts, deletions, assetsChanged: assetsChangedOnDisk(record) },
|
|
708
|
+
options,
|
|
709
|
+
);
|
|
710
|
+
}
|
|
711
|
+
|
|
712
|
+
module.exports = {
|
|
713
|
+
DEFAULT_MAX_CHANGE_RATIO,
|
|
714
|
+
edgesBySource,
|
|
715
|
+
indexInputs,
|
|
716
|
+
indexUpdateStats,
|
|
717
|
+
linkSets,
|
|
718
|
+
refreshIndex,
|
|
719
|
+
updateIndex,
|
|
720
|
+
};
|