@mfdaves/okf-mcp 0.9.0 → 0.12.0-rc.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (97) hide show
  1. package/CHANGELOG.md +69 -0
  2. package/README.md +176 -19
  3. package/bin/okf-mcp.js +6 -1
  4. package/okf/okf-mcp/policies/authoring-safety.md +11 -1
  5. package/package.json +17 -5
  6. package/server.json +2 -2
  7. package/src/assets.js +165 -73
  8. package/src/authoring/candidates.js +391 -0
  9. package/src/authoring/git-preflight.js +225 -0
  10. package/src/authoring/git-publish.js +776 -0
  11. package/src/authoring/live.js +789 -0
  12. package/src/authoring/paths.js +112 -0
  13. package/src/authoring/policy-hook.js +221 -0
  14. package/src/authoring/proposal-policy.js +137 -0
  15. package/src/authoring/receipts.js +329 -0
  16. package/src/authoring/render.js +38 -0
  17. package/src/authoring/shared.js +75 -0
  18. package/src/authoring/staging.js +216 -0
  19. package/src/authoring.js +386 -287
  20. package/src/cache/files.js +147 -0
  21. package/src/cache/lock.js +201 -0
  22. package/src/cache/paths.js +197 -0
  23. package/src/cache/store.js +703 -0
  24. package/src/catalog-text.js +19 -0
  25. package/src/cli.js +184 -151
  26. package/src/computation.js +99 -61
  27. package/src/context/assemble.js +567 -0
  28. package/src/context/cli-options.js +88 -0
  29. package/src/context/score.js +234 -0
  30. package/src/context/sections.js +274 -0
  31. package/src/embed/interface.js +160 -0
  32. package/src/embed/openai.js +715 -0
  33. package/src/embed/prepare.js +182 -0
  34. package/src/embed/units.js +96 -0
  35. package/src/git-source.js +112 -52
  36. package/src/graph.js +95 -37
  37. package/src/http-server.js +36 -17
  38. package/src/index.js +15 -0
  39. package/src/indexer/assemble.js +566 -0
  40. package/src/indexer/build.js +290 -0
  41. package/src/indexer/cache.js +81 -0
  42. package/src/indexer/edges.js +430 -0
  43. package/src/indexer/index.js +44 -0
  44. package/src/indexer/paths.js +178 -0
  45. package/src/indexer/resolve.js +282 -0
  46. package/src/indexer/update.js +720 -0
  47. package/src/indexer.js +3 -1068
  48. package/src/live-authoring.js +4 -2329
  49. package/src/markdown.js +21 -16
  50. package/src/mcp/registry.js +328 -0
  51. package/src/mcp/resources.js +45 -0
  52. package/src/mcp/server.js +472 -0
  53. package/src/mcp/tools/catalog.js +244 -0
  54. package/src/mcp/tools/computation.js +83 -0
  55. package/src/mcp/tools/context.js +133 -0
  56. package/src/mcp/tools/graph.js +167 -0
  57. package/src/mcp/tools/live-authoring.js +316 -0
  58. package/src/mcp/tools/migration.js +86 -0
  59. package/src/mcp/tools/producers.js +79 -0
  60. package/src/mcp/tools/proposals.js +213 -0
  61. package/src/mcp/tools/provenance.js +87 -0
  62. package/src/mcp/tools/remote.js +79 -0
  63. package/src/mcp/tools/search.js +242 -0
  64. package/src/mcp-server.js +3 -1611
  65. package/src/migration.js +431 -253
  66. package/src/parser.js +245 -130
  67. package/src/plugins.js +71 -48
  68. package/src/policy/write-policy.js +713 -0
  69. package/src/producer-publisher.js +195 -77
  70. package/src/producers.js +265 -101
  71. package/src/project.js +277 -93
  72. package/src/remote.js +96 -61
  73. package/src/search/cli-options.js +107 -0
  74. package/src/search/confidence.js +61 -0
  75. package/src/search/dense.js +304 -0
  76. package/src/search/embedding-cli.js +57 -0
  77. package/src/search/embedding-config.js +217 -0
  78. package/src/search/fusion.js +249 -0
  79. package/src/search/hybrid.js +526 -0
  80. package/src/search/languages.js +200 -0
  81. package/src/search/lexical.js +695 -0
  82. package/src/search/query.js +620 -0
  83. package/src/search/stem/among.js +43 -0
  84. package/src/search/stem/base-stemmer.js +311 -0
  85. package/src/search/stem/en-regions.js +456 -0
  86. package/src/search/stem/en-suffixes.js +664 -0
  87. package/src/search/stem/en-tables.js +176 -0
  88. package/src/search/stem/en.js +238 -0
  89. package/src/search/stem/it-regions.js +459 -0
  90. package/src/search/stem/it-suffixes.js +471 -0
  91. package/src/search/stem/it-tables.js +257 -0
  92. package/src/search/stem/it.js +155 -0
  93. package/src/search-index.js +2 -81
  94. package/src/search.js +2 -252
  95. package/src/store.js +134 -81
  96. package/src/v02.js +301 -105
  97. package/src/validation.js +24 -20
@@ -0,0 +1,720 @@
1
+ "use strict";
2
+
3
+ const fs = require("node:fs");
4
+ const { isDeepStrictEqual } = require("node:util");
5
+ const { collectAssetReferences } = require("../assets");
6
+ const { normalizeSlashes, safeRelativePath } = require("../parser");
7
+ const { assetKeys, completeIndex, documentKeys, indexRecord } = require("./assemble");
8
+ const { attachProject, buildIndex, dayKey, missingRootError, parseEntry } = require("./build");
9
+ const {
10
+ bundleAllowsPath,
11
+ documentAbsolutePath,
12
+ documentOverrideKey,
13
+ fileStat,
14
+ sameStat,
15
+ walkMarkdown,
16
+ } = require("./paths");
17
+
18
+ // Above this share of changed documents an update rebuilds the index: the
19
+ // bookkeeping of an incremental update no longer pays for itself, and the
20
+ // producer and remote-bundle paths, which replace whole bundles, land here.
21
+ const DEFAULT_MAX_CHANGE_RATIO = 0.2;
22
+
23
+ // Derived views over an index, memoised per index object. An index is never
24
+ // mutated after it is built, so a WeakMap keyed by the index object is a sound
25
+ // cache: a rebuilt or updated index is a new object with a fresh entry, and
26
+ // the entry of an index that is garbage-collected goes with it.
27
+ const LINK_SETS = new WeakMap();
28
+ const EDGES_BY_SOURCE = new WeakMap();
29
+
30
+ // Outbound and inbound neighbour sets over the index's non-broken edges, keyed
31
+ // by concept URI. Filters that ask "linked to" or "linked from" read these.
32
+ function linkSets(index) {
33
+ let sets = LINK_SETS.get(index);
34
+ if (sets) {
35
+ return sets;
36
+ }
37
+ const outbound = new Map();
38
+ const inbound = new Map();
39
+ (index.edges || []).forEach((edge) => {
40
+ if (edge.broken) {
41
+ return;
42
+ }
43
+ if (!outbound.has(edge.source)) {
44
+ outbound.set(edge.source, new Set());
45
+ }
46
+ if (!inbound.has(edge.target)) {
47
+ inbound.set(edge.target, new Set());
48
+ }
49
+ outbound.get(edge.source).add(edge.target);
50
+ inbound.get(edge.target).add(edge.source);
51
+ });
52
+ sets = { outbound, inbound };
53
+ LINK_SETS.set(index, sets);
54
+ return sets;
55
+ }
56
+
57
+ // Every edge of the index grouped by its source URI, in index edge order.
58
+ function edgesBySource(index) {
59
+ let grouped = EDGES_BY_SOURCE.get(index);
60
+ if (grouped) {
61
+ return grouped;
62
+ }
63
+ grouped = new Map();
64
+ (index.edges || []).forEach((edge) => {
65
+ const list = grouped.get(edge.source);
66
+ if (list) {
67
+ list.push(edge);
68
+ } else {
69
+ grouped.set(edge.source, [edge]);
70
+ }
71
+ });
72
+ EDGES_BY_SOURCE.set(index, grouped);
73
+ return grouped;
74
+ }
75
+
76
+ // What the last build or update of this index did: { mode: "build" |
77
+ // "incremental" | "full", changed, total, ratio, derived, documents, ... }.
78
+ // Null for an object that did not come out of buildIndex or updateIndex.
79
+ function indexUpdateStats(index) {
80
+ const record = indexRecord(index);
81
+ return record ? Object.assign({}, record.stats) : null;
82
+ }
83
+
84
+ // Every input that shaped an index, for a host deciding whether an index
85
+ // built elsewhere can stand in for a build of its own: the bundles (id,
86
+ // root, remote, include and exclude patterns), the relation types and flags,
87
+ // the date (null when it floats with the calendar) and the languages set on
88
+ // the index. Null for an object that did not come out of buildIndex.
89
+ function indexInputs(index) {
90
+ const record = indexRecord(index);
91
+ if (!record) {
92
+ return null;
93
+ }
94
+ return {
95
+ bundles: record.bundles.map(({ bundle }) => ({
96
+ id: bundle.id,
97
+ root: bundle.root,
98
+ remote: Boolean(bundle.remote),
99
+ include: (bundle.include || []).map(String),
100
+ exclude: (bundle.exclude || []).map(String),
101
+ })),
102
+ relationTypes: Array.from(record.config.edgeConfig.allowedRelationTypes),
103
+ allowCustomRelationTypes: record.config.edgeConfig.allowCustomRelationTypes,
104
+ strictLinks: record.config.strictLinks,
105
+ asOf: record.config.floatingDate ? null : record.config.asOf,
106
+ languages: index.languages === undefined ? null : index.languages,
107
+ };
108
+ }
109
+
110
+ function normalizeRelativePath(value) {
111
+ const relativePath = normalizeSlashes(String(value || "").trim());
112
+ const parts = relativePath.split("/");
113
+ if (
114
+ !relativePath ||
115
+ relativePath.startsWith("/") ||
116
+ /^[A-Za-z]:\//.test(relativePath) ||
117
+ parts.some((part) => !part || part === "." || part === "..")
118
+ ) {
119
+ throw new TypeError(`Index update paths must be safe bundle-relative paths: ${value}`);
120
+ }
121
+ if (!relativePath.toLowerCase().endsWith(".md")) {
122
+ throw new TypeError(`Index update paths must name Markdown documents: ${value}`);
123
+ }
124
+ return relativePath;
125
+ }
126
+
127
+ function isParsedDocument(value) {
128
+ return (
129
+ value &&
130
+ typeof value === "object" &&
131
+ typeof value.pathUri === "string" &&
132
+ typeof value.bundle === "string" &&
133
+ typeof value.path === "string" &&
134
+ typeof value.text === "string"
135
+ );
136
+ }
137
+
138
+ function targetFromUri(index, uri) {
139
+ const doc = index.byPathUri?.get(uri) || index.byUri?.get(uri) || null;
140
+ return doc ? { bundleId: doc.bundle, relativePath: doc.path } : null;
141
+ }
142
+
143
+ function targetFromEntry(entry, label) {
144
+ const bundleId = String(entry.bundleId || entry.bundle || "");
145
+ if (!bundleId) {
146
+ throw new TypeError(`Index update ${label} entries need a bundleId.`);
147
+ }
148
+ return { bundleId, relativePath: normalizeRelativePath(entry.relativePath || entry.path) };
149
+ }
150
+
151
+ // A deletion names a document by URI (canonical, alias or path form), by
152
+ // { bundleId, relativePath }, as a parsed document, or as a bare relative
153
+ // path when the index has exactly one local bundle. An unknown URI deletes
154
+ // nothing.
155
+ function deletionTarget(index, record, entry) {
156
+ if (typeof entry === "string") {
157
+ if (entry.startsWith("okf://")) {
158
+ return targetFromUri(index, entry);
159
+ }
160
+ const locals = record.bundles.filter((bundleRecord) => !bundleRecord.bundle.remote);
161
+ if (locals.length !== 1) {
162
+ throw new TypeError(
163
+ "A bare relative path deletion needs exactly one local bundle; pass { bundleId, relativePath }.",
164
+ );
165
+ }
166
+ return { bundleId: locals[0].bundle.id, relativePath: normalizeRelativePath(entry) };
167
+ }
168
+ if (entry && typeof entry === "object") {
169
+ if (isParsedDocument(entry)) {
170
+ return { bundleId: entry.bundle, relativePath: normalizeRelativePath(entry.path) };
171
+ }
172
+ if (typeof entry.uri === "string" && !entry.bundleId && !entry.relativePath) {
173
+ return targetFromUri(index, entry.uri);
174
+ }
175
+ return targetFromEntry(entry, "deletion");
176
+ }
177
+ throw new TypeError(
178
+ "Index update deletions must be URIs, paths, parsed documents, or { bundleId, relativePath }.",
179
+ );
180
+ }
181
+
182
+ // An upsert is a parsed document, { bundleId, relativePath, text }, or
183
+ // { bundleId, relativePath } to read the document back from the bundle root.
184
+ function upsertTarget(entry) {
185
+ if (isParsedDocument(entry)) {
186
+ return { bundleId: entry.bundle, relativePath: normalizeRelativePath(entry.path), doc: entry };
187
+ }
188
+ if (!entry || typeof entry !== "object") {
189
+ throw new TypeError(
190
+ "Index update upserts must be parsed documents or { bundleId, relativePath, text? }.",
191
+ );
192
+ }
193
+ const target = targetFromEntry(entry, "upsert");
194
+ target.text = entry.text === undefined ? undefined : String(entry.text);
195
+ return target;
196
+ }
197
+
198
+ function bundleRecordFor(record, bundleId) {
199
+ const found = record.bundles.find((bundleRecord) => bundleRecord.bundle.id === bundleId);
200
+ if (!found) {
201
+ throw new TypeError(`Unknown bundle in index update: ${bundleId}`);
202
+ }
203
+ if (found.bundle.remote) {
204
+ throw new TypeError(`Remote bundles cannot be updated in place: ${bundleId}`);
205
+ }
206
+ return found;
207
+ }
208
+
209
+ // Groups the requested changes per bundle. Deletions apply before upserts, so
210
+ // a path in both is an upsert; paths the bundle policy excludes are dropped,
211
+ // as a build would drop them; deletions of unknown paths count for nothing.
212
+ // `assetsChanged` says the asset files may differ from what the index saw,
213
+ // so the registry is rebuilt and the documents that depend on assets are
214
+ // re-derived.
215
+ function planChanges(index, record, changes) {
216
+ const input = changes || {};
217
+ const plan = { bundles: new Map(), changed: 0, assetsChanged: Boolean(input.assetsChanged) };
218
+ const touch = (bundleId) => {
219
+ let state = plan.bundles.get(bundleId);
220
+ if (!state) {
221
+ const bundleRecord = bundleRecordFor(record, bundleId);
222
+ state = {
223
+ bundleRecord,
224
+ known: new Set(bundleRecord.entries.map((entry) => entry.relativePath)),
225
+ deletions: new Set(),
226
+ upserts: new Map(),
227
+ };
228
+ plan.bundles.set(bundleId, state);
229
+ }
230
+ return state;
231
+ };
232
+ (Array.isArray(input.deletions) ? input.deletions : []).forEach((raw) => {
233
+ const target = deletionTarget(index, record, raw);
234
+ if (!target) {
235
+ return;
236
+ }
237
+ const state = touch(target.bundleId);
238
+ if (state.known.has(target.relativePath)) {
239
+ state.deletions.add(target.relativePath);
240
+ }
241
+ });
242
+ (Array.isArray(input.upserts) ? input.upserts : []).forEach((raw) => {
243
+ const target = upsertTarget(raw);
244
+ const state = touch(target.bundleId);
245
+ if (!bundleAllowsPath(state.bundleRecord.bundle, target.relativePath)) {
246
+ return;
247
+ }
248
+ state.deletions.delete(target.relativePath);
249
+ state.upserts.set(target.relativePath, target);
250
+ });
251
+ plan.bundles.forEach((state) => {
252
+ plan.changed += state.deletions.size + state.upserts.size;
253
+ });
254
+ return plan;
255
+ }
256
+
257
+ // A full build with the same inputs as the index, for the given day. What the
258
+ // index holds apart from the files (documents kept as text, documents removed
259
+ // while their files remain) and the requested changes are folded into the
260
+ // document overrides and deletions of this one build, so the rebuilt index
261
+ // describes the same state; the rebuilt record keeps the original options and
262
+ // carries the overlay state on its entries and in overlayDeletions, never as
263
+ // pins that would outlive the files.
264
+ function rebuildIndex(record, plan, stats, asOf) {
265
+ const overrides = new Map(record.config.documentOverrides);
266
+ const deletions = new Set(record.config.documentDeletions);
267
+ const overlayDeletions = new Set(record.overlayDeletions);
268
+ overlayDeletions.forEach((key) => {
269
+ deletions.add(key);
270
+ });
271
+ record.bundles.forEach((bundleRecord) => {
272
+ bundleRecord.entries.forEach((entry) => {
273
+ if (entry.source === "text") {
274
+ overrides.set(documentOverrideKey(bundleRecord.bundle.id, entry.relativePath), entry.text);
275
+ }
276
+ });
277
+ });
278
+ plan.bundles.forEach((state, bundleId) => {
279
+ state.deletions.forEach((relativePath) => {
280
+ const key = documentOverrideKey(bundleId, relativePath);
281
+ overrides.delete(key);
282
+ deletions.add(key);
283
+ if (fs.existsSync(documentAbsolutePath(state.bundleRecord.bundle, relativePath))) {
284
+ overlayDeletions.add(key);
285
+ } else {
286
+ overlayDeletions.delete(key);
287
+ }
288
+ });
289
+ state.upserts.forEach((upsert, relativePath) => {
290
+ const key = documentOverrideKey(bundleId, relativePath);
291
+ deletions.delete(key);
292
+ overlayDeletions.delete(key);
293
+ if (upsert.doc) {
294
+ overrides.set(key, upsert.doc.text);
295
+ } else if (upsert.text !== undefined) {
296
+ overrides.set(key, upsert.text);
297
+ } else {
298
+ overrides.delete(key);
299
+ }
300
+ });
301
+ });
302
+ const rebuilt = buildIndex(
303
+ record.bundleArgs,
304
+ Object.assign({}, record.options, {
305
+ documentOverrides: overrides,
306
+ documentDeletions: deletions,
307
+ asOf,
308
+ }),
309
+ );
310
+ if (record.project) {
311
+ attachProject(rebuilt, record.project);
312
+ }
313
+ const next = indexRecord(rebuilt);
314
+ next.options = record.options;
315
+ next.config = Object.assign({}, record.config, { asOf });
316
+ next.overlayDeletions = overlayDeletions;
317
+ next.stats = Object.assign(stats, { mode: "full" });
318
+ return rebuilt;
319
+ }
320
+
321
+ function insertSorted(entries, entry) {
322
+ let position = 0;
323
+ while (
324
+ position < entries.length &&
325
+ (entries[position] === null || !(entries[position].absolutePath > entry.absolutePath))
326
+ ) {
327
+ position += 1;
328
+ }
329
+ entries.splice(position, 0, entry);
330
+ }
331
+
332
+ // Applies one bundle's deletions and upserts to a copy of its entries. A
333
+ // replaced document keeps its position; a new one is inserted where a walk
334
+ // would have listed it, after every replacement, because an insertion shifts
335
+ // the positions of the entries behind it. Returns the documents that left
336
+ // and entered.
337
+ function applyBundleChanges(bundleRecord, state, config, changes) {
338
+ const bundle = bundleRecord.bundle;
339
+ if (!fs.existsSync(bundle.root)) {
340
+ bundleRecord.entries.forEach((entry) => {
341
+ if (entry.doc) {
342
+ changes.removed.push({ relativePath: entry.relativePath, entry });
343
+ }
344
+ changes.restored.add(documentOverrideKey(bundle.id, entry.relativePath));
345
+ });
346
+ bundleRecord.entries = [];
347
+ bundleRecord.rootMissing = missingRootError(bundle);
348
+ return;
349
+ }
350
+ bundleRecord.rootMissing = null;
351
+ const entries = bundleRecord.entries;
352
+ const positions = new Map(entries.map((entry, position) => [entry.relativePath, position]));
353
+ state.deletions.forEach((relativePath) => {
354
+ const position = positions.get(relativePath);
355
+ const old = entries[position];
356
+ if (old.doc) {
357
+ changes.removed.push({ relativePath, entry: old });
358
+ }
359
+ entries[position] = null;
360
+ // A document removed while its file remains is an overlay deletion that
361
+ // a later full rebuild has to repeat; one whose file is gone is not.
362
+ const key = documentOverrideKey(bundle.id, relativePath);
363
+ if (fs.existsSync(documentAbsolutePath(bundle, relativePath))) {
364
+ changes.overlayDeleted.add(key);
365
+ } else {
366
+ changes.restored.add(key);
367
+ }
368
+ });
369
+ const inserted = [];
370
+ state.upserts.forEach((upsert, relativePath) => {
371
+ const absolutePath = documentAbsolutePath(bundle, relativePath);
372
+ let entry;
373
+ if (upsert.doc) {
374
+ entry = {
375
+ relativePath,
376
+ absolutePath,
377
+ doc: upsert.doc,
378
+ parseError: null,
379
+ stat: null,
380
+ source: "text",
381
+ text: upsert.doc.text,
382
+ derived: null,
383
+ };
384
+ } else {
385
+ entry = parseEntry(bundle, relativePath, absolutePath, upsert.text, config);
386
+ changes.owned.add(entry);
387
+ }
388
+ const position = positions.get(relativePath);
389
+ if (position !== undefined) {
390
+ const old = entries[position];
391
+ if (old?.doc) {
392
+ changes.removed.push({ relativePath, entry: old });
393
+ }
394
+ entries[position] = entry;
395
+ } else {
396
+ inserted.push(entry);
397
+ }
398
+ changes.added.push({ relativePath, entry });
399
+ changes.restored.add(documentOverrideKey(bundle.id, relativePath));
400
+ });
401
+ inserted.forEach((entry) => {
402
+ insertSorted(entries, entry);
403
+ });
404
+ bundleRecord.entries = entries.filter(Boolean);
405
+ }
406
+
407
+ function assetReferencesChanged(changes) {
408
+ const before = new Map();
409
+ changes.removed.forEach(({ entry }) => {
410
+ before.set(`${entry.doc.bundle}\u0000${entry.relativePath}`, entry.derived.assetReferences);
411
+ });
412
+ const after = new Map();
413
+ changes.added.forEach(({ entry }) => {
414
+ if (entry.doc) {
415
+ after.set(
416
+ `${entry.doc.bundle}\u0000${entry.relativePath}`,
417
+ collectAssetReferences(entry.doc),
418
+ );
419
+ }
420
+ });
421
+ const keys = new Set([...before.keys(), ...after.keys()]);
422
+ for (const key of keys) {
423
+ if (!isDeepStrictEqual(before.get(key) || [], after.get(key) || [])) {
424
+ return true;
425
+ }
426
+ }
427
+ return false;
428
+ }
429
+
430
+ function applyIncrementally(record, plan, stats) {
431
+ const changes = {
432
+ owned: new Set(),
433
+ removed: [],
434
+ added: [],
435
+ overlayDeleted: new Set(),
436
+ restored: new Set(),
437
+ };
438
+ const bundleRecords = record.bundles.map((bundleRecord) => {
439
+ const copy = Object.assign({}, bundleRecord, { entries: bundleRecord.entries.slice() });
440
+ const state = plan.bundles.get(bundleRecord.bundle.id);
441
+ if (state) {
442
+ applyBundleChanges(copy, state, record.config, changes);
443
+ }
444
+ return copy;
445
+ });
446
+ const dirty = new Set();
447
+ const changedUris = new Set();
448
+ changes.removed.forEach(({ entry }) => {
449
+ documentKeys(entry.doc).forEach((key) => {
450
+ dirty.add(key);
451
+ });
452
+ changedUris.add(entry.doc.uri);
453
+ });
454
+ changes.added.forEach(({ entry }) => {
455
+ if (entry.doc) {
456
+ documentKeys(entry.doc).forEach((key) => {
457
+ dirty.add(key);
458
+ });
459
+ changedUris.add(entry.doc.uri);
460
+ }
461
+ });
462
+ const registryChanged = plan.assetsChanged || assetReferencesChanged(changes);
463
+ if (registryChanged) {
464
+ assetKeys(record.assets.byAssetKey, record.assets.byAssetUri).forEach((key) => {
465
+ dirty.add(key);
466
+ });
467
+ }
468
+ const changedEntries = new Set(changes.added.map(({ entry }) => entry));
469
+ const previousDuplicates = record.registration.duplicateErrors.length > 0;
470
+ const overlayDeletions = new Set(record.overlayDeletions);
471
+ changes.restored.forEach((key) => {
472
+ overlayDeletions.delete(key);
473
+ });
474
+ changes.overlayDeleted.forEach((key) => {
475
+ overlayDeletions.add(key);
476
+ });
477
+ const next = {
478
+ bundleArgs: record.bundleArgs,
479
+ options: record.options,
480
+ config: record.config,
481
+ project: record.project,
482
+ duplicateBundleErrors: record.duplicateBundleErrors,
483
+ bundles: bundleRecords,
484
+ registration: null,
485
+ assets: null,
486
+ overlayDeletions,
487
+ stats,
488
+ };
489
+ const selectAffected = ({ registration, assetState, located }) => {
490
+ if (registryChanged) {
491
+ assetKeys(assetState.byAssetKey, assetState.byAssetUri).forEach((key) => {
492
+ dirty.add(key);
493
+ });
494
+ }
495
+ const pooled = previousDuplicates || registration.duplicateErrors.length > 0;
496
+ const affected = [];
497
+ const chosen = new Set();
498
+ // The entry object is shared with the input index; re-derive a copy.
499
+ const rederive = (item) => {
500
+ const copy = Object.assign({}, item.entry);
501
+ item.bundleRecord.entries[item.position] = copy;
502
+ item.entry = copy;
503
+ chosen.add(item);
504
+ affected.push(item);
505
+ };
506
+ located.forEach((item) => {
507
+ if (changedEntries.has(item.entry)) {
508
+ chosen.add(item);
509
+ affected.push(item);
510
+ return;
511
+ }
512
+ const derived = item.entry.derived;
513
+ let hit = pooled && changedUris.has(item.entry.doc.uri);
514
+ if (!hit) {
515
+ for (const key of derived.dependsOn) {
516
+ if (dirty.has(key)) {
517
+ hit = true;
518
+ break;
519
+ }
520
+ }
521
+ }
522
+ if (hit) {
523
+ rederive(item);
524
+ }
525
+ });
526
+ if (pooled) {
527
+ // Readiness pools the edges of every document sharing a canonical URI,
528
+ // so when one owner is re-derived the others are too.
529
+ const affectedUris = new Set(affected.map((item) => item.entry.doc.uri));
530
+ located.forEach((item) => {
531
+ if (!chosen.has(item) && affectedUris.has(item.entry.doc.uri)) {
532
+ rederive(item);
533
+ }
534
+ });
535
+ }
536
+ stats.dependents = affected.length - changes.added.filter(({ entry }) => entry.doc).length;
537
+ return affected;
538
+ };
539
+ stats.mode = "incremental";
540
+ stats.registry = registryChanged ? "rebuilt" : "carried";
541
+ return completeIndex(next, {
542
+ owned: changes.owned,
543
+ assets: registryChanged ? null : record.assets,
544
+ selectAffected,
545
+ pooledReadiness: previousDuplicates,
546
+ });
547
+ }
548
+
549
+ // Settings a host puts on the index object itself rather than through the
550
+ // project (root mode sets the stemming `languages` there) travel to the index
551
+ // an update produces; a value assembly already took from the project is kept.
552
+ function carryIndexSettings(from, to) {
553
+ if (to !== from && to.languages === undefined && from.languages !== undefined) {
554
+ to.languages = from.languages;
555
+ }
556
+ return to;
557
+ }
558
+
559
+ // Returns a new index with the given documents added, replaced or removed.
560
+ // The input index is never mutated; unchanged documents, edges and derived
561
+ // lists are shared with it. Only the changed documents and the documents
562
+ // whose edges resolved through a key the change dirtied are re-derived;
563
+ // lookup maps are re-registered in index order (a linear pass); the asset
564
+ // registry is carried unless a changed document's asset references changed
565
+ // or `changes.assetsChanged` says the asset files did; validation is
566
+ // recomputed in full. Above `maxChangeRatio` (changed documents over
567
+ // documents in the index, default 0.2) the index is rebuilt instead, and so
568
+ // is an index built without an asOf when `options.asOf` (default: now) falls
569
+ // on another UTC day than the one its documents were stamped with: one full
570
+ // build per day per process, which in the live service runs under the
571
+ // writer lock, so the first batch of a day pays a build's latency.
572
+ function updateIndex(index, changes, options) {
573
+ const record = indexRecord(index);
574
+ if (!record) {
575
+ throw new TypeError("updateIndex needs an index produced by buildIndex.");
576
+ }
577
+ const config = options || {};
578
+ const maxChangeRatio =
579
+ config.maxChangeRatio === undefined ? DEFAULT_MAX_CHANGE_RATIO : Number(config.maxChangeRatio);
580
+ if (!(maxChangeRatio >= 0)) {
581
+ throw new TypeError("maxChangeRatio must be a non-negative number.");
582
+ }
583
+ const plan = planChanges(index, record, changes);
584
+ const asOf = record.config.floatingDate ? dayKey(config.asOf) : record.config.asOf;
585
+ const dateChanged = asOf !== record.config.asOf;
586
+ if (plan.changed === 0 && !plan.assetsChanged && !dateChanged) {
587
+ return index;
588
+ }
589
+ const total = index.documents.length;
590
+ const ratio = plan.changed / Math.max(1, total);
591
+ const stats = {
592
+ mode: null,
593
+ changed: plan.changed,
594
+ total,
595
+ ratio,
596
+ maxChangeRatio,
597
+ asOf,
598
+ assets: plan.assetsChanged ? "changed" : "unchanged",
599
+ };
600
+ if (dateChanged || ratio > maxChangeRatio) {
601
+ stats.reason = dateChanged ? "date" : "ratio";
602
+ return carryIndexSettings(index, rebuildIndex(record, plan, stats, asOf));
603
+ }
604
+ return carryIndexSettings(index, applyIncrementally(record, plan, stats));
605
+ }
606
+
607
+ // Whether the file on disk is what the entry already describes. A disk-backed
608
+ // entry is compared by size and mtime (nanoseconds). An entry held as text is
609
+ // compared by content; when the file carries the same text the entry becomes
610
+ // disk-backed in place (a cache fill that changes nothing the entry
611
+ // describes), so later refreshes need only a stat. A file that vanished
612
+ // mid-walk counts as changed and the update records whatever it finds.
613
+ //
614
+ // Limit: a rewrite that keeps a file's size and mtime is invisible here, for
615
+ // documents and (in assetsChangedOnDisk) for assets alike. The write path
616
+ // covers the documents a batch targets by comparing content hashes and
617
+ // re-reading on a mismatch (live.js, updateCandidate); the dependencies of a
618
+ // batch are not re-read. Before the incremental index, the write path built
619
+ // from disk every time, so every file was read; now a file whose stat is
620
+ // unchanged is trusted.
621
+ function fileMatchesEntry(entry, filePath) {
622
+ const stat = fileStat(filePath);
623
+ if (!stat) {
624
+ return false;
625
+ }
626
+ if (entry.source === "disk") {
627
+ return sameStat(entry.stat, stat);
628
+ }
629
+ if (entry.source !== "text") {
630
+ return false;
631
+ }
632
+ let text;
633
+ try {
634
+ text = new TextDecoder("utf-8", { fatal: true }).decode(fs.readFileSync(filePath));
635
+ } catch {
636
+ return false;
637
+ }
638
+ if (text !== entry.text) {
639
+ return false;
640
+ }
641
+ entry.stat = stat;
642
+ entry.source = "disk";
643
+ return true;
644
+ }
645
+
646
+ // Whether any asset the documents reference was modified, removed or added
647
+ // since the index recorded it, judged by size and mtime of the watched paths.
648
+ function assetsChangedOnDisk(record) {
649
+ for (const [absolutePath, recorded] of record.assets.watch) {
650
+ if (!sameStat(recorded, fileStat(absolutePath))) {
651
+ return true;
652
+ }
653
+ }
654
+ return false;
655
+ }
656
+
657
+ // Brings an index up to date with its local bundle roots by comparing each
658
+ // Markdown file's and each referenced asset's size and mtime with what the
659
+ // index recorded, then applying the differences through updateIndex (which
660
+ // also notices a changed day; `options` are its options). Returns the same
661
+ // index object when nothing changed. Paths pinned by document overrides or
662
+ // deletions at build time are left as built.
663
+ function refreshIndex(index, options) {
664
+ const record = indexRecord(index);
665
+ if (!record) {
666
+ throw new TypeError("refreshIndex needs an index produced by buildIndex.");
667
+ }
668
+ const upserts = [];
669
+ const deletions = [];
670
+ record.bundles.forEach((bundleRecord) => {
671
+ const bundle = bundleRecord.bundle;
672
+ if (bundle.remote) {
673
+ return;
674
+ }
675
+ const pinned = (key) =>
676
+ record.config.documentOverrides.has(key) || record.config.documentDeletions.has(key);
677
+ const seen = new Set();
678
+ if (fs.existsSync(bundle.root)) {
679
+ const byPath = new Map(bundleRecord.entries.map((entry) => [entry.relativePath, entry]));
680
+ walkMarkdown(bundle.root).forEach((filePath) => {
681
+ const relativePath = safeRelativePath(bundle.root, filePath) || normalizeSlashes(filePath);
682
+ if (!bundleAllowsPath(bundle, relativePath)) {
683
+ return;
684
+ }
685
+ const key = documentOverrideKey(bundle.id, relativePath);
686
+ if (pinned(key)) {
687
+ return;
688
+ }
689
+ seen.add(relativePath);
690
+ const entry = byPath.get(relativePath);
691
+ if (!entry || !fileMatchesEntry(entry, filePath)) {
692
+ upserts.push({ bundleId: bundle.id, relativePath });
693
+ }
694
+ });
695
+ }
696
+ bundleRecord.entries.forEach((entry) => {
697
+ if (
698
+ !seen.has(entry.relativePath) &&
699
+ !pinned(documentOverrideKey(bundle.id, entry.relativePath))
700
+ ) {
701
+ deletions.push({ bundleId: bundle.id, relativePath: entry.relativePath });
702
+ }
703
+ });
704
+ });
705
+ return updateIndex(
706
+ index,
707
+ { upserts, deletions, assetsChanged: assetsChangedOnDisk(record) },
708
+ options,
709
+ );
710
+ }
711
+
712
+ module.exports = {
713
+ DEFAULT_MAX_CHANGE_RATIO,
714
+ edgesBySource,
715
+ indexInputs,
716
+ indexUpdateStats,
717
+ linkSets,
718
+ refreshIndex,
719
+ updateIndex,
720
+ };