@mfdaves/okf-mcp 0.9.1 → 0.13.0-rc.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +100 -0
- package/README.md +452 -26
- package/bin/okf-mcp.js +6 -1
- package/okf/okf-mcp/distribution/reference-bundle.md +10 -3
- package/okf/okf-mcp/index.md +2 -1
- package/okf/okf-mcp/interfaces/http-authoring-api.md +11 -2
- package/okf/okf-mcp/interfaces/mcp-http.md +64 -0
- package/okf/okf-mcp/interfaces/mcp-tools.md +25 -6
- package/okf/okf-mcp/overview/okf-mcp.md +16 -6
- package/okf/okf-mcp/policies/authoring-safety.md +28 -5
- package/okf/okf-mcp/runtime/indexer.md +14 -1
- package/okf/okf-mcp/runtime/mcp-server.md +16 -8
- package/okf/okf-mcp/specs/producer-api.md +16 -2
- package/package.json +17 -5
- package/server.json +38 -2
- package/src/assets.js +165 -73
- package/src/authoring/candidates.js +434 -0
- package/src/authoring/git-preflight.js +225 -0
- package/src/authoring/git-publish.js +783 -0
- package/src/authoring/live.js +757 -0
- package/src/authoring/paths.js +112 -0
- package/src/authoring/policy-hook.js +221 -0
- package/src/authoring/proposal-policy.js +159 -0
- package/src/authoring/receipts.js +373 -0
- package/src/authoring/render.js +38 -0
- package/src/authoring/shared.js +75 -0
- package/src/authoring/staging.js +216 -0
- package/src/authoring.js +386 -287
- package/src/cache/files.js +147 -0
- package/src/cache/lock.js +201 -0
- package/src/cache/paths.js +197 -0
- package/src/cache/store.js +703 -0
- package/src/catalog-text.js +19 -0
- package/src/cli/commands.js +448 -0
- package/src/cli/http-options.js +193 -0
- package/src/cli/options.js +554 -0
- package/src/cli/usage.js +96 -0
- package/src/cli.js +18 -932
- package/src/computation.js +99 -61
- package/src/context/assemble.js +567 -0
- package/src/context/cli-options.js +88 -0
- package/src/context/score.js +234 -0
- package/src/context/sections.js +274 -0
- package/src/embed/interface.js +160 -0
- package/src/embed/openai.js +715 -0
- package/src/embed/prepare.js +182 -0
- package/src/embed/units.js +96 -0
- package/src/error-details.js +53 -0
- package/src/git-source.js +112 -52
- package/src/graph.js +95 -37
- package/src/http-server.js +36 -17
- package/src/index.js +15 -0
- package/src/indexer/assemble.js +566 -0
- package/src/indexer/build.js +290 -0
- package/src/indexer/cache.js +81 -0
- package/src/indexer/edges.js +430 -0
- package/src/indexer/index.js +44 -0
- package/src/indexer/paths.js +178 -0
- package/src/indexer/resolve.js +282 -0
- package/src/indexer/update.js +720 -0
- package/src/indexer.js +3 -1068
- package/src/live-authoring.js +4 -2329
- package/src/markdown.js +21 -16
- package/src/mcp/auth.js +409 -0
- package/src/mcp/http.js +317 -0
- package/src/mcp/principal.js +59 -0
- package/src/mcp/registry.js +356 -0
- package/src/mcp/resources.js +45 -0
- package/src/mcp/server.js +567 -0
- package/src/mcp/tools/catalog.js +247 -0
- package/src/mcp/tools/computation.js +83 -0
- package/src/mcp/tools/context.js +133 -0
- package/src/mcp/tools/graph.js +167 -0
- package/src/mcp/tools/live-authoring.js +316 -0
- package/src/mcp/tools/migration.js +86 -0
- package/src/mcp/tools/producers.js +79 -0
- package/src/mcp/tools/proposals.js +225 -0
- package/src/mcp/tools/provenance.js +87 -0
- package/src/mcp/tools/remote.js +79 -0
- package/src/mcp/tools/search.js +242 -0
- package/src/mcp-server.js +3 -1611
- package/src/migration.js +431 -253
- package/src/parser.js +245 -130
- package/src/plugins.js +71 -48
- package/src/policy/write-policy.js +713 -0
- package/src/producer-publisher.js +202 -78
- package/src/producers.js +311 -103
- package/src/project.js +346 -93
- package/src/remote.js +96 -61
- package/src/search/cli-options.js +107 -0
- package/src/search/confidence.js +61 -0
- package/src/search/dense.js +304 -0
- package/src/search/embedding-cli.js +57 -0
- package/src/search/embedding-config.js +217 -0
- package/src/search/fusion.js +249 -0
- package/src/search/hybrid.js +526 -0
- package/src/search/languages.js +200 -0
- package/src/search/lexical.js +695 -0
- package/src/search/query.js +620 -0
- package/src/search/stem/among.js +43 -0
- package/src/search/stem/base-stemmer.js +311 -0
- package/src/search/stem/en-regions.js +456 -0
- package/src/search/stem/en-suffixes.js +664 -0
- package/src/search/stem/en-tables.js +176 -0
- package/src/search/stem/en.js +238 -0
- package/src/search/stem/it-regions.js +459 -0
- package/src/search/stem/it-suffixes.js +471 -0
- package/src/search/stem/it-tables.js +257 -0
- package/src/search/stem/it.js +155 -0
- package/src/search-index.js +2 -81
- package/src/search.js +2 -252
- package/src/store.js +134 -81
- package/src/tool-packs.js +346 -0
- package/src/v02.js +301 -105
- package/src/validation.js +24 -20
|
@@ -0,0 +1,182 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
|
|
3
|
+
// Incremental preparation of a bundle's vectors: compute the retrieval units,
|
|
4
|
+
// embed only the hashes the cache does not hold, store them as each chunk
|
|
5
|
+
// arrives so an interrupted run keeps its progress, and drop rows whose text
|
|
6
|
+
// no longer exists. The receipt says what left the machine (O-2) and what it
|
|
7
|
+
// cost, never what the text was. A unit the provider rejects even after the
|
|
8
|
+
// embedder's own recovery (skipRejected, src/embed/interface.js) is listed
|
|
9
|
+
// under `skipped` and the run continues: one bad document never costs the
|
|
10
|
+
// rest of the catalog its vectors. It is retried by the next preparation.
|
|
11
|
+
|
|
12
|
+
const { performance } = require("node:perf_hooks");
|
|
13
|
+
|
|
14
|
+
const { EmbedderError, describeEmbedder, validateEmbedder } = require("./interface");
|
|
15
|
+
const { UNIT_HASH_PATTERN, conceptUnits } = require("./units");
|
|
16
|
+
|
|
17
|
+
const USAGE_FIELDS = ["requests", "retries", "promptTokens", "totalTokens"];
|
|
18
|
+
|
|
19
|
+
function normalizeUnits(units) {
|
|
20
|
+
if (!Array.isArray(units)) {
|
|
21
|
+
throw new EmbedderError("The units function must return an array.", "invalid_units");
|
|
22
|
+
}
|
|
23
|
+
return units.map((unit, position) => {
|
|
24
|
+
if (
|
|
25
|
+
!unit ||
|
|
26
|
+
typeof unit.id !== "string" ||
|
|
27
|
+
!unit.id ||
|
|
28
|
+
typeof unit.text !== "string" ||
|
|
29
|
+
typeof unit.hash !== "string" ||
|
|
30
|
+
!UNIT_HASH_PATTERN.test(unit.hash)
|
|
31
|
+
) {
|
|
32
|
+
throw new EmbedderError(
|
|
33
|
+
`Unit ${position} must carry an id, a text and a sha256 hash.`,
|
|
34
|
+
"invalid_units",
|
|
35
|
+
{ position },
|
|
36
|
+
);
|
|
37
|
+
}
|
|
38
|
+
return { id: unit.id, text: unit.text, hash: unit.hash };
|
|
39
|
+
});
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
function usageSnapshot(embedder) {
|
|
43
|
+
if (typeof embedder.usage !== "function") return null;
|
|
44
|
+
const usage = embedder.usage();
|
|
45
|
+
if (!usage || typeof usage !== "object") return null;
|
|
46
|
+
return Object.fromEntries(
|
|
47
|
+
USAGE_FIELDS.map((field) => [field, Number.isFinite(usage[field]) ? usage[field] : 0]),
|
|
48
|
+
);
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
function usageDelta(before, after) {
|
|
52
|
+
if (!before || !after) return null;
|
|
53
|
+
return Object.fromEntries(
|
|
54
|
+
USAGE_FIELDS.map((field) => [field, Math.max(0, after[field] - before[field])]),
|
|
55
|
+
);
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
function assertCacheMatches(cache, embedder) {
|
|
59
|
+
if (
|
|
60
|
+
!cache ||
|
|
61
|
+
typeof cache.has !== "function" ||
|
|
62
|
+
typeof cache.putMany !== "function" ||
|
|
63
|
+
typeof cache.prune !== "function"
|
|
64
|
+
) {
|
|
65
|
+
throw new EmbedderError("prepareEmbeddings requires an open vector cache.", "invalid_cache");
|
|
66
|
+
}
|
|
67
|
+
if (cache.embedderId !== embedder.id || cache.dimension !== embedder.dimension) {
|
|
68
|
+
throw new EmbedderError(
|
|
69
|
+
"The vector cache was opened for a different embedder or dimension.",
|
|
70
|
+
"cache_mismatch",
|
|
71
|
+
{
|
|
72
|
+
cache: { embedderId: cache.embedderId, dimension: cache.dimension },
|
|
73
|
+
embedder: { id: embedder.id, dimension: embedder.dimension },
|
|
74
|
+
},
|
|
75
|
+
);
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
// options.units replaces the unit function (sections instead of concepts);
|
|
80
|
+
// options.onProgress receives { embedded, total, units } after each chunk.
|
|
81
|
+
async function prepareEmbeddings(index, embedder, cache, options) {
|
|
82
|
+
const config = options || {};
|
|
83
|
+
validateEmbedder(embedder);
|
|
84
|
+
assertCacheMatches(cache, embedder);
|
|
85
|
+
const unitsOf = typeof config.units === "function" ? config.units : conceptUnits;
|
|
86
|
+
const started = performance.now();
|
|
87
|
+
const units = normalizeUnits(unitsOf(index));
|
|
88
|
+
|
|
89
|
+
const keep = new Set();
|
|
90
|
+
const pending = new Map();
|
|
91
|
+
let reused = 0;
|
|
92
|
+
for (const unit of units) {
|
|
93
|
+
keep.add(unit.hash);
|
|
94
|
+
if (cache.has(unit.hash)) {
|
|
95
|
+
reused += 1;
|
|
96
|
+
} else if (!pending.has(unit.hash)) {
|
|
97
|
+
pending.set(unit.hash, unit);
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
// Each embed() call carries at most maxBatch units (the interface's limit
|
|
102
|
+
// per call); up to `concurrency` calls run at once, and every batch is
|
|
103
|
+
// persisted as it completes so an interrupted run keeps its progress.
|
|
104
|
+
const parallel =
|
|
105
|
+
Number.isSafeInteger(embedder.concurrency) && embedder.concurrency > 0
|
|
106
|
+
? embedder.concurrency
|
|
107
|
+
: 1;
|
|
108
|
+
const queue = Array.from(pending.values());
|
|
109
|
+
const batches = [];
|
|
110
|
+
for (let offset = 0; offset < queue.length; offset += embedder.maxBatch) {
|
|
111
|
+
batches.push(queue.slice(offset, offset + embedder.maxBatch));
|
|
112
|
+
}
|
|
113
|
+
const usageBefore = usageSnapshot(embedder);
|
|
114
|
+
let embedded = 0;
|
|
115
|
+
let calls = 0;
|
|
116
|
+
const skipped = [];
|
|
117
|
+
let next = 0;
|
|
118
|
+
let failure = null;
|
|
119
|
+
const worker = async () => {
|
|
120
|
+
while (next < batches.length && failure === null) {
|
|
121
|
+
const batch = batches[next];
|
|
122
|
+
next += 1;
|
|
123
|
+
try {
|
|
124
|
+
calls += 1;
|
|
125
|
+
// One request bound for the whole run: this pool is the parallelism,
|
|
126
|
+
// so each call runs its own requests one at a time.
|
|
127
|
+
const vectors = await embedder.embed(
|
|
128
|
+
batch.map((unit) => unit.text),
|
|
129
|
+
{ concurrency: 1, skipRejected: true },
|
|
130
|
+
);
|
|
131
|
+
if (!Array.isArray(vectors) || vectors.length !== batch.length) {
|
|
132
|
+
throw new EmbedderError(
|
|
133
|
+
"The embedder returned a different number of vectors than inputs.",
|
|
134
|
+
"embedder_bad_output",
|
|
135
|
+
);
|
|
136
|
+
}
|
|
137
|
+
const rows = [];
|
|
138
|
+
batch.forEach((unit, position) => {
|
|
139
|
+
if (vectors[position] === null) {
|
|
140
|
+
skipped.push({ id: unit.id, reason: "input_rejected" });
|
|
141
|
+
} else {
|
|
142
|
+
rows.push({ id: unit.id, hash: unit.hash, vector: vectors[position] });
|
|
143
|
+
}
|
|
144
|
+
});
|
|
145
|
+
if (rows.length) await cache.putMany(rows);
|
|
146
|
+
embedded += rows.length;
|
|
147
|
+
if (typeof config.onProgress === "function") {
|
|
148
|
+
config.onProgress({ embedded, total: queue.length, units: units.length });
|
|
149
|
+
}
|
|
150
|
+
} catch (error) {
|
|
151
|
+
if (failure === null) failure = error;
|
|
152
|
+
}
|
|
153
|
+
}
|
|
154
|
+
};
|
|
155
|
+
await Promise.all(Array.from({ length: Math.min(parallel, batches.length) }, worker));
|
|
156
|
+
if (failure !== null) throw failure;
|
|
157
|
+
const pruned = await cache.prune(keep);
|
|
158
|
+
const usage = usageDelta(usageBefore, usageSnapshot(embedder));
|
|
159
|
+
|
|
160
|
+
return {
|
|
161
|
+
units: units.length,
|
|
162
|
+
embedded,
|
|
163
|
+
reused,
|
|
164
|
+
deduplicated: units.length - reused - embedded - skipped.length,
|
|
165
|
+
skipped: skipped.sort((left, right) => (left.id < right.id ? -1 : left.id > right.id ? 1 : 0)),
|
|
166
|
+
pruned,
|
|
167
|
+
requests: usage ? usage.requests : calls,
|
|
168
|
+
usage,
|
|
169
|
+
embedder: describeEmbedder(embedder),
|
|
170
|
+
cache: {
|
|
171
|
+
dir: cache.dir,
|
|
172
|
+
rows: cache.size,
|
|
173
|
+
rebuilt: cache.opened ? cache.opened.rebuilt === true : false,
|
|
174
|
+
reason: cache.opened ? cache.opened.reason : null,
|
|
175
|
+
},
|
|
176
|
+
durationMs: Math.round((performance.now() - started) * 1000) / 1000,
|
|
177
|
+
};
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
module.exports = {
|
|
181
|
+
prepareEmbeddings,
|
|
182
|
+
};
|
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
|
|
3
|
+
const crypto = require("node:crypto");
|
|
4
|
+
|
|
5
|
+
// A retrieval unit is the piece of text one vector stands for:
|
|
6
|
+
// { id, text, hash }. `id` is the locator a search result cites (a concept URI
|
|
7
|
+
// today, `uri#heading-slug` once WP3a lands sections), `text` is what gets
|
|
8
|
+
// embedded, and `hash` is the SHA-256 of `text` that keys the vector cache
|
|
9
|
+
// (R-9). prepareEmbeddings() takes the unit function as an option so the
|
|
10
|
+
// section-level units can replace these without touching the cache or the
|
|
11
|
+
// client.
|
|
12
|
+
|
|
13
|
+
const UNIT_HASH_PATTERN = /^[a-f0-9]{64}$/;
|
|
14
|
+
|
|
15
|
+
function hashUnitText(text) {
|
|
16
|
+
return crypto.createHash("sha256").update(String(text), "utf8").digest("hex");
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
// Title, description and body joined by newlines. Reserved documents
|
|
20
|
+
// (index.md, log.md) are navigation, not knowledge, and stay out.
|
|
21
|
+
function conceptUnitText(doc) {
|
|
22
|
+
return [doc.title, doc.description, doc.body]
|
|
23
|
+
.map((part) => (typeof part === "string" ? part : ""))
|
|
24
|
+
.join("\n");
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
function conceptUnits(index) {
|
|
28
|
+
const documents = index?.documents || [];
|
|
29
|
+
return documents
|
|
30
|
+
.filter((doc) => doc?.kind === "concept" && !doc.reserved)
|
|
31
|
+
.map((doc) => {
|
|
32
|
+
const text = conceptUnitText(doc);
|
|
33
|
+
return { id: doc.uri, text, hash: hashUnitText(text) };
|
|
34
|
+
});
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
// Contextual units (WP3c-2): a bounded one-line prefix naming where a text sits in the catalog,
|
|
38
|
+
// "part · part · part", put before the text that is embedded. A bare section ("Steps", "Limits")
|
|
39
|
+
// carries no system context of its own; the prefix gives it its concept's title, type and path.
|
|
40
|
+
// Empty parts are dropped and whitespace runs collapse to one space. Longer than
|
|
41
|
+
// CONTEXT_PREFIX_MAX characters, the first part is shortened with "…" so the later ones (type
|
|
42
|
+
// and path) stay whole; if they alone do not fit, the whole prefix is cut. The contextual text
|
|
43
|
+
// has its own hash, so switching unit functions re-embeds once and plain units keep theirs.
|
|
44
|
+
const CONTEXT_PREFIX_MAX = 160;
|
|
45
|
+
const CONTEXT_SEPARATOR = " · ";
|
|
46
|
+
// Below this many characters a shortened first part says nothing; cut the whole prefix instead.
|
|
47
|
+
const MIN_FIRST_PART = 12;
|
|
48
|
+
|
|
49
|
+
// The first `length` UTF-16 units of `text`, never ending on half a surrogate pair.
|
|
50
|
+
function cut(text, length) {
|
|
51
|
+
const end = Math.max(0, length);
|
|
52
|
+
const code = text.charCodeAt(end - 1);
|
|
53
|
+
return text.slice(0, code >= 0xd800 && code <= 0xdbff ? end - 1 : end);
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
function contextPrefix(parts, max) {
|
|
57
|
+
const limit = Number.isSafeInteger(max) && max > 1 ? max : CONTEXT_PREFIX_MAX;
|
|
58
|
+
const clean = (parts || [])
|
|
59
|
+
.map((part) =>
|
|
60
|
+
String(part ?? "")
|
|
61
|
+
.replace(/\s+/g, " ")
|
|
62
|
+
.trim(),
|
|
63
|
+
)
|
|
64
|
+
.filter(Boolean);
|
|
65
|
+
const whole = clean.join(CONTEXT_SEPARATOR);
|
|
66
|
+
if (whole.length <= limit) return whole;
|
|
67
|
+
if (clean.length > 1) {
|
|
68
|
+
const rest = `${CONTEXT_SEPARATOR}${clean.slice(1).join(CONTEXT_SEPARATOR)}`;
|
|
69
|
+
const room = limit - rest.length - 1;
|
|
70
|
+
if (room >= MIN_FIRST_PART) return `${cut(clean[0], room).trimEnd()}…${rest}`;
|
|
71
|
+
}
|
|
72
|
+
return `${cut(whole, limit - 1).trimEnd()}…`;
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
// The concept text after a `type · path` prefix (the title already leads the concept text).
|
|
76
|
+
function contextualConceptUnits(index) {
|
|
77
|
+
const documents = index?.documents || [];
|
|
78
|
+
return documents
|
|
79
|
+
.filter((doc) => doc?.kind === "concept" && !doc.reserved)
|
|
80
|
+
.map((doc) => {
|
|
81
|
+
const prefix = contextPrefix([doc.type, doc.path]);
|
|
82
|
+
const body = conceptUnitText(doc);
|
|
83
|
+
const text = prefix ? `${prefix}\n${body}` : body;
|
|
84
|
+
return { id: doc.uri, text, hash: hashUnitText(text) };
|
|
85
|
+
});
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
module.exports = {
|
|
89
|
+
CONTEXT_PREFIX_MAX,
|
|
90
|
+
UNIT_HASH_PATTERN,
|
|
91
|
+
conceptUnitText,
|
|
92
|
+
conceptUnits,
|
|
93
|
+
contextPrefix,
|
|
94
|
+
contextualConceptUnits,
|
|
95
|
+
hashUnitText,
|
|
96
|
+
};
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
|
|
3
|
+
// Errors that cross from extension code (producers, tool packs) to a caller
|
|
4
|
+
// carry only a bounded code and scalar details; their message is never
|
|
5
|
+
// forwarded (B-6).
|
|
6
|
+
|
|
7
|
+
const PUBLIC_ERROR_CODE = /^[a-z][a-z0-9_]{0,63}$/;
|
|
8
|
+
const MAX_ERROR_DETAILS = 16;
|
|
9
|
+
const MAX_ERROR_DETAIL_CHARS = 200;
|
|
10
|
+
// Tool packs pass on messages a caller needs to correct its request (a
|
|
11
|
+
// database's error for a query, for instance): longer strings, truncated.
|
|
12
|
+
const TOOL_PACK_DETAILS = Object.freeze({ maxChars: 500, truncate: true });
|
|
13
|
+
|
|
14
|
+
// The error's own code when it is a bounded identifier, else null.
|
|
15
|
+
function publicErrorCode(error) {
|
|
16
|
+
const code = error && typeof error.code === "string" ? error.code : "";
|
|
17
|
+
return PUBLIC_ERROR_CODE.test(code) ? code : null;
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
// The error's `details` reduced to at most 16 entries under identifier keys
|
|
21
|
+
// whose values are null, booleans, finite numbers, or single-line strings
|
|
22
|
+
// holding none of `secrets`; null when none remain. A string longer than
|
|
23
|
+
// `options.maxChars` (default 200) is dropped, or with `options.truncate`
|
|
24
|
+
// cut to that length ending in "…".
|
|
25
|
+
function boundedErrorDetails(error, secrets, options) {
|
|
26
|
+
const maxChars = options?.maxChars || MAX_ERROR_DETAIL_CHARS;
|
|
27
|
+
const raw =
|
|
28
|
+
error?.details && typeof error.details === "object" && !Array.isArray(error.details)
|
|
29
|
+
? error.details
|
|
30
|
+
: {};
|
|
31
|
+
const details = {};
|
|
32
|
+
for (const [key, value] of Object.entries(raw)) {
|
|
33
|
+
if (Object.keys(details).length >= MAX_ERROR_DETAILS) break;
|
|
34
|
+
if (!/^[A-Za-z][A-Za-z0-9_]{0,63}$/.test(key)) continue;
|
|
35
|
+
if (
|
|
36
|
+
value === null ||
|
|
37
|
+
typeof value === "boolean" ||
|
|
38
|
+
(typeof value === "number" && Number.isFinite(value))
|
|
39
|
+
) {
|
|
40
|
+
details[key] = value;
|
|
41
|
+
} else if (
|
|
42
|
+
typeof value === "string" &&
|
|
43
|
+
(value.length <= maxChars || options?.truncate) &&
|
|
44
|
+
!/[\p{Cc}]/u.test(value) &&
|
|
45
|
+
!(secrets || []).some((secret) => secret && value.includes(secret))
|
|
46
|
+
) {
|
|
47
|
+
details[key] = value.length <= maxChars ? value : `${value.slice(0, maxChars - 1)}…`;
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
return Object.keys(details).length ? details : null;
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
module.exports = { TOOL_PACK_DETAILS, boundedErrorDetails, publicErrorCode };
|
package/src/git-source.js
CHANGED
|
@@ -62,15 +62,26 @@ function normalizeLines(value) {
|
|
|
62
62
|
|
|
63
63
|
function safeGitPath(value) {
|
|
64
64
|
if (typeof value !== "string" || !value || value !== value.trim()) {
|
|
65
|
-
throw sourceError(
|
|
65
|
+
throw sourceError(
|
|
66
|
+
"git_source_invalid_path",
|
|
67
|
+
"git.path must be a non-empty repository-relative POSIX path.",
|
|
68
|
+
);
|
|
66
69
|
}
|
|
67
70
|
if (value.includes("\\") || /[\x00-\x1f\x7f]/.test(value) || path.posix.isAbsolute(value)) {
|
|
68
|
-
throw sourceError(
|
|
71
|
+
throw sourceError(
|
|
72
|
+
"git_source_invalid_path",
|
|
73
|
+
"git.path must be a safe repository-relative POSIX path.",
|
|
74
|
+
);
|
|
69
75
|
}
|
|
70
76
|
const segments = value.split("/");
|
|
71
|
-
if (
|
|
72
|
-
|
|
73
|
-
|
|
77
|
+
if (
|
|
78
|
+
segments.some((segment) => !segment || segment === "." || segment === "..") ||
|
|
79
|
+
path.posix.normalize(value) !== value
|
|
80
|
+
) {
|
|
81
|
+
throw sourceError(
|
|
82
|
+
"git_source_invalid_path",
|
|
83
|
+
"git.path cannot contain empty, dot, or parent components.",
|
|
84
|
+
);
|
|
74
85
|
}
|
|
75
86
|
if (Buffer.byteLength(value, "utf8") > 4096) {
|
|
76
87
|
throw sourceError("git_source_invalid_path", "git.path exceeds the 4096-byte safety limit.");
|
|
@@ -92,9 +103,8 @@ function validateGitSource(source) {
|
|
|
92
103
|
throw sourceError("git_source_metadata_missing", "A Git source entry requires a git mapping.");
|
|
93
104
|
}
|
|
94
105
|
|
|
95
|
-
const requestedRevision =
|
|
96
|
-
? source.git.revision.trim()
|
|
97
|
-
: "";
|
|
106
|
+
const requestedRevision =
|
|
107
|
+
typeof source.git.revision === "string" ? source.git.revision.trim() : "";
|
|
98
108
|
const pinned = /^(?:[0-9a-fA-F]{40}|[0-9a-fA-F]{64})$/.test(requestedRevision);
|
|
99
109
|
return {
|
|
100
110
|
repositoryConceptId: source.resource.trim(),
|
|
@@ -109,8 +119,10 @@ function mappingFor(repositoryMappings, conceptId) {
|
|
|
109
119
|
if (repositoryMappings instanceof Map) {
|
|
110
120
|
return repositoryMappings.has(conceptId) ? repositoryMappings.get(conceptId) : undefined;
|
|
111
121
|
}
|
|
112
|
-
if (
|
|
113
|
-
|
|
122
|
+
if (
|
|
123
|
+
isPlainObject(repositoryMappings) &&
|
|
124
|
+
Object.prototype.hasOwnProperty.call(repositoryMappings, conceptId)
|
|
125
|
+
) {
|
|
114
126
|
return repositoryMappings[conceptId];
|
|
115
127
|
}
|
|
116
128
|
return undefined;
|
|
@@ -212,11 +224,12 @@ function runGit(repositoryPath, args, options, maxBuffer) {
|
|
|
212
224
|
throw sourceError("git_source_process_error", "Git returned no process result.");
|
|
213
225
|
}
|
|
214
226
|
if (result.error) {
|
|
215
|
-
const code =
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
227
|
+
const code =
|
|
228
|
+
result.error.code === "ETIMEDOUT"
|
|
229
|
+
? "git_source_timeout"
|
|
230
|
+
: result.error.code === "ENOBUFS"
|
|
231
|
+
? "git_source_output_too_large"
|
|
232
|
+
: "git_source_process_error";
|
|
220
233
|
throw sourceError(code, `Git source inspection failed: ${result.error.message}`);
|
|
221
234
|
}
|
|
222
235
|
if (result.signal) {
|
|
@@ -229,9 +242,11 @@ function runGit(repositoryPath, args, options, maxBuffer) {
|
|
|
229
242
|
}
|
|
230
243
|
|
|
231
244
|
function decodeAscii(buffer, code, label) {
|
|
232
|
-
if (
|
|
233
|
-
|
|
234
|
-
|
|
245
|
+
if (
|
|
246
|
+
!Array.from(buffer).every(
|
|
247
|
+
(byte) => byte === 9 || byte === 10 || byte === 13 || (byte >= 32 && byte <= 126),
|
|
248
|
+
)
|
|
249
|
+
) {
|
|
235
250
|
throw sourceError(code, `${label} was not valid ASCII.`);
|
|
236
251
|
}
|
|
237
252
|
return buffer.toString("ascii");
|
|
@@ -262,10 +277,16 @@ function parseTreeEntry(buffer, expectedPath) {
|
|
|
262
277
|
try {
|
|
263
278
|
returnedPath = new TextDecoder("utf-8", { fatal: true }).decode(record.subarray(tab + 1));
|
|
264
279
|
} catch (error) {
|
|
265
|
-
throw sourceError(
|
|
280
|
+
throw sourceError(
|
|
281
|
+
"git_source_invalid_tree_path",
|
|
282
|
+
"Git returned a tree path that is not valid UTF-8.",
|
|
283
|
+
);
|
|
266
284
|
}
|
|
267
285
|
if (returnedPath !== expectedPath) {
|
|
268
|
-
throw sourceError(
|
|
286
|
+
throw sourceError(
|
|
287
|
+
"git_source_tree_path_mismatch",
|
|
288
|
+
"Git returned a different path than the requested source path.",
|
|
289
|
+
);
|
|
269
290
|
}
|
|
270
291
|
return {
|
|
271
292
|
mode: match[1],
|
|
@@ -292,7 +313,10 @@ function sliceInclusiveLines(text, lines) {
|
|
|
292
313
|
return text;
|
|
293
314
|
}
|
|
294
315
|
if (!text.length) {
|
|
295
|
-
throw sourceError(
|
|
316
|
+
throw sourceError(
|
|
317
|
+
"git_source_lines_out_of_range",
|
|
318
|
+
"git.lines cannot select from an empty file.",
|
|
319
|
+
);
|
|
296
320
|
}
|
|
297
321
|
const starts = [0];
|
|
298
322
|
for (let index = 0; index < text.length; index += 1) {
|
|
@@ -353,7 +377,10 @@ function readGitSource(source, repositoryMappings, options) {
|
|
|
353
377
|
if (!revisionType.ok) {
|
|
354
378
|
return unavailable(normalized, "revision_unavailable");
|
|
355
379
|
}
|
|
356
|
-
if (
|
|
380
|
+
if (
|
|
381
|
+
decodeAscii(revisionType.stdout, "git_source_invalid_object_type", "Git object type").trim() !==
|
|
382
|
+
"commit"
|
|
383
|
+
) {
|
|
357
384
|
return unavailable(normalized, "revision_not_commit");
|
|
358
385
|
}
|
|
359
386
|
|
|
@@ -377,7 +404,10 @@ function readGitSource(source, repositoryMappings, options) {
|
|
|
377
404
|
throw sourceError("git_source_submodule", "Git source paths cannot resolve to submodules.");
|
|
378
405
|
}
|
|
379
406
|
if (!["100644", "100755"].includes(entry.mode) || entry.type !== "blob") {
|
|
380
|
-
throw sourceError(
|
|
407
|
+
throw sourceError(
|
|
408
|
+
"git_source_not_regular_file",
|
|
409
|
+
"Git source paths must resolve to regular file blobs.",
|
|
410
|
+
);
|
|
381
411
|
}
|
|
382
412
|
|
|
383
413
|
const sized = runGit(repositoryPath, ["cat-file", "-s", entry.oid], config, metadataBuffer);
|
|
@@ -405,12 +435,20 @@ function readGitSource(source, repositoryMappings, options) {
|
|
|
405
435
|
);
|
|
406
436
|
}
|
|
407
437
|
|
|
408
|
-
const fetched = runGit(
|
|
438
|
+
const fetched = runGit(
|
|
439
|
+
repositoryPath,
|
|
440
|
+
["cat-file", "blob", entry.oid],
|
|
441
|
+
config,
|
|
442
|
+
maxBytes + metadataBuffer,
|
|
443
|
+
);
|
|
409
444
|
if (!fetched.ok) {
|
|
410
445
|
return unavailable(normalized, "blob_unavailable");
|
|
411
446
|
}
|
|
412
447
|
if (fetched.stdout.length !== size || fetched.stdout.length > maxBytes) {
|
|
413
|
-
throw sourceError(
|
|
448
|
+
throw sourceError(
|
|
449
|
+
"git_source_size_mismatch",
|
|
450
|
+
"Git blob content does not match its inspected size.",
|
|
451
|
+
);
|
|
414
452
|
}
|
|
415
453
|
const objectAlgorithm = entry.oid.length === 64 ? "sha256" : "sha1";
|
|
416
454
|
if (blobObjectId(fetched.stdout, objectAlgorithm) !== entry.oid) {
|
|
@@ -456,25 +494,29 @@ function readGitSources(sources, repositoryMappings, options) {
|
|
|
456
494
|
if (!source || source.git === undefined) {
|
|
457
495
|
return results;
|
|
458
496
|
}
|
|
459
|
-
results.push(
|
|
497
|
+
results.push(
|
|
498
|
+
Object.assign({ sourceIndex: index }, readGitSource(source, repositoryMappings, options)),
|
|
499
|
+
);
|
|
460
500
|
return results;
|
|
461
501
|
}, []);
|
|
462
502
|
}
|
|
463
503
|
|
|
464
504
|
function declaredSource(document, sourceId) {
|
|
465
|
-
const sources =
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
505
|
+
const sources =
|
|
506
|
+
document && document.frontmatter && Array.isArray(document.frontmatter.sources)
|
|
507
|
+
? document.frontmatter.sources
|
|
508
|
+
: [];
|
|
509
|
+
const source = sources.find(
|
|
510
|
+
(entry) =>
|
|
511
|
+
entry &&
|
|
512
|
+
typeof entry === "object" &&
|
|
513
|
+
!Array.isArray(entry) &&
|
|
514
|
+
String(entry.id || "") === String(sourceId || ""),
|
|
515
|
+
);
|
|
474
516
|
if (!source) {
|
|
475
517
|
throw sourceError(
|
|
476
518
|
"git_source_not_declared",
|
|
477
|
-
`Concept ${document && document.conceptId || "<unknown>"} has no source with id ${sourceId || "<missing>"}.`,
|
|
519
|
+
`Concept ${(document && document.conceptId) || "<unknown>"} has no source with id ${sourceId || "<missing>"}.`,
|
|
478
520
|
);
|
|
479
521
|
}
|
|
480
522
|
if (!source.git) {
|
|
@@ -485,7 +527,12 @@ function declaredSource(document, sourceId) {
|
|
|
485
527
|
|
|
486
528
|
function repositoryForSource(index, source) {
|
|
487
529
|
const repository = resolveConcept(index, source && source.resource);
|
|
488
|
-
if (
|
|
530
|
+
if (
|
|
531
|
+
!repository ||
|
|
532
|
+
!repository.valid ||
|
|
533
|
+
repository.reserved ||
|
|
534
|
+
repository.type !== "Git Repository"
|
|
535
|
+
) {
|
|
489
536
|
throw sourceError(
|
|
490
537
|
"git_source_repository_invalid",
|
|
491
538
|
"A Git source resource must resolve to a valid concept with type Git Repository.",
|
|
@@ -497,34 +544,47 @@ function repositoryForSource(index, source) {
|
|
|
497
544
|
function readConceptGitSource(index, conceptLocator, sourceId, repositoryMappings, options) {
|
|
498
545
|
const document = resolveConcept(index, conceptLocator);
|
|
499
546
|
if (!document || !document.valid || document.reserved) {
|
|
500
|
-
throw sourceError(
|
|
547
|
+
throw sourceError(
|
|
548
|
+
"git_source_concept_invalid",
|
|
549
|
+
`Unknown valid OKF concept: ${conceptLocator || "<missing>"}`,
|
|
550
|
+
);
|
|
501
551
|
}
|
|
502
552
|
const source = declaredSource(document, sourceId);
|
|
503
553
|
const repository = repositoryForSource(index, source);
|
|
504
|
-
const result = readGitSource(
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
|
|
554
|
+
const result = readGitSource(
|
|
555
|
+
Object.assign({}, source, {
|
|
556
|
+
resource: repository.conceptId,
|
|
557
|
+
}),
|
|
558
|
+
repositoryMappings,
|
|
559
|
+
options,
|
|
560
|
+
);
|
|
561
|
+
return Object.assign(
|
|
562
|
+
{
|
|
563
|
+
conceptId: document.conceptId,
|
|
564
|
+
sourceId: String(sourceId),
|
|
565
|
+
repository: {
|
|
566
|
+
conceptId: repository.conceptId,
|
|
567
|
+
resource: repository.frontmatter.resource || null,
|
|
568
|
+
},
|
|
513
569
|
},
|
|
514
|
-
|
|
570
|
+
result,
|
|
571
|
+
);
|
|
515
572
|
}
|
|
516
573
|
|
|
517
574
|
function describeConceptGitSources(index, document, repositoryMappings) {
|
|
518
|
-
const sources =
|
|
519
|
-
|
|
520
|
-
|
|
575
|
+
const sources =
|
|
576
|
+
document && document.frontmatter && Array.isArray(document.frontmatter.sources)
|
|
577
|
+
? document.frontmatter.sources
|
|
578
|
+
: [];
|
|
521
579
|
return sources.reduce((output, source) => {
|
|
522
580
|
if (!source || typeof source !== "object" || Array.isArray(source) || !source.git) {
|
|
523
581
|
return output;
|
|
524
582
|
}
|
|
525
583
|
try {
|
|
526
584
|
const repository = repositoryForSource(index, source);
|
|
527
|
-
const normalized = validateGitSource(
|
|
585
|
+
const normalized = validateGitSource(
|
|
586
|
+
Object.assign({}, source, { resource: repository.conceptId }),
|
|
587
|
+
);
|
|
528
588
|
output.push({
|
|
529
589
|
id: source.id || null,
|
|
530
590
|
repositoryConceptId: repository.conceptId,
|