@hraness/wordcell 0.23.0 → 0.24.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +29 -16
- package/dist/authoring.js +2 -1
- package/dist/cli.js +10 -9
- package/dist/clip/jobs.js +5 -1
- package/dist/evaluation-builder.js +5 -4
- package/dist/evaluation-kb.js +5 -4
- package/dist/graph-authority.js +2 -1
- package/dist/graph-percolation.js +3 -2
- package/dist/{index-1er88ckw.js → index-3gc2yk4k.js} +10 -5
- package/dist/{index-mnq2wy34.js → index-59bcxqee.js} +1 -1
- package/dist/{index-p70t8j13.js → index-7drercgk.js} +1 -1
- package/dist/{index-tcaq1c7f.js → index-e26mdwxz.js} +1 -1
- package/dist/{index-r4qs6vq4.js → index-hz2wyggk.js} +1 -1
- package/dist/{index-vtwbapjr.js → index-jqkkb4fr.js} +72 -12
- package/dist/{index-tvd2sqrx.js → index-mcpppwbq.js} +439 -4
- package/dist/index-qkesh4c6.js +7 -0
- package/dist/{index-b0khj0h5.js → index-qt2zwza8.js} +6 -1
- package/dist/{index-0eacgvpv.js → index-t0fh8a1q.js} +7 -3
- package/dist/index.js +6 -5
- package/dist/portfolio.js +5 -4
- package/dist/sdk.js +4 -3
- package/dist/{support-c8hy7qfa.js → support-07jpt1z4.js} +9 -8
- package/dist/url-intelligence.js +2 -1
- package/dist/workflows/decision-context.js +5 -4
- package/dist/workflows/index.js +5 -4
- package/package.json +12 -8
- package/skills/wordcell/SKILL.md +1 -1
- package/skills/wordcell/references/publish.md +2 -1
- package/skills/wordcell/references/query.md +17 -3
- package/src/authoring-platform.ts +3 -0
- package/src/cli-help.ts +1 -0
- package/src/cli-program.ts +41 -3
- package/src/clip/jobs.ts +2 -1
- package/src/clip/metadata-search-tool/Cargo.lock +2 -0
- package/src/clip/metadata-search-tool/Cargo.toml +2 -0
- package/src/clip/metadata-search-tool/runner.ts +4 -1
- package/src/clip/metadata-search-tool/src/duckduckgo.rs +534 -0
- package/src/clip/metadata-search-tool/src/main.rs +31 -18
- package/src/clip/metadata-search.ts +4 -2
- package/src/clip/url-metadata.ts +2 -1
- package/src/directory-sync.ts +9 -0
- package/src/graph-authority.ts +5 -2
- package/src/mcp-tools.ts +17 -1
- package/src/sdk.ts +149 -1
- package/src/selected-passage.ts +409 -0
|
@@ -19,10 +19,10 @@ import {
|
|
|
19
19
|
} from "./index-j70m75wd.js";
|
|
20
20
|
import {
|
|
21
21
|
percolateWithGraph
|
|
22
|
-
} from "./index-
|
|
22
|
+
} from "./index-e26mdwxz.js";
|
|
23
23
|
import {
|
|
24
24
|
openGraphAuthority
|
|
25
|
-
} from "./index-
|
|
25
|
+
} from "./index-3gc2yk4k.js";
|
|
26
26
|
import {
|
|
27
27
|
validateGraphPercolationOptions,
|
|
28
28
|
validateGraphQueryRequest
|
|
@@ -58,6 +58,317 @@ import {
|
|
|
58
58
|
|
|
59
59
|
// src/sdk.ts
|
|
60
60
|
import { resolve } from "path";
|
|
61
|
+
|
|
62
|
+
// src/selected-passage.ts
|
|
63
|
+
import { createHash } from "crypto";
|
|
64
|
+
var SELECTED_PASSAGE_MAX_BYTES = 512;
|
|
65
|
+
var SELECTED_PASSAGE_MAX_HEADING_BYTES = 256;
|
|
66
|
+
var SELECTED_PASSAGE_MAX_SOURCE_BYTES = 2 * 1024 * 1024;
|
|
67
|
+
var SELECTED_PASSAGE_MAX_SEARCH_BYTES = 8 * 1024 * 1024;
|
|
68
|
+
var MAX_QUERY_BYTES = 16 * 1024;
|
|
69
|
+
var MAX_QUERY_TERMS = 64;
|
|
70
|
+
var MAX_CANDIDATES = 1e4;
|
|
71
|
+
var WORD = /[\p{L}\p{N}][\p{L}\p{N}\p{M}]*/gu;
|
|
72
|
+
var STOP_WORDS = new Set(("a an and are as at be been being but by can could did do does doing each either " + "for from had has have having he her here hers herself him himself his how i " + "if in into is it its itself may me might mine more most must my myself neither " + "no nor not of on once only or our ours ourselves out own same she should so " + "some such than that the their theirs them themselves then there these they " + "this those through to too under until up us very was we were what when where " + "which while who whom why will with would you your yours yourself yourselves").split(" "));
|
|
73
|
+
var SELECTED_PASSAGE_NONE_REASONS = Object.freeze([
|
|
74
|
+
"no-query-terms",
|
|
75
|
+
"no-lexical-match"
|
|
76
|
+
]);
|
|
77
|
+
var SELECTED_PASSAGE_UNAVAILABLE_REASONS = Object.freeze([
|
|
78
|
+
"query-limit",
|
|
79
|
+
"source-limit",
|
|
80
|
+
"search-budget",
|
|
81
|
+
"candidate-limit",
|
|
82
|
+
"grapheme-limit",
|
|
83
|
+
"invalid-source",
|
|
84
|
+
"display-sanitized"
|
|
85
|
+
]);
|
|
86
|
+
function createSelectedPassageBudget(maximumBytes = SELECTED_PASSAGE_MAX_SEARCH_BYTES) {
|
|
87
|
+
if (!Number.isSafeInteger(maximumBytes) || maximumBytes < 0) {
|
|
88
|
+
throw new RangeError("Selected passage budget must be a non-negative safe integer.");
|
|
89
|
+
}
|
|
90
|
+
let remaining = maximumBytes;
|
|
91
|
+
return Object.freeze({
|
|
92
|
+
take: (bytes) => {
|
|
93
|
+
if (bytes > remaining)
|
|
94
|
+
return false;
|
|
95
|
+
remaining -= bytes;
|
|
96
|
+
return true;
|
|
97
|
+
}
|
|
98
|
+
});
|
|
99
|
+
}
|
|
100
|
+
function unavailable(reason) {
|
|
101
|
+
return Object.freeze({ status: "unavailable", reason });
|
|
102
|
+
}
|
|
103
|
+
function none(reason) {
|
|
104
|
+
return Object.freeze({ status: "none", reason });
|
|
105
|
+
}
|
|
106
|
+
function normalize(value) {
|
|
107
|
+
return value.normalize("NFC").toLocaleLowerCase("en-US");
|
|
108
|
+
}
|
|
109
|
+
function wellFormed(value) {
|
|
110
|
+
return !/\p{Surrogate}/u.test(value);
|
|
111
|
+
}
|
|
112
|
+
function lowerBound(values, target) {
|
|
113
|
+
let low = 0;
|
|
114
|
+
let high = values.length;
|
|
115
|
+
while (low < high) {
|
|
116
|
+
const middle = low + high >>> 1;
|
|
117
|
+
if (values[middle] < target)
|
|
118
|
+
low = middle + 1;
|
|
119
|
+
else
|
|
120
|
+
high = middle;
|
|
121
|
+
}
|
|
122
|
+
return low;
|
|
123
|
+
}
|
|
124
|
+
function upperBound(values, target) {
|
|
125
|
+
let low = 0;
|
|
126
|
+
let high = values.length;
|
|
127
|
+
while (low < high) {
|
|
128
|
+
const middle = low + high >>> 1;
|
|
129
|
+
if (values[middle] <= target)
|
|
130
|
+
low = middle + 1;
|
|
131
|
+
else
|
|
132
|
+
high = middle;
|
|
133
|
+
}
|
|
134
|
+
return low;
|
|
135
|
+
}
|
|
136
|
+
function sourceLayout(text) {
|
|
137
|
+
const lineStarts = [0];
|
|
138
|
+
for (let index = text.indexOf(`
|
|
139
|
+
`);index >= 0; index = text.indexOf(`
|
|
140
|
+
`, index + 1)) {
|
|
141
|
+
lineStarts.push(index + 1);
|
|
142
|
+
}
|
|
143
|
+
const lineEnd = (index) => {
|
|
144
|
+
const next = lineStarts[index + 1];
|
|
145
|
+
let end = next === undefined ? text.length : next - 1;
|
|
146
|
+
if (end > lineStarts[index] && text[end - 1] === "\r")
|
|
147
|
+
end--;
|
|
148
|
+
return end;
|
|
149
|
+
};
|
|
150
|
+
let firstBodyLine = 0;
|
|
151
|
+
if (text.slice(0, lineEnd(0)).trim() === "---") {
|
|
152
|
+
for (let index = 1;index < lineStarts.length; index++) {
|
|
153
|
+
if (text.slice(lineStarts[index], lineEnd(index)).trim() === "---") {
|
|
154
|
+
firstBodyLine = index + 1;
|
|
155
|
+
break;
|
|
156
|
+
}
|
|
157
|
+
}
|
|
158
|
+
}
|
|
159
|
+
const bodyStart = lineStarts[firstBodyLine] ?? text.length;
|
|
160
|
+
const paragraphs = [];
|
|
161
|
+
const headings = [];
|
|
162
|
+
let fence;
|
|
163
|
+
let current;
|
|
164
|
+
for (let index = firstBodyLine;index < lineStarts.length; index++) {
|
|
165
|
+
const start = lineStarts[index];
|
|
166
|
+
const end = lineEnd(index);
|
|
167
|
+
const line = text.slice(start, end);
|
|
168
|
+
if (line.trim() === "") {
|
|
169
|
+
if (current !== undefined)
|
|
170
|
+
paragraphs.push(current);
|
|
171
|
+
current = undefined;
|
|
172
|
+
continue;
|
|
173
|
+
}
|
|
174
|
+
const delimiter = /^\s{0,3}(`{3,}|~{3,})/u.exec(line)?.[1];
|
|
175
|
+
const marker = fence === undefined ? /^\s{0,3}(#{1,6})(?:\s|$)/u.exec(line)?.[1] : undefined;
|
|
176
|
+
const heading = marker !== undefined;
|
|
177
|
+
if (heading)
|
|
178
|
+
headings.push({ start, end, level: marker.length, line: index + 1 });
|
|
179
|
+
if (delimiter !== undefined) {
|
|
180
|
+
if (fence === undefined)
|
|
181
|
+
fence = { character: delimiter[0], length: delimiter.length };
|
|
182
|
+
else if (delimiter[0] === fence.character && delimiter.length >= fence.length && line.trim() === delimiter)
|
|
183
|
+
fence = undefined;
|
|
184
|
+
}
|
|
185
|
+
if (current === undefined)
|
|
186
|
+
current = { start, end, headingOnly: heading };
|
|
187
|
+
else {
|
|
188
|
+
current.end = end;
|
|
189
|
+
current.headingOnly &&= heading;
|
|
190
|
+
}
|
|
191
|
+
}
|
|
192
|
+
if (current !== undefined)
|
|
193
|
+
paragraphs.push(current);
|
|
194
|
+
return { bodyStart, lineStarts, paragraphs, headings };
|
|
195
|
+
}
|
|
196
|
+
function graphemePrefix(value, maximumBytes) {
|
|
197
|
+
if (Buffer.byteLength(value, "utf8") <= maximumBytes)
|
|
198
|
+
return { text: value, truncated: false };
|
|
199
|
+
let end = 0;
|
|
200
|
+
let bytes = 0;
|
|
201
|
+
for (const part of new Intl.Segmenter("en-US", { granularity: "grapheme" }).segment(value)) {
|
|
202
|
+
const width = Buffer.byteLength(part.segment, "utf8");
|
|
203
|
+
if (bytes + width > maximumBytes)
|
|
204
|
+
break;
|
|
205
|
+
bytes += width;
|
|
206
|
+
end = part.index + part.segment.length;
|
|
207
|
+
}
|
|
208
|
+
return { text: value.slice(0, end), truncated: true };
|
|
209
|
+
}
|
|
210
|
+
function enclosingHeadings(text, headings, passageStart, passageEnd) {
|
|
211
|
+
const stack = [];
|
|
212
|
+
const pop = (level) => {
|
|
213
|
+
while (stack.length > 0 && stack[stack.length - 1].level >= level)
|
|
214
|
+
stack.pop();
|
|
215
|
+
};
|
|
216
|
+
for (const heading of headings) {
|
|
217
|
+
if (heading.start >= passageEnd)
|
|
218
|
+
break;
|
|
219
|
+
if (heading.start >= passageStart) {
|
|
220
|
+
const before = text.slice(passageStart, heading.start).split(`
|
|
221
|
+
`);
|
|
222
|
+
if (!before.every((line) => line.trim() === "" || /^\s{0,3}#{1,6}(?:\s|$)/u.test(line)))
|
|
223
|
+
break;
|
|
224
|
+
pop(heading.level);
|
|
225
|
+
continue;
|
|
226
|
+
}
|
|
227
|
+
pop(heading.level);
|
|
228
|
+
stack.push(heading);
|
|
229
|
+
}
|
|
230
|
+
return Object.freeze(stack.map((heading) => {
|
|
231
|
+
const end = Math.min(heading.end, passageStart);
|
|
232
|
+
const clipped = graphemePrefix(text.slice(heading.start, end), SELECTED_PASSAGE_MAX_HEADING_BYTES);
|
|
233
|
+
const startByte = Buffer.byteLength(text.slice(0, heading.start), "utf8");
|
|
234
|
+
return Object.freeze({
|
|
235
|
+
level: heading.level,
|
|
236
|
+
text: clipped.text,
|
|
237
|
+
startByte,
|
|
238
|
+
endByte: startByte + Buffer.byteLength(clipped.text, "utf8"),
|
|
239
|
+
startLine: heading.line,
|
|
240
|
+
endLine: heading.line,
|
|
241
|
+
truncated: clipped.truncated || end < heading.end
|
|
242
|
+
});
|
|
243
|
+
}));
|
|
244
|
+
}
|
|
245
|
+
function selectSearchPassage(input) {
|
|
246
|
+
const { query, text, budget } = input;
|
|
247
|
+
if (typeof query !== "string" || Buffer.byteLength(query, "utf8") > MAX_QUERY_BYTES) {
|
|
248
|
+
return unavailable("query-limit");
|
|
249
|
+
}
|
|
250
|
+
if (typeof text !== "string" || !wellFormed(text))
|
|
251
|
+
return unavailable("invalid-source");
|
|
252
|
+
const allTerms = new Set([...normalize(query).matchAll(WORD)].map((match) => match[0]));
|
|
253
|
+
if (allTerms.size > MAX_QUERY_TERMS)
|
|
254
|
+
return unavailable("query-limit");
|
|
255
|
+
const terms = new Set([...allTerms].filter((term) => !STOP_WORDS.has(term)));
|
|
256
|
+
if (terms.size === 0)
|
|
257
|
+
return none("no-query-terms");
|
|
258
|
+
const sourceBytes = Buffer.byteLength(text, "utf8");
|
|
259
|
+
if (sourceBytes > SELECTED_PASSAGE_MAX_SOURCE_BYTES)
|
|
260
|
+
return unavailable("source-limit");
|
|
261
|
+
if (!budget.take(sourceBytes))
|
|
262
|
+
return unavailable("search-budget");
|
|
263
|
+
if (text.length === 0)
|
|
264
|
+
return none("no-lexical-match");
|
|
265
|
+
const maxBytes = SELECTED_PASSAGE_MAX_BYTES;
|
|
266
|
+
const { bodyStart, lineStarts, paragraphs, headings } = sourceLayout(text);
|
|
267
|
+
const headingStarts = new Set(headings.map(({ start }) => start));
|
|
268
|
+
const headingOnlyFile = paragraphs.every((paragraph) => paragraph.headingOnly);
|
|
269
|
+
const positions = [bodyStart];
|
|
270
|
+
const bytePositions = [Buffer.byteLength(text.slice(0, bodyStart), "utf8")];
|
|
271
|
+
let bytes = bytePositions[0];
|
|
272
|
+
for (const part of new Intl.Segmenter("en-US", { granularity: "grapheme" }).segment(text.slice(bodyStart))) {
|
|
273
|
+
const width = Buffer.byteLength(part.segment, "utf8");
|
|
274
|
+
if (width > maxBytes)
|
|
275
|
+
return unavailable("grapheme-limit");
|
|
276
|
+
bytes += width;
|
|
277
|
+
positions.push(bodyStart + part.index + part.segment.length);
|
|
278
|
+
bytePositions.push(bytes);
|
|
279
|
+
}
|
|
280
|
+
const tokens = [];
|
|
281
|
+
const tokenStarts = [];
|
|
282
|
+
for (const match of text.slice(bodyStart).matchAll(WORD)) {
|
|
283
|
+
const start = bodyStart + match.index;
|
|
284
|
+
const term = normalize(match[0]);
|
|
285
|
+
const inHeading = headingStarts.has(lineStarts[upperBound(lineStarts, start) - 1]);
|
|
286
|
+
tokenStarts.push(start);
|
|
287
|
+
tokens.push({
|
|
288
|
+
start,
|
|
289
|
+
end: start + match[0].length,
|
|
290
|
+
term: terms.has(term) && (headingOnlyFile || !inHeading) ? term : undefined
|
|
291
|
+
});
|
|
292
|
+
}
|
|
293
|
+
const regions = [];
|
|
294
|
+
let pendingHeading;
|
|
295
|
+
for (const paragraph of paragraphs) {
|
|
296
|
+
if (paragraph.headingOnly) {
|
|
297
|
+
pendingHeading = pendingHeading === undefined ? paragraph : { ...pendingHeading, end: paragraph.end };
|
|
298
|
+
} else {
|
|
299
|
+
regions.push({ start: pendingHeading?.start ?? paragraph.start, end: paragraph.end });
|
|
300
|
+
pendingHeading = undefined;
|
|
301
|
+
}
|
|
302
|
+
}
|
|
303
|
+
if (regions.length === 0 && pendingHeading !== undefined)
|
|
304
|
+
regions.push(pendingHeading);
|
|
305
|
+
let best;
|
|
306
|
+
let considered = 0;
|
|
307
|
+
for (const region of regions) {
|
|
308
|
+
const first = lowerBound(positions, region.start);
|
|
309
|
+
const last = upperBound(positions, region.end) - 1;
|
|
310
|
+
if (first >= last)
|
|
311
|
+
continue;
|
|
312
|
+
let startIndex = first;
|
|
313
|
+
for (;; ) {
|
|
314
|
+
const endIndex = Math.min(last, upperBound(bytePositions, bytePositions[startIndex] + maxBytes) - 1);
|
|
315
|
+
if (endIndex <= startIndex)
|
|
316
|
+
return unavailable("grapheme-limit");
|
|
317
|
+
if (++considered > MAX_CANDIDATES)
|
|
318
|
+
return unavailable("candidate-limit");
|
|
319
|
+
const start = positions[startIndex];
|
|
320
|
+
const end = positions[endIndex];
|
|
321
|
+
const matched = new Set;
|
|
322
|
+
let tokenCount = 0;
|
|
323
|
+
for (let index = lowerBound(tokenStarts, start);index < tokens.length; index++) {
|
|
324
|
+
const token = tokens[index];
|
|
325
|
+
if (token.start >= end)
|
|
326
|
+
break;
|
|
327
|
+
if (token.end > end)
|
|
328
|
+
continue;
|
|
329
|
+
tokenCount++;
|
|
330
|
+
if (token.term !== undefined)
|
|
331
|
+
matched.add(token.term);
|
|
332
|
+
}
|
|
333
|
+
if (matched.size > 0) {
|
|
334
|
+
const score = matched.size + matched.size / (tokenCount + 1) / 10;
|
|
335
|
+
if (best === undefined || score > best.score) {
|
|
336
|
+
best = {
|
|
337
|
+
start,
|
|
338
|
+
end,
|
|
339
|
+
startByte: bytePositions[startIndex],
|
|
340
|
+
endByte: bytePositions[endIndex],
|
|
341
|
+
regionStart: positions[first],
|
|
342
|
+
regionEnd: positions[last],
|
|
343
|
+
score
|
|
344
|
+
};
|
|
345
|
+
}
|
|
346
|
+
}
|
|
347
|
+
if (endIndex === last)
|
|
348
|
+
break;
|
|
349
|
+
const stepped = lowerBound(bytePositions, bytePositions[startIndex] + Math.max(1, Math.floor(maxBytes / 2)));
|
|
350
|
+
const finalStart = lowerBound(bytePositions, bytePositions[last] - maxBytes);
|
|
351
|
+
startIndex = Math.max(startIndex + 1, Math.min(stepped, finalStart));
|
|
352
|
+
}
|
|
353
|
+
}
|
|
354
|
+
if (best === undefined)
|
|
355
|
+
return none("no-lexical-match");
|
|
356
|
+
return Object.freeze({
|
|
357
|
+
status: "selected",
|
|
358
|
+
text: text.slice(best.start, best.end),
|
|
359
|
+
startByte: best.startByte,
|
|
360
|
+
endByte: best.endByte,
|
|
361
|
+
startLine: upperBound(lineStarts, best.start),
|
|
362
|
+
endLine: upperBound(lineStarts, best.end - 1),
|
|
363
|
+
sourceSha256: createHash("sha256").update(text, "utf8").digest("hex"),
|
|
364
|
+
sourceEncoding: "utf8-snapshot",
|
|
365
|
+
clippedStart: best.start > best.regionStart,
|
|
366
|
+
clippedEnd: best.end < best.regionEnd,
|
|
367
|
+
headings: enclosingHeadings(text, headings, best.start, best.end)
|
|
368
|
+
});
|
|
369
|
+
}
|
|
370
|
+
|
|
371
|
+
// src/sdk.ts
|
|
61
372
|
var MAX_SEARCH_RESULTS = 100;
|
|
62
373
|
var MAX_SEARCH_CANDIDATES = 500;
|
|
63
374
|
var DEFAULT_SEARCH_RESULTS = 10;
|
|
@@ -332,6 +643,9 @@ async function openKnowledgeBase(options, dependencies = {}) {
|
|
|
332
643
|
const { query: requestedQuery } = validateSearchQuery(searchOptions.query);
|
|
333
644
|
const expansion = searchRules === null ? { request: searchOptions, alias: null } : expandSearchRequest(searchOptions, searchRules);
|
|
334
645
|
const effectiveOptions = expansion.request;
|
|
646
|
+
if (effectiveOptions.selectedPassage !== undefined && typeof effectiveOptions.selectedPassage !== "boolean") {
|
|
647
|
+
throw new TypeError("Search selectedPassage must be a boolean.");
|
|
648
|
+
}
|
|
335
649
|
const { query: effectiveQuery } = validateSearchQuery(effectiveOptions.query);
|
|
336
650
|
const mode = checkedSearchMode(effectiveOptions.mode);
|
|
337
651
|
const ordering = checkedSearchOrdering(effectiveOptions.ordering);
|
|
@@ -561,6 +875,18 @@ async function openKnowledgeBase(options, dependencies = {}) {
|
|
|
561
875
|
results = Object.freeze(selectedHits.map((hit, index) => hit.rank === index + 1 ? hit : { ...hit, rank: index + 1 }));
|
|
562
876
|
}
|
|
563
877
|
}
|
|
878
|
+
if (effectiveOptions.selectedPassage === true) {
|
|
879
|
+
const budget = createSelectedPassageBudget();
|
|
880
|
+
results = Object.freeze(results.map((hit) => {
|
|
881
|
+
const note = notesById.get(hit.id);
|
|
882
|
+
if (note === undefined)
|
|
883
|
+
throw new Error("Selected search hit has no snapshot note.");
|
|
884
|
+
return Object.freeze({
|
|
885
|
+
...hit,
|
|
886
|
+
selectedPassage: selectSearchPassage({ query: effectiveQuery, text: note.content, budget })
|
|
887
|
+
});
|
|
888
|
+
}));
|
|
889
|
+
}
|
|
564
890
|
let graph = null;
|
|
565
891
|
if (graphOptions !== null) {
|
|
566
892
|
try {
|
|
@@ -793,7 +1119,25 @@ Evidence: `))
|
|
|
793
1119
|
|
|
794
1120
|
`))
|
|
795
1121
|
return writer.result();
|
|
796
|
-
|
|
1122
|
+
const selected = selectedPassageForContext(hit, `Knowledge-base search result ${hit.rank}`);
|
|
1123
|
+
if (selected?.status === "selected") {
|
|
1124
|
+
for (const heading of selected.headings) {
|
|
1125
|
+
if (!append(`Section (level ${heading.level}, lines ${heading.startLine}-${heading.endLine}${heading.truncated ? ", shortened" : ""}):
|
|
1126
|
+
`))
|
|
1127
|
+
return writer.result();
|
|
1128
|
+
if (!append(heading.text))
|
|
1129
|
+
return writer.result();
|
|
1130
|
+
if (!append(`
|
|
1131
|
+
|
|
1132
|
+
`))
|
|
1133
|
+
return writer.result();
|
|
1134
|
+
}
|
|
1135
|
+
if (!append(`Selected source (UTF-8 snapshot ${selected.sourceSha256}; bytes ${selected.startByte}-${selected.endByte}; lines ${selected.startLine}-${selected.endLine}${selected.clippedStart || selected.clippedEnd ? "; excerpt clipped" : ""}):
|
|
1136
|
+
`))
|
|
1137
|
+
return writer.result();
|
|
1138
|
+
if (!append(selected.text))
|
|
1139
|
+
return writer.result();
|
|
1140
|
+
} else if (!append(hit.snippet))
|
|
797
1141
|
return writer.result();
|
|
798
1142
|
}
|
|
799
1143
|
const graph = result.graph;
|
|
@@ -954,6 +1298,95 @@ function optionalDataNumber(record, key, label) {
|
|
|
954
1298
|
return;
|
|
955
1299
|
return dataNumber(record[key], `${label}.${key}`);
|
|
956
1300
|
}
|
|
1301
|
+
function dataCount(value, label, minimum = 0) {
|
|
1302
|
+
if (typeof value !== "number" || !Number.isSafeInteger(value) || value < minimum) {
|
|
1303
|
+
throw new TypeError(`${label} must be an integer of at least ${minimum}.`);
|
|
1304
|
+
}
|
|
1305
|
+
return value;
|
|
1306
|
+
}
|
|
1307
|
+
function dataSourceText(value, label, maximumBytes) {
|
|
1308
|
+
const text = dataString(value, label);
|
|
1309
|
+
if (/\p{Surrogate}/u.test(text))
|
|
1310
|
+
throw new TypeError(`${label} must be well-formed Unicode.`);
|
|
1311
|
+
if (Buffer.byteLength(text, "utf8") > maximumBytes) {
|
|
1312
|
+
throw new RangeError(`${label} exceeds ${maximumBytes} UTF-8 bytes.`);
|
|
1313
|
+
}
|
|
1314
|
+
return text;
|
|
1315
|
+
}
|
|
1316
|
+
function sourceSpan(record, label, text) {
|
|
1317
|
+
const startByte = dataCount(requiredData(record, "startByte", label), `${label}.startByte`);
|
|
1318
|
+
const endByte = dataCount(requiredData(record, "endByte", label), `${label}.endByte`);
|
|
1319
|
+
const startLine = dataCount(requiredData(record, "startLine", label), `${label}.startLine`, 1);
|
|
1320
|
+
const endLine = dataCount(requiredData(record, "endLine", label), `${label}.endLine`, startLine);
|
|
1321
|
+
if (endByte - startByte !== Buffer.byteLength(text, "utf8")) {
|
|
1322
|
+
throw new TypeError(`${label} byte span must match its text.`);
|
|
1323
|
+
}
|
|
1324
|
+
return { startByte, endByte, startLine, endLine };
|
|
1325
|
+
}
|
|
1326
|
+
function selectedPassageData(value, label) {
|
|
1327
|
+
const record = inspectedDataRecord(value, label);
|
|
1328
|
+
const status = requiredData(record, "status", label);
|
|
1329
|
+
if (status === "none" || status === "unavailable") {
|
|
1330
|
+
const reason = requiredData(record, "reason", label);
|
|
1331
|
+
const reasons = status === "none" ? SELECTED_PASSAGE_NONE_REASONS : SELECTED_PASSAGE_UNAVAILABLE_REASONS;
|
|
1332
|
+
if (typeof reason !== "string" || !reasons.includes(reason)) {
|
|
1333
|
+
throw new TypeError(`${label}.reason is not a known ${status} reason.`);
|
|
1334
|
+
}
|
|
1335
|
+
return Object.freeze({ status, reason });
|
|
1336
|
+
}
|
|
1337
|
+
if (status !== "selected")
|
|
1338
|
+
throw new TypeError(`${label}.status is not a known selected-passage status.`);
|
|
1339
|
+
const text = dataSourceText(requiredData(record, "text", label), `${label}.text`, SELECTED_PASSAGE_MAX_BYTES);
|
|
1340
|
+
if (text === "")
|
|
1341
|
+
throw new TypeError(`${label}.text must not be empty.`);
|
|
1342
|
+
const span = sourceSpan(record, label, text);
|
|
1343
|
+
const sourceSha256 = dataString(requiredData(record, "sourceSha256", label), `${label}.sourceSha256`);
|
|
1344
|
+
if (!/^[0-9a-f]{64}$/u.test(sourceSha256))
|
|
1345
|
+
throw new TypeError(`${label}.sourceSha256 must be a SHA-256 digest.`);
|
|
1346
|
+
if (requiredData(record, "sourceEncoding", label) !== "utf8-snapshot") {
|
|
1347
|
+
throw new TypeError(`${label}.sourceEncoding must be utf8-snapshot.`);
|
|
1348
|
+
}
|
|
1349
|
+
let previousLevel = 0;
|
|
1350
|
+
let previousEnd = 0;
|
|
1351
|
+
const headings = inspectedDataArray(requiredData(record, "headings", label), `${label}.headings`, 6).map((entry, index) => {
|
|
1352
|
+
const headingLabel = `${label}.headings[${index}]`;
|
|
1353
|
+
const heading = inspectedDataRecord(entry, headingLabel);
|
|
1354
|
+
const level = dataCount(requiredData(heading, "level", headingLabel), `${headingLabel}.level`, previousLevel + 1);
|
|
1355
|
+
if (level > 6)
|
|
1356
|
+
throw new TypeError(`${headingLabel}.level must be at most 6.`);
|
|
1357
|
+
const headingText = dataSourceText(requiredData(heading, "text", headingLabel), `${headingLabel}.text`, SELECTED_PASSAGE_MAX_HEADING_BYTES);
|
|
1358
|
+
const headingSpan = sourceSpan(heading, headingLabel, headingText);
|
|
1359
|
+
if (headingSpan.startByte < previousEnd || headingSpan.endByte > span.startByte) {
|
|
1360
|
+
throw new TypeError(`${headingLabel} must precede the selected passage in source order.`);
|
|
1361
|
+
}
|
|
1362
|
+
previousLevel = level;
|
|
1363
|
+
previousEnd = headingSpan.endByte;
|
|
1364
|
+
return Object.freeze({
|
|
1365
|
+
level,
|
|
1366
|
+
text: headingText,
|
|
1367
|
+
...headingSpan,
|
|
1368
|
+
truncated: dataBoolean(requiredData(heading, "truncated", headingLabel), `${headingLabel}.truncated`)
|
|
1369
|
+
});
|
|
1370
|
+
});
|
|
1371
|
+
return Object.freeze({
|
|
1372
|
+
status,
|
|
1373
|
+
text,
|
|
1374
|
+
...span,
|
|
1375
|
+
sourceSha256,
|
|
1376
|
+
sourceEncoding: "utf8-snapshot",
|
|
1377
|
+
clippedStart: dataBoolean(requiredData(record, "clippedStart", label), `${label}.clippedStart`),
|
|
1378
|
+
clippedEnd: dataBoolean(requiredData(record, "clippedEnd", label), `${label}.clippedEnd`),
|
|
1379
|
+
headings: Object.freeze(headings)
|
|
1380
|
+
});
|
|
1381
|
+
}
|
|
1382
|
+
function selectedPassageForContext(hit, label) {
|
|
1383
|
+
const descriptor = Object.getOwnPropertyDescriptor(hit, "selectedPassage");
|
|
1384
|
+
if (descriptor === undefined)
|
|
1385
|
+
return;
|
|
1386
|
+
if (!("value" in descriptor))
|
|
1387
|
+
throw new TypeError(`${label} contains an accessor property.`);
|
|
1388
|
+
return descriptor.value === undefined ? undefined : selectedPassageData(descriptor.value, `${label}.selectedPassage`);
|
|
1389
|
+
}
|
|
957
1390
|
function searchEvidenceSummary(value, label) {
|
|
958
1391
|
const evidence = inspectedDataRecord(value, label);
|
|
959
1392
|
return Object.freeze({
|
|
@@ -966,6 +1399,7 @@ function searchHitContextRecord(value, index) {
|
|
|
966
1399
|
const hit = inspectedDataRecord(value, label);
|
|
967
1400
|
const evidence = inspectedDataArray(requiredData(hit, "evidence", label), `${label}.evidence`, 32).map((entry, evidenceIndex) => searchEvidenceSummary(entry, `${label}.evidence[${evidenceIndex}]`));
|
|
968
1401
|
const line = optionalDataNumber(hit, "line", label);
|
|
1402
|
+
const selectedPassage = selectedPassageForContext(hit, label);
|
|
969
1403
|
return Object.freeze({
|
|
970
1404
|
kind: "search-result",
|
|
971
1405
|
id: dataString(requiredData(hit, "id", label), `${label}.id`),
|
|
@@ -976,7 +1410,8 @@ function searchHitContextRecord(value, index) {
|
|
|
976
1410
|
identity: dataBoolean(requiredData(hit, "identity", label), `${label}.identity`),
|
|
977
1411
|
...line === undefined ? {} : { line },
|
|
978
1412
|
evidence: Object.freeze(evidence),
|
|
979
|
-
snippet: dataString(requiredData(hit, "snippet", label), `${label}.snippet`)
|
|
1413
|
+
snippet: dataString(requiredData(hit, "snippet", label), `${label}.snippet`),
|
|
1414
|
+
...selectedPassage === undefined ? {} : { selectedPassage }
|
|
980
1415
|
});
|
|
981
1416
|
}
|
|
982
1417
|
function graphContextRecords(value) {
|
|
@@ -2,6 +2,9 @@
|
|
|
2
2
|
import {
|
|
3
3
|
acquireNoteLock
|
|
4
4
|
} from "./index-3rm7cz6h.js";
|
|
5
|
+
import {
|
|
6
|
+
syncsDirectories
|
|
7
|
+
} from "./index-qkesh4c6.js";
|
|
5
8
|
import {
|
|
6
9
|
isCanonicalNoteId,
|
|
7
10
|
isCanonicalRelationPredicate,
|
|
@@ -844,7 +847,9 @@ async function cleanupTemporary(temporaryPath, identity) {
|
|
|
844
847
|
throw error;
|
|
845
848
|
}
|
|
846
849
|
}
|
|
847
|
-
async function fsyncDirectory(path, openDirectory = open) {
|
|
850
|
+
async function fsyncDirectory(path, openDirectory = open, platform = process.platform) {
|
|
851
|
+
if (!syncsDirectories(platform))
|
|
852
|
+
return;
|
|
848
853
|
const handle = await openDirectory(path, constants.O_RDONLY);
|
|
849
854
|
try {
|
|
850
855
|
await handle.sync();
|
|
@@ -14,6 +14,9 @@ import {
|
|
|
14
14
|
import {
|
|
15
15
|
sanitizeArtifactUrl
|
|
16
16
|
} from "./index-mxxxytys.js";
|
|
17
|
+
import {
|
|
18
|
+
syncsDirectories
|
|
19
|
+
} from "./index-qkesh4c6.js";
|
|
17
20
|
|
|
18
21
|
// src/clip/metadata-search.ts
|
|
19
22
|
import { spawn } from "child_process";
|
|
@@ -320,7 +323,7 @@ async function resolveEngineHosts(resolver, timeoutMs) {
|
|
|
320
323
|
allowPrivateNetwork: false,
|
|
321
324
|
timeoutMs
|
|
322
325
|
});
|
|
323
|
-
if (addresses.length === 0
|
|
326
|
+
if (addresses.length === 0) {
|
|
324
327
|
throw new Error("metadata search engine DNS answer count is invalid");
|
|
325
328
|
}
|
|
326
329
|
const unique = new Map;
|
|
@@ -333,7 +336,7 @@ async function resolveEngineHosts(resolver, timeoutMs) {
|
|
|
333
336
|
throw new Error("metadata search engine DNS answer is duplicated");
|
|
334
337
|
return Object.freeze({
|
|
335
338
|
hostname,
|
|
336
|
-
addresses: Object.freeze([...unique.values()].sort((left, right) => left.family - right.family || (left.address < right.address ? -1 : left.address > right.address ? 1 : 0)))
|
|
339
|
+
addresses: Object.freeze([...unique.values()].sort((left, right) => left.family - right.family || (left.address < right.address ? -1 : left.address > right.address ? 1 : 0)).slice(0, MAX_ENGINE_ADDRESSES))
|
|
337
340
|
});
|
|
338
341
|
}));
|
|
339
342
|
return Object.freeze(resolved);
|
|
@@ -1524,7 +1527,8 @@ function writeUrlMetadataDocument(saved, document) {
|
|
|
1524
1527
|
throw new Error(`URL metadata sidecar failed post-install verification: ${saved.sidecarPath}`);
|
|
1525
1528
|
}
|
|
1526
1529
|
assertHeldUrlMetadataWriteLockCurrent(heldLock);
|
|
1527
|
-
|
|
1530
|
+
if (syncsDirectories())
|
|
1531
|
+
fsyncSync2(directoryDescriptor);
|
|
1528
1532
|
return { changed: true, path: saved.sidecarPath };
|
|
1529
1533
|
} catch (error) {
|
|
1530
1534
|
operationError = error;
|
package/dist/index.js
CHANGED
|
@@ -37,7 +37,7 @@ import {
|
|
|
37
37
|
knowledgeBaseEvaluationRetrieverIds,
|
|
38
38
|
openKnowledgeBaseEvaluation,
|
|
39
39
|
verifyFrozenEvaluationSnapshot
|
|
40
|
-
} from "./index-
|
|
40
|
+
} from "./index-59bcxqee.js";
|
|
41
41
|
import {
|
|
42
42
|
DEFAULT_SEARCH_RESULTS,
|
|
43
43
|
MAX_SEARCH_CANDIDATES,
|
|
@@ -50,7 +50,7 @@ import {
|
|
|
50
50
|
packUntrustedSearchContext,
|
|
51
51
|
searchEvidenceRank,
|
|
52
52
|
validateKnowledgeBaseSearchHistory
|
|
53
|
-
} from "./index-
|
|
53
|
+
} from "./index-mcpppwbq.js";
|
|
54
54
|
import"./index-adx6khj5.js";
|
|
55
55
|
import {
|
|
56
56
|
MAX_EMBEDDING_MODEL_BYTES,
|
|
@@ -94,7 +94,7 @@ import {
|
|
|
94
94
|
} from "./index-b88v3vtm.js";
|
|
95
95
|
import {
|
|
96
96
|
percolateWithGraph
|
|
97
|
-
} from "./index-
|
|
97
|
+
} from "./index-e26mdwxz.js";
|
|
98
98
|
import {
|
|
99
99
|
DEFAULT_PERCOLATION_LIMIT,
|
|
100
100
|
DEFAULT_PERCOLATION_MIN_SUPPORT,
|
|
@@ -122,7 +122,7 @@ import {
|
|
|
122
122
|
queryGraph,
|
|
123
123
|
rebuildGraph,
|
|
124
124
|
verifyGraph
|
|
125
|
-
} from "./index-
|
|
125
|
+
} from "./index-3gc2yk4k.js";
|
|
126
126
|
import {
|
|
127
127
|
GRAPH_LIMITS,
|
|
128
128
|
GraphAuthorityError,
|
|
@@ -297,8 +297,9 @@ import {
|
|
|
297
297
|
noteRevision,
|
|
298
298
|
removeNoteRelation,
|
|
299
299
|
updateNoteBody
|
|
300
|
-
} from "./index-
|
|
300
|
+
} from "./index-qt2zwza8.js";
|
|
301
301
|
import"./index-3rm7cz6h.js";
|
|
302
|
+
import"./index-qkesh4c6.js";
|
|
302
303
|
import {
|
|
303
304
|
MAX_ANALYZED_NOTES,
|
|
304
305
|
MAX_CONNECTION_OBSERVATIONS,
|
package/dist/portfolio.js
CHANGED
|
@@ -20,15 +20,15 @@ import {
|
|
|
20
20
|
selectAuthorizedVaults,
|
|
21
21
|
snapshotPortfolioRegistry,
|
|
22
22
|
validateResolvedPortfolioVaults
|
|
23
|
-
} from "./index-
|
|
24
|
-
import"./index-
|
|
23
|
+
} from "./index-hz2wyggk.js";
|
|
24
|
+
import"./index-mcpppwbq.js";
|
|
25
25
|
import"./index-adx6khj5.js";
|
|
26
26
|
import"./index-fp732bgg.js";
|
|
27
27
|
import"./index-4j3tt0c3.js";
|
|
28
28
|
import"./index-j70m75wd.js";
|
|
29
|
-
import"./index-
|
|
29
|
+
import"./index-e26mdwxz.js";
|
|
30
30
|
import"./index-bdwcjvr4.js";
|
|
31
|
-
import"./index-
|
|
31
|
+
import"./index-3gc2yk4k.js";
|
|
32
32
|
import"./index-pgtm2nhf.js";
|
|
33
33
|
import"./index-1gwbassd.js";
|
|
34
34
|
import"./index-ahyhryb8.js";
|
|
@@ -40,6 +40,7 @@ import"./index-06c9ctr6.js";
|
|
|
40
40
|
import"./index-5vwpzb5a.js";
|
|
41
41
|
import"./index-x3fthpsc.js";
|
|
42
42
|
import"./index-3rm7cz6h.js";
|
|
43
|
+
import"./index-qkesh4c6.js";
|
|
43
44
|
import {
|
|
44
45
|
MAX_DOCUMENT_ID_BYTES,
|
|
45
46
|
MAX_PORTFOLIO_NAME_BYTES,
|
package/dist/sdk.js
CHANGED
|
@@ -11,14 +11,14 @@ import {
|
|
|
11
11
|
packUntrustedSearchContext,
|
|
12
12
|
searchEvidenceRank,
|
|
13
13
|
validateKnowledgeBaseSearchHistory
|
|
14
|
-
} from "./index-
|
|
14
|
+
} from "./index-mcpppwbq.js";
|
|
15
15
|
import"./index-adx6khj5.js";
|
|
16
16
|
import"./index-fp732bgg.js";
|
|
17
17
|
import"./index-4j3tt0c3.js";
|
|
18
18
|
import"./index-j70m75wd.js";
|
|
19
|
-
import"./index-
|
|
19
|
+
import"./index-e26mdwxz.js";
|
|
20
20
|
import"./index-bdwcjvr4.js";
|
|
21
|
-
import"./index-
|
|
21
|
+
import"./index-3gc2yk4k.js";
|
|
22
22
|
import"./index-pgtm2nhf.js";
|
|
23
23
|
import"./index-1gwbassd.js";
|
|
24
24
|
import"./index-ahyhryb8.js";
|
|
@@ -29,6 +29,7 @@ import"./index-48pz4jpc.js";
|
|
|
29
29
|
import"./index-06c9ctr6.js";
|
|
30
30
|
import"./index-5vwpzb5a.js";
|
|
31
31
|
import"./index-3rm7cz6h.js";
|
|
32
|
+
import"./index-qkesh4c6.js";
|
|
32
33
|
import"./index-jvb7w0gg.js";
|
|
33
34
|
import"./index-z1w83f81.js";
|
|
34
35
|
export {
|