@opentf/afw-docs 0.0.0-stage → 0.30.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/build/blog-posts-plugin.js +141 -0
- package/build/docs-nav-plugin.js +196 -0
- package/build/feed.js +147 -0
- package/build/frontmatter.js +45 -0
- package/build/host.js +54 -0
- package/build/index.js +12 -0
- package/build/last-updated-plugin.js +99 -0
- package/build/last-updated.js +40 -0
- package/build/llms.js +176 -0
- package/build/reading-time.js +16 -0
- package/build/search.js +27 -0
- package/components/BlogLayout.jsx +90 -0
- package/components/Breadcrumbs.jsx +71 -0
- package/components/Callout.jsx +29 -0
- package/components/Card.jsx +21 -0
- package/components/Cards.jsx +11 -0
- package/components/CodeBlock.jsx +39 -0
- package/components/DocsLayout.jsx +106 -0
- package/components/Footer.jsx +25 -0
- package/components/LastUpdated.jsx +22 -0
- package/components/NavIcon.jsx +34 -0
- package/components/Navbar.jsx +64 -0
- package/components/NavbarLink.jsx +40 -0
- package/components/Pagination.jsx +49 -0
- package/components/PostBanner.jsx +25 -0
- package/components/PostCard.jsx +19 -0
- package/components/PostList.jsx +17 -0
- package/components/PostMeta.jsx +28 -0
- package/components/ReadingTime.jsx +6 -0
- package/components/Search.jsx +191 -0
- package/components/SearchTrigger.jsx +32 -0
- package/components/Sidebar.jsx +199 -0
- package/components/SidebarNode.jsx +143 -0
- package/components/SidebarToggle.jsx +47 -0
- package/components/Steps.jsx +14 -0
- package/components/Table.jsx +10 -0
- package/components/Tabs.jsx +77 -0
- package/components/ThemeToggle.jsx +164 -0
- package/components/Toc.jsx +79 -0
- package/components/Tooltip.jsx +19 -0
- package/components/format.js +21 -0
- package/components/post-dates.js +21 -0
- package/components/sidebar-collapse.js +38 -0
- package/config.js +77 -0
- package/index.js +67 -0
- package/nav-virtual.js +18 -0
- package/package.json +75 -4
- package/posts-virtual.js +17 -0
- package/search.js +285 -0
- package/theme/index.css +1749 -0
- package/types/config.d.ts +37 -0
- package/types/index.d.ts +66 -0
- package/types/nav.d.ts +9 -0
- package/types/posts.d.ts +17 -0
- package/types/search.d.ts +28 -0
- package/types/updated.d.ts +4 -0
- package/updated-virtual.js +10 -0
- package/README.md +0 -3
package/search.js
ADDED
|
@@ -0,0 +1,285 @@
|
|
|
1
|
+
// DOM-free reader for the AFW Search binary format.
|
|
2
|
+
/** @typedef {{ limit?: number, maxSectionsPerPage?: number, signal?: AbortSignal }} SearchOptions */
|
|
3
|
+
/** @typedef {{ url: string, title: string, text: string, section: string, excerpt: string, highlights: number[][], meta: Record<string, string>, anchors: Array<{ id: string, text: string, pos: number }>, score: number }} SearchResult */
|
|
4
|
+
const encoder = new TextEncoder();
|
|
5
|
+
// Rust's String ordering is UTF-8 byte ordering, not locale collation.
|
|
6
|
+
function compare(a, b) {
|
|
7
|
+
const x = encoder.encode(a), y = encoder.encode(b);
|
|
8
|
+
for (let i = 0; i < Math.min(x.length, y.length); i++) if (x[i] !== y[i]) return x[i] - y[i];
|
|
9
|
+
return x.length - y.length;
|
|
10
|
+
}
|
|
11
|
+
function cursor(bytes, position = 0) {
|
|
12
|
+
return {
|
|
13
|
+
position,
|
|
14
|
+
read() {
|
|
15
|
+
let value = 0;
|
|
16
|
+
for (let shift = 0; shift <= 49; shift += 7) {
|
|
17
|
+
if (this.position >= bytes.length) throw new Error("Truncated search index");
|
|
18
|
+
const byte = bytes[this.position++];
|
|
19
|
+
value += (byte & 127) * 2 ** shift;
|
|
20
|
+
if (!Number.isSafeInteger(value)) throw new Error("Invalid search integer");
|
|
21
|
+
if (!(byte & 128)) return value;
|
|
22
|
+
}
|
|
23
|
+
throw new Error("Invalid search integer");
|
|
24
|
+
},
|
|
25
|
+
};
|
|
26
|
+
}
|
|
27
|
+
function decodeChunk(bytes) {
|
|
28
|
+
if (new TextDecoder().decode(bytes.slice(0, 4)) !== "AFWI" || bytes[4] !== 1) throw new Error("Unsupported search term chunk");
|
|
29
|
+
const input = cursor(bytes, 5), count = input.read(), rows = [];
|
|
30
|
+
const decoder = new TextDecoder("utf-8", { fatal: true });
|
|
31
|
+
let previous = new Uint8Array();
|
|
32
|
+
for (let i = 0; i < count; i++) {
|
|
33
|
+
const prefix = input.read(), length = input.read();
|
|
34
|
+
if (prefix > previous.length || length > bytes.length - input.position) throw new Error("Invalid search term table");
|
|
35
|
+
const term = new Uint8Array(prefix + length);
|
|
36
|
+
term.set(previous.subarray(0, prefix));
|
|
37
|
+
term.set(bytes.subarray(input.position, input.position + length), prefix);
|
|
38
|
+
input.position += length;
|
|
39
|
+
rows.push([decoder.decode(term), input.read(), input.read()]);
|
|
40
|
+
previous = term;
|
|
41
|
+
}
|
|
42
|
+
const postings = new Map(), blob = input.position;
|
|
43
|
+
for (const [term, count, offset] of rows) {
|
|
44
|
+
const list = [], input = cursor(bytes, blob + offset);
|
|
45
|
+
let doc = 0;
|
|
46
|
+
for (let i = 0; i < count; i++) {
|
|
47
|
+
doc += input.read();
|
|
48
|
+
const tf = input.read(), positions = [];
|
|
49
|
+
let position = 0;
|
|
50
|
+
for (let j = 0; j < tf; j++) {
|
|
51
|
+
const packed = input.read();
|
|
52
|
+
position += Math.floor(packed / 8);
|
|
53
|
+
positions.push([position, packed % 8]);
|
|
54
|
+
}
|
|
55
|
+
list.push([doc, tf, positions]);
|
|
56
|
+
}
|
|
57
|
+
postings.set(term, list);
|
|
58
|
+
}
|
|
59
|
+
return postings;
|
|
60
|
+
}
|
|
61
|
+
function wordGroups(text) {
|
|
62
|
+
return (text.match(/[\p{L}\p{N}\p{M}_.-]+/gu) || []).map((word) => {
|
|
63
|
+
const normalized = word.normalize("NFKC"), whole = normalized.toLowerCase();
|
|
64
|
+
if (!/[\p{L}\p{N}]/u.test(whole)) return [];
|
|
65
|
+
const folded = whole.normalize("NFD").replace(/\p{M}/gu, "");
|
|
66
|
+
const parts = normalized.replace(/([a-z])([A-Z])/g, "$1 $2").split(/[_.\-\s]+/).filter(Boolean).map((part) => part.toLowerCase());
|
|
67
|
+
return [...new Set([whole, folded, ...parts])];
|
|
68
|
+
}).filter((group) => group.length);
|
|
69
|
+
}
|
|
70
|
+
export function tokenize(text) {
|
|
71
|
+
return [...new Set(wordGroups(text).flat())];
|
|
72
|
+
}
|
|
73
|
+
// Build a prose-only preview within the selected section. Original offsets remain
|
|
74
|
+
// untouched in the index; omission ranges are mapped only for card rendering.
|
|
75
|
+
function excerptFor(fragment, groups, match) {
|
|
76
|
+
const { text, anchors = [], excerptOmit = [] } = fragment;
|
|
77
|
+
const sectionStart = match?.anchor?.pos ?? 0;
|
|
78
|
+
const sectionEnd = anchors.filter((anchor) => anchor.pos > sectionStart).sort((a, b) => a.pos - b.pos)[0]?.pos ?? text.length;
|
|
79
|
+
let source = sectionStart, clean = "", position = 0;
|
|
80
|
+
const target = match?.position ?? sectionStart;
|
|
81
|
+
const append = (end) => {
|
|
82
|
+
if (target >= source) position = clean.length + Math.min(target - source, end - source);
|
|
83
|
+
clean += text.slice(source, end);
|
|
84
|
+
source = end;
|
|
85
|
+
};
|
|
86
|
+
for (const [from, to] of [...excerptOmit].sort((a, b) => a[0] - b[0])) {
|
|
87
|
+
if (to <= source || from >= sectionEnd) continue;
|
|
88
|
+
append(Math.max(source, from));
|
|
89
|
+
source = Math.min(to, sectionEnd);
|
|
90
|
+
}
|
|
91
|
+
append(sectionEnd);
|
|
92
|
+
position = clean.slice(0, position).replace(/\s+/g, " ").length;
|
|
93
|
+
clean = clean.replace(/\s+/g, " ");
|
|
94
|
+
const start = Math.max(0, clean.lastIndexOf(" ", Math.max(0, position - 70)) + 1);
|
|
95
|
+
let end = Math.min(clean.length, start + 220);
|
|
96
|
+
const boundary = clean.indexOf(" ", end);
|
|
97
|
+
if (boundary >= 0) end = boundary;
|
|
98
|
+
const excerpt = clean.slice(start, end).trim(), highlights = [];
|
|
99
|
+
// Highlight complete visible words; normalization handles accents and identifiers
|
|
100
|
+
// without using normalized string offsets against the original display text.
|
|
101
|
+
for (const word of excerpt.matchAll(/[\p{L}\p{N}\p{M}_.-]+/gu)) {
|
|
102
|
+
const variants = wordGroups(word[0])[0] || [];
|
|
103
|
+
if (groups.some((group) => variants.some((variant) => variant.startsWith(group[0]) ||
|
|
104
|
+
variant.startsWith(group[0].normalize("NFD").replace(/\p{M}/gu, ""))))) {
|
|
105
|
+
highlights.push([word.index, word.index + word[0].length]);
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
return { excerpt, highlights };
|
|
109
|
+
}
|
|
110
|
+
// Render highlighted excerpts as text nodes and <mark>, never injected HTML.
|
|
111
|
+
export function excerptParts(excerpt, highlights = []) {
|
|
112
|
+
const parts = [];
|
|
113
|
+
let position = 0;
|
|
114
|
+
for (const [start, end] of [...highlights].sort((a, b) => a[0] - b[0])) {
|
|
115
|
+
if (start < position || end <= start || end > excerpt.length) continue;
|
|
116
|
+
if (start > position) parts.push({ text: excerpt.slice(position, start), match: false });
|
|
117
|
+
parts.push({ text: excerpt.slice(start, end), match: true });
|
|
118
|
+
position = end;
|
|
119
|
+
}
|
|
120
|
+
if (position < excerpt.length) parts.push({ text: excerpt.slice(position), match: false });
|
|
121
|
+
return parts;
|
|
122
|
+
}
|
|
123
|
+
// Posting offsets already account for Unicode normalization and identifier variants.
|
|
124
|
+
// Rank distinct sections, keeping the link and excerpt at the same location.
|
|
125
|
+
// When query words only match across separate sections, keep the best page card
|
|
126
|
+
// rather than presenting every incomplete section as a complete query match.
|
|
127
|
+
function sectionMatches(fragment, groups, fieldWeights) {
|
|
128
|
+
const anchors = (fragment.anchors || []).filter((anchor) => anchor.id).sort((a, b) => a.pos - b.pos);
|
|
129
|
+
const sections = new Map();
|
|
130
|
+
groups.forEach((positions, group) => {
|
|
131
|
+
for (const [position, field] of positions) {
|
|
132
|
+
const anchor = anchors.findLast((anchor) => anchor.pos <= position);
|
|
133
|
+
const key = anchor?.id || "";
|
|
134
|
+
if (!sections.has(key)) sections.set(key, { anchor, weights: new Map(), position, weight: 0 });
|
|
135
|
+
const section = sections.get(key), weight = fieldWeights[field];
|
|
136
|
+
section.weights.set(group, Math.max(section.weights.get(group) || 0, weight));
|
|
137
|
+
if (weight > section.weight || (weight === section.weight && position < section.position)) {
|
|
138
|
+
section.position = position;
|
|
139
|
+
section.weight = weight;
|
|
140
|
+
}
|
|
141
|
+
}
|
|
142
|
+
});
|
|
143
|
+
const ranked = [...sections.values()].sort((a, b) => b.weights.size - a.weights.size ||
|
|
144
|
+
[...b.weights.values()].reduce((sum, weight) => sum + weight, 0) - [...a.weights.values()].reduce((sum, weight) => sum + weight, 0) ||
|
|
145
|
+
a.position - b.position);
|
|
146
|
+
return ranked[0]?.weights.size === groups.length ? ranked.filter(section => section.weights.size === groups.length) : ranked.slice(0, 1);
|
|
147
|
+
}
|
|
148
|
+
/**
|
|
149
|
+
* Create a reusable client for one static `_search/` directory.
|
|
150
|
+
* `limit` bounds cards; `total` counts matching pages. Each page's best section
|
|
151
|
+
* comes first, then more matches, up to `maxSectionsPerPage` (default 3).
|
|
152
|
+
* @param {{ base?: string }} options
|
|
153
|
+
* @returns {{ preload(): Promise<void>, query(query: string, options?: SearchOptions): Promise<{ results: SearchResult[], total: number, partial: boolean }> }}
|
|
154
|
+
*/
|
|
155
|
+
export function createSearch({ base = "/_search/" } = {}) {
|
|
156
|
+
const root = base.replace(/\/$/, "");
|
|
157
|
+
let initialized;
|
|
158
|
+
const chunkCache = new Map();
|
|
159
|
+
const throwIfAborted = (signal) => {
|
|
160
|
+
if (signal?.aborted) throw new DOMException("Search query aborted", "AbortError");
|
|
161
|
+
};
|
|
162
|
+
const fetchIndex = async (path, options) => {
|
|
163
|
+
const response = await fetch(`${root}/${path}`, options);
|
|
164
|
+
if (!response.ok) {
|
|
165
|
+
if (response.status === 404 && path !== "manifest.json") { initialized = undefined; chunkCache.clear(); }
|
|
166
|
+
throw new Error(`Search index unavailable (${response.status})`);
|
|
167
|
+
}
|
|
168
|
+
return response;
|
|
169
|
+
};
|
|
170
|
+
const init = () => (initialized ||= (async () => {
|
|
171
|
+
const manifest = await (await fetchIndex("manifest.json", { cache: "no-cache" })).json();
|
|
172
|
+
if (manifest.v !== undefined && manifest.v !== 1) throw new Error("Unsupported search index version");
|
|
173
|
+
if (!Array.isArray(manifest.chunks) || !manifest.docsFile) throw new Error("Invalid search manifest");
|
|
174
|
+
for (let i = 1; i < manifest.chunks.length; i++) {
|
|
175
|
+
if (compare(manifest.chunks[i - 1].first, manifest.chunks[i].first) >= 0) throw new Error("Unsorted search shards");
|
|
176
|
+
}
|
|
177
|
+
const bytes = new Uint8Array(await (await fetchIndex(manifest.docsFile)).arrayBuffer());
|
|
178
|
+
const input = cursor(bytes), lengths = [];
|
|
179
|
+
while (input.position < bytes.length) lengths.push(input.read());
|
|
180
|
+
if (manifest.docs !== undefined && manifest.docs !== lengths.length) throw new Error("Invalid search document table");
|
|
181
|
+
return { manifest, lengths };
|
|
182
|
+
})().catch((error) => { initialized = undefined; throw error; }));
|
|
183
|
+
const loadChunk = (file) => {
|
|
184
|
+
if (!chunkCache.has(file)) chunkCache.set(file, fetchIndex(file).then((r) => r.arrayBuffer())
|
|
185
|
+
.then((bytes) => decodeChunk(new Uint8Array(bytes)))
|
|
186
|
+
.catch((error) => { chunkCache.delete(file); throw error; }));
|
|
187
|
+
return chunkCache.get(file);
|
|
188
|
+
};
|
|
189
|
+
const findChunk = (chunks, term) => {
|
|
190
|
+
let low = 0, high = chunks.length - 1, answer = 0;
|
|
191
|
+
while (low <= high) {
|
|
192
|
+
const mid = (low + high) >>> 1;
|
|
193
|
+
if (compare(chunks[mid].first, term) <= 0) { answer = mid; low = mid + 1; }
|
|
194
|
+
else high = mid - 1;
|
|
195
|
+
}
|
|
196
|
+
return answer;
|
|
197
|
+
};
|
|
198
|
+
return {
|
|
199
|
+
async preload() { await init(); },
|
|
200
|
+
async query(query, { limit = 10, maxSectionsPerPage = 3, signal } = {}) {
|
|
201
|
+
throwIfAborted(signal);
|
|
202
|
+
const groups = wordGroups(query);
|
|
203
|
+
if (!groups.length) return { results: [], total: 0, partial: false };
|
|
204
|
+
const state = await init(), { chunks } = state.manifest;
|
|
205
|
+
if (!chunks.length && !state.lengths.length) return { results: [], total: 0, partial: false };
|
|
206
|
+
if (!chunks.length) throw new Error("Search index has no term chunks");
|
|
207
|
+
const selected = new Set();
|
|
208
|
+
groups.forEach((group, groupIndex) => group.forEach((word) => {
|
|
209
|
+
const index = findChunk(chunks, word);
|
|
210
|
+
selected.add(chunks[index].file);
|
|
211
|
+
// Prefixes can span more than two shards.
|
|
212
|
+
if (groupIndex === groups.length - 1) {
|
|
213
|
+
for (let i = index + 1; i < chunks.length && chunks[i].first.startsWith(word); i++) selected.add(chunks[i].file);
|
|
214
|
+
}
|
|
215
|
+
}));
|
|
216
|
+
const postings = new Map();
|
|
217
|
+
for (const shard of await Promise.all([...selected].map(loadChunk))) for (const [term, list] of shard) postings.set(term, list);
|
|
218
|
+
throwIfAborted(signal);
|
|
219
|
+
const scores = new Map(), matches = new Map(), locations = new Map(), matchedPositions = new Map();
|
|
220
|
+
const totalDocs = state.lengths.length, avgdl = state.manifest.avgdl || 1;
|
|
221
|
+
const fieldWeights = [8, 5, 4, 2.5, 3, 2, 1, 0.5];
|
|
222
|
+
groups.forEach((group, groupIndex) => {
|
|
223
|
+
const best = new Map();
|
|
224
|
+
const candidates = [...postings].filter(([term]) => group.some((word) => term === word ||
|
|
225
|
+
(groupIndex === groups.length - 1 && term.startsWith(word))))
|
|
226
|
+
.sort((a, b) => Number(group.includes(b[0])) - Number(group.includes(a[0])) || a[1].length - b[1].length || compare(a[0], b[0])).slice(0, 32);
|
|
227
|
+
for (const [term, list] of candidates) {
|
|
228
|
+
const idf = Math.log(1 + (totalDocs - list.length + 0.5) / (list.length + 0.5));
|
|
229
|
+
for (const [doc, tf, positions] of list) {
|
|
230
|
+
if (doc >= totalDocs || !tf) throw new Error("Invalid search posting");
|
|
231
|
+
const weight = Math.max(...positions.map(([, field]) => fieldWeights[field]));
|
|
232
|
+
const exact = term === group[0] ? 4 : group.includes(term) ? 0.75 : 0.5;
|
|
233
|
+
const score = exact * weight * idf * (tf * 2.2) / (tf + 1.2 * (0.25 + 0.75 * state.lengths[doc] / avgdl));
|
|
234
|
+
const previous = best.get(doc);
|
|
235
|
+
if (!previous) best.set(doc, { score, positions: [...positions] });
|
|
236
|
+
else {
|
|
237
|
+
previous.score = Math.max(previous.score, score);
|
|
238
|
+
// Prefix and identifier alternatives can match different sections.
|
|
239
|
+
// Keep every location while counting this query word only once.
|
|
240
|
+
previous.positions.push(...positions);
|
|
241
|
+
}
|
|
242
|
+
}
|
|
243
|
+
}
|
|
244
|
+
for (const [doc, { score, positions }] of best) {
|
|
245
|
+
positions.sort((a, b) => a[0] - b[0] || a[1] - b[1]);
|
|
246
|
+
scores.set(doc, (scores.get(doc) || 0) + score);
|
|
247
|
+
matches.set(doc, (matches.get(doc) || 0) + 1);
|
|
248
|
+
const perDoc = locations.get(doc) || [];
|
|
249
|
+
perDoc.push(positions.map(([position]) => position)); locations.set(doc, perDoc);
|
|
250
|
+
const perDocMatches = matchedPositions.get(doc) || [];
|
|
251
|
+
perDocMatches.push(positions); matchedPositions.set(doc, perDocMatches);
|
|
252
|
+
}
|
|
253
|
+
});
|
|
254
|
+
for (const [doc, groups] of locations) if (groups.length >= 2) {
|
|
255
|
+
const points = groups.map((group) => group[0]).sort((a, b) => a - b);
|
|
256
|
+
scores.set(doc, scores.get(doc) + 1 / (1 + points.at(-1) - points[0]));
|
|
257
|
+
}
|
|
258
|
+
let ranked = [...scores].filter(([doc]) => matches.get(doc) === groups.length);
|
|
259
|
+
const partial = ranked.length === 0 && scores.size > 0;
|
|
260
|
+
if (partial) ranked = [...scores];
|
|
261
|
+
const resultLimit = Math.max(0, Math.floor(limit));
|
|
262
|
+
const sectionLimit = Number.isFinite(maxSectionsPerPage) ? Math.max(1, Math.floor(maxSectionsPerPage)) : 3;
|
|
263
|
+
const top = ranked.sort((a, b) => b[1] - a[1] || a[0] - b[0]).slice(0, resultLimit);
|
|
264
|
+
throwIfAborted(signal);
|
|
265
|
+
const pages = await Promise.all(top.map(async ([doc, score]) => {
|
|
266
|
+
const fragment = await (await fetchIndex(`${state.manifest.fragmentsDir || "f"}/${doc}.json`, { signal })).json();
|
|
267
|
+
return sectionMatches(fragment, matchedPositions.get(doc) || [], fieldWeights).slice(0, sectionLimit).map(match => {
|
|
268
|
+
const url = match.anchor ? `${fragment.url.split("#")[0]}#${encodeURIComponent(match.anchor.id)}` : fragment.url;
|
|
269
|
+
return { ...fragment, url, section: match.anchor?.text || "", ...excerptFor(fragment, groups, match), score };
|
|
270
|
+
});
|
|
271
|
+
}));
|
|
272
|
+
throwIfAborted(signal);
|
|
273
|
+
// Give each relevant page its best card before adding more sections.
|
|
274
|
+
const results = [];
|
|
275
|
+
for (let section = 0; section < Math.min(sectionLimit, resultLimit); section++) {
|
|
276
|
+
for (const page of pages) {
|
|
277
|
+
if (page[section]) results.push(page[section]);
|
|
278
|
+
if (results.length === resultLimit) break;
|
|
279
|
+
}
|
|
280
|
+
if (results.length === resultLimit) break;
|
|
281
|
+
}
|
|
282
|
+
return { results, total: ranked.length, partial };
|
|
283
|
+
},
|
|
284
|
+
};
|
|
285
|
+
}
|