@hraness/kb 0.17.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (200) hide show
  1. package/DISCLOSURE +43 -0
  2. package/LICENSE +21 -0
  3. package/README.md +545 -0
  4. package/dist/agent-context.js +35 -0
  5. package/dist/agent-guide-audit.js +18 -0
  6. package/dist/attachments.js +19 -0
  7. package/dist/authoring.js +33 -0
  8. package/dist/benchmark.js +19 -0
  9. package/dist/capture.js +49 -0
  10. package/dist/cli.js +3646 -0
  11. package/dist/clip/acquire.js +57 -0
  12. package/dist/clip/args.js +21 -0
  13. package/dist/clip/bounded-byte-buffer.js +9 -0
  14. package/dist/clip/browser-profiles.js +17 -0
  15. package/dist/clip/bundle-reader.js +11 -0
  16. package/dist/clip/cli.js +27 -0
  17. package/dist/clip/cookies.js +21 -0
  18. package/dist/clip/defuddle-worker.js +84 -0
  19. package/dist/clip/doctor.js +32 -0
  20. package/dist/clip/jobs.js +784 -0
  21. package/dist/clip/network-proxy.js +9 -0
  22. package/dist/clip/network.js +28 -0
  23. package/dist/clip/persist.js +30 -0
  24. package/dist/clip/refresh.js +11 -0
  25. package/dist/clip/terminal.js +11 -0
  26. package/dist/evaluation-builder.js +12238 -0
  27. package/dist/evaluation-kb.js +26 -0
  28. package/dist/evaluation.js +35 -0
  29. package/dist/git.js +39 -0
  30. package/dist/graph.js +39 -0
  31. package/dist/index-01jj6rbv.js +1103 -0
  32. package/dist/index-06c9ctr6.js +738 -0
  33. package/dist/index-0kavxzqj.js +2480 -0
  34. package/dist/index-1gwbassd.js +743 -0
  35. package/dist/index-1n418kb9.js +827 -0
  36. package/dist/index-1vrd1rmn.js +56 -0
  37. package/dist/index-1xxnjn0d.js +171 -0
  38. package/dist/index-3rm7cz6h.js +373 -0
  39. package/dist/index-3v2z4f0q.js +328 -0
  40. package/dist/index-48pz4jpc.js +392 -0
  41. package/dist/index-4j3tt0c3.js +234 -0
  42. package/dist/index-5n05se68.js +307 -0
  43. package/dist/index-5vwpzb5a.js +764 -0
  44. package/dist/index-6g2pv9d2.js +370 -0
  45. package/dist/index-7fzc32gf.js +712 -0
  46. package/dist/index-84x0vjjp.js +432 -0
  47. package/dist/index-adx6khj5.js +488 -0
  48. package/dist/index-b88v3vtm.js +805 -0
  49. package/dist/index-cv6fh7z5.js +462 -0
  50. package/dist/index-cxfrakt7.js +1067 -0
  51. package/dist/index-d13v9ckt.js +150 -0
  52. package/dist/index-dycjbhw2.js +1943 -0
  53. package/dist/index-dyqwejk5.js +531 -0
  54. package/dist/index-e5fbsywq.js +658 -0
  55. package/dist/index-f984hw45.js +927 -0
  56. package/dist/index-gh719d91.js +63 -0
  57. package/dist/index-hgve9rh2.js +2046 -0
  58. package/dist/index-hya40gb2.js +474 -0
  59. package/dist/index-j4zgmzjr.js +152 -0
  60. package/dist/index-jsmvyyvf.js +1166 -0
  61. package/dist/index-kh0pwvdd.js +1974 -0
  62. package/dist/index-mqx4nd6v.js +119 -0
  63. package/dist/index-mxxxytys.js +1254 -0
  64. package/dist/index-n5dd7r0v.js +711 -0
  65. package/dist/index-npg9z1a4.js +431 -0
  66. package/dist/index-pj501bh1.js +140 -0
  67. package/dist/index-qry4vhxk.js +1041 -0
  68. package/dist/index-s2gw5aw9.js +199 -0
  69. package/dist/index-vvdj6kca.js +70 -0
  70. package/dist/index-w2zc0vwa.js +571 -0
  71. package/dist/index-x3fthpsc.js +585 -0
  72. package/dist/index-zr53sf63.js +44 -0
  73. package/dist/index-zxdy5pby.js +2728 -0
  74. package/dist/index-zzhgcwyt.js +966 -0
  75. package/dist/index.js +453 -0
  76. package/dist/navigation.js +13 -0
  77. package/dist/pdf.js +85 -0
  78. package/dist/percolate.js +24 -0
  79. package/dist/portfolio.js +77 -0
  80. package/dist/query.js +29 -0
  81. package/dist/repository-memory.js +60 -0
  82. package/dist/sdk.js +36 -0
  83. package/dist/search-rules.js +34 -0
  84. package/dist/search.js +22 -0
  85. package/dist/semantic.js +39 -0
  86. package/dist/source-inbox.js +17 -0
  87. package/dist/untrusted-content.js +32 -0
  88. package/dist/url-intelligence.js +74 -0
  89. package/dist/workflow.js +23 -0
  90. package/dist/workflows/decision-context.js +20 -0
  91. package/dist/workflows/explain-change.js +8 -0
  92. package/dist/workflows/index.js +28 -0
  93. package/dist/workflows/plan-radar.js +10 -0
  94. package/package.json +409 -0
  95. package/skills/kb/AGENTS.md +19 -0
  96. package/skills/kb/SKILL.md +82 -0
  97. package/skills/kb/agents/openai.yaml +4 -0
  98. package/skills/kb/references/pdf-review.md +51 -0
  99. package/skills/kb/references/percolate.md +144 -0
  100. package/skills/kb/references/plan-structure.md +105 -0
  101. package/skills/kb/references/plan.md +115 -0
  102. package/skills/kb/references/query.md +231 -0
  103. package/skills/kb/references/refresh.md +133 -0
  104. package/skills/kb/references/save-pdf.md +125 -0
  105. package/skills/kb/references/save-url.md +178 -0
  106. package/skills/kb/references/url-authentication.md +75 -0
  107. package/skills/kb/references/url-platforms.md +25 -0
  108. package/src/agent-context.ts +1241 -0
  109. package/src/agent-guide-audit.ts +750 -0
  110. package/src/attachments.ts +723 -0
  111. package/src/authoring.ts +1725 -0
  112. package/src/benchmark.ts +362 -0
  113. package/src/capture.ts +68 -0
  114. package/src/cli.ts +3660 -0
  115. package/src/clip/acquire.ts +1273 -0
  116. package/src/clip/archive-today.ts +402 -0
  117. package/src/clip/args.ts +423 -0
  118. package/src/clip/assets.ts +262 -0
  119. package/src/clip/bounded-byte-buffer.ts +75 -0
  120. package/src/clip/browser-profiles.ts +372 -0
  121. package/src/clip/bundle-reader.ts +608 -0
  122. package/src/clip/capture.ts +1054 -0
  123. package/src/clip/cli.ts +181 -0
  124. package/src/clip/cookies.ts +512 -0
  125. package/src/clip/doctor.ts +1110 -0
  126. package/src/clip/extract.ts +868 -0
  127. package/src/clip/jobs.ts +959 -0
  128. package/src/clip/lib.ts +861 -0
  129. package/src/clip/media.ts +1474 -0
  130. package/src/clip/metadata-search-tool/Cargo.lock +2498 -0
  131. package/src/clip/metadata-search-tool/Cargo.toml +18 -0
  132. package/src/clip/metadata-search-tool/runner.ts +685 -0
  133. package/src/clip/metadata-search-tool/src/main.rs +389 -0
  134. package/src/clip/metadata-search.ts +687 -0
  135. package/src/clip/network-proxy.ts +627 -0
  136. package/src/clip/network.ts +950 -0
  137. package/src/clip/package-root.ts +54 -0
  138. package/src/clip/persist.ts +1472 -0
  139. package/src/clip/platforms.ts +1824 -0
  140. package/src/clip/refresh.ts +197 -0
  141. package/src/clip/structured.ts +432 -0
  142. package/src/clip/terminal.ts +197 -0
  143. package/src/clip/url-intelligence.ts +581 -0
  144. package/src/clip/url-metadata-backfill.ts +616 -0
  145. package/src/clip/url-metadata-cli.ts +251 -0
  146. package/src/clip/url-metadata.ts +1397 -0
  147. package/src/evaluation-analysis-v2.ts +1440 -0
  148. package/src/evaluation-builder.ts +2539 -0
  149. package/src/evaluation-corpus-authoring.ts +2052 -0
  150. package/src/evaluation-evidence.ts +1147 -0
  151. package/src/evaluation-execution-v2.ts +426 -0
  152. package/src/evaluation-implementation.ts +131 -0
  153. package/src/evaluation-kb-closure.ts +1430 -0
  154. package/src/evaluation-kb-runner-v2.ts +735 -0
  155. package/src/evaluation-kb-v2.ts +2567 -0
  156. package/src/evaluation-kb.ts +1018 -0
  157. package/src/evaluation-measurement-v2.ts +1462 -0
  158. package/src/evaluation-packing-v2.ts +452 -0
  159. package/src/evaluation-redaction.ts +28 -0
  160. package/src/evaluation-v2.ts +3857 -0
  161. package/src/evaluation.ts +1220 -0
  162. package/src/git.ts +1276 -0
  163. package/src/graph.ts +1487 -0
  164. package/src/index.ts +20 -0
  165. package/src/init.ts +123 -0
  166. package/src/navigation.ts +277 -0
  167. package/src/note-lock.ts +529 -0
  168. package/src/pdf/args.ts +202 -0
  169. package/src/pdf/capture.ts +298 -0
  170. package/src/pdf/cli.ts +159 -0
  171. package/src/pdf/extract.ts +303 -0
  172. package/src/pdf/layout.ts +618 -0
  173. package/src/pdf/markdown.ts +154 -0
  174. package/src/pdf/model.ts +234 -0
  175. package/src/pdf/ocr.ts +190 -0
  176. package/src/pdf/persist.ts +314 -0
  177. package/src/pdf/source.ts +135 -0
  178. package/src/pdf/tools.ts +146 -0
  179. package/src/pdf.ts +92 -0
  180. package/src/percolate.ts +877 -0
  181. package/src/portfolio-audit.ts +574 -0
  182. package/src/portfolio-identity.ts +162 -0
  183. package/src/portfolio-registry.ts +473 -0
  184. package/src/portfolio.ts +765 -0
  185. package/src/query.ts +646 -0
  186. package/src/repository-memory.ts +1150 -0
  187. package/src/sdk.ts +1452 -0
  188. package/src/search-rules.ts +712 -0
  189. package/src/search.ts +710 -0
  190. package/src/semantic-runtime.ts +1139 -0
  191. package/src/semantic.ts +2480 -0
  192. package/src/source-inbox.ts +232 -0
  193. package/src/untrusted-content.ts +401 -0
  194. package/src/url-intelligence.ts +104 -0
  195. package/src/vault.ts +636 -0
  196. package/src/workflow.ts +653 -0
  197. package/src/workflows/decision-context.ts +81 -0
  198. package/src/workflows/explain-change.ts +73 -0
  199. package/src/workflows/index.ts +3 -0
  200. package/src/workflows/plan-radar.ts +101 -0
@@ -0,0 +1,581 @@
1
+ import { isPrivateHostname } from "./network.js";
2
+
3
+ export const MAX_METADATA_SEARCH_RESULTS = 20;
4
+ export const MAX_METADATA_SEARCH_ENGINES = 8;
5
+ export const MAX_METADATA_SEARCH_QUERY_UTF8_BYTES = 4 * 1_024;
6
+ export const MAX_METADATA_SEARCH_TITLE_UTF8_BYTES = 2 * 1_024;
7
+ export const MAX_METADATA_SEARCH_SNIPPET_UTF8_BYTES = 8 * 1_024;
8
+ export const MAX_URL_INTELLIGENCE_URL_UTF8_BYTES = 16 * 1_024;
9
+ export const MAX_METADATA_SEARCH_TEXT_UTF8_BYTES = 512 * 1_024;
10
+ /** Fetch adapters should reject a response body above this limit before JSON parsing. */
11
+ export const MAX_METADATA_SEARCH_RESPONSE_BYTES = 2 * 1_024 * 1_024;
12
+
13
+ export const MAX_ARCHIVE_TIMEMAP_UTF8_BYTES = 512 * 1_024;
14
+ export const MAX_ARCHIVE_TIMEMAP_ENTRIES = 512;
15
+ export const MAX_ARCHIVE_TIMEMAP_PARAMETERS_PER_ENTRY = 16;
16
+
17
+ export const ARCHIVE_TODAY_HOSTS = Object.freeze([
18
+ "archive.today",
19
+ "archive.is",
20
+ "archive.ph",
21
+ "archive.fo",
22
+ "archive.li",
23
+ "archive.md",
24
+ "archive.vn",
25
+ ] as const);
26
+
27
+ export type ArchiveTodayHost = (typeof ARCHIVE_TODAY_HOSTS)[number];
28
+ export type MetadataSearchEngineStatus = "complete" | "partial" | "unavailable";
29
+
30
+ export type MetadataSearchResult = {
31
+ readonly title: string;
32
+ readonly url: string;
33
+ readonly snippet: string | null;
34
+ readonly engines: readonly string[];
35
+ readonly score: number;
36
+ };
37
+
38
+ export type MetadataSearchResponse = {
39
+ readonly query: string;
40
+ readonly results: readonly MetadataSearchResult[];
41
+ readonly enginesQueried: readonly string[];
42
+ readonly enginesFailed: readonly string[];
43
+ readonly engineStatus: MetadataSearchEngineStatus;
44
+ };
45
+
46
+ export type RankMetadataSearchOptions = {
47
+ readonly targetUrl?: string | URL;
48
+ readonly limit?: number;
49
+ };
50
+
51
+ export type RankedMetadataSearchResult = MetadataSearchResult & {
52
+ readonly rank: number;
53
+ readonly sourceIdentity: string;
54
+ readonly exactTarget: boolean;
55
+ };
56
+
57
+ export type ArchiveTodayMemento = {
58
+ readonly url: string;
59
+ readonly archiveHost: ArchiveTodayHost;
60
+ readonly timestamp: string;
61
+ readonly capturedAt: string;
62
+ readonly originalUrl: string;
63
+ };
64
+
65
+ export type ArchiveTodayTimeMap = {
66
+ readonly originalUrl: string;
67
+ readonly mementos: readonly ArchiveTodayMemento[];
68
+ readonly newest: ArchiveTodayMemento | null;
69
+ };
70
+
71
+ export type ParseArchiveTodayMementoUrlOptions = {
72
+ readonly originalUrl: string | URL;
73
+ readonly now: Date;
74
+ };
75
+
76
+ export type ParseArchiveTodayTimeMapOptions = ParseArchiveTodayMementoUrlOptions;
77
+
78
+ const archiveTodayHosts = new Set<string>(ARCHIVE_TODAY_HOSTS);
79
+ const tokenCharacter = /^[!#$%&'*+.^_`|~0-9A-Za-z-]$/u;
80
+ const engineName = /^[a-z0-9][a-z0-9._-]{0,127}$/u;
81
+
82
+ type TextBudget = { bytes: number };
83
+ type ParsedLink = {
84
+ readonly target: string;
85
+ readonly parameters: ReadonlyMap<string, string>;
86
+ };
87
+
88
+ function fail(label: string, message: string): never {
89
+ throw new TypeError(`${label} ${message}`);
90
+ }
91
+
92
+ function isUnsafeControlCodePoint(
93
+ codePoint: number,
94
+ rejectSurrogates = false,
95
+ allowLinkWhitespace = false,
96
+ ): boolean {
97
+ return (codePoint <= 0x1f
98
+ && !(allowLinkWhitespace && (codePoint === 0x09 || codePoint === 0x0a || codePoint === 0x0d)))
99
+ || (codePoint >= 0x7f && codePoint <= 0x9f)
100
+ || codePoint === 0x061c
101
+ || codePoint === 0x200e
102
+ || codePoint === 0x200f
103
+ || (codePoint >= 0x202a && codePoint <= 0x202e)
104
+ || (codePoint >= 0x2066 && codePoint <= 0x2069)
105
+ || (rejectSurrogates && codePoint >= 0xd800 && codePoint <= 0xdfff);
106
+ }
107
+
108
+ function hasUnsafeControls(
109
+ value: string,
110
+ rejectSurrogates = false,
111
+ allowLinkWhitespace = false,
112
+ ): boolean {
113
+ return Array.from(value).some((character) =>
114
+ isUnsafeControlCodePoint(character.codePointAt(0) as number, rejectSurrogates, allowLinkWhitespace));
115
+ }
116
+
117
+ function record(value: unknown, label: string): Readonly<Record<string, unknown>> {
118
+ if (value === null || typeof value !== "object" || Array.isArray(value)) {
119
+ return fail(label, "must be an object.");
120
+ }
121
+ return value as Readonly<Record<string, unknown>>;
122
+ }
123
+
124
+ function strictKeys(value: Readonly<Record<string, unknown>>, allowed: readonly string[], label: string): void {
125
+ const allowedKeys = new Set(allowed);
126
+ const unknown = Object.keys(value).filter((key) => !allowedKeys.has(key)).toSorted();
127
+ if (unknown.length > 0) fail(label, `has unknown fields: ${unknown.join(", ")}.`);
128
+ }
129
+
130
+ function boundedString(
131
+ value: unknown,
132
+ label: string,
133
+ maximumBytes: number,
134
+ budget?: TextBudget,
135
+ ): string {
136
+ if (typeof value !== "string") return fail(label, "must be a string.");
137
+ const bytes = Buffer.byteLength(value, "utf8");
138
+ if (bytes > maximumBytes) fail(label, `must be at most ${maximumBytes} UTF-8 bytes.`);
139
+ if (budget !== undefined) {
140
+ budget.bytes += bytes;
141
+ if (budget.bytes > MAX_METADATA_SEARCH_TEXT_UTF8_BYTES) {
142
+ fail("Metadata search response", `may contain at most ${MAX_METADATA_SEARCH_TEXT_UTF8_BYTES} UTF-8 bytes of text.`);
143
+ }
144
+ }
145
+ return value;
146
+ }
147
+
148
+ function cleanDisplayText(value: unknown, label: string, maximumBytes: number, budget: TextBudget): string {
149
+ const text = boundedString(value, label, maximumBytes, budget);
150
+ return Array.from(text, (character) =>
151
+ isUnsafeControlCodePoint(character.codePointAt(0) as number) ? " " : character)
152
+ .join("")
153
+ .replace(/ +/gu, " ")
154
+ .trim();
155
+ }
156
+
157
+ function checkedEngine(value: unknown, label: string, budget: TextBudget): string {
158
+ const engine = boundedString(value, label, 128, budget);
159
+ if (!engineName.test(engine)) fail(label, "must be a lowercase engine identifier.");
160
+ return engine;
161
+ }
162
+
163
+ function uniqueEngineArray(value: unknown, label: string, budget: TextBudget, allowEmpty: boolean): readonly string[] {
164
+ if (!Array.isArray(value) || value.length > MAX_METADATA_SEARCH_ENGINES || (!allowEmpty && value.length === 0)) {
165
+ return fail(label, `must be ${allowEmpty ? "an" : "a non-empty"} array with at most ${MAX_METADATA_SEARCH_ENGINES} entries.`);
166
+ }
167
+ const engines = value.map((entry, index) => checkedEngine(entry, `${label}[${index}]`, budget));
168
+ if (new Set(engines).size !== engines.length) fail(label, "must not contain duplicates.");
169
+ return Object.freeze(engines);
170
+ }
171
+
172
+ function publicHttpUrl(value: string | URL): URL | null {
173
+ const raw = value instanceof URL ? value.href : value;
174
+ if (
175
+ typeof raw !== "string"
176
+ || raw === ""
177
+ || raw !== raw.trim()
178
+ || hasUnsafeControls(raw)
179
+ || Buffer.byteLength(raw, "utf8") > MAX_URL_INTELLIGENCE_URL_UTF8_BYTES
180
+ ) return null;
181
+ let parsed: URL;
182
+ try {
183
+ parsed = new URL(raw);
184
+ } catch {
185
+ return null;
186
+ }
187
+ if (
188
+ (parsed.protocol !== "http:" && parsed.protocol !== "https:")
189
+ || parsed.username !== ""
190
+ || parsed.password !== ""
191
+ || parsed.hostname === ""
192
+ || isPrivateHostname(parsed.hostname)
193
+ ) return null;
194
+ return parsed;
195
+ }
196
+
197
+ /**
198
+ * Produce a conservative source identity. Only URL syntax semantics are normalized:
199
+ * scheme and host casing, default ports, and fragments. Path case, non-root trailing
200
+ * slashes, and the complete query sequence remain significant.
201
+ */
202
+ export function normalizeSourceUrlIdentity(value: string | URL): string | null {
203
+ const parsed = publicHttpUrl(value);
204
+ if (parsed === null) return null;
205
+ parsed.hash = "";
206
+ return parsed.href;
207
+ }
208
+
209
+ /** Match only URLs with the same conservative source identity. */
210
+ export function isExactSourceTarget(candidate: string | URL, target: string | URL): boolean {
211
+ const candidateIdentity = normalizeSourceUrlIdentity(candidate);
212
+ const targetIdentity = normalizeSourceUrlIdentity(target);
213
+ return candidateIdentity !== null && targetIdentity !== null && candidateIdentity === targetIdentity;
214
+ }
215
+
216
+ function parseMetadataResult(
217
+ value: unknown,
218
+ index: number,
219
+ queried: ReadonlySet<string>,
220
+ failed: ReadonlySet<string>,
221
+ budget: TextBudget,
222
+ ): MetadataSearchResult {
223
+ const label = `Metadata search response.results[${index}]`;
224
+ const input = record(value, label);
225
+ strictKeys(input, ["title", "url", "snippet", "engines", "score"], label);
226
+ const title = cleanDisplayText(input.title, `${label}.title`, MAX_METADATA_SEARCH_TITLE_UTF8_BYTES, budget);
227
+ if (title === "") fail(`${label}.title`, "must contain visible text.");
228
+ const rawUrl = boundedString(input.url, `${label}.url`, MAX_URL_INTELLIGENCE_URL_UTF8_BYTES, budget);
229
+ const url = normalizeSourceUrlIdentity(rawUrl);
230
+ if (url === null) fail(`${label}.url`, "must be a public HTTP or HTTPS URL without credentials or controls.");
231
+ const cleanedSnippet = input.snippet === null
232
+ ? null
233
+ : cleanDisplayText(input.snippet, `${label}.snippet`, MAX_METADATA_SEARCH_SNIPPET_UTF8_BYTES, budget);
234
+ const snippet = cleanedSnippet === "" ? null : cleanedSnippet;
235
+ const engines = uniqueEngineArray(input.engines, `${label}.engines`, budget, false);
236
+ for (const engine of engines) {
237
+ if (!queried.has(engine)) fail(`${label}.engines`, `contains unqueried engine ${engine}.`);
238
+ if (failed.has(engine)) fail(`${label}.engines`, `contains failed engine ${engine}.`);
239
+ }
240
+ if (typeof input.score !== "number" || !Number.isFinite(input.score) || input.score <= 0) {
241
+ fail(`${label}.score`, "must be a positive finite number.");
242
+ }
243
+ return Object.freeze({ title, url, snippet, engines, score: input.score });
244
+ }
245
+
246
+ /** Strictly parse one successful metadata-search-engine-rs `GET /search` body. */
247
+ export function parseMetadataSearchResponse(value: unknown): MetadataSearchResponse {
248
+ const input = record(value, "Metadata search response");
249
+ strictKeys(input, ["query", "results", "engines_queried", "engines_failed"], "Metadata search response");
250
+ const budget: TextBudget = { bytes: 0 };
251
+ const query = boundedString(input.query, "Metadata search response.query", MAX_METADATA_SEARCH_QUERY_UTF8_BYTES, budget);
252
+ if (query === "" || query !== query.trim() || hasUnsafeControls(query)) {
253
+ fail("Metadata search response.query", "must be non-empty, trimmed, and free of controls.");
254
+ }
255
+ const enginesQueried = uniqueEngineArray(input.engines_queried, "Metadata search response.engines_queried", budget, false);
256
+ const enginesFailed = uniqueEngineArray(input.engines_failed, "Metadata search response.engines_failed", budget, true);
257
+ const queried = new Set(enginesQueried);
258
+ for (const engine of enginesFailed) {
259
+ if (!queried.has(engine)) fail("Metadata search response.engines_failed", `contains unqueried engine ${engine}.`);
260
+ }
261
+ if (!Array.isArray(input.results) || input.results.length > MAX_METADATA_SEARCH_RESULTS) {
262
+ fail("Metadata search response.results", `must be an array with at most ${MAX_METADATA_SEARCH_RESULTS} entries.`);
263
+ }
264
+ const failed = new Set(enginesFailed);
265
+ const results = Object.freeze(input.results.map((result, index) =>
266
+ parseMetadataResult(result, index, queried, failed, budget)));
267
+ return Object.freeze({
268
+ query,
269
+ results,
270
+ enginesQueried,
271
+ enginesFailed,
272
+ engineStatus: enginesFailed.length === 0
273
+ ? "complete"
274
+ : enginesFailed.length === enginesQueried.length ? "unavailable" : "partial",
275
+ });
276
+ }
277
+
278
+ /** Rank exact targets first, then upstream score, with input order as the stable tie-breaker. */
279
+ export function rankMetadataSearchResults(
280
+ results: readonly MetadataSearchResult[],
281
+ options: RankMetadataSearchOptions = {},
282
+ ): readonly RankedMetadataSearchResult[] {
283
+ const limit = options.limit ?? MAX_METADATA_SEARCH_RESULTS;
284
+ if (!Number.isSafeInteger(limit) || limit < 0 || limit > MAX_METADATA_SEARCH_RESULTS) {
285
+ throw new RangeError(`Metadata search rank limit must be from 0 through ${MAX_METADATA_SEARCH_RESULTS}.`);
286
+ }
287
+ if (limit === 0) return Object.freeze([]);
288
+ const targetIdentity = options.targetUrl === undefined
289
+ ? null
290
+ : normalizeSourceUrlIdentity(options.targetUrl);
291
+ const decorated = results.flatMap((result, index) => {
292
+ const sourceIdentity = normalizeSourceUrlIdentity(result.url);
293
+ return sourceIdentity === null ? [] : [{ result, index, sourceIdentity, exactTarget: sourceIdentity === targetIdentity }];
294
+ }).toSorted((left, right) =>
295
+ Number(right.exactTarget) - Number(left.exactTarget)
296
+ || right.result.score - left.result.score
297
+ || left.index - right.index);
298
+ const seen = new Set<string>();
299
+ const ranked: RankedMetadataSearchResult[] = [];
300
+ for (const item of decorated) {
301
+ if (seen.has(item.sourceIdentity)) continue;
302
+ seen.add(item.sourceIdentity);
303
+ ranked.push(Object.freeze({
304
+ ...item.result,
305
+ rank: ranked.length + 1,
306
+ sourceIdentity: item.sourceIdentity,
307
+ exactTarget: item.exactTarget,
308
+ }));
309
+ if (ranked.length === limit) break;
310
+ }
311
+ return Object.freeze(ranked);
312
+ }
313
+
314
+ function isWhitespace(character: string | undefined): boolean {
315
+ return character === " " || character === "\t" || character === "\r" || character === "\n";
316
+ }
317
+
318
+ function parseLinkFormat(source: string): readonly ParsedLink[] {
319
+ let cursor = 0;
320
+ const links: ParsedLink[] = [];
321
+ const skipWhitespace = (): void => {
322
+ while (isWhitespace(source[cursor])) cursor += 1;
323
+ };
324
+ const syntax = (message: string): never => fail("Archive.today TimeMap", `${message} at offset ${cursor}.`);
325
+
326
+ skipWhitespace();
327
+ while (cursor < source.length) {
328
+ if (links.length >= MAX_ARCHIVE_TIMEMAP_ENTRIES) {
329
+ fail("Archive.today TimeMap", `may contain at most ${MAX_ARCHIVE_TIMEMAP_ENTRIES} entries.`);
330
+ }
331
+ if (source[cursor] !== "<") syntax("must start each link with '<'");
332
+ cursor += 1;
333
+ const targetStart = cursor;
334
+ while (cursor < source.length && source[cursor] !== ">") {
335
+ const character = source[cursor];
336
+ if (character === "<" || character === '"' || character === "\\" || isWhitespace(character)) {
337
+ syntax("contains an invalid target character");
338
+ }
339
+ cursor += 1;
340
+ }
341
+ if (cursor >= source.length) syntax("has an unterminated target");
342
+ const target = source.slice(targetStart, cursor);
343
+ cursor += 1;
344
+ if (target === "" || Buffer.byteLength(target, "utf8") > MAX_URL_INTELLIGENCE_URL_UTF8_BYTES) {
345
+ syntax("has an empty or oversized target");
346
+ }
347
+
348
+ const parameters = new Map<string, string>();
349
+ skipWhitespace();
350
+ while (source[cursor] === ";") {
351
+ if (parameters.size >= MAX_ARCHIVE_TIMEMAP_PARAMETERS_PER_ENTRY) {
352
+ fail("Archive.today TimeMap", `entries may contain at most ${MAX_ARCHIVE_TIMEMAP_PARAMETERS_PER_ENTRY} parameters.`);
353
+ }
354
+ cursor += 1;
355
+ skipWhitespace();
356
+ const nameStart = cursor;
357
+ while (tokenCharacter.test(source[cursor] ?? "")) cursor += 1;
358
+ if (cursor === nameStart) syntax("has an invalid parameter name");
359
+ const name = source.slice(nameStart, cursor).toLowerCase();
360
+ skipWhitespace();
361
+ if (source[cursor] !== "=") syntax("requires '=' after a parameter name");
362
+ cursor += 1;
363
+ skipWhitespace();
364
+ let parameterValue = "";
365
+ if (source[cursor] === '"') {
366
+ cursor += 1;
367
+ let terminated = false;
368
+ while (cursor < source.length) {
369
+ const character = source[cursor] ?? "";
370
+ cursor += 1;
371
+ if (character === '"') {
372
+ terminated = true;
373
+ break;
374
+ }
375
+ if (character === "\\") {
376
+ if (cursor >= source.length) syntax("has an unterminated quoted escape");
377
+ const escaped = source[cursor] ?? "";
378
+ if (hasUnsafeControls(escaped)) syntax("has a control in a quoted escape");
379
+ parameterValue += escaped;
380
+ cursor += 1;
381
+ } else {
382
+ if (hasUnsafeControls(character)) syntax("has a control in a quoted value");
383
+ parameterValue += character;
384
+ }
385
+ }
386
+ if (!terminated) syntax("has an unterminated quoted value");
387
+ } else {
388
+ const valueStart = cursor;
389
+ while (tokenCharacter.test(source[cursor] ?? "")) cursor += 1;
390
+ if (cursor === valueStart) syntax("has an invalid parameter value");
391
+ parameterValue = source.slice(valueStart, cursor);
392
+ }
393
+ if (Buffer.byteLength(parameterValue, "utf8") > MAX_URL_INTELLIGENCE_URL_UTF8_BYTES) {
394
+ syntax("has an oversized parameter value");
395
+ }
396
+ if (parameters.has(name)) syntax(`repeats parameter ${name}`);
397
+ parameters.set(name, parameterValue);
398
+ skipWhitespace();
399
+ }
400
+ links.push(Object.freeze({ target, parameters }));
401
+ if (cursor === source.length) break;
402
+ if (source[cursor] !== ",") syntax("must separate links with ','");
403
+ cursor += 1;
404
+ skipWhitespace();
405
+ if (cursor === source.length) syntax("must not end with ','");
406
+ }
407
+ return Object.freeze(links);
408
+ }
409
+
410
+ const months: ReadonlyMap<string, number> = new Map([
411
+ ["Jan", 0], ["Feb", 1], ["Mar", 2], ["Apr", 3], ["May", 4], ["Jun", 5],
412
+ ["Jul", 6], ["Aug", 7], ["Sep", 8], ["Oct", 9], ["Nov", 10], ["Dec", 11],
413
+ ] as const);
414
+ const weekdays = ["Sun", "Mon", "Tue", "Wed", "Thu", "Fri", "Sat"] as const;
415
+
416
+ function epochFromTimestamp(timestamp: string, label: string): number {
417
+ const match = /^(\d{4})(\d{2})(\d{2})(\d{2})(\d{2})(\d{2})$/u.exec(timestamp);
418
+ if (match === null) return fail(label, "must contain a 14-digit UTC timestamp.");
419
+ const [year, month, day, hour, minute, second] = match.slice(1).map(Number) as [number, number, number, number, number, number];
420
+ const epoch = Date.UTC(year, month - 1, day, hour, minute, second);
421
+ const date = new Date(epoch);
422
+ if (
423
+ year < 1900
424
+ || date.getUTCFullYear() !== year
425
+ || date.getUTCMonth() !== month - 1
426
+ || date.getUTCDate() !== day
427
+ || date.getUTCHours() !== hour
428
+ || date.getUTCMinutes() !== minute
429
+ || date.getUTCSeconds() !== second
430
+ ) return fail(label, "contains an invalid UTC timestamp.");
431
+ return epoch;
432
+ }
433
+
434
+ function epochFromHttpDate(value: string, label: string): number {
435
+ const match = /^(Sun|Mon|Tue|Wed|Thu|Fri|Sat), (\d{2}) (Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec) (\d{4}) (\d{2}):(\d{2}):(\d{2}) GMT$/u.exec(value);
436
+ if (match === null) return fail(label, "must be an RFC 1123 date in GMT.");
437
+ const month = months.get(match[3] ?? "");
438
+ if (month === undefined) return fail(label, "has an invalid month.");
439
+ const year = Number(match[4]);
440
+ const day = Number(match[2]);
441
+ const hour = Number(match[5]);
442
+ const minute = Number(match[6]);
443
+ const second = Number(match[7]);
444
+ const epoch = Date.UTC(year, month, day, hour, minute, second);
445
+ const date = new Date(epoch);
446
+ if (
447
+ year < 1900
448
+ || date.getUTCFullYear() !== year
449
+ || date.getUTCMonth() !== month
450
+ || date.getUTCDate() !== day
451
+ || date.getUTCHours() !== hour
452
+ || date.getUTCMinutes() !== minute
453
+ || date.getUTCSeconds() !== second
454
+ || weekdays[date.getUTCDay()] !== match[1]
455
+ ) return fail(label, "contains an invalid RFC 1123 date.");
456
+ return epoch;
457
+ }
458
+
459
+ function checkedNow(now: Date): number {
460
+ if (!(now instanceof Date) || !Number.isFinite(now.getTime())) {
461
+ return fail("Archive.today now", "must be a valid injected Date.");
462
+ }
463
+ return now.getTime();
464
+ }
465
+
466
+ function parsedArchiveTodayMementoUrl(
467
+ value: string | URL,
468
+ originalIdentity: string,
469
+ nowEpoch: number,
470
+ expectedEpoch?: number,
471
+ ): ArchiveTodayMemento {
472
+ const raw = value instanceof URL ? value.href : value;
473
+ if (typeof raw !== "string" || raw !== raw.trim() || hasUnsafeControls(raw)
474
+ || Buffer.byteLength(raw, "utf8") > MAX_URL_INTELLIGENCE_URL_UTF8_BYTES) {
475
+ return fail("Archive.today memento URL", "must be a bounded control-free URL.");
476
+ }
477
+ let archiveUrl: URL;
478
+ try {
479
+ archiveUrl = new URL(raw);
480
+ } catch {
481
+ return fail("Archive.today memento URL", "must be a valid URL.");
482
+ }
483
+ if (
484
+ (archiveUrl.protocol !== "http:" && archiveUrl.protocol !== "https:")
485
+ || archiveUrl.username !== ""
486
+ || archiveUrl.password !== ""
487
+ || archiveUrl.port !== ""
488
+ || archiveUrl.hash !== ""
489
+ || !archiveTodayHosts.has(archiveUrl.hostname)
490
+ ) return fail("Archive.today memento URL", "must use an allowlisted archive host without credentials, a port, or a fragment.");
491
+ const path = /^\/(\d{14})\/(https?:\/\/.*)$/u.exec(archiveUrl.pathname);
492
+ if (path === null || path[1] === undefined || path[2] === undefined) {
493
+ return fail("Archive.today memento URL", "must use a timestamped read-only snapshot path.");
494
+ }
495
+ const timestamp = path[1];
496
+ const capturedEpoch = epochFromTimestamp(timestamp, "Archive.today memento URL timestamp");
497
+ if (capturedEpoch > nowEpoch) fail("Archive.today memento URL timestamp", "must not be in the future.");
498
+ if (expectedEpoch !== undefined && capturedEpoch !== expectedEpoch) {
499
+ fail("Archive.today memento URL timestamp", "must equal its Memento datetime.");
500
+ }
501
+ const embeddedIdentity = normalizeSourceUrlIdentity(`${path[2]}${archiveUrl.search}`);
502
+ if (embeddedIdentity === null || embeddedIdentity !== originalIdentity) {
503
+ fail("Archive.today memento URL", "must embed the exact bound original URL.");
504
+ }
505
+ const archiveHost = archiveUrl.hostname as ArchiveTodayHost;
506
+ // Upgrade only after the alias, timestamped path, and embedded source all validate.
507
+ archiveUrl.protocol = "https:";
508
+ return Object.freeze({
509
+ url: archiveUrl.href,
510
+ archiveHost,
511
+ timestamp,
512
+ capturedAt: new Date(capturedEpoch).toISOString(),
513
+ originalUrl: originalIdentity,
514
+ });
515
+ }
516
+
517
+ /** Validate one read-only timestamped archive.today-family snapshot URL. */
518
+ export function parseArchiveTodayMementoUrl(
519
+ value: string | URL,
520
+ options: ParseArchiveTodayMementoUrlOptions,
521
+ ): ArchiveTodayMemento {
522
+ const originalIdentity = normalizeSourceUrlIdentity(options.originalUrl);
523
+ if (originalIdentity === null) fail("Archive.today original URL", "must be a public HTTP or HTTPS URL without credentials.");
524
+ return parsedArchiveTodayMementoUrl(value, originalIdentity, checkedNow(options.now));
525
+ }
526
+
527
+ /** Deterministically select the newest snapshot, breaking timestamp ties by URL. */
528
+ export function selectNewestArchiveTodayMemento(
529
+ mementos: readonly ArchiveTodayMemento[],
530
+ ): ArchiveTodayMemento | null {
531
+ return mementos.toSorted((left, right) =>
532
+ right.timestamp.localeCompare(left.timestamp) || left.url.localeCompare(right.url))[0] ?? null;
533
+ }
534
+
535
+ /** Parse and bind an archive.today-family Memento `application/link-format` TimeMap. */
536
+ export function parseArchiveTodayTimeMap(
537
+ value: unknown,
538
+ options: ParseArchiveTodayTimeMapOptions,
539
+ ): ArchiveTodayTimeMap {
540
+ if (typeof value !== "string") fail("Archive.today TimeMap", "must be text.");
541
+ if (Buffer.byteLength(value, "utf8") > MAX_ARCHIVE_TIMEMAP_UTF8_BYTES) {
542
+ fail("Archive.today TimeMap", `must be at most ${MAX_ARCHIVE_TIMEMAP_UTF8_BYTES} UTF-8 bytes.`);
543
+ }
544
+ if (hasUnsafeControls(value, true, true)) {
545
+ fail("Archive.today TimeMap", "contains forbidden controls.");
546
+ }
547
+ const originalIdentity = normalizeSourceUrlIdentity(options.originalUrl);
548
+ if (originalIdentity === null) fail("Archive.today original URL", "must be a public HTTP or HTTPS URL without credentials.");
549
+ const nowEpoch = checkedNow(options.now);
550
+ const links = parseLinkFormat(value);
551
+ const originals = links.filter((link) => link.parameters.get("rel")?.toLowerCase() === "original");
552
+ if (originals.length !== 1) fail("Archive.today TimeMap", "must contain exactly one rel=original link.");
553
+ const declaredOriginal = normalizeSourceUrlIdentity(originals[0]?.target ?? "");
554
+ if (declaredOriginal === null || declaredOriginal !== originalIdentity) {
555
+ fail("Archive.today TimeMap rel=original", "must exactly match the requested original URL.");
556
+ }
557
+
558
+ const mementos: ArchiveTodayMemento[] = [];
559
+ const seen = new Set<string>();
560
+ for (const [index, link] of links.entries()) {
561
+ const relations = (link.parameters.get("rel") ?? "").trim().toLowerCase().split(/[ \t]+/u).filter(Boolean);
562
+ if (!relations.includes("memento")) continue;
563
+ if (relations.includes("original")) fail(`Archive.today TimeMap entry ${index}`, "cannot be both original and memento.");
564
+ const datetime = link.parameters.get("datetime");
565
+ if (datetime === undefined) fail(`Archive.today TimeMap entry ${index}`, "requires a datetime parameter.");
566
+ const expectedEpoch = epochFromHttpDate(datetime, `Archive.today TimeMap entry ${index} datetime`);
567
+ if (expectedEpoch > nowEpoch) fail(`Archive.today TimeMap entry ${index} datetime`, "must not be in the future.");
568
+ const memento = parsedArchiveTodayMementoUrl(link.target, originalIdentity, nowEpoch, expectedEpoch);
569
+ if (!seen.has(memento.url)) {
570
+ seen.add(memento.url);
571
+ mementos.push(memento);
572
+ }
573
+ }
574
+ const sorted = Object.freeze(mementos.toSorted((left, right) =>
575
+ right.timestamp.localeCompare(left.timestamp) || left.url.localeCompare(right.url)));
576
+ return Object.freeze({
577
+ originalUrl: originalIdentity,
578
+ mementos: sorted,
579
+ newest: selectNewestArchiveTodayMemento(sorted),
580
+ });
581
+ }