@hraness/kb 0.17.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/DISCLOSURE +43 -0
- package/LICENSE +21 -0
- package/README.md +545 -0
- package/dist/agent-context.js +35 -0
- package/dist/agent-guide-audit.js +18 -0
- package/dist/attachments.js +19 -0
- package/dist/authoring.js +33 -0
- package/dist/benchmark.js +19 -0
- package/dist/capture.js +49 -0
- package/dist/cli.js +3646 -0
- package/dist/clip/acquire.js +57 -0
- package/dist/clip/args.js +21 -0
- package/dist/clip/bounded-byte-buffer.js +9 -0
- package/dist/clip/browser-profiles.js +17 -0
- package/dist/clip/bundle-reader.js +11 -0
- package/dist/clip/cli.js +27 -0
- package/dist/clip/cookies.js +21 -0
- package/dist/clip/defuddle-worker.js +84 -0
- package/dist/clip/doctor.js +32 -0
- package/dist/clip/jobs.js +784 -0
- package/dist/clip/network-proxy.js +9 -0
- package/dist/clip/network.js +28 -0
- package/dist/clip/persist.js +30 -0
- package/dist/clip/refresh.js +11 -0
- package/dist/clip/terminal.js +11 -0
- package/dist/evaluation-builder.js +12238 -0
- package/dist/evaluation-kb.js +26 -0
- package/dist/evaluation.js +35 -0
- package/dist/git.js +39 -0
- package/dist/graph.js +39 -0
- package/dist/index-01jj6rbv.js +1103 -0
- package/dist/index-06c9ctr6.js +738 -0
- package/dist/index-0kavxzqj.js +2480 -0
- package/dist/index-1gwbassd.js +743 -0
- package/dist/index-1n418kb9.js +827 -0
- package/dist/index-1vrd1rmn.js +56 -0
- package/dist/index-1xxnjn0d.js +171 -0
- package/dist/index-3rm7cz6h.js +373 -0
- package/dist/index-3v2z4f0q.js +328 -0
- package/dist/index-48pz4jpc.js +392 -0
- package/dist/index-4j3tt0c3.js +234 -0
- package/dist/index-5n05se68.js +307 -0
- package/dist/index-5vwpzb5a.js +764 -0
- package/dist/index-6g2pv9d2.js +370 -0
- package/dist/index-7fzc32gf.js +712 -0
- package/dist/index-84x0vjjp.js +432 -0
- package/dist/index-adx6khj5.js +488 -0
- package/dist/index-b88v3vtm.js +805 -0
- package/dist/index-cv6fh7z5.js +462 -0
- package/dist/index-cxfrakt7.js +1067 -0
- package/dist/index-d13v9ckt.js +150 -0
- package/dist/index-dycjbhw2.js +1943 -0
- package/dist/index-dyqwejk5.js +531 -0
- package/dist/index-e5fbsywq.js +658 -0
- package/dist/index-f984hw45.js +927 -0
- package/dist/index-gh719d91.js +63 -0
- package/dist/index-hgve9rh2.js +2046 -0
- package/dist/index-hya40gb2.js +474 -0
- package/dist/index-j4zgmzjr.js +152 -0
- package/dist/index-jsmvyyvf.js +1166 -0
- package/dist/index-kh0pwvdd.js +1974 -0
- package/dist/index-mqx4nd6v.js +119 -0
- package/dist/index-mxxxytys.js +1254 -0
- package/dist/index-n5dd7r0v.js +711 -0
- package/dist/index-npg9z1a4.js +431 -0
- package/dist/index-pj501bh1.js +140 -0
- package/dist/index-qry4vhxk.js +1041 -0
- package/dist/index-s2gw5aw9.js +199 -0
- package/dist/index-vvdj6kca.js +70 -0
- package/dist/index-w2zc0vwa.js +571 -0
- package/dist/index-x3fthpsc.js +585 -0
- package/dist/index-zr53sf63.js +44 -0
- package/dist/index-zxdy5pby.js +2728 -0
- package/dist/index-zzhgcwyt.js +966 -0
- package/dist/index.js +453 -0
- package/dist/navigation.js +13 -0
- package/dist/pdf.js +85 -0
- package/dist/percolate.js +24 -0
- package/dist/portfolio.js +77 -0
- package/dist/query.js +29 -0
- package/dist/repository-memory.js +60 -0
- package/dist/sdk.js +36 -0
- package/dist/search-rules.js +34 -0
- package/dist/search.js +22 -0
- package/dist/semantic.js +39 -0
- package/dist/source-inbox.js +17 -0
- package/dist/untrusted-content.js +32 -0
- package/dist/url-intelligence.js +74 -0
- package/dist/workflow.js +23 -0
- package/dist/workflows/decision-context.js +20 -0
- package/dist/workflows/explain-change.js +8 -0
- package/dist/workflows/index.js +28 -0
- package/dist/workflows/plan-radar.js +10 -0
- package/package.json +409 -0
- package/skills/kb/AGENTS.md +19 -0
- package/skills/kb/SKILL.md +82 -0
- package/skills/kb/agents/openai.yaml +4 -0
- package/skills/kb/references/pdf-review.md +51 -0
- package/skills/kb/references/percolate.md +144 -0
- package/skills/kb/references/plan-structure.md +105 -0
- package/skills/kb/references/plan.md +115 -0
- package/skills/kb/references/query.md +231 -0
- package/skills/kb/references/refresh.md +133 -0
- package/skills/kb/references/save-pdf.md +125 -0
- package/skills/kb/references/save-url.md +178 -0
- package/skills/kb/references/url-authentication.md +75 -0
- package/skills/kb/references/url-platforms.md +25 -0
- package/src/agent-context.ts +1241 -0
- package/src/agent-guide-audit.ts +750 -0
- package/src/attachments.ts +723 -0
- package/src/authoring.ts +1725 -0
- package/src/benchmark.ts +362 -0
- package/src/capture.ts +68 -0
- package/src/cli.ts +3660 -0
- package/src/clip/acquire.ts +1273 -0
- package/src/clip/archive-today.ts +402 -0
- package/src/clip/args.ts +423 -0
- package/src/clip/assets.ts +262 -0
- package/src/clip/bounded-byte-buffer.ts +75 -0
- package/src/clip/browser-profiles.ts +372 -0
- package/src/clip/bundle-reader.ts +608 -0
- package/src/clip/capture.ts +1054 -0
- package/src/clip/cli.ts +181 -0
- package/src/clip/cookies.ts +512 -0
- package/src/clip/doctor.ts +1110 -0
- package/src/clip/extract.ts +868 -0
- package/src/clip/jobs.ts +959 -0
- package/src/clip/lib.ts +861 -0
- package/src/clip/media.ts +1474 -0
- package/src/clip/metadata-search-tool/Cargo.lock +2498 -0
- package/src/clip/metadata-search-tool/Cargo.toml +18 -0
- package/src/clip/metadata-search-tool/runner.ts +685 -0
- package/src/clip/metadata-search-tool/src/main.rs +389 -0
- package/src/clip/metadata-search.ts +687 -0
- package/src/clip/network-proxy.ts +627 -0
- package/src/clip/network.ts +950 -0
- package/src/clip/package-root.ts +54 -0
- package/src/clip/persist.ts +1472 -0
- package/src/clip/platforms.ts +1824 -0
- package/src/clip/refresh.ts +197 -0
- package/src/clip/structured.ts +432 -0
- package/src/clip/terminal.ts +197 -0
- package/src/clip/url-intelligence.ts +581 -0
- package/src/clip/url-metadata-backfill.ts +616 -0
- package/src/clip/url-metadata-cli.ts +251 -0
- package/src/clip/url-metadata.ts +1397 -0
- package/src/evaluation-analysis-v2.ts +1440 -0
- package/src/evaluation-builder.ts +2539 -0
- package/src/evaluation-corpus-authoring.ts +2052 -0
- package/src/evaluation-evidence.ts +1147 -0
- package/src/evaluation-execution-v2.ts +426 -0
- package/src/evaluation-implementation.ts +131 -0
- package/src/evaluation-kb-closure.ts +1430 -0
- package/src/evaluation-kb-runner-v2.ts +735 -0
- package/src/evaluation-kb-v2.ts +2567 -0
- package/src/evaluation-kb.ts +1018 -0
- package/src/evaluation-measurement-v2.ts +1462 -0
- package/src/evaluation-packing-v2.ts +452 -0
- package/src/evaluation-redaction.ts +28 -0
- package/src/evaluation-v2.ts +3857 -0
- package/src/evaluation.ts +1220 -0
- package/src/git.ts +1276 -0
- package/src/graph.ts +1487 -0
- package/src/index.ts +20 -0
- package/src/init.ts +123 -0
- package/src/navigation.ts +277 -0
- package/src/note-lock.ts +529 -0
- package/src/pdf/args.ts +202 -0
- package/src/pdf/capture.ts +298 -0
- package/src/pdf/cli.ts +159 -0
- package/src/pdf/extract.ts +303 -0
- package/src/pdf/layout.ts +618 -0
- package/src/pdf/markdown.ts +154 -0
- package/src/pdf/model.ts +234 -0
- package/src/pdf/ocr.ts +190 -0
- package/src/pdf/persist.ts +314 -0
- package/src/pdf/source.ts +135 -0
- package/src/pdf/tools.ts +146 -0
- package/src/pdf.ts +92 -0
- package/src/percolate.ts +877 -0
- package/src/portfolio-audit.ts +574 -0
- package/src/portfolio-identity.ts +162 -0
- package/src/portfolio-registry.ts +473 -0
- package/src/portfolio.ts +765 -0
- package/src/query.ts +646 -0
- package/src/repository-memory.ts +1150 -0
- package/src/sdk.ts +1452 -0
- package/src/search-rules.ts +712 -0
- package/src/search.ts +710 -0
- package/src/semantic-runtime.ts +1139 -0
- package/src/semantic.ts +2480 -0
- package/src/source-inbox.ts +232 -0
- package/src/untrusted-content.ts +401 -0
- package/src/url-intelligence.ts +104 -0
- package/src/vault.ts +636 -0
- package/src/workflow.ts +653 -0
- package/src/workflows/decision-context.ts +81 -0
- package/src/workflows/explain-change.ts +73 -0
- package/src/workflows/index.ts +3 -0
- package/src/workflows/plan-radar.ts +101 -0
|
@@ -0,0 +1,712 @@
|
|
|
1
|
+
// @bun
|
|
2
|
+
import {
|
|
3
|
+
assertSafeNetworkUrl,
|
|
4
|
+
decodeBytes,
|
|
5
|
+
isPrivateHostname,
|
|
6
|
+
safeFetch
|
|
7
|
+
} from "./index-e5fbsywq.js";
|
|
8
|
+
import {
|
|
9
|
+
sanitizeArtifactUrl
|
|
10
|
+
} from "./index-mxxxytys.js";
|
|
11
|
+
|
|
12
|
+
// src/clip/url-intelligence.ts
|
|
13
|
+
var MAX_METADATA_SEARCH_RESULTS = 20;
|
|
14
|
+
var MAX_METADATA_SEARCH_ENGINES = 8;
|
|
15
|
+
var MAX_METADATA_SEARCH_QUERY_UTF8_BYTES = 4 * 1024;
|
|
16
|
+
var MAX_METADATA_SEARCH_TITLE_UTF8_BYTES = 2 * 1024;
|
|
17
|
+
var MAX_METADATA_SEARCH_SNIPPET_UTF8_BYTES = 8 * 1024;
|
|
18
|
+
var MAX_URL_INTELLIGENCE_URL_UTF8_BYTES = 16 * 1024;
|
|
19
|
+
var MAX_METADATA_SEARCH_TEXT_UTF8_BYTES = 512 * 1024;
|
|
20
|
+
var MAX_METADATA_SEARCH_RESPONSE_BYTES = 2 * 1024 * 1024;
|
|
21
|
+
var MAX_ARCHIVE_TIMEMAP_UTF8_BYTES = 512 * 1024;
|
|
22
|
+
var MAX_ARCHIVE_TIMEMAP_ENTRIES = 512;
|
|
23
|
+
var MAX_ARCHIVE_TIMEMAP_PARAMETERS_PER_ENTRY = 16;
|
|
24
|
+
var ARCHIVE_TODAY_HOSTS = Object.freeze([
|
|
25
|
+
"archive.today",
|
|
26
|
+
"archive.is",
|
|
27
|
+
"archive.ph",
|
|
28
|
+
"archive.fo",
|
|
29
|
+
"archive.li",
|
|
30
|
+
"archive.md",
|
|
31
|
+
"archive.vn"
|
|
32
|
+
]);
|
|
33
|
+
var archiveTodayHosts = new Set(ARCHIVE_TODAY_HOSTS);
|
|
34
|
+
var tokenCharacter = /^[!#$%&'*+.^_`|~0-9A-Za-z-]$/u;
|
|
35
|
+
var engineName = /^[a-z0-9][a-z0-9._-]{0,127}$/u;
|
|
36
|
+
function fail(label, message) {
|
|
37
|
+
throw new TypeError(`${label} ${message}`);
|
|
38
|
+
}
|
|
39
|
+
function isUnsafeControlCodePoint(codePoint, rejectSurrogates = false, allowLinkWhitespace = false) {
|
|
40
|
+
return codePoint <= 31 && !(allowLinkWhitespace && (codePoint === 9 || codePoint === 10 || codePoint === 13)) || codePoint >= 127 && codePoint <= 159 || codePoint === 1564 || codePoint === 8206 || codePoint === 8207 || codePoint >= 8234 && codePoint <= 8238 || codePoint >= 8294 && codePoint <= 8297 || rejectSurrogates && codePoint >= 55296 && codePoint <= 57343;
|
|
41
|
+
}
|
|
42
|
+
function hasUnsafeControls(value, rejectSurrogates = false, allowLinkWhitespace = false) {
|
|
43
|
+
return Array.from(value).some((character) => isUnsafeControlCodePoint(character.codePointAt(0), rejectSurrogates, allowLinkWhitespace));
|
|
44
|
+
}
|
|
45
|
+
function record(value, label) {
|
|
46
|
+
if (value === null || typeof value !== "object" || Array.isArray(value)) {
|
|
47
|
+
return fail(label, "must be an object.");
|
|
48
|
+
}
|
|
49
|
+
return value;
|
|
50
|
+
}
|
|
51
|
+
function strictKeys(value, allowed, label) {
|
|
52
|
+
const allowedKeys = new Set(allowed);
|
|
53
|
+
const unknown = Object.keys(value).filter((key) => !allowedKeys.has(key)).toSorted();
|
|
54
|
+
if (unknown.length > 0)
|
|
55
|
+
fail(label, `has unknown fields: ${unknown.join(", ")}.`);
|
|
56
|
+
}
|
|
57
|
+
function boundedString(value, label, maximumBytes, budget) {
|
|
58
|
+
if (typeof value !== "string")
|
|
59
|
+
return fail(label, "must be a string.");
|
|
60
|
+
const bytes = Buffer.byteLength(value, "utf8");
|
|
61
|
+
if (bytes > maximumBytes)
|
|
62
|
+
fail(label, `must be at most ${maximumBytes} UTF-8 bytes.`);
|
|
63
|
+
if (budget !== undefined) {
|
|
64
|
+
budget.bytes += bytes;
|
|
65
|
+
if (budget.bytes > MAX_METADATA_SEARCH_TEXT_UTF8_BYTES) {
|
|
66
|
+
fail("Metadata search response", `may contain at most ${MAX_METADATA_SEARCH_TEXT_UTF8_BYTES} UTF-8 bytes of text.`);
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
return value;
|
|
70
|
+
}
|
|
71
|
+
function cleanDisplayText(value, label, maximumBytes, budget) {
|
|
72
|
+
const text = boundedString(value, label, maximumBytes, budget);
|
|
73
|
+
return Array.from(text, (character) => isUnsafeControlCodePoint(character.codePointAt(0)) ? " " : character).join("").replace(/ +/gu, " ").trim();
|
|
74
|
+
}
|
|
75
|
+
function checkedEngine(value, label, budget) {
|
|
76
|
+
const engine = boundedString(value, label, 128, budget);
|
|
77
|
+
if (!engineName.test(engine))
|
|
78
|
+
fail(label, "must be a lowercase engine identifier.");
|
|
79
|
+
return engine;
|
|
80
|
+
}
|
|
81
|
+
function uniqueEngineArray(value, label, budget, allowEmpty) {
|
|
82
|
+
if (!Array.isArray(value) || value.length > MAX_METADATA_SEARCH_ENGINES || !allowEmpty && value.length === 0) {
|
|
83
|
+
return fail(label, `must be ${allowEmpty ? "an" : "a non-empty"} array with at most ${MAX_METADATA_SEARCH_ENGINES} entries.`);
|
|
84
|
+
}
|
|
85
|
+
const engines = value.map((entry, index) => checkedEngine(entry, `${label}[${index}]`, budget));
|
|
86
|
+
if (new Set(engines).size !== engines.length)
|
|
87
|
+
fail(label, "must not contain duplicates.");
|
|
88
|
+
return Object.freeze(engines);
|
|
89
|
+
}
|
|
90
|
+
function publicHttpUrl(value) {
|
|
91
|
+
const raw = value instanceof URL ? value.href : value;
|
|
92
|
+
if (typeof raw !== "string" || raw === "" || raw !== raw.trim() || hasUnsafeControls(raw) || Buffer.byteLength(raw, "utf8") > MAX_URL_INTELLIGENCE_URL_UTF8_BYTES)
|
|
93
|
+
return null;
|
|
94
|
+
let parsed;
|
|
95
|
+
try {
|
|
96
|
+
parsed = new URL(raw);
|
|
97
|
+
} catch {
|
|
98
|
+
return null;
|
|
99
|
+
}
|
|
100
|
+
if (parsed.protocol !== "http:" && parsed.protocol !== "https:" || parsed.username !== "" || parsed.password !== "" || parsed.hostname === "" || isPrivateHostname(parsed.hostname))
|
|
101
|
+
return null;
|
|
102
|
+
return parsed;
|
|
103
|
+
}
|
|
104
|
+
function normalizeSourceUrlIdentity(value) {
|
|
105
|
+
const parsed = publicHttpUrl(value);
|
|
106
|
+
if (parsed === null)
|
|
107
|
+
return null;
|
|
108
|
+
parsed.hash = "";
|
|
109
|
+
return parsed.href;
|
|
110
|
+
}
|
|
111
|
+
function isExactSourceTarget(candidate, target) {
|
|
112
|
+
const candidateIdentity = normalizeSourceUrlIdentity(candidate);
|
|
113
|
+
const targetIdentity = normalizeSourceUrlIdentity(target);
|
|
114
|
+
return candidateIdentity !== null && targetIdentity !== null && candidateIdentity === targetIdentity;
|
|
115
|
+
}
|
|
116
|
+
function parseMetadataResult(value, index, queried, failed, budget) {
|
|
117
|
+
const label = `Metadata search response.results[${index}]`;
|
|
118
|
+
const input = record(value, label);
|
|
119
|
+
strictKeys(input, ["title", "url", "snippet", "engines", "score"], label);
|
|
120
|
+
const title = cleanDisplayText(input.title, `${label}.title`, MAX_METADATA_SEARCH_TITLE_UTF8_BYTES, budget);
|
|
121
|
+
if (title === "")
|
|
122
|
+
fail(`${label}.title`, "must contain visible text.");
|
|
123
|
+
const rawUrl = boundedString(input.url, `${label}.url`, MAX_URL_INTELLIGENCE_URL_UTF8_BYTES, budget);
|
|
124
|
+
const url = normalizeSourceUrlIdentity(rawUrl);
|
|
125
|
+
if (url === null)
|
|
126
|
+
fail(`${label}.url`, "must be a public HTTP or HTTPS URL without credentials or controls.");
|
|
127
|
+
const cleanedSnippet = input.snippet === null ? null : cleanDisplayText(input.snippet, `${label}.snippet`, MAX_METADATA_SEARCH_SNIPPET_UTF8_BYTES, budget);
|
|
128
|
+
const snippet = cleanedSnippet === "" ? null : cleanedSnippet;
|
|
129
|
+
const engines = uniqueEngineArray(input.engines, `${label}.engines`, budget, false);
|
|
130
|
+
for (const engine of engines) {
|
|
131
|
+
if (!queried.has(engine))
|
|
132
|
+
fail(`${label}.engines`, `contains unqueried engine ${engine}.`);
|
|
133
|
+
if (failed.has(engine))
|
|
134
|
+
fail(`${label}.engines`, `contains failed engine ${engine}.`);
|
|
135
|
+
}
|
|
136
|
+
if (typeof input.score !== "number" || !Number.isFinite(input.score) || input.score <= 0) {
|
|
137
|
+
fail(`${label}.score`, "must be a positive finite number.");
|
|
138
|
+
}
|
|
139
|
+
return Object.freeze({ title, url, snippet, engines, score: input.score });
|
|
140
|
+
}
|
|
141
|
+
function parseMetadataSearchResponse(value) {
|
|
142
|
+
const input = record(value, "Metadata search response");
|
|
143
|
+
strictKeys(input, ["query", "results", "engines_queried", "engines_failed"], "Metadata search response");
|
|
144
|
+
const budget = { bytes: 0 };
|
|
145
|
+
const query = boundedString(input.query, "Metadata search response.query", MAX_METADATA_SEARCH_QUERY_UTF8_BYTES, budget);
|
|
146
|
+
if (query === "" || query !== query.trim() || hasUnsafeControls(query)) {
|
|
147
|
+
fail("Metadata search response.query", "must be non-empty, trimmed, and free of controls.");
|
|
148
|
+
}
|
|
149
|
+
const enginesQueried = uniqueEngineArray(input.engines_queried, "Metadata search response.engines_queried", budget, false);
|
|
150
|
+
const enginesFailed = uniqueEngineArray(input.engines_failed, "Metadata search response.engines_failed", budget, true);
|
|
151
|
+
const queried = new Set(enginesQueried);
|
|
152
|
+
for (const engine of enginesFailed) {
|
|
153
|
+
if (!queried.has(engine))
|
|
154
|
+
fail("Metadata search response.engines_failed", `contains unqueried engine ${engine}.`);
|
|
155
|
+
}
|
|
156
|
+
if (!Array.isArray(input.results) || input.results.length > MAX_METADATA_SEARCH_RESULTS) {
|
|
157
|
+
fail("Metadata search response.results", `must be an array with at most ${MAX_METADATA_SEARCH_RESULTS} entries.`);
|
|
158
|
+
}
|
|
159
|
+
const failed = new Set(enginesFailed);
|
|
160
|
+
const results = Object.freeze(input.results.map((result, index) => parseMetadataResult(result, index, queried, failed, budget)));
|
|
161
|
+
return Object.freeze({
|
|
162
|
+
query,
|
|
163
|
+
results,
|
|
164
|
+
enginesQueried,
|
|
165
|
+
enginesFailed,
|
|
166
|
+
engineStatus: enginesFailed.length === 0 ? "complete" : enginesFailed.length === enginesQueried.length ? "unavailable" : "partial"
|
|
167
|
+
});
|
|
168
|
+
}
|
|
169
|
+
function rankMetadataSearchResults(results, options = {}) {
|
|
170
|
+
const limit = options.limit ?? MAX_METADATA_SEARCH_RESULTS;
|
|
171
|
+
if (!Number.isSafeInteger(limit) || limit < 0 || limit > MAX_METADATA_SEARCH_RESULTS) {
|
|
172
|
+
throw new RangeError(`Metadata search rank limit must be from 0 through ${MAX_METADATA_SEARCH_RESULTS}.`);
|
|
173
|
+
}
|
|
174
|
+
if (limit === 0)
|
|
175
|
+
return Object.freeze([]);
|
|
176
|
+
const targetIdentity = options.targetUrl === undefined ? null : normalizeSourceUrlIdentity(options.targetUrl);
|
|
177
|
+
const decorated = results.flatMap((result, index) => {
|
|
178
|
+
const sourceIdentity = normalizeSourceUrlIdentity(result.url);
|
|
179
|
+
return sourceIdentity === null ? [] : [{ result, index, sourceIdentity, exactTarget: sourceIdentity === targetIdentity }];
|
|
180
|
+
}).toSorted((left, right) => Number(right.exactTarget) - Number(left.exactTarget) || right.result.score - left.result.score || left.index - right.index);
|
|
181
|
+
const seen = new Set;
|
|
182
|
+
const ranked = [];
|
|
183
|
+
for (const item of decorated) {
|
|
184
|
+
if (seen.has(item.sourceIdentity))
|
|
185
|
+
continue;
|
|
186
|
+
seen.add(item.sourceIdentity);
|
|
187
|
+
ranked.push(Object.freeze({
|
|
188
|
+
...item.result,
|
|
189
|
+
rank: ranked.length + 1,
|
|
190
|
+
sourceIdentity: item.sourceIdentity,
|
|
191
|
+
exactTarget: item.exactTarget
|
|
192
|
+
}));
|
|
193
|
+
if (ranked.length === limit)
|
|
194
|
+
break;
|
|
195
|
+
}
|
|
196
|
+
return Object.freeze(ranked);
|
|
197
|
+
}
|
|
198
|
+
function isWhitespace(character) {
|
|
199
|
+
return character === " " || character === "\t" || character === "\r" || character === `
|
|
200
|
+
`;
|
|
201
|
+
}
|
|
202
|
+
function parseLinkFormat(source) {
|
|
203
|
+
let cursor = 0;
|
|
204
|
+
const links = [];
|
|
205
|
+
const skipWhitespace = () => {
|
|
206
|
+
while (isWhitespace(source[cursor]))
|
|
207
|
+
cursor += 1;
|
|
208
|
+
};
|
|
209
|
+
const syntax = (message) => fail("Archive.today TimeMap", `${message} at offset ${cursor}.`);
|
|
210
|
+
skipWhitespace();
|
|
211
|
+
while (cursor < source.length) {
|
|
212
|
+
if (links.length >= MAX_ARCHIVE_TIMEMAP_ENTRIES) {
|
|
213
|
+
fail("Archive.today TimeMap", `may contain at most ${MAX_ARCHIVE_TIMEMAP_ENTRIES} entries.`);
|
|
214
|
+
}
|
|
215
|
+
if (source[cursor] !== "<")
|
|
216
|
+
syntax("must start each link with '<'");
|
|
217
|
+
cursor += 1;
|
|
218
|
+
const targetStart = cursor;
|
|
219
|
+
while (cursor < source.length && source[cursor] !== ">") {
|
|
220
|
+
const character = source[cursor];
|
|
221
|
+
if (character === "<" || character === '"' || character === "\\" || isWhitespace(character)) {
|
|
222
|
+
syntax("contains an invalid target character");
|
|
223
|
+
}
|
|
224
|
+
cursor += 1;
|
|
225
|
+
}
|
|
226
|
+
if (cursor >= source.length)
|
|
227
|
+
syntax("has an unterminated target");
|
|
228
|
+
const target = source.slice(targetStart, cursor);
|
|
229
|
+
cursor += 1;
|
|
230
|
+
if (target === "" || Buffer.byteLength(target, "utf8") > MAX_URL_INTELLIGENCE_URL_UTF8_BYTES) {
|
|
231
|
+
syntax("has an empty or oversized target");
|
|
232
|
+
}
|
|
233
|
+
const parameters = new Map;
|
|
234
|
+
skipWhitespace();
|
|
235
|
+
while (source[cursor] === ";") {
|
|
236
|
+
if (parameters.size >= MAX_ARCHIVE_TIMEMAP_PARAMETERS_PER_ENTRY) {
|
|
237
|
+
fail("Archive.today TimeMap", `entries may contain at most ${MAX_ARCHIVE_TIMEMAP_PARAMETERS_PER_ENTRY} parameters.`);
|
|
238
|
+
}
|
|
239
|
+
cursor += 1;
|
|
240
|
+
skipWhitespace();
|
|
241
|
+
const nameStart = cursor;
|
|
242
|
+
while (tokenCharacter.test(source[cursor] ?? ""))
|
|
243
|
+
cursor += 1;
|
|
244
|
+
if (cursor === nameStart)
|
|
245
|
+
syntax("has an invalid parameter name");
|
|
246
|
+
const name = source.slice(nameStart, cursor).toLowerCase();
|
|
247
|
+
skipWhitespace();
|
|
248
|
+
if (source[cursor] !== "=")
|
|
249
|
+
syntax("requires '=' after a parameter name");
|
|
250
|
+
cursor += 1;
|
|
251
|
+
skipWhitespace();
|
|
252
|
+
let parameterValue = "";
|
|
253
|
+
if (source[cursor] === '"') {
|
|
254
|
+
cursor += 1;
|
|
255
|
+
let terminated = false;
|
|
256
|
+
while (cursor < source.length) {
|
|
257
|
+
const character = source[cursor] ?? "";
|
|
258
|
+
cursor += 1;
|
|
259
|
+
if (character === '"') {
|
|
260
|
+
terminated = true;
|
|
261
|
+
break;
|
|
262
|
+
}
|
|
263
|
+
if (character === "\\") {
|
|
264
|
+
if (cursor >= source.length)
|
|
265
|
+
syntax("has an unterminated quoted escape");
|
|
266
|
+
const escaped = source[cursor] ?? "";
|
|
267
|
+
if (hasUnsafeControls(escaped))
|
|
268
|
+
syntax("has a control in a quoted escape");
|
|
269
|
+
parameterValue += escaped;
|
|
270
|
+
cursor += 1;
|
|
271
|
+
} else {
|
|
272
|
+
if (hasUnsafeControls(character))
|
|
273
|
+
syntax("has a control in a quoted value");
|
|
274
|
+
parameterValue += character;
|
|
275
|
+
}
|
|
276
|
+
}
|
|
277
|
+
if (!terminated)
|
|
278
|
+
syntax("has an unterminated quoted value");
|
|
279
|
+
} else {
|
|
280
|
+
const valueStart = cursor;
|
|
281
|
+
while (tokenCharacter.test(source[cursor] ?? ""))
|
|
282
|
+
cursor += 1;
|
|
283
|
+
if (cursor === valueStart)
|
|
284
|
+
syntax("has an invalid parameter value");
|
|
285
|
+
parameterValue = source.slice(valueStart, cursor);
|
|
286
|
+
}
|
|
287
|
+
if (Buffer.byteLength(parameterValue, "utf8") > MAX_URL_INTELLIGENCE_URL_UTF8_BYTES) {
|
|
288
|
+
syntax("has an oversized parameter value");
|
|
289
|
+
}
|
|
290
|
+
if (parameters.has(name))
|
|
291
|
+
syntax(`repeats parameter ${name}`);
|
|
292
|
+
parameters.set(name, parameterValue);
|
|
293
|
+
skipWhitespace();
|
|
294
|
+
}
|
|
295
|
+
links.push(Object.freeze({ target, parameters }));
|
|
296
|
+
if (cursor === source.length)
|
|
297
|
+
break;
|
|
298
|
+
if (source[cursor] !== ",")
|
|
299
|
+
syntax("must separate links with ','");
|
|
300
|
+
cursor += 1;
|
|
301
|
+
skipWhitespace();
|
|
302
|
+
if (cursor === source.length)
|
|
303
|
+
syntax("must not end with ','");
|
|
304
|
+
}
|
|
305
|
+
return Object.freeze(links);
|
|
306
|
+
}
|
|
307
|
+
var months = new Map([
|
|
308
|
+
["Jan", 0],
|
|
309
|
+
["Feb", 1],
|
|
310
|
+
["Mar", 2],
|
|
311
|
+
["Apr", 3],
|
|
312
|
+
["May", 4],
|
|
313
|
+
["Jun", 5],
|
|
314
|
+
["Jul", 6],
|
|
315
|
+
["Aug", 7],
|
|
316
|
+
["Sep", 8],
|
|
317
|
+
["Oct", 9],
|
|
318
|
+
["Nov", 10],
|
|
319
|
+
["Dec", 11]
|
|
320
|
+
]);
|
|
321
|
+
var weekdays = ["Sun", "Mon", "Tue", "Wed", "Thu", "Fri", "Sat"];
|
|
322
|
+
function epochFromTimestamp(timestamp, label) {
|
|
323
|
+
const match = /^(\d{4})(\d{2})(\d{2})(\d{2})(\d{2})(\d{2})$/u.exec(timestamp);
|
|
324
|
+
if (match === null)
|
|
325
|
+
return fail(label, "must contain a 14-digit UTC timestamp.");
|
|
326
|
+
const [year, month, day, hour, minute, second] = match.slice(1).map(Number);
|
|
327
|
+
const epoch = Date.UTC(year, month - 1, day, hour, minute, second);
|
|
328
|
+
const date = new Date(epoch);
|
|
329
|
+
if (year < 1900 || date.getUTCFullYear() !== year || date.getUTCMonth() !== month - 1 || date.getUTCDate() !== day || date.getUTCHours() !== hour || date.getUTCMinutes() !== minute || date.getUTCSeconds() !== second)
|
|
330
|
+
return fail(label, "contains an invalid UTC timestamp.");
|
|
331
|
+
return epoch;
|
|
332
|
+
}
|
|
333
|
+
function epochFromHttpDate(value, label) {
|
|
334
|
+
const match = /^(Sun|Mon|Tue|Wed|Thu|Fri|Sat), (\d{2}) (Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec) (\d{4}) (\d{2}):(\d{2}):(\d{2}) GMT$/u.exec(value);
|
|
335
|
+
if (match === null)
|
|
336
|
+
return fail(label, "must be an RFC 1123 date in GMT.");
|
|
337
|
+
const month = months.get(match[3] ?? "");
|
|
338
|
+
if (month === undefined)
|
|
339
|
+
return fail(label, "has an invalid month.");
|
|
340
|
+
const year = Number(match[4]);
|
|
341
|
+
const day = Number(match[2]);
|
|
342
|
+
const hour = Number(match[5]);
|
|
343
|
+
const minute = Number(match[6]);
|
|
344
|
+
const second = Number(match[7]);
|
|
345
|
+
const epoch = Date.UTC(year, month, day, hour, minute, second);
|
|
346
|
+
const date = new Date(epoch);
|
|
347
|
+
if (year < 1900 || date.getUTCFullYear() !== year || date.getUTCMonth() !== month || date.getUTCDate() !== day || date.getUTCHours() !== hour || date.getUTCMinutes() !== minute || date.getUTCSeconds() !== second || weekdays[date.getUTCDay()] !== match[1])
|
|
348
|
+
return fail(label, "contains an invalid RFC 1123 date.");
|
|
349
|
+
return epoch;
|
|
350
|
+
}
|
|
351
|
+
function checkedNow(now) {
|
|
352
|
+
if (!(now instanceof Date) || !Number.isFinite(now.getTime())) {
|
|
353
|
+
return fail("Archive.today now", "must be a valid injected Date.");
|
|
354
|
+
}
|
|
355
|
+
return now.getTime();
|
|
356
|
+
}
|
|
357
|
+
function parsedArchiveTodayMementoUrl(value, originalIdentity, nowEpoch, expectedEpoch) {
|
|
358
|
+
const raw = value instanceof URL ? value.href : value;
|
|
359
|
+
if (typeof raw !== "string" || raw !== raw.trim() || hasUnsafeControls(raw) || Buffer.byteLength(raw, "utf8") > MAX_URL_INTELLIGENCE_URL_UTF8_BYTES) {
|
|
360
|
+
return fail("Archive.today memento URL", "must be a bounded control-free URL.");
|
|
361
|
+
}
|
|
362
|
+
let archiveUrl;
|
|
363
|
+
try {
|
|
364
|
+
archiveUrl = new URL(raw);
|
|
365
|
+
} catch {
|
|
366
|
+
return fail("Archive.today memento URL", "must be a valid URL.");
|
|
367
|
+
}
|
|
368
|
+
if (archiveUrl.protocol !== "http:" && archiveUrl.protocol !== "https:" || archiveUrl.username !== "" || archiveUrl.password !== "" || archiveUrl.port !== "" || archiveUrl.hash !== "" || !archiveTodayHosts.has(archiveUrl.hostname))
|
|
369
|
+
return fail("Archive.today memento URL", "must use an allowlisted archive host without credentials, a port, or a fragment.");
|
|
370
|
+
const path = /^\/(\d{14})\/(https?:\/\/.*)$/u.exec(archiveUrl.pathname);
|
|
371
|
+
if (path === null || path[1] === undefined || path[2] === undefined) {
|
|
372
|
+
return fail("Archive.today memento URL", "must use a timestamped read-only snapshot path.");
|
|
373
|
+
}
|
|
374
|
+
const timestamp = path[1];
|
|
375
|
+
const capturedEpoch = epochFromTimestamp(timestamp, "Archive.today memento URL timestamp");
|
|
376
|
+
if (capturedEpoch > nowEpoch)
|
|
377
|
+
fail("Archive.today memento URL timestamp", "must not be in the future.");
|
|
378
|
+
if (expectedEpoch !== undefined && capturedEpoch !== expectedEpoch) {
|
|
379
|
+
fail("Archive.today memento URL timestamp", "must equal its Memento datetime.");
|
|
380
|
+
}
|
|
381
|
+
const embeddedIdentity = normalizeSourceUrlIdentity(`${path[2]}${archiveUrl.search}`);
|
|
382
|
+
if (embeddedIdentity === null || embeddedIdentity !== originalIdentity) {
|
|
383
|
+
fail("Archive.today memento URL", "must embed the exact bound original URL.");
|
|
384
|
+
}
|
|
385
|
+
const archiveHost = archiveUrl.hostname;
|
|
386
|
+
archiveUrl.protocol = "https:";
|
|
387
|
+
return Object.freeze({
|
|
388
|
+
url: archiveUrl.href,
|
|
389
|
+
archiveHost,
|
|
390
|
+
timestamp,
|
|
391
|
+
capturedAt: new Date(capturedEpoch).toISOString(),
|
|
392
|
+
originalUrl: originalIdentity
|
|
393
|
+
});
|
|
394
|
+
}
|
|
395
|
+
function parseArchiveTodayMementoUrl(value, options) {
|
|
396
|
+
const originalIdentity = normalizeSourceUrlIdentity(options.originalUrl);
|
|
397
|
+
if (originalIdentity === null)
|
|
398
|
+
fail("Archive.today original URL", "must be a public HTTP or HTTPS URL without credentials.");
|
|
399
|
+
return parsedArchiveTodayMementoUrl(value, originalIdentity, checkedNow(options.now));
|
|
400
|
+
}
|
|
401
|
+
function selectNewestArchiveTodayMemento(mementos) {
|
|
402
|
+
return mementos.toSorted((left, right) => right.timestamp.localeCompare(left.timestamp) || left.url.localeCompare(right.url))[0] ?? null;
|
|
403
|
+
}
|
|
404
|
+
function parseArchiveTodayTimeMap(value, options) {
|
|
405
|
+
if (typeof value !== "string")
|
|
406
|
+
fail("Archive.today TimeMap", "must be text.");
|
|
407
|
+
if (Buffer.byteLength(value, "utf8") > MAX_ARCHIVE_TIMEMAP_UTF8_BYTES) {
|
|
408
|
+
fail("Archive.today TimeMap", `must be at most ${MAX_ARCHIVE_TIMEMAP_UTF8_BYTES} UTF-8 bytes.`);
|
|
409
|
+
}
|
|
410
|
+
if (hasUnsafeControls(value, true, true)) {
|
|
411
|
+
fail("Archive.today TimeMap", "contains forbidden controls.");
|
|
412
|
+
}
|
|
413
|
+
const originalIdentity = normalizeSourceUrlIdentity(options.originalUrl);
|
|
414
|
+
if (originalIdentity === null)
|
|
415
|
+
fail("Archive.today original URL", "must be a public HTTP or HTTPS URL without credentials.");
|
|
416
|
+
const nowEpoch = checkedNow(options.now);
|
|
417
|
+
const links = parseLinkFormat(value);
|
|
418
|
+
const originals = links.filter((link) => link.parameters.get("rel")?.toLowerCase() === "original");
|
|
419
|
+
if (originals.length !== 1)
|
|
420
|
+
fail("Archive.today TimeMap", "must contain exactly one rel=original link.");
|
|
421
|
+
const declaredOriginal = normalizeSourceUrlIdentity(originals[0]?.target ?? "");
|
|
422
|
+
if (declaredOriginal === null || declaredOriginal !== originalIdentity) {
|
|
423
|
+
fail("Archive.today TimeMap rel=original", "must exactly match the requested original URL.");
|
|
424
|
+
}
|
|
425
|
+
const mementos = [];
|
|
426
|
+
const seen = new Set;
|
|
427
|
+
for (const [index, link] of links.entries()) {
|
|
428
|
+
const relations = (link.parameters.get("rel") ?? "").trim().toLowerCase().split(/[ \t]+/u).filter(Boolean);
|
|
429
|
+
if (!relations.includes("memento"))
|
|
430
|
+
continue;
|
|
431
|
+
if (relations.includes("original"))
|
|
432
|
+
fail(`Archive.today TimeMap entry ${index}`, "cannot be both original and memento.");
|
|
433
|
+
const datetime = link.parameters.get("datetime");
|
|
434
|
+
if (datetime === undefined)
|
|
435
|
+
fail(`Archive.today TimeMap entry ${index}`, "requires a datetime parameter.");
|
|
436
|
+
const expectedEpoch = epochFromHttpDate(datetime, `Archive.today TimeMap entry ${index} datetime`);
|
|
437
|
+
if (expectedEpoch > nowEpoch)
|
|
438
|
+
fail(`Archive.today TimeMap entry ${index} datetime`, "must not be in the future.");
|
|
439
|
+
const memento = parsedArchiveTodayMementoUrl(link.target, originalIdentity, nowEpoch, expectedEpoch);
|
|
440
|
+
if (!seen.has(memento.url)) {
|
|
441
|
+
seen.add(memento.url);
|
|
442
|
+
mementos.push(memento);
|
|
443
|
+
}
|
|
444
|
+
}
|
|
445
|
+
const sorted = Object.freeze(mementos.toSorted((left, right) => right.timestamp.localeCompare(left.timestamp) || left.url.localeCompare(right.url)));
|
|
446
|
+
return Object.freeze({
|
|
447
|
+
originalUrl: originalIdentity,
|
|
448
|
+
mementos: sorted,
|
|
449
|
+
newest: selectNewestArchiveTodayMemento(sorted)
|
|
450
|
+
});
|
|
451
|
+
}
|
|
452
|
+
|
|
453
|
+
// src/clip/archive-today.ts
|
|
454
|
+
var archiveTodayDiscoveryOrigin = "https://archive.ph";
|
|
455
|
+
var archiveTodayDiscoveryPrefix = `${archiveTodayDiscoveryOrigin}/newest/`;
|
|
456
|
+
var defaultUserAgent = "@hraness/kb archive-today fallback";
|
|
457
|
+
var defaultTimeoutMs = 1e4;
|
|
458
|
+
var defaultMaximumBytes = 8 * 1024 * 1024;
|
|
459
|
+
var discoveryMaximumBytes = 256 * 1024;
|
|
460
|
+
var maximumSourceUrlBytes = 16 * 1024;
|
|
461
|
+
var maximumUserAgentBytes = 512;
|
|
462
|
+
var failureMessages = Object.freeze({
|
|
463
|
+
"invalid-source-url": "Archive.today requires a bounded public HTTP(S) source URL.",
|
|
464
|
+
"invalid-snapshot": "Archive.today returned a snapshot that could not be bound to the source URL.",
|
|
465
|
+
"invalid-clock": "Archive.today validation requires a valid current time.",
|
|
466
|
+
"invalid-options": "Archive.today adapter options are outside their supported bounds.",
|
|
467
|
+
"request-failed": "Archive.today could not be reached.",
|
|
468
|
+
"not-found": "The validated Archive.today snapshot is no longer available.",
|
|
469
|
+
throttled: "Archive.today throttled the request.",
|
|
470
|
+
"unexpected-response": "Archive.today returned an unexpected response.",
|
|
471
|
+
"unsupported-content": "Archive.today did not return a non-empty HTML document."
|
|
472
|
+
});
|
|
473
|
+
|
|
474
|
+
class ArchiveTodayFailure extends Error {
|
|
475
|
+
code;
|
|
476
|
+
constructor(code) {
|
|
477
|
+
super(failureMessages[code]);
|
|
478
|
+
this.name = "ArchiveTodayFailure";
|
|
479
|
+
this.code = code;
|
|
480
|
+
}
|
|
481
|
+
}
|
|
482
|
+
function utf8Length(value) {
|
|
483
|
+
return new TextEncoder().encode(value).byteLength;
|
|
484
|
+
}
|
|
485
|
+
function containsUnsafeLocationControl(value) {
|
|
486
|
+
for (const character of value) {
|
|
487
|
+
const codePoint = character.codePointAt(0) ?? 0;
|
|
488
|
+
if (codePoint <= 31 || codePoint >= 127 && codePoint <= 159 || codePoint === 1564 || codePoint === 8206 || codePoint === 8207 || codePoint >= 8234 && codePoint <= 8238 || codePoint >= 8294 && codePoint <= 8297)
|
|
489
|
+
return true;
|
|
490
|
+
}
|
|
491
|
+
return false;
|
|
492
|
+
}
|
|
493
|
+
function boundedInteger(value, fallback, minimum, maximum) {
|
|
494
|
+
const selected = value ?? fallback;
|
|
495
|
+
if (!Number.isSafeInteger(selected) || selected < minimum || selected > maximum) {
|
|
496
|
+
throw new ArchiveTodayFailure("invalid-options");
|
|
497
|
+
}
|
|
498
|
+
return selected;
|
|
499
|
+
}
|
|
500
|
+
function resolveDependencies(dependencies) {
|
|
501
|
+
let now;
|
|
502
|
+
try {
|
|
503
|
+
now = dependencies.now?.() ?? new Date;
|
|
504
|
+
} catch {
|
|
505
|
+
throw new ArchiveTodayFailure("invalid-clock");
|
|
506
|
+
}
|
|
507
|
+
if (!(now instanceof Date) || !Number.isFinite(now.getTime())) {
|
|
508
|
+
throw new ArchiveTodayFailure("invalid-clock");
|
|
509
|
+
}
|
|
510
|
+
const userAgent = dependencies.userAgent ?? defaultUserAgent;
|
|
511
|
+
if (userAgent.trim() === "" || utf8Length(userAgent) > maximumUserAgentBytes || containsUnsafeLocationControl(userAgent)) {
|
|
512
|
+
throw new ArchiveTodayFailure("invalid-options");
|
|
513
|
+
}
|
|
514
|
+
return Object.freeze({
|
|
515
|
+
fetch: dependencies.fetch ?? safeFetch,
|
|
516
|
+
assertNetworkUrl: dependencies.assertNetworkUrl ?? assertSafeNetworkUrl,
|
|
517
|
+
now: new Date(now.getTime()),
|
|
518
|
+
monotonicNow: dependencies.monotonicNow ?? Date.now,
|
|
519
|
+
timeoutMs: boundedInteger(dependencies.timeoutMs, defaultTimeoutMs, 250, 60000),
|
|
520
|
+
maxBytes: boundedInteger(dependencies.maxBytes, defaultMaximumBytes, 1, 64 * 1024 * 1024),
|
|
521
|
+
userAgent
|
|
522
|
+
});
|
|
523
|
+
}
|
|
524
|
+
function monotonicTime(resolved) {
|
|
525
|
+
let value;
|
|
526
|
+
try {
|
|
527
|
+
value = resolved.monotonicNow();
|
|
528
|
+
} catch {
|
|
529
|
+
throw new ArchiveTodayFailure("invalid-clock");
|
|
530
|
+
}
|
|
531
|
+
if (!Number.isFinite(value))
|
|
532
|
+
throw new ArchiveTodayFailure("invalid-clock");
|
|
533
|
+
return value;
|
|
534
|
+
}
|
|
535
|
+
function remainingTimeout(deadline, resolved) {
|
|
536
|
+
return Math.floor(deadline - monotonicTime(resolved));
|
|
537
|
+
}
|
|
538
|
+
function normalizeSourceUrl(value) {
|
|
539
|
+
const identity = normalizeSourceUrlIdentity(value);
|
|
540
|
+
if (identity === null || utf8Length(identity) > maximumSourceUrlBytes) {
|
|
541
|
+
throw new ArchiveTodayFailure("invalid-source-url");
|
|
542
|
+
}
|
|
543
|
+
if (sanitizeArtifactUrl(identity) !== identity) {
|
|
544
|
+
throw new ArchiveTodayFailure("invalid-source-url");
|
|
545
|
+
}
|
|
546
|
+
return new URL(identity);
|
|
547
|
+
}
|
|
548
|
+
function parseMemento(value, source, now) {
|
|
549
|
+
try {
|
|
550
|
+
return parseArchiveTodayMementoUrl(value, { originalUrl: source, now });
|
|
551
|
+
} catch {
|
|
552
|
+
throw new ArchiveTodayFailure("invalid-snapshot");
|
|
553
|
+
}
|
|
554
|
+
}
|
|
555
|
+
function snapshotFromMemento(memento) {
|
|
556
|
+
return Object.freeze({
|
|
557
|
+
url: memento.url,
|
|
558
|
+
capturedAt: memento.capturedAt,
|
|
559
|
+
sourceUrl: memento.originalUrl,
|
|
560
|
+
discovery: "newest"
|
|
561
|
+
});
|
|
562
|
+
}
|
|
563
|
+
function unavailable(sourceUrl, reason) {
|
|
564
|
+
return Object.freeze({ status: "unavailable", sourceUrl, reason });
|
|
565
|
+
}
|
|
566
|
+
async function discoverArchiveTodaySnapshot(sourceUrl, dependencies = {}) {
|
|
567
|
+
if (dependencies.fetch !== undefined && dependencies.assertNetworkUrl === undefined) {
|
|
568
|
+
throw new ArchiveTodayFailure("invalid-options");
|
|
569
|
+
}
|
|
570
|
+
const source = normalizeSourceUrl(sourceUrl);
|
|
571
|
+
const resolved = resolveDependencies(dependencies);
|
|
572
|
+
const deadline = monotonicTime(resolved) + resolved.timeoutMs;
|
|
573
|
+
try {
|
|
574
|
+
await resolved.assertNetworkUrl(source, false, resolved.timeoutMs);
|
|
575
|
+
} catch {
|
|
576
|
+
return unavailable(source.href, "request-failed");
|
|
577
|
+
}
|
|
578
|
+
const remaining = remainingTimeout(deadline, resolved);
|
|
579
|
+
if (remaining <= 0)
|
|
580
|
+
return unavailable(source.href, "request-failed");
|
|
581
|
+
const lookupUrl = new URL(`${archiveTodayDiscoveryPrefix}${source.href}`);
|
|
582
|
+
let response;
|
|
583
|
+
try {
|
|
584
|
+
response = await resolved.fetch(lookupUrl, {
|
|
585
|
+
timeoutMs: remaining,
|
|
586
|
+
maxBytes: discoveryMaximumBytes,
|
|
587
|
+
allowPrivateNetwork: false,
|
|
588
|
+
userAgent: resolved.userAgent,
|
|
589
|
+
accept: "text/html,application/xhtml+xml;q=0.9,*/*;q=0.1",
|
|
590
|
+
retries: 0,
|
|
591
|
+
maxRedirects: 0,
|
|
592
|
+
redirect: "manual",
|
|
593
|
+
acceptStatuses: [403, 404, 429]
|
|
594
|
+
});
|
|
595
|
+
} catch {
|
|
596
|
+
return unavailable(source.href, "request-failed");
|
|
597
|
+
}
|
|
598
|
+
if (response.status === 404) {
|
|
599
|
+
return Object.freeze({ status: "not-found", sourceUrl: source.href });
|
|
600
|
+
}
|
|
601
|
+
if (response.status === 403 || response.status === 429) {
|
|
602
|
+
return Object.freeze({ status: "throttled", sourceUrl: source.href });
|
|
603
|
+
}
|
|
604
|
+
if (response.status < 300 || response.status >= 400) {
|
|
605
|
+
return unavailable(source.href, "unexpected-response");
|
|
606
|
+
}
|
|
607
|
+
if (typeof response.location !== "string" || response.location.trim() === "") {
|
|
608
|
+
return unavailable(source.href, "invalid-snapshot");
|
|
609
|
+
}
|
|
610
|
+
try {
|
|
611
|
+
if (response.location !== response.location.trim() || containsUnsafeLocationControl(response.location) || utf8Length(response.location) > MAX_URL_INTELLIGENCE_URL_UTF8_BYTES)
|
|
612
|
+
throw new ArchiveTodayFailure("invalid-snapshot");
|
|
613
|
+
const location = new URL(response.location, lookupUrl);
|
|
614
|
+
const memento = parseMemento(location, source, resolved.now);
|
|
615
|
+
return Object.freeze({
|
|
616
|
+
status: "found",
|
|
617
|
+
sourceUrl: source.href,
|
|
618
|
+
snapshot: snapshotFromMemento(memento)
|
|
619
|
+
});
|
|
620
|
+
} catch {
|
|
621
|
+
return unavailable(source.href, "invalid-snapshot");
|
|
622
|
+
}
|
|
623
|
+
}
|
|
624
|
+
function isHtmlContentType(value) {
|
|
625
|
+
if (value === null)
|
|
626
|
+
return false;
|
|
627
|
+
const mediaType = value.split(";", 1)[0]?.trim().toLowerCase();
|
|
628
|
+
return mediaType === "text/html" || mediaType === "application/xhtml+xml";
|
|
629
|
+
}
|
|
630
|
+
async function acquireArchiveTodaySnapshot(sourceUrl, snapshotUrl, dependencies = {}) {
|
|
631
|
+
if (dependencies.fetch !== undefined && dependencies.assertNetworkUrl === undefined) {
|
|
632
|
+
throw new ArchiveTodayFailure("invalid-options");
|
|
633
|
+
}
|
|
634
|
+
const source = normalizeSourceUrl(sourceUrl);
|
|
635
|
+
const resolved = resolveDependencies(dependencies);
|
|
636
|
+
const requested = parseMemento(snapshotUrl, source, resolved.now);
|
|
637
|
+
let current = new URL(requested.url);
|
|
638
|
+
let response = null;
|
|
639
|
+
const deadline = monotonicTime(resolved) + resolved.timeoutMs;
|
|
640
|
+
try {
|
|
641
|
+
await resolved.assertNetworkUrl(source, false, resolved.timeoutMs);
|
|
642
|
+
} catch {
|
|
643
|
+
throw new ArchiveTodayFailure("request-failed");
|
|
644
|
+
}
|
|
645
|
+
for (let redirects = 0;redirects <= 4; redirects += 1) {
|
|
646
|
+
const remaining = remainingTimeout(deadline, resolved);
|
|
647
|
+
if (remaining <= 0)
|
|
648
|
+
throw new ArchiveTodayFailure("request-failed");
|
|
649
|
+
try {
|
|
650
|
+
response = await resolved.fetch(current, {
|
|
651
|
+
timeoutMs: remaining,
|
|
652
|
+
maxBytes: resolved.maxBytes,
|
|
653
|
+
allowPrivateNetwork: false,
|
|
654
|
+
userAgent: resolved.userAgent,
|
|
655
|
+
accept: "text/html,application/xhtml+xml;q=0.9",
|
|
656
|
+
retries: 0,
|
|
657
|
+
maxRedirects: 0,
|
|
658
|
+
redirect: "manual",
|
|
659
|
+
acceptStatuses: [403, 404, 429]
|
|
660
|
+
});
|
|
661
|
+
} catch {
|
|
662
|
+
throw new ArchiveTodayFailure("request-failed");
|
|
663
|
+
}
|
|
664
|
+
if (response.finalUrl.href !== current.href)
|
|
665
|
+
throw new ArchiveTodayFailure("invalid-snapshot");
|
|
666
|
+
if (response.status < 300 || response.status >= 400)
|
|
667
|
+
break;
|
|
668
|
+
if (redirects === 4 || typeof response.location !== "string") {
|
|
669
|
+
throw new ArchiveTodayFailure("invalid-snapshot");
|
|
670
|
+
}
|
|
671
|
+
if (response.location !== response.location.trim() || containsUnsafeLocationControl(response.location) || utf8Length(response.location) > MAX_URL_INTELLIGENCE_URL_UTF8_BYTES)
|
|
672
|
+
throw new ArchiveTodayFailure("invalid-snapshot");
|
|
673
|
+
let next;
|
|
674
|
+
try {
|
|
675
|
+
next = new URL(response.location, current);
|
|
676
|
+
} catch {
|
|
677
|
+
throw new ArchiveTodayFailure("invalid-snapshot");
|
|
678
|
+
}
|
|
679
|
+
const redirected = parseMemento(next, source, resolved.now);
|
|
680
|
+
if (redirected.timestamp !== requested.timestamp)
|
|
681
|
+
throw new ArchiveTodayFailure("invalid-snapshot");
|
|
682
|
+
current = new URL(redirected.url);
|
|
683
|
+
}
|
|
684
|
+
if (response === null)
|
|
685
|
+
throw new ArchiveTodayFailure("request-failed");
|
|
686
|
+
if (response.status === 403 || response.status === 429) {
|
|
687
|
+
throw new ArchiveTodayFailure("throttled");
|
|
688
|
+
}
|
|
689
|
+
if (response.status === 404)
|
|
690
|
+
throw new ArchiveTodayFailure("not-found");
|
|
691
|
+
if (response.status < 200 || response.status >= 300) {
|
|
692
|
+
throw new ArchiveTodayFailure("unexpected-response");
|
|
693
|
+
}
|
|
694
|
+
const finalMemento = parseMemento(current, source, resolved.now);
|
|
695
|
+
if (finalMemento.timestamp !== requested.timestamp)
|
|
696
|
+
throw new ArchiveTodayFailure("invalid-snapshot");
|
|
697
|
+
if (!isHtmlContentType(response.contentType)) {
|
|
698
|
+
throw new ArchiveTodayFailure("unsupported-content");
|
|
699
|
+
}
|
|
700
|
+
const body = decodeBytes(response.bytes, response.contentType);
|
|
701
|
+
if (body.trim() === "")
|
|
702
|
+
throw new ArchiveTodayFailure("unsupported-content");
|
|
703
|
+
return Object.freeze({
|
|
704
|
+
body,
|
|
705
|
+
contentType: response.contentType,
|
|
706
|
+
finalUrl: new URL(finalMemento.url),
|
|
707
|
+
method: "archive-is",
|
|
708
|
+
warnings: Object.freeze(["Captured from a validated Archive.today snapshot."])
|
|
709
|
+
});
|
|
710
|
+
}
|
|
711
|
+
|
|
712
|
+
export { ARCHIVE_TODAY_HOSTS, normalizeSourceUrlIdentity, isExactSourceTarget, parseMetadataSearchResponse, rankMetadataSearchResults, parseArchiveTodayMementoUrl, selectNewestArchiveTodayMemento, parseArchiveTodayTimeMap, ArchiveTodayFailure, discoverArchiveTodaySnapshot, acquireArchiveTodaySnapshot };
|