@hraness/kb 0.17.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (200) hide show
  1. package/DISCLOSURE +43 -0
  2. package/LICENSE +21 -0
  3. package/README.md +545 -0
  4. package/dist/agent-context.js +35 -0
  5. package/dist/agent-guide-audit.js +18 -0
  6. package/dist/attachments.js +19 -0
  7. package/dist/authoring.js +33 -0
  8. package/dist/benchmark.js +19 -0
  9. package/dist/capture.js +49 -0
  10. package/dist/cli.js +3646 -0
  11. package/dist/clip/acquire.js +57 -0
  12. package/dist/clip/args.js +21 -0
  13. package/dist/clip/bounded-byte-buffer.js +9 -0
  14. package/dist/clip/browser-profiles.js +17 -0
  15. package/dist/clip/bundle-reader.js +11 -0
  16. package/dist/clip/cli.js +27 -0
  17. package/dist/clip/cookies.js +21 -0
  18. package/dist/clip/defuddle-worker.js +84 -0
  19. package/dist/clip/doctor.js +32 -0
  20. package/dist/clip/jobs.js +784 -0
  21. package/dist/clip/network-proxy.js +9 -0
  22. package/dist/clip/network.js +28 -0
  23. package/dist/clip/persist.js +30 -0
  24. package/dist/clip/refresh.js +11 -0
  25. package/dist/clip/terminal.js +11 -0
  26. package/dist/evaluation-builder.js +12238 -0
  27. package/dist/evaluation-kb.js +26 -0
  28. package/dist/evaluation.js +35 -0
  29. package/dist/git.js +39 -0
  30. package/dist/graph.js +39 -0
  31. package/dist/index-01jj6rbv.js +1103 -0
  32. package/dist/index-06c9ctr6.js +738 -0
  33. package/dist/index-0kavxzqj.js +2480 -0
  34. package/dist/index-1gwbassd.js +743 -0
  35. package/dist/index-1n418kb9.js +827 -0
  36. package/dist/index-1vrd1rmn.js +56 -0
  37. package/dist/index-1xxnjn0d.js +171 -0
  38. package/dist/index-3rm7cz6h.js +373 -0
  39. package/dist/index-3v2z4f0q.js +328 -0
  40. package/dist/index-48pz4jpc.js +392 -0
  41. package/dist/index-4j3tt0c3.js +234 -0
  42. package/dist/index-5n05se68.js +307 -0
  43. package/dist/index-5vwpzb5a.js +764 -0
  44. package/dist/index-6g2pv9d2.js +370 -0
  45. package/dist/index-7fzc32gf.js +712 -0
  46. package/dist/index-84x0vjjp.js +432 -0
  47. package/dist/index-adx6khj5.js +488 -0
  48. package/dist/index-b88v3vtm.js +805 -0
  49. package/dist/index-cv6fh7z5.js +462 -0
  50. package/dist/index-cxfrakt7.js +1067 -0
  51. package/dist/index-d13v9ckt.js +150 -0
  52. package/dist/index-dycjbhw2.js +1943 -0
  53. package/dist/index-dyqwejk5.js +531 -0
  54. package/dist/index-e5fbsywq.js +658 -0
  55. package/dist/index-f984hw45.js +927 -0
  56. package/dist/index-gh719d91.js +63 -0
  57. package/dist/index-hgve9rh2.js +2046 -0
  58. package/dist/index-hya40gb2.js +474 -0
  59. package/dist/index-j4zgmzjr.js +152 -0
  60. package/dist/index-jsmvyyvf.js +1166 -0
  61. package/dist/index-kh0pwvdd.js +1974 -0
  62. package/dist/index-mqx4nd6v.js +119 -0
  63. package/dist/index-mxxxytys.js +1254 -0
  64. package/dist/index-n5dd7r0v.js +711 -0
  65. package/dist/index-npg9z1a4.js +431 -0
  66. package/dist/index-pj501bh1.js +140 -0
  67. package/dist/index-qry4vhxk.js +1041 -0
  68. package/dist/index-s2gw5aw9.js +199 -0
  69. package/dist/index-vvdj6kca.js +70 -0
  70. package/dist/index-w2zc0vwa.js +571 -0
  71. package/dist/index-x3fthpsc.js +585 -0
  72. package/dist/index-zr53sf63.js +44 -0
  73. package/dist/index-zxdy5pby.js +2728 -0
  74. package/dist/index-zzhgcwyt.js +966 -0
  75. package/dist/index.js +453 -0
  76. package/dist/navigation.js +13 -0
  77. package/dist/pdf.js +85 -0
  78. package/dist/percolate.js +24 -0
  79. package/dist/portfolio.js +77 -0
  80. package/dist/query.js +29 -0
  81. package/dist/repository-memory.js +60 -0
  82. package/dist/sdk.js +36 -0
  83. package/dist/search-rules.js +34 -0
  84. package/dist/search.js +22 -0
  85. package/dist/semantic.js +39 -0
  86. package/dist/source-inbox.js +17 -0
  87. package/dist/untrusted-content.js +32 -0
  88. package/dist/url-intelligence.js +74 -0
  89. package/dist/workflow.js +23 -0
  90. package/dist/workflows/decision-context.js +20 -0
  91. package/dist/workflows/explain-change.js +8 -0
  92. package/dist/workflows/index.js +28 -0
  93. package/dist/workflows/plan-radar.js +10 -0
  94. package/package.json +409 -0
  95. package/skills/kb/AGENTS.md +19 -0
  96. package/skills/kb/SKILL.md +82 -0
  97. package/skills/kb/agents/openai.yaml +4 -0
  98. package/skills/kb/references/pdf-review.md +51 -0
  99. package/skills/kb/references/percolate.md +144 -0
  100. package/skills/kb/references/plan-structure.md +105 -0
  101. package/skills/kb/references/plan.md +115 -0
  102. package/skills/kb/references/query.md +231 -0
  103. package/skills/kb/references/refresh.md +133 -0
  104. package/skills/kb/references/save-pdf.md +125 -0
  105. package/skills/kb/references/save-url.md +178 -0
  106. package/skills/kb/references/url-authentication.md +75 -0
  107. package/skills/kb/references/url-platforms.md +25 -0
  108. package/src/agent-context.ts +1241 -0
  109. package/src/agent-guide-audit.ts +750 -0
  110. package/src/attachments.ts +723 -0
  111. package/src/authoring.ts +1725 -0
  112. package/src/benchmark.ts +362 -0
  113. package/src/capture.ts +68 -0
  114. package/src/cli.ts +3660 -0
  115. package/src/clip/acquire.ts +1273 -0
  116. package/src/clip/archive-today.ts +402 -0
  117. package/src/clip/args.ts +423 -0
  118. package/src/clip/assets.ts +262 -0
  119. package/src/clip/bounded-byte-buffer.ts +75 -0
  120. package/src/clip/browser-profiles.ts +372 -0
  121. package/src/clip/bundle-reader.ts +608 -0
  122. package/src/clip/capture.ts +1054 -0
  123. package/src/clip/cli.ts +181 -0
  124. package/src/clip/cookies.ts +512 -0
  125. package/src/clip/doctor.ts +1110 -0
  126. package/src/clip/extract.ts +868 -0
  127. package/src/clip/jobs.ts +959 -0
  128. package/src/clip/lib.ts +861 -0
  129. package/src/clip/media.ts +1474 -0
  130. package/src/clip/metadata-search-tool/Cargo.lock +2498 -0
  131. package/src/clip/metadata-search-tool/Cargo.toml +18 -0
  132. package/src/clip/metadata-search-tool/runner.ts +685 -0
  133. package/src/clip/metadata-search-tool/src/main.rs +389 -0
  134. package/src/clip/metadata-search.ts +687 -0
  135. package/src/clip/network-proxy.ts +627 -0
  136. package/src/clip/network.ts +950 -0
  137. package/src/clip/package-root.ts +54 -0
  138. package/src/clip/persist.ts +1472 -0
  139. package/src/clip/platforms.ts +1824 -0
  140. package/src/clip/refresh.ts +197 -0
  141. package/src/clip/structured.ts +432 -0
  142. package/src/clip/terminal.ts +197 -0
  143. package/src/clip/url-intelligence.ts +581 -0
  144. package/src/clip/url-metadata-backfill.ts +616 -0
  145. package/src/clip/url-metadata-cli.ts +251 -0
  146. package/src/clip/url-metadata.ts +1397 -0
  147. package/src/evaluation-analysis-v2.ts +1440 -0
  148. package/src/evaluation-builder.ts +2539 -0
  149. package/src/evaluation-corpus-authoring.ts +2052 -0
  150. package/src/evaluation-evidence.ts +1147 -0
  151. package/src/evaluation-execution-v2.ts +426 -0
  152. package/src/evaluation-implementation.ts +131 -0
  153. package/src/evaluation-kb-closure.ts +1430 -0
  154. package/src/evaluation-kb-runner-v2.ts +735 -0
  155. package/src/evaluation-kb-v2.ts +2567 -0
  156. package/src/evaluation-kb.ts +1018 -0
  157. package/src/evaluation-measurement-v2.ts +1462 -0
  158. package/src/evaluation-packing-v2.ts +452 -0
  159. package/src/evaluation-redaction.ts +28 -0
  160. package/src/evaluation-v2.ts +3857 -0
  161. package/src/evaluation.ts +1220 -0
  162. package/src/git.ts +1276 -0
  163. package/src/graph.ts +1487 -0
  164. package/src/index.ts +20 -0
  165. package/src/init.ts +123 -0
  166. package/src/navigation.ts +277 -0
  167. package/src/note-lock.ts +529 -0
  168. package/src/pdf/args.ts +202 -0
  169. package/src/pdf/capture.ts +298 -0
  170. package/src/pdf/cli.ts +159 -0
  171. package/src/pdf/extract.ts +303 -0
  172. package/src/pdf/layout.ts +618 -0
  173. package/src/pdf/markdown.ts +154 -0
  174. package/src/pdf/model.ts +234 -0
  175. package/src/pdf/ocr.ts +190 -0
  176. package/src/pdf/persist.ts +314 -0
  177. package/src/pdf/source.ts +135 -0
  178. package/src/pdf/tools.ts +146 -0
  179. package/src/pdf.ts +92 -0
  180. package/src/percolate.ts +877 -0
  181. package/src/portfolio-audit.ts +574 -0
  182. package/src/portfolio-identity.ts +162 -0
  183. package/src/portfolio-registry.ts +473 -0
  184. package/src/portfolio.ts +765 -0
  185. package/src/query.ts +646 -0
  186. package/src/repository-memory.ts +1150 -0
  187. package/src/sdk.ts +1452 -0
  188. package/src/search-rules.ts +712 -0
  189. package/src/search.ts +710 -0
  190. package/src/semantic-runtime.ts +1139 -0
  191. package/src/semantic.ts +2480 -0
  192. package/src/source-inbox.ts +232 -0
  193. package/src/untrusted-content.ts +401 -0
  194. package/src/url-intelligence.ts +104 -0
  195. package/src/vault.ts +636 -0
  196. package/src/workflow.ts +653 -0
  197. package/src/workflows/decision-context.ts +81 -0
  198. package/src/workflows/explain-change.ts +73 -0
  199. package/src/workflows/index.ts +3 -0
  200. package/src/workflows/plan-radar.ts +101 -0
@@ -0,0 +1,1943 @@
1
+ // @bun
2
+ import {
3
+ discoverArchiveTodaySnapshot,
4
+ normalizeSourceUrlIdentity,
5
+ parseArchiveTodayMementoUrl,
6
+ parseMetadataSearchResponse,
7
+ rankMetadataSearchResults
8
+ } from "./index-7fzc32gf.js";
9
+ import {
10
+ assertSafeNetworkUrl,
11
+ isPrivateHostname,
12
+ resolveSafeNetworkTarget
13
+ } from "./index-e5fbsywq.js";
14
+ import {
15
+ sanitizeArtifactUrl
16
+ } from "./index-mxxxytys.js";
17
+
18
+ // src/clip/metadata-search.ts
19
+ import { spawn } from "child_process";
20
+ import { createHash } from "crypto";
21
+ import {
22
+ chmodSync,
23
+ closeSync,
24
+ constants,
25
+ fstatSync,
26
+ fsyncSync,
27
+ lstatSync,
28
+ mkdirSync,
29
+ mkdtempSync,
30
+ openSync,
31
+ readSync,
32
+ realpathSync,
33
+ rmSync,
34
+ writeSync
35
+ } from "fs";
36
+ import { tmpdir } from "os";
37
+ import { isAbsolute, join } from "path";
38
+ var REQUEST_SCHEMA_VERSION = 1;
39
+ var MIN_RESULTS = 1;
40
+ var MAX_RESULTS = 20;
41
+ var MIN_TIMEOUT_MS = 500;
42
+ var MAX_TIMEOUT_MS = 15000;
43
+ var MAX_QUERY_BYTES = 4 * 1024;
44
+ var DEFAULT_RESULTS = 10;
45
+ var DEFAULT_TIMEOUT_MS = 15000;
46
+ var DEFAULT_PROCESS_GRACE_MS = 500;
47
+ var MAX_PROCESS_GRACE_MS = 2000;
48
+ var DEFAULT_MAX_STDOUT_BYTES = 2 * 1024 * 1024;
49
+ var DEFAULT_MAX_STDERR_BYTES = 64 * 1024;
50
+ var DEFAULT_MAX_BINARY_BYTES = 64 * 1024 * 1024;
51
+ var MAX_CONFIGURED_BINARY_BYTES = 256 * 1024 * 1024;
52
+ var KILL_GRACE_MS = 100;
53
+ var HASH_BUFFER_BYTES = 64 * 1024;
54
+ var MAX_ENGINE_ADDRESSES = 8;
55
+ var METADATA_SEARCH_ENGINE_HOSTS = Object.freeze([
56
+ "html.duckduckgo.com",
57
+ "search.brave.com",
58
+ "www.startpage.com",
59
+ "search.yahoo.com"
60
+ ]);
61
+ function failure(category, message) {
62
+ return Object.freeze({ status: "failure", category, message });
63
+ }
64
+ function signalAborted(signal) {
65
+ return signal?.aborted === true;
66
+ }
67
+ function checkedInteger(value, fallback, minimum, maximum, label) {
68
+ const resolved = value ?? fallback;
69
+ if (!Number.isSafeInteger(resolved) || resolved < minimum || resolved > maximum) {
70
+ throw new TypeError(`${label} must be an integer from ${minimum} through ${maximum}.`);
71
+ }
72
+ return resolved;
73
+ }
74
+ function utf8Length(value) {
75
+ return Buffer.byteLength(value, "utf8");
76
+ }
77
+ function hasUnsafeQueryCodeUnits(value) {
78
+ for (let index = 0;index < value.length; index += 1) {
79
+ const code = value.charCodeAt(index);
80
+ if (code === 0)
81
+ return true;
82
+ if (code >= 55296 && code <= 56319) {
83
+ const next = value.charCodeAt(index + 1);
84
+ if (next < 56320 || next > 57343)
85
+ return true;
86
+ index += 1;
87
+ } else if (code >= 56320 && code <= 57343) {
88
+ return true;
89
+ }
90
+ }
91
+ return false;
92
+ }
93
+ function validateRequest(request, defaults) {
94
+ if (typeof request.query !== "string" || request.query.trim() === "" || utf8Length(request.query.trim()) > MAX_QUERY_BYTES || hasUnsafeQueryCodeUnits(request.query))
95
+ return null;
96
+ const maxResults = request.maxResults ?? defaults.maxResults;
97
+ const timeoutMs = request.timeoutMs ?? defaults.timeoutMs;
98
+ if (!Number.isSafeInteger(maxResults) || maxResults < MIN_RESULTS || maxResults > MAX_RESULTS || !Number.isSafeInteger(timeoutMs) || timeoutMs < MIN_TIMEOUT_MS || timeoutMs > MAX_TIMEOUT_MS)
99
+ return null;
100
+ return Object.freeze({
101
+ query: request.query.trim(),
102
+ maxResults,
103
+ timeoutMs,
104
+ ...request.signal === undefined ? {} : { signal: request.signal }
105
+ });
106
+ }
107
+ function sameOpenedIdentity(left, right) {
108
+ return left.dev === right.dev && left.ino === right.ino && left.size === right.size && left.mode === right.mode && left.uid === right.uid && left.mtimeNs === right.mtimeNs && left.ctimeNs === right.ctimeNs;
109
+ }
110
+ function inspectBinary(binaryPath, maxBytes) {
111
+ const pathMetadata = lstatSync(binaryPath, { bigint: true });
112
+ if (pathMetadata.isSymbolicLink() || !pathMetadata.isFile() || pathMetadata.nlink !== 1n) {
113
+ throw new Error("The metadata search binary does not have a trusted file identity.");
114
+ }
115
+ const ownerExecutable = (pathMetadata.mode & 0o100n) !== 0n;
116
+ const writableByAnotherUser = (pathMetadata.mode & 0o022n) !== 0n;
117
+ const privilegedExecutable = (pathMetadata.mode & 0o6000n) !== 0n;
118
+ const processOwner = typeof process.getuid === "function" ? BigInt(process.getuid()) : pathMetadata.uid;
119
+ if (!ownerExecutable || writableByAnotherUser || privilegedExecutable || pathMetadata.uid !== processOwner || pathMetadata.size <= 0n || pathMetadata.size > BigInt(maxBytes))
120
+ throw new Error("The metadata search binary does not satisfy the executable policy.");
121
+ const descriptor = openSync(binaryPath, constants.O_RDONLY | constants.O_NOFOLLOW);
122
+ try {
123
+ const opened = fstatSync(descriptor, { bigint: true });
124
+ if (!sameOpenedIdentity(pathMetadata, opened) || opened.nlink !== 1n || !opened.isFile()) {
125
+ throw new Error("The metadata search binary identity changed while it was inspected.");
126
+ }
127
+ const hash = createHash("sha256");
128
+ const buffer = Buffer.allocUnsafe(HASH_BUFFER_BYTES);
129
+ let offset = 0;
130
+ while (offset <= maxBytes) {
131
+ const bytesRead = readSync(descriptor, buffer, 0, Math.min(buffer.byteLength, maxBytes + 1 - offset), offset);
132
+ if (bytesRead === 0)
133
+ break;
134
+ hash.update(buffer.subarray(0, bytesRead));
135
+ offset += bytesRead;
136
+ }
137
+ if (offset > maxBytes)
138
+ throw new Error("The metadata search binary exceeds its byte limit.");
139
+ const finished = fstatSync(descriptor, { bigint: true });
140
+ if (!sameOpenedIdentity(opened, finished) || BigInt(offset) !== finished.size) {
141
+ throw new Error("The metadata search binary identity changed while it was read.");
142
+ }
143
+ return Object.freeze({
144
+ realPath: realpathSync(binaryPath),
145
+ device: opened.dev,
146
+ inode: opened.ino,
147
+ size: opened.size,
148
+ mode: opened.mode,
149
+ owner: opened.uid,
150
+ modifiedNanoseconds: opened.mtimeNs,
151
+ changedNanoseconds: opened.ctimeNs,
152
+ sha256: hash.digest("hex")
153
+ });
154
+ } finally {
155
+ closeSync(descriptor);
156
+ }
157
+ }
158
+ function identitiesEqual(left, right) {
159
+ return left.realPath === right.realPath && left.device === right.device && left.inode === right.inode && left.size === right.size && left.mode === right.mode && left.owner === right.owner && left.modifiedNanoseconds === right.modifiedNanoseconds && left.changedNanoseconds === right.changedNanoseconds && left.sha256 === right.sha256;
160
+ }
161
+ function openedIdentityMatches(opened, expected) {
162
+ return opened.dev === expected.device && opened.ino === expected.inode && opened.size === expected.size && opened.mode === expected.mode && opened.uid === expected.owner && opened.mtimeNs === expected.modifiedNanoseconds && opened.ctimeNs === expected.changedNanoseconds && opened.nlink === 1n && opened.isFile();
163
+ }
164
+ function materializePinnedBinary(expected, runDirectory) {
165
+ const source = openSync(expected.realPath, constants.O_RDONLY | constants.O_NOFOLLOW);
166
+ const destinationPath = join(runDirectory, "metadata-search");
167
+ let destination = null;
168
+ try {
169
+ if (!openedIdentityMatches(fstatSync(source, { bigint: true }), expected)) {
170
+ throw new Error("metadata search binary identity changed");
171
+ }
172
+ destination = openSync(destinationPath, constants.O_WRONLY | constants.O_CREAT | constants.O_EXCL | constants.O_NOFOLLOW, 320);
173
+ const hash = createHash("sha256");
174
+ const buffer = Buffer.allocUnsafe(HASH_BUFFER_BYTES);
175
+ let offset = 0;
176
+ const expectedBytes = Number(expected.size);
177
+ while (offset < expectedBytes) {
178
+ const bytesRead = readSync(source, buffer, 0, Math.min(buffer.byteLength, expectedBytes - offset), offset);
179
+ if (bytesRead === 0)
180
+ throw new Error("metadata search binary ended early");
181
+ hash.update(buffer.subarray(0, bytesRead));
182
+ let written = 0;
183
+ while (written < bytesRead) {
184
+ written += writeSync(destination, buffer, written, bytesRead - written, offset + written);
185
+ }
186
+ offset += bytesRead;
187
+ }
188
+ if (hash.digest("hex") !== expected.sha256 || !openedIdentityMatches(fstatSync(source, { bigint: true }), expected))
189
+ throw new Error("metadata search binary identity changed");
190
+ fsyncSync(destination);
191
+ return destinationPath;
192
+ } finally {
193
+ if (destination !== null)
194
+ closeSync(destination);
195
+ closeSync(source);
196
+ }
197
+ }
198
+ function createPrivateRunDirectory() {
199
+ const directory = mkdtempSync(join(tmpdir(), "hraness-kb-metadata-search-"));
200
+ chmodSync(directory, 448);
201
+ for (const name of ["home", "config", "cache", "data", "tmp"]) {
202
+ mkdirSync(join(directory, name), { mode: 448 });
203
+ }
204
+ return directory;
205
+ }
206
+ function isolatedMetadataSearchEnvironment(source, runDirectory) {
207
+ const environment = {};
208
+ for (const key of ["PATH", "SystemRoot", "WINDIR", "COMSPEC", "PATHEXT"]) {
209
+ const value = source[key];
210
+ if (value !== undefined)
211
+ environment[key] = value;
212
+ }
213
+ environment.HOME = join(runDirectory, "home");
214
+ environment.XDG_CONFIG_HOME = join(runDirectory, "config");
215
+ environment.XDG_CACHE_HOME = join(runDirectory, "cache");
216
+ environment.XDG_DATA_HOME = join(runDirectory, "data");
217
+ environment.TMPDIR = join(runDirectory, "tmp");
218
+ environment.TMP = join(runDirectory, "tmp");
219
+ environment.TEMP = join(runDirectory, "tmp");
220
+ return Object.freeze(environment);
221
+ }
222
+ function appendBounded(capture, chunk, maximumBytes, onExceeded, retain) {
223
+ const bytes = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk);
224
+ if (capture.bytes > maximumBytes)
225
+ return;
226
+ capture.bytes += bytes.byteLength;
227
+ if (capture.bytes > maximumBytes) {
228
+ capture.chunks.length = 0;
229
+ onExceeded();
230
+ return;
231
+ }
232
+ if (retain)
233
+ capture.chunks.push(bytes);
234
+ }
235
+ async function runProcess(options) {
236
+ let child = null;
237
+ let closed = null;
238
+ let timeout = null;
239
+ let forceKill = null;
240
+ let termination;
241
+ let onAbort = null;
242
+ try {
243
+ child = spawn(options.binaryPath, [], {
244
+ cwd: options.cwd,
245
+ env: options.environment,
246
+ shell: false,
247
+ stdio: ["pipe", "pipe", "pipe"],
248
+ windowsHide: true
249
+ });
250
+ const runningChild = child;
251
+ const stdout = { chunks: [], bytes: 0 };
252
+ const stderr = { chunks: [], bytes: 0 };
253
+ let spawnFailed = false;
254
+ const terminate = (reason) => {
255
+ if (termination !== undefined)
256
+ return;
257
+ termination = reason;
258
+ runningChild.kill("SIGTERM");
259
+ forceKill = setTimeout(() => runningChild.kill("SIGKILL"), KILL_GRACE_MS);
260
+ };
261
+ runningChild.stdout.on("data", (chunk) => {
262
+ appendBounded(stdout, chunk, options.maxStdoutBytes, () => terminate("output-limit"), true);
263
+ });
264
+ runningChild.stderr.on("data", (chunk) => {
265
+ appendBounded(stderr, chunk, options.maxStderrBytes, () => terminate("output-limit"), false);
266
+ });
267
+ runningChild.stdin.on("error", () => {});
268
+ closed = new Promise((resolve) => {
269
+ runningChild.once("error", () => {
270
+ spawnFailed = true;
271
+ });
272
+ runningChild.once("close", (exitCode, processSignal) => {
273
+ resolve(Object.freeze({
274
+ stdout: Buffer.concat(stdout.chunks),
275
+ exitCode,
276
+ signal: processSignal,
277
+ spawnFailed,
278
+ ...termination === undefined ? {} : { termination }
279
+ }));
280
+ });
281
+ });
282
+ timeout = setTimeout(() => terminate("timeout"), options.timeoutMs);
283
+ if (options.signal !== undefined) {
284
+ onAbort = () => terminate("aborted");
285
+ options.signal.addEventListener("abort", onAbort, { once: true });
286
+ if (options.signal.aborted)
287
+ terminate("aborted");
288
+ }
289
+ runningChild.stdin.end(options.stdin, "utf8");
290
+ return await closed;
291
+ } finally {
292
+ if (timeout !== null)
293
+ clearTimeout(timeout);
294
+ if (forceKill !== null)
295
+ clearTimeout(forceKill);
296
+ if (onAbort !== null && options.signal !== undefined) {
297
+ options.signal.removeEventListener("abort", onAbort);
298
+ }
299
+ if (child !== null && child.exitCode === null && child.signalCode === null)
300
+ child.kill("SIGKILL");
301
+ if (closed !== null)
302
+ await closed;
303
+ }
304
+ }
305
+ function deterministicResponse(response, maximumResults) {
306
+ const compareText = (left, right) => left < right ? -1 : left > right ? 1 : 0;
307
+ const enginesQueried = Object.freeze([...response.enginesQueried].sort());
308
+ const results = response.results.map((result) => Object.freeze({ ...result, engines: Object.freeze([...result.engines].sort()) })).sort((left, right) => right.score - left.score || compareText(left.title, right.title) || compareText(left.url, right.url)).slice(0, maximumResults);
309
+ return Object.freeze({
310
+ ...response,
311
+ results: Object.freeze(results),
312
+ enginesQueried,
313
+ enginesFailed: Object.freeze([...response.enginesFailed].sort()),
314
+ engineStatus: response.engineStatus
315
+ });
316
+ }
317
+ async function resolveEngineHosts(resolver, timeoutMs) {
318
+ const resolved = await Promise.all(METADATA_SEARCH_ENGINE_HOSTS.map(async (hostname) => {
319
+ const addresses = await resolver(new URL(`https://${hostname}/`), {
320
+ allowPrivateNetwork: false,
321
+ timeoutMs
322
+ });
323
+ if (addresses.length === 0 || addresses.length > MAX_ENGINE_ADDRESSES) {
324
+ throw new Error("metadata search engine DNS answer count is invalid");
325
+ }
326
+ const unique = new Map;
327
+ for (const address of addresses) {
328
+ if (address.family !== 4 && address.family !== 6 || typeof address.address !== "string" || address.address.trim() !== address.address || address.address === "")
329
+ throw new Error("metadata search engine DNS answer is invalid");
330
+ unique.set(`${address.family}:${address.address}`, Object.freeze({ ...address }));
331
+ }
332
+ if (unique.size !== addresses.length)
333
+ throw new Error("metadata search engine DNS answer is duplicated");
334
+ return Object.freeze({
335
+ hostname,
336
+ addresses: Object.freeze([...unique.values()].sort((left, right) => left.family - right.family || (left.address < right.address ? -1 : left.address > right.address ? 1 : 0)))
337
+ });
338
+ }));
339
+ return Object.freeze(resolved);
340
+ }
341
+ function createExactUrlSearchQuery(value) {
342
+ let url;
343
+ try {
344
+ url = value instanceof URL ? new URL(value.href) : new URL(value);
345
+ } catch {
346
+ return null;
347
+ }
348
+ if (url.protocol !== "https:" && url.protocol !== "http:" || normalizeSourceUrlIdentity(url) === null) {
349
+ return null;
350
+ }
351
+ url.hash = "";
352
+ const exactUrl = url.href;
353
+ let sanitized;
354
+ try {
355
+ sanitized = sanitizeArtifactUrl(exactUrl);
356
+ } catch {
357
+ return null;
358
+ }
359
+ if (sanitized !== exactUrl || utf8Length(exactUrl) > MAX_QUERY_BYTES - 2)
360
+ return null;
361
+ return `"${exactUrl}"`;
362
+ }
363
+ function createRustMetadataSearchProvider(options) {
364
+ if (!isAbsolute(options.binaryPath)) {
365
+ throw new TypeError("The metadata search binary path must be absolute.");
366
+ }
367
+ const defaultMaxResults = checkedInteger(options.defaultMaxResults, DEFAULT_RESULTS, MIN_RESULTS, MAX_RESULTS, "defaultMaxResults");
368
+ const defaultTimeoutMs = checkedInteger(options.defaultTimeoutMs, DEFAULT_TIMEOUT_MS, MIN_TIMEOUT_MS, MAX_TIMEOUT_MS, "defaultTimeoutMs");
369
+ const processGraceMs = checkedInteger(options.processGraceMs, DEFAULT_PROCESS_GRACE_MS, 0, MAX_PROCESS_GRACE_MS, "processGraceMs");
370
+ const maxStdoutBytes = checkedInteger(options.maxStdoutBytes, DEFAULT_MAX_STDOUT_BYTES, 1, 16 * 1024 * 1024, "maxStdoutBytes");
371
+ const maxStderrBytes = checkedInteger(options.maxStderrBytes, DEFAULT_MAX_STDERR_BYTES, 1, 2 * 1024 * 1024, "maxStderrBytes");
372
+ const maxBinaryBytes = checkedInteger(options.maxBinaryBytes, DEFAULT_MAX_BINARY_BYTES, 1, MAX_CONFIGURED_BINARY_BYTES, "maxBinaryBytes");
373
+ let pinnedIdentity;
374
+ try {
375
+ pinnedIdentity = inspectBinary(options.binaryPath, maxBinaryBytes);
376
+ } catch {
377
+ throw new Error("The metadata search binary is unavailable or untrusted.");
378
+ }
379
+ const sourceEnvironment = options.environment ?? process.env;
380
+ const resolveNetworkTarget = options.resolveNetworkTarget ?? resolveSafeNetworkTarget;
381
+ return async (request) => {
382
+ const validated = validateRequest(request, { maxResults: defaultMaxResults, timeoutMs: defaultTimeoutMs });
383
+ if (validated === null)
384
+ return failure("invalid-request", "The metadata search request is invalid.");
385
+ if (signalAborted(validated.signal))
386
+ return failure("aborted", "Metadata search was aborted.");
387
+ let engineHosts;
388
+ try {
389
+ engineHosts = await resolveEngineHosts(resolveNetworkTarget, validated.timeoutMs);
390
+ } catch {
391
+ return failure("unavailable", "Metadata search network targets are unavailable.");
392
+ }
393
+ if (signalAborted(validated.signal))
394
+ return failure("aborted", "Metadata search was aborted.");
395
+ let currentIdentity;
396
+ try {
397
+ currentIdentity = inspectBinary(options.binaryPath, maxBinaryBytes);
398
+ } catch {
399
+ return failure("unavailable", "Metadata search is unavailable.");
400
+ }
401
+ if (!identitiesEqual(pinnedIdentity, currentIdentity)) {
402
+ return failure("unavailable", "Metadata search is unavailable.");
403
+ }
404
+ let runDirectory;
405
+ try {
406
+ runDirectory = createPrivateRunDirectory();
407
+ } catch {
408
+ return failure("unavailable", "Metadata search is unavailable.");
409
+ }
410
+ try {
411
+ const input = JSON.stringify({
412
+ schema_version: REQUEST_SCHEMA_VERSION,
413
+ query: validated.query,
414
+ max_results: validated.maxResults,
415
+ timeout_ms: validated.timeoutMs,
416
+ engine_hosts: engineHosts
417
+ });
418
+ let result;
419
+ try {
420
+ const executablePath = materializePinnedBinary(currentIdentity, runDirectory);
421
+ result = await runProcess({
422
+ binaryPath: executablePath,
423
+ cwd: runDirectory,
424
+ environment: isolatedMetadataSearchEnvironment(sourceEnvironment, runDirectory),
425
+ stdin: input,
426
+ timeoutMs: validated.timeoutMs + processGraceMs,
427
+ maxStdoutBytes,
428
+ maxStderrBytes,
429
+ ...validated.signal === undefined ? {} : { signal: validated.signal }
430
+ });
431
+ } catch {
432
+ return failure("process", "Metadata search process failed.");
433
+ }
434
+ if (result.termination === "aborted")
435
+ return failure("aborted", "Metadata search was aborted.");
436
+ if (result.termination === "timeout")
437
+ return failure("timeout", "Metadata search timed out.");
438
+ if (result.termination === "output-limit") {
439
+ return failure("protocol", "Metadata search returned an invalid response.");
440
+ }
441
+ if (result.spawnFailed)
442
+ return failure("unavailable", "Metadata search is unavailable.");
443
+ if (result.exitCode !== 0 || result.signal !== null) {
444
+ return failure("process", "Metadata search process failed.");
445
+ }
446
+ let decoded;
447
+ try {
448
+ decoded = new TextDecoder("utf-8", { fatal: true }).decode(result.stdout);
449
+ } catch {
450
+ return failure("protocol", "Metadata search returned an invalid response.");
451
+ }
452
+ try {
453
+ const parsed = parseMetadataSearchResponse(JSON.parse(decoded));
454
+ if (parsed.query !== validated.query) {
455
+ return failure("protocol", "Metadata search returned an invalid response.");
456
+ }
457
+ return Object.freeze({
458
+ status: "success",
459
+ response: deterministicResponse(parsed, validated.maxResults)
460
+ });
461
+ } catch {
462
+ return failure("protocol", "Metadata search returned an invalid response.");
463
+ }
464
+ } finally {
465
+ rmSync(runDirectory, { recursive: true, force: true, maxRetries: 3 });
466
+ }
467
+ };
468
+ }
469
+
470
+ // src/clip/url-metadata.ts
471
+ import { execFileSync } from "child_process";
472
+ import { randomUUID } from "crypto";
473
+ import {
474
+ closeSync as closeSync2,
475
+ constants as constants2,
476
+ fchmodSync,
477
+ fstatSync as fstatSync2,
478
+ ftruncateSync,
479
+ fsyncSync as fsyncSync2,
480
+ lstatSync as lstatSync2,
481
+ openSync as openSync2,
482
+ readFileSync,
483
+ readSync as readSync2,
484
+ readdirSync,
485
+ realpathSync as realpathSync2,
486
+ renameSync,
487
+ unlinkSync,
488
+ writeFileSync,
489
+ writeSync as writeSync2
490
+ } from "fs";
491
+ import { hostname } from "os";
492
+ import { basename, dirname, join as join2, relative, resolve, sep } from "path";
493
+ import { parseDocument } from "yaml";
494
+ var URL_METADATA_FILENAME = "url-metadata.json";
495
+ var URL_METADATA_SCHEMA_VERSION = 1;
496
+ var METADATA_SEARCH_ENGINE_ID = "metadata-search-engine-rs";
497
+ var METADATA_SEARCH_ENGINE_VERSION = "0.1.3";
498
+ var METADATA_SEARCH_ENGINE_REVISION = "f40a00ea67a857ee996e1caba1ebab3ee7a14a47";
499
+ var MAX_ARTICLE_DIRECTORIES = 4096;
500
+ var MAX_DIRECTORY_ENTRIES = 256;
501
+ var MAX_MARKDOWN_BYTES = 2 * 1024 * 1024;
502
+ var MAX_SIDECAR_BYTES = 1024 * 1024;
503
+ var MAX_CANDIDATES = 20;
504
+ var MAX_ARCHIVES = 32;
505
+ var MAX_ATTEMPTS = 16;
506
+ var MAX_WARNINGS = 32;
507
+ var MAX_URL_BYTES = 16 * 1024;
508
+ var MAX_TITLE_BYTES = 2 * 1024;
509
+ var MAX_SNIPPET_BYTES = 8 * 1024;
510
+ var MAX_MESSAGE_BYTES = 2 * 1024;
511
+ var MAX_ENGINE_COUNT = 8;
512
+ var MAX_LOCK_BYTES = 4 * 1024;
513
+ var MALFORMED_LOCK_RECOVERY_AGE_MILLISECONDS = 30000;
514
+ var LOCK_EXCLUSIVE = 2;
515
+ var LOCK_NONBLOCKING = 4;
516
+ var F_SET_FILE_DESCRIPTOR = 2;
517
+ var FILE_DESCRIPTOR_CLOSE_ON_EXEC = 1;
518
+ var INTERRUPTED_SYSTEM_CALL = 4;
519
+ var engineNamePattern = /^[a-z0-9][a-z0-9._-]{0,63}$/u;
520
+ var lockTokenPattern = /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/u;
521
+ var nativeFlock;
522
+ var nativeFcntl;
523
+ var nativeErrnoLocation;
524
+ var nativeFfiRead;
525
+ var nativeLockLibraries = [];
526
+ var savedUrlRecordIdentities = new WeakMap;
527
+ function requiredNativeSymbol(symbols, name) {
528
+ const symbol = symbols[name];
529
+ if (symbol === undefined)
530
+ throw new Error(`URL metadata locking could not bind ${name}.`);
531
+ return symbol;
532
+ }
533
+ function linuxLibcCandidates(processMaps, architecture = process.arch) {
534
+ const candidates = [];
535
+ const append = (candidate) => {
536
+ if (!candidates.includes(candidate))
537
+ candidates.push(candidate);
538
+ };
539
+ for (const line of processMaps.split(`
540
+ `)) {
541
+ const pathStart = line.indexOf("/");
542
+ if (pathStart < 0 || line.endsWith(" (deleted)"))
543
+ continue;
544
+ const path = line.slice(pathStart).replace(/\\([0-7]{3})/gu, (_, octal) => String.fromCodePoint(Number.parseInt(octal, 8)));
545
+ if (/\/(?:libc(?:-[^/]+)?\.so(?:\.[0-9]+)*|libc\.musl-[^/]+\.so(?:\.[0-9]+)*|ld-musl-[^/]+\.so(?:\.[0-9]+)*)$/u.test(path))
546
+ append(path);
547
+ }
548
+ append("libc.so.6");
549
+ if (architecture === "x64")
550
+ append("/lib/ld-musl-x86_64.so.1");
551
+ if (architecture === "arm64")
552
+ append("/lib/ld-musl-aarch64.so.1");
553
+ return candidates;
554
+ }
555
+ function initializeNativeLocking() {
556
+ if (nativeFlock !== undefined && nativeFcntl !== undefined && nativeErrnoLocation !== undefined)
557
+ return;
558
+ const ffi = globalThis.Bun?.FFI;
559
+ if (ffi === undefined) {
560
+ throw new Error("URL metadata writes require Bun's native file-locking runtime.");
561
+ }
562
+ nativeFfiRead = ffi.read;
563
+ if (process.platform === "darwin") {
564
+ const library = ffi.dlopen("/usr/lib/libSystem.B.dylib", {
565
+ __error: { args: [], returns: "ptr" },
566
+ fcntl: { args: ["i32", "i32", "i32"], returns: "i32" },
567
+ flock: { args: ["i32", "i32"], returns: "i32" }
568
+ });
569
+ nativeLockLibraries.push(library);
570
+ const errnoLocation = requiredNativeSymbol(library.symbols, "__error");
571
+ const fcntl = requiredNativeSymbol(library.symbols, "fcntl");
572
+ const flock = requiredNativeSymbol(library.symbols, "flock");
573
+ nativeErrnoLocation = () => errnoLocation();
574
+ nativeFcntl = (descriptor, command, argument) => fcntl(descriptor, command, argument);
575
+ nativeFlock = (descriptor, operation) => flock(descriptor, operation);
576
+ return;
577
+ }
578
+ if (process.platform === "linux") {
579
+ let processMaps = "";
580
+ try {
581
+ processMaps = readFileSync("/proc/self/maps", "utf8");
582
+ } catch {}
583
+ const failures = [];
584
+ for (const candidate of linuxLibcCandidates(processMaps)) {
585
+ try {
586
+ const library = ffi.dlopen(candidate, {
587
+ __errno_location: { args: [], returns: "ptr" },
588
+ fcntl: { args: ["i32", "i32", "i32"], returns: "i32" },
589
+ flock: { args: ["i32", "i32"], returns: "i32" }
590
+ });
591
+ nativeLockLibraries.push(library);
592
+ const errnoLocation = requiredNativeSymbol(library.symbols, "__errno_location");
593
+ const fcntl = requiredNativeSymbol(library.symbols, "fcntl");
594
+ const flock = requiredNativeSymbol(library.symbols, "flock");
595
+ nativeErrnoLocation = () => errnoLocation();
596
+ nativeFcntl = (descriptor, command, argument) => fcntl(descriptor, command, argument);
597
+ nativeFlock = (descriptor, operation) => flock(descriptor, operation);
598
+ return;
599
+ } catch (error) {
600
+ failures.push(`${candidate}: ${error instanceof Error ? error.message : String(error)}`);
601
+ }
602
+ }
603
+ throw new Error(`URL metadata locking could not load the host libc (${failures.join("; ")}).`);
604
+ }
605
+ throw new Error(`URL metadata locking is unsupported on ${process.platform}.`);
606
+ }
607
+ function currentNativeErrno() {
608
+ initializeNativeLocking();
609
+ const pointer = nativeErrnoLocation?.();
610
+ if (pointer === undefined || pointer === null)
611
+ throw new Error("URL metadata locking could not read errno.");
612
+ if (nativeFfiRead === undefined)
613
+ throw new Error("URL metadata locking lost its native read binding.");
614
+ return nativeFfiRead.i32(pointer);
615
+ }
616
+ function tryExclusiveFileLock(descriptor) {
617
+ initializeNativeLocking();
618
+ for (;; ) {
619
+ if (nativeFlock?.(descriptor, LOCK_EXCLUSIVE | LOCK_NONBLOCKING) === 0)
620
+ return true;
621
+ const errno = currentNativeErrno();
622
+ if (errno === INTERRUPTED_SYSTEM_CALL)
623
+ continue;
624
+ if (process.platform === "darwin" && errno === 35 || process.platform === "linux" && errno === 11) {
625
+ return false;
626
+ }
627
+ throw new Error(`URL metadata flock failed with errno ${errno}.`);
628
+ }
629
+ }
630
+ function setDescriptorCloseOnExec(descriptor) {
631
+ initializeNativeLocking();
632
+ if (nativeFcntl?.(descriptor, F_SET_FILE_DESCRIPTOR, FILE_DESCRIPTOR_CLOSE_ON_EXEC) !== 0) {
633
+ throw new Error(`URL metadata fcntl failed with errno ${currentNativeErrno()}.`);
634
+ }
635
+ }
636
+ function compareText(left, right) {
637
+ return left < right ? -1 : left > right ? 1 : 0;
638
+ }
639
+ function errorFromUnknown(value, message) {
640
+ return value instanceof Error ? value : new Error(message, { cause: value });
641
+ }
642
+ function throwCleanupErrorWithoutOperationError(operationError, cleanupError) {
643
+ if (operationError === undefined && cleanupError !== undefined) {
644
+ throw errorFromUnknown(cleanupError, "URL metadata write cleanup failed.");
645
+ }
646
+ }
647
+ function utf8Length2(value) {
648
+ return new TextEncoder().encode(value).byteLength;
649
+ }
650
+ function record(value, label) {
651
+ if (typeof value !== "object" || value === null || Array.isArray(value)) {
652
+ throw new TypeError(`${label} must be an object.`);
653
+ }
654
+ return value;
655
+ }
656
+ function exactKeys(value, keys, label) {
657
+ const expected = new Set(keys);
658
+ for (const key of Object.keys(value)) {
659
+ if (!expected.has(key))
660
+ throw new TypeError(`${label} contains unknown key ${JSON.stringify(key)}.`);
661
+ }
662
+ for (const key of keys) {
663
+ if (!Object.hasOwn(value, key))
664
+ throw new TypeError(`${label} is missing ${JSON.stringify(key)}.`);
665
+ }
666
+ }
667
+ function boundedText(value, label, maximumBytes) {
668
+ if (typeof value !== "string" || value === "" || utf8Length2(value) > maximumBytes) {
669
+ throw new TypeError(`${label} must be a non-empty string of at most ${maximumBytes} UTF-8 bytes.`);
670
+ }
671
+ if (/\p{Cc}|\p{Cs}/u.test(value))
672
+ throw new TypeError(`${label} contains unsafe control characters.`);
673
+ return value;
674
+ }
675
+ function boundedTextList(value, label, maximumCount, maximumBytes) {
676
+ if (!Array.isArray(value) || value.length > maximumCount) {
677
+ throw new TypeError(`${label} must be an array with at most ${maximumCount} entries.`);
678
+ }
679
+ const parsed = value.map((item, index) => boundedText(item, `${label}[${index}]`, maximumBytes));
680
+ if (new Set(parsed).size !== parsed.length)
681
+ throw new TypeError(`${label} contains duplicate entries.`);
682
+ return Object.freeze(parsed);
683
+ }
684
+ function engineList(value, label) {
685
+ const engines = boundedTextList(value, label, MAX_ENGINE_COUNT, 64);
686
+ for (const engine of engines) {
687
+ if (!engineNamePattern.test(engine)) {
688
+ throw new TypeError(`${label} contains an invalid engine identifier.`);
689
+ }
690
+ }
691
+ return engines;
692
+ }
693
+ function normalizedPublicUrl(value, label) {
694
+ const text = boundedText(value, label, MAX_URL_BYTES);
695
+ let url;
696
+ try {
697
+ url = new URL(text);
698
+ } catch (error) {
699
+ throw new TypeError(`${label} must be an absolute HTTP(S) URL.`, { cause: error });
700
+ }
701
+ if (url.protocol !== "https:" && url.protocol !== "http:" || url.username !== "" || url.password !== "") {
702
+ throw new TypeError(`${label} must be a credential-free HTTP(S) URL.`);
703
+ }
704
+ if (isPrivateHostname(url.hostname)) {
705
+ throw new TypeError(`${label} must not target a private network.`);
706
+ }
707
+ url.hash = "";
708
+ const normalized = url.href;
709
+ if (sanitizeArtifactUrl(normalized) !== normalized) {
710
+ throw new TypeError(`${label} must not contain credential-shaped path or query data.`);
711
+ }
712
+ return normalized;
713
+ }
714
+ function isoTimestamp(value, label) {
715
+ const text = boundedText(value, label, 128);
716
+ const parsed = new Date(text);
717
+ if (!Number.isFinite(parsed.getTime()) || parsed.toISOString() !== text) {
718
+ throw new TypeError(`${label} must be a canonical ISO timestamp.`);
719
+ }
720
+ return text;
721
+ }
722
+ function parseUrlMetadataWriteLockOwner(value) {
723
+ const input = record(value, "URL metadata write lock owner");
724
+ exactKeys(input, [
725
+ "schemaVersion",
726
+ "kind",
727
+ "host",
728
+ "pid",
729
+ "processIdentity",
730
+ "token",
731
+ "acquiredAt"
732
+ ], "URL metadata write lock owner");
733
+ if (input.schemaVersion !== 1 || input.kind !== "url-metadata-write-lock") {
734
+ throw new TypeError("URL metadata write lock owner has an unsupported identity.");
735
+ }
736
+ if (typeof input.pid !== "number" || !Number.isSafeInteger(input.pid) || input.pid <= 0) {
737
+ throw new TypeError("URL metadata write lock owner.pid must be a positive safe integer.");
738
+ }
739
+ const token = boundedText(input.token, "URL metadata write lock owner.token", 64);
740
+ if (!lockTokenPattern.test(token))
741
+ throw new TypeError("URL metadata write lock owner.token is invalid.");
742
+ return Object.freeze({
743
+ schemaVersion: 1,
744
+ kind: "url-metadata-write-lock",
745
+ host: boundedText(input.host, "URL metadata write lock owner.host", 255),
746
+ pid: input.pid,
747
+ processIdentity: boundedText(input.processIdentity, "URL metadata write lock owner.processIdentity", 512),
748
+ token,
749
+ acquiredAt: isoTimestamp(input.acquiredAt, "URL metadata write lock owner.acquiredAt")
750
+ });
751
+ }
752
+ function operatingSystemProcessIdentity(pid) {
753
+ if (!Number.isSafeInteger(pid) || pid <= 0)
754
+ return null;
755
+ if (process.platform === "linux") {
756
+ try {
757
+ const stat = readFileSync(`/proc/${pid}/stat`, "utf8");
758
+ const commandEnd = stat.lastIndexOf(")");
759
+ if (commandEnd < 0)
760
+ return null;
761
+ const fields = stat.slice(commandEnd + 1).trim().split(/\s+/u);
762
+ const state = fields[0];
763
+ const startTicks = fields[19];
764
+ if (state === "Z" || startTicks === undefined || !/^\d+$/u.test(startTicks))
765
+ return null;
766
+ const bootId = readFileSync("/proc/sys/kernel/random/boot_id", "utf8").trim();
767
+ if (!/^[0-9a-f-]{36}$/u.test(bootId))
768
+ return null;
769
+ return `linux:${bootId}:${startTicks}`;
770
+ } catch {
771
+ return null;
772
+ }
773
+ }
774
+ if (process.platform === "darwin") {
775
+ try {
776
+ const started = execFileSync("/bin/ps", ["-o", "lstart=", "-p", String(pid)], {
777
+ encoding: "utf8",
778
+ maxBuffer: 4096,
779
+ stdio: ["ignore", "pipe", "ignore"],
780
+ timeout: 1000
781
+ }).trim();
782
+ return started === "" ? null : `darwin:${started.replace(/\s+/gu, " ")}`;
783
+ } catch {
784
+ return null;
785
+ }
786
+ }
787
+ return null;
788
+ }
789
+ function currentWriteLockOwner() {
790
+ const processIdentity = operatingSystemProcessIdentity(process.pid);
791
+ if (processIdentity === null) {
792
+ throw new Error("URL metadata locking could not verify the current process identity.");
793
+ }
794
+ return parseUrlMetadataWriteLockOwner({
795
+ schemaVersion: 1,
796
+ kind: "url-metadata-write-lock",
797
+ host: hostname(),
798
+ pid: process.pid,
799
+ processIdentity,
800
+ token: randomUUID(),
801
+ acquiredAt: new Date().toISOString()
802
+ });
803
+ }
804
+ function parseCandidate(value, label) {
805
+ const input = record(value, label);
806
+ exactKeys(input, ["title", "url", "snippet", "engines", "score"], label);
807
+ const score = input.score;
808
+ if (typeof score !== "number" || !Number.isFinite(score) || score < 0) {
809
+ throw new TypeError(`${label}.score must be a finite non-negative number.`);
810
+ }
811
+ const snippet = input.snippet === null ? null : boundedText(input.snippet, `${label}.snippet`, MAX_SNIPPET_BYTES);
812
+ return Object.freeze({
813
+ title: boundedText(input.title, `${label}.title`, MAX_TITLE_BYTES),
814
+ url: normalizedPublicUrl(input.url, `${label}.url`),
815
+ snippet,
816
+ engines: boundedTextList(input.engines, `${label}.engines`, MAX_ENGINE_COUNT, 64),
817
+ score
818
+ });
819
+ }
820
+ function parseAttempt(value, label) {
821
+ const input = record(value, label);
822
+ exactKeys(input, ["provider", "outcome", "message"], label);
823
+ if (input.provider !== "metadata-search-engine-rs" && input.provider !== "archive-today") {
824
+ throw new TypeError(`${label}.provider is unsupported.`);
825
+ }
826
+ if (input.outcome !== "succeeded" && input.outcome !== "partial" && input.outcome !== "not-found" && input.outcome !== "failed" && input.outcome !== "skipped")
827
+ throw new TypeError(`${label}.outcome is unsupported.`);
828
+ return Object.freeze({
829
+ provider: input.provider,
830
+ outcome: input.outcome,
831
+ message: boundedText(input.message, `${label}.message`, MAX_MESSAGE_BYTES)
832
+ });
833
+ }
834
+ function parseArchive(value, label, subjectUrl, generatedAt) {
835
+ const input = record(value, label);
836
+ exactKeys(input, ["url", "capturedAt", "discovery"], label);
837
+ if (input.discovery !== "newest" && input.discovery !== "timemap" && input.discovery !== "metadata-search")
838
+ throw new TypeError(`${label}.discovery is unsupported.`);
839
+ const url = normalizedPublicUrl(input.url, `${label}.url`);
840
+ const capturedAt = isoTimestamp(input.capturedAt, `${label}.capturedAt`);
841
+ let memento;
842
+ try {
843
+ memento = parseArchiveTodayMementoUrl(url, {
844
+ originalUrl: subjectUrl,
845
+ now: new Date(generatedAt)
846
+ });
847
+ } catch (error) {
848
+ throw new TypeError(`${label} must be an exact timestamped Archive.today-family snapshot.`, { cause: error });
849
+ }
850
+ if (memento.capturedAt !== capturedAt) {
851
+ throw new TypeError(`${label}.capturedAt must match its snapshot timestamp.`);
852
+ }
853
+ return Object.freeze({
854
+ url: memento.url,
855
+ capturedAt,
856
+ discovery: input.discovery
857
+ });
858
+ }
859
+ function parseSelectedField(value, label, maximumBytes) {
860
+ if (value === null)
861
+ return null;
862
+ const input = record(value, label);
863
+ exactKeys(input, ["value", "sourceUrl", "provider"], label);
864
+ if (input.provider !== METADATA_SEARCH_ENGINE_ID)
865
+ throw new TypeError(`${label}.provider is unsupported.`);
866
+ return Object.freeze({
867
+ value: boundedText(input.value, `${label}.value`, maximumBytes),
868
+ sourceUrl: normalizedPublicUrl(input.sourceUrl, `${label}.sourceUrl`),
869
+ provider: METADATA_SEARCH_ENGINE_ID
870
+ });
871
+ }
872
+ function metadataStatus(options) {
873
+ if (options.hasCandidate)
874
+ return options.partial ? "partial" : "matched";
875
+ if (options.hasArchive)
876
+ return "partial";
877
+ if (options.queriedEngines === 0 || options.failedEngines === options.queriedEngines)
878
+ return "unavailable";
879
+ return options.partial ? "partial" : "not-found";
880
+ }
881
+ function parseUrlMetadataDocument(value) {
882
+ const input = record(value, "URL metadata document");
883
+ exactKeys(input, [
884
+ "schemaVersion",
885
+ "kind",
886
+ "subjectUrl",
887
+ "generatedAt",
888
+ "status",
889
+ "provider",
890
+ "attempts",
891
+ "candidates",
892
+ "selected",
893
+ "archives",
894
+ "warnings"
895
+ ], "URL metadata document");
896
+ if (input.schemaVersion !== URL_METADATA_SCHEMA_VERSION || input.kind !== "url-metadata") {
897
+ throw new TypeError("URL metadata document has an unsupported identity.");
898
+ }
899
+ if (input.status !== "matched" && input.status !== "not-found" && input.status !== "partial" && input.status !== "unavailable")
900
+ throw new TypeError("URL metadata document has an unsupported status.");
901
+ const provider = record(input.provider, "URL metadata document.provider");
902
+ exactKeys(provider, ["id", "version", "revision", "enginesQueried", "enginesFailed"], "URL metadata document.provider");
903
+ if (provider.id !== METADATA_SEARCH_ENGINE_ID || provider.version !== METADATA_SEARCH_ENGINE_VERSION || provider.revision !== METADATA_SEARCH_ENGINE_REVISION)
904
+ throw new TypeError("URL metadata document has an unsupported provider identity.");
905
+ if (!Array.isArray(input.attempts) || input.attempts.length > MAX_ATTEMPTS) {
906
+ throw new TypeError(`URL metadata document.attempts must have at most ${MAX_ATTEMPTS} entries.`);
907
+ }
908
+ if (!Array.isArray(input.candidates) || input.candidates.length > MAX_CANDIDATES) {
909
+ throw new TypeError(`URL metadata document.candidates must have at most ${MAX_CANDIDATES} entries.`);
910
+ }
911
+ if (!Array.isArray(input.archives) || input.archives.length > MAX_ARCHIVES) {
912
+ throw new TypeError(`URL metadata document.archives must have at most ${MAX_ARCHIVES} entries.`);
913
+ }
914
+ const selected = record(input.selected, "URL metadata document.selected");
915
+ exactKeys(selected, ["title", "description"], "URL metadata document.selected");
916
+ const subjectUrl = normalizedPublicUrl(input.subjectUrl, "URL metadata document.subjectUrl");
917
+ const generatedAt = isoTimestamp(input.generatedAt, "URL metadata document.generatedAt");
918
+ const enginesQueried = engineList(provider.enginesQueried, "URL metadata document.provider.enginesQueried");
919
+ const enginesFailed = engineList(provider.enginesFailed, "URL metadata document.provider.enginesFailed");
920
+ const queried = new Set(enginesQueried);
921
+ for (const engine of enginesFailed) {
922
+ if (!queried.has(engine)) {
923
+ throw new TypeError("URL metadata document.provider.enginesFailed contains an unqueried engine.");
924
+ }
925
+ }
926
+ const failed = new Set(enginesFailed);
927
+ const attempts = Object.freeze(input.attempts.map((item, index) => parseAttempt(item, `URL metadata document.attempts[${index}]`)));
928
+ const candidates = Object.freeze(input.candidates.map((item, index) => parseCandidate(item, `URL metadata document.candidates[${index}]`)));
929
+ const candidateUrls = new Set;
930
+ for (const candidate of candidates) {
931
+ if (candidate.url !== subjectUrl) {
932
+ throw new TypeError("URL metadata document candidate must exactly match subjectUrl.");
933
+ }
934
+ if (candidateUrls.has(candidate.url)) {
935
+ throw new TypeError("URL metadata document candidates contain a duplicate source URL.");
936
+ }
937
+ candidateUrls.add(candidate.url);
938
+ for (const engine of candidate.engines) {
939
+ if (!queried.has(engine) || failed.has(engine)) {
940
+ throw new TypeError("URL metadata document candidate references an unavailable or unqueried engine.");
941
+ }
942
+ }
943
+ }
944
+ const title = parseSelectedField(selected.title, "URL metadata document.selected.title", MAX_TITLE_BYTES);
945
+ const description = parseSelectedField(selected.description, "URL metadata document.selected.description", MAX_SNIPPET_BYTES);
946
+ const best = candidates[0] ?? null;
947
+ if (best === null && (title !== null || description !== null) || best !== null && (title?.value !== best.title || title.sourceUrl !== subjectUrl) || best?.snippet === null && description !== null || best?.snippet !== null && best !== null && (description?.value !== best.snippet || description.sourceUrl !== subjectUrl)) {
948
+ throw new TypeError("URL metadata document selected fields do not match the best exact candidate.");
949
+ }
950
+ const archives = Object.freeze(input.archives.map((item, index) => parseArchive(item, `URL metadata document.archives[${index}]`, subjectUrl, generatedAt)));
951
+ const archiveCaptures = new Set;
952
+ for (const archive of archives) {
953
+ if (archiveCaptures.has(archive.capturedAt)) {
954
+ throw new TypeError("URL metadata document archives contain a duplicate capture timestamp.");
955
+ }
956
+ archiveCaptures.add(archive.capturedAt);
957
+ }
958
+ const partial = enginesFailed.length > 0 || attempts.some(({ outcome }) => outcome === "failed" || outcome === "partial");
959
+ const expectedStatus = metadataStatus({
960
+ hasCandidate: best !== null,
961
+ hasArchive: archives.length > 0,
962
+ queriedEngines: enginesQueried.length,
963
+ failedEngines: enginesFailed.length,
964
+ partial
965
+ });
966
+ if (input.status !== expectedStatus) {
967
+ throw new TypeError(`URL metadata document status must be ${expectedStatus}.`);
968
+ }
969
+ return Object.freeze({
970
+ schemaVersion: URL_METADATA_SCHEMA_VERSION,
971
+ kind: "url-metadata",
972
+ subjectUrl,
973
+ generatedAt,
974
+ status: expectedStatus,
975
+ provider: Object.freeze({
976
+ id: METADATA_SEARCH_ENGINE_ID,
977
+ version: METADATA_SEARCH_ENGINE_VERSION,
978
+ revision: METADATA_SEARCH_ENGINE_REVISION,
979
+ enginesQueried,
980
+ enginesFailed
981
+ }),
982
+ attempts,
983
+ candidates,
984
+ selected: Object.freeze({
985
+ title,
986
+ description
987
+ }),
988
+ archives,
989
+ warnings: boundedTextList(input.warnings, "URL metadata document.warnings", MAX_WARNINGS, MAX_MESSAGE_BYTES)
990
+ });
991
+ }
992
+ function stableCandidateOrder(left, right) {
993
+ return right.score - left.score || right.engines.length - left.engines.length || compareText(left.title, right.title) || compareText(left.url, right.url);
994
+ }
995
+ function createUrlMetadataDocument(input) {
996
+ const candidates = [...input.candidates].sort(stableCandidateOrder).slice(0, MAX_CANDIDATES);
997
+ const best = candidates[0] ?? null;
998
+ const archives = [...input.archives ?? []].sort((left, right) => compareText(right.capturedAt, left.capturedAt) || compareText(left.url, right.url)).slice(0, MAX_ARCHIVES);
999
+ const partial = input.enginesFailed.length > 0 || input.attempts.some(({ outcome }) => outcome === "failed" || outcome === "partial");
1000
+ const status = metadataStatus({
1001
+ hasCandidate: best !== null,
1002
+ hasArchive: archives.length > 0,
1003
+ queriedEngines: input.enginesQueried.length,
1004
+ failedEngines: input.enginesFailed.length,
1005
+ partial
1006
+ });
1007
+ return parseUrlMetadataDocument({
1008
+ schemaVersion: URL_METADATA_SCHEMA_VERSION,
1009
+ kind: "url-metadata",
1010
+ subjectUrl: input.subjectUrl,
1011
+ generatedAt: input.generatedAt,
1012
+ status,
1013
+ provider: {
1014
+ id: METADATA_SEARCH_ENGINE_ID,
1015
+ version: METADATA_SEARCH_ENGINE_VERSION,
1016
+ revision: METADATA_SEARCH_ENGINE_REVISION,
1017
+ enginesQueried: [...input.enginesQueried],
1018
+ enginesFailed: [...input.enginesFailed]
1019
+ },
1020
+ attempts: [...input.attempts],
1021
+ candidates,
1022
+ selected: {
1023
+ title: best === null ? null : {
1024
+ value: best.title,
1025
+ sourceUrl: best.url,
1026
+ provider: METADATA_SEARCH_ENGINE_ID
1027
+ },
1028
+ description: best?.snippet === null || best?.snippet === undefined ? null : {
1029
+ value: best.snippet,
1030
+ sourceUrl: best.url,
1031
+ provider: METADATA_SEARCH_ENGINE_ID
1032
+ }
1033
+ },
1034
+ archives,
1035
+ warnings: [...input.warnings ?? []]
1036
+ });
1037
+ }
1038
+ function renderUrlMetadataDocument(document) {
1039
+ return `${JSON.stringify(parseUrlMetadataDocument(document), null, 2)}
1040
+ `;
1041
+ }
1042
+ function frontmatterObject(markdown, path) {
1043
+ const match = /^---(?:\r?\n)([\s\S]*?)(?:\r?\n)---(?:\r?\n|$)/u.exec(markdown);
1044
+ if (match === null)
1045
+ return {};
1046
+ const parsed = parseDocument(match[1] ?? "", { uniqueKeys: true });
1047
+ if (parsed.errors.length > 0)
1048
+ throw new Error(`Invalid YAML frontmatter in ${path}: ${parsed.errors[0]?.message ?? "unknown error"}`);
1049
+ const value = parsed.toJS({ maxAliasCount: 32 });
1050
+ if (value === null)
1051
+ return {};
1052
+ return record(value, `Frontmatter in ${path}`);
1053
+ }
1054
+ function absoluteSource(value, label, allowLocalPdf) {
1055
+ if (value === undefined)
1056
+ return null;
1057
+ if (allowLocalPdf && value === "source.pdf")
1058
+ return null;
1059
+ if (typeof value !== "string") {
1060
+ throw new TypeError(`${label} must be an absolute HTTP(S) URL${allowLocalPdf ? " or source.pdf" : ""}.`);
1061
+ }
1062
+ let parsed;
1063
+ try {
1064
+ parsed = new URL(value);
1065
+ } catch (error) {
1066
+ throw new TypeError(`${label} must be an absolute HTTP(S) URL${allowLocalPdf ? " or source.pdf" : ""}.`, { cause: error });
1067
+ }
1068
+ if (parsed.protocol !== "http:" && parsed.protocol !== "https:") {
1069
+ throw new TypeError(`${label} must use HTTP or HTTPS.`);
1070
+ }
1071
+ return normalizedPublicUrl(value, label);
1072
+ }
1073
+ function confined(root, path, label) {
1074
+ const pathFromRoot = relative(root, path);
1075
+ if (pathFromRoot === "" || pathFromRoot === ".." || pathFromRoot.startsWith(`..${sep}`) || pathFromRoot.includes(`\x00`)) {
1076
+ throw new Error(`${label} escapes the articles root.`);
1077
+ }
1078
+ }
1079
+ function identity(metadata) {
1080
+ return Object.freeze({
1081
+ device: metadata.dev,
1082
+ inode: metadata.ino,
1083
+ size: metadata.size,
1084
+ mode: metadata.mode,
1085
+ links: metadata.nlink,
1086
+ modifiedNanoseconds: metadata.mtimeNs,
1087
+ changedNanoseconds: metadata.ctimeNs
1088
+ });
1089
+ }
1090
+ function sameIdentity(left, right) {
1091
+ return left.device === right.device && left.inode === right.inode && left.size === right.size && left.mode === right.mode && left.links === right.links && left.modifiedNanoseconds === right.modifiedNanoseconds && left.changedNanoseconds === right.changedNanoseconds;
1092
+ }
1093
+ function sameInode(left, right) {
1094
+ return left.device === right.device && left.inode === right.inode;
1095
+ }
1096
+ function validateOwnedLockFile(metadata, path) {
1097
+ const effectiveUserId = process.geteuid?.();
1098
+ if (!metadata.isFile() || metadata.isSymbolicLink() || metadata.nlink !== 1n || effectiveUserId !== undefined && metadata.uid !== BigInt(effectiveUserId)) {
1099
+ throw new Error(`URL metadata write lock must be an owned regular single-link file: ${path}`);
1100
+ }
1101
+ return identity(metadata);
1102
+ }
1103
+ function readDescriptorText(descriptor, maximumBytes, label) {
1104
+ const metadata = fstatSync2(descriptor, { bigint: true });
1105
+ if (!metadata.isFile() || metadata.isSymbolicLink() || metadata.nlink !== 1n) {
1106
+ throw new Error(`${label} descriptor must be a regular single-link file.`);
1107
+ }
1108
+ if (metadata.size > BigInt(maximumBytes))
1109
+ throw new Error(`${label} exceeds ${maximumBytes} bytes.`);
1110
+ const bytes = Buffer.alloc(Number(metadata.size));
1111
+ let offset = 0;
1112
+ while (offset < bytes.byteLength) {
1113
+ const count = readSync2(descriptor, bytes, offset, bytes.byteLength - offset, offset);
1114
+ if (count === 0)
1115
+ throw new Error(`${label} ended before its validated size.`);
1116
+ offset += count;
1117
+ }
1118
+ try {
1119
+ return new TextDecoder("utf-8", { fatal: true }).decode(bytes);
1120
+ } catch (error) {
1121
+ throw new Error(`${label} must be valid UTF-8.`, { cause: error });
1122
+ }
1123
+ }
1124
+ function lockOwnerFromDescriptor(descriptor) {
1125
+ let value;
1126
+ try {
1127
+ value = JSON.parse(readDescriptorText(descriptor, MAX_LOCK_BYTES, "URL metadata write lock"));
1128
+ } catch (error) {
1129
+ throw new Error("URL metadata write lock owner metadata is malformed.", { cause: error });
1130
+ }
1131
+ return parseUrlMetadataWriteLockOwner(value);
1132
+ }
1133
+ function writeLockOwner(descriptor, owner) {
1134
+ const bytes = Buffer.from(`${JSON.stringify(owner)}
1135
+ `, "utf8");
1136
+ if (bytes.byteLength > MAX_LOCK_BYTES)
1137
+ throw new Error("URL metadata write lock owner metadata is oversized.");
1138
+ ftruncateSync(descriptor, 0);
1139
+ let offset = 0;
1140
+ while (offset < bytes.byteLength) {
1141
+ const written = writeSync2(descriptor, bytes, offset, bytes.byteLength - offset, offset);
1142
+ if (written === 0)
1143
+ throw new Error("URL metadata write lock owner metadata could not be fully written.");
1144
+ offset += written;
1145
+ }
1146
+ ftruncateSync(descriptor, bytes.byteLength);
1147
+ fchmodSync(descriptor, 384);
1148
+ fsyncSync2(descriptor);
1149
+ const parsed = lockOwnerFromDescriptor(descriptor);
1150
+ if (parsed.host !== owner.host || parsed.pid !== owner.pid || parsed.processIdentity !== owner.processIdentity || parsed.token !== owner.token || parsed.acquiredAt !== owner.acquiredAt)
1151
+ throw new Error("URL metadata write lock owner metadata failed verification.");
1152
+ return validateOwnedLockFile(fstatSync2(descriptor, { bigint: true }), "opened lock descriptor");
1153
+ }
1154
+ function assertMalformedLockIsOldEnoughToRecover(descriptor, identity2, path) {
1155
+ try {
1156
+ lockOwnerFromDescriptor(descriptor);
1157
+ return;
1158
+ } catch (error) {
1159
+ const ageNanoseconds = BigInt(Date.now()) * 1000000n - identity2.modifiedNanoseconds;
1160
+ const recoveryAgeNanoseconds = BigInt(MALFORMED_LOCK_RECOVERY_AGE_MILLISECONDS) * 1000000n;
1161
+ if (ageNanoseconds < recoveryAgeNanoseconds) {
1162
+ throw new Error(`Refusing to reclaim a fresh malformed URL metadata write lock: ${path}`, { cause: error });
1163
+ }
1164
+ }
1165
+ }
1166
+ function activeLockOwnerDescription(path, openedIdentity) {
1167
+ try {
1168
+ const publishedIdentity = validateOwnedLockFile(lstatSync2(path, { bigint: true }), path);
1169
+ if (!sameInode(openedIdentity, publishedIdentity))
1170
+ return "an active writer after concurrent lock replacement";
1171
+ const owner = parseUrlMetadataWriteLockOwner(JSON.parse(readBoundedSingleLinkFile(path, MAX_LOCK_BYTES, "URL metadata write lock").text));
1172
+ if (owner.host === hostname() && operatingSystemProcessIdentity(owner.pid) === owner.processIdentity)
1173
+ return `active writer pid ${owner.pid} (${owner.token})`;
1174
+ } catch {}
1175
+ return "an active writer with unverified owner metadata";
1176
+ }
1177
+ function acquireUrlMetadataWriteLock(path) {
1178
+ const owner = currentWriteLockOwner();
1179
+ let descriptor = null;
1180
+ let existedBeforeOpen = true;
1181
+ try {
1182
+ try {
1183
+ validateOwnedLockFile(lstatSync2(path, { bigint: true }), path);
1184
+ } catch (error) {
1185
+ const cause = error;
1186
+ if (cause.code !== "ENOENT")
1187
+ throw error;
1188
+ existedBeforeOpen = false;
1189
+ }
1190
+ descriptor = openSync2(path, constants2.O_RDWR | constants2.O_CREAT | constants2.O_NOFOLLOW, 384);
1191
+ setDescriptorCloseOnExec(descriptor);
1192
+ const openedIdentity = validateOwnedLockFile(fstatSync2(descriptor, { bigint: true }), path);
1193
+ const publishedIdentity = validateOwnedLockFile(lstatSync2(path, { bigint: true }), path);
1194
+ if (!sameInode(openedIdentity, publishedIdentity)) {
1195
+ throw new Error(`URL metadata write lock identity changed while opening: ${path}`);
1196
+ }
1197
+ if (!tryExclusiveFileLock(descriptor)) {
1198
+ throw new Error(`URL metadata sidecar is locked by ${activeLockOwnerDescription(path, openedIdentity)}.`);
1199
+ }
1200
+ if (existedBeforeOpen)
1201
+ assertMalformedLockIsOldEnoughToRecover(descriptor, openedIdentity, path);
1202
+ const identity2 = writeLockOwner(descriptor, owner);
1203
+ const ownedPathIdentity = validateOwnedLockFile(lstatSync2(path, { bigint: true }), path);
1204
+ if (!sameIdentity(identity2, ownedPathIdentity)) {
1205
+ throw new Error(`URL metadata write lock changed while publishing its owner: ${path}`);
1206
+ }
1207
+ const held = Object.freeze({ descriptor, identity: identity2, owner, path });
1208
+ descriptor = null;
1209
+ return held;
1210
+ } finally {
1211
+ if (descriptor !== null)
1212
+ closeSync2(descriptor);
1213
+ }
1214
+ }
1215
+ function releaseUrlMetadataWriteLock(held) {
1216
+ let releaseError;
1217
+ try {
1218
+ assertHeldUrlMetadataWriteLockCurrent(held);
1219
+ unlinkSync(held.path);
1220
+ } catch (error) {
1221
+ releaseError = errorFromUnknown(error, "URL metadata write lock release failed.");
1222
+ }
1223
+ try {
1224
+ closeSync2(held.descriptor);
1225
+ } catch (error) {
1226
+ releaseError ??= errorFromUnknown(error, "URL metadata write lock descriptor close failed.");
1227
+ }
1228
+ if (releaseError !== undefined)
1229
+ throw releaseError;
1230
+ }
1231
+ function assertHeldUrlMetadataWriteLockCurrent(held) {
1232
+ const openedIdentity = validateOwnedLockFile(fstatSync2(held.descriptor, { bigint: true }), held.path);
1233
+ const publishedIdentity = validateOwnedLockFile(lstatSync2(held.path, { bigint: true }), held.path);
1234
+ const owner = lockOwnerFromDescriptor(held.descriptor);
1235
+ if (!sameIdentity(held.identity, openedIdentity) || !sameIdentity(held.identity, publishedIdentity) || owner.token !== held.owner.token || owner.pid !== held.owner.pid || owner.processIdentity !== held.owner.processIdentity)
1236
+ throw new Error(`URL metadata write lock was replaced while held: ${held.path}`);
1237
+ }
1238
+ function removeOwnedTemporaryFile(path, expected) {
1239
+ let published;
1240
+ try {
1241
+ published = validateRegularSingleLink(lstatSync2(path, { bigint: true }), path, MAX_SIDECAR_BYTES, "URL metadata temporary sidecar");
1242
+ } catch (error) {
1243
+ const cause = error;
1244
+ if (cause.code === "ENOENT")
1245
+ return;
1246
+ throw error;
1247
+ }
1248
+ if (!sameIdentity(expected, published)) {
1249
+ throw new Error(`Refusing to remove a replaced URL metadata temporary sidecar: ${path}`);
1250
+ }
1251
+ unlinkSync(path);
1252
+ }
1253
+ function directoryIdentity(metadata) {
1254
+ return Object.freeze({
1255
+ device: metadata.dev,
1256
+ inode: metadata.ino,
1257
+ mode: metadata.mode
1258
+ });
1259
+ }
1260
+ function sameDirectoryIdentity(left, right) {
1261
+ return left.device === right.device && left.inode === right.inode && left.mode === right.mode;
1262
+ }
1263
+ function validateArticleDirectory(path) {
1264
+ const metadata = lstatSync2(path, { bigint: true });
1265
+ if (!metadata.isDirectory() || metadata.isSymbolicLink() || realpathSync2(path) !== path) {
1266
+ throw new Error(`Article directory must be a real directory: ${path}`);
1267
+ }
1268
+ return directoryIdentity(metadata);
1269
+ }
1270
+ function validateOpenedArticleDirectory(descriptor, path) {
1271
+ const metadata = fstatSync2(descriptor, { bigint: true });
1272
+ if (!metadata.isDirectory() || metadata.isSymbolicLink()) {
1273
+ throw new Error(`Article directory descriptor is no longer a directory: ${path}`);
1274
+ }
1275
+ return directoryIdentity(metadata);
1276
+ }
1277
+ function validateRegularSingleLink(metadata, path, maximumBytes, label) {
1278
+ if (!metadata.isFile() || metadata.isSymbolicLink() || metadata.nlink !== 1n) {
1279
+ throw new Error(`${label} must be a regular single-link file: ${path}`);
1280
+ }
1281
+ if (metadata.size > BigInt(maximumBytes))
1282
+ throw new Error(`${label} exceeds ${maximumBytes} bytes: ${path}`);
1283
+ return identity(metadata);
1284
+ }
1285
+ function readBoundedSingleLinkFile(path, maximumBytes, label) {
1286
+ const pathIdentity = validateRegularSingleLink(lstatSync2(path, { bigint: true }), path, maximumBytes, label);
1287
+ const descriptor = openSync2(path, constants2.O_RDONLY | constants2.O_NOFOLLOW);
1288
+ try {
1289
+ const openedIdentity = validateRegularSingleLink(fstatSync2(descriptor, { bigint: true }), path, maximumBytes, label);
1290
+ if (!sameIdentity(pathIdentity, openedIdentity))
1291
+ throw new Error(`${label} identity changed before read: ${path}`);
1292
+ const bytes = Buffer.alloc(Number(openedIdentity.size));
1293
+ let offset = 0;
1294
+ while (offset < bytes.byteLength) {
1295
+ const count = readSync2(descriptor, bytes, offset, bytes.byteLength - offset, offset);
1296
+ if (count === 0)
1297
+ throw new Error(`${label} ended before its validated size: ${path}`);
1298
+ offset += count;
1299
+ }
1300
+ const finishedIdentity = validateRegularSingleLink(fstatSync2(descriptor, { bigint: true }), path, maximumBytes, label);
1301
+ const finalPathIdentity = validateRegularSingleLink(lstatSync2(path, { bigint: true }), path, maximumBytes, label);
1302
+ if (!sameIdentity(openedIdentity, finishedIdentity) || !sameIdentity(openedIdentity, finalPathIdentity)) {
1303
+ throw new Error(`${label} identity changed during read: ${path}`);
1304
+ }
1305
+ let text;
1306
+ try {
1307
+ text = new TextDecoder("utf-8", { fatal: true }).decode(bytes);
1308
+ } catch (error) {
1309
+ throw new Error(`${label} must be valid UTF-8: ${path}`, { cause: error });
1310
+ }
1311
+ return Object.freeze({ text, identity: openedIdentity });
1312
+ } finally {
1313
+ closeSync2(descriptor);
1314
+ }
1315
+ }
1316
+ function discoverSavedUrlRecords(vaultRoot) {
1317
+ const resolvedVault = realpathSync2(resolve(vaultRoot));
1318
+ const vaultMetadata = lstatSync2(resolvedVault);
1319
+ if (!vaultMetadata.isDirectory() || vaultMetadata.isSymbolicLink())
1320
+ throw new Error("The vault root must be a real directory.");
1321
+ const articlesRoot = join2(resolvedVault, "articles");
1322
+ const articlesMetadata = lstatSync2(articlesRoot);
1323
+ if (!articlesMetadata.isDirectory() || articlesMetadata.isSymbolicLink())
1324
+ throw new Error("The articles root must be a real directory.");
1325
+ if (realpathSync2(articlesRoot) !== articlesRoot)
1326
+ throw new Error("The articles root must resolve to itself.");
1327
+ const entries = readdirSync(articlesRoot, { withFileTypes: true });
1328
+ const directories = entries.filter((entry) => entry.isDirectory());
1329
+ if (directories.length > MAX_ARTICLE_DIRECTORIES) {
1330
+ throw new Error(`The articles root exceeds ${MAX_ARTICLE_DIRECTORIES} directories.`);
1331
+ }
1332
+ for (const entry of entries) {
1333
+ if (entry.isSymbolicLink())
1334
+ throw new Error(`The articles root contains a symbolic link: ${entry.name}`);
1335
+ }
1336
+ const records = [];
1337
+ for (const entry of directories.sort((left, right) => compareText(left.name, right.name))) {
1338
+ const directory = join2(articlesRoot, entry.name);
1339
+ confined(articlesRoot, directory, "Article directory");
1340
+ const capturedDirectoryIdentity = validateArticleDirectory(directory);
1341
+ const children = readdirSync(directory, { withFileTypes: true });
1342
+ if (children.length > MAX_DIRECTORY_ENTRIES) {
1343
+ throw new Error(`Article directory exceeds ${MAX_DIRECTORY_ENTRIES} entries: ${directory}`);
1344
+ }
1345
+ if (children.some((child) => child.isSymbolicLink())) {
1346
+ throw new Error(`Article directory contains a symbolic link: ${directory}`);
1347
+ }
1348
+ const urlMarkdown = [];
1349
+ for (const child of children) {
1350
+ if (!child.isFile() || !child.name.endsWith(".md") || child.name === "AGENTS.md")
1351
+ continue;
1352
+ const markdownPath = join2(directory, child.name);
1353
+ const markdown = readBoundedSingleLinkFile(markdownPath, MAX_MARKDOWN_BYTES, "Article Markdown");
1354
+ const metadata = frontmatterObject(markdown.text, markdownPath);
1355
+ const source = absoluteSource(metadata.source, `${markdownPath} source`, true);
1356
+ const sourceUrl = absoluteSource(metadata.source_url, `${markdownPath} source_url`, false);
1357
+ if (source !== null && sourceUrl !== null && source !== sourceUrl) {
1358
+ throw new Error(`Article Markdown has conflicting source and source_url values: ${markdownPath}`);
1359
+ }
1360
+ const url = source ?? sourceUrl;
1361
+ if (url !== null)
1362
+ urlMarkdown.push({ path: markdownPath, identity: markdown.identity, url });
1363
+ }
1364
+ if (urlMarkdown.length > 1)
1365
+ throw new Error(`Article directory contains multiple URL-bearing Markdown files: ${directory}`);
1366
+ const selected = urlMarkdown[0];
1367
+ if (selected === undefined)
1368
+ continue;
1369
+ if (!sameDirectoryIdentity(capturedDirectoryIdentity, validateArticleDirectory(directory))) {
1370
+ throw new Error(`Article directory identity changed during inventory: ${directory}`);
1371
+ }
1372
+ const saved = {
1373
+ articleId: entry.name,
1374
+ directory,
1375
+ markdownPath: selected.path,
1376
+ sidecarPath: join2(directory, URL_METADATA_FILENAME),
1377
+ subjectUrl: selected.url
1378
+ };
1379
+ savedUrlRecordIdentities.set(saved, Object.freeze({
1380
+ directory: capturedDirectoryIdentity,
1381
+ markdown: selected.identity
1382
+ }));
1383
+ records.push(Object.freeze(saved));
1384
+ }
1385
+ return Object.freeze(records);
1386
+ }
1387
+ function fileIdentity(path) {
1388
+ return identity(lstatSync2(path, { bigint: true }));
1389
+ }
1390
+ function savedUrlRecordIdentity(saved) {
1391
+ const owned = savedUrlRecordIdentities.get(saved);
1392
+ if (owned === undefined)
1393
+ throw new Error("Saved URL record was not produced by the validated inventory.");
1394
+ return owned;
1395
+ }
1396
+ function assertSavedUrlRecordCurrent(saved) {
1397
+ const expected = savedUrlRecordIdentity(saved);
1398
+ if (dirname(saved.markdownPath) !== saved.directory || !basename(saved.markdownPath).endsWith(".md")) {
1399
+ throw new Error("Saved URL Markdown path is not owned by its article directory.");
1400
+ }
1401
+ const currentDirectoryIdentity = validateArticleDirectory(saved.directory);
1402
+ if (!sameDirectoryIdentity(expected.directory, currentDirectoryIdentity)) {
1403
+ throw new Error(`Article directory identity changed before metadata write: ${saved.directory}`);
1404
+ }
1405
+ const markdown = readBoundedSingleLinkFile(saved.markdownPath, MAX_MARKDOWN_BYTES, "Article Markdown");
1406
+ if (!sameIdentity(expected.markdown, markdown.identity)) {
1407
+ throw new Error(`Article Markdown identity changed before metadata write: ${saved.markdownPath}`);
1408
+ }
1409
+ const metadata = frontmatterObject(markdown.text, saved.markdownPath);
1410
+ const source = absoluteSource(metadata.source, `${saved.markdownPath} source`, true);
1411
+ const sourceUrl = absoluteSource(metadata.source_url, `${saved.markdownPath} source_url`, false);
1412
+ if (source !== null && sourceUrl !== null && source !== sourceUrl) {
1413
+ throw new Error(`Article Markdown has conflicting source and source_url values: ${saved.markdownPath}`);
1414
+ }
1415
+ if ((source ?? sourceUrl) !== saved.subjectUrl) {
1416
+ throw new Error(`Article Markdown source changed before metadata write: ${saved.markdownPath}`);
1417
+ }
1418
+ }
1419
+ function readOwnedSidecar(path, subjectUrl) {
1420
+ const read = readBoundedSingleLinkFile(path, MAX_SIDECAR_BYTES, "URL metadata sidecar");
1421
+ let value;
1422
+ try {
1423
+ value = JSON.parse(read.text);
1424
+ } catch (error) {
1425
+ throw new Error(`Refusing to replace malformed URL metadata sidecar: ${path}`, { cause: error });
1426
+ }
1427
+ const document = parseUrlMetadataDocument(value);
1428
+ if (document.subjectUrl !== subjectUrl) {
1429
+ throw new Error(`Refusing to replace URL metadata sidecar for a different subject: ${path}`);
1430
+ }
1431
+ return { document, identity: read.identity };
1432
+ }
1433
+ function readUrlMetadataDocument(saved) {
1434
+ if (dirname(saved.sidecarPath) !== saved.directory || basename(saved.sidecarPath) !== URL_METADATA_FILENAME) {
1435
+ throw new Error("URL metadata sidecar path is not owned by its article directory.");
1436
+ }
1437
+ assertSavedUrlRecordCurrent(saved);
1438
+ const document = readOwnedSidecar(saved.sidecarPath, saved.subjectUrl).document;
1439
+ assertSavedUrlRecordCurrent(saved);
1440
+ return document;
1441
+ }
1442
+ function writeUrlMetadataDocument(saved, document) {
1443
+ const parsed = parseUrlMetadataDocument(document);
1444
+ if (parsed.subjectUrl !== saved.subjectUrl)
1445
+ throw new Error("URL metadata document does not match its saved URL record.");
1446
+ if (dirname(saved.sidecarPath) !== saved.directory || basename(saved.sidecarPath) !== URL_METADATA_FILENAME) {
1447
+ throw new Error("URL metadata sidecar path is not owned by its article directory.");
1448
+ }
1449
+ assertSavedUrlRecordCurrent(saved);
1450
+ const rendered = renderUrlMetadataDocument(parsed);
1451
+ if (utf8Length2(rendered) > MAX_SIDECAR_BYTES)
1452
+ throw new Error("URL metadata sidecar exceeds its byte limit.");
1453
+ const lockPath = join2(saved.directory, `.${URL_METADATA_FILENAME}.lock`);
1454
+ let heldLock = null;
1455
+ let temporaryPath = null;
1456
+ let temporaryDescriptor = null;
1457
+ let temporaryIdentity = null;
1458
+ let directoryDescriptor = null;
1459
+ let operationError;
1460
+ try {
1461
+ heldLock = acquireUrlMetadataWriteLock(lockPath);
1462
+ temporaryPath = join2(saved.directory, `.${URL_METADATA_FILENAME}.${process.pid}.${heldLock.owner.token}.tmp`);
1463
+ assertSavedUrlRecordCurrent(saved);
1464
+ let existing = null;
1465
+ try {
1466
+ existing = readOwnedSidecar(saved.sidecarPath, saved.subjectUrl);
1467
+ } catch (error) {
1468
+ const cause = error;
1469
+ if (cause.code !== "ENOENT")
1470
+ throw error;
1471
+ }
1472
+ if (existing !== null && renderUrlMetadataDocument(existing.document) === rendered) {
1473
+ assertSavedUrlRecordCurrent(saved);
1474
+ return { changed: false, path: saved.sidecarPath };
1475
+ }
1476
+ temporaryDescriptor = openSync2(temporaryPath, constants2.O_WRONLY | constants2.O_CREAT | constants2.O_EXCL | constants2.O_NOFOLLOW, 384);
1477
+ setDescriptorCloseOnExec(temporaryDescriptor);
1478
+ temporaryIdentity = validateRegularSingleLink(fstatSync2(temporaryDescriptor, { bigint: true }), temporaryPath, MAX_SIDECAR_BYTES, "URL metadata temporary sidecar");
1479
+ const publishedTemporaryIdentity = validateRegularSingleLink(lstatSync2(temporaryPath, { bigint: true }), temporaryPath, MAX_SIDECAR_BYTES, "URL metadata temporary sidecar");
1480
+ if (!sameIdentity(temporaryIdentity, publishedTemporaryIdentity)) {
1481
+ throw new Error(`URL metadata temporary sidecar identity changed while opening: ${temporaryPath}`);
1482
+ }
1483
+ writeFileSync(temporaryDescriptor, rendered, "utf8");
1484
+ fchmodSync(temporaryDescriptor, 420);
1485
+ fsyncSync2(temporaryDescriptor);
1486
+ temporaryIdentity = validateRegularSingleLink(fstatSync2(temporaryDescriptor, { bigint: true }), temporaryPath, MAX_SIDECAR_BYTES, "URL metadata temporary sidecar");
1487
+ if (!sameIdentity(temporaryIdentity, validateRegularSingleLink(lstatSync2(temporaryPath, { bigint: true }), temporaryPath, MAX_SIDECAR_BYTES, "URL metadata temporary sidecar")))
1488
+ throw new Error(`URL metadata temporary sidecar changed while writing: ${temporaryPath}`);
1489
+ closeSync2(temporaryDescriptor);
1490
+ temporaryDescriptor = null;
1491
+ if (existing === null) {
1492
+ try {
1493
+ lstatSync2(saved.sidecarPath);
1494
+ throw new Error(`URL metadata sidecar appeared concurrently: ${saved.sidecarPath}`);
1495
+ } catch (error) {
1496
+ const cause = error;
1497
+ if (cause.code !== "ENOENT")
1498
+ throw error;
1499
+ }
1500
+ } else if (!sameIdentity(existing.identity, fileIdentity(saved.sidecarPath))) {
1501
+ throw new Error(`URL metadata sidecar changed concurrently: ${saved.sidecarPath}`);
1502
+ }
1503
+ directoryDescriptor = openSync2(saved.directory, constants2.O_RDONLY | constants2.O_NOFOLLOW);
1504
+ if (!sameDirectoryIdentity(savedUrlRecordIdentity(saved).directory, validateOpenedArticleDirectory(directoryDescriptor, saved.directory))) {
1505
+ throw new Error(`Article directory identity changed before metadata install: ${saved.directory}`);
1506
+ }
1507
+ assertSavedUrlRecordCurrent(saved);
1508
+ assertHeldUrlMetadataWriteLockCurrent(heldLock);
1509
+ if (!sameIdentity(temporaryIdentity, validateRegularSingleLink(lstatSync2(temporaryPath, { bigint: true }), temporaryPath, MAX_SIDECAR_BYTES, "URL metadata temporary sidecar")))
1510
+ throw new Error(`URL metadata temporary sidecar changed before install: ${temporaryPath}`);
1511
+ renameSync(temporaryPath, saved.sidecarPath);
1512
+ temporaryPath = null;
1513
+ temporaryIdentity = null;
1514
+ const installed = readOwnedSidecar(saved.sidecarPath, saved.subjectUrl);
1515
+ if (renderUrlMetadataDocument(installed.document) !== rendered) {
1516
+ throw new Error(`URL metadata sidecar failed post-install verification: ${saved.sidecarPath}`);
1517
+ }
1518
+ assertHeldUrlMetadataWriteLockCurrent(heldLock);
1519
+ fsyncSync2(directoryDescriptor);
1520
+ return { changed: true, path: saved.sidecarPath };
1521
+ } catch (error) {
1522
+ operationError = error;
1523
+ throw error;
1524
+ } finally {
1525
+ let cleanupError;
1526
+ if (directoryDescriptor !== null) {
1527
+ try {
1528
+ closeSync2(directoryDescriptor);
1529
+ } catch (error) {
1530
+ cleanupError ??= error;
1531
+ }
1532
+ }
1533
+ if (temporaryDescriptor !== null) {
1534
+ try {
1535
+ closeSync2(temporaryDescriptor);
1536
+ } catch (error) {
1537
+ cleanupError ??= error;
1538
+ }
1539
+ }
1540
+ if (temporaryPath !== null && temporaryIdentity !== null) {
1541
+ try {
1542
+ removeOwnedTemporaryFile(temporaryPath, temporaryIdentity);
1543
+ } catch (error) {
1544
+ cleanupError ??= error;
1545
+ }
1546
+ }
1547
+ if (heldLock !== null) {
1548
+ try {
1549
+ releaseUrlMetadataWriteLock(heldLock);
1550
+ } catch (error) {
1551
+ cleanupError ??= error;
1552
+ }
1553
+ }
1554
+ throwCleanupErrorWithoutOperationError(operationError, cleanupError);
1555
+ }
1556
+ }
1557
+
1558
+ // src/clip/url-metadata-backfill.ts
1559
+ var DEFAULT_MAX_RESULTS = 20;
1560
+ var DEFAULT_SEARCH_TIMEOUT_MS = 15000;
1561
+ var DEFAULT_NETWORK_VALIDATION_TIMEOUT_MS = 5000;
1562
+ var MAX_INTER_REQUEST_DELAY_MS = 60000;
1563
+ function compareText2(left, right) {
1564
+ return left < right ? -1 : left > right ? 1 : 0;
1565
+ }
1566
+ function boundedInteger(value, fallback, minimum, maximum, label) {
1567
+ const selected = value ?? fallback;
1568
+ if (!Number.isSafeInteger(selected) || selected < minimum || selected > maximum) {
1569
+ throw new TypeError(`${label} must be an integer from ${minimum} through ${maximum}.`);
1570
+ }
1571
+ return selected;
1572
+ }
1573
+ function resolveOptions(options) {
1574
+ if (typeof options.vaultRoot !== "string" || options.vaultRoot.trim() === "") {
1575
+ throw new TypeError("The URL metadata backfill vault root must be a non-empty path.");
1576
+ }
1577
+ return Object.freeze({
1578
+ vaultRoot: options.vaultRoot,
1579
+ refresh: options.refresh ?? false,
1580
+ discoverArchives: options.discoverArchives ?? false,
1581
+ interRequestDelayMs: boundedInteger(options.interRequestDelayMs, 0, 0, MAX_INTER_REQUEST_DELAY_MS, "URL metadata backfill inter-request delay"),
1582
+ maxResults: boundedInteger(options.maxResults, DEFAULT_MAX_RESULTS, 1, 20, "URL metadata backfill result limit"),
1583
+ searchTimeoutMs: boundedInteger(options.searchTimeoutMs, DEFAULT_SEARCH_TIMEOUT_MS, 500, 15000, "URL metadata backfill search timeout"),
1584
+ networkValidationTimeoutMs: boundedInteger(options.networkValidationTimeoutMs, DEFAULT_NETWORK_VALIDATION_TIMEOUT_MS, 250, 60000, "URL metadata backfill network validation timeout"),
1585
+ ...options.signal === undefined ? {} : { signal: options.signal }
1586
+ });
1587
+ }
1588
+ function abortAwareSleep(milliseconds, signal) {
1589
+ if (milliseconds === 0)
1590
+ return Promise.resolve();
1591
+ const abortError = () => signal?.reason instanceof Error ? signal.reason : new Error("URL metadata backfill aborted.");
1592
+ if (signal?.aborted === true)
1593
+ return Promise.reject(abortError());
1594
+ return new Promise((resolve2, reject) => {
1595
+ const timeout = setTimeout(finish, milliseconds);
1596
+ const abort = () => finish(abortError());
1597
+ function finish(error) {
1598
+ clearTimeout(timeout);
1599
+ signal?.removeEventListener("abort", abort);
1600
+ if (error === undefined)
1601
+ resolve2();
1602
+ else
1603
+ reject(error);
1604
+ }
1605
+ signal?.addEventListener("abort", abort, { once: true });
1606
+ });
1607
+ }
1608
+ function resolveDependencies(dependencies) {
1609
+ if (typeof dependencies.searchProvider !== "function") {
1610
+ throw new TypeError("URL metadata backfill requires a search provider.");
1611
+ }
1612
+ return Object.freeze({
1613
+ searchProvider: dependencies.searchProvider,
1614
+ discoverRecords: dependencies.discoverRecords ?? discoverSavedUrlRecords,
1615
+ readMetadata: dependencies.readMetadata ?? readUrlMetadataDocument,
1616
+ writeMetadata: dependencies.writeMetadata ?? writeUrlMetadataDocument,
1617
+ assertNetworkUrl: dependencies.assertNetworkUrl ?? assertSafeNetworkUrl,
1618
+ discoverArchive: dependencies.discoverArchive ?? discoverArchiveTodaySnapshot,
1619
+ now: dependencies.now ?? (() => new Date),
1620
+ sleep: dependencies.sleep ?? abortAwareSleep
1621
+ });
1622
+ }
1623
+ function canonicalNow(now) {
1624
+ let value;
1625
+ try {
1626
+ value = now();
1627
+ } catch (error) {
1628
+ throw new Error("URL metadata backfill could not read its clock.", { cause: error });
1629
+ }
1630
+ if (!(value instanceof Date) || !Number.isFinite(value.getTime())) {
1631
+ throw new TypeError("URL metadata backfill now must return a valid Date.");
1632
+ }
1633
+ return new Date(value.getTime());
1634
+ }
1635
+ function sortedInventory(records) {
1636
+ const sorted = [...records].sort((left, right) => compareText2(left.articleId, right.articleId) || compareText2(left.subjectUrl, right.subjectUrl) || compareText2(left.markdownPath, right.markdownPath));
1637
+ const sidecars = new Set;
1638
+ for (const record2 of sorted) {
1639
+ if (sidecars.has(record2.sidecarPath)) {
1640
+ throw new Error(`URL metadata inventory repeats a sidecar path: ${record2.sidecarPath}`);
1641
+ }
1642
+ sidecars.add(record2.sidecarPath);
1643
+ }
1644
+ return Object.freeze(sorted);
1645
+ }
1646
+ function isMissingFile(error) {
1647
+ return typeof error === "object" && error !== null && "code" in error && error.code === "ENOENT";
1648
+ }
1649
+ function isAborted(signal) {
1650
+ return signal?.aborted === true;
1651
+ }
1652
+ function readExisting(saved, readMetadata) {
1653
+ try {
1654
+ return readMetadata(saved);
1655
+ } catch (error) {
1656
+ if (isMissingFile(error))
1657
+ return null;
1658
+ throw error;
1659
+ }
1660
+ }
1661
+ function sortedStrings(values) {
1662
+ return Object.freeze([...values].sort(compareText2));
1663
+ }
1664
+ function addArchive(archives, memento, discovery) {
1665
+ const candidate = Object.freeze({
1666
+ url: memento.url,
1667
+ capturedAt: memento.capturedAt,
1668
+ discovery
1669
+ });
1670
+ const previous = archives.get(memento.timestamp);
1671
+ if (previous === undefined || previous.discovery === "metadata-search" && discovery === "newest" || previous.discovery === discovery && compareText2(candidate.url, previous.url) < 0)
1672
+ archives.set(memento.timestamp, candidate);
1673
+ }
1674
+ function collectSuccessfulSearch(subjectUrl, query, response, now, metadata, resultLimit) {
1675
+ if (response.query !== query) {
1676
+ metadata.attempts.push(Object.freeze({
1677
+ provider: "metadata-search-engine-rs",
1678
+ outcome: "failed",
1679
+ message: "Metadata search returned a response for a different query."
1680
+ }));
1681
+ return;
1682
+ }
1683
+ metadata.enginesQueried = sortedStrings(response.enginesQueried);
1684
+ metadata.enginesFailed = sortedStrings(response.enginesFailed);
1685
+ const ranked = rankMetadataSearchResults(response.results, { targetUrl: subjectUrl, limit: resultLimit });
1686
+ let archiveMatches = 0;
1687
+ let discarded = 0;
1688
+ for (const result of ranked) {
1689
+ if (result.exactTarget) {
1690
+ metadata.candidates.push(Object.freeze({
1691
+ title: result.title,
1692
+ url: result.sourceIdentity,
1693
+ snippet: result.snippet,
1694
+ engines: sortedStrings(result.engines),
1695
+ score: result.score
1696
+ }));
1697
+ continue;
1698
+ }
1699
+ try {
1700
+ const memento = parseArchiveTodayMementoUrl(result.url, { originalUrl: subjectUrl, now });
1701
+ addArchive(metadata.archives, memento, "metadata-search");
1702
+ archiveMatches += 1;
1703
+ } catch {
1704
+ discarded += 1;
1705
+ }
1706
+ }
1707
+ if (discarded > 0) {
1708
+ metadata.warnings.push(`Discarded ${discarded} search result${discarded === 1 ? "" : "s"} without an exact source binding.`);
1709
+ }
1710
+ const unavailable = response.engineStatus === "unavailable" || response.enginesFailed.length === response.enginesQueried.length;
1711
+ const partial = !unavailable && (response.engineStatus === "partial" || response.enginesFailed.length > 0);
1712
+ const usefulMatches = metadata.candidates.length + archiveMatches;
1713
+ metadata.attempts.push(Object.freeze({
1714
+ provider: "metadata-search-engine-rs",
1715
+ outcome: unavailable ? "failed" : partial ? "partial" : usefulMatches > 0 ? "succeeded" : "not-found",
1716
+ message: unavailable ? "Metadata search failed because all queried engines were unavailable." : partial ? `Metadata search completed with ${response.enginesFailed.length} failed engine${response.enginesFailed.length === 1 ? "" : "s"}.` : usefulMatches > 0 ? `Metadata search returned ${metadata.candidates.length} exact source match${metadata.candidates.length === 1 ? "" : "es"} and ${archiveMatches} bound archive match${archiveMatches === 1 ? "" : "es"}.` : "Metadata search returned no exact source or bound archive matches."
1717
+ }));
1718
+ }
1719
+ function addSearchFailure(metadata, category) {
1720
+ metadata.attempts.push(Object.freeze({
1721
+ provider: "metadata-search-engine-rs",
1722
+ outcome: "failed",
1723
+ message: `Metadata search failed (${category}).`
1724
+ }));
1725
+ }
1726
+ async function runArchiveDiscovery(saved, now, discoverArchive, metadata) {
1727
+ let outcome;
1728
+ try {
1729
+ outcome = await discoverArchive(saved.subjectUrl, { now: () => new Date(now.getTime()) });
1730
+ } catch {
1731
+ metadata.attempts.push(Object.freeze({
1732
+ provider: "archive-today",
1733
+ outcome: "failed",
1734
+ message: "Archive.today discovery failed."
1735
+ }));
1736
+ return;
1737
+ }
1738
+ if (outcome.status === "found") {
1739
+ try {
1740
+ const memento = parseArchiveTodayMementoUrl(outcome.snapshot.url, {
1741
+ originalUrl: saved.subjectUrl,
1742
+ now
1743
+ });
1744
+ addArchive(metadata.archives, memento, "newest");
1745
+ metadata.attempts.push(Object.freeze({
1746
+ provider: "archive-today",
1747
+ outcome: "succeeded",
1748
+ message: "Archive.today returned a validated newest snapshot."
1749
+ }));
1750
+ } catch {
1751
+ metadata.attempts.push(Object.freeze({
1752
+ provider: "archive-today",
1753
+ outcome: "failed",
1754
+ message: "Archive.today returned an invalid newest snapshot."
1755
+ }));
1756
+ }
1757
+ return;
1758
+ }
1759
+ if (outcome.status === "not-found") {
1760
+ metadata.attempts.push(Object.freeze({
1761
+ provider: "archive-today",
1762
+ outcome: "not-found",
1763
+ message: "Archive.today has no discoverable snapshot for this source."
1764
+ }));
1765
+ return;
1766
+ }
1767
+ if (outcome.status === "throttled") {
1768
+ metadata.attempts.push(Object.freeze({
1769
+ provider: "archive-today",
1770
+ outcome: "failed",
1771
+ message: "Archive.today throttled snapshot discovery."
1772
+ }));
1773
+ return;
1774
+ }
1775
+ metadata.attempts.push(Object.freeze({
1776
+ provider: "archive-today",
1777
+ outcome: "failed",
1778
+ message: `Archive.today snapshot discovery was unavailable (${outcome.reason}).`
1779
+ }));
1780
+ }
1781
+ function newRecordMetadata() {
1782
+ return {
1783
+ attempts: [],
1784
+ candidates: [],
1785
+ archives: new Map,
1786
+ warnings: [],
1787
+ enginesQueried: Object.freeze([]),
1788
+ enginesFailed: Object.freeze([])
1789
+ };
1790
+ }
1791
+ function skipUnsafeProviders(metadata, includeArchive, reason) {
1792
+ const explanation = reason === "network" ? "the source failed network-safety validation" : "the source could not form a disclosure-safe exact query";
1793
+ metadata.attempts.push(Object.freeze({
1794
+ provider: "metadata-search-engine-rs",
1795
+ outcome: "skipped",
1796
+ message: `Metadata search was skipped because ${explanation}.`
1797
+ }));
1798
+ if (includeArchive) {
1799
+ metadata.attempts.push(Object.freeze({
1800
+ provider: "archive-today",
1801
+ outcome: "skipped",
1802
+ message: `Archive.today discovery was skipped because ${explanation}.`
1803
+ }));
1804
+ }
1805
+ metadata.warnings.push(`No URL was disclosed because ${explanation}.`);
1806
+ }
1807
+ function item(saved, action, document) {
1808
+ return Object.freeze({
1809
+ articleId: saved.articleId,
1810
+ subjectUrl: saved.subjectUrl,
1811
+ sidecarPath: saved.sidecarPath,
1812
+ action,
1813
+ status: document.status
1814
+ });
1815
+ }
1816
+ function report(generatedAt, totalRecords, items, interrupted) {
1817
+ const skippedRecords = items.filter(({ action }) => action === "skipped").length;
1818
+ const writtenRecords = items.filter(({ action }) => action === "written").length;
1819
+ const unchangedRecords = items.filter(({ action }) => action === "unchanged").length;
1820
+ const counts = Object.freeze({
1821
+ matched: items.filter(({ status }) => status === "matched").length,
1822
+ notFound: items.filter(({ status }) => status === "not-found").length,
1823
+ partial: items.filter(({ status }) => status === "partial").length,
1824
+ unavailable: items.filter(({ status }) => status === "unavailable").length
1825
+ });
1826
+ return Object.freeze({
1827
+ generatedAt,
1828
+ totalRecords,
1829
+ processedRecords: writtenRecords + unchangedRecords,
1830
+ skippedRecords,
1831
+ writtenRecords,
1832
+ unchangedRecords,
1833
+ remainingRecords: totalRecords - items.length,
1834
+ aborted: interrupted,
1835
+ statusCounts: counts,
1836
+ items: Object.freeze([...items])
1837
+ });
1838
+ }
1839
+ async function backfillSavedUrlMetadata(options, dependencies) {
1840
+ const resolvedOptions = resolveOptions(options);
1841
+ const resolved = resolveDependencies(dependencies);
1842
+ const now = canonicalNow(resolved.now);
1843
+ const generatedAt = now.toISOString();
1844
+ const records = sortedInventory(resolved.discoverRecords(resolvedOptions.vaultRoot));
1845
+ const items = [];
1846
+ let interrupted = isAborted(resolvedOptions.signal);
1847
+ let outboundRequestStarted = false;
1848
+ const waitForRequestSlot = async () => {
1849
+ if (isAborted(resolvedOptions.signal))
1850
+ return false;
1851
+ if (outboundRequestStarted && resolvedOptions.interRequestDelayMs > 0) {
1852
+ try {
1853
+ await resolved.sleep(resolvedOptions.interRequestDelayMs, resolvedOptions.signal);
1854
+ } catch (error) {
1855
+ if (isAborted(resolvedOptions.signal))
1856
+ return false;
1857
+ throw error;
1858
+ }
1859
+ if (isAborted(resolvedOptions.signal))
1860
+ return false;
1861
+ }
1862
+ outboundRequestStarted = true;
1863
+ return true;
1864
+ };
1865
+ for (const saved of records) {
1866
+ if (interrupted || isAborted(resolvedOptions.signal)) {
1867
+ interrupted = true;
1868
+ break;
1869
+ }
1870
+ const existing = readExisting(saved, resolved.readMetadata);
1871
+ if (existing !== null && !resolvedOptions.refresh) {
1872
+ items.push(item(saved, "skipped", existing));
1873
+ continue;
1874
+ }
1875
+ const metadata = newRecordMetadata();
1876
+ let safeForDisclosure = true;
1877
+ try {
1878
+ await resolved.assertNetworkUrl(new URL(saved.subjectUrl), false, resolvedOptions.networkValidationTimeoutMs);
1879
+ } catch {
1880
+ safeForDisclosure = false;
1881
+ skipUnsafeProviders(metadata, resolvedOptions.discoverArchives, "network");
1882
+ }
1883
+ const query = safeForDisclosure ? createExactUrlSearchQuery(saved.subjectUrl) : null;
1884
+ if (safeForDisclosure && query === null) {
1885
+ safeForDisclosure = false;
1886
+ skipUnsafeProviders(metadata, resolvedOptions.discoverArchives, "query");
1887
+ }
1888
+ if (safeForDisclosure && query !== null) {
1889
+ if (!await waitForRequestSlot()) {
1890
+ interrupted = true;
1891
+ break;
1892
+ }
1893
+ try {
1894
+ const outcome = await resolved.searchProvider({
1895
+ query,
1896
+ maxResults: resolvedOptions.maxResults,
1897
+ timeoutMs: resolvedOptions.searchTimeoutMs,
1898
+ ...resolvedOptions.signal === undefined ? {} : { signal: resolvedOptions.signal }
1899
+ });
1900
+ if (isAborted(resolvedOptions.signal)) {
1901
+ interrupted = true;
1902
+ break;
1903
+ }
1904
+ if (outcome.status === "failure")
1905
+ addSearchFailure(metadata, outcome.category);
1906
+ else
1907
+ collectSuccessfulSearch(saved.subjectUrl, query, outcome.response, now, metadata, resolvedOptions.maxResults);
1908
+ } catch {
1909
+ if (isAborted(resolvedOptions.signal)) {
1910
+ interrupted = true;
1911
+ break;
1912
+ }
1913
+ addSearchFailure(metadata, "provider-threw");
1914
+ }
1915
+ if (resolvedOptions.discoverArchives) {
1916
+ if (!await waitForRequestSlot()) {
1917
+ interrupted = true;
1918
+ break;
1919
+ }
1920
+ await runArchiveDiscovery(saved, now, resolved.discoverArchive, metadata);
1921
+ if (isAborted(resolvedOptions.signal)) {
1922
+ interrupted = true;
1923
+ break;
1924
+ }
1925
+ }
1926
+ }
1927
+ const document = createUrlMetadataDocument({
1928
+ subjectUrl: saved.subjectUrl,
1929
+ generatedAt,
1930
+ enginesQueried: metadata.enginesQueried,
1931
+ enginesFailed: metadata.enginesFailed,
1932
+ attempts: metadata.attempts,
1933
+ candidates: metadata.candidates,
1934
+ archives: [...metadata.archives.values()],
1935
+ warnings: metadata.warnings
1936
+ });
1937
+ const write = resolved.writeMetadata(saved, document);
1938
+ items.push(item(saved, write.changed ? "written" : "unchanged", document));
1939
+ }
1940
+ return report(generatedAt, records.length, items, interrupted);
1941
+ }
1942
+
1943
+ export { isolatedMetadataSearchEnvironment, createExactUrlSearchQuery, createRustMetadataSearchProvider, parseUrlMetadataDocument, createUrlMetadataDocument, renderUrlMetadataDocument, discoverSavedUrlRecords, readUrlMetadataDocument, writeUrlMetadataDocument, backfillSavedUrlMetadata };