@zeldrisho/pi-web-fetch 0.10.2 → 0.11.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +30 -1
- package/LICENSE +1 -1
- package/README.md +15 -15
- package/package.json +3 -2
- package/src/{fetch.ts → acquisition/fetch.ts} +258 -72
- package/src/{llms.ts → acquisition/llms.ts} +12 -9
- package/src/challenges/anubis-solver.ts +255 -0
- package/src/challenges/anubis-worker.ts +18 -0
- package/src/challenges/anubis.ts +555 -0
- package/src/{content.ts → document/content.ts} +1 -1
- package/src/{evidence.ts → document/evidence.ts} +32 -1
- package/src/{extract.ts → document/extract.ts} +53 -3
- package/src/{fragments.ts → document/fragments.ts} +1 -1
- package/src/index.ts +27 -11
- package/src/network/cookies.ts +260 -0
- package/src/{network-redirects.ts → network/network-redirects.ts} +12 -4
- package/src/{network-transport.ts → network/network-transport.ts} +37 -1
- package/src/{network.ts → network/network.ts} +3 -2
- package/src/service.ts +7 -7
- package/workers/anubis-solver.js +164 -0
- package/workers/anubis-worker.js +15 -0
- /package/src/{fetch-metadata.ts → acquisition/fetch-metadata.ts} +0 -0
- /package/src/{focus.ts → document/focus.ts} +0 -0
- /package/src/{fragment-index.ts → document/fragment-index.ts} +0 -0
- /package/src/{markdown-cleanup.ts → document/markdown-cleanup.ts} +0 -0
- /package/src/{outline.ts → document/outline.ts} +0 -0
- /package/src/{network-policy.ts → network/network-policy.ts} +0 -0
|
@@ -0,0 +1,255 @@
|
|
|
1
|
+
import { createHash } from "node:crypto";
|
|
2
|
+
|
|
3
|
+
/** Maximum length in bytes of the server-provided random challenge data. */
|
|
4
|
+
export const ANUBIS_MAX_RANDOM_DATA_BYTES = 4_096;
|
|
5
|
+
|
|
6
|
+
/** Maximum size in bytes of a server-provided WebAssembly challenge module. */
|
|
7
|
+
export const ANUBIS_MAX_WASM_BYTES = 4 * 1_024 * 1_024;
|
|
8
|
+
|
|
9
|
+
/** A solved Anubis proof of work, using the nonce and hash the server expects. */
|
|
10
|
+
export interface AnubisSolution {
|
|
11
|
+
nonce: number;
|
|
12
|
+
hash: string;
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
/** A request to solve one Anubis challenge inside a worker thread. */
|
|
16
|
+
export type AnubisWorkerRequest =
|
|
17
|
+
| {
|
|
18
|
+
kind: "sha256";
|
|
19
|
+
randomData: string;
|
|
20
|
+
difficulty: number;
|
|
21
|
+
deadline: number;
|
|
22
|
+
}
|
|
23
|
+
| {
|
|
24
|
+
kind: "wasm";
|
|
25
|
+
moduleBytes: Uint8Array;
|
|
26
|
+
randomData: string;
|
|
27
|
+
difficulty: number;
|
|
28
|
+
deadline: number;
|
|
29
|
+
};
|
|
30
|
+
|
|
31
|
+
/** The host function Anubis baseline WebAssembly modules import. */
|
|
32
|
+
const ANUBIS_WASM_IMPORT = { module: "anubis", name: "anubis_update_nonce" } as const;
|
|
33
|
+
|
|
34
|
+
/** Exports required from an Anubis baseline WebAssembly module. */
|
|
35
|
+
const ANUBIS_WASM_EXPORTS = [
|
|
36
|
+
"data_ptr",
|
|
37
|
+
"set_data_length",
|
|
38
|
+
"anubis_work",
|
|
39
|
+
"result_hash_ptr",
|
|
40
|
+
"result_hash_size",
|
|
41
|
+
] as const;
|
|
42
|
+
|
|
43
|
+
/** Decodes a lowercase or uppercase hexadecimal string, rejecting odd or non-hex input. */
|
|
44
|
+
function decodeHex(value: string): Uint8Array | undefined {
|
|
45
|
+
if (value.length === 0 || value.length % 2 !== 0 || !/^[0-9a-fA-F]+$/.test(value))
|
|
46
|
+
return undefined;
|
|
47
|
+
|
|
48
|
+
const bytes = new Uint8Array(value.length / 2);
|
|
49
|
+
|
|
50
|
+
for (let index = 0; index < bytes.length; index += 1) {
|
|
51
|
+
bytes[index] = Number.parseInt(value.slice(index * 2, index * 2 + 2), 16);
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
return bytes;
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
/** Returns a defensive snapshot of a WebAssembly memory range. */
|
|
58
|
+
function readMemory(memory: WebAssembly.Memory, pointer: number, size: number): Uint8Array {
|
|
59
|
+
const offset = pointer >>> 0;
|
|
60
|
+
|
|
61
|
+
if (size < 0 || offset + size > memory.buffer.byteLength)
|
|
62
|
+
throw new Error("Anubis challenge module returned an out-of-bounds pointer.");
|
|
63
|
+
|
|
64
|
+
return new Uint8Array(memory.buffer, offset, size).slice();
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
/** Copies bytes into a validated WebAssembly memory range. */
|
|
68
|
+
function writeMemory(memory: WebAssembly.Memory, pointer: number, bytes: Uint8Array): void {
|
|
69
|
+
const offset = pointer >>> 0;
|
|
70
|
+
|
|
71
|
+
if (offset + bytes.byteLength > memory.buffer.byteLength)
|
|
72
|
+
throw new Error("Anubis challenge module returned an out-of-bounds pointer.");
|
|
73
|
+
|
|
74
|
+
new Uint8Array(memory.buffer, offset, bytes.byteLength).set(bytes);
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
/** Returns the number of leading zero bits in a byte sequence. */
|
|
78
|
+
function leadingZeroBits(bytes: Uint8Array): number {
|
|
79
|
+
let bits = 0;
|
|
80
|
+
|
|
81
|
+
for (const byte of bytes) {
|
|
82
|
+
if (byte === 0) {
|
|
83
|
+
bits += 8;
|
|
84
|
+
continue;
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
bits += Math.clz32(byte) - 24;
|
|
88
|
+
break;
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
return bits;
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
/** Narrows a WebAssembly export value to a numeric function. */
|
|
95
|
+
function isWasmFunction(
|
|
96
|
+
value: WebAssembly.ExportValue | undefined,
|
|
97
|
+
): value is (...args: number[]) => number {
|
|
98
|
+
return typeof value === "function";
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
/** Reads a required numeric function export from an instantiated module. */
|
|
102
|
+
function requireFunction(
|
|
103
|
+
exports: WebAssembly.Exports,
|
|
104
|
+
name: string,
|
|
105
|
+
): (...args: number[]) => number {
|
|
106
|
+
const value = exports[name];
|
|
107
|
+
|
|
108
|
+
if (!isWasmFunction(value)) throw new Error(`Anubis challenge module is missing ${name}.`);
|
|
109
|
+
|
|
110
|
+
return value;
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
/**
|
|
114
|
+
* Verifies that a module only imports the Anubis host function and exports the
|
|
115
|
+
* interface the baseline solver needs, so a hostile server cannot pass a module
|
|
116
|
+
* that reaches outside its sandbox.
|
|
117
|
+
*/
|
|
118
|
+
function assertSupportedModule(module: WebAssembly.Module): void {
|
|
119
|
+
const imports = WebAssembly.Module.imports(module);
|
|
120
|
+
|
|
121
|
+
if (
|
|
122
|
+
imports.length !== 1 ||
|
|
123
|
+
imports[0].module !== ANUBIS_WASM_IMPORT.module ||
|
|
124
|
+
imports[0].name !== ANUBIS_WASM_IMPORT.name ||
|
|
125
|
+
imports[0].kind !== "function"
|
|
126
|
+
) {
|
|
127
|
+
throw new Error("Anubis challenge module requested unsupported imports.");
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
const exported = new Set(
|
|
131
|
+
WebAssembly.Module.exports(module)
|
|
132
|
+
.filter((entry) => entry.kind === "function")
|
|
133
|
+
.map((entry) => entry.name),
|
|
134
|
+
);
|
|
135
|
+
|
|
136
|
+
for (const name of ANUBIS_WASM_EXPORTS) {
|
|
137
|
+
if (!exported.has(name)) throw new Error(`Anubis challenge module is missing ${name}.`);
|
|
138
|
+
}
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
/**
|
|
142
|
+
* Solves a `fast` or `slow` challenge by searching for a nonce whose
|
|
143
|
+
* `sha256(randomData + nonce)` has `difficulty` leading zero hexadecimal digits.
|
|
144
|
+
*
|
|
145
|
+
* @param randomData - The hex challenge string supplied by the server.
|
|
146
|
+
* @param difficulty - Required number of leading zero hex digits.
|
|
147
|
+
* @param deadline - Epoch milliseconds after which the search aborts.
|
|
148
|
+
* @returns The matching nonce and its hex digest.
|
|
149
|
+
*/
|
|
150
|
+
export function solveAnubisSha256Hex(
|
|
151
|
+
randomData: string,
|
|
152
|
+
difficulty: number,
|
|
153
|
+
deadline: number,
|
|
154
|
+
): AnubisSolution {
|
|
155
|
+
if (!Number.isInteger(difficulty) || difficulty < 0 || difficulty > 64)
|
|
156
|
+
throw new Error("Anubis challenge requested an unsupported difficulty.");
|
|
157
|
+
|
|
158
|
+
const prefix = "0".repeat(difficulty);
|
|
159
|
+
|
|
160
|
+
for (let nonce = 0; nonce <= 0xffff_ffff; nonce += 1) {
|
|
161
|
+
if ((nonce & 0xfff) === 0 && Date.now() > deadline)
|
|
162
|
+
throw new Error("Anubis proof of work exceeded the time budget.");
|
|
163
|
+
|
|
164
|
+
const hash = createHash("sha256").update(randomData).update(String(nonce)).digest("hex");
|
|
165
|
+
|
|
166
|
+
if (hash.startsWith(prefix)) return { nonce, hash };
|
|
167
|
+
}
|
|
168
|
+
|
|
169
|
+
throw new Error("Anubis proof of work exhausted the nonce space.");
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
/**
|
|
173
|
+
* Solves a `sha256`, `hashx`, or `argon2id` challenge by running the
|
|
174
|
+
* server-provided baseline WebAssembly module inside this process.
|
|
175
|
+
*
|
|
176
|
+
* The module is validated before instantiation: it may import only the
|
|
177
|
+
* `anubis_update_nonce` host function and must export the numeric interface the
|
|
178
|
+
* baseline solver uses. The returned hash is checked against the requested
|
|
179
|
+
* difficulty before it is submitted.
|
|
180
|
+
*
|
|
181
|
+
* @param moduleBytes - Compiled bytes of the baseline module.
|
|
182
|
+
* @param randomData - The hex challenge string supplied by the server.
|
|
183
|
+
* @param difficulty - Required number of leading zero bits in the result hash.
|
|
184
|
+
* @param deadline - Epoch milliseconds after which the solve is abandoned.
|
|
185
|
+
* @returns The matching nonce and its hex digest.
|
|
186
|
+
*/
|
|
187
|
+
export function solveAnubisWasm(
|
|
188
|
+
moduleBytes: Uint8Array,
|
|
189
|
+
randomData: string,
|
|
190
|
+
difficulty: number,
|
|
191
|
+
deadline: number,
|
|
192
|
+
): AnubisSolution {
|
|
193
|
+
if (moduleBytes.byteLength === 0 || moduleBytes.byteLength > ANUBIS_MAX_WASM_BYTES)
|
|
194
|
+
throw new Error("Anubis challenge module exceeded the size limit.");
|
|
195
|
+
|
|
196
|
+
if (!Number.isInteger(difficulty) || difficulty < 0 || difficulty > 0xffff_ffff)
|
|
197
|
+
throw new Error("Anubis challenge requested an unsupported difficulty.");
|
|
198
|
+
|
|
199
|
+
const data = decodeHex(randomData);
|
|
200
|
+
|
|
201
|
+
if (!data || data.byteLength > ANUBIS_MAX_RANDOM_DATA_BYTES)
|
|
202
|
+
throw new Error("Anubis challenge supplied unsupported random data.");
|
|
203
|
+
|
|
204
|
+
if (Date.now() > deadline) throw new Error("Anubis proof of work exceeded the time budget.");
|
|
205
|
+
|
|
206
|
+
const module = new WebAssembly.Module(new Uint8Array(moduleBytes));
|
|
207
|
+
assertSupportedModule(module);
|
|
208
|
+
|
|
209
|
+
const instance = new WebAssembly.Instance(module, {
|
|
210
|
+
anubis: { anubis_update_nonce: () => {} },
|
|
211
|
+
});
|
|
212
|
+
|
|
213
|
+
const memory = instance.exports.memory;
|
|
214
|
+
|
|
215
|
+
if (!(memory instanceof WebAssembly.Memory))
|
|
216
|
+
throw new Error("Anubis challenge module is missing memory.");
|
|
217
|
+
|
|
218
|
+
const dataPtr = requireFunction(instance.exports, "data_ptr");
|
|
219
|
+
const setDataLength = requireFunction(instance.exports, "set_data_length");
|
|
220
|
+
const work = requireFunction(instance.exports, "anubis_work");
|
|
221
|
+
const resultHashPtr = requireFunction(instance.exports, "result_hash_ptr");
|
|
222
|
+
const resultHashSize = requireFunction(instance.exports, "result_hash_size");
|
|
223
|
+
|
|
224
|
+
writeMemory(memory, dataPtr(), data);
|
|
225
|
+
setDataLength(data.byteLength);
|
|
226
|
+
const nonce = work(difficulty, 0, 1);
|
|
227
|
+
const hashSize = resultHashSize() >>> 0;
|
|
228
|
+
|
|
229
|
+
if (hashSize < 1 || hashSize > 4_096)
|
|
230
|
+
throw new Error("Anubis challenge module returned an invalid hash size.");
|
|
231
|
+
|
|
232
|
+
const hashBytes = readMemory(memory, resultHashPtr(), hashSize);
|
|
233
|
+
|
|
234
|
+
if (Date.now() > deadline) throw new Error("Anubis proof of work exceeded the time budget.");
|
|
235
|
+
|
|
236
|
+
if (difficulty > 0 && leadingZeroBits(hashBytes) < difficulty)
|
|
237
|
+
throw new Error("Anubis proof of work produced an invalid solution.");
|
|
238
|
+
|
|
239
|
+
const hash = Array.from(hashBytes, (byte) => byte.toString(16).padStart(2, "0")).join("");
|
|
240
|
+
|
|
241
|
+
return { nonce: nonce >>> 0, hash };
|
|
242
|
+
}
|
|
243
|
+
|
|
244
|
+
/** Dispatches one worker solve request to the matching pure solver. */
|
|
245
|
+
export function solveAnubisRequest(request: AnubisWorkerRequest): AnubisSolution {
|
|
246
|
+
if (request.kind === "sha256")
|
|
247
|
+
return solveAnubisSha256Hex(request.randomData, request.difficulty, request.deadline);
|
|
248
|
+
|
|
249
|
+
return solveAnubisWasm(
|
|
250
|
+
request.moduleBytes,
|
|
251
|
+
request.randomData,
|
|
252
|
+
request.difficulty,
|
|
253
|
+
request.deadline,
|
|
254
|
+
);
|
|
255
|
+
}
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
/* v8 ignore file -- executes only inside a worker thread; covered through the parent's integration tests. */
|
|
2
|
+
import { parentPort, workerData } from "node:worker_threads";
|
|
3
|
+
import { solveAnubisRequest, type AnubisWorkerRequest } from "./anubis-solver.ts";
|
|
4
|
+
|
|
5
|
+
/** Solves the challenge supplied as worker data and reports the outcome to the parent. */
|
|
6
|
+
function run(): void {
|
|
7
|
+
const request: AnubisWorkerRequest = workerData;
|
|
8
|
+
|
|
9
|
+
try {
|
|
10
|
+
parentPort?.postMessage({ kind: "solved", solution: solveAnubisRequest(request) });
|
|
11
|
+
} catch (error) {
|
|
12
|
+
const message = error instanceof Error ? error.message : "Anubis worker failed.";
|
|
13
|
+
|
|
14
|
+
parentPort?.postMessage({ kind: "error", message });
|
|
15
|
+
}
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
run();
|