@crawlcheck/sdk 1.0.5 → 1.0.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +30 -0
- package/dist/index.d.ts +172 -2
- package/dist/index.js +84 -2
- package/dist/log.d.ts +141 -0
- package/dist/log.js +194 -0
- package/dist/staple.d.ts +21 -0
- package/dist/staple.js +88 -0
- package/dist/verify.d.ts +1 -0
- package/dist/verify.js +53 -2
- package/package.json +1 -1
- package/test/fixtures/split-view.json +1 -0
- package/test/fixtures/staple.json +7 -0
- package/test/split-view.test.js +34 -0
- package/test/staple.test.js +14 -0
package/README.md
CHANGED
|
@@ -37,6 +37,19 @@ await cc.get("/api/explain", { domain: "example.com", code: "NO_LLMS_TXT" });
|
|
|
37
37
|
await cc.get("/api/explain", { website: "x" }); // compile error: not a parameter of this operation
|
|
38
38
|
```
|
|
39
39
|
|
|
40
|
+
## Use the site's staple
|
|
41
|
+
|
|
42
|
+
A site can serve its own signed verdict in the `CrawlCheck-Staple` header (and at `/.well-known/crawlcheck-staple.json`). `resolveFor` uses it when it verifies, with no request to CrawlCheck, and falls back to a signed lookup when it is missing, expired, for another domain or tampered:
|
|
43
|
+
|
|
44
|
+
```js
|
|
45
|
+
const res = await fetch("https://shop.example.com/");
|
|
46
|
+
const v = await client.resolveFor("shop.example.com", { response: res });
|
|
47
|
+
v.source; // "staple" or "network"
|
|
48
|
+
v.decisions.transact; // allow | warn | require_confirmation | block | unsupported
|
|
49
|
+
```
|
|
50
|
+
|
|
51
|
+
`stapleCheck(record, { host, publishedKids })` does the offline check on its own. Spec: https://crawlcheck.io/spec/staple
|
|
52
|
+
|
|
40
53
|
## Do not trust the API: check the evidence yourself
|
|
41
54
|
|
|
42
55
|
```js
|
|
@@ -66,6 +79,23 @@ suite.ok; // every fixture gave exactly its expecte
|
|
|
66
79
|
|
|
67
80
|
`npm test` in the installed package runs the conformance suite and live checks against crawlcheck.io.
|
|
68
81
|
|
|
82
|
+
## Catch a split view
|
|
83
|
+
|
|
84
|
+
Every Resolve answer names its leaf in CrawlCheck's transparency log and, once merged, carries the signed tree head of its batch. The client keeps every head it sees in `client.log` (plain JSON) and checks each new one against its neighbours: two heads for one batch, or a head that does not chain to the one before, is a **fork**, kept with both signed heads as proof. `guard()` refuses to proceed while the client holds one.
|
|
85
|
+
|
|
86
|
+
```js
|
|
87
|
+
import fs from "node:fs";
|
|
88
|
+
import { CrawlCheck, newLogStore } from "@crawlcheck/sdk";
|
|
89
|
+
const store = fs.existsSync("cc-log.json") ? JSON.parse(fs.readFileSync("cc-log.json", "utf8")) : newLogStore();
|
|
90
|
+
const client = new CrawlCheck({ logStore: store });
|
|
91
|
+
await client.resolve("example.com"); // the answer's head is checked against the store
|
|
92
|
+
const r = await client.logCheck(); // current head + GitHub's witness co-signature + the chain down to your heads
|
|
93
|
+
console.log(r.summary); // "consistent: ..." or "SPLIT VIEW: ..."
|
|
94
|
+
fs.writeFileSync("cc-log.json", JSON.stringify(client.log));
|
|
95
|
+
```
|
|
96
|
+
|
|
97
|
+
`logCheck` verifies the outside witness: a GitHub Actions run re-checks the chain and co-signs the newest head with a GitHub OIDC token (RS256, GitHub's key) whose audience is the head's digest. A head the witness's chain does not contain shows up as a fork. Heads newer than the witnessed one are listed as `unwitnessed`. Clients can also swap heads directly: `logHeads(store)` on one side, `client.logGossip(heads)` on the other. Forks are reported to `/api/v1/log/gossip` (pass `{ report: false }` to skip).
|
|
98
|
+
|
|
69
99
|
## Errors
|
|
70
100
|
|
|
71
101
|
A refused request throws `CrawlCheckError` with `status`, `body` and `locked` (true for "this needs a licence for the domain").
|
package/dist/index.d.ts
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import type { paths, components } from "./gen/openapi.js";
|
|
2
|
+
import * as S from "./staple.js";
|
|
2
3
|
export type { paths, components };
|
|
3
4
|
/** One verifier check. ok: true passed, false FAILED, null could not run (why says why) - never a pass. */
|
|
4
5
|
export interface VerifyCheck {
|
|
@@ -222,6 +223,139 @@ export interface GuardOptions extends PreflightOptions {
|
|
|
222
223
|
/** Also require the signing key to be in the published key directory (default true). */
|
|
223
224
|
onlineKeys?: boolean;
|
|
224
225
|
}
|
|
226
|
+
export interface LogHead {
|
|
227
|
+
kind: "crawlcheck-log-sth";
|
|
228
|
+
log_id: string;
|
|
229
|
+
batch: number;
|
|
230
|
+
first_index: number;
|
|
231
|
+
batch_size: number;
|
|
232
|
+
tree_size: number;
|
|
233
|
+
batch_root: string;
|
|
234
|
+
prev_sth_sha256: string | null;
|
|
235
|
+
created_at: string;
|
|
236
|
+
sha256: string;
|
|
237
|
+
signature: JsonObject;
|
|
238
|
+
[k: string]: unknown;
|
|
239
|
+
}
|
|
240
|
+
export interface LogFork {
|
|
241
|
+
kind: "two_heads_for_one_batch" | "chain_break" | "size_mismatch" | "inconsistent_head";
|
|
242
|
+
batch: number;
|
|
243
|
+
why: string;
|
|
244
|
+
a: LogHead;
|
|
245
|
+
b: LogHead;
|
|
246
|
+
detected_at: string;
|
|
247
|
+
proof: string;
|
|
248
|
+
}
|
|
249
|
+
export interface LogStore {
|
|
250
|
+
v: 1;
|
|
251
|
+
log_id: string;
|
|
252
|
+
heads: Record<string, {
|
|
253
|
+
batch: number;
|
|
254
|
+
sha256: string;
|
|
255
|
+
first_index: number;
|
|
256
|
+
batch_size: number;
|
|
257
|
+
tree_size: number;
|
|
258
|
+
prev_sth_sha256: string | null;
|
|
259
|
+
created_at: string | null;
|
|
260
|
+
sth: LogHead;
|
|
261
|
+
}>;
|
|
262
|
+
forks: LogFork[];
|
|
263
|
+
witnessed: {
|
|
264
|
+
batch: number;
|
|
265
|
+
sha256: string;
|
|
266
|
+
at: string | null;
|
|
267
|
+
run_id: string | null;
|
|
268
|
+
} | null;
|
|
269
|
+
}
|
|
270
|
+
export interface LogObservation {
|
|
271
|
+
state: "new" | "known" | "fork" | "invalid" | "none";
|
|
272
|
+
why?: string;
|
|
273
|
+
fork?: LogFork;
|
|
274
|
+
}
|
|
275
|
+
export interface LogCheckResult {
|
|
276
|
+
consistent: boolean;
|
|
277
|
+
forks: LogFork[];
|
|
278
|
+
all_forks: number;
|
|
279
|
+
witnessed: {
|
|
280
|
+
batch: number;
|
|
281
|
+
sha256: string;
|
|
282
|
+
at: string | null;
|
|
283
|
+
run_id: string | null;
|
|
284
|
+
token: boolean | null;
|
|
285
|
+
why: string;
|
|
286
|
+
observed: string;
|
|
287
|
+
warning?: string;
|
|
288
|
+
} | null;
|
|
289
|
+
unwitnessed: {
|
|
290
|
+
batch: number;
|
|
291
|
+
sha256: string;
|
|
292
|
+
created_at: string | null;
|
|
293
|
+
}[];
|
|
294
|
+
stale_unwitnessed: {
|
|
295
|
+
batch: number;
|
|
296
|
+
sha256: string;
|
|
297
|
+
created_at: string | null;
|
|
298
|
+
}[];
|
|
299
|
+
summary: string;
|
|
300
|
+
}
|
|
301
|
+
export declare const LOG_ID: string;
|
|
302
|
+
/** A new, empty transparency-log store (plain JSON: persist it between runs). */
|
|
303
|
+
export declare const newLogStore: () => LogStore;
|
|
304
|
+
/** Check one signed tree head against the store and keep it. A fork carries both contradicting signed heads. */
|
|
305
|
+
export declare const logObserve: (store: LogStore, sth: unknown, opts?: {
|
|
306
|
+
publishedKids?: string[];
|
|
307
|
+
maxHeads?: number;
|
|
308
|
+
}) => Promise<LogObservation>;
|
|
309
|
+
/** Check the head inside a Resolve answer (transparency.sth), if it carries one. */
|
|
310
|
+
export declare const logObserveAnswer: (store: LogStore, answer: unknown, opts?: {
|
|
311
|
+
publishedKids?: string[];
|
|
312
|
+
}) => Promise<LogObservation>;
|
|
313
|
+
/** The signed heads a store holds, newest first: what a client sends its peers. */
|
|
314
|
+
export declare const logHeads: (store: LogStore, limit?: number) => LogHead[];
|
|
315
|
+
/** Gossip: check another client's heads against this store. */
|
|
316
|
+
export declare const logGossip: (store: LogStore, sths: unknown[], opts?: {
|
|
317
|
+
publishedKids?: string[];
|
|
318
|
+
}) => Promise<{
|
|
319
|
+
forks: LogFork[];
|
|
320
|
+
results: (LogObservation & {
|
|
321
|
+
batch: number;
|
|
322
|
+
sha256: string;
|
|
323
|
+
})[];
|
|
324
|
+
}>;
|
|
325
|
+
/** Verify the outside witness's co-signature: a GitHub OIDC JWT (RS256) whose audience is crawlcheck-sth:<head sha256>. */
|
|
326
|
+
export declare const verifyWitnessToken: (token: string, sthSha256: string, jwks: unknown) => Promise<{
|
|
327
|
+
ok: boolean | null;
|
|
328
|
+
why: string;
|
|
329
|
+
claims?: JsonObject;
|
|
330
|
+
}>;
|
|
331
|
+
export declare const GITHUB_OIDC_JWKS = "https://token.actions.githubusercontent.com/.well-known/jwks";
|
|
332
|
+
export type StapleDecisions = {
|
|
333
|
+
read: string | null;
|
|
334
|
+
cite: string | null;
|
|
335
|
+
connect: string | null;
|
|
336
|
+
transact: string | null;
|
|
337
|
+
};
|
|
338
|
+
export interface StapleCheck {
|
|
339
|
+
valid: boolean;
|
|
340
|
+
reason: string | null;
|
|
341
|
+
expired: boolean;
|
|
342
|
+
record: Record<string, string> | null;
|
|
343
|
+
decisions: StapleDecisions | null;
|
|
344
|
+
}
|
|
345
|
+
/** Parse a ccr1 record (CrawlCheck-Staple header value or <domain>._v.crawlcheck.io TXT). */
|
|
346
|
+
export declare const parseStaple: (record: string) => {
|
|
347
|
+
fields: Record<string, string>;
|
|
348
|
+
body: string;
|
|
349
|
+
sig: string;
|
|
350
|
+
};
|
|
351
|
+
/** The CrawlCheck-Staple header of a response (fetch Response, Headers or a plain header map), or null. */
|
|
352
|
+
export declare const stapleFromResponse: (res: unknown) => string | null;
|
|
353
|
+
/** Verify a staple offline: published key, signature, the record names opts.host (or a parent), fresh_until in the future. Expired or foreign staples are never valid. */
|
|
354
|
+
export declare const stapleCheck: (record: string, opts: {
|
|
355
|
+
host: string;
|
|
356
|
+
publishedKids: string[];
|
|
357
|
+
now?: number;
|
|
358
|
+
}) => Promise<StapleCheck>;
|
|
225
359
|
export interface ClientOptions {
|
|
226
360
|
/** Licence key (cc_ + 32 hex). Without one, licence-gated fields come back withheld, exactly as on the website. */
|
|
227
361
|
key?: string;
|
|
@@ -231,6 +365,8 @@ export interface ClientOptions {
|
|
|
231
365
|
fetch?: typeof fetch;
|
|
232
366
|
/** Extra headers for every request. */
|
|
233
367
|
headers?: Record<string, string>;
|
|
368
|
+
/** Transparency-log store (newLogStore()). Every Resolve answer's signed tree head is checked against it; keep it across runs to catch a split view. Default: a fresh in-memory store. */
|
|
369
|
+
logStore?: LogStore;
|
|
234
370
|
}
|
|
235
371
|
export declare class CrawlCheckError extends Error {
|
|
236
372
|
readonly status: number;
|
|
@@ -252,15 +388,48 @@ export declare class CrawlCheck {
|
|
|
252
388
|
private readonly key?;
|
|
253
389
|
private readonly f;
|
|
254
390
|
private readonly extra;
|
|
391
|
+
/** The transparency-log store this client checks every signed tree head against. Persist it (JSON) between runs. */
|
|
392
|
+
readonly log: LogStore;
|
|
255
393
|
constructor(opts?: ClientOptions);
|
|
256
394
|
/** Any documented GET, typed from the OpenAPI document: client.get("/api/explain", { domain: "example.com" }). */
|
|
257
395
|
get<P extends keyof paths>(path: P, query?: QueryOf<P, "get">): Promise<ResponseOf<P, "get">>;
|
|
258
396
|
/** Any documented POST, typed from the OpenAPI document. */
|
|
259
397
|
post<P extends keyof paths>(path: P, body: BodyOf<P> | JsonObject, query?: QueryOf<P, "post">): Promise<ResponseOf<P, "post">>;
|
|
260
398
|
/** The signed ResolveV1 answer: crawl policy, delivery, machine files, capabilities, entity, findings, freshness and a decision per action. */
|
|
261
|
-
resolve(domain: string): Promise<
|
|
399
|
+
resolve(domain: string): Promise<ResponseOf<"/api/v1/resolve", "get">>;
|
|
400
|
+
private kidCache;
|
|
401
|
+
/** The key ids CrawlCheck publishes (/.well-known/http-message-signatures-directory), cached for an hour. */
|
|
402
|
+
publishedKids(): Promise<string[]>;
|
|
403
|
+
/**
|
|
404
|
+
* The verdict for a domain, preferring a valid staple the site sent (opts.staple, or opts.response's CrawlCheck-Staple
|
|
405
|
+
* header): no request to CrawlCheck is made when the staple verifies. An expired, foreign or invalid staple is ignored
|
|
406
|
+
* and the signed answer is fetched instead; source says which was used and staple says why.
|
|
407
|
+
*/
|
|
408
|
+
resolveFor(domain: string, opts?: {
|
|
409
|
+
staple?: string | null;
|
|
410
|
+
response?: unknown;
|
|
411
|
+
}): Promise<{
|
|
412
|
+
source: "staple" | "network";
|
|
413
|
+
decisions: StapleDecisions;
|
|
414
|
+
staple: StapleCheck | null;
|
|
415
|
+
answer?: unknown;
|
|
416
|
+
}>;
|
|
262
417
|
/** Up to 100 domains in one call; each answer is a full signed ResolveV1 document. */
|
|
263
|
-
resolveBatch(domains: string[]): Promise<
|
|
418
|
+
resolveBatch(domains: string[]): Promise<any>;
|
|
419
|
+
/** Split-view check: the current head, GitHub's witness co-signature on the last witnessed head, and the chain from it down to every head this client holds. Reports the heads it holds to /api/v1/log/gossip unless report is false. */
|
|
420
|
+
logCheck(opts?: {
|
|
421
|
+
report?: boolean;
|
|
422
|
+
maxWalk?: number;
|
|
423
|
+
maxUnwitnessedMinutes?: number;
|
|
424
|
+
}): Promise<LogCheckResult>;
|
|
425
|
+
/** Check heads another client sent (logHeads on their side) against this client's store. */
|
|
426
|
+
logGossip(sths: unknown[]): Promise<{
|
|
427
|
+
forks: LogFork[];
|
|
428
|
+
results: (LogObservation & {
|
|
429
|
+
batch: number;
|
|
430
|
+
sha256: string;
|
|
431
|
+
})[];
|
|
432
|
+
}>;
|
|
264
433
|
/** The policy engine: a signed decision receipt (allow | warn | require_confirmation | block | unsupported). A policy can only make the decision stricter. */
|
|
265
434
|
preflight(domain: string, action?: Action, opts?: PreflightOptions): Promise<DecisionReceipt>;
|
|
266
435
|
/** Verify a signed document here. With onlineKeys (default), its signing key must also be in the published directory. */
|
|
@@ -271,6 +440,7 @@ export declare class CrawlCheck {
|
|
|
271
440
|
receipt: DecisionReceipt;
|
|
272
441
|
verification: DocumentVerification & Trust;
|
|
273
442
|
header: Record<string, string> | null;
|
|
443
|
+
split_view?: LogFork[];
|
|
274
444
|
}>;
|
|
275
445
|
/** Look a domain up without sending it: only the first prefixLen hex characters of sha256(domain) leave this process. found false with ready true means no public measurement. */
|
|
276
446
|
/** Make a signed lockfile for an MCP server (CrawlCheck reads tools/list; no tool is called). Save it; check with checkLock before every connect. */
|
package/dist/index.js
CHANGED
|
@@ -1,4 +1,6 @@
|
|
|
1
1
|
import * as V from "./verify.js";
|
|
2
|
+
import * as L from "./log.js";
|
|
3
|
+
import * as S from "./staple.js";
|
|
2
4
|
function withTrust(v, kid, published) {
|
|
3
5
|
let checks = v.checks;
|
|
4
6
|
let issuer = null;
|
|
@@ -61,6 +63,26 @@ export async function checkLock(lock, tools) {
|
|
|
61
63
|
const same = lock?.tools_sha256 === now.tools_sha256 && !added.length && !removed.length && !modified.length;
|
|
62
64
|
return { verdict: same ? "unchanged" : "changed", decision: same ? "connect" : "block", approved_tools_sha256: lock?.tools_sha256 ?? "", current_tools_sha256: now.tools_sha256, added, removed, modified };
|
|
63
65
|
}
|
|
66
|
+
export const LOG_ID = L.LOG_ID;
|
|
67
|
+
/** A new, empty transparency-log store (plain JSON: persist it between runs). */
|
|
68
|
+
export const newLogStore = L.newLogStore;
|
|
69
|
+
/** Check one signed tree head against the store and keep it. A fork carries both contradicting signed heads. */
|
|
70
|
+
export const logObserve = L.logObserve;
|
|
71
|
+
/** Check the head inside a Resolve answer (transparency.sth), if it carries one. */
|
|
72
|
+
export const logObserveAnswer = L.logObserveAnswer;
|
|
73
|
+
/** The signed heads a store holds, newest first: what a client sends its peers. */
|
|
74
|
+
export const logHeads = L.logHeads;
|
|
75
|
+
/** Gossip: check another client's heads against this store. */
|
|
76
|
+
export const logGossip = L.logGossip;
|
|
77
|
+
/** Verify the outside witness's co-signature: a GitHub OIDC JWT (RS256) whose audience is crawlcheck-sth:<head sha256>. */
|
|
78
|
+
export const verifyWitnessToken = L.verifyWitnessToken;
|
|
79
|
+
export const GITHUB_OIDC_JWKS = "https://token.actions.githubusercontent.com/.well-known/jwks";
|
|
80
|
+
/** Parse a ccr1 record (CrawlCheck-Staple header value or <domain>._v.crawlcheck.io TXT). */
|
|
81
|
+
export const parseStaple = S.parseStaple;
|
|
82
|
+
/** The CrawlCheck-Staple header of a response (fetch Response, Headers or a plain header map), or null. */
|
|
83
|
+
export const stapleFromResponse = S.stapleFromResponse;
|
|
84
|
+
/** Verify a staple offline: published key, signature, the record names opts.host (or a parent), fresh_until in the future. Expired or foreign staples are never valid. */
|
|
85
|
+
export const stapleCheck = S.stapleCheck;
|
|
64
86
|
export class CrawlCheckError extends Error {
|
|
65
87
|
status;
|
|
66
88
|
body;
|
|
@@ -81,10 +103,13 @@ export class CrawlCheck {
|
|
|
81
103
|
key;
|
|
82
104
|
f;
|
|
83
105
|
extra;
|
|
106
|
+
/** The transparency-log store this client checks every signed tree head against. Persist it (JSON) between runs. */
|
|
107
|
+
log;
|
|
84
108
|
constructor(opts = {}) {
|
|
85
109
|
this.base = (opts.base ?? "https://crawlcheck.io").replace(/\/+$/, "");
|
|
86
110
|
this.key = opts.key;
|
|
87
111
|
this.extra = opts.headers ?? {};
|
|
112
|
+
this.log = opts.logStore ?? newLogStore();
|
|
88
113
|
const f = opts.fetch ?? globalThis.fetch;
|
|
89
114
|
if (typeof f !== "function")
|
|
90
115
|
throw new Error("no fetch available: pass { fetch } (Node 18+ and every browser have one)");
|
|
@@ -100,9 +125,62 @@ export class CrawlCheck {
|
|
|
100
125
|
}
|
|
101
126
|
// ── read before acting ──
|
|
102
127
|
/** The signed ResolveV1 answer: crawl policy, delivery, machine files, capabilities, entity, findings, freshness and a decision per action. */
|
|
103
|
-
resolve(domain) {
|
|
128
|
+
async resolve(domain) {
|
|
129
|
+
const a = await this.get("/api/v1/resolve", { domain });
|
|
130
|
+
await logObserveAnswer(this.log, a).catch(() => null); // a head that contradicts one this client already holds is recorded in this.log.forks
|
|
131
|
+
return a;
|
|
132
|
+
}
|
|
133
|
+
kidCache = null;
|
|
134
|
+
/** The key ids CrawlCheck publishes (/.well-known/http-message-signatures-directory), cached for an hour. */
|
|
135
|
+
async publishedKids() {
|
|
136
|
+
if (this.kidCache && Date.now() - this.kidCache.at < 3600e3)
|
|
137
|
+
return this.kidCache.kids;
|
|
138
|
+
const r = await this.f(this.base + "/.well-known/http-message-signatures-directory", { headers: { accept: "application/json" } });
|
|
139
|
+
const j = await r.json();
|
|
140
|
+
const kids = ((j && j.keys) || []).map((k) => k.kid).filter(Boolean);
|
|
141
|
+
this.kidCache = { at: Date.now(), kids };
|
|
142
|
+
return kids;
|
|
143
|
+
}
|
|
144
|
+
/**
|
|
145
|
+
* The verdict for a domain, preferring a valid staple the site sent (opts.staple, or opts.response's CrawlCheck-Staple
|
|
146
|
+
* header): no request to CrawlCheck is made when the staple verifies. An expired, foreign or invalid staple is ignored
|
|
147
|
+
* and the signed answer is fetched instead; source says which was used and staple says why.
|
|
148
|
+
*/
|
|
149
|
+
async resolveFor(domain, opts = {}) {
|
|
150
|
+
const rec = opts.staple || (opts.response ? stapleFromResponse(opts.response) : null);
|
|
151
|
+
let chk = null;
|
|
152
|
+
if (rec) {
|
|
153
|
+
let kids = [];
|
|
154
|
+
try {
|
|
155
|
+
kids = await this.publishedKids();
|
|
156
|
+
}
|
|
157
|
+
catch {
|
|
158
|
+
kids = [];
|
|
159
|
+
}
|
|
160
|
+
chk = await stapleCheck(rec, { host: normalizeDomain(domain), publishedKids: kids });
|
|
161
|
+
if (chk.valid && chk.decisions)
|
|
162
|
+
return { source: "staple", decisions: chk.decisions, staple: chk };
|
|
163
|
+
}
|
|
164
|
+
const a = await this.resolve(domain);
|
|
165
|
+
const act = (k) => (a && a.actions && a.actions[k] && a.actions[k].decision) || null;
|
|
166
|
+
return { source: "network", decisions: { read: act("read"), cite: act("cite"), connect: act("connect"), transact: act("transact") }, staple: chk, answer: a };
|
|
167
|
+
}
|
|
104
168
|
/** Up to 100 domains in one call; each answer is a full signed ResolveV1 document. */
|
|
105
|
-
resolveBatch(domains) {
|
|
169
|
+
async resolveBatch(domains) {
|
|
170
|
+
const r = await this.post("/api/v1/resolve", { domains });
|
|
171
|
+
for (const a of (r && r.answers) || [])
|
|
172
|
+
await logObserveAnswer(this.log, a).catch(() => null);
|
|
173
|
+
return r;
|
|
174
|
+
}
|
|
175
|
+
/** Split-view check: the current head, GitHub's witness co-signature on the last witnessed head, and the chain from it down to every head this client holds. Reports the heads it holds to /api/v1/log/gossip unless report is false. */
|
|
176
|
+
async logCheck(opts = {}) {
|
|
177
|
+
const kids = await this.publishedKeyIds();
|
|
178
|
+
return L.logCheck(this.log, (p) => this.req("GET", p), { publishedKids: kids, maxWalk: opts.maxWalk, maxUnwitnessedMinutes: opts.maxUnwitnessedMinutes,
|
|
179
|
+
fetchJwks: async () => { const r = await this.f(GITHUB_OIDC_JWKS); return r.ok ? r.json() : null; },
|
|
180
|
+
post: opts.report === false ? undefined : (p, b) => this.req("POST", p, undefined, b) });
|
|
181
|
+
}
|
|
182
|
+
/** Check heads another client sent (logHeads on their side) against this client's store. */
|
|
183
|
+
logGossip(sths) { return logGossip(this.log, sths); }
|
|
106
184
|
/** The policy engine: a signed decision receipt (allow | warn | require_confirmation | block | unsupported). A policy can only make the decision stricter. */
|
|
107
185
|
async preflight(domain, action = "read", opts = {}) {
|
|
108
186
|
const b = { domain, action };
|
|
@@ -121,6 +199,10 @@ export class CrawlCheck {
|
|
|
121
199
|
/** Preflight, verify the receipt, and decide. Proceeds on allow; warn only with allowWarn; require_confirmation only if confirm() says yes; block and unsupported never. */
|
|
122
200
|
async guard(domain, action = "read", opts = {}) {
|
|
123
201
|
const receipt = await this.preflight(domain, action, opts);
|
|
202
|
+
if (this.log.forks.length) { // the log has shown this client two contradicting histories: no answer from it is trusted for action
|
|
203
|
+
const verification = await this.verifyDocument(receipt, opts.onlineKeys ?? true);
|
|
204
|
+
return { proceed: false, receipt, verification, header: null, split_view: this.log.forks.slice() };
|
|
205
|
+
}
|
|
124
206
|
const verification = await this.verifyDocument(receipt, opts.onlineKeys ?? true);
|
|
125
207
|
if (!verification.accepted && (opts.onlineKeys ?? true))
|
|
126
208
|
return { proceed: false, receipt, verification, header: null };
|
package/dist/log.d.ts
ADDED
|
@@ -0,0 +1,141 @@
|
|
|
1
|
+
/** A new, empty store. It is plain JSON: keep it across runs (a file, a KV value) so later heads are checked against earlier ones. */
|
|
2
|
+
export function newLogStore(): {
|
|
3
|
+
v: number;
|
|
4
|
+
log_id: string;
|
|
5
|
+
heads: {};
|
|
6
|
+
forks: never[];
|
|
7
|
+
witnessed: null;
|
|
8
|
+
};
|
|
9
|
+
/**
|
|
10
|
+
* Check one signed tree head (kind crawlcheck-log-sth) against everything the store already holds, then keep it.
|
|
11
|
+
* Returns { state: "new" | "known" | "fork" | "invalid", why?, fork? }. invalid means the document is not a head
|
|
12
|
+
* CrawlCheck signed (it proves nothing either way); fork means two signed heads that contradict each other.
|
|
13
|
+
* opts.publishedKids: key ids from the published directory; a head signed by another key is invalid.
|
|
14
|
+
*/
|
|
15
|
+
export function logObserve(store: any, sth: any, opts?: {}): Promise<{
|
|
16
|
+
state: string;
|
|
17
|
+
fork: {
|
|
18
|
+
kind: any;
|
|
19
|
+
batch: any;
|
|
20
|
+
why: any;
|
|
21
|
+
a: any;
|
|
22
|
+
b: any;
|
|
23
|
+
detected_at: string;
|
|
24
|
+
proof: string;
|
|
25
|
+
};
|
|
26
|
+
} | {
|
|
27
|
+
state: string;
|
|
28
|
+
why: string;
|
|
29
|
+
} | {
|
|
30
|
+
state: string;
|
|
31
|
+
why?: undefined;
|
|
32
|
+
}>;
|
|
33
|
+
/** Observe the head inside a Resolve answer (transparency.sth), if it has one. */
|
|
34
|
+
export function logObserveAnswer(store: any, answer: any, opts?: {}): Promise<{
|
|
35
|
+
state: string;
|
|
36
|
+
fork: {
|
|
37
|
+
kind: any;
|
|
38
|
+
batch: any;
|
|
39
|
+
why: any;
|
|
40
|
+
a: any;
|
|
41
|
+
b: any;
|
|
42
|
+
detected_at: string;
|
|
43
|
+
proof: string;
|
|
44
|
+
};
|
|
45
|
+
} | {
|
|
46
|
+
state: string;
|
|
47
|
+
why: string;
|
|
48
|
+
} | {
|
|
49
|
+
state: string;
|
|
50
|
+
why?: undefined;
|
|
51
|
+
} | {
|
|
52
|
+
state: string;
|
|
53
|
+
}>;
|
|
54
|
+
/** The signed heads this store holds, newest first: what a client sends its peers. */
|
|
55
|
+
export function logHeads(store: any, limit?: number): any[];
|
|
56
|
+
/** Gossip: check heads another client holds against this store. Returns { forks, results }. */
|
|
57
|
+
export function logGossip(store: any, sths: any, opts?: {}): Promise<{
|
|
58
|
+
forks: any;
|
|
59
|
+
results: (({
|
|
60
|
+
batch: any;
|
|
61
|
+
sha256: any;
|
|
62
|
+
} & {
|
|
63
|
+
state: string;
|
|
64
|
+
fork: {
|
|
65
|
+
kind: any;
|
|
66
|
+
batch: any;
|
|
67
|
+
why: any;
|
|
68
|
+
a: any;
|
|
69
|
+
b: any;
|
|
70
|
+
detected_at: string;
|
|
71
|
+
proof: string;
|
|
72
|
+
};
|
|
73
|
+
}) | ({
|
|
74
|
+
batch: any;
|
|
75
|
+
sha256: any;
|
|
76
|
+
} & {
|
|
77
|
+
state: string;
|
|
78
|
+
why: string;
|
|
79
|
+
}) | ({
|
|
80
|
+
batch: any;
|
|
81
|
+
sha256: any;
|
|
82
|
+
} & {
|
|
83
|
+
state: string;
|
|
84
|
+
why?: undefined;
|
|
85
|
+
}))[];
|
|
86
|
+
}>;
|
|
87
|
+
/**
|
|
88
|
+
* Verify a witness co-signature: the JWT's RS256 signature against GitHub's published keys (jwks: the JSON from
|
|
89
|
+
* https://token.actions.githubusercontent.com/.well-known/jwks), its issuer, repository, workflow and audience
|
|
90
|
+
* crawlcheck-sth:<sthSha256>. exp is not checked: the token is a record of what the witness saw when it ran.
|
|
91
|
+
* Returns { ok: true | false | null, why, claims? }; null when GitHub no longer publishes the key that signed it.
|
|
92
|
+
*/
|
|
93
|
+
export function verifyWitnessToken(token: any, sthSha256: any, jwks: any): Promise<{
|
|
94
|
+
ok: boolean;
|
|
95
|
+
why: string;
|
|
96
|
+
claims?: undefined;
|
|
97
|
+
} | {
|
|
98
|
+
ok: null;
|
|
99
|
+
why: string;
|
|
100
|
+
claims: any;
|
|
101
|
+
} | {
|
|
102
|
+
ok: boolean;
|
|
103
|
+
why: string;
|
|
104
|
+
claims: any;
|
|
105
|
+
}>;
|
|
106
|
+
/**
|
|
107
|
+
* Online check of a store against the log and its outside witness. get(path) must return parsed JSON from
|
|
108
|
+
* https://crawlcheck.io (the SDK clients pass their own). Steps: observe the current head; fetch the last witnessed
|
|
109
|
+
* head and verify GitHub's co-signature; walk the chain from the witnessed head down to the oldest head this store
|
|
110
|
+
* holds (at most opts.maxWalk batches), observing every link, so any head this client was shown that the witness's
|
|
111
|
+
* chain does not contain surfaces as a fork. Heads newer than the witnessed one are listed as unwitnessed.
|
|
112
|
+
*/
|
|
113
|
+
export function logCheck(store: any, get: any, opts?: {}): Promise<{
|
|
114
|
+
consistent: boolean;
|
|
115
|
+
forks: any;
|
|
116
|
+
all_forks: any;
|
|
117
|
+
witnessed: {
|
|
118
|
+
batch: any;
|
|
119
|
+
sha256: any;
|
|
120
|
+
at: any;
|
|
121
|
+
run_id: any;
|
|
122
|
+
token: boolean | null;
|
|
123
|
+
why: string;
|
|
124
|
+
observed: string;
|
|
125
|
+
} | null;
|
|
126
|
+
unwitnessed: {
|
|
127
|
+
batch: number;
|
|
128
|
+
sha256: any;
|
|
129
|
+
created_at: any;
|
|
130
|
+
}[];
|
|
131
|
+
stale_unwitnessed: {
|
|
132
|
+
batch: number;
|
|
133
|
+
sha256: any;
|
|
134
|
+
created_at: any;
|
|
135
|
+
}[];
|
|
136
|
+
summary: string;
|
|
137
|
+
}>;
|
|
138
|
+
export const LOG_ID: "crawlcheck-resolve-log-v1";
|
|
139
|
+
export const GITHUB_OIDC_ISSUER: "https://token.actions.githubusercontent.com";
|
|
140
|
+
export const WITNESS_REPOSITORY: "emmanuelorta/crawlcheck";
|
|
141
|
+
export const WITNESS_WORKFLOW_PREFIX: "emmanuelorta/crawlcheck/.github/workflows/log-witness.yml@refs/heads/";
|
package/dist/log.js
ADDED
|
@@ -0,0 +1,194 @@
|
|
|
1
|
+
// Split-view detection for the CrawlCheck Resolve transparency log.
|
|
2
|
+
// A log that shows one client a different history than another (a "split view") has to sign two different heads for
|
|
3
|
+
// the same batch, or a head that does not chain to the one before. Both are caught here, three ways:
|
|
4
|
+
// 1. every head a client sees (in Resolve answers, from the log, from peers) is kept in a store and checked against
|
|
5
|
+
// its neighbours: same batch with two digests, a broken prev link, or broken size arithmetic is a fork;
|
|
6
|
+
// 2. gossip: clients swap the signed heads they hold (logHeads / logGossip), so a head shown only to one of them
|
|
7
|
+
// collides with the other's store;
|
|
8
|
+
// 3. the outside witness: a GitHub Actions run re-checks the chain and co-signs the newest head with a GitHub-signed
|
|
9
|
+
// OIDC token whose audience is that head's digest. GitHub's key, not CrawlCheck's. A client that walks the chain
|
|
10
|
+
// from the witnessed head back to its own heads detects any head the witness never saw.
|
|
11
|
+
// A fork comes with both signed heads: two documents carrying CrawlCheck's own signature that cannot both be true.
|
|
12
|
+
// Zero dependencies; WebCrypto only.
|
|
13
|
+
import { verifyDataDoc } from "./verify.js";
|
|
14
|
+
export const LOG_ID = "crawlcheck-resolve-log-v1";
|
|
15
|
+
export const GITHUB_OIDC_ISSUER = "https://token.actions.githubusercontent.com";
|
|
16
|
+
export const WITNESS_REPOSITORY = "emmanuelorta/crawlcheck";
|
|
17
|
+
export const WITNESS_WORKFLOW_PREFIX = "emmanuelorta/crawlcheck/.github/workflows/log-witness.yml@refs/heads/";
|
|
18
|
+
const MAX_HEADS = 2000;
|
|
19
|
+
/** A new, empty store. It is plain JSON: keep it across runs (a file, a KV value) so later heads are checked against earlier ones. */
|
|
20
|
+
export function newLogStore() { return { v: 1, log_id: LOG_ID, heads: {}, forks: [], witnessed: null }; }
|
|
21
|
+
function brief(s) { return { batch: s.batch, sha256: s.sha256, first_index: s.first_index, batch_size: s.batch_size, tree_size: s.tree_size, prev_sth_sha256: s.prev_sth_sha256 ?? null, created_at: s.created_at ?? null }; }
|
|
22
|
+
function forkOf(store, kind, batch, a, b, why) {
|
|
23
|
+
const f = { kind, batch, why, a: a, b: b, detected_at: new Date().toISOString(),
|
|
24
|
+
proof: "both documents carry CrawlCheck's own signature and cannot both belong to one append-only log; publish them or send them to " + "https://crawlcheck.io/api/v1/log/gossip" };
|
|
25
|
+
const key = kind + ":" + (a && a.sha256) + ":" + (b && b.sha256);
|
|
26
|
+
if (!store.forks.some((x) => x.kind + ":" + (x.a && x.a.sha256) + ":" + (x.b && x.b.sha256) === key))
|
|
27
|
+
store.forks.push(f);
|
|
28
|
+
return { state: "fork", fork: f };
|
|
29
|
+
}
|
|
30
|
+
/**
|
|
31
|
+
* Check one signed tree head (kind crawlcheck-log-sth) against everything the store already holds, then keep it.
|
|
32
|
+
* Returns { state: "new" | "known" | "fork" | "invalid", why?, fork? }. invalid means the document is not a head
|
|
33
|
+
* CrawlCheck signed (it proves nothing either way); fork means two signed heads that contradict each other.
|
|
34
|
+
* opts.publishedKids: key ids from the published directory; a head signed by another key is invalid.
|
|
35
|
+
*/
|
|
36
|
+
export async function logObserve(store, sth, opts = {}) {
|
|
37
|
+
if (!store || typeof store !== "object" || !store.heads)
|
|
38
|
+
throw new Error("pass a store from newLogStore()");
|
|
39
|
+
if (!sth || sth.kind !== "crawlcheck-log-sth" || !Number.isInteger(sth.batch) || sth.batch < 0)
|
|
40
|
+
return { state: "invalid", why: "not a signed tree head" };
|
|
41
|
+
let v = null;
|
|
42
|
+
try {
|
|
43
|
+
v = await verifyDataDoc(sth, opts.publishedKids ? { publishedKids: opts.publishedKids } : undefined);
|
|
44
|
+
}
|
|
45
|
+
catch (e) {
|
|
46
|
+
v = null;
|
|
47
|
+
}
|
|
48
|
+
if (!v || !v.verified)
|
|
49
|
+
return { state: "invalid", why: "the head does not verify" + (v ? ": " + v.summary : "") };
|
|
50
|
+
if (opts.publishedKids && !opts.publishedKids.includes(sth.signature && sth.signature.kid))
|
|
51
|
+
return { state: "invalid", why: "signed by a key CrawlCheck does not publish" };
|
|
52
|
+
if (sth.log_id !== (store.log_id || LOG_ID))
|
|
53
|
+
return { state: "invalid", why: "a head of another log (" + sth.log_id + ")" };
|
|
54
|
+
// A head CrawlCheck signed whose own numbers do not add up is itself misbehaviour.
|
|
55
|
+
const arith = Number.isInteger(sth.batch_size) && sth.batch_size > 0 && sth.tree_size === sth.first_index + sth.batch_size && (sth.batch > 0 || (sth.first_index === 0 && sth.prev_sth_sha256 == null)) && (sth.batch === 0 || /^[0-9a-f]{64}$/.test(String(sth.prev_sth_sha256)));
|
|
56
|
+
if (!arith)
|
|
57
|
+
return forkOf(store, "inconsistent_head", sth.batch, sth, sth, "batch " + sth.batch + ": a signed head whose sizes or chain link are malformed");
|
|
58
|
+
const k = sth.batch, have = store.heads[k];
|
|
59
|
+
if (have) {
|
|
60
|
+
if (have.sha256 === sth.sha256)
|
|
61
|
+
return { state: "known" };
|
|
62
|
+
return forkOf(store, "two_heads_for_one_batch", k, have.sth, sth, "batch " + k + " has two signed heads: " + have.sha256.slice(0, 16) + "… and " + sth.sha256.slice(0, 16) + "…");
|
|
63
|
+
}
|
|
64
|
+
const prev = store.heads[k - 1], next = store.heads[k + 1];
|
|
65
|
+
if (prev && sth.prev_sth_sha256 !== prev.sha256)
|
|
66
|
+
return forkOf(store, "chain_break", k, prev.sth, sth, "batch " + k + " does not chain to the batch " + (k - 1) + " head this client holds");
|
|
67
|
+
if (prev && sth.first_index !== prev.tree_size)
|
|
68
|
+
return forkOf(store, "size_mismatch", k, prev.sth, sth, "batch " + k + " starts at " + sth.first_index + ", batch " + (k - 1) + " ended at " + prev.tree_size);
|
|
69
|
+
if (next && next.prev_sth_sha256 !== sth.sha256)
|
|
70
|
+
return forkOf(store, "chain_break", k + 1, sth, next.sth, "the batch " + (k + 1) + " head this client holds does not chain to this batch " + k + " head");
|
|
71
|
+
if (next && next.first_index !== sth.tree_size)
|
|
72
|
+
return forkOf(store, "size_mismatch", k + 1, sth, next.sth, "batch " + (k + 1) + " starts at " + next.first_index + ", this head ends at " + sth.tree_size);
|
|
73
|
+
store.heads[k] = Object.assign(brief(sth), { sth });
|
|
74
|
+
const ks = Object.keys(store.heads).map(Number).sort((a, b) => a - b);
|
|
75
|
+
while (ks.length > (opts.maxHeads || MAX_HEADS))
|
|
76
|
+
delete store.heads[ks.shift()];
|
|
77
|
+
return { state: "new" };
|
|
78
|
+
}
|
|
79
|
+
/** Observe the head inside a Resolve answer (transparency.sth), if it has one. */
|
|
80
|
+
export async function logObserveAnswer(store, answer, opts = {}) {
|
|
81
|
+
const t = answer && answer.transparency;
|
|
82
|
+
return t && t.state === "included" && t.sth ? logObserve(store, t.sth, opts) : { state: "none" };
|
|
83
|
+
}
|
|
84
|
+
/** The signed heads this store holds, newest first: what a client sends its peers. */
|
|
85
|
+
export function logHeads(store, limit = 50) {
|
|
86
|
+
return Object.keys(store.heads).map(Number).sort((a, b) => b - a).slice(0, limit).map((k) => store.heads[k].sth);
|
|
87
|
+
}
|
|
88
|
+
/** Gossip: check heads another client holds against this store. Returns { forks, results }. */
|
|
89
|
+
export async function logGossip(store, sths, opts = {}) {
|
|
90
|
+
const results = [], before = store.forks.length;
|
|
91
|
+
for (const s of Array.isArray(sths) ? sths : [])
|
|
92
|
+
results.push(Object.assign({ batch: s && s.batch, sha256: s && s.sha256 }, await logObserve(store, s, opts)));
|
|
93
|
+
return { forks: store.forks.slice(before), results };
|
|
94
|
+
}
|
|
95
|
+
// ── the outside witness: a GitHub Actions OIDC token whose audience is the head's digest ──
|
|
96
|
+
const b64uBytes = (s) => { const t = String(s).replace(/-/g, "+").replace(/_/g, "/"); const b = atob(t + "=".repeat((4 - t.length % 4) % 4)); const o = new Uint8Array(b.length); for (let i = 0; i < b.length; i++)
|
|
97
|
+
o[i] = b.charCodeAt(i); return o; };
|
|
98
|
+
/**
|
|
99
|
+
* Verify a witness co-signature: the JWT's RS256 signature against GitHub's published keys (jwks: the JSON from
|
|
100
|
+
* https://token.actions.githubusercontent.com/.well-known/jwks), its issuer, repository, workflow and audience
|
|
101
|
+
* crawlcheck-sth:<sthSha256>. exp is not checked: the token is a record of what the witness saw when it ran.
|
|
102
|
+
* Returns { ok: true | false | null, why, claims? }; null when GitHub no longer publishes the key that signed it.
|
|
103
|
+
*/
|
|
104
|
+
export async function verifyWitnessToken(token, sthSha256, jwks) {
|
|
105
|
+
const parts = String(token || "").split(".");
|
|
106
|
+
if (parts.length !== 3)
|
|
107
|
+
return { ok: false, why: "not a JWT" };
|
|
108
|
+
let h, c;
|
|
109
|
+
try {
|
|
110
|
+
h = JSON.parse(new TextDecoder().decode(b64uBytes(parts[0])));
|
|
111
|
+
c = JSON.parse(new TextDecoder().decode(b64uBytes(parts[1])));
|
|
112
|
+
}
|
|
113
|
+
catch (e) {
|
|
114
|
+
return { ok: false, why: "unreadable token" };
|
|
115
|
+
}
|
|
116
|
+
if (h.alg !== "RS256")
|
|
117
|
+
return { ok: false, why: "unexpected algorithm " + h.alg };
|
|
118
|
+
const jwk = (jwks && jwks.keys || []).find((k) => k.kid === h.kid);
|
|
119
|
+
if (!jwk)
|
|
120
|
+
return { ok: null, why: "GitHub no longer publishes key " + h.kid + "; the co-signature cannot be checked now", claims: c };
|
|
121
|
+
let sigOk = false;
|
|
122
|
+
try {
|
|
123
|
+
const key = await crypto.subtle.importKey("jwk", { kty: "RSA", n: jwk.n, e: jwk.e, alg: "RS256", ext: true }, { name: "RSASSA-PKCS1-v1_5", hash: "SHA-256" }, false, ["verify"]);
|
|
124
|
+
sigOk = await crypto.subtle.verify("RSASSA-PKCS1-v1_5", key, b64uBytes(parts[2]), new TextEncoder().encode(parts[0] + "." + parts[1]));
|
|
125
|
+
}
|
|
126
|
+
catch (e) {
|
|
127
|
+
return { ok: null, why: "this runtime cannot verify RS256: " + (e && e.message || e), claims: c };
|
|
128
|
+
}
|
|
129
|
+
if (!sigOk)
|
|
130
|
+
return { ok: false, why: "GitHub's signature does NOT verify", claims: c };
|
|
131
|
+
if (c.iss !== GITHUB_OIDC_ISSUER)
|
|
132
|
+
return { ok: false, why: "issuer is " + c.iss, claims: c };
|
|
133
|
+
if (c.repository !== WITNESS_REPOSITORY || String(c.job_workflow_ref || "").indexOf(WITNESS_WORKFLOW_PREFIX) !== 0)
|
|
134
|
+
return { ok: false, why: "issued to " + c.job_workflow_ref + ", not the log-witness workflow", claims: c };
|
|
135
|
+
if (c.runner_environment && c.runner_environment !== "github-hosted")
|
|
136
|
+
return { ok: false, why: "not a GitHub-hosted runner", claims: c };
|
|
137
|
+
const aud = Array.isArray(c.aud) ? c.aud : [c.aud];
|
|
138
|
+
if (!aud.includes("crawlcheck-sth:" + sthSha256))
|
|
139
|
+
return { ok: false, why: "the token's audience does not name this head", claims: c };
|
|
140
|
+
return { ok: true, why: "GitHub signed a token for run " + c.run_id + " of the log-witness workflow, audience this head's digest", claims: c };
|
|
141
|
+
}
|
|
142
|
+
/**
|
|
143
|
+
* Online check of a store against the log and its outside witness. get(path) must return parsed JSON from
|
|
144
|
+
* https://crawlcheck.io (the SDK clients pass their own). Steps: observe the current head; fetch the last witnessed
|
|
145
|
+
* head and verify GitHub's co-signature; walk the chain from the witnessed head down to the oldest head this store
|
|
146
|
+
* holds (at most opts.maxWalk batches), observing every link, so any head this client was shown that the witness's
|
|
147
|
+
* chain does not contain surfaces as a fork. Heads newer than the witnessed one are listed as unwitnessed.
|
|
148
|
+
*/
|
|
149
|
+
export async function logCheck(store, get, opts = {}) {
|
|
150
|
+
const before = store.forks.length, o = { publishedKids: opts.publishedKids };
|
|
151
|
+
const log = await get("/api/v1/log");
|
|
152
|
+
if (log && log.head)
|
|
153
|
+
await logObserve(store, log.head, o);
|
|
154
|
+
let witnessed = null;
|
|
155
|
+
const w = await get("/api/v1/log/witness/latest").catch(() => null);
|
|
156
|
+
if (w && Number.isInteger(w.batch)) {
|
|
157
|
+
const doc = await get("/api/v1/log/sth/" + w.batch);
|
|
158
|
+
const sth = doc && doc.sth, rec = (doc && doc.witnesses || [])[0] || w;
|
|
159
|
+
const jwks = opts.jwks || (opts.fetchJwks ? await opts.fetchJwks() : null);
|
|
160
|
+
const tv = jwks ? await verifyWitnessToken(rec.token, sth && sth.sha256, jwks) : { ok: null, why: "no GitHub key set given" };
|
|
161
|
+
const r = sth ? await logObserve(store, sth, o) : { state: "invalid" };
|
|
162
|
+
witnessed = { batch: w.batch, sha256: sth && sth.sha256, at: rec.at || null, run_id: tv.claims ? tv.claims.run_id : (rec.run && rec.run.run_id) || null, token: tv.ok, why: tv.why, observed: r.state };
|
|
163
|
+
if (tv.ok === false)
|
|
164
|
+
witnessed.warning = "the witness record does not carry a valid GitHub co-signature for this head";
|
|
165
|
+
if (tv.ok === true && r.state !== "invalid")
|
|
166
|
+
store.witnessed = { batch: w.batch, sha256: sth.sha256, at: witnessed.at, run_id: witnessed.run_id };
|
|
167
|
+
const held = Object.keys(store.heads).map(Number).filter((k) => k < w.batch);
|
|
168
|
+
const lo = held.length ? Math.max(Math.min(...held), w.batch - (opts.maxWalk || 200)) : w.batch;
|
|
169
|
+
for (let k = w.batch - 1; k >= lo; k--) {
|
|
170
|
+
const d = await get("/api/v1/log/sth/" + k).catch(() => null);
|
|
171
|
+
if (d && d.sth)
|
|
172
|
+
await logObserve(store, d.sth, o);
|
|
173
|
+
}
|
|
174
|
+
}
|
|
175
|
+
const wb = store.witnessed ? store.witnessed.batch : -1, now = Date.now(), maxAge = (opts.maxUnwitnessedMinutes || 120) * 60000;
|
|
176
|
+
const unwitnessed = Object.keys(store.heads).map(Number).filter((k) => k > wb).sort((a, b) => a - b).map((k) => ({ batch: k, sha256: store.heads[k].sha256, created_at: store.heads[k].created_at }));
|
|
177
|
+
const stale = unwitnessed.filter((u) => u.created_at && now - Date.parse(u.created_at) > maxAge);
|
|
178
|
+
const forks = store.forks.slice(before);
|
|
179
|
+
if (opts.post) { // tell CrawlCheck which heads this client was shown; a fork reported here is recorded publicly at /api/v1/log/gossip
|
|
180
|
+
const sths = logHeads(store, 20);
|
|
181
|
+
for (const f of forks) {
|
|
182
|
+
if (f.a)
|
|
183
|
+
sths.push(f.a);
|
|
184
|
+
if (f.b && f.b !== f.a)
|
|
185
|
+
sths.push(f.b);
|
|
186
|
+
}
|
|
187
|
+
try {
|
|
188
|
+
await opts.post("/api/v1/log/gossip", { sths: sths.slice(0, 50) });
|
|
189
|
+
}
|
|
190
|
+
catch (e) { }
|
|
191
|
+
}
|
|
192
|
+
return { consistent: store.forks.length === 0, forks, all_forks: store.forks.length, witnessed, unwitnessed, stale_unwitnessed: stale,
|
|
193
|
+
summary: store.forks.length ? "SPLIT VIEW: " + store.forks.length + " pair(s) of contradicting signed heads" : stale.length ? "no fork seen, but " + stale.length + " head(s) older than " + (opts.maxUnwitnessedMinutes || 120) + " min are not covered by a witness" : "consistent: every head this client holds is on one chain" + (witnessed && witnessed.token === true ? " up to the head GitHub's witness co-signed (batch " + witnessed.batch + ")" : "") };
|
|
194
|
+
}
|
package/dist/staple.d.ts
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
/** Parse a ccr1 record ("v=ccr1;d=...;s=..."); quoted TXT strings from dig are joined. */
|
|
2
|
+
export function parseStaple(record: any): {
|
|
3
|
+
fields: {};
|
|
4
|
+
body: string;
|
|
5
|
+
sig: string;
|
|
6
|
+
};
|
|
7
|
+
/** The staple a response carries, if any (a fetch Response, a Headers object or a plain header map). */
|
|
8
|
+
export function stapleFromResponse(res: any): any;
|
|
9
|
+
/**
|
|
10
|
+
* Check a staple offline. opts.host: the host the agent is talking to (required for a usable result);
|
|
11
|
+
* opts.publishedKids: the key ids CrawlCheck publishes (from /.well-known/http-message-signatures-directory);
|
|
12
|
+
* opts.now: epoch ms (default Date.now()).
|
|
13
|
+
* Returns { valid, reason, expired, record, decisions } and never throws.
|
|
14
|
+
*/
|
|
15
|
+
export function stapleCheck(record: any, opts?: {}): Promise<{
|
|
16
|
+
valid: boolean;
|
|
17
|
+
reason: null;
|
|
18
|
+
expired: boolean;
|
|
19
|
+
record: null;
|
|
20
|
+
decisions: null;
|
|
21
|
+
}>;
|
package/dist/staple.js
ADDED
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
// Stapled Resolve and Resolve over DNS: the compact ccr1 record a site sends in the CrawlCheck-Staple header (or that
|
|
2
|
+
// <domain>._v.crawlcheck.io TXT carries). Verified offline: key id = RFC 7638 thumbprint of the public key, the key is
|
|
3
|
+
// one CrawlCheck publishes, the Ed25519 signature covers the record, the record names the host (or a parent domain),
|
|
4
|
+
// and fresh_until is in the future. An expired or foreign staple is never used.
|
|
5
|
+
const STAPLE_PREFIX = "crawlcheck-dns-v1\n";
|
|
6
|
+
const te = new TextEncoder();
|
|
7
|
+
function b64uToBytes(s) { s = String(s || "").replace(/-/g, "+").replace(/_/g, "/"); s += "===".slice((s.length + 3) % 4); const b = atob(s), u = new Uint8Array(b.length); for (let i = 0; i < b.length; i++)
|
|
8
|
+
u[i] = b.charCodeAt(i); return u; }
|
|
9
|
+
function bytesToB64u(u) { let s = ""; for (let i = 0; i < u.length; i++)
|
|
10
|
+
s += String.fromCharCode(u[i]); return btoa(s).replace(/\+/g, "-").replace(/\//g, "_").replace(/=+$/, ""); }
|
|
11
|
+
/** Parse a ccr1 record ("v=ccr1;d=...;s=..."); quoted TXT strings from dig are joined. */
|
|
12
|
+
export function parseStaple(record) {
|
|
13
|
+
const t = String(record || "").trim().replace(/"\s*"/g, "").replace(/^"|"$/g, "");
|
|
14
|
+
const i = t.lastIndexOf(";s=");
|
|
15
|
+
const body = i > 0 ? t.slice(0, i) : "", sig = i > 0 ? t.slice(i + 3) : "";
|
|
16
|
+
const f = {};
|
|
17
|
+
body.split(";").forEach((p) => { const j = p.indexOf("="); if (j > 0)
|
|
18
|
+
f[p.slice(0, j)] = p.slice(j + 1); });
|
|
19
|
+
return { fields: f, body, sig };
|
|
20
|
+
}
|
|
21
|
+
/** The staple a response carries, if any (a fetch Response, a Headers object or a plain header map). */
|
|
22
|
+
export function stapleFromResponse(res) {
|
|
23
|
+
const h = res && (res.headers || res);
|
|
24
|
+
if (!h)
|
|
25
|
+
return null;
|
|
26
|
+
if (typeof h.get === "function")
|
|
27
|
+
return h.get("crawlcheck-staple");
|
|
28
|
+
for (const k of Object.keys(h))
|
|
29
|
+
if (k.toLowerCase() === "crawlcheck-staple")
|
|
30
|
+
return h[k];
|
|
31
|
+
return null;
|
|
32
|
+
}
|
|
33
|
+
/**
|
|
34
|
+
* Check a staple offline. opts.host: the host the agent is talking to (required for a usable result);
|
|
35
|
+
* opts.publishedKids: the key ids CrawlCheck publishes (from /.well-known/http-message-signatures-directory);
|
|
36
|
+
* opts.now: epoch ms (default Date.now()).
|
|
37
|
+
* Returns { valid, reason, expired, record, decisions } and never throws.
|
|
38
|
+
*/
|
|
39
|
+
export async function stapleCheck(record, opts = {}) {
|
|
40
|
+
const out = { valid: false, reason: null, expired: false, record: null, decisions: null };
|
|
41
|
+
try {
|
|
42
|
+
const { fields: f, body, sig } = parseStaple(record);
|
|
43
|
+
out.record = f;
|
|
44
|
+
if (f.v !== "ccr1" || !f.d || !f.k || !f.x || !sig) {
|
|
45
|
+
out.reason = "not a ccr1 record";
|
|
46
|
+
return out;
|
|
47
|
+
}
|
|
48
|
+
out.decisions = { read: f.read || null, cite: f.cite || null, connect: f.connect || null, transact: f.transact || null };
|
|
49
|
+
const now = opts.now || Date.now(), fu = Date.parse(f.f || "");
|
|
50
|
+
if (!isFinite(fu) || fu <= now) {
|
|
51
|
+
out.expired = true;
|
|
52
|
+
out.reason = "expired at " + (f.f || "?") + ": ask CrawlCheck instead";
|
|
53
|
+
return out;
|
|
54
|
+
}
|
|
55
|
+
const host = String(opts.host || "").toLowerCase().replace(/^www\./, "").replace(/\.$/, "");
|
|
56
|
+
const d = String(f.d).toLowerCase();
|
|
57
|
+
if (!host) {
|
|
58
|
+
out.reason = "no host given: a staple is only valid for the host it was served by";
|
|
59
|
+
return out;
|
|
60
|
+
}
|
|
61
|
+
if (host !== d && !host.endsWith("." + d)) {
|
|
62
|
+
out.reason = "the staple names " + d + ", not " + host;
|
|
63
|
+
return out;
|
|
64
|
+
}
|
|
65
|
+
const thumb = bytesToB64u(new Uint8Array(await crypto.subtle.digest("SHA-256", te.encode(JSON.stringify({ crv: "Ed25519", kty: "OKP", x: f.x })))));
|
|
66
|
+
if (thumb !== f.k) {
|
|
67
|
+
out.reason = "key id does not match the public key";
|
|
68
|
+
return out;
|
|
69
|
+
}
|
|
70
|
+
if (!Array.isArray(opts.publishedKids) || opts.publishedKids.indexOf(f.k) < 0) {
|
|
71
|
+
out.reason = Array.isArray(opts.publishedKids) ? "key " + f.k + " is not one CrawlCheck publishes" : "no published key list given (opts.publishedKids)";
|
|
72
|
+
return out;
|
|
73
|
+
}
|
|
74
|
+
const pk = await crypto.subtle.importKey("jwk", { kty: "OKP", crv: "Ed25519", x: f.x }, { name: "Ed25519" }, false, ["verify"]);
|
|
75
|
+
const ok = await crypto.subtle.verify({ name: "Ed25519" }, pk, b64uToBytes(sig), te.encode(STAPLE_PREFIX + body));
|
|
76
|
+
if (!ok) {
|
|
77
|
+
out.reason = "signature does not verify";
|
|
78
|
+
return out;
|
|
79
|
+
}
|
|
80
|
+
out.valid = true;
|
|
81
|
+
out.reason = "valid until " + f.f;
|
|
82
|
+
return out;
|
|
83
|
+
}
|
|
84
|
+
catch (e) {
|
|
85
|
+
out.reason = "could not check: " + (e && e.message || e);
|
|
86
|
+
return out;
|
|
87
|
+
}
|
|
88
|
+
}
|
package/dist/verify.d.ts
CHANGED
|
@@ -9,6 +9,7 @@ export function dspVerdict(m: any): {
|
|
|
9
9
|
effect: string;
|
|
10
10
|
rule: string;
|
|
11
11
|
};
|
|
12
|
+
export function rfcRootV(leafHex: any, index: any, size: any, path: any): Promise<string | null>;
|
|
12
13
|
export function trustOf(res: any, kid: any, publishedKids: any): any;
|
|
13
14
|
export function publishedKeyIds(base: any): Promise<string[]>;
|
|
14
15
|
export function verifyBundle(b: any, opts: any): Promise<any>;
|
package/dist/verify.js
CHANGED
|
@@ -487,6 +487,38 @@ async function verifyResolution0(x) {
|
|
|
487
487
|
resolution: { id: x.resolution_id, dispute: x.dispute_id, domain: x.subject && x.subject.domain, code: x.subject && x.subject.finding && x.subject.finding.code, verdict: said.verdict, effect: said.effect, observers: obs.map((o) => o.observer_id + "@" + (o.network_class || "?")) }, checks,
|
|
488
488
|
note: "Every check used only the resolution itself. To confirm the key is CrawlCheck's, compare its id with " + ((x.issuer && x.issuer.key && x.issuer.key.published_in) || "the published key directory") + "; to check a reading, open the record or the signed observation it links." };
|
|
489
489
|
}
|
|
490
|
+
// tl1: the Resolve log's leaf content and RFC 9162 inclusion check (the same rules the log publishes).
|
|
491
|
+
function tlContentV(doc) {
|
|
492
|
+
const a = doc && doc.actions || {}, acts = {};
|
|
493
|
+
Object.keys(a).sort().forEach((k) => { const x = a[k] || {}; acts[k] = { decision: x.decision || null, doubts: (Array.isArray(x.doubts) ? x.doubts : []).slice().sort() }; });
|
|
494
|
+
return { log: "crawlcheck-resolve-log-v1", domain: doc && doc.domain || null, status: doc && doc.status || null, actions: acts,
|
|
495
|
+
report: doc && doc.evidence && doc.evidence.report || null, observed_at: doc && doc.freshness && doc.freshness.observed_at || null,
|
|
496
|
+
incident: doc && doc.incident && doc.incident.since || null, impersonation: doc && doc.impersonation && doc.impersonation.verdict === "suspected" ? "suspected" : null,
|
|
497
|
+
high_risk_pending: doc && doc.high_risk_fields && Array.isArray(doc.high_risk_fields.pending) ? doc.high_risk_fields.pending.map((p) => p.field).sort() : [] };
|
|
498
|
+
}
|
|
499
|
+
export async function rfcRootV(leafHex, index, size, path) {
|
|
500
|
+
if (!(size > 0) || !(index >= 0) || index >= size)
|
|
501
|
+
return null;
|
|
502
|
+
let fn = index, sn = size - 1, r = unhex(leafHex);
|
|
503
|
+
for (const p of path) {
|
|
504
|
+
if (sn === 0)
|
|
505
|
+
return null;
|
|
506
|
+
if ((fn & 1) === 1 || fn === sn) {
|
|
507
|
+
r = await sha256(cat(cat(new Uint8Array([1]), unhex(p)), r));
|
|
508
|
+
if ((fn & 1) === 0) {
|
|
509
|
+
while ((fn & 1) === 0 && fn !== 0) {
|
|
510
|
+
fn >>= 1;
|
|
511
|
+
sn >>= 1;
|
|
512
|
+
}
|
|
513
|
+
}
|
|
514
|
+
}
|
|
515
|
+
else
|
|
516
|
+
r = await sha256(cat(cat(new Uint8Array([1]), r), unhex(p)));
|
|
517
|
+
fn >>= 1;
|
|
518
|
+
sn >>= 1;
|
|
519
|
+
}
|
|
520
|
+
return sn === 0 ? hex(r) : null;
|
|
521
|
+
}
|
|
490
522
|
// ── data inventories, export parts and deletion records ─────────────────────────────────────────────────────────
|
|
491
523
|
// Signed over sha256(canonical JSON of every field except sha256 and signature) with the prefix below. A deletion
|
|
492
524
|
// record carries counts per family and the digest of the sorted deleted-key list, never the data.
|
|
@@ -496,7 +528,7 @@ async function verifyDataDoc0(x) {
|
|
|
496
528
|
throw new Error("this runtime has no WebCrypto (crypto.subtle)");
|
|
497
529
|
const checks = [];
|
|
498
530
|
const C = (id, ok, why) => checks.push({ id, ok, why });
|
|
499
|
-
if (!x || !/^crawlcheck-(data-inventory|data-export-part|deletion-record|resolve|decision|snapshot|visitor|conduct|traffic-index|anchor|anchor-check|content-clock|origin|claims|read-equivalence|rights|rights-feed|mcp-scan|mcp-lock|mcp-lock-check)$/.test(String(x.kind)) || !x.signature)
|
|
531
|
+
if (!x || !/^crawlcheck-(data-inventory|data-export-part|deletion-record|resolve|decision|snapshot|visitor|conduct|traffic-index|anchor|anchor-check|content-clock|origin|claims|read-equivalence|rights|rights-feed|mcp-scan|mcp-lock|mcp-lock-check|mcp-shadow|mcp-auth|impersonation|task-canaries|incident|revocations|high-risk|log-sth)$/.test(String(x.kind)) || !x.signature)
|
|
500
532
|
throw new Error("not a CrawlCheck data inventory, export part, deletion record, resolve answer, decision receipt, snapshot manifest, visitor verdict, conduct record, traffic index, citation anchor, content clock, origin decision, claim ledger, read-equivalence certificate, rights answer, rights-feed manifest, MCP tool scan, MCP lockfile or lockfile check");
|
|
501
533
|
const body = {};
|
|
502
534
|
for (const k of Object.keys(x))
|
|
@@ -545,6 +577,25 @@ async function verifyDataDoc0(x) {
|
|
|
545
577
|
const ps = Array.isArray(x.parts) ? x.parts : [];
|
|
546
578
|
C("parts", ps.length > 0 && ps.every((p) => /^part-\d{4}\.ndjson\.gz$/.test(String(p.path)) && /^[0-9a-f]{64}$/.test(String(p.sha256)) && p.records > 0) && ps.reduce((a, p) => a + p.records, 0) === x.records, ps.reduce((a, p) => a + (p.records || 0), 0) === x.records ? ps.length + " parts, " + x.records + " records; check each downloaded part with sha256sum against its sha256, then each line with this tool" : "the parts' record counts do not add up to records (" + x.records + ")");
|
|
547
579
|
}
|
|
580
|
+
if (x.kind === "crawlcheck-resolve" && x.transparency) { // tl1: the answer's place in the Resolve transparency log
|
|
581
|
+
const t = x.transparency, leaf = hex(await sha256(cat(new Uint8Array([0]), enc.encode(canon(tlContentV(x))))));
|
|
582
|
+
C("log_leaf", leaf === t.leaf_hash, leaf === t.leaf_hash ? "the answer's decision content hashes to its log leaf " + leaf.slice(0, 16) + "…" : "the decision content hashes to " + leaf + ", the answer names " + t.leaf_hash);
|
|
583
|
+
if (t.state === "included" && t.sth) {
|
|
584
|
+
const root = await rfcRootV(leaf, t.index, t.batch_size, t.path || []);
|
|
585
|
+
C("log_inclusion", root !== null && root === t.sth.batch_root, root === t.sth.batch_root ? "the leaf folds along its " + (t.path || []).length + "-step path to batch " + t.batch + "'s root (RFC 9162)" : "the inclusion path does NOT lead to the signed batch root");
|
|
586
|
+
let sv = null;
|
|
587
|
+
try {
|
|
588
|
+
sv = await verifyDataDoc0(t.sth);
|
|
589
|
+
}
|
|
590
|
+
catch (e) {
|
|
591
|
+
sv = null;
|
|
592
|
+
}
|
|
593
|
+
const same = !x.signature || !t.sth.signature || t.sth.signature.kid === x.signature.kid;
|
|
594
|
+
C("log_head_signature", !!(sv && sv.verified) && t.sth.kind === "crawlcheck-log-sth" && t.sth.batch === t.batch && same, sv && sv.verified ? "batch " + t.batch + "'s signed tree head verifies (tree size " + t.sth.tree_size + ", chained to " + String(t.sth.prev_sth_sha256 || "genesis").slice(0, 16) + "…)" : "the signed tree head does NOT verify");
|
|
595
|
+
}
|
|
596
|
+
else
|
|
597
|
+
C("log_inclusion", null, "pending: the leaf is queued and merged within " + (t.merged_within_minutes || 30) + " minutes; fetch the proof at " + (t.proof || "the log"));
|
|
598
|
+
}
|
|
548
599
|
if (x.kind === "crawlcheck-deletion-record")
|
|
549
600
|
C("counts", (x.deleted || []).reduce((a, f) => a + (f.keys || 0), 0) === x.deleted_total, "the per-family counts add up to deleted_total (" + x.deleted_total + ")");
|
|
550
601
|
const ran = checks.filter((c) => c.ok !== null), failed = checks.filter((c) => c.ok === false);
|
|
@@ -610,7 +661,7 @@ if (isMain) {
|
|
|
610
661
|
const isRc = doc && (doc.kind === "crawlcheck-remediation-receipt" || (doc.receipt && doc.receipt.kind === "crawlcheck-remediation-receipt"));
|
|
611
662
|
const isEx = doc && doc.kind === "crawlcheck-evidence-export";
|
|
612
663
|
const isDr = doc && doc.kind === "crawlcheck-dispute-resolution";
|
|
613
|
-
const isDt = doc && /^crawlcheck-(data-inventory|data-export-part|deletion-record|resolve|decision|snapshot|visitor|conduct|traffic-index|anchor|anchor-check|content-clock|origin|claims|read-equivalence|rights|rights-feed|mcp-scan|mcp-lock|mcp-lock-check)$/.test(String(doc.kind));
|
|
664
|
+
const isDt = doc && /^crawlcheck-(data-inventory|data-export-part|deletion-record|resolve|decision|snapshot|visitor|conduct|traffic-index|anchor|anchor-check|content-clock|origin|claims|read-equivalence|rights|rights-feed|mcp-scan|mcp-lock|mcp-lock-check|mcp-shadow|mcp-auth|impersonation|task-canaries|incident|revocations|high-risk|log-sth)$/.test(String(doc.kind));
|
|
614
665
|
let kids;
|
|
615
666
|
if (online) {
|
|
616
667
|
try {
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@crawlcheck/sdk",
|
|
3
|
-
"version": "1.0.
|
|
3
|
+
"version": "1.0.7",
|
|
4
4
|
"description": "Typed client for the CrawlCheck API (generated from its OpenAPI document) and the zero-dependency offline verifier for evidence bundles and remediation receipts.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "./dist/index.js",
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"kid":"wlMUoXULLLsGJfT7xs0dC4rlNB6xckeP7Of-hL3fS8g","A":[{"kind":"crawlcheck-log-sth","v":1,"log_id":"crawlcheck-resolve-log-v1","batch":0,"first_index":0,"batch_size":3,"tree_size":3,"batch_root":"f5c7cf1ce5c25073318b00bc6c0dc78dee92e55a633f7c7bc5fc71e5d1e6f08c","prev_sth_sha256":null,"created_at":"2026-10-07T12:43:36.031Z","rules":{"leaf":"SHA-256(0x00 || canonical JSON of the answer's decision content)","tree":"RFC 9162 Merkle tree hash over this batch's leaf hashes, sorted","chain":"prev_sth_sha256 is the sha256 of the previous batch's signed tree head; batch numbers are consecutive and tree_size grows by batch_size"},"sha256":"7b92d6860b6e0b95e8ece2980ba7153a3c437e9b375f500289b1346bd639f744","signature":{"v":1,"alg":"Ed25519","kid":"wlMUoXULLLsGJfT7xs0dC4rlNB6xckeP7Of-hL3fS8g","over":"sha256","message":"crawlcheck-data-v1\n7b92d6860b6e0b95e8ece2980ba7153a3c437e9b375f500289b1346bd639f744","sig":"sFrMZhW3RhQfqKgkTVhzjrcvlxTikEuH9Hj8HxFMyD3P4aa53ix0aQoLmykCTBBmnDf1zKEfxCsUe27GnQSqDA","signed_at":"2026-10-07T12:43:36.031Z","key":{"kid":"wlMUoXULLLsGJfT7xs0dC4rlNB6xckeP7Of-hL3fS8g","jwk":{"kty":"OKP","crv":"Ed25519","x":"Ei2Hob8P3o4hhakpwltAFAesqr9N23RnBndK-h5Wkdc"},"published_in":"https://crawlcheck.io/.well-known/http-message-signatures-directory"},"canonical":"sha256 over canonical JSON of every field except sha256 and signature"}},{"kind":"crawlcheck-log-sth","v":1,"log_id":"crawlcheck-resolve-log-v1","batch":1,"first_index":3,"batch_size":1,"tree_size":4,"batch_root":"0ca77e88d0074680c0ed45ab5ef88ab99a6f4f23533e58cbb8fad46625c3e98a","prev_sth_sha256":"7b92d6860b6e0b95e8ece2980ba7153a3c437e9b375f500289b1346bd639f744","created_at":"2026-10-07T12:43:36.034Z","rules":{"leaf":"SHA-256(0x00 || canonical JSON of the answer's decision content)","tree":"RFC 9162 Merkle tree hash over this batch's leaf hashes, sorted","chain":"prev_sth_sha256 is the sha256 of the previous batch's signed tree head; batch numbers are consecutive and tree_size grows by batch_size"},"sha256":"865b63bd6fba326a5dcdfa5c58d46817a6a163209ba7a7357e091f522af8a4b7","signature":{"v":1,"alg":"Ed25519","kid":"wlMUoXULLLsGJfT7xs0dC4rlNB6xckeP7Of-hL3fS8g","over":"sha256","message":"crawlcheck-data-v1\n865b63bd6fba326a5dcdfa5c58d46817a6a163209ba7a7357e091f522af8a4b7","sig":"oB5mXAvuYKReLJLosI06hxajPW4L8sbnCnJTtdRlFuOi2XyuOm13V60BfNI27uHUL7vJqV6dPwkf2PBVSWMvAQ","signed_at":"2026-10-07T12:43:36.034Z","key":{"kid":"wlMUoXULLLsGJfT7xs0dC4rlNB6xckeP7Of-hL3fS8g","jwk":{"kty":"OKP","crv":"Ed25519","x":"Ei2Hob8P3o4hhakpwltAFAesqr9N23RnBndK-h5Wkdc"},"published_in":"https://crawlcheck.io/.well-known/http-message-signatures-directory"},"canonical":"sha256 over canonical JSON of every field except sha256 and signature"}},{"kind":"crawlcheck-log-sth","v":1,"log_id":"crawlcheck-resolve-log-v1","batch":2,"first_index":4,"batch_size":1,"tree_size":5,"batch_root":"18af4f71e5ff99faf5e0cbe20dfd6f226a26318a95785017257376639535f802","prev_sth_sha256":"865b63bd6fba326a5dcdfa5c58d46817a6a163209ba7a7357e091f522af8a4b7","created_at":"2026-10-07T12:43:36.036Z","rules":{"leaf":"SHA-256(0x00 || canonical JSON of the answer's decision content)","tree":"RFC 9162 Merkle tree hash over this batch's leaf hashes, sorted","chain":"prev_sth_sha256 is the sha256 of the previous batch's signed tree head; batch numbers are consecutive and tree_size grows by batch_size"},"sha256":"091d4d852dd45ae637d93368c0bf1602465cd50d0f4a81a225cb0b6f65833d6f","signature":{"v":1,"alg":"Ed25519","kid":"wlMUoXULLLsGJfT7xs0dC4rlNB6xckeP7Of-hL3fS8g","over":"sha256","message":"crawlcheck-data-v1\n091d4d852dd45ae637d93368c0bf1602465cd50d0f4a81a225cb0b6f65833d6f","sig":"gfifnwojp5ep9a6DpqWPlFXuNd1h1jQ1-ZvS_FsRKIl3m6U6cyjZfuDcDGx29Ej4I_GpH8GE75QzjwxZAXhrCQ","signed_at":"2026-10-07T12:43:36.037Z","key":{"kid":"wlMUoXULLLsGJfT7xs0dC4rlNB6xckeP7Of-hL3fS8g","jwk":{"kty":"OKP","crv":"Ed25519","x":"Ei2Hob8P3o4hhakpwltAFAesqr9N23RnBndK-h5Wkdc"},"published_in":"https://crawlcheck.io/.well-known/http-message-signatures-directory"},"canonical":"sha256 over canonical JSON of every field except sha256 and signature"}}],"B":[{"kind":"crawlcheck-log-sth","v":1,"log_id":"crawlcheck-resolve-log-v1","batch":0,"first_index":0,"batch_size":3,"tree_size":3,"batch_root":"f5c7cf1ce5c25073318b00bc6c0dc78dee92e55a633f7c7bc5fc71e5d1e6f08c","prev_sth_sha256":null,"created_at":"2026-10-07T12:43:36.031Z","rules":{"leaf":"SHA-256(0x00 || canonical JSON of the answer's decision content)","tree":"RFC 9162 Merkle tree hash over this batch's leaf hashes, sorted","chain":"prev_sth_sha256 is the sha256 of the previous batch's signed tree head; batch numbers are consecutive and tree_size grows by batch_size"},"sha256":"7b92d6860b6e0b95e8ece2980ba7153a3c437e9b375f500289b1346bd639f744","signature":{"v":1,"alg":"Ed25519","kid":"wlMUoXULLLsGJfT7xs0dC4rlNB6xckeP7Of-hL3fS8g","over":"sha256","message":"crawlcheck-data-v1\n7b92d6860b6e0b95e8ece2980ba7153a3c437e9b375f500289b1346bd639f744","sig":"sFrMZhW3RhQfqKgkTVhzjrcvlxTikEuH9Hj8HxFMyD3P4aa53ix0aQoLmykCTBBmnDf1zKEfxCsUe27GnQSqDA","signed_at":"2026-10-07T12:43:36.031Z","key":{"kid":"wlMUoXULLLsGJfT7xs0dC4rlNB6xckeP7Of-hL3fS8g","jwk":{"kty":"OKP","crv":"Ed25519","x":"Ei2Hob8P3o4hhakpwltAFAesqr9N23RnBndK-h5Wkdc"},"published_in":"https://crawlcheck.io/.well-known/http-message-signatures-directory"},"canonical":"sha256 over canonical JSON of every field except sha256 and signature"}},{"kind":"crawlcheck-log-sth","v":1,"log_id":"crawlcheck-resolve-log-v1","batch":1,"first_index":3,"batch_size":1,"tree_size":4,"batch_root":"1421832f5ae223370e493d079607cf891dffef5b1a21f73834dfdd600a5305b5","prev_sth_sha256":"7b92d6860b6e0b95e8ece2980ba7153a3c437e9b375f500289b1346bd639f744","created_at":"2026-10-07T12:43:36.039Z","rules":{"leaf":"SHA-256(0x00 || canonical JSON of the answer's decision content)","tree":"RFC 9162 Merkle tree hash over this batch's leaf hashes, sorted","chain":"prev_sth_sha256 is the sha256 of the previous batch's signed tree head; batch numbers are consecutive and tree_size grows by batch_size"},"sha256":"2b816b346f43a05913a1a6426c1e2d5a59e8d2a6bb03c520a44936035f0bf75f","signature":{"v":1,"alg":"Ed25519","kid":"wlMUoXULLLsGJfT7xs0dC4rlNB6xckeP7Of-hL3fS8g","over":"sha256","message":"crawlcheck-data-v1\n2b816b346f43a05913a1a6426c1e2d5a59e8d2a6bb03c520a44936035f0bf75f","sig":"x_wDhe08acY2zWMcztpnCx3jguE7Vtap7_8jel8mTIPixsh1VEIYcqsl971sdQSImi1ypU9OFyzy3Hf-o9fbCQ","signed_at":"2026-10-07T12:43:36.039Z","key":{"kid":"wlMUoXULLLsGJfT7xs0dC4rlNB6xckeP7Of-hL3fS8g","jwk":{"kty":"OKP","crv":"Ed25519","x":"Ei2Hob8P3o4hhakpwltAFAesqr9N23RnBndK-h5Wkdc"},"published_in":"https://crawlcheck.io/.well-known/http-message-signatures-directory"},"canonical":"sha256 over canonical JSON of every field except sha256 and signature"}},{"kind":"crawlcheck-log-sth","v":1,"log_id":"crawlcheck-resolve-log-v1","batch":2,"first_index":4,"batch_size":1,"tree_size":5,"batch_root":"05b87125340036bc8c40639ee045cf93e7614a2f70cb6738775bbd08b6abbef1","prev_sth_sha256":"2b816b346f43a05913a1a6426c1e2d5a59e8d2a6bb03c520a44936035f0bf75f","created_at":"2026-10-07T12:43:36.041Z","rules":{"leaf":"SHA-256(0x00 || canonical JSON of the answer's decision content)","tree":"RFC 9162 Merkle tree hash over this batch's leaf hashes, sorted","chain":"prev_sth_sha256 is the sha256 of the previous batch's signed tree head; batch numbers are consecutive and tree_size grows by batch_size"},"sha256":"316bf3e517d3d02615bd6bc0d49699324ef63951b710c72f42b14ca4e63324d4","signature":{"v":1,"alg":"Ed25519","kid":"wlMUoXULLLsGJfT7xs0dC4rlNB6xckeP7Of-hL3fS8g","over":"sha256","message":"crawlcheck-data-v1\n316bf3e517d3d02615bd6bc0d49699324ef63951b710c72f42b14ca4e63324d4","sig":"9fmnQ9AikQJQluM6BICr9lnhvlfUCiqJMPv7LSdX62Yn2eMMfiIeLV4dkCn-Cul2k_teq0QV3ZjySgOA-vObDQ","signed_at":"2026-10-07T12:43:36.041Z","key":{"kid":"wlMUoXULLLsGJfT7xs0dC4rlNB6xckeP7Of-hL3fS8g","jwk":{"kty":"OKP","crv":"Ed25519","x":"Ei2Hob8P3o4hhakpwltAFAesqr9N23RnBndK-h5Wkdc"},"published_in":"https://crawlcheck.io/.well-known/http-message-signatures-directory"},"canonical":"sha256 over canonical JSON of every field except sha256 and signature"}}],"jwks":{"keys":[{"key_ops":["verify"],"ext":true,"alg":"RS256","kty":"RSA","n":"xJlkjrfM9UG4lgK5lTiBodFR9xMTojA_h45ndmTqFe_WOd_c8g8z-f9vXdgaZhXEDBzTdAA-ln1vuE04Xwxl2nQQpK-8pJkKyUTm5ONlf0a1vUUioWVhaUisEcxkpELFDq42D9StkrMi8S6y9Q9AYwXCsmgzLGFLG665Z9udOpHhH6SKHt0_C8RU9Tqxhi9CHttpiVfKENssB25lskvezm4e698G2O-Bb4xKHKEGgqTq7NeX3uw7aUKMXOZNucvOEjYteZMLzFchmops2BrTyjrCw1zu7QEsX-myjNJ_KIrGUVhkCADmFOKnFkM6Qh7UezFgdWSs708zh5z6H7Qspw","e":"AQAB","kid":"test-gh","use":"sig"}]},"witness":{"batch":2,"sth_sha256":"091d4d852dd45ae637d93368c0bf1602465cd50d0f4a81a225cb0b6f65833d6f","at":"2026-10-07T12:43:36.119Z","witness":"github-actions:test","run":{"run_id":"424242"},"token":"eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCIsImtpZCI6InRlc3QtZ2gifQ.eyJpc3MiOiJodHRwczovL3Rva2VuLmFjdGlvbnMuZ2l0aHVidXNlcmNvbnRlbnQuY29tIiwiYXVkIjoiY3Jhd2xjaGVjay1zdGg6MDkxZDRkODUyZGQ0NWFlNjM3ZDkzMzY4YzBiZjE2MDI0NjVjZDUwZDBmNGE4MWEyMjVjYjBiNmY2NTgzM2Q2ZiIsInJlcG9zaXRvcnkiOiJlbW1hbnVlbG9ydGEvY3Jhd2xjaGVjayIsImpvYl93b3JrZmxvd19yZWYiOiJlbW1hbnVlbG9ydGEvY3Jhd2xjaGVjay8uZ2l0aHViL3dvcmtmbG93cy9sb2ctd2l0bmVzcy55bWxAcmVmcy9oZWFkcy9tYWluIiwicnVubmVyX2Vudmlyb25tZW50IjoiZ2l0aHViLWhvc3RlZCIsInJ1bl9pZCI6IjQyNDI0MiIsImlhdCI6MSwiZXhwIjoyfQ.WxmmvJMCe5vK41vSzLd84uIy69wxOni4NICfCh19aaFUoZylcYxf1z7s_8LdyEIItHaSmf6paHNgxo7kpaFUviJqH9pRW1evOikoluiYVEWzjIgrBvS5rk1QbT_YokSNuN1HXuliYLQu1KCWzWe8bA8C1jUprTETpWd2bUGHQMsSGH3MkCTH4NpZwTTxFCNU1Dhk0QZQASNmz0gPXEWmnGoYNvfqbVAjENo1wT4P9p4ZaQ2YXwC25jHTMPVBKdIVkqpdfTxwEGKd6PtOD83oRqZI32gdiVGmUPlfmJKbWfcXkUW2cL3kxQTPVEMbwZ4scly55-c6uIEJbhbfWAv0Cg"}}
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
{
|
|
2
|
+
"kid": "pH8ZDz3tgRtZvHTZdqEmYHGpPI72pmDrjFpMSrNt3VA",
|
|
3
|
+
"valid": "v=ccr1;d=example.com;st=measured;read=allow;cite=warn;connect=require_confirmation;transact=block;o=2026-10-07T00:00:00.000Z;f=2099-01-01T00:00:00.000Z;l=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa;k=pH8ZDz3tgRtZvHTZdqEmYHGpPI72pmDrjFpMSrNt3VA;x=ozx_4zIp-D-K_X4h1VRwWAMfOnzsBchF2lImNSJPMSY;s=7UNCtVpvzFOPfXFrWh1TzhjtWty_DE9CcKEQmFTRdY83b2fKPMNYLfn0aHV4-5ja8Pm0OfgkptK8wzoHSpa_Cw",
|
|
4
|
+
"expired": "v=ccr1;d=example.com;st=measured;read=allow;cite=warn;connect=require_confirmation;transact=block;o=2026-10-07T00:00:00.000Z;f=2020-01-01T00:00:00.000Z;l=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa;k=pH8ZDz3tgRtZvHTZdqEmYHGpPI72pmDrjFpMSrNt3VA;x=ozx_4zIp-D-K_X4h1VRwWAMfOnzsBchF2lImNSJPMSY;s=cZTwNwGE2j88S1Q7em7FRQrCv66v0mATGMd1zXt55fcDX77MEmB1IJzCG88HkPtLQvy1cGv_zHg4D974Yx40DQ",
|
|
5
|
+
"foreign": "v=ccr1;d=other.example;st=measured;read=allow;cite=warn;connect=require_confirmation;transact=block;o=2026-10-07T00:00:00.000Z;f=2099-01-01T00:00:00.000Z;l=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa;k=pH8ZDz3tgRtZvHTZdqEmYHGpPI72pmDrjFpMSrNt3VA;x=ozx_4zIp-D-K_X4h1VRwWAMfOnzsBchF2lImNSJPMSY;s=rUaiUZvO9JpJ4SJbTV_37-e5NbLhhcV0rQ7XrsQlSkjy8vzdETGIWNiohxq0BzmQ0U7vI8NeC3Gh48BBEFOSAw",
|
|
6
|
+
"tampered": "v=ccr1;d=example.com;st=measured;read=allow;cite=warn;connect=require_confirmation;transact=allow;o=2026-10-07T00:00:00.000Z;f=2099-01-01T00:00:00.000Z;l=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa;k=pH8ZDz3tgRtZvHTZdqEmYHGpPI72pmDrjFpMSrNt3VA;x=ozx_4zIp-D-K_X4h1VRwWAMfOnzsBchF2lImNSJPMSY;s=7UNCtVpvzFOPfXFrWh1TzhjtWty_DE9CcKEQmFTRdY83b2fKPMNYLfn0aHV4-5ja8Pm0OfgkptK8wzoHSpa_Cw"
|
|
7
|
+
}
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
// Split-view detection: a forced fork of the Resolve transparency log (one signing key, two histories, made by the
|
|
2
|
+
// Worker's own log code in a test harness) must be detected offline, and the live log must check out consistent.
|
|
3
|
+
import test from "node:test"; import assert from "node:assert/strict"; import fs from "node:fs";
|
|
4
|
+
import { CrawlCheck, newLogStore, logObserve, logGossip, logHeads, verifyWitnessToken, verifyDocument } from "../dist/index.js";
|
|
5
|
+
const F = JSON.parse(fs.readFileSync(new URL("./fixtures/split-view.json", import.meta.url), "utf8"));
|
|
6
|
+
|
|
7
|
+
test("a forked log is detected by gossip: two signed heads for one batch", async () => {
|
|
8
|
+
const a = newLogStore(), b = newLogStore();
|
|
9
|
+
for (const s of F.A) assert.equal((await logObserve(a, s, { publishedKids: [F.kid] })).state, "new");
|
|
10
|
+
for (const s of F.B) await logObserve(b, s, { publishedKids: [F.kid] });
|
|
11
|
+
assert.equal(a.forks.length + b.forks.length, 0, "each history alone is consistent");
|
|
12
|
+
const g = await logGossip(a, logHeads(b), { publishedKids: [F.kid] });
|
|
13
|
+
assert.deepEqual(g.forks.map((f) => f.kind + "@" + f.batch).sort(), ["two_heads_for_one_batch@1", "two_heads_for_one_batch@2"]);
|
|
14
|
+
assert.ok((await verifyDocument(g.forks[0].a)).verified && (await verifyDocument(g.forks[0].b)).verified, "both heads of the proof verify on their own");
|
|
15
|
+
});
|
|
16
|
+
test("a head that does not chain to one held is a fork", async () => {
|
|
17
|
+
const c = newLogStore(); await logObserve(c, F.A[1]);
|
|
18
|
+
const r = await logObserve(c, F.B[2]);
|
|
19
|
+
assert.equal(r.state, "fork"); assert.equal(r.fork.kind, "chain_break");
|
|
20
|
+
});
|
|
21
|
+
test("a tampered head proves nothing (invalid, not a fork)", async () => {
|
|
22
|
+
const s = Object.assign({}, F.B[1], { batch_root: "0".repeat(64) });
|
|
23
|
+
assert.equal((await logObserve(newLogStore(), s)).state, "invalid");
|
|
24
|
+
});
|
|
25
|
+
test("the witness token binds one head: valid for it, refused for the forked head", async () => {
|
|
26
|
+
assert.equal((await verifyWitnessToken(F.witness.token, F.A[2].sha256, F.jwks)).ok, true);
|
|
27
|
+
assert.equal((await verifyWitnessToken(F.witness.token, F.B[2].sha256, F.jwks)).ok, false);
|
|
28
|
+
});
|
|
29
|
+
test("live: the production log is consistent and GitHub's witness co-signature verifies", async () => {
|
|
30
|
+
const c = new CrawlCheck(); await c.resolve("github.com");
|
|
31
|
+
const r = await c.logCheck({ report: false });
|
|
32
|
+
assert.equal(r.consistent, true, r.summary);
|
|
33
|
+
assert.ok(r.witnessed && r.witnessed.token === true, JSON.stringify(r.witnessed));
|
|
34
|
+
});
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
import test from "node:test"; import assert from "node:assert/strict"; import fs from "node:fs";
|
|
2
|
+
import { stapleCheck, stapleFromResponse, CrawlCheck } from "../dist/index.js";
|
|
3
|
+
const F = JSON.parse(fs.readFileSync(new URL("./fixtures/staple.json", import.meta.url), "utf8"));
|
|
4
|
+
const kids = [F.kid];
|
|
5
|
+
test("valid staple verifies offline", async () => { const c = await stapleCheck(F.valid, { host: "example.com", publishedKids: kids }); assert.equal(c.valid, true); assert.equal(c.decisions.transact, "block"); });
|
|
6
|
+
test("subdomain of the stapled domain is covered", async () => { assert.equal((await stapleCheck(F.valid, { host: "shop.example.com", publishedKids: kids })).valid, true); });
|
|
7
|
+
test("expired staple is ignored", async () => { const c = await stapleCheck(F.expired, { host: "example.com", publishedKids: kids }); assert.equal(c.valid, false); assert.equal(c.expired, true); });
|
|
8
|
+
test("staple for another domain is refused", async () => { assert.equal((await stapleCheck(F.foreign, { host: "example.com", publishedKids: kids })).valid, false); });
|
|
9
|
+
test("tampered staple fails the signature", async () => { const c = await stapleCheck(F.tampered, { host: "example.com", publishedKids: kids }); assert.equal(c.valid, false); assert.match(c.reason, /signature/); });
|
|
10
|
+
test("unpublished key is refused", async () => { assert.equal((await stapleCheck(F.valid, { host: "example.com", publishedKids: ["someone-else"] })).valid, false); });
|
|
11
|
+
test("header is read from a response", () => { assert.equal(stapleFromResponse({ headers: new Headers({ "CrawlCheck-Staple": F.valid }) }), F.valid); assert.equal(stapleFromResponse({ "crawlcheck-staple": "x" }), "x"); });
|
|
12
|
+
function mockClient(calls) { return new CrawlCheck({ fetch: async (u) => { calls.push(String(u)); if (String(u).includes("signatures-directory")) return new Response(JSON.stringify({ keys: [{ kid: F.kid }] })); return new Response(JSON.stringify({ kind: "crawlcheck-resolve", actions: { read: { decision: "warn" }, cite: { decision: "warn" }, connect: { decision: "unsupported" }, transact: { decision: "require_confirmation" } } })); } }); }
|
|
13
|
+
test("resolveFor prefers a valid staple and makes no resolve request", async () => { const calls = []; const r = await mockClient(calls).resolveFor("example.com", { staple: F.valid }); assert.equal(r.source, "staple"); assert.equal(calls.some((u) => u.includes("/api/v1/resolve")), false); });
|
|
14
|
+
test("resolveFor ignores an expired staple and asks CrawlCheck", async () => { const calls = []; const r = await mockClient(calls).resolveFor("example.com", { staple: F.expired }); assert.equal(r.source, "network"); assert.equal(r.decisions.read, "warn"); assert.equal(r.staple.expired, true); assert.equal(calls.some((u) => u.includes("/api/v1/resolve")), true); });
|