@crawlcheck/sdk 1.0.7 → 1.0.8

package/dist/verify.d.ts

@@ -10,10 +10,20 @@ export function dspVerdict(m: any): {
     rule: string;
 };
 export function rfcRootV(leafHex: any, index: any, size: any, path: any): Promise<string | null>;
+export function tvReachV(envelope: any): any;
+export function verifyThresholdSigners(x: any): Promise<{
+    observer_id: any;
+    network: any;
+    signature: boolean;
+    binding: boolean;
+    values_match: boolean;
+    why: never[];
+}[]>;
 export function trustOf(res: any, kid: any, publishedKids: any): any;
 export function publishedKeyIds(base: any): Promise<string[]>;
 export function verifyBundle(b: any, opts: any): Promise<any>;
 export function verifyReceipt(rc: any, opts: any): Promise<any>;
 export function verifyExport(x: any, opts: any): Promise<any>;
 export function verifyResolution(x: any, opts: any): Promise<any>;
+export function verifyDnsTxt(txt: any, opts: any): Promise<any>;
 export function verifyDataDoc(x: any, opts: any): Promise<any>;

package/dist/verify.js

@@ -519,6 +519,90 @@ export async function rfcRootV(leafHex, index, size, path) {
     }
     return sn === 0 ? hex(r) : null;
 }
+// tv1: threshold-signed verdicts. Every signer is checked on its own: its Ed25519 signature over its envelope, the
+// binding of its key to its identity (GitHub's RS256 token, a self-signed enrolment, or the published key directory),
+// and the field values recomputed from its envelope. The verdict holds when signers on at least k distinct networks pass.
+const TV_OBS_PREFIX = "crawlcheck-observation-v1\n", TV_ENROL_PREFIX = "crawlcheck-observer-enrol-v1\n";
+function tvFinalV(u) { try {
+    const x = new URL(u);
+    return x.hostname.toLowerCase().replace(/^www\./, "") + (x.pathname.replace(/\/+$/, "") || "/");
+}
+catch (e) {
+    return null;
+} }
+export function tvReachV(envelope) {
+    return (envelope && Array.isArray(envelope.fetches) ? envelope.fetches : []).map((f) => ({ identity: String(f.identity), status: typeof f.status === "number" ? f.status : null, final: f.url_final ? tvFinalV(f.url_final) : null, failed: !!f.error }))
+        .sort((a, b) => (a.identity < b.identity ? -1 : a.identity > b.identity ? 1 : 0));
+}
+async function tvEd(x, msg, sig) { try {
+    const pk = await subtle.importKey("jwk", { kty: "OKP", crv: "Ed25519", x }, { name: "Ed25519" }, false, ["verify"]);
+    return await subtle.verify({ name: "Ed25519" }, pk, b64(sig), enc.encode(msg));
+}
+catch (e) {
+    return null;
+} }
+async function tvJwt(token, jwk) {
+    const p = String(token || "").split(".");
+    if (p.length !== 3 || !jwk || jwk.kty !== "RSA")
+        return { ok: false };
+    let h, c;
+    try {
+        h = JSON.parse(new TextDecoder().decode(b64(p[0])));
+        c = JSON.parse(new TextDecoder().decode(b64(p[1])));
+    }
+    catch (e) {
+        return { ok: false };
+    }
+    if (h.alg !== "RS256" || (jwk.kid && h.kid !== jwk.kid))
+        return { ok: false, claims: c };
+    try {
+        const k = await subtle.importKey("jwk", { kty: "RSA", n: jwk.n, e: jwk.e, alg: "RS256", ext: true }, { name: "RSASSA-PKCS1-v1_5", hash: "SHA-256" }, false, ["verify"]);
+        return { ok: await subtle.verify("RSASSA-PKCS1-v1_5", k, b64(p[2]), enc.encode(p[0] + "." + p[1])), claims: c, kid: h.kid };
+    }
+    catch (e) {
+        return { ok: null, claims: c };
+    }
+}
+export async function verifyThresholdSigners(x) {
+    const st = x.statement || {}, want = canon(st.values || []), rows = [];
+    for (const s of Array.isArray(x.signers) ? x.signers : []) {
+        const r = { observer_id: s.observer_id, network: s.network && s.network.class || null, signature: false, binding: false, values_match: false, why: [] };
+        const ev = s.envelope || {}, key = ev.key || {};
+        r.signature = !!(await tvEd(key.x, TV_OBS_PREFIX + canon(ev), s.sig));
+        if (!r.signature)
+            r.why.push("signature does not verify under the envelope key");
+        const b = s.binding || {};
+        if (b.kind === "github_oidc") {
+            const j = await tvJwt(b.token, b.github_key), c = j.claims || {};
+            const aud = "crawlcheck-observer:" + b64u(await sha256(b64(key.x || "")));
+            r.binding = j.ok === true && c.iss === "https://token.actions.githubusercontent.com" && c.repository === b.repository && String(c.job_workflow_ref || c.workflow_ref || "").indexOf(b.workflow_ref_prefix || "\u0000") === 0 && (Array.isArray(c.aud) ? c.aud : [c.aud]).includes(aud) && (!c.runner_environment || c.runner_environment === "github-hosted");
+            r.github_kid = j.kid || null;
+            if (!r.binding)
+                r.why.push("GitHub token does not bind this key to the observer workflow");
+        }
+        else if (b.kind === "enrolled_ed25519") {
+            const q = b.enrolment && b.enrolment.request;
+            r.binding = !!(b.key && b.key.x === key.x && q && q.observer_id === s.observer_id && q.key && q.key.x === key.x && await tvEd(key.x, TV_ENROL_PREFIX + canon(q), b.enrolment.sig));
+            if (!r.binding)
+                r.why.push("the key is not the one this observer enrolled with");
+        }
+        else if (b.kind === "published_key_directory") {
+            const thumb = b64u(await sha256(enc.encode(JSON.stringify({ crv: "Ed25519", kty: "OKP", x: key.x }))));
+            r.binding = thumb === b.kid && thumb === (x.signature && x.signature.kid);
+            r.kid = thumb;
+            if (!r.binding)
+                r.why.push("key thumbprint is not the stated (published) key id");
+        }
+        else
+            r.why.push("unknown binding " + b.kind);
+        r.values_match = canon(tvReachV(ev)) === want;
+        if (!r.values_match)
+            r.why.push("the values in this envelope differ from the statement");
+        r.passes = r.signature && r.binding && r.values_match;
+        rows.push(r);
+    }
+    return rows;
+}
 // ── data inventories, export parts and deletion records ─────────────────────────────────────────────────────────
 // Signed over sha256(canonical JSON of every field except sha256 and signature) with the prefix below. A deletion
 // record carries counts per family and the digest of the sorted deleted-key list, never the data.
@@ -528,7 +612,7 @@ async function verifyDataDoc0(x) {
         throw new Error("this runtime has no WebCrypto (crypto.subtle)");
     const checks = [];
     const C = (id, ok, why) => checks.push({ id, ok, why });
-    if (!x || !/^crawlcheck-(data-inventory|data-export-part|deletion-record|resolve|decision|snapshot|visitor|conduct|traffic-index|anchor|anchor-check|content-clock|origin|claims|read-equivalence|rights|rights-feed|mcp-scan|mcp-lock|mcp-lock-check|mcp-shadow|mcp-auth|impersonation|task-canaries|incident|revocations|high-risk|log-sth)$/.test(String(x.kind)) || !x.signature)
+    if (!x || !/^crawlcheck-(data-inventory|data-export-part|deletion-record|resolve|decision|snapshot|visitor|conduct|traffic-index|anchor|anchor-check|content-clock|origin|claims|read-equivalence|rights|rights-feed|mcp-scan|mcp-lock|mcp-lock-check|mcp-shadow|mcp-auth|impersonation|task-canaries|agent-safety-benchmark|compliance-pack|share-of-preflight|incident|revocations|high-risk|log-sth|threshold-verdict|resolve-as-of)$/.test(String(x.kind)) || !x.signature)
         throw new Error("not a CrawlCheck data inventory, export part, deletion record, resolve answer, decision receipt, snapshot manifest, visitor verdict, conduct record, traffic index, citation anchor, content clock, origin decision, claim ledger, read-equivalence certificate, rights answer, rights-feed manifest, MCP tool scan, MCP lockfile or lockfile check");
     const body = {};
     for (const k of Object.keys(x))
@@ -577,6 +661,10 @@ async function verifyDataDoc0(x) {
         const ps = Array.isArray(x.parts) ? x.parts : [];
         C("parts", ps.length > 0 && ps.every((p) => /^part-\d{4}\.ndjson\.gz$/.test(String(p.path)) && /^[0-9a-f]{64}$/.test(String(p.sha256)) && p.records > 0) && ps.reduce((a, p) => a + p.records, 0) === x.records, ps.reduce((a, p) => a + (p.records || 0), 0) === x.records ? ps.length + " parts, " + x.records + " records; check each downloaded part with sha256sum against its sha256, then each line with this tool" : "the parts' record counts do not add up to records (" + x.records + ")");
     }
+    if (x.kind === "crawlcheck-resolve" && Array.isArray(x.absences)) { // sa1: every absence states what was checked, from where and when
+        const bad = x.absences.filter((a) => !(a && a.what && (a.result === "absent" || a.result === "none") && a.at && !isNaN(Date.parse(a.at)) && a.from && a.from.observer && typeof a.scope === "string" && Array.isArray(a.checked) && a.checked.length && a.checked.every((c) => c && c.method && (a.result !== "absent" || c.status === undefined || c.status === 404 || c.status === 410))));
+        C("absences_scoped", bad.length === 0, x.absences.length + " absence record(s)" + (x.absences.length ? " (" + x.absences.map((a) => a.what).join(", ") + ")" : "") + (bad.length ? "; " + bad.length + " lack a definite check, a time, a vantage or a scope" : ", each naming what was requested, the answer, the vantage and the time"));
+    }
     if (x.kind === "crawlcheck-resolve" && x.transparency) { // tl1: the answer's place in the Resolve transparency log
         const t = x.transparency, leaf = hex(await sha256(cat(new Uint8Array([0]), enc.encode(canon(tlContentV(x))))));
         C("log_leaf", leaf === t.leaf_hash, leaf === t.leaf_hash ? "the answer's decision content hashes to its log leaf " + leaf.slice(0, 16) + "…" : "the decision content hashes to " + leaf + ", the answer names " + t.leaf_hash);
@@ -596,6 +684,41 @@ async function verifyDataDoc0(x) {
         else
             C("log_inclusion", null, "pending: the leaf is queued and merged within " + (t.merged_within_minutes || 30) + " minutes; fetch the proof at " + (t.proof || "the log"));
     }
+    if (x.kind === "crawlcheck-resolve-as-of") { // ah1: the answer in force at as_of, as signed then
+        let av = null;
+        try {
+            av = await verifyDataDoc0(x.answer);
+        }
+        catch (e) {
+            av = null;
+        }
+        C("answer_signature", !!(av && av.verified) && !!x.answer && x.answer.kind === "crawlcheck-resolve" && x.answer.domain === x.domain, av && av.verified ? "the archived answer verifies on its own (signed " + (x.answer.signature && x.answer.signature.signed_at) + ")" : "the archived answer does NOT verify");
+        const leaf = x.answer ? hex(await sha256(cat(new Uint8Array([0]), enc.encode(canon(tlContentV(x.answer)))))) : null;
+        C("answer_leaf", leaf === x.leaf_hash, leaf === x.leaf_hash ? "the answer's decision content is leaf " + String(leaf).slice(0, 16) + "…" : "the answer's content hashes to " + leaf + ", not the stated leaf");
+        const w = x.in_force || {}, t = Date.parse(x.as_of);
+        C("in_force_window", Date.parse(w.from) <= t && (w.until == null || t < Date.parse(w.until)), "as_of " + x.as_of + " falls in [" + w.from + ", " + (w.until || "now") + ")");
+        const pr = x.log_inclusion;
+        if (pr && pr.sth) {
+            const root = await rfcRootV(x.leaf_hash, pr.index, pr.batch_size, pr.path || []);
+            let sv = null;
+            try {
+                sv = await verifyDataDoc0(pr.sth);
+            }
+            catch (e) {
+                sv = null;
+            }
+            C("log_inclusion", root !== null && root === pr.sth.batch_root && !!(sv && sv.verified), root === pr.sth.batch_root ? "the leaf is in batch " + pr.batch + " of the signed log (head created " + pr.sth.created_at + ")" : "the inclusion proof does NOT reach the signed batch root");
+        }
+        else
+            C("log_inclusion", null, "not merged into the log yet; merged within 30 minutes of first appearing");
+    }
+    if (x.kind === "crawlcheck-threshold-verdict") { // tv1
+        const rows = await verifyThresholdSigners(x), k = x.threshold && x.threshold.k || 2;
+        rows.forEach((r) => C("signer:" + r.observer_id, r.passes, r.passes ? "signature, key binding (" + (x.signers.find((s) => s.observer_id === r.observer_id).binding.kind) + ") and recomputed " + (x.statement && x.statement.family) + " values all check (network " + r.network + ")" : r.why.join("; ")));
+        const nets = new Set(rows.filter((r) => r.passes).map((r) => r.network));
+        C("threshold", nets.size >= k, nets.size + " distinct network(s) signed identical values (" + [...nets].join(", ") + "); the verdict needs " + k);
+        C("key_bindings_online", null, "offline the bindings are checked against keys the document carries; online, compare the GitHub key id with " + "https://token.actions.githubusercontent.com/.well-known/jwks and enrolled keys with https://crawlcheck.io/.well-known/crawlcheck-observers.json");
+    }
     if (x.kind === "crawlcheck-deletion-record")
         C("counts", (x.deleted || []).reduce((a, f) => a + (f.keys || 0), 0) === x.deleted_total, "the per-family counts add up to deleted_total (" + x.deleted_total + ")");
     const ran = checks.filter((c) => c.ok !== null), failed = checks.filter((c) => c.ok === false);
@@ -640,6 +763,38 @@ export async function verifyBundle(b, opts) { return trustOf(await verifyBundle0
 export async function verifyReceipt(rc, opts) { return trustOf(await verifyReceipt0(rc), rc && rc.signature && rc.signature.kid, opts && opts.publishedKids); }
 export async function verifyExport(x, opts) { return trustOf(await verifyExport0(x), kidOf(x), opts && opts.publishedKids); }
 export async function verifyResolution(x, opts) { return trustOf(await verifyResolution0(x), kidOf(x), opts && opts.publishedKids); }
+// dns1: a Resolve-over-DNS TXT record (ccr1). Strings joined without spaces; quotes from dig output are stripped.
+async function verifyDnsTxt0(txt) {
+    if (!subtle)
+        throw new Error("this runtime has no WebCrypto (crypto.subtle)");
+    const checks = [], C = (id, ok, why) => checks.push({ id, ok, why });
+    const t = String(txt || "").trim().replace(/"\s*"/g, "").replace(/^"|"$/g, "").replace(/\;/g, ";");
+    const i = t.lastIndexOf(";s="), body = i > 0 ? t.slice(0, i) : "", sig = i > 0 ? t.slice(i + 3) : "";
+    const f = {};
+    body.split(";").forEach((p) => { const j = p.indexOf("="); if (j > 0)
+        f[p.slice(0, j)] = p.slice(j + 1); });
+    C("format", f.v === "ccr1" && !!f.d && !!f.k && !!f.x && !!sig, f.v === "ccr1" ? "ccr1 record for " + (f.d || "?") + " (" + (f.st || "?") + ")" : "not a ccr1 record");
+    if (f.x && f.k) {
+        const thumb = b64u(await sha256(enc.encode(JSON.stringify({ crv: "Ed25519", kty: "OKP", x: f.x }))));
+        C("key_id", thumb === f.k, thumb === f.k ? "public key thumbprint (RFC 7638) is the key id " + thumb.slice(0, 12) + "…" : "thumbprint " + thumb + " does not match key id " + f.k);
+        let ok = null;
+        try {
+            const pk = await subtle.importKey("jwk", { kty: "OKP", crv: "Ed25519", x: f.x }, { name: "Ed25519" }, false, ["verify"]);
+            ok = await subtle.verify({ name: "Ed25519" }, pk, b64(sig), enc.encode("crawlcheck-dns-v1\n" + body));
+        }
+        catch (e) {
+            ok = null;
+            C("signature", null, "this runtime cannot verify Ed25519: " + (e && e.message || e));
+        }
+        if (ok !== null)
+            C("signature", ok, ok ? "Ed25519 signature over the record verifies" : "signature does NOT verify over this record");
+    }
+    const fu = Date.parse(f.f || "");
+    C("fresh", isFinite(fu) ? fu > Date.now() : false, isFinite(fu) ? (fu > Date.now() ? "fresh until " + f.f : "expired at " + f.f + ": ask again") : "no fresh_until");
+    C("key_published", null, "offline this cannot be decided: compare key id " + (f.k || "?") + " with the ids published at https://crawlcheck.io/.well-known/http-message-signatures-directory");
+    return { record: f, decisions: { read: f.read || null, cite: f.cite || null, connect: f.connect || null, transact: f.transact || null }, checks };
+}
+export async function verifyDnsTxt(txt, opts) { const r = await verifyDnsTxt0(txt); return trustOf(r, r.record && r.record.k, opts && opts.publishedKids); }
 export async function verifyDataDoc(x, opts) { return trustOf(await verifyDataDoc0(x), kidOf(x), opts && opts.publishedKids); }
 const isMain = typeof process !== "undefined" && !!(process.argv && process.argv[1]) && await (async () => { try {
     const { pathToFileURL } = await import("node:url");
@@ -651,9 +806,32 @@ catch (e) {
 } })();
 if (isMain) {
     const online = process.argv.includes("--online");
+    if (process.argv.includes("--dns")) { // dns1: a Resolve-over-DNS TXT record
+        let txt = process.argv.slice(2).filter((a) => a !== "--online" && a !== "--dns").join(" ");
+        if (!txt) {
+            const { readFileSync } = await import("node:fs");
+            txt = readFileSync(0, "utf8");
+        }
+        let kd;
+        if (online) {
+            try {
+                kd = await publishedKeyIds();
+            }
+            catch (e) {
+                console.log("could not read the published key directory: " + e.message);
+            }
+        }
+        const r = await verifyDnsTxt(txt, { publishedKids: kd });
+        console.log("CrawlCheck Resolve over DNS: " + (r.record.d || "?") + " " + (r.record.st || "?") + " · read " + r.decisions.read + " · cite " + r.decisions.cite + " · connect " + r.decisions.connect + " · transact " + r.decisions.transact);
+        for (const c of r.checks)
+            console.log((c.ok === true ? "PASS" : c.ok === false ? "FAIL" : " -- ") + "  " + c.id.padEnd(23) + c.why);
+        console.log(r.summary);
+        console.log(r.accepted ? "ACCEPTED" : "NOT ACCEPTED" + (r.integrity_valid && r.issuer_trusted === null ? ": issuer unknown offline (run with --online)" : ""));
+        process.exit(r.checks.some((c) => c.ok === false) ? 1 : 0);
+    }
     const f = process.argv.slice(2).filter((a) => a !== "--online")[0];
     if (!f) {
-        console.log("usage: node crawlcheck-verify.mjs <bundle.json | receipt.json | export.json | resolution.json | deletion.json | answer.json | decision.json | manifest.json>");
+        console.log("usage: node crawlcheck-verify.mjs [--online] [--dns \"<ccr1 TXT record>\"] <bundle.json | receipt.json | export.json | resolution.json | deletion.json | answer.json | decision.json | manifest.json>");
         process.exit(2);
     }
     const { readFile } = await import("node:fs/promises");
@@ -661,7 +839,7 @@ if (isMain) {
     const isRc = doc && (doc.kind === "crawlcheck-remediation-receipt" || (doc.receipt && doc.receipt.kind === "crawlcheck-remediation-receipt"));
     const isEx = doc && doc.kind === "crawlcheck-evidence-export";
     const isDr = doc && doc.kind === "crawlcheck-dispute-resolution";
-    const isDt = doc && /^crawlcheck-(data-inventory|data-export-part|deletion-record|resolve|decision|snapshot|visitor|conduct|traffic-index|anchor|anchor-check|content-clock|origin|claims|read-equivalence|rights|rights-feed|mcp-scan|mcp-lock|mcp-lock-check|mcp-shadow|mcp-auth|impersonation|task-canaries|incident|revocations|high-risk|log-sth)$/.test(String(doc.kind));
+    const isDt = doc && /^crawlcheck-(data-inventory|data-export-part|deletion-record|resolve|decision|snapshot|visitor|conduct|traffic-index|anchor|anchor-check|content-clock|origin|claims|read-equivalence|rights|rights-feed|mcp-scan|mcp-lock|mcp-lock-check|mcp-shadow|mcp-auth|impersonation|task-canaries|agent-safety-benchmark|compliance-pack|share-of-preflight|incident|revocations|high-risk|log-sth|threshold-verdict|resolve-as-of)$/.test(String(doc.kind));
     let kids;
     if (online) {
         try {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@crawlcheck/sdk",
-  "version": "1.0.7",
+  "version": "1.0.8",
   "description": "Typed client for the CrawlCheck API (generated from its OpenAPI document) and the zero-dependency offline verifier for evidence bundles and remediation receipts.",
   "type": "module",
   "main": "./dist/index.js",