@crawlcheck/sdk 1.0.4 → 1.0.6

package/dist/index.d.ts

@@ -123,6 +123,49 @@ export declare function normalizeDomain(v: string): string;
 export declare function receiptHeader(receipt: {
     sha256: string;
 }): Record<string, string>;
+/** MCP tool lockfile (https://crawlcheck.io/spec/mcp-lockfile). A tool's hash is sha256 of the canonical JSON of {name, description, inputSchema, annotations}. */
+export interface McpTool {
+    name?: string;
+    description?: string;
+    inputSchema?: unknown;
+    input_schema?: unknown;
+    annotations?: unknown;
+    [k: string]: unknown;
+}
+export interface McpLock {
+    tools_sha256: string;
+    tools: {
+        name: string;
+        sha256: string;
+    }[];
+    descriptions?: Record<string, string>;
+    server?: {
+        url?: string | null;
+    } | null;
+    approved_at?: string;
+    [k: string]: unknown;
+}
+export interface McpLockCheck {
+    verdict: "unchanged" | "changed";
+    decision: "connect" | "block";
+    approved_tools_sha256: string;
+    current_tools_sha256: string;
+    added: string[];
+    removed: string[];
+    modified: string[];
+}
+export declare function lockToolHash(t: McpTool): Promise<string>;
+/** Hash a tools/list result the way a lockfile does: per tool, and the sorted per-tool hashes joined by \n. No network, no key. */
+export declare function lockFromTools(tools: McpTool[]): Promise<{
+    tools: {
+        name: string;
+        sha256: string;
+    }[];
+    tools_sha256: string;
+    count: number;
+}>;
+/** Compare a lockfile with the tools a server lists now. Anything added, removed or changed means "block" until a person re-approves. Offline. */
+export declare function checkLock(lock: McpLock, tools: McpTool[]): Promise<McpLockCheck>;
 export interface PrivateLookup {
     found: boolean;
     ready: boolean;
@@ -179,6 +222,112 @@ export interface GuardOptions extends PreflightOptions {
     /** Also require the signing key to be in the published key directory (default true). */
     onlineKeys?: boolean;
 }
+export interface LogHead {
+    kind: "crawlcheck-log-sth";
+    log_id: string;
+    batch: number;
+    first_index: number;
+    batch_size: number;
+    tree_size: number;
+    batch_root: string;
+    prev_sth_sha256: string | null;
+    created_at: string;
+    sha256: string;
+    signature: JsonObject;
+    [k: string]: unknown;
+}
+export interface LogFork {
+    kind: "two_heads_for_one_batch" | "chain_break" | "size_mismatch" | "inconsistent_head";
+    batch: number;
+    why: string;
+    a: LogHead;
+    b: LogHead;
+    detected_at: string;
+    proof: string;
+}
+export interface LogStore {
+    v: 1;
+    log_id: string;
+    heads: Record<string, {
+        batch: number;
+        sha256: string;
+        first_index: number;
+        batch_size: number;
+        tree_size: number;
+        prev_sth_sha256: string | null;
+        created_at: string | null;
+        sth: LogHead;
+    }>;
+    forks: LogFork[];
+    witnessed: {
+        batch: number;
+        sha256: string;
+        at: string | null;
+        run_id: string | null;
+    } | null;
+}
+export interface LogObservation {
+    state: "new" | "known" | "fork" | "invalid" | "none";
+    why?: string;
+    fork?: LogFork;
+}
+export interface LogCheckResult {
+    consistent: boolean;
+    forks: LogFork[];
+    all_forks: number;
+    witnessed: {
+        batch: number;
+        sha256: string;
+        at: string | null;
+        run_id: string | null;
+        token: boolean | null;
+        why: string;
+        observed: string;
+        warning?: string;
+    } | null;
+    unwitnessed: {
+        batch: number;
+        sha256: string;
+        created_at: string | null;
+    }[];
+    stale_unwitnessed: {
+        batch: number;
+        sha256: string;
+        created_at: string | null;
+    }[];
+    summary: string;
+}
+export declare const LOG_ID: string;
+/** A new, empty transparency-log store (plain JSON: persist it between runs). */
+export declare const newLogStore: () => LogStore;
+/** Check one signed tree head against the store and keep it. A fork carries both contradicting signed heads. */
+export declare const logObserve: (store: LogStore, sth: unknown, opts?: {
+    publishedKids?: string[];
+    maxHeads?: number;
+}) => Promise<LogObservation>;
+/** Check the head inside a Resolve answer (transparency.sth), if it carries one. */
+export declare const logObserveAnswer: (store: LogStore, answer: unknown, opts?: {
+    publishedKids?: string[];
+}) => Promise<LogObservation>;
+/** The signed heads a store holds, newest first: what a client sends its peers. */
+export declare const logHeads: (store: LogStore, limit?: number) => LogHead[];
+/** Gossip: check another client's heads against this store. */
+export declare const logGossip: (store: LogStore, sths: unknown[], opts?: {
+    publishedKids?: string[];
+}) => Promise<{
+    forks: LogFork[];
+    results: (LogObservation & {
+        batch: number;
+        sha256: string;
+    })[];
+}>;
+/** Verify the outside witness's co-signature: a GitHub OIDC JWT (RS256) whose audience is crawlcheck-sth:<head sha256>. */
+export declare const verifyWitnessToken: (token: string, sthSha256: string, jwks: unknown) => Promise<{
+    ok: boolean | null;
+    why: string;
+    claims?: JsonObject;
+}>;
+export declare const GITHUB_OIDC_JWKS = "https://token.actions.githubusercontent.com/.well-known/jwks";
 export interface ClientOptions {
     /** Licence key (cc_ + 32 hex). Without one, licence-gated fields come back withheld, exactly as on the website. */
     key?: string;
@@ -188,6 +337,8 @@ export interface ClientOptions {
     fetch?: typeof fetch;
     /** Extra headers for every request. */
     headers?: Record<string, string>;
+    /** Transparency-log store (newLogStore()). Every Resolve answer's signed tree head is checked against it; keep it across runs to catch a split view. Default: a fresh in-memory store. */
+    logStore?: LogStore;
 }
 export declare class CrawlCheckError extends Error {
     readonly status: number;
@@ -209,15 +360,31 @@ export declare class CrawlCheck {
     private readonly key?;
     private readonly f;
     private readonly extra;
+    /** The transparency-log store this client checks every signed tree head against. Persist it (JSON) between runs. */
+    readonly log: LogStore;
     constructor(opts?: ClientOptions);
     /** Any documented GET, typed from the OpenAPI document: client.get("/api/explain", { domain: "example.com" }). */
     get<P extends keyof paths>(path: P, query?: QueryOf<P, "get">): Promise<ResponseOf<P, "get">>;
     /** Any documented POST, typed from the OpenAPI document. */
     post<P extends keyof paths>(path: P, body: BodyOf<P> | JsonObject, query?: QueryOf<P, "post">): Promise<ResponseOf<P, "post">>;
     /** The signed ResolveV1 answer: crawl policy, delivery, machine files, capabilities, entity, findings, freshness and a decision per action. */
-    resolve(domain: string): Promise<JsonObject>;
+    resolve(domain: string): Promise<ResponseOf<"/api/v1/resolve", "get">>;
     /** Up to 100 domains in one call; each answer is a full signed ResolveV1 document. */
-    resolveBatch(domains: string[]): Promise<JsonObject>;
+    resolveBatch(domains: string[]): Promise<any>;
+    /** Split-view check: the current head, GitHub's witness co-signature on the last witnessed head, and the chain from it down to every head this client holds. Reports the heads it holds to /api/v1/log/gossip unless report is false. */
+    logCheck(opts?: {
+        report?: boolean;
+        maxWalk?: number;
+        maxUnwitnessedMinutes?: number;
+    }): Promise<LogCheckResult>;
+    /** Check heads another client sent (logHeads on their side) against this client's store. */
+    logGossip(sths: unknown[]): Promise<{
+        forks: LogFork[];
+        results: (LogObservation & {
+            batch: number;
+            sha256: string;
+        })[];
+    }>;
     /** The policy engine: a signed decision receipt (allow | warn | require_confirmation | block | unsupported). A policy can only make the decision stricter. */
     preflight(domain: string, action?: Action, opts?: PreflightOptions): Promise<DecisionReceipt>;
     /** Verify a signed document here. With onlineKeys (default), its signing key must also be in the published directory. */
@@ -228,8 +395,15 @@ export declare class CrawlCheck {
         receipt: DecisionReceipt;
         verification: DocumentVerification & Trust;
         header: Record<string, string> | null;
+        split_view?: LogFork[];
     }>;
     /** Look a domain up without sending it: only the first prefixLen hex characters of sha256(domain) leave this process. found false with ready true means no public measurement. */
+    /** Make a signed lockfile for an MCP server (CrawlCheck reads tools/list; no tool is called). Save it; check with checkLock before every connect. */
+    mcpLock(url: string): Promise<McpLock & JsonObject>;
+    /** Read the server's tools now (through CrawlCheck) and compare with a lockfile; the answer is signed. For a fully offline check, list the tools yourself and call checkLock. */
+    mcpLockCheck(lock: McpLock, url?: string): Promise<McpLockCheck & JsonObject>;
+    /** Tool-poisoning scan of a server URL or a tool list. */
+    mcpScan(target: string | McpTool[]): Promise<JsonObject>;
     resolvePrivate(domain: string, prefixLen?: number): Promise<PrivateLookup>;
     /** For sites: check a CrawlCheck-Receipt header received with a request to host. accept only when the receipt is held, names host, has not expired, decided allow or warn, and verifies with a published key. */
     checkReceiptHeader(headerValue: string, host: string): Promise<ReceiptCheck>;

package/dist/index.js

@@ -1,4 +1,5 @@
 import * as V from "./verify.js";
+import * as L from "./log.js";
 function withTrust(v, kid, published) {
     let checks = v.checks;
     let issuer = null;
@@ -39,6 +40,42 @@ async function sha256Hex(s) {
     const b = await globalThis.crypto.subtle.digest("SHA-256", new TextEncoder().encode(s));
     return Array.from(new Uint8Array(b), (x) => x.toString(16).padStart(2, "0")).join("");
 }
+export async function lockToolHash(t) {
+    return sha256Hex(V.canon({ name: t?.name ?? null, description: t?.description ?? null, inputSchema: t?.inputSchema ?? t?.input_schema ?? null, annotations: t?.annotations ?? null }));
+}
+/** Hash a tools/list result the way a lockfile does: per tool, and the sorted per-tool hashes joined by \n. No network, no key. */
+export async function lockFromTools(tools) {
+    const rows = [];
+    for (const t of (tools ?? []).slice(0, 300))
+        rows.push({ name: String(t?.name ?? "").slice(0, 120), sha256: await lockToolHash(t) });
+    rows.sort((a, b) => (a.name < b.name ? -1 : a.name > b.name ? 1 : a.sha256 < b.sha256 ? -1 : 1));
+    return { tools: rows, tools_sha256: await sha256Hex(rows.map((r) => r.sha256).sort().join("\n")), count: rows.length };
+}
+/** Compare a lockfile with the tools a server lists now. Anything added, removed or changed means "block" until a person re-approves. Offline. */
+export async function checkLock(lock, tools) {
+    const now = await lockFromTools(tools);
+    const was = {};
+    (lock?.tools ?? []).forEach((r) => { was[r.name] = r.sha256; });
+    const cur = {};
+    now.tools.forEach((r) => { cur[r.name] = r.sha256; });
+    const added = Object.keys(cur).filter((n) => !(n in was)), removed = Object.keys(was).filter((n) => !(n in cur)), modified = Object.keys(cur).filter((n) => n in was && was[n] !== cur[n]);
+    const same = lock?.tools_sha256 === now.tools_sha256 && !added.length && !removed.length && !modified.length;
+    return { verdict: same ? "unchanged" : "changed", decision: same ? "connect" : "block", approved_tools_sha256: lock?.tools_sha256 ?? "", current_tools_sha256: now.tools_sha256, added, removed, modified };
+}
+export const LOG_ID = L.LOG_ID;
+/** A new, empty transparency-log store (plain JSON: persist it between runs). */
+export const newLogStore = L.newLogStore;
+/** Check one signed tree head against the store and keep it. A fork carries both contradicting signed heads. */
+export const logObserve = L.logObserve;
+/** Check the head inside a Resolve answer (transparency.sth), if it carries one. */
+export const logObserveAnswer = L.logObserveAnswer;
+/** The signed heads a store holds, newest first: what a client sends its peers. */
+export const logHeads = L.logHeads;
+/** Gossip: check another client's heads against this store. */
+export const logGossip = L.logGossip;
+/** Verify the outside witness's co-signature: a GitHub OIDC JWT (RS256) whose audience is crawlcheck-sth:<head sha256>. */
+export const verifyWitnessToken = L.verifyWitnessToken;
+export const GITHUB_OIDC_JWKS = "https://token.actions.githubusercontent.com/.well-known/jwks";
 export class CrawlCheckError extends Error {
     status;
     body;
@@ -59,10 +96,13 @@ export class CrawlCheck {
     key;
     f;
     extra;
+    /** The transparency-log store this client checks every signed tree head against. Persist it (JSON) between runs. */
+    log;
     constructor(opts = {}) {
         this.base = (opts.base ?? "https://crawlcheck.io").replace(/\/+$/, "");
         this.key = opts.key;
         this.extra = opts.headers ?? {};
+        this.log = opts.logStore ?? newLogStore();
         const f = opts.fetch ?? globalThis.fetch;
         if (typeof f !== "function")
             throw new Error("no fetch available: pass { fetch } (Node 18+ and every browser have one)");
@@ -78,9 +118,27 @@ export class CrawlCheck {
     }
     // ── read before acting ──
     /** The signed ResolveV1 answer: crawl policy, delivery, machine files, capabilities, entity, findings, freshness and a decision per action. */
-    resolve(domain) { return this.get("/api/v1/resolve", { domain }); }
+    async resolve(domain) {
+        const a = await this.get("/api/v1/resolve", { domain });
+        await logObserveAnswer(this.log, a).catch(() => null); // a head that contradicts one this client already holds is recorded in this.log.forks
+        return a;
+    }
     /** Up to 100 domains in one call; each answer is a full signed ResolveV1 document. */
-    resolveBatch(domains) { return this.post("/api/v1/resolve", { domains }); }
+    async resolveBatch(domains) {
+        const r = await this.post("/api/v1/resolve", { domains });
+        for (const a of (r && r.answers) || [])
+            await logObserveAnswer(this.log, a).catch(() => null);
+        return r;
+    }
+    /** Split-view check: the current head, GitHub's witness co-signature on the last witnessed head, and the chain from it down to every head this client holds. Reports the heads it holds to /api/v1/log/gossip unless report is false. */
+    async logCheck(opts = {}) {
+        const kids = await this.publishedKeyIds();
+        return L.logCheck(this.log, (p) => this.req("GET", p), { publishedKids: kids, maxWalk: opts.maxWalk, maxUnwitnessedMinutes: opts.maxUnwitnessedMinutes,
+            fetchJwks: async () => { const r = await this.f(GITHUB_OIDC_JWKS); return r.ok ? r.json() : null; },
+            post: opts.report === false ? undefined : (p, b) => this.req("POST", p, undefined, b) });
+    }
+    /** Check heads another client sent (logHeads on their side) against this client's store. */
+    logGossip(sths) { return logGossip(this.log, sths); }
     /** The policy engine: a signed decision receipt (allow | warn | require_confirmation | block | unsupported). A policy can only make the decision stricter. */
     async preflight(domain, action = "read", opts = {}) {
         const b = { domain, action };
@@ -99,6 +157,10 @@ export class CrawlCheck {
     /** Preflight, verify the receipt, and decide. Proceeds on allow; warn only with allowWarn; require_confirmation only if confirm() says yes; block and unsupported never. */
     async guard(domain, action = "read", opts = {}) {
         const receipt = await this.preflight(domain, action, opts);
+        if (this.log.forks.length) { // the log has shown this client two contradicting histories: no answer from it is trusted for action
+            const verification = await this.verifyDocument(receipt, opts.onlineKeys ?? true);
+            return { proceed: false, receipt, verification, header: null, split_view: this.log.forks.slice() };
+        }
         const verification = await this.verifyDocument(receipt, opts.onlineKeys ?? true);
         if (!verification.accepted && (opts.onlineKeys ?? true))
             return { proceed: false, receipt, verification, header: null };
@@ -111,6 +173,12 @@ export class CrawlCheck {
         return { proceed, receipt, verification, header: proceed ? receiptHeader(receipt) : null };
     }
     /** Look a domain up without sending it: only the first prefixLen hex characters of sha256(domain) leave this process. found false with ready true means no public measurement. */
+    /** Make a signed lockfile for an MCP server (CrawlCheck reads tools/list; no tool is called). Save it; check with checkLock before every connect. */
+    async mcpLock(url) { return this.req("GET", "/api/v1/mcp/lock", { url }); }
+    /** Read the server's tools now (through CrawlCheck) and compare with a lockfile; the answer is signed. For a fully offline check, list the tools yourself and call checkLock. */
+    async mcpLockCheck(lock, url) { return this.req("POST", "/api/v1/mcp/lock/check", undefined, { lock, url: url ?? lock?.server?.url ?? undefined }); }
+    /** Tool-poisoning scan of a server URL or a tool list. */
+    async mcpScan(target) { return (typeof target === "string" ? this.req("GET", "/api/v1/mcp/scan", { url: target }) : this.req("POST", "/api/v1/mcp/scan", undefined, { tools: target })); }
     async resolvePrivate(domain, prefixLen = 2) {
         const d = normalizeDomain(domain);
         if (!d)

package/dist/log.d.ts

@@ -0,0 +1,141 @@
+/** A new, empty store. It is plain JSON: keep it across runs (a file, a KV value) so later heads are checked against earlier ones. */
+export function newLogStore(): {
+    v: number;
+    log_id: string;
+    heads: {};
+    forks: never[];
+    witnessed: null;
+};
+/**
+ * Check one signed tree head (kind crawlcheck-log-sth) against everything the store already holds, then keep it.
+ * Returns { state: "new" | "known" | "fork" | "invalid", why?, fork? }. invalid means the document is not a head
+ * CrawlCheck signed (it proves nothing either way); fork means two signed heads that contradict each other.
+ * opts.publishedKids: key ids from the published directory; a head signed by another key is invalid.
+ */
+export function logObserve(store: any, sth: any, opts?: {}): Promise<{
+    state: string;
+    fork: {
+        kind: any;
+        batch: any;
+        why: any;
+        a: any;
+        b: any;
+        detected_at: string;
+        proof: string;
+    };
+} | {
+    state: string;
+    why: string;
+} | {
+    state: string;
+    why?: undefined;
+}>;
+/** Observe the head inside a Resolve answer (transparency.sth), if it has one. */
+export function logObserveAnswer(store: any, answer: any, opts?: {}): Promise<{
+    state: string;
+    fork: {
+        kind: any;
+        batch: any;
+        why: any;
+        a: any;
+        b: any;
+        detected_at: string;
+        proof: string;
+    };
+} | {
+    state: string;
+    why: string;
+} | {
+    state: string;
+    why?: undefined;
+} | {
+    state: string;
+}>;
+/** The signed heads this store holds, newest first: what a client sends its peers. */
+export function logHeads(store: any, limit?: number): any[];
+/** Gossip: check heads another client holds against this store. Returns { forks, results }. */
+export function logGossip(store: any, sths: any, opts?: {}): Promise<{
+    forks: any;
+    results: (({
+        batch: any;
+        sha256: any;
+    } & {
+        state: string;
+        fork: {
+            kind: any;
+            batch: any;
+            why: any;
+            a: any;
+            b: any;
+            detected_at: string;
+            proof: string;
+        };
+    }) | ({
+        batch: any;
+        sha256: any;
+    } & {
+        state: string;
+        why: string;
+    }) | ({
+        batch: any;
+        sha256: any;
+    } & {
+        state: string;
+        why?: undefined;
+    }))[];
+}>;
+/**
+ * Verify a witness co-signature: the JWT's RS256 signature against GitHub's published keys (jwks: the JSON from
+ * https://token.actions.githubusercontent.com/.well-known/jwks), its issuer, repository, workflow and audience
+ * crawlcheck-sth:<sthSha256>. exp is not checked: the token is a record of what the witness saw when it ran.
+ * Returns { ok: true | false | null, why, claims? }; null when GitHub no longer publishes the key that signed it.
+ */
+export function verifyWitnessToken(token: any, sthSha256: any, jwks: any): Promise<{
+    ok: boolean;
+    why: string;
+    claims?: undefined;
+} | {
+    ok: null;
+    why: string;
+    claims: any;
+} | {
+    ok: boolean;
+    why: string;
+    claims: any;
+}>;
+/**
+ * Online check of a store against the log and its outside witness. get(path) must return parsed JSON from
+ * https://crawlcheck.io (the SDK clients pass their own). Steps: observe the current head; fetch the last witnessed
+ * head and verify GitHub's co-signature; walk the chain from the witnessed head down to the oldest head this store
+ * holds (at most opts.maxWalk batches), observing every link, so any head this client was shown that the witness's
+ * chain does not contain surfaces as a fork. Heads newer than the witnessed one are listed as unwitnessed.
+ */
+export function logCheck(store: any, get: any, opts?: {}): Promise<{
+    consistent: boolean;
+    forks: any;
+    all_forks: any;
+    witnessed: {
+        batch: any;
+        sha256: any;
+        at: any;
+        run_id: any;
+        token: boolean | null;
+        why: string;
+        observed: string;
+    } | null;
+    unwitnessed: {
+        batch: number;
+        sha256: any;
+        created_at: any;
+    }[];
+    stale_unwitnessed: {
+        batch: number;
+        sha256: any;
+        created_at: any;
+    }[];
+    summary: string;
+}>;
+export const LOG_ID: "crawlcheck-resolve-log-v1";
+export const GITHUB_OIDC_ISSUER: "https://token.actions.githubusercontent.com";
+export const WITNESS_REPOSITORY: "emmanuelorta/crawlcheck";
+export const WITNESS_WORKFLOW_PREFIX: "emmanuelorta/crawlcheck/.github/workflows/log-witness.yml@refs/heads/";