codecartographer-pi 0.25.0 → 0.26.0

package/.codecarto/broadside/SKILL.md

@@ -222,7 +222,10 @@ is about the *presence* of a hardcoded credential at that location; the
 value was never sent. This is a safety net against an accidental upload
 with deliberately low-false-positive patterns, not a secret scanner:
 anything it does not recognise goes as written. `redact_secrets: false` in
-`config.yaml` turns the content pass off (the by-name skip stays).
+`config.yaml` turns the content pass off (the by-name skip stays). The same
+pass runs over every line `verify`'s read-only tools hand the model — a key in
+an ordinary source file is exactly what a finding points a verifier at — and
+the verify report says how many values it redacted.
 
 The `max_cost` guardrail is an **estimate-based pre-flight limit**, distinct
 from OpenRouter's runtime cost tracking: it predicts from file sizes before

package/.codecarto/workflow/scaffold-version.yaml

@@ -3,4 +3,4 @@
 # workspace's framework-owned files (GUIDE.md, templates/, workflow/ pipelines
 # and VALIDATE.md) predate the running release. Written at release time and
 # copied verbatim by init — never edit by hand.
-scaffold_version: 0.25.0
+scaffold_version: 0.26.0

package/README.md

@@ -20,7 +20,7 @@
 ```
 
 <p align="center">
-  <img src="docs/demo-dashboard-hero.png" alt="CodeCartographer dashboard — a mid-run pipeline with architecture and defect-scan phases complete, contracts in progress, per-phase token and tool-use telemetry.">
+  <img src="docs/demo-dashboard-hero.png" alt="CodeCartographer's dashboard after running the full deep-audit pipeline on its own repository — 7/7 phases complete, 50M tokens and 618 tool uses across 77 minutes, seven open questions and twelve post-pipeline items routed, no blocking artifact gaps.">
 </p>
 
 ---
@@ -305,7 +305,7 @@ Beyond the slash commands, the Pi extension layers on:
 
 **Per-phase usage tracking.** Each phase run is appended to `.codecarto/workflow/.usage.local.yaml`. `/codecarto-usage` reports cumulative + per-phase token, runtime, tool-use, and compaction totals, including threshold/overflow/manual triggers and successful/failed/aborted outcomes.
 
-**Tool interception.** `bash` is blocked outright; `edit` and `write` are confined to `.codecarto/`, plus the configured, marker-validated CodeCartographer library when one is configured. Same rules apply to phase sub-agents.
+**Tool interception.** `bash` is blocked outright; `edit` and `write` are confined to `.codecarto/`, plus the configured, marker-validated CodeCartographer library when one is configured. Phase sub-agents get the same hook over a narrower root — `.codecarto/` alone, since a phase writes findings and handoffs and never publishes.
 
 ### Slash commands
 

package/dist/core/broadside/client.d.ts

@@ -0,0 +1,56 @@
+import { type BroadsideLensId } from "./constants.ts";
+import { type BatchRequest } from "./types.ts";
+export type FetchLike = (url: string, init: Record<string, unknown>) => Promise<Response>;
+export declare function submitBatch(batchRequests: BatchRequest[], apiKey: string, fetcher?: FetchLike, model?: string): Promise<{
+    batchId: string;
+    status: string;
+    error?: unknown;
+}>;
+export declare function fetchBatch(batchId: string, apiKey: string, fetcher?: FetchLike): Promise<Record<string, unknown>>;
+export declare function pollBatchUntilTerminal(batchId: string, apiKey: string, opts?: {
+    deadlineMs?: number;
+    onStatus?: (status: string, counts: Record<string, unknown>) => void;
+    fetcher?: FetchLike;
+    pollIntervalMs?: number;
+    /**
+     * Stops polling early with the same synthetic `timeout` a spent budget
+     * returns: the batch keeps running server-side and a later collect
+     * claims it. The MCP server aborts when its client disconnects (#322).
+     */
+    signal?: AbortSignal;
+}): Promise<Record<string, unknown>>;
+/**
+ * Poll several batch ids in parallel against one shared deadline. Collect
+ * previously polled one lens at a time, so a slow first lens serialized the
+ * wall clock for lenses that had already finished server-side (#136). The
+ * onStatus callback identifies the lens so progress output stays readable
+ * even while the polls interleave.
+ */
+export declare function pollBatchesConcurrently(entries: Array<{
+    lensId: BroadsideLensId;
+    batchId: string;
+}>, apiKey: string, opts?: {
+    deadlineMs?: number;
+    fetcher?: FetchLike;
+    pollIntervalMs?: number;
+    signal?: AbortSignal;
+    onStatus?: (lensId: string, status: string, counts: Record<string, unknown>) => void;
+}): Promise<Map<string, Record<string, unknown>>>;
+export declare const NO_BATCH_ENDPOINT_RE: RegExp;
+/** The refusal for a full per-account concurrent batch-job quota. */
+export declare const BATCH_QUOTA_RE: RegExp;
+/** One line of a batch's error field, whatever shape the provider gave it. */
+export declare function describeBatchError(error: unknown): string | null;
+/**
+ * A provider refusal plus what to do about it, for the two refusals a batch
+ * run meets in practice and cannot fix by itself (#141):
+ *
+ * - `Model '<id>' does not have a :batch endpoint.` — the catalog advertises a
+ *   `:batch` id that OpenRouter runs no batch endpoint for. Nothing in the
+ *   catalog distinguishes these; the `models` action marks ids this
+ *   repository has seen refused.
+ * - `job-submission-count … in use: 16, quota: 16` — the per-account limit
+ *   on concurrent batch jobs. Broad-Side submits one job per lens, so a few
+ *   runs in flight on the same key fill it; the refusal costs nothing.
+ */
+export declare function explainBatchError(error: unknown): string | null;

package/dist/core/broadside/client.js

@@ -0,0 +1,200 @@
+// The OpenRouter Batch API client: submit, fetch, poll one or many batches to a terminal status.
+//
+// Split out of core/broadside.ts (#339); the barrel there re-exports every
+// name, so `core/index.ts` and the tests see one module as before.
+import { sleep } from "../utils.js";
+import { BROADSIDE_BATCH_URL, BROADSIDE_DEAD_BATCH_STATUSES, BROADSIDE_DEFAULT_POLL_BUDGET_MS, BROADSIDE_MODEL, BROADSIDE_POLL_INTERVAL_MS } from "./constants.js";
+export async function submitBatch(batchRequests, apiKey, fetcher = fetch, model = BROADSIDE_MODEL) {
+    // The OpenRouter batch endpoint stream-parses the body and requires
+    // `endpoint` and `model` to serialize before `requests` — key order matters.
+    const payload = {
+        endpoint: "/v1/chat/completions",
+        model,
+        requests: batchRequests,
+    };
+    const resp = await fetcher(BROADSIDE_BATCH_URL, {
+        method: "POST",
+        headers: {
+            Authorization: `Bearer ${apiKey}`,
+            "Content-Type": "application/json",
+        },
+        body: JSON.stringify(payload),
+        signal: AbortSignal.timeout(30_000),
+    });
+    const data = (await resp.json());
+    if (resp.status !== 202) {
+        return { batchId: "", status: "rejected", error: data };
+    }
+    return { batchId: String(data.id), status: String(data.status) };
+}
+export async function fetchBatch(batchId, apiKey, fetcher = fetch) {
+    const resp = await fetcher(`${BROADSIDE_BATCH_URL}/${batchId}`, {
+        method: "GET",
+        headers: { Authorization: `Bearer ${apiKey}` },
+        signal: AbortSignal.timeout(30_000),
+    });
+    let data;
+    try {
+        data = (await resp.json());
+    }
+    catch (error) {
+        // A gateway error page is not JSON. It used to throw out of here and
+        // be retried as if the network were down; keep the status instead.
+        data = { error: `non-JSON response (${error instanceof Error ? error.message : String(error)})` };
+    }
+    if (!data || typeof data !== "object")
+        data = { error: "empty response" };
+    // Surface the HTTP status so the poller can bail fast on auth expiry
+    // instead of retrying a dead key for the whole budget.
+    data.http_status = resp.status;
+    return data;
+}
+export async function pollBatchUntilTerminal(batchId, apiKey, opts = {}) {
+    const deadline = Date.now() + (opts.deadlineMs ?? BROADSIDE_DEFAULT_POLL_BUDGET_MS);
+    const intervalMs = opts.pollIntervalMs ?? BROADSIDE_POLL_INTERVAL_MS;
+    const fetcher = opts.fetcher ?? fetch;
+    // A poll that runs out of budget without one good response is not a slow
+    // batch. The last thing that went wrong rides on the timeout so the report
+    // can tell a dead network or a failing gateway from a batch still running.
+    let lastError = null;
+    let sawBatch = false;
+    const timedOut = () => ({
+        id: batchId,
+        status: "timeout",
+        ...(lastError && !sawBatch && { error: `no successful poll response; last error: ${lastError}` }),
+        ...(lastError && sawBatch && { last_error: lastError }),
+    });
+    for (;;) {
+        if (opts.signal?.aborted)
+            return { ...timedOut(), aborted: true };
+        let batch;
+        try {
+            batch = await fetchBatch(batchId, apiKey, fetcher);
+        }
+        catch (error) {
+            lastError = `fetch failed (${error instanceof Error ? error.message : String(error)})`;
+            if (Date.now() >= deadline)
+                return timedOut();
+            await sleep(intervalMs);
+            continue;
+        }
+        const httpStatus = Number(batch.http_status ?? 200);
+        if (httpStatus === 401 || httpStatus === 403) {
+            return { id: batchId, status: "auth-failed", error: batch.error ?? batch };
+        }
+        if (httpStatus >= 400) {
+            // A gateway or server error: retry within the budget, remembered.
+            const detail = typeof batch.error === "string" ? batch.error : JSON.stringify(batch.error ?? "");
+            lastError = `HTTP ${httpStatus}${detail ? ` (${detail.slice(0, 200)})` : ""}`;
+            if (Date.now() >= deadline)
+                return timedOut();
+            await sleep(intervalMs);
+            continue;
+        }
+        sawBatch = true;
+        const status = String(batch.status ?? "unknown");
+        const counts = (batch.request_counts ?? {});
+        opts.onStatus?.(status, counts);
+        if (status === "completed" || BROADSIDE_DEAD_BATCH_STATUSES.includes(status))
+            return batch;
+        if (Date.now() >= deadline)
+            return timedOut();
+        await sleepUnlessAborted(intervalMs, opts.signal);
+    }
+}
+/** Sleep, but wake at once when the signal fires so an abort is not a poll interval late. */
+function sleepUnlessAborted(ms, signal) {
+    if (!signal)
+        return sleep(ms);
+    if (signal.aborted)
+        return Promise.resolve();
+    return new Promise((resolve) => {
+        const timer = setTimeout(() => {
+            signal.removeEventListener("abort", onAbort);
+            resolve();
+        }, ms);
+        const onAbort = () => {
+            clearTimeout(timer);
+            resolve();
+        };
+        signal.addEventListener("abort", onAbort, { once: true });
+    });
+}
+/**
+ * Poll several batch ids in parallel against one shared deadline. Collect
+ * previously polled one lens at a time, so a slow first lens serialized the
+ * wall clock for lenses that had already finished server-side (#136). The
+ * onStatus callback identifies the lens so progress output stays readable
+ * even while the polls interleave.
+ */
+export async function pollBatchesConcurrently(entries, apiKey, opts = {}) {
+    const results = new Map();
+    const deadlineMs = opts.deadlineMs ?? BROADSIDE_DEFAULT_POLL_BUDGET_MS;
+    await Promise.all(entries.map(async ({ lensId, batchId }) => {
+        const batch = await pollBatchUntilTerminal(batchId, apiKey, {
+            deadlineMs,
+            fetcher: opts.fetcher,
+            pollIntervalMs: opts.pollIntervalMs,
+            signal: opts.signal,
+            onStatus: (status, counts) => opts.onStatus?.(lensId, status, counts),
+        });
+        results.set(batchId, batch);
+    }));
+    return results;
+}
+// ---------- provider errors ----------
+export const NO_BATCH_ENDPOINT_RE = /does not have a :batch endpoint/i;
+/** The refusal for a full per-account concurrent batch-job quota. */
+export const BATCH_QUOTA_RE = /job-submission-count/i;
+/** One line of a batch's error field, whatever shape the provider gave it. */
+export function describeBatchError(error) {
+    if (error === undefined || error === null || error === "")
+        return null;
+    if (typeof error === "string")
+        return error.slice(0, 300);
+    if (typeof error === "object") {
+        const message = error.message;
+        if (typeof message === "string" && message)
+            return message.slice(0, 300);
+        // OpenRouter wraps a submit refusal as `{ error: { message } }`.
+        const nested = error.error;
+        if (nested && typeof nested === "object") {
+            const inner = nested.message;
+            if (typeof inner === "string" && inner)
+                return inner.slice(0, 300);
+        }
+        if (typeof nested === "string" && nested)
+            return nested.slice(0, 300);
+        try {
+            return JSON.stringify(error).slice(0, 300);
+        }
+        catch {
+            return String(error);
+        }
+    }
+    return String(error);
+}
+/**
+ * A provider refusal plus what to do about it, for the two refusals a batch
+ * run meets in practice and cannot fix by itself (#141):
+ *
+ * - `Model '<id>' does not have a :batch endpoint.` — the catalog advertises a
+ *   `:batch` id that OpenRouter runs no batch endpoint for. Nothing in the
+ *   catalog distinguishes these; the `models` action marks ids this
+ *   repository has seen refused.
+ * - `job-submission-count … in use: 16, quota: 16` — the per-account limit
+ *   on concurrent batch jobs. Broad-Side submits one job per lens, so a few
+ *   runs in flight on the same key fill it; the refusal costs nothing.
+ */
+export function explainBatchError(error) {
+    const message = describeBatchError(error);
+    if (!message)
+        return null;
+    if (NO_BATCH_ENDPOINT_RE.test(message)) {
+        return `${message} — the catalog lists this id, but OpenRouter runs no batch endpoint for it. Nothing was charged; pick another model (the models action marks ids this repository has seen refused).`;
+    }
+    if (BATCH_QUOTA_RE.test(message)) {
+        return `${message} — OpenRouter's per-account limit on concurrent batch jobs is full. Broad-Side submits one job per lens, so a few runs in flight on this key (in any repository) fill it. Nothing was charged; collect or wait out the runs in flight, then re-submit.`;
+    }
+    return message;
+}

package/dist/core/broadside/collect.d.ts

@@ -0,0 +1,68 @@
+import { type BroadsideCollectResult, type BroadsideStateFile } from "./types.ts";
+import { type FetchLike } from "./client.ts";
+/**
+ * One verdict from a run's `verified.json` (written by the verify pass in
+ * `verify.ts`), reduced to what the post-passes are told.
+ */
+export type PostPassVerdict = {
+    lensId: string;
+    customId: string;
+    severity: string;
+    title: string;
+    location: string;
+    verdict: string;
+    confidence: string;
+    evidence: Array<{
+        file: string;
+        lines: string;
+        note: string;
+    }>;
+    reasoning: string;
+};
+/**
+ * The verdicts a verify pass left in the run directory, or null when none
+ * has run (#338). A file that does not parse is treated as absent: the
+ * post-passes then run from the findings alone, which is what they did
+ * before verdicts existed, and `status` shows the pass carried no verdicts.
+ */
+export declare function loadPostPassVerdicts(runDir: string): Promise<PostPassVerdict[] | null>;
+/**
+ * The verdicts as a section of the post-pass user message: one line per
+ * finding with the verdict, the evidence the verifier cited, and its
+ * reasoning, so the pass can rank on them rather than on the batch model's
+ * own severities (#338).
+ */
+export declare function renderPostPassVerdicts(verdicts: PostPassVerdict[]): string;
+export declare function runBroadsideCollect(cwd: string, apiKey: string, opts?: {
+    waitMs?: number;
+    includeSynthesis?: boolean;
+    includeTriage?: boolean;
+    /** Re-submit truncated slices once with a doubled output cap (#133). */
+    retryTruncated?: boolean;
+    onStatus?: (lensId: string, status: string, counts: Record<string, unknown>) => void;
+    fetcher?: FetchLike;
+    /**
+     * Which run to collect. Absent, the most recent — which used to be the
+     * only choice, so an older run still in flight could not be collected
+     * once a newer submit existed (#268). `status` lists the ids.
+     */
+    runId?: string;
+    /**
+     * Stops polling and submits nothing further once fired; what was
+     * already submitted keeps running server-side for a later collect to
+     * claim. The MCP server fires it when its client disconnects (#322).
+     */
+    signal?: AbortSignal;
+    /** Poll cadence override; tests drive the loop faster than 15 s. */
+    pollIntervalMs?: number;
+    /**
+     * Reset the wanted post-passes of a collected run and run them again
+     * (#338) — after a `verify`, so the executive report and the work order
+     * are built from the verdicts. A pass still in flight is left to finish;
+     * a run whose lens batches are still running is refused.
+     */
+    regeneratePostPasses?: boolean;
+}): Promise<BroadsideCollectResult>;
+export declare function runBroadsideStatus(cwd: string): Promise<{
+    state: BroadsideStateFile;
+}>;