codecartographer-pi 0.24.1 → 0.26.0

package/dist/core/broadside/requests.js

@@ -0,0 +1,71 @@
+// Batch request building and cost estimation for a lens's slices.
+//
+// Split out of core/broadside.ts (#339); the barrel there re-exports every
+// name, so `core/index.ts` and the tests see one module as before.
+import { BROADSIDE_MODEL } from "./constants.js";
+import { defaultReasoningFor } from "./types.js";
+import { SCHEMAS } from "./schemas.js";
+import { sanitizeId } from "./repo.js";
+// ---------- request building ----------
+export function buildBatchRequest(lens, info, slice, index, sliceCount, model = BROADSIDE_MODEL, maxTokensOverride, reasoningOverride) {
+    const moduleTag = sanitizeId(slice.moduleName);
+    const customId = sliceCount > 1 ? `${lens.id}-${moduleTag}-${index + 1}` : `${lens.id}-${moduleTag}`;
+    return {
+        custom_id: customId,
+        body: {
+            model,
+            messages: [
+                { role: "system", content: lens.systemPrompt(info) },
+                {
+                    role: "user",
+                    content: 
+                    // A fallback scan is not "server source files": say what it is,
+                    // so the model judges the trust boundary wherever it appears
+                    // and does not report the missing server/ as a finding (#319).
+                    (slice.fallback
+                        ? `NOTE: this repository has no source files under the paths this lens usually reads (${slice.fallback}). ` +
+                            "What follows is every source file it has, after anything those paths did match; locate the trust boundary and the request-handling code wherever they live.\n\n"
+                        : "") + lens.userPrompt(info, slice.content, slice.moduleName),
+                },
+            ],
+            response_format: { type: "json_schema", json_schema: SCHEMAS[lens.schemaName] },
+            max_tokens: maxTokensOverride ?? lens.maxTokens,
+            // Always sent, never inherited: an absent field means the model's
+            // own default, and that default is what truncated the JSON.
+            reasoning: reasoningOverride ?? lens.reasoning ?? defaultReasoningFor(),
+        },
+    };
+}
+/**
+ * Pre-flight cost estimate for one lens.
+ *
+ * Every slice is its own batch request, so both halves scale with the slice
+ * count. The output half used to be a single `maxTokens * 0.75` for the whole
+ * lens no matter how many requests it sent — on a repository that sliced into
+ * 13 modules that budgeted one request's output and shipped thirteen, and a
+ * live run came in at roughly 3x its estimate. Since this number is what
+ * `max_cost` binds against, under-counting it lets a run outspend the cap the
+ * user set.
+ *
+ * @param info - Repo info, when the caller has it: lets the estimate include
+ *   the system prompt and JSON schema each request carries. Omitted, the
+ *   estimate covers slice content only, which is what the old signature did.
+ */
+export function estimateCost(lens, slices, pricing, maxTokensOverride, info) {
+    // With repo info, size each request from the user prompt that would be
+    // sent, which is what the architecture lens is made of: it used to be
+    // estimated at a flat 6,000 chars while the entry point, manifest, README
+    // excerpt and file tree it carries ran to whatever they ran to (#249).
+    // Without info, the slice content alone is what the old signature covered.
+    const sliceChars = slices.reduce((sum, s) => sum + (info ? lens.userPrompt(info, s.content, s.moduleName).length : lens.maxChars === 0 ? 6000 : s.chars), 0);
+    // The system prompt and the response schema ride on every request, so they
+    // are paid once per slice rather than once per lens.
+    const perRequestOverhead = info
+        ? (lens.systemPrompt(info)?.length ?? 0) + JSON.stringify(SCHEMAS[lens.schemaName] ?? {}).length
+        : 0;
+    const inputTokens = Math.ceil((sliceChars + perRequestOverhead * slices.length) / 4);
+    const outputTokens = slices.length * Math.ceil((maxTokensOverride ?? lens.maxTokens) * 0.75);
+    const cost = (inputTokens / 1_000_000) * pricing.inputPerM +
+        (outputTokens / 1_000_000) * pricing.outputPerM;
+    return { inputTokens, outputTokens, cost };
+}

package/dist/core/broadside/results.d.ts

@@ -0,0 +1,36 @@
+import { type BroadsideLensId } from "./constants.ts";
+import { type BatchRequest } from "./types.ts";
+export declare function renderFindingsMarkdown(content: string): string;
+export type StoredLensResult = {
+    lensId: BroadsideLensId;
+    customId: string;
+    moduleName: string;
+    content: string;
+    raw: Record<string, unknown>;
+    /** True when the content is not parseable JSON even after fence stripping —
+     * the telltale of an output cut off at max_tokens. */
+    truncated: boolean;
+};
+export declare function extractContent(result: Record<string, unknown>): string | null;
+/**
+ * Parse lens content as JSON, tolerating the markdown code fences some models
+ * wrap structured output in (the same tolerance OpenRouter's headless-agent
+ * scaffold ships for --output-schema). Returns null when the content is not
+ * JSON at all — which for a strict json_schema request means the output was
+ * truncated at max_tokens, not that the model chose prose.
+ */
+export declare function parseLensJson(content: string): unknown | null;
+export declare function saveLensResults(runDir: string, lensId: BroadsideLensId, batch: Record<string, unknown>): Promise<StoredLensResult[]>;
+/**
+ * Rebuild lens results from what a previous collect already wrote to disk.
+ *
+ * The post-passes are gated on having lens findings in hand, and a collect
+ * only holds the ones *it* polled. When an earlier collect saved every lens
+ * and then died before synthesis and triage ran — the batch window is long
+ * and a poll can easily be interrupted — the next collect finds every lens
+ * already terminal, skips them all, and would otherwise reach the post-pass
+ * gate with nothing to hand it. Reading the saved results back is what makes
+ * "a resumed collect can finish whichever is still pending" true.
+ */
+export declare function loadSavedLensResults(runDir: string, lenses: BroadsideLensId[]): Promise<StoredLensResult[]>;
+export declare function loadStoredRequests(runDir: string): Promise<Record<string, BatchRequest>>;

package/dist/core/broadside/results.js

@@ -0,0 +1,163 @@
+// Lens results as they come back and as they are stored: content extraction
+// from a batch result, JSON parsing that tolerates code fences, markdown
+// rendering, and the per-run files collect writes and later reads back.
+//
+// Split out of core/broadside.ts (#339); the barrel there re-exports every
+// name, so `core/index.ts` and the tests see one module as before.
+import { readFile, readdir, writeFile } from "node:fs/promises";
+import { join } from "node:path";
+import { pathExists } from "../utils.js";
+import { sanitizeId } from "./repo.js";
+export function renderFindingsMarkdown(content) {
+    const parsed = parseLensJson(content);
+    if (parsed === null)
+        return content;
+    return formatAsMarkdown(parsed);
+}
+function formatAsMarkdown(value, depth = 0) {
+    const indent = "\t".repeat(depth);
+    if (Array.isArray(value)) {
+        const lines = [];
+        for (let i = 0; i < value.length; i++) {
+            const item = value[i];
+            if (item && typeof item === "object") {
+                const title = (item.title ?? item.name ?? item.module ?? item.area ?? item.platform ?? "");
+                lines.push(`${indent}${i + 1}. ${title}`);
+                lines.push(formatAsMarkdown(item, depth + 1));
+            }
+            else {
+                lines.push(`${indent}- ${String(item)}`);
+            }
+        }
+        return lines.join("\n");
+    }
+    if (value && typeof value === "object") {
+        const lines = [];
+        for (const [key, entryValue] of Object.entries(value)) {
+            if (entryValue && typeof entryValue === "object") {
+                lines.push(`${indent}**${key}**:`);
+                lines.push(formatAsMarkdown(entryValue, depth + 1));
+            }
+            else {
+                lines.push(`${indent}- **${key}**: ${String(entryValue)}`);
+            }
+        }
+        return lines.join("\n");
+    }
+    return `${indent}${String(value)}`;
+}
+export function extractContent(result) {
+    const response = result.response;
+    if (!response?.body)
+        return null;
+    const body = response.body;
+    const choices = body.choices;
+    const message = choices?.[0]?.message;
+    return typeof message?.content === "string" ? message.content : null;
+}
+/**
+ * Parse lens content as JSON, tolerating the markdown code fences some models
+ * wrap structured output in (the same tolerance OpenRouter's headless-agent
+ * scaffold ships for --output-schema). Returns null when the content is not
+ * JSON at all — which for a strict json_schema request means the output was
+ * truncated at max_tokens, not that the model chose prose.
+ */
+export function parseLensJson(content) {
+    const trimmed = content.trim();
+    const fenced = /^```(?:json)?\s*\n?([\s\S]*?)\n?```\s*$/.exec(trimmed);
+    const candidate = fenced ? fenced[1].trim() : trimmed;
+    if (!candidate.startsWith("{") && !candidate.startsWith("["))
+        return null;
+    try {
+        return JSON.parse(candidate);
+    }
+    catch {
+        return null;
+    }
+}
+export async function saveLensResults(runDir, lensId, batch) {
+    const results = Array.isArray(batch.results) ? batch.results : [];
+    const out = [];
+    for (const result of results) {
+        const customId = String(result.custom_id ?? "unknown");
+        const content = extractContent(result);
+        if (content === null) {
+            if (result.error) {
+                await writeFile(join(runDir, `${sanitizeId(customId)}.error.json`), `${JSON.stringify(result.error, null, "\t")}\n`, "utf8");
+            }
+            continue;
+        }
+        const parsed = parseLensJson(content);
+        const truncated = parsed === null;
+        if (parsed !== null) {
+            await writeFile(join(runDir, `${sanitizeId(customId)}.json`), `${JSON.stringify(parsed, null, "\t")}\n`, "utf8");
+        }
+        else {
+            // Save the raw bytes verbatim so nothing is lost, but name the
+            // gap: an unparseable strict-schema response is a truncation.
+            await writeFile(join(runDir, `${sanitizeId(customId)}.json`), `${content}\n`, "utf8");
+        }
+        await writeFile(join(runDir, `${sanitizeId(customId)}.md`), renderFindingsMarkdown(content), "utf8");
+        out.push({
+            lensId,
+            customId,
+            moduleName: String(customId).replace(/^[a-z]+-/, ""),
+            content,
+            raw: result,
+            truncated,
+        });
+    }
+    return out;
+}
+/**
+ * Rebuild lens results from what a previous collect already wrote to disk.
+ *
+ * The post-passes are gated on having lens findings in hand, and a collect
+ * only holds the ones *it* polled. When an earlier collect saved every lens
+ * and then died before synthesis and triage ran — the batch window is long
+ * and a poll can easily be interrupted — the next collect finds every lens
+ * already terminal, skips them all, and would otherwise reach the post-pass
+ * gate with nothing to hand it. Reading the saved results back is what makes
+ * "a resumed collect can finish whichever is still pending" true.
+ */
+export async function loadSavedLensResults(runDir, lenses) {
+    if (!(await pathExists(runDir)))
+        return [];
+    const reserved = new Set(["requests.json", "run-meta.json", "synthesis.json", "triage.json"]);
+    const out = [];
+    // Longest lens id first: no id is a prefix of another today, but ordering
+    // keeps that from becoming a silent misattribution if one ever is.
+    const ordered = [...lenses].sort((a, b) => b.length - a.length);
+    for (const name of (await readdir(runDir)).sort()) {
+        if (!name.endsWith(".json") || name.endsWith(".error.json") || reserved.has(name) || name.startsWith("raw-"))
+            continue;
+        const customId = name.slice(0, -".json".length);
+        const lensId = ordered.find((id) => customId === id || customId.startsWith(`${id}-`));
+        if (!lensId)
+            continue;
+        const content = await readFile(join(runDir, name), "utf8").catch(() => null);
+        if (content === null)
+            continue;
+        out.push({
+            lensId,
+            customId,
+            moduleName: customId.replace(/^[a-z]+-/, ""),
+            content,
+            raw: {},
+            truncated: parseLensJson(content) === null,
+        });
+    }
+    return out;
+}
+export async function loadStoredRequests(runDir) {
+    const path = join(runDir, "requests.json");
+    if (!(await pathExists(path)))
+        return {};
+    try {
+        const parsed = JSON.parse(await readFile(path, "utf8"));
+        return parsed && typeof parsed === "object" ? parsed : {};
+    }
+    catch {
+        return {};
+    }
+}

package/dist/core/broadside/schemas.d.ts

@@ -0,0 +1,2 @@
+import { type JsonSchemaDef } from "./types.ts";
+export declare const SCHEMAS: Record<string, JsonSchemaDef>;

package/dist/core/broadside/schemas.js

@@ -0,0 +1,342 @@
+// The structured-output JSON schemas, one per lens plus synthesis and triage.
+//
+// Split out of core/broadside.ts (#339); the barrel there re-exports every
+// name, so `core/index.ts` and the tests see one module as before.
+// ---------- JSON schemas (one per lens, plus synthesis) ----------
+export const SCHEMAS = {
+    architecture: {
+        name: "architecture_report",
+        strict: true,
+        schema: {
+            type: "object",
+            properties: {
+                tech_stack: {
+                    type: "object",
+                    properties: {
+                        language: { type: "string" },
+                        version: { type: "string" },
+                        build_system: { type: "string" },
+                        key_dependencies: { type: "array", items: { type: "string" } },
+                    },
+                    required: ["language", "build_system"],
+                    additionalProperties: false,
+                },
+                module_architecture: {
+                    type: "array",
+                    items: {
+                        type: "object",
+                        properties: {
+                            name: { type: "string" },
+                            role: { type: "string" },
+                            file_count: { type: "integer" },
+                            depends_on: { type: "array", items: { type: "string" } },
+                        },
+                        required: ["name", "role"],
+                        additionalProperties: false,
+                    },
+                },
+                data_flow: { type: "string" },
+                entry_points: { type: "array", items: { type: "string" } },
+                notable_patterns: { type: "array", items: { type: "string" } },
+            },
+            required: ["tech_stack", "module_architecture", "data_flow", "entry_points"],
+            additionalProperties: false,
+        },
+    },
+    api_surface: {
+        name: "api_surface_report",
+        strict: true,
+        schema: {
+            type: "object",
+            properties: {
+                endpoints: {
+                    type: "array",
+                    items: {
+                        type: "object",
+                        properties: {
+                            method: { type: "string" },
+                            path: { type: "string" },
+                            handler: { type: "string" },
+                            auth_required: { type: "boolean" },
+                            description: { type: "string" },
+                        },
+                        required: ["method", "path", "handler", "auth_required"],
+                        additionalProperties: false,
+                    },
+                },
+                data_types: {
+                    type: "array",
+                    items: {
+                        type: "object",
+                        properties: {
+                            name: { type: "string" },
+                            kind: { type: "string" },
+                            fields_summary: { type: "string" },
+                        },
+                        required: ["name", "kind"],
+                        additionalProperties: false,
+                    },
+                },
+                authentication_flow: { type: "string" },
+                error_handling: { type: "string" },
+            },
+            required: ["endpoints"],
+            additionalProperties: false,
+        },
+    },
+    security: {
+        name: "security_review_report",
+        strict: true,
+        schema: {
+            type: "object",
+            properties: {
+                findings: {
+                    type: "array",
+                    items: {
+                        type: "object",
+                        properties: {
+                            severity: { type: "string", enum: ["critical", "high", "medium", "low"] },
+                            category: { type: "string" },
+                            title: { type: "string" },
+                            location: { type: "string" },
+                            description: { type: "string" },
+                        },
+                        required: ["severity", "title", "description"],
+                        additionalProperties: false,
+                    },
+                },
+                overall_assessment: { type: "string" },
+                coverage_note: { type: "string" },
+            },
+            required: ["findings", "overall_assessment"],
+            additionalProperties: false,
+        },
+    },
+    defect_mechanical: {
+        name: "defect_scan_report",
+        strict: true,
+        schema: {
+            type: "object",
+            properties: {
+                module: { type: "string" },
+                findings: {
+                    type: "array",
+                    items: {
+                        type: "object",
+                        properties: {
+                            severity: { type: "string", enum: ["high", "medium", "low"] },
+                            pattern: { type: "string" },
+                            title: { type: "string" },
+                            location: { type: "string" },
+                            description: { type: "string" },
+                            suggestion: { type: "string" },
+                        },
+                        required: ["severity", "pattern", "title", "description"],
+                        additionalProperties: false,
+                    },
+                },
+                patterns_checked: { type: "array", items: { type: "string" } },
+                files_scanned: { type: "integer" },
+                overall_notes: { type: "string" },
+            },
+            required: ["module", "findings", "patterns_checked", "files_scanned"],
+            additionalProperties: false,
+        },
+    },
+    conventions: {
+        name: "conventions_report",
+        strict: true,
+        schema: {
+            type: "object",
+            properties: {
+                module: { type: "string" },
+                naming_conventions: {
+                    type: "object",
+                    properties: {
+                        packages: { type: "string" },
+                        types: { type: "string" },
+                        functions: { type: "string" },
+                        variables: { type: "string" },
+                        files: { type: "string" },
+                        tests: { type: "string" },
+                    },
+                    additionalProperties: false,
+                },
+                error_handling_pattern: { type: "string" },
+                logging_approach: { type: "string" },
+                test_patterns: { type: "string" },
+                code_organization: { type: "string" },
+                idioms: { type: "array", items: { type: "string" } },
+                inconsistencies: {
+                    type: "array",
+                    items: {
+                        type: "object",
+                        properties: {
+                            description: { type: "string" },
+                            locations: { type: "array", items: { type: "string" } },
+                        },
+                        required: ["description"],
+                        additionalProperties: false,
+                    },
+                },
+                promotable_conventions: {
+                    type: "array",
+                    items: {
+                        type: "object",
+                        properties: {
+                            title: { type: "string" },
+                            rule: { type: "string" },
+                            evidence: { type: "string" },
+                        },
+                        required: ["title", "rule"],
+                        additionalProperties: false,
+                    },
+                },
+                files_scanned: { type: "integer" },
+            },
+            required: ["module", "naming_conventions", "files_scanned"],
+            additionalProperties: false,
+        },
+    },
+    porting: {
+        name: "porting_surface_report",
+        strict: true,
+        schema: {
+            type: "object",
+            properties: {
+                module: { type: "string" },
+                platform_coupling: {
+                    type: "array",
+                    items: {
+                        type: "object",
+                        properties: {
+                            platform: { type: "string" },
+                            mechanisms: { type: "array", items: { type: "string" } },
+                            files: { type: "array", items: { type: "string" } },
+                        },
+                        required: ["platform", "mechanisms"],
+                        additionalProperties: false,
+                    },
+                },
+                external_dependencies: {
+                    type: "array",
+                    items: {
+                        type: "object",
+                        properties: {
+                            name: { type: "string" },
+                            role: { type: "string" },
+                            replaceability: { type: "string" },
+                        },
+                        required: ["name"],
+                        additionalProperties: false,
+                    },
+                },
+                build_system_complexity: { type: "string" },
+                porting_risk_areas: {
+                    type: "array",
+                    items: {
+                        type: "object",
+                        properties: {
+                            area: { type: "string" },
+                            risk: { type: "string", enum: ["low", "medium", "high"] },
+                            notes: { type: "string" },
+                        },
+                        required: ["area", "risk"],
+                        additionalProperties: false,
+                    },
+                },
+                files_scanned: { type: "integer" },
+            },
+            required: ["module", "platform_coupling", "files_scanned"],
+            additionalProperties: false,
+        },
+    },
+    synthesis: {
+        name: "synthesis_report",
+        strict: true,
+        schema: {
+            type: "object",
+            properties: {
+                executive_summary: { type: "string" },
+                severity_summary: {
+                    type: "object",
+                    properties: {
+                        critical: { type: "integer" },
+                        high: { type: "integer" },
+                        medium: { type: "integer" },
+                        low: { type: "integer" },
+                    },
+                    required: ["critical", "high", "medium", "low"],
+                    additionalProperties: false,
+                },
+                top_findings: {
+                    type: "array",
+                    items: {
+                        type: "object",
+                        properties: {
+                            title: { type: "string" },
+                            severity: { type: "string" },
+                            source_lens: { type: "string" },
+                            summary: { type: "string" },
+                        },
+                        required: ["title", "severity", "source_lens", "summary"],
+                        additionalProperties: false,
+                    },
+                },
+                module_assessments: {
+                    type: "array",
+                    items: {
+                        type: "object",
+                        properties: {
+                            module: { type: "string" },
+                            quality_notes: { type: "string" },
+                            risk_level: { type: "string", enum: ["low", "medium", "high"] },
+                        },
+                        required: ["module", "risk_level"],
+                        additionalProperties: false,
+                    },
+                },
+                porting_readiness: { type: "string" },
+                gaps_and_unknowns: { type: "array", items: { type: "string" } },
+                coverage: { type: "string" },
+            },
+            required: ["executive_summary", "severity_summary", "top_findings"],
+            additionalProperties: false,
+        },
+    },
+    triage: {
+        name: "triage_report",
+        strict: true,
+        schema: {
+            type: "object",
+            properties: {
+                summary: { type: "string" },
+                items: {
+                    type: "array",
+                    items: {
+                        type: "object",
+                        properties: {
+                            title: { type: "string" },
+                            severity: { type: "string" },
+                            module: { type: "string" },
+                            impact: { type: "string", enum: ["high", "medium", "low"] },
+                            difficulty: { type: "string", enum: ["high", "medium", "low"] },
+                            priority: { type: "string" },
+                            effort_estimate: { type: "string" },
+                            rationale: { type: "string" },
+                        },
+                        required: ["title", "severity", "module", "impact", "difficulty", "priority", "rationale"],
+                        additionalProperties: false,
+                    },
+                },
+                omitted: {
+                    type: "array",
+                    items: { type: "string" },
+                    description: "Leads deliberately dropped from the queue and why (duplicates, too vague, out of scope)",
+                },
+            },
+            required: ["summary", "items"],
+            additionalProperties: false,
+        },
+    },
+};