codex-plugin-doctor 1.46.1 → 1.48.0

package/dist/core/corpus-quality-metrics.js

@@ -0,0 +1,433 @@
+import { stat } from "node:fs/promises";
+import path from "node:path";
+import { buildGenericMcpDoctor } from "../mcp/generic-mcp-doctor.js";
+import { packageVersion } from "../version.js";
+import { buildPackageFingerprint } from "./attestation.js";
+import { readJsonFile } from "./read-json-file.js";
+import { validatePlugin } from "./validate-plugin.js";
+export class CorpusMetricsManifestError extends Error {
+    constructor(message) {
+        super(message);
+        this.name = "CorpusMetricsManifestError";
+    }
+}
+const manifestKeys = new Set(["schemaVersion", "thresholds", "targets"]);
+const thresholdKeys = new Set(["minPrecision", "minRecall", "maxFalsePositiveRate"]);
+const targetKeys = new Set([
+    "id", "profile", "sourceType", "disclosure", "path", "mode",
+    "contentDigest", "source", "reviews"
+]);
+const sourceKeys = new Set(["repository", "revision"]);
+const reviewKeys = new Set(["findingId", "fingerprint", "classification"]);
+const profiles = new Set(["healthy", "broken", "edge-case"]);
+const sourceTypes = new Set([
+    "public-package", "local-snapshot", "derived-fixture"
+]);
+const modes = new Set(["codex-plugin", "generic-mcp"]);
+const classifications = new Set([
+    "true_positive", "false_positive", "unclear"
+]);
+const sha256Digest = /^sha256:[a-f0-9]{64}$/;
+const fingerprintPattern = /^[a-f0-9]{64}$/;
+const immutableRevision = /^[a-f0-9]{40}([a-f0-9]{24})?$/;
+const publicSafeId = /^[a-z0-9][a-z0-9._-]{0,63}$/;
+function isRecord(value) {
+    return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+function rejectUnknownKeys(value, allowed, label) {
+    const unknown = Object.keys(value).filter((key) => !allowed.has(key));
+    if (unknown.length > 0) {
+        throw new CorpusMetricsManifestError(`${label} contains unsupported field${unknown.length === 1 ? "" : "s"}: ${unknown.join(", ")}.`);
+    }
+}
+function requireString(value, label) {
+    if (typeof value !== "string" || value.length === 0) {
+        throw new CorpusMetricsManifestError(`${label} must be a non-empty string.`);
+    }
+    return value;
+}
+function parseThresholds(value) {
+    if (value === undefined)
+        return undefined;
+    if (!isRecord(value))
+        throw new CorpusMetricsManifestError("Metrics thresholds must be an object.");
+    rejectUnknownKeys(value, thresholdKeys, "Metrics thresholds");
+    const result = {};
+    for (const key of thresholdKeys) {
+        const candidate = value[key];
+        if (candidate === undefined)
+            continue;
+        if (typeof candidate !== "number" || !Number.isFinite(candidate) || candidate < 0 || candidate > 1) {
+            throw new CorpusMetricsManifestError(`Metrics threshold ${key} must be between 0 and 1.`);
+        }
+        result[key] = candidate;
+    }
+    return result;
+}
+function parseSource(value, targetId) {
+    if (value === undefined)
+        return undefined;
+    if (!isRecord(value))
+        throw new CorpusMetricsManifestError(`Metrics target ${targetId} source must be an object.`);
+    rejectUnknownKeys(value, sourceKeys, `Metrics target ${targetId} source`);
+    const repository = requireString(value.repository, `Metrics target ${targetId} source repository`);
+    const revision = requireString(value.revision, `Metrics target ${targetId} source revision`);
+    let parsed;
+    try {
+        parsed = new URL(repository);
+    }
+    catch {
+        throw new CorpusMetricsManifestError(`Metrics target ${targetId} source repository must use HTTPS.`);
+    }
+    if (parsed.protocol !== "https:" || parsed.username || parsed.password || parsed.search || parsed.hash) {
+        throw new CorpusMetricsManifestError(`Metrics target ${targetId} source repository must use HTTPS.`);
+    }
+    if (!immutableRevision.test(revision)) {
+        throw new CorpusMetricsManifestError(`Metrics target ${targetId} source revision must be immutable.`);
+    }
+    return { repository, revision };
+}
+function parseReview(value, targetId, index) {
+    if (!isRecord(value))
+        throw new CorpusMetricsManifestError(`Metrics target ${targetId} review ${index} must be an object.`);
+    rejectUnknownKeys(value, reviewKeys, `Metrics target ${targetId} review ${index}`);
+    const findingId = requireString(value.findingId, `Metrics target ${targetId} review ${index} findingId`);
+    const fingerprint = requireString(value.fingerprint, `Metrics target ${targetId} review ${index} fingerprint`);
+    if (!fingerprintPattern.test(fingerprint)) {
+        throw new CorpusMetricsManifestError(`Metrics target ${targetId} review ${index} fingerprint must be a sha256 fingerprint.`);
+    }
+    if (typeof value.classification !== "string" ||
+        !classifications.has(value.classification)) {
+        throw new CorpusMetricsManifestError(`Metrics target ${targetId} review ${index} has an unsupported classification.`);
+    }
+    return { findingId, fingerprint, classification: value.classification };
+}
+function resolveContainedTarget(manifestDirectory, targetPath) {
+    if (path.posix.isAbsolute(targetPath) || path.win32.isAbsolute(targetPath)) {
+        throw new CorpusMetricsManifestError("Metrics target path must be relative to the manifest file.");
+    }
+    const root = path.resolve(manifestDirectory);
+    const resolved = path.resolve(root, targetPath);
+    const relative = path.relative(root, resolved);
+    if (relative === ".." || relative.startsWith(`..${path.sep}`) || path.isAbsolute(relative)) {
+        throw new CorpusMetricsManifestError("Metrics target path must remain beneath the manifest directory.");
+    }
+    return resolved;
+}
+async function parseTarget(value, manifestDirectory, index) {
+    if (!isRecord(value))
+        throw new CorpusMetricsManifestError(`Metrics target ${index} must be an object.`);
+    rejectUnknownKeys(value, targetKeys, `Metrics target ${index}`);
+    const id = requireString(value.id, `Metrics target ${index} id`);
+    if (!publicSafeId.test(id)) {
+        throw new CorpusMetricsManifestError(`Metrics target ${index} id must be public-safe.`);
+    }
+    const targetPath = requireString(value.path, `Metrics target ${id} path`);
+    if (typeof value.profile !== "string" || !profiles.has(value.profile)) {
+        throw new CorpusMetricsManifestError(`Metrics target ${id} has an unsupported profile.`);
+    }
+    if (typeof value.sourceType !== "string" || !sourceTypes.has(value.sourceType)) {
+        throw new CorpusMetricsManifestError(`Metrics target ${id} has an unsupported sourceType.`);
+    }
+    if (value.disclosure !== "anonymized") {
+        throw new CorpusMetricsManifestError(`Metrics target ${id} disclosure must be anonymized.`);
+    }
+    if (typeof value.mode !== "string" || !modes.has(value.mode)) {
+        throw new CorpusMetricsManifestError(`Metrics target ${id} has an unsupported mode.`);
+    }
+    if (typeof value.contentDigest !== "string" || !sha256Digest.test(value.contentDigest)) {
+        throw new CorpusMetricsManifestError(`Metrics target ${id} contentDigest must be a sha256 digest.`);
+    }
+    if (!Array.isArray(value.reviews)) {
+        throw new CorpusMetricsManifestError(`Metrics target ${id} reviews must be an array.`);
+    }
+    const reviews = value.reviews.map((review, reviewIndex) => parseReview(review, id, reviewIndex));
+    const reviewKeysSeen = new Set();
+    for (const review of reviews) {
+        const key = findingKey(review.findingId, review.fingerprint);
+        if (reviewKeysSeen.has(key)) {
+            throw new CorpusMetricsManifestError(`Metrics target ${id} contains a duplicate finding review.`);
+        }
+        reviewKeysSeen.add(key);
+    }
+    const resolvedPath = resolveContainedTarget(manifestDirectory, targetPath);
+    try {
+        if (!(await stat(resolvedPath)).isDirectory())
+            throw new Error("not a directory");
+    }
+    catch {
+        throw new CorpusMetricsManifestError(`Metrics target ${id} directory does not exist.`);
+    }
+    const source = parseSource(value.source, id);
+    return {
+        id,
+        profile: value.profile,
+        sourceType: value.sourceType,
+        disclosure: "anonymized",
+        path: targetPath,
+        mode: value.mode,
+        contentDigest: value.contentDigest,
+        ...(source ? { source } : {}),
+        reviews,
+        resolvedPath
+    };
+}
+export async function loadCorpusMetricsManifest(manifestPath) {
+    let value;
+    try {
+        value = await readJsonFile(path.resolve(manifestPath));
+    }
+    catch {
+        throw new CorpusMetricsManifestError("Metrics manifest could not be read as JSON.");
+    }
+    if (!isRecord(value))
+        throw new CorpusMetricsManifestError("Metrics manifest must be an object.");
+    rejectUnknownKeys(value, manifestKeys, "Metrics manifest");
+    if (value.schemaVersion !== "1.0.0") {
+        throw new CorpusMetricsManifestError("Unsupported metrics manifest schemaVersion.");
+    }
+    if (!Array.isArray(value.targets) || value.targets.length === 0) {
+        throw new CorpusMetricsManifestError("Metrics manifest targets must be a non-empty array.");
+    }
+    const manifestDirectory = path.dirname(path.resolve(manifestPath));
+    const targets = await Promise.all(value.targets.map((target, index) => parseTarget(target, manifestDirectory, index)));
+    const ids = new Set();
+    for (const target of targets) {
+        if (ids.has(target.id))
+            throw new CorpusMetricsManifestError(`Metrics target id ${target.id} is duplicated.`);
+        ids.add(target.id);
+    }
+    const thresholds = parseThresholds(value.thresholds);
+    return { schemaVersion: "1.0.0", ...(thresholds ? { thresholds } : {}), targets };
+}
+function findingKey(findingId, fingerprint) {
+    return `${findingId}:${fingerprint}`;
+}
+function roundMetric(value) {
+    return value === null ? null : Math.round(value * 1_000_000) / 1_000_000;
+}
+function calculateRawMetrics(counts) {
+    const precisionDenominator = counts.truePositives + counts.falsePositives;
+    const recallDenominator = counts.truePositives + counts.falseNegatives;
+    return {
+        precision: precisionDenominator === 0 ? null : counts.truePositives / precisionDenominator,
+        recall: recallDenominator === 0 ? null : counts.truePositives / recallDenominator,
+        falsePositiveRate: precisionDenominator === 0 ? null : counts.falsePositives / precisionDenominator
+    };
+}
+function calculateMetrics(counts) {
+    const raw = calculateRawMetrics(counts);
+    return {
+        precision: roundMetric(raw.precision),
+        recall: roundMetric(raw.recall),
+        falsePositiveRate: roundMetric(raw.falsePositiveRate)
+    };
+}
+export function reconcileCorpusMetricFindings(reviews, actualFindings) {
+    const reviewsByKey = new Map(reviews.map((review) => [findingKey(review.findingId, review.fingerprint), review]));
+    const actualKeys = new Set(actualFindings.map((finding) => findingKey(finding.findingId, finding.fingerprint)));
+    const outcomes = actualFindings.map((finding) => ({
+        ...finding,
+        classification: reviewsByKey.get(findingKey(finding.findingId, finding.fingerprint))?.classification ?? "unreviewed",
+        emitted: true
+    }));
+    for (const review of reviews) {
+        if (!actualKeys.has(findingKey(review.findingId, review.fingerprint))) {
+            outcomes.push({ ...review, emitted: false });
+        }
+    }
+    outcomes.sort((left, right) => findingKey(left.findingId, left.fingerprint)
+        .localeCompare(findingKey(right.findingId, right.fingerprint)));
+    const counts = {
+        truePositives: outcomes.filter((item) => item.emitted && item.classification === "true_positive").length,
+        falsePositives: outcomes.filter((item) => item.emitted && item.classification === "false_positive").length,
+        falseNegatives: outcomes.filter((item) => !item.emitted && item.classification === "true_positive").length,
+        resolvedFalsePositives: outcomes.filter((item) => !item.emitted && item.classification === "false_positive").length,
+        unreviewed: outcomes.filter((item) => item.emitted && item.classification === "unreviewed").length,
+        unclear: outcomes.filter((item) => item.classification === "unclear").length
+    };
+    return {
+        counts,
+        metrics: calculateMetrics(counts),
+        complete: counts.unreviewed === 0 && counts.unclear === 0,
+        outcomes
+    };
+}
+function addCounts(left, right) {
+    return {
+        truePositives: left.truePositives + right.truePositives,
+        falsePositives: left.falsePositives + right.falsePositives,
+        falseNegatives: left.falseNegatives + right.falseNegatives,
+        resolvedFalsePositives: left.resolvedFalsePositives + right.resolvedFalsePositives,
+        unreviewed: left.unreviewed + right.unreviewed,
+        unclear: left.unclear + right.unclear
+    };
+}
+async function defaultAnalyzeTarget(targetPath, mode, environment) {
+    const report = mode === "generic-mcp"
+        ? await buildGenericMcpDoctor(targetPath, environment)
+        : await validatePlugin(targetPath);
+    return {
+        findings: report.findings.map((finding) => {
+            if (!finding.fingerprint) {
+                throw new CorpusMetricsManifestError("Metrics analysis produced a finding without a fingerprint.");
+            }
+            return { findingId: finding.id, fingerprint: finding.fingerprint };
+        })
+    };
+}
+function buildThresholdChecks(metrics, thresholds) {
+    const definitions = [
+        ["minPrecision", "precision", ">="],
+        ["minRecall", "recall", ">="],
+        ["maxFalsePositiveRate", "falsePositiveRate", "<="]
+    ];
+    return definitions.flatMap(([thresholdKey, metric, operator]) => {
+        const threshold = thresholds[thresholdKey];
+        if (threshold === undefined)
+            return [];
+        const actual = metrics[metric];
+        const passed = actual !== null && (operator === ">=" ? actual >= threshold : actual <= threshold);
+        return [{ metric, operator, threshold, actual, passed }];
+    });
+}
+export async function buildCorpusQualityMetricsReport(manifestPath, options = {}) {
+    const manifest = await loadCorpusMetricsManifest(manifestPath);
+    const thresholds = { ...(manifest.thresholds ?? {}), ...(options.thresholds ?? {}) };
+    const analyzeTarget = options.analyzeTarget ?? ((targetPath, mode) => defaultAnalyzeTarget(targetPath, mode, options.environment));
+    const fingerprintBuilder = options.buildFingerprint ?? buildPackageFingerprint;
+    const targets = [];
+    for (const target of manifest.targets) {
+        const [analysis, fingerprint] = await Promise.all([
+            analyzeTarget(target.resolvedPath, target.mode),
+            fingerprintBuilder(target.resolvedPath)
+        ]);
+        const reconciliation = reconcileCorpusMetricFindings(target.reviews, analysis.findings);
+        const digestMatched = fingerprint.digest === target.contentDigest;
+        targets.push({
+            id: target.id,
+            profile: target.profile,
+            sourceType: target.sourceType,
+            disclosure: target.disclosure,
+            mode: target.mode,
+            ...(target.source ? { source: target.source } : {}),
+            digestMatched,
+            counts: reconciliation.counts,
+            metrics: reconciliation.metrics,
+            complete: reconciliation.complete && digestMatched,
+            outcomes: reconciliation.outcomes
+        });
+    }
+    const emptyCounts = {
+        truePositives: 0, falsePositives: 0, falseNegatives: 0,
+        resolvedFalsePositives: 0, unreviewed: 0, unclear: 0
+    };
+    const counts = targets.reduce((total, target) => addCounts(total, target.counts), emptyCounts);
+    const metrics = calculateMetrics(counts);
+    const rawMetrics = calculateRawMetrics(counts);
+    const completeTargets = targets.filter((target) => target.complete).length;
+    const thresholdChecks = buildThresholdChecks(rawMetrics, thresholds).map((check) => ({
+        ...check,
+        actual: metrics[check.metric]
+    }));
+    const incomplete = completeTargets !== targets.length;
+    const failedThreshold = thresholdChecks.some((check) => !check.passed);
+    const status = incomplete ? "incomplete" : failedThreshold ? "fail" : "pass";
+    const exitCode = incomplete ? 2 : failedThreshold ? 1 : 0;
+    return {
+        schemaVersion: "1.0.0",
+        kind: "doctor.validation.corpus.metrics",
+        generatedAt: new Date().toISOString(),
+        version: packageVersion,
+        status,
+        exitCode,
+        summary: {
+            targetCount: targets.length,
+            completeTargets,
+            incompleteTargets: targets.length - completeTargets,
+            ...counts,
+            ...metrics
+        },
+        thresholds,
+        thresholdChecks,
+        targets
+    };
+}
+function formatMetric(value) {
+    return value === null ? "N/A" : `${(value * 100).toFixed(2)}%`;
+}
+function metricLabel(metric) {
+    if (metric === "falsePositiveRate")
+        return "False-positive rate";
+    return metric[0].toUpperCase() + metric.slice(1);
+}
+export function renderCorpusQualityMetricsJson(report) {
+    return JSON.stringify(report, null, 2);
+}
+export function renderCorpusQualityMetricsText(report, options = {}) {
+    const lines = [
+        "Doctor Corpus Quality Metrics",
+        "=============================",
+        `Version: ${report.version}`,
+        `Status: ${report.status.toUpperCase()}`,
+        `Targets: ${report.summary.targetCount}`,
+        `Precision: ${formatMetric(report.summary.precision)}`,
+        `Recall: ${formatMetric(report.summary.recall)}`,
+        `False-positive rate: ${formatMetric(report.summary.falsePositiveRate)}`,
+        `TP: ${report.summary.truePositives}  FP: ${report.summary.falsePositives}  FN: ${report.summary.falseNegatives}`
+    ];
+    if (options.outputPath)
+        lines.push(`Output: ${options.outputPath}`);
+    const failed = report.thresholdChecks.filter((check) => !check.passed);
+    lines.push("", "Failed thresholds", "-----------------");
+    if (failed.length === 0) {
+        lines.push("None.");
+    }
+    else {
+        for (const check of failed) {
+            lines.push(`${metricLabel(check.metric)}: ${formatMetric(check.actual)} ${check.operator} ${formatMetric(check.threshold)} required`);
+        }
+    }
+    lines.push("", "Targets", "-------");
+    for (const target of report.targets) {
+        lines.push(`${target.id}: ${target.complete ? "COMPLETE" : "INCOMPLETE"} ` +
+            `(precision ${formatMetric(target.metrics.precision)}, recall ${formatMetric(target.metrics.recall)})`);
+    }
+    return lines.join("\n");
+}
+export function renderCorpusQualityMetricsMarkdown(report) {
+    const lines = [
+        "# Doctor Corpus Quality Metrics",
+        "",
+        `- Version: ${report.version}`,
+        `- Status: ${report.status.toUpperCase()}`,
+        `- Targets: ${report.summary.targetCount}`,
+        "",
+        "| Metric | Value |",
+        "| --- | ---: |",
+        `| Precision | ${formatMetric(report.summary.precision)} |`,
+        `| Recall | ${formatMetric(report.summary.recall)} |`,
+        `| False-positive rate | ${formatMetric(report.summary.falsePositiveRate)} |`,
+        `| True positives | ${report.summary.truePositives} |`,
+        `| False positives | ${report.summary.falsePositives} |`,
+        `| False negatives | ${report.summary.falseNegatives} |`,
+        "",
+        "## Failed Thresholds",
+        ""
+    ];
+    const failed = report.thresholdChecks.filter((check) => !check.passed);
+    if (failed.length === 0) {
+        lines.push("None.");
+    }
+    else {
+        for (const check of failed) {
+            lines.push(`- ${metricLabel(check.metric)}: ${formatMetric(check.actual)} ${check.operator} ${formatMetric(check.threshold)} required`);
+        }
+    }
+    lines.push("", "## Targets", "", "| Target | Review | Precision | Recall |", "| --- | --- | ---: | ---: |");
+    for (const target of report.targets) {
+        lines.push(`| ${target.id} | ${target.complete ? "complete" : "incomplete"} | ` +
+            `${formatMetric(target.metrics.precision)} | ${formatMetric(target.metrics.recall)} |`);
+    }
+    return lines.join("\n");
+}

package/dist/core/external-validation-corpus.d.ts

@@ -0,0 +1,88 @@
+export type ExternalCorpusProfile = "healthy" | "broken" | "edge-case";
+export type ExternalCorpusSourceType = "public-package" | "local-snapshot" | "derived-fixture";
+export type ExternalCorpusMode = "codex-plugin" | "generic-mcp";
+export type FindingReviewClassification = "true_positive" | "false_positive" | "unclear";
+export interface ExternalCorpusFindingReview {
+    findingId: string;
+    fingerprint: string;
+    classification: FindingReviewClassification;
+}
+export interface ExternalCorpusTarget {
+    id: string;
+    profile: ExternalCorpusProfile;
+    sourceType: ExternalCorpusSourceType;
+    disclosure: "anonymized";
+    path: string;
+    mode: ExternalCorpusMode;
+    contentDigest: string;
+    expectedStatus: "pass" | "warn" | "fail";
+    reviews: ExternalCorpusFindingReview[];
+}
+export interface ExternalCorpusManifest {
+    schemaVersion: "1.0.0";
+    targets: ExternalCorpusTarget[];
+}
+export interface LoadedExternalCorpusTarget extends ExternalCorpusTarget {
+    resolvedPath: string;
+}
+export interface LoadedExternalCorpusManifest extends ExternalCorpusManifest {
+    targets: LoadedExternalCorpusTarget[];
+}
+export interface ExternalCorpusReportedFinding {
+    findingId: string;
+    fingerprint: string;
+    classification: FindingReviewClassification | "unreviewed";
+}
+export interface ExternalCorpusCaseResult {
+    id: string;
+    profile: ExternalCorpusProfile;
+    sourceType: ExternalCorpusSourceType;
+    disclosure: "anonymized";
+    mode: ExternalCorpusMode;
+    expected: {
+        status: ExternalCorpusTarget["expectedStatus"];
+        contentDigest: string;
+        findings: ExternalCorpusFindingReview[];
+    };
+    actual: {
+        status: ExternalCorpusTarget["expectedStatus"];
+        contentDigest: string;
+        findings: ExternalCorpusReportedFinding[];
+    };
+    digestMatched: boolean;
+    classificationCounts: ExternalCorpusClassificationCounts;
+    expectationMatched: boolean;
+}
+export interface ExternalCorpusClassificationCounts {
+    truePositive: number;
+    falsePositive: number;
+    unclear: number;
+    missingExpectedFinding: number;
+}
+export interface ExternalValidationCorpusReport {
+    schemaVersion: "1.0.0";
+    kind: "doctor.validation.corpus";
+    corpusType: "external";
+    generatedAt: string;
+    version: string;
+    summary: {
+        status: "pass" | "fail";
+        caseCount: number;
+        passedExpectations: number;
+        failedExpectations: number;
+        runtimeCases: 0;
+        classificationCounts: ExternalCorpusClassificationCounts;
+    };
+    cases: ExternalCorpusCaseResult[];
+}
+export interface BuildExternalValidationCorpusOptions {
+    environment?: {
+        env?: NodeJS.ProcessEnv;
+        platform?: NodeJS.Platform;
+    };
+}
+export declare class ExternalCorpusManifestError extends Error {
+    constructor(message: string);
+}
+export declare function loadExternalCorpusManifest(manifestPath: string): Promise<LoadedExternalCorpusManifest>;
+export declare function buildExternalValidationCorpusReport(manifestPath: string, options?: BuildExternalValidationCorpusOptions): Promise<ExternalValidationCorpusReport>;