@stupify/cli 0.0.2 → 0.0.3

package/dist/experiment.js

@@ -0,0 +1,225 @@
+import { execFile } from "node:child_process";
+import { mkdir, readFile, writeFile } from "node:fs/promises";
+import path from "node:path";
+import { promisify } from "node:util";
+const execFileAsync = promisify(execFile);
+export async function runExperiment(configPath) {
+    const config = await readConfig(configPath);
+    const startedAt = new Date();
+    const outputDir = path.join(process.cwd(), "experiments", "results", `${safeSegment(config.name)}-${timestamp(startedAt)}`);
+    await mkdir(outputDir, { recursive: true });
+    const cwd = resolveCwd(config.cwd);
+    const cliPath = path.resolve(process.argv[1] ?? "packages/cli/dist/stupify.js");
+    const summaries = [];
+    for (const run of config.runs) {
+        const args = ensureJson([...config.baseCommand, ...run.args]);
+        const command = [process.execPath, cliPath, ...args].join(" ");
+        const summary = await runOneExperiment({
+            id: run.id,
+            args,
+            command,
+            cwd,
+            cliPath,
+            outputDir,
+        });
+        summaries.push(summary);
+        await writeFile(path.join(outputDir, "summary.json"), `${JSON.stringify({ name: config.name, cwd, runs: summaries }, null, 2)}\n`);
+    }
+    return outputDir;
+}
+async function readConfig(configPath) {
+    const fullPath = path.resolve(configPath);
+    const raw = await readFile(fullPath, "utf8");
+    const parsed = JSON.parse(raw);
+    if (!parsed.name)
+        throw new Error("Experiment config requires name.");
+    if (!Array.isArray(parsed.baseCommand))
+        throw new Error("Experiment config requires baseCommand array.");
+    if (!Array.isArray(parsed.runs))
+        throw new Error("Experiment config requires runs array.");
+    return {
+        name: parsed.name,
+        cwd: parsed.cwd,
+        baseCommand: parsed.baseCommand,
+        runs: parsed.runs,
+    };
+}
+async function runOneExperiment(input) {
+    const startedAt = Date.now();
+    try {
+        const { stdout, stderr } = await execFileAsync(process.execPath, [input.cliPath, ...input.args], {
+            cwd: input.cwd,
+            maxBuffer: 128 * 1024 * 1024,
+        });
+        if (stderr.trim()) {
+            await writeFile(path.join(input.outputDir, `${safeSegment(input.id)}.stderr.txt`), stderr);
+        }
+        await writeFile(path.join(input.outputDir, `${safeSegment(input.id)}.json`), stdout);
+        const report = JSON.parse(stdout);
+        const summary = summarizeReport(input.id, input.command, Date.now() - startedAt, report, []);
+        await writeFile(path.join(input.outputDir, `${safeSegment(input.id)}-findings.md`), findingsMarkdown(summary, report));
+        return summary;
+    }
+    catch (error) {
+        const details = errorDetails(error);
+        if (details.stdout)
+            await writeFile(path.join(input.outputDir, `${safeSegment(input.id)}.stdout.txt`), details.stdout);
+        if (details.stderr)
+            await writeFile(path.join(input.outputDir, `${safeSegment(input.id)}.stderr.txt`), details.stderr);
+        const summary = summarizeReport(input.id, input.command, Date.now() - startedAt, {}, [details.message]);
+        await writeFile(path.join(input.outputDir, `${safeSegment(input.id)}-findings.md`), findingsMarkdown(summary, {}));
+        return summary;
+    }
+}
+function summarizeReport(id, command, elapsedMs, report, errors) {
+    const run = objectValue(report.run);
+    const auditStats = objectValue(run.auditStats);
+    const traceEvents = arrayValue(run.traceEvents).map(objectValue);
+    const findings = arrayValue(report.findings).map(objectValue);
+    const targetPreview = arrayValue(run.debugTargets).map(objectValue).map(debugTargetRecord);
+    const contextPackEvents = traceEvents.filter((event) => event.name === "context.pack");
+    const auditBatchEvents = traceEvents.filter((event) => event.name === "audit.batch");
+    const auditInputTokens = auditBatchEvents.map((event) => detailNumber(event.detail, "input_tokens")).filter(isNumber);
+    return {
+        id,
+        command,
+        totalMs: numberValue(objectValue(run.timingsMs).total, elapsedMs),
+        entitiesScanned: numberValue(run.entitiesScanned, 0),
+        targets: numberValue(run.auditedCandidateCount, 0),
+        targetsByCheck: recordOfNumbers(run.targetsByCheck),
+        auditContext: stringValue(run.auditContext, "unknown"),
+        auditPrompt: stringValue(run.auditPrompt, "unknown"),
+        repomixFiles: maxNumber(contextPackEvents.map((event) => numberValue(event.count, 0))),
+        repomixTokens: maxNumber(contextPackEvents.map((event) => detailNumber(event.detail, "pack_tokens") ?? 0)),
+        auditCalls: auditBatchEvents.length,
+        auditInputTokens,
+        auditMs: numberValue(objectValue(run.timingsMs).audit, 0),
+        findings: findings.length,
+        uncertain: numberValue(auditStats.uncertain, 0),
+        clean: numberValue(auditStats.clean, 0),
+        findingsByCheck: countBy(findings, "checkId"),
+        uncertainByCheck: {},
+        targetPreview,
+        errors,
+    };
+}
+function findingsMarkdown(summary, report) {
+    const findings = arrayValue(report.findings).map(objectValue);
+    const lines = [
+        `# ${summary.id}`,
+        "",
+        `Runtime: ${Math.round(summary.totalMs / 1000)}s`,
+        `Targets: ${summary.targets}`,
+        `Findings: ${summary.findings}`,
+        `Uncertain: ${summary.uncertain}`,
+        "",
+        "## Targets",
+        ...targetMarkdown(summary.targetPreview),
+        "",
+        "## Findings",
+    ];
+    if (findings.length === 0) {
+        lines.push("None.");
+    }
+    else {
+        findings.forEach((finding, index) => {
+            lines.push(`${index + 1}. ${stringValue(finding.checkId, "unknown")}`, `why: ${stringValue(finding.why, "")}`, `proof: ${stringValue(finding.proof, "")}`, "Manual label: [good/maybe/bad]", "");
+        });
+    }
+    if (summary.errors.length > 0) {
+        lines.push("", "## Errors", ...summary.errors.map((error) => `- ${error}`));
+    }
+    return `${lines.join("\n")}\n`;
+}
+function targetMarkdown(targets) {
+    if (targets.length === 0)
+        return ["None recorded."];
+    return targets.map((target) => (`- ${target.targetId} ${target.checkId} ${target.entityKind ?? "unknown"}/${target.changeKind ?? "unknown"} ${target.entityId}
+  reason: ${target.scoutReason ?? ""}`));
+}
+function countBy(items, key) {
+    const counts = {};
+    for (const item of items) {
+        const value = stringValue(item[key], "");
+        if (!value)
+            continue;
+        counts[value] = (counts[value] ?? 0) + 1;
+    }
+    return counts;
+}
+function recordOfNumbers(value) {
+    const input = objectValue(value);
+    const output = {};
+    for (const [key, count] of Object.entries(input)) {
+        if (typeof count === "number" && Number.isFinite(count))
+            output[key] = count;
+    }
+    return output;
+}
+function debugTargetRecord(value) {
+    return {
+        targetId: stringValue(value.targetId, ""),
+        checkId: stringValue(value.checkId, ""),
+        entityId: stringValue(value.entityId, ""),
+        entityKind: optionalString(value.entityKind),
+        changeKind: optionalString(value.changeKind),
+        scoutReason: optionalString(value.scoutReason),
+        sourceLabel: optionalString(value.sourceLabel),
+    };
+}
+function optionalString(value) {
+    return typeof value === "string" ? value : undefined;
+}
+function ensureJson(args) {
+    return args.includes("--json") ? args : [...args, "--json"];
+}
+function resolveCwd(value) {
+    if (!value)
+        return process.cwd();
+    if (value.startsWith("$")) {
+        const envName = value.slice(1);
+        const envValue = process.env[envName];
+        if (!envValue)
+            throw new Error(`Experiment cwd env var ${envName} is not set.`);
+        return path.resolve(envValue);
+    }
+    return path.resolve(value);
+}
+function objectValue(value) {
+    return value && typeof value === "object" && !Array.isArray(value) ? value : {};
+}
+function arrayValue(value) {
+    return Array.isArray(value) ? value : [];
+}
+function stringValue(value, fallback) {
+    return typeof value === "string" ? value : fallback;
+}
+function numberValue(value, fallback) {
+    return typeof value === "number" && Number.isFinite(value) ? value : fallback;
+}
+function detailNumber(value, key) {
+    if (typeof value !== "string")
+        return null;
+    const match = new RegExp(`${key}=([0-9]+)`).exec(value);
+    return match ? Number(match[1]) : null;
+}
+function maxNumber(values) {
+    return values.length === 0 ? 0 : Math.max(...values);
+}
+function isNumber(value) {
+    return value !== null;
+}
+function errorDetails(error) {
+    const candidate = objectValue(error);
+    return {
+        message: error instanceof Error ? error.message : String(error),
+        stdout: typeof candidate.stdout === "string" ? candidate.stdout : undefined,
+        stderr: typeof candidate.stderr === "string" ? candidate.stderr : undefined,
+    };
+}
+function timestamp(date) {
+    return date.toISOString().replace(/[:.]/g, "-");
+}
+function safeSegment(value) {
+    return value.replace(/[^A-Za-z0-9._-]+/g, "-").replace(/^-+|-+$/g, "") || "experiment";
+}

package/dist/git.d.ts

@@ -0,0 +1,8 @@
+import { type NetDiff, type SourceRange } from "./types.ts";
+export declare function netDiffSince(since: string): Promise<NetDiff>;
+export declare function netDiffForCommit(commit: string): Promise<NetDiff>;
+export declare function netDiffForRecentCommits(count: number): Promise<NetDiff>;
+export declare function sourceRangeSince(since: string): Promise<SourceRange>;
+export declare function sourceRangeForCommit(commit: string): Promise<SourceRange>;
+export declare function sourceRangeForRecentCommits(count: number): Promise<SourceRange>;
+export declare function netDiffFromStdin(text: string): Promise<NetDiff>;

package/dist/git.js

@@ -0,0 +1,219 @@
+import { execFile } from "node:child_process";
+import { promisify } from "node:util";
+import { sourceId } from "./types.js";
+const execFileAsync = promisify(execFile);
+export async function netDiffSince(since) {
+    const range = await sourceRangeSince(since);
+    return netDiff(range.base, range.target, range.label, range.id);
+}
+export async function netDiffForCommit(commit) {
+    const range = await sourceRangeForCommit(commit);
+    return netDiff(range.base, range.target, range.label, range.id);
+}
+export async function netDiffForRecentCommits(count) {
+    const range = await sourceRangeForRecentCommits(count);
+    return netDiff(range.base, range.target, range.label, range.id);
+}
+export async function sourceRangeSince(since) {
+    const [base, target] = await Promise.all([baseBefore(since), revParse("HEAD")]);
+    return sourceRange(base, target, `last ${since}`);
+}
+export async function sourceRangeForCommit(commit) {
+    const [base, target, shortTarget, message] = await Promise.all([
+        revParse(`${commit}^1`),
+        revParse(commit),
+        shortCommit(commit),
+        commitMessage(commit),
+    ]);
+    return sourceRange(base, target, firstLine(message) || shortTarget, sourceId(shortTarget));
+}
+export async function sourceRangeForRecentCommits(count) {
+    const commits = await recentCommits(count);
+    if (commits.length === 0)
+        throw new Error("No non-merge commits found.");
+    const oldest = commits[0];
+    const newest = commits[commits.length - 1];
+    const [base, target, shortBase, shortTarget] = await Promise.all([
+        revParse(`${oldest}^1`),
+        revParse(newest),
+        shortCommit(`${oldest}^1`),
+        shortCommit(newest),
+    ]);
+    return sourceRange(base, target, `${commits.length} recent commits`, sourceId(`range:${shortBase}..${shortTarget}`));
+}
+export async function netDiffFromStdin(text) {
+    if (!text.trim())
+        throw new Error("No diff received on stdin.");
+    return {
+        id: sourceId("stdin"),
+        label: "stdin",
+        base: "stdin",
+        target: "stdin",
+        text,
+        stats: statsFromDiff(text),
+    };
+}
+async function netDiff(base, target, label, id) {
+    const [text, stats, shortBase, shortTarget] = await Promise.all([
+        diff(base, target),
+        diffStats(base, target),
+        shortCommit(base),
+        shortCommit(target),
+    ]);
+    return {
+        id: id ?? sourceId(`net:${shortBase}..${shortTarget}`),
+        label,
+        base,
+        target,
+        text,
+        stats,
+    };
+}
+async function sourceRange(base, target, label, id) {
+    const [stats, shortBase, shortTarget] = await Promise.all([
+        diffStats(base, target),
+        shortCommit(base),
+        shortCommit(target),
+    ]);
+    return {
+        id: id ?? sourceId(`net:${shortBase}..${shortTarget}`),
+        label,
+        base,
+        target,
+        stats,
+    };
+}
+async function baseBefore(since) {
+    try {
+        const { stdout } = await execFileAsync("git", [
+            "log",
+            "--first-parent",
+            "--before",
+            since,
+            "-1",
+            "--format=%H",
+        ]);
+        const commit = stdout.trim();
+        if (commit)
+            return commit;
+        return rootCommit();
+    }
+    catch {
+        throw new Error(`Could not resolve base commit before ${since}.`);
+    }
+}
+async function rootCommit() {
+    try {
+        const { stdout } = await execFileAsync("git", ["rev-list", "--max-parents=0", "HEAD"]);
+        return stdout.trim().split(/\r?\n/, 1)[0] ?? "";
+    }
+    catch {
+        throw new Error("Could not resolve repository root commit.");
+    }
+}
+async function diff(base, target) {
+    try {
+        const { stdout } = await execFileAsync("git", [
+            "diff",
+            "--no-ext-diff",
+            "--no-color",
+            "--unified=8",
+            base,
+            target,
+            "--",
+        ], { maxBuffer: 128 * 1024 * 1024 });
+        if (!stdout.trim())
+            throw new Error("empty diff");
+        return stdout;
+    }
+    catch {
+        throw new Error(`No diff found for ${base}..${target}.`);
+    }
+}
+async function diffStats(base, target) {
+    try {
+        const { stdout } = await execFileAsync("git", ["diff", "--numstat", base, target, "--"], {
+            maxBuffer: 16 * 1024 * 1024,
+        });
+        return statsFromNumstat(stdout);
+    }
+    catch {
+        return { filesChanged: 0, additions: 0, deletions: 0 };
+    }
+}
+function statsFromDiff(diffText) {
+    const files = new Set();
+    let additions = 0;
+    let deletions = 0;
+    for (const line of diffText.split(/\r?\n/)) {
+        const fileMatch = /^diff --git a\/.+ b\/(.+)$/.exec(line);
+        if (fileMatch)
+            files.add(fileMatch[1]);
+        else if (line.startsWith("+") && !line.startsWith("+++"))
+            additions += 1;
+        else if (line.startsWith("-") && !line.startsWith("---"))
+            deletions += 1;
+    }
+    return { filesChanged: files.size, additions, deletions };
+}
+function statsFromNumstat(numstat) {
+    let filesChanged = 0;
+    let additions = 0;
+    let deletions = 0;
+    for (const line of numstat.split(/\r?\n/)) {
+        if (!line.trim())
+            continue;
+        const [added, deleted] = line.split(/\s+/, 3);
+        filesChanged += 1;
+        additions += Number(added) || 0;
+        deletions += Number(deleted) || 0;
+    }
+    return { filesChanged, additions, deletions };
+}
+async function recentCommits(count) {
+    try {
+        const { stdout } = await execFileAsync("git", [
+            "log",
+            "--first-parent",
+            "--no-merges",
+            "--format=%H",
+            `-${count}`,
+        ]);
+        return stdout.split(/\r?\n/).filter(Boolean).reverse();
+    }
+    catch {
+        throw new Error(`Could not read last ${count} commits.`);
+    }
+}
+async function revParse(rev) {
+    try {
+        const { stdout } = await execFileAsync("git", ["rev-parse", rev]);
+        return stdout.trim();
+    }
+    catch {
+        throw new Error(`Could not resolve ${rev}.`);
+    }
+}
+async function shortCommit(commit) {
+    try {
+        const { stdout } = await execFileAsync("git", ["rev-parse", "--short", commit]);
+        return stdout.trim();
+    }
+    catch {
+        throw new Error(`Could not resolve commit ${commit}.`);
+    }
+}
+async function commitMessage(commit) {
+    try {
+        const { stdout } = await execFileAsync("git", ["show", "--no-patch", "--format=%B", commit], {
+            maxBuffer: 1024 * 1024,
+        });
+        return stdout;
+    }
+    catch {
+        throw new Error(`Could not read commit message for ${commit}.`);
+    }
+}
+function firstLine(value) {
+    return value.trim().split(/\r?\n/, 1)[0]?.trim() ?? "";
+}

package/dist/index.d.ts

@@ -0,0 +1 @@
+export { main } from "./stupify.ts";

package/dist/index.js

@@ -0,0 +1 @@
+export { main } from "./stupify.js";

package/dist/model.d.ts

@@ -0,0 +1,24 @@
+import type { ModelId } from "./types.ts";
+export type ModelProfile = "scout" | "audit";
+export type LocalModel = Readonly<{
+    id: ModelId;
+    name: string;
+    baseUrl: string;
+    profile: ModelProfile;
+}>;
+export declare function loadLocalModels(modelId: ModelId): Promise<{
+    scoutModel: Readonly<{
+        id: ModelId;
+        name: string;
+        baseUrl: string;
+        profile: ModelProfile;
+    }>;
+    auditModel: Readonly<{
+        id: ModelId;
+        name: string;
+        baseUrl: string;
+        profile: ModelProfile;
+    }>;
+}>;
+export declare function firstRunModelBootstrap(modelId: ModelId): Promise<string>;
+export declare function loadLocalModel(modelPath: string, modelId: ModelId, profile?: ModelProfile): Promise<LocalModel>;