@juzser/blacksmith 0.1.0

package/factory/orchestrator/dist/providers/cli-transport.js

@@ -0,0 +1,208 @@
+// Generic CLI judge transport (architecture §6; design decision: Codex runs
+// headless via `codex exec`, ChatGPT-subscription auth — docs/runbooks/
+// providers.md). Prompt delivery: STDIN, not a temp file — no filesystem
+// cleanup/race to manage, and a CLI judge that reads a prompt off stdin
+// (same as an interactive paste) needs no extra flag wiring per provider.
+import { spawn } from 'node:child_process';
+import { extractAndValidate } from './schema-validate.js';
+import { ProviderError } from './types.js';
+const NUDGE = '\n\nReturn only valid JSON per schema.';
+const DIAGNOSTIC_TAIL_BYTES = 400;
+/** Last few hundred bytes of a stream, collapsed to one line, for an error message. */
+function tail(text) {
+    const trimmed = text.trim();
+    const cut = trimmed.length > DIAGNOSTIC_TAIL_BYTES ? trimmed.slice(-DIAGNOSTIC_TAIL_BYTES) : trimmed;
+    return cut.replace(/\s+/g, ' ');
+}
+/**
+ * What to quote back to the operator when a judge process failed.
+ *
+ * Do NOT prefer one stream over the other. Observed against the real binary:
+ * `codex exec --json` wrote progress to stderr ("Reading prompt from stdin...")
+ * and the actual refusal to stdout as a `{"type":"error"}` event, so a
+ * stderr-first rule quotes the chatter and drops the reason. That is why the
+ * `refuse` and `refuse-codex` fixtures are exact mirrors of each other.
+ *
+ * crosscheck.yml no longer passes `--json` (D-118), and plain `exec` splits the
+ * streams differently again — the whole prompt is echoed to stderr, the answer
+ * goes to stdout. Which only reinforces the rule: no CLI owes us a convention
+ * about where it says things, so the combined buffer is the only safe input.
+ *
+ * The combined tail is the answer in the common case. stderr is prepended only
+ * when the tail window no longer shows it, which happens when a chatty judge
+ * floods stdout after writing a short reason to stderr.
+ */
+function diagnosticFor(outcome) {
+    const combined = tail(outcome.combined);
+    const errOnly = tail(outcome.stderr);
+    if (errOnly === '' || combined.includes(errOnly))
+        return combined;
+    return combined === '' ? errOnly : `${errOnly} | ${combined}`;
+}
+/**
+ * One CLI judge invocation: detached process group (POSIX `setsid` via
+ * `detached: true`) + group-kill on timeout or size-cap breach — mirrors
+ * testgate.ts's runOne() exactly (a judge CLI can spawn its own child
+ * processes; a plain per-child kill would orphan them the same way
+ * testgate's own regression note describes).
+ */
+function spawnOnce(config, prompt, budget) {
+    return new Promise((resolve) => {
+        const child = spawn(config.command, config.args, {
+            detached: true,
+            stdio: ['pipe', 'pipe', 'pipe'],
+        });
+        let combined = '';
+        let stderr = '';
+        let totalBytes = 0;
+        let timedOut = false;
+        let sizeExceeded = false;
+        let settled = false;
+        let spawnError;
+        const killGroup = () => {
+            if (child.pid !== undefined) {
+                try {
+                    process.kill(-child.pid, 'SIGKILL');
+                }
+                catch {
+                    // Group already gone — fine.
+                }
+            }
+        };
+        const timer = setTimeout(() => {
+            timedOut = true;
+            killGroup();
+        }, budget.timeout_ms);
+        const onChunk = (chunk) => {
+            totalBytes += chunk.length;
+            if (totalBytes > budget.max_output_bytes) {
+                if (!sizeExceeded) {
+                    sizeExceeded = true;
+                    killGroup();
+                }
+                return;
+            }
+            combined += chunk.toString('utf8');
+        };
+        child.stdout?.on('data', onChunk);
+        child.stderr?.on('data', (chunk) => {
+            // Still folded into `combined` for extraction — a judge that writes its
+            // verdict to stderr has always worked and must keep working. The second
+            // copy exists only so a refusal reason can be reported on its own.
+            stderr += chunk.toString('utf8');
+            onChunk(chunk);
+        });
+        child.stdin?.on('error', () => {
+            // Judge exited before consuming stdin (e.g. a fixture that errors
+            // immediately) — writing to a closed pipe would otherwise raise an
+            // unhandled EPIPE; the close handler below still resolves correctly.
+        });
+        child.stdin?.write(prompt);
+        child.stdin?.end();
+        const finish = (exitCode, signal) => {
+            if (settled)
+                return;
+            settled = true;
+            clearTimeout(timer);
+            resolve({ combined, stderr, exitCode, signal, spawnError, timedOut, sizeExceeded });
+        };
+        child.on('error', (err) => {
+            // The child never ran (ENOENT, EACCES, …). Distinct from a child that
+            // ran and failed: 'close' carries no code in this case.
+            spawnError = err;
+            finish(null, null);
+        });
+        child.on('close', finish);
+    });
+}
+/**
+ * The buffer minus the prompt this process just sent, for the extractor to
+ * scan.
+ *
+ * D-195: plain `codex exec` echoes the prompt it read off stdin back onto
+ * stderr (see diagnosticFor above), and stderr is folded into `combined`. Every
+ * prompt this factory builds interpolates free text an earlier agent wrote — a
+ * finding's summary, its claimed failure scenario, a reviewer contract that
+ * spells out `[]` for a clean diff — so an answer-shaped string sitting in that
+ * text lands in the buffer AHEAD of the judge's own answer and wins
+ * extractAndValidate's first-valid-candidate rule. A reviewer who writes
+ * `{"verdict": "refute"}` into a summary refutes the finding against itself.
+ *
+ * The bytes removed are bytes this process sent, which is a fact about the
+ * exchange rather than a guess about the judge. Preferring the LAST candidate
+ * would be such a guess and breaks on a chatty judge or a protocol trailer;
+ * discarding any candidate whose text also occurs in the prompt would throw
+ * away a judge's legitimate `[]` clean review, turning a clean review into a
+ * hard failure.
+ */
+function withoutEcho(combined, prompt) {
+    if (prompt === '')
+        return combined;
+    // Newline, not empty: candidates either side of an excised span must not be
+    // glued into one unparseable run.
+    return combined.split(prompt).join('\n');
+}
+/** One attempt: spawn, then extract+validate; throws on timeout/size-cap, returns the validation outcome otherwise. */
+async function attempt(provider, config, prompt, request, attemptLabel) {
+    const outcome = await spawnOnce(config, prompt, request.budget);
+    if (outcome.timedOut) {
+        throw new ProviderError('provider.timeout', `Provider "${provider}" CLI judge timed out after ${request.budget.timeout_ms}ms${attemptLabel}.`, { provider });
+    }
+    if (outcome.sizeExceeded) {
+        throw new ProviderError('provider.output-too-large', `Provider "${provider}" CLI judge output exceeded the ${request.budget.max_output_bytes}-byte cap${attemptLabel}.`, { provider });
+    }
+    const answered = withoutEcho(outcome.combined, prompt);
+    return { validation: extractAndValidate(answered, request.schemaName), spawned: outcome };
+}
+/**
+ * D-116: the process either refused to run or ran and failed. Either way the
+ * defect is not in the prompt, so the caller must not nudge-and-retry — that
+ * spends a second call on an already-exhausted quota and then reports a schema
+ * critique of whatever protocol chatter the CLI printed before dying. Returns
+ * undefined when the process itself was healthy.
+ */
+function processFailure(provider, config, outcome) {
+    if (outcome.spawnError !== undefined) {
+        const reason = outcome.spawnError.code ?? outcome.spawnError.message;
+        return new ProviderError('provider.cli-unavailable', `Provider "${provider}" CLI judge could not be started: ${config.command} (${reason}). ` +
+            'Check that the binary is installed and on PATH — see docs/runbooks/providers.md.', { provider, command: config.command, reason });
+    }
+    if (outcome.exitCode !== null && outcome.exitCode !== 0) {
+        const diagnostic = diagnosticFor(outcome);
+        return new ProviderError('provider.cli-failed', `Provider "${provider}" CLI judge failed with exit ${outcome.exitCode} and no usable verdict` +
+            `${diagnostic ? `: ${diagnostic}` : '.'}`, { provider, exitCode: outcome.exitCode, diagnostic });
+    }
+    if (outcome.signal !== null) {
+        return new ProviderError('provider.cli-failed', `Provider "${provider}" CLI judge was killed by ${outcome.signal} without a usable verdict.`, { provider, signal: outcome.signal });
+    }
+    return undefined;
+}
+export async function runCliJudge(provider, config, request) {
+    const start = Date.now();
+    const first = await attempt(provider, config, request.prompt, request, '');
+    let result = first.validation;
+    if (!result.valid) {
+        // A judge that answered is a judge that answered, whatever its exit code —
+        // so this check comes *after* validation, never before it. But if the
+        // output is unusable AND the process itself failed, the prompt is not the
+        // suspect and a retry is waste: report what actually happened.
+        const processError = processFailure(provider, config, first.spawned);
+        if (processError !== undefined)
+            throw processError;
+        // ONE retry with a nudge appended — a fresh spawn (the failed attempt's process is already dead).
+        const second = await attempt(provider, config, `${request.prompt}${NUDGE}`, request, ' on retry');
+        result = second.validation;
+        if (!result.valid) {
+            const retryProcessError = processFailure(provider, config, second.spawned);
+            if (retryProcessError !== undefined)
+                throw retryProcessError;
+            throw new ProviderError('provider.invalid-output', `Provider "${provider}" CLI judge returned invalid output after one retry: ${result.reason}.`, { provider, reason: result.reason, errors: 'errors' in result ? result.errors : undefined });
+        }
+    }
+    return {
+        provider,
+        kind: request.kind,
+        output: result.value,
+        latency_ms: Date.now() - start,
+    };
+}

package/factory/orchestrator/dist/providers/index.js

@@ -0,0 +1,54 @@
+// Provider registry (architecture §6: the judge tier is "provider-agnostic by
+// contract", so "any model that can honor the contract can serve"). Adding a
+// provider is a crosscheck.yml entry and nothing else — no file here, no
+// branch below.
+//
+// It did not start that way. This function used to open with two name-keyed
+// branches, `codex` and `deepseek`, each dispatching to a one-line module that
+// forwarded to the same generic transport this file ends with, and each
+// refusing to run if its name was paired with the other transport. The
+// forwarding modules added no behaviour, and the refusals were the real cost:
+// they turned two ordinary strings into reserved words with opinions. An
+// operator reaching Codex over an OpenAI-compatible endpoint, or running
+// DeepSeek's open weights as a local command, wrote a correct config and was
+// told it was misconfigured — and an operator whose own provider happened to
+// be called either name inherited a vendor's transport by coincidence of
+// spelling. Both are configurations someone will write, and neither is wrong.
+//
+// So the name is now data all the way through: it selects a config entry and
+// labels the result, and `transport` alone decides what runs. Keyed by
+// crosscheck.yml's provider names; `claude` (kind: native) is never dispatched
+// through here — it's the in-process judge already wired through
+// gate.ts/findings.ts.
+import { loadCrosscheckPolicy } from '../crosscheck.js';
+import { runApiJudge } from './api-transport.js';
+import { runCliJudge } from './cli-transport.js';
+import { ProviderError } from './types.js';
+export async function runJudge(providerName, request, opts = {}) {
+    const policy = opts.policy ?? loadCrosscheckPolicy();
+    const config = policy.providers[providerName];
+    if (!config) {
+        throw new ProviderError('provider.unknown', `Unknown provider "${providerName}" (not in crosscheck.yml).`, { provider: providerName });
+    }
+    if (config.kind === 'native') {
+        throw new ProviderError('provider.not-external', `Provider "${providerName}" is native (in-process); it has no CLI/API transport to run.`, { provider: providerName });
+    }
+    if (!config.enabled) {
+        throw new ProviderError('provider.disabled', `Provider "${providerName}" is disabled in crosscheck.yml (enabled: false) — never invoked.`, { provider: providerName });
+    }
+    // Every enabled, non-native provider lands here, whatever it is called.
+    // `transport` is validated when crosscheck.yml is parsed, so the two arms
+    // below are exhaustive and a provider that reached this point has the fields
+    // its arm reads.
+    if (config.transport === 'cli') {
+        return runCliJudge(providerName, { command: config.command, args: config.args }, request);
+    }
+    return runApiJudge(providerName, {
+        baseUrl: config.baseUrl,
+        model: config.model,
+        apiKeyEnv: config.apiKeyEnv,
+        responseFormatJsonObject: config.responseFormatJsonObject,
+    }, request, opts.fetchImpl);
+}
+export { ProviderError } from './types.js';
+export { loadCrosscheckPolicy };

package/factory/orchestrator/dist/providers/schema-validate.js

@@ -0,0 +1,152 @@
+// Shared by cli-transport.ts and api-transport.ts: pull the first balanced
+// JSON value out of a judge's raw text response (tolerating surrounding
+// prose — LLMs routinely wrap JSON in a sentence or a markdown fence) and
+// schema-validate it via the same schemas.ts/taxonomy.ts pipeline the rest
+// of the orchestrator uses (gate.ts, findings.ts) — no bespoke validation
+// path for provider output.
+import { compileSchemas, validateRecord, } from '../schemas.js';
+import { loadTaxonomy } from '../taxonomy.js';
+let cachedTaxonomy;
+let cachedSchemas;
+function resolve(opts) {
+    if (cachedTaxonomy === undefined)
+        cachedTaxonomy = loadTaxonomy();
+    if (cachedSchemas === undefined)
+        cachedSchemas = compileSchemas(cachedTaxonomy);
+    return { taxonomy: opts.taxonomy ?? cachedTaxonomy, schemas: opts.schemas ?? cachedSchemas };
+}
+/**
+ * Every substring of `text` that starts at a `{`/`[`, balances to its matching
+ * `}`/`]` (respecting quoted strings/escapes), and parses as JSON — yielded in
+ * the order they appear. Candidate starts that balance but fail to parse are
+ * skipped rather than ending the scan.
+ *
+ * D-118: there is deliberately no "the first one is the answer" rule here.
+ * Real `codex exec` echoes the prompt back on stderr, the transport merges
+ * stderr into this buffer for extraction, and a prompt that mentions `[]` or
+ * shows a schema template therefore plants parseable JSON *ahead* of the
+ * verdict. Only the caller's schema can say which candidate is the answer.
+ */
+function* balancedJsonCandidates(text) {
+    for (let i = 0; i < text.length; i++) {
+        const startChar = text[i];
+        if (startChar !== '{' && startChar !== '[')
+            continue;
+        const closeChar = startChar === '{' ? '}' : ']';
+        let depth = 0;
+        let inString = false;
+        let isEscaped = false;
+        for (let j = i; j < text.length; j++) {
+            const c = text[j];
+            if (inString) {
+                if (isEscaped)
+                    isEscaped = false;
+                else if (c === '\\')
+                    isEscaped = true;
+                else if (c === '"')
+                    inString = false;
+                continue;
+            }
+            if (c === '"') {
+                inString = true;
+                continue;
+            }
+            // Only the START bracket's own family is depth-counted. Valid JSON
+            // never lets `{`/`[` cross-nest without also closing in order (e.g.
+            // "{[}]" is not legal JSON), so tracking one family is sufficient to
+            // find where the top-level value actually ends; JSON.parse below is
+            // the real correctness check regardless.
+            if (c === startChar)
+                depth++;
+            else if (c === closeChar) {
+                depth--;
+                if (depth === 0) {
+                    const candidate = text.slice(i, j + 1);
+                    let parsed;
+                    try {
+                        parsed = JSON.parse(candidate);
+                    }
+                    catch {
+                        break; // not valid JSON after all — resume scanning from i + 1
+                    }
+                    yield parsed;
+                    // Resume *after* this value rather than inside it: a nested object
+                    // is part of the value already yielded, never a sibling answer.
+                    i = j;
+                    break;
+                }
+            }
+        }
+    }
+}
+/**
+ * The first balanced, parseable JSON value in `text`, or null if there is none.
+ * Unaware of any schema — kept for callers that genuinely want "whatever JSON
+ * this text contains". Judge output goes through `extractAndValidate`, which
+ * picks by schema; see D-118 on why the difference matters.
+ */
+export function extractBalancedJson(text) {
+    for (const candidate of balancedJsonCandidates(text))
+        return candidate;
+    return null;
+}
+/**
+ * Schemas that describe ONE item a `kind: review` judge returns MANY of. Both
+ * of them: `finding` is the stored record (the native reviewer's contract,
+ * .claude/agents/reviewer.md), `finding-evidence` is the judge's half of it —
+ * the same evidence without the six identity fields the orchestrator owns.
+ * A set, not a comparison, because the second entry was added a year after the
+ * first and the `=== 'finding'` it replaced would have silently validated an
+ * array of evidence as a single malformed object.
+ */
+const ARRAY_VALUED_SCHEMAS = new Set(['finding', 'finding-evidence']);
+/**
+ * Extract + schema-validate a judge's raw text response. `finding.schema.json`
+ * describes ONE finding, but a `kind: review` judge's contract
+ * (.claude/agents/reviewer.md) returns an ARRAY of findings — so an array
+ * top-level value against an ARRAY_VALUED_SCHEMAS name validates element-wise;
+ * every other schema (e.g. "judge-verdict") validates the parsed value
+ * directly against its own object shape.
+ */
+export function extractAndValidate(rawText, schemaName, opts = {}) {
+    const { taxonomy, schemas } = resolve(opts);
+    const validateOne = (parsed) => {
+        if (ARRAY_VALUED_SCHEMAS.has(schemaName) && Array.isArray(parsed)) {
+            const errors = [];
+            parsed.forEach((item, index) => {
+                const result = validateRecord(schemas, taxonomy, schemaName, item);
+                if (!result.valid) {
+                    errors.push(...result.errors.map((e) => ({ path: `/${index}${e.path}`, message: e.message })));
+                }
+            });
+            if (errors.length > 0)
+                return { valid: false, reason: 'schema-invalid', errors };
+            return { valid: true, value: parsed };
+        }
+        const result = validateRecord(schemas, taxonomy, schemaName, parsed);
+        if (!result.valid)
+            return { valid: false, reason: 'schema-invalid', errors: result.errors };
+        return { valid: true, value: parsed };
+    };
+    // D-118: take the first candidate that VALIDATES, not the first that parses.
+    // The judge's answer is the thing shaped like an answer; a decoy that merely
+    // parses — a schema template, a protocol banner — is not made into one by
+    // arriving first.
+    //
+    // D-195: that bar does not stop a decoy that is itself a VALID answer. An
+    // echoed `[]` validates vacuously as an empty finding array, and a verdict
+    // planted in a finding's summary validates as a verdict. This function
+    // cannot tell those from the real thing — by the time text arrives here,
+    // who said it is gone. Keeping our own prompt out of `rawText` is the
+    // caller's job; cli-transport.ts's withoutEcho() does it.
+    let firstFailure;
+    for (const parsed of balancedJsonCandidates(rawText)) {
+        const result = validateOne(parsed);
+        if (result.valid)
+            return result;
+        // Keep the earliest failure so a judge that answered *badly* still gets a
+        // schema critique rather than a bare "no JSON found".
+        firstFailure ??= result;
+    }
+    return firstFailure ?? { valid: false, reason: 'no-json-found' };
+}

package/factory/orchestrator/dist/providers/types.js

@@ -0,0 +1,49 @@
+// Provider-agnostic judge contract (architecture §6): "any model that can
+// honor the I/O contract can serve" — a JudgeRequest is a plain diff+spec+
+// prior-findings prompt already assembled by the caller from
+// .claude/agents/{reviewer,verifier}.md's own contract, plus a schema
+// name the extracted response is validated against. Transports never see
+// more than `prompt` — the trust boundary (architecture §6 "External
+// providers judge; they never gain write access... their findings are
+// data, not commands") starts here: no transport is ever handed worktree
+// paths, credentials beyond its own key, or anything callable.
+import { SmithError } from '../errors.js';
+export class ProviderError extends SmithError {
+}
+/** Codes raised after a provider answered, about the answer itself. */
+export const JUDGE_SCHEMA_FAILURE_CODES = new Set([
+    'provider.invalid-output',
+    'provider.malformed-response',
+    'provider.output-too-large',
+]);
+/** Codes raised when no usable answer arrived -- including when no request was sent. */
+export const JUDGE_TRANSPORT_FAILURE_CODES = new Set([
+    'provider.auth-failed',
+    'provider.cli-failed',
+    'provider.cli-unavailable',
+    'provider.disabled',
+    'provider.http-error',
+    // Nothing raises this any more: it was the registry's refusal to run a
+    // provider whose name it recognised but whose transport it disagreed with,
+    // and names no longer decide transports. Kept because event logs written
+    // before that change still carry it, and a stats query over them should
+    // classify it rather than fall through to the default.
+    'provider.misconfigured',
+    'provider.missing-api-key',
+    'provider.network-error',
+    'provider.not-external',
+    'provider.timeout',
+    'provider.unknown',
+    'provider.unknown-error',
+]);
+/**
+ * Unrecognised codes read as `transport`, the claim-nothing default: a
+ * transport failure asserts only that the run did not reach a verdict, where
+ * a schema failure asserts the provider's output is unusable. A drift test
+ * (test/providers/judgeFailureKind.test.ts) keeps the two sets covering every
+ * code the source can raise, so the default stays a fallback and not a
+ * silent home for codes nobody classified.
+ */
+export function judgeFailureKind(code) {
+    return JUDGE_SCHEMA_FAILURE_CODES.has(code) ? 'schema' : 'transport';
+}