@llm4ts/shell 2.32.0 → 2.34.0

package/flows/lib/port.js

@@ -0,0 +1,147 @@
+// Shared by the port flows (ADR 0028): the pack as a porting pack, the
+// manifest of sources and their targets, the rulebook and pitfall cards
+// the implementer reads, and the prompts. The seats and the queue are the
+// flows' own business.
+import { join } from "node:path";
+import * as Effect from "effect/Effect";
+import { autonomyContract } from "@llm4ts/flow/AutonomyContract";
+import { cap } from "@llm4ts/flow/Context";
+import { FlowAborted } from "@llm4ts/flow/FlowError";
+import { loadPatternCards } from "@llm4ts/flow/Patterns";
+import { PortEntry, linesOf, parseLedger, portStatusInstruction, targetPathOf } from "@llm4ts/flow/Port";
+import { matchingFiles } from "@llm4ts/flow/SpecChecks";
+/** Where the port flows keep their state, under the target repository. */
+export const portStateDir = (workDir) => join(workDir, ".llm4ts", "port");
+export const defaultRulebook = [
+    "You translate one source file to its target language, faithfully and mechanically:",
+    "the same structure, the same names, the same control flow, so a reviewer can read source",
+    "and target side by side. Translate what the source does, never what you think it should",
+    "do. Flag anything you cannot translate confidently with a TODO(port): <reason> comment",
+    "instead of guessing: a flag is better than wrong code."
+].join("\n");
+/** The pack read as a porting pack: `target:` is required, the rulebook is `prompts/porting.md`. */
+export const asPortingPack = Effect.fn("port.asPortingPack")(function* (opened) {
+    const pack = opened.pack;
+    if (pack.target === undefined) {
+        return yield* FlowAborted.make({
+            message: `pack '${pack.name}' has no 'target:' template (e.g. target: {{dir}}/{{base}}.rs); it is not a porting pack`
+        });
+    }
+    const cards = yield* loadPatternCards(opened.workspace, `${opened.dir}/patterns`);
+    const pitfalls = cards
+        .filter((card) => card.id.startsWith("pitfalls"))
+        .map((card) => `### ${card.id}\n${card.body.trim()}`)
+        .join("\n\n");
+    return {
+        pack,
+        target: pack.target,
+        comment: pack.comment ?? "//",
+        rulebook: pack.prompt("porting") ?? defaultRulebook,
+        pitfalls
+    };
+});
+/** Every source the pack matches, with its line count and target; sorted by path. */
+export const portManifest = Effect.fn("port.manifest")(function* (repo, files, workDir, porting) {
+    const sources = yield* matchingFiles(repo, porting.pack.sources ?? "(?!)", porting.pack.exclude);
+    const entries = [];
+    for (const source of sources) {
+        const text = (yield* files.read(join(workDir, source))) ?? "";
+        entries.push(PortEntry.make({
+            id: source,
+            source,
+            target: targetPathOf(porting.target, source),
+            loc: linesOf(text)
+        }));
+    }
+    return entries;
+});
+/** The denials that keep an implementer on its one file (the Bun port's prompt, in spirit). */
+export const implementerDenials = [
+    "Read exactly one source file: the one named below. Do NOT read other source files for",
+    "context: the rulebook's maps are authoritative for cross-file references. Do NOT run a",
+    "build, a test or any git command: the flow builds and commits. Write only the target file.",
+    "Match the source's structure: same function names, same field order, same control flow."
+].join("\n");
+export const implementerSystem = (porting) => [
+    autonomyContract("minimal"),
+    "",
+    "# Porting rulebook",
+    porting.rulebook,
+    ...(porting.pitfalls.length === 0
+        ? []
+        : ["", "# Pitfalls: syntactically alike, semantically different", porting.pitfalls]),
+    "",
+    implementerDenials
+].join("\n");
+export const implementerPrompt = (entry, sourceText, porting, sourceChars, ledgerRows) => [
+    `Port \`${entry.source}\` (${entry.loc} lines). Write the ported file at: ${entry.target}`,
+    "The draft need not compile; it must capture the logic faithfully.",
+    ...(ledgerRows === undefined
+        ? []
+        : [
+            "",
+            "Precomputed classifications for this file (cross-file analysis; trust them over local guessing):",
+            ledgerRows
+        ]),
+    "",
+    portStatusInstruction(porting.comment),
+    "",
+    `Source file \`${entry.source}\`:`,
+    "```",
+    cap(sourceText, sourceChars).text,
+    "```"
+].join("\n");
+/** The fixer's brief for a drafted file: apply findings against the source, nothing else. */
+export const portFixerSystem = (porting) => [
+    autonomyContract("minimal"),
+    "",
+    "You apply review findings to a ported file another coder drafted. Apply the findings and",
+    "nothing else; where a finding conflicts with the source file's behaviour, the source wins.",
+    "Keep the PORT STATUS trailer at the end and update its confidence and todos honestly.",
+    "If a finding is wrong (the reviewer misread the code), skip it and say so in one line.",
+    "",
+    "# Porting rulebook",
+    porting.rulebook
+].join("\n");
+/** The compile fixer's brief: one unit, its diagnostics, no build, no git. */
+export const compileFixerSystem = (porting) => [
+    autonomyContract("minimal"),
+    "",
+    "You fix compiler diagnostics in ONE unit of a mechanically ported code base. Edit only the",
+    "files the diagnostics name and their unit. Other units are being fixed by other coders at",
+    "the same time: a broken import from another unit is EXPECTED, leave it. Do NOT run a build",
+    "or any git command: the flow rebuilds once per round and commits. Never stub a function,",
+    "weaken a type to `any` or its equivalent, or add a comment explaining why a workaround is",
+    "acceptable: if the fix needs a paragraph of justification, the fix is wrong. Where a fix",
+    "needs another unit to change, skip it and say which unit in one line.",
+    "",
+    "# Porting rulebook",
+    porting.rulebook
+].join("\n");
+/** The ledger the pack's `port-ledger` writes, beside the specs; `undefined` when none. */
+export const ledgerPath = (porting) => `${porting.pack.specsDir}/ledger.tsv`;
+export const readLedgerRows = (files, workDir, porting) => Effect.map(files.read(join(workDir, ledgerPath(porting))), (text) => text === undefined ? [] : parseLedger(text));
+/** A native engineer's brief for the trial port: no rulebook, idiomatic target code. */
+export const nativeImplementerSystem = (porting) => [
+    autonomyContract("minimal"),
+    "",
+    `You are a senior engineer in the target language of pack ${porting.pack.name}. Port the`,
+    "source file the way you would write it natively and idiomatically, with no rulebook: your",
+    "draft is compared with a rules-following draft to find what the rulebook forgot. Read",
+    "exactly one source file; do not run a build or any git command; write only the target file.",
+    "",
+    implementerDenials
+].join("\n");
+export const portEnv = (environment) => {
+    const int = (raw, fallback, min = 1) => {
+        const value = Number.parseInt(raw ?? "", 10);
+        return Number.isFinite(value) && value >= min ? value : fallback;
+    };
+    return {
+        pilot: int(environment.LLM4TS_PORT_PILOT, 0, 1),
+        concurrency: int(environment.LLM4TS_PORT_CONCURRENCY, 4),
+        batch: int(environment.LLM4TS_PORT_BATCH, 100),
+        votes: int(environment.LLM4TS_REVIEW_VOTES, 2),
+        sourceChars: int(environment.LLM4TS_PORT_SOURCE_CHARS, 120_000, 1_000)
+    };
+};

package/flows/port-compile.js

@@ -0,0 +1,139 @@
+// Compiler diagnostics as a work queue (ADR 0028, the Bun port's Phase B/D):
+// the pack's `## Diagnostics` command runs once per round, its output is
+// grouped by unit, one fixer per unit edits only that unit (no build, no
+// git), two adversarial votes review, the flow rebuilds and commits. Until
+// dry, or until a round lowers nothing (typed Stalled).
+//
+//   LLM4TS_PACK=zig-rust llm4ts run port-compile --repo ~/src/bun
+//
+// Knobs: LLM4TS_PORT_CONCURRENCY (4), LLM4TS_REVIEW_VOTES (2 here),
+// LLM4TS_PORT_COMPILE_ROUNDS (6).
+import { join } from "node:path";
+import * as Effect from "effect/Effect";
+import * as Ref from "effect/Ref";
+import { makeChat } from "@llm4ts/flow/Chat";
+import { groupDiagnostics, parseDiagnostics, renderDiagnostics } from "@llm4ts/flow/Diagnostics";
+import { FlowAborted, ProcessError, Stalled } from "@llm4ts/flow/FlowError";
+import { Info } from "@llm4ts/flow/FlowEvents";
+import { adversarialReviewer, reviewAndFixLoop } from "@llm4ts/flow/Review";
+import { runQueue } from "@llm4ts/flow/WorkQueue";
+import { asReadOnly, coderFromEnv, nodePlainFileStore, nodeProcessExecutor, openPack, resolveFlowInput, runFlowMain, runNode, stage } from "@llm4ts/runner";
+import { asPortingPack, compileFixerSystem, portEnv, portStateDir } from "./lib/port.js";
+const program = Effect.gen(function* () {
+    const input = yield* resolveFlowInput("Fix every compiler diagnostic, one unit at a time");
+    const coder = coderFromEnv(process.env);
+    const files = nodePlainFileStore;
+    const knobs = portEnv(process.env);
+    const maxRounds = Math.max(1, Number.parseInt(process.env.LLM4TS_PORT_COMPILE_ROUNDS ?? "6", 10) || 6);
+    const stateDir = portStateDir(input.workDir);
+    yield* runNode({
+        workDir: input.workDir,
+        workspace: input.workspace,
+        userPrompt: input.prompt,
+        coder,
+        reasoning: asReadOnly(coder),
+        reviewers: [asReadOnly(coder)],
+        environment: process.env
+    }, (context) => Effect.gen(function* () {
+        const events = context.events;
+        const opened = yield* stage(events, "pack", openPack({
+            environment: process.env,
+            launchDir: input.workspace,
+            flowDir: import.meta.dirname
+        }));
+        const porting = yield* asPortingPack(opened);
+        const diagnostics = porting.pack.diagnostics;
+        if (diagnostics === undefined) {
+            return yield* FlowAborted.make({
+                message: `pack '${porting.pack.name}' has no '## Diagnostics' section (- command: …, - format: json | cargo)`
+            });
+        }
+        const survey = Effect.gen(function* () {
+            const ran = yield* nodeProcessExecutor
+                .run(diagnostics.command, input.workDir, {})
+                .pipe(Effect.mapError((cause) => ProcessError.make({ message: diagnostics.command.join(" "), detail: cause.message })));
+            return parseDiagnostics([...ran.stdout, ...ran.stderr].join("\n"), diagnostics.format);
+        });
+        let previous;
+        let flat = 0;
+        for (let round = 1; round <= maxRounds; round += 1) {
+            const found = yield* stage(events, `diagnostics round ${round}`, survey);
+            const groups = groupDiagnostics(found);
+            yield* files.writeAtomic(join(stateDir, `diagnostics-${round}.md`), [
+                `# Diagnostics, round ${round}`,
+                "",
+                ...groups.map((group) => `## ${group.unit}\n\n${renderDiagnostics(group.diagnostics, 200)}`)
+            ].join("\n"));
+            if (found.length === 0) {
+                yield* events.publish(Info.make({ message: `compile: clean after ${round - 1} round(s)` }));
+                return;
+            }
+            if (previous !== undefined && found.length >= previous) {
+                flat += 1;
+                if (flat >= 2) {
+                    return yield* Stalled.make({
+                        signal: "no-progress",
+                        detail: `compile: ${found.length} diagnostic(s) after round ${round - 1}, no lower than the round before`
+                    });
+                }
+            }
+            else {
+                flat = 0;
+            }
+            previous = found.length;
+            yield* events.publish(Info.make({
+                message: `compile: round ${round}, ${found.length} diagnostic(s) in ${groups.length} unit(s)`
+            }));
+            const finished = yield* Ref.make(new Set());
+            const units = groups.map((group) => ({
+                id: group.unit,
+                diagnostics: group.diagnostics
+            }));
+            yield* runQueue({
+                label: `compile round ${round}`,
+                items: units,
+                done: (unit) => Effect.map(Ref.get(finished), (set) => set.has(unit.id)),
+                concurrency: knobs.concurrency,
+                maxRounds: 1,
+                events,
+                ledger: { files, path: join(stateDir, "compile-ledger.jsonl") },
+                work: (unit) => Effect.gen(function* () {
+                    const chat = yield* makeChat(context.coder, {
+                        system: compileFixerSystem(porting),
+                        events,
+                        agent: "coder",
+                        stall: { repeats: 5 }
+                    });
+                    yield* chat.ask([
+                        `Unit \`${unit.id}\`: ${unit.diagnostics.length} diagnostic(s) from \`${diagnostics.command.join(" ")}\`.`,
+                        "Fix them in this unit's files. This file is your only runtime evidence; do not rebuild.",
+                        "",
+                        renderDiagnostics(unit.diagnostics)
+                    ].join("\n"));
+                    yield* reviewAndFixLoop({
+                        reviewers: [adversarialReviewer, ...porting.pack.lenses],
+                        reviewerService: context.reviewers[0] ?? context.reasoning,
+                        coder: chat,
+                        taskTitle: `compile ${unit.id}`,
+                        currentDiff: context.git.diffAll,
+                        events,
+                        maxRounds: 2,
+                        votes: knobs.votes
+                    });
+                    yield* Ref.update(finished, (set) => new Set([...set, unit.id]));
+                    return { note: `${unit.diagnostics.length} diagnostic(s) addressed` };
+                })
+            }).pipe(
+            // A unit the fixer could not finish is the next survey's business.
+            Effect.catchTag("Stalled", () => Effect.void));
+            const dirty = yield* context.git.uncommittedFiles;
+            if (dirty.length > 0) {
+                yield* context.git.commitAll(`compile: round ${round}, ${units.length} unit(s)`);
+            }
+        }
+        yield* events.publish(Info.make({
+            message: `compile: ${maxRounds} round(s) done, diagnostics remain — see ${stateDir}`
+        }));
+    }));
+});
+runFlowMain(program);

package/flows/port-files.js

@@ -0,0 +1,173 @@
+// Port a code base file by file (ADR 0028, the Bun port's Phase A): every
+// source the pack matches gets a draft at its target path, written by one
+// implementer that reads the rulebook, the pitfall cards and exactly one
+// source file; reviewed by two adversarial votes; fixed by a separate fixer;
+// ending in a machine-readable PORT STATUS trailer the ledger records. Not
+// clean-room: the source stays in the tree as the spec. Resumable: a target
+// that exists is done.
+//
+//   LLM4TS_PACK=zig-rust llm4ts run port-files --repo ~/src/bun
+//   LLM4TS_PORT_PILOT=3 … first: three files, a pilot report, an approval
+//
+// Knobs: LLM4TS_PORT_PILOT, LLM4TS_PORT_CONCURRENCY (4), LLM4TS_PORT_BATCH
+// (100), LLM4TS_REVIEW_VOTES (2 here), LLM4TS_PORT_SOURCE_CHARS (120000).
+import { join } from "node:path";
+import * as Clock from "effect/Clock";
+import * as Effect from "effect/Effect";
+import { ApprovedMarker, requireApproval } from "@llm4ts/flow/Approval";
+import { makeChat } from "@llm4ts/flow/Chat";
+import { Info } from "@llm4ts/flow/FlowEvents";
+import { asAddedFileDiff, batchesOf, ledgerRowsFor, portStatusIn, renderPilotReport } from "@llm4ts/flow/Port";
+import { adversarialReviewer, reviewAndFixLoop } from "@llm4ts/flow/Review";
+import { runQueue } from "@llm4ts/flow/WorkQueue";
+import { asReadOnly, coderFromEnv, makeNodeWorkspace, nodePlainFileStore, openPack, resolveFlowInput, runFlowMain, runNode, stage } from "@llm4ts/runner";
+import { asPortingPack, implementerPrompt, implementerSystem, ledgerPath, portEnv, readLedgerRows, portFixerSystem, portManifest, portStateDir } from "./lib/port.js";
+const renderReport = (outcomes, total, pending) => {
+    const done = outcomes.filter((outcome) => outcome.status === "done");
+    const failed = outcomes.filter((outcome) => outcome.status === "failed");
+    const by = (level) => done.filter((outcome) => outcome.confidence === level).length;
+    const low = done.filter((outcome) => outcome.confidence === "low");
+    return [
+        "# Port report",
+        "",
+        `- sources: ${total}; drafted this run: ${done.length}; failed: ${failed.length}; still pending: ${pending}`,
+        `- confidence: ${by("high")} high, ${by("medium")} medium, ${by("low")} low; TODO(port) markers: ${done.reduce((sum, outcome) => sum + (outcome.todos ?? 0), 0)}`,
+        "",
+        ...(low.length === 0
+            ? []
+            : [
+                "## Re-read against the source first (confidence: low)",
+                "",
+                ...low.map((outcome) => `- ${outcome.id}${outcome.note === undefined ? "" : ` — ${outcome.note}`}`),
+                ""
+            ]),
+        ...(failed.length === 0
+            ? []
+            : ["## Failed", "", ...failed.map((outcome) => `- ${outcome.id}: ${outcome.note ?? ""}`), ""])
+    ].join("\n");
+};
+const program = Effect.gen(function* () {
+    const input = yield* resolveFlowInput("Port every source file the pack matches to its target");
+    const coder = coderFromEnv(process.env);
+    const files = nodePlainFileStore;
+    const knobs = portEnv(process.env);
+    const stateDir = portStateDir(input.workDir);
+    const pilotPath = join(stateDir, "pilot.md");
+    yield* runNode({
+        workDir: input.workDir,
+        workspace: input.workspace,
+        userPrompt: input.prompt,
+        coder,
+        reasoning: asReadOnly(coder),
+        reviewers: [asReadOnly(coder)],
+        environment: process.env
+    }, (context) => Effect.gen(function* () {
+        const events = context.events;
+        const repo = yield* makeNodeWorkspace(input.workDir);
+        const opened = yield* stage(events, "pack", openPack({
+            environment: process.env,
+            launchDir: input.workspace,
+            flowDir: import.meta.dirname
+        }));
+        const porting = yield* asPortingPack(opened);
+        const manifest = yield* stage(events, "manifest", portManifest(repo, files, input.workDir, porting));
+        const exists = (target) => Effect.map(files.read(join(input.workDir, target)), (text) => text !== undefined);
+        const pending = [];
+        for (const entry of manifest) {
+            if (!(yield* exists(entry.target)))
+                pending.push(entry);
+        }
+        yield* events.publish(Info.make({
+            message: `port: ${manifest.length} source(s), ${manifest.length - pending.length} already ported, ${pending.length} pending (pack ${porting.pack.name}, target ${porting.target})`
+        }));
+        if (knobs.pilot === 0 && (yield* files.read(pilotPath)) !== undefined) {
+            // A pilot ran: the rest of the port waits for its approval (R7).
+            yield* requireApproval(files, pilotPath);
+        }
+        const items = knobs.pilot > 0 ? pending.slice(0, knobs.pilot) : pending;
+        // A ledger written by port-ledger rides into every implementer prompt.
+        const ledger = yield* readLedgerRows(files, input.workDir, porting);
+        if (ledger.length > 0) {
+            yield* events.publish(Info.make({
+                message: `port: ${ledger.length} ledger row(s) from ${ledgerPath(porting)}`
+            }));
+        }
+        const batches = batchesOf(items, { files: knobs.batch });
+        const started = yield* Clock.currentTimeMillis;
+        const outcomes = [];
+        for (const [index, batch] of batches.entries()) {
+            const report = yield* runQueue({
+                label: `port batch ${index + 1}/${batches.length}`,
+                items: batch,
+                done: (entry) => exists(entry.target),
+                concurrency: knobs.concurrency,
+                maxRounds: 2,
+                events,
+                ledger: { files, path: join(stateDir, "ledger.jsonl") },
+                afterRound: (round, done) => done.length === 0
+                    ? Effect.void
+                    : Effect.asVoid(context.git.commitPaths(`port: ${done.length} file(s) drafted (batch ${index + 1}, round ${round})`, done.map((entry) => entry.target))),
+                work: (entry) => Effect.gen(function* () {
+                    const source = (yield* files.read(join(input.workDir, entry.source))) ?? "";
+                    const chat = yield* makeChat(context.coder, {
+                        system: implementerSystem(porting),
+                        events,
+                        agent: "coder",
+                        stall: { repeats: 5 }
+                    });
+                    yield* chat.ask(implementerPrompt(entry, source, porting, knobs.sourceChars, ledgerRowsFor(ledger, entry.source)));
+                    const targetPath = join(input.workDir, entry.target);
+                    if ((yield* files.read(targetPath)) === undefined) {
+                        return { note: `nothing written at ${entry.target}` };
+                    }
+                    yield* reviewAndFixLoop({
+                        reviewers: [adversarialReviewer, ...porting.pack.lenses],
+                        reviewerService: context.reviewers[0] ?? context.reasoning,
+                        coder: chat,
+                        taskTitle: `port ${entry.source}`,
+                        currentDiff: Effect.map(files.read(targetPath), (text) => asAddedFileDiff(entry.target, text ?? "")),
+                        events,
+                        maxRounds: 2,
+                        votes: knobs.votes,
+                        fixWith: (prompt) => Effect.flatMap(makeChat(context.coder, {
+                            system: portFixerSystem(porting),
+                            events,
+                            agent: "coder",
+                            stall: { repeats: 5 }
+                        }), (fixer) => fixer.ask(`${prompt}\n\nThe ported file is ${entry.target}; its source is ${entry.source}.`))
+                    });
+                    const status = portStatusIn((yield* files.read(targetPath)) ?? "");
+                    return {
+                        ...(status.confidence === undefined ? {} : { confidence: status.confidence }),
+                        ...(status.todos === undefined ? {} : { todos: status.todos }),
+                        ...(status.notes === undefined || status.notes === "none"
+                            ? {}
+                            : { note: status.notes })
+                    };
+                })
+            });
+            outcomes.push(...report.outcomes);
+        }
+        const stillPending = pending.length - outcomes.filter((outcome) => outcome.status === "done").length;
+        yield* files.writeAtomic(join(stateDir, "report.md"), renderReport(outcomes, manifest.length, stillPending));
+        if (knobs.pilot > 0) {
+            const pilot = renderPilotReport({
+                outcomes,
+                piloted: items.length,
+                remaining: pending.length - items.length,
+                elapsedMs: (yield* Clock.currentTimeMillis) - started,
+                estimatedCostUsd: undefined
+            });
+            yield* files.writeAtomic(pilotPath, pilot);
+            yield* events.publish(Info.make({
+                message: `port: pilot of ${items.length} file(s) done — read ${pilotPath}, tick \`${ApprovedMarker}\`, then run again without LLM4TS_PORT_PILOT to port the remaining ${pending.length - items.length}`
+            }));
+        }
+        else {
+            yield* events.publish(Info.make({
+                message: `port: ${outcomes.filter((o) => o.status === "done").length} drafted, ${stillPending} still pending — report at ${join(stateDir, "report.md")}`
+            }));
+        }
+    }));
+});
+runFlowMain(program);