@shardflux/cli 0.15.2 → 0.16.0

package/dist/worktree-twin-engine.js

@@ -0,0 +1,402 @@
+/**
+ * The device helper's live twin loop (0.16.0+): watches the API (conversations and agent sessions, never a VM) for
+ * conversations whose turn ended with committed work and lands each into the local branch it started from, with no
+ * input. One landing at a time, in the order the turns finished; anything unsafe pauses and retries. A conversation
+ * that ended while this computer was away (its VM reclaimed) lands from the project hub. After being offline or asleep
+ * the loop retries at once (runTwinLoop). See worktree-twin.ts.
+ */
+import { connect } from 'node:net';
+import { networkInterfaces } from 'node:os';
+import { existsSync } from 'node:fs';
+import { setTimeout as delay } from 'node:timers/promises';
+import { out } from "./commands.js";
+import { conversationClient } from "./conversation-commands.js";
+import { conversationName, conversationsUnavailable, listConversations, localConversations } from "./conversations.js";
+import { UsageError } from "./errors.js";
+import { projectIdentity } from "./project-identity.js";
+import { TwinLedger, backupRoot, inspectWorktree, landingRound, listWorktrees, pauseReason, restoreBackup } from "./worktree-twin.js";
+import { readdirSync, readFileSync } from 'node:fs';
+import { join } from 'node:path';
+/** Its VM is gone: the conversation ended (archived after the retention window, or done). */
+export const ended = (c) => !!c.ended_at || c.state === 'archived';
+const revision = (c) => JSON.stringify([c.changes, c.summary]);
+/** A turn ended with committed work only; the agent is not asking anything. */
+export function finished(c, agents) {
+    if (ended(c))
+        return 'state';
+    const agent = agents.find(a => a.workspace_id === c.workspace_id && c.harness_session_id && a.id === `${c.workspace_id}:${c.harness_session_id}`);
+    if (agent)
+        return agent.state === 'working' ? null : agent;
+    return c.state === 'unresolved' || c.state === 'done' ? 'state' : null;
+}
+export function eligible(c, ledger, now) {
+    if (c.route === 'codex-cloud' || c.join_mode === 'reviewer' || c.state === 'needs_you')
+        return false;
+    if (!c.changes || c.changes.commits <= 0 || c.changes.uncommitted || c.landed?.how === 'discarded')
+        return false;
+    const st = ledger.conversation(c.id);
+    // An ended conversation is landed here unless someone landed it some other way before it ended.
+    if (ended(c) && c.landed && !st.landed_revision)
+        return false;
+    if ((st.retry_at ?? 0) > now)
+        return false;
+    if (st.landed_revision === revision(c) && (ended(c) || c.landed?.how === 'merged' || c.landed?.how === 'pr'))
+        return false;
+    return st.conflict_revision !== revision(c);
+}
+export function conflictMessage(into, files) {
+    const list = files.length > 8 ? `${files.slice(0, 8).join(', ')} and ${files.length - 8} more` : files.join(', ');
+    return `Shardflux could not land your commits into ${into} on the user's computer: ${list} also changed in the user's newer commits on ${into}. `
+        + `Merge their branch into yours and commit: git merge refs/shard/twin-in^1 (refs/shard/twin-in is the user's working state as of now; its first parent is the tip of their ${into}). `
+        + `Resolve each conflict without asking: where you and the user changed the same lines, keep the user's lines and keep your change only where it fits around them; then run the checks and commit. `
+        + `Your work lands on the user's computer automatically when you finish. Ask the user only if the result cannot work either way.`;
+}
+export class TwinEngine {
+    #deps;
+    #reported = new Map();
+    #posted = new Map();
+    constructor(deps) { this.#deps = deps; }
+    #report(key, line) { if (this.#reported.get(key) !== line) {
+        this.#reported.set(key, line);
+        this.#deps.log(line);
+    } }
+    /** One pass over every project with a local conversation; returns the suggested wait before the next. */
+    async tick(signal) {
+        const d = this.#deps, records = d.records().filter(r => !r.handed_off);
+        let wait = 1000;
+        for (const project of new Set(records.map(r => r.project_id))) {
+            signal?.throwIfAborted();
+            const mode = d.ledger.mode(project);
+            if (mode === 'off')
+                continue;
+            const mine = new Map(records.filter(r => r.project_id === project).map(r => [r.conversation.id, r]));
+            const candidates = (await d.conversations(project)).filter(c => mine.has(c.id) && eligible(c, d.ledger, d.now()));
+            if (!candidates.length)
+                continue;
+            const agents = candidates.some(c => !ended(c)) ? await d.agents(project) : [];
+            // In the order the turns finished (several may have finished while this computer was away); each lands against
+            // the checkout as it is when its turn comes.
+            const ready = candidates.map(c => ({ c, turn: finished(c, agents) })).filter((x) => x.turn !== null)
+                .map(x => ({ ...x, at: x.turn === 'state' ? Date.parse(x.c.last_activity_at ?? x.c.created_at) : Date.parse(x.turn.since) }))
+                .sort((a, b) => (a.at || 0) - (b.at || 0));
+            for (const { c, turn } of ready) {
+                const next = await this.landOne(c, mine.get(c.id), mode, turn === 'state' ? null : turn, signal);
+                if (next !== undefined)
+                    wait = Math.min(wait, next);
+            }
+        }
+        return wait;
+    }
+    /** Back online or awake: every conversation's retry backoff is cleared, so the next pass tries all of them. */
+    resetBackoff() {
+        this.#deps.ledger.update(doc => { for (const st of Object.values(doc.conversations ?? {})) {
+            st.failures = 0;
+            st.retry_at = 0;
+        } });
+        this.#reported.clear();
+    }
+    async landOne(c, record, mode, agent, signal) {
+        const d = this.#deps, st = d.ledger.conversation(c.id), rev = revision(c);
+        const into = c.base_branch ?? record.base_branch ?? st.base_branch;
+        const name = c.name ?? conversationName(c);
+        const label = `${c.title || name} (${c.id})`;
+        if (!into || into.startsWith('shard/')) {
+            this.#report(c.id, `Twin: ${label} has no local base branch to land into.`);
+            return;
+        }
+        if (!existsSync(record.folder)) {
+            this.#report(c.id, `Twin: ${record.folder} is gone; ${label} stays in the cloud.`);
+            return;
+        }
+        // Landed at this revision already; only the API record is missing (a later report reset it on an older guest).
+        if (st.landed_revision === rev) {
+            if (ended(c))
+                return;
+            if ((this.#posted.get(c.id) ?? 0) + 30_000 < d.now()) {
+                this.#posted.set(c.id, d.now());
+                await d.land(c, st.pr_url ? 'pr' : 'merge', st.pr_url ?? into);
+            }
+            return;
+        }
+        const branch = `refs/heads/${into}`;
+        let worktree = null;
+        const target = (await listWorktrees(record.folder)).find(w => w.branch === branch);
+        if (target) {
+            worktree = await inspectWorktree(target.root);
+            if (pauseReason(worktree))
+                return 1000;
+        }
+        const started = d.now();
+        let vm, fromHub = false;
+        try {
+            vm = ended(c) ? null : await d.cell(c);
+            // The VM is gone (reclaimed while this computer was away): its branch is in the project hub.
+            if (!vm) {
+                vm = await d.hubCell(c.project_id);
+                fromHub = true;
+            }
+        }
+        catch (error) {
+            if (signal?.aborted)
+                throw error;
+            return this.#fail(c, label, error instanceof Error ? error.message : String(error));
+        }
+        let outcome, keep = false;
+        try {
+            for (let attempt = 0; attempt < 3; attempt++) {
+                if (target && attempt)
+                    worktree = await inspectWorktree(target.root);
+                outcome = await landingRound(vm.cell, { conversation: { id: c.id, name, title: c.title, summary: c.summary, branch: c.branch, base_commit: c.base_commit, worktree_path: c.worktree_path },
+                    into, mode: mode === 'pr' ? 'pr' : 'auto', device: d.device, repo: target?.root ?? record.folder, worktree, stateDir: d.stateDir, backupRoot: backupRoot(d.stateDir),
+                    upto: st.upto, known: [st.tip, st.upto].filter((x) => !!x), hub: d.hub, vmTmp: d.vmTmp, quietMs: d.quietMs, signal, now: () => d.now(),
+                    ...(fromHub ? { fromHub: { ...(c.repo_remote ? { repoRemote: c.repo_remote } : {}) } } : {}) });
+                if (outcome.state !== 'moved')
+                    break;
+            }
+            keep = outcome?.state === 'landed';
+        }
+        catch (error) {
+            if (signal?.aborted)
+                throw error;
+            return this.#fail(c, label, error instanceof Error ? error.message : String(error));
+        }
+        finally {
+            if (!keep)
+                vm.close();
+        }
+        const post = async (action, ref) => {
+            // An ended conversation cannot record a landing any more (the API refuses); the ledger keeps it.
+            if (ended(c))
+                return;
+            this.#posted.set(c.id, d.now());
+            await d.land(c, action, ref);
+        };
+        const o = outcome;
+        switch (o.state) {
+            case 'landed': {
+                // The VM records the landing and copies it to the project hub on its own time; the connection closes after.
+                void o.confirmed.catch((error) => { this.#report(c.id, `Twin: ${label} landed; recording it in its VM failed: ${error instanceof Error ? error.message : String(error)}.`); })
+                    .finally(() => vm.close());
+                const latency = agent ? o.applied.at - Date.parse(agent.since) : undefined;
+                d.ledger.patch(c.id, { upto: o.land.upto, tip: o.land.tip, landed_revision: rev, landed_at: new Date(o.applied.at).toISOString(), method: o.land.method,
+                    failures: 0, retry_at: 0, last_error: undefined, ...(latency !== undefined ? { latency_ms: latency } : {}) });
+                await post('merge', into);
+                d.log(`Twin: landed ${label} into ${into} in ${target?.root ?? record.folder}${fromHub ? ' from the project hub' : ''} (${o.land.method}; ${o.applied.files.length} file${o.applied.files.length === 1 ? '' : 's'}`
+                    + `${o.applied.raced.length ? `, kept your version of ${o.applied.raced.join(', ')}` : ''}${o.land.shadowed?.length ? `, your edits win in ${o.land.shadowed.join(', ')}` : ''}`
+                    + `${latency !== undefined ? `; ${(latency / 1000).toFixed(1)} s after the turn ended, round started at ${((started - Date.parse(agent.since)) / 1000).toFixed(1)} s` : ''}`
+                    + `; round ${Object.entries(o.timings).map(([k, v]) => `${k} ${v} ms`).join(', ')}; in the VM ${Object.entries(o.land.phases ?? {}).map(([k, v]) => `${k} ${v}`).join(', ')}).`);
+                return;
+            }
+            case 'noop':
+                d.ledger.patch(c.id, { upto: o.land.upto, landed_revision: rev, failures: 0, retry_at: 0 });
+                if (c.landed?.how !== 'merged')
+                    await post('merge', into);
+                return;
+            case 'pr':
+                d.ledger.patch(c.id, { tip: o.land.tip, landed_revision: rev, pr_url: o.land.url, failures: 0, retry_at: 0 });
+                await post('pr', o.land.url);
+                d.log(`Twin: opened ${o.land.url} for ${label}.`);
+                return;
+            case 'conflict':
+                if (st.conflict_tip !== o.land.conversation_tip) {
+                    // Its agent resolves it; an ended conversation is restored into a fresh VM first.
+                    if (ended(c))
+                        await d.resume(c);
+                    await d.message(c, conflictMessage(into, o.land.conflicts ?? []));
+                }
+                d.ledger.patch(c.id, { conflict_tip: o.land.conversation_tip, conflict_revision: rev });
+                d.log(`Twin: ${label} conflicts with your commits on ${into} (${(o.land.conflicts ?? []).join(', ')}); its agent is resolving it.`);
+                return;
+            case 'busy':
+                d.ledger.patch(c.id, { retry_at: d.now() + (o.waitMs ?? 1000) });
+                return o.waitMs ?? 1000;
+            case 'paused': return 1000;
+            case 'moved':
+                d.ledger.patch(c.id, { retry_at: d.now() + 2000 });
+                return 2000;
+            case 'error': return this.#fail(c, label, `${o.land.reason}: ${o.land.message ?? ''}`);
+        }
+    }
+    #fail(c, label, message) {
+        const d = this.#deps, failures = (d.ledger.conversation(c.id).failures ?? 0) + 1, wait = Math.min(300_000, 5000 * 2 ** Math.min(failures - 1, 6));
+        d.ledger.patch(c.id, { failures, retry_at: d.now() + wait, last_error: message });
+        this.#report(c.id, `Twin: ${label} could not land yet: ${message} (retrying in ${Math.round(wait / 1000)} s).`);
+        return wait;
+    }
+}
+const PROBE_MS = 5000, JUMP_MS = 5000;
+/**
+ * Runs the twin's passes. A failing pass (offline) backs off up to 5 minutes, but the wait ends at once when the
+ * computer wakes from sleep (the wall clock jumps past the monotonic one), when its network changes, or when the API
+ * host answers again (probed every 5 s while failing); every conversation's retry backoff is then cleared and a pass
+ * runs immediately. The first pass that succeeds after failures clears them too and runs again at once.
+ */
+export async function runTwinLoop(deps, signal) {
+    let failures = 0, net = deps.network();
+    const reported = new Set();
+    while (!signal.aborted) {
+        let wait;
+        try {
+            wait = await deps.tick(signal);
+            if (failures) {
+                failures = 0;
+                deps.resetBackoff();
+                wait = 0;
+            }
+        }
+        catch (error) {
+            if (signal.aborted)
+                break;
+            wait = Math.min(300_000, 5000 * 2 ** Math.min(failures++, 6));
+            const message = error instanceof Error ? error.message : String(error);
+            if (!conversationsUnavailable(error) && !reported.has(message)) {
+                reported.add(message);
+                deps.log(`Twin: ${message} (retrying in ${wait / 1000} s).`);
+            }
+        }
+        let left = wait, probed = deps.mono();
+        while (left > 0 && !signal.aborted) {
+            const slice = Math.min(1000, left), wall = deps.wall(), mono = deps.mono();
+            await deps.sleep(slice, signal);
+            left -= slice;
+            const woke = deps.wall() - wall - (deps.mono() - mono) > JUMP_MS;
+            const now = deps.network(), changed = now !== net;
+            net = now;
+            let back = false;
+            if (failures && deps.mono() - probed >= PROBE_MS) {
+                probed = deps.mono();
+                back = await deps.reachable().catch(() => false);
+            }
+            if (woke || changed || back) {
+                deps.resetBackoff();
+                left = 0;
+            }
+        }
+    }
+}
+function networkSignature() {
+    const all = [];
+    for (const [name, list] of Object.entries(networkInterfaces()))
+        for (const a of list ?? [])
+            if (!a.internal)
+                all.push(`${name} ${a.address}`);
+    return all.sort().join(',');
+}
+function reachable(url) {
+    const u = new URL(url), port = Number(u.port || (u.protocol === 'http:' ? 80 : 443));
+    return new Promise(resolve => {
+        const socket = connect({ host: u.hostname, port, timeout: 3000 });
+        const done = (ok) => { socket.destroy(); resolve(ok); };
+        socket.once('connect', () => done(true));
+        socket.once('timeout', () => done(false));
+        socket.once('error', () => done(false));
+    });
+}
+const gone = (error) => error.status === 404 || error.details?.reason === 'workspace_deleted';
+/** The device helper's twin. Nothing here asks the user anything. */
+export async function runWorktreeTwin(ctx, device, signal) {
+    const client = () => conversationClient(ctx);
+    const open = (ws) => {
+        const cell = ws.cell({ agentLabel: 'shard-twin' });
+        return { cell, close: () => cell.close() };
+    };
+    const engine = new TwinEngine({
+        records: () => Object.values(localConversations(ctx)),
+        conversations: async (projectId) => await listConversations(ctx, projectId),
+        agents: async (projectId) => {
+            // A signed-in session without a project key has no agent list here; the conversation state then decides.
+            const api = client(), all = [];
+            if (!('agents' in api))
+                return all;
+            let cursor;
+            do {
+                const page = await api.agents.list({ projectId, limit: 200, ...(cursor ? { cursor } : {}) });
+                all.push(...page.data);
+                cursor = page.nextCursor ?? undefined;
+            } while (cursor);
+            return all;
+        },
+        cell: async (c) => {
+            let ws;
+            try {
+                ws = await client().workspaces.get(c.workspace_id);
+            }
+            catch (error) {
+                if (gone(error))
+                    return null;
+                throw error;
+            }
+            if (ws.data.deleted_at)
+                return null;
+            if (!ws.ready)
+                await ws.wake({ timeoutMs: ctx.wakeTimeoutMs, signal });
+            return open(ws);
+        },
+        hubCell: async (projectId) => {
+            // An awake VM of the project reads the hub without waking anything; else the project's small space reader.
+            const all = (await listConversations(ctx, projectId)).filter(c => !ended(c))
+                .sort((a, b) => (b.last_activity_at ?? '').localeCompare(a.last_activity_at ?? ''));
+            for (const c of all) {
+                try {
+                    const ws = await client().workspaces.get(c.workspace_id);
+                    if (!ws.data.deleted_at && ws.ready)
+                        return open(ws);
+                }
+                catch { /* another may be awake */ }
+            }
+            const reader = await client().conversations.spaceReader(projectId, { signal });
+            const ws = await client().workspaces.get(reader.workspace.id);
+            if (!ws.ready)
+                await ws.wake({ timeoutMs: ctx.wakeTimeoutMs, signal });
+            return open(ws);
+        },
+        resume: async (c) => { await client().conversations.resume(c.id, { signal }); },
+        land: async (c, action, into) => { await client().conversations.land(c.id, action, { into, signal }); },
+        message: async (c, text) => { await client().conversations.send(c.id, text, { signal }); },
+        ledger: new TwinLedger(ctx.store.dir), stateDir: ctx.store.dir, device,
+        log: line => { ctx.io.stderr.write(line + '\n'); }, now: Date.now,
+    });
+    await runTwinLoop({
+        tick: s => engine.tick(s), resetBackoff: () => engine.resetBackoff(),
+        sleep: (ms, s) => delay(ms, undefined, { signal: s }).catch(() => undefined),
+        wall: Date.now, mono: () => performance.now(), network: networkSignature, reachable: () => reachable(ctx.apiUrl),
+        log: line => { ctx.io.stderr.write(line + '\n'); },
+    }, signal);
+}
+const MODES = ['auto', 'pr', 'off'];
+function projectOf(ctx, path) {
+    const folder = projectIdentity(path ?? process.cwd()).folder;
+    const local = Object.values(localConversations(ctx)).find(r => r.folder === folder);
+    const project = local?.project_id ?? ctx.store.profile(ctx.apiUrl).project_id;
+    if (!project)
+        throw new UsageError('No Shardflux project for this checkout: start a conversation here first, or run shard setup.');
+    return project;
+}
+export const TWIN_HANDLERS = {
+    /** shard conversations landing [auto|pr|off|restore <backup>|backups] */
+    async 'conversations landing'(ctx, values, [action, arg]) {
+        const ledger = new TwinLedger(ctx.store.dir), project = projectOf(ctx, typeof values.path === 'string' ? values.path : undefined), root = backupRoot(ctx.store.dir);
+        if (action === 'restore') {
+            if (!arg || !/^[\w.:-]+$/.test(arg))
+                throw new UsageError('Name a backup: shard conversations landing backups');
+            const r = await restoreBackup(join(root, arg), { backupRoot: root });
+            out(ctx, `Restored ${r.restored.length} file${r.restored.length === 1 ? '' : 's'}. The replaced versions are in backup ${r.backup.split('/').at(-1)}.\n`, r);
+            return 0;
+        }
+        if (action === 'backups') {
+            const list = (existsSync(root) ? readdirSync(root) : []).filter(n => existsSync(join(root, n, 'manifest.json'))).sort().reverse()
+                .map(id => { const m = JSON.parse(readFileSync(join(root, id, 'manifest.json'), 'utf8')); return { id, created_at: m.created_at, worktree: m.worktree, title: m.title, files: m.files.map(f => f.path) }; });
+            out(ctx, list.map(b => `${b.id}\t${b.title}\t${b.files.length} files\t${b.worktree}`).join('\n') + (list.length ? '\n' : 'No landing backups.\n'), { backups: list });
+            return 0;
+        }
+        if (action !== undefined) {
+            if (!MODES.includes(action))
+                throw new UsageError('Choose auto, pr or off (or restore <backup>, backups).');
+            ledger.setMode(project, action);
+        }
+        const mode = ledger.mode(project);
+        const text = mode === 'auto' ? 'Finished conversations land in your local branch automatically.' : mode === 'pr' ? 'Finished conversations open a pull request instead of landing locally.' : 'Automatic landing is off for this project.';
+        out(ctx, `${text}\n`, { project_id: project, landing: mode });
+        return 0;
+    },
+};