@alecsibilia/luca 14.0.0-alpha.2 → 14.0.0-alpha.21

package/board/server/board-server-engine.ts

@@ -0,0 +1,129 @@
+/**
+ * Running the engine for a run: the environment it gets, finding it from
+ * the settings, the one-at-a-time lock on engine work, and spawning it with
+ * `--resume`.
+ */
+import type { RunStore } from './board-server-store'
+import type { BoardServerConfig, RunMemory } from './board-server-types'
+import { errorText } from './board-server-util'
+import { resolveEngine } from './engine-launch'
+import { engineStoppedText, resumeArgs } from './engine-watch'
+
+import {
+    EngineSettingsSchema,
+    type EngineSettings,
+} from '../shared/engine-settings'
+
+export const createEngineProcess = ({
+    config,
+    store,
+}: {
+    config: BoardServerConfig
+    store: RunStore
+}) => {
+    const {
+        env,
+        read_settings,
+        log,
+        home_dir,
+        file_exists,
+        real_path,
+        spawn_engine,
+    } = config
+    const { enqueue, endRun } = store
+
+    /** The daemon's env for the engine, plus the run's token when given. */
+    const childEnv = ({
+        token,
+    }: {
+        token: string | null
+    }): Record<string, string> => {
+        const child_env: Record<string, string> = {}
+        for (const [key, value] of Object.entries(env)) {
+            if (value !== undefined) child_env[key] = value
+        }
+        if (token !== null) child_env.LUCA_BOARD_TOKEN = token
+        return child_env
+    }
+
+    /** Finds the engine from the current settings, as `run.start` does. */
+    const findEngine = async () => {
+        let settings: EngineSettings = EngineSettingsSchema.parse({})
+        try {
+            settings = await read_settings()
+        } catch (error) {
+            log(`Couldn't read the engine settings: ${errorText({ error })}`)
+        }
+        return resolveEngine({
+            settings,
+            env,
+            home_dir,
+            file_exists,
+            real_path,
+        })
+    }
+
+    /**
+     * Engine checks and resumes run one at a time, so a resume and a check
+     * never both start an engine for the same run.
+     */
+    let engine_work: Promise<unknown> = Promise.resolve()
+    const exclusive = <Result>(work: () => Promise<Result>) => {
+        const result = engine_work.then(work)
+        engine_work = result.catch(() => undefined)
+        return result
+    }
+
+    /** Spawns a run's engine with `--resume` and its token; returns its pid. */
+    const spawnResume = ({
+        memory,
+        engine,
+        on_failed,
+    }: {
+        memory: RunMemory
+        engine: { command: string; lead_args: string[] }
+        /** What the log says when the process fails after the spawn. */
+        on_failed: string
+    }): number | null => {
+        const { run_id, repo, token, log_path } = memory.entry
+        return spawn_engine({
+            command: engine.command,
+            args: [...engine.lead_args, ...resumeArgs({ run_id, repo })],
+            env: childEnv({ token }),
+            cwd: repo,
+            log_path,
+            on_error: (error) => {
+                log(`[${run_id}] ${on_failed}: ${errorText({ error })}`)
+                void enqueue({
+                    run_id,
+                    work: () =>
+                        endRun({
+                            memory,
+                            ended: {
+                                ok: false,
+                                message: engineStoppedText({
+                                    why: {
+                                        kind: 'spawn_failed',
+                                        error: errorText({ error }),
+                                    },
+                                    run_id,
+                                    log_path,
+                                }),
+                            },
+                        }),
+                })
+            },
+        }).pid
+    }
+
+    return {
+        childEnv,
+        findEngine,
+        exclusive,
+        spawnResume,
+        /** The engine work in flight (or done), to wait for. */
+        working: () => engine_work,
+    }
+}
+
+export type EngineProcess = ReturnType<typeof createEngineProcess>

package/board/server/board-server-events.ts

@@ -0,0 +1,134 @@
+/**
+ * `engine.event`: applies a run's journal records, sent by the engine, in
+ * seq order, and adds the rows they make.
+ */
+import { timingSafeEqual } from 'node:crypto'
+
+import type { RunStore } from './board-server-store'
+import type { RunMemory } from './board-server-types'
+import { applyEnded } from './reduce-board'
+
+import type { BoardRow } from '../shared/board-rows'
+import type {
+    EngineEventInput,
+    EngineEventOutput,
+    EngineRecord,
+} from '../shared/board-rpc'
+
+const sameToken = ({ left, right }: { left: string; right: string }) => {
+    const a = Buffer.from(left)
+    const b = Buffer.from(right)
+    return a.length === b.length && timingSafeEqual(a, b)
+}
+
+export const createEngineEvents = ({ store }: { store: RunStore }) => {
+    const {
+        runs,
+        enqueue,
+        applyInOrder,
+        appendRows,
+        header,
+        updateEntry,
+        ready,
+    } = store
+
+    /**
+     * The kept rows of records rebuilt from disk that the engine sent again.
+     * The engine sends from the first record it hasn't had an answer for, so
+     * the chat already has the rows of every lower seq: those are dropped.
+     */
+    const takeUnsentRows = ({
+        memory,
+        records,
+    }: {
+        memory: RunMemory
+        records: EngineRecord[]
+    }): { rows: BoardRow[]; taken: number } => {
+        const rows: BoardRow[] = []
+        let taken = 0
+        if (memory.unsent_rows.size === 0 || records.length === 0) {
+            return { rows, taken }
+        }
+        const lowest = Math.min(...records.map((record) => record.seq))
+        const sent = new Set(records.map((record) => record.seq))
+        for (const [seq, seq_rows] of memory.unsent_rows) {
+            if (seq < lowest) {
+                memory.unsent_rows.delete(seq)
+            } else if (sent.has(seq)) {
+                rows.push(...seq_rows)
+                memory.unsent_rows.delete(seq)
+                taken += 1
+            }
+        }
+        return { rows, taken }
+    }
+
+    /**
+     * `engine.event`: applies a run's journal records in seq order from its
+     * `next_seq`. Lower seqs are duplicates and skipped (but a record rebuilt
+     * from disk adds its rows the first time the engine sends it); a gap
+     * stops there and the reply's `next_seq` asks the engine to resend from it.
+     */
+    const handleEngineEvent = async (
+        input: EngineEventInput
+    ): Promise<EngineEventOutput> => {
+        await ready
+        // A run this plugin didn't start is read-only: it isn't in `runs`.
+        const memory = runs.get(input.run_id)
+        if (!memory) {
+            return {
+                ok: false,
+                next_seq: 0,
+                message: `The board doesn't know run ${input.run_id}.`,
+            }
+        }
+        if (!sameToken({ left: input.token, right: memory.entry.token })) {
+            return {
+                ok: false,
+                next_seq: 0,
+                message: `Wrong token for run ${input.run_id}.`,
+            }
+        }
+        return enqueue({
+            run_id: input.run_id,
+            work: async () => {
+                const unsent = takeUnsentRows({
+                    memory,
+                    records: input.records,
+                })
+                const {
+                    rows: new_rows,
+                    applied,
+                    gap,
+                } = applyInOrder({
+                    memory,
+                    records: input.records,
+                })
+                const rows = [...unsent.rows, ...[...new_rows.values()].flat()]
+                if (gap === null && input.ended) {
+                    memory.state = applyEnded({
+                        state: memory.state,
+                        ended: input.ended,
+                    })
+                    memory.unsent_rows.clear()
+                    updateEntry({ memory, change: { ended: input.ended } })
+                }
+                if (
+                    unsent.taken > 0 ||
+                    applied > 0 ||
+                    (gap === null && input.ended)
+                ) {
+                    rows.push(header({ memory }))
+                }
+                await appendRows({ memory, rows })
+                const message =
+                    gap === null
+                        ? `Applied ${applied} records.`
+                        : `Applied ${applied} records, then found seq ${gap} while waiting for ${memory.next_seq}: resend from ${memory.next_seq}.`
+                return { ok: true, next_seq: memory.next_seq, message }
+            },
+        })
+    }
+
+    return { handleEngineEvent }
+}

package/board/server/board-server-read.ts

@@ -0,0 +1,190 @@
+/**
+ * What the side panel reads and posts: `board.read` (the runs started from
+ * a workspace's chats and the runs the plugin didn't start, read from their
+ * journals), and `reply.post`.
+ */
+import { resolve } from 'node:path'
+
+import { z } from 'zod'
+
+import type { RunStore } from './board-server-store'
+import type { BoardServerConfig, OutsideRun } from './board-server-types'
+import { readLiveFile, withLive } from './live-file'
+import { createBoardState } from './reduce-board'
+import type { ReplyRun } from './reply-post'
+import { listJournals } from './run-journals'
+
+import type {
+    BoardReadInput,
+    BoardReadOutput,
+    EngineRecord,
+    ReplyPostInput,
+    ReplyPostOutput,
+} from '../shared/board-rpc'
+import { needsEngine } from '../shared/board-state'
+
+const RunRepoSchema = z.object({ repo: z.string().min(1) })
+
+/** The repo a run's `run_started` record names, if any. */
+const repoOf = ({ records }: { records: EngineRecord[] }): string | null => {
+    const started = records.find((record) => record.kind === 'run_started')
+    const parsed = RunRepoSchema.safeParse(started?.content)
+    return parsed.success ? resolve(parsed.data.repo) : null
+}
+
+/** How many runs the plugin didn't start it shows, the latest changed first. */
+const OUTSIDE_RUNS_SHOWN = 50
+
+export const createBoardReads = ({
+    config,
+    store,
+}: {
+    config: BoardServerConfig
+    store: RunStore
+}) => {
+    const { runs_dir = null } = config
+    const { runs, poster, applyInOrder, journalOf, ready } = store
+    const outside = new Map<string, OutsideRun>()
+
+    /**
+     * Reads the journals of runs this plugin didn't start, the latest
+     * changed first: a new or changed journal is read again and its new
+     * records applied.
+     */
+    const readOutside = async () => {
+        if (runs_dir === null) return
+        const journals = (await listJournals({ runs_dir }))
+            .filter((journal) => !runs.has(journal.run_id))
+            .slice(0, OUTSIDE_RUNS_SHOWN)
+        const shown = new Set(journals.map((journal) => journal.run_id))
+        for (const run_id of outside.keys()) {
+            if (!shown.has(run_id)) outside.delete(run_id)
+        }
+        await Promise.all(
+            journals.map(async ({ run_id, size, changed_ms }) => {
+                const known = outside.get(run_id)
+                if (known?.size === size && known.changed_ms === changed_ms) {
+                    return
+                }
+                const records = await journalOf({ run_id })
+                const first = records[0]
+                if (!first) return
+                const run: OutsideRun = known ?? {
+                    state: createBoardState({
+                        run_id,
+                        spec_number: null,
+                        demo: false,
+                        started_at: first.time,
+                        log_path: null,
+                    }),
+                    next_seq: 1,
+                    size,
+                    changed_ms,
+                    repo: null,
+                    engine_gone: false,
+                }
+                applyInOrder({ memory: run, records })
+                run.size = size
+                run.changed_ms = changed_ms
+                run.repo ??= repoOf({ records })
+                outside.set(run_id, run)
+            })
+        )
+    }
+
+    /**
+     * `board.read`: the runs started from the workspace's chats and the runs
+     * this plugin didn't start whose repo is the workspace's folder, newest
+     * first, and one of them in full, with its live file laid over it.
+     */
+    const readBoard = async (
+        input: BoardReadInput
+    ): Promise<BoardReadOutput> => {
+        await ready
+        await readOutside()
+        const directory = input.directory ? resolve(input.directory) : null
+        const list = [
+            ...[...runs.values()].filter(
+                (memory) => memory.entry.workspace_id === input.workspace_id
+            ),
+            ...[...outside.values()]
+                .filter((run) => directory !== null && run.repo === directory)
+                .map((run) => ({
+                    state: {
+                        ...run.state,
+                        engine_gone:
+                            run.engine_gone &&
+                            needsEngine({ state: run.state }),
+                    },
+                })),
+        ]
+            .map(({ state }) => state)
+            .toSorted((left, right) =>
+                right.run.started_at === left.run.started_at
+                    ? right.run.run_id.localeCompare(left.run.run_id)
+                    : right.run.started_at.localeCompare(left.run.started_at)
+            )
+        const selected =
+            (input.run_id
+                ? list.find((state) => state.run.run_id === input.run_id)
+                : undefined) ?? list[0]
+        const selected_id = selected?.run.run_id ?? null
+        const live =
+            runs_dir === null || selected_id === null
+                ? null
+                : await readLiveFile({ runs_dir, run_id: selected_id })
+        return {
+            runs: list.map((state) => ({
+                run_id: state.run.run_id,
+                spec_number: state.run.spec_number,
+                spec_title: state.run.spec_title,
+                demo: state.run.demo,
+                status: state.run.status,
+                started_at: state.run.started_at,
+                needs_you: state.needs_you.length,
+            })),
+            selected:
+                selected === undefined
+                    ? null
+                    : withLive({ state: selected, live }),
+            chat_agent_id:
+                selected_id === null
+                    ? null
+                    : (runs.get(selected_id)?.entry.agent_id ?? null),
+            posted:
+                selected_id === null
+                    ? []
+                    : poster.postedOf({ run_id: selected_id }),
+        }
+    }
+
+    /**
+     * `reply.post` (#503): posts a reply button's word on the run's spec
+     * issue. Works for runs this plugin started and for runs it only reads
+     * (their journal is read again first); see `createReplyPoster`.
+     */
+    const postReply = async (
+        input: ReplyPostInput
+    ): Promise<ReplyPostOutput> => {
+        await ready
+        await readOutside()
+        const memory = runs.get(input.run_id)
+        const other = outside.get(input.run_id)
+        const run: ReplyRun | null = memory
+            ? {
+                  state: memory.state,
+                  repo_dir: memory.entry.repo,
+                  next_seq: memory.next_seq,
+              }
+            : other
+              ? {
+                    state: other.state,
+                    repo_dir: other.repo,
+                    next_seq: other.next_seq,
+                }
+              : null
+        return poster.postReply({ run, input })
+    }
+
+    return { outside, readOutside, readBoard, postReply }
+}

package/board/server/board-server-resume.ts

@@ -0,0 +1,164 @@
+/**
+ * `/luca-run resume <run id>` (#493): going on with a run this plugin
+ * started, from its journal.
+ */
+import { randomBytes } from 'node:crypto'
+
+import type { EngineProcess } from './board-server-engine'
+import type { RunStore } from './board-server-store'
+import type { BoardServerConfig } from './board-server-types'
+import { errorText } from './board-server-util'
+import { liveRunIds, RESUMED_TEXT, resumeRow } from './engine-watch'
+
+import type { RunStartInput, RunStartOutput } from '../shared/board-rpc'
+import { OVER_PHASES } from '../shared/board-state'
+
+export const createResumer = ({
+    config,
+    store,
+    engine,
+}: {
+    config: BoardServerConfig
+    store: RunStore
+    engine: EngineProcess
+}) => {
+    const { list_processes, log, now } = config
+    const { runs, enqueue, appendRows, header, updateEntry } = store
+    const { findEngine, exclusive, spawnResume } = engine
+
+    /**
+     * `/luca-run resume <run id>` (#493): goes on with a run this plugin
+     * started, from its journal, with the board attached again. Refused for
+     * an unknown run, a demo, a run that is over, a run whose engine is
+     * still running, or when `ps` fails. Otherwise the run gets a new token,
+     * loses its `ended`, and starts its restart count again at 0; it keeps
+     * its chat, workspace, repo, and log. The engine is spawned with
+     * `--resume`, and the run's chat gets a row saying so.
+     */
+    const resumeRun = async ({
+        run_id,
+        input,
+    }: {
+        run_id: string
+        input: RunStartInput
+    }): Promise<RunStartOutput> => {
+        const refuse = (message: string): RunStartOutput => ({
+            ok: false,
+            message,
+            run_id: null,
+        })
+        const memory = runs.get(run_id)
+        if (!memory) {
+            return refuse(
+                `The board doesn't know run ${run_id}, so it can't resume it. Check the id: the run's header row and the board panel show it. A run the board didn't start goes on from a terminal with luca-run --resume ${run_id}, without the board.`
+            )
+        }
+        if (memory.entry.demo) {
+            return refuse(
+                `Run ${run_id} is a demo, and a demo can't be picked up again. Start a new one with /luca-run demo.`
+            )
+        }
+        if (OVER_PHASES.has(memory.state.run.phase)) {
+            return refuse(
+                `Run ${run_id} is over, so there is nothing to resume.`
+            )
+        }
+        const engine = await findEngine()
+        if (!engine.ok) return refuse(engine.message)
+
+        return exclusive(async () => {
+            let command_lines: string[]
+            try {
+                command_lines = await list_processes()
+            } catch (error) {
+                return refuse(
+                    `Couldn't check whether run ${run_id}'s engine is still running (${errorText({ error })}), so it wasn't resumed: two engines on one run would be far worse. Try again.`
+                )
+            }
+            if (liveRunIds({ command_lines }).has(run_id)) {
+                return refuse(
+                    `Run ${run_id} is still running (its engine is live), so there is nothing to resume.`
+                )
+            }
+            return enqueue({
+                run_id,
+                work: async (): Promise<RunStartOutput> => {
+                    const before = memory.entry
+                    // The new token is kept first: the engine may send
+                    // before the spawn returns.
+                    updateEntry({
+                        memory,
+                        change: {
+                            token: randomBytes(24).toString('hex'),
+                            ended: null,
+                            restarts: 0,
+                        },
+                    })
+                    let pid: number | null = null
+                    try {
+                        pid = spawnResume({
+                            memory,
+                            engine,
+                            on_failed: 'the resumed engine failed',
+                        })
+                    } catch (error) {
+                        updateEntry({
+                            memory,
+                            change: {
+                                token: before.token,
+                                ended: before.ended,
+                                restarts: before.restarts,
+                            },
+                        })
+                        return refuse(
+                            `Couldn't start the engine: ${errorText({ error })}`
+                        )
+                    }
+                    const { log_path, spec, agent_id } = memory.entry
+                    log(
+                        `[${run_id}] resumed with /luca-run resume: ${engine.command} with --resume (pid ${pid ?? '?'}), log ${log_path}`
+                    )
+                    memory.rows_stopped = false
+                    memory.append_failures = 0
+                    memory.state = {
+                        ...memory.state,
+                        run: { ...memory.state.run, engine_ended: null },
+                        latest: RESUMED_TEXT,
+                        latest_at: now().toISOString(),
+                    }
+                    void enqueue({
+                        run_id,
+                        work: () =>
+                            appendRows({
+                                memory,
+                                rows: [
+                                    resumeRow({
+                                        run_id,
+                                        time: now().toISOString(),
+                                    }),
+                                    header({ memory }),
+                                ],
+                            }),
+                    })
+                    const what =
+                        spec === null
+                            ? `run ${run_id}`
+                            : `spec #${spec} (run ${run_id})`
+                    const where =
+                        agent_id === input.agent_id
+                            ? ''
+                            : ' Its rows go on in the chat it started in.'
+                    return {
+                        ok: true,
+                        run_id,
+                        message: `Resumed ${what} from its journal.${where} Its log is ${log_path}.`,
+                    }
+                },
+            })
+        })
+    }
+
+    return { resumeRun }
+}
+
+export type Resumer = ReturnType<typeof createResumer>