@alecsibilia/luca 14.0.0-alpha.2 → 14.0.0-alpha.21

package/board/server/board-server-start.ts

@@ -0,0 +1,174 @@
+/**
+ * `run.start`: parses the args, finds the engine, records the run, spawns
+ * the engine detached, and adds the chat's header row.
+ */
+import { randomBytes, randomInt } from 'node:crypto'
+import { join } from 'node:path'
+
+import type { EngineProcess } from './board-server-engine'
+import type { Resumer } from './board-server-resume'
+import type { RunStore } from './board-server-store'
+import type { BoardServerConfig } from './board-server-types'
+import { errorText } from './board-server-util'
+import { engineArgs, mintRunId, parseRunArgs } from './engine-launch'
+import type { RunEntry } from './run-registry'
+
+import type { RunStartInput, RunStartOutput } from '../shared/board-rpc'
+
+/** The longest `run.start` waits for the header row before it returns. */
+const HEADER_WAIT_MS = 3000
+
+const SUFFIX_ALPHABET = 'abcdefghijklmnopqrstuvwxyz0123456789'
+
+const randomSuffix = (): string =>
+    Array.from(
+        { length: 4 },
+        () => SUFFIX_ALPHABET[randomInt(SUFFIX_ALPHABET.length)] ?? 'x'
+    ).join('')
+
+export const createStarter = ({
+    config,
+    store,
+    engine,
+    resumer,
+}: {
+    config: BoardServerConfig
+    store: RunStore
+    engine: EngineProcess
+    resumer: Resumer
+}) => {
+    const { spawn_engine, log, now, log_dir } = config
+    const {
+        runs,
+        registry,
+        queues,
+        remember,
+        enqueue,
+        endRun,
+        appendRows,
+        header,
+        ready,
+    } = store
+    const { childEnv, findEngine } = engine
+    const { resumeRun } = resumer
+
+    /** Waits for rows being added, but at most `HEADER_WAIT_MS`. */
+    const waitForRows = async (adding: Promise<unknown>) => {
+        let timer: ReturnType<typeof setTimeout> | undefined
+        await Promise.race([
+            adding,
+            new Promise((resolve) => {
+                timer = setTimeout(resolve, HEADER_WAIT_MS)
+            }),
+        ])
+        clearTimeout(timer)
+    }
+
+    /**
+     * `run.start`: parses the args, finds the engine, records the run, spawns
+     * the engine detached, and adds the chat's header row. Returns at once.
+     * `resume <run id>` goes on with a run instead (see `resumeRun`).
+     */
+    const startRun = async (input: RunStartInput): Promise<RunStartOutput> => {
+        await ready
+        const parsed = parseRunArgs({ args: input.args })
+        if (!parsed.ok)
+            return { ok: false, message: parsed.message, run_id: null }
+        if (parsed.target.kind === 'resume') {
+            const output = await resumeRun({
+                run_id: parsed.target.run_id,
+                input,
+            })
+            const rows = queues.get(parsed.target.run_id)
+            if (output.ok && rows) await waitForRows(rows)
+            return output
+        }
+
+        const engine = await findEngine()
+        if (!engine.ok)
+            return { ok: false, message: engine.message, run_id: null }
+
+        let run_id = mintRunId({ now: now(), suffix: randomSuffix() })
+        while (runs.has(run_id)) {
+            run_id = mintRunId({ now: now(), suffix: randomSuffix() })
+        }
+        const token = randomBytes(24).toString('hex')
+        const log_path = join(log_dir, `${run_id}.log`)
+        const target = parsed.target
+        const entry: RunEntry = {
+            run_id,
+            token,
+            agent_id: input.agent_id,
+            workspace_id: input.workspace_id,
+            repo: input.cwd,
+            spec: target.kind === 'spec' ? target.spec : null,
+            demo: target.kind === 'demo',
+            started_at: now().toISOString(),
+            log_path,
+            ended: null,
+            restarts: 0,
+        }
+        registry.add({ entry })
+        const memory = remember({ entry, records: [] })
+
+        const child_env = childEnv({ token })
+
+        let pid: number | null = null
+        try {
+            pid = spawn_engine({
+                command: engine.command,
+                args: [
+                    ...engine.lead_args,
+                    ...engineArgs({ target, repo: input.cwd, run_id }),
+                ],
+                env: child_env,
+                cwd: input.cwd,
+                log_path,
+                on_error: (error) => {
+                    log(
+                        `[${run_id}] the engine process failed: ${errorText({ error })}`
+                    )
+                    void enqueue({
+                        run_id,
+                        work: () =>
+                            endRun({
+                                memory,
+                                ended: {
+                                    ok: false,
+                                    message: `The engine couldn't start: ${errorText({ error })}`,
+                                },
+                            }),
+                    })
+                },
+            }).pid
+        } catch (error) {
+            registry.remove({ run_id })
+            runs.delete(run_id)
+            return {
+                ok: false,
+                message: `Couldn't start the engine: ${errorText({ error })}`,
+                run_id: null,
+            }
+        }
+        log(
+            `[${run_id}] started ${engine.command} (pid ${pid ?? '?'}), log ${log_path}`
+        )
+
+        await waitForRows(
+            enqueue({
+                run_id,
+                work: () => appendRows({ memory, rows: [header({ memory })] }),
+            })
+        )
+
+        const what =
+            target.kind === 'demo' ? 'a demo run' : `spec #${target.spec}`
+        return {
+            ok: true,
+            run_id,
+            message: `Started ${what} as ${run_id}. Its log is ${log_path}.`,
+        }
+    }
+
+    return { startRun }
+}

package/board/server/board-server-store.ts

@@ -0,0 +1,281 @@
+/**
+ * The board server's runs in memory: each run the plugin started, its board
+ * built from its journal records, the queue that keeps a run's work in
+ * order, and the rows it adds to the chat.
+ */
+import type { BoardServerConfig, Replay, RunMemory } from './board-server-types'
+import { errorText } from './board-server-util'
+import { readRecord } from './board-vocabulary'
+import { describeRecord, headerRow, rowsForRecord } from './make-rows'
+import { applyEnded, applyRecord, createBoardState } from './reduce-board'
+import { createReplyPoster } from './reply-post'
+import { readJournal } from './run-journals'
+import { createRunRegistry, type RunEntry } from './run-registry'
+
+import type { BoardRow } from '../shared/board-rows'
+import type { EngineRecord } from '../shared/board-rpc'
+import type { EngineEnded } from '../shared/board-state'
+
+/** After this many failed appends in a row, a run stops sending rows. */
+const APPEND_FAILURES_BEFORE_STOP = 3
+
+/**
+ * The runs this plugin started, their queues, and the work on them. With a
+ * `runs_dir`, every run in the registry is rebuilt from its journal there,
+ * and `ready` resolves once that is done.
+ */
+export const createRunStore = ({ config }: { config: BoardServerConfig }) => {
+    const {
+        registry_path,
+        runs_dir = null,
+        append_row,
+        run_command,
+        env,
+        home_dir,
+        file_exists,
+        now,
+        log,
+    } = config
+    const registry = createRunRegistry({ path: registry_path, log })
+    const runs = new Map<string, RunMemory>()
+    const queues = new Map<string, Promise<unknown>>()
+    const poster = createReplyPoster({
+        run_command,
+        env,
+        home_dir,
+        file_exists,
+        now,
+        log,
+    })
+
+    /** Runs `work` after everything already queued for the same run. */
+    const enqueue = <Result>({
+        run_id,
+        work,
+    }: {
+        run_id: string
+        work: () => Promise<Result>
+    }): Promise<Result> => {
+        const result = (queues.get(run_id) ?? Promise.resolve()).then(work)
+        queues.set(
+            run_id,
+            result.catch(() => undefined)
+        )
+        return result
+    }
+
+    const appendRows = async ({
+        memory,
+        rows,
+    }: {
+        memory: RunMemory
+        rows: BoardRow[]
+    }) => {
+        for (const row of rows) {
+            if (memory.rows_stopped) return
+            try {
+                await append_row({ agent_id: memory.entry.agent_id, row })
+                memory.append_failures = 0
+            } catch (error) {
+                memory.append_failures += 1
+                log(
+                    `[${memory.entry.run_id}] couldn't add row ${row.id} to chat ${memory.entry.agent_id}: ${errorText({ error })}`
+                )
+                if (memory.append_failures >= APPEND_FAILURES_BEFORE_STOP) {
+                    memory.rows_stopped = true
+                    log(
+                        `[${memory.entry.run_id}] stopped adding rows: chat ${memory.entry.agent_id} seems gone. The side panel still updates.`
+                    )
+                }
+            }
+        }
+    }
+
+    const header = ({ memory }: { memory: RunMemory }): BoardRow =>
+        headerRow({ state: memory.state, updated_at: now().toISOString() })
+
+    /** Applies one record to a run's board; returns the rows it makes. */
+    const applyOne = ({
+        memory,
+        record,
+    }: {
+        memory: Replay
+        record: EngineRecord
+    }): BoardRow[] => {
+        const { run_id } = memory.state.run
+        const read = readRecord({ record })
+        if (read.status !== 'known') {
+            if (read.status === 'bad_content') {
+                log(
+                    `[${run_id}] skipped record ${record.seq} (${record.kind}): its content doesn't fit.\n${read.error}`
+                )
+            }
+            memory.state = {
+                ...memory.state,
+                event_count: memory.state.event_count + 1,
+            }
+            return []
+        }
+        const before = memory.state
+        const after = applyRecord({ state: before, record: read.record })
+        if (read.record.kind === 'reply_received') {
+            poster.noteTaken({
+                run_id,
+                seq: read.record.seq,
+                time: read.record.time,
+                word: read.record.content.word,
+                ticket: read.record.content.ticket ?? read.record.ticket,
+            })
+        }
+        const described = describeRecord({
+            before,
+            after,
+            record: read.record,
+        })
+        memory.state = described
+            ? { ...after, latest: described.text, latest_at: read.record.time }
+            : after
+        return rowsForRecord({ run_id, before, after, record: read.record })
+    }
+
+    /**
+     * Applies records in seq order from the run's `next_seq`. Lower seqs are
+     * duplicates and skipped; a gap stops there (`gap` is the seq found).
+     */
+    const applyInOrder = ({
+        memory,
+        records,
+    }: {
+        memory: Replay
+        records: EngineRecord[]
+    }): {
+        rows: Map<number, BoardRow[]>
+        applied: number
+        gap: number | null
+    } => {
+        const rows = new Map<number, BoardRow[]>()
+        let applied = 0
+        for (const record of records.toSorted(
+            (left, right) => left.seq - right.seq
+        )) {
+            if (record.seq < memory.next_seq) continue
+            if (record.seq > memory.next_seq) {
+                return { rows, applied, gap: record.seq }
+            }
+            rows.set(record.seq, applyOne({ memory, record }))
+            memory.next_seq += 1
+            applied += 1
+        }
+        return { rows, applied, gap: null }
+    }
+
+    /**
+     * Knows a run this plugin started, rebuilt from its journal `records`.
+     * Their rows aren't added now; while the engine may still send, they are
+     * kept until it sends those records again (see `handleEngineEvent`).
+     */
+    const remember = ({
+        entry,
+        records,
+    }: {
+        entry: RunEntry
+        records: EngineRecord[]
+    }): RunMemory => {
+        const memory: RunMemory = {
+            entry,
+            state: createBoardState({
+                run_id: entry.run_id,
+                spec_number: entry.spec,
+                demo: entry.demo,
+                started_at: entry.started_at,
+                log_path: entry.log_path,
+            }),
+            next_seq: 1,
+            append_failures: 0,
+            rows_stopped: false,
+            unsent_rows: new Map(),
+        }
+        const { rows } = applyInOrder({ memory, records })
+        // A run whose engine ended stays ended after a plugin restart.
+        if (entry.ended) {
+            memory.state = applyEnded({
+                state: memory.state,
+                ended: entry.ended,
+            })
+        } else {
+            memory.unsent_rows = rows
+        }
+        runs.set(entry.run_id, memory)
+        return memory
+    }
+
+    /** A run's journal records on disk; none without a runs folder. */
+    const journalOf = ({ run_id }: { run_id: string }) =>
+        runs_dir === null
+            ? Promise.resolve([])
+            : readJournal({ runs_dir, run_id, log })
+
+    /** Rebuilds every run in the registry from its journal. */
+    const load = async () => {
+        const entries = registry.list()
+        const journals = await Promise.all(
+            entries.map(({ run_id }) => journalOf({ run_id }))
+        )
+        entries.forEach((entry, index) =>
+            remember({ entry, records: journals[index] ?? [] })
+        )
+    }
+
+    /** Changes a run's registry entry, and its copy in memory. */
+    const updateEntry = ({
+        memory,
+        change,
+    }: {
+        memory: RunMemory
+        change: Partial<Pick<RunEntry, 'ended' | 'restarts' | 'token'>>
+    }) => {
+        memory.entry = registry.update({
+            run_id: memory.entry.run_id,
+            change,
+        }) ?? { ...memory.entry, ...change }
+    }
+
+    /** Marks a run's engine as ended (kept in the registry) and updates its header. */
+    const endRun = async ({
+        memory,
+        ended,
+    }: {
+        memory: RunMemory
+        ended: EngineEnded
+    }) => {
+        memory.state = applyEnded({ state: memory.state, ended })
+        memory.unsent_rows.clear()
+        updateEntry({ memory, change: { ended } })
+        await appendRows({ memory, rows: [header({ memory })] })
+    }
+
+    // Every method waits for the registry's runs to be rebuilt first.
+    const ready = load().catch((error: unknown) => {
+        log(
+            `Couldn't rebuild the runs from their journals: ${errorText({ error })}`
+        )
+    })
+
+    return {
+        registry,
+        runs,
+        queues,
+        poster,
+        enqueue,
+        appendRows,
+        header,
+        applyInOrder,
+        remember,
+        journalOf,
+        updateEntry,
+        endRun,
+        ready,
+    }
+}
+
+export type RunStore = ReturnType<typeof createRunStore>

package/board/server/board-server-types.ts

@@ -0,0 +1,114 @@
+/**
+ * The board server's types: what it spawns and runs, what it is given, and
+ * what it keeps in memory for each run.
+ */
+import type { RunEntry } from './run-registry'
+
+import type { BoardRow } from '../shared/board-rows'
+import type { BoardState } from '../shared/board-state'
+import type { EngineSettings } from '../shared/engine-settings'
+
+/** What the server asks the host to spawn: detached, output to a log file. */
+export type SpawnRequest = {
+    command: string
+    args: string[]
+    env: Record<string, string>
+    cwd: string
+    log_path: string
+    /** Called if the process fails to start after `spawn_engine` returned. */
+    on_error: (error: unknown) => void
+}
+
+export type SpawnEngine = (request: SpawnRequest) => { pid: number | null }
+
+/** A short command the server runs and waits for, such as `luca-run --unfinished`. */
+export type CommandRequest = {
+    command: string
+    args: string[]
+    cwd: string
+    env: Record<string, string>
+    timeout_ms: number
+}
+
+/** How a command ended: `exit_code` is `null` when it was killed or timed out. */
+export type CommandResult = {
+    exit_code: number | null
+    stdout: string
+    stderr: string
+}
+
+/** Runs a command to its end. Never throws: a failure is in the result. */
+export type RunCommand = (request: CommandRequest) => Promise<CommandResult>
+
+/** Every running process's full command line. */
+export type ListProcesses = () => Promise<string[]>
+
+/** What one engine check did, by run id. */
+export type EngineCheck = { restarted: string[]; stopped: string[] }
+
+/** Appends (or replaces, by id) one row in a chat's timeline. */
+export type AppendRow = ({
+    agent_id,
+    row,
+}: {
+    agent_id: string
+    row: BoardRow
+}) => Promise<void>
+
+/** A run's board, built from its journal records so far. */
+export type Replay = {
+    state: BoardState
+    /** The next journal seq the board wants (starts at 1). */
+    next_seq: number
+}
+
+/** One run the plugin started: its registry entry and its board in memory. */
+export type RunMemory = Replay & {
+    entry: RunEntry
+    append_failures: number
+    rows_stopped: boolean
+    /**
+     * The rows of records rebuilt from disk, by seq, that the chat may not
+     * have: the engine may have written them while the plugin was down.
+     * Added when the engine sends those records again.
+     */
+    unsent_rows: Map<number, BoardRow[]>
+}
+
+/**
+ * A run the plugin didn't start (such as one from the command line), read
+ * from its journal on disk. Read-only: no chat rows, no engine events, no
+ * restarts. `size` and `changed_ms` are its journal's when last read.
+ * `repo` is the repo its `run_started` names, `null` when it names none.
+ * `engine_gone` is set by the engine check (#635): no process names the
+ * run and its live file isn't fresh. It only labels the run.
+ */
+export type OutsideRun = Replay & {
+    size: number
+    changed_ms: number
+    repo: string | null
+    engine_gone: boolean
+}
+
+/**
+ * What `createBoardServer` is given. Every side effect is injected, so tests
+ * drive the server without Paseo.
+ */
+export type BoardServerConfig = {
+    registry_path: string
+    /** The engine's runs folder (`<runs_dir>/<run_id>/journal.jsonl`). */
+    runs_dir?: string | null
+    append_row: AppendRow
+    spawn_engine: SpawnEngine
+    list_processes: ListProcesses
+    run_command: RunCommand
+    read_settings: () => Promise<EngineSettings>
+    file_exists: ({ path }: { path: string }) => boolean
+    /** Follows an installed `luca-run`'s symlink; the path as is without it. */
+    real_path?: ({ path }: { path: string }) => string
+    home_dir: string
+    env: Record<string, string | undefined>
+    log_dir: string
+    now: () => Date
+    log: (message: string) => void
+}

package/board/server/board-server-util.ts

@@ -0,0 +1,4 @@
+/** Small things the board server's modules share. */
+
+export const errorText = ({ error }: { error: unknown }): string =>
+    error instanceof Error ? error.message : String(error)