@alecsibilia/luca 14.0.0-alpha.28 → 14.0.0-alpha.29

package/CHANGELOG.md

@@ -1,5 +1,9 @@
 # @alecsibilia/luca
 
+## 14.0.0-alpha.29
+
+No changes in this release.
+
 ## 14.0.0-alpha.28
 
 No changes in this release.

package/board/client/board-look.ts

@@ -144,6 +144,7 @@ export const statusColor = ({
             return theme.colors.statusDanger
         case 'limit_wait':
         case 'usage_line_wait':
+        case 'network_wait':
             return theme.colors.statusWarning
         case 'starting':
         case 'intake':
@@ -195,6 +196,8 @@ export const statusText = ({ status }: { status: RunStatus }): string => {
             return 'limit wait'
         case 'usage_line_wait':
             return 'paused at the usage line'
+        case 'network_wait':
+            return 'Waiting for GitHub to be reachable'
         case 'done':
             return 'done'
         case 'refused':

package/board/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@luca/board",
-    "version": "14.0.0-alpha.28",
+    "version": "14.0.0-alpha.29",
     "private": true,
     "type": "module"
 }

package/board/server/board-server-check.ts

@@ -2,10 +2,12 @@
  * The engine check: finds runs whose engine process is gone although the run
  * hasn't ended, restarts the ones that can go on from their journal, and
  * marks the rest as stopped, with why. The runs the plugin didn't start are
- * only labelled (#635), never restarted.
+ * only labelled (#635), never restarted. A run marked ended whose journal
+ * grew and that has a live engine again comes back (#701).
  */
 
 import type { EngineProcess } from './board-server-engine'
+import type { JournalFollow } from './board-server-follow'
 import type { RunStore } from './board-server-store'
 import type {
     BoardServerConfig,
@@ -34,15 +36,17 @@ export const createEngineChecks = ({
     store,
     engine,
     reads,
+    follow,
 }: {
     config: BoardServerConfig
     store: RunStore
     engine: EngineProcess
-    /** The runs the plugin didn't start, and how to read them again. */
+    /** The runs the plugin didn't start, and how to read every journal again. */
     reads: {
         outside: Map<string, OutsideRun>
-        readOutside: () => Promise<void>
+        readJournals: (args?: { release_held?: boolean }) => Promise<void>
     }
+    follow: JournalFollow
 }) => {
     const {
         list_processes,
@@ -171,7 +175,8 @@ export const createEngineChecks = ({
     /** One check: see `checkEngines`. */
     const runCheck = async (): Promise<EngineCheck> => {
         await ready
-        await reads.readOutside()
+        // Rows held since the plugin loaded stop waiting for a push (#701).
+        await reads.readJournals({ release_held: true })
         const summary: EngineCheck = { restarted: [], stopped: [] }
         const candidates = [...runs.values()].filter((memory) =>
             needsEngine({ state: memory.state })
@@ -179,7 +184,14 @@ export const createEngineChecks = ({
         const others = [...reads.outside.values()].filter((run) =>
             needsEngine({ state: run.state })
         )
-        if (candidates.length === 0 && others.length === 0) return summary
+        const returning = follow.returning()
+        if (
+            candidates.length === 0 &&
+            others.length === 0 &&
+            returning.length === 0
+        ) {
+            return summary
+        }
         let command_lines: string[]
         try {
             command_lines = await list_processes()
@@ -192,6 +204,13 @@ export const createEngineChecks = ({
             return summary
         }
         const live = liveRunIds({ command_lines })
+        // An ended run whose journal grew and that has a live engine (such
+        // as `luca-run --resume`) is live again (#701).
+        await Promise.all(
+            returning
+                .filter((memory) => live.has(memory.entry.run_id))
+                .map((memory) => follow.comeBack({ memory }))
+        )
         await labelOutside({ others, live })
         const dead = candidates.filter(
             (memory) => !live.has(memory.entry.run_id)

package/board/server/board-server-follow.ts

@@ -0,0 +1,143 @@
+/**
+ * Following the runs this plugin started on disk (#701): their journals are
+ * read again when they grow, so a run whose engine can't push (such as one
+ * resumed with `luca-run --resume`) still shows live. A run marked ended
+ * comes back when its journal grows and a live engine runs it.
+ */
+import type { RunStore } from './board-server-store'
+import type { BoardServerConfig, RunMemory } from './board-server-types'
+import { RESUMED_CLI_TEXT, resumedCliRow } from './engine-watch'
+import { settle } from './reduce-board-settle'
+import type { JournalFile } from './run-journals'
+
+import type { BoardRow } from '../shared/board-rows'
+
+export const createJournalFollow = ({
+    config,
+    store,
+}: {
+    config: BoardServerConfig
+    store: RunStore
+}) => {
+    const { log, now } = config
+    const {
+        runs,
+        enqueue,
+        applyInOrder,
+        appendRows,
+        header,
+        journalOf,
+        updateEntry,
+    } = store
+
+    /** Takes the rows a run still holds from its rebuild at plugin load. */
+    const takeHeld = ({ memory }: { memory: RunMemory }): BoardRow[] => {
+        const rows = [...memory.unsent_rows.values()].flat()
+        memory.unsent_rows.clear()
+        return rows
+    }
+
+    /**
+     * Reads again the journal of each run this plugin started that changed
+     * since the last read, and applies the records past its `next_seq`
+     * (the cursor pushes share), adding their rows to the chat at once.
+     * The rows a run still holds from its rebuild at plugin load
+     * (`unsent_rows`) go first, so they reach the chat even when no push
+     * comes: with the first rows read from disk, or on an engine check
+     * (`release_held`) after the first read since the load.
+     */
+    const readFollowed = async ({
+        journals,
+        release_held = false,
+    }: {
+        journals: JournalFile[]
+        release_held?: boolean
+    }) => {
+        await Promise.all(
+            journals.map(({ run_id, size, changed_ms }) => {
+                const memory = runs.get(run_id)
+                if (!memory) return
+                const changed =
+                    memory.size !== size || memory.changed_ms !== changed_ms
+                // The first read since the load leaves a push its chance.
+                const release =
+                    release_held &&
+                    memory.size !== null &&
+                    memory.unsent_rows.size > 0
+                if (!changed && !release) return
+                return enqueue({
+                    run_id,
+                    work: async () => {
+                        const { rows, applied } = applyInOrder({
+                            memory,
+                            records: changed ? await journalOf({ run_id }) : [],
+                        })
+                        memory.size = size
+                        memory.changed_ms = changed_ms
+                        const held =
+                            applied > 0 || release ? takeHeld({ memory }) : []
+                        if (applied === 0 && held.length === 0) return
+                        await appendRows({
+                            memory,
+                            rows: [
+                                ...held,
+                                ...[...rows.values()].flat(),
+                                header({ memory }),
+                            ],
+                        })
+                    },
+                })
+            })
+        )
+    }
+
+    /** The runs marked ended whose journal grew past `ended_at_seq`. */
+    const returning = (): RunMemory[] =>
+        [...runs.values()].filter(
+            ({ entry, next_seq }) =>
+                entry.ended !== null &&
+                entry.ended_at_seq !== null &&
+                next_seq - 1 > entry.ended_at_seq
+        )
+
+    /**
+     * Brings back a run marked ended that has a live engine: clears its
+     * `ended`, re-adds its header, and says so in its chat. Its restart
+     * count is kept.
+     */
+    const comeBack = ({ memory }: { memory: RunMemory }) =>
+        enqueue({
+            run_id: memory.entry.run_id,
+            work: async () => {
+                if (memory.entry.ended === null) return
+                const { run_id } = memory.entry
+                const seq = memory.next_seq - 1
+                updateEntry({ memory, change: { ended: null } })
+                const time = now().toISOString()
+                memory.rows_stopped = false
+                memory.append_failures = 0
+                memory.state = settle({
+                    state: {
+                        ...memory.state,
+                        run: { ...memory.state.run, engine_ended: null },
+                        latest: RESUMED_CLI_TEXT,
+                        latest_at: time,
+                    },
+                })
+                log(
+                    `[${run_id}] came back: its journal grew to seq ${seq} and a live engine runs it.`
+                )
+                await appendRows({
+                    memory,
+                    rows: [
+                        resumedCliRow({ run_id, seq, time }),
+                        header({ memory }),
+                    ],
+                })
+            },
+        })
+
+    return { readFollowed, returning, comeBack }
+}
+
+export type JournalFollow = ReturnType<typeof createJournalFollow>

package/board/server/board-server-read.ts

@@ -1,17 +1,19 @@
 /**
  * What the side panel reads and posts: `board.read` (the runs started from
- * a workspace's chats and the runs the plugin didn't start, read from their
- * journals), and `reply.post`.
+ * a workspace's chats and the runs the plugin didn't start, all followed on
+ * disk through their journals), and `reply.post`.
  */
 import { resolve } from 'node:path'
 
 import { z } from 'zod'
 
+import type { JournalFollow } from './board-server-follow'
 import type { RunStore } from './board-server-store'
 import type { BoardServerConfig, OutsideRun } from './board-server-types'
 import { readLiveFile, withLive } from './live-file'
 import { createBoardState } from './reduce-board'
 import type { ReplyRun } from './reply-post'
+import { createRepoMatch } from './repo-match'
 import { listJournals } from './run-journals'
 
 import type {
@@ -38,22 +40,31 @@ const OUTSIDE_RUNS_SHOWN = 50
 export const createBoardReads = ({
     config,
     store,
+    follow,
 }: {
     config: BoardServerConfig
     store: RunStore
+    follow: JournalFollow
 }) => {
-    const { runs_dir = null } = config
+    const { runs_dir = null, run_command, env, home_dir } = config
     const { runs, poster, applyInOrder, journalOf, ready } = store
     const outside = new Map<string, OutsideRun>()
+    const { sameRepo } = createRepoMatch({ run_command, env, home_dir })
 
     /**
-     * Reads the journals of runs this plugin didn't start, the latest
-     * changed first: a new or changed journal is read again and its new
-     * records applied.
+     * Reads the journals on disk, the latest changed first: a new or changed
+     * journal is read again and its new records applied. The runs this
+     * plugin started are all followed (see `readFollowed`, which
+     * `release_held` passes to); of the others, the latest
+     * `OUTSIDE_RUNS_SHOWN`.
      */
-    const readOutside = async () => {
+    const readJournals = async ({
+        release_held = false,
+    }: { release_held?: boolean } = {}) => {
         if (runs_dir === null) return
-        const journals = (await listJournals({ runs_dir }))
+        const all = await listJournals({ runs_dir })
+        await follow.readFollowed({ journals: all, release_held })
+        const journals = all
             .filter((journal) => !runs.has(journal.run_id))
             .slice(0, OUTSIDE_RUNS_SHOWN)
         const shown = new Set(journals.map((journal) => journal.run_id))
@@ -92,31 +103,47 @@ export const createBoardReads = ({
         )
     }
 
+    /** The runs this plugin didn't start that are in `directory`'s repo. */
+    const outsideIn = async ({
+        directory,
+    }: {
+        directory: string | null
+    }): Promise<OutsideRun[]> => {
+        if (directory === null) return []
+        const others = [...outside.values()]
+        const shown = await Promise.all(
+            others.map((run) =>
+                run.repo === null
+                    ? false
+                    : sameRepo({ directory, repo: run.repo })
+            )
+        )
+        return others.filter((_, index) => shown[index])
+    }
+
     /**
      * `board.read`: the runs started from the workspace's chats and the runs
-     * this plugin didn't start whose repo is the workspace's folder, newest
-     * first, and one of them in full, with its live file laid over it.
+     * this plugin didn't start whose repo is in the workspace folder's git
+     * repo, newest first, and one of them in full, with its live file laid
+     * over it.
      */
     const readBoard = async (
         input: BoardReadInput
     ): Promise<BoardReadOutput> => {
         await ready
-        await readOutside()
+        await readJournals()
         const directory = input.directory ? resolve(input.directory) : null
         const list = [
             ...[...runs.values()].filter(
                 (memory) => memory.entry.workspace_id === input.workspace_id
             ),
-            ...[...outside.values()]
-                .filter((run) => directory !== null && run.repo === directory)
-                .map((run) => ({
-                    state: {
-                        ...run.state,
-                        engine_gone:
-                            run.engine_gone &&
-                            needsEngine({ state: run.state }),
-                    },
-                })),
+            ...(await outsideIn({ directory })).map((run) => ({
+                state: {
+                    ...run.state,
+                    engine_gone:
+                        run.engine_gone && needsEngine({ state: run.state }),
+                },
+            })),
         ]
             .map(({ state }) => state)
             .toSorted((left, right) =>
@@ -167,7 +194,7 @@ export const createBoardReads = ({
         input: ReplyPostInput
     ): Promise<ReplyPostOutput> => {
         await ready
-        await readOutside()
+        await readJournals()
         const memory = runs.get(input.run_id)
         const other = outside.get(input.run_id)
         const run: ReplyRun | null = memory
@@ -186,5 +213,5 @@ export const createBoardReads = ({
         return poster.postReply({ run, input })
     }
 
-    return { outside, readOutside, readBoard, postReply }
+    return { outside, readJournals, readBoard, postReply }
 }

package/board/server/board-server-start.ts

@@ -106,6 +106,7 @@ export const createStarter = ({
             started_at: now().toISOString(),
             log_path,
             ended: null,
+            ended_at_seq: null,
             restarts: 0,
         }
         registry.add({ entry })

package/board/server/board-server-store.ts

@@ -172,7 +172,8 @@ export const createRunStore = ({ config }: { config: BoardServerConfig }) => {
     /**
      * Knows a run this plugin started, rebuilt from its journal `records`.
      * Their rows aren't added now; while the engine may still send, they are
-     * kept until it sends those records again (see `handleEngineEvent`).
+     * kept until it sends those records again (see `handleEngineEvent`) or
+     * the disk follow adds them (see `readFollowed`).
      */
     const remember = ({
         entry,
@@ -194,6 +195,8 @@ export const createRunStore = ({ config }: { config: BoardServerConfig }) => {
             append_failures: 0,
             rows_stopped: false,
             unsent_rows: new Map(),
+            size: null,
+            changed_ms: null,
         }
         const { rows } = applyInOrder({ memory, records })
         // A run whose engine ended stays ended after a plugin restart.
@@ -221,23 +224,44 @@ export const createRunStore = ({ config }: { config: BoardServerConfig }) => {
         const journals = await Promise.all(
             entries.map(({ run_id }) => journalOf({ run_id }))
         )
-        entries.forEach((entry, index) =>
-            remember({ entry, records: journals[index] ?? [] })
-        )
+        entries.forEach((entry, index) => {
+            const memory = remember({ entry, records: journals[index] ?? [] })
+            // An entry ended before `ended_at_seq` existed ended at its
+            // journal's newest seq.
+            if (entry.ended && entry.ended_at_seq === null) {
+                updateEntry({
+                    memory,
+                    change: { ended_at_seq: memory.next_seq - 1 },
+                })
+            }
+        })
     }
 
-    /** Changes a run's registry entry, and its copy in memory. */
+    /**
+     * Changes a run's registry entry, and its copy in memory. Setting
+     * `ended` also sets `ended_at_seq` to the newest seq applied; clearing it
+     * clears that too.
+     */
     const updateEntry = ({
         memory,
         change,
     }: {
         memory: RunMemory
-        change: Partial<Pick<RunEntry, 'ended' | 'restarts' | 'token'>>
+        change: Partial<
+            Pick<RunEntry, 'ended' | 'ended_at_seq' | 'restarts' | 'token'>
+        >
     }) => {
+        const full =
+            change.ended === undefined
+                ? change
+                : {
+                      ...change,
+                      ended_at_seq: change.ended ? memory.next_seq - 1 : null,
+                  }
         memory.entry = registry.update({
             run_id: memory.entry.run_id,
-            change,
-        }) ?? { ...memory.entry, ...change }
+            change: full,
+        }) ?? { ...memory.entry, ...full }
     }
 
     /** Marks a run's engine as ended (kept in the registry) and updates its header. */

package/board/server/board-server-types.ts

@@ -68,11 +68,19 @@ export type RunMemory = Replay & {
     append_failures: number
     rows_stopped: boolean
     /**
-     * The rows of records rebuilt from disk, by seq, that the chat may not
-     * have: the engine may have written them while the plugin was down.
-     * Added when the engine sends those records again.
+     * The rows of records rebuilt from disk at plugin load, by seq, that the
+     * chat may not have: the engine may have written them while the plugin
+     * was down. Added when the engine sends those records again, or, as an
+     * engine that can't push never does, by the disk follow (#701): with the
+     * first rows it reads, or on an engine check after its first read.
      */
     unsent_rows: Map<number, BoardRow[]>
+    /**
+     * Its journal's size and change time when the board last read it from
+     * disk (#701), `null` before the first read; a change reads it again.
+     */
+    size: number | null
+    changed_ms: number | null
 }
 
 /**

package/board/server/board-server.ts

@@ -7,6 +7,8 @@
  * - `board-server-start.ts` and `board-server-resume.ts`: `run.start` and
  *   `/luca-run resume`.
  * - `board-server-events.ts`: `engine.event`.
+ * - `board-server-follow.ts`: following the plugin's runs on disk, and an
+ *   ended run that comes back.
  * - `board-server-read.ts`: `board.read` and `reply.post`, and the runs the
  *   plugin didn't start.
  * - `board-server-check.ts`: the engine check, which also labels the runs
@@ -15,6 +17,7 @@
 import { createEngineChecks } from './board-server-check'
 import { createEngineProcess } from './board-server-engine'
 import { createEngineEvents } from './board-server-events'
+import { createJournalFollow } from './board-server-follow'
 import { createBoardReads } from './board-server-read'
 import { createResumer } from './board-server-resume'
 import { createStarter } from './board-server-start'
@@ -40,10 +43,11 @@ export type {
  * checks (`checkEngines`) read processes through `list_processes` and ask
  * the engine through `run_command`.
  *
- * With a `runs_dir`, every run is rebuilt from its journal there: on start
- * the runs this plugin started, and on each `board.read` the runs it didn't
- * start (shown read-only in the workspace whose folder is their repo). Without one, a restarted
- * plugin knows its runs only from the registry and the engine's next send.
+ * With a `runs_dir`, every run is rebuilt from its journal there on start,
+ * and followed there on each `board.read` and engine check: the runs this
+ * plugin started, and the runs it didn't start (shown read-only in each
+ * workspace in their repo). Without one, a restarted plugin knows its runs
+ * only from the registry and the engine's next send.
  *
  * Each run's `engine.event` work goes through its own queue, so rows never
  * interleave. A failed row append is logged and never loses board state.
@@ -73,8 +77,9 @@ export const createBoardServer = (config: BoardServerConfig) => {
     const resumer = createResumer({ config, store, engine })
     const starter = createStarter({ config, store, engine, resumer })
     const events = createEngineEvents({ store })
-    const reads = createBoardReads({ config, store })
-    const checks = createEngineChecks({ config, store, engine, reads })
+    const follow = createJournalFollow({ config, store })
+    const reads = createBoardReads({ config, store, follow })
+    const checks = createEngineChecks({ config, store, engine, reads, follow })
 
     /** Resolves when every queued piece of work (and any check) has finished. */
     const idle = async () => {

package/board/server/board-vocabulary-run.ts

@@ -52,6 +52,10 @@ export const RUN_VOCABULARY = {
     }),
     /** No earlier run on the repo is going anymore. */
     repo_wait_ended: z.looseObject({}),
+    /** A step can't reach GitHub; the run waits for it (#701). */
+    network_wait_started: z.looseObject({ step: z.string() }),
+    /** The step's wait for GitHub is over: it worked, or the run gave up. */
+    network_wait_ended: z.looseObject({ step: z.string() }),
     /** The engine started again on the run's journal, on Luca's `luca_version`. */
     engine_resumed: z.looseObject({ luca_version: z.string() }),
     /**

package/board/server/board-vocabulary.ts

@@ -34,6 +34,8 @@ const BoardEntrySchema = z.discriminatedUnion('kind', [
     entry({ kind: 'run_started' }),
     entry({ kind: 'repo_wait_started' }),
     entry({ kind: 'repo_wait_ended' }),
+    entry({ kind: 'network_wait_started' }),
+    entry({ kind: 'network_wait_ended' }),
     entry({ kind: 'engine_resumed' }),
     entry({ kind: 'config_reloaded' }),
     entry({ kind: 'config_reload_refused' }),

package/board/server/describe-record.ts

@@ -376,9 +376,12 @@ export const describeRecord = ({
                 text: `The ${record.content.count} new ${record.content.count === 1 ? 'memory went' : 'memories went'} on the spec issue, since there is no PR.`,
                 tone: 'info',
             })
-        // The repo wait shows as a banner on the panel (#632).
+        // The repo wait shows as a banner on the panel (#632), the network
+        // wait as the run's status (#701).
         case 'repo_wait_started':
         case 'repo_wait_ended':
+        case 'network_wait_started':
+        case 'network_wait_ended':
         case 'intake_read':
         case 'ticket_snapshot':
         case 'baseline_tests':

package/board/server/engine-watch.ts

@@ -134,8 +134,8 @@ export type StopWhy =
 /**
  * How the owner resumes run `run_id` with the board attached: the
  * `/luca-run resume` slash command (see `resumeRow`). A bare
- * `luca-run --resume` would run without the board's token, so the board
- * would show it read-only.
+ * `luca-run --resume` runs without the board's token: the board follows it
+ * from its journal on disk, but the engine can't push to it.
  *
  * @example
  * resumeHint({ run_id: 'r1' }) // 'type /luca-run resume r1 in a Paseo chat'
@@ -212,6 +212,37 @@ export const restartRow = ({
 export const RESUMED_TEXT =
     'The run was resumed from its journal with /luca-run resume.'
 
+/** The words of the chat row for an ended run that came back (#701). */
+export const RESUMED_CLI_TEXT = 'Resumed from the command line.'
+
+/**
+ * The chat row for an ended run that came back with a live engine the board
+ * didn't start, such as `luca-run --resume` (#701). Its id ends in the
+ * newest seq then, so a second comeback gets its own row.
+ *
+ * @example
+ * resumedCliRow({ run_id, seq: 1755, time: new Date().toISOString() })
+ * // id `${run_id}-resumed-cli-1755`
+ */
+export const resumedCliRow = ({
+    run_id,
+    seq,
+    time,
+}: {
+    run_id: string
+    seq: number
+    time: string
+}): BoardRow => ({
+    id: `${run_id}-resumed-cli-${seq}`,
+    kind: ROW_KIND.event,
+    data: {
+        time,
+        ticket: null,
+        text: RESUMED_CLI_TEXT,
+        tone: 'info',
+    },
+})
+
 /**
  * The chat row for a run resumed with `/luca-run resume`; its id is unique
  * per run and time, so each resume gets its own row.

package/board/server/luca-version.ts

@@ -1,2 +1,2 @@
 /** The Luca version of this board, stamped when the package was packed. */
-export const LUCA_VERSION: string = "14.0.0-alpha.28"
+export const LUCA_VERSION: string = "14.0.0-alpha.29"

package/board/server/reduce-board-create.ts

@@ -80,6 +80,7 @@ export const createBoardState = ({
     limit_wait: null,
     usage_line_wait: null,
     repo_wait: null,
+    network_waits: [],
     run_plan_used: [],
     needs_you: [],
     tickets: [],