@joshuafolkken/kit 1.1379.0 → 1.1387.0

package/scripts/run/wake/run-wake-cli.ts

@@ -3,12 +3,16 @@ import { randomUUID } from 'node:crypto'
 import path from 'node:path'
 import { fileURLToPath } from 'node:url'
 import type { AgentProfile } from '#scripts/agent/agent-role-profile'
+import { backlog_budget } from '#scripts/backlog/backlog-budget'
 import { cli_flags } from '#scripts/lib/cli-flags'
 import { poll } from '#scripts/lib/poll'
 import { telegram_notify } from '#scripts/notify/telegram-notify'
 import { run_carry, type CarryRead } from '#scripts/run/carry/run-carry'
+import { run_carry_ended } from '#scripts/run/carry/run-carry-ended'
 import { run_event_stream } from '#scripts/run/event/run-event-stream'
+import { run_event_stream_emit } from '#scripts/run/event/run-event-stream-emit'
 import { run_headless } from '#scripts/run/run-headless'
+import { run_stranded_detect, type WatchedRun } from '#scripts/run/run-stranded-detect'
 import { run_wake, type RunWake, type WakeStopReason, type WakeTidyResult } from './run-wake'
 import { run_wake_describe, type WakeContext } from './run-wake-describe'
 import { run_wake_driver } from './run-wake-driver'
@@ -34,7 +38,6 @@ const FAILURE_EXIT_CODE = 1
 const ARGV_OFFSET = 2
 const ONE_GROUP = 1
 const MS_PER_SECOND = 1000
-const DEFAULT_INTERVAL_SECONDS = 60
 // Whole and positive. A zero interval is a busy loop against the temp directory, not a faster
 // supervisor.
 const INTERVAL_PATTERN = /^[1-9]\d*$/u
@@ -71,17 +74,15 @@ const WARNING_RECOVERY = `Check the wake command, then restart with \`pnpm josh
 // `stopped` is a person's own `--stop` or a supervisor being superseded, and a warning channel that
 // fired on those is one that stops being read.
 //
-// **`expired` and `unreadable` warn**, because both end the supervisor while a carry record is still
-// sitting there handed off, so the run would otherwise be left asleep with nothing anywhere saying so.
+// **`unreadable` warns**, because it ends the supervisor while a carry record is still sitting there
+// handed off, so the run would otherwise be left asleep with nothing anywhere saying so.
 // **Every reason is listed, and the silent two are listed as `undefined`.** A partial map would let a
-// sixth stop reason compile straight past this and reach nobody, which is the defect class this
+// fifth stop reason compile straight past this and reach nobody, which is the defect class this
 // section exists to close.
 const STOP_BODIES: Record<WakeStopReason, string | undefined> = {
 	ended: undefined,
 	stopped: undefined,
 	failed: 'The supervisor could not continue the run and has stopped.',
-	expired:
-		'The carried run passed its whole-run bound, so the supervisor stopped without continuing it.',
 	unreadable:
 		'The carry record could not be read, so the supervisor stopped rather than guess at it.',
 }
@@ -118,7 +119,7 @@ function note_to_stderr(note: string): void {
 }
 
 function to_interval_ms(value: string | boolean | undefined): number | undefined {
-	if (typeof value !== 'string') return DEFAULT_INTERVAL_SECONDS * MS_PER_SECOND
+	if (typeof value !== 'string') return backlog_budget.OFFER_INTERVAL_MS
 	if (!INTERVAL_PATTERN.test(value)) return undefined
 
 	const seconds = Number(value)
@@ -147,6 +148,7 @@ async function resolve_context(): Promise<WakeContext | undefined> {
 
 	return {
 		carry_target: run_carry.carry_path(directory),
+		ended_target: run_carry_ended.ended_path(directory),
 		wake_target: run_wake.wake_path(directory),
 		log_target,
 		event_target: run_event_stream.target_of(directory),
@@ -182,10 +184,13 @@ function wake_session(
 	return run_wake_session.launch({ argv: built.argv, ...target, profile }, note_to_stderr)
 }
 
+// **A record this supervisor owns itself is not a session at work.** A failed driver leaves the record
+// adopted for this process, so reading that owner as live would have every restarted pass wait on
+// itself and never drive again.
 function ports_for(context: WakeContext, profile: AgentProfile): LoopPorts {
 	return {
 		read_carry: () => run_carry.read_carry(context.carry_target),
-		is_owner_live: (read) => read.kind === 'carried' && run_carry.is_owner_live(read.carry),
+		is_owner_live: (read) => read.kind === 'carried' && run_carry.is_held_by_another(read.carry),
 		has_work: async () => true,
 		new_session_id: () => randomUUID(),
 		hand_off: () => {
@@ -213,17 +218,48 @@ function warning_body(stop: LoopStop, body: string, context: WakeContext): strin
 	return [body, log_hint(context), stop.note].filter(Boolean).join('\n')
 }
 
-async function warn_of_stop(stop: LoopStop, body: string, context: WakeContext): Promise<void> {
-	await telegram_notify.warn({
+// Whether the warning reached a person — the answer `record_failure` leaves its marker on.
+async function warn_of_stop(stop: LoopStop, body: string, context: WakeContext): Promise<boolean> {
+	return await telegram_notify.warn({
 		issue_title: WARNING_TITLE,
 		body: warning_body(stop, body, context),
 		recovery: WARNING_RECOVERY,
 	})
 }
 
-// **Only a failed wake exits non-zero; `expired` and `unreadable` warn but exit zero.** The
+// **A failed supervisor that leaves the run stranded says so on the stream, with its cause.** Nothing
+// else writes there when the supervisor gives up, so a reader of the events would see the last merge
+// and a run that looks as if it were still watching. The cause is the note's first line — the
+// driver's own `error …` — because the rest is a stderr tail the log already holds.
+//
+// **Only a strand, and only one a person was told of.** A `failed` stop is not always a strand: a
+// `run:carry --end` under a live driver fails it too, and that run ended cleanly. And the event is the
+// marker `run_stranded_detect` holds its own notification back on, so it is left only when the warning
+// was sent — after a failed send the stop-time detector still finds no marker, and notifies.
+function is_strand_told(stop: LoopStop, watched: WatchedRun, is_warned: boolean): boolean {
+	if (!is_warned || stop.reason !== run_wake_loop.FAILED_REASON) return false
+
+	return run_stranded_detect.is_left_stranded(watched)
+}
+
+async function record_failure(
+	stop: LoopStop,
+	watched: WatchedRun,
+	is_warned: boolean,
+): Promise<void> {
+	if (!is_strand_told(stop, watched, is_warned)) return
+
+	const [cause = ''] = (stop.note ?? '').split('\n', 1)
+
+	await run_event_stream_emit.emit(
+		run_event_stream.EVENT_KIND.STRANDED,
+		`The supervisor failed and stopped watching the run — ${cause}`,
+	)
+}
+
+// **Only a failed wake exits non-zero; `unreadable` warns but exits zero.** The
 // verdict-and-exit-code table in `docs/josh-commands-automation.md` is a contract callers branch on,
-// and what those two need is the notification, not a different exit code.
+// and what that one needs is the notification, not a different exit code.
 function exit_code_of(reason: WakeStopReason): number {
 	return reason === run_wake_loop.FAILED_REASON ? FAILURE_EXIT_CODE : SUCCESS_EXIT_CODE
 }
@@ -236,13 +272,17 @@ function stop_body(reason: WakeStopReason): string | undefined {
 	return STOP_BODIES[reason]
 }
 
-async function finish(stop: LoopStop, context: WakeContext): Promise<number> {
+// `started_at` is the run this supervisor took on — what `record_failure` tells an ended run by once
+// the carry record that held it no longer reads.
+async function finish(stop: LoopStop, context: WakeContext, started_at: string): Promise<number> {
 	const body = stop_body(stop.reason)
 
 	if (body === undefined) return report(stop.reason)
 
 	console.error(stop.note ?? body)
-	await warn_of_stop(stop, body, context)
+	const is_warned = await warn_of_stop(stop, body, context)
+
+	await record_failure(stop, { ...context, started_at }, is_warned)
 
 	return report(stop.reason, exit_code_of(stop.reason))
 }
@@ -338,7 +378,7 @@ async function loop(context: WakeContext, interval_ms: number): Promise<number>
 
 	tidy_up(context.wake_target)
 
-	return await finish(stop, context)
+	return await finish(stop, context, read.carry.started_at)
 }
 
 // **The verdict tracks whether the supervisor is actually running, not merely whether a record is
@@ -421,6 +461,9 @@ const run_wake_cli = {
 	SUPERVISING_VERDICT,
 	UNKNOWN_VERDICT,
 	USAGE,
+	finish,
+	ports_for,
+	record_failure,
 	run,
 	stop_body,
 	warning_body,

package/scripts/run/wake/run-wake-describe.ts

@@ -19,6 +19,8 @@ const UNREADABLE_TIME = 'an unreadable time'
 // checkout — a lane must never be where the next session resumes, because a lane belongs to one child.
 interface WakeContext {
 	carry_target: string
+	// Where `run:carry --end` records the run it removes — what tells an ended run from an unread one.
+	ended_target: string
 	wake_target: string
 	// Where everything this supervisor starts writes its output. It is named
 	// in `--list` and in every warning, because a path nobody is told is a file nobody reads.
@@ -32,7 +34,7 @@ interface WakeContext {
 }
 
 function carry_cuts(read: CarryRead): string {
-	if (read.kind === 'carried' || read.kind === 'expired') return String(read.carry.cuts)
+	if (read.kind === 'carried') return String(read.carry.cuts)
 
 	return UNKNOWN_CUTS
 }
@@ -70,15 +72,13 @@ function deadline(marked_at: string | undefined, limit_ms: number): string {
 }
 
 // A live owner that is this supervisor is its own driver at work; any other is a session spending the
-// budget, which has no deadline of its own short of the carry record's whole-run expiry.
+// budget, which has no deadline of its own.
 function owner_wait(wake: RunWake, carry: RunCarry): string {
 	if (carry.owner_pid === wake.pid) {
 		return 'waiting: the driver holds the carry record and is running'
 	}
 
-	const expiry = deadline(carry.started_at, run_carry.CARRY_MAX_AGE_MS)
-
-	return `waiting: process ${String(carry.owner_pid)} holds the carry record; the driver starts at its next cut or once it exits, and the record expires at ${expiry}`
+	return `waiting: process ${String(carry.owner_pid)} holds the carry record; the driver starts at its next cut or once it exits`
 }
 
 const WAIT_LINES: Partial<Record<WakeDecision['kind'], (wake: RunWake) => string>> = {

package/scripts/run/wake/run-wake-driver.ts

@@ -1,5 +1,6 @@
 import { josh_command, type JoshResult } from '#scripts/josh/josh-run'
 import { run_carry, type CarryRead } from '#scripts/run/carry/run-carry'
+import { run_event_setup } from '#scripts/run/event/run-event-setup'
 import { run_invocation } from '#scripts/run/run-invocation'
 import type { DriveResult } from './run-wake-loop'
 
@@ -125,10 +126,16 @@ function read_result(
 	return driver_result(result.out, run_carry.read_carry(target), invocation, result.err)
 }
 
+// **A record this process already owns is claimed again.** A driver that failed leaves the record
+// adopted for the supervisor and not handed off, and the restarted pass re-drives it from here; only a
+// record another live process holds is refused.
 function claim_record(target: string, invocation: string): boolean {
 	const current = run_carry.read_carry(target)
 	if (current.kind !== 'carried' || current.carry.invocation !== invocation) return false
-	if (current.carry.is_handed_off !== true && run_carry.is_owner_live(current.carry)) return false
+
+	if (current.carry.is_handed_off !== true && run_carry.is_held_by_another(current.carry)) {
+		return false
+	}
 
 	return run_carry.adopt_carry(target, current.carry, run_carry.owner_of(process.pid)) !== undefined
 }
@@ -159,9 +166,9 @@ function is_mechanical(result: JoshResult): boolean {
 	)
 }
 
-// A record that expired or was removed during the pause answers `undefined`: its stale verdict must not
-// reach a session, so the supervisor's own next pass re-reads the record and ends the run as `expired`
-// or `stopped`; a rerun would only fail on it and report the run as `failed`.
+// A record that was removed during the pause answers `undefined`: its stale verdict must not reach a
+// session, so the supervisor's own next pass re-reads the record and ends the run as `ended` or
+// `stopped`; a rerun would only fail on it and report the run as `failed`.
 async function run_driver(
 	target: string,
 	args: ReadonlyArray<string>,
@@ -188,6 +195,8 @@ async function drive(target: string, wait: Pause = pause): Promise<DriveResult>
 		return { kind: 'failed', note: 'The driver could not claim the carry record.' }
 	}
 
+	await run_event_setup.emit(run_event_setup.STAGE.HANDING_OFF)
+
 	return read_result(
 		await run_driver(target, args, wait, RERUN_LIMIT),
 		target,

package/scripts/run/wake/run-wake-handoff.ts

@@ -21,7 +21,7 @@ function hand_off(target: string): void {
 
 	if (read.kind !== 'carried' || read.carry.is_handed_off === true) return
 
-	stamp_file.write_stamp(target, { ...read.carry, is_handed_off: true })
+	stamp_file.replace_stamp(target, { ...read.carry, is_handed_off: true })
 }
 
 const run_wake_handoff = { hand_off }

package/scripts/run/wake/run-wake-loop.ts

@@ -24,12 +24,6 @@ const STOPPED_REASON: WakeStopReason = 'stopped'
 const FAILED_REASON: WakeStopReason = 'failed'
 const NO_ATTEMPTS = 0
 const STOPPED_BY_PERSON: LoopStop = { reason: STOPPED_REASON, note: undefined }
-// **The back-off after repeated whiffs, and its ceiling.** While a launch is
-// deferred for want of work, each wait is as long as the idle stretch so far — so the polling interval
-// doubles pass by pass — capped at this multiple of the configured one. Eight times the one-minute
-// default is eight minutes: far fewer backlog reads over a long empty stretch, and still well inside
-// the idle ceiling, so work that appears is picked up within one capped wait.
-const MAX_BACKOFF_FACTOR = 8
 
 interface LoopPorts {
 	// The carry record, re-read every pass rather than cached: it is written by the parent loop of a
@@ -157,18 +151,6 @@ async function decide_pass(wake: RunWake, ports: LoopPorts): Promise<WakeDecisio
 	return run_wake.decide({ ...input, has_work: await ports.has_work(input.read) })
 }
 
-// The wait before the next pass: the configured interval, stretched while a launch is being deferred
-// to the length of the idle stretch so far and capped at `MAX_BACKOFF_FACTOR` times the interval.
-function next_interval(wake: RunWake | undefined, interval_ms: number, now: Date): number {
-	const idle_since = wake?.idle_since === undefined ? NaN : Date.parse(wake.idle_since)
-
-	if (Number.isNaN(idle_since)) return interval_ms
-
-	const stretched = Math.max(interval_ms, now.getTime() - idle_since)
-
-	return Math.min(stretched, interval_ms * MAX_BACKOFF_FACTOR)
-}
-
 // The record is re-read at the top of every pass rather than carried in a variable, so a `--stop` that
 // removed it ends the loop at the next interval and a `--list` reads what the loop actually wrote.
 // One pass: decide, act, write the result back. `undefined` means carry on to the next interval.
@@ -187,6 +169,9 @@ async function run_pass(
 // The record is re-read at the top of every pass rather than carried in a variable, so a `--stop` that
 // removed it ends the loop at the next interval and a `--list` reads what the loop actually wrote.
 //
+// **The wait between two passes is the configured interval, however long the run has been idle**: a
+// launch deferred for want of work is asked about again as soon as a running one would be.
+//
 // **`read_own_wake` rather than `read_wake`, and the difference is a whole pass.**
 // A record that has become a *successor's* is not this loop's to act on, and reading it back would
 // have this pass decide from another supervisor's counters — spawning a session for a cut that
@@ -203,7 +188,7 @@ async function run_loop(target: string, ports: LoopPorts, interval_ms: number):
 		if (stop !== undefined) return stop
 
 		// eslint-disable-next-line no-await-in-loop -- polling: each read waits on the state the previous one saw
-		await ports.sleep(next_interval(run_wake.read_own_wake(target), interval_ms, ports.now()))
+		await ports.sleep(interval_ms)
 		wake = run_wake.read_own_wake(target)
 	}
 
@@ -212,8 +197,6 @@ async function run_loop(target: string, ports: LoopPorts, interval_ms: number):
 
 const run_wake_loop = {
 	FAILED_REASON,
-	MAX_BACKOFF_FACTOR,
-	next_interval,
 	run_loop,
 }
 

package/scripts/run/wake/run-wake-restart.ts

@@ -15,7 +15,7 @@ interface RestartPorts {
 	// One run of the loop over the record the caller claimed once and holds across every restart, so a
 	// `--stop` landing in the pause is read by the next pass as the person's stop.
 	pass: () => Promise<LoopStop>
-	// Whether the carry record can still be continued — `carried`, neither `expired` nor gone.
+	// Whether the carry record can still be continued — `carried`, neither unreadable nor gone.
 	is_resumable: () => boolean
 	pause: () => Promise<void>
 	note: (text: string) => void

package/scripts/run/wake/run-wake.ts

@@ -9,7 +9,7 @@ import { z } from 'zod'
 
 // The process outside the conversation that starts the next session after a `backlogrun` cut.
 // Whether to wake is the carry record's answer, never a judgement: `carried` and handed off wakes;
-// `none`, `expired` (the whole-run bound) and `unreadable` stop. A hand-off is set by `run:carry --cut`,
+// `none` and `unreadable` stop. A hand-off is set by `run:carry --cut`,
 // or by this supervisor when it recovers a record whose owner died, so a live session never reads as
 // waiting. The supervisor owns the record while its deterministic driver spends the budget; a judgment
 // branch hands it off so one AI session can adopt it.
@@ -23,8 +23,8 @@ const WAKE_LOG_SUFFIX = '.log'
 const WAKE_GRACE_MS = 600_000
 // Retried before failing: stopping ends a run nobody is watching.
 const MAX_WAKE_ATTEMPTS = 3
-// An empty backlog still needs one session to finish the run, so a deferral is bounded by the
-// `backlogrun` idle watch's own default.
+// A run with nothing to start still needs one session to finish it, so a deferral is bounded by the
+// `backlogrun` idle wait itself.
 const IDLE_CEILING_MS = backlog_budget.DEFAULT_IDLE_MS
 const NO_WAKES = 0
 const ONE_WAKE = 1
@@ -49,11 +49,11 @@ interface RunWake {
 	woke_pid?: number | undefined
 	// Every `--session-id` this forced, so `josh time --run` attributes whiffs only to its own sessions.
 	spawned?: ReadonlyArray<string> | undefined
-	// Start of one unbroken idle stretch: bounds the deferral and backs off the polling interval.
+	// Start of one unbroken idle stretch: bounds the deferral.
 	idle_since?: string | undefined
 }
 
-type WakeStopReason = 'ended' | 'expired' | 'unreadable' | 'failed' | 'stopped'
+type WakeStopReason = 'ended' | 'unreadable' | 'failed' | 'stopped'
 
 // `superseded`: another supervisor holds the record and this process is the replaced one.
 type WakeTidyResult = 'absent' | 'removed' | 'superseded'
@@ -87,9 +87,8 @@ const WAIT_DECISION: WakeDecision = { kind: 'wait' }
 const PENDING_DECISION: WakeDecision = { kind: 'pending' }
 const FAILED_DECISION: WakeDecision = { kind: 'failed' }
 
-const STOP_REASONS: Record<'none' | 'expired' | 'unreadable', WakeStopReason> = {
+const STOP_REASONS: Record<'none' | 'unreadable', WakeStopReason> = {
 	none: 'ended',
-	expired: 'expired',
 	unreadable: 'unreadable',
 }
 

package/scripts/backlog/backlog-idle.ts

@@ -1,88 +0,0 @@
-import { backlog_budget, type BudgetInput } from './backlog-budget'
-
-// The idle watch as a window a person can read: when the run began waiting on
-// an empty backlog, when the wait ends, and which bound ends it. Without it the moment the wait began
-// would live only in `backlog:drive`'s memory as `--active`, so a reader of the event stream could
-// tell the run was waiting but not until when. The budget names the
-// window, `backlog:offer` records it as an `idle` event, and `run:board` reads it back — all through the
-// text written and parsed here, so the three never spell it differently.
-
-type IdleBound = 'idle' | 'whole-run'
-
-interface IdleWindow {
-	since_ms: number
-	until_ms: number
-	// Which bound ends the wait: the idle budget running out, or the whole-run bound arriving first.
-	bound: IdleBound
-	// When the watching loop last asked the backlog — the poll the next one is counted from.
-	asked_ms: number
-}
-
-const TEXT_PATTERN =
-	/^idle since (?<since>\S+) until (?<until>\S+) \((?<bound>idle|whole-run)\) asked (?<asked>\S+)$/u
-const IDLE_BOUND: IdleBound = 'idle'
-const WHOLE_RUN_BOUND: IdleBound = 'whole-run'
-
-// The window an idle watch opened at `active_at_ms` runs for, as asked at `now_ms`, or `undefined` when
-// the watch is off. The wait ends at the earlier of the idle budget and the whole-run bound — the order
-// `decide` applies.
-function window_of(input: BudgetInput): IdleWindow | undefined {
-	if (input.idle_budget_ms === undefined) return undefined
-
-	const idle_until = input.active_at_ms + input.idle_budget_ms
-	const run_until = input.started_at_ms + backlog_budget.WHOLE_RUN_BUDGET_MS
-	const bound = run_until < idle_until ? WHOLE_RUN_BOUND : IDLE_BOUND
-
-	return {
-		since_ms: input.active_at_ms,
-		until_ms: Math.min(idle_until, run_until),
-		bound,
-		asked_ms: input.now_ms,
-	}
-}
-
-function iso_of(ms: number): string {
-	return new Date(ms).toISOString()
-}
-
-function text_of(window: IdleWindow): string {
-	const { since_ms, until_ms, bound, asked_ms } = window
-
-	return `idle since ${iso_of(since_ms)} until ${iso_of(until_ms)} (${bound}) asked ${iso_of(asked_ms)}`
-}
-
-function is_bound(value: string | undefined): value is IdleBound {
-	return value === IDLE_BOUND || value === WHOLE_RUN_BOUND
-}
-
-// A malformed stamp reads as no window rather than as `NaN`, so a mangled line is never drawn.
-function window_from(groups: Record<string, string>, bound: IdleBound): IdleWindow | undefined {
-	const since_ms = Date.parse(String(groups['since']))
-	const until_ms = Date.parse(String(groups['until']))
-	const asked_ms = Date.parse(String(groups['asked']))
-
-	return Number.isNaN(since_ms + until_ms + asked_ms)
-		? undefined
-		: { since_ms, until_ms, bound, asked_ms }
-}
-
-function parse(text: string): IdleWindow | undefined {
-	const groups = TEXT_PATTERN.exec(text)?.groups
-
-	if (groups === undefined || !is_bound(groups['bound'])) return undefined
-
-	return window_from(groups, groups['bound'])
-}
-
-// The next time the watching loop asks the backlog again: one `IDLE_POLL_MINUTES` after the ask the
-// stream recorded, never past the end of the wait — the interval `idle_watch_reason` tells the loop.
-function next_check_ms(window: IdleWindow): number {
-	const poll_ms = backlog_budget.IDLE_POLL_MINUTES * backlog_budget.MS_PER_MINUTE
-
-	return Math.min(window.asked_ms + poll_ms, window.until_ms)
-}
-
-const backlog_idle = { next_check_ms, parse, text_of, window_of }
-
-export { backlog_idle }
-export type { IdleBound, IdleWindow }