@meistrari/agent-core 0.1.10 → 0.1.11

package/docs/supervisor-interruption-recovery-2026-09-14.md

@@ -0,0 +1,34 @@
+# Supervisor interruption recovery
+
+Live Codex 0.1.10 validation reproduced a ready replacement connection with an
+old command and tool still running after SIGKILL. No output marker was written.
+Chat cancellation subsequently terminated the sandbox; its independent terminal
+session reconciliation now finalizes the chat command.
+
+Unreleased core changes persist active turn identity in the step projector.
+When the provider factory reopens a persisted provider session, recovery first
+drains the step journal, then journals explicit interrupted terminal facts for
+unfinished turns. Step closure precedes the turn fact and retains incomplete
+JSON details. Repeated recovery does not create another logical terminal fact.
+A failure during emission reuses the saved frames and deterministic identity.
+Ordinary reconnects to the same in-memory runtime do not trigger interruption.
+
+This is not yet ready for release or acceptance:
+
+- Dispatch receipts now fence the prompt-accepted / command-applied crash window
+  against resubmission. The new process atomically marks the command failed and
+  records that provider acceptance is uncertain. Agent-api now validates the
+  command-failure identity and projects failed prompt dispatch onto terminal
+  product/run state, with a pre-commit crash/deduplication Postgres regression.
+- Recovery readiness now gates command dispatch and uncertain-failure receipts.
+  Child turns close before the main terminal fact. Barrier and child-ordering
+  regressions pass; lint/typecheck pass, with 340 full-suite tests plus the
+  subsequently added child-ordering regression verified in the projector suite.
+- Missing starts, child turns, normal terminal sessions, provider readiness,
+  and failed oversized publication now have explicit regression coverage.
+  Failed uploads retain the identical JSON bytes/hash and do not emit a terminal
+  turn before publication succeeds.
+- Repeat active crash validation using a new exact npm artifact and image.
+- Do not infer that external tool side effects are exactly once.
+
+No running template or bound sandbox was hot-updated with these source changes.

package/package.json

@@ -1,7 +1,7 @@
 {
     "name": "@meistrari/agent-core",
     "type": "module",
-    "version": "0.1.10",
+    "version": "0.1.11",
     "packageManager": "bun@1.3.12",
     "description": "Shared contracts and runtime modules for Tela coding-agent sandboxes: agent protocol, supervisor wire protocol, resident supervisor, worker runtime client, and Claude/Codex harness adapters.",
     "license": "UNLICENSED",

package/src/provenance.gen.ts

@@ -1,6 +1,6 @@
 // Generated by scripts/write-provenance.ts. Do not edit by hand.
 export const generatedProvenance = {
-    version: "0.1.10",
-    sha: "af84feab98a91cd1cd31661c9fa3c0a72521ccf6",
-    buildTime: "2026-09-14T15:18:33.258Z",
+    version: "0.1.11",
+    sha: "8a060e09b239547c1fb7731ab7e73d2439619a38",
+    buildTime: "2026-09-14T17:32:26.510Z",
 } as const

package/src/steps/projector.ts

@@ -23,6 +23,7 @@ export class DurableStepProjector {
         this.db.exec(`pragma journal_mode=WAL; pragma synchronous=FULL;
             create table if not exists steps(id text primary key, body text not null, last_event_id text);
             create table if not exists journal(id text primary key, event text, frames text);
+            create table if not exists active_turns(identity text primary key, event text not null);
             create table if not exists turn_commands(
                 session_id text not null, provider_session_id text not null,
                 actor_id text not null, turn_id text not null, command_id text not null,
@@ -39,15 +40,63 @@ export class DurableStepProjector {
 
     close(): void { this.db.close() }
 
-    async recover(emit: (body: WireEventBody) => void): Promise<void> {
+    async recover(emit: (body: WireEventBody) => void, options?: { interruptUnfinishedTurns: boolean }): Promise<void> {
         for (;;) {
             const row = this.db.query<{ id: string, event: string, frames: string | null }, []>(
                 'select id, event, frames from journal where event is not null order by rowid limit 1',
             ).get()
             if (!row)
-                return
+                break
             await this.deliver(row.id, JSON.parse(row.event) as AgentEvent, row.frames, emit)
         }
+        if (!options?.interruptUnfinishedTurns)
+            return
+        // A provider can expose a tool/message without a turn-start receipt.
+        // Recover its known identity from unfinished assembly, never fabricate
+        // missing content or emit a synthetic successful start.
+        for (;;) {
+            const orphan = this.db.query<{ body: string }, []>(`
+                select body from steps where json_extract(body,'$.status')='running'
+                and not exists (select 1 from active_turns where identity=json_array(
+                    json_extract(body,'$.sessionId'),json_extract(body,'$.providerSessionId'),
+                    json_extract(body,'$.actor.actorId'),json_extract(body,'$.turnId')))
+                order by id limit 1
+            `).get()
+            if (!orphan)
+                break
+            const step = JSON.parse(orphan.body) as StepDetailV1
+            const identity = JSON.stringify([step.sessionId, step.providerSessionId, step.actor.actorId, step.turnId])
+            this.db.query('insert into active_turns(identity,event) values (?,?)').run(identity, JSON.stringify({
+                type: 'turn.started',
+                eventId: `recovered-context:${step.stepId}`,
+                sequence: 0,
+                provider: step.provider,
+                sessionId: step.sessionId,
+                providerSessionId: step.providerSessionId,
+                actor: step.actor,
+                turnId: step.turnId,
+                timestamp: step.startedAt,
+                payload: {},
+            }))
+        }
+        // A newly opened local provider process cannot own the old active turn.
+        // Journal synthetic terminal facts before emitting; never resend prompts.
+        for (;;) {
+            const row = this.db.query<{ identity: string, event: string }, []>(`
+                select identity,event from active_turns
+                order by (json_extract(event,'$.actor.type')='main'), identity limit 1
+            `).get()
+            if (!row)
+                return
+            const previous = JSON.parse(row.event) as Extract<AgentEvent, { type: 'turn.started' }>
+            await this.process({
+                ...previous,
+                eventId: `supervisor-recovery:${createHash('sha256').update(row.identity).digest('hex')}`,
+                type: 'turn.ended',
+                timestamp: new Date().toISOString(),
+                payload: { status: 'interrupted', reason: 'supervisor_process_restarted' },
+            }, emit)
+        }
     }
 
     async process(event: AgentEvent, emit: (body: WireEventBody) => void): Promise<void> {
@@ -92,6 +141,10 @@ export class DurableStepProjector {
 
     private async project(event: AgentEvent): Promise<WireEventBody[]> {
         const commandId = this.correlateCommand(event)
+        if (event.type === 'turn.started') {
+            this.db.query('insert into active_turns(identity,event) values (?,?) on conflict(identity) do nothing')
+                .run(JSON.stringify([event.sessionId, event.providerSessionId, event.actor.actorId, event.turnId]), JSON.stringify(event))
+        }
         const kind = event.type.startsWith('message.')
             ? 'message'
             : event.type.startsWith('tool.call.')
@@ -113,6 +166,14 @@ export class DurableStepProjector {
                 }
             }
             frames.push(wrapAgentEvent(event))
+            if (event.type === 'turn.ended') {
+                this.db.query('delete from active_turns where identity=?')
+                    .run(JSON.stringify([event.sessionId, event.providerSessionId, event.actor.actorId, event.turnId]))
+            }
+            if (event.type === 'session.ended') {
+                this.db.query('delete from active_turns where json_extract(event,\'$.sessionId\')=? and json_extract(event,\'$.providerSessionId\')=?')
+                    .run(event.sessionId, event.providerSessionId)
+            }
             return frames
         }
         const payload = event.payload as Record<string, unknown>

package/src/supervisor/agent-provider-factory.ts

@@ -17,7 +17,7 @@ export interface AgentProviderBindingStore {
 
 export interface AgentSupervisorProviderFactoryOptions {
     eventProcessor?: {
-        recover: (emit: (body: WireEventBody) => void) => Promise<void>
+        recover: (emit: (body: WireEventBody) => void, options?: { interruptUnfinishedTurns: boolean }) => Promise<void>
         process: (event: AgentEvent, emit: (body: WireEventBody) => void) => Promise<void>
     }
     createProvider: (input: {
@@ -76,6 +76,7 @@ export function createAgentSupervisorProviderFactory(
             emit: input.emit,
             onFatal: input.onFatal,
             eventProcessor: options.eventProcessor,
+            interruptUnfinishedTurns: Boolean(previousProviderSessionId),
             prepareInputAttachments: options.prepareInputAttachments,
             onSecretsRefreshed: options.onSecretsRefreshed,
         })
@@ -84,6 +85,7 @@ export function createAgentSupervisorProviderFactory(
 
 class AgentSupervisorRuntime implements SupervisorAgentRuntime {
     private readonly attached = Promise.withResolvers<void>()
+    private readonly recovered = Promise.withResolvers<void>()
     private state: SupervisorAgentRunSnapshot = { status: 'attached', agentState: null }
     private emit: ((body: WireEventBody) => void) | undefined
     private readonly pendingEvents: WireEventBody[] = []
@@ -95,12 +97,17 @@ class AgentSupervisorRuntime implements SupervisorAgentRuntime {
         emit?: (body: WireEventBody) => void
         onFatal?: (error: unknown) => void
         eventProcessor?: AgentSupervisorProviderFactoryOptions['eventProcessor']
+        interruptUnfinishedTurns?: boolean
         prepareInputAttachments?: AgentSupervisorProviderFactoryOptions['prepareInputAttachments']
         onSecretsRefreshed?: AgentSupervisorProviderFactoryOptions['onSecretsRefreshed']
     }) {
+        // Recovery may fail before the owner awaits ready; keep the rejection
+        // observed while preserving it for the readiness/command boundary.
+        void this.recovered.promise.catch(() => undefined)
         if (input.emit)
             this.attachEmitter(input.emit)
         this.eventPump = this.consumeEvents().catch((error: unknown) => {
+            this.recovered.reject(error)
             this.eventFailure = error
             input.onFatal?.(error)
         })
@@ -110,10 +117,13 @@ class AgentSupervisorRuntime implements SupervisorAgentRuntime {
         return this.state
     }
 
+    async ready(): Promise<void> { await this.recovered.promise }
+
     async handle(input: Parameters<SupervisorAgentRuntime['handle']>[0]): Promise<void> {
+        this.attachEmitter(input.emit)
+        await this.ready()
         if (this.eventFailure)
             throw this.eventFailure
-        this.attachEmitter(input.emit)
         const metadata = this.input.run.metadata
         if (input.body.type === 'agent.send-prompt') {
             const inputAttachmentPreparation = await this.input.prepareInputAttachments?.({
@@ -198,8 +208,9 @@ class AgentSupervisorRuntime implements SupervisorAgentRuntime {
             await this.attached.promise
             if (!this.emit)
                 return
-            await this.input.eventProcessor.recover(body => this.emit!(body))
+            await this.input.eventProcessor.recover(body => this.emit!(body), { interruptUnfinishedTurns: Boolean(this.input.interruptUnfinishedTurns) })
         }
+        this.recovered.resolve()
         for await (const event of this.input.run.events) {
             if (event.type === 'session.state.changed')
                 this.state = { status: 'attached', agentState: event.payload.state }

package/src/supervisor/persistence/supervisor-store.ts

@@ -65,6 +65,7 @@ export class SupervisorStore {
                 store.initializeSchema()
             else if (version !== schemaVersion)
                 throw new Error(`Unsupported supervisor SQLite schema version: ${version}.`)
+            store.database.exec('CREATE TABLE IF NOT EXISTS command_dispatches (seq INTEGER PRIMARY KEY REFERENCES commands(seq))')
             return store
         }
         catch (cause) {
@@ -203,7 +204,40 @@ export class SupervisorStore {
     }
 
     markCommand(input: { commandSeq: number, status: 'applied' | 'failed' }): void {
-        this.database.query(`UPDATE commands SET process_status = ? WHERE seq = ?`).run(input.status, input.commandSeq)
+        this.database.transaction(() => {
+            this.database.query(`UPDATE commands SET process_status = ? WHERE seq = ?`).run(input.status, input.commandSeq)
+            this.database.query('DELETE FROM command_dispatches WHERE seq = ?').run(input.commandSeq)
+        })()
+    }
+
+    /** Commit before calling a provider: a process crash makes the outcome uncertain. */
+    beginCommandDispatch(commandSeq: number): void {
+        this.database.query('INSERT INTO command_dispatches(seq) VALUES (?)').run(commandSeq)
+    }
+
+    /** New process only. Receipt and failure share a transaction; no prompt retry. */
+    recoverCommandDispatches(): void {
+        this.database.transaction(() => {
+            const rows = this.database.query<CommandRow, []>(`
+                SELECT command_id, commands.seq, body_json, body_hash, process_status FROM commands
+                JOIN command_dispatches ON command_dispatches.seq=commands.seq ORDER BY commands.seq
+            `).all()
+            for (const row of rows) {
+                this.appendDurableEvent({
+                    occurredAt: new Date().toISOString(),
+                    body: {
+                        type: 'supervisor.command.failed',
+                        payload: {
+                            commandId: row.command_id,
+                            commandSeq: row.seq,
+                            bodyType: commandBodyFromRow(row).type,
+                            failure: { type: 'command', detail: 'Supervisor restarted during dispatch; provider acceptance is uncertain. Command was not resubmitted.' },
+                        },
+                    },
+                })
+                this.markCommand({ commandSeq: row.seq, status: 'failed' })
+            }
+        })()
     }
 
     lastReceivedCommandSeq(): number {

package/src/supervisor/provider-factory.ts

@@ -5,6 +5,7 @@ import type { SupervisorAgentRunSnapshot } from '../supervisor-protocol/supervis
 import type { SupervisorRpcClient } from './rpc-client'
 
 export interface SupervisorAgentRuntime {
+    ready?: () => Promise<void>
     snapshot: () => SupervisorAgentRunSnapshot
     handle: (input: {
         commandId: string

package/src/supervisor/runtime-handler.ts

@@ -188,6 +188,11 @@ export class SupervisorRuntimeHandler implements RuntimeConnectionHandler {
                 emit: body => this.emit(body),
                 onFatal: error => this.dependencies.onFatal(error),
             }).then(async (agent) => {
+                await agent.ready?.()
+                const previousEventMax = this.dependencies.store.localEventMax()
+                this.dependencies.store.recoverCommandDispatches()
+                for (const event of this.dependencies.store.eventsAfter(previousEventMax))
+                    this.sendEvent(event)
                 if ((this.dependencies.credentials !== initialSecrets.credentials
                     || this.dependencies.gitToken !== initialSecrets.gitToken) && agent.refreshSecrets) {
                     await agent.refreshSecrets({
@@ -210,6 +215,7 @@ export class SupervisorRuntimeHandler implements RuntimeConnectionHandler {
         command: CommandEnvelope & { body: WireCommandBody }
     }): Promise<void> {
         try {
+            this.dependencies.store.beginCommandDispatch(input.command.commandSeq)
             await input.agent.handle({
                 commandId: input.command.commandId,
                 commandSeq: input.command.commandSeq,