agl 40.5.1 → 41.0.0

package/ts/fleet/classes.workers.ts

@@ -1,9 +1,9 @@
 import * as plugins from '../plugins.js';
-import { controllerRuntimeIdKey } from '../../ts_interfaces/index.js';
-import type { IFleetWorker, IFleetWorkerProfile, IFleetWorkerJob, IFleetNativeMessageReceipt, TFleetEventKind, TFleetWorkerAction, TFleetWorkerRequest, TFleetWorkerResponse, TFleetWorkerJobAction, IFleetWorkerReport, IFleetWorkerDispatchHolder } from '../../ts_interfaces/fleetworkers.js';
+import { controllerRuntimeIdKey, type IWorkerClaudeJobRequest, type TWorkerClaudeJobObservation } from '../../ts_interfaces/index.js';
+import type { IFleetEvent, IFleetWorker, IFleetWorkerProfile, IFleetWorkerJob, IFleetNativeMessageReceipt, TFleetEventKind, TFleetWorkerAction, TFleetWorkerRequest, TFleetWorkerResponse, TFleetWorkerJobAction, IFleetWorkerReport, IFleetWorkerDispatchHolder } from '../../ts_interfaces/fleetworkers.js';
 import type { IFleetConfiguration, IFleetOperation, IFleetTicketSync } from '../../ts_interfaces/fleet.js';
 import type { ITicketActor } from '../../ts_interfaces/tickets.js';
-import type { FleetTicketEffects } from './classes.ticketeffects.js';
+import type { FleetTicketEffects, IFleetTicketReport } from './classes.ticketeffects.js';
 import type { IControllerSystemMetrics, IControllerFleetWorkerView, IReq_ControllerFleetWorkerRead, IControllerPermission, IControllerQuestion, TControllerPermissionReply } from '../../ts_interfaces/index.js';
 import { FleetWorkerStore, type IFleetWorkerStore, type IFleetWorkerDocument } from './classes.workerstore.js';
 import { FleetEvents } from './classes.events.js';
@@ -14,6 +14,7 @@ import { FleetDispatchError, fleetMessageDigest, fleetReportMarker, type IFleetW
 import { assertFleetWorkerRequest, validateFleetWorkerProfiles } from './functions.workervalidation.js';
 import { readControllerProcessIdentity, controllerProcessIdentityIsLive } from '../classes.processinspection.js';
 import { ticketActorSchema } from '../functions.ticketvalidation.js';
+import { isWorkerCallerProofRefusal, readWorkerProcessSnapshot } from '../functions.workerprocessproof.js';
 
 /** A binding that stayed on `workerName` though its native session lives on `nextName` now. */
 export interface IFleetSessionBindingFailure { workerName: string; nextName: string; error: string }
@@ -80,6 +81,10 @@ export class FleetWorkers {
   private lastLoad = 0;
   private readonly controllerId = plugins.crypto.randomUUID();
   private holder?: Promise<IFleetWorkerDispatchHolder>;
+  /** The chain the caller job re-proofs read their inventories on, one at a time. */
+  private callerJobs: Promise<unknown> = Promise.resolve();
+  /** Set once the controller began to stop: no new worker job is admitted. */
+  private admissionClosed = false;
   constructor(private readonly options: IFleetWorkersOptions = {}) {
     this.store = options.store ?? new FleetWorkerStore(); this.events = options.events ?? new FleetEvents();
     this.backends = options.backends ?? { claude: new FleetClaudeWorkers(this.abort.signal, undefined, undefined, options.callers, options.claudeReleases),
@@ -110,15 +115,21 @@ export class FleetWorkers {
       try {
         const backend = this.backends[worker.harness], observed = backend.observed?.(worker), excerpt = backend.observedExcerpt?.(worker);
         const events = await this.backends[worker.harness].poll(worker, worker.profile);
+        const ticketed = this.options.ticketEffects && worker.ticketKey && worker.ticketActor;
         for (const { nativeId: _nativeId, ...event } of events) {
-          this.events.publish({ ...event, workerId: worker.id, workerName: worker.name, ...(worker.ticketKey ? { ticketKey: worker.ticketKey } : {}) }, owner);
-          if (this.options.ticketEffects && worker.ticketKey && worker.ticketActor) {
-            if (event.kind === 'worker.report.mid_turn' && event.text && event.reportId) await this.reportTicket(owner, worker, { reportId: event.reportId, text: event.text, truncated: event.truncated ?? false });
-            else if (event.kind === 'worker.turn.ended') {
-              const report = await this.backends[worker.harness].observe(worker, worker.profile);
-              if (report.reportId && report.text.trim()) await this.reportTicket(owner, worker, report);
-            }
+          const published = { ...event, workerId: worker.id, workerName: worker.name, ...(worker.ticketKey ? { ticketKey: worker.ticketKey } : {}) };
+          if (ticketed && event.kind === 'worker.report.mid_turn' && event.text && event.reportId) {
+            await this.reportTicket(owner, worker, { reportId: event.reportId, text: event.text, truncated: event.truncated ?? false }, published);
+            continue;
+          }
+          if (ticketed && event.kind === 'worker.turn.ended') {
+            let report: IFleetWorkerReport;
+            // The turn's end is published whether or not its report can be observed.
+            try { report = await this.backends[worker.harness].observe(worker, worker.profile); }
+            catch (error) { this.events.publish(published, owner); throw error; }
+            if (report.reportId && report.text.trim()) { await this.reportTicket(owner, worker, report, published); continue; }
           }
+          this.events.publish(published, owner);
         }
         if (this.options.ticketEffects && worker.ticketKey && worker.ticketActor) {
           const baseline = `${worker.id}:${worker.nativeId}:${worker.ticketKey}`;
@@ -157,8 +168,9 @@ export class FleetWorkers {
     let report: IFleetWorkerReport;
     try { report = await backend.observe(worker, worker.profile); } catch { return; }
     if (!report.reportId || !report.text.trim() || !fleetReportMarker.test(report.text)) return;
-    this.events.publish({ kind: 'worker.report.mid_turn', workerId: worker.id, workerName: worker.name, ...ticket, reportId: report.reportId, text: report.text, truncated: report.truncated, status: report.status }, owner);
-    if (this.options.ticketEffects && worker.ticketKey && worker.ticketActor) await this.reportTicket(owner, worker, report);
+    const event: Omit<IFleetEvent, 'id' | 'occurredAt'> = { kind: 'worker.report.mid_turn', workerId: worker.id, workerName: worker.name, ...ticket, reportId: report.reportId, text: report.text, truncated: report.truncated, status: report.status };
+    if (this.options.ticketEffects && worker.ticketKey && worker.ticketActor) await this.reportTicket(owner, worker, report, event);
+    else this.events.publish(event, owner);
   }
   public releaseEvent(operation: IFleetOperation, owner: string, repo?: string): void {
     if (operation.operation !== 'release' || !['succeeded', 'failed'].includes(operation.state)) return;
@@ -170,11 +182,16 @@ export class FleetWorkers {
     this.events.publish({ kind: ticketSync.state === 'applied' ? 'ticket.applied' : ticketSync.state === 'failed' ? 'ticket.failed' : 'ticket.unknown',
       ticketKey, ticketSync, ...(workerId ? { workerId } : {}), ...(operationId ? { operationId } : {}) }, owner);
   }
-  private async reportTicket(owner: string, worker: IFleetWorker, report: { reportId?: string; text: string; truncated: boolean }): Promise<IFleetTicketSync> {
-    let sync: IFleetTicketSync;
-    try { sync = await this.options.ticketEffects!.report(owner, worker.ticketKey!, worker.ticketActor!, worker, report, this.abort.signal); }
-    catch (error) { sync = { state: 'unknown', effects: [], error: this.error(error) }; }
-    this.ticketEvent(owner, worker.ticketKey!, sync, worker.id); return sync;
+  /**
+   * Comments the report on the worker's ticket first, then publishes `event`, naming the comment that holds its whole
+   * text when the board wrote it, and then the comment's ticket event.
+   */
+  private async reportTicket(owner: string, worker: IFleetWorker, report: { reportId?: string; text: string; truncated: boolean }, event?: Omit<IFleetEvent, 'id' | 'occurredAt'>): Promise<IFleetTicketSync> {
+    let written: IFleetTicketReport;
+    try { written = await this.options.ticketEffects!.report(owner, worker.ticketKey!, worker.ticketActor!, worker, report, this.abort.signal); }
+    catch (error) { written = { sync: { state: 'unknown', effects: [], error: this.error(error) } }; }
+    if (event) this.events.publish({ ...event, ...(written.commentId ? { commentId: written.commentId } : {}) }, owner);
+    this.ticketEvent(owner, worker.ticketKey!, written.sync, worker.id); return written.sync;
   }
   /** `reportsTo` names the direct report whose session asks; a worker it starts or adopts reports to it. */
   public async execute<T extends TFleetWorkerAction>(action: T, request: TFleetWorkerRequest<T>, configuration: IFleetConfiguration, owner: string, signal?: AbortSignal, actor?: ITicketActor, reportsTo?: string): Promise<TFleetWorkerResponse<T>> {
@@ -298,11 +315,43 @@ export class FleetWorkers {
   public async nativeAgents(home: string | undefined, maxAgeMs: number): Promise<IFleetNativeAgent[]> {
     return await this.backends.claude.inventory?.(home, plugins.os.homedir(), maxAgeMs) ?? [];
   }
+  /**
+   * What a fresh inventory shows of the Claude background job a worker's caller launch is bound to, for the controller
+   * to prove the job's next process. One read at a time, however many bridges ask.
+   */
+  public observeCallerJob(request: IWorkerClaudeJobRequest): Promise<TWorkerClaudeJobObservation> {
+    const task = this.callerJobs.then(() => this.readCallerJob(request));
+    this.callerJobs = task.then(() => undefined, () => undefined);
+    return task;
+  }
+  private async readCallerJob(request: IWorkerClaudeJobRequest): Promise<TWorkerClaudeJobObservation> {
+    if (this.abort.signal.aborted) return { state: 'unproven', reason: 'the fleet is closing' };
+    if (!request.workerId.startsWith(`${request.projectId}:`)) return { state: 'ended', reason: 'its launch names no worker of its project' };
+    let worker: IFleetWorker | undefined;
+    try { worker = (await this.store.list(request.projectId)).find((doc) => doc.id === request.workerId)?.worker; }
+    catch (error) { return { state: 'unproven', reason: `the fleet registry could not be read (${this.error(error)})` }; }
+    if (!worker || worker.state === 'archived') return { state: 'ended', reason: `fleet worker ${request.workerId} is ${worker ? 'archived' : 'gone'}` };
+    if (worker.harness !== 'claude' || worker.nativeId !== request.nativeSessionId) return { state: 'ended', reason: 'its worker continues in another session' };
+    let agents: IFleetNativeAgent[] | undefined;
+    try { agents = await this.backends.claude.inventory?.(worker.profile.home, plugins.os.homedir(), 0); }
+    catch (error) { return { state: 'unproven', reason: `the Claude inventory could not be read (${this.error(error)})` }; }
+    const listed = (agents ?? []).filter((agent) => agent.sessionId === request.nativeSessionId);
+    if (listed.length !== 1) return { state: 'unproven', reason: `the Claude inventory lists its session ${listed.length ? 'more than once' : 'nowhere'}` };
+    const agent = listed[0]!;
+    if (agent.kind !== 'background' || agent.id !== request.jobId) return { state: 'ended', reason: 'its session continues as another Claude job' };
+    if (agent.pid === undefined) return { state: 'settled' };
+    try { return { state: 'running', process: { jobId: request.jobId, pid: agent.pid, startTicks: (await readWorkerProcessSnapshot(agent.pid)).startTicks } }; }
+    catch (error) {
+      if (!isWorkerCallerProofRefusal(error)) throw error;
+      return { state: 'unproven', reason: 'the process of its Claude job could not be read' };
+    }
+  }
   /** The newest root threads on the host's shared Codex app-server, read at most `maxAgeMs` ago. */
   public async nativeThreads(maxAgeMs: number): Promise<IFleetNativeThread[]> {
     return await this.backends.codex.threads?.(plugins.os.homedir(), maxAgeMs) ?? [];
   }
   private admitJob(): void {
+    if (this.admissionClosed || this.abort.signal.aborted) throw new Error('The fleet is closing and admits no new worker job.');
     if (this.jobs.size < 512) return;
     for (const [id, job] of this.jobs) if (['succeeded', 'failed'].includes(job.snapshot.state)) { this.jobs.delete(id); break; }
     if (this.jobs.size >= 512) throw new Error('Fleet worker job capacity reached.');
@@ -779,6 +828,19 @@ export class FleetWorkers {
       throw error;
     }
   }
+  /**
+   * Admits no new worker job and lets the running ones finish, for at most `waitMs`; the ones still running then are
+   * aborted and settle. The controller calls it before it tears down the caller hooks the jobs prepare, bind and
+   * revoke launches through, so no job meets them gone.
+   */
+  public async closeAdmission(waitMs = 10_000): Promise<void> {
+    this.admissionClosed = true;
+    const settled = Promise.allSettled([...this.jobs.values()].map((job) => job.completion));
+    let timer: NodeJS.Timeout | undefined;
+    const waited = await Promise.race([settled.then(() => true), new Promise<false>((resolve) => { timer = setTimeout(() => resolve(false), waitMs); })]);
+    if (timer) clearTimeout(timer);
+    if (!waited) { this.abort.abort(); await settled; }
+  }
   public async close(): Promise<void> {
     if (this.pollTimer) clearInterval(this.pollTimer); this.abort.abort(); this.events.close(); this.viewListeners.clear();
     await Promise.allSettled([...this.jobs.values()].map((job) => job.completion)); await Promise.allSettled([this.polling]);

package/ts/fleet/functions.workervalidation.ts

@@ -117,7 +117,7 @@ const job = plugins.z.object({ id: text, action: plugins.z.enum(['start', 'adopt
 const event = plugins.z.object({ id: plugins.z.number().int().positive(),
   kind: plugins.z.enum(['worker.changed', 'worker.turn.ended', 'worker.report.mid_turn', 'worker.reaped', 'worker.credential.failed', 'worker.observation.failed', 'release.finished', 'release.failed', 'ticket.applied', 'ticket.failed', 'ticket.unknown',
     'ticket.changed', 'ticket.notified', 'ticket.notification.failed', 'ticket.stalled', 'harness.release.discovered', 'host.load']),
-  occurredAt: text, workerId: text.optional(), workerName: name.optional(), sessionId: text.optional(), reportId: text.optional(), turnId: text.optional(), status: text.optional(),
+  occurredAt: text, workerId: text.optional(), workerName: name.optional(), sessionId: text.optional(), reportId: text.optional(), commentId: text.optional(), turnId: text.optional(), status: text.optional(),
   ticketKey: ticketKey.optional(),
   ticketSync: ticketSync.optional(),
   text: plugins.z.string().max(64 * 1024).optional(), truncated: plugins.z.boolean().optional(), error: plugins.z.string().max(4096).optional(),