@moretea-labs/forge 1.7.0 → 1.7.2

package/src/runtime/control-plane/facade/controller-round-relay.ts

@@ -1,4 +1,4 @@
-import { createHash } from 'crypto';
+import { createHash, randomUUID } from 'crypto';
 import { withControllerLock } from '../../../cli/repositories/locks';
 import { readRequirement } from '../persistence/requirement-store';
 import {
@@ -7,6 +7,7 @@ import {
   writeControlPlaneRecord,
   type ControlPlaneRecord,
 } from '../persistence/sqlite-store';
+import { workHasActiveExecution } from '../../execution/work-activity';
 import { controllerSessionBlocksRecovery, getControllerSession } from './controller-session-store';
 import { getHandoffItem, listHandoffItems } from './handoff-inbox-store';
 import { getWorkContract, readWorkContractStore } from './work-contract-store';
@@ -25,6 +26,7 @@ export const CONTROLLER_ROUND_DISPOSITIONS = [
 ] as const;
 export type ControllerRoundDisposition = (typeof CONTROLLER_ROUND_DISPOSITIONS)[number];
 
+export const CONTROLLER_RELAY_ABANDONED_RELEASE_ERROR = 'CONTROLLER_RELAY_CLAIM_RELEASED_WITHOUT_DISPOSITION';
 const CONTROLLER_DISPOSITION_COMPATIBILITY_PREFIX = 'controller.disposition:';
 
 export function parseControllerDispositionCompatibilityCapability(
@@ -77,6 +79,8 @@ export interface ControllerRoundRelayRecord {
   controllerInstanceId: string;
   sessionId: string;
   claimGeneration: number;
+  /** Opaque per-round capability. Rotates when Forge dispatches a new ChatGPT round. */
+  authorityId?: string;
   stateFingerprint: string;
   roundCount: number;
   repeatedStateCount: number;
@@ -90,6 +94,7 @@ export interface ControllerRoundRelayRecord {
   conversationUrl?: string;
   blockedReason?: string;
   lastError?: string;
+  nextRecoveryAt?: string;
   submittedAt: string;
   updatedAt: string;
   dispatchedAt?: string;
@@ -147,11 +152,17 @@ const DEFAULT_MAX_REPEATED_STATE = 2;
 const DEFAULT_MAX_FAILURES = 3;
 const DEFAULT_UNCLOSED_ROUND_GRACE_MS = 10 * 60_000;
 const MAX_UNCLOSED_ROUND_GRACE_MS = 60 * 60_000;
+const DEFAULT_STALLED_RECOVERY_BACKOFF_MS = 60_000;
+const MAX_STALLED_RECOVERY_BACKOFF_MS = 15 * 60_000;
 
 function nowIso(options: ControllerRoundRelayStoreOptions): string {
   return options.now?.() ?? new Date().toISOString();
 }
 
+function newControllerRoundAuthorityId(): string {
+  return `cra_${randomUUID().replace(/-/g, '')}`;
+}
+
 function bounded(value: string | undefined, max: number): string | undefined {
   const normalized = value?.trim();
   return normalized ? normalized.slice(0, max) : undefined;
@@ -515,6 +526,7 @@ export function submitControllerRoundDisposition(
       controllerInstanceId: authority.controllerInstanceId,
       sessionId: authority.sessionId,
       claimGeneration: authority.claimGeneration,
+      authorityId: existing.value.authorityId,
       stateFingerprint,
       roundCount,
       repeatedStateCount,
@@ -567,6 +579,8 @@ export function beginInitialControllerRoundDispatch(
     if (previous && ['pending_release', 'dispatching', 'dispatched', 'claimed'].includes(previous.status)) {
       throw new Error(`CONTROLLER_RELAY_ROUND_ALREADY_OPEN: ${relayScopeId}`);
     }
+    const abandonedReleasedRound = previous?.status === 'failed'
+      && previous.lastError === CONTROLLER_RELAY_ABANDONED_RELEASE_ERROR;
     const requestedMaxRounds = boundedInteger(input.maxRounds, DEFAULT_MAX_ROUNDS, 1, 32);
     const requestedMaxRepeatedState = boundedInteger(input.maxRepeatedState, DEFAULT_MAX_REPEATED_STATE, 1, 8);
     const requestedMaxFailures = boundedInteger(input.maxFailures, DEFAULT_MAX_FAILURES, 1, 8);
@@ -574,16 +588,16 @@ export function beginInitialControllerRoundDispatch(
     const maxRepeatedState = previous ? Math.min(previous.maxRepeatedState, requestedMaxRepeatedState) : requestedMaxRepeatedState;
     const maxFailures = previous ? Math.min(previous.maxFailures, requestedMaxFailures) : requestedMaxFailures;
     const stateFingerprint = mechanicalStateFingerprint(options, work, requirementId, relayScopeId);
-    // beginInitialControllerRoundDispatch is entered only for a fresh external
-    // wake after any prior round has closed. Mechanical round/repeated-state
-    // budgets fence one immediate relay chain; carrying them across an explicit
-    // later schedule/manual occurrence eventually blocks healthy periodic Work
-    // whose repository state intentionally remains unchanged. Open-round
-    // duplicate suppression above still rejects a second wake for the same live
-    // chain, while submit/recovery paths continue to accumulate these budgets.
-    const roundCount = 1;
-    const repeatedStateCount = 0;
-    const consecutiveFailures = previous?.consecutiveFailures ?? 0;
+    // Ordinary later schedule/manual occurrences intentionally receive a fresh
+    // mechanical budget. A claimed round explicitly released without any semantic
+    // disposition is different: launcher_start is recovering the same abandoned
+    // relay chain, so it must consume round/repeated-state budget rather than reset
+    // it and thereby weaken repeated-state fencing.
+    const roundCount = abandonedReleasedRound ? previous!.roundCount + 1 : 1;
+    const repeatedStateCount = abandonedReleasedRound
+      ? (previous!.stateFingerprint === stateFingerprint ? previous!.repeatedStateCount + 1 : 0)
+      : 0;
+    const consecutiveFailures = abandonedReleasedRound ? previous!.consecutiveFailures : 0;
     let blockedReason: string | undefined;
     if (roundCount > maxRounds) blockedReason = `round_budget_exhausted:${roundCount}>${maxRounds}`;
     else if (repeatedStateCount >= maxRepeatedState) blockedReason = `repeated_state:${repeatedStateCount}>=${maxRepeatedState}`;
@@ -605,6 +619,7 @@ export function beginInitialControllerRoundDispatch(
       controllerInstanceId: input.identity.controllerInstanceId.trim().slice(0, 240),
       sessionId: input.identity.sessionId.trim().slice(0, 240),
       claimGeneration: 0,
+      authorityId: newControllerRoundAuthorityId(),
       stateFingerprint,
       roundCount,
       repeatedStateCount,
@@ -612,7 +627,7 @@ export function beginInitialControllerRoundDispatch(
       maxRounds,
       maxRepeatedState,
       maxFailures,
-      reason: 'launcher_start_requested_continuation',
+      reason: abandonedReleasedRound ? 'launcher_start_recovered_abandoned_claim' : 'launcher_start_requested_continuation',
       ...(bounded(input.browserSessionId, 500) ? { browserSessionId: bounded(input.browserSessionId, 500) } : previous?.browserSessionId ? { browserSessionId: previous.browserSessionId } : {}),
       ...(bounded(input.conversationUrl, 2_000) ? { conversationUrl: bounded(input.conversationUrl, 2_000) } : previous?.conversationUrl ? { conversationUrl: previous.conversationUrl } : {}),
       ...(blockedReason ? { blockedReason } : {}),
@@ -651,7 +666,14 @@ export function beginControllerRoundRelayAfterRelease(
       throw new Error(`CONTROLLER_RELAY_RELEASE_FENCE_MISMATCH: ${input.workId}`);
     }
     const at = nowIso(options);
-    const dispatching: ControllerRoundRelayRecord = { ...record, status: 'dispatching', updatedAt: at };
+    // A released round is semantically closed. The successor dispatch receives a
+    // fresh capability so a stale tab/conversation cannot mutate the next round.
+    const dispatching: ControllerRoundRelayRecord = {
+      ...record,
+      authorityId: newControllerRoundAuthorityId(),
+      status: 'dispatching',
+      updatedAt: at,
+    };
     writeControlPlaneRecord(options.controllerHome, {
       namespace: NAMESPACE,
       scope: options.repoId,
@@ -665,22 +687,82 @@ export function beginControllerRoundRelayAfterRelease(
   });
 }
 
+/**
+ * Mechanically close a claimed ChatGPT round only after its exact controller
+ * epoch has been released and durable state proves no controller lease remains.
+ * This never invents a semantic disposition; launcher_start must open a new round.
+ */
+export function reconcileControllerRoundAfterAbandonedRelease(
+  options: ControllerRoundRelayStoreOptions,
+  input: { workId: string; releasedSession: ControllerSession },
+): ControllerRoundRelayRecord | undefined {
+  const initial = readRelayRecord(options, input.workId);
+  if (!initial || initial.value.status !== 'claimed') return undefined;
+  return relayLock(options, initial.value.relayScopeId, `controller-relay-abandoned-release:${input.releasedSession.controllerId}`, () => {
+    const current = readRelayRecord(options, input.workId);
+    if (!current || current.value.status !== 'claimed') return undefined;
+    const record = current.value;
+    const releasedPrincipal = input.releasedSession.principalId?.trim() || input.releasedSession.controllerId;
+    const releasedInstanceId = input.releasedSession.controllerInstanceId?.trim() || '';
+    if (
+      record.claimGeneration !== input.releasedSession.claimGeneration
+      || record.controllerId !== input.releasedSession.controllerId
+      || record.principalId !== releasedPrincipal
+      || record.controllerInstanceId !== releasedInstanceId
+      || record.sessionId !== input.releasedSession.sessionId
+    ) {
+      throw new Error(`CONTROLLER_RELAY_RELEASE_FENCE_MISMATCH: ${input.workId}`);
+    }
+    if (getControllerSession(options, input.workId)) {
+      throw new Error(`CONTROLLER_RELAY_ABANDONED_RELEASE_ACTIVE_CLAIM: ${input.workId}`);
+    }
+    const work = getWorkContract(options, input.workId);
+    if (!work || isTerminalWorkContractStatus(work.status)) return undefined;
+
+    const at = nowIso(options);
+    const abandoned: ControllerRoundRelayRecord = {
+      ...record,
+      status: 'failed',
+      lastError: CONTROLLER_RELAY_ABANDONED_RELEASE_ERROR,
+      claimedAt: undefined,
+      updatedAt: at,
+    };
+    writeControlPlaneRecord(options.controllerHome, {
+      namespace: NAMESPACE,
+      scope: options.repoId,
+      key: input.workId,
+      schemaVersion: SCHEMA_VERSION,
+      value: abandoned,
+      action: 'controller_round_relay_claim_abandoned_after_release',
+      expectedRevision: current.revision,
+    });
+    return abandoned;
+  });
+}
+
 export function finishControllerRoundRelayDispatch(
   options: ControllerRoundRelayStoreOptions,
-  input: { workId: string; ok: boolean; browserSessionId?: string; conversationUrl?: string; error?: string },
+  input: { workId: string; ok: boolean; browserSessionId?: string; conversationUrl?: string; error?: string; recovery?: boolean; nowMs?: number },
 ): ControllerRoundRelayRecord | undefined {
   const initial = readRelayRecord(options, input.workId);
   if (!initial || initial.value.status !== 'dispatching') return initial?.value;
   return relayLock(options, initial.value.relayScopeId, `controller-relay-finish:${input.workId}`, () => {
     const current = readRelayRecord(options, input.workId);
     if (!current || current.value.status !== 'dispatching') return current?.value;
-    const at = nowIso(options);
+    const at = typeof input.nowMs === 'number' ? new Date(input.nowMs).toISOString() : nowIso(options);
+    const nextFailureCount = current.value.consecutiveFailures + 1;
+    const recoveryBlocked = input.recovery === true && nextFailureCount >= current.value.maxFailures;
+    const recoveryDelayMs = Math.min(
+      MAX_STALLED_RECOVERY_BACKOFF_MS,
+      DEFAULT_STALLED_RECOVERY_BACKOFF_MS * 2 ** Math.max(0, nextFailureCount - 1),
+    );
     const next: ControllerRoundRelayRecord = input.ok
       ? {
           ...current.value,
           status: 'dispatched',
           consecutiveFailures: 0,
           lastError: undefined,
+          nextRecoveryAt: undefined,
           blockedReason: undefined,
           tabSettlementStatus: 'round_open',
           tabSettlementAt: undefined,
@@ -690,20 +772,39 @@ export function finishControllerRoundRelayDispatch(
           dispatchedAt: at,
           updatedAt: at,
         }
-      : {
-          ...current.value,
-          status: 'failed',
-          consecutiveFailures: current.value.consecutiveFailures + 1,
-          lastError: bounded(input.error, 2_000) ?? 'CONTROLLER_RELAY_DISPATCH_FAILED',
-          updatedAt: at,
-        };
+      : input.recovery
+        ? {
+            ...current.value,
+            status: recoveryBlocked ? 'blocked' : 'dispatching',
+            consecutiveFailures: nextFailureCount,
+            lastError: bounded(input.error, 2_000) ?? 'CHATGPT_CONTROLLER_RELAY_RECOVERY_FAILED',
+            blockedReason: recoveryBlocked
+              ? `consecutive_failures:${nextFailureCount}>=${current.value.maxFailures}`
+              : undefined,
+            nextRecoveryAt: recoveryBlocked
+              ? undefined
+              : new Date(Date.parse(at) + recoveryDelayMs).toISOString(),
+            updatedAt: at,
+          }
+        : {
+            ...current.value,
+            status: 'failed',
+            consecutiveFailures: nextFailureCount,
+            nextRecoveryAt: undefined,
+            lastError: bounded(input.error, 2_000) ?? 'CONTROLLER_RELAY_DISPATCH_FAILED',
+            updatedAt: at,
+          };
     writeControlPlaneRecord(options.controllerHome, {
       namespace: NAMESPACE,
       scope: options.repoId,
       key: input.workId,
       schemaVersion: SCHEMA_VERSION,
       value: next,
-      action: input.ok ? 'controller_round_relay_dispatched' : 'controller_round_relay_failed',
+      action: input.ok
+        ? 'controller_round_relay_dispatched'
+        : input.recovery
+          ? recoveryBlocked ? 'controller_round_relay_recovery_blocked' : 'controller_round_relay_recovery_retry_scheduled'
+          : 'controller_round_relay_failed',
       expectedRevision: current.revision,
     });
     return next;
@@ -724,7 +825,7 @@ export function acknowledgeControllerRoundClaim(
     ) return initial.value;
   } else if (initial.value.status === 'blocked') {
     if (!initial.value.blockedReason?.startsWith('repeated_state:')) return initial.value;
-  } else if (initial.value.status !== 'dispatched') return initial.value;
+  } else if (!['dispatching', 'dispatched'].includes(initial.value.status)) return initial.value;
   if (input.session.controllerType !== 'chatgpt') throw new Error(`CONTROLLER_RELAY_CHATGPT_CLAIM_REQUIRED: ${input.workId}`);
 
   return relayLock(options, initial.value.relayScopeId, `controller-relay-claim:${input.session.controllerId}`, () => {
@@ -834,7 +935,7 @@ export function acknowledgeControllerRoundClaim(
       });
       return rearmed;
     }
-    if (current.value.status !== 'dispatched') return current.value;
+    if (!['dispatching', 'dispatched'].includes(current.value.status)) return current.value;
     const owner = getControllerSession(options, input.workId);
     if (!owner) throw new Error(`CONTROLLER_RELAY_ACTIVE_CLAIM_REQUIRED: ${input.workId}`);
     if (
@@ -886,26 +987,36 @@ export function claimStalledControllerRoundRelays(
   for (const candidate of latestRelayRecordsByScope(options)) {
     if (claimed.length >= limit) break;
     if (!['pending_release', 'dispatching', 'dispatched', 'claimed'].includes(candidate.status)) continue;
-    const roundOpenedAtMs = Date.parse(candidate.claimedAt ?? candidate.dispatchedAt ?? candidate.updatedAt);
-    if (!Number.isFinite(roundOpenedAtMs) || nowMs - roundOpenedAtMs < graceMs) continue;
+    const scheduledRecoveryAtMs = candidate.nextRecoveryAt ? Date.parse(candidate.nextRecoveryAt) : Number.NaN;
+    if (Number.isFinite(scheduledRecoveryAtMs)) {
+      if (nowMs < scheduledRecoveryAtMs) continue;
+    } else {
+      const roundOpenedAtMs = Date.parse(candidate.claimedAt ?? candidate.dispatchedAt ?? candidate.updatedAt);
+      if (!Number.isFinite(roundOpenedAtMs) || nowMs - roundOpenedAtMs < graceMs) continue;
+    }
     const requirement = requirementForRelay(options, candidate.requirementId);
     if (requirement && !['planned', 'active'].includes(requirement.state)) continue;
     const candidateWorks = relevantWork(options, candidate);
     const activeCandidateWorks = candidateWorks.filter((work) => !isTerminalWorkContractStatus(work.status));
     if (activeCandidateWorks.length === 0) continue;
-    if (activeCandidateWorks.some((work) => controllerSessionBlocksRecovery(options, work.workId, { nowMs, graceMs }))) continue;
+    if (activeCandidateWorks.some((work) => workHasActiveExecution(options.controllerHome, options.repoId, work.workId) || controllerSessionBlocksRecovery(options, work.workId, { nowMs, graceMs }))) continue;
 
     const next = relayLock(options, candidate.relayScopeId, `controller-relay-recover:${candidate.originWorkId}`, () => {
       const latest = relayHistory(options, candidate.relayScopeId)[0];
       if (!latest || latest.originWorkId !== candidate.originWorkId || latest.updatedAt !== candidate.updatedAt || latest.status !== candidate.status) return undefined;
-      const latestRoundOpenedAtMs = Date.parse(latest.claimedAt ?? latest.dispatchedAt ?? latest.updatedAt);
-      if (!Number.isFinite(latestRoundOpenedAtMs) || nowMs - latestRoundOpenedAtMs < graceMs) return undefined;
+      const latestRecoveryAtMs = latest.nextRecoveryAt ? Date.parse(latest.nextRecoveryAt) : Number.NaN;
+      if (Number.isFinite(latestRecoveryAtMs)) {
+        if (nowMs < latestRecoveryAtMs) return undefined;
+      } else {
+        const latestRoundOpenedAtMs = Date.parse(latest.claimedAt ?? latest.dispatchedAt ?? latest.updatedAt);
+        if (!Number.isFinite(latestRoundOpenedAtMs) || nowMs - latestRoundOpenedAtMs < graceMs) return undefined;
+      }
       const latestRequirement = requirementForRelay(options, latest.requirementId);
       if (latestRequirement && !['planned', 'active'].includes(latestRequirement.state)) return undefined;
       const works = relevantWork(options, latest);
       const activeWorks = works.filter((work) => !isTerminalWorkContractStatus(work.status));
       if (activeWorks.length === 0) return undefined;
-      if (activeWorks.some((work) => controllerSessionBlocksRecovery(options, work.workId, { nowMs, graceMs }))) return undefined;
+      if (activeWorks.some((work) => workHasActiveExecution(options.controllerHome, options.repoId, work.workId) || controllerSessionBlocksRecovery(options, work.workId, { nowMs, graceMs }))) return undefined;
 
       const currentRecord = readRelayRecord(options, latest.originWorkId);
       if (!currentRecord || currentRecord.value.updatedAt !== latest.updatedAt || currentRecord.value.status !== latest.status) return undefined;
@@ -925,15 +1036,25 @@ export function claimStalledControllerRoundRelays(
       const at = new Date(nowMs).toISOString();
       const recovered: ControllerRoundRelayRecord = {
         ...latest,
+        // Re-dispatching an already dispatched/claimed round creates a new
+        // controller epoch. Retrying an in-flight dispatch keeps its capability.
+        authorityId: blockedReason
+          ? latest.authorityId
+          : latest.status === 'dispatching' && latest.authorityId
+            ? latest.authorityId
+            : newControllerRoundAuthorityId(),
         status: blockedReason ? 'blocked' : 'dispatching',
         stateFingerprint,
         roundCount,
         repeatedStateCount,
-        lastError: latest.status === 'pending_release'
-          ? 'CONTROLLER_RELAY_RELEASE_TRANSITION_INCOMPLETE'
-          : latest.status === 'dispatching'
-            ? 'CONTROLLER_RELAY_DISPATCH_TRANSITION_INCOMPLETE'
-            : latest.status === 'claimed' ? 'CONTROLLER_RELAY_CLAIMED_ROUND_UNCLOSED' : 'CONTROLLER_RELAY_ROUND_UNCLOSED',
+        lastError: latest.nextRecoveryAt
+          ? latest.lastError
+          : latest.status === 'pending_release'
+            ? 'CONTROLLER_RELAY_RELEASE_TRANSITION_INCOMPLETE'
+            : latest.status === 'dispatching'
+              ? 'CONTROLLER_RELAY_DISPATCH_TRANSITION_INCOMPLETE'
+              : latest.status === 'claimed' ? 'CONTROLLER_RELAY_CLAIMED_ROUND_UNCLOSED' : 'CONTROLLER_RELAY_ROUND_UNCLOSED',
+        nextRecoveryAt: undefined,
         claimedAt: undefined,
         ...(blockedReason ? { blockedReason } : { blockedReason: undefined }),
         updatedAt: at,
@@ -957,6 +1078,7 @@ export function claimStalledControllerRoundRelays(
 export function buildControllerRoundRelayPrompt(
   options: ControllerRoundRelayStoreOptions,
   record: ControllerRoundRelayRecord,
+  promptOptions: { exactOriginWork?: boolean } = {},
 ): string {
   const requirement = requirementForRelay(options, record.requirementId);
   const relevantWorks = relevantWork(options, record);
@@ -980,8 +1102,13 @@ export function buildControllerRoundRelayPrompt(
     requirementLine,
     `Linked Work snapshot:\n${workLines}`,
     `Active Handoff snapshot:\n${handoffLines}`,
-    `Previous relay origin Work: ${record.originWorkId}. Do not assume the next action must continue that Work; select, start, or claim the appropriate Work from the latest semantic state.`,
+    promptOptions.exactOriginWork
+      ? `This is a Work-bound scheduled round. Claim and advance only origin Work ${record.originWorkId}. Do not select, start, delegate, or resume a sibling Work, create another schedule, or widen scope. If this Work cannot safely advance after one bounded diagnostic or repair attempt, record the exact evidence, submit wait or wait_for_user, release ownership, and end the round.`
+      : `Previous relay origin Work: ${record.originWorkId}. Do not assume the next action must continue that Work; select, start, or claim the appropriate Work from the latest semantic state.`,
     `Mechanical relay budget: round=${record.roundCount}/${record.maxRounds}; repeated_state=${record.repeatedStateCount}/${record.maxRepeatedState}; consecutive_failures=${record.consecutiveFailures}/${record.maxFailures}.`,
+    record.authorityId
+      ? `This round's durable controller authority is controller_authority_id=${record.authorityId}. Pass this exact controller_authority_id together with relay_scope_id=${record.relayScopeId} on controller_claim, continue, finalize, stop, and controller_release. The opaque authority survives MCP transport/Runtime rotation within this round and rotates before a successor round; never substitute another Work/conversation's authority.`
+      : `This is a legacy relay record without a controller authority capability. Forge permits only the exact already-claimed MCP/controller epoch to finish it; a replacement transport must reopen/recover the round before mutating Work lifecycle.`,
     ...(['CONTROLLER_RELAY_ROUND_UNCLOSED', 'CONTROLLER_RELAY_CLAIMED_ROUND_UNCLOSED'].includes(record.lastError ?? '')
       ? ['The previous ChatGPT round did not submit an explicit disposition before its liveness grace elapsed. Forge is only recovering liveness: reread durable state, make the semantic decision in ChatGPT, and close this round explicitly.']
       : []),