oh-my-codex 0.12.4 → 0.12.6

package/src/scripts/notify-hook/team-dispatch.ts

@@ -97,6 +97,7 @@ async function writeJsonAtomic(path, value) {
 
 // Keep stale-timeout semantics aligned with src/team/state.ts LOCK_STALE_MS.
 const DISPATCH_LOCK_STALE_MS = 5 * 60 * 1000;
+const DISPATCH_REQUEST_LEASE_STALE_MS = 30 * 1000;
 const DEFAULT_ISSUE_DISPATCH_COOLDOWN_MS = 15 * 60 * 1000;
 const ISSUE_DISPATCH_COOLDOWN_ENV = 'OMX_TEAM_DISPATCH_ISSUE_COOLDOWN_MS';
 const DEFAULT_DISPATCH_TRIGGER_COOLDOWN_MS = 30 * 1000;
@@ -183,6 +184,31 @@ function parseTriggerCooldownEntry(entry) {
   };
 }
 
+function reserveDispatchCooldowns({
+  issueCooldownMs,
+  triggerCooldownMs,
+  issueCooldownByIssue,
+  triggerCooldownByKey,
+  issueKey,
+  triggerKey,
+  requestId,
+  reservedAt = Date.now(),
+}) {
+  let mutated = false;
+  if (issueKey && issueCooldownMs > 0) {
+    issueCooldownByIssue[issueKey] = reservedAt;
+    mutated = true;
+  }
+  if (triggerKey && triggerCooldownMs > 0) {
+    triggerCooldownByKey[triggerKey] = {
+      at: reservedAt,
+      last_request_id: safeString(requestId).trim(),
+    };
+    mutated = true;
+  }
+  return mutated;
+}
+
 async function withLockDirectory(lockDir, timeoutError, fn) {
   const ownerPath = join(lockDir, 'owner');
   const ownerToken = `${process.pid}.${Date.now()}.${Math.random().toString(16).slice(2)}`;
@@ -245,6 +271,49 @@ async function withMailboxLock(teamDirPath, workerName, fn) {
   );
 }
 
+function dispatchRequestLeaseDir(teamDirPath, requestId) {
+  return join(teamDirPath, 'dispatch', `.processing-${safeString(requestId).trim()}`);
+}
+
+async function tryAcquireDispatchRequestLease(teamDirPath, requestId) {
+  const lockDir = dispatchRequestLeaseDir(teamDirPath, requestId);
+  const ownerPath = join(lockDir, 'owner');
+  const ownerToken = `${process.pid}.${Date.now()}.${Math.random().toString(16).slice(2)}`;
+  await mkdir(dirname(lockDir), { recursive: true });
+
+  while (true) {
+    try {
+      await mkdir(lockDir, { recursive: false });
+      await writeFile(ownerPath, ownerToken, 'utf8');
+      return { lockDir, ownerPath, ownerToken, requestId };
+    } catch (error) {
+      if (error?.code !== 'EEXIST') throw error;
+      try {
+        const info = await stat(lockDir);
+        if (Date.now() - info.mtimeMs > DISPATCH_REQUEST_LEASE_STALE_MS) {
+          await rm(lockDir, { recursive: true, force: true });
+          continue;
+        }
+      } catch {
+        // best effort
+      }
+      return null;
+    }
+  }
+}
+
+async function releaseDispatchRequestLease(lease) {
+  if (!lease?.lockDir || !lease?.ownerPath || !lease?.ownerToken) return;
+  try {
+    const currentOwner = await readFile(lease.ownerPath, 'utf8');
+    if (currentOwner.trim() === lease.ownerToken) {
+      await rm(lease.lockDir, { recursive: true, force: true });
+    }
+  } catch {
+    // best effort
+  }
+}
+
 function resolveLeaderPaneId(config) {
   return safeString(config?.leader_pane_id).trim();
 }
@@ -336,6 +405,213 @@ async function appendLeaderNotificationDeferredEvent({
   await appendFile(eventsPath, JSON.stringify(event) + '\n').catch(() => {});
 }
 
+async function finalizeClaimedDispatchRequest({
+  claim,
+  result,
+  teamName,
+  teamDirPath,
+  config,
+  cwd,
+  stateDir,
+  logsDir,
+  issueCooldownMs,
+  triggerCooldownMs,
+}) {
+  const requestsPath = join(teamDirPath, 'dispatch', 'requests.json');
+  const issueKey = extractIssueKey(claim.request.trigger_message);
+  const triggerKey = normalizeTriggerKey(claim.request.trigger_message);
+  let summary = { processed: 0, skipped: 0, failed: 0 };
+
+  await withDispatchLock(teamDirPath, async () => {
+    const bridgeRequests = await readBridgeDispatchRequests(stateDir, teamName);
+    const usingLegacyRequests = bridgeRequests === null;
+    const requests = usingLegacyRequests ? await readJson(requestsPath, []) : bridgeRequests;
+    if (!Array.isArray(requests)) return;
+
+    const index = requests.findIndex((entry) => safeString(entry?.request_id).trim() === claim.request.request_id);
+    if (index < 0) return;
+
+    const request = requests[index];
+    if (!request || typeof request !== 'object' || shouldSkipRequest(request) || request.status !== 'pending') return;
+
+    const issueCooldownState = await readIssueCooldownState(teamDirPath);
+    const triggerCooldownState = await readTriggerCooldownState(teamDirPath);
+    const issueCooldownByIssue = issueCooldownState.by_issue || {};
+    const triggerCooldownByKey = triggerCooldownState.by_trigger || {};
+    const nowIso = new Date().toISOString();
+    let mutated = false;
+
+    mutated = reserveDispatchCooldowns({
+      issueCooldownMs,
+      triggerCooldownMs,
+      issueCooldownByIssue,
+      triggerCooldownByKey,
+      issueKey,
+      triggerKey,
+      requestId: request.request_id,
+    }) || mutated;
+
+    request.attempt_count = Number.isFinite(request.attempt_count) ? Math.max(0, request.attempt_count + 1) : 1;
+    request.updated_at = nowIso;
+
+    if (result.ok) {
+      const MAX_UNCONFIRMED_ATTEMPTS = 3;
+      if (result.reason === 'tmux_send_keys_unconfirmed' && request.attempt_count < MAX_UNCONFIRMED_ATTEMPTS) {
+        request.last_reason = result.reason;
+        summary.skipped += 1;
+        mutated = true;
+        await appendDispatchLog(logsDir, {
+          type: 'dispatch_unconfirmed_retry',
+          team: teamName,
+          request_id: request.request_id,
+          worker: request.to_worker,
+          attempt: request.attempt_count,
+          reason: result.reason,
+          ...buildDispatchAttemptEvidence(result),
+        });
+        await appendDeliveryTelemetry(logsDir, {
+          event: 'dispatch_result',
+          team: teamName,
+          request_id: request.request_id,
+          message_id: request.message_id || null,
+          to_worker: request.to_worker,
+          transport: 'send-keys',
+          result: 'retry',
+          reason: result.reason,
+        });
+        await emitOperationalHookEvent(cwd, 'retry-needed', {
+          team: teamName,
+          worker: request.to_worker,
+          request_id: request.request_id,
+          attempt: request.attempt_count,
+          command: request.trigger_message,
+          reason: result.reason,
+          status: 'retry-needed',
+        });
+      } else if (result.reason === 'tmux_send_keys_unconfirmed') {
+        request.status = 'failed';
+        request.failed_at = nowIso;
+        request.last_reason = 'unconfirmed_after_max_retries';
+        runtimeExec({ command: 'MarkFailed', request_id: request.request_id, reason: 'unconfirmed_after_max_retries' }, stateDir);
+        summary.processed += 1;
+        summary.failed += 1;
+        mutated = true;
+        await appendDispatchLog(logsDir, {
+          type: 'dispatch_failed',
+          team: teamName,
+          request_id: request.request_id,
+          worker: request.to_worker,
+          message_id: request.message_id || null,
+          reason: request.last_reason,
+          ...buildDispatchAttemptEvidence(result),
+        });
+        await appendDeliveryTelemetry(logsDir, {
+          event: 'dispatch_result',
+          team: teamName,
+          request_id: request.request_id,
+          message_id: request.message_id || null,
+          to_worker: request.to_worker,
+          transport: 'send-keys',
+          result: 'failed',
+          reason: request.last_reason,
+        });
+        await emitOperationalHookEvent(cwd, 'failed', {
+          team: teamName,
+          worker: request.to_worker,
+          request_id: request.request_id,
+          message_id: request.message_id || null,
+          command: request.trigger_message,
+          reason: request.last_reason,
+          error_summary: request.last_reason,
+          status: 'failed',
+        });
+      } else {
+        request.status = 'notified';
+        request.notified_at = nowIso;
+        request.last_reason = result.reason;
+        runtimeExec({ command: 'MarkNotified', request_id: request.request_id, channel: 'tmux' }, stateDir);
+        if (request.kind === 'mailbox' && request.message_id) {
+          runtimeExec({ command: 'MarkMailboxNotified', message_id: request.message_id }, stateDir);
+          if (usingLegacyRequests) {
+            await updateMailboxNotified(stateDir, teamName, request.to_worker, request.message_id).catch(() => {});
+          }
+        }
+        summary.processed += 1;
+        mutated = true;
+        await appendDispatchLog(logsDir, {
+          type: 'dispatch_notified',
+          team: teamName,
+          request_id: request.request_id,
+          worker: request.to_worker,
+          message_id: request.message_id || null,
+          reason: result.reason,
+          ...buildDispatchAttemptEvidence(result),
+        });
+        await appendDeliveryTelemetry(logsDir, {
+          event: 'dispatch_result',
+          team: teamName,
+          request_id: request.request_id,
+          message_id: request.message_id || null,
+          to_worker: request.to_worker,
+          transport: 'send-keys',
+          result: 'notified',
+          reason: result.reason,
+        });
+      }
+    } else {
+      request.status = 'failed';
+      request.failed_at = nowIso;
+      request.last_reason = result.reason;
+      runtimeExec({ command: 'MarkFailed', request_id: request.request_id, reason: result.reason }, stateDir);
+      summary.processed += 1;
+      summary.failed += 1;
+      mutated = true;
+      await appendDispatchLog(logsDir, {
+        type: 'dispatch_failed',
+        team: teamName,
+        request_id: request.request_id,
+        worker: request.to_worker,
+        message_id: request.message_id || null,
+        reason: result.reason,
+        ...buildDispatchAttemptEvidence(result),
+      });
+      await appendDeliveryTelemetry(logsDir, {
+        event: 'dispatch_result',
+        team: teamName,
+        request_id: request.request_id,
+        message_id: request.message_id || null,
+        to_worker: request.to_worker,
+        transport: 'send-keys',
+        result: 'failed',
+        reason: result.reason,
+      });
+      await emitOperationalHookEvent(cwd, result.reason === LEADER_PANE_MISSING_DEFERRED_REASON ? 'handoff-needed' : 'failed', {
+        team: teamName,
+        worker: request.to_worker,
+        request_id: request.request_id,
+        message_id: request.message_id || null,
+        command: request.trigger_message,
+        reason: result.reason,
+        ...(result.reason === LEADER_PANE_MISSING_DEFERRED_REASON
+          ? { status: 'handoff-needed' }
+          : { status: 'failed', error_summary: result.reason }),
+      });
+    }
+
+    if (!mutated) return;
+    issueCooldownState.by_issue = issueCooldownByIssue;
+    await writeJsonAtomic(issueCooldownStatePath(teamDirPath), issueCooldownState);
+    triggerCooldownState.by_trigger = triggerCooldownByKey;
+    await writeJsonAtomic(triggerCooldownStatePath(teamDirPath), triggerCooldownState);
+    await writeJsonAtomic(requestsPath, requests);
+    if (!usingLegacyRequests) {
+      await writeBridgeDispatchCompat(stateDir, teamName, requests);
+    }
+  });
+
+  return summary;
+}
+
 function resolveWorkerCliForRequest(request, config) {
   const workers = Array.isArray(config?.workers) ? config.workers : [];
   const idx = Number.isFinite(request?.worker_index) ? Number(request.worker_index) : null;
@@ -602,8 +878,8 @@ export async function drainPendingTeamDispatch({
     const manifestPath = join(teamDirPath, 'manifest.v2.json');
     const configPath = join(teamDirPath, 'config.json');
     const requestsPath = join(teamDirPath, 'dispatch', 'requests.json');
-
     const config = await readJson(existsSync(manifestPath) ? manifestPath : configPath, {});
+    const claims = [];
     await withDispatchLock(teamDirPath, async () => {
       const bridgeRequests = await readBridgeDispatchRequests(stateDir, teamName);
       const usingLegacyRequests = bridgeRequests === null;
@@ -617,7 +893,7 @@ export async function drainPendingTeamDispatch({
 
       let mutated = false;
       for (const request of requests) {
-        if (processed >= maxPerTick) break;
+        if (processed + claims.length >= maxPerTick) break;
         if (!request || typeof request !== 'object') continue;
         if (shouldSkipRequest(request)) {
           skipped += 1;
@@ -660,9 +936,6 @@ export async function drainPendingTeamDispatch({
               result: 'deferred',
               reason: LEADER_PANE_MISSING_DEFERRED_REASON,
             });
-            // On the legacy fallback lane, requests.json still carries the queue
-            // state for this deferred request; this event stays a progress
-            // artifact for hook/watcher readers.
             await appendLeaderNotificationDeferredEvent({
               stateDir,
               teamName,
@@ -697,170 +970,24 @@ export async function drainPendingTeamDispatch({
           }
         }
 
-        const result = await injector(request, config, resolve(cwd), stateDir);
-        if (issueKey && issueCooldownMs > 0) {
-          issueCooldownByIssue[issueKey] = Date.now();
-          mutated = true;
-        }
-        if (triggerKey && triggerCooldownMs > 0) {
-          triggerCooldownByKey[triggerKey] = {
-            at: Date.now(),
-            last_request_id: safeString(request.request_id).trim(),
-          };
-          mutated = true;
-        }
-        const nowIso = new Date().toISOString();
-        request.attempt_count = Number.isFinite(request.attempt_count) ? Math.max(0, request.attempt_count + 1) : 1;
-        request.updated_at = nowIso;
-
-        if (result.ok) {
-          // Unconfirmed sends: trigger text was still visible after retry
-          // rounds. Leave as pending for the next tick to retry (up to 3
-          // total attempts) rather than marking notified. Fixes #391.
-          const MAX_UNCONFIRMED_ATTEMPTS = 3;
-          if (result.reason === 'tmux_send_keys_unconfirmed' && request.attempt_count < MAX_UNCONFIRMED_ATTEMPTS) {
-            request.last_reason = result.reason;
-            mutated = true;
-            skipped += 1;
-            await appendDispatchLog(logsDir, {
-              type: 'dispatch_unconfirmed_retry',
-              team: teamName,
-              request_id: request.request_id,
-              worker: request.to_worker,
-              attempt: request.attempt_count,
-              reason: result.reason,
-              ...buildDispatchAttemptEvidence(result),
-            });
-            await appendDeliveryTelemetry(logsDir, {
-              event: 'dispatch_result',
-              team: teamName,
-              request_id: request.request_id,
-              message_id: request.message_id || null,
-              to_worker: request.to_worker,
-              transport: 'send-keys',
-              result: 'retry',
-              reason: result.reason,
-            });
-            await emitOperationalHookEvent(cwd, 'retry-needed', {
-              team: teamName,
-              worker: request.to_worker,
-              request_id: request.request_id,
-              attempt: request.attempt_count,
-              command: request.trigger_message,
-              reason: result.reason,
-              status: 'retry-needed',
-            });
-            continue;
-          }
-          if (result.reason === 'tmux_send_keys_unconfirmed') {
-            request.status = 'failed';
-            request.failed_at = nowIso;
-            request.last_reason = 'unconfirmed_after_max_retries';
-            runtimeExec({ command: 'MarkFailed', request_id: request.request_id, reason: 'unconfirmed_after_max_retries' }, stateDir);
-            processed += 1;
-            failed += 1;
-            mutated = true;
-            await appendDispatchLog(logsDir, {
-              type: 'dispatch_failed',
-              team: teamName,
-              request_id: request.request_id,
-              worker: request.to_worker,
-              message_id: request.message_id || null,
-              reason: request.last_reason,
-              ...buildDispatchAttemptEvidence(result),
-            });
-            await appendDeliveryTelemetry(logsDir, {
-              event: 'dispatch_result',
-              team: teamName,
-              request_id: request.request_id,
-              message_id: request.message_id || null,
-              to_worker: request.to_worker,
-              transport: 'send-keys',
-              result: 'failed',
-              reason: request.last_reason,
-            });
-            await emitOperationalHookEvent(cwd, 'failed', {
-              team: teamName,
-              worker: request.to_worker,
-              request_id: request.request_id,
-              message_id: request.message_id || null,
-              command: request.trigger_message,
-              reason: request.last_reason,
-              error_summary: request.last_reason,
-              status: 'failed',
-            });
-            continue;
-          }
-          request.status = 'notified';
-          request.notified_at = nowIso;
-          request.last_reason = result.reason;
-          runtimeExec({ command: 'MarkNotified', request_id: request.request_id, channel: 'tmux' }, stateDir);
-          if (request.kind === 'mailbox' && request.message_id) {
-            runtimeExec({ command: 'MarkMailboxNotified', message_id: request.message_id }, stateDir);
-            if (usingLegacyRequests) {
-              await updateMailboxNotified(stateDir, teamName, request.to_worker, request.message_id).catch(() => {});
-            }
-          }
-          processed += 1;
-          mutated = true;
-          await appendDispatchLog(logsDir, {
-            type: 'dispatch_notified',
-            team: teamName,
-            request_id: request.request_id,
-            worker: request.to_worker,
-            message_id: request.message_id || null,
-            reason: result.reason,
-            ...buildDispatchAttemptEvidence(result),
-          });
-          await appendDeliveryTelemetry(logsDir, {
-            event: 'dispatch_result',
-            team: teamName,
-            request_id: request.request_id,
-            message_id: request.message_id || null,
-            to_worker: request.to_worker,
-            transport: 'send-keys',
-            result: 'notified',
-            reason: result.reason,
-          });
-        } else {
-          request.status = 'failed';
-          request.failed_at = nowIso;
-          request.last_reason = result.reason;
-          runtimeExec({ command: 'MarkFailed', request_id: request.request_id, reason: result.reason }, stateDir);
-          processed += 1;
-          failed += 1;
-          mutated = true;
-          await appendDispatchLog(logsDir, {
-            type: 'dispatch_failed',
-            team: teamName,
-            request_id: request.request_id,
-            worker: request.to_worker,
-            message_id: request.message_id || null,
-            reason: result.reason,
-            ...buildDispatchAttemptEvidence(result),
-          });
-          await appendDeliveryTelemetry(logsDir, {
-            event: 'dispatch_result',
-            team: teamName,
-            request_id: request.request_id,
-            message_id: request.message_id || null,
-            to_worker: request.to_worker,
-            transport: 'send-keys',
-            result: 'failed',
-            reason: result.reason,
-          });
-          await emitOperationalHookEvent(cwd, result.reason === LEADER_PANE_MISSING_DEFERRED_REASON ? 'handoff-needed' : 'failed', {
-            team: teamName,
-            worker: request.to_worker,
-            request_id: request.request_id,
-            message_id: request.message_id || null,
-            command: request.trigger_message,
-            reason: result.reason,
-            ...(result.reason === LEADER_PANE_MISSING_DEFERRED_REASON
-              ? { status: 'handoff-needed' }
-              : { status: 'failed', error_summary: result.reason }),
-          });
+        const lease = await tryAcquireDispatchRequestLease(teamDirPath, request.request_id);
+        if (!lease) {
+          skipped += 1;
+          continue;
         }
+        mutated = reserveDispatchCooldowns({
+          issueCooldownMs,
+          triggerCooldownMs,
+          issueCooldownByIssue,
+          triggerCooldownByKey,
+          issueKey,
+          triggerKey,
+          requestId: request.request_id,
+        }) || mutated;
+        claims.push({
+          request: { ...request },
+          lease,
+        });
       }
 
       if (mutated) {
@@ -874,6 +1001,37 @@ export async function drainPendingTeamDispatch({
         }
       }
     });
+
+    try {
+      for (const claim of claims) {
+        try {
+          const result = await injector(claim.request, config, resolve(cwd), stateDir);
+          const delta = await finalizeClaimedDispatchRequest({
+            claim,
+            result,
+            teamName,
+            teamDirPath,
+            config,
+            cwd,
+            stateDir,
+            logsDir,
+            issueCooldownMs,
+            triggerCooldownMs,
+          });
+          processed += delta.processed;
+          skipped += delta.skipped;
+          failed += delta.failed;
+        } finally {
+          claim.released = true;
+          await releaseDispatchRequestLease(claim.lease);
+        }
+      }
+    } finally {
+      for (const claim of claims) {
+        if (claim.released) continue;
+        await releaseDispatchRequestLease(claim.lease);
+      }
+    }
   }
 
   return { processed, skipped, failed };