@moretea-labs/forge 1.6.0 → 1.7.1

package/src/runtime/standalone-recovery/entry.ts

@@ -1,6 +1,6 @@
 import { createHash, randomUUID } from 'crypto';
 import { createServer, type IncomingMessage, type ServerResponse } from 'http';
-import { basename, resolve } from 'path';
+import { basename, join, resolve } from 'path';
 import { pathToFileURL } from 'url';
 import { readMcpServiceOAuthPassphrase } from '../../cli/mcp/auth';
 import { FORGE_VERSION } from '../../version';
@@ -18,6 +18,7 @@ import {
   repairPublicTunnel,
   restartPrimaryConnector,
   restartPrimaryRuntime,
+  restartRecoveryWatchdog,
   stageAndActivateConfiguredRuntimeRelease,
   rollbackPrevious,
   secureEqual,
@@ -27,6 +28,12 @@ import {
   type WatchdogState,
   type RecoveryConfig,
 } from './core';
+import {
+  createRecoveryWatchdogHeartbeat,
+  observeRecoveryWatchdogHealth,
+  writeRecoveryWatchdogHeartbeat,
+  type RecoveryWatchdogHeartbeat,
+} from './watchdog-heartbeat';
 import {
   RECOVERY_RELEASE_ROLE_CANARY_ARG,
   writeRecoveryRuntimeIdentity,
@@ -139,12 +146,24 @@ export function resetWatchdogStateForRecoveryRelease(
     // budget. Those counters are bound to the Runtime release and survive both
     // watchdog process exits and immutable Recovery release activation.
     recoveryGatewayRestartUsed: false,
+    primaryConnectorFailures: 0,
+    primaryConnectorFirstFailureAt: undefined,
+    primaryConnectorRestartAttempts: 0,
+    primaryConnectorRestartFailures: 0,
+    primaryConnectorRestartLastAttemptAt: undefined,
     recoveryReleaseRevision: releaseRevision,
   };
 }
 
 async function startWatchdog(config: RecoveryConfig): Promise<void> {
   const runtimeIdentity = writeRecoveryRuntimeIdentity(config.controllerHome, 'watchdog');
+  let heartbeat: RecoveryWatchdogHeartbeat = createRecoveryWatchdogHeartbeat(runtimeIdentity);
+  const persistHeartbeat = (patch: Partial<RecoveryWatchdogHeartbeat> = {}) => {
+    heartbeat = writeRecoveryWatchdogHeartbeat(config.controllerHome, { ...heartbeat, ...patch });
+  };
+  persistHeartbeat();
+  const pulse = setInterval(() => persistHeartbeat({ lastPulseAt: new Date().toISOString() }), 5_000);
+  pulse.unref?.();
   process.stdout.write(JSON.stringify({ status: 'ready', role: 'watchdog', runtimeIdentity }) + '\n');
   const loadedState = loadWatchdogState(config);
   let state = runtimeIdentity?.releaseRevision
@@ -152,9 +171,13 @@ async function startWatchdog(config: RecoveryConfig): Promise<void> {
     : loadedState;
   if (state !== loadedState) state = saveWatchdogState(config, state);
   for (;;) {
+    const tickStartedAt = new Date().toISOString();
+    persistHeartbeat({ lastPulseAt: tickStartedAt, lastTickStartedAt: tickStartedAt, lastError: undefined });
     try {
       const result = await watchdogTick(config, state);
       state = saveWatchdogState(config, result.state);
+      const tickCompletedAt = new Date().toISOString();
+      persistHeartbeat({ lastPulseAt: tickCompletedAt, lastTickCompletedAt: tickCompletedAt, lastError: undefined });
       process.stdout.write(JSON.stringify({
         at: new Date().toISOString(),
         action: result.decision.action,
@@ -170,10 +193,14 @@ async function startWatchdog(config: RecoveryConfig): Promise<void> {
         primaryRuntimeRecoveryDetail: result.primaryRuntimeRecovery?.detail,
         rollbackDetail: result.rollback?.detail,
         publicTunnelDetail: result.publicTunnelRepair?.detail,
+        primaryConnectorRestartDetail: result.primaryConnectorRestart?.detail,
       }) + '\n');
     } catch (error) {
       state = saveWatchdogState(config, { ...state, failures: state.failures + 1, firstFailureAt: state.firstFailureAt ?? Date.now() });
-      process.stderr.write(`watchdog probe failed: ${error instanceof Error ? error.message : String(error)}\n`);
+      const detail = error instanceof Error ? error.message : String(error);
+      const failedAt = new Date().toISOString();
+      persistHeartbeat({ lastPulseAt: failedAt, lastTickFailedAt: failedAt, lastError: detail.slice(0, 500) });
+      process.stderr.write(`watchdog probe failed: ${detail}\n`);
     }
     await new Promise((resolveSleep) => setTimeout(resolveSleep, 5_000));
   }
@@ -228,7 +255,7 @@ export const RECOVERY_TOOLS = [
   { name: 'restart_primary_runtime', description: 'Restart the installed canonical Forge Runtime service and require whole-Runtime verification.', inputSchema: { type: 'object', properties: { request_id: { type: 'string', minLength: 8, maxLength: 120 } }, required: ['request_id'], additionalProperties: false } },
   { name: 'restart_primary_connector', description: 'Restart the explicitly configured primary OAuth/Connector launchd service only after local Canonical Runtime verification succeeds.', inputSchema: { type: 'object', properties: { request_id: { type: 'string', minLength: 8, maxLength: 120 } }, required: ['request_id'], additionalProperties: false } },
   { name: 'recover_primary_runtime', description: 'Stop the canonical Runtime, restore the attested previous whole release and SQLite backup, restart it, and require verification.', inputSchema: { type: 'object', properties: { request_id: { type: 'string', minLength: 8, maxLength: 120 } }, required: ['request_id'], additionalProperties: false } },
-  { name: 'activate_runtime_release', description: 'Activate an already staged immutable Runtime release only if the caller-observed active release/authority revision are still current. Reverse activation of current.previous is rejected; use rollback_previous/recover_primary_runtime instead.', inputSchema: { type: 'object', properties: { request_id: { type: 'string', minLength: 8, maxLength: 120 }, release_path: { type: 'string', minLength: 8, maxLength: 1024 }, expected_active_release_id: { type: 'string', minLength: 1, maxLength: 256 }, expected_authority_revision: { type: 'integer', minimum: 1 } }, required: ['request_id', 'release_path', 'expected_active_release_id', 'expected_authority_revision'], additionalProperties: false } },
+  { name: 'activate_runtime_release', description: 'Activate an already staged immutable Runtime release only if the caller-observed active release/authority revision are still current. Reverse activation of current.previous is rejected; use rollback_previous/recover_primary_runtime instead.', inputSchema: { type: 'object', properties: { request_id: { type: 'string', minLength: 8, maxLength: 120 }, release_path: { type: 'string', minLength: 8, maxLength: 1024, description: 'Absolute path to the staged immutable Runtime release directory.' }, expected_active_release_id: { type: 'string', minLength: 1, maxLength: 256 }, expected_authority_revision: { type: 'integer', minimum: 1 } }, required: ['request_id', 'release_path', 'expected_active_release_id', 'expected_authority_revision'], additionalProperties: false } },
   { name: 'stage_and_activate_runtime_release', description: 'Build one immutable Runtime release from the fixed Recovery-configured source root, then activate it transactionally with rollback protection. No arbitrary source path is accepted.', inputSchema: { type: 'object', properties: { request_id: { type: 'string', minLength: 8, maxLength: 120 } }, required: ['request_id'], additionalProperties: false } },
   { name: 'restart_public_tunnel', description: 'Restart the explicitly configured public tunnel only after local runtime verification succeeds and the external endpoint is unavailable.', inputSchema: { type: 'object', properties: { request_id: { type: 'string', minLength: 8, maxLength: 120 } }, required: ['request_id'], additionalProperties: false } },
   { name: 'reconnect_primary_connector', description: 'Check canonical Runtime Gateway and primary MCP reconnection readiness without publishing a release.', inputSchema: { type: 'object', additionalProperties: false } },
@@ -277,7 +304,8 @@ function requestUrl(request: IncomingMessage): URL {
   return new URL(request.url ?? '/', `${proto}://${host}`);
 }
 
-function publicOrigin(request: IncomingMessage): string {
+function publicOrigin(request: IncomingMessage, config?: Pick<RecoveryConfig, 'recoveryPublicUrl'>): string {
+  if (config?.recoveryPublicUrl) return new URL(config.recoveryPublicUrl).origin;
   const url = requestUrl(request);
   return `${url.protocol}//${url.host}`;
 }
@@ -289,12 +317,12 @@ function resourcePathFromRequest(_request: IncomingMessage): '/recovery/mcp' {
   return '/recovery/mcp';
 }
 
-function recoveryResource(request: IncomingMessage): string {
-  return `${publicOrigin(request)}${resourcePathFromRequest(request)}`;
+function recoveryResource(request: IncomingMessage, config: Pick<RecoveryConfig, 'recoveryPublicUrl'>): string {
+  return `${publicOrigin(request, config)}${resourcePathFromRequest(request)}`;
 }
 
-function recoveryAuthorizationServerMetadata(request: IncomingMessage): Record<string, unknown> {
-  const origin = publicOrigin(request);
+function recoveryAuthorizationServerMetadata(request: IncomingMessage, config: Pick<RecoveryConfig, 'recoveryPublicUrl'>): Record<string, unknown> {
+  const origin = publicOrigin(request, config);
   return {
     issuer: origin,
     authorization_endpoint: `${origin}/recovery/oauth/authorize`,
@@ -308,19 +336,37 @@ function recoveryAuthorizationServerMetadata(request: IncomingMessage): Record<s
   };
 }
 
-function recoveryProtectedResourceMetadata(request: IncomingMessage): Record<string, unknown> {
+function recoveryProtectedResourceMetadata(request: IncomingMessage, config: Pick<RecoveryConfig, 'recoveryPublicUrl'>): Record<string, unknown> {
+  const origin = publicOrigin(request, config);
   return {
-    resource: recoveryResource(request),
-    authorization_servers: [publicOrigin(request)],
+    resource: recoveryResource(request, config),
+    authorization_servers: [origin],
     scopes_supported: [RECOVERY_OAUTH_SCOPE],
     bearer_methods_supported: ['header'],
-    resource_documentation: `${publicOrigin(request)}/recovery/health`,
+    resource_documentation: `${origin}/recovery/health`,
   };
 }
 
-function recoveryWwwAuthenticate(request: IncomingMessage): string {
-  const metadata = `${publicOrigin(request)}/.well-known/oauth-protected-resource${resourcePathFromRequest(request)}`;
-  return `Bearer realm="forge-recovery", resource_metadata="${metadata}", scope="${RECOVERY_OAUTH_SCOPE}"`;
+export function recoveryWwwAuthenticate(request: IncomingMessage, config: Pick<RecoveryConfig, 'recoveryPublicUrl'>): string {
+  const metadata = `${publicOrigin(request, config)}/.well-known/oauth-protected-resource${resourcePathFromRequest(request)}`;
+  return `Bearer error="invalid_token", error_description="Missing Authorization header", resource_metadata="${metadata}"`;
+}
+
+export function recoveryUnauthorizedBody(): { error: string; message: string } {
+  return { error: 'invalid_token', message: 'Missing Authorization header' };
+}
+
+export type RecoveryMcpRequestClassification = 'not_mcp' | 'auth_required' | 'method_not_supported' | 'mcp';
+
+export function classifyRecoveryMcpRequest(
+  request: Pick<IncomingMessage, 'method' | 'url' | 'headers'>,
+  expectedToken: string | undefined,
+): RecoveryMcpRequestClassification {
+  if (!matchesAnyPath(request.url, ['/mcp', '/recovery/mcp'])) return 'not_mcp';
+  const supplied = request.headers.authorization?.replace(/^Bearer\s+/i, '').trim();
+  if (!expectedToken || !supplied || !secureEqual(supplied, expectedToken)) return 'auth_required';
+  if (request.method !== 'POST') return 'method_not_supported';
+  return 'mcp';
 }
 
 function parseUrlEncoded(input: string): URLSearchParams {
@@ -462,7 +508,7 @@ export async function dispatchRecoveryTool(config: RecoveryConfig, name: string,
     }
     case 'restart_primary_connector': {
       if (!requestId(args.request_id)) throw new Error('RECOVERY_REQUEST_ID_REQUIRED');
-      return restartPrimaryConnector(config);
+      return restartPrimaryConnector(config, { requestId: `recovery-gateway:${args.request_id}` });
     }
     case 'recover_primary_runtime': {
       if (!requestId(args.request_id)) throw new Error('RECOVERY_REQUEST_ID_REQUIRED');
@@ -473,7 +519,9 @@ export async function dispatchRecoveryTool(config: RecoveryConfig, name: string,
       if (typeof args.release_path !== 'string' || !args.release_path.trim()) throw new Error('RECOVERY_RELEASE_PATH_REQUIRED');
       if (typeof args.expected_active_release_id !== 'string' || !args.expected_active_release_id.trim()) throw new Error('RECOVERY_EXPECTED_ACTIVE_RELEASE_REQUIRED');
       if (!Number.isInteger(args.expected_authority_revision) || Number(args.expected_authority_revision) < 1) throw new Error('RECOVERY_EXPECTED_AUTHORITY_REVISION_REQUIRED');
-      return activateRuntimeRelease(config, args.release_path.trim(), {}, {
+      const releasePath = args.release_path.trim();
+      const manifestPath = basename(releasePath) === 'manifest.json' ? releasePath : join(releasePath, 'manifest.json');
+      return activateRuntimeRelease(config, manifestPath, {}, {
         requestId: `recovery-gateway:${args.request_id}`,
         expectedActiveReleaseId: args.expected_active_release_id.trim(),
         expectedAuthorityRevision: Number(args.expected_authority_revision),
@@ -498,6 +546,7 @@ async function startGateway(config: RecoveryConfig): Promise<void> {
     throw new Error('RECOVERY_GATEWAY_CONFIG_INVALID');
   }
   let runtimeIdentity: RecoveryRuntimeIdentity | undefined;
+  let watchdogRestartInFlight = false;
   const recentMutations = new Map<string, number[]>();
   const oauthCodes = new Map<string, PendingOAuthCode>();
   const oauthClients = new Map<string, OAuthClient>();
@@ -509,9 +558,18 @@ async function startGateway(config: RecoveryConfig): Promise<void> {
       return;
     }
     if (request.method === 'GET' && matchesAnyPath(request.url, ['/health', '/recovery/health'])) {
+      const watchdog = observeRecoveryWatchdogHealth(config.controllerHome);
       json(response, 200, {
-        status: 'ok',
+        status: watchdog.ok ? 'ok' : 'degraded',
         service: 'forge-standalone-recovery',
+        watchdog: {
+          ok: watchdog.ok,
+          detail: watchdog.detail,
+          pulseAgeMs: watchdog.pulseAgeMs,
+          tickAgeMs: watchdog.tickAgeMs,
+          releaseRevision: watchdog.runtimeIdentity?.releaseRevision,
+          pid: watchdog.runtimeIdentity?.pid,
+        },
         version: FORGE_VERSION,
         ...(runtimeIdentity ? {
           releasePath: runtimeIdentity.releasePath,
@@ -522,8 +580,8 @@ async function startGateway(config: RecoveryConfig): Promise<void> {
       });
       return;
     }
-    if (request.method === 'GET' && isProtectedResourceMetadataPath(request)) { json(response, 200, recoveryProtectedResourceMetadata(request)); return; }
-    if (request.method === 'GET' && isOAuthMetadataPath(request)) { json(response, 200, recoveryAuthorizationServerMetadata(request)); return; }
+    if (request.method === 'GET' && isProtectedResourceMetadataPath(request)) { json(response, 200, recoveryProtectedResourceMetadata(request, config)); return; }
+    if (request.method === 'GET' && isOAuthMetadataPath(request)) { json(response, 200, recoveryAuthorizationServerMetadata(request, config)); return; }
     if ((request.method === 'GET' || request.method === 'POST') && isAuthorizePath(request)) {
       const params = request.method === 'GET'
         ? requestUrl(request).searchParams
@@ -640,10 +698,9 @@ async function startGateway(config: RecoveryConfig): Promise<void> {
       });
       return;
     }
-    if (request.method !== 'POST' || !matchesAnyPath(request.url, ['/mcp', '/recovery/mcp'])) { json(response, 404, { error: 'NOT_FOUND' }); return; }
-    const expected = gatewayToken(config);
-    const supplied = request.headers.authorization?.replace(/^Bearer\s+/i, '').trim();
-    if (!expected || !supplied || !secureEqual(supplied, expected)) { response.setHeader('www-authenticate', recoveryWwwAuthenticate(request)); json(response, 401, { error: 'RECOVERY_AUTH_REQUIRED' }); return; }
+    const mcpRequest = classifyRecoveryMcpRequest(request, gatewayToken(config));
+    if (mcpRequest === 'not_mcp' || mcpRequest === 'method_not_supported') { json(response, 404, { error: 'NOT_FOUND' }); return; }
+    if (mcpRequest === 'auth_required') { response.setHeader('www-authenticate', recoveryWwwAuthenticate(request, config)); json(response, 401, recoveryUnauthorizedBody()); return; }
     if (!/^application\/json(?:\s*;|$)/i.test(String(request.headers['content-type'] ?? ''))) { json(response, 415, { error: 'RECOVERY_CONTENT_TYPE_REQUIRED' }); return; }
     let message: { id?: unknown; method?: unknown; params?: { name?: unknown; arguments?: unknown } };
     try { message = JSON.parse(await readBody(request)) as typeof message; } catch { json(response, 400, rpcError(null, -32700, 'Invalid JSON.')); return; }
@@ -675,6 +732,25 @@ async function startGateway(config: RecoveryConfig): Promise<void> {
   });
   await new Promise<void>((resolveListen, reject) => { server.once('error', reject); server.listen(gateway.port, gateway.host, () => resolveListen()); });
   runtimeIdentity = writeRecoveryRuntimeIdentity(config.controllerHome, 'gateway');
+  const superviseWatchdog = async () => {
+    if (!runtimeIdentity || watchdogRestartInFlight) return;
+    const watchdog = observeRecoveryWatchdogHealth(config.controllerHome);
+    if (watchdog.ok) return;
+    watchdogRestartInFlight = true;
+    try {
+      const recovery = await restartRecoveryWatchdog(config);
+      auditGateway({
+        watchdog_supervision: recovery.ok ? 'recovered' : 'failed',
+        detail: recovery.detail,
+        attempted: recovery.attempted,
+        serviceTarget: recovery.serviceTarget,
+      });
+    } finally {
+      watchdogRestartInFlight = false;
+    }
+  };
+  const watchdogSupervisor = setInterval(() => { void superviseWatchdog(); }, 15_000);
+  watchdogSupervisor.unref?.();
   process.stdout.write(JSON.stringify({ status: 'ready', host: gateway.host, port: gateway.port, runtimeIdentity }) + '\n');
 }
 

package/src/runtime/standalone-recovery/installer.ts

@@ -25,6 +25,8 @@ import {
 import { isProcessAlive } from '../shared/process-tree';
 import { FORGE_VERSION } from '../../version';
 import { initializeStandaloneRecovery, loadRecoveryConfig, type PrimaryConnectorServiceConfig, type PrimaryRuntimeServiceConfig, type PublicTunnelServiceConfig, type RecoveryConfig } from './core';
+import { RECOVERY_GATEWAY_LABEL, RECOVERY_WATCHDOG_LABEL } from './service-labels';
+export { RECOVERY_GATEWAY_LABEL, RECOVERY_WATCHDOG_LABEL } from './service-labels';
 import {
   RECOVERY_RELEASE_BINARIES,
   RECOVERY_RELEASE_ROLE_CANARY_ARG,
@@ -44,9 +46,6 @@ import {
   type RecoveryRuntimeRole,
 } from './release';
 
-export const RECOVERY_GATEWAY_LABEL = 'com.moretea.forge-recovery-gateway';
-export const RECOVERY_WATCHDOG_LABEL = 'com.moretea.forge-recovery-watchdog';
-
 function defaultRecoveryLaunchctl(args: string[]): ReturnType<LaunchctlCommandRunner> {
   const result = runProcess('launchctl', args, { timeoutMs: 10_000, maxOutputBytes: 64 * 1024 });
   return { ok: result.ok, stdout: result.stdout, stderr: result.stderr, exitCode: result.status ?? 0 };
@@ -341,7 +340,7 @@ function recoveryPlist(input: {
 }): string {
   const xml = (value: string) => value.replace(/&/g, '&amp;').replace(/</g, '&lt;').replace(/>/g, '&gt;');
   const argumentsList = [
-    '/usr/bin/env', '-i', 'PATH=/usr/bin:/bin:/usr/sbin:/sbin', `FORGE_BUILD_VERSION=${FORGE_VERSION}`,
+    '/usr/bin/env', '-i', 'PATH=/usr/bin:/bin:/usr/sbin:/sbin', `FORGE_BUILD_VERSION=${FORGE_VERSION}`, `FORGE_CONNECTOR_EXECUTABLE=${process.execPath}`,
     input.executable, input.command, '--controller-home', input.controllerHome,
   ];
   return `<?xml version="1.0" encoding="UTF-8"?>\n<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">\n<plist version="1.0"><dict><key>Label</key><string>${xml(input.label)}</string><key>ProgramArguments</key><array>${argumentsList.map((argument) => `<string>${xml(argument)}</string>`).join('')}</array><key>RunAtLoad</key><true/><key>KeepAlive</key><true/><key>ThrottleInterval</key><integer>5</integer><key>StandardOutPath</key><string>${xml(input.logPath)}</string><key>StandardErrorPath</key><string>${xml(input.logPath)}</string></dict></plist>\n`;
@@ -682,6 +681,12 @@ export function inspectPrimaryConnectorLaunchdContract(
   return inspectLaunchdRestartContract(service, true);
 }
 
+export function inspectPrimaryPublicTunnelLaunchdContract(
+  service: Pick<PublicTunnelServiceConfig, 'platform' | 'label' | 'plistPath'>,
+): RecoveryTunnelLaunchdContract {
+  return inspectLaunchdRestartContract(service, true);
+}
+
 export async function installStandaloneRecovery(input: {
   controllerHome: string;
   repoRoot: string;
@@ -690,6 +695,7 @@ export async function installStandaloneRecovery(input: {
   publicMcpUrl?: string;
   recoveryPublicUrl?: string;
   recoveryTunnelService?: PublicTunnelServiceConfig;
+  primaryPublicTunnelService?: PublicTunnelServiceConfig;
   primaryRuntimeService?: PrimaryRuntimeServiceConfig;
   primaryConnectorService?: PrimaryConnectorServiceConfig;
   stageOnly?: boolean;
@@ -705,6 +711,15 @@ export async function installStandaloneRecovery(input: {
       throw new Error('RECOVERY_TUNNEL_LAUNCHD_RESTART_CONTRACT_REQUIRED: RunAtLoad=true and KeepAlive=true');
     }
   }
+  if (input.primaryPublicTunnelService?.platform === 'launchd') {
+    const tunnelContract = inspectPrimaryPublicTunnelLaunchdContract(input.primaryPublicTunnelService);
+    if (!tunnelContract.plistInstalled) {
+      throw new Error(`RECOVERY_PRIMARY_TUNNEL_LAUNCHD_PLIST_MISSING: ${tunnelContract.plistPath}`);
+    }
+    if (!tunnelContract.restartSafe) {
+      throw new Error('RECOVERY_PRIMARY_TUNNEL_LAUNCHD_RESTART_CONTRACT_REQUIRED: RunAtLoad=true and KeepAlive=true or KeepAlive.SuccessfulExit=false');
+    }
+  }
   if (input.primaryConnectorService?.platform === 'launchd') {
     const connectorContract = inspectPrimaryConnectorLaunchdContract(input.primaryConnectorService);
     if (!connectorContract.plistInstalled) {
@@ -718,6 +733,7 @@ export async function installStandaloneRecovery(input: {
     ...(input.publicMcpUrl ? { publicMcpUrl: input.publicMcpUrl } : {}),
     ...(input.recoveryPublicUrl ? { recoveryPublicUrl: input.recoveryPublicUrl } : {}),
     ...(input.recoveryTunnelService ? { recoveryTunnelService: input.recoveryTunnelService } : {}),
+    ...(input.primaryPublicTunnelService ? { primaryPublicTunnelService: input.primaryPublicTunnelService } : {}),
     ...(input.primaryRuntimeService ? { primaryRuntimeService: input.primaryRuntimeService } : {}),
     primaryRuntimeSourceRoot: resolve(input.repoRoot),
     ...(input.primaryConnectorService ? { primaryConnectorService: input.primaryConnectorService } : {}),

package/src/runtime/standalone-recovery/service-labels.ts

@@ -0,0 +1,2 @@
+export const RECOVERY_GATEWAY_LABEL = 'com.moretea.forge-recovery-gateway';
+export const RECOVERY_WATCHDOG_LABEL = 'com.moretea.forge-recovery-watchdog';

package/src/runtime/standalone-recovery/watchdog-heartbeat.ts

@@ -0,0 +1,160 @@
+import { readFileSync } from 'fs';
+import { join } from 'path';
+import { writeJsonAtomic } from '../shared/json-files';
+import {
+  readCurrentRecoveryRelease,
+  readRecoveryRuntimeIdentity,
+  recoveryRoot,
+  type RecoveryReleaseDescriptor,
+  type RecoveryRuntimeIdentity,
+} from './release';
+
+export const RECOVERY_WATCHDOG_MAX_PULSE_AGE_MS = 20_000;
+export const RECOVERY_WATCHDOG_MAX_TICK_AGE_MS = 180_000;
+
+export interface RecoveryWatchdogHeartbeat {
+  schemaVersion: 1;
+  pid: number;
+  startedAt: string;
+  releasePath: string;
+  releaseRevision: string;
+  sourceCommit: string;
+  manifestSha256: string;
+  lastPulseAt: string;
+  lastTickStartedAt?: string;
+  lastTickCompletedAt?: string;
+  lastTickFailedAt?: string;
+  lastError?: string;
+}
+
+export interface RecoveryWatchdogHealth {
+  ok: boolean;
+  detail: string;
+  pulseAgeMs?: number;
+  tickAgeMs?: number;
+  heartbeat?: RecoveryWatchdogHeartbeat;
+  runtimeIdentity?: RecoveryRuntimeIdentity;
+  currentReleaseRevision?: string;
+}
+
+export function recoveryWatchdogHeartbeatPath(controllerHome: string): string {
+  return join(recoveryRoot(controllerHome), 'state', 'watchdog-heartbeat.json');
+}
+
+export function createRecoveryWatchdogHeartbeat(
+  identity: RecoveryRuntimeIdentity | undefined,
+  nowMs = Date.now(),
+): RecoveryWatchdogHeartbeat {
+  const now = new Date(nowMs).toISOString();
+  return {
+    schemaVersion: 1,
+    pid: identity?.pid ?? process.pid,
+    startedAt: identity?.startedAt ?? now,
+    releasePath: identity?.releasePath ?? 'unmanaged-source',
+    releaseRevision: identity?.releaseRevision ?? 'unmanaged-source',
+    sourceCommit: identity?.sourceCommit ?? 'unmanaged-source',
+    manifestSha256: identity?.manifestSha256 ?? 'unmanaged-source',
+    lastPulseAt: now,
+  };
+}
+
+export function writeRecoveryWatchdogHeartbeat(
+  controllerHome: string,
+  heartbeat: RecoveryWatchdogHeartbeat,
+): RecoveryWatchdogHeartbeat {
+  writeJsonAtomic(recoveryWatchdogHeartbeatPath(controllerHome), heartbeat);
+  return heartbeat;
+}
+
+export function readRecoveryWatchdogHeartbeat(controllerHome: string): RecoveryWatchdogHeartbeat | undefined {
+  try {
+    const parsed = JSON.parse(readFileSync(recoveryWatchdogHeartbeatPath(controllerHome), 'utf8')) as RecoveryWatchdogHeartbeat;
+    if (
+      parsed.schemaVersion !== 1
+      || !Number.isInteger(parsed.pid)
+      || typeof parsed.lastPulseAt !== 'string'
+      || typeof parsed.releaseRevision !== 'string'
+      || typeof parsed.manifestSha256 !== 'string'
+    ) return undefined;
+    return parsed;
+  } catch {
+    return undefined;
+  }
+}
+
+function pidExists(pid: number): boolean {
+  if (!Number.isInteger(pid) || pid <= 0) return false;
+  try {
+    process.kill(pid, 0);
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+function latestTerminalTickAt(heartbeat: RecoveryWatchdogHeartbeat): number {
+  return Math.max(
+    heartbeat.lastTickCompletedAt ? Date.parse(heartbeat.lastTickCompletedAt) : 0,
+    heartbeat.lastTickFailedAt ? Date.parse(heartbeat.lastTickFailedAt) : 0,
+  );
+}
+
+export function evaluateRecoveryWatchdogHealth(input: {
+  heartbeat?: RecoveryWatchdogHeartbeat;
+  runtimeIdentity?: RecoveryRuntimeIdentity;
+  currentRelease?: Pick<RecoveryReleaseDescriptor, 'releasePath' | 'releaseRevision' | 'sourceCommit' | 'manifestSha256'>;
+  nowMs?: number;
+  maxPulseAgeMs?: number;
+  maxTickAgeMs?: number;
+  pidAlive?: (pid: number) => boolean;
+}): RecoveryWatchdogHealth {
+  const heartbeat = input.heartbeat;
+  const runtimeIdentity = input.runtimeIdentity;
+  const current = input.currentRelease;
+  if (!current) return { ok: false, detail: 'current immutable Recovery release is unavailable' };
+  if (!runtimeIdentity) return { ok: false, detail: 'Recovery Watchdog runtime identity is unavailable', currentReleaseRevision: current.releaseRevision };
+  if (!heartbeat) return { ok: false, detail: 'Recovery Watchdog heartbeat is unavailable', runtimeIdentity, currentReleaseRevision: current.releaseRevision };
+  if (heartbeat.pid !== runtimeIdentity.pid) {
+    return { ok: false, detail: 'Recovery Watchdog heartbeat PID does not match runtime identity', heartbeat, runtimeIdentity, currentReleaseRevision: current.releaseRevision };
+  }
+  if (
+    runtimeIdentity.releasePath !== current.releasePath
+    || runtimeIdentity.releaseRevision !== current.releaseRevision
+    || runtimeIdentity.sourceCommit !== current.sourceCommit
+    || runtimeIdentity.manifestSha256 !== current.manifestSha256
+    || heartbeat.releasePath !== current.releasePath
+    || heartbeat.releaseRevision !== current.releaseRevision
+    || heartbeat.sourceCommit !== current.sourceCommit
+    || heartbeat.manifestSha256 !== current.manifestSha256
+  ) {
+    return { ok: false, detail: 'Recovery Watchdog is not running the current immutable Recovery release', heartbeat, runtimeIdentity, currentReleaseRevision: current.releaseRevision };
+  }
+  if (!(input.pidAlive ?? pidExists)(runtimeIdentity.pid)) {
+    return { ok: false, detail: 'Recovery Watchdog process is not alive', heartbeat, runtimeIdentity, currentReleaseRevision: current.releaseRevision };
+  }
+  const now = input.nowMs ?? Date.now();
+  const pulseAt = Date.parse(heartbeat.lastPulseAt);
+  const pulseAgeMs = Number.isFinite(pulseAt) ? Math.max(0, now - pulseAt) : Number.POSITIVE_INFINITY;
+  const maxPulseAgeMs = Math.max(1_000, input.maxPulseAgeMs ?? RECOVERY_WATCHDOG_MAX_PULSE_AGE_MS);
+  if (pulseAgeMs > maxPulseAgeMs) {
+    return { ok: false, detail: 'Recovery Watchdog heartbeat is stale', pulseAgeMs, heartbeat, runtimeIdentity, currentReleaseRevision: current.releaseRevision };
+  }
+  const tickStartedAt = heartbeat.lastTickStartedAt ? Date.parse(heartbeat.lastTickStartedAt) : 0;
+  const tickTerminalAt = latestTerminalTickAt(heartbeat);
+  if (tickStartedAt > tickTerminalAt) {
+    const tickAgeMs = Math.max(0, now - tickStartedAt);
+    const maxTickAgeMs = Math.max(maxPulseAgeMs, input.maxTickAgeMs ?? RECOVERY_WATCHDOG_MAX_TICK_AGE_MS);
+    if (tickAgeMs > maxTickAgeMs) {
+      return { ok: false, detail: 'Recovery Watchdog tick is stuck beyond its bounded recovery window', pulseAgeMs, tickAgeMs, heartbeat, runtimeIdentity, currentReleaseRevision: current.releaseRevision };
+    }
+  }
+  return { ok: true, detail: 'Recovery Watchdog heartbeat, PID, and immutable release identity are current', pulseAgeMs, heartbeat, runtimeIdentity, currentReleaseRevision: current.releaseRevision };
+}
+
+export function observeRecoveryWatchdogHealth(controllerHome: string): RecoveryWatchdogHealth {
+  return evaluateRecoveryWatchdogHealth({
+    heartbeat: readRecoveryWatchdogHeartbeat(controllerHome),
+    runtimeIdentity: readRecoveryRuntimeIdentity(controllerHome, 'watchdog'),
+    currentRelease: readCurrentRecoveryRelease(controllerHome),
+  });
+}

package/src/runtime/workflow/schedules/browser-probe.ts

@@ -95,10 +95,70 @@ async function browserAction(
   });
 }
 
+interface ScheduledBrowserSessionMetadata {
+  url?: string;
+  ownership?: string;
+  provider?: string;
+  activeMode?: string;
+}
+
+export type ScheduledBrowserProbeNavigationAction = 'navigate' | 'reload' | 'wait_for_load_state';
+
+export function scheduledBrowserProbeNavigationAction(
+  ownership: string | undefined,
+  hasUrl: boolean,
+): ScheduledBrowserProbeNavigationAction {
+  if (ownership === 'user_owned') return 'wait_for_load_state';
+  return hasUrl ? 'navigate' : 'reload';
+}
+
+export function scheduledBrowserProbeManagedRehydrateIntent(
+  savedSession: Pick<ScheduledBrowserSessionMetadata, 'ownership' | 'provider' | 'activeMode'> | undefined,
+): Record<string, true> {
+  return savedSession?.ownership === 'plugin_owned'
+    && savedSession.provider === 'playwright-persistent-context'
+    && savedSession.activeMode === 'managed_persistent'
+    ? { __forge_allow_managed_session_rehydrate: true }
+    : {};
+}
+
+function comparableProbeUrl(value: string): string {
+  try {
+    const parsed = new URL(value);
+    parsed.hash = '';
+    return parsed.toString();
+  } catch {
+    return value;
+  }
+}
+
+async function scheduledBrowserSessionMetadata(
+  controllerHome: string,
+  repository: RepositoryRecord,
+  occurrenceId: string,
+  sessionId: string,
+): Promise<ScheduledBrowserSessionMetadata | undefined> {
+  const inventory = await browserAction(controllerHome, repository, occurrenceId, 'list_sessions', {});
+  const sessions = Array.isArray(inventory.sessions) ? inventory.sessions : [];
+  const saved = sessions
+    .map((entry) => recordValue(entry))
+    .find((entry) => stringValue(entry.sessionId) === sessionId);
+  if (!saved) return undefined;
+  const browser = recordValue(saved.browser);
+  const tab = recordValue(browser.tab);
+  return {
+    url: stringValue(saved.url),
+    ownership: stringValue(tab.ownership),
+    provider: stringValue(browser.provider),
+    activeMode: stringValue(browser.activeMode),
+  };
+}
+
 /**
  * Bounded read-only browser observation for Schedule. It never interprets page
- * meaning: it only refreshes/navigates, extracts bounded text, projects matching
- * lines, detects configured login markers, and returns a stable fingerprint.
+ * meaning: it only refreshes/navigates plugin-owned targets, observes user-owned
+ * targets without mutating them, extracts bounded text, projects matching lines,
+ * detects configured login markers, and returns a stable fingerprint.
  */
 export async function executeScheduledBrowserProbe(input: {
   controllerHome: string;
@@ -112,20 +172,38 @@ export async function executeScheduledBrowserProbe(input: {
 
   const waitUntil = stringValue(input.args.wait_until) ?? 'domcontentloaded';
   const timeoutMs = boundedNumber(input.args.timeout_ms, 60_000, 1_000, 120_000);
+  const savedSession = requestedSessionId
+    ? await scheduledBrowserSessionMetadata(input.controllerHome, input.repository, input.occurrenceId, requestedSessionId)
+    : undefined;
+  const navigationAction = scheduledBrowserProbeNavigationAction(savedSession?.ownership, Boolean(url));
+  const managedRehydrateIntent = scheduledBrowserProbeManagedRehydrateIntent(savedSession);
   let navigation: Record<string, unknown>;
-  if (url) {
+  if (navigationAction === 'wait_for_load_state') {
+    if (!savedSession) throw new Error('SCHEDULE_BROWSER_PROBE_USER_SESSION_STATE_UNAVAILABLE');
+    if (url && savedSession.url && comparableProbeUrl(url) !== comparableProbeUrl(savedSession.url)) {
+      throw new Error('SCHEDULE_BROWSER_PROBE_USER_SESSION_NAVIGATION_REFUSED');
+    }
+    navigation = await browserAction(input.controllerHome, input.repository, input.occurrenceId, 'wait_for_load_state', {
+      session_id: requestedSessionId!,
+      state: waitUntil,
+      timeout_ms: timeoutMs,
+      ...managedRehydrateIntent,
+    });
+  } else if (navigationAction === 'navigate') {
     navigation = await browserAction(input.controllerHome, input.repository, input.occurrenceId, 'navigate', {
       ...(requestedSessionId ? { session_id: requestedSessionId } : {}),
       url,
       wait_until: waitUntil,
       timeout_ms: timeoutMs,
       retries: 1,
+      ...managedRehydrateIntent,
     });
   } else {
     navigation = await browserAction(input.controllerHome, input.repository, input.occurrenceId, 'reload', {
       session_id: requestedSessionId!,
       wait_until: waitUntil,
       timeout_ms: timeoutMs,
+      ...managedRehydrateIntent,
     });
   }