@moretea-labs/forge 1.6.0 → 1.7.1

package/src/runtime/control-plane/global-scheduler/worker-lifecycle.ts

@@ -0,0 +1,150 @@
+import type { ExecutionWorkerLifecycle } from '../../execution/jobs/types';
+import type { SchedulerWorkerLaunchDescriptor } from './worker-launch';
+
+export type SchedulerWorkerReleaseIdentity = Pick<
+  ExecutionWorkerLifecycle,
+  'runtimeInstanceId' | 'releaseAuthorityRevision' | 'releaseId' | 'artifactIdentity' | 'workerProtocolVersion'
+>;
+
+export function buildSchedulerWorkerSpawnFailureLifecycle(input: {
+  executable: string;
+  cwd: string;
+  environment: Record<string, string | undefined>;
+  ownerPid: number;
+  attempt: number;
+  maxAttempts: number;
+  spawnedAt?: string;
+}): ExecutionWorkerLifecycle {
+  return {
+    executable: input.executable,
+    args: [],
+    cwd: input.cwd,
+    environment: input.environment,
+    ownerPid: input.ownerPid,
+    attempt: input.attempt,
+    maxAttempts: input.maxAttempts,
+    spawnedAt: input.spawnedAt ?? new Date().toISOString(),
+    startupState: 'spawn_failed',
+  };
+}
+
+export function buildSchedulerWorkerSpawnedLifecycle(input: {
+  launch: SchedulerWorkerLaunchDescriptor;
+  ownerPid: number;
+  releaseIdentity?: SchedulerWorkerReleaseIdentity;
+  attempt: number;
+  maxAttempts: number;
+  stderrPath: string;
+  spawnedAt?: string;
+}): ExecutionWorkerLifecycle {
+  return {
+    executable: input.launch.executable,
+    args: input.launch.args,
+    cwd: input.launch.cwd,
+    environment: input.launch.lifecycleEnvironment,
+    ownerPid: input.ownerPid,
+    ...(input.releaseIdentity ?? {}),
+    attempt: input.attempt,
+    maxAttempts: input.maxAttempts,
+    spawnedAt: input.spawnedAt ?? new Date().toISOString(),
+    stderrPath: input.stderrPath,
+    startupState: 'spawned',
+  };
+}
+
+export function buildSchedulerWorkerExitedLifecycle(input: {
+  lifecycle: ExecutionWorkerLifecycle;
+  childPid?: number;
+  platform?: NodeJS.Platform;
+  exitCode: number | null;
+  signal: string | null;
+  stderr: string;
+  stderrTruncated: boolean;
+  startupError?: string;
+  exitedAt?: string;
+}): ExecutionWorkerLifecycle {
+  const platform = input.platform ?? process.platform;
+  return {
+    ...input.lifecycle,
+    exitedAt: input.exitedAt ?? new Date().toISOString(),
+    exitCode: input.exitCode,
+    signal: input.signal,
+    workerPid: input.childPid ?? input.lifecycle.workerPid,
+    processGroupId: input.lifecycle.processGroupId ?? (platform !== 'win32' ? input.childPid : undefined),
+    stderr: input.stderr,
+    stderrTruncated: input.stderrTruncated,
+    startupState: input.startupError ? 'spawn_failed' : 'exited',
+  };
+}
+
+export function buildSchedulerWorkerRegisteredLifecycle(input: {
+  lifecycle: ExecutionWorkerLifecycle;
+  currentLifecycle?: ExecutionWorkerLifecycle;
+  workerPid: number;
+  platform?: NodeJS.Platform;
+  attachedAt?: string;
+}): ExecutionWorkerLifecycle {
+  const platform = input.platform ?? process.platform;
+  return {
+    ...(input.currentLifecycle ?? input.lifecycle),
+    attachedAt: input.attachedAt ?? new Date().toISOString(),
+    processGroupId: platform !== 'win32' ? input.workerPid : undefined,
+    workerPid: input.workerPid,
+    startupState: 'registered',
+  };
+}
+
+export interface SchedulerWorkerExitFailure {
+  retryable: boolean;
+  error: {
+    code: 'WORKER_START_FAILED' | 'WORKER_EXITED';
+    message: string;
+    retryable: boolean;
+    details: Record<string, unknown>;
+  };
+}
+
+export function buildSchedulerWorkerExitFailure(input: {
+  lifecycle: ExecutionWorkerLifecycle;
+  attempt: number;
+  maxAttempts: number;
+  exitCode: number | null;
+  signal: string | null;
+  stderr: string;
+  stderrTruncated: boolean;
+  startupError?: string;
+}): SchedulerWorkerExitFailure {
+  const retryable = input.attempt < input.maxAttempts;
+  const details: Record<string, unknown> = {
+    workerLostReason: input.startupError ? 'spawn_failed' : 'process_exit',
+    executable: input.lifecycle.executable,
+    cwd: input.lifecycle.cwd,
+    exitCode: input.exitCode,
+    signal: input.signal,
+    stderr: input.stderr,
+    stderrTruncated: input.stderrTruncated,
+    stderrPath: input.lifecycle.stderrPath,
+    processGroupId: input.lifecycle.processGroupId,
+    ownerPid: input.lifecycle.ownerPid,
+    runtimeInstanceId: input.lifecycle.runtimeInstanceId,
+    releaseAuthorityRevision: input.lifecycle.releaseAuthorityRevision,
+    releaseId: input.lifecycle.releaseId,
+    artifactIdentity: input.lifecycle.artifactIdentity,
+    workerProtocolVersion: input.lifecycle.workerProtocolVersion,
+    attempt: input.attempt,
+    maxAttempts: input.maxAttempts,
+    ...(input.startupError ? { startupError: input.startupError } : {}),
+  };
+  const stderrSummary = input.stderr.trim() ? ` Worker stderr: ${input.stderr.trim()}` : '';
+  const startupSummary = input.startupError ? ` Startup error: ${input.startupError}.` : '';
+  const message = `Execution Worker ${input.lifecycle.executable} exited before completion (cwd ${input.lifecycle.cwd}, exit code ${input.exitCode ?? 'unknown'}${input.signal ? `, signal ${input.signal}` : ''}).${startupSummary}${stderrSummary}`;
+  return {
+    retryable,
+    error: {
+      code: input.startupError ? 'WORKER_START_FAILED' : 'WORKER_EXITED',
+      message,
+      retryable,
+      details,
+    },
+  };
+}

package/src/runtime/control-plane/global-scheduler/worker-process.ts

@@ -0,0 +1,119 @@
+import { spawn, type ChildProcess, type SpawnOptions } from 'child_process';
+import { terminateProcessTree } from '../../shared/process-tree';
+import type { SchedulerWorkerLaunchDescriptor } from './worker-launch';
+import type { SchedulerWorkerStderrCapture } from './worker-stderr';
+
+export interface SchedulerWorkerSpawnDependencies {
+  spawnProcess: typeof spawn;
+  platform: NodeJS.Platform;
+}
+
+const DEFAULT_SPAWN_DEPENDENCIES: SchedulerWorkerSpawnDependencies = {
+  spawnProcess: spawn,
+  platform: process.platform,
+};
+
+export type SchedulerWorkerSpawnResult =
+  | { ok: true; child: ChildProcess }
+  | { ok: false; startupError: string };
+
+export function spawnSchedulerWorkerProcess(
+  launch: SchedulerWorkerLaunchDescriptor,
+  dependencies: SchedulerWorkerSpawnDependencies = DEFAULT_SPAWN_DEPENDENCIES,
+): SchedulerWorkerSpawnResult {
+  const options: SpawnOptions = {
+    cwd: launch.cwd,
+    stdio: ['ignore', 'ignore', 'pipe'],
+    detached: dependencies.platform !== 'win32',
+    env: launch.environment,
+  };
+  try {
+    return {
+      ok: true,
+      child: dependencies.spawnProcess(launch.executable, launch.args, options),
+    };
+  } catch (error) {
+    return {
+      ok: false,
+      startupError: error instanceof Error ? error.message : String(error),
+    };
+  }
+}
+
+export interface SchedulerWorkerRegistrationDependencies {
+  terminateWorker(pid: number | undefined): Promise<unknown>;
+}
+
+const DEFAULT_REGISTRATION_DEPENDENCIES: SchedulerWorkerRegistrationDependencies = {
+  terminateWorker: terminateProcessTree,
+};
+
+export function registerSchedulerWorkerProcess(input: {
+  jobId: string;
+  child: ChildProcess;
+  children: Map<string, ChildProcess>;
+  attach(workerPid: number): boolean;
+}, dependencies: SchedulerWorkerRegistrationDependencies = DEFAULT_REGISTRATION_DEPENDENCIES): boolean {
+  const { child } = input;
+  if (!child.pid) {
+    child.unref();
+    return false;
+  }
+
+  input.children.set(input.jobId, child);
+  if (!input.attach(child.pid)) {
+    input.children.delete(input.jobId);
+    void dependencies.terminateWorker(child.pid);
+    child.unref();
+    return false;
+  }
+
+  child.unref();
+  return true;
+}
+
+export async function cleanupSchedulerWorkerProcesses(
+  children: Map<string, ChildProcess>,
+  dependencies: SchedulerWorkerRegistrationDependencies = DEFAULT_REGISTRATION_DEPENDENCIES,
+): Promise<void> {
+  const workers = [...children.values()];
+  children.clear();
+  await Promise.all(workers.map((child) => dependencies.terminateWorker(child.pid)));
+}
+
+export interface SchedulerWorkerProcessExit {
+  exitCode: number | null;
+  signal: string | null;
+  stderr: string;
+  stderrTruncated: boolean;
+  startupError?: string;
+}
+
+export function wireSchedulerWorkerProcess(input: {
+  jobId: string;
+  child: ChildProcess;
+  children: Map<string, ChildProcess>;
+  stderrCapture: SchedulerWorkerStderrCapture;
+  onExit(exit: SchedulerWorkerProcessExit): void;
+}): void {
+  const { child } = input;
+  child.stderr?.on('data', (chunk: Buffer | string) => input.stderrCapture.append(chunk));
+
+  let finalized = false;
+  const finalize = (exitCode: number | null, signal: string | null, startupError?: string) => {
+    if (finalized) return;
+    finalized = true;
+    if (input.children.get(input.jobId) === child) input.children.delete(input.jobId);
+    const snapshot = input.stderrCapture.snapshot();
+    input.onExit({
+      exitCode,
+      signal,
+      stderr: snapshot.stderr,
+      stderrTruncated: snapshot.stderrTruncated,
+      startupError,
+    });
+  };
+
+  child.once('error', (error) => finalize(null, null, error.message));
+  child.once('close', (code, signal) => finalize(code, signal));
+}

package/src/runtime/control-plane/global-scheduler/worker-stderr.ts

@@ -0,0 +1,60 @@
+import { appendFileSync, mkdirSync, writeFileSync } from 'fs';
+import { dirname, join } from 'path';
+import { executionJobRoot } from '../../execution/jobs/store';
+
+export const MAX_SCHEDULER_WORKER_STDERR_BYTES = 16 * 1024;
+
+export interface SchedulerWorkerStderrSnapshot {
+  stderr: string;
+  stderrTruncated: boolean;
+}
+
+export interface SchedulerWorkerStderrCapture {
+  path: string;
+  append(chunk: Buffer | string): void;
+  snapshot(): SchedulerWorkerStderrSnapshot;
+}
+
+export function createSchedulerWorkerStderrCapture(input: {
+  controllerHome: string;
+  repoId: string;
+  jobId: string;
+  attempt: number;
+  maxBytes?: number;
+}): SchedulerWorkerStderrCapture {
+  const path = join(
+    executionJobRoot(input.controllerHome, input.repoId),
+    'worker-stderr',
+    `${input.jobId}-attempt-${input.attempt}.log`,
+  );
+  mkdirSync(dirname(path), { recursive: true });
+  writeFileSync(path, '', 'utf8');
+
+  const maxBytes = input.maxBytes ?? MAX_SCHEDULER_WORKER_STDERR_BYTES;
+  let stderr = '';
+  let stderrBytes = 0;
+  let stderrTruncated = false;
+
+  return {
+    path,
+    append(chunk) {
+      const text = typeof chunk === 'string' ? chunk : chunk.toString('utf8');
+      const bytes = Buffer.byteLength(text);
+      const remaining = maxBytes - stderrBytes;
+      const accepted = remaining > 0 ? Buffer.from(text).subarray(0, remaining).toString('utf8') : '';
+      if (accepted) {
+        stderr += accepted;
+        stderrBytes += Buffer.byteLength(accepted);
+        try {
+          appendFileSync(path, accepted, 'utf8');
+        } catch {
+          stderrTruncated = true;
+        }
+      }
+      if (bytes > Math.max(0, remaining)) stderrTruncated = true;
+    },
+    snapshot() {
+      return { stderr, stderrTruncated };
+    },
+  };
+}

package/src/runtime/control-plane/launcher/chatgpt-work-binding-store.ts

@@ -47,6 +47,19 @@ export function parseChatgptConversationIdentity(value: string): { conversationU
   return { conversationUrl: url.toString(), conversationId };
 }
 
+export function hasChatgptConversationIdentity(value: string): boolean {
+  try {
+    parseChatgptConversationIdentity(value);
+    return true;
+  } catch (error) {
+    // ChatGPT root and Project URLs are valid launch seeds, but they are not
+    // durable conversation identities. Persist only the /c/<id> URL observed
+    // after the browser confirms submission.
+    if (error instanceof Error && error.message === 'CHATGPT_WORK_CONVERSATION_ID_MISSING') return false;
+    throw error;
+  }
+}
+
 function record(options: ChatgptWorkBindingStoreOptions, workId: string) {
   return readControlPlaneRecord<ChatgptWorkConversationBinding>(
     options.controllerHome,
@@ -63,6 +76,56 @@ export function getChatgptWorkConversationBinding(
   return record(options, workId)?.value;
 }
 
+export function rebindChatgptWorkConversation(
+  options: ChatgptWorkBindingStoreOptions,
+  input: {
+    workId: string;
+    previousConversationId: string;
+    conversationUrl: string;
+    latestBrowserSessionId?: string;
+    localAlias?: string;
+  },
+): ChatgptWorkConversationBinding {
+  if (!input.workId.trim()) throw new Error('CHATGPT_WORK_BINDING_WORK_REQUIRED');
+  if (!input.previousConversationId.trim()) throw new Error('CHATGPT_WORK_REBIND_PREVIOUS_CONVERSATION_REQUIRED');
+  const identity = parseChatgptConversationIdentity(input.conversationUrl);
+  return withControllerLock(
+    options.controllerHome,
+    { scope: 'task', repoId: options.repoId, taskId: `chatgpt-work-binding-${input.workId}` },
+    `chatgpt-work-rebind:${input.workId}`,
+    () => {
+      const existing = record(options, input.workId);
+      if (!existing) throw new Error(`CHATGPT_WORK_CONVERSATION_BINDING_NOT_FOUND: ${input.workId}`);
+      if (existing.value.conversationId !== input.previousConversationId.trim()) {
+        throw new Error(`CHATGPT_WORK_CONVERSATION_REBIND_STALE: ${input.workId}:${existing.value.conversationId}`);
+      }
+      const now = nowIso(options);
+      const binding: ChatgptWorkConversationBinding = {
+        schemaVersion: 1,
+        repoId: options.repoId,
+        workId: input.workId,
+        conversationUrl: identity.conversationUrl,
+        conversationId: identity.conversationId,
+        localAlias: (input.localAlias?.trim() || existing.value.localAlias).slice(0, 180),
+        latestBrowserSessionId: input.latestBrowserSessionId ?? existing.value.latestBrowserSessionId,
+        createdAt: existing.value.createdAt,
+        updatedAt: now,
+        lastContinuedAt: now,
+      };
+      writeControlPlaneRecord(options.controllerHome, {
+        namespace: NAMESPACE,
+        scope: options.repoId,
+        key: input.workId,
+        schemaVersion: 1,
+        value: binding,
+        action: 'chatgpt_work_conversation_rebind',
+        expectedRevision: existing.revision,
+      });
+      return binding;
+    },
+  );
+}
+
 export function bindChatgptWorkConversation(
   options: ChatgptWorkBindingStoreOptions,
   input: {