@treeseed/sdk 0.10.28 → 0.11.0

package/dist/operations/services/railway-deploy.js

@@ -3,6 +3,7 @@ import { dirname, relative, resolve } from "node:path";
 import { spawnSync } from "node:child_process";
 import { loadCliDeployConfig } from "./runtime-tools.js";
 import { createPersistentDeployTarget, resolveTreeseedResourceIdentity } from "./deploy.js";
+import { discoverTreeseedApplications } from "../../hosting/apps.js";
 import { runPrefixedCommand, sleep } from "./bootstrap-runner.js";
 import { resolveTreeseedToolCommand } from "../../managed-dependencies.js";
 import {
@@ -11,6 +12,7 @@ import {
   ensureRailwayService,
   ensureRailwayServiceInstanceConfiguration,
   ensureRailwayServiceVolume,
+  deployRailwayServiceInstance,
   getRailwayServiceInstance,
   listRailwayEnvironments,
   listRailwayProjects,
@@ -31,37 +33,49 @@ function normalizeScope(scope) {
 function resolveRailwayEnvironmentForScope(scope, configuredEnvironment) {
   return normalizeRailwayEnvironmentName(configuredEnvironment || normalizeScope(scope));
 }
-const RAILWAY_SERVICE_KEYS = ["api", "marketOperationsRunner"];
+const RAILWAY_SERVICE_KEYS = ["api", "operationsRunner"];
 const HOSTED_PROJECT_SERVICE_KEYS = ["api"];
 const WORKER_RUNNER_BOOTSTRAP_INDEX = 1;
 const WORKER_RUNNER_VOLUME_MOUNT_PATH = "/data";
-const MARKET_OPERATIONS_RUNNER_BOOTSTRAP_COUNT = 2;
+const OPERATIONS_RUNNER_BOOTSTRAP_COUNT = 2;
+function isTreeseedOperationsRunnerResourceName(value) {
+  const normalized = String(value ?? "").trim().toLowerCase();
+  if (!normalized) {
+    return false;
+  }
+  if (normalized.startsWith("market-ops")) {
+    return true;
+  }
+  return normalized.includes("operations-runner");
+}
+function findStaleTreeseedOperationsRunnerResources(resources, desiredNames) {
+  const desired = new Set([...desiredNames].map((value) => String(value ?? "").trim()).filter(Boolean));
+  return resources.filter((resource) => {
+    const name = String(resource?.name ?? "").trim();
+    return name && isTreeseedOperationsRunnerResourceName(name) && !desired.has(name);
+  });
+}
 function shouldManageRailwaySchedules(scope, phase = "deploy") {
   const environment = normalizeRailwayEnvironmentName(scope);
   return phase === "deploy" && (environment === "staging" || environment === "production");
 }
 function railwayServiceNameSuffix(serviceKey) {
-  return serviceKey === "workdayManager" ? "workday-manager" : serviceKey === "workerRunner" ? "worker-runner" : serviceKey === "marketOperationsRunner" ? "market-operations-runner" : serviceKey;
+  return serviceKey === "workdayManager" ? "workday-manager" : serviceKey === "workerRunner" ? "worker-runner" : serviceKey === "operationsRunner" ? "operations-runner" : serviceKey;
 }
 function deriveRailwayWorkerRunnerServiceName(projectSlug, index = WORKER_RUNNER_BOOTSTRAP_INDEX) {
   const normalizedIndex = Math.max(1, Number.parseInt(String(index), 10) || WORKER_RUNNER_BOOTSTRAP_INDEX);
   return `${projectSlug}-worker-runner-${String(normalizedIndex).padStart(2, "0")}`;
 }
-function deriveRailwayMarketOperationsRunnerServiceName(baseServiceName, index = WORKER_RUNNER_BOOTSTRAP_INDEX) {
+function deriveRailwayOperationsRunnerServiceName(baseServiceName, index = WORKER_RUNNER_BOOTSTRAP_INDEX) {
   const normalizedIndex = Math.max(1, Number.parseInt(String(index), 10) || WORKER_RUNNER_BOOTSTRAP_INDEX);
-  const base = String(baseServiceName ?? "").trim().replace(/-\d+$/u, "") || "treeseed-market-operations-runner";
+  const base = String(baseServiceName ?? "").trim().replace(/-\d+$/u, "").replace(/-\d{2}$/u, "") || "treeseed-api-operations-runner";
   return `${base}-${String(normalizedIndex).padStart(2, "0")}`;
 }
 function deriveRailwayWorkerRunnerVolumeName(serviceName, environmentName = "") {
-  const environment = normalizeRailwayEnvironmentName(environmentName);
-  const environmentSuffix = environment === "production" ? "-prod" : environment ? `-${environment}` : "";
-  return `${serviceName}${environmentSuffix}-data`;
+  return `${serviceName}-volume`;
 }
-function deriveRailwayMarketOperationsRunnerVolumeName(serviceName, environmentName = "") {
-  const environment = normalizeRailwayEnvironmentName(environmentName);
-  const environmentSuffix = environment === "production" ? "-prod" : environment ? `-${environment}` : "";
-  const index = String(serviceName ?? "").match(/-(\d+)$/u)?.[1] ?? "01";
-  return `market-ops-runner-${index}${environmentSuffix}-data`;
+function deriveRailwayOperationsRunnerVolumeName(serviceName, environmentName = "") {
+  return `${serviceName}-volume`;
 }
 function railwayServiceRuntimeStartCommand(service) {
   return service.startCommand;
@@ -87,6 +101,10 @@ function configuredEnvValue(env, name) {
   const value = env?.[name];
   return typeof value === "string" && value.trim() ? value.trim() : "";
 }
+function railwayDeployTransport(env) {
+  const configured = configuredEnvValue(env, "TREESEED_RAILWAY_DEPLOY_TRANSPORT").toLowerCase();
+  return configured === "cli-fallback" ? "cli-fallback" : "api";
+}
 async function timedRailwayPhase(timings, name, run, metadata) {
   const startMs = performance.now();
   try {
@@ -149,6 +167,10 @@ function railwayStatusDeploymentSettled(status) {
   const normalized = String(status ?? "").trim().toUpperCase();
   return normalized === "SUCCESS" || normalized === "SLEEPING";
 }
+function railwayStatusDeploymentTerminalFailure(status) {
+  const normalized = String(status ?? "").trim().toUpperCase();
+  return ["FAILED", "CRASHED", "REMOVED"].includes(normalized);
+}
 function formatRailwayDeploymentStatusSummary(scope, checks) {
   const aliases = {
     api: "api",
@@ -219,12 +241,14 @@ function collectRailwayDeploymentStatusChecks(statusPayload, scope, services) {
     const status = String(deployment?.status ?? "").trim().toUpperCase();
     const instanceStatuses = Array.isArray(deployment?.instances) ? deployment.instances.map((entry) => String(entry?.status ?? "").trim()).filter(Boolean) : [];
     const ok = railwayStatusDeploymentSettled(status);
+    const terminalFailure = railwayStatusDeploymentTerminalFailure(status);
     return {
       type: "deployment-status",
       service: service.key,
       serviceName: service.serviceName,
       environment: normalizeRailwayEnvironmentName(environment.name),
       ok,
+      terminalFailure,
       status: status || "missing_deployment",
       observed: {
         status: status || null,
@@ -234,74 +258,10 @@ function collectRailwayDeploymentStatusChecks(statusPayload, scope, services) {
         instanceStatuses,
         volumeMounts: Array.isArray(deployment?.meta?.volumeMounts) ? deployment.meta.volumeMounts : []
       },
-      message: ok ? void 0 : `Railway deployment for ${service.serviceName} is not settled yet; observed ${status || "missing deployment status"}.`
+      message: ok ? void 0 : terminalFailure ? `Railway deployment for ${service.serviceName} failed with terminal status ${status}.` : `Railway deployment for ${service.serviceName} is not settled yet; observed ${status || "missing deployment status"}.`
     };
   });
 }
-function normalizeRailwayCliVolume(value, { serviceId, serviceName, environmentId, fallbackName, fallbackMountPath }) {
-  if (!value || typeof value !== "object") {
-    return null;
-  }
-  const record = value;
-  const id = typeof record.id === "string" && record.id.trim() ? record.id.trim() : "";
-  if (!id) {
-    return null;
-  }
-  const listedServiceName = typeof record.serviceName === "string" && record.serviceName.trim() ? record.serviceName.trim() : "";
-  if (listedServiceName && serviceName && listedServiceName !== serviceName) {
-    return null;
-  }
-  const name = typeof record.name === "string" && record.name.trim() ? record.name.trim() : fallbackName;
-  const mountPath = typeof record.mountPath === "string" && record.mountPath.trim() ? record.mountPath.trim() : fallbackMountPath;
-  const sizeMb = typeof record.sizeMB === "number" ? record.sizeMB : null;
-  const currentSizeMb = typeof record.currentSizeMB === "number" ? record.currentSizeMB : null;
-  return {
-    id,
-    name,
-    projectId: null,
-    instances: [{
-      id,
-      serviceId,
-      environmentId,
-      mountPath,
-      state: "READY",
-      sizeGb: sizeMb === null ? null : sizeMb / 1e3,
-      usedGb: currentSizeMb === null ? null : currentSizeMb / 1e3
-    }]
-  };
-}
-function normalizeRailwayCliVolumeList(value, options) {
-  if (!value || typeof value !== "object" || !Array.isArray(value.volumes)) {
-    return [];
-  }
-  return value.volumes.map((entry) => normalizeRailwayCliVolume(entry, options)).filter(Boolean);
-}
-function listRailwayServiceVolumesWithCli({
-  cwd,
-  serviceId,
-  serviceName,
-  environmentId,
-  name,
-  mountPath,
-  env = process.env
-}) {
-  const listResult = runRailway(["volume", "--service", serviceId, "--environment", environmentId, "list", "--json"], {
-    cwd,
-    capture: true,
-    allowFailure: true,
-    env
-  });
-  if ((listResult.status ?? 1) !== 0) {
-    return [];
-  }
-  return normalizeRailwayCliVolumeList(parseRailwayJsonOutput(listResult.stdout ?? ""), {
-    serviceId,
-    serviceName,
-    environmentId,
-    fallbackName: name,
-    fallbackMountPath: mountPath
-  });
-}
 function isUsableRailwayToken(value) {
   return typeof value === "string" && value.trim().length >= 8;
 }
@@ -324,6 +284,8 @@ function buildRailwayDeployCommandEnv(env = process.env) {
   const merged = buildRailwayCommandEnv(env);
   if (shouldAttachRailwayDeployLogs(merged)) {
     merged.CI = "true";
+  } else {
+    merged.CI = void 0;
   }
   return merged;
 }
@@ -489,7 +451,7 @@ mutation TreeseedScheduleUpdate($id: String!, $name: String!, $schedule: String!
 `.trim()
   };
 }
-function runRailway(args, { cwd, capture = false, allowFailure = false, input, env, retryTransient = true, retryAttempts = 3, retryDelayMs = 2e3 } = {}) {
+function runRailway(args, { cwd, capture = false, allowFailure = false, input, env, retryTransient = true, retryAttempts = 3, retryDelayMs = 2e3, timeoutMs = 12e4 } = {}) {
   const effectiveEnv = buildRailwayCommandEnv({ ...process.env, ...env ?? {} });
   const railway = resolveTreeseedToolCommand("railway", { env: effectiveEnv });
   if (!railway) {
@@ -500,7 +462,8 @@ function runRailway(args, { cwd, capture = false, allowFailure = false, input, e
     stdio: input !== void 0 ? ["pipe", capture ? "pipe" : "inherit", capture ? "pipe" : "inherit"] : capture ? "pipe" : "inherit",
     encoding: "utf8",
     env: spawnEnv,
-    input
+    input,
+    timeout: Number.isFinite(timeoutMs) && Number(timeoutMs) > 0 ? Number(timeoutMs) : void 0
   });
   let result = null;
   const maxAttempts = retryTransient && !allowFailure ? Math.max(1, Number(retryAttempts) || 1) : 1;
@@ -511,6 +474,14 @@ function runRailway(args, { cwd, capture = false, allowFailure = false, input, e
     }
     sleepSync((Number(retryDelayMs) || 0) * attempt);
   }
+  if (result?.error) {
+    const errorMessage = result.error instanceof Error ? result.error.message : String(result.error);
+    const timeoutMessage = /timed out|ETIMEDOUT/iu.test(errorMessage) ? `railway ${args.join(" ")} timed out after ${Math.round(Number(timeoutMs) / 1e3)}s` : errorMessage;
+    if (!allowFailure) {
+      throw new Error(timeoutMessage);
+    }
+    result.stderr = result.stderr || timeoutMessage;
+  }
   if (result?.status !== 0 && !allowFailure) {
     throw new Error(result.stderr?.trim() || result.stdout?.trim() || `railway ${args.join(" ")} failed`);
   }
@@ -602,6 +573,26 @@ async function waitForRailwayManagedDeploymentsSettled(tenantRoot, scope, {
         }
       };
     }
+    if (checks.some((entry) => entry.terminalFailure === true)) {
+      return {
+        ok: false,
+        checks: checks.map((check) => ({
+          ...check,
+          settle: {
+            durationMs: elapsedMs(startMs),
+            pollCount,
+            finalStatus: check.status,
+            terminalFailure: check.terminalFailure === true
+          }
+        })),
+        settle: {
+          durationMs: elapsedMs(startMs),
+          pollCount,
+          status: "failed"
+        },
+        message: "Railway deployment reached a terminal failed state."
+      };
+    }
     if (Date.now() >= deadline) {
       return {
         ok: false,
@@ -923,8 +914,7 @@ function ensureRailwayProjectContext(service, { env = process.env, allowFailure
   }
   return null;
 }
-function configuredRailwayServices(tenantRoot, scope) {
-  const deployConfig = loadCliDeployConfig(tenantRoot);
+function configuredRailwayServicesForConfig(tenantRoot, scope, deployConfig, application = null) {
   const normalizedScope = normalizeScope(scope);
   let identity;
   try {
@@ -944,7 +934,7 @@ function configuredRailwayServices(tenantRoot, scope) {
     if (!service || service.enabled === false || (service.provider ?? "railway") !== "railway") {
       return [];
     }
-    const defaultRootDir = ["api", "marketOperationsRunner"].includes(serviceKey) ? "." : "packages/core";
+    const defaultRootDir = ["api", "operationsRunner"].includes(serviceKey) ? "." : "packages/core";
     const serviceRoot = resolve(tenantRoot, service.railway?.rootDir ?? service.rootDir ?? defaultRootDir);
     const railwayEnvironment = resolveRailwayEnvironmentForScope(
       normalizedScope,
@@ -953,28 +943,29 @@ function configuredRailwayServices(tenantRoot, scope) {
     const publicBaseUrl = service.environments?.[normalizedScope]?.baseUrl ?? service.publicBaseUrl ?? null;
     const configuredServiceName = service.railway?.serviceName ?? (serviceKey === "workerRunner" ? deriveRailwayWorkerRunnerServiceName(identity.deploymentKey) : `${identity.deploymentKey}-${railwayServiceNameSuffix(serviceKey)}`);
     const configuredRunnerPool = service.railway?.runnerPool && typeof service.railway.runnerPool === "object" ? service.railway.runnerPool : null;
-    const runnerPool = serviceKey === "marketOperationsRunner" ? {
-      bootstrapCount: Math.max(1, Number.parseInt(String(configuredRunnerPool?.bootstrapCount ?? MARKET_OPERATIONS_RUNNER_BOOTSTRAP_COUNT), 10) || MARKET_OPERATIONS_RUNNER_BOOTSTRAP_COUNT),
-      maxRunners: Math.max(1, Number.parseInt(String(configuredRunnerPool?.maxRunners ?? configuredRunnerPool?.bootstrapCount ?? MARKET_OPERATIONS_RUNNER_BOOTSTRAP_COUNT), 10) || MARKET_OPERATIONS_RUNNER_BOOTSTRAP_COUNT),
+    const runnerPool = serviceKey === "operationsRunner" ? {
+      bootstrapCount: Math.max(1, Number.parseInt(String(configuredRunnerPool?.bootstrapCount ?? OPERATIONS_RUNNER_BOOTSTRAP_COUNT), 10) || OPERATIONS_RUNNER_BOOTSTRAP_COUNT),
+      maxRunners: Math.max(1, Number.parseInt(String(configuredRunnerPool?.maxRunners ?? configuredRunnerPool?.bootstrapCount ?? OPERATIONS_RUNNER_BOOTSTRAP_COUNT), 10) || OPERATIONS_RUNNER_BOOTSTRAP_COUNT),
       volumeMountPath: service.railway?.volumeMountPath ?? configuredRunnerPool?.volumeMountPath ?? WORKER_RUNNER_VOLUME_MOUNT_PATH
     } : serviceKey === "workerRunner" ? {
       bootstrapIndex: WORKER_RUNNER_BOOTSTRAP_INDEX,
       volumeMountPath: WORKER_RUNNER_VOLUME_MOUNT_PATH
     } : null;
-    const instanceCount = serviceKey === "marketOperationsRunner" ? runnerPool.bootstrapCount : 1;
+    const instanceCount = serviceKey === "operationsRunner" ? runnerPool.bootstrapCount : 1;
     return Array.from({ length: instanceCount }, (_, offset) => {
       const runnerIndex = offset + 1;
-      const serviceName = serviceKey === "marketOperationsRunner" ? deriveRailwayMarketOperationsRunnerServiceName(configuredServiceName, runnerIndex) : configuredServiceName;
+      const serviceName = serviceKey === "operationsRunner" ? deriveRailwayOperationsRunnerServiceName(configuredServiceName, runnerIndex) : configuredServiceName;
       return {
         key: serviceKey,
-        instanceKey: serviceKey === "marketOperationsRunner" ? `${serviceKey}:${runnerIndex}` : serviceKey,
-        runnerIndex: serviceKey === "marketOperationsRunner" ? runnerIndex : null,
+        instanceKey: serviceKey === "operationsRunner" ? `${serviceKey}:${runnerIndex}` : serviceKey,
+        runnerIndex: serviceKey === "operationsRunner" ? runnerIndex : null,
+        serviceConfig: service,
         scope: normalizedScope,
         projectId: service.railway?.projectId ?? null,
         projectName: service.railway?.projectName ?? identity.deploymentKey,
         serviceId: service.railway?.serviceId ?? null,
         serviceName,
-        runnerId: serviceKey === "marketOperationsRunner" ? serviceName : null,
+        runnerId: serviceKey === "operationsRunner" ? serviceName : null,
         rootDir: serviceRoot,
         publicBaseUrl,
         railwayEnvironment,
@@ -985,14 +976,30 @@ function configuredRailwayServices(tenantRoot, scope) {
         healthcheckIntervalSeconds: service.railway?.healthcheckIntervalSeconds ?? null,
         restartPolicy: service.railway?.restartPolicy ?? null,
         runtimeMode: service.railway?.runtimeMode ?? null,
-        volumeMountPath: serviceKey === "marketOperationsRunner" ? runnerPool.volumeMountPath : service.railway?.volumeMountPath ?? null,
+        volumeMountPath: serviceKey === "operationsRunner" ? runnerPool.volumeMountPath : service.railway?.volumeMountPath ?? null,
         schedule: normalizeScheduleExpressions(service.railway?.schedule),
         hostingKind,
-        runnerPool
+        runnerPool,
+        application
       };
     });
   }).filter(Boolean);
 }
+function configuredRailwayServices(tenantRoot, scope) {
+  const deployConfig = loadCliDeployConfig(tenantRoot);
+  const direct = configuredRailwayServicesForConfig(tenantRoot, scope, deployConfig);
+  const nested = discoverTreeseedApplications(tenantRoot).filter((application) => application.root !== resolve(tenantRoot)).flatMap((application) => configuredRailwayServicesForConfig(
+    application.root,
+    scope,
+    application.config,
+    {
+      id: application.id,
+      root: application.root,
+      relativeRoot: application.relativeRoot
+    }
+  ));
+  return [...direct, ...nested];
+}
 function configuredRailwayScheduledJobs(tenantRoot, scope, { phase = "deploy" } = {}) {
   if (!shouldManageRailwaySchedules(scope, phase)) {
     return [];
@@ -1427,6 +1434,39 @@ function shouldUseVerboseRailwayDeploy(env = process.env) {
   }
   return shouldAttachRailwayDeployLogs(env);
 }
+function railwayDeployCommandTimeoutMs(env = process.env) {
+  const configured = Number.parseInt(configuredEnvValue(env, "TREESEED_RAILWAY_DEPLOY_COMMAND_TIMEOUT_MS"), 10);
+  return Number.isFinite(configured) && configured > 0 ? configured : 3e5;
+}
+function railwayPhaseTimeoutMs(env = process.env, phase = "default") {
+  const configured = Number.parseInt(configuredEnvValue(env, `TREESEED_RAILWAY_${String(phase).toUpperCase().replace(/[^A-Z0-9]+/gu, "_")}_TIMEOUT_MS`), 10);
+  if (Number.isFinite(configured) && configured > 0) {
+    return configured;
+  }
+  const defaultConfigured = Number.parseInt(configuredEnvValue(env, "TREESEED_RAILWAY_PHASE_TIMEOUT_MS"), 10);
+  if (Number.isFinite(defaultConfigured) && defaultConfigured > 0) {
+    return defaultConfigured;
+  }
+  if (phase === "sync_runtime_config") {
+    return 6e5;
+  }
+  return phase === "deploy" ? 3e5 : 18e4;
+}
+async function withRailwayPhaseTimeout(run, timeoutMs, message) {
+  let timer = null;
+  try {
+    return await Promise.race([
+      Promise.resolve().then(run),
+      new Promise((_, reject) => {
+        timer = setTimeout(() => reject(new Error(message)), timeoutMs);
+      })
+    ]);
+  } finally {
+    if (timer) {
+      clearTimeout(timer);
+    }
+  }
+}
 function shouldIncludeRailwayIgnoredFiles(env = process.env) {
   const configured = configuredEnvValue(env, "TREESEED_RAILWAY_DEPLOY_INCLUDE_IGNORED");
   return configured === "1" || configured === "true";
@@ -1699,15 +1739,23 @@ async function resolveRailwayDeployProjectContext(service, { env = process.env }
     projectName: project.name ?? service.projectName
   };
 }
-async function syncRailwayServiceRuntimeConfigurationAfterDeploy(tenantRoot, service, { env = process.env } = {}) {
+async function syncRailwayServiceRuntimeConfigurationAfterDeploy(tenantRoot, service, { env = process.env, writePhase = null } = {}) {
+  const writeSyncPhase = (stage, message) => {
+    if (typeof writePhase === "function") {
+      writePhase(`sync-runtime-config:${stage}`, message);
+    }
+  };
   const wantsInstanceConfig = service.buildCommand || service.startCommand || service.rootDir || service.healthcheckPath || service.healthcheckTimeoutSeconds !== null || service.healthcheckTimeoutSeconds !== void 0 || service.healthcheckIntervalSeconds !== null || service.healthcheckIntervalSeconds !== void 0 || service.restartPolicy || service.runtimeMode;
   const wantsRunnerVolume = service.key === "workerRunner" || Boolean(service.volumeMountPath);
   if (!wantsInstanceConfig && !wantsRunnerVolume) {
+    writeSyncPhase("skip", "No runtime configuration changes requested.");
     return null;
   }
+  writeSyncPhase("workspace", "Resolving Railway workspace.");
   const workspace = await resolveRailwayWorkspaceContext({ env });
   let project = null;
   if (service.projectId) {
+    writeSyncPhase("project", `Resolving Railway project ${service.projectName ?? service.projectId}.`);
     project = await ensureRailwayProject({
       projectId: service.projectId,
       projectName: service.projectName,
@@ -1716,9 +1764,11 @@ async function syncRailwayServiceRuntimeConfigurationAfterDeploy(tenantRoot, ser
       workspace: workspace.id
     }).then((result) => result.project);
   } else {
+    writeSyncPhase("project", `Looking up Railway project ${service.projectName}.`);
     const projects = await listRailwayProjects({ env, workspaceId: workspace.id });
     project = projects.find((entry) => entry.name === service.projectName) ?? null;
     if (!project) {
+      writeSyncPhase("project", `Creating Railway project ${service.projectName}.`);
       project = await ensureRailwayProject({
         projectName: service.projectName,
         defaultEnvironmentName: service.railwayEnvironment,
@@ -1730,6 +1780,7 @@ async function syncRailwayServiceRuntimeConfigurationAfterDeploy(tenantRoot, ser
   const environmentName = normalizeRailwayEnvironmentName(service.railwayEnvironment);
   let environment = project.environments.find((entry) => entry.name === environmentName || entry.id === environmentName) ?? null;
   if (!environment) {
+    writeSyncPhase("environment", `Creating Railway environment ${environmentName}.`);
     environment = await ensureRailwayEnvironment({
       projectId: project.id,
       environmentName,
@@ -1738,6 +1789,7 @@ async function syncRailwayServiceRuntimeConfigurationAfterDeploy(tenantRoot, ser
   }
   let railwayService = project.services.find((entry) => entry.id === service.serviceId || entry.name === service.serviceName) ?? null;
   if (!railwayService) {
+    writeSyncPhase("service", `Creating Railway service ${service.serviceName ?? service.key}.`);
     railwayService = await ensureRailwayService({
       projectId: project.id,
       serviceId: service.serviceId,
@@ -1745,56 +1797,63 @@ async function syncRailwayServiceRuntimeConfigurationAfterDeploy(tenantRoot, ser
       env
     }).then((result) => result.service);
   }
+  if (wantsInstanceConfig) {
+    writeSyncPhase("instance", "Ensuring Railway service instance configuration.");
+  }
   const runtimeConfiguration = wantsInstanceConfig ? await ensureRailwayServiceInstanceConfiguration({
     serviceId: railwayService.id,
     environmentId: environment.id,
     buildCommand: service.buildCommand,
     startCommand: railwayServiceRuntimeStartCommand(service),
     cronSchedule: service.schedule?.[0] ?? null,
-    rootDirectory: relativeRailwayRootDir(tenantRoot, service.rootDir),
+    rootDirectory: ".",
     healthcheckPath: service.healthcheckPath,
     healthcheckTimeoutSeconds: service.healthcheckTimeoutSeconds,
     healthcheckIntervalSeconds: service.healthcheckIntervalSeconds,
     restartPolicy: service.restartPolicy,
     runtimeMode: service.runtimeMode,
+    deploymentRegion: wantsRunnerVolume ? configuredEnvValue(env, "TREESEED_RAILWAY_STATEFUL_REGION") || "us-west2" : null,
     env
   }) : null;
+  writeSyncPhase("variables", "Upserting Railway runtime variables.");
   await upsertRailwayVariables({
     projectId: project.id,
     environmentId: environment.id,
     serviceId: railwayService.id,
     variables: {
       TREESEED_SKIP_PACKAGE_PREPARE: "1",
-      ...service.key === "marketOperationsRunner" ? {
+      ...service.key === "operationsRunner" ? {
         NIXPACKS_APT_PKGS: "git",
         NIXPACKS_PKGS: "git",
         TREESEED_PLATFORM_RUNNER_ID: service.runnerId ?? railwayService.name,
         TREESEED_PLATFORM_RUNNER_DATA_DIR: service.volumeMountPath ?? WORKER_RUNNER_VOLUME_MOUNT_PATH,
         TREESEED_PLATFORM_RUNNER_ENVIRONMENT: normalizeScope(service.scope) === "prod" ? "production" : normalizeScope(service.scope),
-        TREESEED_MARKET_ID: normalizeScope(service.scope),
+        TREESEED_MANAGER_ID: normalizeScope(service.scope),
+        ...configuredEnvValue(env, "RAILWAY_API_TOKEN") ? { RAILWAY_API_TOKEN: configuredEnvValue(env, "RAILWAY_API_TOKEN") } : {},
+        ...configuredEnvValue(env, "TREESEED_RAILWAY_WORKSPACE") ? { TREESEED_RAILWAY_WORKSPACE: configuredEnvValue(env, "TREESEED_RAILWAY_WORKSPACE") } : {},
         ...configuredEnvValue(env, "TREESEED_PLATFORM_RUNNER_SECRET") ? { TREESEED_PLATFORM_RUNNER_SECRET: configuredEnvValue(env, "TREESEED_PLATFORM_RUNNER_SECRET") } : {},
-        ...configuredEnvValue(env, "TREESEED_MARKET_API_BASE_URL") || configuredEnvValue(env, "TREESEED_MARKET_URL") ? {
-          TREESEED_MARKET_API_BASE_URL: configuredEnvValue(env, "TREESEED_MARKET_API_BASE_URL") || configuredEnvValue(env, "TREESEED_MARKET_URL")
+        ...configuredEnvValue(env, "TREESEED_API_BASE_URL") || configuredEnvValue(env, "TREESEED_URL") ? {
+          TREESEED_API_BASE_URL: configuredEnvValue(env, "TREESEED_API_BASE_URL") || configuredEnvValue(env, "TREESEED_URL")
         } : {}
       } : {}
     },
     env
   });
   const volumeMountPath = service.volumeMountPath ?? service.runnerPool?.volumeMountPath ?? WORKER_RUNNER_VOLUME_MOUNT_PATH;
-  const volumeConfiguration = wantsRunnerVolume ? await ensureRailwayServiceVolumeWithCliFallback({
-    tenantRoot,
+  if (wantsRunnerVolume) {
+    writeSyncPhase("volume", `Ensuring Railway volume mounted at ${volumeMountPath}.`);
+  }
+  const volumeConfiguration = wantsRunnerVolume ? await ensureRailwayServiceVolume({
     projectId: project.id,
     environmentId: environment.id,
-    environmentName: environment.name,
     serviceId: railwayService.id,
-    serviceName: railwayService.name,
-    name: service.key === "marketOperationsRunner" ? deriveRailwayMarketOperationsRunnerVolumeName(railwayService.name, environment.name) : deriveRailwayWorkerRunnerVolumeName(railwayService.name, environment.name),
+    name: service.key === "operationsRunner" ? deriveRailwayOperationsRunnerVolumeName(railwayService.name, environment.name) : deriveRailwayWorkerRunnerVolumeName(railwayService.name, environment.name),
     mountPath: volumeMountPath,
-    preferCli: service.key === "marketOperationsRunner",
     env
   }) : null;
   if (wantsRunnerVolume) {
     if (service.key === "workerRunner") {
+      writeSyncPhase("volume-vars", "Upserting Railway worker volume variables.");
       await upsertRailwayVariables({
         projectId: project.id,
         environmentId: environment.id,
@@ -1810,6 +1869,7 @@ async function syncRailwayServiceRuntimeConfigurationAfterDeploy(tenantRoot, ser
       });
     }
   }
+  writeSyncPhase("done", "Runtime configuration is synchronized.");
   return {
     projectId: project.id,
     projectName: project.name ?? service.projectName ?? null,
@@ -1828,163 +1888,6 @@ async function syncRailwayServiceRuntimeConfigurationAfterDeploy(tenantRoot, ser
     } : null
   };
 }
-async function ensureRailwayServiceVolumeWithCliFallback({
-  tenantRoot,
-  projectId,
-  environmentId,
-  environmentName,
-  serviceId,
-  serviceName,
-  name,
-  mountPath,
-  preferCli = false,
-  env = process.env
-}) {
-  if (!preferCli) {
-    try {
-      return await ensureRailwayServiceVolume({
-        projectId,
-        environmentId,
-        serviceId,
-        name,
-        mountPath,
-        env
-      });
-    } catch (error) {
-      const message = error instanceof Error ? error.message : String(error);
-      if (!message.includes("Problem processing request")) {
-        throw error;
-      }
-    }
-  }
-  const cliOptions = {
-    cwd: tenantRoot,
-    capture: true,
-    env
-  };
-  ensureRailwayProjectContext({
-    key: serviceName,
-    projectId,
-    serviceName,
-    rootDir: tenantRoot,
-    railwayEnvironment: environmentName
-  }, {
-    env,
-    capture: true
-  });
-  const volumeArgs = ["volume", "--service", serviceId, "--environment", environmentId];
-  const listResult = runRailway([...volumeArgs, "list", "--json"], cliOptions);
-  const existingVolumes = normalizeRailwayCliVolumeList(parseRailwayJsonOutput(listResult.stdout ?? ""), {
-    serviceId,
-    serviceName,
-    environmentId,
-    fallbackName: name,
-    fallbackMountPath: mountPath
-  });
-  let volume = existingVolumes.find((entry) => entry.name === name) ?? existingVolumes.find((entry) => entry.instances.some((instance2) => instance2.mountPath === mountPath)) ?? existingVolumes[0] ?? null;
-  let created = false;
-  let updated = false;
-  if (!volume) {
-    const createResult = runRailway([...volumeArgs, "add", "--mount-path", mountPath, "--json"], cliOptions);
-    volume = normalizeRailwayCliVolume(parseRailwayJsonOutput(createResult.stdout ?? ""), {
-      serviceId,
-      serviceName,
-      environmentId,
-      fallbackName: name,
-      fallbackMountPath: mountPath
-    });
-    if (!volume) {
-      throw new Error(`Railway CLI volume add did not return a usable volume for ${serviceName} in ${environmentName}.`);
-    }
-    created = true;
-  }
-  let instance = volume.instances.find((entry) => entry.serviceId === serviceId && entry.environmentId === environmentId) ?? volume.instances[0] ?? null;
-  if (!instance || instance.mountPath !== mountPath) {
-    const attachResult = runRailway([...volumeArgs, "attach", "--volume", volume.id, "--yes", "--json"], {
-      ...cliOptions,
-      allowFailure: true
-    });
-    if ((attachResult.status ?? 1) !== 0) {
-      const attachMessage = attachResult.stderr?.trim() || attachResult.stdout?.trim() || "";
-      if (!/already mounted/iu.test(attachMessage)) {
-        throw new Error(attachMessage || `Railway volume attach failed for ${serviceName} in ${environmentName}.`);
-      }
-    }
-    const attachedVolume = (attachResult.status ?? 1) === 0 ? normalizeRailwayCliVolume(parseRailwayJsonOutput(attachResult.stdout ?? ""), {
-      serviceId,
-      serviceName,
-      environmentId,
-      fallbackName: name,
-      fallbackMountPath: mountPath
-    }) : null;
-    volume = attachedVolume ?? {
-      ...volume,
-      instances: [{
-        ...instance ?? {
-          id: volume.id,
-          serviceId,
-          environmentId,
-          state: "READY",
-          sizeGb: null,
-          usedGb: null
-        },
-        serviceId,
-        environmentId,
-        mountPath
-      }]
-    };
-    instance = volume.instances.find((entry) => entry.serviceId === serviceId && entry.environmentId === environmentId) ?? volume.instances[0] ?? null;
-    updated = true;
-  }
-  const apiVolume = await waitForRailwayServiceVolumeMount({
-    projectId,
-    volumeId: volume.id,
-    volumeName: name,
-    serviceId,
-    environmentId,
-    mountPath,
-    env
-  });
-  if (apiVolume) {
-    volume = apiVolume;
-  } else {
-    throw new Error(`Railway volume ${name} was not attached to ${serviceName} at ${mountPath}.`);
-  }
-  return {
-    volume,
-    instance: volume.instances.find((entry) => entry.serviceId === serviceId && entry.environmentId === environmentId) ?? volume.instances[0] ?? null,
-    created,
-    updated
-  };
-}
-async function waitForRailwayServiceVolumeMount({
-  projectId,
-  volumeId,
-  volumeName,
-  serviceId,
-  environmentId,
-  mountPath,
-  env
-}) {
-  for (let attempt = 0; attempt <= 24; attempt += 1) {
-    const volumes = await listRailwayVolumes({ projectId, env });
-    const mounted = volumes.find(
-      (entry) => entry.instances.some(
-        (instance) => instance.serviceId === serviceId && instance.environmentId === environmentId && instance.mountPath === mountPath
-      )
-    ) ?? null;
-    const match = mounted ?? volumes.find((entry) => entry.id === volumeId) ?? volumes.find((entry) => entry.name === volumeName) ?? null;
-    if (match?.instances.some(
-      (instance) => instance.serviceId === serviceId && instance.environmentId === environmentId && instance.mountPath === mountPath
-    )) {
-      return match;
-    }
-    if (attempt < 24) {
-      await sleep(5e3);
-    }
-  }
-  return null;
-}
 async function deployRailwayService(tenantRoot, service, {
   dryRun = false,
   write,
@@ -2000,7 +1903,13 @@ async function deployRailwayService(tenantRoot, service, {
       command: [plan2.command, ...plan2.args].join(" "),
       cwd: plan2.cwd,
       publicBaseUrl: service.publicBaseUrl,
-      timings
+      timings,
+      transport: {
+        railway: {
+          reconcile: "api",
+          deploy: railwayDeployTransport(env)
+        }
+      }
     };
   }
   const deployService = await timedRailwayPhase(timings, "railway:resolve-context", () => resolveRailwayDeployProjectContext(service, { env }), {
@@ -2008,19 +1917,23 @@ async function deployRailwayService(tenantRoot, service, {
   });
   const commandEnv = buildRailwayCommandEnv({ ...process.env, ...env });
   let railwayDeployEnv = buildRailwayDeployCommandEnv(commandEnv);
-  const railway = resolveTreeseedToolCommand("railway", { env: commandEnv });
-  if (!railway) {
-    throw new Error("Railway CLI is unavailable.");
-  }
+  const deployTransport = railwayDeployTransport(commandEnv);
   const taskPrefix = prefix ?? {
     scope: normalizeScope(deployService.scope ?? deployService.railwayEnvironment ?? "railway"),
     system: deployService.key === "api" ? "api" : "agents",
     task: `${deployService.key}-railway-deploy`,
     stage: "deploy"
   };
-  const runtimeConfiguration = await timedRailwayPhase(timings, "railway:sync-runtime-config", () => syncRailwayServiceRuntimeConfigurationAfterDeploy(tenantRoot, deployService, {
-    env: commandEnv
-  }), { service: deployService.key });
+  const writePhase = (stage, message) => {
+    write ? write(`[${taskPrefix.scope}][${taskPrefix.system}][${taskPrefix.task}][${stage}] ${message}`, "stdout") : null;
+  };
+  writePhase("resolve-context", `Resolved Railway service ${deployService.serviceName ?? deployService.serviceId ?? deployService.key}.`);
+  writePhase("sync-runtime-config", "Syncing Railway runtime configuration.");
+  const runtimeConfiguration = await timedRailwayPhase(timings, "railway:sync-runtime-config", () => withRailwayPhaseTimeout(
+    () => syncRailwayServiceRuntimeConfigurationAfterDeploy(tenantRoot, deployService, { env: commandEnv, writePhase }),
+    railwayPhaseTimeoutMs(commandEnv, "sync_runtime_config"),
+    `Railway runtime configuration sync timed out for ${deployService.serviceName ?? deployService.key}.`
+  ), { service: deployService.key });
   const cliDeployService = {
     ...deployService,
     projectId: runtimeConfiguration?.projectId ?? deployService.projectId,
@@ -2030,16 +1943,71 @@ async function deployRailwayService(tenantRoot, service, {
     serviceName: runtimeConfiguration?.serviceName ?? deployService.serviceName,
     railwayEnvironment: runtimeConfiguration?.environmentName ?? runtimeConfiguration?.environmentId ?? deployService.railwayEnvironment
   };
-  await timedRailwayPhase(timings, "railway:device-login-vars", () => syncRailwayApiDeviceLoginVariables(cliDeployService, commandEnv, write, taskPrefix), {
+  writePhase("device-login-vars", "Syncing Railway device-login variables.");
+  await timedRailwayPhase(timings, "railway:device-login-vars", () => withRailwayPhaseTimeout(
+    () => syncRailwayApiDeviceLoginVariables(cliDeployService, commandEnv, write, taskPrefix),
+    railwayPhaseTimeoutMs(commandEnv, "device_login_vars"),
+    `Railway device-login variable sync timed out for ${cliDeployService.serviceName ?? cliDeployService.key}.`
+  ), {
     service: cliDeployService.key
   });
+  if (deployService.buildCommand && shouldRunRailwayPredeployBuild(commandEnv)) {
+    const buildResult = await timedRailwayPhase(timings, "railway:predeploy-build", () => runPrefixedCommand("bash", ["-lc", deployService.buildCommand], {
+      cwd: deployService.rootDir,
+      env: commandEnv,
+      write,
+      prefix: { ...taskPrefix, stage: "build" }
+    }), { service: deployService.key });
+    if (buildResult.status !== 0) {
+      throw new Error(`Railway ${deployService.key} build command failed.`);
+    }
+  }
+  if (deployTransport !== "cli-fallback") {
+    writePhase("deploy", `Deploying Railway service ${cliDeployService.serviceName ?? cliDeployService.serviceId ?? cliDeployService.key} through the Railway API.`);
+    const apiDeploy = await timedRailwayPhase(timings, "railway:api-deploy", () => withRailwayPhaseTimeout(
+      () => deployRailwayServiceInstance({
+        serviceId: cliDeployService.serviceId,
+        environmentId: cliDeployService.environmentId,
+        env: commandEnv
+      }),
+      railwayPhaseTimeoutMs(commandEnv, "deploy"),
+      `Railway API deploy phase timed out for ${cliDeployService.serviceName ?? cliDeployService.key}.`
+    ), { service: cliDeployService.key });
+    return {
+      service: deployService.key,
+      status: "deployed",
+      command: "railway-api serviceInstanceDeployV2",
+      cwd: deployService.rootDir,
+      publicBaseUrl: deployService.publicBaseUrl,
+      timings,
+      deploymentId: apiDeploy.deploymentId,
+      transport: {
+        railway: {
+          reconcile: "api",
+          deploy: "api"
+        }
+      },
+      runtimeConfiguration: runtimeConfiguration ? {
+        updated: runtimeConfiguration.updated,
+        healthcheckPath: runtimeConfiguration.instance?.healthcheckPath ?? null,
+        healthcheckTimeoutSeconds: runtimeConfiguration.instance?.healthcheckTimeoutSeconds ?? null,
+        runtimeMode: runtimeConfiguration.instance?.runtimeMode ?? null,
+        volume: runtimeConfiguration.volume ?? null
+      } : null
+    };
+  }
+  const railway = resolveTreeseedToolCommand("railway", { env: commandEnv });
+  if (!railway) {
+    throw new Error("Railway CLI deploy fallback requested, but Railway CLI is unavailable.");
+  }
   railwayDeployEnv = buildRailwayCliContextEnv(railwayDeployEnv, cliDeployService);
   const hasCommandApiToken = Boolean(configuredEnvValue(commandEnv, "RAILWAY_API_TOKEN"));
   let usesProjectToken = Boolean(configuredEnvValue(railwayDeployEnv, "RAILWAY_TOKEN"));
   if (usesProjectToken) {
     railwayDeployEnv = { ...railwayDeployEnv, RAILWAY_API_TOKEN: void 0 };
   }
-  await timedRailwayPhase(timings, "railway:project-token", async () => {
+  writePhase("project-token", usesProjectToken || hasCommandApiToken ? "Using configured Railway authentication." : "Creating Railway project token.");
+  await timedRailwayPhase(timings, "railway:project-token", () => withRailwayPhaseTimeout(async () => {
     if (usesProjectToken || hasCommandApiToken) {
       return null;
     }
@@ -2051,25 +2019,15 @@ async function deployRailwayService(tenantRoot, service, {
       throw new Error(`Railway CI deploy requires a project token for ${cliDeployService.serviceName ?? cliDeployService.key}. Automatic project token creation did not return a token.`);
     }
     return null;
-  }, { service: cliDeployService.key });
+  }, railwayPhaseTimeoutMs(commandEnv, "project_token"), `Railway project-token phase timed out for ${cliDeployService.serviceName ?? cliDeployService.key}.`), { service: cliDeployService.key });
   const linkPlan = planRailwayServiceLink(cliDeployService, { env: commandEnv });
   const plan = planRailwayServiceDeploy(cliDeployService, { env, projectTokenMode: usesProjectToken });
-  if (deployService.buildCommand && shouldRunRailwayPredeployBuild(commandEnv)) {
-    const buildResult = await timedRailwayPhase(timings, "railway:predeploy-build", () => runPrefixedCommand("bash", ["-lc", deployService.buildCommand], {
-      cwd: deployService.rootDir,
-      env: commandEnv,
-      write,
-      prefix: { ...taskPrefix, stage: "build" }
-    }), { service: deployService.key });
-    if (buildResult.status !== 0) {
-      throw new Error(`Railway ${deployService.key} build command failed.`);
-    }
-  }
   const hasRailwayApiToken = Boolean(configuredEnvValue(commandEnv, "RAILWAY_API_TOKEN"));
   const cliConfig = configuredEnvValue(commandEnv, "CI") === "true" ? writeRailwayCliProjectConfig(cliDeployService, { env: railwayDeployEnv, cwd: plan.cwd }) : null;
   const effectiveLinkPlan = hasRailwayApiToken ? linkPlan : usesProjectToken ? planRailwayProjectEnvironmentLink(cliDeployService) : linkPlan;
   const railwayLinkEnv = hasRailwayApiToken ? buildRailwayLinkCommandEnv(commandEnv, cliDeployService) : railwayDeployEnv;
-  await timedRailwayPhase(timings, "railway:link", async () => {
+  writePhase("link", `Linking Railway project context for ${cliDeployService.serviceName ?? cliDeployService.serviceId ?? cliDeployService.key}.`);
+  await timedRailwayPhase(timings, "railway:link", () => withRailwayPhaseTimeout(async () => {
     if (cliConfig) {
       write ? write(`[${taskPrefix.scope}][${taskPrefix.system}][${taskPrefix.task}][link] Wrote Railway CLI project context for ${cliConfig.projectPath}.`, "stdout") : null;
       return;
@@ -2083,15 +2041,19 @@ async function deployRailwayService(tenantRoot, service, {
     if (linkResult.status !== 0) {
       throw new Error(linkResult.stderr?.trim() || linkResult.stdout?.trim() || `railway ${effectiveLinkPlan.args.join(" ")} failed with exit code ${linkResult.status ?? "unknown"} in ${effectiveLinkPlan.cwd}`);
     }
-  }, { service: cliDeployService.key });
-  await timedRailwayPhase(timings, "railway:deploy", async () => {
+  }, railwayPhaseTimeoutMs(commandEnv, "link"), `Railway link phase timed out for ${cliDeployService.serviceName ?? cliDeployService.key}.`), { service: cliDeployService.key });
+  await timedRailwayPhase(timings, "railway:deploy", () => withRailwayPhaseTimeout(async () => {
     let lastFailure = null;
     for (let attempt = 1; attempt <= 5; attempt += 1) {
+      if (write && attempt === 1) {
+        write(`[${taskPrefix.scope}][${taskPrefix.system}][${taskPrefix.task}][deploy] $ railway ${plan.args.join(" ")}`, "stdout");
+      }
       const result = await runPrefixedCommand(railway.command, [...railway.argsPrefix, ...plan.args], {
         cwd: plan.cwd,
         env: railwayDeployEnv,
         write,
-        prefix: taskPrefix
+        prefix: taskPrefix,
+        timeoutMs: railwayDeployCommandTimeoutMs(commandEnv)
       });
       if (result.status === 0) {
         lastFailure = null;
@@ -2109,7 +2071,7 @@ async function deployRailwayService(tenantRoot, service, {
     if (lastFailure) {
       throw new Error(lastFailure.stderr?.trim() || lastFailure.stdout?.trim() || `railway ${plan.args.join(" ")} failed`);
     }
-  }, { service: cliDeployService.key });
+  }, railwayPhaseTimeoutMs(commandEnv, "deploy"), `Railway deploy phase timed out for ${cliDeployService.serviceName ?? cliDeployService.key}.`), { service: cliDeployService.key });
   return {
     service: deployService.key,
     status: "deployed",
@@ -2117,6 +2079,12 @@ async function deployRailwayService(tenantRoot, service, {
     cwd: plan.cwd,
     publicBaseUrl: deployService.publicBaseUrl,
     timings,
+    transport: {
+      railway: {
+        reconcile: "api",
+        deploy: "cli-fallback"
+      }
+    },
     runtimeConfiguration: runtimeConfiguration ? {
       updated: runtimeConfiguration.updated,
       healthcheckPath: runtimeConfiguration.instance?.healthcheckPath ?? null,
@@ -2134,8 +2102,8 @@ export {
   configuredRailwayScheduledJobs,
   configuredRailwayServices,
   deployRailwayService,
-  deriveRailwayMarketOperationsRunnerServiceName,
-  deriveRailwayMarketOperationsRunnerVolumeName,
+  deriveRailwayOperationsRunnerServiceName,
+  deriveRailwayOperationsRunnerVolumeName,
   deriveRailwayWorkerRunnerServiceName,
   deriveRailwayWorkerRunnerVolumeName,
   ensureRailwayDatabaseServiceExists,
@@ -2144,10 +2112,10 @@ export {
   ensureRailwayProjectExists,
   ensureRailwayScheduledJobs,
   ensureRailwayServiceExists,
-  ensureRailwayServiceVolumeWithCliFallback,
+  findStaleTreeseedOperationsRunnerResources,
   isRailwayTransientFailure,
+  isTreeseedOperationsRunnerResourceName,
   isUsableRailwayToken,
-  listRailwayServiceVolumesWithCli,
   parseRailwayJsonOutput,
   planRailwayServiceDeploy,
   planRailwayServiceLink,