@treeseed/sdk 0.11.0 → 0.12.4

package/dist/operations/services/railway-api.js

@@ -1,4 +1,5 @@
 import { request as httpsRequest } from "node:https";
+import { resolveTreeseedRailwayApiToken } from "../../service-credentials.js";
 const DEFAULT_RAILWAY_API_URL = "https://backboard.railway.com/graphql/v2";
 const DEFAULT_RAILWAY_WORKSPACE = "knowledge-coop";
 function normalizeRailwayEnvironmentName(value) {
@@ -16,7 +17,7 @@ function isUsableRailwayToken(value) {
   return typeof value === "string" && value.trim().length >= 8;
 }
 function resolveRailwayApiToken(env = process.env) {
-  const token = configuredEnvValue(env, "RAILWAY_API_TOKEN");
+  const token = resolveTreeseedRailwayApiToken(env);
   return isUsableRailwayToken(token) ? token : "";
 }
 function resolveRailwayApiUrl(env = process.env) {
@@ -111,15 +112,36 @@ function normalizeProject(node) {
   if (!id || !name) {
     return null;
   }
+  const services = /* @__PURE__ */ new Map();
+  for (const service of normalizeConnectionNodes(node.services, normalizeService)) {
+    services.set(service.id, service);
+  }
+  for (const environment of normalizeConnectionNodes(node.environments, (entry) => entry)) {
+    for (const instance of normalizeConnectionNodes(environment.serviceInstances, (entry) => entry)) {
+      const serviceId = railwayConnectionLabel(instance.serviceId);
+      const serviceName = railwayConnectionLabel(instance.serviceName);
+      if (serviceId && serviceName) {
+        services.set(serviceId, { id: serviceId, name: serviceName });
+      }
+    }
+  }
   return {
     id,
     name,
     workspaceId: railwayConnectionLabel(node.workspaceId) || null,
     deletedAt: railwayConnectionLabel(node.deletedAt) || null,
     environments: normalizeConnectionNodes(node.environments, normalizeEnvironment),
-    services: normalizeConnectionNodes(node.services, normalizeService)
+    services: [...services.values()]
   };
 }
+function normalizeServiceInstanceService(node) {
+  const id = railwayConnectionLabel(node.serviceId);
+  const name = railwayConnectionLabel(node.serviceName);
+  if (!id || !name) {
+    return null;
+  }
+  return { id, name };
+}
 function normalizeVariableMap(value) {
   if (!value) {
     return {};
@@ -231,13 +253,15 @@ function normalizeRailwayVolumeInstance(node) {
     environmentId: railwayConnectionLabel(node.environmentId) || railwayConnectionLabel(node.environment?.id) || null,
     mountPath: railwayConnectionLabel(node.mountPath) || railwayConnectionLabel(node.mount_path) || null,
     state: railwayConnectionLabel(node.state) || null,
+    isPendingDeletion: node.isPendingDeletion === true || node.pendingDeletion === true,
+    deletedAt: railwayConnectionLabel(node.deletedAt) || null,
     sizeGb,
     usedGb
   };
 }
 function isActiveRailwayVolumeInstance(instance) {
   const state = String(instance.state ?? "READY").toUpperCase();
-  return state !== "DELETING" && state !== "DELETED";
+  return !instance.isPendingDeletion && !(typeof instance.deletedAt === "string" && instance.deletedAt.trim()) && state !== "DELETING" && state !== "DELETED";
 }
 function normalizeVolumeInstances(value) {
   const direct = Array.isArray(value) ? value : null;
@@ -300,6 +324,13 @@ function collectRailwayVolumes(value, seen = /* @__PURE__ */ new Set()) {
   }
   return [...byId.values()];
 }
+function railwayApiTimeoutMs(env, explicitTimeoutMs) {
+  if (typeof explicitTimeoutMs === "number" && Number.isFinite(explicitTimeoutMs) && explicitTimeoutMs > 0) {
+    return explicitTimeoutMs;
+  }
+  const configured = Number.parseInt(String(env.TREESEED_RAILWAY_API_TIMEOUT_MS ?? "").trim(), 10);
+  return Number.isFinite(configured) && configured > 0 ? configured : 12e4;
+}
 async function railwayGraphqlRequest({
   query,
   variables,
@@ -307,19 +338,20 @@ async function railwayGraphqlRequest({
   apiToken,
   apiUrl,
   fetchImpl = fetch,
-  timeoutMs = 3e4,
-  retries = 8
+  timeoutMs,
+  retries = 4
 }) {
   const token = apiToken || resolveRailwayApiToken(env);
   if (!token) {
-    throw new Error("Configure RAILWAY_API_TOKEN before invoking Railway APIs.");
+    throw new Error("Configure TREESEED_RAILWAY_API_TOKEN before invoking Railway APIs.");
   }
+  const requestTimeoutMs = railwayApiTimeoutMs(env, timeoutMs);
   let attempt = 0;
   for (; ; ) {
     const controller = new AbortController();
     let timer = null;
     try {
-      const response = fetchImpl === fetch ? await railwayGraphqlHttpsRequest(apiUrl || resolveRailwayApiUrl(env), token, { query, variables }, timeoutMs) : await Promise.race([
+      const response = fetchImpl === fetch ? await railwayGraphqlHttpsRequest(apiUrl || resolveRailwayApiUrl(env), token, { query, variables }, requestTimeoutMs) : await Promise.race([
         fetchImpl(apiUrl || resolveRailwayApiUrl(env), {
           method: "POST",
           headers: {
@@ -337,8 +369,8 @@ async function railwayGraphqlRequest({
         new Promise((_, reject) => {
           timer = setTimeout(() => {
             controller.abort();
-            reject(markRailwayTransientError(new Error(`Railway API request timed out after ${timeoutMs}ms.`)));
-          }, timeoutMs);
+            reject(markRailwayTransientError(new Error(`Railway API request timed out after ${requestTimeoutMs}ms.`)));
+          }, requestTimeoutMs);
         })
       ]);
       const payload = response.payload;
@@ -372,7 +404,24 @@ async function railwayGraphqlRequest({
 async function railwayGraphqlHttpsRequest(url, token, body, timeoutMs) {
   const rawBody = JSON.stringify(body);
   return new Promise((resolve, reject) => {
-    const req = httpsRequest(url, {
+    let settled = false;
+    let req = null;
+    const hardTimeout = setTimeout(() => {
+      if (settled) return;
+      settled = true;
+      const error = markRailwayTransientError(new Error(`Railway API request timed out after ${timeoutMs}ms.`));
+      if (req) {
+        req.destroy(error);
+      }
+      reject(error);
+    }, timeoutMs);
+    const finish = (callback) => {
+      if (settled) return;
+      settled = true;
+      clearTimeout(hardTimeout);
+      callback();
+    };
+    req = httpsRequest(url, {
       method: "POST",
       headers: {
         authorization: `Bearer ${token}`,
@@ -394,18 +443,21 @@ async function railwayGraphqlHttpsRequest(url, token, body, timeoutMs) {
         }
         const status = res.statusCode ?? 0;
         const retryAfterHeader = res.headers["retry-after"];
-        resolve({
+        finish(() => resolve({
           ok: status >= 200 && status < 300,
           status,
           payload,
           retryAfter: Array.isArray(retryAfterHeader) ? retryAfterHeader[0] ?? null : retryAfterHeader ?? null
-        });
+        }));
       });
     });
     req.on("timeout", () => {
-      req.destroy(markRailwayTransientError(new Error(`Railway API request timed out after ${timeoutMs}ms.`)));
+      const error = markRailwayTransientError(new Error(`Railway API request timed out after ${timeoutMs}ms.`));
+      if (!settled) {
+        req?.destroy(error);
+      }
     });
-    req.on("error", reject);
+    req.on("error", (error) => finish(() => reject(error)));
     req.write(rawBody);
     req.end();
   });
@@ -660,28 +712,71 @@ query TreeseedRailwayProjectEnvironments($projectId: String!) {
   });
   return normalizeConnectionNodes(payload.data?.project ? payload.data.project.environments : null, normalizeEnvironment);
 }
+async function listRailwayEnvironmentServices({
+  environmentId,
+  env = process.env,
+  fetchImpl = fetch
+}) {
+  const payload = await railwayGraphqlRequest({
+    query: `
+query TreeseedRailwayEnvironmentServices($environmentId: String!) {
+	environment(id: $environmentId) {
+		id
+		name
+		serviceInstances(first: 100) {
+			edges {
+				node {
+					id
+					serviceId
+					serviceName
+					environmentId
+				}
+			}
+		}
+	}
+}
+`.trim(),
+    variables: { environmentId },
+    env,
+    fetchImpl
+  });
+  return normalizeConnectionNodes(
+    payload.data?.environment ? payload.data.environment.serviceInstances : null,
+    normalizeServiceInstanceService
+  );
+}
 async function ensureRailwayService({
   projectId,
   serviceName,
   serviceId,
   environmentId,
   imageRef,
+  sourceRepo,
+  sourceBranch,
   env = process.env,
   fetchImpl = fetch
 }) {
   const services = await listRailwayServices({ projectId, env, fetchImpl });
   const desiredServiceName = railwayConnectionLabel(serviceName);
   const desiredServiceId = railwayConnectionLabel(serviceId);
-  const existing = services.find(
+  let existing = services.find(
     (service2) => desiredServiceId && service2.id === desiredServiceId || desiredServiceName && service2.name === desiredServiceName
   ) ?? null;
+  if (!existing && environmentId) {
+    const environmentServices = await listRailwayEnvironmentServices({ environmentId, env, fetchImpl }).catch(() => []);
+    existing = environmentServices.find(
+      (service2) => desiredServiceId && service2.id === desiredServiceId || desiredServiceName && service2.name === desiredServiceName
+    ) ?? null;
+  }
   if (existing) {
     const desiredImageRef = railwayConnectionLabel(imageRef);
-    if (desiredImageRef) {
+    const desiredSourceRepo = railwayConnectionLabel(sourceRepo);
+    if (desiredSourceRepo) {
       try {
-        await updateRailwayServiceImageSource({
+        await updateRailwayServiceGitSource({
           serviceId: existing.id,
-          imageRef: desiredImageRef,
+          sourceRepo: desiredSourceRepo,
+          sourceBranch,
           env,
           fetchImpl
         });
@@ -689,17 +784,27 @@ async function ensureRailwayService({
         if (!looksLikeRailwayImageSourceUpdateUnsupported(error)) {
           throw error;
         }
-        await deleteRailwayService({ serviceId: existing.id, env, fetchImpl });
-        await waitForRailwayServiceMissing({ projectId, serviceId: existing.id, env, fetchImpl });
-        const replacement = await createRailwayImageService({
-          projectId,
-          environmentId,
-          serviceName: desiredServiceName || existing.name,
+        throw new Error(
+          `Railway Git source update for existing service ${existing.name} (${existing.id}) is unsupported; refusing to delete and recreate an existing service. Repair the service in place or use a provider-supported source update.`
+        );
+      }
+      return { service: existing, created: false };
+    }
+    if (desiredImageRef) {
+      try {
+        await updateRailwayServiceImageSource({
+          serviceId: existing.id,
           imageRef: desiredImageRef,
           env,
           fetchImpl
         });
-        return { service: replacement, created: true, replaced: true };
+      } catch (error) {
+        if (!looksLikeRailwayImageSourceUpdateUnsupported(error)) {
+          throw error;
+        }
+        throw new Error(
+          `Railway image source update for existing service ${existing.name} (${existing.id}) is unsupported; refusing to delete and recreate an existing service. Repair the service in place or use a provider-supported image source update.`
+        );
       }
     }
     return { service: existing, created: false };
@@ -712,6 +817,8 @@ async function ensureRailwayService({
     environmentId,
     serviceName: desiredServiceName,
     imageRef,
+    sourceRepo,
+    sourceBranch,
     env,
     fetchImpl
   });
@@ -722,9 +829,13 @@ async function createRailwayImageService({
   serviceName,
   environmentId,
   imageRef,
+  sourceRepo,
+  sourceBranch,
   env = process.env,
   fetchImpl = fetch
 }) {
+  const desiredSourceRepo = railwayConnectionLabel(sourceRepo);
+  const desiredImageRef = railwayConnectionLabel(imageRef);
   const created = await railwayGraphqlRequest({
     query: `
 mutation TreeseedRailwayServiceCreate($input: ServiceCreateInput!) {
@@ -739,9 +850,14 @@ mutation TreeseedRailwayServiceCreate($input: ServiceCreateInput!) {
         projectId,
         name: serviceName,
         ...railwayConnectionLabel(environmentId) ? { environmentId: railwayConnectionLabel(environmentId) } : {},
-        ...railwayConnectionLabel(imageRef) ? {
+        ...desiredSourceRepo ? {
+          source: {
+            repo: desiredSourceRepo
+          },
+          ...railwayConnectionLabel(sourceBranch) ? { branch: railwayConnectionLabel(sourceBranch) } : {}
+        } : desiredImageRef ? {
           source: {
-            image: railwayConnectionLabel(imageRef)
+            image: desiredImageRef
           }
         } : {}
       }
@@ -755,23 +871,6 @@ mutation TreeseedRailwayServiceCreate($input: ServiceCreateInput!) {
   }
   return service;
 }
-async function waitForRailwayServiceMissing({
-  projectId,
-  serviceId,
-  env = process.env,
-  fetchImpl = fetch,
-  attempts = 20,
-  delayMs = 1500
-}) {
-  for (let attempt = 0; attempt < attempts; attempt += 1) {
-    const services = await listRailwayServices({ projectId, env, fetchImpl }).catch(() => []);
-    if (!services.some((service) => service.id === serviceId)) {
-      return true;
-    }
-    await new Promise((resolve) => setTimeout(resolve, delayMs));
-  }
-  throw new Error(`Railway service ${serviceId} was not deleted before image source replacement.`);
-}
 function looksLikeRailwayImageSourceUpdateUnsupported(error) {
   const message = error instanceof Error ? error.message : String(error ?? "");
   return /Problem processing request|source|image|ServiceUpdateInput/iu.test(message);
@@ -788,8 +887,8 @@ async function updateRailwayServiceImageSource({
   }
   const payload = await railwayGraphqlRequest({
     query: `
-mutation TreeseedRailwayServiceImageSourceUpdate($id: String!, $input: ServiceUpdateInput!) {
-	serviceUpdate(id: $id, input: $input) {
+mutation TreeseedRailwayServiceImageSourceUpdate($id: String!, $input: ServiceConnectInput!) {
+	serviceConnect(id: $id, input: $input) {
 		id
 		name
 	}
@@ -798,20 +897,67 @@ mutation TreeseedRailwayServiceImageSourceUpdate($id: String!, $input: ServiceUp
     variables: {
       id: serviceId,
       input: {
-        source: {
-          image: desiredImage
-        }
+        image: desiredImage
       }
     },
     env,
     fetchImpl
   });
-  const service = payload.data?.serviceUpdate ? normalizeService(payload.data.serviceUpdate) : null;
+  const service = payload.data?.serviceConnect ? normalizeService(payload.data.serviceConnect) : null;
   if (!service) {
     throw new Error(`Railway service image source update did not return a usable service for ${serviceId}.`);
   }
   return service;
 }
+async function updateRailwayServiceGitSource({
+  serviceId,
+  sourceRepo,
+  sourceBranch,
+  env = process.env,
+  fetchImpl = fetch
+}) {
+  const desiredRepo = railwayConnectionLabel(sourceRepo);
+  if (!serviceId || !desiredRepo) {
+    throw new Error("Railway service Git source update requires a service id and repository slug.");
+  }
+  const connect = async (repo) => {
+    const payload2 = await railwayGraphqlRequest({
+      query: `
+mutation TreeseedRailwayServiceGitSourceUpdate($id: String!, $input: ServiceConnectInput!) {
+	serviceConnect(id: $id, input: $input) {
+		id
+		name
+	}
+}
+`.trim(),
+      variables: {
+        id: serviceId,
+        input: {
+          repo,
+          ...railwayConnectionLabel(sourceBranch) ? { branch: railwayConnectionLabel(sourceBranch) } : {}
+        }
+      },
+      env,
+      fetchImpl
+    });
+    return payload2;
+  };
+  let payload;
+  try {
+    payload = await connect(desiredRepo);
+  } catch (error) {
+    const message = error instanceof Error ? error.message : String(error ?? "");
+    if (!/User does not have access to the repo/iu.test(message) || /^https?:\/\//iu.test(desiredRepo)) {
+      throw error;
+    }
+    payload = await connect(`https://github.com/${desiredRepo}`);
+  }
+  const service = payload.data?.serviceConnect ? normalizeService(payload.data.serviceConnect) : null;
+  if (!service) {
+    throw new Error(`Railway service Git source update did not return a usable service for ${serviceId}.`);
+  }
+  return service;
+}
 async function ensureRailwayGeneratedServiceDomain({
   projectId,
   environmentId,
@@ -986,7 +1132,9 @@ async function ensureRailwayPostgresService({
     if (proof.ok) {
       return { service: existing, created: false, proof };
     }
-    await deleteRailwayService({ serviceId: existing.id, env, fetchImpl });
+    throw new Error(
+      `Railway Postgres service ${existing.name} (${existing.id}) failed proof; refusing to delete and recreate an existing service. Repair the existing database service in place and rerun reconciliation.`
+    );
   }
   const template = await getRailwayTemplateByCode({ code: "postgres", env, fetchImpl });
   await deployRailwayTemplate({
@@ -1157,6 +1305,7 @@ query TreeseedRailwayServiceDeploymentHealth($serviceId: String!, $environmentId
 		latestDeployment {
 			status
 			deploymentStopped
+			meta
 			instances {
 				status
 			}
@@ -1176,6 +1325,12 @@ query TreeseedRailwayServiceDeploymentHealth($serviceId: String!, $environmentId
   return {
     ok,
     status,
+    branch: railwayConnectionLabel(deployment?.meta?.branch) || null,
+    repo: railwayConnectionLabel(deployment?.meta?.repo) || null,
+    rootDirectory: railwayConnectionLabel(deployment?.meta?.rootDirectory) || null,
+    commitHash: railwayConnectionLabel(deployment?.meta?.commitHash) || null,
+    requiredMountPath: railwayConnectionLabel(deployment?.meta?.serviceManifest?.deploy?.requiredMountPath) || null,
+    volumeMounts: Array.isArray(deployment?.meta?.volumeMounts) ? deployment.meta.volumeMounts.map((entry) => railwayConnectionLabel(entry)).filter(Boolean) : [],
     message: ok ? "Deployment is healthy." : `Latest deployment status is ${status ?? "unknown"}${stopped ? " and stopped" : ""}${instanceStatuses.length ? `; instances=${instanceStatuses.join(",")}` : ""}.`
   };
 }
@@ -1215,6 +1370,8 @@ async function getRailwayServiceInstance({
   const legacySummary = {
     id: null,
     buildCommand: null,
+    dockerfilePath: null,
+    railwayConfigFile: null,
     startCommand: null,
     cronSchedule: null,
     rootDirectory: null,
@@ -1233,6 +1390,8 @@ query TreeseedRailwayServiceInstance($serviceId: String!, $environmentId: String
 	serviceInstance(serviceId: $serviceId, environmentId: $environmentId) {
 		id
 		buildCommand
+		dockerfilePath
+		railwayConfigFile
 		startCommand
 		cronSchedule
 		rootDirectory
@@ -1250,6 +1409,8 @@ query TreeseedRailwayServiceInstance($serviceId: String!, $environmentId: String
     return {
       id: railwayConnectionLabel(instance?.id) || null,
       buildCommand: railwayConnectionLabel(instance?.buildCommand) || null,
+      dockerfilePath: railwayConnectionLabel(instance?.dockerfilePath) || null,
+      railwayConfigFile: railwayConnectionLabel(instance?.railwayConfigFile) || null,
       startCommand: railwayConnectionLabel(instance?.startCommand) || null,
       cronSchedule: railwayConnectionLabel(instance?.cronSchedule) || null,
       rootDirectory: railwayConnectionLabel(instance?.rootDirectory) || null,
@@ -1270,6 +1431,8 @@ query TreeseedRailwayServiceInstanceLegacy($serviceId: String!, $environmentId:
 	serviceInstance(serviceId: $serviceId, environmentId: $environmentId) {
 		id
 		buildCommand
+		dockerfilePath
+		railwayConfigFile
 		startCommand
 		cronSchedule
 		rootDirectory
@@ -1285,6 +1448,8 @@ query TreeseedRailwayServiceInstanceLegacy($serviceId: String!, $environmentId:
         ...legacySummary,
         id: railwayConnectionLabel(instance?.id) || null,
         buildCommand: railwayConnectionLabel(instance?.buildCommand) || null,
+        dockerfilePath: railwayConnectionLabel(instance?.dockerfilePath) || null,
+        railwayConfigFile: railwayConnectionLabel(instance?.railwayConfigFile) || null,
         startCommand: railwayConnectionLabel(instance?.startCommand) || null,
         cronSchedule: railwayConnectionLabel(instance?.cronSchedule) || null,
         rootDirectory: railwayConnectionLabel(instance?.rootDirectory) || null
@@ -1300,6 +1465,8 @@ async function ensureRailwayServiceInstanceConfiguration({
   serviceId,
   environmentId,
   buildCommand,
+  dockerfilePath,
+  railwayConfigFile,
   startCommand,
   cronSchedule,
   rootDirectory,
@@ -1328,6 +1495,8 @@ async function ensureRailwayServiceInstanceConfiguration({
   const desiredDeploymentRegion = railwayConnectionLabel(deploymentRegion) || null;
   const desired = {
     buildCommand: railwayConnectionLabel(buildCommand) || null,
+    dockerfilePath: railwayConnectionLabel(dockerfilePath) || null,
+    railwayConfigFile: railwayConnectionLabel(railwayConfigFile) || null,
     startCommand: railwayConnectionLabel(startCommand) || null,
     cronSchedule: railwayConnectionLabel(cronSchedule) || null,
     rootDirectory: railwayConnectionLabel(rootDirectory) || null,
@@ -1349,7 +1518,7 @@ async function ensureRailwayServiceInstanceConfiguration({
   if (desired.restartPolicy !== null) {
     throw new Error("Railway service instance restart policies are unsupported by the current Railway API schema.");
   }
-  const drifted = desired.buildCommand !== null && desired.buildCommand !== current.buildCommand || desired.startCommand !== null && desired.startCommand !== current.startCommand || desired.cronSchedule !== null && desired.cronSchedule !== current.cronSchedule || desired.rootDirectory !== null && desired.rootDirectory !== current.rootDirectory || desired.healthcheckPath !== null && desired.healthcheckPath !== current.healthcheckPath || desired.healthcheckTimeoutSeconds !== null && desired.healthcheckTimeoutSeconds !== current.healthcheckTimeoutSeconds || desired.runtimeMode !== null && desired.runtimeMode !== current.runtimeMode || desired.deploymentRegion !== null;
+  const drifted = desired.buildCommand !== null && desired.buildCommand !== current.buildCommand || desired.dockerfilePath !== null && desired.dockerfilePath !== current.dockerfilePath || desired.railwayConfigFile !== null && desired.railwayConfigFile !== current.railwayConfigFile || desired.startCommand !== null && desired.startCommand !== current.startCommand || desired.cronSchedule !== null && desired.cronSchedule !== current.cronSchedule || desired.rootDirectory !== null && desired.rootDirectory !== current.rootDirectory || desired.healthcheckPath !== null && desired.healthcheckPath !== current.healthcheckPath || desired.healthcheckTimeoutSeconds !== null && desired.healthcheckTimeoutSeconds !== current.healthcheckTimeoutSeconds || desired.runtimeMode !== null && desired.runtimeMode !== current.runtimeMode || desired.deploymentRegion !== null;
   if (!drifted) {
     return { instance: current, updated: false };
   }
@@ -1370,6 +1539,8 @@ mutation TreeseedRailwayServiceInstanceUpdateLegacy($serviceId: String!, $enviro
         environmentId,
         input: {
           ...desired.buildCommand !== null ? { buildCommand: desired.buildCommand } : {},
+          ...desired.dockerfilePath !== null ? { dockerfilePath: desired.dockerfilePath } : {},
+          ...desired.railwayConfigFile !== null ? { railwayConfigFile: desired.railwayConfigFile } : {},
           ...desired.startCommand !== null ? { startCommand: desired.startCommand } : {},
           ...desired.cronSchedule !== null ? { cronSchedule: desired.cronSchedule } : {},
           ...desired.rootDirectory !== null ? { rootDirectory: desired.rootDirectory } : {},
@@ -1474,6 +1645,21 @@ mutation TreeseedRailwayVariableCollectionUpsert($input: VariableCollectionUpser
     replace: false,
     skipDeploys: true
   };
+  const upsertOne = (key, value) => railwayGraphqlRequest({
+    query,
+    variables: {
+      input: {
+        projectId,
+        environmentId,
+        serviceId: serviceId || null,
+        variables: { [key]: value },
+        replace: false,
+        skipDeploys: true
+      }
+    },
+    env,
+    fetchImpl
+  });
   try {
     await railwayGraphqlRequest({
       query,
@@ -1487,23 +1673,29 @@ mutation TreeseedRailwayVariableCollectionUpsert($input: VariableCollectionUpser
       throw error;
     }
     for (const [key, value] of Object.entries(variables)) {
-      await railwayGraphqlRequest({
-        query,
-        variables: {
-          input: {
-            projectId,
-            environmentId,
-            serviceId: serviceId || null,
-            variables: { [key]: value },
-            replace: false,
-            skipDeploys: true
-          }
-        },
-        env,
-        fetchImpl
-      });
+      await upsertOne(key, value);
+    }
+  }
+  const observed = await listRailwayVariables({ projectId, environmentId, serviceId, env, fetchImpl }).catch(() => ({}));
+  const missing = Object.keys(variables).filter((key) => observed[key] == null);
+  for (const key of missing) {
+    await upsertOne(key, variables[key]);
+  }
+  let retried = missing.length > 0 ? await listRailwayVariables({ projectId, environmentId, serviceId, env, fetchImpl }).catch(() => ({})) : observed;
+  let stillMissing = Object.keys(variables).filter((key) => retried[key] == null);
+  for (let attempt = 0; stillMissing.length > 0 && attempt < 12; attempt += 1) {
+    await new Promise((resolve) => setTimeout(resolve, 2500));
+    retried = await listRailwayVariables({ projectId, environmentId, serviceId, env, fetchImpl }).catch(() => ({}));
+    stillMissing = Object.keys(variables).filter((key) => retried[key] == null);
+    if (stillMissing.length > 0 && attempt === 5) {
+      for (const key of stillMissing) {
+        await upsertOne(key, variables[key]);
+      }
     }
   }
+  if (stillMissing.length > 0) {
+    throw new Error(`Railway variable upsert did not persist: ${stillMissing.join(", ")}.`);
+  }
 }
 async function listRailwayVolumes({
   projectId,
@@ -1528,6 +1720,8 @@ query TreeseedRailwayVolumeList($projectId: String!) {
 								environmentId
 								mountPath
 								state
+								isPendingDeletion
+								deletedAt
 							}
 						}
 					}
@@ -1562,15 +1756,17 @@ mutation TreeseedRailwayVolumeCreate($input: VolumeCreateInput!) {
 		projectId
 		volumeInstances {
 			edges {
-				node {
-					id
-					serviceId
-					environmentId
-					mountPath
-								state
+					node {
+						id
+						serviceId
+						environmentId
+						mountPath
+						state
+						isPendingDeletion
+						deletedAt
+					}
 				}
 			}
-		}
 	}
 }
 `.trim();
@@ -1626,15 +1822,17 @@ mutation TreeseedRailwayVolumeUpdate($volumeId: String!, $input: VolumeUpdateInp
 		projectId
 		volumeInstances {
 			edges {
-				node {
-					id
-					serviceId
-					environmentId
-					mountPath
-								state
+					node {
+						id
+						serviceId
+						environmentId
+						mountPath
+						state
+						isPendingDeletion
+						deletedAt
+					}
 				}
 			}
-		}
 	}
 }
 `.trim();
@@ -1666,7 +1864,7 @@ mutation TreeseedRailwayVolumeInstanceUpdate($volumeId: String!, $input: VolumeI
     variables: {
       volumeId,
       input: {
-        ...serviceId ? { serviceId } : {},
+        ...serviceId !== void 0 ? { serviceId } : {},
         mountPath
       }
     },
@@ -1710,6 +1908,8 @@ async function ensureRailwayServiceVolume({
   }
   let volume = findRailwayVolumeForService(volumes, serviceId, environmentId) ?? findRailwayVolumeForService(volumes, serviceId) ?? activeVolumes.find(
     (candidate) => candidate.name === name && candidate.instances.some((instance2) => instance2.environmentId === environmentId)
+  ) ?? activeVolumes.find(
+    (candidate) => candidate.name === name && (candidate.instances.length === 0 || candidate.instances.some((instance2) => instance2.environmentId === environmentId))
   ) ?? null;
   let created = false;
   let updated = false;
@@ -1732,7 +1932,13 @@ async function ensureRailwayServiceVolume({
       for (let attempt = 0; attempt < 8; attempt += 1) {
         await new Promise((resolve) => setTimeout(resolve, 1500));
         const refreshed = await listRailwayVolumes({ projectId, env, fetchImpl });
-        const existing = findRailwayVolumeForService(refreshed, serviceId, environmentId) ?? findRailwayVolumeForService(refreshed, serviceId) ?? findSoleActiveRailwayVolumeForEnvironment(refreshed, environmentId);
+        const activeRefreshed = refreshed.map((candidate) => ({
+          ...candidate,
+          instances: candidate.instances.filter(isActiveRailwayVolumeInstance)
+        })).filter((candidate) => candidate.instances.length > 0);
+        const existing = findRailwayVolumeForService(refreshed, serviceId, environmentId) ?? findRailwayVolumeForService(refreshed, serviceId) ?? activeRefreshed.find(
+          (candidate) => candidate.name === name && (candidate.instances.length === 0 || candidate.instances.some((instance2) => instance2.environmentId === environmentId))
+        ) ?? findSoleActiveRailwayVolumeForEnvironment(refreshed, environmentId);
         if (existing) {
           return existing;
         }
@@ -1745,7 +1951,9 @@ async function ensureRailwayServiceVolume({
   if (!volume) {
     volume = await createReplacementVolume();
   }
-  const desiredNameInUse = volumes.some((candidate) => candidate.id !== volume?.id && candidate.name === name);
+  const desiredNameInUse = volumes.some(
+    (candidate) => candidate.id !== volume?.id && candidate.name === name && candidate.instances.some(isActiveRailwayVolumeInstance)
+  );
   if (volume.name && volume.name !== name && !desiredNameInUse) {
     try {
       volume = await updateRailwayVolumeName({ volumeId: volume.id, name, env, fetchImpl }) ?? { ...volume, name };
@@ -1869,6 +2077,15 @@ function findRailwayVolumeForService(volumes, serviceId, environmentId) {
     )
   ) ?? null;
 }
+function findSoleActiveRailwayVolumeForEnvironment(volumes, environmentId) {
+  const active = volumes.map((candidate) => ({
+    ...candidate,
+    instances: candidate.instances.filter(
+      (instance) => instance.environmentId === environmentId && isActiveRailwayVolumeInstance(instance)
+    )
+  })).filter((candidate) => candidate.instances.length > 0);
+  return active.length === 1 ? active[0] : null;
+}
 function looksLikeRailwayVolumeCreateRace(error) {
   const message = error instanceof Error ? error.message : String(error ?? "");
   return /already has a volume attached|would have \d+ volumes attached|can only have one volume|volume named .* already exists|already exists in this project|not authorized/iu.test(message);
@@ -1986,6 +2203,10 @@ function looksLikeRailwayMissingResource(error) {
   const message = error instanceof Error ? error.message : String(error ?? "");
   return /not found|does not exist|could not find|unknown|invalid .*id/iu.test(message);
 }
+function looksLikeRailwayOperationInProgress(error) {
+  const message = error instanceof Error ? error.message : String(error ?? "");
+  return /operation is already in progress/iu.test(message);
+}
 async function railwayDeleteMutation({
   query,
   variables,
@@ -1993,24 +2214,32 @@ async function railwayDeleteMutation({
   fetchImpl,
   missingResult
 }) {
-  try {
-    const payload = await railwayGraphqlRequest({
-      query,
-      variables,
-      env,
-      fetchImpl
-    });
-    const mutationResult = Object.values(payload.data ?? {})[0];
-    if (mutationResult === false || mutationResult == null) {
-      throw new Error("Railway delete mutation returned no successful deletion result.");
-    }
-    return { status: "deleted" };
-  } catch (error) {
-    if (looksLikeRailwayMissingResource(error)) {
-      return missingResult;
+  let lastError = null;
+  for (let attempt = 0; attempt < 6; attempt += 1) {
+    try {
+      const payload = await railwayGraphqlRequest({
+        query,
+        variables,
+        env,
+        fetchImpl
+      });
+      const mutationResult = Object.values(payload.data ?? {})[0];
+      if (mutationResult === false || mutationResult == null) {
+        throw new Error("Railway delete mutation returned no successful deletion result.");
+      }
+      return { status: "deleted" };
+    } catch (error) {
+      if (looksLikeRailwayMissingResource(error)) {
+        return missingResult;
+      }
+      if (!looksLikeRailwayOperationInProgress(error) || attempt >= 5) {
+        throw error;
+      }
+      lastError = error;
+      await new Promise((resolve) => setTimeout(resolve, 2500 * (attempt + 1)));
     }
-    throw error;
   }
+  throw lastError instanceof Error ? lastError : new Error(String(lastError ?? "Railway delete mutation did not complete."));
 }
 async function deleteRailwayCustomDomain({
   domainId,
@@ -2175,6 +2404,7 @@ export {
   inspectRailwayServiceDeploymentHealth,
   isUsableRailwayToken,
   listRailwayCustomDomains,
+  listRailwayEnvironmentServices,
   listRailwayEnvironments,
   listRailwayProjects,
   listRailwayServiceDomains,
@@ -2187,6 +2417,7 @@ export {
   resolveRailwayApiUrl,
   resolveRailwayWorkspace,
   resolveRailwayWorkspaceContext,
+  updateRailwayServiceGitSource,
   updateRailwayServiceImageSource,
   updateRailwayServiceName,
   upsertRailwayVariables