@treeseed/sdk 0.12.59 → 0.12.61
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/guarantees/index.js +59 -56
- package/dist/hosting/contracts.d.ts +0 -19
- package/dist/hosting/graph.d.ts +1 -86
- package/dist/hosting/graph.js +96 -247
- package/dist/local-dev/managed-dev.js +30 -8
- package/dist/managed-dependencies.d.ts +3 -0
- package/dist/managed-dependencies.js +294 -20
- package/dist/operations/services/deploy.js +6 -6
- package/dist/operations/services/deployment-readiness.js +5 -4
- package/dist/operations/services/git-runner.d.ts +2 -0
- package/dist/operations/services/git-runner.js +23 -2
- package/dist/operations/services/hosted-service-checks.js +28 -0
- package/dist/operations/services/live-hosted-service-checks.js +56 -14
- package/dist/operations/services/local-cleanup.d.ts +1 -0
- package/dist/operations/services/local-cleanup.js +28 -9
- package/dist/operations/services/package-adapters.js +3 -3
- package/dist/operations/services/railway-api.d.ts +72 -28
- package/dist/operations/services/railway-api.js +321 -876
- package/dist/operations/services/railway-cli.d.ts +47 -0
- package/dist/operations/services/railway-cli.js +142 -0
- package/dist/operations/services/railway-deploy.d.ts +5 -0
- package/dist/operations/services/railway-deploy.js +47 -75
- package/dist/operations/services/railway-source-policy.d.ts +6 -0
- package/dist/operations/services/railway-source-policy.js +52 -9
- package/dist/operations/services/repository-save-orchestrator.d.ts +2 -0
- package/dist/operations/services/repository-save-orchestrator.js +45 -14
- package/dist/operations-types.d.ts +3 -1
- package/dist/platform/contracts.d.ts +1 -0
- package/dist/platform/deploy-config.js +2 -1
- package/dist/reconcile/builtin-adapters.js +524 -680
- package/dist/reconcile/desired-state.js +5 -3
- package/dist/reconcile/engine.js +34 -28
- package/dist/reconcile/live-acceptance.js +2 -11
- package/dist/reconcile/providers/railway-iac.d.ts +148 -0
- package/dist/reconcile/providers/railway-iac.js +294 -18
- package/dist/scenes/runner.js +11 -11
- package/dist/scripts/build-dist.js +22 -0
- package/dist/workflow/operations.d.ts +12 -0
- package/dist/workflow/operations.js +265 -90
- package/dist/workflow/runs.d.ts +4 -0
- package/dist/workflow/runs.js +25 -0
- package/dist/workflow-support.d.ts +1 -1
- package/dist/workflow-support.js +3 -1
- package/package.json +1 -2
|
@@ -1,8 +1,7 @@
|
|
|
1
1
|
import { createHmac, randomBytes, randomUUID } from "node:crypto";
|
|
2
2
|
import { spawnSync } from "node:child_process";
|
|
3
|
-
import {
|
|
3
|
+
import { existsSync, mkdirSync, statSync } from "node:fs";
|
|
4
4
|
import { readdir, readFile } from "node:fs/promises";
|
|
5
|
-
import { tmpdir } from "node:os";
|
|
6
5
|
import { dirname, relative, resolve } from "node:path";
|
|
7
6
|
import { resolveCapacityProviderLaunchPlan } from "../capacity-provider.js";
|
|
8
7
|
import { TreeDxClient } from "../treedx-client.js";
|
|
@@ -43,21 +42,23 @@ import {
|
|
|
43
42
|
configuredRailwayServices,
|
|
44
43
|
findStaleTreeseedOperationsRunnerResources,
|
|
45
44
|
isTreeseedOperationsRunnerResourceName,
|
|
45
|
+
railwayObsoleteAliasCleanupPolicy,
|
|
46
46
|
validateRailwayDeployPrerequisites
|
|
47
47
|
} from "../operations/services/railway-deploy.js";
|
|
48
48
|
import { shouldExposeManagedHostRuntimeSecret } from "../operations/services/managed-host-security.js";
|
|
49
|
+
import { assertNoRailwaySourceIdentityCollisions } from "../operations/services/railway-source-policy.js";
|
|
50
|
+
import {
|
|
51
|
+
attachRailwayVolumeWithCli,
|
|
52
|
+
detachRailwayVolumeWithCli,
|
|
53
|
+
updateRailwayVolumeWithCli
|
|
54
|
+
} from "../operations/services/railway-cli.js";
|
|
49
55
|
import {
|
|
50
56
|
ensureRailwayEnvironment,
|
|
51
|
-
ensureRailwayCustomDomain as ensureRailwayCustomDomainViaApi,
|
|
52
57
|
ensureRailwayProject,
|
|
53
|
-
|
|
54
|
-
ensureRailwayServiceInstanceConfiguration,
|
|
55
|
-
ensureRailwayServiceVolume,
|
|
56
|
-
ensureRailwayPostgresService,
|
|
58
|
+
ensureRailwayCustomDomain,
|
|
57
59
|
deleteRailwayService,
|
|
58
60
|
deleteRailwayCustomDomain,
|
|
59
61
|
deleteRailwayVolume,
|
|
60
|
-
deployRailwayServiceInstance,
|
|
61
62
|
inspectRailwayServiceDeploymentHealth,
|
|
62
63
|
listRailwayEnvironmentServices,
|
|
63
64
|
getRailwayServiceInstance,
|
|
@@ -69,12 +70,28 @@ import {
|
|
|
69
70
|
listRailwayServices,
|
|
70
71
|
listRailwayVolumes,
|
|
71
72
|
listRailwayVariables,
|
|
72
|
-
resolveRailwayWorkspaceContext
|
|
73
|
-
updateRailwayServiceGitSource,
|
|
74
|
-
updateRailwayServiceImageSource
|
|
73
|
+
resolveRailwayWorkspaceContext
|
|
75
74
|
} from "../operations/services/railway-api.js";
|
|
76
75
|
import { loadTreeseedReconcileState } from "./state.js";
|
|
77
76
|
import { createTreeseedReconcileUnitId } from "./units.js";
|
|
77
|
+
import {
|
|
78
|
+
applyRailwayIacProject,
|
|
79
|
+
cleanupRailwayIacRender,
|
|
80
|
+
detachRetainedRailwayCustomDomains,
|
|
81
|
+
detachRetainedRailwayVolumeBindings,
|
|
82
|
+
findRailwayPendingVolumeNameCollisions,
|
|
83
|
+
planRailwayIacProject,
|
|
84
|
+
railwayIacApplyFailure,
|
|
85
|
+
renderRailwayIacProject,
|
|
86
|
+
resolveRailwayIacVolumeBindings,
|
|
87
|
+
waitForRailwayVolumeAdoptionResources,
|
|
88
|
+
waitForRailwayVolumeDetachment,
|
|
89
|
+
waitForRailwayVolumeName,
|
|
90
|
+
waitForRailwayServiceAbsence,
|
|
91
|
+
waitForRailwayServices,
|
|
92
|
+
selectRailwayIacRetainedResources,
|
|
93
|
+
validateRailwayIacChangeSet
|
|
94
|
+
} from "./providers/railway-iac.js";
|
|
78
95
|
import { discoverTreeseedApplications } from "../hosting/apps.js";
|
|
79
96
|
import { findTreeseedPackageAdapter, syncTreeseedPackageWorkflows } from "../operations/services/package-adapters.js";
|
|
80
97
|
import { resolveGitHubCredentialForRepository } from "../operations/services/github-credentials.js";
|
|
@@ -2348,6 +2365,8 @@ function normalizeRailwayDomainPayload(value) {
|
|
|
2348
2365
|
return {
|
|
2349
2366
|
id: typeof record.id === "string" ? record.id.trim() : null,
|
|
2350
2367
|
domain,
|
|
2368
|
+
serviceId: typeof record.serviceId === "string" ? record.serviceId.trim() : null,
|
|
2369
|
+
serviceName: typeof record.serviceName === "string" ? record.serviceName.trim() : null,
|
|
2351
2370
|
serviceDomain,
|
|
2352
2371
|
verified: record.verified === true || status.verified === true || verification.verified === true,
|
|
2353
2372
|
certificateStatus: firstRailwayDomainString(
|
|
@@ -2360,94 +2379,29 @@ function normalizeRailwayDomainPayload(value) {
|
|
|
2360
2379
|
dnsRecords: effectiveDnsRecords
|
|
2361
2380
|
};
|
|
2362
2381
|
}
|
|
2363
|
-
async function ensureRailwayCustomDomain(input, service, domain, env, identifiers) {
|
|
2364
|
-
if (!identifiers?.projectId || !identifiers.environmentId || !identifiers.serviceId) {
|
|
2365
|
-
throw new Error(`Railway custom domain ${domain} cannot be reconciled without project, environment, and service ids.`);
|
|
2366
|
-
}
|
|
2367
|
-
const existing = await listRailwayCustomDomains({
|
|
2368
|
-
projectId: identifiers.projectId,
|
|
2369
|
-
environmentId: identifiers.environmentId,
|
|
2370
|
-
serviceId: identifiers.serviceId,
|
|
2371
|
-
env
|
|
2372
|
-
});
|
|
2373
|
-
const matched = existing.find((entry) => entry.domain === domain) ?? null;
|
|
2374
|
-
if (matched) {
|
|
2375
|
-
return matched;
|
|
2376
|
-
}
|
|
2377
|
-
const environmentServices = await listRailwayEnvironmentServices({
|
|
2378
|
-
environmentId: identifiers.environmentId,
|
|
2379
|
-
env
|
|
2380
|
-
});
|
|
2381
|
-
for (const candidate of environmentServices) {
|
|
2382
|
-
const candidateServiceId = typeof candidate.id === "string" ? candidate.id.trim() : "";
|
|
2383
|
-
if (!candidateServiceId || candidateServiceId === identifiers.serviceId) {
|
|
2384
|
-
continue;
|
|
2385
|
-
}
|
|
2386
|
-
const candidateDomains = await listRailwayCustomDomains({
|
|
2387
|
-
projectId: identifiers.projectId,
|
|
2388
|
-
environmentId: identifiers.environmentId,
|
|
2389
|
-
serviceId: candidateServiceId,
|
|
2390
|
-
env
|
|
2391
|
-
});
|
|
2392
|
-
const staleAttachment = candidateDomains.find((entry) => entry.domain === domain) ?? null;
|
|
2393
|
-
if (!staleAttachment) {
|
|
2394
|
-
continue;
|
|
2395
|
-
}
|
|
2396
|
-
if (!staleAttachment.id) {
|
|
2397
|
-
throw new Error(`Railway custom domain ${domain} is attached to service ${candidateServiceId}, but Railway did not return its domain id for exact-state reattachment.`);
|
|
2398
|
-
}
|
|
2399
|
-
await deleteRailwayCustomDomain({ domainId: staleAttachment.id, env });
|
|
2400
|
-
let detached = false;
|
|
2401
|
-
for (let attempt = 0; attempt < 12; attempt += 1) {
|
|
2402
|
-
const remaining = await listRailwayCustomDomains({
|
|
2403
|
-
projectId: identifiers.projectId,
|
|
2404
|
-
environmentId: identifiers.environmentId,
|
|
2405
|
-
serviceId: candidateServiceId,
|
|
2406
|
-
env
|
|
2407
|
-
});
|
|
2408
|
-
if (!remaining.some((entry) => entry.domain === domain)) {
|
|
2409
|
-
detached = true;
|
|
2410
|
-
break;
|
|
2411
|
-
}
|
|
2412
|
-
await new Promise((resolve2) => setTimeout(resolve2, 1e3));
|
|
2413
|
-
}
|
|
2414
|
-
if (!detached) {
|
|
2415
|
-
throw new Error(`Railway custom domain ${domain} remained attached to service ${candidateServiceId} after deletion.`);
|
|
2416
|
-
}
|
|
2417
|
-
break;
|
|
2418
|
-
}
|
|
2419
|
-
const ensured = await ensureRailwayCustomDomainViaApi({
|
|
2420
|
-
projectId: identifiers.projectId,
|
|
2421
|
-
environmentId: identifiers.environmentId,
|
|
2422
|
-
serviceId: identifiers.serviceId,
|
|
2423
|
-
domain,
|
|
2424
|
-
env
|
|
2425
|
-
});
|
|
2426
|
-
if (!ensured.domain) {
|
|
2427
|
-
throw new Error(`Railway API custom domain reconciliation did not return ${domain}.`);
|
|
2428
|
-
}
|
|
2429
|
-
return ensured.domain;
|
|
2430
|
-
}
|
|
2431
2382
|
async function observeRailwayCustomDomainLive(input, domain) {
|
|
2432
2383
|
if (!domain) {
|
|
2433
2384
|
return null;
|
|
2434
2385
|
}
|
|
2435
2386
|
const scope = input.context.target.kind === "persistent" ? input.context.target.scope : "staging";
|
|
2436
2387
|
const serviceKey = String(input.unit.metadata.serviceKey ?? "api").trim();
|
|
2437
|
-
const topology = await
|
|
2438
|
-
refresh:
|
|
2439
|
-
serviceKeys: [serviceKey],
|
|
2388
|
+
const topology = await resolveRailwayUnitTopology(input, scope, {
|
|
2389
|
+
refresh: false,
|
|
2440
2390
|
includeInstances: false,
|
|
2441
2391
|
includeVariables: false
|
|
2442
2392
|
});
|
|
2443
|
-
const entry = topology
|
|
2393
|
+
const entry = findRailwayTopologyEntry(topology, serviceKey);
|
|
2444
2394
|
const identifiers = {
|
|
2445
2395
|
projectId: entry?.project?.id ?? null,
|
|
2446
2396
|
environmentId: entry?.environment?.id ?? null,
|
|
2447
2397
|
serviceId: entry?.service?.id ?? null
|
|
2448
2398
|
};
|
|
2449
2399
|
if (!identifiers.projectId || !identifiers.environmentId || !identifiers.serviceId) {
|
|
2450
|
-
return
|
|
2400
|
+
return {
|
|
2401
|
+
domain: null,
|
|
2402
|
+
serviceId: null,
|
|
2403
|
+
serviceName: entry?.configuredService?.serviceName ?? null
|
|
2404
|
+
};
|
|
2451
2405
|
}
|
|
2452
2406
|
const [customDomains, serviceDomains] = await Promise.all([
|
|
2453
2407
|
listRailwayCustomDomains({
|
|
@@ -2470,9 +2424,15 @@ async function observeRailwayCustomDomainLive(input, domain) {
|
|
|
2470
2424
|
const serviceDomain = serviceDomains.find((entry2) => entry2.kind === "service") ?? null;
|
|
2471
2425
|
const normalized = normalizeRailwayDomainPayload({
|
|
2472
2426
|
...customDomain,
|
|
2427
|
+
serviceId: identifiers.serviceId,
|
|
2428
|
+
serviceName: entry?.configuredService?.serviceName ?? null,
|
|
2473
2429
|
serviceDomain: serviceDomain?.domain ?? null
|
|
2474
2430
|
});
|
|
2475
|
-
return normalized?.domain ? normalized :
|
|
2431
|
+
return normalized?.domain ? normalized : {
|
|
2432
|
+
...customDomain,
|
|
2433
|
+
serviceId: identifiers.serviceId,
|
|
2434
|
+
serviceName: entry?.configuredService?.serviceName ?? null
|
|
2435
|
+
};
|
|
2476
2436
|
}
|
|
2477
2437
|
function railwayCustomDomainHasDnsRequirements(live) {
|
|
2478
2438
|
if (!live) {
|
|
@@ -3420,7 +3380,6 @@ async function ensureRailwayEnvironmentForService({
|
|
|
3420
3380
|
}
|
|
3421
3381
|
async function resolveRailwayTopologyForScope(input, scope, {
|
|
3422
3382
|
ensure = false,
|
|
3423
|
-
ensureServices = ensure,
|
|
3424
3383
|
refresh = false,
|
|
3425
3384
|
serviceKeys,
|
|
3426
3385
|
includeInstances = ensure,
|
|
@@ -3548,71 +3507,14 @@ async function resolveRailwayTopologyForScope(input, scope, {
|
|
|
3548
3507
|
traceRailwayReconcile(env, "topology:environment-services:resolved", `${service.key}:${environmentServices.map((entry) => entry.name).join(",")}`);
|
|
3549
3508
|
}
|
|
3550
3509
|
}
|
|
3551
|
-
|
|
3552
|
-
if (project && !resolvedService && ensureServices) {
|
|
3553
|
-
traceRailwayReconcile(env, "topology:railway-service:ensure", `${service.key}:${resolvedServiceName || resolvedServiceId}`);
|
|
3554
|
-
resolvedService = (await ensureRailwayService({
|
|
3555
|
-
projectId: project.id,
|
|
3556
|
-
serviceId: resolvedServiceId,
|
|
3557
|
-
serviceName: resolvedServiceName,
|
|
3558
|
-
environmentId: environment?.id,
|
|
3559
|
-
imageRef: service.imageRef,
|
|
3560
|
-
sourceRepo: service.sourceRepo,
|
|
3561
|
-
sourceBranch: service.sourceBranch,
|
|
3562
|
-
env
|
|
3563
|
-
})).service;
|
|
3564
|
-
project = {
|
|
3565
|
-
...project,
|
|
3566
|
-
services: [...project.services.filter((entry) => entry.id !== resolvedService?.id), resolvedService]
|
|
3567
|
-
};
|
|
3568
|
-
projectsByKey.set(project.id, project);
|
|
3569
|
-
projectsByKey.set(project.name, project);
|
|
3570
|
-
} else if (project && resolvedService && ensureServices && (service.imageRef || service.sourceRepo)) {
|
|
3571
|
-
traceRailwayReconcile(env, "topology:railway-service:repair-source", `${service.key}:${resolvedService.name}`);
|
|
3572
|
-
try {
|
|
3573
|
-
resolvedService = (await ensureRailwayService({
|
|
3574
|
-
projectId: project.id,
|
|
3575
|
-
serviceId: resolvedService.id,
|
|
3576
|
-
serviceName: resolvedServiceName || resolvedService.name,
|
|
3577
|
-
environmentId: environment?.id,
|
|
3578
|
-
imageRef: service.imageRef,
|
|
3579
|
-
sourceRepo: service.sourceRepo,
|
|
3580
|
-
sourceBranch: service.sourceBranch,
|
|
3581
|
-
env
|
|
3582
|
-
})).service;
|
|
3583
|
-
} catch (error) {
|
|
3584
|
-
if (!looksLikeRailwayExistingSourceRepairUnsupported(error)) {
|
|
3585
|
-
throw error;
|
|
3586
|
-
}
|
|
3587
|
-
traceRailwayReconcile(env, "topology:railway-service:repair-source-skipped", `${service.key}:${resolvedService.name}`);
|
|
3588
|
-
}
|
|
3589
|
-
}
|
|
3510
|
+
const resolvedService = project?.services.find((entry) => entry.id === resolvedServiceId || entry.name === resolvedServiceName) ?? null;
|
|
3590
3511
|
let instance = null;
|
|
3591
3512
|
if (includeInstances && resolvedService && environment) {
|
|
3592
|
-
|
|
3593
|
-
|
|
3594
|
-
|
|
3595
|
-
|
|
3596
|
-
|
|
3597
|
-
buildCommand: service.buildCommand,
|
|
3598
|
-
dockerfilePath: !service.imageRef ? service.dockerfilePath ?? (existsSync(resolve(service.rootDir, "Dockerfile")) ? "/Dockerfile" : null) : null,
|
|
3599
|
-
startCommand: service.startCommand,
|
|
3600
|
-
rootDirectory: service.imageRef || service.sourceMode === "image" ? null : railwayServiceRootDirectory(input.context.tenantRoot, service),
|
|
3601
|
-
healthcheckPath: service.healthcheckPath,
|
|
3602
|
-
healthcheckTimeoutSeconds: service.healthcheckTimeoutSeconds,
|
|
3603
|
-
healthcheckIntervalSeconds: service.healthcheckIntervalSeconds,
|
|
3604
|
-
restartPolicy: service.restartPolicy,
|
|
3605
|
-
runtimeMode: service.runtimeMode,
|
|
3606
|
-
clearSourceConfiguration: service.sourceMode === "image" || Boolean(service.imageRef),
|
|
3607
|
-
env
|
|
3608
|
-
})).instance;
|
|
3609
|
-
} else {
|
|
3610
|
-
instance = await getRailwayServiceInstance({
|
|
3611
|
-
serviceId: resolvedService.id,
|
|
3612
|
-
environmentId: environment.id,
|
|
3613
|
-
env
|
|
3614
|
-
});
|
|
3615
|
-
}
|
|
3513
|
+
instance = await getRailwayServiceInstance({
|
|
3514
|
+
serviceId: resolvedService.id,
|
|
3515
|
+
environmentId: environment.id,
|
|
3516
|
+
env
|
|
3517
|
+
});
|
|
3616
3518
|
}
|
|
3617
3519
|
const currentVariables = includeVariables && project && environment && resolvedService ? await listRailwayVariables({
|
|
3618
3520
|
projectId: project.id,
|
|
@@ -3620,7 +3522,7 @@ async function resolveRailwayTopologyForScope(input, scope, {
|
|
|
3620
3522
|
serviceId: resolvedService.id,
|
|
3621
3523
|
env
|
|
3622
3524
|
}) : {};
|
|
3623
|
-
resolvedServices.set(service.key, {
|
|
3525
|
+
resolvedServices.set(service.instanceKey ?? service.serviceName ?? service.key, {
|
|
3624
3526
|
configuredService: service,
|
|
3625
3527
|
project,
|
|
3626
3528
|
environment,
|
|
@@ -3710,6 +3612,16 @@ function configuredRailwaySiblingResourceNames(input, scope, projectName) {
|
|
|
3710
3612
|
resolveReconcileEnvironmentValues(input, siblingScope),
|
|
3711
3613
|
{ identityOnly: true }
|
|
3712
3614
|
).filter((service) => service.enabled !== false).filter((service) => !isRailwayCapacityProviderService(service)).filter((service) => service.projectName === projectName);
|
|
3615
|
+
const currentServices = configuredRailwayServices(
|
|
3616
|
+
input.context.tenantRoot,
|
|
3617
|
+
scope,
|
|
3618
|
+
resolveReconcileEnvironmentValues(input, scope),
|
|
3619
|
+
{ identityOnly: true }
|
|
3620
|
+
).filter((service) => service.enabled !== false).filter((service) => !isRailwayCapacityProviderService(service)).filter((service) => service.projectName === projectName);
|
|
3621
|
+
assertNoRailwaySourceIdentityCollisions([
|
|
3622
|
+
...currentServices.map((service) => ({ ...service, environment: scope })),
|
|
3623
|
+
...siblingServices.map((service) => ({ ...service, environment: siblingScope }))
|
|
3624
|
+
]);
|
|
3713
3625
|
const database = configuredRailwayIacDatabase(input, siblingServices);
|
|
3714
3626
|
return [.../* @__PURE__ */ new Set([
|
|
3715
3627
|
...siblingServices.flatMap((service) => [
|
|
@@ -3736,6 +3648,14 @@ function railwayIacServiceInput(input, sync, service, scope) {
|
|
|
3736
3648
|
...service.sourceBranch ? { [`${deployVariablePrefix}_DEPLOY_SOURCE_BRANCH`]: service.sourceBranch } : {},
|
|
3737
3649
|
...service.sourceCommit ? { [`${deployVariablePrefix}_DEPLOY_SOURCE_COMMIT`]: service.sourceCommit } : {}
|
|
3738
3650
|
} : {};
|
|
3651
|
+
let customDomain = null;
|
|
3652
|
+
if (service.key === "api" && service.publicBaseUrl) {
|
|
3653
|
+
try {
|
|
3654
|
+
customDomain = new URL(service.publicBaseUrl).hostname || null;
|
|
3655
|
+
} catch {
|
|
3656
|
+
customDomain = null;
|
|
3657
|
+
}
|
|
3658
|
+
}
|
|
3739
3659
|
return {
|
|
3740
3660
|
key: service.instanceKey ?? service.key,
|
|
3741
3661
|
serviceName: service.serviceName,
|
|
@@ -3752,6 +3672,7 @@ function railwayIacServiceInput(input, sync, service, scope) {
|
|
|
3752
3672
|
healthcheckTimeoutSeconds: service.healthcheckTimeoutSeconds,
|
|
3753
3673
|
runtimeMode: service.runtimeMode,
|
|
3754
3674
|
volumeMountPath: service.volumeMountPath,
|
|
3675
|
+
customDomains: customDomain ? [customDomain] : [],
|
|
3755
3676
|
variables: {
|
|
3756
3677
|
...serviceSync.variables,
|
|
3757
3678
|
...sourceVariables
|
|
@@ -3791,26 +3712,6 @@ function activeAttachedRailwayVolumeIds(volumes, serviceId, environmentId, desir
|
|
|
3791
3712
|
(instance) => instance.serviceId === serviceId && instance.environmentId === environmentId && (volume.name !== desiredVolumeName || activeRailwayVolumeInstances({ instances: [instance] }).length === 0)
|
|
3792
3713
|
)).map((volume) => volume.id);
|
|
3793
3714
|
}
|
|
3794
|
-
function blockedPendingRailwayVolumeAttachments(volumes, serviceIdByName, environmentId) {
|
|
3795
|
-
const serviceNameById = new Map([...serviceIdByName.entries()].filter(([, serviceId]) => serviceId).map(([serviceName, serviceId]) => [serviceId, serviceName]));
|
|
3796
|
-
const blocked = [];
|
|
3797
|
-
for (const volume of volumes) {
|
|
3798
|
-
for (const instance of volume.instances) {
|
|
3799
|
-
if (!instance.serviceId || instance.environmentId !== environmentId) {
|
|
3800
|
-
continue;
|
|
3801
|
-
}
|
|
3802
|
-
const serviceName = serviceNameById.get(instance.serviceId);
|
|
3803
|
-
if (!serviceName) {
|
|
3804
|
-
continue;
|
|
3805
|
-
}
|
|
3806
|
-
if (activeRailwayVolumeInstances({ instances: [instance] }).length > 0) {
|
|
3807
|
-
continue;
|
|
3808
|
-
}
|
|
3809
|
-
blocked.push(`${serviceName}: volume ${volume.name ?? volume.id} (${volume.id}) is pending/deleted while still attached to service ${instance.serviceId} at ${instance.mountPath ?? "(unknown mount)"}; deletedAt=${instance.deletedAt ?? "(none)"} state=${instance.state ?? "(unknown)"}`);
|
|
3810
|
-
}
|
|
3811
|
-
}
|
|
3812
|
-
return blocked;
|
|
3813
|
-
}
|
|
3814
3715
|
async function reconcileStaleOperationsRunnerResourcesForScope(input, topology) {
|
|
3815
3716
|
const scope = input.context.target.kind === "persistent" ? input.context.target.scope : "staging";
|
|
3816
3717
|
const desiredRunners = configuredRailwayServicesForInput(input, scope).filter((service) => service.key === "operationsRunner").filter((service) => service.enabled !== false);
|
|
@@ -3916,11 +3817,11 @@ async function syncRailwayEnvironmentForScope(input, { planOnly = false, service
|
|
|
3916
3817
|
}
|
|
3917
3818
|
let workspace = "";
|
|
3918
3819
|
const syncedServices = [];
|
|
3820
|
+
const projectPlans = [];
|
|
3919
3821
|
for (const projectServices of projectGroups) {
|
|
3920
3822
|
const serviceKeySet = [...new Set(projectServices.map((service) => service.key))];
|
|
3921
3823
|
const topology = await resolveRailwayTopologyForScope(input, scope, {
|
|
3922
3824
|
ensure: !planOnly,
|
|
3923
|
-
ensureServices: false,
|
|
3924
3825
|
serviceKeys: serviceKeySet,
|
|
3925
3826
|
includeInstances: false,
|
|
3926
3827
|
includeVariables: false,
|
|
@@ -3934,46 +3835,29 @@ async function syncRailwayEnvironmentForScope(input, { planOnly = false, service
|
|
|
3934
3835
|
for (const service of projectServices) {
|
|
3935
3836
|
railwayIacServiceInput(input, sync, service, scope);
|
|
3936
3837
|
}
|
|
3937
|
-
if (planOnly) {
|
|
3938
|
-
syncedServices.push(...projectServices);
|
|
3939
|
-
continue;
|
|
3940
|
-
}
|
|
3941
3838
|
if (!project || !environment) {
|
|
3942
|
-
throw new Error(`Railway IaC reconciliation could not resolve project/environment for ${projectServices[0]?.projectName ?? "(unknown project)"}/${projectServices[0]?.railwayEnvironment ?? "(unknown environment)"}.`);
|
|
3839
|
+
throw new Error(`Railway IaC ${planOnly ? "plan" : "reconciliation"} could not resolve project/environment for ${projectServices[0]?.projectName ?? "(unknown project)"}/${projectServices[0]?.railwayEnvironment ?? "(unknown environment)"}.`);
|
|
3943
3840
|
}
|
|
3944
3841
|
const liveServices = await listRailwayEnvironmentServices({ environmentId: environment.id, env: topology.env }).catch(() => project.services);
|
|
3945
3842
|
const liveServiceByName = new Map(liveServices.map((service) => [service.name, service]));
|
|
3946
3843
|
const liveVolumes = await listRailwayVolumes({ projectId: project.id, env: topology.env }).catch(() => []);
|
|
3947
3844
|
const databaseDescriptor = configuredRailwayIacDatabase(input, projectServices);
|
|
3948
|
-
const
|
|
3949
|
-
|
|
3950
|
-
|
|
3951
|
-
|
|
3952
|
-
|
|
3953
|
-
|
|
3954
|
-
|
|
3955
|
-
|
|
3956
|
-
|
|
3957
|
-
|
|
3958
|
-
|
|
3959
|
-
|
|
3960
|
-
|
|
3961
|
-
|
|
3962
|
-
status: "blocked",
|
|
3963
|
-
projectId: project.id,
|
|
3964
|
-
environmentId: environment.id,
|
|
3965
|
-
reason
|
|
3966
|
-
});
|
|
3967
|
-
}
|
|
3968
|
-
throw new Error(`Railway has pending-deletion volume attachments that block official IaC reconciliation. ${pendingVolumeBlocks.join("; ")}`);
|
|
3845
|
+
const baseIacServices = projectServices.map((service) => railwayIacServiceInput(input, sync, service, scope));
|
|
3846
|
+
const pendingVolumeNameCollisions = findRailwayPendingVolumeNameCollisions({
|
|
3847
|
+
services: baseIacServices,
|
|
3848
|
+
liveServices,
|
|
3849
|
+
volumes: liveVolumes
|
|
3850
|
+
});
|
|
3851
|
+
const volumeBindingResult = resolveRailwayIacVolumeBindings({
|
|
3852
|
+
environmentId: environment.id,
|
|
3853
|
+
services: baseIacServices,
|
|
3854
|
+
liveServices,
|
|
3855
|
+
volumes: liveVolumes
|
|
3856
|
+
});
|
|
3857
|
+
if (volumeBindingResult.blockedReasons.length > 0) {
|
|
3858
|
+
throw new Error(`Railway volume lineage blocks reconciliation: ${volumeBindingResult.blockedReasons.join("; ")}`);
|
|
3969
3859
|
}
|
|
3970
|
-
const
|
|
3971
|
-
const liveService = liveServiceByName.get(service.serviceName) ?? project.services.find((candidate) => candidate.name === service.serviceName || candidate.id === service.serviceId) ?? null;
|
|
3972
|
-
return [
|
|
3973
|
-
service.serviceName,
|
|
3974
|
-
activeAttachedRailwayVolumeIds(liveVolumes, liveService?.id, environment.id, `${service.serviceName}-volume`)
|
|
3975
|
-
];
|
|
3976
|
-
}));
|
|
3860
|
+
const volumeBindingByServiceName = new Map(volumeBindingResult.bindings.map((binding) => [binding.serviceName, binding]));
|
|
3977
3861
|
const databaseLiveService = databaseDescriptor ? liveServiceByName.get(databaseDescriptor.serviceName) ?? project.services.find((candidate) => candidate.name === databaseDescriptor.serviceName) ?? null : null;
|
|
3978
3862
|
const databaseDetachVolumeIds = databaseDescriptor ? activeAttachedRailwayVolumeIds(liveVolumes, databaseLiveService?.id, environment.id, `${databaseDescriptor.serviceName}-volume`) : [];
|
|
3979
3863
|
const databaseForIac = databaseDescriptor ? { ...databaseDescriptor, detachVolumeIds: databaseDetachVolumeIds, useNativePostgres: false } : null;
|
|
@@ -3990,21 +3874,46 @@ async function syncRailwayEnvironmentForScope(input, { planOnly = false, service
|
|
|
3990
3874
|
environmentId: environment.id,
|
|
3991
3875
|
railwayApiToken: token,
|
|
3992
3876
|
railwayApiUrl: String(topology.env.TREESEED_RAILWAY_API_URL ?? topology.env.RAILWAY_API_URL ?? "").trim() || null,
|
|
3993
|
-
services:
|
|
3994
|
-
...
|
|
3995
|
-
|
|
3877
|
+
services: baseIacServices.map((service) => ({
|
|
3878
|
+
...service,
|
|
3879
|
+
volumeName: volumeBindingByServiceName.get(service.serviceName)?.canonicalVolumeName ?? null,
|
|
3880
|
+
volumeAddress: (() => {
|
|
3881
|
+
const binding = volumeBindingByServiceName.get(service.serviceName);
|
|
3882
|
+
return binding && binding.volumeName !== binding.canonicalVolumeName ? `volume.${binding.volumeName}` : null;
|
|
3883
|
+
})()
|
|
3996
3884
|
})),
|
|
3997
3885
|
database: databaseForIac
|
|
3998
3886
|
};
|
|
3999
|
-
const
|
|
4000
|
-
|
|
4001
|
-
|
|
4002
|
-
|
|
4003
|
-
|
|
4004
|
-
|
|
4005
|
-
|
|
4006
|
-
|
|
4007
|
-
|
|
3887
|
+
const projectEnvironmentServices = await Promise.all(project.environments.map(async (candidate) => ({
|
|
3888
|
+
environment: candidate,
|
|
3889
|
+
services: await listRailwayEnvironmentServices({ environmentId: candidate.id, env: topology.env }).catch(() => [])
|
|
3890
|
+
})));
|
|
3891
|
+
const activeProjectEnvironmentServices = projectEnvironmentServices.flatMap((entry) => entry.services);
|
|
3892
|
+
const siblingEnvironmentName = scope === "prod" ? "staging" : scope === "staging" ? "production" : null;
|
|
3893
|
+
const siblingEnvironmentIds = new Set(project.environments.filter((candidate) => siblingEnvironmentName && candidate.name === siblingEnvironmentName).map((candidate) => candidate.id));
|
|
3894
|
+
const activeSiblingServices = projectEnvironmentServices.filter((entry) => siblingEnvironmentIds.has(entry.environment.id)).flatMap((entry) => entry.services);
|
|
3895
|
+
const activeSiblingServiceIds = new Set(activeSiblingServices.map((service) => service.id));
|
|
3896
|
+
const activeSiblingResourceNames = [
|
|
3897
|
+
...activeSiblingServices.map((service) => service.name),
|
|
3898
|
+
...liveVolumes.filter((volume) => activeRailwayVolumeInstances(volume).some((instance) => siblingEnvironmentIds.has(instance.environmentId ?? "") || activeSiblingServiceIds.has(instance.serviceId ?? ""))).map((volume) => volume.name).filter((name) => Boolean(name))
|
|
3899
|
+
];
|
|
3900
|
+
const aliasCleanup = railwayObsoleteAliasCleanupPolicy(
|
|
3901
|
+
scope,
|
|
3902
|
+
projectServices,
|
|
3903
|
+
project.services.map((service) => service.name),
|
|
3904
|
+
activeProjectEnvironmentServices.map((service) => service.name)
|
|
3905
|
+
);
|
|
3906
|
+
const protectedSiblingResourceNames = [.../* @__PURE__ */ new Set([
|
|
3907
|
+
...configuredRailwaySiblingResourceNames(input, scope, project.name),
|
|
3908
|
+
...activeSiblingResourceNames
|
|
3909
|
+
])];
|
|
3910
|
+
const siblingResourceNames = [
|
|
3911
|
+
...protectedSiblingResourceNames,
|
|
3912
|
+
...aliasCleanup.retainedResourceNames
|
|
3913
|
+
];
|
|
3914
|
+
const allowedResourceDeletions = [
|
|
3915
|
+
...aliasCleanup.allowedResourceDeletions.filter((name) => !protectedSiblingResourceNames.includes(name))
|
|
3916
|
+
];
|
|
4008
3917
|
let effectiveIacInput = iacInput;
|
|
4009
3918
|
let rendered = renderRailwayIacProject(effectiveIacInput);
|
|
4010
3919
|
try {
|
|
@@ -4023,7 +3932,9 @@ async function syncRailwayEnvironmentForScope(input, { planOnly = false, service
|
|
|
4023
3932
|
serviceSourceRefs: Object.fromEntries(effectiveIacInput.services.map((service) => [
|
|
4024
3933
|
service.serviceName,
|
|
4025
3934
|
service.sourceMode === "git" && service.sourceRepo ? `github:${service.sourceRepo}:${service.sourceBranch ?? ""}:${service.sourceRootDirectory ?? ""}:${service.sourceCommit ?? ""}` : service.sourceMode === "image" && service.imageRef ? `image:${service.imageRef}` : null
|
|
4026
|
-
]))
|
|
3935
|
+
])),
|
|
3936
|
+
allowedResourceDeletions,
|
|
3937
|
+
protectedResourceNames: protectedSiblingResourceNames
|
|
4027
3938
|
});
|
|
4028
3939
|
if (!validation.ok && effectiveIacInput.database && !effectiveIacInput.database.useNativePostgres && railwayIacPlanDeletesResource(plan.changeSet, effectiveIacInput.database.serviceName)) {
|
|
4029
3940
|
traceRailwayReconcile(topology.env, "sync:iac-native-postgres-adopt", `${project.name}/${environment.name}:${effectiveIacInput.database.serviceName}`);
|
|
@@ -4050,10 +3961,18 @@ async function syncRailwayEnvironmentForScope(input, { planOnly = false, service
|
|
|
4050
3961
|
serviceSourceRefs: Object.fromEntries(effectiveIacInput.services.map((service) => [
|
|
4051
3962
|
service.serviceName,
|
|
4052
3963
|
service.sourceMode === "git" && service.sourceRepo ? `github:${service.sourceRepo}:${service.sourceBranch ?? ""}:${service.sourceRootDirectory ?? ""}:${service.sourceCommit ?? ""}` : service.sourceMode === "image" && service.imageRef ? `image:${service.imageRef}` : null
|
|
4053
|
-
]))
|
|
3964
|
+
])),
|
|
3965
|
+
allowedResourceDeletions,
|
|
3966
|
+
protectedResourceNames: protectedSiblingResourceNames
|
|
4054
3967
|
});
|
|
4055
3968
|
}
|
|
4056
|
-
const retainedResources =
|
|
3969
|
+
const retainedResources = detachRetainedRailwayCustomDomains(
|
|
3970
|
+
detachRetainedRailwayVolumeBindings(
|
|
3971
|
+
selectRailwayIacRetainedResources(plan, siblingResourceNames),
|
|
3972
|
+
volumeBindingResult.bindings
|
|
3973
|
+
),
|
|
3974
|
+
effectiveIacInput.services.flatMap((service) => service.customDomains ?? [])
|
|
3975
|
+
);
|
|
4057
3976
|
if (retainedResources.length > 0) {
|
|
4058
3977
|
traceRailwayReconcile(
|
|
4059
3978
|
topology.env,
|
|
@@ -4077,16 +3996,220 @@ async function syncRailwayEnvironmentForScope(input, { planOnly = false, service
|
|
|
4077
3996
|
serviceSourceRefs: Object.fromEntries(effectiveIacInput.services.map((service) => [
|
|
4078
3997
|
service.serviceName,
|
|
4079
3998
|
service.sourceMode === "git" && service.sourceRepo ? `github:${service.sourceRepo}:${service.sourceBranch ?? ""}:${service.sourceRootDirectory ?? ""}:${service.sourceCommit ?? ""}` : service.sourceMode === "image" && service.imageRef ? `image:${service.imageRef}` : null
|
|
4080
|
-
]))
|
|
3999
|
+
])),
|
|
4000
|
+
allowedResourceDeletions,
|
|
4001
|
+
protectedResourceNames: protectedSiblingResourceNames
|
|
4081
4002
|
});
|
|
4082
4003
|
}
|
|
4083
4004
|
if (!validation.ok) {
|
|
4084
4005
|
throw new Error(`Railway IaC plan rejected for ${project.name}/${environment.name}: ${validation.blockedReasons.join("; ")}`);
|
|
4085
4006
|
}
|
|
4007
|
+
projectPlans.push({
|
|
4008
|
+
projectName: project.name,
|
|
4009
|
+
environmentName: environment.name,
|
|
4010
|
+
changes: (plan.changeSet?.changes ?? []).map((change) => ({
|
|
4011
|
+
kind: String(change.kind ?? ""),
|
|
4012
|
+
path: String(change.path ?? change.address ?? ""),
|
|
4013
|
+
summary: String(change.summary ?? "")
|
|
4014
|
+
})),
|
|
4015
|
+
volumeBindings: volumeBindingResult.bindings
|
|
4016
|
+
});
|
|
4017
|
+
if (planOnly) {
|
|
4018
|
+
syncedServices.push(...projectServices);
|
|
4019
|
+
continue;
|
|
4020
|
+
}
|
|
4021
|
+
for (const collision of pendingVolumeNameCollisions) {
|
|
4022
|
+
const serviceId = liveServiceByName.get(collision.serviceName)?.id ?? collision.serviceId ?? liveServices[0]?.id;
|
|
4023
|
+
if (!serviceId) {
|
|
4024
|
+
throw new Error(`Railway pending volume ${collision.canonicalVolumeName} cannot be released because no environment service is available for CLI scope.`);
|
|
4025
|
+
}
|
|
4026
|
+
const tombstoneName = `pending-delete-${collision.volumeId.slice(0, 8)}`;
|
|
4027
|
+
traceRailwayReconcile(topology.env, "sync:volume:release-pending-name", `${collision.canonicalVolumeName}:${tombstoneName}`);
|
|
4028
|
+
await updateRailwayVolumeWithCli({
|
|
4029
|
+
projectId: project.id,
|
|
4030
|
+
environmentId: environment.id,
|
|
4031
|
+
serviceId,
|
|
4032
|
+
volumeId: collision.volumeId,
|
|
4033
|
+
name: tombstoneName,
|
|
4034
|
+
mountPath: collision.mountPath,
|
|
4035
|
+
env: topology.env
|
|
4036
|
+
});
|
|
4037
|
+
const released = await waitForRailwayVolumeName({
|
|
4038
|
+
volumeId: collision.volumeId,
|
|
4039
|
+
expectedName: tombstoneName,
|
|
4040
|
+
load: () => listRailwayVolumes({ projectId: project.id, env: topology.env }),
|
|
4041
|
+
sleep: async (milliseconds) => {
|
|
4042
|
+
traceRailwayReconcile(topology.env, "sync:volume:release-pending-settle", `${collision.canonicalVolumeName}:${milliseconds}ms`);
|
|
4043
|
+
await new Promise((resolve2) => setTimeout(resolve2, milliseconds));
|
|
4044
|
+
}
|
|
4045
|
+
});
|
|
4046
|
+
if (!released) {
|
|
4047
|
+
throw new Error(`Railway pending volume name ${collision.canonicalVolumeName} remained locked after 12 observations.`);
|
|
4048
|
+
}
|
|
4049
|
+
const desiredServiceId = liveServiceByName.get(collision.serviceName)?.id ?? null;
|
|
4050
|
+
if (desiredServiceId === serviceId && collision.serviceId === desiredServiceId) {
|
|
4051
|
+
traceRailwayReconcile(topology.env, "sync:volume:replace-pending-service", `${collision.serviceName}:${serviceId}`);
|
|
4052
|
+
await deleteRailwayService({
|
|
4053
|
+
projectId: project.id,
|
|
4054
|
+
environmentId: environment.id,
|
|
4055
|
+
serviceId,
|
|
4056
|
+
env: topology.env
|
|
4057
|
+
});
|
|
4058
|
+
const absent = await waitForRailwayServiceAbsence({
|
|
4059
|
+
serviceId,
|
|
4060
|
+
load: () => listRailwayEnvironmentServices({ environmentId: environment.id, env: topology.env }),
|
|
4061
|
+
sleep: async (milliseconds) => {
|
|
4062
|
+
traceRailwayReconcile(topology.env, "sync:volume:replace-pending-service-settle", `${collision.serviceName}:${milliseconds}ms`);
|
|
4063
|
+
await new Promise((resolve2) => setTimeout(resolve2, milliseconds));
|
|
4064
|
+
}
|
|
4065
|
+
});
|
|
4066
|
+
if (!absent) throw new Error(`Railway service ${collision.serviceName} remained after pending-volume replacement was requested.`);
|
|
4067
|
+
} else {
|
|
4068
|
+
traceRailwayReconcile(topology.env, "sync:volume:release-pending-detach", `${tombstoneName}:${serviceId}`);
|
|
4069
|
+
await detachRailwayVolumeWithCli({
|
|
4070
|
+
projectId: project.id,
|
|
4071
|
+
environmentId: environment.id,
|
|
4072
|
+
serviceId,
|
|
4073
|
+
volumeId: collision.volumeId,
|
|
4074
|
+
env: topology.env
|
|
4075
|
+
});
|
|
4076
|
+
const detached = await waitForRailwayVolumeDetachment({
|
|
4077
|
+
volumeId: collision.volumeId,
|
|
4078
|
+
environmentId: environment.id,
|
|
4079
|
+
serviceId,
|
|
4080
|
+
load: () => listRailwayVolumes({ projectId: project.id, env: topology.env }),
|
|
4081
|
+
sleep: async (milliseconds) => {
|
|
4082
|
+
traceRailwayReconcile(topology.env, "sync:volume:release-pending-detach-settle", `${tombstoneName}:${milliseconds}ms`);
|
|
4083
|
+
await new Promise((resolve2) => setTimeout(resolve2, milliseconds));
|
|
4084
|
+
}
|
|
4085
|
+
});
|
|
4086
|
+
if (!detached) throw new Error(`Railway pending volume ${tombstoneName} remained attached after 12 observations.`);
|
|
4087
|
+
}
|
|
4088
|
+
}
|
|
4089
|
+
if (pendingVolumeNameCollisions.length > 0) {
|
|
4090
|
+
cleanupRailwayIacRender(rendered);
|
|
4091
|
+
rendered = renderRailwayIacProject(effectiveIacInput);
|
|
4092
|
+
plan = await planRailwayIacProject(effectiveIacInput, rendered);
|
|
4093
|
+
if (!plan.ok) {
|
|
4094
|
+
const diagnostics = plan.diagnostics.map((entry) => entry.message).filter(Boolean).join("; ");
|
|
4095
|
+
throw new Error(`Railway IaC recovery replan failed for ${project.name}/${environment.name}${diagnostics ? `: ${diagnostics}` : "."}`);
|
|
4096
|
+
}
|
|
4097
|
+
validation = validateRailwayIacChangeSet(plan.changeSet, {
|
|
4098
|
+
services: rendered.serviceNames,
|
|
4099
|
+
volumes: rendered.volumeNames,
|
|
4100
|
+
database: rendered.databaseName,
|
|
4101
|
+
scope,
|
|
4102
|
+
serviceSourceModes: Object.fromEntries(effectiveIacInput.services.map((service) => [service.serviceName, service.sourceMode ?? null])),
|
|
4103
|
+
serviceSourceRefs: Object.fromEntries(effectiveIacInput.services.map((service) => [service.serviceName, service.sourceMode === "image" ? `image:${service.imageRef ?? ""}` : service.sourceMode === "git" ? `github:${service.sourceRepo ?? ""}:${service.sourceBranch ?? ""}:${service.sourceRootDirectory ?? ""}:${service.sourceCommit ?? ""}` : null])),
|
|
4104
|
+
allowedResourceDeletions,
|
|
4105
|
+
protectedResourceNames: protectedSiblingResourceNames
|
|
4106
|
+
});
|
|
4107
|
+
if (!validation.ok) throw new Error(`Railway IaC recovery plan rejected for ${project.name}/${environment.name}: ${validation.blockedReasons.join("; ")}`);
|
|
4108
|
+
}
|
|
4086
4109
|
const apply = await applyRailwayIacProject(effectiveIacInput, rendered);
|
|
4087
|
-
|
|
4088
|
-
|
|
4089
|
-
|
|
4110
|
+
const applyFailure = railwayIacApplyFailure(apply);
|
|
4111
|
+
if (applyFailure) throw new Error(`Railway IaC apply failed for ${project.name}/${environment.name}: ${applyFailure}`);
|
|
4112
|
+
for (const binding of volumeBindingResult.bindings) {
|
|
4113
|
+
const desiredConfig = effectiveIacInput.services.find((service) => service.serviceName === binding.serviceName);
|
|
4114
|
+
if (!desiredConfig?.volumeMountPath) {
|
|
4115
|
+
throw new Error(`Railway volume adoption could not resolve ${binding.canonicalVolumeName} and its desired service.`);
|
|
4116
|
+
}
|
|
4117
|
+
const settled = await waitForRailwayVolumeAdoptionResources({
|
|
4118
|
+
serviceName: binding.serviceName,
|
|
4119
|
+
volumeId: binding.volumeId,
|
|
4120
|
+
load: async () => ({
|
|
4121
|
+
services: await listRailwayEnvironmentServices({ environmentId: environment.id, env: topology.env }),
|
|
4122
|
+
volumes: await listRailwayVolumes({ projectId: project.id, env: topology.env })
|
|
4123
|
+
}),
|
|
4124
|
+
sleep: async (milliseconds) => {
|
|
4125
|
+
traceRailwayReconcile(topology.env, "sync:volume:settle", `${binding.canonicalVolumeName}:${milliseconds}ms`);
|
|
4126
|
+
await new Promise((resolve2) => setTimeout(resolve2, milliseconds));
|
|
4127
|
+
}
|
|
4128
|
+
});
|
|
4129
|
+
if (!settled) {
|
|
4130
|
+
throw new Error(`Railway volume adoption could not resolve ${binding.canonicalVolumeName} and its desired service after 12 observations.`);
|
|
4131
|
+
}
|
|
4132
|
+
const desiredService = settled.service;
|
|
4133
|
+
const liveVolume = settled.volume;
|
|
4134
|
+
const volumesAfterApply = settled.volumes;
|
|
4135
|
+
const activeEnvironmentAttachments = liveVolume.instances.filter(
|
|
4136
|
+
(instance) => instance.environmentId === environment.id && !instance.isPendingDeletion && !(typeof instance.deletedAt === "string" && instance.deletedAt.trim()) && !["DELETING", "DELETED"].includes(String(instance.state ?? "").toUpperCase())
|
|
4137
|
+
);
|
|
4138
|
+
const nameCollisions = volumesAfterApply.filter(
|
|
4139
|
+
(volume) => volume.id !== liveVolume.id && volume.name === binding.canonicalVolumeName
|
|
4140
|
+
);
|
|
4141
|
+
for (const collision of nameCollisions) {
|
|
4142
|
+
const pendingDeletion = collision.instances.length > 0 && collision.instances.every(
|
|
4143
|
+
(instance) => instance.isPendingDeletion === true || Boolean(typeof instance.deletedAt === "string" && instance.deletedAt.trim()) || ["DELETING", "DELETED"].includes(String(instance.state ?? "").toUpperCase())
|
|
4144
|
+
);
|
|
4145
|
+
if (!pendingDeletion) {
|
|
4146
|
+
throw new Error(`Railway volume name ${binding.canonicalVolumeName} is owned by active volume ${collision.id}; refusing ambiguous adoption.`);
|
|
4147
|
+
}
|
|
4148
|
+
const tombstoneName = `pending-delete-${collision.id.slice(0, 8)}`;
|
|
4149
|
+
traceRailwayReconcile(topology.env, "sync:volume:rename-tombstone", `${binding.canonicalVolumeName}:${tombstoneName}`);
|
|
4150
|
+
await updateRailwayVolumeWithCli({
|
|
4151
|
+
projectId: project.id,
|
|
4152
|
+
environmentId: environment.id,
|
|
4153
|
+
serviceId: desiredService.id,
|
|
4154
|
+
volumeId: collision.id,
|
|
4155
|
+
name: tombstoneName,
|
|
4156
|
+
mountPath: collision.instances[0]?.mountPath || desiredConfig.volumeMountPath,
|
|
4157
|
+
env: topology.env
|
|
4158
|
+
});
|
|
4159
|
+
}
|
|
4160
|
+
for (const attachment of activeEnvironmentAttachments.filter((instance) => instance.serviceId && instance.serviceId !== desiredService.id)) {
|
|
4161
|
+
traceRailwayReconcile(topology.env, "sync:volume:detach-stale", `${liveVolume.name ?? liveVolume.id}:${attachment.serviceId}`);
|
|
4162
|
+
await detachRailwayVolumeWithCli({
|
|
4163
|
+
projectId: project.id,
|
|
4164
|
+
environmentId: environment.id,
|
|
4165
|
+
serviceId: attachment.serviceId,
|
|
4166
|
+
volumeId: liveVolume.id,
|
|
4167
|
+
env: topology.env
|
|
4168
|
+
});
|
|
4169
|
+
}
|
|
4170
|
+
const desiredAttachment = activeEnvironmentAttachments.find((instance) => instance.serviceId === desiredService.id);
|
|
4171
|
+
if (liveVolume.name !== binding.canonicalVolumeName || desiredAttachment?.mountPath !== desiredConfig.volumeMountPath) {
|
|
4172
|
+
traceRailwayReconcile(topology.env, "sync:volume:update", `${liveVolume.name ?? liveVolume.id}:${binding.canonicalVolumeName}:${desiredConfig.volumeMountPath}`);
|
|
4173
|
+
await updateRailwayVolumeWithCli({
|
|
4174
|
+
projectId: project.id,
|
|
4175
|
+
environmentId: environment.id,
|
|
4176
|
+
serviceId: desiredService.id,
|
|
4177
|
+
volumeId: liveVolume.id,
|
|
4178
|
+
name: binding.canonicalVolumeName,
|
|
4179
|
+
mountPath: desiredConfig.volumeMountPath,
|
|
4180
|
+
env: topology.env
|
|
4181
|
+
});
|
|
4182
|
+
}
|
|
4183
|
+
const refreshedVolume = (await listRailwayVolumes({ projectId: project.id, env: topology.env })).find((volume) => volume.id === liveVolume.id);
|
|
4184
|
+
const attachedAfterUpdate = refreshedVolume?.instances.some(
|
|
4185
|
+
(instance) => instance.environmentId === environment.id && instance.serviceId === desiredService.id && !instance.isPendingDeletion && !(typeof instance.deletedAt === "string" && instance.deletedAt.trim()) && !["DELETING", "DELETED"].includes(String(instance.state ?? "").toUpperCase())
|
|
4186
|
+
);
|
|
4187
|
+
if (!attachedAfterUpdate) {
|
|
4188
|
+
traceRailwayReconcile(topology.env, "sync:volume:attach", `${binding.canonicalVolumeName}:${desiredService.name}`);
|
|
4189
|
+
await attachRailwayVolumeWithCli({
|
|
4190
|
+
projectId: project.id,
|
|
4191
|
+
environmentId: environment.id,
|
|
4192
|
+
serviceId: desiredService.id,
|
|
4193
|
+
volumeId: liveVolume.id,
|
|
4194
|
+
env: topology.env
|
|
4195
|
+
});
|
|
4196
|
+
}
|
|
4197
|
+
}
|
|
4198
|
+
const appliedServiceObservation = await waitForRailwayServices({
|
|
4199
|
+
serviceNames: effectiveIacInput.services.map((service) => service.serviceName),
|
|
4200
|
+
load: () => listRailwayEnvironmentServices({ environmentId: environment.id, env: topology.env }),
|
|
4201
|
+
sleep: async (milliseconds) => {
|
|
4202
|
+
traceRailwayReconcile(topology.env, "sync:service:settle", `${project.name}/${environment.name}:${milliseconds}ms`);
|
|
4203
|
+
await new Promise((resolve2) => setTimeout(resolve2, milliseconds));
|
|
4204
|
+
}
|
|
4205
|
+
});
|
|
4206
|
+
const appliedServices = appliedServiceObservation?.services ?? [];
|
|
4207
|
+
const appliedServiceByName = new Map(appliedServices.map((service) => [service.name, service]));
|
|
4208
|
+
for (const service of effectiveIacInput.services) {
|
|
4209
|
+
const appliedService = appliedServiceByName.get(service.serviceName);
|
|
4210
|
+
if (!appliedService) {
|
|
4211
|
+
throw new Error(`Railway IaC did not create or retain expected service ${service.serviceName} in ${environment.name}.`);
|
|
4212
|
+
}
|
|
4090
4213
|
}
|
|
4091
4214
|
assertNoBlockedRailwayProviderDrift(input, scope);
|
|
4092
4215
|
syncedServices.push(...projectServices);
|
|
@@ -4101,406 +4224,14 @@ async function syncRailwayEnvironmentForScope(input, { planOnly = false, service
|
|
|
4101
4224
|
secrets: Object.keys(sync.secrets),
|
|
4102
4225
|
variables: Object.keys(sync.variables),
|
|
4103
4226
|
planOnly,
|
|
4104
|
-
workspace
|
|
4227
|
+
workspace,
|
|
4228
|
+
projectPlans
|
|
4105
4229
|
};
|
|
4106
4230
|
}
|
|
4107
|
-
function
|
|
4108
|
-
|
|
4109
|
-
|
|
4110
|
-
|
|
4111
|
-
if (!entry.project?.id || !entry.environment?.name || !entry.service?.name) {
|
|
4112
|
-
throw new Error(`Railway source upload deploy requires resolved project, environment, and service for ${entry.configuredService.key}.`);
|
|
4113
|
-
}
|
|
4114
|
-
const rootDir = String(entry.configuredService.rootDir ?? "").trim();
|
|
4115
|
-
if (!rootDir || !existsSync(rootDir)) {
|
|
4116
|
-
throw new Error(`Railway source upload deploy root does not exist for ${entry.configuredService.key}: ${rootDir || "(unset)"}.`);
|
|
4117
|
-
}
|
|
4118
|
-
const uploadContext = prepareRailwaySourceUploadContext(input, entry.configuredService, rootDir);
|
|
4119
|
-
const uploadRoot = uploadContext.rootDir;
|
|
4120
|
-
const args = [
|
|
4121
|
-
"trsd",
|
|
4122
|
-
"railway",
|
|
4123
|
-
"--environment",
|
|
4124
|
-
scope,
|
|
4125
|
-
"--",
|
|
4126
|
-
"up",
|
|
4127
|
-
"--service",
|
|
4128
|
-
entry.service.name,
|
|
4129
|
-
"--project",
|
|
4130
|
-
entry.project.id,
|
|
4131
|
-
"--environment",
|
|
4132
|
-
entry.environment.name,
|
|
4133
|
-
"--ci",
|
|
4134
|
-
"--detach",
|
|
4135
|
-
"--path-as-root",
|
|
4136
|
-
...uploadContext.noGitignore ? ["--no-gitignore"] : [],
|
|
4137
|
-
"--verbose",
|
|
4138
|
-
"--message",
|
|
4139
|
-
`trsd staging source upload ${entry.configuredService.key}`,
|
|
4140
|
-
uploadRoot
|
|
4141
|
-
];
|
|
4142
|
-
const commandEnv = {
|
|
4143
|
-
...process.env,
|
|
4144
|
-
...env,
|
|
4145
|
-
TREESEED_RECONCILE_TRACE: env.TREESEED_RECONCILE_TRACE ?? process.env.TREESEED_RECONCILE_TRACE
|
|
4146
|
-
};
|
|
4147
|
-
try {
|
|
4148
|
-
const result = spawnSync("npx", args, {
|
|
4149
|
-
cwd: input.context.tenantRoot,
|
|
4150
|
-
env: commandEnv,
|
|
4151
|
-
encoding: "utf8",
|
|
4152
|
-
stdio: ["ignore", "pipe", "pipe"],
|
|
4153
|
-
maxBuffer: 1024 * 1024 * 20
|
|
4154
|
-
});
|
|
4155
|
-
if ((result.status ?? 1) !== 0) {
|
|
4156
|
-
const stderr = String(result.stderr ?? "").split("\n").slice(-120).join("\n");
|
|
4157
|
-
const stdout = String(result.stdout ?? "").split("\n").slice(-120).join("\n");
|
|
4158
|
-
throw new Error(`Railway source upload deploy failed for ${entry.configuredService.key} with status ${result.status ?? 1}.
|
|
4159
|
-
${stderr || stdout}`);
|
|
4160
|
-
}
|
|
4161
|
-
} finally {
|
|
4162
|
-
if (uploadContext.cleanupDir) {
|
|
4163
|
-
rmSync(uploadContext.cleanupDir, { recursive: true, force: true });
|
|
4164
|
-
}
|
|
4165
|
-
}
|
|
4166
|
-
}
|
|
4167
|
-
function prepareRailwaySourceUploadContext(input, service, rootDir) {
|
|
4168
|
-
const role = service.key === "capacityProviderManager" ? "manager" : service.key === "capacityProviderRunner" ? "runner" : null;
|
|
4169
|
-
if (!role) {
|
|
4170
|
-
return { rootDir, cleanupDir: null, noGitignore: false };
|
|
4171
|
-
}
|
|
4172
|
-
const cleanupDir = mkdtempSync(resolve(tmpdir(), `trsd-railway-agent-${role}-`));
|
|
4173
|
-
const contextRoot = resolve(cleanupDir, "context");
|
|
4174
|
-
mkdirSync(contextRoot, { recursive: true });
|
|
4175
|
-
for (const entry of ["src", "scripts", "templates", "docs"]) {
|
|
4176
|
-
const source = resolve(rootDir, entry);
|
|
4177
|
-
if (existsSync(source)) {
|
|
4178
|
-
cpSync(source, resolve(contextRoot, entry), { recursive: true });
|
|
4179
|
-
}
|
|
4180
|
-
}
|
|
4181
|
-
for (const entry of ["docker-entrypoint.sh", "package.json", "package-lock.json", "tsconfig.json", "tsconfig.dist.json"]) {
|
|
4182
|
-
const source = resolve(rootDir, entry);
|
|
4183
|
-
if (existsSync(source)) {
|
|
4184
|
-
cpSync(source, resolve(contextRoot, entry));
|
|
4185
|
-
}
|
|
4186
|
-
}
|
|
4187
|
-
const sdkRoot = resolve(rootDir, "..", "sdk");
|
|
4188
|
-
let sdkTarballPrepared = false;
|
|
4189
|
-
if (existsSync(resolve(sdkRoot, "package.json"))) {
|
|
4190
|
-
const pack = spawnSync("npm", ["pack", "--ignore-scripts", "--pack-destination", contextRoot], {
|
|
4191
|
-
cwd: sdkRoot,
|
|
4192
|
-
env: process.env,
|
|
4193
|
-
encoding: "utf8",
|
|
4194
|
-
stdio: ["ignore", "pipe", "pipe"],
|
|
4195
|
-
maxBuffer: 1024 * 1024 * 20
|
|
4196
|
-
});
|
|
4197
|
-
if ((pack.status ?? 1) !== 0) {
|
|
4198
|
-
const stderr = String(pack.stderr ?? "").split("\n").slice(-80).join("\n");
|
|
4199
|
-
const stdout = String(pack.stdout ?? "").split("\n").slice(-80).join("\n");
|
|
4200
|
-
throw new Error(`Failed to pack local SDK for Agent ${role} Railway source upload context.
|
|
4201
|
-
${stderr || stdout}`);
|
|
4202
|
-
}
|
|
4203
|
-
const filename = String(pack.stdout ?? "").split(/\r?\n/u).map((line) => line.trim()).filter(Boolean).at(-1);
|
|
4204
|
-
if (!filename || !existsSync(resolve(contextRoot, filename))) {
|
|
4205
|
-
throw new Error(`Local SDK pack did not produce a tarball for Agent ${role} Railway source upload context.`);
|
|
4206
|
-
}
|
|
4207
|
-
cpSync(resolve(contextRoot, filename), resolve(contextRoot, "treeseed-sdk.tgz"));
|
|
4208
|
-
if (filename !== "treeseed-sdk.tgz") {
|
|
4209
|
-
rmSync(resolve(contextRoot, filename), { force: true });
|
|
4210
|
-
}
|
|
4211
|
-
sdkTarballPrepared = true;
|
|
4212
|
-
}
|
|
4213
|
-
const dockerfile = [
|
|
4214
|
-
"FROM node:22-alpine AS builder",
|
|
4215
|
-
"WORKDIR /app",
|
|
4216
|
-
"RUN apk add --no-cache ca-certificates git openssh-client",
|
|
4217
|
-
sdkTarballPrepared ? "COPY package.json package-lock.json treeseed-sdk.tgz ./" : "COPY package.json package-lock.json ./",
|
|
4218
|
-
sdkTarballPrepared ? "RUN npm ci --include=dev --ignore-scripts \\\n && npm install --include=dev --ignore-scripts ./treeseed-sdk.tgz" : "RUN npm ci --include=dev --ignore-scripts",
|
|
4219
|
-
"COPY . .",
|
|
4220
|
-
"RUN npm run build:dist \\",
|
|
4221
|
-
" && npm prune --omit=dev --ignore-scripts \\",
|
|
4222
|
-
" && rm -rf node_modules/@github node_modules/@openai node_modules/@cloudflare node_modules/@railway \\",
|
|
4223
|
-
" node_modules/wrangler node_modules/miniflare node_modules/workerd \\",
|
|
4224
|
-
" node_modules/playwright node_modules/playwright-core node_modules/gpt-tokenizer \\",
|
|
4225
|
-
" node_modules/@remotion node_modules/remotion node_modules/@rspack \\",
|
|
4226
|
-
" node_modules/@repomix node_modules/repomix node_modules/@esbuild node_modules/esbuild \\",
|
|
4227
|
-
" node_modules/webpack node_modules/typescript node_modules/@mediabunny node_modules/mediabunny \\",
|
|
4228
|
-
" node_modules/web-tree-sitter node_modules/caniuse-lite node_modules/@img node_modules/@secretlint \\",
|
|
4229
|
-
" node_modules/lightningcss* node_modules/vite \\",
|
|
4230
|
-
" && npm cache clean --force \\",
|
|
4231
|
-
" && rm -rf src scripts test .git .github",
|
|
4232
|
-
"FROM node:22-alpine AS runtime",
|
|
4233
|
-
"WORKDIR /app",
|
|
4234
|
-
"ENV NODE_ENV=production TREESEED_PROVIDER_DATA_DIR=/data",
|
|
4235
|
-
"RUN apk add --no-cache ca-certificates tini util-linux git openssh-client \\",
|
|
4236
|
-
" && mkdir -p /data",
|
|
4237
|
-
"COPY --from=builder --chown=65532:65532 /app/package.json /app/package-lock.json ./",
|
|
4238
|
-
"COPY --from=builder --chown=65532:65532 /app/node_modules ./node_modules",
|
|
4239
|
-
"COPY --from=builder --chown=65532:65532 /app/dist ./dist",
|
|
4240
|
-
"COPY --from=builder --chown=65532:65532 /app/templates ./templates",
|
|
4241
|
-
"COPY --from=builder --chown=65532:65532 /app/docs ./docs",
|
|
4242
|
-
"COPY --from=builder --chown=65532:65532 /app/docker-entrypoint.sh ./docker-entrypoint.sh",
|
|
4243
|
-
"RUN chmod 0755 /app/docker-entrypoint.sh \\",
|
|
4244
|
-
" && chown -R 65532:65532 /data",
|
|
4245
|
-
"EXPOSE 3100",
|
|
4246
|
-
'ENTRYPOINT ["tini", "--", "/app/docker-entrypoint.sh"]',
|
|
4247
|
-
`ENV TREESEED_PROVIDER_ROLE=${role}`,
|
|
4248
|
-
`CMD ["${role}"]`,
|
|
4249
|
-
""
|
|
4250
|
-
].join("\n");
|
|
4251
|
-
writeFileSync(
|
|
4252
|
-
resolve(contextRoot, "Dockerfile"),
|
|
4253
|
-
dockerfile
|
|
4254
|
-
);
|
|
4255
|
-
return { rootDir: contextRoot, cleanupDir, noGitignore: true };
|
|
4256
|
-
}
|
|
4257
|
-
async function deployRailwayServiceInstanceWithSourceRepair(entry, env) {
|
|
4258
|
-
if (!entry.environment || !entry.service) {
|
|
4259
|
-
return;
|
|
4260
|
-
}
|
|
4261
|
-
const deploy = () => deployRailwayServiceInstance({
|
|
4262
|
-
serviceId: entry.service.id,
|
|
4263
|
-
environmentId: entry.environment.id,
|
|
4264
|
-
env
|
|
4265
|
-
});
|
|
4266
|
-
if (entry.configuredService.sourceMode === "git" && entry.configuredService.sourceRepo && !entry.configuredService.imageRef) {
|
|
4267
|
-
await repairRailwayServiceSource(entry, env);
|
|
4268
|
-
}
|
|
4269
|
-
try {
|
|
4270
|
-
await deploy();
|
|
4271
|
-
return;
|
|
4272
|
-
} catch (error) {
|
|
4273
|
-
const message = error instanceof Error ? error.message : String(error ?? "");
|
|
4274
|
-
if (/Problem processing request/iu.test(message)) {
|
|
4275
|
-
traceRailwayReconcile(env, "sync:deploy-skipped", `${entry.configuredService.key}:${entry.service.name}:problem-processing-request`);
|
|
4276
|
-
return;
|
|
4277
|
-
}
|
|
4278
|
-
if (!/Deployment not found/iu.test(message)) {
|
|
4279
|
-
throw error;
|
|
4280
|
-
}
|
|
4281
|
-
}
|
|
4282
|
-
const repaired = await repairRailwayServiceSource(entry, env);
|
|
4283
|
-
if (repaired) {
|
|
4284
|
-
try {
|
|
4285
|
-
await deploy();
|
|
4286
|
-
return;
|
|
4287
|
-
} catch (error) {
|
|
4288
|
-
const message = error instanceof Error ? error.message : String(error ?? "");
|
|
4289
|
-
if (/Problem processing request/iu.test(message)) {
|
|
4290
|
-
traceRailwayReconcile(env, "sync:deploy-skipped", `${entry.configuredService.key}:${entry.service.name}:problem-processing-request`);
|
|
4291
|
-
return;
|
|
4292
|
-
}
|
|
4293
|
-
if (!/Deployment not found/iu.test(message)) {
|
|
4294
|
-
throw error;
|
|
4295
|
-
}
|
|
4296
|
-
}
|
|
4297
|
-
}
|
|
4298
|
-
throw new Error(
|
|
4299
|
-
`Railway deployment for existing service ${entry.service.name} (${entry.service.id}) was not found; refusing to delete and recreate an existing service. Repair the service in place or restore a deployable service source before rerunning reconciliation.`
|
|
4300
|
-
);
|
|
4301
|
-
}
|
|
4302
|
-
async function repairRailwayServiceSource(entry, env) {
|
|
4303
|
-
if (!entry.service) {
|
|
4304
|
-
return false;
|
|
4305
|
-
}
|
|
4306
|
-
const sourceRepo = String(entry.configuredService.sourceRepo ?? "").trim();
|
|
4307
|
-
if (sourceRepo) {
|
|
4308
|
-
await updateRailwayServiceGitSource({
|
|
4309
|
-
serviceId: entry.service.id,
|
|
4310
|
-
sourceRepo,
|
|
4311
|
-
sourceBranch: entry.configuredService.sourceBranch,
|
|
4312
|
-
env
|
|
4313
|
-
});
|
|
4314
|
-
return true;
|
|
4315
|
-
}
|
|
4316
|
-
const imageRef = String(entry.configuredService.imageRef ?? "").trim();
|
|
4317
|
-
if (imageRef) {
|
|
4318
|
-
await updateRailwayServiceImageSource({
|
|
4319
|
-
serviceId: entry.service.id,
|
|
4320
|
-
imageRef,
|
|
4321
|
-
env
|
|
4322
|
-
});
|
|
4323
|
-
return true;
|
|
4324
|
-
}
|
|
4325
|
-
return false;
|
|
4326
|
-
}
|
|
4327
|
-
function looksLikeRailwayExistingSourceRepairUnsupported(error) {
|
|
4328
|
-
const message = error instanceof Error ? error.message : String(error ?? "");
|
|
4329
|
-
return /source update .*existing service|existing service .*source update|provider-supported .*source update|ServiceUpdateInput|Problem processing request|User does not have access to the repo/iu.test(message);
|
|
4330
|
-
}
|
|
4331
|
-
async function ensureRailwayMarketDatabaseForScope(input, topology) {
|
|
4332
|
-
const scope = input.context.target.kind === "persistent" ? input.context.target.scope : "staging";
|
|
4333
|
-
const treeseedDatabase = configuredMarketDatabaseService(input.context.tenantRoot, input.context.deployConfig);
|
|
4334
|
-
const treeseedDatabaseService = treeseedDatabase?.service;
|
|
4335
|
-
if (!treeseedDatabaseService || treeseedDatabaseService.enabled === false || treeseedDatabaseService.provider !== "railway" || treeseedDatabaseService.railway?.resourceType !== "postgres") {
|
|
4336
|
-
return;
|
|
4337
|
-
}
|
|
4338
|
-
const serviceName = treeseedDatabase.serviceName;
|
|
4339
|
-
const firstService = [...topology.services.values()].find(
|
|
4340
|
-
(entry) => entry.project && entry.environment && ["api", "operationsRunner"].includes(entry.configuredService.key)
|
|
4341
|
-
);
|
|
4342
|
-
if (!firstService?.project || !firstService.environment) {
|
|
4343
|
-
await reconcileAccidentalMarketDatabaseServices(input, topology, serviceName, null);
|
|
4344
|
-
return;
|
|
4345
|
-
}
|
|
4346
|
-
await reconcileAccidentalMarketDatabaseServices(input, topology, serviceName, firstService.project.id);
|
|
4347
|
-
let postgresService;
|
|
4348
|
-
try {
|
|
4349
|
-
postgresService = (await ensureRailwayPostgresService({
|
|
4350
|
-
projectId: firstService.project.id,
|
|
4351
|
-
environmentId: firstService.environment.id,
|
|
4352
|
-
serviceName,
|
|
4353
|
-
env: topology.env
|
|
4354
|
-
})).service;
|
|
4355
|
-
} catch (error) {
|
|
4356
|
-
const detail = error instanceof Error ? error.message : String(error ?? "");
|
|
4357
|
-
throw new Error(`Railway API Postgres provisioning unsupported for ${serviceName}. ${detail}`);
|
|
4358
|
-
}
|
|
4359
|
-
if (!postgresService) {
|
|
4360
|
-
throw new Error(`Railway API Postgres provisioning unsupported for ${serviceName}. No service was returned.`);
|
|
4361
|
-
}
|
|
4362
|
-
try {
|
|
4363
|
-
const postgresVolume = await ensureRailwayServiceVolume({
|
|
4364
|
-
projectId: firstService.project.id,
|
|
4365
|
-
environmentId: firstService.environment.id,
|
|
4366
|
-
serviceId: postgresService.id,
|
|
4367
|
-
name: `${serviceName}-volume`,
|
|
4368
|
-
mountPath: "/var/lib/postgresql/data",
|
|
4369
|
-
env: topology.env
|
|
4370
|
-
});
|
|
4371
|
-
if (postgresVolume.instance?.serviceId !== postgresService.id || postgresVolume.instance?.environmentId !== firstService.environment.id || postgresVolume.instance?.mountPath !== "/var/lib/postgresql/data") {
|
|
4372
|
-
throw new Error(`observed volume ${postgresVolume.volume.name ?? postgresVolume.volume.id ?? "(unknown)"} is not attached to ${serviceName} at /var/lib/postgresql/data`);
|
|
4373
|
-
}
|
|
4374
|
-
} catch (error) {
|
|
4375
|
-
const detail = error instanceof Error ? error.message : String(error ?? "");
|
|
4376
|
-
throw new Error(`Railway provider limitation while reconciling PostgreSQL volume name for ${serviceName}: ${detail}`);
|
|
4377
|
-
}
|
|
4378
|
-
const services = await listRailwayServices({ projectId: firstService.project.id, env: topology.env });
|
|
4379
|
-
for (const service of services) {
|
|
4380
|
-
if (service.id === postgresService.id || service.name === serviceName) {
|
|
4381
|
-
continue;
|
|
4382
|
-
}
|
|
4383
|
-
const variables = await listRailwayVariables({
|
|
4384
|
-
projectId: firstService.project.id,
|
|
4385
|
-
environmentId: firstService.environment.id,
|
|
4386
|
-
serviceId: service.id,
|
|
4387
|
-
env: topology.env
|
|
4388
|
-
}).catch(() => ({}));
|
|
4389
|
-
const looksLikePostgres = typeof variables.DATABASE_URL === "string" && typeof variables.PGHOST === "string" && typeof variables.PGUSER === "string" && typeof variables.PGPASSWORD === "string";
|
|
4390
|
-
if (looksLikePostgres) {
|
|
4391
|
-
traceRailwayReconcile(topology.env, "sync:postgres:block-duplicate", `${service.name}:${service.id}`);
|
|
4392
|
-
recordRailwayProviderDrift(input, scope, {
|
|
4393
|
-
kind: "railway.duplicate-postgres-service",
|
|
4394
|
-
action: "manual-repair-required",
|
|
4395
|
-
status: "blocked",
|
|
4396
|
-
projectId: firstService.project.id,
|
|
4397
|
-
environmentId: firstService.environment.id,
|
|
4398
|
-
serviceId: service.id,
|
|
4399
|
-
serviceName: service.name,
|
|
4400
|
-
reason: `Railway reports an extra PostgreSQL service in ${firstService.project.name}. Reconciliation refuses to delete existing services; adopt, retain, or remove it through an explicit destroy workflow.`
|
|
4401
|
-
});
|
|
4402
|
-
}
|
|
4403
|
-
}
|
|
4404
|
-
const detachedPostgresVolumes = await listRailwayVolumes({
|
|
4405
|
-
projectId: firstService.project.id,
|
|
4406
|
-
env: topology.env
|
|
4407
|
-
});
|
|
4408
|
-
for (const volume of detachedPostgresVolumes) {
|
|
4409
|
-
if (isRetainedRailwayDetachedPostgresVolume(volume.name)) {
|
|
4410
|
-
traceRailwayReconcile(topology.env, "sync:postgres:retain-detached-volume", `${volume.name ?? volume.id}:${volume.id}`);
|
|
4411
|
-
continue;
|
|
4412
|
-
}
|
|
4413
|
-
const postgresInstances = volume.instances.filter(
|
|
4414
|
-
(instance) => instance.environmentId === firstService.environment?.id && instance.mountPath === "/var/lib/postgresql/data"
|
|
4415
|
-
);
|
|
4416
|
-
const activePostgresInstances = postgresInstances.filter((instance) => Boolean(instance.serviceId));
|
|
4417
|
-
const attachedToCanonical = postgresInstances.some((instance) => instance.serviceId === postgresService.id);
|
|
4418
|
-
const detached = postgresInstances.length > 0 && postgresInstances.every((instance) => !instance.serviceId);
|
|
4419
|
-
const detachedCanonicalVolume = volume.name === `${serviceName}-volume` && activePostgresInstances.length === 0;
|
|
4420
|
-
if ((detached || detachedCanonicalVolume) && !attachedToCanonical) {
|
|
4421
|
-
traceRailwayReconcile(topology.env, "sync:postgres:block-detached-volume", `${volume.name ?? volume.id}:${volume.id}`);
|
|
4422
|
-
recordRailwayProviderDrift(input, scope, {
|
|
4423
|
-
kind: "railway.detached-postgres-volume",
|
|
4424
|
-
action: "manual-repair-required",
|
|
4425
|
-
status: "blocked",
|
|
4426
|
-
projectId: firstService.project.id,
|
|
4427
|
-
environmentId: firstService.environment.id,
|
|
4428
|
-
volumeId: volume.id,
|
|
4429
|
-
volumeName: volume.name ?? null,
|
|
4430
|
-
reason: "Railway reports a detached PostgreSQL volume. Reconciliation refuses destructive cleanup during repair; adopt, retain, or remove it through an explicit destroy workflow."
|
|
4431
|
-
});
|
|
4432
|
-
}
|
|
4433
|
-
}
|
|
4434
|
-
for (let attempt = 0; attempt < 20; attempt += 1) {
|
|
4435
|
-
const existingVolumes = await listRailwayVolumes({
|
|
4436
|
-
projectId: firstService.project.id,
|
|
4437
|
-
env: topology.env
|
|
4438
|
-
});
|
|
4439
|
-
const attached = existingVolumes.some((volume) => volume.instances.some(
|
|
4440
|
-
(instance) => instance.serviceId === postgresService.id && instance.environmentId === firstService.environment?.id && instance.mountPath === "/var/lib/postgresql/data"
|
|
4441
|
-
));
|
|
4442
|
-
if (attached) {
|
|
4443
|
-
return;
|
|
4444
|
-
}
|
|
4445
|
-
await new Promise((resolve2) => setTimeout(resolve2, 3e3));
|
|
4446
|
-
}
|
|
4447
|
-
}
|
|
4448
|
-
async function reconcileAccidentalMarketDatabaseServices(input, topology, serviceName, ownerProjectId) {
|
|
4449
|
-
const scope = input.context.target.kind === "persistent" ? input.context.target.scope : "staging";
|
|
4450
|
-
const projectEntries = [...topology.services.values()].filter((entry) => entry.project?.id && entry.environment?.id).map((entry) => ({ project: entry.project, environment: entry.environment }));
|
|
4451
|
-
const projects = new Map(projectEntries.map((entry) => [entry.project.id, entry]));
|
|
4452
|
-
for (const { project, environment } of projects.values()) {
|
|
4453
|
-
if (ownerProjectId && project.id === ownerProjectId) {
|
|
4454
|
-
continue;
|
|
4455
|
-
}
|
|
4456
|
-
const services = await listRailwayServices({ projectId: project.id, env: topology.env }).catch(() => []);
|
|
4457
|
-
for (const service of services.filter((entry) => entry.name === serviceName)) {
|
|
4458
|
-
traceRailwayReconcile(topology.env, "sync:postgres:block-accidental-service", `${project.name}:${service.name}:${service.id}`);
|
|
4459
|
-
recordRailwayProviderDrift(input, scope, {
|
|
4460
|
-
kind: "railway.accidental-market-database-service",
|
|
4461
|
-
action: "manual-repair-required",
|
|
4462
|
-
status: "blocked",
|
|
4463
|
-
projectId: project.id,
|
|
4464
|
-
environmentId: environment.id,
|
|
4465
|
-
serviceId: service.id,
|
|
4466
|
-
serviceName: service.name,
|
|
4467
|
-
reason: `Railway reports undeclared Market database service ${service.name} in non-owner project ${project.name}. Reconciliation refuses to delete existing services; adopt, retain, or remove it through an explicit destroy workflow.`
|
|
4468
|
-
});
|
|
4469
|
-
}
|
|
4470
|
-
const volumes = await listRailwayVolumes({ projectId: project.id, env: topology.env }).catch(() => []);
|
|
4471
|
-
for (const volume of volumes) {
|
|
4472
|
-
if (isRetainedRailwayDetachedPostgresVolume(volume.name)) {
|
|
4473
|
-
continue;
|
|
4474
|
-
}
|
|
4475
|
-
const postgresInstances = volume.instances.filter(
|
|
4476
|
-
(instance) => instance.environmentId === environment.id && instance.mountPath === "/var/lib/postgresql/data"
|
|
4477
|
-
);
|
|
4478
|
-
if (postgresInstances.length === 0) {
|
|
4479
|
-
continue;
|
|
4480
|
-
}
|
|
4481
|
-
const activeServiceIds = new Set(services.map((service) => service.id));
|
|
4482
|
-
const stillAttached = postgresInstances.some((instance) => instance.serviceId && activeServiceIds.has(instance.serviceId));
|
|
4483
|
-
if (stillAttached) {
|
|
4484
|
-
continue;
|
|
4485
|
-
}
|
|
4486
|
-
traceRailwayReconcile(topology.env, "sync:postgres:block-accidental-volume", `${project.name}:${volume.name ?? volume.id}:${volume.id}`);
|
|
4487
|
-
recordRailwayProviderDrift(input, scope, {
|
|
4488
|
-
kind: "railway.accidental-market-database-volume",
|
|
4489
|
-
action: "manual-repair-required",
|
|
4490
|
-
status: "blocked",
|
|
4491
|
-
projectId: project.id,
|
|
4492
|
-
environmentId: environment.id,
|
|
4493
|
-
volumeId: volume.id,
|
|
4494
|
-
volumeName: volume.name ?? null,
|
|
4495
|
-
reason: `Railway reports undeclared Market database volume ${volume.name ?? volume.id} in non-owner project ${project.name}. Reconciliation refuses destructive cleanup during repair; adopt, retain, or remove it through an explicit destroy workflow.`
|
|
4496
|
-
});
|
|
4497
|
-
}
|
|
4498
|
-
}
|
|
4499
|
-
}
|
|
4500
|
-
function isRetainedRailwayDetachedPostgresVolume(value) {
|
|
4501
|
-
return String(value ?? "").trim().startsWith("retained-");
|
|
4502
|
-
}
|
|
4503
|
-
async function observeRailwayUnit(input, { refresh = false } = {}) {
|
|
4231
|
+
async function observeRailwayUnit(input, {
|
|
4232
|
+
refresh = false,
|
|
4233
|
+
topology: providedTopology
|
|
4234
|
+
} = {}) {
|
|
4504
4235
|
let attempt = 0;
|
|
4505
4236
|
for (; ; ) {
|
|
4506
4237
|
try {
|
|
@@ -4508,13 +4239,13 @@ async function observeRailwayUnit(input, { refresh = false } = {}) {
|
|
|
4508
4239
|
const serviceKey = String(input.unit.metadata.serviceKey ?? "").trim();
|
|
4509
4240
|
const state = loadDeployState(input.context.tenantRoot, input.context.deployConfig, { target: toDeployTarget(input.context.target) });
|
|
4510
4241
|
const persisted = state.services?.[serviceKey] ?? {};
|
|
4511
|
-
const topology = await
|
|
4242
|
+
const topology = providedTopology ?? await resolveRailwayUnitTopology(input, scope, {
|
|
4512
4243
|
refresh,
|
|
4513
|
-
serviceKeys: [serviceKey],
|
|
4514
4244
|
includeInstances: false,
|
|
4515
|
-
includeVariables: false
|
|
4245
|
+
includeVariables: false,
|
|
4246
|
+
cacheSuffix: refresh ? "refresh" : "observe"
|
|
4516
4247
|
});
|
|
4517
|
-
const entry = topology
|
|
4248
|
+
const entry = findRailwayTopologyEntry(topology, railwayUnitServiceIdentity(input));
|
|
4518
4249
|
const configuredService = entry?.configuredService ?? null;
|
|
4519
4250
|
const usesPortableSource = Boolean(
|
|
4520
4251
|
configuredService?.imageRef || configuredService?.sourceMode === "git" && configuredService?.sourceRepo
|
|
@@ -4867,8 +4598,7 @@ async function observeCustomDomainUnit(input) {
|
|
|
4867
4598
|
}
|
|
4868
4599
|
case "custom-domain:api": {
|
|
4869
4600
|
const domain = String(input.unit.spec.domain ?? "").trim();
|
|
4870
|
-
const
|
|
4871
|
-
const live = (cached && railwayCustomDomainHasDnsRequirements(cached) ? cached : null) ?? await observeRailwayCustomDomainLive(input, domain) ?? cached ?? input.persistedState?.lastObservedState ?? input.persistedState?.lastReconciledState ?? null;
|
|
4601
|
+
const live = await observeRailwayCustomDomainLive(input, domain);
|
|
4872
4602
|
if (live?.domain) {
|
|
4873
4603
|
storeCustomDomainState(input, "railway", domain, live);
|
|
4874
4604
|
}
|
|
@@ -4878,6 +4608,8 @@ async function observeCustomDomainUnit(input) {
|
|
|
4878
4608
|
live: live ?? {},
|
|
4879
4609
|
locators: {
|
|
4880
4610
|
domain: domain || null,
|
|
4611
|
+
serviceId: typeof live?.serviceId === "string" ? live.serviceId : null,
|
|
4612
|
+
serviceName: typeof live?.serviceName === "string" ? live.serviceName : null,
|
|
4881
4613
|
serviceDomain: typeof live?.serviceDomain === "string" ? live.serviceDomain : null
|
|
4882
4614
|
},
|
|
4883
4615
|
warnings: []
|
|
@@ -4929,8 +4661,7 @@ async function verifyCustomDomainUnit(input) {
|
|
|
4929
4661
|
}
|
|
4930
4662
|
case "custom-domain:api": {
|
|
4931
4663
|
const domain = String(input.unit.spec.domain ?? "").trim();
|
|
4932
|
-
const
|
|
4933
|
-
const live = (cached && railwayCustomDomainHasDnsRequirements(cached) ? cached : null) ?? await observeRailwayCustomDomainLive(input, domain) ?? cached ?? getPersistedCustomDomainState(input, "railway", domain) ?? input.persistedState?.lastObservedState ?? input.persistedState?.lastReconciledState ?? null;
|
|
4664
|
+
const live = await observeRailwayCustomDomainLive(input, domain);
|
|
4934
4665
|
const dnsRecords = Array.isArray(live?.dnsRecords) ? live.dnsRecords : [];
|
|
4935
4666
|
const certificateStatus = typeof live?.certificateStatus === "string" ? live.certificateStatus.trim().toUpperCase() : null;
|
|
4936
4667
|
const verified = live?.verified === true;
|
|
@@ -4946,6 +4677,13 @@ async function verifyCustomDomainUnit(input) {
|
|
|
4946
4677
|
observed: typeof live?.domain === "string" ? live.domain : null,
|
|
4947
4678
|
issues: live?.domain ? [] : [`Railway custom domain ${domain || "(unset)"} is missing.`]
|
|
4948
4679
|
}),
|
|
4680
|
+
verificationCheck("custom-domain.service", "Railway custom domain is attached to the desired service", "api", {
|
|
4681
|
+
exists: Boolean(live?.serviceId),
|
|
4682
|
+
configured: String(live?.serviceName ?? "") === String(input.unit.spec.serviceName ?? ""),
|
|
4683
|
+
expected: input.unit.spec.serviceName ?? null,
|
|
4684
|
+
observed: live?.serviceName ?? null,
|
|
4685
|
+
issues: String(live?.serviceName ?? "") === String(input.unit.spec.serviceName ?? "") ? [] : [`Railway custom domain ${domain || "(unset)"} is not attached to ${String(input.unit.spec.serviceName ?? "(unset service)")}.`]
|
|
4686
|
+
}),
|
|
4949
4687
|
verificationCheck("custom-domain.dns-requirements", "Railway custom domain exposes DNS requirements", "api", {
|
|
4950
4688
|
exists: hasDnsRequirements,
|
|
4951
4689
|
expected: true,
|
|
@@ -5034,30 +4772,72 @@ async function reconcileCustomDomainUnit(input, diff) {
|
|
|
5034
4772
|
}
|
|
5035
4773
|
case "custom-domain:api": {
|
|
5036
4774
|
const scope = input.context.target.kind === "persistent" ? input.context.target.scope : "staging";
|
|
4775
|
+
const domain = String(input.unit.spec.domain ?? "").trim();
|
|
5037
4776
|
const serviceKey = String(input.unit.metadata.serviceKey ?? "api").trim();
|
|
5038
|
-
const topology = await
|
|
5039
|
-
|
|
5040
|
-
serviceKeys: [serviceKey],
|
|
4777
|
+
const topology = await resolveRailwayUnitTopology(input, scope, {
|
|
4778
|
+
refresh: false,
|
|
5041
4779
|
includeInstances: false,
|
|
5042
|
-
includeVariables: false
|
|
4780
|
+
includeVariables: false,
|
|
4781
|
+
cacheSuffix: "post-apply"
|
|
5043
4782
|
});
|
|
5044
|
-
const entry = topology
|
|
5045
|
-
if (!entry?.
|
|
5046
|
-
throw new Error(`Railway
|
|
4783
|
+
const entry = findRailwayTopologyEntry(topology, serviceKey);
|
|
4784
|
+
if (!entry?.project?.id || !entry.environment?.id || !entry.service?.id || !domain) {
|
|
4785
|
+
throw new Error(`Railway custom domain reconciliation could not resolve ${domain || "(unset domain)"} and its desired service.`);
|
|
5047
4786
|
}
|
|
5048
|
-
const
|
|
5049
|
-
|
|
5050
|
-
|
|
5051
|
-
|
|
5052
|
-
|
|
5053
|
-
|
|
5054
|
-
|
|
5055
|
-
|
|
5056
|
-
|
|
4787
|
+
const projectServices = await listRailwayServices({ projectId: entry.project.id, env: topology.env });
|
|
4788
|
+
const detachedServiceIds = [];
|
|
4789
|
+
for (const candidate of projectServices) {
|
|
4790
|
+
if (candidate.id === entry.service.id) continue;
|
|
4791
|
+
const domains = await listRailwayCustomDomains({
|
|
4792
|
+
projectId: entry.project.id,
|
|
4793
|
+
environmentId: entry.environment.id,
|
|
4794
|
+
serviceId: candidate.id,
|
|
4795
|
+
env: topology.env
|
|
4796
|
+
}).catch(() => []);
|
|
4797
|
+
for (const stale of domains.filter((candidateDomain) => candidateDomain.domain === domain)) {
|
|
4798
|
+
if (!stale.id) throw new Error(`Railway custom domain ${domain} is attached to ${candidate.name} but has no provider id for safe transfer.`);
|
|
4799
|
+
traceRailwayReconcile(topology.env, "sync:custom-domain:detach-stale", `${domain}:${candidate.name}`);
|
|
4800
|
+
await deleteRailwayCustomDomain({
|
|
4801
|
+
projectId: entry.project.id,
|
|
4802
|
+
environmentId: entry.environment.id,
|
|
4803
|
+
serviceId: candidate.id,
|
|
4804
|
+
domainId: stale.id,
|
|
4805
|
+
env: topology.env
|
|
4806
|
+
});
|
|
4807
|
+
detachedServiceIds.push(candidate.id);
|
|
5057
4808
|
}
|
|
5058
|
-
|
|
5059
|
-
|
|
5060
|
-
|
|
4809
|
+
}
|
|
4810
|
+
for (const detachedServiceId of detachedServiceIds) {
|
|
4811
|
+
let stillAttached = true;
|
|
4812
|
+
for (let detachAttempt = 0; detachAttempt < 12 && stillAttached; detachAttempt += 1) {
|
|
4813
|
+
const domains = await listRailwayCustomDomains({
|
|
4814
|
+
projectId: entry.project.id,
|
|
4815
|
+
environmentId: entry.environment.id,
|
|
4816
|
+
serviceId: detachedServiceId,
|
|
4817
|
+
env: topology.env
|
|
4818
|
+
});
|
|
4819
|
+
stillAttached = domains.some((candidateDomain) => candidateDomain.domain === domain);
|
|
4820
|
+
if (stillAttached) sleepMs(2e3);
|
|
4821
|
+
}
|
|
4822
|
+
if (stillAttached) {
|
|
4823
|
+
throw new Error(`Railway custom domain ${domain} remained attached to stale service ${detachedServiceId} after deletion.`);
|
|
4824
|
+
}
|
|
4825
|
+
}
|
|
4826
|
+
await ensureRailwayCustomDomain({
|
|
4827
|
+
projectId: entry.project.id,
|
|
4828
|
+
environmentId: entry.environment.id,
|
|
4829
|
+
serviceId: entry.service.id,
|
|
4830
|
+
domain,
|
|
4831
|
+
env: topology.env
|
|
4832
|
+
});
|
|
4833
|
+
let observed = await observeCustomDomainUnit(input);
|
|
4834
|
+
for (let domainAttempt = 0; domainAttempt < 12 && (!observed.exists || observed.locators.serviceId !== entry.service.id); domainAttempt += 1) {
|
|
4835
|
+
sleepMs(2e3);
|
|
4836
|
+
observed = await observeCustomDomainUnit(input);
|
|
4837
|
+
}
|
|
4838
|
+
if (!observed.exists || observed.locators.serviceId !== entry.service.id) {
|
|
4839
|
+
throw new Error(`Railway did not attach custom domain ${domain} to ${entry.service.name}.`);
|
|
4840
|
+
}
|
|
5061
4841
|
return {
|
|
5062
4842
|
unit: input.unit,
|
|
5063
4843
|
observed,
|
|
@@ -5065,7 +4845,7 @@ async function reconcileCustomDomainUnit(input, diff) {
|
|
|
5065
4845
|
action: diff.action === "create" || diff.action === "update" ? "update" : diff.action,
|
|
5066
4846
|
warnings: observed.warnings,
|
|
5067
4847
|
resourceLocators: observed.locators,
|
|
5068
|
-
state,
|
|
4848
|
+
state: observed.live,
|
|
5069
4849
|
verification: null
|
|
5070
4850
|
};
|
|
5071
4851
|
}
|
|
@@ -5114,13 +4894,13 @@ async function verifyRailwayUnit(input) {
|
|
|
5114
4894
|
try {
|
|
5115
4895
|
const scope = input.context.target.kind === "persistent" ? input.context.target.scope : "staging";
|
|
5116
4896
|
const serviceKey = String(input.unit.metadata.serviceKey ?? "").trim();
|
|
5117
|
-
const topology = await
|
|
5118
|
-
serviceKeys: [serviceKey],
|
|
4897
|
+
const topology = await resolveRailwayUnitTopology(input, scope, {
|
|
5119
4898
|
includeInstances: true,
|
|
5120
4899
|
includeVariables: true,
|
|
5121
|
-
refresh: true
|
|
4900
|
+
refresh: true,
|
|
4901
|
+
cacheSuffix: "verify"
|
|
5122
4902
|
});
|
|
5123
|
-
const entry = topology
|
|
4903
|
+
const entry = findRailwayTopologyEntry(topology, railwayUnitServiceIdentity(input));
|
|
5124
4904
|
const service = entry?.configuredService ?? null;
|
|
5125
4905
|
if (!service || !entry) {
|
|
5126
4906
|
return summarizeVerification(input.unit.unitId, [
|
|
@@ -5313,19 +5093,28 @@ async function verifyRailwayUnit(input) {
|
|
|
5313
5093
|
const volumes = entry.project?.id ? await listRailwayVolumes({ projectId: entry.project.id, env: topology.env }) : [];
|
|
5314
5094
|
const expectedServiceId = entry.service?.id ?? null;
|
|
5315
5095
|
const expectedEnvironmentId = entry.environment?.id ?? null;
|
|
5316
|
-
const
|
|
5096
|
+
const mountedVolumes = volumes.filter((volume) => volume.instances.some(
|
|
5317
5097
|
(instance) => instance.serviceId === expectedServiceId && instance.environmentId === expectedEnvironmentId && instance.mountPath === service.volumeMountPath && !instance.isPendingDeletion && !(typeof instance.deletedAt === "string" && instance.deletedAt.trim()) && !["DELETING", "DELETED"].includes(String(instance.state ?? "").toUpperCase())
|
|
5318
|
-
))
|
|
5098
|
+
));
|
|
5099
|
+
const expectedVolumeName = `${service.serviceName}-volume`;
|
|
5100
|
+
const mountedVolume = mountedVolumes.find((volume) => volume.name === expectedVolumeName) ?? null;
|
|
5101
|
+
const volumeIssues = [
|
|
5102
|
+
mountedVolume ? null : `Railway service ${service.serviceName ?? service.key} is not mounted to canonical volume ${expectedVolumeName} at ${service.volumeMountPath}.`,
|
|
5103
|
+
mountedVolumes.length === 1 ? null : `Railway service ${service.serviceName ?? service.key} has ${mountedVolumes.length} active volumes mounted at ${service.volumeMountPath}; expected exactly one.`
|
|
5104
|
+
].filter((issue) => Boolean(issue));
|
|
5319
5105
|
checks.push(verificationCheck("railway.volume:data", "Railway service has persistent data volume mounted", "api", {
|
|
5320
|
-
exists:
|
|
5321
|
-
configured:
|
|
5322
|
-
|
|
5106
|
+
exists: mountedVolumes.length > 0,
|
|
5107
|
+
configured: volumeIssues.length === 0,
|
|
5108
|
+
ready: volumeIssues.length === 0,
|
|
5109
|
+
verified: volumeIssues.length === 0,
|
|
5110
|
+
expected: { name: expectedVolumeName, mountPath: service.volumeMountPath, count: 1 },
|
|
5323
5111
|
observed: mountedVolume ? {
|
|
5324
5112
|
name: mountedVolume.name,
|
|
5325
5113
|
mountPath: service.volumeMountPath,
|
|
5114
|
+
count: mountedVolumes.length,
|
|
5326
5115
|
source: "api"
|
|
5327
|
-
} :
|
|
5328
|
-
issues:
|
|
5116
|
+
} : mountedVolumes.map((volume) => ({ name: volume.name, mountPath: service.volumeMountPath })),
|
|
5117
|
+
issues: volumeIssues
|
|
5329
5118
|
}));
|
|
5330
5119
|
}
|
|
5331
5120
|
for (const [key, value] of Object.entries(sync.variables)) {
|
|
@@ -5373,7 +5162,7 @@ async function verifyRailwayUnit(input) {
|
|
|
5373
5162
|
}
|
|
5374
5163
|
function railwayVerificationMaySettle(verification) {
|
|
5375
5164
|
return verification.checks.some(
|
|
5376
|
-
(check) => !check.verified && (check.key === "railway.instance" || check.key.startsWith("railway.instance.") || check.key === "railway.volume:data")
|
|
5165
|
+
(check) => !check.verified && (check.key === "railway.instance" || check.key.startsWith("railway.instance.") || check.key === "railway.service.source-mode" || check.key === "railway.volume:data")
|
|
5377
5166
|
);
|
|
5378
5167
|
}
|
|
5379
5168
|
function railwayStartCommandMatches(serviceKey, observed, expected) {
|
|
@@ -5400,13 +5189,54 @@ function railwayStartCommandMatches(serviceKey, observed, expected) {
|
|
|
5400
5189
|
}
|
|
5401
5190
|
return remainder === normalizedExpected;
|
|
5402
5191
|
}
|
|
5403
|
-
function
|
|
5192
|
+
function railwayProjectServiceKeys(input, scope, configuredService) {
|
|
5193
|
+
return configuredRailwayServicesForInput(input, scope).filter((candidate) => candidate.projectName === configuredService.projectName && candidate.railwayEnvironment === configuredService.railwayEnvironment).map((candidate) => candidate.key);
|
|
5194
|
+
}
|
|
5195
|
+
function railwayUnitServiceIdentity(input) {
|
|
5196
|
+
return String(input.unit.metadata.serviceInstanceKey ?? input.unit.spec.serviceName ?? input.unit.metadata.serviceKey ?? "").trim();
|
|
5197
|
+
}
|
|
5198
|
+
function configuredRailwayServiceForUnit(input, scope) {
|
|
5199
|
+
const identity = railwayUnitServiceIdentity(input);
|
|
5200
|
+
const serviceKey = String(input.unit.metadata.serviceKey ?? "").trim();
|
|
5201
|
+
return configuredRailwayServicesForInput(input, scope).find((candidate) => candidate.instanceKey === identity || candidate.serviceName === identity || candidate.key === identity || candidate.key === serviceKey) ?? null;
|
|
5202
|
+
}
|
|
5203
|
+
function findRailwayTopologyEntry(topology, identity) {
|
|
5204
|
+
return topology.services.get(identity) ?? [...topology.services.values()].find((entry) => entry.configuredService.instanceKey === identity || entry.configuredService.serviceName === identity || entry.configuredService.key === identity) ?? null;
|
|
5205
|
+
}
|
|
5206
|
+
async function resolveRailwayUnitTopology(input, scope, options) {
|
|
5207
|
+
const configuredService = configuredRailwayServiceForUnit(input, scope);
|
|
5208
|
+
const serviceKeys = configuredService ? railwayProjectServiceKeys(input, scope, configuredService) : [String(input.unit.metadata.serviceKey ?? "").trim()];
|
|
5209
|
+
const projectKey = configuredService ? `${configuredService.projectName}:${configuredService.railwayEnvironment}` : serviceKeys.join(",");
|
|
5210
|
+
const load = () => resolveRailwayTopologyForScope(input, scope, {
|
|
5211
|
+
refresh: options.refresh,
|
|
5212
|
+
serviceKeys,
|
|
5213
|
+
includeInstances: options.includeInstances,
|
|
5214
|
+
includeVariables: options.includeVariables
|
|
5215
|
+
});
|
|
5216
|
+
return options.cacheSuffix ? await providerCache(input, `railway:unit-topology:${scope}:${projectKey}:${options.cacheSuffix}`, load) : await load();
|
|
5217
|
+
}
|
|
5218
|
+
async function buildRailwayDiff(input, observed) {
|
|
5219
|
+
const scope = input.context.target.kind === "persistent" ? input.context.target.scope : "staging";
|
|
5220
|
+
const serviceKey = String(input.unit.metadata.serviceKey ?? "").trim();
|
|
5221
|
+
const configuredService = configuredRailwayServiceForUnit(input, scope);
|
|
5222
|
+
const projectServiceKeys = configuredService ? railwayProjectServiceKeys(input, scope, configuredService) : [];
|
|
5223
|
+
const projectPlan = configuredService ? await providerCache(input, `railway:plan:${scope}:${configuredService.projectName}:${configuredService.railwayEnvironment}`, () => syncRailwayEnvironmentForScope(input, { planOnly: true, serviceKeys: projectServiceKeys })) : null;
|
|
5224
|
+
const plannedChanges = projectPlan?.projectPlans?.flatMap((plan) => plan.changes) ?? [];
|
|
5225
|
+
const planReasons = plannedChanges.map((change) => change.summary || `${change.kind} ${change.path}`);
|
|
5404
5226
|
if (!observed.exists) {
|
|
5405
5227
|
return {
|
|
5406
5228
|
action: "create",
|
|
5407
|
-
reasons: ["service missing from configured topology"],
|
|
5229
|
+
reasons: ["service missing from configured topology", ...planReasons],
|
|
5408
5230
|
before: observed.live,
|
|
5409
|
-
after: input.unit.spec
|
|
5231
|
+
after: { ...input.unit.spec, railwayProjectChanges: plannedChanges }
|
|
5232
|
+
};
|
|
5233
|
+
}
|
|
5234
|
+
if (plannedChanges.length > 0) {
|
|
5235
|
+
return {
|
|
5236
|
+
action: "update",
|
|
5237
|
+
reasons: planReasons,
|
|
5238
|
+
before: observed.live,
|
|
5239
|
+
after: { ...input.unit.spec, railwayProjectChanges: plannedChanges }
|
|
5410
5240
|
};
|
|
5411
5241
|
}
|
|
5412
5242
|
return {
|
|
@@ -5419,9 +5249,9 @@ function buildRailwayDiff(input, observed) {
|
|
|
5419
5249
|
async function reconcileRailwayUnit(input, diff) {
|
|
5420
5250
|
const scope = input.context.target.kind === "persistent" ? input.context.target.scope : "staging";
|
|
5421
5251
|
const serviceKey = String(input.unit.metadata.serviceKey ?? "").trim();
|
|
5422
|
-
const configuredService =
|
|
5252
|
+
const configuredService = configuredRailwayServiceForUnit(input, scope);
|
|
5423
5253
|
const requiresProjectLevelSync = Boolean(configuredService);
|
|
5424
|
-
const serviceKeys =
|
|
5254
|
+
const serviceKeys = configuredService ? railwayProjectServiceKeys(input, scope, configuredService) : serviceKey ? [serviceKey] : void 0;
|
|
5425
5255
|
const cacheKey = requiresProjectLevelSync ? `railway:sync:${scope}:project:${configuredService?.projectName ?? "default"}:${configuredService?.railwayEnvironment ?? scope}` : `railway:sync:${scope}:${serviceKey || "all"}`;
|
|
5426
5256
|
let attempt = 0;
|
|
5427
5257
|
for (; ; ) {
|
|
@@ -5432,11 +5262,6 @@ async function reconcileRailwayUnit(input, diff) {
|
|
|
5432
5262
|
});
|
|
5433
5263
|
break;
|
|
5434
5264
|
} catch (error) {
|
|
5435
|
-
const message = error instanceof Error ? error.message : String(error ?? "");
|
|
5436
|
-
if (attempt >= 2 && /Problem processing request/iu.test(message)) {
|
|
5437
|
-
traceRailwayReconcile(buildRailwayEnv(input, scope), "sync:skipped-provider-error", `${serviceKey || "all"}:problem-processing-request`);
|
|
5438
|
-
break;
|
|
5439
|
-
}
|
|
5440
5265
|
if (attempt >= 2 || !isTransientRailwayReconcileError(error)) {
|
|
5441
5266
|
throw error;
|
|
5442
5267
|
}
|
|
@@ -5444,12 +5269,20 @@ async function reconcileRailwayUnit(input, diff) {
|
|
|
5444
5269
|
sleepMs(1e3 * attempt);
|
|
5445
5270
|
}
|
|
5446
5271
|
}
|
|
5447
|
-
|
|
5448
|
-
|
|
5449
|
-
|
|
5272
|
+
const postApplyTopology = await providerCache(input, `${cacheKey}:post-apply-topology`, async () => {
|
|
5273
|
+
for (const key of input.context.session.keys()) {
|
|
5274
|
+
if (key.startsWith(`railway:topology:${scope}:`) || key.startsWith(`railway:unit-topology:${scope}:`)) {
|
|
5275
|
+
input.context.session.delete(key);
|
|
5276
|
+
}
|
|
5450
5277
|
}
|
|
5451
|
-
|
|
5452
|
-
|
|
5278
|
+
return resolveRailwayUnitTopology(input, scope, {
|
|
5279
|
+
refresh: true,
|
|
5280
|
+
includeInstances: false,
|
|
5281
|
+
includeVariables: false,
|
|
5282
|
+
cacheSuffix: "post-apply"
|
|
5283
|
+
});
|
|
5284
|
+
});
|
|
5285
|
+
const refreshed = await observeRailwayUnit(input, { topology: await postApplyTopology });
|
|
5453
5286
|
return {
|
|
5454
5287
|
unit: input.unit,
|
|
5455
5288
|
observed: refreshed,
|
|
@@ -5482,7 +5315,7 @@ async function destroyRailwayUnit(input) {
|
|
|
5482
5315
|
);
|
|
5483
5316
|
if (!attached) continue;
|
|
5484
5317
|
traceRailwayReconcile(env, "destroy:volume", `${serviceName}:${volume.name ?? volume.id}:${volume.id}`);
|
|
5485
|
-
const result = await deleteRailwayVolume({ volumeId: volume.id, env });
|
|
5318
|
+
const result = await deleteRailwayVolume({ projectId, environmentId, volumeId: volume.id, env });
|
|
5486
5319
|
deletedVolumes.push({
|
|
5487
5320
|
id: volume.id,
|
|
5488
5321
|
name: volume.name ?? null,
|
|
@@ -5493,7 +5326,7 @@ async function destroyRailwayUnit(input) {
|
|
|
5493
5326
|
let deletedService = null;
|
|
5494
5327
|
if (serviceId) {
|
|
5495
5328
|
traceRailwayReconcile(env, "destroy:service", `${serviceName}:${serviceId}`);
|
|
5496
|
-
deletedService = await deleteRailwayService({ serviceId, env });
|
|
5329
|
+
deletedService = await deleteRailwayService({ projectId, environmentId, serviceId, env });
|
|
5497
5330
|
}
|
|
5498
5331
|
for (const key of input.context.session.keys()) {
|
|
5499
5332
|
if (key.startsWith(`railway:topology:${scope}:`)) {
|
|
@@ -5586,8 +5419,19 @@ function buildCustomDomainAdapter(unitType, providerId) {
|
|
|
5586
5419
|
diff(input) {
|
|
5587
5420
|
return buildAttachmentDiff(input, input.observed);
|
|
5588
5421
|
},
|
|
5589
|
-
apply(input) {
|
|
5590
|
-
|
|
5422
|
+
async apply(input) {
|
|
5423
|
+
let attempt = 0;
|
|
5424
|
+
for (; ; ) {
|
|
5425
|
+
try {
|
|
5426
|
+
return await reconcileCustomDomainUnit(input, input.diff);
|
|
5427
|
+
} catch (error) {
|
|
5428
|
+
if (providerId !== "railway" || attempt >= 2 || !isTransientRailwayReconcileError(error)) throw error;
|
|
5429
|
+
attempt += 1;
|
|
5430
|
+
process.stderr.write(`[trsd][railway][custom-domain] domain=${String(input.unit.spec.domain ?? "")} retry=${attempt}/2
|
|
5431
|
+
`);
|
|
5432
|
+
await new Promise((resolve2) => setTimeout(resolve2, 1e3 * attempt));
|
|
5433
|
+
}
|
|
5434
|
+
}
|
|
5591
5435
|
},
|
|
5592
5436
|
verify(input) {
|
|
5593
5437
|
return verifyCustomDomainUnit(input);
|