@wtfalch/foundry 0.62.0 → 0.63.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bootstrap/declare.js +4 -1
- package/dist/bootstrap/versions.d.ts +2 -0
- package/dist/index.d.ts +6 -2
- package/dist/index.js +2 -1
- package/dist/migrations/0067_requested_changes_kosmos.sql +6 -0
- package/dist/migrations/0068_platform_registry.sql +17 -0
- package/dist/native-build.d.ts +7 -27
- package/dist/native-build.js +7 -27
- package/dist/plan/archon-blue-green.d.ts +5 -0
- package/dist/plan/archon-blue-green.js +2 -0
- package/dist/plan/blue-green-apply.d.ts +5 -0
- package/dist/plan/blue-green-apply.js +197 -22
- package/dist/plan/blue-green.d.ts +61 -2
- package/dist/plan/blue-green.js +80 -6
- package/dist/plan/boule.d.ts +1 -1
- package/dist/plan/boule.js +2 -0
- package/dist/plan/broker-entries.d.ts +9 -0
- package/dist/plan/broker-entries.js +29 -0
- package/dist/plan/diff.d.ts +5 -0
- package/dist/plan/diff.js +4 -0
- package/dist/plan/edge-access.d.ts +2 -3
- package/dist/plan/edge-access.js +7 -0
- package/dist/plan/executor.d.ts +71 -1
- package/dist/plan/executor.js +426 -24
- package/dist/plan/index.d.ts +2 -0
- package/dist/plan/index.js +1 -0
- package/dist/plan/jobs-scheduler-http.d.ts +2 -0
- package/dist/plan/jobs-scheduler-http.js +11 -0
- package/dist/plan/own-limits.d.ts +6 -0
- package/dist/plan/own-limits.js +1 -0
- package/dist/plan/plan-apply.d.ts +51 -1
- package/dist/plan/plan-apply.js +298 -25
- package/dist/plan/platform-diff.d.ts +6 -0
- package/dist/plan/platform-diff.js +74 -0
- package/dist/plan/platform-org.d.ts +2 -0
- package/dist/plan/platform-providers.d.ts +46 -1
- package/dist/plan/platform-providers.js +156 -1
- package/dist/plan/platform-registry-keys.d.ts +58 -0
- package/dist/plan/platform-registry-keys.js +69 -0
- package/dist/plan/platform-registry.d.ts +67 -0
- package/dist/plan/platform-registry.js +298 -0
- package/dist/plan/release-host.d.ts +31 -2
- package/dist/plan/release-host.js +114 -6
- package/dist/plan/stage-copy-host.d.ts +104 -0
- package/dist/plan/stage-copy-host.js +259 -0
- package/dist/plan/stage-copy.js +44 -0
- package/dist/plan/stage-env.d.ts +9 -0
- package/dist/plan/stage-env.js +38 -0
- package/dist/plan/stage-guards.d.ts +57 -1
- package/dist/plan/stage-guards.js +106 -0
- package/dist/plan/stage-isolation.js +56 -0
- package/dist/plan/stage-services.d.ts +93 -0
- package/dist/plan/stage-services.js +153 -0
- package/dist/plan/stage-signin.d.ts +152 -0
- package/dist/plan/stage-signin.js +133 -0
- package/dist/plan/stage-sink.d.ts +2 -0
- package/dist/plan/stage-sink.js +6 -1
- package/dist/platform.d.ts +7 -1
- package/dist/platform.js +1 -0
- package/dist/release/decide.d.ts +12 -0
- package/dist/release/decide.js +37 -2
- package/dist/release/run.d.ts +71 -1
- package/dist/release/run.js +92 -8
- package/dist/release/state.d.ts +14 -1
- package/dist/release/state.js +43 -0
- package/dist/release/types.d.ts +20 -0
- package/dist/release/types.js +2 -0
- package/dist/requested-changes.d.ts +11 -2
- package/dist/requested-changes.js +11 -0
- package/dist/store/postgres.js +8 -0
- package/dist/store/schema.d.ts +51 -0
- package/dist/store/schema.js +4 -0
- package/package.json +3 -2
- package/versions.digests.json +1 -0
- package/versions.json +1 -0
|
@@ -0,0 +1,104 @@
|
|
|
1
|
+
import type { StageCopyOutcome } from '../release/run.js';
|
|
2
|
+
import type { BlueGreenSubject } from './blue-green.js';
|
|
3
|
+
import type { ReleaseContext } from './release-host.js';
|
|
4
|
+
import { type StageCopyPortDeps } from './stage-copy-port.js';
|
|
5
|
+
import { type StageCopyCoolify, type StageCopyPort, type StageCopyTimeouts } from './stage-copy.js';
|
|
6
|
+
/** What the host supplies for the copy (`FoundryStepExecutorPorts.stageCopy`). Foundry builds the port itself. */
|
|
7
|
+
export interface StageCopyHostConfig {
|
|
8
|
+
fetchDump: StageCopyPortDeps['fetchDump'];
|
|
9
|
+
/** The registered backup storage the production schedule saves to. */
|
|
10
|
+
storageUuid: string;
|
|
11
|
+
}
|
|
12
|
+
/** Everything one subject's copy needs, built by the executor that knows the subject's recorded keys. */
|
|
13
|
+
export interface StageCopyHostPorts {
|
|
14
|
+
restore: StageCopyPort;
|
|
15
|
+
coolify: StageCopyCoolify;
|
|
16
|
+
org: {
|
|
17
|
+
slug: string;
|
|
18
|
+
};
|
|
19
|
+
storageUuid: string;
|
|
20
|
+
secret: (value: string) => void;
|
|
21
|
+
/** The name in the backup keys; goes into dump errors. */
|
|
22
|
+
databaseKey: string;
|
|
23
|
+
/** Recorded key of production's Coolify database; null: the subject has none. */
|
|
24
|
+
databaseScopeKey: string | null;
|
|
25
|
+
/** Recorded key of its backup schedule; null or unrecorded: the schedule is found live. */
|
|
26
|
+
backupScopeKey: string | null;
|
|
27
|
+
projectScopeKey: string;
|
|
28
|
+
serverScopeKey: string;
|
|
29
|
+
timeouts?: StageCopyTimeouts;
|
|
30
|
+
/** Lock length and heartbeat; tests shorten them. */
|
|
31
|
+
lockMs?: number;
|
|
32
|
+
heartbeatMs?: number;
|
|
33
|
+
now?: () => number;
|
|
34
|
+
}
|
|
35
|
+
/** A copy can take ~45 minutes: the lock outlives it, and the lease is renewed every five minutes meanwhile. */
|
|
36
|
+
export declare const STAGE_COPY_LOCK_MS: number;
|
|
37
|
+
export declare const STAGE_COPY_HEARTBEAT_MS: number;
|
|
38
|
+
/** Builds the ports for a subject, or `undefined` when the host supplies no `stageCopy`. */
|
|
39
|
+
export declare function buildStageCopyHostPorts(input: {
|
|
40
|
+
host: StageCopyHostConfig | undefined;
|
|
41
|
+
coolify: StageCopyCoolify;
|
|
42
|
+
context: Pick<ReleaseContext, 'recordedIds'>;
|
|
43
|
+
org: {
|
|
44
|
+
slug: string;
|
|
45
|
+
};
|
|
46
|
+
secret: (value: string) => void;
|
|
47
|
+
databaseKey: string;
|
|
48
|
+
databaseScopeKey: string | null;
|
|
49
|
+
backupScopeKey: string | null;
|
|
50
|
+
projectScopeKey: string;
|
|
51
|
+
serverScopeKey: string;
|
|
52
|
+
}): StageCopyHostPorts | undefined;
|
|
53
|
+
/** The recorded lock value: `<expiry ms>|<request id>|<run id>`. The run id tells a run from a later re-claim. */
|
|
54
|
+
export declare const stageCopyLockValue: (expiry: number, requestId: number | string, runId: string) => string;
|
|
55
|
+
/** The parts of a recorded lock value; absent parts are undefined. */
|
|
56
|
+
export declare function parseStageCopyLock(value: string | undefined): {
|
|
57
|
+
expiry: number;
|
|
58
|
+
requestId?: string;
|
|
59
|
+
runId?: string;
|
|
60
|
+
};
|
|
61
|
+
/**
|
|
62
|
+
* Takes the copy's own lock `bg.<id>.stagecopylock` and persists it, unless a live one is recorded. The caller
|
|
63
|
+
* holds the lease that fences the persist. A copy then runs without the org's lease, guarded by this lock alone.
|
|
64
|
+
*/
|
|
65
|
+
export declare function claimStageCopyLock(input: {
|
|
66
|
+
subjectId: string;
|
|
67
|
+
context: ReleaseContext;
|
|
68
|
+
requestId: number | string;
|
|
69
|
+
/** This run's token, written into the lock. */
|
|
70
|
+
runId: string;
|
|
71
|
+
now?: () => number;
|
|
72
|
+
lockMs?: number;
|
|
73
|
+
}): Promise<'claimed' | 'busy'>;
|
|
74
|
+
/**
|
|
75
|
+
* Runs one `copy_prod_to_stage` or `undo_stage_copy` request. Exclusion: the recorded lock
|
|
76
|
+
* `bg.<id>.stagecopylock` (`<expiry ms>|<request id>`, one hour). The caller normally took it already
|
|
77
|
+
* ({@link claimStageCopyLock}, under the org's lease) and runs this without that lease; a lock for this request
|
|
78
|
+
* that is already live counts as taken. Otherwise it is taken here, persisted before anything starts, and
|
|
79
|
+
* removed in the same persist as the result, so a second process leaves the request open while it is held.
|
|
80
|
+
* `renew` is called every five minutes to keep the lock alive and throws when it was taken over.
|
|
81
|
+
* `bg.<id>.stagecopyhandled` is written in that same persist: a crash before the request is consumed replays
|
|
82
|
+
* into a consume, not a second copy.
|
|
83
|
+
*
|
|
84
|
+
* A failure halfway leaves what `stageCopy`/`undoStageCopy` documents (the new database deleted, stage on its
|
|
85
|
+
* own database). The lock is dropped and persisted; if that persist fails the lock expires on its own.
|
|
86
|
+
* A failure after the switch (recording the result, or recording an adopted switch) is not a failure of the
|
|
87
|
+
* copy: stage runs on the new database, the records are in memory, and the caller's persist lands them. It is
|
|
88
|
+
* reported as `ok`, so the request is not retried at once. If that persist fails too (or the process dies),
|
|
89
|
+
* nothing is recorded and the lock lapses; the request then runs again, but `stageCopy` first finds the switch
|
|
90
|
+
* from stage's live env and records it (kept = the recorded database) without copying again. The guarantee is
|
|
91
|
+
* "never a second copy over a landed switch", not "always recorded on the first try".
|
|
92
|
+
* Every outcome is a reason, a step or an id: no URL, no secret, no cause text.
|
|
93
|
+
*/
|
|
94
|
+
export declare function runStageCopyRequest(input: {
|
|
95
|
+
subject: Pick<BlueGreenSubject, 'id' | 'base' | 'database'>;
|
|
96
|
+
context: ReleaseContext;
|
|
97
|
+
ports: StageCopyHostPorts | undefined;
|
|
98
|
+
renew: () => Promise<void>;
|
|
99
|
+
}, request: {
|
|
100
|
+
mode: 'copy' | 'undo';
|
|
101
|
+
requestId: number;
|
|
102
|
+
actor: string;
|
|
103
|
+
runId?: string;
|
|
104
|
+
}): Promise<StageCopyOutcome>;
|
|
@@ -0,0 +1,259 @@
|
|
|
1
|
+
import { randomUUID } from 'node:crypto';
|
|
2
|
+
import { releaseKeys } from '../release/state.js';
|
|
3
|
+
import { StaleLeaseError } from '../store.js';
|
|
4
|
+
import { bgKey } from './blue-green.js';
|
|
5
|
+
import { withTimeout } from './call-timeout.js';
|
|
6
|
+
import { createStageCopyPort } from './stage-copy-port.js';
|
|
7
|
+
import { StageCopyFailedError, StageCopyRefusedError, stageCopy, undoStageCopy, } from './stage-copy.js';
|
|
8
|
+
/** A copy can take ~45 minutes: the lock outlives it, and the lease is renewed every five minutes meanwhile. */
|
|
9
|
+
export const STAGE_COPY_LOCK_MS = 60 * 60_000;
|
|
10
|
+
export const STAGE_COPY_HEARTBEAT_MS = 5 * 60_000;
|
|
11
|
+
const CALL_MS = 30_000;
|
|
12
|
+
/** Builds the ports for a subject, or `undefined` when the host supplies no `stageCopy`. */
|
|
13
|
+
export function buildStageCopyHostPorts(input) {
|
|
14
|
+
const { host, coolify } = input;
|
|
15
|
+
if (!host)
|
|
16
|
+
return undefined;
|
|
17
|
+
const restore = createStageCopyPort({
|
|
18
|
+
// Read live on each call: production's address is never held or recorded.
|
|
19
|
+
productionUrls: async () => {
|
|
20
|
+
const uuid = input.databaseScopeKey
|
|
21
|
+
? input.context.recordedIds()[input.databaseScopeKey]
|
|
22
|
+
: undefined;
|
|
23
|
+
if (!uuid)
|
|
24
|
+
return [];
|
|
25
|
+
const database = await coolify.getDatabase(uuid);
|
|
26
|
+
return database.internal_db_url ? [database.internal_db_url] : [];
|
|
27
|
+
},
|
|
28
|
+
fetchDump: host.fetchDump,
|
|
29
|
+
});
|
|
30
|
+
return {
|
|
31
|
+
restore,
|
|
32
|
+
coolify,
|
|
33
|
+
org: input.org,
|
|
34
|
+
storageUuid: host.storageUuid,
|
|
35
|
+
secret: input.secret,
|
|
36
|
+
databaseKey: input.databaseKey,
|
|
37
|
+
databaseScopeKey: input.databaseScopeKey,
|
|
38
|
+
backupScopeKey: input.backupScopeKey,
|
|
39
|
+
projectScopeKey: input.projectScopeKey,
|
|
40
|
+
serverScopeKey: input.serverScopeKey,
|
|
41
|
+
};
|
|
42
|
+
}
|
|
43
|
+
function lockHeld(value, now) {
|
|
44
|
+
const expiry = Number((value ?? '').split('|')[0]);
|
|
45
|
+
return Number.isFinite(expiry) && expiry > now;
|
|
46
|
+
}
|
|
47
|
+
/** The recorded lock value: `<expiry ms>|<request id>|<run id>`. The run id tells a run from a later re-claim. */
|
|
48
|
+
export const stageCopyLockValue = (expiry, requestId, runId) => `${expiry}|${requestId}|${runId}`;
|
|
49
|
+
/** The parts of a recorded lock value; absent parts are undefined. */
|
|
50
|
+
export function parseStageCopyLock(value) {
|
|
51
|
+
const [expiry, requestId, runId] = (value ?? '').split('|');
|
|
52
|
+
return {
|
|
53
|
+
expiry: Number(expiry),
|
|
54
|
+
...(requestId ? { requestId } : {}),
|
|
55
|
+
...(runId ? { runId } : {}),
|
|
56
|
+
};
|
|
57
|
+
}
|
|
58
|
+
/**
|
|
59
|
+
* Takes the copy's own lock `bg.<id>.stagecopylock` and persists it, unless a live one is recorded. The caller
|
|
60
|
+
* holds the lease that fences the persist. A copy then runs without the org's lease, guarded by this lock alone.
|
|
61
|
+
*/
|
|
62
|
+
export async function claimStageCopyLock(input) {
|
|
63
|
+
const keys = releaseKeys(input.subjectId);
|
|
64
|
+
const now = input.now ?? Date.now;
|
|
65
|
+
if (lockHeld(input.context.recordedIds()[keys.stageCopyLock], now()))
|
|
66
|
+
return 'busy';
|
|
67
|
+
input.context.recordScopeId(keys.stageCopyLock, stageCopyLockValue(now() + (input.lockMs ?? STAGE_COPY_LOCK_MS), input.requestId, input.runId));
|
|
68
|
+
await input.context.persist();
|
|
69
|
+
return 'claimed';
|
|
70
|
+
}
|
|
71
|
+
/** Whether a step of the run finished after the switch: stage already runs on the other database. */
|
|
72
|
+
const switched = (record) => record.steps.some((s) => (s.step === 'switch' || s.step === 'adopt') && s.outcome === 'ok');
|
|
73
|
+
/**
|
|
74
|
+
* Runs one `copy_prod_to_stage` or `undo_stage_copy` request. Exclusion: the recorded lock
|
|
75
|
+
* `bg.<id>.stagecopylock` (`<expiry ms>|<request id>`, one hour). The caller normally took it already
|
|
76
|
+
* ({@link claimStageCopyLock}, under the org's lease) and runs this without that lease; a lock for this request
|
|
77
|
+
* that is already live counts as taken. Otherwise it is taken here, persisted before anything starts, and
|
|
78
|
+
* removed in the same persist as the result, so a second process leaves the request open while it is held.
|
|
79
|
+
* `renew` is called every five minutes to keep the lock alive and throws when it was taken over.
|
|
80
|
+
* `bg.<id>.stagecopyhandled` is written in that same persist: a crash before the request is consumed replays
|
|
81
|
+
* into a consume, not a second copy.
|
|
82
|
+
*
|
|
83
|
+
* A failure halfway leaves what `stageCopy`/`undoStageCopy` documents (the new database deleted, stage on its
|
|
84
|
+
* own database). The lock is dropped and persisted; if that persist fails the lock expires on its own.
|
|
85
|
+
* A failure after the switch (recording the result, or recording an adopted switch) is not a failure of the
|
|
86
|
+
* copy: stage runs on the new database, the records are in memory, and the caller's persist lands them. It is
|
|
87
|
+
* reported as `ok`, so the request is not retried at once. If that persist fails too (or the process dies),
|
|
88
|
+
* nothing is recorded and the lock lapses; the request then runs again, but `stageCopy` first finds the switch
|
|
89
|
+
* from stage's live env and records it (kept = the recorded database) without copying again. The guarantee is
|
|
90
|
+
* "never a second copy over a landed switch", not "always recorded on the first try".
|
|
91
|
+
* Every outcome is a reason, a step or an id: no URL, no secret, no cause text.
|
|
92
|
+
*/
|
|
93
|
+
export async function runStageCopyRequest(input, request) {
|
|
94
|
+
const { subject, context, ports } = input;
|
|
95
|
+
if (!ports)
|
|
96
|
+
return { outcome: 'refused', reason: 'no-copy-port' };
|
|
97
|
+
const id = subject.id;
|
|
98
|
+
const keys = releaseKeys(id);
|
|
99
|
+
const now = ports.now ?? Date.now;
|
|
100
|
+
const requestId = String(request.requestId);
|
|
101
|
+
const recordedNow = context.recordedIds();
|
|
102
|
+
// Its result was already persisted by an earlier run that crashed before the request was consumed.
|
|
103
|
+
if (recordedNow[keys.stageCopyHandled] === requestId) {
|
|
104
|
+
return { outcome: 'ok', detail: request.mode === 'copy' ? 'copy' : 'undo', recorded: true };
|
|
105
|
+
}
|
|
106
|
+
const lock = recordedNow[keys.stageCopyLock];
|
|
107
|
+
// Mine means this run's token, not only this request: a re-claim after the lock lapsed is another run.
|
|
108
|
+
const parsed = parseStageCopyLock(lock);
|
|
109
|
+
const mine = lockHeld(lock, now()) &&
|
|
110
|
+
request.runId !== undefined &&
|
|
111
|
+
parsed.requestId === requestId &&
|
|
112
|
+
parsed.runId === request.runId;
|
|
113
|
+
if (!mine) {
|
|
114
|
+
const claimed = await claimStageCopyLock({
|
|
115
|
+
subjectId: id,
|
|
116
|
+
context,
|
|
117
|
+
requestId,
|
|
118
|
+
runId: request.runId ?? randomUUID(),
|
|
119
|
+
now,
|
|
120
|
+
...(ports.lockMs !== undefined ? { lockMs: ports.lockMs } : {}),
|
|
121
|
+
});
|
|
122
|
+
if (claimed === 'busy')
|
|
123
|
+
return { outcome: 'busy' };
|
|
124
|
+
}
|
|
125
|
+
let stale = false;
|
|
126
|
+
const heartbeat = setInterval(() => {
|
|
127
|
+
input.renew().catch(() => {
|
|
128
|
+
stale = true;
|
|
129
|
+
});
|
|
130
|
+
}, ports.heartbeatMs ?? STAGE_COPY_HEARTBEAT_MS);
|
|
131
|
+
heartbeat.unref?.();
|
|
132
|
+
const recorded = () => context.recordedIds();
|
|
133
|
+
// The last persist of a successful run carries the result, the handled mark and the lock's removal together.
|
|
134
|
+
const finalKey = request.mode === 'copy' ? bgKey.stageCopy(id) : bgKey.stageKept(id);
|
|
135
|
+
const base = {
|
|
136
|
+
subject,
|
|
137
|
+
recorded,
|
|
138
|
+
record: (key, value) => {
|
|
139
|
+
context.recordScopeId(key, value);
|
|
140
|
+
if (key === finalKey && (request.mode === 'copy' || value === '')) {
|
|
141
|
+
context.recordScopeId(keys.stageCopyHandled, requestId);
|
|
142
|
+
context.forgetId(keys.stageCopyLock);
|
|
143
|
+
}
|
|
144
|
+
},
|
|
145
|
+
persist: async () => {
|
|
146
|
+
try {
|
|
147
|
+
await context.persist();
|
|
148
|
+
}
|
|
149
|
+
catch (error) {
|
|
150
|
+
// A persist refused because the lock is another run's must not be read as a failed record.
|
|
151
|
+
if (error instanceof StaleLeaseError)
|
|
152
|
+
stale = true;
|
|
153
|
+
throw error;
|
|
154
|
+
}
|
|
155
|
+
},
|
|
156
|
+
coolify: ports.coolify,
|
|
157
|
+
secret: ports.secret,
|
|
158
|
+
requestedBy: request.actor,
|
|
159
|
+
now,
|
|
160
|
+
...(ports.timeouts ? { timeouts: ports.timeouts } : {}),
|
|
161
|
+
};
|
|
162
|
+
let outcome;
|
|
163
|
+
try {
|
|
164
|
+
const run = request.mode === 'copy'
|
|
165
|
+
? await stageCopy({
|
|
166
|
+
...base,
|
|
167
|
+
org: ports.org,
|
|
168
|
+
production: await resolveProduction(input.subject, context, ports),
|
|
169
|
+
restore: ports.restore,
|
|
170
|
+
projectUuid: recorded()[ports.projectScopeKey],
|
|
171
|
+
serverUuid: recorded()[ports.serverScopeKey],
|
|
172
|
+
})
|
|
173
|
+
: await undoStageCopy({ ...base, restore: ports.restore });
|
|
174
|
+
outcome = {
|
|
175
|
+
outcome: 'ok',
|
|
176
|
+
detail: request.mode === 'copy' ? (run.dumpExecutionId ?? 'copy') : 'undo',
|
|
177
|
+
recorded: request.mode === 'copy',
|
|
178
|
+
};
|
|
179
|
+
}
|
|
180
|
+
catch (error) {
|
|
181
|
+
if (stale) {
|
|
182
|
+
// Another run holds the lock now: record nothing, report it below.
|
|
183
|
+
outcome = { outcome: 'busy' };
|
|
184
|
+
}
|
|
185
|
+
else if (error instanceof StageCopyFailedError && switched(error.record)) {
|
|
186
|
+
outcome = {
|
|
187
|
+
outcome: 'ok',
|
|
188
|
+
detail: request.mode === 'copy' ? (error.record.dumpExecutionId ?? 'copy') : 'undo',
|
|
189
|
+
recorded: request.mode === 'copy',
|
|
190
|
+
};
|
|
191
|
+
console.error('stage copy: stage runs on the new database but recording it failed');
|
|
192
|
+
}
|
|
193
|
+
else {
|
|
194
|
+
await dropLock(context, keys, requestId);
|
|
195
|
+
if (error instanceof StageCopyRefusedError) {
|
|
196
|
+
outcome = { outcome: 'refused', reason: error.reason };
|
|
197
|
+
}
|
|
198
|
+
else if (error instanceof StageCopyFailedError) {
|
|
199
|
+
const failed = [...error.record.steps].reverse().find((s) => s.outcome === 'failed');
|
|
200
|
+
outcome = { outcome: 'failed', step: failed?.step ?? 'unknown' };
|
|
201
|
+
}
|
|
202
|
+
else {
|
|
203
|
+
console.error(`stage copy: request failed (${error instanceof Error ? error.name : 'unknown'})`);
|
|
204
|
+
outcome = { outcome: 'failed', step: 'unexpected' };
|
|
205
|
+
}
|
|
206
|
+
}
|
|
207
|
+
}
|
|
208
|
+
finally {
|
|
209
|
+
clearInterval(heartbeat);
|
|
210
|
+
}
|
|
211
|
+
// A lock that could not be kept means another run may hold the copy now: report it, not an outcome.
|
|
212
|
+
if (stale)
|
|
213
|
+
throw new StaleLeaseError('stage copy');
|
|
214
|
+
return outcome;
|
|
215
|
+
}
|
|
216
|
+
/** Undoes the in-memory marks of a run that did not finish and persists the lock's removal. */
|
|
217
|
+
async function dropLock(context, keys, requestId) {
|
|
218
|
+
if (context.recordedIds()[keys.stageCopyHandled] === requestId) {
|
|
219
|
+
context.forgetId(keys.stageCopyHandled);
|
|
220
|
+
}
|
|
221
|
+
context.forgetId(keys.stageCopyLock);
|
|
222
|
+
try {
|
|
223
|
+
await context.persist();
|
|
224
|
+
}
|
|
225
|
+
catch (error) {
|
|
226
|
+
// The lock expires on its own; the next run waits for that.
|
|
227
|
+
console.error(`stage copy: removing the lock failed (${error instanceof Error ? error.name : 'unknown'})`);
|
|
228
|
+
}
|
|
229
|
+
}
|
|
230
|
+
async function resolveProduction(subject, context, ports) {
|
|
231
|
+
if (!subject.database || !ports.databaseScopeKey)
|
|
232
|
+
return null;
|
|
233
|
+
const recorded = context.recordedIds();
|
|
234
|
+
const databaseUuid = recorded[ports.databaseScopeKey];
|
|
235
|
+
if (!databaseUuid)
|
|
236
|
+
return null;
|
|
237
|
+
let backupUuid = ports.backupScopeKey ? recorded[ports.backupScopeKey] : undefined;
|
|
238
|
+
if (!backupUuid) {
|
|
239
|
+
let schedules;
|
|
240
|
+
try {
|
|
241
|
+
schedules = await withTimeout('stage copy: list the backup schedules', CALL_MS, () => ports.coolify.listDatabaseBackups(databaseUuid));
|
|
242
|
+
}
|
|
243
|
+
catch (error) {
|
|
244
|
+
console.error(`stage copy: reading the backup schedules failed (${error instanceof Error ? error.name : 'unknown'})`);
|
|
245
|
+
return null;
|
|
246
|
+
}
|
|
247
|
+
backupUuid = schedules.find((s) => s.enabled !== false &&
|
|
248
|
+
s.save_s3 &&
|
|
249
|
+
(s.s3_storage_uuid === undefined || s.s3_storage_uuid === ports.storageUuid))?.uuid;
|
|
250
|
+
}
|
|
251
|
+
if (!backupUuid)
|
|
252
|
+
return null;
|
|
253
|
+
return {
|
|
254
|
+
databaseKey: ports.databaseKey,
|
|
255
|
+
databaseUuid,
|
|
256
|
+
backupUuid,
|
|
257
|
+
storageUuid: ports.storageUuid,
|
|
258
|
+
};
|
|
259
|
+
}
|
package/dist/plan/stage-copy.js
CHANGED
|
@@ -217,6 +217,30 @@ async function deleteQuietly(base, cfg, uuid) {
|
|
|
217
217
|
logFailure('deleting a database', error);
|
|
218
218
|
}
|
|
219
219
|
}
|
|
220
|
+
/**
|
|
221
|
+
* The copy database stage really runs on when the records name another: a database with the copy prefix whose
|
|
222
|
+
* address is stage's live owner URL. Unreadable databases do not count. `undefined`: none.
|
|
223
|
+
*/
|
|
224
|
+
async function switchedWithoutRecord(base, cfg, recordedDbId, liveOwnerUrl) {
|
|
225
|
+
const live = dbIdentity(liveOwnerUrl);
|
|
226
|
+
if (live === null)
|
|
227
|
+
return undefined;
|
|
228
|
+
const prefix = COPY_PREFIX(base.subject.base);
|
|
229
|
+
const listed = await withTimeout('stage copy: list databases', cfg.callMs, () => base.coolify.listDatabases());
|
|
230
|
+
for (const database of listed) {
|
|
231
|
+
if (database.uuid === recordedDbId || !database.name.startsWith(prefix))
|
|
232
|
+
continue;
|
|
233
|
+
try {
|
|
234
|
+
const { internal_db_url: url } = await withTimeout('stage copy: read a database', cfg.callMs, () => base.coolify.getDatabase(database.uuid));
|
|
235
|
+
if (url !== undefined && dbIdentity(url) === live)
|
|
236
|
+
return database.uuid;
|
|
237
|
+
}
|
|
238
|
+
catch (error) {
|
|
239
|
+
logFailure('reading a database', error);
|
|
240
|
+
}
|
|
241
|
+
}
|
|
242
|
+
return undefined;
|
|
243
|
+
}
|
|
220
244
|
function checkInput(base) {
|
|
221
245
|
requireText('requestedBy', base.requestedBy);
|
|
222
246
|
requireText('subject.id', base.subject.id);
|
|
@@ -297,6 +321,26 @@ export async function stageCopy(input) {
|
|
|
297
321
|
const run = new Run(input.requestedBy, input.now ?? Date.now, stageDbId);
|
|
298
322
|
const pgName = pgNameOf(subject.base);
|
|
299
323
|
const runtimeRole = `${pgName}_rt`;
|
|
324
|
+
// An earlier run may have switched stage and lost its records (the persist failed, or the process died). Stage
|
|
325
|
+
// then runs on a copy database the records do not name. Record that switch instead of copying again, so the
|
|
326
|
+
// database stage ran on stays the undo target and no second copy is made.
|
|
327
|
+
const earlierId = await switchedWithoutRecord(input, cfg, stageDbId, liveOwnerUrl);
|
|
328
|
+
if (earlierId) {
|
|
329
|
+
run.liveDatabaseId = earlierId;
|
|
330
|
+
run.ok('adopt');
|
|
331
|
+
input.record(bgKey.stageKept(id), stageDbId);
|
|
332
|
+
input.record(bgKey.stageDatabase(id), earlierId);
|
|
333
|
+
input.record(bgKey.stageCopy(id), `${new Date(run.now()).toISOString()}|adopted|ok`);
|
|
334
|
+
try {
|
|
335
|
+
await input.persist();
|
|
336
|
+
}
|
|
337
|
+
catch (error) {
|
|
338
|
+
logFailure('recording the database stage runs on', error);
|
|
339
|
+
throw run.fail('record', 'stage runs on the new database but recording it failed');
|
|
340
|
+
}
|
|
341
|
+
run.ok('record');
|
|
342
|
+
return run.record();
|
|
343
|
+
}
|
|
300
344
|
let dump;
|
|
301
345
|
try {
|
|
302
346
|
dump = await dumpNow(coolify, {
|
package/dist/plan/stage-env.d.ts
CHANGED
|
@@ -15,8 +15,17 @@ export interface StageSignIn {
|
|
|
15
15
|
}
|
|
16
16
|
/** The names a stage's own sign-in sets; a change to one of them redeploys stage. */
|
|
17
17
|
export declare const STAGE_SIGN_IN_NAMES: readonly string[];
|
|
18
|
+
/**
|
|
19
|
+
* Agora's stage config: production's `AGORA_CONFIG` (written by `desiredAgoraEnv`, so the shape follows the
|
|
20
|
+
* builder production uses) with the stage client id, chat and calls off, and only the fields named here. Files
|
|
21
|
+
* ids, the in-app sign-in flag and any other field production's builder adds later are left out, so nothing
|
|
22
|
+
* that points at production leaks through. `null` when production's value is not a JSON object.
|
|
23
|
+
*/
|
|
24
|
+
export declare function stageAgoraConfig(prodConfig: string, stageClientId: string): string | null;
|
|
18
25
|
export interface StageEnvInput {
|
|
19
26
|
kind: StageKind;
|
|
27
|
+
/** Agora only: the id of the stage's own sign-in client. Without it an Agora stage gets no `AGORA_CONFIG`. */
|
|
28
|
+
stageClientId?: string;
|
|
20
29
|
/** Where the stage app is reached: its own container address, since stage has no hostname. */
|
|
21
30
|
appUrl: string;
|
|
22
31
|
logDir: string;
|
package/dist/plan/stage-env.js
CHANGED
|
@@ -25,6 +25,36 @@ export const STAGE_SIGN_IN_NAMES = [
|
|
|
25
25
|
];
|
|
26
26
|
/** The shortest `AUTH_APP_KEY` the broker accepts. */
|
|
27
27
|
const MIN_APP_KEY_LENGTH = 32;
|
|
28
|
+
/**
|
|
29
|
+
* Agora's stage config: production's `AGORA_CONFIG` (written by `desiredAgoraEnv`, so the shape follows the
|
|
30
|
+
* builder production uses) with the stage client id, chat and calls off, and only the fields named here. Files
|
|
31
|
+
* ids, the in-app sign-in flag and any other field production's builder adds later are left out, so nothing
|
|
32
|
+
* that points at production leaks through. `null` when production's value is not a JSON object.
|
|
33
|
+
*/
|
|
34
|
+
export function stageAgoraConfig(prodConfig, stageClientId) {
|
|
35
|
+
let parsed;
|
|
36
|
+
try {
|
|
37
|
+
parsed = JSON.parse(prodConfig);
|
|
38
|
+
}
|
|
39
|
+
catch {
|
|
40
|
+
return null;
|
|
41
|
+
}
|
|
42
|
+
if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed))
|
|
43
|
+
return null;
|
|
44
|
+
const prod = parsed;
|
|
45
|
+
const keep = (name) => prod[name] === undefined ? {} : { [name]: prod[name] };
|
|
46
|
+
return JSON.stringify({
|
|
47
|
+
...keep('name'),
|
|
48
|
+
...keep('theme'),
|
|
49
|
+
...keep('issuer'),
|
|
50
|
+
clientId: stageClientId,
|
|
51
|
+
...keep('organisationId'),
|
|
52
|
+
...keep('mailProjectId'),
|
|
53
|
+
...keep('messagingProjectId'),
|
|
54
|
+
chat: { enabled: false },
|
|
55
|
+
calls: { enabled: false },
|
|
56
|
+
});
|
|
57
|
+
}
|
|
28
58
|
/**
|
|
29
59
|
* The env of a stage app, built from the kind's allowlist and never from a copy of the live app's env. Every
|
|
30
60
|
* name is allowed and none is denied, or this throws a fixed text with the name.
|
|
@@ -60,6 +90,14 @@ export function buildStageEnv(input) {
|
|
|
60
90
|
}
|
|
61
91
|
for (const { key, value } of input.signIn?.env ?? [])
|
|
62
92
|
set(key, value);
|
|
93
|
+
// Agora's stage config is production's with the stage client; `SIGNIN_UPSTREAM` is never written: it names
|
|
94
|
+
// production's login, and the stage host is not bound there, so stage signs in through the issuer.
|
|
95
|
+
const prodAgoraConfig = prod.get('AGORA_CONFIG');
|
|
96
|
+
if (kind === 'agora' && prodAgoraConfig !== undefined && input.stageClientId) {
|
|
97
|
+
const config = stageAgoraConfig(prodAgoraConfig, input.stageClientId);
|
|
98
|
+
if (config)
|
|
99
|
+
set('AGORA_CONFIG', config);
|
|
100
|
+
}
|
|
63
101
|
for (const key of MINTED) {
|
|
64
102
|
const wanted = prod.has(key) || (input.signIn !== undefined && key === 'AUTH_COOKIE_SECRET');
|
|
65
103
|
if (wanted && isAllowed(kind, key))
|
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
import { type StageServicesPort } from './stage-services.js';
|
|
2
|
+
import { type StageSignInPort } from './stage-signin.js';
|
|
1
3
|
/**
|
|
2
4
|
* The guards that must hold before stage gets an exact copy of production (campaign T3 section 5, G2 and G3).
|
|
3
5
|
* Each is checked here and recorded under a `bg.<id>.…` scope key only after it was verified live. Nothing in
|
|
@@ -104,14 +106,22 @@ export declare function stageProbesRecord(result: StageProbeResult, isolationHas
|
|
|
104
106
|
export interface StageGate {
|
|
105
107
|
/** The stage host the gate covers. */
|
|
106
108
|
hostname: string;
|
|
107
|
-
/**
|
|
109
|
+
/**
|
|
110
|
+
* Makes the gate (no bypass and no service-token rule) and returns its id. `membersHash` is the hash of the
|
|
111
|
+
* list it was made with, present only when the members were read; recorded as `bg.<id>.stagegatemembers`.
|
|
112
|
+
*/
|
|
108
113
|
create: () => Promise<{
|
|
109
114
|
id: string;
|
|
115
|
+
membersHash?: string;
|
|
110
116
|
}>;
|
|
111
117
|
}
|
|
112
118
|
export interface StageGuardDeps {
|
|
113
119
|
probes?: StageProbePort;
|
|
120
|
+
/** Stage's own sign-in client, broker entry key and app key. Absent: not set up, and the copy waits. */
|
|
121
|
+
signIn?: StageSignInPort;
|
|
114
122
|
gate?: StageGate;
|
|
123
|
+
/** Stage's own files, ai and jobs organisations and keys. A service with no port is recorded absent, never minted. */
|
|
124
|
+
services?: StageServicesPort;
|
|
115
125
|
}
|
|
116
126
|
/** The key of the stage gate in the company (a DNS label, like a declared gate's). */
|
|
117
127
|
export declare const stageGateKey: (id: string) => string;
|
|
@@ -122,3 +132,49 @@ export declare function stageGuardsReady(recorded: Readonly<Record<string, strin
|
|
|
122
132
|
ready: boolean;
|
|
123
133
|
missing: string[];
|
|
124
134
|
};
|
|
135
|
+
/** Milliseconds the host may take to name the people holding `releases:preview`. */
|
|
136
|
+
export declare const STAGE_MEMBERS_TIMEOUT_MS = 10000;
|
|
137
|
+
/** What the host answers for the people who may open stage: `releases:preview` holders of `orgId` (`platform` for Archon). */
|
|
138
|
+
export type ReleasePreviewMembers = (input: {
|
|
139
|
+
orgId: string;
|
|
140
|
+
}) => Promise<{
|
|
141
|
+
emails: string[];
|
|
142
|
+
}>;
|
|
143
|
+
/** Trimmed, lower-case, unique and sorted; anything that is not an email address is dropped. */
|
|
144
|
+
export declare function normaliseEmails(list: readonly unknown[]): string[];
|
|
145
|
+
/** The hash recorded for a sorted list. */
|
|
146
|
+
export declare const membersHash: (sorted: readonly string[]) => string;
|
|
147
|
+
/**
|
|
148
|
+
* The stage gate's list: `owners` always, plus the people `read` names. `members` is false when the read failed,
|
|
149
|
+
* timed out or the host has no port; the list is then the owners alone. One fixed log line names the subject and
|
|
150
|
+
* the error name, never an address.
|
|
151
|
+
*/
|
|
152
|
+
export declare function stageGateEmails(input: {
|
|
153
|
+
label: string;
|
|
154
|
+
subjectId: string;
|
|
155
|
+
orgId: string;
|
|
156
|
+
owners: readonly string[];
|
|
157
|
+
read?: ReleasePreviewMembers;
|
|
158
|
+
timeoutMs?: number;
|
|
159
|
+
}): Promise<{
|
|
160
|
+
emails: string[];
|
|
161
|
+
members: boolean;
|
|
162
|
+
}>;
|
|
163
|
+
/**
|
|
164
|
+
* One release tick for one subject whose stage gate is recorded: the gate's list becomes owners plus the people
|
|
165
|
+
* holding `releases:preview`. Nothing is done while the read fails, the list is unchanged since the last write
|
|
166
|
+
* (`bg.<id>.stagegatemembers`) or the Access call fails; the hash is recorded only after the Access call
|
|
167
|
+
* succeeded. Never throws.
|
|
168
|
+
*/
|
|
169
|
+
export declare function syncStageGateMembers(input: {
|
|
170
|
+
label: string;
|
|
171
|
+
subjectId: string;
|
|
172
|
+
orgId: string;
|
|
173
|
+
owners: readonly string[];
|
|
174
|
+
read?: ReleasePreviewMembers;
|
|
175
|
+
recorded: Readonly<Record<string, string>>;
|
|
176
|
+
record: (key: string, value: string) => void;
|
|
177
|
+
persist: () => Promise<void>;
|
|
178
|
+
setPeople?: (gateId: string, emails: string[]) => Promise<void>;
|
|
179
|
+
timeoutMs?: number;
|
|
180
|
+
}): Promise<void>;
|