@wtfalch/foundry 0.42.1 → 0.44.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bootstrap/versions.d.ts +2 -0
- package/dist/index.d.ts +8 -4
- package/dist/index.js +4 -2
- package/dist/migrations/0055_release_requests.sql +20 -0
- package/dist/migrations/0056_platform_archon_blue_green.sql +7 -0
- package/dist/migrations/0057_org_customers.sql +29 -0
- package/dist/org.d.ts +55 -0
- package/dist/org.js +90 -27
- package/dist/plan/archon-blue-green.d.ts +54 -0
- package/dist/plan/archon-blue-green.js +72 -0
- package/dist/plan/blue-green-apply.d.ts +82 -0
- package/dist/plan/blue-green-apply.js +360 -0
- package/dist/plan/blue-green.d.ts +223 -0
- package/dist/plan/blue-green.js +444 -0
- package/dist/plan/boule.d.ts +43 -2
- package/dist/plan/boule.js +48 -1
- package/dist/plan/broker-entries.d.ts +28 -0
- package/dist/plan/broker-entries.js +46 -1
- package/dist/plan/cloudflared-connector.d.ts +26 -0
- package/dist/plan/cloudflared-connector.js +37 -0
- package/dist/plan/diff.d.ts +20 -0
- package/dist/plan/diff.js +207 -20
- package/dist/plan/executor.d.ts +134 -2
- package/dist/plan/executor.js +517 -25
- package/dist/plan/hostnames.d.ts +7 -0
- package/dist/plan/hostnames.js +17 -2
- package/dist/plan/locked-hostnames.d.ts +11 -0
- package/dist/plan/locked-hostnames.js +13 -0
- package/dist/plan/mcp.d.ts +57 -0
- package/dist/plan/mcp.js +131 -0
- package/dist/plan/org-names.d.ts +2 -2
- package/dist/plan/org-names.js +16 -1
- package/dist/plan/plan-apply.d.ts +77 -0
- package/dist/plan/plan-apply.js +267 -5
- package/dist/plan/platform-diff.d.ts +9 -0
- package/dist/plan/platform-diff.js +24 -1
- package/dist/plan/platform-providers.d.ts +15 -1
- package/dist/plan/platform-providers.js +90 -5
- package/dist/plan/proxy-labels.d.ts +2 -2
- package/dist/plan/proxy-labels.js +14 -4
- package/dist/plan/release-host.d.ts +59 -0
- package/dist/plan/release-host.js +239 -0
- package/dist/plan/remote-database.d.ts +124 -0
- package/dist/plan/remote-database.js +373 -0
- package/dist/plan/worker-app-database.d.ts +148 -0
- package/dist/plan/worker-app-database.js +424 -0
- package/dist/plan/worker-app.d.ts +320 -0
- package/dist/plan/worker-app.js +792 -0
- package/dist/plan/zitadel-machine-user.d.ts +13 -1
- package/dist/plan/zitadel-machine-user.js +80 -0
- package/dist/platform.d.ts +11 -1
- package/dist/platform.js +4 -1
- package/dist/release/decide.d.ts +95 -0
- package/dist/release/decide.js +242 -0
- package/dist/release/health.d.ts +48 -0
- package/dist/release/health.js +61 -0
- package/dist/release/run.d.ts +40 -0
- package/dist/release/run.js +56 -0
- package/dist/release/state.d.ts +64 -0
- package/dist/release/state.js +126 -0
- package/dist/release/types.d.ts +54 -0
- package/dist/release/types.js +4 -0
- package/dist/requested-changes.d.ts +12 -3
- package/dist/requested-changes.js +22 -0
- package/dist/store/postgres.d.ts +8 -0
- package/dist/store/postgres.js +67 -8
- package/dist/store/schema.d.ts +188 -0
- package/dist/store/schema.js +15 -0
- package/dist/worker-app-config.d.ts +151 -0
- package/dist/worker-app-config.js +438 -0
- package/package.json +1 -1
- package/versions.digests.json +2 -1
- package/versions.json +2 -1
|
@@ -76,6 +76,18 @@ export declare class ZitadelLoginClientRoleConflictError extends Error {
|
|
|
76
76
|
readonly roles: readonly string[];
|
|
77
77
|
constructor(userId: string, roles: readonly string[]);
|
|
78
78
|
}
|
|
79
|
+
/** The read-only organisation role a directory reader holds. It lists the organisation's users and changes nothing. */
|
|
80
|
+
export declare const ORG_DIRECTORY_READER_ROLE = "ORG_OWNER_VIEWER";
|
|
81
|
+
/**
|
|
82
|
+
* C3: thrown by `mintOrgDirectoryReader` when the user already exists with an organisation role other
|
|
83
|
+
* than the read-only one. That user was not made by foundry; granting it more or handing out its PAT
|
|
84
|
+
* would widen access nobody asked for. A person resolves it by hand.
|
|
85
|
+
*/
|
|
86
|
+
export declare class ZitadelDirectoryRoleConflictError extends Error {
|
|
87
|
+
readonly userId: string;
|
|
88
|
+
readonly roles: readonly string[];
|
|
89
|
+
constructor(userId: string, roles: readonly string[]);
|
|
90
|
+
}
|
|
79
91
|
export interface ZitadelMinterOptions {
|
|
80
92
|
/** The new instance's base URL, e.g. `https://auth.companyincluded.com`. */
|
|
81
93
|
url: string;
|
|
@@ -84,4 +96,4 @@ export interface ZitadelMinterOptions {
|
|
|
84
96
|
fetchImpl?: typeof fetch;
|
|
85
97
|
instanceHost?: string;
|
|
86
98
|
}
|
|
87
|
-
export declare function createZitadelAdminMinter(options: ZitadelMinterOptions): Pick<ZitadelPort, 'mintAdminMachineUser' | 'mintLoginClientMachineUser' | 'listLoginClientPats' | 'reserveUserName'>;
|
|
99
|
+
export declare function createZitadelAdminMinter(options: ZitadelMinterOptions): Pick<ZitadelPort, 'mintAdminMachineUser' | 'mintLoginClientMachineUser' | 'listLoginClientPats' | 'reserveUserName' | 'mintOrgDirectoryReader' | 'revokeOrgDirectoryReader'>;
|
|
@@ -84,6 +84,23 @@ export class ZitadelLoginClientRoleConflictError extends Error {
|
|
|
84
84
|
this.name = 'ZitadelLoginClientRoleConflictError';
|
|
85
85
|
}
|
|
86
86
|
}
|
|
87
|
+
/** The read-only organisation role a directory reader holds. It lists the organisation's users and changes nothing. */
|
|
88
|
+
export const ORG_DIRECTORY_READER_ROLE = 'ORG_OWNER_VIEWER';
|
|
89
|
+
/**
|
|
90
|
+
* C3: thrown by `mintOrgDirectoryReader` when the user already exists with an organisation role other
|
|
91
|
+
* than the read-only one. That user was not made by foundry; granting it more or handing out its PAT
|
|
92
|
+
* would widen access nobody asked for. A person resolves it by hand.
|
|
93
|
+
*/
|
|
94
|
+
export class ZitadelDirectoryRoleConflictError extends Error {
|
|
95
|
+
userId;
|
|
96
|
+
roles;
|
|
97
|
+
constructor(userId, roles) {
|
|
98
|
+
super(`ZITADEL machine user ${userId} already exists with organisation role(s) [${roles.join(', ') || 'none'}], not only [${ORG_DIRECTORY_READER_ROLE}] -- refusing to reuse it as a directory reader`);
|
|
99
|
+
this.userId = userId;
|
|
100
|
+
this.roles = roles;
|
|
101
|
+
this.name = 'ZitadelDirectoryRoleConflictError';
|
|
102
|
+
}
|
|
103
|
+
}
|
|
87
104
|
export function createZitadelAdminMinter(options) {
|
|
88
105
|
const fetchImpl = options.fetchImpl ?? globalThis.fetch;
|
|
89
106
|
async function call(method, path, body, orgId) {
|
|
@@ -263,7 +280,70 @@ export function createZitadelAdminMinter(options) {
|
|
|
263
280
|
}
|
|
264
281
|
return { userId };
|
|
265
282
|
}
|
|
283
|
+
/**
|
|
284
|
+
* C3: a machine user inside one ZITADEL organisation with the read-only org role
|
|
285
|
+
* {@link ORG_DIRECTORY_READER_ROLE} and nothing else, plus a fresh PAT. Every PAT it already holds is
|
|
286
|
+
* revoked first, so a rerun leaves exactly one live token. An existing user that holds any other
|
|
287
|
+
* org role is refused, not reused. The calls follow ZITADEL's published v1 management API and have
|
|
288
|
+
* not been run against a live instance.
|
|
289
|
+
*/
|
|
290
|
+
async function mintOrgDirectoryReader(orgId, input) {
|
|
291
|
+
const found = await call('POST', '/management/v1/users/_search', {
|
|
292
|
+
queries: [
|
|
293
|
+
{ userNameQuery: { userName: input.userName, method: 'TEXT_QUERY_METHOD_EQUALS' } },
|
|
294
|
+
],
|
|
295
|
+
}, orgId);
|
|
296
|
+
const existing = found.result?.[0]?.id;
|
|
297
|
+
let userId = existing;
|
|
298
|
+
if (!userId) {
|
|
299
|
+
const created = await call('POST', '/management/v1/users/machine', {
|
|
300
|
+
userName: input.userName,
|
|
301
|
+
name: input.name,
|
|
302
|
+
accessTokenType: 'ACCESS_TOKEN_TYPE_BEARER',
|
|
303
|
+
}, orgId);
|
|
304
|
+
userId = typeof created.userId === 'string' ? created.userId : undefined;
|
|
305
|
+
}
|
|
306
|
+
if (!userId)
|
|
307
|
+
throw new Error('ZITADEL did not return the machine user id');
|
|
308
|
+
if (existing) {
|
|
309
|
+
const members = await call('POST', '/management/v1/orgs/me/members/_search', { queries: [{ userIdQuery: { userId } }] }, orgId);
|
|
310
|
+
const roles = (members.result ?? [])
|
|
311
|
+
.filter((m) => m.userId === userId)
|
|
312
|
+
.flatMap((m) => m.roles ?? []);
|
|
313
|
+
if (roles.some((role) => role !== ORG_DIRECTORY_READER_ROLE)) {
|
|
314
|
+
throw new ZitadelDirectoryRoleConflictError(userId, roles);
|
|
315
|
+
}
|
|
316
|
+
}
|
|
317
|
+
try {
|
|
318
|
+
await call('POST', '/management/v1/orgs/me/members', { userId, roles: [ORG_DIRECTORY_READER_ROLE] }, orgId);
|
|
319
|
+
}
|
|
320
|
+
catch (e) {
|
|
321
|
+
if (!(e instanceof ZitadelHttpError && e.status === 409))
|
|
322
|
+
throw e;
|
|
323
|
+
}
|
|
324
|
+
await revokeAllPats(orgId, userId);
|
|
325
|
+
const pat = await call('POST', `/management/v1/users/${userId}/pats`, {}, orgId);
|
|
326
|
+
const patId = typeof pat.tokenId === 'string' && pat.tokenId !== '' ? pat.tokenId : undefined;
|
|
327
|
+
if (typeof pat.token !== 'string' || pat.token === '' || !patId) {
|
|
328
|
+
throw new Error(`ZITADEL returned no token for machine user ${userId}`);
|
|
329
|
+
}
|
|
330
|
+
return { userId, token: pat.token, patId };
|
|
331
|
+
}
|
|
332
|
+
/** Deletes every PAT of a user; a PAT already gone (404) is the outcome wanted. */
|
|
333
|
+
async function revokeAllPats(orgId, userId) {
|
|
334
|
+
for (const patId of await listPats(orgId, userId)) {
|
|
335
|
+
try {
|
|
336
|
+
await call('DELETE', `/management/v1/users/${userId}/pats/${patId}`, undefined, orgId);
|
|
337
|
+
}
|
|
338
|
+
catch (e) {
|
|
339
|
+
if (!(e instanceof ZitadelHttpError && e.status === 404))
|
|
340
|
+
throw e;
|
|
341
|
+
}
|
|
342
|
+
}
|
|
343
|
+
}
|
|
266
344
|
return {
|
|
345
|
+
mintOrgDirectoryReader,
|
|
346
|
+
revokeOrgDirectoryReader: revokeAllPats,
|
|
267
347
|
mintAdminMachineUser: (orgId, input) => mintMachineUser(orgId, input, 'IAM_OWNER').then(({ userId, token }) => ({ userId, token })),
|
|
268
348
|
mintLoginClientMachineUser: (orgId, input) => mintMachineUser(orgId, input, 'IAM_LOGIN_CLIENT'),
|
|
269
349
|
// #216 round 2: lets the caller (`platform-providers.ts`) re-check, after
|
package/dist/platform.d.ts
CHANGED
|
@@ -12,6 +12,16 @@ export interface PlatformService {
|
|
|
12
12
|
hostname: string;
|
|
13
13
|
gitRef: string;
|
|
14
14
|
}
|
|
15
|
+
/**
|
|
16
|
+
* `archon`'s own extra field: presence turns blue/green on for platform-archon, through the same code as a hosting
|
|
17
|
+
* entry (`plan/blue-green.ts`). The tunnel moves to the balancer only once a balancer is recorded.
|
|
18
|
+
*/
|
|
19
|
+
export interface PlatformArchonService extends PlatformService {
|
|
20
|
+
blueGreen?: {
|
|
21
|
+
steps?: number[];
|
|
22
|
+
stepHoldMinutes?: number;
|
|
23
|
+
};
|
|
24
|
+
}
|
|
15
25
|
/**
|
|
16
26
|
* `sfu`'s own extra field. A host that changes what the SFU VM is reached
|
|
17
27
|
* with (its SSH key, host key or tunnel token) sets a new `connectionRevision`
|
|
@@ -97,7 +107,7 @@ export interface DeclaredPlatform {
|
|
|
97
107
|
ai: PlatformService;
|
|
98
108
|
jobs: PlatformService;
|
|
99
109
|
integrations: PlatformService;
|
|
100
|
-
archon:
|
|
110
|
+
archon: PlatformArchonService;
|
|
101
111
|
/** #317 (F7). No deploy step reads this service's own `gitRef` yet -- `N1` is deployed by hand until it exists. */
|
|
102
112
|
edge: PlatformService;
|
|
103
113
|
/**
|
package/dist/platform.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { isIPv4 } from 'node:net';
|
|
2
|
-
import { validateDomain } from './org.js';
|
|
2
|
+
import { validateBlueGreen, validateDomain } from './org.js';
|
|
3
3
|
/** Where Archon publishes the Worker script for `tag` (a private repo; the host's `fetchArtifact` fetches it). */
|
|
4
4
|
export function uptimeArtifactUrl(tag) {
|
|
5
5
|
return `https://github.com/wtfalch/archon/releases/download/${tag}/archon-uptime-worker.js`;
|
|
@@ -69,6 +69,9 @@ export function validatePlatform(platform) {
|
|
|
69
69
|
return `${key}.gitRef must not be empty once ${key}.enabled is true`;
|
|
70
70
|
}
|
|
71
71
|
}
|
|
72
|
+
const blueGreenReason = validateBlueGreen(platform.archon.blueGreen, 'archon');
|
|
73
|
+
if (blueGreenReason)
|
|
74
|
+
return blueGreenReason;
|
|
72
75
|
const revision = platform.sfu.connectionRevision;
|
|
73
76
|
if (revision != null && !/^[A-Za-z0-9:.-]{0,64}$/.test(revision)) {
|
|
74
77
|
return 'sfu.connectionRevision must be at most 64 characters of A-Z, a-z, 0-9, ":", "." and "-"';
|
|
@@ -0,0 +1,95 @@
|
|
|
1
|
+
import { type Color, type ReleaseAction, type ReleaseSignal, type WindowState } from './types.js';
|
|
2
|
+
/** ADR 0001 rule 5: the default steps and the least time on each. */
|
|
3
|
+
export declare const DEFAULT_STEPS: readonly number[];
|
|
4
|
+
export declare const DEFAULT_STEP_HOLD_MINUTES = 60;
|
|
5
|
+
/** Rule 7: errors this share of requests, with at least HARD_FAIL_MIN_REQUESTS, roll back at any level. */
|
|
6
|
+
export declare const HARD_FAIL_ERROR_RATE = 0.9;
|
|
7
|
+
export declare const HARD_FAIL_MIN_REQUESTS = 2;
|
|
8
|
+
/** Rule 8: a relative fail needs this many requests and this many failed checks in a row. */
|
|
9
|
+
export declare const RELATIVE_MIN_REQUESTS = 20;
|
|
10
|
+
export declare const RELATIVE_FAILED_CHECKS = 3;
|
|
11
|
+
/** A relative check counts only this long after the last counted one, so runs close together (applies also run releases) are one check. */
|
|
12
|
+
export declare const CHECK_INTERVAL_MINUTES = 10;
|
|
13
|
+
/** The new color matches the baseline while its error rate is at most baseline plus this (absolute). */
|
|
14
|
+
export declare const ERROR_RATE_MARGIN = 0.02;
|
|
15
|
+
/** Rules 12 and 13: the window at 100% closes after this long, and at most this long. */
|
|
16
|
+
export declare const WINDOW_MIN_HOURS = 24;
|
|
17
|
+
export declare const WINDOW_CAP_DAYS = 7;
|
|
18
|
+
/** The new color must be running and answer on its own address within this time of the deploy. */
|
|
19
|
+
export declare const DEPLOY_TIMEOUT_MINUTES = 30;
|
|
20
|
+
export interface ActiveRelease {
|
|
21
|
+
from: Color;
|
|
22
|
+
to: Color;
|
|
23
|
+
digest: string;
|
|
24
|
+
/** Relative-fail checks in a row. */
|
|
25
|
+
fails: number;
|
|
26
|
+
/** When the last counted relative check ran (ISO). Absent: none yet in this step. */
|
|
27
|
+
checkedAt?: string | undefined;
|
|
28
|
+
}
|
|
29
|
+
export interface Recorded {
|
|
30
|
+
live: Color;
|
|
31
|
+
digests: Record<Color, string | null>;
|
|
32
|
+
stageDigest: string | null;
|
|
33
|
+
weights: Record<Color, number>;
|
|
34
|
+
window: WindowState;
|
|
35
|
+
/** Set while window.kind is step, inconclusive, window or cap. */
|
|
36
|
+
release: ActiveRelease | null;
|
|
37
|
+
held: string[];
|
|
38
|
+
}
|
|
39
|
+
export interface ColorCounts {
|
|
40
|
+
requests: number;
|
|
41
|
+
errors: number;
|
|
42
|
+
}
|
|
43
|
+
/** The reporting health read (C1), once validated. */
|
|
44
|
+
export interface HealthRead {
|
|
45
|
+
colors: Record<Color, ColorCounts>;
|
|
46
|
+
daily: {
|
|
47
|
+
days: number;
|
|
48
|
+
requestsPerDay: number;
|
|
49
|
+
errorsPerDay: number;
|
|
50
|
+
};
|
|
51
|
+
}
|
|
52
|
+
export interface Observation {
|
|
53
|
+
/** null when the health read failed: counted as inconclusive, never as a pass. */
|
|
54
|
+
health: HealthRead | null;
|
|
55
|
+
/** The new color's Coolify status. */
|
|
56
|
+
status: 'running' | 'crash_loop' | 'other';
|
|
57
|
+
/** The new color's own address, checked over HTTP. */
|
|
58
|
+
address: 'ok' | 'failed' | 'unchecked';
|
|
59
|
+
}
|
|
60
|
+
export type ReleaseEffect = {
|
|
61
|
+
kind: 'deploy';
|
|
62
|
+
color: Color;
|
|
63
|
+
digest: string;
|
|
64
|
+
} | {
|
|
65
|
+
kind: 'stop';
|
|
66
|
+
color: Color;
|
|
67
|
+
};
|
|
68
|
+
export interface Decision {
|
|
69
|
+
recorded: Recorded;
|
|
70
|
+
signal: ReleaseSignal | null;
|
|
71
|
+
effects: ReleaseEffect[];
|
|
72
|
+
/** Blocked items for the operator, fixed text. */
|
|
73
|
+
blocked: string[];
|
|
74
|
+
/** Actions that changed nothing, with a fixed reason. */
|
|
75
|
+
refused: string[];
|
|
76
|
+
}
|
|
77
|
+
export declare function normalizeSteps(steps: readonly number[] | undefined): number[];
|
|
78
|
+
/** The time a health read should cover: from the step or window start to now. */
|
|
79
|
+
export declare function healthRange(r: Recorded, now: Date): {
|
|
80
|
+
from: string;
|
|
81
|
+
to: string;
|
|
82
|
+
} | null;
|
|
83
|
+
/**
|
|
84
|
+
* The whole release decision for one run: from the recorded state, the
|
|
85
|
+
* actions requested, one observation and the time, the next recorded state.
|
|
86
|
+
* Pure: it reads nothing and writes nothing (ADR 0001 rules 5, 7-13, 16).
|
|
87
|
+
*/
|
|
88
|
+
export declare function decide(input: {
|
|
89
|
+
recorded: Recorded;
|
|
90
|
+
actions: readonly ReleaseAction[];
|
|
91
|
+
steps?: readonly number[] | undefined;
|
|
92
|
+
stepHoldMinutes?: number | undefined;
|
|
93
|
+
now: Date;
|
|
94
|
+
observation: Observation;
|
|
95
|
+
}): Decision;
|
|
@@ -0,0 +1,242 @@
|
|
|
1
|
+
import { otherColor, } from './types.js';
|
|
2
|
+
/** ADR 0001 rule 5: the default steps and the least time on each. */
|
|
3
|
+
export const DEFAULT_STEPS = [5, 10, 25, 50, 100];
|
|
4
|
+
export const DEFAULT_STEP_HOLD_MINUTES = 60;
|
|
5
|
+
/** Rule 7: errors this share of requests, with at least HARD_FAIL_MIN_REQUESTS, roll back at any level. */
|
|
6
|
+
export const HARD_FAIL_ERROR_RATE = 0.9;
|
|
7
|
+
export const HARD_FAIL_MIN_REQUESTS = 2;
|
|
8
|
+
/** Rule 8: a relative fail needs this many requests and this many failed checks in a row. */
|
|
9
|
+
export const RELATIVE_MIN_REQUESTS = 20;
|
|
10
|
+
export const RELATIVE_FAILED_CHECKS = 3;
|
|
11
|
+
/** A relative check counts only this long after the last counted one, so runs close together (applies also run releases) are one check. */
|
|
12
|
+
export const CHECK_INTERVAL_MINUTES = 10;
|
|
13
|
+
/** The new color matches the baseline while its error rate is at most baseline plus this (absolute). */
|
|
14
|
+
export const ERROR_RATE_MARGIN = 0.02;
|
|
15
|
+
/** Rules 12 and 13: the window at 100% closes after this long, and at most this long. */
|
|
16
|
+
export const WINDOW_MIN_HOURS = 24;
|
|
17
|
+
export const WINDOW_CAP_DAYS = 7;
|
|
18
|
+
/** The new color must be running and answer on its own address within this time of the deploy. */
|
|
19
|
+
export const DEPLOY_TIMEOUT_MINUTES = 30;
|
|
20
|
+
const MINUTE = 60_000;
|
|
21
|
+
const HOUR = 60 * MINUTE;
|
|
22
|
+
const DAY = 24 * HOUR;
|
|
23
|
+
export function normalizeSteps(steps) {
|
|
24
|
+
const wanted = steps && steps.length > 0 ? steps : DEFAULT_STEPS;
|
|
25
|
+
const valid = [...new Set(wanted.filter((n) => Number.isInteger(n) && n >= 1 && n <= 100))].sort((a, b) => a - b);
|
|
26
|
+
if (valid.length === 0)
|
|
27
|
+
return [...DEFAULT_STEPS];
|
|
28
|
+
if (valid[valid.length - 1] !== 100)
|
|
29
|
+
valid.push(100);
|
|
30
|
+
return valid;
|
|
31
|
+
}
|
|
32
|
+
function weightsFor(to, percent) {
|
|
33
|
+
return { [to]: percent, [otherColor(to)]: 100 - percent };
|
|
34
|
+
}
|
|
35
|
+
function rollback(r, now, reason) {
|
|
36
|
+
const rel = r.release;
|
|
37
|
+
if (!rel)
|
|
38
|
+
return r;
|
|
39
|
+
return {
|
|
40
|
+
...r,
|
|
41
|
+
live: rel.from,
|
|
42
|
+
weights: weightsFor(rel.from, 100),
|
|
43
|
+
window: { kind: 'rolled_back', at: now.toISOString(), reason },
|
|
44
|
+
release: null,
|
|
45
|
+
held: r.held.includes(rel.digest) ? r.held : [...r.held, rel.digest],
|
|
46
|
+
};
|
|
47
|
+
}
|
|
48
|
+
function rate(c) {
|
|
49
|
+
return c.requests > 0 ? c.errors / c.requests : 0;
|
|
50
|
+
}
|
|
51
|
+
function dailyRate(h) {
|
|
52
|
+
return h.daily.requestsPerDay > 0 ? h.daily.errorsPerDay / h.daily.requestsPerDay : 0;
|
|
53
|
+
}
|
|
54
|
+
/** The time a health read should cover: from the step or window start to now. */
|
|
55
|
+
export function healthRange(r, now) {
|
|
56
|
+
const w = r.window;
|
|
57
|
+
if (w.kind === 'step' || w.kind === 'inconclusive' || w.kind === 'window' || w.kind === 'cap') {
|
|
58
|
+
if (w.kind === 'inconclusive' && w.percent === 0)
|
|
59
|
+
return null;
|
|
60
|
+
return { from: w.since, to: now.toISOString() };
|
|
61
|
+
}
|
|
62
|
+
return null;
|
|
63
|
+
}
|
|
64
|
+
function applyActions(r, actions, now, out) {
|
|
65
|
+
let cur = r;
|
|
66
|
+
let promoted = false;
|
|
67
|
+
let ended = false;
|
|
68
|
+
for (const a of actions) {
|
|
69
|
+
if (a.kind === 'clear_held') {
|
|
70
|
+
cur = { ...cur, held: cur.held.filter((d) => d !== a.digest) };
|
|
71
|
+
}
|
|
72
|
+
else if (a.kind === 'promote') {
|
|
73
|
+
const idle = !cur.release;
|
|
74
|
+
const digest = cur.stageDigest;
|
|
75
|
+
if (!idle)
|
|
76
|
+
out.refused.push('promote refused: a release is running');
|
|
77
|
+
else if (!digest)
|
|
78
|
+
out.refused.push('promote refused: stage has no digest');
|
|
79
|
+
else if (cur.held.includes(digest))
|
|
80
|
+
out.refused.push('promote refused: digest is held');
|
|
81
|
+
else if (digest === cur.digests[cur.live])
|
|
82
|
+
out.refused.push('promote refused: digest is already live');
|
|
83
|
+
else {
|
|
84
|
+
const to = otherColor(cur.live);
|
|
85
|
+
cur = {
|
|
86
|
+
...cur,
|
|
87
|
+
digests: { ...cur.digests, [to]: digest },
|
|
88
|
+
window: { kind: 'inconclusive', percent: 0, since: now.toISOString() },
|
|
89
|
+
release: { from: cur.live, to, digest, fails: 0 },
|
|
90
|
+
};
|
|
91
|
+
out.effects.push({ kind: 'deploy', color: to, digest });
|
|
92
|
+
promoted = true;
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
else if (a.kind === 'rollback') {
|
|
96
|
+
if (!cur.release)
|
|
97
|
+
out.refused.push('rollback refused: no release is running');
|
|
98
|
+
else {
|
|
99
|
+
cur = rollback(cur, now, 'operator');
|
|
100
|
+
ended = true;
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
else if (a.kind === 'decide_at_cap') {
|
|
104
|
+
if (cur.window.kind !== 'cap' || !cur.release)
|
|
105
|
+
out.refused.push('decision refused: no release is at the cap');
|
|
106
|
+
else if (a.decision === 'rollback') {
|
|
107
|
+
cur = rollback(cur, now, 'operator at cap');
|
|
108
|
+
ended = true;
|
|
109
|
+
}
|
|
110
|
+
else {
|
|
111
|
+
out.effects.push({ kind: 'stop', color: cur.release.from });
|
|
112
|
+
cur = { ...cur, window: { kind: 'passed', at: now.toISOString() }, release: null };
|
|
113
|
+
ended = true;
|
|
114
|
+
}
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
return { recorded: cur, promoted, ended };
|
|
118
|
+
}
|
|
119
|
+
/**
|
|
120
|
+
* The whole release decision for one run: from the recorded state, the
|
|
121
|
+
* actions requested, one observation and the time, the next recorded state.
|
|
122
|
+
* Pure: it reads nothing and writes nothing (ADR 0001 rules 5, 7-13, 16).
|
|
123
|
+
*/
|
|
124
|
+
export function decide(input) {
|
|
125
|
+
const { now, observation } = input;
|
|
126
|
+
const steps = normalizeSteps(input.steps);
|
|
127
|
+
const holdMs = (input.stepHoldMinutes ?? DEFAULT_STEP_HOLD_MINUTES) * MINUTE;
|
|
128
|
+
const out = { effects: [], refused: [] };
|
|
129
|
+
const applied = applyActions(input.recorded, input.actions, now, out);
|
|
130
|
+
let r = applied.recorded;
|
|
131
|
+
let signal = null;
|
|
132
|
+
const blocked = [];
|
|
133
|
+
const rel = r.release;
|
|
134
|
+
if (!applied.promoted && !applied.ended && rel) {
|
|
135
|
+
const since = Date.parse('since' in r.window ? r.window.since : '');
|
|
136
|
+
const elapsed = now.getTime() - since;
|
|
137
|
+
const hard = observation.status === 'crash_loop'
|
|
138
|
+
? 'hard fail: crash loop'
|
|
139
|
+
: observation.address === 'failed'
|
|
140
|
+
? 'hard fail: address check failed'
|
|
141
|
+
: null;
|
|
142
|
+
if (r.window.kind === 'inconclusive' && r.window.percent === 0) {
|
|
143
|
+
if (hard)
|
|
144
|
+
r = rollback(r, now, hard);
|
|
145
|
+
else if (observation.status === 'running' && observation.address === 'ok') {
|
|
146
|
+
const first = steps[0];
|
|
147
|
+
r = stepTo(r, rel, first, now);
|
|
148
|
+
}
|
|
149
|
+
else if (elapsed >= DEPLOY_TIMEOUT_MINUTES * MINUTE)
|
|
150
|
+
r = rollback(r, now, 'deploy timeout');
|
|
151
|
+
}
|
|
152
|
+
else {
|
|
153
|
+
const health = observation.health;
|
|
154
|
+
const newC = health?.colors[rel.to] ?? { requests: 0, errors: 0 };
|
|
155
|
+
const atWindow = r.window.kind === 'window' || r.window.kind === 'cap';
|
|
156
|
+
let baseline = null;
|
|
157
|
+
if (health) {
|
|
158
|
+
const old = health.colors[rel.from];
|
|
159
|
+
baseline = !atWindow && old.requests > 0 ? rate(old) : dailyRate(health);
|
|
160
|
+
}
|
|
161
|
+
const hardRate = newC.requests >= HARD_FAIL_MIN_REQUESTS && rate(newC) >= HARD_FAIL_ERROR_RATE;
|
|
162
|
+
let verdict;
|
|
163
|
+
if (hard || hardRate)
|
|
164
|
+
verdict = 'hard_fail';
|
|
165
|
+
else if (!health || baseline === null || newC.requests < RELATIVE_MIN_REQUESTS)
|
|
166
|
+
verdict = 'inconclusive';
|
|
167
|
+
else
|
|
168
|
+
verdict = rate(newC) > baseline + ERROR_RATE_MARGIN ? 'fail' : 'pass';
|
|
169
|
+
signal = { requests: newC.requests, errors: newC.errors, baselineRate: baseline, verdict };
|
|
170
|
+
if (verdict === 'hard_fail') {
|
|
171
|
+
r = rollback(r, now, hard ?? 'hard fail: error rate');
|
|
172
|
+
}
|
|
173
|
+
else {
|
|
174
|
+
const spaced = !rel.checkedAt ||
|
|
175
|
+
now.getTime() - Date.parse(rel.checkedAt) >= CHECK_INTERVAL_MINUTES * MINUTE;
|
|
176
|
+
const counted = spaced && verdict !== 'inconclusive';
|
|
177
|
+
const fails = !counted ? rel.fails : verdict === 'fail' ? rel.fails + 1 : 0;
|
|
178
|
+
if (fails >= RELATIVE_FAILED_CHECKS) {
|
|
179
|
+
r = rollback(r, now, 'relative fail: failed checks in a row');
|
|
180
|
+
}
|
|
181
|
+
else {
|
|
182
|
+
r = {
|
|
183
|
+
...r,
|
|
184
|
+
release: { ...rel, fails, ...(counted ? { checkedAt: now.toISOString() } : {}) },
|
|
185
|
+
};
|
|
186
|
+
if (!atWindow) {
|
|
187
|
+
const pct = r.window.percent;
|
|
188
|
+
const holdDone = elapsed >= holdMs;
|
|
189
|
+
if (holdDone && verdict !== 'fail') {
|
|
190
|
+
const next = steps.find((s) => s > pct);
|
|
191
|
+
r = stepTo(r, rel, next, now);
|
|
192
|
+
}
|
|
193
|
+
else {
|
|
194
|
+
const kind = verdict === 'inconclusive' ? 'inconclusive' : 'step';
|
|
195
|
+
r = {
|
|
196
|
+
...r,
|
|
197
|
+
window: { kind, percent: pct, since: r.window.since },
|
|
198
|
+
};
|
|
199
|
+
}
|
|
200
|
+
}
|
|
201
|
+
else {
|
|
202
|
+
const required = Math.max(1, Math.ceil(health?.daily.requestsPerDay ?? 0));
|
|
203
|
+
const closable = health !== null &&
|
|
204
|
+
baseline !== null &&
|
|
205
|
+
elapsed >= WINDOW_MIN_HOURS * HOUR &&
|
|
206
|
+
newC.requests >= required &&
|
|
207
|
+
rate(newC) <= baseline + ERROR_RATE_MARGIN;
|
|
208
|
+
if (closable) {
|
|
209
|
+
out.effects.push({ kind: 'stop', color: rel.from });
|
|
210
|
+
r = { ...r, window: { kind: 'passed', at: now.toISOString() }, release: null };
|
|
211
|
+
}
|
|
212
|
+
else if (elapsed >= WINDOW_CAP_DAYS * DAY) {
|
|
213
|
+
r = { ...r, window: { kind: 'cap', since: new Date(since).toISOString() } };
|
|
214
|
+
}
|
|
215
|
+
}
|
|
216
|
+
}
|
|
217
|
+
}
|
|
218
|
+
}
|
|
219
|
+
}
|
|
220
|
+
if (r.window.kind === 'cap')
|
|
221
|
+
blocked.push('release window reached the cap without enough data: pass or roll back in Archon');
|
|
222
|
+
return { recorded: r, signal, effects: out.effects, blocked, refused: out.refused };
|
|
223
|
+
}
|
|
224
|
+
function stepTo(r, rel, percent, now) {
|
|
225
|
+
const at = now.toISOString();
|
|
226
|
+
const fresh = { from: rel.from, to: rel.to, digest: rel.digest, fails: 0 };
|
|
227
|
+
if (percent >= 100) {
|
|
228
|
+
return {
|
|
229
|
+
...r,
|
|
230
|
+
live: rel.to,
|
|
231
|
+
weights: weightsFor(rel.to, 100),
|
|
232
|
+
window: { kind: 'window', since: at },
|
|
233
|
+
release: fresh,
|
|
234
|
+
};
|
|
235
|
+
}
|
|
236
|
+
return {
|
|
237
|
+
...r,
|
|
238
|
+
weights: weightsFor(rel.to, percent),
|
|
239
|
+
window: { kind: 'step', percent, since: at },
|
|
240
|
+
release: fresh,
|
|
241
|
+
};
|
|
242
|
+
}
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
import { z } from 'zod';
|
|
2
|
+
import type { HealthRead } from './decide.js';
|
|
3
|
+
/** The reporting health read has this long to answer. */
|
|
4
|
+
export declare const HEALTH_READ_TIMEOUT_MS = 10000;
|
|
5
|
+
/** The color's own address check has this long to answer. */
|
|
6
|
+
export declare const ADDRESS_CHECK_TIMEOUT_MS = 10000;
|
|
7
|
+
/** C1's 200 body. */
|
|
8
|
+
export declare const healthBody: z.ZodObject<{
|
|
9
|
+
colors: z.ZodObject<{
|
|
10
|
+
blue: z.ZodObject<{
|
|
11
|
+
requests: z.ZodNumber;
|
|
12
|
+
errors: z.ZodNumber;
|
|
13
|
+
}, z.core.$strip>;
|
|
14
|
+
green: z.ZodObject<{
|
|
15
|
+
requests: z.ZodNumber;
|
|
16
|
+
errors: z.ZodNumber;
|
|
17
|
+
}, z.core.$strip>;
|
|
18
|
+
}, z.core.$strip>;
|
|
19
|
+
daily: z.ZodObject<{
|
|
20
|
+
days: z.ZodNumber;
|
|
21
|
+
requestsPerDay: z.ZodNumber;
|
|
22
|
+
errorsPerDay: z.ZodNumber;
|
|
23
|
+
}, z.core.$strip>;
|
|
24
|
+
from: z.ZodString;
|
|
25
|
+
to: z.ZodString;
|
|
26
|
+
}, z.core.$strip>;
|
|
27
|
+
/**
|
|
28
|
+
* `GET <address>/api/reporting/release-health?from&to` with the bearer token.
|
|
29
|
+
* Returns null for any failure (network, timeout, status, shape): the caller
|
|
30
|
+
* counts that as inconclusive, never as a pass. Nothing about the response or
|
|
31
|
+
* the token reaches a log or an error.
|
|
32
|
+
*/
|
|
33
|
+
export declare function readReleaseHealth(input: {
|
|
34
|
+
address: string;
|
|
35
|
+
token: string;
|
|
36
|
+
from: string;
|
|
37
|
+
to: string;
|
|
38
|
+
/** Extra request headers, e.g. the load balancer's color pin. */
|
|
39
|
+
headers?: Record<string, string>;
|
|
40
|
+
fetchImpl?: typeof fetch;
|
|
41
|
+
}): Promise<HealthRead | null>;
|
|
42
|
+
/** An HTTP check of the color's health path on its own address. */
|
|
43
|
+
export declare function checkColorAddress(input: {
|
|
44
|
+
address: string;
|
|
45
|
+
healthPath: string | null;
|
|
46
|
+
headers?: Record<string, string>;
|
|
47
|
+
fetchImpl?: typeof fetch;
|
|
48
|
+
}): Promise<'ok' | 'failed'>;
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
import { z } from 'zod';
|
|
2
|
+
/** The reporting health read has this long to answer. */
|
|
3
|
+
export const HEALTH_READ_TIMEOUT_MS = 10_000;
|
|
4
|
+
/** The color's own address check has this long to answer. */
|
|
5
|
+
export const ADDRESS_CHECK_TIMEOUT_MS = 10_000;
|
|
6
|
+
const counts = z.object({
|
|
7
|
+
requests: z.number().int().nonnegative(),
|
|
8
|
+
errors: z.number().int().nonnegative(),
|
|
9
|
+
});
|
|
10
|
+
/** C1's 200 body. */
|
|
11
|
+
export const healthBody = z.object({
|
|
12
|
+
colors: z.object({ blue: counts, green: counts }),
|
|
13
|
+
daily: z.object({
|
|
14
|
+
days: z.number().nonnegative(),
|
|
15
|
+
requestsPerDay: z.number().nonnegative(),
|
|
16
|
+
errorsPerDay: z.number().nonnegative(),
|
|
17
|
+
}),
|
|
18
|
+
from: z.string(),
|
|
19
|
+
to: z.string(),
|
|
20
|
+
});
|
|
21
|
+
/**
|
|
22
|
+
* `GET <address>/api/reporting/release-health?from&to` with the bearer token.
|
|
23
|
+
* Returns null for any failure (network, timeout, status, shape): the caller
|
|
24
|
+
* counts that as inconclusive, never as a pass. Nothing about the response or
|
|
25
|
+
* the token reaches a log or an error.
|
|
26
|
+
*/
|
|
27
|
+
export async function readReleaseHealth(input) {
|
|
28
|
+
try {
|
|
29
|
+
const url = new URL('/api/reporting/release-health', input.address);
|
|
30
|
+
url.searchParams.set('from', input.from);
|
|
31
|
+
url.searchParams.set('to', input.to);
|
|
32
|
+
const res = await (input.fetchImpl ?? fetch)(url, {
|
|
33
|
+
headers: { ...input.headers, authorization: `Bearer ${input.token}` },
|
|
34
|
+
signal: AbortSignal.timeout(HEALTH_READ_TIMEOUT_MS),
|
|
35
|
+
});
|
|
36
|
+
if (!res.ok)
|
|
37
|
+
return null;
|
|
38
|
+
const parsed = healthBody.safeParse(await res.json());
|
|
39
|
+
if (!parsed.success)
|
|
40
|
+
return null;
|
|
41
|
+
return { colors: parsed.data.colors, daily: parsed.data.daily };
|
|
42
|
+
}
|
|
43
|
+
catch {
|
|
44
|
+
// Network error, timeout or invalid JSON: all the same to the caller.
|
|
45
|
+
return null;
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
/** An HTTP check of the color's health path on its own address. */
|
|
49
|
+
export async function checkColorAddress(input) {
|
|
50
|
+
try {
|
|
51
|
+
const res = await (input.fetchImpl ?? fetch)(new URL(input.healthPath ?? '/', input.address), {
|
|
52
|
+
...(input.headers ? { headers: input.headers } : {}),
|
|
53
|
+
signal: AbortSignal.timeout(ADDRESS_CHECK_TIMEOUT_MS),
|
|
54
|
+
});
|
|
55
|
+
return res.ok ? 'ok' : 'failed';
|
|
56
|
+
}
|
|
57
|
+
catch {
|
|
58
|
+
// Unreachable counts as a failed check.
|
|
59
|
+
return 'failed';
|
|
60
|
+
}
|
|
61
|
+
}
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
import { type Decision, type Observation, type ReleaseEffect } from './decide.js';
|
|
2
|
+
import { type ReleaseStore } from './state.js';
|
|
3
|
+
import type { Color } from './types.js';
|
|
4
|
+
/** What the runner needs from the outside. Each call is bounded by its own timeout. */
|
|
5
|
+
export interface ReleaseDeps {
|
|
6
|
+
/** The reporting health read for the range, or null when it failed. */
|
|
7
|
+
readHealth(range: {
|
|
8
|
+
from: string;
|
|
9
|
+
to: string;
|
|
10
|
+
}): Promise<Observation['health']>;
|
|
11
|
+
/** The Coolify status and the own-address check of a color. */
|
|
12
|
+
observeColor(color: Color): Promise<Pick<Observation, 'status' | 'address'>>;
|
|
13
|
+
effect(effect: ReleaseEffect): Promise<void>;
|
|
14
|
+
}
|
|
15
|
+
export interface ReleaseRunResult {
|
|
16
|
+
/** Scope keys to record (null removes one). The caller persists them with the rest of the run. */
|
|
17
|
+
writes: Record<string, string | null>;
|
|
18
|
+
blocked: string[];
|
|
19
|
+
refused: string[];
|
|
20
|
+
decision: Decision | null;
|
|
21
|
+
/** The requests this run acted on. The caller consumes them with `store.consumeRequests` after it persists `writes`. */
|
|
22
|
+
requestIds: number[];
|
|
23
|
+
}
|
|
24
|
+
/**
|
|
25
|
+
* One run for one entry. Reads the recorded keys fresh and the open
|
|
26
|
+
* requests (it consumes none), decides and runs the effects. A failure, or a
|
|
27
|
+
* crash before the caller persists and consumes, leaves the requests open and
|
|
28
|
+
* the recorded keys as they were: the next run decides again from them.
|
|
29
|
+
*/
|
|
30
|
+
export declare function runRelease(input: {
|
|
31
|
+
store: ReleaseStore;
|
|
32
|
+
orgId: string;
|
|
33
|
+
entryKey: string;
|
|
34
|
+
entry: {
|
|
35
|
+
steps?: number[];
|
|
36
|
+
stepHoldMinutes?: number;
|
|
37
|
+
};
|
|
38
|
+
now: Date;
|
|
39
|
+
deps: ReleaseDeps;
|
|
40
|
+
}): Promise<ReleaseRunResult>;
|