anbaric-cloud-hosting 1.59.0 → 1.61.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +7 -7
- package/src/app-management/BaseBuildLayer.ts +3 -0
- package/src/app-management/BuildLayer.ts +6 -0
- package/src/app-management/appAddress.ts +19 -0
- package/src/app-management/appHostname.ts +16 -23
- package/src/auth/HttpMembershipService.ts +2 -2
- package/src/auth/InMemoryMembershipService.ts +2 -1
- package/src/auth/MembershipService.ts +5 -1
- package/src/data-store/EntitlementStore.ts +23 -1
- package/src/data-store/InMemoryEntitlementStore.ts +25 -1
- package/src/data-store/PlatformMigrations.ts +26 -0
- package/src/data-store/PostgresEntitlementStore.ts +40 -1
- package/src/data-store/PostgresJobPersistence.ts +29 -2
- package/src/data-store/StallSweep.ts +46 -0
- package/src/hosting/HostingServer.ts +26 -4
- package/src/hosting/handlers/AppsHandler.ts +65 -6
- package/src/hosting/handlers/JobsHandler.ts +6 -0
- package/src/hosting/handlers/MyAppsHandler.ts +48 -0
- package/src/hosting/handlers/auth/InvitationsHandler.ts +44 -4
- package/src/hosting/middleware/AuthenticationMiddleware.ts +20 -5
- package/src/hosting/pages/platform-ui.html +3 -3
- package/src/index.ts +1 -0
- package/src/main.ts +40 -2
- package/src/subdomains/HttpSubdomains.ts +91 -0
- package/src/subdomains/Subdomains.ts +30 -0
- package/src/usage/UsageReporter.ts +63 -0
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "anbaric-cloud-hosting",
|
|
3
|
-
"version": "1.
|
|
3
|
+
"version": "1.61.0",
|
|
4
4
|
"description": "Anbaric Cloud hosting service: Postgres-backed job persistence and queuing exposed over an HTTP API",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"author": "chris@anbaric.ai",
|
|
@@ -18,18 +18,18 @@
|
|
|
18
18
|
"@aws-sdk/client-s3": "^3.1110.0",
|
|
19
19
|
"@aws-sdk/client-secrets-manager": "^3.700.0",
|
|
20
20
|
"@aws-sdk/client-servicediscovery": "^3.1110.0",
|
|
21
|
-
"anbaric-data-store": "^1.
|
|
22
|
-
"anbaric-plugins": "^1.
|
|
23
|
-
"anbaric-tsapi": "^1.
|
|
24
|
-
"anbaric-web": "^1.
|
|
21
|
+
"anbaric-data-store": "^1.61.0",
|
|
22
|
+
"anbaric-plugins": "^1.61.0",
|
|
23
|
+
"anbaric-tsapi": "^1.61.0",
|
|
24
|
+
"anbaric-web": "^1.61.0",
|
|
25
25
|
"esbuild": "^0.28.2",
|
|
26
26
|
"pg": "^8.16.0"
|
|
27
27
|
},
|
|
28
28
|
"devDependencies": {
|
|
29
29
|
"@types/node": "^26.2.0",
|
|
30
30
|
"@types/pg": "^8.15.0",
|
|
31
|
-
"anbaric-impl-cloud": "^1.
|
|
32
|
-
"anbaric-state-machine": "^1.
|
|
31
|
+
"anbaric-impl-cloud": "^1.61.0",
|
|
32
|
+
"anbaric-state-machine": "^1.61.0",
|
|
33
33
|
"tsx": "^4.20.0",
|
|
34
34
|
"typescript": "^7.0.2"
|
|
35
35
|
},
|
|
@@ -39,6 +39,8 @@ abstract class BaseBuildLayer implements BuildLayer {
|
|
|
39
39
|
protected deployments = new Map<string, Deployment>();
|
|
40
40
|
protected docGenerator? : DocGenerator;
|
|
41
41
|
consumerUrlsFor : (appName : string) => Array<string> = () => [];
|
|
42
|
+
|
|
43
|
+
appRemoved : (appName : string) => void = () => {};
|
|
42
44
|
private nextAppIndex = 0;
|
|
43
45
|
private hydration? : Promise<void>;
|
|
44
46
|
|
|
@@ -116,6 +118,7 @@ abstract class BaseBuildLayer implements BuildLayer {
|
|
|
116
118
|
deployment.status = "stopped";
|
|
117
119
|
await this.stop(deployment);
|
|
118
120
|
if (this.deployments.get(appName) === deployment) this.deployments.delete(appName);
|
|
121
|
+
this.appRemoved(appName);
|
|
119
122
|
};
|
|
120
123
|
|
|
121
124
|
if (deployment.status === "running" && this.consumerUrlsFor(appName).length > 0) {
|
|
@@ -37,6 +37,12 @@ interface BuildLayer {
|
|
|
37
37
|
// to drain; the hosting server supplies it from its consumer registry.
|
|
38
38
|
consumerUrlsFor : (appName : string) => Array<string>;
|
|
39
39
|
|
|
40
|
+
/* Called once an app is really gone, which for one that drained is some
|
|
41
|
+
minutes after the teardown returned. Whatever outlives the app - its
|
|
42
|
+
public address, in a hosted platform - is let go here, because this is
|
|
43
|
+
the only moment both paths out of teardown pass through. */
|
|
44
|
+
appRemoved : (appName : string) => void;
|
|
45
|
+
|
|
40
46
|
}
|
|
41
47
|
|
|
42
48
|
export type { BuildLayer, DeploymentStatus, DeploymentSummary, Draining };
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
/* Where an app is served, decided in one place because the console, the CLI
|
|
2
|
+
and anything else that shows a person a link must all show the same one.
|
|
3
|
+
|
|
4
|
+
Its own address, allocated by the control plane, which is the only form on
|
|
5
|
+
Anbaric Cloud. Then the app's name under the configured host suffix, which
|
|
6
|
+
is what a self-hoster gets who has pointed their own wildcard at the
|
|
7
|
+
platform and has no control plane to allocate anything - one platform serves
|
|
8
|
+
one tenant there, so the name alone is unambiguous. Then the path form,
|
|
9
|
+
which always works and is what a platform run locally uses.
|
|
10
|
+
|
|
11
|
+
An app's own hostname serves it at the root, so links, assets and fetches
|
|
12
|
+
inside it behave as they did on the machine it was built on. */
|
|
13
|
+
const appAddress = (appName : string, subdomain? : string, hostSuffix : string = process.env.ANBARIC_APP_HOST_SUFFIX ?? "") : string => {
|
|
14
|
+
if (! hostSuffix) return `/app/${appName}`;
|
|
15
|
+
|
|
16
|
+
return `https://${subdomain ?? appName}.${hostSuffix}`;
|
|
17
|
+
};
|
|
18
|
+
|
|
19
|
+
export { appAddress };
|
|
@@ -1,39 +1,32 @@
|
|
|
1
1
|
/* An app is reachable at a hostname of its own, so that it is served at the
|
|
2
|
-
root of it and works exactly as it did when it was built locally.
|
|
3
|
-
will both have a hello-world, so the tenant is part of the name.
|
|
2
|
+
root of it and works exactly as it did when it was built locally.
|
|
4
3
|
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
4
|
+
The address itself is one DNS label and belongs to the control plane, which
|
|
5
|
+
is the only thing that can see every tenant and so the only thing that can
|
|
6
|
+
keep an address unique. What is checked here is narrower and local: that the
|
|
7
|
+
app's name could be a hostname label at all, since it is the first address
|
|
8
|
+
the control plane will be asked for.
|
|
10
9
|
|
|
11
|
-
|
|
10
|
+
A hosted app whose name is already taken is given a different address rather
|
|
11
|
+
than refused, so this is about shape, never about availability. */
|
|
12
12
|
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
13
|
+
/* A DNS label may not exceed 63 characters, and the control plane may have to
|
|
14
|
+
add a distinguishing word to a name two tenants both wanted - so a name is
|
|
15
|
+
held well short of the limit to leave room for one. */
|
|
16
|
+
const NAME_LIMIT = 48;
|
|
16
17
|
|
|
17
18
|
const SHAPE = /^[a-z0-9]([a-z0-9-]*[a-z0-9])?$/;
|
|
18
19
|
|
|
19
|
-
const hostLabelFor = (appName : string, tenant : string) : string => `${appName}${SEPARATOR}${tenant}`;
|
|
20
|
-
|
|
21
20
|
/* Why this name cannot have a hostname, or undefined when it can. Returned as a
|
|
22
21
|
sentence because it is shown to whoever ran the deploy. */
|
|
23
|
-
const hostnameObjection = (appName : string
|
|
22
|
+
const hostnameObjection = (appName : string) : string | undefined => {
|
|
24
23
|
if (! SHAPE.test(appName)) {
|
|
25
24
|
return `"${appName}" must be lower-case letters, numbers and hyphens, starting and ending with a letter or number`;
|
|
26
25
|
}
|
|
27
|
-
if (appName.
|
|
28
|
-
return `"${appName}"
|
|
29
|
-
}
|
|
30
|
-
if (! tenant) return undefined;
|
|
31
|
-
|
|
32
|
-
const label = hostLabelFor(appName, tenant);
|
|
33
|
-
if (label.length > LABEL_LIMIT) {
|
|
34
|
-
return `"${appName}" is too long for a hostname alongside tenant "${tenant}": ${label.length} characters, and a hostname label allows ${LABEL_LIMIT}`;
|
|
26
|
+
if (appName.length > NAME_LIMIT) {
|
|
27
|
+
return `"${appName}" is too long for an app name: ${appName.length} characters, and the limit is ${NAME_LIMIT}`;
|
|
35
28
|
}
|
|
36
29
|
return undefined;
|
|
37
30
|
};
|
|
38
31
|
|
|
39
|
-
export {
|
|
32
|
+
export { hostnameObjection, NAME_LIMIT };
|
|
@@ -31,8 +31,8 @@ class HttpMembershipService implements MembershipService {
|
|
|
31
31
|
return found;
|
|
32
32
|
}
|
|
33
33
|
|
|
34
|
-
async invite(email : string, role : TenantRole, invitedBy : Inviter) : Promise<PendingInvitation> {
|
|
35
|
-
const response = await this.call("POST", "/invitations", { email, role, invitedBy });
|
|
34
|
+
async invite(email : string, role : TenantRole, invitedBy : Inviter, landingApp? : string) : Promise<PendingInvitation> {
|
|
35
|
+
const response = await this.call("POST", "/invitations", { email, role, invitedBy, landingApp });
|
|
36
36
|
return await response.json() as PendingInvitation;
|
|
37
37
|
}
|
|
38
38
|
|
|
@@ -17,13 +17,14 @@ class InMemoryMembershipService implements MembershipService {
|
|
|
17
17
|
return this.members.get(userId);
|
|
18
18
|
}
|
|
19
19
|
|
|
20
|
-
async invite(email : string, role : TenantRole, invitedBy : Inviter) : Promise<PendingInvitation> {
|
|
20
|
+
async invite(email : string, role : TenantRole, invitedBy : Inviter, landingApp? : string) : Promise<PendingInvitation> {
|
|
21
21
|
const token = randomUUID();
|
|
22
22
|
const now = new Date();
|
|
23
23
|
const invitation : PendingInvitation = {
|
|
24
24
|
token,
|
|
25
25
|
email: email.trim().toLowerCase(),
|
|
26
26
|
role,
|
|
27
|
+
landingApp,
|
|
27
28
|
invitedBy: invitedBy.id,
|
|
28
29
|
invitedByName: invitedBy.name,
|
|
29
30
|
link: `${this.linkBase}/${token}`,
|
|
@@ -5,6 +5,7 @@ type PendingInvitation = {
|
|
|
5
5
|
token : string,
|
|
6
6
|
email : string,
|
|
7
7
|
role : TenantRole,
|
|
8
|
+
landingApp? : string,
|
|
8
9
|
invitedBy : string,
|
|
9
10
|
invitedByName? : string,
|
|
10
11
|
link : string,
|
|
@@ -29,7 +30,10 @@ interface MembershipService {
|
|
|
29
30
|
|
|
30
31
|
roleFor(userId : string) : Promise<TenantRole | undefined>;
|
|
31
32
|
|
|
32
|
-
|
|
33
|
+
/* landingApp is the app the invitation should open on rather than the
|
|
34
|
+
person's app directory. The control plane turns it into an address when
|
|
35
|
+
the invitation is accepted, so a later change of address still works. */
|
|
36
|
+
invite(email : string, role : TenantRole, invitedBy : Inviter, landingApp? : string) : Promise<PendingInvitation>;
|
|
33
37
|
|
|
34
38
|
pending(asking : Inviter) : Promise<Array<PendingInvitation>>;
|
|
35
39
|
|
|
@@ -22,6 +22,18 @@ type EntitlementGrant = {
|
|
|
22
22
|
|
|
23
23
|
};
|
|
24
24
|
|
|
25
|
+
/* An entitlement chosen for somebody who has not signed in yet. An invitation
|
|
26
|
+
names an email address; a grant names a user id, which that person does not
|
|
27
|
+
have until they arrive. This is the gap between the two. */
|
|
28
|
+
type EntitlementOffer = {
|
|
29
|
+
|
|
30
|
+
email : string,
|
|
31
|
+
appId : string | null,
|
|
32
|
+
entitlementId : string,
|
|
33
|
+
notes : string,
|
|
34
|
+
|
|
35
|
+
};
|
|
36
|
+
|
|
25
37
|
/* The platform's view of entitlements: what apps can do (register, has) plus
|
|
26
38
|
the management the console performs on their behalf. */
|
|
27
39
|
interface EntitlementStore extends Entitlements {
|
|
@@ -36,6 +48,16 @@ interface EntitlementStore extends Entitlements {
|
|
|
36
48
|
|
|
37
49
|
listGrants(userId? : string) : Promise<Array<EntitlementGrant>>;
|
|
38
50
|
|
|
51
|
+
// Held against an email address until whoever holds it signs in.
|
|
52
|
+
offer(offers : Array<EntitlementOffer>, offeredBy : string) : Promise<void>;
|
|
53
|
+
|
|
54
|
+
offersFor(email : string) : Promise<Array<EntitlementOffer>>;
|
|
55
|
+
|
|
56
|
+
/* Turns everything offered to this email into real grants and forgets the
|
|
57
|
+
offers. Returns how many were made, and is safe to call on every sign-in
|
|
58
|
+
because the second call has nothing left to claim. */
|
|
59
|
+
claim(email : string, userId : string) : Promise<number>;
|
|
60
|
+
|
|
39
61
|
}
|
|
40
62
|
|
|
41
|
-
export type { EntitlementStore, EntitlementDefinition, EntitlementGrant }
|
|
63
|
+
export type { EntitlementStore, EntitlementDefinition, EntitlementGrant, EntitlementOffer }
|
|
@@ -1,10 +1,11 @@
|
|
|
1
1
|
import {randomUUID} from "node:crypto";
|
|
2
|
-
import {EntitlementDefinition, EntitlementGrant, EntitlementStore} from "./EntitlementStore";
|
|
2
|
+
import {EntitlementDefinition, EntitlementGrant, EntitlementOffer, EntitlementStore} from "./EntitlementStore";
|
|
3
3
|
|
|
4
4
|
class InMemoryEntitlementStore implements EntitlementStore {
|
|
5
5
|
|
|
6
6
|
private definitions = new Map<string, EntitlementDefinition>();
|
|
7
7
|
private grants = new Map<string, EntitlementGrant>();
|
|
8
|
+
private offers = new Map<string, { offer : EntitlementOffer, offeredBy : string }>();
|
|
8
9
|
|
|
9
10
|
constructor() {
|
|
10
11
|
this.definitions.set(this.key(null, "access"), { appId: null, entitlementId: "access", notes: "Global access" });
|
|
@@ -46,6 +47,29 @@ class InMemoryEntitlementStore implements EntitlementStore {
|
|
|
46
47
|
.map(grant => ({ ...grant }));
|
|
47
48
|
}
|
|
48
49
|
|
|
50
|
+
// Held against an email address; see PostgresEntitlementStore.
|
|
51
|
+
async offer(offers : Array<EntitlementOffer>, offeredBy : string) : Promise<void> {
|
|
52
|
+
for (const offer of offers) {
|
|
53
|
+
this.offers.set(`${offer.email} ${this.key(offer.appId, offer.entitlementId)}`, { offer, offeredBy });
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
async offersFor(email : string) : Promise<Array<EntitlementOffer>> {
|
|
58
|
+
return [...this.offers.values()]
|
|
59
|
+
.filter(held => held.offer.email === email)
|
|
60
|
+
.map(held => ({ ...held.offer }));
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
async claim(email : string, userId : string) : Promise<number> {
|
|
64
|
+
const owed = [...this.offers.entries()].filter(([, held]) => held.offer.email === email);
|
|
65
|
+
|
|
66
|
+
for (const [key, held] of owed) {
|
|
67
|
+
await this.grant(userId, held.offer.appId, held.offer.entitlementId, held.offer.notes, held.offeredBy);
|
|
68
|
+
this.offers.delete(key);
|
|
69
|
+
}
|
|
70
|
+
return owed.length;
|
|
71
|
+
}
|
|
72
|
+
|
|
49
73
|
private key(appId : string | null, entitlementId : string) : string {
|
|
50
74
|
return `${appId ?? ""}/${entitlementId}`;
|
|
51
75
|
}
|
|
@@ -230,6 +230,32 @@ const PLATFORM_MIGRATIONS : Array<Migration> = [
|
|
|
230
230
|
"ALTER TABLE jobs DROP COLUMN properties",
|
|
231
231
|
),
|
|
232
232
|
|
|
233
|
+
/* A long-running step heartbeats its job; a job whose heartbeat has gone
|
|
234
|
+
quiet is marked Stalled. The index is what the stall sweep reads. */
|
|
235
|
+
statements("004-job-heartbeat",
|
|
236
|
+
"ALTER TABLE jobs ADD COLUMN heartbeat_at TIMESTAMPTZ",
|
|
237
|
+
"CREATE INDEX jobs_heartbeat ON jobs (heartbeat_at) WHERE heartbeat_at IS NOT NULL",
|
|
238
|
+
),
|
|
239
|
+
|
|
240
|
+
/* Entitlements chosen for someone who has not arrived yet. An invitation
|
|
241
|
+
goes to an email address and a grant is made against a user id, which
|
|
242
|
+
that person does not have until the first time they sign in - so what
|
|
243
|
+
the inviter chose waits here under the email and is turned into real
|
|
244
|
+
grants at that moment. Rows are deleted as they are claimed, so this
|
|
245
|
+
table holds only what is still owed. */
|
|
246
|
+
statements("005-entitlement-invites",
|
|
247
|
+
`CREATE TABLE IF NOT EXISTS entitlement_invites (
|
|
248
|
+
email TEXT NOT NULL,
|
|
249
|
+
app_id TEXT,
|
|
250
|
+
entitlement_id TEXT NOT NULL,
|
|
251
|
+
notes TEXT NOT NULL DEFAULT '',
|
|
252
|
+
offered_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
|
253
|
+
offered_by TEXT NOT NULL
|
|
254
|
+
)`,
|
|
255
|
+
`CREATE UNIQUE INDEX IF NOT EXISTS entitlement_invites_key
|
|
256
|
+
ON entitlement_invites (email, COALESCE(app_id, ''), entitlement_id)`,
|
|
257
|
+
),
|
|
258
|
+
|
|
233
259
|
];
|
|
234
260
|
|
|
235
261
|
export { PLATFORM_MIGRATIONS };
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import {randomUUID} from "node:crypto";
|
|
2
2
|
import {Pool} from "pg";
|
|
3
|
-
import {EntitlementDefinition, EntitlementGrant, EntitlementStore} from "./EntitlementStore";
|
|
3
|
+
import {EntitlementDefinition, EntitlementGrant, EntitlementOffer, EntitlementStore} from "./EntitlementStore";
|
|
4
4
|
|
|
5
5
|
class PostgresEntitlementStore implements EntitlementStore {
|
|
6
6
|
|
|
@@ -64,6 +64,45 @@ class PostgresEntitlementStore implements EntitlementStore {
|
|
|
64
64
|
}));
|
|
65
65
|
}
|
|
66
66
|
|
|
67
|
+
/* Held against an email address until whoever holds it signs in, because
|
|
68
|
+
an invitation names an email and a grant names a user id. */
|
|
69
|
+
async offer(offers : Array<EntitlementOffer>, offeredBy : string) : Promise<void> {
|
|
70
|
+
for (const offer of offers) {
|
|
71
|
+
await this.pool.query(
|
|
72
|
+
`INSERT INTO entitlement_invites (email, app_id, entitlement_id, notes, offered_by)
|
|
73
|
+
VALUES ($1, $2, $3, $4, $5)
|
|
74
|
+
ON CONFLICT (email, COALESCE(app_id, ''), entitlement_id)
|
|
75
|
+
DO UPDATE SET notes = EXCLUDED.notes, offered_by = EXCLUDED.offered_by`,
|
|
76
|
+
[offer.email, offer.appId, offer.entitlementId, offer.notes, offeredBy]);
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
async offersFor(email : string) : Promise<Array<EntitlementOffer>> {
|
|
81
|
+
const result = await this.pool.query(
|
|
82
|
+
`SELECT email, app_id, entitlement_id, notes FROM entitlement_invites
|
|
83
|
+
WHERE lower(email) = lower($1) ORDER BY app_id NULLS FIRST, entitlement_id`,
|
|
84
|
+
[email]);
|
|
85
|
+
return result.rows.map(row => ({
|
|
86
|
+
email: row.email, appId: row.app_id, entitlementId: row.entitlement_id, notes: row.notes,
|
|
87
|
+
}));
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
/* Claimed in one statement so a second sign-in arriving at the same moment
|
|
91
|
+
cannot grant the same thing twice: the delete is what decides who owns
|
|
92
|
+
each row, and only rows it actually removed become grants. */
|
|
93
|
+
async claim(email : string, userId : string) : Promise<number> {
|
|
94
|
+
const result = await this.pool.query(
|
|
95
|
+
`WITH claimed AS (
|
|
96
|
+
DELETE FROM entitlement_invites WHERE lower(email) = lower($1)
|
|
97
|
+
RETURNING app_id, entitlement_id, notes, offered_by
|
|
98
|
+
)
|
|
99
|
+
INSERT INTO entitlement_grants (id, user_id, app_id, entitlement_id, notes, granted_by)
|
|
100
|
+
SELECT gen_random_uuid()::text, $2, app_id, entitlement_id, notes, offered_by FROM claimed
|
|
101
|
+
RETURNING id`,
|
|
102
|
+
[email, userId]);
|
|
103
|
+
return result.rowCount ?? 0;
|
|
104
|
+
}
|
|
105
|
+
|
|
67
106
|
private async upsertDefinition(appId : string | null, entitlementId : string, notes : string) : Promise<void> {
|
|
68
107
|
await this.pool.query(
|
|
69
108
|
`INSERT INTO entitlements (app_id, entitlement_id, notes)
|
|
@@ -13,6 +13,7 @@ type JobRow = {
|
|
|
13
13
|
status : string,
|
|
14
14
|
waiting_for? : string,
|
|
15
15
|
await_metadata? : SerializedWaitForInput,
|
|
16
|
+
heartbeat_at? : Date,
|
|
16
17
|
};
|
|
17
18
|
|
|
18
19
|
/* The await a job is parked on is normalised into the awaits table: jobs
|
|
@@ -20,7 +21,7 @@ type JobRow = {
|
|
|
20
21
|
job's awaitMetadata is rejoined on read. */
|
|
21
22
|
const JOB_SELECT =
|
|
22
23
|
`SELECT j.id, j.state, j.workflow_id, j.app_id, j.started_at, j.started_by, j.last_updated,
|
|
23
|
-
j.killed, j.status, j.waiting_for, a.metadata AS await_metadata
|
|
24
|
+
j.killed, j.status, j.waiting_for, j.heartbeat_at, a.metadata AS await_metadata
|
|
24
25
|
FROM jobs j LEFT JOIN awaits a ON a.id = j.waiting_for`;
|
|
25
26
|
|
|
26
27
|
/* Properties live one row each in job_properties, so a read fetches only the
|
|
@@ -153,6 +154,31 @@ class PostgresJobPersistence extends JobPersistence {
|
|
|
153
154
|
return result.rows.map(row => this.deserializeRow(row, new JobProperties(properties.get(row.id) ?? new Map())));
|
|
154
155
|
}
|
|
155
156
|
|
|
157
|
+
protected async heartbeatInternal(id : string, running : boolean) : Promise<void> {
|
|
158
|
+
// A heartbeat is proof the step is running, so one from a job marked
|
|
159
|
+
// Stalled - a process that came back, or a step slower than the
|
|
160
|
+
// sweep - returns it to active.
|
|
161
|
+
await this.pool.query(running
|
|
162
|
+
? `UPDATE jobs SET heartbeat_at = now(),
|
|
163
|
+
status = CASE WHEN status = $2 THEN $3 ELSE status END
|
|
164
|
+
WHERE id = $1`
|
|
165
|
+
: "UPDATE jobs SET heartbeat_at = NULL WHERE id = $1",
|
|
166
|
+
running ? [id, Job.Status.STALLED, Job.Status.ACTIVE] : [id]);
|
|
167
|
+
}
|
|
168
|
+
|
|
169
|
+
/* Marks Stalled every active job whose long-running step has not
|
|
170
|
+
heartbeated for `staleMs`. The heartbeat is left as it was, so the
|
|
171
|
+
record shows when the step was last heard from; a save that moves the
|
|
172
|
+
job on sets its status again. Returns how many were marked. */
|
|
173
|
+
async markStalled(staleMs : number = Job.STALL_AFTER_MS) : Promise<number> {
|
|
174
|
+
const result = await this.pool.query(
|
|
175
|
+
`UPDATE jobs SET status = $1, last_updated = now()
|
|
176
|
+
WHERE status = $2 AND killed = false AND heartbeat_at IS NOT NULL AND heartbeat_at < now() - ($3::text || ' milliseconds')::interval`,
|
|
177
|
+
[Job.Status.STALLED, Job.Status.ACTIVE, String(staleMs)],
|
|
178
|
+
);
|
|
179
|
+
return result.rowCount ?? 0;
|
|
180
|
+
}
|
|
181
|
+
|
|
156
182
|
protected async killInternal(id : string) : Promise<void> {
|
|
157
183
|
await this.pool.query("UPDATE jobs SET killed = true, last_updated = now() WHERE id = $1", [id]);
|
|
158
184
|
}
|
|
@@ -185,9 +211,10 @@ class PostgresJobPersistence extends JobPersistence {
|
|
|
185
211
|
status: row.status,
|
|
186
212
|
waitingFor: row.waiting_for ?? undefined,
|
|
187
213
|
awaitMetadata: row.await_metadata ?? undefined,
|
|
214
|
+
heartbeatAt: row.heartbeat_at?.toISOString(),
|
|
188
215
|
});
|
|
189
216
|
return new Job(shape.id, properties, shape.state, shape.workflowId, shape.appId, shape.startedBy, shape.startedAt,
|
|
190
|
-
shape.lastUpdated, shape.killed, shape.status, shape.awaitMetadata, shape.waitingFor);
|
|
217
|
+
shape.lastUpdated, shape.killed, shape.status, shape.awaitMetadata, shape.waitingFor, shape.heartbeatAt);
|
|
191
218
|
}
|
|
192
219
|
|
|
193
220
|
}
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
import {PostgresJobPersistence} from "./PostgresJobPersistence";
|
|
2
|
+
|
|
3
|
+
const SWEEP_INTERVAL_MS = 60_000;
|
|
4
|
+
|
|
5
|
+
/* Marks a job Stalled when the long-running step that was heartbeating it has
|
|
6
|
+
gone quiet - the process running it died, most likely mid-deploy or on a
|
|
7
|
+
crash. Marking is all it does: the platform never re-runs a long step on its
|
|
8
|
+
own, because only the app knows what a half-finished one has already done.
|
|
9
|
+
Someone sees the status and decides. */
|
|
10
|
+
class StallSweep {
|
|
11
|
+
|
|
12
|
+
private ticker? : NodeJS.Timeout;
|
|
13
|
+
private sweeping = false;
|
|
14
|
+
|
|
15
|
+
constructor(private persistence : PostgresJobPersistence,
|
|
16
|
+
private intervalMs : number = SWEEP_INTERVAL_MS) {}
|
|
17
|
+
|
|
18
|
+
start() : void {
|
|
19
|
+
if (this.ticker) return;
|
|
20
|
+
this.ticker = setInterval(() => void this.sweep(), this.intervalMs);
|
|
21
|
+
this.ticker.unref();
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
async sweep() : Promise<number> {
|
|
25
|
+
if (this.sweeping) return 0;
|
|
26
|
+
this.sweeping = true;
|
|
27
|
+
try {
|
|
28
|
+
const stalled = await this.persistence.markStalled();
|
|
29
|
+
if (stalled > 0) console.warn(`[jobs] marked ${stalled} job(s) stalled: no heartbeat from a long-running step`);
|
|
30
|
+
return stalled;
|
|
31
|
+
} catch (error) {
|
|
32
|
+
console.error(`[jobs] the stall sweep failed: ${error instanceof Error ? error.message : error}`);
|
|
33
|
+
return 0;
|
|
34
|
+
} finally {
|
|
35
|
+
this.sweeping = false;
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
async cleanUp() : Promise<void> {
|
|
40
|
+
if (this.ticker) clearInterval(this.ticker);
|
|
41
|
+
this.ticker = undefined;
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
export { StallSweep }
|
|
@@ -9,6 +9,8 @@ import {ConsumerRegistry} from "../queuing/ConsumerRegistry";
|
|
|
9
9
|
import {AppProxyHandler} from "./handlers/AppProxyHandler";
|
|
10
10
|
import {AppLinkFallbackHandler} from "./handlers/AppLinkFallbackHandler";
|
|
11
11
|
import {AppsHandler} from "./handlers/AppsHandler";
|
|
12
|
+
import {MyAppsHandler} from "./handlers/MyAppsHandler";
|
|
13
|
+
import {Subdomains} from "../subdomains/Subdomains";
|
|
12
14
|
import {AuditsHandler} from "./handlers/AuditsHandler";
|
|
13
15
|
import {AuthorizeCliHandler} from "./handlers/auth/AuthorizeCliHandler";
|
|
14
16
|
import {KeysHandler} from "./handlers/auth/KeysHandler";
|
|
@@ -74,7 +76,8 @@ class HostingServer {
|
|
|
74
76
|
userDirectory? : UserDirectory,
|
|
75
77
|
memberships? : MembershipService,
|
|
76
78
|
promptManagerFor? : (appId : string) => PromptManager,
|
|
77
|
-
fileStorageFor? : (appId : string) => FileStorage
|
|
79
|
+
fileStorageFor? : (appId : string) => FileStorage,
|
|
80
|
+
subdomains? : Subdomains) {
|
|
78
81
|
const pages = new PagesHandler();
|
|
79
82
|
const ping = new PingHandler(tenant);
|
|
80
83
|
const jobs = new JobsHandler(persistence);
|
|
@@ -107,7 +110,7 @@ class HostingServer {
|
|
|
107
110
|
if (notifier) publicRouter.registerApi("notifications", new NotificationsHandler(notifier));
|
|
108
111
|
if (entitlements) publicRouter.registerApi("entitlements", new EntitlementsAdminHandler(entitlements));
|
|
109
112
|
if (userDirectory) publicRouter.registerApi("users", new UsersHandler(userDirectory));
|
|
110
|
-
if (memberships) publicRouter.registerApi("invitations", new InvitationsHandler(memberships));
|
|
113
|
+
if (memberships) publicRouter.registerApi("invitations", new InvitationsHandler(memberships, entitlements));
|
|
111
114
|
const prompts = promptManagerFor && new PromptsHandler(promptManagerFor);
|
|
112
115
|
if (prompts) publicRouter.registerApi("prompts", prompts);
|
|
113
116
|
if (cliAuthorizer) {
|
|
@@ -120,8 +123,15 @@ class HostingServer {
|
|
|
120
123
|
// are wherever they registered themselves.
|
|
121
124
|
buildLayer.consumerUrlsFor = appName =>
|
|
122
125
|
[...new Set(registry.list().filter(consumer => consumer.appId === appName).map(consumer => consumer.url))];
|
|
123
|
-
|
|
126
|
+
buildLayer.appRemoved = appName => void subdomains?.release(appName);
|
|
127
|
+
publicRouter.registerApi("apps", new AppsHandler(buildLayer, tenant, subdomains));
|
|
124
128
|
publicRouter.register("app", appProxy);
|
|
129
|
+
|
|
130
|
+
// The directory a person who is not a builder sees instead of the
|
|
131
|
+
// console, so it only exists where entitlements decide access.
|
|
132
|
+
if (entitlements) {
|
|
133
|
+
publicRouter.registerApi("my-apps", new MyAppsHandler(buildLayer, entitlements, subdomains));
|
|
134
|
+
}
|
|
125
135
|
}
|
|
126
136
|
// An unrouted absolute path carrying an app Referer is an app-internal
|
|
127
137
|
// link the proxy's prefix-stripping left bare; send it back to its app.
|
|
@@ -151,10 +161,22 @@ class HostingServer {
|
|
|
151
161
|
request.url.pathname === "/favicon.ico" ||
|
|
152
162
|
(request.method === "GET" && CLI_KEY_POLL.test(request.url.pathname));
|
|
153
163
|
|
|
164
|
+
const authentication =
|
|
165
|
+
new AuthenticationMiddleware(authenticator, tokenAuthenticator, openRequests, undefined, userDirectory, memberships);
|
|
166
|
+
|
|
167
|
+
/* Entitlements an inviter chose before this person had a user id were
|
|
168
|
+
put aside under their email address; their first sign-in is when
|
|
169
|
+
they can finally be granted. */
|
|
170
|
+
if (entitlements) {
|
|
171
|
+
authentication.claimWhatIsOwed = async user => {
|
|
172
|
+
if (user.email) await entitlements.claim(user.email, user.id);
|
|
173
|
+
};
|
|
174
|
+
}
|
|
175
|
+
|
|
154
176
|
this.publicServer = new Server(publicRouter, [
|
|
155
177
|
new SessionMiddleware(),
|
|
156
178
|
new TenantRoutingMiddleware(tenant),
|
|
157
|
-
|
|
179
|
+
authentication,
|
|
158
180
|
/* After authentication and before routing: an app reached by its own
|
|
159
181
|
hostname owns every path on it, so nothing of the platform's would
|
|
160
182
|
match anyway - but a nicer address must not also be a way in
|