@wildo-ai/saas-workers-node-lib 1.1.11 → 1.1.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/esm/index.d.ts +1 -2
- package/dist/esm/index.d.ts.map +1 -1
- package/dist/esm/index.js +1 -2
- package/dist/esm/index.js.map +1 -1
- package/dist/esm/inject/models/container.d.ts.map +1 -1
- package/dist/esm/inject/models/container.js +1 -2
- package/dist/esm/inject/models/container.js.map +1 -1
- package/dist/esm/inject/models/service-types.d.ts.map +1 -1
- package/dist/esm/inject/models/service-types.js.map +1 -1
- package/dist/esm/runtime/middlewares/worker-jwt-verifier.middleware.d.ts +24 -3
- package/dist/esm/runtime/middlewares/worker-jwt-verifier.middleware.d.ts.map +1 -1
- package/dist/esm/runtime/middlewares/worker-jwt-verifier.middleware.js +84 -20
- package/dist/esm/runtime/middlewares/worker-jwt-verifier.middleware.js.map +1 -1
- package/dist/esm/runtime/middlewares/worker-request-context.middleware.d.ts.map +1 -1
- package/dist/esm/runtime/middlewares/worker-request-context.middleware.js.map +1 -1
- package/dist/esm/runtime/services/define-worker-init.d.ts +5 -2
- package/dist/esm/runtime/services/define-worker-init.d.ts.map +1 -1
- package/dist/esm/runtime/services/define-worker-init.js.map +1 -1
- package/dist/esm/runtime/services/read-worker-bootstrap-env.d.ts.map +1 -1
- package/dist/esm/runtime/services/read-worker-bootstrap-env.js.map +1 -1
- package/dist/esm/runtime/services/read-worker-http-surface-env.d.ts.map +1 -1
- package/dist/esm/runtime/services/read-worker-http-surface-env.js.map +1 -1
- package/dist/esm/runtime/services/start-worker-application.backend.service.d.ts +31 -20
- package/dist/esm/runtime/services/start-worker-application.backend.service.d.ts.map +1 -1
- package/dist/esm/runtime/services/start-worker-application.backend.service.js +45 -10
- package/dist/esm/runtime/services/start-worker-application.backend.service.js.map +1 -1
- package/dist/esm/runtime/services/worker-http-server.backend.service.d.ts +6 -5
- package/dist/esm/runtime/services/worker-http-server.backend.service.d.ts.map +1 -1
- package/dist/esm/runtime/services/worker-http-server.backend.service.js +6 -6
- package/dist/esm/runtime/services/worker-http-server.backend.service.js.map +1 -1
- package/dist/tsconfig.build.tsbuildinfo +1 -1
- package/package.json +7 -7
- package/src/index.ts +1 -2
- package/src/inject/models/container.ts +1 -2
- package/src/runtime/middlewares/__tests__/worker-jwt-verifier.rotation-cutoff.middleware.test.ts +104 -9
- package/src/runtime/middlewares/worker-jwt-verifier.middleware.ts +90 -22
- package/src/runtime/services/__tests__/start-worker-application.backend.service.test.ts +115 -2
- package/src/runtime/services/define-worker-init.ts +5 -2
- package/src/runtime/services/start-worker-application.backend.service.ts +61 -21
- package/src/runtime/services/worker-http-server.backend.service.ts +8 -5
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@wildo-ai/saas-workers-node-lib",
|
|
3
|
-
"version": "1.1.
|
|
3
|
+
"version": "1.1.13",
|
|
4
4
|
"description": "Node worker runtime host boundary for Wildo SaaS applications",
|
|
5
5
|
"license": "SEE LICENSE IN LICENSE",
|
|
6
6
|
"author": "Wildo AI",
|
|
@@ -30,10 +30,10 @@
|
|
|
30
30
|
"test:watch": "vitest"
|
|
31
31
|
},
|
|
32
32
|
"dependencies": {
|
|
33
|
-
"@wildo-ai/platform-config-lib": "1.1.
|
|
34
|
-
"@wildo-ai/platform-core-shared": "1.1.
|
|
35
|
-
"@wildo-ai/saas-backend-lib": "1.1.
|
|
36
|
-
"@wildo-ai/saas-models": "1.1.
|
|
33
|
+
"@wildo-ai/platform-config-lib": "1.1.13",
|
|
34
|
+
"@wildo-ai/platform-core-shared": "1.1.13",
|
|
35
|
+
"@wildo-ai/saas-backend-lib": "1.1.13",
|
|
36
|
+
"@wildo-ai/saas-models": "1.1.13",
|
|
37
37
|
"express": "^5.1.0",
|
|
38
38
|
"helmet": "^8.1.0",
|
|
39
39
|
"jsonwebtoken": "^9.0.2",
|
|
@@ -46,11 +46,11 @@
|
|
|
46
46
|
"@types/node": "^24.0.15",
|
|
47
47
|
"@types/express": "^5.0.3",
|
|
48
48
|
"@types/jsonwebtoken": "^9.0.9",
|
|
49
|
-
"rimraf": "^6.
|
|
49
|
+
"rimraf": "^6.1.4",
|
|
50
50
|
"tsc-alias": "1.8.16",
|
|
51
51
|
"tsc-watch": "7.2.1",
|
|
52
52
|
"typescript": "5.9.3",
|
|
53
|
-
"vitest": "^1.
|
|
53
|
+
"vitest": "^4.1.12"
|
|
54
54
|
},
|
|
55
55
|
"repository": {
|
|
56
56
|
"type": "git",
|
package/src/index.ts
CHANGED
|
@@ -2,8 +2,7 @@
|
|
|
2
2
|
* Node worker runtime-host boundary.
|
|
3
3
|
*
|
|
4
4
|
* Owns the worker-side surface that intentionally lives OUTSIDE
|
|
5
|
-
* `saas-backend-lib
|
|
6
|
-
* `.claude/plans/minions-and-workers-implementation.md`: the worker is a
|
|
5
|
+
* `saas-backend-lib`: the worker is a
|
|
7
6
|
* stateless HTTP-only runtime with no backing-service access (Mongo /
|
|
8
7
|
* Redis / RabbitMQ MUST NOT bleed in).
|
|
9
8
|
*
|
|
@@ -15,8 +15,7 @@ let globalWorkersContainer: Container | undefined;
|
|
|
15
15
|
/**
|
|
16
16
|
* Composes the worker DI graph on top of `setupSaasBackendInjection()`.
|
|
17
17
|
*
|
|
18
|
-
*
|
|
19
|
-
* worker boundary deliberately does NOT consume backing-service bindings
|
|
18
|
+
* The worker boundary deliberately does NOT consume backing-service bindings
|
|
20
19
|
* (Mongo / Redis / RabbitMQ) at runtime. The shared backend container is
|
|
21
20
|
* still composed underneath because it carries the JWT signer family,
|
|
22
21
|
* the logger, the error builder, and `WildoWorkerBootstrapClientService`
|
package/src/runtime/middlewares/__tests__/worker-jwt-verifier.rotation-cutoff.middleware.test.ts
CHANGED
|
@@ -3,8 +3,9 @@ import { generateKeyPairSync } from 'node:crypto';
|
|
|
3
3
|
import jwt from 'jsonwebtoken';
|
|
4
4
|
import { afterEach, describe, expect, it, vi } from 'vitest';
|
|
5
5
|
|
|
6
|
+
import { signingKeyIdOfPublicKey, UNKNOWN_SIGNING_KEY_ID_REFRESH_MIN_INTERVAL_MS } from '@wildo-ai/saas-backend-lib';
|
|
6
7
|
import { CryptoKey_Purpose, singleJwsTypeOf } from '@wildo-ai/saas-models';
|
|
7
|
-
import { createWorkerJwtVerifierMiddleware } from '../worker-jwt-verifier.middleware';
|
|
8
|
+
import { createWorkerJwtVerifierMiddleware, WorkerJwtRefusal } from '../worker-jwt-verifier.middleware';
|
|
8
9
|
|
|
9
10
|
function generateSigningPair(): { readonly privateKey: string; readonly publicKey: string } {
|
|
10
11
|
return generateKeyPairSync('ec', {
|
|
@@ -14,10 +15,16 @@ function generateSigningPair(): { readonly privateKey: string; readonly publicKe
|
|
|
14
15
|
});
|
|
15
16
|
}
|
|
16
17
|
|
|
17
|
-
|
|
18
|
-
|
|
18
|
+
/** A worker request signed as the backend signs one: ES256, the worker-request type, and the signing key's thumbprint as `kid`. */
|
|
19
|
+
function signWorkerToken(
|
|
20
|
+
pair: { readonly privateKey: string; readonly publicKey: string },
|
|
21
|
+
typ: string = singleJwsTypeOf(CryptoKey_Purpose.APPLICATION_WORKER_REQUEST),
|
|
22
|
+
/** `null`: sign with no `kid` at all. */
|
|
23
|
+
kid: string | null | undefined = signingKeyIdOfPublicKey(pair.publicKey),
|
|
24
|
+
): string {
|
|
25
|
+
return jwt.sign({ sub: 'worker-caller' }, pair.privateKey, {
|
|
19
26
|
algorithm: 'ES256',
|
|
20
|
-
header: { alg: 'ES256', typ },
|
|
27
|
+
header: { alg: 'ES256', typ, ...(kid ? { kid } : {}) },
|
|
21
28
|
issuer: 'wildo-application',
|
|
22
29
|
audience: 'APPLICATION_WORKER',
|
|
23
30
|
expiresIn: '5m',
|
|
@@ -40,7 +47,7 @@ describe('worker JWT rotated-key retirement', () => {
|
|
|
40
47
|
vi.setSystemTime(new Date('2026-08-12T12:00:00.000Z'));
|
|
41
48
|
const current = generateSigningPair();
|
|
42
49
|
const rotated = generateSigningPair();
|
|
43
|
-
const token = signWorkerToken(rotated
|
|
50
|
+
const token = signWorkerToken(rotated);
|
|
44
51
|
let notAfter = new Date('2026-08-12T12:01:00.000Z');
|
|
45
52
|
const middleware = createWorkerJwtVerifierMiddleware({
|
|
46
53
|
getCurrentPublicKey: () => current.publicKey,
|
|
@@ -59,18 +66,106 @@ describe('worker JWT rotated-key retirement', () => {
|
|
|
59
66
|
await middleware({ headers: { authorization: `Bearer ${token}` } } as never, refused.response as never, refusedNext);
|
|
60
67
|
expect(refusedNext).not.toHaveBeenCalled();
|
|
61
68
|
expect(refused.response.status).toHaveBeenCalledWith(401);
|
|
62
|
-
expect(refused.response.json).toHaveBeenCalledWith({ error:
|
|
69
|
+
expect(refused.response.json).toHaveBeenCalledWith({ error: WorkerJwtRefusal.UNKNOWN_KID });
|
|
63
70
|
});
|
|
64
71
|
|
|
65
72
|
// #2892 — mutation: drop the typ check → a token another of the application's keys signed for another use (same
|
|
66
73
|
// issuer, same audience forged) is admitted by a worker holding only the worker-request public key.
|
|
67
74
|
it('refuses a token whose type is not the worker-request type, even under the trusted key', async () => {
|
|
68
|
-
const
|
|
69
|
-
const middleware = createWorkerJwtVerifierMiddleware({ getCurrentPublicKey: () => publicKey });
|
|
75
|
+
const pair = generateSigningPair();
|
|
76
|
+
const middleware = createWorkerJwtVerifierMiddleware({ getCurrentPublicKey: () => pair.publicKey });
|
|
70
77
|
const { response } = responseDouble();
|
|
71
78
|
const next = vi.fn();
|
|
72
|
-
await middleware({ headers: { authorization: `Bearer ${signWorkerToken(
|
|
79
|
+
await middleware({ headers: { authorization: `Bearer ${signWorkerToken(pair, 'wildo-access+jwt')}` } } as never, response as never, next);
|
|
73
80
|
expect(next).not.toHaveBeenCalled();
|
|
74
81
|
expect(response.status).toHaveBeenCalledWith(401);
|
|
75
82
|
});
|
|
76
83
|
});
|
|
84
|
+
|
|
85
|
+
async function run(middleware: ReturnType<typeof createWorkerJwtVerifierMiddleware>, token: string) {
|
|
86
|
+
const { response } = responseDouble();
|
|
87
|
+
const next = vi.fn();
|
|
88
|
+
await middleware({ headers: { authorization: `Bearer ${token}` } } as never, response as never, next);
|
|
89
|
+
return { admitted: next.mock.calls.length === 1, response };
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
// #2895 — the key is selected by the token's kid; no other key is tried, and an unknown kid fetches the bootstrap again
|
|
93
|
+
// at most once per interval.
|
|
94
|
+
describe('worker JWT key selection by kid (#2895)', () => {
|
|
95
|
+
afterEach(() => {
|
|
96
|
+
vi.useRealTimers();
|
|
97
|
+
});
|
|
98
|
+
|
|
99
|
+
it('verifies under the key the kid names, and tries no other', async () => {
|
|
100
|
+
const current = generateSigningPair();
|
|
101
|
+
const retired = generateSigningPair();
|
|
102
|
+
const middleware = createWorkerJwtVerifierMiddleware({
|
|
103
|
+
getCurrentPublicKey: () => current.publicKey,
|
|
104
|
+
getRotatedPublicKeys: () => [{ kid: 'label-not-read', publicKey: retired.publicKey }],
|
|
105
|
+
});
|
|
106
|
+
expect((await run(middleware, signWorkerToken(current))).admitted).toBe(true);
|
|
107
|
+
expect((await run(middleware, signWorkerToken(retired))).admitted).toBe(true);
|
|
108
|
+
// Signed by the retired key but naming the current one: only the current key is tried, and it refuses.
|
|
109
|
+
const misnamed = await run(middleware, signWorkerToken(retired, undefined, signingKeyIdOfPublicKey(current.publicKey)));
|
|
110
|
+
expect(misnamed.admitted).toBe(false);
|
|
111
|
+
expect(misnamed.response.json).toHaveBeenCalledWith({ error: WorkerJwtRefusal.INVALID_JWT });
|
|
112
|
+
});
|
|
113
|
+
|
|
114
|
+
it('refuses a token that names no key', async () => {
|
|
115
|
+
const current = generateSigningPair();
|
|
116
|
+
const middleware = createWorkerJwtVerifierMiddleware({ getCurrentPublicKey: () => current.publicKey });
|
|
117
|
+
const outcome = await run(middleware, signWorkerToken(current, undefined, null));
|
|
118
|
+
expect(outcome.admitted).toBe(false);
|
|
119
|
+
expect(outcome.response.json).toHaveBeenCalledWith({ error: WorkerJwtRefusal.MISSING_KID });
|
|
120
|
+
});
|
|
121
|
+
|
|
122
|
+
it('fetches the bootstrap again for an unknown kid, at most once per interval, then admits the new key', async () => {
|
|
123
|
+
vi.useFakeTimers();
|
|
124
|
+
vi.setSystemTime(new Date('2026-11-09T12:00:00.000Z'));
|
|
125
|
+
const loaded = generateSigningPair();
|
|
126
|
+
const rotatedTo = generateSigningPair();
|
|
127
|
+
let held = loaded.publicKey;
|
|
128
|
+
const refresh = vi.fn(async () => {
|
|
129
|
+
held = rotatedTo.publicKey;
|
|
130
|
+
});
|
|
131
|
+
const middleware = createWorkerJwtVerifierMiddleware({ getCurrentPublicKey: () => held, refreshKeysForUnknownKid: refresh });
|
|
132
|
+
|
|
133
|
+
// The backend already signs with the new key: the first request fetches the bootstrap and is admitted.
|
|
134
|
+
expect((await run(middleware, signWorkerToken(rotatedTo))).admitted).toBe(true);
|
|
135
|
+
expect(refresh).toHaveBeenCalledOnce();
|
|
136
|
+
|
|
137
|
+
// A key nobody publishes: refused, and inside the interval it costs no further fetch.
|
|
138
|
+
const stranger = generateSigningPair();
|
|
139
|
+
const refused = await run(middleware, signWorkerToken(stranger));
|
|
140
|
+
expect(refused.response.json).toHaveBeenCalledWith({ error: WorkerJwtRefusal.UNKNOWN_KID });
|
|
141
|
+
expect(refresh).toHaveBeenCalledOnce();
|
|
142
|
+
|
|
143
|
+
vi.setSystemTime(new Date(Date.now() + UNKNOWN_SIGNING_KEY_ID_REFRESH_MIN_INTERVAL_MS));
|
|
144
|
+
await run(middleware, signWorkerToken(stranger));
|
|
145
|
+
expect(refresh).toHaveBeenCalledTimes(2);
|
|
146
|
+
});
|
|
147
|
+
|
|
148
|
+
it('concurrent requests naming the same new key share one fetch', async () => {
|
|
149
|
+
const loaded = generateSigningPair();
|
|
150
|
+
const rotatedTo = generateSigningPair();
|
|
151
|
+
let held = loaded.publicKey;
|
|
152
|
+
let release!: () => void;
|
|
153
|
+
const refresh = vi.fn(
|
|
154
|
+
() =>
|
|
155
|
+
new Promise<void>((resolve) => {
|
|
156
|
+
release = () => {
|
|
157
|
+
held = rotatedTo.publicKey;
|
|
158
|
+
resolve();
|
|
159
|
+
};
|
|
160
|
+
}),
|
|
161
|
+
);
|
|
162
|
+
const middleware = createWorkerJwtVerifierMiddleware({ getCurrentPublicKey: () => held, refreshKeysForUnknownKid: refresh });
|
|
163
|
+
const first = run(middleware, signWorkerToken(rotatedTo));
|
|
164
|
+
const second = run(middleware, signWorkerToken(rotatedTo));
|
|
165
|
+
await vi.waitFor(() => expect(refresh).toHaveBeenCalledOnce());
|
|
166
|
+
release();
|
|
167
|
+
expect((await first).admitted).toBe(true);
|
|
168
|
+
expect((await second).admitted).toBe(true);
|
|
169
|
+
expect(refresh).toHaveBeenCalledOnce();
|
|
170
|
+
});
|
|
171
|
+
});
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import type { Request, RequestHandler, Response, NextFunction } from 'express';
|
|
2
2
|
import jwt from 'jsonwebtoken';
|
|
3
|
-
import { JwtVerifyOptions } from '@wildo-ai/saas-backend-lib';
|
|
3
|
+
import { JwtVerifyOptions, signingKeyIdOfPublicKey, UNKNOWN_SIGNING_KEY_ID_REFRESH_MIN_INTERVAL_MS } from '@wildo-ai/saas-backend-lib';
|
|
4
4
|
import { CryptoKey_Purpose } from '@wildo-ai/saas-models';
|
|
5
5
|
|
|
6
6
|
/**
|
|
@@ -37,11 +37,28 @@ export interface WorkerJwtVerifierFactoryInput {
|
|
|
37
37
|
*/
|
|
38
38
|
getCurrentPublicKey: () => string;
|
|
39
39
|
/**
|
|
40
|
-
* Optional set of
|
|
41
|
-
*
|
|
42
|
-
* Provided by the bootstrap response payload.
|
|
40
|
+
* Optional set of retired public keys still trusted during a rotation window, each until its `notAfter`. Provided by
|
|
41
|
+
* the bootstrap response payload. The `kid` label is not read: every key is filed under its own RFC 7638 thumbprint.
|
|
43
42
|
*/
|
|
44
43
|
getRotatedPublicKeys?: () => Array<{ kid: string; publicKey: string; notAfter?: Date | string }>;
|
|
44
|
+
/**
|
|
45
|
+
* Fetch the bootstrap again, because a token named a key this worker does not hold (#2895): the first requests after
|
|
46
|
+
* a rotation look exactly like that. Called at most once per {@link UNKNOWN_SIGNING_KEY_ID_REFRESH_MIN_INTERVAL_MS},
|
|
47
|
+
* concurrent requests sharing the one call. Absent, an unknown `kid` is refused at once.
|
|
48
|
+
*/
|
|
49
|
+
refreshKeysForUnknownKid?: () => Promise<void>;
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
/** Why a worker request was refused: the `error` of the `401` body. */
|
|
53
|
+
export enum WorkerJwtRefusal {
|
|
54
|
+
MISSING_AUTHORIZATION = 'missing_authorization',
|
|
55
|
+
EMPTY_AUTHORIZATION_TOKEN = 'empty_authorization_token',
|
|
56
|
+
/** The token names no key: every application signer writes its key's `kid`. */
|
|
57
|
+
MISSING_KID = 'missing_kid',
|
|
58
|
+
/** The token names a key this worker does not hold, even after fetching the bootstrap again. */
|
|
59
|
+
UNKNOWN_KID = 'unknown_kid',
|
|
60
|
+
/** The named key does not verify the token, or a claim, the type or the lifetime is wrong. */
|
|
61
|
+
INVALID_JWT = 'invalid_jwt',
|
|
45
62
|
}
|
|
46
63
|
|
|
47
64
|
/**
|
|
@@ -49,6 +66,11 @@ export interface WorkerJwtVerifierFactoryInput {
|
|
|
49
66
|
* `Authorization: Bearer <jwt>` headers against the trusted
|
|
50
67
|
* `APPLICATION_WORKER_REQUEST` public key (Q3 LOCKED — dynamic resolver).
|
|
51
68
|
*
|
|
69
|
+
* #2895 — the token is verified under the ONE key its header's `kid` names: the current key or a retired key still
|
|
70
|
+
* before its `notAfter`, each filed under its RFC 7638 thumbprint (`signingKeyIdOfPublicKey`, the same function the
|
|
71
|
+
* backend names its signing key with). No other key is tried: trying every key would accept a token under a key it does
|
|
72
|
+
* not name. An unknown `kid` fetches the bootstrap again (bounded) and is looked up once more before it is refused.
|
|
73
|
+
*
|
|
52
74
|
* Independent of any DI container: takes plain functions returning the
|
|
53
75
|
* current trusted key(s). The host wires those functions to the
|
|
54
76
|
* bootstrap response managed by `StartWorkerApplicationBackendService`.
|
|
@@ -59,37 +81,83 @@ export function createWorkerJwtVerifierMiddleware(
|
|
|
59
81
|
// The worker-request key signs nothing else, and its tokens carry their purpose's own type (#2892).
|
|
60
82
|
const { expectedTyp, ...verifyOptions } = JwtVerifyOptions[CryptoKey_Purpose.APPLICATION_WORKER_REQUEST];
|
|
61
83
|
|
|
84
|
+
const kidByPublicKey = new Map<string, string | undefined>();
|
|
85
|
+
const kidOf = (publicKey: string): string | undefined => {
|
|
86
|
+
if (!kidByPublicKey.has(publicKey)) {
|
|
87
|
+
let kid: string | undefined;
|
|
88
|
+
try {
|
|
89
|
+
kid = signingKeyIdOfPublicKey(publicKey);
|
|
90
|
+
} catch {
|
|
91
|
+
// A value that is not a public key names no kid.
|
|
92
|
+
kid = undefined;
|
|
93
|
+
}
|
|
94
|
+
kidByPublicKey.set(publicKey, kid);
|
|
95
|
+
}
|
|
96
|
+
return kidByPublicKey.get(publicKey);
|
|
97
|
+
};
|
|
98
|
+
const findKey = (kid: string): string | undefined => {
|
|
99
|
+
const now = Date.now();
|
|
100
|
+
const candidates = [
|
|
101
|
+
input.getCurrentPublicKey(),
|
|
102
|
+
...(input.getRotatedPublicKeys?.() ?? [])
|
|
103
|
+
.filter((candidate) => !candidate.notAfter || new Date(candidate.notAfter).getTime() > now)
|
|
104
|
+
.map((candidate) => candidate.publicKey),
|
|
105
|
+
];
|
|
106
|
+
return candidates.find((candidate) => kidOf(candidate) === kid);
|
|
107
|
+
};
|
|
108
|
+
|
|
109
|
+
let lastRefreshAt: number | undefined;
|
|
110
|
+
let refreshInFlight: Promise<void> | undefined;
|
|
111
|
+
const refreshForUnknownKid = async (): Promise<boolean> => {
|
|
112
|
+
if (!input.refreshKeysForUnknownKid) return false;
|
|
113
|
+
if (!refreshInFlight) {
|
|
114
|
+
const now = Date.now();
|
|
115
|
+
if (lastRefreshAt !== undefined && now - lastRefreshAt < UNKNOWN_SIGNING_KEY_ID_REFRESH_MIN_INTERVAL_MS) return false;
|
|
116
|
+
lastRefreshAt = now;
|
|
117
|
+
refreshInFlight = input
|
|
118
|
+
.refreshKeysForUnknownKid()
|
|
119
|
+
.catch(() => {
|
|
120
|
+
// A failed fetch keeps the keys held; the request is then refused as naming an unknown key.
|
|
121
|
+
})
|
|
122
|
+
.finally(() => {
|
|
123
|
+
refreshInFlight = undefined;
|
|
124
|
+
});
|
|
125
|
+
}
|
|
126
|
+
await refreshInFlight;
|
|
127
|
+
return true;
|
|
128
|
+
};
|
|
129
|
+
|
|
62
130
|
return async (req: Request, res: Response, next: NextFunction): Promise<void> => {
|
|
63
131
|
const authHeader = req.headers.authorization ?? req.headers.Authorization;
|
|
64
132
|
const headerValue = Array.isArray(authHeader) ? authHeader[0] : authHeader;
|
|
65
133
|
if (!headerValue || typeof headerValue !== 'string' || !headerValue.startsWith('Bearer ')) {
|
|
66
|
-
res.status(401).json({ error:
|
|
134
|
+
res.status(401).json({ error: WorkerJwtRefusal.MISSING_AUTHORIZATION });
|
|
67
135
|
return;
|
|
68
136
|
}
|
|
69
137
|
const token = headerValue.slice('Bearer '.length).trim();
|
|
70
138
|
if (!token) {
|
|
71
|
-
res.status(401).json({ error:
|
|
139
|
+
res.status(401).json({ error: WorkerJwtRefusal.EMPTY_AUTHORIZATION_TOKEN });
|
|
72
140
|
return;
|
|
73
141
|
}
|
|
74
142
|
|
|
75
|
-
const
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
143
|
+
const kid = jwt.decode(token, { complete: true })?.header?.kid;
|
|
144
|
+
if (!kid) {
|
|
145
|
+
res.status(401).json({ error: WorkerJwtRefusal.MISSING_KID });
|
|
146
|
+
return;
|
|
147
|
+
}
|
|
148
|
+
const publicKey = findKey(kid) ?? ((await refreshForUnknownKid()) ? findKey(kid) : undefined);
|
|
149
|
+
if (!publicKey) {
|
|
150
|
+
res.status(401).json({ error: WorkerJwtRefusal.UNKNOWN_KID });
|
|
151
|
+
return;
|
|
152
|
+
}
|
|
81
153
|
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
return;
|
|
88
|
-
} catch {
|
|
89
|
-
// try the next candidate; only emit 401 once all keys have failed
|
|
90
|
-
}
|
|
154
|
+
try {
|
|
155
|
+
req.workerJwt = await verifyOnce(token, publicKey, verifyOptions, expectedTyp);
|
|
156
|
+
} catch {
|
|
157
|
+
res.status(401).json({ error: WorkerJwtRefusal.INVALID_JWT });
|
|
158
|
+
return;
|
|
91
159
|
}
|
|
92
|
-
|
|
160
|
+
next();
|
|
93
161
|
};
|
|
94
162
|
}
|
|
95
163
|
|
|
@@ -1,10 +1,25 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { existsSync, mkdtempSync, readFileSync, rmSync } from 'node:fs';
|
|
2
|
+
import { createServer } from 'node:http';
|
|
3
|
+
import type { AddressInfo } from 'node:net';
|
|
4
|
+
import { tmpdir } from 'node:os';
|
|
5
|
+
import { join } from 'node:path';
|
|
6
|
+
|
|
7
|
+
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
|
2
8
|
import type { WildoWorkerBootstrapClientService } from '@wildo-ai/saas-backend-lib';
|
|
3
9
|
import {
|
|
4
10
|
PROVIDER_RUNTIME_ENV_KEYS,
|
|
5
11
|
type AppWorkerBootstrapResponse,
|
|
6
12
|
} from '@wildo-ai/platform-config-lib';
|
|
7
|
-
import { BackgroundRuntimeType } from '@wildo-ai/saas-models';
|
|
13
|
+
import { BackgroundRuntimeType, ProviderDeploymentModel } from '@wildo-ai/saas-models';
|
|
14
|
+
import {
|
|
15
|
+
ApplicationWorkerRuntimeEnvKeys,
|
|
16
|
+
BACKEND_START_OUTCOME_DIRECTORY,
|
|
17
|
+
BackendStartFailureCause,
|
|
18
|
+
BackendStartOutcome,
|
|
19
|
+
BackendStartOutcomeRecord_Schema,
|
|
20
|
+
backendStartOutcomeFileName,
|
|
21
|
+
type BackendStartOutcomeRecord,
|
|
22
|
+
} from '@wildo-ai/platform-core-shared';
|
|
8
23
|
|
|
9
24
|
import {
|
|
10
25
|
defineWorkerInit,
|
|
@@ -179,6 +194,38 @@ describe('StartWorkerApplicationBackendService', () => {
|
|
|
179
194
|
}));
|
|
180
195
|
});
|
|
181
196
|
|
|
197
|
+
it('forwards the declared data residency narrowed to this worker\'s providers, so its loader re-applies it (#2294)', async () => {
|
|
198
|
+
const { service } = instantiate();
|
|
199
|
+
const providerRuntime = {
|
|
200
|
+
...createWorkerProviderRuntimeArtifact(),
|
|
201
|
+
providerDataResidency: {
|
|
202
|
+
territories: ['eu'],
|
|
203
|
+
choices: {
|
|
204
|
+
'app-illustration': { deploymentModel: ProviderDeploymentModel.VENDOR_HOSTED, processingRegion: 'EU' },
|
|
205
|
+
'sibling-provider': { deploymentModel: ProviderDeploymentModel.VENDOR_HOSTED, processingRegion: 'US' },
|
|
206
|
+
},
|
|
207
|
+
},
|
|
208
|
+
};
|
|
209
|
+
withMatchingProviderRuntimeEnv({ refs: providerRuntime.providerRefs!, digest: providerRuntime.sourceDigest });
|
|
210
|
+
|
|
211
|
+
await service.initialize({
|
|
212
|
+
applicationId: 'app-1',
|
|
213
|
+
platformApplicationPrimarySecret: TEST_PLATFORM_PRIMARY_SECRET,
|
|
214
|
+
config: defineWorkerInit({
|
|
215
|
+
workerName: 'imageWorker',
|
|
216
|
+
serviceKind: BackgroundRuntimeType.WORKER,
|
|
217
|
+
http: { port: 4010, healthPath: '/healthz', readyPath: '/readyz' },
|
|
218
|
+
appsManagerBootstrapUrl: 'https://apps-manager.example.com',
|
|
219
|
+
providerRuntime,
|
|
220
|
+
}),
|
|
221
|
+
});
|
|
222
|
+
|
|
223
|
+
expect(service.getProviderRuntime()?.providerDataResidency).toEqual({
|
|
224
|
+
territories: ['eu'],
|
|
225
|
+
choices: { 'app-illustration': { deploymentModel: ProviderDeploymentModel.VENDOR_HOSTED, processingRegion: 'EU' } },
|
|
226
|
+
});
|
|
227
|
+
});
|
|
228
|
+
|
|
182
229
|
it('fails fast when the generated worker runtime omits the current worker scope', async () => {
|
|
183
230
|
const { workerHttpServer, service } = instantiate();
|
|
184
231
|
|
|
@@ -317,3 +364,69 @@ describe('StartWorkerApplicationBackendService', () => {
|
|
|
317
364
|
}));
|
|
318
365
|
});
|
|
319
366
|
});
|
|
367
|
+
|
|
368
|
+
/**
|
|
369
|
+
* A node worker records how its start ended (#3288), so the dev supervisor can name a process holding the worker's port
|
|
370
|
+
* as it does for the backend, instead of crash-looping into CRASH_RESPAWN_EXHAUSTED.
|
|
371
|
+
*/
|
|
372
|
+
describe('StartWorkerApplicationBackendService records its start outcome (#3288)', () => {
|
|
373
|
+
let signalsHome: string;
|
|
374
|
+
beforeEach(() => {
|
|
375
|
+
signalsHome = mkdtempSync(join(tmpdir(), 'worker-start-record-'));
|
|
376
|
+
vi.stubEnv(ApplicationWorkerRuntimeEnvKeys.APPLICATION_RUNTIME_SIGNALS_HOME, signalsHome);
|
|
377
|
+
});
|
|
378
|
+
afterEach(() => {
|
|
379
|
+
vi.unstubAllEnvs();
|
|
380
|
+
rmSync(signalsHome, { recursive: true, force: true });
|
|
381
|
+
});
|
|
382
|
+
|
|
383
|
+
const initializeWith = (service: StartWorkerApplicationBackendService): Promise<void> => service.initialize({
|
|
384
|
+
applicationId: 'app-1',
|
|
385
|
+
platformApplicationPrimarySecret: TEST_PLATFORM_PRIMARY_SECRET,
|
|
386
|
+
config: defineWorkerInit({
|
|
387
|
+
workerName: 'imageWorker',
|
|
388
|
+
serviceKind: BackgroundRuntimeType.WORKER,
|
|
389
|
+
http: { port: 4010, healthPath: '/healthz', readyPath: '/readyz' },
|
|
390
|
+
appsManagerBootstrapUrl: 'https://apps-manager.example.com',
|
|
391
|
+
}),
|
|
392
|
+
});
|
|
393
|
+
const record = (): BackendStartOutcomeRecord => BackendStartOutcomeRecord_Schema.parse(
|
|
394
|
+
JSON.parse(readFileSync(join(signalsHome, BACKEND_START_OUTCOME_DIRECTORY, backendStartOutcomeFileName('workers.imageWorker')), 'utf8')),
|
|
395
|
+
);
|
|
396
|
+
|
|
397
|
+
it('records STARTED, under this process, once the HTTP server listens', async () => {
|
|
398
|
+
const { service } = instantiate();
|
|
399
|
+
await initializeWith(service);
|
|
400
|
+
expect(record()).toMatchObject({ outcome: BackendStartOutcome.STARTED, serviceIdentifier: 'workers.imageWorker', pid: process.pid });
|
|
401
|
+
});
|
|
402
|
+
|
|
403
|
+
it('records a held port as port-in-use WITH the port, and still throws the error', async () => {
|
|
404
|
+
const { service, workerHttpServer } = instantiate();
|
|
405
|
+
// The REAL error a Node HTTP server rejects with when its port is held, as the worker's own server forwards it
|
|
406
|
+
// (`server.on('error', reject)`): taken by binding a port twice, so the classification is read off Node's shape.
|
|
407
|
+
const holder = createServer();
|
|
408
|
+
await new Promise<void>((resolve) => holder.listen(0, '127.0.0.1', () => resolve()));
|
|
409
|
+
const port = (holder.address() as AddressInfo).port;
|
|
410
|
+
const held = await new Promise<Error>((resolve) => {
|
|
411
|
+
const second = createServer();
|
|
412
|
+
second.on('error', resolve);
|
|
413
|
+
second.listen(port, '127.0.0.1');
|
|
414
|
+
});
|
|
415
|
+
await new Promise<void>((resolve) => holder.close(() => resolve()));
|
|
416
|
+
vi.mocked(workerHttpServer.initialize).mockRejectedValueOnce(held);
|
|
417
|
+
await expect(initializeWith(service)).rejects.toBe(held);
|
|
418
|
+
expect(record()).toMatchObject({
|
|
419
|
+
outcome: BackendStartOutcome.FAILED,
|
|
420
|
+
failureCause: BackendStartFailureCause.PORT_IN_USE,
|
|
421
|
+
port,
|
|
422
|
+
pid: process.pid,
|
|
423
|
+
});
|
|
424
|
+
});
|
|
425
|
+
|
|
426
|
+
it('writes nothing where no runtime-signals home is set (every non-local deployment)', async () => {
|
|
427
|
+
vi.unstubAllEnvs();
|
|
428
|
+
const { service } = instantiate();
|
|
429
|
+
await initializeWith(service);
|
|
430
|
+
expect(existsSync(join(signalsHome, BACKEND_START_OUTCOME_DIRECTORY))).toBe(false);
|
|
431
|
+
});
|
|
432
|
+
});
|
|
@@ -21,6 +21,10 @@ export interface WorkerProviderRuntimeArtifact {
|
|
|
21
21
|
providerRuntimeImports?: ProviderRuntimeArtifact['providerRuntimeImports'];
|
|
22
22
|
/** Mirrored for the same reason: the projection narrows and forwards it (#1672). */
|
|
23
23
|
providerRuntimeAugmentations?: ProviderRuntimeArtifact['providerRuntimeAugmentations'];
|
|
24
|
+
/** Mirrored for the same reason: the projection narrows and forwards it (#3513). */
|
|
25
|
+
providerProcessingRegions?: ProviderRuntimeArtifact['providerProcessingRegions'];
|
|
26
|
+
/** Mirrored for the same reason: the projection narrows and forwards it (#2294), and the loader re-applies it. */
|
|
27
|
+
providerDataResidency?: ProviderRuntimeArtifact['providerDataResidency'];
|
|
24
28
|
providerGraphDigest?: ProviderRuntimeArtifact['providerGraphDigest'];
|
|
25
29
|
lockfileSectionDigest?: ProviderRuntimeArtifact['lockfileSectionDigest'];
|
|
26
30
|
artifactKind: 'nodeWorkers';
|
|
@@ -29,8 +33,7 @@ export interface WorkerProviderRuntimeArtifact {
|
|
|
29
33
|
}
|
|
30
34
|
|
|
31
35
|
/**
|
|
32
|
-
* Inputs the worker host needs to bootstrap. Intentionally minimal
|
|
33
|
-
* of `.claude/plans/minions-and-workers-implementation.md`: the worker has
|
|
36
|
+
* Inputs the worker host needs to bootstrap. Intentionally minimal: the worker has
|
|
34
37
|
* no backing-service access (no Mongo / Redis / RabbitMQ), only:
|
|
35
38
|
* - the four-env bootstrap-env-contract
|
|
36
39
|
* (`APPLICATION_ID`, `PLATFORM_APPLICATION_PRIMARY_SECRET`,
|