@pikku/core 0.12.78 → 0.12.79
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +73 -0
- package/dist/services/in-memory-workflow-service.d.ts +10 -0
- package/dist/services/in-memory-workflow-service.js +23 -0
- package/dist/utils/node-host-resolver.d.ts +12 -0
- package/dist/utils/node-host-resolver.js +16 -0
- package/dist/utils/safe-fetch.d.ts +18 -0
- package/dist/utils/safe-fetch.js +167 -29
- package/dist/wirings/gateway/gateway-runner.js +32 -3
- package/dist/wirings/secret/validate-secret-definitions.js +2 -0
- package/dist/wirings/workflow/pikku-workflow-service.d.ts +52 -0
- package/dist/wirings/workflow/pikku-workflow-service.js +104 -0
- package/knowledge/decisions/security/gateway-handlers-run-through-the-function-runner-gate.md +13 -3
- package/package.json +2 -1
- package/src/services/in-memory-workflow-service.ts +23 -0
- package/src/utils/node-host-resolver.ts +20 -0
- package/src/utils/safe-fetch.test.ts +143 -1
- package/src/utils/safe-fetch.ts +200 -25
- package/src/wirings/gateway/gateway-authorization.test.ts +131 -0
- package/src/wirings/gateway/gateway-runner.ts +35 -3
- package/src/wirings/secret/validate-secret-definitions.test.ts +47 -0
- package/src/wirings/secret/validate-secret-definitions.ts +2 -0
- package/src/wirings/workflow/pikku-workflow-service.ts +131 -0
- package/src/wirings/workflow/workflow-dispatch-relay.test.ts +128 -0
- package/tsconfig.tsbuildinfo +1 -1
|
@@ -146,6 +146,20 @@ const WORKFLOW_TERMINAL_STATES = new Set([
|
|
|
146
146
|
const DEFAULT_STALLED_RUN_MS = 5 * 60_000;
|
|
147
147
|
/** Runs re-driven per `recoverStalledRuns` call, so one sweep is bounded. */
|
|
148
148
|
const DEFAULT_STALLED_RUN_LIMIT = 100;
|
|
149
|
+
/**
|
|
150
|
+
* How long a step may sit `pending` before the relay assumes its dispatch was
|
|
151
|
+
* lost. This is a bet that no healthy queue takes this long to move a job from
|
|
152
|
+
* `pending` to `running`; set it above the observed p99 of that latency.
|
|
153
|
+
*/
|
|
154
|
+
const DEFAULT_UNDISPATCHED_STEP_MS = 30_000;
|
|
155
|
+
/** Steps re-driven per `relayUndispatchedSteps` call, so one tick is bounded. */
|
|
156
|
+
const DEFAULT_UNDISPATCHED_STEP_LIMIT = 100;
|
|
157
|
+
/** First wait before a step already re-dispatched once is re-dispatched again. */
|
|
158
|
+
const REDISPATCH_BACKOFF_MS = 30_000;
|
|
159
|
+
/** Ceiling on the doubling backoff, so a permanently stuck step still gets swept. */
|
|
160
|
+
const REDISPATCH_BACKOFF_MAX_MS = 10 * 60_000;
|
|
161
|
+
/** Bound on the in-process backoff map, so a long-lived process cannot grow it without bound. */
|
|
162
|
+
const REDISPATCH_BACKOFF_MAX_ENTRIES = 10_000;
|
|
149
163
|
const WORKFLOW_POLL_MIN_MS = 10;
|
|
150
164
|
const WORKFLOW_POLL_FACTOR = 1.6;
|
|
151
165
|
const WORKFLOW_CHILD_POLL_MAX_MS = 500;
|
|
@@ -486,6 +500,96 @@ export class PikkuWorkflowService {
|
|
|
486
500
|
}
|
|
487
501
|
return { resumed };
|
|
488
502
|
}
|
|
503
|
+
/**
|
|
504
|
+
* Runs holding a step that has sat `pending` since before `before`, paired
|
|
505
|
+
* with the step that flagged them.
|
|
506
|
+
*
|
|
507
|
+
* Returns nothing by default so a store that cannot express the query keeps
|
|
508
|
+
* working unchanged — and, because it does not opt in, gains no re-dispatches
|
|
509
|
+
* either. A store must have an atomic `withStepLock` before overriding this,
|
|
510
|
+
* or no concurrency for one to exclude: the relay makes duplicate dispatch
|
|
511
|
+
* routine, and the claim in `executeWorkflowStepInner` is what keeps a
|
|
512
|
+
* duplicate from becoming a second execution. `kysely-postgres` and
|
|
513
|
+
* `kysely-mysql` qualify on the lock, `in-memory` on being inline and
|
|
514
|
+
* single-process; `mongodb` and `kysely-sqlite` qualify on neither.
|
|
515
|
+
*/
|
|
516
|
+
async findUndispatchedSteps(_before, _limit) {
|
|
517
|
+
return [];
|
|
518
|
+
}
|
|
519
|
+
/**
|
|
520
|
+
* Re-drive steps whose dispatch was lost, and report which runs were nudged.
|
|
521
|
+
*
|
|
522
|
+
* Arming a step is two writes to two systems: the step row lands `pending`,
|
|
523
|
+
* then a queue or scheduler job is published. Nothing spans both, so a crash
|
|
524
|
+
* in between leaves a durable row that nothing will ever pick up — the run
|
|
525
|
+
* neither finishes nor fails. (Seen on a `workflow.sleep()`: a deploy restart
|
|
526
|
+
* landed between the sleep step's insert and its timer.)
|
|
527
|
+
*
|
|
528
|
+
* The row is the outbox record and this is the relay. Age is the only signal
|
|
529
|
+
* available — a step `pending` because its dispatch was lost is
|
|
530
|
+
* indistinguishable from one whose job is merely still queued — so a step
|
|
531
|
+
* past `undispatchedAfterMs` is re-dispatched regardless, and correctness
|
|
532
|
+
* rests on the claim rather than on the guess being right. A redundant
|
|
533
|
+
* dispatch costs one queue message: the loser reads `running` and returns
|
|
534
|
+
* without invoking anything.
|
|
535
|
+
*
|
|
536
|
+
* Re-dispatches back off per step (doubling from 30s, capped at 10m) so a
|
|
537
|
+
* genuine queue backlog is not amplified by a tick that keeps firing at the
|
|
538
|
+
* steps the backlog is already delaying. The backoff is per process and
|
|
539
|
+
* advisory — losing it on restart costs extra dispatches, never correctness.
|
|
540
|
+
*
|
|
541
|
+
* This is not self-starting. Call it from a scheduled task; ~30s suits a
|
|
542
|
+
* queue whose `pending`→`running` latency is well under that.
|
|
543
|
+
*/
|
|
544
|
+
async relayUndispatchedSteps(options) {
|
|
545
|
+
const before = new Date(Date.now() -
|
|
546
|
+
(options?.undispatchedAfterMs ?? DEFAULT_UNDISPATCHED_STEP_MS));
|
|
547
|
+
const steps = await this.findUndispatchedSteps(before, options?.limit ?? DEFAULT_UNDISPATCHED_STEP_LIMIT);
|
|
548
|
+
// Backoff is keyed by run, not step, because the run is the unit of
|
|
549
|
+
// re-drive: `resumeWorkflow` replays the whole run and re-dispatches every
|
|
550
|
+
// step still owed a job. Holding off a single step while resuming its run
|
|
551
|
+
// would not suppress anything.
|
|
552
|
+
const now = Date.now();
|
|
553
|
+
const runIds = new Set();
|
|
554
|
+
for (const { runId } of steps) {
|
|
555
|
+
const eligibleAt = this.redispatchBackoff.get(runId);
|
|
556
|
+
if (eligibleAt !== undefined && eligibleAt > now) {
|
|
557
|
+
continue;
|
|
558
|
+
}
|
|
559
|
+
this.noteRedispatch(runId, now);
|
|
560
|
+
runIds.add(runId);
|
|
561
|
+
}
|
|
562
|
+
const redispatched = [];
|
|
563
|
+
for (const runId of runIds) {
|
|
564
|
+
try {
|
|
565
|
+
await this.resumeWorkflow(runId);
|
|
566
|
+
redispatched.push(runId);
|
|
567
|
+
}
|
|
568
|
+
catch (err) {
|
|
569
|
+
// One unresumable run must not stop the tick from relaying the rest.
|
|
570
|
+
getSingletonServices()?.logger?.error(`Failed to re-dispatch workflow run ${runId}: ${err instanceof Error ? err.message : String(err)}`);
|
|
571
|
+
}
|
|
572
|
+
}
|
|
573
|
+
return { redispatched };
|
|
574
|
+
}
|
|
575
|
+
/** Advisory, per-process record of when a run may next be re-dispatched. */
|
|
576
|
+
redispatchBackoff = new Map();
|
|
577
|
+
redispatchDelays = new Map();
|
|
578
|
+
noteRedispatch(runId, now) {
|
|
579
|
+
const previous = this.redispatchDelays.get(runId);
|
|
580
|
+
const delay = Math.min(previous === undefined ? REDISPATCH_BACKOFF_MS : previous * 2, REDISPATCH_BACKOFF_MAX_MS);
|
|
581
|
+
// A run that settles is never returned again, so entries are only evicted by
|
|
582
|
+
// this bound — oldest first, which is also least recently re-dispatched.
|
|
583
|
+
if (this.redispatchBackoff.size >= REDISPATCH_BACKOFF_MAX_ENTRIES) {
|
|
584
|
+
const oldest = this.redispatchBackoff.keys().next();
|
|
585
|
+
if (!oldest.done) {
|
|
586
|
+
this.redispatchBackoff.delete(oldest.value);
|
|
587
|
+
this.redispatchDelays.delete(oldest.value);
|
|
588
|
+
}
|
|
589
|
+
}
|
|
590
|
+
this.redispatchDelays.set(runId, delay);
|
|
591
|
+
this.redispatchBackoff.set(runId, now + delay);
|
|
592
|
+
}
|
|
489
593
|
resolveStepJobOptions(stepOptions) {
|
|
490
594
|
const retries = stepOptions?.retries ?? DEFAULT_STEP_RETRIES;
|
|
491
595
|
const retryDelay = stepOptions?.retryDelay;
|
package/knowledge/decisions/security/gateway-handlers-run-through-the-function-runner-gate.md
CHANGED
|
@@ -17,7 +17,16 @@ runner's gate is the only thing that evaluates a function's declared `auth`,
|
|
|
17
17
|
`scopes` and `permissions`; a direct call runs the handler with none of them
|
|
18
18
|
checked.
|
|
19
19
|
|
|
20
|
-
The
|
|
20
|
+
The synthetic registration **inherits the wired function's own metadata** —
|
|
21
|
+
`sessionless`, input and output schema names, `scopes`, tag middleware — read
|
|
22
|
+
back from `pikkuState(null, 'gateway', 'meta')[name].pikkuFuncId`, which the
|
|
23
|
+
inspector already records. The synthetic id exists so the gate runs; it is not a
|
|
24
|
+
licence to run the handler under metadata its author never wrote. A handler
|
|
25
|
+
declared with `pikkuFunc` says session-required in meta rather than through an
|
|
26
|
+
`auth` property, and that declaration is honoured.
|
|
27
|
+
|
|
28
|
+
When nothing was declared — no inspector entry, as when a gateway is wired by
|
|
29
|
+
hand — the handler falls back to `sessionless: true`. Gateway inbound traffic is
|
|
21
30
|
authenticated by the platform adapter (webhook signature verification, platform
|
|
22
31
|
tokens), not by a user session, so defaulting to session-required would reject
|
|
23
32
|
every legitimate webhook. `CoreGateway.auth` and the handler's own `auth: true`
|
|
@@ -26,6 +35,7 @@ whenever declared, session or not.
|
|
|
26
35
|
|
|
27
36
|
**What this rules out:** invoking `config.func` directly from any gateway
|
|
28
37
|
transport as an optimisation, and "simplifying" the synthetic function
|
|
29
|
-
registration away. It also rules out
|
|
30
|
-
|
|
38
|
+
registration away. It also rules out fabricating the handler's metadata rather
|
|
39
|
+
than inheriting it, and flipping the _fallback_ to session-required as a
|
|
40
|
+
hardening measure — that breaks every webhook that declared nothing, rather than
|
|
31
41
|
securing it; require auth per gateway via `CoreGateway.auth` instead.
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@pikku/core",
|
|
3
|
-
"version": "0.12.
|
|
3
|
+
"version": "0.12.79",
|
|
4
4
|
"description": "The Pikku runtime — functions, wirings, services, middleware and types",
|
|
5
5
|
"author": "yasser.fadl@gmail.com",
|
|
6
6
|
"license": "MIT",
|
|
@@ -37,6 +37,7 @@
|
|
|
37
37
|
"./trigger": "./dist/wirings/trigger/index.js",
|
|
38
38
|
"./rpc": "./dist/wirings/rpc/index.js",
|
|
39
39
|
"./safe-fetch": "./dist/utils/safe-fetch.js",
|
|
40
|
+
"./node-host-resolver": "./dist/utils/node-host-resolver.js",
|
|
40
41
|
"./mcp": "./dist/wirings/mcp/index.js",
|
|
41
42
|
"./ai-agent": "./dist/wirings/ai-agent/index.js",
|
|
42
43
|
"./gateway": "./dist/wirings/gateway/index.js",
|
|
@@ -301,6 +301,29 @@ export class InMemoryWorkflowService
|
|
|
301
301
|
return newStep
|
|
302
302
|
}
|
|
303
303
|
|
|
304
|
+
/**
|
|
305
|
+
* Safe to opt in despite the pass-through `withStepLock`: this service wires
|
|
306
|
+
* no queues and runs every step inline in one process, so the relay's
|
|
307
|
+
* redundant dispatch has no second holder to race with. See
|
|
308
|
+
* knowledge: decisions/internals/the-in-memory-workflow-service-is-inline-only-and-single-process.md
|
|
309
|
+
*/
|
|
310
|
+
protected async findUndispatchedSteps(
|
|
311
|
+
before: Date,
|
|
312
|
+
limit: number
|
|
313
|
+
): Promise<Array<{ runId: string; stepId: string }>> {
|
|
314
|
+
const undispatched: Array<{ runId: string; stepId: string }> = []
|
|
315
|
+
for (const [runId, run] of this.runs) {
|
|
316
|
+
if (run.status !== 'running') continue
|
|
317
|
+
for (const step of this.stepHistory.get(runId) ?? []) {
|
|
318
|
+
if (step.status !== 'pending') continue
|
|
319
|
+
if (step.updatedAt >= before) continue
|
|
320
|
+
undispatched.push({ runId, stepId: step.stepId })
|
|
321
|
+
if (undispatched.length >= limit) return undispatched
|
|
322
|
+
}
|
|
323
|
+
}
|
|
324
|
+
return undispatched
|
|
325
|
+
}
|
|
326
|
+
|
|
304
327
|
protected async findStalledRunIds(
|
|
305
328
|
before: Date,
|
|
306
329
|
limit: number
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
import { lookup } from 'node:dns/promises'
|
|
2
|
+
|
|
3
|
+
import { setDefaultHostResolver, type HostResolver } from './safe-fetch.js'
|
|
4
|
+
|
|
5
|
+
/**
|
|
6
|
+
* Resolves a hostname through the platform resolver, returning every address it
|
|
7
|
+
* points at so `safeFetch` can reject a public name aimed at an internal one.
|
|
8
|
+
*
|
|
9
|
+
* This module is Node-only and is never imported by core itself — Workers has no
|
|
10
|
+
* DNS API, and a static `node:dns` import in the shared path would break that
|
|
11
|
+
* build.
|
|
12
|
+
*/
|
|
13
|
+
export const nodeHostResolver: HostResolver = async (hostname) => {
|
|
14
|
+
const results = await lookup(hostname, { all: true, verbatim: true })
|
|
15
|
+
return results.map(({ address }) => address)
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
/** Installs {@link nodeHostResolver} as the default for every `safeFetch`. */
|
|
19
|
+
export const installNodeHostResolver = () =>
|
|
20
|
+
setDefaultHostResolver(nodeHostResolver)
|
|
@@ -41,15 +41,54 @@ describe('isPrivateHost', () => {
|
|
|
41
41
|
}
|
|
42
42
|
})
|
|
43
43
|
|
|
44
|
+
test('flags reserved IPv4 blocks that are not routable public space', () => {
|
|
45
|
+
for (const host of [
|
|
46
|
+
'100.64.0.1', // carrier-grade NAT
|
|
47
|
+
'100.100.100.200', // Alibaba Cloud metadata endpoint
|
|
48
|
+
'100.127.255.255', // top of 100.64.0.0/10
|
|
49
|
+
'192.0.0.192', // IETF protocol assignments
|
|
50
|
+
'198.18.0.1', // benchmarking
|
|
51
|
+
'198.19.255.255', // top of 198.18.0.0/15
|
|
52
|
+
'192.88.99.1', // 6to4 anycast
|
|
53
|
+
'192.0.2.1', // TEST-NET-1
|
|
54
|
+
'198.51.100.1', // TEST-NET-2
|
|
55
|
+
'203.0.113.1', // TEST-NET-3
|
|
56
|
+
'224.0.0.1', // multicast
|
|
57
|
+
'240.0.0.1', // reserved
|
|
58
|
+
'255.255.255.255', // broadcast
|
|
59
|
+
]) {
|
|
60
|
+
assert.equal(isPrivateHost(host), true, `${host} should be private`)
|
|
61
|
+
}
|
|
62
|
+
})
|
|
63
|
+
|
|
64
|
+
test('flags IPv6 forms that tunnel to an internal IPv4 address', () => {
|
|
65
|
+
for (const host of [
|
|
66
|
+
'64:ff9b::a9fe:a9fe', // NAT64 wrapping 169.254.169.254
|
|
67
|
+
'[64:ff9b::169.254.169.254]', // same, dotted-quad tail
|
|
68
|
+
'2002:a9fe:a9fe::', // 6to4 wrapping 169.254.169.254
|
|
69
|
+
'2002:6464:64c8::', // 6to4 wrapping 100.100.100.200
|
|
70
|
+
'fec0::1', // deprecated site-local
|
|
71
|
+
'ff02::1', // multicast
|
|
72
|
+
'100::1', // discard-only
|
|
73
|
+
]) {
|
|
74
|
+
assert.equal(isPrivateHost(host), true, `${host} should be private`)
|
|
75
|
+
}
|
|
76
|
+
})
|
|
77
|
+
|
|
44
78
|
test('allows public hosts', () => {
|
|
45
79
|
for (const host of [
|
|
46
80
|
'example.com',
|
|
47
81
|
'8.8.8.8',
|
|
48
82
|
'172.32.0.1',
|
|
49
83
|
'11.0.0.1',
|
|
84
|
+
'100.63.255.255', // just below the CGNAT block
|
|
85
|
+
'100.128.0.0', // just above the CGNAT block
|
|
86
|
+
'198.17.255.255', // just below the benchmarking block
|
|
87
|
+
'198.20.0.0', // just above the benchmarking block
|
|
50
88
|
'134744072', // decimal-encoded 8.8.8.8 — public
|
|
51
89
|
'2001:db8::1', // documentation range — public
|
|
52
|
-
'
|
|
90
|
+
'2002:808:808::', // 6to4 wrapping public 8.8.8.8
|
|
91
|
+
'64:ff9b::808:808', // NAT64 wrapping public 8.8.8.8
|
|
53
92
|
]) {
|
|
54
93
|
assert.equal(isPrivateHost(host), false, `${host} should be public`)
|
|
55
94
|
}
|
|
@@ -138,6 +177,109 @@ describe('safeFetch', () => {
|
|
|
138
177
|
)
|
|
139
178
|
})
|
|
140
179
|
|
|
180
|
+
test('refuses a public hostname that resolves to a private address', async () => {
|
|
181
|
+
await withStubbedFetch(
|
|
182
|
+
() => new Response('ok', { status: 200 }),
|
|
183
|
+
async (calls) => {
|
|
184
|
+
// The shape the literal-only check misses: a name that parses as public
|
|
185
|
+
// but whose A record points at the cloud metadata endpoint.
|
|
186
|
+
await assert.rejects(
|
|
187
|
+
safeFetch(
|
|
188
|
+
'https://169-254-169-254.nip.io/latest/meta-data/',
|
|
189
|
+
{},
|
|
190
|
+
{
|
|
191
|
+
resolveHost: async () => ['169.254.169.254'],
|
|
192
|
+
}
|
|
193
|
+
),
|
|
194
|
+
/resolves to 169\.254\.169\.254/
|
|
195
|
+
)
|
|
196
|
+
assert.equal(calls.length, 0)
|
|
197
|
+
}
|
|
198
|
+
)
|
|
199
|
+
})
|
|
200
|
+
|
|
201
|
+
test('refuses when any one resolved address is private', async () => {
|
|
202
|
+
await withStubbedFetch(
|
|
203
|
+
() => new Response('ok', { status: 200 }),
|
|
204
|
+
async (calls) => {
|
|
205
|
+
await assert.rejects(
|
|
206
|
+
safeFetch(
|
|
207
|
+
'https://split-horizon.example.com/',
|
|
208
|
+
{},
|
|
209
|
+
{
|
|
210
|
+
resolveHost: async () => ['93.184.216.34', '10.0.0.5'],
|
|
211
|
+
}
|
|
212
|
+
),
|
|
213
|
+
/resolves to 10\.0\.0\.5/
|
|
214
|
+
)
|
|
215
|
+
assert.equal(calls.length, 0)
|
|
216
|
+
}
|
|
217
|
+
)
|
|
218
|
+
})
|
|
219
|
+
|
|
220
|
+
test('re-resolves the host of a redirect hop', async () => {
|
|
221
|
+
await withStubbedFetch(
|
|
222
|
+
(url) =>
|
|
223
|
+
url.includes('start.com')
|
|
224
|
+
? new Response(null, {
|
|
225
|
+
status: 302,
|
|
226
|
+
headers: { location: 'https://rebound.example.com/next' },
|
|
227
|
+
})
|
|
228
|
+
: new Response('ok', { status: 200 }),
|
|
229
|
+
async (calls) => {
|
|
230
|
+
await assert.rejects(
|
|
231
|
+
safeFetch(
|
|
232
|
+
'https://start.com',
|
|
233
|
+
{},
|
|
234
|
+
{
|
|
235
|
+
resolveHost: async (hostname) =>
|
|
236
|
+
hostname === 'start.com' ? ['93.184.216.34'] : ['127.0.0.1'],
|
|
237
|
+
}
|
|
238
|
+
),
|
|
239
|
+
/resolves to 127\.0\.0\.1/
|
|
240
|
+
)
|
|
241
|
+
assert.equal(calls.length, 1)
|
|
242
|
+
}
|
|
243
|
+
)
|
|
244
|
+
})
|
|
245
|
+
|
|
246
|
+
test('allows a public hostname that resolves to public addresses', async () => {
|
|
247
|
+
await withStubbedFetch(
|
|
248
|
+
() => new Response('ok', { status: 200 }),
|
|
249
|
+
async (calls) => {
|
|
250
|
+
const res = await safeFetch(
|
|
251
|
+
'https://example.com/',
|
|
252
|
+
{},
|
|
253
|
+
{
|
|
254
|
+
resolveHost: async () => ['93.184.216.34'],
|
|
255
|
+
}
|
|
256
|
+
)
|
|
257
|
+
assert.equal(res.status, 200)
|
|
258
|
+
assert.equal(calls.length, 1)
|
|
259
|
+
}
|
|
260
|
+
)
|
|
261
|
+
})
|
|
262
|
+
|
|
263
|
+
test('does not resolve an IP literal or an allowlisted host', async () => {
|
|
264
|
+
await withStubbedFetch(
|
|
265
|
+
() => new Response('ok', { status: 200 }),
|
|
266
|
+
async () => {
|
|
267
|
+
let resolverCalls = 0
|
|
268
|
+
const resolveHost = async () => {
|
|
269
|
+
resolverCalls++
|
|
270
|
+
return ['10.0.0.1']
|
|
271
|
+
}
|
|
272
|
+
await safeFetch('https://93.184.216.34/', {}, { resolveHost })
|
|
273
|
+
await safeFetch(
|
|
274
|
+
'https://internal.example.com/',
|
|
275
|
+
{},
|
|
276
|
+
{ resolveHost, allowedHosts: ['internal.example.com'] }
|
|
277
|
+
)
|
|
278
|
+
assert.equal(resolverCalls, 0)
|
|
279
|
+
}
|
|
280
|
+
)
|
|
281
|
+
})
|
|
282
|
+
|
|
141
283
|
test('follows a redirect to another public host', async () => {
|
|
142
284
|
await withStubbedFetch(
|
|
143
285
|
(url) =>
|
package/src/utils/safe-fetch.ts
CHANGED
|
@@ -32,6 +32,134 @@ function parseIPv4Octets(
|
|
|
32
32
|
return octets as [number, number, number, number]
|
|
33
33
|
}
|
|
34
34
|
|
|
35
|
+
/**
|
|
36
|
+
* IPv4 blocks that must never be reachable from user-supplied URLs: private,
|
|
37
|
+
* loopback, link-local (cloud metadata), carrier-grade NAT (Alibaba's
|
|
38
|
+
* `100.100.100.200` metadata endpoint), IETF protocol assignments, benchmarking,
|
|
39
|
+
* 6to4 anycast, the documentation TEST-NETs, multicast and reserved space.
|
|
40
|
+
*/
|
|
41
|
+
const PRIVATE_IPV4_BLOCKS: ReadonlyArray<readonly [string, number]> = [
|
|
42
|
+
['0.0.0.0', 8],
|
|
43
|
+
['10.0.0.0', 8],
|
|
44
|
+
['100.64.0.0', 10],
|
|
45
|
+
['127.0.0.0', 8],
|
|
46
|
+
['169.254.0.0', 16],
|
|
47
|
+
['172.16.0.0', 12],
|
|
48
|
+
['192.0.0.0', 24],
|
|
49
|
+
['192.0.2.0', 24],
|
|
50
|
+
['192.88.99.0', 24],
|
|
51
|
+
['192.168.0.0', 16],
|
|
52
|
+
['198.18.0.0', 15],
|
|
53
|
+
['198.51.100.0', 24],
|
|
54
|
+
['203.0.113.0', 24],
|
|
55
|
+
['224.0.0.0', 4],
|
|
56
|
+
['240.0.0.0', 4],
|
|
57
|
+
]
|
|
58
|
+
|
|
59
|
+
const toUint32 = (octets: [number, number, number, number]): number =>
|
|
60
|
+
((octets[0] << 24) | (octets[1] << 16) | (octets[2] << 8) | octets[3]) >>> 0
|
|
61
|
+
|
|
62
|
+
const PRIVATE_IPV4_RANGES = PRIVATE_IPV4_BLOCKS.map(([base, prefix]) => {
|
|
63
|
+
const mask = prefix === 0 ? 0 : (0xffffffff << (32 - prefix)) >>> 0
|
|
64
|
+
return [(toUint32(parseIPv4Octets(base)!) & mask) >>> 0, mask] as const
|
|
65
|
+
})
|
|
66
|
+
|
|
67
|
+
const isPrivateIPv4 = (octets: [number, number, number, number]): boolean => {
|
|
68
|
+
const addr = toUint32(octets)
|
|
69
|
+
return PRIVATE_IPV4_RANGES.some(
|
|
70
|
+
([base, mask]) => (addr & mask) >>> 0 === base
|
|
71
|
+
)
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
/**
|
|
75
|
+
* Expands an IPv6 literal into its eight 16-bit groups, handling `::` elision
|
|
76
|
+
* and a trailing dotted-quad. `null` when not a well-formed IPv6 literal.
|
|
77
|
+
*/
|
|
78
|
+
function parseIPv6Groups(host: string): number[] | null {
|
|
79
|
+
const zoneless = host.split('%')[0]!
|
|
80
|
+
const halves = zoneless.split('::')
|
|
81
|
+
if (halves.length > 2) return null
|
|
82
|
+
|
|
83
|
+
const parseSide = (side: string): number[] | null => {
|
|
84
|
+
if (side === '') return []
|
|
85
|
+
const parts = side.split(':')
|
|
86
|
+
const groups: number[] = []
|
|
87
|
+
for (let i = 0; i < parts.length; i++) {
|
|
88
|
+
const part = parts[i]!
|
|
89
|
+
if (i === parts.length - 1 && part.includes('.')) {
|
|
90
|
+
const v4 = parseIPv4Octets(part)
|
|
91
|
+
if (!v4) return null
|
|
92
|
+
groups.push((v4[0] << 8) | v4[1], (v4[2] << 8) | v4[3])
|
|
93
|
+
continue
|
|
94
|
+
}
|
|
95
|
+
if (!/^[0-9a-f]{1,4}$/.test(part)) return null
|
|
96
|
+
groups.push(parseInt(part, 16))
|
|
97
|
+
}
|
|
98
|
+
return groups
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
const head = parseSide(halves[0]!)
|
|
102
|
+
if (head === null) return null
|
|
103
|
+
|
|
104
|
+
if (halves.length === 1) {
|
|
105
|
+
return head.length === 8 ? head : null
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
const tail = parseSide(halves[1]!)
|
|
109
|
+
if (tail === null) return null
|
|
110
|
+
if (head.length + tail.length > 7) return null
|
|
111
|
+
return [...head, ...new Array(8 - head.length - tail.length).fill(0), ...tail]
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
const embeddedIPv4 = (
|
|
115
|
+
hi: number,
|
|
116
|
+
lo: number
|
|
117
|
+
): [number, number, number, number] => [
|
|
118
|
+
(hi >> 8) & 0xff,
|
|
119
|
+
hi & 0xff,
|
|
120
|
+
(lo >> 8) & 0xff,
|
|
121
|
+
lo & 0xff,
|
|
122
|
+
]
|
|
123
|
+
|
|
124
|
+
function isPrivateIPv6(groups: number[]): boolean {
|
|
125
|
+
const [g0, g1, g2, g3, g4, g5, g6, g7] = groups as [
|
|
126
|
+
number,
|
|
127
|
+
number,
|
|
128
|
+
number,
|
|
129
|
+
number,
|
|
130
|
+
number,
|
|
131
|
+
number,
|
|
132
|
+
number,
|
|
133
|
+
number,
|
|
134
|
+
]
|
|
135
|
+
|
|
136
|
+
if (groups.every((g) => g === 0)) return true // :: unspecified
|
|
137
|
+
if (groups.slice(0, 7).every((g) => g === 0) && g7 === 1) return true // ::1
|
|
138
|
+
|
|
139
|
+
// IPv4-mapped ::ffff:0:0/96 and IPv4-compatible ::/96
|
|
140
|
+
if (g0 === 0 && g1 === 0 && g2 === 0 && g3 === 0 && g4 === 0) {
|
|
141
|
+
if (g5 === 0xffff || g5 === 0) return isPrivateIPv4(embeddedIPv4(g6, g7))
|
|
142
|
+
}
|
|
143
|
+
// NAT64 well-known prefix 64:ff9b::/96
|
|
144
|
+
if (
|
|
145
|
+
g0 === 0x64 &&
|
|
146
|
+
g1 === 0xff9b &&
|
|
147
|
+
g2 === 0 &&
|
|
148
|
+
g3 === 0 &&
|
|
149
|
+
g4 === 0 &&
|
|
150
|
+
g5 === 0
|
|
151
|
+
)
|
|
152
|
+
return isPrivateIPv4(embeddedIPv4(g6, g7))
|
|
153
|
+
// 6to4 2002::/16 carries the IPv4 address in the next 32 bits
|
|
154
|
+
if (g0 === 0x2002) return isPrivateIPv4(embeddedIPv4(g1, g2))
|
|
155
|
+
if (g0 === 0x100 && g1 === 0 && g2 === 0 && g3 === 0) return true // discard-only 100::/64
|
|
156
|
+
if ((g0 & 0xffc0) === 0xfe80) return true // link-local fe80::/10
|
|
157
|
+
if ((g0 & 0xfe00) === 0xfc00) return true // unique-local fc00::/7
|
|
158
|
+
if ((g0 & 0xffc0) === 0xfec0) return true // deprecated site-local fec0::/10
|
|
159
|
+
if ((g0 & 0xff00) === 0xff00) return true // multicast ff00::/8
|
|
160
|
+
return false
|
|
161
|
+
}
|
|
162
|
+
|
|
35
163
|
/**
|
|
36
164
|
* Whether a hostname is an obvious internal target. Best-effort literal
|
|
37
165
|
* matching only: it cannot catch a public hostname that *resolves* to a
|
|
@@ -46,31 +174,30 @@ export function isPrivateHost(hostname: string): boolean {
|
|
|
46
174
|
return true
|
|
47
175
|
|
|
48
176
|
if (host.includes(':')) {
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
if (mappedV4) return isPrivateHost(mappedV4[1]!)
|
|
52
|
-
const mappedHex = host.match(/^::ffff:([0-9a-f]{1,4}):([0-9a-f]{1,4})$/)
|
|
53
|
-
if (mappedHex) {
|
|
54
|
-
const hi = parseInt(mappedHex[1]!, 16)
|
|
55
|
-
const lo = parseInt(mappedHex[2]!, 16)
|
|
56
|
-
return isPrivateHost(
|
|
57
|
-
`${(hi >> 8) & 0xff}.${hi & 0xff}.${(lo >> 8) & 0xff}.${lo & 0xff}`
|
|
58
|
-
)
|
|
59
|
-
}
|
|
60
|
-
if (/^fe[89ab]/.test(host)) return true // link-local fe80::/10
|
|
61
|
-
if (host.startsWith('fc') || host.startsWith('fd')) return true // unique-local fc00::/7
|
|
62
|
-
return false
|
|
177
|
+
const groups = parseIPv6Groups(host)
|
|
178
|
+
return groups ? isPrivateIPv6(groups) : false
|
|
63
179
|
}
|
|
64
180
|
|
|
65
181
|
const v4 = parseIPv4Octets(host)
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
182
|
+
return v4 ? isPrivateIPv4(v4) : false
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
/** Resolves a hostname to the IP addresses it points at. */
|
|
186
|
+
export type HostResolver = (hostname: string) => Promise<string[]>
|
|
187
|
+
|
|
188
|
+
let defaultHostResolver: HostResolver | undefined
|
|
189
|
+
|
|
190
|
+
/**
|
|
191
|
+
* Installs the resolver {@link safeFetch} uses when a call passes no
|
|
192
|
+
* `resolveHost` of its own.
|
|
193
|
+
*
|
|
194
|
+
* Core cannot resolve DNS itself — Workers has no DNS API — so without a
|
|
195
|
+
* resolver the guard is literal-only and a public name pointing at
|
|
196
|
+
* `169.254.169.254` passes. Node runtimes install
|
|
197
|
+
* `nodeHostResolver` from `@pikku/core/node-host-resolver` at startup.
|
|
198
|
+
*/
|
|
199
|
+
export function setDefaultHostResolver(resolver: HostResolver | undefined) {
|
|
200
|
+
defaultHostResolver = resolver
|
|
74
201
|
}
|
|
75
202
|
|
|
76
203
|
export interface SafeFetchOptions {
|
|
@@ -81,6 +208,50 @@ export interface SafeFetchOptions {
|
|
|
81
208
|
allowedHosts?: string[]
|
|
82
209
|
/** Maximum redirect hops to follow (each re-validated). Defaults to 3. */
|
|
83
210
|
maxRedirects?: number
|
|
211
|
+
/**
|
|
212
|
+
* Resolves a hostname so a *public* name pointing at a private address is
|
|
213
|
+
* refused. Defaults to whatever {@link setDefaultHostResolver} installed;
|
|
214
|
+
* pass `null` to opt a call out of resolution entirely.
|
|
215
|
+
*/
|
|
216
|
+
resolveHost?: HostResolver | null
|
|
217
|
+
}
|
|
218
|
+
|
|
219
|
+
/** Whether a hostname is already an IP literal, which the sync check covers. */
|
|
220
|
+
function isIpLiteral(hostname: string): boolean {
|
|
221
|
+
const host = hostname.replace(/^\[|\]$/g, '').replace(/\.$/, '')
|
|
222
|
+
if (host.includes(':')) return parseIPv6Groups(host.toLowerCase()) !== null
|
|
223
|
+
return parseIPv4Octets(host) !== null
|
|
224
|
+
}
|
|
225
|
+
|
|
226
|
+
/**
|
|
227
|
+
* Rejects a hostname that resolves to an internal address.
|
|
228
|
+
*
|
|
229
|
+
* Resolution happens once per hop and the connection is not pinned to the
|
|
230
|
+
* address checked, so a rebind between this check and the socket connecting is
|
|
231
|
+
* still possible; catching that needs a runtime-level connect hook.
|
|
232
|
+
*/
|
|
233
|
+
async function assertResolvedHostAllowed(
|
|
234
|
+
hostname: string,
|
|
235
|
+
options: SafeFetchOptions
|
|
236
|
+
): Promise<void> {
|
|
237
|
+
if (options.allowedHosts) return
|
|
238
|
+
const resolver =
|
|
239
|
+
options.resolveHost === null
|
|
240
|
+
? undefined
|
|
241
|
+
: (options.resolveHost ?? defaultHostResolver)
|
|
242
|
+
if (!resolver || isIpLiteral(hostname)) return
|
|
243
|
+
|
|
244
|
+
const addresses = await resolver(hostname)
|
|
245
|
+
if (addresses.length === 0) {
|
|
246
|
+
throw new Error(`Refusing to fetch: '${hostname}' resolved to no addresses`)
|
|
247
|
+
}
|
|
248
|
+
for (const address of addresses) {
|
|
249
|
+
if (isPrivateHost(address)) {
|
|
250
|
+
throw new Error(
|
|
251
|
+
`Refusing to fetch from a private/internal host: '${hostname}' resolves to ${address}`
|
|
252
|
+
)
|
|
253
|
+
}
|
|
254
|
+
}
|
|
84
255
|
}
|
|
85
256
|
|
|
86
257
|
export function assertFetchableUrl(
|
|
@@ -142,7 +313,9 @@ export async function safeFetch(
|
|
|
142
313
|
options: SafeFetchOptions = {}
|
|
143
314
|
): Promise<Response> {
|
|
144
315
|
const maxRedirects = options.maxRedirects ?? 3
|
|
145
|
-
|
|
316
|
+
const initial = assertFetchableUrl(url, options)
|
|
317
|
+
await assertResolvedHostAllowed(initial.hostname, options)
|
|
318
|
+
let currentUrl = initial.toString()
|
|
146
319
|
let currentInit = init
|
|
147
320
|
|
|
148
321
|
for (let hop = 0; ; hop++) {
|
|
@@ -157,10 +330,12 @@ export async function safeFetch(
|
|
|
157
330
|
if (!location || hop >= maxRedirects) {
|
|
158
331
|
return response
|
|
159
332
|
}
|
|
160
|
-
const
|
|
333
|
+
const next = assertFetchableUrl(
|
|
161
334
|
new URL(location, currentUrl).toString(),
|
|
162
335
|
options
|
|
163
|
-
)
|
|
336
|
+
)
|
|
337
|
+
await assertResolvedHostAllowed(next.hostname, options)
|
|
338
|
+
const nextUrl = next.toString()
|
|
164
339
|
await response.body?.cancel()
|
|
165
340
|
let nextInit = redirectInit(response.status, currentInit)
|
|
166
341
|
if (new URL(nextUrl).origin !== new URL(currentUrl).origin) {
|