deepline 0.1.109 → 0.1.111
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli/index.js +2634 -1532
- package/dist/cli/index.mjs +2547 -1451
- package/dist/index.d.mts +21 -14
- package/dist/index.d.ts +21 -14
- package/dist/index.js +97 -23
- package/dist/index.mjs +97 -23
- package/dist/repo/apps/play-runner-workers/src/coordinator-entry.ts +192 -121
- package/dist/repo/apps/play-runner-workers/src/entry.ts +254 -65
- package/dist/repo/apps/play-runner-workers/src/runtime/receipts.ts +18 -27
- package/dist/repo/apps/play-runner-workers/src/workflow-instance-create.ts +44 -0
- package/dist/repo/apps/play-runner-workers/src/workflow-retry.ts +7 -11
- package/dist/repo/sdk/src/client.ts +35 -12
- package/dist/repo/sdk/src/errors.ts +2 -2
- package/dist/repo/sdk/src/http.ts +87 -7
- package/dist/repo/sdk/src/play.ts +1 -1
- package/dist/repo/sdk/src/plays/bundle-play-file.ts +5 -1
- package/dist/repo/sdk/src/release.ts +13 -10
- package/dist/repo/sdk/src/tool-output.ts +2 -2
- package/dist/repo/sdk/src/types.ts +9 -6
- package/dist/repo/shared_libs/play-runtime/fullenrich-batching.ts +229 -0
- package/dist/repo/shared_libs/play-runtime/governor/policy.ts +1 -1
- package/dist/repo/shared_libs/play-runtime/play-runtime-batching-registry.ts +20 -0
- package/dist/repo/shared_libs/play-runtime/run-failure.ts +20 -12
- package/dist/repo/shared_libs/play-runtime/run-ledger.ts +147 -70
- package/dist/repo/shared_libs/play-runtime/scheduler-backend.ts +6 -2
- package/dist/repo/shared_libs/play-runtime/secret-redaction.ts +15 -0
- package/dist/repo/shared_libs/play-runtime/work-receipts.ts +1 -0
- package/dist/repo/shared_libs/plays/bundling/index.ts +193 -21
- package/dist/repo/shared_libs/plays/static-pipeline.ts +1 -3
- package/dist/repo/shared_libs/security/outbound-url-policy.ts +238 -0
- package/dist/repo/shared_libs/security/safe-fetch.ts +118 -0
- package/dist/viewer/viewer.css +617 -0
- package/dist/viewer/viewer.js +1496 -0
- package/package.json +5 -1
|
@@ -7,8 +7,12 @@ const PRIVATE_KEY_RE =
|
|
|
7
7
|
/-----BEGIN [A-Z0-9 ]*PRIVATE KEY-----[\s\S]*?-----END [A-Z0-9 ]*PRIVATE KEY-----/g;
|
|
8
8
|
const COMMON_SECRET_RE =
|
|
9
9
|
/\b(?:sk|pk|rk|pat|ghp|github_pat|xox[baprs]|key|token|secret|api[_-]?key)[A-Za-z0-9_./+=:-]{12,}\b/gi;
|
|
10
|
+
const URL_SECRET_VALUE_RE =
|
|
11
|
+
/\b(?:sk|pk|rk|pat|ghp|github_pat|xox)[A-Za-z0-9_./+=:-]{12,}\b/i;
|
|
10
12
|
const HIGH_ENTROPY_ASSIGNMENT_RE =
|
|
11
13
|
/\b(?:api[_-]?key|token|secret|password|authorization|access[_-]?token|refresh[_-]?token)\b\s*[:=]\s*["']?[^"',\s]{16,}["']?/gi;
|
|
14
|
+
const SENSITIVE_URL_PARAM_RE =
|
|
15
|
+
/[?#&](?:\w*(?:key|token)|secret|password|authorization|credential|signature|sig)(?:=|$)/i;
|
|
12
16
|
|
|
13
17
|
function escapeRegExp(value: string): string {
|
|
14
18
|
return value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
|
|
@@ -19,6 +23,17 @@ function isRecord(value: unknown): value is Record<string, unknown> {
|
|
|
19
23
|
}
|
|
20
24
|
|
|
21
25
|
export function redactSecretLikeString(value: string): string {
|
|
26
|
+
const trimmed = value.trim();
|
|
27
|
+
if (/^https?:\/\/\S+$/i.test(trimmed)) {
|
|
28
|
+
if (
|
|
29
|
+
/^https?:\/\/[^/?#@]+@/i.test(trimmed) ||
|
|
30
|
+
SENSITIVE_URL_PARAM_RE.test(trimmed)
|
|
31
|
+
) {
|
|
32
|
+
return SECRET_REDACTION_PLACEHOLDER;
|
|
33
|
+
}
|
|
34
|
+
if (!URL_SECRET_VALUE_RE.test(value)) return value;
|
|
35
|
+
}
|
|
36
|
+
|
|
22
37
|
return value
|
|
23
38
|
.replace(PRIVATE_KEY_RE, SECRET_REDACTION_PLACEHOLDER)
|
|
24
39
|
.replace(BEARER_TOKEN_RE, `Bearer ${SECRET_REDACTION_PLACEHOLDER}`)
|
|
@@ -8,6 +8,7 @@ import {
|
|
|
8
8
|
extname,
|
|
9
9
|
isAbsolute,
|
|
10
10
|
join,
|
|
11
|
+
relative,
|
|
11
12
|
resolve,
|
|
12
13
|
} from 'node:path';
|
|
13
14
|
import { builtinModules } from 'node:module';
|
|
@@ -96,6 +97,7 @@ export type PlayBundlingAdapter = {
|
|
|
96
97
|
sdkWorkersEntryFile: string;
|
|
97
98
|
workersHarnessEntryFile: string;
|
|
98
99
|
workersHarnessFilesDir: string;
|
|
100
|
+
workersRuntimeFingerprintDirs?: string[];
|
|
99
101
|
discoverPackagedLocalFiles(
|
|
100
102
|
filePath: string,
|
|
101
103
|
): Promise<PlayLocalFileDiscoveryResult>;
|
|
@@ -415,6 +417,106 @@ export function extractDefinedPlayName(sourceCode: string): string | null {
|
|
|
415
417
|
return null;
|
|
416
418
|
}
|
|
417
419
|
|
|
420
|
+
function canonicalizeRootPlayNameForWorkersRuntimeHash(
|
|
421
|
+
sourceCode: string,
|
|
422
|
+
): string {
|
|
423
|
+
const source = stripCommentsToSpaces(sourceCode);
|
|
424
|
+
const callPattern =
|
|
425
|
+
/(?:\b[A-Za-z_$][\w$]*\s*\.\s*)?\b(?:definePlay|defineWorkflow)\s*\(/g;
|
|
426
|
+
const match = callPattern.exec(source);
|
|
427
|
+
if (!match) return sourceCode;
|
|
428
|
+
|
|
429
|
+
const openParen = match.index + match[0].length - 1;
|
|
430
|
+
const firstArgStart = openParen + 1;
|
|
431
|
+
const firstNonSpace = source.slice(firstArgStart).search(/\S/);
|
|
432
|
+
if (firstNonSpace < 0) return sourceCode;
|
|
433
|
+
|
|
434
|
+
const argIndex = firstArgStart + firstNonSpace;
|
|
435
|
+
const quote = source[argIndex];
|
|
436
|
+
if (quote === '"' || quote === "'") {
|
|
437
|
+
const literalMatch = source
|
|
438
|
+
.slice(argIndex)
|
|
439
|
+
.match(/^(['"])(?:\\.|(?!\1)[\s\S])*\1/);
|
|
440
|
+
if (!literalMatch) return sourceCode;
|
|
441
|
+
const replacement = `${quote}__deepline_runtime_play_name__${quote}`;
|
|
442
|
+
return `${sourceCode.slice(0, argIndex)}${replacement}${sourceCode.slice(argIndex + literalMatch[0].length)}`;
|
|
443
|
+
}
|
|
444
|
+
|
|
445
|
+
if (quote !== '{') return sourceCode;
|
|
446
|
+
const closeBrace = findMatchingBrace(source, argIndex);
|
|
447
|
+
if (closeBrace < 0) return sourceCode;
|
|
448
|
+
const objectSource = source.slice(argIndex + 1, closeBrace);
|
|
449
|
+
const idMatch = objectSource.match(
|
|
450
|
+
/(^|[,{\s])((?:id|['"]id['"])\s*:\s*)(['"])([\s\S]*?)\3/,
|
|
451
|
+
);
|
|
452
|
+
if (!idMatch || idMatch.index === undefined) return sourceCode;
|
|
453
|
+
const idValueStart =
|
|
454
|
+
argIndex +
|
|
455
|
+
1 +
|
|
456
|
+
idMatch.index +
|
|
457
|
+
idMatch[1]!.length +
|
|
458
|
+
idMatch[2]!.length +
|
|
459
|
+
idMatch[3]!.length;
|
|
460
|
+
return `${sourceCode.slice(0, idValueStart)}__deepline_runtime_play_name__${sourceCode.slice(idValueStart + idMatch[4]!.length)}`;
|
|
461
|
+
}
|
|
462
|
+
|
|
463
|
+
function workersRuntimeGraphFilePath(input: {
|
|
464
|
+
entryFile: string;
|
|
465
|
+
filePath: string;
|
|
466
|
+
adapter: PlayBundlingAdapter;
|
|
467
|
+
}): string {
|
|
468
|
+
if (input.filePath === input.entryFile) return '<entry>';
|
|
469
|
+
const entryRelative = relative(dirname(input.entryFile), input.filePath);
|
|
470
|
+
if (entryRelative && !entryRelative.startsWith('..')) {
|
|
471
|
+
return entryRelative.replaceAll('\\', '/');
|
|
472
|
+
}
|
|
473
|
+
return `<project>/${relative(input.adapter.projectRoot, input.filePath).replaceAll('\\', '/')}`;
|
|
474
|
+
}
|
|
475
|
+
|
|
476
|
+
function buildWorkersRuntimeGraphHash(input: {
|
|
477
|
+
analysis: SourceGraphAnalysis;
|
|
478
|
+
entryFile: string;
|
|
479
|
+
adapter: PlayBundlingAdapter;
|
|
480
|
+
exportName: string;
|
|
481
|
+
}): string {
|
|
482
|
+
const sourceFiles = Object.entries(input.analysis.sourceFiles)
|
|
483
|
+
.map(([filePath, contents]) => ({
|
|
484
|
+
filePath: workersRuntimeGraphFilePath({
|
|
485
|
+
entryFile: input.entryFile,
|
|
486
|
+
filePath,
|
|
487
|
+
adapter: input.adapter,
|
|
488
|
+
}),
|
|
489
|
+
hash: sha256(
|
|
490
|
+
filePath === input.entryFile
|
|
491
|
+
? canonicalizeRootPlayNameForWorkersRuntimeHash(contents)
|
|
492
|
+
: contents,
|
|
493
|
+
),
|
|
494
|
+
}))
|
|
495
|
+
.sort((left, right) => left.filePath.localeCompare(right.filePath));
|
|
496
|
+
|
|
497
|
+
return sha256(
|
|
498
|
+
JSON.stringify({
|
|
499
|
+
entryFile: '<entry>',
|
|
500
|
+
entryExport: input.exportName,
|
|
501
|
+
localFiles: sourceFiles,
|
|
502
|
+
nodeBuiltins: [...input.analysis.importPolicy.nodeBuiltins].sort(),
|
|
503
|
+
packages: input.analysis.importPolicy.packages
|
|
504
|
+
.map(({ name, version }) => ({ name, version }))
|
|
505
|
+
.sort((left, right) => left.name.localeCompare(right.name)),
|
|
506
|
+
importedPlayDependencies: input.analysis.importedPlayDependencies
|
|
507
|
+
.map((dependency) => ({
|
|
508
|
+
filePath: workersRuntimeGraphFilePath({
|
|
509
|
+
entryFile: input.entryFile,
|
|
510
|
+
filePath: dependency.filePath,
|
|
511
|
+
adapter: input.adapter,
|
|
512
|
+
}),
|
|
513
|
+
playName: dependency.playName,
|
|
514
|
+
}))
|
|
515
|
+
.sort((left, right) => left.filePath.localeCompare(right.filePath)),
|
|
516
|
+
}),
|
|
517
|
+
);
|
|
518
|
+
}
|
|
519
|
+
|
|
418
520
|
function readPackageVersionFromPackageJson(
|
|
419
521
|
packageJsonPath: string,
|
|
420
522
|
packageName: string,
|
|
@@ -1063,19 +1165,69 @@ async function analyzeSourceGraph(
|
|
|
1063
1165
|
}
|
|
1064
1166
|
|
|
1065
1167
|
/**
|
|
1066
|
-
* Fingerprint of
|
|
1067
|
-
*
|
|
1068
|
-
*
|
|
1069
|
-
*
|
|
1070
|
-
*
|
|
1071
|
-
*
|
|
1072
|
-
* caching: that's deliberate — caching the fingerprint is exactly the
|
|
1073
|
-
* stale-state bug this exists to prevent.
|
|
1168
|
+
* Fingerprint of Worker runtime sources bundled into every esm_workers play
|
|
1169
|
+
* artifact. Harness, shared runtime, or provider-owned batching edits must
|
|
1170
|
+
* invalidate the bundle cache and force a fresh CF deploy. Computed fresh on
|
|
1171
|
+
* every bundle call so dev edits are picked up on the next `play run` without
|
|
1172
|
+
* restarting the dev server. No caching: that's deliberate — caching the
|
|
1173
|
+
* fingerprint is exactly the stale-state bug this exists to prevent.
|
|
1074
1174
|
*/
|
|
1075
1175
|
async function computeWorkersHarnessFingerprintWithAdapter(
|
|
1076
1176
|
adapter: PlayBundlingAdapter,
|
|
1077
1177
|
): Promise<string> {
|
|
1078
1178
|
const { readdir } = await import('node:fs/promises');
|
|
1179
|
+
const addFilePart = async (
|
|
1180
|
+
parts: Array<{ name: string; hash: string }>,
|
|
1181
|
+
rootDir: string,
|
|
1182
|
+
filePath: string,
|
|
1183
|
+
) => {
|
|
1184
|
+
const contents = await readFile(filePath, 'utf-8');
|
|
1185
|
+
parts.push({
|
|
1186
|
+
name: `${basename(rootDir)}:${filePath.slice(rootDir.length + 1)}`,
|
|
1187
|
+
hash: sha256(contents),
|
|
1188
|
+
});
|
|
1189
|
+
};
|
|
1190
|
+
const collectTopLevelTsFiles = async (
|
|
1191
|
+
rootDir: string,
|
|
1192
|
+
parts: Array<{ name: string; hash: string }>,
|
|
1193
|
+
) => {
|
|
1194
|
+
if (!(await fileExists(rootDir))) return;
|
|
1195
|
+
const entries = await readdir(rootDir, { withFileTypes: true });
|
|
1196
|
+
const tsFiles = entries
|
|
1197
|
+
.filter((entry) => entry.isFile() && /\.[cm]?ts$/.test(entry.name))
|
|
1198
|
+
.map((entry) => entry.name)
|
|
1199
|
+
.sort();
|
|
1200
|
+
for (const name of tsFiles) {
|
|
1201
|
+
await addFilePart(parts, rootDir, join(rootDir, name));
|
|
1202
|
+
}
|
|
1203
|
+
};
|
|
1204
|
+
const collectIntegrationBatchingFiles = async (
|
|
1205
|
+
rootDir: string,
|
|
1206
|
+
parts: Array<{ name: string; hash: string }>,
|
|
1207
|
+
) => {
|
|
1208
|
+
if (!(await fileExists(rootDir))) return;
|
|
1209
|
+
const entries = await readdir(rootDir, { withFileTypes: true });
|
|
1210
|
+
const filePaths: string[] = [];
|
|
1211
|
+
for (const entry of entries) {
|
|
1212
|
+
if (
|
|
1213
|
+
entry.isFile() &&
|
|
1214
|
+
(entry.name === 'play-runtime-batching-registry.ts' ||
|
|
1215
|
+
/^batching.*\.ts$/.test(entry.name))
|
|
1216
|
+
) {
|
|
1217
|
+
filePaths.push(join(rootDir, entry.name));
|
|
1218
|
+
}
|
|
1219
|
+
if (entry.isDirectory()) {
|
|
1220
|
+
const batchingFile = join(rootDir, entry.name, 'batching.ts');
|
|
1221
|
+
if (await fileExists(batchingFile)) {
|
|
1222
|
+
filePaths.push(batchingFile);
|
|
1223
|
+
}
|
|
1224
|
+
}
|
|
1225
|
+
}
|
|
1226
|
+
for (const filePath of filePaths.sort()) {
|
|
1227
|
+
await addFilePart(parts, rootDir, filePath);
|
|
1228
|
+
}
|
|
1229
|
+
};
|
|
1230
|
+
|
|
1079
1231
|
const entries = await readdir(adapter.workersHarnessFilesDir, {
|
|
1080
1232
|
withFileTypes: true,
|
|
1081
1233
|
});
|
|
@@ -1085,11 +1237,18 @@ async function computeWorkersHarnessFingerprintWithAdapter(
|
|
|
1085
1237
|
.sort();
|
|
1086
1238
|
const parts: Array<{ name: string; hash: string }> = [];
|
|
1087
1239
|
for (const name of tsFiles) {
|
|
1088
|
-
|
|
1240
|
+
await addFilePart(
|
|
1241
|
+
parts,
|
|
1242
|
+
adapter.workersHarnessFilesDir,
|
|
1089
1243
|
join(adapter.workersHarnessFilesDir, name),
|
|
1090
|
-
'utf-8',
|
|
1091
1244
|
);
|
|
1092
|
-
|
|
1245
|
+
}
|
|
1246
|
+
for (const dir of adapter.workersRuntimeFingerprintDirs ?? []) {
|
|
1247
|
+
if (basename(dir) === 'integrations') {
|
|
1248
|
+
await collectIntegrationBatchingFiles(dir, parts);
|
|
1249
|
+
} else {
|
|
1250
|
+
await collectTopLevelTsFiles(dir, parts);
|
|
1251
|
+
}
|
|
1093
1252
|
}
|
|
1094
1253
|
return sha256(JSON.stringify(parts));
|
|
1095
1254
|
}
|
|
@@ -1430,9 +1589,15 @@ export async function bundlePlayFile(
|
|
|
1430
1589
|
|
|
1431
1590
|
try {
|
|
1432
1591
|
const analysis = await analyzeSourceGraph(absolutePath, adapter);
|
|
1433
|
-
analysis.graphHash =
|
|
1434
|
-
|
|
1435
|
-
|
|
1592
|
+
analysis.graphHash =
|
|
1593
|
+
target === PLAY_ARTIFACT_KINDS.esmWorkers
|
|
1594
|
+
? buildWorkersRuntimeGraphHash({
|
|
1595
|
+
analysis,
|
|
1596
|
+
entryFile: absolutePath,
|
|
1597
|
+
adapter,
|
|
1598
|
+
exportName,
|
|
1599
|
+
})
|
|
1600
|
+
: sha256(`${analysis.graphHash}\nentry-export:${exportName}`);
|
|
1436
1601
|
// For esm_workers builds, the harness source files (entry.ts +
|
|
1437
1602
|
// peer DO/coordinator types it imports) are bundled INTO every play
|
|
1438
1603
|
// artifact. So any harness edit must produce a different graphHash so
|
|
@@ -1479,11 +1644,10 @@ export async function bundlePlayFile(
|
|
|
1479
1644
|
// Cache lookup happens after validation because a bundle cache hit is keyed
|
|
1480
1645
|
// by source and target, while cloud descriptor typecheck results also depend
|
|
1481
1646
|
// on generated tool metadata.
|
|
1482
|
-
const
|
|
1483
|
-
|
|
1484
|
-
target,
|
|
1485
|
-
|
|
1486
|
-
);
|
|
1647
|
+
const canUseArtifactCache = target !== PLAY_ARTIFACT_KINDS.esmWorkers;
|
|
1648
|
+
const cachedArtifact = canUseArtifactCache
|
|
1649
|
+
? await readArtifactCache(analysis.graphHash, target, adapter)
|
|
1650
|
+
: null;
|
|
1487
1651
|
const discoveredFiles =
|
|
1488
1652
|
await adapter.discoverPackagedLocalFiles(absolutePath);
|
|
1489
1653
|
if (cachedArtifact) {
|
|
@@ -1502,7 +1666,13 @@ export async function bundlePlayFile(
|
|
|
1502
1666
|
|
|
1503
1667
|
return {
|
|
1504
1668
|
success: true,
|
|
1505
|
-
artifact: {
|
|
1669
|
+
artifact: {
|
|
1670
|
+
...cachedArtifact,
|
|
1671
|
+
entryFile: absolutePath,
|
|
1672
|
+
sourceHash: analysis.sourceHash,
|
|
1673
|
+
importPolicy: analysis.importPolicy,
|
|
1674
|
+
cacheHit: true,
|
|
1675
|
+
},
|
|
1506
1676
|
sourceCode: analysis.sourceCode,
|
|
1507
1677
|
sourceFiles: analysis.sourceFiles,
|
|
1508
1678
|
filePath: absolutePath,
|
|
@@ -1568,7 +1738,9 @@ export async function bundlePlayFile(
|
|
|
1568
1738
|
cacheHit: false,
|
|
1569
1739
|
};
|
|
1570
1740
|
|
|
1571
|
-
|
|
1741
|
+
if (canUseArtifactCache) {
|
|
1742
|
+
await writeArtifactCache(artifact, adapter);
|
|
1743
|
+
}
|
|
1572
1744
|
|
|
1573
1745
|
return {
|
|
1574
1746
|
success: true,
|
|
@@ -911,9 +911,7 @@ export function compileSheetContract(pipeline: PlayStaticPipeline): {
|
|
|
911
911
|
});
|
|
912
912
|
};
|
|
913
913
|
|
|
914
|
-
const inputFields = pipeline.inputFields?.length
|
|
915
|
-
? pipeline.inputFields
|
|
916
|
-
: [tableNamespace];
|
|
914
|
+
const inputFields = pipeline.inputFields?.length ? pipeline.inputFields : [];
|
|
917
915
|
const rowKeyFieldSet = new Set(pipeline.rowKeyFields ?? []);
|
|
918
916
|
for (const inputField of inputFields) {
|
|
919
917
|
addColumn({
|
|
@@ -0,0 +1,238 @@
|
|
|
1
|
+
const BLOCKED_HOSTNAMES = new Set(['localhost']);
|
|
2
|
+
|
|
3
|
+
const BLOCKED_IPV4_CIDRS = [
|
|
4
|
+
['0.0.0.0', 8],
|
|
5
|
+
['10.0.0.0', 8],
|
|
6
|
+
['100.64.0.0', 10],
|
|
7
|
+
['127.0.0.0', 8],
|
|
8
|
+
['169.254.0.0', 16],
|
|
9
|
+
['172.16.0.0', 12],
|
|
10
|
+
['192.0.0.0', 24],
|
|
11
|
+
['192.0.2.0', 24],
|
|
12
|
+
['192.168.0.0', 16],
|
|
13
|
+
['198.18.0.0', 15],
|
|
14
|
+
['198.51.100.0', 24],
|
|
15
|
+
['203.0.113.0', 24],
|
|
16
|
+
['224.0.0.0', 4],
|
|
17
|
+
['240.0.0.0', 4],
|
|
18
|
+
] as const;
|
|
19
|
+
|
|
20
|
+
const BLOCKED_IPV6_CIDRS = [
|
|
21
|
+
['64:ff9b::', 96],
|
|
22
|
+
['64:ff9b:1::', 48],
|
|
23
|
+
['100::', 64],
|
|
24
|
+
['2001::', 23],
|
|
25
|
+
['2001:db8::', 32],
|
|
26
|
+
['2002::', 16],
|
|
27
|
+
['fc00::', 7],
|
|
28
|
+
['fe80::', 10],
|
|
29
|
+
['fec0::', 10],
|
|
30
|
+
['ff00::', 8],
|
|
31
|
+
] as const;
|
|
32
|
+
|
|
33
|
+
export class UnsafeOutboundUrlError extends Error {
|
|
34
|
+
constructor(message: string) {
|
|
35
|
+
super(message);
|
|
36
|
+
this.name = 'UnsafeOutboundUrlError';
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
function ipv4ToInt(ip: string): number | null {
|
|
41
|
+
const parts = ip.split('.');
|
|
42
|
+
if (parts.length !== 4) return null;
|
|
43
|
+
let value = 0;
|
|
44
|
+
for (const part of parts) {
|
|
45
|
+
if (!/^\d{1,3}$/.test(part)) return null;
|
|
46
|
+
const numeric = Number.parseInt(part, 10);
|
|
47
|
+
if (numeric < 0 || numeric > 255) return null;
|
|
48
|
+
value = (value << 8) + numeric;
|
|
49
|
+
}
|
|
50
|
+
return value >>> 0;
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
function isBlockedIpv4(ip: string): boolean {
|
|
54
|
+
const numericIp = ipv4ToInt(ip);
|
|
55
|
+
if (numericIp === null) return false;
|
|
56
|
+
return BLOCKED_IPV4_CIDRS.some(([network, prefix]) => {
|
|
57
|
+
const numericNetwork = ipv4ToInt(network);
|
|
58
|
+
if (numericNetwork === null) return false;
|
|
59
|
+
const mask = prefix >= 32 ? 0xffffffff : (~0 << (32 - prefix)) >>> 0;
|
|
60
|
+
return (numericIp & mask) === (numericNetwork & mask);
|
|
61
|
+
});
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
function ipv4IntToAddress(value: number): string {
|
|
65
|
+
return [
|
|
66
|
+
(value >>> 24) & 0xff,
|
|
67
|
+
(value >>> 16) & 0xff,
|
|
68
|
+
(value >>> 8) & 0xff,
|
|
69
|
+
value & 0xff,
|
|
70
|
+
].join('.');
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
function parseIpv6Part(part: string): number[] | null {
|
|
74
|
+
if (!part) return [];
|
|
75
|
+
const groups: number[] = [];
|
|
76
|
+
for (const segment of part.split(':')) {
|
|
77
|
+
if (!segment) return null;
|
|
78
|
+
if (segment.includes('.')) {
|
|
79
|
+
const ipv4 = ipv4ToInt(segment);
|
|
80
|
+
if (ipv4 === null) return null;
|
|
81
|
+
groups.push((ipv4 >>> 16) & 0xffff, ipv4 & 0xffff);
|
|
82
|
+
continue;
|
|
83
|
+
}
|
|
84
|
+
if (!/^[0-9a-f]{1,4}$/i.test(segment)) return null;
|
|
85
|
+
groups.push(Number.parseInt(segment, 16));
|
|
86
|
+
}
|
|
87
|
+
return groups;
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
function expandIpv6(ip: string): number[] | null {
|
|
91
|
+
const normalized = normalizeUrlHostname(ip).toLowerCase();
|
|
92
|
+
const pieces = normalized.split('::');
|
|
93
|
+
if (pieces.length > 2) return null;
|
|
94
|
+
|
|
95
|
+
const left = parseIpv6Part(pieces[0] ?? '');
|
|
96
|
+
const right = parseIpv6Part(pieces[1] ?? '');
|
|
97
|
+
if (!left || !right) return null;
|
|
98
|
+
|
|
99
|
+
if (pieces.length === 1) {
|
|
100
|
+
return left.length === 8 ? left : null;
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
const zeroCount = 8 - left.length - right.length;
|
|
104
|
+
if (zeroCount < 1) return null;
|
|
105
|
+
return [...left, ...Array.from({ length: zeroCount }, () => 0), ...right];
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
function maybeIpv4MappedIpv6(ip: string): string | null {
|
|
109
|
+
const groups = expandIpv6(ip);
|
|
110
|
+
if (!groups) return null;
|
|
111
|
+
|
|
112
|
+
const ipv4Value = ((groups[6] ?? 0) << 16) + (groups[7] ?? 0);
|
|
113
|
+
const isIpv4Mapped =
|
|
114
|
+
groups.slice(0, 5).every((group) => group === 0) && groups[5] === 0xffff;
|
|
115
|
+
if (!isIpv4Mapped) return null;
|
|
116
|
+
|
|
117
|
+
return ipv4IntToAddress(ipv4Value >>> 0);
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
function isIpv4CompatibleIpv6(ip: string): boolean {
|
|
121
|
+
const groups = expandIpv6(ip);
|
|
122
|
+
if (!groups) return false;
|
|
123
|
+
return groups.slice(0, 6).every((group) => group === 0);
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
function ipv6InCidr(
|
|
127
|
+
groups: number[],
|
|
128
|
+
network: number[],
|
|
129
|
+
prefix: number,
|
|
130
|
+
): boolean {
|
|
131
|
+
let remainingBits = prefix;
|
|
132
|
+
for (let index = 0; index < groups.length; index += 1) {
|
|
133
|
+
if (remainingBits <= 0) return true;
|
|
134
|
+
const bits = Math.min(16, remainingBits);
|
|
135
|
+
const mask = bits === 16 ? 0xffff : (0xffff << (16 - bits)) & 0xffff;
|
|
136
|
+
if ((groups[index]! & mask) !== (network[index]! & mask)) {
|
|
137
|
+
return false;
|
|
138
|
+
}
|
|
139
|
+
remainingBits -= bits;
|
|
140
|
+
}
|
|
141
|
+
return true;
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
function isBlockedIpv6(ip: string): boolean {
|
|
145
|
+
const normalized = normalizeUrlHostname(ip).toLowerCase();
|
|
146
|
+
if (isIpv4CompatibleIpv6(normalized)) return true;
|
|
147
|
+
|
|
148
|
+
const mappedIpv4 = maybeIpv4MappedIpv6(normalized);
|
|
149
|
+
if (mappedIpv4) return isBlockedIpv4(mappedIpv4);
|
|
150
|
+
|
|
151
|
+
const groups = expandIpv6(normalized);
|
|
152
|
+
if (!groups) return false;
|
|
153
|
+
|
|
154
|
+
const publicUnicast = expandIpv6('2000::');
|
|
155
|
+
if (!publicUnicast || !ipv6InCidr(groups, publicUnicast, 3)) {
|
|
156
|
+
return true;
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
return BLOCKED_IPV6_CIDRS.some(([network, prefix]) => {
|
|
160
|
+
const networkGroups = expandIpv6(network);
|
|
161
|
+
return networkGroups !== null && ipv6InCidr(groups, networkGroups, prefix);
|
|
162
|
+
});
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
export function normalizeUrlHostname(hostname: string): string {
|
|
166
|
+
return hostname
|
|
167
|
+
.trim()
|
|
168
|
+
.toLowerCase()
|
|
169
|
+
.replace(/^\[(.*)\]$/, '$1')
|
|
170
|
+
.replace(/\.$/, '');
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
export function isBlockedIpAddress(ip: string): boolean {
|
|
174
|
+
const normalized = normalizeUrlHostname(ip);
|
|
175
|
+
if (normalized.includes(':')) {
|
|
176
|
+
return isBlockedIpv6(normalized);
|
|
177
|
+
}
|
|
178
|
+
return isBlockedIpv4(normalized);
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
export function isIpAddressLiteral(hostname: string): boolean {
|
|
182
|
+
const normalized = normalizeUrlHostname(hostname);
|
|
183
|
+
return ipv4ToInt(normalized) !== null || expandIpv6(normalized) !== null;
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
export function isBlockedOutboundHostname(hostname: string): boolean {
|
|
187
|
+
const normalized = normalizeUrlHostname(hostname);
|
|
188
|
+
return (
|
|
189
|
+
!normalized ||
|
|
190
|
+
BLOCKED_HOSTNAMES.has(normalized) ||
|
|
191
|
+
normalized.endsWith('.localhost') ||
|
|
192
|
+
normalized.endsWith('.local') ||
|
|
193
|
+
isBlockedIpAddress(normalized)
|
|
194
|
+
);
|
|
195
|
+
}
|
|
196
|
+
|
|
197
|
+
export function assertPublicHttpUrl(rawUrl: string | URL): URL {
|
|
198
|
+
let url: URL;
|
|
199
|
+
try {
|
|
200
|
+
url = rawUrl instanceof URL ? new URL(rawUrl.toString()) : new URL(rawUrl);
|
|
201
|
+
} catch {
|
|
202
|
+
throw new UnsafeOutboundUrlError('url must be a valid absolute URL.');
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
if (url.protocol !== 'http:' && url.protocol !== 'https:') {
|
|
206
|
+
throw new UnsafeOutboundUrlError('Only http and https URLs are allowed.');
|
|
207
|
+
}
|
|
208
|
+
if (url.username || url.password) {
|
|
209
|
+
throw new UnsafeOutboundUrlError(
|
|
210
|
+
'Credentials in URLs are not allowed. Use headers instead.',
|
|
211
|
+
);
|
|
212
|
+
}
|
|
213
|
+
|
|
214
|
+
const hostname = normalizeUrlHostname(url.hostname);
|
|
215
|
+
if (!hostname) {
|
|
216
|
+
throw new UnsafeOutboundUrlError('URL hostname is required.');
|
|
217
|
+
}
|
|
218
|
+
if (isBlockedOutboundHostname(hostname)) {
|
|
219
|
+
throw new UnsafeOutboundUrlError(
|
|
220
|
+
`Target host "${hostname}" is not allowed.`,
|
|
221
|
+
);
|
|
222
|
+
}
|
|
223
|
+
|
|
224
|
+
return url;
|
|
225
|
+
}
|
|
226
|
+
|
|
227
|
+
export function resolveRedirectUrl(location: string, currentUrl: URL): URL {
|
|
228
|
+
try {
|
|
229
|
+
return assertPublicHttpUrl(new URL(location, currentUrl));
|
|
230
|
+
} catch (error) {
|
|
231
|
+
if (error instanceof UnsafeOutboundUrlError) throw error;
|
|
232
|
+
throw new UnsafeOutboundUrlError('redirect location must be a valid URL.');
|
|
233
|
+
}
|
|
234
|
+
}
|
|
235
|
+
|
|
236
|
+
export function isRedirectStatus(status: number): boolean {
|
|
237
|
+
return [301, 302, 303, 307, 308].includes(status);
|
|
238
|
+
}
|
|
@@ -0,0 +1,118 @@
|
|
|
1
|
+
import {
|
|
2
|
+
assertPublicHttpUrl,
|
|
3
|
+
isRedirectStatus,
|
|
4
|
+
resolveRedirectUrl,
|
|
5
|
+
} from './outbound-url-policy';
|
|
6
|
+
|
|
7
|
+
export type SafeFetchImplementation = (
|
|
8
|
+
input: string | URL,
|
|
9
|
+
init?: RequestInit,
|
|
10
|
+
) => Promise<Response>;
|
|
11
|
+
|
|
12
|
+
export type SafeFetchOptions = {
|
|
13
|
+
fetchImpl?: SafeFetchImplementation;
|
|
14
|
+
maxRedirects?: number;
|
|
15
|
+
sensitiveHeaders?: Iterable<string>;
|
|
16
|
+
};
|
|
17
|
+
|
|
18
|
+
function removeHeader(headers: Headers, name: string) {
|
|
19
|
+
if (headers.has(name)) headers.delete(name);
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
function requestInitForRedirect(
|
|
23
|
+
init: RequestInit,
|
|
24
|
+
from: URL,
|
|
25
|
+
to: URL,
|
|
26
|
+
status: number,
|
|
27
|
+
sensitiveHeaders: Iterable<string>,
|
|
28
|
+
): RequestInit {
|
|
29
|
+
const headers = new Headers(init.headers);
|
|
30
|
+
let method = String(init.method ?? 'GET').toUpperCase();
|
|
31
|
+
let body = init.body;
|
|
32
|
+
|
|
33
|
+
if (
|
|
34
|
+
status === 303 ||
|
|
35
|
+
((status === 301 || status === 302) && method === 'POST')
|
|
36
|
+
) {
|
|
37
|
+
method = 'GET';
|
|
38
|
+
body = undefined;
|
|
39
|
+
removeHeader(headers, 'content-length');
|
|
40
|
+
removeHeader(headers, 'content-type');
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
if (from.origin !== to.origin) {
|
|
44
|
+
removeHeader(headers, 'authorization');
|
|
45
|
+
removeHeader(headers, 'cookie');
|
|
46
|
+
removeHeader(headers, 'proxy-authorization');
|
|
47
|
+
for (const header of sensitiveHeaders) {
|
|
48
|
+
removeHeader(headers, header);
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
return {
|
|
53
|
+
...init,
|
|
54
|
+
method,
|
|
55
|
+
body,
|
|
56
|
+
headers,
|
|
57
|
+
redirect: 'manual',
|
|
58
|
+
};
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
export async function safePublicFetch(
|
|
62
|
+
input: string | URL,
|
|
63
|
+
init: RequestInit = {},
|
|
64
|
+
options: SafeFetchOptions = {},
|
|
65
|
+
): Promise<Response> {
|
|
66
|
+
const redirectMode = init.redirect ?? 'follow';
|
|
67
|
+
const maxRedirects = options.maxRedirects ?? 10;
|
|
68
|
+
const fetchImpl = options.fetchImpl ?? fetch;
|
|
69
|
+
const sensitiveHeaders = options.sensitiveHeaders ?? [];
|
|
70
|
+
let currentUrl = assertPublicHttpUrl(input);
|
|
71
|
+
let currentInit: RequestInit = {
|
|
72
|
+
...init,
|
|
73
|
+
redirect: 'manual',
|
|
74
|
+
};
|
|
75
|
+
|
|
76
|
+
for (
|
|
77
|
+
let redirectCount = 0;
|
|
78
|
+
redirectCount <= maxRedirects;
|
|
79
|
+
redirectCount += 1
|
|
80
|
+
) {
|
|
81
|
+
const response = await fetchImpl(currentUrl, currentInit);
|
|
82
|
+
if (!isRedirectStatus(response.status)) {
|
|
83
|
+
return response;
|
|
84
|
+
}
|
|
85
|
+
if (redirectMode === 'error') {
|
|
86
|
+
throw new Error(
|
|
87
|
+
`Redirect blocked while fetching ${currentUrl.toString()}.`,
|
|
88
|
+
);
|
|
89
|
+
}
|
|
90
|
+
if (redirectMode !== 'follow') {
|
|
91
|
+
return response;
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
const location = response.headers.get('location');
|
|
95
|
+
if (!location) {
|
|
96
|
+
return response;
|
|
97
|
+
}
|
|
98
|
+
if (redirectCount === maxRedirects) {
|
|
99
|
+
throw new Error(
|
|
100
|
+
`Too many redirects while fetching ${currentUrl.toString()}.`,
|
|
101
|
+
);
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
const nextUrl = resolveRedirectUrl(location, currentUrl);
|
|
105
|
+
currentInit = requestInitForRedirect(
|
|
106
|
+
currentInit,
|
|
107
|
+
currentUrl,
|
|
108
|
+
nextUrl,
|
|
109
|
+
response.status,
|
|
110
|
+
sensitiveHeaders,
|
|
111
|
+
);
|
|
112
|
+
currentUrl = nextUrl;
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
throw new Error(
|
|
116
|
+
`Too many redirects while fetching ${currentUrl.toString()}.`,
|
|
117
|
+
);
|
|
118
|
+
}
|