deepline 0.1.109 → 0.1.110
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli/index.js +2226 -1271
- package/dist/cli/index.mjs +2303 -1355
- package/dist/index.d.mts +21 -14
- package/dist/index.d.ts +21 -14
- package/dist/index.js +97 -23
- package/dist/index.mjs +97 -23
- package/dist/repo/apps/play-runner-workers/src/coordinator-entry.ts +42 -20
- package/dist/repo/apps/play-runner-workers/src/entry.ts +135 -45
- package/dist/repo/apps/play-runner-workers/src/runtime/receipts.ts +18 -27
- package/dist/repo/apps/play-runner-workers/src/workflow-instance-create.ts +44 -0
- package/dist/repo/apps/play-runner-workers/src/workflow-retry.ts +7 -11
- package/dist/repo/sdk/src/client.ts +35 -12
- package/dist/repo/sdk/src/errors.ts +2 -2
- package/dist/repo/sdk/src/http.ts +87 -7
- package/dist/repo/sdk/src/play.ts +1 -1
- package/dist/repo/sdk/src/plays/bundle-play-file.ts +5 -1
- package/dist/repo/sdk/src/release.ts +13 -10
- package/dist/repo/sdk/src/tool-output.ts +2 -2
- package/dist/repo/sdk/src/types.ts +9 -6
- package/dist/repo/shared_libs/play-runtime/fullenrich-batching.ts +229 -0
- package/dist/repo/shared_libs/play-runtime/governor/policy.ts +1 -1
- package/dist/repo/shared_libs/play-runtime/play-runtime-batching-registry.ts +20 -0
- package/dist/repo/shared_libs/play-runtime/run-failure.ts +20 -12
- package/dist/repo/shared_libs/play-runtime/run-ledger.ts +107 -56
- package/dist/repo/shared_libs/play-runtime/scheduler-backend.ts +4 -2
- package/dist/repo/shared_libs/play-runtime/secret-redaction.ts +15 -0
- package/dist/repo/shared_libs/play-runtime/work-receipts.ts +1 -0
- package/dist/repo/shared_libs/plays/bundling/index.ts +69 -11
- package/dist/repo/shared_libs/plays/static-pipeline.ts +1 -3
- package/dist/repo/shared_libs/security/outbound-url-policy.ts +238 -0
- package/dist/repo/shared_libs/security/safe-fetch.ts +118 -0
- package/dist/viewer/viewer.css +617 -0
- package/dist/viewer/viewer.js +1496 -0
- package/package.json +5 -1
|
@@ -549,22 +549,17 @@ function appendLogLines(
|
|
|
549
549
|
|
|
550
550
|
/**
|
|
551
551
|
* Terminal-status precedence. Re-delivery of the same terminal status is a
|
|
552
|
-
* benign no-op handled by the regular reduction.
|
|
553
|
-
*
|
|
554
|
-
*
|
|
555
|
-
*
|
|
556
|
-
*
|
|
557
|
-
*
|
|
558
|
-
* (compute billing finalize), so a billing business denial there — e.g. the
|
|
559
|
-
* per-run credit cap (maxCreditsPerRun) — arrives as a later run.failed
|
|
560
|
-
* that MUST fail the run. Blanket first-terminal-wins silently completed
|
|
561
|
-
* capped runs (regression pinned by
|
|
562
|
-
* tests/v2-plays/plays/44-compute-billing-cap.play.ts). A cancelled run
|
|
563
|
-
* stays cancelled, and a failed run can never be flipped to completed.
|
|
552
|
+
* benign no-op handled by the regular reduction. Conflicting terminal events
|
|
553
|
+
* use newest-terminal-wins by event time: older events are ignored and logged,
|
|
554
|
+
* newer events reconcile the snapshot. This is the idempotency model callers
|
|
555
|
+
* expect when duplicate/retried work completes out of order. It also preserves
|
|
556
|
+
* post-completion billing cap demotion because that denial arrives as a newer
|
|
557
|
+
* run.failed event.
|
|
564
558
|
*/
|
|
565
559
|
function conflictingTerminalSnapshot(
|
|
566
560
|
base: PlayRunLedgerSnapshot,
|
|
567
561
|
eventType: keyof typeof TERMINAL_STATUS_BY_EVENT_TYPE,
|
|
562
|
+
occurredAt: number,
|
|
568
563
|
): PlayRunLedgerSnapshot | null {
|
|
569
564
|
if (!isTerminalPlayRunLedgerStatus(base.status)) {
|
|
570
565
|
return null;
|
|
@@ -572,14 +567,16 @@ function conflictingTerminalSnapshot(
|
|
|
572
567
|
if (TERMINAL_STATUS_BY_EVENT_TYPE[eventType] === base.status) {
|
|
573
568
|
return null;
|
|
574
569
|
}
|
|
575
|
-
|
|
576
|
-
|
|
577
|
-
//
|
|
570
|
+
const terminalAt = base.finishedAt ?? base.updatedAt ?? 0;
|
|
571
|
+
if (occurredAt > terminalAt) {
|
|
572
|
+
// Newer terminal evidence reconciles the run. This covers replay/receipt
|
|
573
|
+
// races where an earlier attempt failed but a later attempt recovered and
|
|
574
|
+
// completed with the durable result.
|
|
578
575
|
return null;
|
|
579
576
|
}
|
|
580
577
|
return withTiming(
|
|
581
578
|
appendLogLines(base, [
|
|
582
|
-
`[ledger] conflicting terminal event ${eventType} ignored; status already ${base.status}`,
|
|
579
|
+
`[ledger] stale conflicting terminal event ${eventType} ignored; status already ${base.status}`,
|
|
583
580
|
]),
|
|
584
581
|
);
|
|
585
582
|
}
|
|
@@ -617,6 +614,23 @@ function settleRunningStepsOnTerminal(
|
|
|
617
614
|
return changed ? { ...snapshot, stepsById } : snapshot;
|
|
618
615
|
}
|
|
619
616
|
|
|
617
|
+
function terminalFinishedAt(
|
|
618
|
+
base: PlayRunLedgerSnapshot,
|
|
619
|
+
eventType: keyof typeof TERMINAL_STATUS_BY_EVENT_TYPE,
|
|
620
|
+
occurredAt: number,
|
|
621
|
+
): number {
|
|
622
|
+
const nextStatus = TERMINAL_STATUS_BY_EVENT_TYPE[eventType];
|
|
623
|
+
const terminalAt = base.finishedAt ?? base.updatedAt ?? 0;
|
|
624
|
+
if (
|
|
625
|
+
isTerminalPlayRunLedgerStatus(base.status) &&
|
|
626
|
+
nextStatus !== base.status &&
|
|
627
|
+
occurredAt > terminalAt
|
|
628
|
+
) {
|
|
629
|
+
return occurredAt;
|
|
630
|
+
}
|
|
631
|
+
return base.finishedAt ?? occurredAt;
|
|
632
|
+
}
|
|
633
|
+
|
|
620
634
|
export function reducePlayRunLedgerEvent(
|
|
621
635
|
snapshot: PlayRunLedgerSnapshot,
|
|
622
636
|
event: PlayRunLedgerEvent,
|
|
@@ -654,52 +668,46 @@ export function reducePlayRunLedgerEvent(
|
|
|
654
668
|
});
|
|
655
669
|
case 'run.completed':
|
|
656
670
|
return (
|
|
657
|
-
conflictingTerminalSnapshot(base, event.type) ??
|
|
658
|
-
withTiming(
|
|
659
|
-
|
|
660
|
-
|
|
661
|
-
|
|
662
|
-
|
|
663
|
-
|
|
664
|
-
|
|
665
|
-
|
|
666
|
-
|
|
667
|
-
|
|
668
|
-
|
|
669
|
-
},
|
|
670
|
-
)
|
|
671
|
+
conflictingTerminalSnapshot(base, event.type, occurredAt) ??
|
|
672
|
+
withTiming({
|
|
673
|
+
...settleRunningStepsOnTerminal(base, 'completed', occurredAt),
|
|
674
|
+
status: 'completed',
|
|
675
|
+
error: null,
|
|
676
|
+
startedAt: base.startedAt ?? occurredAt,
|
|
677
|
+
finishedAt: terminalFinishedAt(base, event.type, occurredAt),
|
|
678
|
+
activeStepId: null,
|
|
679
|
+
activeArtifactTableNamespace: null,
|
|
680
|
+
result: event.result ?? base.result,
|
|
681
|
+
resultSummary: event.resultSummary ?? base.resultSummary,
|
|
682
|
+
})
|
|
671
683
|
);
|
|
672
684
|
case 'run.failed':
|
|
673
685
|
return (
|
|
674
|
-
conflictingTerminalSnapshot(base, event.type) ??
|
|
675
|
-
withTiming(
|
|
676
|
-
|
|
677
|
-
|
|
678
|
-
|
|
679
|
-
|
|
680
|
-
|
|
681
|
-
|
|
682
|
-
|
|
683
|
-
|
|
684
|
-
|
|
685
|
-
},
|
|
686
|
-
)
|
|
686
|
+
conflictingTerminalSnapshot(base, event.type, occurredAt) ??
|
|
687
|
+
withTiming({
|
|
688
|
+
...settleRunningStepsOnTerminal(base, 'failed', occurredAt),
|
|
689
|
+
status: 'failed',
|
|
690
|
+
error: event.error ?? base.error ?? null,
|
|
691
|
+
startedAt: base.startedAt ?? occurredAt,
|
|
692
|
+
finishedAt: terminalFinishedAt(base, event.type, occurredAt),
|
|
693
|
+
activeStepId: null,
|
|
694
|
+
activeArtifactTableNamespace: null,
|
|
695
|
+
result: event.result ?? base.result,
|
|
696
|
+
})
|
|
687
697
|
);
|
|
688
698
|
case 'run.cancelled':
|
|
689
699
|
return (
|
|
690
|
-
conflictingTerminalSnapshot(base, event.type) ??
|
|
691
|
-
withTiming(
|
|
692
|
-
|
|
693
|
-
|
|
694
|
-
|
|
695
|
-
|
|
696
|
-
|
|
697
|
-
|
|
698
|
-
|
|
699
|
-
|
|
700
|
-
|
|
701
|
-
},
|
|
702
|
-
)
|
|
700
|
+
conflictingTerminalSnapshot(base, event.type, occurredAt) ??
|
|
701
|
+
withTiming({
|
|
702
|
+
...settleRunningStepsOnTerminal(base, 'failed', occurredAt),
|
|
703
|
+
status: 'cancelled',
|
|
704
|
+
error: event.error ?? base.error ?? null,
|
|
705
|
+
startedAt: base.startedAt ?? occurredAt,
|
|
706
|
+
finishedAt: terminalFinishedAt(base, event.type, occurredAt),
|
|
707
|
+
activeStepId: null,
|
|
708
|
+
activeArtifactTableNamespace: null,
|
|
709
|
+
result: event.result ?? base.result,
|
|
710
|
+
})
|
|
703
711
|
);
|
|
704
712
|
case 'log.appended':
|
|
705
713
|
return withTiming(
|
|
@@ -927,6 +935,20 @@ export function slicePositionalLogLines(input: {
|
|
|
927
935
|
return { lines: [...lines], channelOffset: sendFromOffset };
|
|
928
936
|
}
|
|
929
937
|
|
|
938
|
+
function terminalLogReplayChannelOffset(input: {
|
|
939
|
+
liveLogTotalCount?: number;
|
|
940
|
+
liveLogsLength: number;
|
|
941
|
+
}): number | null {
|
|
942
|
+
if (
|
|
943
|
+
typeof input.liveLogTotalCount !== 'number' ||
|
|
944
|
+
!Number.isFinite(input.liveLogTotalCount) ||
|
|
945
|
+
input.liveLogTotalCount < input.liveLogsLength
|
|
946
|
+
) {
|
|
947
|
+
return null;
|
|
948
|
+
}
|
|
949
|
+
return input.liveLogTotalCount - input.liveLogsLength;
|
|
950
|
+
}
|
|
951
|
+
|
|
930
952
|
/**
|
|
931
953
|
* Forward producer log lines as one `log.appended` event.
|
|
932
954
|
*
|
|
@@ -968,6 +990,35 @@ export function buildPlayRunLedgerEventsFromLogLines(input: {
|
|
|
968
990
|
];
|
|
969
991
|
}
|
|
970
992
|
|
|
993
|
+
/**
|
|
994
|
+
* Re-send a terminal retained log tail through terminal transport recovery.
|
|
995
|
+
*
|
|
996
|
+
* Live worker flushes use positional channel offsets. Terminal replay is a
|
|
997
|
+
* different transport: it replays the runner's retained output after terminal
|
|
998
|
+
* state is known. When the producer knows the retained tail's offset, Convex
|
|
999
|
+
* uses that offset only to bound occurrence-count reconciliation to the same
|
|
1000
|
+
* tail window; it does not route the replay through the worker cursor.
|
|
1001
|
+
*/
|
|
1002
|
+
export function buildTerminalLogReplayEvents(input: {
|
|
1003
|
+
runId: string;
|
|
1004
|
+
lines: readonly string[];
|
|
1005
|
+
source?: PlayRunLedgerEventSource;
|
|
1006
|
+
occurredAt?: number;
|
|
1007
|
+
liveLogTotalCount?: number;
|
|
1008
|
+
}): PlayRunLedgerEvent[] {
|
|
1009
|
+
const channelOffset = terminalLogReplayChannelOffset({
|
|
1010
|
+
liveLogTotalCount: input.liveLogTotalCount,
|
|
1011
|
+
liveLogsLength: input.lines.length,
|
|
1012
|
+
});
|
|
1013
|
+
return buildPlayRunLedgerEventsFromLogLines({
|
|
1014
|
+
runId: input.runId,
|
|
1015
|
+
lines: input.lines,
|
|
1016
|
+
source: 'coordinator',
|
|
1017
|
+
occurredAt: input.occurredAt,
|
|
1018
|
+
channelOffset,
|
|
1019
|
+
});
|
|
1020
|
+
}
|
|
1021
|
+
|
|
971
1022
|
export function buildPlayRunLedgerEventsFromStatusPatch(input: {
|
|
972
1023
|
patch: PlayRunLedgerStatusPatch;
|
|
973
1024
|
previousSnapshot: PlayRunLedgerSnapshot;
|
|
@@ -4,8 +4,8 @@
|
|
|
4
4
|
* One of three pluggable axes (alongside runner-backends and dedup-backends).
|
|
5
5
|
* Selected per-run via PlayExecutionProfile.
|
|
6
6
|
*
|
|
7
|
-
*
|
|
8
|
-
*
|
|
7
|
+
* Cloudflare Workflows is the default production scheduler through the
|
|
8
|
+
* workers_edge profile. Hatchet is selected explicitly by the hatchet profile.
|
|
9
9
|
*
|
|
10
10
|
* Customer plays are unaffected — this is purely the orchestration layer.
|
|
11
11
|
*/
|
|
@@ -18,6 +18,7 @@ import type {
|
|
|
18
18
|
import type { ExecutionPlan } from './execution-plan';
|
|
19
19
|
import type { PlayRuntimeManifestMap } from '../plays/compiler-manifest';
|
|
20
20
|
import type { PreloadedRuntimeDbSession } from './db-session';
|
|
21
|
+
import type { PlayRunnerRuntimeTiming } from './protocol';
|
|
21
22
|
|
|
22
23
|
export const PLAY_SCHEDULER_BACKENDS = {
|
|
23
24
|
temporal: 'temporal',
|
|
@@ -195,6 +196,7 @@ export type PlaySchedulerResultEnvelope = {
|
|
|
195
196
|
finalCheckpoint?: PlayCheckpoint;
|
|
196
197
|
totalRows?: number;
|
|
197
198
|
durationMs?: number;
|
|
199
|
+
runtimeTiming?: PlayRunnerRuntimeTiming;
|
|
198
200
|
};
|
|
199
201
|
|
|
200
202
|
export interface PlaySchedulerBackend {
|
|
@@ -7,8 +7,12 @@ const PRIVATE_KEY_RE =
|
|
|
7
7
|
/-----BEGIN [A-Z0-9 ]*PRIVATE KEY-----[\s\S]*?-----END [A-Z0-9 ]*PRIVATE KEY-----/g;
|
|
8
8
|
const COMMON_SECRET_RE =
|
|
9
9
|
/\b(?:sk|pk|rk|pat|ghp|github_pat|xox[baprs]|key|token|secret|api[_-]?key)[A-Za-z0-9_./+=:-]{12,}\b/gi;
|
|
10
|
+
const URL_SECRET_VALUE_RE =
|
|
11
|
+
/\b(?:sk|pk|rk|pat|ghp|github_pat|xox)[A-Za-z0-9_./+=:-]{12,}\b/i;
|
|
10
12
|
const HIGH_ENTROPY_ASSIGNMENT_RE =
|
|
11
13
|
/\b(?:api[_-]?key|token|secret|password|authorization|access[_-]?token|refresh[_-]?token)\b\s*[:=]\s*["']?[^"',\s]{16,}["']?/gi;
|
|
14
|
+
const SENSITIVE_URL_PARAM_RE =
|
|
15
|
+
/[?#&](?:\w*(?:key|token)|secret|password|authorization|credential|signature|sig)(?:=|$)/i;
|
|
12
16
|
|
|
13
17
|
function escapeRegExp(value: string): string {
|
|
14
18
|
return value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
|
|
@@ -19,6 +23,17 @@ function isRecord(value: unknown): value is Record<string, unknown> {
|
|
|
19
23
|
}
|
|
20
24
|
|
|
21
25
|
export function redactSecretLikeString(value: string): string {
|
|
26
|
+
const trimmed = value.trim();
|
|
27
|
+
if (/^https?:\/\/\S+$/i.test(trimmed)) {
|
|
28
|
+
if (
|
|
29
|
+
/^https?:\/\/[^/?#@]+@/i.test(trimmed) ||
|
|
30
|
+
SENSITIVE_URL_PARAM_RE.test(trimmed)
|
|
31
|
+
) {
|
|
32
|
+
return SECRET_REDACTION_PLACEHOLDER;
|
|
33
|
+
}
|
|
34
|
+
if (!URL_SECRET_VALUE_RE.test(value)) return value;
|
|
35
|
+
}
|
|
36
|
+
|
|
22
37
|
return value
|
|
23
38
|
.replace(PRIVATE_KEY_RE, SECRET_REDACTION_PLACEHOLDER)
|
|
24
39
|
.replace(BEARER_TOKEN_RE, `Bearer ${SECRET_REDACTION_PLACEHOLDER}`)
|
|
@@ -96,6 +96,7 @@ export type PlayBundlingAdapter = {
|
|
|
96
96
|
sdkWorkersEntryFile: string;
|
|
97
97
|
workersHarnessEntryFile: string;
|
|
98
98
|
workersHarnessFilesDir: string;
|
|
99
|
+
workersRuntimeFingerprintDirs?: string[];
|
|
99
100
|
discoverPackagedLocalFiles(
|
|
100
101
|
filePath: string,
|
|
101
102
|
): Promise<PlayLocalFileDiscoveryResult>;
|
|
@@ -1063,19 +1064,69 @@ async function analyzeSourceGraph(
|
|
|
1063
1064
|
}
|
|
1064
1065
|
|
|
1065
1066
|
/**
|
|
1066
|
-
* Fingerprint of
|
|
1067
|
-
*
|
|
1068
|
-
*
|
|
1069
|
-
*
|
|
1070
|
-
*
|
|
1071
|
-
*
|
|
1072
|
-
* caching: that's deliberate — caching the fingerprint is exactly the
|
|
1073
|
-
* stale-state bug this exists to prevent.
|
|
1067
|
+
* Fingerprint of Worker runtime sources bundled into every esm_workers play
|
|
1068
|
+
* artifact. Harness, shared runtime, or provider-owned batching edits must
|
|
1069
|
+
* invalidate the bundle cache and force a fresh CF deploy. Computed fresh on
|
|
1070
|
+
* every bundle call so dev edits are picked up on the next `play run` without
|
|
1071
|
+
* restarting the dev server. No caching: that's deliberate — caching the
|
|
1072
|
+
* fingerprint is exactly the stale-state bug this exists to prevent.
|
|
1074
1073
|
*/
|
|
1075
1074
|
async function computeWorkersHarnessFingerprintWithAdapter(
|
|
1076
1075
|
adapter: PlayBundlingAdapter,
|
|
1077
1076
|
): Promise<string> {
|
|
1078
1077
|
const { readdir } = await import('node:fs/promises');
|
|
1078
|
+
const addFilePart = async (
|
|
1079
|
+
parts: Array<{ name: string; hash: string }>,
|
|
1080
|
+
rootDir: string,
|
|
1081
|
+
filePath: string,
|
|
1082
|
+
) => {
|
|
1083
|
+
const contents = await readFile(filePath, 'utf-8');
|
|
1084
|
+
parts.push({
|
|
1085
|
+
name: `${basename(rootDir)}:${filePath.slice(rootDir.length + 1)}`,
|
|
1086
|
+
hash: sha256(contents),
|
|
1087
|
+
});
|
|
1088
|
+
};
|
|
1089
|
+
const collectTopLevelTsFiles = async (
|
|
1090
|
+
rootDir: string,
|
|
1091
|
+
parts: Array<{ name: string; hash: string }>,
|
|
1092
|
+
) => {
|
|
1093
|
+
if (!(await fileExists(rootDir))) return;
|
|
1094
|
+
const entries = await readdir(rootDir, { withFileTypes: true });
|
|
1095
|
+
const tsFiles = entries
|
|
1096
|
+
.filter((entry) => entry.isFile() && /\.[cm]?ts$/.test(entry.name))
|
|
1097
|
+
.map((entry) => entry.name)
|
|
1098
|
+
.sort();
|
|
1099
|
+
for (const name of tsFiles) {
|
|
1100
|
+
await addFilePart(parts, rootDir, join(rootDir, name));
|
|
1101
|
+
}
|
|
1102
|
+
};
|
|
1103
|
+
const collectIntegrationBatchingFiles = async (
|
|
1104
|
+
rootDir: string,
|
|
1105
|
+
parts: Array<{ name: string; hash: string }>,
|
|
1106
|
+
) => {
|
|
1107
|
+
if (!(await fileExists(rootDir))) return;
|
|
1108
|
+
const entries = await readdir(rootDir, { withFileTypes: true });
|
|
1109
|
+
const filePaths: string[] = [];
|
|
1110
|
+
for (const entry of entries) {
|
|
1111
|
+
if (
|
|
1112
|
+
entry.isFile() &&
|
|
1113
|
+
(entry.name === 'play-runtime-batching-registry.ts' ||
|
|
1114
|
+
/^batching.*\.ts$/.test(entry.name))
|
|
1115
|
+
) {
|
|
1116
|
+
filePaths.push(join(rootDir, entry.name));
|
|
1117
|
+
}
|
|
1118
|
+
if (entry.isDirectory()) {
|
|
1119
|
+
const batchingFile = join(rootDir, entry.name, 'batching.ts');
|
|
1120
|
+
if (await fileExists(batchingFile)) {
|
|
1121
|
+
filePaths.push(batchingFile);
|
|
1122
|
+
}
|
|
1123
|
+
}
|
|
1124
|
+
}
|
|
1125
|
+
for (const filePath of filePaths.sort()) {
|
|
1126
|
+
await addFilePart(parts, rootDir, filePath);
|
|
1127
|
+
}
|
|
1128
|
+
};
|
|
1129
|
+
|
|
1079
1130
|
const entries = await readdir(adapter.workersHarnessFilesDir, {
|
|
1080
1131
|
withFileTypes: true,
|
|
1081
1132
|
});
|
|
@@ -1085,11 +1136,18 @@ async function computeWorkersHarnessFingerprintWithAdapter(
|
|
|
1085
1136
|
.sort();
|
|
1086
1137
|
const parts: Array<{ name: string; hash: string }> = [];
|
|
1087
1138
|
for (const name of tsFiles) {
|
|
1088
|
-
|
|
1139
|
+
await addFilePart(
|
|
1140
|
+
parts,
|
|
1141
|
+
adapter.workersHarnessFilesDir,
|
|
1089
1142
|
join(adapter.workersHarnessFilesDir, name),
|
|
1090
|
-
'utf-8',
|
|
1091
1143
|
);
|
|
1092
|
-
|
|
1144
|
+
}
|
|
1145
|
+
for (const dir of adapter.workersRuntimeFingerprintDirs ?? []) {
|
|
1146
|
+
if (basename(dir) === 'integrations') {
|
|
1147
|
+
await collectIntegrationBatchingFiles(dir, parts);
|
|
1148
|
+
} else {
|
|
1149
|
+
await collectTopLevelTsFiles(dir, parts);
|
|
1150
|
+
}
|
|
1093
1151
|
}
|
|
1094
1152
|
return sha256(JSON.stringify(parts));
|
|
1095
1153
|
}
|
|
@@ -911,9 +911,7 @@ export function compileSheetContract(pipeline: PlayStaticPipeline): {
|
|
|
911
911
|
});
|
|
912
912
|
};
|
|
913
913
|
|
|
914
|
-
const inputFields = pipeline.inputFields?.length
|
|
915
|
-
? pipeline.inputFields
|
|
916
|
-
: [tableNamespace];
|
|
914
|
+
const inputFields = pipeline.inputFields?.length ? pipeline.inputFields : [];
|
|
917
915
|
const rowKeyFieldSet = new Set(pipeline.rowKeyFields ?? []);
|
|
918
916
|
for (const inputField of inputFields) {
|
|
919
917
|
addColumn({
|
|
@@ -0,0 +1,238 @@
|
|
|
1
|
+
const BLOCKED_HOSTNAMES = new Set(['localhost']);
|
|
2
|
+
|
|
3
|
+
const BLOCKED_IPV4_CIDRS = [
|
|
4
|
+
['0.0.0.0', 8],
|
|
5
|
+
['10.0.0.0', 8],
|
|
6
|
+
['100.64.0.0', 10],
|
|
7
|
+
['127.0.0.0', 8],
|
|
8
|
+
['169.254.0.0', 16],
|
|
9
|
+
['172.16.0.0', 12],
|
|
10
|
+
['192.0.0.0', 24],
|
|
11
|
+
['192.0.2.0', 24],
|
|
12
|
+
['192.168.0.0', 16],
|
|
13
|
+
['198.18.0.0', 15],
|
|
14
|
+
['198.51.100.0', 24],
|
|
15
|
+
['203.0.113.0', 24],
|
|
16
|
+
['224.0.0.0', 4],
|
|
17
|
+
['240.0.0.0', 4],
|
|
18
|
+
] as const;
|
|
19
|
+
|
|
20
|
+
const BLOCKED_IPV6_CIDRS = [
|
|
21
|
+
['64:ff9b::', 96],
|
|
22
|
+
['64:ff9b:1::', 48],
|
|
23
|
+
['100::', 64],
|
|
24
|
+
['2001::', 23],
|
|
25
|
+
['2001:db8::', 32],
|
|
26
|
+
['2002::', 16],
|
|
27
|
+
['fc00::', 7],
|
|
28
|
+
['fe80::', 10],
|
|
29
|
+
['fec0::', 10],
|
|
30
|
+
['ff00::', 8],
|
|
31
|
+
] as const;
|
|
32
|
+
|
|
33
|
+
export class UnsafeOutboundUrlError extends Error {
|
|
34
|
+
constructor(message: string) {
|
|
35
|
+
super(message);
|
|
36
|
+
this.name = 'UnsafeOutboundUrlError';
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
function ipv4ToInt(ip: string): number | null {
|
|
41
|
+
const parts = ip.split('.');
|
|
42
|
+
if (parts.length !== 4) return null;
|
|
43
|
+
let value = 0;
|
|
44
|
+
for (const part of parts) {
|
|
45
|
+
if (!/^\d{1,3}$/.test(part)) return null;
|
|
46
|
+
const numeric = Number.parseInt(part, 10);
|
|
47
|
+
if (numeric < 0 || numeric > 255) return null;
|
|
48
|
+
value = (value << 8) + numeric;
|
|
49
|
+
}
|
|
50
|
+
return value >>> 0;
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
function isBlockedIpv4(ip: string): boolean {
|
|
54
|
+
const numericIp = ipv4ToInt(ip);
|
|
55
|
+
if (numericIp === null) return false;
|
|
56
|
+
return BLOCKED_IPV4_CIDRS.some(([network, prefix]) => {
|
|
57
|
+
const numericNetwork = ipv4ToInt(network);
|
|
58
|
+
if (numericNetwork === null) return false;
|
|
59
|
+
const mask = prefix >= 32 ? 0xffffffff : (~0 << (32 - prefix)) >>> 0;
|
|
60
|
+
return (numericIp & mask) === (numericNetwork & mask);
|
|
61
|
+
});
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
function ipv4IntToAddress(value: number): string {
|
|
65
|
+
return [
|
|
66
|
+
(value >>> 24) & 0xff,
|
|
67
|
+
(value >>> 16) & 0xff,
|
|
68
|
+
(value >>> 8) & 0xff,
|
|
69
|
+
value & 0xff,
|
|
70
|
+
].join('.');
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
function parseIpv6Part(part: string): number[] | null {
|
|
74
|
+
if (!part) return [];
|
|
75
|
+
const groups: number[] = [];
|
|
76
|
+
for (const segment of part.split(':')) {
|
|
77
|
+
if (!segment) return null;
|
|
78
|
+
if (segment.includes('.')) {
|
|
79
|
+
const ipv4 = ipv4ToInt(segment);
|
|
80
|
+
if (ipv4 === null) return null;
|
|
81
|
+
groups.push((ipv4 >>> 16) & 0xffff, ipv4 & 0xffff);
|
|
82
|
+
continue;
|
|
83
|
+
}
|
|
84
|
+
if (!/^[0-9a-f]{1,4}$/i.test(segment)) return null;
|
|
85
|
+
groups.push(Number.parseInt(segment, 16));
|
|
86
|
+
}
|
|
87
|
+
return groups;
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
function expandIpv6(ip: string): number[] | null {
|
|
91
|
+
const normalized = normalizeUrlHostname(ip).toLowerCase();
|
|
92
|
+
const pieces = normalized.split('::');
|
|
93
|
+
if (pieces.length > 2) return null;
|
|
94
|
+
|
|
95
|
+
const left = parseIpv6Part(pieces[0] ?? '');
|
|
96
|
+
const right = parseIpv6Part(pieces[1] ?? '');
|
|
97
|
+
if (!left || !right) return null;
|
|
98
|
+
|
|
99
|
+
if (pieces.length === 1) {
|
|
100
|
+
return left.length === 8 ? left : null;
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
const zeroCount = 8 - left.length - right.length;
|
|
104
|
+
if (zeroCount < 1) return null;
|
|
105
|
+
return [...left, ...Array.from({ length: zeroCount }, () => 0), ...right];
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
function maybeIpv4MappedIpv6(ip: string): string | null {
|
|
109
|
+
const groups = expandIpv6(ip);
|
|
110
|
+
if (!groups) return null;
|
|
111
|
+
|
|
112
|
+
const ipv4Value = ((groups[6] ?? 0) << 16) + (groups[7] ?? 0);
|
|
113
|
+
const isIpv4Mapped =
|
|
114
|
+
groups.slice(0, 5).every((group) => group === 0) && groups[5] === 0xffff;
|
|
115
|
+
if (!isIpv4Mapped) return null;
|
|
116
|
+
|
|
117
|
+
return ipv4IntToAddress(ipv4Value >>> 0);
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
function isIpv4CompatibleIpv6(ip: string): boolean {
|
|
121
|
+
const groups = expandIpv6(ip);
|
|
122
|
+
if (!groups) return false;
|
|
123
|
+
return groups.slice(0, 6).every((group) => group === 0);
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
function ipv6InCidr(
|
|
127
|
+
groups: number[],
|
|
128
|
+
network: number[],
|
|
129
|
+
prefix: number,
|
|
130
|
+
): boolean {
|
|
131
|
+
let remainingBits = prefix;
|
|
132
|
+
for (let index = 0; index < groups.length; index += 1) {
|
|
133
|
+
if (remainingBits <= 0) return true;
|
|
134
|
+
const bits = Math.min(16, remainingBits);
|
|
135
|
+
const mask = bits === 16 ? 0xffff : (0xffff << (16 - bits)) & 0xffff;
|
|
136
|
+
if ((groups[index]! & mask) !== (network[index]! & mask)) {
|
|
137
|
+
return false;
|
|
138
|
+
}
|
|
139
|
+
remainingBits -= bits;
|
|
140
|
+
}
|
|
141
|
+
return true;
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
function isBlockedIpv6(ip: string): boolean {
|
|
145
|
+
const normalized = normalizeUrlHostname(ip).toLowerCase();
|
|
146
|
+
if (isIpv4CompatibleIpv6(normalized)) return true;
|
|
147
|
+
|
|
148
|
+
const mappedIpv4 = maybeIpv4MappedIpv6(normalized);
|
|
149
|
+
if (mappedIpv4) return isBlockedIpv4(mappedIpv4);
|
|
150
|
+
|
|
151
|
+
const groups = expandIpv6(normalized);
|
|
152
|
+
if (!groups) return false;
|
|
153
|
+
|
|
154
|
+
const publicUnicast = expandIpv6('2000::');
|
|
155
|
+
if (!publicUnicast || !ipv6InCidr(groups, publicUnicast, 3)) {
|
|
156
|
+
return true;
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
return BLOCKED_IPV6_CIDRS.some(([network, prefix]) => {
|
|
160
|
+
const networkGroups = expandIpv6(network);
|
|
161
|
+
return networkGroups !== null && ipv6InCidr(groups, networkGroups, prefix);
|
|
162
|
+
});
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
export function normalizeUrlHostname(hostname: string): string {
|
|
166
|
+
return hostname
|
|
167
|
+
.trim()
|
|
168
|
+
.toLowerCase()
|
|
169
|
+
.replace(/^\[(.*)\]$/, '$1')
|
|
170
|
+
.replace(/\.$/, '');
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
export function isBlockedIpAddress(ip: string): boolean {
|
|
174
|
+
const normalized = normalizeUrlHostname(ip);
|
|
175
|
+
if (normalized.includes(':')) {
|
|
176
|
+
return isBlockedIpv6(normalized);
|
|
177
|
+
}
|
|
178
|
+
return isBlockedIpv4(normalized);
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
export function isIpAddressLiteral(hostname: string): boolean {
|
|
182
|
+
const normalized = normalizeUrlHostname(hostname);
|
|
183
|
+
return ipv4ToInt(normalized) !== null || expandIpv6(normalized) !== null;
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
export function isBlockedOutboundHostname(hostname: string): boolean {
|
|
187
|
+
const normalized = normalizeUrlHostname(hostname);
|
|
188
|
+
return (
|
|
189
|
+
!normalized ||
|
|
190
|
+
BLOCKED_HOSTNAMES.has(normalized) ||
|
|
191
|
+
normalized.endsWith('.localhost') ||
|
|
192
|
+
normalized.endsWith('.local') ||
|
|
193
|
+
isBlockedIpAddress(normalized)
|
|
194
|
+
);
|
|
195
|
+
}
|
|
196
|
+
|
|
197
|
+
export function assertPublicHttpUrl(rawUrl: string | URL): URL {
|
|
198
|
+
let url: URL;
|
|
199
|
+
try {
|
|
200
|
+
url = rawUrl instanceof URL ? new URL(rawUrl.toString()) : new URL(rawUrl);
|
|
201
|
+
} catch {
|
|
202
|
+
throw new UnsafeOutboundUrlError('url must be a valid absolute URL.');
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
if (url.protocol !== 'http:' && url.protocol !== 'https:') {
|
|
206
|
+
throw new UnsafeOutboundUrlError('Only http and https URLs are allowed.');
|
|
207
|
+
}
|
|
208
|
+
if (url.username || url.password) {
|
|
209
|
+
throw new UnsafeOutboundUrlError(
|
|
210
|
+
'Credentials in URLs are not allowed. Use headers instead.',
|
|
211
|
+
);
|
|
212
|
+
}
|
|
213
|
+
|
|
214
|
+
const hostname = normalizeUrlHostname(url.hostname);
|
|
215
|
+
if (!hostname) {
|
|
216
|
+
throw new UnsafeOutboundUrlError('URL hostname is required.');
|
|
217
|
+
}
|
|
218
|
+
if (isBlockedOutboundHostname(hostname)) {
|
|
219
|
+
throw new UnsafeOutboundUrlError(
|
|
220
|
+
`Target host "${hostname}" is not allowed.`,
|
|
221
|
+
);
|
|
222
|
+
}
|
|
223
|
+
|
|
224
|
+
return url;
|
|
225
|
+
}
|
|
226
|
+
|
|
227
|
+
export function resolveRedirectUrl(location: string, currentUrl: URL): URL {
|
|
228
|
+
try {
|
|
229
|
+
return assertPublicHttpUrl(new URL(location, currentUrl));
|
|
230
|
+
} catch (error) {
|
|
231
|
+
if (error instanceof UnsafeOutboundUrlError) throw error;
|
|
232
|
+
throw new UnsafeOutboundUrlError('redirect location must be a valid URL.');
|
|
233
|
+
}
|
|
234
|
+
}
|
|
235
|
+
|
|
236
|
+
export function isRedirectStatus(status: number): boolean {
|
|
237
|
+
return [301, 302, 303, 307, 308].includes(status);
|
|
238
|
+
}
|