deepline 0.1.210 → 0.1.212
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bundling-sources/apps/play-runner-workers/src/entry.ts +203 -12
- package/dist/bundling-sources/apps/play-runner-workers/src/runtime/customer-console.ts +23 -0
- package/dist/bundling-sources/apps/play-runner-workers/src/runtime/map-chunk-plan.ts +15 -0
- package/dist/bundling-sources/apps/play-runner-workers/src/runtime/map-latency-profile.ts +90 -0
- package/dist/bundling-sources/apps/play-runner-workers/src/runtime/tool-dispatch.ts +162 -100
- package/dist/bundling-sources/sdk/src/client.ts +118 -3
- package/dist/bundling-sources/sdk/src/play.ts +2 -0
- package/dist/bundling-sources/sdk/src/plays/bundle-play-file.ts +1 -1
- package/dist/bundling-sources/sdk/src/release.ts +2 -2
- package/dist/bundling-sources/sdk/src/types.ts +47 -0
- package/dist/bundling-sources/shared_libs/play-runtime/app-runtime-api.ts +253 -30
- package/dist/bundling-sources/shared_libs/play-runtime/context.ts +365 -103
- package/dist/bundling-sources/shared_libs/play-runtime/ctx-types.ts +12 -0
- package/dist/bundling-sources/shared_libs/play-runtime/dataset-id.ts +10 -4
- package/dist/bundling-sources/shared_libs/play-runtime/db-session.ts +9 -0
- package/dist/bundling-sources/shared_libs/play-runtime/durable-receipt-execution.ts +45 -8
- package/dist/bundling-sources/shared_libs/play-runtime/ledger-safe-payload.ts +14 -2
- package/dist/bundling-sources/shared_libs/play-runtime/output-size-limits.ts +4 -2
- package/dist/bundling-sources/shared_libs/play-runtime/receipt-sql.ts +21 -1
- package/dist/bundling-sources/shared_libs/play-runtime/run-ledger.ts +128 -6
- package/dist/bundling-sources/shared_libs/play-runtime/run-snapshot-stream.ts +7 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-payload-transport.ts +5 -1
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona.ts +37 -5
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/types.ts +19 -10
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-api.ts +63 -7
- package/dist/bundling-sources/shared_libs/play-runtime/secret-capability.ts +18 -4
- package/dist/bundling-sources/shared_libs/play-runtime/single-flight.ts +31 -0
- package/dist/bundling-sources/shared_libs/play-runtime/test-runtime-seams.ts +12 -1
- package/dist/bundling-sources/shared_libs/play-runtime/tool-execute-retry-policy.ts +29 -9
- package/dist/bundling-sources/shared_libs/play-runtime/work-receipt-state-machine.ts +22 -1
- package/dist/bundling-sources/shared_libs/plays/bundling/index.ts +79 -1
- package/dist/bundling-sources/shared_libs/plays/dataset.ts +41 -5
- package/dist/cli/index.js +728 -239
- package/dist/cli/index.mjs +728 -239
- package/dist/index.d.mts +93 -7
- package/dist/index.d.ts +93 -7
- package/dist/index.js +245 -46
- package/dist/index.mjs +245 -46
- package/dist/plays/bundle-play-file.mjs +65 -4
- package/package.json +1 -1
|
@@ -9,16 +9,25 @@ import type {
|
|
|
9
9
|
PlayRunnerResult,
|
|
10
10
|
} from '@shared_libs/play-runtime/protocol';
|
|
11
11
|
|
|
12
|
-
export type PlayRunnerRuntimeResource =
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
12
|
+
export type PlayRunnerRuntimeResource = {
|
|
13
|
+
kind: 'daytona_sandbox';
|
|
14
|
+
sandboxId: string;
|
|
15
|
+
billingStartedAt: number;
|
|
16
|
+
billingEndedAt?: number | null;
|
|
17
|
+
terminalReason?: RuntimeResourceTerminalReason | null;
|
|
18
|
+
terminalAt?: number | null;
|
|
19
|
+
cpu?: number | null;
|
|
20
|
+
memoryGiB?: number | null;
|
|
21
|
+
diskGiB?: number | null;
|
|
22
|
+
};
|
|
23
|
+
|
|
24
|
+
export type RuntimeResourceTerminalReason =
|
|
25
|
+
| 'completed'
|
|
26
|
+
| 'runner_failed'
|
|
27
|
+
| 'sandbox_missing'
|
|
28
|
+
| 'sandbox_oom'
|
|
29
|
+
| 'cancelled'
|
|
30
|
+
| 'lease_expired';
|
|
22
31
|
|
|
23
32
|
export class RuntimeResourceFenceLostError extends Error {
|
|
24
33
|
constructor(message: string) {
|
|
@@ -127,6 +127,8 @@ type RuntimeApiContext = {
|
|
|
127
127
|
postgresSessionUnwrapKey?: string | null;
|
|
128
128
|
};
|
|
129
129
|
|
|
130
|
+
const RUNTIME_RUN_DATASET_CATALOG_TABLE = '_deepline_run_datasets';
|
|
131
|
+
|
|
130
132
|
type DirectRuntimeTestFaultState = {
|
|
131
133
|
headerValue: string;
|
|
132
134
|
counts: Partial<Record<RuntimeTestFaultName, number>>;
|
|
@@ -1572,8 +1574,17 @@ async function withRuntimePostgres<T>(
|
|
|
1572
1574
|
requestLocalPool = pool;
|
|
1573
1575
|
}
|
|
1574
1576
|
client = await connectRuntimePostgresPool(pool);
|
|
1577
|
+
if (session.executionRole) {
|
|
1578
|
+
await client.query(
|
|
1579
|
+
`SET ROLE ${quoteIdentifier(session.executionRole)}`,
|
|
1580
|
+
);
|
|
1581
|
+
}
|
|
1575
1582
|
break;
|
|
1576
1583
|
} catch (error) {
|
|
1584
|
+
if (client) {
|
|
1585
|
+
client.release();
|
|
1586
|
+
client = null;
|
|
1587
|
+
}
|
|
1577
1588
|
if (cachePool) {
|
|
1578
1589
|
const pool = postgresPools.get(session.postgresUrl);
|
|
1579
1590
|
postgresPools.delete(session.postgresUrl);
|
|
@@ -1604,7 +1615,13 @@ async function withRuntimePostgres<T>(
|
|
|
1604
1615
|
try {
|
|
1605
1616
|
return await fn(client);
|
|
1606
1617
|
} finally {
|
|
1607
|
-
|
|
1618
|
+
try {
|
|
1619
|
+
if (session.executionRole) {
|
|
1620
|
+
await client.query('RESET ROLE');
|
|
1621
|
+
}
|
|
1622
|
+
} finally {
|
|
1623
|
+
client.release();
|
|
1624
|
+
}
|
|
1608
1625
|
if (requestLocalPool) {
|
|
1609
1626
|
await Promise.resolve(requestLocalPool.end()).catch(() => {});
|
|
1610
1627
|
}
|
|
@@ -1833,6 +1850,38 @@ function columnSummaryTable(session: RuntimePostgresSession): string {
|
|
|
1833
1850
|
return fqRuntimeTable(session, session.postgres.columnSummaryTable);
|
|
1834
1851
|
}
|
|
1835
1852
|
|
|
1853
|
+
function runDatasetCatalogTable(session: RuntimePostgresSession): string {
|
|
1854
|
+
return fqRuntimeTable(session, RUNTIME_RUN_DATASET_CATALOG_TABLE);
|
|
1855
|
+
}
|
|
1856
|
+
|
|
1857
|
+
async function registerRuntimeDataset(
|
|
1858
|
+
session: RuntimePostgresSession,
|
|
1859
|
+
input: { playName: string; tableNamespace: string; runId: string },
|
|
1860
|
+
): Promise<void> {
|
|
1861
|
+
const tableNamespace = normalizeTableNamespace(input.tableNamespace);
|
|
1862
|
+
const playName = normalizePlayNameForSheet(input.playName);
|
|
1863
|
+
const datasetId = createRuntimeDatasetId(playName, tableNamespace);
|
|
1864
|
+
await withRuntimePostgres(session, async (client) => {
|
|
1865
|
+
await client.query(
|
|
1866
|
+
`INSERT INTO ${runDatasetCatalogTable(session)} (
|
|
1867
|
+
run_id, dataset_id, play_name, table_namespace, public_path
|
|
1868
|
+
) VALUES ($1, $2, $3, $4, $5)
|
|
1869
|
+
ON CONFLICT (run_id, dataset_id) DO UPDATE SET
|
|
1870
|
+
play_name = EXCLUDED.play_name,
|
|
1871
|
+
table_namespace = EXCLUDED.table_namespace,
|
|
1872
|
+
public_path = EXCLUDED.public_path,
|
|
1873
|
+
updated_at = now()`,
|
|
1874
|
+
[
|
|
1875
|
+
input.runId,
|
|
1876
|
+
datasetId,
|
|
1877
|
+
playName,
|
|
1878
|
+
tableNamespace,
|
|
1879
|
+
`datasets.${tableNamespace}`,
|
|
1880
|
+
],
|
|
1881
|
+
);
|
|
1882
|
+
});
|
|
1883
|
+
}
|
|
1884
|
+
|
|
1836
1885
|
function workReceiptTable(session: RuntimePostgresSession): string {
|
|
1837
1886
|
return fqRuntimeTable(
|
|
1838
1887
|
session,
|
|
@@ -2022,8 +2071,11 @@ async function withRuntimeWorkReceiptClient<T>(
|
|
|
2022
2071
|
): Promise<T> {
|
|
2023
2072
|
// Receipt tables are part of customer Postgres bootstrap, so the hot path
|
|
2024
2073
|
// should not pay DDL on every fresh runtime isolate. Try the receipt query
|
|
2025
|
-
// first, then self-heal once on a missing relation/column.
|
|
2026
|
-
//
|
|
2074
|
+
// first, then self-heal once on a missing relation/column. Permission loss
|
|
2075
|
+
// during an operational receipt query must fail loud: silently repairing it
|
|
2076
|
+
// can let provider work continue after durable receipt persistence is gone.
|
|
2077
|
+
// Only the schema-migration branch below may repair ownership before its
|
|
2078
|
+
// retry. A genuinely missing schema or second failure also rethrows loudly.
|
|
2027
2079
|
const runWithSelfHeal = async (client: RuntimeQueryClient): Promise<T> => {
|
|
2028
2080
|
try {
|
|
2029
2081
|
return await operation(client);
|
|
@@ -2048,10 +2100,6 @@ async function withRuntimeWorkReceiptClient<T>(
|
|
|
2048
2100
|
playName: context.playName?.trim() || session.playName,
|
|
2049
2101
|
});
|
|
2050
2102
|
}
|
|
2051
|
-
} else if (isPostgresPermissionDeniedError(error)) {
|
|
2052
|
-
await repairRuntimeStorageGrants(context, {
|
|
2053
|
-
playName: context.playName?.trim() || session.playName,
|
|
2054
|
-
});
|
|
2055
2103
|
} else {
|
|
2056
2104
|
throw error;
|
|
2057
2105
|
}
|
|
@@ -5188,6 +5236,14 @@ export async function startRuntimeSheetDataset(
|
|
|
5188
5236
|
ms: Date.now() - sessionStartedAt,
|
|
5189
5237
|
rows: input.rows.length,
|
|
5190
5238
|
});
|
|
5239
|
+
// Register the exact run-scoped Dataset Handle before any row can become
|
|
5240
|
+
// durable. A crash after a later sheet write can therefore never leave
|
|
5241
|
+
// customer data discoverable only through static pipeline inference.
|
|
5242
|
+
await registerRuntimeDataset(session, {
|
|
5243
|
+
playName,
|
|
5244
|
+
tableNamespace: input.tableNamespace,
|
|
5245
|
+
runId: input.runId,
|
|
5246
|
+
});
|
|
5191
5247
|
const normalizeStartedAt = Date.now();
|
|
5192
5248
|
const uniqueRows = new Map<
|
|
5193
5249
|
string,
|
|
@@ -39,10 +39,24 @@ export function isSecretAuth(value: unknown): value is SecretAuth {
|
|
|
39
39
|
}
|
|
40
40
|
|
|
41
41
|
export function valueContainsSecret(value: unknown): boolean {
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
42
|
+
const pending: unknown[] = [value];
|
|
43
|
+
const seen = new WeakSet<object>();
|
|
44
|
+
while (pending.length > 0) {
|
|
45
|
+
const candidate = pending.pop();
|
|
46
|
+
if (isSecretHandle(candidate) || isSecretAuth(candidate)) return true;
|
|
47
|
+
if (typeof candidate === 'string') {
|
|
48
|
+
if (SECRET_HANDLE_MARKER_RE.test(candidate)) return true;
|
|
49
|
+
continue;
|
|
50
|
+
}
|
|
51
|
+
if (!candidate || typeof candidate !== 'object') continue;
|
|
52
|
+
if (seen.has(candidate)) continue;
|
|
53
|
+
seen.add(candidate);
|
|
54
|
+
if (Array.isArray(candidate)) {
|
|
55
|
+
for (const entry of candidate) pending.push(entry);
|
|
56
|
+
} else if (isRecord(candidate)) {
|
|
57
|
+
for (const entry of Object.values(candidate)) pending.push(entry);
|
|
58
|
+
}
|
|
59
|
+
}
|
|
46
60
|
return false;
|
|
47
61
|
}
|
|
48
62
|
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
export type SingleFlight<K, V> = {
|
|
2
|
+
run(key: K, create: () => Promise<V>): Promise<V>;
|
|
3
|
+
has(key: K): boolean;
|
|
4
|
+
activeCount(): number;
|
|
5
|
+
};
|
|
6
|
+
|
|
7
|
+
/** Coalesce concurrent work for one key without caching its settled result. */
|
|
8
|
+
export function createSingleFlight<K, V>(): SingleFlight<K, V> {
|
|
9
|
+
const active = new Map<K, Promise<V>>();
|
|
10
|
+
|
|
11
|
+
return {
|
|
12
|
+
run(key, create) {
|
|
13
|
+
const existing = active.get(key);
|
|
14
|
+
if (existing) return existing;
|
|
15
|
+
|
|
16
|
+
const pending = Promise.resolve().then(create);
|
|
17
|
+
active.set(key, pending);
|
|
18
|
+
const clear = () => {
|
|
19
|
+
if (active.get(key) === pending) active.delete(key);
|
|
20
|
+
};
|
|
21
|
+
void pending.then(clear, clear);
|
|
22
|
+
return pending;
|
|
23
|
+
},
|
|
24
|
+
has(key) {
|
|
25
|
+
return active.has(key);
|
|
26
|
+
},
|
|
27
|
+
activeCount() {
|
|
28
|
+
return active.size;
|
|
29
|
+
},
|
|
30
|
+
};
|
|
31
|
+
}
|
|
@@ -34,6 +34,8 @@ type ValidatedRuntimeTestFaultHeader =
|
|
|
34
34
|
| { ok: false; status: 400 | 403; error: string };
|
|
35
35
|
|
|
36
36
|
export type RuntimeTestPolicyOverrides = {
|
|
37
|
+
/** Opt-in bounded runner map latency profile for local/preview diagnosis. */
|
|
38
|
+
mapLatencyProfile?: boolean;
|
|
37
39
|
receiptLeaseTtlMs?: number;
|
|
38
40
|
sheetAttemptLeaseMs?: number;
|
|
39
41
|
heartbeatIntervalMs?: number;
|
|
@@ -170,7 +172,8 @@ function parseRuntimeTestPolicyOverrides(
|
|
|
170
172
|
const unknownKeys = Object.keys(record).filter(
|
|
171
173
|
(key) =>
|
|
172
174
|
!RUNTIME_TEST_POLICY_MS_FIELDS.has(key) &&
|
|
173
|
-
key !== 'workBudgetYieldLimits'
|
|
175
|
+
key !== 'workBudgetYieldLimits' &&
|
|
176
|
+
key !== 'mapLatencyProfile',
|
|
174
177
|
);
|
|
175
178
|
if (unknownKeys.length > 0) {
|
|
176
179
|
return {
|
|
@@ -179,6 +182,14 @@ function parseRuntimeTestPolicyOverrides(
|
|
|
179
182
|
}
|
|
180
183
|
|
|
181
184
|
const overrides: RuntimeTestPolicyOverrides = {};
|
|
185
|
+
if ('mapLatencyProfile' in record) {
|
|
186
|
+
if (typeof record.mapLatencyProfile !== 'boolean') {
|
|
187
|
+
return {
|
|
188
|
+
error: 'testPolicyOverrides.mapLatencyProfile must be a boolean.',
|
|
189
|
+
};
|
|
190
|
+
}
|
|
191
|
+
overrides.mapLatencyProfile = record.mapLatencyProfile;
|
|
192
|
+
}
|
|
182
193
|
for (const field of RUNTIME_TEST_POLICY_MS_FIELDS) {
|
|
183
194
|
if (!(field in record)) continue;
|
|
184
195
|
const parsed = readPositiveIntegerField({
|
|
@@ -88,14 +88,15 @@ function decideToolExecuteHttpRetry(input: {
|
|
|
88
88
|
hasRetryAfterHeader?: boolean;
|
|
89
89
|
transientHttpRetrySafe?: boolean;
|
|
90
90
|
}): ToolExecuteHttpRetryDecision {
|
|
91
|
+
if (input.hardBillingFailure) {
|
|
92
|
+
return {
|
|
93
|
+
retryable: false,
|
|
94
|
+
attemptCap:
|
|
95
|
+
input.status === 429 ? TOOL_EXECUTE_RATE_LIMIT_MAX_ATTEMPTS : 1,
|
|
96
|
+
reason: 'hard_billing_error',
|
|
97
|
+
};
|
|
98
|
+
}
|
|
91
99
|
if (input.status === 429) {
|
|
92
|
-
if (input.hardBillingFailure) {
|
|
93
|
-
return {
|
|
94
|
-
retryable: false,
|
|
95
|
-
attemptCap: TOOL_EXECUTE_RATE_LIMIT_MAX_ATTEMPTS,
|
|
96
|
-
reason: 'hard_billing_error',
|
|
97
|
-
};
|
|
98
|
-
}
|
|
99
100
|
if (!input.hasRetryAfterHeader) {
|
|
100
101
|
return {
|
|
101
102
|
retryable: true,
|
|
@@ -178,6 +179,7 @@ export function classifyToolExecuteHttpFailure(input: {
|
|
|
178
179
|
bodyText: string;
|
|
179
180
|
retryAfterHeader?: string | null;
|
|
180
181
|
transientHttpRetrySafe?: boolean;
|
|
182
|
+
providerLatencyMs?: number;
|
|
181
183
|
nowMs?: number;
|
|
182
184
|
}): ToolExecuteHttpFailureOutcome {
|
|
183
185
|
const transientHttpRetrySafe = input.transientHttpRetrySafe === true;
|
|
@@ -196,15 +198,33 @@ export function classifyToolExecuteHttpFailure(input: {
|
|
|
196
198
|
maxAttempts: initialRetryDecision.attemptCap,
|
|
197
199
|
bodyText: input.bodyText,
|
|
198
200
|
});
|
|
201
|
+
if (
|
|
202
|
+
typeof input.providerLatencyMs === 'number' &&
|
|
203
|
+
Number.isFinite(input.providerLatencyMs) &&
|
|
204
|
+
input.providerLatencyMs >= 0
|
|
205
|
+
) {
|
|
206
|
+
error.message = `${error.message} (provider call ${Math.round(input.providerLatencyMs)}ms)`;
|
|
207
|
+
}
|
|
208
|
+
const hardBillingFailure = isHardBillingToolHttpError(error);
|
|
199
209
|
const retryDecision = decideToolExecuteHttpRetry({
|
|
200
210
|
status: input.status,
|
|
201
|
-
hardBillingFailure
|
|
211
|
+
hardBillingFailure,
|
|
202
212
|
hasRetryAfterHeader,
|
|
203
213
|
transientHttpRetrySafe,
|
|
204
214
|
});
|
|
205
215
|
const shouldRetry =
|
|
206
216
|
retryDecision.retryable && input.attempt < retryDecision.attemptCap;
|
|
207
|
-
|
|
217
|
+
// In-attempt retry safety and later-run repairability are different
|
|
218
|
+
// decisions. A non-idempotent 5xx must not be repeated automatically inside
|
|
219
|
+
// this execution attempt, but an explicit later `plays run` must be able to
|
|
220
|
+
// repair that failed work while completed calls remain reusable. Otherwise
|
|
221
|
+
// one provider gateway failure poisons the semantic call forever.
|
|
222
|
+
if (
|
|
223
|
+
!hardBillingFailure &&
|
|
224
|
+
(retryDecision.retryable || (input.status >= 500 && input.status < 600))
|
|
225
|
+
) {
|
|
226
|
+
error.receiptFailureKind = 'repairable';
|
|
227
|
+
}
|
|
208
228
|
const retryAfterMs = parseToolExecuteRetryAfterMs(
|
|
209
229
|
input.retryAfterHeader,
|
|
210
230
|
input.nowMs,
|
|
@@ -9,6 +9,7 @@ export type WorkReceiptStateStatus =
|
|
|
9
9
|
export type WorkReceiptStateInput = {
|
|
10
10
|
status: WorkReceiptStateStatus;
|
|
11
11
|
runId?: string | null;
|
|
12
|
+
error?: string | null;
|
|
12
13
|
failureKind?: 'terminal' | 'repairable' | null;
|
|
13
14
|
leaseId?: string | null;
|
|
14
15
|
leaseOwnerRunId?: string | null;
|
|
@@ -16,6 +17,25 @@ export type WorkReceiptStateInput = {
|
|
|
16
17
|
leaseExpiresAt?: string | null;
|
|
17
18
|
};
|
|
18
19
|
|
|
20
|
+
export function isLegacyRepairableWorkReceiptError(
|
|
21
|
+
error: string | null | undefined,
|
|
22
|
+
): boolean {
|
|
23
|
+
const message = error?.trim().toLowerCase() ?? '';
|
|
24
|
+
if (!message) return false;
|
|
25
|
+
if (
|
|
26
|
+
message.includes('billing cap') ||
|
|
27
|
+
message.includes('insufficient credits') ||
|
|
28
|
+
message.includes('monthly billing limit')
|
|
29
|
+
) {
|
|
30
|
+
return false;
|
|
31
|
+
}
|
|
32
|
+
return (
|
|
33
|
+
/(?:^|\D)5\d\d(?:\D|$)/.test(message) ||
|
|
34
|
+
message.includes('transport failed calling') ||
|
|
35
|
+
message.includes('runtime api call timed out')
|
|
36
|
+
);
|
|
37
|
+
}
|
|
38
|
+
|
|
19
39
|
export type WorkReceiptLeaseState =
|
|
20
40
|
| { kind: 'none' }
|
|
21
41
|
| {
|
|
@@ -254,7 +274,8 @@ export function decideWorkReceiptClaim(input: {
|
|
|
254
274
|
const claimantRunAttempt = normalizeAttempt(input.claimantRunAttempt);
|
|
255
275
|
const ownerAttempt = normalizeAttempt(input.receipt.leaseOwnerAttempt);
|
|
256
276
|
if (
|
|
257
|
-
input.receipt.failureKind === 'repairable'
|
|
277
|
+
(input.receipt.failureKind === 'repairable' ||
|
|
278
|
+
isLegacyRepairableWorkReceiptError(input.receipt.error)) &&
|
|
258
279
|
claimantRunId !== null &&
|
|
259
280
|
ownerRunId !== null &&
|
|
260
281
|
(claimantRunId !== ownerRunId || ownerAttempt < claimantRunAttempt)
|
|
@@ -31,7 +31,7 @@ import { buildPlayContractCompatibility } from '../contracts';
|
|
|
31
31
|
import { validatePlaySourceFilesHaveNoInlineSecrets } from '../secret-guardrails';
|
|
32
32
|
import { MAX_ESM_WORKERS_BUNDLE_BYTES, MAX_PLAY_BUNDLE_BYTES } from './limits';
|
|
33
33
|
|
|
34
|
-
const PLAY_BUNDLE_CACHE_VERSION =
|
|
34
|
+
const PLAY_BUNDLE_CACHE_VERSION = 25;
|
|
35
35
|
const PLAY_ARTIFACT_CACHE_DIR = join(
|
|
36
36
|
tmpdir(),
|
|
37
37
|
`deepline-play-artifacts-v${PLAY_BUNDLE_CACHE_VERSION}`,
|
|
@@ -1223,6 +1223,72 @@ function workersPlayEntryAliasPlugin(playFilePath: string): Plugin {
|
|
|
1223
1223
|
};
|
|
1224
1224
|
}
|
|
1225
1225
|
|
|
1226
|
+
const CUSTOMER_CONSOLE_CAPTURE_GLOBAL = '__deeplineCaptureCustomerConsole';
|
|
1227
|
+
|
|
1228
|
+
function sourceLoaderForPath(path: string): 'ts' | 'tsx' | 'js' | 'jsx' {
|
|
1229
|
+
const extension = extname(path).toLowerCase();
|
|
1230
|
+
if (extension === '.tsx') return 'tsx';
|
|
1231
|
+
if (extension === '.jsx') return 'jsx';
|
|
1232
|
+
if (extension === '.js' || extension === '.mjs' || extension === '.cjs') {
|
|
1233
|
+
return 'js';
|
|
1234
|
+
}
|
|
1235
|
+
return 'ts';
|
|
1236
|
+
}
|
|
1237
|
+
|
|
1238
|
+
/**
|
|
1239
|
+
* Give the customer's entry module a lexical console without replacing the
|
|
1240
|
+
* Worker's process-wide console. Runtime diagnostics therefore keep flowing
|
|
1241
|
+
* to Cloudflare while console.* calls authored in the play join its Run Log
|
|
1242
|
+
* Stream. The lazy global lookup is required because ESM dependencies are
|
|
1243
|
+
* evaluated before the harness installs the request-scoped capture function.
|
|
1244
|
+
*/
|
|
1245
|
+
function workersCustomerConsolePlugin(
|
|
1246
|
+
customerSourceFilePaths: readonly string[],
|
|
1247
|
+
): Plugin {
|
|
1248
|
+
const customerSourceFiles = new Set(
|
|
1249
|
+
customerSourceFilePaths
|
|
1250
|
+
.filter((path) => extname(path).toLowerCase() !== '.json')
|
|
1251
|
+
.map((path) => resolve(path)),
|
|
1252
|
+
);
|
|
1253
|
+
return {
|
|
1254
|
+
name: 'deepline-workers-customer-console',
|
|
1255
|
+
setup(buildContext) {
|
|
1256
|
+
buildContext.onLoad({ filter: /./ }, (args) => {
|
|
1257
|
+
if (!customerSourceFiles.has(resolve(args.path))) return undefined;
|
|
1258
|
+
const source = readFileSync(args.path, 'utf8');
|
|
1259
|
+
const declaresOwnConsole =
|
|
1260
|
+
/(?:^|\n)\s*(?:(?:const|let|var|class|function)\s+console\b|import[^\n]*\bconsole\b)/m.test(
|
|
1261
|
+
source,
|
|
1262
|
+
);
|
|
1263
|
+
if (declaresOwnConsole) {
|
|
1264
|
+
return {
|
|
1265
|
+
contents: source,
|
|
1266
|
+
loader: sourceLoaderForPath(args.path),
|
|
1267
|
+
resolveDir: dirname(args.path),
|
|
1268
|
+
};
|
|
1269
|
+
}
|
|
1270
|
+
const prelude = `
|
|
1271
|
+
const console = new Proxy(globalThis.console, {
|
|
1272
|
+
get(target, property) {
|
|
1273
|
+
const capture = globalThis.${CUSTOMER_CONSOLE_CAPTURE_GLOBAL};
|
|
1274
|
+
if (typeof capture === 'function') {
|
|
1275
|
+
return (...args) => capture(String(property), args);
|
|
1276
|
+
}
|
|
1277
|
+
const value = Reflect.get(target, property);
|
|
1278
|
+
return typeof value === 'function' ? value.bind(target) : value;
|
|
1279
|
+
},
|
|
1280
|
+
});
|
|
1281
|
+
`;
|
|
1282
|
+
return {
|
|
1283
|
+
contents: `${prelude}\n${source}`,
|
|
1284
|
+
loader: sourceLoaderForPath(args.path),
|
|
1285
|
+
resolveDir: dirname(args.path),
|
|
1286
|
+
};
|
|
1287
|
+
});
|
|
1288
|
+
},
|
|
1289
|
+
};
|
|
1290
|
+
}
|
|
1291
|
+
|
|
1226
1292
|
function workersNamedPlayEntryAliasPlugin(
|
|
1227
1293
|
playFilePath: string,
|
|
1228
1294
|
exportName: string,
|
|
@@ -1989,6 +2055,7 @@ type PlayArtifactTargetAdapter = {
|
|
|
1989
2055
|
includeWorkersHarnessInGraphHash: boolean;
|
|
1990
2056
|
runEsbuild(input: {
|
|
1991
2057
|
entryFile: string;
|
|
2058
|
+
customerSourceFilePaths: string[];
|
|
1992
2059
|
importedPlayDependencies: ImportedPlayDependency[];
|
|
1993
2060
|
adapter: PlayBundlingAdapter;
|
|
1994
2061
|
exportName: string;
|
|
@@ -1997,6 +2064,7 @@ type PlayArtifactTargetAdapter = {
|
|
|
1997
2064
|
|
|
1998
2065
|
async function runEsbuildForCjsNode(
|
|
1999
2066
|
entryFile: string,
|
|
2067
|
+
_customerSourceFilePaths: string[],
|
|
2000
2068
|
importedPlayDependencies: ImportedPlayDependency[],
|
|
2001
2069
|
adapter: PlayBundlingAdapter,
|
|
2002
2070
|
exportName: string,
|
|
@@ -2048,6 +2116,7 @@ async function runEsbuildForCjsNode(
|
|
|
2048
2116
|
|
|
2049
2117
|
async function runEsbuildForEsmWorkers(
|
|
2050
2118
|
playEntryFile: string,
|
|
2119
|
+
customerSourceFilePaths: string[],
|
|
2051
2120
|
importedPlayDependencies: ImportedPlayDependency[],
|
|
2052
2121
|
adapter: PlayBundlingAdapter,
|
|
2053
2122
|
exportName: string,
|
|
@@ -2058,6 +2127,9 @@ async function runEsbuildForEsmWorkers(
|
|
|
2058
2127
|
exportName === 'default'
|
|
2059
2128
|
? workersPlayEntryAliasPlugin(playEntryFile)
|
|
2060
2129
|
: workersNamedPlayEntryAliasPlugin(playEntryFile, exportName);
|
|
2130
|
+
const customerConsolePlugin = workersCustomerConsolePlugin(
|
|
2131
|
+
customerSourceFilePaths,
|
|
2132
|
+
);
|
|
2061
2133
|
const result = await build({
|
|
2062
2134
|
// Entry is the Workers harness; it imports the play via the virtual
|
|
2063
2135
|
// `deepline-play-entry` alias resolved by workersPlayEntryAliasPlugin.
|
|
@@ -2102,6 +2174,7 @@ async function runEsbuildForEsmWorkers(
|
|
|
2102
2174
|
sdkAliasPlugin,
|
|
2103
2175
|
playProxyPlugin,
|
|
2104
2176
|
playEntryAlias,
|
|
2177
|
+
customerConsolePlugin,
|
|
2105
2178
|
workersNodeBuiltinStubPlugin(),
|
|
2106
2179
|
// Strip non-English zod locale data from the bundle. zod's locales
|
|
2107
2180
|
// are re-exported as a static namespace from `zod/v4/core`, so
|
|
@@ -2136,12 +2209,14 @@ const PLAY_ARTIFACT_TARGET_ADAPTERS: Record<
|
|
|
2136
2209
|
includeWorkersHarnessInGraphHash: false,
|
|
2137
2210
|
runEsbuild: ({
|
|
2138
2211
|
entryFile,
|
|
2212
|
+
customerSourceFilePaths,
|
|
2139
2213
|
importedPlayDependencies,
|
|
2140
2214
|
adapter,
|
|
2141
2215
|
exportName,
|
|
2142
2216
|
}) =>
|
|
2143
2217
|
runEsbuildForCjsNode(
|
|
2144
2218
|
entryFile,
|
|
2219
|
+
customerSourceFilePaths,
|
|
2145
2220
|
importedPlayDependencies,
|
|
2146
2221
|
adapter,
|
|
2147
2222
|
exportName,
|
|
@@ -2153,12 +2228,14 @@ const PLAY_ARTIFACT_TARGET_ADAPTERS: Record<
|
|
|
2153
2228
|
includeWorkersHarnessInGraphHash: true,
|
|
2154
2229
|
runEsbuild: ({
|
|
2155
2230
|
entryFile,
|
|
2231
|
+
customerSourceFilePaths,
|
|
2156
2232
|
importedPlayDependencies,
|
|
2157
2233
|
adapter,
|
|
2158
2234
|
exportName,
|
|
2159
2235
|
}) =>
|
|
2160
2236
|
runEsbuildForEsmWorkers(
|
|
2161
2237
|
entryFile,
|
|
2238
|
+
customerSourceFilePaths,
|
|
2162
2239
|
importedPlayDependencies,
|
|
2163
2240
|
adapter,
|
|
2164
2241
|
exportName,
|
|
@@ -2288,6 +2365,7 @@ export async function bundlePlayFile(
|
|
|
2288
2365
|
|
|
2289
2366
|
const buildOutcome = await targetAdapter.runEsbuild({
|
|
2290
2367
|
entryFile: absolutePath,
|
|
2368
|
+
customerSourceFilePaths: analysis.importPolicy.localFiles,
|
|
2291
2369
|
importedPlayDependencies: analysis.importedPlayDependencies,
|
|
2292
2370
|
adapter,
|
|
2293
2371
|
exportName,
|
|
@@ -125,7 +125,7 @@ export interface PlayDataset<T> extends AsyncIterable<T> {
|
|
|
125
125
|
* Large datasets should flow by handle through Neon-backed storage, not
|
|
126
126
|
* through worker memory as giant arrays.
|
|
127
127
|
*/
|
|
128
|
-
materialize(
|
|
128
|
+
materialize(options?: number | PlayDatasetMaterializeOptions): Promise<T[]>;
|
|
129
129
|
toJSON(): {
|
|
130
130
|
kind: 'dataset';
|
|
131
131
|
datasetKind: PlayDatasetKind;
|
|
@@ -146,9 +146,19 @@ type PlayDatasetResolvers<T> = {
|
|
|
146
146
|
count: () => Promise<number>;
|
|
147
147
|
peek: (limit: number) => Promise<T[]>;
|
|
148
148
|
materialize: (limit?: number) => Promise<T[]>;
|
|
149
|
+
materializeFullPersistedDataset?: (limit?: number) => Promise<T[]>;
|
|
149
150
|
iterate: () => AsyncIterable<T>;
|
|
150
151
|
};
|
|
151
152
|
|
|
153
|
+
export type PlayDatasetMaterializeScope = 'result' | 'full_persisted_dataset';
|
|
154
|
+
|
|
155
|
+
export type PlayDatasetMaterializeOptions = {
|
|
156
|
+
/** Rows returned by this operation, or every current row in its persisted dataset. */
|
|
157
|
+
scope?: PlayDatasetMaterializeScope;
|
|
158
|
+
/** Maximum number of rows to load into memory. */
|
|
159
|
+
limit?: number;
|
|
160
|
+
};
|
|
161
|
+
|
|
152
162
|
type PlayDatasetTransform<T, U = T> =
|
|
153
163
|
| {
|
|
154
164
|
kind: 'map';
|
|
@@ -164,7 +174,7 @@ type PlayDatasetTransform<T, U = T> =
|
|
|
164
174
|
end?: number;
|
|
165
175
|
};
|
|
166
176
|
|
|
167
|
-
function resolveMaterializeLimitCap(): number {
|
|
177
|
+
export function resolveMaterializeLimitCap(): number {
|
|
168
178
|
const raw = process.env.DEEPLINE_PLAY_DATASET_MATERIALIZE_LIMIT;
|
|
169
179
|
const parsed = raw ? Number(raw) : NaN;
|
|
170
180
|
if (Number.isFinite(parsed) && parsed > 0) {
|
|
@@ -301,10 +311,25 @@ class DeferredPlayDataset<T> implements PlayDataset<T> {
|
|
|
301
311
|
return this.slice(0, limit, options);
|
|
302
312
|
}
|
|
303
313
|
|
|
304
|
-
async materialize(
|
|
314
|
+
async materialize(
|
|
315
|
+
options?: number | PlayDatasetMaterializeOptions,
|
|
316
|
+
): Promise<T[]> {
|
|
317
|
+
const scope =
|
|
318
|
+
typeof options === 'object' ? (options.scope ?? 'result') : 'result';
|
|
319
|
+
const limit = typeof options === 'number' ? options : options?.limit;
|
|
305
320
|
const requestedLimit =
|
|
306
321
|
limit !== undefined ? Math.max(0, Math.floor(limit)) : undefined;
|
|
307
322
|
const cap = resolveMaterializeLimitCap();
|
|
323
|
+
const materialize =
|
|
324
|
+
scope === 'full_persisted_dataset'
|
|
325
|
+
? this.resolvers.materializeFullPersistedDataset
|
|
326
|
+
: this.resolvers.materialize;
|
|
327
|
+
if (!materialize) {
|
|
328
|
+
throw new Error(
|
|
329
|
+
'PlayDataset.materialize({ scope: "full_persisted_dataset" }) is only available ' +
|
|
330
|
+
'for a persisted dataset returned by ctx.dataset(...).run().',
|
|
331
|
+
);
|
|
332
|
+
}
|
|
308
333
|
if (requestedLimit !== undefined) {
|
|
309
334
|
if (requestedLimit > cap) {
|
|
310
335
|
throw new Error(
|
|
@@ -312,7 +337,18 @@ class DeferredPlayDataset<T> implements PlayDataset<T> {
|
|
|
312
337
|
'Return the dataset handle instead, or request a smaller bounded slice.',
|
|
313
338
|
);
|
|
314
339
|
}
|
|
315
|
-
return await
|
|
340
|
+
return await materialize(requestedLimit);
|
|
341
|
+
}
|
|
342
|
+
|
|
343
|
+
if (scope === 'full_persisted_dataset') {
|
|
344
|
+
const rows = await materialize(cap + 1);
|
|
345
|
+
if (rows.length > cap) {
|
|
346
|
+
throw new Error(
|
|
347
|
+
'PlayDataset.materialize({ scope: "full_persisted_dataset" }) refuses to load ' +
|
|
348
|
+
`more than ${cap} rows into memory. Pass an explicit bounded limit.`,
|
|
349
|
+
);
|
|
350
|
+
}
|
|
351
|
+
return rows;
|
|
316
352
|
}
|
|
317
353
|
|
|
318
354
|
const count = await this.count();
|
|
@@ -322,7 +358,7 @@ class DeferredPlayDataset<T> implements PlayDataset<T> {
|
|
|
322
358
|
`The hard limit is ${cap}. Return the dataset handle instead or call materialize(limit).`,
|
|
323
359
|
);
|
|
324
360
|
}
|
|
325
|
-
return await
|
|
361
|
+
return await materialize();
|
|
326
362
|
}
|
|
327
363
|
|
|
328
364
|
async *[Symbol.asyncIterator](): AsyncIterator<T> {
|