deepline 0.1.62 → 0.1.64
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli/index.js +702 -339
- package/dist/cli/index.mjs +727 -348
- package/dist/index.d.mts +7 -1
- package/dist/index.d.ts +7 -1
- package/dist/index.js +77 -51
- package/dist/index.mjs +77 -51
- package/dist/repo/apps/play-runner-workers/src/coordinator-entry.ts +9 -10
- package/dist/repo/apps/play-runner-workers/src/entry.ts +55 -0
- package/dist/repo/apps/play-runner-workers/src/runtime/dataset-handles.ts +36 -27
- package/dist/repo/apps/play-runner-workers/src/runtime/tool-http-errors.ts +5 -2
- package/dist/repo/sdk/src/client.ts +71 -63
- package/dist/repo/sdk/src/errors.ts +5 -1
- package/dist/repo/sdk/src/http.ts +25 -17
- package/dist/repo/sdk/src/plays/local-file-discovery.ts +93 -24
- package/dist/repo/sdk/src/release.ts +2 -2
- package/dist/repo/sdk/src/tool-output.ts +40 -20
- package/dist/repo/shared_libs/play-runtime/batch-runtime.ts +10 -3
- package/dist/repo/shared_libs/play-runtime/batching-types.ts +15 -4
- package/dist/repo/shared_libs/play-runtime/coordinator-headers.ts +2 -1
- package/dist/repo/shared_libs/play-runtime/dedup-backend.ts +0 -0
- package/dist/repo/shared_libs/play-runtime/default-batch-strategies.ts +3 -4
- package/dist/repo/shared_libs/play-runtime/run-failure.ts +1 -3
- package/dist/repo/shared_libs/play-runtime/step-lifecycle-tracker.ts +4 -1
- package/dist/repo/shared_libs/play-runtime/tool-batch-executor.ts +4 -1
- package/dist/repo/shared_libs/play-runtime/tool-result.ts +28 -15
- package/dist/repo/shared_libs/plays/dataset.ts +10 -11
- package/package.json +1 -1
|
@@ -35,7 +35,10 @@ export type WorkerDatasetCapabilities = {
|
|
|
35
35
|
materialization: 'bounded' | 'in_memory';
|
|
36
36
|
};
|
|
37
37
|
|
|
38
|
-
const datasetChunkReaders = new WeakMap<
|
|
38
|
+
const datasetChunkReaders = new WeakMap<
|
|
39
|
+
object,
|
|
40
|
+
DatasetChunkReader<DatasetRow>
|
|
41
|
+
>();
|
|
39
42
|
const datasetCountHints = new WeakMap<object, number | null>();
|
|
40
43
|
const datasetCapabilities = new WeakMap<object, WorkerDatasetCapabilities>();
|
|
41
44
|
|
|
@@ -154,11 +157,7 @@ export function createPersistedDatasetHandle<T extends DatasetRow>(input: {
|
|
|
154
157
|
sourceLabel?: string | null;
|
|
155
158
|
workProgress?: PlayDatasetWorkProgressSummary;
|
|
156
159
|
readRows: (input: { limit: number; offset: number }) => Promise<readonly T[]>;
|
|
157
|
-
trace?: (
|
|
158
|
-
phase: string,
|
|
159
|
-
ms: number,
|
|
160
|
-
extra?: Record<string, unknown>,
|
|
161
|
-
) => void;
|
|
160
|
+
trace?: (phase: string, ms: number, extra?: Record<string, unknown>) => void;
|
|
162
161
|
nowMs: () => number;
|
|
163
162
|
}): WorkerDatasetHandle<T> {
|
|
164
163
|
const datasetKind = input.datasetKind ?? 'map';
|
|
@@ -202,7 +201,9 @@ export function createPersistedDatasetHandle<T extends DatasetRow>(input: {
|
|
|
202
201
|
return rows.map(internalDatasetRow);
|
|
203
202
|
}
|
|
204
203
|
|
|
205
|
-
async function* readChunks(
|
|
204
|
+
async function* readChunks(
|
|
205
|
+
chunkSize: number,
|
|
206
|
+
): AsyncGenerator<T[], void, void> {
|
|
206
207
|
const size = normalizedChunkSize(chunkSize);
|
|
207
208
|
let offset = 0;
|
|
208
209
|
while (offset < count) {
|
|
@@ -255,13 +256,18 @@ export function createPersistedDatasetHandle<T extends DatasetRow>(input: {
|
|
|
255
256
|
}) as AsyncIterable<T>,
|
|
256
257
|
},
|
|
257
258
|
});
|
|
258
|
-
return registerChunkReader(
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
259
|
+
return registerChunkReader(
|
|
260
|
+
dataset,
|
|
261
|
+
(chunkSize) => readChunks(chunkSize),
|
|
262
|
+
count,
|
|
263
|
+
{
|
|
264
|
+
storage: 'runtime_sheet',
|
|
265
|
+
chunkReadable: true,
|
|
266
|
+
pageBacked: true,
|
|
267
|
+
replayable: true,
|
|
268
|
+
materialization: 'bounded',
|
|
269
|
+
},
|
|
270
|
+
);
|
|
265
271
|
}
|
|
266
272
|
|
|
267
273
|
export function createCsvDatasetHandle<T extends DatasetRow>(input: {
|
|
@@ -274,17 +280,15 @@ export function createCsvDatasetHandle<T extends DatasetRow>(input: {
|
|
|
274
280
|
byteChunks: AsyncIterable<Uint8Array>,
|
|
275
281
|
chunkSize: number,
|
|
276
282
|
) => AsyncIterable<T[]>;
|
|
277
|
-
trace?: (
|
|
278
|
-
phase: string,
|
|
279
|
-
ms: number,
|
|
280
|
-
extra?: Record<string, unknown>,
|
|
281
|
-
) => void;
|
|
283
|
+
trace?: (phase: string, ms: number, extra?: Record<string, unknown>) => void;
|
|
282
284
|
nowMs: () => number;
|
|
283
285
|
}): WorkerDatasetHandle<T> {
|
|
284
286
|
const datasetId = `csv:${input.name}`;
|
|
285
287
|
let cachedCount: number | null = null;
|
|
286
288
|
|
|
287
|
-
async function* readChunks(
|
|
289
|
+
async function* readChunks(
|
|
290
|
+
chunkSize: number,
|
|
291
|
+
): AsyncGenerator<T[], void, void> {
|
|
288
292
|
const startedAt = input.nowMs();
|
|
289
293
|
let yieldedRows = 0;
|
|
290
294
|
let yieldedChunks = 0;
|
|
@@ -386,13 +390,18 @@ export function createCsvDatasetHandle<T extends DatasetRow>(input: {
|
|
|
386
390
|
}) as AsyncIterable<T>,
|
|
387
391
|
},
|
|
388
392
|
});
|
|
389
|
-
return registerChunkReader(
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
393
|
+
return registerChunkReader(
|
|
394
|
+
dataset,
|
|
395
|
+
(chunkSize) => readChunks(chunkSize),
|
|
396
|
+
null,
|
|
397
|
+
{
|
|
398
|
+
storage: 'csv_stream',
|
|
399
|
+
chunkReadable: true,
|
|
400
|
+
pageBacked: false,
|
|
401
|
+
replayable: true,
|
|
402
|
+
materialization: 'bounded',
|
|
403
|
+
},
|
|
404
|
+
);
|
|
396
405
|
}
|
|
397
406
|
|
|
398
407
|
export function createInlineDatasetHandle<T extends DatasetRow>(
|
|
@@ -79,7 +79,8 @@ function normalizeHardBillingPayload(
|
|
|
79
79
|
: 'MONTHLY_BILLING_LIMIT_EXCEEDED',
|
|
80
80
|
error_category: 'billing',
|
|
81
81
|
failure_origin:
|
|
82
|
-
typeof payload.failure_origin === 'string' &&
|
|
82
|
+
typeof payload.failure_origin === 'string' &&
|
|
83
|
+
payload.failure_origin.trim()
|
|
83
84
|
? payload.failure_origin
|
|
84
85
|
: 'deepline_billing',
|
|
85
86
|
message:
|
|
@@ -194,5 +195,7 @@ export function extractErrorBilling(
|
|
|
194
195
|
}
|
|
195
196
|
|
|
196
197
|
export function isHardBillingToolHttpError(error: unknown): boolean {
|
|
197
|
-
return
|
|
198
|
+
return (
|
|
199
|
+
error instanceof ToolHttpError && isHardBillingFailurePayload(error.billing)
|
|
200
|
+
);
|
|
198
201
|
}
|
|
@@ -183,11 +183,11 @@ function isRecord(value: unknown): value is Record<string, unknown> {
|
|
|
183
183
|
function isPlayRunPackage(value: unknown): value is PlayRunPackage {
|
|
184
184
|
return Boolean(
|
|
185
185
|
value &&
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
186
|
+
typeof value === 'object' &&
|
|
187
|
+
!Array.isArray(value) &&
|
|
188
|
+
(value as Record<string, unknown>).kind === 'play_run' &&
|
|
189
|
+
(value as Record<string, unknown>).run &&
|
|
190
|
+
typeof (value as { run?: { id?: unknown } }).run?.id === 'string',
|
|
191
191
|
);
|
|
192
192
|
}
|
|
193
193
|
|
|
@@ -209,7 +209,7 @@ function normalizePlayStatus(raw: Record<string, unknown>): PlayStatus {
|
|
|
209
209
|
? raw.runId
|
|
210
210
|
: typeof raw.workflowId === 'string'
|
|
211
211
|
? raw.workflowId
|
|
212
|
-
: packageRun?.id ?? '';
|
|
212
|
+
: (packageRun?.id ?? '');
|
|
213
213
|
return {
|
|
214
214
|
...(raw as unknown as Omit<PlayStatus, 'runId' | 'status'>),
|
|
215
215
|
runId,
|
|
@@ -228,7 +228,9 @@ function normalizePlayRunStart(raw: Record<string, unknown>): PlayRunStart {
|
|
|
228
228
|
return raw as unknown as PlayRunStart;
|
|
229
229
|
}
|
|
230
230
|
const status =
|
|
231
|
-
typeof runPackage.run.status === 'string'
|
|
231
|
+
typeof runPackage.run.status === 'string'
|
|
232
|
+
? runPackage.run.status
|
|
233
|
+
: 'running';
|
|
232
234
|
return {
|
|
233
235
|
workflowId: runPackage.run.id,
|
|
234
236
|
name: runPackage.run.playName,
|
|
@@ -236,9 +238,7 @@ function normalizePlayRunStart(raw: Record<string, unknown>): PlayRunStart {
|
|
|
236
238
|
...(runPackage.run.dashboardUrl
|
|
237
239
|
? { dashboardUrl: runPackage.run.dashboardUrl }
|
|
238
240
|
: {}),
|
|
239
|
-
...(TERMINAL_PLAY_STATUSES.has(status)
|
|
240
|
-
? { finalStatus: runPackage }
|
|
241
|
-
: {}),
|
|
241
|
+
...(TERMINAL_PLAY_STATUSES.has(status) ? { finalStatus: runPackage } : {}),
|
|
242
242
|
package: runPackage,
|
|
243
243
|
};
|
|
244
244
|
}
|
|
@@ -267,7 +267,9 @@ type PlayLiveStatusState = {
|
|
|
267
267
|
latest: PlayStatus | null;
|
|
268
268
|
};
|
|
269
269
|
|
|
270
|
-
function getPlayLiveEventPayload(
|
|
270
|
+
function getPlayLiveEventPayload(
|
|
271
|
+
event: PlayLiveEvent,
|
|
272
|
+
): Record<string, unknown> {
|
|
271
273
|
return event.payload && typeof event.payload === 'object'
|
|
272
274
|
? (event.payload as Record<string, unknown>)
|
|
273
275
|
: {};
|
|
@@ -309,16 +311,14 @@ function updatePlayLiveStatusState(
|
|
|
309
311
|
? payload.runId
|
|
310
312
|
: isPlayRunPackage(payload)
|
|
311
313
|
? payload.run.id
|
|
312
|
-
|
|
314
|
+
: state.runId;
|
|
313
315
|
const status =
|
|
314
316
|
normalizeLiveStatus(payload.status) ??
|
|
315
317
|
(isPlayRunPackage(payload)
|
|
316
318
|
? normalizeLiveStatus(payload.run.status)
|
|
317
319
|
: null) ??
|
|
318
320
|
state.status;
|
|
319
|
-
const progressPayload = isRecord(payload.progress)
|
|
320
|
-
? payload.progress
|
|
321
|
-
: {};
|
|
321
|
+
const progressPayload = isRecord(payload.progress) ? payload.progress : {};
|
|
322
322
|
const payloadLogs = readStringArray(payload.logs);
|
|
323
323
|
const progressLogs = readStringArray(progressPayload.logs);
|
|
324
324
|
const logs = payloadLogs.length > 0 ? payloadLogs : progressLogs;
|
|
@@ -342,7 +342,10 @@ function updatePlayLiveStatusState(
|
|
|
342
342
|
|
|
343
343
|
const progressRecord = progressPayload;
|
|
344
344
|
const next: PlayStatus = {
|
|
345
|
-
...(payload as unknown as Omit<
|
|
345
|
+
...(payload as unknown as Omit<
|
|
346
|
+
PlayStatus,
|
|
347
|
+
'runId' | 'status' | 'progress'
|
|
348
|
+
>),
|
|
346
349
|
runId,
|
|
347
350
|
status,
|
|
348
351
|
...(isPlayRunPackage(payload)
|
|
@@ -485,7 +488,9 @@ export class DeeplineClient {
|
|
|
485
488
|
return `deepline plays run ${target} --input '{...}' --watch`;
|
|
486
489
|
}
|
|
487
490
|
|
|
488
|
-
private starterPlayPath(
|
|
491
|
+
private starterPlayPath(
|
|
492
|
+
play: Pick<PlayListItem, 'name' | 'reference'>,
|
|
493
|
+
): string {
|
|
489
494
|
const target = play.reference || play.name;
|
|
490
495
|
const unqualifiedName = target.split('/').pop() || play.name;
|
|
491
496
|
const safeName = unqualifiedName
|
|
@@ -741,47 +746,50 @@ export class DeeplineClient {
|
|
|
741
746
|
* artifactStorageKey: 'plays/v1/orgs/acme/plays/my-play/artifacts/playgraph_abc123.json',
|
|
742
747
|
* });
|
|
743
748
|
* ```
|
|
744
|
-
|
|
749
|
+
*/
|
|
745
750
|
async startPlayRun(request: StartPlayRunRequest): Promise<PlayRunStart> {
|
|
746
|
-
const response = await this.http.post<Record<string, unknown>>(
|
|
747
|
-
|
|
748
|
-
|
|
749
|
-
|
|
750
|
-
? {
|
|
751
|
-
|
|
752
|
-
|
|
753
|
-
|
|
754
|
-
|
|
755
|
-
? {
|
|
756
|
-
|
|
757
|
-
|
|
758
|
-
|
|
759
|
-
|
|
760
|
-
? {
|
|
761
|
-
|
|
762
|
-
|
|
763
|
-
|
|
764
|
-
|
|
765
|
-
|
|
766
|
-
|
|
767
|
-
|
|
768
|
-
|
|
769
|
-
|
|
770
|
-
|
|
771
|
-
|
|
772
|
-
|
|
773
|
-
|
|
774
|
-
? {
|
|
775
|
-
|
|
776
|
-
|
|
777
|
-
|
|
778
|
-
? {
|
|
779
|
-
|
|
780
|
-
|
|
781
|
-
|
|
782
|
-
|
|
783
|
-
|
|
784
|
-
|
|
751
|
+
const response = await this.http.post<Record<string, unknown>>(
|
|
752
|
+
'/api/v2/plays/run',
|
|
753
|
+
{
|
|
754
|
+
...(request.name ? { name: request.name } : {}),
|
|
755
|
+
...(request.revisionId ? { revisionId: request.revisionId } : {}),
|
|
756
|
+
...(request.artifactStorageKey
|
|
757
|
+
? { artifactStorageKey: request.artifactStorageKey }
|
|
758
|
+
: {}),
|
|
759
|
+
...(request.sourceCode ? { sourceCode: request.sourceCode } : {}),
|
|
760
|
+
...(request.sourceFiles ? { sourceFiles: request.sourceFiles } : {}),
|
|
761
|
+
...('staticPipeline' in request
|
|
762
|
+
? { staticPipeline: request.staticPipeline }
|
|
763
|
+
: {}),
|
|
764
|
+
...(request.artifactHash ? { artifactHash: request.artifactHash } : {}),
|
|
765
|
+
...(request.graphHash ? { graphHash: request.graphHash } : {}),
|
|
766
|
+
...(request.runtimeArtifact
|
|
767
|
+
? { runtimeArtifact: request.runtimeArtifact }
|
|
768
|
+
: {}),
|
|
769
|
+
...(request.compilerManifest
|
|
770
|
+
? { compilerManifest: request.compilerManifest }
|
|
771
|
+
: {}),
|
|
772
|
+
...(request.inputFileUpload
|
|
773
|
+
? { inputFileUpload: request.inputFileUpload }
|
|
774
|
+
: {}),
|
|
775
|
+
...(request.packagedFileUploads?.length
|
|
776
|
+
? { packagedFileUploads: request.packagedFileUploads }
|
|
777
|
+
: {}),
|
|
778
|
+
...(request.input ? { input: request.input } : {}),
|
|
779
|
+
...(request.inputFile ? { inputFile: request.inputFile } : {}),
|
|
780
|
+
...(request.packagedFiles?.length
|
|
781
|
+
? { packagedFiles: request.packagedFiles }
|
|
782
|
+
: {}),
|
|
783
|
+
...(request.force ? { force: true } : {}),
|
|
784
|
+
...(typeof request.waitForCompletionMs === 'number'
|
|
785
|
+
? { waitForCompletionMs: request.waitForCompletionMs }
|
|
786
|
+
: {}),
|
|
787
|
+
// Profile selection is the API's job, not the CLI's. The server
|
|
788
|
+
// hardcodes workers_edge as the default; tests that want a
|
|
789
|
+
// different profile pass `request.profile` explicitly.
|
|
790
|
+
...(request.profile ? { profile: request.profile } : {}),
|
|
791
|
+
},
|
|
792
|
+
);
|
|
785
793
|
return normalizePlayRunStart(response);
|
|
786
794
|
}
|
|
787
795
|
|
|
@@ -1173,10 +1181,9 @@ export class DeeplineClient {
|
|
|
1173
1181
|
}
|
|
1174
1182
|
return formData;
|
|
1175
1183
|
};
|
|
1176
|
-
const response = await this.http.postFormData<{
|
|
1177
|
-
|
|
1178
|
-
|
|
1179
|
-
);
|
|
1184
|
+
const response = await this.http.postFormData<{
|
|
1185
|
+
files: PlayStagedFileRef[];
|
|
1186
|
+
}>('/api/v2/plays/files/stage', buildFormData);
|
|
1180
1187
|
return response.files;
|
|
1181
1188
|
}
|
|
1182
1189
|
|
|
@@ -1733,8 +1740,9 @@ export class DeeplineClient {
|
|
|
1733
1740
|
options?.onProgress?.(status);
|
|
1734
1741
|
|
|
1735
1742
|
if (TERMINAL_PLAY_STATUSES.has(status.status)) {
|
|
1736
|
-
const finalStatus = await this.getPlayStatus(
|
|
1737
|
-
.
|
|
1743
|
+
const finalStatus = await this.getPlayStatus(
|
|
1744
|
+
status.runId || workflowId,
|
|
1745
|
+
).catch(() => status);
|
|
1738
1746
|
return playRunResultFromStatus(finalStatus, start, workflowId);
|
|
1739
1747
|
}
|
|
1740
1748
|
}
|
|
@@ -92,7 +92,11 @@ export class RateLimitError extends DeeplineError {
|
|
|
92
92
|
public retryAfterMs: number;
|
|
93
93
|
|
|
94
94
|
constructor(retryAfterMs = 5000, message?: string) {
|
|
95
|
-
super(
|
|
95
|
+
super(
|
|
96
|
+
message ?? `Rate limited. Retry after ${retryAfterMs}ms.`,
|
|
97
|
+
429,
|
|
98
|
+
'RATE_LIMIT',
|
|
99
|
+
);
|
|
96
100
|
this.name = 'RateLimitError';
|
|
97
101
|
this.retryAfterMs = retryAfterMs;
|
|
98
102
|
}
|
|
@@ -63,40 +63,45 @@ export class HttpClient {
|
|
|
63
63
|
|
|
64
64
|
private authHeaders(extra?: Record<string, string>): Record<string, string> {
|
|
65
65
|
const headers: Record<string, string> = {
|
|
66
|
-
|
|
66
|
+
Authorization: `Bearer ${this.config.apiKey}`,
|
|
67
67
|
'User-Agent': `deepline-ts-sdk/${SDK_VERSION}`,
|
|
68
68
|
'X-Deepline-SDK-Version': SDK_VERSION,
|
|
69
69
|
'X-Deepline-API-Contract': SDK_API_CONTRACT,
|
|
70
70
|
...extra,
|
|
71
71
|
};
|
|
72
|
-
const bypassToken =
|
|
73
|
-
|
|
74
|
-
|
|
72
|
+
const bypassToken =
|
|
73
|
+
typeof process !== 'undefined'
|
|
74
|
+
? process.env?.VERCEL_PROTECTION_BYPASS_TOKEN
|
|
75
|
+
: undefined;
|
|
75
76
|
if (bypassToken) {
|
|
76
77
|
headers['x-vercel-protection-bypass'] = bypassToken;
|
|
77
78
|
}
|
|
78
|
-
const playArtifactR2Prefix =
|
|
79
|
-
|
|
80
|
-
|
|
79
|
+
const playArtifactR2Prefix =
|
|
80
|
+
typeof process !== 'undefined'
|
|
81
|
+
? process.env?.DEEPLINE_PLAY_ARTIFACT_R2_PREFIX
|
|
82
|
+
: undefined;
|
|
81
83
|
if (playArtifactR2Prefix) {
|
|
82
84
|
headers['x-deepline-play-artifact-r2-prefix'] = playArtifactR2Prefix;
|
|
83
85
|
}
|
|
84
|
-
const coordinatorUrl =
|
|
85
|
-
|
|
86
|
-
|
|
86
|
+
const coordinatorUrl =
|
|
87
|
+
typeof process !== 'undefined'
|
|
88
|
+
? process.env?.DEEPLINE_COORDINATOR_URL
|
|
89
|
+
: undefined;
|
|
87
90
|
if (coordinatorUrl?.trim()) {
|
|
88
91
|
headers[COORDINATOR_URL_OVERRIDE_HEADER] = coordinatorUrl.trim();
|
|
89
|
-
const coordinatorInternalToken =
|
|
90
|
-
|
|
91
|
-
|
|
92
|
+
const coordinatorInternalToken =
|
|
93
|
+
typeof process !== 'undefined'
|
|
94
|
+
? process.env?.DEEPLINE_INTERNAL_TOKEN
|
|
95
|
+
: undefined;
|
|
92
96
|
if (coordinatorInternalToken?.trim()) {
|
|
93
97
|
headers[COORDINATOR_INTERNAL_TOKEN_HEADER] =
|
|
94
98
|
coordinatorInternalToken.trim();
|
|
95
99
|
}
|
|
96
100
|
}
|
|
97
|
-
const workerCallbackUrl =
|
|
98
|
-
|
|
99
|
-
|
|
101
|
+
const workerCallbackUrl =
|
|
102
|
+
typeof process !== 'undefined'
|
|
103
|
+
? process.env?.DEEPLINE_WORKER_CALLBACK_URL
|
|
104
|
+
: undefined;
|
|
100
105
|
if (workerCallbackUrl?.trim()) {
|
|
101
106
|
headers[WORKER_CALLBACK_URL_OVERRIDE_HEADER] = workerCallbackUrl.trim();
|
|
102
107
|
}
|
|
@@ -114,7 +119,10 @@ export class HttpClient {
|
|
|
114
119
|
* @throws {@link RateLimitError} on HTTP 429 after all retries exhausted
|
|
115
120
|
* @throws {@link DeeplineError} on other API errors or connection failures
|
|
116
121
|
*/
|
|
117
|
-
async request<T = unknown>(
|
|
122
|
+
async request<T = unknown>(
|
|
123
|
+
path: string,
|
|
124
|
+
options?: RequestOptions,
|
|
125
|
+
): Promise<T> {
|
|
118
126
|
const baseUrl = this.config.baseUrl;
|
|
119
127
|
const url = `${baseUrl}${path}`;
|
|
120
128
|
const method = options?.method ?? 'GET';
|
|
@@ -1,6 +1,14 @@
|
|
|
1
1
|
import { createHash } from 'node:crypto';
|
|
2
2
|
import { readFile, stat } from 'node:fs/promises';
|
|
3
|
-
import {
|
|
3
|
+
import {
|
|
4
|
+
basename,
|
|
5
|
+
dirname,
|
|
6
|
+
extname,
|
|
7
|
+
isAbsolute,
|
|
8
|
+
join,
|
|
9
|
+
relative,
|
|
10
|
+
resolve,
|
|
11
|
+
} from 'node:path';
|
|
4
12
|
|
|
5
13
|
export interface PlayLocalFileReference {
|
|
6
14
|
sourceFragment: string;
|
|
@@ -33,7 +41,17 @@ export interface PlayStagedFileRef {
|
|
|
33
41
|
|
|
34
42
|
type ConstMap = Map<string, string>;
|
|
35
43
|
|
|
36
|
-
const SOURCE_EXTENSIONS = [
|
|
44
|
+
const SOURCE_EXTENSIONS = [
|
|
45
|
+
'.ts',
|
|
46
|
+
'.tsx',
|
|
47
|
+
'.mts',
|
|
48
|
+
'.cts',
|
|
49
|
+
'.js',
|
|
50
|
+
'.jsx',
|
|
51
|
+
'.mjs',
|
|
52
|
+
'.cjs',
|
|
53
|
+
'.json',
|
|
54
|
+
];
|
|
37
55
|
|
|
38
56
|
function sha256(buffer: Buffer): string {
|
|
39
57
|
return createHash('sha256').update(buffer).digest('hex');
|
|
@@ -50,19 +68,28 @@ function contentTypeForFile(filePath: string): string {
|
|
|
50
68
|
function stripCommentsToSpaces(source: string): string {
|
|
51
69
|
return source
|
|
52
70
|
.replace(/\/\*[\s\S]*?\*\//g, (match) => match.replace(/[^\n]/g, ' '))
|
|
53
|
-
.replace(
|
|
54
|
-
|
|
71
|
+
.replace(
|
|
72
|
+
/(^|[^:])\/\/.*$/gm,
|
|
73
|
+
(match, prefix: string) =>
|
|
74
|
+
prefix + ' '.repeat(Math.max(0, match.length - prefix.length)),
|
|
55
75
|
);
|
|
56
76
|
}
|
|
57
77
|
|
|
58
78
|
function unquoteStringLiteral(literal: string): string | null {
|
|
59
79
|
const trimmed = literal.trim();
|
|
60
80
|
const quote = trimmed[0];
|
|
61
|
-
if (
|
|
81
|
+
if (
|
|
82
|
+
(quote !== '"' && quote !== "'") ||
|
|
83
|
+
trimmed[trimmed.length - 1] !== quote
|
|
84
|
+
) {
|
|
62
85
|
return null;
|
|
63
86
|
}
|
|
64
87
|
try {
|
|
65
|
-
return JSON.parse(
|
|
88
|
+
return JSON.parse(
|
|
89
|
+
quote === '"'
|
|
90
|
+
? trimmed
|
|
91
|
+
: `"${trimmed.slice(1, -1).replace(/"/g, '\\"')}"`,
|
|
92
|
+
);
|
|
66
93
|
} catch {
|
|
67
94
|
return trimmed.slice(1, -1);
|
|
68
95
|
}
|
|
@@ -116,7 +143,10 @@ function isRuntimeInputExpression(expression: string): boolean {
|
|
|
116
143
|
return /(^|[^\w$])input([^\w$]|$)/.test(expression);
|
|
117
144
|
}
|
|
118
145
|
|
|
119
|
-
function resolveStringExpression(
|
|
146
|
+
function resolveStringExpression(
|
|
147
|
+
expression: string,
|
|
148
|
+
constants: ConstMap,
|
|
149
|
+
): string | null {
|
|
120
150
|
const value = stripOuterParens(expression);
|
|
121
151
|
if (/^(['"])(?:\\.|(?!\1)[\s\S])*\1$/.test(value)) {
|
|
122
152
|
return unquoteStringLiteral(value);
|
|
@@ -129,7 +159,9 @@ function resolveStringExpression(expression: string, constants: ConstMap): strin
|
|
|
129
159
|
}
|
|
130
160
|
const parts = splitTopLevelPlus(value);
|
|
131
161
|
if (parts) {
|
|
132
|
-
const resolved = parts.map((part) =>
|
|
162
|
+
const resolved = parts.map((part) =>
|
|
163
|
+
resolveStringExpression(part, constants),
|
|
164
|
+
);
|
|
133
165
|
return resolved.every((part): part is string => part != null)
|
|
134
166
|
? resolved.join('')
|
|
135
167
|
: null;
|
|
@@ -140,7 +172,9 @@ function resolveStringExpression(expression: string, constants: ConstMap): strin
|
|
|
140
172
|
function collectTopLevelStringConstants(sourceCode: string): ConstMap {
|
|
141
173
|
const constants: ConstMap = new Map();
|
|
142
174
|
const source = stripCommentsToSpaces(sourceCode);
|
|
143
|
-
for (const match of source.matchAll(
|
|
175
|
+
for (const match of source.matchAll(
|
|
176
|
+
/(?:^|\n)\s*const\s+([A-Za-z_$][\w$]*)\s*=\s*([^;\n]+)/g,
|
|
177
|
+
)) {
|
|
144
178
|
const resolved = resolveStringExpression(match[2]!, constants);
|
|
145
179
|
if (resolved != null) {
|
|
146
180
|
constants.set(match[1]!, resolved);
|
|
@@ -190,7 +224,10 @@ function findCallOpenParen(source: string, afterCsvIndex: number): number {
|
|
|
190
224
|
return source[index] === '(' ? index : -1;
|
|
191
225
|
}
|
|
192
226
|
|
|
193
|
-
function firstCallArgument(
|
|
227
|
+
function firstCallArgument(
|
|
228
|
+
source: string,
|
|
229
|
+
openParen: number,
|
|
230
|
+
): { text: string; start: number; end: number } | null {
|
|
194
231
|
let depth = 0;
|
|
195
232
|
let quote: string | null = null;
|
|
196
233
|
let escaped = false;
|
|
@@ -235,7 +272,9 @@ function localImportSpecifiers(sourceCode: string): string[] {
|
|
|
235
272
|
)) {
|
|
236
273
|
if (match[1]?.startsWith('.')) specifiers.push(match[1]);
|
|
237
274
|
}
|
|
238
|
-
for (const match of source.matchAll(
|
|
275
|
+
for (const match of source.matchAll(
|
|
276
|
+
/\brequire\s*\(\s*(['"])(\.[^'"]*)\1\s*\)/g,
|
|
277
|
+
)) {
|
|
239
278
|
specifiers.push(match[2]!);
|
|
240
279
|
}
|
|
241
280
|
return specifiers;
|
|
@@ -252,20 +291,34 @@ async function fileExists(filePath: string): Promise<boolean> {
|
|
|
252
291
|
|
|
253
292
|
function isPathInsideDirectory(filePath: string, directory: string): boolean {
|
|
254
293
|
const relativePath = relative(directory, filePath);
|
|
255
|
-
return
|
|
294
|
+
return (
|
|
295
|
+
relativePath === '' ||
|
|
296
|
+
(!relativePath.startsWith('..') && !isAbsolute(relativePath))
|
|
297
|
+
);
|
|
256
298
|
}
|
|
257
299
|
|
|
258
|
-
async function resolveLocalImport(
|
|
259
|
-
|
|
300
|
+
async function resolveLocalImport(
|
|
301
|
+
fromFile: string,
|
|
302
|
+
specifier: string,
|
|
303
|
+
): Promise<string> {
|
|
304
|
+
const base = isAbsolute(specifier)
|
|
305
|
+
? resolve(specifier)
|
|
306
|
+
: resolve(dirname(fromFile), specifier);
|
|
260
307
|
const candidates: string[] = [base];
|
|
261
308
|
const explicitExtension = extname(base).toLowerCase();
|
|
262
309
|
|
|
263
310
|
if (!explicitExtension) {
|
|
264
|
-
candidates.push(
|
|
265
|
-
|
|
311
|
+
candidates.push(
|
|
312
|
+
...SOURCE_EXTENSIONS.map((extension) => `${base}${extension}`),
|
|
313
|
+
);
|
|
314
|
+
candidates.push(
|
|
315
|
+
...SOURCE_EXTENSIONS.map((extension) => join(base, `index${extension}`)),
|
|
316
|
+
);
|
|
266
317
|
} else if (['.js', '.jsx', '.mjs', '.cjs'].includes(explicitExtension)) {
|
|
267
318
|
const stem = base.slice(0, -explicitExtension.length);
|
|
268
|
-
candidates.push(
|
|
319
|
+
candidates.push(
|
|
320
|
+
...SOURCE_EXTENSIONS.map((extension) => `${stem}${extension}`),
|
|
321
|
+
);
|
|
269
322
|
}
|
|
270
323
|
|
|
271
324
|
for (const candidate of candidates) {
|
|
@@ -274,7 +327,9 @@ async function resolveLocalImport(fromFile: string, specifier: string): Promise<
|
|
|
274
327
|
}
|
|
275
328
|
}
|
|
276
329
|
|
|
277
|
-
throw new Error(
|
|
330
|
+
throw new Error(
|
|
331
|
+
`Could not resolve local import "${specifier}" from ${fromFile}`,
|
|
332
|
+
);
|
|
278
333
|
}
|
|
279
334
|
|
|
280
335
|
export async function discoverPackagedLocalFiles(
|
|
@@ -298,12 +353,17 @@ export async function discoverPackagedLocalFiles(
|
|
|
298
353
|
const constants = collectTopLevelStringConstants(sourceCode);
|
|
299
354
|
const childVisits: Promise<void>[] = [];
|
|
300
355
|
|
|
301
|
-
for (const match of scanSource.matchAll(
|
|
356
|
+
for (const match of scanSource.matchAll(
|
|
357
|
+
/\b([A-Za-z_$][\w$]*)\s*\.\s*csv\b/g,
|
|
358
|
+
)) {
|
|
302
359
|
const target = match[1]!;
|
|
303
360
|
if (target !== 'ctx' && !target.endsWith('Ctx')) {
|
|
304
361
|
continue;
|
|
305
362
|
}
|
|
306
|
-
const openParen = findCallOpenParen(
|
|
363
|
+
const openParen = findCallOpenParen(
|
|
364
|
+
scanSource,
|
|
365
|
+
match.index! + match[0].length,
|
|
366
|
+
);
|
|
307
367
|
if (openParen < 0) {
|
|
308
368
|
continue;
|
|
309
369
|
}
|
|
@@ -317,15 +377,22 @@ export async function discoverPackagedLocalFiles(
|
|
|
317
377
|
const resolvedPath = resolveStringExpression(argument.text, constants);
|
|
318
378
|
if (resolvedPath == null) {
|
|
319
379
|
unresolved.push({
|
|
320
|
-
sourceFragment: sourceCode
|
|
380
|
+
sourceFragment: sourceCode
|
|
381
|
+
.slice(argument.start, argument.end)
|
|
382
|
+
.trim(),
|
|
321
383
|
message:
|
|
322
384
|
'Could not resolve this ctx.csv(...) path at submit time. Use a string literal, a top-level const string, or pass a runtime input like input.file.',
|
|
323
385
|
});
|
|
324
386
|
} else {
|
|
325
387
|
const absoluteCsvPath = resolve(dirname(absolutePath), resolvedPath);
|
|
326
|
-
if (
|
|
388
|
+
if (
|
|
389
|
+
isAbsolute(resolvedPath) ||
|
|
390
|
+
!isPathInsideDirectory(absoluteCsvPath, packagingRoot)
|
|
391
|
+
) {
|
|
327
392
|
unresolved.push({
|
|
328
|
-
sourceFragment: sourceCode
|
|
393
|
+
sourceFragment: sourceCode
|
|
394
|
+
.slice(argument.start, argument.end)
|
|
395
|
+
.trim(),
|
|
329
396
|
message:
|
|
330
397
|
'ctx.csv(...) packaged file paths must be relative paths inside the play directory. Pass external files at runtime with input.file instead.',
|
|
331
398
|
});
|
|
@@ -334,7 +401,9 @@ export async function discoverPackagedLocalFiles(
|
|
|
334
401
|
const buffer = await readFile(absoluteCsvPath);
|
|
335
402
|
const stats = await stat(absoluteCsvPath);
|
|
336
403
|
files.set(absoluteCsvPath, {
|
|
337
|
-
sourceFragment: sourceCode
|
|
404
|
+
sourceFragment: sourceCode
|
|
405
|
+
.slice(argument.start, argument.end)
|
|
406
|
+
.trim(),
|
|
338
407
|
logicalPath: resolvedPath,
|
|
339
408
|
absolutePath: absoluteCsvPath,
|
|
340
409
|
bytes: stats.size,
|