deepline 0.3.247 → 0.3.249

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (67) hide show
  1. package/dist/bundling-sources/sdk/src/client.ts +177 -13
  2. package/dist/bundling-sources/sdk/src/index.ts +8 -0
  3. package/dist/bundling-sources/sdk/src/plays/input-example.ts +40 -0
  4. package/dist/bundling-sources/sdk/src/release.ts +1 -1
  5. package/dist/bundling-sources/sdk/src/runs/overview.ts +457 -0
  6. package/dist/bundling-sources/sdk/src/runs/receipts.ts +23 -0
  7. package/dist/bundling-sources/sdk/src/tool-calls.ts +21 -1
  8. package/dist/bundling-sources/sdk/src/types.ts +16 -1
  9. package/dist/bundling-sources/shared_libs/observability/scheduled-jobs.json +19 -1
  10. package/dist/bundling-sources/shared_libs/play-data-plane/column-names.ts +17 -3
  11. package/dist/bundling-sources/shared_libs/play-data-plane/sheet-contract.ts +40 -8
  12. package/dist/bundling-sources/shared_libs/play-runtime/app-runtime-api.ts +26 -1
  13. package/dist/bundling-sources/shared_libs/play-runtime/context.ts +144 -14
  14. package/dist/bundling-sources/shared_libs/play-runtime/ctx-types.ts +2 -1
  15. package/dist/bundling-sources/shared_libs/play-runtime/daytona-runtime-config.ts +6 -3
  16. package/dist/bundling-sources/shared_libs/play-runtime/daytona-sandbox-janitor.ts +26 -21
  17. package/dist/bundling-sources/shared_libs/play-runtime/daytona-sandbox-reaper.ts +23 -21
  18. package/dist/bundling-sources/shared_libs/play-runtime/deployment/fly-machines.ts +17 -0
  19. package/dist/bundling-sources/shared_libs/play-runtime/log-context.ts +105 -0
  20. package/dist/bundling-sources/shared_libs/play-runtime/output-size-limits.ts +8 -1
  21. package/dist/bundling-sources/shared_libs/play-runtime/play-runs/terminal/job.ts +1 -0
  22. package/dist/bundling-sources/shared_libs/play-runtime/play-runs/terminal/ports.ts +1 -0
  23. package/dist/bundling-sources/shared_libs/play-runtime/protocol.ts +9 -0
  24. package/dist/bundling-sources/shared_libs/play-runtime/run-ledger-event-transport.ts +40 -4
  25. package/dist/bundling-sources/shared_libs/play-runtime/run-ledger.ts +50 -5
  26. package/dist/bundling-sources/shared_libs/play-runtime/run-lifecycle-policy.ts +3 -1
  27. package/dist/bundling-sources/shared_libs/play-runtime/run-log-buffer.ts +28 -1
  28. package/dist/bundling-sources/shared_libs/play-runtime/runner-app/index.ts +14 -1
  29. package/dist/bundling-sources/shared_libs/play-runtime/runner-app/runtime-terminal-push.ts +56 -16
  30. package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-control-transport.ts +184 -0
  31. package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-launch-transport.ts +327 -0
  32. package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-lifecycle.ts +87 -30
  33. package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-session-execution.ts +24 -13
  34. package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-start-admission.ts +23 -6
  35. package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona.ts +76 -33
  36. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-runtime-worker.ts +90 -35
  37. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres-pool.ts +16 -0
  38. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres-progress.ts +112 -50
  39. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres.ts +447 -77
  40. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/receipt-gateway.ts +22 -4
  41. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/run-ledger-outbox-recovery.ts +105 -121
  42. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/runtime-event-search.ts +1648 -0
  43. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/runtime-queue-health.ts +16 -19
  44. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/runtime-receipt-value-storage.ts +14 -6
  45. package/dist/bundling-sources/shared_libs/play-runtime/tool-execute-retry-policy.ts +16 -0
  46. package/dist/bundling-sources/shared_libs/play-runtime/tool-http-errors.ts +40 -0
  47. package/dist/bundling-sources/shared_libs/plays/authoring-contract.ts +5 -2
  48. package/dist/bundling-sources/shared_libs/plays/dataset-summary.ts +11 -7
  49. package/dist/bundling-sources/shared_libs/plays/static-pipeline-storage-codec.ts +5 -2
  50. package/dist/bundling-sources/shared_libs/plays/static-pipeline.ts +11 -2
  51. package/dist/cli/index.js +3504 -2368
  52. package/dist/cli/index.mjs +3188 -2052
  53. package/dist/{compiler-manifest-n8ZtPv2h.d.mts → compiler-manifest-DavmLa8y.d.mts} +1 -0
  54. package/dist/{compiler-manifest-n8ZtPv2h.d.ts → compiler-manifest-DavmLa8y.d.ts} +1 -0
  55. package/dist/index.d.mts +97 -5
  56. package/dist/index.d.ts +97 -5
  57. package/dist/index.js +135 -15
  58. package/dist/index.mjs +135 -15
  59. package/dist/install-integrity.json +9 -2
  60. package/dist/plays/bundle-play-file.d.mts +2 -2
  61. package/dist/plays/bundle-play-file.d.ts +2 -2
  62. package/dist/plays/bundle-play-file.mjs +1 -1
  63. package/dist/release.d.mts +1 -1
  64. package/dist/release.d.ts +1 -1
  65. package/dist/release.js +1 -1
  66. package/dist/release.mjs +1 -1
  67. package/package.json +1 -1
@@ -1035,26 +1035,23 @@ export async function readRuntimeQueueHealthFromClient(
1035
1035
  }),
1036
1036
  );
1037
1037
  }
1038
- const [outbox, computeBilling, cleanup] = await Promise.all([
1039
- client.query<CountRow>(
1040
- outboxProjectionQuery(schema, input.includeRetirementCandidates === true),
1041
- [now, sampleLimit + 1, sampleLimit],
1042
- ),
1043
- client.query<CountRow>(
1044
- computeBillingProjectionQuery(
1045
- schema,
1046
- input.includeRetirementCandidates === true,
1047
- ),
1048
- [now, sampleLimit + 1, sampleLimit],
1049
- ),
1050
- client.query<CountRow>(
1051
- cleanupProjectionQuery(
1052
- schema,
1053
- input.includeRetirementCandidates === true,
1054
- ),
1055
- [now, sampleLimit + 1, sampleLimit],
1038
+ // One checkout owns this read. pg serializes its wire protocol, so await
1039
+ // each query instead of concurrently submitting work to the same client.
1040
+ const outbox = await client.query<CountRow>(
1041
+ outboxProjectionQuery(schema, input.includeRetirementCandidates === true),
1042
+ [now, sampleLimit + 1, sampleLimit],
1043
+ );
1044
+ const computeBilling = await client.query<CountRow>(
1045
+ computeBillingProjectionQuery(
1046
+ schema,
1047
+ input.includeRetirementCandidates === true,
1056
1048
  ),
1057
- ]);
1049
+ [now, sampleLimit + 1, sampleLimit],
1050
+ );
1051
+ const cleanup = await client.query<CountRow>(
1052
+ cleanupProjectionQuery(schema, input.includeRetirementCandidates === true),
1053
+ [now, sampleLimit + 1, sampleLimit],
1054
+ );
1058
1055
  projections.push(
1059
1056
  projectionFromRow({
1060
1057
  id: 'runtime:run-ledger-outbox',
@@ -22,6 +22,9 @@ export type RuntimeValueBlobStore = {
22
22
  remove(key: string): Promise<void>;
23
23
  };
24
24
 
25
+ /** A receipt read only needs immutable GET access to the object store. */
26
+ export type RuntimeValueBlobReader = Pick<RuntimeValueBlobStore, 'get'>;
27
+
25
28
  /** Backward-compatible receipt-specific name for the shared object store. */
26
29
  export type RuntimeReceiptValueBlobStore = RuntimeValueBlobStore;
27
30
 
@@ -64,9 +67,9 @@ function transientStoreError(
64
67
  * injects its object-store adapter. This keeps R2 credentials and transport
65
68
  * mechanics at the gateway edge, without a reverse package dependency.
66
69
  */
67
- export function requireRuntimeValueBlobStore(
68
- store: RuntimeValueBlobStore | undefined,
69
- ): RuntimeValueBlobStore {
70
+ export function requireRuntimeValueBlobStore<
71
+ T extends Pick<RuntimeValueBlobStore, 'get'>,
72
+ >(store: T | undefined): T {
70
73
  if (!store) {
71
74
  throw new Error(
72
75
  'Runtime value storage requires an explicit object-store adapter.',
@@ -304,7 +307,7 @@ export function createRuntimeReceiptValuePersister(
304
307
  * while allowing independent immutable objects to use the connection pool.
305
308
  */
306
309
  export function createRuntimeReceiptValueReader(
307
- input: { store?: RuntimeReceiptValueBlobStore } = {},
310
+ input: { store?: RuntimeValueBlobReader } = {},
308
311
  ): RuntimeReceiptValueReader {
309
312
  type Read = {
310
313
  orgId: string;
@@ -384,10 +387,15 @@ export async function readRuntimeReceiptValue(input: {
384
387
  orgId: string;
385
388
  receiptKey: string;
386
389
  reference: RuntimeReceiptValueReference;
387
- store?: RuntimeReceiptValueBlobStore;
390
+ store?: RuntimeValueBlobReader;
388
391
  }): Promise<unknown> {
389
392
  assertRuntimeReceiptValueReferenceScope(input);
390
- const store = requireRuntimeValueBlobStore(input.store);
393
+ const store = input.store;
394
+ if (!store) {
395
+ throw new Error(
396
+ 'Runtime receipt value read requires an object-store reader.',
397
+ );
398
+ }
391
399
  let body: Buffer;
392
400
  try {
393
401
  body = await store.get(input.reference.key);
@@ -1,6 +1,7 @@
1
1
  import {
2
2
  isHardBillingToolHttpError,
3
3
  normalizeToolHttpErrorMessage,
4
+ readExplicitToolHttpRetryability,
4
5
  type ToolHttpError,
5
6
  } from './tool-http-errors';
6
7
  import {
@@ -302,6 +303,7 @@ function decideToolExecuteHttpRetry(input: {
302
303
  hardBillingFailure?: boolean;
303
304
  hasRetryAfterHeader?: boolean;
304
305
  transientHttpRetrySafe?: boolean;
306
+ explicitRetryable?: boolean;
305
307
  }): ToolExecuteHttpRetryDecision {
306
308
  if (input.hardBillingFailure) {
307
309
  return {
@@ -371,6 +373,13 @@ function decideToolExecuteHttpRetry(input: {
371
373
  };
372
374
  }
373
375
  if (input.status >= 500 && input.status < 600) {
376
+ if (input.explicitRetryable === false) {
377
+ return {
378
+ retryable: false,
379
+ attemptCap: 1,
380
+ reason: 'non_retryable_status',
381
+ };
382
+ }
374
383
  if (!input.transientHttpRetrySafe) {
375
384
  return {
376
385
  retryable: false,
@@ -440,6 +449,10 @@ export function createToolExecuteHttpFailureAttemptTracker(): ToolExecuteHttpFai
440
449
  }),
441
450
  hasRetryAfterHeader: true,
442
451
  transientHttpRetrySafe: input.transientHttpRetrySafe === true,
452
+ explicitRetryable: readExplicitToolHttpRetryability({
453
+ toolId: input.toolId,
454
+ bodyText: input.bodyText ?? '',
455
+ }),
443
456
  });
444
457
  attemptsByReason[decision.reason] += 1;
445
458
  return attemptsByReason[decision.reason];
@@ -495,6 +508,7 @@ export function classifyToolExecuteHttpFailure(input: {
495
508
  const concurrencyBackpressure = isConcurrencyBackpressureResponse(input);
496
509
  const runtimeSchedulerCapacity = isRuntimeSchedulerCapacityResponse(input);
497
510
  const providerPacingWait = isProviderPacingWaitResponse(input);
511
+ const explicitRetryable = readExplicitToolHttpRetryability(input);
498
512
  const initialRetryDecision = decideToolExecuteHttpRetry({
499
513
  status: input.status,
500
514
  idempotencyInProgress,
@@ -505,6 +519,7 @@ export function classifyToolExecuteHttpFailure(input: {
505
519
  providerPacingWait,
506
520
  hasRetryAfterHeader,
507
521
  transientHttpRetrySafe,
522
+ explicitRetryable,
508
523
  });
509
524
  const providerPacingWaitBody = providerPacingWait
510
525
  ? parseJsonObject(input.bodyText)
@@ -568,6 +583,7 @@ export function classifyToolExecuteHttpFailure(input: {
568
583
  hardBillingFailure,
569
584
  hasRetryAfterHeader,
570
585
  transientHttpRetrySafe,
586
+ explicitRetryable,
571
587
  });
572
588
  const shouldRetry =
573
589
  providerOwnedRateLimit ||
@@ -359,6 +359,46 @@ function formatPublicToolErrorPayload(input: {
359
359
  ).slice(0, 1_500);
360
360
  }
361
361
 
362
+ /** Undefined means the response made no authoritative retryability claim. */
363
+ export function readExplicitToolHttpRetryability(input: {
364
+ toolId: string;
365
+ bodyText: string;
366
+ schemaVersion?: ToolExecutionErrorSchemaVersion;
367
+ }): boolean | undefined {
368
+ let parsed: Record<string, unknown> | null;
369
+ try {
370
+ const value = JSON.parse(input.bodyText);
371
+ parsed = isRecord(value) ? value : null;
372
+ } catch {
373
+ return undefined;
374
+ }
375
+ const serialized = getObjectField(parsed, 'tool_error');
376
+ const hydrated = deserializeToolExecutionFailure(
377
+ '',
378
+ serialized,
379
+ input.schemaVersion ?? TOOL_EXECUTION_ERROR_SCHEMA_VERSION,
380
+ );
381
+ if (hydrated) {
382
+ // Deserialization defaults missing/unknown retryability to false. Only
383
+ // the original attributed boolean is a decision supplied by the sender.
384
+ return hydrated.origin !== 'unknown'
385
+ ? (getBooleanField(serialized, 'retryable') ?? undefined)
386
+ : undefined;
387
+ }
388
+ const origin = normalizeToolExecutionOrigin(
389
+ getStringField(parsed, 'failure_origin'),
390
+ {
391
+ provider: getStringField(parsed, 'provider'),
392
+ operation: getStringField(parsed, 'operation') ?? input.toolId,
393
+ },
394
+ );
395
+ // Older readers do not recognize every category (notably model_output),
396
+ // but that must not erase an attributed sender's explicit retry refusal.
397
+ return origin !== 'unknown'
398
+ ? (getBooleanField(parsed, 'retryable') ?? undefined)
399
+ : undefined;
400
+ }
401
+
362
402
  export function normalizeToolHttpErrorMessage(input: {
363
403
  toolId: string;
364
404
  status: number;
@@ -1612,7 +1612,10 @@ export interface PlayAuthoringRuntimeContext {
1612
1612
  */
1613
1613
  log(
1614
1614
  message: string,
1615
- options?: { level?: 'debug' | 'info' | 'warn' | 'error' },
1615
+ options?: {
1616
+ level?: 'debug' | 'info' | 'warn' | 'error';
1617
+ context?: Readonly<Record<string, string | number | boolean>>;
1618
+ },
1616
1619
  ): void;
1617
1620
  sleep(ms: number): Promise<void>;
1618
1621
  }
@@ -3259,7 +3262,7 @@ export const PLAY_AUTHORING_CLOUD_TYPE_DECLARATIONS = [
3259
3262
  ` runPlay<TOutput = unknown>(key: string, playRef: ${cloudReferenceType('ctx.runPlay.playRef')}, input: ${cloudReferenceType('ctx.runPlay.input')}, options: PlayCallOptions): Promise<TOutput>;`,
3260
3263
  ' runPlayAsync<TInput extends Record<string, unknown>>(key: string, playRef: PlayReferenceLike<TInput>, input: NoInfer<TInput>, options: PlayAsyncCallOptions): Promise<AsyncPlayRunRef>;',
3261
3264
  ' runPlayAsync(key: string, playRef: string, input: Record<string, unknown>, options: PlayAsyncCallOptions): Promise<AsyncPlayRunRef>;',
3262
- " log(message: string, options?: { level?: 'debug' | 'info' | 'warn' | 'error' }): void;",
3265
+ " log(message: string, options?: { level?: 'debug' | 'info' | 'warn' | 'error'; context?: Readonly<Record<string, string | number | boolean>> }): void;",
3263
3266
  ` sleep(ms: ${cloudReferenceType('ctx.sleep.ms')}): Promise<void>;`,
3264
3267
  '}',
3265
3268
  "export type DefinePlayConfig<TInput, TOutput extends PlayReturnObject> = { id: string; description?: string; input: PlayInputContract<TInput>; run: (ctx: DeeplinePlayRuntimeContext, input: TInput) => Promise<TOutput>; bindings?: PlayBindings<TInput>; billing?: PlayBindings<TInput>['billing']; runtime?: PlayBindings<TInput>['runtime']; compatibility?: PlayBindings<TInput>['compatibility'] };",
@@ -1,11 +1,15 @@
1
+ export const DATASET_EXECUTION_BUCKETS = [
2
+ 'queued',
3
+ 'running',
4
+ 'completed:executed',
5
+ 'completed:reused',
6
+ 'skipped:condition',
7
+ 'skipped:missed',
8
+ 'failed',
9
+ ] as const;
10
+
1
11
  export type DatasetColumnExecutionBucket =
2
- | 'queued'
3
- | 'running'
4
- | 'completed:executed'
5
- | 'completed:reused'
6
- | 'skipped:condition'
7
- | 'skipped:missed'
8
- | 'failed';
12
+ (typeof DATASET_EXECUTION_BUCKETS)[number];
9
13
 
10
14
  export type DatasetColumnExecutionStats = Record<
11
15
  DatasetColumnExecutionBucket,
@@ -3,7 +3,6 @@ import {
3
3
  createStoredStaticPipelineV2,
4
4
  hydrateStoredStaticPipelineV2,
5
5
  readStoredStaticPipeline,
6
- createStaticPipelineV1CompatibilityProjection,
7
6
  type PlayStaticPipeline,
8
7
  } from './static-pipeline';
9
8
  import {
@@ -46,7 +45,11 @@ function executionStorageFields(
46
45
  retainedPipelineGraph: RetainedPipelineGraph;
47
46
  };
48
47
  return {
49
- staticPipeline: createStaticPipelineV1CompatibilityProjection(pipeline),
48
+ // Shared-graph readers own the complete presentation and execution view.
49
+ // Retaining a recursive V1 mirror would expand the same child once per
50
+ // call again, even when the authoritative graph is compact. Keep only the
51
+ // historical field's fixed empty shape; never use it as an execution root.
52
+ staticPipeline: pipeline === null ? null : { fields: [], substeps: [] },
50
53
  retainedPipelineGraph: compact.retainedPipelineGraph,
51
54
  };
52
55
  }
@@ -1,5 +1,8 @@
1
1
  import { normalizeTableNamespace } from './row-identity';
2
- import { sqlSafePlayColumnName } from '../play-data-plane/column-names';
2
+ import {
3
+ sqlSafePlayColumnName,
4
+ sqlSafePlayStepColumnName,
5
+ } from '../play-data-plane/column-names';
3
6
  import type {
4
7
  PlaySheetColumnSource,
5
8
  PlaySheetColumnContract,
@@ -1879,7 +1882,13 @@ export function compileSheetContract(pipeline: PlayStaticPipeline): {
1879
1882
  }
1880
1883
  columns.push({
1881
1884
  ...column,
1882
- sqlName: sqlSafePlayColumnName(column.id),
1885
+ sqlName:
1886
+ column.source === 'waterfallStep' && column.stepId
1887
+ ? sqlSafePlayStepColumnName(
1888
+ column.field ?? column.waterfallId ?? column.id,
1889
+ column.stepId,
1890
+ )
1891
+ : sqlSafePlayColumnName(column.id),
1883
1892
  });
1884
1893
  };
1885
1894