deepline 0.3.17 → 0.3.18

package/dist/index.js

@@ -783,7 +783,7 @@ var SDK_RELEASE = {
   // 0.3.0 introduces raw-v2: complete scrubbed provider responses are
   // available at toolResponse.rawV2 while toolResponse.raw and all declared
   // getters keep their established compatibility behavior.
-  version: "0.3.17",
+  version: "0.3.18",
   updateSummary: "New raw-v2 tool responses preserve complete scrubbed provider envelopes at toolResponse.rawV2, including JSON:API included resources, links, cursors, and totals. Existing toolResponse.raw and declared getters keep working unchanged.",
   contracts: {
     api: {
@@ -981,8 +981,259 @@ var RUNTIME_ENVIRONMENT_TOKEN_HEADER = "x-deepline-runtime-environment-token";
 var ABSURD_RELEASE_OVERRIDE_HEADER = "x-deepline-absurd-release";
 var SYNTHETIC_RUN_HEADER = "x-deepline-synthetic-run";
 
+// ../shared_libs/play-runtime/runtime-incident-drills.ts
+var RUNTIME_TEST_FAULT_IDS = [
+  "receipt_complete_write_fail",
+  "receipt_fail_write_fail",
+  "worker_receipt_complete_write_fail",
+  "invocation_response_delivery_abort",
+  "receipt_gateway_hold_ms",
+  "receipt_claim_query_hold_once_ms",
+  "receipt_claim_response_timeout",
+  "modal_sandbox_create_resource_exhausted",
+  "runtime_sheet_page_tail_hold_ms"
+];
+var LOCAL_TOPOLOGY = ["local_daytona"];
+var LOCAL_AND_PREVIEW_TOPOLOGIES = [
+  "local_daytona",
+  "preview_fly",
+  "preview_direct_modal"
+];
+var MODAL_PREVIEW_TOPOLOGY = ["preview_direct_modal"];
+var RUNTIME_INCIDENT_DRILL_CATALOG = [
+  {
+    id: "local_worker_process_outage",
+    description: "Interrupt the scheduler worker in the selected non-production runtime topology.",
+    topologies: LOCAL_AND_PREVIEW_TOPOLOGIES,
+    control: {
+      kind: "process",
+      component: "worker",
+      modes: ["pause", "crash"]
+    },
+    expected: {
+      outcome: "The same logical run stays durable through the worker outage and completes after the selected topology recovers.",
+      safetyFacts: ["same_logical_run", "no_false_success"]
+    },
+    verification: {
+      kind: "local_black_box",
+      target: "local-component-outage-recovery-e2e"
+    }
+  },
+  {
+    id: "local_gateway_process_outage",
+    description: "Interrupt the receipt gateway in the selected non-production runtime topology.",
+    topologies: LOCAL_AND_PREVIEW_TOPOLOGIES,
+    control: {
+      kind: "process",
+      component: "gateway",
+      modes: ["pause", "crash"]
+    },
+    expected: {
+      outcome: "In-flight work recovers only at a safe receipt boundary and the same logical run completes after the gateway recovers.",
+      safetyFacts: [
+        "same_logical_run",
+        "no_false_success",
+        "bounded_provider_execution"
+      ]
+    },
+    verification: {
+      kind: "local_black_box",
+      target: "local-component-outage-recovery-e2e"
+    }
+  },
+  {
+    id: "local_runtime_process_outage",
+    description: "Interrupt both runtime components in the selected non-production topology.",
+    topologies: LOCAL_AND_PREVIEW_TOPOLOGIES,
+    control: { kind: "process", component: "all", modes: ["pause", "crash"] },
+    expected: {
+      outcome: "Durable work waits through the combined outage; it never becomes a false success or creates a second logical run.",
+      safetyFacts: ["same_logical_run", "no_false_success"]
+    },
+    verification: {
+      kind: "local_black_box",
+      target: "local-component-outage-recovery-e2e"
+    }
+  },
+  {
+    id: "receipt_complete_write_fail",
+    description: "Fail the next durable receipt-completion write.",
+    topologies: LOCAL_AND_PREVIEW_TOPOLOGIES,
+    control: {
+      kind: "runtime_test_fault",
+      defaultValue: 1,
+      valueMeaning: "occurrence"
+    },
+    expected: {
+      outcome: "The failed run is loud, and a later bounded recovery may repeat the affected provider operation if its prior completion was not durable.",
+      safetyFacts: ["retry_or_explicit_recovery", "bounded_provider_execution"]
+    },
+    verification: {
+      kind: "local_black_box",
+      target: "receipt-persist-failure-run-fatal-e2e"
+    }
+  },
+  {
+    id: "receipt_fail_write_fail",
+    description: "Fail the next durable receipt-failure write.",
+    topologies: LOCAL_AND_PREVIEW_TOPOLOGIES,
+    control: {
+      kind: "runtime_test_fault",
+      defaultValue: 1,
+      valueMeaning: "occurrence"
+    },
+    expected: {
+      outcome: "The failure is loud; recovery remains bounded and may repeat an operation whose prior outcome was not durably recorded.",
+      safetyFacts: ["retry_or_explicit_recovery", "bounded_provider_execution"]
+    },
+    verification: {
+      kind: "focused_regression",
+      target: "tests/lib/plays/runtime-test-fault-registry.test.ts"
+    }
+  },
+  {
+    id: "worker_receipt_complete_write_fail",
+    description: "Fail the worker-side receipt-completion write once.",
+    topologies: LOCAL_AND_PREVIEW_TOPOLOGIES,
+    control: {
+      kind: "runtime_test_fault",
+      defaultValue: 1,
+      valueMeaning: "occurrence"
+    },
+    expected: {
+      outcome: "The worker reports the persistence failure loudly; bounded recovery may repeat work whose completion was not durable.",
+      safetyFacts: ["retry_or_explicit_recovery", "bounded_provider_execution"]
+    },
+    verification: {
+      kind: "focused_regression",
+      target: "tests/lib/plays/runtime-test-fault-registry.test.ts"
+    }
+  },
+  {
+    id: "invocation_response_delivery_abort",
+    description: "Abort one receipt-gateway response after invocation delivery.",
+    topologies: LOCAL_AND_PREVIEW_TOPOLOGIES,
+    control: {
+      kind: "runtime_test_fault",
+      defaultValue: 1,
+      valueMeaning: "occurrence"
+    },
+    expected: {
+      outcome: "The same logical run completes through receipt replay; recovery remains bounded if the provider outcome must be retried.",
+      safetyFacts: ["same_logical_run", "bounded_provider_execution"]
+    },
+    verification: {
+      kind: "local_black_box",
+      target: "invocation-delivery-replay-e2e"
+    }
+  },
+  {
+    id: "receipt_gateway_hold_ms",
+    description: "Hold one checked-out receipt-gateway scheduler client.",
+    topologies: LOCAL_AND_PREVIEW_TOPOLOGIES,
+    control: {
+      kind: "runtime_test_fault",
+      defaultValue: 500,
+      valueMeaning: "milliseconds"
+    },
+    expected: {
+      outcome: "The request is delayed inside the real gateway pool and either completes within its deadline or follows bounded transport recovery.",
+      safetyFacts: ["bounded_provider_execution"]
+    },
+    verification: {
+      kind: "local_black_box",
+      target: "agent-heavy-gateway-load-e2e"
+    }
+  },
+  {
+    id: "receipt_claim_query_hold_once_ms",
+    description: "Hold one physical receipt-claim query.",
+    topologies: LOCAL_AND_PREVIEW_TOPOLOGIES,
+    control: {
+      kind: "runtime_test_fault",
+      defaultValue: 500,
+      valueMeaning: "milliseconds"
+    },
+    expected: {
+      outcome: "The checked-out claim either completes or times out through bounded recovery; an ambiguous outcome may be retried within its budget.",
+      safetyFacts: ["bounded_provider_execution"]
+    },
+    verification: {
+      kind: "local_black_box",
+      target: "receipt-claim-deadline-recovery-e2e"
+    }
+  },
+  {
+    id: "receipt_claim_response_timeout",
+    description: "Report one fully received receipt-claim response as timed out.",
+    topologies: LOCAL_AND_PREVIEW_TOPOLOGIES,
+    control: {
+      kind: "runtime_test_fault",
+      defaultValue: 1,
+      valueMeaning: "occurrence"
+    },
+    expected: {
+      outcome: "The same logical run completes by replaying the committed receipt response or uses bounded recovery if that response is unavailable.",
+      safetyFacts: ["same_logical_run", "bounded_provider_execution"]
+    },
+    verification: {
+      kind: "local_black_box",
+      target: "receipt-ambiguous-response-recovery-e2e"
+    }
+  },
+  {
+    id: "modal_sandbox_create_resource_exhausted",
+    description: "Make Modal sandbox creation return RESOURCE_EXHAUSTED before user code.",
+    topologies: MODAL_PREVIEW_TOPOLOGY,
+    control: {
+      kind: "runtime_test_fault",
+      defaultValue: 1,
+      valueMeaning: "occurrence"
+    },
+    expected: {
+      outcome: "The same logical run enters bounded pre-code defer/recovery and later retries sandbox startup; no user code or provider call precedes the retry.",
+      safetyFacts: [
+        "same_logical_run",
+        "no_user_code_before_recovery",
+        "bounded_provider_execution"
+      ]
+    },
+    verification: {
+      kind: "preview_black_box",
+      target: "modal-sandbox-recovery-e2e"
+    }
+  },
+  {
+    id: "runtime_sheet_page_tail_hold_ms",
+    description: "Hold one runtime-sheet tail-page operation.",
+    topologies: LOCAL_TOPOLOGY,
+    control: {
+      kind: "runtime_test_fault",
+      defaultValue: 500,
+      valueMeaning: "milliseconds"
+    },
+    expected: {
+      outcome: "Page-tail work stays pending during the bounded hold, then completes without losing or duplicating rows.",
+      safetyFacts: ["all_rows_settle_once"]
+    },
+    verification: {
+      kind: "local_black_box",
+      target: "page-tail-sheet-write-hold-e2e"
+    }
+  }
+];
+function isRuntimeTestFaultDrill(drill) {
+  return drill.control.kind === "runtime_test_fault" && RUNTIME_TEST_FAULT_IDS.some((faultId) => faultId === drill.id);
+}
+function runtimeTestFaultDrills() {
+  return RUNTIME_INCIDENT_DRILL_CATALOG.filter(isRuntimeTestFaultDrill);
+}
+
 // ../shared_libs/play-runtime/test-runtime-seams.ts
 var PLAY_RUNTIME_TEST_FAULT_HEADER = "x-deepline-test-fault";
+var SUPPORTED_RUNTIME_TEST_FAULTS = new Set(
+  runtimeTestFaultDrills().map(({ id }) => id)
+);
 var MAX_RUNTIME_TEST_POLICY_MS = 10 * 6e4;
 
 // src/http.ts

package/dist/index.mjs

@@ -706,7 +706,7 @@ var SDK_RELEASE = {
   // 0.3.0 introduces raw-v2: complete scrubbed provider responses are
   // available at toolResponse.rawV2 while toolResponse.raw and all declared
   // getters keep their established compatibility behavior.
-  version: "0.3.17",
+  version: "0.3.18",
   updateSummary: "New raw-v2 tool responses preserve complete scrubbed provider envelopes at toolResponse.rawV2, including JSON:API included resources, links, cursors, and totals. Existing toolResponse.raw and declared getters keep working unchanged.",
   contracts: {
     api: {
@@ -904,8 +904,259 @@ var RUNTIME_ENVIRONMENT_TOKEN_HEADER = "x-deepline-runtime-environment-token";
 var ABSURD_RELEASE_OVERRIDE_HEADER = "x-deepline-absurd-release";
 var SYNTHETIC_RUN_HEADER = "x-deepline-synthetic-run";
 
+// ../shared_libs/play-runtime/runtime-incident-drills.ts
+var RUNTIME_TEST_FAULT_IDS = [
+  "receipt_complete_write_fail",
+  "receipt_fail_write_fail",
+  "worker_receipt_complete_write_fail",
+  "invocation_response_delivery_abort",
+  "receipt_gateway_hold_ms",
+  "receipt_claim_query_hold_once_ms",
+  "receipt_claim_response_timeout",
+  "modal_sandbox_create_resource_exhausted",
+  "runtime_sheet_page_tail_hold_ms"
+];
+var LOCAL_TOPOLOGY = ["local_daytona"];
+var LOCAL_AND_PREVIEW_TOPOLOGIES = [
+  "local_daytona",
+  "preview_fly",
+  "preview_direct_modal"
+];
+var MODAL_PREVIEW_TOPOLOGY = ["preview_direct_modal"];
+var RUNTIME_INCIDENT_DRILL_CATALOG = [
+  {
+    id: "local_worker_process_outage",
+    description: "Interrupt the scheduler worker in the selected non-production runtime topology.",
+    topologies: LOCAL_AND_PREVIEW_TOPOLOGIES,
+    control: {
+      kind: "process",
+      component: "worker",
+      modes: ["pause", "crash"]
+    },
+    expected: {
+      outcome: "The same logical run stays durable through the worker outage and completes after the selected topology recovers.",
+      safetyFacts: ["same_logical_run", "no_false_success"]
+    },
+    verification: {
+      kind: "local_black_box",
+      target: "local-component-outage-recovery-e2e"
+    }
+  },
+  {
+    id: "local_gateway_process_outage",
+    description: "Interrupt the receipt gateway in the selected non-production runtime topology.",
+    topologies: LOCAL_AND_PREVIEW_TOPOLOGIES,
+    control: {
+      kind: "process",
+      component: "gateway",
+      modes: ["pause", "crash"]
+    },
+    expected: {
+      outcome: "In-flight work recovers only at a safe receipt boundary and the same logical run completes after the gateway recovers.",
+      safetyFacts: [
+        "same_logical_run",
+        "no_false_success",
+        "bounded_provider_execution"
+      ]
+    },
+    verification: {
+      kind: "local_black_box",
+      target: "local-component-outage-recovery-e2e"
+    }
+  },
+  {
+    id: "local_runtime_process_outage",
+    description: "Interrupt both runtime components in the selected non-production topology.",
+    topologies: LOCAL_AND_PREVIEW_TOPOLOGIES,
+    control: { kind: "process", component: "all", modes: ["pause", "crash"] },
+    expected: {
+      outcome: "Durable work waits through the combined outage; it never becomes a false success or creates a second logical run.",
+      safetyFacts: ["same_logical_run", "no_false_success"]
+    },
+    verification: {
+      kind: "local_black_box",
+      target: "local-component-outage-recovery-e2e"
+    }
+  },
+  {
+    id: "receipt_complete_write_fail",
+    description: "Fail the next durable receipt-completion write.",
+    topologies: LOCAL_AND_PREVIEW_TOPOLOGIES,
+    control: {
+      kind: "runtime_test_fault",
+      defaultValue: 1,
+      valueMeaning: "occurrence"
+    },
+    expected: {
+      outcome: "The failed run is loud, and a later bounded recovery may repeat the affected provider operation if its prior completion was not durable.",
+      safetyFacts: ["retry_or_explicit_recovery", "bounded_provider_execution"]
+    },
+    verification: {
+      kind: "local_black_box",
+      target: "receipt-persist-failure-run-fatal-e2e"
+    }
+  },
+  {
+    id: "receipt_fail_write_fail",
+    description: "Fail the next durable receipt-failure write.",
+    topologies: LOCAL_AND_PREVIEW_TOPOLOGIES,
+    control: {
+      kind: "runtime_test_fault",
+      defaultValue: 1,
+      valueMeaning: "occurrence"
+    },
+    expected: {
+      outcome: "The failure is loud; recovery remains bounded and may repeat an operation whose prior outcome was not durably recorded.",
+      safetyFacts: ["retry_or_explicit_recovery", "bounded_provider_execution"]
+    },
+    verification: {
+      kind: "focused_regression",
+      target: "tests/lib/plays/runtime-test-fault-registry.test.ts"
+    }
+  },
+  {
+    id: "worker_receipt_complete_write_fail",
+    description: "Fail the worker-side receipt-completion write once.",
+    topologies: LOCAL_AND_PREVIEW_TOPOLOGIES,
+    control: {
+      kind: "runtime_test_fault",
+      defaultValue: 1,
+      valueMeaning: "occurrence"
+    },
+    expected: {
+      outcome: "The worker reports the persistence failure loudly; bounded recovery may repeat work whose completion was not durable.",
+      safetyFacts: ["retry_or_explicit_recovery", "bounded_provider_execution"]
+    },
+    verification: {
+      kind: "focused_regression",
+      target: "tests/lib/plays/runtime-test-fault-registry.test.ts"
+    }
+  },
+  {
+    id: "invocation_response_delivery_abort",
+    description: "Abort one receipt-gateway response after invocation delivery.",
+    topologies: LOCAL_AND_PREVIEW_TOPOLOGIES,
+    control: {
+      kind: "runtime_test_fault",
+      defaultValue: 1,
+      valueMeaning: "occurrence"
+    },
+    expected: {
+      outcome: "The same logical run completes through receipt replay; recovery remains bounded if the provider outcome must be retried.",
+      safetyFacts: ["same_logical_run", "bounded_provider_execution"]
+    },
+    verification: {
+      kind: "local_black_box",
+      target: "invocation-delivery-replay-e2e"
+    }
+  },
+  {
+    id: "receipt_gateway_hold_ms",
+    description: "Hold one checked-out receipt-gateway scheduler client.",
+    topologies: LOCAL_AND_PREVIEW_TOPOLOGIES,
+    control: {
+      kind: "runtime_test_fault",
+      defaultValue: 500,
+      valueMeaning: "milliseconds"
+    },
+    expected: {
+      outcome: "The request is delayed inside the real gateway pool and either completes within its deadline or follows bounded transport recovery.",
+      safetyFacts: ["bounded_provider_execution"]
+    },
+    verification: {
+      kind: "local_black_box",
+      target: "agent-heavy-gateway-load-e2e"
+    }
+  },
+  {
+    id: "receipt_claim_query_hold_once_ms",
+    description: "Hold one physical receipt-claim query.",
+    topologies: LOCAL_AND_PREVIEW_TOPOLOGIES,
+    control: {
+      kind: "runtime_test_fault",
+      defaultValue: 500,
+      valueMeaning: "milliseconds"
+    },
+    expected: {
+      outcome: "The checked-out claim either completes or times out through bounded recovery; an ambiguous outcome may be retried within its budget.",
+      safetyFacts: ["bounded_provider_execution"]
+    },
+    verification: {
+      kind: "local_black_box",
+      target: "receipt-claim-deadline-recovery-e2e"
+    }
+  },
+  {
+    id: "receipt_claim_response_timeout",
+    description: "Report one fully received receipt-claim response as timed out.",
+    topologies: LOCAL_AND_PREVIEW_TOPOLOGIES,
+    control: {
+      kind: "runtime_test_fault",
+      defaultValue: 1,
+      valueMeaning: "occurrence"
+    },
+    expected: {
+      outcome: "The same logical run completes by replaying the committed receipt response or uses bounded recovery if that response is unavailable.",
+      safetyFacts: ["same_logical_run", "bounded_provider_execution"]
+    },
+    verification: {
+      kind: "local_black_box",
+      target: "receipt-ambiguous-response-recovery-e2e"
+    }
+  },
+  {
+    id: "modal_sandbox_create_resource_exhausted",
+    description: "Make Modal sandbox creation return RESOURCE_EXHAUSTED before user code.",
+    topologies: MODAL_PREVIEW_TOPOLOGY,
+    control: {
+      kind: "runtime_test_fault",
+      defaultValue: 1,
+      valueMeaning: "occurrence"
+    },
+    expected: {
+      outcome: "The same logical run enters bounded pre-code defer/recovery and later retries sandbox startup; no user code or provider call precedes the retry.",
+      safetyFacts: [
+        "same_logical_run",
+        "no_user_code_before_recovery",
+        "bounded_provider_execution"
+      ]
+    },
+    verification: {
+      kind: "preview_black_box",
+      target: "modal-sandbox-recovery-e2e"
+    }
+  },
+  {
+    id: "runtime_sheet_page_tail_hold_ms",
+    description: "Hold one runtime-sheet tail-page operation.",
+    topologies: LOCAL_TOPOLOGY,
+    control: {
+      kind: "runtime_test_fault",
+      defaultValue: 500,
+      valueMeaning: "milliseconds"
+    },
+    expected: {
+      outcome: "Page-tail work stays pending during the bounded hold, then completes without losing or duplicating rows.",
+      safetyFacts: ["all_rows_settle_once"]
+    },
+    verification: {
+      kind: "local_black_box",
+      target: "page-tail-sheet-write-hold-e2e"
+    }
+  }
+];
+function isRuntimeTestFaultDrill(drill) {
+  return drill.control.kind === "runtime_test_fault" && RUNTIME_TEST_FAULT_IDS.some((faultId) => faultId === drill.id);
+}
+function runtimeTestFaultDrills() {
+  return RUNTIME_INCIDENT_DRILL_CATALOG.filter(isRuntimeTestFaultDrill);
+}
+
 // ../shared_libs/play-runtime/test-runtime-seams.ts
 var PLAY_RUNTIME_TEST_FAULT_HEADER = "x-deepline-test-fault";
+var SUPPORTED_RUNTIME_TEST_FAULTS = new Set(
+  runtimeTestFaultDrills().map(({ id }) => id)
+);
 var MAX_RUNTIME_TEST_POLICY_MS = 10 * 6e4;
 
 // src/http.ts

package/dist/install-integrity.json

@@ -100,6 +100,7 @@
     "dist/bundling-sources/shared_libs/play-runtime/play-input.ts",
     "dist/bundling-sources/shared_libs/play-runtime/play-latency-trace.ts",
     "dist/bundling-sources/shared_libs/play-runtime/play-node-scope.ts",
+    "dist/bundling-sources/shared_libs/play-runtime/play-run-recovery-policy.ts",
     "dist/bundling-sources/shared_libs/play-runtime/play-runtime-batching-registry.ts",
     "dist/bundling-sources/shared_libs/play-runtime/postgres-json.ts",
     "dist/bundling-sources/shared_libs/play-runtime/profiles.ts",
@@ -143,12 +144,14 @@
     "dist/bundling-sources/shared_libs/play-runtime/runtime-constraints.ts",
     "dist/bundling-sources/shared_libs/play-runtime/runtime-contract.ts",
     "dist/bundling-sources/shared_libs/play-runtime/runtime-environment.ts",
+    "dist/bundling-sources/shared_libs/play-runtime/runtime-incident-drills.ts",
     "dist/bundling-sources/shared_libs/play-runtime/runtime-infra-mode.ts",
     "dist/bundling-sources/shared_libs/play-runtime/runtime-pg-driver-pg.ts",
     "dist/bundling-sources/shared_libs/play-runtime/runtime-pg-driver.ts",
     "dist/bundling-sources/shared_libs/play-runtime/runtime-postgres-admission.ts",
     "dist/bundling-sources/shared_libs/play-runtime/runtime-receipt-store-adapter.ts",
     "dist/bundling-sources/shared_libs/play-runtime/runtime-receipt-writer.ts",
+    "dist/bundling-sources/shared_libs/play-runtime/runtime-reliability-policy.ts",
     "dist/bundling-sources/shared_libs/play-runtime/runtime-sandbox-placement-policy.ts",
     "dist/bundling-sources/shared_libs/play-runtime/runtime-scheduler-topology.ts",
     "dist/bundling-sources/shared_libs/play-runtime/runtime-sheet-attempt-state-machine.ts",
@@ -156,6 +159,7 @@
     "dist/bundling-sources/shared_libs/play-runtime/runtime-sheet-row-transition.ts",
     "dist/bundling-sources/shared_libs/play-runtime/runtime-sheet-row-writer.ts",
     "dist/bundling-sources/shared_libs/play-runtime/runtime-sheet-session.ts",
+    "dist/bundling-sources/shared_libs/play-runtime/runtime-traffic-policy.ts",
     "dist/bundling-sources/shared_libs/play-runtime/sandbox-compute-usage.ts",
     "dist/bundling-sources/shared_libs/play-runtime/sandbox-runtime-limits.ts",
     "dist/bundling-sources/shared_libs/play-runtime/scheduler-backend.ts",

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "deepline",
-  "version": "0.3.17",
+  "version": "0.3.18",
   "description": "GTM data CLI and TypeScript SDK for coding agents",
   "license": "MIT",
   "homepage": "https://code.deepline.com",