deepline 0.3.16 → 0.3.18

package/dist/bundling-sources/shared_libs/play-runtime/runtime-incident-drills.ts

@@ -0,0 +1,378 @@
+/**
+ * Canonical incident-drill catalog for the Play Runtime.
+ *
+ * A drill is not a retry policy and it never changes a Play by itself. It
+ * states a bounded fault we can deliberately exercise, the topology where it
+ * is meaningful, and the durable safety facts an observer must verify. Test
+ * seams, the local admin CLI, V2 scenarios, and documentation are Adapters to
+ * this Module; they must not invent their own outcome descriptions.
+ */
+
+export const RUNTIME_TEST_FAULT_IDS = [
+  'receipt_complete_write_fail',
+  'receipt_fail_write_fail',
+  'worker_receipt_complete_write_fail',
+  'invocation_response_delivery_abort',
+  'receipt_gateway_hold_ms',
+  'receipt_claim_query_hold_once_ms',
+  'receipt_claim_response_timeout',
+  'modal_sandbox_create_resource_exhausted',
+  'runtime_sheet_page_tail_hold_ms',
+] as const;
+
+export type RuntimeTestFaultName = (typeof RUNTIME_TEST_FAULT_IDS)[number];
+
+export type RuntimeIncidentDrillId =
+  | 'local_worker_process_outage'
+  | 'local_gateway_process_outage'
+  | 'local_runtime_process_outage'
+  | RuntimeTestFaultName;
+
+export type RuntimeIncidentTopology =
+  | 'local_daytona'
+  | 'preview_fly'
+  | 'preview_direct_modal';
+
+export type RuntimeIncidentSafetyFact =
+  | 'same_logical_run'
+  | 'no_false_success'
+  | 'bounded_provider_execution'
+  | 'retry_or_explicit_recovery'
+  | 'no_user_code_before_recovery'
+  | 'all_rows_settle_once';
+
+/**
+ * The automated evidence that proves this drill.  A focused regression owns a
+ * narrow storage/protocol invariant; a black-box scenario drives the full
+ * local (or Preview) runtime.  Keeping this on the catalog means on-call and
+ * CI see the same answer to “how do we prove this?” as the injector.
+ */
+export type RuntimeIncidentVerification = Readonly<{
+  kind: 'focused_regression' | 'local_black_box' | 'preview_black_box';
+  target: string;
+}>;
+
+export type RuntimeIncidentDrill = Readonly<{
+  id: RuntimeIncidentDrillId;
+  description: string;
+  topologies: readonly RuntimeIncidentTopology[];
+  control:
+    | Readonly<{
+        kind: 'process';
+        component: 'worker' | 'gateway' | 'all';
+        modes: readonly ('pause' | 'crash')[];
+      }>
+    | Readonly<{
+        kind: 'runtime_test_fault';
+        defaultValue: number;
+        valueMeaning: 'occurrence' | 'milliseconds';
+      }>;
+  expected: Readonly<{
+    outcome: string;
+    safetyFacts: readonly RuntimeIncidentSafetyFact[];
+  }>;
+  verification: RuntimeIncidentVerification;
+}>;
+
+const LOCAL_TOPOLOGY = ['local_daytona'] as const;
+const LOCAL_AND_PREVIEW_TOPOLOGIES = [
+  'local_daytona',
+  'preview_fly',
+  'preview_direct_modal',
+] as const;
+const MODAL_PREVIEW_TOPOLOGY = ['preview_direct_modal'] as const;
+
+/**
+ * The single catalog that binds an injected fault to its required proof.
+ *
+ * `valueMeaning` is deliberately explicit: callers cannot accidentally pass
+ * a millisecond delay where the runtime interprets an occurrence count.
+ */
+export const RUNTIME_INCIDENT_DRILL_CATALOG: readonly RuntimeIncidentDrill[] = [
+  {
+    id: 'local_worker_process_outage',
+    description:
+      'Interrupt the scheduler worker in the selected non-production runtime topology.',
+    topologies: LOCAL_AND_PREVIEW_TOPOLOGIES,
+    control: {
+      kind: 'process',
+      component: 'worker',
+      modes: ['pause', 'crash'],
+    },
+    expected: {
+      outcome:
+        'The same logical run stays durable through the worker outage and completes after the selected topology recovers.',
+      safetyFacts: ['same_logical_run', 'no_false_success'],
+    },
+    verification: {
+      kind: 'local_black_box',
+      target: 'local-component-outage-recovery-e2e',
+    },
+  },
+  {
+    id: 'local_gateway_process_outage',
+    description:
+      'Interrupt the receipt gateway in the selected non-production runtime topology.',
+    topologies: LOCAL_AND_PREVIEW_TOPOLOGIES,
+    control: {
+      kind: 'process',
+      component: 'gateway',
+      modes: ['pause', 'crash'],
+    },
+    expected: {
+      outcome:
+        'In-flight work recovers only at a safe receipt boundary and the same logical run completes after the gateway recovers.',
+      safetyFacts: [
+        'same_logical_run',
+        'no_false_success',
+        'bounded_provider_execution',
+      ],
+    },
+    verification: {
+      kind: 'local_black_box',
+      target: 'local-component-outage-recovery-e2e',
+    },
+  },
+  {
+    id: 'local_runtime_process_outage',
+    description:
+      'Interrupt both runtime components in the selected non-production topology.',
+    topologies: LOCAL_AND_PREVIEW_TOPOLOGIES,
+    control: { kind: 'process', component: 'all', modes: ['pause', 'crash'] },
+    expected: {
+      outcome:
+        'Durable work waits through the combined outage; it never becomes a false success or creates a second logical run.',
+      safetyFacts: ['same_logical_run', 'no_false_success'],
+    },
+    verification: {
+      kind: 'local_black_box',
+      target: 'local-component-outage-recovery-e2e',
+    },
+  },
+  {
+    id: 'receipt_complete_write_fail',
+    description: 'Fail the next durable receipt-completion write.',
+    topologies: LOCAL_AND_PREVIEW_TOPOLOGIES,
+    control: {
+      kind: 'runtime_test_fault',
+      defaultValue: 1,
+      valueMeaning: 'occurrence',
+    },
+    expected: {
+      outcome:
+        'The failed run is loud, and a later bounded recovery may repeat the affected provider operation if its prior completion was not durable.',
+      safetyFacts: ['retry_or_explicit_recovery', 'bounded_provider_execution'],
+    },
+    verification: {
+      kind: 'local_black_box',
+      target: 'receipt-persist-failure-run-fatal-e2e',
+    },
+  },
+  {
+    id: 'receipt_fail_write_fail',
+    description: 'Fail the next durable receipt-failure write.',
+    topologies: LOCAL_AND_PREVIEW_TOPOLOGIES,
+    control: {
+      kind: 'runtime_test_fault',
+      defaultValue: 1,
+      valueMeaning: 'occurrence',
+    },
+    expected: {
+      outcome:
+        'The failure is loud; recovery remains bounded and may repeat an operation whose prior outcome was not durably recorded.',
+      safetyFacts: ['retry_or_explicit_recovery', 'bounded_provider_execution'],
+    },
+    verification: {
+      kind: 'focused_regression',
+      target: 'tests/lib/plays/runtime-test-fault-registry.test.ts',
+    },
+  },
+  {
+    id: 'worker_receipt_complete_write_fail',
+    description: 'Fail the worker-side receipt-completion write once.',
+    topologies: LOCAL_AND_PREVIEW_TOPOLOGIES,
+    control: {
+      kind: 'runtime_test_fault',
+      defaultValue: 1,
+      valueMeaning: 'occurrence',
+    },
+    expected: {
+      outcome:
+        'The worker reports the persistence failure loudly; bounded recovery may repeat work whose completion was not durable.',
+      safetyFacts: ['retry_or_explicit_recovery', 'bounded_provider_execution'],
+    },
+    verification: {
+      kind: 'focused_regression',
+      target: 'tests/lib/plays/runtime-test-fault-registry.test.ts',
+    },
+  },
+  {
+    id: 'invocation_response_delivery_abort',
+    description:
+      'Abort one receipt-gateway response after invocation delivery.',
+    topologies: LOCAL_AND_PREVIEW_TOPOLOGIES,
+    control: {
+      kind: 'runtime_test_fault',
+      defaultValue: 1,
+      valueMeaning: 'occurrence',
+    },
+    expected: {
+      outcome:
+        'The same logical run completes through receipt replay; recovery remains bounded if the provider outcome must be retried.',
+      safetyFacts: ['same_logical_run', 'bounded_provider_execution'],
+    },
+    verification: {
+      kind: 'local_black_box',
+      target: 'invocation-delivery-replay-e2e',
+    },
+  },
+  {
+    id: 'receipt_gateway_hold_ms',
+    description: 'Hold one checked-out receipt-gateway scheduler client.',
+    topologies: LOCAL_AND_PREVIEW_TOPOLOGIES,
+    control: {
+      kind: 'runtime_test_fault',
+      defaultValue: 500,
+      valueMeaning: 'milliseconds',
+    },
+    expected: {
+      outcome:
+        'The request is delayed inside the real gateway pool and either completes within its deadline or follows bounded transport recovery.',
+      safetyFacts: ['bounded_provider_execution'],
+    },
+    verification: {
+      kind: 'local_black_box',
+      target: 'agent-heavy-gateway-load-e2e',
+    },
+  },
+  {
+    id: 'receipt_claim_query_hold_once_ms',
+    description: 'Hold one physical receipt-claim query.',
+    topologies: LOCAL_AND_PREVIEW_TOPOLOGIES,
+    control: {
+      kind: 'runtime_test_fault',
+      defaultValue: 500,
+      valueMeaning: 'milliseconds',
+    },
+    expected: {
+      outcome:
+        'The checked-out claim either completes or times out through bounded recovery; an ambiguous outcome may be retried within its budget.',
+      safetyFacts: ['bounded_provider_execution'],
+    },
+    verification: {
+      kind: 'local_black_box',
+      target: 'receipt-claim-deadline-recovery-e2e',
+    },
+  },
+  {
+    id: 'receipt_claim_response_timeout',
+    description:
+      'Report one fully received receipt-claim response as timed out.',
+    topologies: LOCAL_AND_PREVIEW_TOPOLOGIES,
+    control: {
+      kind: 'runtime_test_fault',
+      defaultValue: 1,
+      valueMeaning: 'occurrence',
+    },
+    expected: {
+      outcome:
+        'The same logical run completes by replaying the committed receipt response or uses bounded recovery if that response is unavailable.',
+      safetyFacts: ['same_logical_run', 'bounded_provider_execution'],
+    },
+    verification: {
+      kind: 'local_black_box',
+      target: 'receipt-ambiguous-response-recovery-e2e',
+    },
+  },
+  {
+    id: 'modal_sandbox_create_resource_exhausted',
+    description:
+      'Make Modal sandbox creation return RESOURCE_EXHAUSTED before user code.',
+    topologies: MODAL_PREVIEW_TOPOLOGY,
+    control: {
+      kind: 'runtime_test_fault',
+      defaultValue: 1,
+      valueMeaning: 'occurrence',
+    },
+    expected: {
+      outcome:
+        'The same logical run enters bounded pre-code defer/recovery and later retries sandbox startup; no user code or provider call precedes the retry.',
+      safetyFacts: [
+        'same_logical_run',
+        'no_user_code_before_recovery',
+        'bounded_provider_execution',
+      ],
+    },
+    verification: {
+      kind: 'preview_black_box',
+      target: 'modal-sandbox-recovery-e2e',
+    },
+  },
+  {
+    id: 'runtime_sheet_page_tail_hold_ms',
+    description: 'Hold one runtime-sheet tail-page operation.',
+    topologies: LOCAL_TOPOLOGY,
+    control: {
+      kind: 'runtime_test_fault',
+      defaultValue: 500,
+      valueMeaning: 'milliseconds',
+    },
+    expected: {
+      outcome:
+        'Page-tail work stays pending during the bounded hold, then completes without losing or duplicating rows.',
+      safetyFacts: ['all_rows_settle_once'],
+    },
+    verification: {
+      kind: 'local_black_box',
+      target: 'page-tail-sheet-write-hold-e2e',
+    },
+  },
+];
+
+export type RuntimeTestFaultDrill = Omit<
+  RuntimeIncidentDrill,
+  'id' | 'control'
+> &
+  Readonly<{
+    id: RuntimeTestFaultName;
+    control: Readonly<{
+      kind: 'runtime_test_fault';
+      defaultValue: number;
+      valueMeaning: 'occurrence' | 'milliseconds';
+    }>;
+  }>;
+
+function isRuntimeTestFaultDrill(
+  drill: RuntimeIncidentDrill,
+): drill is RuntimeTestFaultDrill {
+  return (
+    drill.control.kind === 'runtime_test_fault' &&
+    RUNTIME_TEST_FAULT_IDS.some((faultId) => faultId === drill.id)
+  );
+}
+
+export function runtimeTestFaultDrills(): readonly RuntimeTestFaultDrill[] {
+  return RUNTIME_INCIDENT_DRILL_CATALOG.filter(isRuntimeTestFaultDrill);
+}
+
+export function findRuntimeIncidentDrill(
+  id: string,
+): RuntimeIncidentDrill | null {
+  return (
+    RUNTIME_INCIDENT_DRILL_CATALOG.find((drill) => drill.id === id) ?? null
+  );
+}
+
+export function findRuntimeTestFaultDrill(
+  id: string,
+): RuntimeTestFaultDrill | null {
+  const drill = findRuntimeIncidentDrill(id);
+  return drill && isRuntimeTestFaultDrill(drill) ? drill : null;
+}
+
+export function runtimeIncidentDrillSupportsTopology(
+  drill: RuntimeIncidentDrill,
+  topology: RuntimeIncidentTopology,
+): boolean {
+  return drill.topologies.includes(topology);
+}