deepline 0.3.90 → 0.3.91

package/dist/bundling-sources/shared_libs/play-runtime/deployment/runtime-topology.ts

@@ -0,0 +1,303 @@
+/**
+ * Declarative Fly runtime topology.
+ *
+ * This is the only place that defines the durable platform (app identities,
+ * ingress, Machine shapes, and release ownership). Release code selects an
+ * immutable image and replaces only the declared release unit; it does not
+ * invent an app, IP, queue, or controller relationship.
+ */
+export type RuntimeGatewayRole = 'receipt' | 'execution';
+export type RuntimeGatewayTopologyProfile = 'production' | 'preview';
+
+export type RuntimeGatewayMachineConfig = {
+  env: Record<string, string>;
+  guest: {
+    cpu_kind: 'shared' | 'performance';
+    cpus: number;
+    memory_mb: number;
+  };
+  restart: { policy: 'always' };
+  stop_config: { signal: 'SIGTERM'; timeout: string };
+  metrics: { port: number; path: '/metrics' };
+  services: Array<Record<string, unknown>>;
+};
+
+type GatewayTopology = {
+  app?: string;
+  region: 'iad';
+  minimumMachines: number;
+  processGroup: 'app';
+  machine: RuntimeGatewayMachineConfig;
+};
+
+type ProductionGatewayTopology = GatewayTopology & {
+  app: string;
+  machine: RuntimeGatewayMachineConfig & {
+    guest: { cpu_kind: 'performance'; cpus: number; memory_mb: number };
+  };
+};
+
+const httpService = (minimumMachines: number): Record<string, unknown> => ({
+  protocol: 'tcp',
+  internal_port: 8788,
+  autostop: false,
+  autostart: true,
+  min_machines_running: minimumMachines,
+  ports: [
+    { port: 80, handlers: ['http'], force_https: true },
+    { port: 443, handlers: ['http', 'tls'] },
+  ],
+  checks: [
+    {
+      type: 'http',
+      interval: '15s',
+      timeout: '5s',
+      grace_period: '15s',
+      // Fly owns liveness only. Version/schema compatibility is a deliberate
+      // cordoned-candidate proof, never a service-check side effect.
+      path: '/healthz',
+    },
+  ],
+  concurrency: { type: 'requests', soft_limit: 1, hard_limit: 64 },
+});
+
+function gatewayMachine(
+  profile: RuntimeGatewayTopologyProfile,
+  role: RuntimeGatewayRole,
+  minimumMachines: number,
+): RuntimeGatewayMachineConfig {
+  const production = profile === 'production';
+  return {
+    env: {
+      PORT: '8788',
+      GATEWAY_METRICS_PORT: '8789',
+      DEEPLINE_RUNTIME_GATEWAY_ROLE: role,
+      FLY_PROCESS_GROUP: 'app',
+      ...(production
+        ? { DEEPLINE_RUNTIME_JWKS_ORIGIN: 'https://code.deepline.com' }
+        : {
+            DEEPLINE_ENV: 'dev',
+            DEEPLINE_TEST_RUNTIME_SEAMS: '1',
+            DEEPLINE_RUNTIME_SCHEDULER_SCHEMA:
+              'runtime_scheduler_absurd_preview',
+          }),
+    },
+    guest: production
+      ? { cpu_kind: 'performance', cpus: 2, memory_mb: 4096 }
+      : { cpu_kind: 'shared', cpus: 1, memory_mb: 1024 },
+    restart: { policy: 'always' },
+    stop_config: {
+      signal: 'SIGTERM',
+      timeout: production ? '5m0s' : '30s',
+    },
+    metrics: { port: 8789, path: '/metrics' },
+    services: [httpService(minimumMachines)],
+  };
+}
+
+function productionGatewayMachine(
+  role: RuntimeGatewayRole,
+  minimumMachines: number,
+): ProductionGatewayTopology['machine'] {
+  return gatewayMachine(
+    'production',
+    role,
+    minimumMachines,
+  ) as ProductionGatewayTopology['machine'];
+}
+
+const productionGateways: Record<
+  RuntimeGatewayRole,
+  ProductionGatewayTopology
+> = {
+  receipt: {
+    app: 'deepline-runtime-receipt-gateway',
+    region: 'iad',
+    minimumMachines: 2,
+    processGroup: 'app',
+    machine: productionGatewayMachine('receipt', 2),
+  },
+  execution: {
+    app: 'deepline-runtime-execution-relay',
+    region: 'iad',
+    minimumMachines: 2,
+    processGroup: 'app',
+    machine: productionGatewayMachine('execution', 2),
+  },
+};
+
+const previewGateways: Record<RuntimeGatewayRole, GatewayTopology> = {
+  receipt: {
+    region: 'iad',
+    minimumMachines: 2,
+    processGroup: 'app',
+    machine: gatewayMachine('preview', 'receipt', 2),
+  },
+  execution: {
+    region: 'iad',
+    minimumMachines: 2,
+    processGroup: 'app',
+    machine: gatewayMachine('preview', 'execution', 2),
+  },
+};
+
+export const RUNTIME_TOPOLOGY = {
+  production: {
+    components: {
+      receiptGateway: {
+        ...productionGateways.receipt,
+        release: 'machine-set',
+        publicIngress: true,
+        image: {
+          kind: 'gateway',
+          config: 'deploy/fly/runtime-worker/receipt-gateway.toml',
+          dockerfile: 'deploy/fly/runtime-worker/receipt-gateway.Dockerfile',
+        },
+      },
+      executionRelay: {
+        ...productionGateways.execution,
+        release: 'machine-set',
+        publicIngress: true,
+        image: {
+          kind: 'gateway',
+          config: 'deploy/fly/runtime-worker/execution-relay.toml',
+          dockerfile: 'deploy/fly/runtime-worker/receipt-gateway.Dockerfile',
+        },
+      },
+      worker: {
+        app: 'deepline-absurd-worker-prod',
+        release: 'immutable-lane',
+        publicIngress: false,
+        image: {
+          kind: 'worker',
+          config: 'deploy/fly/runtime-worker/absurd-worker.toml',
+          dockerfile: 'deploy/fly/runtime-worker/Dockerfile.absurd',
+        },
+      },
+      scaler: {
+        app: 'deepline-absurd-worker-prod-scaler',
+        release: 'lease-holder',
+        publicIngress: false,
+        image: {
+          kind: 'scaler',
+          config: 'deploy/fly/runtime-worker/absurd-scaler.toml',
+          dockerfile: 'deploy/fly/runtime-worker/Dockerfile.absurd-scaler',
+        },
+        processGroups: {
+          app: { desiredStarted: 1, healthCheck: 'scaler_ready' },
+          projector: { desiredStarted: 2, healthCheck: 'projector_ready' },
+        },
+      },
+      workerLogShipper: {
+        app: 'deepline-runtime-log-shipper',
+        release: 'singleton',
+        publicIngress: false,
+        config: 'deploy/fly/runtime-log-shipper/fly.toml',
+      },
+      receiptGatewayLogShipper: {
+        app: 'deepline-runtime-receipt-gateway-log-shipper',
+        release: 'singleton',
+        publicIngress: false,
+        config: 'deploy/fly/runtime-log-shipper/receipt-gateway.fly.toml',
+      },
+      executionRelayLogShipper: {
+        app: 'deepline-runtime-execution-relay-log-shipper',
+        release: 'singleton',
+        publicIngress: false,
+        config: 'deploy/fly/runtime-log-shipper/execution-relay.fly.toml',
+      },
+    },
+  },
+  preview: { gateways: previewGateways },
+} as const;
+
+export type RuntimeComponent =
+  keyof typeof RUNTIME_TOPOLOGY.production.components;
+
+type ProductionComponent =
+  (typeof RUNTIME_TOPOLOGY.production.components)[RuntimeComponent];
+
+type SingletonComponent = ProductionComponent & {
+  release: 'singleton';
+  config: string;
+};
+
+export function productionRuntimeAppNames(): string[] {
+  return Object.values(RUNTIME_TOPOLOGY.production.components).map(
+    (component) => component.app,
+  );
+}
+
+/**
+ * Singleton components are the only fixed services that can be deployed as a
+ * group. Adding another declared singleton automatically adds its app shell
+ * and its no-public-IP deployment to topology reconciliation.
+ */
+export function productionSingletonComponents(): SingletonComponent[] {
+  return Object.values(RUNTIME_TOPOLOGY.production.components).filter(
+    (component): component is SingletonComponent =>
+      component.release === 'singleton' &&
+      'config' in component &&
+      typeof component.config === 'string',
+  );
+}
+
+export function assertRuntimeTopologyInvariants(): void {
+  const components = Object.values(
+    RUNTIME_TOPOLOGY.production.components,
+  ) as Array<{ app: string; release: string; publicIngress: boolean }>;
+  const apps = components.map((component) => component.app);
+  if (new Set(apps).size !== apps.length) {
+    throw new Error(
+      'Every production runtime component requires a unique app.',
+    );
+  }
+  for (const component of components) {
+    if (component.release === 'machine-set' && !component.publicIngress) {
+      throw new Error(
+        `Machine-set component ${component.app} must declare public ingress.`,
+      );
+    }
+    if (component.release === 'immutable-lane' && component.publicIngress) {
+      throw new Error(
+        `Immutable-lane component ${component.app} cannot declare public ingress.`,
+      );
+    }
+  }
+}
+
+assertRuntimeTopologyInvariants();
+
+export function runtimeGatewayTopology(
+  profile: RuntimeGatewayTopologyProfile,
+  role: RuntimeGatewayRole,
+): GatewayTopology {
+  if (profile === 'production') return productionGateways[role];
+  return previewGateways[role];
+}
+
+export function productionGatewayTopology(
+  role: RuntimeGatewayRole,
+): ProductionGatewayTopology {
+  return productionGateways[role];
+}
+
+export function gatewayMachineConfig(input: {
+  profile: RuntimeGatewayTopologyProfile;
+  role: RuntimeGatewayRole;
+  image: string;
+  metadata: Record<string, string>;
+  env?: Record<string, string>;
+}): Record<string, unknown> {
+  const machine = runtimeGatewayTopology(input.profile, input.role).machine;
+  return {
+    image: input.image,
+    env: { ...machine.env, ...input.env },
+    guest: machine.guest,
+    restart: machine.restart,
+    stop_config: machine.stop_config,
+    metrics: machine.metrics,
+    services: machine.services,
+    metadata: input.metadata,
+  };
+}

package/dist/bundling-sources/shared_libs/play-runtime/deployment/runtime-tuple-publication-guardian.ts

@@ -0,0 +1,70 @@
+import { spawn } from 'node:child_process';
+import {
+  createRuntimeTuplePanicRecovery,
+  type RuntimeTupleRecoveryCommand,
+} from './runtime-tuple-recovery';
+
+function required(name: string): string {
+  const value = process.env[name]?.trim();
+  if (!value) {
+    throw new Error(`${name} is required for runtime tuple recovery.`);
+  }
+  return value;
+}
+
+function command(
+  input: Parameters<RuntimeTupleRecoveryCommand>[0],
+): Promise<void> {
+  return new Promise((resolve, reject) => {
+    const child = spawn(input.executable, [...input.args], {
+      env: { ...process.env, ...input.environment },
+      stdio: 'inherit',
+    });
+    child.once('error', reject);
+    child.once('close', (code, signal) => {
+      if (code === 0) {
+        resolve();
+        return;
+      }
+      reject(
+        new Error(
+          signal
+            ? `${input.description} was terminated by ${signal}.`
+            : `${input.description} failed with exit code ${code ?? '<unknown>'}.`,
+        ),
+      );
+    });
+  });
+}
+
+function recovery() {
+  return createRuntimeTuplePanicRecovery({
+    controlUrl: required('RUNTIME_TUPLE_CONTROL_URL'),
+    internalToken: required('DEEPLINE_INTERNAL_TOKEN'),
+    vercelProtectionBypassToken: process.env.VERCEL_PROTECTION_BYPASS_TOKEN,
+    vercel: {
+      token: required('VERCEL_TOKEN'),
+      teamId: required('VERCEL_TEAM_ID'),
+      projectId: required('VERCEL_PROJECT_ID'),
+    },
+    gatewayFlyToken: required('RUNTIME_GATEWAY_FAS_API_TOKEN'),
+    command,
+    log: (event, fields) =>
+      console.info(`[runtime-tuple-guardian] ${event}`, fields),
+  });
+}
+
+export async function activeRuntimeTuplePublication() {
+  return await recovery().activePublication();
+}
+
+/**
+ * One idempotent convergence pass. The scaler invokes this on every tick; a
+ * provider outage leaves the durable operation in panic and the next tick
+ * resumes from the same baseline rather than declaring recovery failed.
+ */
+export async function reconcileRuntimeTuplePublicationPanic(input: {
+  owner: string;
+}): Promise<boolean> {
+  return await recovery().reconcile(input);
+}