deepline 0.3.89 → 0.3.91

package/dist/bundling-sources/shared_libs/play-runtime/deployment/fly-registry.ts

@@ -0,0 +1,21 @@
+import { spawnSync } from 'node:child_process';
+
+function run(command: string, args: string[], quiet = false): boolean {
+  const result = spawnSync(command, args, {
+    encoding: 'utf8',
+    stdio: quiet ? 'pipe' : 'inherit',
+  });
+  return result.status === 0;
+}
+
+/** Shared immutable-tag presence probe for image builds and rollback planning. */
+export function runtimeImageManifestExists(image: string): boolean {
+  return run('docker', ['manifest', 'inspect', image], true);
+}
+
+/** Authenticate the local docker client against the Fly registry. */
+export function authenticateFlyRegistry(): void {
+  if (!run('flyctl', ['auth', 'docker'])) {
+    throw new Error('Fly registry authentication failed.');
+  }
+}

package/dist/bundling-sources/shared_libs/play-runtime/deployment/process-tree.ts

@@ -0,0 +1,40 @@
+import type { ChildProcess } from 'node:child_process';
+
+type CancellableChildProcess = Pick<
+  ChildProcess,
+  'exitCode' | 'kill' | 'pid' | 'signalCode'
+>;
+
+/**
+ * On POSIX, a detached child leads a new process group. Signalling its
+ * negative pid reaches the command and everything it started (notably the
+ * synchronous `flyctl` calls inside Bun scripts). That prevents recovery from
+ * racing an orphaned provider mutation after GitHub cancels the parent job.
+ */
+export function terminateChildProcessTree(
+  child: CancellableChildProcess,
+  signal: NodeJS.Signals,
+): void {
+  if (child.exitCode !== null || child.signalCode !== null) return;
+
+  if (process.platform !== 'win32' && child.pid !== undefined) {
+    try {
+      process.kill(-child.pid, signal);
+      return;
+    } catch (error) {
+      if (error instanceof Error && 'code' in error && error.code === 'ESRCH') {
+        return;
+      }
+      throw new Error(
+        `Could not send ${signal} to process group ${child.pid}: ${error instanceof Error ? error.message : String(error)}`,
+        { cause: error },
+      );
+    }
+  }
+
+  child.kill(signal);
+}
+
+export function shouldCreateDetachedProcessGroup(): boolean {
+  return process.platform !== 'win32';
+}

package/dist/bundling-sources/shared_libs/play-runtime/deployment/runtime-gateway-drain.ts

@@ -0,0 +1,262 @@
+/**
+ * Exact-machine gateway drain primitives.
+ *
+ * A Fly cordon is the admission boundary.  It prevents new public ingress but
+ * intentionally does not terminate an accepted request.  This module makes
+ * deletion depend on the old Machine's own live-work gauge instead of a sleep
+ * or the deploy runner still being alive.
+ */
+import { FlyMachinesClient } from './fly-machines';
+import type { FlyMachine } from './fly-machine-lifecycle';
+
+export const GATEWAY_REQUESTS_ACTIVE_METRIC =
+  'deepline_runtime_gateway_requests_active';
+export const GATEWAY_PRIVATE_METRICS_RETRY_DELAYS_MS = [250, 750] as const;
+
+/**
+ * The release scaler runs inside Fly's private network and addresses a
+ * draining Machine directly over 6PN.
+ *
+ * Do not "simplify" this to a public `fly-force-instance-id` probe. A drain
+ * proof must read the gauge on one exact Machine with no possibility of
+ * another Machine answering, and 6PN is the only addressing that cannot be
+ * re-routed by the proxy. (An earlier comment here claimed a cordoned Machine
+ * is unreachable via `fly-force-instance-id`; the receipt gateway proves the
+ * opposite on every release, so that is not the reason.)
+ */
+export function gatewayPrivateMetricsUrl(input: {
+  app: string;
+  machineId: string;
+}): string {
+  return `http://${input.machineId}.vm.${input.app}.internal:8789/metrics`;
+}
+
+export function parseGatewayRequestsActiveGauge(metrics: string): number {
+  const match = metrics.match(
+    new RegExp(`^${GATEWAY_REQUESTS_ACTIVE_METRIC}\\s+(\\d+)\\s*$`, 'm'),
+  );
+  if (!match) {
+    throw new Error(
+      'Gateway drain proof is missing the live-requests-active metric. This Machine predates the drain contract and must not be deleted by the generic controller.',
+    );
+  }
+  const value = Number(match[1]);
+  if (!Number.isSafeInteger(value) || value < 0) {
+    throw new Error(
+      'Gateway drain proof reported an invalid live request count.',
+    );
+  }
+  return value;
+}
+
+export type GatewayDrainMachine = Pick<FlyMachine, 'id' | 'state' | 'cordoned'>;
+
+function isRetryablePrivateMetricsProbeError(error: unknown): boolean {
+  if (error instanceof DOMException && error.name === 'AbortError') return true;
+  const message = error instanceof Error ? error.message : String(error);
+  return /unable to connect|fetch failed|econnreset|econnrefused|etimedout|network|socket/i.test(
+    message,
+  );
+}
+
+async function readPrivateGatewayMetrics(input: {
+  app: string;
+  machineId: string;
+  fetchImpl: typeof fetch;
+  retryDelaysMs?: readonly number[];
+  sleep?: (delayMs: number) => Promise<void>;
+}): Promise<string> {
+  const retryDelaysMs =
+    input.retryDelaysMs ?? GATEWAY_PRIVATE_METRICS_RETRY_DELAYS_MS;
+  const sleep =
+    input.sleep ??
+    ((delayMs: number) =>
+      new Promise<void>((resolve) => setTimeout(resolve, delayMs)));
+  let lastError: unknown;
+
+  for (let attempt = 0; attempt <= retryDelaysMs.length; attempt += 1) {
+    try {
+      const response = await input.fetchImpl(
+        gatewayPrivateMetricsUrl({
+          app: input.app,
+          machineId: input.machineId,
+        }),
+        { signal: AbortSignal.timeout(10_000) },
+      );
+      if (!response.ok) {
+        throw new Error(
+          `Gateway drain metrics probe for ${input.machineId} failed with HTTP ${response.status}.`,
+        );
+      }
+      if (
+        response.headers.get('x-deepline-gateway-machine-id') !==
+        input.machineId
+      ) {
+        throw new Error(
+          `Gateway drain metrics probe did not reach exact Machine ${input.machineId}.`,
+        );
+      }
+      return await response.text();
+    } catch (error) {
+      lastError = error;
+      if (
+        attempt === retryDelaysMs.length ||
+        !isRetryablePrivateMetricsProbeError(error)
+      ) {
+        throw error;
+      }
+      await sleep(retryDelaysMs[attempt]!);
+    }
+  }
+
+  throw lastError;
+}
+
+/**
+ * Return only source Machines which are safely proven empty.  An absent source
+ * is already provider-confirmed gone.  A source that is still public is a
+ * sequencing fault, never an excuse to cordon/delete it implicitly.
+ */
+export async function selectDrainedGatewaySourceMachines(input: {
+  sourceMachineIds: readonly string[];
+  inventory: readonly GatewayDrainMachine[];
+  readMetrics: (machineId: string) => Promise<string>;
+}): Promise<string[]> {
+  const uniqueSourceIds = [...new Set(input.sourceMachineIds)];
+  if (uniqueSourceIds.length !== input.sourceMachineIds.length) {
+    throw new Error('Gateway drain source identities must be unique.');
+  }
+  const live = uniqueSourceIds
+    .map((id) => input.inventory.find((machine) => machine.id === id))
+    .filter((machine): machine is GatewayDrainMachine => Boolean(machine));
+  if (live.some((machine) => machine.cordoned !== true)) {
+    throw new Error(
+      'Gateway drain refuses a source Machine that is still public; cut over and cordon it first.',
+    );
+  }
+  // A provider-confirmed stopped/suspended Machine cannot retain a live
+  // request.  Treat it as a deletion-resume candidate without probing its now
+  // unreachable metrics endpoint; otherwise an interruption between stop and
+  // delete would strand the exact source forever.
+  const alreadyStopped = live
+    .filter((machine) => ['stopped', 'suspended'].includes(machine.state ?? ''))
+    .map((machine) => machine.id!);
+  const observed: string[] = [];
+  for (const machine of live.filter(
+    (entry) => !['stopped', 'suspended'].includes(entry.state ?? ''),
+  )) {
+    if (
+      parseGatewayRequestsActiveGauge(await input.readMetrics(machine.id!)) ===
+      0
+    ) {
+      observed.push(machine.id!);
+    }
+  }
+  return [...alreadyStopped, ...observed];
+}
+
+/** One bounded reconciliation pass.  Nonzero work is normal and is retained
+ * for the next controller tick; metric/identity errors are intentionally loud
+ * so no release can quietly fall back to a timeout-based delete. */
+export async function reconcileGatewaySourceDrain(input: {
+  app: string;
+  client: FlyMachinesClient;
+  sourceMachineIds: readonly string[];
+  fetchImpl?: typeof fetch;
+  inventory?: readonly FlyMachine[];
+  retryDelaysMs?: readonly number[];
+  sleep?: (delayMs: number) => Promise<void>;
+}): Promise<{ deletedMachineIds: string[]; pendingMachineIds: string[] }> {
+  const inventory = [
+    ...(input.inventory ?? (await input.client.listMachines())),
+  ];
+  const sourceIds = [...new Set(input.sourceMachineIds)];
+  const fetchImpl = input.fetchImpl ?? fetch;
+  const deletedMachineIds = await selectDrainedGatewaySourceMachines({
+    sourceMachineIds: sourceIds,
+    inventory,
+    readMetrics: async (machineId) => {
+      return await readPrivateGatewayMetrics({
+        app: input.app,
+        machineId,
+        fetchImpl,
+        retryDelaysMs: input.retryDelaysMs,
+        sleep: input.sleep,
+      });
+    },
+  });
+  if (deletedMachineIds.length > 0) {
+    // Fly's client is asynchronous, so keep the provider inventory fresh
+    // between stop and delete rather than pretending the pre-stop snapshot is
+    // authoritative.  Deletion is still exact-ID and provider-confirmed.
+    await Promise.all(
+      deletedMachineIds
+        .filter((machineId) => {
+          const machine = inventory.find((entry) => entry.id === machineId);
+          return !['stopped', 'suspended'].includes(machine?.state ?? '');
+        })
+        .map((machineId) =>
+          input.client.stopMachine(machineId, {
+            signal: 'SIGTERM',
+            timeout: '300s',
+          }),
+        ),
+    );
+    const deadline = Date.now() + 10 * 60_000;
+    for (;;) {
+      const current = await input.client.listMachines();
+      const remaining = current.filter((machine) =>
+        deletedMachineIds.includes(machine.id ?? ''),
+      );
+      if (
+        remaining.every((machine) =>
+          ['stopped', 'suspended'].includes(machine.state ?? ''),
+        )
+      ) {
+        if (remaining.some((machine) => machine.cordoned !== true)) {
+          throw new Error(
+            'Gateway drain source Machine became public before deletion.',
+          );
+        }
+        await Promise.all(
+          remaining.map((machine) =>
+            input.client.deleteMachine(machine.id!, false),
+          ),
+        );
+        break;
+      }
+      if (Date.now() >= deadline) {
+        throw new Error(
+          `Gateway drain sources did not stop within 600000 ms: ${remaining.map((machine) => `${machine.id}=${machine.state}`).join(', ')}.`,
+        );
+      }
+      await new Promise((resolve) => setTimeout(resolve, 5_000));
+    }
+    for (;;) {
+      const providerMachineIds = new Set(
+        (await input.client.listMachines())
+          .map((machine) => machine.id)
+          .filter((id): id is string => Boolean(id)),
+      );
+      if (
+        deletedMachineIds.every(
+          (machineId) => !providerMachineIds.has(machineId),
+        )
+      ) {
+        break;
+      }
+      if (Date.now() >= deadline) {
+        throw new Error('Gateway source deletion is not provider-confirmed.');
+      }
+      await new Promise((resolve) => setTimeout(resolve, 5_000));
+    }
+  }
+  return {
+    deletedMachineIds,
+    pendingMachineIds: sourceIds.filter(
+      (id) =>
+        inventory.some((machine) => machine.id === id) &&
+        !deletedMachineIds.includes(id),
+    ),
+  };
+}

package/dist/bundling-sources/shared_libs/play-runtime/deployment/runtime-surface.ts

@@ -0,0 +1,98 @@
+import { createHash } from 'node:crypto';
+import { existsSync, readdirSync, readFileSync, statSync } from 'node:fs';
+import { join, relative } from 'node:path';
+
+/**
+ * A deliberately broad, content-addressed definition of code that can affect
+ * the Vercel ↔ Fly runtime boundary. This is release evidence and an exact
+ * cleanup selector, not a versioned customer contract: changing any input
+ * automatically changes the fingerprint, so there is no edition number to
+ * remember to bump.
+ *
+ * Every production revision builds an immutable lane. Keep this broad so the
+ * evidence continues to describe all code material available to that worker.
+ */
+export const RUNTIME_SURFACE_INPUTS = [
+  'packages/integrations/bettercontact/BetterContactOpenAPI.json',
+  // Keep these in step with the runtime-worker Dockerfile COPY inputs. The
+  // worker does not necessarily import every one of these files today, but
+  // hashing the material available to it makes omitted future imports safe.
+  '.dockerignore',
+  'apps/deepline-api/package.json',
+  'apps/deepline-api/src/lib',
+  'apps/play-runner',
+  'apps/absurd-runtime-worker',
+  'apps/absurd-release-scaler',
+  'apps/ledger-outbox-projector',
+  'apps/runtime-execution-relay',
+  'apps/runtime-receipt-gateway',
+  'apps/ojos/scripts/build-usage-catalog.ts',
+  'bunfig.toml',
+  'contracts',
+  'deploy',
+  'packages',
+  'patches',
+  'scripts',
+  'shared_libs',
+  'apps/slack-agent/package.json',
+  'apps/deepline-api/src/app/api',
+  // Includes scheduler, billing, integration, PlanetScale/Neon, and runtime
+  // code without coupling the gate to a hand-maintained subdirectory list.
+  'src/lib',
+  '.github/workflows/fly-absurd-worker.yml',
+  'bun.lock',
+  'package.json',
+  'tsconfig.apps.play-runner.json',
+  'tsconfig.json',
+  'turbo.json',
+  'vercel.sh',
+] as const;
+
+const RUNTIME_SURFACE_EXCLUDED_DIRECTORY_NAMES = new Set(['node_modules']);
+
+function collectFiles(root: string, path: string, files: string[]): void {
+  const absolutePath = join(root, path);
+  if (!existsSync(absolutePath)) return;
+  if (statSync(absolutePath).isFile()) {
+    files.push(path);
+    return;
+  }
+  for (const entry of readdirSync(absolutePath, { withFileTypes: true }).sort(
+    (left, right) => left.name.localeCompare(right.name),
+  )) {
+    // Dependency installation may materialize workspace-local node_modules
+    // after release planning. Fly excludes those directories from the Docker
+    // context, so including them here would make identical source revisions
+    // produce different planned and built fingerprints.
+    if (
+      entry.isDirectory() &&
+      RUNTIME_SURFACE_EXCLUDED_DIRECTORY_NAMES.has(entry.name)
+    ) {
+      continue;
+    }
+    const child = relative(root, join(absolutePath, entry.name));
+    if (entry.isDirectory()) collectFiles(root, child, files);
+    else if (entry.isFile()) files.push(child);
+  }
+}
+
+export function runtimeSurfaceFiles(root = process.cwd()): string[] {
+  const files: string[] = [];
+  for (const input of RUNTIME_SURFACE_INPUTS) collectFiles(root, input, files);
+  return [...new Set(files)].sort();
+}
+
+export function runtimeSurfaceFingerprint(root = process.cwd()): {
+  sha256: string;
+  files: string[];
+} {
+  const files = runtimeSurfaceFiles(root);
+  const hash = createHash('sha256');
+  for (const file of files) {
+    hash.update(file);
+    hash.update('\0');
+    hash.update(readFileSync(join(root, file)));
+    hash.update('\0');
+  }
+  return { sha256: hash.digest('hex'), files };
+}