deepline 0.3.128 → 0.3.130

package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/run-terminal-transition.ts

@@ -33,6 +33,7 @@ export async function acceptRunnerTerminal(
     attempt: number;
     authority?: RunnerTerminalAttemptAuthority | null;
     result: unknown;
+    terminalProgressEvents?: readonly unknown[];
   },
   options?: { schema?: string },
 ): Promise<RunnerTerminalIngressOutcome> {

package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/runtime-receipt-value-storage.ts

@@ -0,0 +1,371 @@
+import { createHash } from 'node:crypto';
+import {
+  normalizeRuntimeReceiptValueInlineFields,
+  RuntimeReceiptValueReadError,
+  type RuntimeReceiptValueInlineFields,
+  type RuntimeReceiptValueReference,
+} from '../runtime-receipt-value';
+import { BoundedBatchFunction } from '../bounded-batch-function';
+
+/**
+ * Keep small receipts in the same Postgres row. Larger JSON bodies move once
+ * to the receipt-owned object namespace, so the row and gateway batch keep a
+ * compact manifest. The boundary is deliberately an injected policy in tests.
+ */
+export const RUNTIME_RECEIPT_VALUE_INLINE_MAX_BYTES = 256 * 1024;
+
+/** Shared object-store surface for all immutable runtime JSON values. */
+export type RuntimeValueBlobStore = {
+  exists(key: string): Promise<boolean>;
+  put(input: { key: string; body: Buffer; contentType: string }): Promise<void>;
+  get(key: string): Promise<Buffer>;
+  remove(key: string): Promise<void>;
+};
+
+/** Backward-compatible receipt-specific name for the shared object store. */
+export type RuntimeReceiptValueBlobStore = RuntimeValueBlobStore;
+
+/**
+ * Foundation owns value references and bounded I/O admission; the deployment
+ * injects its object-store adapter. This keeps R2 credentials and transport
+ * mechanics at the gateway edge, without a reverse package dependency.
+ */
+export function requireRuntimeValueBlobStore(
+  store: RuntimeValueBlobStore | undefined,
+): RuntimeValueBlobStore {
+  if (!store) {
+    throw new Error(
+      'Runtime value storage requires an explicit object-store adapter.',
+    );
+  }
+  return store;
+}
+
+type RuntimeReceiptValueStorageInput = {
+  orgId: string;
+  receiptKey: string;
+  serialized: string;
+  inlineFields?: RuntimeReceiptValueInlineFields;
+  inlineMaxBytes?: number;
+  store?: RuntimeReceiptValueBlobStore;
+};
+
+export type PersistedRuntimeReceiptValue =
+  | { kind: 'inline'; outputJson: string }
+  | {
+      kind: 'reference';
+      outputJson: null;
+      outputRef: RuntimeReceiptValueReference;
+    };
+
+export type RuntimeReceiptValuePersister = (input: {
+  orgId: string;
+  receiptKey: string;
+  serialized: string;
+  inlineFields?: RuntimeReceiptValueInlineFields;
+}) => Promise<PersistedRuntimeReceiptValue>;
+
+export type RuntimeReceiptValueReader = (input: {
+  orgId: string;
+  receiptKey: string;
+  reference: RuntimeReceiptValueReference;
+}) => Promise<unknown>;
+
+function sha256(value: string | Buffer): string {
+  return createHash('sha256').update(value).digest('hex');
+}
+
+function requiredNonempty(value: string, label: string): string {
+  const normalized = value.trim();
+  if (!normalized) throw new Error(`${label} must be non-empty.`);
+  return normalized;
+}
+
+function nonnegativeInteger(value: number, label: string): number {
+  if (!Number.isSafeInteger(value) || value < 0) {
+    throw new Error(`${label} must be a non-negative safe integer.`);
+  }
+  return value;
+}
+
+function positiveInteger(value: number, label: string): number {
+  if (!Number.isSafeInteger(value) || value <= 0) {
+    throw new Error(`${label} must be a positive safe integer.`);
+  }
+  return value;
+}
+
+function receiptValueBindings(input: { orgId: string; receiptKey: string }): {
+  orgBinding: string;
+  receiptBinding: string;
+} {
+  return {
+    orgBinding: sha256(`runtime-receipt-value/org/v1/${input.orgId}`),
+    receiptBinding: sha256(
+      `runtime-receipt-value/receipt/v1/${input.orgId}\0${input.receiptKey}`,
+    ),
+  };
+}
+
+export function runtimeReceiptValueStorageKey(input: {
+  orgId: string;
+  receiptKey: string;
+  digest: string;
+}): string {
+  const orgId = requiredNonempty(input.orgId, 'orgId');
+  const receiptKey = requiredNonempty(input.receiptKey, 'receiptKey');
+  if (!/^[a-f0-9]{64}$/i.test(input.digest)) {
+    throw new Error('digest must be a SHA-256 hex digest.');
+  }
+  const { orgBinding, receiptBinding } = receiptValueBindings({
+    orgId,
+    receiptKey,
+  });
+  return `runtime/receipt-values/v1/${orgBinding}/${receiptBinding}/${input.digest}.json`;
+}
+
+function referenceForBody(input: {
+  orgId: string;
+  receiptKey: string;
+  body: Buffer;
+  inlineFields?: RuntimeReceiptValueInlineFields;
+}): RuntimeReceiptValueReference {
+  const orgId = requiredNonempty(input.orgId, 'orgId');
+  const receiptKey = requiredNonempty(input.receiptKey, 'receiptKey');
+  const digest = sha256(input.body);
+  const { orgBinding, receiptBinding } = receiptValueBindings({
+    orgId,
+    receiptKey,
+  });
+  return {
+    storage: 'r2',
+    codec: 'json',
+    key: runtimeReceiptValueStorageKey({ orgId, receiptKey, digest }),
+    digest,
+    byteLength: input.body.byteLength,
+    orgBinding,
+    receiptBinding,
+    ...(input.inlineFields ? { inlineFields: input.inlineFields } : {}),
+  };
+}
+
+/**
+ * Store a body under its deterministic immutable key before its receipt row
+ * points at it. If a PUT response is lost after object creation, the exact
+ * object is discovered and reused; a caller never regenerates provider work to
+ * repair that ambiguity.
+ */
+export async function persistRuntimeReceiptValue(
+  input: RuntimeReceiptValueStorageInput,
+): Promise<PersistedRuntimeReceiptValue> {
+  const orgId = requiredNonempty(input.orgId, 'orgId');
+  const receiptKey = requiredNonempty(input.receiptKey, 'receiptKey');
+  const body = Buffer.from(input.serialized, 'utf8');
+  const inlineMaxBytes = positiveInteger(
+    input.inlineMaxBytes ?? RUNTIME_RECEIPT_VALUE_INLINE_MAX_BYTES,
+    'inlineMaxBytes',
+  );
+  const inlineFields =
+    input.inlineFields === undefined
+      ? undefined
+      : normalizeRuntimeReceiptValueInlineFields(input.inlineFields);
+  if (body.byteLength <= inlineMaxBytes) {
+    return { kind: 'inline', outputJson: input.serialized };
+  }
+
+  const store = requireRuntimeValueBlobStore(input.store);
+  const outputRef = referenceForBody({ orgId, receiptKey, body, inlineFields });
+  try {
+    await store.put({
+      key: outputRef.key,
+      body,
+      contentType: 'application/json',
+    });
+  } catch (error) {
+    // An immutable deterministic object that is observable after a failed PUT
+    // is the same attempted publication. Treat it as a recovered write, never
+    // as permission to call the provider again.
+    if (!(await store.exists(outputRef.key))) throw error;
+  }
+  return { kind: 'reference', outputJson: null, outputRef };
+}
+
+/**
+ * Build one process-scoped persister for a Fly gateway. It batches admission
+ * only: R2 has no multi-PUT, so each physical call remains one immutable PUT,
+ * while the common primitive bounds queued and active bytes across requests.
+ */
+export function createRuntimeReceiptValuePersister(
+  input: {
+    store?: RuntimeReceiptValueBlobStore;
+    inlineMaxBytes?: number;
+  } = {},
+): RuntimeReceiptValuePersister {
+  const inlineMaxBytes =
+    input.inlineMaxBytes ?? RUNTIME_RECEIPT_VALUE_INLINE_MAX_BYTES;
+  type Write = {
+    orgId: string;
+    receiptKey: string;
+    serialized: string;
+    inlineFields?: RuntimeReceiptValueInlineFields;
+  };
+  const writes = new BoundedBatchFunction<
+    Write,
+    PersistedRuntimeReceiptValue,
+    'r2'
+  >({
+    keyId: () => 'r2',
+    estimateBytes: (value) => Buffer.byteLength(value.serialized, 'utf8'),
+    maxBatchItems: 4,
+    maxBatchBytes: 12 * 1024 * 1024,
+    maxBatchUnits: 4,
+    maxRetainedItems: 32,
+    maxRetainedBytes: 24 * 1024 * 1024,
+    maxRetainedUnits: 32,
+    maxInFlightBatches: 2,
+    flushMs: 0,
+    execute: async ({ values }) =>
+      await Promise.all(
+        values.map(
+          async (value) =>
+            await persistRuntimeReceiptValue({
+              ...value,
+              inlineMaxBytes,
+              ...(input.store ? { store: input.store } : {}),
+            }),
+        ),
+      ),
+  });
+  return async (value) => {
+    if (Buffer.byteLength(value.serialized, 'utf8') <= inlineMaxBytes) {
+      return await persistRuntimeReceiptValue({
+        ...value,
+        inlineMaxBytes,
+        ...(input.store ? { store: input.store } : {}),
+      });
+    }
+    return await writes.submit({ key: 'r2', value });
+  };
+}
+
+/**
+ * Build one process-scoped reader for a Fly gateway. R2 has no multi-GET,
+ * so the batch is an admission boundary rather than a protocol claim: it
+ * keeps active and queued body bytes bounded across every inbound request
+ * while allowing independent immutable objects to use the connection pool.
+ */
+export function createRuntimeReceiptValueReader(
+  input: { store?: RuntimeReceiptValueBlobStore } = {},
+): RuntimeReceiptValueReader {
+  type Read = {
+    orgId: string;
+    receiptKey: string;
+    reference: RuntimeReceiptValueReference;
+  };
+  const reads = new BoundedBatchFunction<Read, unknown, 'r2'>({
+    keyId: () => 'r2',
+    estimateBytes: (value) => value.reference.byteLength,
+    maxBatchItems: 4,
+    maxBatchBytes: 12 * 1024 * 1024,
+    maxBatchUnits: 4,
+    maxRetainedItems: 32,
+    maxRetainedBytes: 24 * 1024 * 1024,
+    maxRetainedUnits: 32,
+    maxInFlightBatches: 2,
+    flushMs: 0,
+    execute: async ({ values }) =>
+      await Promise.all(
+        values.map(
+          async (value) =>
+            await readRuntimeReceiptValue({
+              ...value,
+              ...(input.store ? { store: input.store } : {}),
+            }),
+        ),
+      ),
+  });
+  return async (value) => await reads.submit({ key: 'r2', value });
+}
+
+/**
+ * Verify that a compact reference is bound to the authenticated receipt that
+ * is about to store it. Execution relays use the same proof before a receipt
+ * gateway accepts their already-persisted result reference.
+ */
+export function assertRuntimeReceiptValueReferenceScope(input: {
+  orgId: string;
+  receiptKey: string;
+  reference: RuntimeReceiptValueReference;
+}): void {
+  const orgId = requiredNonempty(input.orgId, 'orgId');
+  const receiptKey = requiredNonempty(input.receiptKey, 'receiptKey');
+  const bindings = receiptValueBindings({ orgId, receiptKey });
+  if (
+    input.reference.orgBinding !== bindings.orgBinding ||
+    input.reference.receiptBinding !== bindings.receiptBinding ||
+    input.reference.key !==
+      runtimeReceiptValueStorageKey({
+        orgId,
+        receiptKey,
+        digest: input.reference.digest,
+      })
+  ) {
+    throw new RuntimeReceiptValueReadError(
+      'scope_mismatch',
+      'Runtime receipt value reference is outside the authenticated receipt scope.',
+    );
+  }
+}
+
+function isMissingRuntimeReceiptValue(error: unknown): boolean {
+  if (!error || typeof error !== 'object') return false;
+  const record = error as Record<string, unknown>;
+  const code = String(record.code ?? record.name ?? '');
+  if (code === 'NoSuchKey' || code === 'NotFound') return true;
+  const metadata = record.$metadata;
+  return (
+    Boolean(metadata) &&
+    typeof metadata === 'object' &&
+    (metadata as Record<string, unknown>).httpStatusCode === 404
+  );
+}
+
+/** Read, authenticate, size-check, and decode a referenced JSON receipt. */
+export async function readRuntimeReceiptValue(input: {
+  orgId: string;
+  receiptKey: string;
+  reference: RuntimeReceiptValueReference;
+  store?: RuntimeReceiptValueBlobStore;
+}): Promise<unknown> {
+  assertRuntimeReceiptValueReferenceScope(input);
+  const store = requireRuntimeValueBlobStore(input.store);
+  let body: Buffer;
+  try {
+    body = await store.get(input.reference.key);
+  } catch (error) {
+    throw new RuntimeReceiptValueReadError(
+      isMissingRuntimeReceiptValue(error) ? 'missing' : 'unavailable',
+      `Runtime receipt value could not be read: ${
+        error instanceof Error ? error.message : 'unknown storage error'
+      }`,
+    );
+  }
+  if (
+    body.byteLength !==
+      nonnegativeInteger(input.reference.byteLength, 'byteLength') ||
+    sha256(body) !== input.reference.digest
+  ) {
+    throw new RuntimeReceiptValueReadError(
+      'corrupt',
+      'Runtime receipt value failed its stored size or digest check.',
+    );
+  }
+  try {
+    return JSON.parse(body.toString('utf8')) as unknown;
+  } catch {
+    throw new RuntimeReceiptValueReadError(
+      'corrupt',
+      'Runtime receipt value is not valid JSON for its declared codec.',
+    );
+  }
+}

package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/worker-resource-metrics.ts

@@ -1,5 +1,6 @@
 import { statfsSync } from 'node:fs';
 import { cpus } from 'node:os';
+import { getHeapStatistics } from 'node:v8';
 
 type ReadFile = (path: string, encoding: 'utf8') => string;
 
@@ -8,13 +9,62 @@ export type WorkerResourceMetrics = {
   cpuLimitCores: number;
   memoryUsageBytes: number;
   memoryLimitBytes: number;
+  memoryLimitSource: 'cgroup' | 'process_fallback';
   memoryUtilizationRatio: number;
+  heapUsedBytes: number;
+  heapLimitBytes: number;
+  heapUtilizationRatio: number;
   filesystemUsageBytes: number;
   filesystemAvailableBytes: number;
   filesystemTotalBytes: number;
   filesystemUtilizationRatio: number;
 };
 
+export const WORKER_CPU_PRESSURE_RATIO = 0.95;
+export const WORKER_MEMORY_PRESSURE_RATIO = 0.9;
+export const WORKER_HEAP_PRESSURE_RATIO = 0.88;
+
+export type WorkerResourcePressureReason = 'cpu' | 'memory' | 'heap';
+
+/**
+ * Resource pressure is an admission signal, not a terminal outcome. A worker
+ * should stop claiming new durable work while the scaler replaces it; the
+ * engine still owns already-claimed tasks and drains them before close.
+ *
+ * Memory pressure requires a real cgroup limit. A local process fallback uses
+ * RSS as both the observed value and its fallback limit, so it must never
+ * masquerade as a saturated container.
+ */
+export function workerResourcePressureReasons(
+  metrics: Pick<
+    WorkerResourceMetrics,
+    | 'cpuUtilizationRatio'
+    | 'memoryLimitSource'
+    | 'memoryUtilizationRatio'
+    | 'heapUtilizationRatio'
+  >,
+  thresholds: Readonly<{
+    cpu?: number;
+    memory?: number;
+    heap?: number;
+  }> = {},
+): WorkerResourcePressureReason[] {
+  const cpuThreshold = thresholds.cpu ?? WORKER_CPU_PRESSURE_RATIO;
+  const memoryThreshold =
+    thresholds.memory ?? WORKER_MEMORY_PRESSURE_RATIO;
+  const heapThreshold = thresholds.heap ?? WORKER_HEAP_PRESSURE_RATIO;
+  const reasons: WorkerResourcePressureReason[] = [];
+  if (metrics.cpuUtilizationRatio >= cpuThreshold) reasons.push('cpu');
+  if (
+    metrics.memoryLimitSource === 'cgroup' &&
+    metrics.memoryUtilizationRatio >= memoryThreshold
+  ) {
+    reasons.push('memory');
+  }
+  if (metrics.heapUtilizationRatio >= heapThreshold) reasons.push('heap');
+  return reasons;
+}
+
 /** A zero-sized filesystem report means the probe itself is unavailable; a
  * nonzero filesystem with zero free bytes is the most urgent pressure state. */
 export function hasFilesystemPressure(
@@ -60,6 +110,15 @@ export function renderRuntimeProcessResourceMetrics(input: {
     '# HELP deepline_runtime_process_memory_utilization_percent Container memory utilization normalized to its memory limit.',
     '# TYPE deepline_runtime_process_memory_utilization_percent gauge',
     `deepline_runtime_process_memory_utilization_percent{${labels}} ${input.metrics.memoryUtilizationRatio * 100}`,
+    '# HELP deepline_runtime_process_heap_used_bytes V8 heap currently used.',
+    '# TYPE deepline_runtime_process_heap_used_bytes gauge',
+    `deepline_runtime_process_heap_used_bytes{${labels}} ${input.metrics.heapUsedBytes}`,
+    '# HELP deepline_runtime_process_heap_limit_bytes V8 heap limit.',
+    '# TYPE deepline_runtime_process_heap_limit_bytes gauge',
+    `deepline_runtime_process_heap_limit_bytes{${labels}} ${input.metrics.heapLimitBytes}`,
+    '# HELP deepline_runtime_process_heap_utilization_percent V8 heap utilization normalized to its heap limit.',
+    '# TYPE deepline_runtime_process_heap_utilization_percent gauge',
+    `deepline_runtime_process_heap_utilization_percent{${labels}} ${input.metrics.heapUtilizationRatio * 100}`,
     '# HELP deepline_runtime_process_filesystem_usage_bytes Local /tmp filesystem space used.',
     '# TYPE deepline_runtime_process_filesystem_usage_bytes gauge',
     `deepline_runtime_process_filesystem_usage_bytes{${labels}} ${input.metrics.filesystemUsageBytes}`,
@@ -158,11 +217,13 @@ export function createWorkerResourceMetricsReader(input: {
   now?: () => number;
   cpuCount?: () => number;
   memoryUsage?: () => NodeJS.MemoryUsage;
+  heapLimit?: () => number;
   statfs?: (path: string) => FileSystemStats;
 }) {
   const now = input.now ?? Date.now;
   const cpuCount = input.cpuCount ?? (() => cpus().length || 1);
   const memoryUsage = input.memoryUsage ?? process.memoryUsage;
+  const heapLimit = input.heapLimit ?? (() => getHeapStatistics().heap_size_limit);
   const statfs = input.statfs ?? statfsSync;
   let previousCpu: CpuSample | null = null;
 
@@ -208,7 +269,8 @@ export function createWorkerResourceMetricsReader(input: {
       previousCpu = { usageMicroseconds: cpuUsageMicroseconds, sampledAtMs };
     }
 
-    const fallbackMemory = memoryUsage().rss;
+    const processMemory = memoryUsage();
+    const fallbackMemory = processMemory.rss;
     const filesystem = (() => {
       try {
         return parseFileSystemStats(statfs('/tmp'));
@@ -216,27 +278,44 @@ export function createWorkerResourceMetricsReader(input: {
         return parseFileSystemStats(null);
       }
     })();
-    const memoryUsageBytes = parseCgroupLimit(
+    const cgroupMemoryUsage =
       read('/sys/fs/cgroup/memory.current') ??
-        read('/sys/fs/cgroup/memory/memory.usage_in_bytes') ??
-        '',
+      read('/sys/fs/cgroup/memory/memory.usage_in_bytes');
+    const cgroupMemoryLimit =
+      read('/sys/fs/cgroup/memory.max') ??
+      read('/sys/fs/cgroup/memory/memory.limit_in_bytes');
+    const memoryUsageBytes = parseCgroupLimit(
+      cgroupMemoryUsage ?? '',
       fallbackMemory,
     );
     const memoryLimitBytes = parseCgroupLimit(
-      read('/sys/fs/cgroup/memory.max') ??
-        read('/sys/fs/cgroup/memory/memory.limit_in_bytes') ??
-        '',
+      cgroupMemoryLimit ?? '',
       fallbackMemory,
     );
+    const heapUsedBytes = finiteNonNegative(
+      processMemory.heapUsed,
+      0,
+    );
+    const heapLimitBytes = finiteNonNegative(heapLimit(), 0);
     return {
       cpuUtilizationRatio,
       cpuLimitCores,
       memoryUsageBytes,
       memoryLimitBytes,
+      memoryLimitSource:
+        cgroupMemoryLimit === null ||
+        cgroupMemoryLimit === undefined ||
+        cgroupMemoryLimit.trim() === 'max'
+          ? 'process_fallback'
+          : 'cgroup',
       memoryUtilizationRatio:
         memoryLimitBytes > 0
           ? Math.min(1, memoryUsageBytes / memoryLimitBytes)
           : 0,
+      heapUsedBytes,
+      heapLimitBytes,
+      heapUtilizationRatio:
+        heapLimitBytes > 0 ? Math.min(1, heapUsedBytes / heapLimitBytes) : 0,
       filesystemUsageBytes: filesystem.usageBytes,
       filesystemAvailableBytes: filesystem.availableBytes,
       filesystemTotalBytes: filesystem.totalBytes,