deepline 0.3.128 → 0.3.130

package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/integration-operation-value-retirement.ts

@@ -0,0 +1,193 @@
+import { randomUUID } from 'node:crypto';
+
+import {
+  isIntegrationOperationValueReference,
+  type IntegrationOperationValueReference,
+} from './integration-operation-value-reference';
+
+/**
+ * The value-retirement journal needs only the scheduler client's query
+ * surface. Keeping that structural contract here lets the Fly gateway own its
+ * janitor without importing the application scheduler implementation.
+ */
+type PostgresSchedulerQueryClient = {
+  query<Row extends Record<string, unknown> = Record<string, unknown>>(
+    sql: string,
+    params?: unknown[],
+  ): Promise<{ rows: Row[] }>;
+};
+
+const DEFAULT_SCHEMA = 'runtime_scheduler';
+
+function quoteIdent(value: string): string {
+  return `"${value.replace(/"/g, '""')}"`;
+}
+
+function tableName(schema: string): string {
+  return `${quoteIdent(schema)}.${quoteIdent(
+    'runtime_integration_operation_value_retirements',
+  )}`;
+}
+
+function required(value: string, label: string): string {
+  const normalized = value.trim();
+  if (!normalized) throw new Error(`${label} must be non-empty.`);
+  return normalized;
+}
+
+function limit(value: number | undefined): number {
+  const normalized = value ?? 32;
+  if (!Number.isSafeInteger(normalized) || normalized < 1 || normalized > 256) {
+    throw new Error('limit must be a safe integer between 1 and 256.');
+  }
+  return normalized;
+}
+
+type RetirementRow = {
+  org_id: string;
+  operation_key: string;
+  object_key: string;
+  object_digest: string;
+  reference_json: unknown;
+  claim_token: string;
+};
+
+export type ClaimedIntegrationOperationValueRetirement = {
+  orgId: string;
+  operationKey: string;
+  objectKey: string;
+  objectDigest: string;
+  reference: IntegrationOperationValueReference;
+  claimToken: string;
+};
+
+/**
+ * Claims a bounded set of detached immutable values. A claim remains durable
+ * until it is confirmed or released, so a process crash after object deletion
+ * cannot quietly permit a second writer to reuse the same deterministic key.
+ */
+export async function claimPostgresIntegrationOperationValueRetirements(input: {
+  client: PostgresSchedulerQueryClient;
+  schema?: string;
+  limit?: number;
+  claimToken?: string;
+}): Promise<ClaimedIntegrationOperationValueRetirement[]> {
+  const schema = input.schema?.trim() || DEFAULT_SCHEMA;
+  const claimToken = required(input.claimToken ?? randomUUID(), 'claimToken');
+  const claimed = await input.client.query<RetirementRow>(
+    `WITH candidates AS (
+       SELECT ctid
+         FROM ${tableName(schema)}
+        WHERE (
+            state = 'pending'
+            OR (
+              state = 'deleting'
+              AND updated_at < now() - interval '15 minutes'
+            )
+          )
+          AND jsonb_typeof(reference_json) = 'object'
+          AND reference_json ->> 'storage' = 'r2'
+          AND reference_json ->> 'codec' IN ('json', 'json-v1')
+          AND NULLIF(reference_json ->> 'key', '') IS NOT NULL
+          AND (reference_json ->> 'digest') ~ '^[A-Fa-f0-9]{64}$'
+        ORDER BY updated_at ASC, object_key ASC
+        LIMIT $1
+        FOR UPDATE SKIP LOCKED
+     )
+     UPDATE ${tableName(schema)} AS retirements
+        SET state = 'deleting',
+            claim_token = $2,
+            updated_at = now()
+       FROM candidates
+      WHERE retirements.ctid = candidates.ctid
+      RETURNING retirements.org_id, retirements.operation_key,
+                retirements.object_key, retirements.object_digest,
+                retirements.reference_json, retirements.claim_token`,
+    [limit(input.limit), claimToken],
+  );
+  return claimed.rows.map((row) => {
+    if (!isIntegrationOperationValueReference(row.reference_json)) {
+      throw new Error(
+        `Integration Operation value retirement ${row.object_key} has an invalid reference.`,
+      );
+    }
+    if (
+      row.reference_json.key !== row.object_key ||
+      row.reference_json.digest.toLowerCase() !== row.object_digest.toLowerCase()
+    ) {
+      throw new Error(
+        `Integration Operation value retirement ${row.object_key} does not match its immutable reference.`,
+      );
+    }
+    return {
+      orgId: row.org_id,
+      operationKey: row.operation_key,
+      objectKey: row.object_key,
+      objectDigest: row.object_digest,
+      reference: row.reference_json,
+      claimToken: row.claim_token,
+    };
+  });
+}
+
+export async function confirmPostgresIntegrationOperationValueRetirement(input: {
+  client: PostgresSchedulerQueryClient;
+  schema?: string;
+  orgId: string;
+  objectKey: string;
+  objectDigest: string;
+  claimToken: string;
+}): Promise<boolean> {
+  const schema = input.schema?.trim() || DEFAULT_SCHEMA;
+  const confirmed = await input.client.query<{ confirmed: number }>(
+    `UPDATE ${tableName(schema)}
+        SET state = 'deleted',
+            claim_token = NULL,
+            deletion_confirmed_at = now(),
+            updated_at = now()
+      WHERE org_id = $1
+        AND object_key = $2
+        AND object_digest = $3
+        AND state = 'deleting'
+        AND claim_token = $4
+      RETURNING 1 AS confirmed`,
+    [
+      required(input.orgId, 'orgId'),
+      required(input.objectKey, 'objectKey'),
+      required(input.objectDigest, 'objectDigest').toLowerCase(),
+      required(input.claimToken, 'claimToken'),
+    ],
+  );
+  return confirmed.rows.length === 1;
+}
+
+/** Release an unsuccessful delete so a later bounded janitor pass can retry. */
+export async function releasePostgresIntegrationOperationValueRetirement(input: {
+  client: PostgresSchedulerQueryClient;
+  schema?: string;
+  orgId: string;
+  objectKey: string;
+  objectDigest: string;
+  claimToken: string;
+}): Promise<boolean> {
+  const schema = input.schema?.trim() || DEFAULT_SCHEMA;
+  const released = await input.client.query<{ released: number }>(
+    `UPDATE ${tableName(schema)}
+        SET state = 'pending',
+            claim_token = NULL,
+            updated_at = now()
+      WHERE org_id = $1
+        AND object_key = $2
+        AND object_digest = $3
+        AND state = 'deleting'
+        AND claim_token = $4
+      RETURNING 1 AS released`,
+    [
+      required(input.orgId, 'orgId'),
+      required(input.objectKey, 'objectKey'),
+      required(input.objectDigest, 'objectDigest').toLowerCase(),
+      required(input.claimToken, 'claimToken'),
+    ],
+  );
+  return released.rows.length === 1;
+}

package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/integration-operation-value-storage.ts

@@ -0,0 +1,420 @@
+import { createHash } from 'node:crypto';
+import {
+  IntegrationOperationValueReadError,
+  isIntegrationOperationValueReference,
+  type IntegrationOperationValueReference,
+} from './integration-operation-value-reference';
+import type { IntegrationOperationValueReader } from './integration-operation-store-contract';
+import { BoundedBatchFunction } from '../bounded-batch-function';
+import {
+  requireRuntimeValueBlobStore,
+  type RuntimeValueBlobStore,
+} from './runtime-receipt-value-storage';
+
+/** Keep small canonical outcomes in the transaction row. */
+export const INTEGRATION_OPERATION_VALUE_INLINE_MAX_BYTES = 256 * 1024;
+
+export type PersistedIntegrationOperationValue =
+  | { kind: 'inline' }
+  | { kind: 'reference'; outputRef: IntegrationOperationValueReference };
+
+export type PreparedIntegrationOperationValue =
+  | { kind: 'inline' }
+  | {
+      kind: 'reference';
+      outputRef: IntegrationOperationValueReference;
+      /** Private until the operation store has reserved its durable owner. */
+      body: Buffer;
+    };
+
+type IntegrationOperationValueInput = {
+  orgId: string;
+  operationKey: string;
+  output: unknown;
+};
+
+/**
+ * A prepared value separates deterministic serialization from object I/O.
+ * Gateway composition can therefore fence pending ownership before handing a
+ * large body to the shared R2 admission queue.
+ */
+export type IntegrationOperationValuePersister = ((
+  input: IntegrationOperationValueInput,
+) => Promise<PersistedIntegrationOperationValue>) & {
+  prepare(input: IntegrationOperationValueInput): PreparedIntegrationOperationValue;
+  persistPrepared(
+    input: PreparedIntegrationOperationValue,
+  ): Promise<PersistedIntegrationOperationValue>;
+};
+
+export type IntegrationOperationValueRetirer = (input: {
+  reference: IntegrationOperationValueReference;
+}) => Promise<void>;
+
+function sha256(value: string | Buffer): string {
+  return createHash('sha256').update(value).digest('hex');
+}
+
+function nonempty(value: string, label: string): string {
+  const normalized = value.trim();
+  if (!normalized) throw new Error(`${label} must be non-empty.`);
+  return normalized;
+}
+
+function nonnegativeInteger(value: number, label: string): number {
+  if (!Number.isSafeInteger(value) || value < 0) {
+    throw new Error(`${label} must be a non-negative safe integer.`);
+  }
+  return value;
+}
+
+function positiveInteger(value: number, label: string): number {
+  if (!Number.isSafeInteger(value) || value <= 0) {
+    throw new Error(`${label} must be a positive safe integer.`);
+  }
+  return value;
+}
+
+function serializeOutput(output: unknown): string {
+  const serialized = JSON.stringify(output);
+  if (typeof serialized !== 'string') {
+    throw new Error('Integration Operation output must be JSON serializable.');
+  }
+  return serialized;
+}
+
+function bindings(input: { orgId: string; operationKey: string }): {
+  orgBinding: string;
+  operationBinding: string;
+} {
+  return {
+    orgBinding: sha256(
+      `runtime-integration-operation-value/org/v1/${input.orgId}`,
+    ),
+    operationBinding: sha256(
+      `runtime-integration-operation-value/operation/v1/${input.orgId}\0${input.operationKey}`,
+    ),
+  };
+}
+
+export function integrationOperationValueStorageKey(input: {
+  orgId: string;
+  operationKey: string;
+  digest: string;
+}): string {
+  const orgId = nonempty(input.orgId, 'orgId');
+  const operationKey = nonempty(input.operationKey, 'operationKey');
+  if (!/^[a-f0-9]{64}$/i.test(input.digest)) {
+    throw new Error('digest must be a SHA-256 hex digest.');
+  }
+  const { orgBinding, operationBinding } = bindings({ orgId, operationKey });
+  return `runtime/integration-operation-values/v1/${orgBinding}/${operationBinding}/${input.digest}.json`;
+}
+
+function referenceFor(input: {
+  orgId: string;
+  operationKey: string;
+  body: Buffer;
+}): IntegrationOperationValueReference {
+  const orgId = nonempty(input.orgId, 'orgId');
+  const operationKey = nonempty(input.operationKey, 'operationKey');
+  const digest = sha256(input.body);
+  const { orgBinding, operationBinding } = bindings({ orgId, operationKey });
+  return {
+    storage: 'r2',
+    codec: 'json',
+    key: integrationOperationValueStorageKey({ orgId, operationKey, digest }),
+    digest,
+    byteLength: input.body.byteLength,
+    orgBinding,
+    operationBinding,
+  };
+}
+
+/** Serialize a value and derive its immutable reference without touching R2. */
+export function prepareIntegrationOperationValue(input: {
+  orgId: string;
+  operationKey: string;
+  output: unknown;
+  inlineMaxBytes?: number;
+}): PreparedIntegrationOperationValue {
+  const orgId = nonempty(input.orgId, 'orgId');
+  const operationKey = nonempty(input.operationKey, 'operationKey');
+  const body = Buffer.from(serializeOutput(input.output), 'utf8');
+  const inlineMaxBytes = positiveInteger(
+    input.inlineMaxBytes ?? INTEGRATION_OPERATION_VALUE_INLINE_MAX_BYTES,
+    'inlineMaxBytes',
+  );
+  if (body.byteLength <= inlineMaxBytes) return { kind: 'inline' };
+
+  return {
+    kind: 'reference',
+    outputRef: referenceFor({ orgId, operationKey, body }),
+    body,
+  };
+}
+
+/**
+ * Write a prepared immutable body. A lost PUT acknowledgement observes the
+ * deterministic object; it never becomes permission to repeat provider work.
+ */
+export async function persistPreparedIntegrationOperationValue(input: {
+  value: PreparedIntegrationOperationValue;
+  store?: RuntimeValueBlobStore;
+}): Promise<PersistedIntegrationOperationValue> {
+  if (input.value.kind === 'inline') return input.value;
+
+  const store = requireRuntimeValueBlobStore(input.store);
+  try {
+    await store.put({
+      key: input.value.outputRef.key,
+      body: input.value.body,
+      contentType: 'application/json',
+    });
+  } catch (error) {
+    if (!(await store.exists(input.value.outputRef.key))) throw error;
+  }
+  return { kind: 'reference', outputRef: input.value.outputRef };
+}
+
+/**
+ * Convenience path for callers without an operation-store ownership fence.
+ * The gateway uses prepare + reserve + persistPrepared instead.
+ */
+export async function persistIntegrationOperationValue(input: {
+  orgId: string;
+  operationKey: string;
+  output: unknown;
+  inlineMaxBytes?: number;
+  store?: RuntimeValueBlobStore;
+}): Promise<PersistedIntegrationOperationValue> {
+  return await persistPreparedIntegrationOperationValue({
+    value: prepareIntegrationOperationValue(input),
+    ...(input.store ? { store: input.store } : {}),
+  });
+}
+
+/**
+ * One process-scoped admission budget for Integration Operation object PUTs.
+ * R2 has no multi-PUT; batching bounds retained work and concurrent physical
+ * requests without claiming a nonexistent storage transaction.
+ */
+export function createIntegrationOperationValuePersister(
+  input: {
+    store?: RuntimeValueBlobStore;
+    inlineMaxBytes?: number;
+  } = {},
+): IntegrationOperationValuePersister {
+  const inlineMaxBytes =
+    input.inlineMaxBytes ?? INTEGRATION_OPERATION_VALUE_INLINE_MAX_BYTES;
+  type Write = Extract<PreparedIntegrationOperationValue, { kind: 'reference' }>;
+  const writes = new BoundedBatchFunction<
+    Write,
+    PersistedIntegrationOperationValue,
+    'r2'
+  >({
+    keyId: () => 'r2',
+    estimateBytes: (value) => value.body.byteLength,
+    maxBatchItems: 4,
+    maxBatchBytes: 12 * 1024 * 1024,
+    maxBatchUnits: 4,
+    maxRetainedItems: 32,
+    maxRetainedBytes: 24 * 1024 * 1024,
+    maxRetainedUnits: 32,
+    maxInFlightBatches: 2,
+    flushMs: 0,
+    execute: async ({ values }) =>
+      await Promise.all(
+        values.map(
+          async (value) =>
+            await persistPreparedIntegrationOperationValue({
+              value,
+              ...(input.store ? { store: input.store } : {}),
+            }),
+        ),
+      ),
+  });
+  const persistPrepared = async (
+    value: PreparedIntegrationOperationValue,
+  ): Promise<PersistedIntegrationOperationValue> => {
+    if (value.kind === 'inline') return value;
+    return await writes.submit({ key: 'r2', value });
+  };
+  const persister = (async (value: IntegrationOperationValueInput) =>
+    await persistPrepared(
+      prepareIntegrationOperationValue({ ...value, inlineMaxBytes }),
+    )) as IntegrationOperationValuePersister;
+  persister.prepare = (value) =>
+    prepareIntegrationOperationValue({ ...value, inlineMaxBytes });
+  persister.persistPrepared = persistPrepared;
+  return persister;
+}
+
+/**
+ * Delete an immutable value only after its durable retirement claim is held.
+ * A successful DELETE acknowledgement is insufficient: object stores can lose
+ * it, so absence must be observed before the caller records confirmation.
+ */
+export async function deleteIntegrationOperationValue(input: {
+  reference: IntegrationOperationValueReference;
+  store?: RuntimeValueBlobStore;
+}): Promise<void> {
+  if (!isIntegrationOperationValueReference(input.reference)) {
+    throw new Error('Integration Operation value deletion has an invalid reference.');
+  }
+  const store = requireRuntimeValueBlobStore(input.store);
+  try {
+    await store.remove(input.reference.key);
+  } catch (error) {
+    if (await store.exists(input.reference.key)) throw error;
+  }
+  if (await store.exists(input.reference.key)) {
+    throw new Error(
+      'Integration Operation value deletion was not confirmed absent from object storage.',
+    );
+  }
+}
+
+/** One process-scoped bounded admission budget for immutable R2 DELETEs. */
+export function createIntegrationOperationValueRetirer(
+  input: { store?: RuntimeValueBlobStore } = {},
+): IntegrationOperationValueRetirer {
+  const deletes = new BoundedBatchFunction<
+    IntegrationOperationValueReference,
+    void,
+    'r2'
+  >({
+    keyId: () => 'r2',
+    estimateBytes: (value) => value.byteLength,
+    maxBatchItems: 4,
+    maxBatchBytes: 12 * 1024 * 1024,
+    maxBatchUnits: 4,
+    maxRetainedItems: 32,
+    maxRetainedBytes: 24 * 1024 * 1024,
+    maxRetainedUnits: 32,
+    maxInFlightBatches: 2,
+    flushMs: 0,
+    execute: async ({ values }) =>
+      await Promise.all(
+        values.map(
+          async (reference) =>
+            await deleteIntegrationOperationValue({
+              reference,
+              ...(input.store ? { store: input.store } : {}),
+            }),
+        ),
+      ),
+  });
+  return async ({ reference }) => await deletes.submit({ key: 'r2', value: reference });
+}
+
+function assertScope(input: {
+  orgId: string;
+  operationKey: string;
+  reference: IntegrationOperationValueReference;
+}): void {
+  const orgId = nonempty(input.orgId, 'orgId');
+  const operationKey = nonempty(input.operationKey, 'operationKey');
+  const expected = bindings({ orgId, operationKey });
+  if (
+    input.reference.orgBinding !== expected.orgBinding ||
+    input.reference.operationBinding !== expected.operationBinding ||
+    input.reference.key !==
+      integrationOperationValueStorageKey({
+        orgId,
+        operationKey,
+        digest: input.reference.digest,
+      })
+  ) {
+    throw new IntegrationOperationValueReadError(
+      'scope_mismatch',
+      'Integration Operation value reference is outside the authenticated operation scope.',
+    );
+  }
+}
+
+function isMissing(error: unknown): boolean {
+  if (!error || typeof error !== 'object') return false;
+  const record = error as Record<string, unknown>;
+  const code = String(record.code ?? record.name ?? '');
+  if (code === 'NoSuchKey' || code === 'NotFound') return true;
+  const metadata = record.$metadata;
+  return (
+    Boolean(metadata) &&
+    typeof metadata === 'object' &&
+    (metadata as Record<string, unknown>).httpStatusCode === 404
+  );
+}
+
+export async function readIntegrationOperationValue(input: {
+  orgId: string;
+  operationKey: string;
+  reference: IntegrationOperationValueReference;
+  store?: RuntimeValueBlobStore;
+}): Promise<unknown> {
+  assertScope(input);
+  const store = requireRuntimeValueBlobStore(input.store);
+  let body: Buffer;
+  try {
+    body = await store.get(input.reference.key);
+  } catch (error) {
+    throw new IntegrationOperationValueReadError(
+      isMissing(error) ? 'missing' : 'unavailable',
+      `Integration Operation value could not be read: ${
+        error instanceof Error ? error.message : 'unknown storage error'
+      }`,
+    );
+  }
+  if (
+    body.byteLength !==
+      nonnegativeInteger(input.reference.byteLength, 'byteLength') ||
+    sha256(body) !== input.reference.digest
+  ) {
+    throw new IntegrationOperationValueReadError(
+      'corrupt',
+      'Integration Operation value failed its stored size or digest check.',
+    );
+  }
+  try {
+    return JSON.parse(body.toString('utf8')) as unknown;
+  } catch {
+    throw new IntegrationOperationValueReadError(
+      'corrupt',
+      'Integration Operation value is not valid JSON for its declared codec.',
+    );
+  }
+}
+
+/** One process-scoped bounded admission budget for immutable R2 GETs. */
+export function createIntegrationOperationValueReader(
+  input: { store?: RuntimeValueBlobStore } = {},
+): IntegrationOperationValueReader {
+  type Read = {
+    orgId: string;
+    operationKey: string;
+    reference: IntegrationOperationValueReference;
+  };
+  const reads = new BoundedBatchFunction<Read, unknown, 'r2'>({
+    keyId: () => 'r2',
+    estimateBytes: (value) => value.reference.byteLength,
+    maxBatchItems: 4,
+    maxBatchBytes: 12 * 1024 * 1024,
+    maxBatchUnits: 4,
+    maxRetainedItems: 32,
+    maxRetainedBytes: 24 * 1024 * 1024,
+    maxRetainedUnits: 32,
+    maxInFlightBatches: 2,
+    flushMs: 0,
+    execute: async ({ values }) =>
+      await Promise.all(
+        values.map(
+          async (value) =>
+            await readIntegrationOperationValue({
+              ...value,
+              ...(input.store ? { store: input.store } : {}),
+            }),
+        ),
+      ),
+  });
+  return async (value) => await reads.submit({ key: 'r2', value });
+}