deepline 0.3.461 → 0.3.463

package/dist/bundling-sources/shared_libs/play-runtime/array-input-store.ts

@@ -0,0 +1,319 @@
+import { createHash } from 'node:crypto';
+import { setPlayDatasetSourceRowIdentity } from '../plays/dataset';
+import { stableStringify } from '../plays/row-identity';
+import { getCsvProjectedFields } from './csv-rename';
+
+/** Durable, bounded source snapshots for arrays that exceed map materialization memory. */
+export type RuntimeArrayInputScope = {
+  playName: string;
+  runId: string;
+  tableNamespace: string;
+  mapInvocationId: string;
+};
+
+export type RuntimeArrayInputOccurrence = {
+  data: Record<string, unknown>;
+  sourceIdentity?: { datasetId: string; rowIndex: number };
+};
+
+export type RuntimeArrayInputManifest =
+  | { state: 'absent' }
+  | {
+      state: 'staging';
+      sourceFingerprint: string;
+      sourceRows: number;
+      hasSourceIdentities: boolean;
+    }
+  | {
+      state: 'sealed';
+      sourceFingerprint: string;
+      sourceRows: number;
+      pageCount: number;
+      hasSourceIdentities: boolean;
+    };
+
+export type RuntimeArrayInputStore = {
+  getManifest(
+    scope: RuntimeArrayInputScope,
+  ): Promise<RuntimeArrayInputManifest>;
+  begin(
+    input: RuntimeArrayInputScope & {
+      sourceFingerprint: string;
+      sourceRows: number;
+      hasSourceIdentities: boolean;
+    },
+  ): Promise<void>;
+  putPage(
+    input: RuntimeArrayInputScope & {
+      sourceFingerprint: string;
+      ordinal: number;
+      offset: number;
+      rows: RuntimeArrayInputOccurrence[];
+    },
+  ): Promise<void>;
+  seal(
+    input: RuntimeArrayInputScope & {
+      sourceFingerprint: string;
+      sourceRows: number;
+      pageCount: number;
+    },
+  ): Promise<void>;
+  readPage(
+    input: RuntimeArrayInputScope & { ordinal: number },
+  ): Promise<{ offset: number; rows: RuntimeArrayInputOccurrence[] }>;
+};
+
+// Durable array pages use JSONB. Decline a fresh snapshot when its authored
+// values would change under that encoding; the existing materialization path
+// then preserves the original value or raises its existing memory limit.
+export function isLosslessArrayInputValue(value: unknown): boolean {
+  const seen = new WeakSet<object>();
+  const validString = (text: string) => {
+    for (let index = 0; index < text.length; index += 1) {
+      const unit = text.charCodeAt(index);
+      if (unit === 0) return false;
+      if (unit >= 0xd800 && unit <= 0xdbff) {
+        const next = text.charCodeAt(++index);
+        if (!(next >= 0xdc00 && next <= 0xdfff)) return false;
+      } else if (unit >= 0xdc00 && unit <= 0xdfff) {
+        return false;
+      }
+    }
+    return true;
+  };
+  const visit = (current: unknown): boolean => {
+    if (current === null || typeof current === 'boolean') return true;
+    if (typeof current === 'string') return validString(current);
+    if (typeof current === 'number')
+      return Number.isFinite(current) && !Object.is(current, -0);
+    if (typeof current !== 'object' || seen.has(current)) return false;
+    seen.add(current);
+    if (Array.isArray(current)) {
+      if (Object.getPrototypeOf(current) !== Array.prototype) return false;
+      if (Object.getOwnPropertySymbols(current).length > 0) return false;
+      if (Object.getOwnPropertyNames(current).length !== current.length + 1)
+        return false;
+      for (let index = 0; index < current.length; index += 1) {
+        const descriptor = Object.getOwnPropertyDescriptor(current, index);
+        if (
+          !descriptor ||
+          !('value' in descriptor) ||
+          !descriptor.enumerable ||
+          !descriptor.writable ||
+          !descriptor.configurable ||
+          !visit(descriptor.value)
+        )
+          return false;
+      }
+      return true;
+    }
+    if (Object.getPrototypeOf(current) !== Object.prototype) return false;
+    const row = current as Record<string, unknown>;
+    const projected = getCsvProjectedFields(row);
+    const symbols = Object.getOwnPropertySymbols(current);
+    if (
+      symbols.some(
+        (symbol) =>
+          symbol !== Symbol.for('deepline.play.csv.projected_fields') ||
+          !projected,
+      )
+    )
+      return false;
+    for (const key of Object.getOwnPropertyNames(current)) {
+      if (!validString(key)) return false;
+      const descriptor = Object.getOwnPropertyDescriptor(current, key);
+      if (!descriptor || !('value' in descriptor)) return false;
+      if (!descriptor.writable || !descriptor.configurable) return false;
+      if (!descriptor.enumerable && !projected?.has(key)) return false;
+      if (!visit(descriptor.value)) return false;
+    }
+    return true;
+  };
+  return visit(value);
+}
+
+/** Only the exact old-receiver unsupported-action response may retain legacy admission. */
+export class RuntimeArrayInputStoreUnavailableError extends Error {
+  constructor() {
+    super('Runtime array input store is unavailable on this receiver.');
+    this.name = 'RuntimeArrayInputStoreUnavailableError';
+  }
+}
+
+const MAX_PAGE_ROWS = 1_000;
+// The backend checks the complete JSON envelope against 1 MiB. Keep room for
+// scope, field names and transport framing without reducing the row memory cap.
+const MAX_PAGE_ROW_BYTES = 1024 * 1024 - 64 * 1024;
+
+function sourceFingerprint<T>(input: {
+  rows: readonly T[];
+  normalize: (row: T, index: number) => RuntimeArrayInputOccurrence;
+}): {
+  digest: string;
+  hasSourceIdentities: boolean;
+  largestRowMemoryBytes: number;
+} {
+  const hash = createHash('sha256');
+  let hasSourceIdentities = false;
+  let largestRowMemoryBytes = 0;
+  hash.update(`runtime-array-input-v1:${input.rows.length}:`);
+  for (let index = 0; index < input.rows.length; index += 1) {
+    const occurrence = input.normalize(input.rows[index]!, index);
+    hasSourceIdentities ||= occurrence.sourceIdentity !== undefined;
+    const serialized = stableStringify(occurrence);
+    const bytes = Buffer.byteLength(serialized, 'utf8');
+    largestRowMemoryBytes = Math.max(largestRowMemoryBytes, bytes * 4 + 512);
+    hash.update(`${bytes}:`);
+    hash.update(serialized);
+  }
+  return {
+    digest: hash.digest('hex'),
+    hasSourceIdentities,
+    largestRowMemoryBytes,
+  };
+}
+
+/** Stage every occurrence before the first sheet admission or resolver call. */
+export async function sealRuntimeArrayInput<T>(input: {
+  scope: RuntimeArrayInputScope;
+  rows: readonly T[];
+  store: RuntimeArrayInputStore;
+  normalize: (row: T, index: number) => RuntimeArrayInputOccurrence;
+  manifest: RuntimeArrayInputManifest;
+  maxPageMemoryBytes?: number;
+}): Promise<Extract<RuntimeArrayInputManifest, { state: 'sealed' }>> {
+  if (input.manifest.state === 'sealed') return input.manifest;
+
+  const source = sourceFingerprint(input);
+  if (
+    input.maxPageMemoryBytes !== undefined &&
+    source.largestRowMemoryBytes > input.maxPageMemoryBytes
+  ) {
+    throw new Error(
+      `Dataset ${input.scope.tableNamespace} has a row that exceeds the remaining map memory budget for a durable input page.`,
+    );
+  }
+  if (
+    input.manifest.state === 'staging' &&
+    (input.manifest.sourceFingerprint !== source.digest ||
+      input.manifest.sourceRows !== input.rows.length ||
+      input.manifest.hasSourceIdentities !== source.hasSourceIdentities)
+  ) {
+    throw new Error(
+      `Dataset ${input.scope.tableNamespace} source changed before its array input was sealed. Retry with the original source.`,
+    );
+  }
+  await input.store.begin({
+    ...input.scope,
+    sourceFingerprint: source.digest,
+    sourceRows: input.rows.length,
+    hasSourceIdentities: source.hasSourceIdentities,
+  });
+
+  const secondPass = createHash('sha256');
+  secondPass.update(`runtime-array-input-v1:${input.rows.length}:`);
+  let pageRows: RuntimeArrayInputOccurrence[] = [];
+  let pageBytes = 0;
+  let pageMemoryBytes = 0;
+  let pageOffset = 0;
+  let pageCount = 0;
+  const flushPage = async () => {
+    if (pageRows.length === 0) return;
+    await input.store.putPage({
+      ...input.scope,
+      sourceFingerprint: source.digest,
+      ordinal: pageCount,
+      offset: pageOffset,
+      rows: pageRows,
+    });
+    pageOffset += pageRows.length;
+    pageCount += 1;
+    pageRows = [];
+    pageBytes = 0;
+    pageMemoryBytes = 0;
+  };
+  for (let index = 0; index < input.rows.length; index += 1) {
+    const occurrence = input.normalize(input.rows[index]!, index);
+    const serialized = stableStringify(occurrence);
+    const bytes = Buffer.byteLength(serialized, 'utf8');
+    const rowMemoryBytes = bytes * 4 + 512;
+    secondPass.update(`${bytes}:`);
+    secondPass.update(serialized);
+    if (bytes > MAX_PAGE_ROW_BYTES) {
+      throw new Error(
+        `Dataset ${input.scope.tableNamespace} row ${index} exceeds the 1 MiB durable input page limit.`,
+      );
+    }
+    if (
+      input.maxPageMemoryBytes !== undefined &&
+      rowMemoryBytes > input.maxPageMemoryBytes
+    ) {
+      throw new Error(
+        `Dataset ${input.scope.tableNamespace} row ${index} exceeds the remaining map memory budget for a durable input page.`,
+      );
+    }
+    if (
+      pageRows.length > 0 &&
+      (pageRows.length >= MAX_PAGE_ROWS ||
+        pageBytes + bytes > MAX_PAGE_ROW_BYTES ||
+        (input.maxPageMemoryBytes !== undefined &&
+          pageMemoryBytes + rowMemoryBytes > input.maxPageMemoryBytes))
+    ) {
+      await flushPage();
+    }
+    pageRows.push(occurrence);
+    pageBytes += bytes;
+    pageMemoryBytes += rowMemoryBytes;
+  }
+  await flushPage();
+  if (secondPass.digest('hex') !== source.digest) {
+    throw new Error(
+      `Dataset ${input.scope.tableNamespace} source changed while staging its array input. Retry with the original source.`,
+    );
+  }
+  await input.store.seal({
+    ...input.scope,
+    sourceFingerprint: source.digest,
+    sourceRows: input.rows.length,
+    pageCount,
+  });
+  return {
+    state: 'sealed',
+    sourceFingerprint: source.digest,
+    sourceRows: input.rows.length,
+    pageCount,
+    hasSourceIdentities: source.hasSourceIdentities,
+  };
+}
+
+/** Read the sealed sequence by ordinal, retaining only one backend page. */
+export async function* iterateSealedRuntimeArrayInput(
+  store: RuntimeArrayInputStore,
+  scope: RuntimeArrayInputScope,
+  manifest: Extract<RuntimeArrayInputManifest, { state: 'sealed' }>,
+): AsyncIterable<Record<string, unknown>> {
+  let offset = 0;
+  for (let ordinal = 0; ordinal < manifest.pageCount; ordinal += 1) {
+    const page = await store.readPage({ ...scope, ordinal });
+    if (page.offset !== offset || page.rows.length === 0) {
+      throw new Error(
+        `Dataset ${scope.tableNamespace} durable input page ${ordinal} is missing or out of order.`,
+      );
+    }
+    for (const occurrence of page.rows) {
+      const row = occurrence.sourceIdentity
+        ? setPlayDatasetSourceRowIdentity(
+            occurrence.data,
+            occurrence.sourceIdentity,
+          )
+        : occurrence.data;
+      yield row;
+    }
+    offset += page.rows.length;
+  }
+  if (offset !== manifest.sourceRows) {
+    throw new Error(
+      `Dataset ${scope.tableNamespace} durable input ended at ${offset}/${manifest.sourceRows} rows.`,
+    );
+  }
+}