deepline 0.2.53 → 0.2.55

package/dist/bundling-sources/shared_libs/play-runtime/docflow-node-io.ts

@@ -0,0 +1,634 @@
+import {
+  createSecretRedactionContext,
+  type SecretRedactionContext,
+} from './secret-redaction';
+
+export const DOCFLOW_NODE_IO_LIMITS = {
+  maxPaths: 16,
+  maxDepth: 3,
+  maxObjectFields: 12,
+  maxArrayItems: 3,
+  maxStringBytes: 256,
+  maxPhaseBytes: 2_000,
+  maxErrorBytes: 512,
+} as const;
+
+export type PlayDocflowNodeValuePreview =
+  | { kind: 'null' }
+  | { kind: 'scalar'; value: string | number | boolean }
+  | {
+      kind: 'dataset';
+      datasetId: string;
+      datasetKind: 'csv' | 'map';
+      tableNamespace?: string | null;
+      rowCount?: number;
+    }
+  | {
+      kind: 'array';
+      items: PlayDocflowNodeValuePreview[];
+      totalItems: number;
+      truncated?: boolean;
+    }
+  | {
+      kind: 'object';
+      fields: Record<string, PlayDocflowNodeValuePreview>;
+      totalFields: number;
+      truncated?: boolean;
+    }
+  | {
+      kind: 'unavailable';
+      reason:
+        | 'undefined'
+        | 'function'
+        | 'symbol'
+        | 'bigint'
+        | 'cycle'
+        | 'depth'
+        | 'access_error'
+        | 'limit'
+        | 'unsupported';
+    };
+
+export type PlayDocflowNodeIoPreviewMap = Record<
+  string,
+  PlayDocflowNodeValuePreview
+>;
+
+export type PlayDocflowNodeInputCapture = {
+  path: string;
+  /** Reads only the lexical root. TDZ and missing bindings are contained. */
+  readRoot: () => unknown;
+  /** Traversed through data descriptors so authored getters are never invoked. */
+  properties: readonly string[];
+};
+
+export type PlayDocflowNodeIoState = {
+  attempt: number;
+  invocationId?: string;
+  inputs?: PlayDocflowNodeIoPreviewMap;
+  outputs?: PlayDocflowNodeIoPreviewMap;
+  inputsTruncated?: boolean;
+  outputsTruncated?: boolean;
+  error?: string | null;
+};
+
+type PreviewOptions = {
+  redactor?: SecretRedactionContext;
+};
+
+const utf8Encoder = new TextEncoder();
+const PLAY_DATASET_BRAND = Symbol.for('deepline.play.dataset');
+
+/**
+ * Preview-map keys must be persistable everywhere the ledger lands, and Convex
+ * reserves field names beginning with '$' (a terminal flush carrying one 500s,
+ * dropping the WHOLE terminal event batch). `$output` is the wrapper's
+ * whole-result path; strip the sigil so previews survive every store. Applied
+ * at every build/normalize seam so live events, durable observations, and
+ * replayed historical payloads all converge on the safe representation.
+ */
+export function ledgerSafeDocflowPreviewKey(key: string): string {
+  if (!key.startsWith('$')) return key;
+  const stripped = key.replace(/^\$+/, '');
+  return stripped || 'output';
+}
+
+function utf8Bytes(value: string): number {
+  return utf8Encoder.encode(value).byteLength;
+}
+
+function truncateUtf8(value: string, maxBytes: number): string {
+  if (utf8Bytes(value) <= maxBytes) return value;
+  let output = '';
+  for (const character of value) {
+    if (utf8Bytes(`${output}${character}…`) > maxBytes) break;
+    output += character;
+  }
+  return `${output}…`;
+}
+
+function redactString(
+  redactor: SecretRedactionContext,
+  value: string,
+  key?: string,
+): string {
+  if (!key) return redactor.redactString(value);
+  const wrapped = redactor.redact({ [key]: value }) as Record<string, unknown>;
+  return typeof wrapped[key] === 'string'
+    ? wrapped[key]
+    : redactor.redactString(value);
+}
+
+function safeString(
+  value: unknown,
+  redactor: SecretRedactionContext,
+  key?: string,
+): string | undefined {
+  if (typeof value !== 'string') return undefined;
+  return truncateUtf8(
+    redactString(redactor, value, key),
+    DOCFLOW_NODE_IO_LIMITS.maxStringBytes,
+  );
+}
+
+function previewValue(
+  value: unknown,
+  input: {
+    depth: number;
+    ancestors: WeakSet<object>;
+    redactor: SecretRedactionContext;
+    key?: string;
+  },
+): PlayDocflowNodeValuePreview {
+  if (value === null) return { kind: 'null' };
+  if (value === undefined) return { kind: 'unavailable', reason: 'undefined' };
+  if (typeof value === 'string') {
+    return {
+      kind: 'scalar',
+      value: safeString(value, input.redactor, input.key) ?? '',
+    };
+  }
+  if (typeof value === 'number') {
+    return Number.isFinite(value)
+      ? { kind: 'scalar', value }
+      : { kind: 'unavailable', reason: 'unsupported' };
+  }
+  if (typeof value === 'boolean') return { kind: 'scalar', value };
+  if (typeof value === 'bigint') {
+    return { kind: 'unavailable', reason: 'bigint' };
+  }
+  if (typeof value === 'function') {
+    return { kind: 'unavailable', reason: 'function' };
+  }
+  if (typeof value === 'symbol') {
+    return { kind: 'unavailable', reason: 'symbol' };
+  }
+  if (typeof value !== 'object') {
+    return { kind: 'unavailable', reason: 'unsupported' };
+  }
+
+  try {
+    const brand = readDataProperty(value, PLAY_DATASET_BRAND);
+    const datasetKind = readDataProperty(value, 'datasetKind');
+    const datasetId = readDataProperty(value, 'datasetId');
+    const tableNamespace = readDataProperty(value, 'tableNamespace');
+    if (
+      brand.ok &&
+      brand.value === true &&
+      datasetKind.ok &&
+      (datasetKind.value === 'csv' || datasetKind.value === 'map') &&
+      datasetId.ok &&
+      typeof datasetId.value === 'string'
+    ) {
+      return {
+        kind: 'dataset',
+        datasetId:
+          safeString(datasetId.value, input.redactor, 'datasetId') ?? 'dataset',
+        datasetKind: datasetKind.value,
+        tableNamespace:
+          tableNamespace.ok && tableNamespace.value === null
+            ? null
+            : tableNamespace.ok
+              ? safeString(
+                  tableNamespace.value,
+                  input.redactor,
+                  'tableNamespace',
+                )
+              : undefined,
+      };
+    }
+    const kind = readDataProperty(value, 'kind');
+    const count = readDataProperty(value, 'count');
+    const preview = readDataProperty(value, 'preview');
+    if (
+      kind.ok &&
+      kind.value === 'dataset' &&
+      datasetKind.ok &&
+      (datasetKind.value === 'csv' || datasetKind.value === 'map') &&
+      datasetId.ok &&
+      typeof datasetId.value === 'string' &&
+      count.ok &&
+      typeof count.value === 'number' &&
+      preview.ok &&
+      Array.isArray(preview.value)
+    ) {
+      return {
+        kind: 'dataset',
+        datasetId:
+          safeString(datasetId.value, input.redactor, 'datasetId') ?? 'dataset',
+        datasetKind: datasetKind.value,
+        tableNamespace:
+          tableNamespace.ok && tableNamespace.value === null
+            ? null
+            : tableNamespace.ok
+              ? safeString(
+                  tableNamespace.value,
+                  input.redactor,
+                  'tableNamespace',
+                )
+              : undefined,
+        rowCount: Math.max(0, Math.floor(count.value)),
+      };
+    }
+  } catch {
+    return { kind: 'unavailable', reason: 'access_error' };
+  }
+
+  if (input.ancestors.has(value)) {
+    return { kind: 'unavailable', reason: 'cycle' };
+  }
+  if (input.depth >= DOCFLOW_NODE_IO_LIMITS.maxDepth) {
+    return { kind: 'unavailable', reason: 'depth' };
+  }
+
+  input.ancestors.add(value);
+  try {
+    if (Array.isArray(value)) {
+      const items: PlayDocflowNodeValuePreview[] = [];
+      for (
+        let index = 0;
+        index < Math.min(value.length, DOCFLOW_NODE_IO_LIMITS.maxArrayItems);
+        index += 1
+      ) {
+        const descriptor = Object.getOwnPropertyDescriptor(
+          value,
+          String(index),
+        );
+        items.push(
+          descriptor && 'value' in descriptor
+            ? previewValue(descriptor.value, {
+                ...input,
+                depth: input.depth + 1,
+              })
+            : { kind: 'unavailable', reason: 'access_error' },
+        );
+      }
+      return {
+        kind: 'array',
+        items,
+        totalItems: value.length,
+        ...(value.length > items.length ? { truncated: true } : {}),
+      };
+    }
+
+    let descriptors: Record<string, PropertyDescriptor>;
+    try {
+      descriptors = Object.getOwnPropertyDescriptors(value);
+    } catch {
+      return { kind: 'unavailable', reason: 'access_error' };
+    }
+    const entries = Object.entries(descriptors).filter(
+      ([, descriptor]) => descriptor.enumerable,
+    );
+    const selected = entries.slice(0, DOCFLOW_NODE_IO_LIMITS.maxObjectFields);
+    const fields: Record<string, PlayDocflowNodeValuePreview> = {};
+    for (const [key, descriptor] of selected) {
+      const safeKey = truncateUtf8(
+        ledgerSafeDocflowPreviewKey(input.redactor.redactString(key)),
+        DOCFLOW_NODE_IO_LIMITS.maxStringBytes,
+      );
+      fields[safeKey] =
+        'value' in descriptor
+          ? previewValue(descriptor.value, {
+              ...input,
+              depth: input.depth + 1,
+              key,
+            })
+          : { kind: 'unavailable', reason: 'access_error' };
+    }
+    return {
+      kind: 'object',
+      fields,
+      totalFields: entries.length,
+      ...(entries.length > selected.length ? { truncated: true } : {}),
+    };
+  } catch {
+    return { kind: 'unavailable', reason: 'access_error' };
+  } finally {
+    input.ancestors.delete(value);
+  }
+}
+
+export function buildPlayDocflowNodeIoPreviewMap(
+  values: Record<string, unknown>,
+  options: PreviewOptions = {},
+): { values: PlayDocflowNodeIoPreviewMap; truncated: boolean } {
+  const redactor = options.redactor ?? createSecretRedactionContext();
+  const previews: PlayDocflowNodeIoPreviewMap = {};
+  let truncated = false;
+  let entries: Array<[string, unknown]>;
+  try {
+    entries = Object.entries(values);
+  } catch {
+    return {
+      values: { value: { kind: 'unavailable', reason: 'access_error' } },
+      truncated: true,
+    };
+  }
+  for (const [path, value] of entries) {
+    if (Object.keys(previews).length >= DOCFLOW_NODE_IO_LIMITS.maxPaths) {
+      truncated = true;
+      break;
+    }
+    const safePath = truncateUtf8(
+      ledgerSafeDocflowPreviewKey(redactor.redactString(path)),
+      DOCFLOW_NODE_IO_LIMITS.maxStringBytes,
+    );
+    const preview = previewValue(value, {
+      depth: 0,
+      ancestors: new WeakSet(),
+      redactor,
+      key: path.split('.').at(-1),
+    });
+    const candidate = { ...previews, [safePath]: preview };
+    if (
+      utf8Bytes(JSON.stringify(candidate)) >
+      DOCFLOW_NODE_IO_LIMITS.maxPhaseBytes
+    ) {
+      truncated = true;
+      break;
+    }
+    previews[safePath] = preview;
+  }
+  if (entries.length > Object.keys(previews).length) truncated = true;
+  return { values: previews, truncated };
+}
+
+function readDataProperty(
+  value: unknown,
+  property: PropertyKey,
+): {
+  ok: boolean;
+  value?: unknown;
+} {
+  if (
+    (typeof value !== 'object' || value === null) &&
+    typeof value !== 'function'
+  ) {
+    return { ok: false };
+  }
+  try {
+    let owner: object | null = value as object;
+    while (owner) {
+      const descriptor = Object.getOwnPropertyDescriptor(owner, property);
+      if (descriptor) {
+        return 'value' in descriptor
+          ? { ok: true, value: descriptor.value }
+          : { ok: false };
+      }
+      owner = Object.getPrototypeOf(owner) as object | null;
+    }
+  } catch {
+    return { ok: false };
+  }
+  return { ok: false };
+}
+
+export function buildPlayDocflowNodeInputPreviewMap(
+  captures: readonly PlayDocflowNodeInputCapture[],
+  options: PreviewOptions = {},
+): { values: PlayDocflowNodeIoPreviewMap; truncated: boolean } {
+  const redactor = options.redactor ?? createSecretRedactionContext();
+  const values: PlayDocflowNodeIoPreviewMap = {};
+  let truncated = false;
+  for (const capture of captures.slice(0, DOCFLOW_NODE_IO_LIMITS.maxPaths)) {
+    let current: unknown;
+    let available = true;
+    try {
+      current = capture.readRoot();
+    } catch {
+      available = false;
+    }
+    if (available) {
+      for (const property of capture.properties) {
+        const next = readDataProperty(current, property);
+        if (!next.ok) {
+          available = false;
+          break;
+        }
+        current = next.value;
+      }
+    }
+    const safePath = truncateUtf8(
+      ledgerSafeDocflowPreviewKey(redactor.redactString(capture.path)),
+      DOCFLOW_NODE_IO_LIMITS.maxStringBytes,
+    );
+    const preview: PlayDocflowNodeValuePreview = available
+      ? previewValue(current, {
+          depth: 0,
+          ancestors: new WeakSet(),
+          redactor,
+          key: capture.properties.at(-1) ?? capture.path,
+        })
+      : { kind: 'unavailable', reason: 'access_error' };
+    const candidate = { ...values, [safePath]: preview };
+    if (
+      utf8Bytes(JSON.stringify(candidate)) >
+      DOCFLOW_NODE_IO_LIMITS.maxPhaseBytes
+    ) {
+      truncated = true;
+      break;
+    }
+    values[safePath] = preview;
+  }
+  if (captures.length > Object.keys(values).length) truncated = true;
+  return { values, truncated };
+}
+
+export function buildPlayDocflowNodeErrorPreview(
+  error: unknown,
+  redactor: SecretRedactionContext = createSecretRedactionContext(),
+): string {
+  try {
+    const message = error instanceof Error ? error.message : String(error);
+    return truncateUtf8(
+      redactor.redactString(message),
+      DOCFLOW_NODE_IO_LIMITS.maxErrorBytes,
+    );
+  } catch {
+    return 'Non-Error value thrown (message unavailable)';
+  }
+}
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+  return Boolean(value && typeof value === 'object' && !Array.isArray(value));
+}
+
+function normalizePreview(
+  value: unknown,
+  depth: number,
+): PlayDocflowNodeValuePreview | null {
+  if (!isRecord(value) || typeof value.kind !== 'string') return null;
+  if (value.kind === 'null') return { kind: 'null' };
+  if (value.kind === 'scalar') {
+    if (
+      typeof value.value === 'string' ||
+      typeof value.value === 'boolean' ||
+      (typeof value.value === 'number' && Number.isFinite(value.value))
+    ) {
+      return previewValue(value.value, {
+        depth: 0,
+        ancestors: new WeakSet(),
+        redactor: createSecretRedactionContext(),
+      });
+    }
+    return null;
+  }
+  if (value.kind === 'unavailable') {
+    const reason = value.reason;
+    return typeof reason === 'string' &&
+      [
+        'undefined',
+        'function',
+        'symbol',
+        'bigint',
+        'cycle',
+        'depth',
+        'access_error',
+        'limit',
+        'unsupported',
+      ].includes(reason)
+      ? {
+          kind: 'unavailable',
+          reason: reason as Extract<
+            PlayDocflowNodeValuePreview,
+            { kind: 'unavailable' }
+          >['reason'],
+        }
+      : null;
+  }
+  if (value.kind === 'dataset') {
+    const datasetId = safeString(
+      value.datasetId,
+      createSecretRedactionContext(),
+      'datasetId',
+    );
+    if (
+      !datasetId ||
+      (value.datasetKind !== 'csv' && value.datasetKind !== 'map')
+    ) {
+      return null;
+    }
+    return {
+      kind: 'dataset',
+      datasetId,
+      datasetKind: value.datasetKind,
+      ...(value.tableNamespace === null
+        ? { tableNamespace: null }
+        : typeof value.tableNamespace === 'string'
+          ? {
+              tableNamespace: safeString(
+                value.tableNamespace,
+                createSecretRedactionContext(),
+                'tableNamespace',
+              ),
+            }
+          : {}),
+      ...(typeof value.rowCount === 'number' && Number.isFinite(value.rowCount)
+        ? { rowCount: Math.max(0, Math.floor(value.rowCount)) }
+        : {}),
+    };
+  }
+  if (depth >= DOCFLOW_NODE_IO_LIMITS.maxDepth) {
+    return { kind: 'unavailable', reason: 'depth' };
+  }
+  if (value.kind === 'array' && Array.isArray(value.items)) {
+    const items = value.items
+      .slice(0, DOCFLOW_NODE_IO_LIMITS.maxArrayItems)
+      .map((item) => normalizePreview(item, depth + 1))
+      .filter((item): item is PlayDocflowNodeValuePreview => item !== null);
+    const totalItems =
+      typeof value.totalItems === 'number' && Number.isFinite(value.totalItems)
+        ? Math.max(items.length, Math.floor(value.totalItems))
+        : items.length;
+    return {
+      kind: 'array',
+      items,
+      totalItems,
+      ...(value.truncated === true || totalItems > items.length
+        ? { truncated: true }
+        : {}),
+    };
+  }
+  if (value.kind === 'object' && isRecord(value.fields)) {
+    const fields: Record<string, PlayDocflowNodeValuePreview> = {};
+    for (const [key, field] of Object.entries(value.fields).slice(
+      0,
+      DOCFLOW_NODE_IO_LIMITS.maxObjectFields,
+    )) {
+      const normalized = normalizePreview(field, depth + 1);
+      if (normalized) fields[ledgerSafeDocflowPreviewKey(key)] = normalized;
+    }
+    const totalFields =
+      typeof value.totalFields === 'number' &&
+      Number.isFinite(value.totalFields)
+        ? Math.max(Object.keys(fields).length, Math.floor(value.totalFields))
+        : Object.keys(fields).length;
+    return {
+      kind: 'object',
+      fields,
+      totalFields,
+      ...(value.truncated === true || totalFields > Object.keys(fields).length
+        ? { truncated: true }
+        : {}),
+    };
+  }
+  return null;
+}
+
+export function normalizePlayDocflowNodeIoState(
+  value: unknown,
+): PlayDocflowNodeIoState | undefined {
+  if (!isRecord(value)) return undefined;
+  const attempt =
+    typeof value.attempt === 'number' && Number.isFinite(value.attempt)
+      ? Math.max(0, Math.floor(value.attempt))
+      : 0;
+  const normalizeMap = (
+    raw: unknown,
+  ): PlayDocflowNodeIoPreviewMap | undefined => {
+    if (!isRecord(raw)) return undefined;
+    const output: PlayDocflowNodeIoPreviewMap = {};
+    for (const [path, preview] of Object.entries(raw).slice(
+      0,
+      DOCFLOW_NODE_IO_LIMITS.maxPaths,
+    )) {
+      const normalized = normalizePreview(preview, 0);
+      if (!normalized) continue;
+      const safePath = ledgerSafeDocflowPreviewKey(path);
+      const candidate = { ...output, [safePath]: normalized };
+      if (
+        utf8Bytes(JSON.stringify(candidate)) >
+        DOCFLOW_NODE_IO_LIMITS.maxPhaseBytes
+      ) {
+        break;
+      }
+      output[safePath] = normalized;
+    }
+    return output;
+  };
+  const inputs = normalizeMap(value.inputs);
+  const outputs = normalizeMap(value.outputs);
+  const error =
+    typeof value.error === 'string'
+      ? buildPlayDocflowNodeErrorPreview(value.error)
+      : value.error === null
+        ? null
+        : undefined;
+  return {
+    attempt,
+    ...(typeof value.invocationId === 'string' && value.invocationId.trim()
+      ? {
+          invocationId: truncateUtf8(
+            value.invocationId.trim(),
+            DOCFLOW_NODE_IO_LIMITS.maxStringBytes,
+          ),
+        }
+      : {}),
+    ...(inputs ? { inputs } : {}),
+    ...(outputs ? { outputs } : {}),
+    ...(value.inputsTruncated === true ? { inputsTruncated: true } : {}),
+    ...(value.outputsTruncated === true ? { outputsTruncated: true } : {}),
+    ...(error !== undefined ? { error } : {}),
+  };
+}

package/dist/bundling-sources/shared_libs/play-runtime/docflow-observation.ts

@@ -0,0 +1,64 @@
+/**
+ * Durable docflow observation record (ADR 0016 rule 2).
+ *
+ * An observation is one authored-node invocation recorded on the same durable
+ * wire receipts use (the runtime receipt gateway) into the scheduler schema's
+ * `observations` table. It is keyed like a receipt — org/run scope proven by the
+ * executor token, fenced by the per-attempt authority — plus the node id and the
+ * wrapper's per-attempt invocation sequence.
+ *
+ * Observations are a projection of the live `docflow.node.*` execution events
+ * onto durable storage; the wrapper still emits those events for live streaming.
+ * They carry only BOUNDED input/output previews (per `docflow-node-io`), never
+ * per-row map truth — maps observe at the node level while per-row outcomes live
+ * in the sheets.
+ */
+import type { PlayDocflowNodeIoPreviewMap } from './docflow-node-io';
+
+export type DocflowObservationStatus = 'started' | 'completed' | 'failed';
+
+/**
+ * A single upsert of one node invocation's state. The gateway upserts on
+ * (run_id, attempt, node_id, invocation_seq); a later `completed`/`failed`
+ * write for the same key supersedes an earlier `started` (status precedence
+ * started < completed/failed, never regressing to `started`).
+ */
+export type DocflowObservationUpsert = {
+  runId: string;
+  attempt: number;
+  nodeId: string;
+  invocationSeq: number;
+  invocationId?: string | null;
+  status: DocflowObservationStatus;
+  inputs?: PlayDocflowNodeIoPreviewMap | null;
+  outputs?: PlayDocflowNodeIoPreviewMap | null;
+  inputsTruncated?: boolean;
+  outputsTruncated?: boolean;
+  error?: string | null;
+  /** Epoch ms; started writes carry it, terminal writes carry a settled ts. */
+  at: number;
+};
+
+/** A durable observation row read back for terminal ledger reconciliation. */
+export type DocflowObservationRecord = {
+  runId: string;
+  attempt: number;
+  nodeId: string;
+  invocationSeq: number;
+  invocationId: string | null;
+  status: DocflowObservationStatus;
+  inputs: PlayDocflowNodeIoPreviewMap | null;
+  outputs: PlayDocflowNodeIoPreviewMap | null;
+  inputsTruncated: boolean;
+  outputsTruncated: boolean;
+  error: string | null;
+  startedAt: number | null;
+  settledAt: number | null;
+};
+
+/** Numeric precedence so a late `started` never clobbers a settled write. */
+export function docflowObservationStatusRank(
+  status: DocflowObservationStatus,
+): number {
+  return status === 'started' ? 0 : 1;
+}

package/dist/bundling-sources/shared_libs/play-runtime/dynamic-worker-version.ts

@@ -1,2 +1,2 @@
 export const DYNAMIC_PLAY_WORKER_ARTIFACT_VERSION =
-  'h24-map-dispatch-batch-metadata';
+  'h26-docflow-node-io-runtime';