deepline 0.3.392 → 0.3.394

package/dist/bundling-sources/shared_libs/integration-contracts/request-warnings.ts

@@ -0,0 +1,64 @@
+/** Declarative request predicates authored by the server, with no provider knowledge in clients. */
+export type ToolRequestCondition =
+  | {
+      field: string;
+      operator: 'equals';
+      value: string | number | boolean | null;
+    }
+  | { field: string; operator: 'number_equals'; value: number }
+  | { field: string; operator: 'present_nonblank' }
+  | { all: readonly ToolRequestCondition[] }
+  | { any: readonly ToolRequestCondition[] };
+
+export type ToolRequestWarningRule = {
+  code: string;
+  /** Customer-safe Deepline billing guidance; never upstream spend. */
+  message: string;
+  when: ToolRequestCondition;
+};
+
+export function matchesToolRequestCondition(
+  condition: ToolRequestCondition,
+  payload: Record<string, unknown> | null | undefined,
+): boolean {
+  if ('all' in condition) {
+    return condition.all.every((entry) =>
+      matchesToolRequestCondition(entry, payload),
+    );
+  }
+  if ('any' in condition) {
+    return condition.any.some((entry) =>
+      matchesToolRequestCondition(entry, payload),
+    );
+  }
+  const value = payload?.[condition.field];
+  switch (condition.operator) {
+    case 'equals':
+      return value === condition.value;
+    case 'number_equals':
+      try {
+        return Number(value) === condition.value;
+      } catch {
+        // Metadata warnings run before server input validation. A malformed
+        // JSON object must still reach that structured validation boundary.
+        return false;
+      }
+    case 'present_nonblank':
+      // A present non-string value is significant, including an empty array.
+      // Provider policies choose this predicate explicitly when that is their
+      // existing request contract; clients must not reinterpret it as truthy.
+      return typeof value === 'string'
+        ? value.trim().length > 0
+        : value !== undefined && value !== null;
+    default:
+      throw new Error('Unsupported tool request warning condition.');
+  }
+}
+
+/** Ordered first-match semantics preserve provider precedence and emit at most one warning. */
+export function firstMatchingToolRequestWarning(
+  rules: readonly ToolRequestWarningRule[] | null | undefined,
+  payload: Record<string, unknown> | null | undefined,
+): ToolRequestWarningRule | undefined {
+  return rules?.find((rule) => matchesToolRequestCondition(rule.when, payload));
+}

package/dist/bundling-sources/shared_libs/play-data-plane/runtime-sheet-storage-contract.ts

@@ -1,5 +1,6 @@
 /** Compatibility seam for callers that used the cell-meta-only contract. */
 import { RUNTIME_SHEET_CELL_META_COLUMN } from './runtime-sheet-orm';
+import type { RuntimeSheetOrmQuery } from './runtime-sheet-orm';
 
 export { RUNTIME_SHEET_CELL_META_COLUMN } from './runtime-sheet-orm';
 
@@ -45,3 +46,317 @@ export function isRuntimeSheetCellMetaColumnReady(
     state.column_default != null
   );
 }
+
+/** Expand-only immutable history. Writers enable after tenant migration proof. */
+export const RUNTIME_SHEET_RUN_MEMBERSHIP_TABLE =
+  '_deepline_run_row_memberships';
+export const RUNTIME_SHEET_RUN_MEMBERSHIP_SCHEMA_VERSION = 9;
+/** Reader/expand release is inert; enable in its independently reviewed PR. */
+export const RUNTIME_SHEET_RUN_MEMBERSHIP_WRITES_ENABLED = false;
+
+/**
+ * Optional history is available only after all expansion DDL and grants land.
+ * Inspect its own bounded capability; never consult the admission schema floor.
+ */
+export async function isRuntimeSheetRunMembershipStorageAvailable(
+  query: RuntimeSheetOrmQuery,
+  schema = 'storage',
+  requireCaptureAccess = false,
+): Promise<boolean> {
+  const result = await query<{ available: boolean }>(
+    `WITH relations AS (
+      SELECT c.oid, c.relname, c.relkind, n.oid AS schema_oid
+      FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace
+      WHERE n.nspname = $1 AND c.relname IN ('${RUNTIME_SHEET_RUN_MEMBERSHIP_TABLE}', '_deepline_run_datasets')
+    ), required_columns (relation, name, type, required) AS (
+      VALUES
+        ('${RUNTIME_SHEET_RUN_MEMBERSHIP_TABLE}', 'run_id', 'text', true),
+        ('${RUNTIME_SHEET_RUN_MEMBERSHIP_TABLE}', 'dataset_id', 'text', true),
+        ('${RUNTIME_SHEET_RUN_MEMBERSHIP_TABLE}', 'row_key', 'text', true),
+        ('${RUNTIME_SHEET_RUN_MEMBERSHIP_TABLE}', 'phase', 'text', true),
+        ('${RUNTIME_SHEET_RUN_MEMBERSHIP_TABLE}', 'generation', 'text', true),
+        ('${RUNTIME_SHEET_RUN_MEMBERSHIP_TABLE}', 'attempt_seq', 'integer', false),
+        ('${RUNTIME_SHEET_RUN_MEMBERSHIP_TABLE}', 'row_version', 'bigint', false),
+        ('${RUNTIME_SHEET_RUN_MEMBERSHIP_TABLE}', 'input_index', 'bigint', false),
+        ('${RUNTIME_SHEET_RUN_MEMBERSHIP_TABLE}', 'input_digest', 'text', true),
+        ('${RUNTIME_SHEET_RUN_MEMBERSHIP_TABLE}', 'status', 'text', true),
+        ('${RUNTIME_SHEET_RUN_MEMBERSHIP_TABLE}', 'row_data', 'jsonb', true),
+        ('${RUNTIME_SHEET_RUN_MEMBERSHIP_TABLE}', 'source_ref', 'jsonb', true),
+        ('${RUNTIME_SHEET_RUN_MEMBERSHIP_TABLE}', 'recorded_at', 'timestamp with time zone', true),
+        ('_deepline_run_datasets', 'membership_expected_rows', 'bigint', false),
+        ('_deepline_run_datasets', 'membership_started_at', 'timestamp with time zone', false),
+        ('_deepline_run_datasets', 'membership_completed_at', 'timestamp with time zone', false)
+    )
+    SELECT coalesce(
+      (SELECT count(*) = 2 FROM relations WHERE relkind = 'r')
+      AND NOT EXISTS (
+        SELECT 1 FROM required_columns required
+        LEFT JOIN relations relation ON relation.relname = required.relation
+        LEFT JOIN pg_attribute attribute ON attribute.attrelid = relation.oid
+          AND attribute.attname = required.name AND NOT attribute.attisdropped
+        WHERE attribute.attnum IS NULL OR attribute.attnum <= 0
+          OR format_type(attribute.atttypid, attribute.atttypmod) <> required.type
+          OR attribute.attnotnull <> required.required
+      )
+      AND EXISTS (
+        SELECT 1 FROM relations relation JOIN pg_attribute attribute ON attribute.attrelid = relation.oid
+        JOIN pg_attrdef column_default ON column_default.adrelid = relation.oid AND column_default.adnum = attribute.attnum
+        WHERE relation.relname = '${RUNTIME_SHEET_RUN_MEMBERSHIP_TABLE}' AND attribute.attname = 'recorded_at' AND NOT attribute.attisdropped
+      )
+      AND EXISTS (
+        SELECT 1 FROM relations relation JOIN pg_constraint constraint_row ON constraint_row.conrelid = relation.oid
+        WHERE relation.relname = '${RUNTIME_SHEET_RUN_MEMBERSHIP_TABLE}' AND constraint_row.contype = 'p'
+          AND (SELECT array_agg(attribute.attname::text ORDER BY keys.ordinality)
+            FROM unnest(constraint_row.conkey) WITH ORDINALITY keys(attnum, ordinality)
+            JOIN pg_attribute attribute ON attribute.attrelid = relation.oid AND attribute.attnum = keys.attnum)
+            = ARRAY['run_id', 'dataset_id', 'row_key', 'phase', 'generation']::text[]
+      )
+      AND NOT EXISTS (
+        SELECT 1 FROM relations relation
+        WHERE NOT has_schema_privilege(relation.schema_oid, 'USAGE')
+          OR NOT has_table_privilege(relation.oid, 'SELECT')
+          OR ($2::boolean AND relation.relname = '${RUNTIME_SHEET_RUN_MEMBERSHIP_TABLE}' AND NOT has_table_privilege(relation.oid, 'INSERT'))
+          OR ($2::boolean AND relation.relname = '${RUNTIME_SHEET_RUN_MEMBERSHIP_TABLE}' AND NOT has_table_privilege(relation.oid, 'UPDATE'))
+          OR ($2::boolean AND relation.relname = '_deepline_run_datasets' AND NOT has_table_privilege(relation.oid, 'UPDATE'))
+      ), false) AS available`,
+    [schema, requireCaptureAccess],
+  );
+  return result.rows[0]?.available === true;
+}
+
+export type RuntimeSheetRunRowMembership = {
+  runId: string;
+  datasetId: string;
+  rowKey: string;
+  phase: 'admitted' | 'terminal';
+  /** Durable terminal row version distinguishes repair within an attempt. */
+  generation: string;
+  attemptSeq: number | null;
+  rowVersion: number | null;
+  inputIndex: number | null;
+  inputDigest: string;
+  status: 'pending' | 'enriched' | 'failed' | 'cancelled';
+  data: Record<string, unknown>;
+  source: Record<string, unknown>;
+  recordedAt: string;
+};
+
+export type RuntimeSheetRunRowMembershipPage = {
+  entries: RuntimeSheetRunRowMembership[];
+  nextCursor: string | null;
+  coverage: {
+    complete: boolean;
+    expectedRows: number | null;
+    captureStartedAt: string | null;
+    captureCompletedAt: string | null;
+    reason:
+      | 'storage_not_expanded'
+      | 'legacy_or_capture_not_enabled'
+      | 'capture_in_progress'
+      | 'complete';
+  };
+};
+
+/** The existing sheet/history erasure owner clears snapshots atomically too. */
+export async function clearRuntimeSheetRunRowMembership(
+  query: RuntimeSheetOrmQuery,
+  input: { playName: string; tableNamespace: string },
+): Promise<void> {
+  const relation = await query<{ present: boolean }>(
+    'SELECT to_regclass($1) IS NOT NULL AS present',
+    [`storage.${RUNTIME_SHEET_RUN_MEMBERSHIP_TABLE}`],
+  );
+  if (relation.rows[0]?.present !== true) return;
+  await query(
+    `DELETE FROM storage.${RUNTIME_SHEET_RUN_MEMBERSHIP_TABLE} history
+    USING storage._deepline_run_datasets catalog
+    WHERE history.run_id = catalog.run_id AND history.dataset_id = catalog.dataset_id
+      AND catalog.play_name = $1 AND catalog.table_namespace = $2`,
+    [input.playName, input.tableNamespace],
+  );
+  await query(
+    `UPDATE storage._deepline_run_datasets
+    SET membership_expected_rows = NULL, membership_started_at = NULL, membership_completed_at = NULL
+    WHERE play_name = $1 AND table_namespace = $2`,
+    [input.playName, input.tableNamespace],
+  );
+}
+
+function membershipCursor(
+  value: string | undefined,
+): { rowKey: string; phase: string; generation: string } | null {
+  if (!value) return null;
+  try {
+    const parsed = JSON.parse(Buffer.from(value, 'base64url').toString('utf8'));
+    if (
+      parsed.version !== 1 ||
+      typeof parsed.rowKey !== 'string' ||
+      !['admitted', 'terminal'].includes(parsed.phase) ||
+      typeof parsed.generation !== 'string'
+    )
+      throw new Error();
+    return {
+      rowKey: parsed.rowKey,
+      phase: parsed.phase,
+      generation: parsed.generation,
+    };
+  } catch {
+    throw new Error('Invalid run membership cursor.');
+  }
+}
+
+/** Read customer rows behind a cursor, never embed them in run/status metadata. */
+export async function readRuntimeSheetRunRowMembershipPage(
+  query: RuntimeSheetOrmQuery,
+  input: { runId: string; datasetId: string; limit?: number; cursor?: string },
+): Promise<RuntimeSheetRunRowMembershipPage> {
+  const limit = input.limit ?? 200;
+  if (!Number.isSafeInteger(limit) || limit < 1 || limit > 1_000)
+    throw new Error('Run membership limit must be an integer from 1 to 1000.');
+  const after = membershipCursor(input.cursor);
+  if (!(await isRuntimeSheetRunMembershipStorageAvailable(query)))
+    return {
+      entries: [],
+      nextCursor: null,
+      coverage: {
+        complete: false,
+        expectedRows: null,
+        captureStartedAt: null,
+        captureCompletedAt: null,
+        reason: 'storage_not_expanded',
+      },
+    };
+  const manifest = await query<{
+    membership_expected_rows: number | string | null;
+    membership_started_at: Date | string | null;
+    membership_completed_at: Date | string | null;
+  }>(
+    `SELECT membership_expected_rows, membership_started_at, membership_completed_at
+       FROM storage._deepline_run_datasets WHERE run_id = $1 AND dataset_id = $2`,
+    [input.runId, input.datasetId],
+  );
+  const coverage = manifest.rows[0];
+  const result = await query<{
+    run_id: string;
+    dataset_id: string;
+    row_key: string;
+    phase: 'admitted' | 'terminal';
+    generation: string;
+    attempt_seq: number | null;
+    row_version: number | string | null;
+    input_index: number | string | null;
+    input_digest: string;
+    status: RuntimeSheetRunRowMembership['status'];
+    row_data: Record<string, unknown>;
+    source_ref: Record<string, unknown>;
+    recorded_at: Date | string;
+  }>(
+    `SELECT run_id, dataset_id, row_key, phase, generation, attempt_seq, row_version, input_index, input_digest, status, row_data, source_ref, recorded_at
+       FROM storage.${RUNTIME_SHEET_RUN_MEMBERSHIP_TABLE}
+      WHERE run_id = $1 AND dataset_id = $2
+        AND ($3::text IS NULL OR (row_key, phase, generation) > ($3::text, $4::text, $5::text))
+      ORDER BY row_key ASC, phase ASC, generation ASC LIMIT $6`,
+    [
+      input.runId,
+      input.datasetId,
+      after?.rowKey ?? null,
+      after?.phase ?? null,
+      after?.generation ?? null,
+      limit + 1,
+    ],
+  );
+  const entries = result.rows.slice(0, limit).map((row) => ({
+    runId: row.run_id,
+    datasetId: row.dataset_id,
+    rowKey: row.row_key,
+    phase: row.phase,
+    generation: row.generation,
+    attemptSeq: row.attempt_seq === null ? null : Number(row.attempt_seq),
+    rowVersion: row.row_version === null ? null : Number(row.row_version),
+    inputIndex: row.input_index === null ? null : Number(row.input_index),
+    inputDigest: row.input_digest,
+    status: row.status,
+    data: row.row_data,
+    source: row.source_ref,
+    recordedAt: new Date(row.recorded_at).toISOString(),
+  }));
+  const last = entries.at(-1);
+  const complete =
+    coverage?.membership_started_at != null &&
+    coverage.membership_completed_at != null &&
+    coverage.membership_expected_rows != null;
+  return {
+    entries,
+    nextCursor:
+      result.rows.length > limit && last
+        ? Buffer.from(
+            JSON.stringify({
+              version: 1,
+              rowKey: last.rowKey,
+              phase: last.phase,
+              generation: last.generation,
+            }),
+          ).toString('base64url')
+        : null,
+    coverage: {
+      complete,
+      expectedRows:
+        coverage?.membership_expected_rows == null
+          ? null
+          : Number(coverage.membership_expected_rows),
+      captureStartedAt: coverage?.membership_started_at
+        ? new Date(coverage.membership_started_at).toISOString()
+        : null,
+      captureCompletedAt: coverage?.membership_completed_at
+        ? new Date(coverage.membership_completed_at).toISOString()
+        : null,
+      reason: complete
+        ? 'complete'
+        : coverage?.membership_started_at
+          ? 'capture_in_progress'
+          : 'legacy_or_capture_not_enabled',
+    },
+  };
+}
+
+/**
+ * Future enabled writers call this inside the SAME transaction as the fenced
+ * sheet write. No runtime calls it in the expand release. Retry preserves the
+ * first snapshot; a different immutable input/result fails loudly.
+ */
+export async function appendRuntimeSheetRunRowMembership(
+  query: RuntimeSheetOrmQuery,
+  row: Omit<RuntimeSheetRunRowMembership, 'recordedAt'>,
+): Promise<void> {
+  return appendRuntimeSheetRunRowMembershipBatch(query, [row]);
+}
+
+/** A single bounded SQL write per canonical sheet batch, in its transaction. */
+export async function appendRuntimeSheetRunRowMembershipBatch(
+  query: RuntimeSheetOrmQuery,
+  rows: ReadonlyArray<Omit<RuntimeSheetRunRowMembership, 'recordedAt'>>,
+  schema = 'storage',
+): Promise<void> {
+  if (rows.length === 0) return;
+  const table = `"${schema.replace(/"/g, '""')}"."${RUNTIME_SHEET_RUN_MEMBERSHIP_TABLE}"`;
+  const result = await query<{ row_key: string }>(
+    `INSERT INTO ${table} AS target
+    (run_id, dataset_id, row_key, phase, generation, attempt_seq, row_version, input_index, input_digest, status, row_data, source_ref)
+    SELECT "runId", "datasetId", "rowKey", phase, generation, "attemptSeq", "rowVersion", "inputIndex", "inputDigest", status, data, source
+    FROM jsonb_to_recordset($1::jsonb) AS snapshots(
+      "runId" text, "datasetId" text, "rowKey" text, phase text, generation text,
+      "attemptSeq" integer, "rowVersion" bigint, "inputIndex" bigint, "inputDigest" text, status text, data jsonb, source jsonb)
+    ON CONFLICT (run_id, dataset_id, row_key, phase, generation) DO UPDATE SET row_key = EXCLUDED.row_key
+    WHERE target.input_index IS NOT DISTINCT FROM EXCLUDED.input_index
+      AND target.input_digest = EXCLUDED.input_digest AND target.status = EXCLUDED.status
+      AND target.row_data = EXCLUDED.row_data AND target.source_ref = EXCLUDED.source_ref
+      AND target.attempt_seq IS NOT DISTINCT FROM EXCLUDED.attempt_seq
+      AND target.row_version IS NOT DISTINCT FROM EXCLUDED.row_version
+    RETURNING row_key`,
+    [JSON.stringify(rows)],
+  );
+  if (result.rows.length !== rows.length)
+    throw new Error(
+      'Immutable run row membership conflicts with its retained input or result.',
+    );
+}

package/dist/bundling-sources/shared_libs/play-runtime/async-operation.ts

@@ -1,3 +1,5 @@
+import { TOOL_EXECUTE_PROVIDER_PACING_WAIT_CODE } from './tool-execute-retry-policy';
+
 export type AsyncOperationJsonValue =
   | string
   | number
@@ -6,6 +8,15 @@ export type AsyncOperationJsonValue =
   | readonly AsyncOperationJsonValue[]
   | { readonly [key: string]: AsyncOperationJsonValue };
 
+/** Shared timeout wording for the provider boundary and its keyed runtime continuation. */
+export function formatAsyncOperationTimeout(input: {
+  startAction: string;
+  jobId: string | null;
+  timeoutDescription: string;
+}): string {
+  return `Async operation ${input.startAction} (${input.jobId}) did not reach a terminal state within ${input.timeoutDescription}.`;
+}
+
 export type AsyncOperationTerminalOutcome =
   | 'running'
   | 'succeeded'
@@ -1111,7 +1122,7 @@ export async function executeAsyncOperationLifecycle<Response>(input: {
   }): Promise<
     { kind: 'response'; response: Response } | { kind: 'timed_out' }
   > => {
-    let pacingAttempt = 0;
+    let pacingRefusals = 0;
     while (true) {
       const remainingTimeoutMs = Math.max(
         0,
@@ -1131,18 +1142,40 @@ export async function executeAsyncOperationLifecycle<Response>(input: {
           : { kind: 'response', response };
       } catch (error) {
         if (abortSignal.aborted) return { kind: 'timed_out' };
-        if (!(error instanceof AsyncOperationCompanionPacingError)) throw error;
-        // This refusal applies only to the current companion read. The launch
-        // already owns its job and hold. Stay inside their original deadline
-        // and repeat this exact action/page, never the enclosing start request.
-        pacingAttempt += 1;
+        const refusal =
+          error && typeof error === 'object'
+            ? (error as Record<string, unknown>)
+            : null;
+        // This is the companion's admission refusal, never evidence that the
+        // original launch was not sent. Retain the current job, terminal poll
+        // and result page while retrying only this undispatched action.
+        if (
+          !(error instanceof AsyncOperationCompanionPacingError) &&
+          (refusal?.status !== 503 ||
+            refusal.code !== TOOL_EXECUTE_PROVIDER_PACING_WAIT_CODE ||
+            refusal.retryable !== true ||
+            refusal.failureOrigin !== 'deepline_rate_limit' ||
+            refusal.providerRequestDispatched !== false)
+        ) {
+          throw error;
+        }
+        pacingRefusals += 1;
+        const retryAfterMs =
+          error instanceof AsyncOperationCompanionPacingError
+            ? error.retryAfterMs
+            : refusal?.retryAfterMs;
         const delayMs = Math.max(
           1,
           asyncOperationRateLimitPollDelayMs({
             contract: effectiveContract,
             pollAttempt: request.pollAttempt,
-            rateLimitAttempt: pacingAttempt,
-            retryAfterMs: error.retryAfterMs,
+            rateLimitAttempt: pacingRefusals,
+            retryAfterMs:
+              typeof retryAfterMs === 'number' &&
+              Number.isFinite(retryAfterMs) &&
+              retryAfterMs >= 0
+                ? retryAfterMs
+                : null,
           }),
         );
         const remainingAfterRefusal = Math.max(