deepline 0.3.174 → 0.3.176

package/dist/bundling-sources/sdk/src/client.ts

@@ -107,6 +107,7 @@ import type { PlayCompilerManifest } from '../../shared_libs/plays/compiler-mani
 import type { EnrichCompiledConfig } from './cli/enrich-play-compiler.js';
 import { RUNTIME_ENVIRONMENT_TOKEN_HEADER } from '../../shared_libs/play-runtime/coordinator-headers.js';
 import { resolveTheirstackClientTimeoutMs } from '../../shared_libs/integration-contracts/theirstack/execution-policy.js';
+import { resolveWaterfallClientTimeoutMs } from '../../shared_libs/integration-contracts/waterfall/execution-policy.js';
 import {
   BETTERCONTACT_LAUNCHER_CLIENT_TIMEOUT_MS,
   usesExtendedBetterContactLauncherBudget,
@@ -724,6 +725,8 @@ function resolveToolExecuteTimeoutMs(
     input,
   );
   if (theirstackTimeoutMs !== null) return theirstackTimeoutMs;
+  const waterfallTimeoutMs = resolveWaterfallClientTimeoutMs(normalized);
+  if (waterfallTimeoutMs !== null) return waterfallTimeoutMs;
   if (usesExtendedBetterContactLauncherBudget(normalized)) {
     return BETTERCONTACT_LAUNCHER_CLIENT_TIMEOUT_MS;
   }

package/dist/bundling-sources/sdk/src/release.ts

@@ -200,7 +200,7 @@ export const SDK_RELEASE = {
   // getters keep their established compatibility behavior.
   // 0.3.1 deprecated the legacy `deepline enrich` command in favor of Plays.
   // 0.3.90 is the first deliberately versioned SDK release for API v3.
-  version: '0.3.174',
+  version: '0.3.176',
   updateSummary:
     'Play grep now returns concise results with the matched fields and terms. Automatic CLI updates are enabled by default; use `deepline settings autoupdate off` to opt out, `deepline settings autoupdate on` to re-enable updates, or `deepline settings autoupdate pin <version>` to hold an exact release.',
   packageCapabilities: {

package/dist/bundling-sources/shared_libs/integration-contracts/index.ts

@@ -1,3 +1,4 @@
 export * from './bettercontact/execution-policy';
 export * from './theirstack/execution-policy';
+export * from './waterfall/execution-policy';
 export * from './monitor-input-contract';

package/dist/bundling-sources/shared_libs/integration-contracts/waterfall/execution-policy.ts

@@ -0,0 +1,18 @@
+/**
+ * Waterfall launcher responses include a bounded launch-and-poll window. The
+ * SDK default is only 60 seconds, which can abandon a valid prospector job
+ * while the server is still polling it. Keep this budget specific to the
+ * launcher instead of widening every integration request.
+ */
+export const WATERFALL_PROSPECTOR_CLIENT_TIMEOUT_MS = 180_000;
+
+const WATERFALL_PROSPECTOR_LAUNCHER_TOOL_IDS = new Set([
+  'prospector',
+  'deepline_native_prospector',
+]);
+
+export function resolveWaterfallClientTimeoutMs(toolId: string): number | null {
+  return WATERFALL_PROSPECTOR_LAUNCHER_TOOL_IDS.has(toolId.trim().toLowerCase())
+    ? WATERFALL_PROSPECTOR_CLIENT_TIMEOUT_MS
+    : null;
+}

package/dist/bundling-sources/shared_libs/play-data-plane/index.ts

@@ -2,3 +2,4 @@ export * from './column-names';
 export * from './contracts';
 export * from './sheet-contract';
 export * from './runtime-sheet-storage-contract';
+export * from './runtime-sheet-orm';

package/dist/bundling-sources/shared_libs/play-data-plane/runtime-sheet-orm.ts

@@ -0,0 +1,489 @@
+/**
+ * Typed owner for the physical Runtime Sheet table contract.
+ *
+ * This is intentionally a small ORM seam, not a general SQL builder. It owns
+ * the facts that every Runtime Sheet writer needs to agree on: which columns
+ * belong to Deepline, what shape they have, how readiness is determined, and
+ * which additive/default repairs are safe. Application and worker code supply
+ * the query adapter and keep their transport-specific concerns outside this
+ * Module.
+ */
+
+export const RUNTIME_SHEET_VERSION_SEQUENCE = '_deepline_sheet_version_seq';
+
+export const RUNTIME_SHEET_CELL_META_COLUMN = {
+  name: '_cell_meta',
+  dataType: 'jsonb',
+  nullable: 'NO',
+  defaultSql: "'{}'::jsonb",
+} as const;
+
+type RuntimeSheetNullable = 'YES' | 'NO';
+
+export type RuntimeSheetColumnContract = {
+  name: string;
+  sqlType: string;
+  dataType: string;
+  nullable: RuntimeSheetNullable;
+  defaultSql: string | null;
+  defaultKind?: 'version_sequence';
+  /** A missing column can be added without guessing at customer data. */
+  repairable: boolean;
+  /** The column must have a default because writers intentionally omit it. */
+  requiresDefault: boolean;
+};
+
+export type RuntimeSheetColumnState = {
+  column_name: string;
+  data_type: string;
+  is_nullable: string;
+  column_default: string | null;
+};
+
+export type RuntimeSheetSchemaSnapshot = {
+  sequenceExists: boolean;
+  columns: ReadonlyMap<string, RuntimeSheetColumnState>;
+};
+
+export type RuntimeSheetSchemaReadiness = {
+  ready: boolean;
+  missingColumns: string[];
+  incompatibleColumns: string[];
+  missingDefaults: string[];
+  sequenceReady: boolean;
+};
+
+export type RuntimeSheetOrmQuery = <Row = Record<string, unknown>>(
+  sql: string,
+  params?: readonly unknown[],
+) => Promise<{ rows: Row[] }>;
+
+export type RuntimeSheetOrm = {
+  inspect(): Promise<RuntimeSheetSchemaSnapshot>;
+  readiness(
+    snapshot?: RuntimeSheetSchemaSnapshot,
+  ): Promise<RuntimeSheetSchemaReadiness>;
+  repairOwnerMetadata(): Promise<void>;
+  isOwnerMetadataConstraintError(error: unknown): boolean;
+};
+
+export function isRuntimeSheetOwnerMetadataConstraintError(
+  error: unknown,
+  input: { table: string },
+): boolean {
+  if (!error || typeof error !== 'object') return false;
+  const record = error as Record<string, unknown>;
+  if (String(record.code ?? '') !== '23502') return false;
+  const table = typeof record.table === 'string' ? record.table : null;
+  const message = typeof record.message === 'string' ? record.message : '';
+  const tableMatches =
+    table === input.table || message.includes(`relation "${input.table}"`);
+  const column = readColumnName(error);
+  return (
+    tableMatches &&
+    column !== null &&
+    RUNTIME_SHEET_REPAIRABLE_COLUMNS.some(
+      (contract) => contract.name === column,
+    )
+  );
+}
+
+const RUNTIME_SHEET_COLUMN_CONTRACTS: readonly RuntimeSheetColumnContract[] = [
+  {
+    name: '_key',
+    sqlType: 'text',
+    dataType: 'text',
+    nullable: 'NO',
+    defaultSql: null,
+    repairable: false,
+    requiresDefault: false,
+  },
+  {
+    name: '_status',
+    sqlType: 'text NOT NULL',
+    dataType: 'text',
+    nullable: 'NO',
+    defaultSql: null,
+    repairable: false,
+    requiresDefault: false,
+  },
+  {
+    name: '_run_id',
+    sqlType: 'text',
+    dataType: 'text',
+    nullable: 'YES',
+    defaultSql: null,
+    repairable: true,
+    requiresDefault: false,
+  },
+  {
+    name: '_error',
+    sqlType: 'text',
+    dataType: 'text',
+    nullable: 'YES',
+    defaultSql: null,
+    repairable: true,
+    requiresDefault: false,
+  },
+  {
+    name: '_stage',
+    sqlType: 'text',
+    dataType: 'text',
+    nullable: 'YES',
+    defaultSql: null,
+    repairable: true,
+    requiresDefault: false,
+  },
+  {
+    name: '_provider',
+    sqlType: 'text',
+    dataType: 'text',
+    nullable: 'YES',
+    defaultSql: null,
+    repairable: true,
+    requiresDefault: false,
+  },
+  {
+    name: '_input_index',
+    sqlType: 'bigint',
+    dataType: 'bigint',
+    nullable: 'YES',
+    defaultSql: null,
+    repairable: true,
+    requiresDefault: false,
+  },
+  {
+    name: '_attempt_id',
+    sqlType: 'text',
+    dataType: 'text',
+    nullable: 'YES',
+    defaultSql: null,
+    repairable: true,
+    requiresDefault: false,
+  },
+  {
+    name: '_attempt_owner_run_id',
+    sqlType: 'text',
+    dataType: 'text',
+    nullable: 'YES',
+    defaultSql: null,
+    repairable: true,
+    requiresDefault: false,
+  },
+  {
+    name: '_attempt_expires_at',
+    sqlType: 'timestamptz',
+    dataType: 'timestamp with time zone',
+    nullable: 'YES',
+    defaultSql: null,
+    repairable: true,
+    requiresDefault: false,
+  },
+  {
+    name: '_attempt_seq',
+    sqlType: 'integer',
+    dataType: 'integer',
+    nullable: 'YES',
+    defaultSql: null,
+    repairable: true,
+    requiresDefault: false,
+  },
+  {
+    name: '_write_version',
+    sqlType: 'bigint NOT NULL DEFAULT 0',
+    dataType: 'bigint',
+    nullable: 'NO',
+    defaultSql: '0',
+    repairable: true,
+    requiresDefault: true,
+  },
+  {
+    name: '_writer_run_id',
+    sqlType: 'text',
+    dataType: 'text',
+    nullable: 'YES',
+    defaultSql: null,
+    repairable: true,
+    requiresDefault: false,
+  },
+  {
+    name: '_created_at',
+    sqlType: 'timestamptz NOT NULL DEFAULT now()',
+    dataType: 'timestamp with time zone',
+    nullable: 'NO',
+    defaultSql: 'now()',
+    repairable: true,
+    requiresDefault: true,
+  },
+  {
+    name: '_updated_at',
+    sqlType: 'timestamptz NOT NULL DEFAULT now()',
+    dataType: 'timestamp with time zone',
+    nullable: 'NO',
+    defaultSql: 'now()',
+    repairable: true,
+    requiresDefault: true,
+  },
+  {
+    name: '_version',
+    sqlType: 'bigint NOT NULL',
+    dataType: 'bigint',
+    nullable: 'NO',
+    defaultSql: null,
+    defaultKind: 'version_sequence',
+    repairable: true,
+    requiresDefault: true,
+  },
+  {
+    ...RUNTIME_SHEET_CELL_META_COLUMN,
+    sqlType: `jsonb NOT NULL DEFAULT ${RUNTIME_SHEET_CELL_META_COLUMN.defaultSql}`,
+    repairable: true,
+    requiresDefault: true,
+  },
+];
+
+export const RUNTIME_SHEET_ATTEMPT_COLUMNS =
+  RUNTIME_SHEET_COLUMN_CONTRACTS.filter((column) =>
+    column.name.startsWith('_attempt_'),
+  ).map((column) => column.name);
+
+export const RUNTIME_SHEET_WRITE_VERSION_COLUMNS =
+  RUNTIME_SHEET_COLUMN_CONTRACTS.filter(
+    (column) =>
+      column.name === '_write_version' || column.name === '_writer_run_id',
+  ).map((column) => column.name);
+
+export const RUNTIME_SHEET_TIMESTAMP_COLUMNS =
+  RUNTIME_SHEET_COLUMN_CONTRACTS.filter(
+    (column) => column.name === '_created_at' || column.name === '_updated_at',
+  ).map((column) => column.name);
+
+export const RUNTIME_SHEET_SYSTEM_COLUMNS = new Set(
+  RUNTIME_SHEET_COLUMN_CONTRACTS.map((column) => column.name),
+);
+
+const RUNTIME_SHEET_REPAIRABLE_COLUMNS = RUNTIME_SHEET_COLUMN_CONTRACTS.filter(
+  (column) => column.repairable,
+);
+
+export function runtimeSheetColumnContracts(): readonly RuntimeSheetColumnContract[] {
+  return RUNTIME_SHEET_COLUMN_CONTRACTS;
+}
+
+export function runtimeSheetRepairableColumnContracts(): readonly RuntimeSheetColumnContract[] {
+  return RUNTIME_SHEET_REPAIRABLE_COLUMNS;
+}
+
+export function isRuntimeSheetSystemColumn(columnName: string): boolean {
+  return RUNTIME_SHEET_SYSTEM_COLUMNS.has(columnName);
+}
+
+/**
+ * SQL projection for adapters that already have a larger relation-readiness
+ * query. The expected rows are generated from the contract, so adding an
+ * owner column expands this projection automatically instead of requiring a
+ * new field-specific boolean in every adapter.
+ */
+export function runtimeSheetOwnerColumnsReadinessSql(input: {
+  schemaParam: string;
+  tableParam: string;
+}): string {
+  const expectedRows = RUNTIME_SHEET_COLUMN_CONTRACTS.map(
+    (column) =>
+      `(${quoteLiteral(column.name)}, ${quoteLiteral(column.dataType)}, ${quoteLiteral(column.nullable)}, ${column.requiresDefault ? 'TRUE' : 'FALSE'})`,
+  ).join(',\n          ');
+  return `(
+        SELECT count(*) = ${RUNTIME_SHEET_COLUMN_CONTRACTS.length}
+        FROM information_schema.columns AS actual
+        JOIN (VALUES
+          ${expectedRows}
+        ) AS expected(column_name, data_type, is_nullable, requires_default)
+          ON expected.column_name = actual.column_name
+        WHERE actual.table_schema = ${input.schemaParam}
+          AND actual.table_name = ${input.tableParam}
+          AND actual.data_type = expected.data_type
+          AND actual.is_nullable = expected.is_nullable
+          AND (NOT expected.requires_default OR actual.column_default IS NOT NULL)
+      )`;
+}
+
+export function isRuntimeSheetColumnReady(
+  contract: RuntimeSheetColumnContract,
+  state: RuntimeSheetColumnState | undefined,
+): boolean {
+  return (
+    state?.data_type === contract.dataType &&
+    state.is_nullable === contract.nullable &&
+    (!contract.requiresDefault || state.column_default != null)
+  );
+}
+
+export function evaluateRuntimeSheetSchema(
+  snapshot: RuntimeSheetSchemaSnapshot,
+): RuntimeSheetSchemaReadiness {
+  const missingColumns: string[] = [];
+  const incompatibleColumns: string[] = [];
+  const missingDefaults: string[] = [];
+
+  for (const contract of RUNTIME_SHEET_COLUMN_CONTRACTS) {
+    const state = snapshot.columns.get(contract.name);
+    if (!state) {
+      missingColumns.push(contract.name);
+      continue;
+    }
+    if (
+      state.data_type !== contract.dataType ||
+      state.is_nullable !== contract.nullable
+    ) {
+      incompatibleColumns.push(contract.name);
+      continue;
+    }
+    if (contract.requiresDefault && state.column_default == null) {
+      missingDefaults.push(contract.name);
+    }
+  }
+
+  const sequenceReady = snapshot.sequenceExists;
+  return {
+    ready:
+      missingColumns.length === 0 &&
+      incompatibleColumns.length === 0 &&
+      missingDefaults.length === 0 &&
+      sequenceReady,
+    missingColumns,
+    incompatibleColumns,
+    missingDefaults,
+    sequenceReady,
+  };
+}
+
+function quoteIdentifier(identifier: string): string {
+  return `"${identifier.replace(/"/g, '""')}"`;
+}
+
+function quoteLiteral(value: string): string {
+  return `'${value.replace(/'/g, "''")}'`;
+}
+
+function relation(schema: string, table: string): string {
+  return `${quoteIdentifier(schema)}.${quoteIdentifier(table)}`;
+}
+
+function versionDefaultSql(schema: string): string {
+  return `nextval(${quoteLiteral(`${schema}.${RUNTIME_SHEET_VERSION_SEQUENCE}`)}::regclass)`;
+}
+
+function columnDefaultSql(
+  column: RuntimeSheetColumnContract,
+  schema: string,
+): string | null {
+  if (column.defaultKind === 'version_sequence') {
+    return versionDefaultSql(schema);
+  }
+  return column.defaultSql;
+}
+
+function readColumnName(error: unknown): string | null {
+  if (!error || typeof error !== 'object') return null;
+  const record = error as Record<string, unknown>;
+  if (typeof record.column === 'string') return record.column;
+  if (typeof record.message !== 'string') return null;
+  return /column ['"]([^'"]+)['"]/i.exec(record.message)?.[1] ?? null;
+}
+
+export function createRuntimeSheetOrm(input: {
+  schema: string;
+  table: string;
+  query: RuntimeSheetOrmQuery;
+}): RuntimeSheetOrm {
+  const tableReference = relation(input.schema, input.table);
+  const versionSequenceReference = `${input.schema}.${RUNTIME_SHEET_VERSION_SEQUENCE}`;
+
+  async function inspect(): Promise<RuntimeSheetSchemaSnapshot> {
+    const [columns, sequence] = await Promise.all([
+      input.query<RuntimeSheetColumnState>(
+        `
+          SELECT column_name, data_type, is_nullable, column_default
+          FROM information_schema.columns
+          WHERE table_schema = $1
+            AND table_name = $2
+            AND column_name = ANY($3::text[])
+        `,
+        [input.schema, input.table, [...RUNTIME_SHEET_SYSTEM_COLUMNS]],
+      ),
+      input.query<{ exists: boolean }>(
+        'SELECT to_regclass($1) IS NOT NULL AS exists',
+        [versionSequenceReference],
+      ),
+    ]);
+    return {
+      sequenceExists: sequence.rows[0]?.exists === true,
+      columns: new Map(columns.rows.map((row) => [row.column_name, row])),
+    };
+  }
+
+  async function readiness(
+    snapshot?: RuntimeSheetSchemaSnapshot,
+  ): Promise<RuntimeSheetSchemaReadiness> {
+    return evaluateRuntimeSheetSchema(snapshot ?? (await inspect()));
+  }
+
+  async function repairOwnerMetadata(): Promise<void> {
+    await input.query(`
+      ALTER TABLE ${tableReference}
+      ${RUNTIME_SHEET_REPAIRABLE_COLUMNS.map(
+        (column) =>
+          `ADD COLUMN IF NOT EXISTS ${quoteIdentifier(column.name)} ${column.dataType}`,
+      ).join(',\n      ')}
+    `);
+
+    const defaultedColumns = RUNTIME_SHEET_REPAIRABLE_COLUMNS.filter(
+      (column) => column.requiresDefault,
+    );
+    await input.query(`
+      ALTER TABLE ${tableReference}
+      ${defaultedColumns
+        .map(
+          (column) =>
+            `ALTER COLUMN ${quoteIdentifier(column.name)} SET DEFAULT ${columnDefaultSql(column, input.schema)}`,
+        )
+        .join(',\n      ')}
+    `);
+
+    for (const column of defaultedColumns) {
+      const defaultSql = columnDefaultSql(column, input.schema);
+      if (defaultSql === null) continue;
+      await input.query(`
+        UPDATE ${tableReference}
+        SET ${quoteIdentifier(column.name)} = ${defaultSql}
+        WHERE ${quoteIdentifier(column.name)} IS NULL
+      `);
+    }
+
+    const nonNullableColumns = RUNTIME_SHEET_REPAIRABLE_COLUMNS.filter(
+      (column) => column.nullable === 'NO',
+    );
+    await input.query(`
+      ALTER TABLE ${tableReference}
+      ${nonNullableColumns
+        .map(
+          (column) =>
+            `ALTER COLUMN ${quoteIdentifier(column.name)} SET NOT NULL`,
+        )
+        .join(',\n      ')}
+    `);
+  }
+
+  function isOwnerMetadataConstraintError(error: unknown): boolean {
+    return isRuntimeSheetOwnerMetadataConstraintError(error, {
+      table: input.table,
+    });
+  }
+
+  return {
+    inspect,
+    readiness,
+    repairOwnerMetadata,
+    isOwnerMetadataConstraintError,
+  };
+}

package/dist/bundling-sources/shared_libs/play-data-plane/runtime-sheet-storage-contract.ts

@@ -1,16 +1,7 @@
-/**
- * Canonical contract for the runtime sheet cell-provenance column.
- *
- * Runtime sheet writers intentionally omit this column and rely on the
- * database default. Keep the type, nullability, and default in one Module so
- * table creation, readiness probes, migrations, and diagnostics cannot drift.
- */
-export const RUNTIME_SHEET_CELL_META_COLUMN = {
-  name: '_cell_meta',
-  dataType: 'jsonb',
-  nullable: 'NO',
-  defaultSql: "'{}'::jsonb",
-} as const;
+/** Compatibility seam for callers that used the cell-meta-only contract. */
+import { RUNTIME_SHEET_CELL_META_COLUMN } from './runtime-sheet-orm';
+
+export { RUNTIME_SHEET_CELL_META_COLUMN } from './runtime-sheet-orm';
 
 export type RuntimeSheetCellMetaColumnState = {
   data_type: string;