deepline 0.3.227 → 0.3.229

package/dist/bundling-sources/sdk/src/release.ts

@@ -200,7 +200,7 @@ export const SDK_RELEASE = {
   // getters keep their established compatibility behavior.
   // 0.3.1 deprecated the legacy `deepline enrich` command in favor of Plays.
   // 0.3.90 is the first deliberately versioned SDK release for API v3.
-  version: '0.3.227',
+  version: '0.3.229',
   updateSummary:
     'Play grep now returns concise results with the matched fields and terms. Automatic CLI updates are enabled by default; use `deepline settings autoupdate off` to opt out, `deepline settings autoupdate on` to re-enable updates, or `deepline settings autoupdate pin <version>` to hold an exact release.',
   packageCapabilities: {

package/dist/bundling-sources/shared_libs/play-data-plane/contracts.ts

@@ -1,3 +1,14 @@
+/** Source evidence is advisory; stored values determine lossless SQL encoding. */
+export interface PlayDatasetColumnTypeEvidence {
+  postgresTypeId: number;
+  valueType: 'boolean' | 'number' | 'string' | 'json';
+}
+
+export type PlayDatasetColumnTypes = Record<
+  string,
+  PlayDatasetColumnTypeEvidence
+>;
+
 export type PlaySheetColumnSource =
   | 'input'
   | 'datasetColumn'
@@ -17,6 +28,7 @@ export interface PlaySheetColumnContract {
   stepId?: string;
   toolId?: string;
   isRowKey?: boolean;
+  typeEvidence?: PlayDatasetColumnTypeEvidence;
 }
 
 export interface PlaySheetContract {

package/dist/bundling-sources/shared_libs/play-data-plane/index.ts

@@ -3,3 +3,4 @@ export * from './contracts';
 export * from './sheet-contract';
 export * from './runtime-sheet-storage-contract';
 export * from './runtime-sheet-orm';
+export * from './sheet-column-storage';

package/dist/bundling-sources/shared_libs/play-data-plane/sheet-column-storage.ts

@@ -0,0 +1,252 @@
+import type { PhysicalSheetColumnProjection } from './sheet-contract';
+
+type QueryClient = {
+  query(
+    sql: string,
+    params?: unknown[],
+  ): Promise<{ rows: Record<string, unknown>[] }>;
+};
+
+type SheetIdentity = { schema: string; table: string };
+type StorageType = 'boolean' | 'double precision' | 'text' | 'jsonb';
+type ColumnState = { name: string; type: string; marker: string | null };
+export type RuntimeSheetColumnTypes = ReadonlyMap<string, string>;
+
+const MARKER = 'deepline:runtime-sheet-column:';
+const UNDECIDED = `${MARKER}undecided`;
+const NATIVE = `${MARKER}native`;
+const MIXED = `${MARKER}jsonb`;
+
+const identifier = (value: string) => `"${value.replaceAll('"', '""')}"`;
+const literal = (value: string) => `'${value.replaceAll("'", "''")}'`;
+const relation = (sheet: SheetIdentity) =>
+  `${identifier(sheet.schema)}.${identifier(sheet.table)}`;
+
+function combine(
+  a: StorageType | null,
+  b: StorageType | null,
+): StorageType | null {
+  return a === null ? b : b === null || a === b ? a : 'jsonb';
+}
+
+function valueType(value: unknown): StorageType | null {
+  if (value == null) return null;
+  if (typeof value === 'boolean') return 'boolean';
+  if (typeof value === 'string') return 'text';
+  if (typeof value === 'number' && Number.isFinite(value))
+    return 'double precision';
+  return 'jsonb';
+}
+
+async function lock(client: QueryClient, sheet: SheetIdentity): Promise<void> {
+  // The caller holds this transaction through DML. Both SQL writers use the
+  // same lock, so type inspection, widening and the affected write are atomic.
+  await client.query('SELECT pg_advisory_xact_lock(hashtextextended($1, 0))', [
+    `runtime-sheet-columns:${relation(sheet)}`,
+  ]);
+}
+
+async function inspect(
+  client: QueryClient,
+  sheet: SheetIdentity,
+): Promise<ColumnState[]> {
+  const result = await client.query(
+    `SELECT a.attname AS name, format_type(a.atttypid, NULL) AS type,
+            col_description(a.attrelid, a.attnum) AS marker
+       FROM pg_attribute a
+       WHERE a.attrelid = to_regclass($1)
+         AND a.attnum > 0 AND NOT a.attisdropped`,
+    [relation(sheet)],
+  );
+  return result.rows.map((row) => ({
+    name: String(row.name),
+    type: String(row.type),
+    marker: typeof row.marker === 'string' ? row.marker : null,
+  }));
+}
+
+async function mark(
+  client: QueryClient,
+  sheet: SheetIdentity,
+  column: string,
+  marker: string,
+) {
+  await client.query(
+    `COMMENT ON COLUMN ${relation(sheet)}.${identifier(column)} IS ${literal(marker)}`,
+  );
+}
+
+async function changeType(
+  client: QueryClient,
+  sheet: SheetIdentity,
+  column: ColumnState,
+  desired: StorageType | null,
+): Promise<void> {
+  if (desired === null || column.marker === MIXED) return;
+  // Unmarked columns predate this feature (or belong to customers). Never
+  // reinterpret them merely because their current SQL type happens to match.
+  if (column.marker !== UNDECIDED && column.marker !== NATIVE) return;
+  let next =
+    column.marker === UNDECIDED
+      ? desired
+      : combine(column.type as StorageType, desired)!;
+  if (column.marker === NATIVE && next === column.type) return;
+  const quoted = identifier(column.name);
+
+  if (column.marker === UNDECIDED && next !== 'jsonb') {
+    // Another compatible old writer may have filled a provisioned JSONB
+    // column before its first inference. Prove the whole stored column rather
+    // than relying on a preview or assuming the new marker means no data.
+    const jsonType =
+      next === 'text' ? 'string' : next === 'boolean' ? 'boolean' : 'number';
+    const incompatible = await client.query(
+      `SELECT EXISTS(SELECT 1 FROM ${relation(sheet)}
+         WHERE ${quoted} IS NOT NULL AND ${quoted} <> 'null'::jsonb
+           AND jsonb_typeof(${quoted}) <> $1) AS incompatible`,
+      [jsonType],
+    );
+    if (incompatible.rows[0]?.incompatible === true) next = 'jsonb';
+    if (next === 'double precision') {
+      // Existing arbitrary JSON numeric values may exceed IEEE-754 precision.
+      // CASE guards the cast; SQL predicate evaluation order is not a guard.
+      const numeric = `(${quoted} #>> '{}')::numeric`;
+      const loss = await client.query(
+        `SELECT EXISTS(SELECT 1 FROM ${relation(sheet)}
+           WHERE ${quoted} IS NOT NULL AND ${quoted} <> 'null'::jsonb
+             AND CASE WHEN abs(${numeric}) > 1.7976931348623157e308::numeric
+                       OR (${numeric} <> 0 AND abs(${numeric}) < 5e-324::numeric)
+                  THEN true
+                  ELSE to_jsonb((${quoted} #>> '{}')::double precision) <> ${quoted}
+                 END) AS incompatible`,
+      );
+      if (loss.rows[0]?.incompatible === true) next = 'jsonb';
+    }
+  }
+
+  if (next !== column.type) {
+    const conversion =
+      next === 'jsonb'
+        ? `to_jsonb(${quoted})`
+        : `(${quoted} #>> '{}')::${next}`;
+    await client.query(
+      `ALTER TABLE ${relation(sheet)} ALTER COLUMN ${quoted} TYPE ${next} USING ${conversion}`,
+    );
+  }
+  await mark(client, sheet, column.name, next === 'jsonb' ? MIXED : NATIVE);
+  column.type = next;
+  column.marker = next === 'jsonb' ? MIXED : NATIVE;
+}
+
+/**
+ * Runtime Sheet column storage owns conservative, monotonic SQL typing.
+ * Methods that mutate storage require an existing transaction, held through
+ * the corresponding row write. JSON values remain the transport contract.
+ */
+export const RuntimeSheetColumns = {
+  lockForWrite: lock,
+  async markNewColumns(
+    client: QueryClient,
+    input: SheetIdentity & { columns: readonly string[] },
+  ): Promise<void> {
+    // Caller proves these columns were created in its schema transaction.
+    for (const column of input.columns)
+      await mark(client, input, column, UNDECIDED);
+  },
+
+  async prepareWrite(
+    client: QueryClient,
+    input: SheetIdentity & {
+      rows:
+        | readonly Record<string, unknown>[]
+        | (() => Promise<readonly Record<string, unknown>[]>);
+      columns: readonly PhysicalSheetColumnProjection[];
+    },
+  ): Promise<RuntimeSheetColumnTypes> {
+    await lock(client, input);
+    const rows =
+      typeof input.rows === 'function' ? await input.rows() : input.rows;
+    const columns = new Map(
+      (await inspect(client, input)).map((column) => [column.name, column]),
+    );
+    for (const projection of rows.length > 0 ? input.columns : []) {
+      const column = columns.get(projection.sqlName);
+      if (!column)
+        throw new Error(
+          `Runtime Sheet column is missing: ${projection.sqlName}`,
+        );
+      let desired: StorageType | null =
+        projection.typeEvidence?.valueType === 'json' ? 'jsonb' : null;
+      for (const row of rows)
+        desired = combine(desired, valueType(row[projection.fieldName]));
+      await changeType(client, input, column, desired);
+    }
+    return new Map([...columns].map(([name, column]) => [name, column.type]));
+  },
+
+  async prepareClone(
+    client: QueryClient,
+    input: SheetIdentity & {
+      sourceTable: string;
+      columns: readonly string[];
+      columnMappings?: readonly {
+        sourceSqlName: string;
+        targetSqlName: string;
+      }[];
+    },
+  ): Promise<RuntimeSheetColumnTypes> {
+    const source = { schema: input.schema, table: input.sourceTable };
+    const sheets = [input, source].sort((a, b) =>
+      relation(a) < relation(b) ? -1 : relation(a) > relation(b) ? 1 : 0,
+    );
+    for (const sheet of sheets) await lock(client, sheet);
+    const sourceColumns = new Map(
+      (await inspect(client, source)).map((column) => [column.name, column]),
+    );
+    const targetColumns = new Map(
+      (await inspect(client, input)).map((column) => [column.name, column]),
+    );
+    const mappings =
+      input.columnMappings ??
+      input.columns.map((name) => ({
+        sourceSqlName: name,
+        targetSqlName: name,
+      }));
+    for (const mapping of mappings) {
+      const sourceColumn = sourceColumns.get(mapping.sourceSqlName);
+      const targetColumn = targetColumns.get(mapping.targetSqlName);
+      if (!sourceColumn || !targetColumn)
+        throw new Error(
+          `Runtime Sheet clone column is missing: ${mapping.sourceSqlName} -> ${mapping.targetSqlName}`,
+        );
+      const type = sourceColumn.type;
+      await changeType(
+        client,
+        input,
+        targetColumn,
+        type === 'boolean' || type === 'text' || type === 'double precision'
+          ? type
+          : 'jsonb',
+      );
+    }
+    return new Map(
+      [...targetColumns].map(([name, column]) => [name, column.type]),
+    );
+  },
+
+  jsonValueSql(
+    jsonExpression: string,
+    tableReference: string,
+    columnName: string,
+    types: RuntimeSheetColumnTypes,
+  ): string {
+    const type = types.get(columnName);
+    if (!type)
+      throw new Error(`Runtime Sheet column is missing: ${columnName}`);
+    // jsonb_populate_record maps JSON null to SQL NULL. A legacy JSONB cell
+    // must retain explicit JSON null, distinct from an omitted/SQL NULL cell.
+    if (type === 'jsonb') return jsonExpression;
+    // Resolve the actual row type inside PostgreSQL, never from a cached
+    // contract. This also preserves JSON scalar strings for legacy JSONB.
+    return `(jsonb_populate_record(NULL::${tableReference}, jsonb_build_object(${literal(columnName)}, ${jsonExpression}))).${identifier(columnName)}`;
+  },
+};

package/dist/bundling-sources/shared_libs/play-data-plane/sheet-contract.ts

@@ -1,4 +1,8 @@
-import type { PlaySheetContract } from './contracts';
+import type {
+  PlayDatasetColumnTypeEvidence,
+  PlayDatasetColumnTypes,
+  PlaySheetContract,
+} from './contracts';
 import {
   legacyPostgresTruncatedPlayColumnName,
   sqlSafePlayColumnName,
@@ -7,6 +11,7 @@ import {
 export type PhysicalSheetColumnProjection = {
   sqlName: string;
   fieldName: string;
+  typeEvidence?: PlayDatasetColumnTypeEvidence;
 };
 
 export type LegacyPhysicalSheetColumnBackfill = {
@@ -37,13 +42,11 @@ export function isDatasetPayloadField(field: string): boolean {
 
 export function physicalSheetColumnProjections(
   sheetContract: PlaySheetContract | null | undefined,
+  physicalColumns?: readonly string[],
 ): PhysicalSheetColumnProjection[] {
-  if (!sheetContract) {
-    return [];
-  }
   const seen = new Set<string>();
   const columns: PhysicalSheetColumnProjection[] = [];
-  for (const column of sheetContract.columns) {
+  for (const column of sheetContract?.columns ?? []) {
     const sqlName = column.sqlName.trim();
     const fieldName =
       column.source === 'input' || column.source === 'datasetColumn'
@@ -53,7 +56,18 @@ export function physicalSheetColumnProjections(
       continue;
     }
     seen.add(sqlName);
-    columns.push({ sqlName, fieldName });
+    columns.push({
+      sqlName,
+      fieldName,
+      ...(column.typeEvidence ? { typeEvidence: column.typeEvidence } : {}),
+    });
+  }
+  // A prior run may have introduced fields absent from the current artifact.
+  // Keep their live storage projection so reads and writes do not hide them.
+  for (const sqlName of physicalColumns ?? []) {
+    if (!sqlName || sqlName.startsWith('_') || seen.has(sqlName)) continue;
+    seen.add(sqlName);
+    columns.push({ sqlName, fieldName: sqlName });
   }
   return columns;
 }
@@ -141,7 +155,11 @@ export function outputPhysicalSheetColumnProjections(
       continue;
     }
     seen.add(sqlName);
-    columns.push({ sqlName, fieldName: column.field });
+    columns.push({
+      sqlName,
+      fieldName: column.field,
+      ...(column.typeEvidence ? { typeEvidence: column.typeEvidence } : {}),
+    });
   }
   return columns;
 }
@@ -158,6 +176,7 @@ export function augmentSheetContractWithDatasetFields(input: {
   contract: PlaySheetContract;
   rows: readonly Record<string, unknown>[];
   outputFields?: readonly string[];
+  columnTypes?: PlayDatasetColumnTypes;
 }): PlaySheetContract {
   const outputFields = new Set(input.outputFields ?? []);
   const candidateFields = new Set<string>();
@@ -198,7 +217,9 @@ export function augmentSheetContractWithDatasetFields(input: {
       existingDatasetPayloadFields.add(field);
     }
     existingSqlNames.add(sqlName);
-    target.push(column);
+    const typeEvidence =
+      column.source === 'input' ? input.columnTypes?.[field] : undefined;
+    target.push(typeEvidence ? { ...column, typeEvidence } : column);
   };
 
   for (const column of input.contract.columns) {