deepline 0.1.174 → 0.1.176

package/dist/bundling-sources/shared_libs/play-runtime/runtime-api.ts

@@ -2224,7 +2224,10 @@ async function prepareRuntimeSheetDatasetRows(
     normalizedTableNamespace: string;
     physicalInsertColumnsSql: string;
     physicalInsertValuesSql: string;
+    physicalRefreshSetSql: string;
+    physicalUpsertSetSql: string;
     outputPhysicalColumns: PhysicalSheetColumnProjection[];
+    force?: boolean;
   },
 ): Promise<{ inserted: number; pendingKeys: string[] }> {
   let inserted = 0;
@@ -2259,10 +2262,14 @@ async function prepareRuntimeSheetDatasetRows(
           UPDATE ${sheetTable(session)} AS target
           SET _input_index = input_rows._input_index,
               _updated_at = now(),
-              _version = ${nextRuntimeSheetVersionExpression(session)}
+              _version = ${nextRuntimeSheetVersionExpression(session)}${input.physicalRefreshSetSql}
           FROM input_rows
           WHERE target._key = input_rows._key
-            AND target._input_index IS DISTINCT FROM input_rows._input_index
+            AND target._status <> 'stale'
+            AND (
+              $7::boolean
+              OR target._input_index IS DISTINCT FROM input_rows._input_index
+            )
           RETURNING target._key
         ),
         inserted_rows AS (
@@ -2274,7 +2281,7 @@ async function prepareRuntimeSheetDatasetRows(
             _run_id = EXCLUDED._run_id,
             _input_index = EXCLUDED._input_index,
             _updated_at = now(),
-            _version = ${nextRuntimeSheetVersionExpression(session)}
+            _version = ${nextRuntimeSheetVersionExpression(session)}${input.physicalUpsertSetSql}
           WHERE ${sheetTable(session)}._status = 'stale'
           RETURNING _key
         ),
@@ -2375,6 +2382,7 @@ async function prepareRuntimeSheetDatasetRows(
         input.runId,
         input.normalizedPlayName,
         input.normalizedTableNamespace,
+        input.force === true,
       ],
     );
     inserted += Number(rows[0]?.inserted_count ?? 0);
@@ -2398,30 +2406,35 @@ async function buildRuntimeSheetDatasetStartResult(
     runId: string;
     inserted: number;
     pendingKeys: string[];
+    force?: boolean;
     timings?: RuntimeSheetTiming[];
   },
 ): Promise<PrepareRuntimeSheetResult> {
+  const datasetFields = input.sheetContract.columns.flatMap((column) =>
+    column.source === 'datasetColumn' && typeof column.field === 'string'
+      ? [column.field]
+      : [],
+  );
+  const buildFreshPendingRows = () => {
+    return input.rowEntries.map((entry) => {
+      const row: Record<string, unknown> = {
+        ...sanitizePostgresJsonValue(entry.row),
+        __deeplineRowKey: entry.key,
+      };
+      for (const field of datasetFields) {
+        if (!Object.prototype.hasOwnProperty.call(row, field)) {
+          row[field] = null;
+        }
+      }
+      return row;
+    });
+  };
+
   if (input.inserted === input.rowEntries.length) {
-    const datasetFields = input.sheetContract.columns.flatMap((column) =>
-      column.source === 'datasetColumn' && typeof column.field === 'string'
-        ? [column.field]
-        : [],
-    );
     return {
       inserted: input.inserted,
       skipped: input.sourceRowsLength - input.rowEntries.length,
-      pendingRows: input.rowEntries.map((entry) => {
-        const row: Record<string, unknown> = {
-          ...sanitizePostgresJsonValue(entry.row),
-          __deeplineRowKey: entry.key,
-        };
-        for (const field of datasetFields) {
-          if (!Object.prototype.hasOwnProperty.call(row, field)) {
-            row[field] = null;
-          }
-        }
-        return row;
-      }),
+      pendingRows: buildFreshPendingRows(),
       completedRows: [],
       tableNamespace: input.tableNamespace,
     };
@@ -2434,7 +2447,24 @@ async function buildRuntimeSheetDatasetStartResult(
     runId: input.runId,
     normalizedPlayName: input.normalizedPlayName,
     normalizedTableNamespace: input.normalizedTableNamespace,
+    outputFields: input.force === true ? datasetFields : [],
   });
+  if (pendingKeys.size > 0) {
+    input.timings?.push({
+      phase: 'mark_rows_pending_for_recompute',
+      ms: Date.now() - startedAt,
+      rows: pendingKeys.size,
+    });
+  }
+  if (input.force === true) {
+    return {
+      inserted: input.inserted,
+      skipped: input.sourceRowsLength - input.rowEntries.length,
+      pendingRows: buildFreshPendingRows(),
+      completedRows: [],
+      tableNamespace: input.tableNamespace,
+    };
+  }
   const persistedRows = await readRuntimeRowsByKey(
     client,
     session,
@@ -2444,13 +2474,6 @@ async function buildRuntimeSheetDatasetStartResult(
   const persistedRowsByKey = new Map(
     persistedRows.map((row) => [row.key, row.data]),
   );
-  if (pendingKeys.size > 0) {
-    input.timings?.push({
-      phase: 'mark_rows_pending_for_recompute',
-      ms: Date.now() - startedAt,
-      rows: pendingKeys.size,
-    });
-  }
   const buildMergedRow = (entry: RuntimeDatasetRowEntry) => {
     const merged = { ...entry.row };
     for (const [field, value] of Object.entries(
@@ -2489,12 +2512,14 @@ async function markRuntimeRowsPendingForRecompute(
     runId: string;
     normalizedPlayName: string;
     normalizedTableNamespace: string;
+    outputFields?: string[];
   },
 ): Promise<void> {
   if (input.keys.length === 0) return;
+  const outputFields = [...new Set(input.outputFields ?? [])];
   await client.query(
     `WITH target_rows AS (
-       SELECT _key, _status
+       SELECT _key, _status, _cell_meta
          FROM ${sheetTable(session)}
         WHERE _key = ANY($1::text[])
         FOR UPDATE
@@ -2511,7 +2536,8 @@ async function markRuntimeRowsPendingForRecompute(
            target._status <> 'pending'
            OR target._run_id IS DISTINCT FROM $2::text
          )
-       RETURNING target_rows._status AS previous_status
+       RETURNING target_rows._status AS previous_status,
+                 target_rows._cell_meta AS previous_cell_meta
      ),
      summary_counts AS (
        SELECT
@@ -2520,6 +2546,18 @@ async function markRuntimeRowsPendingForRecompute(
          count(*) FILTER (WHERE previous_status = 'running')::int AS running_to_pending
        FROM updated
      ),
+     column_delta_counts AS (
+       SELECT field_values.field,
+              count(*) FILTER (
+                WHERE previous_cell_meta -> field_values.field ->> 'status' = 'completed'
+              )::int AS completed_to_pending,
+              count(*) FILTER (
+                WHERE previous_cell_meta -> field_values.field ->> 'status' = 'failed'
+              )::int AS failed_to_pending
+         FROM updated
+         JOIN unnest($5::text[]) AS field_values(field) ON true
+        GROUP BY field_values.field
+     ),
      summary_delta AS (
        INSERT INTO ${summaryTable(session)} AS target (play_name, table_namespace, total, queued, running, completed, failed)
        SELECT
@@ -2547,6 +2585,27 @@ async function markRuntimeRowsPendingForRecompute(
          })},
          _updated_at = now()
        RETURNING 1
+     ),
+     column_summary_delta AS (
+       INSERT INTO ${columnSummaryTable(session)} AS target (
+         play_name,
+         table_namespace,
+         field,
+         completed,
+         failed
+       )
+       SELECT $3::text,
+              $4::text,
+              field,
+              -completed_to_pending,
+              -failed_to_pending
+         FROM column_delta_counts
+        WHERE completed_to_pending > 0 OR failed_to_pending > 0
+       ON CONFLICT (play_name, table_namespace, field) DO UPDATE SET
+         completed = GREATEST(target.completed + EXCLUDED.completed, 0),
+         failed = GREATEST(target.failed + EXCLUDED.failed, 0),
+         _updated_at = now()
+       RETURNING 1
      )
      SELECT 1`,
     [
@@ -2554,6 +2613,7 @@ async function markRuntimeRowsPendingForRecompute(
       input.runId,
       input.normalizedPlayName,
       input.normalizedTableNamespace,
+      outputFields,
     ],
   );
 }
@@ -3237,6 +3297,7 @@ export async function startRuntimeSheetDataset(
     rows: Record<string, unknown>[];
     runId: string;
     inputOffset?: number;
+    force?: boolean;
   },
 ): Promise<PrepareRuntimeSheetResult> {
   const totalStartedAt = Date.now();
@@ -3334,6 +3395,24 @@ export async function startRuntimeSheetDataset(
           .map((column) => `payload -> ${quoteLiteral(column)}`)
           .join(', ')}`
       : '';
+  const physicalRefreshSetSql =
+    physicalColumns.length > 0
+      ? `, ${physicalColumns
+          .map(
+            (column) =>
+              `${quoteIdentifier(column)} = input_rows.payload -> ${quoteLiteral(column)}`,
+          )
+          .join(', ')}`
+      : '';
+  const physicalUpsertSetSql =
+    physicalColumns.length > 0
+      ? `, ${physicalColumns
+          .map(
+            (column) =>
+              `${quoteIdentifier(column)} = EXCLUDED.${quoteIdentifier(column)}`,
+          )
+          .join(', ')}`
+      : '';
   const outputPhysicalColumns = outputPhysicalSheetColumnProjections(
     input.sheetContract,
   );
@@ -3364,15 +3443,26 @@ export async function startRuntimeSheetDataset(
         normalizedTableNamespace,
         physicalInsertColumnsSql,
         physicalInsertValuesSql,
+        physicalRefreshSetSql:
+          input.force === true ? physicalRefreshSetSql : '',
+        physicalUpsertSetSql,
         outputPhysicalColumns,
+        force: input.force === true,
       });
+      const effectivePrepared = input.force
+        ? {
+            inserted: prepared.inserted,
+            pendingKeys: rowEntries.map((entry) => entry.key),
+          }
+        : prepared;
       timings.push({
         phase: 'prepare_rows_sql',
         ms: Date.now() - prepareStartedAt,
         rows: rowEntries.length,
         chunks: chunks.length,
-        inserted: prepared.inserted,
-        pending: prepared.pendingKeys.length,
+        inserted: effectivePrepared.inserted,
+        pending: effectivePrepared.pendingKeys.length,
+        ...(input.force ? { force: true } : {}),
       });
       const buildStartedAt = Date.now();
       const built = await buildRuntimeSheetDatasetStartResult(client, session, {
@@ -3384,7 +3474,8 @@ export async function startRuntimeSheetDataset(
         normalizedTableNamespace,
         runId: input.runId,
         timings,
-        ...prepared,
+        force: input.force === true,
+        ...effectivePrepared,
       });
       timings.push({
         phase: 'build_result',

package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backend.ts

@@ -83,6 +83,8 @@ export type PlaySchedulerSubmitInput = {
   artifactHash: string;
   graphHash: string;
   input: Record<string, unknown>;
+  /** Run-level --force cache bypass for dataset rows and ctx.tools.execute receipts. */
+  forceToolRefresh?: boolean | null;
   inputFile?: {
     name?: string;
     path?: string;