deepline 0.3.408 → 0.3.410

package/dist/bundling-sources/shared_libs/play-runtime/runtime-api.ts

@@ -1,3 +1,11 @@
+import {
+  assertDatasetAdmissionHistory,
+  type DatasetPreviousCells,
+} from './dataset-row-input';
+import {
+  previousCellFromValue,
+  type CellStalenessMeta,
+} from '../plays/cell-staleness';
 import {
   RuntimeSheetAttemptSupersededError,
   RuntimeSheetWriteIntentUnavailableError,
@@ -497,6 +505,7 @@ export type ResolvedRuntimePlay = {
 };
 
 export type PrepareRuntimeSheetResult = {
+  previousCells?: DatasetPreviousCells[];
   inserted: number;
   skipped: number;
   pendingRows: Record<string, unknown>[];
@@ -4612,6 +4621,7 @@ async function buildRuntimeSheetDatasetStartResult(
     inserted: number;
     rowDispositions: RuntimeSheetPreparedRowDisposition[];
     force?: boolean;
+    includePreviousCells?: boolean;
     timings?: RuntimeSheetTiming[];
   },
 ): Promise<PrepareRuntimeSheetResult> {
@@ -4620,6 +4630,33 @@ async function buildRuntimeSheetDatasetStartResult(
       ? [column.field]
       : [],
   );
+  const historicalCells = (
+    rows: RuntimeApiRowRecord[],
+  ): DatasetPreviousCells[] =>
+    rows.flatMap(({ key, data }) => {
+      const meta = parseRuntimeCellMeta(data[DEEPLINE_CELL_META_FIELD]);
+      const cells = Object.fromEntries(
+        datasetFields.flatMap((field) => {
+          const cellMeta = parseRuntimeCellMeta(meta[field]);
+          if (
+            (cellMeta.status !== 'completed' && cellMeta.status !== 'cached') ||
+            !Object.prototype.hasOwnProperty.call(data, field)
+          )
+            return [];
+          return [
+            [
+              field,
+              previousCellFromValue({
+                hasValue: true,
+                value: data[field],
+                meta: cellMeta as CellStalenessMeta,
+              })!,
+            ],
+          ];
+        }),
+      );
+      return Object.keys(cells).length ? [{ key, cells }] : [];
+    });
   const buildFreshRow = (entry: RuntimeDatasetRowEntry) => {
     const row: Record<string, unknown> = {
       ...sanitizePostgresJsonValue(entry.row),
@@ -4640,6 +4677,7 @@ async function buildRuntimeSheetDatasetStartResult(
     return {
       inserted: input.inserted,
       skipped: input.sourceRowsLength - input.rowEntries.length,
+      ...(input.includePreviousCells ? { previousCells: [] } : {}),
       pendingRows: buildFreshPendingRows(),
       completedRows: [],
       blockedRows: [],
@@ -4721,6 +4759,9 @@ async function buildRuntimeSheetDatasetStartResult(
     return {
       inserted: input.inserted,
       skipped: input.sourceRowsLength - input.rowEntries.length,
+      ...(input.includePreviousCells
+        ? { previousCells: historicalCells(completedData) }
+        : {}),
       pendingRows,
       completedRows,
       blockedRows,
@@ -4771,6 +4812,9 @@ async function buildRuntimeSheetDatasetStartResult(
   return {
     inserted: input.inserted,
     skipped: input.sourceRowsLength - input.rowEntries.length,
+    ...(input.includePreviousCells
+      ? { previousCells: historicalCells(persistedRows) }
+      : {}),
     pendingRows,
     completedRows,
     blockedRows,
@@ -6970,6 +7014,9 @@ export async function startRuntimeSheetDataset(
     writeVersion?: number | null;
     /** Retain this run/table's first admission across pages, retries and wakes. Absence keeps legacy allocation. */
     stableWriteIntent?: boolean;
+    /** Return completed durable cells separately from merged source rows. */
+    includePreviousCells?: boolean;
+    previousCellsBudgetBytes?: number;
     inputOffset?: number;
     force?: boolean;
     mode?: RuntimeSheetDatasetMode;
@@ -6977,16 +7024,25 @@ export async function startRuntimeSheetDataset(
     pendingRerunAdmission?: boolean;
   },
 ): Promise<PrepareRuntimeSheetResult> {
+  const checkedResult = (
+    result: PrepareRuntimeSheetResult,
+  ): PrepareRuntimeSheetResult => {
+    if (input.includePreviousCells)
+      assertDatasetAdmissionHistory(result, input.previousCellsBudgetBytes);
+    return result;
+  };
   if (context.dbSessionStrategy === 'gateway_only') {
-    return await postRuntimeApi<PrepareRuntimeSheetResult>(context, {
-      // A named operation prevents an old gateway from silently ignoring the
-      // intent opt-in and re-admitting this run under a new version.
-      action:
-        input.stableWriteIntent === true
-          ? 'runtime_sheet_start_with_intent'
-          : 'runtime_sheet_start',
-      input: { ...input, pendingRerunAdmission: true },
-    });
+    return checkedResult(
+      await postRuntimeApi<PrepareRuntimeSheetResult>(context, {
+        // A named operation prevents an old gateway from silently ignoring the
+        // intent opt-in and re-admitting this run under a new version.
+        action:
+          input.stableWriteIntent === true
+            ? 'runtime_sheet_start_with_intent'
+            : 'runtime_sheet_start',
+        input: { ...input, pendingRerunAdmission: true },
+      }),
+    );
   }
   const totalStartedAt = Date.now();
   const timings: RuntimeSheetTiming[] = [];
@@ -7088,9 +7144,10 @@ export async function startRuntimeSheetDataset(
     rows: input.rows.length,
   });
   if (rowEntries.length === 0) {
-    return {
+    return checkedResult({
       inserted: 0,
       skipped: input.rows.length,
+      ...(input.includePreviousCells ? { previousCells: [] } : {}),
       pendingRows: [],
       completedRows: [],
       blockedRows: [],
@@ -7103,7 +7160,7 @@ export async function startRuntimeSheetDataset(
           rows: input.rows.length,
         },
       ],
-    };
+    });
   }
 
   const physicalColumns = physicalSheetColumnNames(input.sheetContract);
@@ -7311,6 +7368,7 @@ export async function startRuntimeSheetDataset(
         writeVersion,
         timings,
         force: input.force === true,
+        includePreviousCells: input.includePreviousCells,
         ...prepared,
       });
       const membershipCapture = await runtimeSheetMembershipCaptureAvailable(
@@ -7395,7 +7453,7 @@ export async function startRuntimeSheetDataset(
     pending: result.pendingRows.length,
     completed: result.completedRows.length,
   });
-  return {
+  return checkedResult({
     ...result,
     attemptId,
     attemptOwnerRunId,
@@ -7403,7 +7461,7 @@ export async function startRuntimeSheetDataset(
     attemptSeq,
     writeVersion: result.writeVersion,
     timings,
-  };
+  });
 }
 
 export async function heartbeatRuntimeSheetAttempt(

package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-execution-queries.ts

@@ -0,0 +1,100 @@
+/** Every candidate after an org's first B eligible runs has at least B runs
+ * from that same org ahead of it in the global total order. It therefore cannot
+ * enter the global first B. Enumerate every org, including NULL, without rotating
+ * a subset; retain the engine's fixed chosen IDs and under-lock rechecks. */
+export function optimizeAbsurdFairClaimSql(noCapClaim: string): string {
+  const start = noCapClaim.indexOf(
+    "  v_sql := format(\n    'with inflight as (",
+  );
+  const end = noCapClaim.indexOf('\n  return query execute v_sql', start);
+  if (start < 0 || end < 0)
+    throw new Error('Absurd fair claim SQL markers are malformed.');
+  const original = noCapClaim.slice(start, end);
+  const from = original.indexOf(
+    '          from absurd.%1$I r\n          join absurd.%2$I t',
+  );
+  const until = original.indexOf('\n     ),\n     ranked as (', from);
+  if (from < 0 || until < 0)
+    throw new Error('Absurd fair claim candidate markers are malformed.');
+  const eligibility = original
+    .slice(from, until)
+    .replace(
+      '          left join inflight i on i.org_key is not distinct from r.org_key\n',
+      '',
+    );
+  const bounded = `with recursive org_keys(org_key) as (
+        select min(org_key) from absurd.%1$I
+         where state in (''pending'', ''sleeping'') and org_key is not null
+        union all
+        select (select min(r.org_key) from absurd.%1$I r
+                 where r.state in (''pending'', ''sleeping'') and r.org_key > o.org_key)
+          from org_keys o where o.org_key is not null
+     ), orgs as (
+        select org_key from org_keys where org_key is not null
+     ), bounded_ready as (
+        select head.* from orgs o cross join lateral (
+          select r.*
+${eligibility}
+           and r.org_key = o.org_key
+           order by r.priority desc, r.available_at, r.created_at, r.run_id
+           limit $2
+        ) head
+        union all
+        select null_head.* from (
+          select r.*
+${eligibility}
+           and r.org_key is null
+           order by r.priority desc, r.available_at, r.created_at, r.run_id
+           limit $2
+        ) null_head
+     ), inflight as (`;
+  const optimized = original
+    .replace('with inflight as (', bounded)
+    .replace(
+      original.slice(from, until),
+      '          from bounded_ready r\n          left join inflight i on i.org_key is not distinct from r.org_key',
+    );
+  const choice = `  -- A false result proves there is no due run. Conservative run-only
+  -- probes retain task and continuation eligibility in the existing plans.
+  -- This must follow BOTH maintenance sweeps: fail_run can create ready work.
+  execute format('select exists (select 1 from absurd.%1$I where state = ''pending'' and available_at <= $1) or exists (select 1 from absurd.%1$I where state = ''sleeping'' and available_at <= $1)', 'r_' || p_queue_name)
+    into v_has_ready_run using v_now;
+  if not v_has_ready_run then
+    return;
+  end if;
+
+  -- An index changes only the query plan. Unmigrated queues keep the
+  -- identical fair ranking; no scheduler admission depends on migration state.
+  execute format('select exists (select 1 from absurd.%I where state in (''pending'', ''sleeping'', ''running'') and cancel_at is not null and cancel_at <= $1)', 't_' || p_queue_name)
+    into v_due_cancellations using v_now;
+  if exists (
+    select 1 from pg_index ix
+      join pg_class idx on idx.oid = ix.indexrelid
+      join pg_class tbl on tbl.oid = ix.indrelid
+      join pg_stats stats on stats.schemaname = 'absurd'
+        and stats.tablename = tbl.relname and stats.attname = 'org_key'
+        and stats.tablename = 'r_' || p_queue_name
+        and (stats.inherited or tbl.relkind <> 'p')
+    where ix.indexrelid = to_regclass(format('absurd.%I', 'r_' || p_queue_name ||
+        case when v_worker_id like '%:continuations' then '_fc' else '_fq' end))
+      and ix.indisvalid and ix.indisready
+      -- Require an eightfold candidate reduction to avoid sparse-tenant
+      -- random lookup amplification relative to the original window plan.
+      -- Catalog estimates select only an equivalent plan, never admission.
+      and (case when idx.relkind = 'I' then (
+        select coalesce(sum(greatest(leaf.reltuples, 0)), 0)
+          from pg_partition_tree(ix.indexrelid) tree
+          join pg_class leaf on leaf.oid = tree.relid where tree.isleaf
+        ) else idx.reltuples end) > 8::double precision * v_qty * greatest(1,
+        case when stats.n_distinct < 0 then -stats.n_distinct * tbl.reltuples
+             else stats.n_distinct end)
+  ) and not v_due_cancellations then
+${optimized}
+  else
+${original}
+  end if;`;
+  return (noCapClaim.slice(0, start) + choice + noCapClaim.slice(end)).replace(
+    '  v_sql text;',
+    '  v_sql text;\n  v_due_cancellations boolean;\n  v_has_ready_run boolean;',
+  );
+}

package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-pool.ts

@@ -12,6 +12,7 @@
  */
 import { createRequire } from 'node:module';
 import { AsyncLocalStorage } from 'node:async_hooks';
+import { optimizeAbsurdFairClaimSql } from './absurd-execution-queries';
 import { readFileSync } from 'node:fs';
 import path from 'node:path';
 import {
@@ -142,10 +143,7 @@ export function readAbsurdSchemaSql(): string {
  * claim function deliberately ignores the retired setting, so it cannot limit
  * scheduler claims even while its compatibility-only column exists.
  */
-export function readAbsurdSchemaNoSchedulerOrgCapCompatibilitySql(): string {
-  if (cachedNoSchedulerOrgCapCompatibilitySql !== null) {
-    return cachedNoSchedulerOrgCapCompatibilitySql;
-  }
+export function readAbsurdSchemaUncappedClaimSql(): string {
   const sql = readAbsurdSchemaSql();
   const claim = schemaSection(
     sql,
@@ -188,10 +186,16 @@ export function readAbsurdSchemaNoSchedulerOrgCapCompatibilitySql(): string {
   }
   // The installed v4 claim function has the identical signature and result
   // type. Replacing only its body is safe for old and new processes alike.
-  cachedNoSchedulerOrgCapCompatibilitySql = noCapClaim.replace(
+  return noCapClaim.replace(
     'create function absurd.claim_task',
     'create or replace function absurd.claim_task',
   );
+}
+
+export function readAbsurdSchemaNoSchedulerOrgCapCompatibilitySql(): string {
+  cachedNoSchedulerOrgCapCompatibilitySql ??= optimizeAbsurdFairClaimSql(
+    readAbsurdSchemaUncappedClaimSql(),
+  );
   return cachedNoSchedulerOrgCapCompatibilitySql;
 }