deepline 0.3.286 → 0.3.287

package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/runtime-event-search.ts

@@ -22,6 +22,11 @@ const EVENTS_TABLE = 'runtime_observability.events';
 const RUNTIME_EVENT_LOG_DEDUPE_INDEX = 'runtime_events_log_attempt_digest_idx';
 const RUNTIME_EVENT_LOG_POSITION_INDEX =
   'runtime_events_log_attempt_position_idx';
+const RUNTIME_EVENT_LOG_POSITION_CHANNELS = [
+  'worker',
+  'replay',
+  'envelope',
+] as const;
 const RUNTIME_EVENT_LOG_LEGACY_INDEX =
   'runtime_events_legacy_log_normalize_idx';
 const RUNTIME_EVENT_LOG_BODY_PAGE_INDEX = 'runtime_events_run_log_body_seq_idx';
@@ -51,7 +56,7 @@ const RUNTIME_EVENT_LOG_POSITION_EXPRESSION = `jsonb_set(
 // Migration readiness tracks this projection body independently of scheduler
 // wire and schema versions. Bump when a changed trigger must be reinstalled.
 export const RUNTIME_EVENT_SEARCH_OUTBOX_PROJECTION_MARKER =
-  'runtime-event-search-log-context-v13-index-first-position';
+  'runtime-event-search-log-context-v14-btree-position';
 export const RUNTIME_EVENT_CANONICAL_RETENTION_INDEX_READY_MARKER =
   'runtime-event-canonical-retention-v1';
 export const RUNTIME_EVENTS_RETENTION_MS = 7 * 24 * 60 * 60 * 1000;
@@ -1263,10 +1268,7 @@ export async function attachRuntimeEventSearchTriggers(
                   'logPositions', jsonb_build_object(log_channel, log_position));
                 position_alias := jsonb_build_object(
                   'producerAttempt', (item->>'producerAttempt')::integer,
-                  '_runtimeScope:' || ${quoteLiteral(scope)}, true,
-                  '_orgId:' || run_org, true,
-                  '_runId:' || observation.run_id, true)
-                  || jsonb_build_object(log_channel, log_position);
+                  log_channel, log_position);
                 matched_event_id := NULL;
                 position_event_ids := ARRAY[
                   legacy_position_prefix || log_position::text,
@@ -1293,24 +1295,26 @@ export async function attachRuntimeEventSearchTriggers(
                  ORDER BY event.event_id
                  LIMIT 1;
                 IF matched_event_id IS NULL THEN
-                  -- A transport may already be aliased onto an event whose
-                  -- primary key belongs to another channel. Find that alias
-                  -- independently of content so changed bytes still fail the
-                  -- immutable-position check below. Materialize the scoped
-                  -- position lookup before applying run filters: otherwise
-                  -- the planner can choose the run-history index and inspect
-                  -- every old log for each incoming position, even with a
-                  -- custom plan. The indexed document already binds scope,
-                  -- org, run, attempt and channel position; retain the outer
-                  -- identity checks as defense in depth.
+                  -- Pack scope, organization and run into one indexed key.
+                  -- The materialization keeps generic plans from choosing a
+                  -- run-history scan based on the outer identity checks.
                   WITH position_matches AS MATERIALIZED (
+                    ${RUNTIME_EVENT_LOG_POSITION_CHANNELS.map(
+                      (channel) => `
                     SELECT event.event_id, event.source_digest, event.search_doc,
-                           event.runtime_scope, event.org_id, event.run_id, event.log_seq
+                           event.log_seq, event.runtime_scope, event.org_id, event.run_id
                       FROM runtime_observability.events AS event
-                     WHERE event.kind = ${RUN_EVENT_KIND_SQL_LITERAL.logAppended}
+                     WHERE log_channel = '${channel}'
+                       AND ARRAY[event.runtime_scope, event.org_id, event.run_id]
+                         = ARRAY[${quoteLiteral(scope)}, run_org, observation.run_id]
+                       AND event.kind = ${RUN_EVENT_KIND_SQL_LITERAL.logAppended}
                        AND event.search_doc ? 'producerAttempt'
                        AND event.search_doc ? 'logPositions'
-                       AND (${RUNTIME_EVENT_LOG_POSITION_EXPRESSION.replaceAll('search_doc', 'event.search_doc')}) @> position_alias
+                       AND (event.search_doc->'logPositions') ? '${channel}'
+                       AND event.search_doc->'producerAttempt' = position_alias->'producerAttempt'
+                       AND event.search_doc->'logPositions'->'${channel}' = position_alias->'${channel}'
+                    `,
+                    ).join(' UNION ALL ')}
                   )
                   SELECT event.event_id, event.source_digest, event.search_doc
                     INTO matched_event_id, existing_digest, matched_doc
@@ -2034,8 +2038,27 @@ const RUNTIME_EVENT_LOG_POSITION_INDEX_SPEC: RuntimeEventIndexExpectation = {
       WHERE ${RUNTIME_EVENT_LOG_POSITION_PREDICATE}`,
 };
 
+const RUNTIME_EVENT_LOG_CHANNEL_POSITION_INDEXES: RuntimeEventIndexExpectation[] =
+  RUNTIME_EVENT_LOG_POSITION_CHANNELS.map((channel) => ({
+    name: `runtime_events_log_${channel}_position_idx`,
+    keys: [
+      'ARRAY[runtime_scope, org_id, run_id]',
+      "(search_doc -> 'producerAttempt'::text)",
+      `(search_doc -> 'logPositions'::text) -> '${channel}'::text`,
+    ],
+    predicate: `${RUNTIME_EVENT_LOG_POSITION_PREDICATE}
+      AND (search_doc->'logPositions') ? '${channel}'`,
+    createSql: `CREATE INDEX CONCURRENTLY IF NOT EXISTS runtime_events_log_${channel}_position_idx
+      ON runtime_observability.events
+        ((ARRAY[runtime_scope, org_id, run_id]), (search_doc->'producerAttempt'),
+         (search_doc->'logPositions'->'${channel}'))
+      WHERE ${RUNTIME_EVENT_LOG_POSITION_PREDICATE}
+        AND (search_doc->'logPositions') ? '${channel}'`,
+  }));
+
 const RUNTIME_EVENT_QUERY_INDEXES: RuntimeEventIndexExpectation[] = [
   RUNTIME_EVENT_LOG_POSITION_INDEX_SPEC,
+  ...RUNTIME_EVENT_LOG_CHANNEL_POSITION_INDEXES,
   {
     name: RUNTIME_EVENT_LOG_DEDUPE_INDEX,
     keys: [
@@ -2167,20 +2190,25 @@ async function prepareRuntimeEventQueryIndex(
 export async function hasRuntimeEventLogPositionIndex(
   client: PostgresSchedulerQueryClient,
 ): Promise<boolean> {
-  return hasRuntimeEventQueryIndex(
-    client,
+  for (const expected of [
     RUNTIME_EVENT_LOG_POSITION_INDEX_SPEC,
-  );
+    ...RUNTIME_EVENT_LOG_CHANNEL_POSITION_INDEXES,
+  ]) {
+    if (!(await hasRuntimeEventQueryIndex(client, expected))) return false;
+  }
+  return true;
 }
 
-/** Prepare the existing position index before replacing the projection body. */
+/** Prepare old and new position indexes before replacing the projection body. */
 export async function prepareRuntimeEventLogPositionIndex(
   client: PostgresSchedulerQueryClient,
 ): Promise<void> {
-  await prepareRuntimeEventQueryIndex(
-    client,
+  for (const expected of [
     RUNTIME_EVENT_LOG_POSITION_INDEX_SPEC,
-  );
+    ...RUNTIME_EVENT_LOG_CHANNEL_POSITION_INDEXES,
+  ]) {
+    await prepareRuntimeEventQueryIndex(client, expected);
+  }
 }
 
 /** Build only operator-owned indexes, concurrently and outside a transaction. */

package/dist/bundling-sources/shared_libs/play-runtime/step-progress.ts

@@ -1,4 +1,17 @@
-import type { PlayStaticSubstep } from '../plays/static-pipeline';
+import {
+  getTopLevelPipelineSubsteps,
+  type PlayStaticPipeline,
+  type PlayStaticSubstep,
+} from '../plays/static-pipeline';
+import {
+  docflowOutputRoots,
+  flattenSubsteps,
+  resolveDocflowOwnerDataset,
+} from '../plays/docflow-binding-owner';
+import {
+  normalizePlayConditionCounts,
+  type PlayConditionCounts,
+} from './cell-provenance';
 import {
   normalizePlayDocflowNodeIoState,
   type PlayDocflowNodeIoState,
@@ -32,6 +45,7 @@ export type PlayVisualNodeProgress = {
   completedAt?: number | null;
   artifactTableNamespace?: string | null;
   nodeIo?: PlayDocflowNodeIoState;
+  conditionCounts?: PlayConditionCounts;
 };
 
 export type PlayVisualNodeState = {
@@ -162,6 +176,9 @@ export function normalizePlayVisualNodeProgressMap(
           ? null
           : undefined;
     const nodeIo = normalizePlayDocflowNodeIoState(rawProgress.nodeIo);
+    const conditionCounts = normalizePlayConditionCounts(
+      rawProgress.conditionCounts,
+    );
 
     normalized[nodeId] = {
       ...(completed !== undefined ? { completed } : {}),
@@ -180,11 +197,76 @@ export function normalizePlayVisualNodeProgressMap(
         ? { artifactTableNamespace }
         : {}),
       ...(nodeIo ? { nodeIo } : {}),
+      ...(conditionCounts ? { conditionCounts } : {}),
     };
   }
   return normalized;
 }
 
+/** Join observed predicates to authored decision nodes by their owning column. */
+export function projectDocflowConditionCounts(
+  pipeline: PlayStaticPipeline | null | undefined,
+  sheets: readonly {
+    tableNamespace: string;
+    conditions?: Record<string, PlayConditionCounts>;
+  }[],
+): Record<string, PlayConditionCounts> {
+  if (!pipeline?.docflow) return {};
+  const substeps = getTopLevelPipelineSubsteps(pipeline);
+  const namespaces = flattenSubsteps(substeps)
+    .filter((step) => step.type === 'dataset' || step.type === 'csv')
+    .map((step) =>
+      step.type === 'dataset'
+        ? (step.tableNamespace ?? step.field)
+        : step.field,
+    )
+    .sort((a, b) => b.length - a.length);
+  const decisions = new Set(
+    pipeline.docflow.nodes
+      .filter((node) => node.kind === 'decision' && !node.sketch)
+      .map((node) => node.id),
+  );
+  const result: Record<string, PlayConditionCounts> = {};
+  for (const binding of pipeline.docflow.bindings) {
+    if (!decisions.has(binding.nodeId)) continue;
+    const owner = resolveDocflowOwnerDataset(binding, substeps).substep;
+    if (!owner) continue;
+    const namespace = owner.tableNamespace ?? owner.field;
+    const roots = docflowOutputRoots(binding);
+    // Several output columns can carry different predicates. Do not combine them.
+    if (roots.length !== 1) continue;
+    const root = roots[0]!;
+    const column = owner.sheetContract?.columns.find(
+      (candidate) =>
+        candidate.field === root ||
+        candidate.id === root ||
+        candidate.sqlName === root,
+    );
+    const key = column?.sqlName ?? root;
+    for (const sheet of sheets) {
+      const sheetOwner = namespaces.find(
+        (candidate) =>
+          sheet.tableNamespace === candidate ||
+          sheet.tableNamespace.startsWith(`${candidate}_`),
+      );
+      if (sheetOwner !== namespace) continue;
+      const counts = normalizePlayConditionCounts(
+        sheet.conditions?.[key] ?? sheet.conditions?.[root],
+      );
+      if (!counts) continue;
+      const current = result[`docflow:${binding.nodeId}`] ?? {
+        true: 0,
+        false: 0,
+      };
+      result[`docflow:${binding.nodeId}`] = {
+        true: current.true + counts.true,
+        false: current.false + counts.false,
+      };
+    }
+  }
+  return result;
+}
+
 export function applyVisualNodeProgress(input: {
   nodeStates: readonly PlayVisualNodeState[];
   progressByNode: PlayVisualNodeProgressMap;

package/dist/bundling-sources/shared_libs/play-runtime/worker-api-types.ts

@@ -1,5 +1,6 @@
 import type { PlayDocflowNodeIoState } from './docflow-node-io';
 import type { PlayMapRowActivitySource } from './run-ledger';
+import type { PlayConditionCounts } from './cell-provenance';
 
 export type PlayStagedFileRef = {
   storageKind: 'r2';
@@ -33,6 +34,7 @@ export type PlayVisualNodeProgress = {
   completedAt?: number;
   artifactTableNamespace?: string | null;
   nodeIo?: PlayDocflowNodeIoState;
+  conditionCounts?: PlayConditionCounts;
 };
 
 export type PlayVisualNodeProgressMap = Record<string, PlayVisualNodeProgress>;