deepline 0.3.286 → 0.3.288

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (37) hide show
  1. package/dist/bundling-sources/sdk/src/client.ts +47 -24
  2. package/dist/bundling-sources/sdk/src/index.ts +4 -0
  3. package/dist/bundling-sources/sdk/src/release.ts +7 -1
  4. package/dist/bundling-sources/sdk/src/types.ts +36 -0
  5. package/dist/bundling-sources/shared_libs/play-runtime/cell-provenance.ts +16 -0
  6. package/dist/bundling-sources/shared_libs/play-runtime/context.ts +17 -12
  7. package/dist/bundling-sources/shared_libs/play-runtime/docflow-node-io.ts +22 -54
  8. package/dist/bundling-sources/shared_libs/play-runtime/ledger-safe-payload.ts +7 -26
  9. package/dist/bundling-sources/shared_libs/play-runtime/live-state-contract.ts +2 -0
  10. package/dist/bundling-sources/shared_libs/play-runtime/run-ledger.ts +38 -3
  11. package/dist/bundling-sources/shared_libs/play-runtime/run-observation.ts +12 -0
  12. package/dist/bundling-sources/shared_libs/play-runtime/run-snapshot-stream.ts +3 -0
  13. package/dist/bundling-sources/shared_libs/play-runtime/runtime-api.ts +43 -1
  14. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-runtime-worker.ts +19 -0
  15. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres-progress.ts +88 -1
  16. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres.ts +3 -1
  17. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/runtime-event-search.ts +53 -25
  18. package/dist/bundling-sources/shared_libs/play-runtime/secret-redaction.ts +47 -1
  19. package/dist/bundling-sources/shared_libs/play-runtime/step-progress.ts +83 -1
  20. package/dist/bundling-sources/shared_libs/play-runtime/worker-api-types.ts +2 -0
  21. package/dist/bundling-sources/shared_libs/security/secret-redaction.ts +1 -89
  22. package/dist/cli/index.js +580 -629
  23. package/dist/cli/index.mjs +457 -507
  24. package/dist/{client-B7ic-kY5.d.ts → client-FR_u0dxT.d.ts} +38 -1
  25. package/dist/{client-Dzf_KovW.d.mts → client-phK1Ixcl.d.mts} +38 -1
  26. package/dist/index.d.mts +2 -2
  27. package/dist/index.d.ts +2 -2
  28. package/dist/index.js +49 -26
  29. package/dist/index.mjs +49 -26
  30. package/dist/install-integrity.json +2 -2
  31. package/dist/plays/run-request.d.mts +1 -1
  32. package/dist/plays/run-request.d.ts +1 -1
  33. package/dist/release.d.mts +5 -1
  34. package/dist/release.d.ts +5 -1
  35. package/dist/release.js +6 -1
  36. package/dist/release.mjs +6 -1
  37. package/package.json +1 -1
@@ -116,6 +116,7 @@ import {
116
116
  resolveMapRowOutcomeKey,
117
117
  } from './map-row-outcome';
118
118
  import { DEEPLINE_CELL_META_FIELD } from '../plays/cell-staleness';
119
+ import type { PlayConditionCounts } from './cell-provenance';
119
120
  import {
120
121
  LEGACY_RUNTIME_RECEIPT_WIRE_FORMAT,
121
122
  LEGACY_RUNTIME_RECEIPT_WIRE_HEADER,
@@ -9207,6 +9208,7 @@ export type RuntimeSheetRowOutcomeCounts = {
9207
9208
  completed: number;
9208
9209
  failed: number;
9209
9210
  total: number;
9211
+ conditions?: Record<string, PlayConditionCounts>;
9210
9212
  };
9211
9213
 
9212
9214
  /**
@@ -9231,6 +9233,7 @@ export async function countRuntimeSheetRowOutcomes(
9231
9233
  runId: string;
9232
9234
  playName?: string;
9233
9235
  tableNamespace?: string;
9236
+ includeConditions?: boolean;
9234
9237
  },
9235
9238
  ): Promise<RuntimeSheetRowOutcomeCounts> {
9236
9239
  const { rows } = await client.query<{
@@ -9247,13 +9250,51 @@ export async function countRuntimeSheetRowOutcomes(
9247
9250
  [input.runId],
9248
9251
  );
9249
9252
  const row = rows[0];
9253
+ const conditions = input.includeConditions
9254
+ ? await countRuntimeSheetConditionOutcomes(client, input)
9255
+ : undefined;
9250
9256
  return {
9251
9257
  completed: Math.max(0, Number(row?.completed ?? 0) || 0),
9252
9258
  failed: Math.max(0, Number(row?.failed ?? 0) || 0),
9253
9259
  total: Math.max(0, Number(row?.total ?? 0) || 0),
9260
+ ...(conditions ? { conditions } : {}),
9254
9261
  };
9255
9262
  }
9256
9263
 
9264
+ /** Aggregate durable witnesses, including false/skipped and true/failed cells. */
9265
+ export async function countRuntimeSheetConditionOutcomes(
9266
+ client: {
9267
+ query<Row extends Record<string, unknown> = Record<string, unknown>>(
9268
+ sql: string,
9269
+ params?: unknown[],
9270
+ ): Promise<{ rows: Row[] }>;
9271
+ },
9272
+ input: { schema: string; table: string; runId: string },
9273
+ ): Promise<Record<string, PlayConditionCounts>> {
9274
+ const { rows } = await client.query<{
9275
+ field: string;
9276
+ true_count: number | string;
9277
+ false_count: number | string;
9278
+ }>(
9279
+ `SELECT cell.key AS field,
9280
+ count(*) FILTER (WHERE cell.value #>> '{decide,branch}' = 'run') AS true_count,
9281
+ count(*) FILTER (WHERE cell.value #>> '{decide,branch}' = 'else') AS false_count
9282
+ FROM ${quoteIdentifier(input.schema)}.${quoteIdentifier(input.table)} AS sheet
9283
+ CROSS JOIN LATERAL jsonb_each(coalesce(sheet._cell_meta, '{}'::jsonb)) AS cell(key, value)
9284
+ WHERE sheet._run_id = $1::text
9285
+ AND cell.value ->> 'runId' = $1::text
9286
+ AND cell.value #>> '{decide,branch}' IN ('run', 'else')
9287
+ GROUP BY cell.key`,
9288
+ [input.runId],
9289
+ );
9290
+ return Object.fromEntries(
9291
+ rows.map((row) => [
9292
+ row.field,
9293
+ { true: Number(row.true_count), false: Number(row.false_count) },
9294
+ ]),
9295
+ );
9296
+ }
9297
+
9257
9298
  /**
9258
9299
  * Read the settled row-outcome counts for a dataset's runtime sheet through the
9259
9300
  * run's preloaded DB session. Used by the scheduler worker at terminal finalize
@@ -9265,7 +9306,7 @@ export async function countRuntimeSheetRowOutcomes(
9265
9306
  */
9266
9307
  export async function readRuntimeSheetRowOutcomeCounts(
9267
9308
  context: RuntimeApiContext & { playName: string; runId: string },
9268
- input: { tableNamespace: string },
9309
+ input: { tableNamespace: string; includeConditions?: boolean },
9269
9310
  ): Promise<RuntimeSheetRowOutcomeCounts> {
9270
9311
  if (context.dbSessionStrategy === 'gateway_only') {
9271
9312
  throw new Error(
@@ -9290,6 +9331,7 @@ export async function readRuntimeSheetRowOutcomeCounts(
9290
9331
  runId: context.runId,
9291
9332
  playName: context.playName,
9292
9333
  tableNamespace: input.tableNamespace,
9334
+ includeConditions: input.includeConditions,
9293
9335
  }),
9294
9336
  );
9295
9337
  }
@@ -1947,6 +1947,10 @@ async function readTerminalSheetRowOutcomes(input: {
1947
1947
  completed: number;
1948
1948
  failed: number;
1949
1949
  total: number;
1950
+ conditions?: Record<
1951
+ string,
1952
+ import('../cell-provenance').PlayConditionCounts
1953
+ >;
1950
1954
  }>
1951
1955
  > {
1952
1956
  const sheetNamespaces = [
@@ -1981,11 +1985,26 @@ async function readTerminalSheetRowOutcomes(input: {
1981
1985
  completed: number;
1982
1986
  failed: number;
1983
1987
  total: number;
1988
+ conditions?: Record<
1989
+ string,
1990
+ import('../cell-provenance').PlayConditionCounts
1991
+ >;
1984
1992
  }> = [];
1985
1993
  for (const tableNamespace of sheetNamespaces) {
1986
1994
  try {
1987
1995
  const counts = await readRuntimeSheetRowOutcomeCounts(runtimeContext, {
1988
1996
  tableNamespace,
1997
+ includeConditions: Boolean(
1998
+ (
1999
+ input.launch.contractSnapshot as {
2000
+ staticPipeline?: {
2001
+ docflow?: { nodes?: Array<{ kind: string; sketch?: boolean }> };
2002
+ };
2003
+ } | null
2004
+ )?.staticPipeline?.docflow?.nodes?.some(
2005
+ (node) => node.kind === 'decision' && !node.sketch,
2006
+ ),
2007
+ ),
1989
2008
  });
1990
2009
  outcomes.push({ tableNamespace, ...counts });
1991
2010
  } catch (error) {
@@ -43,7 +43,12 @@ import {
43
43
  type PlayStaticPipeline,
44
44
  type PlayStaticSubstep,
45
45
  } from '../../plays/static-pipeline';
46
- import { getStaticSubstepNodeId } from '../step-progress';
46
+ import {
47
+ getStaticSubstepNodeId,
48
+ projectDocflowConditionCounts,
49
+ } from '../step-progress';
50
+ import type { PlayConditionCounts } from '../cell-provenance';
51
+ import { readRuntimeSheetRowOutcomeCounts } from '../runtime-api';
47
52
  import { runtimeApiBaseUrlFromLaunch } from './runtime-api-origin';
48
53
 
49
54
  // Patterns mirrored from runner events — they parse
@@ -232,6 +237,7 @@ export type PostgresSchedulerProgressReporter = {
232
237
  completed: number;
233
238
  failed: number;
234
239
  total: number;
240
+ conditions?: Record<string, PlayConditionCounts>;
235
241
  }>,
236
242
  ): void;
237
243
  /**
@@ -737,6 +743,9 @@ export function createPostgresSchedulerProgressReporter(input: {
737
743
  ? { completedAt: current.completedAt }
738
744
  : {}),
739
745
  ...(current.nodeIo !== undefined ? { nodeIo: current.nodeIo } : {}),
746
+ ...(current.conditionCounts !== undefined
747
+ ? { conditionCounts: current.conditionCounts }
748
+ : {}),
740
749
  ...(patch.completed !== undefined
741
750
  ? { completed: patch.completed }
742
751
  : {}),
@@ -783,6 +792,9 @@ export function createPostgresSchedulerProgressReporter(input: {
783
792
  ? { completedAt: patch.completedAt }
784
793
  : {}),
785
794
  ...(patch.nodeIo !== undefined ? { nodeIo: patch.nodeIo } : {}),
795
+ ...(patch.conditionCounts !== undefined
796
+ ? { conditionCounts: patch.conditionCounts }
797
+ : {}),
786
798
  updatedAt: Date.now(),
787
799
  },
788
800
  };
@@ -1474,6 +1486,7 @@ export function createPostgresSchedulerProgressReporter(input: {
1474
1486
  completed: number;
1475
1487
  failed: number;
1476
1488
  total: number;
1489
+ conditions?: Record<string, PlayConditionCounts>;
1477
1490
  }> = [];
1478
1491
 
1479
1492
  const applyTerminalSheetOutcomes = (
@@ -1482,6 +1495,7 @@ export function createPostgresSchedulerProgressReporter(input: {
1482
1495
  completed: number;
1483
1496
  failed: number;
1484
1497
  total: number;
1498
+ conditions?: Record<string, PlayConditionCounts>;
1485
1499
  }>,
1486
1500
  ): void => {
1487
1501
  terminalSheetOutcomes = outcomes;
@@ -1491,6 +1505,13 @@ export function createPostgresSchedulerProgressReporter(input: {
1491
1505
  // the two disagree; also the only terminal source when frames are missing
1492
1506
  // (detached push runs) and liveNodeProgress is empty (zero-event runs).
1493
1507
  const reconcileTerminalMapProgressFromSheetOutcomes = (): void => {
1508
+ const conditionCounts = projectDocflowConditionCounts(
1509
+ staticPipelineFromContractSnapshot(launch.contractSnapshot),
1510
+ terminalSheetOutcomes,
1511
+ );
1512
+ for (const [nodeId, counts] of Object.entries(conditionCounts)) {
1513
+ setNodeProgress(nodeId, { conditionCounts: counts });
1514
+ }
1494
1515
  for (const outcome of terminalSheetOutcomes) {
1495
1516
  const tableNamespace = outcome.tableNamespace.trim();
1496
1517
  if (!tableNamespace) continue;
@@ -1650,6 +1671,7 @@ export function buildFailedTerminalSettledProgressEvents(input: {
1650
1671
  tableNamespace: string;
1651
1672
  completed: number;
1652
1673
  failed: number;
1674
+ conditions?: Record<string, PlayConditionCounts>;
1653
1675
  }>;
1654
1676
  occurredAt: number;
1655
1677
  }): PlayRunLedgerEvent[] {
@@ -1680,6 +1702,21 @@ export function buildFailedTerminalSettledProgressEvents(input: {
1680
1702
  },
1681
1703
  });
1682
1704
  }
1705
+ for (const [stepId, conditionCounts] of Object.entries(
1706
+ projectDocflowConditionCounts(
1707
+ staticPipelineFromContractSnapshot(input.launch.contractSnapshot),
1708
+ input.outcomes,
1709
+ ),
1710
+ )) {
1711
+ events.push({
1712
+ type: 'step.progress',
1713
+ runId: input.launch.runId,
1714
+ source: 'worker',
1715
+ occurredAt: input.occurredAt,
1716
+ stepId,
1717
+ progress: { conditionCounts },
1718
+ });
1719
+ }
1683
1720
  return events;
1684
1721
  }
1685
1722
 
@@ -1724,6 +1761,56 @@ export async function collectGatewayRunnerTerminalProgressLedgerEvents(input: {
1724
1761
  captured.update = update;
1725
1762
  },
1726
1763
  });
1764
+ // The gateway can win terminal before the worker reconciles the sheet.
1765
+ // Carry predicate summaries in that same durable terminal outbox prefix.
1766
+ const pipeline = staticPipelineFromContractSnapshot(
1767
+ input.launch.contractSnapshot,
1768
+ );
1769
+ if (
1770
+ pipeline?.docflow?.nodes.some(
1771
+ (node) => node.kind === 'decision' && !node.sketch,
1772
+ )
1773
+ ) {
1774
+ const outcomes = [];
1775
+ const namespaces = new Set(
1776
+ (input.launch.preloadedDbSessions ?? [])
1777
+ .filter((session) => session.logicalTable === 'sheet_rows')
1778
+ .map((session) => session.tableNamespace),
1779
+ );
1780
+ for (const tableNamespace of namespaces) {
1781
+ try {
1782
+ outcomes.push({
1783
+ tableNamespace,
1784
+ ...(await readRuntimeSheetRowOutcomeCounts(
1785
+ {
1786
+ baseUrl: input.launch.baseUrl,
1787
+ executorToken: input.launch.executorToken,
1788
+ orgId: input.launch.orgId,
1789
+ playName: input.launch.playName,
1790
+ runId: input.launch.runId,
1791
+ dbSessionStrategy: 'preloaded',
1792
+ preloadedDbSessions:
1793
+ input.launch.preloadedDbSessions ?? undefined,
1794
+ postgresSessionUnwrapKey:
1795
+ input.launch.preloadedDbSessionUnwrapKey ??
1796
+ input.launch.executorToken,
1797
+ },
1798
+ { tableNamespace, includeConditions: true },
1799
+ )),
1800
+ });
1801
+ } catch (error) {
1802
+ console.warn(
1803
+ '[postgres-scheduler.terminal_condition_reconcile_failed]',
1804
+ {
1805
+ runId: input.launch.runId,
1806
+ tableNamespace,
1807
+ error: error instanceof Error ? error.message : String(error),
1808
+ },
1809
+ );
1810
+ }
1811
+ }
1812
+ reporter.applyTerminalSheetOutcomes(outcomes);
1813
+ }
1727
1814
  for (const event of input.events) reporter.onRunnerEvent(event);
1728
1815
  await reporter.finalize(input.result, {
1729
1816
  status: 'running',
@@ -334,7 +334,9 @@ const POSTGRES_SCHEDULER_SCHEMA_INIT_STATEMENT_TIMEOUT_MS = 60_000;
334
334
  // index stays operator-prepared concurrently before marked-run admission.
335
335
  // v56 adds a release-leading index for fixed-controller runner-terminal
336
336
  // recovery, which scans only active runs for each release lane.
337
- export const POSTGRES_SCHEDULER_SCHEMA_VERSION = 56;
337
+ // v57 refreshes log-position lookups after the operator concurrently prepares
338
+ // channel-specific indexes; outgoing readers retain the existing GIN index.
339
+ export const POSTGRES_SCHEDULER_SCHEMA_VERSION = 57;
338
340
  const POSTGRES_SCHEDULER_SCHEMA_MIGRATIONS_TABLE =
339
341
  'scheduler_schema_migrations';
340
342
  export const POSTGRES_SCHEDULER_OPEN_RECOVERY_INDEX =
@@ -22,6 +22,11 @@ const EVENTS_TABLE = 'runtime_observability.events';
22
22
  const RUNTIME_EVENT_LOG_DEDUPE_INDEX = 'runtime_events_log_attempt_digest_idx';
23
23
  const RUNTIME_EVENT_LOG_POSITION_INDEX =
24
24
  'runtime_events_log_attempt_position_idx';
25
+ const RUNTIME_EVENT_LOG_POSITION_CHANNELS = [
26
+ 'worker',
27
+ 'replay',
28
+ 'envelope',
29
+ ] as const;
25
30
  const RUNTIME_EVENT_LOG_LEGACY_INDEX =
26
31
  'runtime_events_legacy_log_normalize_idx';
27
32
  const RUNTIME_EVENT_LOG_BODY_PAGE_INDEX = 'runtime_events_run_log_body_seq_idx';
@@ -51,7 +56,7 @@ const RUNTIME_EVENT_LOG_POSITION_EXPRESSION = `jsonb_set(
51
56
  // Migration readiness tracks this projection body independently of scheduler
52
57
  // wire and schema versions. Bump when a changed trigger must be reinstalled.
53
58
  export const RUNTIME_EVENT_SEARCH_OUTBOX_PROJECTION_MARKER =
54
- 'runtime-event-search-log-context-v13-index-first-position';
59
+ 'runtime-event-search-log-context-v14-btree-position';
55
60
  export const RUNTIME_EVENT_CANONICAL_RETENTION_INDEX_READY_MARKER =
56
61
  'runtime-event-canonical-retention-v1';
57
62
  export const RUNTIME_EVENTS_RETENTION_MS = 7 * 24 * 60 * 60 * 1000;
@@ -1263,10 +1268,7 @@ export async function attachRuntimeEventSearchTriggers(
1263
1268
  'logPositions', jsonb_build_object(log_channel, log_position));
1264
1269
  position_alias := jsonb_build_object(
1265
1270
  'producerAttempt', (item->>'producerAttempt')::integer,
1266
- '_runtimeScope:' || ${quoteLiteral(scope)}, true,
1267
- '_orgId:' || run_org, true,
1268
- '_runId:' || observation.run_id, true)
1269
- || jsonb_build_object(log_channel, log_position);
1271
+ log_channel, log_position);
1270
1272
  matched_event_id := NULL;
1271
1273
  position_event_ids := ARRAY[
1272
1274
  legacy_position_prefix || log_position::text,
@@ -1293,24 +1295,26 @@ export async function attachRuntimeEventSearchTriggers(
1293
1295
  ORDER BY event.event_id
1294
1296
  LIMIT 1;
1295
1297
  IF matched_event_id IS NULL THEN
1296
- -- A transport may already be aliased onto an event whose
1297
- -- primary key belongs to another channel. Find that alias
1298
- -- independently of content so changed bytes still fail the
1299
- -- immutable-position check below. Materialize the scoped
1300
- -- position lookup before applying run filters: otherwise
1301
- -- the planner can choose the run-history index and inspect
1302
- -- every old log for each incoming position, even with a
1303
- -- custom plan. The indexed document already binds scope,
1304
- -- org, run, attempt and channel position; retain the outer
1305
- -- identity checks as defense in depth.
1298
+ -- Pack scope, organization and run into one indexed key.
1299
+ -- The materialization keeps generic plans from choosing a
1300
+ -- run-history scan based on the outer identity checks.
1306
1301
  WITH position_matches AS MATERIALIZED (
1302
+ ${RUNTIME_EVENT_LOG_POSITION_CHANNELS.map(
1303
+ (channel) => `
1307
1304
  SELECT event.event_id, event.source_digest, event.search_doc,
1308
- event.runtime_scope, event.org_id, event.run_id, event.log_seq
1305
+ event.log_seq, event.runtime_scope, event.org_id, event.run_id
1309
1306
  FROM runtime_observability.events AS event
1310
- WHERE event.kind = ${RUN_EVENT_KIND_SQL_LITERAL.logAppended}
1307
+ WHERE log_channel = '${channel}'
1308
+ AND ARRAY[event.runtime_scope, event.org_id, event.run_id]
1309
+ = ARRAY[${quoteLiteral(scope)}, run_org, observation.run_id]
1310
+ AND event.kind = ${RUN_EVENT_KIND_SQL_LITERAL.logAppended}
1311
1311
  AND event.search_doc ? 'producerAttempt'
1312
1312
  AND event.search_doc ? 'logPositions'
1313
- AND (${RUNTIME_EVENT_LOG_POSITION_EXPRESSION.replaceAll('search_doc', 'event.search_doc')}) @> position_alias
1313
+ AND (event.search_doc->'logPositions') ? '${channel}'
1314
+ AND event.search_doc->'producerAttempt' = position_alias->'producerAttempt'
1315
+ AND event.search_doc->'logPositions'->'${channel}' = position_alias->'${channel}'
1316
+ `,
1317
+ ).join(' UNION ALL ')}
1314
1318
  )
1315
1319
  SELECT event.event_id, event.source_digest, event.search_doc
1316
1320
  INTO matched_event_id, existing_digest, matched_doc
@@ -2034,8 +2038,27 @@ const RUNTIME_EVENT_LOG_POSITION_INDEX_SPEC: RuntimeEventIndexExpectation = {
2034
2038
  WHERE ${RUNTIME_EVENT_LOG_POSITION_PREDICATE}`,
2035
2039
  };
2036
2040
 
2041
+ const RUNTIME_EVENT_LOG_CHANNEL_POSITION_INDEXES: RuntimeEventIndexExpectation[] =
2042
+ RUNTIME_EVENT_LOG_POSITION_CHANNELS.map((channel) => ({
2043
+ name: `runtime_events_log_${channel}_position_idx`,
2044
+ keys: [
2045
+ 'ARRAY[runtime_scope, org_id, run_id]',
2046
+ "(search_doc -> 'producerAttempt'::text)",
2047
+ `(search_doc -> 'logPositions'::text) -> '${channel}'::text`,
2048
+ ],
2049
+ predicate: `${RUNTIME_EVENT_LOG_POSITION_PREDICATE}
2050
+ AND (search_doc->'logPositions') ? '${channel}'`,
2051
+ createSql: `CREATE INDEX CONCURRENTLY IF NOT EXISTS runtime_events_log_${channel}_position_idx
2052
+ ON runtime_observability.events
2053
+ ((ARRAY[runtime_scope, org_id, run_id]), (search_doc->'producerAttempt'),
2054
+ (search_doc->'logPositions'->'${channel}'))
2055
+ WHERE ${RUNTIME_EVENT_LOG_POSITION_PREDICATE}
2056
+ AND (search_doc->'logPositions') ? '${channel}'`,
2057
+ }));
2058
+
2037
2059
  const RUNTIME_EVENT_QUERY_INDEXES: RuntimeEventIndexExpectation[] = [
2038
2060
  RUNTIME_EVENT_LOG_POSITION_INDEX_SPEC,
2061
+ ...RUNTIME_EVENT_LOG_CHANNEL_POSITION_INDEXES,
2039
2062
  {
2040
2063
  name: RUNTIME_EVENT_LOG_DEDUPE_INDEX,
2041
2064
  keys: [
@@ -2167,20 +2190,25 @@ async function prepareRuntimeEventQueryIndex(
2167
2190
  export async function hasRuntimeEventLogPositionIndex(
2168
2191
  client: PostgresSchedulerQueryClient,
2169
2192
  ): Promise<boolean> {
2170
- return hasRuntimeEventQueryIndex(
2171
- client,
2193
+ for (const expected of [
2172
2194
  RUNTIME_EVENT_LOG_POSITION_INDEX_SPEC,
2173
- );
2195
+ ...RUNTIME_EVENT_LOG_CHANNEL_POSITION_INDEXES,
2196
+ ]) {
2197
+ if (!(await hasRuntimeEventQueryIndex(client, expected))) return false;
2198
+ }
2199
+ return true;
2174
2200
  }
2175
2201
 
2176
- /** Prepare the existing position index before replacing the projection body. */
2202
+ /** Prepare old and new position indexes before replacing the projection body. */
2177
2203
  export async function prepareRuntimeEventLogPositionIndex(
2178
2204
  client: PostgresSchedulerQueryClient,
2179
2205
  ): Promise<void> {
2180
- await prepareRuntimeEventQueryIndex(
2181
- client,
2206
+ for (const expected of [
2182
2207
  RUNTIME_EVENT_LOG_POSITION_INDEX_SPEC,
2183
- );
2208
+ ...RUNTIME_EVENT_LOG_CHANNEL_POSITION_INDEXES,
2209
+ ]) {
2210
+ await prepareRuntimeEventQueryIndex(client, expected);
2211
+ }
2184
2212
  }
2185
2213
 
2186
2214
  /** Build only operator-owned indexes, concurrently and outside a transaction. */
@@ -188,6 +188,7 @@ export type SecretRedactionContext = {
188
188
  /** Redacts only values registered by the secret resolver, preserving safe diagnostics. */
189
189
  redactRegisteredSecrets(value: string): string;
190
190
  redactString(value: string): string;
191
+ /** Redacts registered credential values recursively, without guessing from field names or patterns. */
191
192
  redactKnownSecrets<T>(value: T): T;
192
193
  redact<T>(value: T): T;
193
194
  };
@@ -335,7 +336,7 @@ export function createSecretRedactionContext(
335
336
  }
336
337
 
337
338
  function redactKnownSecrets<T>(value: T): T {
338
- if (typeof value === 'string') return redactString(value) as T;
339
+ if (typeof value === 'string') return redactRegisteredSecrets(value) as T;
339
340
  if (Array.isArray(value)) {
340
341
  return value.map((entry) => redactKnownSecrets(entry)) as T;
341
342
  }
@@ -360,3 +361,48 @@ export function createSecretRedactionContext(
360
361
  redact,
361
362
  };
362
363
  }
364
+
365
+ const REQUEST_CREDENTIAL_HEADER_NAMES = new Set([
366
+ 'authorization',
367
+ 'cookie',
368
+ 'proxy-authorization',
369
+ 'x-api-key',
370
+ 'api-key',
371
+ 'apikey',
372
+ 'x-auth',
373
+ 'auth-token',
374
+ 'access-token',
375
+ 'refresh-token',
376
+ 'session-token',
377
+ 'x-auth-token',
378
+ 'x-access-token',
379
+ 'x-refresh-token',
380
+ 'x-session-token',
381
+ ]);
382
+
383
+ export function createRequestHeaderSecretRedactionContext(
384
+ headers: Readonly<Record<string, string>>,
385
+ sensitiveHeaders: readonly string[] = [],
386
+ ): SecretRedactionContext {
387
+ const redactor = createSecretRedactionContext();
388
+ const declared = new Set(sensitiveHeaders.map((name) => name.toLowerCase()));
389
+ for (const [name, value] of Object.entries(headers)) {
390
+ const normalizedName = name.toLowerCase();
391
+ if (
392
+ !REQUEST_CREDENTIAL_HEADER_NAMES.has(normalizedName) &&
393
+ !declared.has(normalizedName)
394
+ ) {
395
+ continue;
396
+ }
397
+ redactor.register(value);
398
+ const auth = /^(?:bearer|basic)\s+(.+)$/i.exec(value.trim());
399
+ if (auth?.[1]) redactor.register(auth[1]);
400
+ if (normalizedName === 'cookie') {
401
+ for (const pair of value.split(';')) {
402
+ const separator = pair.indexOf('=');
403
+ if (separator > 0) redactor.register(pair.slice(separator + 1).trim());
404
+ }
405
+ }
406
+ }
407
+ return redactor;
408
+ }
@@ -1,4 +1,17 @@
1
- import type { PlayStaticSubstep } from '../plays/static-pipeline';
1
+ import {
2
+ getTopLevelPipelineSubsteps,
3
+ type PlayStaticPipeline,
4
+ type PlayStaticSubstep,
5
+ } from '../plays/static-pipeline';
6
+ import {
7
+ docflowOutputRoots,
8
+ flattenSubsteps,
9
+ resolveDocflowOwnerDataset,
10
+ } from '../plays/docflow-binding-owner';
11
+ import {
12
+ normalizePlayConditionCounts,
13
+ type PlayConditionCounts,
14
+ } from './cell-provenance';
2
15
  import {
3
16
  normalizePlayDocflowNodeIoState,
4
17
  type PlayDocflowNodeIoState,
@@ -32,6 +45,7 @@ export type PlayVisualNodeProgress = {
32
45
  completedAt?: number | null;
33
46
  artifactTableNamespace?: string | null;
34
47
  nodeIo?: PlayDocflowNodeIoState;
48
+ conditionCounts?: PlayConditionCounts;
35
49
  };
36
50
 
37
51
  export type PlayVisualNodeState = {
@@ -162,6 +176,9 @@ export function normalizePlayVisualNodeProgressMap(
162
176
  ? null
163
177
  : undefined;
164
178
  const nodeIo = normalizePlayDocflowNodeIoState(rawProgress.nodeIo);
179
+ const conditionCounts = normalizePlayConditionCounts(
180
+ rawProgress.conditionCounts,
181
+ );
165
182
 
166
183
  normalized[nodeId] = {
167
184
  ...(completed !== undefined ? { completed } : {}),
@@ -180,11 +197,76 @@ export function normalizePlayVisualNodeProgressMap(
180
197
  ? { artifactTableNamespace }
181
198
  : {}),
182
199
  ...(nodeIo ? { nodeIo } : {}),
200
+ ...(conditionCounts ? { conditionCounts } : {}),
183
201
  };
184
202
  }
185
203
  return normalized;
186
204
  }
187
205
 
206
+ /** Join observed predicates to authored decision nodes by their owning column. */
207
+ export function projectDocflowConditionCounts(
208
+ pipeline: PlayStaticPipeline | null | undefined,
209
+ sheets: readonly {
210
+ tableNamespace: string;
211
+ conditions?: Record<string, PlayConditionCounts>;
212
+ }[],
213
+ ): Record<string, PlayConditionCounts> {
214
+ if (!pipeline?.docflow) return {};
215
+ const substeps = getTopLevelPipelineSubsteps(pipeline);
216
+ const namespaces = flattenSubsteps(substeps)
217
+ .filter((step) => step.type === 'dataset' || step.type === 'csv')
218
+ .map((step) =>
219
+ step.type === 'dataset'
220
+ ? (step.tableNamespace ?? step.field)
221
+ : step.field,
222
+ )
223
+ .sort((a, b) => b.length - a.length);
224
+ const decisions = new Set(
225
+ pipeline.docflow.nodes
226
+ .filter((node) => node.kind === 'decision' && !node.sketch)
227
+ .map((node) => node.id),
228
+ );
229
+ const result: Record<string, PlayConditionCounts> = {};
230
+ for (const binding of pipeline.docflow.bindings) {
231
+ if (!decisions.has(binding.nodeId)) continue;
232
+ const owner = resolveDocflowOwnerDataset(binding, substeps).substep;
233
+ if (!owner) continue;
234
+ const namespace = owner.tableNamespace ?? owner.field;
235
+ const roots = docflowOutputRoots(binding);
236
+ // Several output columns can carry different predicates. Do not combine them.
237
+ if (roots.length !== 1) continue;
238
+ const root = roots[0]!;
239
+ const column = owner.sheetContract?.columns.find(
240
+ (candidate) =>
241
+ candidate.field === root ||
242
+ candidate.id === root ||
243
+ candidate.sqlName === root,
244
+ );
245
+ const key = column?.sqlName ?? root;
246
+ for (const sheet of sheets) {
247
+ const sheetOwner = namespaces.find(
248
+ (candidate) =>
249
+ sheet.tableNamespace === candidate ||
250
+ sheet.tableNamespace.startsWith(`${candidate}_`),
251
+ );
252
+ if (sheetOwner !== namespace) continue;
253
+ const counts = normalizePlayConditionCounts(
254
+ sheet.conditions?.[key] ?? sheet.conditions?.[root],
255
+ );
256
+ if (!counts) continue;
257
+ const current = result[`docflow:${binding.nodeId}`] ?? {
258
+ true: 0,
259
+ false: 0,
260
+ };
261
+ result[`docflow:${binding.nodeId}`] = {
262
+ true: current.true + counts.true,
263
+ false: current.false + counts.false,
264
+ };
265
+ }
266
+ }
267
+ return result;
268
+ }
269
+
188
270
  export function applyVisualNodeProgress(input: {
189
271
  nodeStates: readonly PlayVisualNodeState[];
190
272
  progressByNode: PlayVisualNodeProgressMap;
@@ -1,5 +1,6 @@
1
1
  import type { PlayDocflowNodeIoState } from './docflow-node-io';
2
2
  import type { PlayMapRowActivitySource } from './run-ledger';
3
+ import type { PlayConditionCounts } from './cell-provenance';
3
4
 
4
5
  export type PlayStagedFileRef = {
5
6
  storageKind: 'r2';
@@ -33,6 +34,7 @@ export type PlayVisualNodeProgress = {
33
34
  completedAt?: number;
34
35
  artifactTableNamespace?: string | null;
35
36
  nodeIo?: PlayDocflowNodeIoState;
37
+ conditionCounts?: PlayConditionCounts;
36
38
  };
37
39
 
38
40
  export type PlayVisualNodeProgressMap = Record<string, PlayVisualNodeProgress>;