@stigmer/server 3.25.0 → 3.26.0

package/src/domain/workflowexecution/controller.ts

@@ -22,8 +22,6 @@ import type { ConnectRouter, HandlerContext } from "@connectrpc/connect";
 import { WorkflowExecutionSchema } from "@stigmer/protos/ai/stigmer/agentic/workflowexecution/v1/api_pb";
 import type { WorkflowExecution } from "@stigmer/protos/ai/stigmer/agentic/workflowexecution/v1/api_pb";
 import { WorkflowExecutionCommandController } from "@stigmer/protos/ai/stigmer/agentic/workflowexecution/v1/command_pb";
-import { ExecutionSortField } from "@stigmer/protos/ai/stigmer/agentic/workflowexecution/v1/io_pb";
-import { WorkflowExecutionListSchema } from "@stigmer/protos/ai/stigmer/agentic/workflowexecution/v1/io_pb";
 import type {
   ListWorkflowExecutionsByWorkflowRequest,
   ListWorkflowExecutionsRequest,
@@ -40,7 +38,6 @@ import type { Authorizer } from "../../extensions/authorizer.js";
 import type { ResolvedGateSteps } from "../../extensions/gate-slots.js";
 import { stepsForSlot } from "../../extensions/gate-slots.js";
 import type { ListReadScope } from "../../extensions/list-read-scope.js";
-import { restrictListByReadScope } from "../../extensions/list-read-scope.js";
 import type { CallerIdentity } from "../../extensions/identity.js";
 import type { ResourceAuthorizationLifecycle } from "../../extensions/resource-authorization.js";
 import { internalError, invalidArgumentError } from "../../pipeline/errors.js";
@@ -102,9 +99,8 @@ import { newCreateExecutionContextStep } from "./create-execution-context-step.j
 import type { WorkflowExecutionEngineStateProvider } from "./engine.js";
 import { newEnsureEngineAvailableStep } from "./engine.js";
 import {
-  applyFilterCriteria,
-  applyLegacyPhaseFilter,
-  applySortField,
+  matchesFilterCriteria,
+  matchesLegacyPhase,
 } from "./execution-filter.js";
 import {
   cancelExecution,
@@ -119,7 +115,7 @@ import { sendSignal } from "./send-signal.js";
 import { getEventLog } from "./get-event-log.js";
 import { getExecutionSummary } from "./get-execution-summary.js";
 import { listPendingApprovals } from "./list-pending-approvals.js";
-import { loadAllWorkflowExecutions } from "./queries.js";
+import { readWorkflowExecutionList } from "./queries.js";
 import { workflowExecutionRunTarget } from "./run-target.js";
 import { workflowExecutionSearchExtractor } from "./search-extractor.js";
 import type { SandboxLane } from "../../sandbox/lane.js";
@@ -484,60 +480,48 @@ async function get(
 }
 
 /**
- * List — list.go: full scan, the legacy top-level phase filter (only when
- * filter.phases is absent), structured filter criteria (T13), and the
- * sort default started_at descending. No pagination (total_pages
- * placeholder 1). With a composed ListReadScope (20260830.01, census
- * lane 6) the list narrows to the caller's authorized executions and the
- * request `org` narrows when non-blank (the Java
- * WorkflowExecutionListHandler baseline: blank = permission-bounded
- * across orgs); no scope = the full scan with `org` a deliberate no-op
- * on this single-tenant edition — byte-identical. The scope runs AFTER
- * the phase and criteria filters (stigmer-cloud 20260913.04 T02: the
- * scope is the last per-row predicate), so a composed driver is asked
- * about the rows the request keeps, not the kind.
+ * List — list.go: the request's org through the index (every org when
+ * blank), the legacy top-level phase filter (only when filter.phases is
+ * absent), structured filter criteria (T13), the read scope last (census
+ * lane 6; no scope = every matching execution). Newest created first and
+ * paged; another sort field sorts the whole set (queries.ts).
  */
 async function list(
   deps: WorkflowExecutionControllerDeps,
   req: ListWorkflowExecutionsRequest,
   identity: CallerIdentity,
 ): Promise<WorkflowExecutionList> {
-  let executions = await loadAllWorkflowExecutions(
-    deps.store,
-    deps.logger,
-    "failed to list workflow executions",
-  );
-
-  if (req.filter === undefined || req.filter.phases.length === 0) {
-    executions = applyLegacyPhaseFilter(executions, req.phase);
-  }
-  executions = applyFilterCriteria(executions, req.filter);
-  executions = await restrictListByReadScope(
-    deps.listReadScope,
+  const legacyPhase =
+    req.filter === undefined || req.filter.phases.length === 0;
+  return readWorkflowExecutionList({
+    store: deps.store,
+    logger: deps.logger,
+    listReadScope: deps.listReadScope,
     identity,
-    ApiResourceKind.workflow_execution,
-    executions,
-    req.org,
-  );
-
-  let sortField = req.sortField;
-  let ascending = req.sortAscending;
-  if (sortField === ExecutionSortField.UNSPECIFIED) {
-    sortField = ExecutionSortField.STARTED_AT;
-    ascending = false;
-  }
-  applySortField(executions, sortField, ascending);
-
-  return create(WorkflowExecutionListSchema, {
-    entries: executions,
-    totalPages: 1,
+    query: { org: req.org },
+    request: req,
+    sortField: req.sortField,
+    sortAscending: req.sortAscending,
+    keep: (execution) =>
+      (!legacyPhase || matchesLegacyPhase(execution, req.phase)) &&
+      matchesFilterCriteria(execution, req.filter),
+    fingerprint: {
+      lane: "workflowExecution.list",
+      org: req.org,
+      phase: req.phase,
+      tags: req.tags,
+      filter: req.filter,
+      sortField: req.sortField,
+      sortAscending: req.sortAscending,
+    },
   });
 }
 
 /**
  * ListByWorkflow — list_by_workflow.go: the request field accepts either
- * a Workflow ID or a WorkflowInstance ID, so both spec fields are
- * matched. Same filter/sort tail as list; no pagination.
+ * a Workflow ID or a WorkflowInstance ID, so both of the index's keys are
+ * read. Same filter, scope and order as list (census lane 7; bounded by
+ * the workflow id, org never consulted).
  */
 async function listByWorkflow(
   deps: WorkflowExecutionControllerDeps,
@@ -547,39 +531,27 @@ async function listByWorkflow(
   if (req.workflowId === "") {
     throw invalidArgumentError("workflow_id is required");
   }
-
-  // Census lane 7: workflow filter ∩ criteria ∩ authorized ids, the scope
-  // last (bounded by the workflow id; the Java handler never consults org
-  // on this lane).
-  const all = await loadAllWorkflowExecutions(
-    deps.store,
-    deps.logger,
-    "failed to list workflow executions",
-  );
-  let executions = all.filter(
-    (execution) =>
-      execution.spec?.workflowInstanceId === req.workflowId ||
-      execution.spec?.workflowId === req.workflowId,
-  );
-  executions = applyFilterCriteria(executions, req.filter);
-  executions = await restrictListByReadScope(
-    deps.listReadScope,
+  return readWorkflowExecutionList({
+    store: deps.store,
+    logger: deps.logger,
+    listReadScope: deps.listReadScope,
     identity,
-    ApiResourceKind.workflow_execution,
-    executions,
-    "",
-  );
-
-  let sortField = req.sortField;
-  let ascending = req.sortAscending;
-  if (sortField === ExecutionSortField.UNSPECIFIED) {
-    sortField = ExecutionSortField.STARTED_AT;
-    ascending = false;
-  }
-  applySortField(executions, sortField, ascending);
-
-  return create(WorkflowExecutionListSchema, {
-    entries: executions,
-    totalPages: 1,
+    query: {
+      anyKey: [
+        { name: "workflow", value: req.workflowId },
+        { name: "workflow_instance", value: req.workflowId },
+      ],
+    },
+    request: req,
+    sortField: req.sortField,
+    sortAscending: req.sortAscending,
+    keep: (execution) => matchesFilterCriteria(execution, req.filter),
+    fingerprint: {
+      lane: "workflowExecution.listByWorkflow",
+      workflowId: req.workflowId,
+      filter: req.filter,
+      sortField: req.sortField,
+      sortAscending: req.sortAscending,
+    },
   });
 }

package/src/domain/workflowexecution/execution-filter.ts

@@ -63,6 +63,14 @@ export function applyFilterCriteria(
   return executions.filter((execution) => matchesFilter(execution, filter));
 }
 
+/** The same criteria for one execution, for a page read's per-row filter. */
+export function matchesFilterCriteria(
+  execution: WorkflowExecution,
+  filter: ExecutionFilterCriteria | undefined,
+): boolean {
+  return filter === undefined || matchesFilter(execution, filter);
+}
+
 function matchesFilter(
   execution: WorkflowExecution,
   f: ExecutionFilterCriteria,
@@ -244,12 +252,17 @@ export function applyLegacyPhaseFilter(
   executions: WorkflowExecution[],
   phase: ExecutionPhase,
 ): WorkflowExecution[] {
-  if (phase === ExecutionPhase.EXECUTION_PHASE_UNSPECIFIED) {
-    return executions;
-  }
-  return executions.filter(
-    (execution) =>
-      (execution.status?.phase ??
-        ExecutionPhase.EXECUTION_PHASE_UNSPECIFIED) === phase,
+  return executions.filter((execution) => matchesLegacyPhase(execution, phase));
+}
+
+/** The legacy phase filter for one execution. */
+export function matchesLegacyPhase(
+  execution: WorkflowExecution,
+  phase: ExecutionPhase,
+): boolean {
+  return (
+    phase === ExecutionPhase.EXECUTION_PHASE_UNSPECIFIED ||
+    (execution.status?.phase ?? ExecutionPhase.EXECUTION_PHASE_UNSPECIFIED) ===
+      phase
   );
 }

package/src/domain/workflowexecution/get-execution-summary.ts

@@ -10,16 +10,15 @@
  * deliberately absent when nothing completed (an average over nothing is
  * not 0).
  *
- * With a composed ListReadScope (C2 Stage 4's ExecutionReadScope,
- * absorbed by 20260830.01 — the multi-tenant
- * edition), the scan narrows to the caller's authorized ids intersected
- * with the requested org (the Java GetExecutionSummary baseline: without
- * the org filter a member of several organizations sees every org's
- * numbers on every dashboard), and an EMPTY authorized set answers the
- * proto default instance — the conformance-pinned multi-tenant zero shape
- * (success_rate 0, NO cost summary) falls out of the scoping, never a
- * special case. No scope composed = the full scan above, byte-identical
- * (the request's org deliberately not consulted — single-user semantics).
+ * The rows are the requested org's, created within the time window, read
+ * through the list index (queries.ts) — every edition honours the org, so
+ * a member of several organizations never sees every org's numbers on
+ * every dashboard (the Java GetExecutionSummary baseline). With a composed
+ * ListReadScope those rows are offered to its restrict verb — exact, one
+ * read of the kind — and when the caller may see NONE of them the answer
+ * is the proto default instance: the conformance-pinned multi-tenant zero
+ * shape (success_rate 0, NO cost summary) falls out of the scoping, never
+ * a special case. No scope composed = every row of the org and window.
  *
  * Tie order in the two ranked lists is not wire-stable in Go (map
  * iteration feeds a stable sort), so ties here — deterministic first-seen
@@ -50,11 +49,13 @@ import { ApiResourceKind } from "@stigmer/protos/ai/stigmer/commons/apiresource/
 
 import type { Logger } from "../../boot/logger.js";
 import type { ListReadScope } from "../../extensions/list-read-scope.js";
+import { restrictListByReadScope } from "../../extensions/list-read-scope.js";
 import type { CallerIdentity } from "../../extensions/identity.js";
 import type { Store } from "../../store/interface.js";
+import { listIndexInstantOfMillis } from "../../store/list-index.js";
 
 import { parseRfc3339Ms } from "./execution-filter.js";
-import { loadAllWorkflowExecutions } from "./queries.js";
+import { loadWorkflowExecutions } from "./queries.js";
 
 const RANK_LIMIT = 10;
 
@@ -70,30 +71,36 @@ export async function getExecutionSummary(
   req: GetExecutionSummaryRequest,
   identity: CallerIdentity,
 ): Promise<ExecutionSummary> {
-  let executions = await loadAllWorkflowExecutions(
+  // The store's window is one millisecond wider than the loop's check
+  // below, which stays the authority: it only ever narrows the read.
+  const cutoffMs = resolveTimeCutoffMs(req.timeWindow);
+  let executions = await loadWorkflowExecutions(
     deps.store,
     deps.logger,
+    {
+      org: req.org,
+      ...(cutoffMs === undefined
+        ? {}
+        : { createdAtOrAfter: listIndexInstantOfMillis(cutoffMs - 1) }),
+    },
     "failed to list workflow executions for summary",
   );
 
   if (deps.listReadScope !== undefined) {
-    const authorizedIds = await deps.listReadScope.authorizedResourceIds(
+    executions = await restrictListByReadScope(
+      deps.listReadScope,
       identity,
       ApiResourceKind.workflow_execution,
+      executions,
+      "",
     );
-    if (authorizedIds.size === 0) {
+    if (executions.length === 0) {
       // The Java baseline's empty arm: the default instance, before any
       // aggregation — success_rate 0 and NO cost summary by construction.
       return create(ExecutionSummarySchema);
     }
-    executions = executions.filter(
-      (execution) =>
-        authorizedIds.has(execution.metadata?.id ?? "") &&
-        execution.metadata?.org === req.org,
-    );
   }
 
-  const cutoffMs = resolveTimeCutoffMs(req.timeWindow);
   const workflowFilter = req.workflowId;
 
   const phaseCounts: Record<number, number> = {};
@@ -203,9 +210,7 @@ export async function getExecutionSummary(
     for (const durationMs of completedDurationsMs) {
       totalMs += durationMs;
     }
-    summary.avgDuration = durationFromMs(
-      totalMs / completedDurationsMs.length,
-    );
+    summary.avgDuration = durationFromMs(totalMs / completedDurationsMs.length);
   }
 
   summary.topFailingWorkflows = buildFailureRanks(
@@ -255,7 +260,9 @@ function isActivePhase(phase: ExecutionPhase): boolean {
 }
 
 /** Go auditCreatedAt: nil timestamp → zero time (undefined here). */
-function auditCreatedAtMs(createdAt: Timestamp | undefined): number | undefined {
+function auditCreatedAtMs(
+  createdAt: Timestamp | undefined,
+): number | undefined {
   if (createdAt === undefined) {
     return undefined;
   }

package/src/domain/workflowexecution/list-index.ts

@@ -0,0 +1,22 @@
+/**
+ * The workflow-execution list index (store/list-index.ts): a run names
+ * its workflow and, when it ran through one, its instance;
+ * `listByWorkflow` accepts either id, so it reads both keys at once.
+ *
+ * A change to `keys` bumps `revision` (boot/__tests__/list-indexes.test.ts
+ * pins the pair).
+ */
+import { WorkflowExecutionSchema } from "@stigmer/protos/ai/stigmer/agentic/workflowexecution/v1/api_pb";
+import { ApiResourceKind } from "@stigmer/protos/ai/stigmer/commons/apiresource/apiresourcekind/api_resource_kind_pb";
+
+import { declareListIndex, field } from "../../store/list-index.js";
+
+export const workflowExecutionListIndex = declareListIndex({
+  kind: ApiResourceKind.workflow_execution,
+  schema: WorkflowExecutionSchema,
+  revision: 1,
+  keys: {
+    workflow: field("spec.workflow_id"),
+    workflow_instance: field("spec.workflow_instance_id"),
+  },
+});

package/src/domain/workflowexecution/list-pending-approvals.ts

@@ -1,8 +1,12 @@
 /**
  * ListPendingApprovals — ports list_pending_approvals.go (T14 dashboard):
- * scans IN_PROGRESS executions for tasks in WORKFLOW_TASK_WAITING_APPROVAL
- * and projects them into PendingApproval entries. total_count is the
- * pre-truncation total; only the entries list is trimmed to the page.
+ * reads the org's IN_PROGRESS executions for tasks in
+ * WORKFLOW_TASK_WAITING_APPROVAL and projects them into PendingApproval
+ * entries, newest execution first. total_count is the whole set's; the
+ * entries are one page (default 20, at most 100), continued by a token
+ * whose position is an execution's list-index cursor plus the approval's
+ * place among its waiting tasks — stable while approvals land and resolve
+ * between pages, where an offset would skip or repeat them.
  */
 import { create } from "@bufbuild/protobuf";
 import type { Timestamp } from "@bufbuild/protobuf/wkt";
@@ -29,14 +33,25 @@ import type { Logger } from "../../boot/logger.js";
 import type { CallerIdentity } from "../../extensions/identity.js";
 import type { ListReadScope } from "../../extensions/list-read-scope.js";
 import { restrictListByReadScope } from "../../extensions/list-read-scope.js";
+import { invalidArgumentError } from "../../pipeline/errors.js";
+import {
+  decodeListPageToken,
+  encodeListPageToken,
+  listPageFingerprint,
+} from "../../pipeline/steps/list-page.js";
+import type { ListPageTokenCursor } from "../../pipeline/steps/list-page.js";
 import type { Store } from "../../store/interface.js";
+import {
+  compareListIndexOrder,
+  listIndexInstant,
+} from "../../store/list-index.js";
 
 import {
   DEFAULT_PENDING_APPROVALS_PAGE_SIZE,
   MAX_PENDING_APPROVALS_PAGE_SIZE,
 } from "./constants.js";
 import { parseRfc3339Ms } from "./execution-filter.js";
-import { loadAllWorkflowExecutions } from "./queries.js";
+import { loadWorkflowExecutions } from "./queries.js";
 
 export interface PendingApprovalsDeps {
   readonly store: Store;
@@ -50,44 +65,68 @@ export async function listPendingApprovals(
   req: ListPendingApprovalsRequest,
   identity: CallerIdentity,
 ): Promise<PendingApprovalsList> {
-  // Census lane 8 (20260830.01): the scope narrows the EXECUTIONS — before
-  // the approvals projection and its pagination, the Java
-  // WorkflowExecutionListPendingApprovalsHandler order — and the request
-  // org narrows when non-blank (blank = permission-bounded across orgs).
-  // It runs over the executions that CAN carry a pending approval (in
-  // progress, a task waiting) rather than the org's whole history: the
-  // predicate is per-row, so the scope is last (stigmer-cloud 20260913.04
-  // T02) and a composed driver is asked about a handful of rows.
+  // Census lane 8: the org's executions that CAN carry a
+  // pending approval (in progress, a task waiting) through the list index,
+  // then the scope — the last per-row predicate, so a composed driver is
+  // asked about a handful of rows — before the approvals projection and
+  // its pagination, the Java WorkflowExecutionListPendingApprovalsHandler
+  // order.
+  if (req.pageSize < 0) {
+    throw invalidArgumentError("page_size must not be negative");
+  }
+  const fingerprint = listPageFingerprint({
+    lane: "workflowExecution.listPendingApprovals",
+    org: req.org,
+  });
+  const after =
+    req.pageToken === ""
+      ? undefined
+      : decodeListPageToken(req.pageToken, fingerprint);
+
   const executions = await restrictListByReadScope(
     deps.listReadScope,
     identity,
     ApiResourceKind.workflow_execution,
     (
-      await loadAllWorkflowExecutions(
+      await loadWorkflowExecutions(
         deps.store,
         deps.logger,
+        { org: req.org },
         "failed to list workflow executions for pending approvals",
       )
     ).filter(mayHavePendingApproval),
-    req.org,
+    "",
   );
 
   let pageSize = req.pageSize;
-  if (pageSize <= 0) {
+  if (pageSize === 0) {
     pageSize = DEFAULT_PENDING_APPROVALS_PAGE_SIZE;
   }
   if (pageSize > MAX_PENDING_APPROVALS_PAGE_SIZE) {
     pageSize = MAX_PENDING_APPROVALS_PAGE_SIZE;
   }
 
-  let approvals: PendingApproval[] = [];
+  // Newest execution first (the index's order), an execution's waiting
+  // tasks in their order; each approval's position is its execution's
+  // cursor and its place among that execution's waiting tasks.
+  const approvals: Array<{
+    readonly approval: PendingApproval;
+    readonly at: ListPageTokenCursor & { readonly position: number };
+  }> = [];
   for (const execution of executions) {
+    const cursor = {
+      createdAt: listIndexInstant(
+        execution.status?.audit?.specAudit?.createdAt,
+      ),
+      id: execution.metadata?.id ?? "",
+    };
+    let position = 0;
     for (const task of execution.status?.tasks ?? []) {
       if (task.status !== WorkflowTaskStatus.WORKFLOW_TASK_WAITING_APPROVAL) {
         continue;
       }
-      approvals.push(
-        create(PendingApprovalSchema, {
+      approvals.push({
+        approval: create(PendingApprovalSchema, {
           executionId: execution.metadata?.id ?? "",
           workflowName: execution.metadata?.name ?? "",
           // task_name, not task_id: the composite task_id ("gate:2")
@@ -98,21 +137,39 @@ export async function listPendingApprovals(
           requestedAt: parseTimestampString(task.startedAt),
           uiHint: task.uiHint,
         }),
-      );
+        at: { cursor, position },
+      });
+      position += 1;
     }
   }
 
-  const totalCount = approvals.length;
-  if (approvals.length > pageSize) {
-    approvals = approvals.slice(0, pageSize);
-  }
+  const start =
+    after === undefined
+      ? 0
+      : approvals.findIndex((a) => comesAfter(a.at, after));
+  const remaining = start < 0 ? [] : approvals.slice(start);
+  const page = remaining.slice(0, pageSize);
+  const last = page[page.length - 1];
 
   return create(PendingApprovalsListSchema, {
-    entries: approvals,
-    totalCount,
+    entries: page.map((a) => a.approval),
+    totalCount: approvals.length,
+    nextPageToken:
+      remaining.length > pageSize && last !== undefined
+        ? encodeListPageToken(last.at.cursor, fingerprint, last.at.position)
+        : "",
   });
 }
 
+/** Whether an approval comes strictly after a token's position. */
+function comesAfter(
+  at: ListPageTokenCursor & { readonly position: number },
+  after: ListPageTokenCursor,
+): boolean {
+  const order = compareListIndexOrder(at.cursor, after.cursor);
+  return order > 0 || (order === 0 && at.position > (after.position ?? -1));
+}
+
 /** Go parseTimestampString: RFC3339 or nothing (nil on parse failure). */
 function parseTimestampString(value: string): Timestamp | undefined {
   const ms = parseRfc3339Ms(value);