@stigmer/server 3.25.0 → 3.26.0

package/src/domain/workflowexecution/queries.ts

@@ -1,48 +1,159 @@
 /**
- * Shared full-scan load for the workflowexecution read surfaces (list,
- * listByWorkflow, getExecutionSummary, listPendingApprovals) — Go's
- * store.ListResources + proto.Unmarshal-continue loop, where malformed
- * rows are skipped rather than failing the read.
+ * The workflowexecution read surfaces' shared reads (list,
+ * listByWorkflow, getExecutionSummary, listPendingApprovals), all through
+ * the list index (list-index.ts beside this file; the store's contract in
+ * store/interface.ts) — the request's org or workflow key narrows in the
+ * store, newest created first. Malformed rows are skipped rather than
+ * failing the read, Go's proto.Unmarshal-continue loop.
+ *
+ * The two lists share one reader: the default order pages
+ * (pipeline/steps/list-page.ts), with the lane's filters and then the read
+ * scope on each batch; another sort field sorts the whole matching set and
+ * returns its first page_size entries with no token, because only the
+ * creation order is an index order.
  */
-import { fromBinary } from "@bufbuild/protobuf";
+import { create, fromBinary } from "@bufbuild/protobuf";
 
 import type { WorkflowExecution } from "@stigmer/protos/ai/stigmer/agentic/workflowexecution/v1/api_pb";
 import { WorkflowExecutionSchema } from "@stigmer/protos/ai/stigmer/agentic/workflowexecution/v1/api_pb";
+import {
+  ExecutionSortField,
+  WorkflowExecutionListSchema,
+} from "@stigmer/protos/ai/stigmer/agentic/workflowexecution/v1/io_pb";
+import type { WorkflowExecutionList } from "@stigmer/protos/ai/stigmer/agentic/workflowexecution/v1/io_pb";
 import { ApiResourceKind } from "@stigmer/protos/ai/stigmer/commons/apiresource/apiresourcekind/api_resource_kind_pb";
 
 import type { Logger } from "../../boot/logger.js";
-import { internalError } from "../../pipeline/errors.js";
+import type { CallerIdentity } from "../../extensions/identity.js";
+import type { ListReadScope } from "../../extensions/list-read-scope.js";
+import { restrictListByReadScope } from "../../extensions/list-read-scope.js";
+import { internalError, invalidArgumentError } from "../../pipeline/errors.js";
+import {
+  LIST_PAGE_MAX_SIZE,
+  listPageFingerprint,
+  readListPage,
+} from "../../pipeline/steps/list-page.js";
+import type { ListPageRequest } from "../../pipeline/steps/list-page.js";
 import type { Store } from "../../store/interface.js";
+import type { ListIndexQuery, ListIndexRow } from "../../store/list-index.js";
+import { applySortField } from "./execution-filter.js";
+import { workflowExecutionListIndex } from "./list-index.js";
+
+/** The keys the workflow-execution index declares. */
+export type WorkflowExecutionListKey = "workflow" | "workflow_instance";
+
+/** A stored row, or undefined (logged) when it does not decode. */
+export function decodeWorkflowExecution(
+  data: Uint8Array,
+  logger: Logger,
+): WorkflowExecution | undefined {
+  try {
+    return fromBinary(WorkflowExecutionSchema, data);
+  } catch (error) {
+    logger.warn("Failed to unmarshal workflow execution, skipping", {
+      error: error instanceof Error ? error.message : String(error),
+    });
+    return undefined;
+  }
+}
 
 /**
- * Loads every workflow execution. The Internal message varies by caller in
- * Go (each handler wraps the same store error with its own text), so the
- * caller supplies it.
+ * Every workflow execution the query matches, newest created first. The
+ * Internal message varies by caller in Go (each handler wraps the same
+ * store error with its own text), so the caller supplies it.
  */
-export async function loadAllWorkflowExecutions(
+export async function loadWorkflowExecutions(
   store: Store,
   logger: Logger,
+  query: ListIndexQuery<WorkflowExecutionListKey>,
   internalMessage: string,
 ): Promise<WorkflowExecution[]> {
-  let rows: Uint8Array[];
+  let rows: ListIndexRow[];
   try {
-    rows = await store.listResources(ApiResourceKind.workflow_execution);
+    rows = await store.queryResources(workflowExecutionListIndex, query);
   } catch (error) {
     throw internalError(error, internalMessage);
   }
-
   const executions: WorkflowExecution[] = [];
-  for (const data of rows) {
-    let execution: WorkflowExecution;
-    try {
-      execution = fromBinary(WorkflowExecutionSchema, data);
-    } catch (error) {
-      logger.warn("Failed to unmarshal workflow execution, skipping", {
-        error: error instanceof Error ? error.message : String(error),
-      });
-      continue;
+  for (const row of rows) {
+    const execution = decodeWorkflowExecution(row.data, logger);
+    if (execution !== undefined) {
+      executions.push(execution);
     }
-    executions.push(execution);
   }
   return executions;
 }
+
+export interface WorkflowExecutionListRead {
+  readonly store: Store;
+  readonly logger: Logger;
+  readonly listReadScope: ListReadScope | undefined;
+  readonly identity: CallerIdentity;
+  readonly query: ListIndexQuery<WorkflowExecutionListKey>;
+  readonly request: ListPageRequest;
+  readonly sortField: ExecutionSortField;
+  readonly sortAscending: boolean;
+  /** The lane's filters beyond the indexed predicates. */
+  readonly keep: (execution: WorkflowExecution) => boolean;
+  /** The request's other fields, which its token is bound to. */
+  readonly fingerprint: Readonly<Record<string, unknown>>;
+}
+
+/** One list response: a page in the default order, or the whole set sorted by another field. */
+export async function readWorkflowExecutionList(
+  read: WorkflowExecutionListRead,
+): Promise<WorkflowExecutionList> {
+  const scope = (
+    executions: WorkflowExecution[],
+  ): Promise<WorkflowExecution[]> =>
+    restrictListByReadScope(
+      read.listReadScope,
+      read.identity,
+      ApiResourceKind.workflow_execution,
+      executions,
+      "",
+    );
+
+  if (read.sortField === ExecutionSortField.UNSPECIFIED) {
+    const page = await readListPage({
+      store: read.store,
+      declaration: workflowExecutionListIndex,
+      query: read.query,
+      request: read.request,
+      fingerprint: listPageFingerprint(read.fingerprint),
+      decode: (data) => decodeWorkflowExecution(data, read.logger),
+      keep: read.keep,
+      scope,
+      failure: "failed to list workflow executions",
+    });
+    return create(WorkflowExecutionListSchema, {
+      entries: page.entries,
+      nextPageToken: page.nextPageToken,
+      totalPages: page.nextPageToken === "" ? 1 : 0,
+    });
+  }
+
+  // Another sort field never issues a token, so none can be continued.
+  if (read.request.pageToken !== "") {
+    throw invalidArgumentError("invalid page_token");
+  }
+  if (read.request.pageSize < 0) {
+    throw invalidArgumentError("page_size must not be negative");
+  }
+  const executions = await scope(
+    (
+      await loadWorkflowExecutions(
+        read.store,
+        read.logger,
+        read.query,
+        "failed to list workflow executions",
+      )
+    ).filter(read.keep),
+  );
+  applySortField(executions, read.sortField, read.sortAscending);
+  const size = Math.min(read.request.pageSize, LIST_PAGE_MAX_SIZE);
+  return create(WorkflowExecutionListSchema, {
+    entries: size === 0 ? executions : executions.slice(0, size),
+    totalPages: 1,
+  });
+}

package/src/extensions/list-read-scope.ts

@@ -81,18 +81,15 @@
  *     driver's cost is proportional to the candidates offered (the cloud
  *     checks each one against its authorization engine), and the whole
  *     kind across every tenant is what rolled back on 2026-09-14; the
- *     request's org is tens of rows. Two idioms carry the rule and both
- *     keep their reason: a lane whose Go/Java contract org-narrows in
- *     BOTH editions (environment, memory, schedule, agent-instance,
- *     agent-share and agent-channel lists; the getBy* parent filters)
- *     applies its own filter ABOVE the helper call and passes `""`; a lane
- *     whose contract is cross-org on OSS (`agentExecution.list`,
- *     `workflowExecution.list`, `listPendingApprovals`) hands the request
- *     org to the helper, which applies it before the scope ONLY when a
- *     scope is composed — a scope-less server still returns the input
- *     untouched with the org unconsulted (the first line above). Two lanes
- *     have no org on the wire (`session.list`, `apiKey.findAll`) and offer
- *     the kind until P1 entry 12 pages them.
+ *     request's org is tens of rows. The lanes over the session, execution
+ *     and artifact kinds narrow by the request's org or parent in the
+ *     store's indexed read (store/list-index.ts; every posture honours the
+ *     request) and page there (pipeline/steps/list-page.ts, one batch per
+ *     scope call), passing `""`; the small org-scoped lists (environment,
+ *     memory, schedule, agent-instance, agent-share and agent-channel; the
+ *     getBy* parent filters) apply their own filter above the helper call
+ *     and pass `""` too. `apiKey.findAll` has no org on the wire and offers
+ *     the kind: a few rows per person.
  *   - A KIND WHOSE AUTHORIZATION IS ITS PARENT'S CARRIES THAT PARENT ON
  *     EVERY CANDIDATE (stigmer-cloud 20260913.04 T04, Q-LB-33..36). When
  *     `kind_meta` declares PARENT scope with INHERITED owner

package/src/index.ts

@@ -380,11 +380,20 @@ export { newResolveSlugStep } from "./pipeline/steps/slug.js";
 // The driver interfaces and the store-fault classes the ratified mapping
 // keys on (typed not-found → NotFound; anything else rethrows as an
 // infrastructure fault — the guidelines' instanceof idiom).
-export type { Store } from "./store/interface.js";
+export type { Store, StoreOpenOptions } from "./store/interface.js";
 export {
   AuditNotFoundError,
   ResourceNotFoundError,
 } from "./store/interface.js";
+// The list index's read shapes, which `Store.queryResources` speaks
+// (store/list-index.ts). Declaring an index stays internal: the list is
+// the composition root's (boot/list-indexes.ts), one per server.
+export type {
+  ListIndexCursor,
+  ListIndexDeclaration,
+  ListIndexQuery,
+  ListIndexRow,
+} from "./store/list-index.js";
 // The maintenance-surface row shape (20260830.04 Stage 1, ruling Q3):
 // what findResourcesRawOrderedAfter pages and what
 // replaceResourceDataIfUnchanged guards on — the secret-convergence

package/src/pipeline/steps/__tests__/list-page.test.ts

@@ -0,0 +1,186 @@
+/**
+ * Pins the page loop every paged list lane shares (../list-page.ts),
+ * against a real store opened with the server's list indexes:
+ * `page_size` 0 reads the whole list with no token; a positive size walks
+ * every row in the store's order with no gap and no duplicate, the last
+ * page carrying no token (also when the rows end exactly on a page); a
+ * scope that refuses rows still fills pages and the cut resumes after the
+ * last row returned; a caller who may see nothing gets short pages bounded
+ * by the examine budget, each with a token, until the rows run out; a size
+ * above the contract's maximum is capped; and the token's refusals — a
+ * negative size, malformed text, a token issued for another request.
+ */
+import { create } from "@bufbuild/protobuf";
+import { ConnectError, Code } from "@connectrpc/connect";
+import { afterEach, beforeEach, describe, expect, it } from "vitest";
+
+import { SessionSchema } from "@stigmer/protos/ai/stigmer/agentic/session/v1/api_pb";
+import type { Session } from "@stigmer/protos/ai/stigmer/agentic/session/v1/api_pb";
+import { ApiResourceKind } from "@stigmer/protos/ai/stigmer/commons/apiresource/apiresourcekind/api_resource_kind_pb";
+import { fromBinary } from "@bufbuild/protobuf";
+
+import { sessionListIndex } from "../../../domain/session/list-index.js";
+import { tempStore } from "../../../store/sqlite/__tests__/support.js";
+import type { TempStore } from "../../../store/sqlite/__tests__/support.js";
+import {
+  LIST_PAGE_MAX_SIZE,
+  examineBudgetOf,
+  listPageFingerprint,
+  readListPage,
+} from "../list-page.js";
+import type { ListPageParams } from "../list-page.js";
+
+let temp: TempStore;
+
+beforeEach(() => {
+  temp = tempStore();
+});
+
+afterEach(async () => {
+  await temp.cleanup();
+});
+
+async function seed(count: number): Promise<string[]> {
+  const ids: string[] = [];
+  for (let i = 0; i < count; i++) {
+    const id = `ses_${String(i).padStart(4, "0")}`;
+    ids.push(id);
+    await temp.store.saveResource(
+      ApiResourceKind.session,
+      id,
+      SessionSchema,
+      create(SessionSchema, {
+        metadata: { id, org: "acme" },
+        status: {
+          audit: {
+            specAudit: { createdAt: { seconds: BigInt(1_000 + i), nanos: 0 } },
+          },
+        },
+      }),
+    );
+  }
+  // Newest first: the highest creation instant first.
+  return ids.reverse();
+}
+
+function params(
+  pageSize: number,
+  pageToken: string,
+  scope: (rows: Session[]) => Session[] = (rows) => rows,
+): ListPageParams<Session, "agent_instance" | "channel"> {
+  return {
+    store: temp.store,
+    declaration: sessionListIndex,
+    query: { org: "acme" },
+    request: { pageSize, pageToken },
+    fingerprint: listPageFingerprint({ lane: "test", org: "acme" }),
+    decode: (data) => fromBinary(SessionSchema, data),
+    scope: (rows) => Promise.resolve(scope(rows)),
+    failure: "failed to list sessions",
+  };
+}
+
+async function walk(
+  pageSize: number,
+  scope?: (rows: Session[]) => Session[],
+): Promise<{ ids: string[]; pages: number }> {
+  const ids: string[] = [];
+  let token = "";
+  let pages = 0;
+  do {
+    const page = await readListPage(params(pageSize, token, scope));
+    ids.push(...page.entries.map((s) => s.metadata!.id));
+    token = page.nextPageToken;
+    pages += 1;
+  } while (token !== "");
+  return { ids, pages };
+}
+
+async function refusal(promise: Promise<unknown>): Promise<ConnectError> {
+  try {
+    await promise;
+  } catch (error) {
+    if (error instanceof ConnectError) {
+      return error;
+    }
+    throw error;
+  }
+  throw new Error("expected a refusal");
+}
+
+describe("readListPage", () => {
+  it("reads the whole list with no token at page_size 0", async () => {
+    const all = await seed(7);
+    const page = await readListPage(params(0, ""));
+    expect(page.entries.map((s) => s.metadata!.id)).toEqual(all);
+    expect(page.nextPageToken).toBe("");
+  });
+
+  it("walks every row with no gap and no duplicate, the last page without a token", async () => {
+    const all = await seed(7);
+    const { ids, pages } = await walk(3);
+    expect(ids).toEqual(all);
+    expect(pages).toBe(3);
+  });
+
+  it("issues no token when the rows end exactly on a page", async () => {
+    const all = await seed(6);
+    const { ids, pages } = await walk(3);
+    expect(ids).toEqual(all);
+    expect(pages).toBe(2);
+  });
+
+  it("fills pages around rows the scope refuses and resumes after the last row returned", async () => {
+    const all = await seed(12);
+    const refused = new Set(["ses_0010", "ses_0007", "ses_0006", "ses_0001"]);
+    const scope = (rows: Session[]) =>
+      rows.filter((s) => !refused.has(s.metadata!.id));
+    const first = await readListPage(params(3, "", scope));
+    expect(first.entries).toHaveLength(3);
+    const { ids } = await walk(3, scope);
+    expect(ids).toEqual(all.filter((id) => !refused.has(id)));
+  });
+
+  it("bounds what a caller who may see nothing examines, with a token until the rows run out", async () => {
+    await seed(250);
+    const noneVisible = () => [];
+    const first = await readListPage(params(20, "", noneVisible));
+    expect(first.entries).toEqual([]);
+    expect(first.nextPageToken).not.toBe("");
+    expect(examineBudgetOf(20)).toBe(100);
+    const { ids, pages } = await walk(20, noneVisible);
+    expect(ids).toEqual([]);
+    expect(pages).toBe(3);
+  });
+
+  it("caps a page at the contract's maximum", async () => {
+    await seed(LIST_PAGE_MAX_SIZE + 5);
+    const page = await readListPage(params(1_000, ""));
+    expect(page.entries).toHaveLength(LIST_PAGE_MAX_SIZE);
+    expect(page.nextPageToken).not.toBe("");
+  });
+
+  it("refuses a negative size, a malformed token and a token issued for another request", async () => {
+    await seed(4);
+    expect((await refusal(readListPage(params(-1, "")))).code).toBe(
+      Code.InvalidArgument,
+    );
+    const bad = await refusal(readListPage(params(2, "not a token")));
+    expect([bad.code, bad.rawMessage]).toEqual([
+      Code.InvalidArgument,
+      "invalid page_token",
+    ]);
+
+    const token = (await readListPage(params(2, ""))).nextPageToken;
+    const other = await refusal(
+      readListPage({
+        ...params(2, token),
+        fingerprint: listPageFingerprint({ lane: "test", org: "other" }),
+      }),
+    );
+    expect([other.code, other.rawMessage]).toEqual([
+      Code.InvalidArgument,
+      "page_token was issued for a different request",
+    ]);
+  });
+});