@stigmer/server 3.25.0 → 3.26.0

package/src/pipeline/steps/list-page.ts

@@ -0,0 +1,284 @@
+/**
+ * One page of a list lane read through the list index — the one
+ * implementation of the page loop every paged lane calls, so the rules
+ * below hold everywhere by construction:
+ *
+ * - The store narrows by the request's own predicates (organization, key,
+ *   creation window) and orders newest first (store/list-index.ts). The
+ *   lane's per-row filter runs on each batch, then the read scope, which
+ *   stays the last per-row predicate (extensions/list-read-scope.ts);
+ *   neither reorders, so a page is in the store's order.
+ * - `page_size` 0 reads every matching row and returns no token, the
+ *   behaviour every list lane had before it paged; a positive size is
+ *   capped at `LIST_PAGE_MAX_SIZE`.
+ * - A page fills batch by batch until it is full, the rows run out, or
+ *   `examineBudgetOf(size)` rows have been examined. A caller who may see
+ *   few of the organization's rows gets a short page, possibly empty, with
+ *   a token (the timeline's cursor contract, agentchannel/v1
+ *   conversation_io.proto), never an unbounded walk.
+ * - The token's cursor is the last row RETURNED when the page was cut,
+ *   else the last row EXAMINED, so no row is skipped and no refused row is
+ *   read twice. The token is opaque and versioned, carries a fingerprint
+ *   of the request's other fields (a mismatch is refused), and carries no
+ *   authority: every page re-runs the scope.
+ *
+ * Proven by __tests__/list-page.test.ts (the loop, the budget, the cut,
+ * the token) and end-to-end by the paging arms of the conformance suites.
+ */
+import { createHash } from "node:crypto";
+
+import type { Store } from "../../store/interface.js";
+import type {
+  ListIndexCursor,
+  ListIndexDeclaration,
+  ListIndexQuery,
+  ListIndexRow,
+} from "../../store/list-index.js";
+import { internalError, invalidArgumentError } from "../errors.js";
+
+/** The most entries one page returns, the contract's word on every paged request. */
+export const LIST_PAGE_MAX_SIZE = 100;
+
+/**
+ * The most rows one request examines for a page of `size`: five pages'
+ * worth, at least 100, at most 500. It bounds what one request asks of a
+ * composed read scope (on the hosted edition one authorization key per
+ * row, 50 per round trip) when the caller can see few of the rows.
+ */
+export function examineBudgetOf(size: number): number {
+  return Math.min(500, Math.max(100, size * 5));
+}
+
+/** The request fields every paged lane carries. */
+export interface ListPageRequest {
+  readonly pageSize: number;
+  readonly pageToken: string;
+}
+
+export interface ListPageParams<T, K extends string> {
+  readonly store: Store;
+  readonly declaration: ListIndexDeclaration<K>;
+  /** The request's own predicates, pushed into the store's indexed read. */
+  readonly query: Omit<ListIndexQuery<K>, "after" | "limit">;
+  readonly request: ListPageRequest;
+  /**
+   * The request's other fields, as text: a token is valid only for the
+   * request it was issued to.
+   */
+  readonly fingerprint: string;
+  /** Decodes a row; undefined skips it (the lane logs, as it always has). */
+  readonly decode: (data: Uint8Array) => T | undefined;
+  /** The lane's per-row filter beyond the indexed predicates. */
+  readonly keep?: (row: T) => boolean;
+  /** The read scope, bound by the lane to its caller and kind. */
+  readonly scope: (rows: T[]) => Promise<T[]>;
+  /** The failure copy for a store fault. */
+  readonly failure: string;
+}
+
+export interface ListPage<T> {
+  readonly entries: T[];
+  /** "" when the list is complete. */
+  readonly nextPageToken: string;
+}
+
+/** Reads one page (or, at `page_size` 0, the whole list) through the list index. */
+export async function readListPage<T, K extends string>(
+  params: ListPageParams<T, K>,
+): Promise<ListPage<T>> {
+  const { request } = params;
+  if (request.pageSize < 0) {
+    throw invalidArgumentError("page_size must not be negative");
+  }
+  let after =
+    request.pageToken === ""
+      ? undefined
+      : decodeListPageToken(request.pageToken, params.fingerprint).cursor;
+
+  if (request.pageSize === 0) {
+    const rows = await queryOrFail(params, after, undefined);
+    return {
+      entries: await admit(params, rows).then((a) => a.map((e) => e.row)),
+      nextPageToken: "",
+    };
+  }
+
+  const size = Math.min(request.pageSize, LIST_PAGE_MAX_SIZE);
+  const budget = examineBudgetOf(size);
+  const kept: Array<Admitted<T>> = [];
+  let examined = 0;
+  for (;;) {
+    // One row past the batch is read only to learn whether more follow,
+    // so a list that ends exactly on a page carries no token (and no
+    // empty page behind it); the next batch reads that row again.
+    const batchSize = Math.min(size, budget - examined);
+    const read = await queryOrFail(params, after, batchSize + 1);
+    const more = read.length > batchSize;
+    const rows = more ? read.slice(0, batchSize) : read;
+    examined += rows.length;
+    kept.push(...(await admit(params, rows)));
+    const last = rows[rows.length - 1];
+    if (last !== undefined) {
+      after = last.cursor;
+    }
+
+    if (kept.length >= size) {
+      const page = kept.slice(0, size);
+      const exhausted = !more && kept.length === size;
+      return {
+        entries: page.map((e) => e.row),
+        nextPageToken: exhausted
+          ? ""
+          : encodeListPageToken(
+              page[page.length - 1]!.cursor,
+              params.fingerprint,
+            ),
+      };
+    }
+    if (!more) {
+      return { entries: kept.map((e) => e.row), nextPageToken: "" };
+    }
+    if (examined >= budget) {
+      return {
+        entries: kept.map((e) => e.row),
+        nextPageToken: encodeListPageToken(after!, params.fingerprint),
+      };
+    }
+  }
+}
+
+interface Admitted<T> {
+  readonly row: T;
+  readonly cursor: ListIndexCursor;
+}
+
+async function queryOrFail<T, K extends string>(
+  params: ListPageParams<T, K>,
+  after: ListIndexCursor | undefined,
+  limit: number | undefined,
+): Promise<ListIndexRow[]> {
+  try {
+    return await params.store.queryResources(params.declaration, {
+      ...params.query,
+      ...(after === undefined ? {} : { after }),
+      ...(limit === undefined ? {} : { limit }),
+    });
+  } catch (error) {
+    throw internalError(error, params.failure);
+  }
+}
+
+/** A batch through the lane's filter and the scope, each row keeping its cursor. */
+async function admit<T, K extends string>(
+  params: ListPageParams<T, K>,
+  rows: ReadonlyArray<ListIndexRow>,
+): Promise<Array<Admitted<T>>> {
+  const decoded: Array<Admitted<T>> = [];
+  for (const row of rows) {
+    const value = params.decode(row.data);
+    if (
+      value !== undefined &&
+      (params.keep === undefined || params.keep(value))
+    ) {
+      decoded.push({ row: value, cursor: row.cursor });
+    }
+  }
+  if (decoded.length === 0) {
+    return [];
+  }
+  const allowed = new Set(await params.scope(decoded.map((d) => d.row)));
+  return decoded.filter((d) => allowed.has(d.row));
+}
+
+// =============================================================================
+// The token
+// =============================================================================
+
+const TOKEN_VERSION = 1;
+
+/**
+ * The position a token carries: the list index's cursor, plus, for a lane
+ * whose entries are finer than rows (the approvals of one execution), the
+ * entry's position within its row.
+ */
+export interface ListPageTokenCursor {
+  readonly cursor: ListIndexCursor;
+  readonly position?: number;
+}
+
+function fingerprintHash(fingerprint: string): string {
+  return createHash("sha256")
+    .update(fingerprint)
+    .digest("base64url")
+    .slice(0, 16);
+}
+
+/** An opaque, versioned token for the position after `cursor`. */
+export function encodeListPageToken(
+  cursor: ListIndexCursor,
+  fingerprint: string,
+  position?: number,
+): string {
+  const body = {
+    v: TOKEN_VERSION,
+    c: cursor.createdAt,
+    i: cursor.id,
+    f: fingerprintHash(fingerprint),
+    ...(position === undefined ? {} : { p: position }),
+  };
+  return Buffer.from(JSON.stringify(body)).toString("base64url");
+}
+
+/**
+ * The position a token carries, refused as `invalid page_token` (the copy
+ * every paged lane already answers with) when it is malformed, of a
+ * version this server never issued, or issued for another request.
+ */
+export function decodeListPageToken(
+  token: string,
+  fingerprint: string,
+): ListPageTokenCursor {
+  let body: unknown;
+  try {
+    const bytes = Buffer.from(token, "base64url");
+    if (bytes.toString("base64url") !== token) {
+      throw new Error("not canonical base64url");
+    }
+    body = JSON.parse(bytes.toString("utf8"));
+  } catch {
+    throw invalidArgumentError("invalid page_token");
+  }
+  if (
+    typeof body !== "object" ||
+    body === null ||
+    (body as { v?: unknown }).v !== TOKEN_VERSION ||
+    typeof (body as { c?: unknown }).c !== "string" ||
+    typeof (body as { i?: unknown }).i !== "string" ||
+    typeof (body as { f?: unknown }).f !== "string"
+  ) {
+    throw invalidArgumentError("invalid page_token");
+  }
+  const parsed = body as { c: string; i: string; f: string; p?: unknown };
+  if (parsed.f !== fingerprintHash(fingerprint)) {
+    throw invalidArgumentError("page_token was issued for a different request");
+  }
+  if (
+    parsed.p !== undefined &&
+    !(Number.isSafeInteger(parsed.p) && (parsed.p as number) >= 0)
+  ) {
+    throw invalidArgumentError("invalid page_token");
+  }
+  return {
+    cursor: { createdAt: parsed.c, id: parsed.i },
+    ...(parsed.p === undefined ? {} : { position: parsed.p as number }),
+  };
+}
+
+/** A request's other fields as the text its token is bound to. */
+export function listPageFingerprint(
+  fields: Readonly<Record<string, unknown>>,
+): string {
+  return JSON.stringify(fields, (_key, value: unknown) =>
+    typeof value === "bigint" ? value.toString() : value,
+  );
+}

package/src/query/activity/handler.ts

@@ -5,18 +5,15 @@
  *
  * This is the OSS twin of the cloud's ListRecentActivityHandler; the
  * merge, ordering, projection, and filtering semantics are deliberately
- * identical (stigmer#461). What differs is only what single-tenancy
- * removes: with NO ListReadScope composed there is no id enumeration
- * (the candidate set is every stored row) and the request's org is a
- * no-op (a recents filter stricter than the per-kind lists it summarizes
- * would hide locally-owned rows); load-all-then-sort-in-memory replaces
- * per-kind SQL LIMIT windows (each kind's newest page_size rows are a
- * superset of its contribution to the merged page — the identical final
- * list; the full scan is this store's contract, the same pattern every
- * OSS list handler uses). With a composed ListReadScope (20260830.01,
- * census lane 22) both loads narrow to the caller's authorized ids ∩
- * the request's org when non-blank — the Java handler's two
- * listAuthorizedResourceIds calls and its findRecentByIdsAndOrg org arm.
+ * identical (stigmer#461). Both loads read the request's org through the
+ * list index (every org when blank) — the same narrowing the per-kind
+ * lists it summarizes apply, so recents is never stricter than they are —
+ * and, with a composed ListReadScope (census lane 22), offer those rows to
+ * its restrict verb: one read per kind, exact, where the Java handler
+ * enumerated the caller's authorized ids and then scanned. Recents orders
+ * by the last update, not by creation, so each kind's org is read whole
+ * and merged in memory (each kind's newest page_size rows are a superset
+ * of its contribution to the merged page).
  *
  * Proven by __tests__/handler.test.ts (Go's handler_test.go arms) and
  * activity.conformance.test.ts on local.
@@ -35,7 +32,9 @@ import type {
   RecentActivityEntry,
 } from "@stigmer/protos/ai/stigmer/activity/v1/io_pb";
 import { SessionSchema } from "@stigmer/protos/ai/stigmer/agentic/session/v1/api_pb";
+import type { Session } from "@stigmer/protos/ai/stigmer/agentic/session/v1/api_pb";
 import { WorkflowExecutionSchema } from "@stigmer/protos/ai/stigmer/agentic/workflowexecution/v1/api_pb";
+import type { WorkflowExecution } from "@stigmer/protos/ai/stigmer/agentic/workflowexecution/v1/api_pb";
 import { ExecutionPhase } from "@stigmer/protos/ai/stigmer/agentic/workflowexecution/v1/enum_pb";
 import { ApiResourceKind } from "@stigmer/protos/ai/stigmer/commons/apiresource/apiresourcekind/api_resource_kind_pb";
 import type { ApiResourceAudit } from "@stigmer/protos/ai/stigmer/commons/apiresource/status_pb";
@@ -44,6 +43,9 @@ import type { ApiResourceMetadata } from "@stigmer/protos/ai/stigmer/commons/api
 import type { Logger } from "../../boot/logger.js";
 import type { CallerIdentity } from "../../extensions/identity.js";
 import type { ListReadScope } from "../../extensions/list-read-scope.js";
+import { restrictListByReadScope } from "../../extensions/list-read-scope.js";
+import { sessionListIndex } from "../../domain/session/list-index.js";
+import { workflowExecutionListIndex } from "../../domain/workflowexecution/list-index.js";
 import type { Store } from "../../store/interface.js";
 
 /**
@@ -91,10 +93,9 @@ export class ActivityHandler {
   ) {}
 
   /**
-   * Go ListRecentActivity: load both kinds, project to sidebar entries,
-   * merge-sort newest-first, trim to the page. With a composed scope the
-   * per-kind loads narrow to the caller's authorized ids ∩ the request's
-   * org (blank = permission-bounded across orgs, the repo convention).
+   * Go ListRecentActivity: load both kinds (the request's org, every org
+   * when blank; the scope's restrict verb when one is composed), project
+   * to sidebar entries, merge-sort newest-first, trim to the page.
    */
   async listRecentActivity(
     request: ListRecentActivityRequest,
@@ -102,21 +103,8 @@ export class ActivityHandler {
   ): Promise<ListRecentActivityResponse> {
     const pageSize = normalizePageSize(request.pageSize);
 
-    let sessionScope: ReadonlySet<string> | undefined;
-    let executionScope: ReadonlySet<string> | undefined;
-    if (this.listReadScope !== undefined) {
-      sessionScope = await this.listReadScope.authorizedResourceIds(
-        identity,
-        ApiResourceKind.session,
-      );
-      executionScope = await this.listReadScope.authorizedResourceIds(
-        identity,
-        ApiResourceKind.workflow_execution,
-      );
-    }
-
-    const sessions = await this.loadSessions(sessionScope, request.org);
-    const executions = await this.loadExecutions(executionScope, request.org);
+    const sessions = await this.loadSessions(identity, request.org);
+    const executions = await this.loadExecutions(identity, request.org);
 
     // Sessions before executions, then a stable sort: entries with equal
     // timestamps keep this insertion order — the same tie-break the
@@ -146,32 +134,36 @@ export class ActivityHandler {
   }
 
   /**
-   * Go loadSessions: every stored personal session projected to a recents
-   * entry; runtime-originated sessions excluded.
+   * Go loadSessions: the org's personal sessions the caller may see,
+   * projected to recents entries; runtime-originated sessions excluded.
    */
   private async loadSessions(
-    authorizedIds: ReadonlySet<string> | undefined,
+    identity: CallerIdentity,
     org: string,
   ): Promise<RecentActivityEntry[]> {
-    const rows = await this.store.listResources(ApiResourceKind.session);
-    const entries: RecentActivityEntry[] = [];
+    const rows = await this.store.queryResources(sessionListIndex, { org });
+    const personal: Session[] = [];
     for (const row of rows) {
-      let session;
+      let session: Session;
       try {
-        session = fromBinary(SessionSchema, row);
+        session = fromBinary(SessionSchema, row.data);
       } catch {
         this.logger.warn("Skipping undecodable session row in recent activity");
         continue;
       }
-      if (hasRuntimeOriginLabel(session.metadata)) {
-        continue;
-      }
-      if (
-        authorizedIds !== undefined &&
-        !isVisibleUnderScope(authorizedIds, org, session.metadata)
-      ) {
-        continue;
+      if (!hasRuntimeOriginLabel(session.metadata)) {
+        personal.push(session);
       }
+    }
+    const visible = await restrictListByReadScope(
+      this.listReadScope,
+      identity,
+      ApiResourceKind.session,
+      personal,
+      "",
+    );
+    const entries: RecentActivityEntry[] = [];
+    for (const session of visible) {
       entries.push(
         create(RecentActivityEntrySchema, {
           id: session.metadata?.id ?? "",
@@ -184,31 +176,33 @@ export class ActivityHandler {
     return entries;
   }
 
-  /** Go loadExecutions: every stored workflow execution projected. */
+  /** Go loadExecutions: the org's workflow executions the caller may see, projected. */
   private async loadExecutions(
-    authorizedIds: ReadonlySet<string> | undefined,
+    identity: CallerIdentity,
     org: string,
   ): Promise<RecentActivityEntry[]> {
-    const rows = await this.store.listResources(
-      ApiResourceKind.workflow_execution,
-    );
-    const entries: RecentActivityEntry[] = [];
+    const rows = await this.store.queryResources(workflowExecutionListIndex, {
+      org,
+    });
+    const decoded: WorkflowExecution[] = [];
     for (const row of rows) {
-      let execution;
       try {
-        execution = fromBinary(WorkflowExecutionSchema, row);
+        decoded.push(fromBinary(WorkflowExecutionSchema, row.data));
       } catch {
         this.logger.warn(
           "Skipping undecodable workflow execution row in recent activity",
         );
-        continue;
-      }
-      if (
-        authorizedIds !== undefined &&
-        !isVisibleUnderScope(authorizedIds, org, execution.metadata)
-      ) {
-        continue;
       }
+    }
+    const visible = await restrictListByReadScope(
+      this.listReadScope,
+      identity,
+      ApiResourceKind.workflow_execution,
+      decoded,
+      "",
+    );
+    const entries: RecentActivityEntry[] = [];
+    for (const execution of visible) {
       const name = execution.metadata?.name ?? "";
       entries.push(
         create(RecentActivityEntrySchema, {
@@ -217,7 +211,8 @@ export class ActivityHandler {
           subject: name === "" ? UNTITLED_EXECUTION_SUBJECT : name,
           updatedAt: extractUpdatedAt(execution.status?.audit),
           status: resolvePhase(
-            execution.status?.phase ?? ExecutionPhase.EXECUTION_PHASE_UNSPECIFIED,
+            execution.status?.phase ??
+              ExecutionPhase.EXECUTION_PHASE_UNSPECIFIED,
           ),
         }),
       );
@@ -226,22 +221,6 @@ export class ActivityHandler {
   }
 }
 
-/**
- * The scoped-visibility test both loads share when a scope is composed:
- * the row's id must be authorized and, when the request names an org, the
- * row must belong to it (the Java findRecentByIdsAndOrg arm).
- */
-function isVisibleUnderScope(
-  authorizedIds: ReadonlySet<string>,
-  org: string,
-  metadata: ApiResourceMetadata | undefined,
-): boolean {
-  if (!authorizedIds.has(metadata?.id ?? "")) {
-    return false;
-  }
-  return org === "" || (metadata?.org ?? "") === org;
-}
-
 /** Go normalizePageSize: ≤0 → default 30; >100 → cap 100. */
 export function normalizePageSize(requested: number): number {
   if (requested <= 0) {

package/src/store/README.md

@@ -1,40 +1,59 @@
 # store/ — the persistence layer
 
 Ports `backend/libs/go/store` (D2 §3, DD-003): `interface.ts` is the
-driver-agnostic contract (surface-for-surface with Go's `store.Store`, plus
-the consolidated members Go kept behind the `DB()` escape hatch — bootstrap
-state, signal dedupe, MCP OAuth). Two drivers implement it:
+driver-agnostic contract (surface-for-surface with Go's `store.Store`, plus the
+consolidated members Go kept behind the `DB()` escape hatch — bootstrap state,
+signal dedupe, MCP OAuth). Two drivers implement it:
 
-- `sqlite/` — the laptop-tier default (`node:sqlite`, zero-config, one
-  file) with the versioned migration chain (v1–v6 adopted from Go
-  DDL-faithful, v7 = the OD-3 consolidation).
+- `sqlite/` — the laptop-tier default (`node:sqlite`, zero-config, one file)
+  with its versioned migration chain (v1–v6 adopted from Go DDL-faithful, then
+  this port's own steps, each named at its constant in `sqlite/migrations.ts`).
 - `postgres/` — the self-host/team tier (DD-010: `pg` pool, row-level
-  `FOR UPDATE` atomicity, its own independent migration chain v1 with real
-  indexes, `tsvector`/`tsquery` search per DD-009).
+  `FOR UPDATE` atomicity, its own independent migration chain with real indexes,
+  `tsvector`/`tsquery` search per DD-009).
 
-Selection is boot config (`boot/config.ts`): `DATABASE_URL` set → Postgres
-(it wins when both are set — `DB_PATH` always has a default value); else
-sqlite on `DB_PATH`. Domain code depends on `interface.ts` only — never on
-a driver directory. Driver-neutral helpers shared by both drivers live
-here (`proto-fields.ts` reflection + scan semantics, `logger.ts`).
+Selection is boot config (`boot/config.ts`): `DATABASE_URL` set → Postgres (it
+wins when both are set — `DB_PATH` always has a default value); else sqlite on
+`DB_PATH`. Domain code depends on `interface.ts` only — never on a driver
+directory. Driver-neutral helpers shared by both drivers live here
+(`proto-fields.ts` reflection + scan semantics, `list-index.ts`, `logger.ts`).
+
+## The list index
+
+A list lane reads one organization's or one parent's rows in creation order
+through `Store.queryResources`, never by decoding the whole kind.
+`list-index.ts` is the one derivation: a kind's declaration (beside its lanes,
+`domain/<kind>/list-index.ts`) names its keys as data, the composition root
+opens the store with the one list of declarations (`boot/list-indexes.ts`), and
+every driver keeps a declared row's facts (organization, creation instant, keys)
+in the same statement as the row.
+
+Reads are exact whoever wrote the rows. A row whose facts are not proven current
+— written by a binary that does not know the index, as the old pod does while a
+roll overlaps it with the new one, or under another revision of the declaration
+— is evaluated from its bytes and repaired by a compare-and-set on those bytes.
+The migrations that add the index are schema only; the reconciliation each
+driver runs at open derives every unproven row, so the unproven set is empty in
+steady state and reads stay one index range. Every read shape names or forces
+its index, because a freshly written table has no planner statistics.
+
+## Contract and continuity
 
 The behavioral contract both drivers must satisfy identically is
-`__tests__/store-contract.ts`, invoked by each driver's
-`store-contract.test.ts` (sqlite always; Postgres under
-`TEST_DATABASE_URL` — visible skips locally, a real service container in
-CI). Driver-physical behavior stays in each driver's own tests. One
-deliberate semantic difference is recorded in DD-010: sqlite serializes
-ALL writes globally as a side effect of its single synchronous connection;
-Postgres guarantees per-resource atomicity only — the interface contract
-is the narrower one.
-
-Schema continuity across the Go cutover (any Go-created database adopts
-forward) is proven by `sqlite/__tests__/migrations.test.ts` against a real
-Go-created fixture. The fixture is FROZEN: the Go server retired
-(go-server-retirement, D4 #25), its schema can no longer change, and the
-committed v6 dump is the permanent record of what real pre-cutover
-databases look like. The generator script
-(`scripts/regen-go-db-fixture.sh`) lived until #25 and remains in git
-history should the fixture ever need forensic regeneration. The Postgres
-chain has no adoption story by construction — no Postgres database
-predates its driver.
+`__tests__/store-contract.ts`, invoked by each driver's `store-contract.test.ts`
+(sqlite always; Postgres under `TEST_DATABASE_URL` — visible skips locally, a
+real service container in CI). Driver-physical behavior stays in each driver's
+own tests. One deliberate semantic difference is recorded in DD-010: sqlite
+serializes ALL writes globally as a side effect of its single synchronous
+connection; Postgres guarantees per-resource atomicity only — the interface
+contract is the narrower one.
+
+Schema continuity across the Go cutover (any Go-created database adopts forward)
+is proven by `sqlite/__tests__/migrations.test.ts` against a real Go-created
+fixture. The fixture is FROZEN: the Go server retired (go-server-retirement, D4
+#25), its schema can no longer change, and the committed v6 dump is the
+permanent record of what real pre-cutover databases look like. The generator
+script (`scripts/regen-go-db-fixture.sh`) lived until #25 and remains in git
+history should the fixture ever need forensic regeneration. The Postgres chain
+has no adoption story by construction — no Postgres database predates its
+driver.