@stigmer/server 3.25.0 → 3.26.0

package/src/store/__tests__/list-index.measure.test.ts

@@ -0,0 +1,293 @@
+/**
+ * What a list read costs at production's shape, measured on both drivers.
+ * Gated on `STORE_MEASURE=1` and SKIPPED otherwise (the
+ * `AUTHORIZATION_MEASURE` idiom of authorization/__tests__/
+ * list-read-scope.measure.test.ts): seeding tens of megabytes takes a
+ * while, and the numbers are recorded beside the change that moves them,
+ * not asserted — no budget is ruled for open source.
+ *
+ * The seed is the hosted edition's population as read on 2026-09-23:
+ * 1,279 agent executions averaging about 37 KB each (the kind's rows are
+ * fat: every message, tool call and todo of a run rides the row), 865
+ * sessions of about half a kilobyte, twelve organizations with one third
+ * of every kind in the busiest. The payload is uniform where production's
+ * is skewed (its largest execution is 4 MB), so these numbers understate
+ * the tail, never the median.
+ *
+ * Each shape is timed as the lane pays for it — the store read, the
+ * decode, the lane's own predicate — as the median of five runs after one
+ * warm-up. The "today" shapes read the whole kind the way every lane did
+ * before the list index; the "indexed" shapes read the same answer through
+ * `queryResources`. The status-write shape times the run's hottest write,
+ * an `updateResource` that changes no list key, which the index must not
+ * slow; it uses only the store API that predates the index, so the same
+ * file measures the code before and after.
+ */
+import { create, fromBinary } from "@bufbuild/protobuf";
+import { afterAll, describe, expect, it } from "vitest";
+
+import type { AgentExecution } from "@stigmer/protos/ai/stigmer/agentic/agentexecution/v1/api_pb";
+import { AgentExecutionSchema } from "@stigmer/protos/ai/stigmer/agentic/agentexecution/v1/api_pb";
+import { MessageType } from "@stigmer/protos/ai/stigmer/agentic/agentexecution/v1/enum_pb";
+import { SessionSchema } from "@stigmer/protos/ai/stigmer/agentic/session/v1/api_pb";
+import { ApiResourceKind } from "@stigmer/protos/ai/stigmer/commons/apiresource/apiresourcekind/api_resource_kind_pb";
+
+import { LIST_INDEXES } from "../../boot/list-indexes.js";
+import { agentExecutionListIndex } from "../../domain/agentexecution/list-index.js";
+import { sessionListIndex } from "../../domain/session/list-index.js";
+import type { Store } from "../interface.js";
+import { PostgresStore } from "../postgres/store.js";
+import {
+  createTestDatabase,
+  testDatabaseAdminUrl,
+} from "../postgres/__tests__/support.js";
+import type { TestDatabase } from "../postgres/__tests__/support.js";
+import { tempStore } from "../sqlite/__tests__/support.js";
+
+const MEASURE = process.env["STORE_MEASURE"] === "1";
+
+const ORGS = 12;
+const EXECUTIONS = 1_279;
+const SESSIONS = 865;
+const MESSAGES_PER_EXECUTION = 40;
+const MESSAGE_CHARS = 900;
+const BUSY_ORG = "org-00";
+const RUNS = 5;
+
+function orgOf(index: number): string {
+  // One third of every kind in the busiest organization, the rest spread.
+  return index % 3 === 0
+    ? BUSY_ORG
+    : `org-${String((index % (ORGS - 1)) + 1).padStart(2, "0")}`;
+}
+
+function sessionIdOf(index: number): string {
+  return `ses_${String(index).padStart(26, "0")}`;
+}
+
+function createdAt(index: number): { seconds: bigint; nanos: number } {
+  return { seconds: BigInt(1_790_000_000 + index), nanos: 0 };
+}
+
+async function seed(store: Store): Promise<void> {
+  const content = "x".repeat(MESSAGE_CHARS);
+  for (let i = 0; i < SESSIONS; i++) {
+    const id = sessionIdOf(i);
+    await store.saveResource(
+      ApiResourceKind.session,
+      id,
+      SessionSchema,
+      create(SessionSchema, {
+        metadata: { id, org: orgOf(i), name: `session ${i}` },
+        spec: { subject: `subject ${i}` },
+        status: {
+          audit: { specAudit: { createdAt: createdAt(i) } },
+        },
+      }),
+    );
+  }
+  for (let i = 0; i < EXECUTIONS; i++) {
+    const id = `aex_${String(i).padStart(26, "0")}`;
+    const session = i % SESSIONS;
+    await store.saveResource(
+      ApiResourceKind.agent_execution,
+      id,
+      AgentExecutionSchema,
+      create(AgentExecutionSchema, {
+        metadata: { id, org: orgOf(session) },
+        spec: { sessionId: sessionIdOf(session), message: "run it" },
+        status: {
+          messages: Array.from({ length: MESSAGES_PER_EXECUTION }, (_, m) => ({
+            type:
+              m % 2 === 0 ? MessageType.MESSAGE_HUMAN : MessageType.MESSAGE_AI,
+            content,
+            timestamp: "2026-09-23T10:00:00Z",
+          })),
+          audit: { specAudit: { createdAt: createdAt(i) } },
+        },
+      }),
+    );
+  }
+}
+
+async function median(
+  work: () => Promise<number>,
+): Promise<{ ms: number; rows: number }> {
+  await work();
+  const samples: number[] = [];
+  let rows = 0;
+  for (let run = 0; run < RUNS; run++) {
+    const started = performance.now();
+    rows = await work();
+    samples.push(performance.now() - started);
+  }
+  samples.sort((a, b) => a - b);
+  return { ms: Math.round(samples[Math.floor(RUNS / 2)] ?? 0), rows };
+}
+
+function decodeExecutions(rows: ReadonlyArray<Uint8Array>): AgentExecution[] {
+  return rows.map((row) => fromBinary(AgentExecutionSchema, row));
+}
+
+interface Shape {
+  readonly name: string;
+  run(store: Store): Promise<number>;
+}
+
+/** What each lane pays today: the whole kind, decoded, then its predicate. */
+const TODAY: ReadonlyArray<Shape> = [
+  {
+    name: "agentExecution.list(org) — whole kind, org in memory",
+    async run(store) {
+      const rows = await store.listResources(ApiResourceKind.agent_execution);
+      return decodeExecutions(rows).filter((e) => e.metadata?.org === BUSY_ORG)
+        .length;
+    },
+  },
+  {
+    name: "agentExecution.listBySession — whole kind, session in memory",
+    async run(store) {
+      const rows = await store.listResources(ApiResourceKind.agent_execution);
+      return decodeExecutions(rows).filter(
+        (e) => e.spec?.sessionId === sessionIdOf(0),
+      ).length;
+    },
+  },
+  {
+    name: "session.list — whole kind",
+    async run(store) {
+      const rows = await store.listResources(ApiResourceKind.session);
+      return rows.map((row) => fromBinary(SessionSchema, row)).length;
+    },
+  },
+];
+
+/** The same answers read through the list index. */
+const INDEXED: ReadonlyArray<Shape> = [
+  {
+    name: "agentExecution.list(org) — the organization's rows, every one",
+    async run(store) {
+      const rows = await store.queryResources(agentExecutionListIndex, {
+        org: BUSY_ORG,
+      });
+      return decodeExecutions(rows.map((row) => row.data)).length;
+    },
+  },
+  {
+    name: "agentExecution.list(org) — one page of 50",
+    async run(store) {
+      const rows = await store.queryResources(agentExecutionListIndex, {
+        org: BUSY_ORG,
+        limit: 50,
+      });
+      return decodeExecutions(rows.map((row) => row.data)).length;
+    },
+  },
+  {
+    name: "agentExecution.listBySession — the session key",
+    async run(store) {
+      const rows = await store.queryResources(agentExecutionListIndex, {
+        anyKey: [{ name: "session", value: sessionIdOf(0) }],
+      });
+      return decodeExecutions(rows.map((row) => row.data)).length;
+    },
+  },
+  {
+    name: "session.list(org) — one page of 50",
+    async run(store) {
+      const rows = await store.queryResources(sessionListIndex, {
+        org: BUSY_ORG,
+        limit: 50,
+      });
+      return rows.map((row) => fromBinary(SessionSchema, row.data)).length;
+    },
+  },
+];
+
+const STATUS_WRITES = 200;
+
+/** The run's hottest write: a status persist that changes no list key. */
+async function timeStatusWrites(store: Store): Promise<number> {
+  const id = `aex_${String(1).padStart(26, "0")}`;
+  const started = performance.now();
+  for (let i = 0; i < STATUS_WRITES; i++) {
+    await store.updateResource(
+      ApiResourceKind.agent_execution,
+      id,
+      AgentExecutionSchema,
+      (execution) => {
+        execution.status!.error = `write ${i}`;
+      },
+    );
+  }
+  return (performance.now() - started) / STATUS_WRITES;
+}
+
+let postgresDatabase: TestDatabase | undefined;
+
+const DRIVERS: ReadonlyArray<{
+  readonly name: string;
+  readonly skip: boolean;
+  open(): Promise<{ store: Store; close(): Promise<void> }>;
+}> = [
+  {
+    name: "sqlite",
+    skip: false,
+    async open() {
+      const temp = tempStore();
+      return { store: temp.store, close: () => temp.cleanup() };
+    },
+  },
+  {
+    name: "postgres",
+    skip: testDatabaseAdminUrl() === undefined,
+    async open() {
+      postgresDatabase = await createTestDatabase();
+      const store = await PostgresStore.open(
+        postgresDatabase.databaseUrl,
+        undefined,
+        {
+          listIndexes: LIST_INDEXES,
+        },
+      );
+      return { store, close: () => store.close() };
+    },
+  },
+];
+
+afterAll(async () => {
+  await postgresDatabase?.drop();
+});
+
+describe.skipIf(!MEASURE)("list reads at production's shape", () => {
+  describe.each(DRIVERS)("$name", (driver) => {
+    it.skipIf(driver.skip)(
+      "times every shape",
+      async () => {
+        const opened = await driver.open();
+        try {
+          await seed(opened.store);
+          for (const [label, shapes] of [
+            ["today", TODAY],
+            ["indexed", INDEXED],
+          ] as const) {
+            for (const shape of shapes) {
+              const { ms, rows } = await median(() => shape.run(opened.store));
+              console.log(
+                `[measure] ${driver.name} ${label} ${shape.name}: ${ms} ms (${rows} rows kept)`,
+              );
+              expect(rows).toBeGreaterThan(0);
+            }
+          }
+          const perWrite = await timeStatusWrites(opened.store);
+          console.log(
+            `[measure] ${driver.name} status write (updateResource, no key change): ${perWrite.toFixed(2)} ms per write over ${STATUS_WRITES}`,
+          );
+        } finally {
+          await opened.close();
+        }
+      },
+      600_000,
+    );
+  });
+});

package/src/store/__tests__/list-index.test.ts

@@ -0,0 +1,328 @@
+/**
+ * Pins the list index's one derivation (../list-index.ts), the pieces
+ * both drivers share: a declaration refuses a key its schema cannot
+ * answer at load; a row's facts are its organization, its creation
+ * instant as fixed-width text and every key it names; the instant sorts
+ * as text exactly as `compareCreatedAtDesc` orders timestamps (absent
+ * last); the query predicate and the merge the drivers finish with; and
+ * the registry's refusals. The drivers' SQL is pinned per driver by the
+ * list-index arms of store-contract.ts.
+ */
+import { create } from "@bufbuild/protobuf";
+import { describe, expect, it } from "vitest";
+
+import { AgentExecutionSchema } from "@stigmer/protos/ai/stigmer/agentic/agentexecution/v1/api_pb";
+import { ArtifactSchema } from "@stigmer/protos/ai/stigmer/agentic/artifact/v1/api_pb";
+import { SessionSchema } from "@stigmer/protos/ai/stigmer/agentic/session/v1/api_pb";
+import { ApiResourceKind } from "@stigmer/protos/ai/stigmer/commons/apiresource/apiresourcekind/api_resource_kind_pb";
+
+import { compareCreatedAtDesc } from "../../pipeline/steps/helpers.js";
+import {
+  ListIndexRegistry,
+  compareListIndexOrder,
+  declareListIndex,
+  field,
+  label,
+  listIndexFactsOf,
+  listIndexInstant,
+  listIndexInstantOfMillis,
+  matchesListIndexQuery,
+  mergeListIndexRows,
+  sameListKeyRows,
+} from "../list-index.js";
+import type { ListIndexFacts, ListIndexRow } from "../list-index.js";
+
+const sessions = declareListIndex({
+  kind: ApiResourceKind.session,
+  schema: SessionSchema,
+  revision: 1,
+  keys: {
+    agent_instance: field("spec.agent_instance_id"),
+    channel: label("stigmer.ai/channel-id"),
+  },
+});
+
+function row(createdAt: string, id: string): ListIndexRow {
+  return { id, data: new Uint8Array(), cursor: { createdAt, id } };
+}
+
+function facts(overrides: Partial<ListIndexFacts> = {}): ListIndexFacts {
+  return {
+    org: "acme",
+    createdAt: "2026-09-23T10:00:00.000000000Z",
+    revision: 1,
+    keys: [],
+    ...overrides,
+  };
+}
+
+describe("declareListIndex", () => {
+  it("refuses a path with no such field", () => {
+    expect(() =>
+      declareListIndex({
+        kind: ApiResourceKind.session,
+        schema: SessionSchema,
+        revision: 1,
+        keys: { wrong: field("spec.agent_instance") },
+      }),
+    ).toThrow(
+      "list index for session, key 'wrong': 'spec.agent_instance' has no field 'agent_instance'",
+    );
+  });
+
+  it("refuses a path that ends on a message, not a string", () => {
+    expect(() =>
+      declareListIndex({
+        kind: ApiResourceKind.session,
+        schema: SessionSchema,
+        revision: 1,
+        keys: { spec: field("spec") },
+      }),
+    ).toThrow("'spec' is not a string field");
+  });
+
+  it("refuses a path that walks through a scalar", () => {
+    expect(() =>
+      declareListIndex({
+        kind: ApiResourceKind.session,
+        schema: SessionSchema,
+        revision: 1,
+        keys: { deep: field("spec.agent_instance_id.nested") },
+      }),
+    ).toThrow(
+      "'agent_instance_id' in 'spec.agent_instance_id.nested' is not a message",
+    );
+  });
+
+  it("refuses a revision that is not a positive integer", () => {
+    expect(() =>
+      declareListIndex({
+        kind: ApiResourceKind.session,
+        schema: SessionSchema,
+        revision: 0,
+        keys: {},
+      }),
+    ).toThrow("revision must be a positive integer");
+  });
+});
+
+describe("listIndexFactsOf", () => {
+  it("reads the organization, the creation instant and every named key", () => {
+    const session = create(SessionSchema, {
+      metadata: {
+        id: "ses_1",
+        org: "acme",
+        labels: { "stigmer.ai/channel-id": "ach_1" },
+      },
+      spec: { agentInstanceId: "ain_1" },
+      status: {
+        audit: {
+          specAudit: { createdAt: { seconds: 1_790_000_000n, nanos: 5 } },
+        },
+      },
+    });
+    expect(listIndexFactsOf(sessions, session)).toEqual({
+      org: "acme",
+      createdAt: "2026-09-21T14:13:20.000000005Z",
+      revision: 1,
+      keys: [
+        { key: "agent_instance", value: "ain_1" },
+        { key: "channel", value: "ach_1" },
+      ],
+    });
+  });
+
+  it("leaves out a key the row does not name, and reads absences as empty", () => {
+    expect(listIndexFactsOf(sessions, create(SessionSchema, {}))).toEqual({
+      org: "",
+      createdAt: "",
+      revision: 1,
+      keys: [],
+    });
+  });
+
+  it("follows a nested field path", () => {
+    const artifacts = declareListIndex({
+      kind: ApiResourceKind.artifact,
+      schema: ArtifactSchema,
+      revision: 1,
+      keys: { agent_execution: field("spec.source.agent_execution_id") },
+    });
+    const artifact = create(ArtifactSchema, {
+      spec: { source: { agentExecutionId: "aex_1" } },
+    });
+    expect(listIndexFactsOf(artifacts, artifact).keys).toEqual([
+      { key: "agent_execution", value: "aex_1" },
+    ]);
+  });
+
+  it("refuses a declaration that was not made by declareListIndex", () => {
+    expect(() =>
+      listIndexFactsOf(
+        {
+          kind: ApiResourceKind.agent_execution,
+          schema: AgentExecutionSchema,
+          revision: 1,
+          keys: {},
+        },
+        create(AgentExecutionSchema, {}),
+      ),
+    ).toThrow("was not made by declareListIndex");
+  });
+});
+
+describe("listIndexInstant", () => {
+  it("is fixed-width UTC text to the nanosecond", () => {
+    expect(listIndexInstant({ seconds: 0n, nanos: 0 })).toBe(
+      "1970-01-01T00:00:00.000000000Z",
+    );
+    expect(listIndexInstantOfMillis(1_500)).toBe(
+      "1970-01-01T00:00:01.500000000Z",
+    );
+  });
+
+  it("is empty for an absent or unrepresentable stamp", () => {
+    expect(listIndexInstant(undefined)).toBe("");
+    expect(listIndexInstant({ seconds: 300_000_000_000n, nanos: 0 })).toBe("");
+  });
+
+  it("sorts as text exactly as compareCreatedAtDesc orders the timestamps", () => {
+    const stamps = [
+      { seconds: 10n, nanos: 2 },
+      undefined,
+      { seconds: 10n, nanos: 10 },
+      { seconds: 9n, nanos: 999_999_999 },
+      { seconds: 1_790_000_000n, nanos: 0 },
+    ];
+    const byTimestamp = [...stamps].sort((a, b) =>
+      compareCreatedAtDesc(a as never, b as never),
+    );
+    const byText = [...stamps].sort((a, b) =>
+      compareListIndexOrder(
+        { createdAt: listIndexInstant(a), id: "" },
+        { createdAt: listIndexInstant(b), id: "" },
+      ),
+    );
+    expect(byText).toEqual(byTimestamp);
+  });
+});
+
+describe("matchesListIndexQuery", () => {
+  const withChannel = facts({ keys: [{ key: "channel", value: "ach_1" }] });
+
+  it("matches an organization only exactly, and every organization when absent or empty", () => {
+    expect(matchesListIndexQuery("x", withChannel, { org: "acme" })).toBe(true);
+    expect(matchesListIndexQuery("x", withChannel, { org: "other" })).toBe(
+      false,
+    );
+    expect(matchesListIndexQuery("x", withChannel, { org: "" })).toBe(true);
+  });
+
+  it("matches any one of the listed keys", () => {
+    expect(
+      matchesListIndexQuery("x", withChannel, {
+        anyKey: [
+          { name: "agent_instance", value: "ain_9" },
+          { name: "channel", value: "ach_1" },
+        ],
+      }),
+    ).toBe(true);
+    expect(
+      matchesListIndexQuery("x", withChannel, {
+        anyKey: [{ name: "channel", value: "ach_2" }],
+      }),
+    ).toBe(false);
+  });
+
+  it("keeps rows created at or after the bound, and rows with no stamp", () => {
+    const bound = "2026-09-23T10:00:00.000000000Z";
+    expect(
+      matchesListIndexQuery("x", facts(), { createdAtOrAfter: bound }),
+    ).toBe(true);
+    expect(
+      matchesListIndexQuery(
+        "x",
+        facts({ createdAt: "2026-09-22T00:00:00.000000000Z" }),
+        {
+          createdAtOrAfter: bound,
+        },
+      ),
+    ).toBe(false);
+    expect(
+      matchesListIndexQuery("x", facts({ createdAt: "" }), {
+        createdAtOrAfter: bound,
+      }),
+    ).toBe(true);
+  });
+
+  it("keeps only rows strictly after the cursor in newest-first order", () => {
+    const at = "2026-09-23T10:00:00.000000000Z";
+    const after = { createdAt: at, id: "m" };
+    expect(
+      matchesListIndexQuery("a", facts({ createdAt: at }), { after }),
+    ).toBe(true);
+    expect(
+      matchesListIndexQuery("m", facts({ createdAt: at }), { after }),
+    ).toBe(false);
+    expect(
+      matchesListIndexQuery("z", facts({ createdAt: at }), { after }),
+    ).toBe(false);
+    expect(
+      matchesListIndexQuery(
+        "z",
+        facts({ createdAt: "2026-09-22T00:00:00.000000000Z" }),
+        { after },
+      ),
+    ).toBe(true);
+  });
+});
+
+describe("mergeListIndexRows", () => {
+  it("merges in order, keeps one row per id and cuts to the limit", () => {
+    const merged = mergeListIndexRows(
+      [row("2026-09-23", "b"), row("2026-09-21", "d")],
+      [row("2026-09-22", "c"), row("2026-09-23", "b"), row("", "e")],
+      3,
+    );
+    expect(merged.map((r) => r.id)).toEqual(["b", "c", "d"]);
+    expect(
+      mergeListIndexRows(
+        [row("", "e")],
+        [row("2026-09-23", "a")],
+        undefined,
+      ).map((r) => r.id),
+    ).toEqual(["a", "e"]);
+  });
+});
+
+describe("sameListKeyRows", () => {
+  it("is equal only when the instant and every key agree", () => {
+    const a = facts({ keys: [{ key: "channel", value: "ach_1" }] });
+    expect(
+      sameListKeyRows(a, facts({ keys: [{ key: "channel", value: "ach_1" }] })),
+    ).toBe(true);
+    expect(
+      sameListKeyRows(a, facts({ keys: [{ key: "channel", value: "ach_2" }] })),
+    ).toBe(false);
+    expect(sameListKeyRows(a, facts({ createdAt: "", keys: a.keys }))).toBe(
+      false,
+    );
+  });
+});
+
+describe("ListIndexRegistry", () => {
+  it("refuses a kind declared twice", () => {
+    expect(() => new ListIndexRegistry([sessions, sessions])).toThrow(
+      "list index for session is declared twice",
+    );
+  });
+
+  it("refuses a read through a declaration it was not opened with", () => {
+    const other = declareListIndex({ ...sessions, keys: {} });
+    const registry = new ListIndexRegistry([sessions]);
+    expect(() => registry.require(sessions)).not.toThrow();
+    expect(() => registry.require(other)).toThrow(
+      "list index for session is not registered with this store",
+    );
+  });
+});