@opengeni/db 4.4.1 → 4.5.0

package/src/session-history-tool-identities.ts

@@ -0,0 +1,55 @@
+import { sql } from "drizzle-orm";
+import { rawRows, type Database } from "./database";
+import { fromPostgresLosslessJson } from "./lossless-json";
+import { historyCallId, historyItemType } from "./session-tool-call-settlement";
+
+/** Receipt proof metadata only. Canonical model history is never projected here. */
+export async function readSessionHistoryToolIdentityPage(
+  tx: Pick<Database, "execute">,
+  scope: { workspaceId: string; sessionId: string; turnId: string },
+  before: number | null,
+  limit: number,
+): Promise<Array<{ position: number; callId: string | null; type: string | null }>> {
+  const rows = await rawRows<{
+    position: string;
+    item: Record<string, unknown>;
+    itemCodecVersion: number | null;
+  }>(
+    tx,
+    sql`
+    select page.position, page.item_codec_version as "itemCodecVersion",
+      jsonb_build_object(
+        'type', identity.type, 'callId', identity."callId", 'call_id', identity.call_id,
+        'id', identity.id, 'providerData', jsonb_build_object(
+          'callId', identity."providerData"->'callId',
+          'call_id', identity."providerData"->'call_id'
+        )
+      ) as item
+    from (
+      select position, item, item_codec_version from session_history_items
+      where workspace_id = ${scope.workspaceId}::uuid
+        and session_id = ${scope.sessionId}::uuid and turn_id = ${scope.turnId}::uuid
+        ${before === null ? sql`` : sql`and position < ${before}`}
+      order by position desc limit ${limit}
+    ) page
+    cross join lateral jsonb_to_record(
+      case when jsonb_typeof(page.item) in ('object', 'null') then page.item else '{}'::jsonb end
+    ) as identity(type jsonb, "callId" jsonb, call_id jsonb, id jsonb, "providerData" jsonb)
+    order by page.position desc
+  `,
+  );
+  // The synchronized JSONB mirror has the same field values as item_ordered.
+  // Decode only these fields with the original row's codec, then use the same
+  // identity precedence/type checks as the full-history reader. jsonb_to_record
+  // extracts them in one pass rather than detoasting a large output per field.
+  // A malformed JSON null still rejects the read; other primitive/array rows
+  // have no named identities, just as with the previous JavaScript lookup.
+  return rows.map((row) => {
+    const item = fromPostgresLosslessJson(row.item, row.itemCodecVersion);
+    return {
+      position: Number(row.position),
+      callId: historyCallId(item),
+      type: historyItemType(item),
+    };
+  });
+}

package/src/session-message-search.ts

@@ -1,5 +1,5 @@
 import { createHash } from "node:crypto";
-import { and, asc, eq, sql, type SQL } from "drizzle-orm";
+import { and, eq, sql, type SQL } from "drizzle-orm";
 import { z } from "zod";
 import type {
   SessionMessageSearchRequest,
@@ -35,6 +35,19 @@ const Cursor = z
   })
   .strict();
 type Position = z.infer<typeof Position>;
+type CandidateRow = {
+  visited: number;
+  rank: number | null;
+  sessionId: string | null;
+  eventId: string | null;
+  sequence: number | null;
+  turnId: string | null;
+  type: "user.message" | "agent.message.completed" | null;
+  sessionTitle: string | null;
+  messageId: unknown;
+  codec: number | null;
+  smallText: unknown;
+};
 
 export class SessionMessageSearchCursorError extends Error {
   constructor() {
@@ -50,6 +63,82 @@ export function sessionMessageLiteralPattern(query: string): RegExp {
   return new RegExp(query.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"), "iu");
 }
 
+// Highest code point with a Unicode case mapping (Adlam ends at U+1E943).
+const LAST_CASED_CODE_POINT = 0x1ffff;
+let caseClasses: ReadonlyMap<number, readonly number[]> | undefined;
+
+/** Every code point that `sessionMessageLiteralPattern` treats as the same
+ * character, keyed by each member. Characters absent from the map only match
+ * themselves. Built once from the runtime's own case mappings and verified
+ * against its `iu` matching, so the database filter agrees with the exact
+ * JavaScript matcher (including K/k/U+212A, s/S/U+017F and the Greek sigmas).
+ */
+export function sessionMessageCaseClasses(): ReadonlyMap<number, readonly number[]> {
+  if (caseClasses) return caseClasses;
+  const edges = new Map<number, Set<number>>();
+  const link = (a: number, b: number) => {
+    let set = edges.get(a);
+    if (!set) edges.set(a, (set = new Set()));
+    set.add(b);
+  };
+  for (let cp = 0; cp <= LAST_CASED_CODE_POINT; cp++) {
+    if (cp >= 0xd800 && cp <= 0xdfff) continue;
+    const char = String.fromCodePoint(cp);
+    for (const mapped of [char.toLowerCase(), char.toUpperCase()]) {
+      if (mapped === char) continue;
+      const target = mapped.codePointAt(0)!;
+      if (String.fromCodePoint(target) !== mapped) continue;
+      if (!sessionMessageLiteralPattern(char).test(mapped)) continue;
+      link(cp, target);
+      link(target, cp);
+    }
+  }
+  const classes = new Map<number, readonly number[]>();
+  for (const start of edges.keys()) {
+    if (classes.has(start)) continue;
+    const members = new Set([start]);
+    for (const member of members) for (const next of edges.get(member) ?? []) members.add(next);
+    const sorted = [...members].sort((a, b) => a - b);
+    for (const member of sorted) classes.set(member, sorted);
+  }
+  return (caseClasses = classes);
+}
+
+/** A PostgreSQL advanced regular expression matching exactly the strings that
+ * contain `query` under `sessionMessageLiteralPattern`. Every character is a
+ * \u/\U escape, so no character is special and the database collation plays
+ * no part. Null when the query holds a character PostgreSQL text cannot store
+ * (NUL or a lone surrogate): only losslessly encoded text can contain it.
+ */
+export function sessionMessagePostgresPattern(query: string): string | null {
+  const escape = (cp: number) =>
+    cp <= 0xffff
+      ? `\\u${cp.toString(16).padStart(4, "0")}`
+      : `\\U${cp.toString(16).padStart(8, "0")}`;
+  let pattern = "";
+  for (const char of query) {
+    const cp = char.codePointAt(0)!;
+    if (cp === 0 || (cp >= 0xd800 && cp <= 0xdfff)) return null;
+    const members = sessionMessageCaseClasses().get(cp);
+    pattern += members ? `[${members.map(escape).join("")}]` : escape(cp);
+  }
+  return pattern;
+}
+
+/** Database tests over stored message text: `matched` is exact for ordinary
+ * text; `encoded` marks losslessly encoded strings, which only the decoding
+ * reader can inspect and so always remain candidates. */
+export function sessionMessageTextMatchSql(
+  query: string,
+  text: SQL,
+): { matched: SQL; encoded: SQL } {
+  const pattern = sessionMessagePostgresPattern(query);
+  return {
+    matched: pattern === null ? sql`false` : sql`(${text} ~ ${pattern})`,
+    encoded: sql`(left(${text}, ${LOSSLESS_JSON_STRING_PREFIX.length}) = ${LOSSLESS_JSON_STRING_PREFIX})`,
+  };
+}
+
 export function sessionMessageSearchSnippet(text: string, start: number, length: number) {
   let left = Math.max(0, start - 120);
   let right = Math.min(text.length, start + length + 120);
@@ -76,7 +165,6 @@ export async function scanSessionMessages(
   signal?: AbortSignal,
 ): Promise<SessionMessageSearchResponse> {
   signal?.throwIfAborted();
-  const started = performance.now();
   const binding = createHash("sha256")
     .update(
       JSON.stringify([
@@ -127,6 +215,13 @@ export async function scanSessionMessages(
         ? sql`${e.sessionId} > ${position.sessionId}::uuid`
         : sql`(${e.sessionId} > ${position.sessionId}::uuid or (${e.sessionId} = ${position.sessionId}::uuid and ${position.done ? sql`${e.sequence} > ${position.sequence}` : sql`${e.sequence} >= ${position.sequence}`}))`
       : sql`true`;
+    // One statement checks every remaining message's text in PostgreSQL and
+    // returns only the first candidates that contain the literal. The cheap
+    // list/visibility predicates and the text test run once per message; the
+    // correlated canonical-copy and unclaimed-prompt checks run only for
+    // candidates, lazily in key order, until the batch is full. A no-hit scan
+    // finishes in one request instead of one request per 33 messages.
+    //
     // Correlate the exact durable message identity, not turn status or latest
     // model context. The latest full completion for a provider message wins,
     // so an earlier shorter completion cannot hide later retained text. Same-
@@ -135,27 +230,15 @@ export async function scanSessionMessages(
     // Suppression requires the competing copy to retain searchable string text
     // itself: a later text-less completion (failed/empty retry sharing the
     // provider message id) must never hide the earlier complete text.
-    const identities = await db
-      .select({
-        sessionId: e.sessionId,
-        eventId: e.id,
-        sequence: e.sequence,
-        turnId: e.turnId,
-        type: sql<"user.message" | "agent.message.completed">`${e.type}`,
-        sessionTitle: sql<
-          string | null
-        >`case when octet_length(${schema.sessions.title}) <= 2048 then ${schema.sessions.title} else null end`,
-        messageId: sql<unknown>`case when octet_length((${e.payload}->'messageId')::text) <= 2048 then ${e.payload}->'messageId' else null end`,
-        codec: e.payloadCodecVersion,
-        smallText: sql<unknown>`case when octet_length(${e.payload}->>'text') <= 8192 then ${e.payload}->'text' else null end`,
-      })
-      .from(e)
-      .innerJoin(
-        schema.sessions,
-        and(eq(schema.sessions.workspaceId, e.workspaceId), eq(schema.sessions.id, e.sessionId)),
-      )
-      .where(
-        and(
+    const textMatch = sessionMessageTextMatchSql(request.query, sql`${e.payload}->>'text'`);
+    const rows: CandidateRow[] = await db.execute<CandidateRow>(sql`
+      with eligible as materialized (
+        select ${e.id} as id, ${e.sessionId} as session_id, ${e.sequence} as sequence,
+          ${textMatch.matched} as matched, ${textMatch.encoded} as encoded
+        from ${e}
+        inner join ${schema.sessions}
+          on ${schema.sessions.workspaceId} = ${e.workspaceId} and ${schema.sessions.id} = ${e.sessionId}
+        where ${and(
           eq(e.workspaceId, workspaceId),
           ...filters,
           boundary,
@@ -164,8 +247,41 @@ export async function scanSessionMessages(
           sql`(${e.turnAssociation} is null or ${e.turnAssociation} = 'current')`,
           sql`${e.duplicateOfEventId} is null`,
           sql`jsonb_typeof(${e.payload}->'text') = 'string'`,
-          excludeUnclaimedHumanPromptEventFilter(workspaceId),
-          sql`(${e.type} <> 'agent.message.completed' or not exists (
+        )}
+      ), visited as (
+        select count(*)::int as visited from eligible
+      )
+      select visited.visited, candidate.*
+      from visited
+      left join lateral (
+        select * from (
+          select valid.*${
+            request.groupBy === "session"
+              ? sql`, (count(*) filter (where valid.definite) over (partition by valid."sessionId"
+                  order by valid.sequence rows between unbounded preceding and 1 preceding))::int as earlier`
+              : sql``
+          }
+          from (
+        select ranked.rank, ranked.session_id as "sessionId", ${e.id} as "eventId",
+          ranked.sequence as sequence, ${e.turnId} as "turnId", ${e.type} as type,
+          ranked.matched and not ranked.encoded as definite,
+          case when octet_length(${schema.sessions.title}) <= 2048 then ${schema.sessions.title} else null end as "sessionTitle",
+          case when octet_length((${e.payload}->'messageId')::text) <= 2048 then ${e.payload}->'messageId' else null end as "messageId",
+          ${e.payloadCodecVersion} as codec,
+          case when octet_length(${e.payload}->>'text') <= 8192 then ${e.payload}->'text' else null end as "smallText"
+        from (
+          select eligible.id, eligible.session_id, eligible.sequence, eligible.matched, eligible.encoded,
+            (row_number() over (order by eligible.session_id, eligible.sequence))::int as rank
+          from eligible
+          order by eligible.session_id, eligible.sequence
+          offset 0
+        ) ranked
+        inner join ${e} on ${e.workspaceId} = ${workspaceId} and ${e.id} = ranked.id
+        inner join ${schema.sessions}
+          on ${schema.sessions.workspaceId} = ${e.workspaceId} and ${schema.sessions.id} = ${e.sessionId}
+        where (ranked.matched or ranked.encoded)
+          and ${excludeUnclaimedHumanPromptEventFilter(workspaceId)}
+          and (${e.type} <> 'agent.message.completed' or not exists (
         select 1 from ${schema.sessionEvents} other_message
         where other_message.workspace_id = ${workspaceId} and other_message.session_id = ${e.sessionId}
           and other_message.turn_id = ${e.turnId}
@@ -181,15 +297,46 @@ export async function scanSessionMessages(
               and other_message.sequence < ${e.sequence}
               and other_message.payload->'text' = ${e.payload}->'text'
               and (other_message.payload_codec_version is not distinct from ${e.payloadCodecVersion}
-                or left(${e.payload}->>'text', ${LOSSLESS_JSON_STRING_PREFIX.length}) <> ${LOSSLESS_JSON_STRING_PREFIX})))))`,
-        ),
-      )
-      .orderBy(asc(e.sessionId), asc(e.sequence))
-      .limit(33);
+                or left(${e.payload}->>'text', ${LOSSLESS_JSON_STRING_PREFIX.length}) <> ${LOSSLESS_JSON_STRING_PREFIX})))))
+        order by ranked.session_id, ranked.sequence
+        offset 0
+          ) valid
+        ) grouped
+        -- Grouping needs only each session's first certain hit. Encoded text
+        -- stays a candidate because only the decoding reader can confirm it.
+        where ${request.groupBy === "session" ? sql`grouped.earlier = 0` : sql`true`}
+        order by grouped."sessionId", grouped.sequence
+        limit 33
+      ) candidate on true
+      order by candidate.rank
+    `);
+    const visited = rows[0]?.visited ?? 0;
+    const identities = rows.flatMap((row: CandidateRow) =>
+      row.rank === null ||
+      row.sessionId === null ||
+      row.eventId === null ||
+      row.sequence === null ||
+      row.type === null
+        ? []
+        : [
+            {
+              ...row,
+              rank: row.rank,
+              sessionId: row.sessionId,
+              eventId: row.eventId,
+              sequence: row.sequence,
+              type: row.type,
+            },
+          ],
+    );
+    // The cursor's message being resumed mid-text is the first visited row
+    // again; it was already counted by the request that started it.
+    const visitedBefore = scanned - (position && !position.done ? 1 : 0);
+    const budgetStarted = performance.now();
     let identityIndex = 0;
     for (let window = 0; window < 32 && matches.length < (request.limit ?? 20); window++) {
       signal?.throwIfAborted();
-      if (window > 0 && performance.now() - started >= 1_500) break;
+      if (window > 0 && performance.now() - budgetStarted >= 1_500) break;
       const identity = identities[identityIndex];
       if (!identity) {
         // Grouping can drain a full candidate batch in fewer than 32 windows.
@@ -203,7 +350,7 @@ export async function scanSessionMessages(
         position.sessionId === identity.sessionId &&
         position.sequence === identity.sequence;
       if (!continuing) {
-        scanned++;
+        scanned = visitedBefore + identity.rank;
         position = {
           sessionId: identity.sessionId,
           sequence: identity.sequence,
@@ -326,6 +473,7 @@ export async function scanSessionMessages(
       };
     }
     if (identityIndex >= identities.length && identities.length < 33) exhausted = true;
+    if (exhausted) scanned = visitedBefore + visited;
   }
   signal?.throwIfAborted();
   return {
@@ -339,7 +487,14 @@ export async function scanSessionMessages(
       exhausted || !position
         ? null
         : Buffer.from(
-            JSON.stringify({ version: 1, binding, position, scanned, matched, occurrences }),
+            JSON.stringify({
+              version: 1,
+              binding,
+              position,
+              scanned,
+              matched,
+              occurrences,
+            }),
           ).toString("base64url"),
   };
 }

package/src/subscription-core/administered-connections.ts

@@ -9,7 +9,7 @@
  *
  * Two scopes, each with its own authority and nothing borrowed:
  * - `organization_administrator`: the organization context (no workspace),
- *   the organization membership lifecycle lock, the caller's subject and the
+ *   a shared organization membership lifecycle lock, the caller's subject and the
  *   organization administrator check (the predicate the connection policies
  *   use), in every transaction. Only a shared, organization-managed
  *   connection (not one a workspace manages) of the provider's registered
@@ -200,15 +200,17 @@ export const subscriptionCoreAdministeredConnections = memoByProvider(
               { accountId: scope.accountId, workspaceId: null },
               async (tx) => {
                 // The organization membership lifecycle key, before the
-                // administrator check, so a demotion cannot commit while
-                // this transaction acts on its result. A bounded wait: a
-                // busy organization refuses the read rather than queueing.
+                // administrator check, shared between inspections of independent
+                // connections. Lifecycle writers remain exclusive, so a demotion
+                // cannot commit while this transaction acts on its result. The
+                // bounded wait refuses a busy organization rather than queueing
+                // indefinitely.
                 await tx.execute(
                   sql`select set_config('lock_timeout', ${ORGANIZATION_MEMBERSHIP_LOCK_TIMEOUT}, true)`,
                 );
                 try {
                   await tx.execute(
-                    sql`select pg_advisory_xact_lock(hashtextextended(${`organization-membership:${scope.accountId}`}, 0))`,
+                    sql`select pg_advisory_xact_lock_shared(hashtextextended(${`organization-membership:${scope.accountId}`}, 0))`,
                   );
                 } catch (error) {
                   if (nestedPostgresSqlState(error) === "55P03") throw new NotVisible();

package/src/subscription-usage-pool.ts

@@ -44,11 +44,11 @@ export function blockedSubscriptionWindows(
   provider: Provider,
   usage: ReturnType<typeof projectSubscriptionCoreAccountUsage>,
   now: number,
-): Partial<Record<Window, number>> {
-  const blocked: Partial<Record<Window, number>> = {};
+): Partial<Record<Window, number | null>> {
+  const blocked: Partial<Record<Window, number | null>> = {};
   if (!usage) return blocked;
   const block = (id: Window, until: number) => {
-    if (until > now) blocked[id] = Math.max(blocked[id] ?? 0, until);
+    if (until > now && blocked[id] !== null) blocked[id] = Math.max(blocked[id] ?? 0, until);
   };
   const windows = usage.windows.flatMap((window) => {
     const id = windowId(provider, window.id);
@@ -56,8 +56,10 @@ export function blockedSubscriptionWindows(
     return id === null ? [] : [{ id, resetsAt, status: window.status }];
   });
   for (const window of windows) {
-    if (window.status === "exhausted" && window.resetsAt !== null)
-      block(window.id, window.resetsAt);
+    if (window.status === "exhausted") {
+      if (window.resetsAt === null) blocked[window.id] = null;
+      else block(window.id, window.resetsAt);
+    }
   }
   const until = epoch(usage.exhaustedUntil);
   if (until !== null && until > now) {

package/src/workspace-artifacts.ts

@@ -11,7 +11,7 @@ import { parseVerifiedAttemptToolCatalog } from "@opengeni/codemode";
 import { and, desc, eq, lt, or, sql } from "drizzle-orm";
 import { isDeepStrictEqual } from "node:util";
 import type { Database } from "./database";
-import { withRlsContext, withWorkspaceRls } from "./database";
+import { withRlsContext, withWorkspaceRls, withWorkspaceSubjectRls } from "./database";
 import * as schema from "./schema";
 import { hydrateStoredAttemptToolCatalog } from "./session-content-blobs";
 import { sameSitePublicationRequest, sitePublicationRequest } from "./site-publication-request";
@@ -185,6 +185,50 @@ export class WorkspaceArtifactNotFoundError extends Error {
   readonly name = "WorkspaceArtifactNotFoundError";
 }
 
+/**
+ * Records that a person opened a Site in the console, for the order of their
+ * own Site rows. Only a Site this viewer can see is recorded; the row grants
+ * nothing. Repeated opens only move the time forward.
+ */
+export async function recordSiteOpened(
+  db: Database,
+  input: {
+    workspaceId: string;
+    subjectId: string;
+    viewer: WorkspaceArtifactViewer;
+    siteId: string;
+  },
+): Promise<{ openedAt: string }> {
+  return await withWorkspaceSubjectRls(db, input.workspaceId, input.subjectId, async (scoped) => {
+    const [site] = await scoped
+      .select({ accountId: schema.workspaceArtifacts.accountId })
+      .from(schema.workspaceArtifacts)
+      .where(
+        and(
+          eq(schema.workspaceArtifacts.workspaceId, input.workspaceId),
+          eq(schema.workspaceArtifacts.id, input.siteId),
+          visibleTo(input.viewer),
+        ),
+      )
+      .limit(1);
+    if (!site) throw new WorkspaceArtifactNotFoundError("Site not found");
+    const [row] = await scoped
+      .insert(schema.siteOpens)
+      .values({
+        accountId: site.accountId,
+        workspaceId: input.workspaceId,
+        subjectId: input.subjectId,
+        siteId: input.siteId,
+      })
+      .onConflictDoUpdate({
+        target: [schema.siteOpens.workspaceId, schema.siteOpens.subjectId, schema.siteOpens.siteId],
+        set: { openedAt: sql`greatest(${schema.siteOpens.openedAt}, now())` },
+      })
+      .returning({ openedAt: schema.siteOpens.openedAt });
+    return { openedAt: row!.openedAt.toISOString() };
+  });
+}
+
 export class WorkspaceArtifactConflictError extends Error {
   readonly name = "WorkspaceArtifactConflictError";
   constructor(