@opengeni/db 0.7.2 → 0.9.3

package/src/memory-domain.ts

@@ -54,7 +54,7 @@ export const MEMORY_KIND_SECTION_TITLES: Record<KnowledgeMemoryKind, string> = {
 };
 
 // ---------------------------------------------------------------------------
-// Canonical prompt surface (dossier §10b) — the prompts ARE the product.
+// Canonical prompt surface — the prompts ARE the product.
 // ---------------------------------------------------------------------------
 
 export const WORKSPACE_MEMORY_BLOCK_HEADER_POPULATED = `## Workspace memory

package/src/migrate.ts

@@ -5,6 +5,24 @@ import postgres from "postgres";
 
 const DEFAULT_DATABASE_URL = "postgres://opengeni:opengeni@127.0.0.1:5432/opengeni";
 const concurrentIndexDirective = /^-- opengeni:concurrent-index lock-timeout=(\d+(?:ms|s|min))$/;
+const concurrentIndexStatement =
+  /^CREATE\s+(?:UNIQUE\s+)?INDEX\s+CONCURRENTLY\s+(?:(IF\s+NOT\s+EXISTS)\s+)?(?:"((?:[^"]|"")+)"|([A-Za-z_][A-Za-z0-9_]*))\s+ON\b/is;
+const governedLegacyConcurrentIndexMigrations = new Set([
+  "0066_session_interruption_attempt_lookup.sql",
+  "0070_session_event_type_sequence_lookup.sql",
+  "0071_session_event_monitoring_tail.sql",
+  "0072_sessions_workspace_created_id_idx.sql",
+  "0073_sessions_workspace_updated_id_idx.sql",
+  "0075_sessions_workspace_activity_revision_idx.sql",
+  "0077_session_attempt_latest_lookup.sql",
+]);
+
+export interface ConcurrentIndexMigration {
+  indexName: string;
+  lockTimeout: string;
+  skipWhenValid: boolean;
+  statement: string;
+}
 
 /** A bare Postgres identifier (schema/role name) safe to interpolate into DDL. */
 function assertIdentifier(name: string, value: string): string {
@@ -20,45 +38,90 @@ function assertIdentifier(name: string, value: string): string {
  * into one narrowly validated transactionless statement with:
  *
  *   -- opengeni:concurrent-index lock-timeout=5s
- *   CREATE [UNIQUE] INDEX CONCURRENTLY ...;
+ *   CREATE [UNIQUE] INDEX CONCURRENTLY IF NOT EXISTS ...;
  *
  * The directive is deliberately not a generic "no transaction" escape hatch:
- * only one concurrent-index statement is accepted, and lock acquisition is
- * always bounded. This keeps additive large-table indexes online without making
- * arbitrary partially-applied migration scripts possible.
+ * only one idempotent concurrent-index statement is accepted, lock acquisition
+ * is always bounded, and an invalid artifact left by a failed concurrent build
+ * is removed before retry. Seven governed historical migrations predate the
+ * IF NOT EXISTS rule; only those exact filenames may use their immutable bare
+ * statements, and the runner makes their retries idempotent by skipping an
+ * already-valid index. This keeps additive large-table indexes online without
+ * rewriting shipped history or making arbitrary partially-applied migration
+ * scripts possible.
  */
-async function executeMigrationFile(
-  sql: postgres.Sql,
+export function parseConcurrentIndexMigration(
   file: string,
   sqlText: string,
-): Promise<void> {
-  const [firstLine = "", ...remainingLines] = sqlText.replaceAll("\r\n", "\n").split("\n");
-  const directive = concurrentIndexDirective.exec(firstLine.trim());
+): ConcurrentIndexMigration | null {
+  const lines = sqlText.replaceAll("\r\n", "\n").split("\n");
+  const firstLine = lines[0]?.trim() ?? "";
+  const deploymentPrefixed = /^-- deployment-mode: (?:rolling|maintenance)$/.test(firstLine);
+  const directiveIndex = deploymentPrefixed ? 1 : 0;
+  const directiveLine = lines[directiveIndex]?.trim() ?? "";
+  const directive = concurrentIndexDirective.exec(directiveLine);
   if (!directive) {
-    if (firstLine.trim().startsWith("-- opengeni:")) {
+    if (directiveLine.startsWith("-- opengeni:")) {
       throw new Error(`Unsupported OpenGeni migration directive in ${file}`);
     }
-    await sql.unsafe(sqlText);
-    return;
+    return null;
   }
 
   const lockTimeout = directive[1]!;
-  const statement = remainingLines.join("\n").trim();
+  const statement = lines
+    .slice(directiveIndex + 1)
+    .join("\n")
+    .trim();
   const withoutTrailingSemicolon = statement.endsWith(";")
     ? statement.slice(0, -1).trimEnd()
     : statement;
-  if (
-    !/^CREATE\s+(?:UNIQUE\s+)?INDEX\s+CONCURRENTLY\b/is.test(withoutTrailingSemicolon) ||
-    withoutTrailingSemicolon.includes(";")
-  ) {
+  const parsedStatement = concurrentIndexStatement.exec(withoutTrailingSemicolon);
+  if (!parsedStatement || withoutTrailingSemicolon.includes(";")) {
     throw new Error(
-      `${file}: opengeni:concurrent-index requires exactly one CREATE [UNIQUE] INDEX CONCURRENTLY statement`,
+      `${file}: opengeni:concurrent-index requires exactly one CREATE [UNIQUE] INDEX CONCURRENTLY IF NOT EXISTS statement with an unqualified index name`,
     );
   }
+  const idempotentInSql = parsedStatement[1] !== undefined;
+  if (!idempotentInSql && !governedLegacyConcurrentIndexMigrations.has(file)) {
+    throw new Error(
+      `${file}: opengeni:concurrent-index requires IF NOT EXISTS; bare statements are supported only for governed historical migrations`,
+    );
+  }
+  return {
+    indexName: (parsedStatement[2] ?? parsedStatement[3]!).replaceAll('""', '"'),
+    lockTimeout,
+    skipWhenValid: !idempotentInSql,
+    statement,
+  };
+}
 
-  await sql`select set_config('lock_timeout', ${lockTimeout}, false)`;
+async function executeMigrationFile(
+  sql: postgres.Sql,
+  file: string,
+  sqlText: string,
+): Promise<void> {
+  const concurrentIndex = parseConcurrentIndexMigration(file, sqlText);
+  if (!concurrentIndex) {
+    await sql.unsafe(sqlText);
+    return;
+  }
+
+  await sql`select set_config('lock_timeout', ${concurrentIndex.lockTimeout}, false)`;
   try {
-    await sql.unsafe(statement);
+    const [existing] = await sql<Array<{ valid: boolean; ready: boolean }>>`
+      select i.indisvalid as valid, i.indisready as ready
+      from pg_catalog.pg_class c
+      join pg_catalog.pg_namespace n on n.oid = c.relnamespace
+      join pg_catalog.pg_index i on i.indexrelid = c.oid
+      where n.nspname = current_schema() and c.relname = ${concurrentIndex.indexName}
+    `;
+    if (existing && (!existing.valid || !existing.ready)) {
+      const quotedIndexName = `"${concurrentIndex.indexName.replaceAll('"', '""')}"`;
+      await sql.unsafe(`DROP INDEX CONCURRENTLY ${quotedIndexName}`);
+    } else if (existing && concurrentIndex.skipWhenValid) {
+      return;
+    }
+    await sql.unsafe(concurrentIndex.statement);
   } finally {
     await sql`select set_config('lock_timeout', '0', false)`;
   }
@@ -73,18 +136,18 @@ async function executeMigrationFile(
  * byte-for-byte historical behavior — the migration test suite calls
  * `migrate(DB_URL)` and is unaffected.
  *
- * EMBEDDED (Step I, §7.8 runtime/SDK half): pass a `schema` (or set
+ * EMBEDDED SCHEMA MODE: pass a `schema` (or set
  * `OPENGENI_DB_SCHEMA`). The migrate session then `CREATE SCHEMA IF NOT EXISTS`
  * for both `<schema>` and `opengeni_private`, and sets
  * `search_path = "<schema>", "opengeni_private", "public"`, so EVERY unqualified
  * DDL statement lands in the dedicated schema with NO per-statement SQL rewrite
- * (the SPIKE-1 F1 result). Two things make this work and stay idempotent:
+ * (the schema-isolation contract). Two things make this work and stay idempotent:
  *   1. The policy-existence guards in the migration SQL use `current_schema()`
  *      (not a hardcoded `'public'`) — so a re-run finds the policy it already
  *      created in `<schema>` and DROP/CREATEs idempotently instead of failing
  *      with "policy already exists". (This guard substitution is the migrate-
  *      time enabler for the runtime search_path approach; without it the SDK
- *      entry point silently fails on re-run — the Fork-6 hazard.)
+ *      entry point silently fails on re-run — the migration replay hazard.)
  *   2. `public` stays LAST on the path so `gen_random_uuid()` (pgcrypto) and the
  *      `vector` type — both installed into `public` by 0000 — still resolve. The
  *      `opengeni_private.*` helpers are always called with an absolute prefix.

package/src/persistence-errors.ts

@@ -0,0 +1,252 @@
+import { sanitizeEventString } from "./event-payload-sanitizer";
+
+export type DatabaseFailureCode = "db_deadlock" | "db_serialization_failure" | "db_failure";
+
+export type PersistenceRetryOutcome = "not_retryable" | "exhausted";
+
+export type SafeDatabaseErrorFacts = {
+  severity?: string;
+  schema?: string;
+  table?: string;
+  column?: string;
+  dataType?: string;
+  constraint?: string;
+  routine?: string;
+};
+
+export type PersistenceFailureDetails = {
+  code: DatabaseFailureCode;
+  sqlState: string | null;
+  stage: string;
+  eventTypes: string[];
+  correlationId: string;
+  attempts: number;
+  retryOutcome: PersistenceRetryOutcome;
+  database: SafeDatabaseErrorFacts;
+};
+
+const SQLSTATE_KEYS = ["sqlState", "sqlstate", "code"] as const;
+const NESTED_ERROR_KEYS = ["cause", "original", "driverError", "error", "errors"] as const;
+const DATABASE_ERROR_NAMES = new Set(["DatabaseError", "DrizzleQueryError", "PostgresError"]);
+const DATABASE_DIAGNOSTIC_KEYS = [
+  "severity",
+  "schema_name",
+  "table_name",
+  "column_name",
+  "data_type_name",
+  "constraint_name",
+  "routine",
+] as const;
+const SAFE_FACT_KEYS = [
+  ["severity", "severity"],
+  ["schema_name", "schema"],
+  ["schema", "schema"],
+  ["table_name", "table"],
+  ["table", "table"],
+  ["column_name", "column"],
+  ["column", "column"],
+  ["data_type_name", "dataType"],
+  ["dataType", "dataType"],
+  ["constraint_name", "constraint"],
+  ["constraint", "constraint"],
+  ["routine", "routine"],
+] as const;
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+  return Boolean(value && typeof value === "object");
+}
+
+function safeFact(value: unknown): string | undefined {
+  if (typeof value !== "string" || value.length === 0) return undefined;
+  return sanitizeEventString(value).slice(0, 256);
+}
+
+/** Find the driver SQLSTATE even when Drizzle wrapped it under nested causes. */
+export function nestedPostgresSqlState(error: unknown): string | null {
+  const queue: unknown[] = [error];
+  const seen = new Set<unknown>();
+  let fallback: string | null = null;
+  while (queue.length > 0 && seen.size < 64) {
+    const current = queue.shift();
+    if (!isRecord(current) || seen.has(current)) continue;
+    seen.add(current);
+    for (const key of SQLSTATE_KEYS) {
+      const value = current[key];
+      if (typeof value !== "string" || !/^[0-9A-Z]{5}$/i.test(value)) continue;
+      const normalized = value.toUpperCase();
+      if (normalized === "40P01" || normalized === "40001") return normalized;
+      fallback ??= normalized;
+    }
+    for (const key of NESTED_ERROR_KEYS) {
+      const nested = current[key];
+      if (Array.isArray(nested)) queue.push(...nested);
+      else if (nested !== undefined) queue.push(nested);
+    }
+  }
+  return fallback;
+}
+
+export function databaseFailureCode(sqlState: string | null): DatabaseFailureCode {
+  if (sqlState === "40P01") return "db_deadlock";
+  if (sqlState === "40001") return "db_serialization_failure";
+  return "db_failure";
+}
+
+export function isRetryablePersistenceSqlState(sqlState: string | null): boolean {
+  return sqlState === "40P01" || sqlState === "40001";
+}
+
+/**
+ * Distinguish database/ORM failures from expected domain exceptions when a
+ * driver omitted SQLSTATE. This checks shape only and never retains query text,
+ * bound parameters, or a raw driver cause.
+ */
+export function isDatabasePersistenceFailure(error: unknown): boolean {
+  if (nestedPostgresSqlState(error) !== null) return true;
+
+  const queue: unknown[] = [error];
+  const seen = new Set<unknown>();
+  while (queue.length > 0 && seen.size < 64) {
+    const current = queue.shift();
+    if (!isRecord(current) || seen.has(current)) continue;
+    seen.add(current);
+
+    if (typeof current.name === "string" && DATABASE_ERROR_NAMES.has(current.name)) {
+      return true;
+    }
+    if (
+      typeof current.query === "string" &&
+      (Object.hasOwn(current, "params") || Object.hasOwn(current, "parameters"))
+    ) {
+      return true;
+    }
+    if (DATABASE_DIAGNOSTIC_KEYS.some((key) => Object.hasOwn(current, key))) {
+      return true;
+    }
+
+    for (const key of NESTED_ERROR_KEYS) {
+      const nested = current[key];
+      if (Array.isArray(nested)) queue.push(...nested);
+      else if (nested !== undefined) queue.push(nested);
+    }
+  }
+  return false;
+}
+
+/** Extract only PostgreSQL diagnostic identifiers; never query text/parameters. */
+export function safeDatabaseErrorFacts(error: unknown): SafeDatabaseErrorFacts {
+  const queue: unknown[] = [error];
+  const seen = new Set<unknown>();
+  const facts: SafeDatabaseErrorFacts = {};
+  while (queue.length > 0 && seen.size < 64) {
+    const current = queue.shift();
+    if (!isRecord(current) || seen.has(current)) continue;
+    seen.add(current);
+    for (const [source, destination] of SAFE_FACT_KEYS) {
+      if (facts[destination] !== undefined) continue;
+      const value = safeFact(current[source]);
+      if (value !== undefined) facts[destination] = value;
+    }
+    for (const key of NESTED_ERROR_KEYS) {
+      const nested = current[key];
+      if (Array.isArray(nested)) queue.push(...nested);
+      else if (nested !== undefined) queue.push(nested);
+    }
+  }
+  return facts;
+}
+
+/** Public-safe cause that preserves database classification without driver data. */
+export class SanitizedDatabasePersistenceCause extends Error {
+  readonly name = "SanitizedDatabasePersistenceCause";
+
+  constructor(
+    readonly sqlState: string | null,
+    readonly database: SafeDatabaseErrorFacts,
+  ) {
+    super(sqlState === null ? "Database driver failure" : `PostgreSQL failure ${sqlState}`);
+  }
+}
+
+/**
+ * Public-safe replacement for a raw Drizzle/postgres-js failure. Its `cause`
+ * is a newly constructed sanitized projection; the original driver cause can
+ * contain full SQL and bound parameters and is never retained.
+ */
+export class SessionEventPersistenceError extends Error {
+  readonly name = "SessionEventPersistenceError";
+  readonly cause: SanitizedDatabasePersistenceCause;
+
+  constructor(readonly details: PersistenceFailureDetails) {
+    const label =
+      details.code === "db_deadlock"
+        ? "Database deadlock"
+        : details.code === "db_serialization_failure"
+          ? "Database serialization failure"
+          : "Database failure";
+    super(`${label} while persisting ${details.eventTypes.join(", ") || "session events"}`);
+    this.cause = new SanitizedDatabasePersistenceCause(details.sqlState, details.database);
+  }
+
+  get code(): DatabaseFailureCode {
+    return this.details.code;
+  }
+}
+
+export function isSessionEventPersistenceError(
+  error: unknown,
+): error is SessionEventPersistenceError {
+  return error instanceof SessionEventPersistenceError;
+}
+
+export type IdempotentPersistenceTransactionOptions = {
+  stage: string;
+  eventTypes?: string[];
+  maxAttempts?: number;
+  correlationId?: string;
+  onRetry?: (input: { attempt: number; sqlState: "40P01" | "40001" }) => void | Promise<void>;
+};
+
+/**
+ * Retry only the supplied idempotent database transaction/savepoint. Provider
+ * inference, tools, NATS, and all other external effects must remain outside
+ * this function. A single correlation ID follows every persistence attempt.
+ */
+export async function runIdempotentPersistenceTransaction<T>(
+  options: IdempotentPersistenceTransactionOptions,
+  transaction: (attempt: number) => Promise<T>,
+): Promise<T> {
+  const maxAttempts = options.maxAttempts ?? 3;
+  if (!Number.isInteger(maxAttempts) || maxAttempts < 1) {
+    throw new Error("Persistence maxAttempts must be a positive integer");
+  }
+  const correlationId = options.correlationId ?? crypto.randomUUID();
+  const eventTypes = [...new Set(options.eventTypes ?? [])].sort();
+  for (let attempt = 1; attempt <= maxAttempts; attempt += 1) {
+    try {
+      return await transaction(attempt);
+    } catch (error) {
+      const sqlState = nestedPostgresSqlState(error);
+      const retryable = isRetryablePersistenceSqlState(sqlState);
+      if (retryable && attempt < maxAttempts) {
+        await options.onRetry?.({
+          attempt,
+          sqlState: sqlState as "40P01" | "40001",
+        });
+        continue;
+      }
+      if (!isDatabasePersistenceFailure(error)) throw error;
+      throw new SessionEventPersistenceError({
+        code: databaseFailureCode(sqlState),
+        sqlState,
+        stage: options.stage,
+        eventTypes,
+        correlationId,
+        attempts: attempt,
+        retryOutcome: retryable ? "exhausted" : "not_retryable",
+        database: safeDatabaseErrorFacts(error),
+      });
+    }
+  }
+  throw new Error("Unreachable persistence retry state");
+}

package/src/provision-roles.ts

@@ -3,6 +3,7 @@ import type { RlsStrategy } from "./index";
 
 export type ProvisionResult = {
   appRole: string | null;
+  hostExportRole: string | null;
   temporalRole: string | null;
   temporalDatabases: string[];
   schema: string;
@@ -27,6 +28,16 @@ export type ProvisionRolesOptions = {
   rlsStrategy?: RlsStrategy;
   appRole?: string;
   appPassword?: string;
+  /**
+   * Optional cross-workspace projection role. It receives schema USAGE and
+   * EXECUTE only on the host-export API; it receives no table privileges.
+   * Provision it after the first migration run so the schema exists. The
+   * provisioner also registers same-owner default privileges for future
+   * host-export functions; shipped migrations preserve existing exporter ACLs
+   * when a migration-only upgrade adds a function.
+   */
+  hostExportRole?: string;
+  hostExportPassword?: string;
   temporalRole?: string;
   temporalPassword?: string;
   temporalDatabases?: string[];
@@ -59,6 +70,13 @@ export async function provisionRoles(
     options.appRole ?? (process.env.OPENGENI_APP_DATABASE_USER?.trim() || "opengeni_app"),
   );
   const appPassword = options.appPassword ?? process.env.OPENGENI_APP_DATABASE_PASSWORD;
+  const hostExportRole = validateIdentifier(
+    "hostExportRole",
+    options.hostExportRole ??
+      (process.env.OPENGENI_HOST_EXPORT_DATABASE_USER?.trim() || "opengeni_host_exporter"),
+  );
+  const hostExportPassword =
+    options.hostExportPassword ?? process.env.OPENGENI_HOST_EXPORT_DATABASE_PASSWORD;
   const temporalRole = validateIdentifier(
     "temporalRole",
     options.temporalRole ??
@@ -95,12 +113,18 @@ export async function provisionRoles(
       }
     }
 
+    if (hostExportPassword) {
+      await ensureLoginRole(sql, hostExportRole, hostExportPassword);
+      await grantHostExportRoleIfSchemaExists(sql, hostExportRole);
+    }
+
     if (rlsStrategy === "force") {
       await grantAppRoleIfSchemaExists(sql, appRole, schema);
     }
 
     return {
       appRole: provisionedAppRole,
+      hostExportRole: hostExportPassword ? hostExportRole : null,
       temporalRole: temporalPassword ? temporalRole : null,
       temporalDatabases: temporalPassword ? temporalDatabases : [],
       schema,
@@ -111,6 +135,24 @@ export async function provisionRoles(
   }
 }
 
+/**
+ * The exporter is intentionally separate from `opengeni_app`: its functions
+ * project every workspace into a host-owned sink and therefore cannot be made
+ * available to the tenant-scoped application role.
+ */
+async function grantHostExportRoleIfSchemaExists(sql: postgres.Sql, role: string): Promise<void> {
+  await sql.unsafe(`
+DO $$
+BEGIN
+  IF EXISTS (SELECT 1 FROM pg_namespace WHERE nspname = 'opengeni_host_export') THEN
+    EXECUTE format('GRANT USAGE ON SCHEMA opengeni_host_export TO %I', ${literal(role)});
+    EXECUTE format('GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA opengeni_host_export TO %I', ${literal(role)});
+    EXECUTE format('ALTER DEFAULT PRIVILEGES IN SCHEMA opengeni_host_export GRANT EXECUTE ON FUNCTIONS TO %I', ${literal(role)});
+  END IF;
+END $$;
+`);
+}
+
 async function ensureLoginRole(sql: postgres.Sql, role: string, password: string): Promise<void> {
   await sql.unsafe(`
 DO $$