@memberjunction/server 6.2.0-edge.1 → 6.2.0-edge.3

package/src/index.ts

@@ -31,6 +31,8 @@ import { WebSocketServer } from 'ws';
 import { RealtimeProxyServer } from './realtimeProxy/RealtimeProxyServer.js';
 import buildApolloServer from './apolloServer/index.js';
 import { configInfo, configFilePath, dbDatabase, dbHost, dbPort, dbUsername, graphqlPort, graphqlRootPath, mj_core_schema, websiteRunFromPackage, RESTApiOptions } from './config.js';
+import { TranslateBracketsToPG } from './postgresqlCompat.js';
+import { BuildPostgreSQLConnectionConfig, DescribeReadOnlyLoginOverreach, PostgreSQLReadOnlyPool, ResolvePostgreSQLEndpoint, ResolvePostgreSQLReadOnlyCredentials, ToPGPoolConfig } from './postgresqlPoolSettings.js';
 import { default as jwt } from 'jsonwebtoken';
 import { contextFunction, CreateUnifiedAuthMiddleware, getUserPayload } from './context.js';
 import { UserPayload } from './types.js';
@@ -61,12 +63,26 @@ import { GetAPIKeyEngine } from '@memberjunction/api-keys';
 import { RedisLocalStorageProvider } from '@memberjunction/redis-provider';
 import { GenericDatabaseProvider } from '@memberjunction/generic-database-provider';
 import { PubSubManager } from './generic/PubSubManager.js';
+import { ReconcileOrphanedConversationDetails } from './generic/OrphanedConversationDetailReconciler.js';
+import {
+  PUSH_STATUS_UPDATES_TOPIC,
+  SetPushStatusPublishHook,
+  ParseReplicatedStatusUpdate,
+} from './generic/PushStatusResolver.js';
+import {
+  HANDOFF_OFFER_FANOUT_CHANNEL,
+  HANDOFF_OFFER_TOPIC,
+  SetHandoffOfferPublishHook,
+  ParseReplicatedHandoffOfferUpdate,
+} from './resolvers/HumanHandoffResolver.js';
+import { RoomHandoffEngine } from '@memberjunction/livekit-room-server';
 import { IntegrationProgressEmitter } from '@memberjunction/integration-progress-artifacts';
 import { PublishIntegrationProgress } from './resolvers/IntegrationProgressResolver.js';
 import { RegisterRSUProgressBridge } from './integration/RSUProgressBridge.js';
 import { ClientToolRequestManager, AgentRunWatchdog } from '@memberjunction/ai-agents';
 import { SessionJanitor } from './agentSessions/index.js';
 import { StartTaskGraphDispatcher } from './services/StartTaskGraphDispatcher.js';
+import { MJServerWorkQueueProviderSource, StartWorkQueueHost } from './services/WorkQueueHostService.js';
 import { GetAttachmentService } from '@memberjunction/aiengine';
 import { MJStorageBlobStore } from './services/MJStorageBlobStore.js';
 import { CACHE_INVALIDATION_TOPIC, MayBroadcastRecordData, ConfigureRecordDataBroadcast } from './generic/CacheInvalidationResolver.js';
@@ -167,7 +183,9 @@ export * from './generic/refusalExtensions.js';
 export * from './generic/RunViewResolver.js';
 export * from './resolvers/RunTemplateResolver.js';
 export * from './resolvers/RunAIPromptResolver.js';
+export * from './resolvers/RunDecisionResolver.js';
 export * from './resolvers/RunAIAgentResolver.js';
+export { AgentRunStatusPublisher } from './resolvers/AgentRunStatusPublisher.js';
 export * from './resolvers/VectorizeEntityResolver.js';
 export * from './resolvers/SearchKnowledgeResolver.js';
 export * from './resolvers/SearchKnowledgeStreamResolver.js';
@@ -224,6 +242,7 @@ export * from './rest/MediaAccessKeys.js';
 export * from './rest/MediaStreamHandler.js';
 export * from './resolvers/InfoResolver.js';
 export * from './resolvers/PotentialDuplicateRecordResolver.js';
+export * from './resolvers/DuplicateEntryCheckResolver.js';
 export * from './resolvers/RunTestResolver.js';
 export * from './resolvers/SearchEntitiesResolver.js';
 export * from './resolvers/UserFavoriteResolver.js';
@@ -234,6 +253,7 @@ export * from './resolvers/CurrentUserContextResolver.js';
 export * from './resolvers/RSUResolver.js';
 export * from './resolvers/AgentSessionResolver.js';
 export * from './resolvers/RealtimeClientSessionResolver.js';
+export * from './resolvers/MeetingResolver.js';
 export * from './resolvers/RemoteBrowserActionResolver.js';
 export * from './agentSessions/index.js';
 export { GetReadOnlyDataSource, GetReadWriteDataSource, GetReadWriteProvider, GetReadOnlyProvider } from './util.js';
@@ -287,11 +307,108 @@ function resolveServerVersion(): string | undefined {
     const pkgPath = fileURLToPath(new URL('../package.json', import.meta.url));
     const pkg = JSON.parse(readFileSync(pkgPath, 'utf-8')) as { version?: string };
     return pkg.version;
-  } catch {
+  } catch (err) {
+    LogError('Failed to resolve server version from package.json', undefined, err);
     return undefined;
   }
 }
 
+/** How often to re-check for conversation details left behind by finished runs. */
+const ORPHAN_DETAIL_SWEEP_INTERVAL_MS = 5 * 60_000;
+
+/** Redis channel carrying replicated push-status updates between server instances. */
+const PUSH_STATUS_FANOUT_CHANNEL = 'push-status-updates';
+
+/**
+ * Replicate push-status updates across server instances over Redis (MJ #4222).
+ *
+ * Outbound: every locally-published update is forwarded on a shared channel. Inbound: a message
+ * from another instance is republished onto THIS instance's local topic, where the normal
+ * subscription filter decides who receives it — so the identity gate (`ownerUserId` vs. the
+ * connection's authenticated user) still applies to a replicated message exactly as it does to a
+ * local one. The replica has no say in who sees what.
+ *
+ * Republishing goes straight to `PubSubManager`, never back through `publishStatusUpdate`, so an
+ * inbound message cannot be re-broadcast and loop. `SourceServerId` guards the remaining case: a
+ * publisher also receives its own message from Redis.
+ */
+async function wirePushStatusFanOut(redisProvider: RedisLocalStorageProvider, startupLog: StartupLogger): Promise<void> {
+  try {
+    await redisProvider.SubscribeToChannel(PUSH_STATUS_FANOUT_CHANNEL, (raw: string) => {
+      try {
+        const payload = ParseReplicatedStatusUpdate(raw, MJGlobal.Instance.ProcessUUID);
+        if (!payload) {
+          return;
+        }
+        // Rebuilt as a plain record: the topic's publish signature takes an index-signature type,
+        // and listing the fields keeps the wire shape explicit at the one place it crosses hosts.
+        PubSubManager.Instance.Publish(PUSH_STATUS_UPDATES_TOPIC, {
+          sessionId: payload.sessionId,
+          ownerUserId: payload.ownerUserId,
+          message: payload.message,
+          SourceServerId: payload.SourceServerId,
+        });
+      } catch (err) {
+        // A malformed message on a shared channel must not take down the subscriber.
+        LogError('Error processing push-status fan-out message', undefined, err);
+      }
+    });
+
+    SetPushStatusPublishHook((payload) => {
+      redisProvider.PublishMessage(PUSH_STATUS_FANOUT_CHANNEL, JSON.stringify(payload));
+    });
+
+    // Printed unconditionally, not verbose-gated. "Is fan-out actually on?" is the first question
+    // anyone debugging a hung conversation behind a load balancer asks, and a silent default left
+    // no way to answer it.
+    console.log('[MJAPI] Push-status updates: cross-instance fan-out enabled via Redis');
+  } catch (err) {
+    // Single-instance delivery still works, and the durable tail query covers the rest. Degraded,
+    // not broken — so this must not stop the server from starting.
+    console.warn(`Push-status fan-out unavailable: ${(err as Error).message}`);
+  }
+}
+
+/**
+ * Replicate handoff-offer updates across server instances over Redis.
+ *
+ * Outbound: every locally-published offer change is forwarded on a shared Redis channel.
+ * Inbound: a message from another instance is republished onto THIS instance's local GraphQL
+ * topic (where the subscription filter scopes to the target user) and delivered to RoomHandoffEngine
+ * so the call-hosting instance can transition its local flow (e.g. wait for the user to join the room).
+ */
+async function wireHandoffOfferFanOut(redisProvider: RedisLocalStorageProvider, startupLog: StartupLogger): Promise<void> {
+  try {
+    await redisProvider.SubscribeToChannel(HANDOFF_OFFER_FANOUT_CHANNEL, (raw: string) => {
+      try {
+        const payload = ParseReplicatedHandoffOfferUpdate(raw, MJGlobal.Instance.ProcessUUID);
+        if (!payload) {
+          return;
+        }
+        PubSubManager.Instance.Publish(HANDOFF_OFFER_TOPIC, {
+          UserID: payload.UserID,
+          Kind: payload.Kind,
+          Offer: payload.Offer,
+        });
+        RoomHandoffEngine.Instance.OnRemoteOfferChange(payload);
+      } catch (err) {
+        // A malformed message on a shared channel must not take down the subscriber.
+        LogError('Error processing handoff-offer fan-out message', undefined, err);
+      }
+    });
+
+    SetHandoffOfferPublishHook((payload) => {
+      redisProvider.PublishMessage(HANDOFF_OFFER_FANOUT_CHANNEL, JSON.stringify(payload));
+    });
+
+    console.log('[MJAPI] Handoff-offer updates: cross-instance fan-out enabled via Redis');
+    startupLog.LogIf('verbose', 'Handoff-offer updates: cross-instance fan-out enabled via Redis');
+  } catch (err) {
+    console.warn(`Handoff-offer fan-out unavailable: ${err instanceof Error ? err.message : String(err)}`);
+  }
+}
+
+
 // Bind MJStorage as the conversation-attachment blob store. The attachment service itself no longer
 // imports `@memberjunction/storage` — that dependency made it unusable from any browser or React
 // Native client, which is why the same attachment rules had been reimplemented three times.
@@ -338,25 +455,13 @@ const setupComplete$ = new ReplaySubject(1);
     startupLog.BeginPhase('Connecting to database');
     startupLog.LogIf('verbose', 'Database type: PostgreSQL');
     const pg = await import('pg');
-    const { PostgreSQLDataProvider, PostgreSQLProviderConfigData } = await import('@memberjunction/postgresql-dataprovider');
-
-    const pgHost = process.env.PG_HOST || process.env.DB_HOST || 'localhost';
-    const pgPort = parseInt(process.env.PG_PORT || process.env.DB_PORT || '5432', 10);
-    const pgUser = process.env.PG_USERNAME || process.env.DB_USERNAME || 'postgres';
-    const pgPass = process.env.PG_PASSWORD || process.env.DB_PASSWORD || '';
-    const pgDatabase = process.env.PG_DATABASE || process.env.DB_DATABASE || '';
+    const { PostgreSQLDataProvider, PostgreSQLProviderConfigData, MJPostgresTypes } = await import('@memberjunction/postgresql-dataprovider');
 
-    const pgPool = new pg.default.Pool({
-      host: pgHost,
-      port: pgPort,
-      user: pgUser,
-      password: pgPass,
-      database: pgDatabase,
-      max: configInfo.databaseSettings.connectionPool?.max ?? 50,
-      min: configInfo.databaseSettings.connectionPool?.min ?? 5,
-      idleTimeoutMillis: configInfo.databaseSettings.connectionPool?.idleTimeoutMillis ?? 30000,
-      connectionTimeoutMillis: configInfo.databaseSettings.connectionPool?.acquireTimeoutMillis ?? 30000,
-    });
+    const pgEndpoint = ResolvePostgreSQLEndpoint();
+    const { Host: pgHost, Port: pgPort, User: pgUser, Database: pgDatabase } = pgEndpoint;
+    // Every API pool carries the statement and idle-in-transaction timeouts from connection #1.
+    const pgConnectionConfig = BuildPostgreSQLConnectionConfig(pgEndpoint, configInfo.databaseSettings, 'api');
+    const pgPool = new pg.default.Pool(ToPGPoolConfig(pgConnectionConfig));
 
     // Verify connection
     const testClient = await pgPool.connect();
@@ -364,6 +469,28 @@ const setupComplete$ = new ReplaySubject(1);
     testClient.release();
     startupLog.LogIf('verbose', `PostgreSQL pool connected to ${pgHost}:${pgPort}/${pgDatabase}`);
 
+    // A read-only pool opened with the read-only login, as SQL Server has. Read-only per-request
+    // providers (TestQuerySQL and other caller-supplied SQL) share it instead of the primary pool.
+    const pgReadOnlyCredentials = ResolvePostgreSQLReadOnlyCredentials(configInfo);
+    if (pgReadOnlyCredentials) {
+      // A pool a provider runs on must carry the provider's type parsers, so BIGINT and NUMERIC
+      // come back as numbers here as they do on the provider's own pool.
+      const readOnlyPgPool = new pg.default.Pool({
+        ...ToPGPoolConfig(BuildPostgreSQLConnectionConfig({ ...pgEndpoint, ...pgReadOnlyCredentials }, configInfo.databaseSettings, 'read-only')),
+        types: MJPostgresTypes,
+      });
+      const readOnlyTestClient = await readOnlyPgPool.connect();
+      try {
+        for (const warning of await DescribeReadOnlyLoginOverreach(readOnlyTestClient, mj_core_schema)) {
+          LogStatus(`WARNING: ${warning}`);
+        }
+      } finally {
+        readOnlyTestClient.release();
+      }
+      PostgreSQLReadOnlyPool.Instance.Pool = readOnlyPgPool;
+      startupLog.LogIf('verbose', 'Read-only PostgreSQL pool has been initialized.');
+    }
+
     // Create a DataSourceInfo with a MSSQL-compatible wrapper around pg.Pool
     // This allows existing code (types, util, context) to work without changes
     const mssqlCompatPool = createMSSQLCompatPool(pgPool);
@@ -377,15 +504,6 @@ const setupComplete$ = new ReplaySubject(1);
     }));
 
     // Set up the PostgreSQL provider
-    const pgConnectionConfig = {
-      Host: pgHost,
-      Port: pgPort,
-      Database: pgDatabase,
-      User: pgUser,
-      Password: pgPass,
-      MaxConnections: configInfo.databaseSettings.connectionPool?.max ?? 50,
-      MinConnections: configInfo.databaseSettings.connectionPool?.min ?? 5,
-    };
     const pgConfigData = new PostgreSQLProviderConfigData(
       pgConnectionConfig,
       mj_core_schema,
@@ -419,7 +537,7 @@ const setupComplete$ = new ReplaySubject(1);
       if (poolAny._pgPool) {
         const thePgPool = poolAny._pgPool as import('pg').Pool;
         // Translate SQL Server bracket syntax to PostgreSQL double-quote syntax
-        const pgQuery = translateBracketsToPG(query);
+        const pgQuery = TranslateBracketsToPG(query);
         const result = await thePgPool.query(pgQuery);
         return result.rows;
       }
@@ -438,21 +556,21 @@ const setupComplete$ = new ReplaySubject(1);
     const pgCodegenPass = process.env.CODEGEN_DB_PASSWORD;
     if (pgCodegenUser && pgCodegenPass) {
       try {
-        const codegenPgPool = new pg.default.Pool({
-          host: pgHost,
-          port: pgPort,
-          user: pgCodegenUser,
-          password: pgCodegenPass,
-          database: pgDatabase,
-          max: 10,
-        });
+        // CodeGen and DDL work runs long statements across every entity, so this pool gets the
+        // long CodeGen timeout rather than the API one, as the SQL Server CodeGen pool does.
+        const codegenPgConnectionConfig = BuildPostgreSQLConnectionConfig(
+          { ...pgEndpoint, User: pgCodegenUser, Password: pgCodegenPass },
+          configInfo.databaseSettings,
+          'codegen',
+        );
+        const codegenPgPool = new pg.default.Pool(ToPGPoolConfig(codegenPgConnectionConfig));
         const codegenTestClient = await codegenPgPool.connect();
         await codegenTestClient.query('SELECT 1');
         codegenTestClient.release();
 
         const { RuntimeSchemaManager } = await import('@memberjunction/schema-engine');
         const codegenPgConfigData = new PostgreSQLProviderConfigData(
-          { Host: pgHost, Port: pgPort, Database: pgDatabase, User: pgCodegenUser, Password: pgCodegenPass },
+          codegenPgConnectionConfig,
           mj_core_schema,
           cacheRefreshInterval / 1000, // ms → seconds
         );
@@ -720,6 +838,16 @@ const setupComplete$ = new ReplaySubject(1);
         }
     });
 
+    // Fan push-status updates across server instances (MJ #4222).
+    //
+    // Behind a load balancer the browser's WebSocket lives on one replica while the mutation that
+    // drives the agent can be handled by another. The push topic is an in-process PubSub, so a
+    // completion published on replica B never reaches a subscriber on replica A — the browser waits
+    // forever for an event that was delivered to nobody. Replicating progress and completion over
+    // Redis closes that, and the durable tail query remains the backstop if Redis is down.
+    await wirePushStatusFanOut(redisProvider, startupLog);
+    await wireHandoffOfferFanOut(redisProvider, startupLog);
+
     startupLog.LogIf('verbose', `Redis cache provider connected: ${process.env.REDIS_URL}`);
   }
 
@@ -1075,6 +1203,15 @@ const setupComplete$ = new ReplaySubject(1);
     }
   });
 
+  // graphql-ws's ws integration takes its keepalive as a THIRD POSITIONAL argument to useServer,
+  // defaulting to 12_000 when omitted. That default was doing real work here while being invisible
+  // at the call site: the server pings every 12s and terminates the socket after an unanswered
+  // pong, which is why MJAPI notices a half-open link in ~12-24s while the browser — whose
+  // graphql-ws client does nothing on an unanswered pong — noticed nothing at all (MJ #4222).
+  // Stated explicitly so the behaviour is visible and tunable, and so the next reader does not
+  // conclude from the call site that no server-side heartbeat exists. Value unchanged.
+  const WS_SERVER_KEEPALIVE_MS = 12_000;
+
   // Track per-connection expiry timers so we can clean them up on close
   const expiryTimers = new WeakMap<object, ReturnType<typeof setTimeout>>();
 
@@ -1149,7 +1286,8 @@ const setupComplete$ = new ReplaySubject(1);
         console.error('WebSocket error:', errors);
       },
     },
-    webSocketServer
+    webSocketServer,
+    WS_SERVER_KEEPALIVE_MS
   );
 
   const apolloServer = buildApolloServer(
@@ -1284,13 +1422,22 @@ const setupComplete$ = new ReplaySubject(1);
   // Backwards-compatibility shim: synthesize ServerExtensionConfig entries from legacy configInfo.telephony
   const telephonyExtensionConfigs: ServerExtensionConfig[] = [];
   if (configInfo.telephony?.enabled) {
+    // Settings every carrier shares (inbound run-as user, call cap, outbound policy). Each carrier's own block is
+    // spread AFTER them, so a carrier can override one explicitly.
+    const sharedTelephonySettings: Record<string, unknown> = {
+      inboundRunAsUserEmail: configInfo.telephony.inboundRunAsUserEmail,
+      maxCallSeconds: configInfo.telephony.maxCallSeconds,
+      maxConcurrentCalls: configInfo.telephony.maxConcurrentCalls,
+      transferTargets: configInfo.telephony.transferTargets,
+      outbound: configInfo.telephony.outbound,
+    };
     if (configInfo.telephony.twilio) {
       telephonyExtensionConfigs.push({
         Enabled: true,
         DriverClass: 'TwilioTelephonyExtension',
         RootPath: '/telephony/twilio',
         Phase: 'pre-auth',
-        Settings: configInfo.telephony.twilio as unknown as Record<string, unknown>,
+        Settings: { ...sharedTelephonySettings, ...configInfo.telephony.twilio },
       });
     }
     if (configInfo.telephony.vonage) {
@@ -1299,7 +1446,7 @@ const setupComplete$ = new ReplaySubject(1);
         DriverClass: 'VonageTelephonyExtension',
         RootPath: '/telephony/vonage',
         Phase: 'pre-auth',
-        Settings: configInfo.telephony.vonage as unknown as Record<string, unknown>,
+        Settings: { ...sharedTelephonySettings, ...configInfo.telephony.vonage },
       });
     }
     if (configInfo.telephony.ringcentral) {
@@ -1308,7 +1455,16 @@ const setupComplete$ = new ReplaySubject(1);
         DriverClass: 'RingCentralTelephonyExtension',
         RootPath: '/telephony/ringcentral',
         Phase: 'pre-auth',
-        Settings: configInfo.telephony.ringcentral as unknown as Record<string, unknown>,
+        Settings: { ...sharedTelephonySettings, ...configInfo.telephony.ringcentral },
+      });
+    }
+    if (configInfo.telephony.livekitSip) {
+      telephonyExtensionConfigs.push({
+        Enabled: true,
+        DriverClass: 'LiveKitSipExtension',
+        RootPath: '/telephony/livekit-sip',
+        Phase: 'pre-auth',
+        Settings: { ...sharedTelephonySettings, ...configInfo.telephony.livekitSip },
       });
     }
     if (configInfo.telephony.teams?.enabled) {
@@ -1580,8 +1736,25 @@ const setupComplete$ = new ReplaySubject(1);
   // instance is still heart-beating. The watchdog also self-registers for graceful-shutdown
   // cancellation (via ShutdownRegistry) once it begins tracking this process's first live run.
   if (resumeUser && Metadata.Provider instanceof DatabaseProviderBase) { // global-provider-ok: server startup recovery — one-shot orphaned-run sweep at boot
-    AgentRunWatchdog.SweepOrphanedRuns(Metadata.Provider, resumeUser) // global-provider-ok: server startup recovery — one-shot orphaned-run sweep at boot
+    const sweepUser = resumeUser;
+    const sweepProvider = Metadata.Provider; // global-provider-ok: server startup recovery — one-shot orphaned-run sweep at boot
+    AgentRunWatchdog.SweepOrphanedRuns(sweepProvider, sweepUser)
       .catch(err => console.warn(`[AgentRunWatchdog] Startup sweep failed: ${err}`));
+
+    // The watchdog repairs the RUN; nothing repaired the conversation detail, which is the row the
+    // chat actually renders from (MJ #4222). A process that dies mid-run leaves a terminal run
+    // beside a message that still claims to be generating, and it spins forever for anyone who
+    // opens it. Runs at boot (closes restart orphans) and on a timer (closes mid-life orphans),
+    // mirroring the watchdog's own two-phase shape.
+    const reconcileOrphans = () =>
+      ReconcileOrphanedConversationDetails(sweepProvider, sweepUser)
+        .catch(err => console.warn(`[OrphanDetailReconciler] Pass failed: ${err}`));
+    void reconcileOrphans();
+    const orphanDetailTimer = setInterval(() => void reconcileOrphans(), ORPHAN_DETAIL_SWEEP_INTERVAL_MS);
+    ShutdownRegistry.Instance.Register({
+      ShutdownName: 'OrphanDetailReconciler',
+      Shutdown: () => { clearInterval(orphanDetailTimer); },
+    });
   }
 
   // Launch the AI Agent Session janitor: run own-host orphan recovery once at boot, then keep a
@@ -1623,6 +1796,18 @@ const setupComplete$ = new ReplaySubject(1);
       .catch(err => console.warn(`[TaskGraphDispatcher] Startup failed: ${err}`));
   }
 
+  // Start the durable work-queue host where enabled. It plans which subscriptions this instance runs,
+  // re-plans on a timer, and self-registers with ShutdownRegistry, so gracefulShutdown's awaited
+  // ShutdownAll() drains it (up to 2 × shutdownDrainMs) before the HTTP server closes.
+  // Not awaited: a slow engine load must not delay readiness, and a failure never stops the API.
+  const workQueueProvider = Metadata.Provider; // global-provider-ok: server startup — the work-queue host runs on the server's own provider
+  if (configInfo.workQueue?.enabled && workQueueProvider instanceof DatabaseProviderBase) {
+    const workQueuePool = dataSources[0]?.dataSource;
+    const providerSource = new MJServerWorkQueueProviderSource(workQueuePool instanceof sql.ConnectionPool ? workQueuePool : null, workQueueProvider);
+    StartWorkQueueHost(configInfo.workQueue, workQueueProvider, providerSource)
+      .catch(error => console.error('❌ Failed to start the work queue host:', error));
+  }
+
 
   // Set up graceful shutdown handlers
   const gracefulShutdown = async (signal: string) => {
@@ -2106,11 +2291,3 @@ function createMSSQLCompatPool(pgPool: import('pg').Pool): sql.ConnectionPool {
   return wrapper as unknown as sql.ConnectionPool;
 }
 
-/**
- * Translates SQL Server bracket-quoted identifiers to PostgreSQL double-quoted identifiers.
- * Converts [schema].[table] to "schema"."table" and handles common T-SQL patterns.
- */
-function translateBracketsToPG(sql: string): string {
-  // Replace [identifier] with "identifier"
-  return sql.replace(/\[([^\]]+)\]/g, '"$1"');
-}

package/src/postgresqlCompat.ts

@@ -0,0 +1,21 @@
+import { LexSQL } from '@memberjunction/sql-parser';
+import { SQLServerDialect } from '@memberjunction/sql-dialect';
+
+const sqlServer = new SQLServerDialect();
+
+/**
+ * Translates SQL Server bracket-quoted identifiers (`[Name]`) to PostgreSQL double-quoted ones
+ * (`"Name"`), for SQL written in SQL Server syntax that runs on PostgreSQL. Only identifiers
+ * change: brackets inside string literals and comments are left as written. A `]]` in a name
+ * becomes `]`, and a `"` in a name is doubled.
+ */
+export function TranslateBracketsToPG(sql: string): string {
+    return LexSQL(sql, sqlServer)
+        .map(t => t.Kind === 'identifier' && t.Text.startsWith('[') ? toDoubleQuoted(t.Text) : t.Text)
+        .join('');
+}
+
+function toDoubleQuoted(bracketed: string): string {
+    const name = bracketed.slice(1, bracketed.endsWith(']') ? -1 : undefined).replace(/]]/g, ']');
+    return `"${name.replace(/"/g, '""')}"`;
+}

package/src/postgresqlPoolSettings.ts

@@ -0,0 +1,213 @@
+/**
+ * Connection settings for MJAPI's PostgreSQL pools, derived from the same `databaseSettings`
+ * values SQL Server reads, with the same defaults.
+ *
+ * | Setting                              | SQL Server                       | PostgreSQL                                   |
+ * |--------------------------------------|----------------------------------|----------------------------------------------|
+ * | `requestTimeout`                     | driver request timeout           | `statement_timeout` on every backend         |
+ * | `connectionPool.max` / `min`         | pool size                        | pool size                                    |
+ * | `connectionPool.idleTimeoutMillis`   | idle connection eviction         | idle connection eviction                     |
+ * | `connectionPool.acquireTimeoutMillis`| wait for a pooled connection     | `connectionTimeoutMillis` (see below)        |
+ * | `connectionTimeout`                  | time to open a connection        | not used                                     |
+ *
+ * Where the two cannot match exactly:
+ * - The `pg` pool has one timeout covering both waiting for a free slot and opening a connection.
+ *   It is fed from `acquireTimeoutMillis`; `connectionTimeout` is not used on PostgreSQL.
+ * - SQL Server's `requestTimeout` is a client-side timer over the whole request, including reading
+ *   rows. PostgreSQL's `statement_timeout` is server-side execution time per statement, so a query
+ *   that executes quickly and returns its rows slowly is not cut off on PostgreSQL.
+ * - `idle_in_transaction_session_timeout` has no SQL Server counterpart. It is derived as twice the
+ *   statement timeout, so a transaction left open by an abandoned request is ended and its locks
+ *   released.
+ * - MJAPI opens a raw compatibility pool and a provider pool, each up to `connectionPool.max`, plus
+ *   a CodeGen pool of 10 and, when a read-only login is configured, a read-only pool of at most 10,
+ *   where SQL Server opens one pool (and a read-only one). PostgreSQL's default `max_connections`
+ *   is 100, so the read-only and CodeGen pools are kept small.
+ */
+import type { PGConnectionConfig } from '@memberjunction/postgresql-dataprovider';
+import type { Pool, PoolConfig } from 'pg';
+import { BaseSingleton } from '@memberjunction/global';
+import type { DatabaseSettingsInfo } from './config.js';
+
+/** Statement timeout for the CodeGen/DDL pool, matching the SQL Server CodeGen pool. */
+export const CODEGEN_STATEMENT_TIMEOUT_MS = 600000;
+
+/** Pool size for the CodeGen/DDL pool. */
+const CODEGEN_MAX_CONNECTIONS = 10;
+
+/** Largest pool size for the read-only pool, which serves only caller-supplied SQL. */
+const READ_ONLY_MAX_CONNECTIONS = 10;
+
+/**
+ * Which pool the settings are for: API traffic, the read-only pool for caller-supplied SQL, or
+ * CodeGen and DDL work.
+ */
+export type PostgreSQLPoolPurpose = 'api' | 'read-only' | 'codegen';
+
+/** Where a PostgreSQL pool connects and as whom. */
+export interface PostgreSQLEndpoint {
+    Host: string;
+    Port: number;
+    Database: string;
+    User: string;
+    Password: string;
+}
+
+/** Reads the PostgreSQL endpoint from `PG_*` environment variables, then `DB_*` ones. */
+export function ResolvePostgreSQLEndpoint(env: NodeJS.ProcessEnv = process.env): PostgreSQLEndpoint {
+    return {
+        Host: env.PG_HOST || env.DB_HOST || 'localhost',
+        Port: parseInt(env.PG_PORT || env.DB_PORT || '5432', 10),
+        Database: env.PG_DATABASE || env.DB_DATABASE || '',
+        User: env.PG_USERNAME || env.DB_USERNAME || 'postgres',
+        Password: env.PG_PASSWORD || env.DB_PASSWORD || ''
+    };
+}
+
+/** The idle-in-transaction limit for a given statement timeout; 0 means none. */
+export function IdleInTransactionTimeoutFor(statementTimeoutMs: number): number {
+    return statementTimeoutMs > 0 ? statementTimeoutMs * 2 : 0;
+}
+
+/** Builds the provider connection config for one of MJAPI's PostgreSQL pools. */
+export function BuildPostgreSQLConnectionConfig(
+    endpoint: PostgreSQLEndpoint,
+    settings: DatabaseSettingsInfo,
+    purpose: PostgreSQLPoolPurpose
+): PGConnectionConfig {
+    const pool = settings.connectionPool;
+    if (purpose === 'codegen') {
+        return {
+            ...endpoint,
+            MaxConnections: CODEGEN_MAX_CONNECTIONS,
+            StatementTimeoutMs: CODEGEN_STATEMENT_TIMEOUT_MS,
+            IdleInTransactionSessionTimeoutMs: 0
+        };
+    }
+    const statementTimeoutMs = Math.max(0, settings.requestTimeout);
+    const apiMax = pool?.max ?? 50;
+    const apiMin = pool?.min ?? 5;
+    const readOnly = purpose === 'read-only';
+    return {
+        ...endpoint,
+        MaxConnections: readOnly ? Math.min(READ_ONLY_MAX_CONNECTIONS, apiMax) : apiMax,
+        MinConnections: readOnly ? 0 : apiMin,
+        IdleTimeoutMillis: pool?.idleTimeoutMillis ?? 30000,
+        ConnectionTimeoutMillis: pool?.acquireTimeoutMillis ?? 30000,
+        StatementTimeoutMs: statementTimeoutMs,
+        IdleInTransactionSessionTimeoutMs: IdleInTransactionTimeoutFor(statementTimeoutMs)
+    };
+}
+
+/**
+ * Maps a provider connection config onto `pg.Pool` options, for the pools MJAPI creates directly
+ * rather than through the provider. Timeouts that are unset or 0 are left out.
+ */
+export function ToPGPoolConfig(config: PGConnectionConfig): PoolConfig {
+    const pool: PoolConfig = {
+        host: config.Host,
+        port: config.Port,
+        database: config.Database,
+        user: config.User,
+        password: config.Password,
+        max: config.MaxConnections
+    };
+    if (config.MinConnections !== undefined) pool.min = config.MinConnections;
+    if (config.IdleTimeoutMillis !== undefined) pool.idleTimeoutMillis = config.IdleTimeoutMillis;
+    if (config.ConnectionTimeoutMillis !== undefined) pool.connectionTimeoutMillis = config.ConnectionTimeoutMillis;
+    if (config.StatementTimeoutMs) pool.statement_timeout = config.StatementTimeoutMs;
+    if (config.IdleInTransactionSessionTimeoutMs) pool.idle_in_transaction_session_timeout = config.IdleInTransactionSessionTimeoutMs;
+    return pool;
+}
+
+/** Credentials for a read-only database login. */
+export interface ReadOnlyCredentials {
+    User: string;
+    Password: string;
+}
+
+/**
+ * The read-only login for PostgreSQL: the `dbReadOnlyUsername` / `dbReadOnlyPassword` settings
+ * SQL Server uses (which default from `DB_READ_ONLY_USERNAME` / `DB_READ_ONLY_PASSWORD`), then
+ * `PG_READ_ONLY_USERNAME` / `PG_READ_ONLY_PASSWORD`. `null` when neither pair is complete.
+ */
+export function ResolvePostgreSQLReadOnlyCredentials(
+    configured: { dbReadOnlyUsername?: string; dbReadOnlyPassword?: string },
+    env: NodeJS.ProcessEnv = process.env
+): ReadOnlyCredentials | null {
+    if (configured.dbReadOnlyUsername && configured.dbReadOnlyPassword) {
+        return { User: configured.dbReadOnlyUsername, Password: configured.dbReadOnlyPassword };
+    }
+    if (env.PG_READ_ONLY_USERNAME && env.PG_READ_ONLY_PASSWORD) {
+        return { User: env.PG_READ_ONLY_USERNAME, Password: env.PG_READ_ONLY_PASSWORD };
+    }
+    return null;
+}
+
+/**
+ * Holds MJAPI's PostgreSQL read-only pool, opened once at startup with the read-only login.
+ * Per-request read-only providers share it, so they connect as the read-only user rather than
+ * through the read-write pool.
+ */
+export class PostgreSQLReadOnlyPool extends BaseSingleton<PostgreSQLReadOnlyPool> {
+    private _pool: Pool | null = null;
+
+    /** Use {@link PostgreSQLReadOnlyPool.Instance}. */
+    public constructor() {
+        super();
+    }
+
+    public static get Instance(): PostgreSQLReadOnlyPool {
+        return PostgreSQLReadOnlyPool.getInstance<PostgreSQLReadOnlyPool>();
+    }
+
+    /** The read-only pool, or `null` when no read-only login is configured. */
+    public get Pool(): Pool | null {
+        return this._pool;
+    }
+
+    public set Pool(pool: Pool | null) {
+        this._pool = pool;
+    }
+}
+
+/** What {@link DescribeReadOnlyLoginOverreach} reads about the read-only login. */
+export interface ReadOnlyLoginGrants {
+    ReadsServerFiles: boolean;
+    ReadableBaseTables: number;
+}
+
+/** The one method {@link DescribeReadOnlyLoginOverreach} needs from a connection; a `pg` client has it. */
+export interface ReadOnlyLoginCheckClient {
+    query(sql: string, params: unknown[]): Promise<{ rows: ReadOnlyLoginGrants[] }>;  // case-violation-ok-legacy-back-compat: mirrors the pg client's query method, which a pg client must satisfy as-is
+}
+
+/**
+ * What the PostgreSQL read-only login can do beyond reading entity base views, as warnings to log at
+ * startup. Ad-hoc SQL and `TestQuerySQL` run as this login, and SQL can reach data through functions
+ * that no check of its table references sees (`query_to_xml`, file readers), so the login's own
+ * grants are the only complete limit. It should hold `SELECT` on the entity base views and nothing
+ * else: not on the base tables, and not membership of `pg_read_server_files`. Empty when the login
+ * is that narrow.
+ *
+ * @param client - A connection opened as the read-only login.
+ * @param coreSchema - The MJ core schema, whose base tables the login should not read.
+ */
+export async function DescribeReadOnlyLoginOverreach(client: ReadOnlyLoginCheckClient, coreSchema: string): Promise<string[]> {
+    const result = await client.query(
+        `SELECT pg_has_role(current_user, 'pg_read_server_files', 'MEMBER') AS "ReadsServerFiles",
+                (SELECT count(*)::int FROM pg_catalog.pg_tables t
+                  WHERE t.schemaname = $1
+                    AND has_table_privilege(current_user, quote_ident(t.schemaname) || '.' || quote_ident(t.tablename), 'SELECT')) AS "ReadableBaseTables"`,
+        [coreSchema],
+    );
+    const row = result.rows[0];
+    const warnings: string[] = [];
+    if (row?.ReadableBaseTables > 0) {
+        warnings.push(`The PostgreSQL read-only login can read ${row.ReadableBaseTables} base table(s) in schema "${coreSchema}". Ad-hoc SQL and TestQuerySQL run as this login; grant it SELECT on the entity base views only.`);
+    }
+    if (row?.ReadsServerFiles) {
+        warnings.push('The PostgreSQL read-only login is a member of pg_read_server_files, so SQL run as it can read files on the database server. Revoke that membership.');
+    }
+    return warnings;
+}