@memberjunction/server 6.2.0-edge.6 → 6.2.0-edge.8

package/src/services/TaskGraphAgentRunner.ts

@@ -43,6 +43,25 @@ export class TaskGraphAgentRunner implements TaskAgentRunner {
                     };
                 }
             }
+
+            let configurationId: string | undefined;
+            const requestedPreset = params.ConfigurationPresetName?.trim();
+            if (requestedPreset) {
+                const matched = AIEngine.Instance.FindActiveAgentConfigurationPreset(agent.ID, requestedPreset);
+                if (!matched) {
+                    const presets = AIEngine.Instance.GetAgentConfigurationPresets(agent.ID, true);
+                    const available = presets.map((p) => p.Name).join(', ') || 'none';
+                    return {
+                        Success: false,
+                        ErrorMessage: `Agent '${agent.Name || params.AgentID}' has no active preset '${requestedPreset}'. Available: ${available}.`,
+                    };
+                }
+                configurationId = matched.AIConfigurationID ?? undefined;
+            } else {
+                configurationId = parentRun?.ConfigurationID ?? undefined;
+            }
+
+            const parentAgentHierarchy = this.buildParentAgentHierarchy(parentRun, params);
             const taskContext = await this.resolveTaskContext(params);
             const conversationDetailId =
                 parentRun?.ConversationDetailID ?? taskContext.conversationDetailId ?? undefined;
@@ -66,6 +85,8 @@ export class TaskGraphAgentRunner implements TaskAgentRunner {
                 // rather than starting the chain over. Without it MAX_REINVOKE_DEPTH never fires.
                 continuationDepth: params.ContinuationDepth ?? 0,
                 parentRun,
+                parentAgentHierarchy,
+                configurationId,
                 // The bridge from the agent's own progress stream up to graph-level observers: the
                 // dispatcher turns these into rate-limited `NodeProgress` frames, which is how a
                 // running Agent step shows "what it is doing right now" on the workflow canvas.
@@ -221,6 +242,28 @@ export class TaskGraphAgentRunner implements TaskAgentRunner {
         }
     }
 
+    /**
+     * Reads the root-first ancestor agent name hierarchy carried on the task from when its graph was
+     * submitted (`params.ParentAgentHierarchy`), falling back to the submitting run's own agent name
+     * when no hierarchy was persisted on the parent task.
+     */
+    private buildParentAgentHierarchy(
+        parentRun: MJAIAgentRunEntityExtended | undefined,
+        params: TaskAgentRunParams,
+    ): string[] | undefined {
+        if (Array.isArray(params.ParentAgentHierarchy)) {
+            const carried = params.ParentAgentHierarchy
+                .filter((name): name is string => typeof name === 'string')
+                .map((name) => name.trim())
+                .filter((name) => name.length > 0);
+            if (carried.length > 0) {
+                return carried;
+            }
+        }
+        const submittingAgentName = parentRun?.Agent?.trim();
+        return submittingAgentName ? [submittingAgentName] : undefined;
+    }
+
     /**
      * Builds the agent's instruction from the task's name/description, its structured input payload,
      * and its dependencies' outputs.

package/src/services/TaskGraphContinuationDeliverer.ts

@@ -491,6 +491,7 @@ export class TaskGraphContinuationDeliverer implements TaskContinuationDeliverer
                     Fields: ['ID'],
                     OrderBy: '__mj_CreatedAt DESC',
                     MaxRows: 1,
+                    SkipTotalRowCount: true,
                     ResultType: 'simple',
                     BypassCache: true,
                 },

package/src/services/WorkQueueHostService.ts

@@ -28,13 +28,25 @@ export interface StartableWorkQueueHost {
 export interface WorkQueueHostStartDependencies {
     Env: EnvironmentVariables;
     FindUserByEmail(email: string): UserInfo | undefined;
+    FindSystemUser?(): UserInfo | undefined;
     ConfigureEngine(user: UserInfo, provider: DatabaseProviderBase): Promise<WorkQueueEngine>;
     CreateHost(config: WorkQueueHostConfig, engine: WorkQueueEngine, user: UserInfo, provider: DatabaseProviderBase, providerSource: WorkQueueProviderSource): StartableWorkQueueHost;
 }
 
+const DEFAULT_SYSTEM_USER_EMAILS = new Set(['system@memberjunction.org', 'not.set@nowhere.com']);
+
+type IndependentProviderCapable = IMetadataProvider & {
+    CreateIndependentInstance(): Promise<IMetadataProvider>;
+};
+
+function hasCreateIndependentInstance(provider: IMetadataProvider): provider is IndependentProviderCapable {
+    return 'CreateIndependentInstance' in provider && typeof provider.CreateIndependentInstance === 'function';
+}
+
 export const DEFAULT_WORK_QUEUE_HOST_START_DEPENDENCIES: WorkQueueHostStartDependencies = {
     Env: process.env,
     FindUserByEmail: email => UserCache.Users.find(user => user.Email?.trim().toLowerCase() === email.trim().toLowerCase()),
+    FindSystemUser: () => UserCache.Instance.GetSystemUser(),
     ConfigureEngine: async (user, provider) => {
         await WorkQueueEngine.Instance.Config(false, user, provider);
         await ReportTopologyIssues(WorkQueueEngine.Instance);
@@ -89,7 +101,7 @@ export function BuildWorkQueueHostConfig(config: WorkQueueConfig, instanceID: st
     };
 }
 
-/** A fresh SQL Server provider per delivery over the shared pool; the server provider otherwise. */
+/** A fresh SQL Server or PostgreSQL provider per delivery over the shared pool; the server provider otherwise. */
 export class MJServerWorkQueueProviderSource implements WorkQueueProviderSource {
     private readonly factory: TaskGraphProviderFactory | null;
 
@@ -97,8 +109,14 @@ export class MJServerWorkQueueProviderSource implements WorkQueueProviderSource
         this.factory = pool ? new TaskGraphProviderFactory(pool) : null;
     }
 
-    public CreateProvider(): Promise<IMetadataProvider> {
-        return this.factory ? this.factory.CreateProvider() : Promise.resolve(this.fallback);
+    public async CreateProvider(): Promise<IMetadataProvider> {
+        if (this.factory) {
+            return this.factory.CreateProvider();
+        }
+        if (hasCreateIndependentInstance(this.fallback)) {
+            return this.fallback.CreateIndependentInstance();
+        }
+        return this.fallback;
     }
 }
 
@@ -116,7 +134,11 @@ export async function StartWorkQueueHost(
         LogStatus(`[WorkQueue] Host disabled by ${WORK_QUEUE_DISABLE_ENV}=1 — this process publishes but runs no subscriptions.`);
         return null;
     }
-    const user = dependencies.FindUserByEmail(config.systemUserEmail);
+    const configuredEmail = config.systemUserEmail?.trim() ?? '';
+    let user = configuredEmail ? dependencies.FindUserByEmail(configuredEmail) : undefined;
+    if (!user && (!configuredEmail || DEFAULT_SYSTEM_USER_EMAILS.has(configuredEmail.toLowerCase()))) {
+        user = dependencies.FindSystemUser?.();
+    }
     if (!user) {
         throw new Error(`[WorkQueue] System user not found with email: ${config.systemUserEmail} — set workQueue.systemUserEmail in mj.config.cjs`);
     }

package/src/services/workQueueConfig.ts

@@ -18,7 +18,7 @@ export const WorkQueueSubscriptionEntrySchema = z.object({
  */
 export const WorkQueueSchema = z
   .object({
-    enabled: z.boolean().optional().default(false),
+    enabled: z.boolean().optional().default(true),
     systemUserEmail: z.string().optional().default('system@memberjunction.org'),
     subscriptions: z.array(WorkQueueSubscriptionEntrySchema).optional().default([{ name: '*', concurrency: 4 }]),
     idlePollMinMs: z.number().int().positive().optional().default(250),
@@ -36,12 +36,12 @@ export const WorkQueueSchema = z
 export type WorkQueueConfig = z.infer<typeof WorkQueueSchema>;
 
 /**
- * The merge base for mj.config.cjs. Like scheduledJobs and integrationSyncWorker, the EFFECTIVE default system user
- * is the placeholder: set workQueue.systemUserEmail explicitly, or StartWorkQueueHost fails fast and says so.
+ * The merge base for mj.config.cjs. Enabled by default with the Database transport out of the box; StartWorkQueueHost
+ * resolves `systemUserEmail` first and falls back to `UserCache.Instance.GetSystemUser()` when the default is unchanged.
  */
 export const DEFAULT_WORK_QUEUE_CONFIG: WorkQueueConfig = {
-  enabled: false,
-  systemUserEmail: 'not.set@nowhere.com',
+  enabled: true,
+  systemUserEmail: 'system@memberjunction.org',
   subscriptions: [{ name: '*', concurrency: 4 }],
   idlePollMinMs: 250,
   idlePollMaxMs: 5000,

package/src/sharedCache.ts

@@ -238,7 +238,7 @@ function relayRunViewChangeToBrowsers(event: CacheChangedEvent): void {
     PubSubManager.Instance.Publish(CACHE_INVALIDATION_TOPIC, {
         entityName,
         primaryKeyValues: null, // entity-level invalidation
-        action: event.Action,
+        action: event.Action === 'row_delta' ? 'set' : event.Action,
         sourceServerId: event.SourceServerId || 'unknown',
         timestamp: new Date(),
     });

package/src/types.ts

@@ -73,6 +73,19 @@ export type AppContext = {
    * Used only to key abuse limits (e.g. realtime verification sends) — never as an identity.
    */
   ClientIp?: string;
+
+  /**
+   * The request's `Origin` header as a normalized origin — scheme, host and port, lowercased, no
+   * trailing slash (e.g. `https://app.example.com:8443`). Normalized by `NormalizeOrigin`, the same
+   * helper the realtime widget allowlist uses, so it compares to allowlist entries normalized the
+   * same way with plain string equality. Ports are kept as sent; browsers omit default ports.
+   *
+   * Undefined when the request sent no `Origin` (server-to-server calls, many same-origin GETs),
+   * sent the opaque `null` origin, or sent a value that is not an http(s) origin. Always absent on
+   * WebSocket (subscription) contexts. Treat undefined as "origin unknown": an allowlist check
+   * must fail closed on it.
+   */
+  RequestOrigin?: string;
 };
 
 export class ProviderInfo {
@@ -145,6 +158,11 @@ export type RunViewGenericParams = {
    * Propagated to `RunViewParams.IncludeBinaryFields`.
    */
   includeBinaryFields?: boolean;
+  /**
+   * When true, no count runs to populate `TotalRowCount`, which comes back undefined.
+   * Propagated to `RunViewParams.SkipTotalRowCount`.
+   */
+  skipTotalRowCount?: boolean;
   /**
    * Optional registered Alternate Base View logical Name (from `EntityAlternateBaseView.Name`)
    * for this entity. Propagated to `RunViewParams.AlternateBaseView`.

package/src/util.ts

@@ -6,7 +6,8 @@ import { URL } from 'url';
 import { z } from 'zod';
 import { DataSourceInfo, ProviderInfo } from './types';
 import sql from 'mssql';
-import { DatabaseProviderBase } from '@memberjunction/core';
+import { DatabaseProviderBase, GetDiagnosticContext, IsDiagnosticCaptureSuppressed } from '@memberjunction/core';
+import { GetGlobalObjectStore } from '@memberjunction/global';
 
 const gzip = promisify(gzipCallback);
 
@@ -249,8 +250,59 @@ export async function sendPostRequest(url: string, payload: any, useCompression:
           sqlQuery = sqlQuery.replace(/\?/g, () => `@p${paramIndex++}`);
         }
       }
-      const result = await request.query(sqlQuery);
-      return result.recordset || [];
+      const startTime = performance.now();
+      const notifyPoolSqlObservers = (rowCount: number, errorMessage?: string): void => {
+        if (typeof IsDiagnosticCaptureSuppressed === 'function' && IsDiagnosticCaptureSuppressed()) {
+          return;
+        }
+        const observers = GetGlobalObjectStore()?.['___MJ_SQL_OBSERVERS___'] as
+          | Set<{ onSqlExecuted: (obs: Record<string, unknown>) => void }>
+          | undefined;
+        if (!observers || observers.size === 0) {
+          return;
+        }
+        const ctx = typeof GetDiagnosticContext === 'function' ? GetDiagnosticContext() : undefined;
+        const obs: Record<string, unknown> = {
+          ts: Date.now(),
+          sql: sqlQuery,
+          parameterTypes: Array.isArray(parameters)
+            ? parameters.map((v, idx) => ({
+                name: `p${idx}`,
+                sqlType: v === null ? 'null' : typeof v,
+                byteLength: typeof v === 'string' ? v.length : 0,
+              }))
+            : [],
+          durationMs: performance.now() - startTime,
+          rowCount,
+          isMutation: false,
+          description: 'ConnectionPool.query',
+          ...(errorMessage ? { errorMessage } : {}),
+          ...(ctx
+            ? {
+                diagnosticContext: ctx,
+                sessionId: ctx.sessionId ?? ctx.SessionID,
+                traceId: ctx.traceId ?? ctx.TraceID,
+                spanId: ctx.spanId ?? ctx.SpanID,
+              }
+            : {}),
+        };
+        for (const observer of observers) {
+          try {
+            observer.onSqlExecuted(obs);
+          } catch {
+            // Observer errors must never affect SQL execution
+          }
+        }
+      };
+      try {
+        const result = await request.query(sqlQuery);
+        const rows = result.recordset || [];
+        notifyPoolSqlObservers(rows.length);
+        return rows;
+      } catch (err) {
+        notifyPoolSqlObservers(0, err instanceof Error ? err.message : String(err));
+        throw err;
+      }
     };
     return extendedPool;
   }

package/src/services/TaskOrchestration-Integration.md

@@ -1,188 +0,0 @@
-# Task Orchestration Integration Guide
-
-## Overview
-This document explains how to integrate the TaskOrchestrator into the Conversation Manager agent flow.
-
-## Components Created
-
-1. **TaskOrchestrator** (`/services/TaskOrchestrator.ts`)
-   - Creates tasks from LLM task graphs
-   - Manages task dependencies
-   - Executes tasks in proper order
-   - Passes outputs between tasks
-
-2. **Updated Conversation Manager Prompt** (`/metadata/prompts/templates/conversations/conversation-manager-agent.template.md`)
-   - Now supports returning task graphs
-   - Backward compatible with simple agent delegation
-
-## Integration Points
-
-### 1. Where to Add Task Orchestration Logic
-
-When the Conversation Manager agent returns a response, check the `payloadChangeRequest.newElements` for either:
-
-**Simple delegation (existing):**
-```typescript
-if (newElements.invokeAgent) {
-    // Existing flow - delegate to single agent
-    const agentName = newElements.invokeAgent;
-    // ... invoke agent ...
-}
-```
-
-**Multi-step orchestration (new):**
-```typescript
-if (newElements.taskGraph) {
-    const taskGraph = newElements.taskGraph;
-
-    if (taskGraph.isMultiStep) {
-        // Create tasks in database
-        const orchestrator = new TaskOrchestrator(contextUser);
-        const taskIdMap = await orchestrator.createTasksFromGraph(
-            taskGraph,
-            conversationDetailId,
-            environmentId
-        );
-
-        // Execute tasks (respecting dependencies)
-        const results = await orchestrator.executeTasksForConversation(
-            conversationDetailId
-        );
-
-        // Send results back to conversation
-        // ... format and return results ...
-    }
-}
-```
-
-### 2. Suggested Integration Location
-
-**Option A: In RunAIAgent Resolver**
-- File: `/packages/MJServer/src/resolvers/RunAIAgentResolver.ts`
-- After agent execution, check the result payload for taskGraph
-- Create and execute tasks if multi-step
-
-**Option B: In AskSkip Resolver**
-- File: `/packages/MJServer/src/resolvers/AskSkipResolver.ts`
-- When processing conversation manager responses
-- Check for taskGraph in the response payload
-
-**Option C: New Task Resolver (Recommended)**
-- Create `/packages/MJServer/src/resolvers/TaskResolver.ts`
-- Add mutation: `ExecuteTaskGraph(taskGraph, conversationDetailId, environmentId)`
-- Keep task orchestration logic separate and reusable
-
-### 3. Example Integration Code
-
-```typescript
-import { TaskOrchestrator, TaskGraphResponse } from '../services/TaskOrchestrator.js';
-
-// In your resolver after Conversation Manager responds
-async handleConversationManagerResponse(
-    agentResult: ExecuteAgentResult,
-    conversationDetailId: string,
-    environmentId: string,
-    contextUser: UserInfo
-): Promise<void> {
-    // Check if response contains a task graph
-    if (agentResult.payload?.taskGraph) {
-        const taskGraph: TaskGraphResponse = agentResult.payload.taskGraph;
-
-        if (taskGraph.isMultiStep) {
-            LogStatus(`Multi-step workflow detected: ${taskGraph.tasks.length} tasks`);
-
-            // Create task orchestrator
-            const orchestrator = new TaskOrchestrator(contextUser);
-
-            // Create tasks and dependencies
-            const taskIdMap = await orchestrator.createTasksFromGraph(
-                taskGraph,
-                conversationDetailId,
-                environmentId
-            );
-
-            LogStatus(`Created ${taskIdMap.size} tasks with dependencies`);
-
-            // Execute tasks in proper order
-            const results = await orchestrator.executeTasksForConversation(
-                conversationDetailId
-            );
-
-            // Log results
-            for (const result of results) {
-                if (result.success) {
-                    LogStatus(`Task ${result.taskId} completed successfully`);
-                } else {
-                    LogError(`Task ${result.taskId} failed: ${result.error}`);
-                }
-            }
-
-            // Return task completion summary to user
-            return {
-                success: true,
-                message: `Completed ${results.filter(r => r.success).length} of ${results.length} tasks`,
-                results: results
-            };
-        }
-    }
-
-    // Simple agent delegation (existing flow)
-    if (agentResult.payload?.invokeAgent) {
-        // ... existing single-agent delegation logic ...
-    }
-}
-```
-
-### 4. UI Integration
-
-The existing task UI components already work with TaskEntity:
-- `/packages/Angular/Generic/conversations/src/lib/components/task/task-list.component.ts`
-- Tasks will automatically appear in the task list
-- Users can see progress, dependencies, and results
-
-### 5. Testing
-
-**Test Single-Step (Backward Compatibility):**
-```
-User: "Analyze our sales data"
-Expected: Conversation Manager delegates to Analysis Agent (existing behavior)
-```
-
-**Test Multi-Step:**
-```
-User: "Research associations with 5-30M revenue in USA, then create a GTM report"
-Expected:
-- Conversation Manager returns taskGraph
-- Task 1 (Research) executes
-- Task 2 (GTM Report) waits for Task 1
-- Task 2 receives Task 1 output
-- Both complete successfully
-```
-
-## Database Schema
-
-No changes needed! TaskEntity and TaskDependencyEntity already exist with all required fields.
-
-**Note:** For production, consider adding these optional columns to Task table:
-- `InputPayload` NVARCHAR(MAX) - Better than embedding in Description
-- `OutputPayload` NVARCHAR(MAX) - Better than embedding in Description
-
-Current implementation uses Description field with `__TASK_METADATA__` and `__TASK_OUTPUT__` markers as a workaround.
-
-## Future Enhancements
-
-1. **Parallel Execution**: Modify executeTasksForConversation() to run independent tasks concurrently
-2. **Progress Streaming**: Use WebSockets to stream task progress to UI
-3. **Task Templates**: Save common workflows as reusable templates
-4. **Error Recovery**: Add retry logic and error handling strategies
-5. **Task Cancellation**: Allow users to cancel in-progress task chains
-6. **Visualization**: Show task dependency graph in UI
-
-## Questions?
-
-- TaskOrchestrator handles all task lifecycle management
-- Conversation Manager LLM determines when to use multi-step
-- Existing UI components display tasks automatically
-- Fully backward compatible with simple delegation
-
-Integration should take ~1-2 hours for experienced developer familiar with the codebase.