@memberjunction/ai-agent-manager 6.2.0-edge.1 → 6.2.0-edge.2

package/src/__tests__/agent-spec-sync-decision-roundtrip.test.ts

@@ -0,0 +1,218 @@
+/**
+ * Round-trip coverage for Decision steps in AgentSpecSync.
+ *
+ * Verifies that a Decision step's PromptID (whether custom or omitted) and Configuration
+ * (whether supplied as JSON string or parsed object) survive save and load round trips.
+ */
+import { describe, it, expect, vi, beforeEach } from 'vitest';
+
+const { capturedStepEntities, mockGetEntityObject, mockRunView, mockRunViews } = vi.hoisted(() => {
+    const capturedStepEntities: Record<string, unknown>[] = [];
+
+    const makeEntity = (entityName: string) => {
+        const entity: Record<string, unknown> = {
+            Load: vi.fn().mockResolvedValue(true),
+            Save: vi.fn().mockResolvedValue(true),
+            Delete: vi.fn().mockResolvedValue(true),
+            NewRecord: vi.fn(),
+            Validate: vi.fn().mockReturnValue({ Success: true, Errors: [] }),
+            ID: 'saved-id-1',
+            Name: 'Test Agent',
+            TypeID: '4F6A189B-C068-4736-9F23-3FF540B40FDD',
+            Status: 'Active',
+            StartingPayloadValidationMode: 'Fail',
+        };
+
+        if (entityName === 'MJ: AI Agent Steps') {
+            capturedStepEntities.push(entity);
+        }
+
+        return entity;
+    };
+
+    const empty = { Success: true, Results: [], RowCount: 0 };
+    return {
+        capturedStepEntities,
+        mockGetEntityObject: vi.fn((name: string) => Promise.resolve(makeEntity(name))),
+        mockRunView: vi.fn(() => Promise.resolve(empty)),
+        mockRunViews: vi.fn(() => Promise.resolve([empty, empty, empty, empty, empty])),
+    };
+});
+
+vi.mock('@memberjunction/core', () => {
+    const rvInstance = { RunView: mockRunView, RunViews: mockRunViews };
+    return {
+        Metadata: Object.assign(
+            vi.fn().mockImplementation(() => ({ GetEntityObject: mockGetEntityObject })),
+            { Provider: { GetEntityObject: mockGetEntityObject } }
+        ),
+        RunView: Object.assign(
+            vi.fn().mockImplementation(() => rvInstance),
+            { FromMetadataProvider: vi.fn(() => rvInstance) }
+        ),
+        UserInfo: vi.fn(),
+        LogError: vi.fn(),
+    };
+});
+
+vi.mock('@memberjunction/core-entities', () => ({
+    MJAIAgentEntity: vi.fn(),
+    MJAIAgentActionEntity: vi.fn(),
+    MJAIAgentRelationshipEntity: vi.fn(),
+    MJAIAgentStepEntity: vi.fn(),
+    MJAIAgentStepPathEntity: vi.fn(),
+}));
+
+vi.mock('@memberjunction/ai-core-plus', () => ({
+    AgentSpec: vi.fn(),
+    AgentActionSpec: vi.fn(),
+    SubAgentSpec: vi.fn(),
+}));
+
+import { UserInfo } from '@memberjunction/core';
+import { AgentSpecSync } from '../agent-spec-sync';
+
+describe('AgentSpecSync Decision step round-trip', () => {
+    // UserInfo is mocked above, so this is the mock's instance.
+    const mockUser = new UserInfo();
+
+    beforeEach(() => {
+        vi.clearAllMocks();
+        capturedStepEntities.length = 0;
+    });
+
+    it('persists a Decision step with custom PromptID and object Configuration', async () => {
+        const decisionConfig = {
+            key: 'triage',
+            questions: {
+                category: {
+                    instructions: 'What category?',
+                    kind: 'Choice',
+                    options: [
+                        { value: 'billing', description: 'Billing' },
+                        { value: 'tech', description: 'Technical' },
+                    ],
+                },
+            },
+        };
+
+        const sync = new AgentSpecSync(
+            {
+                ID: 'agent-1',
+                Name: 'Decision Flow Agent',
+                TypeID: '4F6A189B-C068-4736-9F23-3FF540B40FDD',
+                Steps: [
+                    {
+                        ID: 'step-1',
+                        Name: 'Triage Issue',
+                        StepType: 'Decision',
+                        StartingStep: true,
+                        PromptID: 'custom-prompt-uuid-999',
+                        Configuration: decisionConfig,
+                    },
+                ],
+            },
+            mockUser
+        );
+
+        const result = await sync.SaveToDatabase();
+        expect(result.success).toBe(true);
+
+        const savedStep = capturedStepEntities.find(s => s.Name === 'Triage Issue');
+        expect(savedStep).toBeDefined();
+        expect(savedStep?.StepType).toBe('Decision');
+        expect(savedStep?.PromptID).toBe('custom-prompt-uuid-999');
+        expect(savedStep?.Configuration).toBe(JSON.stringify(decisionConfig));
+    });
+
+    it('persists a Decision step with null PromptID when none is provided', async () => {
+        const sync = new AgentSpecSync(
+            {
+                ID: 'agent-2',
+                Name: 'Decision Agent Without Prompt',
+                TypeID: '4F6A189B-C068-4736-9F23-3FF540B40FDD',
+                Steps: [
+                    {
+                        ID: 'step-2',
+                        Name: 'Default Decision Step',
+                        StepType: 'Decision',
+                        StartingStep: true,
+                        Configuration: JSON.stringify({ key: 'route', questions: {} }),
+                    },
+                ],
+            },
+            mockUser
+        );
+
+        const result = await sync.SaveToDatabase();
+        expect(result.success).toBe(true);
+
+        const savedStep = capturedStepEntities.find(s => s.Name === 'Default Decision Step');
+        expect(savedStep).toBeDefined();
+        expect(savedStep?.PromptID).toBeNull();
+        expect(savedStep?.Configuration).toBe(JSON.stringify({ key: 'route', questions: {} }));
+    });
+
+    it('loads a Decision step from database preserving PromptID and Configuration', async () => {
+        const decisionConfigJson = JSON.stringify({
+            key: 'triage',
+            questions: {
+                priority: {
+                    text: 'What priority?',
+                    kind: 'Rating',
+                    minRating: 1,
+                    maxRating: 5,
+                },
+            },
+        });
+
+        const mockAgentEntity = {
+            Load: vi.fn().mockResolvedValue(true),
+            ID: 'agent-3',
+            Name: 'Loaded Decision Agent',
+            TypeID: '4F6A189B-C068-4736-9F23-3FF540B40FDD',
+            Status: 'Active',
+            StartingPayloadValidationMode: 'Fail',
+        };
+
+        const mockStepEntity = {
+            ID: 'step-3',
+            AgentID: 'agent-3',
+            Name: 'Rate Priority',
+            StepType: 'Decision',
+            StartingStep: true,
+            PromptID: 'decision-prompt-777',
+            Configuration: decisionConfigJson,
+            Status: 'Active',
+        };
+
+        mockGetEntityObject.mockImplementation((name: string) => {
+            if (name === 'MJ: AI Agents') {
+                return Promise.resolve(mockAgentEntity);
+            }
+            return Promise.resolve({
+                Load: vi.fn().mockResolvedValue(true),
+                Save: vi.fn().mockResolvedValue(true),
+                Validate: vi.fn().mockReturnValue({ Success: true, Errors: [] }),
+            });
+        });
+
+        // Set up RunViews return: [actions, childAgents, relatedAgents, prompts, steps]
+        const empty = { Success: true, Results: [], RowCount: 0 };
+        const stepsResult = { Success: true, Results: [mockStepEntity], RowCount: 1 };
+        mockRunViews.mockResolvedValueOnce([empty, empty, empty, empty, stepsResult]);
+        // mockRunView for paths
+        mockRunView.mockResolvedValueOnce(empty);
+
+        const loadedSync = await AgentSpecSync.LoadFromDatabase('agent-3', mockUser);
+
+        expect(loadedSync.Spec.Steps).toBeDefined();
+        expect(loadedSync.Spec.Steps).toHaveLength(1);
+
+        const loadedStep = loadedSync.Spec.Steps![0];
+        expect(loadedStep.Name).toBe('Rate Priority');
+        expect(loadedStep.StepType).toBe('Decision');
+        expect(loadedStep.PromptID).toBe('decision-prompt-777');
+        expect(loadedStep.Configuration).toBe(decisionConfigJson);
+    });
+});

package/src/__tests__/architect-agent-example.test.ts

@@ -0,0 +1,218 @@
+import { describe, it, expect, beforeEach, vi } from 'vitest';
+import * as fs from 'fs';
+import * as path from 'path';
+import type { AgentSpec, AgentStep, AgentStepPath, ExecuteAgentParams, MJAIPromptEntityExtended } from '@memberjunction/ai-core-plus';
+import type { MJAIAgentTypeEntity, MJAIModelTypeEntity } from '@memberjunction/core-entities';
+import { AIEngine } from '@memberjunction/aiengine';
+import { AgentArchitectAgent } from '../agents/architect-agent';
+import { ValidateFlowGraph } from '../flow-step-validation';
+
+/** The seeded agent types the Architect's template names by ID. */
+const FLOW_TYPE_ID = '4F6A189B-C068-4736-9F23-3FF540B40FDD';
+const LOOP_TYPE_ID = 'F7926101-5099-4FA5-836A-479D9707C818';
+
+const LLM_MODEL_TYPE_ID = 'E8A5CCEC-6A37-EF11-86D4-000D3A4E707E';
+const DECISION_MODEL_TYPE_ID = 'D3C15E6A-0000-4000-8000-00000000DEC1';
+
+const DECISION_PROMPT_ID = 'D3C15E6A-0000-4000-8000-000000000001';
+const CHAT_PROMPT_ID = 'D3C15E6A-0000-4000-8000-000000000002';
+
+function agentType(id: string, name: string): MJAIAgentTypeEntity {
+    return { ID: id, Name: name } as MJAIAgentTypeEntity;
+}
+
+function modelType(id: string, name: string): MJAIModelTypeEntity {
+    return { ID: id, Name: name } as MJAIModelTypeEntity;
+}
+
+function prompt(id: string, name: string, modelTypeID: string | null): MJAIPromptEntityExtended {
+    return { ID: id, Name: name, AIModelTypeID: modelTypeID } as MJAIPromptEntityExtended;
+}
+
+class TestArchitectAgent extends AgentArchitectAgent {
+    public async testValidateAgentSpec(
+        spec: AgentSpec,
+        params: ExecuteAgentParams = {} as ExecuteAgentParams
+    ): Promise<{ errors: string[]; correctedSpec?: AgentSpec }> {
+        return this.validateAgentSpec(spec, params);
+    }
+
+    protected override async validateActions(): Promise<{ errors: string[] }> {
+        return { errors: [] };
+    }
+}
+
+/** The Architect's example outputs, keyed by example. */
+type ArchitectExamples = Record<string, { output: AgentSpec }>;
+
+const examplePath = path.resolve(
+    __dirname,
+    '../../../../../../metadata/prompts/output/agent-manager/architect-agent.example.json'
+);
+
+/** A fresh copy of example_4's spec, so a test can change it. */
+function example4(): AgentSpec {
+    const examples: ArchitectExamples = JSON.parse(fs.readFileSync(examplePath, 'utf8'));
+    return examples.example_4_flow_agent_with_decision_step.output;
+}
+
+async function validateSpec(spec: AgentSpec): Promise<string[]> {
+    return (await new TestArchitectAgent().testValidateAgentSpec(spec)).errors;
+}
+
+describe('Architect Agent Example Output and Decision Step Validation', () => {
+    beforeEach(() => {
+        vi.restoreAllMocks();
+        // validateAgentSpec resolves a spec's TypeID (a GUID) to its type's name through AIEngine, and
+        // a Decision step's PromptID to its prompt and model type.
+        vi.spyOn(AIEngine.Instance, 'AgentTypes', 'get').mockReturnValue([agentType(FLOW_TYPE_ID, 'Flow'), agentType(LOOP_TYPE_ID, 'Loop')]);
+        vi.spyOn(AIEngine.Instance, 'ModelTypes', 'get').mockReturnValue([modelType(LLM_MODEL_TYPE_ID, 'LLM'), modelType(DECISION_MODEL_TYPE_ID, 'Decision')]);
+        vi.spyOn(AIEngine.Instance, 'Prompts', 'get').mockReturnValue([
+            prompt(DECISION_PROMPT_ID, 'Fast Triage Decision', DECISION_MODEL_TYPE_ID),
+            prompt(CHAT_PROMPT_ID, 'Summarize Ticket', LLM_MODEL_TYPE_ID),
+        ]);
+    });
+
+    it('parses architect-agent.example.json as valid JSON', () => {
+        const rawContent = fs.readFileSync(examplePath, 'utf8');
+        expect(() => JSON.parse(rawContent)).not.toThrow();
+    });
+
+    it('compiles and validates example_4_flow_agent_with_decision_step as the runtime does', () => {
+        const spec = example4();
+        expect(spec.Steps?.find(s => s.StepType === 'Decision')?.Name).toBe('Triage Issue');
+        expect(ValidateFlowGraph(spec.Steps ?? [], spec.Paths ?? [], spec.Name)).toEqual([]);
+    });
+
+    it('passes full AgentArchitectAgent.validateAgentSpec for example_4', async () => {
+        expect(await validateSpec(example4())).toEqual([]);
+    });
+
+    it('catches an incomplete Choice fork inside validateAgentSpec', async () => {
+        const spec = example4();
+        spec.Paths = spec.Paths?.filter(p => !p.Condition?.includes('general'));
+
+        const errors = (await validateSpec(spec)).join(' ');
+        expect(errors).toContain('[IncompleteFork]');
+        expect(errors).toContain('"general"');
+    });
+
+    it('accepts a single conditional path from a Decision step, which the runtime runs', async () => {
+        const spec = example4();
+        spec.Steps = spec.Steps?.filter(s => s.Name === 'Triage Issue' || s.Name === 'Handle Billing');
+        spec.Paths = spec.Paths?.filter(p => p.DestinationStepID === 'Handle Billing');
+
+        expect(await validateSpec(spec)).toEqual([]);
+    });
+
+    // The three specs a review probed: the Architect passed each one, and the runtime refused it.
+    it.each([
+        {
+            what: 'a path from a later step reading an unknown Decision key',
+            code: '[UnknownDecisionKey]',
+            change: (spec: AgentSpec): void => {
+                const summarize: AgentStep = { ID: '', Name: 'Summarize', StepType: 'Prompt', StartingStep: false, PromptText: 'Summarize the issue.', PromptName: 'Summarize Issue' };
+                spec.Steps = [...(spec.Steps ?? []), summarize];
+                const fromTriage: AgentStepPath = { ID: '', OriginStepID: 'Handle General', DestinationStepID: 'Summarize', Priority: 0 };
+                const fromSummarize: AgentStepPath = { ID: '', OriginStepID: 'Summarize', DestinationStepID: 'Handle Billing', Condition: "decisions.triag.category.value === 'billing'", Priority: 0 };
+                spec.Paths = [...(spec.Paths ?? []), fromTriage, fromSummarize];
+            },
+        },
+        {
+            what: 'a Likelihood read through .value',
+            code: '[InvalidCondition]',
+            change: (spec: AgentSpec): void => addUrgentGate(spec, 'decisions.triage.urgent.value >= 0.8'),
+        },
+        {
+            what: 'a question the Decision step does not ask',
+            code: '[InvalidCondition]',
+            change: (spec: AgentSpec): void => addUrgentGate(spec, 'decisions.triage.urgency.probability >= 0.8'),
+        },
+    ])('refuses $what inside validateAgentSpec', async ({ code, change }) => {
+        const spec = example4();
+        change(spec);
+        expect((await validateSpec(spec)).join('\n')).toContain(code);
+    });
+
+    it('validates the Decision steps of a Flow child sub-agent', async () => {
+        const child: AgentSpec = {
+            ID: '',
+            Name: 'Ticket Router',
+            TypeID: FLOW_TYPE_ID,
+            StartingPayloadValidationMode: 'Fail',
+            Prompts: [],
+            Steps: [
+                { ID: '', Name: 'Route', StepType: 'Decision', StartingStep: true, Configuration: { key: '1bad' } },
+                { ID: '', Name: 'Escalate', StepType: 'Action', StartingStep: false, ActionID: 'escalate-action-guid' },
+            ],
+            Paths: [{ ID: '', OriginStepID: 'Route', DestinationStepID: 'Escalate', Condition: 'decisions.nope.q', Priority: 0 }],
+        };
+        const parent: AgentSpec = {
+            ID: '',
+            Name: 'Support Desk',
+            TypeID: LOOP_TYPE_ID,
+            StartingPayloadValidationMode: 'Fail',
+            Prompts: [{ ID: '', PromptID: '', PromptName: 'Support Desk', PromptDescription: '', PromptText: 'Help the customer.', PromptTypeID: '' }],
+            SubAgents: [{ Type: 'child', SubAgent: child }],
+        };
+
+        const errors = await validateSpec(parent);
+        const childErrors = errors.filter(e => e.startsWith('Flow SubAgent[0] "Ticket Router" -> '));
+        expect(childErrors.join('\n')).toContain('[InvalidDecisionStep]');
+        expect(childErrors.join('\n')).toContain('it asks no questions');
+        expect(childErrors.join('\n')).toContain('[UnknownDecisionKey]');
+        expect(childErrors).toHaveLength(errors.length);
+    });
+
+    it('accepts a Decision step whose PromptID is a Decision prompt', async () => {
+        const spec = example4();
+        setTriagePrompt(spec, DECISION_PROMPT_ID.toLowerCase());
+        expect(await validateSpec(spec)).toEqual([]);
+    });
+
+    it('refuses a Decision step whose PromptID is not a prompt', async () => {
+        const spec = example4();
+        setTriagePrompt(spec, 'D3C15E6A-0000-4000-8000-0000000000FF');
+        expect((await validateSpec(spec)).join('\n')).toContain('which is not a prompt');
+    });
+
+    it('refuses a Decision step whose prompt runs on another model type', async () => {
+        const spec = example4();
+        setTriagePrompt(spec, CHAT_PROMPT_ID);
+        const errors = (await validateSpec(spec)).join('\n');
+        expect(errors).toContain('uses the prompt "Summarize Ticket", whose model type is "LLM"');
+        expect(errors).toContain('must run on Decision models');
+    });
+});
+
+/** example_4's triage, asking a Likelihood as well as the Choice. */
+const TRIAGE_WITH_URGENT = {
+    key: 'triage',
+    questions: {
+        category: {
+            instructions: 'What category best describes this issue?',
+            kind: 'Choice',
+            options: [
+                { value: 'billing', description: 'Billing inquiry or invoice problem' },
+                { value: 'technical', description: 'Technical defect or system error' },
+                { value: 'general', description: 'General question or account update' },
+            ],
+        },
+        urgent: { instructions: 'The customer cannot work until this is resolved.', kind: 'Likelihood' },
+    },
+};
+
+/** Has the triage ask whether the issue is urgent, and gates a new step on the answer. */
+function addUrgentGate(spec: AgentSpec, condition: string): void {
+    const triage = spec.Steps?.find(s => s.Name === 'Triage Issue');
+    if (triage) triage.Configuration = TRIAGE_WITH_URGENT;
+    const escalate: AgentStep = { ID: '', Name: 'Escalate', StepType: 'Action', StartingStep: false, ActionID: 'escalate-action-guid' };
+    spec.Steps = [...(spec.Steps ?? []), escalate];
+    const gate: AgentStepPath = { ID: '', OriginStepID: 'Handle Technical', DestinationStepID: 'Escalate', Condition: condition, Priority: 0 };
+    spec.Paths = [...(spec.Paths ?? []), gate];
+}
+
+function setTriagePrompt(spec: AgentSpec, promptID: string): void {
+    const triage = spec.Steps?.find(s => s.Name === 'Triage Issue');
+    if (triage) triage.PromptID = promptID;
+}