@memberjunction/server 5.34.1 → 5.35.0

package/src/resolvers/RunAIAgentResolver.ts

@@ -1,7 +1,8 @@
 import { Resolver, Mutation, Query, Arg, Ctx, ObjectType, Field, PubSub, PubSubEngine, Subscription, Root, ResolverFilterData, ID, Int } from 'type-graphql';
 import { AppContext, UserPayload } from '../types.js';
 import { DatabaseProviderBase, LogError, LogStatus, Metadata, RunView, UserInfo, IMetadataProvider } from '@memberjunction/core';
-import { MJConversationDetailEntity, MJConversationDetailAttachmentEntity, MJConversationDetailArtifactEntity, MJArtifactVersionEntity, MJAIAgentRequestEntity } from '@memberjunction/core-entities';
+import { MJConversationDetailEntity, MJConversationDetailAttachmentEntity, MJConversationDetailArtifactEntity, MJArtifactVersionEntity, MJAIAgentRequestEntity, ArtifactMetadataEngine } from '@memberjunction/core-entities';
+import { RouteArtifact } from './artifact-routing.js';
 import { AgentRunner, ArtifactToolManager } from '@memberjunction/ai-agents';
 import { MJAIAgentEntityExtended, MJAIAgentRunEntityExtended, ExecuteAgentResult, ConversationUtility, AttachmentData } from '@memberjunction/ai-core-plus';
 import { AIEngine } from '@memberjunction/aiengine';
@@ -14,6 +15,15 @@ import { SafeJSONParse, UUIDsEqual } from '@memberjunction/global';
 import { GetAttachmentService } from '@memberjunction/aiengine';
 import { NotificationEngine } from '@memberjunction/notifications';
 
+/**
+ * Absolute server-side cap for inline content blocks emitted to the LLM, in
+ * bytes. Defense in depth: even if an Artifact Type is configured for Inline
+ * delivery, anything past this size falls back to the tool dispatch path with
+ * a visible annotation on the manifest. Single source of truth — replaces the
+ * pre-existing per-call MAX_INLINE_ARTIFACT_CHARS / maxInlineChars constants.
+ */
+const INLINE_SIZE_CAP = 100 * 1024;
+
 @ObjectType()
 export class AIAgentRunResult {
     @Field()
@@ -1310,84 +1320,132 @@ export class RunAIAgentResolver extends ResolverBase {
             );
             const attachmentDataResults = await Promise.all(attachmentDataPromises);
 
-            // Filter out nulls and convert to AttachmentData format.
-            // IMPORTANT: Skip large document attachments that are handled by artifact tools.
-            // These file types (PDF, Excel, Word) are accessible via ArtifactToolManager
-            // and embedding their multi-MB base64 content in the conversation causes
-            // context overflow on follow-up messages when conversation history is replayed.
-            // Images are kept inline since LLMs handle them natively via multimodal.
-            const ARTIFACT_TOOL_MIME_PREFIXES = [
-                'application/pdf',
-                'application/vnd.openxmlformats-officedocument', // .docx, .xlsx, .pptx
-                'application/vnd.ms-excel', // .xls
-                'application/msword', // .doc
-            ];
-
+            // Decide inline vs. tools per-attachment via the artifact-type registry
+            // and the pure RouteArtifact() function. See plans/artifact-attachment-unification.md.
+            //
+            // Note on the modality check: this resolver doesn't know which model will
+            // ultimately run, so RouteArtifact's modality predicate is passed as a no-op.
+            // Model-specific modality enforcement is the driver layer's responsibility;
+            // a follow-up PR can thread the active model through here to surface modality
+            // mismatches at the hard-error path described in plan §4.
             const validAttachments: AttachmentData[] = [];
 
             for (const result of attachmentDataResults) {
                 if (!result) continue;
+                // Storage-unified attachments link forward to their artifact version
+                // via ArtifactVersionID; the artifact path handles delivery, so skip
+                // the attachment row here to avoid double-processing.
+                if (result.attachment.ArtifactVersionID) {
+                    continue;
+                }
                 const mime = result.attachment.MimeType || '';
-                const isArtifactToolType = ARTIFACT_TOOL_MIME_PREFIXES.some(prefix => mime.startsWith(prefix));
-                if (isArtifactToolType) {
-                    // Skip raw file embedding — the agent accesses the file via
-                    // artifact tools (manifest injected into prompt) and/or native
-                    // file input (resolved per-driver in AIPromptRunner).
-                    // Do NOT add a placeholder attachment: 'Document' maps to 'file_url'
-                    // content blocks, and drivers attempt base64 decoding of the text,
-                    // causing API errors.
-                } else {
-                    validAttachments.push({
-                        type: ConversationUtility.GetAttachmentTypeFromMime(result.attachment.MimeType),
-                        mimeType: result.attachment.MimeType,
-                        fileName: result.attachment.FileName ?? undefined,
-                        sizeBytes: result.attachment.FileSizeBytes ?? undefined,
-                        width: result.attachment.Width ?? undefined,
-                        height: result.attachment.Height ?? undefined,
-                        durationSeconds: result.attachment.DurationSeconds ?? undefined,
-                        content: result.contentUrl
-                    });
+                const fileName = result.attachment.FileName ?? '';
+                const ext = fileName.includes('.') ? fileName.split('.').pop() : undefined;
+                const artifactType = ArtifactMetadataEngine.Instance.GetArtifactTypeByMimeType(mime, ext);
+
+                const decision = RouteArtifact({
+                    typeDefault: artifactType?.DefaultDeliveryMode ?? 'ToolsOnly',
+                    forceToolsOnly: false,
+                    mimeType: mime,
+                    sizeBytes: result.attachment.FileSizeBytes ?? 0,
+                    inlineSizeCap: INLINE_SIZE_CAP,
+                    modelSupportsModality: () => true,
+                    modelName: '<resolver>',
+                    artifactTypeName: artifactType?.Name ?? mime,
+                });
+
+                if (decision.delivery !== 'inline') {
+                    // Tools or error: skip inline embedding. The agent reaches the
+                    // bytes via artifact tools. Driver layer handles modality enforcement.
+                    if (decision.delivery === 'tools' && decision.annotation) {
+                        LogStatus(`[RunAIAgentResolver] ${decision.annotation}`);
+                    }
+                    continue;
                 }
+                validAttachments.push({
+                    type: ConversationUtility.GetAttachmentTypeFromMime(result.attachment.MimeType),
+                    mimeType: result.attachment.MimeType,
+                    fileName: result.attachment.FileName ?? undefined,
+                    sizeBytes: result.attachment.FileSizeBytes ?? undefined,
+                    width: result.attachment.Width ?? undefined,
+                    height: result.attachment.Height ?? undefined,
+                    durationSeconds: result.attachment.DurationSeconds ?? undefined,
+                    content: result.contentUrl
+                });
             }
 
-            // Get input artifacts for this message and convert to AttachmentData.
-            // Like regular attachments above, skip file-backed artifacts whose MIME
-            // types are handled by ArtifactToolManager — embedding their multi-MB
-            // base64 content in every conversation replay causes context overflow.
+            // Get input artifacts for this message — same routing logic, plus ForceToolsOnly.
             const inputArtifacts = inputArtifactsByDetailId.get(detail.ID) || [];
             for (const artifactVersion of inputArtifacts) {
                 const artifactMime = artifactVersion.MimeType || '';
-                const isArtifactToolHandled = ARTIFACT_TOOL_MIME_PREFIXES.some(
-                    prefix => artifactMime.startsWith(prefix)
-                );
+                const fileName = artifactVersion.FileName ?? '';
+                const ext = fileName.includes('.') ? fileName.split('.').pop() : undefined;
+                const artifactType = ArtifactMetadataEngine.Instance.GetArtifactTypeByMimeType(artifactMime, ext);
+
+                const decision = RouteArtifact({
+                    typeDefault: artifactType?.DefaultDeliveryMode ?? 'ToolsOnly',
+                    forceToolsOnly: artifactVersion.ForceToolsOnly,
+                    mimeType: artifactMime,
+                    sizeBytes: artifactVersion.ContentSizeBytes ?? 0,
+                    inlineSizeCap: INLINE_SIZE_CAP,
+                    modelSupportsModality: () => true,
+                    modelName: '<resolver>',
+                    artifactTypeName: artifactType?.Name ?? artifactMime,
+                });
 
                 if (artifactVersion.ContentMode === 'File' && artifactVersion.FileID) {
-                    if (isArtifactToolHandled) {
-                        // Skip raw file embedding — the agent accesses the file via
-                        // artifact tools (manifest injected into prompt) and/or native
-                        // file input (resolved per-driver in AIPromptRunner).
-                        // Do NOT add a placeholder attachment here: 'Document' maps to
-                        // 'file_url' content blocks, and drivers attempt base64 decoding
-                        // of the placeholder text, causing API errors.
-                    } else {
-                        // Non-artifact-tool file types: embed normally
-                        const fileContent = await this.downloadArtifactFileContent(artifactVersion, contextUser, provider);
-                        if (fileContent) {
-                            validAttachments.push({
-                                type: ConversationUtility.GetAttachmentTypeFromMime(artifactMime),
-                                mimeType: artifactMime || 'application/octet-stream',
-                                fileName: artifactVersion.FileName || artifactVersion.Name || undefined,
-                                sizeBytes: artifactVersion.ContentSizeBytes || undefined,
-                                content: fileContent
-                            });
+                    if (decision.delivery !== 'inline') {
+                        if (decision.delivery === 'tools' && decision.annotation) {
+                            LogStatus(`[RunAIAgentResolver] ${decision.annotation}`);
                         }
+                        continue;
+                    }
+                    const fileContent = await this.downloadArtifactFileContent(artifactVersion, contextUser, provider);
+                    if (fileContent) {
+                        validAttachments.push({
+                            type: ConversationUtility.GetAttachmentTypeFromMime(artifactMime),
+                            mimeType: artifactMime || 'application/octet-stream',
+                            fileName: artifactVersion.FileName || artifactVersion.Name || undefined,
+                            sizeBytes: artifactVersion.ContentSizeBytes || undefined,
+                            content: fileContent
+                        });
                     }
                 } else if (artifactVersion.Content) {
-                    // Text artifact — use BuildInlinePreview for large DataSnapshots
-                    // to preserve table structure instead of raw substring truncation.
+                    // Text-mode artifact (ContentMode = 'Text'). Honor the
+                    // routing decision the same way as for file-mode.
+                    if (decision.delivery !== 'inline') {
+                        if (decision.delivery === 'tools' && decision.annotation) {
+                            LogStatus(`[RunAIAgentResolver] ${decision.annotation}`);
+                        }
+                        continue;
+                    }
+
                     const textContent = artifactVersion.Content;
-                    const MAX_INLINE_ARTIFACT_CHARS = 10_000;
 
+                    // Media artifacts (image / audio / video) stored inline by
+                    // the server hook arrive here as `Text` mode with a base64
+                    // data URL in Content. We must route them as their native
+                    // modality, not as text — otherwise the LLM sees the raw
+                    // base64 as text content, can't process it, and either
+                    // hallucinates or admits confusion. Use the artifact's
+                    // declared MIME (not the wrapper string) so
+                    // ConversationUtility builds the right content block type.
+                    const mediaModality = artifactMime.startsWith('image/')
+                        || artifactMime.startsWith('audio/')
+                        || artifactMime.startsWith('video/');
+
+                    if (mediaModality && textContent.startsWith('data:')) {
+                        validAttachments.push({
+                            type: ConversationUtility.GetAttachmentTypeFromMime(artifactMime),
+                            mimeType: artifactMime,
+                            fileName: artifactVersion.FileName || artifactVersion.Name || undefined,
+                            sizeBytes: artifactVersion.ContentSizeBytes || undefined,
+                            content: textContent,
+                        });
+                        continue;
+                    }
+
+                    const MAX_INLINE_ARTIFACT_CHARS = 10_000;
                     if (textContent.length > MAX_INLINE_ARTIFACT_CHARS) {
                         const preview = ArtifactToolManager.BuildInlinePreview(textContent, 5);
                         validAttachments.push({

package/src/resolvers/__tests__/artifact-routing.test.ts

@@ -0,0 +1,88 @@
+import { describe, it, expect, vi } from 'vitest';
+import { RouteArtifact, type ArtifactRoutingInput } from '../artifact-routing';
+
+const baseInput = (overrides: Partial<ArtifactRoutingInput> = {}): ArtifactRoutingInput => ({
+    typeDefault: 'Inline',
+    forceToolsOnly: false,
+    mimeType: 'image/png',
+    sizeBytes: 5_000,
+    inlineSizeCap: 100 * 1024,
+    modelSupportsModality: () => true,
+    modelName: 'TestModel',
+    artifactTypeName: 'Image',
+    ...overrides,
+});
+
+describe('RouteArtifact', () => {
+    it('routes Inline default + modality supported + under cap to inline', () => {
+        const result = RouteArtifact(baseInput());
+        expect(result).toEqual({ delivery: 'inline' });
+    });
+
+    it('routes ToolsOnly type default to tools', () => {
+        const result = RouteArtifact(baseInput({ typeDefault: 'ToolsOnly' }));
+        expect(result).toEqual({ delivery: 'tools' });
+    });
+
+    it('routes ForceToolsOnly per-instance override to tools regardless of type default', () => {
+        const result = RouteArtifact(baseInput({ forceToolsOnly: true }));
+        expect(result).toEqual({ delivery: 'tools' });
+    });
+
+    it('returns an error when the model lacks modality support for an Inline type', () => {
+        const result = RouteArtifact(baseInput({
+            modelSupportsModality: () => false,
+        }));
+        expect(result.delivery).toBe('error');
+        if (result.delivery !== 'error') return;
+        expect(result.message).toContain('Image');
+        expect(result.message).toContain('TestModel');
+        expect(result.message).toContain('image/png');
+        // The error message lists all three remediation paths.
+        expect(result.message).toMatch(/ToolsOnly/);
+        expect(result.message).toMatch(/ForceToolsOnly/);
+        expect(result.message).toMatch(/switch to a model/);
+    });
+
+    it('falls back to tools with annotation when size exceeds the cap', () => {
+        const result = RouteArtifact(baseInput({
+            sizeBytes: 200 * 1024,
+            inlineSizeCap: 100 * 1024,
+        }));
+        expect(result.delivery).toBe('tools');
+        if (result.delivery !== 'tools') return;
+        expect(result.annotation).toBeDefined();
+        expect(result.annotation).toMatch(/exceeds the inline cap/);
+        expect(result.annotation).toContain('204800');
+        expect(result.annotation).toContain('102400');
+    });
+
+    it('checks ToolsOnly before modality (modality check is irrelevant when type is ToolsOnly)', () => {
+        const modelSupportsModality = vi.fn(() => false);
+        const result = RouteArtifact(baseInput({
+            typeDefault: 'ToolsOnly',
+            modelSupportsModality,
+        }));
+        expect(result).toEqual({ delivery: 'tools' });
+        expect(modelSupportsModality).not.toHaveBeenCalled();
+    });
+
+    it('checks modality before size (modality error wins over size fallback)', () => {
+        const result = RouteArtifact(baseInput({
+            modelSupportsModality: () => false,
+            sizeBytes: 200 * 1024,
+        }));
+        expect(result.delivery).toBe('error');
+    });
+
+    it('ForceToolsOnly bypasses both modality and size checks', () => {
+        const modelSupportsModality = vi.fn(() => false);
+        const result = RouteArtifact(baseInput({
+            forceToolsOnly: true,
+            modelSupportsModality,
+            sizeBytes: 200 * 1024,
+        }));
+        expect(result).toEqual({ delivery: 'tools' });
+        expect(modelSupportsModality).not.toHaveBeenCalled();
+    });
+});

package/src/resolvers/artifact-routing.ts

@@ -0,0 +1,80 @@
+/**
+ * Pure routing function: decides whether an artifact reaches the LLM via an
+ * inline content block (image_url, audio_url, file_url) or via the artifact
+ * tool dispatch path. Has no entity-type or framework dependency — operates
+ * on plain inputs and returns a discriminated result. Lives next to the
+ * resolver but is independently testable.
+ *
+ * See plans/artifact-attachment-unification.md §4 for the contract.
+ */
+
+export type ArtifactDeliveryMode = 'Inline' | 'ToolsOnly';
+
+export interface ArtifactRoutingInput {
+    /** The Artifact Type's DefaultDeliveryMode. */
+    typeDefault: ArtifactDeliveryMode;
+    /** Per-instance opt-out — `true` forces tools regardless of typeDefault. */
+    forceToolsOnly: boolean;
+    /** MIME type of the artifact content (e.g. 'image/png'). */
+    mimeType: string;
+    /** Size of the content in bytes. */
+    sizeBytes: number;
+    /** Maximum inline size in bytes; over this, even Inline-default artifacts go to tools. */
+    inlineSizeCap: number;
+    /** Predicate: does the active model driver support the given MIME modality inline? */
+    modelSupportsModality: (mimeType: string) => boolean;
+    /** Model name used in error messages — never used to make decisions. */
+    modelName: string;
+    /** Artifact type name used in error messages. */
+    artifactTypeName: string;
+}
+
+export type ArtifactRoutingDecision =
+    | { delivery: 'inline' }
+    | { delivery: 'tools'; annotation?: string }
+    | { delivery: 'error'; message: string };
+
+export function RouteArtifact(input: ArtifactRoutingInput): ArtifactRoutingDecision {
+    const {
+        typeDefault,
+        forceToolsOnly,
+        mimeType,
+        sizeBytes,
+        inlineSizeCap,
+        modelSupportsModality,
+        modelName,
+        artifactTypeName,
+    } = input;
+
+    // Path 1: ToolsOnly default or per-instance opt-out — always tools.
+    if (typeDefault === 'ToolsOnly' || forceToolsOnly) {
+        return { delivery: 'tools' };
+    }
+
+    // Path 2: Inline default + modality mismatch — hard error.
+    // The admin / user has paired an Inline-default type with a model that does
+    // not support the modality. There is no defensible runtime fix, so surface
+    // it with a remediable message rather than silently falling back to tools.
+    if (!modelSupportsModality(mimeType)) {
+        return {
+            delivery: 'error',
+            message:
+                `Artifact type "${artifactTypeName}" is configured for Inline delivery but model "${modelName}" does not support modality "${mimeType}". ` +
+                `Either configure the type as ToolsOnly, set ForceToolsOnly on this instance, or switch to a model that supports this modality.`,
+        };
+    }
+
+    // Path 3: Inline default + over size cap — documented, annotated fallback.
+    // Not silent: the manifest entry carries a visible note and the caller is
+    // expected to log at WARN. Both the LLM and the operator can see it
+    // happened, so this isn't the same as the silent-fallback antipattern.
+    if (sizeBytes >= inlineSizeCap) {
+        return {
+            delivery: 'tools',
+            annotation: `Artifact type "${artifactTypeName}" is configured for Inline delivery but content size (${sizeBytes} bytes) exceeds the inline cap (${inlineSizeCap} bytes); delivered via tools instead.`,
+        };
+    }
+
+    // Path 4: Inline, modality supported, under cap — emit inline content block.
+    return { delivery: 'inline' };
+}

package/src/types.ts

@@ -101,6 +101,12 @@ export type RunViewGenericParams = {
   resultType?: string;
   userPayload?: UserPayload;
   aggregates?: AggregateExpression[];
+  /**
+   * When true, the server-side cache layer is bypassed for this view run —
+   * neither the pre-check cache lookup nor the post-query cache write
+   * happens. Propagated to `RunViewParams.BypassCache`.
+   */
+  bypassCache?: boolean;
 };