@mxf-dev/core 3.1.0 → 3.2.0

package/src/config/MeilisearchIngressLimits.ts

@@ -0,0 +1,63 @@
+/**
+ * Copyright 2024 Brad Anderson
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * @author Brad Anderson <BradA1878@pm.me>
+ * @repository https://github.com/BradA1878/model-exchange-framework
+ * @documentation https://mxf-dev.github.io/mxf/
+ */
+
+/**
+ * Size limits on the conversation search index's socket ingress.
+ *
+ * The server enforces these in MeilisearchIngressPolicy on every
+ * `meilisearch:index:request` and `meilisearch:backfill:request`. The SDK
+ * builds its memory-load backfill batches from the same numbers, so a batch
+ * it sends is one the server accepts. Before this module existed the SDK
+ * batched by count alone while the server rejected by bytes, and an agent
+ * whose stored history was dense could never load it again: the same batch
+ * was refused on every connect.
+ */
+
+/**
+ * Largest single message content the index accepts, in UTF-8 bytes. A
+ * larger message is not indexed: the live path drops it after the server
+ * refuses it, and the backfill skips it without sending.
+ */
+export const MAX_MEILISEARCH_MESSAGE_BYTES = 64 * 1024;
+
+/** Most messages one backfill request may carry. */
+export const MAX_MEILISEARCH_BACKFILL_MESSAGES = 50;
+
+/**
+ * Most message content one backfill request may carry, as the sum of each
+ * message's UTF-8 bytes.
+ */
+export const MAX_MEILISEARCH_BACKFILL_CONTENT_BYTES = 512 * 1024;
+
+/**
+ * Most bytes one backfill request event may occupy once serialized for the
+ * socket, envelope included. JSON escaping makes a message's wire size larger
+ * than its content bytes (a newline or quote is two bytes on the wire), so
+ * the content limit alone does not bound the frame. Engine.IO closes a socket
+ * that sends a frame above the server's message limit
+ * (MXF_SOCKET_MAX_HTTP_BUFFER_BYTES, default 1 MiB), which would put the
+ * agent in a reconnect loop instead of a policy rejection; the server refuses
+ * to boot with a message limit this budget does not fit under.
+ */
+export const MAX_MEILISEARCH_BACKFILL_WIRE_BYTES = 768 * 1024;
+
+/** UTF-8 byte length of message content — the measure every limit above uses. */
+export const meilisearchContentBytes = (content: string): number =>
+    Buffer.byteLength(content, 'utf8');

package/src/events/event-definitions/MeilisearchEvents.ts

@@ -23,7 +23,7 @@
  * Events for server-side indexing with embeddings
  */
 
-import { BaseEventPayload } from '../../schemas/EventPayloadSchema.js';
+import { BaseEventPayload, MeilisearchBackfillEventData } from '../../schemas/EventPayloadSchema.js';
 
 /**
  * Meilisearch event names
@@ -39,6 +39,11 @@ export const MeilisearchEvents = {
     BACKFILL_COMPLETE: 'meilisearch:backfill:complete',
     BACKFILL_PARTIAL: 'meilisearch:backfill:partial',
     BACKFILL_ERROR: 'meilisearch:backfill:error',
+
+    // Report from the SDK to the server: its memory-load backfill has settled
+    // (every batch answered or abandoned). Carries the final counts so the
+    // server can mark the agent ready for search tools and log the outcome.
+    BACKFILL_SETTLED: 'meilisearch:backfill:settled',
 } as const;
 
 /**
@@ -52,4 +57,5 @@ export interface MeilisearchPayloads {
     'meilisearch:backfill:complete': BaseEventPayload<any>;
     'meilisearch:backfill:partial': BaseEventPayload<any>;
     'meilisearch:backfill:error': BaseEventPayload<any>;
+    'meilisearch:backfill:settled': BaseEventPayload<MeilisearchBackfillEventData>;
 }

package/src/index.ts

@@ -30,3 +30,10 @@ export {
     buildServerUrl,
 } from './config/ServerConfig.js';
 export type { ServerConfig } from './config/ServerConfig.js';
+export {
+    MAX_MEILISEARCH_MESSAGE_BYTES,
+    MAX_MEILISEARCH_BACKFILL_MESSAGES,
+    MAX_MEILISEARCH_BACKFILL_CONTENT_BYTES,
+    MAX_MEILISEARCH_BACKFILL_WIRE_BYTES,
+    meilisearchContentBytes,
+} from './config/MeilisearchIngressLimits.js';

package/src/models/task.ts

@@ -90,7 +90,11 @@ export interface ITask extends Document {
     dependsOn?: string[];
     blockedBy?: string[];
     
-    // Results and outcomes - MongoDB version with Date objects
+    // Results and outcomes - MongoDB version with Date objects. `output` is
+    // whatever the caller passed to completeTask() or the REST completion
+    // route, or a TaskCompletionOutput (see types/TaskTypes.ts) when an agent
+    // completed the task by calling task_complete. There is no `result.summary`
+    // - use getTaskCompletionOutput() to read the summary safely.
     result?: {
         success?: boolean;
         output?: any;

package/src/schemas/EventPayloadSchema.ts

@@ -3226,6 +3226,7 @@ export interface MeilisearchBackfillEventData {
     totalDocuments: number;       // Total documents to backfill
     indexedDocuments: number;     // Number of documents successfully indexed
     failedDocuments: number;      // Number of documents that failed to index
+    skippedDocuments?: number;    // Documents the SDK never sent because they exceed the per-message limit
     duration: number;             // Total time taken in milliseconds
     success: boolean;             // Whether backfill completed successfully
     source: 'mongodb' | 'memory' | 'other'; // Source of backfilled data
@@ -3302,6 +3303,9 @@ export function createMeilisearchBackfillEventPayload(
     validator.assertIsNumber(data.totalDocuments, 'totalDocuments must be a number');
     validator.assertIsNumber(data.indexedDocuments, 'indexedDocuments must be a number');
     validator.assertIsNumber(data.failedDocuments, 'failedDocuments must be a number');
+    if (data.skippedDocuments !== undefined) {
+        validator.assertIsNumber(data.skippedDocuments, 'skippedDocuments must be a number');
+    }
     validator.assertIsNumber(data.duration, 'duration must be a number');
     validator.assertIsBoolean(data.success);
     validator.assertIsNonEmptyString(data.source, 'source');

package/src/services/MxfMeilisearchService.ts

@@ -112,6 +112,11 @@ export interface SearchResult<T> {
  */
 export type EmbeddingGenerator = (text: string, options?: { model?: string; dimensions?: number }) => Promise<number[]>;
 
+/** The part of a Meilisearch enqueued-task promise the index methods use. */
+interface IndexTaskPromise {
+    waitTask: () => Promise<{ uid: number; status: string; error: { message?: string } | null }>;
+}
+
 /**
  * Meilisearch Service Configuration
  */
@@ -364,8 +369,13 @@ export class MxfMeilisearchService {
     }
 
     /**
-     * Generate embedding for text using provided embedding generator
-     * Respects SYSTEMLLM_PROVIDER configuration from server
+     * Embed text with the generator the server installed.
+     *
+     * Returns undefined only when embeddings are off or no generator is
+     * installed (keyword-only mode). A generator failure is thrown with the
+     * provider's reason: a document indexed without the vector its caller
+     * expects would still be counted as indexed, and semantic searches would
+     * quietly miss it, so the failure has to reach the caller.
      */
     private async generateEmbedding(text: string): Promise<number[] | undefined> {
         if (!this.config.enableEmbeddings || !this.embeddingGenerator) {
@@ -373,26 +383,42 @@ export class MxfMeilisearchService {
         }
 
         try {
-            // Call the embedding generator function (provided by server)
-            const embedding = await this.embeddingGenerator(text, {
+            return await this.embeddingGenerator(text, {
                 model: this.config.embeddingModel,
                 dimensions: this.config.embeddingDimensions
             });
-
-            return embedding;
         } catch (error) {
-            //this.logger.error(`Embedding generation failed:`, error);
-            return undefined;
+            const reason = error instanceof Error ? error.message : String(error);
+            throw new Error(`Embedding generation failed (${this.config.embeddingModel}): ${reason}`);
         }
     }
 
     /**
-     * Index a conversation message
+     * Wait for an indexing task and throw when Meilisearch did not complete
+     * it. waitTask() resolves for a task that ended `failed` or `canceled` —
+     * the outcome is on the task, not in the promise.
+     */
+    private async awaitIndexTask(taskPromise: IndexTaskPromise, what: string): Promise<void> {
+        const task = await taskPromise.waitTask();
+        if (task.status !== 'succeeded') {
+            throw new Error(
+                `Meilisearch ${what} task ${task.uid} ${task.status}: ${task.error?.message ?? 'no error detail'}`
+            );
+        }
+    }
+
+    /**
+     * Index a conversation message.
+     *
+     * Throws when the embedding cannot be generated, the document cannot be
+     * enqueued, or Meilisearch fails the task. The caller decides what a
+     * missing document means: the server reports it to the SDK, and the SDK
+     * counts it against the backfill or drops it from the live index queue.
      */
     public async indexConversation(message: ConversationMessage): Promise<void> {
         try {
             const embedding = await this.generateEmbedding(message.content);
-            
+
             // Build document with proper _vectors format for Meilisearch
             const document: ConversationDocument = {
                 id: message.id,
@@ -407,14 +433,11 @@ export class MxfMeilisearchService {
             };
 
             const index = this.client.index(MeilisearchIndex.CONVERSATIONS);
-            const taskPromise = index.addDocuments([document]);
-
-            // Wait for indexing task to complete so documents are immediately searchable
-            await taskPromise.waitTask();
-
+            // Wait for the task so the document is searchable when this resolves.
+            await this.awaitIndexTask(index.addDocuments([document]), `conversation ${message.id}`);
         } catch (error) {
-            this.logger.error('Failed to index conversation', error);
-            // Don't throw - indexing failures shouldn't break the main flow
+            this.logger.error(`Failed to index conversation ${message.id}`, error);
+            throw error;
         }
     }
 
@@ -444,13 +467,10 @@ export class MxfMeilisearchService {
             };
 
             const index = this.client.index(MeilisearchIndex.ACTIONS);
-            const taskPromise = index.addDocuments([document]);
-
-            // Wait for indexing task to complete
-            await taskPromise.waitTask();
-
+            await this.awaitIndexTask(index.addDocuments([document]), `action ${action.id}`);
         } catch (error) {
-            this.logger.error('Failed to index action', error);
+            this.logger.error(`Failed to index action ${action.id}`, error);
+            throw error;
         }
     }
 
@@ -480,13 +500,10 @@ export class MxfMeilisearchService {
             };
 
             const index = this.client.index(MeilisearchIndex.PATTERNS);
-            const taskPromise = index.addDocuments([document]);
-
-            // Wait for indexing task to complete
-            await taskPromise.waitTask();
-
+            await this.awaitIndexTask(index.addDocuments([document]), `pattern ${pattern.patternId}`);
         } catch (error) {
-            this.logger.error('Failed to index pattern', error);
+            this.logger.error(`Failed to index pattern ${pattern.patternId}`, error);
+            throw error;
         }
     }
 
@@ -535,21 +552,16 @@ export class MxfMeilisearchService {
                 searchParams.attributesToHighlight = params.attributesToHighlight;
             }
 
-            // Enable hybrid search if embeddings are available, generator is configured, and ratio is set
-            // For user-provided embeddings, we must generate and pass the query embedding
+            // Hybrid search needs the query embedded by the same generator that
+            // embedded the documents. A generator failure is thrown (through the
+            // catch below) rather than quietly answering a semantic request with
+            // keyword-only results.
             if (this.config.enableEmbeddings && this.embeddingGenerator && params.hybridRatio !== undefined) {
-                // Generate embedding for the search query
-                const queryEmbedding = await this.generateEmbedding(params.query);
-
-                if (queryEmbedding) {
-                    searchParams.hybrid = {
-                        semanticRatio: params.hybridRatio,
-                        embedder: 'default'
-                    };
-                    // Provide the query embedding for user-provided embedder configuration
-                    searchParams.vector = queryEmbedding;
-                }
-                // If embedding generation fails, fall back to keyword-only search
+                searchParams.hybrid = {
+                    semanticRatio: params.hybridRatio,
+                    embedder: 'default'
+                };
+                searchParams.vector = await this.generateEmbedding(params.query);
             }
 
             const result = await index.search<T>(params.query, searchParams);

package/src/types/TaskTypes.ts

@@ -109,7 +109,11 @@ export interface ChannelTask {
     dependsOn?: string[];               // Task IDs this task depends on
     blockedBy?: string[];               // Task IDs that block this task
     
-    // Results and outcomes
+    // Results and outcomes. `output` is whatever the caller passed to
+    // completeTask() or the REST completion route, or a TaskCompletionOutput
+    // when an agent completed the task by calling task_complete. There is no
+    // `result.summary` — use getTaskCompletionOutput() to read the summary
+    // safely regardless of which completion path produced this task.
     result?: {
         success?: boolean;
         output?: any;
@@ -117,10 +121,73 @@ export interface ChannelTask {
         completedAt?: number;
         completedBy?: string;
     };
-    
+
     assignmentStrategy: AssignmentStrategy;
 }
 
+/**
+ * What `task_complete` stores in `ChannelTask.result.output`.
+ *
+ * This is the shape TaskService.handleTaskCompletion writes when an agent
+ * completes a task through the task_complete tool. It is not the only shape
+ * `result.output` can hold: a caller can also set `result.output` directly
+ * through completeTask() or the REST completion route, so this type is a
+ * guard to check against, not a blanket narrowing of `output`.
+ */
+export interface TaskCompletionOutput {
+    agentId: string;
+    summary: string;
+    details?: Record<string, unknown>;
+    nextSteps?: string;
+    reportedSuccess: boolean;
+    requestId: string;
+}
+
+/**
+ * Checks whether a value matches the TaskCompletionOutput shape.
+ * Used to read `result.output` from a task without assuming which
+ * completion path (task_complete vs. completeTask()/REST) produced it.
+ */
+export function isTaskCompletionOutput(value: unknown): value is TaskCompletionOutput {
+    if (typeof value !== 'object' || value === null) {
+        return false;
+    }
+    const candidate = value as Record<string, unknown>;
+    if (typeof candidate.agentId !== 'string' || candidate.agentId.length === 0) {
+        return false;
+    }
+    if (typeof candidate.summary !== 'string') {
+        return false;
+    }
+    if (typeof candidate.reportedSuccess !== 'boolean') {
+        return false;
+    }
+    if (typeof candidate.requestId !== 'string' || candidate.requestId.length === 0) {
+        return false;
+    }
+    if (candidate.details !== undefined) {
+        if (typeof candidate.details !== 'object' || candidate.details === null || Array.isArray(candidate.details)) {
+            return false;
+        }
+    }
+    if (candidate.nextSteps !== undefined && typeof candidate.nextSteps !== 'string') {
+        return false;
+    }
+    return true;
+}
+
+/**
+ * Reads `task.result.output` as a TaskCompletionOutput, or returns undefined
+ * when the task has no result, no output, or an output that does not match
+ * (e.g. a value a caller passed directly to completeTask()/REST).
+ */
+export function getTaskCompletionOutput(
+    task: Pick<ChannelTask, 'result'> | null | undefined
+): TaskCompletionOutput | undefined {
+    const output = task?.result?.output;
+    return isTaskCompletionOutput(output) ? output : undefined;
+}
+
 /**
  * Task creation request
  */