@juspay/neurolink 10.10.6 → 10.10.8

package/dist/lib/utils/errorHandling.d.ts

@@ -39,6 +39,9 @@ export declare const ERROR_CODES: {
     readonly FILE_PROCESSING_FAILED: "FILE_PROCESSING_FAILED";
     readonly CSV_PROCESSING_FAILED: "CSV_PROCESSING_FAILED";
     readonly PDF_PAGE_LIMIT_EXCEEDED: "PDF_PAGE_LIMIT_EXCEEDED";
+    readonly PDF_AGGREGATE_PAGE_LIMIT_EXCEEDED: "PDF_AGGREGATE_PAGE_LIMIT_EXCEEDED";
+    readonly PDF_AGGREGATE_SIZE_LIMIT_EXCEEDED: "PDF_AGGREGATE_SIZE_LIMIT_EXCEEDED";
+    readonly PDF_PAGE_COUNT_UNVERIFIABLE: "PDF_PAGE_COUNT_UNVERIFIABLE";
     readonly PDF_PASSWORD_REQUIRED: "PDF_PASSWORD_REQUIRED";
     readonly PDF_INCORRECT_PASSWORD: "PDF_INCORRECT_PASSWORD";
     readonly RATE_LIMITER_QUEUE_FULL: "RATE_LIMITER_QUEUE_FULL";
@@ -203,6 +206,24 @@ export declare class ErrorFactory {
      * Create a PDF page limit exceeded error
      */
     static pdfPageLimitExceeded(estimatedPages: number, maxPages: number, provider: string): NeuroLinkError;
+    /**
+     * The combined page count across every PDF in one request exceeds what the
+     * provider accepts (#309). Distinct from `pdfPageLimitExceeded`, which is
+     * per-file — here each document can be individually legal.
+     */
+    static pdfAggregatePageLimitExceeded(fileCount: number, totalPages: number, maxPages: number, provider: string): NeuroLinkError;
+    /**
+     * The combined byte size across every PDF in one request exceeds what the
+     * provider accepts (#309).
+     */
+    static pdfAggregateSizeLimitExceeded(fileCount: number, totalMB: number, maxSizeMB: number, provider: string): NeuroLinkError;
+    /**
+     * A PDF supplied through the untrusted `input.content` surface could not be
+     * parsed for a page count (#309). Caller-supplied `metadata.pages` is not
+     * authoritative there, so an unreadable document is rejected rather than
+     * admitted with an assumed count of zero.
+     */
+    static pdfPageCountUnverifiable(filenames: string[], provider: string): NeuroLinkError;
     /**
      * The PDF is encrypted and no password was supplied (#258).
      */

package/dist/lib/utils/errorHandling.js

@@ -52,6 +52,9 @@ export const ERROR_CODES = {
     CSV_PROCESSING_FAILED: "CSV_PROCESSING_FAILED",
     // PDF validation errors
     PDF_PAGE_LIMIT_EXCEEDED: "PDF_PAGE_LIMIT_EXCEEDED",
+    PDF_AGGREGATE_PAGE_LIMIT_EXCEEDED: "PDF_AGGREGATE_PAGE_LIMIT_EXCEEDED",
+    PDF_AGGREGATE_SIZE_LIMIT_EXCEEDED: "PDF_AGGREGATE_SIZE_LIMIT_EXCEEDED",
+    PDF_PAGE_COUNT_UNVERIFIABLE: "PDF_PAGE_COUNT_UNVERIFIABLE",
     PDF_PASSWORD_REQUIRED: "PDF_PASSWORD_REQUIRED",
     PDF_INCORRECT_PASSWORD: "PDF_INCORRECT_PASSWORD",
     // Rate limiter errors
@@ -557,6 +560,56 @@ export class ErrorFactory {
             },
         });
     }
+    /**
+     * The combined page count across every PDF in one request exceeds what the
+     * provider accepts (#309). Distinct from `pdfPageLimitExceeded`, which is
+     * per-file — here each document can be individually legal.
+     */
+    static pdfAggregatePageLimitExceeded(fileCount, totalPages, maxPages, provider) {
+        return new NeuroLinkError({
+            code: ERROR_CODES.PDF_AGGREGATE_PAGE_LIMIT_EXCEEDED,
+            message: `[PDF] Combined page count across ${fileCount} PDF(s) (${totalPages}) exceeds the ` +
+                `${maxPages}-page limit for ${provider}. ` +
+                `Split the request or reduce the number of PDFs.`,
+            category: ErrorCategory.VALIDATION,
+            severity: ErrorSeverity.MEDIUM,
+            retriable: false,
+            context: { fileCount, totalPages, maxPages, provider },
+        });
+    }
+    /**
+     * The combined byte size across every PDF in one request exceeds what the
+     * provider accepts (#309).
+     */
+    static pdfAggregateSizeLimitExceeded(fileCount, totalMB, maxSizeMB, provider) {
+        return new NeuroLinkError({
+            code: ERROR_CODES.PDF_AGGREGATE_SIZE_LIMIT_EXCEEDED,
+            message: `[PDF] Combined size across ${fileCount} PDF(s) (${totalMB.toFixed(2)}MB) exceeds the ` +
+                `${maxSizeMB}MB limit for ${provider}.`,
+            category: ErrorCategory.VALIDATION,
+            severity: ErrorSeverity.MEDIUM,
+            retriable: false,
+            context: { fileCount, totalMB, maxSizeMB, provider },
+        });
+    }
+    /**
+     * A PDF supplied through the untrusted `input.content` surface could not be
+     * parsed for a page count (#309). Caller-supplied `metadata.pages` is not
+     * authoritative there, so an unreadable document is rejected rather than
+     * admitted with an assumed count of zero.
+     */
+    static pdfPageCountUnverifiable(filenames, provider) {
+        return new NeuroLinkError({
+            code: ERROR_CODES.PDF_PAGE_COUNT_UNVERIFIABLE,
+            message: `[PDF] Cannot verify the page count for ${filenames.length} PDF(s) supplied via ` +
+                `input.content (${filenames.join(", ")}). Provide readable PDFs, or submit them ` +
+                `via input.pdfFiles where page counts are derived during detection.`,
+            category: ErrorCategory.VALIDATION,
+            severity: ErrorSeverity.MEDIUM,
+            retriable: false,
+            context: { filenames, provider },
+        });
+    }
     /**
      * The PDF is encrypted and no password was supplied (#258).
      */

package/dist/lib/utils/fileDetector.js

@@ -344,13 +344,13 @@ export class FileDetector {
                 logger.warn(`[FileDetector] All fallback parsing failed for type "${detection.type}". ` +
                     `Attempted: ${options.allowedTypes.join(", ")}. Falling through to universal handler.`);
                 const csvOptions = options?.csvOptions;
-                const result = await FileDetector.processFile(content, detection, csvOptions, options?.provider);
+                const result = await FileDetector.processFile(content, detection, csvOptions, options?.provider, options?.videoOptions);
                 FileDetector.setFileResultSpanAttributes(span, result, inputFilename, detection.type);
                 return result;
             }
             const content = await FileDetector.loadContent(input, detection, options);
             const csvOptions = options?.csvOptions;
-            const result = await FileDetector.processFile(content, detection, csvOptions, options?.provider);
+            const result = await FileDetector.processFile(content, detection, csvOptions, options?.provider, options?.videoOptions);
             FileDetector.setFileResultSpanAttributes(span, result, inputFilename, detection.type);
             return result;
         });
@@ -967,7 +967,7 @@ export class FileDetector {
     /**
      * Route to appropriate processor
      */
-    static async processFile(content, detection, options, provider) {
+    static async processFile(content, detection, options, provider, videoOptions) {
         switch (detection.type) {
             case "csv":
                 // Pass original extension through to CSV processor; if detection has none,
@@ -985,7 +985,7 @@ export class FileDetector {
                 // AI providers don't support SVG as image format, so we extract text content
                 return await FileDetector.processSvgAsText(content, detection);
             case "video":
-                return await FileDetector.processVideoFile(content, detection);
+                return await FileDetector.processVideoFile(content, detection, videoOptions);
             case "audio":
                 return await FileDetector.processAudioFile(content, detection);
             case "archive":
@@ -1031,7 +1031,7 @@ export class FileDetector {
     /**
      * Process video file: extract metadata, keyframes, and subtitles via VideoProcessor
      */
-    static async processVideoFile(content, detection) {
+    static async processVideoFile(content, detection, videoOptions) {
         const videoFilename = detection.metadata.filename || "video";
         try {
             const videoResult = await (await getVideoProcessor()).processFile({
@@ -1040,7 +1040,10 @@ export class FileDetector {
                 mimetype: detection.mimeType || "video/mp4",
                 size: content.length,
                 buffer: content,
-            });
+            }, 
+            // #478: carry the caller's keyframe budget/quality/format through to
+            // the processor; previously these stopped at the CLI layer.
+            videoOptions);
             if (videoResult.success && videoResult.data) {
                 return {
                     type: "video",

package/dist/lib/utils/messageBuilder.js

@@ -1,6 +1,5 @@
 import { existsSync, readFileSync, statSync } from "fs";
 import { readFile as readFileAsync, stat as statAsync } from "fs/promises";
-import { basename } from "path";
 import { getGlobalDispatcher, interceptors, request } from "undici";
 import { MultimodalLogger, ProviderImageAdapter, } from "../adapters/providerImageAdapter.js";
 import { CONVERSATION_INSTRUCTIONS, STRUCTURED_OUTPUT_INSTRUCTIONS, } from "../config/conversationMemory.js";
@@ -809,6 +808,20 @@ export function mergeMediaFileAliases(input) {
     input.audioFiles = undefined;
     input.videoFiles = undefined;
 }
+/**
+ * #478: `transcribeAudio` (CLI `--transcribe-audio`) is accepted by the options
+ * surface but no video-audio transcription exists yet — VideoProcessor extracts
+ * keyframes and embedded subtitle tracks only, and the transcription step is
+ * still open as #433. Say so once per request rather than letting the caller
+ * believe a transcript was produced and silently omitted.
+ */
+function warnIfVideoTranscriptionRequested(videoOptions) {
+    if (videoOptions?.transcribeAudio) {
+        logger.warn("[NEUROLINK] Video audio transcription was requested but is not implemented yet " +
+            "(tracked as #433). Keyframes and any embedded subtitle tracks are still extracted; " +
+            "spoken audio will not be transcribed.");
+    }
+}
 /**
  * Process the unified files array with auto-detection.
  * Handles lazy file registration, full processing, and preview injection.
@@ -825,6 +838,7 @@ export async function processUnifiedFilesArray(options, maxSize, provider) {
     }
     const totalFiles = options.input.files.length;
     const files = options.input.files;
+    warnIfVideoTranscriptionRequested(options.videoOptions);
     return withSpan({
         name: "neurolink.file.process_all",
         tracer: tracers.file,
@@ -882,6 +896,15 @@ export async function processUnifiedFilesArray(options, maxSize, provider) {
                         "unknown",
                     ],
                     csvOptions: options.csvOptions,
+                    // #478: videos arrive through this unified `files` path, so this is
+                    // where the CLI's frame/quality/format request has to be handed on.
+                    videoOptions: options.videoOptions
+                        ? {
+                            frames: options.videoOptions.frames,
+                            quality: options.videoOptions.quality,
+                            format: options.videoOptions.format,
+                        }
+                        : undefined,
                     provider: provider,
                     mimetypeHint: fileMimetypeHint,
                 });
@@ -1020,6 +1043,83 @@ function enforcePostProcessingBudget(options, provider, model) {
         `budget=${contextWindow.toLocaleString()} tokens, ` +
         `utilization=${contextWindow > 0 ? ((totalContentTokens / contextWindow) * 100).toFixed(1) : "N/A"}%`);
 }
+/**
+ * #309: enforce the provider's page/size ceilings across ALL PDFs in a request,
+ * not just per-file. N files each just under the single-file limit can still
+ * blow past it in aggregate (e.g. three 40-page PDFs → 120 pages for a
+ * 100-page API).
+ *
+ * Shared by both PDF submission surfaces: `input.pdfFiles` (via
+ * `processExplicitPdfFiles`) and `input.content` with `type: "pdf"` (via
+ * `convertContentToProviderFormat`). The latter previously built its own
+ * `pdfFiles` array and reached the provider without ever calling this guard,
+ * so the limit was bypassable by moving the same payload to `input.content`.
+ */
+/**
+ * Basename that strips BOTH separators regardless of host platform.
+ *
+ * `path.basename` only understands the host's separator, so on a POSIX server
+ * a Windows-style filename (`C:\Users\alice\q3-merger.pdf`) comes back
+ * completely unchanged — defeating the point of trimming it before it reaches
+ * a log line, since caller-controlled paths can carry usernames and internal
+ * directory structure.
+ */
+function safeBasename(filename) {
+    const lastSeparator = Math.max(filename.lastIndexOf("/"), filename.lastIndexOf("\\"));
+    const trimmed = lastSeparator === -1 ? filename : filename.slice(lastSeparator + 1);
+    return trimmed || "<unnamed file>";
+}
+async function enforceAggregatePdfLimits(pdfFiles, provider, { trustSuppliedPageCounts }) {
+    const aggregateConfig = PDFProcessor.getProviderConfig(provider);
+    // Only an empty set is exempt. A single PDF must still be checked: on the
+    // `input.content` path nothing else validates it (that path never goes
+    // through FileDetector.detectAndProcess / PDFProcessor.process), so bailing
+    // at length <= 1 let one 200-page document through untouched.
+    if (!aggregateConfig || pdfFiles.length === 0) {
+        return;
+    }
+    // Byte total is free to compute — enforce it BEFORE parsing anything, so an
+    // oversized request is rejected without first spending parser CPU/memory on
+    // every document in it.
+    const totalMB = pdfFiles.reduce((sum, f) => sum + f.buffer.length, 0) / (1024 * 1024);
+    if (totalMB > aggregateConfig.maxSizeMB) {
+        throw ErrorFactory.pdfAggregateSizeLimitExceeded(pdfFiles.length, totalMB, aggregateConfig.maxSizeMB, provider);
+    }
+    // `trustSuppliedPageCounts` is the difference between the two surfaces.
+    // On `input.pdfFiles` the count comes from FileDetector's own detection, so
+    // it is authoritative. On `input.content` it is `metadata.pages` — plain
+    // caller input — and trusting it lets a request declare `pages: 1` for each
+    // of three 40-page PDFs and sail past the ceiling. There, the count is
+    // always re-derived from the bytes and the supplied value is ignored.
+    const pageCounts = await Promise.all(pdfFiles.map(async (f) => trustSuppliedPageCounts && typeof f.pageCount === "number"
+        ? f.pageCount
+        : await PDFProcessor.resolvePageCount(f.buffer)));
+    // Filenames are caller-controlled and may be full paths carrying
+    // PII/internal directory segments — surface only the basename, stripping
+    // both separators so a Windows path is trimmed on a POSIX host too.
+    const unknownFileNames = pdfFiles
+        .filter((_, i) => typeof pageCounts[i] !== "number")
+        .map((f) => (f.filename ? safeBasename(f.filename) : "<unnamed file>"));
+    const totalPages = pageCounts.reduce((sum, p) => sum + (typeof p === "number" ? p : 0), 0);
+    if (unknownFileNames.length > 0) {
+        if (!trustSuppliedPageCounts) {
+            // Untrusted surface: an unreadable count is indistinguishable from an
+            // evasion attempt, and counting it as zero is precisely the hole. Fail
+            // closed rather than admit an unverifiable document.
+            throw ErrorFactory.pdfPageCountUnverifiable(unknownFileNames, provider);
+        }
+        // Trusted surface: detection already vetted these, so one unreadable
+        // count must not fail an otherwise valid request — but it must not be
+        // silent either, since the known sum may undercount the true total.
+        logger.warn(`[PDF] Aggregate page-limit check across ${pdfFiles.length} PDFs could only be ` +
+            `partially verified: ${unknownFileNames.length} file(s) have an unknown ` +
+            `page count (${unknownFileNames.join(", ")}), so the known total (${totalPages}) may ` +
+            `undercount the true combined page count.`);
+    }
+    if (totalPages > aggregateConfig.maxPages) {
+        throw ErrorFactory.pdfAggregatePageLimitExceeded(pdfFiles.length, totalPages, aggregateConfig.maxPages, provider);
+    }
+}
 /**
  * Process explicit PDF files and return structured PDF entries for multimodal processing.
  */
@@ -1059,41 +1159,10 @@ async function processExplicitPdfFiles(options, maxSize, provider) {
             throw error;
         }
     }
-    // #309: enforce the provider's page/size ceilings across ALL PDFs, not just
-    // per-file. N files each just under the single-file limit can still blow past
-    // it in aggregate (e.g. three 40-page PDFs → 120 pages for a 100-page API).
-    const aggregateConfig = PDFProcessor.getProviderConfig(provider);
-    if (aggregateConfig && pdfFiles.length > 1) {
-        // A null pageCount (accurate count unavailable — see
-        // PDFProcessor.getAccuratePageCount) is treated as 0 in the sum below,
-        // which can undercount the aggregate and let a combined request over
-        // the provider's page limit slip through silently. Enforcement still
-        // runs against the known sum — a PDF with an unknown count must not
-        // fail the request outright — but the gap itself must not be silent.
-        const unknownPageCountFiles = pdfFiles.filter((f) => f.pageCount === null || f.pageCount === undefined);
-        const totalPages = pdfFiles.reduce((sum, f) => sum + (f.pageCount ?? 0), 0);
-        const totalMB = pdfFiles.reduce((sum, f) => sum + f.buffer.length, 0) / (1024 * 1024);
-        if (unknownPageCountFiles.length > 0) {
-            // Filenames are caller-controlled and may be full paths carrying
-            // PII/internal directory segments — log only the basename.
-            const unknownFileNames = unknownPageCountFiles
-                .map((f) => (f.filename ? basename(f.filename) : "<unnamed file>"))
-                .join(", ");
-            logger.warn(`[PDF] Aggregate page-limit check across ${pdfFiles.length} PDFs could only be ` +
-                `partially verified: ${unknownPageCountFiles.length} file(s) have an unknown ` +
-                `page count (${unknownFileNames}), so the known total (${totalPages}) may ` +
-                `undercount the true combined page count.`);
-        }
-        if (totalPages > aggregateConfig.maxPages) {
-            throw new Error(`[PDF] Combined page count across ${pdfFiles.length} PDFs (${totalPages}) exceeds the ` +
-                `${aggregateConfig.maxPages}-page limit for ${provider}. ` +
-                `Split the request or reduce the number of PDFs.`);
-        }
-        if (totalMB > aggregateConfig.maxSizeMB) {
-            throw new Error(`[PDF] Combined size across ${pdfFiles.length} PDFs (${totalMB.toFixed(2)}MB) exceeds the ` +
-                `${aggregateConfig.maxSizeMB}MB limit for ${provider}.`);
-        }
-    }
+    // Counts here come from FileDetector's detection, so they are authoritative.
+    await enforceAggregatePdfLimits(pdfFiles, provider, {
+        trustSuppliedPageCounts: true,
+    });
     return pdfFiles;
 }
 /**
@@ -1390,6 +1459,12 @@ async function convertContentToProviderFormat(content, provider, _model, pdfOpti
         password: pdfOptions?.password,
         maxCanvasPixels: pdfOptions?.maxCanvasPixels,
     }));
+    // #309: same aggregate ceiling as `input.pdfFiles`. Without this, moving an
+    // over-limit payload from `input.pdfFiles` to `input.content` skipped the
+    // check entirely and the request went straight to the provider.
+    await enforceAggregatePdfLimits(pdfFiles, provider, {
+        trustSuppliedPageCounts: false,
+    });
     return await convertMultimodalToProviderFormat(text, images, pdfFiles, provider, _model);
 }
 /**

package/dist/lib/utils/pdfProcessor.d.ts

@@ -18,6 +18,17 @@ export declare class PDFProcessor {
      */
     static supportsNativePDF(provider: string): boolean;
     static getProviderConfig(provider: string): PDFProviderConfig | null;
+    /**
+     * Best-effort page count for a PDF whose count the caller did not supply
+     * (#309). Mirrors what `process()` derives for the `input.pdfFiles` path:
+     * the accurate pdfjs count when the document parses, otherwise the header
+     * regex estimate. Returns null when neither can determine a count.
+     *
+     * Exists so the aggregate page-limit guard can enforce against PDFs handed
+     * in via `input.content`, where `metadata.pages` is optional and routinely
+     * omitted — without it, an absent count silently sums as zero.
+     */
+    static resolvePageCount(buffer: Buffer): Promise<number | null>;
     private static isValidPDF;
     private static extractBasicMetadata;
     /**

package/dist/lib/utils/pdfProcessor.js

@@ -212,6 +212,23 @@ export class PDFProcessor {
     static getProviderConfig(provider) {
         return PDF_PROVIDER_CONFIGS[provider] || null;
     }
+    /**
+     * Best-effort page count for a PDF whose count the caller did not supply
+     * (#309). Mirrors what `process()` derives for the `input.pdfFiles` path:
+     * the accurate pdfjs count when the document parses, otherwise the header
+     * regex estimate. Returns null when neither can determine a count.
+     *
+     * Exists so the aggregate page-limit guard can enforce against PDFs handed
+     * in via `input.content`, where `metadata.pages` is optional and routinely
+     * omitted — without it, an absent count silently sums as zero.
+     */
+    static async resolvePageCount(buffer) {
+        const accurate = await PDFProcessor.getAccuratePageCount(buffer);
+        if (accurate !== null) {
+            return accurate;
+        }
+        return PDFProcessor.extractBasicMetadata(buffer).estimatedPages;
+    }
     static isValidPDF(buffer) {
         if (buffer.length < 5) {
             return false;

package/dist/lib/utils/redis.d.ts

@@ -2,7 +2,7 @@
  * Redis Utilities for NeuroLink
  * Helper functions for Redis storage operations
  */
-import type { RedisClient, RedisConversationObject, RedisStorageConfig } from "../types/index.js";
+import type { ChatMessage, RedisClient, RedisConversationObject, RedisStorageConfig } from "../types/index.js";
 /**
  * Get a pooled Redis connection. Multiple callers with the same host:port:db
  * share a single connection, reducing connection count.
@@ -36,6 +36,28 @@ export declare function getUserSessionsKey(config: Required<RedisStorageConfig>,
  * Serializes conversation object for Redis storage
  */
 export declare function serializeConversation(conversation: RedisConversationObject): string;
+/**
+ * True for a complete `ChatMessage` — `id` included, because callers keying
+ * summary pointers and condensation groups off it are entitled to find one.
+ */
+export declare function isStoredChatMessage(value: unknown): value is ChatMessage;
+/**
+ * Coerce a stored entry into a complete `ChatMessage`, or `undefined` when its
+ * shape is unusable.
+ *
+ * Shared by BOTH read paths on purpose. Inline blobs have always been
+ * validated; once messages moved into the companion LIST the split read
+ * bypassed that check, so `null`, a number or a bare string survived
+ * `JSON.parse` and reached callers that dereference `.role`, `.content` and
+ * `.metadata`. The two storage formats must offer the same read guarantee.
+ *
+ * `id` is a separate matter: it is required on `ChatMessage`, but no read path
+ * has ever enforced it, so history written before it existed carries none.
+ * Rejecting those records would empty otherwise-healthy sessions, so a missing
+ * id is backfilled rather than fatal — prefixed, so a synthesized id is never
+ * mistaken for one an existing pointer could reference.
+ */
+export declare function normalizeStoredMessage(value: unknown): ChatMessage | undefined;
 /**
  * Deserializes conversation object from Redis storage
  */
@@ -58,3 +80,40 @@ export declare function scanKeys(client: RedisClient, pattern: string, batchSize
  * Get normalized Redis configuration with defaults
  */
 export declare function getNormalizedConfig(config: RedisStorageConfig): Required<RedisStorageConfig>;
+/**
+ * Suffix of the companion LIST key holding a session's messages.
+ *
+ * Every `storeConversationTurn` used to re-serialize and SET the ENTIRE
+ * conversation, so per-turn write cost grew with history size. Measured
+ * against local Redis: 200 turns of 2KB messages stayed flat at 2ms, but 400
+ * turns of 20KB messages (~16MB blob) went 2ms -> 61ms, a 30x degradation on
+ * exactly the agentic tool-output profile. Splitting messages into an
+ * append-only LIST makes the per-turn write O(1) in conversation size.
+ */
+export declare const MESSAGES_KEY_SUFFIX = ":msgs";
+/**
+ * Marker on a stored blob meaning "messages live in the companion LIST".
+ * A blob WITHOUT it is a legacy record whose inline `messages` array is
+ * authoritative — it is read as-is and converted on its next write, which is
+ * what makes this migration backward compatible.
+ */
+export declare const MESSAGES_IN_LIST_MARKER = "__nlMessagesInList";
+/** Redis key holding a session's messages as an append-only LIST. */
+export declare function getSessionMessagesKey(config: Required<RedisStorageConfig>, sessionId: string, userId?: string): string;
+/**
+ * True for a companion messages LIST key.
+ *
+ * Scan-then-GET paths (`getStats`, session listing) match `${keyPrefix}*`, so
+ * they now also see these LIST keys — and `GET` on a LIST raises WRONGTYPE.
+ * They must filter with this, exactly as they already skip `:sessions` index
+ * keys.
+ */
+export declare function isSessionMessagesKey(key: string): boolean;
+/** Serialize the conversation WITHOUT its messages, flagged for split reads. */
+export declare function serializeConversationMetadata(conversation: RedisConversationObject): string;
+/** True when a deserialized blob's messages live in the companion LIST. */
+export declare function usesSplitMessageStorage(conversation: RedisConversationObject | null | undefined): boolean;
+/** Parse LRANGE entries, skipping any single entry that is not usable. */
+export declare function parseStoredMessages(entries: string[]): ChatMessage[];
+/** Encode messages for RPUSH. */
+export declare function encodeStoredMessages(messages: ChatMessage[]): string[];

package/dist/lib/utils/redis.js

@@ -2,6 +2,7 @@
  * Redis Utilities for NeuroLink
  * Helper functions for Redis storage operations
  */
+import { randomUUID } from "crypto";
 import { createClient } from "redis";
 import { logger } from "./logger.js";
 const SESSION_ONLY_PREFIX = "session-only:";
@@ -198,6 +199,65 @@ export function serializeConversation(conversation) {
         throw error;
     }
 }
+/** The exact role set `ChatMessage` allows. */
+const STORED_MESSAGE_ROLES = new Set([
+    "user",
+    "assistant",
+    "system",
+    "tool_call",
+    "tool_result",
+]);
+function isStoredMessageRole(role) {
+    return typeof role === "string" && STORED_MESSAGE_ROLES.has(role);
+}
+/**
+ * True for a complete `ChatMessage` — `id` included, because callers keying
+ * summary pointers and condensation groups off it are entitled to find one.
+ */
+export function isStoredChatMessage(value) {
+    if (typeof value !== "object" || value === null) {
+        return false;
+    }
+    const candidate = value;
+    return (typeof candidate.id === "string" &&
+        typeof candidate.content === "string" &&
+        isStoredMessageRole(candidate.role));
+}
+/**
+ * Coerce a stored entry into a complete `ChatMessage`, or `undefined` when its
+ * shape is unusable.
+ *
+ * Shared by BOTH read paths on purpose. Inline blobs have always been
+ * validated; once messages moved into the companion LIST the split read
+ * bypassed that check, so `null`, a number or a bare string survived
+ * `JSON.parse` and reached callers that dereference `.role`, `.content` and
+ * `.metadata`. The two storage formats must offer the same read guarantee.
+ *
+ * `id` is a separate matter: it is required on `ChatMessage`, but no read path
+ * has ever enforced it, so history written before it existed carries none.
+ * Rejecting those records would empty otherwise-healthy sessions, so a missing
+ * id is backfilled rather than fatal — prefixed, so a synthesized id is never
+ * mistaken for one an existing pointer could reference.
+ */
+export function normalizeStoredMessage(value) {
+    if (isStoredChatMessage(value)) {
+        return value;
+    }
+    if (typeof value !== "object" || value === null) {
+        return undefined;
+    }
+    const candidate = value;
+    if (typeof candidate.content !== "string" ||
+        !isStoredMessageRole(candidate.role)) {
+        return undefined;
+    }
+    return {
+        ...candidate,
+        id: `legacy-${randomUUID()}`,
+        role: candidate.role,
+        content: candidate.content,
+    };
+}
 /**
  * Deserializes conversation object from Redis storage
  */
@@ -235,18 +295,8 @@ export function deserializeConversation(data) {
             return null;
         }
         // Validate each message in the messages array
-        const isValidHistory = conversation.messages.every((m) => typeof m === "object" &&
-            m !== null &&
-            "role" in m &&
-            "content" in m &&
-            typeof m.role === "string" &&
-            typeof m.content === "string" &&
-            (m.role === "user" ||
-                m.role === "assistant" ||
-                m.role === "system" ||
-                m.role === "tool_call" ||
-                m.role === "tool_result"));
-        if (!isValidHistory) {
+        const normalizedMessages = conversation.messages.map(normalizeStoredMessage);
+        if (normalizedMessages.some((message) => message === undefined)) {
             logger.warn("[redisUtils] Invalid messages structure", {
                 messageCount: conversation.messages.length,
                 firstMessage: conversation.messages.length > 0
@@ -255,6 +305,7 @@ export function deserializeConversation(data) {
             });
             return null;
         }
+        conversation.messages = normalizedMessages.filter((message) => message !== undefined);
         logger.debug("[redisUtils] Conversation deserialized successfully", {
             sessionId: conversation.sessionId,
             userId: conversation.userId,
@@ -396,4 +447,84 @@ export function getNormalizedConfig(config) {
         },
     };
 }
+// ---------------------------------------------------------------------------
+// Split message storage
+// ---------------------------------------------------------------------------
+/**
+ * Suffix of the companion LIST key holding a session's messages.
+ *
+ * Every `storeConversationTurn` used to re-serialize and SET the ENTIRE
+ * conversation, so per-turn write cost grew with history size. Measured
+ * against local Redis: 200 turns of 2KB messages stayed flat at 2ms, but 400
+ * turns of 20KB messages (~16MB blob) went 2ms -> 61ms, a 30x degradation on
+ * exactly the agentic tool-output profile. Splitting messages into an
+ * append-only LIST makes the per-turn write O(1) in conversation size.
+ */
+export const MESSAGES_KEY_SUFFIX = ":msgs";
+/**
+ * Marker on a stored blob meaning "messages live in the companion LIST".
+ * A blob WITHOUT it is a legacy record whose inline `messages` array is
+ * authoritative — it is read as-is and converted on its next write, which is
+ * what makes this migration backward compatible.
+ */
+export const MESSAGES_IN_LIST_MARKER = "__nlMessagesInList";
+/** Redis key holding a session's messages as an append-only LIST. */
+export function getSessionMessagesKey(config, sessionId, userId) {
+    return `${getSessionKey(config, sessionId, userId)}${MESSAGES_KEY_SUFFIX}`;
+}
+/**
+ * True for a companion messages LIST key.
+ *
+ * Scan-then-GET paths (`getStats`, session listing) match `${keyPrefix}*`, so
+ * they now also see these LIST keys — and `GET` on a LIST raises WRONGTYPE.
+ * They must filter with this, exactly as they already skip `:sessions` index
+ * keys.
+ */
+export function isSessionMessagesKey(key) {
+    return key.endsWith(MESSAGES_KEY_SUFFIX);
+}
+/** Serialize the conversation WITHOUT its messages, flagged for split reads. */
+export function serializeConversationMetadata(conversation) {
+    return JSON.stringify({
+        ...conversation,
+        messages: [],
+        [MESSAGES_IN_LIST_MARKER]: true,
+    });
+}
+/** True when a deserialized blob's messages live in the companion LIST. */
+export function usesSplitMessageStorage(conversation) {
+    if (!conversation) {
+        return false;
+    }
+    const record = conversation;
+    return record[MESSAGES_IN_LIST_MARKER] === true;
+}
+/** Parse LRANGE entries, skipping any single entry that is not usable. */
+export function parseStoredMessages(entries) {
+    const messages = [];
+    for (const entry of entries) {
+        try {
+            // `JSON.parse` succeeds for `null`, numbers and bare strings, so the
+            // catch below cannot filter them — the shape has to be checked.
+            const parsed = JSON.parse(entry);
+            const message = normalizeStoredMessage(parsed);
+            if (!message) {
+                logger.warn("[redisUtils] Skipping stored message with invalid shape");
+                continue;
+            }
+            messages.push(message);
+        }
+        catch (error) {
+            // One corrupt entry must not destroy a whole session's history.
+            logger.warn("[redisUtils] Skipping unparseable stored message", {
+                error: error instanceof Error ? error.message : String(error),
+            });
+        }
+    }
+    return messages;
+}
+/** Encode messages for RPUSH. */
+export function encodeStoredMessages(messages) {
+    return messages.map((message) => JSON.stringify(message));
+}
 //# sourceMappingURL=redis.js.map