convex-cms 0.0.1

package/dist/component/lib/ragContentChunker.js

@@ -0,0 +1,897 @@
+/**
+ * RAG Content Chunker
+ *
+ * Utility to extract and structure content from CMS entries for @convex-dev/rag indexing.
+ * This module provides:
+ *
+ * 1. **Content Extraction**: Extracts text from various CMS field types (text, richText, json, etc.)
+ * 2. **Semantic Chunking**: Splits content into meaningful chunks optimized for embedding
+ * 3. **Metadata Tagging**: Attaches relevant metadata (content type, field source, locale, etc.)
+ * 4. **Reference Handling**: Processes embedded references and includes contextual information
+ *
+ * The output is designed to be directly compatible with @convex-dev/rag's `add()` function.
+ *
+ * @example
+ * ```typescript
+ * import { extractContentForRag, chunkContentEntry } from "@convex-cms/core/lib";
+ * import { rag } from "@convex-dev/rag";
+ *
+ * // In a Convex action
+ * const chunks = await chunkContentEntry(ctx, entry, contentType, {
+ *   includeMetadata: true,
+ *   chunkOptions: { maxCharsSoftLimit: 1000 },
+ * });
+ *
+ * await rag.add(ctx, {
+ *   namespace: "cms-content",
+ *   key: entry._id,
+ *   chunks: chunks.map(c => c.text),
+ *   title: chunks[0]?.metadata?.title,
+ * });
+ * ```
+ *
+ * @module
+ */
+// =============================================================================
+// Default Configuration
+// =============================================================================
+const DEFAULT_CHUNK_OPTIONS = {
+    minLines: 1,
+    minCharsSoftLimit: 100,
+    maxCharsSoftLimit: 1000,
+    maxCharsHardLimit: 4000,
+    delimiter: "\n\n",
+    fallbackDelimiters: ["\n", ". ", ", "],
+    preserveHeadingContext: true,
+    overlapChars: 50,
+};
+const DEFAULT_EXTRACTION_OPTIONS = {
+    includeMetadata: true,
+    includeFields: [],
+    excludeFields: [],
+    extractRichText: true,
+    extractJson: true,
+    includeReferenceContext: true,
+    chunkOptions: DEFAULT_CHUNK_OPTIONS,
+    chunkPrefix: "",
+    chunkSuffix: "",
+    createSummaryChunk: false,
+    summaryFields: [],
+};
+// =============================================================================
+// Text Extraction Functions
+// =============================================================================
+/**
+ * Extracts plain text from a rich text field value.
+ *
+ * Handles common rich text formats:
+ * - HTML strings (strips tags)
+ * - ProseMirror/Tiptap JSON structure
+ * - Markdown strings
+ * - Plain text strings
+ *
+ * @param value - The rich text field value
+ * @returns Plain text content
+ */
+export function extractTextFromRichText(value) {
+    if (value === null || value === undefined) {
+        return "";
+    }
+    // Handle string values (HTML, Markdown, or plain text)
+    if (typeof value === "string") {
+        return stripHtmlTags(value);
+    }
+    // Handle ProseMirror/Tiptap JSON structure
+    if (typeof value === "object" && value !== null) {
+        const obj = value;
+        // Check for ProseMirror doc structure
+        if (obj.type === "doc" && Array.isArray(obj.content)) {
+            return extractTextFromProseMirrorDoc(obj);
+        }
+        // Check for array of blocks
+        if (Array.isArray(value)) {
+            return value.map((block) => extractTextFromRichText(block)).join("\n\n");
+        }
+        // Generic object - try to find text content
+        if ("text" in obj && typeof obj.text === "string") {
+            return obj.text;
+        }
+        if ("content" in obj && typeof obj.content === "string") {
+            return obj.content;
+        }
+    }
+    return "";
+}
+/**
+ * Extracts text from a ProseMirror document structure.
+ */
+function extractTextFromProseMirrorDoc(doc) {
+    const content = doc.content;
+    if (!Array.isArray(content)) {
+        return "";
+    }
+    const textParts = [];
+    for (const node of content) {
+        if (typeof node !== "object" || node === null)
+            continue;
+        const nodeObj = node;
+        const nodeType = nodeObj.type;
+        switch (nodeType) {
+            case "paragraph":
+            case "heading":
+                textParts.push(extractTextFromProseMirrorNode(nodeObj));
+                break;
+            case "bulletList":
+            case "orderedList":
+                textParts.push(extractTextFromProseMirrorList(nodeObj));
+                break;
+            case "blockquote": {
+                const quoteText = extractTextFromProseMirrorDoc(nodeObj);
+                textParts.push(`"${quoteText}"`);
+                break;
+            }
+            case "codeBlock":
+                if (nodeObj.content && Array.isArray(nodeObj.content)) {
+                    const codeText = nodeObj.content
+                        .map((c) => c.text || "")
+                        .join("");
+                    textParts.push(codeText);
+                }
+                break;
+            case "horizontalRule":
+                // Skip horizontal rules
+                break;
+            default:
+                // Try generic extraction
+                if (nodeObj.content) {
+                    textParts.push(extractTextFromProseMirrorDoc(nodeObj));
+                }
+        }
+    }
+    return textParts.filter(Boolean).join("\n\n");
+}
+/**
+ * Extracts text from a ProseMirror node with inline content.
+ */
+function extractTextFromProseMirrorNode(node) {
+    const content = node.content;
+    if (!Array.isArray(content)) {
+        return "";
+    }
+    return content
+        .map((child) => {
+        if (typeof child !== "object" || child === null)
+            return "";
+        const childObj = child;
+        if (childObj.type === "text") {
+            return childObj.text || "";
+        }
+        // Handle inline nodes with content
+        if (childObj.content) {
+            return extractTextFromProseMirrorNode(childObj);
+        }
+        return "";
+    })
+        .join("");
+}
+/**
+ * Extracts text from a ProseMirror list node.
+ */
+function extractTextFromProseMirrorList(list) {
+    const items = list.content;
+    if (!Array.isArray(items)) {
+        return "";
+    }
+    return items
+        .map((item, _index) => {
+        if (typeof item !== "object" || item === null)
+            return "";
+        const itemObj = item;
+        const itemText = extractTextFromProseMirrorDoc(itemObj);
+        return `- ${itemText}`;
+    })
+        .join("\n");
+}
+/**
+ * Strips HTML tags from a string, preserving structure where possible.
+ */
+export function stripHtmlTags(html) {
+    if (!html)
+        return "";
+    // First, add newlines for block elements
+    let text = html
+        .replace(/<\/?(p|div|br|h[1-6]|li|tr)[^>]*>/gi, "\n")
+        .replace(/<\/?(ul|ol|table|blockquote)[^>]*>/gi, "\n\n");
+    // Remove remaining HTML tags
+    text = text.replace(/<[^>]*>/g, "");
+    // Decode common HTML entities
+    text = text
+        .replace(/&nbsp;/g, " ")
+        .replace(/&amp;/g, "&")
+        .replace(/&lt;/g, "<")
+        .replace(/&gt;/g, ">")
+        .replace(/&quot;/g, '"')
+        .replace(/&#39;/g, "'")
+        .replace(/&mdash;/g, "—")
+        .replace(/&ndash;/g, "–");
+    // Clean up whitespace
+    text = text
+        .split("\n")
+        .map((line) => line.trim())
+        .join("\n")
+        .replace(/\n{3,}/g, "\n\n")
+        .trim();
+    return text;
+}
+/**
+ * Extracts text from a JSON field value.
+ *
+ * Recursively extracts string values from objects and arrays.
+ * Useful for structured data fields that may contain text content.
+ *
+ * @param value - The JSON field value
+ * @param maxDepth - Maximum recursion depth
+ * @returns Extracted text content
+ */
+export function extractTextFromJson(value, maxDepth = 5) {
+    if (maxDepth <= 0)
+        return "";
+    if (value === null || value === undefined) {
+        return "";
+    }
+    if (typeof value === "string") {
+        return value;
+    }
+    if (typeof value === "number" || typeof value === "boolean") {
+        return String(value);
+    }
+    if (Array.isArray(value)) {
+        return value
+            .map((item) => extractTextFromJson(item, maxDepth - 1))
+            .filter(Boolean)
+            .join(", ");
+    }
+    if (typeof value === "object") {
+        const obj = value;
+        const textParts = [];
+        // Prioritize common text field names
+        const priorityKeys = [
+            "text",
+            "content",
+            "value",
+            "label",
+            "title",
+            "name",
+            "description",
+        ];
+        const seenKeys = new Set();
+        for (const key of priorityKeys) {
+            if (key in obj) {
+                const extracted = extractTextFromJson(obj[key], maxDepth - 1);
+                if (extracted) {
+                    textParts.push(extracted);
+                    seenKeys.add(key);
+                }
+            }
+        }
+        // Then process remaining keys
+        for (const [key, val] of Object.entries(obj)) {
+            if (seenKeys.has(key))
+                continue;
+            // Skip internal/system keys
+            if (key.startsWith("_") || key.startsWith("$"))
+                continue;
+            const extracted = extractTextFromJson(val, maxDepth - 1);
+            if (extracted) {
+                textParts.push(extracted);
+            }
+        }
+        return textParts.join(" ");
+    }
+    return "";
+}
+/**
+ * Extracts text from a select or multiSelect field value.
+ */
+export function extractTextFromSelect(value) {
+    if (value === null || value === undefined) {
+        return "";
+    }
+    if (typeof value === "string") {
+        return value;
+    }
+    if (Array.isArray(value)) {
+        return value.filter((v) => typeof v === "string").join(", ");
+    }
+    return "";
+}
+// =============================================================================
+// Content Extraction
+// =============================================================================
+/**
+ * Extracts text content from a content entry based on its content type schema.
+ *
+ * This function:
+ * 1. Iterates through fields defined in the content type
+ * 2. Extracts text from each field based on its type
+ * 3. Tracks references (content and media)
+ * 4. Builds a combined text representation
+ *
+ * @param entry - The content entry to extract from
+ * @param contentType - The content type definition
+ * @param options - Extraction options
+ * @param resolvedReferences - Optional map of resolved reference information
+ * @returns Extracted content with metadata
+ *
+ * @example
+ * ```typescript
+ * const extracted = extractContent(entry, contentType, {
+ *   includeFields: ["title", "content", "excerpt"],
+ *   extractRichText: true,
+ * });
+ *
+ * console.log(extracted.fullText);
+ * // "My Blog Post\n\nThis is the main content...\n\nA brief excerpt."
+ * ```
+ */
+export function extractContent(entry, contentType, options = {}, resolvedReferences) {
+    const opts = { ...DEFAULT_EXTRACTION_OPTIONS, ...options };
+    const data = entry.data || {};
+    const fieldTexts = {};
+    const sourceInfo = [];
+    const referencedEntryIds = [];
+    const referencedMediaIds = [];
+    let title;
+    // Determine which fields to process
+    const fieldsToProcess = contentType.fields.filter((field) => {
+        // Check include list
+        if (opts.includeFields && opts.includeFields.length > 0) {
+            if (!opts.includeFields.includes(field.name))
+                return false;
+        }
+        // Check exclude list
+        if (opts.excludeFields && opts.excludeFields.length > 0) {
+            if (opts.excludeFields.includes(field.name))
+                return false;
+        }
+        return true;
+    });
+    // Process each field
+    for (const field of fieldsToProcess) {
+        const value = data[field.name];
+        if (value === null || value === undefined)
+            continue;
+        let extractedText = "";
+        switch (field.type) {
+            case "text":
+                extractedText = typeof value === "string" ? value : String(value);
+                break;
+            case "richText":
+                if (opts.extractRichText) {
+                    extractedText = extractTextFromRichText(value);
+                }
+                break;
+            case "json":
+                if (opts.extractJson) {
+                    extractedText = extractTextFromJson(value);
+                }
+                break;
+            case "select":
+            case "multiSelect":
+                extractedText = extractTextFromSelect(value);
+                break;
+            case "reference": {
+                // Track reference IDs
+                const refIds = extractReferenceIds(value, field);
+                referencedEntryIds.push(...refIds);
+                // Include reference context if available
+                if (opts.includeReferenceContext && resolvedReferences) {
+                    const refTexts = refIds
+                        .map((id) => {
+                        const ref = resolvedReferences.get(id);
+                        if (ref && ref.title) {
+                            return `[${ref.title}]`;
+                        }
+                        return null;
+                    })
+                        .filter(Boolean);
+                    if (refTexts.length > 0) {
+                        extractedText = `Referenced: ${refTexts.join(", ")}`;
+                    }
+                }
+                break;
+            }
+            case "media": {
+                // Track media IDs
+                const mediaIds = extractMediaIds(value, field);
+                referencedMediaIds.push(...mediaIds);
+                // Media doesn't contribute to text content
+                break;
+            }
+            case "number":
+            case "boolean":
+            case "date":
+            case "datetime":
+                // These can optionally be included as context
+                extractedText = formatFieldValue(value, field.type);
+                break;
+            default:
+                // Unknown field type - try generic extraction
+                if (typeof value === "string") {
+                    extractedText = value;
+                }
+        }
+        if (extractedText) {
+            fieldTexts[field.name] = extractedText;
+            sourceInfo.push({
+                fieldName: field.name,
+                fieldLabel: field.label,
+                fieldType: field.type,
+                charCount: extractedText.length,
+            });
+            // Extract title from title field
+            if (field.name === contentType.titleField) {
+                title = extractedText;
+            }
+        }
+    }
+    // Build full text with field labels for context
+    const fullTextParts = [];
+    // Add title first if available
+    if (title) {
+        fullTextParts.push(title);
+    }
+    // Add other fields
+    for (const field of fieldsToProcess) {
+        if (field.name === contentType.titleField)
+            continue; // Already added
+        const text = fieldTexts[field.name];
+        if (text) {
+            fullTextParts.push(text);
+        }
+    }
+    return {
+        fullText: fullTextParts.join("\n\n"),
+        fieldTexts,
+        title,
+        referencedEntryIds,
+        referencedMediaIds,
+        sourceInfo,
+    };
+}
+/**
+ * Extracts reference IDs from a reference field value.
+ */
+function extractReferenceIds(value, field) {
+    if (value === null || value === undefined) {
+        return [];
+    }
+    const isMultiple = field.options?.multiple === true;
+    if (isMultiple && Array.isArray(value)) {
+        return value.filter((v) => typeof v === "string");
+    }
+    if (typeof value === "string") {
+        return [value];
+    }
+    return [];
+}
+/**
+ * Extracts media IDs from a media field value.
+ */
+function extractMediaIds(value, field) {
+    if (value === null || value === undefined) {
+        return [];
+    }
+    const isMultiple = field.options?.multiple === true;
+    if (isMultiple && Array.isArray(value)) {
+        return value.filter((v) => typeof v === "string");
+    }
+    if (typeof value === "string") {
+        return [value];
+    }
+    return [];
+}
+/**
+ * Formats a field value for text representation.
+ */
+function formatFieldValue(value, fieldType) {
+    if (value === null || value === undefined) {
+        return "";
+    }
+    switch (fieldType) {
+        case "number":
+            return typeof value === "number" ? value.toString() : String(value);
+        case "boolean":
+            return value ? "Yes" : "No";
+        case "date":
+        case "datetime":
+            if (typeof value === "string") {
+                return value;
+            }
+            if (typeof value === "number") {
+                return new Date(value).toISOString();
+            }
+            return String(value);
+        default:
+            return String(value);
+    }
+}
+// =============================================================================
+// Text Chunking
+// =============================================================================
+/**
+ * Splits text into semantic chunks optimized for embedding.
+ *
+ * The algorithm:
+ * 1. First tries to split on paragraph breaks (default delimiter)
+ * 2. Falls back to line breaks if paragraphs are too large
+ * 3. Falls back to sentence boundaries if lines are too large
+ * 4. Force-splits at hard limit if necessary
+ * 5. Optionally preserves heading context
+ *
+ * @param text - The text to chunk
+ * @param options - Chunking options
+ * @returns Array of text chunks
+ *
+ * @example
+ * ```typescript
+ * const chunks = chunkText(longArticle, {
+ *   maxCharsSoftLimit: 1000,
+ *   preserveHeadingContext: true,
+ * });
+ * ```
+ */
+export function chunkText(text, options = {}) {
+    const opts = { ...DEFAULT_CHUNK_OPTIONS, ...options };
+    if (!text || text.trim().length === 0) {
+        return [];
+    }
+    // If text is small enough, return as single chunk
+    if (text.length <= opts.maxCharsSoftLimit) {
+        return [text.trim()];
+    }
+    const chunks = [];
+    let currentHeading = null;
+    // Split by primary delimiter
+    let segments = text.split(opts.delimiter);
+    // If we have very few segments, try secondary splitting
+    if (segments.length <= 2 && text.length > opts.maxCharsSoftLimit) {
+        for (const fallback of opts.fallbackDelimiters) {
+            const fallbackSegments = text.split(fallback);
+            if (fallbackSegments.length > segments.length) {
+                segments = fallbackSegments;
+                break;
+            }
+        }
+    }
+    let currentChunk = "";
+    for (const segment of segments) {
+        const trimmedSegment = segment.trim();
+        if (!trimmedSegment)
+            continue;
+        // Detect headings (lines that look like titles)
+        const isHeading = detectHeading(trimmedSegment);
+        if (isHeading && opts.preserveHeadingContext) {
+            currentHeading = trimmedSegment;
+        }
+        // Check if adding this segment would exceed soft limit
+        const potentialChunk = currentChunk
+            ? `${currentChunk}\n\n${trimmedSegment}`
+            : trimmedSegment;
+        if (potentialChunk.length > opts.maxCharsSoftLimit && currentChunk) {
+            // Save current chunk
+            chunks.push(finalizeChunk(currentChunk, currentHeading, opts));
+            // Start new chunk, potentially with heading context
+            if (opts.preserveHeadingContext && currentHeading && !isHeading) {
+                currentChunk = `${currentHeading}\n\n${trimmedSegment}`;
+            }
+            else {
+                currentChunk = trimmedSegment;
+            }
+        }
+        else {
+            currentChunk = potentialChunk;
+        }
+        // Handle segments that are too large even alone
+        if (currentChunk.length > opts.maxCharsHardLimit) {
+            const subChunks = forceSplitText(currentChunk, opts);
+            chunks.push(...subChunks.slice(0, -1));
+            currentChunk = subChunks[subChunks.length - 1] || "";
+        }
+    }
+    // Don't forget the last chunk
+    if (currentChunk.trim()) {
+        chunks.push(finalizeChunk(currentChunk, null, opts));
+    }
+    return chunks;
+}
+/**
+ * Detects if a text segment is likely a heading.
+ */
+function detectHeading(text) {
+    const trimmed = text.trim();
+    // Short lines that don't end with sentence punctuation are likely headings
+    if (trimmed.length < 100 && !trimmed.match(/[.!?]$/)) {
+        // Check if it starts with heading patterns
+        if (trimmed.match(/^#{1,6}\s/) || // Markdown headings
+            trimmed.match(/^[A-Z][\w\s]+:?$/) || // Title Case lines
+            trimmed.match(/^\d+\.\s+[A-Z]/)) {
+            // Numbered sections
+            return true;
+        }
+    }
+    return false;
+}
+/**
+ * Finalizes a chunk by adding overlap if needed.
+ */
+function finalizeChunk(chunk, _heading, _opts) {
+    return chunk.trim();
+}
+/**
+ * Force-splits text that exceeds the hard limit.
+ */
+function forceSplitText(text, opts) {
+    const chunks = [];
+    let remaining = text;
+    while (remaining.length > opts.maxCharsHardLimit) {
+        // Try to find a good split point
+        let splitPoint = opts.maxCharsSoftLimit;
+        // Look for sentence boundary
+        const sentenceEnd = remaining.lastIndexOf(". ", splitPoint);
+        if (sentenceEnd > opts.minCharsSoftLimit) {
+            splitPoint = sentenceEnd + 1;
+        }
+        else {
+            // Look for word boundary
+            const spacePoint = remaining.lastIndexOf(" ", splitPoint);
+            if (spacePoint > opts.minCharsSoftLimit) {
+                splitPoint = spacePoint;
+            }
+        }
+        chunks.push(remaining.slice(0, splitPoint).trim());
+        remaining = remaining.slice(splitPoint).trim();
+    }
+    if (remaining) {
+        chunks.push(remaining);
+    }
+    return chunks;
+}
+// =============================================================================
+// Main API Functions
+// =============================================================================
+/**
+ * Processes a content entry into chunks ready for RAG indexing.
+ *
+ * This is the main function to use for preparing CMS content for @convex-dev/rag.
+ * It combines extraction and chunking with full metadata.
+ *
+ * @param entry - The content entry to process
+ * @param contentType - The content type definition
+ * @param options - Extraction and chunking options
+ * @param resolvedReferences - Optional map of resolved references for context
+ * @returns Array of content chunks with metadata
+ *
+ * @example
+ * ```typescript
+ * // In a Convex action
+ * export const indexEntry = action({
+ *   args: { entryId: v.id("contentEntries") },
+ *   handler: async (ctx, { entryId }) => {
+ *     const entry = await ctx.runQuery(api.contentEntries.get, { id: entryId });
+ *     const contentType = await ctx.runQuery(api.contentTypes.get, {
+ *       id: entry.contentTypeId
+ *     });
+ *
+ *     const chunks = chunkContentEntry(entry, contentType, {
+ *       chunkOptions: { maxCharsSoftLimit: 800 },
+ *       includeMetadata: true,
+ *     });
+ *
+ *     // Add to RAG index
+ *     await rag.add(ctx, {
+ *       namespace: `cms:${contentType.name}`,
+ *       key: entryId,
+ *       chunks: chunks.map(c => c.text),
+ *       title: entry.data.title,
+ *     });
+ *
+ *     return { indexed: chunks.length };
+ *   },
+ * });
+ * ```
+ */
+export function chunkContentEntry(entry, contentType, options = {}, resolvedReferences) {
+    const opts = { ...DEFAULT_EXTRACTION_OPTIONS, ...options };
+    // Extract content from the entry
+    const extracted = extractContent(entry, contentType, opts, resolvedReferences);
+    if (!extracted.fullText) {
+        return [];
+    }
+    // Apply prefix/suffix to full text before chunking
+    let textToChunk = extracted.fullText;
+    if (opts.chunkPrefix) {
+        const prefix = opts.chunkPrefix
+            .replace("{contentType}", contentType.displayName)
+            .replace("{title}", extracted.title || entry.slug)
+            .replace("{slug}", entry.slug);
+        textToChunk = `${prefix}\n\n${textToChunk}`;
+    }
+    if (opts.chunkSuffix) {
+        textToChunk = `${textToChunk}\n\n${opts.chunkSuffix}`;
+    }
+    // Chunk the text
+    const textChunks = chunkText(textToChunk, opts.chunkOptions);
+    // Build content chunks with metadata
+    const chunks = textChunks.map((text, index) => {
+        const metadata = {
+            entryId: entry._id,
+            contentType: contentType.name,
+            contentTypeDisplayName: contentType.displayName,
+            slug: entry.slug,
+            status: entry.status,
+            locale: entry.locale,
+            sourceFields: extracted.sourceInfo.map((s) => s.fieldName),
+            chunkIndex: index,
+            totalChunks: textChunks.length,
+            title: extracted.title,
+            createdAt: new Date(entry._creationTime).toISOString(),
+            firstPublishedAt: entry.firstPublishedAt
+                ? new Date(entry.firstPublishedAt).toISOString()
+                : undefined,
+            lastPublishedAt: entry.lastPublishedAt
+                ? new Date(entry.lastPublishedAt).toISOString()
+                : undefined,
+            version: entry.version,
+            referencedEntryIds: extracted.referencedEntryIds.length > 0
+                ? extracted.referencedEntryIds
+                : undefined,
+            referencedMediaIds: extracted.referencedMediaIds.length > 0
+                ? extracted.referencedMediaIds
+                : undefined,
+            semanticType: detectSemanticType(text),
+        };
+        return opts.includeMetadata ? { text, metadata } : { text, metadata };
+    });
+    // Optionally create a summary chunk
+    if (opts.createSummaryChunk && chunks.length > 0) {
+        const summaryChunk = createSummaryChunk(entry, contentType, extracted, chunks.length);
+        chunks.unshift(summaryChunk);
+        // Update chunk indices
+        chunks.forEach((chunk, index) => {
+            chunk.metadata.chunkIndex = index;
+            chunk.metadata.totalChunks = chunks.length;
+        });
+    }
+    return chunks;
+}
+/**
+ * Detects the semantic type of a chunk based on its content.
+ */
+function detectSemanticType(text) {
+    const trimmed = text.trim();
+    // Check for headings
+    if (trimmed.match(/^#{1,6}\s/) ||
+        (trimmed.length < 100 && !trimmed.includes("\n"))) {
+        const lines = trimmed.split("\n");
+        if (lines.length === 1 && !trimmed.match(/[.!?]$/)) {
+            return lines[0].length < 20 ? "title" : "heading";
+        }
+    }
+    // Check for lists
+    if (trimmed.match(/^[-*]\s/m) || trimmed.match(/^\d+\.\s/m)) {
+        return "list";
+    }
+    // Check for quotes
+    if (trimmed.startsWith('"') || trimmed.startsWith(">")) {
+        return "quote";
+    }
+    // Check for code
+    if (trimmed.startsWith("```") || trimmed.match(/^\s{4}/m)) {
+        return "code";
+    }
+    // Default to paragraph or mixed
+    return trimmed.includes("\n\n") ? "mixed" : "paragraph";
+}
+/**
+ * Creates a summary chunk from key fields.
+ */
+function createSummaryChunk(entry, contentType, extracted, totalChunks) {
+    const summaryParts = [];
+    // Add title
+    if (extracted.title) {
+        summaryParts.push(`Title: ${extracted.title}`);
+    }
+    // Add content type
+    summaryParts.push(`Type: ${contentType.displayName}`);
+    // Add status and dates
+    summaryParts.push(`Status: ${entry.status}`);
+    if (entry.lastPublishedAt) {
+        summaryParts.push(`Published: ${new Date(entry.lastPublishedAt).toLocaleDateString()}`);
+    }
+    // Add brief excerpt from first field
+    const firstField = Object.keys(extracted.fieldTexts)[0];
+    if (firstField && extracted.fieldTexts[firstField]) {
+        const excerpt = extracted.fieldTexts[firstField].slice(0, 200);
+        summaryParts.push(`Summary: ${excerpt}${excerpt.length >= 200 ? "..." : ""}`);
+    }
+    return {
+        text: summaryParts.join("\n"),
+        metadata: {
+            entryId: entry._id,
+            contentType: contentType.name,
+            contentTypeDisplayName: contentType.displayName,
+            slug: entry.slug,
+            status: entry.status,
+            locale: entry.locale,
+            sourceFields: ["_summary"],
+            chunkIndex: 0,
+            totalChunks: totalChunks + 1,
+            title: extracted.title,
+            createdAt: new Date(entry._creationTime).toISOString(),
+            firstPublishedAt: entry.firstPublishedAt
+                ? new Date(entry.firstPublishedAt).toISOString()
+                : undefined,
+            lastPublishedAt: entry.lastPublishedAt
+                ? new Date(entry.lastPublishedAt).toISOString()
+                : undefined,
+            version: entry.version,
+            semanticType: "field_value",
+        },
+    };
+}
+// =============================================================================
+// Batch Processing Utilities
+// =============================================================================
+/**
+ * Processes multiple content entries into chunks.
+ *
+ * Useful for batch indexing operations.
+ *
+ * @param entries - Array of content entries
+ * @param contentTypes - Map of content type ID to content type
+ * @param options - Extraction options
+ * @returns Map of entry ID to chunks
+ */
+export function chunkMultipleEntries(entries, contentTypes, options = {}) {
+    const results = new Map();
+    for (const entry of entries) {
+        const contentType = contentTypes.get(entry.contentTypeId);
+        if (!contentType) {
+            console.warn(`Content type not found for entry ${entry._id}`);
+            continue;
+        }
+        const chunks = chunkContentEntry(entry, contentType, options);
+        results.set(entry._id, chunks);
+    }
+    return results;
+}
+/**
+ * Calculates the total character count and chunk count for entries.
+ *
+ * Useful for estimating indexing costs.
+ */
+export function estimateChunkingStats(entries, contentTypes, options = {}) {
+    let totalChunks = 0;
+    let totalCharacters = 0;
+    for (const entry of entries) {
+        const contentType = contentTypes.get(entry.contentTypeId);
+        if (!contentType)
+            continue;
+        const chunks = chunkContentEntry(entry, contentType, options);
+        totalChunks += chunks.length;
+        totalCharacters += chunks.reduce((sum, c) => sum + c.text.length, 0);
+    }
+    return {
+        totalEntries: entries.length,
+        totalChunks,
+        totalCharacters,
+        averageChunksPerEntry: entries.length > 0 ? totalChunks / entries.length : 0,
+        averageCharsPerChunk: totalChunks > 0 ? totalCharacters / totalChunks : 0,
+    };
+}
+// =============================================================================
+// Exports
+// =============================================================================
+export { DEFAULT_CHUNK_OPTIONS, DEFAULT_EXTRACTION_OPTIONS };
+//# sourceMappingURL=ragContentChunker.js.map