@daviddh/llm-markdown-whatsapp 0.0.1

package/packages/core/dist/chatSplit/questionProcessor.js

@@ -0,0 +1,212 @@
+import { COMBINED_LENGTH_THRESHOLD, DOUBLE_NEWLINE_DISTANCE_THRESHOLD, INDEX_OFFSET, LONG_QUESTION_THRESHOLD, SHORT_INTRO_THRESHOLD, } from './constants.js';
+import { isPositionInBulletLine, isPositionInsideParentheses } from './positionHelpers.js';
+import { findPositionAfterEmoji, hasTextContent, isParentheticalClarification, smartTrim, startsWithEmoji, startsWithLowercase, } from './textHelpers.js';
+/** Contiguous questions text length threshold */
+const CONTIGUOUS_QUESTIONS_TEXT_THRESHOLD = 50;
+/** Zero constant */
+const ZERO = 0;
+/** Not found constant */
+const NOT_FOUND = -1;
+/** Double newline offset */
+const DOUBLE_NEWLINE_OFFSET = 2;
+/** Increment constant */
+const INCREMENT = 1;
+/** Get first element of regex match */
+const getFirstMatch = (match) => {
+    const [firstElement] = match;
+    return firstElement;
+};
+/** Get last element of array */
+const getLastElement = (arr) => {
+    const { length } = arr;
+    return length > ZERO ? arr[length - INDEX_OFFSET] : undefined;
+};
+/**
+ * Handles long question splitting
+ */
+function handleLongQuestion(remainingText, chunks, questionPart, afterQuestion) {
+    if (startsWithLowercase(afterQuestion)) {
+        return { splitFound: false, newRemainingText: remainingText };
+    }
+    if (startsWithEmoji(afterQuestion)) {
+        const emojiEndPos = findPositionAfterEmoji(afterQuestion);
+        const emojiPart = afterQuestion.substring(ZERO, emojiEndPos);
+        const textAfterEmoji = afterQuestion.substring(emojiEndPos).trim();
+        if (textAfterEmoji.length > ZERO) {
+            chunks.push(`${questionPart} ${emojiPart}`);
+            return { splitFound: true, newRemainingText: textAfterEmoji };
+        }
+    }
+    else {
+        chunks.push(questionPart);
+        return { splitFound: true, newRemainingText: afterQuestion };
+    }
+    return { splitFound: false, newRemainingText: remainingText };
+}
+/**
+ * Handles short question splitting
+ */
+function handleShortQuestion(remainingText, chunks, questionPart, afterQuestion) {
+    const firstPeriodIndex = afterQuestion.indexOf('.');
+    if (firstPeriodIndex !== NOT_FOUND && firstPeriodIndex < afterQuestion.length - INDEX_OFFSET) {
+        const untilFirstPeriod = afterQuestion.substring(ZERO, firstPeriodIndex + INDEX_OFFSET);
+        const combinedLength = questionPart.length + INDEX_OFFSET + untilFirstPeriod.length;
+        if (combinedLength <= COMBINED_LENGTH_THRESHOLD) {
+            return { splitFound: false, newRemainingText: remainingText };
+        }
+    }
+    if (startsWithLowercase(afterQuestion)) {
+        return { splitFound: false, newRemainingText: remainingText };
+    }
+    if (startsWithEmoji(afterQuestion)) {
+        const emojiEndPos = findPositionAfterEmoji(afterQuestion);
+        const emojiPart = afterQuestion.substring(ZERO, emojiEndPos);
+        const textAfterEmoji = afterQuestion.substring(emojiEndPos).trim();
+        if (textAfterEmoji.length > ZERO) {
+            chunks.push(`${questionPart} ${emojiPart}`);
+            return { splitFound: true, newRemainingText: textAfterEmoji };
+        }
+    }
+    else {
+        chunks.push(questionPart);
+        return { splitFound: true, newRemainingText: afterQuestion };
+    }
+    return { splitFound: false, newRemainingText: remainingText };
+}
+/** Handle parenthetical clarification split */
+function handleParentheticalSplit(ctx, chunks) {
+    const parentheticalMatch = /^\([^\)]+\)\?/v.exec(ctx.afterQuestion);
+    if (parentheticalMatch === null)
+        return null;
+    const firstMatch = getFirstMatch(parentheticalMatch);
+    const { length: parentheticalLength } = firstMatch;
+    const afterParenthetical = smartTrim(ctx.afterQuestion.substring(parentheticalLength));
+    if (!hasTextContent(afterParenthetical))
+        return null;
+    const beforePart = ctx.remainingText.substring(ZERO, ctx.questionIndex + INDEX_OFFSET + ctx.afterQuestionRaw.indexOf(firstMatch) + parentheticalLength);
+    chunks.push(beforePart);
+    return { splitFound: true, newRemainingText: afterParenthetical };
+}
+/**
+ * Processes a single question
+ */
+function processSingleQuestion(remainingText, chunks, questionIndex) {
+    if (questionIndex >= remainingText.length - INDEX_OFFSET) {
+        return { splitFound: false, newRemainingText: remainingText };
+    }
+    const afterQuestionRaw = remainingText.substring(questionIndex + INDEX_OFFSET);
+    const afterQuestion = smartTrim(afterQuestionRaw);
+    if (isParentheticalClarification(afterQuestion)) {
+        const ctx = { remainingText, questionIndex, afterQuestionRaw, afterQuestion };
+        const result = handleParentheticalSplit(ctx, chunks);
+        if (result !== null)
+            return result;
+    }
+    if (!isParentheticalClarification(afterQuestion) && hasTextContent(afterQuestion)) {
+        const questionPart = remainingText.substring(ZERO, questionIndex + INDEX_OFFSET);
+        if (questionPart.length > LONG_QUESTION_THRESHOLD) {
+            return handleLongQuestion(remainingText, chunks, questionPart, afterQuestion);
+        }
+        return handleShortQuestion(remainingText, chunks, questionPart, afterQuestion);
+    }
+    return { splitFound: false, newRemainingText: remainingText };
+}
+/**
+ * Processes contiguous questions
+ */
+function processContiguousQuestions(remainingText, chunks, lastQuestionIdx) {
+    if (lastQuestionIdx >= remainingText.length - INDEX_OFFSET) {
+        return { splitFound: false, newRemainingText: remainingText };
+    }
+    const afterQuestionRaw = remainingText.substring(lastQuestionIdx + INDEX_OFFSET);
+    const afterQuestion = smartTrim(afterQuestionRaw);
+    if (!hasTextContent(afterQuestion)) {
+        return { splitFound: false, newRemainingText: remainingText };
+    }
+    if (startsWithEmoji(afterQuestion)) {
+        const emojiEndPos = findPositionAfterEmoji(afterQuestion);
+        const emojiPart = afterQuestion.substring(ZERO, emojiEndPos);
+        const textAfterEmoji = afterQuestion.substring(emojiEndPos).trim();
+        if (textAfterEmoji.length > ZERO) {
+            const beforePart = remainingText.substring(ZERO, lastQuestionIdx + INDEX_OFFSET);
+            const spaceBeforeEmoji = remainingText.charAt(lastQuestionIdx + INDEX_OFFSET) === ' ' ? ' ' : '';
+            chunks.push(beforePart + spaceBeforeEmoji + emojiPart);
+            return { splitFound: true, newRemainingText: textAfterEmoji };
+        }
+    }
+    else {
+        const beforePart = remainingText.substring(ZERO, lastQuestionIdx + INDEX_OFFSET);
+        const spaceAfterQuestion = remainingText.charAt(lastQuestionIdx + INDEX_OFFSET) === ' ' ? ' ' : '';
+        chunks.push(beforePart + spaceAfterQuestion);
+        return { splitFound: true, newRemainingText: afterQuestion };
+    }
+    return { splitFound: false, newRemainingText: remainingText };
+}
+/** Check if character is a question mark */
+const isQuestionMark = (text, index) => text[index] === '?';
+/** Get first line from text */
+const getFirstLineFromText = (text) => {
+    const firstLineEnd = text.indexOf('\n');
+    return firstLineEnd === NOT_FOUND ? text : text.substring(ZERO, firstLineEnd);
+};
+/** Check if position has response options pattern after double newline */
+const hasResponseOptionsAfterDoubleNewline = (afterQuestion) => {
+    const doubleNewlineAfter = afterQuestion.indexOf('\n\n');
+    if (doubleNewlineAfter === NOT_FOUND || doubleNewlineAfter >= DOUBLE_NEWLINE_DISTANCE_THRESHOLD) {
+        return false;
+    }
+    const afterDoubleNewline = afterQuestion.substring(doubleNewlineAfter + DOUBLE_NEWLINE_OFFSET);
+    const firstLine = getFirstLineFromText(afterDoubleNewline);
+    const trimmedFirstLine = firstLine.trim();
+    const hasResponseOptions = trimmedFirstLine.length < SHORT_INTRO_THRESHOLD &&
+        trimmedFirstLine.endsWith(':') &&
+        afterDoubleNewline.includes('\n-');
+    return hasResponseOptions;
+};
+/** Check if position is valid for question split */
+const isValidQuestionPosition = (remainingText, position) => {
+    if (!isQuestionMark(remainingText, position))
+        return false;
+    if (isPositionInBulletLine(remainingText, position))
+        return false;
+    if (isPositionInsideParentheses(remainingText, position))
+        return false;
+    const afterQuestion = remainingText.substring(position + INDEX_OFFSET);
+    return !hasResponseOptionsAfterDoubleNewline(afterQuestion);
+};
+/**
+ * Finds all question mark indices that are valid split points
+ */
+function findValidQuestionIndices(remainingText) {
+    const questionIndices = [];
+    const { length: textLength } = remainingText;
+    for (let i = ZERO; i < textLength; i += INCREMENT) {
+        if (isValidQuestionPosition(remainingText, i)) {
+            questionIndices.push(i);
+        }
+    }
+    return questionIndices;
+}
+/**
+ * Processes question marks for splitting
+ */
+export function processQuestionMarks(remainingText, chunks) {
+    const questionIndices = findValidQuestionIndices(remainingText);
+    if (questionIndices.length === ZERO) {
+        return { splitFound: false, newRemainingText: remainingText };
+    }
+    const [firstQuestionIdx] = questionIndices;
+    const lastQuestionIdx = getLastElement(questionIndices);
+    if (firstQuestionIdx === undefined || lastQuestionIdx === undefined) {
+        return { splitFound: false, newRemainingText: remainingText };
+    }
+    const areContiguous = questionIndices.length > INDEX_OFFSET &&
+        (() => {
+            const textBetween = remainingText.substring(firstQuestionIdx + INDEX_OFFSET, lastQuestionIdx);
+            return (!textBetween.includes('.') && smartTrim(textBetween).length < CONTIGUOUS_QUESTIONS_TEXT_THRESHOLD);
+        })();
+    if (areContiguous) {
+        return processContiguousQuestions(remainingText, chunks, lastQuestionIdx);
+    }
+    return processSingleQuestion(remainingText, chunks, firstQuestionIdx);
+}

package/packages/core/dist/chatSplit/sections.d.ts

@@ -0,0 +1,23 @@
+/** Result of finding a markdown section */
+export interface MarkdownSectionResult {
+    header: string;
+    content: string;
+    fullSection: string;
+}
+/** Result of finding a list section */
+export interface ListSectionResult {
+    start: number;
+    end: number;
+    type: 'numbered' | 'bullet';
+}
+/**
+ * Detects if text starts with a markdown section header
+ * Returns the header and its content if found
+ */
+export declare const findMarkdownSection: (text: string) => MarkdownSectionResult | null;
+/**
+ * Detects if text is within a list section (numbered or bulleted)
+ * Returns the boundaries of the list if found
+ */
+export declare const findListSection: (text: string) => ListSectionResult | null;
+//# sourceMappingURL=sections.d.ts.map

package/packages/core/dist/chatSplit/sections.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"sections.d.ts","sourceRoot":"","sources":["../../src/chatSplit/sections.ts"],"names":[],"mappings":"AAAA,2CAA2C;AAC3C,MAAM,WAAW,qBAAqB;IACpC,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;IAChB,WAAW,EAAE,MAAM,CAAC;CACrB;AAED,uCAAuC;AACvC,MAAM,WAAW,iBAAiB;IAChC,KAAK,EAAE,MAAM,CAAC;IACd,GAAG,EAAE,MAAM,CAAC;IACZ,IAAI,EAAE,UAAU,GAAG,QAAQ,CAAC;CAC7B;AAiCD;;;GAGG;AACH,eAAO,MAAM,mBAAmB,GAAI,MAAM,MAAM,KAAG,qBAAqB,GAAG,IA4B1E,CAAC;AAgJF;;;GAGG;AACH,eAAO,MAAM,eAAe,GAAI,MAAM,MAAM,KAAG,iBAAiB,GAAG,IAiBlE,CAAC"}

package/packages/core/dist/chatSplit/sections.js

@@ -0,0 +1,153 @@
+/** Constants for section processing */
+const NOT_FOUND = -1;
+const FIRST_ELEMENT = 1;
+const DOUBLE_NEWLINE_LENGTH = 2;
+const ZERO = 0;
+const INCREMENT = 1;
+/** Check if text after double newline starts with a markdown header */
+const startsWithMarkdownHeader = (text) => /^(?:\*[^*\n]+\*|_[^_\n]+_)\s*\n/v.test(text);
+/** Check if text starts with a bullet list item */
+const startsWithBullet = (text) => /^[\-•]\s+/v.test(text.trim());
+/** Determine end index based on content after double newline */
+const determineEndIndex = (afterDoubleNewline, doubleNewlineIndex, defaultEndIndex) => {
+    if (startsWithMarkdownHeader(afterDoubleNewline)) {
+        return doubleNewlineIndex;
+    }
+    if (startsWithBullet(afterDoubleNewline)) {
+        return defaultEndIndex;
+    }
+    return doubleNewlineIndex;
+};
+/** Get line at index safely */
+const getLineAt = (lines, index) => lines[index];
+/**
+ * Detects if text starts with a markdown section header
+ * Returns the header and its content if found
+ */
+export const findMarkdownSection = (text) => {
+    const headerMatch = /^(?<header>\*[^*\n]+\*|_[^_\n]+_)\s*\n/v.exec(text);
+    if (headerMatch === null) {
+        return null;
+    }
+    const { groups } = headerMatch;
+    const header = groups?.header ?? headerMatch[FIRST_ELEMENT] ?? '';
+    const [matchedText = ''] = headerMatch;
+    const afterHeader = text.substring(matchedText.length);
+    const { length: defaultEndIndex } = afterHeader;
+    const doubleNewlineIndex = afterHeader.indexOf('\n\n');
+    const endIndex = doubleNewlineIndex === NOT_FOUND
+        ? defaultEndIndex
+        : determineEndIndex(afterHeader.substring(doubleNewlineIndex + DOUBLE_NEWLINE_LENGTH), doubleNewlineIndex, defaultEndIndex);
+    const content = afterHeader.substring(ZERO, endIndex);
+    const fullSection = matchedText + content;
+    return { header, content, fullSection };
+};
+/** Test if line is a numbered list item */
+const isNumberedListItem = (line) => /^\s*\d{1,2}\.\s+/v.test(line);
+/** Test if line is a bullet list item */
+const isBulletListItem = (line) => /^\s*[\-•]\s+/v.test(line);
+/** Find the next non-empty line index in an array of lines */
+const findNextNonEmptyIndex = (lines, startIndex) => {
+    for (let j = startIndex; j < lines.length; j += INCREMENT) {
+        const line = getLineAt(lines, j);
+        if (line !== undefined && line.trim() !== '') {
+            return j;
+        }
+    }
+    return NOT_FOUND;
+};
+/** Check if next line is a list continuation */
+const isListContinuation = (lines, nextNonEmptyIndex) => {
+    if (nextNonEmptyIndex === NOT_FOUND)
+        return false;
+    const nextLine = getLineAt(lines, nextNonEmptyIndex) ?? '';
+    return isNumberedListItem(nextLine) || isBulletListItem(nextLine);
+};
+/** Process a single line in numbered list - returns new state */
+const processNumberedListLine = (line, currentIndex, lines, currentState) => {
+    const isNumbered = isNumberedListItem(line);
+    const isBullet = isBulletListItem(line);
+    if (isNumbered) {
+        return { action: 'next', newState: { inList: true, endLineIndex: currentIndex } };
+    }
+    if (isBullet && currentState.inList) {
+        return { action: 'next', newState: { ...currentState, endLineIndex: currentIndex } };
+    }
+    if (currentState.inList && line.trim() === '') {
+        const nextNonEmptyIndex = findNextNonEmptyIndex(lines, currentIndex + INCREMENT);
+        const action = isListContinuation(lines, nextNonEmptyIndex) ? 'continue' : 'break';
+        return { action, newState: currentState };
+    }
+    if (currentState.inList) {
+        return { action: 'break', newState: currentState };
+    }
+    return { action: 'next', newState: currentState };
+};
+/** Process numbered list to find end index */
+const processNumberedList = (lines) => {
+    let state = { endLineIndex: NOT_FOUND, inList: false };
+    for (let i = ZERO; i < lines.length; i += INCREMENT) {
+        const line = getLineAt(lines, i);
+        if (line === undefined)
+            continue;
+        const { action, newState } = processNumberedListLine(line, i, lines, state);
+        state = newState;
+        if (action === 'break')
+            break;
+    }
+    return state.endLineIndex >= ZERO
+        ? lines.slice(ZERO, state.endLineIndex + INCREMENT).join('\n').length
+        : ZERO;
+};
+/** Process a single line in bullet list - returns new state */
+const processBulletListLine = (line, currentIndex, lines, currentState) => {
+    const isBullet = /^[\-•]\s+/v.test(line.trim());
+    if (isBullet) {
+        const { length: newEndIndex } = lines.slice(ZERO, currentIndex + INCREMENT).join('\n');
+        return { action: 'next', newState: { inList: true, endIndex: newEndIndex } };
+    }
+    if (currentState.inList && line.trim() === '') {
+        const nextLine = getLineAt(lines, currentIndex + INCREMENT);
+        if (nextLine !== undefined && /^[\-•]\s+/v.test(nextLine.trim())) {
+            return { action: 'continue', newState: currentState };
+        }
+        return { action: 'break', newState: currentState };
+    }
+    if (currentState.inList) {
+        return { action: 'break', newState: currentState };
+    }
+    return { action: 'next', newState: currentState };
+};
+/** Process bullet list to find end index */
+const processBulletList = (lines) => {
+    let state = { endIndex: ZERO, inList: false };
+    for (let i = ZERO; i < lines.length; i += INCREMENT) {
+        const line = getLineAt(lines, i);
+        if (line === undefined)
+            continue;
+        const { action, newState } = processBulletListLine(line, i, lines, state);
+        state = newState;
+        if (action === 'break')
+            break;
+    }
+    return state.endIndex;
+};
+/**
+ * Detects if text is within a list section (numbered or bulleted)
+ * Returns the boundaries of the list if found
+ */
+export const findListSection = (text) => {
+    const numberedListStart = /^\d{1,2}\.\s+/v;
+    const bulletListStart = /^[\-•]\s+/v;
+    if (numberedListStart.test(text.trim())) {
+        const lines = text.split('\n');
+        const endIndex = processNumberedList(lines);
+        return { start: ZERO, end: endIndex, type: 'numbered' };
+    }
+    if (bulletListStart.test(text.trim())) {
+        const lines = text.split('\n');
+        const endIndex = processBulletList(lines);
+        return { start: ZERO, end: endIndex, type: 'bullet' };
+    }
+    return null;
+};

package/packages/core/dist/chatSplit/splitChatText.d.ts

@@ -0,0 +1,6 @@
+/**
+ * Splits chat text into smaller chunks for better readability.
+ * Handles various patterns like lists, questions, markdown sections, etc.
+ */
+export declare const splitChatText: (text: string | null | undefined) => string[];
+//# sourceMappingURL=splitChatText.d.ts.map

package/packages/core/dist/chatSplit/splitChatText.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"splitChatText.d.ts","sourceRoot":"","sources":["../../src/chatSplit/splitChatText.ts"],"names":[],"mappings":"AAoHA;;;GAGG;AACH,eAAO,MAAM,aAAa,GAAI,MAAM,MAAM,GAAG,IAAI,GAAG,SAAS,KAAG,MAAM,EAmCrE,CAAC"}

package/packages/core/dist/chatSplit/splitChatText.js

@@ -0,0 +1,119 @@
+import { processSectionBreaks } from './breakProcessor.js';
+import { ZERO } from './constants.js';
+import { normalizeInlineNumberedList, normalizeInlineProductCardList } from './listNormalization.js';
+import { processListSection } from './listProcessor.js';
+import { mergeSmallChunks } from './mergeProcessor.js';
+import { processLongParagraphSequence, processLongParagraphsAfterIntro, processMarkdownSection, } from './paragraphProcessor.js';
+import { processPeriodSplits } from './periodProcessor.js';
+import { processProductCardLists } from './productCardProcessor.js';
+import { normalizeSpanishPunctuation } from './punctuationNormalization.js';
+import { processQuestionMarks } from './questionProcessor.js';
+import { PERIOD_SPLIT_TEXT_THRESHOLD } from './splitConstants.js';
+import { processIntroWithList, processIntroWithLongParagraphs, processQuestionWithList, } from './splitProcessors.js';
+import { removePeriodsAfterURLs } from './urlNormalization.js';
+/** Run intro and list processors */
+const runIntroAndListProcessors = (remainingText, chunks) => {
+    const introResult = processIntroWithList(remainingText, chunks);
+    if (introResult.splitFound)
+        return introResult;
+    const questionListResult = processQuestionWithList(remainingText, chunks);
+    if (questionListResult.splitFound)
+        return questionListResult;
+    const longParaResult = processIntroWithLongParagraphs(remainingText, chunks);
+    if (longParaResult.splitFound)
+        return longParaResult;
+    return null;
+};
+/** Run content structure processors */
+const runContentStructureProcessors = (remainingText, chunks) => {
+    const productCardResult = processProductCardLists(remainingText, chunks);
+    if (productCardResult.splitFound)
+        return productCardResult;
+    const listResult = processListSection(remainingText, chunks);
+    if (listResult.splitFound)
+        return listResult;
+    const longParaAfterIntroResult = processLongParagraphsAfterIntro(remainingText, chunks);
+    if (longParaAfterIntroResult.splitFound)
+        return longParaAfterIntroResult;
+    const longParagraphResult = processLongParagraphSequence(remainingText, chunks);
+    if (longParagraphResult.splitFound)
+        return longParagraphResult;
+    return null;
+};
+/** Run formatting processors */
+const runFormattingProcessors = (remainingText, chunks) => {
+    const markdownResult = processMarkdownSection(remainingText, chunks);
+    if (markdownResult.splitFound)
+        return markdownResult;
+    const sectionBreakResult = processSectionBreaks(remainingText, chunks);
+    if (sectionBreakResult.splitFound)
+        return sectionBreakResult;
+    return null;
+};
+/**
+ * Runs all split processors on the remaining text
+ */
+const runSplitProcessors = (remainingText, chunks) => {
+    const introResult = runIntroAndListProcessors(remainingText, chunks);
+    if (introResult !== null)
+        return introResult;
+    const contentResult = runContentStructureProcessors(remainingText, chunks);
+    if (contentResult !== null)
+        return contentResult;
+    const formatResult = runFormattingProcessors(remainingText, chunks);
+    if (formatResult !== null)
+        return formatResult;
+    return { splitFound: false, newRemainingText: remainingText };
+};
+/**
+ * Runs question and period processors
+ */
+const runQuestionAndPeriodProcessors = (remainingText, chunks) => {
+    const questionResult = processQuestionMarks(remainingText, chunks);
+    if (questionResult.splitFound) {
+        return questionResult;
+    }
+    if (remainingText.length > PERIOD_SPLIT_TEXT_THRESHOLD) {
+        const periodResult = processPeriodSplits(remainingText, chunks);
+        if (periodResult.splitFound) {
+            return periodResult;
+        }
+    }
+    return { splitFound: false, newRemainingText: remainingText };
+};
+/** Pre-process text before splitting */
+const preProcessText = (text) => {
+    let processedText = removePeriodsAfterURLs(text);
+    processedText = normalizeInlineNumberedList(processedText);
+    processedText = normalizeInlineProductCardList(processedText);
+    return processedText;
+};
+/**
+ * Splits chat text into smaller chunks for better readability.
+ * Handles various patterns like lists, questions, markdown sections, etc.
+ */
+export const splitChatText = (text) => {
+    if (text === null || text === undefined || text.length === ZERO) {
+        return [];
+    }
+    const processedText = preProcessText(text);
+    const chunks = [];
+    let remainingText = processedText;
+    while (remainingText !== '') {
+        const { splitFound, newRemainingText } = runSplitProcessors(remainingText, chunks);
+        if (splitFound) {
+            remainingText = newRemainingText;
+            continue;
+        }
+        const { splitFound: qpSplitFound, newRemainingText: qpNewText } = runQuestionAndPeriodProcessors(remainingText, chunks);
+        if (qpSplitFound) {
+            remainingText = qpNewText;
+            continue;
+        }
+        chunks.push(remainingText);
+        break;
+    }
+    const mergedChunks = mergeSmallChunks(chunks);
+    const normalizedChunks = mergedChunks.map((chunk) => normalizeSpanishPunctuation(chunk));
+    return normalizedChunks;
+};

package/packages/core/dist/chatSplit/splitConstants.d.ts

@@ -0,0 +1,3 @@
+/** Short text threshold for period splitting */
+export declare const PERIOD_SPLIT_TEXT_THRESHOLD = 100;
+//# sourceMappingURL=splitConstants.d.ts.map

package/packages/core/dist/chatSplit/splitConstants.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"splitConstants.d.ts","sourceRoot":"","sources":["../../src/chatSplit/splitConstants.ts"],"names":[],"mappings":"AAAA,gDAAgD;AAChD,eAAO,MAAM,2BAA2B,MAAM,CAAC"}

package/packages/core/dist/chatSplit/splitConstants.js

@@ -0,0 +1,2 @@
+/** Short text threshold for period splitting */
+export const PERIOD_SPLIT_TEXT_THRESHOLD = 100;

package/packages/core/dist/chatSplit/splitProcessors.d.ts

@@ -0,0 +1,22 @@
+/** Split result type */
+export interface SplitResult {
+    splitFound: boolean;
+    newRemainingText: string;
+}
+/**
+ * Checks if text has a question with response options pattern
+ */
+export declare const hasQuestionWithOptionsPattern: (text: string) => boolean;
+/**
+ * Processes intro with list pattern and returns split info
+ */
+export declare const processIntroWithList: (remainingText: string, chunks: string[]) => SplitResult;
+/**
+ * Processes question followed by numbered list (answer options)
+ */
+export declare const processQuestionWithList: (remainingText: string, chunks: string[]) => SplitResult;
+/**
+ * Processes intro followed by long paragraphs
+ */
+export declare const processIntroWithLongParagraphs: (remainingText: string, chunks: string[]) => SplitResult;
+//# sourceMappingURL=splitProcessors.d.ts.map

package/packages/core/dist/chatSplit/splitProcessors.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"splitProcessors.d.ts","sourceRoot":"","sources":["../../src/chatSplit/splitProcessors.ts"],"names":[],"mappings":"AAmBA,wBAAwB;AACxB,MAAM,WAAW,WAAW;IAC1B,UAAU,EAAE,OAAO,CAAC;IACpB,gBAAgB,EAAE,MAAM,CAAC;CAC1B;AAED;;GAEG;AACH,eAAO,MAAM,6BAA6B,GAAI,MAAM,MAAM,KAAG,OACmC,CAAC;AAuCjG;;GAEG;AACH,eAAO,MAAM,oBAAoB,GAAI,eAAe,MAAM,EAAE,QAAQ,MAAM,EAAE,KAAG,WAkB9E,CAAC;AAEF;;GAEG;AACH,eAAO,MAAM,uBAAuB,GAAI,eAAe,MAAM,EAAE,QAAQ,MAAM,EAAE,KAAG,WA0BjF,CAAC;AAMF;;GAEG;AACH,eAAO,MAAM,8BAA8B,GAAI,eAAe,MAAM,EAAE,QAAQ,MAAM,EAAE,KAAG,WAuBxF,CAAC"}

package/packages/core/dist/chatSplit/splitProcessors.js

@@ -0,0 +1,105 @@
+import { FIRST_NEWLINE_SEARCH_LIMIT, INDEX_OFFSET, MAX_INTRO_LENGTH, MAX_QUESTION_WITH_OPTIONS_LENGTH, } from './constants.js';
+/** Long paragraph threshold */
+const LONG_PARAGRAPH_THRESHOLD = 150;
+/** Minimum list items for question with options */
+const MIN_LIST_ITEMS_FOR_OPTIONS = 2;
+/** Constants */
+const NOT_FOUND = -1;
+const ZERO = 0;
+const FIRST_MATCH = 1;
+const SECOND_MATCH = 2;
+/**
+ * Checks if text has a question with response options pattern
+ */
+export const hasQuestionWithOptionsPattern = (text) => /^[^?]+\?\s*\n+[\s\S]*?(?:Puedes responder con|puedes responder con):[\s\S]*?\n+-/v.test(text);
+/** Check if line ends with colon */
+const lineEndsWithColon = (line) => line?.trim().endsWith(':') === true;
+/** Find the last line ending with colon */
+const findLastColonLineIndex = (lines) => {
+    for (let i = lines.length - INDEX_OFFSET; i >= ZERO; i -= INDEX_OFFSET) {
+        if (lineEndsWithColon(lines[i])) {
+            return i;
+        }
+    }
+    return NOT_FOUND;
+};
+/** Extract intro from match groups */
+const extractIntro = (match) => {
+    const { groups } = match;
+    const part1 = groups?.intro ?? match[FIRST_MATCH] ?? '';
+    const part2 = groups?.afterColon ?? match[SECOND_MATCH] ?? '';
+    return part1 + part2;
+};
+/** Adjust intro for response prompts */
+const adjustIntroForResponsePrompt = (intro) => {
+    if (!intro.includes('Puedes responder con:') && !intro.includes('puedes responder con:')) {
+        return intro;
+    }
+    const lines = intro.split('\n');
+    const lastColonIndex = findLastColonLineIndex(lines);
+    if (lastColonIndex >= ZERO) {
+        return lines.slice(ZERO, lastColonIndex + INDEX_OFFSET).join('\n');
+    }
+    return intro;
+};
+/**
+ * Processes intro with list pattern and returns split info
+ */
+export const processIntroWithList = (remainingText, chunks) => {
+    const match = /^(?<intro>.+?:)(?<afterColon>[^\n]*?)\n+(?<listStart>\d{1,2}\.\s+|[\-•]\s+)/v.exec(remainingText);
+    if (match === null) {
+        return { splitFound: false, newRemainingText: remainingText };
+    }
+    const intro = extractIntro(match);
+    const finalIntro = adjustIntroForResponsePrompt(intro);
+    if (finalIntro.length < MAX_INTRO_LENGTH) {
+        chunks.push(finalIntro);
+        return { splitFound: true, newRemainingText: remainingText.substring(finalIntro.length).trim() };
+    }
+    return { splitFound: false, newRemainingText: remainingText };
+};
+/**
+ * Processes question followed by numbered list (answer options)
+ */
+export const processQuestionWithList = (remainingText, chunks) => {
+    const match = /^(?<question>[\s\S]*?\?[^\n]*?)\n(?<list>\d{1,2}\.\s+[\s\S]*)/v.exec(remainingText);
+    if (match === null) {
+        return { splitFound: false, newRemainingText: remainingText };
+    }
+    const { groups } = match;
+    const listPart = groups?.list ?? match[SECOND_MATCH] ?? '';
+    const listLines = listPart.split('\n');
+    const allLinesAreListItems = listLines.every((line) => line.trim() === '' || /^\d{1,2}\.\s+/v.test(line.trim()));
+    const { length: listItemCount } = listLines.filter((l) => /^\d{1,2}\.\s+/v.test(l.trim()));
+    const shouldProcess = allLinesAreListItems &&
+        remainingText.length < MAX_QUESTION_WITH_OPTIONS_LENGTH &&
+        listItemCount >= MIN_LIST_ITEMS_FOR_OPTIONS;
+    if (shouldProcess) {
+        chunks.push(remainingText);
+        return { splitFound: true, newRemainingText: '' };
+    }
+    return { splitFound: false, newRemainingText: remainingText };
+};
+/** Check if intro ends with list pattern */
+const endsWithListPattern = (intro) => /\d{1,2}\.\s*$/v.exec(intro) !== null || /[\-•]\s*$/v.exec(intro) !== null;
+/**
+ * Processes intro followed by long paragraphs
+ */
+export const processIntroWithLongParagraphs = (remainingText, chunks) => {
+    const firstNewline = remainingText.indexOf('\n');
+    if (firstNewline === NOT_FOUND || firstNewline >= FIRST_NEWLINE_SEARCH_LIMIT) {
+        return { splitFound: false, newRemainingText: remainingText };
+    }
+    const intro = remainingText.substring(ZERO, firstNewline).trim();
+    if (!intro.endsWith(':') || endsWithListPattern(intro)) {
+        return { splitFound: false, newRemainingText: remainingText };
+    }
+    const afterIntro = remainingText.substring(firstNewline + INDEX_OFFSET);
+    const [firstParagraphRaw] = afterIntro.split('\n');
+    const firstParagraph = (firstParagraphRaw ?? '').trim();
+    if (firstParagraph.length > LONG_PARAGRAPH_THRESHOLD) {
+        chunks.push(intro);
+        return { splitFound: true, newRemainingText: afterIntro };
+    }
+    return { splitFound: false, newRemainingText: remainingText };
+};