flappa-doormal 2.2.2 → 2.3.0

package/dist/index.mjs

@@ -157,6 +157,26 @@ const makeDiacriticInsensitive = (text) => {
 
 //#endregion
 //#region src/segmentation/breakpoint-utils.ts
+const WINDOW_PREFIX_LENGTHS = [
+	80,
+	60,
+	40,
+	30,
+	20,
+	15
+];
+const JOINER_PREFIX_LENGTHS = [
+	80,
+	60,
+	40,
+	30,
+	20,
+	15,
+	12,
+	10,
+	8,
+	6
+];
 /**
 * Normalizes a breakpoint to the object form.
 * Strings are converted to { pattern: str } with no constraints.
@@ -312,6 +332,120 @@ const expandBreakpoints = (breakpoints, processPattern$1) => breakpoints.map((bp
 	}
 });
 /**
+* Applies a configured joiner at detected page boundaries within a multi-page content chunk.
+*
+* This is used for breakpoint-generated segments which don't have access to the original
+* `pageMap.pageBreaks` offsets. We detect page starts sequentially by searching for each page's
+* prefix after the previous boundary, then replace ONLY the single newline immediately before
+* that page start.
+*
+* This avoids converting real in-page newlines, while still normalizing page joins consistently.
+*/
+const applyPageJoinerBetweenPages = (content, fromIdx, toIdx, pageIds, normalizedPages, joiner) => {
+	if (joiner === "newline" || fromIdx >= toIdx || !content.includes("\n")) return content;
+	let updated = content;
+	let searchFrom = 0;
+	for (let pi = fromIdx + 1; pi <= toIdx; pi++) {
+		const pageData = normalizedPages.get(pageIds[pi]);
+		if (!pageData) continue;
+		const found = findPrefixPositionInContent(updated, pageData.content.trimStart(), searchFrom);
+		if (found > 0 && updated[found - 1] === "\n") updated = `${updated.slice(0, found - 1)} ${updated.slice(found)}`;
+		if (found > 0) searchFrom = found;
+	}
+	return updated;
+};
+/**
+* Finds the position of a page prefix in content, trying multiple prefix lengths.
+*/
+const findPrefixPositionInContent = (content, trimmedPageContent, searchFrom) => {
+	for (const len of JOINER_PREFIX_LENGTHS) {
+		const prefix = trimmedPageContent.slice(0, Math.min(len, trimmedPageContent.length)).trim();
+		if (!prefix) continue;
+		const pos = content.indexOf(prefix, searchFrom);
+		if (pos > 0) return pos;
+	}
+	return -1;
+};
+/**
+* Estimates how far into the current page `remainingContent` begins.
+*
+* During breakpoint processing, `remainingContent` can begin mid-page after a previous split.
+* When that happens, raw cumulative page offsets (computed from full page starts) can overestimate
+* expected boundary positions. This helper computes an approximate starting offset by matching
+* a short prefix of `remainingContent` inside the current page content.
+*/
+const estimateStartOffsetInCurrentPage = (remainingContent, currentFromIdx, pageIds, normalizedPages) => {
+	const currentPageData = normalizedPages.get(pageIds[currentFromIdx]);
+	if (!currentPageData) return 0;
+	const remStart = remainingContent.trimStart().slice(0, Math.min(60, remainingContent.length));
+	const needle = remStart.slice(0, Math.min(30, remStart.length));
+	if (!needle) return 0;
+	const idx = currentPageData.content.indexOf(needle);
+	return idx > 0 ? idx : 0;
+};
+/**
+* Attempts to find the start position of a target page within remainingContent,
+* anchored near an expected boundary position to reduce collisions.
+*
+* This is used to define breakpoint windows in terms of actual content being split, rather than
+* raw per-page offsets which can desync when structural rules strip markers.
+*/
+const findPageStartNearExpectedBoundary = (remainingContent, _currentFromIdx, targetPageIdx, expectedBoundary, pageIds, normalizedPages) => {
+	const targetPageData = normalizedPages.get(pageIds[targetPageIdx]);
+	if (!targetPageData) return -1;
+	const approx = Math.min(Math.max(0, expectedBoundary), remainingContent.length);
+	const searchStart = Math.max(0, approx - 1e4);
+	const searchEnd = Math.min(remainingContent.length, approx + 2e3);
+	const targetTrimmed = targetPageData.content.trimStart();
+	for (const len of WINDOW_PREFIX_LENGTHS) {
+		const prefix = targetTrimmed.slice(0, Math.min(len, targetTrimmed.length)).trim();
+		if (!prefix) continue;
+		let pos = remainingContent.indexOf(prefix, searchStart);
+		while (pos !== -1 && pos <= searchEnd) {
+			if (pos > 0 && /\s/.test(remainingContent[pos - 1] ?? "")) return pos;
+			pos = remainingContent.indexOf(prefix, pos + 1);
+		}
+		const last = remainingContent.lastIndexOf(prefix, approx);
+		if (last > 0) return last;
+	}
+	return -1;
+};
+/**
+* Finds the end position of a breakpoint window inside `remainingContent`.
+*
+* The window end is defined as the start of the page AFTER `windowEndIdx` (i.e. `windowEndIdx + 1`),
+* found within the actual `remainingContent` string being split. This avoids relying on raw page offsets
+* that can diverge when structural rules strip markers (e.g. `lineStartsAfter`).
+*/
+const findBreakpointWindowEndPosition = (remainingContent, currentFromIdx, windowEndIdx, toIdx, pageIds, normalizedPages, cumulativeOffsets) => {
+	if (windowEndIdx >= toIdx) return remainingContent.length;
+	const desiredNextIdx = windowEndIdx + 1;
+	const minNextIdx = currentFromIdx + 1;
+	const maxNextIdx = Math.min(desiredNextIdx, toIdx);
+	const startOffsetInCurrentPage = estimateStartOffsetInCurrentPage(remainingContent, currentFromIdx, pageIds, normalizedPages);
+	for (let nextIdx = maxNextIdx; nextIdx >= minNextIdx; nextIdx--) {
+		const expectedBoundary = cumulativeOffsets[nextIdx] !== void 0 && cumulativeOffsets[currentFromIdx] !== void 0 ? Math.max(0, cumulativeOffsets[nextIdx] - cumulativeOffsets[currentFromIdx] - startOffsetInCurrentPage) : remainingContent.length;
+		const pos = findPageStartNearExpectedBoundary(remainingContent, currentFromIdx, nextIdx, expectedBoundary, pageIds, normalizedPages);
+		if (pos > 0) return pos;
+	}
+	return remainingContent.length;
+};
+/**
+* Finds exclusion-based break position using raw cumulative offsets.
+*
+* This is used to ensure pages excluded by breakpoints are never merged into the same output segment.
+* Returns a break position relative to the start of `remainingContent` (i.e. the currentFromIdx start).
+*/
+const findExclusionBreakPosition = (currentFromIdx, windowEndIdx, toIdx, pageIds, expandedBreakpoints, cumulativeOffsets) => {
+	const startingPageId = pageIds[currentFromIdx];
+	if (expandedBreakpoints.some((bp) => bp.excludeSet.has(startingPageId)) && currentFromIdx < toIdx) return cumulativeOffsets[currentFromIdx + 1] - cumulativeOffsets[currentFromIdx];
+	for (let pageIdx = currentFromIdx + 1; pageIdx <= windowEndIdx; pageIdx++) {
+		const pageId = pageIds[pageIdx];
+		if (expandedBreakpoints.some((bp) => bp.excludeSet.has(pageId))) return cumulativeOffsets[pageIdx] - cumulativeOffsets[currentFromIdx];
+	}
+	return -1;
+};
+/**
 * Finds the actual ending page index by searching backwards for page content prefix.
 * Used to determine which page a segment actually ends on based on content matching.
 *
@@ -414,6 +548,21 @@ const findPatternBreakPosition = (windowContent, regex, prefer) => {
 	return selected.index + selected.length;
 };
 /**
+* Handles page boundary breakpoint (empty pattern).
+* Returns break position or -1 if no valid position found.
+*/
+const handlePageBoundaryBreak = (remainingContent, windowEndIdx, windowEndPosition, toIdx, pageIds, normalizedPages) => {
+	const nextPageIdx = windowEndIdx + 1;
+	if (nextPageIdx <= toIdx) {
+		const nextPageData = normalizedPages.get(pageIds[nextPageIdx]);
+		if (nextPageData) {
+			const pos = findNextPagePosition(remainingContent, nextPageData);
+			if (pos > 0) return Math.min(pos, windowEndPosition, remainingContent.length);
+		}
+	}
+	return Math.min(windowEndPosition, remainingContent.length);
+};
+/**
 * Tries to find a break position within the current window using breakpoint patterns.
 * Returns the break position or -1 if no suitable break was found.
 *
@@ -424,30 +573,165 @@ const findPatternBreakPosition = (windowContent, regex, prefer) => {
 * @param ctx - Breakpoint context with page data and patterns
 * @returns Break position in the content, or -1 if no break found
 */
-const findBreakPosition = (remainingContent, currentFromIdx, toIdx, windowEndIdx, ctx) => {
-	const { pageIds, normalizedPages, cumulativeOffsets, expandedBreakpoints, prefer } = ctx;
+const findBreakPosition = (remainingContent, currentFromIdx, toIdx, windowEndIdx, windowEndPosition, ctx) => {
+	const { pageIds, normalizedPages, expandedBreakpoints, prefer } = ctx;
 	for (const { rule, regex, excludeSet, skipWhenRegex } of expandedBreakpoints) {
 		if (!isInBreakpointRange(pageIds[currentFromIdx], rule)) continue;
 		if (hasExcludedPageInRange(excludeSet, pageIds, currentFromIdx, windowEndIdx)) continue;
 		if (skipWhenRegex?.test(remainingContent)) continue;
-		if (regex === null) {
-			const nextPageIdx = windowEndIdx + 1;
-			if (nextPageIdx <= toIdx) {
-				const nextPageData = normalizedPages.get(pageIds[nextPageIdx]);
-				if (nextPageData) {
-					const pos = findNextPagePosition(remainingContent, nextPageData);
-					if (pos > 0) return pos;
-				}
-			}
-			return Math.min(cumulativeOffsets[windowEndIdx + 1] - cumulativeOffsets[currentFromIdx], remainingContent.length);
-		}
-		const windowEndPosition = Math.min(cumulativeOffsets[windowEndIdx + 1] - cumulativeOffsets[currentFromIdx], remainingContent.length);
-		const breakPos = findPatternBreakPosition(remainingContent.slice(0, windowEndPosition), regex, prefer);
+		if (regex === null) return handlePageBoundaryBreak(remainingContent, windowEndIdx, windowEndPosition, toIdx, pageIds, normalizedPages);
+		const breakPos = findPatternBreakPosition(remainingContent.slice(0, Math.min(windowEndPosition, remainingContent.length)), regex, prefer);
 		if (breakPos > 0) return breakPos;
 	}
 	return -1;
 };
 
+//#endregion
+//#region src/segmentation/breakpoint-processor.ts
+/**
+* Breakpoint post-processing engine extracted from segmenter.ts.
+*
+* This module is intentionally split into small helpers to reduce cognitive complexity
+* and allow unit testing of tricky edge cases (window sizing, next-page advancement, etc.).
+*/
+const buildPageIdToIndexMap = (pageIds) => new Map(pageIds.map((id, i) => [id, i]));
+const buildNormalizedPagesMap = (pages, normalizedContent) => {
+	const normalizedPages = /* @__PURE__ */ new Map();
+	for (let i = 0; i < pages.length; i++) {
+		const content = normalizedContent[i];
+		normalizedPages.set(pages[i].id, {
+			content,
+			index: i,
+			length: content.length
+		});
+	}
+	return normalizedPages;
+};
+const buildCumulativeOffsets = (pageIds, normalizedPages) => {
+	const cumulativeOffsets = [0];
+	let totalOffset = 0;
+	for (let i = 0; i < pageIds.length; i++) {
+		const pageData = normalizedPages.get(pageIds[i]);
+		totalOffset += pageData ? pageData.length : 0;
+		if (i < pageIds.length - 1) totalOffset += 1;
+		cumulativeOffsets.push(totalOffset);
+	}
+	return cumulativeOffsets;
+};
+const hasAnyExclusionsInRange = (expandedBreakpoints, pageIds, fromIdx, toIdx) => expandedBreakpoints.some((bp) => hasExcludedPageInRange(bp.excludeSet, pageIds, fromIdx, toIdx));
+const computeWindowEndIdx = (currentFromIdx, toIdx, pageIds, maxPages) => {
+	const maxWindowPageId = pageIds[currentFromIdx] + maxPages;
+	let windowEndIdx = currentFromIdx;
+	for (let i = currentFromIdx; i <= toIdx; i++) if (pageIds[i] <= maxWindowPageId) windowEndIdx = i;
+	else break;
+	return windowEndIdx;
+};
+const computeRemainingSpan = (currentFromIdx, toIdx, pageIds) => pageIds[toIdx] - pageIds[currentFromIdx];
+const createFinalSegment = (remainingContent, currentFromIdx, toIdx, pageIds, meta, includeMeta) => createSegment(remainingContent, pageIds[currentFromIdx], currentFromIdx !== toIdx ? pageIds[toIdx] : void 0, includeMeta ? meta : void 0);
+const computePiecePages = (pieceContent, currentFromIdx, toIdx, windowEndIdx, pageIds, normalizedPages) => {
+	const actualStartIdx = pieceContent ? findActualStartPage(pieceContent, currentFromIdx, toIdx, pageIds, normalizedPages) : currentFromIdx;
+	return {
+		actualEndIdx: pieceContent ? findActualEndPage(pieceContent, actualStartIdx, windowEndIdx, pageIds, normalizedPages) : currentFromIdx,
+		actualStartIdx
+	};
+};
+const computeNextFromIdx = (remainingContent, actualEndIdx, toIdx, pageIds, normalizedPages) => {
+	let nextFromIdx = actualEndIdx;
+	if (remainingContent && actualEndIdx + 1 <= toIdx) {
+		const nextPageData = normalizedPages.get(pageIds[actualEndIdx + 1]);
+		if (nextPageData) {
+			const nextPrefix = nextPageData.content.slice(0, Math.min(30, nextPageData.length));
+			const remainingPrefix = remainingContent.trimStart().slice(0, Math.min(30, remainingContent.length));
+			if (nextPrefix && (remainingContent.startsWith(nextPrefix) || nextPageData.content.startsWith(remainingPrefix))) nextFromIdx = actualEndIdx + 1;
+		}
+	}
+	return nextFromIdx;
+};
+const createPieceSegment = (pieceContent, actualStartIdx, actualEndIdx, pageIds, meta, includeMeta) => createSegment(pieceContent, pageIds[actualStartIdx], actualEndIdx > actualStartIdx ? pageIds[actualEndIdx] : void 0, includeMeta ? meta : void 0);
+const processOversizedSegment = (segment, fromIdx, toIdx, pageIds, normalizedPages, cumulativeOffsets, expandedBreakpoints, maxPages, prefer, logger) => {
+	const result = [];
+	let remainingContent = segment.content;
+	let currentFromIdx = fromIdx;
+	let isFirstPiece = true;
+	let iterationCount = 0;
+	const maxIterations = 1e4;
+	while (currentFromIdx <= toIdx) {
+		iterationCount++;
+		if (iterationCount > maxIterations) {
+			logger?.error?.("INFINITE LOOP DETECTED! Breaking out, you should report this bug", { iterationCount: maxIterations });
+			break;
+		}
+		const remainingHasExclusions = hasAnyExclusionsInRange(expandedBreakpoints, pageIds, currentFromIdx, toIdx);
+		if (computeRemainingSpan(currentFromIdx, toIdx, pageIds) <= maxPages && !remainingHasExclusions) {
+			const finalSeg = createFinalSegment(remainingContent, currentFromIdx, toIdx, pageIds, segment.meta, isFirstPiece);
+			if (finalSeg) result.push(finalSeg);
+			break;
+		}
+		const windowEndIdx = computeWindowEndIdx(currentFromIdx, toIdx, pageIds, maxPages);
+		const windowEndPosition = findBreakpointWindowEndPosition(remainingContent, currentFromIdx, windowEndIdx, toIdx, pageIds, normalizedPages, cumulativeOffsets);
+		const windowHasExclusions = hasAnyExclusionsInRange(expandedBreakpoints, pageIds, currentFromIdx, windowEndIdx);
+		let breakPosition = -1;
+		if (windowHasExclusions) breakPosition = findExclusionBreakPosition(currentFromIdx, windowEndIdx, toIdx, pageIds, expandedBreakpoints, cumulativeOffsets);
+		if (breakPosition <= 0) breakPosition = findBreakPosition(remainingContent, currentFromIdx, toIdx, windowEndIdx, windowEndPosition, {
+			expandedBreakpoints,
+			normalizedPages,
+			pageIds,
+			prefer
+		});
+		if (breakPosition <= 0) breakPosition = windowEndPosition;
+		const pieceContent = remainingContent.slice(0, breakPosition).trim();
+		const { actualEndIdx, actualStartIdx } = computePiecePages(pieceContent, currentFromIdx, toIdx, windowEndIdx, pageIds, normalizedPages);
+		if (pieceContent) {
+			const pieceSeg = createPieceSegment(pieceContent, actualStartIdx, actualEndIdx, pageIds, segment.meta, isFirstPiece);
+			if (pieceSeg) result.push(pieceSeg);
+		}
+		remainingContent = remainingContent.slice(breakPosition).trim();
+		if (!remainingContent) break;
+		currentFromIdx = computeNextFromIdx(remainingContent, actualEndIdx, toIdx, pageIds, normalizedPages);
+		isFirstPiece = false;
+	}
+	return result;
+};
+/**
+* Applies breakpoints to oversized segments.
+*
+* Note: This is an internal engine used by `segmentPages()`.
+*/
+const applyBreakpoints = (segments, pages, normalizedContent, maxPages, breakpoints, prefer, patternProcessor, logger, pageJoiner = "space") => {
+	const pageIds = pages.map((p) => p.id);
+	const pageIdToIndex = buildPageIdToIndexMap(pageIds);
+	const normalizedPages = buildNormalizedPagesMap(pages, normalizedContent);
+	const cumulativeOffsets = buildCumulativeOffsets(pageIds, normalizedPages);
+	const expandedBreakpoints = expandBreakpoints(breakpoints, patternProcessor);
+	const result = [];
+	logger?.info?.("Starting breakpoint processing", {
+		maxPages,
+		segmentCount: segments.length
+	});
+	for (const segment of segments) {
+		const fromIdx = pageIdToIndex.get(segment.from) ?? -1;
+		const toIdx = segment.to !== void 0 ? pageIdToIndex.get(segment.to) ?? fromIdx : fromIdx;
+		const segmentSpan = (segment.to ?? segment.from) - segment.from;
+		const hasExclusions = hasAnyExclusionsInRange(expandedBreakpoints, pageIds, fromIdx, toIdx);
+		if (segmentSpan <= maxPages && !hasExclusions) {
+			result.push(segment);
+			continue;
+		}
+		const broken = processOversizedSegment(segment, fromIdx, toIdx, pageIds, normalizedPages, cumulativeOffsets, expandedBreakpoints, maxPages, prefer, logger);
+		result.push(...broken.map((s) => {
+			const segFromIdx = pageIdToIndex.get(s.from) ?? -1;
+			const segToIdx = s.to !== void 0 ? pageIdToIndex.get(s.to) ?? segFromIdx : segFromIdx;
+			if (segFromIdx >= 0 && segToIdx > segFromIdx) return {
+				...s,
+				content: applyPageJoinerBetweenPages(s.content, segFromIdx, segToIdx, pageIds, normalizedPages, pageJoiner)
+			};
+			return s;
+		}));
+	}
+	logger?.info?.("Breakpoint processing completed", { resultCount: result.length });
+	return result;
+};
+
 //#endregion
 //#region src/segmentation/match-utils.ts
 /**
@@ -613,28 +897,6 @@ const anyRuleAllowsId = (rules, pageId) => {
 	});
 };
 
-//#endregion
-//#region src/segmentation/textUtils.ts
-/**
-* Strip all HTML tags from content, keeping only text.
-*
-* @param html - HTML content
-* @returns Plain text content
-*/
-const stripHtmlTags = (html) => {
-	return html.replace(/<[^>]*>/g, "");
-};
-/**
-* Normalizes line endings to Unix-style (`\n`).
-*
-* Converts Windows (`\r\n`) and old Mac (`\r`) line endings to Unix style
-* for consistent pattern matching across platforms.
-*
-* @param content - Raw content with potentially mixed line endings
-* @returns Content with all line endings normalized to `\n`
-*/
-const normalizeLineEndings = (content) => content.includes("\r") ? content.replace(/\r\n?/g, "\n") : content;
-
 //#endregion
 //#region src/segmentation/tokens.ts
 /**
@@ -721,6 +983,7 @@ const BASE_TOKENS = {
 	dash: "[-–—ـ]",
 	fasl: ["مسألة", "فصل"].join("|"),
 	harf: "[أ-ي]",
+	harfs: "[أ-ي](?:[أ-ي\\s]*[أ-ي])?",
 	kitab: "كتاب",
 	naql: [
 		"حدثني",
@@ -1004,15 +1267,12 @@ const getAvailableTokens = () => Object.keys(TOKEN_PATTERNS);
 const getTokenPattern = (tokenName) => TOKEN_PATTERNS[tokenName];
 
 //#endregion
-//#region src/segmentation/segmenter.ts
+//#region src/segmentation/rule-regex.ts
 /**
-* Core segmentation engine for splitting Arabic text pages into logical segments.
+* Split rule → compiled regex builder.
 *
-* The segmenter takes an array of pages and applies pattern-based rules to
-* identify split points, producing segments with content, page references,
-* and optional metadata.
-*
-* @module segmenter
+* Extracted from `segmenter.ts` to reduce cognitive complexity and enable
+* independent unit testing of regex compilation and token expansion behavior.
 */
 /**
 * Checks if a regex pattern contains standard (anonymous) capturing groups.
@@ -1023,35 +1283,41 @@ const getTokenPattern = (tokenName) => TOKEN_PATTERNS[tokenName];
 * - Lookbehind assertions `(?<=...)` and `(?<!...)`
 * - Named groups `(?<name>...)` (start with `(?` so excluded here)
 *
-* **Note**: Named capture groups `(?<name>...)` ARE capturing groups but are
-* excluded by this check because they are tracked separately via the
-* `captureNames` array from token expansion. This function only detects
-* anonymous capturing groups like `(.*)`.
-*
-* @param pattern - Regex pattern string to analyze
-* @returns `true` if the pattern contains at least one anonymous capturing group
+* NOTE: Named capture groups are still captures, but they're tracked via `captureNames`.
 */
 const hasCapturingGroup = (pattern) => {
 	return /\((?!\?)/.test(pattern);
 };
 /**
-* Processes a pattern string by expanding tokens and optionally applying fuzzy matching.
+* Extracts named capture group names from a regex pattern.
 *
-* Fuzzy matching makes Arabic text diacritic-insensitive. When enabled, the
-* transform is applied to token patterns BEFORE wrapping with capture groups,
-* ensuring regex metacharacters (`(`, `)`, `|`, etc.) are not corrupted.
-*
-* @param pattern - Pattern string potentially containing `{{token}}` placeholders
-* @param fuzzy - Whether to apply diacritic-insensitive transformation
-* @returns Processed pattern with expanded tokens and capture names
+* Parses patterns like `(?<num>[0-9]+)` and returns `['num']`.
 *
 * @example
-* processPattern('{{raqms:num}} {{dash}}', false)
-* // → { pattern: '(?<num>[٠-٩]+) [-–—ـ]', captureNames: ['num'] }
+* extractNamedCaptureNames('^(?<num>[٠-٩]+)\\s+') // ['num']
+* extractNamedCaptureNames('^(?<a>\\d+)(?<b>\\w+)') // ['a', 'b']
+* extractNamedCaptureNames('^\\d+') // []
+*/
+const extractNamedCaptureNames = (pattern) => {
+	const names = [];
+	for (const match of pattern.matchAll(/\(\?<([^>]+)>/g)) names.push(match[1]);
+	return names;
+};
+/**
+* Safely compiles a regex pattern, throwing a helpful error if invalid.
+*/
+const compileRuleRegex = (pattern) => {
+	try {
+		return new RegExp(pattern, "gmu");
+	} catch (error) {
+		const message = error instanceof Error ? error.message : String(error);
+		throw new Error(`Invalid regex pattern: ${pattern}\n  Cause: ${message}`);
+	}
+};
+/**
+* Processes a pattern string by expanding tokens and optionally applying fuzzy matching.
 *
-* @example
-* processPattern('{{naql}}', true)
-* // → { pattern: 'حَ?دَّ?ثَ?نَ?ا|...', captureNames: [] }
+* Brackets `()[]` outside `{{tokens}}` are auto-escaped.
 */
 const processPattern = (pattern, fuzzy) => {
 	const { pattern: expanded, captureNames } = expandTokensWithCaptures(escapeTemplateBrackets(pattern), fuzzy ? makeDiacriticInsensitive : void 0);
@@ -1060,77 +1326,116 @@ const processPattern = (pattern, fuzzy) => {
 		pattern: expanded
 	};
 };
+const buildLineStartsAfterRegexSource = (patterns, fuzzy) => {
+	const processed = patterns.map((p) => processPattern(p, fuzzy));
+	const union = processed.map((p) => p.pattern).join("|");
+	return {
+		captureNames: processed.flatMap((p) => p.captureNames),
+		regex: `^(?:${union})(.*)`
+	};
+};
+const buildLineStartsWithRegexSource = (patterns, fuzzy) => {
+	const processed = patterns.map((p) => processPattern(p, fuzzy));
+	const union = processed.map((p) => p.pattern).join("|");
+	return {
+		captureNames: processed.flatMap((p) => p.captureNames),
+		regex: `^(?:${union})`
+	};
+};
+const buildLineEndsWithRegexSource = (patterns, fuzzy) => {
+	const processed = patterns.map((p) => processPattern(p, fuzzy));
+	const union = processed.map((p) => p.pattern).join("|");
+	return {
+		captureNames: processed.flatMap((p) => p.captureNames),
+		regex: `(?:${union})$`
+	};
+};
+const buildTemplateRegexSource = (template) => {
+	const { pattern, captureNames } = expandTokensWithCaptures(escapeTemplateBrackets(template));
+	return {
+		captureNames,
+		regex: pattern
+	};
+};
+const determineUsesCapture = (regexSource, _captureNames) => hasCapturingGroup(regexSource);
 /**
 * Builds a compiled regex and metadata from a split rule.
 *
-* Handles all pattern types:
-* - `regex`: Used as-is (no token expansion)
-* - `template`: Tokens expanded via `expandTokensWithCaptures`
-* - `lineStartsWith`: Converted to `^(?:patterns...)`
-* - `lineStartsAfter`: Converted to `^(?:patterns...)(.*)`
-* - `lineEndsWith`: Converted to `(?:patterns...)$`
-*
-* @param rule - Split rule containing pattern and options
-* @returns Compiled regex with capture metadata
+* Behavior mirrors the previous implementation in `segmenter.ts`.
 */
 const buildRuleRegex = (rule) => {
 	const s = { ...rule };
 	const fuzzy = rule.fuzzy ?? false;
 	let allCaptureNames = [];
-	/**
-	* Safely compiles a regex pattern, throwing a helpful error if invalid.
-	*
-	* @remarks
-	* This catches syntax errors only. It does NOT protect against ReDoS
-	* (catastrophic backtracking) from pathological patterns. Avoid compiling
-	* patterns from untrusted sources.
-	*/
-	const compileRegex = (pattern) => {
-		try {
-			return new RegExp(pattern, "gmu");
-		} catch (error) {
-			const message = error instanceof Error ? error.message : String(error);
-			throw new Error(`Invalid regex pattern: ${pattern}\n  Cause: ${message}`);
-		}
-	};
 	if (s.lineStartsAfter?.length) {
-		const processed = s.lineStartsAfter.map((p) => processPattern(p, fuzzy));
-		const patterns = processed.map((p) => p.pattern).join("|");
-		allCaptureNames = processed.flatMap((p) => p.captureNames);
-		s.regex = `^(?:${patterns})(.*)`;
+		const { regex, captureNames } = buildLineStartsAfterRegexSource(s.lineStartsAfter, fuzzy);
+		allCaptureNames = captureNames;
 		return {
 			captureNames: allCaptureNames,
-			regex: compileRegex(s.regex),
+			regex: compileRuleRegex(regex),
 			usesCapture: true,
 			usesLineStartsAfter: true
 		};
 	}
 	if (s.lineStartsWith?.length) {
-		const processed = s.lineStartsWith.map((p) => processPattern(p, fuzzy));
-		const patterns = processed.map((p) => p.pattern).join("|");
-		allCaptureNames = processed.flatMap((p) => p.captureNames);
-		s.regex = `^(?:${patterns})`;
+		const { regex, captureNames } = buildLineStartsWithRegexSource(s.lineStartsWith, fuzzy);
+		s.regex = regex;
+		allCaptureNames = captureNames;
 	}
 	if (s.lineEndsWith?.length) {
-		const processed = s.lineEndsWith.map((p) => processPattern(p, fuzzy));
-		const patterns = processed.map((p) => p.pattern).join("|");
-		allCaptureNames = processed.flatMap((p) => p.captureNames);
-		s.regex = `(?:${patterns})$`;
+		const { regex, captureNames } = buildLineEndsWithRegexSource(s.lineEndsWith, fuzzy);
+		s.regex = regex;
+		allCaptureNames = captureNames;
 	}
 	if (s.template) {
-		const { pattern, captureNames } = expandTokensWithCaptures(escapeTemplateBrackets(s.template));
-		s.regex = pattern;
+		const { regex, captureNames } = buildTemplateRegexSource(s.template);
+		s.regex = regex;
 		allCaptureNames = [...allCaptureNames, ...captureNames];
 	}
 	if (!s.regex) throw new Error("Rule must specify exactly one pattern type: regex, template, lineStartsWith, lineStartsAfter, or lineEndsWith");
-	const usesCapture = hasCapturingGroup(s.regex) || allCaptureNames.length > 0;
+	if (allCaptureNames.length === 0) allCaptureNames = extractNamedCaptureNames(s.regex);
+	const usesCapture = determineUsesCapture(s.regex, allCaptureNames);
 	return {
 		captureNames: allCaptureNames,
-		regex: compileRegex(s.regex),
+		regex: compileRuleRegex(s.regex),
 		usesCapture,
 		usesLineStartsAfter: false
 	};
 };
+
+//#endregion
+//#region src/segmentation/textUtils.ts
+/**
+* Strip all HTML tags from content, keeping only text.
+*
+* @param html - HTML content
+* @returns Plain text content
+*/
+const stripHtmlTags = (html) => {
+	return html.replace(/<[^>]*>/g, "");
+};
+/**
+* Normalizes line endings to Unix-style (`\n`).
+*
+* Converts Windows (`\r\n`) and old Mac (`\r`) line endings to Unix style
+* for consistent pattern matching across platforms.
+*
+* @param content - Raw content with potentially mixed line endings
+* @returns Content with all line endings normalized to `\n`
+*/
+const normalizeLineEndings = (content) => content.includes("\r") ? content.replace(/\r\n?/g, "\n") : content;
+
+//#endregion
+//#region src/segmentation/segmenter.ts
+/**
+* Core segmentation engine for splitting Arabic text pages into logical segments.
+*
+* The segmenter takes an array of pages and applies pattern-based rules to
+* identify split points, producing segments with content, page references,
+* and optional metadata.
+*
+* @module segmenter
+*/
 /**
 * Builds a concatenated content string and page mapping from input pages.
 *
@@ -1200,6 +1505,62 @@ const buildPageMap = (pages) => {
 	};
 };
 /**
+* Deduplicate split points by index, preferring ones with more information.
+*
+* Preference rules (when same index):
+* - Prefer a split with `contentStartOffset` (needed for `lineStartsAfter` marker stripping)
+* - Otherwise prefer a split with `meta` over one without
+*/
+const dedupeSplitPoints = (splitPoints) => {
+	const byIndex = /* @__PURE__ */ new Map();
+	for (const p of splitPoints) {
+		const existing = byIndex.get(p.index);
+		if (!existing) {
+			byIndex.set(p.index, p);
+			continue;
+		}
+		if (p.contentStartOffset !== void 0 && existing.contentStartOffset === void 0 || p.meta !== void 0 && existing.meta === void 0) byIndex.set(p.index, p);
+	}
+	const unique = [...byIndex.values()];
+	unique.sort((a, b) => a.index - b.index);
+	return unique;
+};
+/**
+* If no structural rules produced segments, create a single segment spanning all pages.
+* This allows breakpoint processing to still run.
+*/
+const ensureFallbackSegment = (segments, pages, normalizedContent, pageJoiner) => {
+	if (segments.length > 0 || pages.length === 0) return segments;
+	const firstPage = pages[0];
+	const lastPage = pages[pages.length - 1];
+	const joinChar = pageJoiner === "newline" ? "\n" : " ";
+	const allContent = normalizedContent.join(joinChar).trim();
+	if (!allContent) return segments;
+	const initialSeg = {
+		content: allContent,
+		from: firstPage.id
+	};
+	if (lastPage.id !== firstPage.id) initialSeg.to = lastPage.id;
+	return [initialSeg];
+};
+const collectSplitPointsFromRules = (rules, matchContent, pageMap) => {
+	const collectSplitPointsFromRule = (rule) => {
+		const { regex, usesCapture, captureNames, usesLineStartsAfter } = buildRuleRegex(rule);
+		return filterByOccurrence(filterByConstraints(findMatches(matchContent, regex, usesCapture, captureNames), rule, pageMap.getId), rule.occurrence).map((m) => {
+			const isLineStartsAfter = usesLineStartsAfter && m.captured !== void 0;
+			const markerLength = isLineStartsAfter ? m.end - m.captured.length - m.start : 0;
+			return {
+				capturedContent: isLineStartsAfter ? void 0 : m.captured,
+				contentStartOffset: isLineStartsAfter ? markerLength : void 0,
+				index: (rule.split ?? "at") === "at" ? m.start : m.end,
+				meta: rule.meta,
+				namedCaptures: m.namedCaptures
+			};
+		});
+	};
+	return rules.flatMap(collectSplitPointsFromRule);
+};
+/**
 * Executes a regex against content and extracts match results with capture information.
 *
 * @param content - Full content string to search
@@ -1282,202 +1643,6 @@ const convertPageBreaks = (content, startOffset, pageBreaks) => {
 * @param prefer - 'longer' for last match, 'shorter' for first match
 * @returns Processed segments with oversized ones broken up
 */
-const applyBreakpoints = (segments, pages, normalizedContent, maxPages, breakpoints, prefer, logger) => {
-	const findExclusionBreakPosition = (currentFromIdx, windowEndIdx, toIdx, pageIds$1, expandedBreakpoints$1, cumulativeOffsets$1) => {
-		const startingPageId = pageIds$1[currentFromIdx];
-		if (expandedBreakpoints$1.some((bp) => bp.excludeSet.has(startingPageId)) && currentFromIdx < toIdx) return cumulativeOffsets$1[currentFromIdx + 1] - cumulativeOffsets$1[currentFromIdx];
-		for (let pageIdx = currentFromIdx + 1; pageIdx <= windowEndIdx; pageIdx++) {
-			const pageId = pageIds$1[pageIdx];
-			if (expandedBreakpoints$1.some((bp) => bp.excludeSet.has(pageId))) return cumulativeOffsets$1[pageIdx] - cumulativeOffsets$1[currentFromIdx];
-		}
-		return -1;
-	};
-	const pageIds = pages.map((p) => p.id);
-	const pageIdToIndex = new Map(pageIds.map((id, i) => [id, i]));
-	const normalizedPages = /* @__PURE__ */ new Map();
-	for (let i = 0; i < pages.length; i++) {
-		const content = normalizedContent[i];
-		normalizedPages.set(pages[i].id, {
-			content,
-			index: i,
-			length: content.length
-		});
-	}
-	const cumulativeOffsets = [0];
-	let totalOffset = 0;
-	for (let i = 0; i < pageIds.length; i++) {
-		const pageData = normalizedPages.get(pageIds[i]);
-		totalOffset += pageData ? pageData.length : 0;
-		if (i < pageIds.length - 1) totalOffset += 1;
-		cumulativeOffsets.push(totalOffset);
-	}
-	const patternProcessor = (p) => processPattern(p, false).pattern;
-	const expandedBreakpoints = expandBreakpoints(breakpoints, patternProcessor);
-	const result = [];
-	logger?.info?.("Starting breakpoint processing", {
-		maxPages,
-		segmentCount: segments.length
-	});
-	for (const segment of segments) {
-		const fromIdx = pageIdToIndex.get(segment.from) ?? -1;
-		const toIdx = segment.to !== void 0 ? pageIdToIndex.get(segment.to) ?? fromIdx : fromIdx;
-		logger?.debug?.("Processing segment", {
-			contentLength: segment.content.length,
-			contentPreview: segment.content.slice(0, 100),
-			from: segment.from,
-			fromIdx,
-			to: segment.to,
-			toIdx
-		});
-		const segmentSpan = (segment.to ?? segment.from) - segment.from;
-		const hasExclusions = expandedBreakpoints.some((bp) => hasExcludedPageInRange(bp.excludeSet, pageIds, fromIdx, toIdx));
-		if (segmentSpan <= maxPages && !hasExclusions) {
-			logger?.trace?.("Segment within limit, keeping as-is");
-			result.push(segment);
-			continue;
-		}
-		logger?.debug?.("Segment exceeds limit or has exclusions, breaking it up");
-		let remainingContent = segment.content;
-		let currentFromIdx = fromIdx;
-		let isFirstPiece = true;
-		let iterationCount = 0;
-		const maxIterations = 1e4;
-		while (currentFromIdx <= toIdx) {
-			iterationCount++;
-			if (iterationCount > maxIterations) {
-				logger?.error?.("INFINITE LOOP DETECTED! Breaking out, you should report this bug", { iterationCount: maxIterations });
-				logger?.error?.("Loop state", {
-					currentFromIdx,
-					remainingContentLength: remainingContent.length,
-					toIdx
-				});
-				break;
-			}
-			const remainingSpan = pageIds[toIdx] - pageIds[currentFromIdx];
-			logger?.trace?.("Loop iteration", {
-				currentFromIdx,
-				currentPageId: pageIds[currentFromIdx],
-				iterationCount,
-				remainingContentLength: remainingContent.length,
-				remainingContentPreview: remainingContent.slice(0, 80),
-				remainingSpan,
-				toIdx,
-				toPageId: pageIds[toIdx]
-			});
-			const remainingHasExclusions = expandedBreakpoints.some((bp) => hasExcludedPageInRange(bp.excludeSet, pageIds, currentFromIdx, toIdx));
-			if (remainingSpan <= maxPages && !remainingHasExclusions) {
-				logger?.debug?.("Remaining span within limit, outputting final segment");
-				const finalSeg = createSegment(remainingContent, pageIds[currentFromIdx], currentFromIdx !== toIdx ? pageIds[toIdx] : void 0, isFirstPiece ? segment.meta : void 0);
-				if (finalSeg) result.push(finalSeg);
-				break;
-			}
-			const currentPageId = pageIds[currentFromIdx];
-			const maxWindowPageId = currentPageId + maxPages;
-			let windowEndIdx = currentFromIdx;
-			for (let i = currentFromIdx; i <= toIdx; i++) if (pageIds[i] <= maxWindowPageId) windowEndIdx = i;
-			else break;
-			logger?.trace?.("Window calculation", {
-				currentPageId,
-				maxWindowPageId,
-				windowEndIdx,
-				windowEndPageId: pageIds[windowEndIdx]
-			});
-			const windowHasExclusions = expandedBreakpoints.some((bp) => hasExcludedPageInRange(bp.excludeSet, pageIds, currentFromIdx, windowEndIdx));
-			let breakPosition = -1;
-			if (windowHasExclusions) {
-				logger?.trace?.("Window has exclusions, finding exclusion break position");
-				breakPosition = findExclusionBreakPosition(currentFromIdx, windowEndIdx, toIdx, pageIds, expandedBreakpoints, cumulativeOffsets);
-				logger?.trace?.("Exclusion break position", { breakPosition });
-			}
-			if (breakPosition <= 0) {
-				const breakpointCtx = {
-					cumulativeOffsets,
-					expandedBreakpoints,
-					normalizedPages,
-					pageIds,
-					prefer
-				};
-				logger?.trace?.("Finding break position using patterns...");
-				breakPosition = findBreakPosition(remainingContent, currentFromIdx, toIdx, windowEndIdx, breakpointCtx);
-				logger?.trace?.("Pattern break position", { breakPosition });
-			}
-			if (breakPosition <= 0) {
-				logger?.debug?.("No pattern matched, falling back to page boundary");
-				if (windowEndIdx === currentFromIdx) {
-					logger?.trace?.("Single page window, outputting page and advancing");
-					const pageContent = cumulativeOffsets[currentFromIdx + 1] !== void 0 ? remainingContent.slice(0, cumulativeOffsets[currentFromIdx + 1] - cumulativeOffsets[currentFromIdx]) : remainingContent;
-					const pageSeg = createSegment(pageContent.trim(), pageIds[currentFromIdx], void 0, isFirstPiece ? segment.meta : void 0);
-					if (pageSeg) result.push(pageSeg);
-					remainingContent = remainingContent.slice(pageContent.length).trim();
-					currentFromIdx++;
-					isFirstPiece = false;
-					logger?.trace?.("After single page", {
-						currentFromIdx,
-						remainingContentLength: remainingContent.length
-					});
-					continue;
-				}
-				breakPosition = cumulativeOffsets[windowEndIdx + 1] - cumulativeOffsets[currentFromIdx];
-				logger?.trace?.("Multi-page window, using full window break position", { breakPosition });
-			}
-			const pieceContent = remainingContent.slice(0, breakPosition).trim();
-			logger?.trace?.("Piece extracted", {
-				breakPosition,
-				pieceContentLength: pieceContent.length,
-				pieceContentPreview: pieceContent.slice(0, 80)
-			});
-			const actualStartIdx = pieceContent ? findActualStartPage(pieceContent, currentFromIdx, toIdx, pageIds, normalizedPages) : currentFromIdx;
-			const actualEndIdx = pieceContent ? findActualEndPage(pieceContent, actualStartIdx, windowEndIdx, pageIds, normalizedPages) : currentFromIdx;
-			logger?.trace?.("Actual page indices", {
-				actualEndIdx,
-				actualStartIdx,
-				pieceHasContent: !!pieceContent
-			});
-			if (pieceContent) {
-				const pieceSeg = createSegment(pieceContent, pageIds[actualStartIdx], actualEndIdx > actualStartIdx ? pageIds[actualEndIdx] : void 0, isFirstPiece ? segment.meta : void 0);
-				if (pieceSeg) {
-					result.push(pieceSeg);
-					logger?.debug?.("Created segment", {
-						contentLength: pieceSeg.content.length,
-						from: pieceSeg.from,
-						to: pieceSeg.to
-					});
-				}
-			}
-			const prevRemainingLength = remainingContent.length;
-			remainingContent = remainingContent.slice(breakPosition).trim();
-			logger?.trace?.("After slicing remainingContent", {
-				newLength: remainingContent.length,
-				prevLength: prevRemainingLength,
-				slicedAmount: breakPosition
-			});
-			if (!remainingContent) {
-				logger?.debug?.("No remaining content, breaking out of loop");
-				break;
-			}
-			let nextFromIdx = actualEndIdx;
-			if (remainingContent && actualEndIdx + 1 <= toIdx) {
-				const nextPageData = normalizedPages.get(pageIds[actualEndIdx + 1]);
-				if (nextPageData) {
-					const nextPrefix = nextPageData.content.slice(0, Math.min(30, nextPageData.length));
-					if (nextPrefix && remainingContent.startsWith(nextPrefix)) {
-						nextFromIdx = actualEndIdx + 1;
-						logger?.trace?.("Content starts with next page prefix", { advancingTo: nextFromIdx });
-					}
-				}
-			}
-			logger?.trace?.("End of iteration", {
-				nextFromIdx,
-				prevCurrentFromIdx: currentFromIdx,
-				willAdvance: nextFromIdx !== currentFromIdx
-			});
-			currentFromIdx = nextFromIdx;
-			isFirstPiece = false;
-		}
-	}
-	logger?.info?.("Breakpoint processing completed", { resultCount: result.length });
-	return result;
-};
 /**
 * Segments pages of content based on pattern-matching rules.
 *
@@ -1521,45 +1686,15 @@ const applyBreakpoints = (segments, pages, normalizedContent, maxPages, breakpoi
 * });
 */
 const segmentPages = (pages, options) => {
-	const { rules = [], maxPages, breakpoints, prefer = "longer", logger } = options;
+	const { rules = [], maxPages, breakpoints, prefer = "longer", pageJoiner = "space", logger } = options;
 	if (!pages.length) return [];
 	const { content: matchContent, normalizedPages: normalizedContent, pageMap } = buildPageMap(pages);
-	const splitPoints = [];
-	for (const rule of rules) {
-		const { regex, usesCapture, captureNames, usesLineStartsAfter } = buildRuleRegex(rule);
-		const finalMatches = filterByOccurrence(filterByConstraints(findMatches(matchContent, regex, usesCapture, captureNames), rule, pageMap.getId), rule.occurrence);
-		for (const m of finalMatches) {
-			const isLineStartsAfter = usesLineStartsAfter && m.captured !== void 0;
-			const markerLength = isLineStartsAfter ? m.end - m.captured.length - m.start : 0;
-			splitPoints.push({
-				capturedContent: isLineStartsAfter ? void 0 : m.captured,
-				contentStartOffset: isLineStartsAfter ? markerLength : void 0,
-				index: (rule.split ?? "at") === "at" ? m.start : m.end,
-				meta: rule.meta,
-				namedCaptures: m.namedCaptures
-			});
-		}
-	}
-	const byIndex = /* @__PURE__ */ new Map();
-	for (const p of splitPoints) {
-		const existing = byIndex.get(p.index);
-		if (!existing) byIndex.set(p.index, p);
-		else if (p.contentStartOffset !== void 0 && existing.contentStartOffset === void 0 || p.meta !== void 0 && existing.meta === void 0) byIndex.set(p.index, p);
-	}
-	const unique = [...byIndex.values()];
-	unique.sort((a, b) => a.index - b.index);
-	let segments = buildSegments(unique, matchContent, pageMap, rules);
-	if (segments.length === 0 && pages.length > 0) {
-		const firstPage = pages[0];
-		const lastPage = pages[pages.length - 1];
-		const initialSeg = {
-			content: normalizedContent.join("\n").trim(),
-			from: firstPage.id
-		};
-		if (lastPage.id !== firstPage.id) initialSeg.to = lastPage.id;
-		if (initialSeg.content) segments = [initialSeg];
+	let segments = buildSegments(dedupeSplitPoints(collectSplitPointsFromRules(rules, matchContent, pageMap)), matchContent, pageMap, rules);
+	segments = ensureFallbackSegment(segments, pages, normalizedContent, pageJoiner);
+	if (maxPages !== void 0 && maxPages >= 0 && breakpoints?.length) {
+		const patternProcessor = (p) => processPattern(p, false).pattern;
+		return applyBreakpoints(segments, pages, normalizedContent, maxPages, breakpoints, prefer, patternProcessor, logger, pageJoiner);
 	}
-	if (maxPages !== void 0 && maxPages >= 0 && breakpoints?.length) return applyBreakpoints(segments, pages, normalizedContent, maxPages, breakpoints, prefer, logger);
 	return segments;
 };
 /**