flappa-doormal 2.3.1 → 2.5.0

package/dist/index.mjs

@@ -662,12 +662,24 @@ const processOversizedSegment = (segment, fromIdx, toIdx, pageIds, normalizedPag
 			break;
 		}
 		const remainingHasExclusions = hasAnyExclusionsInRange(expandedBreakpoints, pageIds, currentFromIdx, toIdx);
-		if (computeRemainingSpan(currentFromIdx, toIdx, pageIds) <= maxPages && !remainingHasExclusions) {
+		const remainingSpan = computeRemainingSpan(currentFromIdx, toIdx, pageIds);
+		if (remainingSpan <= maxPages && !remainingHasExclusions) {
 			const finalSeg = createFinalSegment(remainingContent, currentFromIdx, toIdx, pageIds, segment.meta, isFirstPiece);
 			if (finalSeg) result.push(finalSeg);
 			break;
 		}
 		const windowEndIdx = computeWindowEndIdx(currentFromIdx, toIdx, pageIds, maxPages);
+		logger?.debug?.(`[breakpoints] iteration=${iterationCount}`, {
+			currentFromIdx,
+			currentFromPageId: pageIds[currentFromIdx],
+			remainingContentStart: remainingContent.slice(0, 50),
+			remainingContentLength: remainingContent.length,
+			remainingSpan,
+			toIdx,
+			toPageId: pageIds[toIdx],
+			windowEndIdx,
+			windowEndPageId: pageIds[windowEndIdx]
+		});
 		const windowEndPosition = findBreakpointWindowEndPosition(remainingContent, currentFromIdx, windowEndIdx, toIdx, pageIds, normalizedPages, cumulativeOffsets);
 		const windowHasExclusions = hasAnyExclusionsInRange(expandedBreakpoints, pageIds, currentFromIdx, windowEndIdx);
 		let breakPosition = -1;
@@ -680,16 +692,35 @@ const processOversizedSegment = (segment, fromIdx, toIdx, pageIds, normalizedPag
 		});
 		if (breakPosition <= 0) breakPosition = windowEndPosition;
 		const pieceContent = remainingContent.slice(0, breakPosition).trim();
+		logger?.debug?.("[breakpoints] selectedBreak", {
+			breakPosition,
+			pieceContentEnd: pieceContent.slice(-50),
+			pieceContentLength: pieceContent.length,
+			windowEndPosition
+		});
 		const { actualEndIdx, actualStartIdx } = computePiecePages(pieceContent, currentFromIdx, toIdx, windowEndIdx, pageIds, normalizedPages);
 		if (pieceContent) {
 			const pieceSeg = createPieceSegment(pieceContent, actualStartIdx, actualEndIdx, pageIds, segment.meta, isFirstPiece);
 			if (pieceSeg) result.push(pieceSeg);
 		}
 		remainingContent = remainingContent.slice(breakPosition).trim();
-		if (!remainingContent) break;
+		logger?.debug?.("[breakpoints] afterSlice", {
+			actualEndIdx,
+			remainingContentLength: remainingContent.length,
+			remainingContentStart: remainingContent.slice(0, 60)
+		});
+		if (!remainingContent) {
+			logger?.debug?.("[breakpoints] done: no remaining content");
+			break;
+		}
 		currentFromIdx = computeNextFromIdx(remainingContent, actualEndIdx, toIdx, pageIds, normalizedPages);
+		logger?.debug?.("[breakpoints] nextIteration", {
+			currentFromIdx,
+			currentFromPageId: pageIds[currentFromIdx]
+		});
 		isFirstPiece = false;
 	}
+	logger?.debug?.("[breakpoints] processOversizedSegmentDone", { resultCount: result.length });
 	return result;
 };
 /**
@@ -708,6 +739,14 @@ const applyBreakpoints = (segments, pages, normalizedContent, maxPages, breakpoi
 		maxPages,
 		segmentCount: segments.length
 	});
+	logger?.debug?.("[breakpoints] inputSegments", {
+		segmentCount: segments.length,
+		segments: segments.map((s) => ({
+			contentLength: s.content.length,
+			from: s.from,
+			to: s.to
+		}))
+	});
 	for (const segment of segments) {
 		const fromIdx = pageIdToIndex.get(segment.from) ?? -1;
 		const toIdx = segment.to !== void 0 ? pageIdToIndex.get(segment.to) ?? fromIdx : fromIdx;
@@ -732,6 +771,138 @@ const applyBreakpoints = (segments, pages, normalizedContent, maxPages, breakpoi
 	return result;
 };
 
+//#endregion
+//#region src/segmentation/match-utils.ts
+/**
+* Utility functions for regex matching and result processing.
+*
+* These functions were extracted from `segmenter.ts` to reduce complexity
+* and enable independent testing. They handle match filtering, capture
+* extraction, and occurrence-based selection.
+*
+* @module match-utils
+*/
+/**
+* Extracts named capture groups from a regex match.
+*
+* Only includes groups that are in the `captureNames` list and have
+* defined values. This filters out positional captures and ensures
+* only explicitly requested named captures are returned.
+*
+* @param groups - The `match.groups` object from `RegExp.exec()`
+* @param captureNames - List of capture names to extract (from `{{token:name}}` syntax)
+* @returns Object with capture name → value pairs, or `undefined` if none found
+*
+* @example
+* const match = /(?<num>[٠-٩]+) -/.exec('٦٦٩٦ - text');
+* extractNamedCaptures(match.groups, ['num'])
+* // → { num: '٦٦٩٦' }
+*
+* @example
+* // No matching captures
+* extractNamedCaptures({}, ['num'])
+* // → undefined
+*
+* @example
+* // Undefined groups
+* extractNamedCaptures(undefined, ['num'])
+* // → undefined
+*/
+const extractNamedCaptures = (groups, captureNames) => {
+	if (!groups || captureNames.length === 0) return;
+	const namedCaptures = {};
+	for (const name of captureNames) if (groups[name] !== void 0) namedCaptures[name] = groups[name];
+	return Object.keys(namedCaptures).length > 0 ? namedCaptures : void 0;
+};
+/**
+* Gets the last defined positional capture group from a match array.
+*
+* Used for `lineStartsAfter` patterns where the content capture (`.*`)
+* is always at the end of the pattern. Named captures may shift the
+* positional indices, so we iterate backward to find the actual content.
+*
+* @param match - RegExp exec result array
+* @returns The last defined capture group value, or `undefined` if none
+*
+* @example
+* // Pattern: ^(?:(?<num>[٠-٩]+) - )(.*)
+* // Match array: ['٦٦٩٦ - content', '٦٦٩٦', 'content']
+* getLastPositionalCapture(match)
+* // → 'content'
+*
+* @example
+* // No captures
+* getLastPositionalCapture(['full match'])
+* // → undefined
+*/
+const getLastPositionalCapture = (match) => {
+	if (match.length <= 1) return;
+	for (let i = match.length - 1; i >= 1; i--) if (match[i] !== void 0) return match[i];
+};
+/**
+* Filters matches to only include those within page ID constraints.
+*
+* Applies the `min`, `max`, and `exclude` constraints from a rule to filter out
+* matches that occur on pages outside the allowed range or explicitly excluded.
+*
+* @param matches - Array of match results to filter
+* @param rule - Rule containing `min`, `max`, and/or `exclude` page constraints
+* @param getId - Function that returns the page ID for a given offset
+* @returns Filtered array containing only matches within constraints
+*
+* @example
+* const matches = [
+*   { start: 0, end: 10 },   // Page 1
+*   { start: 100, end: 110 }, // Page 5
+*   { start: 200, end: 210 }, // Page 10
+* ];
+* filterByConstraints(matches, { min: 3, max: 8 }, getId)
+* // → [{ start: 100, end: 110 }] (only page 5 match)
+*/
+const filterByConstraints = (matches, rule, getId) => {
+	return matches.filter((m) => {
+		const id = getId(m.start);
+		if (rule.min !== void 0 && id < rule.min) return false;
+		if (rule.max !== void 0 && id > rule.max) return false;
+		if (isPageExcluded(id, rule.exclude)) return false;
+		return true;
+	});
+};
+/**
+* Checks if any rule in the list allows the given page ID.
+*
+* A rule allows an ID if it falls within the rule's `min`/`max` constraints.
+* Rules without constraints allow all page IDs.
+*
+* This is used to determine whether to create a segment for content
+* that appears before any split points (the "first segment").
+*
+* @param rules - Array of rules with optional `min` and `max` constraints
+* @param pageId - Page ID to check
+* @returns `true` if at least one rule allows the page ID
+*
+* @example
+* const rules = [
+*   { min: 5, max: 10 },  // Allows pages 5-10
+*   { min: 20 },          // Allows pages 20+
+* ];
+*
+* anyRuleAllowsId(rules, 7)   // → true (first rule allows)
+* anyRuleAllowsId(rules, 3)   // → false (no rule allows)
+* anyRuleAllowsId(rules, 25)  // → true (second rule allows)
+*
+* @example
+* // Rules without constraints allow everything
+* anyRuleAllowsId([{}], 999) // → true
+*/
+const anyRuleAllowsId = (rules, pageId) => {
+	return rules.some((r) => {
+		const minOk = r.min === void 0 || pageId >= r.min;
+		const maxOk = r.max === void 0 || pageId <= r.max;
+		return minOk && maxOk;
+	});
+};
+
 //#endregion
 //#region src/segmentation/tokens.ts
 /**
@@ -798,19 +969,13 @@ const applyBreakpoints = (segments, pages, normalizedContent, maxPages, breakpoi
 * // → '{{harf}}' (unchanged - no brackets outside tokens)
 */
 const escapeTemplateBrackets = (pattern) => {
-	return pattern.replace(/(\{\{[^}]*\}\})|([()[\]])/g, (match, token, bracket) => {
+	return pattern.replace(/(\{\{[^}]*\}\})|([()[\]])/g, (_match, token, bracket) => {
 		if (token) return token;
 		return `\\${bracket}`;
 	});
 };
-/**
-* Base token definitions mapping human-readable token names to regex patterns.
-*
-* These tokens contain raw regex patterns and do not reference other tokens.
-* For composite tokens that build on these, see `COMPOSITE_TOKENS`.
-*
-* @internal
-*/
+const RUMUZ_ATOM = `(?:خت|خغ|بخ|عخ|مق|مت|عس|سي|كن|مد|قد|خد|فد|دل|كد|غد|صد|تم|فق|دق|[خرزيمنصسدفلتقع]|(?<![\\u0660-\\u0669])٤(?![\\u0660-\\u0669]))`;
+const RUMUZ_BLOCK = `${RUMUZ_ATOM}(?:\\s+${RUMUZ_ATOM})*`;
 const BASE_TOKENS = {
 	bab: "باب",
 	basmalah: ["بسم الله", "﷽"].join("|"),
@@ -818,7 +983,7 @@ const BASE_TOKENS = {
 	dash: "[-–—ـ]",
 	fasl: ["مسألة", "فصل"].join("|"),
 	harf: "[أ-ي]",
-	harfs: "[أ-ي](?:[أ-ي\\s]*[أ-ي])?",
+	harfs: "[أ-ي](?:\\s+[أ-ي])*",
 	kitab: "كتاب",
 	naql: [
 		"حدثني",
@@ -831,6 +996,7 @@ const BASE_TOKENS = {
 	].join("|"),
 	raqm: "[\\u0660-\\u0669]",
 	raqms: "[\\u0660-\\u0669]+",
+	rumuz: RUMUZ_BLOCK,
 	tarqim: "[.!?؟؛]"
 };
 /**
@@ -1090,236 +1256,18 @@ const getAvailableTokens = () => Object.keys(TOKEN_PATTERNS);
 /**
 * Gets the regex pattern for a specific token name.
 *
-* Returns the raw pattern string as defined in `TOKEN_PATTERNS`,
-* without any expansion or capture group wrapping.
-*
-* @param tokenName - The token name to look up (e.g., 'raqms', 'dash')
-* @returns The regex pattern string, or `undefined` if token doesn't exist
-*
-* @example
-* getTokenPattern('raqms')   // → '[\\u0660-\\u0669]+'
-* getTokenPattern('dash')    // → '[-–—ـ]'
-* getTokenPattern('unknown') // → undefined
-*/
-const getTokenPattern = (tokenName) => TOKEN_PATTERNS[tokenName];
-
-//#endregion
-//#region src/segmentation/fast-fuzzy-prefix.ts
-/**
-* Fast-path fuzzy prefix matching for common Arabic line-start markers.
-*
-* This exists to avoid running expensive fuzzy-expanded regex alternations over
-* a giant concatenated string. Instead, we match only at known line-start
-* offsets and perform a small deterministic comparison:
-* - Skip Arabic diacritics in the CONTENT
-* - Treat common equivalence groups as equal (ا/آ/أ/إ, ة/ه, ى/ي)
-*
-* This module is intentionally conservative: it only supports "literal"
-* token patterns (plain text alternation via `|`), not general regex.
-*/
-const isArabicDiacriticCode = (code) => code >= 1611 && code <= 1618;
-const equivKey = (ch) => {
-	switch (ch) {
-		case "آ":
-		case "أ":
-		case "إ": return "ا";
-		case "ه": return "ة";
-		case "ي": return "ى";
-		default: return ch;
-	}
-};
-/**
-* Match a fuzzy literal prefix at a given offset.
-*
-* - Skips diacritics in the content
-* - Applies equivalence groups on both content and literal
-*
-* @returns endOffset (exclusive) in CONTENT if matched; otherwise null.
-*/
-const matchFuzzyLiteralPrefixAt = (content, offset, literal) => {
-	let i = offset;
-	while (i < content.length && isArabicDiacriticCode(content.charCodeAt(i))) i++;
-	for (let j = 0; j < literal.length; j++) {
-		const litCh = literal[j];
-		while (i < content.length && isArabicDiacriticCode(content.charCodeAt(i))) i++;
-		if (i >= content.length) return null;
-		const cCh = content[i];
-		if (equivKey(cCh) !== equivKey(litCh)) return null;
-		i++;
-	}
-	while (i < content.length && isArabicDiacriticCode(content.charCodeAt(i))) i++;
-	return i;
-};
-const isLiteralOnly = (s) => {
-	return !/[\\[\]{}()^$.*+?]/.test(s);
-};
-const compileLiteralAlternation = (pattern) => {
-	if (!pattern) return null;
-	if (!isLiteralOnly(pattern)) return null;
-	const alternatives = pattern.split("|").map((s) => s.trim()).filter(Boolean);
-	if (!alternatives.length) return null;
-	return { alternatives };
-};
-/**
-* Attempt to compile a fast fuzzy rule from a single-token pattern like `{{kitab}}`.
-* Returns null if not eligible.
-*/
-const compileFastFuzzyTokenRule = (tokenTemplate) => {
-	const m = tokenTemplate.match(/^\{\{(\w+)\}\}$/);
-	if (!m) return null;
-	const token = m[1];
-	const tokenPattern = getTokenPattern(token);
-	if (!tokenPattern) return null;
-	const compiled = compileLiteralAlternation(tokenPattern);
-	if (!compiled) return null;
-	return {
-		alternatives: compiled.alternatives,
-		token
-	};
-};
-/**
-* Try matching any alternative for a compiled token at a line-start offset.
-* Returns endOffset (exclusive) on match, else null.
-*/
-const matchFastFuzzyTokenAt = (content, offset, compiled) => {
-	for (const alt of compiled.alternatives) {
-		const end = matchFuzzyLiteralPrefixAt(content, offset, alt);
-		if (end !== null) return end;
-	}
-	return null;
-};
-
-//#endregion
-//#region src/segmentation/match-utils.ts
-/**
-* Utility functions for regex matching and result processing.
-*
-* These functions were extracted from `segmenter.ts` to reduce complexity
-* and enable independent testing. They handle match filtering, capture
-* extraction, and occurrence-based selection.
-*
-* @module match-utils
-*/
-/**
-* Extracts named capture groups from a regex match.
-*
-* Only includes groups that are in the `captureNames` list and have
-* defined values. This filters out positional captures and ensures
-* only explicitly requested named captures are returned.
-*
-* @param groups - The `match.groups` object from `RegExp.exec()`
-* @param captureNames - List of capture names to extract (from `{{token:name}}` syntax)
-* @returns Object with capture name → value pairs, or `undefined` if none found
-*
-* @example
-* const match = /(?<num>[٠-٩]+) -/.exec('٦٦٩٦ - text');
-* extractNamedCaptures(match.groups, ['num'])
-* // → { num: '٦٦٩٦' }
-*
-* @example
-* // No matching captures
-* extractNamedCaptures({}, ['num'])
-* // → undefined
-*
-* @example
-* // Undefined groups
-* extractNamedCaptures(undefined, ['num'])
-* // → undefined
-*/
-const extractNamedCaptures = (groups, captureNames) => {
-	if (!groups || captureNames.length === 0) return;
-	const namedCaptures = {};
-	for (const name of captureNames) if (groups[name] !== void 0) namedCaptures[name] = groups[name];
-	return Object.keys(namedCaptures).length > 0 ? namedCaptures : void 0;
-};
-/**
-* Gets the last defined positional capture group from a match array.
-*
-* Used for `lineStartsAfter` patterns where the content capture (`.*`)
-* is always at the end of the pattern. Named captures may shift the
-* positional indices, so we iterate backward to find the actual content.
-*
-* @param match - RegExp exec result array
-* @returns The last defined capture group value, or `undefined` if none
-*
-* @example
-* // Pattern: ^(?:(?<num>[٠-٩]+) - )(.*)
-* // Match array: ['٦٦٩٦ - content', '٦٦٩٦', 'content']
-* getLastPositionalCapture(match)
-* // → 'content'
-*
-* @example
-* // No captures
-* getLastPositionalCapture(['full match'])
-* // → undefined
-*/
-const getLastPositionalCapture = (match) => {
-	if (match.length <= 1) return;
-	for (let i = match.length - 1; i >= 1; i--) if (match[i] !== void 0) return match[i];
-};
-/**
-* Filters matches to only include those within page ID constraints.
-*
-* Applies the `min`, `max`, and `exclude` constraints from a rule to filter out
-* matches that occur on pages outside the allowed range or explicitly excluded.
-*
-* @param matches - Array of match results to filter
-* @param rule - Rule containing `min`, `max`, and/or `exclude` page constraints
-* @param getId - Function that returns the page ID for a given offset
-* @returns Filtered array containing only matches within constraints
-*
-* @example
-* const matches = [
-*   { start: 0, end: 10 },   // Page 1
-*   { start: 100, end: 110 }, // Page 5
-*   { start: 200, end: 210 }, // Page 10
-* ];
-* filterByConstraints(matches, { min: 3, max: 8 }, getId)
-* // → [{ start: 100, end: 110 }] (only page 5 match)
-*/
-const filterByConstraints = (matches, rule, getId) => {
-	return matches.filter((m) => {
-		const id = getId(m.start);
-		if (rule.min !== void 0 && id < rule.min) return false;
-		if (rule.max !== void 0 && id > rule.max) return false;
-		if (isPageExcluded(id, rule.exclude)) return false;
-		return true;
-	});
-};
-/**
-* Checks if any rule in the list allows the given page ID.
-*
-* A rule allows an ID if it falls within the rule's `min`/`max` constraints.
-* Rules without constraints allow all page IDs.
-*
-* This is used to determine whether to create a segment for content
-* that appears before any split points (the "first segment").
-*
-* @param rules - Array of rules with optional `min` and `max` constraints
-* @param pageId - Page ID to check
-* @returns `true` if at least one rule allows the page ID
-*
-* @example
-* const rules = [
-*   { min: 5, max: 10 },  // Allows pages 5-10
-*   { min: 20 },          // Allows pages 20+
-* ];
+* Returns the raw pattern string as defined in `TOKEN_PATTERNS`,
+* without any expansion or capture group wrapping.
 *
-* anyRuleAllowsId(rules, 7)   // → true (first rule allows)
-* anyRuleAllowsId(rules, 3)   // → false (no rule allows)
-* anyRuleAllowsId(rules, 25)  // → true (second rule allows)
+* @param tokenName - The token name to look up (e.g., 'raqms', 'dash')
+* @returns The regex pattern string, or `undefined` if token doesn't exist
 *
 * @example
-* // Rules without constraints allow everything
-* anyRuleAllowsId([{}], 999) // → true
+* getTokenPattern('raqms')   // → '[\\u0660-\\u0669]+'
+* getTokenPattern('dash')    // → '[-–—ـ]'
+* getTokenPattern('unknown') // → undefined
 */
-const anyRuleAllowsId = (rules, pageId) => {
-	return rules.some((r) => {
-		const minOk = r.min === void 0 || pageId >= r.min;
-		const maxOk = r.max === void 0 || pageId <= r.max;
-		return minOk && maxOk;
-	});
-};
+const getTokenPattern = (tokenName) => TOKEN_PATTERNS[tokenName];
 
 //#endregion
 //#region src/segmentation/rule-regex.ts
@@ -1462,16 +1410,231 @@ const buildRuleRegex = (rule, capturePrefix) => {
 };
 
 //#endregion
-//#region src/segmentation/textUtils.ts
+//#region src/segmentation/fast-fuzzy-prefix.ts
+/**
+* Fast-path fuzzy prefix matching for common Arabic line-start markers.
+*
+* This exists to avoid running expensive fuzzy-expanded regex alternations over
+* a giant concatenated string. Instead, we match only at known line-start
+* offsets and perform a small deterministic comparison:
+* - Skip Arabic diacritics in the CONTENT
+* - Treat common equivalence groups as equal (ا/آ/أ/إ, ة/ه, ى/ي)
+*
+* This module is intentionally conservative: it only supports "literal"
+* token patterns (plain text alternation via `|`), not general regex.
+*/
+const isArabicDiacriticCode = (code) => code >= 1611 && code <= 1618;
+const equivKey = (ch) => {
+	switch (ch) {
+		case "آ":
+		case "أ":
+		case "إ": return "ا";
+		case "ه": return "ة";
+		case "ي": return "ى";
+		default: return ch;
+	}
+};
 /**
-* Strip all HTML tags from content, keeping only text.
+* Match a fuzzy literal prefix at a given offset.
+*
+* - Skips diacritics in the content
+* - Applies equivalence groups on both content and literal
 *
-* @param html - HTML content
-* @returns Plain text content
+* @returns endOffset (exclusive) in CONTENT if matched; otherwise null.
+*/
+const matchFuzzyLiteralPrefixAt = (content, offset, literal) => {
+	let i = offset;
+	while (i < content.length && isArabicDiacriticCode(content.charCodeAt(i))) i++;
+	for (let j = 0; j < literal.length; j++) {
+		const litCh = literal[j];
+		while (i < content.length && isArabicDiacriticCode(content.charCodeAt(i))) i++;
+		if (i >= content.length) return null;
+		const cCh = content[i];
+		if (equivKey(cCh) !== equivKey(litCh)) return null;
+		i++;
+	}
+	while (i < content.length && isArabicDiacriticCode(content.charCodeAt(i))) i++;
+	return i;
+};
+const isLiteralOnly = (s) => {
+	return !/[\\[\]{}()^$.*+?]/.test(s);
+};
+const compileLiteralAlternation = (pattern) => {
+	if (!pattern) return null;
+	if (!isLiteralOnly(pattern)) return null;
+	const alternatives = pattern.split("|").map((s) => s.trim()).filter(Boolean);
+	if (!alternatives.length) return null;
+	return { alternatives };
+};
+/**
+* Attempt to compile a fast fuzzy rule from a single-token pattern like `{{kitab}}`.
+* Returns null if not eligible.
+*/
+const compileFastFuzzyTokenRule = (tokenTemplate) => {
+	const m = tokenTemplate.match(/^\{\{(\w+)\}\}$/);
+	if (!m) return null;
+	const token = m[1];
+	const tokenPattern = getTokenPattern(token);
+	if (!tokenPattern) return null;
+	const compiled = compileLiteralAlternation(tokenPattern);
+	if (!compiled) return null;
+	return {
+		alternatives: compiled.alternatives,
+		token
+	};
+};
+/**
+* Try matching any alternative for a compiled token at a line-start offset.
+* Returns endOffset (exclusive) on match, else null.
 */
-const stripHtmlTags = (html) => {
-	return html.replace(/<[^>]*>/g, "");
+const matchFastFuzzyTokenAt = (content, offset, compiled) => {
+	for (const alt of compiled.alternatives) {
+		const end = matchFuzzyLiteralPrefixAt(content, offset, alt);
+		if (end !== null) return end;
+	}
+	return null;
+};
+
+//#endregion
+//#region src/segmentation/segmenter-rule-utils.ts
+const partitionRulesForMatching = (rules) => {
+	const combinableRules = [];
+	const standaloneRules = [];
+	const fastFuzzyRules = [];
+	rules.forEach((rule, index) => {
+		if (rule.fuzzy && "lineStartsWith" in rule) {
+			const compiled = rule.lineStartsWith.length === 1 ? compileFastFuzzyTokenRule(rule.lineStartsWith[0]) : null;
+			if (compiled) {
+				fastFuzzyRules.push({
+					compiled,
+					kind: "startsWith",
+					rule,
+					ruleIndex: index
+				});
+				return;
+			}
+		}
+		if (rule.fuzzy && "lineStartsAfter" in rule) {
+			const compiled = rule.lineStartsAfter.length === 1 ? compileFastFuzzyTokenRule(rule.lineStartsAfter[0]) : null;
+			if (compiled) {
+				fastFuzzyRules.push({
+					compiled,
+					kind: "startsAfter",
+					rule,
+					ruleIndex: index
+				});
+				return;
+			}
+		}
+		let isCombinable = true;
+		if ("regex" in rule && rule.regex) {
+			const hasNamedCaptures = extractNamedCaptureNames(rule.regex).length > 0;
+			const hasBackreferences = /\\[1-9]/.test(rule.regex);
+			const hasAnonymousCaptures = hasCapturingGroup(rule.regex);
+			if (hasNamedCaptures || hasBackreferences || hasAnonymousCaptures) isCombinable = false;
+		}
+		if (isCombinable) combinableRules.push({
+			index,
+			prefix: `r${index}_`,
+			rule
+		});
+		else standaloneRules.push(rule);
+	});
+	return {
+		combinableRules,
+		fastFuzzyRules,
+		standaloneRules
+	};
+};
+const createPageStartGuardChecker = (matchContent, pageMap) => {
+	const pageStartToBoundaryIndex = /* @__PURE__ */ new Map();
+	for (let i = 0; i < pageMap.boundaries.length; i++) pageStartToBoundaryIndex.set(pageMap.boundaries[i].start, i);
+	const compiledPageStartPrev = /* @__PURE__ */ new Map();
+	const getPageStartPrevRegex = (rule, ruleIndex) => {
+		if (compiledPageStartPrev.has(ruleIndex)) return compiledPageStartPrev.get(ruleIndex) ?? null;
+		const pattern = rule.pageStartGuard;
+		if (!pattern) {
+			compiledPageStartPrev.set(ruleIndex, null);
+			return null;
+		}
+		const expanded = processPattern(pattern, false).pattern;
+		const re = new RegExp(`(?:${expanded})$`, "u");
+		compiledPageStartPrev.set(ruleIndex, re);
+		return re;
+	};
+	const getPrevPageLastNonWsChar = (boundaryIndex) => {
+		if (boundaryIndex <= 0) return "";
+		const prevBoundary = pageMap.boundaries[boundaryIndex - 1];
+		for (let i = prevBoundary.end - 1; i >= prevBoundary.start; i--) {
+			const ch = matchContent[i];
+			if (!ch) continue;
+			if (/\s/u.test(ch)) continue;
+			return ch;
+		}
+		return "";
+	};
+	return (rule, ruleIndex, matchStart) => {
+		const boundaryIndex = pageStartToBoundaryIndex.get(matchStart);
+		if (boundaryIndex === void 0 || boundaryIndex === 0) return true;
+		const prevReq = getPageStartPrevRegex(rule, ruleIndex);
+		if (!prevReq) return true;
+		const lastChar = getPrevPageLastNonWsChar(boundaryIndex);
+		if (!lastChar) return false;
+		return prevReq.test(lastChar);
+	};
+};
+const collectFastFuzzySplitPoints = (matchContent, pageMap, fastFuzzyRules, passesPageStartGuard) => {
+	const splitPointsByRule = /* @__PURE__ */ new Map();
+	if (fastFuzzyRules.length === 0 || pageMap.boundaries.length === 0) return splitPointsByRule;
+	let boundaryIdx = 0;
+	let currentBoundary = pageMap.boundaries[boundaryIdx];
+	const advanceBoundaryTo = (offset) => {
+		while (currentBoundary && offset > currentBoundary.end && boundaryIdx < pageMap.boundaries.length - 1) {
+			boundaryIdx++;
+			currentBoundary = pageMap.boundaries[boundaryIdx];
+		}
+	};
+	const recordSplitPoint = (ruleIndex, sp) => {
+		const arr = splitPointsByRule.get(ruleIndex);
+		if (!arr) {
+			splitPointsByRule.set(ruleIndex, [sp]);
+			return;
+		}
+		arr.push(sp);
+	};
+	const isPageStart = (offset) => offset === currentBoundary?.start;
+	for (let lineStart = 0; lineStart <= matchContent.length;) {
+		advanceBoundaryTo(lineStart);
+		const pageId = currentBoundary?.id ?? 0;
+		if (lineStart >= matchContent.length) break;
+		for (const { compiled, kind, rule, ruleIndex } of fastFuzzyRules) {
+			if (!((rule.min === void 0 || pageId >= rule.min) && (rule.max === void 0 || pageId <= rule.max) && !isPageExcluded(pageId, rule.exclude))) continue;
+			if (isPageStart(lineStart) && !passesPageStartGuard(rule, ruleIndex, lineStart)) continue;
+			const end = matchFastFuzzyTokenAt(matchContent, lineStart, compiled);
+			if (end === null) continue;
+			const splitIndex = (rule.split ?? "at") === "at" ? lineStart : end;
+			if (kind === "startsWith") recordSplitPoint(ruleIndex, {
+				index: splitIndex,
+				meta: rule.meta
+			});
+			else {
+				const markerLength = end - lineStart;
+				recordSplitPoint(ruleIndex, {
+					contentStartOffset: (rule.split ?? "at") === "at" ? markerLength : void 0,
+					index: splitIndex,
+					meta: rule.meta
+				});
+			}
+		}
+		const nextNl = matchContent.indexOf("\n", lineStart);
+		if (nextNl === -1) break;
+		lineStart = nextNl + 1;
+	}
+	return splitPointsByRule;
 };
+
+//#endregion
+//#region src/segmentation/textUtils.ts
 /**
 * Normalizes line endings to Unix-style (`\n`).
 *
@@ -1481,7 +1644,9 @@ const stripHtmlTags = (html) => {
 * @param content - Raw content with potentially mixed line endings
 * @returns Content with all line endings normalized to `\n`
 */
-const normalizeLineEndings = (content) => content.includes("\r") ? content.replace(/\r\n?/g, "\n") : content;
+const normalizeLineEndings = (content) => {
+	return content.includes("\r") ? content.replace(/\r\n?/g, "\n") : content;
+};
 
 //#endregion
 //#region src/segmentation/segmenter.ts
@@ -1602,67 +1767,9 @@ const ensureFallbackSegment = (segments, pages, normalizedContent, pageJoiner) =
 	return [initialSeg];
 };
 const collectSplitPointsFromRules = (rules, matchContent, pageMap) => {
-	const combinableRules = [];
-	const standaloneRules = [];
-	const fastFuzzyRules = [];
-	rules.forEach((rule, index) => {
-		let isCombinable = true;
-		if (rule.fuzzy && "lineStartsWith" in rule && Array.isArray(rule.lineStartsWith)) {
-			const compiled = rule.lineStartsWith.length === 1 ? compileFastFuzzyTokenRule(rule.lineStartsWith[0]) : null;
-			if (compiled) {
-				fastFuzzyRules.push({
-					compiled,
-					rule,
-					ruleIndex: index
-				});
-				return;
-			}
-		}
-		if ("regex" in rule && rule.regex) {
-			const hasNamedCaptures = extractNamedCaptureNames(rule.regex).length > 0;
-			const hasBackreferences = /\\[1-9]/.test(rule.regex);
-			const hasAnonymousCaptures = hasCapturingGroup(rule.regex);
-			if (hasNamedCaptures || hasBackreferences || hasAnonymousCaptures) isCombinable = false;
-		}
-		if (isCombinable) combinableRules.push({
-			index,
-			prefix: `r${index}_`,
-			rule
-		});
-		else standaloneRules.push(rule);
-	});
-	const splitPointsByRule = /* @__PURE__ */ new Map();
-	if (fastFuzzyRules.length > 0) {
-		let boundaryIdx = 0;
-		let currentBoundary = pageMap.boundaries[boundaryIdx];
-		const advanceBoundaryTo = (offset) => {
-			while (currentBoundary && offset > currentBoundary.end && boundaryIdx < pageMap.boundaries.length - 1) {
-				boundaryIdx++;
-				currentBoundary = pageMap.boundaries[boundaryIdx];
-			}
-		};
-		const recordSplitPoint = (ruleIndex, sp) => {
-			if (!splitPointsByRule.has(ruleIndex)) splitPointsByRule.set(ruleIndex, []);
-			splitPointsByRule.get(ruleIndex).push(sp);
-		};
-		for (let lineStart = 0; lineStart <= matchContent.length;) {
-			advanceBoundaryTo(lineStart);
-			const pageId = currentBoundary?.id ?? 0;
-			if (lineStart >= matchContent.length) break;
-			for (const { compiled, rule, ruleIndex } of fastFuzzyRules) {
-				if (!((rule.min === void 0 || pageId >= rule.min) && (rule.max === void 0 || pageId <= rule.max) && !isPageExcluded(pageId, rule.exclude))) continue;
-				const end = matchFastFuzzyTokenAt(matchContent, lineStart, compiled);
-				if (end === null) continue;
-				recordSplitPoint(ruleIndex, {
-					index: (rule.split ?? "at") === "at" ? lineStart : end,
-					meta: rule.meta
-				});
-			}
-			const nextNl = matchContent.indexOf("\n", lineStart);
-			if (nextNl === -1) break;
-			lineStart = nextNl + 1;
-		}
-	}
+	const passesPageStartGuard = createPageStartGuardChecker(matchContent, pageMap);
+	const { combinableRules, fastFuzzyRules, standaloneRules } = partitionRulesForMatching(rules);
+	const splitPointsByRule = collectFastFuzzySplitPoints(matchContent, pageMap, fastFuzzyRules, passesPageStartGuard);
 	if (combinableRules.length > 0) {
 		const ruleRegexes = combinableRules.map(({ rule, prefix }) => {
 			const built = buildRuleRegex(rule, prefix);
@@ -1698,6 +1805,7 @@ const collectSplitPointsFromRules = (rules, matchContent, pageMap) => {
 				const end = m.index + m[0].length;
 				const pageId = pageMap.getId(start);
 				if ((rule.min === void 0 || pageId >= rule.min) && (rule.max === void 0 || pageId <= rule.max) && !isPageExcluded(pageId, rule.exclude)) {
+					if (!passesPageStartGuard(rule, originalIndex, start)) continue;
 					const sp = {
 						capturedContent: void 0,
 						contentStartOffset,
@@ -1715,7 +1823,7 @@ const collectSplitPointsFromRules = (rules, matchContent, pageMap) => {
 	}
 	const collectSplitPointsFromRule = (rule, ruleIndex) => {
 		const { regex, usesCapture, captureNames, usesLineStartsAfter } = buildRuleRegex(rule);
-		const points = filterByConstraints(findMatches(matchContent, regex, usesCapture, captureNames), rule, pageMap.getId).map((m) => {
+		const points = filterByConstraints(findMatches(matchContent, regex, usesCapture, captureNames), rule, pageMap.getId).filter((m) => passesPageStartGuard(rule, ruleIndex, m.start)).map((m) => {
 			const isLineStartsAfter = usesLineStartsAfter && m.captured !== void 0;
 			const markerLength = isLineStartsAfter ? m.end - m.captured.length - m.start : 0;
 			return {
@@ -1869,12 +1977,11 @@ const convertPageBreaks = (content, startOffset, pageBreaks) => {
 * });
 */
 const segmentPages = (pages, options) => {
-	const { rules = [], maxPages, breakpoints, prefer = "longer", pageJoiner = "space", logger } = options;
-	if (!pages.length) return [];
+	const { rules = [], maxPages = 0, breakpoints = [], prefer = "longer", pageJoiner = "space", logger } = options;
 	const { content: matchContent, normalizedPages: normalizedContent, pageMap } = buildPageMap(pages);
 	let segments = buildSegments(dedupeSplitPoints(collectSplitPointsFromRules(rules, matchContent, pageMap)), matchContent, pageMap, rules);
 	segments = ensureFallbackSegment(segments, pages, normalizedContent, pageJoiner);
-	if (maxPages !== void 0 && maxPages >= 0 && breakpoints?.length) {
+	if (maxPages >= 0 && breakpoints.length) {
 		const patternProcessor = (p) => processPattern(p, false).pattern;
 		return applyBreakpoints(segments, pages, normalizedContent, maxPages, breakpoints, prefer, patternProcessor, logger, pageJoiner);
 	}
@@ -1949,7 +2056,232 @@ const buildSegments = (splitPoints, content, pageMap, rules) => {
 };
 
 //#endregion
-//#region src/pattern-detection.ts
+//#region src/analysis.ts
+const countTokenMarkers = (pattern) => (pattern.match(/\{\{/g) ?? []).length;
+const stripWhitespacePlaceholders = (pattern) => pattern.replace(/\\s\*/g, "").replace(/[ \t]+/g, "");
+const computeSpecificity = (pattern) => {
+	const tokenCount = countTokenMarkers(pattern);
+	return {
+		literalLen: stripWhitespacePlaceholders(pattern).length,
+		tokenCount
+	};
+};
+const DEFAULT_OPTIONS = {
+	includeFirstWordFallback: true,
+	lineFilter: void 0,
+	maxExamples: 1,
+	minCount: 3,
+	minLineLength: 6,
+	normalizeArabicDiacritics: true,
+	prefixChars: 60,
+	prefixMatchers: [/^#+/u],
+	sortBy: "specificity",
+	topK: 40,
+	whitespace: "regex"
+};
+const escapeRegexLiteral = (s) => s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+const TOKEN_PRIORITY_ORDER$1 = [
+	"basmalah",
+	"kitab",
+	"bab",
+	"fasl",
+	"naql",
+	"rumuz",
+	"numbered",
+	"raqms",
+	"raqm",
+	"dash",
+	"bullet",
+	"tarqim"
+];
+const buildTokenPriority = () => {
+	const allTokens = new Set(getAvailableTokens());
+	return TOKEN_PRIORITY_ORDER$1.filter((t) => allTokens.has(t));
+};
+const collapseWhitespace = (s) => s.replace(/\s+/g, " ").trim();
+const stripArabicDiacritics = (s) => s.replace(/[\u064B-\u065F\u0670\u06D6-\u06ED\u0640]/gu, "");
+const compileTokenRegexes = (tokenNames) => {
+	const compiled = [];
+	for (const token of tokenNames) {
+		const pat = TOKEN_PATTERNS[token];
+		if (!pat) continue;
+		try {
+			compiled.push({
+				re: new RegExp(pat, "uy"),
+				token
+			});
+		} catch {}
+	}
+	return compiled;
+};
+const appendWs = (out, mode) => {
+	if (!out) return out;
+	if (mode === "space") return out.endsWith(" ") ? out : `${out} `;
+	return out.endsWith("\\s*") ? out : `${out}\\s*`;
+};
+const consumeLeadingPrefixes = (s, pos, out, prefixMatchers, whitespace) => {
+	let matchedAny = false;
+	let currentPos = pos;
+	let currentOut = out;
+	for (const re of prefixMatchers) {
+		if (currentPos >= s.length) break;
+		const m = re.exec(s.slice(currentPos));
+		if (!m || m.index !== 0 || !m[0]) continue;
+		currentOut += escapeRegexLiteral(m[0]);
+		currentPos += m[0].length;
+		matchedAny = true;
+		const wsAfter = /^[ \t]+/u.exec(s.slice(currentPos));
+		if (wsAfter) {
+			currentPos += wsAfter[0].length;
+			currentOut = appendWs(currentOut, whitespace);
+		}
+	}
+	return {
+		matchedAny,
+		out: currentOut,
+		pos: currentPos
+	};
+};
+const findBestTokenMatchAt = (s, pos, compiled, isArabicLetter) => {
+	let best = null;
+	for (const { token, re } of compiled) {
+		re.lastIndex = pos;
+		const m = re.exec(s);
+		if (!m || m.index !== pos) continue;
+		if (!best || m[0].length > best.text.length) best = {
+			text: m[0],
+			token
+		};
+	}
+	if (best?.token === "rumuz") {
+		const end = pos + best.text.length;
+		const next = end < s.length ? s[end] : "";
+		if (next && isArabicLetter(next) && !/\s/u.test(next)) return null;
+	}
+	return best;
+};
+const tokenizeLineStart = (line, tokenNames, prefixChars, includeFirstWordFallback, normalizeArabicDiacritics, prefixMatchers, whitespace) => {
+	const trimmed = collapseWhitespace(line);
+	if (!trimmed) return null;
+	const s = (normalizeArabicDiacritics ? stripArabicDiacritics(trimmed) : trimmed).slice(0, prefixChars);
+	let pos = 0;
+	let out = "";
+	let matchedAny = false;
+	let matchedToken = false;
+	const compiled = compileTokenRegexes(tokenNames);
+	const isArabicLetter = (ch) => /[\u0600-\u06FF]/u.test(ch);
+	const isCommonDelimiter = (ch) => /[:：\-–—ـ،؛.?!؟()[\]{}]/u.test(ch);
+	{
+		const consumed = consumeLeadingPrefixes(s, pos, out, prefixMatchers, whitespace);
+		pos = consumed.pos;
+		out = consumed.out;
+		matchedAny = consumed.matchedAny;
+	}
+	for (let steps = 0; steps < 6 && pos < s.length; steps++) {
+		const wsMatch = /^[ \t]+/u.exec(s.slice(pos));
+		if (wsMatch) {
+			pos += wsMatch[0].length;
+			out = appendWs(out, whitespace);
+			continue;
+		}
+		const best = findBestTokenMatchAt(s, pos, compiled, isArabicLetter);
+		if (best) {
+			if (out && !out.endsWith("\\s*")) {}
+			out += `{{${best.token}}}`;
+			matchedAny = true;
+			matchedToken = true;
+			pos += best.text.length;
+			continue;
+		}
+		if (matchedAny) {
+			const ch = s[pos];
+			if (ch && isCommonDelimiter(ch)) {
+				out += escapeRegexLiteral(ch);
+				pos += 1;
+				continue;
+			}
+		}
+		if (matchedAny) {
+			if (includeFirstWordFallback && !matchedToken) {
+				const firstWord$1 = (s.slice(pos).match(/^[^\s:،؛.?!؟]+/u) ?? [])[0];
+				if (!firstWord$1) break;
+				out += escapeRegexLiteral(firstWord$1);
+			}
+			break;
+		}
+		if (!includeFirstWordFallback) return null;
+		const firstWord = (s.slice(pos).match(/^[^\s:،؛.?!؟]+/u) ?? [])[0];
+		if (!firstWord) return null;
+		out += escapeRegexLiteral(firstWord);
+		return out;
+	}
+	if (!matchedAny) return null;
+	if (whitespace === "regex") while (out.endsWith("\\s*")) out = out.slice(0, -3);
+	else while (out.endsWith(" ")) out = out.slice(0, -1);
+	return out;
+};
+/**
+* Analyze pages and return the most common line-start patterns (top K).
+*
+* This is a pure algorithmic heuristic: it tokenizes common prefixes into a stable
+* template-ish string using the library tokens (e.g., `{{bab}}`, `{{raqms}}`, `{{rumuz}}`).
+*/
+const analyzeCommonLineStarts = (pages, options = {}) => {
+	const o = {
+		...DEFAULT_OPTIONS,
+		...options,
+		lineFilter: options.lineFilter ?? DEFAULT_OPTIONS.lineFilter,
+		prefixMatchers: options.prefixMatchers ?? DEFAULT_OPTIONS.prefixMatchers,
+		whitespace: options.whitespace ?? DEFAULT_OPTIONS.whitespace
+	};
+	const tokenPriority = buildTokenPriority();
+	const counts = /* @__PURE__ */ new Map();
+	for (const page of pages) {
+		const lines = normalizeLineEndings(page.content ?? "").split("\n");
+		for (const line of lines) {
+			const trimmed = collapseWhitespace(line);
+			if (trimmed.length < o.minLineLength) continue;
+			if (o.lineFilter && !o.lineFilter(trimmed, page.id)) continue;
+			const sig = tokenizeLineStart(trimmed, tokenPriority, o.prefixChars, o.includeFirstWordFallback, o.normalizeArabicDiacritics, o.prefixMatchers, o.whitespace);
+			if (!sig) continue;
+			const existing = counts.get(sig);
+			if (!existing) counts.set(sig, {
+				count: 1,
+				examples: [{
+					line: trimmed,
+					pageId: page.id
+				}]
+			});
+			else {
+				existing.count++;
+				if (existing.examples.length < o.maxExamples) existing.examples.push({
+					line: trimmed,
+					pageId: page.id
+				});
+			}
+		}
+	}
+	const compareSpecificityThenCount = (a, b) => {
+		const sa = computeSpecificity(a.pattern);
+		const sb = computeSpecificity(b.pattern);
+		if (sb.tokenCount !== sa.tokenCount) return sb.tokenCount - sa.tokenCount;
+		if (sb.literalLen !== sa.literalLen) return sb.literalLen - sa.literalLen;
+		if (b.count !== a.count) return b.count - a.count;
+		return a.pattern.localeCompare(b.pattern);
+	};
+	const compareCountThenSpecificity = (a, b) => {
+		if (b.count !== a.count) return b.count - a.count;
+		return compareSpecificityThenCount(a, b);
+	};
+	return [...counts.entries()].map(([pattern, v]) => ({
+		count: v.count,
+		examples: v.examples,
+		pattern
+	})).filter((p) => p.count >= o.minCount).sort(o.sortBy === "count" ? compareCountThenSpecificity : compareSpecificityThenCount).slice(0, o.topK);
+};
+
+//#endregion
+//#region src/detection.ts
 /**
 * Pattern detection utilities for recognizing template tokens in Arabic text.
 * Used to auto-detect patterns from user-highlighted text in the segmentation dialog.
@@ -1968,6 +2300,7 @@ const TOKEN_PRIORITY_ORDER = [
 	"bab",
 	"fasl",
 	"naql",
+	"rumuz",
 	"numbered",
 	"raqms",
 	"raqm",
@@ -1986,6 +2319,17 @@ const getTokenPriority = () => {
 	const remaining = allTokens.filter((t) => !TOKEN_PRIORITY_ORDER.includes(t)).sort();
 	return [...prioritized, ...remaining];
 };
+const isRumuzStandalone = (text, startIndex, endIndex) => {
+	const before = startIndex > 0 ? text[startIndex - 1] : "";
+	const after = endIndex < text.length ? text[endIndex] : "";
+	const isWhitespace = (ch) => !!ch && /\s/u.test(ch);
+	const isOpenBracket = (ch) => !!ch && /[([{]/u.test(ch);
+	const isRightDelimiter = (ch) => !!ch && /[:：\-–—ـ،؛.?!؟)\]}]/u.test(ch);
+	const isArabicWordy = (ch) => !!ch && /[\u0600-\u06FF]/u.test(ch);
+	const leftOk = !before || isWhitespace(before) || isOpenBracket(before) || !isArabicWordy(before);
+	const rightOk = !after || isWhitespace(after) || isRightDelimiter(after) || !isArabicWordy(after);
+	return leftOk && rightOk;
+};
 /**
 * Analyzes text and returns all detected token patterns with their positions.
 * Patterns are detected in priority order to avoid partial matches.
@@ -2017,6 +2361,7 @@ const detectTokenPatterns = (text) => {
 			while ((match = regex.exec(text)) !== null) {
 				const startIndex = match.index;
 				const endIndex = startIndex + match[0].length;
+				if (tokenName === "rumuz" && !isRumuzStandalone(text, startIndex, endIndex)) continue;
 				if (isPositionCovered(startIndex, endIndex)) continue;
 				results.push({
 					endIndex,
@@ -2104,5 +2449,5 @@ const analyzeTextForRule = (text) => {
 };
 
 //#endregion
-export { TOKEN_PATTERNS, analyzeTextForRule, containsTokens, detectTokenPatterns, escapeRegex, escapeTemplateBrackets, expandTokens, expandTokensWithCaptures, generateTemplateFromText, getAvailableTokens, getTokenPattern, makeDiacriticInsensitive, normalizeLineEndings, segmentPages, stripHtmlTags, suggestPatternConfig, templateToRegex };
+export { TOKEN_PATTERNS, analyzeCommonLineStarts, analyzeTextForRule, containsTokens, detectTokenPatterns, escapeRegex, escapeTemplateBrackets, expandTokens, expandTokensWithCaptures, generateTemplateFromText, getAvailableTokens, getTokenPattern, makeDiacriticInsensitive, segmentPages, suggestPatternConfig, templateToRegex };
 //# sourceMappingURL=index.mjs.map