flappa-doormal 1.0.0 → 2.1.0

package/dist/index.mjs

@@ -1,517 +1,1735 @@
-import { makeDiacriticInsensitive } from "bitaboom";
-
-//#region src/markers/defaults.ts
+//#region src/segmentation/fuzzy.ts
+/**
+* Fuzzy matching utilities for Arabic text.
+*
+* Provides diacritic-insensitive and character-equivalence matching for Arabic text.
+* This allows matching text regardless of:
+* - Diacritical marks (harakat/tashkeel): فَتْحَة، ضَمَّة، كَسْرَة، سُكُون، شَدَّة، تَنْوين
+* - Character equivalences: ا↔آ↔أ↔إ, ة↔ه, ى↔ي
+*
+* @module fuzzy
+*
+* @example
+* // Make a pattern diacritic-insensitive
+* const pattern = makeDiacriticInsensitive('حدثنا');
+* new RegExp(pattern, 'u').test('حَدَّثَنَا') // → true
+*/
+/**
+* Character class matching all Arabic diacritics (Tashkeel/Harakat).
+*
+* Includes the following diacritical marks:
+* - U+064B: ً (fathatan - double fatha)
+* - U+064C: ٌ (dammatan - double damma)
+* - U+064D: ٍ (kasratan - double kasra)
+* - U+064E: َ (fatha - short a)
+* - U+064F: ُ (damma - short u)
+* - U+0650: ِ (kasra - short i)
+* - U+0651: ّ (shadda - gemination)
+* - U+0652: ْ (sukun - no vowel)
+*
+* @internal
+*/
+const DIACRITICS_CLASS = "[ًٌٍَُِّْ]";
 /**
-* Default numbering style for markers
+* Groups of equivalent Arabic characters.
+*
+* Characters within the same group are considered equivalent for matching purposes.
+* This handles common variations in Arabic text where different characters are
+* used interchangeably or have the same underlying meaning.
+*
+* Equivalence groups:
+* - Alef variants: ا (bare), آ (with madda), أ (with hamza above), إ (with hamza below)
+* - Ta marbuta and Ha: ة ↔ ه (often interchangeable at word endings)
+* - Alef maqsura and Ya: ى ↔ ي (often interchangeable at word endings)
+*
+* @internal
 */
-const DEFAULT_NUMBERING = "arabic-indic";
+const EQUIV_GROUPS = [
+	[
+		"ا",
+		"آ",
+		"أ",
+		"إ"
+	],
+	["ة", "ه"],
+	["ى", "ي"]
+];
 /**
-* Default separator style for markers
+* Escapes a string for safe inclusion in a regular expression.
+*
+* Escapes all regex metacharacters: `.*+?^${}()|[\]\\`
+*
+* @param s - Any string to escape
+* @returns String with regex metacharacters escaped
+*
+* @example
+* escapeRegex('hello.world')   // → 'hello\\.world'
+* escapeRegex('[test]')        // → '\\[test\\]'
+* escapeRegex('a+b*c?')        // → 'a\\+b\\*c\\?'
 */
-const DEFAULT_SEPARATOR = "dash";
+const escapeRegex = (s) => s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
 /**
-* Default separator pattern (used when separator is a custom string)
+* Returns a regex character class for all equivalents of a given character.
+*
+* If the character belongs to one of the predefined equivalence groups
+* (e.g., ا/آ/أ/إ), the returned class will match any member of that group.
+* Otherwise, the original character is simply escaped for safe regex inclusion.
+*
+* @param ch - A single character to expand into its equivalence class
+* @returns A RegExp-safe string representing the character and its equivalents
+*
+* @example
+* getEquivClass('ا') // → '[اآأإ]' (matches any alef variant)
+* getEquivClass('ب') // → 'ب' (no equivalents, just escaped)
+* getEquivClass('.') // → '\\.' (regex metachar escaped)
+*
+* @internal
 */
-const DEFAULT_SEPARATOR_PATTERN = "[-–—ـ]";
+const getEquivClass = (ch) => {
+	for (const group of EQUIV_GROUPS) if (group.includes(ch)) return `[${group.map((c) => escapeRegex(c)).join("")}]`;
+	return escapeRegex(ch);
+};
 /**
-* Numbering patterns mapped by style
+* Performs light normalization on Arabic text for consistent matching.
+*
+* Normalization steps:
+* 1. NFC normalization (canonical decomposition then composition)
+* 2. Remove Zero-Width Joiner (U+200D) and Zero-Width Non-Joiner (U+200C)
+* 3. Collapse multiple whitespace characters to single space
+* 4. Trim leading and trailing whitespace
+*
+* This normalization preserves diacritics and letter forms while removing
+* invisible characters that could interfere with matching.
+*
+* @param str - Arabic text to normalize
+* @returns Normalized string
+*
+* @example
+* normalizeArabicLight('حَدَّثَنَا')           // → 'حَدَّثَنَا' (diacritics preserved)
+* normalizeArabicLight('بسم  الله')          // → 'بسم الله' (spaces collapsed)
+* normalizeArabicLight('  text  ')          // → 'text' (trimmed)
+*
+* @internal
 */
-const NUMBERING_PATTERNS = {
-	"arabic-indic": "[\\u0660-\\u0669]+",
-	"latin": "\\d+"
+const normalizeArabicLight = (str) => {
+	return str.normalize("NFC").replace(/[\u200C\u200D]/g, "").replace(/\s+/g, " ").trim();
 };
 /**
-* Separator patterns mapped by style
+* Creates a diacritic-insensitive regex pattern for Arabic text matching.
+*
+* Transforms input text into a regex pattern that matches the text regardless
+* of diacritical marks (harakat) and character variations. Each character in
+* the input is:
+* 1. Expanded to its equivalence class (if applicable)
+* 2. Followed by an optional diacritics matcher
+*
+* This allows matching:
+* - `حدثنا` with `حَدَّثَنَا` (with full diacritics)
+* - `الإيمان` with `الايمان` (alef variants)
+* - `صلاة` with `صلاه` (ta marbuta ↔ ha)
+*
+* @param text - Input Arabic text to make diacritic-insensitive
+* @returns Regex pattern string that matches the text with or without diacritics
+*
+* @example
+* const pattern = makeDiacriticInsensitive('حدثنا');
+* // Each char gets equivalence class + optional diacritics
+* // Result matches: حدثنا, حَدَّثَنَا, حَدَثَنَا, etc.
+*
+* @example
+* const pattern = makeDiacriticInsensitive('باب');
+* new RegExp(pattern, 'u').test('بَابٌ')  // → true
+* new RegExp(pattern, 'u').test('باب')   // → true
+*
+* @example
+* // Using with split rules
+* {
+*   lineStartsWith: ['باب'],
+*   split: 'at',
+*   fuzzy: true  // Applies makeDiacriticInsensitive internally
+* }
 */
-const SEPARATOR_PATTERNS = {
-	"colon": ":",
-	"dash": "[-–—ـ]",
-	"dot": "\\.",
-	"none": "",
-	"paren": "\\)"
+const makeDiacriticInsensitive = (text) => {
+	const diacriticsMatcher = `${DIACRITICS_CLASS}*`;
+	const norm = normalizeArabicLight(text);
+	return Array.from(norm).map((ch) => getEquivClass(ch) + diacriticsMatcher).join("");
 };
 
 //#endregion
-//#region src/markers/presets.ts
+//#region src/segmentation/breakpoint-utils.ts
+/**
+* Normalizes a breakpoint to the object form.
+* Strings are converted to { pattern: str } with no constraints.
+*
+* @param bp - Breakpoint as string or object
+* @returns Normalized BreakpointRule object
+*
+* @example
+* normalizeBreakpoint('\\n\\n')
+* // → { pattern: '\\n\\n' }
+*
+* normalizeBreakpoint({ pattern: '\\n', min: 10 })
+* // → { pattern: '\\n', min: 10 }
+*/
+const normalizeBreakpoint = (bp) => typeof bp === "string" ? { pattern: bp } : bp;
 /**
-* Default phrase lists for preset marker types.
-* Export these so users can extend them.
+* Checks if a page ID is in an excluded list (single pages or ranges).
+*
+* @param pageId - Page ID to check
+* @param excludeList - List of page IDs or [from, to] ranges to exclude
+* @returns True if page is excluded
+*
+* @example
+* isPageExcluded(5, [1, 5, 10])
+* // → true
+*
+* isPageExcluded(5, [[3, 7]])
+* // → true
+*
+* isPageExcluded(5, [[10, 20]])
+* // → false
 */
+const isPageExcluded = (pageId, excludeList) => {
+	if (!excludeList || excludeList.length === 0) return false;
+	for (const item of excludeList) if (typeof item === "number") {
+		if (pageId === item) return true;
+	} else {
+		const [from, to] = item;
+		if (pageId >= from && pageId <= to) return true;
+	}
+	return false;
+};
 /**
-* Common hadith narrator phrases (diacritic-insensitive)
-* Users can extend: [...DEFAULT_HADITH_PHRASES, 'أَخْبَرَنِي']
+* Checks if a page ID is within a breakpoint's min/max range and not excluded.
+*
+* @param pageId - Page ID to check
+* @param rule - Breakpoint rule with optional min/max/exclude constraints
+* @returns True if page is within valid range
+*
+* @example
+* isInBreakpointRange(50, { pattern: '\\n', min: 10, max: 100 })
+* // → true
+*
+* isInBreakpointRange(5, { pattern: '\\n', min: 10 })
+* // → false (below min)
 */
-const DEFAULT_HADITH_PHRASES = [
-	"حَدَّثَنَا",
-	"حدثنا",
-	"أَخْبَرَنَا",
-	"حدثني",
-	"حدَّثني",
-	"وحدثنا",
-	"حُدِّثت عن",
-	"وحَدَّثَنَا"
-];
+const isInBreakpointRange = (pageId, rule) => {
+	if (rule.min !== void 0 && pageId < rule.min) return false;
+	if (rule.max !== void 0 && pageId > rule.max) return false;
+	return !isPageExcluded(pageId, rule.exclude);
+};
 /**
-* Common basmala patterns
-* Users can extend: [...DEFAULT_BASMALA_PATTERNS, 'customPattern']
+* Builds an exclude set from a PageRange array for O(1) lookups.
+*
+* @param excludeList - List of page IDs or [from, to] ranges
+* @returns Set of all excluded page IDs
+*
+* @remarks
+* This expands ranges into explicit page IDs for fast membership checks. For typical
+* book-scale inputs (thousands of pages), this is small and keeps downstream logic
+* simple and fast. If you expect extremely large ranges (e.g., millions of pages),
+* consider avoiding broad excludes or introducing a range-based membership structure.
+*
+* @example
+* buildExcludeSet([1, 5, [10, 12]])
+* // → Set { 1, 5, 10, 11, 12 }
 */
-const DEFAULT_BASMALA_PATTERNS = [
-	"بسم الله",
-	"\\[بسم",
-	"\\[تم"
-];
+const buildExcludeSet = (excludeList) => {
+	const excludeSet = /* @__PURE__ */ new Set();
+	for (const item of excludeList || []) if (typeof item === "number") excludeSet.add(item);
+	else for (let i = item[0]; i <= item[1]; i++) excludeSet.add(i);
+	return excludeSet;
+};
+/**
+* Creates a segment with optional to and meta fields.
+* Returns null if content is empty after trimming.
+*
+* @param content - Segment content
+* @param fromPageId - Starting page ID
+* @param toPageId - Optional ending page ID (omitted if same as from)
+* @param meta - Optional metadata to attach
+* @returns Segment object or null if empty
+*
+* @example
+* createSegment('Hello world', 1, 3, { chapter: 1 })
+* // → { content: 'Hello world', from: 1, to: 3, meta: { chapter: 1 } }
+*
+* createSegment('   ', 1, undefined, undefined)
+* // → null (empty content)
+*/
+const createSegment = (content, fromPageId, toPageId, meta) => {
+	const trimmed = content.trim();
+	if (!trimmed) return null;
+	const seg = {
+		content: trimmed,
+		from: fromPageId
+	};
+	if (toPageId !== void 0 && toPageId !== fromPageId) seg.to = toPageId;
+	if (meta) seg.meta = meta;
+	return seg;
+};
+/**
+* Expands breakpoint patterns and pre-computes exclude sets.
+*
+* @param breakpoints - Array of breakpoint patterns or rules
+* @param processPattern - Function to expand tokens in patterns
+* @returns Array of expanded breakpoints with compiled regexes
+*
+* @remarks
+* This function compiles regex patterns dynamically. This can be a ReDoS vector
+* if patterns come from untrusted sources. In typical usage, breakpoint rules
+* are application configuration, not user input.
+*/
+const expandBreakpoints = (breakpoints, processPattern$1) => breakpoints.map((bp) => {
+	const rule = normalizeBreakpoint(bp);
+	const excludeSet = buildExcludeSet(rule.exclude);
+	const skipWhenRegex = rule.skipWhen !== void 0 ? (() => {
+		const expandedSkip = processPattern$1(rule.skipWhen);
+		try {
+			return new RegExp(expandedSkip, "mu");
+		} catch (error) {
+			const message = error instanceof Error ? error.message : String(error);
+			throw new Error(`Invalid breakpoint skipWhen regex: ${rule.skipWhen}\n  Cause: ${message}`);
+		}
+	})() : null;
+	if (rule.pattern === "") return {
+		excludeSet,
+		regex: null,
+		rule,
+		skipWhenRegex
+	};
+	const expanded = processPattern$1(rule.pattern);
+	try {
+		return {
+			excludeSet,
+			regex: new RegExp(expanded, "gmu"),
+			rule,
+			skipWhenRegex
+		};
+	} catch (error) {
+		const message = error instanceof Error ? error.message : String(error);
+		throw new Error(`Invalid breakpoint regex: ${rule.pattern}\n  Cause: ${message}`);
+	}
+});
+/**
+* Finds the actual ending page index by searching backwards for page content prefix.
+* Used to determine which page a segment actually ends on based on content matching.
+*
+* @param pieceContent - Content of the segment piece
+* @param currentFromIdx - Current starting index in pageIds
+* @param toIdx - Maximum ending index to search
+* @param pageIds - Array of page IDs
+* @param normalizedPages - Map of page ID to normalized content
+* @returns The actual ending page index
+*/
+const findActualEndPage = (pieceContent, currentFromIdx, toIdx, pageIds, normalizedPages) => {
+	for (let pi = toIdx; pi > currentFromIdx; pi--) {
+		const pageData = normalizedPages.get(pageIds[pi]);
+		if (pageData) {
+			const checkPortion = pageData.content.slice(0, Math.min(30, pageData.length));
+			if (checkPortion.length > 0 && pieceContent.indexOf(checkPortion) > 0) return pi;
+		}
+	}
+	return currentFromIdx;
+};
+/**
+* Finds the actual starting page index by searching forwards for page content prefix.
+* Used to determine which page content actually starts from based on content matching.
+*
+* This is the counterpart to findActualEndPage - it searches forward to find which
+* page the content starts on, rather than which page it ends on.
+*
+* @param pieceContent - Content of the segment piece
+* @param currentFromIdx - Current starting index in pageIds
+* @param toIdx - Maximum ending index to search
+* @param pageIds - Array of page IDs
+* @param normalizedPages - Map of page ID to normalized content
+* @returns The actual starting page index
+*/
+const findActualStartPage = (pieceContent, currentFromIdx, toIdx, pageIds, normalizedPages) => {
+	const trimmedPiece = pieceContent.trimStart();
+	if (!trimmedPiece) return currentFromIdx;
+	for (let pi = currentFromIdx; pi <= toIdx; pi++) {
+		const pageData = normalizedPages.get(pageIds[pi]);
+		if (pageData) {
+			const pagePrefix = pageData.content.slice(0, Math.min(30, pageData.length)).trim();
+			const piecePrefix = trimmedPiece.slice(0, Math.min(30, trimmedPiece.length));
+			if (pagePrefix.length > 0) {
+				if (trimmedPiece.startsWith(pagePrefix)) return pi;
+				if (pageData.content.trimStart().startsWith(piecePrefix)) return pi;
+			}
+		}
+	}
+	return currentFromIdx;
+};
+/**
+* Checks if any page in a range is excluded by the given exclude set.
+*
+* @param excludeSet - Set of excluded page IDs
+* @param pageIds - Array of page IDs
+* @param fromIdx - Start index (inclusive)
+* @param toIdx - End index (inclusive)
+* @returns True if any page in range is excluded
+*/
+const hasExcludedPageInRange = (excludeSet, pageIds, fromIdx, toIdx) => {
+	if (excludeSet.size === 0) return false;
+	for (let pageIdx = fromIdx; pageIdx <= toIdx; pageIdx++) if (excludeSet.has(pageIds[pageIdx])) return true;
+	return false;
+};
+/**
+* Finds the position of the next page content within remaining content.
+* Returns -1 if not found.
+*
+* @param remainingContent - Content to search in
+* @param nextPageData - Normalized data for the next page
+* @returns Position of next page content, or -1 if not found
+*/
+const findNextPagePosition = (remainingContent, nextPageData) => {
+	const searchPrefix = nextPageData.content.trim().slice(0, Math.min(30, nextPageData.length));
+	if (searchPrefix.length === 0) return -1;
+	const pos = remainingContent.indexOf(searchPrefix);
+	return pos > 0 ? pos : -1;
+};
+/**
+* Finds matches within a window and returns the selected position based on preference.
+*
+* @param windowContent - Content to search
+* @param regex - Regex to match
+* @param prefer - 'longer' for last match, 'shorter' for first match
+* @returns Break position after the selected match, or -1 if no matches
+*/
+const findPatternBreakPosition = (windowContent, regex, prefer) => {
+	const matches = [];
+	for (const m of windowContent.matchAll(regex)) matches.push({
+		index: m.index,
+		length: m[0].length
+	});
+	if (matches.length === 0) return -1;
+	const selected = prefer === "longer" ? matches[matches.length - 1] : matches[0];
+	return selected.index + selected.length;
+};
+/**
+* Tries to find a break position within the current window using breakpoint patterns.
+* Returns the break position or -1 if no suitable break was found.
+*
+* @param remainingContent - Content remaining to be segmented
+* @param currentFromIdx - Current starting page index
+* @param toIdx - Ending page index
+* @param windowEndIdx - Maximum window end index
+* @param ctx - Breakpoint context with page data and patterns
+* @returns Break position in the content, or -1 if no break found
+*/
+const findBreakPosition = (remainingContent, currentFromIdx, toIdx, windowEndIdx, ctx) => {
+	const { pageIds, normalizedPages, cumulativeOffsets, expandedBreakpoints, prefer } = ctx;
+	for (const { rule, regex, excludeSet, skipWhenRegex } of expandedBreakpoints) {
+		if (!isInBreakpointRange(pageIds[currentFromIdx], rule)) continue;
+		if (hasExcludedPageInRange(excludeSet, pageIds, currentFromIdx, windowEndIdx)) continue;
+		if (skipWhenRegex?.test(remainingContent)) continue;
+		if (regex === null) {
+			const nextPageIdx = windowEndIdx + 1;
+			if (nextPageIdx <= toIdx) {
+				const nextPageData = normalizedPages.get(pageIds[nextPageIdx]);
+				if (nextPageData) {
+					const pos = findNextPagePosition(remainingContent, nextPageData);
+					if (pos > 0) return pos;
+				}
+			}
+			return Math.min(cumulativeOffsets[windowEndIdx + 1] - cumulativeOffsets[currentFromIdx], remainingContent.length);
+		}
+		const windowEndPosition = Math.min(cumulativeOffsets[windowEndIdx + 1] - cumulativeOffsets[currentFromIdx], remainingContent.length);
+		const breakPos = findPatternBreakPosition(remainingContent.slice(0, windowEndPosition), regex, prefer);
+		if (breakPos > 0) return breakPos;
+	}
+	return -1;
+};
 
 //#endregion
-//#region src/markers/tokens.ts
+//#region src/segmentation/match-utils.ts
+/**
+* Utility functions for regex matching and result processing.
+*
+* These functions were extracted from `segmenter.ts` to reduce complexity
+* and enable independent testing. They handle match filtering, capture
+* extraction, and occurrence-based selection.
+*
+* @module match-utils
+*/
 /**
-* Token definitions for pattern templates.
-* Tokens provide a readable alternative to raw regex patterns.
+* Extracts named capture groups from a regex match.
+*
+* Only includes groups that are in the `captureNames` list and have
+* defined values. This filters out positional captures and ensures
+* only explicitly requested named captures are returned.
+*
+* @param groups - The `match.groups` object from `RegExp.exec()`
+* @param captureNames - List of capture names to extract (from `{{token:name}}` syntax)
+* @returns Object with capture name → value pairs, or `undefined` if none found
+*
+* @example
+* const match = /(?<num>[٠-٩]+) -/.exec('٦٦٩٦ - text');
+* extractNamedCaptures(match.groups, ['num'])
+* // → { num: '٦٦٩٦' }
+*
+* @example
+* // No matching captures
+* extractNamedCaptures({}, ['num'])
+* // → undefined
+*
+* @example
+* // Undefined groups
+* extractNamedCaptures(undefined, ['num'])
+* // → undefined
 */
+const extractNamedCaptures = (groups, captureNames) => {
+	if (!groups || captureNames.length === 0) return;
+	const namedCaptures = {};
+	for (const name of captureNames) if (groups[name] !== void 0) namedCaptures[name] = groups[name];
+	return Object.keys(namedCaptures).length > 0 ? namedCaptures : void 0;
+};
 /**
-* Standard tokens for building marker patterns.
-* Use these in templates like: '{num} {dash}' instead of '[\\u0660-\\u0669]+ [-–—ـ]'
+* Gets the last defined positional capture group from a match array.
+*
+* Used for `lineStartsAfter` patterns where the content capture (`.*`)
+* is always at the end of the pattern. Named captures may shift the
+* positional indices, so we iterate backward to find the actual content.
+*
+* @param match - RegExp exec result array
+* @returns The last defined capture group value, or `undefined` if none
+*
+* @example
+* // Pattern: ^(?:(?<num>[٠-٩]+) - )(.*)
+* // Match array: ['٦٦٩٦ - content', '٦٦٩٦', 'content']
+* getLastPositionalCapture(match)
+* // → 'content'
+*
+* @example
+* // No captures
+* getLastPositionalCapture(['full match'])
+* // → undefined
 */
-const TOKENS = {
-	bullet: "[•*°]",
-	colon: ":",
-	comma: "،",
-	content: "(.*)",
-	dash: "[-–—ـ]",
-	dot: "\\.",
-	latin: "\\d+",
-	letter: "[أ-ي]",
-	num: "[\\u0660-\\u0669]+",
-	paren: "\\)",
-	s: "\\s?",
-	slash: "/",
-	space: "\\s+"
+const getLastPositionalCapture = (match) => {
+	if (match.length <= 1) return;
+	for (let i = match.length - 1; i >= 1; i--) if (match[i] !== void 0) return match[i];
+};
+/**
+* Filters matches to only include those within page ID constraints.
+*
+* Applies the `min`, `max`, and `exclude` constraints from a rule to filter out
+* matches that occur on pages outside the allowed range or explicitly excluded.
+*
+* @param matches - Array of match results to filter
+* @param rule - Rule containing `min`, `max`, and/or `exclude` page constraints
+* @param getId - Function that returns the page ID for a given offset
+* @returns Filtered array containing only matches within constraints
+*
+* @example
+* const matches = [
+*   { start: 0, end: 10 },   // Page 1
+*   { start: 100, end: 110 }, // Page 5
+*   { start: 200, end: 210 }, // Page 10
+* ];
+* filterByConstraints(matches, { min: 3, max: 8 }, getId)
+* // → [{ start: 100, end: 110 }] (only page 5 match)
+*/
+const filterByConstraints = (matches, rule, getId) => {
+	return matches.filter((m) => {
+		const id = getId(m.start);
+		if (rule.min !== void 0 && id < rule.min) return false;
+		if (rule.max !== void 0 && id > rule.max) return false;
+		if (isPageExcluded(id, rule.exclude)) return false;
+		return true;
+	});
+};
+/**
+* Filters matches based on occurrence setting (first, last, or all).
+*
+* Applies occurrence-based selection to a list of matches:
+* - `'all'` or `undefined`: Return all matches (default)
+* - `'first'`: Return only the first match
+* - `'last'`: Return only the last match
+*
+* @param matches - Array of match results to filter
+* @param occurrence - Which occurrence(s) to keep
+* @returns Filtered array based on occurrence setting
+*
+* @example
+* const matches = [{ start: 0 }, { start: 10 }, { start: 20 }];
+*
+* filterByOccurrence(matches, 'first')
+* // → [{ start: 0 }]
+*
+* filterByOccurrence(matches, 'last')
+* // → [{ start: 20 }]
+*
+* filterByOccurrence(matches, 'all')
+* // → [{ start: 0 }, { start: 10 }, { start: 20 }]
+*
+* filterByOccurrence(matches, undefined)
+* // → [{ start: 0 }, { start: 10 }, { start: 20 }] (default: all)
+*/
+const filterByOccurrence = (matches, occurrence) => {
+	if (!matches.length) return [];
+	if (occurrence === "first") return [matches[0]];
+	if (occurrence === "last") return [matches[matches.length - 1]];
+	return matches;
+};
+/**
+* Checks if any rule in the list allows the given page ID.
+*
+* A rule allows an ID if it falls within the rule's `min`/`max` constraints.
+* Rules without constraints allow all page IDs.
+*
+* This is used to determine whether to create a segment for content
+* that appears before any split points (the "first segment").
+*
+* @param rules - Array of rules with optional `min` and `max` constraints
+* @param pageId - Page ID to check
+* @returns `true` if at least one rule allows the page ID
+*
+* @example
+* const rules = [
+*   { min: 5, max: 10 },  // Allows pages 5-10
+*   { min: 20 },          // Allows pages 20+
+* ];
+*
+* anyRuleAllowsId(rules, 7)   // → true (first rule allows)
+* anyRuleAllowsId(rules, 3)   // → false (no rule allows)
+* anyRuleAllowsId(rules, 25)  // → true (second rule allows)
+*
+* @example
+* // Rules without constraints allow everything
+* anyRuleAllowsId([{}], 999) // → true
+*/
+const anyRuleAllowsId = (rules, pageId) => {
+	return rules.some((r) => {
+		const minOk = r.min === void 0 || pageId >= r.min;
+		const maxOk = r.max === void 0 || pageId <= r.max;
+		return minOk && maxOk;
+	});
 };
 
 //#endregion
-//#region src/markers/template-parser.ts
-/**
-* Expands a template string into a regex pattern using named capture groups.
-* Always creates three groups: full (entire match), marker (just the marker), content (clean text).
-* 
-* The content group uses [\s\S]*? (non-greedy) to match across newlines but stop at next marker.
-* 
-* @param template - Template string with {token} placeholders
-* @param options - Optional configuration
-* @returns Regex pattern string with named groups
-* 
-* @example
-* expandTemplate('{num} {dash}')
-* // Returns: ^(?<full>(?<marker>[\\u0660-\\u0669]+\\s?[-–—ـ])(?<content>[\\s\\S]*?))
-*/
-function expandTemplate(template, options) {
-	const tokenMap = options?.tokens || TOKENS;
-	let expandedMarker = template;
-	for (const [token, pattern] of Object.entries(tokenMap)) {
-		const placeholder = `{${token}}`;
-		expandedMarker = expandedMarker.replaceAll(placeholder, pattern);
-	}
-	return String.raw`^(?<full>(?<marker>${expandedMarker})(?<content>[\s\S]*))`;
-}
-/**
-* Create a custom token map by extending the base tokens.
-* 
-* @param customTokens - Custom token definitions
-* @returns Combined token map
-* 
-* @example
-* const myTokens = createTokenMap({
-*   verse: '\\[[\\u0660-\\u0669]+\\]',
-*   tafsir: 'تفسير'
-* });
+//#region src/segmentation/textUtils.ts
+/**
+* Strip all HTML tags from content, keeping only text.
+*
+* @param html - HTML content
+* @returns Plain text content
 */
-function createTokenMap(customTokens) {
-	return {
-		...TOKENS,
-		...customTokens
-	};
-}
-/**
-* Validates a template string.
-* 
-* @param template - Template to validate
-* @param tokens - Token map to validate against
-* @returns Validation result with errors if invalid
-* 
-* @example
-* validateTemplate('{num} {dash}')
-* // Returns: { valid: true }
-* 
-* validateTemplate('{invalid}')
-* // Returns: { valid: false, errors: ['Unknown token: {invalid}'] }
-*/
-function validateTemplate(template, tokens = TOKENS) {
-	const unknownTokens = (template.match(/\{(\w+)\}/g) || []).map((t) => t.slice(1, -1)).filter((name) => !tokens[name]);
-	if (unknownTokens.length > 0) return {
-		valid: false,
-		errors: [`Unknown tokens: ${unknownTokens.map((t) => `{${t}}`).join(", ")}`, `Available tokens: ${Object.keys(tokens).map((t) => `{${t}}`).join(", ")}`]
-	};
-	return { valid: true };
-}
+const stripHtmlTags = (html) => {
+	return html.replace(/<[^>]*>/g, "");
+};
+/**
+* Normalizes line endings to Unix-style (`\n`).
+*
+* Converts Windows (`\r\n`) and old Mac (`\r`) line endings to Unix style
+* for consistent pattern matching across platforms.
+*
+* @param content - Raw content with potentially mixed line endings
+* @returns Content with all line endings normalized to `\n`
+*/
+const normalizeLineEndings = (content) => content.replace(/\r\n?/g, "\n");
 
 //#endregion
-//#region src/markers/type-generators.ts
+//#region src/segmentation/tokens.ts
 /**
-* Generates a regular expression for pattern-type markers.
+* Token-based template system for Arabic text pattern matching.
 *
-* Supports two modes:
-* 1. Template-based: Uses the `template` field with token expansion
-* 2. Pattern-based: Uses the raw `pattern` field as-is
+* This module provides a human-readable way to define regex patterns using
+* `{{token}}` placeholders that expand to their regex equivalents. It supports
+* named capture groups for extracting matched values into metadata.
 *
-* @param config - Marker configuration with either `template` or `pattern` field
-* @returns A compiled RegExp object for matching the pattern
-* @throws {Error} When neither `template` nor `pattern` is provided
+* @module tokens
 *
 * @example
-* // Using template
-* const regex = generatePatternRegex({ type: 'pattern', template: '{num} {dash}' });
+* // Simple token expansion
+* expandTokens('{{raqms}} {{dash}}')
+* // → '[\\u0660-\\u0669]+ [-–—ـ]'
 *
 * @example
-* // Using raw pattern
-* const regex = generatePatternRegex({ type: 'pattern', pattern: '^\\d+' });
+* // Named capture groups
+* expandTokensWithCaptures('{{raqms:num}} {{dash}}')
+* // → { pattern: '(?<num>[\\u0660-\\u0669]+) [-–—ـ]', captureNames: ['num'], hasCaptures: true }
+*/
+/**
+* Token definitions mapping human-readable token names to regex patterns.
+*
+* Tokens are used in template strings with double-brace syntax:
+* - `{{token}}` - Expands to the pattern (non-capturing in context)
+* - `{{token:name}}` - Expands to a named capture group `(?<name>pattern)`
+* - `{{:name}}` - Captures any content with the given name `(?<name>.+)`
+*
+* @remarks
+* These patterns are designed for Arabic text matching. For diacritic-insensitive
+* matching of Arabic patterns, use the `fuzzy: true` option in split rules,
+* which applies `makeDiacriticInsensitive()` to the expanded patterns.
 *
 * @example
-* // Using custom tokens
-* const regex = generatePatternRegex({
-*   type: 'pattern',
-*   template: '{verse}',
-*   tokens: { verse: '\\[[0-9]+\\]' }
-* });
+* // Using tokens in a split rule
+* { lineStartsWith: ['{{kitab}}', '{{bab}}'], split: 'at', fuzzy: true }
+*
+* @example
+* // Using tokens with named captures
+* { lineStartsAfter: ['{{raqms:hadithNum}} {{dash}} '], split: 'at' }
 */
-function generatePatternRegex(config) {
-	if (config.template) {
-		const tokenMap = config.tokens ? createTokenMap(config.tokens) : TOKENS;
-		const pattern = expandTemplate(config.template, { tokens: tokenMap });
-		return new RegExp(pattern, "u");
-	}
-	if (!config.pattern) throw new Error("pattern marker must provide either a template or pattern");
-	return new RegExp(config.pattern, "u");
-}
 /**
-* Generates a regular expression for 'bab' (chapter) markers.
+* Base token definitions mapping human-readable token names to regex patterns.
+*
+* These tokens contain raw regex patterns and do not reference other tokens.
+* For composite tokens that build on these, see `COMPOSITE_TOKENS`.
+*
+* @internal
+*/
+const BASE_TOKENS = {
+	bab: "باب",
+	basmalah: "بسم الله|﷽",
+	bullet: "[•*°]",
+	dash: "[-–—ـ]",
+	fasl: "فصل|مسألة",
+	harf: "[أ-ي]",
+	kitab: "كتاب",
+	naql: "حدثنا|أخبرنا|حدثني|وحدثنا|أنبأنا|سمعت",
+	raqm: "[\\u0660-\\u0669]",
+	raqms: "[\\u0660-\\u0669]+",
+	tarqim: "[.!?؟؛]"
+};
+/**
+* Composite token definitions using template syntax.
+*
+* These tokens reference base tokens using `{{token}}` syntax and are
+* automatically expanded to their final regex patterns at module load time.
+*
+* This provides better abstraction - if base tokens change, composites
+* automatically update on the next build.
+*
+* @internal
+*/
+const COMPOSITE_TOKENS = { numbered: "{{raqms}} {{dash}} " };
+/**
+* Expands base tokens in a template string.
+* Used internally to pre-expand composite tokens.
+*
+* @param template - Template string with `{{token}}` placeholders
+* @returns Expanded pattern with base tokens replaced
+* @internal
+*/
+const expandBaseTokens = (template) => {
+	return template.replace(/\{\{(\w+)\}\}/g, (_, tokenName) => {
+		return BASE_TOKENS[tokenName] ?? `{{${tokenName}}}`;
+	});
+};
+/**
+* Token definitions mapping human-readable token names to regex patterns.
 *
-* Matches Arabic chapter markers like باب, بَابُ, بَابٌ with optional diacritics.
-* The pattern is diacritic-insensitive using bitaboom's makeDiacriticInsensitive.
+* Tokens are used in template strings with double-brace syntax:
+* - `{{token}}` - Expands to the pattern (non-capturing in context)
+* - `{{token:name}}` - Expands to a named capture group `(?<name>pattern)`
+* - `{{:name}}` - Captures any content with the given name `(?<name>.+)`
 *
-* @returns A compiled RegExp with named groups: `full`, `marker`, `content`
+* @remarks
+* These patterns are designed for Arabic text matching. For diacritic-insensitive
+* matching of Arabic patterns, use the `fuzzy: true` option in split rules,
+* which applies `makeDiacriticInsensitive()` to the expanded patterns.
 *
 * @example
-* const regex = generateBabRegex();
-* const match = regex.exec('باب الصلاة');
-* // match.groups.marker -> 'باب'
-* // match.groups.content -> ' الصلاة'
+* // Using tokens in a split rule
+* { lineStartsWith: ['{{kitab}}', '{{bab}}'], split: 'at', fuzzy: true }
+*
+* @example
+* // Using tokens with named captures
+* { lineStartsAfter: ['{{raqms:hadithNum}} {{dash}} '], split: 'at' }
+*
+* @example
+* // Using the numbered convenience token
+* { lineStartsAfter: ['{{numbered}}'], split: 'at' }
+*/
+const TOKEN_PATTERNS = {
+	...BASE_TOKENS,
+	...Object.fromEntries(Object.entries(COMPOSITE_TOKENS).map(([k, v]) => [k, expandBaseTokens(v)]))
+};
+/**
+* Regex pattern for matching tokens with optional named capture syntax.
+*
+* Matches:
+* - `{{token}}` - Simple token (group 1 = token name, group 2 = empty)
+* - `{{token:name}}` - Token with capture (group 1 = token, group 2 = name)
+* - `{{:name}}` - Capture-only (group 1 = empty, group 2 = name)
+*
+* @internal
+*/
+const TOKEN_WITH_CAPTURE_REGEX = /\{\{(\w*):?(\w*)\}\}/g;
+/**
+* Regex pattern for simple token matching (no capture syntax).
+*
+* Matches only `{{token}}` format where token is one or more word characters.
+* Used by `containsTokens()` for quick detection.
+*
+* @internal
 */
-function generateBabRegex() {
-	const babPattern = makeDiacriticInsensitive("باب");
-	const pattern = String.raw`^(?<full>(?<marker>${babPattern}[ًٌٍَُ]?)(?<content>[\s\S]*))`;
-	return new RegExp(pattern, "u");
-}
+const SIMPLE_TOKEN_REGEX = /\{\{(\w+)\}\}/g;
 /**
-* Generates a regular expression for hadith chain (isnad) markers.
+* Checks if a query string contains template tokens.
 *
-* Matches common hadith narrator phrases like حَدَّثَنَا, أَخْبَرَنَا, etc.
-* Uses default phrases from presets or custom phrases from config.
-* All phrases are made diacritic-insensitive.
+* Performs a quick test for `{{token}}` patterns without actually
+* expanding them. Useful for determining whether to apply token
+* expansion to a string.
 *
-* @param config - Marker configuration with optional `phrases` array
-* @returns A compiled RegExp with named groups: `full`, `marker`, `content`
+* @param query - String to check for tokens
+* @returns `true` if the string contains at least one `{{token}}` pattern
 *
 * @example
-* // Using default phrases
-* const regex = generateHadithChainRegex({ type: 'hadith-chain' });
-* const match = regex.exec('حَدَّثَنَا أبو بكر');
+* containsTokens('{{raqms}} {{dash}}') // → true
+* containsTokens('plain text')          // → false
+* containsTokens('[٠-٩]+ - ')           // → false (raw regex, no tokens)
+*/
+const containsTokens = (query) => {
+	SIMPLE_TOKEN_REGEX.lastIndex = 0;
+	return SIMPLE_TOKEN_REGEX.test(query);
+};
+/**
+* Expands template tokens with support for named captures.
+*
+* This is the primary token expansion function that handles all token syntax:
+* - `{{token}}` → Expands to the token's pattern (no capture group)
+* - `{{token:name}}` → Expands to `(?<name>pattern)` (named capture)
+* - `{{:name}}` → Expands to `(?<name>.+)` (capture anything)
+*
+* Unknown tokens are left as-is in the output, allowing for partial templates.
+*
+* @param query - The template string containing tokens
+* @param fuzzyTransform - Optional function to transform Arabic text for fuzzy matching.
+*                         Applied to both token patterns and plain Arabic text between tokens.
+*                         Typically `makeDiacriticInsensitive` from the fuzzy module.
+* @returns Object with expanded pattern, capture names, and capture flag
 *
 * @example
-* // Using custom phrases
-* const regex = generateHadithChainRegex({
-*   type: 'hadith-chain',
-*   phrases: ['قَالَ', 'رَوَى']
-* });
+* // Simple token expansion
+* expandTokensWithCaptures('{{raqms}} {{dash}}')
+* // → { pattern: '[\\u0660-\\u0669]+ [-–—ـ]', captureNames: [], hasCaptures: false }
+*
+* @example
+* // Named capture
+* expandTokensWithCaptures('{{raqms:num}} {{dash}}')
+* // → { pattern: '(?<num>[\\u0660-\\u0669]+) [-–—ـ]', captureNames: ['num'], hasCaptures: true }
+*
+* @example
+* // Capture-only token
+* expandTokensWithCaptures('{{raqms:num}} {{dash}} {{:content}}')
+* // → { pattern: '(?<num>[٠-٩]+) [-–—ـ] (?<content>.+)', captureNames: ['num', 'content'], hasCaptures: true }
+*
+* @example
+* // With fuzzy transform
+* expandTokensWithCaptures('{{bab}}', makeDiacriticInsensitive)
+* // → { pattern: 'بَ?ا?بٌ?', captureNames: [], hasCaptures: false }
 */
-function generateHadithChainRegex(config) {
-	const phrasesPattern = (config.phrases || DEFAULT_HADITH_PHRASES).map((p) => makeDiacriticInsensitive(p)).join("|");
-	const pattern = String.raw`^(?<full>(?<marker>${phrasesPattern})(?<content>[\s\S]*))`;
-	return new RegExp(pattern, "u");
-}
+const expandTokensWithCaptures = (query, fuzzyTransform) => {
+	const captureNames = [];
+	const segments = [];
+	let lastIndex = 0;
+	TOKEN_WITH_CAPTURE_REGEX.lastIndex = 0;
+	let match;
+	while ((match = TOKEN_WITH_CAPTURE_REGEX.exec(query)) !== null) {
+		if (match.index > lastIndex) segments.push({
+			type: "text",
+			value: query.slice(lastIndex, match.index)
+		});
+		segments.push({
+			type: "token",
+			value: match[0]
+		});
+		lastIndex = match.index + match[0].length;
+	}
+	if (lastIndex < query.length) segments.push({
+		type: "text",
+		value: query.slice(lastIndex)
+	});
+	const processedParts = segments.map((segment) => {
+		if (segment.type === "text") {
+			if (fuzzyTransform && /[\u0600-\u06FF]/.test(segment.value)) return fuzzyTransform(segment.value);
+			return segment.value;
+		}
+		TOKEN_WITH_CAPTURE_REGEX.lastIndex = 0;
+		const tokenMatch = TOKEN_WITH_CAPTURE_REGEX.exec(segment.value);
+		if (!tokenMatch) return segment.value;
+		const [, tokenName, captureName] = tokenMatch;
+		if (!tokenName && captureName) {
+			captureNames.push(captureName);
+			return `(?<${captureName}>.+)`;
+		}
+		let tokenPattern = TOKEN_PATTERNS[tokenName];
+		if (!tokenPattern) return segment.value;
+		if (fuzzyTransform) tokenPattern = tokenPattern.split("|").map((part) => /[\u0600-\u06FF]/.test(part) ? fuzzyTransform(part) : part).join("|");
+		if (captureName) {
+			captureNames.push(captureName);
+			return `(?<${captureName}>${tokenPattern})`;
+		}
+		return tokenPattern;
+	});
+	return {
+		captureNames,
+		hasCaptures: captureNames.length > 0,
+		pattern: processedParts.join("")
+	};
+};
 /**
-* Generates a regular expression for basmala markers.
+* Expands template tokens in a query string to their regex equivalents.
 *
-* Matches various forms of بِسْمِ اللَّهِ (In the name of Allah):
-* - بسم الله (without diacritics)
-* - بِسْمِ اللَّهِ (with diacritics)
-* - Special patterns like [بسم, [تم
+* This is the simple version without capture support. It returns only the
+* expanded pattern string, not capture metadata.
 *
-* @returns A compiled RegExp with named groups: `full`, `marker`, `content`
+* Unknown tokens are left as-is, allowing for partial templates.
+*
+* @param query - Template string containing `{{token}}` placeholders
+* @returns Expanded regex pattern string
 *
 * @example
-* const regex = generateBasmalaRegex();
-* const match = regex.exec('بسم الله الرحمن الرحيم');
-* // match.groups.marker -> 'بسم الله'
+* expandTokens('، {{raqms}}')     // → '، [\\u0660-\\u0669]+'
+* expandTokens('{{raqm}}*')       // → '[\\u0660-\\u0669]*'
+* expandTokens('{{dash}}{{raqm}}') // → '[-–—ـ][\\u0660-\\u0669]'
+* expandTokens('{{unknown}}')     // → '{{unknown}}' (left as-is)
+*
+* @see expandTokensWithCaptures for full capture group support
 */
-function generateBasmalaRegex() {
-	const combinedPattern = DEFAULT_BASMALA_PATTERNS.map((p) => makeDiacriticInsensitive(p)).join("|");
-	const pattern = String.raw`^(?<full>(?<marker>${combinedPattern})(?<content>[\s\S]*))`;
-	return new RegExp(pattern, "u");
-}
+const expandTokens = (query) => expandTokensWithCaptures(query).pattern;
 /**
-* Generates a regular expression for custom phrase markers.
+* Converts a template string to a compiled RegExp.
+*
+* Expands all tokens and attempts to compile the result as a RegExp
+* with Unicode flag. Returns `null` if the resulting pattern is invalid.
 *
-* Similar to hadith-chain markers but requires explicit phrase list.
-* All phrases are made diacritic-insensitive.
+* @remarks
+* This function dynamically compiles regular expressions from template strings.
+* If templates may come from untrusted sources, be aware of potential ReDoS
+* (Regular Expression Denial of Service) risks due to catastrophic backtracking.
+* Consider validating pattern complexity or applying execution timeouts when
+* running user-submitted patterns.
 *
-* @param config - Marker configuration with required `phrases` array
-* @returns A compiled RegExp with named groups: `full`, `marker`, `content`
-* @throws {Error} When `phrases` is undefined or empty
+* @param template - Template string containing `{{token}}` placeholders
+* @returns Compiled RegExp with 'u' flag, or `null` if invalid
 *
 * @example
-* const regex = generatePhraseRegex({
-*   type: 'phrase',
-*   phrases: ['فَائِدَةٌ', 'مَسْأَلَةٌ']
-* });
+* templateToRegex('، {{raqms}}')  // → /، [٠-٩]+/u
+* templateToRegex('{{raqms}}+')   // → /[٠-٩]++/u (might be invalid in some engines)
+* templateToRegex('(((')          // → null (invalid regex)
 */
-function generatePhraseRegex(config) {
-	if (!config.phrases || config.phrases.length === 0) throw new Error("phrase marker requires phrases array");
-	const phrasesPattern = config.phrases.map((p) => makeDiacriticInsensitive(p)).join("|");
-	const pattern = String.raw`^(?<full>(?<marker>${phrasesPattern})(?<content>[\s\S]*))`;
-	return new RegExp(pattern, "u");
-}
+const templateToRegex = (template) => {
+	const expanded = expandTokens(template);
+	try {
+		return new RegExp(expanded, "u");
+	} catch {
+		return null;
+	}
+};
 /**
-* Generates a regular expression for square bracket markers.
+* Lists all available token names defined in `TOKEN_PATTERNS`.
 *
-* Matches verse or hadith reference numbers in square brackets:
-* - [٦٥] - Simple bracket
-* - • [٦٥] - With bullet prefix
-* - ° [٦٥] - With degree prefix
+* Useful for documentation, validation, or building user interfaces
+* that show available tokens.
 *
-* @returns A compiled RegExp with named groups: `full`, `marker`, `content`
+* @returns Array of token names (e.g., `['bab', 'basmala', 'bullet', ...]`)
 *
 * @example
-* const regex = generateSquareBracketRegex();
-* const match = regex.exec('[٦٥] نص الحديث');
-* // match.groups.content -> ' نص الحديث'
+* getAvailableTokens()
+* // → ['bab', 'basmala', 'bullet', 'dash', 'harf', 'kitab', 'naql', 'raqm', 'raqms']
 */
-function generateSquareBracketRegex() {
-	const markerPattern = String.raw`[•°]?\s?\[[\u0660-\u0669]+\]\s?`;
-	const pattern = String.raw`^(?<full>(?<marker>${markerPattern})(?<content>[\s\S]*))`;
-	return new RegExp(pattern, "u");
-}
+const getAvailableTokens = () => Object.keys(TOKEN_PATTERNS);
 /**
-* Generates a regular expression for number-letter-separator markers.
+* Gets the regex pattern for a specific token name.
 *
-* Matches patterns like:
-* - ٥ أ - (Arabic-Indic number, Arabic letter, dash)
-* - 5 ب. (Latin number, Arabic letter, dot)
+* Returns the raw pattern string as defined in `TOKEN_PATTERNS`,
+* without any expansion or capture group wrapping.
 *
-* @param config - Configuration with required `numbering` and `separator` fields
-* @returns A compiled RegExp with named groups: `full`, `marker`, `content`
+* @param tokenName - The token name to look up (e.g., 'raqms', 'dash')
+* @returns The regex pattern string, or `undefined` if token doesn't exist
 *
 * @example
-* const regex = generateNumLetterRegex({
-*   numbering: 'arabic-indic',
-*   separator: 'dash'
-* });
-* const match = regex.exec('٥ أ - نص');
+* getTokenPattern('raqms')   // → '[\\u0660-\\u0669]+'
+* getTokenPattern('dash')    // → '[-–—ـ]'
+* getTokenPattern('unknown') // → undefined
 */
-function generateNumLetterRegex(config) {
-	const numPattern = NUMBERING_PATTERNS[config.numbering];
-	const sepPattern = SEPARATOR_PATTERNS[config.separator] ?? config.separator;
-	const markerPattern = String.raw`${numPattern} [أ-ي]\s?${sepPattern}`;
-	const pattern = String.raw`^(?<full>(?<marker>${markerPattern})(?<content>[\s\S]*))`;
-	return new RegExp(pattern, "u");
-}
+const getTokenPattern = (tokenName) => TOKEN_PATTERNS[tokenName];
+
+//#endregion
+//#region src/segmentation/segmenter.ts
 /**
-* Generates a regular expression for number-parenthetical-separator markers.
+* Core segmentation engine for splitting Arabic text pages into logical segments.
 *
-* Matches patterns like:
-* - ٥ (أ) - (number, parenthetical content, separator)
-* - 5 (٦) - (number with parenthetical number)
+* The segmenter takes an array of pages and applies pattern-based rules to
+* identify split points, producing segments with content, page references,
+* and optional metadata.
 *
-* @param config - Configuration with required `numbering` and `separator` fields
-* @returns A compiled RegExp with named groups: `full`, `marker`, `content`
+* @module segmenter
+*/
+/**
+* Checks if a regex pattern contains standard (anonymous) capturing groups.
+*
+* Detects standard capturing groups `(...)` while excluding:
+* - Non-capturing groups `(?:...)`
+* - Lookahead assertions `(?=...)` and `(?!...)`
+* - Lookbehind assertions `(?<=...)` and `(?<!...)`
+* - Named groups `(?<name>...)` (start with `(?` so excluded here)
+*
+* **Note**: Named capture groups `(?<name>...)` ARE capturing groups but are
+* excluded by this check because they are tracked separately via the
+* `captureNames` array from token expansion. This function only detects
+* anonymous capturing groups like `(.*)`.
+*
+* @param pattern - Regex pattern string to analyze
+* @returns `true` if the pattern contains at least one anonymous capturing group
+*/
+const hasCapturingGroup = (pattern) => {
+	return /\((?!\?)/.test(pattern);
+};
+/**
+* Processes a pattern string by expanding tokens and optionally applying fuzzy matching.
+*
+* Fuzzy matching makes Arabic text diacritic-insensitive. When enabled, the
+* transform is applied to token patterns BEFORE wrapping with capture groups,
+* ensuring regex metacharacters (`(`, `)`, `|`, etc.) are not corrupted.
+*
+* @param pattern - Pattern string potentially containing `{{token}}` placeholders
+* @param fuzzy - Whether to apply diacritic-insensitive transformation
+* @returns Processed pattern with expanded tokens and capture names
 *
 * @example
-* const regex = generateNumParenRegex({
-*   numbering: 'arabic-indic',
-*   separator: 'dash'
-* });
-* const match = regex.exec('٥ (أ) - نص');
+* processPattern('{{raqms:num}} {{dash}}', false)
+* // → { pattern: '(?<num>[٠-٩]+) [-–—ـ]', captureNames: ['num'] }
+*
+* @example
+* processPattern('{{naql}}', true)
+* // → { pattern: 'حَ?دَّ?ثَ?نَ?ا|...', captureNames: [] }
 */
-function generateNumParenRegex(config) {
-	const numPattern = NUMBERING_PATTERNS[config.numbering];
-	const sepPattern = SEPARATOR_PATTERNS[config.separator] ?? config.separator;
-	const markerPattern = String.raw`${numPattern}\s*\([\u0600-\u06FF\u0660-\u0669\s]+\)\s?${sepPattern}`;
-	const pattern = String.raw`^(?<full>(?<marker>${markerPattern})(?<content>[\s\S]*))`;
-	return new RegExp(pattern, "u");
-}
+const processPattern = (pattern, fuzzy) => {
+	const { pattern: expanded, captureNames } = expandTokensWithCaptures(pattern, fuzzy ? makeDiacriticInsensitive : void 0);
+	return {
+		captureNames,
+		pattern: expanded
+	};
+};
 /**
-* Generates a regular expression for number-slash-number markers.
+* Builds a compiled regex and metadata from a split rule.
 *
-* Matches patterns like:
-* - ٥/٦ - (number slash number, separator)
-* - ٥ - (single number, separator)
+* Handles all pattern types:
+* - `regex`: Used as-is (no token expansion)
+* - `template`: Tokens expanded via `expandTokensWithCaptures`
+* - `lineStartsWith`: Converted to `^(?:patterns...)`
+* - `lineStartsAfter`: Converted to `^(?:patterns...)(.*)`
+* - `lineEndsWith`: Converted to `(?:patterns...)$`
+*
+* @param rule - Split rule containing pattern and options
+* @returns Compiled regex with capture metadata
+*/
+const buildRuleRegex = (rule) => {
+	const s = { ...rule };
+	const fuzzy = rule.fuzzy ?? false;
+	let allCaptureNames = [];
+	/**
+	* Safely compiles a regex pattern, throwing a helpful error if invalid.
+	*
+	* @remarks
+	* This catches syntax errors only. It does NOT protect against ReDoS
+	* (catastrophic backtracking) from pathological patterns. Avoid compiling
+	* patterns from untrusted sources.
+	*/
+	const compileRegex = (pattern) => {
+		try {
+			return new RegExp(pattern, "gmu");
+		} catch (error) {
+			const message = error instanceof Error ? error.message : String(error);
+			throw new Error(`Invalid regex pattern: ${pattern}\n  Cause: ${message}`);
+		}
+	};
+	if (s.lineStartsAfter?.length) {
+		const processed = s.lineStartsAfter.map((p) => processPattern(p, fuzzy));
+		const patterns = processed.map((p) => p.pattern).join("|");
+		allCaptureNames = processed.flatMap((p) => p.captureNames);
+		s.regex = `^(?:${patterns})(.*)`;
+		return {
+			captureNames: allCaptureNames,
+			regex: compileRegex(s.regex),
+			usesCapture: true,
+			usesLineStartsAfter: true
+		};
+	}
+	if (s.lineStartsWith?.length) {
+		const processed = s.lineStartsWith.map((p) => processPattern(p, fuzzy));
+		const patterns = processed.map((p) => p.pattern).join("|");
+		allCaptureNames = processed.flatMap((p) => p.captureNames);
+		s.template = `^(?:${patterns})`;
+	}
+	if (s.lineEndsWith?.length) {
+		const processed = s.lineEndsWith.map((p) => processPattern(p, fuzzy));
+		const patterns = processed.map((p) => p.pattern).join("|");
+		allCaptureNames = processed.flatMap((p) => p.captureNames);
+		s.template = `(?:${patterns})$`;
+	}
+	if (s.template) {
+		const { pattern, captureNames } = expandTokensWithCaptures(s.template);
+		s.regex = pattern;
+		allCaptureNames = [...allCaptureNames, ...captureNames];
+	}
+	if (!s.regex) throw new Error("Rule must specify exactly one pattern type: regex, template, lineStartsWith, lineStartsAfter, or lineEndsWith");
+	const usesCapture = hasCapturingGroup(s.regex) || allCaptureNames.length > 0;
+	return {
+		captureNames: allCaptureNames,
+		regex: compileRegex(s.regex),
+		usesCapture,
+		usesLineStartsAfter: false
+	};
+};
+/**
+* Builds a concatenated content string and page mapping from input pages.
 *
-* The second number after the slash is optional.
+* Pages are joined with newline characters, and a page map is created to
+* track which page each offset belongs to. This allows pattern matching
+* across page boundaries while preserving page reference information.
 *
-* @param config - Configuration with required `numbering` and `separator` fields
-* @returns A compiled RegExp with named groups: `full`, `marker`, `content`
+* @param pages - Array of input pages with id and content
+* @returns Concatenated content string and page mapping utilities
 *
 * @example
-* const regex = generateNumSlashRegex({
-*   numbering: 'arabic-indic',
-*   separator: 'dash'
-* });
-* const match1 = regex.exec('٥/٦ - نص');
-* const match2 = regex.exec('٥ - نص'); // Also matches
+* const pages = [
+*   { id: 1, content: 'Page 1 text' },
+*   { id: 2, content: 'Page 2 text' }
+* ];
+* const { content, pageMap } = buildPageMap(pages);
+* // content = 'Page 1 text\nPage 2 text'
+* // pageMap.getId(0) = 1
+* // pageMap.getId(12) = 2
+*/
+const buildPageMap = (pages) => {
+	const boundaries = [];
+	const pageBreaks = [];
+	let offset = 0;
+	const parts = [];
+	for (let i = 0; i < pages.length; i++) {
+		const normalized = normalizeLineEndings(pages[i].content);
+		boundaries.push({
+			end: offset + normalized.length,
+			id: pages[i].id,
+			start: offset
+		});
+		parts.push(normalized);
+		if (i < pages.length - 1) {
+			pageBreaks.push(offset + normalized.length);
+			offset += normalized.length + 1;
+		} else offset += normalized.length;
+	}
+	/**
+	* Finds the page boundary containing the given offset using binary search.
+	* O(log n) complexity for efficient lookup with many pages.
+	*
+	* @param off - Character offset to look up
+	* @returns Page boundary or the last boundary as fallback
+	*/
+	const findBoundary = (off) => {
+		let lo = 0;
+		let hi = boundaries.length - 1;
+		while (lo <= hi) {
+			const mid = lo + hi >>> 1;
+			const b = boundaries[mid];
+			if (off < b.start) hi = mid - 1;
+			else if (off > b.end) lo = mid + 1;
+			else return b;
+		}
+		return boundaries[boundaries.length - 1];
+	};
+	return {
+		content: parts.join("\n"),
+		normalizedPages: parts,
+		pageMap: {
+			boundaries,
+			getId: (off) => findBoundary(off)?.id ?? 0,
+			pageBreaks,
+			pageIds: boundaries.map((b) => b.id)
+		}
+	};
+};
+/**
+* Executes a regex against content and extracts match results with capture information.
+*
+* @param content - Full content string to search
+* @param regex - Compiled regex with 'g' flag
+* @param usesCapture - Whether to extract captured content
+* @param captureNames - Names of expected named capture groups
+* @returns Array of match results with positions and captures
+*/
+const findMatches = (content, regex, usesCapture, captureNames) => {
+	const matches = [];
+	regex.lastIndex = 0;
+	let m = regex.exec(content);
+	while (m !== null) {
+		const result = {
+			end: m.index + m[0].length,
+			start: m.index
+		};
+		result.namedCaptures = extractNamedCaptures(m.groups, captureNames);
+		if (usesCapture) result.captured = getLastPositionalCapture(m);
+		matches.push(result);
+		if (m[0].length === 0) regex.lastIndex++;
+		m = regex.exec(content);
+	}
+	return matches;
+};
+/**
+* Finds page breaks within a given offset range using binary search.
+* O(log n + k) where n = total breaks, k = breaks in range.
+*
+* @param startOffset - Start of range (inclusive)
+* @param endOffset - End of range (exclusive)
+* @param sortedBreaks - Sorted array of page break offsets
+* @returns Array of break offsets relative to startOffset
+*/
+const findBreaksInRange = (startOffset, endOffset, sortedBreaks) => {
+	if (sortedBreaks.length === 0) return [];
+	let lo = 0;
+	let hi = sortedBreaks.length;
+	while (lo < hi) {
+		const mid = lo + hi >>> 1;
+		if (sortedBreaks[mid] < startOffset) lo = mid + 1;
+		else hi = mid;
+	}
+	const result = [];
+	for (let i = lo; i < sortedBreaks.length && sortedBreaks[i] < endOffset; i++) result.push(sortedBreaks[i] - startOffset);
+	return result;
+};
+/**
+* Converts page-break newlines to spaces in segment content.
+*
+* When a segment spans multiple pages, the newline characters that were
+* inserted as page separators during concatenation are converted to spaces
+* for more natural reading.
+*
+* Uses binary search for O(log n + k) lookup instead of O(n) iteration.
+*
+* @param content - Segment content string
+* @param startOffset - Starting offset of this content in concatenated string
+* @param pageBreaks - Sorted array of page break offsets
+* @returns Content with page-break newlines converted to spaces
 */
-function generateNumSlashRegex(config) {
-	const numPattern = NUMBERING_PATTERNS[config.numbering];
-	const sepPattern = SEPARATOR_PATTERNS[config.separator] ?? config.separator;
-	const markerPattern = String.raw`${numPattern}(?:\s?/\s?${numPattern})?\s?${sepPattern}`;
-	const pattern = String.raw`^(?<full>(?<marker>${markerPattern})(?<content>[\s\S]*))`;
-	return new RegExp(pattern, "u");
-}
+const convertPageBreaks = (content, startOffset, pageBreaks) => {
+	const breaksInRange = findBreaksInRange(startOffset, startOffset + content.length, pageBreaks);
+	if (breaksInRange.length === 0) return content;
+	const breakSet = new Set(breaksInRange);
+	return content.replace(/\n/g, (match, offset) => breakSet.has(offset) ? " " : match);
+};
 /**
-* Generates a regular expression for numbered markers with optional format template.
+* Applies breakpoints to oversized segments.
+*
+* For each segment that spans more than maxPages, tries the breakpoint patterns
+* in order to find a suitable split point. Structural markers (from rules) are
+* always respected - segments are only broken within their boundaries.
 *
-* Supports two modes:
-* 1. Format template: Uses `format` field with token expansion (e.g., '{bullet}+ {num} {dash}')
-* 2. Default pattern: Uses `numbering` and `separator` to build standard numbered markers
+* @param segments - Initial segments from rule processing
+* @param pages - Original pages for page lookup
+* @param maxPages - Maximum pages before breakpoints apply
+* @param breakpoints - Patterns to try in order (tokens supported)
+* @param prefer - 'longer' for last match, 'shorter' for first match
+* @returns Processed segments with oversized ones broken up
+*/
+const applyBreakpoints = (segments, pages, normalizedContent, maxPages, breakpoints, prefer, logger) => {
+	const findExclusionBreakPosition = (currentFromIdx, windowEndIdx, toIdx, pageIds$1, expandedBreakpoints$1, cumulativeOffsets$1) => {
+		const startingPageId = pageIds$1[currentFromIdx];
+		if (expandedBreakpoints$1.some((bp) => bp.excludeSet.has(startingPageId)) && currentFromIdx < toIdx) return cumulativeOffsets$1[currentFromIdx + 1] - cumulativeOffsets$1[currentFromIdx];
+		for (let pageIdx = currentFromIdx + 1; pageIdx <= windowEndIdx; pageIdx++) {
+			const pageId = pageIds$1[pageIdx];
+			if (expandedBreakpoints$1.some((bp) => bp.excludeSet.has(pageId))) return cumulativeOffsets$1[pageIdx] - cumulativeOffsets$1[currentFromIdx];
+		}
+		return -1;
+	};
+	const pageIds = pages.map((p) => p.id);
+	const pageIdToIndex = new Map(pageIds.map((id, i) => [id, i]));
+	const normalizedPages = /* @__PURE__ */ new Map();
+	for (let i = 0; i < pages.length; i++) {
+		const content = normalizedContent[i];
+		normalizedPages.set(pages[i].id, {
+			content,
+			index: i,
+			length: content.length
+		});
+	}
+	const cumulativeOffsets = [0];
+	let totalOffset = 0;
+	for (let i = 0; i < pageIds.length; i++) {
+		const pageData = normalizedPages.get(pageIds[i]);
+		totalOffset += pageData ? pageData.length : 0;
+		if (i < pageIds.length - 1) totalOffset += 1;
+		cumulativeOffsets.push(totalOffset);
+	}
+	const patternProcessor = (p) => processPattern(p, false).pattern;
+	const expandedBreakpoints = expandBreakpoints(breakpoints, patternProcessor);
+	const result = [];
+	logger?.info?.("Starting breakpoint processing", {
+		maxPages,
+		segmentCount: segments.length
+	});
+	for (const segment of segments) {
+		const fromIdx = pageIdToIndex.get(segment.from) ?? -1;
+		const toIdx = segment.to !== void 0 ? pageIdToIndex.get(segment.to) ?? fromIdx : fromIdx;
+		logger?.debug?.("Processing segment", {
+			contentLength: segment.content.length,
+			contentPreview: segment.content.slice(0, 100),
+			from: segment.from,
+			fromIdx,
+			to: segment.to,
+			toIdx
+		});
+		const segmentSpan = (segment.to ?? segment.from) - segment.from;
+		const hasExclusions = expandedBreakpoints.some((bp) => hasExcludedPageInRange(bp.excludeSet, pageIds, fromIdx, toIdx));
+		if (segmentSpan <= maxPages && !hasExclusions) {
+			logger?.trace?.("Segment within limit, keeping as-is");
+			result.push(segment);
+			continue;
+		}
+		logger?.debug?.("Segment exceeds limit or has exclusions, breaking it up");
+		let remainingContent = segment.content;
+		let currentFromIdx = fromIdx;
+		let isFirstPiece = true;
+		let iterationCount = 0;
+		const maxIterations = 1e4;
+		while (currentFromIdx <= toIdx) {
+			iterationCount++;
+			if (iterationCount > maxIterations) {
+				logger?.error?.("INFINITE LOOP DETECTED! Breaking out", { iterationCount: maxIterations });
+				logger?.error?.("Loop state", {
+					currentFromIdx,
+					remainingContentLength: remainingContent.length,
+					toIdx
+				});
+				break;
+			}
+			const remainingSpan = pageIds[toIdx] - pageIds[currentFromIdx];
+			logger?.trace?.("Loop iteration", {
+				currentFromIdx,
+				currentPageId: pageIds[currentFromIdx],
+				iterationCount,
+				remainingContentLength: remainingContent.length,
+				remainingContentPreview: remainingContent.slice(0, 80),
+				remainingSpan,
+				toIdx,
+				toPageId: pageIds[toIdx]
+			});
+			const remainingHasExclusions = expandedBreakpoints.some((bp) => hasExcludedPageInRange(bp.excludeSet, pageIds, currentFromIdx, toIdx));
+			if (remainingSpan <= maxPages && !remainingHasExclusions) {
+				logger?.debug?.("Remaining span within limit, outputting final segment");
+				const finalSeg = createSegment(remainingContent, pageIds[currentFromIdx], currentFromIdx !== toIdx ? pageIds[toIdx] : void 0, isFirstPiece ? segment.meta : void 0);
+				if (finalSeg) result.push(finalSeg);
+				break;
+			}
+			const currentPageId = pageIds[currentFromIdx];
+			const maxWindowPageId = currentPageId + maxPages;
+			let windowEndIdx = currentFromIdx;
+			for (let i = currentFromIdx; i <= toIdx; i++) if (pageIds[i] <= maxWindowPageId) windowEndIdx = i;
+			else break;
+			logger?.trace?.("Window calculation", {
+				currentPageId,
+				maxWindowPageId,
+				windowEndIdx,
+				windowEndPageId: pageIds[windowEndIdx]
+			});
+			const windowHasExclusions = expandedBreakpoints.some((bp) => hasExcludedPageInRange(bp.excludeSet, pageIds, currentFromIdx, windowEndIdx));
+			let breakPosition = -1;
+			if (windowHasExclusions) {
+				logger?.trace?.("Window has exclusions, finding exclusion break position");
+				breakPosition = findExclusionBreakPosition(currentFromIdx, windowEndIdx, toIdx, pageIds, expandedBreakpoints, cumulativeOffsets);
+				logger?.trace?.("Exclusion break position", { breakPosition });
+			}
+			if (breakPosition <= 0) {
+				const breakpointCtx = {
+					cumulativeOffsets,
+					expandedBreakpoints,
+					normalizedPages,
+					pageIds,
+					prefer
+				};
+				logger?.trace?.("Finding break position using patterns...");
+				breakPosition = findBreakPosition(remainingContent, currentFromIdx, toIdx, windowEndIdx, breakpointCtx);
+				logger?.trace?.("Pattern break position", { breakPosition });
+			}
+			if (breakPosition <= 0) {
+				logger?.debug?.("No pattern matched, falling back to page boundary");
+				if (windowEndIdx === currentFromIdx) {
+					logger?.trace?.("Single page window, outputting page and advancing");
+					const pageContent = cumulativeOffsets[currentFromIdx + 1] !== void 0 ? remainingContent.slice(0, cumulativeOffsets[currentFromIdx + 1] - cumulativeOffsets[currentFromIdx]) : remainingContent;
+					const pageSeg = createSegment(pageContent.trim(), pageIds[currentFromIdx], void 0, isFirstPiece ? segment.meta : void 0);
+					if (pageSeg) result.push(pageSeg);
+					remainingContent = remainingContent.slice(pageContent.length).trim();
+					currentFromIdx++;
+					isFirstPiece = false;
+					logger?.trace?.("After single page", {
+						currentFromIdx,
+						remainingContentLength: remainingContent.length
+					});
+					continue;
+				}
+				breakPosition = cumulativeOffsets[windowEndIdx + 1] - cumulativeOffsets[currentFromIdx];
+				logger?.trace?.("Multi-page window, using full window break position", { breakPosition });
+			}
+			const pieceContent = remainingContent.slice(0, breakPosition).trim();
+			logger?.trace?.("Piece extracted", {
+				breakPosition,
+				pieceContentLength: pieceContent.length,
+				pieceContentPreview: pieceContent.slice(0, 80)
+			});
+			const actualStartIdx = pieceContent ? findActualStartPage(pieceContent, currentFromIdx, toIdx, pageIds, normalizedPages) : currentFromIdx;
+			const actualEndIdx = pieceContent ? findActualEndPage(pieceContent, actualStartIdx, windowEndIdx, pageIds, normalizedPages) : currentFromIdx;
+			logger?.trace?.("Actual page indices", {
+				actualEndIdx,
+				actualStartIdx,
+				pieceHasContent: !!pieceContent
+			});
+			if (pieceContent) {
+				const pieceSeg = createSegment(pieceContent, pageIds[actualStartIdx], actualEndIdx > actualStartIdx ? pageIds[actualEndIdx] : void 0, isFirstPiece ? segment.meta : void 0);
+				if (pieceSeg) {
+					result.push(pieceSeg);
+					logger?.debug?.("Created segment", {
+						contentLength: pieceSeg.content.length,
+						from: pieceSeg.from,
+						to: pieceSeg.to
+					});
+				}
+			}
+			const prevRemainingLength = remainingContent.length;
+			remainingContent = remainingContent.slice(breakPosition).trim();
+			logger?.trace?.("After slicing remainingContent", {
+				newLength: remainingContent.length,
+				prevLength: prevRemainingLength,
+				slicedAmount: breakPosition
+			});
+			if (!remainingContent) {
+				logger?.debug?.("No remaining content, breaking out of loop");
+				break;
+			}
+			let nextFromIdx = actualEndIdx;
+			if (actualEndIdx + 1 <= toIdx) {
+				const nextPageData = normalizedPages.get(pageIds[actualEndIdx + 1]);
+				if (nextPageData) {
+					const nextPrefix = nextPageData.content.slice(0, Math.min(30, nextPageData.length));
+					if (nextPrefix && remainingContent.startsWith(nextPrefix)) {
+						nextFromIdx = actualEndIdx + 1;
+						logger?.trace?.("Content starts with next page prefix", { advancingTo: nextFromIdx });
+					}
+				}
+			}
+			logger?.trace?.("End of iteration", {
+				nextFromIdx,
+				prevCurrentFromIdx: currentFromIdx,
+				willAdvance: nextFromIdx !== currentFromIdx
+			});
+			currentFromIdx = nextFromIdx;
+			isFirstPiece = false;
+		}
+	}
+	logger?.info?.("Breakpoint processing completed", { resultCount: result.length });
+	return result;
+};
+/**
+* Segments pages of content based on pattern-matching rules.
 *
-* When using default pattern:
-* - Separator 'none' generates pattern without separator
-* - Custom separator strings are used as-is or looked up in SEPARATOR_PATTERNS
+* This is the main entry point for the segmentation engine. It takes an array
+* of pages and applies the provided rules to identify split points, producing
+* an array of segments with content, page references, and metadata.
 *
-* @param config - Configuration with `numbering`, `separator`, and optional `format`/`tokens`
-* @returns A compiled RegExp with named groups: `full`, `marker`, `content`
+* @param pages - Array of pages with id and content
+* @param options - Segmentation options including splitting rules
+* @returns Array of segments with content, from/to page references, and optional metadata
 *
 * @example
-* // Using format template
-* const regex = generateNumberedRegex({
-*   numbering: 'arabic-indic',
-*   separator: 'dash',
-*   format: '{bullet}+ {num} {dash}'
+* // Split markdown by headers
+* const segments = segmentPages(pages, {
+*   rules: [
+*     { lineStartsWith: ['## '], split: 'at', meta: { type: 'chapter' } }
+*   ]
 * });
 *
 * @example
-* // Using default pattern
-* const regex = generateNumberedRegex({
-*   numbering: 'arabic-indic',
-*   separator: 'dash'
+* // Split Arabic hadith text with number extraction
+* const segments = segmentPages(pages, {
+*   rules: [
+*     {
+*       lineStartsAfter: ['{{raqms:hadithNum}} {{dash}} '],
+*       split: 'at',
+*       fuzzy: true,
+*       meta: { type: 'hadith' }
+*     }
+*   ]
 * });
-* const match = regex.exec('٥ - نص');
 *
 * @example
-* // With 'none' separator
-* const regex = generateNumberedRegex({
-*   numbering: 'latin',
-*   separator: 'none'
+* // Multiple rules with page constraints
+* const segments = segmentPages(pages, {
+*   rules: [
+*     { lineStartsWith: ['{{kitab}}'], split: 'at', meta: { type: 'book' } },
+*     { lineStartsWith: ['{{bab}}'], split: 'at', min: 10, meta: { type: 'chapter' } },
+*     { regex: '^[٠-٩]+ - ', split: 'at', meta: { type: 'hadith' } }
+*   ]
 * });
-* const match = regex.exec('5 text');
 */
-function generateNumberedRegex(config) {
-	if (config.format) {
-		const tokenMap = config.tokens ? createTokenMap(config.tokens) : TOKENS;
-		const expandedPattern = expandTemplate(config.format, { tokens: tokenMap });
-		return new RegExp(expandedPattern, "u");
+const segmentPages = (pages, options) => {
+	const { rules = [], maxPages, breakpoints, prefer = "longer", logger } = options;
+	if (!pages.length) return [];
+	const { content: matchContent, normalizedPages: normalizedContent, pageMap } = buildPageMap(pages);
+	const splitPoints = [];
+	for (const rule of rules) {
+		const { regex, usesCapture, captureNames, usesLineStartsAfter } = buildRuleRegex(rule);
+		const finalMatches = filterByOccurrence(filterByConstraints(findMatches(matchContent, regex, usesCapture, captureNames), rule, pageMap.getId), rule.occurrence);
+		for (const m of finalMatches) {
+			const isLineStartsAfter = usesLineStartsAfter && m.captured !== void 0;
+			const markerLength = isLineStartsAfter ? m.end - m.captured.length - m.start : 0;
+			splitPoints.push({
+				capturedContent: isLineStartsAfter ? void 0 : m.captured,
+				contentStartOffset: isLineStartsAfter ? markerLength : void 0,
+				index: rule.split === "at" ? m.start : m.end,
+				meta: rule.meta,
+				namedCaptures: m.namedCaptures
+			});
+		}
+	}
+	const byIndex = /* @__PURE__ */ new Map();
+	for (const p of splitPoints) {
+		const existing = byIndex.get(p.index);
+		if (!existing) byIndex.set(p.index, p);
+		else if (p.contentStartOffset !== void 0 && existing.contentStartOffset === void 0 || p.meta !== void 0 && existing.meta === void 0) byIndex.set(p.index, p);
 	}
-	const numPattern = NUMBERING_PATTERNS[config.numbering];
-	const separator = config.separator;
-	const sepPattern = separator !== "none" ? SEPARATOR_PATTERNS[separator] ?? separator : "";
-	const markerPattern = sepPattern ? String.raw`${numPattern}\s?${sepPattern}` : numPattern;
-	const pattern = String.raw`^(?<full>(?<marker>${markerPattern})(?<content>[\s\S]*))`;
-	return new RegExp(pattern, "u");
-}
+	const unique = [...byIndex.values()];
+	unique.sort((a, b) => a.index - b.index);
+	let segments = buildSegments(unique, matchContent, pageMap, rules);
+	if (segments.length === 0 && pages.length > 0) {
+		const firstPage = pages[0];
+		const lastPage = pages[pages.length - 1];
+		const initialSeg = {
+			content: pages.map((p) => normalizeLineEndings(p.content)).join("\n").trim(),
+			from: firstPage.id
+		};
+		if (lastPage.id !== firstPage.id) initialSeg.to = lastPage.id;
+		if (initialSeg.content) segments = [initialSeg];
+	}
+	if (maxPages !== void 0 && maxPages >= 0 && breakpoints?.length) return applyBreakpoints(segments, pages, normalizedContent, maxPages, breakpoints, prefer, logger);
+	return segments;
+};
 /**
-* Generates a regular expression for bullet-point markers.
+* Creates segment objects from split points.
 *
-* Matches common bullet characters:
-* - • (bullet)
-* - * (asterisk)
-* - ° (degree)
-* - - (dash)
+* Handles segment creation including:
+* - Content extraction (with captured content for `lineStartsAfter`)
+* - Page break conversion to spaces
+* - From/to page reference calculation
+* - Metadata merging (static + named captures)
 *
-* @returns A compiled RegExp with named groups: `full`, `marker`, `content`
+* @param splitPoints - Sorted, unique split points
+* @param content - Full concatenated content string
+* @param pageMap - Page mapping utilities
+* @param rules - Original rules (for constraint checking on first segment)
+* @returns Array of segment objects
+*/
+const buildSegments = (splitPoints, content, pageMap, rules) => {
+	/**
+	* Creates a single segment from a content range.
+	*/
+	const createSegment$1 = (start, end, meta, capturedContent, namedCaptures, contentStartOffset) => {
+		const actualStart = start + (contentStartOffset ?? 0);
+		const sliced = content.slice(actualStart, end);
+		let text = capturedContent?.trim() ?? (contentStartOffset ? sliced.trim() : sliced.replace(/[\s\n]+$/, ""));
+		if (!text) return null;
+		if (!capturedContent) text = convertPageBreaks(text, actualStart, pageMap.pageBreaks);
+		const from = pageMap.getId(actualStart);
+		const to = capturedContent ? pageMap.getId(end - 1) : pageMap.getId(actualStart + text.length - 1);
+		const seg = {
+			content: text,
+			from
+		};
+		if (to !== from) seg.to = to;
+		if (meta || namedCaptures) seg.meta = {
+			...meta,
+			...namedCaptures
+		};
+		return seg;
+	};
+	/**
+	* Creates segments from an array of split points.
+	*/
+	const createSegmentsFromSplitPoints = () => {
+		const result = [];
+		for (let i = 0; i < splitPoints.length; i++) {
+			const sp = splitPoints[i];
+			const end = i < splitPoints.length - 1 ? splitPoints[i + 1].index : content.length;
+			const s = createSegment$1(sp.index, end, sp.meta, sp.capturedContent, sp.namedCaptures, sp.contentStartOffset);
+			if (s) result.push(s);
+		}
+		return result;
+	};
+	const segments = [];
+	if (!splitPoints.length) {
+		if (anyRuleAllowsId(rules, pageMap.getId(0))) {
+			const s = createSegment$1(0, content.length);
+			if (s) segments.push(s);
+		}
+		return segments;
+	}
+	if (splitPoints[0].index > 0) {
+		if (anyRuleAllowsId(rules, pageMap.getId(0))) {
+			const s = createSegment$1(0, splitPoints[0].index);
+			if (s) segments.push(s);
+		}
+	}
+	return [...segments, ...createSegmentsFromSplitPoints()];
+};
+
+//#endregion
+//#region src/pattern-detection.ts
+/**
+* Pattern detection utilities for recognizing template tokens in Arabic text.
+* Used to auto-detect patterns from user-highlighted text in the segmentation dialog.
 *
-* @example
-* const regex = generateBulletRegex();
-* const match = regex.exec('• نقطة');
-* // match.groups.content -> 'نقطة'
+* @module pattern-detection
 */
-function generateBulletRegex() {
-	return new RegExp("^(?<full>(?<marker>[•*°\\-]\\s?)(?<content>[\\s\\S]*))", "u");
-}
 /**
-* Generates a regular expression for Markdown-style heading markers.
+* Token detection order - more specific patterns first to avoid partial matches.
+* Example: 'raqms' before 'raqm' so "٣٤" matches 'raqms' not just the first digit.
 *
-* Matches heading levels using hash symbols:
-* - # Heading 1
-* - ## Heading 2
-* - ### Heading 3
-* - etc.
+* Tokens not in this list are appended in alphabetical order from TOKEN_PATTERNS.
+*/
+const TOKEN_PRIORITY_ORDER = [
+	"basmalah",
+	"kitab",
+	"bab",
+	"fasl",
+	"naql",
+	"numbered",
+	"raqms",
+	"raqm",
+	"tarqim",
+	"bullet",
+	"dash",
+	"harf"
+];
+/**
+* Gets the token detection priority order.
+* Returns tokens in priority order, with any TOKEN_PATTERNS not in the priority list appended.
+*/
+const getTokenPriority = () => {
+	const allTokens = getAvailableTokens();
+	const prioritized = TOKEN_PRIORITY_ORDER.filter((t) => allTokens.includes(t));
+	const remaining = allTokens.filter((t) => !TOKEN_PRIORITY_ORDER.includes(t)).sort();
+	return [...prioritized, ...remaining];
+};
+/**
+* Analyzes text and returns all detected token patterns with their positions.
+* Patterns are detected in priority order to avoid partial matches.
 *
-* @returns A compiled RegExp with named groups: `full`, `marker`, `content`
+* @param text - The text to analyze for token patterns
+* @returns Array of detected patterns sorted by position
 *
 * @example
-* const regex = generateHeadingRegex();
-* const match = regex.exec('## عنوان فرعي');
-* // match.groups.marker -> '## '
-* // match.groups.content -> 'عنوان فرعي'
+* detectTokenPatterns("٣٤ - حدثنا")
+* // Returns: [
+* //   { token: 'raqms', match: '٣٤', index: 0, endIndex: 2 },
+* //   { token: 'dash', match: '-', index: 3, endIndex: 4 },
+* //   { token: 'naql', match: 'حدثنا', index: 5, endIndex: 10 }
+* // ]
 */
-function generateHeadingRegex() {
-	return new RegExp("^(?<full>(?<marker>#+\\s?)(?<content>[\\s\\S]*))", "u");
-}
-
-//#endregion
-//#region src/markers/generator.ts
-/**
-* Generates a regex pattern from a marker configuration.
-* Always returns a regex with three named capture groups:
-* - full: Complete match including marker
-* - marker: Just the marker part (for metadata/indexing)
-* - content: Clean content without marker (for LLM processing)
-* 
-* This function applies all default values before delegating to type-specific generators.
-* 
-* @param config - Marker configuration
-* @returns Regular expression with named groups
-* 
-* @example
-* const regex = generateRegexFromMarker({ type: 'numbered' });
-* const match = regex.exec('٥ - نص');
-* match.groups.full    // "٥ - نص"
-* match.groups.marker  // "٥ -"
-* match.groups.content // "نص"
-*/
-function generateRegexFromMarker(config) {
-	const normalized = {
-		numbering: config.numbering ?? DEFAULT_NUMBERING,
-		separator: config.separator ?? DEFAULT_SEPARATOR,
-		...config
+const detectTokenPatterns = (text) => {
+	if (!text) return [];
+	const results = [];
+	const coveredRanges = [];
+	const isPositionCovered = (start, end) => {
+		return coveredRanges.some(([s, e]) => start >= s && start < e || end > s && end <= e || start <= s && end >= e);
 	};
-	switch (normalized.type) {
-		case "pattern": return generatePatternRegex(normalized);
-		case "bab": return generateBabRegex();
-		case "hadith-chain": return generateHadithChainRegex(normalized);
-		case "basmala": return generateBasmalaRegex();
-		case "phrase": return generatePhraseRegex(normalized);
-		case "square-bracket": return generateSquareBracketRegex();
-		case "num-letter": return generateNumLetterRegex(normalized);
-		case "num-paren": return generateNumParenRegex(normalized);
-		case "num-slash": return generateNumSlashRegex(normalized);
-		case "numbered": return generateNumberedRegex(normalized);
-		case "bullet": return generateBulletRegex();
-		case "heading": return generateHeadingRegex();
-		default: {
-			const _exhaustive = normalized.type;
-			throw new Error(`Unknown marker type: ${_exhaustive}`);
-		}
+	for (const tokenName of getTokenPriority()) {
+		const pattern = TOKEN_PATTERNS[tokenName];
+		if (!pattern) continue;
+		try {
+			const regex = new RegExp(`(${pattern})`, "gu");
+			let match;
+			while ((match = regex.exec(text)) !== null) {
+				const startIndex = match.index;
+				const endIndex = startIndex + match[0].length;
+				if (isPositionCovered(startIndex, endIndex)) continue;
+				results.push({
+					endIndex,
+					index: startIndex,
+					match: match[0],
+					token: tokenName
+				});
+				coveredRanges.push([startIndex, endIndex]);
+			}
+		} catch {}
 	}
-}
+	return results.sort((a, b) => a.index - b.index);
+};
+/**
+* Generates a template pattern from text using detected tokens.
+* Replaces matched portions with {{token}} syntax.
+*
+* @param text - Original text
+* @param detected - Array of detected patterns from detectTokenPatterns
+* @returns Template string with tokens, e.g., "{{raqms}} {{dash}} "
+*
+* @example
+* const detected = detectTokenPatterns("٣٤ - ");
+* generateTemplateFromText("٣٤ - ", detected);
+* // Returns: "{{raqms}} {{dash}} "
+*/
+const generateTemplateFromText = (text, detected) => {
+	if (!text || detected.length === 0) return text;
+	let template = text;
+	const sortedByIndexDesc = [...detected].sort((a, b) => b.index - a.index);
+	for (const d of sortedByIndexDesc) template = `${template.slice(0, d.index)}{{${d.token}}}${template.slice(d.endIndex)}`;
+	return template;
+};
+/**
+* Determines the best pattern type for auto-generated rules based on detected patterns.
+*
+* @param detected - Array of detected patterns
+* @returns Suggested pattern type and whether to use fuzzy matching
+*/
+const suggestPatternConfig = (detected) => {
+	const hasStructuralToken = detected.some((d) => [
+		"basmalah",
+		"kitab",
+		"bab",
+		"fasl"
+	].includes(d.token));
+	const hasNumberedPattern = detected.some((d) => [
+		"raqms",
+		"raqm",
+		"numbered"
+	].includes(d.token));
+	if (hasStructuralToken) return {
+		fuzzy: true,
+		metaType: detected.find((d) => [
+			"kitab",
+			"bab",
+			"fasl"
+		].includes(d.token))?.token || "chapter",
+		patternType: "lineStartsWith"
+	};
+	if (hasNumberedPattern) return {
+		fuzzy: false,
+		metaType: "hadith",
+		patternType: "lineStartsAfter"
+	};
+	return {
+		fuzzy: false,
+		patternType: "lineStartsAfter"
+	};
+};
+/**
+* Analyzes text and generates a complete suggested rule configuration.
+*
+* @param text - Highlighted text from the page
+* @returns Suggested rule configuration or null if no patterns detected
+*/
+const analyzeTextForRule = (text) => {
+	const detected = detectTokenPatterns(text);
+	if (detected.length === 0) return null;
+	return {
+		detected,
+		template: generateTemplateFromText(text, detected),
+		...suggestPatternConfig(detected)
+	};
+};
 
 //#endregion
-export { DEFAULT_BASMALA_PATTERNS, DEFAULT_HADITH_PHRASES, DEFAULT_NUMBERING, DEFAULT_SEPARATOR, DEFAULT_SEPARATOR_PATTERN, NUMBERING_PATTERNS, SEPARATOR_PATTERNS, TOKENS, createTokenMap, expandTemplate, generateBabRegex, generateBasmalaRegex, generateBulletRegex, generateHadithChainRegex, generateHeadingRegex, generateNumLetterRegex, generateNumParenRegex, generateNumSlashRegex, generateNumberedRegex, generatePatternRegex, generatePhraseRegex, generateRegexFromMarker, generateSquareBracketRegex, validateTemplate };
+export { TOKEN_PATTERNS, analyzeTextForRule, containsTokens, detectTokenPatterns, escapeRegex, expandTokens, expandTokensWithCaptures, generateTemplateFromText, getAvailableTokens, getTokenPattern, makeDiacriticInsensitive, normalizeLineEndings, segmentPages, stripHtmlTags, suggestPatternConfig, templateToRegex };
 //# sourceMappingURL=index.mjs.map