flappa-doormal 2.9.0 → 2.10.1

package/dist/index.mjs

@@ -155,6 +155,154 @@ const makeDiacriticInsensitive = (text) => {
 	return Array.from(norm).map((ch) => getEquivClass(ch) + diacriticsMatcher).join("");
 };
 
+//#endregion
+//#region src/segmentation/types.ts
+/**
+* Pattern type key names for split rules.
+*
+* Use this array to dynamically iterate over pattern types in UIs,
+* or use the `PatternTypeKey` type for type-safe string unions.
+*
+* @example
+* // Build a dropdown/select in UI
+* PATTERN_TYPE_KEYS.map(key => <option value={key}>{key}</option>)
+*
+* @example
+* // Type-safe pattern key validation
+* const validateKey = (k: string): k is PatternTypeKey =>
+*   (PATTERN_TYPE_KEYS as readonly string[]).includes(k);
+*/
+const PATTERN_TYPE_KEYS = [
+	"lineStartsWith",
+	"lineStartsAfter",
+	"lineEndsWith",
+	"template",
+	"regex"
+];
+
+//#endregion
+//#region src/segmentation/optimize-rules.ts
+/**
+* Rule optimization utilities for merging and sorting split rules.
+*
+* Provides `optimizeRules()` to:
+* 1. Merge compatible rules with same pattern type and options
+* 2. Deduplicate patterns within each rule
+* 3. Sort rules by specificity (longer patterns first)
+*
+* @module optimize-rules
+*/
+const MERGEABLE_KEYS = new Set([
+	"lineStartsWith",
+	"lineStartsAfter",
+	"lineEndsWith"
+]);
+/**
+* Get the pattern type key for a rule.
+*/
+const getPatternKey = (rule) => {
+	for (const key of PATTERN_TYPE_KEYS) if (key in rule) return key;
+	return "regex";
+};
+/**
+* Get the pattern array for a mergeable rule.
+*/
+const getPatternArray = (rule, key) => {
+	const value = rule[key];
+	return Array.isArray(value) ? value : [];
+};
+/**
+* Get a string representation of the pattern value (for specificity scoring).
+*/
+const getPatternString = (rule, key) => {
+	const value = rule[key];
+	if (typeof value === "string") return value;
+	if (Array.isArray(value)) return value.join("\n");
+	return "";
+};
+/**
+* Deduplicate and sort patterns by length (longest first).
+*/
+const normalizePatterns = (patterns) => {
+	return [...new Set(patterns)].sort((a, b) => b.length - a.length || a.localeCompare(b));
+};
+/**
+* Calculate specificity score for a rule (higher = more specific).
+* Based on the longest pattern length.
+*/
+const getSpecificityScore = (rule) => {
+	const key = getPatternKey(rule);
+	if (MERGEABLE_KEYS.has(key)) return getPatternArray(rule, key).reduce((max, p) => Math.max(max, p.length), 0);
+	return getPatternString(rule, key).length;
+};
+/**
+* Create a merge key for a rule based on pattern type and all non-pattern properties.
+* Rules with the same merge key can have their patterns combined.
+*/
+const createMergeKey = (rule) => {
+	const patternKey = getPatternKey(rule);
+	const { [patternKey]: _pattern, ...rest } = rule;
+	return `${patternKey}|${JSON.stringify(rest)}`;
+};
+/**
+* Optimize split rules by merging compatible rules and sorting by specificity.
+*
+* This function:
+* 1. **Merges compatible rules**: Rules with the same pattern type and identical
+*    options (meta, fuzzy, min/max, etc.) have their pattern arrays combined
+* 2. **Deduplicates patterns**: Removes duplicate patterns within each rule
+* 3. **Sorts by specificity**: Rules with longer patterns come first
+*
+* Only array-based pattern types (`lineStartsWith`, `lineStartsAfter`, `lineEndsWith`)
+* can be merged. `template` and `regex` rules are kept separate.
+*
+* @param rules - Array of split rules to optimize
+* @returns Optimized rules and count of merged rules
+*
+* @example
+* import { optimizeRules } from 'flappa-doormal';
+*
+* const { rules, mergedCount } = optimizeRules([
+*   { lineStartsWith: ['{{kitab}}'], fuzzy: true, meta: { type: 'header' } },
+*   { lineStartsWith: ['{{bab}}'], fuzzy: true, meta: { type: 'header' } },
+*   { lineStartsAfter: ['{{numbered}}'], meta: { type: 'entry' } },
+* ]);
+*
+* // rules[0] = { lineStartsWith: ['{{kitab}}', '{{bab}}'], fuzzy: true, meta: { type: 'header' } }
+* // rules[1] = { lineStartsAfter: ['{{numbered}}'], meta: { type: 'entry' } }
+* // mergedCount = 1
+*/
+const optimizeRules = (rules) => {
+	const output = [];
+	const indexByMergeKey = /* @__PURE__ */ new Map();
+	let mergedCount = 0;
+	for (const rule of rules) {
+		const patternKey = getPatternKey(rule);
+		if (!MERGEABLE_KEYS.has(patternKey)) {
+			output.push(rule);
+			continue;
+		}
+		const mergeKey = createMergeKey(rule);
+		const existingIndex = indexByMergeKey.get(mergeKey);
+		if (existingIndex === void 0) {
+			indexByMergeKey.set(mergeKey, output.length);
+			output.push({
+				...rule,
+				[patternKey]: normalizePatterns(getPatternArray(rule, patternKey))
+			});
+			continue;
+		}
+		const existing = output[existingIndex];
+		existing[patternKey] = normalizePatterns([...getPatternArray(existing, patternKey), ...getPatternArray(rule, patternKey)]);
+		mergedCount++;
+	}
+	output.sort((a, b) => getSpecificityScore(b) - getSpecificityScore(a));
+	return {
+		mergedCount,
+		rules: output
+	};
+};
+
 //#endregion
 //#region src/segmentation/tokens.ts
 /**
@@ -626,6 +774,51 @@ const shouldDefaultToFuzzy = (patterns) => {
 		return FUZZY_TOKEN_REGEX.test(p);
 	});
 };
+/**
+* Apply token mappings to a template string.
+*
+* Transforms `{{token}}` into `{{token:name}}` based on the provided mappings.
+* Useful for applying user-configured capture names to a raw template.
+*
+* - Only affects exact matches of `{{token}}`.
+* - Does NOT affect tokens that already have a capture name (e.g. `{{token:existing}}`).
+* - Does NOT affect capture-only tokens (e.g. `{{:name}}`).
+*
+* @param template - The template string to transform
+* @param mappings - Array of mappings from token name to capture name
+* @returns Transformed template string with captures applied
+*
+* @example
+* applyTokenMappings('{{raqms}} {{dash}}', [{ token: 'raqms', name: 'num' }])
+* // → '{{raqms:num}} {{dash}}'
+*/
+const applyTokenMappings = (template, mappings) => {
+	let result = template;
+	for (const { token, name } of mappings) {
+		if (!token || !name) continue;
+		const regex = new RegExp(`\\{\\{${token}\\}\\}`, "g");
+		result = result.replace(regex, `{{${token}:${name}}}`);
+	}
+	return result;
+};
+/**
+* Strip token mappings from a template string.
+*
+* Transforms `{{token:name}}` back into `{{token}}`.
+* Also transforms `{{:name}}` patterns (capture-only) into `{{}}` (which is invalid/empty).
+*
+* Useful for normalizing templates for storage or comparison.
+*
+* @param template - The template string to strip
+* @returns Template string with capture names removed
+*
+* @example
+* stripTokenMappings('{{raqms:num}} {{dash}}')
+* // → '{{raqms}} {{dash}}'
+*/
+const stripTokenMappings = (template) => {
+	return template.replace(/\{\{([^:}]+):[^}]+\}\}/g, "{{$1}}");
+};
 
 //#endregion
 //#region src/segmentation/pattern-validator.ts
@@ -651,6 +844,7 @@ const validatePattern = (pattern, seenPatterns) => {
 	};
 	if (seenPatterns.has(pattern)) return {
 		message: `Duplicate pattern: "${pattern}"`,
+		pattern,
 		type: "duplicate"
 	};
 	seenPatterns.add(pattern);
@@ -660,6 +854,7 @@ const validatePattern = (pattern, seenPatterns) => {
 		if (!KNOWN_TOKENS.has(tokenName)) return {
 			message: `Unknown token: {{${tokenName}}}. Available tokens: ${[...KNOWN_TOKENS].slice(0, 5).join(", ")}...`,
 			suggestion: `Check spelling or use a known token`,
+			token: tokenName,
 			type: "unknown_token"
 		};
 	}
@@ -674,6 +869,7 @@ const validatePattern = (pattern, seenPatterns) => {
 		if (before !== "{{" && after !== "}}") return {
 			message: `Token "${tokenName}" appears to be missing {{}}. Did you mean "{{${fullMatch}}}"?`,
 			suggestion: `{{${fullMatch}}}`,
+			token: tokenName,
 			type: "missing_braces"
 		};
 	}
@@ -742,6 +938,39 @@ const validateRules = (rules) => {
 		return hasIssues ? result : void 0;
 	});
 };
+/**
+* Formats a validation result array into a list of human-readable error messages.
+*
+* Useful for displaying validation errors in UIs.
+*
+* @param results - The result array from `validateRules()`
+* @returns Array of formatted error strings
+*
+* @example
+* const issues = validateRules(rules);
+* const errors = formatValidationReport(issues);
+* // ["Rule 1, lineStartsWith: Missing {{}} around token..."]
+*/
+const formatValidationReport = (results) => {
+	const errors = [];
+	results.forEach((result, ruleIndex) => {
+		if (!result) return;
+		const formatIssue = (issue, location) => {
+			if (!issue) return;
+			const type = issue.type;
+			if (type === "missing_braces" && issue.token) errors.push(`${location}: Missing {{}} around token "${issue.token}"`);
+			else if (type === "unknown_token" && issue.token) errors.push(`${location}: Unknown token "{{${issue.token}}}"`);
+			else if (type === "duplicate" && issue.pattern) errors.push(`${location}: Duplicate pattern "${issue.pattern}"`);
+			else if (issue.message) errors.push(`${location}: ${issue.message}`);
+			else errors.push(`${location}: ${type}`);
+		};
+		for (const [patternType, issues] of Object.entries(result)) {
+			const list = Array.isArray(issues) ? issues : [issues];
+			for (const issue of list) if (issue) formatIssue(issue, `Rule ${ruleIndex + 1}, ${patternType}`);
+		}
+	});
+	return errors;
+};
 
 //#endregion
 //#region src/segmentation/replace.ts
@@ -1049,7 +1278,7 @@ const estimateStartOffsetInCurrentPage = (remainingContent, currentFromIdx, page
 * This is used to define breakpoint windows in terms of actual content being split, rather than
 * raw per-page offsets which can desync when structural rules strip markers.
 */
-const findPageStartNearExpectedBoundary = (remainingContent, _currentFromIdx, targetPageIdx, expectedBoundary, pageIds, normalizedPages) => {
+const findPageStartNearExpectedBoundary = (remainingContent, _currentFromIdx, targetPageIdx, expectedBoundary, pageIds, normalizedPages, logger) => {
 	const targetPageData = normalizedPages.get(pageIds[targetPageIdx]);
 	if (!targetPageData) return -1;
 	const approx = Math.min(Math.max(0, expectedBoundary), remainingContent.length);
@@ -1059,13 +1288,45 @@ const findPageStartNearExpectedBoundary = (remainingContent, _currentFromIdx, ta
 	for (const len of WINDOW_PREFIX_LENGTHS) {
 		const prefix = targetTrimmed.slice(0, Math.min(len, targetTrimmed.length)).trim();
 		if (!prefix) continue;
+		const candidates = [];
 		let pos = remainingContent.indexOf(prefix, searchStart);
 		while (pos !== -1 && pos <= searchEnd) {
-			if (pos > 0 && /\s/.test(remainingContent[pos - 1] ?? "")) return pos;
+			if (pos > 0) {
+				const charBefore = remainingContent[pos - 1];
+				if (charBefore === "\n") candidates.push({
+					isNewline: true,
+					pos
+				});
+				else if (/\s/.test(charBefore)) candidates.push({
+					isNewline: false,
+					pos
+				});
+			}
 			pos = remainingContent.indexOf(prefix, pos + 1);
 		}
-		const last = remainingContent.lastIndexOf(prefix, approx);
-		if (last > 0) return last;
+		if (candidates.length > 0) {
+			const newlineCandidates = candidates.filter((c) => c.isNewline);
+			const pool = newlineCandidates.length > 0 ? newlineCandidates : candidates;
+			let bestCandidate = pool[0];
+			let bestDistance = Math.abs(pool[0].pos - expectedBoundary);
+			for (let i = 1; i < pool.length; i++) {
+				const dist = Math.abs(pool[i].pos - expectedBoundary);
+				if (dist < bestDistance) {
+					bestDistance = dist;
+					bestCandidate = pool[i];
+				}
+			}
+			const MAX_DEVIATION = 2e3;
+			if (bestDistance <= MAX_DEVIATION) return bestCandidate.pos;
+			logger?.debug?.("[breakpoints] findPageStartNearExpectedBoundary: Rejected match exceeding deviation", {
+				targetPageIdx,
+				expectedBoundary,
+				bestDistance,
+				maxDeviation: MAX_DEVIATION,
+				matchPos: bestCandidate.pos,
+				prefixLength: len
+			});
+		}
 	}
 	return -1;
 };
@@ -1085,6 +1346,7 @@ const findPageStartNearExpectedBoundary = (remainingContent, _currentFromIdx, ta
 * @param pageIds - Array of all page IDs
 * @param normalizedPages - Map of page ID to normalized content
 * @param cumulativeOffsets - Cumulative character offsets (for estimates)
+* @param logger - Optional logger for debugging
 * @returns Array where boundaryPositions[i] = start position of page (fromIdx + i),
 *          with a sentinel boundary at segmentContent.length as the last element
 *
@@ -1093,12 +1355,12 @@ const findPageStartNearExpectedBoundary = (remainingContent, _currentFromIdx, ta
 * buildBoundaryPositions(content, 0, 2, pageIds, normalizedPages, offsets)
 * // → [0, 23, 45, 67] where 67 is content.length (sentinel)
 */
-const buildBoundaryPositions = (segmentContent, fromIdx, toIdx, pageIds, normalizedPages, cumulativeOffsets) => {
+const buildBoundaryPositions = (segmentContent, fromIdx, toIdx, pageIds, normalizedPages, cumulativeOffsets, logger) => {
 	const boundaryPositions = [0];
 	const startOffsetInFromPage = estimateStartOffsetInCurrentPage(segmentContent, fromIdx, pageIds, normalizedPages);
 	for (let i = fromIdx + 1; i <= toIdx; i++) {
 		const expectedBoundary = cumulativeOffsets[i] !== void 0 && cumulativeOffsets[fromIdx] !== void 0 ? Math.max(0, cumulativeOffsets[i] - cumulativeOffsets[fromIdx] - startOffsetInFromPage) : segmentContent.length;
-		const pos = findPageStartNearExpectedBoundary(segmentContent, fromIdx, i, expectedBoundary, pageIds, normalizedPages);
+		const pos = findPageStartNearExpectedBoundary(segmentContent, fromIdx, i, expectedBoundary, pageIds, normalizedPages, logger);
 		const prevBoundary = boundaryPositions[boundaryPositions.length - 1];
 		if (pos > 0 && pos > prevBoundary && Math.abs(pos - expectedBoundary) < 2e3) boundaryPositions.push(pos);
 		else {
@@ -1142,18 +1404,20 @@ const findPageIndexForPosition = (position, boundaryPositions, fromIdx) => {
 * found within the actual `remainingContent` string being split. This avoids relying on raw page offsets
 * that can diverge when structural rules strip markers (e.g. `lineStartsAfter`).
 */
-const findBreakpointWindowEndPosition = (remainingContent, currentFromIdx, windowEndIdx, toIdx, pageIds, normalizedPages, cumulativeOffsets) => {
+const findBreakpointWindowEndPosition = (remainingContent, currentFromIdx, windowEndIdx, toIdx, pageIds, normalizedPages, cumulativeOffsets, logger) => {
 	if (windowEndIdx >= toIdx) return remainingContent.length;
 	const desiredNextIdx = windowEndIdx + 1;
 	const minNextIdx = currentFromIdx + 1;
 	const maxNextIdx = Math.min(desiredNextIdx, toIdx);
 	const startOffsetInCurrentPage = estimateStartOffsetInCurrentPage(remainingContent, currentFromIdx, pageIds, normalizedPages);
+	let bestExpectedBoundary = remainingContent.length;
 	for (let nextIdx = maxNextIdx; nextIdx >= minNextIdx; nextIdx--) {
 		const expectedBoundary = cumulativeOffsets[nextIdx] !== void 0 && cumulativeOffsets[currentFromIdx] !== void 0 ? Math.max(0, cumulativeOffsets[nextIdx] - cumulativeOffsets[currentFromIdx] - startOffsetInCurrentPage) : remainingContent.length;
-		const pos = findPageStartNearExpectedBoundary(remainingContent, currentFromIdx, nextIdx, expectedBoundary, pageIds, normalizedPages);
+		if (nextIdx === maxNextIdx) bestExpectedBoundary = expectedBoundary;
+		const pos = findPageStartNearExpectedBoundary(remainingContent, currentFromIdx, nextIdx, expectedBoundary, pageIds, normalizedPages, logger);
 		if (pos > 0) return pos;
 	}
-	return remainingContent.length;
+	return Math.min(bestExpectedBoundary, remainingContent.length);
 };
 /**
 * Finds exclusion-based break position using raw cumulative offsets.
@@ -1231,7 +1495,8 @@ const handlePageBoundaryBreak = (remainingContent, windowEndIdx, windowEndPositi
 		const nextPageData = normalizedPages.get(pageIds[nextPageIdx]);
 		if (nextPageData) {
 			const pos = findNextPagePosition(remainingContent, nextPageData);
-			if (pos > 0) return Math.min(pos, windowEndPosition, remainingContent.length);
+			const tolerance = Math.max(2e3, windowEndPosition * .5);
+			if (pos > 0 && Math.abs(pos - windowEndPosition) <= tolerance) return Math.min(pos, windowEndPosition, remainingContent.length);
 		}
 	}
 	return Math.min(windowEndPosition, remainingContent.length);
@@ -1255,14 +1520,14 @@ const findBreakPosition = (remainingContent, currentFromIdx, toIdx, windowEndIdx
 		if (hasExcludedPageInRange(excludeSet, pageIds, currentFromIdx, windowEndIdx)) continue;
 		if (skipWhenRegex?.test(remainingContent)) continue;
 		if (regex === null) return {
-			breakpointIndex: i,
 			breakPos: handlePageBoundaryBreak(remainingContent, windowEndIdx, windowEndPosition, toIdx, pageIds, normalizedPages),
+			breakpointIndex: i,
 			rule
 		};
 		const breakPos = findPatternBreakPosition(remainingContent.slice(0, Math.min(windowEndPosition, remainingContent.length)), regex, prefer);
 		if (breakPos > 0) return {
-			breakpointIndex: i,
 			breakPos,
+			breakpointIndex: i,
 			rule
 		};
 	}
@@ -1433,7 +1698,7 @@ const processOversizedSegment = (segment, fromIdx, toIdx, pageIds, normalizedPag
 	let currentFromIdx = fromIdx;
 	let isFirstPiece = true;
 	let lastBreakpoint = null;
-	const boundaryPositions = buildBoundaryPositions(fullContent, fromIdx, toIdx, pageIds, normalizedPages, cumulativeOffsets);
+	const boundaryPositions = buildBoundaryPositions(fullContent, fromIdx, toIdx, pageIds, normalizedPages, cumulativeOffsets, logger);
 	logger?.debug?.("[breakpoints] boundaryPositions built", {
 		boundaryPositions,
 		fromIdx,
@@ -1454,7 +1719,7 @@ const processOversizedSegment = (segment, fromIdx, toIdx, pageIds, normalizedPag
 			break;
 		}
 		const windowEndIdx = computeWindowEndIdx(currentFromIdx, toIdx, pageIds, maxPages);
-		const windowEndPosition = findBreakpointWindowEndPosition(remainingContent, currentFromIdx, windowEndIdx, toIdx, pageIds, normalizedPages, cumulativeOffsets);
+		const windowEndPosition = findBreakpointWindowEndPosition(remainingContent, currentFromIdx, windowEndIdx, toIdx, pageIds, normalizedPages, cumulativeOffsets, logger);
 		logger?.debug?.(`[breakpoints] iteration=${i}`, {
 			currentFromIdx,
 			cursorPos,
@@ -2536,7 +2801,7 @@ const buildTokenPriority = () => {
 	return TOKEN_PRIORITY_ORDER$1.filter((t) => allTokens.has(t));
 };
 const collapseWhitespace = (s) => s.replace(/\s+/g, " ").trim();
-const stripArabicDiacritics = (s) => s.replace(/[\u064B-\u065F\u0670\u06D6-\u06ED\u0640]/gu, "");
+const stripArabicDiacritics = (s) => s.replace(/[\u064B-\u065F\u0670\u06D6-\u06ED]/gu, "");
 const compileTokenRegexes = (tokenNames) => {
 	const compiled = [];
 	for (const token of tokenNames) {
@@ -3618,5 +3883,5 @@ function recoverMistakenMarkersForRuns(runs, opts) {
 }
 
 //#endregion
-export { TOKEN_PATTERNS, analyzeCommonLineStarts, analyzeRepeatingSequences, analyzeTextForRule, applyReplacements, containsTokens, detectTokenPatterns, escapeRegex, escapeTemplateBrackets, expandCompositeTokensInTemplate, expandTokens, expandTokensWithCaptures, generateTemplateFromText, getAvailableTokens, getTokenPattern, makeDiacriticInsensitive, recoverMistakenLineStartsAfterMarkers, recoverMistakenMarkersForRuns, segmentPages, suggestPatternConfig, templateToRegex, validateRules };
+export { PATTERN_TYPE_KEYS, TOKEN_PATTERNS, analyzeCommonLineStarts, analyzeRepeatingSequences, analyzeTextForRule, applyReplacements, applyTokenMappings, containsTokens, detectTokenPatterns, escapeRegex, escapeTemplateBrackets, expandCompositeTokensInTemplate, expandTokens, expandTokensWithCaptures, formatValidationReport, generateTemplateFromText, getAvailableTokens, getTokenPattern, makeDiacriticInsensitive, optimizeRules, recoverMistakenLineStartsAfterMarkers, recoverMistakenMarkersForRuns, segmentPages, shouldDefaultToFuzzy, stripTokenMappings, suggestPatternConfig, templateToRegex, validateRules };
 //# sourceMappingURL=index.mjs.map