locai 2.1.0

package/dist/utils/quality/special-characters-checker.js

@@ -0,0 +1,229 @@
+/**
+ * Special Characters Preservation Checker.
+ * Ensures special characters, symbols, and formatting are preserved.
+ */
+class SpecialCharactersChecker {
+    specialCharacterSets;
+    criticalCharacters;
+    constructor() {
+        // Define categories of special characters to preserve
+        this.specialCharacterSets = {
+            // Currency symbols
+            currency: /[$€£¥₹₽¢]/g,
+            // Math symbols
+            math: /[+\-×÷=≠<>≤≥±∞√∑∏]/g,
+            // Arrows
+            arrows: /[←→↑↓↔↕⇐⇒⇑⇓⇔]/g,
+            // Bullets and markers
+            bullets: /[•·●○◆◇■□▪▫]/g,
+            // Quotes and apostrophes
+            quotes: /[""''«»‹›]/g,
+            // Dashes and hyphens
+            dashes: /[–—−]/g,
+            // Special punctuation
+            punctuation: /[¿¡‽…]/g,
+            // Trademark and copyright
+            legal: /[©®™℠]/g,
+            // Fractions
+            fractions: /[¼½¾⅓⅔⅛⅜⅝⅞]/g,
+            // Degree and temperature
+            measurements: /[°℃℉]/g,
+        };
+        // Characters that should never be translated
+        this.criticalCharacters = [
+            ...Array.from("$€£¥₹₽¢"),
+            ...Array.from("©®™℠"),
+            ...Array.from("°℃℉"),
+        ];
+    }
+    /**
+     * Check if special characters are preserved.
+     */
+    checkSpecialCharacters(sourceText, translatedText) {
+        const issues = [];
+        // Check each category of special characters
+        for (const [category, pattern] of Object.entries(this.specialCharacterSets)) {
+            const sourceMatches = sourceText.match(pattern) || [];
+            const translatedMatches = translatedText.match(pattern) || [];
+            if (sourceMatches.length !== translatedMatches.length) {
+                issues.push({
+                    type: "special-characters",
+                    category,
+                    severity: this.criticalCharacters.some((char) => sourceMatches.includes(char))
+                        ? "error"
+                        : "warning",
+                    message: `${category} character count mismatch: source has ${sourceMatches.length}, translation has ${translatedMatches.length}`,
+                    details: {
+                        sourceChars: sourceMatches,
+                        translatedChars: translatedMatches,
+                        missing: sourceMatches.filter((char) => !translatedMatches.includes(char)),
+                        extra: translatedMatches.filter((char) => !sourceMatches.includes(char)),
+                    },
+                });
+            }
+            // Check for character substitutions (e.g., " to ")
+            const sourceUniqueChars = new Set(sourceMatches);
+            const translatedUniqueChars = new Set(translatedMatches);
+            for (const char of sourceUniqueChars) {
+                if (!translatedUniqueChars.has(char)) {
+                    issues.push({
+                        type: "special-character-substitution",
+                        category,
+                        severity: "warning",
+                        message: `Special character "${char}" was replaced or removed`,
+                        details: {
+                            originalChar: char,
+                            charCode: char.charCodeAt(0),
+                        },
+                    });
+                }
+            }
+        }
+        // Check for emoji preservation
+        const emojiIssues = this.checkEmojiPreservation(sourceText, translatedText);
+        issues.push(...emojiIssues);
+        return issues;
+    }
+    /**
+     * Check emoji preservation.
+     */
+    checkEmojiPreservation(sourceText, translatedText) {
+        const issues = [];
+        // Emoji regex pattern
+        const emojiPattern = /[\u{1F600}-\u{1F64F}\u{1F300}-\u{1F5FF}\u{1F680}-\u{1F6FF}\u{1F1E0}-\u{1F1FF}\u{2600}-\u{26FF}\u{2700}-\u{27BF}]/gu;
+        const sourceEmojis = sourceText.match(emojiPattern) || [];
+        const translatedEmojis = translatedText.match(emojiPattern) || [];
+        if (sourceEmojis.length !== translatedEmojis.length) {
+            issues.push({
+                type: "emoji-preservation",
+                severity: "warning",
+                message: `Emoji count mismatch: source has ${sourceEmojis.length}, translation has ${translatedEmojis.length}`,
+                details: {
+                    sourceEmojis,
+                    translatedEmojis,
+                },
+            });
+        }
+        return issues;
+    }
+    /**
+     * Fix special character issues.
+     */
+    fixSpecialCharacters(sourceText, translatedText) {
+        let fixedText = translatedText;
+        const foundIssues = [];
+        const appliedFixes = [];
+        // Extract special characters from source
+        const sourceSpecialChars = this.extractSpecialCharacters(sourceText);
+        // Check what's missing in translation
+        const translatedSpecialChars = this.extractSpecialCharacters(translatedText);
+        // Find missing critical characters
+        for (const [category, chars] of Object.entries(sourceSpecialChars)) {
+            const missingChars = chars.filter((char) => !translatedSpecialChars[category]?.includes(char));
+            if (missingChars.length > 0) {
+                foundIssues.push({
+                    type: "missing-special-chars",
+                    category,
+                    severity: "warning", // Added severity
+                    message: `Missing ${missingChars.length} ${category} characters`,
+                });
+                // Try to restore critical characters
+                if (this.criticalCharacters.some((critical) => missingChars.includes(critical))) {
+                    const result = this.restoreCriticalCharacters(sourceText, fixedText, missingChars);
+                    fixedText = result.text;
+                    appliedFixes.push(...result.appliedFixes);
+                }
+            }
+        }
+        // Fix common character substitutions
+        const substitutionResult = this.fixCommonSubstitutions(sourceText, fixedText);
+        fixedText = substitutionResult.text;
+        foundIssues.push(...substitutionResult.foundIssues);
+        appliedFixes.push(...substitutionResult.appliedFixes);
+        return { text: fixedText, foundIssues, appliedFixes };
+    }
+    /**
+     * Extract special characters by category.
+     */
+    extractSpecialCharacters(text) {
+        const extracted = {};
+        for (const [category, pattern] of Object.entries(this.specialCharacterSets)) {
+            const matches = text.match(pattern) || [];
+            extracted[category] = matches;
+        }
+        return extracted;
+    }
+    /**
+     * Restore critical characters that were lost.
+     */
+    restoreCriticalCharacters(sourceText, translatedText, missingChars) {
+        const appliedFixes = [];
+        const fixedText = translatedText;
+        // This is a simplified restoration
+        // In practice, we'd need context to know where to place the characters
+        for (const char of missingChars) {
+            if (this.criticalCharacters.includes(char)) {
+                // Find where this character appears in source
+                const sourceIndex = sourceText.indexOf(char);
+                if (sourceIndex !== -1) {
+                    appliedFixes.push({
+                        type: "critical-char-restore",
+                        message: `Noted missing critical character: "${char}"`,
+                    });
+                    // Note: Full restoration would require AI to determine placement
+                }
+            }
+        }
+        return { text: fixedText, appliedFixes };
+    }
+    /**
+     * Fix common character substitutions.
+     */
+    fixCommonSubstitutions(sourceText, translatedText) {
+        const foundIssues = [];
+        const appliedFixes = [];
+        let fixedText = translatedText;
+        // Common problematic substitutions
+        const substitutions = [
+            // Smart quotes to regular quotes (and vice versa)
+            { wrong: '"', correct: '"', check: sourceText.includes('"') },
+            { wrong: '"', correct: '"', check: sourceText.includes('"') },
+            { wrong: "'", correct: "'", check: sourceText.includes("'") },
+            { wrong: "'", correct: "'", check: sourceText.includes("'") },
+            // Dashes
+            { wrong: "-", correct: "–", check: sourceText.includes("–") },
+            { wrong: "-", correct: "—", check: sourceText.includes("—") },
+            // Ellipsis
+            { wrong: "...", correct: "…", check: sourceText.includes("…") },
+        ];
+        for (const { wrong, correct, check } of substitutions) {
+            if (check && fixedText.includes(wrong)) {
+                const count = (fixedText.match(new RegExp(wrong, "g")) || []).length;
+                foundIssues.push({
+                    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+                    type: "character-substitution",
+                    severity: "warning", // Added severity
+                    message: `Found ${count} instances of "${wrong}" that should be "${correct}"`,
+                });
+                fixedText = fixedText.replace(new RegExp(wrong, "g"), correct);
+                appliedFixes.push({
+                    type: "character-substitution",
+                    message: `Replaced "${wrong}" with "${correct}" (${count} instances)`,
+                });
+            }
+        }
+        return { text: fixedText, foundIssues, appliedFixes };
+    }
+    /**
+     * Check if text has special characters.
+     */
+    hasSpecialCharacters(text) {
+        for (const pattern of Object.values(this.specialCharacterSets)) {
+            if (pattern.test(text)) {
+                return true;
+            }
+        }
+        return false;
+    }
+}
+export default SpecialCharactersChecker;

package/dist/utils/quality/style-guide-checker.js

@@ -0,0 +1,192 @@
+/**
+ * Style Guide Checker.
+ * Enforces style guide conventions like Oxford comma and sentence case.
+ */
+class StyleGuideChecker {
+    styleGuide;
+    constructor(styleGuide = {}) {
+        this.styleGuide = {
+            formality: styleGuide?.formality || "neutral",
+            toneOfVoice: styleGuide?.toneOfVoice || "neutral",
+            toneProvider: styleGuide?.toneProvider || "openai",
+            enforceTone: styleGuide?.enforceTone || false, // Use the configured value directly
+            analysisOptions: styleGuide?.analysisOptions || {},
+            conventions: {
+                useOxfordComma: styleGuide?.conventions?.useOxfordComma ?? true,
+                useSentenceCase: styleGuide?.conventions?.useSentenceCase ?? true,
+                ...styleGuide?.conventions,
+            },
+        };
+    }
+    /**
+     * Check if text follows style guide conventions.
+     */
+    // eslint-disable-next-line @typescript-eslint/no-unused-vars
+    checkStyleGuide(_sourceText, translatedText) {
+        const issues = [];
+        if (this.styleGuide.conventions.useOxfordComma) {
+            const oxfordIssues = this.checkOxfordComma(translatedText);
+            issues.push(...oxfordIssues);
+        }
+        if (this.styleGuide.conventions.useSentenceCase) {
+            const sentenceCaseIssues = this.checkSentenceCase(translatedText);
+            issues.push(...sentenceCaseIssues);
+        }
+        return issues;
+    }
+    /**
+     * Check for Oxford comma usage in lists.
+     */
+    checkOxfordComma(text) {
+        const issues = [];
+        // Pattern: word, word and word (missing Oxford comma)
+        // Should be: word, word, and word
+        const listPattern = /(\w+),\s+(\w+)\s+(?:and|or)\s+(\w+)/gi;
+        const matches = text.matchAll(listPattern);
+        for (const match of matches) {
+            const fullMatch = match[0];
+            // Check if there's already a comma before 'and/or'
+            if (!fullMatch.match(/,\s+(?:and|or)/)) {
+                issues.push({
+                    type: "oxford-comma",
+                    severity: "warning",
+                    message: `Missing Oxford comma in list: "${fullMatch}"`,
+                    suggestion: fullMatch.replace(/(\w+)\s+(and|or)/, "$1, $2"),
+                });
+            }
+        }
+        return issues;
+    }
+    /**
+     * Check sentence case for headings/titles.
+     */
+    checkSentenceCase(text) {
+        const issues = [];
+        // Check for potential headings (short lines with multiple capital letters)
+        const lines = text.split("\n");
+        for (const line of lines) {
+            const trimmed = line.trim();
+            // Skip empty lines and very long lines (likely not headings)
+            if (!trimmed || trimmed.length > 100)
+                continue;
+            // Check if line looks like a title (multiple consecutive capitals)
+            const allCapsPattern = /^[A-Z\s]+$/;
+            if (allCapsPattern.test(trimmed) && trimmed.length > 5) {
+                issues.push({
+                    type: "sentence-case",
+                    severity: "info",
+                    message: `All caps heading detected: "${trimmed}"`,
+                    suggestion: this.convertToSentenceCase(trimmed),
+                });
+            }
+        }
+        return issues;
+    }
+    /**
+     * Fix style guide issues.
+     */
+    // eslint-disable-next-line @typescript-eslint/no-unused-vars
+    fixStyleGuide(sourceText, translatedText) {
+        let fixedText = translatedText;
+        const foundIssues = [];
+        const appliedFixes = [];
+        if (this.styleGuide.conventions.useOxfordComma) {
+            const result = this.fixOxfordComma(fixedText);
+            fixedText = result.text;
+            foundIssues.push(...result.foundIssues);
+            appliedFixes.push(...result.appliedFixes);
+        }
+        if (this.styleGuide.conventions.useSentenceCase) {
+            const result = this.fixSentenceCase(fixedText);
+            fixedText = result.text;
+            foundIssues.push(...result.foundIssues);
+            appliedFixes.push(...result.appliedFixes);
+        }
+        return { text: fixedText, foundIssues, appliedFixes };
+    }
+    /**
+     * Fix missing Oxford commas.
+     */
+    fixOxfordComma(text) {
+        const foundIssues = [];
+        const appliedFixes = [];
+        let fixedText = text;
+        // Pattern: word, word and word -> word, word, and word
+        const listPattern = /(\w+),\s+(\w+)\s+(and|or)\s+(\w+)/gi;
+        fixedText = fixedText.replace(listPattern, (match, word1, word2, conjunction, word3) => {
+            // Check if comma already exists before conjunction
+            if (!match.includes(`, ${conjunction}`)) {
+                foundIssues.push({
+                    type: "oxford-comma",
+                    severity: "warning", // Added severity
+                    message: `Missing Oxford comma: "${match}"`,
+                });
+                appliedFixes.push({
+                    type: "oxford-comma",
+                    message: `Added Oxford comma before "${conjunction}"`,
+                });
+                return `${word1}, ${word2}, ${conjunction} ${word3}`;
+            }
+            return match;
+        });
+        return { text: fixedText, foundIssues, appliedFixes };
+    }
+    /**
+     * Fix sentence case issues.
+     */
+    fixSentenceCase(text) {
+        const foundIssues = [];
+        const appliedFixes = [];
+        const lines = text.split("\n");
+        const fixedLines = [];
+        for (const line of lines) {
+            const trimmed = line.trim();
+            // Check for all caps headings
+            const allCapsPattern = /^[A-Z\s]+$/;
+            if (allCapsPattern.test(trimmed) && trimmed.length > 5 && trimmed.length < 100) {
+                foundIssues.push({
+                    type: "sentence-case",
+                    severity: "info", // Added severity
+                    message: `All caps heading: "${trimmed}"`,
+                });
+                const sentenceCase = this.convertToSentenceCase(trimmed);
+                fixedLines.push(line.replace(trimmed, sentenceCase));
+                appliedFixes.push({
+                    type: "sentence-case",
+                    message: `Converted to sentence case: "${trimmed}" → "${sentenceCase}"`,
+                });
+            }
+            else {
+                fixedLines.push(line);
+            }
+        }
+        return {
+            text: fixedLines.join("\n"),
+            foundIssues,
+            appliedFixes,
+        };
+    }
+    /**
+     * Convert text to sentence case.
+     */
+    convertToSentenceCase(text) {
+        // Capitalize first letter, lowercase the rest
+        // But preserve acronyms (2-3 letter all-caps words)
+        const words = text.split(/\s+/);
+        return words
+            .map((word, index) => {
+            // Keep short all-caps words (likely acronyms)
+            if (word.length <= 3 && word === word.toUpperCase()) {
+                return word;
+            }
+            // First word gets capital
+            if (index === 0) {
+                return word.charAt(0).toUpperCase() + word.slice(1).toLowerCase();
+            }
+            // Rest are lowercase (unless acronyms)
+            return word.toLowerCase();
+        })
+            .join(" ");
+    }
+}
+export default StyleGuideChecker;

package/dist/utils/quality/text-sanitizer.js

@@ -0,0 +1,120 @@
+/**
+ * Text Sanitizer.
+ * Cleans up translated text by removing AI artifacts, think tags, and unwanted formatting.
+ */
+class TextSanitizer {
+    /**
+     * Sanitize text by applying a sequence of cleaning rules.
+     */
+    sanitize(text) {
+        if (!text)
+            return text;
+        const rules = [
+            this.removeThinkTags,
+            this.removeMarkdownFormatting,
+            this.removeQuotes,
+            this.removeBulletPoints,
+            this.removeExplanations,
+            this.removeAIArtifacts,
+            this.removeAllArtifacts,
+            this.normalizeWhitespace,
+            this.trimSpecialChars,
+        ];
+        return this.applyRules(text, rules);
+    }
+    /**
+     * Apply a list of rules to text.
+     */
+    applyRules(text, rules) {
+        return rules.reduce((processedText, rule) => rule(processedText), text);
+    }
+    /**
+     * Remove <think> tags and their content.
+     */
+    removeThinkTags = (text) => {
+        // More robust think tag removal
+        return text
+            .replace(/<think>[\s\S]*?<\/think>/gi, "")
+            .replace(/<think[\s\S]*?<\/think>/gi, "")
+            .trim();
+    };
+    /**
+     * Remove bold/italic markdown formatting.
+     */
+    removeMarkdownFormatting = (text) => {
+        return text.replace(/\*\*.*?:\*\*/g, "");
+    };
+    /**
+     * Remove leading/trailing quotes.
+     */
+    removeQuotes = (text) => {
+        return text.replace(/^['"]|['"]$/g, "");
+    };
+    /**
+     * Remove bullet points at the start.
+     */
+    removeBulletPoints = (text) => {
+        return text.replace(/^\s*[-•]\s*/g, "");
+    };
+    /**
+     * Remove explanatory text and metadata.
+     */
+    removeExplanations = (text) => {
+        // Remove any text between common explanation markers
+        return text
+            .replace(/<think>[\s\S]*?<\/think>/g, "")
+            .replace(/\[.*?\]/g, "")
+            .replace(/\(.*?\)/g, "")
+            .replace(/^(Translation:|Translated text:|Result:|Output:)/gi, "");
+    };
+    /**
+     * Remove common AI conversational artifacts.
+     */
+    removeAIArtifacts = (text) => {
+        // Remove common AI model output patterns
+        return text
+            .replace(/^(Here's the translation:|The translation is:|I would translate this as:)/gi, "")
+            .replace(/^[A-Za-z]+ translation: /g, "")
+            .replace(/\b(Note|Remember|Important):.+$/gi, "");
+    };
+    /**
+     * Trim special characters from ends.
+     */
+    trimSpecialChars = (text) => {
+        // Remove special characters and extra whitespace
+        return text.replace(/^['"*_~`]+|['"*_~`]+$/g, "").replace(/^\s+|\s+$/g, "");
+    };
+    /**
+     * Normalize whitespace and remove duplicate lines.
+     */
+    normalizeWhitespace = (text) => {
+        // Remove duplicate lines and normalize whitespace
+        const lines = text
+            .split("\n")
+            .map((line) => line.trim())
+            .filter((line) => line); // Remove empty lines
+        // Remove duplicate consecutive lines
+        const uniqueLines = lines.filter((line, index, arr) => line !== arr[index - 1]);
+        return uniqueLines.join("\n");
+    };
+    /**
+     * Aggressive cleanup of all artifacts.
+     */
+    removeAllArtifacts = (text) => {
+        const cleaned = text
+            .replace(/<think>[\s\S]*?<\/think>/gi, "")
+            .replace(/^[A-Za-z]+ translation:[\s\S]*?\n/gim, "")
+            .replace(/^(Here's|This is|The) (the )?translation:?\s*/gim, "")
+            .replace(/^Translation result:?\s*/gim, "")
+            .replace(/^\s*[-•]\s*/gm, "")
+            .replace(/^['"]|['"]$/g, "")
+            .replace(/^\s+|\s+$/gm, "");
+        // Remove duplicate lines
+        const lines = cleaned
+            .split("\n")
+            .filter((line) => line.trim())
+            .filter((line, index, arr) => line !== arr[index - 1]);
+        return lines.join("\n");
+    };
+}
+export default TextSanitizer;

package/dist/utils/quality/tone-verifier.js

@@ -0,0 +1,91 @@
+import ProviderFactory from "../../core/provider-factory.js";
+export class ToneVerifier {
+    provider;
+    initialized = false;
+    constructor() {
+        // Provider will be initialized lazily to avoid startup overhead
+        // or circular dependency issues if ProviderFactory needs full env loaded
+    }
+    async initProvider(providerName) {
+        if (this.initialized && this.provider)
+            return;
+        // We use a lighter model if available, or just the default configured provider.
+        try {
+            // Try to get a specific "judge" provider if configured, otherwise default
+            this.provider = ProviderFactory.getProvider(providerName, true);
+        }
+        catch (error) {
+            // If specific provider fails, might fall back to any available
+            const available = ProviderFactory.getAvailableProviders();
+            if (available.length > 0) {
+                this.provider = ProviderFactory.getProvider(available[0], true);
+            }
+            else {
+                throw new Error("No AI providers available for ToneVerifier");
+            }
+        }
+        this.initialized = true;
+    }
+    async verify(sourceText, translatedText, sourceLang, targetLang, targetTone, providerName = "openai", options = {}) {
+        if (!targetTone) {
+            return { passed: true, score: 10, reasoning: "No target tone specified." };
+        }
+        await this.initProvider(providerName);
+        const systemInstruction = `Role: You are a strict linguistic tone auditor.
+Task: Analyze the tone of the provided translation against the Target Tone.
+
+Instructions:
+1. Analyze if the translation matches the Target Tone in the context of the Source text.
+2. Ignore minor grammatical errors; focus on TONE and PERSONA (e.g., Formal, Casual, Authoritative, Friendly).
+3. Rate the match on a scale of 0-10 (10 = Perfect match, 0 = Complete mismatch/Robot/Wrong tone).
+4. If score is below 7, set passed to false.
+
+Output JSON format ONLY:
+{
+  "passed": boolean,
+  "score": number, // 0-10
+  "reasoning": "brief explanation",
+  "suggestions": "optional suggestion for improvement"
+}`;
+        const userMessage = `
+Target Tone: "${targetTone}"
+Source (${sourceLang}): "${sourceText}"
+Translation (${targetLang}): "${translatedText}"
+`;
+        try {
+            let response;
+            if (this.provider.chat) {
+                response = await this.provider.chat([
+                    { role: "system", content: systemInstruction },
+                    { role: "user", content: userMessage },
+                ], options);
+            }
+            else if (this.provider.analyze) {
+                // Fallback to analyze if chat not available (unlikely for OpenAI)
+                response = await this.provider.analyze(systemInstruction + "\n" + userMessage, options);
+            }
+            else {
+                // Fallback to using translate as a generic prompt runner
+                response = await this.provider.translate(systemInstruction + "\n" + userMessage, "System", "JSON");
+            }
+            // Clean up response if it contains markdown code blocks
+            const cleanResponse = response.replace(/```json\n?|\n?```/g, "").trim();
+            const result = JSON.parse(cleanResponse);
+            return {
+                passed: result.passed,
+                score: result.score,
+                reasoning: result.reasoning || "No reasoning provided",
+                suggestions: result.suggestions,
+            };
+        }
+        catch (error) {
+            console.warn("Tone verification failed:", error);
+            // Fail open - if verification fails, we don't block the translation
+            return {
+                passed: true,
+                score: 0,
+                reasoning: `Verification failed: ${error.message}`,
+            };
+        }
+    }
+}