georgian-hyphenation 2.2.1 → 2.2.3

package/src/georgian_hyphenation/hyphenator.py

@@ -0,0 +1,358 @@
+# -*- coding: utf-8 -*-
+"""
+Georgian Hyphenation Library v2.2.1
+ქართული ენის დამარცვლის ბიბლიოთეკა
+
+Modernized & Optimized
+- Hybrid Engine: Algorithm + Dictionary
+- Harmonic Clusters Support
+- Gemination Handling
+- O(1) Cluster Lookup with Set
+
+Author: Guram Zhgamadze
+"""
+
+import json
+import os
+import re
+from typing import List, Dict, Set
+
+
+class GeorgianHyphenator:
+    """
+    Georgian language hyphenation with hybrid engine
+    
+    Features:
+    - Phonological distance analysis
+    - Dictionary-based exception handling
+    - Harmonic cluster awareness
+    - Gemination (double consonant) handling
+    - Anti-orphan protection
+    """
+    
+    def __init__(self, hyphen_char: str = '\u00AD'):
+        """
+        Initialize Georgian Hyphenator
+        
+        Args:
+            hyphen_char: Character to use for hyphenation (default: soft hyphen U+00AD)
+        """
+        self.hyphen_char = hyphen_char
+        self.vowels = 'აეიოუ'
+        self.left_min = 2
+        self.right_min = 2
+        
+        # v2.2.1: Optimized - Set for O(1) lookup instead of list
+        self.harmonic_clusters: Set[str] = {
+            'ბლ', 'ბრ', 'ბღ', 'ბზ', 'გდ', 'გლ', 'გმ', 'გნ', 'გვ', 'გზ', 'გრ',
+            'დრ', 'თლ', 'თრ', 'თღ', 'კლ', 'კმ', 'კნ', 'კრ', 'კვ', 'მტ', 'პლ', 
+            'პრ', 'ჟღ', 'რგ', 'რლ', 'რმ', 'სწ', 'სხ', 'ტკ', 'ტპ', 'ტრ', 'ფლ', 
+            'ფრ', 'ფქ', 'ფშ', 'ქლ', 'ქნ', 'ქვ', 'ქრ', 'ღლ', 'ღრ', 'ყლ', 'ყრ', 
+            'შთ', 'შპ', 'ჩქ', 'ჩრ', 'ცლ', 'ცნ', 'ცრ', 'ცვ', 'ძგ', 'ძვ', 'ძღ', 
+            'წლ', 'წრ', 'წნ', 'წკ', 'ჭკ', 'ჭრ', 'ჭყ', 'ხლ', 'ხმ', 'ხნ', 'ხვ', 'ჯგ'
+        }
+        
+        # v2.2.1: Dictionary for exception words
+        self.dictionary: Dict[str, str] = {}
+    
+    def _strip_hyphens(self, text: str) -> str:
+        """
+        Remove existing hyphenation symbols (Sanitization)
+        
+        Args:
+            text: Input text
+            
+        Returns:
+            Text without any hyphens
+        """
+        if not text:
+            return ''
+        # Remove soft hyphens and visible hyphens
+        return text.replace('\u00AD', '').replace(self.hyphen_char, '').replace('-', '')
+    
+    def load_library(self, data: Dict[str, str]) -> None:
+        """
+        Load custom dictionary
+        
+        Args:
+            data: Dictionary mapping words to their hyphenation
+                  Example: {"საქართველო": "სა-ქარ-თვე-ლო"}
+        """
+        if data and isinstance(data, dict):
+            self.dictionary.update(data)
+    
+    def load_default_library(self) -> None:
+        """
+        Load default exceptions dictionary from data/exceptions.json
+        
+        Works in both development and installed package modes.
+        Tries multiple locations to find the data file.
+        """
+        try:
+            package_dir = os.path.dirname(__file__)
+            
+            # Try multiple possible locations
+            locations = [
+                # Development mode (root data/ folder)
+                os.path.join(package_dir, '..', '..', 'data', 'exceptions.json'),
+                # Installed via pip (data/ copied to site-packages)
+                os.path.join(os.path.dirname(package_dir), 'data', 'exceptions.json'),
+                # Alternative installed location
+                os.path.join(package_dir, 'data', 'exceptions.json'),
+            ]
+            
+            data_file = None
+            for loc in locations:
+                abs_loc = os.path.abspath(loc)
+                if os.path.exists(abs_loc):
+                    data_file = abs_loc
+                    break
+            
+            if data_file:
+                with open(data_file, 'r', encoding='utf-8') as f:
+                    data = json.load(f)
+                    self.load_library(data)
+                    print(f"Georgian Hyphenation v2.2.1: Dictionary loaded ({len(self.dictionary)} words)")
+            else:
+                print("Georgian Hyphenation v2.2.1: Dictionary not found, using algorithm only")
+        
+        except Exception as e:
+            print(f"Georgian Hyphenation v2.2.1: Could not load dictionary ({e}), using algorithm only")
+    
+    def hyphenate(self, word: str) -> str:
+        """
+        Hyphenate a Georgian word
+        
+        v2.2.1 Behavior: Always strip existing hyphens and re-hyphenate.
+        This corrects any previously incorrect hyphenation.
+        
+        Args:
+            word: Georgian word to hyphenate
+            
+        Returns:
+            Hyphenated word with configured hyphen character
+        """
+        # v2.2.1: Always strip existing hyphens first (sanitization)
+        sanitized_word = self._strip_hyphens(word)
+        
+        # Remove punctuation for dictionary lookup
+        clean_word = re.sub(r'[.,/#!$%^&*;:{}=\-_`~()]', '', sanitized_word)
+        
+        # Check dictionary first (if available)
+        if clean_word in self.dictionary:
+            return self.dictionary[clean_word].replace('-', self.hyphen_char)
+        
+        # Fallback to algorithm
+        return self.apply_algorithm(sanitized_word)
+    
+    def apply_algorithm(self, word: str) -> str:
+        """
+        Apply hyphenation algorithm
+        
+        v2.2.1 Algorithm Features:
+        - Vowel-based syllable detection
+        - Gemination (double consonant) handling
+        - Harmonic cluster preservation
+        - Anti-orphan protection (leftMin=2, rightMin=2)
+        
+        Args:
+            word: Word to hyphenate
+            
+        Returns:
+            Hyphenated word
+        """
+        # Skip short words
+        if len(word) < (self.left_min + self.right_min):
+            return word
+        
+        # Find all vowel positions
+        vowel_indices = [i for i, char in enumerate(word) if char in self.vowels]
+        
+        # Need at least 2 vowels for hyphenation
+        if len(vowel_indices) < 2:
+            return word
+        
+        insert_points = []
+        
+        # Analyze each vowel pair
+        for i in range(len(vowel_indices) - 1):
+            v1 = vowel_indices[i]
+            v2 = vowel_indices[i + 1]
+            distance = v2 - v1 - 1  # Number of consonants between vowels
+            between_substring = word[v1 + 1:v2]
+            
+            candidate_pos = -1
+            
+            if distance == 0:
+                # V-V: Split between vowels (გა-ა-ნა-ლი-ზა)
+                candidate_pos = v1 + 1
+            elif distance == 1:
+                # V-C-V: Split after vowel (მა-მა)
+                candidate_pos = v1 + 1
+            else:
+                # V-CC...C-V: Complex case
+                
+                # v2.2.1: Check for gemination (double consonants)
+                double_consonant_index = -1
+                for j in range(len(between_substring) - 1):
+                    if between_substring[j] == between_substring[j + 1]:
+                        double_consonant_index = j
+                        break
+                
+                if double_consonant_index != -1:
+                    # Split between double consonants (კლას-სი, მას-სა)
+                    candidate_pos = v1 + 1 + double_consonant_index + 1
+                else:
+                    # v2.2.1: Check for harmonic clusters
+                    break_index = -1
+                    if distance >= 2:
+                        last_two = between_substring[distance - 2:distance]
+                        if last_two in self.harmonic_clusters:
+                            break_index = distance - 2
+                    
+                    if break_index != -1:
+                        # Split before harmonic cluster (ას-ტრო-ნო-მი-ა)
+                        candidate_pos = v1 + 1 + break_index
+                    else:
+                        # Default: split after first consonant (ბარ-ბა-რე)
+                        candidate_pos = v1 + 2
+            
+            # Anti-orphan protection: ensure minimum 2 chars on each side
+            if candidate_pos >= self.left_min and (len(word) - candidate_pos) >= self.right_min:
+                insert_points.append(candidate_pos)
+        
+        # Insert hyphens (from right to left to maintain positions)
+        result = list(word)
+        for pos in reversed(insert_points):
+            result.insert(pos, self.hyphen_char)
+        
+        return ''.join(result)
+    
+    def get_syllables(self, word: str) -> List[str]:
+        """
+        Get syllables as a list
+        
+        Args:
+            word: Word to split into syllables
+            
+        Returns:
+            List of syllables without hyphen characters
+        """
+        hyphenated = self.hyphenate(word)
+        return hyphenated.split(self.hyphen_char)
+    
+    def hyphenate_text(self, text: str) -> str:
+        """
+        Hyphenate entire Georgian text
+        
+        Preserves:
+        - Punctuation
+        - Non-Georgian characters
+        - Word boundaries
+        - Whitespace
+        
+        v2.2.1: Strips existing hyphens from entire text first
+        
+        Args:
+            text: Text to hyphenate (can contain multiple words)
+            
+        Returns:
+            Hyphenated text
+        """
+        if not text:
+            return ''
+        
+        # v2.2.1: Strip existing hyphens from entire text
+        sanitized_text = self._strip_hyphens(text)
+        
+        # Split text into Georgian words and other characters
+        # Pattern captures Georgian letter sequences
+        parts = re.split(r'([ა-ჰ]+)', sanitized_text)
+        
+        result = []
+        for part in parts:
+            # Only hyphenate Georgian words with 4+ characters
+            if len(part) >= 4 and re.search(r'[ა-ჰ]', part):
+                result.append(self.hyphenate(part))
+            else:
+                result.append(part)
+        
+        return ''.join(result)
+
+
+# Convenience functions for backward compatibility and quick usage
+
+def hyphenate(word: str, hyphen_char: str = '\u00AD') -> str:
+    """
+    Hyphenate a single Georgian word
+    
+    Args:
+        word: Georgian word
+        hyphen_char: Hyphen character to use
+        
+    Returns:
+        Hyphenated word
+    """
+    h = GeorgianHyphenator(hyphen_char)
+    return h.hyphenate(word)
+
+
+def get_syllables(word: str) -> List[str]:
+    """
+    Get syllables of a Georgian word
+    
+    Args:
+        word: Georgian word
+        
+    Returns:
+        List of syllables
+    """
+    h = GeorgianHyphenator('-')
+    return h.get_syllables(word)
+
+
+def hyphenate_text(text: str, hyphen_char: str = '\u00AD') -> str:
+    """
+    Hyphenate Georgian text
+    
+    Args:
+        text: Text containing Georgian words
+        hyphen_char: Hyphen character to use
+        
+    Returns:
+        Hyphenated text
+    """
+    h = GeorgianHyphenator(hyphen_char)
+    return h.hyphenate_text(text)
+
+
+# Export format converters (v2.0 compatibility)
+
+def to_tex_pattern(word: str) -> str:
+    """
+    Convert to TeX hyphenation pattern format
+    
+    Args:
+        word: Georgian word
+        
+    Returns:
+        TeX pattern (e.g., ".სა1ქარ1თვე1ლო.")
+    """
+    h = GeorgianHyphenator('-')
+    syllables = h.get_syllables(word)
+    return '.' + '1'.join(syllables) + '.'
+
+
+def to_hunspell_format(word: str) -> str:
+    """
+    Convert to Hunspell hyphenation format
+    
+    Args:
+        word: Georgian word
+        
+    Returns:
+        Hunspell format (e.g., "სა=ქარ=თვე=ლო")
+    """
+    h = GeorgianHyphenator('-')
+    hyphenated = h.hyphenate(word)
+    return hyphenated.replace('-', '=')

package/src/georgian_hyphenation/hyphenator.py.backup

@@ -0,0 +1,312 @@
+# -*- coding: utf-8 -*-
+"""
+Georgian Language Hyphenation Library
+ქართული ენის დამარცვლის ბიბლიოთეკა
+
+Supports multiple output formats:
+- Soft hyphens for web/documents
+- TeX hyphenation patterns
+- Hunspell dictionary format
+- Plain syllable lists
+"""
+
+import re
+from functools import reduce
+from typing import List, Dict, Optional
+import json
+
+
+class GeorgianHyphenator:
+    """
+    Main hyphenation class for Georgian language
+    ქართული ენის დამარცვლის ძირითადი კლასი
+    """
+    
+    def __init__(self, hyphen_char: str = '\u00AD'):
+        """
+        Initialize hyphenator with specified hyphen character
+        
+        Args:
+            hyphen_char: Character to use for hyphenation points
+                        Default is soft hyphen (U+00AD)
+        """
+        self.hyphen_char = hyphen_char
+        self.C = '[ბგდვზთკლმნპჟრსტფქღყშჩცძწჭხჯჰ]'  # Consonants
+        self.V = '[აეიოუ]'                          # Vowels
+        self.char = '[ა-ჰ]'                         # All Georgian letters
+    
+    def count_vowels(self, word: str) -> int:
+        """Count vowels in a word"""
+        vowel_counts = [word.count(x) for x in "აეიოუ"]
+        return reduce(lambda x, y: x + y, vowel_counts, 0)
+    
+    def hyphenate(self, word: str) -> str:
+        """
+        Hyphenate a single Georgian word
+        
+        Args:
+            word: Georgian word to hyphenate
+            
+        Returns:
+            Word with hyphenation points inserted
+        """
+        # Don't hyphenate words with 0-1 vowels
+        if self.count_vowels(word) <= 1:
+            return word
+        
+        softhpn = self.hyphen_char
+        
+        # Apply hyphenation rules with different boundary markers
+        result = self._apply_rules(word, softhpn, '^', '$')
+        result = self._apply_rules(result, softhpn, '^', softhpn)
+        result = self._apply_rules(result, softhpn, softhpn, '$')
+        result = self._apply_rules(result, softhpn, softhpn, softhpn)
+        
+        # Remove duplicate hyphens
+        result = re.sub(f"{re.escape(softhpn)}+", softhpn, result, flags=re.U)
+        
+        return result
+    
+    def _apply_rules(self, w: str, softhpn: str, startchar: str, endchar: str) -> str:
+        """Apply hyphenation regex rules"""
+        C, V, char = self.C, self.V, self.char
+        
+        # Rule 1: V+C+C++V → VC|CV
+        t = re.sub(f"({V})({C})({C}+)({V})", 
+                   rf"\1\2{softhpn}\3\4", w, flags=re.U)
+        
+        # Rule 2: V+C+V+C+V → VCV|CV
+        t = re.sub(f"({V})({C})({V})({C})({V})", 
+                   rf"\1\2\3{softhpn}\4\5", t, flags=re.U)
+        
+        # Rule 3: C+V+C+V → CV|CV
+        t = re.sub(f"({C})({V})({C})({V})", 
+                   rf"\1\2{softhpn}\3\4", t, flags=re.U)
+        
+        # Rule 4: V+V+V → VV|V
+        t = re.sub(f"({V})({V})({V})", 
+                   rf"\1\2{softhpn}\3", t, flags=re.U)
+        
+        # Rule 5: Word start - ^VCVCV
+        t = re.sub(f"{startchar}({V})({C})({V})({C})({V})", 
+                   rf"\1\2\3{softhpn}\4\5", t, flags=re.U)
+        
+        # Rule 6: Word start - ^VCVCchar
+        t = re.sub(f"{startchar}({V})({C})({V})({C})({char})", 
+                   rf"\1\2\3{softhpn}\4\5", t, flags=re.U)
+        
+        # Rule 7: Word start - ^C++CVCV
+        t = re.sub(f"{startchar}({C}+)({V})({C})({V})", 
+                   rf"\1\2{softhpn}\3\4", t, flags=re.U)
+        
+        # Rule 8: Word start - ^C++VVchar
+        t = re.sub(f"{startchar}({C}+)({V})({V})({char})", 
+                   rf"\1\2{softhpn}\3\4", t, flags=re.U)
+        
+        # Rule 9: Word end - charVVC++$
+        t = re.sub(f"({char})({V})({V})({C}+){endchar}", 
+                   rf"\1\2{softhpn}\3\4", t, flags=re.U)
+        
+        # Rule 10: Word end - charVCV$
+        t = re.sub(f"({char})({V})({C})({V}){endchar}", 
+                   rf"\1\2{softhpn}\3\4", t, flags=re.U)
+        
+        # Rule 11: Word end - VCC++VC++$
+        t = re.sub(f"({V})({C})({C}+)({V})({C}+){endchar}", 
+                   rf"\1\2{softhpn}\3\4\5", t, flags=re.U)
+        
+        # Rule 12: Word end - charVCVC++$
+        t = re.sub(f"({char})({V})({C})({V}+)({C}+){endchar}", 
+                   rf"\1\2{softhpn}\3\4\5", t, flags=re.U)
+        
+        return t
+    
+    def get_syllables(self, word: str) -> List[str]:
+        """
+        Get list of syllables for a word
+        
+        Args:
+            word: Georgian word
+            
+        Returns:
+            List of syllables
+        """
+        hyphenated = self.hyphenate(word)
+        return hyphenated.split(self.hyphen_char)
+    
+    def hyphenate_text(self, text: str) -> str:
+        """
+        Hyphenate entire text
+        
+        Args:
+            text: Georgian text
+            
+        Returns:
+            Hyphenated text
+        """
+        words = text.split(' ')
+        hyphenated_words = [self.hyphenate(w) for w in words]
+        return ' '.join(hyphenated_words)
+
+
+class TeXPatternGenerator:
+    """Generate TeX hyphenation patterns"""
+    
+    def __init__(self, hyphenator: GeorgianHyphenator):
+        self.hyphenator = hyphenator
+    
+    def word_to_pattern(self, word: str) -> str:
+        """
+        Convert a word to TeX pattern format
+        
+        Example: საქართველო → .სა1ქარ1თვე1ლო
+        """
+        syllables = self.hyphenator.get_syllables(word)
+        if len(syllables) <= 1:
+            return f".{word}"
+        return "." + "1".join(syllables)
+    
+    def generate_patterns_file(self, words: List[str], output_file: str):
+        """
+        Generate complete TeX patterns file
+        
+        Args:
+            words: List of Georgian words
+            output_file: Path to output .tex file
+        """
+        patterns = [self.word_to_pattern(w) for w in words]
+        
+        with open(output_file, 'w', encoding='utf-8') as f:
+            f.write("% Georgian hyphenation patterns\n")
+            f.write("% ქართული დამარცვლის პატერნები\n")
+            f.write("% Generated automatically\n\n")
+            f.write("\\patterns{\n")
+            for pattern in sorted(set(patterns)):
+                f.write(f"  {pattern}\n")
+            f.write("}\n")
+        
+        print(f"TeX patterns saved to {output_file}")
+
+
+class HunspellDictionaryGenerator:
+    """Generate Hunspell dictionary format"""
+    
+    def __init__(self, hyphenator: GeorgianHyphenator):
+        self.hyphenator = hyphenator
+    
+    def word_to_hunspell(self, word: str) -> str:
+        """
+        Convert word to Hunspell format
+        
+        Example: საქართველო → სა=ქარ=თვე=ლო
+        """
+        syllables = self.hyphenator.get_syllables(word)
+        return "=".join(syllables)
+    
+    def generate_dictionary(self, words: List[str], output_prefix: str):
+        """
+        Generate Hunspell .dic file
+        
+        Args:
+            words: List of Georgian words
+            output_prefix: Prefix for output files (e.g., 'hyph_ka_GE')
+        """
+        dic_file = f"{output_prefix}.dic"
+        
+        with open(dic_file, 'w', encoding='utf-8') as f:
+            # Header with word count
+            f.write(f"UTF-8\n{len(words)}\n")
+            for word in words:
+                f.write(self.word_to_hunspell(word) + "\n")
+        
+        print(f"Hunspell dictionary saved to {dic_file}")
+
+
+class HyphenationExporter:
+    """Export hyphenation data in various formats"""
+    
+    def __init__(self, hyphenator: GeorgianHyphenator):
+        self.hyphenator = hyphenator
+    
+    def export_json(self, words: List[str], output_file: str):
+        """Export as JSON for JavaScript usage"""
+        data = {}
+        for word in words:
+            data[word] = {
+                "syllables": self.hyphenator.get_syllables(word),
+                "hyphenated": self.hyphenator.hyphenate(word)
+            }
+        
+        with open(output_file, 'w', encoding='utf-8') as f:
+            json.dump(data, f, ensure_ascii=False, indent=2)
+        
+        print(f"JSON export saved to {output_file}")
+    
+    def export_csv(self, words: List[str], output_file: str):
+        """Export as CSV"""
+        import csv
+        
+        with open(output_file, 'w', encoding='utf-8', newline='') as f:
+            writer = csv.writer(f)
+            writer.writerow(['word', 'syllables', 'syllable_count'])
+            
+            for word in words:
+                syllables = self.hyphenator.get_syllables(word)
+                writer.writerow([word, '-'.join(syllables), len(syllables)])
+        
+        print(f"CSV export saved to {output_file}")
+
+
+# Test and demonstration
+if __name__ == "__main__":
+    # Initialize hyphenator
+    hyphenator = GeorgianHyphenator()
+    
+    # Test words
+    test_words = [
+        "საქართველო",
+        "მთავრობა",
+        "დედაქალაქი",
+        "ტელევიზორი",
+        "კომპიუტერი",
+        "უნივერსიტეტი",
+        "პარლამენტი",
+        "დამოუკიდებლობა",
+        "განათლება",
+        "ეკონომიკა"
+    ]
+    
+    print("=" * 60)
+    print("Georgian Hyphenation Examples")
+    print("ქართული დამარცვლის მაგალითები")
+    print("=" * 60)
+    print()
+    
+    # Test basic hyphenation with visible hyphens
+    visible_hyphenator = GeorgianHyphenator('-')
+    for word in test_words:
+        syllables = visible_hyphenator.get_syllables(word)
+        hyphenated = visible_hyphenator.hyphenate(word)
+        print(f"{word:20} → {hyphenated:25} [{len(syllables)} syllables]")
+    
+    print("\n" + "=" * 60)
+    print("Generating export files...")
+    print("=" * 60)
+    print()
+    
+    # Generate TeX patterns
+    tex_gen = TeXPatternGenerator(hyphenator)
+    tex_gen.generate_patterns_file(test_words, "hyph-ka.tex")
+    
+    # Generate Hunspell dictionary
+    hunspell_gen = HunspellDictionaryGenerator(hyphenator)
+    hunspell_gen.generate_dictionary(test_words, "hyph_ka_GE")
+    
+    # Generate exports
+    exporter = HyphenationExporter(hyphenator)
+    exporter.export_json(test_words, "georgian_hyphenation.json")
+    exporter.export_csv(test_words, "georgian_hyphenation.csv")
+    
+    print("\n" + "=" * 60)
+    print("All files generated successfully!")
+    print("=" * 60)