jtcg_locale_detector 1.0.1

data/python/src/zhon/pinyin.py

@@ -0,0 +1,187 @@
+# -*- coding: utf-8 -*-
+"""Constants for processing Pinyin strings."""
+
+from string import whitespace
+from re import escape
+
+
+_a = "a\u0101\u00E0\u00E1\u01CE"
+_e = "e\u0113\u00E9\u011B\u00E8"
+_i = "i\u012B\u00ED\u01D0\u00EC"
+_o = "o\u014D\u00F3\u01D2\u00F2"
+_u = "u\u016B\u00FA\u01D4\u00F9"
+_v = "v\u00FC\u01D6\u01D8\u01DA\u01DC"
+
+_lowercase_vowels = _a + _e + _i + _o + _u + _v
+_uppercase_vowels = _lowercase_vowels.upper()
+_lowercase_consonants = "bpmfdtnlgkhjqxzcsrwy"
+_uppercase_consonants = _lowercase_consonants.upper()
+
+#: A string containing every Pinyin vowel (lowercase and uppercase).
+vowels = _lowercase_vowels + _uppercase_vowels
+
+#: A string containing every Pinyin consonant (lowercase and uppercase).
+consonants = _lowercase_consonants + _uppercase_consonants
+
+#: A string containing every lowercase Pinyin character.
+lowercase = _lowercase_consonants + _lowercase_vowels
+
+#: A string containing every uppercase Pinyin character.
+uppercase = _uppercase_consonants + _uppercase_vowels
+
+#: A string containing all Pinyin marks that have special meaning:
+#: middle dot and numbers for tones, colon for easily writing \u00FC ('u:'),
+#: hyphen for connecting syllables within words, and apostrophe for
+#: separating a syllable beginning with a vowel from the previous syllable
+#: in its word. All of these marks can be used within a valid Pinyin word.
+marks = "·012345:-'"
+
+#: A string containing valid punctuation marks that are not stops.
+non_stops = """"#$%&'()*+,-/\\:;<=>@[]^_`{|}~"""
+
+#: A string containing valid stop punctuation marks.
+stops = ".!?"
+
+#: A string containing all punctuation marks.
+punctuation = non_stops + stops
+
+#: A string containing all printable Pinyin characters, marks, punctuation,
+#: and whitespace.
+printable = vowels + consonants + marks[:-3] + whitespace + punctuation
+
+_a_vowels = {"a": _a, "e": _e, "i": _i, "o": _o, "u": _u, "v": _v}
+_n_vowels = {"a": "a", "e": "e", "i": "i", "o": "o", "u": "u", "v": "v\u00FC"}
+
+
+def _build_syl(vowels, tone_numbers=False):
+    """Builds a Pinyin syllable re pattern.
+
+    Syllables can be preceded by a middle dot (tone mark). Syllables that end
+    in a consonant are only valid if they aren't followed directly by a vowel
+    with no apostrophe in between.
+
+    The rough approach used to validate a Pinyin syllable is:
+        1. Get the longest valid syllable.
+        2. If it ends in a consonant make sure it's not followed directly by a
+            vowel (hyphens and apostrophes don't count).
+        3. If the above didn't match, repeat for the next longest valid match.
+
+    Lookahead assertions are used to ensure that hyphens and apostrophes are
+    only considered valid if used correctly. This helps to weed out non-Pinyin
+    strings.
+
+    """
+    # This is the end-of-syllable-consonant lookahead assertion.
+    consonant_end = "(?![{a}{e}{i}{o}{u}{v}]|u:)".format(
+        a=_a, e=_e, i=_i, o=_o, u=_u, v=_v
+    )
+    _vowels = vowels.copy()
+    for v, s in _vowels.items():
+        if len(s) > 1:
+            _vowels[v] = "[{}]".format(s)
+    return (
+        "(?:\u00B7|\u2027)?"
+        "(?:"
+        "(?:(?:[zcs]h|[gkh])u%(a)sng%(consonant_end)s)|"
+        "(?:[jqx]i%(o)sng%(consonant_end)s)|"
+        "(?:[nljqx]i%(a)sng%(consonant_end)s)|"
+        "(?:(?:[zcs]h?|[dtnlgkhrjqxy])u%(a)sn%(consonant_end)s)|"
+        "(?:(?:[zcs]h|[gkh])u%(a)si)|"
+        "(?:(?:[zc]h?|[rdtnlgkhsy])%(o)sng%(consonant_end)s)|"
+        "(?:(?:[zcs]h?|[rbpmfdtnlgkhw])?%(e)sng%(consonant_end)s)|"
+        "(?:(?:[zcs]h?|[rbpmfdtnlgkhwy])?%(a)sng%(consonant_end)s)|"
+        "(?:[bpmdtnljqxy]%(i)sng%(consonant_end)s)|"
+        "(?:[bpmdtnljqx]i%(a)sn%(consonant_end)s)|"
+        "(?:[bpmdtnljqx]i%(a)so)|"
+        "(?:[nl](?:v|u:|\u00FC)%(e)s)|"
+        "(?:[nl](?:%(v)s|u:))|"
+        "(?:[jqxy]u%(e)s)|"
+        "(?:[bpmnljqxy]%(i)sn%(consonant_end)s)|"
+        "(?:[mdnljqx]i%(u)s)|"
+        "(?:[bpmdtnljqx]i%(e)s)|"
+        "(?:[dljqx]i%(a)s)|"
+        "(?:(?:[zcs]h?|[rdtnlgkhxqjy])%(u)sn%(consonant_end)s)|"
+        "(?:(?:[zcs]h?|[rdtgkh])u%(i)s)|"
+        "(?:(?:[zcs]h?|[rdtnlgkh])u%(o)s)|"
+        "(?:(?:[zcs]h|[rgkh])u%(a)s)|"
+        "(?:(?:[zcs]h?|[rbpmfdngkhw])?%(e)sn%(consonant_end)s)|"
+        "(?:(?:[zcs]h?|[rbpmfdtnlgkhwy])?%(a)sn%(consonant_end)s)|"
+        "(?:(?:[zcs]h?|[rpmfdtnlgkhy])?%(o)su)|"
+        "(?:(?:[zcs]h?|[rbpmdtnlgkhy])?%(a)so)|"
+        "(?:(?:[zs]h|[bpmfdtnlgkhwz])?%(e)si)|"
+        "(?:(?:[zcs]h?|[bpmdtnlgkhw])?%(a)si)|"
+        "(?:(?:[zcs]h?|[rjqxybpmdtnl])%(i)s)|"
+        "(?:(?:[zcs]h?|[rwbpmfdtnlgkhjqxwy])%(u)s)|"
+        "(?:%(e)s(?:r%(consonant_end)s)?)|"
+        "(?:(?:[zcs]h?|[rmdtnlgkhy])%(e)s)|"
+        "(?:[bpmfwyl]?%(o)s)|"
+        "(?:(?:[zcs]h|[bpmfdtnlgkhzcswy])?%(a)s)|"
+        "(?:r%(consonant_end)s)"
+        ")" + ("[0-5]?" if tone_numbers else "")
+    ) % {
+        "consonant_end": consonant_end,
+        "a": _vowels["a"],
+        "e": _vowels["e"],
+        "i": _vowels["i"],
+        "o": _vowels["o"],
+        "u": _vowels["u"],
+        "v": _vowels["v"],
+    }
+
+
+def _build_word(syl, vowels):
+    """Builds a Pinyin word re pattern from a Pinyin syllable re pattern.
+
+    A word is defined as a series of consecutive valid Pinyin syllables
+    with optional hyphens and apostrophes interspersed. Hyphens must be
+    followed immediately by another valid Pinyin syllable. Apostrophes must be
+    followed by another valid Pinyin syllable that starts with an 'a', 'e', or
+    'o'.
+
+    """
+    return "(?:{syl}(?:-(?={syl})|'(?=[{a}{e}{o}])(?={syl}))?)+".format(
+        syl=syl, a=vowels["a"], e=vowels["e"], o=vowels["o"]
+    )
+
+
+def _build_sentence(word):
+    """Builds a Pinyin sentence re pattern from a Pinyin word re pattern.
+
+    A sentence is defined as a series of valid Pinyin words, punctuation
+    (non-stops), and spaces followed by a single stop and zero or more
+    container-closing punctuation marks (e.g. apostrophe and brackets).
+
+    """
+    return r"(?:{word}|[{non_stops}]|(?<![{stops} ]) )+[{stops}]['\"\]}}\)]*".format(
+        word=word, non_stops=escape(non_stops), stops=escape(stops)
+    )
+
+
+#: A regular expression pattern for a valid accented Pinyin syllable.
+a_syl = acc_syl = accented_syllable = _build_syl(_a_vowels, tone_numbers=False)
+
+#: A regular expression pattern for a valid numbered Pinyin syllable.
+n_syl = num_syl = numbered_syllable = _build_syl(_n_vowels, tone_numbers=True)
+
+#: A regular expression pattern for a valid Pinyin syllable.
+syl = syllable = _build_syl(_a_vowels, tone_numbers=True)
+
+
+#: A regular expression pattern for a valid accented Pinyin word.
+a_word = acc_word = accented_word = _build_word(a_syl, _a_vowels)
+
+#: A regular expression pattern for a valid numbered Pinyin word.
+n_word = num_word = numbered_word = _build_word(n_syl, _n_vowels)
+
+#: A regular expression pattern for a valid Pinyin word.
+word = _build_word(syl, _a_vowels)
+
+
+#: A regular expression pattern for a valid accented Pinyin sentence.
+a_sent = acc_sent = accented_sentence = _build_sentence(a_word)
+
+#: A regular expression pattern for a valid numbered Pinyin sentence.
+n_sent = num_sent = numbered_sentence = _build_sentence(n_word)
+
+#: A regular expression pattern for a valid Pinyin sentence.
+sent = sentence = _build_sentence(word)

data/python/src/zhon/zhuyin.py

@@ -0,0 +1,46 @@
+# -*- coding: utf-8 -*-
+"""Constants for working with Zhuyin (Bopomofo)."""
+
+#: A string containing all Zhuyin characters.
+characters = (
+    "ㄅㄆㄇㄈㄉㄊㄋㄌㄍㄎㄏㄐㄑㄒㄓㄔㄕㄖㄗㄘㄙ" "ㄚㄛㄝㄜㄞㄟㄠㄡㄢㄣㄤㄥㄦㄧㄨㄩㄭ"
+)
+
+#: A string containing all Zhuyin tone marks.
+marks = (
+    "\u02C7"  # Caron
+    "\u02CA"  # Modifier letter accute accent
+    "\u02CB"  # Modifier letter grave accent
+    "\u02D9"  # Dot above
+)
+
+#: A regular expression pattern for a Zhuyin syllable.
+syl = syllable = (
+    "(?:"
+    "[ㄇㄉㄊㄋㄌㄍㄎㄏㄓㄔㄕㄖㄗㄘㄙ]?ㄜ|"
+    "[ㄅㄆㄇㄉㄊㄋㄌㄍㄎㄏㄓㄔㄕㄗㄘㄙㄧ]?ㄞ|"
+    "[ㄅㄆㄇㄈㄉㄋㄌㄍㄏㄓㄕㄗ]?ㄟ|"
+    "[ㄅㄆㄇㄈㄋㄍㄎㄏㄓㄔㄕㄖㄗㄘㄙ]?ㄣ|"
+    "[ㄉㄌㄐㄑㄒ]?ㄧㄚ|"
+    "[ㄅㄆㄇㄈㄉㄊㄋㄌㄍㄎㄏㄓㄔㄕㄗㄘㄙ]?ㄚ|"
+    "[ㄅㄆㄇㄉㄊㄋㄌㄍㄎㄏㄓㄔㄕㄖㄗㄘㄙ]?ㄠ|"
+    "[ㄆㄇㄈㄉㄊㄋㄌㄍㄎㄏㄓㄔㄕㄖㄗㄘㄙ]?ㄡ|"
+    "[ㄅㄆㄇㄈㄉㄊㄋㄌㄍㄎㄏㄓㄔㄕㄖㄗㄘㄙ]?ㄢ|"
+    "[ㄇㄉㄋㄌㄐㄑㄒ]?ㄧㄡ|"
+    "[ㄅㄆㄇㄋㄌㄐㄑㄒ]?ㄧㄣ|"
+    "[ㄐㄑㄒ]?ㄩ[ㄢㄥ]|"
+    "[ㄌㄐㄑㄒ]?ㄩㄣ|"
+    "[ㄋㄌㄐㄑㄒ]?(?:ㄩㄝ?|ㄧㄤ)|"
+    "[ㄅㄆㄇㄈㄌㄧ]?ㄛ|"
+    "[ㄅㄆㄇㄉㄊㄋㄌㄐㄑㄒ]?ㄧ[ㄝㄠㄢㄥ]?|"
+    "[ㄅㄆㄇㄈㄉㄊㄋㄌㄍㄎㄏㄓㄔㄕㄖㄗㄘㄙ]?[ㄤㄥ]|"
+    "[ㄍㄎㄏㄓㄔㄕ]?ㄨ[ㄚㄞㄤ]|"
+    "[ㄉㄊㄋㄌㄍㄎㄏㄓㄔㄕㄖㄗㄘㄙ]?ㄨㄛ|"
+    "[ㄉㄊㄍㄎㄏㄓㄔㄕㄖㄗㄘㄙ]?ㄨㄟ|"
+    "[ㄉㄊㄋㄌㄍㄎㄏㄓㄔㄕㄖㄗㄘㄙ]?ㄨㄢ|"
+    "[ㄉㄊㄌㄍㄎㄏㄓㄔㄕㄖㄗㄘㄙ]?ㄨㄣ|"
+    "[ㄉㄊㄋㄌㄍㄎㄏㄓㄔㄖㄗㄘㄙ]?ㄨㄥ|"
+    "[ㄅㄆㄇㄈㄉㄊㄋㄌㄍㄎㄏㄓㄔㄕㄖㄗㄘㄙ]?ㄨ|"
+    "[ㄓㄔㄕㄖㄗㄘㄙㄝㄦㄧ]"
+    ")[{marks}]?"
+).format(marks=marks)

metadata

@@ -0,0 +1,198 @@
+--- !ruby/object:Gem::Specification
+name: jtcg_locale_detector
+version: !ruby/object:Gem::Version
+  version: 1.0.1
+platform: ruby
+authors:
+- Original mark
+- JTCG Team
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2025-06-01 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: ffi
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.15'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.15'
+- !ruby/object:Gem::Dependency
+  name: json
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.6'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.6'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.17'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.17'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '13.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '13.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.12'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.12'
+- !ruby/object:Gem::Dependency
+  name: rubocop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.50'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.50'
+description: Multi-language locale detector with specialized Chinese variant detection.
+  Uses FastText for initial language identification and multiple algorithms for Chinese
+  variant detection. Ruby gem packaged by JTCG Team.
+email:
+- mark@j-tcg.com
+- enor@j-tcg.com
+executables:
+- locale-detector
+extensions: []
+extra_rdoc_files: []
+files:
+- CHANGELOG.md
+- PACKAGING_SUMMARY.md
+- README.md
+- bin/locale-detector
+- jtcg_locale_detector.gemspec
+- lib/locale_detector.rb
+- lib/locale_detector/client.rb
+- lib/locale_detector/detector.rb
+- lib/locale_detector/version.rb
+- locale_detector.gemspec
+- python/cli.py
+- python/requirements.txt
+- python/src/__init__.py
+- python/src/__pycache__/__init__.cpython-311.pyc
+- python/src/__pycache__/__init__.cpython-313.pyc
+- python/src/__pycache__/locale_data.cpython-311.pyc
+- python/src/__pycache__/locale_data.cpython-313.pyc
+- python/src/__pycache__/locale_detector.cpython-311.pyc
+- python/src/__pycache__/locale_detector.cpython-313.pyc
+- python/src/artifacts/fasttext/lid.176.bin
+- python/src/artifacts/fasttext/lid.176.ftz
+- python/src/download_fasttext.py
+- python/src/locale_data.py
+- python/src/locale_detector.py
+- python/src/locale_detector_c.c
+- python/src/locale_detector_c.h
+- python/src/locale_detector_cy.cpp
+- python/src/locale_detector_cy.cpython-311-darwin.so
+- python/src/locale_detector_cy.cpython-313-darwin.so
+- python/src/locale_detector_cy.html
+- python/src/locale_detector_cy.pyx
+- python/src/utils/__init__.py
+- python/src/utils/__pycache__/__init__.cpython-311.pyc
+- python/src/utils/__pycache__/__init__.cpython-313.pyc
+- python/src/utils/__pycache__/data_utils.cpython-311.pyc
+- python/src/utils/__pycache__/data_utils.cpython-313.pyc
+- python/src/utils/data_utils.py
+- python/src/utils/data_utils_cy.cpp
+- python/src/utils/data_utils_cy.cpython-311-darwin.so
+- python/src/utils/data_utils_cy.cpython-313-darwin.so
+- python/src/utils/data_utils_cy.html
+- python/src/utils/data_utils_cy.pyx
+- python/src/zhon/__init__.py
+- python/src/zhon/__pycache__/__init__.cpython-311.pyc
+- python/src/zhon/__pycache__/hanzi.cpython-311.pyc
+- python/src/zhon/__pycache__/pinyin.cpython-311.pyc
+- python/src/zhon/__pycache__/zhuyin.cpython-311.pyc
+- python/src/zhon/cedict/__init__.py
+- python/src/zhon/cedict/__pycache__/__init__.cpython-311.pyc
+- python/src/zhon/cedict/__pycache__/all.cpython-311.pyc
+- python/src/zhon/cedict/__pycache__/simplified.cpython-311.pyc
+- python/src/zhon/cedict/__pycache__/traditional.cpython-311.pyc
+- python/src/zhon/cedict/all.py
+- python/src/zhon/cedict/simplified.py
+- python/src/zhon/cedict/traditional.py
+- python/src/zhon/hanzi.py
+- python/src/zhon/pinyin.py
+- python/src/zhon/zhuyin.py
+homepage: https://github.com/jtcg/locale-detector
+licenses:
+- MIT
+metadata:
+  homepage_uri: https://github.com/jtcg/locale-detector
+  source_code_uri: https://github.com/jtcg/locale-detector
+  changelog_uri: https://github.com/jtcg/locale-detector/blob/main/CHANGELOG.md
+  packaged_by: JTCG Team <enor@j-tcg.com>
+  original_author: Original mark <mark@j-tcg.com>
+  maintainer: JTCG Team
+  funding_uri: https://github.com/sponsors/jtcg
+  rubygems_mfa_required: 'true'
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 2.6.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.0.3.1
+signing_key: 
+specification_version: 4
+summary: 多語言地區檢測工具，特別優化中文繁簡體檢測
+test_files: []