jtcg_locale_detector 1.0.1

data/lib/locale_detector/client.rb

@@ -0,0 +1,163 @@
+require "json"
+require "open3"
+require "tempfile"
+
+module LocaleDetector
+  class Client
+    attr_reader :options
+
+    def initialize(**options)
+      @options = default_options.merge(options)
+      validate_python_environment
+    end
+
+    # Detect locale for a single text
+    def detect(text)
+      return nil if text.nil? || text.strip.empty?
+
+      result = execute_detection(text: text)
+      parse_result(result)
+    end
+
+    # Detect locales for multiple texts
+    def detect_batch(texts)
+      return [] if texts.nil? || texts.empty?
+
+      texts.map { |text| detect(text) }
+    end
+
+    # Detect locale from file
+    def detect_file(file_path)
+      raise Error, "File not found: #{file_path}" unless File.exist?(file_path)
+
+      result = execute_detection(file: file_path)
+      parse_result(result)
+    end
+
+    # Get detailed detection information
+    def detect_with_details(text)
+      return nil if text.nil? || text.strip.empty?
+
+      result = execute_detection(text: text, details: true)
+      parse_detailed_result(result)
+    end
+
+    private
+
+    def default_options
+      {
+        mode: "ratio",           # conversion, ratio, both
+        low_memory: false,       # use low memory mode
+        output_format: "json",   # simple, json, detailed
+        encoding: "utf-8",       # file encoding
+        python_path: find_python_executable,
+        cli_path: find_cli_path
+      }
+    end
+
+    def find_python_executable
+      # Try to find Python executable
+      %w[python3 python].each do |cmd|
+        _output, status = Open3.capture2e("which #{cmd}")
+        return cmd if status.success?
+      end
+
+      raise Error, "Python executable not found. Please install Python 3.11+."
+    end
+
+    def find_cli_path
+      # Look for the CLI script in the gem directory
+      gem_root = File.expand_path("../..", __dir__)
+      cli_candidates = [
+        File.join(gem_root, "python", "cli.py"),
+        File.join(File.dirname(gem_root), "cli.py"),
+        File.join(gem_root, "..", "cli.py") # Parent directory
+      ]
+
+      cli_candidates.each do |path|
+        return File.expand_path(path) if File.exist?(path)
+      end
+
+      raise Error, "CLI script not found. Please ensure the Python components are installed."
+    end
+
+    def validate_python_environment
+      # Check if Python is available
+      output, status = Open3.capture2e("#{@options[:python_path]} --version")
+      raise Error, "Python is not available: #{output}" unless status.success?
+
+      # Check Python version (should be 3.11+)
+      version_match = output.match(/Python (\d+\.\d+)/)
+      return unless version_match
+
+      version = version_match[1].to_f
+      return unless version < 3.11
+
+      raise Error, "Python version must be 3.11 or higher. Found: #{version}"
+    end
+
+    def execute_detection(text: nil, file: nil, details: false)
+      cmd_args = build_command_args(text: text, file: file, details: details)
+
+      output, error, status = Open3.capture3(*cmd_args)
+
+      raise Error, "Detection failed: #{error}" unless status.success?
+
+      output.strip
+    end
+
+    def build_command_args(text: nil, file: nil, details: false)
+      args = [@options[:python_path], @options[:cli_path]]
+
+      if text
+        args << text
+      elsif file
+        args += ["--file", file]
+      end
+
+      args += ["--mode", @options[:mode]]
+      args += ["--output-format", @options[:output_format]]
+      args += ["--encoding", @options[:encoding]]
+      args << "--low-memory" if @options[:low_memory]
+      args << "--details" if details
+
+      args
+    end
+
+    def parse_result(output)
+      case @options[:output_format]
+      when "json"
+        begin
+          JSON.parse(output)
+        rescue JSON::ParserError => e
+          raise Error, "Failed to parse JSON response: #{e.message}"
+        end
+      when "detailed"
+        parse_detailed_output(output)
+      else # "simple" or any other format
+        { "locale" => output.strip }
+      end
+    end
+
+    def parse_detailed_result(output)
+      JSON.parse(output)
+    rescue JSON::ParserError => e
+      raise Error, "Failed to parse detailed JSON response: #{e.message}"
+    end
+
+    def parse_detailed_output(output)
+      # Parse detailed text output format
+      lines = output.split("\n")
+      result = {}
+
+      lines.each do |line|
+        if line.include?(":")
+          key, value = line.split(":", 2)
+          result[key.strip.downcase.gsub(" ", "_")] = value.strip
+        end
+      end
+
+      result
+    end
+  end
+end

data/lib/locale_detector/detector.rb

@@ -0,0 +1,46 @@
+module LocaleDetector
+  class Detector
+    SUPPORTED_LANGUAGES = %w[
+      af am an ar arz as ast av az azb ba bar bcl be bg bh bn bo bpy br bs bxr
+      ca cbk ce ceb ckb co cs cv cy da de diq dsb dv dz ee el eml en eo es et
+      eu ext fa fi fr frr fy ga gd gl gn gom gu gv he hi hif hr hsb ht hu hy
+      ia id ie ilo io is it ja jbo jv ka kk km kn ko krc ku kv kw ky la lb lez
+      li lmo lo lrc lt lv mai mg mhr min mk ml mn mr mrj ms mt mwl my myv mzn
+      nah nap nds ne new nl nn no oc or os pa pam pfl pl pms pnb ps pt qu rm ro
+      ru rue sa sah sc scn sco sd sh si sk sl so sq sr su sv sw szl ta te tg th
+      tk tl tr tt tyv ug uk ur uz vec vep vi vls vo wa war wuu xal xmf yi yo
+      yue zh zu
+    ].freeze
+
+    CHINESE_VARIANTS = {
+      "zh-TW" => "Traditional Chinese (Taiwan)",
+      "zh-CN" => "Simplified Chinese (China)",
+      "zh-HK" => "Traditional Chinese (Hong Kong)",
+      "zh-MO" => "Traditional Chinese (Macau)",
+      "zh-SG" => "Simplified Chinese (Singapore)"
+    }.freeze
+
+    DETECTION_MODES = %w[conversion ratio both].freeze
+
+    def self.supported_languages
+      SUPPORTED_LANGUAGES
+    end
+
+    def self.chinese_variants
+      CHINESE_VARIANTS
+    end
+
+    def self.detection_modes
+      DETECTION_MODES
+    end
+
+    def self.language_supported?(language_code)
+      base_language = language_code.split("-").first
+      SUPPORTED_LANGUAGES.include?(base_language)
+    end
+
+    def self.chinese_variant?(locale)
+      CHINESE_VARIANTS.key?(locale)
+    end
+  end
+end

data/lib/locale_detector/version.rb

@@ -0,0 +1,3 @@
+module LocaleDetector
+  VERSION = "1.0.1".freeze
+end

data/lib/locale_detector.rb

@@ -0,0 +1,25 @@
+require_relative "locale_detector/version"
+require_relative "locale_detector/client"
+require_relative "locale_detector/detector"
+
+module LocaleDetector
+  class Error < StandardError; end
+
+  # Convenience method for quick detection
+  def self.detect(text, **options)
+    client = Client.new(**options)
+    client.detect(text)
+  end
+
+  # Batch detection
+  def self.detect_batch(texts, **options)
+    client = Client.new(**options)
+    client.detect_batch(texts)
+  end
+
+  # Detect from file
+  def self.detect_file(file_path, **options)
+    client = Client.new(**options)
+    client.detect_file(file_path)
+  end
+end

data/locale_detector.gemspec

@@ -0,0 +1,46 @@
+Gem::Specification.new do |spec|
+  spec.name          = "jtcg_locale_detector"
+  
+  spec.summary       = "多語言地區檢測工具，特別優化中文繁簡體檢測"
+  spec.description   = "Multi-language locale detector with specialized Chinese variant detection. " \
+                       "Uses FastText for initial language identification and multiple algorithms " \
+                       "for Chinese variant detection. Ruby gem packaged by JTCG Team."
+  spec.version       = "1.0.1"
+  spec.authors       = ["JTCG Team"]
+  spec.email         = ["enor@j-tcg.com"]
+
+  spec.summary       = "多語言地區檢測工具，特別優化中文繁簡體檢測"
+  spec.description   = "Multi-language locale detector with specialized Chinese variant detection. " \
+                       "Uses FastText for initial language identification and multiple algorithms " \
+                       "for Chinese variant detection."
+  spec.homepage      = "https://github.com/jtcg/locale-detector"
+  spec.license       = "MIT"
+  spec.required_ruby_version = ">= 2.6.0"
+
+  # 作者角色說明
+  spec.metadata["packaged_by"] = "JTCG Team <enor@j-tcg.com>"
+  spec.metadata["original_author"] = "Original Author <mark@j-tcg.com>"
+  spec.metadata["maintainer"] = "JTCG Team"
+
+  # Specify which files should be added to the gem when it is released.
+  spec.files = Dir.chdir(File.expand_path(__dir__)) do
+    Dir["{bin,lib,python}/**/*", "*.md", "*.txt", "*.gemspec"]
+  end
+
+  spec.bindir        = "bin"
+  spec.executables   = ["locale-detector"]
+  spec.require_paths = ["lib"]
+
+  # Dependencies
+  spec.add_dependency "ffi", "~> 1.15"
+  spec.add_dependency "json", "~> 2.6"
+
+  # Development dependencies
+  spec.add_development_dependency "bundler", "~> 1.17"
+  spec.add_development_dependency "rake", "~> 13.0"
+  spec.add_development_dependency "rspec", "~> 3.12"
+  spec.add_development_dependency "rubocop", "~> 1.50"
+
+  # Metadata
+  spec.metadata["rubygems_mfa_required"] = "true"
+end

data/python/cli.py

@@ -0,0 +1,220 @@
+#!/usr/bin/env python3
+"""
+LocaleDetector CLI - 多語言地區檢測命令列工具
+
+支援檢測多種語言，特別針對中文繁簡體變體檢測進行優化。
+使用 FastText 進行初始語言識別，再對中文進行特化檢測。
+"""
+
+import argparse
+import asyncio
+import json
+import os
+import sys
+
+# 若在 PyInstaller 執行環境，自動將 _internal/src 加入 sys.path
+if hasattr(sys, "_MEIPASS"):
+    cython_mod_path = os.path.join(sys._MEIPASS, "src")
+    if cython_mod_path not in sys.path:
+        sys.path.insert(0, cython_mod_path)
+from pathlib import Path
+from typing import Literal
+
+try:
+    from src.locale_detector_cy import LocaleDetector
+except ImportError:
+    from src.locale_detector import LocaleDetector
+
+
+def create_parser() -> argparse.ArgumentParser:
+    """創建命令列參數解析器"""
+    parser = argparse.ArgumentParser(
+        description="多語言地區檢測工具 - 特別優化中文繁簡體檢測",
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+        epilog="""
+使用範例:
+  %(prog)s "Hello world"                    # 檢測單一文本
+  %(prog)s -f input.txt                     # 從檔案讀取
+  %(prog)s "你好世界" --details              # 顯示詳細信息
+  %(prog)s "中文文本" --mode ratio           # 使用比例分析模式
+  %(prog)s --batch file1.txt file2.txt     # 批次處理多個檔案
+        """,
+    )
+
+    # 文本輸入選項
+    input_group = parser.add_mutually_exclusive_group(required=True)
+    input_group.add_argument("text", nargs="?", help="要檢測的文本內容")
+    input_group.add_argument("-f", "--file", type=Path, help="從檔案讀取文本進行檢測")
+    input_group.add_argument("--batch", nargs="+", type=Path, help="批次處理多個檔案")
+
+    # 檢測模式選項
+    parser.add_argument(
+        "--mode",
+        choices=["conversion", "ratio", "both"],
+        default="ratio",
+        help="中文檢測模式: conversion(快速), ratio(準確), both(混合) [預設: ratio]",
+    )
+
+    # 記憶體選項
+    parser.add_argument("--low-memory", action="store_true", help="使用低記憶體模式 (使用較小的 FastText 模型)")
+
+    # 輸出格式選項
+    parser.add_argument(
+        "--output-format",
+        choices=["simple", "json", "detailed"],
+        default="simple",
+        help="輸出格式: simple(簡單), json(JSON格式), detailed(詳細信息) [預設: simple]",
+    )
+
+    # 詳細信息選項
+    parser.add_argument("--details", action="store_true", help="顯示檢測詳細信息（包含信心分數）")
+
+    # 輸出檔案選項
+    parser.add_argument("-o", "--output", type=Path, help="將結果輸出到檔案")
+
+    # 編碼選項
+    parser.add_argument("--encoding", default="utf-8", help="檔案編碼格式 [預設: utf-8]")
+
+    return parser
+
+
+def read_text_file(file_path: Path, encoding: str = "utf-8") -> str:
+    """讀取文本檔案"""
+    try:
+        return file_path.read_text(encoding=encoding).strip()
+    except UnicodeDecodeError:
+        # 嘗試其他編碼
+        for fallback_encoding in ["gbk", "big5", "latin1"]:
+            try:
+                return file_path.read_text(encoding=fallback_encoding).strip()
+            except UnicodeDecodeError:
+                continue
+        raise ValueError(f"無法解碼檔案 {file_path}，請指定正確的編碼格式")
+
+
+def format_output(result: dict | str, output_format: str, show_details: bool) -> str:
+    """格式化輸出結果"""
+    if isinstance(result, str):
+        # 簡單字串結果
+        if output_format == "json":
+            return json.dumps({"locale": result}, ensure_ascii=False, indent=2)
+        return result
+
+    # 詳細結果字典
+    if output_format == "json":
+        return json.dumps(result, ensure_ascii=False, indent=2)
+    if output_format == "detailed" or show_details:
+        locale = result.get("locale", "unknown")
+        language = result.get("language", "unknown")
+        score = result.get("score", 0.0)
+        return f"地區: {locale} | 語言: {language} | 信心分數: {score:.3f}"
+    return result.get("locale", "unknown")
+
+
+def write_output(content: str, output_file: Path | None, encoding: str = "utf-8") -> None:
+    """寫入輸出"""
+    if output_file:
+        output_file.write_text(content, encoding=encoding)
+        print(f"結果已儲存至: {output_file}")
+    else:
+        print(content)
+
+
+async def process_single_text(
+    detector: LocaleDetector, text: str, mode: Literal["conversion", "ratio", "both"], show_details: bool
+) -> dict | str:
+    """處理單一文本"""
+    if show_details:
+        return await detector.adetect_with_details(text, mode=mode)
+    return await detector.adetect(text)
+
+
+async def process_batch(
+    detector: LocaleDetector, file_paths: list[Path], mode: Literal["conversion", "ratio", "both"], encoding: str, show_details: bool
+) -> list[dict]:
+    """批次處理多個檔案"""
+    results = []
+
+    for file_path in file_paths:
+        try:
+            text = read_text_file(file_path, encoding)
+            if show_details:
+                result = await detector.adetect_with_details(text, mode=mode)
+                result["file"] = str(file_path)
+            else:
+                locale = await detector.adetect(text)
+                result = {"file": str(file_path), "locale": locale}
+            results.append(result)
+        except Exception as e:
+            error_result = {"file": str(file_path), "error": str(e), "locale": "error"}
+            results.append(error_result)
+
+    return results
+
+
+async def main() -> None:
+    """主函數"""
+    parser = create_parser()
+    args = parser.parse_args()
+
+    try:
+        # 初始化檢測器
+        print("正在初始化 LocaleDetector...", file=sys.stderr)
+        detector = LocaleDetector(low_memory=args.low_memory)
+
+        # 處理輸入
+        if args.batch:
+            # 批次處理
+            print(f"正在處理 {len(args.batch)} 個檔案...", file=sys.stderr)
+            results = await process_batch(detector, args.batch, args.mode, args.encoding, args.details)
+
+            if args.output_format == "json":
+                output = json.dumps(results, ensure_ascii=False, indent=2)
+            else:
+                output_lines = []
+                for result in results:
+                    if "error" in result:
+                        line = f"{result['file']}: ERROR - {result['error']}"
+                    else:
+                        formatted = format_output(result, args.output_format, args.details)
+                        if args.details or args.output_format == "detailed":
+                            line = f"{result['file']}: {formatted}"
+                        else:
+                            line = f"{result['file']}: {result['locale']}"
+                    output_lines.append(line)
+                output = "\n".join(output_lines)
+
+        else:
+            # 單一文本處理
+            text = read_text_file(args.file, args.encoding) if args.file else args.text
+
+            if not text.strip():
+                print("錯誤: 輸入文本為空", file=sys.stderr)
+                sys.exit(1)
+
+            print("正在檢測語言...", file=sys.stderr)
+            result = await process_single_text(detector, text, args.mode, args.details)
+            output = format_output(result, args.output_format, args.details)
+
+        # 輸出結果
+        write_output(output, args.output, args.encoding)
+
+    except KeyboardInterrupt:
+        print("\n程式被使用者中斷", file=sys.stderr)
+        sys.exit(1)
+    except Exception as e:
+        print(f"錯誤: {e}", file=sys.stderr)
+        sys.exit(1)
+
+
+def cli_entry_point() -> None:
+    """CLI 入口點，用於打包後的執行檔"""
+    try:
+        asyncio.run(main())
+    except RuntimeError:
+        loop = asyncio.get_event_loop()
+        loop.run_until_complete(main())
+
+
+if __name__ == "__main__":
+    cli_entry_point()

data/python/requirements.txt

@@ -0,0 +1,8 @@
+fasttext-wheel>=0.9.2
+opencc-python-reimplemented>=0.1.7
+zhon>=2.0.2
+requests>=2.31.0
+tqdm>=4.66.0
+numpy<2.0.0
+nest-asyncio>=1.5.6
+psutil>=5.9.0

data/python/src/__init__.py

@@ -0,0 +1,10 @@
+"""
+Locale Detector Package
+
+Multi-language locale detector with specialized Chinese variant detection.
+"""
+
+from .locale_detector import LocaleDetector
+
+__version__ = "1.0.0"
+__all__ = ["LocaleDetector"]

data/python/src/download_fasttext.py

@@ -0,0 +1,69 @@
+import sys
+from pathlib import Path
+
+import requests
+from tqdm import tqdm
+
+
+def download_file(url: str, target_path: Path, desc: str):
+    """
+    Download a file with progress bar
+    """
+    response = requests.get(url, stream=True)
+    total_size = int(response.headers.get("content-length", 0))
+
+    # Ensure the directory exists
+    target_path.parent.mkdir(parents=True, exist_ok=True)
+
+    # Show download progress
+    with (
+        open(target_path, "wb") as file,
+        tqdm(
+            desc=desc,
+            total=total_size,
+            unit="iB",
+            unit_scale=True,
+            unit_divisor=1024,
+        ) as pbar,
+    ):
+        for data in response.iter_content(chunk_size=1024):
+            size = file.write(data)
+            pbar.update(size)
+
+
+def main():
+    # Define the target directory
+    current_dir = Path(__file__).resolve().parents[2]
+    target_dir = current_dir / "tools" / "locales" / "artifacts" / "fasttext"
+
+    # Model URLs and filenames
+    models = {
+        "lid.176.ftz": "https://dl.fbaipublicfiles.com/fasttext/supervised-models/lid.176.ftz",
+        "lid.176.bin": "https://dl.fbaipublicfiles.com/fasttext/supervised-models/lid.176.bin",
+    }
+
+    print(f"Downloading FastText models to: {target_dir}")
+
+    try:
+        # Download each model
+        for filename, url in models.items():
+            target_path = target_dir / filename
+
+            if target_path.exists():
+                print(f"Model {filename} already exists, skipping...")
+                continue
+
+            print(f"Downloading {filename}...")
+            download_file(url, target_path, f"Downloading {filename}")
+            print(f"Successfully downloaded {filename}")
+
+        print("\nAll models downloaded successfully!")
+        print(f"Models are located in: {target_dir}")
+
+    except Exception as e:
+        print(f"Error downloading models: {e!s}", file=sys.stderr)
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()