unihan_lang 0.2.0 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 81d1394f3bee01c607c5440f3682b6401c9aa1d1ff9ba5ad286c196f8eebc54b
-  data.tar.gz: 8be0a8218adbe226b8f2079a893200912e8cbcba6b2c00aa405b0b886dd01f50
+  metadata.gz: 07200fff605b22cae82d18c8b9a16041fb1fbcd53c9106a25582785a779631ad
+  data.tar.gz: 335f78c7b507b547bbad354856bb294655f1042943ed66d3899a607721c40efb
 SHA512:
-  metadata.gz: 01ae746510cad08ab38db9f21752049e7fd99141fdc6886f6a93a2c521feb0554b47fafa8d7d3bfa4fd6aee560bbd1dfa3a08ad0ac477e399583971fa7258ebe
-  data.tar.gz: 302c234bc6616021682ff2b9f86df610f24bc9bd7894e6107369d44a5e2e4164738130ec31d785f45485b5e508fc03934cd50bb5bea52d9423299ca5679e182c
+  metadata.gz: 7691fa9c50fc79921fa069e36a367cb37adc20a2be5fecdd4d768d037389b3fa434fa1078ab4bf05afc026f5a91d1b8215696f0eff0f2feb9987e10264a4bc32
+  data.tar.gz: acff2b9f98d296486e0a83f2b8909db4351042e7ec837c088d0ddf8377b99425ebda8eea4f99709861390fe141162f95717768196c7a022eb29d13030d9708f8

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    unihan_lang (0.1.0)
+    unihan_lang (0.3.0)
 
 GEM
   remote: https://rubygems.org/

data/HOW_TO_UPDATE.md

@@ -0,0 +1,32 @@
+# Setup
+
+```bash
+mkdir ~/.gem
+RUBY_GEMS_ACCOUNT=TODO
+curl -u $RUBY_GEMS_ACCOUNT https://rubygems.org/api/v1/api_key.yaml > ~/.gem/credentials; chmod 0600 ~/.gem/credentials
+```
+
+# Publish
+
+```bash
+# Add release branch
+VERSION=TODO
+git checkout master
+git checkout -b release/$VERSION
+vi lib/unihan_lang/version.rb # edit version
+git commit -am "Bump version $VERSION"
+
+# Publish to rubygems
+gem build unihan_lang.gemspec
+bundle install
+gem push unihan_lang-$VERSION.gem
+
+# GitHub release
+git tag $VERSION
+git push --tags
+open https://github.com/kyubey1228/unihan_lang/releases/new
+
+# merge to master
+git checkout master
+git merge release/$VERSION
+```

data/lib/unihan_lang/chinese_score_analyzer.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+module UnihanLang
+  class ChineseScoreAnalyzer
+    attr_reader :traditional_score, :simplified_score, :total_chinese
+
+    def initialize(text, chinese_processor, variant_mapping)
+      @text = text
+      @chinese_processor = chinese_processor
+      @variant_mapping = variant_mapping
+      @traditional_score = 0
+      @simplified_score = 0
+      analyze
+    end
+
+    def dominant_language
+      return "Unknown" if total_chinese.zero?
+      return "ZH_TW" if traditional_score > simplified_score
+      return "ZH_CN" if simplified_score > traditional_score
+
+      "Unknown"
+    end
+
+    def language_ratio
+      return :unknown if total_chinese != @text.length
+      return :tw if traditional_score > simplified_score
+      return :cn if simplified_score >= traditional_score
+
+      :unknown
+    end
+
+    private
+
+    def analyze
+      @total_chinese = 0
+      @text.chars.each do |char|
+        next unless @chinese_processor.chinese_character?(char)
+
+        @total_chinese += 1
+
+        calculate_character_scores(char)
+      end
+    end
+
+    def calculate_character_scores(char)
+      if @chinese_processor.only_zh_tw?(char)
+        @traditional_score += 2
+      elsif @chinese_processor.only_zh_cn?(char)
+        @simplified_score += 2
+      end
+
+      @traditional_score += 0.5 if @variant_mapping.traditional_variants(char).any?
+      @simplified_score += 0.5 if @variant_mapping.simplified_variants(char).any?
+    end
+  end
+end

data/lib/unihan_lang/variant_mapping.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+module UnihanLang
+  class VariantMapping
+    def initialize
+      @traditional_to_simplified = load_variant_mappings
+      @simplified_to_traditional = {}
+      # 簡体字から繁体字へのマッピングを構築
+      @traditional_to_simplified.each do |trad, simps|
+        simps.each do |simp|
+          @simplified_to_traditional[simp] ||= Set.new
+          @simplified_to_traditional[simp] << trad
+        end
+      end
+    end
+
+    def traditional_variants(char)
+      @simplified_to_traditional[char] || Set.new
+    end
+
+    def simplified_variants(char)
+      @traditional_to_simplified[char] || Set.new
+    end
+
+    private
+
+    def load_variant_mappings
+      traditional_to_simplified = {}
+      file_path = File.join(File.dirname(__FILE__), "..", "..", "data", "Unihan_Variants.txt")
+
+      File.foreach(file_path, encoding: "UTF-8") do |line|
+        next if line.start_with?("#") || line.strip.empty?
+
+        fields = line.strip.split("\t")
+        # kTraditionalVariant フィールドの場合のみ処理
+        if fields.size >= 3 && fields[1] == ("kTraditionalVariant")
+          simp = [fields[0].gsub(/^U\+/, "").hex].pack("U")
+          trad = [fields[2].gsub(/^U\+/, "").hex].pack("U")
+          traditional_to_simplified[trad] ||= Set.new
+          traditional_to_simplified[trad] << simp
+        end
+      end
+      traditional_to_simplified
+    end
+  end
+end

data/lib/unihan_lang/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module UnihanLang
-  VERSION = "0.2.0"
+  VERSION = "0.3.0"
 end

data/lib/unihan_lang.rb

@@ -2,11 +2,14 @@
 
 require_relative "unihan_lang/version"
 require_relative "unihan_lang/chinese_processor"
+require_relative "unihan_lang/variant_mapping"
+require_relative "unihan_lang/chinese_score_analyzer"
 
 module UnihanLang
   class Unihan
     def initialize
       @chinese_processor = ChineseProcessor.new
+      @variant_mapping = VariantMapping.new
     end
 
     def zh_tw?(text)
@@ -49,19 +52,25 @@ module UnihanLang
       end
     end
 
-    private
+    def analyze_with_variants(text)
+      analyzer = ChineseScoreAnalyzer.new(text, @chinese_processor, @variant_mapping)
+      {
+        traditional_score: analyzer.traditional_score,
+        simplified_score: analyzer.simplified_score,
+        total_chinese: analyzer.total_chinese,
+      }
+    end
 
-    # テキストの言語比率を計算し、最も可能性の高い言語を返す
-    def language_ratio(text)
-      only_tw_chars = text.chars.count { |char| @chinese_processor.only_zh_tw?(char) }
-      only_cn_chars = text.chars.count { |char| @chinese_processor.only_zh_cn?(char) }
-      chinese_chars = text.chars.count { |char| @chinese_processor.chinese?(char) }
+    def determine_language_with_variants(text)
+      analyzer = ChineseScoreAnalyzer.new(text, @chinese_processor, @variant_mapping)
+      analyzer.dominant_language
+    end
 
-      return :unknown unless chinese_chars == text.length
-      return :tw if only_tw_chars > only_cn_chars
-      return :cn if only_cn_chars >= only_tw_chars
+    private
 
-      :unknown
+    def language_ratio(text)
+      analyzer = ChineseScoreAnalyzer.new(text, @chinese_processor, @variant_mapping)
+      analyzer.language_ratio
     end
   end
 end

metadata

@@ -1,14 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: unihan_lang
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.3.0
 platform: ruby
 authors:
 - kyubey1228
-autorequire:
 bindir: exe
 cert_chain: []
-date: 2024-10-17 00:00:00.000000000 Z
+date: 2025-02-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -66,6 +65,7 @@ files:
 - ".rubocop.yml"
 - Gemfile
 - Gemfile.lock
+- HOW_TO_UPDATE.md
 - LICENSE.md
 - README.ja.md
 - README.md
@@ -73,13 +73,14 @@ files:
 - data/Unihan_Variants.txt
 - lib/unihan_lang.rb
 - lib/unihan_lang/chinese_processor.rb
+- lib/unihan_lang/chinese_score_analyzer.rb
+- lib/unihan_lang/variant_mapping.rb
 - lib/unihan_lang/version.rb
 - unihan_lang.gemspec
 homepage: https://github.com/kyubey1228/unihan_lang
 licenses:
 - MIT
 metadata: {}
-post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -94,8 +95,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.5.3
-signing_key:
+rubygems_version: 3.6.2
 specification_version: 4
 summary: Language detection for Chinese characters
 test_files: []