unihan_lang 0.2.0 → 0.2.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 81d1394f3bee01c607c5440f3682b6401c9aa1d1ff9ba5ad286c196f8eebc54b
-  data.tar.gz: 8be0a8218adbe226b8f2079a893200912e8cbcba6b2c00aa405b0b886dd01f50
+  metadata.gz: 838b5d8575bc79d05a24ed6555b9fe8d7856779b1f06e11ddbe3ae08e7030f11
+  data.tar.gz: b0e38c1bacc89a87f227ede35cef8b1985688b781bc9ebaf5a4f37ea29b378d7
 SHA512:
-  metadata.gz: 01ae746510cad08ab38db9f21752049e7fd99141fdc6886f6a93a2c521feb0554b47fafa8d7d3bfa4fd6aee560bbd1dfa3a08ad0ac477e399583971fa7258ebe
-  data.tar.gz: 302c234bc6616021682ff2b9f86df610f24bc9bd7894e6107369d44a5e2e4164738130ec31d785f45485b5e508fc03934cd50bb5bea52d9423299ca5679e182c
+  metadata.gz: '0482b7206764fc812703fefcb37badb8a59cdd2a88fb04741ae4e25c7380c83f08e293992a6ec3158ff8d296c2c790d111b16e16fa48bb825a1b4091e1da2c1f'
+  data.tar.gz: 7613e83a5579b1a45c9fdb4ef8d8df549f7f5dd95a5d7110dd5ca36c471cfcbcc410debfb9d15ca59927656699f756877051e9f49649ec5e01d98fcb5631f6b5

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    unihan_lang (0.1.0)
+    unihan_lang (0.2.1)
 
 GEM
   remote: https://rubygems.org/

data/HOW_TO_UPDATE.md

@@ -0,0 +1,13 @@
+# Setup
+
+```bash
+mkdir ~/.gem
+RUBY_GEMS_ACCOUNT=TODO
+curl -u $RUBY_GEMS_ACCOUNT https://rubygems.org/api/v1/api_key.yaml > ~/.gem/credentials; chmod 0600 ~/.gem/credentials
+```
+
+# Publish
+
+```bash
+./bump_new_version.sh 1.2.3
+```

data/README.ja.md

@@ -83,3 +83,4 @@ puts unihan.contains_zh_cn?("這個text不包含簡體字") # => false
 
 このライブラリは、テキストの言語を完全に正確に判定することを保証するものではありません。
 特に、短いテキストや複数の言語が混在するテキストの場合、判定が難しい場合があります。
+判定の際、句読点・数字・空白は無視されます。また、ひらがな・カタカナを含むテキストは中国語ではなく日本語として扱われます。

data/README.md

@@ -71,7 +71,7 @@ puts unihan.contains_zh_cn?("這個text不包含簡體字") # => false
 
 ## Features
 
-- `determine_language(text)`: Determines the language of the text ("ZH_TW", "ZH_CN", "JA", "Unknown").
+- `determine_language(text)`: Determines the language of the text ("zh_TW", "zh_CN", "JA", "Unknown").
 - `zh_tw?(text)`: Checks if the text is in Traditional Chinese.
 - `zh_cn?(text)`: Checks if the text is in Simplified Chinese.
 - `contains_chinese?(text)`: Checks if the text contains Chinese characters.
@@ -86,3 +86,4 @@ puts unihan.contains_zh_cn?("這個text不包含簡體字") # => false
 This library does not guarantee 100% accuracy in language identification.
 Particularly for short texts or texts containing multiple languages, determination may be challenging.
 The distinction between Traditional and Simplified Chinese is based on the Unihan database.
+Punctuation, digits, and whitespace are ignored when judging a sentence, and text containing Hiragana/Katakana is treated as Japanese rather than Chinese.

data/bump_new_version.sh

@@ -0,0 +1,38 @@
+#!/bin/bash
+
+if [ -z "$1" ]; then
+  echo "Usage: $0 <version>"
+  exit 1
+fi
+
+# ~/.gem/credentials
+if [ ! -f ~/.gem/credentials ]; then
+  echo "Error: ~/.gem/credentials not found. Please set up your RubyGems credentials."
+  exit 1
+fi
+
+PROJECT_NAME="unihan_lang"
+GITHUB_REPO="kyubey1228/unihan_lang"
+VERSION=$1
+echo "Start bumping version: $VERSION"
+
+# Publish
+# Add release branch
+git checkout master
+git checkout -b release/$VERSION
+sed -i '' "s/VERSION = \".*\"/VERSION = \"$VERSION\"/" lib/$PROJECT_NAME/version.rb
+git commit -am "Bump version $VERSION"
+
+# Publish to rubygems
+gem build $PROJECT_NAME.gemspec
+bundle install
+gem push $PROJECT_NAME-$VERSION.gem
+
+# GitHub release
+git tag $VERSION
+git push --tags
+open https://github.com/$GITHUB_REPO/releases/new
+
+# merge to master
+git checkout master
+git merge release/$VERSION

data/lib/unihan_lang/chinese_score_analyzer.rb

@@ -0,0 +1,77 @@
+# frozen_string_literal: true
+
+module UnihanLang
+  class ChineseScoreAnalyzer
+    attr_reader :traditional_score, :simplified_score, :total_chinese
+
+    def initialize(text, chinese_processor, variant_mapping)
+      @text = text
+      @chinese_processor = chinese_processor
+      @variant_mapping = variant_mapping
+      @traditional_score = 0
+      @simplified_score = 0
+      @contains_kana = false
+      analyze
+    end
+
+    def dominant_language
+      return "Unknown" if contains_kana?
+      return "Unknown" if total_chinese.zero?
+      return "ZH_TW" if traditional_score > simplified_score
+      return "ZH_CN" if simplified_score > traditional_score
+
+      "Unknown"
+    end
+
+    def language_ratio
+      return :unknown if contains_kana?
+      return :unknown if total_chinese.zero? || total_chinese != meaningful_length
+      return :tw if traditional_score > simplified_score
+      return :cn if simplified_score > traditional_score
+
+      :unknown
+    end
+
+    private
+
+    def contains_kana?
+      @contains_kana
+    end
+
+    def kana_char?(char)
+      ord = char.ord
+      # 0x3040-0x309F: ひらがな, 0x30A0-0x30FF: カタカナ, 0x31F0-0x31FF: カタカナ拡張
+      (ord >= 0x3040 && ord <= 0x309F) || (ord >= 0x30A0 && ord <= 0x30FF) || (ord >= 0x31F0 && ord <= 0x31FF)
+    end
+
+    # Punctuation, digits, and whitespace are ignored so they don't
+    # invalidate an otherwise all-Chinese sentence (e.g. "這是中文。").
+    def meaningful_length
+      @text.chars.count { |char| char.match?(/\p{L}/) }
+    end
+
+    def analyze
+      @total_chinese = 0
+      @text.chars.each do |char|
+        @contains_kana ||= kana_char?(char)
+
+        next unless @chinese_processor.chinese_character?(char)
+
+        @total_chinese += 1
+
+        calculate_character_scores(char)
+      end
+    end
+
+    def calculate_character_scores(char)
+      if @chinese_processor.only_zh_tw?(char)
+        @traditional_score += 2
+      elsif @chinese_processor.only_zh_cn?(char)
+        @simplified_score += 2
+      end
+
+      @traditional_score += 0.5 if @variant_mapping.traditional_variants(char).any?
+      @simplified_score += 0.5 if @variant_mapping.simplified_variants(char).any?
+    end
+  end
+end

data/lib/unihan_lang/variant_mapping.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+module UnihanLang
+  class VariantMapping
+    def initialize
+      @traditional_to_simplified = load_variant_mappings
+      @simplified_to_traditional = {}
+      # 簡体字から繁体字へのマッピングを構築
+      @traditional_to_simplified.each do |trad, simps|
+        simps.each do |simp|
+          @simplified_to_traditional[simp] ||= Set.new
+          @simplified_to_traditional[simp] << trad
+        end
+      end
+    end
+
+    def traditional_variants(char)
+      @simplified_to_traditional[char] || Set.new
+    end
+
+    def simplified_variants(char)
+      @traditional_to_simplified[char] || Set.new
+    end
+
+    private
+
+    def load_variant_mappings
+      traditional_to_simplified = {}
+      file_path = File.join(File.dirname(__FILE__), "..", "..", "data", "Unihan_Variants.txt")
+
+      File.foreach(file_path, encoding: "UTF-8") do |line|
+        next if line.start_with?("#") || line.strip.empty?
+
+        fields = line.strip.split("\t")
+        # kTraditionalVariant フィールドの場合のみ処理
+        if fields.size >= 3 && fields[1] == ("kTraditionalVariant")
+          simp = [fields[0].gsub(/^U\+/, "").hex].pack("U")
+          trad = [fields[2].gsub(/^U\+/, "").hex].pack("U")
+          traditional_to_simplified[trad] ||= Set.new
+          traditional_to_simplified[trad] << simp
+        end
+      end
+      traditional_to_simplified
+    end
+  end
+end

data/lib/unihan_lang/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module UnihanLang
-  VERSION = "0.2.0"
+  VERSION = "0.2.1"
 end

data/lib/unihan_lang.rb

@@ -2,11 +2,14 @@
 
 require_relative "unihan_lang/version"
 require_relative "unihan_lang/chinese_processor"
+require_relative "unihan_lang/variant_mapping"
+require_relative "unihan_lang/chinese_score_analyzer"
 
 module UnihanLang
   class Unihan
     def initialize
       @chinese_processor = ChineseProcessor.new
+      @variant_mapping = VariantMapping.new
     end
 
     def zh_tw?(text)
@@ -49,19 +52,25 @@ module UnihanLang
       end
     end
 
-    private
+    def analyze_with_variants(text)
+      analyzer = ChineseScoreAnalyzer.new(text, @chinese_processor, @variant_mapping)
+      {
+        traditional_score: analyzer.traditional_score,
+        simplified_score: analyzer.simplified_score,
+        total_chinese: analyzer.total_chinese,
+      }
+    end
 
-    # テキストの言語比率を計算し、最も可能性の高い言語を返す
-    def language_ratio(text)
-      only_tw_chars = text.chars.count { |char| @chinese_processor.only_zh_tw?(char) }
-      only_cn_chars = text.chars.count { |char| @chinese_processor.only_zh_cn?(char) }
-      chinese_chars = text.chars.count { |char| @chinese_processor.chinese?(char) }
+    def determine_language_with_variants(text)
+      analyzer = ChineseScoreAnalyzer.new(text, @chinese_processor, @variant_mapping)
+      analyzer.dominant_language
+    end
 
-      return :unknown unless chinese_chars == text.length
-      return :tw if only_tw_chars > only_cn_chars
-      return :cn if only_cn_chars >= only_tw_chars
+    private
 
-      :unknown
+    def language_ratio(text)
+      analyzer = ChineseScoreAnalyzer.new(text, @chinese_processor, @variant_mapping)
+      analyzer.language_ratio
     end
   end
 end

metadata

@@ -1,14 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: unihan_lang
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.2.1
 platform: ruby
 authors:
 - kyubey1228
-autorequire:
 bindir: exe
 cert_chain: []
-date: 2024-10-17 00:00:00.000000000 Z
+date: 2026-08-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -66,20 +65,23 @@ files:
 - ".rubocop.yml"
 - Gemfile
 - Gemfile.lock
+- HOW_TO_UPDATE.md
 - LICENSE.md
 - README.ja.md
 - README.md
 - Rakefile
+- bump_new_version.sh
 - data/Unihan_Variants.txt
 - lib/unihan_lang.rb
 - lib/unihan_lang/chinese_processor.rb
+- lib/unihan_lang/chinese_score_analyzer.rb
+- lib/unihan_lang/variant_mapping.rb
 - lib/unihan_lang/version.rb
 - unihan_lang.gemspec
 homepage: https://github.com/kyubey1228/unihan_lang
 licenses:
 - MIT
 metadata: {}
-post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -94,8 +96,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.5.3
-signing_key:
+rubygems_version: 3.6.2
 specification_version: 4
 summary: Language detection for Chinese characters
 test_files: []