neologdish-normalizer 0.0.1 → 0.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f2f96d57b26c4e44d37d47bcabcec5a820470d4e89beb59b709c2dc36ecd449b
-  data.tar.gz: fc7479594872db9c262cd13acf8e4f717936b289ead7fe136c65f0f211d32417
+  metadata.gz: 2aa7f75228455a5870d56e6f8b74dd74dcb9f8c0619bb86eda89e02518478afd
+  data.tar.gz: b92a3042b6f7c3ad7b2653bb7f04643e2769733104ee78e1c689170776729b53
 SHA512:
-  metadata.gz: 84457e6fd2b5fae0b87d51a86b0e9a5c4aaf808b436ba97654c0939630fe19142c6bb2eb6729baeb0773882726aacffb62f67139be5c03b8c2404e74ca1b2408
-  data.tar.gz: 312f9993dceed61d3443f7ddb6c06904d49b21a52725be48721bf0683e75111b76b6b3bd53cde1d7a34315adb157f91306a75f3561f584af74d269bd71eb19aa
+  metadata.gz: 4479aa95d7ba3d2aca68feecf5a4674dc5af9f71c795ac748bcb6e38558b7d9e73c4d068b071441c020b96a4765b58224ad0328fe4856aafcf934bf6eb4d6e99
+  data.tar.gz: fb794e40955c4460bf592fac2f9baf415bb9c486ff07a288fc9b3a083bb8256a98d949291b497fd36b2e846ea083dee3925afd8001ae6a7ce90e4536badf6836

data/.ruby-version

@@ -1 +1 @@
-3.3
+3.3.5

data/README.md

@@ -1,4 +1,4 @@
-# Neologdish::Normalizer for Ruby
+# Neologdish::Normalizer for Ruby [![Check](https://github.com/moznion/neologdish-normalizer-ruby/actions/workflows/check.yml/badge.svg)](https://github.com/moznion/neologdish-normalizer-ruby/actions/workflows/check.yml) [![Gem Version](https://badge.fury.io/rb/neologdish-normalizer.svg)](https://badge.fury.io/rb/neologdish-normalizer)
 
 A Japanese text normalization library for Ruby follows the conventions of [neologd/mecab-ipadic-neologd](https://github.com/neologd/mecab-ipadic-neologd), with some performance optimizations, without external dependencies. It is designed to preprocess Japanese text before applying NLP techniques.
 
@@ -27,18 +27,16 @@ The benchmark script is here: [./scripts/benchmark.rb](./scripts/benchmark.rb)
 
 ## Installation
 
-TODO: Replace `UPDATE_WITH_YOUR_GEM_NAME_IMMEDIATELY_AFTER_RELEASE_TO_RUBYGEMS_ORG` with your gem name right after releasing it to RubyGems.org. Please do not do it earlier due to security reasons. Alternatively, replace this section with instructions to install your gem from git if you don't plan to release to RubyGems.org.
-
 Install the gem and add to the application's Gemfile by executing:
 
 ```bash
-bundle add UPDATE_WITH_YOUR_GEM_NAME_IMMEDIATELY_AFTER_RELEASE_TO_RUBYGEMS_ORG
+bundle add 'neologdish-normalizer'
 ```
 
 If bundler is not being used to manage dependencies, install the gem by executing:
 
 ```bash
-gem install UPDATE_WITH_YOUR_GEM_NAME_IMMEDIATELY_AFTER_RELEASE_TO_RUBYGEMS_ORG
+gem install 'neologdish-normalizer'
 ```
 
 ## Development

data/Rakefile

@@ -12,6 +12,11 @@ namespace :rbs do
   end
 end
 
+desc 'run benchmark script'
+task :benchmark do
+  sh 'ruby ./scripts/benchmark.rb'
+end
+
 Rake::TestTask.new do |task|
   task.libs = %w[lib test]
   task.test_files = FileList['test/**/*.rb']

data/lib/neologdish/normalizer/version.rb

@@ -2,6 +2,6 @@
 
 module Neologdish
   module Normalizer
-    VERSION = '0.0.1' #: String
+    VERSION = '0.1.0' #: String
   end
 end

data/lib/neologdish/normalizer.rb

@@ -69,15 +69,18 @@ module Neologdish
     # Normalize the given text.
     #
     # @rbs str: String
+    # @rbs override_conversion_map: Hash[String, String]
     # @rbs return: String
-    def normalize(str)
+    def normalize(str, override_conversion_map = {})
+      conversion_map = CONVERSION_MAP.merge(override_conversion_map)
+
       squeezee = ''
       prev_latin = false
       whitespace_encountered = false
       encountered_half_width_kana = nil
       normalized = str.chars.map do |c|
         prefix = ''
-        c = CONVERSION_MAP[c] || c
+        c = conversion_map[c] || c
 
         # normalize the Half-width kana to full-width
         if encountered_half_width_kana
@@ -112,7 +115,7 @@ module Neologdish
           c = ''
         else
           prefix = ' ' if is_latin && whitespace_encountered
-          whitespace_encountered = false
+          whitespace_encountered &&= c == '' # take care for consecutive spaces on the right side
         end
         prev_latin = is_latin
 

data/renovate.json

@@ -0,0 +1,6 @@
+{
+  "$schema": "https://docs.renovatebot.com/renovate-schema.json",
+  "extends": [
+    "config:recommended"
+  ]
+}

data/sig/generated/neologdish/normalizer.rbs

@@ -1,6 +1,7 @@
 # Generated from lib/neologdish/normalizer.rb with RBS::Inline
 
 module Neologdish
+  # A Japanese text normalizer module according to the neologd convention.
   module Normalizer
     CONVERSION_MAP: Hash[String, String]
 
@@ -12,8 +13,11 @@ module Neologdish
 
     HANDAKUON_KANA_MAP: Hash[String, String]
 
+    # Normalize the given text.
+    #
     # @rbs str: String
+    # @rbs override_conversion_map: Hash[String, String]
     # @rbs return: String
-    def normalize: (String str) -> String
+    def normalize: (String str, ?Hash[String, String] override_conversion_map) -> String
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: neologdish-normalizer
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.1.0
 platform: ruby
 authors:
 - moznion
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2024-10-28 00:00:00.000000000 Z
+date: 2024-10-29 00:00:00.000000000 Z
 dependencies: []
 description: A Japanese text normalization library follows the conventions of neologd
   with some performance optimizations. It is designed to preprocess Japanese text
@@ -26,6 +26,7 @@ files:
 - Rakefile
 - lib/neologdish/normalizer.rb
 - lib/neologdish/normalizer/version.rb
+- renovate.json
 - sig/generated/neologdish/normalizer.rbs
 - sig/generated/neologdish/normalizer/version.rbs
 homepage: https://github.com/moznion/neologdish-normalizer
@@ -50,7 +51,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.5.22
+rubygems_version: 3.5.16
 signing_key:
 specification_version: 4
 summary: A Japanese text normalization library follows the conventions of neologd