groonga-synonym 1.0.0

data/lib/groonga-synonym/sudachi.rb

@@ -0,0 +1,122 @@
+# Copyright (C) 2021  Sutou Kouhei <kou@clear-code.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+require "datasets"
+
+require_relative "synonym"
+
+module GroongaSynonym
+  class Sudachi
+    include Enumerable
+
+    def initialize
+      @dataset = Datasets::SudachiSynonymDictionary.new
+    end
+
+    def each
+      return to_enum(__method__) unless block_given?
+
+      groups = {}
+      group_id = nil
+      group = nil
+      @dataset.each do |synonym|
+        if synonym.group_id != group_id
+          emit_synonyms(groups, group)
+          group_id = synonym.group_id
+          group = [synonym]
+        else
+          group << synonym
+        end
+      end
+      emit_synonyms(groups, group)
+      filter_groups(groups) do |term, synonyms|
+        yield(term, synonyms)
+      end
+    end
+
+    private
+    def emit_synonyms(groups, group)
+      return if group.nil?
+      target_synonyms = group.reject do |synonym|
+        synonym.expansion_type == :never
+      end
+      return if target_synonyms.size <= 1
+      target_synonyms.each_with_index do |typical, i|
+        next unless typical.expansion_type == :always
+        term = typical.notation
+        synonyms = []
+        target_synonyms.each_with_index do |synonym, j|
+          if i == j
+            weight = nil
+          elsif synonym.lexeme_id == typical.lexeme_id
+            weight = 0.8
+          else
+            weight = 0.6
+          end
+          synonyms << Synonym.new(synonym.notation, weight)
+        end
+        # e.g.: 働き手
+        if groups.key?(term)
+          groups[term] |= synonyms
+        else
+          groups[term] = synonyms
+        end
+      end
+    end
+
+    def filter_groups(groups)
+      groups.each do |term, synonyms|
+        typical_synonym = nil
+        other_synonyms = []
+        synonyms.each do |synonym|
+          if synonym.weight.nil?
+            typical_synonym = synonym
+          else
+            other_synonyms << synonym
+          end
+        end
+        others_sub_synonyms = []
+        sub_synonyms = []
+        super_synonyms = []
+        other_synonyms.each do |synonym|
+          is_sub_synonym = other_synonyms.any? do |other_synonym|
+            other_synonym != synonym and
+              synonym.term.include?(other_synonym.term)
+          end
+          if is_sub_synonym
+            others_sub_synonyms << synonym
+          elsif term.include?(synonym.term)
+            sub_synonyms << synonym
+          elsif synonym.term.include?(term)
+            super_synonyms << synonym
+          end
+        end
+        synonyms -= others_sub_synonyms
+        synonyms -= super_synonyms
+        unless sub_synonyms.empty?
+          sorted_sub_synonyms = sub_synonyms.sort_by do |synonym|
+            synonym.term.size
+          end
+          typical_sub_synonym, *other_sub_synonyms = sorted_sub_synonyms
+          synonyms -= other_sub_synonyms
+          synonyms.delete(typical_synonym)
+          synonyms << Synonym.new(typical_synonym.term,
+                                  (1.0 - typical_sub_synonym.weight).round(2))
+        end
+        yield(term, synonyms)
+      end
+    end
+  end
+end

data/lib/groonga-synonym/synonym.rb

@@ -0,0 +1,61 @@
+# Copyright (C) 2021  Sutou Kouhei <kou@clear-code.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+module GroongaSynonym
+  class Synonym
+    attr_reader :term
+    attr_reader :weight
+    def initialize(term, weight=nil)
+      @term = term
+      @weight = weight
+    end
+
+    def to_groonga
+      formatted = ""
+      if @weight and @weight != 1.0
+        formatted << ">" << ("%f" % (@weight - 1)).gsub(/0+\z/, "")
+      end
+      formatted << escape_term(@term)
+      formatted
+    end
+
+    def ==(other)
+      other.is_a?(self.class) and
+        @term == other.term and
+        @weight == other.weight
+    end
+
+    def eql?(other)
+      self == other
+    end
+
+    def hash
+      [@term, @weight].hash
+    end
+
+    private
+    def escape_term(term)
+      return "\"#{term}\"" if term == "OR"
+      term = term.gsub(/["()\\*:+-]/) do |matched|
+        "\\#{matched}"
+      end
+      if term.include?(" ")
+        "\"#{term}\""
+      else
+        term
+      end
+    end
+  end
+end

data/lib/groonga-synonym/version.rb

@@ -0,0 +1,18 @@
+# Copyright (C) 2021  Sutou Kouhei <kou@clear-code.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+module GroongaSynonym
+  VERSION = "1.0.0"
+end

metadata

@@ -0,0 +1,71 @@
+--- !ruby/object:Gem::Specification
+name: groonga-synonym
+version: !ruby/object:Gem::Version
+  version: 1.0.0
+platform: ruby
+authors:
+- Sutou Kouhei
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2021-07-15 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: red-datasets
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.1.3
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.1.3
+description: ''
+email:
+- kou@clear-code.com
+executables:
+- groonga-synonym-generate
+extensions: []
+extra_rdoc_files: []
+files:
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/groonga-synonym-generate
+- groonga-synonym.gemspec
+- lib/groonga-synonym.rb
+- lib/groonga-synonym/command-line/generator.rb
+- lib/groonga-synonym/groonga-generator.rb
+- lib/groonga-synonym/pgroonga-generator.rb
+- lib/groonga-synonym/sudachi.rb
+- lib/groonga-synonym/synonym.rb
+- lib/groonga-synonym/version.rb
+homepage: https://github.com/groonga/groonga-synonym
+licenses:
+- GPL-3.0+
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.3.0.dev
+signing_key: 
+specification_version: 4
+summary: Groonga synonym provides tools for synonym of Groonga families.
+test_files: []