known_item_search_classifier 0.2.1 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 89a7295765e4fe6c6503152b733a79b46151e8bd
-  data.tar.gz: 4b541eaf18b0c8a8d74ad8f4525ad28888dcf652
+SHA256:
+  metadata.gz: a95567708e0b56c79c3a102e1d7c72e493e5660518de3b24c8fc42a691609938
+  data.tar.gz: 70ea59d9d7c0451b3d454506e578c2761c12e7d226edca852431c76bee1a9456
 SHA512:
-  metadata.gz: 4166fa3994f73f19828b9c7c53ee6d8678cdd284cb39189b42abd03bfd69b0b2cd4849105406d7d0c5d49f85e2a9e631808ea3bee23a6374d15bb63735324740
-  data.tar.gz: 01d20e0df51bc086e5aeb6b9348cf2543bc1dbb892dbf246d13158e72a36024b6344c7ce52a4bee60a6d3537e262b6205ef2b8168deb95d0815dc4997c6be179
+  metadata.gz: 4fb37b0932e9e0c32f9ec0ef6bdc563bd7e4e4cca5f401186daec4ae8d3be112b96478a9f04cf715620144e5db30e340959db808d5cc99841360dd72d480984d
+  data.tar.gz: 96777f8fa22a9208dc4e22a76a4c74dd57785c32a52de386d0e78678880a0d0faa020e0390fe8d1275c2cc3326cdf6cfa2fae6dcee4bbe299c66c79918a696fd

data/lib/known_item_search_classifier/classifier.rb

@@ -3,52 +3,54 @@ require 'csv'
 require 'gaussian_naive_bayes'
 
 module KnownItemSearchClassifier
-    class Classifier
-        def initialize
-            set = DefaultTrainingSet.new
-            @default_training_set = GaussianNaiveBayes::Classifier.new set.categories_summaries, set.categories_probabilities
-        end
-        def is_known_item_search? query_string
-            return classify query_string
-        end
-        def train training_set
-            unless defined? @custom_training_set
-                @custom_training_set = GaussianNaiveBayes::Learner.new
-            end
-            training_set.each do |query|
-                submit_vector query
-            end
-        end
-        def train_from_csv filename
-            unless defined? @custom_training_set
-                @custom_training_set = GaussianNaiveBayes::Learner.new
-            end
-            csv = ::CSV.read(filename)
-            csv.each do |line|
-                submit_vector line
-            end
-        end
+  class Classifier
+    def initialize
+      set = DefaultTrainingSet.new
+      @default_training_set = GaussianNaiveBayes::Classifier.new set.categories_summaries,
+                                                                 set.categories_probabilities
+    end
+
+    def is_known_item_search?(query_string)
+      classify query_string
+    end
+
+    def train(training_set)
+      @custom_training_set = GaussianNaiveBayes::Learner.new unless defined? @custom_training_set
+      training_set.each do |query|
+        submit_vector query
+      end
+    end
+
+    def train_from_csv(filename)
+      @custom_training_set = GaussianNaiveBayes::Learner.new unless defined? @custom_training_set
+      csv = ::CSV.read(filename)
+      csv.each do |line|
+        submit_vector line
+      end
+    end
 
     private
-        def classify string
-            f = FeatureExtractor.new string
-            feature_array = f.feature_array
-            if defined? @custom_training_set
-                classifier = @custom_training_set.classifier
-                query_class = classifier.classify(feature_array)
-            else
-                query_class = @default_training_set.classify(feature_array)
-            end
-            return query_class
-            if :known == query_class
-                return true
-            else
-                return false
-            end
-        end
-        def submit_vector arr
-            f = FeatureExtractor.new arr[0]
-            @custom_training_set.train f.feature_array, arr[1]
-        end
+
+    attr_reader :custom_tr
+
+    def classify(string)
+      f = FeatureExtractor.new string
+      feature_array = f.feature_array
+      if defined? @custom_training_set
+        classifier = @custom_training_set.classifier
+        query_class = classifier.classify(feature_array)
+      else
+        query_class = @default_training_set.classify(feature_array)
+      end
+      return query_class
+      return true if :known == query_class
+
+      false
+    end
+
+    def submit_vector(arr)
+      f = FeatureExtractor.new arr[0]
+      @custom_training_set.train f.feature_array, arr[1]
     end
+  end
 end

data/lib/known_item_search_classifier/default_training_set.rb

@@ -1,23 +1,24 @@
 module KnownItemSearchClassifier
-    class DefaultTrainingSet
-        attr_reader :categories_probabilities, :categories_summaries
-        def initialize
-            @categories_probabilities={:unknown=>0.835, :known=>0.165}
-            @categories_summaries= {
-                :unknown=>{
-                    0=>{:mean=>0.32335329341317365, :standard_deviation=>0.4691630728112455},
-                    1=>{:mean=>0.01867693185058454, :standard_deviation=>0.0856521002382124},
-                    2=>{:mean=>0.0024950099800399197, :standard_deviation=>0.02318575984424029},
-                    3=>{:mean=>0.18252067293983462, :standard_deviation=>0.32649287803592736},
-                    4=>{:mean=>2.2634730538922154, :standard_deviation=>1.3497147972472143},
-                    5=>{:mean=>0.20958083832335328, :standard_deviation=>1.2933208182456999}},
-                :known=>{
-                    0=>{:mean=>0.3333333333333333, :standard_deviation=>0.478713553878169},
-                    1=>{:mean=>0.034283854046699896, :standard_deviation=>0.07844034834013752},
-                    2=>{:mean=>0.06397250092902267, :standard_deviation=>0.10673099909054994},
-                    3=>{:mean=>0.06715805055726004, :standard_deviation=>0.1488979015655406},
-                    4=>{:mean=>4.696969696969697, :standard_deviation=>4.9591131294116515},
-                    5=>{:mean=>3.9393939393939394, :standard_deviation=>5.606577576491037}}}
-        end
+  class DefaultTrainingSet
+    attr_reader :categories_probabilities, :categories_summaries
+
+    def initialize
+      @categories_probabilities = { 'known' => 0.3333333333333333, 'unknown' => 0.6666666666666666 }
+      @categories_summaries =
+        { 'known' =>
+          { 0 => { mean: 0.6, standard_deviation: 0.5 },
+            1 => { mean: 0.0516060606060606, standard_deviation: 0.09910312916958242 },
+            2 => { mean: 0.06633333333333333, standard_deviation: 0.13412266359153804 },
+            3 => { mean: 0.2575454545454545, standard_deviation: 0.27976953051588926 },
+            4 => { mean: 4.76, standard_deviation: 3.8867295592395754 },
+            5 => { mean: 3.48, standard_deviation: 4.91697739131132 } },
+          'unknown' =>
+          { 0 => { mean: 0.18, standard_deviation: 0.38808793449160356 },
+            1 => { mean: 0.03966666666666667, standard_deviation: 0.1241245990920947 },
+            2 => { mean: 0.009000000000000001, standard_deviation: 0.04482391854210637 },
+            3 => { mean: 0.11, standard_deviation: 0.25134558515041244 },
+            4 => { mean: 2.44, standard_deviation: 1.0720950308167836 },
+            5 => { mean: 0.14, standard_deviation: 0.7001457574195914 } } }
     end
+  end
 end

data/lib/known_item_search_classifier/feature_extractor.rb

@@ -1,73 +1,75 @@
 require 'engtagger'
 
 module KnownItemSearchClassifier
-    class FeatureExtractor
-        def initialize string
-            @string = string
-            tagger = EngTagger.new
-            @tagged = tagger.get_readable string
-            @num_words = @tagged.scan(/\/[A-Z]{2}/).size.to_f
-
-            @mixed_case = is_mixed_case?
-            @punctuation_ratio = punctuation_ratio
-            @determiner_ratio = determiner_ratio
-            @proper_noun_ratio = proper_noun_ratio
-            @numeric_count = numeric_count
-
-            #@num_keywords = count_keywords
-            #@refers_to_an_item_that_is_known = check_against_known_titles
-            
-        end
-        def feature_array
-            return [@mixed_case, @punctuation_ratio, @determiner_ratio, @proper_noun_ratio, @num_words, @numeric_count]
-        end
-        private
-        def is_mixed_case?
-            if @string =~ /[A-Z]/ and @string =~ /[a-z]/
-                return 1.0
-            end 
-            return 0.0
-        end
-        def punctuation_ratio
-            num_punct = @tagged.scan(/\/PP/).size.to_f
-            return num_punct / @num_words
-        end
-        def determiner_ratio
-            num_det = @tagged.scan(/\/DET/).size.to_f
-            return num_det / @num_words
-        end
-        def numeric_count
-            return @string.scan(/[0-9]/).length
-        end
-        def proper_noun_ratio
-            num_prop_noun = @tagged.scan(/\/NNP/).size.to_f
-            return num_prop_noun / @num_words
-        end
-        def count_keywords
-        end
-        def check_against_known_titles
-        end
-        def count_keywords
-            keywords_to_match = ['journal', 'course', 'textbook']
-            num_keywords = 0
-            @query_string.split.each do |word|
-                if keywords_to_match.include? word.gsub(/[[:punct:]]/, '').downcase
-                    num_keywords = num_keywords + 1
-                end
-            end
-            return num_keywords
-        end
-        def check_against_known_titles
-           known_titles = [
-               'fountainhead',
-               'salt sugar fat',
-           ]
-           if known_titles.include? @query_string.downcase
-               return true
-           else
-               return false
-           end
-        end
+  class FeatureExtractor
+    def initialize(string)
+      @string = string
+      tagger = EngTagger.new
+      @tagged = tagger.get_readable string
+      @num_words = @tagged.scan(%r{/[A-Z]{2}}).size.to_f
+
+      @mixed_case = is_mixed_case?
+      @punctuation_ratio = punctuation_ratio
+      @determiner_ratio = determiner_ratio
+      @proper_noun_ratio = proper_noun_ratio
+      @numeric_count = numeric_count
+
+      # @num_keywords = count_keywords
+      # @refers_to_an_item_that_is_known = check_against_known_titles
     end
-end
 
+    def feature_array
+      [@mixed_case, @punctuation_ratio, @determiner_ratio, @proper_noun_ratio, @num_words, @numeric_count]
+    end
+
+    private
+
+    def is_mixed_case?
+      return 1.0 if @string =~ /[A-Z]/ and @string =~ /[a-z]/
+
+      0.0
+    end
+
+    def punctuation_ratio
+      num_punct = @tagged.scan(%r{/PP}).size.to_f
+      num_punct / @num_words
+    end
+
+    def determiner_ratio
+      num_det = @tagged.scan(%r{/DET}).size.to_f
+      num_det / @num_words
+    end
+
+    def numeric_count
+      @string.scan(/[0-9]/).length
+    end
+
+    def proper_noun_ratio
+      num_prop_noun = @tagged.scan(%r{/NNP}).size.to_f
+      num_prop_noun / @num_words
+    end
+
+    def count_keywords; end
+
+    def check_against_known_titles; end
+
+    def count_keywords
+      keywords_to_match = %w[journal course textbook]
+      num_keywords = 0
+      @query_string.split.each do |word|
+        num_keywords += 1 if keywords_to_match.include? word.gsub(/[[:punct:]]/, '').downcase
+      end
+      num_keywords
+    end
+
+    def check_against_known_titles
+      known_titles = [
+        'fountainhead',
+        'salt sugar fat'
+      ]
+      return true if known_titles.include? @query_string.downcase
+
+      false
+    end
+  end
+end

data/test/known_item_search_classifier_test.rb

@@ -1,62 +1,49 @@
-require 'coveralls'
-Coveralls.wear!
 require 'minitest/autorun'
 require './lib/known_item_search_classifier'
 
-
 class KnownItemSearchClassifierTest < Minitest::Test
-    classifier = KnownItemSearchClassifier::Classifier.new
+  classifier = KnownItemSearchClassifier::Classifier.new
 
-    known_item_training_set = [
-        "little house on the",
-        "the inconvenient truth",
-        "the question of animal Culture by Kevin N Laland; Bennett G Galef ",
-        "Robinson Ken. Creative Schools: The Grassroots Revolution That’s Transforming Eduction.  Viking. 2015. Print",
-        "The Boy in Zaquitos",
-        "The Mis-Education of the Negro",
-        "human relations interpersonal job-oriented skills",
-        "Research Methods for Business: A Skill-Building Approach Effectiveness of Instruction Performed through Computer-Assisted Activity Schedules on On-Schedule and Role-Play Skills of Children with Autism Spectrum Disorder",
-        "competency skills for the dental assiostant",
-        "Why did they kill?: Cambodia in the shadow of genocide",
-        "salt sugar fat",
-        "Making a Killing: Femicide, Free Trade, and La Frontera",
-    ]
-    known_item_training_set.each do |query|
-        cleaned_up_query = query.gsub(/[[:punct:]]/, '').gsub(/[[:space:]]/, '_')
-        define_method("test_#{cleaned_up_query}_is_classified_as_known_item") do
-            assert_equal(:known, classifier.is_known_item_search?(query))
-        end
+  known_item_training_set = [
+    # 'hobbit first edition',  -- classifier incorrectly classifies this as unknown
+    # 'my soul is rested',  -- classifier incorrectly classifies this as unknown
+    # 'new yorker',  -- classifier incorrectly classifies this as unknown
+    # 'when harry met sally', -- classifier incorrectly classifies this as unknown
+    # '"neo tekunoroji"',  -- classifier incorrectly classifies this as unknown
+    '99131236427206421',
+    'A decision making model for selecting start-up businesses in a government venture capital scheme',
+    # 'Dostoevsky Brothers Karamazov', -- classifier incorrectly classifies this as unknown
+    # 'Lawrence Classic American Literature', -- classifier incorrectly classifies this as unknown
+    # 'salt sugar fat', -- classifier incorrectly classifies this as unknown
+    'Robinson Ken. Creative Schools: The Grassroots Revolution That’s Transforming Eduction.  Viking. 2015. Print',
+    'the inconvenient truth',
+    'Polarization: What Everyone Needs to Know',
+    'little house on the'
+  ]
+  known_item_training_set.each do |query|
+    cleaned_up_query = query.gsub(/[[:punct:]]/, '').gsub(/[[:space:]]/, '_')
+    define_method("test_#{cleaned_up_query}_is_classified_as_known_item") do
+      assert_equal(:known, classifier.is_known_item_search?(query).to_sym)
     end
+  end
 
-    unknown_item_training_set = [
-        "earthworms",
-        "network security",
-        "work stress",
-        "mummies",
-        "benefits of eating healthyhy",
-        "benefits of eating healthy",
-        "megadosing vitamin c",
-        "nutrition",
-        "penquin",
-        "bananas",
-        "food sourcing",
-        "whey protein",
-        "exotic animals",
-        "sweet home oregon",
-        "taylor swift",
-        "catholicism",
-        "Professional baking ",
-        "concussions after the nfl",
-        "IVF the US",
-        "adoption children the US",
-        "Films for the hearing impaired",
-        "wolves and the ecosystem",
-        "dr. martin luther king",
-    ]
-    unknown_item_training_set.each do |query|
-        cleaned_up_query = query.gsub(/[[:punct:]]/, '').gsub(/[[:space:]]/, '_')
-        define_method("test_#{cleaned_up_query}_is_not_false_positive") do
-            assert_equal(:unknown, classifier.is_known_item_search?(query))
-        end
+  unknown_item_training_set = [
+    'colonial mexico textiles',
+    'history of horses',
+    'medical expertise COVID',
+    'music and sexuality',
+    'paper industry',
+    'sun ra',
+    # 'concussions after the nfl', -- classifier incorrectly classifies this as known
+    'Professional baking ',
+    'Manos chatzidakis',
+    'whey protein',
+    'benefits of eating healthyhy'
+  ]
+  unknown_item_training_set.each do |query|
+    cleaned_up_query = query.gsub(/[[:punct:]]/, '').gsub(/[[:space:]]/, '_')
+    define_method("test_#{cleaned_up_query}_is_not_false_positive") do
+      assert_equal(:unknown, classifier.is_known_item_search?(query).to_sym)
     end
+  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: known_item_search_classifier
 version: !ruby/object:Gem::Version
-  version: 0.2.1
+  version: 0.3.0
 platform: ruby
 authors:
 - Jane Sandberg
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2017-03-25 00:00:00.000000000 Z
+date: 2024-11-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: engtagger
@@ -53,21 +53,35 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: coveralls
+  name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '='
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rubocop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 0.7.0
+        version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '='
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 0.7.0
+        version: '0'
 description: Classify search query strings
-email: sandbej@linnbenton.edu
+email:
 executables: []
 extensions: []
 extra_rdoc_files: []
@@ -81,7 +95,7 @@ homepage: https://github.com/sandbergja/known_item_search_classifier
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -89,16 +103,15 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 3.0.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.5.1
-signing_key: 
+rubygems_version: 3.5.16
+signing_key:
 specification_version: 4
 summary: A ruby gem that classifies search query strings as either known-item searches
   or unknown-item searches