known_item_search_classifier 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: df03a8dc0661439a41c7d3366a49573f463a176f
+  data.tar.gz: 58713a770bba4a173adc4e88bb507f256b4eafc5
+SHA512:
+  metadata.gz: 470565b3b2932df41a0d02b99048746b5f5e97de476b273a9648d473d63d7e73e44d038ff2b35ffd1f0bbd413960d35df5ca1ad8e8b8902038aedb57d64225f8
+  data.tar.gz: 7e81f8262925a653ab33c12844947f9df40b5510cf3e43d801d6a31cab7f3bf3a3de58dc9693ecbd71283ff309dc9bdb6490d1a6be25fe436726ffc09fea28e0

data/lib/known_item_search_classifier/classifier.rb

@@ -0,0 +1,53 @@
+# Classifies search strings as either known-item searches or unknown-item searches
+require 'gaussian_naive_bayes'
+
+module KnownItemSearchClassifier
+    class Classifier
+        def initialize
+            set = DefaultTrainingSet.new
+            @default_training_set = GaussianNaiveBayes::Classifier.new set.categories_summaries, set.categories_probabilities
+        end
+        def is_known_item_search? query_string
+            return classify query_string
+        end
+        def train training_set
+            if defined? @custom_training_set
+                @custom_training_set = GaussianNaiveBayes::Learner.new
+            end
+            training_set.each do |query|
+                submit_vector query
+            end
+        end
+        def train_from_csv filename
+            if defined? @custom_training_set
+                @custom_training_set = GaussianNaiveBayes::Learner.new
+            end
+            csv = CSV.read(filename)
+            csv.each do |line|
+                submit_vector line
+            end
+        end
+
+    private
+        def classify string
+            f = FeatureExtractor.new string
+            feature_array = f.feature_array
+            if defined? @custom_training_set
+                classifier = @custom_training_set.classifier
+                query_class = classifier.classify(feature_array)
+            else
+                query_class = @default_training_set.classify(feature_array)
+            end
+            return query_class
+            if :known == query_class
+                return true
+            else
+                return false
+            end
+        end
+        def submit_vector arr
+            f = FeatureExtractor.new arr[0]
+            @custom_training_set.train f.feature_array, arr[1]
+        end
+    end
+end

data/lib/known_item_search_classifier/default_training_set.rb

@@ -0,0 +1,21 @@
+module KnownItemSearchClassifier
+    class DefaultTrainingSet
+        attr_reader :categories_probabilities, :categories_summaries
+        def initialize
+            @categories_probabilities={:unknown=>0.78, :known=>0.22}
+            @categories_summaries= {
+                :unknown=>{
+                    0=>{:mean=>0.2564102564102564, :standard_deviation=>0.4394771815921655},
+                    1=>{:mean=>0.03418803418803419, :standard_deviation=>0.11344969312798027},
+                    2=>{:mean=>0.002564102564102564, :standard_deviation=>0.0226455406828919},
+                    3=>{:mean=>0.12991452991452992, :standard_deviation=>0.26648206508636013},
+                    4=>{:mean=>2.7948717948717947, :standard_deviation=>2.053561836691609}},
+                :known=>{
+                    0=>{:mean=>0.5454545454545454, :standard_deviation=>0.5096471914376255},
+                    1=>{:mean=>0.051659451659451655, :standard_deviation=>0.07957404805575267},
+                    2=>{:mean=>0.021248196248196245, :standard_deviation=>0.04412470821426937},
+                    3=>{:mean=>0.22550505050505054, :standard_deviation=>0.2520704609787127},
+                    4=>{:mean=>7.590909090909091, :standard_deviation=>5.770690236086651}}}
+        end
+    end
+end

data/lib/known_item_search_classifier/feature_extractor.rb

@@ -0,0 +1,68 @@
+require 'engtagger'
+
+module KnownItemSearchClassifier
+    class FeatureExtractor
+        def initialize string
+            @string = string
+            tagger = EngTagger.new
+            @tagged = tagger.get_readable string
+            @num_words = @tagged.scan(/\/[A-Z]{2}/).size.to_f
+
+            @mixed_case = is_mixed_case?
+            @punctuation_ratio = punctuation_ratio
+            @determiner_ratio = determiner_ratio
+            @proper_noun_ratio = proper_noun_ratio
+
+            #@num_keywords = count_keywords
+            #@refers_to_an_item_that_is_known = check_against_known_titles
+            
+        end
+        def feature_array
+            return [@mixed_case, @punctuation_ratio, @determiner_ratio, @proper_noun_ratio, @num_words]
+        end
+        private
+        def is_mixed_case?
+            if @string =~ /[A-Z]/ and @string =~ /[a-z]/
+                return 1.0
+            end 
+            return 0.0
+        end
+        def punctuation_ratio
+            num_punct = @tagged.scan(/\/PP/).size.to_f
+            return num_punct / @num_words
+        end
+        def determiner_ratio
+            num_det = @tagged.scan(/\/DET/).size.to_f
+            return num_det / @num_words
+        end
+        def proper_noun_ratio
+            num_prop_noun = @tagged.scan(/\/NNP/).size.to_f
+            return num_prop_noun / @num_words
+        end
+        def count_keywords
+        end
+        def check_against_known_titles
+        end
+        def count_keywords
+            keywords_to_match = ['journal', 'course', 'textbook']
+            num_keywords = 0
+            @query_string.split.each do |word|
+                if keywords_to_match.include? word.gsub(/[[:punct:]]/, '').downcase
+                    num_keywords = num_keywords + 1
+                end
+            end
+            return num_keywords
+        end
+        def check_against_known_titles
+           known_titles = [
+               'salt sugar fat',
+           ]
+           if known_titles.include? @query_string.downcase
+               return true
+           else
+               return false
+           end
+        end
+    end
+end
+

data/lib/known_item_search_classifier.rb

@@ -0,0 +1,3 @@
+require 'known_item_search_classifier/default_training_set'
+require 'known_item_search_classifier/feature_extractor'
+require 'known_item_search_classifier/classifier'

data/test/known_item_search_classifier_test.rb

@@ -0,0 +1,62 @@
+require 'coveralls'
+Coveralls.wear!
+require 'minitest/autorun'
+require './lib/known_item_search_classifier'
+
+
+class KnownItemSearchClassifierTest < Minitest::Test
+    classifier = KnownItemSearchClassifier::Classifier.new
+
+    known_item_training_set = [
+        "little house on the",
+        "the inconvenient truth",
+        "the question of animal Culture by Kevin N Laland; Bennett G Galef ",
+        "Robinson Ken. Creative Schools: The Grassroots Revolution That’s Transforming Eduction.  Viking. 2015. Print",
+        "The Boy in Zaquitos",
+        "The Mis-Education of the Negro",
+        "human relations interpersonal job-oriented skills",
+        "Research Methods for Business: A Skill-Building Approach Effectiveness of Instruction Performed through Computer-Assisted Activity Schedules on On-Schedule and Role-Play Skills of Children with Autism Spectrum Disorder",
+        "competency skills for the dental assiostant",
+        "Why did they kill?: Cambodia in the shadow of genocide",
+        "salt sugar fat",
+        "Making a Killing: Femicide, Free Trade, and La Frontera",
+    ]
+    known_item_training_set.each do |query|
+        cleaned_up_query = query.gsub(/[[:punct:]]/, '').gsub(/[[:space:]]/, '_')
+        define_method("test_#{cleaned_up_query}_is_classified_as_known_item") do
+            assert_equal(:known, classifier.is_known_item_search?(query))
+        end
+    end
+
+    unknown_item_training_set = [
+        "earthworms",
+        "network security",
+        "work stress",
+        "mummies",
+        "benefits of eating healthyhy",
+        "benefits of eating healthy",
+        "megadosing vitamin c",
+        "nutrition",
+        "penquin",
+        "bananas",
+        "food sourcing",
+        "whey protein",
+        "exotic animals",
+        "sweet home oregon",
+        "taylor swift",
+        "catholicism",
+        "Professional baking ",
+        "concussions after the nfl",
+        "IVF the US",
+        "adoption children the US",
+        "Films for the hearing impaired",
+        "wolves and the ecosystem",
+        "dr. martin luther king",
+    ]
+    unknown_item_training_set.each do |query|
+        cleaned_up_query = query.gsub(/[[:punct:]]/, '').gsub(/[[:space:]]/, '_')
+        define_method("test_#{cleaned_up_query}_is_not_false_positive") do
+            assert_equal(:unknown, classifier.is_known_item_search?(query))
+        end
+    end
+end

metadata

@@ -0,0 +1,106 @@
+--- !ruby/object:Gem::Specification
+name: known_item_search_classifier
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Jane Sandberg
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2016-11-13 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: engtagger
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.2.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.2.1
+- !ruby/object:Gem::Dependency
+  name: gaussian_naive_bayes
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.1.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.1.1
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: coveralls
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.7.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.7.0
+description: Classify search query strings
+email: sandbej@linnbenton.edu
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/known_item_search_classifier.rb
+- lib/known_item_search_classifier/classifier.rb
+- lib/known_item_search_classifier/default_training_set.rb
+- lib/known_item_search_classifier/feature_extractor.rb
+- test/known_item_search_classifier_test.rb
+homepage: https://github.com/sandbergja/known_item_search_classifier
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.5.1
+signing_key: 
+specification_version: 4
+summary: A ruby gem that classifies search query strings as either known-item searches
+  or unknown-item searches
+test_files:
+- test/known_item_search_classifier_test.rb