site_classifier 0.0.6 → 0.0.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: bdaa0867f7090d45492ae06ae60405f6cbb4af14
-  data.tar.gz: fb2a4af89c55d876d0a0786b2244ce64bdd06c09
+  metadata.gz: 425b9df31050d7e9a4f2e957a4e7bcf85668c4a2
+  data.tar.gz: 79757d84732bcfe3ca38f968af8587d6d5af47e0
 SHA512:
-  metadata.gz: f3b5313c03e1b98496bb8e06ca886e8156afec5c4138090e4a4a43586b83b1e47db4f040e805d865ec05f5d3dbde2c1aef02410ee6d7f621216cfb7c355b2fb7
-  data.tar.gz: 38e2777f8386bdc950996d9d6dac82a394e5e2d7d2b9cb443719ce9d4b788d95aa24460dd5d26aba0f4e413406e8936a2013fdfc5027a394e17bef45a23541c7
+  metadata.gz: df676399410f5aae1a2b7509a8eea697c033b127705ef146f86575b21ae69256b4f02c3949146e13e2312c689a407c0652ffad6ae268dc239f4d522adcb55440
+  data.tar.gz: df848b49799f73aa15124e71da2e33c2e7b4dd1bfec5380ab5bb65b92161956578701fbf51e46ccc6d3279a8120f0e6157ed1244cd9690364752c4f8eeaebd79

data/lib/site_classifier.rb

@@ -22,8 +22,4 @@ module SiteClassifier
   def self.configuration
     @setup ||= SiteClassifier::Configuration.new
   end
-
-  def self.extract!(url)
-    SiteClassifier::Extractor.parse_site(url)
-  end
 end

data/lib/site_classifier/extractor.rb

@@ -30,30 +30,29 @@ module SiteClassifier
     # Extract most significant tags
     def most_significant
       most_sig = []
-
-      # TODO: replace with NLP.
-      # description.to_s.split.each do |word|
-      #   self.word_frequency[word] ||= 0
-      #   self.word_frequency[word] += 1
+      # if !description.nil?
+      #   if tags.any?
+      #     most_sig = tags.select {|tag| self.description.downcase.include?(tag)}.collect {|tag| tag.singularize }
+      #   else
+      #     most_sig = word_frequency.keys.select {|tag| self.description.downcase.include?(tag)}.collect {|tag| tag.singularize }
+      #   end
       # end
 
-      if most_sig.empty?
-        total = self.word_frequency.values.count
-        
-        sum = 0
-        self.word_frequency.values.each do |counter|
-          sum += counter
-        end
+      description.to_s.split.each do |word|
+        self.word_frequency[word] ||= 0
+        self.word_frequency[word] += 1
+      end
 
-        avg = (sum.to_f / total.to_f).floor
+      if most_sig.empty?
+        most_sig = self.word_frequency.reject {|k,v| v < 3}.keys
+        most_sig.flatten!
+      end
 
-        self.tags.each do |tag|
-          self.word_frequency[tag] ||= 0
-          self.word_frequency[tag] += (avg * 2.0)
-        end
-        most_sig = self.word_frequency.reject {|k,v| v < (avg * 2.0).floor }.to_a.uniq.sort_by {|_key, v| v}.reverse.collect(&:first).collect(&:downcase)
-        if most_sig.empty?
-          most_sig = self.word_frequency.reject {|k,v| v < (avg * 0.5).floor }.to_a.uniq.sort_by {|_key, v| v}.reverse.collect(&:first).collect(&:downcase)
+      if description && tags.any?
+        tags.each do |tag|
+          if description.include?(tag)
+            most_sig << tag.singularize
+          end
         end
       end
       
@@ -90,8 +89,7 @@ module SiteClassifier
 
       debug("getting #{url}")
       html = Nokogiri::HTML(self.get(url).parsed_response)
-      html.search("//style").remove
-      html.search("//script").remove
+
       tags = []
       description = nil
       word_hash = {}
@@ -146,37 +144,25 @@ module SiteClassifier
         end
       end
 
-      debug("no tags, parsing body")
-      word_hash = Hash.new(0)
-      all_text = []
-      # all_text = html.search("p").collect {|p| p.text.strip }.collect {|text| text.split.collect(&:strip)}.flatten.reject {|word| word.size < 4}
-      # debug("p's extracts - #{all_text.inspect}")
-      if all_text.empty?
-        all_text = html.search("div").collect {|p| p.text.strip }.collect {|text| text.split.collect(&:strip)}.flatten.reject {|word| word.size < 4}
-        debug("divs extracts - #{all_text.inspect}")
-      end
-      all_text += description.to_s.split
+      if tags.empty?
+        debug("no tags, parsing body")
+        word_hash = Hash.new(0)
+        all_text = []
+        # all_text = html.search("p").collect {|p| p.text.strip }.collect {|text| text.split.collect(&:strip)}.flatten.reject {|word| word.size < 4}
+        # debug("p's extracts - #{all_text.inspect}")
+        if all_text.empty?
+          all_text = html.search("div").collect {|p| p.text.strip }.collect {|text| text.split.collect(&:strip)}.flatten.reject {|word| word.size < 4}
+          debug("divs extracts - #{all_text.inspect}")
+        end
+        all_text += description.to_s.split
 
-      all_text.flatten.each do |word|
-        word_hash[word] += 1
+        all_text.flatten.each do |word|
+          word_hash[word] += 1
+        end
+        debug("final word hash - #{word_hash.inspect}")
+        word_hash.reject! {|k,v| v < 3 || k.size == 1 || k.include?(".") || k.include?("'") || k.include?("(") || k.include?(":") || k.include?("]")}
+        
       end
-      debug("final word hash - #{word_hash.inspect}")
-      word_hash.reject! {|k,v| v < 3 || k.size == 1 || k.include?(".") || k.include?("'") || k.include?("(") || k.include?(":") || k.include?("]")}
-
-      # max_score = word_hash.values.max.to_i
-      # dmoz = HTTParty.get("http://www.dmoz.org/search", query: {q: url})
-      # dmoz_res = Nokogiri::HTML(dmoz.parsed_response)
-
-      # dmoz_classification = dmoz_res.search('//*[@id="bd-cross"]//ol[1]//li//strong').collect(&:text)
-      # if dmoz_classification.any?
-      #   dmoz_classification.each do |dmoz_class|
-      #     dmoz_class.split(": ").reject {|v| ["World"].include?(v)}.each do |tag|
-      #       tag.split(" ").each do |plain_tag|
-      #         word_hash[plain_tag] = max_score + 5
-      #       end
-      #     end
-      #   end
-      # end
       self.new(url, tags, word_hash, description, page_lang)
     end
   end

data/lib/site_classifier/version.rb

@@ -1,3 +1,3 @@
 module SiteClassifier
-  VERSION = "0.0.6"
+  VERSION = "0.0.7"
 end

data/site_classifier.gemspec

@@ -21,8 +21,8 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "bundler", "~> 1.3"
   spec.add_development_dependency "rake"
   spec.add_development_dependency "rspec"
-  spec.add_dependency "httparty", "0.11.0"
-  spec.add_dependency "nokogiri", "1.6.0"
-  spec.add_dependency "easy_translate", "0.3.3"
-  spec.add_dependency "activesupport", "4.0.0"
+  spec.add_dependency "httparty"
+  spec.add_dependency "nokogiri"
+  spec.add_dependency "easy_translate"
+  spec.add_dependency "activesupport", "4.2.3"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: site_classifier
 version: !ruby/object:Gem::Version
-  version: 0.0.6
+  version: 0.0.7
 platform: ruby
 authors:
 - Elad Meidar
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-10-02 00:00:00.000000000 Z
+date: 2016-10-31 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -56,58 +56,58 @@ dependencies:
   name: httparty
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '='
+    - - '>='
       - !ruby/object:Gem::Version
-        version: 0.11.0
+        version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '='
+    - - '>='
       - !ruby/object:Gem::Version
-        version: 0.11.0
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: nokogiri
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '='
+    - - '>='
       - !ruby/object:Gem::Version
-        version: 1.6.0
+        version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '='
+    - - '>='
       - !ruby/object:Gem::Version
-        version: 1.6.0
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: easy_translate
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '='
+    - - '>='
       - !ruby/object:Gem::Version
-        version: 0.3.3
+        version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '='
+    - - '>='
       - !ruby/object:Gem::Version
-        version: 0.3.3
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: activesupport
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 4.0.0
+        version: 4.2.3
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 4.0.0
+        version: 4.2.3
 description: Return a tag list for submitted urls
 email:
 - elad@eizesus.com
@@ -150,7 +150,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.0.6
+rubygems_version: 2.4.8
 signing_key: 
 specification_version: 4
 summary: This gem extracts a list of english tags for a given url
@@ -159,3 +159,4 @@ test_files:
 - spec/models/extractor_spec.rb
 - spec/models/site_classifier_spec.rb
 - spec/spec_helper.rb
+has_rdoc: