site_classifier 0.0.4 → 0.0.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 532c257b74d762bf2bad6c07255e87a5c6e924dc
-  data.tar.gz: 760547db13f4b9c75db9085b778c276fa184efe3
+  metadata.gz: 1a6185e3f8b41f26e38ba1787d9a73ec864960ae
+  data.tar.gz: d53cf863639a4841920e85ed1d68c1e1a6367c2e
 SHA512:
-  metadata.gz: 760921ace478383eee08c03b06b041066d6b60bea86406380b75dd6379445b4ed58ebdda765ebdda16fbcc3f4dee8ff600b7cbd09309b9a86877ad7b8b61ad5d
-  data.tar.gz: 3084a269a2416a40c472befdf20dc7f2b11f5a0554f2f7f2f08256c37448a54159bb8e575be5d3dd84daa5cc779d074bcb765ae9719172df5b12a3664e282651
+  metadata.gz: 5b1e6a8e583c46f23dfdf35c2ce8448ca7539154775acd905a9cf2b212e3a6136b485c6b806ba2dae57ab8727e546b31c82f7cd2e7c879c8d23e002e8ba4f90c
+  data.tar.gz: e7ff3a1003a2baecf41486efcab6768c3ef5ee41fc24a558bdbdaadf65a46da6bbe31f121235ab1851934831937130f1e3d3b09f3fa4ea1f4d2c1ec91e55ee9e

data/lib/site_classifier/configuration.rb

@@ -1,13 +1,18 @@
 module SiteClassifier
   class Configuration
-    attr_accessor :translate, :google_translate_api_key
+    attr_accessor :translate, :google_translate_api_key, :debug
       
     # Instantiate a new class
     def initialize(options = {})
       @translate = options[:translate] || false
       @google_translate_api_key = options[:google_translate_api_key]
+      @debug = options[:debug] || false
     end
 
+    def debug?
+      self.debug == true
+    end
+    
     # Configure by block
     def self.configure(&block)
       new_configuration = SiteClassifier::Configuration.new

data/lib/site_classifier/extractor.rb

@@ -12,6 +12,12 @@ module SiteClassifier
       @lang = lang.downcase
     end
 
+    def self.debug(string)
+      if SiteClassifier.configuration.debug?
+        puts "#{Time.now.to_i} - #{string}"
+      end
+    end
+
     # Normalize site language
     def validate_lang
       if EasyTranslate::LANGUAGES.keys.include?(@lang)
@@ -24,18 +30,34 @@ module SiteClassifier
     # Extract most significant tags
     def most_significant
       most_sig = []
-      if !description.nil?
-        if tags.any?
-          most_sig = tags.select {|tag| self.description.downcase.include?(tag)}.collect {|tag| tag.singularize }
-        else
-          most_sig = word_frequency.keys.select {|tag| self.description.downcase.include?(tag)}.collect {|tag| tag.singularize }
-        end
+      # if !description.nil?
+      #   if tags.any?
+      #     most_sig = tags.select {|tag| self.description.downcase.include?(tag)}.collect {|tag| tag.singularize }
+      #   else
+      #     most_sig = word_frequency.keys.select {|tag| self.description.downcase.include?(tag)}.collect {|tag| tag.singularize }
+      #   end
+      # end
+
+      description.to_s.split.each do |word|
+        self.word_frequency[word] ||= 0
+        self.word_frequency[word] += 1
       end
 
       if most_sig.empty?
-        most_sig = self.word_frequency.keys
+        most_sig = self.word_frequency.reject {|k,v| v < 3}.keys
+        most_sig.flatten!
       end
 
+      if description && tags.any?
+        tags.each do |tag|
+          if description.include?(tag)
+            most_sig << tag.singularize
+          end
+        end
+      end
+      
+      most_sig.uniq!
+
       self.validate_lang
 
       if SiteClassifier.translate_tags?
@@ -65,6 +87,7 @@ module SiteClassifier
     def self.parse_site(url = "")
       return if url == "" || url.nil?
 
+      debug("getting #{url}")
       html = Nokogiri::HTML(self.get(url).parsed_response)
 
       tags = []
@@ -74,30 +97,71 @@ module SiteClassifier
 
       begin
         page_lang = html.search("html").first["lang"].to_s.slice(0..1)
+        debug("found lang in html tag - #{page_lang}")
       rescue
       end
 
       begin
         page_lang = html.search("html").first["xml:lang"].to_s.slice(0..1)
+        debug("found lang in html tag (xml:lang) - #{page_lang}")
       rescue
       end
 
       begin
         tags = html.search('meta[name="keywords"]').first["content"].split(",").collect(&:strip).collect(&:downcase)
+        debug("Tags - #{tags.inspect}")
+      rescue
+        debug("no tags found")
+      end
+
+      if tags.empty?
+        begin
+          tags = html.search('meta[property="keywords"]').first["content"].split(",").collect(&:strip).collect(&:downcase)
+          debug("Tags - #{tags.inspect}")
+        rescue
+          debug("no tags found")
+        end
+      end
+      begin
         description = html.search('meta[name="description"]').first["content"]
+        debug("Decription meta found")
       rescue
       end
 
+      if description.nil?
+        begin
+          description = html.search('meta[property="og:description"]').first["content"]
+          debug("Facebook og:description found")
+        rescue
+        end
+      end
+
+      if description.nil?
+        begin
+          description = html.search('meta[name="og:description"]').first["content"]
+          debug("Facebook og:description found")
+        rescue
+        end
+      end
+
       if tags.empty?
+        debug("no tags, parsing body")
         word_hash = Hash.new(0)
-        all_text = html.search("p").collect {|p| p.text.strip }.collect {|text| text.split.collect(&:strip)}.flatten.reject {|word| word.size < 4}
+        all_text = []
+        # all_text = html.search("p").collect {|p| p.text.strip }.collect {|text| text.split.collect(&:strip)}.flatten.reject {|word| word.size < 4}
+        # debug("p's extracts - #{all_text.inspect}")
         if all_text.empty?
           all_text = html.search("div").collect {|p| p.text.strip }.collect {|text| text.split.collect(&:strip)}.flatten.reject {|word| word.size < 4}
+          debug("divs extracts - #{all_text.inspect}")
         end
-        all_text.each do |word|
+        all_text += description.to_s.split
+
+        all_text.flatten.each do |word|
           word_hash[word] += 1
         end
-        word_hash.reject! {|k,v| v < 2 }
+        debug("final word hash - #{word_hash.inspect}")
+        word_hash.reject! {|k,v| v < 3 || k.size == 1 || k.include?(".") || k.include?("'") || k.include?("(") || k.include?(":") || k.include?("]")}
+        
       end
       self.new(url, tags, word_hash, description, page_lang)
     end

data/lib/site_classifier/version.rb

@@ -1,3 +1,3 @@
 module SiteClassifier
-  VERSION = "0.0.4"
+  VERSION = "0.0.5"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: site_classifier
 version: !ruby/object:Gem::Version
-  version: 0.0.4
+  version: 0.0.5
 platform: ruby
 authors:
 - Elad Meidar