site_classifier 0.0.2

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 43547a8a99af0382b4fea86385db18428fe267ac
+  data.tar.gz: fa4321d7f5ecb4bdd1c415c5c9e179b1418306b5
+SHA512:
+  metadata.gz: f1107cddc78b414c551065976f53e6a25fee56cce3dcc2d9c5245485bad06fdfd099712fd20196aa20e6e041b92fa77c667c924e24a56d8d4ce61721554e6328
+  data.tar.gz: 6914f5718f3fbd9c97bdd28f7b794f564f5a4f44563c26eb7f0e65df11c5b959a11a71d7cd220af402a999a4f7166cedf647954b26aa0b297c838f7797f2af3f

data/.gitignore

@@ -0,0 +1,17 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp

data/.rspec

@@ -0,0 +1,2 @@
+--color
+--format documentation

data/Gemfile

@@ -0,0 +1,5 @@
+source 'https://rubygems.org'
+
+ruby '2.0.0'
+# Specify your gem's dependencies in site_classifier.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2013 Elad Meidar
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,44 @@
+# SiteClassifier
+
+TODO: Write a gem description
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'site_classifier'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install site_classifier
+
+## Configuration
+
+    SiteClassifier.configure do |classifier|
+      classifier.translate = true
+      classifier.google_translate_api_key = "XXX"
+    end
+
+    - translate (true / false) - indicates if tags should be translated to english. if `false` tags will appear in url's native language.
+    - google_translate_api_key (string) - if translate is `true`, supply a Google Translate API key to allow translation.
+
+## Usage
+
+    SiteClassifier::Extractor.parse_site("http://cnn.com") 
+    #=> {:most_significant=>["cnn", "news", "breaking news", "busines", "sport", "entertainment", "special report"], 
+        :language=>"auto", 
+        :url=>"http://cnn.com", 
+        :tags=>["cnn", "cnn news", "cnn international", "cnn international news", "cnn edition", "edition news", "news", "news online", "breaking news", "u.s. news", "world news", "global news", "weather", "business", "cnn money", "sports", "politics", "law", "technology", "entertainment", "education", "travel", "health", "special reports", "autos", "developing story", "news video", "cnn intl", "podcasts", "world blogs"], 
+        :description=>"CNN.com International delivers breaking news from across the globe and information on the latest top stories, business, sports and entertainment headlines. Follow the news as it happens through: special reports, videos, audio, photo galleries plus interactive maps and timelines."}
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/Rakefile

@@ -0,0 +1 @@
+require "bundler/gem_tasks"

data/lib/site_classifier/configuration.rb

@@ -0,0 +1,18 @@
+module SiteClassifier
+  class Configuration
+    attr_accessor :translate, :google_translate_api_key
+      
+    # Instantiate a new class
+    def initialize(options = {})
+      @translate = options[:translate] || false
+      @google_translate_api_key = options[:google_translate_api_key]
+    end
+
+    # Configure by block
+    def self.configure(&block)
+      new_configuration = SiteClassifier::Configuration.new
+      yield new_configuration
+      new_configuration
+    end
+  end
+end

data/lib/site_classifier/extractor.rb

@@ -0,0 +1,105 @@
+module SiteClassifier
+  class Extractor
+    include HTTParty
+
+    attr_accessor :url, :tags, :description, :word_frequency, :lang
+
+    def initialize(url, tags, word_hash, description, lang)
+      @url = url
+      @tags = tags
+      @description = description
+      @word_frequency = word_hash
+      @lang = lang.downcase
+    end
+
+    # Normalize site language
+    def validate_lang
+      if EasyTranslate::LANGUAGES.keys.include?(@lang)
+        @lang
+      else
+        self.lang = "auto"
+      end
+    end
+
+    # Extract most significant tags
+    def most_significant
+      most_sig = []
+      if !description.nil?
+        if tags.any?
+          most_sig = tags.select {|tag| self.description.downcase.include?(tag)}.collect {|tag| tag.singularize }
+        else
+          most_sig = word_frequency.keys.select {|tag| self.description.downcase.include?(tag)}.collect {|tag| tag.singularize }
+        end
+      end
+
+      if most_sig.empty?
+        most_sig = self.word_frequency.keys
+      end
+
+      self.validate_lang
+
+      if SiteClassifier.translate_tags?
+        begin
+          if self.lang == "auto"
+            @lang = EasyTranslate.detect(most_sig.first, key: SiteClassifier.configuration.google_translate_api_key)
+          end
+          EasyTranslate.translate(most_sig, from: self.lang, to: :en, key: SiteClassifier.configuration.google_translate_api_key)
+        rescue
+          return most_sig
+        end
+      else
+        return most_sig
+      end
+    end
+
+    def to_hash
+      {
+        most_significant: most_significant,
+        language: self.lang,
+        url: url,
+        tags: tags,
+        description: description
+      }
+    end
+
+    def self.parse_site(url = "")
+      return if url == "" || url.nil?
+
+      html = Nokogiri::HTML(self.get(url).parsed_response)
+
+      tags = []
+      description = nil
+      word_hash = {}
+      page_lang = "auto"
+
+      begin
+        page_lang = html.search("html").first["lang"].to_s.slice(0..1)
+      rescue
+      end
+
+      begin
+        page_lang = html.search("html").first["xml:lang"].to_s.slice(0..1)
+      rescue
+      end
+
+      begin
+        tags = html.search('meta[name="keywords"]').first["content"].split(",").collect(&:strip).collect(&:downcase)
+        description = html.search('meta[name="description"]').first["content"]
+      rescue
+      end
+
+      if tags.empty?
+        word_hash = Hash.new(0)
+        all_text = html.search("p").collect {|p| p.text.strip }.collect {|text| text.split.collect(&:strip)}.flatten.reject {|word| word.size < 4}
+        if all_text.empty?
+          all_text = html.search("div").collect {|p| p.text.strip }.collect {|text| text.split.collect(&:strip)}.flatten.reject {|word| word.size < 4}
+        end
+        all_text.each do |word|
+          word_hash[word] += 1
+        end
+        word_hash.reject! {|k,v| v < 2 }
+      end
+      self.new(url, tags, word_hash, description, page_lang)
+    end
+  end
+end

data/lib/site_classifier/version.rb

@@ -0,0 +1,3 @@
+module SiteClassifier
+  VERSION = "0.0.2"
+end

data/lib/site_classifier.rb

@@ -0,0 +1,25 @@
+require "site_classifier/version"
+require 'httparty'
+require 'easy_translate'
+require 'nokogiri'
+require 'active_support/inflector'
+
+module SiteClassifier
+
+  autoload :Configuration, 'site_classifier/configuration'
+  autoload :Extractor, 'site_classifier/extractor'
+
+  attr_reader :setup
+
+  def self.configure(&block)
+    @setup = SiteClassifier::Configuration.configure(&block)
+  end
+
+  def self.translate_tags?
+    self.configuration.translate == true
+  end
+
+  def self.configuration
+    @setup ||= SiteClassifier::Configuration.new
+  end
+end

data/site_classifier.gemspec

@@ -0,0 +1,28 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'site_classifier/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "site_classifier"
+  spec.version       = SiteClassifier::VERSION
+  spec.authors       = ["Elad Meidar"]
+  spec.email         = ["elad@eizesus.com"]
+  spec.description   = "Return a tag list for submitted urls"
+  spec.summary       = "This gem extracts a list of english tags for a given url"
+  spec.homepage      = "https://github.com/ShinobiDevs/SiteClassifier"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files`.split($/)
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.3"
+  spec.add_development_dependency "rake"
+  spec.add_development_dependency "rspec"
+  spec.add_dependency "httparty", "0.11.0"
+  spec.add_dependency "nokogiri", "1.6.0"
+  spec.add_dependency "easy_translate", "0.3.3"
+  spec.add_dependency "active_support", "3.0.0"
+end

data/spec/models/configuration_spec.rb

@@ -0,0 +1,23 @@
+require 'spec_helper'
+
+describe SiteClassifier::Configuration do
+  describe "#initialize" do
+    it "should access an option hash" do
+      conf = SiteClassifier::Configuration.new({translate: true, google_translate_api_key: "xxx"})
+      conf.translate.should be_true
+      conf.google_translate_api_key.should eq("xxx")
+    end
+  end
+
+  describe "#configure" do
+    it "should return a valid instance with a configuration block" do
+      conf = SiteClassifier::Configuration.configure do |conf|
+        conf.translate = true
+        conf.google_translate_api_key = "yyy"
+      end
+
+      conf.translate.should be_true
+      conf.google_translate_api_key.should eq("yyy")
+    end
+  end
+end

data/spec/models/extractor_spec.rb

@@ -0,0 +1,50 @@
+require 'spec_helper'
+
+describe SiteClassifier::Extractor do
+  describe "#initialize" do
+    it "should create a valid instance" do
+      extractor = SiteClassifier::Extractor.new("http://cnn.com", ["news"], ["news", "economy"], "This is cnn", "auto")
+
+      extractor.url.should eq("http://cnn.com")
+    end
+  end
+
+  describe "#validate_lang" do
+    before(:all) do
+      @extractor = SiteClassifier::Extractor.new("http://cnn.com", ["news"], ["news", "economy"], "This is cnn", "auto")
+    end
+
+    it "should keep language if it is a known language to google translate" do
+      @extractor.validate_lang
+      @extractor.lang.should eq("auto")
+    end
+
+    it "should reset to 'auto' if language is not a known language to google translate" do
+      @extractor.lang = "bullshit-language"
+      @extractor.validate_lang
+      @extractor.lang.should eq("auto")
+    end
+  end
+
+  describe "#most_significant" do
+    before(:each) {
+      SiteClassifier.configuration.translate = false
+      @extractor = SiteClassifier::Extractor.new("http://cnn.com", ["news"], {"news" => 10, "economy" => 5, "text" => 3, "elad" => 1, "miki" => 1}, "This is cnn news", "auto")
+    }
+
+    it "should return a list of the most common tags that exist in description" do
+      @extractor.most_significant.should eq(["news"])
+    end
+
+    it "should return a list of most frequest used words and exist in description if tags are empty and description exists" do
+      @extractor.tags = []
+      @extractor.most_significant.should eq(["news"])
+    end
+
+    it "should return a list of most frequest used words if tags are empty and description missing" do
+      @extractor.tags = []
+      @extractor.description = nil
+      @extractor.most_significant.should eq(["news", "economy", "text", "elad", "miki"])
+    end
+  end
+end

data/spec/models/site_classifier_spec.rb

@@ -0,0 +1,43 @@
+require 'spec_helper'
+
+describe SiteClassifier do
+
+  describe '#configure' do
+    it "should have a 'configure' method" do
+      SiteClassifier.should respond_to(:configure)
+    end
+
+    it "should accept block configuration" do
+      SiteClassifier.configure do |sc|
+        sc.translate = true
+        sc.google_translate_api_key = "xxx"
+      end
+
+      SiteClassifier.configuration.translate.should be_true
+      SiteClassifier.configuration.google_translate_api_key.should eq("xxx")
+    end
+  end
+
+  describe '#configuration' do
+    it "should have a 'configuration' method" do
+      SiteClassifier.should respond_to(:configuration)
+    end
+  end
+
+  describe "#translate_tags?" do
+    before(:all) do
+      SiteClassifier.configure do |sc|
+        sc.translate = true
+        sc.google_translate_api_key = "xxx"
+      end
+    end
+
+    it "should have a 'translate_tags?' method" do
+      SiteClassifier.should respond_to(:translate_tags?)
+    end
+
+    it "should return true if translate is set to true in configuration" do
+      SiteClassifier.translate_tags?.should be_true
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1 @@
+require 'site_classifier'

metadata

@@ -0,0 +1,161 @@
+--- !ruby/object:Gem::Specification
+name: site_classifier
+version: !ruby/object:Gem::Version
+  version: 0.0.2
+platform: ruby
+authors:
+- Elad Meidar
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-09-29 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.3'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: httparty
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.11.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.11.0
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 1.6.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 1.6.0
+- !ruby/object:Gem::Dependency
+  name: easy_translate
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.3.3
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.3.3
+- !ruby/object:Gem::Dependency
+  name: active_support
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 3.0.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 3.0.0
+description: Return a tag list for submitted urls
+email:
+- elad@eizesus.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- .rspec
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- lib/site_classifier.rb
+- lib/site_classifier/configuration.rb
+- lib/site_classifier/extractor.rb
+- lib/site_classifier/version.rb
+- site_classifier.gemspec
+- spec/models/configuration_spec.rb
+- spec/models/extractor_spec.rb
+- spec/models/site_classifier_spec.rb
+- spec/spec_helper.rb
+homepage: https://github.com/ShinobiDevs/SiteClassifier
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.0.6
+signing_key: 
+specification_version: 4
+summary: This gem extracts a list of english tags for a given url
+test_files:
+- spec/models/configuration_spec.rb
+- spec/models/extractor_spec.rb
+- spec/models/site_classifier_spec.rb
+- spec/spec_helper.rb