socialcrawler 0.0.0 → 0.0.2

data/.coveralls.yml

@@ -0,0 +1 @@
+service_name: travis-ci

data/.travis.yml

@@ -0,0 +1,10 @@
+language: ruby
+
+rvm:
+  - "1.9.3"
+  - jruby-19mode
+  - rbx
+
+addons:
+    climate_control:
+        repo_token: c5d1031f512ff1ec104782e87ba1514dff71d8a564b928076e54bb67226552df

data/Gemfile

@@ -3,4 +3,7 @@ source 'https://rubygems.org'
 # Specify your gem's dependencies in SocialCrawler.gemspec
 gemspec
 
-gem 'nokogiri', '~> 1.6.0'
+gem 'nokogiri', '1.6.4.1'
+# test coverage
+gem 'coveralls', require: false, group: :test
+gem "codeclimate-test-reporter", group: :test, require: nil

data/README.md

@@ -1,6 +1,10 @@
+[![Coverage Status](https://img.shields.io/coveralls/iceraj/socialcrawler.svg)](https://coveralls.io/r/iceraj/socialcrawler)
+[![Build Status](https://travis-ci.org/iceraj/socialcrawler.svg?branch=feature%2FIntial_Development)](https://travis-ci.org/iceraj/socialcrawler)
+[![Code Climate](https://codeclimate.com/github/iceraj/socialcrawler/badges/gpa.svg)](https://codeclimate.com/github/iceraj/socialcrawler)
+
 # Socialcrawler
 
-TODO: Write a gem description
+SocialCrawler looks for social media links for different sites
 
 ## Installation
 
@@ -24,7 +28,7 @@ TODO: Write usage instructions here
 
 ## Contributing
 
-1. Fork it ( https://github.com/[my-github-username]/socialcrawler/fork )
+1. Fork it ( https://github.com/iceraj/socialcrawler/fork )
 2. Create your feature branch (`git checkout -b my-new-feature`)
 3. Commit your changes (`git commit -am 'Add some feature'`)
 4. Push to the branch (`git push origin my-new-feature`)

data/Rakefile

@@ -1,2 +1,9 @@
 require "bundler/gem_tasks"
+require 'rake/testtask'
 
+Rake::TestTask.new do |t|
+  t.libs << 'test'
+end
+
+desc "Run tests"
+task :default => :test

data/lib/socialcrawler.rb

@@ -22,139 +22,155 @@ require 'logger'
 
 module SocialCrawler
 
+  class SocialCrawler
 
-  def self._put( hash, symbol , value , log=nil)
-    log = Logger.new(STDOUT) if log.nil?
-    if not hash.has_key?( symbol)
-      hash[symbol] = value
-    else
-      hash[symbol] = "#{hash[symbol]} #{value}"
-      log.info( "Multiple values for #{symbol} value #{hash[symbol]}")
+    def initialize
+      @map = {
+          twitter: 'twitter.com/',
+          facebook: 'facebook.com/',
+          google_plus: 'plus.google.com/'
+      }
     end
-  end
 
-  def self.crawl_url(url,log=nil)
-    log = Logger.new(STDOUT) if log.nil?
-    log.info( "Crawling #{url}")
-    result = Hash.new('NOT FOUND')
-    begin
-      page = Nokogiri::HTML(open(url))
-      title = page.css('title')
-      if not title.nil?
-        result[:title] = title.text.strip
+    def _put(hash, symbol, value, log=nil)
+      log = Logger.new(STDOUT) if log.nil?
+      if not hash.has_key?(symbol)
+        hash[symbol] = value
+      else
+        hash[symbol] = "#{hash[symbol]} #{value}"
+        log.info("Multiple values for #{symbol} value #{hash[symbol]}")
       end
+    end
+
+    def page_to_result(page, result, log)
       links = page.css('a[href]')
       links.each do |link|
         link_url = link['href']
-
-        if not link_url.index('twitter.com/').nil?
-          log.info( "twitter #{link_url} for #{url}")
-          _put(result,:twitter,link_url,log)
+        @map.each do |k, prefix|
+          if not link_url.index(prefix).nil?
+            _put(result, k, link_url, log)
+          end
         end
-        if not link_url.index('facebook.com/').nil?
-          log.info( "facebook #{link_url} for #{url}")
-          _put(result,:facebook,link_url,log)
+      end
+    end
+
+    def crawl_url(url, log=nil)
+      log = Logger.new(STDOUT) if log.nil?
+      log.info("Crawling #{url}")
+      result = Hash.new('NOT FOUND')
+      begin
+        page = Nokogiri::HTML(open(url))
+        title = page.css('title')
+        if not title.nil?
+          result[:title] = title.text.strip
         end
-        if not link_url.index('plus.google.com/').nil?
-          log.info( "google_plus #{link_url} for #{url}")
-          _put(result,:google_plus,link_url,log)
+        page_to_result(page, result, log)
+        result[:url] = url
+        result[:success] = true
+        result[:message] = ''
+      rescue Exception => e
+        result[:url] = url
+        result[:success] = false
+        result[:message] = "#{e}"
+      end
+      return result
+    end
+
+    def load_status_cache(status_filename, log=nil)
+      status = Hash.new
+      if not status_filename.nil? and File.exists?(status_filename)
+        log.info("Loading previous status from #{status_filename}")
+        CSV.foreach(status_filename) do |row|
+          if row.count < 3
+            next
+          end
+          url = row[0]
+          result = row[1]
+          message = row[2]
+          status[url] = {
+              :url => url,
+              :result => result,
+              :message => message
+          }
         end
+        log.info("Loading previous status from #{status_filename} finished, #{status.keys.length} loaded.")
       end
-      result[:url] = url
-      result[:success] = true
-      result[:message] = ''
-    rescue Exception => e
-      result[:url] = url
-      result[:success] = false
-      result[:message] = "#{e}"
+      return status
     end
-    return result
-  end
 
-  def self.crawl( domain_list_filename, output_list_filename, status_filename=nil , log=nil)
-    log = Logger.new(STDOUT) if log.nil?
-    log.info( "Crawler started")
-    status = Hash.new
-    if not status_filename.nil? and File.exists?(status_filename)
-      log.info( "Loading previous status from #{status_filename}")
-      CSV.foreach( status_filename ) do |row|
-        begin
+    def load_output_cache(output_list_filename, log=nil)
+      data = Hash.new()
+      log.info("Loading previous status from #{output_list_filename}")
+      if not File.exist?(output_list_filename)
+        return data
+      end
+      CSV.foreach(output_list_filename) do |row|
+        log.info("Loading #{row} #{row.count}")
+        if row.count < 5
+          next
+        end
         url = row[0]
-        result = row[1]
-        message = row[2]
-        status[url] = {
+        title= row[1]
+        twitter = row[2]
+        facebook = row[3]
+        google_plus = row[4]
+        data[url] = {
             :url => url,
-            :result => result,
-            :message => message
+            :title => title,
+            :twitter => twitter,
+            :facebook => facebook,
+            :google_plus => google_plus
         }
-        rescue Exception => e
-          log.info("Exception reading file #{e}")
-        end
+        log.info("Loading previous status from #{output_list_filename} finished, #{data.keys.length} loaded.")
       end
-      log.info( "Loading previous status from #{status_filename} finished, #{status.keys.length} loaded.")
+      return data
     end
 
-    data = Hash.new()
-    if File.exist?(output_list_filename)
-      log.info( "Loading previous status from #{output_list_filename}")
-      CSV.open( output_list_filename ) do |row|
-          if row.count >= 5
-            url = row[0]
-            title= row[1]
-            twitter = row[2]
-            facebook = row[3]
-            google_plus = row[4]
-            data[url] = {
-                :url => url,
-                :title => title,
-                :twitter => twitter,
-                :facebook => facebook,
-                :google_plus => google_plus
-            }
-            end
+    def crawl(domain_list_filename, output_list_filename, status_filename=nil, log=nil)
+      log = Logger.new(STDOUT) if log.nil?
+      log.info("Crawler started")
+
+      status = load_status_cache(status_filename, log)
+
+      data = load_output_cache(output_list_filename, log)
+
+      CSV.open(output_list_filename, "wb") do |output|
+        data.each do |k, v|
+          output << [k, v[:title], v[:twitter], v[:facebook], v[:google_plus]]
+        end
+        CSV.open(status_filename, "wb") do |status_line|
+          status.each do |k, v|
+            status_line << [k, v[:success], v[:message]]
+          end
+          crawl_loop(data, domain_list_filename, log, output, status, status_line)
+        end
       end
-      log.info( "Loading previous status from #{output_list_filename} finished, #{data.keys.length} loaded.")
     end
 
-      CSV.foreach( domain_list_filename ) do |row|
+    def crawl_loop(data, domain_list_filename, log, output, status, status_line)
+      CSV.foreach(domain_list_filename) do |row|
         url = row[0]
         if status.has_key?(url)
-          # already visited, skip
-        else
-          result = crawl_url(url,log)
-          if result[:success] == true
-            CSV.open( output_list_filename, "wb") do |output|
-              data.each do |k,v|
-                log.info(k)
-                log.info(v)
-                  output << [k,v[:title],v[:twitter],v[:facebook],v[:google_plus]]
-              end
-              output << [url, result[:title], result[:twitter], result[:facebook], result[:google_plus]]
-              data[url] = result
-            end
-            status[url] = {
-                :url => url,
-                :result => 'success',
-                :message => ''
-            }
-            CSV.open( status_filename, "wb" ) do |status_line|
-              status_line << [url,'success','']
-            end
-          else
-            status[url] = {
-                :url => url,
-                :result => result[:success],
-                :message => result[:message]
-            }
-            CSV.open( status_filename, "wb" ) do |status_line|
-              status_line << [url,result[:success],result[:message]]
-            end
+          next
+        end
+        result = crawl_url(url, log)
+        if result[:success] == true
+          data[url] = result
+          output << [url, result[:title], result[:twitter], result[:facebook], result[:google_plus]]
         end
+        status[url] = {
+            :url => url,
+            :result => result[:success],
+            :message => result[:message]
+        }
+        status_line << [url, result[:success], result[:message]]
       end
     end
   end
 end
 
 if __FILE__ == $0
-  SocialCrawler.crawl(ARGV[0],ARGV[1],ARGV[2])
+  #:nocov:
+  SocialCrawler::SocialCrawler.new.crawl(ARGV[0], ARGV[1], ARGV[2])
+  #:nocov:
 end

data/lib/socialcrawler/version.rb

@@ -1,3 +1,3 @@
 module SocialCrawler
-  VERSION = "0.0.0"
+  VERSION = "0.0.2"
 end

data/test/test_crawler.rb

@@ -0,0 +1,25 @@
+require 'test_helper'
+require "test/unit"
+require 'semantic'
+require 'socialcrawler'
+
+class CrawlewrTest < Test::Unit::TestCase
+
+  def test_version
+    s = SocialCrawler::VERSION
+    v = Semantic::Version.new(s)
+    assert_equal v.to_s, s
+  end
+
+  def test_1
+    File.delete('/tmp/test_out.txt') if File.exists?('/tmp/test_out.txt')
+    File.delete('/tmp/test_status.txt') if File.exists?('/tmp/test_status.txt')
+
+    sc = SocialCrawler::SocialCrawler.new
+    sc.crawl('test/test_url.txt', '/tmp/test_out.txt', '/tmp/test_status.txt')
+
+    sc = SocialCrawler::SocialCrawler.new
+    sc.crawl('test/test_url.txt', '/tmp/test_out.txt', '/tmp/test_status.txt')
+   end
+
+end

data/test/test_helper.rb

@@ -1,2 +1,12 @@
+require "codeclimate-test-reporter"
 require 'simplecov'
-SimpleCov.start
+require 'coveralls'
+SimpleCov.formatter = SimpleCov::Formatter::MultiFormatter[
+    SimpleCov::Formatter::HTMLFormatter,
+    Coveralls::SimpleCov::Formatter
+]
+CodeClimate::TestReporter.start
+SimpleCov.start
+puts "Simple Coverage Started"
+
+

data/test/test_url.txt

@@ -0,0 +1,4 @@
+https://twitter.com/bugaco
+https://plus.google.com/101033631762132540828/posts
+https://www.facebook.com/
+https://localhost/

metadata

@@ -1,83 +1,94 @@
 --- !ruby/object:Gem::Specification
 name: socialcrawler
 version: !ruby/object:Gem::Version
-  version: 0.0.0
+  version: 0.0.2
+  prerelease: 
 platform: ruby
 authors:
 - Ivica Ceraj
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-01-12 00:00:00.000000000 Z
+date: 2015-01-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: '1.7'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: '1.7'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: '10.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: '10.0'
 - !ruby/object:Gem::Dependency
   name: semantic
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: '1.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: '1.0'
 - !ruby/object:Gem::Dependency
   name: simplecov
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: '0.9'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: '0.9'
 - !ruby/object:Gem::Dependency
   name: simplecov-html
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: '0.8'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: '0.8'
 description: It read file containing list of urls and produces output file with domain,
@@ -88,7 +99,9 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- ".gitignore"
+- .coveralls.yml
+- .gitignore
+- .travis.yml
 - Gemfile
 - LICENSE.txt
 - README.md
@@ -96,32 +109,35 @@ files:
 - lib/socialcrawler.rb
 - lib/socialcrawler/version.rb
 - socialcrawler.gemspec
+- test/test_crawler.rb
 - test/test_helper.rb
-- test/unit/semver_test.rb
+- test/test_url.txt
 homepage: http://github.com/iceraj/socialcrawler
 licenses:
 - LGPL 2.1
-metadata: {}
 post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
   requirements:
-  - - ">="
+  - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
   requirements:
-  - - ">="
+  - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.5
+rubygems_version: 1.8.24
 signing_key: 
-specification_version: 4
+specification_version: 3
 summary: SocialCrawler looks for social media links for different sites
 test_files:
+- test/test_crawler.rb
 - test/test_helper.rb
-- test/unit/semver_test.rb
+- test/test_url.txt

checksums.yaml

@@ -1,7 +0,0 @@
----
-SHA1:
-  metadata.gz: 1bfd2a9261b07fb456185b808c9f532128f0fcdc
-  data.tar.gz: 42f27c1214c3b8e850a6b9106bc0002a0f76d862
-SHA512:
-  metadata.gz: 21878df5ce6b2c75e3af24e4f2b950076098af255c88e900d639e3852eedb21587d485c83271df3b08bd2d6a2f363d2faf8e9052b3d497431f46b7f195433de3
-  data.tar.gz: e59148b275a58ec7ad5b692c25f030e94ce2ed585b5b69010d39df19fb17b112a62c65204a59a8e2fafa23485881d67d9cc7027a82dbb00192664ed0056170fd

data/test/unit/semver_test.rb

@@ -1,14 +0,0 @@
-require "test/unit"
-require 'semantic'
-require 'socialcrawler'
-require 'test_helper'
-
-class VersioningTest < Test::Unit::TestCase
-
-  def test_version
-    s = SocialCrawler::VERSION
-    v = Semantic::Version.new(s)
-    assert_equal v.to_s, s
-  end
-
-end