socialcrawler 0.0.3 → 0.0.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 0f04315836dd27d5ceaf9cc4d34490eb92c40bfc
-  data.tar.gz: e1277912203a7ae9b52a4078e73ba48346cd5e0f
+  metadata.gz: 479202e48815c59eb775fc309eb5f771b27c7c16
+  data.tar.gz: b085817b3960a9397e0379561ecb27e81ea1fcd6
 SHA512:
-  metadata.gz: 5c9b1577dc309c82657e6a083fc5fb529b14cea34719c5e7c70ad1e86a9dc5a83ae76ec72ca8c236eb0aec5e97ea2c06925fc7a31e5eb74ebaea9f52a9499a4f
-  data.tar.gz: c5904e1b4a6fdc076f277cb504eab15e3d64f65f67ebb4caac730aff4f233ddfcce8daafbb0d8e3b8450bd1fbe5534543c757545f9c1adc841292715c9a4836a
+  metadata.gz: 38f539ee81079a7000ca42b6074caf4269db86a41992952d40b243d3f64970fd5319acad8aea8f488b113ff199438a986d8c51a407bc293aa92c8d184f2718a5
+  data.tar.gz: 93806c94207115dbe3ea49f0f6192eb8e9ad36fa853a13c2ff7fbfd247b5e9247fed6a975f1eb53ef3130c637686c0dd4eec7518e50c51f99cbfc595f267af99

data/.travis.yml

@@ -1,9 +1,8 @@
 language: ruby
 
 rvm:
-  - "1.9.3"
-  - jruby-19mode
-  - rbx
+  - "2.3.1"
+  - "2.4.0"
 
 addons:
     climate_control:

data/Gemfile

@@ -7,3 +7,4 @@ gem 'nokogiri', '1.6.4.1'
 # test coverage
 gem 'coveralls', require: false, group: :test
 gem "codeclimate-test-reporter", group: :test, require: nil
+gem 'test-unit'

data/lib/socialcrawler.rb

@@ -28,7 +28,12 @@ module SocialCrawler
       @map = {
           twitter: 'twitter.com/',
           facebook: 'facebook.com/',
-          google_plus: 'plus.google.com/'
+          google_plus: 'plus.google.com/',
+          instagram: 'www.instagram.com',
+          you_tube: 'youtube.com/user',
+          pinterest: 'pinterest.com/',
+          linked_in: 'linkedin.com/',
+          flickr: 'flickr.com/'
       }
     end
 
@@ -57,7 +62,7 @@ module SocialCrawler
     def crawl_url(url, log=nil)
       log = Logger.new(STDOUT) if log.nil?
       log.info("Crawling #{url}")
-      result = Hash.new('NOT FOUND')
+      result = Hash.new(:NOT_FOUND)
       begin
         page = Nokogiri::HTML(open(url))
         title = page.css('title')
@@ -81,13 +86,7 @@ module SocialCrawler
       if not status_filename.nil? and File.exists?(status_filename)
         log.info("Loading previous status from #{status_filename}")
         CSV.foreach(status_filename) do |row|
-          if row.count >= 3
-            status[row[0]] = {
-                :url => row[0],
-                :result => row[1],
-                :message => row[2]
-            }
-          end
+          set_status_cache_data(status, row)
         end
         log.info("Loading previous status from #{status_filename} finished, #{status.keys.length} loaded.")
       end
@@ -101,16 +100,7 @@ module SocialCrawler
         return data
       end
       CSV.foreach(output_list_filename) do |row|
-        log.info("Loading #{row} #{row.count}")
-        if row.count >= 5
-          data[row[0]] = {
-              :url => row[0],
-              :title => row[1],
-              :twitter => row[2],
-              :facebook => row[3],
-              :google_plus => row[4]
-          }
-        end
+        set_output_cache_data(data, row)
         log.info("Loading previous status from #{output_list_filename} finished, #{data.keys.length} loaded.")
       end
       return data
@@ -125,13 +115,9 @@ module SocialCrawler
       data = load_output_cache(output_list_filename, log)
 
       CSV.open(output_list_filename, "wb") do |output|
-        data.each do |k, v|
-          output << [k, v[:title], v[:twitter], v[:facebook], v[:google_plus]]
-        end
+        write_data(data, output)
         CSV.open(status_filename, "wb") do |status_line|
-          status.each do |k, v|
-            status_line << [k, v[:success], v[:message]]
-          end
+          write_status(status, status_line)
           crawl_loop(data, domain_list_filename, log, output, status, status_line)
         end
       end
@@ -151,6 +137,18 @@ module SocialCrawler
 
     private
 
+    def write_data(data, output)
+      data.each do |k, v|
+        output << [k, v[:title], v[:twitter], v[:facebook], v[:google_plus]]
+      end
+    end
+
+    def write_status(status, status_line)
+      status.each do |k, v|
+        status_line << [k, v[:success], v[:message]]
+      end
+    end
+
     def set_data(result, url, data, output)
       if result[:success] == true
         data[url] = result
@@ -166,11 +164,33 @@ module SocialCrawler
       }
       status_line << [url, result[:success], result[:message]]
     end
+
+    def set_output_cache_data(data, row)
+      if row.count >= 5
+        data[row[0]] = {
+            :url => row[0],
+            :title => row[1],
+            :twitter => row[2],
+            :facebook => row[3],
+            :google_plus => row[4]
+        }
+      end
+    end
+
+    def set_status_cache_data(status, row)
+      if row.count >= 3
+        status[row[0]] = {
+            :url => row[0],
+            :result => row[1],
+            :message => row[2]
+        }
+      end
+    end
   end
 end
 
 if __FILE__ == $0
-  # :nocov:
+  #:nocov:
   SocialCrawler::SocialCrawler.new.crawl(ARGV[0], ARGV[1], ARGV[2])
-  # :nocov:
+  #:nocov:
 end

data/lib/socialcrawler/version.rb

@@ -1,3 +1,3 @@
 module SocialCrawler
-  VERSION = "0.0.3"
+  VERSION = "0.0.4"
 end

data/test/test_helper.rb

@@ -1,11 +1,9 @@
-require "codeclimate-test-reporter"
 require 'simplecov'
 require 'coveralls'
 SimpleCov.formatter = SimpleCov::Formatter::MultiFormatter[
     SimpleCov::Formatter::HTMLFormatter,
     Coveralls::SimpleCov::Formatter
 ]
-CodeClimate::TestReporter.start
 SimpleCov.start
 puts "Simple Coverage Started"
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: socialcrawler
 version: !ruby/object:Gem::Version
-  version: 0.0.3
+  version: 0.0.4
 platform: ruby
 authors:
 - Ivica Ceraj
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-01-26 00:00:00.000000000 Z
+date: 2017-02-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -101,7 +101,6 @@ files:
 - test/test_crawler.rb
 - test/test_helper.rb
 - test/test_url.txt
-- test_status.txt
 homepage: http://github.com/iceraj/socialcrawler
 licenses:
 - LGPL 2.1
@@ -122,7 +121,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.5
+rubygems_version: 2.5.1
 signing_key: 
 specification_version: 4
 summary: SocialCrawler looks for social media links for different sites

data/test_status.txt

@@ -1,4 +0,0 @@
-https://twitter.com/bugaco,,""
-https://plus.google.com/101033631762132540828/posts,,""
-https://www.facebook.com/,,""
-https://localhost/,,"Connection refused - connect(2) for ""localhost"" port 443"