socialcrawler 0.0.2 → 0.0.3

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 0f04315836dd27d5ceaf9cc4d34490eb92c40bfc
+  data.tar.gz: e1277912203a7ae9b52a4078e73ba48346cd5e0f
+SHA512:
+  metadata.gz: 5c9b1577dc309c82657e6a083fc5fb529b14cea34719c5e7c70ad1e86a9dc5a83ae76ec72ca8c236eb0aec5e97ea2c06925fc7a31e5eb74ebaea9f52a9499a4f
+  data.tar.gz: c5904e1b4a6fdc076f277cb504eab15e3d64f65f67ebb4caac730aff4f233ddfcce8daafbb0d8e3b8450bd1fbe5534543c757545f9c1adc841292715c9a4836a

data/lib/socialcrawler/version.rb

@@ -1,3 +1,3 @@
 module SocialCrawler
-  VERSION = "0.0.2"
+  VERSION = "0.0.3"
 end

data/lib/socialcrawler.rb

@@ -81,17 +81,13 @@ module SocialCrawler
       if not status_filename.nil? and File.exists?(status_filename)
         log.info("Loading previous status from #{status_filename}")
         CSV.foreach(status_filename) do |row|
-          if row.count < 3
-            next
+          if row.count >= 3
+            status[row[0]] = {
+                :url => row[0],
+                :result => row[1],
+                :message => row[2]
+            }
           end
-          url = row[0]
-          result = row[1]
-          message = row[2]
-          status[url] = {
-              :url => url,
-              :result => result,
-              :message => message
-          }
         end
         log.info("Loading previous status from #{status_filename} finished, #{status.keys.length} loaded.")
       end
@@ -106,21 +102,15 @@ module SocialCrawler
       end
       CSV.foreach(output_list_filename) do |row|
         log.info("Loading #{row} #{row.count}")
-        if row.count < 5
-          next
+        if row.count >= 5
+          data[row[0]] = {
+              :url => row[0],
+              :title => row[1],
+              :twitter => row[2],
+              :facebook => row[3],
+              :google_plus => row[4]
+          }
         end
-        url = row[0]
-        title= row[1]
-        twitter = row[2]
-        facebook = row[3]
-        google_plus = row[4]
-        data[url] = {
-            :url => url,
-            :title => title,
-            :twitter => twitter,
-            :facebook => facebook,
-            :google_plus => google_plus
-        }
         log.info("Loading previous status from #{output_list_filename} finished, #{data.keys.length} loaded.")
       end
       return data
@@ -154,23 +144,33 @@ module SocialCrawler
           next
         end
         result = crawl_url(url, log)
-        if result[:success] == true
-          data[url] = result
-          output << [url, result[:title], result[:twitter], result[:facebook], result[:google_plus]]
-        end
-        status[url] = {
-            :url => url,
-            :result => result[:success],
-            :message => result[:message]
-        }
-        status_line << [url, result[:success], result[:message]]
+        set_data(result, url, data, output)
+        set_status(result, url, status, status_line)
+      end
+    end
+
+    private
+
+    def set_data(result, url, data, output)
+      if result[:success] == true
+        data[url] = result
+        output << [url, result[:title], result[:twitter], result[:facebook], result[:google_plus]]
       end
     end
+
+    def set_status(result, url, status, status_line)
+      status[url] = {
+          :url => url,
+          :result => result[:success],
+          :message => result[:message]
+      }
+      status_line << [url, result[:success], result[:message]]
+    end
   end
 end
 
 if __FILE__ == $0
-  #:nocov:
+  # :nocov:
   SocialCrawler::SocialCrawler.new.crawl(ARGV[0], ARGV[1], ARGV[2])
-  #:nocov:
+  # :nocov:
 end

data/test/test_crawler.rb

@@ -20,6 +20,9 @@ class CrawlewrTest < Test::Unit::TestCase
 
     sc = SocialCrawler::SocialCrawler.new
     sc.crawl('test/test_url.txt', '/tmp/test_out.txt', '/tmp/test_status.txt')
+    
+    sc = SocialCrawler::SocialCrawler.new
+    sc.crawl('test/test_url.txt', '/tmp/test_out.txt', 'test_status.txt')
    end
 
 end

data/test_status.txt

@@ -0,0 +1,4 @@
+https://twitter.com/bugaco,,""
+https://plus.google.com/101033631762132540828/posts,,""
+https://www.facebook.com/,,""
+https://localhost/,,"Connection refused - connect(2) for ""localhost"" port 443"

metadata

@@ -1,94 +1,83 @@
 --- !ruby/object:Gem::Specification
 name: socialcrawler
 version: !ruby/object:Gem::Version
-  version: 0.0.2
-  prerelease: 
+  version: 0.0.3
 platform: ruby
 authors:
 - Ivica Ceraj
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-01-16 00:00:00.000000000 Z
+date: 2015-01-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.7'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.7'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '10.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '10.0'
 - !ruby/object:Gem::Dependency
   name: semantic
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.0'
 - !ruby/object:Gem::Dependency
   name: simplecov
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '0.9'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '0.9'
 - !ruby/object:Gem::Dependency
   name: simplecov-html
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '0.8'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '0.8'
 description: It read file containing list of urls and produces output file with domain,
@@ -99,9 +88,9 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- .coveralls.yml
-- .gitignore
-- .travis.yml
+- ".coveralls.yml"
+- ".gitignore"
+- ".travis.yml"
 - Gemfile
 - LICENSE.txt
 - README.md
@@ -112,30 +101,30 @@ files:
 - test/test_crawler.rb
 - test/test_helper.rb
 - test/test_url.txt
+- test_status.txt
 homepage: http://github.com/iceraj/socialcrawler
 licenses:
 - LGPL 2.1
+metadata: {}
 post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
-  - - ! '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
-  - - ! '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 1.8.24
+rubygems_version: 2.4.5
 signing_key: 
-specification_version: 3
+specification_version: 4
 summary: SocialCrawler looks for social media links for different sites
 test_files:
 - test/test_crawler.rb