shared_count-cli 0.0.1 → 0.0.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 0c3111abeaf8c7634190475c25146259569027a4
-  data.tar.gz: 52a4ad77c5fd35502554a1a3a872f9b4f07b0a0e
+  metadata.gz: dbac3484620be6939c0dd0941d049ae62c693901
+  data.tar.gz: af1587d6aef436efdd9aa2d07eb4b3a9073fd27b
 SHA512:
-  metadata.gz: e91498ba39a144ba4478a6d8be707d01937b4d6d245105ddcceb7a49fa3457b880c7d009f7fe3045a962d85ff69cd8442631c7ad003ef4e6ab372550e4af19cb
-  data.tar.gz: 53d75baa64822207eda1fdbae2ba4dd78a49a0a5bfeb1600bd5e8095d032bdc65a98917f59cfdbe1c570f70466f1de462686ef50b763ad2180500ec537df9402
+  metadata.gz: 08ea24413ded85fa2ab27f187769e178459dcaf64ee67690c60e6670c090b1fda30de1d7422abab6635ab45ad4b4f0eaa704b02075f99987b1a8f6eb9c3c4fe8
+  data.tar.gz: 35fd4f6093cab45615bde109e1ae42fda27614a798c950375acdd9bba7ee8668f9773ac52ff30169f5ff7c16283f37962f5981415aeb91fbe2009ebb05e8f029

data/.gitignore

@@ -17,3 +17,4 @@ test/version_tmp
 tmp
 .env
 .rbenv-gemsets
+*.log

data/README.md

@@ -13,13 +13,20 @@ CLI to the shared_count_api gem
 * Redirect the CSV file to stdout
 
 ```ruby
-$ ruby bin/shared_count-cli ~/Desktop/file?.txt # => ~/Desktop/file1.txt ~/Desktop/file2.txt
+$ shared_count-cli ~/Desktop/file?.txt # => ~/Desktop/file1.txt ~/Desktop/file2.txt
 ```
 
 * Redirect the CSV file to a file
 
 ```ruby
-$ ruby bin/shared_count-cli ~/Desktop/file?.txt > ~/Desktop/output.csv
+$ shared_count-cli ~/Desktop/file?.txt > ~/Desktop/output.csv
+```
+
+* Launch it in debug mode
+
+```ruby
+$ DEBUG=true shared_count-cli ~/Desktop/file?.txt > ~/Desktop/output.csv
+$ tail -f shared_count-cli.log
 ```
 
 ## Contributing

data/bin/shared_count-cli

@@ -3,6 +3,7 @@
 require_relative "../lib/shared_count/cli"
 
 urls = ARGF.readlines
+urls.reject! { |url| url.empty? }
 urls.map! { |url| url.start_with?("http") ? url : "http://#{url}" }
 
 csv = SharedCount::Cli.run(urls)

data/lib/shared_count/cli/version.rb

@@ -1,5 +1,5 @@
 module SharedCount
   module Cli
-    VERSION = "0.0.1"
+    VERSION = "0.0.2"
   end
 end

data/lib/shared_count/cli.rb

@@ -1,4 +1,7 @@
 require "csv"
+require "logger"
+require "thread"
+require "uri"
 require "shared_count_api"
 
 require_relative "../../config/initializers/dotenv"
@@ -8,36 +11,111 @@ require_relative "cli/version"
 
 module SharedCount
   module Cli
+    JOIN_TIMEOUT = 5 # seconds
+    SLEEP_TIME = 2.5 # seconds
+    MAX_RETRIES = 3
+    LINES_PER_ITERATION = 1000
+    MAX_CONCURRENCY = 50
+
     class << self
       def run(lines)
         configure_shared_count_client
 
-        CSV.generate do |csv|
-          lines.each_with_index do |url, i|
-            url.chomp!
-            response = nil
-            begin
-              response = SharedCountApi::Client.new(url).response
-            rescue SharedCountApi::Error
-              next
+        iterations, mod =  lines.length.divmod(LINES_PER_ITERATION)
+        iterations += 1 if mod > 0
+        results = Queue.new
+
+        iterations.times do |iteration|
+          logger.error "Iteration ##{iteration + 1}"
+          queue = Queue.new
+          from = LINES_PER_ITERATION * iteration
+          lines[from, LINES_PER_ITERATION].each { |url| queue.push(url) }
+          thread_count = [MAX_CONCURRENCY, lines.length].min
+
+          threads = (0...thread_count).map do |thread|
+            Thread.new(thread) do |thread|
+              error = 0
+
+              url = begin
+                      queue.pop(true)
+                    rescue ThreadError; end
+
+              while url do
+                url.chomp!
+                uri = URI(url)
+                host = uri.host || url[/\Ahttps?:\/\/([^\/]+)/, 1]
+                url = "#{uri.scheme}://#{host}"
+
+                response = nil
+                begin
+                  response = SharedCountApi::Client.new(url).response
+                rescue SharedCountApi::Error
+                  logger.error "[Thread ##{thread}] - error while processing '#{url}'"
+                rescue => err
+                  logger.error "[Thread ##{thread}] - error while processing '#{url}', retry: ##{error} - #{err.inspect}"
+                  error += 1
+                  sleep(SLEEP_TIME)
+                  if error <= MAX_RETRIES
+                    retry
+                  else
+                    queue.push(url)
+                    break
+                  end
+                else
+                  error = 0
+                end
+
+                if response
+                  logger.debug "[Thread ##{thread}] - #{url}"
+
+                  facebook_metrics = response.delete("Facebook")
+                  facebook_metrics = {} unless facebook_metrics.is_a?(Hash)
+                  values = response.values.unshift(url)
+                  results.push(values.concat(facebook_metrics.values))
+                else
+                  logger.warn "[Thread ##{thread}] - no response for '#{url}'"
+                end
+
+                url = begin
+                        queue.pop(true)
+                      rescue ThreadError; end
+              end
             end
+          end
 
-            facebook_metrics = response.delete("Facebook")
-            if i.zero?
-              keys = response.keys.unshift("URL")
-              headers = keys.concat(facebook_metrics.keys)
-              csv << headers
-              csv << []
+          threads.each do |thread|
+            begin
+              thread.join(JOIN_TIMEOUT)
+            rescue => err
+              logger.error "[Thread ##{thread}] - error while joining main thread: #{err.inspect}"
+              logger.error "[Thread ##{thread}] - #{err.backtrace.join("\n")}"
             end
+          end
+        end
+
 
-            values = response.values.unshift(url)
-            csv << values.concat(facebook_metrics.values)
+        CSV.generate do |csv|
+          csv << %w(URL StumbleUpon Reddit Delicious GooglePlusOne Buzz Twitter Diggs Pinterest LinkedIn commentsbox_count click_count total_count comment_count like_count share_count)
+          csv << []
+          loop do
+            begin
+              arr = results.pop(true)
+              csv << arr
+            rescue ThreadError
+              break
+            end
           end
         end
       end
 
     private
 
+      def logger
+        @logger ||= Logger.new("shared_count-cli.log").tap do |logger|
+          logger.level = ENV["DEBUG"] ? Logger::DEBUG : Logger::ERROR
+        end
+      end
+
       def configure_shared_count_client
         SharedCountApi.configure do |config|
           config.apikey = ENV["SHARED_COUNT_APIKEY"]

data/spec/shared_count/cli_spec.rb

@@ -4,18 +4,19 @@ require_relative "../../lib/shared_count/cli"
 
 describe SharedCount::Cli do
   describe ".run" do
-    let(:urls) do
+    let(:lines) do
       ["http://slashdot.org\n", "http://bbc.co.uk\n", "http://www.lanacion.com.ar\n", "http://www.theguardian.com\n"]
     end
 
     it "queries the SharedCount API for each URL passed in" do
-      csv = SharedCount::Cli.run(urls)
+      csv = SharedCount::Cli.run(lines)
 
       arr = CSV.parse(csv, headers: :first_row)
-      arr[1]["URL"].must_equal "http://slashdot.org"
-      arr[2]["URL"].must_equal "http://bbc.co.uk"
-      arr[3]["URL"].must_equal "http://www.lanacion.com.ar"
-      arr[4]["URL"].must_equal "http://www.theguardian.com"
+      urls = [arr[1]["URL"], arr[2]["URL"], arr[3]["URL"], arr[4]["URL"]]
+      lines.each do |line|
+        line.chomp!
+        urls.any? { |url| url == line }.must_be_same_as true
+      end
     end
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: shared_count-cli
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.0.2
 platform: ruby
 authors:
 - Cristian Rasch