britebox 0.0.1 → 0.0.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 933b24652e4d8e01a6cba8e94c78826441a6833b
-  data.tar.gz: 71d0fe270ab25989b2157fbdb5f40bd78b70bdd3
+  metadata.gz: 9e800dec5d7cc7526497042fdd135dfeefb49b08
+  data.tar.gz: caef336a484073e5b13fdbe31ab1c72d92155281
 SHA512:
-  metadata.gz: 7014db8bbe827d9030e69f9e377d2b04fcb43550a64742af704277d6c221204fbe8ff006cd8342ace8960bcd9f97fa84fd3215a1d1a44a735cd049ae3001673d
-  data.tar.gz: 664811ca1f66bb90f5dbc3de04317deebedaebf1dad076c7c5821674d622a0a2a17db5aeaeba8ff5016775dd18d4dcfd29a2854dfdffb29b65732976608f3703
+  metadata.gz: d8cf57385c286d5a807c0a3e04fc5c4ed0ec73754ebe1c0072d73951461b1d3285a63b93fe0f4dc94b5cd887e5209e5fe8fc2bdc80cd95257998e146005a87b6
+  data.tar.gz: a887c75d7e6423e5e17fccb2b226f16a13b97c91839676cab6f11d6856aaa84312a34796247dffffa3f73e1f53c1840d2efba86340320ef457a69fadd5b5c4ac

data/lib/britebox.rb

@@ -1,4 +1,6 @@
 
 require 'britebox/version'
+require 'britebox/helpers'
 require 'britebox/file_job'
-require 'britebox/file_job_pool'
+require 'britebox/file_job_pool'
+require 'britebox/export_buffer'

data/lib/britebox/export_buffer.rb

@@ -0,0 +1,87 @@
+module Britebox
+  class ExportBuffer
+    def initialize(out_file_name, header_line, col_separator)
+      @file = File.new out_file_name, 'w+'
+      @col_separator = col_separator
+      write_header_row(header_line)
+      @last_exported_line_idx = 0
+
+      @buffer = []
+      @backlog = []
+      @semaphore = Mutex.new
+    end
+
+    def buffer_size
+      100
+    end
+
+    # {n: N, line: [...]}
+    def push(item)
+      check_backlog
+
+      @semaphore.synchronize do
+        if item[:n] == @last_exported_line_idx
+          @buffer << item[:line]
+          @last_exported_line_idx += 1
+        else
+          @backlog << item
+        end
+      end
+
+      flush_buffer
+    end
+
+    def flush_backlog
+      @semaphore.synchronize do
+        @backlog.each do |item|
+          @buffer << item[:line]
+          @last_exported_line_idx += 1
+        end
+      end
+      flush_buffer(true)
+    end
+
+    def flush_buffer(force = false)
+      if force || @buffer.size > buffer_size
+        @semaphore.synchronize do
+          @file.write @buffer.map{ |line| CSV.generate_line(line, col_sep: @col_separator) }.join
+          @buffer = []
+        end
+      end
+    end
+
+    def close
+      @file.close
+    end
+
+    def status
+      {
+        buffer: @buffer.size,
+        backlog: @backlog.size,
+        last_exported_line_idx: @last_exported_line_idx,
+      }
+    end
+
+    private
+
+    def check_backlog
+      loop do
+        if (itm = @backlog.detect { |q| q[:n] == @last_exported_line_idx })
+          @semaphore.synchronize do
+            @buffer << itm[:line]
+            @backlog.delete(itm)
+            @last_exported_line_idx += 1
+          end
+        else
+          break
+        end
+      end
+    end
+
+    def write_header_row(row)
+      @file.write(CSV.generate_line(row, col_sep: @col_separator)) if row
+    end
+
+    alias_method :<<, :push
+  end
+end

data/lib/britebox/file_job.rb

@@ -1,45 +1,59 @@
-require 'thread_storm'
 require 'brite-api'
 require 'csv'
 
 module Britebox
   class FileJob
 
-    attr_reader :file_name, :lines_total, :error, :status, :processed_lines
+    attr_reader :file_name, :threads_count, :error, :status, :size_total, :size_processed
 
     EMAIL_PATTERN   = /(\S+)@(\S+)/
     COL_SEPARATORS = [";", "|", "\t"]
 
-    def initialize(file_name, brite_client, thread_pool, options = {})
+    def initialize(file_name, brite_client, options = {})
       @file_name = file_name
       @brite_client = brite_client
-      @thread_pool = thread_pool
-      @processed_lines = []
-      @status = 'pending'
-    end
 
-    def lines_processed
-      @processed_lines.compact.count
-    end
+      @threads_count = options[:threads] || Britebox::FileJobPool::THREAD_NUM_DEFAULT
+      @queue = options[:queue]
 
-    def verify!
-      @status = 'verifying'
+      @status = 'pending'
+
+      @size_processed = 0
 
       unless File.exist?(file_name)
         report_error!("File #{file_name} not found") and return
       end
-      file_data = File.read file_name
-      if file_data.size == 0
+      if (@size_total = File.size(file_name)) == 0
         report_error!("File #{file_name} is empty") and return
       end
 
+      @semaphore = Mutex.new
+    end
+
+    def buffer_size
+      @threads_count * 4
+    end
+
+    def percent_complete
+      if @size_total.to_i > 0
+        (100.0 * @size_processed / @size_total).round(1)
+      else
+        0.0
+      end
+    end
+
+    def verify!(file_name_to)
+      return if @status == 'error'
+
       test_lines = []
       begin
         line_n = 0
-        CSV.parse(file_data) do |line|
+        CSV.foreach(file_name) do |line|
           if line && line.size > 0
             if line_n < 5
               test_lines << line
+            else
+              break
             end
 
             line_n += 1
@@ -51,56 +65,66 @@ module Britebox
 
       autoconfigure(test_lines) || return
 
-      parsed_lines = CSV.parse(file_data, col_sep: @col_separator)
-      file_data = nil # Free up resources
-
-      @lines_total = parsed_lines.count
-      @processed_lines = Array.new(@lines_total)
-      @lines_total -= 1 if is_header_row?(parsed_lines.first)
+      # Block processing if some another FileJob processing lines
+      @queue.pop if @queue
 
+      @status = 'verifying'
 
-      # spin up multiple processing threads
-      jobs = []
-      parsed_lines.each_with_index do |line, idx|
-        next if idx == 0 && is_header_row?(line)
+      @in_buffer = Queue.new
+      @out_buffer = ExportBuffer.new(file_name_to, @header_row, @col_separator)
+
+      @threads = []
+      @threads_count.times do
+        @threads << Thread.new do
+          loop do
+            item = Timeout.timeout(1) { @in_buffer.pop } rescue nil
+            break if item.nil?
+
+            email = item[:line][@email_index]
+            if email.to_s.match(EMAIL_PATTERN)
+              begin
+                contact = @brite_client.contacts.create(email: email)
+                contact.verify!
+                contact_status = [contact.status, contact.response[:email]['disposable'], contact.response[:email]['role_address']]
+              rescue
+                contact_status = ['error', false, false]
+              end
+            else
+              contact_status = ['invalid', false, false]
+            end
 
-        jobs << @thread_pool.execute do
-          email = line[@email_index]
-          begin
-            contact = @brite_client.contacts.create(email: email)
-            contact.verify!
-            # ['email_status', 'disposable', 'role_account']
-            contact_status = [contact.status, contact.response[:email]['disposable'], contact.response[:email]['role_address']]
-          rescue  Exception => ex
-            contact_status = ['error', false, false]
+            @semaphore.synchronize do
+              @out_buffer << {n: item[:n], line: item[:line] + contact_status}
+              @size_processed += CSV.generate_line(item[:line], col_sep: @col_separator).size
+            end
           end
-          # Store processed file
-          @processed_lines[idx] = line + contact_status
         end
       end
 
-      # Wait for all threads
-      jobs.each{ |j| j.join }
+      idx = 0
+      CSV.foreach(file_name, col_sep: @col_separator) do |line|
+        if idx == 0 && is_header_row?(line)
+          @size_processed += CSV.generate_line(line, col_sep: @col_separator).size
+          next
+        end
+        next if line.nil? || line.size == 0
+        # Throttle file reading
+        sleep(0.1) while @in_buffer.size > buffer_size
 
-      # Free up resources
-      parsed_lines = nil
+        @in_buffer << {n: idx, line: line}
+        idx += 1
+      end
 
-      @status = 'exporting'
+      @threads.each{ |m| m.join }
 
-      true
-    end
+      @out_buffer.flush_backlog
+      @out_buffer.close
 
-    def export!(file_name)
-      file = File.new(file_name, "w+")
+      @status = 'complete'
 
-      file.write(CSV.generate_line(@header_row, col_sep: @col_separator)) if @header_row
-      @processed_lines.each do |line|
-        next if line.nil?
-        file.write CSV.generate_line(line, col_sep: @col_separator)
-      end
+      # Release the lock
+      @queue.push(:flag) if @queue
 
-      file.close
-      @status = 'complete'
       true
     end
 
@@ -128,11 +152,16 @@ module Britebox
 
 
     def autoconfigure(test_lines)
+      if test_lines.nil? || test_lines.size == 0
+        report_error! 'Column separator could not be determined'
+        return
+      end
+
       @col_separator = nil
       @header_row = nil
       @email_index = nil
 
-      if test_lines.first.size > 1
+      if test_lines[0].size > 1
         @col_separator = ','
       else
         COL_SEPARATORS.each do |sep|

data/lib/britebox/file_job_pool.rb

@@ -1,6 +1,8 @@
 module Britebox
   # Keep current status for all processing FileJobs
   class FileJobPool
+    include Britebox::Helpers
+
     attr_reader :file_jobs
 
     SPINNERS = ["|", "/", "—", "\\"]
@@ -8,10 +10,14 @@ module Britebox
 
     def initialize(num_threads = nil)
       num_threads ||= THREAD_NUM_DEFAULT
+      @num_threads = num_threads
+
+      # Only single FileJob can own :flag & process own lines
+      @queue = Queue.new
+      @queue << :flag
+
       @file_jobs = []
       @refresh_number = 0
-      @fj_pool = ThreadStorm.new size: num_threads
-      @lines_pool = ThreadStorm.new size: num_threads
     end
 
     def add(file_job)
@@ -23,16 +29,19 @@ module Britebox
       @file_jobs.map{ |fj| fj.file_name }.include? file_name
     end
 
-    def process_file!(file, dir, out_dir, brite_client, fj_options)
-      file_name = File.expand_path(file, dir)
+    def process_file!(file, in_dir, out_dir, brite_client, fj_options = {})
+      file_name = File.expand_path(file, in_dir)
 
       # Do not process same files twice
       return if self.include? file_name
 
-      fj = FileJob.new(file_name, brite_client, @lines_pool, fj_options)
+      fj_options[:queue] = @queue
+      fj_options[:threads] = @num_threads
+
+      fj = FileJob.new(file_name, brite_client, fj_options)
       self.add(fj)
-      @fj_pool.execute do
-        fj.verify!
+      Thread.new do
+        fj.verify!(File.expand_path(file, out_dir))
 
         case fj.status
           when 'error'
@@ -41,10 +50,8 @@ module Britebox
             File.open(File.expand_path(err_name + '.log', out_dir), 'w+') do |f|
               f.write fj.error # TODO: add extended error log
             end
-          when 'exporting'
-            if fj.export! File.expand_path(file, out_dir)
-              File.delete file_name
-            end
+          when 'complete'
+            File.delete file_name
           else
             raise "unexpected error, status: #{fj.status}"
         end
@@ -61,20 +68,23 @@ module Britebox
         print "\r#{clear}#{up}" * @prev_buffer_height
       end
 
-      buffer =  "\nFiles processing: #{@file_jobs.count}"
+      buffer =  "\nFiles in list: #{@file_jobs.count}"
 
       @file_jobs.each do |fj|
         fname = File.basename(fj.file_name)
         if fname.length > 20
           fname = fname[0..16] + '...'
         end
+
         if fj.status == 'error'
           status_str = fj.error
+        elsif fj.status == 'complete'
+          status_str = "100.0 %"
         else
-          status_str = "#{(fj.lines_processed || '-')} / #{fj.lines_total || '-'}"
+          status_str = "#{fj.percent_complete} %".rjust(7)
         end
 
-        buffer << "\n#{fname.ljust(20)} | #{fj.status.to_s.ljust(9)} | #{status_str}"
+        buffer << "\n#{fname.ljust(20)} | #{fj.status.to_s.ljust(9)} | #{human_size(fj.size_total).rjust(8)} | #{status_str}"
       end
 
       buffer << "\n #{SPINNERS[@refresh_number % 4]}\n"

data/lib/britebox/helpers.rb

@@ -0,0 +1,16 @@
+module Britebox
+  module Helpers
+    def human_size(n)
+      count = 0
+      while  n >= 1024 and count < 4
+        n /= 1024.0
+        count += 1
+      end
+      if count == 0
+        n.to_s + 'B'
+      else
+        format("%.2f", n) + %w(B KB MB GB TB)[count]
+      end
+    end
+  end
+end

data/lib/britebox/version.rb

@@ -1,3 +1,3 @@
 module Britebox
-  VERSION = '0.0.1'
+  VERSION = '0.0.2'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: britebox
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.0.2
 platform: ruby
 authors:
 - Alexander Shapiotko
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-07-07 00:00:00.000000000 Z
+date: 2013-07-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: brite-api
@@ -25,20 +25,6 @@ dependencies:
     - - '>='
       - !ruby/object:Gem::Version
         version: 0.0.1
-- !ruby/object:Gem::Dependency
-  name: thread_storm
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ~>
-      - !ruby/object:Gem::Version
-        version: 0.7.1
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ~>
-      - !ruby/object:Gem::Version
-        version: 0.7.1
 - !ruby/object:Gem::Dependency
   name: listen
   requirement: !ruby/object:Gem::Requirement
@@ -62,8 +48,10 @@ extensions: []
 extra_rdoc_files: []
 files:
 - lib/britebox/cli.rb
+- lib/britebox/export_buffer.rb
 - lib/britebox/file_job.rb
 - lib/britebox/file_job_pool.rb
+- lib/britebox/helpers.rb
 - lib/britebox/version.rb
 - lib/britebox.rb
 - README.md