britebox 0.0.6 → 0.0.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: af52b5e18fb92d49ca360ef3a68e4a5b65825e3c
-  data.tar.gz: 1c70ba5f08bdadef985eee62dda15a4ff435aa10
+  metadata.gz: 0949a7ddc1540e0c8374818c3647930eb3560ded
+  data.tar.gz: 24c7939e3c0ca1e318ea9ff8af22c485649558b9
 SHA512:
-  metadata.gz: 83d66b045e53371ab19a2f777cd8336474b41f6fbf6bc146c8af0c7e47b07541bf0fb49a02ef80cd3ead29dba115872f64e0b0107a6a6b03e3f3978293f3b7a1
-  data.tar.gz: 831f8252a4e93a07a4384eacfc88e722feba7b79ce25489ff2fb14b201ba7d010682ff5ab806506d11ef864bd2760f8f9fd686506c82ff029b690d7ebf07bbf1
+  metadata.gz: c0a6695c78f1daed658d42c1826cb7ad5d43efdb9f3fe35bc7e371a6804e726f5c7d05a47f7a98722a897fd6c3a8efb028165367528d93aa0aa22d4a022d9f3f
+  data.tar.gz: b71b27e62b1877698515972d0afe8935edbfd262cac51c2087193fdcd83acfa271eac77ba93e7d6535a20f6ed09fcc46cb8b381c7002ac0e7c5e4158156ac2eb

data/lib/britebox.rb

@@ -3,8 +3,11 @@ require 'britebox/version'
 require 'britebox/helpers'
 require 'britebox/config'
 require 'britebox/file_job'
+require 'britebox/file_job_timer'
 require 'britebox/file_job_pool'
 require 'britebox/export_buffer'
 require 'britebox/event_log'
 require 'britebox/web_ui'
-require 'britebox/filtered_error_io'
+require 'britebox/filtered_error_io'
+require 'britebox/format_recognizer'
+require 'britebox/lines_verifier'

data/lib/britebox/file_job.rb

@@ -4,11 +4,10 @@ require 'csv'
 module Britebox
   class FileJob
 
-    attr_reader :file_name, :threads_count, :error, :status, :size_total, :size_processed,
-                :started_at, :processed_at
+    attr_reader :file_name, :threads_count, :error, :status, :size_total,
+                :timer, :queue, :semaphore, :brite_client
 
-    EMAIL_PATTERN   = /(\S+)@(\S+)/
-    COL_SEPARATORS = [";", "|", "\t"]
+    attr_accessor :size_processed
 
     def initialize(file_name, brite_client, options = {})
       @file_name = file_name
@@ -21,6 +20,9 @@ module Britebox
 
       @size_processed = 0
 
+      @semaphore = Mutex.new
+      @timer = FileJobTimer.new
+
       unless File.exist?(file_name)
         report_error!("File not found")
         return
@@ -29,8 +31,18 @@ module Britebox
         report_error!("File is empty")
         return
       end
+    end
 
-      @semaphore = Mutex.new
+    def started_at
+      @timer.started_at
+    end
+
+    def processed_at
+      @timer.ended_at
+    end
+
+    def duration
+      @timer.duration
     end
 
     def as_json
@@ -41,8 +53,8 @@ module Britebox
         threads: @threads_count,
         size_total: @size_total,
         size_processed: @size_processed,
-        started_at: @started_at,
-        processed_at: @processed_at,
+        started_at: started_at,
+        processed_at: processed_at,
         duration: duration,
         percent_complete: percent_complete,
         error: @error
@@ -57,13 +69,6 @@ module Britebox
       @threads_count * 4
     end
 
-    def duration
-      if @started_at
-        end_time = @processed_at || Time.now
-        end_time - @started_at
-      end
-    end
-
     def percent_complete
       if @size_total.to_i > 0
         (100.0 * @size_processed / @size_total).round(1)
@@ -94,14 +99,31 @@ module Britebox
 
     def status=(new_status)
       return if status == new_status
+      return if verifying? && new_status == 'pending' # avoid double-clicking
+
+      if new_status == 'verifying'
+        raise("status 'verifying' can't be set, use 'pending' instead")
+      end
 
       # Release processing slot
-      if verifying? && ['paused', 'cancelled'].include?(new_status)
-        @queue.push(:flag) if @queue
+      if new_status == 'cancelled'
+        release_flag
       end
 
-      if new_status == 'verifying'
-        raise("status 'verifying' can't be set, use 'pending' instead")
+      if verifying? && new_status == 'paused'
+        release_flag
+      end
+
+      if new_status == 'pending'
+        Thread.new do
+          while @getting_flag
+            sleep(0.1)
+          end
+          get_flag
+          @semaphore.synchronize do
+            @status = 'verifying'
+          end
+        end
       end
 
       @status = new_status
@@ -110,106 +132,36 @@ module Britebox
     def verify!(file_name_to)
       return if error?
 
-      test_lines = []
-      begin
-        line_n = 0
-        CSV.foreach(file_name) do |line|
-          if line && line.size > 0
-            if line_n < 5
-              test_lines << line
-            else
-              break
-            end
-
-            line_n += 1
-          end
-        end
-      rescue Exception => ex
-        report_error!(ex.message.to_s) and return
+      recognizer = FormatRecognizer.new(@file_name)
+      if recognizer.recognize!
+        opts = recognizer.opts
+      else
+        report_error! recognizer.error
+        return
       end
 
-      autoconfigure(test_lines) || return
 
       # Block processing if some another FileJob processing lines
-      @queue.pop if @queue
+      loop do
+        break if verifying?
+        get_flag
 
-      @started_at = Time.now
-      @status = 'verifying'
-
-      @in_buffer = Queue.new
-      @out_buffer = ExportBuffer.new(file_name_to, @header_row, @col_separator)
-
-      @threads = []
-      @threads_count.times do
-        @threads << Thread.new do
-          loop do
-            # Pause all processing
-            loop do
-              break unless paused?
-              sleep(0.1)
-            end
-
-            break if cancelled?
-
-            # Wait for processing slot
-            if pending?
-              @semaphore.synchronize do
-                @queue.pop if @queue && pending?
-                @status = 'verifying'
-              end
-            end
-
-            item = Timeout.timeout(1) { @in_buffer.pop } rescue nil
-            break if item.nil?
-
-            email = item[:line][@email_index]
-            if email.to_s.match(EMAIL_PATTERN)
-              if Britebox::Config.simulate
-                # Do not send real requests in this mode
-                sleep(1)
-                contact_status = ['unknown', false, false]
-              else
-                begin
-                  contact = @brite_client.contacts.create(email: email)
-                  contact.verify!
-                  contact_status = [contact.status, contact.response[:email]['disposable'], contact.response[:email]['role_address']]
-                rescue
-                  contact_status = ['error', false, false]
-                end
-              end
-            else
-              contact_status = ['invalid', false, false]
-            end
-
-            @semaphore.synchronize do
-              @out_buffer << {n: item[:n], line: item[:line] + contact_status}
-              @size_processed += CSV.generate_line(item[:line], col_sep: @col_separator).size
-            end
-          end
-        end
-      end
-
-      idx = 0
-      CSV.foreach(file_name, col_sep: @col_separator) do |line|
-        if idx == 0 && is_header_row?(line)
-          @size_processed += CSV.generate_line(line, col_sep: @col_separator).size
-          next
+        if paused?
+          release_flag
+          sleep 0.3
+        else
+          break
         end
-        next if line.nil? || line.size == 0
-        # Throttle file reading
-        break if cancelled?
-        sleep(0.1) while @in_buffer.size > buffer_size && !cancelled?
-
-        @in_buffer << {n: idx, line: line}
-        idx += 1
       end
 
-      @threads.each{ |m| m.join }
+      if pending? || verifying?
+        @timer.start
+        @status = 'verifying'
 
-      @out_buffer.flush_backlog
-      @out_buffer.close
+        LinesVerifier.new(self).process!(file_name_to, opts)
 
-      @processed_at = Time.now
+        @timer.stop
+      end
 
       if cancelled?
         File.delete file_name_to
@@ -217,126 +169,33 @@ module Britebox
         @status = 'complete'
       end
 
-      # Release the lock
-      @queue.push(:flag) if @queue
+      release_flag
 
       true
     end
 
     private
 
-    def is_header_row?(row)
-      row.each do |v|
-        v = v.first if v.instance_of? Array
-        return false if v.to_s.match(EMAIL_PATTERN)
-      end
-      !row.join(" ").downcase.match("email").nil?
-    end
-
-    def find_email_index(row)
-      row.each_with_index do |value, index|
-        return index if value.to_s.match(EMAIL_PATTERN)
-      end
-      nil
-    end
-
-    def report_error!(text)
-      @status = 'error'
-      @error = text
+    def release_flag
+      @queue.push(:flag) if @queue
     end
 
-
-    def autoconfigure(test_lines)
-      if test_lines.nil? || test_lines.size == 0
-        report_error! 'Column separator could not be determined'
-        return
-      end
-
-      @col_separator = nil
-      @header_row = nil
-      @email_index = nil
-
-      if test_lines[0].size > 1
-        @col_separator = ','
-      else
-        COL_SEPARATORS.each do |sep|
-          if is_separator?(sep, test_lines)
-            @col_separator = sep
-            break
-          end
-        end
+    def get_flag
+      @semaphore.synchronize do
+        @getting_flag = true
       end
 
-      # Single column file
-      if @col_separator.nil? && test_lines.first.size == 1 && !is_separator?(' ', test_lines)
-        @col_separator = ','
-      end
-
-      if @col_separator.nil?
-        report_error! 'Column separator could not be determined'
-        return
-      end
-
-      test_rows = test_lines.map do |line|
-        if @col_separator == ","
-          line
-        else
-          line.first.split(@col_separator)
-        end
-      end
-
-      test_rows.each do |row|
-        if (index = find_email_index row)
-          @email_index = index
-          break
-        end
-      end
-
-      if @email_index.nil?
-        report_error! "Email column could not be determined."
-        return
-      end
-
-      configure_header_row(test_rows)
-
-      true
-    end
-
-
-    def configure_header_row(test_rows)
-      plus_headers = ['email_status', 'disposable', 'role_account']
+      @queue.pop if @queue
 
-      if is_header_row?(test_rows.first)
-        hr = test_rows.first
-      else
-        hr = []
-        test_rows.first.count.times do |i|
-          if i == @email_index
-            hr << "email"
-          else
-            hr << "column_#{i}"
-          end
-        end
+      @semaphore.synchronize do
+        @getting_flag = false
       end
-      @header_row = hr + plus_headers
     end
 
-    def is_separator?(sep, lines)
-      same_count = 0
-      fcount = lines.first.first.split(sep).size
-      if fcount == 1
-        false
-      else
-        lines.each do |line|
-          cc = line.first.split(sep).size
-          same_count += 1 if cc == fcount
-        end
-        same_count == lines.count
-      end
+    def report_error!(text)
+      @status = 'error'
+      @error = text
     end
 
-
-
-
   end
 end

data/lib/britebox/file_job_timer.rb

@@ -0,0 +1,47 @@
+module Britebox
+  class FileJobTimer
+    attr_reader :started_at, :ended_at
+
+    def initialize
+      @banked_time = 0
+      @is_running = false
+    end
+
+    def start
+      @banked_time = 0
+      @started_at = @resumed_at = Time.now
+      @is_running = true
+    end
+
+    def running?
+      @is_running
+    end
+
+    def pause
+      return unless running?
+
+      @is_running = false
+      @banked_time += (Time.now - @resumed_at)
+    end
+
+    def resume
+      return if running?
+
+      @resumed_at = Time.now
+      @is_running = true
+    end
+
+    def stop
+      pause
+      @ended_at = Time.now
+    end
+
+    def duration
+      if running?
+        @banked_time + (Time.now - @resumed_at)
+      else
+        @banked_time
+      end
+    end
+  end
+end

data/lib/britebox/format_recognizer.rb

@@ -0,0 +1,144 @@
+module Britebox
+  class FormatRecognizer
+    attr_reader :opts, :error
+
+    EMAIL_PATTERN   = /(\S+)@(\S+)/
+    COL_SEPARATORS = [";", "|", "\t"]
+    PLUS_HEADERS = ['email_status', 'disposable', 'role_account']
+
+
+
+    def initialize(file_name)
+      @file_name = file_name
+      @opts = {col_separator: nil, header_row: nil, email_index: nil}
+    end
+
+    def recognize!
+      test_lines = []
+      begin
+        line_n = 0
+        CSV.foreach(@file_name) do |line|
+          if line && line.size > 0
+            if line_n < 5
+              test_lines << line
+            else
+              break
+            end
+
+            line_n += 1
+          end
+        end
+      rescue Exception => ex
+        @error = ex.message.to_s
+        return false
+      end
+
+      autoconfigure(test_lines) || return
+
+      true
+    end
+
+
+    def find_email_index(row)
+      row.each_with_index do |value, index|
+        return index if value.to_s.match(EMAIL_PATTERN)
+      end
+      nil
+    end
+
+    def self.is_header_row?(row)
+      row.each do |v|
+        v = v.first if v.instance_of? Array
+        return false if v.to_s.match(EMAIL_PATTERN)
+      end
+      !row.join(" ").downcase.match("email").nil?
+    end
+
+    private
+
+    def autoconfigure(test_lines)
+      if test_lines.nil? || test_lines.size == 0
+        @error = 'Column separator could not be determined'
+        return false
+      end
+
+      if test_lines[0].size > 1
+        @opts[:col_separator] = ','
+      else
+        COL_SEPARATORS.each do |sep|
+          if is_separator?(sep, test_lines)
+            @opts[:col_separator] = sep
+            break
+          end
+        end
+      end
+
+      # Single column file
+      if @opts[:col_separator].nil? && test_lines.first.size == 1 && !is_separator?(' ', test_lines)
+        @opts[:col_separator] = ','
+      end
+
+      if @opts[:col_separator].nil?
+        @error = 'Column separator could not be determined'
+        return false
+      end
+
+      test_rows = test_lines.map do |line|
+        if @opts[:col_separator] == ","
+          line
+        else
+          line.first.split(@opts[:col_separator])
+        end
+      end
+
+      test_rows.each do |row|
+        if (index = find_email_index row)
+          @opts[:email_index] = index
+          break
+        end
+      end
+
+      if  @opts[:email_index].nil?
+        @error = "Email column could not be determined."
+        return false
+      end
+
+      configure_header_row(test_rows)
+
+      true
+    end
+
+    def configure_header_row(test_rows)
+      if self.class.is_header_row?(test_rows.first)
+        hr = test_rows.first
+      else
+        hr = []
+        test_rows.first.count.times do |i|
+          if i == @opts[:email_index]
+            hr << "email"
+          else
+            hr << "column_#{i}"
+          end
+        end
+      end
+      @opts[:header_row] = hr + PLUS_HEADERS
+    end
+
+    def is_separator?(sep, lines)
+      same_count = 0
+      fcount = lines.first.first.split(sep).size
+      if fcount == 1
+        false
+      else
+        lines.each do |line|
+          cc = line.first.split(sep).size
+          same_count += 1 if cc == fcount
+        end
+        same_count == lines.count
+      end
+    end
+
+
+
+  end
+end