britebox 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 933b24652e4d8e01a6cba8e94c78826441a6833b
+  data.tar.gz: 71d0fe270ab25989b2157fbdb5f40bd78b70bdd3
+SHA512:
+  metadata.gz: 7014db8bbe827d9030e69f9e377d2b04fcb43550a64742af704277d6c221204fbe8ff006cd8342ace8960bcd9f97fa84fd3215a1d1a44a735cd049ae3001673d
+  data.tar.gz: 664811ca1f66bb90f5dbc3de04317deebedaebf1dad076c7c5821674d622a0a2a17db5aeaeba8ff5016775dd18d4dcfd29a2854dfdffb29b65732976608f3703

data/README.md

@@ -0,0 +1,4 @@
+britebox
+========
+
+BriteVerify API CLI tool

data/bin/britebox

@@ -0,0 +1,76 @@
+#!/usr/bin/env ruby
+
+require 'optparse'
+require 'britebox'
+require 'britebox/cli'
+
+Thread.abort_on_exception = true
+THREAD_NUM_MAX = 10
+
+commands = ['watch']
+
+if $*.size == 1 && ($*[0] == '-v' || $*[0] == '--version')
+  puts Britebox::VERSION
+  exit 0
+end
+
+if $*.size == 0 || (not commands.include?($*[0]))
+  puts 'usage: britebox COMMAND [OPTIONS]'
+  puts "    COMMAND: #{commands.join(', ')}"
+  puts '    run britebox COMMAND --help to get more information about each command'
+  exit 1
+end
+
+def common_opts(opts)
+
+end
+
+
+@cli = Britebox::CLI.new
+
+command = $*.shift
+if command == 'watch'
+  params = {}
+  options = {}
+
+  opts = OptionParser.new do |opts|
+    opts.banner = "usage: britebox watch [OPTIONS]"
+
+    opts.on('-d', '--dir WATCHDIR', 'Watch this directory for incoming files') do |v|
+      params[:dir] = v
+    end
+
+    opts.on('-o', '--output OUTPUTDIR', 'Place verified files into this directory') do |v|
+      params[:output] = v
+    end
+
+    opts.on('-k', '--apikey APIKEY', 'BriteVerify API Key') do |v|
+      params[:api_key] = v
+    end
+
+    opts.on('-t', '--threads THREADS', "Maximum number of parallel threads used for processing, "+
+      "Default is #{Britebox::FileJobPool::THREAD_NUM_DEFAULT}, " +
+      "Maximum is #{THREAD_NUM_MAX}") do |v|
+      unless (1..THREAD_NUM_MAX).include? v.to_i
+        puts "Threads number should be in range 1..#{THREAD_NUM_MAX}"
+        exit 1
+      end
+      params[:threads] = v.to_i
+    end
+
+    common_opts(opts)
+  end
+
+  begin
+    opts.parse!
+  rescue OptionParser::ParseError
+    puts $!.to_s
+    exit 1
+  end
+
+  @cli.watch(params)
+
+
+end
+
+

data/lib/britebox.rb

@@ -0,0 +1,4 @@
+
+require 'britebox/version'
+require 'britebox/file_job'
+require 'britebox/file_job_pool'

data/lib/britebox/cli.rb

@@ -0,0 +1,63 @@
+require 'listen'
+require 'fileutils'
+
+module Britebox
+  class CLI
+    MONITOR_EXTENSIONS = ['csv', 'txt']
+
+    def initialize
+
+    end
+
+    def watch(params)
+      @api_key = params[:api_key] || raise("Please provide BriteVerify API Key")
+
+      if params[:dir]
+        @dir = File.expand_path params[:dir]
+      else
+        raise("Please provide directory-to-watch")
+      end
+
+      if params[:output]
+        @out_dir = File.expand_path params[:output]
+      else
+        @out_dir = File.expand_path 'verified', @dir
+      end
+
+      fj_options = {threads: params[:threads]}
+
+      puts "Watching directory #{@dir}"
+      puts "Output directory is #{@out_dir}"
+      puts "Press Ctrl-C to quit"
+
+      fj_pool = FileJobPool.new(params[:threads])
+
+      options = {filter: /\.(#{MONITOR_EXTENSIONS.join('|')})$/}
+      if @out_dir.include? @dir
+        options[:ignore] = Regexp.new "^" + @out_dir.sub(@dir + File::SEPARATOR, '')
+      end
+
+      brite_client = BriteAPI::Client.new(@api_key)
+
+      Listen.to(@dir, options) do |modified, added, removed|
+        (modified + added).each do |file|
+          fj_pool.process_file!(file, @dir, @out_dir, brite_client, fj_options)
+        end
+      end
+
+      # Trigger change for existing files
+      MONITOR_EXTENSIONS.each do |ext|
+        FileUtils.touch Dir.glob(File.expand_path("*.#{ext}", @dir)).map{ |f| File.expand_path(f, "*.#{ext}")}
+      end
+
+      # Update current status in cycle
+      loop do
+        fj_pool.print_status
+        sleep 0.5
+      end
+
+
+
+    end
+  end
+end

data/lib/britebox/file_job.rb

@@ -0,0 +1,218 @@
+require 'thread_storm'
+require 'brite-api'
+require 'csv'
+
+module Britebox
+  class FileJob
+
+    attr_reader :file_name, :lines_total, :error, :status, :processed_lines
+
+    EMAIL_PATTERN   = /(\S+)@(\S+)/
+    COL_SEPARATORS = [";", "|", "\t"]
+
+    def initialize(file_name, brite_client, thread_pool, options = {})
+      @file_name = file_name
+      @brite_client = brite_client
+      @thread_pool = thread_pool
+      @processed_lines = []
+      @status = 'pending'
+    end
+
+    def lines_processed
+      @processed_lines.compact.count
+    end
+
+    def verify!
+      @status = 'verifying'
+
+      unless File.exist?(file_name)
+        report_error!("File #{file_name} not found") and return
+      end
+      file_data = File.read file_name
+      if file_data.size == 0
+        report_error!("File #{file_name} is empty") and return
+      end
+
+      test_lines = []
+      begin
+        line_n = 0
+        CSV.parse(file_data) do |line|
+          if line && line.size > 0
+            if line_n < 5
+              test_lines << line
+            end
+
+            line_n += 1
+          end
+        end
+      rescue Exception => ex
+        report_error!(ex.message.to_s) and return
+      end
+
+      autoconfigure(test_lines) || return
+
+      parsed_lines = CSV.parse(file_data, col_sep: @col_separator)
+      file_data = nil # Free up resources
+
+      @lines_total = parsed_lines.count
+      @processed_lines = Array.new(@lines_total)
+      @lines_total -= 1 if is_header_row?(parsed_lines.first)
+
+
+      # spin up multiple processing threads
+      jobs = []
+      parsed_lines.each_with_index do |line, idx|
+        next if idx == 0 && is_header_row?(line)
+
+        jobs << @thread_pool.execute do
+          email = line[@email_index]
+          begin
+            contact = @brite_client.contacts.create(email: email)
+            contact.verify!
+            # ['email_status', 'disposable', 'role_account']
+            contact_status = [contact.status, contact.response[:email]['disposable'], contact.response[:email]['role_address']]
+          rescue  Exception => ex
+            contact_status = ['error', false, false]
+          end
+          # Store processed file
+          @processed_lines[idx] = line + contact_status
+        end
+      end
+
+      # Wait for all threads
+      jobs.each{ |j| j.join }
+
+      # Free up resources
+      parsed_lines = nil
+
+      @status = 'exporting'
+
+      true
+    end
+
+    def export!(file_name)
+      file = File.new(file_name, "w+")
+
+      file.write(CSV.generate_line(@header_row, col_sep: @col_separator)) if @header_row
+      @processed_lines.each do |line|
+        next if line.nil?
+        file.write CSV.generate_line(line, col_sep: @col_separator)
+      end
+
+      file.close
+      @status = 'complete'
+      true
+    end
+
+    private
+
+    def is_header_row?(row)
+      row.each do |v|
+        v = v.first if v.instance_of? Array
+        return false if v.to_s.match(EMAIL_PATTERN)
+      end
+      !row.join(" ").downcase.match("email").nil?
+    end
+
+    def find_email_index(row)
+      row.each_with_index do |value, index|
+        return index if value.to_s.match(EMAIL_PATTERN)
+      end
+      nil
+    end
+
+    def report_error!(text)
+      @status = 'error'
+      @error = text
+    end
+
+
+    def autoconfigure(test_lines)
+      @col_separator = nil
+      @header_row = nil
+      @email_index = nil
+
+      if test_lines.first.size > 1
+        @col_separator = ','
+      else
+        COL_SEPARATORS.each do |sep|
+          if is_separator?(sep, test_lines)
+            @col_separator = sep
+            break
+          end
+        end
+      end
+
+      # Single column file
+      if @col_separator.nil? && test_lines.first.size == 1 && is_separator?(' ', test_lines)
+        @col_separator = ','
+      end
+
+      if @col_separator.nil?
+        report_error! 'Column separator could not be determined'
+        return
+      end
+
+      test_rows = test_lines.map do |line|
+        if @col_separator == ","
+          line
+        else
+          line.first.split(@col_separator)
+        end
+      end
+
+      test_rows.each do |row|
+        if (index = find_email_index row)
+          @email_index = index
+          break
+        end
+      end
+
+      if @email_index.nil?
+        report_error! "Email column could not be determined."
+        return
+      end
+
+      configure_header_row(test_rows)
+
+      true
+    end
+
+
+    def configure_header_row(test_rows)
+      plus_headers = ['email_status', 'disposable', 'role_account']
+
+      if is_header_row?(test_rows.first)
+        hr = test_rows.first
+      else
+        hr = []
+        test_rows.first.count.times do |i|
+          if i == @email_index
+            hr << "email"
+          else
+            hr << "column_#{i}"
+          end
+        end
+      end
+      @header_row = hr + plus_headers
+    end
+
+    def is_separator?(sep, lines)
+      same_count = 0
+      fcount = lines.first.first.split(sep).size
+      if fcount == 1
+        false
+      else
+        lines.each do |line|
+          cc = line.first.split(sep).size
+          same_count += 1 if cc == fcount
+        end
+        same_count == lines.count
+      end
+    end
+
+
+
+
+  end
+end

data/lib/britebox/file_job_pool.rb

@@ -0,0 +1,90 @@
+module Britebox
+  # Keep current status for all processing FileJobs
+  class FileJobPool
+    attr_reader :file_jobs
+
+    SPINNERS = ["|", "/", "—", "\\"]
+    THREAD_NUM_DEFAULT = 8
+
+    def initialize(num_threads = nil)
+      num_threads ||= THREAD_NUM_DEFAULT
+      @file_jobs = []
+      @refresh_number = 0
+      @fj_pool = ThreadStorm.new size: num_threads
+      @lines_pool = ThreadStorm.new size: num_threads
+    end
+
+    def add(file_job)
+      return if include? file_job.file_name
+      @file_jobs << file_job
+    end
+
+    def include?(file_name)
+      @file_jobs.map{ |fj| fj.file_name }.include? file_name
+    end
+
+    def process_file!(file, dir, out_dir, brite_client, fj_options)
+      file_name = File.expand_path(file, dir)
+
+      # Do not process same files twice
+      return if self.include? file_name
+
+      fj = FileJob.new(file_name, brite_client, @lines_pool, fj_options)
+      self.add(fj)
+      @fj_pool.execute do
+        fj.verify!
+
+        case fj.status
+          when 'error'
+            err_name = File.basename(file, File.extname(file)) + '_error' + File.extname(file)
+            File.rename file_name, File.expand_path(err_name, out_dir)
+            File.open(File.expand_path(err_name + '.log', out_dir), 'w+') do |f|
+              f.write fj.error # TODO: add extended error log
+            end
+          when 'exporting'
+            if fj.export! File.expand_path(file, out_dir)
+              File.delete file_name
+            end
+          else
+            raise "unexpected error, status: #{fj.status}"
+        end
+      end
+    end
+
+
+
+    def print_status
+      clear = "\e[K"
+      up = "\e[A"
+
+      if @prev_buffer_height
+        print "\r#{clear}#{up}" * @prev_buffer_height
+      end
+
+      buffer =  "\nFiles processing: #{@file_jobs.count}"
+
+      @file_jobs.each do |fj|
+        fname = File.basename(fj.file_name)
+        if fname.length > 20
+          fname = fname[0..16] + '...'
+        end
+        if fj.status == 'error'
+          status_str = fj.error
+        else
+          status_str = "#{(fj.lines_processed || '-')} / #{fj.lines_total || '-'}"
+        end
+
+        buffer << "\n#{fname.ljust(20)} | #{fj.status.to_s.ljust(9)} | #{status_str}"
+      end
+
+      buffer << "\n #{SPINNERS[@refresh_number % 4]}\n"
+
+      print buffer
+
+      @prev_buffer_height = buffer.count("\n")
+      @refresh_number += 1
+    end
+
+
+  end
+end

data/lib/britebox/version.rb

@@ -0,0 +1,3 @@
+module Britebox
+  VERSION = '0.0.1'
+end

metadata

@@ -0,0 +1,95 @@
+--- !ruby/object:Gem::Specification
+name: britebox
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Alexander Shapiotko
+- BriteVerify
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-07-07 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: brite-api
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 0.0.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 0.0.1
+- !ruby/object:Gem::Dependency
+  name: thread_storm
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.7.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.7.1
+- !ruby/object:Gem::Dependency
+  name: listen
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 0.0.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 0.0.1
+description: BriteVerify API CLI tool
+email:
+- support@briteverify.com
+executables:
+- britebox
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/britebox/cli.rb
+- lib/britebox/file_job.rb
+- lib/britebox/file_job_pool.rb
+- lib/britebox/version.rb
+- lib/britebox.rb
+- README.md
+- LICENSE
+- bin/britebox
+homepage: https://github.com/thousandsofthem/britebox
+licenses: []
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '1.9'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: 1.3.6
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.0.0
+signing_key: 
+specification_version: 4
+summary: BriteVerify API CLI tool
+test_files: []