britebox 0.0.3 → 0.0.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: ae190d938423b74e7df0f20150c509aef201a73f
-  data.tar.gz: ef9bc4daf6cb23fe62984a4b55a5ce63315c9235
+  metadata.gz: 828f33e21107bef719e0cfe79663172057ae4191
+  data.tar.gz: f073980c5916b567d366e502f6830c8c4d233672
 SHA512:
-  metadata.gz: f9f7b8ac141941117e93326f84bb93cda6615b9bc7d2247f97767bc9f03e4e395b958a4884a7dcba5c4eb11801e46da1f3f8362627a1085caa447d128c10f764
-  data.tar.gz: 6f5222d89f56495be4404e8f7907272573e372d93996914e43a5d1e9cbc45c54310bcf44c2986083b2f410bfebf4d576e079936814c93f58dc2bbd3d2b9a1521
+  metadata.gz: 98f40a0354d69a0d3d8f210da9f6835923aa4e6daf75986843e8b1700e84cec8b403395d84e5a6e11666f80a8a0f2a0dbf84e7c0148f2587c4456ba8b6a1e70a
+  data.tar.gz: 62e5d74ced143d6bb1d7d6f9b9739736ad6eea7463a41b83a9cd434e0729fb7d10fc75b95dadde58eea48dbc6627d753284d9548b2429bf8ae2dc14cbec33bf9

data/README.md

@@ -1,4 +1,63 @@
-britebox
-========
+britebox is CLI tool for batch email verification via BriteVerify API
+
+# Installation
+
+Type in terminal:
+
+```sh
+$ gem install britebox
+```
+
+Note: BriteVerify API key is required to use this gem
+
+# Usage
+
+At this moment only "watch" mode is supported. In this mode cli tool will watch target directory for csv and txt files with email column inside (exact column, column separator etc will be autodetected)
+
+There are number of different configuration options supported, to preview them all type
+
+```sh
+$ britebox watch --help
+```
+
+Example:
+
+```sh
+$ britebox watch --apikey api-key --dir /home/user/files
+```
+
+If you tired typing api key and other settings over and over again, use `--save` flag.
+Now all provided settings will be stored in ~/.britebox.json and will be loaded by default,
+just type `britebox watch` next time.
+
+Example:
+
+```sh
+$ britebox watch --apikey api-key --dir /home/user/files --threads 5 --save
+...
+Ctrl-C
+...
+$ britebox watch
+```
+
+# Web UI mode
+
+Launch tool with key `--webui` to enable web interface.
+
+```sh
+$ britebox watch --apikey api-key --dir /home/user/files --webui
+Britebox web interface up and running, open http://localhost:7000/ in browser to see it
+
+Watching directory /home/user/files
+Output directory is /home/user/filesverified
+Press Ctrl-C to quit
+```
+
+![screenshot](http://i.imgur.com/Zl1iAVf.png)
+
+
+
+
+
+
 
-BriteVerify API CLI tool

data/bin/britebox

@@ -8,7 +8,35 @@ Thread.abort_on_exception = true
 THREAD_NUM_MAX = 10
 
 # Register action on Ctrl-C
-trap("SIGINT") { exit! }
+[:SIGINT, :TERM, :INT].each do |sig|
+  trap(sig) { exit! }
+end
+
+# Hack - allow processing files with weird characters inside
+class String
+  alias_method :orig_sub!, :sub!
+  def sub!(*args)
+    begin
+      orig_sub!(*args)
+    rescue ArgumentError
+      encode!("UTF-16BE", undef: :replace, invalid: :replace, replace: "")
+      encode!('UTF-8')
+      orig_sub!(*args)
+    end
+  end
+end
+
+# Hack - fix wdm issues on windows
+module Listen
+  module Adapters
+    class Windows
+      def self.usable?
+        false
+      end
+    end
+  end
+end
+
 
 commands = ['watch']
 
@@ -31,34 +59,56 @@ end
 
 @cli = Britebox::CLI.new
 
+Britebox::Config.load
+
 command = $*.shift
 if command == 'watch'
   params = {}
-  options = {}
 
   opts = OptionParser.new do |opts|
     opts.banner = "usage: britebox watch [OPTIONS]"
 
     opts.on('-d', '--dir WATCHDIR', 'Watch this directory for incoming files') do |v|
-      params[:dir] = v
+      Britebox::Config.watch_dir = v
     end
 
     opts.on('-o', '--output OUTPUTDIR', 'Place verified files into this directory') do |v|
-      params[:output] = v
+      Britebox::Config.out_dir = v
     end
 
     opts.on('-k', '--apikey APIKEY', 'BriteVerify API Key') do |v|
-      params[:api_key] = v
+      Britebox::Config.api_key = v
     end
 
     opts.on('-t', '--threads THREADS', "Maximum number of parallel threads used for processing, "+
-      "Default is #{Britebox::FileJobPool::THREAD_NUM_DEFAULT}, " +
+      "Default is #{Britebox::Config::DEFAULT_OPTIONS[:threads]}, " +
       "Maximum is #{THREAD_NUM_MAX}") do |v|
       unless (1..THREAD_NUM_MAX).include? v.to_i
         puts "Threads number should be in range 1..#{THREAD_NUM_MAX}"
         exit 1
       end
-      params[:threads] = v.to_i
+      Britebox::Config.threads = v.to_i
+    end
+
+    opts.on('--save', 'Save options to config file') do |v|
+      params[:save] = true
+    end
+
+    opts.on('-p', '--port PORT', 'Set port number for web interface, ' +
+      "Default is #{Britebox::Config::DEFAULT_OPTIONS[:port]}") do |v|
+      unless (1..65535).include? v.to_i
+        puts "Port number should be in range 1..65535"
+        exit 1
+      end
+      Britebox::Config.port = v.to_i
+    end
+
+    opts.on('--ui', '--webui [on/off]', 'Enable Web UI') do |v|
+      Britebox::Config.ui_enabled = !['off', 'false', '0'].include?(v.to_s.downcase)
+    end
+
+    opts.on('--simulate [on/off]', 'Do not send requests to api, fake verification') do |v|
+      Britebox::Config.simulate = !['off', 'false', '0'].include?(v.to_s.downcase)
     end
 
     common_opts(opts)
@@ -71,7 +121,12 @@ if command == 'watch'
     exit 1
   end
 
-  @cli.watch(params)
+  if params[:save]
+    Britebox::Config.save
+  end
+
+
+  @cli.watch
 
 
 end

data/lib/britebox.rb

@@ -1,6 +1,10 @@
 
 require 'britebox/version'
 require 'britebox/helpers'
+require 'britebox/config'
 require 'britebox/file_job'
 require 'britebox/file_job_pool'
-require 'britebox/export_buffer'
+require 'britebox/export_buffer'
+require 'britebox/event_log'
+require 'britebox/web_ui'
+require 'britebox/filtered_error_io'

data/lib/britebox/cli.rb

@@ -9,40 +9,60 @@ module Britebox
 
     end
 
-    def watch(params)
-      @api_key = params[:api_key] || raise("Please provide BriteVerify API Key")
+    def watch
+      # Start WebUI
+      if Britebox::Config.ui_enabled
+        with_sanitized_stderr do
+          Thread.new do
+            logger = Logger.new(STDOUT)
+            logger.level = Logger::ERROR
+            Britebox::WebUI.run!(port: Britebox::Config.port, server_settings: {Logger: logger, AccessLog: []})
+            exit!
+          end
+        end
+        puts "Britebox web interface up and running, open http://localhost:#{Britebox::Config.port}/ in browser to see it"
+        puts
+      end
+
 
-      if params[:dir]
-        @dir = File.expand_path params[:dir]
+      @api_key = Config.api_key || raise("Please provide BriteVerify API Key")
+
+      if Config.watch_dir
+        @dir = File.expand_path Config.watch_dir
       else
         raise("Please provide directory-to-watch")
       end
 
       raise("Directory #{@dir} does not exist") unless File.exists? @dir
 
-      if params[:output]
-        @out_dir = File.expand_path params[:output]
+      if Config.out_dir
+        @out_dir = File.expand_path Config.out_dir
       else
         @out_dir = File.expand_path 'verified', @dir
       end
+      Config.out_dir = @out_dir
 
       FileUtils.mkdir_p(@out_dir) unless File.exists? @out_dir
 
-      fj_options = {threads: params[:threads]}
-
       puts "Watching directory #{@dir}"
       puts "Output directory is #{@out_dir}"
+
+      if Britebox::Config.simulate
+        EventLog.add 'WARN', "Running in simulation mode. No real requests to API will be made."
+      end
+
       puts "Press Ctrl-C to quit"
+      puts
 
-      fj_pool = FileJobPool.new(params[:threads])
+      $fj_pool = FileJobPool.new
 
-      options = {filter: /\.(#{MONITOR_EXTENSIONS.join('|')})$/, ignore: /\//}
+      options = {filter: /\.(#{MONITOR_EXTENSIONS.join('|')})$/, ignore: /\//, polling_fallback_message: false}
 
       brite_client = BriteAPI::Client.new(@api_key)
 
       Listen.to(@dir, options) do |modified, added, removed|
         (modified + added).each do |file|
-          fj_pool.process_file!(file, @dir, @out_dir, brite_client, fj_options)
+          $fj_pool.process_file!(file, @dir, @out_dir, brite_client)
         end
       end
 
@@ -53,12 +73,23 @@ module Britebox
 
       # Update current status in cycle
       loop do
-        fj_pool.print_status
+        $fj_pool.print_status
         sleep 0.5
       end
+    end
 
+    private
 
-
+    def with_sanitized_stderr
+      $stderr = Britebox::FilteredErrorIO.new($stderr)
+      begin
+        yield
+      ensure
+        Thread.new do
+          sleep(1)
+          $stderr.restore!
+        end
+      end
     end
   end
 end

data/lib/britebox/config.rb

@@ -0,0 +1,91 @@
+require 'json'
+require 'singleton'
+
+module Britebox
+  class Config
+    include Singleton
+
+    OPTIONS = [:threads, :api_key, :watch_dir, :out_dir, :port, :ui_enabled, :simulate]
+
+    DEFAULT_OPTIONS = {
+      threads: 10,
+      port: 7000
+    }
+
+    CONFIG_PATH = '~/.britebox.json'
+
+    def initialize
+      # load config file
+      init_defaults
+    end
+
+    def self.load
+      data = read_config
+
+      if data && data['config']
+        data['config'].each do |k, v|
+          self.instance.send("#{k}=", v) if self.instance.respond_to?(k)
+        end
+        true
+      else
+        false
+      end
+    end
+
+    def self.read_config
+      path = File.expand_path(CONFIG_PATH)
+      if File.exists? path
+        JSON.parse File.read path rescue nil
+      end
+    end
+
+    def self.values
+      data = {}
+      OPTIONS.each do |k|
+        data[k] = self.instance.send(k)
+      end
+      data
+    end
+
+    def self.save
+      path = File.expand_path(CONFIG_PATH)
+
+      data = read_config
+      data ||= {}
+      data['config'] ||= {}
+      OPTIONS.each do |key|
+        data['config'][key.to_s] = self.instance.send(key)
+      end
+      begin
+        File.open(path, 'w+') do |file|
+          file.write JSON.pretty_generate data
+        end
+        true
+      rescue Exception => ex
+        puts "Can't save config: #{ex.message}"
+        false
+      end
+    end
+
+    def init_defaults
+      DEFAULT_OPTIONS.each do |key, val|
+        send("#{key}=", val) if send(key).nil?
+      end
+    end
+
+    OPTIONS.each do |key|
+      attr_accessor key
+
+      define_singleton_method key do
+        self.instance.send(key)
+      end
+
+      define_singleton_method "#{key}=" do |val|
+        self.instance.send("#{key}=", val)
+      end
+    end
+
+
+
+  end
+end

data/lib/britebox/event_log.rb

@@ -0,0 +1,35 @@
+require 'singleton'
+require 'ostruct'
+
+module Britebox
+  class EventLog
+    include Singleton
+    attr_accessor :log_lines
+
+    def initialize
+      @log_lines = []
+    end
+
+    def self.empty?
+      self.instance.log_lines.empty?
+    end
+
+    def self.push(event_type, message)
+      event = OpenStruct.new(type: event_type, message: message, time: Time.now)
+      self.instance.log_lines.push(event)
+
+      # if Britebox::Config.log ...
+      # save event to log
+
+      true
+    end
+
+    def self.pop
+      self.instance.log_lines.pop
+    end
+
+    class << self
+      alias_method :add, :push
+    end
+  end
+end

data/lib/britebox/file_job.rb

@@ -14,7 +14,7 @@ module Britebox
       @file_name = file_name
       @brite_client = brite_client
 
-      @threads_count = options[:threads] || Britebox::FileJobPool::THREAD_NUM_DEFAULT
+      @threads_count = options[:threads]
       @queue = options[:queue]
 
       @status = 'pending'
@@ -33,6 +33,26 @@ module Britebox
       @semaphore = Mutex.new
     end
 
+    def as_json
+      {
+        id: id,
+        file_name: id,
+        status: @status,
+        threads: @threads_count,
+        size_total: @size_total,
+        size_processed: @size_processed,
+        started_at: @started_at,
+        processed_at: @processed_at,
+        duration: duration,
+        percent_complete: percent_complete,
+        error: @error
+      }
+    end
+
+    def id
+      File.basename(@file_name)
+    end
+
     def buffer_size
       @threads_count * 4
     end
@@ -52,8 +72,43 @@ module Britebox
       end
     end
 
+    def cancelled?
+      status == 'cancelled'
+    end
+
+    def verifying?
+      status == 'verifying'
+    end
+
+    def paused?
+      status == 'paused'
+    end
+
+    def pending?
+      status == 'pending'
+    end
+
+    def error?
+      status == 'error'
+    end
+
+    def status=(new_status)
+      return if status == new_status
+
+      # Release processing slot
+      if verifying? && ['paused', 'cancelled'].include?(new_status)
+        @queue.push(:flag) if @queue
+      end
+
+      if new_status == 'verifying'
+        raise("status 'verifying' can't be set, use 'pending' instead")
+      end
+
+      @status = new_status
+    end
+
     def verify!(file_name_to)
-      return if @status == 'error'
+      return if error?
 
       test_lines = []
       begin
@@ -88,17 +143,39 @@ module Britebox
       @threads_count.times do
         @threads << Thread.new do
           loop do
+            # Pause all processing
+            loop do
+              break unless paused?
+              sleep(0.1)
+            end
+
+            break if cancelled?
+
+            # Wait for processing slot
+            if pending?
+              @semaphore.synchronize do
+                @queue.pop if @queue && pending?
+                @status = 'verifying'
+              end
+            end
+
             item = Timeout.timeout(1) { @in_buffer.pop } rescue nil
             break if item.nil?
 
             email = item[:line][@email_index]
             if email.to_s.match(EMAIL_PATTERN)
-              begin
-                contact = @brite_client.contacts.create(email: email)
-                contact.verify!
-                contact_status = [contact.status, contact.response[:email]['disposable'], contact.response[:email]['role_address']]
-              rescue
-                contact_status = ['error', false, false]
+              if Britebox::Config.simulate
+                # Do not send real requests in this mode
+                sleep(1)
+                contact_status = ['unknown', false, false]
+              else
+                begin
+                  contact = @brite_client.contacts.create(email: email)
+                  contact.verify!
+                  contact_status = [contact.status, contact.response[:email]['disposable'], contact.response[:email]['role_address']]
+                rescue
+                  contact_status = ['error', false, false]
+                end
               end
             else
               contact_status = ['invalid', false, false]
@@ -120,7 +197,8 @@ module Britebox
         end
         next if line.nil? || line.size == 0
         # Throttle file reading
-        sleep(0.1) while @in_buffer.size > buffer_size
+        break if cancelled?
+        sleep(0.1) while @in_buffer.size > buffer_size && !cancelled?
 
         @in_buffer << {n: idx, line: line}
         idx += 1
@@ -132,7 +210,12 @@ module Britebox
       @out_buffer.close
 
       @processed_at = Time.now
-      @status = 'complete'
+
+      if cancelled?
+        File.delete file_name_to
+      else
+        @status = 'complete'
+      end
 
       # Release the lock
       @queue.push(:flag) if @queue