broker 0.1.1 → 0.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 06703642e42f471850e220882e24d2e0b3cd54b7
-  data.tar.gz: 0996bd3519546efc94b153e53ba5e42da45e6570
+  metadata.gz: 616efefdc628ec749aad198ade1c21133dc7081a
+  data.tar.gz: c848d5e2c0460677b12f8a57e153216bf917dcfc
 SHA512:
-  metadata.gz: 18fdb75b3817fa85cfb7f7a23b0daa644fcfcc2bf471bc0a4ee61914d253751dc5abaa813322f406481867a838687c4301feb34d737790d9167ba49fc0e63863
-  data.tar.gz: 463b7df4a4699a1c03295579ce154b88841bfb57cf5d26c4d85afd52c577b51668b9234f6b5ddb5c5aab6eb53424056d9ddf8b5c8e48c8770785720bf78d916a
+  metadata.gz: 3ed6ed4619c2369e1ac87a077327c70def073b2c352b010da89fc8e09b4d1ba15493a05cdce4971234c06134adba9fac3e757e39f27aa31aa1f5ef5896aaf0c8
+  data.tar.gz: 75c5eaae7ccba5d8291a43fb914497f1cfeb215162b17b9b0c65cd4568365d5aa8ed355badf5fd3602bc47d70d3b8d8bf235998007bacf6dde0bc2eb3be3fd42

data/README.md

@@ -22,7 +22,7 @@ Or install it yourself as:
     $ gem install broker
 ```
 
-## Setup
+## Initial Setup
 
 Setup the initializers, config files, and structure by:
 
@@ -34,9 +34,78 @@ The following files will be generated for you:
 
 ```ruby
 /config/secrets.yml
-/config/quickbase_tables
+/config/quickbase_tables.yml
 /config/initializers/broker.rb
-````
+```
+
+Enter your Quickbase connection credentials in `/config/secrets.yml` file.
+
+```ruby
+ # Place your quickbase credentials here
+ # Do not check into version control, keep your secrets safe
+ #
+ # ORG -> if you are using a custom subdomain such as mycompany.quickbase.com,
+ # you will set ORG:  mycompany, otherwise www is the default
+ #
+ #
+ ORG:  www
+ USERNAME:  billy_the_kid@user.com
+ PASSWORD:  mickeymouse
+```
+
+## Generate the Queue
+
+Once running, Broker will poll for new files in a directory structure that resembles your Quickbase apps and tables. Before starting Broker up, you need to generate the Queue.
+
+1. Edit the initializer file in `/config/initializers/broker.rb` to customize the location you want the Queue to be generated, along with changing the default locations for your secrets.yml and quickbase_tables.yml to be.
+
+2. Open up your `quickbase_tables.yml` configuration file and enter your quickbase app/table structure. This convention must be strictly followed as it is the way Broker is able to connect with Quickbase. Create as many app config blocks as you need.
+
+```ruby
+tracker:									# Give each of your apps a simple unique key name
+  name: Job Tracker							# This is the official Quickbase App Name
+  token:  your_apps_token_for_tracker		# Quickbase API token you assigned to the app
+  tables:									# Generic tables key that points to your tables
+    main:  table_dbid						# Give each of your tables a simple unique key name, pointing to the Quickbase table dbid
+    people:  table_dbid
+```
+
+3. Once your `quickbase_table.yml` configuration is done, you are ready to generate the Broker Queue.
+
+```ruby
+$ broker queue -c
+
+$ broker queue -u
+```
+
+Your Queue has been created using your `quickbase_tables.yml` settings.
+
+```ruby
+broker_queue/tracker/main
+broker_queue/tracker/people
+
+broker_processed
+```
+
+## Boot Broker
+
+```ruby
+$ broker start -s
+```
+
+Broker will be running and watching for new files inside your Queue. New files will be swept up, imported to Quickbase, and then moved out to the Processed folder.
+
+
+## In the Works
+
+1. Broker activity - successful/failed import records save to a MongoDB
+
+2. Web UI to monitor Broker activity
+
+3. Mount Broker to a Rails App
+
+4. 
+
 
 ## Contributing
 

data/lib/broker.rb

@@ -3,7 +3,7 @@ require 'broker/version'
 require 'yaml'
 
 module Broker
-  NAME = "Broker: Quickbase Data Intake"
+  NAME = "Broker"
   
   DEFAULTS = {
     secrets_path: 'config/secrets.yml',
@@ -12,6 +12,7 @@ module Broker
     poll_interval: 300,
     queue: 'broker_queue',
     processed_path: 'broker_processed',
+    failed_path: 'broker_failed',
     file_ext: :csv,
     enqueued: []
   }
@@ -96,12 +97,16 @@ module Broker
     options[:processed_path] = pa
   end
   
+  def self.failed_path=(pa)
+    options[:failed_path] = pa
+  end
+  
   def self.poll_interval=(val)
     options[:poll_interval] = val
   end
   
-  def file_ext=(val)
-    options[:file_ext] = val if [:csv, :tab].include? val
+  def self.file_ext=(val)
+    options[:file_ext] = val if [:csv, :txt].include? val
   end
   
 end

data/lib/broker/application.rb

@@ -6,11 +6,11 @@ end
 
 module Broker
   class Application
-    
+
     def self.boot!      
       puts "starting up"
       puts "watching for files in #{Broker.options[:queue]}"
     end
     
-  end
-end
+  end    
+end

data/lib/broker/cli.rb

@@ -154,9 +154,17 @@ module Broker
       say ""
       say "*************************************************"
       say ""
-      say "Update your Broker Q by running:"
+      say "Create your Broker Q by running:"
       say ""
-      say "$ broker update_q"
+      say "$ broker queue -c"
+      say ""
+      say "Populate your Broker Q by running:"
+      say ""
+      say "$ broker queue -u"
+      say ""
+      say "Boot Broker"
+      say ""
+      say "$ broker start -s"
       say ""
     end
     
@@ -194,6 +202,5 @@ module Broker
       end && true
     end
     
-    
   end
 end

data/lib/broker/client/QuickBaseClient.rb

@@ -1499,6 +1499,7 @@ class Client
    
    # shenley - gem's method was flawed and removed nil values, used CSV lib to separate correctly
    def splitString(string, fieldSeparator = ",")
+     string &&= string.encode("UTF-8", "binary", :invalid => :replace, :undef => :replace, :replace => "")
      CSV.parse(string, :col_sep => fieldSeparator).shift
    end
    
@@ -4475,8 +4476,12 @@ class Client
                   csvdata = ""
                   validLines.each{ |line| 
                      if line 
-                        csvdata << line.join( ',' ) 
-                        csvdata << "\n"
+                       csvdata << line.to_csv
+                        # these 2 lines don't handle columns with commas, ex "My field, needs escaped"
+                        # using CSV library's to_csv will correctly escape any quoted fields to preserve the commas 
+                        # and not mistake them for wrong fields
+                        #csvdata << line.join( ',' ) 
+                        #csvdata << "\n"
                      end
                   }
                   clist = targetFieldIDs.join( '.' )

data/lib/broker/export.rb

@@ -1,7 +1,7 @@
 require 'broker/qb/session'
 
 module Broker
-  class Export < Broker::QB::Session
+  class Export < Broker::Session
     
    def initialize(opt={})
      super(opt)

data/lib/broker/launcher.rb

@@ -25,15 +25,18 @@ module Broker
       while !@queue.empty?
         payload = @queue.next
         results = payload.commit(session)
-        puts "#{results.inspect}"
-        @queue.success(payload)
+        results ? @queue.success(payload) : @queue.failure(payload)        
         # Need to handle failure and add payload to queues failed
         # Need to handle success and move file out and alert queue
       end
       puts "Session Terminated: #{session.sign_out}"
+      puts "****************************"
+      puts "Queue Status"
+      puts "****************************"
       puts "Processed: #{@queue.processed}"
       puts "Pending: #{@queue.pending.inspect}"
       puts "Failed: #{@queue.failed.inspect}"
+      puts "****************************"
     end
     
   end

data/lib/broker/payload.rb

@@ -3,7 +3,7 @@ require 'ostruct'
 module Broker
   class Payload
     
-    attr_reader :pkg
+    attr_accessor :pkg, :results
     
     class << self
     
@@ -36,6 +36,7 @@ module Broker
     end
     
     def initialize(opt={})
+      @response = nil
       @pkg = OpenStruct.new(file: opt[:file],
                             dbid: opt[:dbid],
                             app: opt[:app],
@@ -44,7 +45,42 @@ module Broker
     end
    
     def commit(session)
-      session.fire_event(self)
+      begin
+        capture_response(session.fire_event(self))
+      rescue => e
+        # Need to log the failed attempt somewhere
+        puts "#{pkg.file} failed to import!"
+        puts e.message
+        puts e.backtrace.join("\n")
+        capture_response("Failed to Import")
+        return nil
+      end
+      true
+    end
+    
+    def capture_response(results)
+      if results.respond_to?(:capitalize)
+        @response = results
+      else
+        parse_response(results)
+        puts "Captured Payload Result: #{@response.inspect}"
+      end
+    end
+    
+    # Quickbase API returns 2D array with results
+    # First Element -> [num created, num imported, num updated, raw xml, update_id]
+    # Second Element -> [invalid records that didn't get imported]
+    
+    def parse_response(results)
+      details = results.shift
+      res = {
+        records_imported: details[1],
+        records_created: details[0],
+        records_updated: details[2],
+        update_id: details[4],
+        invalid_records: results.shift
+      }
+      @response = res
     end
     
   end

data/lib/broker/queue.rb

@@ -1,7 +1,9 @@
 require 'broker/payload'
+require 'broker/utility'
 
 module Broker
   class Queue
+    include Broker::Utility
     
     attr_reader :processed, :pending, :failed
     
@@ -25,18 +27,19 @@ module Broker
     
     def failure(payload)
       @failed << payload
+      move(payload, Broker.options[:failed_path])
     end
     
     def success(payload)
       @processed +=1
-      move payload
+      move(payload, Broker.options[:processed_path])
     end
     
     private
     
-    def move(payload)
-      f = File.basename(payload.pkg.file)
-      FileUtils.mv payload.pkg.file, File.join(Broker.options[:processed_path],"/", f)
+    def move(payload, dest)
+      f = "#{timestamp}_#{File.basename(payload.pkg.file)}"
+      FileUtils.mv payload.pkg.file, File.join(dest, "/", f)
     end
     
   end

data/lib/broker/session.rb

@@ -1,66 +1,64 @@
 require 'broker/client/quickbase_client'
 
 module Broker
- # module QB
-    class Session
-      
-      attr_reader :client, :app, :ext
+  class Session
     
-      def initialize(opt={})
-        @app = Broker.lookup_appname(opt[:app])
-        @ext = Broker.options[:file_ext]
-      
-        credentials = {
-          "username" => Broker.secrets['USERNAME'],
-          "password" => Broker.secrets['PASSWORD'],
-          "appname"  => @app,
-          "org"      => Broker.secrets['ORG'],
-          "apptoken" => opt[:token] || Broker.secrets['TOKEN']
-        }
-        
-        begin
-          @client = QuickBase::Client.init(credentials)
-        rescue => e
-          puts e.message
-          #raise ArgumentError
-        ensure
-          # We successfully logged into quickbase, but supplied an invalid app name
-          if @client && @client.errcode == "32"
-            @client.signOut
-            raise ArgumentError
-          end
-        end
-      end
-      
-      def fire_event
-        raise NotImplementedError
-      end
-      
-      def sign_out
-        @client.signOut
-      end
+    attr_reader :client, :app, :ext
+  
+    def initialize(opt={})
+      @app = Broker.lookup_appname(opt[:app])
+      @ext = Broker.options[:file_ext]
+    
+      credentials = {
+        "username" => Broker.secrets['USERNAME'],
+        "password" => Broker.secrets['PASSWORD'],
+        "appname"  => @app,
+        "org"      => Broker.secrets['ORG'],
+        "apptoken" => opt[:token] || Broker.secrets['TOKEN']
+      }
       
-      def qb_ready?(name)
-        app_name = Broker.lookup_appname(name)
-        unless app_name == @client.dbname
-          return app_name && change_app(app_name, name)
+      begin
+        @client = QuickBase::Client.init(credentials)
+      rescue => e
+        puts e.message
+        #raise ArgumentError
+      ensure
+        # We successfully logged into quickbase, but supplied an invalid app name
+        if @client && @client.errcode == "32"
+          @client.signOut
+          raise ArgumentError
         end
-        true
       end
+    end
     
-      def field_names(table)
-        table &&= table.to_s
-        db = Broker.tables[@app]['tables'][table]
-        db && @client.getFieldNames(db, "", true)
-      end
-      
-      private
-      
-      def change_app(app_name, app_key)
-        @app = app_key.to_s
-        @client.findDBByname(app_name)
-        @client.dbname == app_name
+    def fire_event
+      raise NotImplementedError
+    end
+    
+    def sign_out
+      @client.signOut
+    end
+    
+    def qb_ready?(name)
+      app_name = Broker.lookup_appname(name)
+      unless app_name == @client.dbname
+        return app_name && change_app(app_name, name)
       end
+      true
+    end
+  
+    def field_names(table)
+      table &&= table.to_s
+      db = Broker.tables[@app]['tables'][table]
+      db && @client.getFieldNames(db, "", true)
+    end
+    
+    private
+    
+    def change_app(app_name, app_key)
+      @app = app_key.to_s
+      @client.findDBByname(app_name)
+      @client.dbname == app_name
     end
- # end
+  end
 end

data/lib/broker/templates/broker.rb

@@ -14,8 +14,11 @@ Broker.setup do |config|
   # Directory where your successful imported files will be moved
   config.processed_path = 'broker_processed'
   
+  # Directory where your failed imported files will be moved
+  config.failed_path = 'broker_failed'
+  
   # Uncomment to change the default file type to use for importing
-  # [:csv, :tab]
+  # [:csv, :txt]
   #config.file_ext = :csv
   
   # Sets polling wait time before checking for new import files

data/lib/broker/templates/secrets.yml

@@ -8,3 +8,4 @@
 ORG:  www
 USERNAME:  billy_the_kid@user.com
 PASSWORD:  mickeymouse
+TOKEN:  your_main_app_token_in_quickbase

data/lib/broker/utility.rb

@@ -12,6 +12,10 @@ module Broker
        watcher(name, &block)
      end
    end
+   
+   def timestamp
+     Time.now.strftime("%Y-%m-%dT%H.%M.%S")
+   end
     
   end
 end

data/lib/broker/version.rb

@@ -1,3 +1,3 @@
 module Broker
-  VERSION = "0.1.1"
+  VERSION = "0.1.2"
 end

data/lib/broker/web_routes.rb

@@ -1,5 +1,5 @@
-require 'broker/import'
-require 'broker/export'
+# require 'broker/import'
+# require 'broker/export'
 
 module Broker
   module WebRoutes
@@ -13,16 +13,17 @@ module Broker
       end
     
       app.get '/exports' do
-        db = params['table']
-        app = params['app']
+        # db = params['table']
+#         app = params['app']
+#         @fields = []
+#         @header = "Exports"
+#         if db && app
+#           qb = Broker::Export.new(:app => app)
+#           @fields = qb.get_field_names(db)
+#           @header = "App: #{app.downcase} | Table: #{db.capitalize}"
+#           qb.sign_out
+#         end
         @fields = []
-        @header = "Exports"
-        if db && app
-          qb = Broker::Export.new(:app => app)
-          @fields = qb.get_field_names(db)
-          @header = "App: #{app.downcase} | Table: #{db.capitalize}"
-          qb.sign_out
-        end
         erb :exports
       end
       

data/web/views/api.erb

@@ -1,14 +1,14 @@
 <% Broker.tables.each do |table, vals| %>
 <div class="row header">
   <div class="col-sm-12">
-    <h3><span class="title"><%= vals['name'] %></span> URL: <%= "/import/#{table}/[table_name]" %></h3>
+    <h3><span class="title"><%= vals['name'] %></span></h3>
 	<h4>Token: <%= vals['token'] %></h4>
 	<table class="workers table table-hover table-bordered table-striped table-white">
 		<thead>
 				<tr>
 					<th>Table Name</th>
 					<th>ID #</th>
-					<th>View Fields</th>					
+					<th>Broker Path</th>					
 				</tr>
 		</thead>
 		<tbody>
@@ -16,7 +16,7 @@
 			<tr>
 				<td><%= key %></td>
 				<td><%= val %></td>
-				<td><a href="/exports?app=<%= table %>&table=<%= key %>">View</a></td>
+				<td><%= "#{Broker.options[:queue]}/#{table}/#{key}" %></td>
 			</tr>
 			<% end %>
 		</tbody>

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: broker
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.1.2
 platform: ruby
 authors:
 - Shawn Henley
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-11-21 00:00:00.000000000 Z
+date: 2015-11-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler