broker 0.0.9 → 0.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 22906356dab990f2e1791980fd4259d5c915c1e1
-  data.tar.gz: b509f861be4137165f8b92e5f7667dbb891fdca7
+  metadata.gz: 06703642e42f471850e220882e24d2e0b3cd54b7
+  data.tar.gz: 0996bd3519546efc94b153e53ba5e42da45e6570
 SHA512:
-  metadata.gz: ed637b3ac16bbb1346fbee41f7bd52e456441cf4b74a47a4a2bfe66423bde82e6876b879657f841e13c1abaf925a25970cda27fe0b07f81e1fb6e9849c4ffd86
-  data.tar.gz: 12de6c2ad9ddc465aed4423e8c4d5eaec4f1bd503e9b49eb57384f029254bea79248b3e654452c86cab6166dd75bedd64da4bbfcbf1b0d03d71b4a259ad76923
+  metadata.gz: 18fdb75b3817fa85cfb7f7a23b0daa644fcfcc2bf471bc0a4ee61914d253751dc5abaa813322f406481867a838687c4301feb34d737790d9167ba49fc0e63863
+  data.tar.gz: 463b7df4a4699a1c03295579ce154b88841bfb57cf5d26c4d85afd52c577b51668b9234f6b5ddb5c5aab6eb53424056d9ddf8b5c8e48c8770785720bf78d916a

data/broker.gemspec

@@ -8,7 +8,7 @@ Gem::Specification.new do |gem|
   gem.version       = Broker::VERSION
   gem.authors       = ["Shawn Henley"]
   gem.email         = ["shawn@luckysailor.net"]
-  gem.description   = gem.summary = "Automate the importing and exporting of data with Quickbase applications, using post requests."
+  gem.description   = gem.summary = "Simple utility to automate the importing of data through the Quickbase API by polling a folder queue for new files, and pushing the data to your Quickbase apps."
   gem.homepage      = "https://github.com/luckysailor/broker"
   gem.license       = "MIT"
 

data/lib/broker/application.rb

@@ -9,7 +9,7 @@ module Broker
     
     def self.boot!      
       puts "starting up"
-      puts "watching for files in #{Broker.queue}"
+      puts "watching for files in #{Broker.options[:queue]}"
     end
     
   end

data/lib/broker/cli.rb

@@ -39,17 +39,18 @@ module Broker
                   banner: "Generate your queue folder structure"
     def queue
       require 'broker'
-      empty_dir = !Dir.exists?(Broker.queue)
+      empty_dir = !Dir.exists?(Broker.options[:queue])
       
       if options[:create] && empty_dir
-        FileUtils.mkdir Broker.queue
+        FileUtils.mkdir Broker.options[:queue]
+        FileUtils.mkdir Broker.options[:processed_path]
         say "Your Q has been created at:"
-        puts "#{Broker.queue}"
+        puts "#{Broker.options[:queue]}"
       end
       
       if options[:update] && !empty_dir
-        base_path = "#{Broker.queue}"
-        roots     = Broker.tables.keys
+        base_path = "#{Broker.options[:queue]}"
+        roots     = Broker.table_keys
         tables    = []
         
         make_tree(base_path, roots)

data/lib/broker/export.rb

@@ -1,7 +1,7 @@
-require 'broker/session'
+require 'broker/qb/session'
 
 module Broker
-  class Export < Broker::Session
+  class Export < Broker::QB::Session
     
    def initialize(opt={})
      super(opt)

data/lib/broker/finder.rb

@@ -0,0 +1,24 @@
+require 'broker/queue'
+
+module Broker
+  class Finder
+  
+    def initialize(q)
+      opt = Broker.options
+      @queue = q
+      @dir = File.join(opt[:queue], "**", "*.#{opt[:file_ext].to_s}")
+    end
+
+    def check
+    	files = get
+    	@queue.push(files) unless files.empty?
+    end
+
+    private
+
+    def get
+      Dir.glob(@dir)
+    end
+   
+  end
+end

data/lib/broker/import.rb

@@ -1,11 +1,22 @@
-require 'broker'
-require 'broker/client/quickbase_client'
+require 'broker/session'
 
 module Broker
-  class Import
-   
-   
-   
+  class Import < Broker::Session
+ 
+    def fire_event(payload)
+      @payload = payload
+      if qb_ready?(@payload.pkg.app_key) && respond_to?(ext)
+        return send(ext)
+      end
+    end
     
+    def csv
+      @client.importCSVFile(@payload.pkg.file, @payload.pkg.dbid)
+    end
+    
+    def txt
+      @client.importTSVFile(@payload.pkg.file, @payload.pkg.dbid)
+    end
+ 
   end
 end

data/lib/broker/launcher.rb

@@ -1,6 +1,43 @@
 require 'broker/utility'
+require 'broker/queue'
+require 'broker/import'
+require 'broker/finder'
 
 module Broker
+  module Event
+    def register(arr)
+      arr || return
+      puts "Event has been registered with: #{arr.inspect}"
+      unless arr.empty?
+        begin
+          session = Broker::Import.new(:app => Broker.any_app)
+        rescue ArgumentError => e
+          @failed = true
+          puts "Cant login to QB, invalid app name"
+          puts e.message
+        end
+        @failed || transport(session)
+      end
+      
+    end
+    
+    def transport(session)
+      while !@queue.empty?
+        payload = @queue.next
+        results = payload.commit(session)
+        puts "#{results.inspect}"
+        @queue.success(payload)
+        # Need to handle failure and add payload to queues failed
+        # Need to handle success and move file out and alert queue
+      end
+      puts "Session Terminated: #{session.sign_out}"
+      puts "Processed: #{@queue.processed}"
+      puts "Pending: #{@queue.pending.inspect}"
+      puts "Failed: #{@queue.failed.inspect}"
+    end
+    
+  end
+  
   class Launcher
     
    def initialize
@@ -23,11 +60,14 @@ module Broker
   
   class Poller
     include Utility
+    include Event
     
     def initialize
-      @finished = false
-      @wait_time = Broker.poll_interval
-      @folder = Broker.queue
+      @finished  = false
+      @wait_time = Broker.options[:poll_interval]
+      @folder    = Broker.options[:queue]
+      @queue     = Broker::Queue.new
+      @finder		 = Broker::Finder.new(@queue)
     end
     
     def terminate
@@ -46,7 +86,7 @@ module Broker
         pause_first
         
         while !@finished
-          puts "Checking for new files in #{@folder}"
+          register(@finder.check)
           wait
         end
       end

data/lib/broker/payload.rb

@@ -0,0 +1,51 @@
+require 'ostruct'
+
+module Broker
+  class Payload
+    
+    attr_reader :pkg
+    
+    class << self
+    
+      def parse(single_path)
+        
+        attrs = {
+          file: single_path,
+          dbid: nil,
+          app: nil,
+          app_key: nil,
+          table: nil
+        }
+        
+        # TODO: Rewrite!
+        # Sloppy, need a more elegant way to pop values off the stack
+        slots = single_path.split("/")
+        slots.pop
+        t = slots.pop
+        a = slots.pop        
+        
+        _id = Broker.lookup_tbid({app: a, table: t})
+        attrs[:dbid], attrs[:app], attrs[:app_key], attrs[:table] = _id, Broker.lookup_appname(a), a, t
+        
+        new(attrs)
+      end
+      
+      def chunk(arr)
+        3.times { yield arr.pop }
+      end
+    end
+    
+    def initialize(opt={})
+      @pkg = OpenStruct.new(file: opt[:file],
+                            dbid: opt[:dbid],
+                            app: opt[:app],
+                            app_key: opt[:app_key],
+                            table: opt[:table])                  
+    end
+   
+    def commit(session)
+      session.fire_event(self)
+    end
+    
+  end
+end

data/lib/broker/queue.rb

@@ -0,0 +1,43 @@
+require 'broker/payload'
+
+module Broker
+  class Queue
+    
+    attr_reader :processed, :pending, :failed
+    
+    def initialize
+      @processed = 0
+      @pending = []
+      @failed = []
+    end
+    
+    def push(paths)
+      paths.each { |p| @pending << Broker::Payload.parse(p) }
+    end
+    
+    def next
+      @pending.pop
+    end
+    
+    def empty?
+      @pending.empty?
+    end
+    
+    def failure(payload)
+      @failed << payload
+    end
+    
+    def success(payload)
+      @processed +=1
+      move payload
+    end
+    
+    private
+    
+    def move(payload)
+      f = File.basename(payload.pkg.file)
+      FileUtils.mv payload.pkg.file, File.join(Broker.options[:processed_path],"/", f)
+    end
+    
+  end
+end

data/lib/broker/session.rb

@@ -1,34 +1,66 @@
-require 'broker'
 require 'broker/client/quickbase_client'
 
 module Broker
-  class Session
-    
-    attr_reader :client, :app
+ # module QB
+    class Session
+      
+      attr_reader :client, :app, :ext
     
-    def initialize(opt={})
-      @app        = opt[:app]
+      def initialize(opt={})
+        @app = Broker.lookup_appname(opt[:app])
+        @ext = Broker.options[:file_ext]
       
-      credentials = {
-        "username" => Broker.secrets['USER'],
-        "password" => Broker.secrets['PASSWORD'],
-        "appname"  => @app,
-        "org"      => Broker.secrets['ORG'],
-        "apptoken" => opt[:token] || Broker.tables[@app]['token']
-      }
+        credentials = {
+          "username" => Broker.secrets['USERNAME'],
+          "password" => Broker.secrets['PASSWORD'],
+          "appname"  => @app,
+          "org"      => Broker.secrets['ORG'],
+          "apptoken" => opt[:token] || Broker.secrets['TOKEN']
+        }
+        
+        begin
+          @client = QuickBase::Client.init(credentials)
+        rescue => e
+          puts e.message
+          #raise ArgumentError
+        ensure
+          # We successfully logged into quickbase, but supplied an invalid app name
+          if @client && @client.errcode == "32"
+            @client.signOut
+            raise ArgumentError
+          end
+        end
+      end
       
-      @client = QuickBase::Client.init(credentials)
-    end
-    
-    def get_field_names(table)
-      table &&= table.to_s
-      db = Broker.tables[@app]['tables'][table]
-      db && @client.getFieldNames(db, "", true)
-    end
+      def fire_event
+        raise NotImplementedError
+      end
+      
+      def sign_out
+        @client.signOut
+      end
+      
+      def qb_ready?(name)
+        app_name = Broker.lookup_appname(name)
+        unless app_name == @client.dbname
+          return app_name && change_app(app_name, name)
+        end
+        true
+      end
     
-    def sign_out
-      @client.signOut
+      def field_names(table)
+        table &&= table.to_s
+        db = Broker.tables[@app]['tables'][table]
+        db && @client.getFieldNames(db, "", true)
+      end
+      
+      private
+      
+      def change_app(app_name, app_key)
+        @app = app_key.to_s
+        @client.findDBByname(app_name)
+        @client.dbname == app_name
+      end
     end
-    
-  end
+ # end
 end

data/lib/broker/templates/broker.rb

@@ -11,6 +11,9 @@ Broker.setup do |config|
   # Directory where your import files will be queued for import
   config.queue = 'broker_queue'
   
+  # Directory where your successful imported files will be moved
+  config.processed_path = 'broker_processed'
+  
   # Uncomment to change the default file type to use for importing
   # [:csv, :tab]
   #config.file_ext = :csv

data/lib/broker/version.rb

@@ -1,3 +1,3 @@
 module Broker
-  VERSION = "0.0.9"
+  VERSION = "0.1.1"
 end

data/lib/broker.rb

@@ -1,4 +1,5 @@
 require 'broker/version'
+#require 'broker/session'
 require 'yaml'
 
 module Broker
@@ -10,23 +11,30 @@ module Broker
     initializer: 'config/initializers/broker.rb',
     poll_interval: 300,
     queue: 'broker_queue',
-    file_ext: :csv
+    processed_path: 'broker_processed',
+    file_ext: :csv,
+    enqueued: []
   }
   
-  @@secrets_path    = @@secrets_path ||= DEFAULTS[:secrets_path]
-  @@tables_path 		= @@tables_path  ||= DEFAULTS[:tables_path]
-  @@initializer 		= @@initializer  ||= DEFAULTS[:initializer]
-  @@poll_interval		= @poll_interval ||= DEFAULTS[:poll_interval]
-  @@queue           = @@queue        ||= DEFAULTS[:queue]
-  @@file_ext        = @@file_ext     ||= DEFAULTS[:file_ext]
+  def self.load_config(dest)
+    YAML.load_file(dest) rescue {}
+  end
+  
+  @@secrets = @@secrets ||= load_config(DEFAULTS[:secrets_path])
+  @@tables  = @@tables  ||= load_config(DEFAULTS[:tables_path])
   
-  @@secrets   			= YAML.load_file(@@secrets_path) rescue {}
-  @@tables    			= YAML.load_file(@@tables_path) rescue {}
+  def self.options
+    @options ||= DEFAULTS.dup
+  end
   
   def self.lookup_tbid(opt={})
     tables[opt[:app]]['tables'][opt[:table]]
   end
   
+  def self.lookup_appname(key)
+    tables[key.to_s]['name']
+  end
+  
   def self.path
     Dir.pwd
   end
@@ -40,76 +48,62 @@ module Broker
   end
   
   def self.config_files
-    [secrets_path, tables_path, initializer]
+    fs = [:secrets_path, :tables_path, :initializer]
+    options.select { |opt| fs.include? opt }.values
   end
   
-  # def self.options
-#     @options ||= DEFAULTS.dup
-#   end
-#
-#   def self.options=(opt)
-#     @options = opt
-#   end
-  
-  def self.poll_interval
-    @@poll_interval
+  def self.secrets
+    @@secrets
   end
   
-  def self.poll_interval=(val)
-    @@poll_interval = val
+  def self.tables
+    @@tables
   end
   
-  def self.queue
-    @@queue
+  def self.table_keys
+    tables.keys
   end
   
-  def self.queue=(val)
-    @@queue = val
+  def self.any_app
+    table_keys.first
   end
   
-  def file_ext
-    @@file_ext
-  end
+  # Initializer setter methods
+  # Loaded at runtime in initializers/broker.rb
+  # Broker.setup do |config|
+  #   config.secrets_path = /my/updated/path.yml
+  # end
   
-  def file_ext=(val)
-    @@file_ext = val if [:csv, :tab].include? val
-  end
-  
-  def self.secrets
-    @@secrets
+  def self.initializer=(val)
+    options[:initializer] = val
   end
   
-  def self.secrets_path
-    @@secrets_path
+  def self.secrets_path=(pa)
+    options[:secrets_path] = pa
+    @@secrets = load_config(pa)
   end
   
-  def self.secrets_path=(val)
-    @@secrets_path = val
-    @@secrets   = YAML.load_file(@@secrets_path) rescue {}
+  def self.tables_path=(pa)
+    options[:tables_path] = pa
+    @@tables = load_config(pa)
   end
   
-  def self.tables
-    @@tables
+  def self.queue=(val)
+    options[:queue] = val
   end
   
-  def self.tables_path
-    @@tables_path
+  def self.processed_path=(pa)
+    options[:processed_path] = pa
   end
   
-  def self.tables_path=(val)
-    @@tables_path = val
-    @@tables = YAML.load_file(@@tables_path) rescue {}
+  def self.poll_interval=(val)
+    options[:poll_interval] = val
   end
   
-  def self.initializer
-    @@initializer
+  def file_ext=(val)
+    options[:file_ext] = val if [:csv, :tab].include? val
   end
   
-  def self.initializer=(val)
-    @@initializer = val
-  end
-    
 end
 
 require 'broker/application'
-

data/web/views/layout.erb

@@ -39,7 +39,7 @@
               <p class="navbar-text" style="color:white;">Broker v<%= Broker::VERSION %></p>
             </li>
             <li>
-              <p class="navbar-text">Queue: <%= Broker.queue %></p>
+              <p class="navbar-text">Queue: <%= Broker.options[:queue] %></p>
             </li>
             <li>
               <p class="navbar-text"><%= 'Time' %>: <%= Time.now.localtime.strftime('%H:%M:%S CST') %></p>

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: broker
 version: !ruby/object:Gem::Version
-  version: 0.0.9
+  version: 0.1.1
 platform: ruby
 authors:
 - Shawn Henley
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-11-20 00:00:00.000000000 Z
+date: 2015-11-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -52,8 +52,9 @@ dependencies:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
-description: Automate the importing and exporting of data with Quickbase applications,
-  using post requests.
+description: Simple utility to automate the importing of data through the Quickbase
+  API by polling a folder queue for new files, and pushing the data to your Quickbase
+  apps.
 email:
 - shawn@luckysailor.net
 executables:
@@ -76,8 +77,11 @@ files:
 - lib/broker/client/QuickBaseMisc.rb
 - lib/broker/client/quickbase_client.rb
 - lib/broker/export.rb
+- lib/broker/finder.rb
 - lib/broker/import.rb
 - lib/broker/launcher.rb
+- lib/broker/payload.rb
+- lib/broker/queue.rb
 - lib/broker/session.rb
 - lib/broker/templates/broker.rb
 - lib/broker/templates/quickbase_tables.yml
@@ -121,6 +125,7 @@ rubyforge_project:
 rubygems_version: 2.0.6
 signing_key: 
 specification_version: 4
-summary: Automate the importing and exporting of data with Quickbase applications,
-  using post requests.
+summary: Simple utility to automate the importing of data through the Quickbase API
+  by polling a folder queue for new files, and pushing the data to your Quickbase
+  apps.
 test_files: []