tq 0.1.3

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: fc0bd8304fbc66b4f921997f047c637f579250b9
+  data.tar.gz: d694a8a9b632e17715d78d32a7a2311176d4fed8
+SHA512:
+  metadata.gz: f0062de8772a7555928676469d0fa9191c9d36429f0b451d8cdf28ff0225e07d8eac29d69b826f74636110639936209dda94908e9df2d18ddaf9dafcc68076c7
+  data.tar.gz: 370c3621670e588217dd84fea9087d94fda54c040bc8776c55ea1f592383835f32c6d0baf366c3ce5052e0f655d729b59e88b0fe3af1069cdb5de706b4ac0983

data/.gems

@@ -0,0 +1,3 @@
+google-api-client -v 0.8.4
+minitest -v 5.5.1
+parallel -v 1.4.1

data/lib/tq.rb

@@ -0,0 +1,5 @@
+require_relative 'version'
+require_relative 'tq/app'
+require_relative 'tq/queue'
+require_relative 'tq/logger'
+

data/lib/tq/app.rb

@@ -0,0 +1,167 @@
+require 'google/api_client'
+require 'google/api_client/client_secrets'
+require 'google/api_client/auth/file_storage'
+require 'google/api_client/auth/installed_app'
+require 'parallel'
+
+require_relative 'queue'
+
+TASKQUEUE_API = 'taskqueue'
+TASKQUEUE_API_VERSION = 'v1beta2'
+TASKQUEUE_API_SCOPES = ['https://www.googleapis.com/auth/taskqueue']
+
+module TQ
+
+  DEFAULT_OPTIONS = { 
+    'concurrency' => 2, 
+    'log' => {
+      'file' => $stderr
+    },
+    'env' => {} 
+  }
+
+  class App
+    
+    attr_reader :id, :worker
+    def initialize(id, worker, options={})
+      @id = id; @worker = worker  
+      @options = DEFAULT_OPTIONS.merge(options)
+    end
+
+    def options(_)
+      App.new @id, @worker, @options.merge(_)
+    end
+
+    def project(_)
+      options({'project' => _})
+    end
+      
+    def log(_)
+      options({'log' => @options['log'].merge(_)})
+    end
+
+    def logger(_)
+      options({'logger' => _})
+    end
+
+    def env(_)
+      options({'env' => @options['env'].merge(_)})
+    end
+      
+    def stdin(_)
+      return stdin({'name' => _}) if String === _ 
+      options({'stdin' => _})
+    end
+    
+    def stdout(_)
+      return stdout({'name' => _}) if String === _ 
+      options({'stdout' => _})
+    end
+    
+    def stderr(_)
+      return stderr({'name' => _}) if String === _ 
+      options({'stderr' => _})
+    end
+    
+    def run!(secrets_file=nil, store_file=nil)
+      setup_logger!
+      _run *(_queues( TQ::Queue.new( *(auth!(secrets_file, store_file)) ).project(@options['project']) ) )
+    end
+
+
+    def service_auth!(issuer, p12_file)
+      key = Google::APIClient::KeyUtils.load_from_pkcs12(p12_file, 'notasecret')
+      client.authorization = Signet::OAuth2::Client.new(
+        :token_credential_uri => 'https://accounts.google.com/o/oauth2/token',
+        :audience => 'https://accounts.google.com/o/oauth2/token',
+        :scope => TASKQUEUE_API_SCOPES,
+        :issuer => issuer,
+        :signing_key => key)
+      client.authorization.fetch_access_token!
+      
+      api = client.discovered_api(TASKQUEUE_API, TASKQUEUE_API_VERSION)
+
+      return client, api
+    end
+    
+    def auth!(secrets_file=nil, store_file=nil)
+      if store_file.nil? || (cred_store = credentials_store(store_file)).authorization.nil?
+        client_secrets = Google::APIClient::ClientSecrets.load(secrets_file)
+        flow = Google::APIClient::InstalledAppFlow.new(
+          :client_id => client_secrets.client_id,
+          :client_secret => client_secrets.client_secret,
+          :scope => TASKQUEUE_API_SCOPES
+        )
+        client.authorization = store_file.nil? ? 
+                                 flow.authorize :
+                                 flow.authorize(cred_store)
+      else
+        client.authorization = cred_store.authorization
+      end
+      
+      api = client.discovered_api(TASKQUEUE_API, TASKQUEUE_API_VERSION)
+
+      return client, api
+    end
+
+    def application_name
+      @id.split('/')[0]
+    end
+    
+    def application_version
+      @id.split('/')[1] || '0.0.0'
+    end
+    
+    private
+    
+    def setup_logger!
+      if logger = @options['logger']
+      else
+        if (log = @options['log']) && (file = log['file'])
+          logger = Logger.new(file)
+          if level = log['level']
+            logger.level = level
+          end
+        end
+      end
+      (Google::APIClient.logger = logger) if logger
+    end
+
+    def client
+      @client ||= Google::APIClient.new(
+                    :application_name => application_name,
+                    :application_version => application_version
+                  )
+    end
+    
+    def credentials_store(file)
+      Google::APIClient::FileStorage.new(file)
+    end
+    
+    def _queues(q)
+      qin  = @options['stdin']  && q.options(@options['stdin'])
+      qout = @options['stdout'] && q.options(@options['stdout'])
+      qerr = @options['stderr'] && q.options(@options['stderr'])
+      return qin, qout, qerr
+    end 
+   
+    # TODO handle uncaught worker errors by qerr.push!(err) and qin.finish!(task)
+    # TODO raise if not qin
+    def _run(qin, qout, qerr)
+      tasks = qin.lease!
+      Parallel.each(tasks, :in_threads => @options['concurrency']) do |task| 
+        @worker.new(qin, qout, qerr, inherited_env).call(task)
+      end
+    end
+
+    # default log/logger options into env
+    def inherited_env
+      env = @options['env']
+      log = @options['log']
+      logger = @options['logger']
+      {'log' => log, 'logger' => logger}.merge(env)
+    end
+
+  end
+
+end

data/lib/tq/logger.rb

@@ -0,0 +1,93 @@
+require 'logger'
+require 'time'
+
+module TQ
+
+  class Logger
+    
+    DEFAULT_OPTIONS = { 
+      'file'  =>  $stderr,
+      'level' =>  ::Logger::WARN
+    }
+
+    def initialize(queue, options={})
+      @queue = queue
+      if Hash === options
+        @log = build_log( DEFAULT_OPTIONS.merge(options) )
+      else
+        @log = options
+      end
+    end
+
+    def level
+      @log.level
+    end
+
+    def level=(severity)
+      @log.level = severity
+    end
+
+    def progname
+      @log.progname
+    end
+
+    def progname=(name)
+      @log.progname = name
+    end
+
+    def add(severity, message=nil, progname=nil, context=nil, &block)
+      t = Time.now       
+      @log.add(severity, message, progname, &block)
+      @queue.push!( 
+        queue_message(t, severity, message, progname, context, &block), 
+        ::Logger::SEV_LABEL[severity].to_s.downcase
+      ) if (severity >= level)
+    end
+
+    alias log add 
+
+    ::Logger::SEV_LABEL.each_with_index do |label,level|
+      define_method(label == 'ANY' ? 'unknown' : label.to_s.downcase) do |progname=nil, context=nil, &block|
+        add(level, nil, progname, context, &block)
+      end
+    end
+
+    private
+
+    def build_log(options)
+      return options if options.respond_to?(:log)
+      logger = ::Logger.new(options['file'], options['shift_age'], options['shift_size'])
+      logger.level = options['level'] if options['level']
+      logger.progname = options['progname']  if options['progname']
+      return logger
+    end
+
+    # damn, the ruby logger interface is weird... this logic is copied almost verbatim
+    # from ::Logger.add
+    def queue_message(t, severity, message, progname, context, &block)
+      severity ||= ::Logger::UNKNOWN
+      progname ||= self.progname
+      if message.nil?
+        if block_given?
+          message = yield
+        else
+          message = progname
+          progname = self.progname
+        end
+      end
+
+      return {
+        time: t.iso8601,
+        timestamp: t.to_i,
+        level: severity,
+        label: ::Logger::SEV_LABEL[severity],
+        message: message,
+        progname: progname,
+        context: context
+      }
+    end
+
+  end
+
+end
+

data/lib/tq/queue.rb

@@ -0,0 +1,138 @@
+
+module TQ
+
+  class Queue
+   
+    DEFAULT_OPTIONS = { 'lease_secs' => 60, 'num_tasks' => 1 }
+
+    attr_reader :client, :api
+    def initialize(client, api, options={})
+      @client, @api = client, api
+      @options = DEFAULT_OPTIONS.merge(options)
+    end
+
+    def options(_)
+      Queue.new @client, @api, @options.merge(_)
+    end
+
+    def project(_)
+      options({'project' => _})
+    end
+
+    def name(_)
+      options({'name' => _})
+    end
+
+    def lease!(opts={})
+      opts = @options.merge(opts)
+      results = client.execute!(  
+                  :api_method => api.tasks.lease,
+                  :parameters => { :leaseSecs => opts['lease_secs'], 
+                                   :project => opts['project'], 
+                                   :taskqueue => opts['name'], 
+                                   :numTasks => opts['num_tasks']
+                                 }
+                )
+      items = (results.data && results.data['items']) || []
+      items.map {|t| new_task(t) } 
+    end
+    
+    # note: does not currently work; filed bug report https://code.google.com/p/googleappengine/issues/detail?id=11838
+    def extend!(task, secs=nil)
+      secs = secs.nil? ? @options['lease_secs'] : secs
+      opts = @options
+      results = client.execute!(
+                  :api_method => api.tasks.update,
+                  :parameters => { :newLeaseSeconds => secs, 
+                                   :project => opts['project'], 
+                                   :taskqueue => opts['name'], 
+                                   :task => task.id
+                                 }
+                )
+      new_task(results.data)
+    end
+      
+    def push!(payload, tag=nil)
+      opts = @options
+      body = { 'queueName'     => opts['name'],
+               'payloadBase64' => encode(payload)
+             }
+      body['tag'] = tag if tag
+      
+      results = client.execute!( 
+                  :api_method => api.tasks.insert,
+                  :parameters => { :project   => opts['project'],
+                                   :taskqueue => opts['name']
+                                 },
+                  :body_object => body
+                )
+      new_task(results.data)
+    end
+
+    # note: you must have previously leased given task
+    def finish!(task)
+      opts = @options
+      client.execute!( :api_method => api.tasks.delete,
+                       :parameters => { :project   => opts['project'],
+                                        :taskqueue => opts['name'],
+                                        :task      => task.id
+                                      }
+                    )
+      return
+    end
+    
+    private
+    
+    def new_task(t)
+      Task.new(self, t['id'], timestamp_time(t['leaseTimestamp']), decode(t.payloadBase64), t) 
+    end
+
+    def timestamp_time(t)
+      Time.at( t / 1000000 )
+    end
+
+    def encode(obj)
+      Base64.urlsafe_encode64(JSON.dump(obj))
+    end
+    
+    def decode(str)
+      JSON.load(Base64.urlsafe_decode64(str))
+    end
+    
+  end
+
+  class Task < Struct.new(:queue, :id, :expires, :payload, :raw)
+    
+    def initialize(*args)
+      super
+      @clock = Time
+    end
+
+    def finish!
+      self.queue.finish!(self)
+    end
+
+    def extend!(secs=nil)
+      self.queue.extend!(self, secs)
+    end
+
+    def clock!(_)
+      @clock = _; return self
+    end
+
+    def reset_clock!
+      @clock = Time; return self
+    end
+
+    def lease_remaining
+      self.expires - @clock.now
+    end
+
+    def lease_expired?
+      self.expires < @clock.now
+    end
+
+  end
+  
+end
+

data/lib/tq/shell.rb

@@ -0,0 +1,85 @@
+require 'optparse'
+require 'json'
+require_relative '../version'
+
+module TQ
+
+  class Shell
+  
+    DEFAULT_OPTIONS = {
+      app: {},
+      auth_secrets_file: './tq-client.json',
+      auth_store_file:   './tq-client-store.json',
+      config_file:       './tq-app.json'
+    }
+
+    def initialize(app, logger=nil)
+      @app = app
+      @logger = logger
+      @summary = []
+    end
+
+    def banner(_)
+      @banner = _; return self
+    end
+
+    def summary(*_)
+      @summary = _; return self
+    end
+
+    def call(argv=ARGV)
+      
+      progname = File.basename(__FILE__,'.rb')
+
+      @logger.info(progname) { "Configuring #{@app.id}" } if @logger
+      opts = parse_args(argv)
+      @logger.debug(progname) { "Configuration: #{opts[:app].inspect}" } if @logger
+
+      @app = @app.options( opts[:app] ).logger(@logger)
+
+      @logger.info(progname) { "Running #{@app.id} using worker #{@app.worker}" } if @logger
+      @app.run!( opts[:auth_secrets_file], opts[:auth_store_file] )
+
+    end
+
+    private
+
+    def parse_args(argv)
+      opts = {}.merge(DEFAULT_OPTIONS)
+
+      OptionParser.new do |shell|
+ 
+        (shell.banner = @banner) if @banner
+        @summary.each do |line|
+          shell.separator line
+        end
+
+        shell.on('-a', '--auth-secrets [FILE]', "Google OAuth2 secrets file") do |given|
+          opts[:auth_secrets_file] = given
+        end
+
+        shell.on('-s', '--auth-store [FILE]', "Google OAuth2 storage file") do |given|
+          opts[:auth_store_file] = given
+        end
+
+        shell.on('-c', '--config [FILE]', "Application config file (json)") do |given|
+          opts[:config_file] = given
+          opts[:app] = JSON.load( File.open(given, 'r') )
+        end
+
+        shell.on('-h', '--help', "Prints this help") do |given|
+          puts shell; exit
+        end
+
+        shell.on('-v', '--version', "Prints TQ version") do |given|
+          puts TQ::VERSION; exit
+        end
+
+      end.parse(argv)
+
+      return opts
+    end
+
+  end
+
+end

data/lib/version.rb

@@ -0,0 +1,4 @@
+
+module TQ
+  VERSION = '0.1.3'
+end

data/test/helper.rb

@@ -0,0 +1,120 @@
+require 'fileutils'
+require 'logger'
+
+gem 'minitest'
+require 'minitest/autorun'
+
+require 'google/api_client'
+
+module TestUtils
+  extend self
+
+  def setup_logger(name="test")
+    file = File.expand_path("log/#{name}.log", File.dirname(__FILE__)) 
+    FileUtils.mkdir_p( File.dirname(file) )
+    logger = Logger.new( File.open(file, 'w' ) )
+    logger.level = ENV['DEBUG'] ? Logger::DEBUG : Logger::INFO
+    Google::APIClient.logger = logger
+  end
+
+  def current_logger
+    Google::APIClient.logger
+  end
+
+  class QueueHelper
+
+    attr_reader :project, :queue
+    def initialize(project,queue)
+      @project, @queue = project, queue
+    end
+
+    def auth_files(secrets,creds)
+      @secrets_file, @creds_file = secrets, creds
+      return self
+    end
+
+    def authorized_client
+      # @authorized_client ||= TQ::App.new('test_app',nil).service_auth!(File.read(SERVICE_ISSUER_FILE).chomp, SERVICE_P12_FILE)
+      @authorized_client ||= TQ::App.new('test_app',nil).auth!(@secrets_file, @creds_file)
+    end
+    
+    # Note: inaccurate, don't use
+    def peek()
+      client, api = authorized_client
+      results = client.execute!( :api_method => api.tasks.list,
+                                 :parameters => { :project => project, :taskqueue => queue }
+                )
+      items = results.data['items'] || []
+    end
+
+    def push!(payload)
+      client, api = authorized_client
+      client.execute!( :api_method => api.tasks.insert,
+                       :parameters => { :project => project, :taskqueue => queue },
+                       :body_object => { 
+                         'queueName' => queue, 
+                         'payloadBase64' => encode(payload)
+                       }
+      )
+    end
+
+    def pop!(n=1)
+      client, api = authorized_client
+      results = client.execute!( :api_method => api.tasks.lease,
+                                 :parameters => { :project => project, :taskqueue => queue, 
+                                                  :leaseSecs => 60, :numTasks => n
+                                 }
+                )
+      items = results.data['items'] || []
+      items.each do |item|
+        client.execute!( :api_method => api.tasks.delete,
+                         :parameters => { :project => project, :taskqueue => queue, :task => item['id'] }
+        )
+      end
+      return items
+    end
+
+    def all_payloads!
+      map! { |t| decode(t['payloadBase64']) }
+    end
+
+    def all_tags!
+      map! { |t| t['tag'] }
+    end
+
+    def all_payloads_and_tags!
+      map! { |t| {:payload => decode(t['payloadBase64']),
+                  :tag => t['tag']
+                 }
+           }
+    end
+
+    def map!(&b)
+      clear!.map(&b) 
+    end
+
+    def clear!
+      client, api = authorized_client
+      done = false
+      all = []
+      while !done do
+        batch = pop!(10)
+        done = batch.empty? || batch.length < 10
+        all = all + batch
+      end
+      all
+    end
+
+    def encode(obj)
+      Base64.urlsafe_encode64(JSON.dump(obj))
+    end
+
+    def decode(str)
+      JSON.load(Base64.urlsafe_decode64(str))
+    end
+
+  end
+
+
+end
+

data/test/suite.rb

@@ -0,0 +1,4 @@
+
+%w[ test_shell test_run test_logger ].each do |rb|
+  require_relative rb
+end

data/test/test_auth.rb

@@ -0,0 +1,63 @@
+require 'fileutils'
+require_relative './helper'
+require_relative '../lib/tq/app'
+
+def setup_logger!(name)
+  TestUtils.setup_logger(name)
+end
+
+def delete_credentials!
+  FileUtils.rm_f(CREDENTIALS_FILE)
+end
+
+# for installed app auth
+CLIENT_SECRETS_FILE = File.expand_path('../config/secrets/test/client_secrets.json', File.dirname(__FILE__))
+CREDENTIALS_FILE    = File.expand_path("../config/secrets/test/#{File.basename(__FILE__,'.rb')}-oauth2.json", File.dirname(__FILE__))
+
+# for service account auth -- not quite working
+SERVICE_ISSUER_FILE = File.expand_path('../config/secrets/test/issuer', File.dirname(__FILE__))
+SERVICE_P12_FILE    = File.expand_path('../config/secrets/test/client.p12', File.dirname(__FILE__))
+
+describe TQ::App do
+
+  describe "auth" do
+    
+    before do
+      setup_logger!('auth')
+      delete_credentials!
+    end
+
+    it "should authorize without cached credentials" do
+      app = TQ::App.new('test_app/0.0.0', nil)
+      app.auth! CLIENT_SECRETS_FILE 
+      assert true
+    end
+
+    # Note: browser window should only appear once for this test
+    # Not sure if it's possible to assert this
+    it "should authorize with cached credentials not existing and existing" do
+      app = TQ::App.new('test_app/0.0.0', nil)
+      app.auth! CLIENT_SECRETS_FILE, CREDENTIALS_FILE
+      assert File.exists?(CREDENTIALS_FILE)
+      app.auth! CLIENT_SECRETS_FILE, CREDENTIALS_FILE
+      assert File.exists?(CREDENTIALS_FILE)
+    end
+
+  end
+
+  describe "service auth" do
+
+    before do
+      setup_logger!('auth')
+    end
+
+    it "should authorize service account" do
+      app = TQ::App.new('test_app/0.0.0', nil)
+      app.service_auth!(File.read(SERVICE_ISSUER_FILE).chomp, SERVICE_P12_FILE)
+      assert true
+    end
+
+  end
+
+end
+

data/test/test_logger.rb

@@ -0,0 +1,184 @@
+# Functional tests of TQ::Logger
+# Please note that these require a deployed GAE app
+#   with two queues: 'test' and 'log'.
+# The GAE project_id is defined in ../config/secrets/test/project_id,
+#   along with other secrets files (see below).
+
+require 'logger'
+
+require_relative './helper'
+require_relative '../lib/tq/app'
+require_relative '../lib/tq/queue'
+require_relative '../lib/tq/logger'
+
+def setup_test_logger!
+  TestUtils.setup_logger( File.basename(__FILE__,'.rb') )
+end
+
+
+class LoggerTests < Minitest::Spec
+
+  # for installed app auth
+  CLIENT_SECRETS_FILE = File.expand_path(
+    '../config/secrets/test/client_secrets.json', File.dirname(__FILE__)
+  )
+  CREDENTIALS_FILE    = File.expand_path(
+    "../config/secrets/test/#{File.basename(__FILE__,'.rb')}-oauth2.json", 
+    File.dirname(__FILE__)
+  )
+
+  # task queue constants
+  TASKQUEUE_PROJECT_ID = File.read(
+    File.expand_path('../config/secrets/test/project_id', File.dirname(__FILE__))
+  ).chomp
+  
+  TASKQUEUE_TEST_QUEUE = 'log'
+
+  def test_logger
+    @test_logger ||=  TestUtils.current_logger
+  end
+
+  def assert_logged( exps, acts )
+    errs = []
+    unless (a = exps.length) == (b = acts.length)
+      errs.push( "Expected #{a} messages, #{b < 2 ? 'were' : 'was'} #{b}" )
+    end
+    exps.each_with_index do |exp,i|
+      act = acts[i]
+      next if act.nil?
+      [:level, :message, :progname].each do |elem|
+        unless (a = exp[elem]) == (b = act[elem.to_s])
+          errs.push( "[#{i}] Expected #{elem} to be #{a}, was #{b}" )
+        end
+      end
+      unless (a = ::Logger::SEV_LABEL[exp[:level]]) == (b = act['label'])
+        errs.push( "[#{i}] Expected label to be #{a}, was #{b}" )
+      end
+    end
+    assert errs.length == 0, errs.join("\n")
+  end
+
+  def assert_logged_tags( exps, acts )
+    errs = []
+    unless (a = exps.length) == (b = acts.length)
+      errs.push( "Expected #{a} messages, #{b < 2 ? 'were' : 'was'} #{b}" )
+    end
+    exps.each_with_index do |exp,i|
+      act = acts[i]
+      next if act.nil?
+      unless (a = exp[:method].to_s) == (b = act)
+        errs.push( "[#{i}] Expected tag to be #{a}, was #{b}" )
+      end
+    end
+    assert errs.length == 0, errs.join("\n")
+  end
+
+  def send_messages_to!(logger, msgs)
+    msgs.each do |msg|
+      logger.__send__(msg[:method], msg[:progname], msg[:context]) { msg[:message] }
+    end
+  end
+
+  def queue_helper(project,queue)
+    TestUtils::QueueHelper.new(project,queue).auth_files(CLIENT_SECRETS_FILE, CREDENTIALS_FILE)
+  end
+
+  def clear_queue!(project,queue)
+    queue_helper(project,queue).clear!
+  end
+
+  def verify_logged_messages_to_level!(expected_messages, minlevel)
+    actual_messages = queue_helper(TASKQUEUE_PROJECT_ID, TASKQUEUE_TEST_QUEUE).all_payloads_and_tags! 
+    $stderr.puts actual_messages.inspect
+
+    selected_messages = expected_messages.select { |msg| msg[:level] >= minlevel } 
+
+    assert_logged( selected_messages,
+                   actual_messages.map { |msg| msg[:payload] }
+                 )
+
+    assert_logged_tags( selected_messages,
+                        actual_messages.map { |msg| msg[:tag] }
+                      )
+  end
+
+  # Note: ideally these tests would also verify the (file) logger output as well
+  # and the timestamps, mocking the clock, etc. But that's a mess of work, so for now
+  # you should just eyeball it in the $stderr output.
+
+  def setup
+    clear_queue!(TASKQUEUE_PROJECT_ID, TASKQUEUE_TEST_QUEUE)
+    
+    app = TQ::App.new('test_app/0.0.0', nil)
+    @queue = TQ::Queue.new( *app.auth!(CLIENT_SECRETS_FILE, CREDENTIALS_FILE) )
+                      .options({ 'project' => TASKQUEUE_PROJECT_ID, 'name' => TASKQUEUE_TEST_QUEUE })
+  end
+
+  it 'default logger should log to queue at warn level' do
+    subject = TQ::Logger.new(@queue)
+
+    expected_messages = [
+      { method: :debug, level: ::Logger::DEBUG, message: 'debug message', progname: 'prog1', context: { key: 1 } },
+      { method: :info,  level: ::Logger::INFO,  message: 'info message',  progname: 'prog2', context: { key: 2 } },
+      { method: :warn,  level: ::Logger::WARN,  message: 'warn message',  progname: 'prog3', context: { key: 3 } },
+      { method: :error,  level: ::Logger::ERROR,  message: 'error message',  progname: 'prog4', context: { key: 4 } }
+    ]
+
+    send_messages_to! subject, expected_messages
+    
+    verify_logged_messages_to_level! expected_messages, ::Logger::WARN
+  
+  end
+
+  it 'after setting level to debug, logger should log to queue at debug level' do
+    subject = TQ::Logger.new(@queue)
+    subject.level = ::Logger::DEBUG
+
+    expected_messages = [
+      { method: :debug, level: ::Logger::DEBUG, message: 'debug message', progname: 'prog1', context: { key: 1 } },
+      { method: :info,  level: ::Logger::INFO,  message: 'info message',  progname: 'prog2', context: { key: 2 } },
+      { method: :warn,  level: ::Logger::WARN,  message: 'warn message',  progname: 'prog3', context: { key: 3 } },
+      { method: :error,  level: ::Logger::ERROR,  message: 'error message',  progname: 'prog4', context: { key: 4 } }
+    ]
+
+    send_messages_to!(subject, expected_messages)
+
+    verify_logged_messages_to_level! expected_messages, ::Logger::DEBUG
+   
+  end
+
+  it 'when setting level to debug in config, logger should log to queue at debug level' do
+    subject = TQ::Logger.new(@queue, {'level' => ::Logger::DEBUG } )
+
+    expected_messages = [
+      { method: :debug, level: ::Logger::DEBUG, message: 'debug message', progname: 'prog1', context: { key: 1 } },
+      { method: :info,  level: ::Logger::INFO,  message: 'info message',  progname: 'prog2', context: { key: 2 } },
+      { method: :warn,  level: ::Logger::WARN,  message: 'warn message',  progname: 'prog3', context: { key: 3 } },
+      { method: :error,  level: ::Logger::ERROR,  message: 'error message',  progname: 'prog4', context: { key: 4 } }
+    ]
+
+    send_messages_to!(subject, expected_messages)
+
+    verify_logged_messages_to_level! expected_messages, ::Logger::DEBUG
+
+  end
+
+  it 'when setting external logger, logger should log to queue at logger\'s level' do
+    subject = TQ::Logger.new(@queue, test_logger )
+
+    expected_messages = [
+      { method: :debug, level: ::Logger::DEBUG, message: 'debug message', progname: 'prog1', context: { key: 1 } },
+      { method: :info,  level: ::Logger::INFO,  message: 'info message',  progname: 'prog2', context: { key: 2 } },
+      { method: :warn,  level: ::Logger::WARN,  message: 'warn message',  progname: 'prog3', context: { key: 3 } },
+      { method: :error,  level: ::Logger::ERROR,  message: 'error message',  progname: 'prog4', context: { key: 4 } }
+    ]
+
+    send_messages_to!(subject, expected_messages)
+
+    verify_logged_messages_to_level! expected_messages, test_logger.level
+
+  end
+
+end
+
+setup_test_logger!

data/test/test_run.rb

@@ -0,0 +1,239 @@
+# Functional tests of TQ::App#run!
+# Please note that these require a deployed GAE app
+#   with two queues: 'test' and 'log'.
+# The GAE project_id is defined in ../config/secrets/test/project_id,
+#   along with other secrets files (see below).
+
+require_relative './helper'
+require_relative '../lib/tq/app'
+
+def setup_test_logger!
+  TestUtils.setup_logger( File.basename(__FILE__,'.rb') )
+end
+
+class AppRunTests < Minitest::Spec
+
+  # for installed app auth
+  CLIENT_SECRETS_FILE = File.expand_path(
+    '../config/secrets/test/client_secrets.json', File.dirname(__FILE__)
+  )
+  CREDENTIALS_FILE    = File.expand_path(
+    "../config/secrets/test/#{File.basename(__FILE__,'.rb')}-oauth2.json", 
+    File.dirname(__FILE__)
+  )
+
+  # for service account auth -- not quite working
+  SERVICE_ISSUER_FILE = File.expand_path(
+    '../config/secrets/test/issuer', File.dirname(__FILE__)
+  )
+  SERVICE_P12_FILE    = File.expand_path(
+    '../config/secrets/test/client.p12', File.dirname(__FILE__)
+  )
+
+  # task queue constants
+  TASKQUEUE_PROJECT_ID = File.read(
+    File.expand_path('../config/secrets/test/project_id', File.dirname(__FILE__))
+  ).chomp
+  
+  TASKQUEUE_LEASE_SECS = 2
+
+
+  def queue_helper(project,queue)
+    TestUtils::QueueHelper.new(project,queue).auth_files(CLIENT_SECRETS_FILE, CREDENTIALS_FILE)
+  end
+
+  def tasks_on_queue(project,queue)
+    queue_helper(project,queue).list
+  end
+
+  def clear_queue!(project,queue)
+    queue_helper(project,queue).clear!
+  end
+
+  def push_tasks!(project,queue,tasks)
+    q = queue_helper(project,queue)
+    tasks.each do |task| q.push!(task) end
+  end
+
+
+  describe "run!" do
+  
+    before do
+      sleep TASKQUEUE_LEASE_SECS + 1  # to wait for lease expiry from previous test
+      clear_queue!(TASKQUEUE_PROJECT_ID,'test')
+      clear_queue!(TASKQUEUE_PROJECT_ID,'log')
+    end
+
+    it "should setup clearing the queue" do
+      cleared = clear_queue!(TASKQUEUE_PROJECT_ID,'test')
+      assert_equal 0, n = cleared.length,
+         "Expected no tasks on queue, #{n < 2 ? 'was' : 'were'} #{n}"
+
+      # unfortunately, queue peeks are horribly inaccurate right now.
+      # assert_equal 0, n = tasks_on_queue(TASKQUEUE_PROJECT_ID,'test').length,
+      #   "Expected no tasks on queue, #{n < 2 ? 'was' : 'were'} #{n}"
+    end
+
+    it "worker should receive input queue and :call with each task on the queue up to the specified number" do
+      
+      # setup
+      expected_tasks = [
+        { 'What is your name?' => 'Sir Lancelot', 'What is your quest?' => 'To seek the holy grail', 'What is your favorite color?' => 'blue' },
+        { 'What is your name?' => 'Sir Robin', 'What is your quest?' => 'To seek the holy grail', 'What is the capital of Assyria?' => nil },
+        { 'What is your name?' => 'Galahad', 'What is your quest?' => 'To seek the grail', 'What is your favorite color?' => ['blue','yellow'] },
+        { 'What is your name?' => 'Arthur', 'What is your quest?' => 'To seek the holy grail', 'What is the air-speed velocity of an unladen swallow?' => 'African or European swallow?' }
+      ]
+      push_tasks!(TASKQUEUE_PROJECT_ID,'test', expected_tasks)
+      
+      # expectations
+      mock_handler_class = MiniTest::Mock.new
+      mock_handler = MiniTest::Mock.new
+
+      ## expect constructor receives task queue as first param -- for each queued task up to expected_instances
+      expected_calls = 3
+      (0...expected_calls).each do
+        mock_handler_class.expect(:new, mock_handler) do |*args|
+          args.first.respond_to?('finish!')  
+        end
+      end
+
+      ## expect :call for each queued task up to expected_calls
+      actual_calls = 0; 
+      (0...expected_calls).each do 
+        mock_handler.expect(:call, true) do |actual_task|
+          actual_calls += 1
+          valid = !!actual_task && actual_task.payload.has_key?('What is your name?')
+          valid.tap do |yes|
+            $stderr.puts("mock_handler.call - received: #{ actual_task.payload }")  if yes
+          end
+        end
+      end
+
+      # execution
+      app = TQ::App.new('test_app/0.0.0', mock_handler_class)
+               .project(TASKQUEUE_PROJECT_ID)
+               .stdin({ 'name' => 'test', 'num_tasks' => expected_calls, 'lease_secs' => TASKQUEUE_LEASE_SECS })
+      app.run! CLIENT_SECRETS_FILE, CREDENTIALS_FILE
+
+      # assertions
+      assert_equal expected_calls, actual_calls,
+        "Expected #{expected_calls} worker calls, was #{actual_calls}"
+
+      mock_handler_class.verify
+      mock_handler.verify
+
+    end
+
+    it 'should put task back on input queue after lease_secs, if not finished' do
+    
+      # setup
+      expected_tasks = [
+        { 'What is your name?' => 'Sir Lancelot', 'What is your quest?' => 'To seek the holy grail', 'What is your favorite color?' => 'blue' }
+      ]
+      push_tasks!(TASKQUEUE_PROJECT_ID,'test', expected_tasks)
+ 
+      class DoNothingWorker
+        def initialize(*args); end
+        def call(task); end
+      end
+
+      # execution
+      app = TQ::App.new('test_app/0.0.0', DoNothingWorker)
+               .project(TASKQUEUE_PROJECT_ID)
+               .stdin({ 'name' => 'test', 'num_tasks' => 1, 'lease_secs' => TASKQUEUE_LEASE_SECS })
+      app.run! CLIENT_SECRETS_FILE, CREDENTIALS_FILE
+    
+      sleep TASKQUEUE_LEASE_SECS + 1
+      actual_tasks = clear_queue!(TASKQUEUE_PROJECT_ID,'test')
+
+      ## queue peeks are inaccurate right now...
+      # actual_tasks = tasks_on_queue(TASKQUEUE_PROJECT_ID,'test')
+      
+      ## assertion
+      assert_equal a = expected_tasks.length, b = actual_tasks.length, 
+         "Expected #{a} #{a == 1 ? 'task' : 'tasks'} on queue, #{b < 2 ? 'was' : 'were'} #{b}"
+      
+      assert (n = actual_tasks.first['retry_count']) > 0,
+         "Expected >0 lease retry count, #{n < 2 ? 'was' : 'were'} #{n}"
+
+    end
+
+    it 'should be able to push to queue from within worker' do
+      
+      # setup
+      expected_tasks = [
+        { 'What is your name?' => 'Sir Lancelot', 'What is your quest?' => 'To seek the holy grail', 'What is your favorite color?' => 'blue' }
+      ]
+      push_tasks!(TASKQUEUE_PROJECT_ID,'test', expected_tasks)
+ 
+      class RelayWorker
+        def initialize(*args)
+          @stdin = args.first; @stdout = args[1]
+        end
+
+        def call(task)
+          @stdout.push!(task.payload)
+          task.finish!
+        end
+      end
+
+      # execution
+      app = TQ::App.new('test_app/0.0.0', RelayWorker)
+               .project(TASKQUEUE_PROJECT_ID)
+               .stdin({ 'name' => 'test', 'num_tasks' => 1, 'lease_secs' => TASKQUEUE_LEASE_SECS })
+               .stdout({ 'name' => 'log' })
+      app.run! CLIENT_SECRETS_FILE, CREDENTIALS_FILE
+    
+      sleep TASKQUEUE_LEASE_SECS + 1
+      actual_output_tasks = clear_queue!(TASKQUEUE_PROJECT_ID,'log')
+      actual_input_tasks = clear_queue!(TASKQUEUE_PROJECT_ID,'test')
+      
+      # assertion
+
+      assert_equal 0, b = actual_input_tasks.length,
+        "Expected no tasks on input queue, #{b < 2 ? 'was' : 'were'} #{b}"
+
+      assert_equal a = expected_tasks.length, b = actual_output_tasks.length, 
+        "Expected #{a} #{a == 1 ? 'task' : 'tasks'} on output queue, #{b < 2 ? 'was' : 'were'} #{b}"
+      
+    end
+
+    it 'should extend a task lease if extended before lease expires' do
+      
+      refute true, "Extending task leases does not currently work"
+
+      # setup
+      expected_tasks = [
+        { 'What is your name?' => 'Sir Lancelot', 'What is your quest?' => 'To seek the holy grail', 'What is your favorite color?' => 'blue' }
+      ]
+      push_tasks!(TASKQUEUE_PROJECT_ID,'test', expected_tasks)
+ 
+      class ExtendWorker
+        def initialize(*args)
+          @stdin = args.first
+        end
+
+        def call(task)
+          $stderr.puts "ExtendWorker - task #{ task.raw.inspect }"
+          ttl = task.lease_remaining
+          # sleep( ttl - 0.5 )
+          task = @stdin.extend!(task)
+          ttl2 = task.lease_remaining
+          $stderr.puts "ExtendWorker - ttl before extend: #{ttl}"
+          $stderr.puts "ExtendWorker - ttl after extend: #{ttl2}"
+        end
+      end
+
+      # execution
+      app = TQ::App.new('test_app/0.0.0', ExtendWorker)
+               .project(TASKQUEUE_PROJECT_ID)
+               .stdin({ 'name' => 'test', 'num_tasks' => 1, 'lease_secs' => TASKQUEUE_LEASE_SECS + 2 })
+      app.run! CLIENT_SECRETS_FILE, CREDENTIALS_FILE
+    
+    end
+
+  end
+
+end
+
+setup_test_logger!

data/test/test_shell.rb

@@ -0,0 +1,128 @@
+require_relative './helper'
+require_relative '../lib/tq/app'
+require_relative '../lib/tq/shell'
+
+def setup_test_logger!
+  TestUtils.setup_logger( File.basename(__FILE__,'.rb') )
+end
+
+class ShellTests < Minitest::Spec
+
+  include 
+  # for installed app auth
+  CLIENT_SECRETS_FILE = File.expand_path(
+    '../config/secrets/test/client_secrets.json', File.dirname(__FILE__)
+  )
+  CREDENTIALS_FILE    = File.expand_path(
+    "../config/secrets/test/#{File.basename(__FILE__,'.rb')}-oauth2.json", 
+    File.dirname(__FILE__)
+  )
+
+  # task queue constants
+  TASKQUEUE_APP_CONFIG = 
+    File.expand_path("../config/secrets/test/#{File.basename(__FILE__,'.rb')}" + 
+                     "-config.json", File.dirname(__FILE__))
+
+  TASKQUEUE_LEASE_SECS = 2
+
+  class EchoWorker
+
+    def initialize(stdin, stdout, stderr, env)
+      @stdin = stdin
+      @env = env
+      @logger = env['logger']
+    end
+
+    def call(task)
+      @logger.info("EchoWorker") { "Received task #{task.id}" }
+      @logger.debug("EchoWorker") { "Task payload: #{task.payload.inspect}" }
+      @logger.debug("EchoWorker") { "Env: #{@env.inspect}" }
+      task.finish!
+    end
+
+  end
+
+  def logger
+    @logger ||= TestUtils.current_logger
+  end
+
+  def queue_helper(project,queue)
+    TestUtils::QueueHelper.new(project,queue).auth_files(CLIENT_SECRETS_FILE, CREDENTIALS_FILE)
+  end
+
+  def app_config
+    @app_config ||= JSON.load( File.open(TASKQUEUE_APP_CONFIG) )
+  end
+
+  def app_project_id
+    app_config['project']
+  end
+
+  def app_stdin_name
+    app_config['stdin']['name']
+  end
+
+  def app_stdin_num_tasks
+    app_config['stdin']['num_tasks']
+  end
+
+  def populate_queue!(tasks)
+    q = queue_helper(app_project_id, app_stdin_name)
+    tasks.each do |task| q.push!(task) end
+  end
+
+  def clear_queue!
+    queue_helper(app_project_id, app_stdin_name).clear!
+  end
+
+  def tasks_on_queue
+    clear_queue!
+  end
+
+  def assert_tasks_on_queue(exp)
+    assert_equal exp, n = tasks_on_queue.length,
+      "Expected #{exp} tasks on input queue, was #{n}"
+  end
+
+  def shell_args
+    [ "--auth-secrets", CLIENT_SECRETS_FILE,
+      "--auth-store", CREDENTIALS_FILE,
+      "--config", TASKQUEUE_APP_CONFIG
+    ]
+  end
+
+  def setup 
+    sleep TASKQUEUE_LEASE_SECS+1 
+    clear_queue!
+    @app = TQ::App.new('test_app_shell/0.0.0', EchoWorker)
+  end
+
+  it 'should execute and complete app-specified number of tasks on input queue' do
+    exps = [
+        { 'What is your name?' => 'Sir Lancelot', 
+          'What is your quest?' => 'To seek the holy grail', 
+          'What is your favorite color?' => 'blue' },
+        { 'What is your name?' => 'Sir Robin', 
+          'What is your quest?' => 'To seek the holy grail', 
+          'What is the capital of Assyria?' => nil },
+        { 'What is your name?' => 'Galahad', 
+          'What is your quest?' => 'To seek the grail', 
+          'What is your favorite color?' => ['blue','yellow'] },
+        { 'What is your name?' => 'Arthur', 
+          'What is your quest?' => 'To seek the holy grail', 
+          'What is the air-speed velocity of an unladen swallow?' => 'African or European swallow?' }
+    ]
+
+    populate_queue! exps
+    
+    TQ::Shell.new(@app, logger).call( shell_args )
+
+    assert_tasks_on_queue(exps.length - app_stdin_num_tasks)
+
+  end
+
+end
+
+
+setup_test_logger!
+

metadata

@@ -0,0 +1,98 @@
+--- !ruby/object:Gem::Specification
+name: tq
+version: !ruby/object:Gem::Version
+  version: 0.1.3
+platform: ruby
+authors:
+- Eric Gjertsen
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2015-04-13 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: google-api-client
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.8.4
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.8.4
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 5.5.1
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 5.5.1
+- !ruby/object:Gem::Dependency
+  name: parallel
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 1.4.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 1.4.1
+description: Provides a simple framework for writing task worker processes
+email: ericgj72@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gems"
+- lib/tq.rb
+- lib/tq/app.rb
+- lib/tq/logger.rb
+- lib/tq/queue.rb
+- lib/tq/shell.rb
+- lib/version.rb
+- test/helper.rb
+- test/suite.rb
+- test/test_auth.rb
+- test/test_logger.rb
+- test/test_run.rb
+- test/test_shell.rb
+homepage: https://github.com/ericgj/tq
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.5
+signing_key: 
+specification_version: 4
+summary: Ruby client for Google App Engine TaskQueue (REST API)
+test_files: []