tq 0.2.1 → 0.3.1

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: tq
 version: !ruby/object:Gem::Version
-  version: 0.2.1
+  version: 0.3.1
 platform: ruby
 authors:
 - Eric Gjertsen
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-08-04 00:00:00.000000000 Z
+date: 2018-01-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: google-api-client
@@ -16,43 +16,43 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 0.8.4
+        version: 0.19.3
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 0.8.4
+        version: 0.19.3
 - !ruby/object:Gem::Dependency
   name: minitest
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 5.5.1
+        version: 5.11.1
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 5.5.1
+        version: 5.11.1
 - !ruby/object:Gem::Dependency
   name: parallel
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 1.4.1
+        version: 1.12.1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 1.4.1
-description: Provides a simple framework for writing task worker processes
+        version: 1.12.1
+description: A simple framework for writing task worker processes
 email: ericgj72@gmail.com
 executables: []
 extensions: []
@@ -60,17 +60,13 @@ extra_rdoc_files: []
 files:
 - ".gems"
 - lib/tq.rb
-- lib/tq/app.rb
 - lib/tq/logger.rb
-- lib/tq/queue.rb
-- lib/tq/shell.rb
 - lib/version.rb
 - test/helper.rb
 - test/suite.rb
 - test/test_auth.rb
 - test/test_logger.rb
 - test/test_run.rb
-- test/test_shell.rb
 homepage: https://github.com/ericgj/tq
 licenses:
 - MIT
@@ -91,8 +87,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.5.1
+rubygems_version: 2.6.13
 signing_key: 
 specification_version: 4
-summary: Ruby client for Google App Engine TaskQueue (REST API)
+summary: Ruby client for Google Cloud Tasks (REST API v2beta2)
 test_files: []

data/lib/tq/app.rb

@@ -1,176 +0,0 @@
-require 'google/api_client'
-require 'google/api_client/client_secrets'
-require 'google/api_client/auth/file_storage'
-require 'google/api_client/auth/installed_app'
-require 'parallel'
-
-require_relative 'queue'
-
-TASKQUEUE_API = 'taskqueue'
-TASKQUEUE_API_VERSION = 'v1beta2'
-TASKQUEUE_API_SCOPES = ['https://www.googleapis.com/auth/taskqueue']
-
-module TQ
-
-  DEFAULT_OPTIONS = { 
-    'concurrency' => 2, 
-    'log' => {
-      'file' => $stderr
-    },
-    'env' => {} 
-  }
-
-  class App
-    
-    attr_reader :id, :worker
-    def initialize(id, worker, options={})
-      @id = id; @worker = worker  
-      @options = DEFAULT_OPTIONS.merge(options)
-    end
-
-    def options(_)
-      App.new @id, @worker, @options.merge(_)
-    end
-
-    def project(_)
-      options({'project' => _})
-    end
-      
-    def log(_)
-      options({'log' => @options['log'].merge(_)})
-    end
-
-    def logger(_)
-      options({'logger' => _})
-    end
-
-    def env(_)
-      options({'env' => @options['env'].merge(_)})
-    end
-      
-    def stdin(_)
-      return stdin({'name' => _}) if String === _ 
-      options({'stdin' => _})
-    end
-    
-    def stdout(_)
-      return stdout({'name' => _}) if String === _ 
-      options({'stdout' => _})
-    end
-    
-    def stderr(_)
-      return stderr({'name' => _}) if String === _ 
-      options({'stderr' => _})
-    end
-    
-    def service_run!(issuer, p12_file)
-      setup_logger!
-      _run *(_queues( TQ::Queue.new( *(service_auth!(issuer, p12_file)) ).project(@options['project']) ) )
-    end
-
-    def run!(secrets_file=nil, store_file=nil)
-      setup_logger!
-      _run *(_queues( TQ::Queue.new( *(auth!(secrets_file, store_file)) ).project(@options['project']) ) )
-    end
-
-    # Note issuer is not a file name but the service account email address
-    def service_auth!(issuer, p12_file)
-      key = Google::APIClient::KeyUtils.load_from_pkcs12(p12_file, 'notasecret')
-      client.authorization = Signet::OAuth2::Client.new(
-        :token_credential_uri => 'https://accounts.google.com/o/oauth2/token',
-        :audience => 'https://accounts.google.com/o/oauth2/token',
-        :scope => TASKQUEUE_API_SCOPES,
-        :issuer => issuer,
-        :signing_key => key)
-      client.authorization.fetch_access_token!
-      
-      api = client.discovered_api(TASKQUEUE_API, TASKQUEUE_API_VERSION)
-
-      return client, api
-    end
-    
-    def auth!(secrets_file=nil, store_file=nil)
-      if store_file.nil? || (cred_store = credentials_store(store_file)).authorization.nil?
-        client_secrets = Google::APIClient::ClientSecrets.load(secrets_file)
-        flow = Google::APIClient::InstalledAppFlow.new(
-          :client_id => client_secrets.client_id,
-          :client_secret => client_secrets.client_secret,
-          :scope => TASKQUEUE_API_SCOPES
-        )
-        client.authorization = store_file.nil? ? 
-                                 flow.authorize :
-                                 flow.authorize(cred_store)
-      else
-        client.authorization = cred_store.authorization
-      end
-      
-      api = client.discovered_api(TASKQUEUE_API, TASKQUEUE_API_VERSION)
-
-      return client, api
-    end
-
-    def application_name
-      @id.split('/')[0]
-    end
-    
-    def application_version
-      @id.split('/')[1] || '0.0.0'
-    end
-    
-    private
-    
-    def setup_logger!
-      if logger = @options['logger']
-      else
-        if (log = @options['log']) && (file = log['file'])
-          logger = Logger.new(file)
-          if level = log['level']
-            logger.level = level
-          end
-        end
-      end
-      (Google::APIClient.logger = logger) if logger
-    end
-
-    def client
-      @client ||= Google::APIClient.new(
-                    :application_name => application_name,
-                    :application_version => application_version
-                  )
-    end
-    
-    def credentials_store(file)
-      Google::APIClient::FileStorage.new(file)
-    end
-    
-    def _queues(q)
-      qin  = @options['stdin']  && q.options(@options['stdin'])
-      qout = @options['stdout'] && q.options(@options['stdout'])
-      qerr = @options['stderr'] && q.options(@options['stderr'])
-      return qin, qout, qerr
-    end 
-   
-    # TODO handle uncaught worker errors by qerr.push!(err) and qin.finish!(task)
-    # TODO raise if not qin
-    def _run(qin, qout, qerr)
-      tasks = qin.lease!
-      Parallel.each(tasks, :in_threads => @options['concurrency']) do |task| 
-        if task.try?
-          @worker.new(qin, qout, qerr, inherited_env).call(task)
-        else
-          qin.finish!(task)
-        end
-      end
-    end
-
-    # default log/logger options into env
-    def inherited_env
-      env = @options['env']
-      log = @options['log']
-      logger = @options['logger']
-      {'log' => log, 'logger' => logger}.merge(env)
-    end
-
-  end
-
-end

data/lib/tq/queue.rb

@@ -1,161 +0,0 @@
-require 'json'
-require 'base64'
-
-module TQ
-
-  class Queue
-   
-    DEFAULT_OPTIONS = { 
-      'lease_secs' => 60, 
-      'num_tasks' => 1, 
-      'max_tries' => -1 
-    }
-
-    attr_reader :client, :api
-    def initialize(client, api, options={})
-      @client, @api = client, api
-      @options = DEFAULT_OPTIONS.merge(options)
-    end
-
-    def options(_)
-      Queue.new @client, @api, @options.merge(_)
-    end
-
-    def project(_)
-      options({'project' => _})
-    end
-
-    def name(_)
-      options({'name' => _})
-    end
-
-    def option(key)
-      @options[key]
-    end
-
-    def lease!(opts={})
-      opts = @options.merge(opts)
-      results = client.execute!(  
-                  :api_method => api.tasks.lease,
-                  :parameters => { :leaseSecs => opts['lease_secs'], 
-                                   :project => opts['project'], 
-                                   :taskqueue => opts['name'], 
-                                   :numTasks => opts['num_tasks']
-                                 }
-                )
-      items = (results.data && results.data['items']) || []
-      items.map {|t| new_task(t) } 
-    end
-    
-    # note: does not currently work; filed bug report https://code.google.com/p/googleappengine/issues/detail?id=11838
-    def extend!(task, secs=nil)
-      secs = secs.nil? ? @options['lease_secs'] : secs
-      opts = @options
-      results = client.execute!(
-                  :api_method => api.tasks.update,
-                  :parameters => { :newLeaseSeconds => secs, 
-                                   :project => opts['project'], 
-                                   :taskqueue => opts['name'], 
-                                   :task => task.id
-                                 }
-                )
-      new_task(results.data)
-    end
-      
-    def push!(payload, tag=nil)
-      opts = @options
-      body = { 'queueName'     => opts['name'],
-               'payloadBase64' => encode(payload)
-             }
-      body['tag'] = tag if tag
-      
-      results = client.execute!( 
-                  :api_method => api.tasks.insert,
-                  :parameters => { :project   => opts['project'],
-                                   :taskqueue => opts['name']
-                                 },
-                  :body_object => body
-                )
-      new_task(results.data)
-    end
-
-    # note: you must have previously leased given task
-    def finish!(task)
-      opts = @options
-      client.execute!( :api_method => api.tasks.delete,
-                       :parameters => { :project   => opts['project'],
-                                        :taskqueue => opts['name'],
-                                        :task      => task.id
-                                      }
-                    )
-      return
-    end
-    
-    private
-    
-    def new_task(t)
-      Task.new(
-        self, 
-        t['id'], 
-        timestamp_time(t['leaseTimestamp']),
-        t['retry_count'],
-        t['tag'],
-        decode(t.payloadBase64), 
-        t
-      ) 
-    end
-
-    def timestamp_time(t)
-      Time.at( t / 1000000 )
-    end
-
-    def encode(obj)
-      Base64.urlsafe_encode64(JSON.dump(obj))
-    end
-    
-    def decode(str)
-      JSON.load(Base64.urlsafe_decode64(str))
-    end
-    
-  end
-
-  class Task < Struct.new(:queue, :id, :expires, :tries, :tag, :payload, :raw)
-    
-    def initialize(*args)
-      super
-      @clock = Time
-    end
-
-    def finish!
-      self.queue.finish!(self)
-    end
-
-    def extend!(secs=nil)
-      self.queue.extend!(self, secs)
-    end
-
-    def clock!(_)
-      @clock = _; return self
-    end
-
-    def reset_clock!
-      @clock = Time; return self
-    end
-
-    def lease_remaining
-      self.expires - @clock.now
-    end
-
-    def lease_expired?
-      self.expires < @clock.now
-    end
-
-    def try?
-      max = self.queue.option('max_tries')
-      return (max == -1 or self.tries < max)
-    end
-
-  end
-  
-end
-

data/lib/tq/shell.rb

@@ -1,104 +0,0 @@
-require 'optparse'
-require 'json'
-require_relative '../version'
-
-module TQ
-
-  class Shell
-  
-    DEFAULT_OPTIONS = {
-      app: {},
-      config_file:       './tq-app.json'
-    }
-
-    def initialize(app, logger=nil)
-      @app = app
-      @logger = logger
-      @summary = []
-    end
-
-    def banner(_)
-      @banner = _; return self
-    end
-
-    def summary(*_)
-      @summary = _; return self
-    end
-
-    def call(argv=ARGV)
-      
-      progname = File.basename(__FILE__,'.rb')
-
-      opts = parse_args(argv)
-      @logger.debug(progname) { "Configuration for #{@app.id}: #{opts.inspect}" } if @logger
-
-      @app = @app.options( opts[:app] )
-      @app = @app.logger(@logger) if @logger
-
-      @logger.info(progname) { "Running #{@app.id} using worker #{@app.worker}" } if @logger
-
-      secrets, store = opts[:auth_secrets_file],        opts[:auth_store_file] 
-      issuer, p12    = opts[:service_auth_issuer_file], opts[:service_auth_p12_file]
-
-      if secrets
-        @app.run!(secrets, store)
-      elsif issuer && p12
-        @app.service_run!(File.read(issuer).chomp, p12)
-      else
-        raise ArgumentError, "You must provide either OAuth2 secrets and credentials store, " +
-                             "or service-account issuer and p12 files."
-      end
-
-      @logger.info(progname) { "Completed #{@app.id}" } if @logger
-
-    end
-
-    private
-
-    def parse_args(argv)
-      opts = {}.merge(DEFAULT_OPTIONS)
-
-      OptionParser.new do |shell|
- 
-        (shell.banner = @banner) if @banner
-        @summary.each do |line|
-          shell.separator line
-        end
-
-        shell.on('-a', '--auth-secrets [FILE]', "Google OAuth2 secrets file") do |given|
-          opts[:auth_secrets_file] = given
-        end
-
-        shell.on('-s', '--auth-store [FILE]', "Google OAuth2 credentials storage file") do |given|
-          opts[:auth_store_file] = given
-        end
-
-        shell.on('-i', '--service-auth-issuer [FILE]', "Google service account issuer file") do |given|
-          opts[:service_auth_issuer_file] = given
-        end
-
-        shell.on('-p', '--service-auth-p12 [FILE]', "Google service account p12 file") do |given|
-          opts[:service_auth_p12_file] = given
-        end
-
-        shell.on('-c', '--config [FILE]', "Application config file (json)") do |given|
-          opts[:config_file] = given
-          opts[:app] = JSON.load( File.open(given, 'r') )
-        end
-
-        shell.on('-h', '--help', "Prints this help") do |given|
-          puts shell; exit
-        end
-
-        shell.on('-v', '--version', "Prints TQ version") do |given|
-          puts TQ::VERSION; exit
-        end
-
-      end.parse(argv)
-
-      return opts
-    end
-
-  end
-
-end