tq 0.2.1 → 0.3.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 877e9730d05610f3b211c7a0d69817f60149bde4
-  data.tar.gz: eb2d73a1f7e5a63659fe53041171bde055e59523
+  metadata.gz: 6c715d2a759e261f57e2f4f3a745e8c641187e8e
+  data.tar.gz: a996d693740eb9a6846bc510e178e6c935639b41
 SHA512:
-  metadata.gz: 9c2539a74728b068c962eab6e1637559664e1a5a74514623893b756166c1246641c5b33dc5892e671ada06a7e2760bba2453137b0ba98f74813c729bbdbaed3b
-  data.tar.gz: 2981fa8c831cf086873d2d9dfd2fef886533dc01baaa5521b0d4ea2f6f6a701ed37a165299f558a92ba3397d98ddab54415f1658bad433192335a261063b9b45
+  metadata.gz: 7c350e6cd8cf00d23926f4bb980af5f8221a6346ba2d3047480dd80ec0052fd81f3441fd0df7b83c26d12bdd6391bfaf8ce39e04a77094acb9c3e689856440c7
+  data.tar.gz: 44cbccddca8ed63e62e06e629b70c81598e43078df946f507e796821246d547f34fd92d414c8135db76c840ffcf59550ab82a6e043f0d5e7b1e906027f4b0aa6

data/.gems

@@ -1,3 +1,3 @@
-google-api-client -v 0.8.4
-minitest -v 5.5.1
-parallel -v 1.4.1
+google-api-client -v 0.19.3
+minitest -v 5.11.1
+parallel -v 1.12.1

data/lib/tq.rb

@@ -1,5 +1,252 @@
+require 'json'
+
+require 'googleauth'
+require 'google/apis'
+require 'google/apis/cloudtasks_v2beta2'
+require 'parallel'
+
 require_relative 'version'
-require_relative 'tq/app'
-require_relative 'tq/queue'
-require_relative 'tq/logger'
 
+module TQ
+
+  CloudTasks = Google::Apis::CloudtasksV2beta2
+
+  API_SCOPES = ['https://www.googleapis.com/auth/cloud-tasks' ]  
+  
+  module Utils
+
+    def options_including(keys, options)
+      options.select { |k,_| keys.include?(k) }
+    end
+
+  end
+
+  class App
+
+    attr_reader :worker, :queue_in, :queue_out, :queue_err, :logger, :concurrency, :env
+    def initialize( worker, queue_in, queue_out: nil, queue_err: nil, 
+                    logger: nil, concurrency: 1, env: {} )
+      @worker = worker
+      @queue_in = queue_in
+      @queue_out = queue_out
+      @queue_err = queue_err
+      @logger = logger
+      @concurrency = concurrency
+      @env = env
+    end
+
+    def call(auth_file=nil)
+      if auth_file
+        run!( service_account_client(auth_file) )
+      else
+        run!( default_client )
+      end
+    end
+
+    def run!(client)
+      setup_api_logger!
+      qin = TQ::Queue.new(client, queue_in)
+      qout = queue_out.nil? ? nil : TQ::Queue.new(client, queue_out)
+      qerr = queue_err.nil? ? nil : TQ::Queue.new(client, queue_err)
+      
+      tasks = qin.lease!
+
+      Parallel.each(tasks, :in_threads => concurrency) do |task| 
+        worker.new(qin, qout, qerr, env_with_logger).call(task)
+      end
+    end
+
+    def default_client
+      CloudTasks::CloudTasksService.new
+    end
+
+    def service_account_client(file)
+      return TQ::ServiceAccount.client(file)
+    end
+
+    def setup_api_logger!
+      Google::Apis.logger = logger
+    end
+
+    def env_with_logger
+      env.merge({ 'logger' => logger })
+    end
+
+  end
+
+  module ServiceAccount
+    extend self
+
+    def client(file)
+      creds = Google::Auth::ServiceAccountCredentials.make_creds(
+         :json_key_io => File.open(file, 'r'),
+         :scope => TQ::API_SCOPES
+      )
+      creds.fetch_access_token!
+
+      client = CloudTasks::CloudTasksService.new
+      client.authorization = creds
+      client
+    end
+
+  end
+
+  class QueueSpec
+    extend TQ::Utils
+
+    def self.from_hash(data)
+      return new( 
+        data['project'], 
+        data['location'], 
+        data['name'],
+        **options_including(['lease_duration','max_tasks'], data)
+       )
+    end
+
+    attr_reader :project, :location, :name, :lease_duration, :max_tasks
+    def initialize(project, location, name,
+                   lease_duration: '60s', max_tasks: 1)
+        @project = project
+        @location = location
+        @name = name
+        @lease_duration = lease_duration
+        @max_tasks = max_tasks
+    end
+
+    def queue_name
+      "projects/#{project}/locations/#{location}/queues/#{name}"
+    end
+
+  end
+
+
+  class Queue
+    
+    attr_reader :client, :queue
+    def initialize(client, spec)
+      @client = client
+      @queue = spec
+    end
+
+    def queue_name
+      queue.queue_name
+    end
+
+    def lease!
+      msg = { :lease_duration => queue.lease_duration, 
+              :max_tasks => queue.max_tasks,
+              :response_view => 'FULL'
+            }
+      results = client.lease_tasks( queue_name,
+          CloudTasks::LeaseTasksRequest.new( **msg )
+      )
+      items = results.tasks || []
+      items.map {|t| new_task(t)}
+    end
+    
+
+    # Note: does not actually extend the time, but sets the total duration
+    # to the given amount (minus any already elapsed time).
+    #
+    def extend!(task, dur)
+      msg = { :lease_duration => dur,
+              :schedule_time => task.schedule_time,
+              :response_view => 'FULL'
+            }
+      results = client.renew_task_lease( task.name,
+          CloudTasks::RenewLeaseRequest.new( **msg )
+      )
+      return new_task(results)
+    end
+      
+    def push!(payload, tag=nil)
+      msg = { :payload => encode(payload), :tag => tag }.reject {|k,v| v.nil?}
+      results = client.create_task( queue_name, 
+          CloudTasks::CreateTaskRequest.new( 
+              :task => CloudTasks::Task.new(
+                  :pull_message => CloudTasks::PullMessage.new( **msg )
+              )
+          )
+      )
+      new_task(results)
+    end
+
+    def finish!(task)
+      results = client.acknowledge_task( task.name,
+          CloudTasks::AcknowledgeTaskRequest.new( :schedule_time => task.schedule_time )
+      )
+      return
+    end
+    
+
+    private
+    
+    def new_task(t)
+      TQ::Task.new self, t
+    end
+
+    def encode(obj)
+      JSON.dump(obj)
+    end
+ 
+  end
+
+  class Task 
+    
+    attr_reader :task
+    def initialize(queue, task)
+      @queue = queue
+      @task = task
+      @clock = Time
+    end
+
+    def name
+      self.task.name
+    end
+
+    def expires
+      DateTime.rfc3339(self.task.schedule_time).to_time
+    end
+
+    def tag
+      self.task.tag
+    end
+
+    def payload
+      decode self.task.pull_message.payload
+    end
+
+    def finish!
+      @queue.finish!(self.task)
+    end
+
+    def extend!(dur)
+      @queue.extend!(self.task, dur)
+    end
+
+    def clock!(_)
+      @clock = _; return self
+    end
+
+    def reset_clock!
+      @clock = Time; return self
+    end
+
+    def lease_remaining
+      self.expires - @clock.now
+    end
+
+    def lease_expired?
+      self.expires < @clock.now
+    end
+
+    private
+
+    def decode(str)
+      JSON.load(str)
+    end
+    
+  end
+
+  
+end

data/lib/version.rb

@@ -1,4 +1,4 @@
 
 module TQ
-  VERSION = '0.2.1'
+  VERSION = '0.3.1'
 end

data/test/helper.rb

@@ -4,7 +4,9 @@ require 'logger'
 gem 'minitest'
 require 'minitest/autorun'
 
-require 'google/api_client'
+require 'googleauth'
+require 'google/apis'
+require 'google/apis/cloudtasks_v2beta2'
 
 module TestUtils
   extend self
@@ -14,122 +16,64 @@ module TestUtils
     FileUtils.mkdir_p( File.dirname(file) )
     logger = Logger.new( File.open(file, 'w' ) )
     logger.level = ENV['DEBUG'] ? Logger::DEBUG : Logger::INFO
-    Google::APIClient.logger = logger
+    Google::Apis.logger = logger
   end
 
   def current_logger
-    Google::APIClient.logger
+    Google::Apis.logger
   end
 
   class QueueHelper
 
-    attr_reader :project, :queue
-    def initialize(project,queue)
-      @project, @queue = project, queue
-      @service_auth = false
-    end
-
-    def auth_files(secrets,creds)
-      @secrets_file, @creds_file = secrets, creds
-      @service_auth = false
-      return self
-    end
-    
-    def service_auth_files(issuer,p12)
-      @secrets_issuer, @secrets_p12 = issuer, p12
-      @service_auth = true
-      return self
-    end
-
-    def authorized_client
-      @authorized_client ||= (
-        if @service_auth
-          TQ::App.new('test_app',nil).service_auth!(
-            File.read(@secrets_issuer).chomp, @secrets_p12
-          )
-        else
-          TQ::App.new('test_app',nil).auth!(@secrets_file, @creds_file)
-        end
-      )
-    end
+    CloudTasks = Google::Apis::CloudtasksV2beta2
 
-    # Note: inaccurate, don't use
-    def peek()
-      client, api = authorized_client
-      results = client.execute!( :api_method => api.tasks.list,
-                                 :parameters => { :project => project, :taskqueue => queue }
-                )
-      items = results.data['items'] || []
+    def initialize(spec, authfile)
+      @queue = spec
+      @authfile = authfile
     end
 
-    def push!(payload)
-      client, api = authorized_client
-      client.execute!( :api_method => api.tasks.insert,
-                       :parameters => { :project => project, :taskqueue => queue },
-                       :body_object => { 
-                         'queueName' => queue, 
-                         'payloadBase64' => encode(payload)
-                       }
+    def purge!
+      client = service_account_client
+      client.purge_queue(@queue.queue_name, 
+          CloudTasks::PurgeQueueRequest.new
       )
+      return
     end
 
-    def pop!(n=1)
-      client, api = authorized_client
-      results = client.execute!( :api_method => api.tasks.lease,
-                                 :parameters => { :project => project, :taskqueue => queue, 
-                                                  :leaseSecs => 60, :numTasks => n
-                                 }
-                )
-      items = results.data['items'] || []
-      items.each do |item|
-        client.execute!( :api_method => api.tasks.delete,
-                         :parameters => { :project => project, :taskqueue => queue, :task => item['id'] }
-        )
+    # Note: longer way to do it, but gets the count
+    def clear!
+      client = service_account_client
+      results = client.list_project_location_queue_tasks(@queue.queue_name)
+      tasks = (results.tasks || [])
+      tasks.each do |t|
+        client.delete_project_location_queue_task( t.name )
       end
-      return items
-    end
-
-    def all_payloads!
-      map! { |t| decode(t['payloadBase64']) }
+      return tasks
     end
 
-    def all_tags!
-      map! { |t| t['tag'] }
+    def push!(data)
+      client = service_account_client
+      q = TQ::Queue.new(client, @queue)
+      return q.push!(data)
     end
 
-    def all_payloads_and_tags!
-      map! { |t| {:payload => decode(t['payloadBase64']),
-                  :tag => t['tag']
-                 }
-           }
-    end
-
-    def map!(&b)
-      clear!.map(&b) 
-    end
-
-    def clear!
-      client, api = authorized_client
-      done = false
-      all = []
-      while !done do
-        batch = pop!(10)
-        done = batch.empty? || batch.length < 10
-        all = all + batch
-      end
-      all
-    end
+    def service_account_client
+      creds = Google::Auth::ServiceAccountCredentials.make_creds(
+         :json_key_io => File.open(@authfile, 'r'),
+         :scope => TQ::API_SCOPES
+      )
+      creds.fetch_access_token!
 
-    def encode(obj)
-      Base64.urlsafe_encode64(JSON.dump(obj))
+      client = CloudTasks::CloudTasksService.new
+      client.authorization = creds
+      client
     end
 
-    def decode(str)
-      JSON.load(Base64.urlsafe_decode64(str))
+    def messages!
+      clear!.map {|t| t.pull_message }
     end
 
   end
 
-
 end
 

data/test/suite.rb

@@ -1,4 +1,4 @@
 
-%w[ test_shell test_run test_logger ].each do |rb|
+%w[ test_auth test_run test_logger ].each do |rb|
   require_relative rb
 end