activejob-google_cloud_pubsub 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 9c7fb77dfcc2712ebff760e45cb9fcfb605230f1
+  data.tar.gz: '086f22928fb681f54ede2c7c114433362b699bc7'
+SHA512:
+  metadata.gz: 0d15086d08144818ae5ca082c0e9f0457455b2d02bd5471cfaf5dad6d90096eaabf0f38317448d2886f7a8a84af9e9d48b8e0601863a1a4187433e6b0afaffd5
+  data.tar.gz: 9a73c4e6a1f361a2346e467f48aa62820f766ed1ac518947166a935c9d169582506a832997e212b915889fe46b02370b87ee743821da1be5504d107e435b36ec

data/.gitignore

@@ -0,0 +1,12 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+
+# rspec failure tracking
+.rspec_status

data/.rspec

@@ -0,0 +1,2 @@
+--format documentation
+--color

data/.travis.yml

@@ -0,0 +1,10 @@
+sudo: false
+language: ruby
+
+rvm:
+  - 2.4.1
+
+before_install:
+  - curl -sL https://dl.google.com/dl/cloudsdk/channels/rapid/google-cloud-sdk.tar.gz | tar xz
+  - google-cloud-sdk/install.sh --override-components beta pubsub-emulator --quiet
+  - export PATH=$PATH:$PWD/google-cloud-sdk/bin

data/Gemfile

@@ -0,0 +1,5 @@
+source 'https://rubygems.org'
+
+gemspec
+
+gem 'concurrent-ruby-ext'

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2017 Keita Urashima
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,110 @@
+# ActiveJob::GoogleCloudPubsub
+
+Google Cloud Pub/Sub adapter and worker for ActiveJob
+
+## Installation
+
+```ruby
+gem 'activejob-google_cloud_pubsub'
+```
+
+## Usage
+
+First, change the ActiveJob backend.
+
+``` ruby
+Rails.application.config.active_job.queue_adapter = :google_cloud_pubsub
+```
+
+Write the Job class and code to use it.
+
+``` ruby
+class HelloJob < ApplicationJob
+  def perform(name)
+    puts "hello, #{name}!"
+  end
+end
+```
+
+``` ruby
+class HelloController < ApplicationController
+  def say
+    HelloJob.perform_later params[:name]
+  end
+end
+```
+
+In order to test the worker in your local environment, it is a good idea to use the Pub/Sub emulator provided by `gcloud` command. Refer to [this document](https://cloud.google.com/pubsub/docs/emulator) for the installation procedure.
+
+When the installation is completed, execute the following command to start up the worker.
+
+``` sh
+$ gcloud beta emulators pubsub start
+
+(Switch to another terminal)
+
+$ eval `gcloud beta emulators pubsub env-init`
+$ cd path/to/your-app
+$ bundle exec activejob-google_cloud_pubsub-worker
+```
+
+If you hit the previous action, the job will be executed.
+(Both the emulator and the worker stop with <kbd>Ctrl+C</kbd>)
+
+## Configuration
+
+### Adapter
+
+When passing options to the adapter, you need to create the object instead of a symbol.
+
+``` ruby
+Rails.application.config.active_job.queue_adapter = ActiveJob::QueueAdapters::GoogleCloudPubsubAdapter.new(options)
+```
+
+All options are passed to [`Google::Cloud::Pubsub.new`](http://googlecloudplatform.github.io/google-cloud-ruby/#/docs/google-cloud-pubsub/v0.23.2/google/cloud/pubsub?method=new-class).
+
+### Worker
+
+The following command line flags can be specified. All flags are optional.
+
+#### `--require=PATH`
+
+The path of the file to load before the worker starts up.
+
+Default: `./config/environment`
+
+#### `--queue=NAME`
+
+The name of the queue the worker handles.
+
+Note: One worker can handle only one queue. If you use multiple queues, you need to launch multiple worker processes.
+
+Default: `default`
+
+#### `--min_threads=N`
+
+Minimum number of worker threads.
+
+Default: `0`
+
+#### `--max_threads=N`
+
+Maximum number of worker threads.
+
+Default: number of logical cores
+
+#### `--project=PROJECT_ID`, `--keyfile=PATH`
+
+Credentials of Google Cloud Platform. Please refer to [the document](https://github.com/GoogleCloudPlatform/google-cloud-ruby/blob/master/AUTHENTICATION.md) for details.
+
+``` sh
+$ bundle exec rake spec
+```
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/ursm/activejob-google_cloud_pubsub.
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT).

data/Rakefile

@@ -0,0 +1,6 @@
+require 'bundler/gem_tasks'
+require 'rspec/core/rake_task'
+
+RSpec::Core::RakeTask.new(:spec)
+
+task :default => :spec

data/activejob-google_cloud_pubsub.gemspec

@@ -0,0 +1,31 @@
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'active_job/google_cloud_pubsub/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = 'activejob-google_cloud_pubsub'
+  spec.version       = ActiveJob::GoogleCloudPubsub::VERSION
+  spec.authors       = ['Keita Urashima']
+  spec.email         = ['ursm@ursm.jp']
+
+  spec.summary       = 'Google Cloud Pub/Sub adapter and worker for ActiveJob'
+  spec.homepage      = 'https://github.com/ursm/activejob-google_cloud_pubsub'
+  spec.license       = 'MIT'
+
+  spec.files         = `git ls-files -z`.split("\x0").reject do |f|
+    f.match(%r{^(test|spec|features)/})
+  end
+
+  spec.bindir        = 'exe'
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ['lib']
+
+  spec.add_runtime_dependency 'activejob'
+  spec.add_runtime_dependency 'activesupport'
+  spec.add_runtime_dependency 'concurrent-ruby'
+  spec.add_runtime_dependency 'google-cloud-pubsub'
+
+  spec.add_development_dependency 'bundler'
+  spec.add_development_dependency 'rake'
+  spec.add_development_dependency 'rspec'
+end

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require 'bundler/setup'
+require 'activejob-google_cloud_pubsub'
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require 'irb'
+IRB.start(__FILE__)

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/exe/activejob-google_cloud_pubsub-worker

@@ -0,0 +1,41 @@
+#!/usr/bin/env ruby
+
+require 'activejob-google_cloud_pubsub'
+require 'google/cloud/pubsub'
+require 'optparse/kwargs'
+
+Version = ActiveJob::GoogleCloudPubsub::VERSION
+
+args = {
+  require: './config/environment'
+}
+
+opt = OptionParser.new
+
+opt.on '--[no-]require=PATH' do |v|
+  args[:require] = v
+end
+
+worker_args = opt.define_by_keywords(
+  {},
+  ActiveJob::GoogleCloudPubsub::Worker.instance_method(:initialize),
+  {
+    min_threads: Integer,
+    max_threads: Integer
+  }
+)
+
+pubsub_args = opt.define_by_keywords(
+  {},
+  Google::Cloud::Pubsub.method(:new),
+  {
+    scope:   Array,
+    timeout: Integer
+  }
+)
+
+opt.parse ARGV
+
+require args[:require] if args[:require]
+
+ActiveJob::GoogleCloudPubsub::Worker.new(**worker_args, **pubsub_args).run

data/lib/active_job/google_cloud_pubsub/adapter.rb

@@ -0,0 +1,29 @@
+require 'active_job/google_cloud_pubsub/naming'
+require 'google/cloud/pubsub'
+require 'json'
+
+module ActiveJob
+  module GoogleCloudPubsub
+    class Adapter
+      include Naming
+
+      def initialize(**pubsub_args)
+        @pubsub = Google::Cloud::Pubsub.new(pubsub_args)
+      end
+
+      def enqueue(job, attributes = {})
+        topic = @pubsub.topic(topic_name(job.queue_name), autocreate: true)
+
+        topic.publish JSON.dump(job.serialize), attributes
+      end
+
+      def enqueue_at(job, timestamp)
+        enqueue job, timestamp: timestamp
+      end
+    end
+  end
+end
+
+require 'active_job'
+
+ActiveJob::QueueAdapters::GoogleCloudPubsubAdapter = ActiveJob::GoogleCloudPubsub::Adapter

data/lib/active_job/google_cloud_pubsub/naming.rb

@@ -0,0 +1,13 @@
+module ActiveJob
+  module GoogleCloudPubsub
+    module Naming
+      def topic_name(queue_name)
+        "activejob-queue-#{queue_name}"
+      end
+
+      def subscription_name(queue_name)
+        "activejob-worker-#{queue_name}"
+      end
+    end
+  end
+end

data/lib/active_job/google_cloud_pubsub/version.rb

@@ -0,0 +1,5 @@
+module ActiveJob
+  module GoogleCloudPubsub
+    VERSION = '0.1.0'
+  end
+end

data/lib/active_job/google_cloud_pubsub/worker.rb

@@ -0,0 +1,98 @@
+require 'active_job/base'
+require 'active_job/google_cloud_pubsub/naming'
+require 'active_support/core_ext/numeric/time'
+require 'concurrent'
+require 'google/cloud/pubsub'
+require 'json'
+require 'logger'
+
+module ActiveJob
+  module GoogleCloudPubsub
+    class Worker
+      include Naming
+
+      MAX_DEADLINE = 10.minutes
+
+      cattr_accessor(:logger) { Logger.new($stdout) }
+
+      def initialize(queue: 'default', min_threads: 0, max_threads: Concurrent.processor_count, **pubsub_args)
+        @queue_name, @min_threads, @max_threads = queue, min_threads, max_threads
+
+        @pubsub = Google::Cloud::Pubsub.new(**pubsub_args)
+      end
+
+      def subscription
+        topic = @pubsub.topic(topic_name(@queue_name), autocreate: true)
+
+        topic.subscription(subscription_name(@queue_name)) || topic.subscribe(subscription_name(@queue_name))
+      end
+      alias ensure_subscription subscription
+
+      def run
+        pool = Concurrent::ThreadPoolExecutor.new(min_threads: @min_threads, max_threads: @max_threads, max_queue: -1)
+
+        subscription.listen do |message|
+          begin
+            Concurrent::Promise.execute(args: message, executor: pool) {|msg|
+              process msg
+            }.rescue {|e|
+              logger.error e
+            }
+          rescue Concurrent::RejectedExecutionError
+            message.delay! 10.seconds.to_i
+          end
+        end
+      end
+
+      private
+
+      def process(message)
+        if timestamp = message.attributes['timestamp']
+          ts = Time.at(timestamp.to_f)
+
+          if ts >= Time.now
+            _process message
+          else
+            message.delay! [ts - Time.now, MAX_DEADLINE.to_i].min
+          end
+        else
+          _process message
+        end
+      end
+
+      def _process(message)
+        timer_opts = {
+          execution_interval: MAX_DEADLINE - 10.seconds,
+          timeout_interval:   5.seconds,
+          run_now:            true
+        }
+
+        delay_timer = Concurrent::TimerTask.execute(timer_opts) {
+          message.delay! MAX_DEADLINE.to_i
+        }
+
+        begin
+          succeeded = false
+          failed    = false
+
+          ActiveJob::Base.execute JSON.parse(message.data)
+
+          succeeded = true
+        rescue Exception
+          failed = true
+
+          raise
+        ensure
+          delay_timer.shutdown
+
+          if succeeded || failed
+            message.acknowledge!
+          else
+            # terminated from outside
+            message.delay! 0
+          end
+        end
+      end
+    end
+  end
+end

data/lib/activejob-google_cloud_pubsub.rb

@@ -0,0 +1,12 @@
+require 'active_job'
+
+module ActiveJob
+  module GoogleCloudPubsub
+    autoload :VERSION, 'active_job/google_cloud_pubsub/version'
+    autoload :Worker,  'active_job/google_cloud_pubsub/worker'
+  end
+
+  module QueueAdapters
+    autoload :GoogleCloudPubsubAdapter, 'active_job/google_cloud_pubsub/adapter'
+  end
+end

metadata

@@ -0,0 +1,159 @@
+--- !ruby/object:Gem::Specification
+name: activejob-google_cloud_pubsub
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Keita Urashima
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2017-04-01 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: activejob
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: concurrent-ruby
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: google-cloud-pubsub
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: 
+email:
+- ursm@ursm.jp
+executables:
+- activejob-google_cloud_pubsub-worker
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rspec"
+- ".travis.yml"
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- activejob-google_cloud_pubsub.gemspec
+- bin/console
+- bin/setup
+- exe/activejob-google_cloud_pubsub-worker
+- lib/active_job/google_cloud_pubsub/adapter.rb
+- lib/active_job/google_cloud_pubsub/naming.rb
+- lib/active_job/google_cloud_pubsub/version.rb
+- lib/active_job/google_cloud_pubsub/worker.rb
+- lib/activejob-google_cloud_pubsub.rb
+homepage: https://github.com/ursm/activejob-google_cloud_pubsub
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.6.11
+signing_key: 
+specification_version: 4
+summary: Google Cloud Pub/Sub adapter and worker for ActiveJob
+test_files: []