delayed_job-ssh_remote_worker 1.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: f194bed80798938fa56694242c4b189775a1cb9c
+  data.tar.gz: 8781173a79be3b715016a9aecf1c6aaeca650172
+SHA512:
+  metadata.gz: d1f1e47cc0c9106a5eb0138acf79288b9a55f6bd3eac58ae7d689a8382151cc98a817386908df2fb8c6f9a105a900f4e45e0f6135018ceaf0b1e101581b61020
+  data.tar.gz: d8848fd43a2e81f200840b552bf9f7bf8a29ce2a0a3b7dd03cac8f9c56a1c4b258e064e174c40340ef85ca122657ba811c6c117761110a7d8310918d6cfbf619

data/.gitignore

@@ -0,0 +1,22 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp
+*.bundle
+*.so
+*.o
+*.a
+mkmf.log

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in delayed_job-ssh_remote_worker.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2014 Alex McHale
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,77 @@
+# DelayedJob::SshRemoteWorker
+
+This gem provides a rake task for executing DelayedJob backed by ActiveRecord
+remotely over an SSH connection.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'delayed_job-ssh_remote_worker'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install delayed_job-ssh_remote_worker
+
+## Usage
+
+Require the gem and either create a `config/remote_database.yml` file that
+contains a record such as below:
+
+```yaml
+production:
+  ssh_hostname: "example.com"
+  ssh_username: "app"
+  adapter:      "postgresql"
+  encoding:     "unicode"
+  database:     "dbname"
+  username:     "dbuser"
+  password:     "pa$$word"
+```
+
+Alternatively, the `ssh_hostname` and `ssh_username` can be simply added to an
+entry in `config/database.yml`. You may specify the SSH port number as `ssh_port`.
+
+The `config/remote_database.yml` top level keys can have names other than your
+`Rails.env`. For example:
+
+```yaml
+fast_server:
+  ssh_hostname: "example.com"
+  ssh_username: "app"
+  ssh_port:     20022
+  adapter:      "postgresql"
+  encoding:     "unicode"
+  database:     "dbname"
+  username:     "dbuser"
+  password:     "pa$$word"
+
+slow_server:
+  ssh_hostname: "example2.com"
+  ssh_username: "app"
+  adapter:      "mysql"
+  encoding:     "unicode"
+  database:     "dbname"
+  username:     "dbuser"
+  password:     "pa$$word"
+```
+
+The key can then be specified in `ENV['REMOTE_ENV']` thusly:
+
+```bash
+RAILS_ENV=production REMOTE_ENV=fast_server bundle exec rake jobs:remote_workoff
+```
+
+If `ENV['REMOTE_ENV']` is not set, `delayed_job-ssh_remote_worker` will default to `Rails.env`.
+
+## Contributing
+
+1. Fork it ( https://github.com/alexmchale/delayed_job-ssh_remote_worker/fork )
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create a new Pull Request

data/Rakefile

@@ -0,0 +1,2 @@
+require "bundler/gem_tasks"
+

data/delayed_job-ssh_remote_worker.gemspec

@@ -0,0 +1,29 @@
+# encoding: utf-8
+
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'delayed/ssh_remote_worker/version'
+
+Gem::Specification.new do |spec|
+
+  spec.name          = "delayed_job-ssh_remote_worker"
+  spec.version       = DelayedJob::SshRemoteWorker::VERSION
+  spec.authors       = ["Alex McHale"]
+  spec.email         = ["alex@anticlever.com"]
+  spec.summary       = %q{Run DelayedJob queues over SSH in a remote database}
+  spec.description   = %q{Provides a few rake tasks to run DelayedJob jobs on a remote server over SSH}
+  spec.homepage      = "http://github.com/alexmchale/delayed_job-ssh_remote_worker"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files -z`.split("\x0")
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_runtime_dependency "delayed_job_active_record", ">= 3.0.0", "< 4.2.0"
+  spec.add_runtime_dependency "net-ssh-gateway", "~> 1.2"
+
+  spec.add_development_dependency "bundler", "~> 1.6"
+  spec.add_development_dependency "rake", "~> 10.3"
+
+end

data/lib/delayed/ssh_remote_worker.rb

@@ -0,0 +1,116 @@
+require "delayed/ssh_remote_worker/version"
+require "net/ssh/gateway"
+
+module Delayed
+  class SshRemoteWorker
+
+    attr_reader :remote_ssh_username, :remote_ssh_hostname, :remote_ssh_port
+    attr_reader :remote_db_hostname, :remote_db_port
+    attr_reader :local_db_hostname, :local_db_port
+    attr_reader :gateway_ssh, :gateway_port
+    attr_reader :gateway_in_rd, :gateway_in_wr
+    attr_reader :gateway_out_rd, :gateway_out_wr
+    attr_reader :child
+    attr_reader :config
+
+    def initialize(config)
+      @config              = Hash[config.map { |k, v| [ k.to_sym, v ] }]
+      @debug               = !!@config[:ssh_debug]
+
+      @remote_ssh_username = @config[:ssh_username]
+      @remote_ssh_hostname = @config[:ssh_hostname]
+      @remote_ssh_port     = @config[:ssh_port].to_i
+      @remote_ssh_port     = 22 if remote_ssh_port <= 0
+
+      @remote_db_hostname  = @config[:host].to_s.strip
+      @remote_db_hostname  = "localhost" if remote_db_hostname == ""
+      @remote_db_port      = @config[:port].to_i
+      @remote_db_port      = default_port_number(@config[:adapter]) if remote_db_port <= 0
+
+      @local_db_hostname = "127.0.0.1"
+      @local_db_port     = nil
+    end
+
+    def create_port_tunnel
+      # Create two pipes - one will be used to get the port number from the child
+      # process, the other will be used to tell the child process when it's time
+      # to exit.
+      ( @gateway_in_rd  , @gateway_in_wr  ) = IO.pipe
+      ( @gateway_out_rd , @gateway_out_wr ) = IO.pipe
+
+      # The child process will establish the SSH connection to the database server.
+      @child = fork {
+        debug "CHILD: Tunneling to #{ remote_db_hostname }:#{ remote_db_port } on #{ remote_ssh_username }@#{ remote_ssh_hostname }:#{ remote_ssh_port }."
+
+        @gateway_ssh  = Net::SSH::Gateway.new(remote_ssh_hostname, remote_ssh_username, port: remote_ssh_port)
+        @gateway_port = gateway_ssh.open(remote_db_hostname, remote_db_port)
+
+        debug "CHILD: Notifying parent process of gateway port number #{ gateway_port }."
+
+        gateway_out_rd.close
+        gateway_out_wr.write(gateway_port.to_s)
+        gateway_out_wr.close
+
+        debug "CHILD: Waiting for parent to close pipe."
+
+        gateway_in_wr.close
+        gateway_in_rd.read
+        gateway_in_rd.close
+
+        debug "CHILD: Parent closed pipe. Exiting."
+
+        gateway_ssh.shutdown!
+      }
+
+      # Get the port number back from the child process once the SSH connection is established.
+      debug "PARENT: Reading port number from child."
+      gateway_out_wr.close
+      @local_db_port = Integer(gateway_out_rd.read)
+      gateway_out_rd.close
+
+      # Log and proceed with the next task.
+      debug "PARENT: Received port number #{ gateway_port } from child. Proceeding with execution."
+    end
+
+    def connect_to_tunneled_database
+      # Load just ActiveRecord. Do this before any part of the native Rails project is loaded.
+      require "active_record"
+
+      # Connect to the remote Postgres database.
+      ActiveRecord::Base.establish_connection(activerecord_config)
+
+      # Log a simple query just to verify that we actually have a connection.
+      debug "Connected to remote Postgres. Found #{ User.count } users in the remote database."
+    end
+
+    def destroy_port_tunnel
+      # Close the pipe with the gateway, indicating that it should close.
+      debug "PARENT: Closing pipe with child to terminate gateway."
+      gateway_in_wr.close
+      gateway_in_rd.close
+      Process.wait(child)
+    end
+
+    private
+
+    def default_port_number(adapter)
+      case adapter.to_s.strip
+      when /mysql/i    then 3306
+      when /postgres/i then 5432
+      else raise "no known default port number for adapter #{ adapter.inspect }"
+      end
+    end
+
+    def activerecord_config
+      config.merge({
+        :host => local_db_hostname ,
+        :port => local_db_port     ,
+      })
+    end
+
+    def debug(message)
+      @debug and puts(message)
+    end
+
+  end
+end

data/lib/delayed/ssh_remote_worker/railtie.rb

@@ -0,0 +1,16 @@
+require 'delayed/ssh_remote_worker'
+require 'rails'
+
+module Delayed
+  class SshRemoteWorker
+    class Railtie < Rails::Railtie
+
+      railtie_name :delayed_job_ssh_remote_worker
+
+      rake_tasks do
+        load "delayed/ssh_remote_worker/tasks.rb"
+      end
+
+    end
+  end
+end

data/lib/delayed/ssh_remote_worker/tasks.rb

@@ -0,0 +1,60 @@
+namespace :jobs do
+
+  task :create_port_tunnel do
+    load_yaml = -> filename {
+      filename =
+        if defined?(Rails)
+          Rails.root.join("config", filename)
+        else
+          File.join("config", filename)
+        end
+
+      if File.file? filename
+        YAML.load File.read filename
+      end
+    }
+
+    get_hash = -> hash, key {
+      hash2 = hash[key.to_s.strip]
+
+      if hash2.kind_of? Hash
+        hash2
+      else
+        hash
+      end
+    }
+
+    config = load_yaml["remote_database.yml"] || load_yaml["database.yml"]
+    config = get_hash[config, ENV["REMOTE_ENV"]]
+    config = get_hash[config, Rails.env.to_s]
+
+    $ssh_remote_worker = Delayed::SshRemoteWorker.new(config)
+    $ssh_remote_worker and $ssh_remote_worker.create_port_tunnel
+  end
+
+  task :connect_to_tunneled_database do
+    $ssh_remote_worker and $ssh_remote_worker.connect_to_tunneled_database
+  end
+
+  task :destroy_port_tunnel do
+    $ssh_remote_worker and $ssh_remote_worker.destroy_port_tunnel
+    $ssh_remote_worker = nil
+  end
+
+  desc "Start a remote delayed_job worker"
+  task :remote_work => %w(
+    jobs:create_port_tunnel
+    jobs:connect_to_tunneled_database
+    jobs:work
+    jobs:destroy_port_tunnel
+  )
+
+  desc "Start a remote delayed_job worker and exit when all available jobs are complete"
+  task :remote_workoff => %w(
+    jobs:create_port_tunnel
+    jobs:connect_to_tunneled_database
+    jobs:workoff
+    jobs:destroy_port_tunnel
+  )
+
+end

data/lib/delayed/ssh_remote_worker/version.rb

@@ -0,0 +1,7 @@
+module DelayedJob
+  class SshRemoteWorker
+
+    VERSION = "1.0.0"
+
+  end
+end

data/lib/delayed_job-ssh_remote_worker.rb

@@ -0,0 +1,2 @@
+require "delayed/ssh_remote_worker"
+require "delayed/ssh_remote_worker/railtie" if defined?(Rails)

metadata

@@ -0,0 +1,118 @@
+--- !ruby/object:Gem::Specification
+name: delayed_job-ssh_remote_worker
+version: !ruby/object:Gem::Version
+  version: 1.0.0
+platform: ruby
+authors:
+- Alex McHale
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-09-03 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: delayed_job_active_record
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 3.0.0
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: 4.2.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 3.0.0
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: 4.2.0
+- !ruby/object:Gem::Dependency
+  name: net-ssh-gateway
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.2'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.6'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.6'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.3'
+description: Provides a few rake tasks to run DelayedJob jobs on a remote server over
+  SSH
+email:
+- alex@anticlever.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- delayed_job-ssh_remote_worker.gemspec
+- lib/delayed/ssh_remote_worker.rb
+- lib/delayed/ssh_remote_worker/railtie.rb
+- lib/delayed/ssh_remote_worker/tasks.rb
+- lib/delayed/ssh_remote_worker/version.rb
+- lib/delayed_job-ssh_remote_worker.rb
+homepage: http://github.com/alexmchale/delayed_job-ssh_remote_worker
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.2.2
+signing_key: 
+specification_version: 4
+summary: Run DelayedJob queues over SSH in a remote database
+test_files: []