jobbie_oci 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 75785a941b0a3e17af5387fbd61e775a881018a4
+  data.tar.gz: f9d3b9416587e292b184efb0015a6a2d5e8ccf41
+SHA512:
+  metadata.gz: 19e62e7a8ef1808507143e01cbbb97524b65b34e32485490a204c7a54eef733e1584ecb7f013f029b10e66998bd4d4e233f02a01231835af4e03c908cb06667f
+  data.tar.gz: 7b736413ac7b793f5111a95df6cf9d5d6a10b4625699c158b6a810bfd127a5259a46e59d7d0ec26a3c9d0d05871463e79b20080d2e3d4bb52408bf55e651bb84

data/.gitignore

@@ -0,0 +1,9 @@
+/.bundle/
+/.yardoc
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+*.swp

data/.rubocop.yml

@@ -0,0 +1,10 @@
+Metrics/AbcSize:
+  Enabled: false
+Metrics/PerceivedComplexity:
+  Enabled: false
+Metrics/CyclomaticComplexity:
+  Enabled: false
+Metrics/MethodLength:
+  Max: 1000
+Metrics/ClassLength:
+  Max: 1000

data/Gemfile

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+source 'https://rubygems.org'
+
+git_source(:github) { |repo_name| "https://github.com/#{repo_name}" }
+
+# Specify your gem's dependencies in jobbie_oci.gemspec
+gemspec

data/Gemfile.lock

@@ -0,0 +1,20 @@
+PATH
+  remote: .
+  specs:
+    jobbie_oci (0.1.0)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    rake (10.5.0)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  bundler (~> 1.16)
+  jobbie_oci!
+  rake (~> 10.0)
+
+BUNDLED WITH
+   1.16.1

data/README.md

@@ -0,0 +1,66 @@
+# JobbieOci
+
+A very simple job queuing system that uses OCI Casper buckets as a backend.
+
+Jobbie is intended to fill a gap in Oracle's OCI cloud whilst there is a lack
+of a queueing service similar to Amazon's SNS/SQS.  As soon as such a service
+is introduced, it's unlikely that Jobbie will serve any further purpose.
+
+Jobs are added to an 'execution group', which is used to group related
+tasks together for consumption by a daemon.  Multiple daemons can run in the
+same execution group, but only one daemon will be able to claim the job.  If
+a daemon is unable to complete the job then it will eventually time out and
+become claimable again.
+
+## Installation
+
+```
+gem install jobbie_oci
+```
+
+## Usage
+
+Ensure that you have a bucket in Casper and that you have permission to create
+objects in it.
+
+Jobbie can be run in one of two modes: Either client mode which can submit jobs
+or list them, or daemon mode which takes jobs from the queue and executes them.
+
+To run jobs in daemon mode, you will use:
+```
+jobbie daemon --exec-group=EXEC_GROUP --script-dir=SCRIPT_DIR
+```
+
+An `execution group` is used to to group related tasks together.  Jobs
+submitted in the same execution group will be executed in the same way.  For
+example, you might create a job in the "cleanup\_users" exec group, and a
+jobbie daemon with a matching exec-group will be used to execute a script
+to cleanup the user.
+
+The script dir is the only directory in which scripts triggered by the
+daemon are allowed to run.  No fully qualified paths are allowed in the script
+name.
+
+## Example
+
+Suppose we have a script directory called `scripts` and that we add to it a
+script called `cleanup_users`.
+
+We could add a job to the queue as follows:
+```
+jobbie add --bucket my-bucket --exec-group=USERMGMT --script=cleanup_users user123
+```
+
+Meanwhile we could write start a daemon as follows:
+```
+jobbie daemon --bucket my-bucket --exec-group=USERMGMT --script-dir=scripts
+```
+.. and the daemon would pick up the job and execute the script as if
+someone were to manually run:
+```
+./scripts/cleanup_users user123
+```
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/[USERNAME]/jobbie_oci.

data/Rakefile

@@ -0,0 +1,4 @@
+# frozen_string_literal: true
+
+require 'bundler/gem_tasks'
+task default: :spec

data/exe/jobbie

@@ -0,0 +1,6 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require 'jobbie_oci/cli'
+
+Jobbie::CLI::Main.start(ARGV)

data/jobbie_oci.gemspec

@@ -0,0 +1,26 @@
+
+# frozen_string_literal: true
+
+lib = File.expand_path('lib', __dir__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'jobbie_oci/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = 'jobbie_oci'
+  spec.version       = JobbieOci::VERSION
+  spec.authors       = ['Stephen Pearson']
+  spec.email         = ['stephen.pearson@oracle.com']
+
+  spec.summary       = 'Simple noddy job queuer that uses OCI Casper backend'
+  spec.homepage      = 'http://github.com/stephenpearson/jobbie_oci'
+
+  spec.files = `git ls-files -z`.split("\x0").reject do |f|
+    f.match(%r{^(test|spec|features)/})
+  end
+  spec.bindir        = 'exe'
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ['lib']
+
+  spec.add_development_dependency 'bundler', '~> 1.16'
+  spec.add_development_dependency 'rake', '~> 10.0'
+end

data/lib/jobbie_oci/cli.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+require 'oci'
+require 'thor'
+
+require 'jobbie_oci/jobs'
+
+module Jobbie
+  module CLI
+    # This is the Main entry point for the CLI
+    class Main < Thor
+      class_option :region,
+                   type: :string,
+                   default: 'phx',
+                   desc: 'Region in which to store jobs'
+      class_option :profile,
+                   type: :string,
+                   default: 'DEFAULT',
+                   desc: 'OCI config profile name'
+      class_option :config,
+                   type: :string,
+                   default: OCI::ConfigFileLoader::DEFAULT_CONFIG_FILE,
+                   desc: 'OCI config file location'
+      class_option :bucket,
+                   type: :string,
+                   required: true
+      class_option :verbose,
+                   type: :boolean,
+                   aliases: :v,
+                   default: false
+
+      desc 'list', 'List queued jobs'
+      option :exec_group, type: :string, default: nil
+      def list
+        job_api = Jobbie::Jobs.new(options)
+        job_api.show_job_list(exec_group: options[:exec_group])
+      end
+
+      desc 'add', 'Add job onto the queue'
+      option :exec_group, type: :string, required: true
+      option :script, type: :string, required: true
+      def add(*params)
+        job_api = Jobbie::Jobs.new(options)
+        job_api.put_job(options[:exec_group], options[:script], params)
+      end
+
+      desc 'daemon', 'Daemon mode: Poll queue for jobs'
+      option :exec_group, type: :string, required: true
+      option :script_dir, type: :string, required: true
+      def daemon
+        job_api = Jobbie::Jobs.new(options)
+        job_api.daemon(options[:exec_group], options[:script_dir])
+      end
+    end
+  end
+end

data/lib/jobbie_oci/jobs.rb

@@ -0,0 +1,162 @@
+# frozen_string_literal: true
+
+require 'json'
+require 'socket'
+require 'terminal-table'
+
+require 'jobbie_oci/oci'
+
+module Jobbie
+  # Process jobs from the queue
+  class Jobs
+    attr_reader :options, :oci
+
+    def initialize(options)
+      @options = options
+      @oci = Jobbie::OciUtil.new(options)
+    end
+
+    def obj_api
+      oci.obj_api
+    end
+
+    def job_list(exec_group: nil)
+      unordered_job_list(exec_group: exec_group).sort_by(&:first)
+    end
+
+    def fqdn
+      Socket.gethostname
+    end
+
+    def job_details(job)
+      job_id = job.first.split('_')
+      params = job[1]['params']
+      params = params.join(' ') if params.is_a? Array
+      {
+        group: job_id.first,
+        id: job_id.last,
+        script: job[1]['script'],
+        params: params,
+        claim: job[1]['claim'],
+        time: Time.at(job[1]['epoch'].to_i)
+      }
+    end
+
+    def show_job_list(exec_group: nil)
+      rows = []
+      headings = if options[:verbose]
+                   ['Exec Group', 'ID', 'Command', 'Date Added', 'Claimed']
+                 else
+                   ['Exec Group', 'Command']
+                 end
+      job_list(exec_group: exec_group).each do |job|
+        details = job_details(job)
+        rows << if options[:verbose]
+                  [details[:group], details[:id],
+                   "#{details[:script]} #{details[:params]}",
+                   details[:time],
+                   details[:claim].nil? ? 'UNCLAIMED' : details[:claim]]
+                else
+                  [details[:group], "#{details[:script]} #{details[:params]}"]
+                end
+      end
+      puts Terminal::Table.new headings: headings, rows: rows
+    end
+
+    def put_job(exec_group, script, params)
+      obj_name = "#{exec_group}_#{Time.now.to_f}"
+      epoch = Time.now.to_i
+      obj = { script: script, params: params, epoch: epoch }
+      puts "Adding job #{obj_name}"
+      oci.add_obj(obj_name, obj.to_json)
+    end
+
+    def claimable_jobs?(exec_group)
+      job_list(exec_group: exec_group).each do |job|
+        obj = JSON.parse(oci.get_obj(job.first))
+        return true if obj['claim'].nil?
+      end
+      false
+    end
+
+    def unclaimed?(obj)
+      if obj['claim']
+        # Claim expires after 10 mins
+        (Time.now.to_i - obj['claimed_at'].to_i) > 300
+      else
+        true
+      end
+    end
+
+    def claim_job(job_id, obj)
+      my_ref = "#{fqdn}.#{Process.pid}"
+      obj[:claim] = my_ref
+      obj[:claimed_at] = Time.now.to_i
+      puts "claiming #{job_id}"
+      begin
+        oci.add_obj(job_id, obj.to_json)
+      rescue OCI::Errors::ServiceError
+        puts 'did not get claim'
+        return false
+      end
+      sleep 0.3
+      begin
+        obj = JSON.parse(oci.get_obj(job_id))
+      rescue TypeError
+        puts 'Could not read back job contents, skipping ..'
+        return false
+      end
+      return true if obj['claim'] == my_ref
+      puts 'did not get claim'
+      false
+    end
+
+    def invalid_script?(script)
+      (script =~ /\A[a-zA-Z0-9\._-]+\Z/).nil?
+    end
+
+    def daemon(exec_group, script_dir)
+      dir = File.expand_path(script_dir)
+      loop do
+        puts 'Looking for jobs ..'
+        job_obj_list(exec_group: exec_group).map(&:name).each do |job_id|
+          raw = oci.get_obj(job_id)
+          next if raw.nil?
+          obj = JSON.parse(raw)
+          if unclaimed?(obj) && claim_job(job_id, obj)
+            if invalid_script?(obj['script'])
+              puts 'Invalid script'
+            else
+              params = obj['params']
+              params = params.join(' ') if params.is_a? Array
+              if system(File.join(dir, obj['script']), params)
+                puts 'Success.  Removing job from queue'
+                oci.del_obj(job_id)
+              else
+                puts "Job #{job_id} returned non-zero exit code"
+              end
+            end
+          end
+        end
+        puts 'Waiting 60 seconds ..'
+        sleep 60
+      end
+    end
+
+    private
+
+    def job_obj_list(exec_group: nil)
+      if exec_group
+        oci.objects.select { |o| o.name.start_with? "#{exec_group}_" }
+      else
+        oci.objects
+      end
+    end
+
+    def unordered_job_list(exec_group: nil)
+      job_obj_list(exec_group: exec_group).map do |job|
+        [job.name, JSON.parse(oci.get_obj(job.name))]
+      end
+    end
+  end
+end

data/lib/jobbie_oci/oci.rb

@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+require 'oci'
+
+module Jobbie
+  # Utility class for interacting with OCI object store
+  class OciUtil
+    attr_reader :options
+
+    def initialize(options)
+      @options = options
+    end
+
+    def region
+      OCI::Regions::REGION_SHORT_NAMES_TO_LONG_NAMES[options[:region].to_sym]
+    end
+
+    def client_conf
+      OCI::ConfigFileLoader.load_config(profile_name: options[:profile])
+    end
+
+    def client_args
+      { config: client_conf, region: region }
+    end
+
+    def obj_api
+      OCI::ObjectStorage::ObjectStorageClient.new(client_args)
+    end
+
+    def namespace
+      @namespace ||= obj_api.get_namespace.data
+    end
+
+    def bucket
+      options[:bucket]
+    end
+
+    def objects
+      obj_api.list_objects(namespace, bucket).flat_map do |resp|
+        resp.data.objects.map do |obj|
+          obj
+        end
+      end
+    end
+
+    def add_obj(obj_name, content)
+      obj_api.put_object(namespace, bucket, obj_name, content)
+    end
+
+    def get_obj(obj_name)
+      obj_api.get_object(namespace, bucket, obj_name).data
+    rescue OCI::Errors::ServiceError
+      nil
+    end
+
+    def del_obj(obj_name)
+      obj_api.delete_object(namespace, bucket, obj_name)
+    end
+  end
+end

data/lib/jobbie_oci/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module JobbieOci
+  VERSION = '0.1.0'
+end

metadata

@@ -0,0 +1,84 @@
+--- !ruby/object:Gem::Specification
+name: jobbie_oci
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Stephen Pearson
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2018-07-12 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.16'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.16'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+description: 
+email:
+- stephen.pearson@oracle.com
+executables:
+- jobbie
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rubocop.yml"
+- Gemfile
+- Gemfile.lock
+- README.md
+- Rakefile
+- exe/jobbie
+- jobbie_oci.gemspec
+- lib/jobbie_oci/cli.rb
+- lib/jobbie_oci/jobs.rb
+- lib/jobbie_oci/oci.rb
+- lib/jobbie_oci/version.rb
+homepage: http://github.com/stephenpearson/jobbie_oci
+licenses: []
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.6.13
+signing_key: 
+specification_version: 4
+summary: Simple noddy job queuer that uses OCI Casper backend
+test_files: []