jobly 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 8ad67b7e96a1ee27e3034c812d05026f922b72585d70d0c8085f9173f8e70d04
+  data.tar.gz: 9614bb05c9da5bf17a9e2d25e3aafdd859bf811828f71c6a511557dc926985ae
+SHA512:
+  metadata.gz: c891c3bbff7c1c37180fdc186b22f63a50730f978125546e48a2231ebeab630bb27312016c3d47fa39afcc48dd79bb0e59bb96c1abafebbc16804f77eec8da42
+  data.tar.gz: f021ecf23d579f53fc713b4657ca2613b429da6e7ff2e503e21bf9edb3d4b9f18bf48362349deafd0e1fab36324d51f029ea2d4e9f8dc5d38b43912ce63dce42

data/README.md

@@ -0,0 +1,76 @@
+Jobly
+==================================================
+
+[![Gem Version](https://badge.fury.io/rb/jobly.svg)](https://badge.fury.io/rb/jobly)
+[![Build Status](https://travis-ci.com/DannyBen/jobly.svg?branch=master)](https://travis-ci.com/DannyBen/jobly)
+
+---
+
+Job server toolbelt with API, CLI, Web UI and a Sidekiq heart.
+
+---
+
+Installation
+--------------------------------------------------
+
+    $ gem install jobly
+
+
+What's in the Box
+--------------------------------------------------
+
+Jobly is a command line utility that wraps several goodies around the 
+sidekiq backgronud jobs system. It includes the following components:
+
+- **Command line interface** - for starting the server, starting the worker, 
+  and for running jobs.
+- **Web API** - for executing jobs.
+- **Web Dashboard** - including job progress and status.
+
+Getting Started
+--------------------------------------------------
+
+TODO
+
+
+Usage
+--------------------------------------------------
+
+### Server
+
+To start the server run `jobly server` and open <http://localhost:3000/> 
+in your browser.
+
+This will start a webserver with two primary entrypoints:
+
+- `/` (root) - a dashboard for your background job processes.
+- `/do/:job?param=value` - an API for executing jobs
+
+
+### Worker
+
+To start processing jobs, run `jobly worker`
+
+
+### Running jobs from the command line
+
+There are three ways to run a job from the command line:
+
+Run the job locally, without going through any of the background job 
+processing chain:
+
+```ruby    
+jobly run JobName param:value
+```
+
+Run the job locally, but wait for a worker to process it.
+
+```ruby
+jobly run --later JobName param:value
+```
+
+Send a job through the API (either localhost or remote).
+
+```ruby
+jobly send JobName param:value
+```

data/bin/jobly

@@ -0,0 +1,16 @@
+#!/usr/bin/env ruby
+require 'colsole'
+require 'jobly'
+require 'jobly/boot'
+
+include Colsole
+
+router = Jobly::CLI.router
+
+begin
+  exit router.run ARGV
+rescue => e
+  puts e.backtrace.reverse if ENV['DEBUG']
+  say! "!txtred!#{e.class}: #{e.message}"
+  exit 1
+end

data/lib/jobly.rb

@@ -0,0 +1,11 @@
+require 'requires'
+require 'byebug' if ENV['BYEBUG']
+
+requires 'jobly/refinements'
+require 'jobly/exceptions'
+require 'jobly/sidekiq'
+require 'jobly/module_functions'
+require 'jobly/cli'
+require 'jobly/job'
+require 'jobly/jobs'
+require 'jobly/server'

data/lib/jobly/boot.rb

@@ -0,0 +1,5 @@
+require 'jobly'
+
+Jobly.load_custom_config
+Jobly::Jobs.load_all
+Jobly::SidekiqBoot.configure

data/lib/jobly/cli.rb

@@ -0,0 +1,23 @@
+require 'mister_bin'
+requires 'commands/base', 'commands'
+
+module Jobly
+
+  # Command line interface router. This is called by bin/jobly.
+  class CLI
+    def self.router
+      router = MisterBin::Runner.new version: VERSION,
+        header: "Jobly",
+        footer: "Run !txtpur!jobly COMMAND --help!txtrst! for more information"
+
+      router.route 'server',    to: Commands::ServerCmd
+      router.route 'worker',    to: Commands::WorkerCmd
+      router.route 'send',      to: Commands::SendCmd
+      router.route 'run',       to: Commands::RunCmd
+      router.route 'config',    to: Commands::ConfigCmd
+
+      router
+    end
+  end
+
+end

data/lib/jobly/commands/base.rb

@@ -0,0 +1,11 @@
+require 'mister_bin'
+require 'colsole'
+require 'lp'
+
+module Jobly
+  module Commands
+    class Base < MisterBin::Command
+      include Colsole
+    end
+  end
+end

data/lib/jobly/commands/config.rb

@@ -0,0 +1,34 @@
+require 'pathname'
+
+module Jobly
+  module Commands
+    class ConfigCmd < Base
+      summary "Show configuration options"
+      usage "jobly config"
+      usage "jobly config (-h|--help)"
+
+      def run
+        line "custom config file", short_config_path, Jobly.custom_config?
+        Jobly.options.each do |key, value|
+          if key.to_s.end_with? '_path'
+            line key, value, !Dir.exist?(value)
+          else
+            line key, value
+          end
+        end
+      end
+
+    private
+
+      def short_config_path
+        Jobly.config_file.sub "#{Dir.pwd}", '.'
+      end
+
+      def line(key, value, attention=false)
+        color = attention ? '!txtred!' : '!txtgrn!'
+        say "#{key.to_s.rjust 20}  #{color}#{value.to_s.strip}"
+      end
+
+    end
+  end
+end

data/lib/jobly/commands/run.rb

@@ -0,0 +1,38 @@
+module Jobly
+  module Commands
+    class RunCmd < Base
+      using ArgumentConverters
+      using ConvertToTyped
+
+      summary "Run a job locally"
+      usage "jobly run [--later] JOB [PARAMS...]"
+      usage "jobly run (-h|--help)"
+      option "-l, --later", "Schedule the job to be executed later by a worker instead of running it immediately"
+      param "JOB", "Job name"
+      param "PARAMS", "Parameters to pass to the job as key:value"
+      example "jobly run Greet name:Bob"
+      example "jobly run --later Greet name:Bob"
+      example "jobly run Deploy env:stage branch:master"
+
+      def run
+        job = args['JOB']
+        params = args['PARAMS'].to_params.convert_to_typed
+        job_class = Jobs.get_class! job
+
+        if args['--later']
+          say "!txtgrn!Scheduling"
+          if params.empty?
+            job_class.perform_async
+          else
+            job_class.perform_async params
+          end
+        
+        else
+          say "!txtgrn!Running"
+          job_class.new.perform params
+        end
+      end
+
+    end
+  end
+end

data/lib/jobly/commands/send.rb

@@ -0,0 +1,38 @@
+require 'http'
+
+module Jobly
+  module Commands
+    class SendCmd < Base
+      using ArgumentConverters
+
+      summary "Send a job to the API"
+      usage "jobly send JOB [PARAMS...]"
+      usage "jobly send (-h|--help)"
+      param "JOB", "Job name"
+      param "PARAMS", "Parameters to pass to the job as key:value"
+      example "jobly send Greet name:Bob"
+      example "jobly send Deploy env:stage branch:master"
+
+      def run
+        job = args['JOB']
+        params = args['PARAMS'].to_params
+        url = "#{Jobly.api_base}/#{job}"
+
+        response = if params.empty?
+          HTTP.get url
+        else
+          HTTP.get url, params: params
+        end
+
+        if response.code != 200
+          say "!txtred!#{response.code}"
+        else
+          say "!txtgrn!200 OK"
+        end
+        
+        lp response.parse
+      end
+
+    end
+  end
+end

data/lib/jobly/commands/server.rb

@@ -0,0 +1,22 @@
+module Jobly
+  module Commands
+    class ServerCmd < Base
+      summary "Start the server"
+      usage "jobly server [--port NUMBER]"
+      usage "jobly server (-h|--help)"
+      option "-p, --port NUMBER", "Set the port number [default: 3000]"
+
+      def run
+        port = args['--port']
+        say "Starting server"
+        exec "rackup --env #{Jobly.environment} --port #{port} --host 0.0.0.0 #{rackup_file}"
+      end
+
+    private
+
+      def rackup_file
+        @rackup_file ||= File.expand_path '../config.ru', __dir__
+      end
+    end
+  end
+end

data/lib/jobly/commands/worker.rb

@@ -0,0 +1,22 @@
+module Jobly
+  module Commands
+    class WorkerCmd < Base
+      summary "Start a job worker"
+      usage "jobly worker [--concurrency COUNT]"
+      usage "jobly worker (-h|--help)"
+      option "-c, --concurrency COUNT", "Number of parallel jobs [default: 4]"
+
+      def run
+        concurrency = args['--concurrency']
+        say "Starting sidekiq"
+        exec "sidekiq --environment #{Jobly.environment} --concurrency #{concurrency} --require #{boot_file}"
+      end
+
+    private
+
+      def boot_file
+        @boot_file ||= File.expand_path '../boot.rb', __dir__
+      end
+    end
+  end
+end

data/lib/jobly/config.ru

@@ -0,0 +1,10 @@
+require 'puma'
+require 'jobly'
+require 'jobly/boot'
+
+mounts = {
+  '/'   => Sidekiq::Web, 
+  '/do' => Jobly::Server
+}
+
+run Rack::URLMap.new mounts

data/lib/jobly/exceptions.rb

@@ -0,0 +1,4 @@
+module Jobly
+  class Error < StandardError; end
+  class JobNotFound < Error; end
+end

data/lib/jobly/job.rb

@@ -0,0 +1,16 @@
+module Jobly
+  class Job
+    include Sidekiq::Worker
+    include Sidekiq::Status::Worker
+    sidekiq_options retry: 5
+
+    def perform(params={})
+      params = params.to_h.transform_keys(&:to_sym)
+      params.empty? ? execute : execute(params)
+    end
+
+    def execute(params={})
+      raise NotImplementedError
+    end
+  end
+end

data/lib/jobly/jobs.rb

@@ -0,0 +1,27 @@
+module Jobly
+  module Jobs
+    def self.get_class(job)
+      Object.const_get namespaced_job(job) rescue nil
+    end
+
+    def self.get_class!(job)
+      Object.const_get namespaced_job(job)
+    rescue NameError
+      raise JobNotFound, job
+    end
+
+    def self.load_all
+      Dir["#{Jobly.full_app_path}/**/*.rb"].each { |file| require file }
+      Dir["#{Jobly.full_jobs_path}/**/*.rb"].each { |file| require file }
+    end
+
+    def self.namespaced_job(job)
+      if Jobly.jobs_namespace
+        "#{Jobly.jobs_namespace}::#{job}"
+      else
+        job
+      end
+    end
+  end
+end
+

data/lib/jobly/module_functions.rb

@@ -0,0 +1,65 @@
+module Jobly
+  def self.configure
+    yield self
+  end
+
+  def self.load_custom_config
+    require config_file if File.exist? config_file
+  end
+
+  def self.default_options
+    {
+      environment: ENV['JOBLY_ENVIRONMENT'] || 'development',
+      api_base: ENV['JOBLY_API_URL'] || 'http://localhost:3000/do',
+      app_path: ENV['JOBLY_APP_PATH'] || 'app',
+      jobs_path: ENV['JOBLY_JOBS_PATH'] || "jobs",
+      config_path: ENV['JOBLY_CONFIG_PATH'] || "config",
+      redis_url: ENV['JOBLY_REDIS_URL'] || "redis://localhost:6379/0",
+      status_expiration: 30,
+      jobs_namespace: nil,
+      logger: nil,
+    }
+  end
+
+  def self.method_missing(method, args=nil, &_block)
+    key = method.to_s
+    assign = key[-1] == "="
+    key = key.chomp('=') if assign
+    key = key.to_sym
+
+    if options.has_key? key
+      assign ? options[key] = args : options[key]
+    else
+      super
+    end
+  end
+
+  def self.respond_to_missing?(method, include_private=false)
+    key = method.to_s.chomp('=').to_sym
+    options.has_key?(key) ? true : super
+  end
+
+  def self.options
+    @options ||= default_options.dup
+  end
+
+  def self.full_app_path
+    File.expand_path app_path
+  end
+
+  def self.full_jobs_path
+    File.expand_path jobs_path
+  end
+
+  def self.full_config_path
+    File.expand_path config_path
+  end
+
+  def self.config_file
+    File.expand_path 'jobly.rb', self.config_path
+  end
+
+  def self.custom_config?
+    File.exist? config_file
+  end
+end

data/lib/jobly/refinements/argument_converters.rb

@@ -0,0 +1,9 @@
+module Jobly
+  module ArgumentConverters
+    refine Array do
+      def to_params
+        map { |pair| pair.split ':' }.to_h.transform_keys(&:to_sym)
+      end
+    end
+  end
+end

data/lib/jobly/refinements/convert_to_typed.rb

@@ -0,0 +1,25 @@
+module Jobly
+  module ConvertToTyped
+    refine Hash do
+      def convert_to_typed
+        transform_values do |element|
+          element.is_a?(String) ? element.convert_to_typed : element
+        end
+      end
+    end
+
+    refine Array do
+      def convert_to_typed
+        map do |element|
+          element.is_a?(String) ? element.convert_to_typed : element
+        end
+      end
+    end
+
+    refine String do
+      def convert_to_typed
+        Integer self rescue self
+      end
+    end
+  end
+end

data/lib/jobly/server.rb

@@ -0,0 +1,77 @@
+require 'jobly'
+require 'sinatra/base'
+require 'sinatra/reloader'
+require 'sinatra/custom_logger'
+
+module Jobly
+
+  class Server < Sinatra::Application
+    helpers Sinatra::CustomLogger
+    using ConvertToTyped
+
+    configure :development, :production do
+      set :server, :puma
+      set :logger, Jobly.logger if Jobly.logger
+    end
+
+    configure :development do
+      register Sinatra::Reloader
+    end
+
+    before do
+      content_type 'application/json'
+    end
+
+    get '/' do
+      {
+        version: VERSION,
+        message: %Q["I'm gonna live till I die" - Frank Sinatra]
+      }.to_json
+    end
+
+    get '/:job' do
+      job = params.delete :job
+      add_job job, params
+    end
+
+    post '/:job' do
+      job = params.delete :job
+      add_job job, params
+    end
+
+  private
+
+    def add_job(job, args={})
+      job_class = Jobs.get_class job
+
+      if !job_class
+        response = {
+          status: 'error',
+          message: 'No such job',
+          job: job,
+          params: args
+        }
+
+        status 404
+        logger.debug "[jobly server] No such job (#{job})"
+        return response.to_json
+      end
+
+      args = args.convert_to_typed
+      if args.empty?
+        job_class.perform_async
+      else
+        job_class.perform_async args
+      end
+      
+      response = {
+        status: 'received',
+        job: job,
+        params: args,
+      }
+
+      logger.debug "[jobly server] Job received (#{job})"
+      response.to_json
+    end
+  end
+end

data/lib/jobly/sidekiq.rb

@@ -0,0 +1,26 @@
+require 'sidekiq'
+require 'sidekiq/web'
+require 'sidekiq-status'
+require 'sidekiq-status/web'
+
+module Jobly
+  module SidekiqBoot
+    def self.configure
+      Sidekiq.configure_client do |config|
+        config.redis = { url: Jobly.redis_url }
+        Sidekiq::Status.configure_client_middleware config, expiration: 60 * 60
+      end
+
+      Sidekiq.configure_server do |config|
+        # :nocov:
+        config.redis = { url: Jobly.redis_url }
+
+        Sidekiq::Status.configure_server_middleware config, expiration: Jobly.status_expiration * 60
+        Sidekiq::Status.configure_client_middleware config, expiration: Jobly.status_expiration * 60
+        # :nocov:
+      end
+
+      Sidekiq::Logging.logger = Jobly.logger if Jobly.logger
+    end
+  end
+end

data/lib/jobly/version.rb

@@ -0,0 +1,3 @@
+module Jobly
+  VERSION = "0.1.0"
+end

metadata

@@ -0,0 +1,218 @@
+--- !ruby/object:Gem::Specification
+name: jobly
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Danny Ben Shitrit
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2019-03-08 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: mister_bin
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.6'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.6'
+- !ruby/object:Gem::Dependency
+  name: colsole
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.5'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.5'
+- !ruby/object:Gem::Dependency
+  name: puma
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.12'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.12'
+- !ruby/object:Gem::Dependency
+  name: rack
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+- !ruby/object:Gem::Dependency
+  name: requires
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.1'
+- !ruby/object:Gem::Dependency
+  name: sidekiq
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.2'
+- !ruby/object:Gem::Dependency
+  name: sidekiq-status
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.1'
+- !ruby/object:Gem::Dependency
+  name: sinatra
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+- !ruby/object:Gem::Dependency
+  name: sinatra-contrib
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+- !ruby/object:Gem::Dependency
+  name: http
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.0'
+- !ruby/object:Gem::Dependency
+  name: lp
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.1'
+description: Execute background jobs and build tasks on this sidekiq-based job server
+email: db@dannyben.com
+executables:
+- jobly
+extensions: []
+extra_rdoc_files: []
+files:
+- README.md
+- bin/jobly
+- lib/jobly.rb
+- lib/jobly/boot.rb
+- lib/jobly/cli.rb
+- lib/jobly/commands/base.rb
+- lib/jobly/commands/config.rb
+- lib/jobly/commands/run.rb
+- lib/jobly/commands/send.rb
+- lib/jobly/commands/server.rb
+- lib/jobly/commands/worker.rb
+- lib/jobly/config.ru
+- lib/jobly/exceptions.rb
+- lib/jobly/job.rb
+- lib/jobly/jobs.rb
+- lib/jobly/module_functions.rb
+- lib/jobly/refinements/argument_converters.rb
+- lib/jobly/refinements/convert_to_typed.rb
+- lib/jobly/server.rb
+- lib/jobly/sidekiq.rb
+- lib/jobly/version.rb
+homepage: https://github.com/dannyben/jobly
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 2.4.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.0.3
+signing_key: 
+specification_version: 4
+summary: Compact job server with API, CLI, Web UI and a Sidekiq heart
+test_files: []