bushpig 0.1.0

data/lib/bushpig.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+require 'bushpig/version'
+
+require 'bushpig/redis_pool'
+require 'bushpig/job'
+require 'bushpig/client'
+require 'bushpig/server'
+
+module Bushpig
+  NAME = 'Bushpig'
+  LICENSE = 'See LICENSE and the MIT License for licensing details.'
+
+  def self.client
+    raise 'Bushpig client not configured' unless @client
+
+    @client
+  end
+
+  def self.client=(client)
+    @client = client
+  end
+
+  def self.server
+    raise 'Bushpig server not configured' unless @server
+
+    @server
+  end
+
+  def self.server=(server)
+    @server = server
+  end
+
+  def self.configure_server
+    yield self if server?
+  end
+
+  def self.configure_client
+    yield self unless server?
+  end
+
+  def self.server?
+    defined?(Bushpig::CLI)
+  end
+
+  def self.set_key(queue)
+    "set:#{queue}"
+  end
+
+  def self.job_key(job_id)
+    "job:#{job_id}"
+  end
+end

data/lib/bushpig/cli.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+require 'thor'
+
+module Bushpig
+  class CLI < Thor
+    package_name 'bushpig'
+
+    def self.exit_on_failure?
+      true
+    end
+
+    desc 'serve SET', 'Run the bushpig server'
+    def serve(queue)
+      require 'rails'
+      require File.expand_path('./config/environment.rb')
+
+      server = Bushpig.server
+      server.serve(queue)
+    end
+  end
+end

data/lib/bushpig/client.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module Bushpig
+  class Client
+    attr_accessor :default_ttl
+
+    def initialize(pool)
+      @pool = pool
+      @default_ttl = nil
+    end
+
+    def redis_pool
+      @pool
+    end
+
+    def default_score
+      Time.now.to_f
+    end
+
+    def submit(queue, job, score: default_score, ttl: default_ttl)
+      redis_pool.with do |conn|
+        conn.set(Bushpig.job_key(job.job_id), job.job_payload, ex: ttl)
+        conn.zadd(Bushpig.set_key(queue), score, job.job_id)
+      end
+    end
+  end
+end

data/lib/bushpig/job.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+require 'digest'
+require 'securerandom'
+require 'json'
+
+module Bushpig
+  module Job
+    def self.job(*args)
+      Struct.new(*args) do
+        def job_id
+          @job_id ||= SecureRandom.hex(32)
+        end
+
+        def job_id=(job_id)
+          @job_id = job_id
+        end
+
+        def job_payload
+          JSON.generate({ class: self.class.name, args: each.to_a })
+        end
+      end
+    end
+
+    def self.keyed(unique_key, *args)
+      Struct.new(*args) do
+        @@unique_key = unique_key
+
+        def job_id
+          @@unique_key.inject(Digest::SHA256.new) do |digest, key|
+            digest.update(self[key].to_s)
+          end.hexdigest
+        end
+
+        def job_id=(job_id); end
+
+        def job_payload
+          JSON.generate({ class: self.class.name, args: each.to_a })
+        end
+      end
+    end
+
+    def self.hydrate(job_id, job_payload)
+      h = JSON.parse(job_payload, symbolize_names: true)
+      klass = const_get(h[:class])
+      job = klass.new(*h[:args])
+      job.job_id = job_id
+      job
+    end
+  end
+end

data/lib/bushpig/redis_pool.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+require 'connection_pool'
+require 'redis'
+
+module Bushpig
+  class RedisPool
+    def initialize(size: 1, timeout: 5, redis_options: {})
+      @size = size
+      @timeout = timeout
+      @redis_options = redis_options
+
+      @pool = ConnectionPool.new(size: @size, timeout: @timeout) { Redis.new(@redis_options) }
+    end
+
+    def with(options = {}, &block)
+      raise ArgumentError, 'requires a block' unless block_given?
+
+      retries = 1
+      begin
+        @pool.with(options, &block)
+      rescue Redis::TimeoutError
+        if retries > 0
+          retries -= 1
+          retry
+        end
+        raise
+      end
+    end
+  end
+end

data/lib/bushpig/server.rb

@@ -0,0 +1,91 @@
+# frozen_string_literal: true
+
+module Bushpig
+  class Server
+    def initialize(pool, timeout: 2, &handler)
+      @pool = pool
+      @timeout = timeout.to_f
+      @handler = handler
+      @done = false
+    end
+
+    def redis_pool
+      @pool
+    end
+
+    def trap_signals
+      Signal.trap('INT') do
+        reset_signals
+        puts 'INT received, shutdown flagged'
+        @done = true
+      end
+    end
+
+    def reset_signals
+      Signal.trap('INT', 'DEFAULT')
+    end
+
+    def serve(queue)
+      puts "Serving queue #{queue}"
+      trap_signals
+
+      until @done
+        job = fetch(queue)
+        next if job.nil?
+
+        handle(job)
+        complete(job)
+      end
+
+      reset_signals
+      puts "Stop serving queue #{queue}"
+    end
+
+    def handle(job)
+      puts "Job starting: jid-#{job.job_id} #{job}"
+      started = monotonic_time
+      @handler.call(job)
+      finished = monotonic_time
+      elapsed = finished - started
+      puts "Job completed: jid-#{job.job_id} #{elapsed} seconds"
+    rescue StandardError => e
+      # Job handler raised exception
+      finished = monotonic_time
+      elapsed = finished - started
+      puts "Job raised exception: jid-#{job.job_id} #{elapsed} seconds: #{e}"
+      # TODO: log exception to honeybadger
+    end
+
+    def monotonic_time
+      Process.clock_gettime(Process::CLOCK_MONOTONIC)
+    end
+
+    def fetch(queue)
+      redis_pool.with do |conn|
+        begin
+          res = conn.bzpopmin(Bushpig.set_key(queue), @timeout)
+        rescue Redis::TimeoutError
+          # TODO: warn user (once) that redis timeout set lower than pop timeout
+          conn.close
+          res = nil
+        end
+        return nil if res.nil?
+
+        (_set, jid, _score) = res
+        # conn.sadd('running', jid)
+
+        payload = conn.get(Bushpig.job_key(jid))
+        return nil if payload.nil? # most likely job expired
+
+        Bushpig::Job.hydrate(jid, payload)
+      end
+    end
+
+    def complete(job)
+      redis_pool.with do |conn|
+        # conn.srem('running', job.job_id)
+        conn.del(Bushpig.job_key(job.job_id))
+      end
+    end
+  end
+end

data/lib/bushpig/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module Bushpig
+  VERSION = '0.1.0'
+end

data/shell.nix

@@ -0,0 +1,4 @@
+let
+  pkgs = import <nixpkgs> {};
+in
+  pkgs.mkShell { buildInputs = with pkgs; [ ruby bundler bundix ]; }

metadata

@@ -0,0 +1,197 @@
+--- !ruby/object:Gem::Specification
+name: bushpig
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Shaun Sharples
+autorequire:
+bindir: bin
+cert_chain: []
+date: 1980-01-01 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.1'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.1'
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: rubocop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: solargraph
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: connection_pool
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.2'
+- !ruby/object:Gem::Dependency
+  name: json
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.8'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.8'
+- !ruby/object:Gem::Dependency
+  name: redis
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.3'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.3'
+- !ruby/object:Gem::Dependency
+  name: thor
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+description: Simple job system.
+email:
+- shaun.sharples@gmail.com
+executables:
+- bushpig
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- Gemfile
+- Gemfile.lock
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/bushpig
+- bin/console
+- bin/setup
+- bushpig.gemspec
+- default.nix
+- gemset.nix
+- lib/bushpig.rb
+- lib/bushpig/cli.rb
+- lib/bushpig/client.rb
+- lib/bushpig/job.rb
+- lib/bushpig/redis_pool.rb
+- lib/bushpig/server.rb
+- lib/bushpig/version.rb
+- shell.nix
+homepage: https://github.com/mpowered/bushpig
+licenses:
+- MIT
+metadata:
+  allowed_push_host: https://rubygems.org
+  homepage_uri: https://github.com/mpowered/bushpig
+  source_code_uri: https://github.com/mpowered/bushpig
+  changelog_uri: https://github.com/mpowered/bushpig/blob/master/CHANGELOG.md
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '2.3'
+  - - "<"
+    - !ruby/object:Gem::Version
+      version: '3'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.1.3
+signing_key:
+specification_version: 4
+summary: Simple job system.
+test_files: []