litejob 0.1.0 → 0.2.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f0cd6ee9a1f3896b0aa44f935258c92d3326661ffb09c42c143095fc60c8ea08
-  data.tar.gz: 302ea0cbf170c7c07c4384983bb5801af794f2d185db77064ff1b855a3693c95
+  metadata.gz: 9235b916ceb5620f30e307e23adb1c3baf256625a3d3cedde1c2851faa9ed490
+  data.tar.gz: 553500fbb22d4ef729a6059277cb56738c87b671e4009ac9c3b7174293f7bfb8
 SHA512:
-  metadata.gz: 8f3d9837dd1ef417645e6dd46e587a23565a0e0ac7e900410b44b009b89d4c8ecbe1b058aad6b5782df550992712ab25fde799b9c249bf153dc0c7beaa7016aa
-  data.tar.gz: 2f9647c30eb7d8af132df8baf0266f409ed9523f1ec848a161919be3de7cec76de0b14003661b140c1b619f229c87df020d44fde6dae93e1db8fae4d38f81ada
+  metadata.gz: 86125548df029a4a0035314ff34f3a429394079d56b88075e7b05ec72a4975a9b7ee32ed130bef939cfe4edd6a6cbc02b46458e011df847fd942efbbdbd36562
+  data.tar.gz: 50f38790e97d1b19462a5926bd2a92fbcbc9485ccadf2c89a9c1f1effc4ccd566650bea57400793d3d11aec0390c647f81e04d2ead1963aa0889f9522918654a

data/.standard.yml

@@ -1,3 +1,6 @@
 # For available configuration options, see:
 #   https://github.com/testdouble/standard
-ruby_version: 2.6
+ruby_version: 3.2
+ignore:
+  - 'test/**/*':
+    - Style/GlobalVars

data/CHANGELOG.md

@@ -1,5 +1,13 @@
 ## [Unreleased]
 
+## [0.2.1] - 2023-08-13
+
+- Fix some bugs with Litejob::Server
+
+## [0.2.0] - 2023-08-13
+
+- Initial release of usable code
+
 ## [0.1.0] - 2023-08-09
 
 - Initial release

data/Gemfile.lock

@@ -1,15 +1,24 @@
 PATH
   remote: .
   specs:
-    litejob (0.1.0)
+    litejob (0.2.1)
+      litequeue (>= 0.2.0)
+      litescheduler (>= 0.2.1)
 
 GEM
   remote: https://rubygems.org/
   specs:
     ast (2.4.2)
+    docile (1.4.0)
     json (2.6.3)
     language_server-protocol (3.17.0.3)
     lint_roller (1.1.0)
+    litedb (0.2.1)
+      litescheduler (>= 0.2.0)
+      sqlite3 (>= 1.5.0)
+    litequeue (0.2.0)
+      litedb (>= 0.2.1)
+    litescheduler (0.2.1)
     minitest (5.19.0)
     parallel (1.23.0)
     parser (3.2.2.3)
@@ -36,6 +45,14 @@ GEM
       rubocop (>= 1.7.0, < 2.0)
       rubocop-ast (>= 0.4.0)
     ruby-progressbar (1.13.0)
+    simplecov (0.22.0)
+      docile (~> 1.1)
+      simplecov-html (~> 0.11)
+      simplecov_json_formatter (~> 0.1)
+    simplecov-html (0.12.3)
+    simplecov_json_formatter (0.1.4)
+    sqlite3 (1.6.3-arm64-darwin)
+    sqlite3 (1.6.3-x86_64-linux)
     standard (1.30.1)
       language_server-protocol (~> 3.17.0.2)
       lint_roller (~> 1.0)
@@ -52,11 +69,13 @@ GEM
 
 PLATFORMS
   arm64-darwin-21
+  x86_64-linux
 
 DEPENDENCIES
   litejob!
   minitest (~> 5.0)
   rake (~> 13.0)
+  simplecov
   standard (~> 1.3)
 
 BUNDLED WITH

data/README.md

@@ -1,5 +1,10 @@
 # Litejob
 
+[![Gem Version](https://badge.fury.io/rb/litejob.svg)](https://rubygems.org/gems/litejob)
+[![Gem Downloads](https://img.shields.io/gem/dt/litejob)](https://rubygems.org/gems/litejob)
+![Tests](https://github.com/litestack-ruby/litejob/actions/workflows/main.yml/badge.svg)
+![Coverage](https://img.shields.io/badge/code_coverage-100%25-brightgreen)
+
 TODO: Delete this and the text below, and describe your gem
 
 Welcome to your new gem! In this directory, you'll find the files you need to be able to package up your Ruby library into a gem. Put your Ruby code in the file `lib/litejob`. To experiment with that code, run `bin/console` for an interactive prompt.

data/Rakefile

@@ -9,6 +9,34 @@ Rake::TestTask.new(:test) do |t|
   t.test_files = FileList["test/**/test_*.rb"]
 end
 
+desc "Update the README code coverage badge"
+task :update_readme_coverage_badge do
+  require "json"
+
+  next unless File.exist?("coverage/.last_run.json")
+
+  last_run_coverage = JSON.load_file("coverage/.last_run.json")
+  line_coverage = last_run_coverage.dig("result", "line")
+  branch_coverage = last_run_coverage.dig("result", "branch")
+  average_coverage = [(branch_coverage * 1), (line_coverage * 1.5)].sum.fdiv(2.5).round
+  badge_color = if average_coverage >= 75
+    :brightgreen
+  else
+    :red
+  end
+
+  coverage_badge_re = /!\[Coverage\]\(https:\/\/img.shields.io\/badge\/code_coverage-(.*?)\)/
+  last_run_coverage_badge = "![Coverage](https://img.shields.io/badge/code_coverage-#{average_coverage}%25-#{badge_color})"
+
+  new_readme = File.read("README.md").gsub(coverage_badge_re, last_run_coverage_badge)
+
+  File.write("README.md", new_readme)
+
+  puts "Updated README code coverage badge to show #{average_coverage}% coverage."
+end
+
+task cov: %i[test update_readme_coverage_badge]
+
 require "standard/rake"
 
 task default: %i[test standard]

data/lib/litejob/client.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+require "json"
+require "litequeue"
+
+module Litejob
+  # Litejob::Client is responsible for pushing job payloads to the SQLite queue.
+  class Client
+    def initialize
+      @queue = Litequeue.instance
+    end
+
+    def push(jobclass, params, options = {})
+      delay = options[:delay] || 0
+      attempts = options[:attempts] || 5
+      queue = options[:queue]
+      payload = JSON.dump({class: jobclass, params: params, attempts: attempts, queue: queue})
+      atomic_push(payload, delay, queue)
+    end
+
+    def delete(id)
+      payload = @queue.delete(id)
+      JSON.parse(payload)
+    end
+
+    private
+
+    def atomic_push(payload, delay, queue)
+      retryable = true
+      begin
+        @queue.push(payload, queue: queue, delay: delay)
+      rescue => exception
+        # Retry once retryable exceptions
+        # https://github.com/sparklemotion/sqlite3-ruby/blob/master/lib/sqlite3/errors.rb
+        if retryable && exception.is_a?(SQLite3::BusyException)
+          retryable = false
+          retry
+        else
+          raise exception
+        end
+      end
+    end
+  end
+end

data/lib/litejob/processor.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+require "json"
+require "litequeue"
+
+module Litejob
+  # Litejob::Processor is responsible for processing job payloads
+  class Processor
+    def initialize(payload)
+      @payload = payload
+      @queue = Litequeue.instance
+    end
+
+    def repush(id, job, delay = 0, queue = nil)
+      @queue.repush(id, JSON.dump(job), queue: queue, delay: delay)
+    end
+
+    def process!
+      id, serialized_job = @payload
+      job_hash = JSON.parse(serialized_job)
+      klass = Object.const_get(job_hash["class"])
+      instance = klass.new
+
+      begin
+        instance.perform(*job_hash["params"])
+      rescue
+        if job_hash["retries_left"] == 0
+          repush(id, job_hash, 0, "_dead")
+        else
+          job_hash["retries_left"] ||= job_hash["attempts"]
+          job_hash["retries_left"] -= 1
+          retry_delay = (job_hash["attempts"] - job_hash["retries_left"]) * 0.1
+          repush(id, job_hash, retry_delay, job_hash["queue"])
+        end
+      end
+    rescue => exception # standard:disable Lint/UselessRescue
+      # this is an error in the extraction of job info, retrying here will not be useful
+      raise exception
+    end
+  end
+end

data/lib/litejob/server.rb

@@ -0,0 +1,70 @@
+# frozen_string_literal: true
+
+require "litequeue"
+require "litescheduler"
+require_relative "processor"
+
+module Litejob
+  # Litejob::Server is responsible for popping job payloads from the SQLite queue.
+  # :nocov:
+  class Server
+    # TODO: make queues use [["default", 1]]
+    # TODO: make queue priorities optional
+    def initialize(queues)
+      @queue = Litequeue.instance
+      @scheduler = Litescheduler.instance
+      @queues = queues
+      # group and order queues according to their priority
+      @prioritized_queues = queues.each_with_object({}) do |(name, priority, spawns), memo|
+        memo[priority] ||= []
+        memo[priority] << [name, spawns == "spawn"]
+      end.sort_by do |priority, _|
+        -priority
+      end
+      @running = true
+      @sleep_intervals = [0.001, 0.005, 0.025, 0.125, 0.625, 1.0, 2.0]
+      run!
+    end
+
+    def pop(queue)
+      result = @queue.pop(queue: queue)
+
+      return result unless result.is_a?(Array)
+      return false if result.empty?
+
+      result
+    end
+
+    def run!
+      @scheduler.spawn do
+        worker_sleep_index = 0
+        while @running
+          processed = 0
+          @prioritized_queues.each do |priority, queues|
+            queues.each do |queue, spawns|
+              batched = 0
+              while (batched < priority) && (payload = pop(queue))
+                batched += 1
+                processed += 1
+
+                processor = Processor.new(payload)
+                processor.process!
+
+                # give other contexts a chance to run here
+                @scheduler.switch
+              end
+            end
+
+            if processed == 0
+              sleep @sleep_intervals[worker_sleep_index]
+              worker_sleep_index += 1 if worker_sleep_index < @sleep_intervals.length - 1
+            else
+              worker_sleep_index = 0 # reset the index
+            end
+          end
+        end
+      end
+    end
+  end
+  # :nocov:
+end

data/lib/litejob/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Litejob
-  VERSION = "0.1.0"
+  VERSION = "0.2.1"
 end

data/lib/litejob.rb

@@ -1,8 +1,53 @@
 # frozen_string_literal: true
 
 require_relative "litejob/version"
+require_relative "litejob/client"
+require_relative "litejob/server"
 
+# Litejob is responsible for providing an interface to job classes
 module Litejob
-  class Error < StandardError; end
-  # Your code goes here...
+  def self.included(klass)
+    klass.extend(ClassMethods)
+  end
+
+  module ClassMethods
+    def perform_async(*params)
+      @litejob_options ||= {}
+      client.push(name, params, @litejob_options.merge(delay: 0, queue: queue_name))
+    end
+
+    def perform_at(time, *params)
+      @litejob_options ||= {}
+      delay = time.to_i - Time.now.to_i
+      client.push(name, params, @litejob_options.merge(delay: delay, queue: queue_name))
+    end
+
+    def perform_in(delay, *params)
+      @litejob_options ||= {}
+      client.push(name, params, @litejob_options.merge(delay: delay, queue: queue_name))
+    end
+    alias_method :perform_after, :perform_in
+
+    def delete(id)
+      client.delete(id)
+    end
+
+    def queue_as(queue_name)
+      @queue_name = queue_name.to_s
+    end
+
+    def litejob_options(options)
+      @litejob_options = options
+    end
+
+    private
+
+    def queue_name
+      @queue_name || "default"
+    end
+
+    def client
+      @client ||= Client.new
+    end
+  end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: litejob
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.1
 platform: ruby
 authors:
 - Mohamed Hassan
@@ -9,8 +9,50 @@ authors:
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2023-08-09 00:00:00.000000000 Z
-dependencies: []
+date: 2023-08-13 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: litescheduler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.2.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.2.1
+- !ruby/object:Gem::Dependency
+  name: litequeue
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.2.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.2.0
+- !ruby/object:Gem::Dependency
+  name: simplecov
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description:
 email:
 - oldmoe@gmail.com
@@ -28,6 +70,9 @@ files:
 - README.md
 - Rakefile
 - lib/litejob.rb
+- lib/litejob/client.rb
+- lib/litejob/processor.rb
+- lib/litejob/server.rb
 - lib/litejob/version.rb
 - sig/litejob.rbs
 homepage: https://github.com/litestack-ruby/litejob