filesystem_queue 0.1.1 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 51d0fe57ac440673ae98c9fa2a782ddfe6552450bf984dd44c4edc0d0f972e2f
-  data.tar.gz: e61ded49f3eb2919f0da55ce20627a5d3a732c22c71bf747b64def5f42f0e2a4
+  metadata.gz: 7954b5b7e75219071fbf43cb6c92efa52477a933a2e2238e189ee67dc8cc87ae
+  data.tar.gz: 5b253de47b288f393474f5ed225429ec48132856e3f1f2b45c040b2dc506286e
 SHA512:
-  metadata.gz: 7ecb0a4780632dcedcbe33593460c9b827c4bc1402b967fe9cd010ca3cbb2ab97f4e7bcdcbe33f707caf7bf8a90d964c65f8e773cf806f2a2e3436e203398c85
-  data.tar.gz: 73b113cb15f2cd5eb285565ceb82c62b8d3b0f6180c49c75c374228becd4ba0f1877dc49794cc01b7ca0e3b4442b43614d229e21f093e43739e4c7ab84da2eed
+  metadata.gz: 4cfbee5995cbcb04f1f5757ee1415744ee2b236d95864c94930c43c50234cd6080d8554b3e06726dba95203292cef1760e65b40afbba1445c4479c1d09500fbc
+  data.tar.gz: 43dcdde986b435fdf1b3c06ee15f598a0b95406d62ecea8754e32b1c8a7d10fb243757df18c83cd0486bd909a408b0eb06a18ae35773138836d76cea697e9e0c

data/.rubocop.yml

@@ -3,7 +3,6 @@ AllCops:
 
 Style/StringLiterals:
   Enabled: true
-  EnforcedStyle: double_quotes
 
 Style/StringLiteralsInInterpolation:
   Enabled: true

data/CHANGELOG.md

@@ -1,8 +1,17 @@
 ## [Unreleased]
 
+## [0.3.0] - 2024-09-20
+- Added `reenqueue_failed_jobs` method to move failed jobs back to the queue
+- Added `reenqueue_completed_jobs` method to move completed jobs back to the queue
+
+## [0.2.0] - 2024-09-16
+- Replaced old index for an in-memory index for job tracking
+- Added `cleanup` method to delete files and directories created by the queue
+
 ## [0.1.1] - 2024-09-13
 - Added unit tests
 - Fixed Github Actions
 - Minor refactors
+
 ## [0.1.0] - 2024-09-12
 - Initial release

data/Gemfile

@@ -1,12 +1,12 @@
 # frozen_string_literal: true
 
-source "https://rubygems.org"
+source 'https://rubygems.org'
 
 # Specify your gem's dependencies in filesystem_queue.gemspec
 gemspec
 
-gem "rake", "~> 13.0"
+gem 'rake', '~> 13.0'
 
-gem "minitest", "~> 5.0"
+gem 'minitest', '~> 5.0'
 
-gem "rubocop", "~> 1.21"
+gem 'rubocop', '~> 1.21'

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    filesystem_queue (0.1.1)
+    filesystem_queue (0.3.0)
 
 GEM
   remote: https://rubygems.org/

data/README.md

@@ -14,9 +14,11 @@
 - **Persistence**: Jobs are stored on the filesystem, ensuring they are not lost between application restarts.
 - **Minimal Overhead**: No need for external dependencies or services.
 - **Ease of Use**: Simple API for enqueuing and dequeuing jobs.
-- **Improved Performance**: Enhances filesystem performance by using an index file that keeps track of files by the time they are written in the queue.
-- **Tailored for Local Use**: This system is designed for local use with a single consumer (at the moment).
-- **Job Management**: Completed and failed jobs are tracked in separate folders for easy management.
+- **In-Memory Index**: Uses an in-memory index for tracking jobs, improving performance by reducing file I/O operations.
+- **Cleanup Method**: Provides a method to delete all files and directories created by the queue.
+- **Queue Size Methods**: Includes methods to get the size of the queue and the number of failed jobs.
+- **Tailored for Local Use**: This system is optimized for local use with a single consumer. If you are looking for a multiple consumer approach you should be looking for larger distributed solutions such as Resque or other distributed Queue systems.
+- **Job Management**: Once a job is dequeued, it's your responsabilty to flag it as `Completed` or `Failed`. Make sure you have fault tolerant logic in your worker to ensure this step. Completed and failed jobs are tracked in separate folders for easy management.
 
 ## Installation
 
@@ -72,6 +74,18 @@ Returns the number of jobs currently in the queue.
 - **Returns**:
   - `Integer`: The number of jobs in the queue.
 
+### `reenqueue_failed_jobs`
+
+Moves all failed jobs back to the queue for reprocessing.
+
+### `reenqueue_completed_jobs`
+
+Moves all completed jobs back to the queue for reprocessing.
+
+### `cleanup`
+
+Removes all files and directories created by the queue.
+
 ## Usage
 
 ```ruby
@@ -117,6 +131,15 @@ puts 'Queue is empty'
 # Check on failed jobs
 puts "Queue size: #{queue.size}"
 puts "Failed jobs: #{queue.failed_size}"
+
+# Reenqueue all failed jobs back to the queue
+queue.reenqueue_failed_jobs
+
+# Reenqueue all completed jobs back to the queue
+queue.reenqueue_completed_jobs
+
+# Cleanup all files and directories created by the queue
+queue.cleanup
 ```
 
 ## Error Handling

data/Rakefile

@@ -1,15 +1,15 @@
 # frozen_string_literal: true
 
-require "bundler/gem_tasks"
-require "rake/testtask"
+require 'bundler/gem_tasks'
+require 'rake/testtask'
 
 Rake::TestTask.new(:test) do |t|
-  t.libs << "test"
-  t.libs << "lib"
-  t.test_files = FileList["test/**/test_*.rb"]
+  t.libs << 'test'
+  t.libs << 'lib'
+  t.test_files = FileList['test/**/test_*.rb']
 end
 
-require "rubocop/rake_task"
+require 'rubocop/rake_task'
 
 RuboCop::RakeTask.new
 

data/filesystem_queue.gemspec

@@ -1,25 +1,25 @@
 # frozen_string_literal: true
 
-require_relative "lib/filesystem_queue/version"
+require_relative 'lib/filesystem_queue/version'
 
 Gem::Specification.new do |spec|
-  spec.name = "filesystem_queue"
+  spec.name = 'filesystem_queue'
   spec.version = FilesystemQueue::VERSION
-  spec.authors = ["Carlos Westman"]
-  spec.email = ["carloswestman@gmail.com"]
+  spec.authors = ['Carlos Westman']
+  spec.email = ['carloswestman@gmail.com']
 
-  spec.summary = "A persistent queue system based on the local filesystem"
-  spec.description = "FilesystemQueue is a Ruby gem that provides a persistent queue system "\
-  "using the local filesystem.It allows you to enqueue and dequeue jobs, and keeps track of completed and failed jobs."
-  spec.homepage = "https://github.com/carloswestman/filesystem_queue"
-  spec.license = "MIT"
-  spec.required_ruby_version = ">= 2.6.0"
+  spec.summary = 'A persistent queue system based on the local filesystem'
+  spec.description = 'FilesystemQueue is a Ruby gem that provides a persistent queue system '\
+  'using the local filesystem.It allows you to enqueue and dequeue jobs, and keeps track of completed and failed jobs.'
+  spec.homepage = 'https://github.com/carloswestman/filesystem_queue'
+  spec.license = 'MIT'
+  spec.required_ruby_version = '>= 2.6.0'
 
-  spec.metadata["allowed_push_host"] = "https://rubygems.org"
+  spec.metadata['allowed_push_host'] = 'https://rubygems.org'
 
-  spec.metadata["homepage_uri"] = spec.homepage
-  spec.metadata["source_code_uri"] = spec.homepage
-  spec.metadata["changelog_uri"] = "https://github.com/carloswestman/filesystem_queue/blob/main/CHANGELOG.md"
+  spec.metadata['homepage_uri'] = spec.homepage
+  spec.metadata['source_code_uri'] = spec.homepage
+  spec.metadata['changelog_uri'] = 'https://github.com/carloswestman/filesystem_queue/blob/main/CHANGELOG.md'
 
   # Specify which files should be added to the gem when it is released.
   # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
@@ -28,7 +28,7 @@ Gem::Specification.new do |spec|
       (File.expand_path(f) == __FILE__) || f.start_with?(*%w[bin/ test/ spec/ features/ .git .circleci appveyor])
     end
   end
-  spec.bindir = "exe"
+  spec.bindir = 'exe'
   spec.executables = spec.files.grep(%r{\Aexe/}) { |f| File.basename(f) }
-  spec.require_paths = ["lib"]
+  spec.require_paths = ['lib']
 end

data/lib/filesystem_queue/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module FilesystemQueue
-  VERSION = "0.1.1"
+  VERSION = '0.3.0'
 end

data/lib/filesystem_queue.rb

@@ -1,8 +1,8 @@
 # frozen_string_literal: true
 
-require "json"
-require "fileutils"
-require_relative "filesystem_queue/version"
+require 'json'
+require 'fileutils'
+require_relative 'filesystem_queue/version'
 
 module FilesystemQueue
   class Error < StandardError; end
@@ -12,27 +12,29 @@ module FilesystemQueue
   class Queue
     def initialize(queue_dir)
       @queue_dir = queue_dir
-      @jobs_dir = File.join(@queue_dir, "jobs")
-      @completed_dir = File.join(@queue_dir, "completed")
-      @failed_dir = File.join(@queue_dir, "failed")
-      @index_file = File.join(@queue_dir, "index.txt")
+      @jobs_dir = File.join(@queue_dir, 'jobs')
+      @completed_dir = File.join(@queue_dir, 'completed')
+      @failed_dir = File.join(@queue_dir, 'failed')
 
       [@jobs_dir, @completed_dir, @failed_dir].each do |dir|
         FileUtils.mkdir_p(dir) unless Dir.exist?(dir)
       end
-      FileUtils.touch(@index_file) unless File.exist?(@index_file)
+
+      @index = rebuild_index
     end
 
     def enqueue(job)
       timestamp = Time.now.to_f.to_s
       job_file = File.join(@jobs_dir, "job_#{timestamp}.json")
       File.write(job_file, job.to_json)
-      File.open(@index_file, "a") { |f| f.puts(job_file) }
+      @index << job_file
     end
 
     def dequeue
-      job_file = extract_job_file_from_index
-      return nil unless job_file && File.exist?(job_file)
+      return nil if @index.empty?
+
+      job_file = @index.shift
+      return nil unless File.exist?(job_file)
 
       job_data = JSON.parse(File.read(job_file), symbolize_names: true)
       [job_file, job_data]
@@ -47,33 +49,56 @@ module FilesystemQueue
     end
 
     def size
-      File.readlines(@index_file).size
+      @index.size
     end
 
     def failed_size
-      Dir[File.join(@failed_dir, "*")].count { |file| File.file?(file) }
+      Dir[File.join(@failed_dir, '*')].count { |file| File.file?(file) }
+    end
+
+    def retry_failed_jobs
+      failed_jobs = Dir.glob(File.join(@failed_dir, '*.json'))
+      failed_jobs.each do |job_file|
+        new_job_file = File.join(@jobs_dir, File.basename(job_file))
+        FileUtils.mv(job_file, new_job_file)
+        @index << new_job_file
+      end
+    end
+
+    def reenqueue_failed_jobs
+      reenqueue_jobs(@failed_dir)
+    end
+
+    def reenqueue_completed_jobs
+      reenqueue_jobs(@completed_dir)
+    end
+
+    # CAUTION: Cleanup the queue directory, removing all files and directories
+    def cleanup
+      [@jobs_dir, @completed_dir, @failed_dir].each do |dir|
+        FileUtils.rm_rf(dir)
+      end
+      FileUtils.rm_rf(@queue_dir)
     end
 
     private
 
+    def rebuild_index
+      Dir.glob(File.join(@jobs_dir, '*.json')).sort
+    end
+
     def move_job(job_file, target_dir)
       FileUtils.mv(job_file, target_dir)
-    rescue StandardError => e
-      puts "Failed to move job file: #{e.message}"
+      @index.delete(job_file)
     end
 
-    def extract_job_file_from_index
-      job_file = nil
-      File.open(@index_file, "r+") do |f|
-        lines = f.each_line.to_a
-        return nil if lines.empty?
-
-        job_file = lines.shift.strip
-        f.rewind
-        f.write(lines.join)
-        f.truncate(f.pos)
+    def reenqueue_jobs(source_dir)
+      job_files = Dir.glob(File.join(source_dir, '*.json'))
+      job_files.each do |job_file|
+        new_job_file = File.join(@jobs_dir, File.basename(job_file))
+        FileUtils.mv(job_file, new_job_file)
+        @index << new_job_file
       end
-      job_file
     end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: filesystem_queue
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.3.0
 platform: ruby
 authors:
 - Carlos Westman
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2024-09-13 00:00:00.000000000 Z
+date: 2024-09-17 00:00:00.000000000 Z
 dependencies: []
 description: FilesystemQueue is a Ruby gem that provides a persistent queue system
   using the local filesystem.It allows you to enqueue and dequeue jobs, and keeps