filesystem_queue 0.1.0 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b9e3ab5ddf19ea8ef07cf949fa4e273c5498c3c71922318be16619f788a648dc
-  data.tar.gz: 51f0f96c5299d04c3777fff7f20fbfa2ec476923464f19fed4e8a11b53f4abbc
+  metadata.gz: 2e960414c06a8d0a9fe7d7fc0b8b1ed90cf604d7648ab74c64f924648a137e94
+  data.tar.gz: 06f0a8e59cbb4c64a2de4af00360b317bb13e8a136f3d1647ff802b8b7617939
 SHA512:
-  metadata.gz: b1999fc8c23de8854d08aadc53993dc1fb128ad8dfd1a682cea44fb7315c5939ac75341d05c8ea2b9eac41820c51e422193924c422ece9dd9c01f109f0caf20e
-  data.tar.gz: 191ad7aa123dcd9cdde4056f820190eeff45204e36fda24ca3c97d0029751b1b63b06f44832e485915903c0bb3a81afe12040330b44ca8897f35f1929155d15b
+  metadata.gz: 6a9d6f4d5c55313ed7af2bc85245c019f97a66a691a4511c67ed78a53e74fa91f6574e513ebaf1e288aaeeb74cca74bbb53de289c294ff4ad781c496a9e4edf1
+  data.tar.gz: a71491c4ac7e6f46b8834ef9ca7f1f7534216d6036c4e98877c2f0ae34a050202699fb471b9886a51f6fb61fa3359aa297980f4debc74e93db90108adbf8f756

data/.rubocop.yml

@@ -3,7 +3,6 @@ AllCops:
 
 Style/StringLiterals:
   Enabled: true
-  EnforcedStyle: double_quotes
 
 Style/StringLiteralsInInterpolation:
   Enabled: true

data/CHANGELOG.md

@@ -1,5 +1,11 @@
 ## [Unreleased]
 
+## [0.2.0] - 2024-09-16
+- Replaced old index for an in-memory index for job tracking
+- Added `cleanup` method to delete files and directories created by the queue
+## [0.1.1] - 2024-09-13
+- Added unit tests
+- Fixed Github Actions
+- Minor refactors
 ## [0.1.0] - 2024-09-12
-
 - Initial release

data/Gemfile

@@ -1,12 +1,12 @@
 # frozen_string_literal: true
 
-source "https://rubygems.org"
+source 'https://rubygems.org'
 
 # Specify your gem's dependencies in filesystem_queue.gemspec
 gemspec
 
-gem "rake", "~> 13.0"
+gem 'rake', '~> 13.0'
 
-gem "minitest", "~> 5.0"
+gem 'minitest', '~> 5.0'
 
-gem "rubocop", "~> 1.21"
+gem 'rubocop', '~> 1.21'

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    filesystem_queue (0.1.0)
+    filesystem_queue (0.2.0)
 
 GEM
   remote: https://rubygems.org/
@@ -35,6 +35,7 @@ GEM
 
 PLATFORMS
   x86_64-darwin-19
+  x86_64-linux
 
 DEPENDENCIES
   filesystem_queue!

data/README.md

@@ -1,18 +1,34 @@
 # FilesystemQueue
 
+[![Gem Version](https://badge.fury.io/rb/filesystem_queue.svg)](https://badge.fury.io/rb/filesystem_queue)
+
 `FileSystemQueue` is a persistent queue system based on the local filesystem. It allows you to enqueue and dequeue jobs, and keeps track of completed and failed jobs.
 
-## Installation
 
-TODO: Replace `UPDATE_WITH_YOUR_GEM_NAME_PRIOR_TO_RELEASE_TO_RUBYGEMS_ORG` with your gem name right after releasing it to RubyGems.org. Please do not do it earlier due to security reasons. Alternatively, replace this section with instructions to install your gem from git if you don't plan to release to RubyGems.org.
+## What is this for?
+
+`FileSystemQueue` is designed for applications that need a simple, embedded queue system with minimal footprint. It consists of just a single class and utilizes your local filesystem for storage. This makes it an excellent choice for lightweight applications or development environments where setting up a full-fledged queue service is overkill. If you are looking to scale up your queue into a service architecture, Ruby offers other solutions like Resque (Redis-based), or various suites from cloud providers such as AWS-SQS.
+
+### Features
+
+- **Persistence**: Jobs are stored on the filesystem, ensuring they are not lost between application restarts.
+- **Minimal Overhead**: No need for external dependencies or services.
+- **Ease of Use**: Simple API for enqueuing and dequeuing jobs.
+- **In-Memory Index**: Uses an in-memory index for tracking jobs, improving performance by reducing file I/O operations.
+- **Cleanup Method**: Provides a method to delete all files and directories created by the queue.
+- **Queue Size Methods**: Includes methods to get the size of the queue and the number of failed jobs.
+- **Tailored for Local Use**: This system is optimized for local use with a single consumer. If you are looking for a multiple consumer approach you should be looking for larger distributed solutions such as Resque or other distributed Queue systems.
+- **Job Management**: Once a job is dequeued, it's your responsabilty to flag it as `Completed` or `Failed`. Make sure you have fault tolerant logic in your worker to ensure this step. Completed and failed jobs are tracked in separate folders for easy management.
+
+## Installation
 
 Install the gem and add to the application's Gemfile by executing:
 
-    $ bundle add UPDATE_WITH_YOUR_GEM_NAME_PRIOR_TO_RELEASE_TO_RUBYGEMS_ORG
+    $ bundle add filesystem_queue
 
 If bundler is not being used to manage dependencies, install the gem by executing:
 
-    $ gem install UPDATE_WITH_YOUR_GEM_NAME_PRIOR_TO_RELEASE_TO_RUBYGEMS_ORG
+    $ gem install filesystem_queue
 
 ## Methods
 
@@ -103,6 +119,9 @@ puts 'Queue is empty'
 # Check on failed jobs
 puts "Queue size: #{queue.size}"
 puts "Failed jobs: #{queue.failed_size}"
+
+# Cleanup all files and directories created by the queue
+queue.cleanup
 ```
 
 ## Error Handling

data/Rakefile

@@ -1,15 +1,15 @@
 # frozen_string_literal: true
 
-require "bundler/gem_tasks"
-require "rake/testtask"
+require 'bundler/gem_tasks'
+require 'rake/testtask'
 
 Rake::TestTask.new(:test) do |t|
-  t.libs << "test"
-  t.libs << "lib"
-  t.test_files = FileList["test/**/test_*.rb"]
+  t.libs << 'test'
+  t.libs << 'lib'
+  t.test_files = FileList['test/**/test_*.rb']
 end
 
-require "rubocop/rake_task"
+require 'rubocop/rake_task'
 
 RuboCop::RakeTask.new
 

data/filesystem_queue.gemspec

@@ -1,24 +1,25 @@
 # frozen_string_literal: true
 
-require_relative "lib/filesystem_queue/version"
+require_relative 'lib/filesystem_queue/version'
 
 Gem::Specification.new do |spec|
-  spec.name = "filesystem_queue"
+  spec.name = 'filesystem_queue'
   spec.version = FilesystemQueue::VERSION
-  spec.authors = ["Carlos Westman"]
-  spec.email = ["carloswestman@gmail.com"]
+  spec.authors = ['Carlos Westman']
+  spec.email = ['carloswestman@gmail.com']
 
-  spec.summary = "A persistent queue system based on the local filesystem"
-  spec.description = "FilesystemQueue is a Ruby gem that provides a persistent queue system using the local filesystem. It allows you to enqueue and dequeue jobs, and keeps track of completed and failed jobs."
-  spec.homepage = "https://github.com/carloswestman/filesystem_queue"
-  spec.license = "MIT"
-  spec.required_ruby_version = ">= 2.6.0"
+  spec.summary = 'A persistent queue system based on the local filesystem'
+  spec.description = 'FilesystemQueue is a Ruby gem that provides a persistent queue system '\
+  'using the local filesystem.It allows you to enqueue and dequeue jobs, and keeps track of completed and failed jobs.'
+  spec.homepage = 'https://github.com/carloswestman/filesystem_queue'
+  spec.license = 'MIT'
+  spec.required_ruby_version = '>= 2.6.0'
 
-  spec.metadata["allowed_push_host"] = "https://rubygems.org"
+  spec.metadata['allowed_push_host'] = 'https://rubygems.org'
 
-  spec.metadata["homepage_uri"] = spec.homepage
-  spec.metadata["source_code_uri"] = spec.homepage
-  spec.metadata["changelog_uri"] = "https://github.com/carloswestman/filesystem_queue/blob/main/CHANGELOG.md"
+  spec.metadata['homepage_uri'] = spec.homepage
+  spec.metadata['source_code_uri'] = spec.homepage
+  spec.metadata['changelog_uri'] = 'https://github.com/carloswestman/filesystem_queue/blob/main/CHANGELOG.md'
 
   # Specify which files should be added to the gem when it is released.
   # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
@@ -27,7 +28,7 @@ Gem::Specification.new do |spec|
       (File.expand_path(f) == __FILE__) || f.start_with?(*%w[bin/ test/ spec/ features/ .git .circleci appveyor])
     end
   end
-  spec.bindir = "exe"
+  spec.bindir = 'exe'
   spec.executables = spec.files.grep(%r{\Aexe/}) { |f| File.basename(f) }
-  spec.require_paths = ["lib"]
+  spec.require_paths = ['lib']
 end

data/lib/filesystem_queue/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module FilesystemQueue
-  VERSION = "0.1.0"
+  VERSION = '0.2.0'
 end

data/lib/filesystem_queue.rb

@@ -1,8 +1,8 @@
 # frozen_string_literal: true
 
-require "json"
-require "fileutils"
-require_relative "filesystem_queue/version"
+require 'json'
+require 'fileutils'
+require_relative 'filesystem_queue/version'
 
 module FilesystemQueue
   class Error < StandardError; end
@@ -12,37 +12,29 @@ module FilesystemQueue
   class Queue
     def initialize(queue_dir)
       @queue_dir = queue_dir
-      @jobs_dir = File.join(@queue_dir, "jobs")
-      @completed_dir = File.join(@queue_dir, "completed")
-      @failed_dir = File.join(@queue_dir, "failed")
-      @index_file = File.join(@queue_dir, "index.txt")
+      @jobs_dir = File.join(@queue_dir, 'jobs')
+      @completed_dir = File.join(@queue_dir, 'completed')
+      @failed_dir = File.join(@queue_dir, 'failed')
 
       [@jobs_dir, @completed_dir, @failed_dir].each do |dir|
         FileUtils.mkdir_p(dir) unless Dir.exist?(dir)
       end
-      FileUtils.touch(@index_file) unless File.exist?(@index_file)
+
+      @index = rebuild_index
     end
 
     def enqueue(job)
       timestamp = Time.now.to_f.to_s
       job_file = File.join(@jobs_dir, "job_#{timestamp}.json")
       File.write(job_file, job.to_json)
-      File.open(@index_file, "a") { |f| f.puts(job_file) }
+      @index << job_file
     end
 
     def dequeue
-      job_file = nil
-      File.open(@index_file, "r+") do |f|
-        lines = f.each_line.to_a
-        return nil if lines.empty?
-
-        job_file = lines.shift.strip
-        f.rewind
-        f.write(lines.join)
-        f.truncate(f.pos)
-      end
+      return nil if @index.empty?
 
-      return nil unless job_file && File.exist?(job_file)
+      job_file = @index.shift
+      return nil unless File.exist?(job_file)
 
       job_data = JSON.parse(File.read(job_file), symbolize_names: true)
       [job_file, job_data]
@@ -57,19 +49,30 @@ module FilesystemQueue
     end
 
     def size
-      File.readlines(@index_file).size
+      @index.size
     end
 
     def failed_size
-      Dir[File.join(@failed_dir, "*")].count { |file| File.file?(file) }
+      Dir[File.join(@failed_dir, '*')].count { |file| File.file?(file) }
+    end
+
+    # CAUTION: Cleanup the queue directory, removing all files and directories
+    def cleanup
+      [@jobs_dir, @completed_dir, @failed_dir].each do |dir|
+        FileUtils.rm_rf(dir)
+      end
+      FileUtils.rm_rf(@queue_dir)
     end
 
     private
 
+    def rebuild_index
+      Dir.glob(File.join(@jobs_dir, '*.json')).sort
+    end
+
     def move_job(job_file, target_dir)
       FileUtils.mv(job_file, target_dir)
-    rescue StandardError => e
-      puts "Failed to move job file: #{e.message}"
+      @index.delete(job_file)
     end
   end
 end

metadata

@@ -1,17 +1,17 @@
 --- !ruby/object:Gem::Specification
 name: filesystem_queue
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.0
 platform: ruby
 authors:
 - Carlos Westman
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2024-09-13 00:00:00.000000000 Z
+date: 2024-09-16 00:00:00.000000000 Z
 dependencies: []
 description: FilesystemQueue is a Ruby gem that provides a persistent queue system
-  using the local filesystem. It allows you to enqueue and dequeue jobs, and keeps
+  using the local filesystem.It allows you to enqueue and dequeue jobs, and keeps
   track of completed and failed jobs.
 email:
 - carloswestman@gmail.com