sqewer 5.0.9 → 5.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 84caf342300d2128e0e128efe4d235413bf011bc
-  data.tar.gz: f695a18f5e6b94a5c1854e4a5eb973ab7066def1
+  metadata.gz: a3b121d612798813c7a9de3e795f82603cc57b9c
+  data.tar.gz: 4fb43eefa14ff0fe3c2838b34d8c7d787ebb8481
 SHA512:
-  metadata.gz: 408c64f831f9441ed93609b07f037dff99de6af2d5ece6daebc95bf604c43a6518e2a9dc18a5c8ec37cdfe5d138f0d1a1b3a82b9b1a6554f4b34832992d275b0
-  data.tar.gz: 9b1c80ccadffe1b7f006a3310272d9fbdc715d3c45e22c1fcf8ace0d9bc8e2d300ac27096a745af0e80d031b41b454f60c46c6830263542c512cb7382312cd4d
+  metadata.gz: ce64c01eae4696d64c4215757c18afee1cefd3d99de906a5c8a12f7f28f265533a33a48aa4bbe434190f01fd76892334016bc493c0160146c90ab049f75e26a9
+  data.tar.gz: c1c8a1f5882fc38da3026cedd2fa3c1bd35ee1317061b49ff3c69dfd50ebe1ebf5880fad7d14d77ceb441d3b68a10c5c17d6a0c3fdf14ed3f100ccd8ceb92684

data/lib/sqewer/connection.rb

@@ -25,11 +25,18 @@ class Sqewer::Connection
   end
 
   # Returns the default adapter, connected to the queue set via the `SQS_QUEUE_URL`
-  # environment variable.
+  # environment variable. Switches to SQLite-backed local queue if the SQS_QUEUE_URL
+  # is prefixed with 'sqlite3://'
   def self.default
-    new(ENV.fetch('SQS_QUEUE_URL'))
+    url_str = ENV.fetch('SQS_QUEUE_URL')
+    uri = URI(url_str)
+    if uri.scheme == 'sqlite3'
+      Sqewer::LocalConnection.new(uri.to_s)
+    else
+      new(uri.to_s)
+    end
   rescue KeyError => e
-    raise "SQS_QUEUE_URL not set in the environment. This is the queue URL that the default that Sqewer uses"
+    raise "SQS_QUEUE_URL not set in the environment. This is the queue URL Sqewer uses by default."
   end
 
   # Initializes a new adapter, with access to the SQS queue at the given URL.

data/lib/sqewer/local_connection.rb

@@ -0,0 +1,154 @@
+require 'rack'
+class Sqewer::LocalConnection < Sqewer::Connection
+  ASSUME_DEADLETTER_AFTER_N_DELIVERIES = 10
+
+  def self.parse_queue_url(queue_url_starting_with_sqlite3_proto)
+    uri = URI.parse(queue_url_starting_with_sqlite3_proto)
+
+    unless uri.scheme == 'sqlite3'
+      raise "The scheme of the SQS queue URL should be with `sqlite3' but was %s" % uri.scheme
+    end
+
+    path_components = ['/', uri.hostname, uri.path].reject(&:nil?).reject(&:empty?).join('/').squeeze('/')
+    dbfile_path = File.expand_path(path_components)
+
+    queue_name = Rack::Utils.parse_nested_query(uri.query).fetch('queue', 'sqewer_local')
+
+    [dbfile_path, queue_name]
+  end
+
+  def initialize(queue_url_with_sqlite3_scheme)
+    require 'sqlite3'
+    @db_path, @queue_name = self.class.parse_queue_url(queue_url_with_sqlite3_scheme)
+    with_db do |db|
+      db.execute("CREATE TABLE IF NOT EXISTS sqewer_messages_v2 (
+        id INTEGER PRIMARY KEY AUTOINCREMENT ,
+        queue_name VARCHAR NOT NULL,
+        receipt_handle VARCHAR NOT NULL,
+        deliver_after_epoch INTEGER,
+        times_delivered_so_far INTEGER DEFAULT 0,
+        last_delivery_at_epoch INTEGER,
+        visible BOOLEAN DEFAULT 't',
+        message_body TEXT)"
+      )
+      db.execute("CREATE INDEX IF NOT EXISTS on_receipt_handle ON sqewer_messages_v2 (receipt_handle)")
+      db.execute("CREATE INDEX IF NOT EXISTS on_queue_name ON sqewer_messages_v2 (queue_name)")
+    end
+  rescue LoadError => e
+    raise e, "You need the sqlite3 gem in your Gemfile to use LocalConnection. Add it to your Gemfile (`gem 'sqlite3'')"
+  end
+
+  # @return [Array<Message>] an array of Message objects 
+  def receive_messages
+    messages = load_receipt_handles_and_bodies
+    messages.map {|message| Message.new(message[0], message[1]) }
+  end
+
+  # @yield [#send_message] the object you can send messages through (will be flushed at method return)
+  # @return [void]
+  def send_multiple_messages
+    buffer = SendBuffer.new
+    yield(buffer)
+    messages = buffer.messages
+    persist_messages(messages)
+  end
+
+  # Deletes multiple messages after they all have been succesfully decoded and processed.
+  #
+  # @yield [#delete_message] an object you can delete an individual message through
+  # @return [void]
+  def delete_multiple_messages
+    buffer = DeleteBuffer.new
+    yield(buffer)
+    delete_persisted_messages(buffer.messages)
+  end
+
+  # Only gets used in tests
+  def truncate!
+    with_db do |db|
+      db.execute("DELETE FROM sqewer_messages_v2 WHERE queue_name = ?", @queue_name)
+    end
+  end
+
+  private
+
+  def with_db(**k)
+    SQLite3::Database.open(@db_path, **k) do |db|
+      db.busy_timeout = 5
+      return yield db
+    end
+  rescue SQLite3::CantOpenException => e
+    message_with_path = [e.message, 'at', @db_path].join(' ')
+    raise SQLite3::CantOpenException.new(message_with_path)
+  end
+
+  def with_readonly_db(&blk)
+    with_db(readonly: true, &blk)
+  end
+
+  def delete_persisted_messages(messages)
+    ids_to_delete = messages.map{|m| m.fetch(:receipt_handle) }
+    with_db do |db|
+      db.execute("BEGIN")
+      ids_to_delete.each do |id|
+        db.execute("DELETE FROM sqewer_messages_v2 WHERE receipt_handle = ?", id)
+      end
+      db.execute("COMMIT")
+    end
+  end
+
+  def load_receipt_handles_and_bodies
+    t = Time.now.to_i
+
+    # First make messages that were previously marked invisible but not deleted visible again
+    with_db do |db|
+      db.execute("BEGIN")
+      # Make messages visible that have to be redelivered
+      db.execute("UPDATE sqewer_messages_v2
+        SET visible = 't' 
+        WHERE queue_name = ? AND visible = 'f' AND last_delivery_at_epoch < ?", @queue_name.to_s, t - 60)
+      # Remove hopeless messages
+      db.execute("DELETE FROM sqewer_messages_v2
+        WHERE queue_name = ? AND times_delivered_so_far >= ?", @queue_name.to_s, ASSUME_DEADLETTER_AFTER_N_DELIVERIES)
+      db.execute("COMMIT")
+    end
+
+    rows = with_readonly_db do |db|
+      db.execute("SELECT id, receipt_handle, message_body FROM sqewer_messages_v2
+        WHERE queue_name = ? AND visible = 't' AND deliver_after_epoch <= ? AND last_delivery_at_epoch <= ?",
+        @queue_name.to_s, t, t)
+    end
+    
+    with_db do |db|
+      db.execute("BEGIN")
+      rows.map do |(id, *_)|
+        db.execute("UPDATE sqewer_messages_v2
+          SET visible = 'f', times_delivered_so_far = times_delivered_so_far + 1, last_delivery_at_epoch = ?
+          WHERE id = ?", t, id)
+      end
+      db.execute("COMMIT")
+    end
+
+    rows.map do |(_, *receipt_handle_and_body)|
+      receipt_handle_and_body
+    end
+  end
+
+  def persist_messages(messages)
+    epoch = Time.now.to_i
+    bodies_and_deliver_afters = messages.map do |msg|
+      [msg.fetch(:message_body), epoch + msg.fetch(:delay_seconds, 0)]
+    end
+
+    with_db do |db|
+      db.execute("BEGIN")
+      bodies_and_deliver_afters.map do |body, deliver_after_epoch|
+        db.execute("INSERT INTO sqewer_messages_v2
+          (queue_name, receipt_handle, message_body, deliver_after_epoch, last_delivery_at_epoch)
+          VALUES(?, ?, ?, ?, ?)",
+          @queue_name.to_s, SecureRandom.uuid, body, deliver_after_epoch, epoch)
+      end
+      db.execute("COMMIT")
+    end
+  end
+end

data/lib/sqewer/version.rb

@@ -1,3 +1,3 @@
 module Sqewer
-  VERSION = '5.0.9'
+  VERSION = '5.1.0'
 end

data/lib/sqewer/worker.rb

@@ -110,7 +110,7 @@ class Sqewer::Worker
               @logger.debug { "[worker] Received and buffered %d messages" % messages.length } if messages.any?
             else
               @logger.debug { "[worker] No messages received" }
-              Thread.pass
+              sleep SLEEP_SECONDS_ON_EMPTY_QUEUE
             end
           else
             @logger.debug { "[worker] Cache is full (%d items), postponing receive" % @execution_queue.length }

data/sqewer.gemspec

@@ -30,6 +30,7 @@ Gem::Specification.new do |spec|
   spec.require_paths = ["lib"]
 
   spec.add_runtime_dependency 'aws-sdk', '~> 2'
+  spec.add_runtime_dependency 'rack'
   spec.add_runtime_dependency 'very_tiny_state_machine'
   spec.add_runtime_dependency 'ks'
   spec.add_runtime_dependency 'retriable'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: sqewer
 version: !ruby/object:Gem::Version
-  version: 5.0.9
+  version: 5.1.0
 platform: ruby
 authors:
 - Julik Tarkhanov
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-06-23 00:00:00.000000000 Z
+date: 2017-08-31 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aws-sdk
@@ -24,6 +24,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '2'
+- !ruby/object:Gem::Dependency
+  name: rack
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: very_tiny_state_machine
   requirement: !ruby/object:Gem::Requirement
@@ -224,6 +238,7 @@ files:
 - lib/sqewer/extensions/active_job_adapter.rb
 - lib/sqewer/extensions/appsignal_wrapper.rb
 - lib/sqewer/extensions/railtie.rb
+- lib/sqewer/local_connection.rb
 - lib/sqewer/middleware_stack.rb
 - lib/sqewer/null_logger.rb
 - lib/sqewer/resubmit.rb