sqewer 6.1.0 → 6.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 1301945308afd9596e1b563dd0ed568f8dd2bd82
-  data.tar.gz: 770c8114bf9ef3bc84b96dcfb6a08fb251d77643
+  metadata.gz: '09e6812e437ea73062add603664517055b6eff54'
+  data.tar.gz: e7f001ecc56fc313c0af256c4bf5b89635318fad
 SHA512:
-  metadata.gz: fdcd703b863c6e5ce2af7adeaf4a6205304d7c42ed0a227dc7d466c270294013efba20d7f2030c644663c6f6a6c98549b364802eeb407664e646056fa9d7adbe
-  data.tar.gz: 408b96e763f411673b914011cb5f70dfde2b791da07a4487f3b018d5dad8ae1b1f01a83cc0a5cf182ec7852672dfadb97a46c92a34e0007e47aa22a5b6673b2f
+  metadata.gz: 14f6ac8a671ec1b6b13f9d24f4e3c13e844c37150ae50ca44c0b20d83590e2ff96173a83e225a63e9e613c6b4c479a2ec3e88e82ddbffa1c62a0817591958b89
+  data.tar.gz: f007a131aa6412f252e46316fdca1eb9a7895ad0eddbc9ae3a2ab0cbeaf8bc6a03ce7d7f5cfc665fe61a1b530858e52894ba7e72b1682d7a4decc7ca6db40872

data/CHANGELOG.md

@@ -1,3 +1,6 @@
+### 6.2.0
+- Store SentTimestamp in SQLite and restore it on execution
+
 ### 6.1.0
 - Pass SQS message attributes through the middleware chain
 - Recover the SentTimestamp attribute and set it as queue start in Appsignal

data/lib/sqewer/extensions/appsignal_wrapper.rb

@@ -4,7 +4,7 @@ module Sqewer
     # to Appsignal and to monitor performance. Will only activate
     # if the Appsignal gem is loaded within the current process and active.
     class AppsignalWrapper
-      
+
       def self.new
         if defined?(Appsignal)
           super
@@ -15,7 +15,7 @@ module Sqewer
 
       # extend Appsignal::Hooks::Helpers
       # and use format_args(args) on the jobargs?
-      
+
       # Acts as a replacement for Appsignal::GenericRequest
       class FakeRequest < Struct.new(:params)
         def initialize; super({}); end

data/lib/sqewer/local_connection.rb

@@ -21,7 +21,7 @@ class Sqewer::LocalConnection < Sqewer::Connection
     require 'sqlite3'
     @db_path, @queue_name = self.class.parse_queue_url(queue_url_with_sqlite3_scheme)
     with_db do |db|
-      db.execute("CREATE TABLE IF NOT EXISTS sqewer_messages_v2 (
+      db.execute("CREATE TABLE IF NOT EXISTS sqewer_messages_v3 (
         id INTEGER PRIMARY KEY AUTOINCREMENT ,
         queue_name VARCHAR NOT NULL,
         receipt_handle VARCHAR NOT NULL,
@@ -29,19 +29,21 @@ class Sqewer::LocalConnection < Sqewer::Connection
         times_delivered_so_far INTEGER DEFAULT 0,
         last_delivery_at_epoch INTEGER,
         visible BOOLEAN DEFAULT 't',
+        sent_timestamp_millis INTEGER,
         message_body TEXT)"
       )
-      db.execute("CREATE INDEX IF NOT EXISTS on_receipt_handle ON sqewer_messages_v2 (receipt_handle)")
-      db.execute("CREATE INDEX IF NOT EXISTS on_queue_name ON sqewer_messages_v2 (queue_name)")
+      db.execute("CREATE INDEX IF NOT EXISTS on_receipt_handle ON sqewer_messages_v3 (receipt_handle)")
+      db.execute("CREATE INDEX IF NOT EXISTS on_queue_name ON sqewer_messages_v3 (queue_name)")
     end
   rescue LoadError => e
     raise e, "You need the sqlite3 gem in your Gemfile to use LocalConnection. Add it to your Gemfile (`gem 'sqlite3'')"
   end
 
-  # @return [Array<Message>] an array of Message objects 
+  # @return [Array<Message>] an array of Message objects
   def receive_messages
-    messages = load_receipt_handles_and_bodies
-    messages.map {|message| Message.new(message[0], message[1]) }
+    load_receipt_handles_bodies_and_timestamps.map do |(receipt_handle, message_body, sent_timestamp_millis)|
+      Message.new(receipt_handle, message_body, {'SentTimestamp' => sent_timestamp_millis})
+    end
   end
 
   # @yield [#send_message] the object you can send messages through (will be flushed at method return)
@@ -66,7 +68,7 @@ class Sqewer::LocalConnection < Sqewer::Connection
   # Only gets used in tests
   def truncate!
     with_db do |db|
-      db.execute("DELETE FROM sqewer_messages_v2 WHERE queue_name = ?", @queue_name)
+      db.execute("DELETE FROM sqewer_messages_v3 WHERE queue_name = ?", @queue_name)
     end
   end
 
@@ -91,51 +93,53 @@ class Sqewer::LocalConnection < Sqewer::Connection
     with_db do |db|
       db.execute("BEGIN")
       ids_to_delete.each do |id|
-        db.execute("DELETE FROM sqewer_messages_v2 WHERE receipt_handle = ?", id)
+        db.execute("DELETE FROM sqewer_messages_v3 WHERE receipt_handle = ?", id)
       end
       db.execute("COMMIT")
     end
   end
 
-  def load_receipt_handles_and_bodies
+  def load_receipt_handles_bodies_and_timestamps
     t = Time.now.to_i
 
     # First make messages that were previously marked invisible but not deleted visible again
     with_db do |db|
       db.execute("BEGIN")
       # Make messages visible that have to be redelivered
-      db.execute("UPDATE sqewer_messages_v2
-        SET visible = 't' 
+      db.execute("UPDATE sqewer_messages_v3
+        SET visible = 't'
         WHERE queue_name = ? AND visible = 'f' AND last_delivery_at_epoch < ?", @queue_name.to_s, t - 60)
       # Remove hopeless messages
-      db.execute("DELETE FROM sqewer_messages_v2
+      db.execute("DELETE FROM sqewer_messages_v3
         WHERE queue_name = ? AND times_delivered_so_far >= ?", @queue_name.to_s, ASSUME_DEADLETTER_AFTER_N_DELIVERIES)
       db.execute("COMMIT")
     end
 
+    # Then select messages to receive
     rows = with_readonly_db do |db|
-      db.execute("SELECT id, receipt_handle, message_body FROM sqewer_messages_v2
+      db.execute("SELECT id, receipt_handle, message_body, sent_timestamp_millis FROM sqewer_messages_v3
         WHERE queue_name = ? AND visible = 't' AND deliver_after_epoch <= ? AND last_delivery_at_epoch <= ?",
         @queue_name.to_s, t, t)
     end
-    
+
     with_db do |db|
       db.execute("BEGIN")
       rows.map do |(id, *_)|
-        db.execute("UPDATE sqewer_messages_v2
+        db.execute("UPDATE sqewer_messages_v3
           SET visible = 'f', times_delivered_so_far = times_delivered_so_far + 1, last_delivery_at_epoch = ?
           WHERE id = ?", t, id)
       end
       db.execute("COMMIT")
     end
 
-    rows.map do |(_, *receipt_handle_and_body)|
-      receipt_handle_and_body
+    rows.map do |(_db_id, receipt_handle, body, timestamp)|
+      [receipt_handle, body, timestamp]
     end
   end
 
   def persist_messages(messages)
     epoch = Time.now.to_i
+    sent_timestamp_millis = (Time.now.to_f * 1000).to_i
     bodies_and_deliver_afters = messages.map do |msg|
       [msg.fetch(:message_body), epoch + msg.fetch(:delay_seconds, 0)]
     end
@@ -143,10 +147,10 @@ class Sqewer::LocalConnection < Sqewer::Connection
     with_db do |db|
       db.execute("BEGIN")
       bodies_and_deliver_afters.map do |body, deliver_after_epoch|
-        db.execute("INSERT INTO sqewer_messages_v2
-          (queue_name, receipt_handle, message_body, deliver_after_epoch, last_delivery_at_epoch)
-          VALUES(?, ?, ?, ?, ?)",
-          @queue_name.to_s, SecureRandom.uuid, body, deliver_after_epoch, epoch)
+        db.execute("INSERT INTO sqewer_messages_v3
+          (queue_name, receipt_handle, message_body, deliver_after_epoch, last_delivery_at_epoch, sent_timestamp_millis)
+          VALUES(?, ?, ?, ?, ?, ?)",
+          @queue_name.to_s, SecureRandom.uuid, body, deliver_after_epoch, epoch, sent_timestamp_millis)
       end
       db.execute("COMMIT")
     end

data/lib/sqewer/version.rb

@@ -1,3 +1,3 @@
 module Sqewer
-  VERSION = '6.1.0'
+  VERSION = '6.2.0'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: sqewer
 version: !ruby/object:Gem::Version
-  version: 6.1.0
+  version: 6.2.0
 platform: ruby
 authors:
 - Julik Tarkhanov
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-07-02 00:00:00.000000000 Z
+date: 2019-07-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aws-sdk-sqs