riverqueue-sequel 0.10.0 → 0.11.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: af0f2602b8323031224709b01961d93e73865df77caffb8099959f411541677d
-  data.tar.gz: b1b720a495fc3a5eaf8413fdc8732eda6a1de2bb7e033a0ec6531337a3d53f3f
+  metadata.gz: c7ade67ed4f568f7dbc60f31ea72eb6e5b6c71b1274869e67bd9d3dbf120a562
+  data.tar.gz: af7100c24950d9a32458928690b0e4a61a1375d1dfdbd1571bddb43efa717619
 SHA512:
-  metadata.gz: 7215adbf69d421de6d65fceb09be674b777d399b14d75ac47e847c79cf6b29eab17b7267392e99813af29d33fc3543d234c2c19ef2a57d3898f4d6a50624e9c0
-  data.tar.gz: 9dbd8f19c8ce23281bf6c78fde501e7dbb5f8969562ef25fc0945bd58ec2879948d7cdd49671e9dee93480f703deb9e417e86fb4cbda089797246c22e23823d0
+  metadata.gz: ae68c96561a0f54df6e181c4e4028f3f0ed1b5a929d2b12428ea4b8f3c53f8bc46f33ae2d77fd5b86e3c21ba8ba193bf21d6bd69433bb2c735152225f0810139
+  data.tar.gz: 1027bd3dcddb61912915dd78214445be02e9ed2400123446a87c4aa364b4f4868811d60d227be878ee1900336692edea75e47899eb5174453998d97d8abd7369

data/lib/driver.rb

@@ -1,21 +1,84 @@
+require "securerandom"
+
 module River::Driver
-  # Provides a Sequel driver for River.
+  # Provides a Sequel driver for River that supports both PostgreSQL and SQLite.
   #
   # Used in conjunction with a River client like:
   #
   #   DB = Sequel.connect("postgres://...")
   #   client = River::Client.new(River::Driver::Sequel.new(DB))
   #
+  # Or with SQLite:
+  #
+  #   DB = Sequel.connect("sqlite://path/to/river.db")
+  #   client = River::Client.new(River::Driver::Sequel.new(DB))
+  #
   class Sequel
+    SQLITE_CONFLICT_WHERE = <<~SQL.chomp
+      unique_key IS NOT NULL
+          AND unique_states IS NOT NULL
+          AND CASE state
+            WHEN 'available' THEN unique_states & (1 << 0)
+            WHEN 'cancelled' THEN unique_states & (1 << 1)
+            WHEN 'completed' THEN unique_states & (1 << 2)
+            WHEN 'discarded' THEN unique_states & (1 << 3)
+            WHEN 'pending'   THEN unique_states & (1 << 4)
+            WHEN 'retryable' THEN unique_states & (1 << 5)
+            WHEN 'running'   THEN unique_states & (1 << 6)
+            WHEN 'scheduled' THEN unique_states & (1 << 7)
+            ELSE 0
+          END >= 1
+    SQL
+    private_constant :SQLITE_CONFLICT_WHERE
+
+    # SQLite 3.45+ may store JSON as binary JSONB. Always project JSON columns
+    # through json() so this driver can read both the current JSONB format and
+    # the text JSON used by River migrations through version 006. Cast times to
+    # text so Sequel doesn't interpret timezone-less SQLite timestamps in the
+    # process timezone.
+    SQLITE_JOB_COLUMNS = <<~SQL.chomp
+      id,
+      json(args) AS args,
+      attempt,
+      CAST(attempted_at AS text) AS attempted_at,
+      json(attempted_by) AS attempted_by,
+      CAST(created_at AS text) AS created_at,
+      json(errors) AS errors,
+      CAST(finalized_at AS text) AS finalized_at,
+      kind,
+      max_attempts,
+      json(metadata) AS metadata,
+      priority,
+      queue,
+      state,
+      CAST(scheduled_at AS text) AS scheduled_at,
+      json(tags) AS tags,
+      unique_key,
+      unique_states
+    SQL
+    private_constant :SQLITE_JOB_COLUMNS
+
+    SQLITE_UNIQUE_NONCE_KEY = "river:unique_nonce"
+    private_constant :SQLITE_UNIQUE_NONCE_KEY
+
     def initialize(db)
       @db = db
-      @db.extension(:pg_array)
-      @db.extension(:pg_json)
+      @is_sqlite = (db.database_type == :sqlite)
+
+      unless @is_sqlite
+        db.extension(:pg_array)
+        db.extension(:pg_json)
+      end
     end
 
     def job_get_by_id(id)
-      data_set = @db[:river_job].where(id: id)
-      data_set.first ? to_job_row(data_set.first) : nil
+      if @is_sqlite
+        row = sqlite_job_rows("WHERE id = ? LIMIT 1", id).first
+        row ? sqlite_to_job_row_from_raw(row) : nil
+      else
+        data_set = @db[:river_job].where(id: id)
+        data_set.first ? to_job_row(data_set.first) : nil
+      end
     end
 
     def job_insert(insert_params)
@@ -23,6 +86,26 @@ module River::Driver
     end
 
     def job_insert_many(insert_params_array)
+      @is_sqlite ? sqlite_job_insert_many(insert_params_array) : postgres_job_insert_many(insert_params_array)
+    end
+
+    def job_list
+      if @is_sqlite
+        sqlite_job_rows("ORDER BY id").map { |row| sqlite_to_job_row_from_raw(row) }
+      else
+        @db[:river_job].order_by(:id).all.map { |job| to_job_row(job) }
+      end
+    end
+
+    def rollback_exception
+      ::Sequel::Rollback
+    end
+
+    def transaction(&)
+      @db.transaction(savepoint: true, &)
+    end
+
+    private def postgres_job_insert_many(insert_params_array)
       @db[:river_job]
         .insert_conflict(
           target: [:unique_key],
@@ -32,24 +115,69 @@ module River::Driver
           update: {kind: ::Sequel[:excluded][:kind]}
         )
         .returning(::Sequel.lit("*, (xmax != 0) AS unique_skipped_as_duplicate"))
-        .multi_insert(insert_params_array.map { |p| insert_params_to_hash(p) })
-        .map { |row| to_insert_result(row) }
+        .multi_insert(insert_params_array.map { |p| postgres_insert_params_to_hash(p) })
+        .map { |row| [to_job_row(row), row[:unique_skipped_as_duplicate]] }
     end
 
-    def job_list
-      data_set = @db[:river_job].order_by(:id)
-      data_set.all.map { |job| to_job_row(job) }
-    end
+    # River's current SQLite driver uses json_each to make a batch a single,
+    # atomic statement. The JSON columns are converted to SQLite JSONB here,
+    # matching migration 007 and newer River databases.
+    private def sqlite_job_insert_many(insert_params_array)
+      return [] if insert_params_array.empty?
 
-    def rollback_exception
-      ::Sequel::Rollback
-    end
+      @db.transaction(savepoint: true) do
+        nonce = SecureRandom.hex(8)
+        jobs = insert_params_array.map { |param| sqlite_insert_params_to_hash(param, nonce) }
 
-    def transaction(&)
-      @db.transaction(savepoint: true, &)
+        sql = <<~SQL
+          INSERT INTO river_job (
+            args,
+            created_at,
+            kind,
+            max_attempts,
+            metadata,
+            priority,
+            queue,
+            scheduled_at,
+            state,
+            tags,
+            unique_key,
+            unique_states
+          )
+          SELECT
+            jsonb(json_extract(value, '$.args')),
+            datetime('now', 'subsec'),
+            cast(json_extract(value, '$.kind') AS text),
+            cast(json_extract(value, '$.max_attempts') AS integer),
+            jsonb(json_extract(value, '$.metadata')),
+            cast(json_extract(value, '$.priority') AS integer),
+            cast(json_extract(value, '$.queue') AS text),
+            coalesce(cast(json_extract(value, '$.scheduled_at') AS text), datetime('now', 'subsec')),
+            cast(json_extract(value, '$.state') AS text),
+            jsonb(json_extract(value, '$.tags')),
+            CASE
+              WHEN length(cast(json_extract(value, '$.unique_key') AS text)) = 0 THEN NULL
+              ELSE unhex(cast(json_extract(value, '$.unique_key') AS text))
+            END,
+            nullif(cast(json_extract(value, '$.unique_states') AS integer), 0)
+          FROM json_each(cast(? AS blob))
+          WHERE true
+          ON CONFLICT (unique_key) WHERE #{SQLITE_CONFLICT_WHERE}
+          DO UPDATE SET kind = EXCLUDED.kind
+          RETURNING #{SQLITE_JOB_COLUMNS}
+        SQL
+
+        rows = @db.fetch(sql, JSON.dump(jobs)).all
+        sqlite_notify_insert(insert_params_array)
+
+        rows.map do |row|
+          metadata = JSON.parse(row[:metadata])
+          [sqlite_to_job_row_from_raw(row), metadata[SQLITE_UNIQUE_NONCE_KEY] != nonce]
+        end
+      end
     end
 
-    private def insert_params_to_hash(insert_params)
+    private def postgres_insert_params_to_hash(insert_params)
       {
         args: insert_params.encoded_args,
         kind: insert_params.kind,
@@ -64,11 +192,32 @@ module River::Driver
       }
     end
 
-    private def to_insert_result(result)
-      [to_job_row(result), result[:unique_skipped_as_duplicate]]
+    private def sqlite_insert_params_to_hash(insert_params, nonce)
+      {
+        args: JSON.parse(insert_params.encoded_args),
+        kind: insert_params.kind,
+        max_attempts: insert_params.max_attempts,
+        metadata: {SQLITE_UNIQUE_NONCE_KEY => nonce},
+        priority: insert_params.priority,
+        queue: insert_params.queue,
+        scheduled_at: insert_params.scheduled_at ? format_time(insert_params.scheduled_at) : nil,
+        state: insert_params.state,
+        tags: insert_params.tags || [],
+        unique_key: insert_params.unique_key&.unpack1("H*"),
+        unique_states: insert_params.unique_states&.to_i(2)
+      }
     end
 
     private def to_job_row(river_job)
+      if @is_sqlite
+        row = sqlite_job_rows("WHERE id = ? LIMIT 1", river_job[:id]).first
+        sqlite_to_job_row_from_raw(row)
+      else
+        postgres_to_job_row(river_job)
+      end
+    end
+
+    private def postgres_to_job_row(river_job)
       River::JobRow.new(
         id: river_job[:id],
         args: river_job[:args].to_h,
@@ -97,5 +246,65 @@ module River::Driver
         unique_states: ::River::UniqueBitmask.to_states(river_job[:unique_states]&.to_i(2))
       )
     end
+
+    private def sqlite_to_job_row_from_raw(river_job)
+      errors = river_job[:errors] ? JSON.parse(river_job[:errors]) : []
+
+      River::JobRow.new(
+        id: river_job[:id],
+        args: JSON.parse(river_job[:args]),
+        attempt: river_job[:attempt],
+        attempted_at: parse_sqlite_time(river_job[:attempted_at]),
+        attempted_by: river_job[:attempted_by] ? JSON.parse(river_job[:attempted_by]) : nil,
+        created_at: parse_sqlite_time(river_job[:created_at]),
+        errors: errors.map { |deserialized_error|
+          River::AttemptError.new(
+            at: Time.parse(deserialized_error["at"]),
+            attempt: deserialized_error["attempt"],
+            error: deserialized_error["error"],
+            trace: deserialized_error["trace"]
+          )
+        },
+        finalized_at: parse_sqlite_time(river_job[:finalized_at]),
+        kind: river_job[:kind],
+        max_attempts: river_job[:max_attempts],
+        metadata: JSON.parse(river_job[:metadata]),
+        priority: river_job[:priority],
+        queue: river_job[:queue],
+        scheduled_at: parse_sqlite_time(river_job[:scheduled_at]),
+        state: river_job[:state],
+        tags: JSON.parse(river_job[:tags]),
+        unique_key: river_job[:unique_key]&.to_s,
+        unique_states: river_job[:unique_states] ? ::River::UniqueBitmask.to_states(river_job[:unique_states]) : nil
+      )
+    end
+
+    private def format_time(time)
+      time.getutc.round(3).strftime("%Y-%m-%d %H:%M:%S.%3N")
+    end
+
+    private def parse_sqlite_time(value)
+      return nil unless value
+
+      value = value.to_s
+      value += " UTC" unless value.match?(/(?:Z|[+-]\d{2}:?\d{2})\z/)
+      Time.parse(value).utc
+    end
+
+    private def sqlite_job_rows(suffix, *binds)
+      @db.fetch("SELECT #{SQLITE_JOB_COLUMNS} FROM river_job #{suffix}", *binds).all
+    end
+
+    private def sqlite_notify_insert(insert_params_array)
+      queues = insert_params_array
+        .select { |param| param.state == ::River::JOB_STATE_AVAILABLE }
+        .map(&:queue)
+        .uniq
+      return if queues.empty?
+
+      @db[:river_notification].multi_insert(queues.map do |queue|
+        {payload: JSON.dump({queue: queue}), topic: "insert"}
+      end)
+    end
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: riverqueue-sequel
 version: !ruby/object:Gem::Version
-  version: 0.10.0
+  version: 0.11.0
 platform: ruby
 authors:
 - Blake Gentry
@@ -10,26 +10,6 @@ bindir: bin
 cert_chain: []
 date: 1980-01-02 00:00:00.000000000 Z
 dependencies:
-- !ruby/object:Gem::Dependency
-  name: pg
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">"
-      - !ruby/object:Gem::Version
-        version: '0'
-    - - "<"
-      - !ruby/object:Gem::Version
-        version: '1000'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">"
-      - !ruby/object:Gem::Version
-        version: '0'
-    - - "<"
-      - !ruby/object:Gem::Version
-        version: '1000'
 - !ruby/object:Gem::Dependency
   name: sequel
   requirement: !ruby/object:Gem::Requirement
@@ -50,8 +30,8 @@ dependencies:
     - - "<"
       - !ruby/object:Gem::Version
         version: '1000'
-description: Sequel driver for the River Ruby gem. Use in conjunction with the riverqueue
-  gem to insert jobs that are worked in Go.
+description: Sequel PostgreSQL and SQLite driver for the River Ruby gem. Use in conjunction
+  with the riverqueue gem to insert jobs that are worked in Go.
 email: brandur@brandur.org
 executables: []
 extensions: []
@@ -79,5 +59,5 @@ required_rubygems_version: !ruby/object:Gem::Requirement
 requirements: []
 rubygems_version: 4.0.9
 specification_version: 4
-summary: Sequel driver for the River Ruby gem.
+summary: Sequel PostgreSQL and SQLite driver for the River Ruby gem.
 test_files: []