riverqueue-sequel 0.11.0 → 0.12.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c7ade67ed4f568f7dbc60f31ea72eb6e5b6c71b1274869e67bd9d3dbf120a562
-  data.tar.gz: af7100c24950d9a32458928690b0e4a61a1375d1dfdbd1571bddb43efa717619
+  metadata.gz: 8672d5a5d5c01059a17a307704e5512496aecc5e62f57afe2dfcd5ddd8569a34
+  data.tar.gz: abca68b736d841506449226a45e7b3d049884b40accdfb2c0e5fecf8cbbf896a
 SHA512:
-  metadata.gz: ae68c96561a0f54df6e181c4e4028f3f0ed1b5a929d2b12428ea4b8f3c53f8bc46f33ae2d77fd5b86e3c21ba8ba193bf21d6bd69433bb2c735152225f0810139
-  data.tar.gz: 1027bd3dcddb61912915dd78214445be02e9ed2400123446a87c4aa364b4f4868811d60d227be878ee1900336692edea75e47899eb5174453998d97d8abd7369
+  metadata.gz: 8af058f89ffef579beb2cc91f428a4e004a7d844c61fd2ae588c8214e6fd54e7dc3b22acf7a01feca0dd2474945c5a6ec408378b42918e4f7bd29359c861c8fe
+  data.tar.gz: 1d16f622b3df74b166ffccabee0c2374d810b43d026d344529df4f7d9eb2fd66a7f0bc2448a4e12619d00632c99e119a312f487635074dc14f35c47a44fbffd3

data/lib/driver.rb

@@ -1,7 +1,9 @@
+# frozen_string_literal: true
+
 require "securerandom"
 
 module River::Driver
-  # Provides a Sequel driver for River that supports both PostgreSQL and SQLite.
+  # Provides a Sequel driver for River supporting PostgreSQL, YugabyteDB, and SQLite.
   #
   # Used in conjunction with a River client like:
   #
@@ -14,7 +16,76 @@ module River::Driver
   #   client = River::Client.new(River::Driver::Sequel.new(DB))
   #
   class Sequel
-    SQLITE_CONFLICT_WHERE = <<~SQL.chomp
+    include River::Driver::Runtime
+
+    # Creates a driver backed by a connected Sequel::Database for PostgreSQL or
+    # SQLite.
+    def initialize(db)
+      @db = db
+      @postgres_capabilities_cache = PostgresCapabilities::Cache.new
+      @is_sqlite = (db.database_type == :sqlite)
+
+      unless @is_sqlite
+        db.extension(:pg_array)
+        db.extension(:pg_json)
+      end
+    end
+
+    def job_get_by_id(id)
+      if @is_sqlite
+        row = sqlite_job_rows("WHERE id = ? LIMIT 1", id).first
+        row ? sqlite_to_job_row_from_raw(row) : nil
+      else
+        row = @db[:river_job].where(id: id).first
+        row ? to_job_row(row) : nil
+      end
+    end
+
+    def job_insert(insert_params)
+      job_insert_many([insert_params]).first
+    end
+
+    def job_insert_many(insert_params_array)
+      return [] if insert_params_array.empty?
+
+      return sqlite_job_insert_many(insert_params_array) if @is_sqlite
+
+      transaction do
+        results = postgres_job_insert_many(insert_params_array)
+        postgres_notify_insert(insert_params_array)
+        results
+      end
+    end
+
+    def job_list(params = :all)
+      return super unless params == :all
+
+      runtime_job_rows("ORDER BY id")
+    end
+
+    def rollback_exception
+      ::Sequel::Rollback
+    end
+
+    # Backend used by River::Migrator.
+    def migration_backend = @is_sqlite ? :sqlite : :postgresql
+
+    # Pins a raw connection for the complete migration operation.
+    def migration_connection
+      @db.synchronize do |connection|
+        raise River::Error, "migrations cannot run inside an application transaction" if @db.in_transaction?
+
+        yield connection
+      end
+    end
+
+    # Runs the block in a Sequel transaction or savepoint and returns the
+    # block's result.
+    def transaction(&)
+      @db.transaction(savepoint: true, &)
+    end
+
+    SQLITE_CONFLICT_WHERE = <<~SQL.chomp.freeze
       unique_key IS NOT NULL
           AND unique_states IS NOT NULL
           AND CASE state
@@ -29,105 +100,147 @@ module River::Driver
             ELSE 0
           END >= 1
     SQL
-    private_constant :SQLITE_CONFLICT_WHERE
 
     # SQLite 3.45+ may store JSON as binary JSONB. Always project JSON columns
     # through json() so this driver can read both the current JSONB format and
     # the text JSON used by River migrations through version 006. Cast times to
     # text so Sequel doesn't interpret timezone-less SQLite timestamps in the
-    # process timezone.
-    SQLITE_JOB_COLUMNS = <<~SQL.chomp
+    # process timezone. Flag 10 validates JSONB contents, not just its header,
+    # so damaged blobs reach the row decoder without aborting the SQL query.
+    SQLITE_JOB_COLUMNS = <<~SQL.chomp.freeze
       id,
-      json(args) AS args,
+      CASE WHEN json_valid(args, 10) THEN json(args) ELSE args END AS args,
       attempt,
       CAST(attempted_at AS text) AS attempted_at,
-      json(attempted_by) AS attempted_by,
+      CASE WHEN json_valid(attempted_by, 10) THEN json(attempted_by) ELSE attempted_by END AS attempted_by,
       CAST(created_at AS text) AS created_at,
-      json(errors) AS errors,
+      CASE WHEN json_valid(errors, 10) THEN json(errors) ELSE errors END AS errors,
       CAST(finalized_at AS text) AS finalized_at,
       kind,
       max_attempts,
-      json(metadata) AS metadata,
+      CASE WHEN json_valid(metadata, 10) THEN json(metadata) ELSE metadata END AS metadata,
       priority,
       queue,
       state,
       CAST(scheduled_at AS text) AS scheduled_at,
-      json(tags) AS tags,
+      CASE WHEN json_valid(tags, 10) THEN json(tags) ELSE tags END AS tags,
       unique_key,
       unique_states
     SQL
-    private_constant :SQLITE_JOB_COLUMNS
 
-    SQLITE_UNIQUE_NONCE_KEY = "river:unique_nonce"
-    private_constant :SQLITE_UNIQUE_NONCE_KEY
+    UNIQUE_INSERT_METADATA_KEY = "river:unique_nonce"
 
-    def initialize(db)
-      @db = db
-      @is_sqlite = (db.database_type == :sqlite)
+    private_constant :SQLITE_CONFLICT_WHERE, :SQLITE_JOB_COLUMNS, :UNIQUE_INSERT_METADATA_KEY
 
-      unless @is_sqlite
-        db.extension(:pg_array)
-        db.extension(:pg_json)
-      end
+    private def format_time(time)
+      time.getutc.round(3).strftime("%Y-%m-%d %H:%M:%S.%3N")
     end
 
-    def job_get_by_id(id)
-      if @is_sqlite
-        row = sqlite_job_rows("WHERE id = ? LIMIT 1", id).first
-        row ? sqlite_to_job_row_from_raw(row) : nil
-      else
-        data_set = @db[:river_job].where(id: id)
-        data_set.first ? to_job_row(data_set.first) : nil
-      end
+    private def postgres_insert_params_to_hash(insert_params, nonce)
+      metadata = insert_params.metadata || {}
+      metadata = metadata.merge(UNIQUE_INSERT_METADATA_KEY => nonce) if nonce
+      {
+        args: insert_params.encoded_args,
+        kind: insert_params.kind,
+        max_attempts: insert_params.max_attempts,
+        metadata: ::Sequel.pg_jsonb(metadata),
+        priority: insert_params.priority,
+        queue: insert_params.queue,
+        scheduled_at: insert_params.scheduled_at,
+        state: insert_params.state,
+        tags: ::Sequel.pg_array(insert_params.tags || [], :text),
+        unique_key: insert_params.unique_key ? ::Sequel.blob(insert_params.unique_key) : nil,
+        unique_states: insert_params.unique_states
+      }
     end
 
-    def job_insert(insert_params)
-      job_insert_many([insert_params]).first
+    private def postgres_job_insert_many(insert_params_array)
+      capabilities = postgres_capabilities
+      nonce = SecureRandom.hex(8) if capabilities.unique_insert_mode == :metadata_nonce
+      @db[:river_job]
+        .insert_conflict(
+          conflict_where: ::Sequel.lit(
+            "unique_key IS NOT NULL AND unique_states IS NOT NULL AND river_job_state_in_bitmask(unique_states, state)"
+          ),
+          target: [:unique_key],
+          update: {kind: ::Sequel[:excluded][:kind]}
+        )
+        .returning(::Sequel.lit("*, #{capabilities.unique_insert_sql} AS unique_skipped_as_duplicate"))
+        .multi_insert(insert_params_array.map { |p| postgres_insert_params_to_hash(p, nonce) })
+        .map do |row|
+          job = to_job_row(row)
+          [job, nonce ? job.metadata[UNIQUE_INSERT_METADATA_KEY] != nonce : row[:unique_skipped_as_duplicate]]
+        end
     end
 
-    def job_insert_many(insert_params_array)
-      @is_sqlite ? sqlite_job_insert_many(insert_params_array) : postgres_job_insert_many(insert_params_array)
+    private def postgres_to_job_row(river_job)
+      runtime_decode_postgres_job(river_job)
+    end
+
+    private def runtime_connection_pool
+      @db
     end
 
-    def job_list
+    private def runtime_execute(sql)
+      @db.run(sql)
+    end
+
+    private def runtime_job_list_without_params
+      job_list(:all)
+    end
+
+    private def runtime_job_rows(suffix)
       if @is_sqlite
-        sqlite_job_rows("ORDER BY id").map { |row| sqlite_to_job_row_from_raw(row) }
+        sqlite_job_rows(suffix).map { |row| sqlite_to_job_row_from_raw(row) }
       else
-        @db[:river_job].order_by(:id).all.map { |job| to_job_row(job) }
+        @db.fetch("SELECT * FROM river_job #{suffix}").map { |row| to_job_row(row) }
       end
     end
 
-    def rollback_exception
-      ::Sequel::Rollback
+    private def runtime_postgres?
+      !@is_sqlite
     end
 
-    def transaction(&)
-      @db.transaction(savepoint: true, &)
+    private def runtime_query_rows(sql)
+      @db.fetch(sql).all
     end
 
-    private def postgres_job_insert_many(insert_params_array)
-      @db[:river_job]
-        .insert_conflict(
-          target: [:unique_key],
-          conflict_where: ::Sequel.lit(
-            "unique_key IS NOT NULL AND unique_states IS NOT NULL AND river_job_state_in_bitmask(unique_states, state)"
-          ),
-          update: {kind: ::Sequel[:excluded][:kind]}
-        )
-        .returning(::Sequel.lit("*, (xmax != 0) AS unique_skipped_as_duplicate"))
-        .multi_insert(insert_params_array.map { |p| postgres_insert_params_to_hash(p) })
-        .map { |row| [to_job_row(row), row[:unique_skipped_as_duplicate]] }
+    private def runtime_quote(value)
+      @db.literal(value)
+    end
+
+    private def runtime_unique_violation_class
+      ::Sequel::UniqueConstraintViolation
+    end
+
+    private def runtime_value(row, key)
+      row[key] || row[key.to_s]
+    end
+
+    private def sqlite_insert_params_to_hash(insert_params, nonce)
+      {
+        args: JSON.parse(insert_params.encoded_args),
+        kind: insert_params.kind,
+        max_attempts: insert_params.max_attempts,
+        metadata: {UNIQUE_INSERT_METADATA_KEY => nonce},
+        priority: insert_params.priority,
+        queue: insert_params.queue,
+        scheduled_at: insert_params.scheduled_at ? format_time(insert_params.scheduled_at) : nil,
+        state: insert_params.state,
+        tags: insert_params.tags || [],
+        unique_key: insert_params.unique_key&.unpack1("H*"),
+        unique_states: insert_params.unique_states&.to_i(2)
+      }.tap { |values| values[:metadata] = (insert_params.metadata || {}).merge(values[:metadata]) }
     end
 
     # River's current SQLite driver uses json_each to make a batch a single,
     # atomic statement. The JSON columns are converted to SQLite JSONB here,
     # matching migration 007 and newer River databases.
     private def sqlite_job_insert_many(insert_params_array)
-      return [] if insert_params_array.empty?
-
+      nonces = sqlite_insert_nonces(insert_params_array)
       @db.transaction(savepoint: true) do
-        nonce = SecureRandom.hex(8)
-        jobs = insert_params_array.map { |param| sqlite_insert_params_to_hash(param, nonce) }
+        jobs = insert_params_array.zip(nonces).map { |param, nonce| sqlite_insert_params_to_hash(param, nonce) }
+        inserted_nonces = nonces.to_h { |nonce| [nonce, true] }
 
         sql = <<~SQL
           INSERT INTO river_job (
@@ -167,130 +280,16 @@ module River::Driver
           RETURNING #{SQLITE_JOB_COLUMNS}
         SQL
 
-        rows = @db.fetch(sql, JSON.dump(jobs)).all
+        rows = @db.fetch(sql, JSON.generate(jobs)).all
         sqlite_notify_insert(insert_params_array)
 
         rows.map do |row|
           metadata = JSON.parse(row[:metadata])
-          [sqlite_to_job_row_from_raw(row), metadata[SQLITE_UNIQUE_NONCE_KEY] != nonce]
+          [sqlite_to_job_row_from_raw(row), !inserted_nonces.key?(metadata[UNIQUE_INSERT_METADATA_KEY])]
         end
       end
     end
 
-    private def postgres_insert_params_to_hash(insert_params)
-      {
-        args: insert_params.encoded_args,
-        kind: insert_params.kind,
-        max_attempts: insert_params.max_attempts,
-        priority: insert_params.priority,
-        queue: insert_params.queue,
-        state: insert_params.state,
-        scheduled_at: insert_params.scheduled_at,
-        tags: ::Sequel.pg_array(insert_params.tags || [], :text),
-        unique_key: insert_params.unique_key ? ::Sequel.blob(insert_params.unique_key) : nil,
-        unique_states: insert_params.unique_states
-      }
-    end
-
-    private def sqlite_insert_params_to_hash(insert_params, nonce)
-      {
-        args: JSON.parse(insert_params.encoded_args),
-        kind: insert_params.kind,
-        max_attempts: insert_params.max_attempts,
-        metadata: {SQLITE_UNIQUE_NONCE_KEY => nonce},
-        priority: insert_params.priority,
-        queue: insert_params.queue,
-        scheduled_at: insert_params.scheduled_at ? format_time(insert_params.scheduled_at) : nil,
-        state: insert_params.state,
-        tags: insert_params.tags || [],
-        unique_key: insert_params.unique_key&.unpack1("H*"),
-        unique_states: insert_params.unique_states&.to_i(2)
-      }
-    end
-
-    private def to_job_row(river_job)
-      if @is_sqlite
-        row = sqlite_job_rows("WHERE id = ? LIMIT 1", river_job[:id]).first
-        sqlite_to_job_row_from_raw(row)
-      else
-        postgres_to_job_row(river_job)
-      end
-    end
-
-    private def postgres_to_job_row(river_job)
-      River::JobRow.new(
-        id: river_job[:id],
-        args: river_job[:args].to_h,
-        attempt: river_job[:attempt],
-        attempted_at: river_job[:attempted_at]&.getutc,
-        attempted_by: river_job[:attempted_by],
-        created_at: river_job[:created_at].getutc,
-        errors: river_job[:errors]&.map { |deserialized_error|
-          River::AttemptError.new(
-            at: Time.parse(deserialized_error["at"]),
-            attempt: deserialized_error["attempt"],
-            error: deserialized_error["error"],
-            trace: deserialized_error["trace"]
-          )
-        },
-        finalized_at: river_job[:finalized_at]&.getutc,
-        kind: river_job[:kind],
-        max_attempts: river_job[:max_attempts],
-        metadata: river_job[:metadata],
-        priority: river_job[:priority],
-        queue: river_job[:queue],
-        scheduled_at: river_job[:scheduled_at].getutc,
-        state: river_job[:state],
-        tags: river_job[:tags].to_a,
-        unique_key: river_job[:unique_key]&.to_s,
-        unique_states: ::River::UniqueBitmask.to_states(river_job[:unique_states]&.to_i(2))
-      )
-    end
-
-    private def sqlite_to_job_row_from_raw(river_job)
-      errors = river_job[:errors] ? JSON.parse(river_job[:errors]) : []
-
-      River::JobRow.new(
-        id: river_job[:id],
-        args: JSON.parse(river_job[:args]),
-        attempt: river_job[:attempt],
-        attempted_at: parse_sqlite_time(river_job[:attempted_at]),
-        attempted_by: river_job[:attempted_by] ? JSON.parse(river_job[:attempted_by]) : nil,
-        created_at: parse_sqlite_time(river_job[:created_at]),
-        errors: errors.map { |deserialized_error|
-          River::AttemptError.new(
-            at: Time.parse(deserialized_error["at"]),
-            attempt: deserialized_error["attempt"],
-            error: deserialized_error["error"],
-            trace: deserialized_error["trace"]
-          )
-        },
-        finalized_at: parse_sqlite_time(river_job[:finalized_at]),
-        kind: river_job[:kind],
-        max_attempts: river_job[:max_attempts],
-        metadata: JSON.parse(river_job[:metadata]),
-        priority: river_job[:priority],
-        queue: river_job[:queue],
-        scheduled_at: parse_sqlite_time(river_job[:scheduled_at]),
-        state: river_job[:state],
-        tags: JSON.parse(river_job[:tags]),
-        unique_key: river_job[:unique_key]&.to_s,
-        unique_states: river_job[:unique_states] ? ::River::UniqueBitmask.to_states(river_job[:unique_states]) : nil
-      )
-    end
-
-    private def format_time(time)
-      time.getutc.round(3).strftime("%Y-%m-%d %H:%M:%S.%3N")
-    end
-
-    private def parse_sqlite_time(value)
-      return nil unless value
-
-      value = value.to_s
-      value += " UTC" unless value.match?(/(?:Z|[+-]\d{2}:?\d{2})\z/)
-      Time.parse(value).utc
-    end
-
     private def sqlite_job_rows(suffix, *binds)
       @db.fetch("SELECT #{SQLITE_JOB_COLUMNS} FROM river_job #{suffix}", *binds).all
     end
@@ -303,8 +302,21 @@ module River::Driver
       return if queues.empty?
 
       @db[:river_notification].multi_insert(queues.map do |queue|
-        {payload: JSON.dump({queue: queue}), topic: "insert"}
+        {payload: JSON.generate({queue: queue}), topic: "river_insert"}
       end)
     end
+
+    private def sqlite_to_job_row_from_raw(row)
+      runtime_decode_sqlite_job(row)
+    end
+
+    private def to_job_row(river_job)
+      if @is_sqlite
+        row = sqlite_job_rows("WHERE id = ? LIMIT 1", river_job[:id]).first
+        sqlite_to_job_row_from_raw(row)
+      else
+        postgres_to_job_row(river_job)
+      end
+    end
   end
 end

data/lib/riverqueue-sequel.rb

@@ -1,4 +1,7 @@
+# frozen_string_literal: true
+
 require "sequel"
+require "riverqueue"
 
 require_relative "driver"
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: riverqueue-sequel
 version: !ruby/object:Gem::Version
-  version: 0.11.0
+  version: 0.12.0
 platform: ruby
 authors:
 - Blake Gentry
@@ -30,8 +30,22 @@ dependencies:
     - - "<"
       - !ruby/object:Gem::Version
         version: '1000'
-description: Sequel PostgreSQL and SQLite driver for the River Ruby gem. Use in conjunction
-  with the riverqueue gem to insert jobs that are worked in Go.
+- !ruby/object:Gem::Dependency
+  name: riverqueue
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.12.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.12.0
+description: Sequel PostgreSQL and SQLite driver for inserting and working River jobs
+  in Ruby.
 email: brandur@brandur.org
 executables: []
 extensions: []
@@ -41,7 +55,7 @@ files:
 - lib/riverqueue-sequel.rb
 homepage: https://riverqueue.com
 licenses:
-- LGPL-3.0-or-later
+- MPL-2.0
 metadata: {}
 rdoc_options: []
 require_paths:
@@ -50,7 +64,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '0'
+      version: '3.2'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="