riverqueue-sequel 0.10.1 → 0.12.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 863659587eec09b4b452469224566c7ca5ba81e34b743326f7ba0e8e7dcd8935
-  data.tar.gz: b1b720a495fc3a5eaf8413fdc8732eda6a1de2bb7e033a0ec6531337a3d53f3f
+  metadata.gz: 8672d5a5d5c01059a17a307704e5512496aecc5e62f57afe2dfcd5ddd8569a34
+  data.tar.gz: abca68b736d841506449226a45e7b3d049884b40accdfb2c0e5fecf8cbbf896a
 SHA512:
-  metadata.gz: 6d47a5886765f27631e57ed847a4f84814bf4814b10f3dc971776d883d781dd4d7bf2610d841b8495af55bc9d4bd5f91424722e2a29a0a8ce84fc254a407a5f0
-  data.tar.gz: 9dbd8f19c8ce23281bf6c78fde501e7dbb5f8969562ef25fc0945bd58ec2879948d7cdd49671e9dee93480f703deb9e417e86fb4cbda089797246c22e23823d0
+  metadata.gz: 8af058f89ffef579beb2cc91f428a4e004a7d844c61fd2ae588c8214e6fd54e7dc3b22acf7a01feca0dd2474945c5a6ec408378b42918e4f7bd29359c861c8fe
+  data.tar.gz: 1d16f622b3df74b166ffccabee0c2374d810b43d026d344529df4f7d9eb2fd66a7f0bc2448a4e12619d00632c99e119a312f487635074dc14f35c47a44fbffd3

data/lib/driver.rb

@@ -1,21 +1,44 @@
+# frozen_string_literal: true
+
+require "securerandom"
+
 module River::Driver
-  # Provides a Sequel driver for River.
+  # Provides a Sequel driver for River supporting PostgreSQL, YugabyteDB, and SQLite.
   #
   # Used in conjunction with a River client like:
   #
   #   DB = Sequel.connect("postgres://...")
   #   client = River::Client.new(River::Driver::Sequel.new(DB))
   #
+  # Or with SQLite:
+  #
+  #   DB = Sequel.connect("sqlite://path/to/river.db")
+  #   client = River::Client.new(River::Driver::Sequel.new(DB))
+  #
   class Sequel
+    include River::Driver::Runtime
+
+    # Creates a driver backed by a connected Sequel::Database for PostgreSQL or
+    # SQLite.
     def initialize(db)
       @db = db
-      @db.extension(:pg_array)
-      @db.extension(:pg_json)
+      @postgres_capabilities_cache = PostgresCapabilities::Cache.new
+      @is_sqlite = (db.database_type == :sqlite)
+
+      unless @is_sqlite
+        db.extension(:pg_array)
+        db.extension(:pg_json)
+      end
     end
 
     def job_get_by_id(id)
-      data_set = @db[:river_job].where(id: id)
-      data_set.first ? to_job_row(data_set.first) : nil
+      if @is_sqlite
+        row = sqlite_job_rows("WHERE id = ? LIMIT 1", id).first
+        row ? sqlite_to_job_row_from_raw(row) : nil
+      else
+        row = @db[:river_job].where(id: id).first
+        row ? to_job_row(row) : nil
+      end
     end
 
     def job_insert(insert_params)
@@ -23,79 +46,277 @@ module River::Driver
     end
 
     def job_insert_many(insert_params_array)
-      @db[:river_job]
-        .insert_conflict(
-          target: [:unique_key],
-          conflict_where: ::Sequel.lit(
-            "unique_key IS NOT NULL AND unique_states IS NOT NULL AND river_job_state_in_bitmask(unique_states, state)"
-          ),
-          update: {kind: ::Sequel[:excluded][:kind]}
-        )
-        .returning(::Sequel.lit("*, (xmax != 0) AS unique_skipped_as_duplicate"))
-        .multi_insert(insert_params_array.map { |p| insert_params_to_hash(p) })
-        .map { |row| to_insert_result(row) }
+      return [] if insert_params_array.empty?
+
+      return sqlite_job_insert_many(insert_params_array) if @is_sqlite
+
+      transaction do
+        results = postgres_job_insert_many(insert_params_array)
+        postgres_notify_insert(insert_params_array)
+        results
+      end
     end
 
-    def job_list
-      data_set = @db[:river_job].order_by(:id)
-      data_set.all.map { |job| to_job_row(job) }
+    def job_list(params = :all)
+      return super unless params == :all
+
+      runtime_job_rows("ORDER BY id")
     end
 
     def rollback_exception
       ::Sequel::Rollback
     end
 
+    # Backend used by River::Migrator.
+    def migration_backend = @is_sqlite ? :sqlite : :postgresql
+
+    # Pins a raw connection for the complete migration operation.
+    def migration_connection
+      @db.synchronize do |connection|
+        raise River::Error, "migrations cannot run inside an application transaction" if @db.in_transaction?
+
+        yield connection
+      end
+    end
+
+    # Runs the block in a Sequel transaction or savepoint and returns the
+    # block's result.
     def transaction(&)
       @db.transaction(savepoint: true, &)
     end
 
-    private def insert_params_to_hash(insert_params)
+    SQLITE_CONFLICT_WHERE = <<~SQL.chomp.freeze
+      unique_key IS NOT NULL
+          AND unique_states IS NOT NULL
+          AND CASE state
+            WHEN 'available' THEN unique_states & (1 << 0)
+            WHEN 'cancelled' THEN unique_states & (1 << 1)
+            WHEN 'completed' THEN unique_states & (1 << 2)
+            WHEN 'discarded' THEN unique_states & (1 << 3)
+            WHEN 'pending'   THEN unique_states & (1 << 4)
+            WHEN 'retryable' THEN unique_states & (1 << 5)
+            WHEN 'running'   THEN unique_states & (1 << 6)
+            WHEN 'scheduled' THEN unique_states & (1 << 7)
+            ELSE 0
+          END >= 1
+    SQL
+
+    # SQLite 3.45+ may store JSON as binary JSONB. Always project JSON columns
+    # through json() so this driver can read both the current JSONB format and
+    # the text JSON used by River migrations through version 006. Cast times to
+    # text so Sequel doesn't interpret timezone-less SQLite timestamps in the
+    # process timezone. Flag 10 validates JSONB contents, not just its header,
+    # so damaged blobs reach the row decoder without aborting the SQL query.
+    SQLITE_JOB_COLUMNS = <<~SQL.chomp.freeze
+      id,
+      CASE WHEN json_valid(args, 10) THEN json(args) ELSE args END AS args,
+      attempt,
+      CAST(attempted_at AS text) AS attempted_at,
+      CASE WHEN json_valid(attempted_by, 10) THEN json(attempted_by) ELSE attempted_by END AS attempted_by,
+      CAST(created_at AS text) AS created_at,
+      CASE WHEN json_valid(errors, 10) THEN json(errors) ELSE errors END AS errors,
+      CAST(finalized_at AS text) AS finalized_at,
+      kind,
+      max_attempts,
+      CASE WHEN json_valid(metadata, 10) THEN json(metadata) ELSE metadata END AS metadata,
+      priority,
+      queue,
+      state,
+      CAST(scheduled_at AS text) AS scheduled_at,
+      CASE WHEN json_valid(tags, 10) THEN json(tags) ELSE tags END AS tags,
+      unique_key,
+      unique_states
+    SQL
+
+    UNIQUE_INSERT_METADATA_KEY = "river:unique_nonce"
+
+    private_constant :SQLITE_CONFLICT_WHERE, :SQLITE_JOB_COLUMNS, :UNIQUE_INSERT_METADATA_KEY
+
+    private def format_time(time)
+      time.getutc.round(3).strftime("%Y-%m-%d %H:%M:%S.%3N")
+    end
+
+    private def postgres_insert_params_to_hash(insert_params, nonce)
+      metadata = insert_params.metadata || {}
+      metadata = metadata.merge(UNIQUE_INSERT_METADATA_KEY => nonce) if nonce
       {
         args: insert_params.encoded_args,
         kind: insert_params.kind,
         max_attempts: insert_params.max_attempts,
+        metadata: ::Sequel.pg_jsonb(metadata),
         priority: insert_params.priority,
         queue: insert_params.queue,
-        state: insert_params.state,
         scheduled_at: insert_params.scheduled_at,
+        state: insert_params.state,
         tags: ::Sequel.pg_array(insert_params.tags || [], :text),
         unique_key: insert_params.unique_key ? ::Sequel.blob(insert_params.unique_key) : nil,
         unique_states: insert_params.unique_states
       }
     end
 
-    private def to_insert_result(result)
-      [to_job_row(result), result[:unique_skipped_as_duplicate]]
+    private def postgres_job_insert_many(insert_params_array)
+      capabilities = postgres_capabilities
+      nonce = SecureRandom.hex(8) if capabilities.unique_insert_mode == :metadata_nonce
+      @db[:river_job]
+        .insert_conflict(
+          conflict_where: ::Sequel.lit(
+            "unique_key IS NOT NULL AND unique_states IS NOT NULL AND river_job_state_in_bitmask(unique_states, state)"
+          ),
+          target: [:unique_key],
+          update: {kind: ::Sequel[:excluded][:kind]}
+        )
+        .returning(::Sequel.lit("*, #{capabilities.unique_insert_sql} AS unique_skipped_as_duplicate"))
+        .multi_insert(insert_params_array.map { |p| postgres_insert_params_to_hash(p, nonce) })
+        .map do |row|
+          job = to_job_row(row)
+          [job, nonce ? job.metadata[UNIQUE_INSERT_METADATA_KEY] != nonce : row[:unique_skipped_as_duplicate]]
+        end
     end
 
-    private def to_job_row(river_job)
-      River::JobRow.new(
-        id: river_job[:id],
-        args: river_job[:args].to_h,
-        attempt: river_job[:attempt],
-        attempted_at: river_job[:attempted_at]&.getutc,
-        attempted_by: river_job[:attempted_by],
-        created_at: river_job[:created_at].getutc,
-        errors: river_job[:errors]&.map { |deserialized_error|
-          River::AttemptError.new(
-            at: Time.parse(deserialized_error["at"]),
-            attempt: deserialized_error["attempt"],
-            error: deserialized_error["error"],
-            trace: deserialized_error["trace"]
+    private def postgres_to_job_row(river_job)
+      runtime_decode_postgres_job(river_job)
+    end
+
+    private def runtime_connection_pool
+      @db
+    end
+
+    private def runtime_execute(sql)
+      @db.run(sql)
+    end
+
+    private def runtime_job_list_without_params
+      job_list(:all)
+    end
+
+    private def runtime_job_rows(suffix)
+      if @is_sqlite
+        sqlite_job_rows(suffix).map { |row| sqlite_to_job_row_from_raw(row) }
+      else
+        @db.fetch("SELECT * FROM river_job #{suffix}").map { |row| to_job_row(row) }
+      end
+    end
+
+    private def runtime_postgres?
+      !@is_sqlite
+    end
+
+    private def runtime_query_rows(sql)
+      @db.fetch(sql).all
+    end
+
+    private def runtime_quote(value)
+      @db.literal(value)
+    end
+
+    private def runtime_unique_violation_class
+      ::Sequel::UniqueConstraintViolation
+    end
+
+    private def runtime_value(row, key)
+      row[key] || row[key.to_s]
+    end
+
+    private def sqlite_insert_params_to_hash(insert_params, nonce)
+      {
+        args: JSON.parse(insert_params.encoded_args),
+        kind: insert_params.kind,
+        max_attempts: insert_params.max_attempts,
+        metadata: {UNIQUE_INSERT_METADATA_KEY => nonce},
+        priority: insert_params.priority,
+        queue: insert_params.queue,
+        scheduled_at: insert_params.scheduled_at ? format_time(insert_params.scheduled_at) : nil,
+        state: insert_params.state,
+        tags: insert_params.tags || [],
+        unique_key: insert_params.unique_key&.unpack1("H*"),
+        unique_states: insert_params.unique_states&.to_i(2)
+      }.tap { |values| values[:metadata] = (insert_params.metadata || {}).merge(values[:metadata]) }
+    end
+
+    # River's current SQLite driver uses json_each to make a batch a single,
+    # atomic statement. The JSON columns are converted to SQLite JSONB here,
+    # matching migration 007 and newer River databases.
+    private def sqlite_job_insert_many(insert_params_array)
+      nonces = sqlite_insert_nonces(insert_params_array)
+      @db.transaction(savepoint: true) do
+        jobs = insert_params_array.zip(nonces).map { |param, nonce| sqlite_insert_params_to_hash(param, nonce) }
+        inserted_nonces = nonces.to_h { |nonce| [nonce, true] }
+
+        sql = <<~SQL
+          INSERT INTO river_job (
+            args,
+            created_at,
+            kind,
+            max_attempts,
+            metadata,
+            priority,
+            queue,
+            scheduled_at,
+            state,
+            tags,
+            unique_key,
+            unique_states
           )
-        },
-        finalized_at: river_job[:finalized_at]&.getutc,
-        kind: river_job[:kind],
-        max_attempts: river_job[:max_attempts],
-        metadata: river_job[:metadata],
-        priority: river_job[:priority],
-        queue: river_job[:queue],
-        scheduled_at: river_job[:scheduled_at].getutc,
-        state: river_job[:state],
-        tags: river_job[:tags].to_a,
-        unique_key: river_job[:unique_key]&.to_s,
-        unique_states: ::River::UniqueBitmask.to_states(river_job[:unique_states]&.to_i(2))
-      )
+          SELECT
+            jsonb(json_extract(value, '$.args')),
+            datetime('now', 'subsec'),
+            cast(json_extract(value, '$.kind') AS text),
+            cast(json_extract(value, '$.max_attempts') AS integer),
+            jsonb(json_extract(value, '$.metadata')),
+            cast(json_extract(value, '$.priority') AS integer),
+            cast(json_extract(value, '$.queue') AS text),
+            coalesce(cast(json_extract(value, '$.scheduled_at') AS text), datetime('now', 'subsec')),
+            cast(json_extract(value, '$.state') AS text),
+            jsonb(json_extract(value, '$.tags')),
+            CASE
+              WHEN length(cast(json_extract(value, '$.unique_key') AS text)) = 0 THEN NULL
+              ELSE unhex(cast(json_extract(value, '$.unique_key') AS text))
+            END,
+            nullif(cast(json_extract(value, '$.unique_states') AS integer), 0)
+          FROM json_each(cast(? AS blob))
+          WHERE true
+          ON CONFLICT (unique_key) WHERE #{SQLITE_CONFLICT_WHERE}
+          DO UPDATE SET kind = EXCLUDED.kind
+          RETURNING #{SQLITE_JOB_COLUMNS}
+        SQL
+
+        rows = @db.fetch(sql, JSON.generate(jobs)).all
+        sqlite_notify_insert(insert_params_array)
+
+        rows.map do |row|
+          metadata = JSON.parse(row[:metadata])
+          [sqlite_to_job_row_from_raw(row), !inserted_nonces.key?(metadata[UNIQUE_INSERT_METADATA_KEY])]
+        end
+      end
+    end
+
+    private def sqlite_job_rows(suffix, *binds)
+      @db.fetch("SELECT #{SQLITE_JOB_COLUMNS} FROM river_job #{suffix}", *binds).all
+    end
+
+    private def sqlite_notify_insert(insert_params_array)
+      queues = insert_params_array
+        .select { |param| param.state == ::River::JOB_STATE_AVAILABLE }
+        .map(&:queue)
+        .uniq
+      return if queues.empty?
+
+      @db[:river_notification].multi_insert(queues.map do |queue|
+        {payload: JSON.generate({queue: queue}), topic: "river_insert"}
+      end)
+    end
+
+    private def sqlite_to_job_row_from_raw(row)
+      runtime_decode_sqlite_job(row)
+    end
+
+    private def to_job_row(river_job)
+      if @is_sqlite
+        row = sqlite_job_rows("WHERE id = ? LIMIT 1", river_job[:id]).first
+        sqlite_to_job_row_from_raw(row)
+      else
+        postgres_to_job_row(river_job)
+      end
     end
   end
 end

data/lib/riverqueue-sequel.rb

@@ -1,4 +1,7 @@
+# frozen_string_literal: true
+
 require "sequel"
+require "riverqueue"
 
 require_relative "driver"
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: riverqueue-sequel
 version: !ruby/object:Gem::Version
-  version: 0.10.1
+  version: 0.12.0
 platform: ruby
 authors:
 - Blake Gentry
@@ -11,7 +11,7 @@ cert_chain: []
 date: 1980-01-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: pg
+  name: sequel
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">"
@@ -31,27 +31,21 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '1000'
 - !ruby/object:Gem::Dependency
-  name: sequel
+  name: riverqueue
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">"
+    - - '='
       - !ruby/object:Gem::Version
-        version: '0'
-    - - "<"
-      - !ruby/object:Gem::Version
-        version: '1000'
+        version: 0.12.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">"
-      - !ruby/object:Gem::Version
-        version: '0'
-    - - "<"
+    - - '='
       - !ruby/object:Gem::Version
-        version: '1000'
-description: Sequel driver for the River Ruby gem. Use in conjunction with the riverqueue
-  gem to insert jobs that are worked in Go.
+        version: 0.12.0
+description: Sequel PostgreSQL and SQLite driver for inserting and working River jobs
+  in Ruby.
 email: brandur@brandur.org
 executables: []
 extensions: []
@@ -61,7 +55,7 @@ files:
 - lib/riverqueue-sequel.rb
 homepage: https://riverqueue.com
 licenses:
-- LGPL-3.0-or-later
+- MPL-2.0
 metadata: {}
 rdoc_options: []
 require_paths:
@@ -70,7 +64,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '0'
+      version: '3.2'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
@@ -79,5 +73,5 @@ required_rubygems_version: !ruby/object:Gem::Requirement
 requirements: []
 rubygems_version: 4.0.9
 specification_version: 4
-summary: Sequel driver for the River Ruby gem.
+summary: Sequel PostgreSQL and SQLite driver for the River Ruby gem.
 test_files: []