workhorse 1.5.2 → 2.0.0.rc1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9ffbb2c24919d7b745fd87bd6a8675fdae5448a6c0e11c3af486d34baf5a9343
-  data.tar.gz: 9cf2270f0c3a65d244d679baf56367033e314e4a8b7ed11cc17f4d07e4a6fe63
+  metadata.gz: 84d0f28bfeb67292f3de373115ada4ef7c2cff27b29e847af29326733ba89841
+  data.tar.gz: f69f1d82c2c9c15f47f87d8f8390c31e655596636383b9cadefc8c240d8f43b6
 SHA512:
-  metadata.gz: 22b02451a64ae383b75616b9556cc05927f479f47db7d7993347df0f520d36ebfda8c7f3f4de49574c84499dfd603285bab83295df72c6100b0f6f438730d576
-  data.tar.gz: '0549ab8a2e2683dd13b60bc85a5d2d85a260dee938a13714c45fe1233af217a433af2e71f899a7056c3afd8f503f6f94e1a5b9cee5e97bbf5fb93fed9ae75a03'
+  metadata.gz: 00c490f9e4aa2b57066b8b974e38286509cb156cb27f79dc7d7a355c9797338eaa8dcf33a5dc9c07e2f3bbe37878d0be5e82b7f098da9e45194ad0fe5b064d6f
+  data.tar.gz: e0116907c47ae31d9fa5a5626e331a503286667966aa974255e8d9f816c363fe3a9e55406e5f79c9a3da7e82abe4b0648b505de3e279992a58e1864d847edaef

data/.github/workflows/ruby.yml

@@ -40,10 +40,146 @@ jobs:
       - name: Run rake tests
         uses: nick-fields/retry@v2
         with:
-          timeout_seconds: 120
+          # Not much above what one run takes on a loaded runner.
+          timeout_seconds: 300
           retry_on: any
           max_attempts: 3
           command: bundle exec rake test TESTOPTS='--verbose'
+          # A timed-out attempt can leave the suite and its daemon workers
+          # running - the workers call setsid, so killing the attempt does not
+          # reach them - and the next attempt then runs against them. KILL, as
+          # a worker traps TERM. Clear their pidfiles as well.
+          on_retry_command: |
+            pkill -KILL -f "^Workhorse .*: $GITHUB_WORKSPACE" || true
+            pkill -KILL -f "[r]ake_test_loader" || true
+            sleep 1
+            rm -f tmp/pids/*
+
+  test-oracle:
+    runs-on: ubuntu-latest
+
+    # Oracle gets a job of its own rather than a matrix entry: it needs a
+    # service container and the Instant Client, and it is run against a single
+    # Ruby version, as what is being covered is the database dialect rather
+    # than the interpreter.
+    services:
+      oracle:
+        image: gvenzl/oracle-free:23-slim
+        env:
+          ORACLE_PASSWORD: oracle
+          APP_USER: workhorse
+          APP_USER_PASSWORD: workhorse
+        ports:
+          - 1521:1521
+        options: >-
+          --health-cmd healthcheck.sh
+          --health-interval 20s
+          --health-timeout 10s
+          --health-retries 30
+
+    env:
+      DB_ADAPTER: oracle_enhanced
+      DB_NAME: FREEPDB1
+      DB_USERNAME: workhorse
+      DB_PASSWORD: workhorse
+      DB_HOST: localhost
+      DB_PORT: 1521
+      LD_LIBRARY_PATH: /opt/oracle/instantclient
+
+    steps:
+      - uses: actions/checkout@v2
+
+      # ruby-oci8 compiles against the Instant Client, so both the runtime
+      # library and the SDK headers are needed; basiclite alone does not carry
+      # oci.h.
+      - name: Install Oracle Instant Client
+        run: |
+          sudo apt-get update
+          # The package was renamed in Ubuntu 24.04, which `ubuntu-latest` now
+          # is; keep working on both.
+          sudo apt-get install -y unzip
+          sudo apt-get install -y libaio1 || sudo apt-get install -y libaio1t64
+          # The rename came with a soname change to libaio.so.1t64, while the
+          # Oracle client still links against libaio.so.1. Without this the
+          # client library cannot be linked, and ruby-oci8's version probe
+          # fails with a TypeError that says nothing about the cause.
+          if ! ldconfig -p | grep -qE 'libaio\.so\.1$|libaio\.so\.1 '; then
+            t64=$(ls /usr/lib/*/libaio.so.1t64 2>/dev/null | head -1)
+            if [ -n "$t64" ]; then sudo ln -sfn "$t64" "$(dirname "$t64")/libaio.so.1"; fi
+          fi
+          sudo mkdir -p /opt/oracle
+          cd /tmp
+          # Pinned to 21.13 rather than taking the current release: ruby-oci8
+          # 2.2 fails its `OCI_MAJOR_VERSION` probe against an Instant Client
+          # 23 header, after finding the library and the SDK correctly. A 21
+          # client talks to the 23 server used below.
+          base=https://download.oracle.com/otn_software/linux/instantclient/2113000
+          curl -sSLO $base/instantclient-basiclite-linux.x64-21.13.0.0.0dbru.zip
+          curl -sSLO $base/instantclient-sdk-linux.x64-21.13.0.0.0dbru.zip
+          sudo unzip -oq 'instantclient-basiclite-linux.x64-*.zip' -d /opt/oracle
+          sudo unzip -oq 'instantclient-sdk-linux.x64-*.zip' -d /opt/oracle
+          # The archives unpack into a versioned directory, whose name changes
+          # with every release; the fixed symlink is what LD_LIBRARY_PATH and
+          # the ruby-oci8 build both point at.
+          sudo ln -sfn "$(find /opt/oracle -maxdepth 1 -type d -name 'instantclient_*' | head -1)" /opt/oracle/instantclient
+          # The basiclite package ships only the versioned libraries, and
+          # linking a conftest against -lclntsh needs the unversioned name.
+          cd /opt/oracle/instantclient
+          for lib in libclntsh libocci; do
+            versioned=$(ls $lib.so.* 2>/dev/null | head -1)
+            if [ -n "$versioned" ]; then sudo ln -sfn "$versioned" "$lib.so"; fi
+          done
+          echo /opt/oracle/instantclient | sudo tee /etc/ld.so.conf.d/oracle-instantclient.conf
+          sudo ldconfig
+
+      - name: Set up Ruby
+        uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: '3.2.1'
+
+      - name: Install bundle including the oracle group
+        run: |
+          bundle config set --local with oracle
+          bundle install --jobs 4 --retry 3
+
+      # extconf.rb only says to consult mkmf.log, which is otherwise lost with
+      # the runner.
+      - name: Show the native extension build log
+        if: failure()
+        run: find / -name mkmf.log -path '*ruby-oci8*' -exec cat {} +
+
+      # The global lock is DBMS_LOCK on Oracle, and execute permission on that
+      # package is not granted to a new schema by default. SYS owns the
+      # package, and only it can grant it, so this connects as SYSDBA.
+      - name: Grant DBMS_LOCK to the test schema
+        run: |
+          bundle exec ruby -e '
+            require "active_record"
+            require "active_record/connection_adapters/oracle_enhanced_adapter"
+            ActiveRecord::Base.establish_connection(
+              adapter: "oracle_enhanced", database: "FREEPDB1",
+              username: "sys", password: "oracle", privilege: "SYSDBA",
+              host: "localhost", port: 1521
+            )
+            ActiveRecord::Base.connection.execute("GRANT EXECUTE ON DBMS_LOCK TO workhorse")
+          '
+
+      - name: Run rake tests
+        uses: nick-fields/retry@v2
+        with:
+          timeout_seconds: 600
+          retry_on: any
+          max_attempts: 3
+          command: bundle exec rake test TESTOPTS='--verbose'
+          # A timed-out attempt can leave the suite and its daemon workers
+          # running - the workers call setsid, so killing the attempt does not
+          # reach them - and the next attempt then runs against them. KILL, as
+          # a worker traps TERM. Clear their pidfiles as well.
+          on_retry_command: |
+            pkill -KILL -f "^Workhorse .*: $GITHUB_WORKSPACE" || true
+            pkill -KILL -f "[r]ake_test_loader" || true
+            sleep 1
+            rm -f tmp/pids/*
 
   linters:
     runs-on: ubuntu-latest

data/CHANGELOG.md

@@ -1,5 +1,155 @@
 # Workhorse Changelog
 
+## 2.0.0.rc1 - 2026-09-30
+
+Sitrox reference: #154443.
+
+### Added
+
+* Oracle 12c or later is supported again, and is now covered by CI against
+  `activerecord-oracle_enhanced-adapter` rather than only by hand. The global
+  lock uses `DBMS_LOCK`, and the generated migrations leave out the index
+  prefix lengths Oracle rejects while naming every index within the 30
+  characters it allows before 12.2. See
+  [Database support](README.md#database-support).
+
+  2.0.0.rc0 dropped Oracle; anyone who took that release up and needs it can
+  move straight to this one. Grant the schema execute permission on
+  `DBMS_LOCK`, as described under Database support.
+
+### Fixed
+
+* On Oracle, jobs are picked up in priority order. 1.x limited the rows it
+  selected by filtering on `ROWNUM`, which Oracle numbers before sorting, so
+  both the job taken from each queue and the jobs a poll picked up were an
+  arbitrary subset regardless of `priority`. Limiting now uses
+  `FETCH FIRST … ROWS ONLY`, which applies after the sort and is why 12c is
+  now the minimum.
+
+* A poll that finds the global lock taken now waits for it with a polling
+  interval below half a second. The lock timeout follows the interval, and
+  MySQL and Oracle take it as whole seconds, so they rounded it down to not
+  waiting at all - only MariaDB honours a fraction. Each such poll gave up
+  at once and counted towards `Workhorse.max_global_lock_fails`, so several
+  workers contending for the lock could trigger its alarm about a crashed
+  worker. The timeout is now rounded up to whole seconds.
+
+* A soft restart (`USR1`) no longer wedges a worker whose `tmp/pids` does not
+  exist yet, as on a fresh checkout or deployment. The worker touched its
+  shutdown file there after it had stopped accepting jobs, so failing on it
+  left a worker that took no jobs and never exited. The directory is now
+  created when missing, and a shutdown file that still cannot be written no
+  longer holds up the shutdown. The heartbeat and the memory-limit shutdown,
+  which write to the same directory, create it as well.
+
+* `Workhorse.clean_stuck_jobs` works on Oracle. It is built on splitting
+  `locked_by` into host, PID and random component, which was written with
+  MySQL's `substring_index` and so raised on Oracle for as long as the option
+  has existed — visible only to an installation that turned it on, as it is
+  off by default.
+
+## 2.0.0.rc0 - 2026-09-29
+
+Sitrox reference: #154443.
+
+### Breaking changes
+
+* **Support for Oracle is dropped.** Workhorse supports MySQL and MariaDB
+  only. The Oracle branches of the global lock, the row limiting and the
+  generated migrations are gone, along with
+  `Workhorse::Poller::ORACLE_LOCK_MODE` and `ORACLE_LOCK_HANDLE`. It was never
+  covered by CI, so it only ever had manual verification. An Oracle
+  installation has no upgrade path and should stay on 1.x. See
+  [Database support](README.md#database-support).
+
+* A forked daemon worker no longer runs the `at_exit` handlers registered by
+  the process that started it. They belong to that process, and one that waits
+  on threads the fork did not inherit hangs a worker which has already
+  finished - which then ignores `TERM`. This skips interpreter finalisation as
+  a whole, so buffered output is dropped too; a worker that dies of an
+  unhandled exception now reports it through `Workhorse.on_exception` and
+  exits non-zero.
+
+### Added
+
+* *Scheduling*: workhorse runs jobs on a cron schedule itself, without an
+  external scheduler process. Each schedule owns a row in the new
+  `workhorse_schedules` table holding the occurrence it is waiting for, so an
+  occurrence whose time passes while nothing is running is not lost, and what
+  happens to it is a per-schedule `catch_up` policy. Timezones and daylight
+  saving are handled. See [Scheduling](README.md#scheduling).
+
+* *Notifications*: enqueuing a job announces it and a waiting worker polls
+  straight away, rather than sleeping out its polling interval. Polling stays
+  the floor. `:file` suits workers sharing a filesystem with the application
+  and `:redis` those that do not; both are off by default. See
+  [Notifications](README.md#notifications).
+
+* Job deadlines and lateness reporting: `expires_at`, `max_lateness`, the new
+  `expired` state, `Workhorse.on_job_expired`, `Workhorse.on_job_late` and
+  `Workhorse::DbJob#lateness`. A job past its deadline is expired rather than
+  performed. See [Lateness and deadlines](README.md#lateness-and-deadlines).
+
+* `Workhorse::Jobs::DetectLateSchedulesJob`, which reports schedules whose
+  next occurrence lies well in the past. Neither callback above can fire for a
+  job that was never created, so this is what catches materialization having
+  stopped. See
+  [Detecting schedules that stopped](README.md#detecting-schedules-that-stopped).
+
+* `Workhorse.shutdown_timeout`, the seconds the daemon's `stop` waits for a
+  worker before killing it. Defaults to 300, `nil` restores the previous
+  behaviour. A worker that ignores `TERM` used to leave `stop` - and whatever
+  waits on it, usually a deployment - looping forever.
+
+* `Workhorse.enqueue_job_class`, the keyword arguments `expires_at:` and
+  `max_lateness:` on `Workhorse.enqueue` and `Workhorse.enqueue_active_job`,
+  `priority:` on the latter, and the scope `Workhorse::DbJob.expired`.
+
+* The composite indexes `[state, perform_at]`, `[state, priority, created_at]`
+  and `[state, expires_at]` in the generated `jobs` migration, replacing the
+  single-column index on `state`. `rails generate workhorse:install` now emits
+  two migrations rather than one.
+
+* `fugit` as a runtime dependency, for parsing cron expressions.
+
+### Changed
+
+* `Workhorse::Jobs::CleanupSucceededJobs` also deletes jobs in the new
+  `expired` state, with a `states` argument to opt out. A schedule using
+  `expires_after` that regularly misses its window would otherwise grow the
+  jobs table without bound.
+
+* Failures to obtain the global lock on a poll that a notification or an
+  instant repoll brought forward no longer count towards
+  `max_global_lock_fails`, and are logged at `debug`. Several workers woken by
+  one announcement race for the lock and all but one lose, which says nothing
+  about a crashed worker.
+
+* `Workhorse::DbJob#reset!` accepts `expired` as the terminal state it is.
+
+### Fixed
+
+* A deadlock between shutting a worker down and the poller posting a job.
+  `Worker#shutdown` held the worker's mutex while waiting for the poller
+  thread, which could be waiting for that same mutex in `Worker#perform`. The
+  worker then ignored `TERM`. A job that was locked but cannot be performed
+  because the worker is shutting down is now reset to `waiting` rather than
+  left locked, where it would have blocked its queue.
+
+* `Worker#shutdown` raising when called concurrently, which the daemon does by
+  sending both `TERM` and `INT`.
+
+### Documentation
+
+* PostgreSQL is documented as unsupported. Workers emit `GET_LOCK` on every
+  poll, which PostgreSQL does not provide, so a worker fails on its first one.
+  The requirements previously listed it as supported, which it never was.
+
+### Upgrading
+
+Nothing breaks without migrating, but the new features need one. See
+[Upgrading from 1.x](README.md#upgrading-from-1x).
+
 ## 1.5.2 - 2026-08-04
 
 * Fix `Poller#valid_queues` raising `NoMethodError` on the Oracle adapter. The

data/Gemfile

@@ -11,5 +11,20 @@ gem 'minitest'
 gem 'mysql2'
 gem 'pry'
 gem 'rake'
-gem 'rubocop', '~> 1.60'
+
+# Pinned to a patch range rather than given a floor: Gemfile.lock is not
+# checked in, so CI resolves the newest version matching this line while a
+# checkout keeps whatever it installed. With a floor, every rubocop release
+# that adds a cop turns the build red for a reason nobody can reproduce
+# locally.
+gem 'rubocop', '~> 1.91.0'
 gem 'trilogy'
+
+# Only needed to run the suite against Oracle, which additionally requires the
+# Oracle Instant Client to be installed - building ruby-oci8 fails without it.
+# The group is optional, so a plain `bundle install` skips it; enable it with
+# `bundle config set --local with oracle`.
+group :oracle, optional: true do
+  gem 'activerecord-oracle_enhanced-adapter', '~> 7.1'
+  gem 'ruby-oci8'
+end