dispatch_policy 0.4.1 → 0.4.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 55753a0af85b649115d306ab790978668958d8a4ebe43b44c4b955a00b525b3b
-  data.tar.gz: 1f12837a2f561ff28f1fa00aedcd5cf92092766b75043a1937658964a0583d90
+  metadata.gz: 0f7f08701682539b873609e4f45432661e9d9e070e5b27593695a90b6266595e
+  data.tar.gz: c32b4f91f257b1e69e0417edc44fb3779c3f0567f4dd152d893fa781ca7d3142
 SHA512:
-  metadata.gz: 98ad50661323d62f22593bf323d61a18716c059a0571fcf8057e0f8b18bdd39dd58214cc399e38b4c96297539cfeb1f841675879733a0ef5d7913261a0c0d42d
-  data.tar.gz: e54665de99a9cb7c63f522bbefeb406958382831bf0a16e41daaa25ecd977fbdf215c12fd0b91411bed8a9981f454238007ea615767914d7046401c78ff9dc81
+  metadata.gz: 982defdd7fda9aae96d31b83666bd291bf4c36dde33e673f43322ee35cbb830c547a9ada0bbcbe142b47a00f10112d6aae24f1631a06f94e97e2e9ef71365239
+  data.tar.gz: e928eb7605905fb7de6867d4933332789ebd36e932be2f6784ed672fdcd485bf3228b659a5947b5965e7e47ee3e2c9089dd6cdd5b6069764008f682eacbacaa2

data/CHANGELOG.md

@@ -1,5 +1,42 @@
 # Changelog
 
+## 0.4.2
+
+### Fixed
+- The engine UI's **"admit"** and **"drain"** buttons now claim and
+  forward jobs inside a single transaction, matching the Tick's
+  at-least-once guarantee. They previously ran
+  `Repository.claim_staged_jobs!` (a `DELETE … RETURNING` that
+  autocommits on its own) and then a bare `Forwarder.dispatch` in a
+  separate statement — so if the forward raised (deserialize, adapter,
+  network), the staged rows were already deleted and the jobs were
+  lost. The atomic primitive now lives in
+  `DispatchPolicy::ManualAdmission.force!` and both controller actions
+  delegate to it.
+- The same UI paths now regenerate `active_job_id` per row before the
+  adapter handoff, as the Tick admission path already did in 0.4.1.
+  Without it the manual buttons could raise
+  `ActiveRecord::RecordNotUnique` against a residual adapter row from a
+  previous admission — which, combined with the missing transaction,
+  both 500'd the request and lost the staged rows.
+- Fixes a latent `NoMethodError` in the `admit` action: the old
+  `rows.size - Forwarder.dispatch(rows).size` raised when the claim
+  came back empty (`dispatch` returns `nil`) and otherwise reported a
+  misleading forwarded count.
+- Tick pass-2 budget redistribution no longer double-spends the
+  `throttle` token bucket. When `tick_admission_budget` is set, pass-2
+  re-evaluates a partition's gates against the in-memory partition
+  hash; the throttle bucket lives in `partitions.gate_state`, which was
+  persisted to the DB in pass-1 but never mirrored back in memory. So
+  pass-2 re-read the pre-pass-1 token count, admitted again from a full
+  bucket (above the configured rate), and persisted a patch computed
+  off the stale base — silently dropping pass-1's consumption, so the
+  effective rate drifted upward tick after tick. The committed
+  `gate_state` patch is now shallow-merged back onto the in-memory
+  partition after each admit. (`concurrency` / `adaptive_concurrency`
+  were unaffected — they re-read their counts from the DB on every
+  evaluate.)
+
 ## 0.4.1
 
 ### Fixed
@@ -26,6 +63,30 @@
   never drain again. -700 still yields a finite ~9.86e-305, which is
   effectively zero for the EWMA.
 
+## 0.4.0
+
+### Added
+- Admin UI dark mode with an auto / light / dark selector in the
+  header that persists across pages.
+- Conceptual logo (chevrons + gate + admitted dot across 3
+  partitions), surfaced in the admin header and README. Theme-aware
+  lockup keeps the wordmark readable on both light and dark
+  backgrounds.
+- Vendored Turbo, served by the engine, so the admin UI no longer
+  depends on the host app shipping Turbo itself.
+- `screenshots` Rake task that regenerates all README screenshots
+  from the dummy app instead of importing static assets.
+
+### Changed
+- Slimmer brand bundle in the gem: only the two masters actually
+  used are shipped; the full art set stays in `arts/` for source.
+- README install section drops the obsolete v2-branch banner — the
+  gem now installs cleanly from RubyGems.
+
+### Fixed
+- Dummy app opts into the gem's `db/migrate` path so its setup
+  picks up new tables without manual copying.
+
 ## 0.3.0
 
 ### Added

data/app/controllers/dispatch_policy/partitions_controller.rb

@@ -67,15 +67,12 @@ module DispatchPolicy
     end
 
     def admit
-      count = Integer(params[:count] || 1)
-      rows = Repository.claim_staged_jobs!(
-        policy_name:      @partition.policy_name,
-        partition_key:    @partition.partition_key,
-        limit:            count,
-        gate_state_patch: {},
-        retry_after:      nil
+      count     = Integer(params[:count] || 1)
+      forwarded = ManualAdmission.force!(
+        policy_name:   @partition.policy_name,
+        partition_key: @partition.partition_key,
+        limit:         count
       )
-      forwarded = rows.size - Forwarder.dispatch(rows).size
       redirect_to partition_path(@partition), notice: "Forwarded #{forwarded} job(s)."
     end
 
@@ -97,17 +94,14 @@ module DispatchPolicy
       drained = 0
       while drained < DRAIN_MAX_PER_REQUEST
         batch_limit = [DRAIN_BATCH_SIZE, DRAIN_MAX_PER_REQUEST - drained].min
-        rows = Repository.claim_staged_jobs!(
-          policy_name:      partition.policy_name,
-          partition_key:    partition.partition_key,
-          limit:            batch_limit,
-          gate_state_patch: {},
-          retry_after:      nil
+        forwarded   = ManualAdmission.force!(
+          policy_name:   partition.policy_name,
+          partition_key: partition.partition_key,
+          limit:         batch_limit
         )
-        break if rows.empty?
+        break if forwarded.zero?
 
-        Forwarder.dispatch(rows)
-        drained += rows.size
+        drained += forwarded
       end
       remaining = partition.class.where(id: partition.id).pick(:pending_count) || 0
       [drained, remaining]

data/lib/dispatch_policy/manual_admission.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+require "securerandom"
+
+module DispatchPolicy
+  # Force-admit staged jobs for a partition, bypassing every gate. Backs
+  # the engine UI's "admit" and "drain" buttons.
+  #
+  # Mirrors the atomicity guarantee of Tick#admit_partition: the DELETE
+  # from staged_jobs (via Repository.claim_staged_jobs!) and the adapter
+  # enqueue (Forwarder.dispatch) run in a SINGLE transaction, so any
+  # failure — deserialize, adapter, network — rolls the claim back and the
+  # staged rows survive. Without this the UI buttons would DELETE staged
+  # rows and then lose them whenever the forward raised, breaking the
+  # gem's at-least-once contract on a path the Tick already protects.
+  #
+  # active_job_id is regenerated per row for the same reason Tick does it
+  # (see Tick#admit_partition): adapters that key their jobs table on
+  # active_job_id (good_job, solid_queue) raise RecordNotUnique against a
+  # residual row from a previous admission, which would abort the TX.
+  module ManualAdmission
+    module_function
+
+    # Force-admit up to `limit` staged jobs for the partition, bypassing
+    # all gates, atomically. Returns the number of jobs forwarded.
+    def force!(policy_name:, partition_key:, limit:)
+      return 0 unless limit.positive?
+
+      forwarded = 0
+      Repository.with_connection do
+        ActiveRecord::Base.transaction(requires_new: true) do
+          rows = Repository.claim_staged_jobs!(
+            policy_name:      policy_name,
+            partition_key:    partition_key,
+            limit:            limit,
+            gate_state_patch: {},
+            retry_after:      nil
+          )
+          next if rows.empty?
+
+          rows.each { |row| row["job_data"]["job_id"] = SecureRandom.uuid }
+          Forwarder.dispatch(rows)
+          forwarded = rows.size
+        end
+      end
+      forwarded
+    end
+  end
+end

data/lib/dispatch_policy/tick.rb

@@ -286,6 +286,20 @@ module DispatchPolicy
         end
       end
 
+      # Reflect the just-committed gate_state patch on the in-memory
+      # partition. `record_partition_admit!` persisted `gate_state || patch`
+      # inside the TX above; without mirroring it here, a pass-2 re-admit
+      # (Tick budget redistribution) would re-evaluate the gates against
+      # the STALE pre-pass-1 snapshot. For the throttle that means reading
+      # the token bucket at its original level and double-spending —
+      # admitting above the configured rate and overwriting pass-1's
+      # consumption. The shallow merge matches Postgres jsonb `||`.
+      # Only runs on a committed admit: if the TX raised we fall through to
+      # the rescue below and never touch the in-memory state.
+      if result.gate_state_patch&.any?
+        partition["gate_state"] = (partition["gate_state"] || {}).merge(result.gate_state_patch)
+      end
+
       if admitted.zero?
         { admitted: 0, failures: 0, reasons: ["no_rows_claimed"] }
       else

data/lib/dispatch_policy/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module DispatchPolicy
-  VERSION = "0.4.1"
+  VERSION = "0.4.2"
 end

data/lib/dispatch_policy.rb

@@ -21,6 +21,7 @@ require_relative "dispatch_policy/cursor_pagination"
 require_relative "dispatch_policy/pipeline"
 require_relative "dispatch_policy/repository"
 require_relative "dispatch_policy/forwarder"
+require_relative "dispatch_policy/manual_admission"
 require_relative "dispatch_policy/inflight_tracker"
 require_relative "dispatch_policy/tick"
 require_relative "dispatch_policy/tick_loop"

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: dispatch_policy
 version: !ruby/object:Gem::Version
-  version: 0.4.1
+  version: 0.4.2
 platform: ruby
 authors:
 - José Galisteo
@@ -233,6 +233,7 @@ files:
 - lib/dispatch_policy/gates/throttle.rb
 - lib/dispatch_policy/inflight_tracker.rb
 - lib/dispatch_policy/job_extension.rb
+- lib/dispatch_policy/manual_admission.rb
 - lib/dispatch_policy/operator_hints.rb
 - lib/dispatch_policy/pipeline.rb
 - lib/dispatch_policy/policy.rb