trellis-pg 0.1.0.pre.alpha → 0.1.0.pre.beta

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f1baafc626d091c77635ab5de6f1bb321200d27b3ce659465f2b69cf34e948ab
-  data.tar.gz: c9c7d5f91121f749391c06e331732a375e98f149ef7ca2259aa2e965640ee6e2
+  metadata.gz: 040d3074f1223a81e28989c5d0a2b7825e84806ea0b088894fc1ea0daaf68312
+  data.tar.gz: e5bbfe7ce9446bd67d7b01666e555255a6ab39687c2c7415080789ddb8a934ac
 SHA512:
-  metadata.gz: 685a0b5d491c23b32a9ed8784153d68ae60191f2901633030e74cd5da8f6fe19dfb3f09c1fdbfbf1d425e3a7023845f63726d7c37079f78d97086694de11e531
-  data.tar.gz: c520497842ef2b4ecd252b3a307ea33c5a01b0fdd818f3f74561141c8b28408abb7288b50971383099979d6e074451febbcdfde047e6f3473f3e5b20eb31fb48
+  metadata.gz: 0f56d84b4f68c1619f480040e659513872e4c6d9f93358eff375b400b7e5b22eb9c2c29080c5433cc24ae9ab33cb8467f735c151e36c6634fea094fa2e04a5de
+  data.tar.gz: cc1c7b5b6ed393b91c6ed3a866a0172ddb085b297e4c7914ec4a3de56f32163734449d68a2d28e40192bc147c3b797cc16222ff4e7d2b943fbb97a84ac498847

data/README.md

@@ -144,10 +144,33 @@ Trellis.shutdown
 | `staging:` | `false` | run the staging worker (installs change capture, starts backfills) here |
 | `drain_threads:` | `0` | threads applying staged changes to the targets |
 | `worker_threads:` | `2` | the threads of each Rust runtime a handle owns (its calls', and its background client's), invisible to Ruby's own sizing |
-
-The defaults run nothing in the background. Exactly one process in a fleet
-should set `staging: true`, and some process must run drain threads, or no
-definition ever reaches `:live`.
+| `max_call_pool_size:` | `20` | the most connections the handle's call pool opens; the background pool is sized by `staging:` and `drain_threads:` |
+| `call_pool_wait_timeout_ms:` | `30_000` | how long a call waits for a free call-pool connection before a `ConnectivityError` |
+
+The defaults run nothing in the background. Some process in a fleet must set
+`staging: true` and some process must run drain threads, or no definition ever
+reaches `:live`. Any number of processes may set `staging: true`: one takes the
+producer lock and leads, the rest stand by and take over when it goes away, so a
+rolling deploy needs no operator step. Under a preforking Puma with
+`TRELLIS_WORKER=1`, every Puma worker connects with `staging: true`, so all but
+one stand by; that is harmless, and each polls the lock every 5 seconds (see
+[Rolling deploys](../../docs/embedding.md#rolling-deploys)).
+
+### Resources
+
+At the defaults, a handle holds up to 20 Postgres connections (its call pool,
+opened as calls need them and closed after 60 seconds idle) and 3 threads. A
+worker handle (`staging: true, drain_threads: N`) also owns a background pool
+of up to `2N + 5` connections (`2N + 4` without staging, `1` with staging
+alone), closed after 60 seconds idle, and two connections that stay open: the
+staging session and one `LISTEN` shared by its drain threads. It holds 6
+threads: drain threads are tasks on its runtime, not OS threads. Its memory is
+bounded by row counts, not bytes: about `N` × 100,000 changed rows' images at
+its peak. `max_call_pool_size:` lowers the 20 (the background pool is sized by
+the handle's roles, not by an option), and `worker_threads:` sets the threads.
+Every handle (each `Trellis::Instance` too) costs this again. The full
+accounting is in
+[`docs/embedding.md`](../../docs/embedding.md#what-a-handle-uses).
 
 ### Several instances
 

data/ext/trellis_ruby/Cargo.lock

@@ -1729,9 +1729,10 @@ dependencies = [
 
 [[package]]
 name = "trellis"
-version = "0.1.0-alpha"
-source = "git+https://github.com/salesforce-misc/trellis?rev=31985aab1f7ddb22513973b0e85882c3eca9583f#31985aab1f7ddb22513973b0e85882c3eca9583f"
+version = "0.1.0-beta"
+source = "git+https://github.com/salesforce-misc/trellis?rev=1820fd7db091e540990d1efe190a687fb4dfeb59#1820fd7db091e540990d1efe190a687fb4dfeb59"
 dependencies = [
+ "deadpool",
  "deadpool-postgres",
  "getrandom 0.4.3",
  "metrics",
@@ -1751,8 +1752,8 @@ dependencies = [
 
 [[package]]
 name = "trellis-embed"
-version = "0.1.0-alpha"
-source = "git+https://github.com/salesforce-misc/trellis?rev=31985aab1f7ddb22513973b0e85882c3eca9583f#31985aab1f7ddb22513973b0e85882c3eca9583f"
+version = "0.1.0-beta"
+source = "git+https://github.com/salesforce-misc/trellis?rev=1820fd7db091e540990d1efe190a687fb4dfeb59#1820fd7db091e540990d1efe190a687fb4dfeb59"
 dependencies = [
  "tracing",
  "trellis",
@@ -1760,7 +1761,7 @@ dependencies = [
 
 [[package]]
 name = "trellis_ruby"
-version = "0.1.0-alpha"
+version = "0.1.0-beta"
 dependencies = [
  "magnus",
  "rb-sys",

data/ext/trellis_ruby/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "trellis_ruby"
-version = "0.1.0-alpha"
+version = "0.1.0-beta"
 edition = "2024"
 publish = false
 
@@ -28,8 +28,8 @@ doctest = false
 ruby = ["dep:magnus", "dep:rb-sys"]
 
 [dependencies]
-trellis = { git = "https://github.com/salesforce-misc/trellis", rev = "31985aab1f7ddb22513973b0e85882c3eca9583f" }
-trellis-embed = { git = "https://github.com/salesforce-misc/trellis", rev = "31985aab1f7ddb22513973b0e85882c3eca9583f" }
+trellis = { git = "https://github.com/salesforce-misc/trellis", rev = "1820fd7db091e540990d1efe190a687fb4dfeb59" }
+trellis-embed = { git = "https://github.com/salesforce-misc/trellis", rev = "1820fd7db091e540990d1efe190a687fb4dfeb59" }
 magnus = { version = "0.9", optional = true, default-features = false, features = ["rb-sys"] }
 rb-sys = { version = "0.9", optional = true }
 

data/ext/trellis_ruby/src/engine.rs

@@ -1,4 +1,4 @@
-//! Knowing whether a forked child may connect (issue #600).
+//! Knowing whether a forked child may connect.
 //!
 //! `fork()` copies the whole address space but only the calling thread. The
 //! engine keeps process-wide state behind locks (the enum type-name

data/ext/trellis_ruby/src/lib.rs

@@ -24,7 +24,7 @@
 //!   and raises `Trellis::ForkedHandleError`, and a forked copy is never
 //!   dropped either (see [`Handle`]'s `Drop`).
 //! - **A process forked while its parent had an engine running can't
-//!   connect one of its own** (issue #600): it may have inherited a lock one
+//!   connect one of its own**: it may have inherited a lock one
 //!   of the parent's engine threads held, which nothing in it can release.
 //!   [`connect`] raises `Trellis::ForkedHandleError` instead of risking the
 //!   hang. See the [`engine`] module.
@@ -331,7 +331,7 @@ impl Handle {
     }
 
     /// Pauses now the readers of every table whose capture was changed
-    /// outside Trellis (#1181), and returns the tables.
+    /// outside Trellis, and returns the tables.
     fn detect_capture_disruptions(ruby: &Ruby, rb_self: &Self) -> Result<Vec<String>, Error> {
         rb_self.call(ruby, BlockingTrellis::start_detect_capture_disruptions)
     }
@@ -388,18 +388,21 @@ impl Handle {
     }
 
     /// The configuration the handle connected with, all but the connection
-    /// string (see [`PlainConfig`]).
+    /// string, and its call pool's limits (see [`PlainConfig`]).
     fn config(ruby: &Ruby, rb_self: &Self) -> Result<RHash, Error> {
-        let config = rb_self.with_engine(ruby, |trellis| PlainConfig::from(trellis.config()))?;
+        let config = rb_self.with_engine(ruby, |trellis| PlainConfig::from(trellis))?;
         record(
             ruby,
             [
                 ("schema", ruby.into_value(config.schema)),
                 ("target_schema", ruby.into_value(config.target_schema)),
-                ("pool_max_size", ruby.into_value(config.pool_max_size)),
                 (
-                    "pool_wait_timeout_ms",
-                    ruby.into_value(config.pool_wait_timeout_ms),
+                    "max_call_pool_size",
+                    ruby.into_value(config.max_call_pool_size),
+                ),
+                (
+                    "call_pool_wait_timeout_ms",
+                    ruby.into_value(config.call_pool_wait_timeout_ms),
                 ),
             ],
         )
@@ -432,7 +435,11 @@ impl Handle {
 /// `lib/trellis/pg.rb`'s to document, and nothing falls back to the environment.
 ///
 /// Raises `Trellis::ForkedHandleError`, before starting anything, in a
-/// process forked while its parent had an engine running (issue #600).
+/// process forked while its parent had an engine running.
+#[allow(
+    clippy::too_many_arguments,
+    reason = "`function!` binds each Ruby argument to its own parameter"
+)]
 fn connect(
     ruby: &Ruby,
     url: String,
@@ -441,6 +448,8 @@ fn connect(
     staging: bool,
     drain_threads: usize,
     worker_threads: usize,
+    max_call_pool_size: usize,
+    call_pool_wait_timeout_ms: u64,
 ) -> Result<Handle, Error> {
     engine::check().map_err(|err| forked_handle_error(ruby, forked_while_running(&err)))?;
     // Connecting can wait on the database (a staging worker installs its
@@ -460,6 +469,8 @@ fn connect(
                     staging,
                     drain_threads,
                     worker_threads: Some(worker_threads),
+                    max_call_pool_size,
+                    call_pool_wait_timeout: Duration::from_millis(call_pool_wait_timeout_ms),
                     ..Default::default()
                 };
                 // Can't fail after `check` passed (a process's answer never
@@ -1229,7 +1240,7 @@ fn init(ruby: &Ruby) -> Result<(), Error> {
     }
 
     let native = ruby.define_module("Trellis")?.define_module("Native")?;
-    native.define_module_function("connect", function!(connect, 6))?;
+    native.define_module_function("connect", function!(connect, 8))?;
     native.define_module_function("error_codes", function!(error_codes, 0))?;
     native.define_module_function("status_names", function!(status_names, 0))?;
     native.define_module_function("quarantine_states", function!(quarantine_states, 0))?;

data/lib/trellis/error.rb

@@ -82,7 +82,7 @@ module Trellis
   #
   # Also Trellis.connect's error in a process forked while its parent had a
   # handle running, which may have inherited a lock that handle's threads
-  # held (issue #600): shut down before forking (Puma's before_fork).
+  # held: shut down before forking (Puma's before_fork).
   class ForkedHandleError < ValidationError
   end
 

data/lib/trellis/instance.rb

@@ -98,7 +98,7 @@ module Trellis
   # forked child's calls on an instance it inherited raise
   # Trellis::ForkedHandleError, and its shutdown leaves the parent's handle
   # alone and does nothing. So does Instance.connect in a child forked while
-  # any instance of its parent was running (issue #600).
+  # any instance of its parent was running.
   class Instance
     extend Checks
     include Checks
@@ -111,7 +111,7 @@ module Trellis
       # Connects a new instance and returns it. Raises ForkedHandleError in
       # a process forked while its parent had an instance running
       # (connected, connecting, or not yet fully shut down): it may have
-      # inherited a lock one of that instance's threads held (issue #600).
+      # inherited a lock one of that instance's threads held.
       #
       # - url: where the database is, as a libpq connection string
       #   ("host=... dbname=...") or URL. Nothing is read from the environment.
@@ -121,8 +121,10 @@ module Trellis
       # - target_schema: the schema a bare target table name is created in.
       # - staging: whether this process runs the instance's staging worker,
       #   which installs change capture on the source tables and starts each
-      #   new transform's backfill. Exactly one process in a fleet should,
-      #   per instance.
+      #   new transform's backfill. Any number of processes may set it: one
+      #   takes the instance's producer lock and leads, the rest stand by and
+      #   take over when it goes away, so a rolling deploy needs no
+      #   operator step.
       # - drain_threads: how many threads apply staged changes to the targets.
       #   Some process must run at least one per instance, or no definition
       #   reaches :live.
@@ -130,13 +132,20 @@ module Trellis
       #   instance owns (its calls', and its background client's), invisible
       #   to Ruby's own thread sizing, so kept small and explicit. Each
       #   instance has its own; nothing is shared between instances.
+      # - max_call_pool_size: the most connections this instance's call pool
+      #   (its API calls) opens at once. Connections open as calls need them
+      #   and close after 60 seconds idle. The background client's pool is
+      #   sized by staging and drain_threads, not by this.
+      # - call_pool_wait_timeout_ms: how long a call waits for a free call-pool
+      #   connection before failing with a ConnectivityError.
       #
       # The defaults (staging: false, drain_threads: 0) run nothing in the
       # background, which is what a migration step or a console wants. The
       # staging worker reads Trellis's tables as it starts, so a staging: true
       # connect to an unmigrated schema fails: migrate first.
       def connect(url:, schema: "trellis", target_schema: "public", staging: false,
-                  drain_threads: 0, worker_threads: 2)
+                  drain_threads: 0, worker_threads: 2, max_call_pool_size: 20,
+                  call_pool_wait_timeout_ms: 30_000)
         string_option!(:url, url)
         string_option!(:schema, schema)
         string_option!(:target_schema, target_schema)
@@ -152,7 +161,19 @@ module Trellis
                 "worker_threads must be a positive Integer, got: #{worker_threads.inspect}"
         end
 
-        handle = Native.connect(url, schema, target_schema, staging, drain_threads, worker_threads)
+        unless max_call_pool_size.is_a?(Integer) && max_call_pool_size >= 1
+          raise ValidationError,
+                "max_call_pool_size must be a positive Integer, got: #{max_call_pool_size.inspect}"
+        end
+        unless call_pool_wait_timeout_ms.is_a?(Integer) &&
+               call_pool_wait_timeout_ms.between?(0, MAX_TIMEOUT_MS)
+          raise ValidationError,
+                "call_pool_wait_timeout_ms must be a non-negative Integer, got: " \
+                "#{call_pool_wait_timeout_ms.inspect}"
+        end
+
+        handle = Native.connect(url, schema, target_schema, staging, drain_threads, worker_threads,
+                                max_call_pool_size, call_pool_wait_timeout_ms)
         instance = new(handle, schema)
         register(instance)
         instance

data/lib/trellis/pg.rb

@@ -71,7 +71,7 @@ require_relative "instance"
 # calls on a handle it inherited raise Trellis::ForkedHandleError, except
 # Trellis.shutdown, which leaves the parent's handle alone and does nothing.
 # So does Trellis.connect in a child forked while its parent's handle was
-# running (issue #600).
+# running.
 module Trellis
   @default = nil
   @lock = Mutex.new

data/lib/trellis/railtie.rb

@@ -49,7 +49,7 @@ module Trellis
   # - config.trellis.connect_on_boot = false, for a server that forks with
   #   no hook to shut the handle down first (see below).
   #
-  # A handle doesn't survive fork (issue #600). A preforking server that
+  # A handle doesn't survive fork. A preforking server that
   # loads the app before forking (Puma's preload_app!, on by default with
   # more than one worker) must shut the boot handle down before it forks,
   # and connect one in each worker:

data/lib/trellis/schema_dump.rb

@@ -10,7 +10,7 @@ module Trellis
   # appends the `INSERT`s that describe how the schema came to be. A database
   # built from the file (`db:schema:load`, `db:prepare`, the test database
   # `maintain_test_schema!` loads) then has every transform defined, with no
-  # Trellis step to run: Trellis's worker finds the definitions unbuilt and builds them (issue #644, ADR-0010).
+  # Trellis step to run: Trellis's worker finds the definitions unbuilt and builds them (ADR-0010).
   #
   # Needs `config.active_record.schema_format = :sql`: a `schema.rb` can't
   # carry another schema's tables or triggers. With `:ruby`, the dump raises
@@ -156,7 +156,7 @@ module Trellis
 
       # The keywords of a libpq connection string or URI. A failure names
       # what was being read and never the string or libpq's message, which
-      # can quote part of a password (as Trellis's own parse does, #608).
+      # can quote part of a password (as Trellis's own parse does).
       def conninfo(url, what)
         PG::Connection.conninfo_parse(url.to_s).to_h { |entry| [entry[:keyword], entry[:val]] }
       rescue PG::Error

data/lib/trellis/values.rb

@@ -376,11 +376,11 @@ module Trellis
   Divergence = Data.define(:kind, :key, :column, :persisted, :recomputed, :table, :detail)
 
   # The configuration this process's handle connected with, as Trellis.config
-  # returns it: connect's schema and target_schema, and the connection pool's
-  # size cap and how long a call waits for a free connection.
+  # returns it: connect's schema and target_schema, and the handle's call pool
+  # as it was built: max_call_pool_size and call_pool_wait_timeout_ms.
   #
   # Not the url: a connection string can carry a password, and a Config is
   # the kind of value that ends up in a log line whole. The caller already
   # has the url it connected with.
-  Config = Data.define(:schema, :target_schema, :pool_max_size, :pool_wait_timeout_ms)
+  Config = Data.define(:schema, :target_schema, :max_call_pool_size, :call_pool_wait_timeout_ms)
 end

data/lib/trellis/version.rb

@@ -3,5 +3,5 @@
 module Trellis
   # The trellis-pg gem's version: the Trellis release's, which a
   # `v<VERSION>` tag releases (docs/releasing.md).
-  VERSION = "0.1.0-alpha"
+  VERSION = "0.1.0-beta"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: trellis-pg
 version: !ruby/object:Gem::Version
-  version: 0.1.0.pre.alpha
+  version: 0.1.0.pre.beta
 platform: ruby
 authors:
 - Salesforce