trellis-pg 0.1.0.pre.alpha → 0.1.0.pre.gamma

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f1baafc626d091c77635ab5de6f1bb321200d27b3ce659465f2b69cf34e948ab
-  data.tar.gz: c9c7d5f91121f749391c06e331732a375e98f149ef7ca2259aa2e965640ee6e2
+  metadata.gz: 8839af0acc8f206000a1348bbba21b4ea1d06784bf7d677c0244f0c630863c00
+  data.tar.gz: 77f8ffb6fc607c81fcbb614a6238aa9c21770616a525aeda1df4a007247c0d59
 SHA512:
-  metadata.gz: 685a0b5d491c23b32a9ed8784153d68ae60191f2901633030e74cd5da8f6fe19dfb3f09c1fdbfbf1d425e3a7023845f63726d7c37079f78d97086694de11e531
-  data.tar.gz: c520497842ef2b4ecd252b3a307ea33c5a01b0fdd818f3f74561141c8b28408abb7288b50971383099979d6e074451febbcdfde047e6f3473f3e5b20eb31fb48
+  metadata.gz: fc0f00f2deb17dcbb647ca2ff60d0da513e77987fc9d4206cf1c4f723412284d44f337120dc1c667a7d6149a07f144d97825a6f9cf473d2f1c3f18630e5c1e00
+  data.tar.gz: 8c00d355d02a8437815fedfa97e519c4ce2fe576e46a4614c273bd1c290c89729bc1f83e1c0176b315f5a26a1566ab74a01b5453b59f25a411308e415c93674e

data/README.md

@@ -103,6 +103,9 @@ Nothing in the binding uses it.
   and `self_check_job` returns `nil` for the replaced job or one whose
   transform was dropped. With no drain worker anywhere in the fleet a job
   stays `:queued`.
+- `Trellis::Metrics.render_prometheus`: the process-wide metrics registry as
+  Prometheus text, for a `/metrics` route the app already serves (the
+  binding opens no port; see [Metrics](#metrics)). It needs no handle.
 
 ```ruby
 require "trellis/pg"
@@ -144,10 +147,59 @@ Trellis.shutdown
 | `staging:` | `false` | run the staging worker (installs change capture, starts backfills) here |
 | `drain_threads:` | `0` | threads applying staged changes to the targets |
 | `worker_threads:` | `2` | the threads of each Rust runtime a handle owns (its calls', and its background client's), invisible to Ruby's own sizing |
+| `max_call_pool_size:` | `20` | the most connections the handle's call pool opens; the background pool is sized by `staging:` and `drain_threads:` |
+| `call_pool_wait_timeout_ms:` | `30_000` | how long a call waits for a free call-pool connection before a `ConnectivityError` |
+
+The defaults run nothing in the background. Some process in a fleet must set
+`staging: true` and some process must run drain threads, or no definition ever
+reaches `:live`. Any number of processes may set `staging: true`: one takes the
+producer lock and leads, the rest stand by and take over when it goes away, so a
+rolling deploy needs no operator step. Under a preforking Puma with
+`TRELLIS_WORKER=1`, every Puma worker connects with `staging: true`, so all but
+one stand by; that is harmless, and each polls the lock every 5 seconds (see
+[Rolling deploys](../../docs/embedding.md#rolling-deploys)).
+
+### Resources
+
+At the defaults, a handle holds up to 20 Postgres connections (its call pool,
+opened as calls need them and closed after 60 seconds idle) and 3 threads. A
+worker handle (`staging: true, drain_threads: N`) also owns a background pool
+of up to `2N + 5` connections (`2N + 4` without staging, `1` with staging
+alone), closed after 60 seconds idle, and two connections that stay open: the
+staging session and one `LISTEN` shared by its drain threads. It holds 6
+threads: drain threads are tasks on its runtime, not OS threads. Its memory is
+bounded by row counts, not bytes: about `N` × 100,000 changed rows' images at
+its peak. `max_call_pool_size:` lowers the 20 (the background pool is sized by
+the handle's roles, not by an option), and `worker_threads:` sets the threads.
+Every handle (each `Trellis::Instance` too) costs this again. The full
+accounting is in
+[`docs/embedding.md`](../../docs/embedding.md#what-a-handle-uses).
+
+### Metrics
+
+The engine's metrics live in one registry per process, holding the series of
+every handle in it; `Trellis::Metrics.render_prometheus` renders them in
+Prometheus's text format. Every series carries a `trellis_instance` label
+(`<database>/<catalog schema>`), so instances in one process stay apart, and
+`trellis_instance_up` is 1 while an instance's handle runs and 0 after it
+shuts down. The metrics themselves are listed in
+[`docs/observability.md`](../../docs/observability.md).
+
+The binding opens no port, so serve the text from a route of the app's own:
 
-The defaults run nothing in the background. Exactly one process in a fleet
-should set `staging: true`, and some process must run drain threads, or no
-definition ever reaches `:live`.
+```ruby
+# config/routes.rb
+get "/metrics", to: proc {
+  [200, { "content-type" => "text/plain; version=0.0.4" }, [Trellis::Metrics.render_prometheus]]
+}
+```
+
+Under a forking server each worker process has its own registry, holding
+only its own handles' series, so Prometheus scrapes each worker (or the
+worker handle, `staging: true`, runs in a process of its own that it
+scrapes). A process forked while its parent had a handle running can't
+render the registry, as it can't connect: it raises
+`Trellis::ForkedHandleError` (see [Forking servers](#forking-servers)).
 
 ### Several instances
 
@@ -501,6 +553,7 @@ source rows; a test database has none, so that is immediate. Runtime state
 ## Layout
 
 - `lib/`: the `Trellis` module and `Trellis::Instance`, its error classes and its `Data` values,
+  `Trellis::Metrics`,
   the Rails integration (`trellis/railtie.rb`, `trellis/migration.rb`),
   loaded only in an app that has Rails, and the test helpers
   (`trellis/test_support.rb`), loaded only when required.

data/ext/trellis_ruby/Cargo.lock

@@ -1729,9 +1729,10 @@ dependencies = [
 
 [[package]]
 name = "trellis"
-version = "0.1.0-alpha"
-source = "git+https://github.com/salesforce-misc/trellis?rev=31985aab1f7ddb22513973b0e85882c3eca9583f#31985aab1f7ddb22513973b0e85882c3eca9583f"
+version = "0.1.0-gamma"
+source = "git+https://github.com/salesforce-misc/trellis?rev=c271ef149159d9309174ca476d792c89128bce99#c271ef149159d9309174ca476d792c89128bce99"
 dependencies = [
+ "deadpool",
  "deadpool-postgres",
  "getrandom 0.4.3",
  "metrics",
@@ -1751,8 +1752,8 @@ dependencies = [
 
 [[package]]
 name = "trellis-embed"
-version = "0.1.0-alpha"
-source = "git+https://github.com/salesforce-misc/trellis?rev=31985aab1f7ddb22513973b0e85882c3eca9583f#31985aab1f7ddb22513973b0e85882c3eca9583f"
+version = "0.1.0-gamma"
+source = "git+https://github.com/salesforce-misc/trellis?rev=c271ef149159d9309174ca476d792c89128bce99#c271ef149159d9309174ca476d792c89128bce99"
 dependencies = [
  "tracing",
  "trellis",
@@ -1760,7 +1761,7 @@ dependencies = [
 
 [[package]]
 name = "trellis_ruby"
-version = "0.1.0-alpha"
+version = "0.1.0-gamma"
 dependencies = [
  "magnus",
  "rb-sys",

data/ext/trellis_ruby/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "trellis_ruby"
-version = "0.1.0-alpha"
+version = "0.1.0-gamma"
 edition = "2024"
 publish = false
 
@@ -28,8 +28,8 @@ doctest = false
 ruby = ["dep:magnus", "dep:rb-sys"]
 
 [dependencies]
-trellis = { git = "https://github.com/salesforce-misc/trellis", rev = "31985aab1f7ddb22513973b0e85882c3eca9583f" }
-trellis-embed = { git = "https://github.com/salesforce-misc/trellis", rev = "31985aab1f7ddb22513973b0e85882c3eca9583f" }
+trellis = { git = "https://github.com/salesforce-misc/trellis", rev = "c271ef149159d9309174ca476d792c89128bce99" }
+trellis-embed = { git = "https://github.com/salesforce-misc/trellis", rev = "c271ef149159d9309174ca476d792c89128bce99" }
 magnus = { version = "0.9", optional = true, default-features = false, features = ["rb-sys"] }
 rb-sys = { version = "0.9", optional = true }
 

data/ext/trellis_ruby/src/engine.rs

@@ -1,4 +1,4 @@
-//! Knowing whether a forked child may connect (issue #600).
+//! Knowing whether a forked child may connect.
 //!
 //! `fork()` copies the whole address space but only the calling thread. The
 //! engine keeps process-wide state behind locks (the enum type-name

data/ext/trellis_ruby/src/lib.rs

@@ -24,7 +24,7 @@
 //!   and raises `Trellis::ForkedHandleError`, and a forked copy is never
 //!   dropped either (see [`Handle`]'s `Drop`).
 //! - **A process forked while its parent had an engine running can't
-//!   connect one of its own** (issue #600): it may have inherited a lock one
+//!   connect one of its own**: it may have inherited a lock one
 //!   of the parent's engine threads held, which nothing in it can release.
 //!   [`connect`] raises `Trellis::ForkedHandleError` instead of risking the
 //!   hang. See the [`engine`] module.
@@ -331,7 +331,7 @@ impl Handle {
     }
 
     /// Pauses now the readers of every table whose capture was changed
-    /// outside Trellis (#1181), and returns the tables.
+    /// outside Trellis, and returns the tables.
     fn detect_capture_disruptions(ruby: &Ruby, rb_self: &Self) -> Result<Vec<String>, Error> {
         rb_self.call(ruby, BlockingTrellis::start_detect_capture_disruptions)
     }
@@ -388,18 +388,21 @@ impl Handle {
     }
 
     /// The configuration the handle connected with, all but the connection
-    /// string (see [`PlainConfig`]).
+    /// string, and its call pool's limits (see [`PlainConfig`]).
     fn config(ruby: &Ruby, rb_self: &Self) -> Result<RHash, Error> {
-        let config = rb_self.with_engine(ruby, |trellis| PlainConfig::from(trellis.config()))?;
+        let config = rb_self.with_engine(ruby, |trellis| PlainConfig::from(trellis))?;
         record(
             ruby,
             [
                 ("schema", ruby.into_value(config.schema)),
                 ("target_schema", ruby.into_value(config.target_schema)),
-                ("pool_max_size", ruby.into_value(config.pool_max_size)),
                 (
-                    "pool_wait_timeout_ms",
-                    ruby.into_value(config.pool_wait_timeout_ms),
+                    "max_call_pool_size",
+                    ruby.into_value(config.max_call_pool_size),
+                ),
+                (
+                    "call_pool_wait_timeout_ms",
+                    ruby.into_value(config.call_pool_wait_timeout_ms),
                 ),
             ],
         )
@@ -432,7 +435,11 @@ impl Handle {
 /// `lib/trellis/pg.rb`'s to document, and nothing falls back to the environment.
 ///
 /// Raises `Trellis::ForkedHandleError`, before starting anything, in a
-/// process forked while its parent had an engine running (issue #600).
+/// process forked while its parent had an engine running.
+#[allow(
+    clippy::too_many_arguments,
+    reason = "`function!` binds each Ruby argument to its own parameter"
+)]
 fn connect(
     ruby: &Ruby,
     url: String,
@@ -441,6 +448,8 @@ fn connect(
     staging: bool,
     drain_threads: usize,
     worker_threads: usize,
+    max_call_pool_size: usize,
+    call_pool_wait_timeout_ms: u64,
 ) -> Result<Handle, Error> {
     engine::check().map_err(|err| forked_handle_error(ruby, forked_while_running(&err)))?;
     // Connecting can wait on the database (a staging worker installs its
@@ -460,6 +469,8 @@ fn connect(
                     staging,
                     drain_threads,
                     worker_threads: Some(worker_threads),
+                    max_call_pool_size,
+                    call_pool_wait_timeout: Duration::from_millis(call_pool_wait_timeout_ms),
                     ..Default::default()
                 };
                 // Can't fail after `check` passed (a process's answer never
@@ -1220,6 +1231,20 @@ fn statement_kinds() -> Vec<&'static str> {
         .collect()
 }
 
+/// The process-wide metrics registry in Prometheus text exposition format:
+/// `trellis::Metrics::render_prometheus`, the series of every handle in this
+/// process. It needs no handle.
+///
+/// The registry sits behind a lock the engine's threads take, so a process
+/// forked while its parent had an engine running may have inherited it held:
+/// rendering there raises `Trellis::ForkedHandleError`, as [`connect`] does,
+/// rather than risk the hang. Rendering is a short walk of the registry,
+/// so it keeps the GVL.
+fn render_prometheus(ruby: &Ruby) -> Result<String, Error> {
+    engine::check().map_err(|err| forked_handle_error(ruby, forked_while_running(&err)))?;
+    Ok(trellis::Metrics::new().render_prometheus())
+}
+
 #[magnus::init]
 fn init(ruby: &Ruby) -> Result<(), Error> {
     // Interns every symbol a result can carry up front, from the closed
@@ -1229,7 +1254,7 @@ fn init(ruby: &Ruby) -> Result<(), Error> {
     }
 
     let native = ruby.define_module("Trellis")?.define_module("Native")?;
-    native.define_module_function("connect", function!(connect, 6))?;
+    native.define_module_function("connect", function!(connect, 8))?;
     native.define_module_function("error_codes", function!(error_codes, 0))?;
     native.define_module_function("status_names", function!(status_names, 0))?;
     native.define_module_function("quarantine_states", function!(quarantine_states, 0))?;
@@ -1241,6 +1266,7 @@ fn init(ruby: &Ruby) -> Result<(), Error> {
     native.define_module_function("capture_failure_kinds", function!(capture_failure_kinds, 0))?;
     native.define_module_function("index_roles", function!(index_roles, 0))?;
     native.define_module_function("statement_kinds", function!(statement_kinds, 0))?;
+    native.define_module_function("render_prometheus", function!(render_prometheus, 0))?;
 
     let handle = native.define_class("Handle", ruby.class_object())?;
     handle.define_method("owner_pid", method!(Handle::owner_pid, 0))?;

data/lib/trellis/error.rb

@@ -82,7 +82,7 @@ module Trellis
   #
   # Also Trellis.connect's error in a process forked while its parent had a
   # handle running, which may have inherited a lock that handle's threads
-  # held (issue #600): shut down before forking (Puma's before_fork).
+  # held: shut down before forking (Puma's before_fork).
   class ForkedHandleError < ValidationError
   end
 

data/lib/trellis/instance.rb

@@ -98,7 +98,7 @@ module Trellis
   # forked child's calls on an instance it inherited raise
   # Trellis::ForkedHandleError, and its shutdown leaves the parent's handle
   # alone and does nothing. So does Instance.connect in a child forked while
-  # any instance of its parent was running (issue #600).
+  # any instance of its parent was running.
   class Instance
     extend Checks
     include Checks
@@ -111,7 +111,7 @@ module Trellis
       # Connects a new instance and returns it. Raises ForkedHandleError in
       # a process forked while its parent had an instance running
       # (connected, connecting, or not yet fully shut down): it may have
-      # inherited a lock one of that instance's threads held (issue #600).
+      # inherited a lock one of that instance's threads held.
       #
       # - url: where the database is, as a libpq connection string
       #   ("host=... dbname=...") or URL. Nothing is read from the environment.
@@ -121,8 +121,10 @@ module Trellis
       # - target_schema: the schema a bare target table name is created in.
       # - staging: whether this process runs the instance's staging worker,
       #   which installs change capture on the source tables and starts each
-      #   new transform's backfill. Exactly one process in a fleet should,
-      #   per instance.
+      #   new transform's backfill. Any number of processes may set it: one
+      #   takes the instance's producer lock and leads, the rest stand by and
+      #   take over when it goes away, so a rolling deploy needs no
+      #   operator step.
       # - drain_threads: how many threads apply staged changes to the targets.
       #   Some process must run at least one per instance, or no definition
       #   reaches :live.
@@ -130,13 +132,20 @@ module Trellis
       #   instance owns (its calls', and its background client's), invisible
       #   to Ruby's own thread sizing, so kept small and explicit. Each
       #   instance has its own; nothing is shared between instances.
+      # - max_call_pool_size: the most connections this instance's call pool
+      #   (its API calls) opens at once. Connections open as calls need them
+      #   and close after 60 seconds idle. The background client's pool is
+      #   sized by staging and drain_threads, not by this.
+      # - call_pool_wait_timeout_ms: how long a call waits for a free call-pool
+      #   connection before failing with a ConnectivityError.
       #
       # The defaults (staging: false, drain_threads: 0) run nothing in the
       # background, which is what a migration step or a console wants. The
       # staging worker reads Trellis's tables as it starts, so a staging: true
       # connect to an unmigrated schema fails: migrate first.
       def connect(url:, schema: "trellis", target_schema: "public", staging: false,
-                  drain_threads: 0, worker_threads: 2)
+                  drain_threads: 0, worker_threads: 2, max_call_pool_size: 20,
+                  call_pool_wait_timeout_ms: 30_000)
         string_option!(:url, url)
         string_option!(:schema, schema)
         string_option!(:target_schema, target_schema)
@@ -152,7 +161,19 @@ module Trellis
                 "worker_threads must be a positive Integer, got: #{worker_threads.inspect}"
         end
 
-        handle = Native.connect(url, schema, target_schema, staging, drain_threads, worker_threads)
+        unless max_call_pool_size.is_a?(Integer) && max_call_pool_size >= 1
+          raise ValidationError,
+                "max_call_pool_size must be a positive Integer, got: #{max_call_pool_size.inspect}"
+        end
+        unless call_pool_wait_timeout_ms.is_a?(Integer) &&
+               call_pool_wait_timeout_ms.between?(0, MAX_TIMEOUT_MS)
+          raise ValidationError,
+                "call_pool_wait_timeout_ms must be a non-negative Integer, got: " \
+                "#{call_pool_wait_timeout_ms.inspect}"
+        end
+
+        handle = Native.connect(url, schema, target_schema, staging, drain_threads, worker_threads,
+                                max_call_pool_size, call_pool_wait_timeout_ms)
         instance = new(handle, schema)
         register(instance)
         instance

data/lib/trellis/metrics.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+module Trellis
+  # The engine's metrics, in Prometheus text exposition format: the Rust
+  # crate's trellis::Metrics#render_prometheus
+  # (docs/decisions/0009-observability-decisions.md).
+  #
+  # The registry is process-wide: one body holds the series of every Trellis
+  # handle in this process, so it needs no handle and works before
+  # Trellis.connect and after Trellis.shutdown. Every series carries a
+  # trellis_instance="<database>/<catalog schema>" label, so instances in one
+  # process stay apart. trellis_instance_up is 1 while an instance has a
+  # running handle and 0 after its last one shuts down; the other gauges of a
+  # stopped instance disappear after 60 seconds. See docs/observability.md.
+  #
+  # The binding opens no port. Serve the text from a route the app already
+  # has, for example in Rails:
+  #
+  #   # config/routes.rb
+  #   get "/metrics", to: proc {
+  #     [200, { "content-type" => "text/plain; version=0.0.4" }, [Trellis::Metrics.render_prometheus]]
+  #   }
+  #
+  # In a forking server each worker process has its own registry, holding
+  # only its own handles' series, so scrape every worker (or run the worker
+  # handle, staging: true, in a process of its own and scrape that).
+  #
+  # A process forked while its parent had an engine running may have
+  # inherited the registry's lock held, so rendering there raises
+  # Trellis::ForkedHandleError, as Trellis.connect does.
+  module Metrics
+    # Renders the registry's current contents: # HELP and # TYPE lines, then
+    # each series' samples.
+    def self.render_prometheus
+      Native.render_prometheus
+    end
+  end
+end

data/lib/trellis/pg.rb

@@ -24,6 +24,7 @@ else
 end
 
 require_relative "instance"
+require_relative "metrics"
 
 # Embedded Trellis for Ruby apps: a native extension over the `trellis`
 # crate's BlockingTrellis, so a Rails app can define and run streaming
@@ -48,6 +49,7 @@ require_relative "instance"
 # - Operations: request_backfill, detect_capture_disruptions, has_live_drain_workers?,
 #   has_live_staging_worker?, watermark_token, await_converged, self_check,
 #   self_check_job.
+# - Metrics: Trellis::Metrics.render_prometheus, which needs no handle.
 #
 #   # A deploy's migration step: the defaults run nothing in the background.
 #   Trellis.connect(url: "host=localhost dbname=app")
@@ -71,7 +73,7 @@ require_relative "instance"
 # calls on a handle it inherited raise Trellis::ForkedHandleError, except
 # Trellis.shutdown, which leaves the parent's handle alone and does nothing.
 # So does Trellis.connect in a child forked while its parent's handle was
-# running (issue #600).
+# running.
 module Trellis
   @default = nil
   @lock = Mutex.new

data/lib/trellis/railtie.rb

@@ -49,7 +49,7 @@ module Trellis
   # - config.trellis.connect_on_boot = false, for a server that forks with
   #   no hook to shut the handle down first (see below).
   #
-  # A handle doesn't survive fork (issue #600). A preforking server that
+  # A handle doesn't survive fork. A preforking server that
   # loads the app before forking (Puma's preload_app!, on by default with
   # more than one worker) must shut the boot handle down before it forks,
   # and connect one in each worker:

data/lib/trellis/schema_dump.rb

@@ -10,7 +10,7 @@ module Trellis
   # appends the `INSERT`s that describe how the schema came to be. A database
   # built from the file (`db:schema:load`, `db:prepare`, the test database
   # `maintain_test_schema!` loads) then has every transform defined, with no
-  # Trellis step to run: Trellis's worker finds the definitions unbuilt and builds them (issue #644, ADR-0010).
+  # Trellis step to run: Trellis's worker finds the definitions unbuilt and builds them (ADR-0010).
   #
   # Needs `config.active_record.schema_format = :sql`: a `schema.rb` can't
   # carry another schema's tables or triggers. With `:ruby`, the dump raises
@@ -156,7 +156,7 @@ module Trellis
 
       # The keywords of a libpq connection string or URI. A failure names
       # what was being read and never the string or libpq's message, which
-      # can quote part of a password (as Trellis's own parse does, #608).
+      # can quote part of a password (as Trellis's own parse does).
       def conninfo(url, what)
         PG::Connection.conninfo_parse(url.to_s).to_h { |entry| [entry[:keyword], entry[:val]] }
       rescue PG::Error

data/lib/trellis/values.rb

@@ -376,11 +376,11 @@ module Trellis
   Divergence = Data.define(:kind, :key, :column, :persisted, :recomputed, :table, :detail)
 
   # The configuration this process's handle connected with, as Trellis.config
-  # returns it: connect's schema and target_schema, and the connection pool's
-  # size cap and how long a call waits for a free connection.
+  # returns it: connect's schema and target_schema, and the handle's call pool
+  # as it was built: max_call_pool_size and call_pool_wait_timeout_ms.
   #
   # Not the url: a connection string can carry a password, and a Config is
   # the kind of value that ends up in a log line whole. The caller already
   # has the url it connected with.
-  Config = Data.define(:schema, :target_schema, :pool_max_size, :pool_wait_timeout_ms)
+  Config = Data.define(:schema, :target_schema, :max_call_pool_size, :call_pool_wait_timeout_ms)
 end

data/lib/trellis/version.rb

@@ -3,5 +3,5 @@
 module Trellis
   # The trellis-pg gem's version: the Trellis release's, which a
   # `v<VERSION>` tag releases (docs/releasing.md).
-  VERSION = "0.1.0-alpha"
+  VERSION = "0.1.0-gamma"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: trellis-pg
 version: !ruby/object:Gem::Version
-  version: 0.1.0.pre.alpha
+  version: 0.1.0.pre.gamma
 platform: ruby
 authors:
 - Salesforce
@@ -41,6 +41,7 @@ files:
 - ext/trellis_ruby/src/lib.rs
 - lib/trellis/error.rb
 - lib/trellis/instance.rb
+- lib/trellis/metrics.rb
 - lib/trellis/migration.rb
 - lib/trellis/pg.rb
 - lib/trellis/railtie.rb