trellis-pg 0.1.0-aarch64-linux
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/README.md +620 -0
- data/lib/trellis/3.3/trellis_ruby.so +0 -0
- data/lib/trellis/3.4/trellis_ruby.so +0 -0
- data/lib/trellis/4.0/trellis_ruby.so +0 -0
- data/lib/trellis/error.rb +103 -0
- data/lib/trellis/instance.rb +561 -0
- data/lib/trellis/metrics.rb +38 -0
- data/lib/trellis/migration.rb +186 -0
- data/lib/trellis/pg.rb +142 -0
- data/lib/trellis/railtie.rb +293 -0
- data/lib/trellis/schema_dump.rb +278 -0
- data/lib/trellis/test_support.rb +161 -0
- data/lib/trellis/values.rb +386 -0
- data/lib/trellis/version.rb +7 -0
- metadata +67 -0
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Trellis
|
|
4
|
+
# The engine's metrics, in Prometheus text exposition format: the Rust
|
|
5
|
+
# crate's trellis::Metrics#render_prometheus
|
|
6
|
+
# (docs/decisions/0009-observability-decisions.md).
|
|
7
|
+
#
|
|
8
|
+
# The registry is process-wide: one body holds the series of every Trellis
|
|
9
|
+
# handle in this process, so it needs no handle and works before
|
|
10
|
+
# Trellis.connect and after Trellis.shutdown. Every series carries a
|
|
11
|
+
# trellis_instance="<database>/<catalog schema>" label, so instances in one
|
|
12
|
+
# process stay apart. trellis_instance_up is 1 while an instance has a
|
|
13
|
+
# running handle and 0 after its last one shuts down; the other gauges of a
|
|
14
|
+
# stopped instance disappear after 60 seconds. See docs/observability.md.
|
|
15
|
+
#
|
|
16
|
+
# The binding opens no port. Serve the text from a route the app already
|
|
17
|
+
# has, for example in Rails:
|
|
18
|
+
#
|
|
19
|
+
# # config/routes.rb
|
|
20
|
+
# get "/metrics", to: proc {
|
|
21
|
+
# [200, { "content-type" => "text/plain; version=0.0.4" }, [Trellis::Metrics.render_prometheus]]
|
|
22
|
+
# }
|
|
23
|
+
#
|
|
24
|
+
# In a forking server each worker process has its own registry, holding
|
|
25
|
+
# only its own handles' series, so scrape every worker (or run the worker
|
|
26
|
+
# handle, staging: true, in a process of its own and scrape that).
|
|
27
|
+
#
|
|
28
|
+
# A process forked while its parent had an engine running may have
|
|
29
|
+
# inherited the registry's lock held, so rendering there raises
|
|
30
|
+
# Trellis::ForkedHandleError, as Trellis.connect does.
|
|
31
|
+
module Metrics
|
|
32
|
+
# Renders the registry's current contents: # HELP and # TYPE lines, then
|
|
33
|
+
# each series' samples.
|
|
34
|
+
def self.render_prometheus
|
|
35
|
+
Native.render_prometheus
|
|
36
|
+
end
|
|
37
|
+
end
|
|
38
|
+
end
|
|
@@ -0,0 +1,186 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "active_record"
|
|
4
|
+
require "trellis/pg"
|
|
5
|
+
|
|
6
|
+
module Trellis
|
|
7
|
+
# Trellis statements in an ActiveRecord migration, so a transform is
|
|
8
|
+
# defined, and dropped, the way the tables it reads are created.
|
|
9
|
+
#
|
|
10
|
+
# class DefineWidgetPrices < ActiveRecord::Migration[8.1]
|
|
11
|
+
# include Trellis::Migration
|
|
12
|
+
#
|
|
13
|
+
# # Required: see "Transactions" below.
|
|
14
|
+
# disable_ddl_transaction!
|
|
15
|
+
#
|
|
16
|
+
# def up
|
|
17
|
+
# create_table :widgets do |t|
|
|
18
|
+
# t.integer :price
|
|
19
|
+
# end
|
|
20
|
+
#
|
|
21
|
+
# define "TRANSFORM widget_prices FROM widgets SELECT price AS price"
|
|
22
|
+
# end
|
|
23
|
+
#
|
|
24
|
+
# def down
|
|
25
|
+
# apply "PAUSE TRANSFORM widget_prices"
|
|
26
|
+
# apply "DROP TRANSFORM widget_prices"
|
|
27
|
+
# drop_table :widgets
|
|
28
|
+
# end
|
|
29
|
+
# end
|
|
30
|
+
#
|
|
31
|
+
# Three helpers, each a thin call through to the Trellis method of the
|
|
32
|
+
# same name:
|
|
33
|
+
#
|
|
34
|
+
# - define: a `TRANSFORM` statement, and nothing else (Trellis.define).
|
|
35
|
+
# - apply: any statement of the grammar (Trellis.apply), such as the
|
|
36
|
+
# `PAUSE TRANSFORM` and `DROP TRANSFORM` a `down` undoes a define with.
|
|
37
|
+
# (`DROP` refuses a transform that isn't paused, and drops its target
|
|
38
|
+
# table.)
|
|
39
|
+
# - status: the Status of the transform writing a target, or nil
|
|
40
|
+
# (Trellis.status).
|
|
41
|
+
#
|
|
42
|
+
# Each runs as it's called, in order with the migration's own statements,
|
|
43
|
+
# so a transform can read a table created earlier in the same migration.
|
|
44
|
+
# Each raises the Trellis::Error if its call fails, which fails the
|
|
45
|
+
# migration. (ActiveRecord's migrator re-raises it as a StandardError
|
|
46
|
+
# naming the migration, with the Trellis::Error as its cause.)
|
|
47
|
+
#
|
|
48
|
+
# == Transactions
|
|
49
|
+
#
|
|
50
|
+
# Trellis doesn't join the migration's transaction. It runs every call on
|
|
51
|
+
# connections of its own, and each call commits before it returns. If the
|
|
52
|
+
# migration's transaction rolled back after a define, the host's changes
|
|
53
|
+
# would be undone and the definition and its target table would not.
|
|
54
|
+
#
|
|
55
|
+
# So a migration that includes this module has to turn its transaction off
|
|
56
|
+
# with `disable_ddl_transaction!`. define and apply check, as they run,
|
|
57
|
+
# that the migration's connection has no transaction open (a `transaction
|
|
58
|
+
# do` block inside the migration opens one too), and raise
|
|
59
|
+
# ActiveRecord::MigrationError before applying anything if it does. In a
|
|
60
|
+
# migration that forgot `disable_ddl_transaction!`, the migration's
|
|
61
|
+
# transaction then rolls back with nothing applied on either side.
|
|
62
|
+
#
|
|
63
|
+
# Without the transaction, a migration that fails partway leaves whatever
|
|
64
|
+
# ran before the failure in place, on both sides. Give a migration that
|
|
65
|
+
# defines transforms little else to do.
|
|
66
|
+
#
|
|
67
|
+
# == up and down, not change
|
|
68
|
+
#
|
|
69
|
+
# ActiveRecord can reverse `create_table` on its own, but not a Trellis
|
|
70
|
+
# statement: the reverse of a define is a pause and a drop of a target
|
|
71
|
+
# that only the statement names. So a migration that includes this module
|
|
72
|
+
# must define `up` and `down`. One that defines a public `change` raises
|
|
73
|
+
# ActiveRecord::MigrationError before running anything, in either
|
|
74
|
+
# direction, rather than roll back partway. define and apply inside a
|
|
75
|
+
# `revert` block raise ActiveRecord::IrreversibleMigration.
|
|
76
|
+
#
|
|
77
|
+
# == Which Trellis
|
|
78
|
+
#
|
|
79
|
+
# If this process's handle is connected, the helpers use it. Otherwise
|
|
80
|
+
# (the usual case: `rails db:migrate` gets no boot handle, see
|
|
81
|
+
# Trellis::Railtie) each call connects a handle of its own from
|
|
82
|
+
# config.trellis.connect, runs Trellis.migrate on it (creating Trellis's
|
|
83
|
+
# own tables the first time), makes its one call, and shuts the handle
|
|
84
|
+
# down. A migration's own handle never runs background work: staging and
|
|
85
|
+
# drain_threads are always the defaults, whatever the configuration says.
|
|
86
|
+
#
|
|
87
|
+
# The helpers run on the default instance. A migration for an instance
|
|
88
|
+
# named in config.trellis.instances runs its statements on that instance
|
|
89
|
+
# itself, in the same transaction-free way:
|
|
90
|
+
#
|
|
91
|
+
# Trellis::Railtie.with_handle(:reports) do |reports|
|
|
92
|
+
# reports.define "TRANSFORM monthly_totals FROM orders SELECT total AS total"
|
|
93
|
+
# end
|
|
94
|
+
#
|
|
95
|
+
# == Running a migration once
|
|
96
|
+
#
|
|
97
|
+
# A define isn't idempotent: a second define of a target that exists
|
|
98
|
+
# raises ConflictError. ActiveRecord's schema_migrations already records
|
|
99
|
+
# which migrations have run, so a migration's define runs once per
|
|
100
|
+
# database, and these helpers add no guard of their own. A migration that
|
|
101
|
+
# has to tolerate a transform defined some other way (by hand, or by a
|
|
102
|
+
# deploy step) can guard the define itself:
|
|
103
|
+
#
|
|
104
|
+
# def up
|
|
105
|
+
# define "TRANSFORM widget_prices FROM widgets SELECT price AS price" if status("widget_prices").nil?
|
|
106
|
+
# end
|
|
107
|
+
#
|
|
108
|
+
# The guard checks the target's name, not its definition: a changed
|
|
109
|
+
# statement for a target that exists is an `ALTER TRANSFORM`.
|
|
110
|
+
module Migration
|
|
111
|
+
# The helpers' Trellis::Railtie.with_handle: runs the block with this
|
|
112
|
+
# process's handle, or one connected from config.trellis.connect for the
|
|
113
|
+
# length of the block, running Trellis.migrate first either way, and
|
|
114
|
+
# returns the block's value. Without Rails, it uses this process's
|
|
115
|
+
# handle. Raises ActiveRecord::MigrationError if there's neither.
|
|
116
|
+
def self.with_handle(&)
|
|
117
|
+
if Trellis.connected?
|
|
118
|
+
Trellis.migrate
|
|
119
|
+
return yield
|
|
120
|
+
end
|
|
121
|
+
|
|
122
|
+
unless defined?(Trellis::Railtie) && Trellis::Railtie.connect_options
|
|
123
|
+
raise ActiveRecord::MigrationError,
|
|
124
|
+
"there's no Trellis to run this against: set config.trellis.connect to " \
|
|
125
|
+
"Trellis.connect's options (config.trellis.connect = { url: ENV.fetch(\"DATABASE_URL\") }), " \
|
|
126
|
+
"or call Trellis.connect before migrating"
|
|
127
|
+
end
|
|
128
|
+
|
|
129
|
+
Trellis::Railtie.with_handle(&)
|
|
130
|
+
end
|
|
131
|
+
|
|
132
|
+
# Registers a transform from a `TRANSFORM` statement and returns its
|
|
133
|
+
# target table: Trellis.define, run as the migration reaches it.
|
|
134
|
+
def define(statement)
|
|
135
|
+
trellis_statement("define", statement) { Trellis.define(statement) }
|
|
136
|
+
end
|
|
137
|
+
|
|
138
|
+
# Runs any statement of Trellis's grammar and returns its Applied:
|
|
139
|
+
# Trellis.apply, run as the migration reaches it.
|
|
140
|
+
def apply(statement)
|
|
141
|
+
trellis_statement("apply", statement) { Trellis.apply(statement) }
|
|
142
|
+
end
|
|
143
|
+
|
|
144
|
+
# The Status of the transform writing target_table, or nil when none
|
|
145
|
+
# does: Trellis.status. A read, so it runs whether or not a transaction
|
|
146
|
+
# is open.
|
|
147
|
+
def status(target_table)
|
|
148
|
+
Migration.with_handle { Trellis.status(target_table) }
|
|
149
|
+
end
|
|
150
|
+
|
|
151
|
+
# ActiveRecord's entry point for running this migration in a direction:
|
|
152
|
+
# refuses a change method before anything runs.
|
|
153
|
+
def exec_migration(conn, direction)
|
|
154
|
+
if respond_to?(:change)
|
|
155
|
+
raise ActiveRecord::MigrationError,
|
|
156
|
+
"#{self.class.name} includes Trellis::Migration, so it must define up and down " \
|
|
157
|
+
"rather than change: ActiveRecord can't reverse a Trellis statement. Undo a " \
|
|
158
|
+
"define in down with apply \"PAUSE TRANSFORM <target>\" and " \
|
|
159
|
+
"apply \"DROP TRANSFORM <target>\"."
|
|
160
|
+
end
|
|
161
|
+
super
|
|
162
|
+
end
|
|
163
|
+
|
|
164
|
+
private
|
|
165
|
+
|
|
166
|
+
def trellis_statement(helper, statement)
|
|
167
|
+
if reverting?
|
|
168
|
+
raise ActiveRecord::IrreversibleMigration,
|
|
169
|
+
"cannot reverse a Trellis statement: #{statement}. Undo a define in down with " \
|
|
170
|
+
"PAUSE TRANSFORM and DROP TRANSFORM statements."
|
|
171
|
+
end
|
|
172
|
+
refuse_open_transaction!("this statement wasn't applied: #{statement}")
|
|
173
|
+
|
|
174
|
+
say_with_time("#{helper}(#{statement.inspect})") { Migration.with_handle { yield } }
|
|
175
|
+
end
|
|
176
|
+
|
|
177
|
+
def refuse_open_transaction!(what)
|
|
178
|
+
return unless connection.transaction_open?
|
|
179
|
+
|
|
180
|
+
raise ActiveRecord::MigrationError,
|
|
181
|
+
"Trellis doesn't join the migration's transaction, so #{what}: if the " \
|
|
182
|
+
"transaction rolled back, a definition would stay. Add `disable_ddl_transaction!` " \
|
|
183
|
+
"to the migration, and keep Trellis statements out of `transaction` blocks."
|
|
184
|
+
end
|
|
185
|
+
end
|
|
186
|
+
end
|
data/lib/trellis/pg.rb
ADDED
|
@@ -0,0 +1,142 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "error"
|
|
4
|
+
require_relative "values"
|
|
5
|
+
require_relative "version"
|
|
6
|
+
require "rbconfig"
|
|
7
|
+
|
|
8
|
+
# The native extension (ext/trellis_ruby). Installing the source gem, or
|
|
9
|
+
# `rake compile`, builds the one this Ruby needs into lib/trellis/. A platform
|
|
10
|
+
# gem instead carries one build per Ruby minor version, each in
|
|
11
|
+
# lib/trellis/<major.minor>/, and none in lib/trellis/ itself.
|
|
12
|
+
#
|
|
13
|
+
# A lib/trellis/ build wins when both are there: in a checkout, a
|
|
14
|
+
# per-version directory is only ever left over from building a platform gem
|
|
15
|
+
# (it's ignored by git, so nothing shows it), and `rake test` must load the
|
|
16
|
+
# extension it just compiled, not that one.
|
|
17
|
+
dlext = RbConfig::CONFIG.fetch("DLEXT")
|
|
18
|
+
per_version = "#{RUBY_VERSION[/\A\d+\.\d+/]}/trellis_ruby"
|
|
19
|
+
if !File.exist?(File.join(__dir__, "trellis_ruby.#{dlext}")) &&
|
|
20
|
+
File.exist?(File.join(__dir__, "#{per_version}.#{dlext}"))
|
|
21
|
+
require_relative per_version
|
|
22
|
+
else
|
|
23
|
+
require_relative "trellis_ruby"
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
require_relative "instance"
|
|
27
|
+
require_relative "metrics"
|
|
28
|
+
|
|
29
|
+
# Embedded Trellis for Ruby apps: a native extension over the `trellis`
|
|
30
|
+
# crate's BlockingTrellis, so a Rails app can define and run streaming
|
|
31
|
+
# transforms without deploying a separate service
|
|
32
|
+
# (docs/decisions/0010-embeddable-clients.md).
|
|
33
|
+
#
|
|
34
|
+
# A Trellis::Instance is one catalog schema's handle, and a process holds as
|
|
35
|
+
# many as it has schemas (Trellis::Instance.connect). This module is the
|
|
36
|
+
# common case's shortcut: Trellis.connect opens the process's default
|
|
37
|
+
# instance, every other method here is that instance's, and Trellis.shutdown
|
|
38
|
+
# closes it. An at_exit hook shuts down every instance the process still
|
|
39
|
+
# holds, the default included.
|
|
40
|
+
#
|
|
41
|
+
# The surface mirrors the Rust crate's BlockingTrellis, and is documented on
|
|
42
|
+
# Trellis::Instance:
|
|
43
|
+
#
|
|
44
|
+
# - Lifecycle: connect, connected?, migrate, config, shutdown.
|
|
45
|
+
# - Every statement form: apply, and define for a TRANSFORM statement alone.
|
|
46
|
+
# - Reads: status, definitions, dump_definitions_sql.
|
|
47
|
+
# - Quarantine: quarantined, quarantine_status, sample_quarantined,
|
|
48
|
+
# poisoned_since, release_key.
|
|
49
|
+
# - Operations: request_backfill, detect_capture_disruptions, has_live_drain_workers?,
|
|
50
|
+
# has_live_staging_worker?, watermark_token, await_converged, self_check,
|
|
51
|
+
# self_check_job.
|
|
52
|
+
# - Metrics: Trellis::Metrics.render_prometheus, which needs no handle.
|
|
53
|
+
#
|
|
54
|
+
# # A deploy's migration step: the defaults run nothing in the background.
|
|
55
|
+
# Trellis.connect(url: "host=localhost dbname=app")
|
|
56
|
+
# Trellis.migrate
|
|
57
|
+
# Trellis.shutdown
|
|
58
|
+
#
|
|
59
|
+
# # The app, once migrated.
|
|
60
|
+
# Trellis.connect(url: "host=localhost dbname=app", staging: true, drain_threads: 2)
|
|
61
|
+
# Trellis.define("TRANSFORM widget_prices FROM widgets SELECT price AS price")
|
|
62
|
+
# Trellis.status("widget_prices").status # => :live, once it has backfilled
|
|
63
|
+
#
|
|
64
|
+
# # A second instance, in a schema of its own.
|
|
65
|
+
# reports = Trellis::Instance.connect(url: "host=localhost dbname=app", schema: "reports")
|
|
66
|
+
# reports.status("monthly_totals")
|
|
67
|
+
# reports.shutdown # the default instance keeps running
|
|
68
|
+
#
|
|
69
|
+
# A handle doesn't survive `fork`. Shut down before forking
|
|
70
|
+
# (Trellis::Instance.shutdown_all; Puma's before_fork) and connect after
|
|
71
|
+
# (Puma's before_worker_boot, Passenger's starting_worker_process); in a
|
|
72
|
+
# Rails app, Trellis::Railtie.connect does the connecting. A forked child's
|
|
73
|
+
# calls on a handle it inherited raise Trellis::ForkedHandleError, except
|
|
74
|
+
# Trellis.shutdown, which leaves the parent's handle alone and does nothing.
|
|
75
|
+
# So does Trellis.connect in a child forked while its parent's handle was
|
|
76
|
+
# running.
|
|
77
|
+
module Trellis
|
|
78
|
+
@default = nil
|
|
79
|
+
@lock = Mutex.new
|
|
80
|
+
|
|
81
|
+
class << self
|
|
82
|
+
# Connects this process's default instance, with the options
|
|
83
|
+
# Trellis::Instance.connect takes. Raises ValidationError if the default
|
|
84
|
+
# instance is already connected: shut it down first. A second instance
|
|
85
|
+
# is Trellis::Instance.connect's. Raises ForkedHandleError in a process
|
|
86
|
+
# forked while its parent had an instance running.
|
|
87
|
+
def connect(**options)
|
|
88
|
+
@lock.synchronize do
|
|
89
|
+
if connected?
|
|
90
|
+
raise ValidationError,
|
|
91
|
+
"this process's default instance is already connected: call Trellis.shutdown " \
|
|
92
|
+
"first (Trellis::Instance.connect opens another instance)"
|
|
93
|
+
end
|
|
94
|
+
@default = Instance.connect(**options)
|
|
95
|
+
end
|
|
96
|
+
nil
|
|
97
|
+
end
|
|
98
|
+
|
|
99
|
+
# Whether this process's default instance is connected. False before
|
|
100
|
+
# connect, after shutdown, and in a forked child that hasn't connected
|
|
101
|
+
# its own.
|
|
102
|
+
def connected?
|
|
103
|
+
default = @default
|
|
104
|
+
!default.nil? && default.connected?
|
|
105
|
+
end
|
|
106
|
+
|
|
107
|
+
# The default instance Trellis.connect opened, for code that takes an
|
|
108
|
+
# Instance. Raises ValidationError if there is none.
|
|
109
|
+
def default_instance
|
|
110
|
+
@default or raise ValidationError, "Trellis is not connected: call Trellis.connect first"
|
|
111
|
+
end
|
|
112
|
+
|
|
113
|
+
# Stops the default instance (see Trellis::Instance#shutdown). Does
|
|
114
|
+
# nothing if it is not connected, which includes a forked child holding
|
|
115
|
+
# only the one it inherited.
|
|
116
|
+
def shutdown
|
|
117
|
+
@lock.synchronize do
|
|
118
|
+
return nil unless connected?
|
|
119
|
+
|
|
120
|
+
begin
|
|
121
|
+
@default.shutdown
|
|
122
|
+
ensure
|
|
123
|
+
@default = nil
|
|
124
|
+
end
|
|
125
|
+
end
|
|
126
|
+
nil
|
|
127
|
+
end
|
|
128
|
+
|
|
129
|
+
# Trellis::Instance's calls, on the default instance.
|
|
130
|
+
Instance.public_instance_methods(false).each do |name|
|
|
131
|
+
next if %i[connected? shutdown inspect to_s].include?(name)
|
|
132
|
+
|
|
133
|
+
define_method(name) { |*args, **options, &block| default_instance.public_send(name, *args, **options, &block) }
|
|
134
|
+
end
|
|
135
|
+
end
|
|
136
|
+
end
|
|
137
|
+
|
|
138
|
+
# The Rails integration, only when Rails is loaded first (Bundler.require in
|
|
139
|
+
# config/application.rb). Without Rails, nothing here requires a gem.
|
|
140
|
+
# Trellis::Migration (lib/trellis/migration.rb) needs ActiveRecord: the
|
|
141
|
+
# Railtie loads it with ActiveRecord, and an app without Rails requires it.
|
|
142
|
+
require_relative "railtie" if defined?(::Rails::Railtie)
|
|
@@ -0,0 +1,293 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "rails/railtie"
|
|
4
|
+
|
|
5
|
+
module Trellis
|
|
6
|
+
# Connects this process's handle as a Rails app boots, from the app's
|
|
7
|
+
# `config.trellis`. `require "trellis/pg"` loads it when Rails is loaded
|
|
8
|
+
# first, which is what Bundler.require in config/application.rb does.
|
|
9
|
+
#
|
|
10
|
+
# # config/application.rb, or config/environments/*.rb
|
|
11
|
+
# config.trellis.connect = {
|
|
12
|
+
# url: ENV.fetch("DATABASE_URL"),
|
|
13
|
+
# staging: ENV["TRELLIS_WORKER"] == "1",
|
|
14
|
+
# drain_threads: ENV["TRELLIS_WORKER"] == "1" ? 2 : 0
|
|
15
|
+
# }
|
|
16
|
+
#
|
|
17
|
+
# config.trellis.connect takes Trellis.connect's options, and is the one
|
|
18
|
+
# place the Rails integration reads the default instance's from: the boot
|
|
19
|
+
# connect, the migration helpers (Trellis::Migration) and `rails
|
|
20
|
+
# trellis:migrate` all use it. Leave it nil (the default) and the Railtie
|
|
21
|
+
# does nothing for the default instance.
|
|
22
|
+
#
|
|
23
|
+
# An app that runs more instances (a second catalog schema, or another
|
|
24
|
+
# database) names the extras in config.trellis.instances, each with
|
|
25
|
+
# Trellis::Instance.connect's options. The Railtie connects them with the
|
|
26
|
+
# default instance, `rails trellis:migrate` migrates them, and
|
|
27
|
+
# Trellis::Railtie.named_instance(:reports) returns one:
|
|
28
|
+
#
|
|
29
|
+
# config.trellis.instances = {
|
|
30
|
+
# reports: { url: ENV.fetch("REPORTS_DATABASE_URL"), schema: "reports" }
|
|
31
|
+
# }
|
|
32
|
+
#
|
|
33
|
+
# With `config.active_record.schema_format = :sql`, the same config puts
|
|
34
|
+
# the definitions in `structure.sql` (Trellis::SchemaDump). The database
|
|
35
|
+
# they dump into is config.trellis.dump_database, "primary" by default.
|
|
36
|
+
#
|
|
37
|
+
# When either is set, the app's handles connect in an after_initialize hook,
|
|
38
|
+
# except in these two cases:
|
|
39
|
+
#
|
|
40
|
+
# - An app booted by a rake task (`rails db:migrate`, `rails db:create`,
|
|
41
|
+
# ...): like Ecto, which migrates without starting the app, those get no
|
|
42
|
+
# boot handle. The database may not exist yet, and a migration run must
|
|
43
|
+
# not start the worker's background work. The migration helpers connect
|
|
44
|
+
# a handle of their own; a task of your own that calls Trellis calls
|
|
45
|
+
# Trellis::Railtie.connect. It's the task that counts, not the command:
|
|
46
|
+
# `rails test` loads the tasks to run test:prepare, then boots the app
|
|
47
|
+
# outside any task, so it gets its handle (but `bin/rails db:test:prepare
|
|
48
|
+
# test` boots it inside the first task, so it doesn't).
|
|
49
|
+
# - config.trellis.connect_on_boot = false, for a server that forks with
|
|
50
|
+
# no hook to shut the handle down first (see below).
|
|
51
|
+
#
|
|
52
|
+
# A handle doesn't survive fork. A preforking server that
|
|
53
|
+
# loads the app before forking (Puma's preload_app!, on by default with
|
|
54
|
+
# more than one worker) must shut the boot handle down before it forks,
|
|
55
|
+
# and connect one in each worker:
|
|
56
|
+
#
|
|
57
|
+
# # config/puma.rb
|
|
58
|
+
# before_fork { Trellis::Instance.shutdown_all }
|
|
59
|
+
# before_worker_boot { Trellis::Railtie.connect } # on_worker_boot before Puma 7
|
|
60
|
+
#
|
|
61
|
+
# With Puma's fork_worker, worker 0 forks the others, so it needs
|
|
62
|
+
# before_worker_fork and after_worker_fork hooks as well: see "Forking
|
|
63
|
+
# servers" in the README.
|
|
64
|
+
class Railtie < ::Rails::Railtie
|
|
65
|
+
config.trellis = ActiveSupport::OrderedOptions.new
|
|
66
|
+
config.trellis.connect = nil
|
|
67
|
+
config.trellis.instances = {}
|
|
68
|
+
config.trellis.connect_on_boot = true
|
|
69
|
+
config.trellis.dump_database = "primary"
|
|
70
|
+
|
|
71
|
+
rake_tasks do
|
|
72
|
+
namespace :trellis do
|
|
73
|
+
desc "Create or upgrade Trellis's own tables, for config.trellis.connect and config.trellis.instances"
|
|
74
|
+
task migrate: :environment do
|
|
75
|
+
Trellis::Railtie.migrate_all
|
|
76
|
+
end
|
|
77
|
+
end
|
|
78
|
+
end
|
|
79
|
+
|
|
80
|
+
initializer "trellis.migration" do
|
|
81
|
+
ActiveSupport.on_load(:active_record) do
|
|
82
|
+
require "trellis/migration"
|
|
83
|
+
require "trellis/schema_dump"
|
|
84
|
+
end
|
|
85
|
+
end
|
|
86
|
+
|
|
87
|
+
config.after_initialize { Trellis::Railtie.instance.send(:boot) }
|
|
88
|
+
|
|
89
|
+
# The instances Railtie.connect opened for config.trellis.instances,
|
|
90
|
+
# by name.
|
|
91
|
+
@instances = {}
|
|
92
|
+
|
|
93
|
+
class << self
|
|
94
|
+
# Connects this process's instances from the config: the default
|
|
95
|
+
# instance from config.trellis.connect (when set), and each of
|
|
96
|
+
# config.trellis.instances. In a forked worker (Puma's
|
|
97
|
+
# before_worker_boot, Unicorn's after_fork, Passenger's
|
|
98
|
+
# starting_worker_process), or a rake task of your own. Raises
|
|
99
|
+
# ValidationError if neither is set, or if a named instance is already
|
|
100
|
+
# connected (as Trellis.connect does for the default one), and whatever
|
|
101
|
+
# Trellis.connect and Trellis::Instance.connect raise; the instances
|
|
102
|
+
# this call had connected are shut down first, so a failure leaves
|
|
103
|
+
# nothing half connected.
|
|
104
|
+
def connect
|
|
105
|
+
default = connect_options
|
|
106
|
+
named = instance_options
|
|
107
|
+
if default.nil? && named.empty?
|
|
108
|
+
raise ValidationError,
|
|
109
|
+
"config.trellis.connect is not set: give it Trellis.connect's options"
|
|
110
|
+
end
|
|
111
|
+
named.each_key do |name|
|
|
112
|
+
next unless @instances[name]&.connected?
|
|
113
|
+
|
|
114
|
+
raise ValidationError,
|
|
115
|
+
"the instance named #{name.inspect} is already connected: call " \
|
|
116
|
+
"Trellis::Instance.shutdown_all first"
|
|
117
|
+
end
|
|
118
|
+
|
|
119
|
+
opened = []
|
|
120
|
+
begin
|
|
121
|
+
if default
|
|
122
|
+
Trellis.connect(**default)
|
|
123
|
+
opened << -> { Trellis.shutdown }
|
|
124
|
+
end
|
|
125
|
+
named.each do |name, options|
|
|
126
|
+
instance = Trellis::Instance.connect(**options)
|
|
127
|
+
@instances[name] = instance
|
|
128
|
+
opened << -> { @instances.delete(name)&.shutdown }
|
|
129
|
+
end
|
|
130
|
+
rescue Exception # rubocop:disable Lint/RescueException
|
|
131
|
+
opened.reverse_each { |undo| ignoring_errors(&undo) }
|
|
132
|
+
raise
|
|
133
|
+
end
|
|
134
|
+
nil
|
|
135
|
+
end
|
|
136
|
+
|
|
137
|
+
# The instance config.trellis.instances names `name`, connected by
|
|
138
|
+
# Railtie.connect. Raises ValidationError if there is none.
|
|
139
|
+
def named_instance(name)
|
|
140
|
+
instance = @instances[name.to_sym]
|
|
141
|
+
return instance if instance&.connected?
|
|
142
|
+
|
|
143
|
+
raise ValidationError,
|
|
144
|
+
"no connected instance named #{name.inspect}: name it in config.trellis.instances " \
|
|
145
|
+
"and call Trellis::Railtie.connect"
|
|
146
|
+
end
|
|
147
|
+
|
|
148
|
+
# Runs the block with an instance, running its migrate first, and
|
|
149
|
+
# returns the block's value. With no name, the default instance,
|
|
150
|
+
# yielded as well as reachable as the Trellis module; with a name, that
|
|
151
|
+
# instance of config.trellis.instances. If the instance isn't connected
|
|
152
|
+
# (a rake task gets no boot handle), connects one for the length of
|
|
153
|
+
# the block from its config, with staging: false and drain_threads: 0 so
|
|
154
|
+
# it starts no background work, and shuts it down after. What `rails
|
|
155
|
+
# trellis:migrate` and Trellis::Migration's helpers run on, for a script
|
|
156
|
+
# or rake task of your own that needs Trellis the way a migration does.
|
|
157
|
+
# Needs no ActiveRecord.
|
|
158
|
+
#
|
|
159
|
+
# Raises what the block raises. If the shutdown fails too, that's a
|
|
160
|
+
# warning, so it doesn't hide the block's error; after a block that
|
|
161
|
+
# returned, the shutdown's error is raised. Raises ValidationError if
|
|
162
|
+
# the instance isn't connected and its options aren't configured.
|
|
163
|
+
def with_handle(name = nil)
|
|
164
|
+
instance = name ? @instances[name.to_sym] : (Trellis.default_instance if Trellis.connected?)
|
|
165
|
+
if instance&.connected?
|
|
166
|
+
instance.migrate
|
|
167
|
+
return yield(instance)
|
|
168
|
+
end
|
|
169
|
+
|
|
170
|
+
options = name ? named_options(name) : required_connect_options
|
|
171
|
+
options = options.merge(staging: false, drain_threads: 0)
|
|
172
|
+
instance = name ? Trellis::Instance.connect(**options) : connect_default(options)
|
|
173
|
+
primary = nil
|
|
174
|
+
begin
|
|
175
|
+
instance.migrate
|
|
176
|
+
yield instance
|
|
177
|
+
rescue Exception => e # any exception, Interrupt too: it's re-raised
|
|
178
|
+
primary = e
|
|
179
|
+
raise
|
|
180
|
+
ensure
|
|
181
|
+
shutdown_after(name, instance, primary)
|
|
182
|
+
end
|
|
183
|
+
end
|
|
184
|
+
|
|
185
|
+
# Runs Trellis.migrate on the default instance (when
|
|
186
|
+
# config.trellis.connect is set) and on each of
|
|
187
|
+
# config.trellis.instances, each as with_handle does. Raises
|
|
188
|
+
# ValidationError if neither is set.
|
|
189
|
+
def migrate_all
|
|
190
|
+
default = connect_options
|
|
191
|
+
named = instance_options
|
|
192
|
+
if default.nil? && named.empty?
|
|
193
|
+
raise ValidationError,
|
|
194
|
+
"config.trellis.connect is not set: give it Trellis.connect's options"
|
|
195
|
+
end
|
|
196
|
+
|
|
197
|
+
with_handle { nil } if default
|
|
198
|
+
named.each_key { |name| with_handle(name) { nil } }
|
|
199
|
+
nil
|
|
200
|
+
end
|
|
201
|
+
|
|
202
|
+
# config.trellis.connect, as Trellis.connect's keyword options, or nil
|
|
203
|
+
# when it isn't set (or there's no Rails app).
|
|
204
|
+
def connect_options
|
|
205
|
+
app = ::Rails.respond_to?(:application) && ::Rails.application or return nil
|
|
206
|
+
options = app.config.trellis.connect or return nil
|
|
207
|
+
|
|
208
|
+
keyword_options("config.trellis.connect", options)
|
|
209
|
+
end
|
|
210
|
+
|
|
211
|
+
# config.trellis.instances, as each name's Trellis::Instance.connect
|
|
212
|
+
# keyword options; empty when it isn't set (or there's no Rails app).
|
|
213
|
+
def instance_options
|
|
214
|
+
app = ::Rails.respond_to?(:application) && ::Rails.application or return {}
|
|
215
|
+
instances = app.config.trellis.instances
|
|
216
|
+
return {} if instances.nil? || instances.empty?
|
|
217
|
+
|
|
218
|
+
unless instances.respond_to?(:to_h)
|
|
219
|
+
raise ValidationError,
|
|
220
|
+
"config.trellis.instances must be a Hash of names to Trellis::Instance.connect's " \
|
|
221
|
+
"options, got: #{instances.inspect}"
|
|
222
|
+
end
|
|
223
|
+
|
|
224
|
+
instances.to_h.to_h do |name, options|
|
|
225
|
+
[name.to_sym, keyword_options("config.trellis.instances[#{name.inspect}]", options)]
|
|
226
|
+
end
|
|
227
|
+
end
|
|
228
|
+
|
|
229
|
+
private
|
|
230
|
+
|
|
231
|
+
def keyword_options(what, options)
|
|
232
|
+
unless options.respond_to?(:to_h)
|
|
233
|
+
raise ValidationError,
|
|
234
|
+
"#{what} must be a Hash of Trellis.connect's options, got: #{options.inspect}"
|
|
235
|
+
end
|
|
236
|
+
|
|
237
|
+
options.to_h.transform_keys(&:to_sym)
|
|
238
|
+
end
|
|
239
|
+
|
|
240
|
+
def required_connect_options
|
|
241
|
+
connect_options or
|
|
242
|
+
raise ValidationError, "config.trellis.connect is not set: give it Trellis.connect's options"
|
|
243
|
+
end
|
|
244
|
+
|
|
245
|
+
def named_options(name)
|
|
246
|
+
instance_options.fetch(name.to_sym) do
|
|
247
|
+
raise ValidationError, "config.trellis.instances has no #{name.inspect}"
|
|
248
|
+
end
|
|
249
|
+
end
|
|
250
|
+
|
|
251
|
+
def connect_default(options)
|
|
252
|
+
Trellis.connect(**options)
|
|
253
|
+
Trellis.default_instance
|
|
254
|
+
end
|
|
255
|
+
|
|
256
|
+
def ignoring_errors
|
|
257
|
+
yield
|
|
258
|
+
rescue StandardError => e
|
|
259
|
+
warn "trellis: shutting down after a failed connect failed: #{e.class}: #{e.message}"
|
|
260
|
+
end
|
|
261
|
+
|
|
262
|
+
# Shuts with_handle's own instance down. With a primary error on its
|
|
263
|
+
# way out, a failed shutdown is only a warning: raising would replace
|
|
264
|
+
# it.
|
|
265
|
+
def shutdown_after(name, instance, primary)
|
|
266
|
+
name ? instance.shutdown : Trellis.shutdown
|
|
267
|
+
rescue StandardError => e
|
|
268
|
+
raise unless primary
|
|
269
|
+
|
|
270
|
+
warn "trellis: shutting down after #{primary.class} failed: #{e.class}: #{e.message}"
|
|
271
|
+
end
|
|
272
|
+
end
|
|
273
|
+
|
|
274
|
+
private
|
|
275
|
+
|
|
276
|
+
def boot
|
|
277
|
+
return if rake_running?
|
|
278
|
+
|
|
279
|
+
settings = ::Rails.application.config.trellis
|
|
280
|
+
return unless settings.connect_on_boot && (settings.connect || !self.class.instance_options.empty?)
|
|
281
|
+
|
|
282
|
+
self.class.connect
|
|
283
|
+
end
|
|
284
|
+
|
|
285
|
+
# Whether the app is booting inside a rake run (`rails db:migrate`,
|
|
286
|
+
# `rake db:create`), not just in a process that loaded the tasks:
|
|
287
|
+
# `rails test` loads them to run test:prepare, and only then, outside
|
|
288
|
+
# any task, boots the app.
|
|
289
|
+
def rake_running?
|
|
290
|
+
defined?(::Rake.application) && ::Rake.application.top_level_tasks.any?
|
|
291
|
+
end
|
|
292
|
+
end
|
|
293
|
+
end
|