yamine 0.15.2 → 0.17.0

data/lib/yamine/cli/boot.rb

@@ -116,27 +116,29 @@ module Yamine
         say opts, "yamine (#{service})"
         say opts, "--"
 
-        # Per-worktree database: one database per directory so concurrent
-        # agents never share tables. Resolved once, injected into every
-        # process as DATABASE_URL, created on demand with the app's own
-        # schema-load command. SQLite needs nothing (relative paths
-        # already isolate); the branch plays no part (dirs are stable,
-        # branches hop).
+        # Per-worktree databases: one isolated set per directory so
+        # concurrent agents never share tables — every database the app
+        # declares (a Rails multi-database app may have five). The
+        # claim records the whole set; setup creates what's missing,
+        # loads schemas on first creation, and hands back the env to
+        # inject into every process. SQLite needs nothing (relative
+        # paths already isolate); the branch plays no part (dirs are
+        # stable, branches hop).
         db_name = Database.name_for(Dir.pwd, env: rails_env,
           state_dir: ctx.store.dir)
-        db_created, db_url = setup_database(ctx, resolved, db_name, events: events)
-        events&.note("[db] #{db_name}#{db_created ? " (created)" : ""}") if db_url
+        db_created, db_env = setup_database(ctx, resolved, db_name, events: events)
+        events&.note("[db] #{db_name}#{db_created ? " (created)" : ""}") if db_env && !db_env.empty?
 
         with_wait = !opts[:no_wait]
-        spawn_plan = collect_spawns(ctx, runner, resolved, opts, db_url, children)
+        spawn_plan = collect_spawns(ctx, runner, resolved, opts, db_env, children)
 
         begin
           if with_wait
             boot_concurrent(ctx, runner, resolved, opts, spawn_plan, children,
-              routes_registered, db_url, db_name, db_created, events: events)
+              routes_registered, db_env, db_name, db_created, events: events)
           else
             boot_sequential(ctx, runner, resolved, opts, spawn_plan, children,
-              routes_registered, db_url, events: events)
+              routes_registered, db_env, events: events)
           end
         rescue StandardError
           # A raise mid-boot (route conflict, quota, unexpected error)
@@ -198,14 +200,14 @@ module Yamine
       # (no route to wait on), HTTP processes become a spawn plan the
       # boot mode executes. Compound-line refusal happens here so both
       # paths share one rule.
-      def collect_spawns(ctx, runner, resolved, opts, db_url, children)
+      def collect_spawns(ctx, runner, resolved, opts, db_env, children)
         plan = []
         processes = resolved.processes
         processes.each do |proc_name, entry|
           hostname = Resolver.hostname_for(resolved, proc_name)
           if entry["proxy"] == false
             boot_background(ctx, runner, resolved, proc_name, entry, opts,
-              children, db_url)
+              children, db_env)
             next
           end
           next unless hostname
@@ -226,19 +228,20 @@ module Yamine
 
       # Default: sequential boot, route registered per process as before.
       def boot_sequential(ctx, runner, resolved, opts, plan, children,
-        routes_registered, db_url, events: nil)
+        routes_registered, db_env, events: nil)
         plan.each do |item|
           say opts, "  [#{item[:name]}] #{item[:url]}"
           # Always allow the proxied hostname in Rails dev (Rails ignores
           # this env var when not a Rails app — safe for every framework).
-          # DATABASE_URL points at this worktree's own database; non-Ruby
-          # frameworks that honor it get isolation for free, others ignore it.
-          # boot_run registers the route + backend sidecar itself
-          # (register: true default); no separate adopt step needed.
+          # db_env points every process at this worktree's own databases;
+          # non-Ruby frameworks that honor DATABASE_URL get isolation for
+          # free, others ignore it. boot_run registers the route + backend
+          # sidecar itself (register: true default); no separate adopt
+          # step needed.
           app = runner.boot_run(name: item[:name], hostname: item[:hostname],
             url: item[:url], dir: Dir.pwd, command: item[:command],
             port: item[:port], force: opts[:force],
-            rails_dev_host: item[:hostname], database_url: db_url,
+            rails_dev_host: item[:hostname], database_env: db_env,
             subdomains: resolved.subdomains,
             extra_env: build_env(resolved, item[:entry], proc_name: item[:name]))
           routes_registered << { hostnames: item[:hostnames], app: app }
@@ -253,14 +256,14 @@ module Yamine
       # never leak into the proxy. A backend that dies before its route
       # registers is cleaned up like any other failed process.
       def boot_concurrent(ctx, runner, resolved, opts, plan, children,
-        routes_registered, db_url, db_name, db_created, events: nil)
+        routes_registered, db_env, db_name, db_created, events: nil)
         apps = {}
         plan.each do |item|
           say opts, "  [#{item[:name]}] #{item[:url]}"
           app = runner.spawn_http(name: item[:name], hostname: item[:hostname],
             url: item[:url], dir: Dir.pwd, command: item[:command],
             port: item[:port], rails_dev_host: item[:hostname],
-            database_url: db_url, force: opts[:force],
+            database_env: db_env, force: opts[:force],
             extra_env: build_env(resolved, item[:entry], proc_name: item[:name]))
           # Tracked before registration so any later failure (a raise
           # during adopt, a crash while another process waits) can
@@ -285,7 +288,7 @@ module Yamine
           end
         end
         finish_wait(ctx, resolved, apps, wait_result, failed, routes_registered,
-          children, db_name, db_url, db_created, json: opts[:json])
+          children, db_name, db_env, db_created, json: opts[:json])
       end
 
       # --wait epilogue. Success: the summary line and, with --json, the
@@ -294,13 +297,14 @@ module Yamine
       # and removed, the failed phase + that process's log tail is
       # reported, and start exits 1 — no half-booted routes left behind.
       def finish_wait(ctx, resolved, apps, wait_result, failed, routes_registered,
-        children, db_name, db_url, db_created, json: false)
+        children, db_name, db_env, db_created, json: false)
         urls = apps.transform_values { |slot| slot[:item][:url] }
         summary = "ready: #{urls.map { |n, u| "#{n}=#{u}" }.join(" ")}"
 
         if failed.empty?
           payload = Yamine::WaitPayload.success(resolved, wait_result, urls: urls,
-            db_name: db_name, db_url: db_url, created: db_created)
+            db_name: db_name, db_url: db_env && db_env["DATABASE_URL"],
+            created: db_created)
           if json
             $stderr.puts summary
             puts JSON.generate(payload)
@@ -349,10 +353,10 @@ module Yamine
 
       # A background process (proxy: false) is spawned, logged, and
       # supervised exactly like an HTTP one — it just gets no route.
-      def boot_background(ctx, runner, resolved, proc_name, entry, opts, children, db_url)
+      def boot_background(ctx, runner, resolved, proc_name, entry, opts, children, db_env)
         cmd = entry["cmd"].to_s
         if cmd.match?(Yamine::Procfile::COMPOUND)
-          $stderr.puts "  [#{proc_name}] ERROR: compound line (&&, ||, |, ;) — split it into separate processes, or wrap in a script and point cmd: at it"
+          $stderr.puts "  [#{proc_name}] ERROR: compound line (&&, ||, |, ;) — split it into separate processes, or wrap it in a script and point cmd: at it"
           return
         end
         port = opts[:app_port] || Ports.find_free
@@ -360,7 +364,7 @@ module Yamine
         app = runner.boot_run(name: proc_name, hostname: "#{resolved.app}.#{proc_name}.internal",
           url: url, dir: Dir.pwd, command: ["sh", "-c", cmd], port: port,
           force: opts[:force], rails_dev_host: nil, register: false,
-          database_url: db_url,
+          database_env: db_env,
           extra_env: build_env(resolved, entry, proc_name: proc_name))
         children << { name: proc_name, pid: app.pid }
         say opts, "  [#{proc_name}] background (pid #{app.pid})"
@@ -447,23 +451,41 @@ module Yamine
         ENV.fetch("RAILS_ENV", "development")
       end
 
-      # Resolve this worktree's database, create it if missing, and run
-      # the app's schema-load command on first creation. Returns
-      # [created, url]: created is true only when this boot created the
+      # Resolve this checkout's databases, create what's missing, and
+      # run the app's schema-load command on first creation. Returns
+      # [created, env]: created is true only when this boot created a
       # database (so --wait can report provenance without a second
-      # probe), url is nil when there is nothing to inject.
+      # probe), env is the hash of variables to inject into every
+      # process (DATABASE_URL for primary, NAME_DATABASE_URL for the
+      # rest — Rails' own convention) or nil when there is nothing to
+      # inject.
       #
-      # Template sources: ENV DATABASE_URL, then config env.clear
-      # DATABASE_URL (top-level env, falling back to per-process env
-      # for configs written before top-level env existed). Injected
-      # DATABASE_URL overrides database.yml/credentials at connection
-      # time (ActiveRecord merges env over file config), so credentials
-      # apps work — they just need the template declared in config.
+      # Three shapes, decided in order:
+      #   * a multi-database claim (worktree recorded at add time) —
+      #     the whole set is ensured and injected;
+      #   * the main checkout with no template — nothing to isolate:
+      #     the app's own databases ARE its databases. Silence, not
+      #     the old false alarm;
+      #   * a DATABASE_URL template (ENV or config env.clear) — the
+      #     single-database path, unchanged. Injected DATABASE_URL
+      #     overrides database.yml/credentials at connection time
+      #     (ActiveRecord merges env over file config), so credentials
+      #     apps work — they just need the template declared in config.
       def setup_database(ctx, resolved, db_name, events: nil)
         return [false, nil] if db_opted_out?(resolved)
 
+        claim = Database.multi_claim(ctx.store.dir, db_name)
+        return setup_multidb(resolved, claim, events: events) if claim
+
         template = ENV["DATABASE_URL"] || database_template_from_config(resolved)
         if template.nil? || template.strip.empty?
+          if Database.main_checkout?(Dir.pwd) && rails_app?
+            Readiness.phase(:db, "main checkout — the app's own databases", sink: events) do
+              "unsuffixed; worktrees get isolated sets at `yamine worktree add`"
+            end
+            return [false, nil]
+          end
+
           warn_missing_template(resolved)
           return [false, nil]
         end
@@ -483,15 +505,258 @@ module Yamine
         end
         created = db_event.detail.to_s.start_with?("created")
 
+        db_env = { "DATABASE_URL" => url }
         schema_event = Readiness.phase(:schema, "load #{db_name}", sink: events) do
           next "skipped (already exists)" unless created
 
-          run_schema_load(resolved, url)
+          run_schema_load(resolved, db_env)
         end
         unless schema_event.status == "ok"
           $stderr.puts "  [db] WARNING: schema load #{schema_event.status}: #{schema_event.detail}."
         end
-        [created, url]
+        [created, db_env]
+      end
+
+      # Multi-database worktree: the claim carries every database name
+      # plus the app's own URL for each. Create what's missing, hand
+      # back the env for the whole set, load schemas once on first
+      # creation.
+      #
+      # On creation failure the env is STILL handed back. A boot
+      # pointed at missing databases fails loudly at connect time,
+      # while a boot falling back to the shared base databases would
+      # silently run another checkout's tables — the exact bug
+      # isolation exists to kill. Loud and broken beats quiet and
+      # wrong.
+      def setup_multidb(resolved, claim, events: nil)
+        names = claim["names"]
+        bases = claim["bases"] || {}
+        env = db_env_for(names, bases)
+        created = false
+
+        db_event = Readiness.phase(:db, "ensure #{names.size} databases", sink: events) do
+          missing = names.reject { |cfg, actual| Database.exists?(actual, base_for(bases, cfg)) }
+          failed = missing.filter_map do |cfg, actual|
+            actual unless Database.ensure_exists(actual, base_for(bases, cfg))
+          end
+          raise "could not create #{failed.join(', ')} — is postgres/mysql running?" unless failed.empty?
+
+          created = missing.any?
+          created ? "created #{missing.size}" : "already exist"
+        end
+        unless db_event.status == "ok"
+          $stderr.puts "  [db] WARNING: #{db_event.detail}."
+          $stderr.puts "    The isolated URLs stay in effect — the app cannot connect until they exist."
+          $stderr.puts "    Start the database server and re-run `yamine start`."
+        end
+
+        if created
+          schema_event = Readiness.phase(:schema, "load #{names.size} schemas", sink: events) do
+            run_schema_load(resolved, env)
+          end
+          unless schema_event.status == "ok"
+            $stderr.puts "  [db] WARNING: schema load #{schema_event.status}: #{schema_event.detail}."
+          end
+        end
+        [created, env]
+      end
+
+      # The env for a whole claim: DATABASE_URL for primary (the name
+      # every non-Rails framework reads too), NAME_DATABASE_URL per
+      # config — Rails resolves those over database.yml all by
+      # themselves, so even an app whose database.yml never heard of
+      # yamine gets full isolation in supervised processes.
+      def db_env_for(names, bases)
+        names.each_with_object({}) do |(cfg, actual), env|
+          url = Database.url_for(actual, base_for(bases, cfg))
+          next unless url
+
+          env["DATABASE_URL"] = url if cfg == "primary"
+          env["#{cfg.upcase}_DATABASE_URL"] = url
+        end
+      end
+
+      # All of an app's databases live on one server in every layout
+      # yamine has seen; a per-config base with a missing entry falls
+      # back to the first.
+      def base_for(bases, cfg)
+        bases[cfg] || bases.values.first
+      end
+
+      # Worktree-add (and `yamine db create` in a worktree): turn a
+      # probe of the app's real databases into a provisioned,
+      # self-describing claim. Names come from the app, the suffix
+      # from the directory; claim + .env files are written BEFORE any
+      # database is created, so a failure mid-way leaves a state the
+      # next `yamine db create` or boot can finish from — never a
+      # half-named set with no record of it.
+      #
+      # Returns the final names; raises Error only for conditions the
+      # user must fix (name budget, server down, .env breaking the
+      # app's boot) — those keep the worktree and the claim.
+      def provision_multidb(ctx, dir, claim_key, rows)
+        env_name = rails_env
+        # Re-provisioning a worktree whose .env exists: the app (via
+        # dotenv) resolves SUFFIXED names — strip the claim's stored
+        # suffix before fitting, or this pass would double it and
+        # yamine would provision a set the app never connects to.
+        prior = Database.load_map(ctx.store.dir)[claim_key]
+        prior_suffix = prior.is_a?(Hash) ? prior["suffix"] : nil
+        rows = strip_suffix(rows, prior_suffix)
+        # The test environment's database joins the claim: worktree
+        # tests run against it (.env.test carries its URL), and
+        # teardown must drop it — a leaked test database per worktree
+        # is exactly the leftover this exists to prevent. Test configs
+        # are flat (implicitly named "primary"), so the claim key is
+        # namespaced under "test" to not collide with the dev primary.
+        # A test env the app cannot answer (missing test credentials,
+        # say) degrades to a dev-only claim; `yamine db create` heals.
+        test_rows = Probe.rails_databases(dir, env: "test")
+        test_rows = strip_suffix(test_rows, prior_suffix)
+        test_pairs = (test_rows && Probe.server_backed(test_rows) || [])
+          .map { |r| [env_namespaced_key(r["name"], "test"), r] }
+
+        bases = rows.to_h { |r| [r["name"], r["url"]] }
+          .merge(test_pairs.to_h { |key, r| [key, r["url"]] })
+        suffix = Database.suffix_for(claim_key, env_name)
+        all_bases = rows.map { |r| r["database"] } + test_pairs.map { |_, r| r["database"] }
+        fitted = Database.fit_suffix(suffix, dir, all_bases)
+        unless fitted
+          longest = all_bases.max_by(&:to_s)&.bytesize
+          raise Error,
+            "a database name of #{longest} bytes leaves no room for a worktree " \
+            "suffix (#{Database::MAX_IDENTIFIER_BYTES}-byte limit): #{all_bases.inspect}\n" \
+            "  Shorten the base names in the app's database configuration."
+        end
+        names = rows.to_h { |r| [r["name"], Database.multiname(r["database"], fitted)] }
+          .merge(test_pairs.to_h { |key, r| [key, Database.multiname(r["database"], fitted)] })
+
+        map = Database.load_map(ctx.store.dir)
+        map[claim_key] = (map[claim_key] || {}).merge(
+          "dir" => File.expand_path(dir),
+          "claimed_at" => Time.now.utc.iso8601,
+          "suffix" => fitted, "names" => names, "bases" => bases)
+        Database.save_map(ctx.store.dir, map)
+        # The environment files: what hand-run commands read. Written
+        # before creation so the verify probe below can see them.
+        dev_env = db_env_for(
+          names.reject { |cfg, _| cfg == "test" || cfg.start_with?("test_") }, bases)
+        test_url = names["test"] &&
+          Database.url_for(names["test"], base_for(bases, "test"))
+        Database.write_env_files(dir, dev_env, test_url: test_url)
+        Database.exclude_files(dir, Database::ENV_FILES)
+
+        # An empty test database is worse than none: Rails won't
+        # auto-load schema into it (verified — first `rails test`
+        # fails on missing schema_migrations, not a clear
+        # NoDatabaseError). So when THIS pass created the test DB,
+        # prepare it too — after that, tests just run.
+        test_needs_schema = names["test"] &&
+          !Database.exists?(names["test"], base_for(bases, "test"))
+
+        Dir.chdir(dir) { setup_database(ctx, Resolver.resolve(dir), claim_key) }
+
+        missing = names.reject { |cfg, actual| Database.exists?(actual, base_for(bases, cfg)) }
+        unless missing.empty?
+          raise Error,
+            "could not create #{missing.values.join(', ')} — is the database server running?\n" \
+            "  The worktree and its claim are kept; start postgres/mysql and run `yamine db create` here."
+        end
+
+        if test_needs_schema
+          # RAILS_ENV=test, not development: dotenv must load .env.test
+          # (PRIMARY_DATABASE_URL = this worktree's test URL) AND the
+          # environment-override merge only applies to the CURRENT
+          # env's configs — under development the flat test config
+          # would resolve to the base name and purge+load the MAIN
+          # checkout's test database instead of this worktree's.
+          prepared = Dir.chdir(dir) do
+            system({ "RAILS_ENV" => "test" }, "sh", "-c", "bin/rails db:test:prepare",
+              out: File::NULL, err: File::NULL)
+          end
+          if prepared
+            puts "  test database schema ready — tests run as-is"
+          else
+            warn "  WARNING: could not prepare the test database — run `bin/rails db:test:prepare` in #{dir} before running tests"
+          end
+        end
+
+        verify_env_loading(dir, names)
+        names
+      end
+
+      def strip_suffix(rows, suffix)
+        return rows if rows.nil? || suffix.nil? || suffix.empty?
+
+        rows.map do |r|
+          if r["database"].end_with?("_#{suffix}")
+            r.merge("database" => r["database"].delete_suffix("_#{suffix}"))
+          else
+            r
+          end
+        end
+      end
+
+      # Flat configs (a lone `test:` section) resolve as "primary" —
+      # the claim key must not collide with the development primary.
+      def env_namespaced_key(cfg_name, env)
+        cfg_name == "primary" ? env : "#{env}_#{cfg_name}"
+      end
+
+      # The proof, not the promise: re-probe now that .env exists.
+      # Matching names mean something inside the app loads .env
+      # (dotenv-rails or any dotenv loader) — hand-run console/test/
+      # migrate commands are isolated too. Mismatch is not fatal
+      # (supervised boots isolate via injected env regardless), but
+      # it is the one gap worth shouting about at the exact moment of
+      # creation. A probe that cannot run at all after writing .env
+      # IS fatal: the app no longer boots, and .env's own authorship
+      # is the likely reason.
+      def verify_env_loading(dir, names)
+        rows = Probe.rails_databases(dir)
+        unless rows
+          raise Error,
+            "the app no longer boots after writing .env — " \
+            "check the generated .env files in #{dir}"
+        end
+
+        actual = rows.to_h { |r| [r["name"], r["database"]] }
+        # Keys absent from this probe are env-namespaced (the test
+        # database, probed separately) — the dev probe cannot vouch
+        # for them either way; skip, don't flag.
+        mismatched = names.reject { |cfg, want| !actual.key?(cfg) || actual[cfg] == want }
+        if mismatched.empty?
+          puts "  .env loaded — hand-run commands are isolated too"
+        else
+          cfg, want = mismatched.first
+          env_url = rows.first["env_database_url"]
+          if env_url && !env_url.empty?
+            # The loader DID run — DATABASE_URL reached the app — yet
+            # resolution ignored it. database.yml supplies `url:` keys
+            # (credentials-style), and for those Rails gives the FILE
+            # precedence: merge_db_environment_variables skips configs
+            # that are already URL-shaped. No environment channel,
+            # spawned or loaded, can redirect this app in that form.
+            warn "  WARNING: #{cfg} resolves to #{actual[cfg].inspect} even though " \
+                 "DATABASE_URL says #{env_url.split("/").last.inspect}."
+            warn "    database.yml's `url:` keys take precedence over the environment in this configuration —"
+            warn "    neither injected env nor .env can redirect them. Switch development/test to"
+            warn "    component form (database:, host:, username:) so DATABASE_URL and .env apply —"
+            warn "    see the yamine README's multi-database section."
+          else
+            warn "  WARNING: this app does not load .env " \
+                 "(#{cfg} resolves to #{actual[cfg].inspect}, expected #{want.inspect})."
+            warn "    Supervised boots are still isolated via injected env, but `rails console`, " \
+                 "`rails test`, and `db:migrate` run by hand in this worktree would use the main checkout's databases."
+            warn "    Fix once: add `gem \"dotenv-rails\", groups: [:development, :test]` to the Gemfile — " \
+                 "any dotenv loader works."
+          end
+        end
+      rescue StandardError => e
+        raise Error, e.message if e.is_a?(Error)
+
+        raise Error,
+          "the app no longer boots after writing .env: #{e.message}"
       end
 
       # Top-level `db: false` opts out of per-worktree databases.
@@ -548,13 +813,17 @@ module Yamine
       # Schema-load on first boot uses the app's own command when declared
       # (top-level db.schema_load in config/local.yml), else a Rails
       # default when a Rails app is detected, else nothing (frameworks
-      # without a schema concept need no step). Returns a detail string.
-      def run_schema_load(resolved, url)
+      # without a schema concept need no step). db_env is the whole
+      # injection hash — with a multi-database claim, Rails' schema
+      # tasks iterate every config and each resolves to its own
+      # isolated database. Returns a detail string.
+      def run_schema_load(resolved, db_env)
         cmd = db_schema_load(resolved) ||
           ("bin/rails db:schema:load" if rails_app?)
         return "no schema-load command" unless cmd
 
-        ok = system({ "DATABASE_URL" => url, "RAILS_ENV" => rails_env },
+        env = { "RAILS_ENV" => rails_env }.merge(db_env || {})
+        ok = system(env,
           "sh", "-c", cmd, chdir: Dir.pwd, out: File::NULL, err: File::NULL)
         ok ? "loaded schema via #{cmd}" : raise("schema-load exited non-zero: #{cmd}")
       end