dockside 0.1.0

data/lib/dockside/dependency.rb

@@ -0,0 +1,216 @@
+require "fileutils"
+
+module Dockside
+  # One service of the compose file in one environment: start, stop, reset, exec.
+  class Dependency
+    LOG_LINES_ON_ERROR = 50
+
+    attr_reader :name, :project, :settings, :config
+
+    def initialize(name:, project:, settings:, config:, shared: false)
+      @name = name
+      @project = project
+      @settings = settings
+      @config = config
+      @shared = shared
+    end
+
+    def env
+      project.env
+    end
+
+    def shared?
+      @shared
+    end
+
+    def autostart?
+      settings.autostart?
+    end
+
+    def port
+      published = config.dig("ports", 0, "published")
+      published&.to_s&.[](/\d+/)&.to_i
+    end
+
+    def container_port
+      config.dig("ports", 0, "target")
+    end
+
+    def url
+      "http://localhost:#{port}" if port
+    end
+
+    def container_name
+      "#{project.name}-#{name}-1"
+    end
+
+    def running?
+      container&.running? || false
+    end
+
+    def ready?
+      running? && probe.ready?
+    end
+
+    def start(build: false, pull: false)
+      with_lock do
+        docker.preflight!
+        create_bind_mount_directories
+        return if !build && !pull && reuse_running_container
+
+        Dockside.log "starting #{name} (#{env})"
+        compose_up(build: build, pull: pull)
+        wait_until_ready
+        provisioner.run(container.id)
+        remember_config
+        Dockside.log "#{name} ready at #{url}"
+      end
+    end
+
+    def ensure_running!
+      start
+    end
+
+    def stop
+      compose.stop(name)
+    end
+
+    def reset
+      remove
+      start
+    end
+
+    # Removes the container, its volumes and the content of its data folders under tmp/.
+    def remove
+      compose.remove(name)
+      remove_shared_container
+      named_volumes.each { |volume| docker.remove_volume(volume) }
+      bind_mount_sources.select { |source| source.to_s.start_with?(project.root.join("tmp").to_s) }.each { |source| wipe(source) }
+      FileUtils.rm_f(provisioner.marker)
+    end
+
+    def remove_shared_container
+      return unless shared?
+
+      other = docker.find_container(project: "#{project.app_name}-#{project.other_env}", service: name)
+      docker.remove_container(other.id) if other
+    end
+
+    def exec(command, environment: {}, stdin: nil, workdir: nil, user: nil, allow_failure: false)
+      result = compose.exec(name, command, environment: environment, stdin: stdin, workdir: workdir, user: user)
+      raise CommandFailed.new(result) unless result.success? || allow_failure
+
+      result.stdout
+    end
+
+    def exec_succeeds?(command)
+      compose.exec(name, command).success?
+    end
+
+    def logs(tail: 100)
+      compose.logs(name, tail: tail).stdout
+    end
+
+    def probe
+      @probe ||= Readiness.probe_for(self, settings.ready)
+    end
+
+    def provisioner
+      @provisioner ||= Provisioner.new(self)
+    end
+
+    private
+
+    def compose
+      project.compose
+    end
+
+    def docker
+      project.docker
+    end
+
+    def container
+      docker.find_container(project: project.name, service: name)
+    end
+
+    def with_lock
+      project.work_dir.mkpath
+      File.open(project.work_dir.join(".lock"), File::RDWR | File::CREAT) do |file|
+        file.flock(File::LOCK_EX)
+        yield
+      end
+    end
+
+    def reuse_running_container
+      if !config_changed? && ready?
+        Dockside.log "#{name} already running at #{url}"
+        return true
+      end
+      return false unless shared?
+
+      other = docker.find_container(project: "#{project.app_name}-#{project.other_env}", service: name)
+      return false unless other&.running? && probe.ready?
+
+      Dockside.log "#{name} shared with #{project.other_env} at #{url}"
+      true
+    end
+
+    def compose_up(build:, pull:)
+      result = compose.up(name, build: build, pull: pull, timeout: settings.timeout)
+      return if result.success?
+
+      output = result.stdout
+      if output.match?(/port is already allocated|address already in use/i)
+        raise PortInUse, "#{name} (#{env}) could not start because port #{port} is in use. " \
+          "Run `docker ps` to see whether another container holds it.\n#{output}"
+      end
+
+      raise CommandFailed.new(result, "#{name} (#{env}) failed to start:\n#{output}\nLast #{LOG_LINES_ON_ERROR} log lines:\n#{last_logs}")
+    end
+
+    def wait_until_ready
+      Readiness.wait(probe, timeout: settings.timeout)
+    rescue ReadyTimeout
+      raise ReadyTimeout, "#{name} (#{env}) did not become ready within #{settings.timeout}s " \
+        "(waited until #{probe}). Last #{LOG_LINES_ON_ERROR} log lines:\n#{last_logs}"
+    end
+
+    def last_logs
+      logs(tail: LOG_LINES_ON_ERROR).chomp
+    end
+
+    def config_file
+      project.work_dir.join("#{name}.config.json")
+    end
+
+    def config_changed?
+      !config_file.exist? || config_file.read != JSON.generate(config)
+    end
+
+    def remember_config
+      config_file.write(JSON.generate(config))
+    end
+
+    def create_bind_mount_directories
+      bind_mount_sources.select { |source| source.to_s.start_with?(project.root.to_s) }.each do |source|
+        FileUtils.mkdir_p(source)
+        FileUtils.chmod(0o777, source)
+      end
+    end
+
+    # Empties the folder but keeps it: Docker Desktop keeps a deleted folder mounted as a stale mount.
+    def wipe(directory)
+      return unless directory.directory?
+
+      directory.children.each { |child| FileUtils.rm_rf(child) }
+    end
+
+    def bind_mount_sources
+      Array(config["volumes"]).select { |volume| volume["type"] == "bind" }.map { |volume| Pathname.new(volume["source"]) }
+    end
+
+    def named_volumes
+      Array(config["volumes"]).select { |volume| volume["type"] == "volume" }.map { |volume| "#{project.name}_#{volume["source"]}" }
+    end
+  end
+end

data/lib/dockside/docker.rb

@@ -0,0 +1,61 @@
+module Dockside
+  # The docker commands the gem needs besides compose: checks, container lookup, removal.
+  class Docker
+    Container = Struct.new(:id, :state) do
+      def running?
+        state == "running"
+      end
+    end
+
+    def initialize(runner)
+      @runner = runner
+      @checked = false
+    end
+
+    def preflight!
+      return if @checked
+
+      check_daemon
+      check_compose
+      @checked = true
+    end
+
+    # The container compose created for the service in the project, or nil.
+    def find_container(project:, service:)
+      result = @runner.run!([
+        "docker", "ps", "--all",
+        "--filter", "label=com.docker.compose.project=#{project}",
+        "--filter", "label=com.docker.compose.service=#{service}",
+        "--format", "{{.ID}} {{.State}}"
+      ])
+      id, state = result.stdout.lines.first&.split
+      Container.new(id, state) if id
+    end
+
+    def remove_container(id)
+      @runner.run!(["docker", "rm", "--force", "--volumes", id])
+    end
+
+    def remove_volume(name)
+      @runner.run!(["docker", "volume", "rm", "--force", name])
+    end
+
+    private
+
+    def check_daemon
+      result = @runner.run(["docker", "version", "--format", "{{.Server.Version}}"])
+      return if result.success?
+
+      raise DockerUnavailable, "Docker is installed but the daemon does not answer: #{result.stderr.strip}\n" \
+        "Start Docker, or start with DOCKSIDE_AUTOSTART=0 to skip the containers."
+    end
+
+    def check_compose
+      result = @runner.run(["docker", "compose", "version"])
+      return if result.success?
+
+      raise ComposeMissing, "The docker compose plugin is missing. Install it from " \
+        "https://docs.docker.com/compose/install/ (docker-compose v1 is not supported)."
+    end
+  end
+end

data/lib/dockside/errors.rb

@@ -0,0 +1,42 @@
+module Dockside
+  class Error < StandardError; end
+
+  class ConfigError < Error; end
+
+  class UnknownDependency < Error; end
+
+  class DockerMissing < Error; end
+
+  class DockerUnavailable < Error; end
+
+  class ComposeMissing < Error; end
+
+  class ReadyTimeout < Error; end
+
+  class PortInUse < Error; end
+
+  class CommandFailed < Error
+    attr_reader :argv, :stdout, :stderr, :exit_status
+
+    def initialize(result, message = nil)
+      @argv = result.argv
+      @stdout = result.stdout
+      @stderr = result.stderr
+      @exit_status = result.exit_status
+      super(message || default_message)
+    end
+
+    # The last lines the command printed, stderr first.
+    def output_tail(lines = 10)
+      [stderr, stdout].map(&:strip).reject(&:empty?).join("\n").lines.last(lines).join.strip
+    end
+
+    private
+
+    def default_message
+      message = "`#{argv.join(" ")}` failed with status #{exit_status}"
+      details = [stderr, stdout].map(&:strip).reject(&:empty?).join("\n")
+      details.empty? ? message : "#{message}:\n#{details}"
+    end
+  end
+end

data/lib/dockside/override.rb

@@ -0,0 +1,51 @@
+module Dockside
+  # The compose override file the gem generates for one environment: the environment block of every
+  # service plus the conventions (image tag for built services, host.docker.internal).
+  class Override
+    def initialize(project)
+      @project = project
+    end
+
+    def to_hash
+      services = @project.raw_services.to_h do |service, definition|
+        [service, service_override(service, definition)]
+      end
+      {"services" => services}
+    end
+
+    def to_yaml
+      tree = Psych::Visitors::YAMLTree.create
+      tree << to_hash
+      stream = tree.tree
+      mark_environment_lists_as_override(stream.children.first.root)
+      stream.to_yaml
+    end
+
+    private
+
+    def service_override(service, definition)
+      override = Settings.compose_overrides(definition, @project.env)
+      override["extra_hosts"] = Array(override["extra_hosts"] || definition["extra_hosts"]) | [Project::HOST_GATEWAY]
+      override["image"] = "#{@project.app_name}-#{service}" if definition.key?("build") && !definition.key?("image")
+      override
+    end
+
+    def environment_keys(service)
+      Settings.compose_overrides(@project.raw_services.fetch(service), @project.env).keys
+    end
+
+    # Lists in an environment block replace the list of the service instead of being appended to it.
+    def mark_environment_lists_as_override(root)
+      services = root.children.last
+      services.children.each_slice(2) do |name, service|
+        replaced = environment_keys(name.value)
+        service.children.each_slice(2) do |key, value|
+          next unless value.is_a?(Psych::Nodes::Sequence) && replaced.include?(key.value)
+
+          value.tag = "!override"
+          value.implicit = false
+        end
+      end
+    end
+  end
+end

data/lib/dockside/project.rb

@@ -0,0 +1,133 @@
+module Dockside
+  # The app's compose file and the files the gem derives from it for one environment.
+  class Project
+    ENVIRONMENTS = %w[development test].freeze
+    HOST_GATEWAY = "host.docker.internal:host-gateway".freeze
+
+    attr_reader :root, :env, :app_name, :runner
+
+    def initialize(root:, env:, app_name:, runner:)
+      @root = Pathname.new(root)
+      @env = env.to_s
+      @app_name = app_name
+      @runner = runner
+    end
+
+    def name
+      "#{app_name}-#{env}"
+    end
+
+    def other_env
+      (env == "test") ? "development" : "test"
+    end
+
+    def compose_file
+      root.join("config/dockside.yml")
+    end
+
+    def env_file
+      file = root.join("config/dockside.#{env}.yml")
+      file if file.exist?
+    end
+
+    def work_dir
+      root.join("tmp/dockside", env)
+    end
+
+    def override_file
+      work_dir.join("override.yml")
+    end
+
+    # The app's compose file without the x-dockside part, so compose does not interpolate
+    # ${...} inside after_start commands or warn about them.
+    def plain_file
+      work_dir.join("dockside.yml")
+    end
+
+    def files
+      [plain_file, env_file, override_file].compact
+    end
+
+    def compose_env
+      {"RAILS_ENV" => env, "DOCKSIDE_ENV" => env}
+    end
+
+    def compose
+      @compose ||= Compose.new(self)
+    end
+
+    def docker
+      @docker ||= Docker.new(runner)
+    end
+
+    def service_names
+      raw_services.keys
+    end
+
+    def settings(service)
+      Settings.for(service, raw_services.fetch(service), env)
+    end
+
+    # The first host port the service publishes in the given environment, read from the file
+    # before compose resolves it. Used to find out whether two environments share a container.
+    def host_port(service, environment)
+      definition = raw_services.fetch(service)
+      ports = definition.dig("x-dockside", environment, "ports") || definition["ports"] || []
+      first = ports.first
+      return if first.nil?
+
+      published = first.is_a?(Hash) ? first["published"] : first.to_s.split(":")[-2]
+      published&.to_s
+    end
+
+    # Writes the gem's files and lets compose merge everything. Returns the resolved services.
+    def resolve!
+      write_files
+      compose.config.fetch("services")
+    end
+
+    # The compose file with everything merged, ready for a plain `docker compose -f`.
+    def resolved_yaml
+      write_files
+      compose.config_yaml
+    end
+
+    def write_files
+      work_dir.mkpath
+      plain_file.write(plain_document.to_yaml)
+      override_file.write(Override.new(self).to_yaml)
+    end
+
+    def raw_services
+      raw_document.fetch("services")
+    end
+
+    private
+
+    def raw_document
+      @raw_document ||= load_raw_document
+    end
+
+    def load_raw_document
+      raise ConfigError, "#{compose_file} does not exist. Run `bin/rails generate dockside:install`." unless compose_file.exist?
+
+      document = YAML.safe_load(compose_file.read, aliases: true) || {}
+      document["services"] ||= {}
+      document["services"].each { |service, definition| validate(service, definition) }
+      document
+    end
+
+    def plain_document
+      services = raw_services.transform_values { |definition| definition.except("x-dockside") }
+      raw_document.merge("services" => services)
+    end
+
+    def validate(service, definition)
+      if definition.key?("container_name")
+        raise ConfigError, "Service #{service} sets container_name. dockside names containers " \
+          "#{app_name}-<environment>-#{service}-1, remove container_name."
+      end
+      Settings.validate(service, definition["x-dockside"])
+    end
+  end
+end

data/lib/dockside/provisioner.rb

@@ -0,0 +1,81 @@
+module Dockside
+  # Runs the after_start steps of a dependency once per container.
+  class Provisioner
+    Step = Struct.new(:exec, :run, :environment, :workdir, :user, :stdin, :allow_failure, :always)
+
+    class Step
+      KEYS = %w[exec run environment workdir user stdin allow_failure always].freeze
+
+      def self.parse_all(service, steps)
+        Array(steps).map { |step| parse(service, step) }
+      end
+
+      def self.parse(service, step)
+        unless step.is_a?(Hash) && (step.keys - KEYS).empty? && [step["exec"], step["run"]].compact.one?
+          raise ConfigError, "#{service}: every after_start step needs exec: or run:, " \
+            "with the options #{(KEYS - %w[exec run]).join(", ")}. Got #{step.inspect}"
+        end
+
+        new(**step.transform_keys(&:to_sym))
+      end
+
+      def description
+        exec ? "exec #{Array(exec).join(" ")}" : "run #{Array(run).join(" ")}"
+      end
+    end
+
+    def initialize(dependency)
+      @dependency = dependency
+    end
+
+    def marker
+      @dependency.project.work_dir.join("#{@dependency.name}.provisioned")
+    end
+
+    def provisioned?(container_id)
+      marker.exist? && marker.read.strip == container_id
+    end
+
+    # Runs every step for a new container, only the `always` steps for a known one.
+    def run(container_id)
+      due = !provisioned?(container_id)
+      @dependency.settings.after_start.each do |step|
+        run_step(step) if due || step.always
+      end
+      marker.write(container_id) if due
+    end
+
+    private
+
+    def run_step(step)
+      Dockside.log "#{@dependency.name}: #{step.description}"
+      environment = injected_environment.merge((step.environment || {}).transform_keys(&:to_s))
+      stdin = step.stdin && @dependency.project.root.join(step.stdin).read
+      if step.exec
+        @dependency.exec(step.exec, environment: environment, stdin: stdin, workdir: step.workdir, user: step.user)
+      else
+        run_on_host(step, environment, stdin)
+      end
+    rescue CommandFailed => error
+      raise unless step.allow_failure
+
+      Dockside.log "#{@dependency.name}: the step failed, moving on because of allow_failure:\n#{error.output_tail}"
+    end
+
+    def run_on_host(step, environment, stdin)
+      command = step.run.is_a?(Array) ? step.run : ["sh", "-c", step.run]
+      result = @dependency.project.runner.run(command, env: environment, stdin: stdin, chdir: @dependency.project.root)
+      raise CommandFailed.new(result) unless result.success?
+    end
+
+    def injected_environment
+      {
+        "DOCKSIDE_NAME" => @dependency.name,
+        "DOCKSIDE_ENV" => @dependency.env,
+        "DOCKSIDE_PORT" => @dependency.port.to_s,
+        "DOCKSIDE_URL" => @dependency.url.to_s,
+        "DOCKSIDE_CONTAINER" => @dependency.container_name
+      }
+    end
+  end
+end

data/lib/dockside/railtie.rb

@@ -0,0 +1,16 @@
+require "rails/railtie"
+
+module Dockside
+  class Railtie < Rails::Railtie
+    config.dockside = ActiveSupport::OrderedOptions.new
+    config.dockside.autostart = nil
+
+    rake_tasks do
+      load File.expand_path("../tasks/dockside.rake", __dir__)
+    end
+
+    config.after_initialize do |app|
+      Autostart.boot(app.config.dockside, env: Rails.env, server_process: defined?(Rails::Server))
+    end
+  end
+end