dockerfile-rails 1.3.0 → 1.4.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8780a8a7015d339452edf4dff330981a748ad72c72081e0cfa473c835121637c
-  data.tar.gz: ab37e68fe9d3f82f9368e45738a12f171eaaabcff8b311451d89bdbce1217d95
+  metadata.gz: 70cab322c4f52cdddb9934de220be781a87ef306af7a21bf1db8a15c8db16a3a
+  data.tar.gz: a8363fcbdd774e3a2b3254f274366c0ad0f700eabdc90502a189c727005de25e
 SHA512:
-  metadata.gz: 8a1bce2d42a5916ae7bf4cc49f23d0b9077533e1447015f35bfd4835397438a8bb4bc13e48a774a7e80130cf0f99dada597fda1fa0cada1674e52073d7846fc1
-  data.tar.gz: 9793fdc06ffe7a84a1e0f40282570ea5e4ad6e08514ff8210f805254a093208cc4b98f683c44fce299dc0b71ff04c2418b847c5e33f05da407cefd617412521b
+  metadata.gz: 10fe62c57fe8f2088bbe0b58feafc4199e2591cf214d39e410d1e78be3f5039187ee28ed9279f774eb02ec991bd46318a014fa324bfc57fe16c9398c51844e10
+  data.tar.gz: 9bcc4bca316a3c4eb96a00060e1f3cbc45072975a28c16248353947da1e9259121fdbf7213e3a382cd407f48a1eb0d1b3baec65a9461242f1269f376a887d291

data/README.md

@@ -52,6 +52,7 @@ Generally the dockerfile generator will be able to determine what dependencies y
 are actually using.  But should you be using DATABASE_URL, for example, at runtime
 additional support may be needed:
 
+* `--litefs` - use [LiteFS](https://fly.io/docs/litefs/)
 * `--mysql` - add mysql libraries
 * `--postgresql` - add postgresql libraries
 * `--redis` - add redis libraries
@@ -110,9 +111,9 @@ If you are running a single test, the following environment variables settings m
  * `TEST_CAPTURE=1` will capture test results.
  * `TEST_KEEP=1` will leave the test app behind for inspection after the test completes.
 
-## Links
+## Historical Links
 
-The following links relate to the current development status with respect to Rails 7.1 and will be removed once that is resolved.
+The following links relate to the coordination between this package and Rails 7.1.
 
 * [Preparations for Rails 7.1](https://community.fly.io/t/preparations-for-rails-7-1/9512) - [Fly.io](https://fly.io/)'s plans and initial discussions with DHH
 * [Rails Dockerfile futures](https://discuss.rubyonrails.org/t/rails-dockerfile-futures/82091/1) - rationale for a generator

data/lib/generators/dockerfile_generator.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require "erb"
+require "json"
 require_relative "../dockerfile-rails/scanner.rb"
 
 class DockerfileGenerator < Rails::Generators::Base
@@ -15,6 +16,7 @@ class DockerfileGenerator < Rails::Generators::Base
     "jemalloc" => false,
     "label" => {},
     "link" => true,
+    "litefs" => false,
     "lock" => true,
     "max-idle" => nil,
     "mysql" => false,
@@ -111,6 +113,9 @@ class DockerfileGenerator < Rails::Generators::Base
   class_option :sqlite3, aliases: "--sqlite", type: :boolean, default: OPTION_DEFAULTS.sqlite3,
     desc: "include sqlite3 libraries"
 
+  class_option :litefs, type: :boolean, default: OPTION_DEFAULTS.litefs,
+    desc: "replicate sqlite3 databases using litefs"
+
   class_option :postgresql, aliases: "--postgres", type: :boolean, default: OPTION_DEFAULTS.postgresql,
     desc: "include postgresql libraries"
 
@@ -233,6 +238,23 @@ class DockerfileGenerator < Rails::Generators::Base
 
     template "docker-compose.yml.erb", "docker-compose.yml" if options.compose
 
+    if using_litefs?
+      template "litefs.yml.erb", "config/litefs.yml"
+
+      fly_attach_consul
+    end
+
+    if fly_processes # therefore File.exist?('fly.toml')
+      if File.stat("fly.toml").size > 0
+        template "fly.toml.erb", "fly.toml"
+      else
+        toml = fly_make_processes(fly_processes)
+        if toml != IO.read("fly.toml")
+          File.write "fly.toml", toml
+        end
+      end
+    end
+
     if @gemfile.include?("vite_ruby")
       package = JSON.load_file("package.json")
       unless package.dig("scripts", "build")
@@ -296,6 +318,10 @@ private
     options.root?
   end
 
+  def using_litefs?
+    options.litefs?
+  end
+
   def using_node?
     return @using_node if @using_node != nil
     @using_node = File.exist? "package.json"
@@ -495,6 +521,9 @@ private
     packages << "default-mysql-client" if options.mysql? || @mysql
     packages << "libjemalloc2" if options.jemalloc? && !options.fullstaq?
 
+    # litefs
+    packages += ["ca-certificates", "fuse3", "sudo"] if options.litefs?
+
     # ActiveStorage preview support
     packages << "libvips" if @gemfile.include? "ruby-vips"
 
@@ -601,13 +630,21 @@ private
   def deploy_env
     env = {}
 
-    env["PORT"] = "3001" if options.nginx? && !using_passenger?
+    env["PORT"] = "3001" if (options.nginx? && !using_passenger?) || using_litefs?
 
     if Rails::VERSION::MAJOR < 7 || Rails::VERSION::STRING.start_with?("7.0")
       env["RAILS_LOG_TO_STDOUT"] = "1"
       env["RAILS_SERVE_STATIC_FILES"] = "true" unless options.nginx?
     end
 
+    if deploy_database == "sqlite3"
+      if using_litefs?
+        env["DATABASE_URL"] = "sqlite3:///litefs/production.sqlite3"
+      else
+        env["DATABASE_URL"] = "sqlite3:///data/production.sqlite3"
+      end
+    end
+
     if options.yjit?
       env["RUBY_YJIT_ENABLE"] = "1"
     end
@@ -639,7 +676,7 @@ private
 
     env.merge! @@vars["base"] if @@vars["base"]
 
-    env.map { |key, value| "#{key}=#{value.inspect}" }
+    env.map { |key, value| "#{key}=#{value.inspect}" }.sort
   end
 
   def base_args
@@ -716,9 +753,11 @@ private
   end
 
   def deploy_database
-    if options.postgresql? || @postgresql
+    # note: as database can be overridden at runtime via DATABASE_URL,
+    # use presence of "pg" or "mysql2" in the bundle as evidence of intent.
+    if options.postgresql? || @postgresql || @gemfile.include?("pg")
       "postgresql"
-    elsif options.mysql? || @mysql
+    elsif options.mysql? || @mysql || @gemfile.include?("mysql2")
       "mysql"
     else
       "sqlite3"
@@ -823,6 +862,21 @@ private
     end
   end
 
+  def fly_processes
+    return unless File.exist? "fly.toml"
+    return unless using_sidekiq?
+
+    if procfile.size > 1
+      list = { "app" => "foreman start --procfile=Procfile.prod" }
+    else
+      list = { "app" => procfile.values.first }
+    end
+
+    list["sidekiq"] = "bundle exec sidekiq"
+
+    list
+  end
+
   def more_docker_ignores
     more = ""
 
@@ -853,4 +907,54 @@ private
   rescue ArgumentError
     nil
   end
+
+  # if running on fly v2, make a best effort to attach consul
+  def fly_attach_consul
+    # certainly not fly unless there is a fly.toml
+    return unless File.exist? "fly.toml"
+
+    # Check fly.toml to guess if v1 or v2
+    toml = File.read("fly.toml")
+    return if toml.include?("enable_consul") # v1-ism
+    return unless toml.include?("primary_region") # v2
+
+    # see if flyctl is in the path
+    paths = ENV["PATH"].split(File::PATH_SEPARATOR)
+    cmds = %w(flyctl)
+    exts = ENV["PATHEXT"] ? ENV["PATHEXT"].split(";") : [""]
+    flyctl = Enumerator.product(paths, cmds, exts).
+      map { |path, cmd, ext| File.join(path, "#{cmd}#{ext}") }.
+      find { |path| File.executable? path }
+    return unless flyctl
+
+    # see if secret is already set?
+    begin
+      secrets = JSON.parse(`#{flyctl} secrets list --json`)
+      return if secrets.any? { |secret| secret["Name"] == "FLY_CONSUL_URL" }
+    rescue
+      return # likely got an error like "Could not find App"
+    end
+
+    # attach consul
+    say_status :execute, "flyctl consul attach", :green
+    system "#{flyctl} consul attach"
+  end
+
+  def fly_make_processes(list)
+    toml = File.read("fly.toml")
+
+    if toml.include? "[processes]"
+      toml.sub!(/\[processes\].*?(\n\n|\n?\z)/m, "[processes]\n" +
+        list.map { |name, cmd| "  #{name} = #{cmd.inspect}" }.join("\n") + '\1')
+    else
+      toml += "\n[processes]\n" +
+        list.map { |name, cmd| "  #{name} = #{cmd.inspect}\n" }.join
+
+      app = list.has_key?("app") ? "app" : list.keys.first
+
+      toml.sub! "[http_service]\n", "\\0  processes = [#{app.inspect}]\n"
+    end
+
+    toml
+  end
 end

data/lib/generators/templates/Dockerfile.erb

@@ -134,6 +134,12 @@ RUN SECRET_KEY_BASE<%= Rails::VERSION::MAJOR<7 || Rails::VERSION::STRING.start_w
 # Final stage for app image
 FROM base
 
+<% end -%>
+<% if using_litefs? -%>
+# Install, configure litefs
+COPY --from=flyio/litefs:0.4.0 /usr/local/bin/litefs /usr/local/bin/litefs
+COPY<% if options.link? %> --link<% end %> config/litefs.yml /etc/litefs.yml
+
 <% end -%>
 <% unless deploy_args.empty? -%>
 # Deployment build arguments
@@ -166,7 +172,11 @@ COPY --from=client /rails/<%= api_client_dir %>/build /rails/public
 <% end -%>
 
 <% end -%>
-<% unless run_as_root? -%>
+<% if run_as_root? -%>
+<% if deploy_database == 'sqlite3' -%>
+RUN mkdir /data
+<% end -%>
+<% else -%>
 # Run and own only the runtime files as a non-root user for security
 <% if options.compose? -%>
 ARG UID=1000 \
@@ -182,11 +192,23 @@ RUN useradd rails --create-home --shell /bin/bash && \
 <% if deploy_packages.include?("sudo") && options.sudo? -%>
     sed -i 's/env_reset/env_keep="*"/' /etc/sudoers && \
 <% end -%>
+<% if deploy_database == 'sqlite3' -%>
+    mkdir /data<% if using_litefs? %> /litefs<% end %> && \
+    chown -R rails:rails <%= Dir[*%w(db log storage tmp)].join(" ") %> /data<% if using_litefs? %> /litefs<% end %>
+<% else -%>
     chown -R rails:rails <%= Dir[*%w(db log storage tmp)].join(" ") %>
-<% unless options.swap? or using_passenger? -%>
+<% end -%>
+<% unless options.swap? or using_passenger? or using_litefs? -%>
 USER rails:rails
 <% end -%>
 
+<% end -%>
+<% if using_litefs? and !run_as_root? -%>
+# Authorize rails user to launch litefs
+COPY <<-"EOF" /etc/sudoers.d/rails
+rails ALL=(root) /usr/local/bin/litefs
+EOF
+
 <% end -%>
 <% unless deploy_env.empty? -%>
 # Deployment options
@@ -208,11 +230,16 @@ COPY <<-"EOF" /rails/Procfile.prod
 <% end -%>
 EOF
 
+<% end -%>
 # Start the server by default, this can be overwritten at runtime
 EXPOSE 3000
+<% if deploy_database == 'sqlite3' -%>
+VOLUME /data
+<% end -%>
+<% unless fly_processes -%>
+<% if procfile.size > 1 -%>
 CMD ["foreman", "start", "--procfile=Procfile.prod"]
 <% else -%>
-# Start the server by default, this can be overwritten at runtime
-EXPOSE 3000
 CMD <%= procfile.values.first.split(" ").inspect %>
 <% end -%>
+<% end -%>

data/lib/generators/templates/docker-compose.yml.erb

@@ -82,7 +82,7 @@ services:
 
   sidekiq:
     build: .
-    command: bin/sidekiq
+    command: bunde exec sidekiq
     environment:
       - RAILS_MASTER_KEY=$RAILS_MASTER_KEY
       - REDIS_URL=redis://redis-db:6379

data/lib/generators/templates/docker-entrypoint.erb

@@ -14,11 +14,21 @@ if [ $UID -eq 0 ]; then
 <%= @space %>echo 10 > /proc/sys/vm/swappiness
 <%= @space %>swapon /swapfile
 <%= @space %>echo 1 > /proc/sys/vm/overcommit_memory
+<% if using_litefs? -%>
+
+<%= @space %># mount litefs
+<%= @space %>litefs mount &
+<% end -%>
 <% unless run_as_root? or using_passenger? -%>
+
   exec su rails $0 $@
 fi
 <% end -%>
 
+<% elsif using_litefs? -%>
+# mount litefs
+<% unless run_as_root? %>sudo -E <% end %>litefs mount &
+
 <% end -%>
 <% if options.prepare -%>
 <% if procfile.size > 1 -%>
@@ -26,7 +36,7 @@ fi
 if [ "${*}" == "foreman start --procfile=Procfile.prod" ]; then
 <% else -%>
 # If running the rails server then create or migrate existing database
-if [ "${*}" == <%= procfile.values.first.inspect %> ]; then
+if [ "${*}" == <%= procfile.values.first.inspect %> <% if using_litefs? %>-a "$FLY_REGION" == "$PRIMARY_REGION" <%end%>]; then
 <% end -%>
 <% if options.precompile == "defer" -%>
   ./bin/rails assets:precompile

data/lib/generators/templates/fly.toml.erb

@@ -0,0 +1 @@
+<%= fly_make_processes(fly_processes) -%>

data/lib/generators/templates/litefs.yml.erb

@@ -0,0 +1,116 @@
+# based on: https://github.com/superfly/litefs/blob/main/cmd/litefs/etc/litefs.yml
+
+# The FUSE section handles settings on the FUSE file system. FUSE
+# provides a layer for intercepting SQLite transactions on the
+# primary node so they can be shipped to replica nodes transparently.
+fuse:
+  # Required. This is the mount directory that applications will
+  # use to access their SQLite databases.
+  dir: "/litefs"
+
+  # Set this flag to true to allow non-root users to access mount.
+  # You must set the "user_allow_other" option in /etc/fuse.conf first.
+  allow-other: false
+
+  # The debug flag enables debug logging of all FUSE API calls.
+  # This will produce a lot of logging. Not for general use.
+  debug: false
+
+# The data section specifies where internal LiteFS data is stored
+# and how long to retain the transaction files.
+# 
+# Transaction files are used to ship changes to replica nodes so
+# they should persist long enough for replicas to retrieve them,
+# even in the face of a short network interruption or a redeploy.
+# Under high load, these files can grow large so it's not advised
+# to extend retention too long.
+data:
+  # Path to internal data storage.
+  dir: "/data"
+
+  # Duration to keep LTX files. Latest LTX file is always kept.
+  retention: "10m"
+
+  # Frequency with which to check for LTX files to delete.
+  retention-monitor-interval: "1m"
+
+# If true, then LiteFS will not wait until the node becomes the
+# primary or connects to the primary before starting the subprocess.
+skip-sync: false
+
+# If true, then LiteFS will not exit if there is a validation
+# issue on startup. This can be useful for debugging issues as
+# it avoids constantly restarting the node on ephemeral hosting.
+exit-on-error: false
+
+# This section defines settings for the LiteFS HTTP API server.
+# This API server is how nodes communicate with each other.
+http:
+  # Specifies the bind address of the HTTP API server.
+  addr: ":20202"
+
+# This section defines settings for the option HTTP proxy.
+# This proxy can handle primary forwarding & replica consistency
+# for applications that use a single SQLite database.
+proxy:
+  # Specifies the bind address of the proxy server.
+  addr: ":3000"
+
+  # The hostport of the target application.
+  target: "localhost:3001"
+
+  # The name of the database used for TXID tracking.
+  db: "production.sqlite3"
+
+  # If true, enables verbose logging of requests by the proxy.
+  debug: false
+
+  # List of paths that are ignored by the proxy. The asterisk is
+  # the only available wildcard. These requests are passed
+  # through to the target as-is.
+  passthrough: []
+
+# The lease section defines how LiteFS creates a cluster and
+# implements leader election. For dynamic clusters, use the
+# "consul". This allows the primary to change automatically when
+# the current primary goes down. For a simpler setup, use
+# "static" which assigns a single node to be the primary and does
+# not failover.
+lease:
+  # Required. Must be either "consul" or "static".
+  type: "consul"
+
+  # Required. The URL for this node's LiteFS API.
+  # Should match HTTP port.
+  advertise-url: "http://${HOSTNAME}.vm.${FLY_APP_NAME}.internal:20202"
+
+  # Specifies whether the node can become the primary. If using
+  # "static" leasing, this should be set to true on the primary
+  # and false on the replicas.
+  candidate: ${FLY_REGION == PRIMARY_REGION}
+
+  # A Consul server provides leader election and ensures that the
+  # responsibility of the primary node can be moved in the event
+  # of a deployment or a failure.
+  consul:
+    # Required. The base URL of the Consul server.
+    url: "${FLY_CONSUL_URL}"
+
+    # Required. The key used for obtaining a lease by the primary.
+    # This must be unique for each cluster of LiteFS servers
+    key: "litefs/${FLY_APP_NAME}"
+
+    # Length of time before a lease expires. The primary will
+    # automatically renew the lease while it is alive, however,
+    # if it fails to renew in time then a new primary may be
+    # elected after the TTL. This only occurs for unexpected loss
+    # of the leader as normal operation will allow the leader to
+    # handoff the lease to another replica without downtime.
+    #
+    # Consul does not allow a TTL of less than 10 seconds.
+    ttl: "10s"
+
+    # Length of time after the lease expires before a candidate
+    # can become leader. This buffer is intended to prevent
+    # overlap in leadership due to clock skew or in-flight calls.
+    lock-delay: "1s"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: dockerfile-rails
 version: !ruby/object:Gem::Version
-  version: 1.3.0
+  version: 1.4.1
 platform: ruby
 authors:
 - Sam Ruby
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-05-18 00:00:00.000000000 Z
+date: 2023-05-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rails
@@ -49,12 +49,14 @@ files:
 - lib/generators/templates/docker-entrypoint.erb
 - lib/generators/templates/dockerfile.yml.erb
 - lib/generators/templates/dockerignore.erb
+- lib/generators/templates/fly.toml.erb
+- lib/generators/templates/litefs.yml.erb
 - lib/generators/templates/node-version.erb
-homepage: https://github.com/rubys/dockerfile-rails
+homepage: https://github.com/fly-apps/dockerfile-rails
 licenses:
 - MIT
 metadata:
-  homepage_uri: https://github.com/rubys/dockerfile-rails
+  homepage_uri: https://github.com/fly-apps/dockerfile-rails
 post_install_message:
 rdoc_options: []
 require_paths: