fly.io-rails 0.1.3 → 0.1.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b0cfcab66d8f161efea6825e771f1bd95f3f57af5e6163dbdec933281a372f78
-  data.tar.gz: 84f1312372f746f150b3dc5dd0d5b35a31d180573c541a42228ae2e8b903c5d7
+  metadata.gz: '09430ace636e3b85c5b26ea93685d04c6ac9849a739d4bf9112737b14db0c066'
+  data.tar.gz: ae3546d0dc0d4091bb80397ea064a685b755de8f22cfe6873822f74d13c5b9b8
 SHA512:
-  metadata.gz: c8dc7129a3b3a06ef98234c590b20e7d14d5d4091744079f353a308e703bcca1bb1aa89ece7c713c7d63723e42c68b90f4a5cfed1bd00cb9f78ebcfdd2411210
-  data.tar.gz: fa3c2b69e0672cc5c7c2115b17a0bcf5e4fd0adc46a0d1f0b6ee5a4cfc909b56dfe8ce4d5115435e59a903285357a4260521716e90b24397415af01f6798c2c0
+  metadata.gz: b677c704e6cb60d17705509b10b1ac1bc301f032ecef1997605d00851bafa3fd58d3bd431c76f897eebade03bb2205ce33c472c425ca55eb4407b0a053314fd7
+  data.tar.gz: 81907b62d795adae29bce7479de234cc9de8e172b20d3ed5d7d3cf881f7e926b3571db7d38bb1dbabf058982f80d725627a45256f9132b5e50eb685149c3f917

data/README.md

@@ -2,11 +2,28 @@
 
 Add [Fly.io](https://fly.io) support to [Rails](https://rubyonrails.org/).
 
-## Status
+## Usage
+
+For usage instructions, see the following guides:
+
+  * [Machine API](https://fly.io/docs/rails/advanced-guides/machine/)
+  * [Lite FS](https://fly.io/docs/rails/advanced-guides/litefs/)
+  * [Terraform](https://fly.io/docs/rails/advanced-guides/terraform/)
+
+## Key files
+
+  * Entrypoints: [lib/tasks/fly.rake](./lib/tasks/fly.rake), [lib/generators/app_generator.rb](./lib/generators/app_generator.rb), [lib/generators/terraform_generator.rb](.lib/generators/terraform_generator.rb) contain the deploy task, fly:app generator and
+  fly:terraform generator respectively.
+  * [lib/fly.io-rails/actions.rb](./lib/fly.io-rails/actions.rb) contains Thor actions used by the
+  rake task and generators.  Does some funky stuff to allow Thor actions to
+  be called from Rake.
+  * [lib/fly.io-rails/machines.rb](./lib/fly.io-rails/machines.rb) wraps Fly.io's machine API as a Ruby module.
+  * [lib/generators/templates](./lib/generators/templates) contains erb
+  templates for all of the files produced primarily by the generator, but also
+  by the deploy task.
+  * [Rakefile](./Rakefile) used to build gems.  Includes native binaries for each supported platform.
 
-<u>pre-alpha</u>.
 
-For usage instructions, see [Fly.io Rails Advanced Guide: Terraform](https://fly.io/docs/rails/advanced-guides/terraform/).
 
 ## Build instructions
 
@@ -19,3 +36,46 @@ every supported platform as well as an additional gem that doesn't
 include a binary.
 
 To download new binaries, run `rake clobber` then `rake package` agein.
+
+## Debugging instructions
+
+This gem provides a Railtie, with rake tasks and a generator that uses
+Thor and templates.  Being in Ruby, there is no "compile" step.  That
+coupled with Bundler "local overrides" makes testing a breeze.  And
+Rails 7 applications without node dependencies are quick to create.
+
+A script like the following will destroy previous fly applications,
+create a new rails app, add and then override this gem, and finally
+copy in any files that would need to be manually edited.
+
+```
+if [ -e welcome/fly.toml ]; then
+  app=$(awk -e '/^app\s+=/ { print $3 }' welcome/fly.toml | sed 's/"//g')
+  fly apps destroy -y $app
+fi
+rm -rf welcome
+rails new welcome
+cd welcome
+bundle config disable_local_branch_check true
+bundle config set --local local.fly.io-rails /Users/rubys/git/fly.io-rails
+bundle add fly.io-rails --git https://github.com/rubys/fly.io-rails.git
+cp ../routes.rb config
+# bin/rails generate terraform
+# fly secrets set FLY_API_TOKEN=$(fly auth token)
+# bin/rails generate controller job start complete status
+# bin/rails generate job machine
+# cp ../job_controller.rb app/controllers/
+# cp ../machine_job.rb app/jobs/
+```
+
+Once created, I rerun using:
+
+```
+cd ..; sh redo-welcome; cd welcome; 
+```
+
+Generally after the finaly semicolon, I have commands like
+`bin/rails generate fly:app --litefs; fly deploy`.  Rerunning
+after I make a change is a matter of pushing the up arrow until
+I find this command and then pressing enter.
+

data/lib/fly.io-rails/actions.rb

@@ -16,9 +16,18 @@ module Fly
     include Fly::Scanner
     attr_accessor :options
 
-    def initialize(app, regions = nil)
+    def initialize(app, options={})
+      # placate thor
+      @options = {}
+      @destination_stack = [Dir.pwd]
+
+      # extract options
       self.app = app
+      regions = options[:region]&.flatten || []
+      @litefs = options[:litefs]
+      @nomad = options[:nomad]
 
+      # prepare template variablesl
       @ruby_version = RUBY_VERSION
       @bundler_version = Bundler::VERSION
       @node = File.exist? 'node_modules'
@@ -26,9 +35,9 @@ module Fly
       @node_version = @node ? `node --version`.chomp.sub(/^v/, '') : '16.17.0'
       @org = Fly::Machines.org
 
-      @options = {}
-      @destination_stack = [Dir.pwd]
+      @set_stage = @nomad ? 'set' : 'set --stage'
 
+      # determine region
       if !regions or regions.empty?
         @regions = JSON.parse(`flyctl regions list --json --app #{app}`)['Regions'].
           map {|region| region['Code']} rescue []
@@ -37,12 +46,15 @@ module Fly
       end
 
       @region = @regions.first || 'iad'
+      @regions = [@region] if @regions.empty?
 
+      # Process DSL
       @config = Fly::DSL::Config.new
       if File.exist? 'config/fly.rb'
         @config.instance_eval IO.read('config/fly.rb')
       end
 
+      # set additional variables based on application source
       scan_rails_app
     end
 
@@ -87,6 +99,15 @@ module Fly
       app_template 'fly.rake.erb', 'lib/tasks/fly.rake'
     end
 
+    def generate_procfile
+      return unless @sidekiq
+      app_template 'Procfile.fly.erb', 'Procfile.fly'
+    end
+
+    def generate_litefs
+      app_template 'litefs.yml.erb', 'config/litefs.yml'
+    end
+
     def generate_key
       credentials = nil
       if File.exist? 'config/credentials/production.key'
@@ -96,8 +117,8 @@ module Fly
       end
   
       if credentials
-        say_status :run, "flyctl secrets set --stage RAILS_MASTER_KEY from #{credentials}"
-        system "flyctl secrets set --stage RAILS_MASTER_KEY=#{IO.read(credentials).chomp}"
+        say_status :run, "flyctl secrets #{@set_stage} RAILS_MASTER_KEY from #{credentials}"
+        system "flyctl secrets #{@set_stage} RAILS_MASTER_KEY=#{IO.read(credentials).chomp}"
         puts
       end
   
@@ -126,9 +147,9 @@ module Fly
     def create_volume(app, region, size)
       volume = "#{app.gsub('-', '_')}_volume"
       volumes = JSON.parse(`flyctl volumes list --json`).
-        map {|volume| volume['Name']}
+        map {|volume| [volume['Name'], volume['Region']]}
 
-      unless volumes.include? volume
+      unless volumes.include? [volume, region]
         cmd = "flyctl volumes create #{volume} --app #{app} --region #{region} --size #{size}"
         say_status :run, cmd
         system cmd
@@ -139,6 +160,7 @@ module Fly
 
     def create_postgres(app, org, region, vm_size, volume_size, cluster_size)
       cmd = "flyctl postgres create --name #{app}-db --org #{org} --region #{region} --vm-size #{vm_size} --volume-size #{volume_size} --initial-cluster-size #{cluster_size}"
+      cmd += ' --machines' unless @nomad
       say_status :run, cmd
       output = FlyIoRails::Utils.tee(cmd)
       output[%r{postgres://\S+}]
@@ -204,7 +226,13 @@ module Fly
       end
 
       if @sqlite3
-        @volume = create_volume(app, @region, @config.sqlite3.size) 
+        if @litefs
+          @regions.each do |region|
+            @volume = create_volume(app, region, @config.sqlite3.size) 
+          end
+        else
+          @volume = create_volume(app, @region, @config.sqlite3.size) 
+        end
       elsif @postgresql and not secrets.include? 'DATABASE_URL'
         secret = create_postgres(app, @org, @region,
           @config.postgres.vm_size,
@@ -212,7 +240,7 @@ module Fly
           @config.postgres.initial_cluster_size)
 
         if secret
-          cmd = "flyctl secrets set --stage DATABASE_URL=#{secret}"
+          cmd = "flyctl secrets #{@set_stage} DATABASE_URL=#{secret}"
           say_status :run, cmd
           system cmd
         end
@@ -225,7 +253,7 @@ module Fly
         secret = create_redis(app, @org, @region, eviction)
 
         if secret
-          cmd = "flyctl secrets set --stage REDIS_URL=#{secret}"
+          cmd = "flyctl secrets #{@set_stage} REDIS_URL=#{secret}"
           say_status :run, cmd
           system cmd
         end
@@ -293,6 +321,10 @@ module Fly
         config[:env] = {
           "DATABASE_URL" => "sqlite3:///mnt/volume/production.sqlite3"
         }
+
+        if @litefs
+          config[:env]['DATABASE_URL'] = "sqlite3:///data/production.sqlite3"
+        end
       end
 
       # process toml overrides

data/lib/fly.io-rails/machines.rb

@@ -10,7 +10,7 @@ module Fly
   # the caller is trusted and does pass through unsanitized user input.
   #
   module Machines
-    @@api_token = nil
+    @@api_token = ENV['FLY_API_TOKEN']
     @@fly_api_hostname = nil
 
     # determine fly api hostname.  Returns nil if no proxy is running
@@ -160,14 +160,25 @@ module Fly
       api(path) {|uri| request = Net::HTTP::Delete.new(uri) }
     end
 
+    # graphql -- see https://til.simonwillison.net/fly/undocumented-graphql-api
+    def self.graphql(query)
+      api('/graphql', 'https://api.fly.io/') do |path|
+        request = Net::HTTP::Post.new(path)
+        request.body = { query: query }.to_json
+        request
+      end
+    end
+
     # common processing for all APIs
-    def self.api(path, &make_request)
-      uri = URI("http://#{fly_api_hostname}#{path}")
+    def self.api(path, host=nil, &make_request)
+      host ||= "http://#{fly_api_hostname}"
+      uri = URI.join(host, path)
       http = Net::HTTP.new(uri.host, uri.port)
+      http.use_ssl = true if uri.instance_of? URI::HTTPS
 
       request = make_request.call(uri.request_uri)
 
-      @@api_token ||= `fly auth token`.chomp
+      @@api_token ||= `flyctl auth token`.chomp
       headers = {
         "Authorization" => "Bearer #{@@api_token}",
         "Content-Type" => "application/json",

data/lib/fly.io-rails/scanner.rb

@@ -12,6 +12,8 @@ module Fly
         @postgresql = true
       end
 
+      @sidekiq = IO.read('Gemfile').include? 'sidekiq' rescue false
+
       if (YAML.load_file('config/cable.yml').dig('production', 'adapter') rescue false)
         @redis_cable = true
       end
@@ -20,7 +22,7 @@ module Fly
         @redis_cache = true
       end
 
-      @redis = @redis_cable || @redis_cache
+      @redis = @redis_cable || @redis_cache || @sidekiq
     end
   end
 end

data/lib/fly.io-rails/utils.rb

@@ -28,12 +28,14 @@ module FlyIoRails
       data.join
     end
 
-    def create_app(name=nil, org='personal', regions=[])
+    def create_app(name: nil, org: 'personal', regions: [], nomad: false, **rest)
       cmd = if name
         "flyctl apps create #{name.inspect} --org #{org.inspect} --machines"
       else
         "flyctl apps create --generate-name --org #{org.inspect} --machines"
       end
+
+      cmd.sub! ' --machines', '' if nomad
   
       output = tee cmd
       exit 1 unless output =~ /^New app created: /

data/lib/fly.io-rails/version.rb

@@ -1,3 +1,3 @@
 module Fly_io
-  VERSION = '0.1.3'
+  VERSION = '0.1.5'
 end

data/lib/fly.io-rails.rb

@@ -3,9 +3,25 @@ require 'fly.io-rails/generators'
 require 'fly.io-rails/utils'
 
 class FlyIoRailtie < Rails::Railtie
+  # load rake tasks
   rake_tasks do
     Dir[File.expand_path('tasks/*.rake', __dir__)].each do |file|
       load file
     end
   end
+
+  # set FLY_IMAGE_NAME on Nomad VMs
+  if not ENV['FLY_IMAGE_NAME'] and ENV['FLY_APP_NAME'] and ENV['FLY_API_TOKEN']
+    require 'fly.io-rails/machines'
+
+    ENV['FLY_IMAGE_REF'] = Fly::Machines.graphql(%{
+      query {
+	app(name: "#{ENV['FLY_APP_NAME']}") {
+	  currentRelease {
+	    imageRef
+	  }
+	}
+      }
+    }).dig(:data, :app, :currentRelease, :imageRef)
+  end
 end

data/lib/generators/fly/app_generator.rb

@@ -7,19 +7,24 @@ class AppGenerator < Rails::Generators::Base
   class_option :name, type: :string, required: false
   class_option :org, type: :string, default: 'personal'
   class_option :region, type: :array, repeatable: true, default: []
+  class_option :nomad, type: :boolean, default: false
+
+  class_option :litefs, type: :boolean, default: false
 
   def generate_app
     source_paths.push File.expand_path('../templates', __dir__)
 
-    create_app(options[:name], options[:org], options)
+    create_app(**options.symbolize_keys)
 
-    action = Fly::Actions.new(@app, options[:region])
+    action = Fly::Actions.new(@app, options)
 
     action.generate_toml
     action.generate_fly_config unless File.exist? 'config/fly.rb'
     action.generate_dockerfile unless File.exist? 'Dockerfile'
     action.generate_dockerignore unless File.exist? '.dockerignore'
     action.generate_raketask unless File.exist? 'lib/tasks/fly.rake'
+    action.generate_procfile unless File.exist? 'Procfile.rake'
+    action.generate_litefs if options[:litefs] and not File.exist? 'config/litefs'
     action.generate_patches
     action.generate_ipv4
     action.generate_ipv6
@@ -27,4 +32,4 @@ class AppGenerator < Rails::Generators::Base
     action.launch(@app)
   end
 end
-end
+end

data/lib/generators/fly/terraform_generator.rb

@@ -8,18 +8,21 @@ class TerraformGenerator < Rails::Generators::Base
   class_option :org, type: :string, default: 'personal'
   class_option :region, type: :array, repeatable: true, default: []
 
+  class_option :litefs, type: :boolean, default: false
+
   def terraform
     source_paths.push File.expand_path('../templates', __dir__)
 
-    create_app(options[:name], options[:org], options[:region])
+    create_app(**options.symbolize_keys)
 
-    action = Fly::Actions.new(@app, options[:region])
+    action = Fly::Actions.new(@app, options)
     
     action.generate_toml
     action.generate_dockerfile
     action.generate_dockerignore
     action.generate_terraform
     action.generate_raketask
+    action.generate_procfile
     action.generate_patches
 
     action.generate_key
@@ -27,4 +30,4 @@ class TerraformGenerator < Rails::Generators::Base
     tee 'terraform init'
   end
 end
-end
+end

data/lib/generators/templates/Dockerfile.erb

@@ -21,6 +21,8 @@ ARG RUBY_VERSION=<%= @ruby_version %>
 ARG VARIANT=jemalloc-slim
 FROM quay.io/evl.ms/fullstaq-ruby:${RUBY_VERSION}-${VARIANT} as base
 
+LABEL fly_launch_runtime="rails"
+
 <% if @node -%>
 ARG NODE_VERSION=<%= @node_version %>
 <% end -%>
@@ -54,7 +56,12 @@ RUN volta install node@${NODE_VERSION} yarn
 
 FROM base as build_deps
 
-ARG BUILD_PACKAGES="git build-essential libpq-dev wget vim curl gzip xz-utils libsqlite3-dev"
+<% 
+  @build_packages = %w(git build-essential wget vim curl gzip xz-utils)
+  @build_packages << 'libpq-dev' if @postgresql
+  @build_packages << 'libsqlite3-dev' if @sqlite3
+-%>
+ARG BUILD_PACKAGES=<%= @build_packages.join(' ').inspect %>
 ENV BUILD_PACKAGES ${BUILD_PACKAGES}
 
 RUN --mount=type=cache,id=dev-apt-cache,sharing=locked,target=/var/cache/apt \
@@ -74,6 +81,9 @@ RUN gem update --system --no-document && \
 
 COPY Gemfile* ./
 RUN bundle install &&  rm -rf vendor/bundle/ruby/*/cache
+<% if @sidekiq -%>
+RUN gem install foreman
+<% end -%>
 
 <% if @node -%>
 #######################################################################
@@ -92,12 +102,24 @@ RUN npm install
 
 <% end -%>
 #######################################################################
+<% if @litefs -%>
+
+# Fetch the LiteFS binary
+FROM flyio/litefs:pr-109 AS litefs
 
+#######################################################################
+<% end %>
 # install deployment packages
 
 FROM base
 
-ARG DEPLOY_PACKAGES="postgresql-client file vim curl gzip libsqlite3-0"
+<%
+  @deploy_packages = %w(file vim curl gzip)
+  @deploy_packages << 'postgresql-client' if @postgresql
+  @deploy_packages << 'libsqlite3-0' if @sqlite3
+  @deploy_packages << 'fuse' if @litefs
+-%>
+ARG DEPLOY_PACKAGES=<%= @deploy_packages.join(' ').inspect %>
 ENV DEPLOY_PACKAGES=${DEPLOY_PACKAGES}
 
 RUN --mount=type=cache,id=prod-apt-cache,sharing=locked,target=/var/cache/apt \
@@ -116,6 +138,16 @@ COPY --from=gems /usr/local/bundle /usr/local/bundle
 # copy installed node modules
 COPY --from=node_modules /app/node_modules /app/node_modules
 
+<% end -%>
+<% if @litefs -%>
+# copy litefs binary
+COPY --from=litefs /usr/local/bin/litefs /usr/local/bin/litefs
+
+# Copy our LiteFS configuration.
+ADD config/litefs.yml /etc/litefs.yml
+
+# Create mount point
+RUN mkdir /data
 <% end -%>
 #######################################################################
 
@@ -140,6 +172,10 @@ RUN ${BUILD_COMMAND}
 
 # Default server start instructions.  Generally Overridden by fly.toml.
 ENV PORT 8080
+<% if @litefs -%>
+ARG SERVER_COMMAND="litefs"
+<% else -%>
 ARG SERVER_COMMAND="bin/rails fly:server"
+<% end -%>
 ENV SERVER_COMMAND ${SERVER_COMMAND}
 CMD ${SERVER_COMMAND}

data/lib/generators/templates/Procfile.fly.erb

@@ -0,0 +1,4 @@
+web: bin/rails server
+<% if @sidekiq -%>
+worker: bundle exec sidekiq
+<% end -%>

data/lib/generators/templates/fly.rake.erb

@@ -6,6 +6,7 @@ namespace :fly do
   #  - Failures here prevent deployment
   task :build => 'assets:precompile'
 
+<% unless @litefs -%>
   # RELEASE step:
   #  - changes to the filesystem made here are DISCARDED
   #  - full access to secrets, databases
@@ -16,6 +17,7 @@ namespace :fly do
   task :release => 'db:migrate'
   <%- end -%>
 
+<% end -%>
   # SERVER step:
   #  - changes to the filesystem made here are deployed
   #  - full access to secrets, databases
@@ -26,7 +28,13 @@ namespace :fly do
   <%- else -%>
   task :server => :swapfile do
   <%- end -%>
+  <%- if @sidekiq -%>
+    Bundler.with_original_env do
+      sh 'foreman start --procfile=Procfile.fly'
+    end
+  <%- else -%>
     sh 'bin/rails server'
+  <%- end -%>
   end
 
   # optional SWAPFILE task:

data/lib/generators/templates/fly.toml.erb

@@ -6,15 +6,23 @@ processes = []
 [build]
   [build.args]
     BUILD_COMMAND = "bin/rails fly:build"
+<% if @litefs -%>
+    SERVER_COMMAND = "litefs"
+<% else -%>
     SERVER_COMMAND = "bin/rails fly:server"
 
 [deploy]
   release_command = "bin/rails fly:release"
+<% end -%>
 
 [env]
   PORT = "8080"
 <% if @sqlite3 -%>
+<% if @litefs -%>
+  DATABASE_URL = "sqlite3:///data/production.sqlite3"
+<% else -%>
   DATABASE_URL = "sqlite3:///mnt/volume/production.sqlite3"
+<% end -%>
 
 [mounts]
 source = <%= "#{app.gsub('-', '_')}_volume".inspect %>
@@ -24,6 +32,9 @@ destination = "/mnt/volume"
 [experimental]
   allowed_public_ports = []
   auto_rollback = true
+<% if @litefs -%>
+  enable_consul = true
+<% end  -%>
 
 [[services]]
   http_checks = []

data/lib/generators/templates/litefs.yml.erb

@@ -0,0 +1,14 @@
+# The path to where the SQLite database will be accessed.
+mount-dir: "/data"
+
+# The path to where the underlying volume mount is.
+data-dir: "/mnt/volume"
+
+# Execute this subprocess once LiteFS connects to the cluster. 
+exec: "bin/rails fly:server"
+
+# These environment variables will be available in your Fly.io application.
+# You must specify "experiement.enable_consul" for FLY_CONSUL_URL to be available.
+consul:
+  url: "${FLY_CONSUL_URL}"
+  advertise-url: "http://${HOSTNAME}.vm.${FLY_APP_NAME}.internal:20202"

data/lib/tasks/fly.rake

@@ -22,6 +22,7 @@ namespace :fly do
     action.generate_dockerfile unless File.exist? 'Dockerfile'
     action.generate_dockerignore unless File.exist? '.dockerignore'
     action.generate_raketask unless File.exist? 'lib/tasks/fly.rake'
+    action.generate_procfile unless File.exist? 'Procfile.fly'
 
     # build and push an image
     out = FlyIoRails::Utils.tee 'fly deploy --build-only --push'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fly.io-rails
 version: !ruby/object:Gem::Version
-  version: 0.1.3
+  version: 0.1.5
 platform: ruby
 authors:
 - Sam Ruby
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2022-09-25 00:00:00.000000000 Z
+date: 2022-10-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fly-ruby
@@ -62,10 +62,12 @@ files:
 - lib/generators/fly/app_generator.rb
 - lib/generators/fly/terraform_generator.rb
 - lib/generators/templates/Dockerfile.erb
+- lib/generators/templates/Procfile.fly.erb
 - lib/generators/templates/dockerignore.erb
 - lib/generators/templates/fly.rake.erb
 - lib/generators/templates/fly.rb.erb
 - lib/generators/templates/fly.toml.erb
+- lib/generators/templates/litefs.yml.erb
 - lib/generators/templates/main.tf.erb
 - lib/generators/templates/patches/action_cable.rb
 - lib/tasks/fly.rake