pups 1.0.3 → 1.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6637fcde0de413ee3d9925e644f84547563f86ba44d67692daa9f431520ab598
-  data.tar.gz: 52ed370e7071a7eb53cd87bc455dec4f48ca84c691ac28f1429e6985673a4d21
+  metadata.gz: e76a5c569771df90ca70c5e87fcef1f1252192b487fa0e772f43adcc7118adc0
+  data.tar.gz: 71a81d1ed1c2a4fe0dc1618b399dab1fc3a0e4b47d4afd9e8a5fe03b17411cf1
 SHA512:
-  metadata.gz: dc4828cccbe30f2d64d265c5bbed1558e17754a1866ebee549fc590eb26e80de8388deb40905fd3715af029e68c7e5592520b6eaeb1a61904f36114fae1f49ff
-  data.tar.gz: 9f5da00c947a36c01d574a497afad7f9d218870ab898195d2427622817b0f260f3e8ccf2650113ba0f20bf585d9673a80af9d6174d81b4cd6301cacfa3ce84af
+  metadata.gz: 2cb538356ede8e3554047bd59818500533dfa4ab7ab2446020d481c38121e7f88a3bfac481ae3985f45dde11384c3cf915db2be60dba1f5fee54e09886f53337
+  data.tar.gz: fedb98ab255593cb8804d05bed9fc1367135e5d891e3ea813795dafe4da5407c59123fdf52f99d42a402695a47c74dc828fac9be15d475d1dfc3a1317958d125

data/.github/workflows/ci.yml

@@ -0,0 +1,29 @@
+name: Tests
+
+on:
+  push:
+    branches:
+      - master
+  pull_request: {}
+
+jobs:
+  test:
+    name: "pups tests"
+    runs-on: ${{ matrix.os }}
+    timeout-minutes: 5
+
+    strategy:
+      fail-fast: true
+      matrix:
+        os: [ubuntu-latest]
+        ruby: ["2.7"]
+
+    steps:
+      - uses: actions/checkout@v2
+      - uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: ${{ matrix.ruby }}
+          bundler-cache: true
+      - name: Run minitest
+        run: |
+          rake test

data/.github/workflows/lint.yml

@@ -0,0 +1,27 @@
+name: Lint
+
+on:
+  push:
+    branches:
+      - master
+  pull_request: {}
+
+jobs:
+  lint:
+    name: "pups lint"
+    runs-on: ${{ matrix.os }}
+    timeout-minutes: 5
+
+    strategy:
+      fail-fast: true
+      matrix:
+        os: [ubuntu-latest]
+        ruby: ["2.7"]
+
+    steps:
+      - uses: actions/checkout@v2
+      - uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: ${{ matrix.ruby }}
+          bundler-cache: true
+      - run: bundle exec rubocop

data/.rubocop.yml

@@ -0,0 +1,3 @@
+inherit_gem:
+  rubocop-discourse: default.yml
+

data/Gemfile

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 source 'https://rubygems.org'
 
 # Specify your gem's dependencies in pups.gemspec

data/Guardfile

@@ -1,6 +1,8 @@
+# frozen_string_literal: true
+
 guard :minitest do
   # with Minitest::Unit
-  watch(%r{^test/(.*)\/?(.*)_test\.rb$})
+  watch(%r{^test/(.*)/?(.*)_test\.rb$})
   watch(%r{^lib/(.*/)?([^/]+)\.rb$})     { |m| "test/#{m[2]}_test.rb" }
   watch(%r{^test/test_helper\.rb$})      { 'test' }
 end

data/README.md

@@ -20,6 +20,19 @@ Or install it yourself as:
 
 pups is a small library that allows you to automate the process of creating Unix images.
 
+```
+Usage: pups [options] [FILE|--stdin]
+        --stdin                      Read input from stdin.
+        --quiet                      Don't print any logs.
+        --ignore <elements>          Ignore specific configuration elements, multiple elements can be provided (comma-delimited).
+                                     Useful if you want to skip over config in a pups execution.
+                                     e.g. `--ignore env,params`.
+        --gen-docker-run-args        Output arguments from the pups configuration for input into a docker run command. All other pups config is ignored.
+    -h, --help
+```
+
+pups requires input either via a stdin stream or a filename. The entire input is parsed prior to any templating or command execution.
+
 Example:
 
 ```
@@ -35,6 +48,14 @@ Running: `pups somefile.yaml` will execute the shell script resulting in a file
 
 ### Features
 
+#### Docker run argument generation
+
+The `--gen-docker-run-args` argument is used to make pups output arguments be in the format of `docker run <arguments output>`. Specifically, pups
+will take any `env`, `volume`, `labels`, `links`, and `expose` configuration, and coerce that into the format expected by `docker run`. This can be useful
+when pups is being used to configure an image (e.g. by executing a series of commands) that is then going to be run as a container. That way, the runtime and image
+configuration can be specified within the same yaml files.
+
+
 #### Environment Variables
 
 By default, pups automatically imports your environment variables and includes them as params.
@@ -149,15 +170,31 @@ Will merge the yaml file with the inline contents.
 
 #### A common environment
 
-This is implemented in discourse_docker's launcher, not in pups - therefore it does not work in standalone pups.
+Environment variables can be specified under the `env` key, which will be included in the environment for the template.
+
+```
+env:
+  MY_ENV: "a couple of words"
+run:
+  - exec: echo $MY_ENV > tmpfile
+```
+
+`tmpfile` will contain `a couple of words`.
+
+You can also specify variables to be templated within the environment, such as:
 
 ```
 env:
-   MY_ENV: 1
+  greeting: "hello, {{location}}!"
+env_template:
+  location: world
 ```
 
-All executions will get this environment set up
+In this example, the `greeting` environment variable will be set to `hello, world!` during initialisation as the `{{location}}` variable will be templated as `world`.
+Pups will also look in the environment itself at runtime for template variables, prefixed with `env_template_<variable name>`.
+Note that strings should be quoted to prevent YAML from parsing the `{ }` characters.
 
+All commands executed will inherit the environment once parsing and variable interpolation has been completed.
 
 ## Contributing
 

data/Rakefile

@@ -1,10 +1,12 @@
-require "bundler/gem_tasks"
-require "rake/testtask"
+# frozen_string_literal: true
+
+require 'bundler/gem_tasks'
+require 'rake/testtask'
 
 Rake::TestTask.new do |t|
-  t.libs << "test"
-  t.libs << "lib"
-  t.test_files = FileList["test/*_test.rb"]
+  t.libs << 'test'
+  t.libs << 'lib'
+  t.test_files = FileList['test/*_test.rb']
 end
 
-task :default => :test
+task default: :test

data/bin/pups

@@ -1,9 +1,9 @@
 #!/usr/bin/env ruby
+# frozen_string_literal: true
 
-$LOAD_PATH.unshift File.dirname(__FILE__) + "/../lib"
+$LOAD_PATH.unshift "#{File.dirname(__FILE__)}/../lib"
 
-require "pups"
-require "pups/cli"
+require 'pups'
+require 'pups/cli'
 
 Pups::Cli.run(ARGV)
-

data/lib/pups.rb

@@ -1,15 +1,17 @@
-require "logger"
-require "yaml"
+# frozen_string_literal: true
 
-require "pups/version"
-require "pups/config"
-require "pups/command"
-require "pups/exec_command"
-require "pups/merge_command"
-require "pups/replace_command"
-require "pups/file_command"
+require 'logger'
+require 'yaml'
 
-require "pups/runit"
+require 'pups/version'
+require 'pups/config'
+require 'pups/command'
+require 'pups/exec_command'
+require 'pups/merge_command'
+require 'pups/replace_command'
+require 'pups/file_command'
+require 'pups/docker'
+require 'pups/runit'
 
 module Pups
   class ExecError < RuntimeError
@@ -18,10 +20,18 @@ module Pups
 
   def self.log
     # at the moment docker likes this
-    @logger ||= Logger.new(STDERR)
+    @logger ||= Logger.new($stderr)
   end
 
   def self.log=(logger)
     @logger = logger
   end
+
+  def self.silence
+    if @logger
+      @logger.close
+    end
+
+    @logger = Logger.new(File.open(File::NULL, "w"))
+  end
 end

data/lib/pups/cli.rb

@@ -1,35 +1,70 @@
-class Pups::Cli
+# frozen_string_literal: true
 
-  def self.usage
-    puts "Usage: pups FILE or pups --stdin"
-    exit 1
-  end
-  def self.run(args)
-    if args.length != 1
-      usage
+require 'optparse'
+
+module Pups
+  class Cli
+    def self.opts
+      OptionParser.new do |opts|
+        opts.banner = 'Usage: pups [FILE|--stdin]'
+        opts.on('--stdin', 'Read input from stdin.')
+        opts.on('--quiet', "Don't print any logs.")
+        opts.on('--ignore <element(s)>', Array, "Ignore these template configuration elements, multiple elements can be provided (comma-delimited).")
+        opts.on('--gen-docker-run-args', 'Output arguments from the pups configuration for input into a docker run command. All other pups config is ignored.')
+        opts.on('-h', '--help') do
+          puts opts
+          exit
+        end
+      end
     end
 
-    Pups.log.info("Loading #{args[0]}")
-    if args[0] == "--stdin"
-      conf = STDIN.readlines.join
-      split = conf.split("_FILE_SEPERATOR_")
-
-      conf = nil
-      split.each do |data|
-        current = YAML.load(data.strip)
-        if conf
-          conf = Pups::MergeCommand.deep_merge(conf, current, :merge_arrays)
-        else
-          conf = current
+    def self.parse_args(args)
+      options = {}
+      opts.parse!(args, into: options)
+      options
+    end
+
+    def self.run(args)
+      options = parse_args(args)
+      input_file = options[:stdin] ? 'stdin' : args.last
+      unless input_file
+        puts opts.parse!(%w[--help])
+        exit
+      end
+
+      if options[:quiet]
+        Pups.silence
+      end
+
+      Pups.log.info("Reading from #{input_file}")
+
+      if options[:stdin]
+        conf = $stdin.readlines.join
+        split = conf.split('_FILE_SEPERATOR_')
+
+        conf = nil
+        split.each do |data|
+          current = YAML.safe_load(data.strip)
+          conf = if conf
+            Pups::MergeCommand.deep_merge(conf, current, :merge_arrays)
+          else
+            current
+          end
         end
+
+        config = Pups::Config.new(conf, options[:ignore])
+      else
+        config = Pups::Config.load_file(input_file, options[:ignore])
+      end
+
+      if options[:"gen-docker-run-args"]
+        print config.generate_docker_run_arguments
+        return
       end
 
-      config = Pups::Config.new(conf)
-    else
-      config = Pups::Config.load_file(args[0])
+      config.run
+    ensure
+      Pups::ExecCommand.terminate_async
     end
-    config.run
-  ensure
-    Pups::ExecCommand.terminate_async
   end
 end

data/lib/pups/command.rb

@@ -1,17 +1,20 @@
-class Pups::Command
+# frozen_string_literal: true
 
-  def self.run(command,params)
-    case command
-      when String then self.from_str(command,params).run
-      when Hash then self.from_hash(command,params).run
+module Pups
+  class Command
+    def self.run(command, params)
+      case command
+      when String then from_str(command, params).run
+      when Hash then from_hash(command, params).run
+      end
     end
-  end
 
-  def self.interpolate_params(cmd,params)
-    Pups::Config.interpolate_params(cmd,params)
-  end
+    def self.interpolate_params(cmd, params)
+      Pups::Config.interpolate_params(cmd, params)
+    end
 
-  def interpolate_params(cmd)
-    Pups::Command.interpolate_params(cmd,@params)
+    def interpolate_params(cmd)
+      Pups::Command.interpolate_params(cmd, @params)
+    end
   end
 end

data/lib/pups/config.rb

@@ -1,128 +1,171 @@
-class Pups::Config
+# frozen_string_literal: true
 
-  attr_reader :config, :params
+module Pups
+  class Config
+    attr_reader :config, :params
 
-  def self.load_file(config_file)
-    begin
-      new YAML.load_file(config_file)
+    def initialize(config, ignored = nil)
+      @config = config
+
+      # remove any ignored config elements prior to any more processing
+      ignored&.each { |e| @config.delete(e) }
+
+      # set some defaults to prevent checks in various functions
+      ['env_template', 'env', 'labels', 'params'].each { |key| @config[key] = {} unless @config.has_key?(key) }
+
+      # Order here is important.
+      Pups::Config.combine_template_and_process_env(@config, ENV)
+      Pups::Config.prepare_env_template_vars(@config['env_template'], ENV)
+
+      # Templating is supported in env and label variables.
+      Pups::Config.transform_config_with_templated_vars(@config['env_template'], ENV)
+      Pups::Config.transform_config_with_templated_vars(@config['env_template'], @config['env'])
+      Pups::Config.transform_config_with_templated_vars(@config['env_template'], @config['labels'])
+
+      @params = @config["params"]
+      ENV.each do |k, v|
+        @params["$ENV_#{k}"] = v
+      end
+      inject_hooks
+    end
+
+    def self.load_file(config_file, ignored = nil)
+      Config.new(YAML.load_file(config_file), ignored)
     rescue Exception
-      STDERR.puts "Failed to parse #{config_file}"
-      STDERR.puts "This is probably a formatting error in #{config_file}"
-      STDERR.puts "Cannot continue. Edit #{config_file} and try again."
+      warn "Failed to parse #{config_file}"
+      warn "This is probably a formatting error in #{config_file}"
+      warn "Cannot continue. Edit #{config_file} and try again."
       raise
     end
-  end
 
-  def self.load_config(config)
-    new YAML.load(config)
-  end
+    def self.load_config(config, ignored = nil)
+      Config.new(YAML.safe_load(config), ignored)
+    end
 
-  def initialize(config)
-    @config = config
-    validate!(@config)
-    @params = @config["params"]
-    @params ||= {}
-    ENV.each do |k,v|
-      @params["$ENV_#{k}"] = v
+    def self.prepare_env_template_vars(env_template, env)
+      # Merge env_template variables from env and templates.
+      env.each do |k, v|
+        if k.include?('env_template_')
+          key = k.gsub('env_template_', '')
+          env_template[key] = v.to_s
+        end
+      end
     end
-    inject_hooks
-  end
 
-  def validate!(conf)
-    # raise proper errors if nodes are missing etc
-  end
+    def self.transform_config_with_templated_vars(env_template, to_transform)
+      # Transform any templated variables prior to copying to params.
+      # This has no effect if no env_template was provided.
+      env_template.each do |k, v|
+        to_transform.each do |key, val|
+          if val.to_s.include?("{{#{k}}}")
+            to_transform[key] = val.gsub("{{#{k}}}", v.to_s)
+          end
+        end
+      end
+    end
+
+    def self.combine_template_and_process_env(config, env)
+      # Merge all template env variables and process env variables, so that env
+      # variables can be provided both by configuration and runtime variables.
+      config["env"].each { |k, v| env[k] = v.to_s }
+    end
 
-  def inject_hooks
-    return unless hooks = @config["hooks"]
+    def inject_hooks
+      return unless hooks = @config['hooks']
 
-    run = @config["run"]
+      run = @config['run']
+
+      positions = {}
+      run.each do |row|
+        next unless row.is_a?(Hash)
 
-    positions = {}
-    run.each do |row|
-      if Hash === row
         command = row.first
-        if Hash === command[1]
-          hook = command[1]["hook"]
+        if command[1].is_a?(Hash)
+          hook = command[1]['hook']
           positions[hook] = row if hook
         end
       end
-    end
-
-    hooks.each do |full, list|
 
-      offset = nil
-      name = nil
+      hooks.each do |full, list|
+        offset = nil
+        name = nil
 
-      if full =~ /^after_/
-        name = full[6..-1]
-        offset = 1
-      end
+        if full =~ /^after_/
+          name = full[6..-1]
+          offset = 1
+        end
 
-      if full =~ /^before_/
-        name = full[7..-1]
-        offset = 0
-      end
+        if full =~ /^before_/
+          name = full[7..-1]
+          offset = 0
+        end
 
-      index = run.index(positions[name])
+        index = run.index(positions[name])
 
-      if index && index >= 0
-        run.insert(index + offset, *list)
-      else
-        Pups.log.info "Skipped missing #{full} hook"
+        if index && index >= 0
+          run.insert(index + offset, *list)
+        else
+          Pups.log.info "Skipped missing #{full} hook"
+        end
       end
-
     end
-  end
 
-  def run
-    run_commands
-  rescue => e
-    exit_code = 1
-    if Pups::ExecError === e
-      exit_code = e.exit_code
+    def generate_docker_run_arguments
+      output = []
+      output << Pups::Docker.generate_env_arguments(config['env'])
+      output << Pups::Docker.generate_link_arguments(config['links'])
+      output << Pups::Docker.generate_expose_arguments(config['expose'])
+      output << Pups::Docker.generate_volume_arguments(config['volumes'])
+      output << Pups::Docker.generate_label_arguments(config['labels'])
+      output.sort!.join(" ").strip
     end
-    unless exit_code == 77
-      puts
-      puts
-      puts "FAILED"
-      puts "-" * 20
-      puts "#{e.class}: #{e}"
-      puts "Location of failure: #{e.backtrace[0]}"
-      if @last_command
-        puts "#{@last_command[:command]} failed with the params #{@last_command[:params].inspect}"
+
+    def run
+      run_commands
+    rescue StandardError => e
+      exit_code = 1
+      exit_code = e.exit_code if e.is_a?(Pups::ExecError)
+      unless exit_code == 77
+        puts
+        puts
+        puts 'FAILED'
+        puts '-' * 20
+        puts "#{e.class}: #{e}"
+        puts "Location of failure: #{e.backtrace[0]}"
+        puts "#{@last_command[:command]} failed with the params #{@last_command[:params].inspect}" if @last_command
       end
+      exit exit_code
     end
-    exit exit_code
-  end
 
-  def run_commands
-    @config["run"].each do |item|
-      item.each do |k,v|
-        type = case k
-                  when "exec" then Pups::ExecCommand
-                  when "merge" then Pups::MergeCommand
-                  when "replace" then Pups::ReplaceCommand
-                  when "file" then Pups::FileCommand
-                  else raise SyntaxError.new("Invalid run command #{k}")
-              end
-
-        @last_command = { command: k, params: v }
-        type.run(v, @params)
+    def run_commands
+      @config['run']&.each do |item|
+        item.each do |k, v|
+          type = case k
+                 when 'exec' then Pups::ExecCommand
+                 when 'merge' then Pups::MergeCommand
+                 when 'replace' then Pups::ReplaceCommand
+                 when 'file' then Pups::FileCommand
+                 else raise SyntaxError, "Invalid run command #{k}"
+          end
+
+          @last_command = { command: k, params: v }
+          type.run(v, @params)
+        end
       end
     end
-  end
 
-  def interpolate_params(cmd)
-    self.class.interpolate_params(cmd,@params)
-  end
+    def interpolate_params(cmd)
+      self.class.interpolate_params(cmd, @params)
+    end
+
+    def self.interpolate_params(cmd, params)
+      return unless cmd
 
-  def self.interpolate_params(cmd, params)
-    return unless cmd
-    processed = cmd.dup
-    params.each do |k,v|
-      processed.gsub!("$#{k}", v.to_s)
+      processed = cmd.dup
+      params.each do |k, v|
+        processed.gsub!("$#{k}", v.to_s)
+      end
+      processed
     end
-    processed
   end
-
 end