opswalrus 1.0.0

data/lib/opswalrus/runtime_environment.rb

@@ -0,0 +1,258 @@
+require 'json'
+require 'shellwords'
+require 'socket'
+require 'sshkit'
+
+require_relative 'traversable'
+require_relative 'walrus_lang'
+
+module OpsWalrus
+
+  class ImportReference
+    attr_accessor :local_name
+
+    def initialize(local_name)
+      @local_name = local_name
+    end
+  end
+
+  class PackageDependencyReference < ImportReference
+    attr_accessor :package_reference
+    def initialize(local_name, package_reference)
+      super(local_name)
+      @package_reference = package_reference
+    end
+  end
+
+  class DirectoryReference < ImportReference
+    attr_accessor :dirname
+    def initialize(local_name, dirname)
+      super(local_name)
+      @dirname = dirname
+    end
+  end
+
+  class OpsFileReference < ImportReference
+    attr_accessor :ops_file_path
+    def initialize(local_name, ops_file_path)
+      super(local_name)
+      @ops_file_path = ops_file_path
+    end
+  end
+
+  # Namespace is really just a Map of symbol_name -> (Namespace | OpsFile) pairs
+  class Namespace
+    attr_accessor :runtime_env
+    attr_accessor :dirname
+    attr_accessor :symbol_table
+
+    # dirname is an absolute path
+    def initialize(runtime_env, dirname)
+      @runtime_env = runtime_env
+      @dirname = dirname
+      @symbol_table = {}    # "symbol_name" => ops_file_or_child_namespace
+    end
+
+    def add(symbol_name, ops_file_or_child_namespace)
+      @symbol_table[symbol_name.to_s] = ops_file_or_child_namespace
+    end
+
+    def resolve_symbol(symbol_name)
+      @symbol_table[symbol_name.to_s]
+    end
+
+    def method_missing(name, *args, **kwargs, &block)
+      resolved_symbol = resolve_symbol(name)
+      case resolved_symbol
+      when Namespace
+        resolved_symbol
+      when OpsFile
+        params_hash = resolved_symbol.build_params_hash(*args, **kwargs)
+        resolved_symbol.invoke(runtime_env, params_hash)
+      end
+    end
+  end
+
+    # the assumption is that we have a bundle directory with all the packages in it
+  # and the bundle directory is the root directory
+  class LoadPath
+    include Traversable
+
+    attr_accessor :dir
+    attr_accessor :runtime_env
+
+    def initialize(runtime_env, dir)
+      @runtime_env = runtime_env
+      @dir = dir
+      @root_namespace = build_symbol_resolution_tree(@dir)
+      @path_map = build_path_map(@root_namespace)
+    end
+
+    # returns a tree of Namespace -> {Namespace* -> {Namespace* -> ..., OpsFile*}, OpsFile*}
+    def build_symbol_resolution_tree(directory_path)
+      namespace = Namespace.new(runtime_env, directory_path)
+
+      directory_path.glob("*.ops").each do |ops_file_path|
+        ops_file = OpsFile.new(@app, ops_file_path)
+        namespace.add(ops_file.basename, ops_file)
+      end
+
+      directory_path.glob("*").
+                     select(&:directory?).
+                     reject {|dir| dir.basename.to_s.downcase == Bundler::BUNDLE_DIR }.
+                     each do |dir_path|
+        dir_basename = dir_path.basename
+        unless namespace.resolve_symbol(dir_basename)
+          child_namespace = build_symbol_resolution_tree(dir_path)
+          namespace.add(dir_basename, child_namespace)
+        end
+      end
+
+      namespace
+    end
+
+    # returns a Map of path -> (Namespace | OpsFile) pairs
+    def build_path_map(root_namespace)
+      path_map = {}
+
+      pre_order_traverse(root_namespace) do |namespace_or_ops_file|
+        case namespace_or_ops_file
+        when Namespace
+          path_map[namespace_or_ops_file.dirname] = namespace_or_ops_file
+        when OpsFile
+          path_map[namespace_or_ops_file.ops_file_path] = namespace_or_ops_file
+        end
+
+        namespace_or_ops_file.symbol_table.values if namespace_or_ops_file.is_a?(Namespace)
+      end
+
+      path_map
+    end
+
+    # returns a Namespace
+    def lookup_namespace(ops_file)
+      @path_map[ops_file.dirname]
+    end
+
+    # returns a Namespace or OpsFile
+    def resolve_symbol(origin_ops_file, symbol_name)
+      lookup_namespace(origin_ops_file)&.resolve_symbol(symbol_name)
+    end
+
+    # returns a Namespace | OpsFile
+    def resolve_import_reference(origin_ops_file, import_reference)
+      case import_reference
+      when PackageDependencyReference
+        # puts "root namespace: #{@root_namespace.symbol_table}"
+        @root_namespace.resolve_symbol(import_reference.package_reference.local_name)   # returns the Namespace associated with the bundled package dirname (i.e. the local name)
+      when DirectoryReference
+        @path_map[import_reference.dirname]
+      when OpsFileReference
+        @path_map[import_reference.ops_file_path]
+      end
+    end
+  end
+
+  class RuntimeEnvironment
+    include Traversable
+
+    attr_accessor :app
+    attr_accessor :pty
+
+    def initialize(app)
+      @app = app
+      @bundle_load_path = LoadPath.new(self, @app.bundle_dir)
+      @app_load_path = LoadPath.new(self, @app.pwd)
+
+      configure_sshkit
+    end
+
+    # configure sshkit globally
+    def configure_sshkit
+      SSHKit.config.use_format :blackhole
+      SSHKit.config.output_verbosity = :info
+
+      if app.debug?
+        SSHKit.config.use_format :pretty
+        # SSHKit.config.use_format :simpletext
+        SSHKit.config.output_verbosity = :debug
+      elsif app.verbose?
+        # SSHKit.config.use_format :dot
+        SSHKit.config.output_verbosity = :info
+      end
+
+      SSHKit::Backend::Netssh.configure do |ssh|
+        ssh.pty = true                # necessary for interaction with sudo on the remote host
+        ssh.connection_timeout = 60   # seconds
+        ssh.ssh_options = {           # this hash is passed in as the options hash (3rd argument) to Net::SSH.start(host, user, options) - see https://net-ssh.github.io/net-ssh/Net/SSH.html
+          auth_methods: %w(publickey password),   # :auth_methods => an array of authentication methods to try
+                                                  # :forward_agent => set to true if you want the SSH agent connection to be forwarded
+                                                  # :keepalive => set to true to send a keepalive packet to the SSH server when there's no traffic between the SSH server and Net::SSH client for the keepalive_interval seconds. Defaults to false.
+                                                  # :keepalive_interval => the interval seconds for keepalive. Defaults to 300 seconds.
+                                                  # :keepalive_maxcount => the maximun number of keepalive packet miss allowed. Defaults to 3
+          timeout: 2,                             # :timeout => how long to wait for the initial connection to be made
+        }
+      end
+      SSHKit::Backend::Netssh.pool.idle_timeout = 1   # seconds
+    end
+
+    def debug?
+      @app.debug?
+    end
+
+    def verbose?
+      @app.verbose?
+    end
+
+    def local_hostname
+      @app.local_hostname
+    end
+
+    def sudo_user
+      @app.sudo_user
+    end
+
+    def sudo_password
+      @app.sudo_password
+    end
+
+    def zip_bundle_path
+      app.zip
+    end
+
+    def run(entry_point_ops_file, params_hash)
+      runtime_env = self
+      SSHKit::Backend::LocalPty.new do
+        runtime_env.pty = self
+        retval = runtime_env.invoke(entry_point_ops_file, params_hash)
+        runtime_env.pty = nil
+        retval
+      end.run
+    end
+
+    def invoke(ops_file, params_hash)
+      ops_file.invoke(self, params_hash)
+    end
+
+    # returns a Namespace or OpsFile
+    def resolve_sibling_symbol(origin_ops_file, symbol_name)
+      @app_load_path.resolve_symbol(origin_ops_file, symbol_name)
+    end
+
+    # returns a Namespace | OpsFile
+    def resolve_import_reference(origin_ops_file, import_reference)
+      case import_reference
+
+      # we know we're dealing with a package dependency reference, so we want to do the lookup in the bundle load path, where package dependencies live
+      when PackageDependencyReference
+        @bundle_load_path.resolve_import_reference(origin_ops_file, import_reference)
+
+      # we know we're dealing with a directory reference or OpsFile reference outside of the bundle dir, so we want to do the lookup in the app load path
+      when DirectoryReference, OpsFileReference
+        @app_load_path.resolve_import_reference(origin_ops_file, import_reference)
+      end
+    end
+
+  end
+
+end

data/lib/opswalrus/sshkit_ext.rb

@@ -0,0 +1,51 @@
+require 'digest'
+require 'logger'
+require 'securerandom'
+require 'sshkit'
+
+require_relative 'local_non_blocking_backend'
+require_relative 'local_pty_backend'
+
+module SSHKit
+  module Backend
+    class Abstract
+      def execute_cmd(*args)
+        # puts "args: #{args.inspect}"
+        options = { verbosity: :debug, strip: true }.merge(args.extract_options!)
+        # puts "options: #{options.inspect}"
+        create_command_and_execute(args, options)
+      end
+    end
+  end
+
+  module Runner
+    class Sequential < Abstract
+      def run_backend(host, &block)
+        backend(host, &block).run
+      # rescue ::StandardError => e
+      #   e2 = ExecuteError.new e
+      #   raise e2, "Exception while executing #{host.user ? "as #{host.user}@" : "on host "}#{host}: #{e.message}"
+      end
+    end
+  end
+
+  class Command
+    # Initialize a new Command object
+    #
+    # @param  [Array] A list of arguments, the first is considered to be the
+    # command name, with optional variadaric args
+    # @return [Command] An un-started command object with no exit staus, and
+    # nothing in stdin or stdout
+    #
+    def initialize(*args)
+      raise ArgumentError, "Must pass arguments to Command.new" if args.empty?
+      @options = default_options.merge(args.extract_options!)
+      # @command = sanitize_command(args.shift)
+      @command = args.shift
+      @args    = args
+      @options.symbolize_keys!    # @options.transform_keys!(&:to_sym)
+      @stdout, @stderr, @full_stdout, @full_stderr = String.new, String.new, String.new, String.new
+      @uuid = Digest::SHA1.hexdigest(SecureRandom.random_bytes(10))[0..7]
+    end
+  end
+end

data/lib/opswalrus/traversable.rb

@@ -0,0 +1,15 @@
+module OpsWalrus
+  module Traversable
+    # the yield block visits the node and returns children that should be visited
+    def pre_order_traverse(root, observed_nodes = Set.new, &visit_fn_block)
+      # there shouldn't be any cycles in a tree, but we're going to make sure!
+      return if observed_nodes.include?(root)
+      observed_nodes << root
+
+      children = visit_fn_block.call(root)
+      children&.each do |child_node|
+        pre_order_traverse(child_node, observed_nodes, &visit_fn_block)
+      end
+    end
+  end
+end

data/lib/opswalrus/version.rb

@@ -0,0 +1,3 @@
+module OpsWalrus
+  VERSION = "1.0.0"
+end

data/lib/opswalrus/walrus_lang.rb

@@ -0,0 +1,83 @@
+require "citrus"
+require "stringio"
+
+module WalrusLang
+  module Templates
+    def render(binding)
+      captures(:template).map{|t| t.render(binding) }.join
+    end
+  end
+
+  module Template
+    def render(binding)
+      s = StringIO.new
+      if mustache = capture(:mustache)
+        s << capture(:pre).value
+        s << mustache.render(binding)
+        s << capture(:post).value
+      else
+        s << capture(:fallthrough).value
+      end
+      s.string
+    end
+  end
+
+  module Mustache
+    def render(binding)
+      eval(capture(:expr).render(binding), binding)
+    end
+  end
+
+  Grammar = <<-GRAMMAR
+    grammar WalrusLang::Parser
+      rule templates
+        (template*) <WalrusLang::Templates>
+      end
+
+      rule template
+        ((pre:(non_mustache?) mustache post:(non_mustache?)) | fallthrough:non_mustache) <WalrusLang::Template>
+      end
+
+      rule before_mustache
+        ~'{{'
+      end
+
+      rule non_mustache
+        ~('{{' | '}}')
+      end
+
+      rule mustache
+        ('{{' expr:templates '}}') <WalrusLang::Mustache>
+      end
+    end
+  GRAMMAR
+
+  Citrus.eval(Grammar)
+
+  def self.render(template, binding)
+    ast = WalrusLang::Parser.parse(template)
+    ast.render(binding)
+  end
+end
+
+class Binding
+  def local_vars_hash
+    local_variables.map {|s| [s.to_s, local_variable_get(s)] }.to_h
+  end
+end
+
+def mustache(&block)
+  template_string = block.call
+  template_string =~ /{{.*}}/ ? WalrusLang.render(block.call, block.binding) : template_string
+end
+
+
+# foo = 1
+# bar = 2
+# # m = TemplateLang.parse("abc; {{ 'foo' * bar }} def ")
+# m = WalrusLang::Parser.parse("abc; {{ 'foo{{1+2}}' * bar }} def {{ 4 * 4 }}; def")
+# # m = TemplateLang.parse("a{{b{{c}}d}}e{{f}}g{{h{{i{{j{{k{{l}}m{{n}}o}}p}}}}}}")
+# # puts m.dump
+# puts m.render(binding)
+
+# puts(mustache { "abc {{ 1 + 2 }} def" })

data/lib/opswalrus/zip.rb

@@ -0,0 +1,57 @@
+require 'zip'
+
+module OpsWalrus
+  class DirZipper
+
+    # Zip the input directory.
+    # returns output_file
+    def self.zip(input_dir, output_file)
+      entries = Dir.entries(input_dir) - %w[. ..]
+
+      ::Zip::File.open(output_file, create: true) do |zipfile|
+        write_entries(input_dir, entries, '', zipfile)
+      end
+
+      output_file
+    end
+
+    # returns output_dir
+    def self.unzip(input_file, output_dir)
+      if !File.exist?(output_dir) || File.directory?(output_dir)
+        FileUtils.mkdir_p(output_dir)
+        ::Zip::File.foreach(input_file) do |entry|
+          path = File.join(output_dir, entry.name)
+          entry.extract(path) unless File.exist?(path)
+        end
+      else
+        raise Error, "#{output_dir} is not a directory"
+      end
+
+      output_dir
+    end
+
+    def self.write_entries(input_dir, entries, path, zipfile)
+      entries.each do |e|
+        zipfile_path = path == '' ? e : File.join(path, e)
+        disk_file_path = File.join(input_dir, zipfile_path)
+
+        if File.directory?(disk_file_path)
+          zipfile.mkdir(zipfile_path)
+          subdir = Dir.entries(disk_file_path) - %w[. ..]
+          write_entries(input_dir, subdir, zipfile_path, zipfile)
+        else
+          zipfile.add(zipfile_path, disk_file_path)
+        end
+      end
+    end
+
+  end
+end
+
+# this is just to test the zip function
+def main
+  OpsWalrus::DirZipper.zip("../example/davidinfra", "test.zip")
+  OpsWalrus::DirZipper.unzip("test.zip", "test")
+end
+
+main if __FILE__ == $0

data/lib/opswalrus.rb

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+require 'rubygems'
+require 'bundler/setup'
+
+require_relative 'opswalrus/app'
+require_relative 'opswalrus/cli'
+require_relative 'opswalrus/version'
+
+module OpsWalrusWalrus
+end

data/opswalrus.gemspec

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+require_relative "lib/opswalrus/version"
+
+Gem::Specification.new do |spec|
+  spec.name = "opswalrus"
+  spec.version = OpsWalrus::VERSION
+  spec.authors = ["David Ellis"]
+  spec.email = ["david@conquerthelawn.com"]
+
+  spec.summary = "opswalrus is a tool that runs scripts against a fleet of hosts"
+  spec.description = "opswalrus is a tool that runs scripts against a fleet of hosts hosts. It's kind of like Ansible, but aims to be simpler to use."
+  spec.homepage = "https://github.com/opswalrus/opswalrus"
+  spec.required_ruby_version = ">= 2.6.0"
+
+  # spec.metadata["allowed_push_host"] = "TODO: Set to your gem server 'https://example.com'"
+
+  spec.metadata["homepage_uri"] = spec.homepage
+  spec.metadata["source_code_uri"] = "https://github.com/opswalrus/opswalrus"
+  # spec.metadata["changelog_uri"] = "TODO: Put your gem's CHANGELOG.md URL here."
+
+  # Specify which files should be added to the gem when it is released.
+  # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
+  spec.files = Dir.chdir(__dir__) do
+    `git ls-files -z`.split("\x0").reject do |f|
+      (File.expand_path(f) == __FILE__) || f.start_with?(*%w[bin/ test/ spec/ features/ .git .circleci appveyor])
+    end
+  end
+  spec.bindir = "exe"
+  spec.executables = spec.files.grep(%r{\Aexe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  # gem dependencies
+  spec.add_dependency "citrus"
+  spec.add_dependency "gli"
+  spec.add_dependency "git"
+  spec.add_dependency "rubyzip"
+
+  spec.add_dependency "bcrypt_pbkdf"
+  spec.add_dependency "ed25519"
+  spec.add_dependency "sshkit"
+
+  # For more information and examples about making a new gem, check out our
+  # guide at: https://bundler.io/guides/creating_gem.html
+end

data/sig/opswalrus.rbs

@@ -0,0 +1,4 @@
+module OpsWalrusWalrus
+  VERSION: String
+  # See the writing guide of rbs: https://github.com/ruby/rbs#guides
+end