opswalrus 1.0.0

data/lib/opswalrus/bootstrap_linux_host2.sh

@@ -0,0 +1,37 @@
+#!/usr/bin/env bash
+
+# there are probably some services that need restarting because they're using old libraries, so we'll just do the easy thing and reboot
+sudo DEBIAN_FRONTEND=noninteractive apt install -yq needrestart
+
+# install basic development tools
+sudo DEBIAN_FRONTEND=noninteractive apt install -yq build-essential
+
+# install ruby dependencies
+sudo DEBIAN_FRONTEND=noninteractive apt install -yq autoconf patch rustc libssl-dev libyaml-dev libreadline6-dev zlib1g-dev libgmp-dev libncurses5-dev libffi-dev libgdbm6 libgdbm-dev libdb-dev uuid-dev
+
+# restart services that need it
+sudo needrestart -q -r a
+
+# vagrant@ubuntu-jammy:~$ sudo needrestart -q -r a
+#  systemctl restart unattended-upgrades.service
+
+# vagrant@ubuntu-jammy:~$ sudo needrestart -r l
+# Scanning processes...
+# Scanning candidates...
+# Scanning linux images...
+#
+# Running kernel seems to be up-to-date.
+#
+# Services to be restarted:
+#
+# Service restarts being deferred:
+#  systemctl restart unattended-upgrades.service
+#
+# No containers need to be restarted.
+#
+# No user sessions are running outdated binaries.
+#
+# No VM guests are running outdated hypervisor (qemu) binaries on this host.
+
+# reboot just in case
+# sudo reboot

data/lib/opswalrus/bootstrap_linux_host3.sh

@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+
+# install homebrew
+NONINTERACTIVE=1 /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)"
+eval "$(/home/linuxbrew/.linuxbrew/bin/brew shellenv)"
+
+# install gcc, rust, ruby
+brew install gcc
+brew install rust
+brew install ruby
+
+# brew install rtx
+# eval "$(rtx activate bash)"   # register a shell hook
+# rtx use -g ruby@3.2   # install ruby via rtx
+
+# download frum for ruby version management
+# curl -L -o frum.tar.gz https://github.com/TaKO8Ki/frum/releases/download/v0.1.2/frum-v0.1.2-x86_64-unknown-linux-musl.tar.gz
+# tar -zxf frum.tar.gz
+# mv frum-v0.1.2-x86_64-unknown-linux-musl/frum ~/bin
+# chmod 755 ~/bin/frum
+

data/lib/opswalrus/bundler.rb

@@ -0,0 +1,175 @@
+require 'git'
+require 'set'
+
+require_relative "package_file"
+require_relative "traversable"
+require_relative "zip"
+
+module OpsWalrus
+  class Bundler
+    BUNDLE_DIR = "opswalrus_bundle"
+
+    include Traversable
+
+    attr_accessor :pwd
+
+    def initialize(working_directory_path)
+      @pwd = working_directory_path.to_pathname
+      @bundle_dir = @pwd.join(BUNDLE_DIR)
+    end
+
+    def bundle_dir
+      @bundle_dir
+    end
+
+    def ensure_package_bundle_directory_exists
+      FileUtils.mkdir_p(@bundle_dir)
+    end
+
+    # # returns the OpsFile within the bundle directory that represents the given ops_file (which is outside of the bundle directory)
+    # def build_bundle_for_ops_file(ops_file)
+    #   if ops_file.package_file   # ops_file is part of larger package
+    #     self_pkg_dir = build_bundle_for_package(ops_file.package_file)
+    #     relative_ops_file_path = ops_file.ops_file_path.relative_path_from(ops_file.package_file.dirname)
+    #   else
+    #     # ops_file is not part of a larger package
+    #     # in this case, we want to bundle the ops_file.dirname into the bundle directory
+    #     update
+    #     self_pkg_dir = include_directory_in_bundle_as_self_pkg(ops_file.dirname)
+    #     relative_ops_file_path = ops_file.ops_file_path.relative_path_from(ops_file.dirname)
+    #   end
+    #   ops_file_in_self_pkg = self_pkg_dir.join(relative_ops_file_path)
+    # end
+
+    # # we want to bundle the package represented by package_file into the bundle directory that
+    # # resides within the package directory that contains the package_file
+    # def build_bundle_for_package(package_file)
+    #   bundler_for_package = Bundler.new(package_file.dirname)
+    #   bundler_for_package.update
+    #   bundler_for_package.include_directory_in_bundle_as_self_pkg(pwd)
+    # end
+
+    def update
+      ensure_package_bundle_directory_exists
+
+      package_yaml_files = pwd.glob("./**/package.yaml") - pwd.glob("./**/#{BUNDLE_DIR}/**/package.yaml")
+      package_files_within_pwd = package_yaml_files.map {|path| PackageFile.new(path.realpath) }
+
+      download_dependency_tree(*package_files_within_pwd)
+    end
+
+    # downloads all transitive package dependencies associated with ops_files
+    # all downloaded packages are placed into @bundle_dir
+    def download_dependency_tree(*ops_files_and_package_files)
+      package_files = ops_files_and_package_files.map(&:package_file).compact.uniq
+
+      package_files.each do |root_package_file|
+        pre_order_traverse(root_package_file) do |package_file|
+          download_package_dependencies(package_file).map do |downloaded_package_directory_path|
+            package_file_path = File.join(downloaded_package_directory_path, "package.yaml")
+            PackageFile.new(package_file_path)
+          end
+        end
+      end
+    end
+
+    # returns the array of the destination directories that the packages that ops_file depend on were downloaded to
+    # e.g. [dir_path1, dir_path2, dir_path3, ...]
+    def download_package_dependencies(package_file)
+      package_file.dependencies.map do |local_name, package_reference|
+        download_package(package_file, package_reference)
+      end
+    end
+
+    # returns the self_pkg directory within the bundle directory
+    # def include_directory_in_bundle_as_self_pkg(dirname = pwd)
+    #   ensure_package_bundle_directory_exists
+
+    #   destination_package_path = @bundle_dir.join("self_pkg")
+
+    #   # recreate the destination package path - self_pkg
+    #   FileUtils.remove_dir(destination_package_path) if destination_package_path.exist?
+    #   FileUtils.mkdir_p(destination_package_path)
+
+    #   # files in dirname except for the BUNDLE_DIR
+    #   files = dirname.glob("*").reject {|f| f.basename.to_s == BUNDLE_DIR }
+    #   files.each do |file|
+    #     FileUtils.cp_r(file, destination_package_path)
+    #   end
+
+    #   destination_package_path
+    # end
+
+    # This method downloads a package_url that is a dependency referenced in the specified package_file
+    # returns the destination directory that the package was downloaded to
+    def download_package(package_file, package_reference)
+      ensure_package_bundle_directory_exists
+
+      local_name = package_reference.local_name
+      package_url = package_reference.package_uri
+      version = package_reference.version
+
+      destination_package_path = @bundle_dir.join(package_reference.dirname)
+      FileUtils.remove_dir(destination_package_path) if destination_package_path.exist?
+
+      case
+      when package_url =~ /\.git/                               # git reference
+        download_git_package(package_url, version, destination_package_path)
+      when package_url.start_with?("file://")                   # local path
+        path = package_url.sub("file://", "")
+        path = path.to_pathname
+        package_path_to_download = if path.relative?            # relative path
+          package_file.containing_directory.join(path)
+        else                                                    # absolute path
+          path.realpath
+        end
+
+        raise Error, "Package not found: #{package_path_to_download}" unless package_path_to_download.exist?
+        FileUtils.cp_r(package_path_to_download, destination_package_path)
+      when package_url.to_pathname.exist? || package_file.containing_directory.join(package_url).exist?     # local path
+        path = package_url.to_pathname
+        package_path_to_download = if path.relative?            # relative path
+          package_file.containing_directory.join(path)
+        else                                                    # absolute path
+          path.realpath
+        end
+
+        raise Error, "Package not found: #{package_path_to_download}" unless File.exist?(package_path_to_download)
+        FileUtils.cp_r(package_path_to_download, destination_package_path)
+      else                                                      # git reference
+        download_git_package(package_url, version, destination_package_path)
+      end
+
+      destination_package_path
+    end
+
+    def download_git_package(package_url, version = nil, destination_package_path = nil)
+      destination_package_path ||= begin
+        package_reference_dirname = sanitize_path(package_url)
+        bundle_dir.join(package_reference_dirname)
+      end
+      FileUtils.remove_dir(destination_package_path) if File.exist?(destination_package_path)
+      if version
+        Git.clone(package_url, destination_package_path, branch: version, config: ['submodule.recurse=true'])
+      else
+        Git.clone(package_url, destination_package_path, config: ['submodule.recurse=true'])
+      end
+      destination_package_path
+    end
+
+    def sanitize_path(path)
+      # found this at https://apidock.com/rails/v5.2.3/ActiveStorage/Filename/sanitized
+      path.encode(Encoding::UTF_8, invalid: :replace, undef: :replace, replace: "�").strip.tr("\u{202E}%$|:;/\t\r\n\\", "-")
+    end
+
+    # returns the directory that the zip file is unzipped into
+    def unzip(zip_bundle_file, output_dir = nil)
+      if zip_bundle_file.to_pathname.exist?
+        output_dir ||= Dir.mktmpdir.to_pathname
+
+        # unzip the bundle into the output_dir directory
+        DirZipper.unzip(zip_bundle_file, output_dir)
+      end
+    end
+  end
+end

data/lib/opswalrus/cli.rb

@@ -0,0 +1,143 @@
+require "gli"
+
+# require_relative "walrus_lang"
+require_relative "app"
+
+module OpsWalrus
+  class Cli
+    extend GLI::App
+
+    pre do |global_options, command, options, args|
+      $app = App.instance(Dir.pwd)
+      $app.set_local_hostname(ENV["WALRUS_LOCAL_HOSTNAME"]) if ENV["WALRUS_LOCAL_HOSTNAME"]
+      true
+    end
+
+    # this is invoked on an unhandled exception or a call to exit_now!
+    on_error do |exception|
+      puts "*" * 80
+      puts "catchall exception handler:"
+      puts exception.message
+      puts exception.backtrace.join("\n")
+      false   # disable built-in exception handling
+    end
+
+    program_desc 'ops is an operation runner'
+
+    desc 'Be verbose'
+    switch [:v, :verbose]
+
+    desc 'Debug'
+    switch [:d, :debug]
+
+    flag [:h, :hosts], multiple: true, desc: "Specify the hosts.yaml file"
+    flag [:t, :tags], multiple: true, desc: "Specify a set of tags to filter the hosts by"
+
+    desc 'Report on the host inventory'
+    long_desc 'Report on the host inventory'
+    command :inventory do |c|
+      c.action do |global_options, options, args|
+        hosts = global_options[:hosts]
+        tags = global_options[:tags]
+
+        $app.set_verbose(global_options[:debug] || global_options[:verbose])
+
+        $app.report_inventory(hosts, tags: tags)
+      end
+    end
+
+    desc 'Run an operation from a package'
+    long_desc 'Run the specified operation found within the specified package'
+    arg_name 'args', :multiple
+    command :run do |c|
+      c.flag [:u, :user], desc: "Specify the user that the operation will run as"
+      c.switch :pass, desc: "Prompt for a sudo password"
+      c.flag [:p, :params], desc: "JSON string that represents the input parameters for the operation. The JSON string must conform to the params schema for the operation."
+
+      c.action do |global_options, options, args|
+        hosts = global_options[:hosts] || []
+        tags = global_options[:tags] || []
+
+        $app.set_inventory_hosts(hosts)
+        $app.set_inventory_tags(tags)
+
+        verbose = case
+        when global_options[:debug]
+          2
+        when global_options[:verbose]
+          1
+        end
+
+        user = options[:user]
+        params = options[:params]
+
+        $app.set_verbose(verbose)
+        $app.set_params(params)
+
+        $app.set_sudo_user(user) if user
+
+        if options[:pass]
+          $app.prompt_sudo_password
+        end
+
+        # puts "verbose"
+        # puts verbose.inspect
+        # puts "user"
+        # puts user.inspect
+        # puts "args"
+        # puts args.inspect
+
+        exit_status = $app.run(args)
+
+        exit_now!("error", exit_status) unless exit_status == 0
+      end
+    end
+
+    desc 'Bundle dependencies'
+    long_desc 'Download and bundle the dependencies for the operations found in the current directory'
+    command :bundle do |c|
+
+      desc 'Update bundle dependencies'
+      long_desc 'Download and bundle the latest versions of dependencies for the current package'
+      c.command :update do |update|
+        update.action do |global_options, options, args|
+          $app.bundle_update
+        end
+      end
+
+      desc 'List bundle dependencies'
+      long_desc 'List bundle dependencies'
+      c.command :status do |status|
+        status.action do |global_options, options, args|
+          $app.bundle_status
+        end
+      end
+
+      desc 'Unzip bundle'
+      long_desc 'Unzip the specified bundle zip file to the specified directory'
+      c.command :unzip do |unzip|
+        unzip.flag [:o, :output], desc: "Specify the output directory"
+
+        unzip.action do |global_options, options, args|
+          output_dir = options[:output]
+          zip_file_path = args.first
+
+          destination_dir = $app.unzip(zip_file_path, output_dir)
+
+          if destination_dir
+            puts destination_dir
+          end
+        end
+      end
+
+      c.default_command :status
+    end
+  end
+end
+
+def main
+  exit_status = OpsWalrus::Cli.run(ARGV)
+  exit exit_status
+end
+
+main if __FILE__ == $0

data/lib/opswalrus/host.rb

@@ -0,0 +1,177 @@
+require "set"
+require "sshkit"
+
+require_relative "interaction_handlers"
+
+module OpsWalrus
+
+  module HostDSL
+    # returns the stdout from the command
+    def sh(desc_or_cmd = nil, cmd = nil, stdin: nil, &block)
+      out, err, status = *shell!(desc_or_cmd, cmd, block, stdin: stdin)
+      out
+    end
+
+    # returns the tuple: [stdout, stderr, exit_status]
+    def shell(desc_or_cmd = nil, cmd = nil, stdin: nil, &block)
+      shell!(desc_or_cmd, cmd, block, stdin: stdin)
+    end
+
+    # returns the tuple: [stdout, stderr, exit_status]
+    def shell!(desc_or_cmd = nil, cmd = nil, block = nil, stdin: nil)
+      # description = nil
+
+      return ["", "", 0] if !desc_or_cmd && !cmd && !block    # we were told to do nothing; like hitting enter at the bash prompt; we can do nothing successfully
+
+      description = desc_or_cmd if cmd || block
+      cmd = block.call if block
+      cmd ||= desc_or_cmd
+
+      if self.alias
+        print "[#{self.alias} | #{host}] "
+      else
+        print "[#{host}] "
+      end
+      print "#{description}: " if description
+      puts cmd
+
+      cmd = WalrusLang.render(cmd, block.binding) if block && cmd =~ /{{.*}}/
+      return unless cmd && !cmd.strip.empty?
+
+      #cmd = Shellwords.escape(cmd)
+
+      # puts "shell: #{cmd}"
+      # puts "shell: #{cmd.inspect}"
+      # puts "sudo_password: #{sudo_password}"
+
+      sshkit_cmd = execute_cmd(cmd)
+
+      [sshkit_cmd.full_stdout, sshkit_cmd.full_stderr, sshkit_cmd.exit_status]
+    end
+
+    # def init_brew
+    #   execute('eval "$(/home/linuxbrew/.linuxbrew/bin/brew shellenv)"')
+    # end
+
+    def run_ops(cmd)
+      shell!("/home/linuxbrew/.linuxbrew/bin/gem exec opswalrus ops #{cmd}")
+    end
+
+  end
+
+  class HostProxy
+    def initialize(host)
+      @host = host
+    end
+    def method_missing()
+    end
+  end
+
+  class Host
+    include HostDSL
+
+    def initialize(name_or_ip_or_cidr, tags = [], props = {})
+      @name_or_ip_or_cidr = name_or_ip_or_cidr
+      @tags = tags.to_set
+      @props = props.is_a?(Array) ? {"tags" => props} : props.to_h
+    end
+
+    def host
+      @name_or_ip_or_cidr
+    end
+
+    def ssh_port
+      @props["port"]
+    end
+
+    def ssh_user
+      @props["user"]
+    end
+
+    def ssh_password
+      @props["password"]
+    end
+
+    def ssh_keys
+      @props["keys"]
+    end
+
+    def hash
+      @name_or_ip_or_cidr.hash
+    end
+
+    def eql?(other)
+      self.class == other.class && self.hash == other.hash
+    end
+
+    def to_s
+      @name_or_ip_or_cidr
+    end
+
+    def alias
+      @props["alias"]
+    end
+
+    def tag!(*tags)
+      enumerables, scalars = tags.partition {|t| Enumerable === t }
+      @tags.merge(scalars)
+      enumerables.each {|enum| @tags.merge(enum) }
+      @tags
+    end
+
+    def tags
+      @tags
+    end
+
+    def summary(verbose = false)
+      report = "#{to_s}\n  tags: #{tags.sort.join(', ')}"
+      if verbose
+        @props.reject{|k,v| k == 'tags' }.each {|k,v| report << "\n  #{k}: #{v}" }
+      end
+      report
+    end
+
+    def sshkit_host
+      @sshkit_host ||= ::SSHKit::Host.new({
+        hostname: host,
+        port: ssh_port || 22,
+        user: ssh_user || raise("No ssh user specified to connect to #{host}"),
+        password: ssh_password,
+        keys: ssh_keys
+      })
+    end
+
+    def set_ssh_connection(sshkit_backend)
+      @sshkit_backend = sshkit_backend
+    end
+
+    def clear_ssh_connection
+      @sskit_backend = nil
+    end
+
+    def execute(*args)
+      # puts "interaction handler responds with: #{ssh_password}"
+      @sshkit_backend.capture(*args, interaction_handler: SudoPasswordMapper.new(ssh_password).interaction_handler, verbosity: :info)
+      # @sshkit_backend.capture(*args, interaction_handler: SudoPromptInteractionHandler.new, verbosity: :info)
+    end
+
+    def execute_cmd(*args)
+      @sshkit_backend.execute_cmd(*args, interaction_handler: SudoPasswordMapper.new(ssh_password).interaction_handler, verbosity: :info)
+    end
+
+    def upload(local_path_or_io, remote_path)
+      source = local_path_or_io.is_a?(IO) ? local_path_or_io : local_path_or_io.to_s
+      @sshkit_backend.upload!(source, remote_path.to_s)
+    end
+
+    def download(remote_path, local_path)
+      @sshkit_backend.download!(remote_path.to_s, local_path.to_s)
+    end
+
+    def run
+
+    end
+
+  end
+
+end

data/lib/opswalrus/hosts_file.rb

@@ -0,0 +1,55 @@
+require "pathname"
+require "yaml"
+require_relative "host"
+
+module OpsWalrus
+  class HostsFile
+    attr_accessor :hosts_file_path
+    attr_accessor :yaml
+
+    def initialize(hosts_file_path)
+      @hosts_file_path = File.absolute_path(hosts_file_path)
+      @yaml = YAML.load(File.read(hosts_file_path)) if File.exist?(hosts_file_path)
+      # puts @yaml.inspect
+    end
+
+    def defaults
+      @defaults ||= (@yaml["defaults"] || @yaml["default"] || {})
+    end
+
+    # returns an Array(Host)
+    def hosts
+      # @yaml is a map of the form:
+      # {
+      #   "198.23.249.13"=>{"hostname"=>"web1", "tags"=>["monopod", "racknerd", "vps", "2.5gb", "web1", "web", "ubuntu22.04"]},
+      #   "107.175.91.150"=>{"tags"=>["monopod", "racknerd", "vps", "2.5gb", "pbx1", "pbx", "ubuntu22.04"]},
+      #   "198.23.249.16"=>{"tags"=>["racknerd", "vps", "4gb", "kvm", "ubuntu20.04", "minecraft"]},
+      #   "198.211.15.34"=>{"tags"=>["racknerd", "vps", "1.5gb", "kvm", "ubuntu20.04", "blog"]},
+      #   "homeassistant.locallan.network"=>{"tags"=>["local", "homeassistant", "home", "rpi"]},
+      #   "synology.locallan.network"=>{"tags"=>["local", "synology", "nas"]},
+      #   "pfsense.locallan.network"=>false,
+      #   "192.168.56.10"=>{"tags"=>["web", "vagrant"]}
+      # }
+      @yaml.map do |host_ref, host_attrs|
+        next if host_ref == "default" || host_ref == "defaults"   # this maps to a nil
+
+        host_params = host_attrs.is_a?(Hash) ? host_attrs : {}
+
+        Host.new(host_ref, tags(host_ref), defaults.merge(host_params))
+      end.compact
+    end
+
+    def tags(host)
+      host_attrs = @yaml[host]
+
+      case host_attrs
+      when Array
+        tags = host_attrs
+        tags.compact.uniq
+      when Hash
+        tags = host_attrs["tags"] || []
+        tags.compact.uniq
+      end || []
+    end
+  end
+end

data/lib/opswalrus/interaction_handlers.rb

@@ -0,0 +1,53 @@
+require 'sshkit'
+
+module OpsWalrus
+
+  class PasswdInteractionHandler
+    def on_data(command, stream_name, data, channel)
+      # puts data
+      case data
+      when '(current) UNIX password: '
+        channel.send_data("old_pw\n")
+      when 'Enter new UNIX password: ', 'Retype new UNIX password: '
+        channel.send_data("new_pw\n")
+      when 'passwd: password updated successfully'
+      else
+        raise "Unexpected stderr #{stderr}"
+    end
+    end
+  end
+
+  class SudoPasswordMapper
+    def initialize(sudo_password)
+      @sudo_password = sudo_password
+    end
+
+    def interaction_handler
+      SSHKit::MappingInteractionHandler.new({
+        /\[sudo\] password for .*?:\s*/ => "#{@sudo_password}\n",
+        # /\s+/ => nil,     # unnecessary
+      }, :info)
+    end
+  end
+
+  class SudoPromptInteractionHandler
+    def on_data(command, stream_name, data, channel)
+      # puts "0" * 80
+      # puts data.inspect
+      case data
+      when /\[sudo\] password for/
+        if channel.respond_to?(:send_data)  # Net::SSH channel
+          channel.send_data("conquer\n")
+        elsif channel.respond_to?(:write)   # IO
+          channel.write("conquer\n")
+        end
+      when /\s+/
+        puts 'space, do nothing'
+      else
+        raise "Unexpected prompt: #{data} on stream #{stream_name} and channel #{channel.inspect}"
+    end
+    end
+  end
+
+
+end