herd-rb 0.3.0 → 0.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f0b080c1095b56bac42536acc7df54cd20c97bac52f2367301e2b951eb8a60be
-  data.tar.gz: 397b3c9636a2dc20310f8e70fbe3911049bde76183fe8650f30d0cff4bb40402
+  metadata.gz: ded7f78ffd316333cc6e7177d7ec84ad9ac91ec6cd9b10a9b3e55c7851fa6e22
+  data.tar.gz: fb2aed4ff5ae882238b6a38905d0685af26a03d11149ae3e1cfe2d4846287e08
 SHA512:
-  metadata.gz: 514947c517caa05fc2229c6d293042bf6065ad2d067574087d479772fac271802279f769a6f98d33ae0c7a09720bd99e6592b475e2e6cc3934c515d970323bc1
-  data.tar.gz: 7962d18779bc695f3858adcbf40dc75350e9d88ec00a29b0ab71b8e9ab8ffd21b18f28e8078c2e012b33612c400941d7394ce5254fa52520be8988bc59d11784
+  metadata.gz: ccaa31336a819b2e845f0a713696303900a049f764d5eb7b1233e554f15fcf497a2d74109a2bf293e3d309a82bc711ad07cd9d38acfac18811f32e9c29869250
+  data.tar.gz: 1088e0d0613804db1ef7a1c2f098bdc63bc01aadf391ab4e712019f65461f4fc8d10ba65d84fd556f6065816c41213e7b5b6aed54b1db29028a2aa1555521d87

data/README.md

@@ -6,7 +6,13 @@ Fast host configuration tool.
 
 * [x] Run with `sudo`
 * [x] Commands with arguments
-* [ ] Reading and writing files
+* [x] Reading and writing files
+* [x] Templates (ERB)
+* [x] Copy dirs
+  * [ ] Compare with Rsync
+* [x] Crontab
+* [x] Log all commands for all hosts
+* [ ] Add user to group
 
 ## Installation
 
@@ -55,6 +61,21 @@ runner.exec("hostname") # ["alpha001\n", "omega001\n"]
 runner.exec { hostname + uptime } # ["alpha001\n2000 years\n", "omega001\2500 years\n"]
 ```
 
+List of hosts can be loaded from the CSV file:
+
+```ruby
+# hosts.csv
+host,port,user,password,some_param1,some_param2
+alpha.tesla.com,2022,elon,T0pS3kr3t,value1,value2
+omega.tesla.com,2023,elon,T0pS3kr3t2,value3,value4
+```
+
+```ruby
+hosts = Herd::Host.from_csv("hosts.csv")
+runner = Herd::Runner.new(hosts)
+...
+```
+
 ### Something more complex
 
 ```ruby
@@ -72,6 +93,68 @@ result = runner.exec do
     puts "Added new key for host #{h}"
   end
 end
+
+# or even simpler
+my_key2 = "ssh-ed25519 ..."
+
+result = runner.exec do
+  authorized_keys_contains_exactly([my_key, my_key2])
+end
+```
+
+### Files and directories
+
+Following example takes file from the `./files/etc/sudoers.d/50-elon`
+and copy content to the remote host with required permissions.
+
+```ruby
+result = runner.exec do
+  file("/etc/sudoers.d/50-elon", "root", "root", 440)
+
+  # or copy dirs
+  dir("/home/elon/projects", "elon", "elon")
+end
+```
+
+### Templates
+
+Following example takes ERB template from the `./templates/home/elon/.env.erb`
+and renders using additional `Herd::Host` values and copies content to the remote host
+`/home/elon/.env`:
+
+```erb
+# File: ./templates/home/elon/.env.erb
+export ALIAS=<%= alias %>
+```
+
+```ruby
+host = Herd::Host.new("tesla.com", "elon", password: "T0pS3kr3t", alias: "alpha001")
+runner = Runner.new([host])
+runner.exec do |values|
+  # values contain named arguments (except password and public_key_path) 
+  # from the host constructor:
+  # { host: "tesla.com", port: 22, user: "elon", alias: "alpha001" }
+  template("/home/elon/.env", "elon", "wheels", values: values)
+end
+```
+
+### Crontab
+
+```ruby
+crontab("* * * * * /some-script.sh")
+```
+
+### Logs
+Herd logs all commands, outputs and errors into the `log/<host>_<port>_<user>/<timestamp>.json` files:
+
+```json
+{
+{"vars":{"alias":"alpha001","port":22,"host":"tesla.com","user":"elon"}},
+{"timestamp":"2025-11-09 18:10:21.134","command":"test -a /home/elon/.herd-version; echo $?"},
+{"timestamp":"2025-11-09 18:10:21.395","command":"test -a /home/elon/.herd-version; echo $?","output":"4\r\n","time":0.261358},
+{"timestamp":"2025-11-09 18:10:22.013","command":"cat /home/home/.herd-version"},
+{"timestamp":"2025-11-09 18:10:22.314","command":"cat /home/home/.herd-version","output":"4\r\n","time":0.301}
+}
 ```
 
 ## Development

data/lib/herd/commands/authorized_keys.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+module Herd
+  module Commands
+    # Commands for inspecting and managing authorized SSH keys.
+    module AuthorizedKeys
+      AUTHORIZED_KEYS_FILE = "~/.ssh/authorized_keys"
+
+      def authorized_keys_contains_exactly(required_keys)
+        required_keys = [required_keys].flatten
+        diff = keys_diff(authorized_keys, required_keys)
+        self.authorized_keys = required_keys
+
+        diff
+      end
+
+      def authorized_keys
+        read_file(AUTHORIZED_KEYS_FILE)&.split(/\r\n|\r|\n/) || []
+      end
+
+      def authorized_keys=(keys)
+        touch(AUTHORIZED_KEYS_FILE)
+        write_to_file(AUTHORIZED_KEYS_FILE, [keys].flatten.join("\n"))
+        file_permissions(AUTHORIZED_KEYS_FILE, 600)
+      end
+
+      def add_authorized_key(key)
+        touch(AUTHORIZED_KEYS_FILE)
+        append_to_file(AUTHORIZED_KEYS_FILE, key)
+        file_permissions(AUTHORIZED_KEYS_FILE, 600)
+      end
+
+      private
+
+      def keys_diff(actual_keys, required_keys)
+        result = Hash.new { |h, k| h[k] = [] }
+        actual_keys.each do |actual_key|
+          if required_keys.include?(actual_key)
+            result[:existing] << actual_key
+          else
+            result[:obsolete] << actual_key
+          end
+        end
+        result.merge({ added: required_keys - actual_keys })
+      end
+    end
+  end
+end

data/lib/herd/commands/crontab.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+module Herd
+  module Commands
+    # Commands for adding new crontab tasks or replace with given one.
+    module Crontab
+      def add_cron(entry)
+        run(%(crontab -l | { cat; echo "#{entry}"; } | crontab -))
+      end
+
+      def crontab(entry)
+        run(%(echo "#{entry}" | crontab -))
+      end
+    end
+  end
+end

data/lib/herd/commands/files.rb

@@ -0,0 +1,134 @@
+# frozen_string_literal: true
+
+require "diff/lcs"
+require "diff/lcs/hunk"
+
+module Herd
+  module Commands
+    FILES = "files"
+
+    # Create, read, write, remove files, check permssions.
+    module Files
+      class PermissionDeniedError < StandardError; end
+
+      def file_exists?(path)
+        run("test -a #{path}; echo $?").chomp == "0"
+      end
+
+      def file_readable?(path)
+        run("test -r #{path}; echo $?").chomp == "0"
+      end
+
+      def file_writable?(path)
+        run("test -w #{path}; echo $?").chomp == "0"
+      end
+
+      def dir(path, user, group, sudo: false)
+        Dir.glob(File.join(FILES, path, "**/*"), File::FNM_DOTMATCH).each do |f|
+          remote_path = f.sub(FILES, "")
+          if File.directory?(f)
+            mkdir_p(remote_path, user, group, sudo: sudo)
+          else
+            file(remote_path, user, group)
+          end
+        end
+      end
+
+      def mkdir_p(path, user, group, sudo: false)
+        if sudo
+          run("sudo mkdir -p #{path}")
+        else
+          run("mkdir -p #{path}")
+        end
+        file_user_and_group(path, user, group)
+      end
+
+      def file(path, user, group, content: nil, mode: nil)
+        required_content = if content.nil?
+                             File.read(File.join(FILES, path))
+                           else
+                             content
+                           end
+
+        expect_file_content_equals(path, required_content)
+
+        file_user_and_group(path, user, group)
+        file_permissions(path, mode) if mode
+      end
+
+      def expect_file_content_equals(path, required_content)
+        if file_exists?(path)
+          actual_content = read_file!(path, sudo: true)
+          unless actual_content.lines(chomp: true) == required_content.lines(chomp: true)
+            # "File has been replaced with diff:\n\n#{diff(actual_content, required_content)}"
+            write_to_file!(path, required_content, sudo: true)
+          end
+        else
+          write_to_file!(path, required_content, sudo: true)
+        end
+      end
+
+      def read_file!(path, sudo: false)
+        if file_readable?(path)
+          read_file(path)
+        elsif sudo
+          read_file(path, sudo: true)
+        else
+          raise PermissionDeniedError, "'#{path}' is not readable"
+        end
+      end
+
+      def read_file(path, sudo: false)
+        command = "cat #{path}"
+        command = "sudo #{command}" if sudo
+
+        result = run(command)&.chomp
+        result = result.sub(/\A(\r\n|\r|\n)/, "") if sudo
+
+        result
+      end
+
+      def write_to_file!(path, content, sudo: false)
+        if file_writable?(path)
+          write_to_file(path, content)
+        elsif sudo
+          write_to_file(path, content, sudo: true)
+        else
+          raise PermissionDeniedError, "'#{path}' is not writable"
+        end
+      end
+
+      def write_to_file(path, content, sudo: false)
+        command = "tee"
+        command = "sudo #{command}" if sudo
+        run(%(#{command} #{path} << EOF
+#{content}
+EOF))
+      end
+
+      def append_to_file(path, content, sudo: false)
+        command = "tee -a"
+        command = "sudo #{command}" if sudo
+        run(%(#{command} #{path} << EOF
+#{content}
+EOF))
+      end
+
+      def file_user_and_group(path, user, group)
+        sudo("chown #{user}:#{group} #{path}")
+      end
+
+      def file_permissions(path, mode)
+        sudo("chmod #{mode} #{path}")
+      end
+
+      def diff(actual, required)
+        actual   = actual.lines(chomp: true)
+        required = required.lines(chomp: true)
+
+        diffs = Diff::LCS.diff actual, required
+        Diff::LCS::Hunk.new(actual, required, diffs[0], 3, 0).diff(:unified)
+      end
+    end
+  end
+end

data/lib/herd/{session/commands → commands}/packages.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 module Herd
-  module SessionCommands
+  module Commands
     # Working with package managers, like apt for Ubuntu
     module Packages
       def install_packages(packages)

data/lib/herd/commands/python.rb

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+module Herd
+  module Commands
+    # Commands for python environment
+    module Python
+      def install_uv
+        run("curl -LsSf https://astral.sh/uv/install.sh | sh")
+      end
+    end
+  end
+end

data/lib/herd/commands/templates.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+require "erb"
+
+module Herd
+  module Commands
+    TEMPLATES = "templates"
+
+    # Render templates using ERB
+    module Templates
+      def template(path, user, group, mode: nil, values: {})
+        erb = ERB.new(File.read(File.join(TEMPLATES, "#{path}.erb")))
+        content = erb.result_with_hash(values)
+        file(path, user, group, mode: mode, content: content)
+      end
+    end
+  end
+end

data/lib/herd/host.rb

@@ -1,36 +1,60 @@
 # frozen_string_literal: true
 
+require "csv"
+require "json"
 require "net/ssh"
 
 module Herd
   # Target host
   class Host
-    attr_reader :host, :user, :ssh_options, :password
+    include Herd::Log
 
-    def initialize(host, user, port: 22, private_key_path: nil, password: nil)
+    attr_reader :host, :user, :ssh_options, :password, :vars, :log
+
+    # port, private_key_path, password are for the ssh connection
+    def initialize(host, user, options)
       @host = host
       @user = user
 
-      @ssh_options = { port: port, timeout: 10 }
-      if private_key_path
-        @ssh_options[:keys] = [private_key_path]
+      create_ssh_options(options)
+
+      @password = options.delete(:password)
+      @vars = options.merge(host: host, user: user, port: ssh_options[:port])
+
+      open_log
+    end
+
+    def create_ssh_options(options)
+      @ssh_options = { port: options[:port] || 22, timeout: 10 }
+      if options[:private_key_path]
+        @ssh_options[:keys] = [options.delete(:private_key_path)]
       else
-        @ssh_options[:password] = password
+        @ssh_options[:password] = options[:password]
       end
-
-      @password = password
     end
 
     def exec(command = nil, &)
       Net::SSH.start(host, user, ssh_options) do |ssh|
-        session = Herd::Session.new(ssh, password)
+        session = Herd::Session.new(ssh, password, log)
 
-        output = nil
         output = session.send(command) if command
-        output = session.instance_exec(&) if block_given?
-
+        output = session.instance_exec(vars, &) if block_given?
         output
       end
+    rescue StandardError => e
+      log_connection_error(e)
+    ensure
+      close_log
+    end
+
+    def self.from_csv(file = "hosts.csv")
+      CSV.read(file, headers: true).map do |csv|
+        h = csv.to_h.transform_keys(&:to_sym)
+        host = h.delete(:host)
+        user = h.delete(:user)
+        port = h.delete(:port).to_i
+        new(host, user, h.merge(port: port))
+      end
     end
   end
 end

data/lib/herd/log.rb

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+require "fileutils"
+require "json"
+
+module Herd
+  # Methods for logging commands, outputs and errors.
+  module Log
+    def log_file_path
+      dir = "log/#{vars[:host]}_#{vars[:port]}_#{vars[:user]}"
+      FileUtils.mkdir_p(dir)
+      "#{File.join(dir, Time.now.strftime("%Y%m%d_%H%M%S"))}.json"
+    end
+
+    def open_log
+      @log = File.open(log_file_path, "w")
+
+      log.puts "{"
+      log.print({ vars: vars }.to_json)
+    end
+
+    def close_log
+      log.puts "\n}"
+      log.close
+    end
+
+    def log_connection_error(error)
+      puts "#{vars.inspect}: #{error.message}"
+      log.puts ","
+      log.print({ error: error.message, error_trace: error.backtrace }.to_json)
+    end
+
+    def log_command_start(timestamp, command)
+      log.puts(",")
+      log.print({ timestamp: time(timestamp), command: command }.to_json)
+    end
+
+    def log_command_output(command, output, started_at)
+      now = Time.now
+      log.puts(",")
+      log.print({ timestamp: time(now), command: command, output: output, time: now - started_at }.to_json)
+    end
+
+    def log_command_error(command, error, started_at)
+      now = Time.now
+      log.puts(",")
+      log.print({ timestamp: time(now), command: command, error: error, time: now - started_at }.to_json)
+    end
+
+    def time(timestamp = Time.now)
+      timestamp.strftime("%Y-%m-%d %H:%M:%S.%L")
+    end
+  end
+end

data/lib/herd/session.rb

@@ -3,14 +3,17 @@
 module Herd
   # Session for executing commands on the remote host
   class Session
+    include Herd::Log
+
     OS_COMMANDS = %i[cat chmod echo hostname touch].freeze
-    CUSTOM_COMMANDS_DIR = File.expand_path("session/commands", __dir__)
+    CUSTOM_COMMANDS_DIR = File.expand_path("commands", __dir__)
 
-    attr_reader :ssh, :password
+    attr_reader :ssh, :password, :log
 
-    def initialize(ssh, password = nil)
+    def initialize(ssh, password, log)
       @ssh = ssh
       @password = password
+      @log = log
     end
 
     def method_missing(cmd, *args)
@@ -27,7 +30,7 @@ module Herd
         channel.request_pty do |ch, success|
           raise ::Herd::CommandError, "could not obtain pty" unless success
 
-          channel_run(ch, command, result)
+          channel_run(ch, command, result, Time.now)
         end
       end
       ssh.loop
@@ -42,7 +45,7 @@ module Herd
       def load_command_modules
         command_files.each { |file| require file }
 
-        session_command_modules.each do |mod|
+        command_modules.each do |mod|
           next if self <= mod
 
           prepend mod
@@ -55,29 +58,45 @@ module Herd
         Dir[File.join(CUSTOM_COMMANDS_DIR, "*.rb")]
       end
 
-      def session_command_modules
-        return [] unless defined?(Herd::SessionCommands)
+      def command_modules
+        return [] unless defined?(Herd::Commands)
 
-        Herd::SessionCommands.constants
-                             .sort
-                             .map { |const_name| Herd::SessionCommands.const_get(const_name) }
-                             .select { |value| value.is_a?(Module) }
+        Herd::Commands.constants
+                      .sort
+                      .map { |const_name| Herd::Commands.const_get(const_name) }
+                      .select { |value| value.is_a?(Module) }
       end
     end
 
     private
 
-    def channel_run(channel, command, result)
+    def channel_run(channel, command, result, started_at)
+      log_command_start(started_at, command)
+
       channel.exec(command) do |c, _|
         c.on_data do |_, data|
-          result << data
+          process_output(c, command, started_at, data, result)
         end
 
         c.on_extended_data do |_, _, data|
-          raise ::Herd::CommandError, data
+          process_error(command, started_at, data)
         end
       end
     end
+
+    def process_output(channel, command, started_at, data, result)
+      if data.include?("[sudo] password for")
+        channel.send_data "#{password}\n"
+      else
+        log_command_output(command, data, started_at)
+        result << data
+      end
+    end
+
+    def process_error(command, started_at, data)
+      log_command_error(command, data, started_at)
+      raise ::Herd::CommandError, data
+    end
   end
 end
 

data/lib/herd/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Herd
-  VERSION = "0.3.0"
+  VERSION = "0.5.0"
 end

data/lib/herd.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require_relative "herd/version"
+require_relative "herd/log"
 require_relative "herd/host"
 require_relative "herd/runner"
 require_relative "herd/session"

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: herd-rb
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.5.0
 platform: ruby
 authors:
 - Yury Kotlyarov
@@ -25,12 +25,18 @@ files:
 - Rakefile
 - TODO.md
 - lib/herd.rb
+- lib/herd/commands/authorized_keys.rb
+- lib/herd/commands/crontab.rb
+- lib/herd/commands/files.rb
+- lib/herd/commands/packages.rb
+- lib/herd/commands/python.rb
+- lib/herd/commands/templates.rb
 - lib/herd/host.rb
+- lib/herd/log.rb
 - lib/herd/runner.rb
 - lib/herd/session.rb
-- lib/herd/session/commands/authorized_keys.rb
-- lib/herd/session/commands/packages.rb
 - lib/herd/version.rb
+- log/.keep
 - sig/herd.rbs
 homepage: https://github.com/yura/herd
 licenses:

data/lib/herd/session/commands/authorized_keys.rb

@@ -1,18 +0,0 @@
-# frozen_string_literal: true
-
-module Herd
-  module SessionCommands
-    # Commands for inspecting and managing authorized SSH keys.
-    module AuthorizedKeys
-      def authorized_keys
-        cat("~/.ssh/authorized_keys")&.chomp&.split("\n") || []
-      end
-
-      def add_authorized_key(key)
-        touch("~/.ssh/authorized_keys")
-        chmod("600 ~/.ssh/authorized_keys")
-        echo("'#{key}' >> ~/.ssh/authorized_keys")
-      end
-    end
-  end
-end