vop 0.3.5 → 0.3.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 2e82b43bd88a68ea52a0cc6fa550fa47fb063cbd
-  data.tar.gz: d88c69b59fb9b4e44d8b17f5d8f8b369ad40cf15
+SHA256:
+  metadata.gz: 701c7d9de56bad966ff63365cd24f6feecf5f63314b0dfbd480a5890f23a98b8
+  data.tar.gz: 83f8ac7420c3d9a95bb5087c19d70ebaf6ebd228716201588b693f6b89667818
 SHA512:
-  metadata.gz: b1eba461ca7c2c0ec1ec030f0bfaa2918adc9e74678284daaf215ec16f69bbb5a8f2be5292253a9d55b6bfc86a5404b50e02761146a7a1e1ff0fcd21d4e52e3e
-  data.tar.gz: b871a90a6488754cf5e5f984cbe893766a2d775e9f75fb86b0619c361354de36a749b619b2f62c42ba73aceb1567afdfb3a35866732a7d5da61469957750ac0e
+  metadata.gz: 8acbbc85ce5502380b618116fbe386675c804997e7bb72cd8f81bbec23ed16d06bd7d308dac28d7c3496d7c8f5cdfa0b9a220f9a509815a3629396b695e5ed3a
+  data.tar.gz: ed870d852b55a179bf4cd47d72058eeb111f81c456d5e338af1e086f9596a90021bc83b109dee537b0ea49cdbfa4ce0d931148229e4936b3f08c2b808a504a6b

data/.github/workflows/ci.yml

@@ -0,0 +1,39 @@
+# https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-ruby
+
+name: CI
+
+on:
+  push:
+    branches:
+      - master
+      - and-tonic
+  pull_request:
+    branches: [ master, and-tonic ]
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+
+    services:
+      # https://docs.github.com/en/actions/using-containerized-services/creating-redis-service-containers
+      redis:
+        image: redis
+        # Set health checks to wait until redis has started
+        options: >-
+          --health-cmd "redis-cli ping"
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+        ports:
+          - 6379:6379
+
+    steps:
+      - uses: actions/checkout@v3
+      - name: Set up Ruby
+        uses: ruby/setup-ruby@359bebbc29cbe6c87da6bc9ea3bc930432750108
+        with:
+          ruby-version: '3.1'
+      - name: Install dependencies
+        run: bundle install
+      - name: Run tests
+        run: bundle exec rake

data/.ruby-version

@@ -0,0 +1 @@
+3.1.2

data/Gemfile.lock

@@ -1,67 +1,54 @@
 PATH
   remote: .
   specs:
-    vop (0.3.5)
-      net-scp
-      net-ssh
-      redis
-      sidekiq
+    vop (0.3.6)
+      byebug
+      json (~> 2.3)
       terminal-table
       xml-simple
 
 GEM
   remote: https://rubygems.org/
   specs:
-    concurrent-ruby (1.0.5)
-    connection_pool (2.2.1)
-    diff-lcs (1.2.5)
-    docile (1.1.5)
-    json (1.8.6)
-    net-scp (1.2.1)
-      net-ssh (>= 2.6.5)
-    net-ssh (4.2.0)
-    rack (2.0.3)
-    rack-protection (2.0.0)
-      rack
-    rake (10.5.0)
-    redis (4.0.1)
-    rspec (3.4.0)
-      rspec-core (~> 3.4.0)
-      rspec-expectations (~> 3.4.0)
-      rspec-mocks (~> 3.4.0)
-    rspec-core (3.4.1)
-      rspec-support (~> 3.4.0)
-    rspec-expectations (3.4.0)
+    byebug (11.1.3)
+    diff-lcs (1.5.0)
+    docile (1.4.0)
+    json (2.7.1)
+    rake (13.1.0)
+    rexml (3.2.6)
+    rspec (3.12.0)
+      rspec-core (~> 3.12.0)
+      rspec-expectations (~> 3.12.0)
+      rspec-mocks (~> 3.12.0)
+    rspec-core (3.12.2)
+      rspec-support (~> 3.12.0)
+    rspec-expectations (3.12.3)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.4.0)
-    rspec-mocks (3.4.0)
+      rspec-support (~> 3.12.0)
+    rspec-mocks (3.12.6)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.4.0)
-    rspec-support (3.4.1)
-    sidekiq (5.0.5)
-      concurrent-ruby (~> 1.0)
-      connection_pool (~> 2.2, >= 2.2.0)
-      rack-protection (>= 1.5.0)
-      redis (>= 3.3.4, < 5)
-    simplecov (0.11.2)
-      docile (~> 1.1.0)
-      json (~> 1.8)
-      simplecov-html (~> 0.10.0)
-    simplecov-html (0.10.0)
-    terminal-table (1.8.0)
-      unicode-display_width (~> 1.1, >= 1.1.1)
-    unicode-display_width (1.3.0)
-    xml-simple (1.1.5)
+      rspec-support (~> 3.12.0)
+    rspec-support (3.12.1)
+    simplecov (0.22.0)
+      docile (~> 1.1)
+      simplecov-html (~> 0.11)
+      simplecov_json_formatter (~> 0.1)
+    simplecov-html (0.12.3)
+    simplecov_json_formatter (0.1.4)
+    terminal-table (3.0.2)
+      unicode-display_width (>= 1.1.1, < 3)
+    unicode-display_width (2.5.0)
+    xml-simple (1.1.9)
+      rexml
 
 PLATFORMS
-  ruby
+  x86_64-linux
 
 DEPENDENCIES
-  bundler (~> 1.15)
-  rake (~> 10.0)
+  rake
   rspec (~> 3.0)
   simplecov
   vop!
 
 BUNDLED WITH
-   1.16.0
+   2.4.10

data/README.md

@@ -1,25 +1,23 @@
+The vop is a scripting framework.
 
-
-The vop is a systems automation scripting framework.
-
-It organizes (ruby) scripts into commands living in plugins, defines services that can be installed and managed, and comes with a shell and a web interface.
-
-# Status: WIP
-
-This is work in progress. Do not assume everything you read to be totally accurate and/or stable.
+It organizes ruby scripts as *commands* living in *plugins*, defines and manages *services*, and can be accessed from the command line or a web interface.
 
 # Installation
 
 ## as a gem:
 
-    $ gem install vop
+  $ gem install vop
+
+## from source
+
+* checkout the following repos from github.com/virtualop: vop, plugins, bundle
+* install Ruby >= 3.1.2
+* run `bundle install` in all working copies
 
 # Usage
 
 Call `vop` to start the shell.
 
-Use the tab key for completion, type "help" for more info, `list_plugins` and `list_commands` for an overview.
-
 # Syntax
 
 ## Plugins
@@ -51,27 +49,42 @@ hook :after_execute do |payload|
 end
 ```
 
+config:
+```
+config_param "foo" [, { options }]
+config_param! "snafoo" [, { options }]
+```
+
+accessing plugin config inside command:
+```
+run do |plugin|
+  plugin.config["foo"]
+end
+```
+
 ## Commands
 
 ...are loaded from the `commands` folder in a plugin.
 
-minimal:
+Minimally, a command needs only a run block:
 ```
 run { 42 }
 ```
 
-optional:
+Also, you can use any of these:
 ```
 description "Roses are red."
 
 read_only   # => cacheable
+
+dont_log
 ```
 
-defining param(eter)s:
+A command can define parameters
 ```
 param "snafoo"    # optional
 param! "snafoo"   # mandatory
-param! :snafoo    # entity
+param! :snafoo    # mandatory entity
 ```
 param syntax:
 ```
@@ -106,12 +119,38 @@ run do |machine|
 end
 ```
 
+There is special handling in place for block parameters.
+
+Declare one with
+```
+block_param
+```
+or
+```
+block_param!
+```
+and a block can be passed to the command like this:
+```
+@op.foo do
+  # whatever you need to do
+end
+```
+From inside your command, you can access the block through the parameter called `block`.
+
 contribute:
 ```
 contribute to: "other_command" do |params|
 end
 ```
 
+collect contributions:
+```
+@op.collect_contributions(
+  command_name: "other_command",
+  raw_params: {}
+)
+```
+
 
 show (display options for shell output):
 ```
@@ -139,7 +178,18 @@ deploy package: <foo>
 deploy package: [ <foo>, <bar>, <baz> ]
 ```
 
-install (configuration for) a package repository:
+install a virtualop service:
+```
+deploy service: "plugin.service"
+```
+
+deploy configuration from a template:
+```
+deploy template: "foo.conf.erb",
+  to: "/etc/foo/conf.d/foo.conf"
+```
+
+install (configuration for) a debian package repository:
 ```
 deploy repository: {
   alias: "funny-name",
@@ -157,6 +207,58 @@ deploy do |machine|
 end
 ```
 
+## Entities
+
+Each entity is a file in the `/entities` subfolder of a plugin.
+
+A minimal entity is an array of hashes, each with a unique "name" attribute:
+
+```
+entity do
+  [
+    {
+      "name" => "foo"
+    }
+  ]
+end
+```
+
+Instead of "name", a differing key attribute can be specified with `key`:
+```
+key "path"
+
+entity do
+  [
+    {
+      "path" => "/bin/false"
+    }
+  ]
+end
+```
+
+The `on` keyword allows to stack entities onto another, e.g. the `log` entity living on a machine:
+```
+key "path"
+
+on :machine
+
+entity do |machine|
+  # ...
+end
+```
+A stacked entity (or rather, the list command generated from the entity) automatically has a parameter for the entity it is stacked on - in the log example:
+```
+>> logs?
+
+logs
+
+syntax:
+  logs <machine>
+
+parameters:
+  machine
+```
+
 # Development
 
 ## required dependencies

data/bin/sidekiq.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+VOP_DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )/.." && pwd )
+
+cd $VOP_DIR
+
+export VOP_ORIGIN="sidekiq"
+bundle exec sidekiq -r ./lib/boot.rb
+
+cd - >/dev/null

data/exe/vop

@@ -2,5 +2,5 @@
 
 require "vop"
 
-@op = Vop.boot()
-Vop::Shell.run(@op)
+@op = Vop.boot(origin: "shell:#{Process.pid}@#{`hostname`.strip}")
+Vop::Shell.run(@op, ARGV.join(" "))

data/lib/core/cache/commands/invalidate_cache.rb

@@ -0,0 +1,9 @@
+param! "command"
+param! "raw_params", default: {}
+
+run do |params|
+  @op.collect_contributions(
+    command_name: "invalidate_cache",
+    raw_params: params
+  )
+end

data/lib/core/{structure → meta}/commands/list_commands.rb

@@ -1,4 +1,5 @@
-param "plugin_filter", description: "name of a plugin by which commands should be filtered"
+param "plugin_filter", description: "name of a plugin by which commands should be filtered",
+  lookup: lambda { @op.list_plugins }
 
 run do |plugin_filter|
   result = @op.commands.values

data/lib/core/meta/commands/list_filters.rb

@@ -0,0 +1,3 @@
+run do
+  @op.filters.keys
+end

data/lib/core/meta/commands/list_plugins.rb

@@ -0,0 +1,3 @@
+run do |plugin|
+  @op.plugins.map(&:name).sort
+end

data/lib/core/meta/commands/new_plugin.rb

@@ -18,25 +18,21 @@ run do |params|
 
   raise "no such path: #{params["path"]}" unless File.exists? params["path"]
 
-  pp params
-
-  # a plugin is a directory
+  # create a new plugin: a plugin is a directory
   plugin_path = File.join(params["path"], params["name"])
   puts "plugin path : #{plugin_path}"
   Dir.mkdir(plugin_path)
 
-  # with subfolders for commands and helpers
+  # ...with subfolders for commands and helpers
   %w|commands helpers|.each do |thing|
     Dir.mkdir(File.join(plugin_path, thing))
   end
 
-  # and a metadata file called "<name>.plugin"
+  # ...and a metadata file called "<name>.plugin"
   plugin_file = params["name"] + ".plugin"
   full_name = File.join(plugin_path, plugin_file)
   FileUtils.touch full_name
 
-  # TODO content is [] - should probably be nil, though
-  #puts "content: >>#{params["content"].pretty_inspect}<<"
   unless params["content"].nil?
     IO.write(full_name, params["content"])
   end

data/lib/core/shell/commands/detail.rb

@@ -0,0 +1,21 @@
+param "index"
+
+show display_type: :raw
+
+run do |shell, index|
+  last_result = shell.last_response&.result
+
+  result = if last_result
+    if last_result.is_a?(Array) && index
+      last_result[index.to_i]
+    else
+      last_result
+    end
+  end
+
+  if result.is_a? ::Vop::Entity
+    result.data
+  else
+    result
+  end
+end

data/lib/core/shell/commands/edit.rb

@@ -2,10 +2,13 @@ param! "name", lookup: lambda { @op.commands.keys }
 
 run do |params, name|
   has_vim = `which vim`
-  editor = ENV["EDITOR"] || "vim" if has_vim
+  editor = ENV["EDITOR"]
+  if editor.nil?
+    editor = "vim" if has_vim
+  end
   raise "please set the EDITOR environment variable" unless editor
 
-  command_file = @op.commands[name].source[:file_name]  
+  command_file = @op.commands[name].source[:file_name]
   system("#{editor} #{command_file}")
 
   @op.reset

data/lib/core/shell/commands/help.rb

@@ -1,6 +1,6 @@
 param! "name",
   description: "a command for which help should be displayed",
-  lookup: lambda { |params| @op.list_commands.split }
+  lookup: lambda { |params| @op.list_commands }
 
 run do |params, name|
   command = @op.commands[params["name"]]

data/lib/core/shell/commands/source.rb

@@ -1,6 +1,7 @@
 param! "name", lookup: lambda { @op.commands.keys + @op.entities.keys + @op.filters.keys }
+param "numbers", default: true
 
-run do |name|
+run do |name, numbers|
   (source, thing) = if @op.commands.keys.include? name
     [ :commands, @op.commands[name] ]
   elsif @op.entities.keys.include? name
@@ -13,9 +14,14 @@ run do |name|
 
   result = []
   result << " "
-  code.lines.each_with_index { |line, idx|
-    result << "%02d %s" % [idx+1, line.chomp]
-  }
+  code.lines.each_with_index do |line, idx|
+    line.chomp!
+    line = numbers ?
+      "%02d %s" % [idx+1, line] :
+      line
+
+    result << line
+  end
   result << " "
   result.join("\n")
 end

data/lib/core/structure/commands/collect_contributions.rb

@@ -31,11 +31,17 @@ run do |command_name, raw_params|
     contribution = @op.execute(short_name, raw_params)
 
     if contribution.nil?
-      $logger.warn "command #{short_name} contributes a nil value"
+      $logger.debug "command #{short_name} contributes a nil value"
     else
       case display_type
       when :table
-        result += contribution
+        if contribution.is_a? Array
+          result += contribution
+        else
+          if target.show_options[:display_type] == :table
+            $logger.warn "contribution from #{short_name} is not an Array (but a #{contribution.class}), though the :table display type would imply that (ignoring)"
+          end
+        end
       when :hash
         result.merge! contribution
       end

data/lib/core/structure/commands/generate_entity_commands.rb

@@ -4,21 +4,23 @@ run do
   result = []
 
   @op.entities.each do |entity_name, definition|
-    list_command_name = definition.name.carefully_pluralize
-    $logger.debug "generating entity list command #{list_command_name} (#{definition.plugin.name})"
+    $logger.debug "generating entity list command #{definition.list_command_name} (#{definition.plugin.name})"
 
     plugin = definition.plugin
 
-    list_command = Command.new(plugin, list_command_name)
-    # TODO list_command.read_only = true
+    list_command = Command.new(plugin, definition.list_command_name)
+    list_command.read_only = definition.read_only
+    list_command.dont_log = true
+    list_command.show_options = definition.show_options
 
     if definition.on
-      list_command.add_param(definition.on.to_s, mandatory: true)
+      list_command.add_param(definition.on.to_s, mandatory: true, entity: true)
     end
 
     list_command.block = lambda do |params, request, context, plugin|
       ex = Executor.new(@op)
       block_param_names = definition.block.parameters.map { |x| x.last }
+
       payload = ex.prepare_payload(request, context, block_param_names)
 
       hash_array = definition.block.call(*payload)
@@ -31,11 +33,15 @@ run do
         entity_array = []
         unless hash_array.empty?
           first = hash_array.first
-          unless first.is_a? Hash
-            raise "entity '#{definition.name}' returned unexpected data type : found #{first.class}, expected Hash"
-          end
-          entity_array = hash_array.map do |row|
-            Entity.new(@op, definition.short_name, definition.key, row)
+          if first.is_a? Entity
+            entity_array = hash_array
+          else
+            unless first.is_a? Hash
+              raise "entity '#{definition.name}' returned unexpected data type : found #{first.class}, expected Hash"
+            end
+            entity_array = hash_array.map do |row|
+              Entity.new(@op, definition, row)
+            end
           end
         end
 
@@ -43,6 +49,9 @@ run do
         ::Vop::Entities.new(entity_array)
       end
     end
+
+    list_command.invalidation_block = definition.invalidation_block
+
     result << list_command
   end
 

data/lib/core/structure/commands/generate_invalidation_commands.rb

@@ -2,26 +2,36 @@ description "all read-only commands get <command>! commands that invalidate auto
 
 run do
   count = 0
-  @op.commands.values.each do |command|
-    # actually, all read-only commands *and* all commands with an invalidation block
-    if command.read_only || command.invalidation_block
-      invalidation_command_name = "#{command.short_name}!"
+
+  def setup_invalidation_command(thing, invalidation_command_name, command_name)
+    if thing.read_only || thing.invalidation_block
       $logger.debug "generating invalidation command #{invalidation_command_name}"
 
-      invalidation_command = Command.new(command.plugin, invalidation_command_name)
-      invalidation_command.params = command.params
+      invalidation_command = Command.new(thing.plugin, invalidation_command_name)
+      invalidation_command.params = thing.params
 
       invalidation_command.block = lambda do |params|
         @op.invalidate_cache(
-          "command" => command.short_name,
+          "command" => command_name,
           "raw_params" => params
         )
-        @op.execute(command.short_name, params)
+        @op.execute(command_name, params)
       end
 
       @op << invalidation_command
-      count += 1
     end
   end
+
+  @op.commands.values.each do |command|
+    setup_invalidation_command(command, "#{command.short_name}!", command.short_name)
+    count += 1
+  end
+
+  # not needed at the moment (because entity list commands are generated with an invalidation block)
+  # @op.entities.values.each do |entity|
+  #   setup_invalidation_command(entity, "#{entity.list_command_name}!", entity.list_command_name)
+  #   count += 1
+  # end
+
   count
 end

data/lib/core/structure/commands/list_contribution_targets.rb

@@ -0,0 +1,9 @@
+description "lists the target commands that a given command contributes to (the contributees so to speak)"
+
+param! "source_command"
+
+run do |plugin, source_command|
+  registry = plugin.state[:contributions] || {}
+
+  registry.select { |k,v| v.include? source_command.to_s.carefully_pluralize }.keys
+end

data/lib/core/structure/commands/list_contributors.rb

@@ -3,7 +3,7 @@ param "command_name", description: "command for which contributors should be dis
 run do |plugin, command_name|
   registry = plugin.state[:contributions] || {}
   if command_name
-    registry[command_name]
+    registry[command_name] || []
   else
     registry
   end

data/lib/core/structure/structure.plugin

@@ -1,4 +1,4 @@
-hook :loading_finished do
+hook :init_complete do
   @op.generate_entity_commands
   @op.generate_invalidation_commands
 end