vop 0.3.1 → 0.3.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 210d42c576321fd27102c17b4a8f43e1cc76497d
-  data.tar.gz: f40f69a7cb5de67a7cfc0587469f58dda3fe43df
+  metadata.gz: 671eba569f41b30868a8026134138d13399d3d5e
+  data.tar.gz: 5791e1031450edc74942ddc3b388ba8c27521c87
 SHA512:
-  metadata.gz: 6924c43e9ea5ab100f6285c71cb960d6c98da41b826668feee5462ef7dd230b1a78ce3ae488a295f72a63ebaa393d25a7af1fe36bcf216b77f77739e4fcc25c3
-  data.tar.gz: 3867865eb15607c54d2291f833a0acd363a256ed52f79bfd24ed4e0ec68e9bcf44f73e1b222bae53ef4a62af38c4870b237b0622a7a00bb9f28bfe16e0891e14
+  metadata.gz: 2e216682fe3dce4eedbec94ff3d4073084652b405747383085430b16d8abebf0e1b20ecaff2632591592de142db52c34dd98103b27ef31a9df84df2135138549
+  data.tar.gz: 62a72d94b60acd3f92dd0d5da33cbc3b14ee02aecb3380a365c5c5d2bffd8400d0adf96962d19474620d0977a52a6b6a687aca3f10203be1d67a77ced488c6bd

data/.gitignore

@@ -48,3 +48,5 @@ build-iPhoneSimulator/
 
 # unless supporting rvm < 1.11.0 or doing something fancy, ignore this:
 .rvmrc
+.rake_tasks~
+.rspec_status

data/Gemfile

@@ -1,3 +1,4 @@
-source 'http://rubygems.org'
+source "https://rubygems.org"
 
+# Specify your gem's dependencies in vop.gemspec
 gemspec

data/Gemfile.lock

@@ -1,37 +1,30 @@
 PATH
   remote: .
   specs:
-    vop (0.3.0)
-      activesupport
-      docopt
+    vop (0.3.4)
       net-scp
       net-ssh
-      pry
+      redis
+      sidekiq
       terminal-table
+      xml-simple
 
 GEM
-  remote: http://rubygems.org/
+  remote: https://rubygems.org/
   specs:
-    activesupport (5.0.0.1)
-      concurrent-ruby (~> 1.0, >= 1.0.2)
-      i18n (~> 0.7)
-      minitest (~> 5.1)
-      tzinfo (~> 1.1)
-    coderay (1.1.1)
-    concurrent-ruby (1.0.2)
+    concurrent-ruby (1.0.5)
+    connection_pool (2.2.1)
     diff-lcs (1.2.5)
-    docopt (0.5.0)
-    i18n (0.7.0)
-    method_source (0.8.2)
-    minitest (5.10.1)
+    docile (1.1.5)
+    json (1.8.6)
     net-scp (1.2.1)
       net-ssh (>= 2.6.5)
-    net-ssh (3.2.0)
-    pry (0.10.3)
-      coderay (~> 1.1.0)
-      method_source (~> 0.8.1)
-      slop (~> 3.4)
-    rake (11.3.0)
+    net-ssh (4.2.0)
+    rack (2.0.3)
+    rack-protection (2.0.0)
+      rack
+    rake (10.5.0)
+    redis (4.0.1)
     rspec (3.4.0)
       rspec-core (~> 3.4.0)
       rspec-expectations (~> 3.4.0)
@@ -45,22 +38,30 @@ GEM
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.4.0)
     rspec-support (3.4.1)
-    slop (3.6.0)
-    terminal-table (1.7.3)
-      unicode-display_width (~> 1.1.1)
-    thread_safe (0.3.5)
-    tzinfo (1.2.2)
-      thread_safe (~> 0.1)
-    unicode-display_width (1.1.1)
+    sidekiq (5.0.5)
+      concurrent-ruby (~> 1.0)
+      connection_pool (~> 2.2, >= 2.2.0)
+      rack-protection (>= 1.5.0)
+      redis (>= 3.3.4, < 5)
+    simplecov (0.11.2)
+      docile (~> 1.1.0)
+      json (~> 1.8)
+      simplecov-html (~> 0.10.0)
+    simplecov-html (0.10.0)
+    terminal-table (1.8.0)
+      unicode-display_width (~> 1.1, >= 1.1.1)
+    unicode-display_width (1.3.0)
+    xml-simple (1.1.5)
 
 PLATFORMS
   ruby
 
 DEPENDENCIES
-  bundler (~> 1.10)
-  rake
-  rspec
+  bundler (~> 1.15)
+  rake (~> 10.0)
+  rspec (~> 3.0)
+  simplecov
   vop!
 
 BUNDLED WITH
-   1.10.6
+   1.16.0

data/README.md

@@ -1,2 +1,184 @@
-# vop
-new and improved, now with twice the vitamins!
+
+
+The vop is a systems automation scripting framework.
+
+It organizes (ruby) scripts into commands living in plugins, defines services that can be installed and managed, and comes with a shell and a web interface.
+
+# Status: WIP
+
+This is work in progress. Do not assume everything you read to be totally accurate and/or stable.
+
+# Installation
+
+## as a gem:
+
+    $ gem install vop
+
+# Usage
+
+Call `vop` to start the shell.
+
+Use the tab key for completion, type "help" for more info, `list_plugins` and `list_commands` for an overview.
+
+# Syntax
+
+## Plugins
+
+Minimally, a plugin is a folder containing a file called "<name>.plugin". An empty file will do.
+
+optional:
+```
+description "really nice plugin, this one"
+
+auto_load false    # default is true
+```
+
+dependencies:
+```
+depends_on :other_plugin
+depends_on [ :many, :other, :plugins ]
+```
+
+hooks:
+```
+hook :before_execute do |payload|
+  request = payload[:request]
+end
+
+hook :after_execute do |payload|
+  request = payload[:request]
+  response = payload[:response]
+end
+```
+
+## Commands
+
+...are loaded from the `commands` folder in a plugin.
+
+minimal:
+```
+run { 42 }
+```
+
+optional:
+```
+description "Roses are red."
+
+read_only   # => cacheable
+```
+
+defining param(eter)s:
+```
+param "snafoo"    # optional
+param! "snafoo"   # mandatory
+param! :snafoo    # entity
+```
+param syntax:
+```
+param[!] ( "name" | :entity ) [, {option: value}]
+```
+full example:
+```
+param! "foo",
+  description: "it's the foo.",
+  multi: true,
+  default: [ "snafoo", "footoristic", "foofoo", "foolosophy" ],
+  default_param: true,
+  lookup: lambda { |params| %w|string array| }  
+```
+
+to access param values, use the `params` hash or named block variables:
+```
+param "foo"
+param "not_really"
+
+run do |params, not_really|
+  puts params["foo"] unless not_really
+end
+```
+
+entity parameters are resolved from the specified key to the entity:
+```
+param :machine
+
+run do |machine|
+  puts machine.name
+end
+```
+
+contribute:
+```
+contribute to: "other_command" do |params|
+end
+```
+
+
+show (display options for shell output):
+```
+show columns: ["name", "number"]
+show sort: false
+show display_type: :raw
+```
+`display_type` can be any of:
+```
+:data
+:raw
+:table
+:list
+:hash
+```
+(see [`shell/formatter.rb`](https://github.com/virtualop/vop/blob/master/lib/vop/parts/shell_formatter.rb))
+
+## Services
+
+Plugin folder: `services`
+
+install one or multiple OS packages:
+```
+deploy package: <foo>
+deploy package: [ <foo>, <bar>, <baz> ]
+```
+
+install (configuration for) a package repository:
+```
+deploy repository: {
+  alias: "funny-name",
+  url: "https://package.repository.somewhere.xxx",
+  dist: "stable/",
+  key: "https://download.somewhere.xxx/the.gpg.key"
+}
+```
+
+run arbitrary vop or ruby code during deployment of a service:
+```
+deploy do |machine|
+
+
+end
+```
+
+# Development
+
+## required dependencies
+
+* (to checkout sources: git)
+* ruby + headers (e.g. packages `ruby` and `ruby-dev` on Ubuntu)
+* bundler (`gem install bundler`)
+
+## setup
+After checking out the repo, run `bin/setup` to install (gem) dependencies.
+Then, run `bundle exec rake spec` to run the tests.
+You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To start the vop from the checked out sources, set the environment variable `VOP_DEV_MODE` to any value, e.g. like this:
+```
+export VOP_DEV_MODE=1
+```
+and then run `exe/vop`.
+
+### packaging
+To install the vop from source as a gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+# Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/virtualop/vop.

data/bin/console

@@ -0,0 +1,17 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "vop"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+@op = Vop.boot
+puts "A Vop instance is available in @op."
+
+require "irb"
+IRB.start(__FILE__)

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/bin/vop.sh

@@ -1,4 +1,8 @@
 #!/bin/bash
 
-DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
-ruby $DIR/vop.rb $*
+VOP_DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )/.." && pwd )
+
+cd $VOP_DIR
+bundle exec exe/vop "$*"
+
+cd - >/dev/null

data/exe/vop

@@ -1,28 +1,6 @@
 #!/usr/bin/env ruby
 
-require 'pathname'
+require "vop"
 
-$: << File.join(Pathname.new(File.join(File.dirname(__FILE__), '..')).realpath, 'lib')
-
-require 'vop/shell'
-require 'docopt'
-require 'pp'
-
-begin
-  shell = Vop::Shell::setup()
-  if shell.options["--execute"]
-    last_response = shell.execute(shell.options["--execute"])
-  else
-    last_response = shell.run_cli
-  end
-
-  # TODO exit_status = last_response && last_response.status == Vop.Status::OK ? 0 : 1
-  exit 0
-rescue => detail
-  if detail.is_a? Docopt::Exit
-    puts detail.message
-  else
-    $stderr.puts "error : #{detail.message}\n#{detail.backtrace[0..9].join("\n")}"
-    exit 42
-  end
-end
+@op = Vop.boot()
+Vop::Shell.run(@op)

data/lib/boot.rb

@@ -0,0 +1,3 @@
+require "vop"
+
+@op = Vop.boot()

data/lib/core/meta/commands/new_command.rb

@@ -0,0 +1 @@
+run { 42 }

data/lib/core/meta/commands/new_plugin.rb

@@ -0,0 +1,47 @@
+param! "name"
+param "path"
+param "content"
+
+require "fileutils"
+
+run do |params|
+  unless params["path"]
+    # try to find a writable plugin directory
+    first_non_usr = @op.plugin_locations.select { |location| not location.start_with? "/usr" }.first
+    if first_non_usr.nil?
+      raise "no writable plugin location found"
+    else
+      $logger.info "new_plugin defaulting to first writable plugin location #{first_non_usr}"
+      params["path"] = first_non_usr
+    end
+  end
+
+  raise "no such path: #{params["path"]}" unless File.exists? params["path"]
+
+  pp params
+
+  # a plugin is a directory
+  plugin_path = File.join(params["path"], params["name"])
+  puts "plugin path : #{plugin_path}"
+  Dir.mkdir(plugin_path)
+
+  # with subfolders for commands and helpers
+  %w|commands helpers|.each do |thing|
+    Dir.mkdir(File.join(plugin_path, thing))
+  end
+
+  # and a metadata file called "<name>.plugin"
+  plugin_file = params["name"] + ".plugin"
+  full_name = File.join(plugin_path, plugin_file)
+  FileUtils.touch full_name
+
+  # TODO content is [] - should probably be nil, though
+  #puts "content: >>#{params["content"].pretty_inspect}<<"
+  unless params["content"].nil?
+    IO.write(full_name, params["content"])
+  end
+
+  $logger.info "created new plugin file #{plugin_file}"
+
+  @op.reset
+end

data/lib/core/meta/commands/search_gems_for_plugins.rb

@@ -0,0 +1,38 @@
+require "rubygems"
+require "find"
+require "digest"
+
+contribute to: "search_path" do |params|
+  gem_list = `gem list`.split("\n")
+  candidates = gem_list.select { |x| x =~ /vop-\w+/ }
+
+  # TODO cache which gems have already been inspected
+  # checksum = Digest::MD5.new
+  # candidates.map do |candidate|
+  #   checksum << [ candidate.name, candidate.version ].join(":")
+  # end
+  # puts "gemspec md5 : #{checksum.hexdigest}"
+
+  result = []
+
+  candidates.each do |candidate|
+    $logger.debug "inspecting #{candidate}"
+
+    matched = /(.+?)\s+\([\d\.]+\)/.match(candidate)
+    unless matched
+      $logger.warn("unexpected line format : #{candidate}")
+      next
+    end
+
+    name = matched.captures.first
+    version = matched.captures.last
+
+    contents = `gem contents #{name}`.split("\n")
+
+    plugin_files = contents.grep /\.plugin$/
+    result += plugin_files.map do |file_name|
+      file_name.split("/")[0..-2].join("/")
+    end
+  end
+  result
+end

data/lib/core/meta/commands/search_path.rb

@@ -0,0 +1,6 @@
+run do |params|
+  @op.collect_contributions(
+    "command_name" => "search_path",
+    "raw_params" => params
+  )
+end

data/lib/core/meta/commands/set.rb

@@ -0,0 +1,12 @@
+param! "plugin"
+param! "key"
+param! "value"
+
+run do |key, value, params|
+  plugin = @op.plugin(params["plugin"])
+
+  plugin.config[key] = value
+  plugin.write_config
+
+  plugin.config
+end

data/lib/core/meta/commands/show.rb

@@ -0,0 +1,6 @@
+param! "plugin_name"
+
+run do |params|
+  plugin = @op.plugin(params["plugin_name"])
+  plugin.config
+end

data/lib/core/meta/commands/show_config.rb

@@ -0,0 +1,9 @@
+show display_type: :raw
+
+run do
+  @op.plugins.map do |plugin|
+    { "name" => plugin.name,
+      "config" => @op.show("plugin_name" => plugin.name)
+    }
+  end
+end

data/lib/core/meta/commands/who_provides.rb

@@ -0,0 +1,6 @@
+# TODO would be nice if we had a :command entity
+param! 'command_name', :lookup => lambda { |params| @op.commands.keys }
+
+run do |command_name|
+  @op.commands[command_name].plugin.name
+end

data/lib/core/meta/meta.plugin

@@ -0,0 +1 @@
+depends_on :structure

data/lib/core/shell/commands/change_loglevel.rb

@@ -0,0 +1,6 @@
+param! "level"
+
+run do |level|
+  new_level = Logger.const_get(level.upcase)
+  $logger.level = new_level
+end

data/lib/core/shell/commands/edit.rb

@@ -0,0 +1,12 @@
+param! "name", lookup: lambda { @op.commands.keys }
+
+run do |params, name|
+  has_vim = `which vim`
+  editor = ENV["EDITOR"] || "vim" if has_vim
+  raise "please set the EDITOR environment variable" unless editor
+
+  command_file = @op.commands[name].source[:file_name]  
+  system("#{editor} #{command_file}")
+
+  @op.reset
+end

data/lib/core/shell/commands/help.rb

@@ -0,0 +1,49 @@
+param! "name",
+  description: "a command for which help should be displayed",
+  lookup: lambda { |params| @op.list_commands.split }
+
+run do |params, name|
+  command = @op.commands[params["name"]]
+  if command.nil?
+    raise "no such command : #{name}"
+  end
+
+  puts
+  puts command.name
+
+  if command.description
+    puts "  #{command.description}"
+  end
+
+  puts
+  puts "syntax:"
+
+  minimal_syntax = command.short_name
+  maximal_syntax = command.short_name
+  command.params.each do |p|
+    example = if p == command.default_param
+      "<#{p.name}>"
+    else
+      "#{p.name}=<#{p.name}>"
+    end
+    minimal_syntax << " #{example}" if p.options[:mandatory]
+    maximal_syntax << " #{example}"
+  end
+  puts "  #{minimal_syntax}"
+  puts "  #{maximal_syntax}" unless minimal_syntax == maximal_syntax
+
+  if command.params.size > 0
+    puts
+    puts "parameters:"
+    max_param_length = command.params.map { |p| p.name.length }.max
+    command.params.each do |p|
+      default_text = ""
+      if p.options.key? :default
+        default_text += " (default: #{p.options[:default]})"
+      end
+      puts "  %-#{max_param_length}s\t%s\t%s" % [p.name, p.options[:description], default_text]
+    end
+  end
+
+  nil
+end

data/lib/core/shell/commands/reset.rb

@@ -0,0 +1,4 @@
+run do
+  @op._reset
+  [ @op.plugins.size, @op.commands.size ]
+end

data/lib/core/shell/commands/source.rb

@@ -0,0 +1,21 @@
+param! "name", lookup: lambda { @op.commands.keys + @op.entities.keys + @op.filters.keys }
+
+run do |name|
+  (source, thing) = if @op.commands.keys.include? name
+    [ :commands, @op.commands[name] ]
+  elsif @op.entities.keys.include? name
+    [ :entities, @op.entities[name] ]
+  elsif @op.filters.keys.include? name
+    [ :filters, @op.filters[name] ]
+  end
+
+  code = thing.source[:code]
+
+  result = []
+  result << " "
+  code.lines.each_with_index { |line, idx|
+    result << "%02d %s" % [idx+1, line.chomp]
+  }
+  result << " "
+  result.join("\n")
+end

data/lib/core/structure/commands/collect_contributions.rb

@@ -0,0 +1,46 @@
+description "This is called by a command accepting contributions - it calls all contributors and merges the result"
+
+param! "command_name", default_param: true
+param "raw_params", default: {}
+
+run do |command_name, raw_params|
+  contributors = @op.list_contributors(command_name: command_name)
+
+  target_name = command_name.split(".").last
+  target = @op.commands[target_name]
+
+  if target.nil?
+    raise "contribution target #{target_name} not found"
+  end
+
+  display_type = target.show_options[:display_type] || :table
+
+  result = case display_type
+  when :table
+    []
+  when :hash
+    {}
+  else
+    raise "contributions not implemented for display_type #{display_type}"
+  end
+
+  contributors.each do |contributor_name|
+    short_name = contributor_name.split(".").last
+    contributor = @op.commands[short_name]
+
+    contribution = @op.execute(short_name, raw_params)
+
+    if contribution.nil?
+      $logger.warn "command #{short_name} contributes a nil value"
+    else
+      case display_type
+      when :table
+        result += contribution
+      when :hash
+        result.merge! contribution
+      end
+    end
+  end
+
+  result
+end