loom-core 0.0.4 → 0.0.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6d528b08eb52c1ed7fca596af8d47b5e1869c5bd98d1bbeba9cb66c27dad3043
-  data.tar.gz: 1445b6f3d508a9be19d32f2aa9253e3e6f7b7eb9b771b73ea482b9b3ca07a846
+  metadata.gz: 4dd2b1e4959e4556b67f1369c3e8de87bdca21c0082ec446d629319cc58ee748
+  data.tar.gz: edb2d6b2f14a01549b6b75820cf3d456d401fb396f7c13469b812922ae700e2b
 SHA512:
-  metadata.gz: 93499a20ce6ba74bcf96dc84d6fbf2c1b1c3fd38901dd9f4c06e8f3ce04d6d44dde3176f1aa88b8becbda43b72b5e93767d7e088d8d4fb1c52e40e3b07212f9e
-  data.tar.gz: f3e2c5a3699f76c3e6d83d41286b421158c9ed2380714dbe2e54658c51c3a466832967f29d1e12601c63beaf2f59d8e83b0b03938cff43459d906e59018d4eec
+  metadata.gz: 8459bc0b442c21f59217a8d4646418c17a82a77fa0321f56f1877e8892bc84be56ee16043f9fc362acd437b713296c66f3a8542df0fe9be64c9a08070453bf55
+  data.tar.gz: 3fb25060598c49d64efcc60e23a44e1ca927304ae861ddf45baa3ded0481da9da8d0f40f358b7c2e8df4c6ec2a1bf9cb2f017285cb0cf63a8ade311d7b149282

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    loom-core (0.0.3)
+    loom-core (0.0.5)
       bcrypt_pbkdf (>= 1.0, < 2.0)
       commander (~> 4.4)
       ed25519 (>= 1.0, < 2.0)

data/bin/loom

@@ -19,10 +19,13 @@ module Loom
       program :name, "Loom - Weaving through infrastructure"
       program :version , Loom::VERSION
       program :description, <<EOS
-A lightweight infrastructure managment tool designed to manage hosts
-through SSH, loosely inspired by Python Fabric - http://www.fabfile.org/.
+A lightweight infrastructure managment tool for managing hosts through SSH,
+inspired by [Python Fabric](http://www.fabfile.org/) with thanks and nods
+[Capistrano](https://capistranorb.com/).
 
 Try `loom weave uptime -H localhost` to see an example.
+
+Want to know how loom works? @see lib/loom/pattern/dsl.rb
 EOS
 
       global_option "-V", "--verbose", "Report verbose results" do |v|
@@ -38,8 +41,12 @@ EOS
         end
       end
 
-      global_option "--dbg [N]", Integer,
-                    "Enable deep debug logging, where N is 0-6, implies --verbose" do |n|
+      global_option("--dbg [N]", Integer, <<EOS
+Enables deep debug logging, where N is 1-6, implies --verbose. The difference
+between -d and --dbg is the former is intended for site.loom authors and the
+latter for authors looking to inspect loom internals.
+EOS
+        ) do |n|
         raise "N must be greater than 0" if n < 0
         Loom.configure do |c|
           c.log_level = n * -1
@@ -116,12 +123,14 @@ EOS
           puts "Loom mods are:"
           puts ""
 
+          # TODO: I think this is broken... fix this.
           Loom::Mods::ModLoader.registered_mods.each do |name, aliases|
             puts aliases.join(", ")
             puts "\t#{name}"
           end
         end
       end
+      alias_command :"m", "mods"
 
       command :"patterns" do |c|
         c.syntax = "loom patterns [pattern]"

data/lib/loom/config.rb

@@ -7,6 +7,8 @@ module Loom
 
   class Config
 
+    # TODO: Add a more module config_var registry mechanism for Modules and
+    # FactProviders to register their own values & defaults.
     CONFIG_VARS = {
       :loom_search_paths => ['/etc/loom', File.join(ENV['HOME'], '.loom'), './.loom'],
       :loom_files => ['site.loom'],

data/lib/loom/dsl.rb

@@ -1,5 +1,10 @@
 require 'socket'
 
+##
+# Links to relevant SSHKit code:
+# https://github.com/capistrano/sshkit/blob/master/lib/sshkit/backends/abstract.rb
+# https://github.com/capistrano/sshkit/blob/master/lib/sshkit/backends/netssh.rb
+# https://github.com/capistrano/sshkit/blob/master/lib/sshkit/backends/local.rb
 module Loom
 
   # TODO: Rename this to something like SSHKitWrapper, DSL is a

data/lib/loom/facts/fact_set.rb

@@ -1,5 +1,12 @@
 module Loom::Facts
 
+  EMPTY = {}
+  class << EMPTY
+    def [](*args)
+      self
+    end
+  end
+
   ##
   # A factset is created by running each registered Loom::Facts::Provider and
   # calling Provider+collect_facts+. See ./fact_file_provider.rb and
@@ -79,14 +86,28 @@ module Loom::Facts
     end
 
     def hostname
+      Loom.log.debug(<<NB
+`facts.hostname ` is actually the name by which SSH knows the host, not
+necessarily the actual host name. e.g. it may be an ip address, /etc/hosts
+alias, or ssh config alias. Use `facts.sshname` for the same value without this
+warning.
+NB
+        )
+      sshname
+    end
+
+    def sshname
       host_spec.hostname
     end
 
     def get(fact_name)
       v = @fact_map[fact_name.to_sym]
       dup = v.dup rescue v
-      dup = FactSet.new(@host_spec, {}) if dup.nil?
-      dup.is_a?(Hash) ? FactSet.new(@host_spec, dup) : dup
+      if dup.nil?
+        EMPTY
+      else
+        dup
+      end
     end
     alias_method :[], :get
 

data/lib/loom/pattern/definition_context.rb

@@ -6,6 +6,8 @@ module Loom::Pattern
   # includes all contexts of parent modules.
   class DefinitionContext
 
+    NilLetValueError = Class.new Loom::LoomError
+
     def initialize(pattern_module, parent_context=nil)
       @fact_map = pattern_module.facts.dup
       @let_map = pattern_module.let_map.dup
@@ -26,7 +28,9 @@ module Loom::Pattern
       host_fact_set.merge merged_fact_map
     end
 
-    # TODO: #define_let_readers is a TERRIBLE name. Rename this method.
+    # TODO: #define_let_readers is a TERRIBLE name. Rename this method. Also
+    # consider moving the instance_exec call to inside RunContext, it's a bit
+    # misplaced here.
     #
     # The method is called by Reference#call with the Reference::RunContext
     # instance. The "let" defined local declarations are added as method
@@ -34,9 +38,16 @@ module Loom::Pattern
     # let definitions, merged from the associated module, up the namespace
     # graph, allowing for inheriting and overriding +let+ declarations.
     def define_let_readers(scope_object, fact_set)
-      @merged_let_map.each do |let_key, block|
-        raise "no let block" unless block
-        value = scope_object.instance_exec fact_set, &block
+      @merged_let_map.each do |let_key, let_map_entry|
+        Loom.log.debug { "evaluating let expression[:#{let_key}]" }
+        value = scope_object.instance_exec fact_set, &let_map_entry.block
+        value = value.nil? ? let_map_entry.default : value
+        Loom.log.debug1(self) { "let[:#{let_key}] => #{value}" }
+
+        if value.nil? || value.equal?(Loom::Facts::EMPTY)
+          Loom.log.e "value of let expression[:#{let_key}] is nil"
+          raise NilLetValueError, let_key
+        end
         scope_object.define_singleton_method(let_key) { value }
       end
     end

data/lib/loom/pattern/dsl.rb

@@ -1,23 +1,43 @@
 =begin
 
-# .loom File DSL
+# TODO: DSL extensions:
+- Pattern+non_idempotent+ marks a pattern as explicitly not idempotent, this
+  let's additional warnings and checks to be added
+- A history module, store a log of each executed command, a hash of the .loom
+  file, and the requisite facts (the let declarations) for each executed pattern
+  on the host it executes. /var/log/loom/history? Create this log on startup.
+  -- add a new set of history commands through the CLI and a history
+     FactProvider exposing host/loom/pattern_slug execution stats.
+- Provide automatic command reversion support with a =Module= DSL that ties in
+  with local revision history.
+  -- allow Module actions/mods to define an "undo" command of itself given the
+     original inputs to the action
+  -- using the history to pull previous params (let defns) into a revert command.
+  -- usages of the raw shell, such as `loom.x` and `loom.capture` would be
+     unsupported. so would accesses to the fact_set in any before/after/pattern
+     blocks.
+  -- however patterns that only used let blocks, and used all "revertable"
+     module methods, could have automatic state reversion and integrity checking
+     managed.
+  -- best practices (encouraged through warnings) to be to heavily discourage
+     uses of loom.execute and loom.capture in .loom files and encourage all
+     accesses to fact_set be done in let expressions (enforce this maybe?)
+  -- Later... before/after hooks can ensure the entire loom execution sequence
+     was "revertable"
+
+## .loom File DSL
+
+See specs/test.loom for a valid .loom file.
+
+I've tried to take inspriation from the RSpec DSL for .loom, so hopefully it
+feels comfortable.
 
 Loom::Pattern::DSL is the mixin that defines the declarative API for all .loom
 file defined modules. It is included into Loom::Pattern by default. The outer
 most module that a .loom file declares has Loom::Pattern mixed in by
 default. Submodules must explicitly include Loom::Pattern, and will receive DSL.
 
-To follow the code path for .loom file loading see:
-
-    Loom::Runner#load
-      -> Loom::Pattern::Loader.load
-         -> Loom::Pattern::ReferenceSet.load_from_file
-            -> Loom::Pattern::ReferenceSet::Builder.create
-
-The Loom::Pattern::ReferenceSet::Builder creates a ReferenceSet from a .loom
-file. A ReferenseSet being a collection of references with uniquely named
-slugs. The slug of a reference is computed from the module namespace and
-instance method name. For example, given the following .loom file:
+For example, given the following .loom file:
 
 ``` ~ruby
 def top_level; end
@@ -42,13 +62,31 @@ It declares a reference set with slugs:
 
 Defining the same pattern slug twice raises a DuplicatPatternRef error.
 
-Module Inner inherits all +let+ declarations from its outer contexts, both ::
-(root) and ::Outer. +before+ hooks are run from a top-down module ordering,
-+after+ hooks are run bottom-up. For example, given the following .loom file:
+#### Code Details
+
+To follow the code path for .loom file loading see:
+
+    Loom::Runner#load
+      -> Loom::Pattern::Loader.load
+         -> Loom::Pattern::ReferenceSet.load_from_file
+            -> Loom::Pattern::ReferenceSet::Builder.create
+
+The Loom::Pattern::ReferenceSet::Builder creates a ReferenceSet from a .loom
+file. A ReferenseSet being a collection of references with uniquely named
+slugs. The slug of a reference is computed from the module namespace and
+instance method name.
+
+### `let`, `before`, and `after`
+
+Module::Inner, from above, inherits all +let+ declarations from its outer
+contexts, both :: (root) and ::Outer. +before+ hooks are run from a top-down
+module ordering, +after+ hooks are run bottom-up. For example, given the
+following .loom file:
 
 ``` ~ruby
 let(:var_1) { "v1 value" }
 let(:var_2) { "v2 value" }
+let(:var_3, "otherwise") { |facts| facts[:a] || facts[:b] }
 
 before { puts "runs first +before+" }
 after { puts "runs last +after+" }
@@ -65,13 +103,19 @@ module Submod
 end
 ```
 
-
 If running `loom submod:a_pattern`, then let declarations would declare values:
 
     { :var_1 => "submod value", :var_2 => "v2 value" }
 
+:var_3 would be set to either fact :a or fact :b. If the let expression
+evaluates to nil?, then "otherwise" will be used as a default.
+
 Each let value is effectively available as an `attr_reader` declaration from
-::Submod#a_pattern. Before and After hook ordering with pattern execution would
+::Submod#a_pattern. Because the attr_reader is defined on the RunContext scope
+object, let expressions are available all before/after hooks and pattern
+expresions.
+
+Before and After hook ordering with pattern execution would
 look like:
 
     => runs first +before+
@@ -83,12 +127,16 @@ look like:
 For the code that executes before hooks, pattern, after hooks see
 Loom::Pattern::Reference::RunContext#run.
 
+#### Code Details
+
 The Loom::Pattern::Reference::RunContext acts as the the binding object for each
-pattern slug. i.e. When running a pattern slug, the RunContext is the self
-object. Let definitions, before and after hooks, and fact maps are unique to
-each RunContext, for each RunContext they are defined in the
+pattern. i.e. RunContext is the self object for all the blocks defined in a loom
+file. Let definitions, before and after hooks, and fact maps are unique to each
+RunContext, for each RunContext they are defined in the
 Loom::Pattern::DefinitionContext. Each DefinitionContext is merged from it's
-parent module, see Loom::Pattern::DefinitionContext#merge_contexts for info.
+parent module, see Loom::Pattern::DefinitionContext#merge_contexts for
+info. Each pattern/host combo gets a unique RunContext instance via
+Loom::Runner+execute_pattern+ -> Loom::Pattern::Reference+call+.
 
 The RunContext#run method is the actual execution of the pattern. A pattern,
 before association to a RunContext instance is an unbound method. During
@@ -96,6 +144,71 @@ RunContext#initialize the pattern is bound to the RunContext instance and
 executed during RunContext#run with the associated Loom::Shell::Api and
 Loom::Facts::FactSet as parameters.
 
+See Loom::Pattern::DefinitionContext for evaluation of `let` blocks and
+before/after context ordering.
+
+### `weave`
+
+The `weave` keyword allows aliasing a sequence of patterns a single
+name. Pattern execution will be flattened and run sequentially before or after
+any other patterns in the `$ loom` invocation.
+
+``` ~ruby
+pattern :step_1 { ... }
+pattern :step_2 { ... }
+
+weave :do_it, [ :step_1, :step_2 ]
+```
+
+This creates pattern :do_it, which when run `$ loom do_it` will run :step_1,
+:step_2. Recursive expansion is explicitly disallowed, only pattern names (not
+weaves), are allowed in the 2nd param of `weave`.
+
+#### Code Details
+
+Weave expansion to pattern slugs is accomplished by creating a
+Loom::Pattern::ExpandingReference via the Loom::Pattern::Loader+load+ path
+invoked via Loom::Runner+load+. Expansion happens on read via
+Loom::Pattern::Loader+patterns+, thus the list of patterns is constant
+throughout all phases of pattern execution.
+
+#### Pattern Execution Phases
+
+Once hosts and patterns are identified in earlier Loom::Runner phases,
+Loom::Runner+run_internal+, per host, initiates an SSH session and command
+execution phases of processing the .loom file. First phase is fact collection
+via +Loom::Facts.fact_set, see Loom::Facts::FactSet for createing, registering,
+and executing Loom::Facts::FactProviders.
+
+The inputs to fact collection are a Loom::Shell::Core, Loom::HostSpec, and
+Loom::Config. Fact collection must be FAST and idempotent (or as reasonably
+possible). No network requests should be made during fact colleciton. Fact
+collection is done prior to EACH pattern/host combination in order to ensure
+having the newest facts from prevoius pattern executions.
+
+After fact collection is pattern block execution, including before and after
+block execution. See comments above for pattern code pointers and other details.
+
+## Decorating the Loom Object with Custom Modules and FactProviders
+
+TODO
+
+See lib/loomext/coremods & lib/loomext/corefacts for examples.
+
+## Facts and Inventory
+
+TODO
+
+## Config
+
+TODO
+
+## Logger
+
+TODO
+
+##
+
 =end
 module Loom::Pattern
 
@@ -121,9 +234,11 @@ module Loom::Pattern
       @facts = yield_result if yield_result.is_a? Hash
     end
 
-    def let(name, &block)
+    def let(name, default: nil, &block)
+      raise "malformed let expression: missing block" unless block_given?
+
       @let_map ||= {}
-      @let_map[name.to_sym] = block
+      @let_map[name.to_sym] = LetMapEntry.new default, &block
     end
 
     def pattern(name, &block)
@@ -143,26 +258,6 @@ module Loom::Pattern
       define_pattern_internal(name) { |_, _| true }
     end
 
-    def define_pattern_internal(name, &block)
-      @pattern_methods ||= []
-      @pattern_method_map ||= {}
-      @pattern_descriptions ||= {}
-
-      method_name = name.to_sym
-
-      @pattern_methods << method_name
-      @pattern_method_map[method_name] = true
-      @pattern_descriptions[method_name] = @next_description
-      @next_description = nil
-
-      define_method method_name, &block
-    end
-
-    def hook(scope, &block)
-      @hooks ||= []
-      @hooks << Hook.new(scope, &block)
-    end
-
     def before(&block)
       hook :before, &block
     end
@@ -203,5 +298,34 @@ module Loom::Pattern
     def let_map
       @let_map || {}
     end
+
+    private
+    def define_pattern_internal(name, &block)
+      @pattern_methods ||= []
+      @pattern_method_map ||= {}
+      @pattern_descriptions ||= {}
+
+      method_name = name.to_sym
+
+      @pattern_methods << method_name
+      @pattern_method_map[method_name] = true
+      @pattern_descriptions[method_name] = @next_description
+      @next_description = nil
+
+      define_method method_name, &block
+    end
+
+    def hook(scope, &block)
+      @hooks ||= []
+      @hooks << Hook.new(scope, &block)
+    end
+  end # DSL
+
+  class LetMapEntry
+    attr_reader :default, :block
+    def initialize(default, &block)
+      @default = default
+      @block = block
+    end
   end
 end

data/lib/loom/pattern/reference.rb

@@ -19,8 +19,12 @@ module Loom::Pattern
       run_context = RunContext.new @unbound_method, @definition_ctx
 
       fact_set = @definition_ctx.fact_set host_fact_set
-      Loom.log.debug5(self) { "fact set for pattern execution => #{fact_set.facts}" }
+      Loom.log.debug5(self) {
+        "fact set for pattern execution => #{fact_set.facts}" }
 
+      # TODO: wrap up this fact_set in a delegator/facade/proxy to eliminate the
+      # .loom file from directly accessing it. Add logging and deprecation
+      # warnings there.... like FactSet+hostname+ currently.
       @definition_ctx.define_let_readers run_context, fact_set
 
       begin

data/lib/loom/runner.rb

@@ -59,11 +59,21 @@ module Loom
         exit code
       rescue PatternExecutionError => e
         num_patterns_failed = @run_failures.size
-        Loom.log.error "error executing #{num_patterns_failed} patterns => #{e}"
-        Loom.log.debug e.backtrace.join "\n"
+        Loom.log.error "error executing #{num_patterns_failed} patterns => #{e}, run with -d for more"
+        Loom.log.debug e.backtrace.join "\n\t"
+        # TODO: I think the max return code is 255. Cap this if so.
         exit 100 + num_patterns_failed
+      rescue SSHKit::Runner::ExecuteError => e
+        Loom.log.error "wrapped SSHKit::Runner::ExecuteError, run with -d for more"
+        Loom.log.error e.cause
+        Loom.log.debug e.cause.backtrace.join "\n\t"
+        Loom.log.debug1(self) { "Original error:" }
+        Loom.log.debug1(self) { e.inspect }
+        Loom.log.debug1(self) { e.backtrace.join "\n\t" }
+        exit 97
       rescue Loom::LoomError => e
-        Loom.log.error "loom error => #{e.inspect}"
+        Loom.log.error "Loom::LoomError => #{e.inspect}, run with -d for more"
+        Loom.log.debug e.cause.backtrace.join "\n\t"
         exit 98
       rescue => e
         Loom.log.fatal "fatal error => #{e.inspect}"
@@ -99,6 +109,8 @@ module Loom
         Loom::Inventory::InventoryList.active_inventory @loom_config
       @active_hosts = @inventory_list.hosts
 
+      # TODO: the naming inconsistency between Pattern::Loader and
+      # Mods::ModLoader bothers me... :(
       pattern_loader = Loom::Pattern::Loader.load @loom_config
       @pattern_refs = pattern_loader.patterns @pattern_slugs
 
@@ -141,11 +153,18 @@ module Loom
             pattern_shell = Loom::Shell.create @mod_loader, sshkit_backend, dry_run
 
             Loom.log.warn "dry run only => #{pattern_description}" if dry_run
-            execute_pattern pattern_ref, pattern_shell, fact_set
+            failures = execute_pattern pattern_ref, pattern_shell, fact_set
+
+            if failures.empty?
+              Loom.log.debug "success on => #{pattern_description}"
+            else
+              Loom.log.error "failures on => #{pattern_description}:\n\t%s" % (
+                failures.join("\n\t"))
+            end
           end
         rescue IOError => e
-          # TODO: Try to patch SSHKit for a more specific error for unexpected SSH
-          # disconnections
+          # TODO: Try to patch SSHKit for a more specific error for unexpected
+          # SSH disconnections
           Loom.log.error "unexpected SSH disconnect => #{hostname}"
           Loom.log.debug e
           handle_host_failure_strategy hostname, e.message
@@ -168,6 +187,7 @@ module Loom
       # when a command fails and pattern execution should stop. All errors
       # should come from exceptions. This needs to be thought about wrt to the
       # defined `failure_strategy`
+      # @see the TODO at Loom::Shell::Core+test+
       run_failure = []
       begin
         pattern_ref.call(shell.shell_api, fact_set)
@@ -187,6 +207,7 @@ module Loom
         @result_reports << result_reporter
         @run_failures << run_failure unless run_failure.empty?
       end
+      run_failure
     end
 
     private

data/lib/loom/shell/core.rb

@@ -40,6 +40,7 @@ module Loom::Shell
       # objects and declare style of reporting & error code handling it
       # has. Commands can be defined to ignore errors and just return their
       # results.
+      # @see the TODO at Loom::Runner+execute_pattern+
       execute *cmd, :is_test => true, **cmd_opts
 
       case check
@@ -100,6 +101,8 @@ module Loom::Shell
       end
     end
 
+    # TODO: finish the harness... avoiding dealing w/ shell escaping, like
+    # from sudo, is the goal.
     # def sudo(user=nil, *sudo_cmd, &block)
     #   # I'm trying to work around crappy double escaping issues caused by
     #   # sudo_legacy... but I'm failing

data/lib/loom/version.rb

@@ -1,3 +1,3 @@
 module Loom
-  VERSION = "0.0.4"
+  VERSION = "0.0.5"
 end

data/lib/loomext/coremods/exec.rb

@@ -7,6 +7,8 @@ module LoomExt::CoreMods
   #
   # loom << :echo, "hello there"
   class Exec < Loom::Mods::Module
+    # TODO: add an "example" DSL to Loom::Mods::Module to automate
+    # documentation.
     register_mod :exec, :alias => [:x, :<<] do |*cmd, **opts|
       shell.execute *cmd, **opts
     end

data/lib/loomext/coremods/package/adapter.rb

@@ -89,6 +89,7 @@ module LoomExt::CoreMods
       end
 
       def uninstall(pkg_name)
+        # TODO: fix all loom.execute calls w/ properly atomized params
         loom << "echo dnf uninstall"
       end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: loom-core
 version: !ruby/object:Gem::Version
-  version: 0.0.4
+  version: 0.0.5
 platform: ruby
 authors:
 - Erick Johnson
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-10-21 00:00:00.000000000 Z
+date: 2018-10-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: sshkit