loom-core 0.0.6 → 0.0.9

data/lib/loom/pattern/reference.rb

@@ -1,22 +1,25 @@
 module Loom::Pattern
+  # See Loom::Pattern::Pattern for the difference between refs and patterns.
   class Reference
 
-    attr_reader :slug, :source_file, :desc
+    attr_reader :slug, :source_file, :desc, :pattern
 
-    def initialize(slug, unbound_method, source_file, definition_ctx, description)
+    def initialize(slug, pattern, source_file, definition_ctx)
       @slug = slug
-      @unbound_method = unbound_method
       @source_file = source_file
       @definition_ctx = definition_ctx
-      @desc = description
+      @desc = pattern.description
+      @pattern = pattern
     end
 
-    def is_expanding?
-      false
+    # Used by Loom::Pattern::Loader to expand weaves. A Pattern::Reference it
+    # just expands to itself.
+    def expand_slugs
+      @slug
     end
 
     def call(shell_api, host_fact_set)
-      run_context = RunContext.new @unbound_method, @definition_ctx
+      run_context = RunContext.new @pattern, @definition_ctx
 
       fact_set = @definition_ctx.fact_set host_fact_set
       Loom.log.debug5(self) {
@@ -30,7 +33,7 @@ module Loom::Pattern
       begin
         run_context.run shell_api, fact_set
       rescue => e
-        error_msg = "error executing '#{slug}' in #{source_file} => #{e} \n%s"
+        error_msg = "error executing '#{slug}' in #{source_file} =>\n\t#{e}\n%s"
         Loom.log.error(error_msg % e.backtrace.join("\n\t"))
         raise
       end
@@ -42,8 +45,8 @@ module Loom::Pattern
     # A small class to bind the unbound_method to and provide context
     # in the case of errors.
     class RunContext
-      def initialize(unbound_method, definition_ctx)
-        @bound_method = unbound_method.bind self
+      def initialize(pattern, definition_ctx)
+        @pattern = pattern
         @definition_ctx = definition_ctx
       end
 
@@ -55,7 +58,7 @@ module Loom::Pattern
           Loom.log.debug1(self) { "before hooks => #{before_hooks}"}
           before_hooks.each do |hook|
             Loom.log.debug2(self) { "executing before hook => #{hook}"}
-            self.instance_exec shell_api, fact_set, &hook.block
+            instance_exec shell_api, fact_set, &hook.block
           end
 
           # This is the entry point into calling patterns.
@@ -64,14 +67,15 @@ module Loom::Pattern
           Loom.log.debug1(self) { "after hooks => #{after_hooks}" }
           after_hooks.each do |hook|
             Loom.log.debug2(self) { "executing after hook => #{hook}"}
-            self.instance_exec shell_api, fact_set, &hook.block
+            instance_exec shell_api, fact_set, &hook.block
           end
         end
       end
 
       private
+
       def apply_pattern(*args)
-        @bound_method.call *args
+        instance_exec(*args, &@pattern.pattern_block)
       end
     end
 

data/lib/loom/pattern/reference_set.rb

@@ -3,10 +3,13 @@
 # into a reference set. There's also a lot of dark magic w/ generating
 # stacktraces to get the correct .loom file line (but I forget if that's done
 # here or in pattern/reference.rb maybe?)
+
+# NB: The use of the word "mod" or "module" in this file probably means a
+# ::Module, not a Loom::Mods::Module
 module Loom::Pattern
 
   DuplicatePatternRef = Class.new Loom::LoomError
-  UnknownPatternMethod = Class.new Loom::LoomError
+  NoReferenceForSlug = Class.new Loom::LoomError
   InvalidPatternNamespace = Class.new Loom::LoomError
 
   ##
@@ -16,7 +19,11 @@ module Loom::Pattern
     class << self
       def load_from_file(path)
         Loom.log.debug1(self) { "loading patterns from file => #{path}" }
-        Builder.create File.read(path), path
+        builder(File.read(path), path).build
+      end
+
+      def builder(file_src, file_path)
+        Builder.create(file_src, file_path)
       end
     end
 
@@ -34,21 +41,20 @@ module Loom::Pattern
 
     def get_pattern_ref(slug)
       ref = @slug_to_ref_map[slug]
-      raise UnknownPatternMethod, slug unless ref
+      raise NoReferenceForSlug, slug unless ref
       ref
     end
     alias_method :[], :get_pattern_ref
 
     def merge!(ref_set)
-      self.add_pattern_refs(ref_set.pattern_refs)
+      add_pattern_refs(ref_set.pattern_refs)
     end
 
     def add_pattern_refs(refs)
-      map = @slug_to_ref_map
       refs.each do |ref|
         Loom.log.debug2(self) { "adding ref to set => #{ref.slug}" }
-        raise DuplicatePatternRef, ref.slug if map[ref.slug]
-        map[ref.slug] = ref
+        raise DuplicatePatternRef, ref.slug if @slug_to_ref_map[ref.slug]
+        @slug_to_ref_map[ref.slug] = ref
       end
     end
 
@@ -57,83 +63,95 @@ module Loom::Pattern
 
       class << self
         def create(ruby_code, source_file)
+          # Creates an anonymous parent module in which to evaluate the .loom
+          # file src. This module acts as a global context for the .loom file.
+          # TODO: How should this be hardened?
           shell_module = Module.new
           shell_module.include Loom::Pattern
           # TODO: I think this is my black magic for capturing stacktrace
           # info... I forget the details. Add documentation.
+          # TODO: This is where I would need to hack into to auto-include
+          # Loom::Pattern in .loom file modules
           shell_module.module_eval ruby_code, source_file, 1
           shell_module.namespace ""
 
-          self.new(shell_module, source_file).build
+          self.new shell_module, source_file
         end
       end
 
       def initialize(shell_module, source_file)
         @shell_module = shell_module
-        @pattern_mod_specs = pattern_mod_specs
         @source_file = source_file
       end
 
       def build
         ref_set = ReferenceSet.new
+
+        dsl_specs = create_dsl_specs
+        pattern_refs = create_pattern_refs dsl_specs, ref_set
+
         ref_set.add_pattern_refs pattern_refs
         ref_set
       end
 
       private
-      def pattern_refs
-        @pattern_mod_specs.map { |mod_spec| refs_for_mod_spec mod_spec }.flatten
+      # TODO: I don't like passing in the ref set build target here.... but
+      # ExpandingReference needs it... Can I do this w/o an instance variable
+      # and w/o plumbing the param through methods?
+      def create_pattern_refs(dsl_specs, builder_target_ref_set)
+        dsl_specs.flat_map do |dsl_spec|
+          create_refs_for_dsl_spec dsl_spec, builder_target_ref_set
+        end
       end
 
-      def refs_for_mod_spec(mod_spec)
-        mod = mod_spec[:module]
-        context = context_for_mod_spec mod_spec
+      def create_refs_for_dsl_spec(dsl_spec, builder_target_ref_set)
+        context = create_defn_context_for_dsl_spec dsl_spec
 
-        mod_spec[:pattern_methods].map do |m|
-          slug = compute_slug mod_spec[:namespace_list], m
+        dsl_spec[:dsl_builder].patterns.map do |pattern|
+          slug = compute_slug dsl_spec[:namespace_list], pattern.name
 
-          if mod.is_weave?(m)
+          case pattern.kind
+          when :weave
             Loom.log.debug2(self) { "adding ExpandingReference for weave: #{slug}" }
-            build_expanding_reference(m, slug, mod)
+            create_expanding_reference(pattern, slug, builder_target_ref_set)
           else
             Loom.log.debug2(self) { "adding Reference for pattern: #{slug}" }
-            build_pattern_reference(m, slug, context, mod)
+            create_pattern_reference(pattern, slug, context)
           end
         end
       end
 
-      def build_expanding_reference(weave_name, slug, mod)
-        desc = mod.pattern_description weave_name
+      def create_expanding_reference(pattern, slug, ref_set)
+        desc = pattern.description
         Loom.log.warn "no descripiton for weave => #{slug}" unless desc
 
-        ExpandingReference.new slug, mod.weave_slugs[weave_name], @source_file, desc
+        ExpandingReference.new slug, pattern, ref_set
       end
 
-      def build_pattern_reference(pattern_name, slug, context, mod)
-        method = mod.pattern_method pattern_name
-        desc = mod.pattern_description pattern_name
+      def create_pattern_reference(pattern, slug, context)
+        desc = pattern.description
         Loom.log.warn "no descripiton for pattern => #{slug}" unless desc
 
-        Reference.new slug, method, @source_file, context, desc
+        Reference.new slug, pattern, @source_file, context
       end
 
-      def context_for_mod_spec(mod_spec)
-        parents = mod_spec[:parent_modules].find_all do |mod|
-          is_pattern_module mod
+      # Creates a DefinitionContext for dsl_module by flattening and mapping
+      # parent ::Modules to a contextualized DefinitionContext.
+      def create_defn_context_for_dsl_spec(dsl_spec)
+        parents = dsl_spec[:parent_modules].find_all do |mod|
+          dsl_module? mod
         end
         parent_context = parents.reduce(nil) do |parent_ctx, parent_mod|
-          DefinitionContext.new parent_mod, parent_ctx
+          DefinitionContext.new parent_mod.dsl_builder, parent_ctx
         end
-
-        mod = mod_spec[:module]
-        DefinitionContext.new mod, parent_context
+        DefinitionContext.new dsl_spec[:dsl_builder], parent_context
       end
 
       def compute_slug(namespace_list, pattern_method_name)
         namespace_list.dup.push(pattern_method_name).join ":"
       end
 
-      def mod_namespace_list(pattern, parent_modules)
+      def create_namespace_list(pattern, parent_modules)
         mods = parent_modules.dup << pattern
         mods.reduce([]) do |memo, mod|
           mod_name = if mod.respond_to?(:namespace) && mod.namespace
@@ -149,32 +167,35 @@ module Loom::Pattern
         end
       end
 
-      def pattern_mod_specs
-        pattern_mods = []
-        traverse_pattern_modules @shell_module do |pattern_mod, parent_modules|
-          Loom.log.debug2(self) { "found pattern module => #{pattern_mod}" }
-          pattern_methods = pattern_mod.pattern_methods
+      def create_dsl_specs
+        dsl_mods = []
+        traverse_dsl_modules @shell_module do |dsl_mod, parent_modules|
+          Loom.log.debug2(self) { "found pattern module => #{dsl_mod}" }
+
+          dsl_builder = dsl_mod.dsl_builder
+          next if dsl_builder.patterns.empty?
 
-          next if pattern_methods.empty?
-          pattern_mods << {
-            :namespace_list => mod_namespace_list(pattern_mod, parent_modules),
-            :pattern_methods => pattern_methods,
-            :module => pattern_mod,
-            :parent_modules => parent_modules.dup
+          dsl_mods << {
+            namespace_list: create_namespace_list(dsl_mod, parent_modules),
+            dsl_builder: dsl_builder,
+            dsl_module: dsl_mod,
+            parent_modules: parent_modules
           }
         end
-        pattern_mods
+        dsl_mods
       end
 
-      def is_pattern_module(mod)
+      def dsl_module?(mod)
         mod.included_modules.include? Loom::Pattern
       end
 
-      def traverse_pattern_modules(mod, pattern_parents=[], visited={}, &block)
+      # Recursive method to walk the tree of dsl_builders representative of the
+      # .loom file pattern modules
+      def traverse_dsl_modules(mod, pattern_parents=[], visited={}, &block)
         return if visited[mod.name] # prevent cycles
         visited[mod.name] = true
 
-        yield mod, pattern_parents.dup if is_pattern_module(mod)
+        yield mod, pattern_parents.dup if dsl_module?(mod)
 
         # Traverse all sub modules, even ones that aren't
         # Loom::Pattern[s], since they might contain more sub modules
@@ -185,7 +206,7 @@ module Loom::Pattern
 
         pattern_parents << mod
         sub_modules.each do |sub_mod|
-          traverse_pattern_modules sub_mod, pattern_parents.dup, visited, &block
+          traverse_dsl_modules sub_mod, pattern_parents.dup, visited, &block
         end
       end
     end

data/lib/loom/runner/all.rb

@@ -0,0 +1,2 @@
+require_relative "execution_context"
+require_relative "sshkit_connector"

data/lib/loom/runner/execution_context.rb

@@ -0,0 +1,9 @@
+module Loom::RunnerModule
+  class ExecutionContext
+    attr_reader :module_accesses, :fact_sets
+
+    def initialize(shell_api)
+    end
+  end
+end
+

data/lib/loom/{dsl.rb → runner/sshkit_connector.rb}

@@ -5,12 +5,8 @@ require 'socket'
 # https://github.com/capistrano/sshkit/blob/master/lib/sshkit/backends/abstract.rb
 # https://github.com/capistrano/sshkit/blob/master/lib/sshkit/backends/netssh.rb
 # https://github.com/capistrano/sshkit/blob/master/lib/sshkit/backends/local.rb
-module Loom
-
-  # TODO: Rename this to something like SSHKitWrapper, DSL is a
-  # terribly misinformative name.
-  module DSL
-
+module Loom::RunnerModule
+  module SSHKitConnector
     UnexpectedHostError = Class.new Loom::LoomError
     SSHConnectionError = Class.new Loom::LoomError
 
@@ -22,7 +18,9 @@ module Loom
     # +&block+ should accept an SSHKit::Backend and SSHKit::Host
     def on_host(host_specs, &run_block)
       host_specs.each do |spec|
-        raise UnexpectedHostError, "not a HostSpec => #{spec}" unless spec.is_a? HostSpec
+        unless spec.is_a? Loom::HostSpec
+          raise UnexpectedHostError "not a HostSpec => #{spec}"
+        end
       end
 
       host_spec_map = host_specs.reduce({}) do |map, spec|

data/lib/loom/runner.rb

@@ -1,10 +1,13 @@
 module Loom
+  # TODO: Make Loom::Runner a module and move this to
+  # Loom::Runner::Core. Loom::RunnerModule is set up as a temporary namespace
+  # until that happens.
   class Runner
 
     PatternExecutionError = Class.new Loom::LoomError
     FailFastExecutionError = Class.new PatternExecutionError
 
-    include Loom::DSL
+    include Loom::RunnerModule::SSHKitConnector
 
     def initialize(loom_config, pattern_slugs=[], other_facts={})
       @pattern_slugs = pattern_slugs
@@ -73,11 +76,13 @@ module Loom
         exit 97
       rescue Loom::LoomError => e
         Loom.log.error "Loom::LoomError => #{e.inspect}, run with -d for more"
-        Loom.log.debug e.cause.backtrace.join "\n\t"
+        backtrace = e.cause.nil? ? e.backtrace : e.cause.backtrace
+        Loom.log.debug backtrace.join "\n\t"
+        Loom.log.debug backtrace.join "\n\t"
         exit 98
       rescue => e
-        Loom.log.fatal "fatal error => #{e.inspect}"
-        Loom.log.fatal e.backtrace.join "\n\t"
+        # TODO: Make error/exception logging look like this everywhere
+        Loom.log.fatal "fatal error =>\n#{e.inspect}\n\t#{e.backtrace.join("\n\t\t")}"
 
         loom_files = @loom_config.files.loom_files
         loom_errors = e.backtrace.select { |line| line =~ /(#{loom_files.join("|")})/ }
@@ -119,7 +124,7 @@ module Loom
       @pattern_refs = pattern_loader.patterns @pattern_slugs
 
       @loom_config[:loomfile_autoloads].each do |path|
-        Loom.log.debug { "autoloading: #{path}" }
+        Loom.log.debug { "requiring config[:loomfile_autoloads]: #{path}" }
         require path
       end
       @mod_loader = Loom::Mods::ModLoader.new @loom_config
@@ -135,40 +140,18 @@ module Loom
 
         begin
           @pattern_refs.each do |pattern_ref|
-            slug = pattern_ref.slug
-            pattern_description = "[#{hostname} => #{slug}]"
-
             if @caught_sig_int
-              Loom.log.warn "caught SIGINT, skipping #{pattern_description}"
+              Loom.log.warn "caught SIGINT, skipping #{log_token(pattern_ref, hostname)}"
               next
             elsif inventory_list.disabled? hostname
               Loom.log.warn "host disabled due to previous failure, " +
-                            "skipping: #{pattern_description}"
+                            "skipping: #{log_token(pattern_ref, hostname)}"
               next
             end
 
-            Loom.log.debug "collecting facts for => #{pattern_description}"
-            # Collects facts for each pattern run on each host, this way if one
-            # pattern run updates would be facts, the next pattern will see the
-            # new fact.
-            fact_shell = Loom::Shell.create @mod_loader, sshkit_backend, dry_run
-            fact_set = Loom::Facts.fact_set(host_spec, fact_shell, @loom_config)
-                         .merge @other_facts
-
-            Loom.log.info "running pattern => #{pattern_description}"
-            # Creates a new shell for executing the pattern, so as not to be
-            # tainted by the fact finding shell above.
-            pattern_shell = Loom::Shell.create @mod_loader, sshkit_backend, dry_run
-
-            Loom.log.warn "dry run only => #{pattern_description}" if dry_run
-            failures = execute_pattern pattern_ref, pattern_shell, fact_set
-
-            if failures.empty?
-              Loom.log.debug "success on => #{pattern_description}"
-            else
-              Loom.log.error "failures on => #{pattern_description}:\n\t%s" % (
-                failures.join("\n\t"))
-            end
+            # wip-patternbuilder is adding this feature
+#            run_analysis_phase(pattern_ref, host_spec, sshkit_backend)
+            run_execution_phase(pattern_ref, host_spec, sshkit_backend, dry_run)
           end
         rescue IOError => e
           # TODO: Try to patch SSHKit for a more specific error for unexpected
@@ -184,6 +167,32 @@ module Loom
       end
     end
 
+    def run_execution_phase(pattern_ref, host_spec, sshkit_backend, dry_run)
+      log_token = log_token(pattern_ref, host_spec.hostname)
+      Loom.log.debug "collecting facts for => #{log_token}"
+      # Collects facts for each pattern run on each host, this way if one
+      # pattern run updates would be facts, the next pattern will see the
+      # new fact.
+      fact_shell = Loom::Shell.create @mod_loader, sshkit_backend, dry_run
+      fact_set = Loom::Facts.fact_set(host_spec, fact_shell, @loom_config)
+        .merge @other_facts
+
+      Loom.log.info "running pattern => #{log_token}"
+      # Creates a new shell for executing the pattern, so as not to be
+      # tainted by the fact finding shell above.
+      pattern_shell = Loom::Shell.create @mod_loader, sshkit_backend, dry_run
+
+      Loom.log.warn "dry run only => #{log_token}" if dry_run
+      failures = execute_pattern pattern_ref, pattern_shell, fact_set
+
+      if failures.empty?
+        Loom.log.debug "success on => #{log_token}"
+      else
+        Loom.log.error "failures on => #{log_token}:\n\t%s" % (
+          failures.join("\n\t"))
+      end
+    end
+
     def execute_pattern(pattern_ref, shell, fact_set)
       shell_session = shell.session
       hostname = fact_set.hostname
@@ -218,7 +227,11 @@ module Loom
       run_failure
     end
 
-    private
+    def log_token(pattern_ref, hostname)
+      slug = pattern_ref.slug
+      "[#{hostname} => #{slug}]"
+    end
+
     def handle_host_failure_strategy(hostname, failure_summary=nil)
       failure_strategy = @loom_config.run_failure_strategy.to_sym
 

data/lib/loom/shell/api.rb

@@ -20,14 +20,14 @@ module Loom::Shell
     end
 
     # This is the entry point for `loom.foo` calls from .loom files.
-    def method_missing(name, *args, &block)
-      Loom.log.debug3(self) { "shell api => #{name} #{args} #{block}" }
+    def method_missing(name, *args, **kwargs, &block)
+      Loom.log.debug3(self) { "shell api => #{name} #{args} #{kwargs} #{block}" }
       # TODO: The relationship between shell and mod_loader seems leaky here, a
       # Shell::Api should have a shell and not care about the mod_loader,
       # currently it seems to violate Demeter. The shell should dispatch to the
       # mod_loader only as an implementation detail. Otherwise this is harder to
       # test.
-      @mod_loader.send name, @shell, *args, &block
+      @mod_loader.send name, @shell, *args, **kwargs, &block
     end
   end
 

data/lib/loom/shell/cmd_wrapper.rb

@@ -1,4 +1,4 @@
-require "shellwords"
+  require "shellwords"
 
 module Loom::Shell
 
@@ -27,7 +27,7 @@ module Loom::Shell
             parts
           end
         end
-        CmdWrapper.new *cmd_parts.flatten, {
+        CmdWrapper.new *cmd_parts.flatten, **{
           :should_quote => should_quote,
           :is_wrapped => true
         }
@@ -39,6 +39,9 @@ module Loom::Shell
     # @param :redirc [Array<CmdRedirect>] STDIO redirection for the command
     # in quotes.
     def initialize(*cmd, should_quote: false, is_wrapped: false, redirect: [])
+      if cmd.last.is_a?(Hash)
+        raise ArgumentError.new "kwargs mixed into cmd"
+      end
       @cmd_parts = cmd.flatten
       @should_quote = should_quote
       @is_wrapped = is_wrapped

data/lib/loom/shell/core.rb

@@ -168,22 +168,22 @@ module Loom::Shell
       cmd_parts.compact!
       raise "empty command passed to execute" if cmd_parts.empty?
 
-      result = if @dry_run
-                 wrap :printf, :first => true do
-                   cmd_result = execute_internal *cmd_parts, **cmd_opts
-                   Loom.log.info do
-                     "\t%s" % prompt_fmt(cmd_result.full_stdout.strip)
-                   end
-                   cmd_result
-                 end
-               else
-                 execute_internal *cmd_parts, **cmd_opts
-               end
-      @session << CmdResult.create_from_sshkit_command(result, is_test, self)
-
-      Loom.log.debug @session.last.stdout unless @session.last.stdout.empty?
-      Loom.log.debug @session.last.stderr unless @session.last.stderr.empty?
-      @session.last
+      sshkit_result = if @dry_run
+                        wrap(:printf, first: true) do
+                          r = execute_internal(*cmd_parts, **cmd_opts)
+                          Loom.log.info { "\t%s" % prompt_fmt(r.full_stdout.strip) }
+                          r
+                        end
+                      else
+                        execute_internal(*cmd_parts, **cmd_opts)
+                      end
+      result =
+        CmdResult.create_from_sshkit_command(sshkit_result, is_test, self)
+
+      @session << result
+      Loom.log.debug result.stdout unless result.stdout.empty?
+      Loom.log.debug result.stderr unless result.stderr.empty?
+      result
     end
     alias_method :exec, :execute
 

data/lib/loom/shell.rb

@@ -3,8 +3,10 @@ module Loom
 
     VerifyError = Class.new Loom::LoomError
 
-    def self.create(*args)
-      Loom::Shell::Core.new *args
+    class << self
+      def create(mod_loader, sshkit_backend, dry_run, no_exec: false)
+        Loom::Shell::Core.new mod_loader, sshkit_backend, dry_run
+      end
     end
   end
 end

data/lib/loom/version.rb

@@ -1,3 +1,3 @@
 module Loom
-  VERSION = "0.0.6"
+  VERSION = "0.0.9"
 end

data/lib/loomext/coremods/exec.rb

@@ -62,6 +62,7 @@ module LoomExt::CoreMods
 
   class Fail < Loom::Mods::Module
     register_mod :fail do |message=nil|
+      # TODO: create a stacktrace that indicates the raise happened from the .loom file.
       raise FailError, message
     end
   end

data/lib/loomext/coremods/systemd/all.rb

@@ -1,2 +1,3 @@
 require_relative "systemd"
+require_relative "hostname"
 require_relative "systemd_units"

data/lib/loomext/coremods/systemd/hostname.rb

@@ -0,0 +1,10 @@
+module LoomExt::CoreMods
+
+  class Hostname < Loom::Mods::Module
+    include SystemdCommon
+
+    register_mod :sethostname do |**opts|
+      shell.execute :hostnamectl, **opts
+    end
+  end
+end