loom-core 0.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (79) hide show
  1. checksums.yaml +7 -0
  2. data/.gitignore +1 -0
  3. data/.rspec +2 -0
  4. data/Gemfile +4 -0
  5. data/Gemfile.lock +99 -0
  6. data/Guardfile +54 -0
  7. data/Rakefile +6 -0
  8. data/bin/loom +185 -0
  9. data/lib/env/development.rb +1 -0
  10. data/lib/loom.rb +44 -0
  11. data/lib/loom/all.rb +20 -0
  12. data/lib/loom/config.rb +106 -0
  13. data/lib/loom/core_ext.rb +37 -0
  14. data/lib/loom/dsl.rb +60 -0
  15. data/lib/loom/facts.rb +13 -0
  16. data/lib/loom/facts/all.rb +2 -0
  17. data/lib/loom/facts/fact_file_provider.rb +86 -0
  18. data/lib/loom/facts/fact_set.rb +138 -0
  19. data/lib/loom/host_spec.rb +32 -0
  20. data/lib/loom/inventory.rb +124 -0
  21. data/lib/loom/logger.rb +141 -0
  22. data/lib/loom/method_signature.rb +174 -0
  23. data/lib/loom/mods.rb +4 -0
  24. data/lib/loom/mods/action_proxy.rb +105 -0
  25. data/lib/loom/mods/all.rb +3 -0
  26. data/lib/loom/mods/mod_loader.rb +80 -0
  27. data/lib/loom/mods/module.rb +113 -0
  28. data/lib/loom/pattern.rb +15 -0
  29. data/lib/loom/pattern/all.rb +7 -0
  30. data/lib/loom/pattern/definition_context.rb +74 -0
  31. data/lib/loom/pattern/dsl.rb +176 -0
  32. data/lib/loom/pattern/hook.rb +28 -0
  33. data/lib/loom/pattern/loader.rb +48 -0
  34. data/lib/loom/pattern/reference.rb +71 -0
  35. data/lib/loom/pattern/reference_set.rb +169 -0
  36. data/lib/loom/pattern/result_reporter.rb +77 -0
  37. data/lib/loom/runner.rb +209 -0
  38. data/lib/loom/shell.rb +12 -0
  39. data/lib/loom/shell/all.rb +10 -0
  40. data/lib/loom/shell/api.rb +48 -0
  41. data/lib/loom/shell/cmd_result.rb +33 -0
  42. data/lib/loom/shell/cmd_wrapper.rb +164 -0
  43. data/lib/loom/shell/core.rb +226 -0
  44. data/lib/loom/shell/harness_blob.rb +26 -0
  45. data/lib/loom/shell/harness_command_builder.rb +50 -0
  46. data/lib/loom/shell/session.rb +25 -0
  47. data/lib/loom/trap.rb +44 -0
  48. data/lib/loom/version.rb +3 -0
  49. data/lib/loomext/all.rb +4 -0
  50. data/lib/loomext/corefacts.rb +6 -0
  51. data/lib/loomext/corefacts/all.rb +8 -0
  52. data/lib/loomext/corefacts/facter_provider.rb +24 -0
  53. data/lib/loomext/coremods.rb +5 -0
  54. data/lib/loomext/coremods/all.rb +13 -0
  55. data/lib/loomext/coremods/exec.rb +50 -0
  56. data/lib/loomext/coremods/files.rb +104 -0
  57. data/lib/loomext/coremods/net.rb +33 -0
  58. data/lib/loomext/coremods/package/adapter.rb +100 -0
  59. data/lib/loomext/coremods/package/package.rb +62 -0
  60. data/lib/loomext/coremods/user.rb +82 -0
  61. data/lib/loomext/coremods/vm.rb +0 -0
  62. data/lib/loomext/coremods/vm/all.rb +6 -0
  63. data/lib/loomext/coremods/vm/vbox.rb +84 -0
  64. data/loom.gemspec +39 -0
  65. data/loom/inventory.yml +13 -0
  66. data/scripts/harness.sh +242 -0
  67. data/spec/loom/host_spec_spec.rb +101 -0
  68. data/spec/loom/inventory_spec.rb +154 -0
  69. data/spec/loom/method_signature_spec.rb +275 -0
  70. data/spec/loom/pattern/dsl_spec.rb +207 -0
  71. data/spec/loom/shell/cmd_wrapper_spec.rb +239 -0
  72. data/spec/loom/shell/harness_blob_spec.rb +42 -0
  73. data/spec/loom/shell/harness_command_builder_spec.rb +36 -0
  74. data/spec/runloom.sh +35 -0
  75. data/spec/scripts/harness_spec.rb +385 -0
  76. data/spec/spec_helper.rb +94 -0
  77. data/spec/test.loom +370 -0
  78. data/spec/test_loom_spec.rb +57 -0
  79. metadata +287 -0
@@ -0,0 +1,71 @@
1
+ module Loom::Pattern
2
+ class Reference
3
+
4
+ attr_reader :slug, :source_file, :desc
5
+
6
+ def initialize(slug, unbound_method, source_file, definition_ctx, description)
7
+ @slug = slug
8
+ @unbound_method = unbound_method
9
+ @source_file = source_file
10
+ @definition_ctx = definition_ctx
11
+ @desc = description
12
+ end
13
+
14
+ def call(shell_api, host_fact_set)
15
+ run_context = RunContext.new @unbound_method, @definition_ctx
16
+
17
+ fact_set = @definition_ctx.fact_set host_fact_set
18
+ Loom.log.debug5(self) { "fact set for pattern execution => #{fact_set.facts}" }
19
+
20
+ @definition_ctx.define_let_readers run_context, fact_set
21
+
22
+ begin
23
+ run_context.run shell_api, fact_set
24
+ rescue => e
25
+ error_msg = "error executing '#{slug}' in #{source_file} => #{e} \n%s"
26
+ Loom.log.error(error_msg % e.backtrace.join("\n\t"))
27
+ raise
28
+ end
29
+ end
30
+
31
+ private
32
+
33
+ ##
34
+ # A small class to bind the unbound_method to and provide context
35
+ # in the case of errors.
36
+ class RunContext
37
+ def initialize(unbound_method, definition_ctx)
38
+ @bound_method = unbound_method.bind self
39
+ @definition_ctx = definition_ctx
40
+ end
41
+
42
+ def run(shell_api, fact_set)
43
+ before_hooks = @definition_ctx.before_hooks
44
+ after_hooks = @definition_ctx.after_hooks
45
+
46
+ begin
47
+ Loom.log.debug1(self) { "before hooks => #{before_hooks}"}
48
+ before_hooks.each do |hook|
49
+ Loom.log.debug2(self) { "executing before hook => #{hook}"}
50
+ self.instance_exec shell_api, fact_set, &hook.block
51
+ end
52
+
53
+ # This is the entry point into calling patterns.
54
+ apply_pattern shell_api, fact_set
55
+ ensure
56
+ Loom.log.debug1(self) { "after hooks => #{after_hooks}" }
57
+ after_hooks.each do |hook|
58
+ Loom.log.debug2(self) { "executing after hook => #{hook}"}
59
+ self.instance_exec shell_api, fact_set, &hook.block
60
+ end
61
+ end
62
+ end
63
+
64
+ private
65
+ def apply_pattern(*args)
66
+ @bound_method.call *args
67
+ end
68
+ end
69
+
70
+ end
71
+ end
@@ -0,0 +1,169 @@
1
+ module Loom::Pattern
2
+
3
+ DuplicatePatternRef = Class.new Loom::LoomError
4
+ UnknownPatternMethod = Class.new Loom::LoomError
5
+ InvalidPatternNamespace = Class.new Loom::LoomError
6
+
7
+ ##
8
+ # A collection of Pattern::Reference objects
9
+ class ReferenceSet
10
+
11
+ class << self
12
+ def load_from_file(path)
13
+ Loom.log.debug1(self) { "loading patterns from file => #{path}" }
14
+ Builder.create File.read(path), path
15
+ end
16
+ end
17
+
18
+ def initialize
19
+ @slug_to_ref_map = {}
20
+ end
21
+
22
+ def slugs
23
+ @slug_to_ref_map.keys
24
+ end
25
+
26
+ def pattern_refs
27
+ @slug_to_ref_map.values
28
+ end
29
+
30
+ def get_pattern_ref(slug)
31
+ ref = @slug_to_ref_map[slug]
32
+ raise UnknownPatternMethod, slug unless ref
33
+ ref
34
+ end
35
+ alias_method :[], :get_pattern_ref
36
+
37
+ def merge!(ref_set)
38
+ self.add_pattern_refs(ref_set.pattern_refs)
39
+ end
40
+
41
+ def add_pattern_refs(refs)
42
+ map = @slug_to_ref_map
43
+ refs.each do |ref|
44
+ Loom.log.debug2(self) { "adding ref to set => #{ref.slug}" }
45
+ raise DuplicatePatternRef, ref.slug if map[ref.slug]
46
+ map[ref.slug] = ref
47
+ end
48
+ end
49
+
50
+ class Builder
51
+ using Loom::CoreExt # using demodulize for namespace creation
52
+
53
+ class << self
54
+ def create(ruby_code, source)
55
+ shell_module = Module.new
56
+ shell_module.include Loom::Pattern
57
+ shell_module.module_eval ruby_code, source, 1
58
+ shell_module.namespace ""
59
+
60
+ self.new(shell_module, source).build
61
+ end
62
+ end
63
+
64
+ def initialize(shell_module, source)
65
+ @shell_module = shell_module
66
+ @pattern_mod_specs = pattern_mod_specs
67
+ @source = source
68
+ end
69
+
70
+ def build
71
+ ref_set = ReferenceSet.new
72
+ ref_set.add_pattern_refs pattern_refs
73
+ ref_set
74
+ end
75
+
76
+ private
77
+ def pattern_refs
78
+ @pattern_mod_specs.map { |mod_spec| refs_for_mod_spec mod_spec }.flatten
79
+ end
80
+
81
+ def refs_for_mod_spec(mod_spec)
82
+ mod = mod_spec[:module]
83
+ context = context_for_mod_spec mod_spec
84
+ source = @source
85
+
86
+ mod_spec[:pattern_methods].map do |m|
87
+ method = mod.pattern_method m
88
+ desc = mod.pattern_description m
89
+ slug = compute_slug mod_spec[:namespace_list], m
90
+
91
+ Loom.log.warn "no descripiton for pattern => #{slug}" unless desc
92
+ Reference.new slug, method, source, context, desc
93
+ end
94
+ end
95
+
96
+ def context_for_mod_spec(mod_spec)
97
+ parents = mod_spec[:parent_modules].find_all do |mod|
98
+ is_pattern_module mod
99
+ end
100
+ parent_context = parents.reduce(nil) do |parent_ctx, parent_mod|
101
+ DefinitionContext.new parent_mod, parent_ctx
102
+ end
103
+
104
+ mod = mod_spec[:module]
105
+ DefinitionContext.new mod, parent_context
106
+ end
107
+
108
+ def compute_slug(namespace_list, pattern_method_name)
109
+ namespace_list.dup.push(pattern_method_name).join ":"
110
+ end
111
+
112
+ def mod_namespace_list(pattern, parent_modules)
113
+ mods = parent_modules.dup << pattern
114
+ mods.reduce([]) do |memo, mod|
115
+ mod_name = if mod.respond_to?(:namespace) && mod.namespace
116
+ mod.namespace
117
+ else
118
+ mod.name.demodulize rescue ''
119
+ end
120
+ if memo.size > 0 && mod_name.empty?
121
+ raise InvalidPatternNamespace, "only the root can have an empty namespace"
122
+ end
123
+ memo << mod_name.downcase unless mod_name.empty?
124
+ memo
125
+ end
126
+ end
127
+
128
+ def pattern_mod_specs
129
+ pattern_mods = []
130
+ traverse_pattern_modules @shell_module do |pattern_mod, parent_modules|
131
+ Loom.log.debug2(self) { "found pattern module => #{pattern_mod}" }
132
+ pattern_methods = pattern_mod.pattern_methods
133
+
134
+ next if pattern_methods.empty?
135
+ pattern_mods << {
136
+ :namespace_list => mod_namespace_list(pattern_mod, parent_modules),
137
+ :pattern_methods => pattern_methods,
138
+ :module => pattern_mod,
139
+ :parent_modules => parent_modules.dup
140
+ }
141
+ end
142
+ pattern_mods
143
+ end
144
+
145
+ def is_pattern_module(mod)
146
+ mod.included_modules.include? Loom::Pattern
147
+ end
148
+
149
+ def traverse_pattern_modules(mod, pattern_parents=[], visited={}, &block)
150
+ return if visited[mod.name] # prevent cycles
151
+ visited[mod.name] = true
152
+
153
+ yield mod, pattern_parents.dup if is_pattern_module(mod)
154
+
155
+ # Traverse all sub modules, even ones that aren't
156
+ # Loom::Pattern[s], since they might contain more sub modules
157
+ # themselves.
158
+ sub_modules = mod.constants
159
+ .map { |c| mod.const_get(c) }
160
+ .find_all { |m| m.is_a? Module }
161
+
162
+ pattern_parents << mod
163
+ sub_modules.each do |sub_mod|
164
+ traverse_pattern_modules sub_mod, pattern_parents.dup, visited, &block
165
+ end
166
+ end
167
+ end
168
+ end
169
+ end
@@ -0,0 +1,77 @@
1
+ module Loom::Pattern
2
+ class ResultReporter
3
+ def initialize(loom_config, pattern_slug, hostname, shell_session)
4
+ @loom_config = loom_config
5
+ @start = Time.now
6
+ @delta_t = nil
7
+ @hostname = hostname
8
+ @pattern_slug = pattern_slug
9
+ @shell_session = shell_session
10
+ end
11
+
12
+ attr_reader :hostname
13
+
14
+ def failure_summary
15
+ return "scenario did not fail" if success?
16
+ scenario_string
17
+ end
18
+
19
+ def write_report
20
+ @delta_t = Time.now - @start
21
+
22
+ report = generate_report.join "\n\t"
23
+ if success?
24
+ Loom.log.info report
25
+ else
26
+ Loom.log.warn report
27
+ end
28
+ end
29
+
30
+ private
31
+ def success?
32
+ @shell_session.success?
33
+ end
34
+
35
+ def scenario_string
36
+ status = success? ? "OK" : "FAILED"
37
+ "#{hostname} => #{@pattern_slug} [Result: #{status}] "
38
+ end
39
+
40
+ def generate_report
41
+ cmds = @shell_session.command_results
42
+
43
+ report = ["--- #{scenario_string}"]
44
+ report << "Completed in: %01.3fs" % @delta_t
45
+
46
+ cmds.find_all { |cmd| !cmd.is_test }.each do |cmd|
47
+ if !cmd.success? || @loom_config.run_verbose
48
+ report.concat generate_cmd_report(cmd)
49
+ end
50
+ end
51
+
52
+ report
53
+ end
54
+
55
+ def generate_cmd_report(cmd)
56
+ status = cmd.success? ? "Success" : "Failed"
57
+
58
+ report = []
59
+ report << ""
60
+ report << "--- #{status} Command ---"
61
+ report << "$ #{cmd.command}"
62
+
63
+ unless cmd.stdout.empty?
64
+ report << cmd.stdout
65
+ end
66
+
67
+ unless cmd.stderr.empty?
68
+ report << "[STDERR]:"
69
+ report << cmd.stderr
70
+ end
71
+
72
+ report << "[EXIT STATUS]: #{cmd.exit_status}"
73
+
74
+ report
75
+ end
76
+ end
77
+ end
@@ -0,0 +1,209 @@
1
+ module Loom
2
+ class Runner
3
+
4
+ PatternExecutionError = Class.new Loom::LoomError
5
+ FailFastExecutionError = Class.new PatternExecutionError
6
+
7
+ include Loom::DSL
8
+
9
+ def initialize(loom_config, pattern_slugs=[], other_facts={})
10
+ @pattern_slugs = pattern_slugs
11
+ @loom_config = loom_config
12
+ @other_facts = other_facts
13
+
14
+ @run_failures = []
15
+ @result_reports = []
16
+
17
+ # these are initialized in +load+
18
+ @inventory_list = nil
19
+ @active_hosts = nil
20
+ @pattern_refs = nil
21
+ @mod_loader = nil
22
+
23
+ Loom.log.debug1(self) do
24
+ "initialized runner with config => #{loom_config.dump}"
25
+ end
26
+
27
+ @caught_sig_int = false
28
+ end
29
+
30
+ def run(dry_run)
31
+ install_signal_traps
32
+
33
+ begin
34
+ load
35
+
36
+ if @pattern_refs.empty?
37
+ Loom.log.warn "no patterns given, there's no work to do"
38
+ return
39
+ end
40
+ if @active_hosts.empty?
41
+ Loom.log.warn "no hosts in the active inventory"
42
+ return
43
+ end
44
+
45
+ hostnames = @active_hosts.map(&:hostname)
46
+ Loom.log.info do
47
+ "executing patterns #{@pattern_slugs} across hosts #{hostnames}"
48
+ end
49
+
50
+ run_internal dry_run
51
+
52
+ unless @run_failures.empty?
53
+ raise PatternExecutionError, @run_failures
54
+ end
55
+ rescue Loom::Trap::SignalExit => e
56
+ Loom.log.error "exiting on signal => #{e.signal}"
57
+ # Exit with the signal code or 40 for unknown Signal
58
+ code = Signal.list[e.signal] || 40
59
+ exit code
60
+ rescue PatternExecutionError => e
61
+ num_patterns_failed = @run_failures.size
62
+ Loom.log.error "error executing #{num_patterns_failed} patterns => #{e}"
63
+ Loom.log.debug e.backtrace.join "\n"
64
+ exit 100 + num_patterns_failed
65
+ rescue Loom::LoomError => e
66
+ Loom.log.error "loom error => #{e.inspect}"
67
+ exit 98
68
+ rescue => e
69
+ Loom.log.fatal "fatal error => #{e.inspect}"
70
+ Loom.log.fatal e.backtrace.join "\n\t"
71
+ exit 99
72
+ end
73
+ end
74
+
75
+ private
76
+
77
+ def install_signal_traps
78
+ signal_handler = Loom::Trap::Handler.new do |sig, count|
79
+ case sig
80
+ when Loom::Trap::Sig::INT
81
+ @caught_sig_int = true
82
+ if count == 1
83
+ puts "Caught #{sig}, exiting after current pattern completion"
84
+ puts "Ctrl-C again to exit immediately"
85
+ else
86
+ puts "Caught #{sig}"
87
+ raise Loom::Trap::SignalExit.new sig
88
+ end
89
+ else
90
+ puts "Caught unhandled signal #{sig}"
91
+ raise Loom::Trap::SignalExit.new sig
92
+ end
93
+ end
94
+ Loom::Trap.install(Loom::Trap::Sig::INT, signal_handler)
95
+ end
96
+
97
+ def load
98
+ @inventory_list =
99
+ Loom::Inventory::InventoryList.active_inventory @loom_config
100
+ @active_hosts = @inventory_list.hosts
101
+
102
+ pattern_loader = Loom::Pattern::Loader.load @loom_config
103
+ @pattern_refs = pattern_loader.patterns @pattern_slugs
104
+
105
+ @mod_loader = Loom::Mods::ModLoader.new @loom_config
106
+ end
107
+
108
+ def run_internal(dry_run)
109
+ # TODO: fix the bindings in the block below so we don't need
110
+ # this alias
111
+ inventory_list = @inventory_list
112
+
113
+ on_host @active_hosts do |sshkit_backend, host_spec|
114
+ hostname = host_spec.hostname
115
+
116
+ begin
117
+ @pattern_refs.each do |pattern_ref|
118
+ slug = pattern_ref.slug
119
+ pattern_description = "[#{hostname} => #{slug}]"
120
+
121
+ if @caught_sig_int
122
+ Loom.log.warn "caught SIGINT, skipping #{pattern_description}"
123
+ next
124
+ elsif inventory_list.disabled? hostname
125
+ Loom.log.warn "host disabled due to previous failure, " +
126
+ "skipping: #{pattern_description}"
127
+ next
128
+ end
129
+
130
+ Loom.log.debug "collecting facts for => #{pattern_description}"
131
+ # Collect facts for each pattern run on each host, this way if one
132
+ # pattern run updates would be facts, the next pattern will see the
133
+ # new fact.
134
+ fact_shell = Loom::Shell.create @mod_loader, sshkit_backend, dry_run
135
+ fact_set = Loom::Facts.fact_set(host_spec, fact_shell, @loom_config)
136
+ .merge @other_facts
137
+
138
+ Loom.log.info "running pattern => #{pattern_description}"
139
+ # Each pattern execution needs its own shell and mod loader to make
140
+ # sure context is reported correctly (this is probably a hack, there
141
+ # should just be a way to clear/ignore state from certain commands -
142
+ # like the fact_finding ones above).
143
+ pattern_shell = Loom::Shell.create @mod_loader, sshkit_backend, dry_run
144
+
145
+ Loom.log.warn "dry run only => #{pattern_description}" if dry_run
146
+ execute_pattern pattern_ref, pattern_shell, fact_set
147
+ end
148
+ rescue IOError => e
149
+ # TODO: Try to patch SSHKit for a more specific error for unexpected SSH
150
+ # disconnections
151
+ Loom.log.error "unexpected SSH disconnect => #{hostname}"
152
+ Loom.log.debug e
153
+ handle_host_failure_strategy hostname, e.message
154
+ rescue Errno::ECONNREFUSED => e
155
+ Loom.log.error "unable to connect to host => #{hostname}"
156
+ Loom.log.debug e
157
+ handle_host_failure_strategy hostname, e.message
158
+ end
159
+ end
160
+ end
161
+
162
+ def execute_pattern(pattern_ref, shell, fact_set)
163
+ shell_session = shell.session
164
+ hostname = fact_set.hostname
165
+ result_reporter = Loom::Pattern::ResultReporter.new(
166
+ @loom_config, pattern_ref.slug, hostname, shell_session)
167
+
168
+ # TODO: This is a crappy mechanism for tracking errors, there should be an
169
+ # exception thrown inside of Shell when a command fails and pattern
170
+ # execution should stop. All errors should come from exceptions.
171
+ run_failure = []
172
+ begin
173
+ pattern_ref.call(shell.shell_api, fact_set)
174
+ rescue Loom::ExecutionError => e
175
+ Loom.log.debug e.backtrace.join "\n\t"
176
+ run_failure << e
177
+ ensure
178
+ # TODO: this prints out [Result: OK] even if an exception is raised
179
+ result_reporter.write_report
180
+
181
+ # TODO: this is not the correct error condition.
182
+ unless shell_session.success?
183
+ run_failure << result_reporter.failure_summary
184
+ handle_host_failure_strategy hostname, result_reporter.failure_summary
185
+ end
186
+ @result_reports << result_reporter
187
+ @run_failures << run_failure unless run_failure.empty?
188
+ end
189
+ end
190
+
191
+ private
192
+ def handle_host_failure_strategy(hostname, failure_summary=nil)
193
+ failure_strategy = @loom_config.run_failure_strategy.to_sym
194
+
195
+ case failure_strategy
196
+ when :exclude_host
197
+ Loom.log.warn "disabling host per :run_failure_strategy => #{failure_strategy}"
198
+ @inventory_list.disable hostname
199
+ when :fail_fast
200
+ Loom.log.error "erroring out of failed scenario per :run_failure_strategy"
201
+ raise FailFastExecutionError, failure_summary
202
+ when :cowboy
203
+ Loom.log.warn "continuing on past failed scenario per :run_failure_strategy"
204
+ else
205
+ raise ConfigError, "unknown failure_strategy: #{failure_stratgy}"
206
+ end
207
+ end
208
+ end
209
+ end