chef-apply 0.1.2

Sign up to get free protection for your applications and to get access to all the features.
Files changed (39) hide show
  1. checksums.yaml +7 -0
  2. data/Gemfile +26 -0
  3. data/Gemfile.lock +423 -0
  4. data/LICENSE +201 -0
  5. data/README.md +41 -0
  6. data/Rakefile +32 -0
  7. data/bin/chef-run +23 -0
  8. data/chef-apply.gemspec +67 -0
  9. data/i18n/en.yml +513 -0
  10. data/lib/chef_apply.rb +20 -0
  11. data/lib/chef_apply/action/base.rb +158 -0
  12. data/lib/chef_apply/action/converge_target.rb +173 -0
  13. data/lib/chef_apply/action/install_chef.rb +30 -0
  14. data/lib/chef_apply/action/install_chef/base.rb +137 -0
  15. data/lib/chef_apply/action/install_chef/linux.rb +38 -0
  16. data/lib/chef_apply/action/install_chef/windows.rb +54 -0
  17. data/lib/chef_apply/action/reporter.rb +39 -0
  18. data/lib/chef_apply/cli.rb +470 -0
  19. data/lib/chef_apply/cli_options.rb +145 -0
  20. data/lib/chef_apply/config.rb +150 -0
  21. data/lib/chef_apply/error.rb +108 -0
  22. data/lib/chef_apply/errors/ccr_failure_mapper.rb +93 -0
  23. data/lib/chef_apply/file_fetcher.rb +70 -0
  24. data/lib/chef_apply/log.rb +42 -0
  25. data/lib/chef_apply/recipe_lookup.rb +117 -0
  26. data/lib/chef_apply/startup.rb +162 -0
  27. data/lib/chef_apply/status_reporter.rb +42 -0
  28. data/lib/chef_apply/target_host.rb +233 -0
  29. data/lib/chef_apply/target_resolver.rb +202 -0
  30. data/lib/chef_apply/telemeter.rb +162 -0
  31. data/lib/chef_apply/telemeter/patch.rb +32 -0
  32. data/lib/chef_apply/telemeter/sender.rb +121 -0
  33. data/lib/chef_apply/temp_cookbook.rb +159 -0
  34. data/lib/chef_apply/text.rb +77 -0
  35. data/lib/chef_apply/ui/error_printer.rb +261 -0
  36. data/lib/chef_apply/ui/plain_text_element.rb +75 -0
  37. data/lib/chef_apply/ui/terminal.rb +94 -0
  38. data/lib/chef_apply/version.rb +20 -0
  39. metadata +376 -0
@@ -0,0 +1,202 @@
1
+ #
2
+ # Copyright:: Copyright (c) 2018 Chef Software Inc.
3
+ # License:: Apache License, Version 2.0
4
+ #
5
+ # Licensed under the Apache License, Version 2.0 (the "License");
6
+ # you may not use this file except in compliance with the License.
7
+ # You may obtain a copy of the License at
8
+ #
9
+ # http://www.apache.org/licenses/LICENSE-2.0
10
+ #
11
+ # Unless required by applicable law or agreed to in writing, software
12
+ # distributed under the License is distributed on an "AS IS" BASIS,
13
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14
+ # See the License for the specific language governing permissions and
15
+ # limitations under the License.
16
+ #
17
+
18
+ require "chef_apply/target_host"
19
+ require "chef_apply/error"
20
+
21
+ module ChefApply
22
+ class TargetResolver
23
+ MAX_EXPANDED_TARGETS = 24
24
+
25
+ def initialize(target, default_protocol, conn_options)
26
+ @default_proto = default_protocol
27
+ @unparsed_target = target
28
+ @split_targets = @unparsed_target.split(",")
29
+ @conn_options = conn_options.dup
30
+ @default_password = @conn_options.delete(:password)
31
+ @default_user = @conn_options.delete(:user)
32
+ end
33
+
34
+ # Returns the list of targets as an array of TargetHost instances,
35
+ # them to account for ranges embedded in the target name.
36
+ def targets
37
+ return @targets unless @targets.nil?
38
+ expanded_urls = []
39
+ @split_targets.each do |target|
40
+ expanded_urls = (expanded_urls | expand_targets(target))
41
+ end
42
+ @targets = expanded_urls.map do |url|
43
+ config = @conn_options.merge(config_for_target(url))
44
+ TargetHost.new(config.delete(:url), config)
45
+ end
46
+ end
47
+
48
+ def config_for_target(url)
49
+ prefix, target = prefix_from_target(url)
50
+
51
+ inline_password = nil
52
+ inline_user = nil
53
+ host = target
54
+ # Default greedy-scan of the regex means that
55
+ # $2 will resolve to content after the final "@"
56
+ # URL credentials will take precedence over the default :user
57
+ # in @conn_opts
58
+ if target =~ /(.*)@(.*)/
59
+ inline_credentials = $1
60
+ host = $2
61
+ # We'll use a non-greedy match to grab everthinmg up to the first ':'
62
+ # as username if there is no :, credentials is just the username
63
+ if inline_credentials =~ /(.+?):(.*)/
64
+ inline_user = $1
65
+ inline_password = $2
66
+ else
67
+ inline_user = inline_credentials
68
+ end
69
+ end
70
+ user, password = make_credentials(inline_user, inline_password)
71
+ { url: "#{prefix}#{host}",
72
+ user: user,
73
+ password: password }
74
+ end
75
+
76
+ # Merge the inline user/pass with the default user/pass, giving
77
+ # precedence to inline.
78
+ def make_credentials(inline_user, inline_password)
79
+ user = inline_user || @default_user
80
+ user = nil if user && user.empty?
81
+ password = (inline_password || @default_password)
82
+ password = nil if password && password.empty?
83
+ [user, password]
84
+ end
85
+
86
+ def prefix_from_target(target)
87
+ if target =~ /^(.+?):\/\/(.*)/
88
+ # We'll store the existing prefix to avoid it interfering
89
+ # with the check further below.
90
+ if ChefApply::Config::SUPPORTED_PROTOCOLS.include? $1.downcase
91
+ prefix = "#{$1}://"
92
+ target = $2
93
+ else
94
+ raise UnsupportedProtocol.new($1)
95
+ end
96
+ else
97
+ prefix = "#{@default_proto}://"
98
+ end
99
+ [prefix, target]
100
+ end
101
+
102
+ def expand_targets(target)
103
+ @current_target = target # Hold onto this for error reporting
104
+ do_parse([target.downcase])
105
+ end
106
+
107
+ private
108
+
109
+ # A string matching PREFIX[x:y]POSTFIX:
110
+ # POSTFIX can contain further ranges itself
111
+ # This uses a greedy match (.*) to get include every character
112
+ # up to the last "[" in PREFIX
113
+ # $1 - prefix; $2 - x, $3 - y, $4 unproccessed/remaining text
114
+ TARGET_WITH_RANGE = /^(.*)\[([\p{Alnum}]+):([\p{Alnum}]+)\](.*)/
115
+
116
+ def do_parse(targets, depth = 0)
117
+ raise TooManyRanges.new(@current_target) if depth > 2
118
+ new_targets = []
119
+ done = false
120
+ targets.each do |target|
121
+ if TARGET_WITH_RANGE =~ target
122
+ # $1 - prefix; $2 - x, $3 - y, $4 unprocessed/remaining text
123
+ expand_range(new_targets, $1, $2, $3, $4)
124
+ else
125
+ # Nothing more to expand
126
+ done = true
127
+ new_targets << target
128
+ end
129
+ end
130
+ if done
131
+ new_targets
132
+ else
133
+ do_parse(new_targets, depth + 1)
134
+ end
135
+ end
136
+
137
+ def expand_range(dest, prefix, start, stop, suffix)
138
+ prefix ||= ""
139
+ suffix ||= ""
140
+ start_is_int = Integer(start) >= 0 rescue false
141
+ stop_is_int = Integer(stop) >= 0 rescue false
142
+
143
+ if (start_is_int && !stop_is_int) || (stop_is_int && !start_is_int)
144
+ raise InvalidRange.new(@current_target, "[#{start}:#{stop}]")
145
+ end
146
+
147
+ # Ensure that a numeric range doesn't get created as a string, which
148
+ # would make the created Range further below fail to iterate for some values
149
+ # because of ASCII sorting.
150
+ if start_is_int
151
+ start = Integer(start)
152
+ end
153
+
154
+ if stop_is_int
155
+ stop = Integer(stop)
156
+ end
157
+
158
+ # For range to iterate correctly, the values must
159
+ # be low,high
160
+ if start > stop
161
+ temp = stop; stop = start; start = temp
162
+ end
163
+ Range.new(start, stop).each do |value|
164
+ # Ranges will resolve only numbers and letters,
165
+ # not other ascii characters that happen to fall between.
166
+ if start_is_int || /^[a-z0-9]/ =~ value
167
+ dest << "#{prefix}#{value}#{suffix}"
168
+ end
169
+ # Stop expanding as soon as we go over limit to prevent
170
+ # making the user wait for a massive accidental expansion
171
+ if dest.length > MAX_EXPANDED_TARGETS
172
+ raise TooManyTargets.new(@split_targets.length, MAX_EXPANDED_TARGETS)
173
+ end
174
+ end
175
+ end
176
+
177
+ class InvalidRange < ErrorNoLogs
178
+ def initialize(unresolved_target, given_range)
179
+ super("CHEFRANGE001", unresolved_target, given_range)
180
+ end
181
+ end
182
+
183
+ class TooManyRanges < ErrorNoLogs
184
+ def initialize(unresolved_target)
185
+ super("CHEFRANGE002", unresolved_target)
186
+ end
187
+ end
188
+
189
+ class TooManyTargets < ErrorNoLogs
190
+ def initialize(num_top_level_targets, max_targets)
191
+ super("CHEFRANGE003", num_top_level_targets, max_targets)
192
+ end
193
+ end
194
+
195
+ class UnsupportedProtocol < ErrorNoLogs
196
+ def initialize(attempted_protocol)
197
+ super("CHEFVAL011", attempted_protocol,
198
+ ChefApply::Config::SUPPORTED_PROTOCOLS.join(" "))
199
+ end
200
+ end
201
+ end
202
+ end
@@ -0,0 +1,162 @@
1
+ #
2
+ # Copyright:: Copyright (c) 2018 Chef Software Inc.
3
+ # License:: Apache License, Version 2.0
4
+ #
5
+ # Licensed under the Apache License, Version 2.0 (the "License");
6
+ # you may not use this file except in compliance with the License.
7
+ # You may obtain a copy of the License at
8
+ #
9
+ # http://www.apache.org/licenses/LICENSE-2.0
10
+ #
11
+ # Unless required by applicable law or agreed to in writing, software
12
+ # distributed under the License is distributed on an "AS IS" BASIS,
13
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14
+ # See the License for the specific language governing permissions and
15
+ # limitations under the License.
16
+ #
17
+
18
+ require "benchmark"
19
+ require "forwardable"
20
+ require "singleton"
21
+ require "json"
22
+ require "digest/sha1"
23
+ require "securerandom"
24
+ require "chef_apply/version"
25
+ require "chef_apply/config"
26
+ require "yaml"
27
+
28
+ module ChefApply
29
+
30
+ # This definites the Telemeter interface. Implementation thoughts for
31
+ # when we unstub it:
32
+ # - let's track the call sequence; most of our calls will be nested inside
33
+ # a main 'timed_capture', and it would be good to see ordering within nested calls.
34
+ class Telemeter
35
+ include Singleton
36
+ DEFAULT_INSTALLATION_GUID = "00000000-0000-0000-0000-000000000000"
37
+
38
+ class << self
39
+ extend Forwardable
40
+ def_delegators :instance, :timed_capture, :capture, :commit, :timed_action_capture, :timed_run_capture
41
+ def_delegators :instance, :pending_event_count, :last_event, :enabled?
42
+ def_delegators :instance, :make_event_payload
43
+ end
44
+
45
+ attr_reader :events_to_send, :run_timestamp
46
+
47
+ def enabled?
48
+ require "telemetry/decision"
49
+ ChefApply::Config.telemetry.enable && !Telemetry::Decision.env_opt_out?
50
+ end
51
+
52
+ def initialize
53
+ @events_to_send = []
54
+ @run_timestamp = Time.now.utc.strftime("%FT%TZ")
55
+ end
56
+
57
+ def timed_action_capture(action, &block)
58
+ # Note: we do not directly capture hostname for privacy concerns, but
59
+ # using a sha1 digest will allow us to anonymously see
60
+ # unique hosts to derive number of hosts affected by a command
61
+ target = action.target_host
62
+ target_data = { platform: {}, hostname_sha1: nil, transport_type: nil }
63
+ if target
64
+ target_data[:platform][:name] = target.base_os # :windows, :linux, eventually :macos
65
+ target_data[:platform][:version] = target.version
66
+ target_data[:platform][:architecture] = target.architecture
67
+ target_data[:hostname_sha1] = Digest::SHA1.hexdigest(target.hostname.downcase)
68
+ target_data[:transport_type] = target.transport_type
69
+ end
70
+ timed_capture(:action, { action: action.name, target: target_data }, &block)
71
+ end
72
+
73
+ def timed_run_capture(arguments, &block)
74
+ timed_capture(:run, arguments: arguments, &block)
75
+ end
76
+
77
+ def capture(name, data = {})
78
+ # Adding it to the head of the list will ensure that the
79
+ # sequence of events is preserved when we send the final payload
80
+ payload = make_event_payload(name, data)
81
+ @events_to_send.unshift payload
82
+ end
83
+
84
+ def timed_capture(name, data = {})
85
+ time = Benchmark.measure { yield }
86
+ data[:duration] = time.real
87
+ capture(name, data)
88
+ end
89
+
90
+ def commit
91
+ if enabled?
92
+ session = convert_events_to_session
93
+ write_session(session)
94
+ end
95
+ @events_to_send = []
96
+ end
97
+
98
+ def make_event_payload(name, data)
99
+ {
100
+ event: name,
101
+ properties: {
102
+ installation_id: installation_id,
103
+ run_timestamp: run_timestamp,
104
+ host_platform: host_platform,
105
+ event_data: data
106
+ }
107
+ }
108
+ end
109
+
110
+ def installation_id
111
+ @installation_id ||=
112
+ begin
113
+ File.read(ChefApply::Config.telemetry_installation_identifier_file).chomp
114
+ rescue
115
+ ChefApply::Log.info "could not read #{ChefApply::Config.telemetry_installation_identifier_file} - using default id"
116
+ DEFAULT_INSTALLATION_GUID
117
+ end
118
+ end
119
+
120
+ # For testing.
121
+ def pending_event_count
122
+ @events_to_send.length
123
+ end
124
+
125
+ def last_event
126
+ @events_to_send.last
127
+ end
128
+
129
+ private
130
+
131
+ def host_platform
132
+ @host_platform ||= case RUBY_PLATFORM
133
+ when /mswin|mingw|windows/
134
+ "windows"
135
+ else
136
+ RUBY_PLATFORM.split("-")[1]
137
+ end
138
+ end
139
+
140
+ def convert_events_to_session
141
+ YAML.dump({ "version" => ChefApply::VERSION,
142
+ "entries" => @events_to_send })
143
+ end
144
+
145
+ def write_session(session)
146
+ File.write(next_filename, convert_events_to_session)
147
+ end
148
+
149
+ def next_filename
150
+ id = 0
151
+ filename = ""
152
+ loop do
153
+ id += 1
154
+ filename = File.join(ChefApply::Config.telemetry_path,
155
+ "telemetry-payload-#{id}.yml")
156
+ break unless File.exist?(filename)
157
+ end
158
+ filename
159
+ end
160
+
161
+ end
162
+ end
@@ -0,0 +1,32 @@
1
+ #
2
+ # Copyright:: Copyright (c) 2018 Chef Software Inc.
3
+ # License:: Apache License, Version 2.0
4
+ #
5
+ # Licensed under the Apache License, Version 2.0 (the "License");
6
+ # you may not use this file except in compliance with the License.
7
+ # You may obtain a copy of the License at
8
+ #
9
+ # http://www.apache.org/licenses/LICENSE-2.0
10
+ #
11
+ # Unless required by applicable law or agreed to in writing, software
12
+ # distributed under the License is distributed on an "AS IS" BASIS,
13
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14
+ # See the License for the specific language governing permissions and
15
+ # limitations under the License.
16
+ #
17
+
18
+ class Telemetry
19
+ class Session
20
+ # The telemetry session data is normally kept in .chef, which we don't have.
21
+ def session_file
22
+ ChefApply::Config.telemetry_session_file.freeze
23
+ end
24
+ end
25
+
26
+ def deliver(data = {})
27
+ if ChefApply::Telemeter.instance.enabled?
28
+ payload = event.prepare(data)
29
+ client.await.fire(payload)
30
+ end
31
+ end
32
+ end
@@ -0,0 +1,121 @@
1
+ #
2
+ # Copyright:: Copyright (c) 2018 Chef Software Inc.
3
+ # License:: Apache License, Version 2.0
4
+ #
5
+ # Licensed under the Apache License, Version 2.0 (the "License");
6
+ # you may not use this file except in compliance with the License.
7
+ # You may obtain a copy of the License at
8
+ #
9
+ # http://www.apache.org/licenses/LICENSE-2.0
10
+ #
11
+ # Unless required by applicable law or agreed to in writing, software
12
+ # distributed under the License is distributed on an "AS IS" BASIS,
13
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14
+ # See the License for the specific language governing permissions and
15
+ # limitations under the License.
16
+ #
17
+
18
+ require "telemetry"
19
+ require "chef_apply/telemeter"
20
+ require "chef_apply/telemeter/patch"
21
+ require "chef_apply/log"
22
+ require "chef_apply/version"
23
+
24
+ module ChefApply
25
+ class Telemeter
26
+ class Sender
27
+ attr_reader :session_files
28
+
29
+ def self.start_upload_thread
30
+ # Find the files before we spawn the thread - otherwise
31
+ # we may accidentally pick up the current run's session file if it
32
+ # finishes before the thread scans for new files
33
+ session_files = Sender.find_session_files
34
+ sender = Sender.new(session_files)
35
+ Thread.new { sender.run }
36
+ end
37
+
38
+ def self.find_session_files
39
+ ChefApply::Log.info("Looking for telemetry data to submit")
40
+ session_search = File.join(ChefApply::Config.telemetry_path, "telemetry-payload-*.yml")
41
+ session_files = Dir.glob(session_search)
42
+ ChefApply::Log.info("Found #{session_files.length} sessions to submit")
43
+ session_files
44
+ end
45
+
46
+ def initialize(session_files)
47
+ @session_files = session_files
48
+ end
49
+
50
+ def run
51
+ if ChefApply::Telemeter.enabled?
52
+ ChefApply::Log.info("Telemetry enabled, beginning upload of previous session(s)")
53
+ # dev mode telemetry gets sent to a different location
54
+ if ChefApply::Config.telemetry.dev
55
+ ENV["CHEF_TELEMETRY_ENDPOINT"] ||= "https://telemetry-acceptance.chef.io"
56
+ end
57
+ session_files.each { |path| process_session(path) }
58
+ else
59
+ # If telemetry is not enabled, just clean up and return. Even though
60
+ # the telemetry gem will not send if disabled, log output saying that we're submitting
61
+ # it when it has been disabled can be alarming.
62
+ ChefApply::Log.info("Telemetry disabled, clearing any existing session captures without sending them.")
63
+ session_files.each { |path| FileUtils.rm_rf(path) }
64
+ end
65
+ FileUtils.rm_rf(ChefApply::Config.telemetry_session_file)
66
+ ChefApply::Log.info("Terminating, nothing more to do.")
67
+ rescue => e
68
+ ChefApply::Log.fatal "Sender thread aborted: '#{e}' failed at #{e.backtrace[0]}"
69
+ end
70
+
71
+ def process_session(path)
72
+ ChefApply::Log.info("Processing telemetry entries from #{path}")
73
+ content = load_and_clear_session(path)
74
+ submit_session(content)
75
+ end
76
+
77
+ def submit_session(content)
78
+ # Each file contains the actions taken within a single run of the chef tool.
79
+ # Each run is one session, so we'll first remove remove the session file
80
+ # to force creating a new one.
81
+ FileUtils.rm_rf(ChefApply::Config.telemetry_session_file)
82
+ # We'll use the version captured in the sesion file
83
+ entries = content["entries"]
84
+ cli_version = content["version"]
85
+ total = entries.length
86
+ telemetry = Telemetry.new(product: "chef-workstation",
87
+ origin: "command-line",
88
+ product_version: cli_version,
89
+ install_context: "omnibus")
90
+ total = entries.length
91
+ entries.each_with_index do |entry, x|
92
+ submit_entry(telemetry, entry, x + 1, total)
93
+ end
94
+ end
95
+
96
+ def submit_entry(telemetry, entry, sequence, total)
97
+ ChefApply::Log.info("Submitting telemetry entry #{sequence}/#{total}: #{entry} ")
98
+ telemetry.deliver(entry)
99
+ ChefApply::Log.info("Entry #{sequence}/#{total} submitted.")
100
+ rescue => e
101
+ # No error handling in telemetry lib, so at least track the failrue
102
+ ChefApply::Log.error("Failed to send entry #{sequence}/#{total}: #{e}")
103
+ ChefApply::Log.error("Backtrace: #{e.backtrace} ")
104
+ end
105
+
106
+ private
107
+
108
+ def load_and_clear_session(path)
109
+ content = File.read(path)
110
+ # We'll remove it now instead of after we parse or submit it -
111
+ # if we fail to deliver, we don't want to be stuck resubmitting it if the problem
112
+ # was due to payload. This is a trade-off - if we get a transient error, the
113
+ # payload will be lost.
114
+ # TODO: Improve error handling so we can intelligently decide whether to
115
+ # retry a failed load or failed submit.
116
+ FileUtils.rm_rf(path)
117
+ YAML.load(content)
118
+ end
119
+ end
120
+ end
121
+ end