chef-apply 0.1.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (39) hide show
  1. checksums.yaml +7 -0
  2. data/Gemfile +26 -0
  3. data/Gemfile.lock +423 -0
  4. data/LICENSE +201 -0
  5. data/README.md +41 -0
  6. data/Rakefile +32 -0
  7. data/bin/chef-run +23 -0
  8. data/chef-apply.gemspec +67 -0
  9. data/i18n/en.yml +513 -0
  10. data/lib/chef_apply.rb +20 -0
  11. data/lib/chef_apply/action/base.rb +158 -0
  12. data/lib/chef_apply/action/converge_target.rb +173 -0
  13. data/lib/chef_apply/action/install_chef.rb +30 -0
  14. data/lib/chef_apply/action/install_chef/base.rb +137 -0
  15. data/lib/chef_apply/action/install_chef/linux.rb +38 -0
  16. data/lib/chef_apply/action/install_chef/windows.rb +54 -0
  17. data/lib/chef_apply/action/reporter.rb +39 -0
  18. data/lib/chef_apply/cli.rb +470 -0
  19. data/lib/chef_apply/cli_options.rb +145 -0
  20. data/lib/chef_apply/config.rb +150 -0
  21. data/lib/chef_apply/error.rb +108 -0
  22. data/lib/chef_apply/errors/ccr_failure_mapper.rb +93 -0
  23. data/lib/chef_apply/file_fetcher.rb +70 -0
  24. data/lib/chef_apply/log.rb +42 -0
  25. data/lib/chef_apply/recipe_lookup.rb +117 -0
  26. data/lib/chef_apply/startup.rb +162 -0
  27. data/lib/chef_apply/status_reporter.rb +42 -0
  28. data/lib/chef_apply/target_host.rb +233 -0
  29. data/lib/chef_apply/target_resolver.rb +202 -0
  30. data/lib/chef_apply/telemeter.rb +162 -0
  31. data/lib/chef_apply/telemeter/patch.rb +32 -0
  32. data/lib/chef_apply/telemeter/sender.rb +121 -0
  33. data/lib/chef_apply/temp_cookbook.rb +159 -0
  34. data/lib/chef_apply/text.rb +77 -0
  35. data/lib/chef_apply/ui/error_printer.rb +261 -0
  36. data/lib/chef_apply/ui/plain_text_element.rb +75 -0
  37. data/lib/chef_apply/ui/terminal.rb +94 -0
  38. data/lib/chef_apply/version.rb +20 -0
  39. metadata +376 -0
@@ -0,0 +1,202 @@
1
+ #
2
+ # Copyright:: Copyright (c) 2018 Chef Software Inc.
3
+ # License:: Apache License, Version 2.0
4
+ #
5
+ # Licensed under the Apache License, Version 2.0 (the "License");
6
+ # you may not use this file except in compliance with the License.
7
+ # You may obtain a copy of the License at
8
+ #
9
+ # http://www.apache.org/licenses/LICENSE-2.0
10
+ #
11
+ # Unless required by applicable law or agreed to in writing, software
12
+ # distributed under the License is distributed on an "AS IS" BASIS,
13
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14
+ # See the License for the specific language governing permissions and
15
+ # limitations under the License.
16
+ #
17
+
18
+ require "chef_apply/target_host"
19
+ require "chef_apply/error"
20
+
21
+ module ChefApply
22
+ class TargetResolver
23
+ MAX_EXPANDED_TARGETS = 24
24
+
25
+ def initialize(target, default_protocol, conn_options)
26
+ @default_proto = default_protocol
27
+ @unparsed_target = target
28
+ @split_targets = @unparsed_target.split(",")
29
+ @conn_options = conn_options.dup
30
+ @default_password = @conn_options.delete(:password)
31
+ @default_user = @conn_options.delete(:user)
32
+ end
33
+
34
+ # Returns the list of targets as an array of TargetHost instances,
35
+ # them to account for ranges embedded in the target name.
36
+ def targets
37
+ return @targets unless @targets.nil?
38
+ expanded_urls = []
39
+ @split_targets.each do |target|
40
+ expanded_urls = (expanded_urls | expand_targets(target))
41
+ end
42
+ @targets = expanded_urls.map do |url|
43
+ config = @conn_options.merge(config_for_target(url))
44
+ TargetHost.new(config.delete(:url), config)
45
+ end
46
+ end
47
+
48
+ def config_for_target(url)
49
+ prefix, target = prefix_from_target(url)
50
+
51
+ inline_password = nil
52
+ inline_user = nil
53
+ host = target
54
+ # Default greedy-scan of the regex means that
55
+ # $2 will resolve to content after the final "@"
56
+ # URL credentials will take precedence over the default :user
57
+ # in @conn_opts
58
+ if target =~ /(.*)@(.*)/
59
+ inline_credentials = $1
60
+ host = $2
61
+ # We'll use a non-greedy match to grab everthinmg up to the first ':'
62
+ # as username if there is no :, credentials is just the username
63
+ if inline_credentials =~ /(.+?):(.*)/
64
+ inline_user = $1
65
+ inline_password = $2
66
+ else
67
+ inline_user = inline_credentials
68
+ end
69
+ end
70
+ user, password = make_credentials(inline_user, inline_password)
71
+ { url: "#{prefix}#{host}",
72
+ user: user,
73
+ password: password }
74
+ end
75
+
76
+ # Merge the inline user/pass with the default user/pass, giving
77
+ # precedence to inline.
78
+ def make_credentials(inline_user, inline_password)
79
+ user = inline_user || @default_user
80
+ user = nil if user && user.empty?
81
+ password = (inline_password || @default_password)
82
+ password = nil if password && password.empty?
83
+ [user, password]
84
+ end
85
+
86
+ def prefix_from_target(target)
87
+ if target =~ /^(.+?):\/\/(.*)/
88
+ # We'll store the existing prefix to avoid it interfering
89
+ # with the check further below.
90
+ if ChefApply::Config::SUPPORTED_PROTOCOLS.include? $1.downcase
91
+ prefix = "#{$1}://"
92
+ target = $2
93
+ else
94
+ raise UnsupportedProtocol.new($1)
95
+ end
96
+ else
97
+ prefix = "#{@default_proto}://"
98
+ end
99
+ [prefix, target]
100
+ end
101
+
102
+ def expand_targets(target)
103
+ @current_target = target # Hold onto this for error reporting
104
+ do_parse([target.downcase])
105
+ end
106
+
107
+ private
108
+
109
+ # A string matching PREFIX[x:y]POSTFIX:
110
+ # POSTFIX can contain further ranges itself
111
+ # This uses a greedy match (.*) to get include every character
112
+ # up to the last "[" in PREFIX
113
+ # $1 - prefix; $2 - x, $3 - y, $4 unproccessed/remaining text
114
+ TARGET_WITH_RANGE = /^(.*)\[([\p{Alnum}]+):([\p{Alnum}]+)\](.*)/
115
+
116
+ def do_parse(targets, depth = 0)
117
+ raise TooManyRanges.new(@current_target) if depth > 2
118
+ new_targets = []
119
+ done = false
120
+ targets.each do |target|
121
+ if TARGET_WITH_RANGE =~ target
122
+ # $1 - prefix; $2 - x, $3 - y, $4 unprocessed/remaining text
123
+ expand_range(new_targets, $1, $2, $3, $4)
124
+ else
125
+ # Nothing more to expand
126
+ done = true
127
+ new_targets << target
128
+ end
129
+ end
130
+ if done
131
+ new_targets
132
+ else
133
+ do_parse(new_targets, depth + 1)
134
+ end
135
+ end
136
+
137
+ def expand_range(dest, prefix, start, stop, suffix)
138
+ prefix ||= ""
139
+ suffix ||= ""
140
+ start_is_int = Integer(start) >= 0 rescue false
141
+ stop_is_int = Integer(stop) >= 0 rescue false
142
+
143
+ if (start_is_int && !stop_is_int) || (stop_is_int && !start_is_int)
144
+ raise InvalidRange.new(@current_target, "[#{start}:#{stop}]")
145
+ end
146
+
147
+ # Ensure that a numeric range doesn't get created as a string, which
148
+ # would make the created Range further below fail to iterate for some values
149
+ # because of ASCII sorting.
150
+ if start_is_int
151
+ start = Integer(start)
152
+ end
153
+
154
+ if stop_is_int
155
+ stop = Integer(stop)
156
+ end
157
+
158
+ # For range to iterate correctly, the values must
159
+ # be low,high
160
+ if start > stop
161
+ temp = stop; stop = start; start = temp
162
+ end
163
+ Range.new(start, stop).each do |value|
164
+ # Ranges will resolve only numbers and letters,
165
+ # not other ascii characters that happen to fall between.
166
+ if start_is_int || /^[a-z0-9]/ =~ value
167
+ dest << "#{prefix}#{value}#{suffix}"
168
+ end
169
+ # Stop expanding as soon as we go over limit to prevent
170
+ # making the user wait for a massive accidental expansion
171
+ if dest.length > MAX_EXPANDED_TARGETS
172
+ raise TooManyTargets.new(@split_targets.length, MAX_EXPANDED_TARGETS)
173
+ end
174
+ end
175
+ end
176
+
177
+ class InvalidRange < ErrorNoLogs
178
+ def initialize(unresolved_target, given_range)
179
+ super("CHEFRANGE001", unresolved_target, given_range)
180
+ end
181
+ end
182
+
183
+ class TooManyRanges < ErrorNoLogs
184
+ def initialize(unresolved_target)
185
+ super("CHEFRANGE002", unresolved_target)
186
+ end
187
+ end
188
+
189
+ class TooManyTargets < ErrorNoLogs
190
+ def initialize(num_top_level_targets, max_targets)
191
+ super("CHEFRANGE003", num_top_level_targets, max_targets)
192
+ end
193
+ end
194
+
195
+ class UnsupportedProtocol < ErrorNoLogs
196
+ def initialize(attempted_protocol)
197
+ super("CHEFVAL011", attempted_protocol,
198
+ ChefApply::Config::SUPPORTED_PROTOCOLS.join(" "))
199
+ end
200
+ end
201
+ end
202
+ end
@@ -0,0 +1,162 @@
1
+ #
2
+ # Copyright:: Copyright (c) 2018 Chef Software Inc.
3
+ # License:: Apache License, Version 2.0
4
+ #
5
+ # Licensed under the Apache License, Version 2.0 (the "License");
6
+ # you may not use this file except in compliance with the License.
7
+ # You may obtain a copy of the License at
8
+ #
9
+ # http://www.apache.org/licenses/LICENSE-2.0
10
+ #
11
+ # Unless required by applicable law or agreed to in writing, software
12
+ # distributed under the License is distributed on an "AS IS" BASIS,
13
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14
+ # See the License for the specific language governing permissions and
15
+ # limitations under the License.
16
+ #
17
+
18
+ require "benchmark"
19
+ require "forwardable"
20
+ require "singleton"
21
+ require "json"
22
+ require "digest/sha1"
23
+ require "securerandom"
24
+ require "chef_apply/version"
25
+ require "chef_apply/config"
26
+ require "yaml"
27
+
28
+ module ChefApply
29
+
30
+ # This definites the Telemeter interface. Implementation thoughts for
31
+ # when we unstub it:
32
+ # - let's track the call sequence; most of our calls will be nested inside
33
+ # a main 'timed_capture', and it would be good to see ordering within nested calls.
34
+ class Telemeter
35
+ include Singleton
36
+ DEFAULT_INSTALLATION_GUID = "00000000-0000-0000-0000-000000000000"
37
+
38
+ class << self
39
+ extend Forwardable
40
+ def_delegators :instance, :timed_capture, :capture, :commit, :timed_action_capture, :timed_run_capture
41
+ def_delegators :instance, :pending_event_count, :last_event, :enabled?
42
+ def_delegators :instance, :make_event_payload
43
+ end
44
+
45
+ attr_reader :events_to_send, :run_timestamp
46
+
47
+ def enabled?
48
+ require "telemetry/decision"
49
+ ChefApply::Config.telemetry.enable && !Telemetry::Decision.env_opt_out?
50
+ end
51
+
52
+ def initialize
53
+ @events_to_send = []
54
+ @run_timestamp = Time.now.utc.strftime("%FT%TZ")
55
+ end
56
+
57
+ def timed_action_capture(action, &block)
58
+ # Note: we do not directly capture hostname for privacy concerns, but
59
+ # using a sha1 digest will allow us to anonymously see
60
+ # unique hosts to derive number of hosts affected by a command
61
+ target = action.target_host
62
+ target_data = { platform: {}, hostname_sha1: nil, transport_type: nil }
63
+ if target
64
+ target_data[:platform][:name] = target.base_os # :windows, :linux, eventually :macos
65
+ target_data[:platform][:version] = target.version
66
+ target_data[:platform][:architecture] = target.architecture
67
+ target_data[:hostname_sha1] = Digest::SHA1.hexdigest(target.hostname.downcase)
68
+ target_data[:transport_type] = target.transport_type
69
+ end
70
+ timed_capture(:action, { action: action.name, target: target_data }, &block)
71
+ end
72
+
73
+ def timed_run_capture(arguments, &block)
74
+ timed_capture(:run, arguments: arguments, &block)
75
+ end
76
+
77
+ def capture(name, data = {})
78
+ # Adding it to the head of the list will ensure that the
79
+ # sequence of events is preserved when we send the final payload
80
+ payload = make_event_payload(name, data)
81
+ @events_to_send.unshift payload
82
+ end
83
+
84
+ def timed_capture(name, data = {})
85
+ time = Benchmark.measure { yield }
86
+ data[:duration] = time.real
87
+ capture(name, data)
88
+ end
89
+
90
+ def commit
91
+ if enabled?
92
+ session = convert_events_to_session
93
+ write_session(session)
94
+ end
95
+ @events_to_send = []
96
+ end
97
+
98
+ def make_event_payload(name, data)
99
+ {
100
+ event: name,
101
+ properties: {
102
+ installation_id: installation_id,
103
+ run_timestamp: run_timestamp,
104
+ host_platform: host_platform,
105
+ event_data: data
106
+ }
107
+ }
108
+ end
109
+
110
+ def installation_id
111
+ @installation_id ||=
112
+ begin
113
+ File.read(ChefApply::Config.telemetry_installation_identifier_file).chomp
114
+ rescue
115
+ ChefApply::Log.info "could not read #{ChefApply::Config.telemetry_installation_identifier_file} - using default id"
116
+ DEFAULT_INSTALLATION_GUID
117
+ end
118
+ end
119
+
120
+ # For testing.
121
+ def pending_event_count
122
+ @events_to_send.length
123
+ end
124
+
125
+ def last_event
126
+ @events_to_send.last
127
+ end
128
+
129
+ private
130
+
131
+ def host_platform
132
+ @host_platform ||= case RUBY_PLATFORM
133
+ when /mswin|mingw|windows/
134
+ "windows"
135
+ else
136
+ RUBY_PLATFORM.split("-")[1]
137
+ end
138
+ end
139
+
140
+ def convert_events_to_session
141
+ YAML.dump({ "version" => ChefApply::VERSION,
142
+ "entries" => @events_to_send })
143
+ end
144
+
145
+ def write_session(session)
146
+ File.write(next_filename, convert_events_to_session)
147
+ end
148
+
149
+ def next_filename
150
+ id = 0
151
+ filename = ""
152
+ loop do
153
+ id += 1
154
+ filename = File.join(ChefApply::Config.telemetry_path,
155
+ "telemetry-payload-#{id}.yml")
156
+ break unless File.exist?(filename)
157
+ end
158
+ filename
159
+ end
160
+
161
+ end
162
+ end
@@ -0,0 +1,32 @@
1
+ #
2
+ # Copyright:: Copyright (c) 2018 Chef Software Inc.
3
+ # License:: Apache License, Version 2.0
4
+ #
5
+ # Licensed under the Apache License, Version 2.0 (the "License");
6
+ # you may not use this file except in compliance with the License.
7
+ # You may obtain a copy of the License at
8
+ #
9
+ # http://www.apache.org/licenses/LICENSE-2.0
10
+ #
11
+ # Unless required by applicable law or agreed to in writing, software
12
+ # distributed under the License is distributed on an "AS IS" BASIS,
13
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14
+ # See the License for the specific language governing permissions and
15
+ # limitations under the License.
16
+ #
17
+
18
+ class Telemetry
19
+ class Session
20
+ # The telemetry session data is normally kept in .chef, which we don't have.
21
+ def session_file
22
+ ChefApply::Config.telemetry_session_file.freeze
23
+ end
24
+ end
25
+
26
+ def deliver(data = {})
27
+ if ChefApply::Telemeter.instance.enabled?
28
+ payload = event.prepare(data)
29
+ client.await.fire(payload)
30
+ end
31
+ end
32
+ end
@@ -0,0 +1,121 @@
1
+ #
2
+ # Copyright:: Copyright (c) 2018 Chef Software Inc.
3
+ # License:: Apache License, Version 2.0
4
+ #
5
+ # Licensed under the Apache License, Version 2.0 (the "License");
6
+ # you may not use this file except in compliance with the License.
7
+ # You may obtain a copy of the License at
8
+ #
9
+ # http://www.apache.org/licenses/LICENSE-2.0
10
+ #
11
+ # Unless required by applicable law or agreed to in writing, software
12
+ # distributed under the License is distributed on an "AS IS" BASIS,
13
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14
+ # See the License for the specific language governing permissions and
15
+ # limitations under the License.
16
+ #
17
+
18
+ require "telemetry"
19
+ require "chef_apply/telemeter"
20
+ require "chef_apply/telemeter/patch"
21
+ require "chef_apply/log"
22
+ require "chef_apply/version"
23
+
24
+ module ChefApply
25
+ class Telemeter
26
+ class Sender
27
+ attr_reader :session_files
28
+
29
+ def self.start_upload_thread
30
+ # Find the files before we spawn the thread - otherwise
31
+ # we may accidentally pick up the current run's session file if it
32
+ # finishes before the thread scans for new files
33
+ session_files = Sender.find_session_files
34
+ sender = Sender.new(session_files)
35
+ Thread.new { sender.run }
36
+ end
37
+
38
+ def self.find_session_files
39
+ ChefApply::Log.info("Looking for telemetry data to submit")
40
+ session_search = File.join(ChefApply::Config.telemetry_path, "telemetry-payload-*.yml")
41
+ session_files = Dir.glob(session_search)
42
+ ChefApply::Log.info("Found #{session_files.length} sessions to submit")
43
+ session_files
44
+ end
45
+
46
+ def initialize(session_files)
47
+ @session_files = session_files
48
+ end
49
+
50
+ def run
51
+ if ChefApply::Telemeter.enabled?
52
+ ChefApply::Log.info("Telemetry enabled, beginning upload of previous session(s)")
53
+ # dev mode telemetry gets sent to a different location
54
+ if ChefApply::Config.telemetry.dev
55
+ ENV["CHEF_TELEMETRY_ENDPOINT"] ||= "https://telemetry-acceptance.chef.io"
56
+ end
57
+ session_files.each { |path| process_session(path) }
58
+ else
59
+ # If telemetry is not enabled, just clean up and return. Even though
60
+ # the telemetry gem will not send if disabled, log output saying that we're submitting
61
+ # it when it has been disabled can be alarming.
62
+ ChefApply::Log.info("Telemetry disabled, clearing any existing session captures without sending them.")
63
+ session_files.each { |path| FileUtils.rm_rf(path) }
64
+ end
65
+ FileUtils.rm_rf(ChefApply::Config.telemetry_session_file)
66
+ ChefApply::Log.info("Terminating, nothing more to do.")
67
+ rescue => e
68
+ ChefApply::Log.fatal "Sender thread aborted: '#{e}' failed at #{e.backtrace[0]}"
69
+ end
70
+
71
+ def process_session(path)
72
+ ChefApply::Log.info("Processing telemetry entries from #{path}")
73
+ content = load_and_clear_session(path)
74
+ submit_session(content)
75
+ end
76
+
77
+ def submit_session(content)
78
+ # Each file contains the actions taken within a single run of the chef tool.
79
+ # Each run is one session, so we'll first remove remove the session file
80
+ # to force creating a new one.
81
+ FileUtils.rm_rf(ChefApply::Config.telemetry_session_file)
82
+ # We'll use the version captured in the sesion file
83
+ entries = content["entries"]
84
+ cli_version = content["version"]
85
+ total = entries.length
86
+ telemetry = Telemetry.new(product: "chef-workstation",
87
+ origin: "command-line",
88
+ product_version: cli_version,
89
+ install_context: "omnibus")
90
+ total = entries.length
91
+ entries.each_with_index do |entry, x|
92
+ submit_entry(telemetry, entry, x + 1, total)
93
+ end
94
+ end
95
+
96
+ def submit_entry(telemetry, entry, sequence, total)
97
+ ChefApply::Log.info("Submitting telemetry entry #{sequence}/#{total}: #{entry} ")
98
+ telemetry.deliver(entry)
99
+ ChefApply::Log.info("Entry #{sequence}/#{total} submitted.")
100
+ rescue => e
101
+ # No error handling in telemetry lib, so at least track the failrue
102
+ ChefApply::Log.error("Failed to send entry #{sequence}/#{total}: #{e}")
103
+ ChefApply::Log.error("Backtrace: #{e.backtrace} ")
104
+ end
105
+
106
+ private
107
+
108
+ def load_and_clear_session(path)
109
+ content = File.read(path)
110
+ # We'll remove it now instead of after we parse or submit it -
111
+ # if we fail to deliver, we don't want to be stuck resubmitting it if the problem
112
+ # was due to payload. This is a trade-off - if we get a transient error, the
113
+ # payload will be lost.
114
+ # TODO: Improve error handling so we can intelligently decide whether to
115
+ # retry a failed load or failed submit.
116
+ FileUtils.rm_rf(path)
117
+ YAML.load(content)
118
+ end
119
+ end
120
+ end
121
+ end