knife-essentials 0.9.8 → 1.0.0.beta1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -81,13 +81,7 @@ module ChefFS
81
81
  EnvironmentsDir.new(self),
82
82
  RestListDir.new("roles", self, nil, ChefFS::DataHandler::RoleDataHandler.new)
83
83
  ]
84
- if repo_mode == 'everything'
85
- result += [
86
- RestListDir.new("clients", self, nil, ChefFS::DataHandler::ClientDataHandler.new),
87
- NodesDir.new(self),
88
- RestListDir.new("users", self, nil, ChefFS::DataHandler::UserDataHandler.new)
89
- ]
90
- elsif repo_mode == 'hosted_everything'
84
+ if repo_mode == 'hosted_everything'
91
85
  result += [
92
86
  AclsDir.new(self),
93
87
  RestListDir.new("clients", self, nil, ChefFS::DataHandler::ClientDataHandler.new),
@@ -95,6 +89,12 @@ module ChefFS
95
89
  RestListDir.new("groups", self, nil, ChefFS::DataHandler::GroupDataHandler.new),
96
90
  NodesDir.new(self)
97
91
  ]
92
+ elsif repo_mode != 'static'
93
+ result += [
94
+ RestListDir.new("clients", self, nil, ChefFS::DataHandler::ClientDataHandler.new),
95
+ NodesDir.new(self),
96
+ RestListDir.new("users", self, nil, ChefFS::DataHandler::UserDataHandler.new)
97
+ ]
98
98
  end
99
99
  result.sort_by { |child| child.name }
100
100
  end
@@ -150,7 +150,7 @@ module ChefFS
150
150
  return [ !exists?, nil, nil ]
151
151
  end
152
152
  are_same = true
153
- ChefFS::CommandLine::diff_entries(self, other, nil, :name_only) do |type, old_entry, new_entry|
153
+ ChefFS::CommandLine::diff_entries(self, other, nil, :name_only).each do |type, old_entry, new_entry|
154
154
  if [ :directory_to_file, :file_to_directory, :deleted, :added, :modified ].include?(type)
155
155
  are_same = false
156
156
  end
@@ -105,6 +105,8 @@ module ChefFS
105
105
  end
106
106
 
107
107
  def compare_to(other)
108
+ # TODO this pair of reads can be parallelized
109
+
108
110
  # Grab the other value
109
111
  begin
110
112
  other_value_json = other.read
@@ -20,6 +20,7 @@ require 'chef_fs/file_system/chef_server_root_dir'
20
20
  require 'chef_fs/file_system/chef_repository_file_system_root_dir'
21
21
  require 'chef_fs/file_pattern'
22
22
  require 'chef_fs/path_utils'
23
+ require 'chef_fs/parallelizer'
23
24
  require 'chef/config'
24
25
 
25
26
  module ChefFS
@@ -27,16 +28,21 @@ module ChefFS
27
28
  def self.common_options
28
29
  option :repo_mode,
29
30
  :long => '--repo-mode MODE',
30
- :description => "Specifies the local repository layout. Values: default, everything, hosted_everything"
31
+ :description => "Specifies the local repository layout. Values: static, everything, hosted_everything. Default: everything/hosted_everything"
31
32
 
32
33
  option :chef_repo_path,
33
34
  :long => '--chef-repo-path PATH',
34
35
  :description => 'Overrides the location of chef repo. Default is specified by chef_repo_path in the config'
36
+
37
+ option :concurrency,
38
+ :long => '--concurrency THREADS',
39
+ :description => 'Maximum number of simultaneous requests to send (default: 10)'
35
40
  end
36
41
 
37
42
  def configure_chef
38
43
  super
39
44
  Chef::Config[:repo_mode] = config[:repo_mode] if config[:repo_mode]
45
+ Chef::Config[:concurrency] = config[:concurrency].to_i if config[:concurrency]
40
46
 
41
47
  # --chef-repo-path overrides all other paths
42
48
  path_variables = %w(acl_path client_path cookbook_path container_path data_bag_path environment_path group_path node_path role_path user_path)
@@ -65,6 +71,15 @@ module ChefFS
65
71
  Chef::Config[:role_path] = nil
66
72
  end
67
73
 
74
+ # Default to getting *everything* from the server.
75
+ if !Chef::Config[:repo_mode]
76
+ if Chef::Config[:chef_server_url] =~ /^[^\/]+\/+organizations\//
77
+ Chef::Config[:repo_mode] = 'hosted_everything'
78
+ else
79
+ Chef::Config[:repo_mode] = 'everything'
80
+ end
81
+ end
82
+
68
83
  # Infer any *_path variables that are not specified
69
84
  if Chef::Config[:chef_repo_path]
70
85
  path_variables.each do |variable_name|
@@ -76,6 +91,8 @@ module ChefFS
76
91
  end
77
92
  end
78
93
  end
94
+
95
+ ChefFS::Parallelizer.threads = (Chef::Config[:concurrency] || 10) - 1
79
96
  end
80
97
 
81
98
  def chef_fs
@@ -91,12 +108,12 @@ module ChefFS
91
108
 
92
109
  result = {}
93
110
  case Chef::Config[:repo_mode]
94
- when 'everything'
95
- object_names = %w(clients cookbooks data_bags environments nodes roles users)
111
+ when 'static'
112
+ object_names = %w(cookbooks data_bags environments roles)
96
113
  when 'hosted_everything'
97
114
  object_names = %w(acls clients cookbooks containers data_bags environments groups nodes roles)
98
115
  else
99
- object_names = %w(cookbooks data_bags environments roles)
116
+ object_names = %w(clients cookbooks data_bags environments nodes roles users)
100
117
  end
101
118
  object_names.each do |object_name|
102
119
  variable_name = "#{object_name[0..-2]}_path" # cookbooks -> cookbook_path
@@ -195,5 +212,8 @@ module ChefFS
195
212
  end
196
213
  end
197
214
 
215
+ def parallelize(inputs, options = {}, &block)
216
+ ChefFS::Parallelizer.parallelize(inputs, options, &block)
217
+ end
198
218
  end
199
219
  end
@@ -0,0 +1,127 @@
1
+ module ChefFS
2
+ class Parallelizer
3
+ @@parallelizer = nil
4
+ @@threads = 0
5
+
6
+ def self.threads=(value)
7
+ if @@threads != value
8
+ @@threads = value
9
+ @@parallelizer = nil
10
+ end
11
+ end
12
+
13
+ def self.parallelize(enumerator, options = {}, &block)
14
+ @@parallelizer ||= Parallelizer.new(@@threads)
15
+ @@parallelizer.parallelize(enumerator, options, &block)
16
+ end
17
+
18
+ def initialize(threads)
19
+ @tasks_mutex = Mutex.new
20
+ @tasks = []
21
+ @threads = []
22
+ 1.upto(threads) do
23
+ @threads << Thread.new { worker_loop }
24
+ end
25
+ end
26
+
27
+ def parallelize(enumerator, options = {}, &block)
28
+ task = ParallelizedResults.new(enumerator, options, &block)
29
+ @tasks_mutex.synchronize do
30
+ @tasks << task
31
+ end
32
+ task
33
+ end
34
+
35
+ class ParallelizedResults
36
+ include Enumerable
37
+
38
+ def initialize(enumerator, options, &block)
39
+ @inputs = enumerator.to_a
40
+ @options = options
41
+ @block = block
42
+
43
+ @mutex = Mutex.new
44
+ @outputs = []
45
+ @status = []
46
+ end
47
+
48
+ def each
49
+ next_index = 0
50
+ while true
51
+ # Report any results that already exist
52
+ while @status.length > next_index && ([:finished, :exception].include?(@status[next_index]))
53
+ if @status[next_index] == :finished
54
+ if @options[:flatten]
55
+ @outputs[next_index].each do |entry|
56
+ yield entry
57
+ end
58
+ else
59
+ yield @outputs[next_index]
60
+ end
61
+ else
62
+ raise @outputs[next_index]
63
+ end
64
+ next_index = next_index + 1
65
+ end
66
+
67
+ # Pick up a result and process it, if there is one. This ensures we
68
+ # move forward even if there are *zero* worker threads available.
69
+ if !process_input
70
+ # Exit if we're done.
71
+ if next_index >= @status.length
72
+ break
73
+ else
74
+ # Ruby 1.8 threading sucks. Wait till we process more things.
75
+ sleep(0.05)
76
+ end
77
+ end
78
+ end
79
+ end
80
+
81
+ def process_input
82
+ # Grab the next one to process
83
+ index, input = @mutex.synchronize do
84
+ index = @status.length
85
+ if index >= @inputs.length
86
+ return nil
87
+ end
88
+ input = @inputs[index]
89
+ @status[index] = :started
90
+ [ index, input ]
91
+ end
92
+
93
+ begin
94
+ @outputs[index] = @block.call(input)
95
+ @status[index] = :finished
96
+ rescue
97
+ @outputs[index] = $!
98
+ @status[index] = :exception
99
+ end
100
+ index
101
+ end
102
+ end
103
+
104
+ private
105
+
106
+ def worker_loop
107
+ while true
108
+ begin
109
+ task = @tasks[0]
110
+ if task
111
+ if !task.process_input
112
+ @tasks_mutex.synchronize do
113
+ @tasks.delete(task)
114
+ end
115
+ end
116
+ else
117
+ # Ruby 1.8 threading sucks. Wait a bit to see if another task comes in.
118
+ sleep(0.05)
119
+ end
120
+ rescue
121
+ puts "ERROR #{$!}"
122
+ puts $!.backtrace
123
+ end
124
+ end
125
+ end
126
+ end
127
+ end
@@ -23,10 +23,15 @@ module ChefFS
23
23
  response_body = chef_rest.decompress_body(response)
24
24
 
25
25
  if response.kind_of?(Net::HTTPSuccess)
26
- response_body
26
+ response_body
27
27
  elsif redirect_location = redirected_to(response)
28
- raise "Redirected to #{create_url(redirect_location)}"
29
- follow_redirect {api_request(:GET, create_url(redirect_location))}
28
+ if [:GET, :HEAD].include?(method)
29
+ chef_rest.follow_redirect do
30
+ api_request(chef_rest, method, chef_rest.create_url(redirect_location))
31
+ end
32
+ else
33
+ raise Exceptions::InvalidRedirect, "#{method} request was redirected from #{url} to #{redirect_location}. Only GET and HEAD support redirects."
34
+ end
30
35
  else
31
36
  # have to decompress the body before making an exception for it. But the body could be nil.
32
37
  response.body.replace(chef_rest.decompress_body(response)) if response.body.respond_to?(:replace)
@@ -57,7 +62,6 @@ module ChefFS
57
62
  response['location']
58
63
  end
59
64
 
60
-
61
65
  def self.build_headers(chef_rest, method, url, headers={}, json_body=false, raw=false)
62
66
  # headers = @default_headers.merge(headers)
63
67
  #headers['Accept'] = "application/json" unless raw
@@ -1,3 +1,3 @@
1
1
  module ChefFS
2
- VERSION = "0.9.8"
2
+ VERSION = "1.0.0.beta1"
3
3
  end
@@ -17,7 +17,7 @@ describe 'chef_repo_path tests' do
17
17
  file 'roles/role1.json', {}
18
18
  file 'users/user1.json', {}
19
19
 
20
- file 'clients2/client1.json', {}
20
+ file 'clients2/client2.json', {}
21
21
  file 'cookbooks2/cookbook2/metadata.rb', ''
22
22
  file 'data_bags2/bag2/item2.json', {}
23
23
  file 'environments2/env2.json', {}
@@ -61,6 +61,8 @@ describe 'chef_repo_path tests' do
61
61
  cwd 'chef_repo2'
62
62
  it 'knife list --local -Rfp lists everything' do
63
63
  knife('list --local -Rfp').should_succeed <<EOM
64
+ clients/
65
+ clients/client2.json
64
66
  cookbooks/
65
67
  cookbooks/cookbook2/
66
68
  cookbooks/cookbook2/metadata.rb
@@ -69,8 +71,12 @@ data_bags/bag2/
69
71
  data_bags/bag2/item2.json
70
72
  environments/
71
73
  environments/env2.json
74
+ nodes/
75
+ nodes/node2.json
72
76
  roles/
73
77
  roles/role2.json
78
+ users/
79
+ users/user2.json
74
80
  EOM
75
81
  end
76
82
  end
@@ -100,6 +106,8 @@ EOM
100
106
  cwd '.'
101
107
  it 'knife list --local -Rfp lists everything' do
102
108
  knife('list --local -Rfp').should_succeed <<EOM
109
+ clients/
110
+ clients/client2.json
103
111
  cookbooks/
104
112
  cookbooks/cookbook2/
105
113
  cookbooks/cookbook2/metadata.rb
@@ -108,8 +116,12 @@ data_bags/bag2/
108
116
  data_bags/bag2/item2.json
109
117
  environments/
110
118
  environments/env2.json
119
+ nodes/
120
+ nodes/node2.json
111
121
  roles/
112
122
  roles/role2.json
123
+ users/
124
+ users/user2.json
113
125
  EOM
114
126
  end
115
127
  end
@@ -165,6 +177,8 @@ EOM
165
177
  cwd 'chef_repo2'
166
178
  it 'knife list --local -Rfp lists everything' do
167
179
  knife('list --local -Rfp').should_succeed <<EOM
180
+ clients/
181
+ clients/client3.json
168
182
  cookbooks/
169
183
  cookbooks/cookbook3/
170
184
  cookbooks/cookbook3/metadata.rb
@@ -173,8 +187,12 @@ data_bags/bag3/
173
187
  data_bags/bag3/item3.json
174
188
  environments/
175
189
  environments/env3.json
190
+ nodes/
191
+ nodes/node3.json
176
192
  roles/
177
193
  roles/role3.json
194
+ users/
195
+ users/user3.json
178
196
  EOM
179
197
  end
180
198
  end
@@ -351,6 +369,9 @@ EOM
351
369
  cwd 'chef_repo2'
352
370
  it 'knife list --local -Rfp lists everything' do
353
371
  knife('list --local -Rfp').should_succeed <<EOM
372
+ clients/
373
+ clients/client1.json
374
+ clients/client2.json
354
375
  cookbooks/
355
376
  cookbooks/cookbook1/
356
377
  cookbooks/cookbook1/metadata.rb
@@ -364,9 +385,15 @@ data_bags/bag2/item2.json
364
385
  environments/
365
386
  environments/env1.json
366
387
  environments/env2.json
388
+ nodes/
389
+ nodes/node1.json
390
+ nodes/node2.json
367
391
  roles/
368
392
  roles/role1.json
369
393
  roles/role2.json
394
+ users/
395
+ users/user1.json
396
+ users/user2.json
370
397
  EOM
371
398
  end
372
399
  end
@@ -399,6 +426,9 @@ EOM
399
426
  cwd '.'
400
427
  it 'knife list --local -Rfp lists everything' do
401
428
  knife('list --local -Rfp').should_succeed <<EOM
429
+ clients/
430
+ clients/client1.json
431
+ clients/client3.json
402
432
  cookbooks/
403
433
  cookbooks/cookbook1/
404
434
  cookbooks/cookbook1/metadata.rb
@@ -412,9 +442,15 @@ data_bags/bag3/item3.json
412
442
  environments/
413
443
  environments/env1.json
414
444
  environments/env3.json
445
+ nodes/
446
+ nodes/node1.json
447
+ nodes/node3.json
415
448
  roles/
416
449
  roles/role1.json
417
450
  roles/role3.json
451
+ users/
452
+ users/user1.json
453
+ users/user3.json
418
454
  EOM
419
455
  end
420
456
  end
@@ -435,6 +471,9 @@ EOM
435
471
  cwd 'chef_repo2'
436
472
  it 'knife list --local -Rfp lists everything' do
437
473
  knife('list --local -Rfp').should_succeed <<EOM
474
+ clients/
475
+ clients/client1.json
476
+ clients/client3.json
438
477
  cookbooks/
439
478
  cookbooks/cookbook1/
440
479
  cookbooks/cookbook1/metadata.rb
@@ -448,9 +487,15 @@ data_bags/bag3/item3.json
448
487
  environments/
449
488
  environments/env1.json
450
489
  environments/env3.json
490
+ nodes/
491
+ nodes/node1.json
492
+ nodes/node3.json
451
493
  roles/
452
494
  roles/role1.json
453
495
  roles/role3.json
496
+ users/
497
+ users/user1.json
498
+ users/user3.json
454
499
  EOM
455
500
  end
456
501
  end
@@ -495,6 +540,8 @@ EOM
495
540
  cwd 'chef_repo2'
496
541
  it 'knife list --local -Rfp lists everything' do
497
542
  knife('list --local -Rfp').should_succeed <<EOM
543
+ clients/
544
+ clients/client3.json
498
545
  cookbooks/
499
546
  cookbooks/cookbook3/
500
547
  cookbooks/cookbook3/metadata.rb
@@ -503,8 +550,12 @@ data_bags/bag3/
503
550
  data_bags/bag3/item3.json
504
551
  environments/
505
552
  environments/env3.json
553
+ nodes/
554
+ nodes/node3.json
506
555
  roles/
507
556
  roles/role3.json
557
+ users/
558
+ users/user3.json
508
559
  EOM
509
560
  end
510
561
  end
@@ -536,6 +587,9 @@ EOM
536
587
  cwd '.'
537
588
  it 'knife list --local -Rfp lists everything' do
538
589
  knife('list --local -Rfp').should_succeed <<EOM
590
+ clients/
591
+ clients/client1.json
592
+ clients/client3.json
539
593
  cookbooks/
540
594
  cookbooks/cookbook1/
541
595
  cookbooks/cookbook1/metadata.rb
@@ -549,9 +603,15 @@ data_bags/bag3/item3.json
549
603
  environments/
550
604
  environments/env1.json
551
605
  environments/env3.json
606
+ nodes/
607
+ nodes/node1.json
608
+ nodes/node3.json
552
609
  roles/
553
610
  roles/role1.json
554
611
  roles/role3.json
612
+ users/
613
+ users/user1.json
614
+ users/user3.json
555
615
  EOM
556
616
  end
557
617
  end
@@ -572,6 +632,9 @@ EOM
572
632
  cwd 'chef_repo2'
573
633
  it 'knife list --local -Rfp lists everything' do
574
634
  knife('list --local -Rfp').should_succeed <<EOM
635
+ clients/
636
+ clients/client1.json
637
+ clients/client3.json
575
638
  cookbooks/
576
639
  cookbooks/cookbook1/
577
640
  cookbooks/cookbook1/metadata.rb
@@ -585,9 +648,15 @@ data_bags/bag3/item3.json
585
648
  environments/
586
649
  environments/env1.json
587
650
  environments/env3.json
651
+ nodes/
652
+ nodes/node1.json
653
+ nodes/node3.json
588
654
  roles/
589
655
  roles/role1.json
590
656
  roles/role3.json
657
+ users/
658
+ users/user1.json
659
+ users/user3.json
591
660
  EOM
592
661
  end
593
662
  end
@@ -635,6 +704,8 @@ EOM
635
704
  cwd 'chef_repo2'
636
705
  it 'knife list --local -Rfp lists everything' do
637
706
  knife('list --local -Rfp').should_succeed <<EOM
707
+ clients/
708
+ clients/client3.json
638
709
  cookbooks/
639
710
  cookbooks/cookbook3/
640
711
  cookbooks/cookbook3/metadata.rb
@@ -643,8 +714,12 @@ data_bags/bag/
643
714
  data_bags/bag/item.json
644
715
  environments/
645
716
  environments/env3.json
717
+ nodes/
718
+ nodes/node3.json
646
719
  roles/
647
720
  roles/role3.json
721
+ users/
722
+ users/user3.json
648
723
  EOM
649
724
  end
650
725
  end