puppet 8.6.0 → 8.7.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (72) hide show
  1. checksums.yaml +4 -4
  2. data/Gemfile +6 -2
  3. data/Gemfile.lock +42 -38
  4. data/Rakefile +45 -22
  5. data/examples/hiera/README.md +68 -57
  6. data/examples/hiera/data/common.yaml +12 -0
  7. data/examples/hiera/data/dc1.yaml +6 -0
  8. data/examples/hiera/hiera.yaml +15 -0
  9. data/examples/hiera/modules/ntp/data/common.yaml +4 -0
  10. data/examples/hiera/modules/ntp/hiera.yaml +9 -0
  11. data/examples/hiera/modules/ntp/manifests/config.pp +16 -4
  12. data/examples/hiera/modules/ntp/templates/ntp.conf.epp +3 -0
  13. data/examples/hiera/modules/users/manifests/common.pp +7 -2
  14. data/examples/hiera/modules/users/manifests/dc1.pp +7 -2
  15. data/examples/hiera/site.pp +1 -1
  16. data/ext/project_data.yaml +0 -45
  17. data/lib/puppet/daemon.rb +1 -0
  18. data/lib/puppet/pops/loader/static_loader.rb +2 -2
  19. data/lib/puppet/pops/lookup/module_data_provider.rb +9 -9
  20. data/lib/puppet/provider/aix_object.rb +1 -1
  21. data/lib/puppet/provider/group/groupadd.rb +30 -9
  22. data/lib/puppet/provider/package/xbps.rb +127 -0
  23. data/lib/puppet/scheduler/splay_job.rb +9 -0
  24. data/lib/puppet/type/exec.rb +8 -0
  25. data/lib/puppet/util/command_line/trollop.rb +20 -2
  26. data/lib/puppet/util/rpm_compare.rb +1 -1
  27. data/lib/puppet/util/windows/com.rb +2 -2
  28. data/lib/puppet/version.rb +1 -1
  29. data/locales/puppet.pot +604 -600
  30. data/man/man5/puppet.conf.5 +2 -2
  31. data/man/man8/puppet-agent.8 +1 -1
  32. data/man/man8/puppet-apply.8 +1 -1
  33. data/man/man8/puppet-catalog.8 +1 -1
  34. data/man/man8/puppet-config.8 +1 -1
  35. data/man/man8/puppet-describe.8 +1 -1
  36. data/man/man8/puppet-device.8 +1 -1
  37. data/man/man8/puppet-doc.8 +1 -1
  38. data/man/man8/puppet-epp.8 +1 -1
  39. data/man/man8/puppet-facts.8 +1 -1
  40. data/man/man8/puppet-filebucket.8 +1 -1
  41. data/man/man8/puppet-generate.8 +1 -1
  42. data/man/man8/puppet-help.8 +1 -1
  43. data/man/man8/puppet-lookup.8 +1 -1
  44. data/man/man8/puppet-module.8 +1 -1
  45. data/man/man8/puppet-node.8 +1 -1
  46. data/man/man8/puppet-parser.8 +1 -1
  47. data/man/man8/puppet-plugin.8 +1 -1
  48. data/man/man8/puppet-report.8 +1 -1
  49. data/man/man8/puppet-resource.8 +1 -1
  50. data/man/man8/puppet-script.8 +1 -1
  51. data/man/man8/puppet-ssl.8 +1 -1
  52. data/man/man8/puppet.8 +2 -2
  53. metadata +13 -23
  54. data/examples/hiera/etc/hiera.yaml +0 -15
  55. data/examples/hiera/etc/hieradb/common.yaml +0 -3
  56. data/examples/hiera/etc/hieradb/dc1.yaml +0 -6
  57. data/examples/hiera/etc/hieradb/development.yaml +0 -2
  58. data/examples/hiera/etc/puppet.conf +0 -3
  59. data/examples/hiera/modules/data/manifests/common.pp +0 -4
  60. data/examples/hiera/modules/ntp/manifests/data.pp +0 -4
  61. data/examples/hiera/modules/ntp/templates/ntp.conf.erb +0 -3
  62. data/examples/hiera/modules/users/manifests/development.pp +0 -4
  63. data/tasks/benchmark.rake +0 -180
  64. data/tasks/cfpropertylist.rake +0 -15
  65. data/tasks/ci.rake +0 -24
  66. data/tasks/generate_ast_model.rake +0 -90
  67. data/tasks/generate_cert_fixtures.rake +0 -199
  68. data/tasks/manpages.rake +0 -67
  69. data/tasks/memwalk.rake +0 -195
  70. data/tasks/parallel.rake +0 -410
  71. data/tasks/parser.rake +0 -22
  72. data/tasks/yard.rake +0 -59
@@ -1,90 +0,0 @@
1
- begin
2
- require 'puppet'
3
- rescue LoadError
4
- #nothing to see here
5
- else
6
- desc "Generate the Pcore model that represents the AST for the Puppet Language"
7
- task :gen_pcore_ast do
8
- Puppet::Pops.generate_ast
9
- end
10
-
11
- module Puppet::Pops
12
- def self.generate_ast
13
- Puppet.initialize_settings
14
- env = Puppet.lookup(:environments).get(Puppet[:environment])
15
- loaders = Loaders.new(env)
16
- ast_pp = Pathname(__FILE__).parent.parent + 'lib/puppet/pops/model/ast.pp'
17
- Puppet.override(:current_environment => env, :loaders => loaders) do
18
- ast_factory = Parser::Parser.new.parse_file(ast_pp.expand_path.to_s)
19
- ast_model = Types::TypeParser.singleton.interpret(
20
- ast_factory.model.body, Loader::PredefinedLoader.new(loaders.find_loader(nil), 'TypeSet loader'))
21
-
22
- ruby = Types::RubyGenerator.new.module_definition_from_typeset(ast_model)
23
-
24
- # Replace ref() constructs to known Pcore types with directly initialized types. ref() cannot be used
25
- # since it requires a parser (chicken-and-egg problem)
26
- ruby.gsub!(/^module Parser\nmodule Locator\n.*\nend\nend\nmodule Model\n/m, "module Model\n")
27
-
28
- # Remove generated RubyMethod annotations. The ruby methods are there now, no need to also have
29
- # the annotations present.
30
- ruby.gsub!(/^\s+'annotations' => \{\n\s+ref\('RubyMethod'\) => \{\n.*\n\s+\}\n\s+\},\n/, '')
31
-
32
- ruby.gsub!(/ref\('([A-Za-z]+)'\)/, 'Types::P\1Type::DEFAULT')
33
- ruby.gsub!(/ref\('Optional\[([0-9A-Za-z_]+)\]'\)/, 'Types::POptionalType.new(Types::P\1Type::DEFAULT)')
34
- ruby.gsub!(/ref\('Array\[([0-9A-Za-z_]+)\]'\)/, 'Types::PArrayType.new(Types::P\1Type::DEFAULT)')
35
- ruby.gsub!(/ref\('Optional\[Array\[([0-9A-Za-z_]+)\]\]'\)/,
36
- 'Types::POptionalType.new(Types::PArrayType.new(Types::P\1Type::DEFAULT))')
37
- ruby.gsub!(/ref\('Enum(\[[^\]]+\])'\)/) do |match|
38
- params = $1
39
- params.gsub!(/\\'/, '\'')
40
- "Types::PEnumType.new(#{params})"
41
- end
42
-
43
- # Replace ref() constructs with references to _pcore_type of the types in the module namespace
44
- ruby.gsub!(/ref\('Puppet::AST::Locator'\)/, 'Parser::Locator::Locator19._pcore_type')
45
- ruby.gsub!(/ref\('Puppet::AST::([0-9A-Za-z_]+)'\)/, '\1._pcore_type')
46
- ruby.gsub!(/ref\('Optional\[Puppet::AST::([0-9A-Za-z_]+)\]'\)/, 'Types::POptionalType.new(\1._pcore_type)')
47
- ruby.gsub!(/ref\('Array\[Puppet::AST::([0-9A-Za-z_]+)\]'\)/, 'Types::PArrayType.new(\1._pcore_type)')
48
- ruby.gsub!(/ref\('Array\[Puppet::AST::([0-9A-Za-z_]+), 1, default\]'\)/,
49
- 'Types::PArrayType.new(\1._pcore_type, Types::PCollectionType::NOT_EMPTY_SIZE)')
50
-
51
- # Remove the generated ref() method. It's not needed by this model
52
- ruby.gsub!(/ def self\.ref\(type_string\)\n.*\n end\n\n/, '')
53
-
54
- # Add Program#current method for backward compatibility
55
- ruby.gsub!(/(attr_reader :body\n attr_reader :definitions\n attr_reader :locator)/, "\\1\n\n def current\n self\n end")
56
-
57
- # Replace the generated registration with a registration that uses the static loader. This will
58
- # become part of the Puppet bootstrap code and there will be no other loader until we have a
59
- # parser.
60
- ruby.gsub!(/^Puppet::Pops::Pcore.register_implementations\((\[[^\]]+\])\)/, <<-RUBY)
61
-
62
- module Model
63
- @@pcore_ast_initialized = false
64
- def self.register_pcore_types
65
- return if @@pcore_ast_initialized
66
- @@pcore_ast_initialized = true
67
- all_types = \\1
68
-
69
- # Create and register a TypeSet that corresponds to all types in the AST model
70
- types_map = {}
71
- all_types.each do |type|
72
- types_map[type._pcore_type.simple_name] = type._pcore_type
73
- end
74
- type_set = Types::PTypeSetType.new({
75
- 'name' => 'Puppet::AST',
76
- 'pcore_version' => '1.0.0',
77
- 'types' => types_map
78
- })
79
- loc = Puppet::Util.path_to_uri("\#{__FILE__}")
80
- Loaders.static_loader.set_entry(Loader::TypedName.new(:type, 'puppet::ast', Pcore::RUNTIME_NAME_AUTHORITY), type_set, URI("\#{loc}?line=1"))
81
- Loaders.register_static_implementations(all_types)
82
- end
83
- end
84
- RUBY
85
- ast_rb = Pathname(__FILE__).parent.parent + 'lib/puppet/pops/model/ast.rb'
86
- File.open(ast_rb.to_s, 'w') { |f| f.write(ruby) }
87
- end
88
- end
89
- end
90
- end
@@ -1,199 +0,0 @@
1
- # Run this rake task to generate cert fixtures used in unit tests. This should
2
- # be run whenever new fixtures are required that derive from the existing ones
3
- # such as to add an extension to client certs, change expiration, etc. All
4
- # regenerated fixtures should be committed together.
5
- desc "Generate cert test fixtures"
6
- task(:gen_cert_fixtures) do
7
- $LOAD_PATH << File.expand_path(File.join(File.dirname(__FILE__), '../spec/lib'))
8
- require 'puppet/test_ca'
9
-
10
- def save(dir, name, x509)
11
- path = File.join(dir, name)
12
- puts "Generating #{path}"
13
- File.open(path, 'w') do |f|
14
- f.write(x509.to_text)
15
- text = if block_given?
16
- yield x509
17
- else
18
- x509.to_pem
19
- end
20
-
21
- f.write(text)
22
- end
23
- end
24
-
25
- # This task generates a PKI consisting of a root CA, intermediate CA and
26
- # several leaf certs. A CRL is generated for each CA. The root CA CRL is
27
- # empty, while the intermediate CA CRL contains the revoked cert's serial
28
- # number. A textual representation of each X509 object is included in the
29
- # fixture as a comment.
30
- #
31
- # Certs
32
- # =====
33
- #
34
- # ca.pem /CN=Test CA
35
- # |
36
- # intermediate.pem +- /CN=Test CA Subauthority
37
- # | |
38
- # signed.pem | +- /CN=signed
39
- # revoked.pem | +- /CN=revoked
40
- # tampered-cert.pem | +- /CN=signed (with different public key)
41
- # ec.pem | +- /CN=ec (with EC private key)
42
- # oid.pem | +- /CN=oid (with custom oid)
43
- # |
44
- # 127.0.0.1.pem +- /CN=127.0.0.1 (with dns alt names)
45
- # |
46
- # intermediate-agent.pem +- /CN=Test CA Agent Subauthority
47
- # | |
48
- # pluto.pem | +- /CN=pluto
49
- # |
50
- # bad-int-basic-constraints.pem +- /CN=Test CA Subauthority (bad isCA constraint)
51
- #
52
- # bad-basic-constraints.pem /CN=Test CA (bad isCA constraint)
53
- #
54
- # unknown-ca.pem /CN=Unknown CA
55
- # |
56
- # unknown-127.0.0.1.pem +- /CN=127.0.0.1
57
- #
58
- # Keys
59
- # ====
60
- #
61
- # The RSA private key for each leaf cert is also generated. In addition,
62
- # `encrypted-key.pem` contains the private key for the `signed` cert.
63
- #
64
- # Requests
65
- # ========
66
- #
67
- # `request.pem` contains a valid CSR for /CN=pending, while `tampered_csr.pem`
68
- # is the same as `request.pem`, but it's public key has been replaced.
69
- #
70
- dir = File.join(RAKE_ROOT, 'spec/fixtures/ssl')
71
-
72
- # Create self-signed CA & key
73
- unknown_ca = Puppet::TestCa.new('Unknown CA')
74
- save(dir, 'unknown-ca.pem', unknown_ca.ca_cert)
75
- save(dir, 'unknown-ca-key.pem', unknown_ca.key)
76
-
77
- # Create an SSL cert for 127.0.0.1
78
- signed = unknown_ca.create_cert('127.0.0.1', unknown_ca.ca_cert, unknown_ca.key, subject_alt_names: 'DNS:127.0.0.1,DNS:127.0.0.2')
79
- save(dir, 'unknown-127.0.0.1.pem', signed[:cert])
80
- save(dir, 'unknown-127.0.0.1-key.pem', signed[:private_key])
81
-
82
- # Create Test CA & CRL
83
- ca = Puppet::TestCa.new
84
- save(dir, 'ca.pem', ca.ca_cert)
85
- save(dir, 'crl.pem', ca.ca_crl)
86
-
87
- # Create Intermediate CA & CRL "Test CA Subauthority" issued by "Test CA"
88
- inter = ca.create_intermediate_cert('Test CA Subauthority', ca.ca_cert, ca.key)
89
- save(dir, 'intermediate.pem', inter[:cert])
90
- save(dir, 'intermediate-key.pem', inter[:private_key])
91
- inter_crl = ca.create_crl(inter[:cert], inter[:private_key])
92
-
93
- # Create a leaf/entity key and cert for host "signed" and issued by "Test CA Subauthority"
94
- signed = ca.create_cert('signed', inter[:cert], inter[:private_key])
95
- save(dir, 'signed.pem', signed[:cert])
96
- save(dir, 'signed-key.pem', signed[:private_key])
97
-
98
- # Create a cert for host "renewed" and issued by "Test CA Subauthority"
99
- renewed = ca.create_cert('renewed', inter[:cert], inter[:private_key], reuse_key: signed[:private_key])
100
- save(dir, 'renewed.pem', renewed[:cert])
101
-
102
- # Create an encrypted version of the above private key for host "signed"
103
- save(dir, 'encrypted-key.pem', signed[:private_key]) do |x509|
104
- # private key password was chosen at random
105
- x509.to_pem(OpenSSL::Cipher::AES.new(128, :CBC), '74695716c8b6')
106
- end
107
-
108
- # Create an SSL cert for 127.0.0.1 with dns_alt_names
109
- signed = ca.create_cert('127.0.0.1', ca.ca_cert, ca.key, subject_alt_names: 'DNS:127.0.0.1,DNS:127.0.0.2')
110
- save(dir, '127.0.0.1.pem', signed[:cert])
111
- save(dir, '127.0.0.1-key.pem', signed[:private_key])
112
-
113
- # Create an SSL cert with extensions containing custom oids
114
- extensions = [
115
- ['1.3.6.1.4.1.34380.1.2.1.1', OpenSSL::ASN1::UTF8String.new('somevalue'), false],
116
- ]
117
- oid = ca.create_cert('oid', inter[:cert], inter[:private_key], extensions: extensions)
118
- save(dir, 'oid.pem', oid[:cert])
119
- save(dir, 'oid-key.pem', oid[:private_key])
120
-
121
- # Create a leaf/entity key and cert for host "revoked", issued by "Test CA Subauthority"
122
- # and revoke the cert
123
- revoked = ca.create_cert('revoked', inter[:cert], inter[:private_key])
124
- ca.revoke(revoked[:cert], inter_crl, inter[:private_key])
125
- save(dir, 'revoked.pem', revoked[:cert])
126
- save(dir, 'revoked-key.pem', revoked[:private_key])
127
-
128
- # Create an EC key and cert, issued by "Test CA Subauthority"
129
- ec = ca.create_cert('ec', inter[:cert], inter[:private_key], key_type: :ec)
130
- save(dir, 'ec.pem', ec[:cert])
131
- save(dir, 'ec-key.pem', ec[:private_key])
132
-
133
- # Create an encrypted version of the above private key for host "ec"
134
- save(dir, 'encrypted-ec-key.pem', ec[:private_key]) do |x509|
135
- # private key password was chosen at random
136
- x509.to_pem(OpenSSL::Cipher::AES.new(128, :CBC), '74695716c8b6')
137
- end
138
-
139
- # Update intermediate CRL now that we've revoked
140
- save(dir, 'intermediate-crl.pem', inter_crl)
141
-
142
- # Create a pending request (CSR) and private key for host "pending"
143
- request = ca.create_request('pending')
144
- save(dir, 'request.pem', request[:csr])
145
- save(dir, 'request-key.pem', request[:private_key])
146
-
147
- # Create an intermediate for agent certs
148
- inter_agent = ca.create_intermediate_cert('Test CA Agent Subauthority', ca.ca_cert, ca.key)
149
- save(dir, 'intermediate-agent.pem', inter_agent[:cert])
150
- inter_agent_crl = ca.create_crl(inter_agent[:cert], inter_agent[:private_key])
151
- save(dir, 'intermediate-agent-crl.pem', inter_agent_crl)
152
-
153
- # Create a leaf/entity key and cert for host "pluto" and issued by "Test CA Agent Subauthority"
154
- pluto = ca.create_cert('pluto', inter_agent[:cert], inter_agent[:private_key])
155
- save(dir, 'pluto.pem', pluto[:cert])
156
- save(dir, 'pluto-key.pem', pluto[:private_key])
157
-
158
- # Create a new root CA cert, but change the "isCA" basic constraint.
159
- # It should not be trusted to act as a CA.
160
- badconstraints = ca.create_cacert('Test CA')[:cert]
161
- badconstraints.public_key = ca.ca_cert.public_key
162
- badconstraints.extensions = []
163
- ca.ca_cert.extensions.each do |ext|
164
- if ext.oid == 'basicConstraints'
165
- ef = OpenSSL::X509::ExtensionFactory.new
166
- badconstraints.add_extension(ef.create_extension("basicConstraints","CA:FALSE", true))
167
- else
168
- badconstraints.add_extension(ext)
169
- end
170
- end
171
- badconstraints.sign(ca.key, OpenSSL::Digest::SHA256.new)
172
- save(dir, 'bad-basic-constraints.pem', badconstraints)
173
-
174
- # Same as above, but create a new intermediate CA
175
- badintconstraints = inter[:cert].dup
176
- badintconstraints.public_key = inter[:cert].public_key
177
- badintconstraints.extensions = []
178
- inter[:cert].extensions.each do |ext|
179
- if ext.oid == 'basicConstraints'
180
- ef = OpenSSL::X509::ExtensionFactory.new
181
- badintconstraints.add_extension(ef.create_extension("basicConstraints","CA:FALSE", true))
182
- else
183
- badintconstraints.add_extension(ext)
184
- end
185
- end
186
- badintconstraints.sign(ca.key, OpenSSL::Digest::SHA256.new)
187
- save(dir, 'bad-int-basic-constraints.pem', badintconstraints)
188
-
189
- # Create a request, but replace its public key after it's signed
190
- tampered_csr = ca.create_request('signed')[:csr]
191
- tampered_csr.public_key = OpenSSL::PKey::RSA.new(2048).public_key
192
- save(dir, 'tampered-csr.pem', tampered_csr)
193
-
194
- # Create a cert issued from the real intermediate CA, but replace its
195
- # public key
196
- tampered_cert = ca.create_cert('signed', inter[:cert], inter[:private_key])[:cert]
197
- tampered_cert.public_key = OpenSSL::PKey::RSA.new(2048).public_key
198
- save(dir, 'tampered-cert.pem', tampered_cert)
199
- end
data/tasks/manpages.rake DELETED
@@ -1,67 +0,0 @@
1
- desc "Build Puppet manpages"
2
- task :gen_manpages do
3
- require 'puppet/face'
4
- require 'fileutils'
5
-
6
- Puppet.initialize_settings
7
- helpface = Puppet::Face[:help, '0.0.1']
8
-
9
- bins = Dir.glob(%w{bin/*})
10
- non_face_applications = helpface.legacy_applications
11
- faces = Puppet::Face.faces.map(&:to_s)
12
- apps = non_face_applications + faces
13
-
14
- ronn_args = '--manual="Puppet manual" --organization="Puppet, Inc." --roff'
15
-
16
- unless ENV['SOURCE_DATE_EPOCH'].nil?
17
- source_date = Time.at(ENV['SOURCE_DATE_EPOCH'].to_i).strftime('%Y-%m-%d')
18
- ronn_args += " --date=#{source_date}"
19
- end
20
-
21
- # Locate ronn
22
- begin
23
- require 'ronn'
24
- rescue LoadError
25
- abort("Run `bundle install --with documentation` to install the `ronn` gem.")
26
- end
27
-
28
- ronn = %x{which ronn}.chomp
29
- unless File.executable?(ronn)
30
- abort("Ronn does not appear to be installed")
31
- end
32
-
33
- %x{mkdir -p ./man/man5 ./man/man8}
34
- %x{RUBYLIB=./lib:$RUBYLIB bin/puppet doc --reference configuration > ./man/man5/puppetconf.5.ronn}
35
- %x{#{ronn} #{ronn_args} ./man/man5/puppetconf.5.ronn}
36
- FileUtils.mv("./man/man5/puppetconf.5", "./man/man5/puppet.conf.5")
37
- FileUtils.rm("./man/man5/puppetconf.5.ronn")
38
-
39
- # Create LEGACY binary man pages (i.e. delete me for 2.8.0)
40
- bins.each do |bin|
41
- b = bin.gsub( /^s?bin\//, "")
42
- %x{RUBYLIB=./lib:$RUBYLIB #{bin} --help > ./man/man8/#{b}.8.ronn}
43
- %x{#{ronn} #{ronn_args} ./man/man8/#{b}.8.ronn}
44
- FileUtils.rm("./man/man8/#{b}.8.ronn")
45
- end
46
-
47
- apps.each do |app|
48
- %x{RUBYLIB=./lib:$RUBYLIB bin/puppet help #{app} --ronn > ./man/man8/puppet-#{app}.8.ronn}
49
- %x{#{ronn} #{ronn_args} ./man/man8/puppet-#{app}.8.ronn}
50
- FileUtils.rm("./man/man8/puppet-#{app}.8.ronn")
51
- end
52
-
53
- # Delete orphaned manpages if binary was deleted
54
- Dir.glob(%w{./man/man8/puppet-*.8}) do |app|
55
- appname = app.match(/puppet-(.*)\.8/)[1]
56
- FileUtils.rm("./man/man8/puppet-#{appname}.8") unless apps.include?(appname)
57
- end
58
-
59
- # Vile hack: create puppet resource man page
60
- # Currently, the useless resource face wins against puppet resource in puppet
61
- # man. (And actually, it even gets removed from the list of legacy
62
- # applications.) So we overwrite it with the correct man page at the end.
63
- %x{RUBYLIB=./lib:$RUBYLIB bin/puppet resource --help > ./man/man8/puppet-resource.8.ronn}
64
- %x{#{ronn} #{ronn_args} ./man/man8/puppet-resource.8.ronn}
65
- FileUtils.rm("./man/man8/puppet-resource.8.ronn")
66
-
67
- end
data/tasks/memwalk.rake DELETED
@@ -1,195 +0,0 @@
1
- # Walks the memory dumped into heap.json, and produces a graph of the memory dumped in diff.json
2
- # If a single argument (a hex address to one object) is given, the graph is limited to this object and what references it
3
- # The heap dumps should be in the format produced by Ruby ObjectSpace in Ruby version 2.1.0 or later.
4
- #
5
- # The command produces a .dot file that can be rendered with graphwiz dot into SVG. If a memwalk is performed for all
6
- # objects in the diff.json, the output file name is memwalk.dot. If it is produced for a single address, the name of the
7
- # output file is memwalk-<address>.dot
8
- #
9
- # The dot file can be rendered with something like: dot -Tsvg -omemwalk.svg memwalk.dot
10
- #
11
- desc "Process a diff.json of object ids, and a heap.json of a Ruby 2.1.0 ObjectSpace dump and produce a graph"
12
- task :memwalk, [:id] do |t, args|
13
- puts "Memwalk"
14
- puts "Computing for #{args[:id] ? args[:id] : 'all'}"
15
- @single_id = args[:id] ? args[:id].to_i(16) : nil
16
-
17
- require 'json'
18
- #require 'debug'
19
-
20
- TYPE = "type".freeze
21
- ROOT = "root".freeze
22
- ROOT_UC = "ROOT".freeze
23
- ADDR = "address".freeze
24
- NODE = "NODE".freeze
25
- STRING = "STRING".freeze
26
- DATA = "DATA".freeze
27
- HASH = "HASH".freeze
28
- ARRAY = "ARRAY".freeze
29
- OBJECT = "OBJECT".freeze
30
- CLASS = "CLASS".freeze
31
-
32
- allocations = {}
33
- # An array of integer addresses of the objects to trace bindings for
34
- diff_index = {}
35
- puts "Reading data"
36
- begin
37
- puts "Reading diff"
38
- lines = 0;
39
- File.readlines("diff.json").each do | line |
40
- lines += 1
41
- diff = JSON.parse(line)
42
- case diff[ TYPE ]
43
- when STRING, DATA, HASH, ARRAY
44
- # skip the strings
45
- else
46
- diff_index[ diff[ ADDR ].to_i(16) ] = diff
47
- end
48
- end
49
- puts "Read #{lines} number of diffs"
50
- rescue => e
51
- raise "ERROR READING DIFF at line #{lines} #{e.message[0, 200]}"
52
- end
53
-
54
- begin
55
- puts "Reading heap"
56
- lines = 0
57
- allocation = nil
58
- File.readlines("heap.json").each do | line |
59
- lines += 1
60
- allocation = JSON.parse(line)
61
- case allocation[ TYPE ]
62
- when ROOT_UC
63
- # Graph for single id must include roots, as it may be a root that holds on to the reference
64
- # a global variable, thread, etc.
65
- #
66
- if @single_id
67
- allocations[ allocation[ ROOT ] ] = allocation
68
- end
69
- when NODE
70
- # skip the NODE objects - they represent the loaded ruby code
71
- when STRING
72
- # skip all strings - they are everywhere
73
- else
74
- allocations[ allocation[ ADDR ].to_i(16) ] = allocation
75
- end
76
- end
77
- puts "Read #{lines} number of entries"
78
- rescue => e
79
- require 'debug'
80
- puts "ERROR READING HEAP #{e.message[0, 200]}"
81
- raise e
82
- end
83
- @heap = allocations
84
-
85
- puts "Building reference index"
86
- # References is an index from a referenced object to an array with addresses to the objects that references it
87
- @references = Hash.new { |h, k| h[k] = [] }
88
- REFERENCES = "references".freeze
89
- allocations.each do |k,v|
90
- refs = v[ REFERENCES ]
91
- if refs.is_a?(Array)
92
- refs.each {|addr| @references[ addr.to_i(16) ] << k }
93
- end
94
- end
95
-
96
- @printed = Set.new()
97
-
98
- def print_object(addr, entry)
99
- # only print each node once
100
- return unless @printed.add?(addr)
101
- begin
102
- if addr.is_a?(String)
103
- @output.write( "x#{node_name(addr)} [label=\"#{node_label(addr, entry)}\\n#{addr}\"];\n")
104
- else
105
- @output.write( "x#{node_name(addr)} [label=\"#{node_label(addr, entry)}\\n#{addr.to_s(16)}\"];\n")
106
- end
107
- rescue => e
108
- require 'debug'
109
- raise e
110
- end
111
- end
112
-
113
- def node_label(addr, entry)
114
- if entry[ TYPE ] == OBJECT
115
- class_ref = entry[ "class" ].to_i(16)
116
- @heap[ class_ref ][ "name" ]
117
- elsif entry[ TYPE ] == CLASS
118
- "CLASS #{entry[ "name"]}"
119
- else
120
- entry[TYPE]
121
- end
122
- end
123
-
124
- def node_name(addr)
125
- return addr if addr.is_a? String
126
- addr.to_s(16)
127
- end
128
-
129
- def print_edge(from_addr, to_addr)
130
- @output.write("x#{node_name(from_addr)}->x#{node_name(to_addr)};\n")
131
- end
132
-
133
- def closure_and_edges(diff)
134
- edges = Set.new()
135
- walked = Set.new()
136
- puts "Number of diffs referenced = #{diff.count {|k,_| @references[k].is_a?(Array) && @references[k].size() > 0 }}"
137
- diff.each {|k,_| walk(k, edges, walked) }
138
- edges.each {|e| print_edge(*e) }
139
- end
140
-
141
- def walk(addr, edges, walked)
142
- if !@heap[ addr ].nil?
143
- print_object(addr, @heap[addr])
144
-
145
- @references [ addr ].each do |r|
146
- walk_to_object(addr, r, edges, walked)
147
- end
148
- end
149
- end
150
-
151
- def walk_to_object(to_addr, cursor, edges, walked)
152
- return unless walked
153
- # if walked to an object, or everything if a single_id is the target
154
- if @heap[ cursor ][ TYPE ] == OBJECT || (@single_id && @heap[ cursor ][ TYPE ] == ROOT_UC || @heap[ cursor ][ TYPE ] == CLASS )
155
- # and the edge is unique
156
- if edges.add?( [ cursor, to_addr ] )
157
- # then we may not have visited objects this objects is being referred from
158
- print_object(cursor, @heap[ cursor ])
159
- # Do not follow what binds a class
160
- if @heap[ cursor ][ TYPE ] != CLASS
161
- @references[ cursor ].each do |r|
162
- walk_to_object(cursor, r, edges, walked.add?(r))
163
- walked.delete(r)
164
- end
165
- end
166
- end
167
- else
168
- # continue search until Object
169
- @references[cursor].each do |r|
170
- walk_to_object(to_addr, r, edges, walked.add?(r))
171
- end
172
- end
173
- end
174
-
175
- def single_closure_and_edges(the_target)
176
- edges = Set.new()
177
- walked = Set.new()
178
- walk(the_target, edges, walked)
179
- edges.each {|e| print_edge(*e) }
180
- end
181
-
182
- puts "creating graph"
183
- if @single_id
184
- @output = File.open("memwalk-#{@single_id.to_s(16)}.dot", "w")
185
- @output.write("digraph root {\n")
186
- single_closure_and_edges(@single_id)
187
- else
188
- @output = File.open("memwalk.dot", "w")
189
- @output.write("digraph root {\n")
190
- closure_and_edges(diff_index)
191
- end
192
- @output.write("}\n")
193
- @output.close
194
- puts "done"
195
- end