inventory_refresh 0.3.5 → 1.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (52) hide show
  1. checksums.yaml +4 -4
  2. data/.codeclimate.yml +25 -30
  3. data/.github/workflows/ci.yaml +58 -0
  4. data/.rubocop.yml +3 -3
  5. data/.rubocop_cc.yml +3 -4
  6. data/.rubocop_local.yml +5 -2
  7. data/.whitesource +3 -0
  8. data/CHANGELOG.md +19 -0
  9. data/Gemfile +10 -4
  10. data/README.md +1 -2
  11. data/Rakefile +2 -2
  12. data/inventory_refresh.gemspec +9 -10
  13. data/lib/inventory_refresh/application_record_iterator.rb +25 -12
  14. data/lib/inventory_refresh/graph/topological_sort.rb +24 -26
  15. data/lib/inventory_refresh/graph.rb +2 -2
  16. data/lib/inventory_refresh/inventory_collection/builder.rb +37 -15
  17. data/lib/inventory_refresh/inventory_collection/data_storage.rb +9 -0
  18. data/lib/inventory_refresh/inventory_collection/helpers/initialize_helper.rb +147 -38
  19. data/lib/inventory_refresh/inventory_collection/helpers/questions_helper.rb +48 -4
  20. data/lib/inventory_refresh/inventory_collection/index/proxy.rb +35 -3
  21. data/lib/inventory_refresh/inventory_collection/index/type/base.rb +8 -0
  22. data/lib/inventory_refresh/inventory_collection/index/type/local_db.rb +2 -0
  23. data/lib/inventory_refresh/inventory_collection/index/type/skeletal.rb +1 -0
  24. data/lib/inventory_refresh/inventory_collection/reference.rb +1 -0
  25. data/lib/inventory_refresh/inventory_collection/references_storage.rb +17 -0
  26. data/lib/inventory_refresh/inventory_collection/scanner.rb +91 -3
  27. data/lib/inventory_refresh/inventory_collection/serialization.rb +16 -10
  28. data/lib/inventory_refresh/inventory_collection.rb +122 -64
  29. data/lib/inventory_refresh/inventory_object.rb +74 -40
  30. data/lib/inventory_refresh/inventory_object_lazy.rb +17 -10
  31. data/lib/inventory_refresh/null_logger.rb +2 -2
  32. data/lib/inventory_refresh/persister.rb +43 -93
  33. data/lib/inventory_refresh/save_collection/base.rb +4 -2
  34. data/lib/inventory_refresh/save_collection/saver/base.rb +114 -15
  35. data/lib/inventory_refresh/save_collection/saver/batch.rb +17 -0
  36. data/lib/inventory_refresh/save_collection/saver/concurrent_safe_batch.rb +129 -51
  37. data/lib/inventory_refresh/save_collection/saver/default.rb +57 -0
  38. data/lib/inventory_refresh/save_collection/saver/partial_upsert_helper.rb +2 -19
  39. data/lib/inventory_refresh/save_collection/saver/retention_helper.rb +68 -3
  40. data/lib/inventory_refresh/save_collection/saver/sql_helper.rb +125 -0
  41. data/lib/inventory_refresh/save_collection/saver/sql_helper_update.rb +10 -6
  42. data/lib/inventory_refresh/save_collection/saver/sql_helper_upsert.rb +28 -16
  43. data/lib/inventory_refresh/save_collection/sweeper.rb +17 -93
  44. data/lib/inventory_refresh/save_collection/topological_sort.rb +5 -5
  45. data/lib/inventory_refresh/save_inventory.rb +5 -12
  46. data/lib/inventory_refresh/target.rb +73 -0
  47. data/lib/inventory_refresh/target_collection.rb +92 -0
  48. data/lib/inventory_refresh/version.rb +1 -1
  49. data/lib/inventory_refresh.rb +2 -0
  50. metadata +42 -39
  51. data/.travis.yml +0 -23
  52. data/lib/inventory_refresh/exception.rb +0 -8
@@ -0,0 +1,92 @@
1
+ require "active_support/core_ext/module/delegation"
2
+
3
+ module InventoryRefresh
4
+ class TargetCollection
5
+ attr_reader :targets
6
+
7
+ delegate :<<, :to => :targets
8
+
9
+ # @param manager [ManageIQ::Providers::BaseManager] manager owning the TargetCollection
10
+ # @param manager_id [Integer] primary key of manager owning the TargetCollection
11
+ # @param event [EmsEvent] EmsEvent associated with the TargetCollection
12
+ # @param targets [Array<InventoryRefresh::Target, ApplicationRecord>] Array of InventoryRefresh::Target objects or
13
+ # ApplicationRecord objects
14
+ def initialize(manager: nil, manager_id: nil, event: nil, targets: [])
15
+ @manager = manager
16
+ @manager_id = manager_id
17
+ @event = event
18
+ @targets = targets
19
+ end
20
+
21
+ # @param association [Symbol] An existing association on Manager, that lists objects represented by a Target, naming
22
+ # should be the same of association of a counterpart InventoryCollection object
23
+ # @param manager_ref [Hash] A Hash that can be used to find_by on a given association and returning a unique object.
24
+ # The keys should be the same as the keys of the counterpart InventoryObject
25
+ # @param manager [ManageIQ::Providers::BaseManager] The Manager owning the Target
26
+ # @param manager_id [Integer] A primary key of the Manager owning the Target
27
+ # @param event_id [Integer] A primary key of the EmsEvent associated with the Target
28
+ # @param options [Hash] A free form options hash
29
+ def add_target(association:, manager_ref:, manager: nil, manager_id: nil, event_id: nil, options: {})
30
+ self << InventoryRefresh::Target.new(:association => association,
31
+ :manager_ref => manager_ref,
32
+ :manager => manager || @manager,
33
+ :manager_id => manager_id || @manager_id || @manager.try(:id),
34
+ :event_id => event_id || @event.try(:id),
35
+ :options => options)
36
+ end
37
+
38
+ # @return [String] A String containing a summary
39
+ def name
40
+ "Collection of #{targets.size} targets"
41
+ end
42
+
43
+ # @return [String] A String containing an id of each target in the TargetCollection
44
+ def id
45
+ "Collection of targets with id: #{targets.collect(&:name)}"
46
+ end
47
+
48
+ # Returns targets in a format:
49
+ # {
50
+ # :vms => {:ems_ref => Set.new(["vm_ref_1", "vm_ref2"])},
51
+ # :network_ports => {:ems_ref => Set.new(["network_port_1", "network_port2"])
52
+ # }
53
+ #
54
+ # Then we can quickly access all objects affected by:
55
+ # NetworkPort.where(target_collection.manager_refs_by_association[:network_ports].to_a) =>
56
+ # return AR objects with ems_refs ["network_port_1", "network_port2"]
57
+ # And we can get a list of ids for the API query by:
58
+ # target_collection.manager_refs_by_association[:network_ports][:ems_ref].to_a =>
59
+ # ["network_port_1", "network_port2"]
60
+ #
61
+ # Only targets of a type InventoryRefresh::Target are processed, any other targets present should be converted to
62
+ # InventoryRefresh::Target, e.g. in the Inventory::Collector code.
63
+ def manager_refs_by_association
64
+ @manager_refs_by_association ||= targets.select { |x| x.kind_of?(InventoryRefresh::Target) }.each_with_object({}) do |x, obj|
65
+ if obj[x.association].blank?
66
+ obj[x.association] = x.manager_ref.each_with_object({}) { |(key, value), hash| hash[key] = Set.new([value]) }
67
+ else
68
+ obj[x.association].each do |key, value|
69
+ value << x.manager_ref[key]
70
+ end
71
+ end
72
+ end
73
+ end
74
+
75
+ # Resets the cached @manager_refs_by_association to enforce reload when calling :manager_refs_by_association method
76
+ def manager_refs_by_association_reset
77
+ @manager_refs_by_association = nil
78
+ end
79
+
80
+ # Returns list of ems_refs
81
+ # @return [Array<String>]
82
+ def references(collection)
83
+ manager_refs_by_association.try(:[], collection).try(:[], :ems_ref)&.to_a || []
84
+ end
85
+
86
+ # Returns list of names
87
+ # @return [Array<String>]
88
+ def name_references(collection)
89
+ manager_refs_by_association.try(:[], collection).try(:[], :name)&.to_a || []
90
+ end
91
+ end
92
+ end
@@ -1,3 +1,3 @@
1
1
  module InventoryRefresh
2
- VERSION = "0.3.5".freeze
2
+ VERSION = "1.1.0".freeze
3
3
  end
@@ -6,4 +6,6 @@ require "inventory_refresh/logging"
6
6
  require "inventory_refresh/null_logger"
7
7
  require "inventory_refresh/persister"
8
8
  require "inventory_refresh/save_inventory"
9
+ require "inventory_refresh/target"
10
+ require "inventory_refresh/target_collection"
9
11
  require "inventory_refresh/version"
metadata CHANGED
@@ -1,43 +1,55 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: inventory_refresh
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.3.5
4
+ version: 1.1.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - ManageIQ Developers
8
8
  autorequire:
9
9
  bindir: exe
10
10
  cert_chain: []
11
- date: 2020-04-20 00:00:00.000000000 Z
11
+ date: 2022-05-03 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: activerecord
15
15
  requirement: !ruby/object:Gem::Requirement
16
16
  requirements:
17
- - - "~>"
17
+ - - ">="
18
18
  - !ruby/object:Gem::Version
19
19
  version: '5.0'
20
+ - - "<"
21
+ - !ruby/object:Gem::Version
22
+ version: '7.0'
20
23
  type: :runtime
21
24
  prerelease: false
22
25
  version_requirements: !ruby/object:Gem::Requirement
23
26
  requirements:
24
- - - "~>"
27
+ - - ">="
25
28
  - !ruby/object:Gem::Version
26
29
  version: '5.0'
30
+ - - "<"
31
+ - !ruby/object:Gem::Version
32
+ version: '7.0'
27
33
  - !ruby/object:Gem::Dependency
28
34
  name: more_core_extensions
29
35
  requirement: !ruby/object:Gem::Requirement
30
36
  requirements:
31
- - - "~>"
37
+ - - ">="
32
38
  - !ruby/object:Gem::Version
33
39
  version: '3.5'
40
+ - - "<"
41
+ - !ruby/object:Gem::Version
42
+ version: '5'
34
43
  type: :runtime
35
44
  prerelease: false
36
45
  version_requirements: !ruby/object:Gem::Requirement
37
46
  requirements:
38
- - - "~>"
47
+ - - ">="
39
48
  - !ruby/object:Gem::Version
40
49
  version: '3.5'
50
+ - - "<"
51
+ - !ruby/object:Gem::Version
52
+ version: '5'
41
53
  - !ruby/object:Gem::Dependency
42
54
  name: pg
43
55
  requirement: !ruby/object:Gem::Requirement
@@ -95,75 +107,61 @@ dependencies:
95
107
  - !ruby/object:Gem::Version
96
108
  version: 4.11.1
97
109
  - !ruby/object:Gem::Dependency
98
- name: rake
110
+ name: manageiq-style
99
111
  requirement: !ruby/object:Gem::Requirement
100
112
  requirements:
101
- - - "~>"
113
+ - - ">="
102
114
  - !ruby/object:Gem::Version
103
- version: '10.0'
115
+ version: '0'
104
116
  type: :development
105
117
  prerelease: false
106
118
  version_requirements: !ruby/object:Gem::Requirement
107
119
  requirements:
108
- - - "~>"
120
+ - - ">="
109
121
  - !ruby/object:Gem::Version
110
- version: '10.0'
122
+ version: '0'
111
123
  - !ruby/object:Gem::Dependency
112
- name: rspec
124
+ name: rake
113
125
  requirement: !ruby/object:Gem::Requirement
114
126
  requirements:
115
127
  - - "~>"
116
128
  - !ruby/object:Gem::Version
117
- version: '3.0'
129
+ version: '10.0'
118
130
  type: :development
119
131
  prerelease: false
120
132
  version_requirements: !ruby/object:Gem::Requirement
121
133
  requirements:
122
134
  - - "~>"
123
135
  - !ruby/object:Gem::Version
124
- version: '3.0'
125
- - !ruby/object:Gem::Dependency
126
- name: rubocop
127
- requirement: !ruby/object:Gem::Requirement
128
- requirements:
129
- - - ">="
130
- - !ruby/object:Gem::Version
131
- version: '0'
132
- type: :development
133
- prerelease: false
134
- version_requirements: !ruby/object:Gem::Requirement
135
- requirements:
136
- - - ">="
137
- - !ruby/object:Gem::Version
138
- version: '0'
136
+ version: '10.0'
139
137
  - !ruby/object:Gem::Dependency
140
- name: rubocop-performance
138
+ name: rspec
141
139
  requirement: !ruby/object:Gem::Requirement
142
140
  requirements:
143
- - - ">="
141
+ - - "~>"
144
142
  - !ruby/object:Gem::Version
145
- version: '0'
143
+ version: '3.0'
146
144
  type: :development
147
145
  prerelease: false
148
146
  version_requirements: !ruby/object:Gem::Requirement
149
147
  requirements:
150
- - - ">="
148
+ - - "~>"
151
149
  - !ruby/object:Gem::Version
152
- version: '0'
150
+ version: '3.0'
153
151
  - !ruby/object:Gem::Dependency
154
152
  name: simplecov
155
153
  requirement: !ruby/object:Gem::Requirement
156
154
  requirements:
157
155
  - - ">="
158
156
  - !ruby/object:Gem::Version
159
- version: '0'
157
+ version: 0.21.2
160
158
  type: :development
161
159
  prerelease: false
162
160
  version_requirements: !ruby/object:Gem::Requirement
163
161
  requirements:
164
162
  - - ">="
165
163
  - !ruby/object:Gem::Version
166
- version: '0'
164
+ version: 0.21.2
167
165
  description: Topological Inventory Persister
168
166
  email:
169
167
  executables: []
@@ -172,13 +170,14 @@ extra_rdoc_files: []
172
170
  files:
173
171
  - ".codeclimate.yml"
174
172
  - ".github/CODEOWNERS"
173
+ - ".github/workflows/ci.yaml"
175
174
  - ".gitignore"
176
175
  - ".rspec"
177
176
  - ".rspec_ci"
178
177
  - ".rubocop.yml"
179
178
  - ".rubocop_cc.yml"
180
179
  - ".rubocop_local.yml"
181
- - ".travis.yml"
180
+ - ".whitesource"
182
181
  - ".yamllint"
183
182
  - CHANGELOG.md
184
183
  - Gemfile
@@ -193,7 +192,6 @@ files:
193
192
  - lib/inventory_refresh.rb
194
193
  - lib/inventory_refresh/application_record_iterator.rb
195
194
  - lib/inventory_refresh/application_record_reference.rb
196
- - lib/inventory_refresh/exception.rb
197
195
  - lib/inventory_refresh/graph.rb
198
196
  - lib/inventory_refresh/graph/topological_sort.rb
199
197
  - lib/inventory_refresh/inventory_collection.rb
@@ -221,7 +219,9 @@ files:
221
219
  - lib/inventory_refresh/persister.rb
222
220
  - lib/inventory_refresh/save_collection/base.rb
223
221
  - lib/inventory_refresh/save_collection/saver/base.rb
222
+ - lib/inventory_refresh/save_collection/saver/batch.rb
224
223
  - lib/inventory_refresh/save_collection/saver/concurrent_safe_batch.rb
224
+ - lib/inventory_refresh/save_collection/saver/default.rb
225
225
  - lib/inventory_refresh/save_collection/saver/partial_upsert_helper.rb
226
226
  - lib/inventory_refresh/save_collection/saver/retention_helper.rb
227
227
  - lib/inventory_refresh/save_collection/saver/sql_helper.rb
@@ -230,11 +230,14 @@ files:
230
230
  - lib/inventory_refresh/save_collection/sweeper.rb
231
231
  - lib/inventory_refresh/save_collection/topological_sort.rb
232
232
  - lib/inventory_refresh/save_inventory.rb
233
+ - lib/inventory_refresh/target.rb
234
+ - lib/inventory_refresh/target_collection.rb
233
235
  - lib/inventory_refresh/version.rb
234
236
  homepage: https://github.com/ManageIQ/inventory_refresh
235
237
  licenses:
236
238
  - Apache-2.0
237
- metadata: {}
239
+ metadata:
240
+ rubygems_mfa_required: 'true'
238
241
  post_install_message:
239
242
  rdoc_options: []
240
243
  require_paths:
@@ -250,7 +253,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
250
253
  - !ruby/object:Gem::Version
251
254
  version: '0'
252
255
  requirements: []
253
- rubygems_version: 3.1.2
256
+ rubygems_version: 3.3.5
254
257
  signing_key:
255
258
  specification_version: 4
256
259
  summary: Topological Inventory Persister
data/.travis.yml DELETED
@@ -1,23 +0,0 @@
1
- dist: xenial
2
- sudo: false
3
- language: ruby
4
- cache: bundler
5
- rvm:
6
- - 2.5.7
7
- - 2.6.5
8
- addons:
9
- postgresql: '10'
10
- env:
11
- global:
12
- - CC_TEST_REPORTER_ID=1ef1a3a3d007395b11083d634a6fdac1e3d979b6428c39d2cd8d58556cdd68f7
13
- before_install:
14
- - gem install bundler -v 2.0.1
15
- - psql -c "CREATE USER root SUPERUSER PASSWORD 'smartvm';" -U postgres
16
- before_script:
17
- - curl -L https://codeclimate.com/downloads/test-reporter/test-reporter-latest-linux-amd64
18
- > ./cc-test-reporter
19
- - chmod +x ./cc-test-reporter
20
- - "./cc-test-reporter before-build"
21
- - bundle exec rake spec:setup
22
- after_script:
23
- - "./cc-test-reporter after-build --exit-code $TRAVIS_TEST_RESULT"
@@ -1,8 +0,0 @@
1
- module InventoryRefresh
2
- module Exception
3
- class SweeperError < StandardError; end
4
- class SweeperNonExistentScopeKeyFoundError < SweeperError; end
5
- class SweeperNonUniformScopeKeyFoundError < SweeperError; end
6
- class SweeperScopeBadFormat < SweeperError; end
7
- end
8
- end