inventory_refresh 0.1.1 → 0.2.2

Sign up to get free protection for your applications and to get access to all the features.
Files changed (38) hide show
  1. checksums.yaml +4 -4
  2. data/.codeclimate.yml +1 -0
  3. data/.gitignore +6 -0
  4. data/.travis.yml +3 -3
  5. data/Gemfile +4 -0
  6. data/inventory_refresh.gemspec +7 -5
  7. data/lib/inventory_refresh.rb +1 -0
  8. data/lib/inventory_refresh/inventory_collection.rb +115 -646
  9. data/lib/inventory_refresh/inventory_collection/builder.rb +249 -0
  10. data/lib/inventory_refresh/inventory_collection/graph.rb +0 -15
  11. data/lib/inventory_refresh/inventory_collection/helpers.rb +6 -0
  12. data/lib/inventory_refresh/inventory_collection/helpers/associations_helper.rb +80 -0
  13. data/lib/inventory_refresh/inventory_collection/helpers/initialize_helper.rb +456 -0
  14. data/lib/inventory_refresh/inventory_collection/helpers/questions_helper.rb +132 -0
  15. data/lib/inventory_refresh/inventory_collection/index/proxy.rb +1 -1
  16. data/lib/inventory_refresh/inventory_collection/index/type/skeletal.rb +5 -5
  17. data/lib/inventory_refresh/inventory_collection/reference.rb +4 -0
  18. data/lib/inventory_refresh/inventory_collection/scanner.rb +111 -18
  19. data/lib/inventory_refresh/inventory_collection/serialization.rb +7 -7
  20. data/lib/inventory_refresh/inventory_collection/unconnected_edge.rb +19 -0
  21. data/lib/inventory_refresh/inventory_object.rb +17 -11
  22. data/lib/inventory_refresh/inventory_object_lazy.rb +20 -10
  23. data/lib/inventory_refresh/persister.rb +212 -0
  24. data/lib/inventory_refresh/save_collection/base.rb +18 -3
  25. data/lib/inventory_refresh/save_collection/saver/base.rb +27 -64
  26. data/lib/inventory_refresh/save_collection/saver/concurrent_safe_batch.rb +73 -225
  27. data/lib/inventory_refresh/save_collection/saver/partial_upsert_helper.rb +226 -0
  28. data/lib/inventory_refresh/save_collection/saver/retention_helper.rb +115 -0
  29. data/lib/inventory_refresh/save_collection/saver/sql_helper.rb +122 -0
  30. data/lib/inventory_refresh/save_collection/saver/sql_helper_update.rb +24 -5
  31. data/lib/inventory_refresh/save_collection/saver/sql_helper_upsert.rb +6 -6
  32. data/lib/inventory_refresh/save_collection/sweeper.rb +69 -0
  33. data/lib/inventory_refresh/save_inventory.rb +18 -8
  34. data/lib/inventory_refresh/target_collection.rb +12 -0
  35. data/lib/inventory_refresh/version.rb +1 -1
  36. metadata +61 -19
  37. data/lib/inventory_refresh/save_collection/recursive.rb +0 -52
  38. data/lib/inventory_refresh/save_collection/saver/concurrent_safe.rb +0 -71
@@ -0,0 +1,249 @@
1
+ module InventoryRefresh
2
+ class InventoryCollection
3
+ class Builder
4
+ class MissingModelClassError < StandardError; end
5
+
6
+ def self.allowed_properties
7
+ %i(all_manager_uuids arel association
8
+ attributes_blacklist attributes_whitelist batch_extra_attributes
9
+ complete create_only custom_save_block
10
+ custom_reconnect_block default_values delete_method
11
+ dependency_attributes check_changed inventory_object_attributes
12
+ manager_ref manager_ref_allowed_nil manager_uuids
13
+ model_class name parent
14
+ parent_inventory_collections retention_strategy strategy
15
+ saver_strategy secondary_refs targeted
16
+ targeted_arel update_only use_ar_object
17
+ assert_graph_integrity).to_set
18
+ end
19
+
20
+ def allowed_properties
21
+ @allowed_properties ||= self.class.allowed_properties
22
+ end
23
+
24
+ # Default options for builder
25
+ # :adv_settings
26
+ # - values from Advanced settings (doesn't overwrite values specified in code)
27
+ # - @see method ManageIQ::Providers::Inventory::Persister.make_builder_settings()
28
+ # :shared_properties
29
+ # - any properties applied if missing (not explicitly specified)
30
+ def self.default_options
31
+ {
32
+ :shared_properties => {},
33
+ }
34
+ end
35
+
36
+ # Entry point
37
+ # Creates builder and builds data for inventory collection
38
+ # @param name [Symbol, Array] InventoryCollection.association value. <name> method not called when Array
39
+ # (optional) method with this name also used for concrete inventory collection specific properties
40
+ # @param persister_class [Class] used for "guessing" model_class
41
+ # @param options [Hash]
42
+ def self.prepare_data(name, persister_class, options = {})
43
+ options = default_options.merge(options)
44
+ builder = new(name, persister_class, options)
45
+ builder.construct_data
46
+
47
+ yield(builder) if block_given?
48
+
49
+ builder
50
+ end
51
+
52
+ # @see prepare_data()
53
+ def initialize(name, persister_class, options = self.class.default_options)
54
+ @name = name
55
+ @persister_class = persister_class
56
+
57
+ @properties = {}
58
+ @inventory_object_attributes = []
59
+ @default_values = {}
60
+ @dependency_attributes = {}
61
+
62
+ @options = options
63
+ skip_auto_inventory_attributes(false) if @options[:auto_inventory_attributes].nil?
64
+ skip_model_class(false) if @options[:without_model_class].nil?
65
+
66
+ @shared_properties = options[:shared_properties] # From persister
67
+ end
68
+
69
+ # Builds data for InventoryCollection
70
+ # Calls method @name (if exists) with specific properties
71
+ # Yields for overwriting provider-specific properties
72
+ def construct_data
73
+ add_properties({:association => @name}, :if_missing)
74
+
75
+ add_properties(@shared_properties, :if_missing)
76
+
77
+ send(@name.to_sym) if @name.respond_to?(:to_sym) && respond_to?(@name.to_sym)
78
+
79
+ if @properties[:model_class].nil?
80
+ add_properties(:model_class => auto_model_class) unless @options[:without_model_class]
81
+ end
82
+ end
83
+
84
+ # Creates InventoryCollection
85
+ def to_inventory_collection
86
+ if @properties[:model_class].nil? && !@options[:without_model_class]
87
+ raise MissingModelClassError, "Missing model_class for :#{@name} (\"#{@name.to_s.classify}\" or subclass expected)."
88
+ end
89
+
90
+ ::InventoryRefresh::InventoryCollection.new(to_hash)
91
+ end
92
+
93
+ #
94
+ # Missing method
95
+ # - add_some_property(value)
96
+ # converted to:
97
+ # - add_properties(:some_property => value)
98
+ #
99
+ def method_missing(method_name, *arguments, &block)
100
+ if method_name.to_s.starts_with?('add_')
101
+ add_properties(
102
+ method_name.to_s.gsub('add_', '').to_sym => arguments[0]
103
+ )
104
+ else
105
+ super
106
+ end
107
+ end
108
+
109
+ def respond_to_missing?(method_name, _include_private = false)
110
+ method_name.to_s.starts_with?('add_')
111
+ end
112
+
113
+ # Merges @properties
114
+ # @see ManagerRefresh::InventoryCollection.initialize for list of properties
115
+ #
116
+ # @param props [Hash]
117
+ # @param mode [Symbol] :overwrite | :if_missing
118
+ def add_properties(props = {}, mode = :overwrite)
119
+ props.each_key { |property_name| assert_allowed_property(property_name) }
120
+
121
+ @properties = merge_hashes(@properties, props, mode)
122
+ end
123
+
124
+ # Adds inventory object attributes (part of @properties)
125
+ def add_inventory_attributes(array)
126
+ @inventory_object_attributes += (array || [])
127
+ end
128
+
129
+ # Removes specified inventory object attributes
130
+ def remove_inventory_attributes(array)
131
+ @inventory_object_attributes -= (array || [])
132
+ end
133
+
134
+ # Clears all inventory object attributes
135
+ def clear_inventory_attributes!
136
+ @options[:auto_inventory_attributes] = false
137
+ @inventory_object_attributes = []
138
+ end
139
+
140
+ # Adds key/values to default values (InventoryCollection.default_values) (part of @properties)
141
+ def add_default_values(params = {}, mode = :overwrite)
142
+ @default_values = merge_hashes(@default_values, params, mode)
143
+ end
144
+
145
+ # Evaluates lambda blocks
146
+ def evaluate_lambdas!(persister)
147
+ @default_values = evaluate_lambdas_on(@default_values, persister)
148
+ @dependency_attributes = evaluate_lambdas_on(@dependency_attributes, persister)
149
+ end
150
+
151
+ # Adds key/values to dependency_attributes (part of @properties)
152
+ def add_dependency_attributes(attrs = {}, mode = :overwrite)
153
+ @dependency_attributes = merge_hashes(@dependency_attributes, attrs, mode)
154
+ end
155
+
156
+ # Deletes key from dependency_attributes
157
+ def remove_dependency_attributes(key)
158
+ @dependency_attributes.delete(key)
159
+ end
160
+
161
+ # Returns whole InventoryCollection properties
162
+ def to_hash
163
+ add_inventory_attributes(auto_inventory_attributes) if @options[:auto_inventory_attributes]
164
+
165
+ @properties[:inventory_object_attributes] ||= @inventory_object_attributes
166
+
167
+ @properties[:default_values] ||= {}
168
+ @properties[:default_values].merge!(@default_values)
169
+
170
+ @properties[:dependency_attributes] ||= {}
171
+ @properties[:dependency_attributes].merge!(@dependency_attributes)
172
+
173
+ @properties
174
+ end
175
+
176
+ protected
177
+
178
+ def assert_allowed_property(name)
179
+ unless allowed_properties.include?(name)
180
+ raise "InventoryCollection property :#{name} is not allowed. Allowed properties are:\n#{self.allowed_properties.to_a.map(&:to_s).join(', ')}"
181
+ end
182
+ end
183
+
184
+ # Extends source hash with
185
+ # - a) all keys from dest (overwrite mode)
186
+ # - b) missing keys (missing mode)
187
+ #
188
+ # @param mode [Symbol] :overwrite | :if_missing
189
+ def merge_hashes(source, dest, mode)
190
+ return source if source.nil? || dest.nil?
191
+
192
+ if mode == :overwrite
193
+ source.merge(dest)
194
+ else
195
+ dest.merge(source)
196
+ end
197
+ end
198
+
199
+ # Derives model_class from @name
200
+ # Can be disabled by options :without_model_class => true
201
+ # @return [Class | nil] when class doesn't exist, returns nil
202
+ def auto_model_class
203
+ "::#{@name.to_s.classify}".safe_constantize
204
+ end
205
+
206
+ # Enables/disables auto_model_class and exception check
207
+ # @param skip [Boolean]
208
+ def skip_model_class(skip = true)
209
+ @options[:without_model_class] = skip
210
+ end
211
+
212
+ # Inventory object attributes are derived from setters
213
+ #
214
+ # Can be disabled by options :auto_inventory_attributes => false
215
+ # - attributes can be manually set via method add_inventory_attributes()
216
+ def auto_inventory_attributes
217
+ return if @properties[:model_class].nil?
218
+
219
+ (@properties[:model_class].new.methods - ar_base_class.methods).grep(/^[\w]+?\=$/).collect do |setter|
220
+ setter.to_s[0..setter.length - 2].to_sym
221
+ end
222
+ end
223
+
224
+ # used for ignoring unrelated auto_inventory_attributes
225
+ def ar_base_class
226
+ ActiveRecord::Base
227
+ end
228
+
229
+ # Enables/disables auto_inventory_attributes
230
+ # @param skip [Boolean]
231
+ def skip_auto_inventory_attributes(skip = true)
232
+ @options[:auto_inventory_attributes] = !skip
233
+ end
234
+
235
+ # Evaluates lambda blocks in @default_values and @dependency_attributes
236
+ # @param values [Hash]
237
+ # @param persister [ManageIQ::Providers::Inventory::Persister]
238
+ def evaluate_lambdas_on(values, persister)
239
+ values&.transform_values do |value|
240
+ if value.respond_to?(:call)
241
+ value.call(persister)
242
+ else
243
+ value
244
+ end
245
+ end
246
+ end
247
+ end
248
+ end
249
+ end
@@ -52,26 +52,11 @@ module InventoryRefresh
52
52
  # depth 10. We should throw a warning maybe asking for simplifying the interconnections in the models.
53
53
  assert_graph!(edges)
54
54
 
55
- self.nodes = sort_nodes(nodes)
56
-
57
55
  self
58
56
  end
59
57
 
60
58
  private
61
59
 
62
- # Sort the nodes putting child nodes as last. Child nodes are InventoryCollection objects having
63
- # :parent_inventory_collections definition.
64
- #
65
- # @param nodes [Array<InventoryRefresh::InventoryCollection>] List of Inventory collection nodes
66
- # @return [Array<InventoryRefresh::InventoryCollection>] Sorted list of Inventory collection nodes
67
- def sort_nodes(nodes)
68
- # Separate to root nodes and child nodes, where child nodes are determined by having parent_inventory_collections
69
- root_nodes, child_nodes = nodes.partition { |node| node.parent_inventory_collections.blank? }
70
- # And order it that root nodes comes first, that way the disconnect of the child nodes should be always done as
71
- # a part of the root nodes, as intended.
72
- root_nodes + child_nodes
73
- end
74
-
75
60
  # Asserts all nodes are of class ::InventoryRefresh::InventoryCollection
76
61
  #
77
62
  # @param inventory_collections [Array<InventoryRefresh::InventoryCollection>] List of Inventory collection nodes
@@ -0,0 +1,6 @@
1
+ module InventoryRefresh
2
+ class InventoryCollection
3
+ module Helpers
4
+ end
5
+ end
6
+ end
@@ -0,0 +1,80 @@
1
+ require_relative "../helpers"
2
+
3
+ module InventoryRefresh
4
+ class InventoryCollection
5
+ module Helpers
6
+ module AssociationsHelper
7
+ # @return [Array<ActiveRecord::Reflection::BelongsToReflection">] All belongs_to associations
8
+ def belongs_to_associations
9
+ model_class.reflect_on_all_associations.select { |x| x.kind_of?(ActiveRecord::Reflection::BelongsToReflection) }
10
+ end
11
+
12
+ # @return [Hash{Symbol => String}] Hash with association name mapped to foreign key column name
13
+ def association_to_foreign_key_mapping
14
+ return {} unless model_class
15
+
16
+ @association_to_foreign_key_mapping ||= belongs_to_associations.each_with_object({}) do |x, obj|
17
+ obj[x.name] = x.foreign_key
18
+ end
19
+ end
20
+
21
+ # @return [Hash{String => Hash}] Hash with foreign_key column name mapped to association name
22
+ def foreign_key_to_association_mapping
23
+ return {} unless model_class
24
+
25
+ @foreign_key_to_association_mapping ||= belongs_to_associations.each_with_object({}) do |x, obj|
26
+ obj[x.foreign_key] = x.name
27
+ end
28
+ end
29
+
30
+ # @return [Hash{Symbol => String}] Hash with association name mapped to polymorphic foreign key type column name
31
+ def association_to_foreign_type_mapping
32
+ return {} unless model_class
33
+
34
+ @association_to_foreign_type_mapping ||= model_class.reflect_on_all_associations.each_with_object({}) do |x, obj|
35
+ obj[x.name] = x.foreign_type if x.polymorphic?
36
+ end
37
+ end
38
+
39
+ # @return [Hash{Symbol => String}] Hash with polymorphic foreign key type column name mapped to association name
40
+ def foreign_type_to_association_mapping
41
+ return {} unless model_class
42
+
43
+ @foreign_type_to_association_mapping ||= model_class.reflect_on_all_associations.each_with_object({}) do |x, obj|
44
+ obj[x.foreign_type] = x.name if x.polymorphic?
45
+ end
46
+ end
47
+
48
+ # @return [Hash{Symbol => String}] Hash with association name mapped to base class of the association
49
+ def association_to_base_class_mapping
50
+ return {} unless model_class
51
+
52
+ @association_to_base_class_mapping ||= model_class.reflect_on_all_associations.each_with_object({}) do |x, obj|
53
+ obj[x.name] = x.klass.base_class.name unless x.polymorphic?
54
+ end
55
+ end
56
+
57
+ # @return [Array<Symbol>] List of all column names that are foreign keys
58
+ def foreign_keys
59
+ return [] unless model_class
60
+
61
+ @foreign_keys_cache ||= belongs_to_associations.map(&:foreign_key).map!(&:to_sym)
62
+ end
63
+
64
+ # @return [Array<Symbol>] List of all column names that are foreign keys and cannot removed, otherwise we couldn't
65
+ # save the record
66
+ def fixed_foreign_keys
67
+ # Foreign keys that are part of a manager_ref must be present, otherwise the record would get lost. This is a
68
+ # minimum check we can do to not break a referential integrity.
69
+ return @fixed_foreign_keys_cache unless @fixed_foreign_keys_cache.nil?
70
+
71
+ manager_ref_set = (manager_ref - manager_ref_allowed_nil)
72
+ @fixed_foreign_keys_cache = manager_ref_set.map { |x| association_to_foreign_key_mapping[x] }.compact
73
+ @fixed_foreign_keys_cache += foreign_keys & manager_ref
74
+ @fixed_foreign_keys_cache.map!(&:to_sym)
75
+ @fixed_foreign_keys_cache
76
+ end
77
+ end
78
+ end
79
+ end
80
+ end
@@ -0,0 +1,456 @@
1
+ require_relative "../helpers"
2
+
3
+ module InventoryRefresh
4
+ class InventoryCollection
5
+ module Helpers
6
+ module InitializeHelper
7
+ # @param association [Symbol] A Rails association callable on a :parent attribute is used for comparing with the
8
+ # objects in the DB, to decide if the InventoryObjects will be created/deleted/updated or used for obtaining
9
+ # the data from a DB, if a DB strategy is used. It returns objects of the :model_class class or its sub STI.
10
+ # @param model_class [Class] A class of an ApplicationRecord model, that we want to persist into the DB or load from
11
+ # the DB.
12
+ # @param name [Symbol] A unique name of the InventoryCollection under a Persister. If not provided, the :association
13
+ # attribute is used. If :association is nil as well, the :name will be inferred from the :model_class.
14
+ # @param parent [ApplicationRecord] An ApplicationRecord object that has a callable :association method returning
15
+ # the objects of a :model_class.
16
+ def init_basic_properties(association, model_class, name, parent)
17
+ @association = association
18
+ @model_class = model_class
19
+ @name = name || association || model_class.to_s.demodulize.tableize
20
+ @parent = parent || nil
21
+ end
22
+
23
+ # @param strategy [Symbol] A strategy of the InventoryCollection that will be used for saving/loading of the
24
+ # InventoryObject objects.
25
+ # Allowed strategies are:
26
+ # - nil => InventoryObject objects of the InventoryCollection will be saved to the DB, only these objects
27
+ # will be referable from the other InventoryCollection objects.
28
+ # - :local_db_cache_all => Loads InventoryObject objects from the database, it loads all the objects that
29
+ # are a result of a [<:parent>.<:association>, :arel] taking
30
+ # first defined in this order. This strategy will not save any objects in the DB.
31
+ # - :local_db_find_references => Loads InventoryObject objects from the database, it loads only objects that
32
+ # were referenced by the other InventoryCollections using a filtered result
33
+ # of a [<:parent>.<:association>, :arel] taking first
34
+ # defined in this order. This strategy will not save any objects in the DB.
35
+ # - :local_db_find_missing_references => InventoryObject objects of the InventoryCollection will be saved to
36
+ # the DB. Then if we reference an object that is not present, it will
37
+ # load them from the db using :local_db_find_references strategy.
38
+ # @param saver_strategy [Symbol] A strategy that will be used for InventoryCollection persisting into the DB.
39
+ # Allowed saver strategies are:
40
+ # - :default => Using Rails saving methods, this way is not safe to run in multiple workers concurrently,
41
+ # since it will lead to non consistent data.
42
+ # - :batch => Using batch SQL queries, this way is not safe to run in multiple workers
43
+ # concurrently, since it will lead to non consistent data.
44
+ # - :concurrent_safe_batch => It uses atomic upsert to avoid data duplication and it uses timestamp based
45
+ # atomic checks to avoid new data being overwritten by the the old data. The upsert/update queries are
46
+ # executed as batched SQL queries, instead of sending 1 query per record.
47
+ # @param retention_strategy [Symbol] A retention strategy for this collection. Allowed values are:
48
+ # - :destroy => Will destroy the inactive records.
49
+ # - :archive => Will archive the inactive records by setting :archived_at timestamp.
50
+ # @param delete_method [Symbol] A delete method that will be used for deleting of the InventoryObject, if the
51
+ # object is marked for deletion. A default is :destroy, the instance method must be defined on the
52
+ # :model_class.
53
+ def init_strategies(strategy, saver_strategy, retention_strategy, delete_method)
54
+ @saver_strategy = process_saver_strategy(saver_strategy)
55
+ @strategy = process_strategy(strategy)
56
+ @retention_strategy = process_retention_strategy(retention_strategy)
57
+ @delete_method = delete_method || :destroy
58
+ end
59
+
60
+ # @param manager_ref [Array] Array of Symbols, that are keys of the InventoryObject's data, inserted into this
61
+ # InventoryCollection. Using these keys, we need to be able to uniquely identify each of the InventoryObject
62
+ # objects inside.
63
+ # @param manager_ref_allowed_nil [Array] Array of symbols having manager_ref columns, that are a foreign key an can
64
+ # be nil. Given the table are shared by many providers, it can happen, that the table is used only partially.
65
+ # Then it can happen we want to allow certain foreign keys to be nil, while being sure the referential
66
+ # integrity is not broken. Of course the DB Foreign Key can't be created in this case, so we should try to
67
+ # avoid this usecase by a proper modeling.
68
+ # Note that InventoryObject's data has to be build with <foreign_key> => nil, it means that key cannot be missing!
69
+ # @param secondary_refs [Hash] TODO
70
+ # @param manager_uuids [Array|Proc] Array of manager_uuids of the InventoryObjects we want to create/update/delete. Using
71
+ # this attribute, the db_collection_for_comparison will be automatically limited by the manager_uuids, in a
72
+ # case of a simple relation. In a case of a complex relation, we can leverage :manager_uuids in a
73
+ # custom :targeted_arel. We can pass also lambda, for lazy_evaluation.
74
+ def init_references(manager_ref, manager_ref_allowed_nil, secondary_refs, manager_uuids)
75
+ @manager_ref = manager_ref || %i(ems_ref)
76
+ @manager_ref_allowed_nil = manager_ref_allowed_nil || []
77
+ @secondary_refs = secondary_refs || {}
78
+ @manager_uuids = manager_uuids || []
79
+ end
80
+
81
+ # @param all_manager_uuids [Array] Array of all manager_uuids of the InventoryObjects. With the :targeted true,
82
+ # having this parameter defined will invoke only :delete_method on a complement of this set, making sure
83
+ # the DB has only this set of data after. This :attribute serves for deleting of top level
84
+ # InventoryCollections, i.e. InventoryCollections having parent_inventory_collections nil. The deleting of
85
+ # child collections is already handled by the scope of the parent_inventory_collections and using Rails
86
+ # :dependent => :destroy,
87
+ # @param all_manager_uuids_scope [Array] A scope limiting the :all_manager_uuids parameter. E.g. we can send
88
+ # all_manager_uuids for 1 region, leading to delete a complement of the entities just under that 1
89
+ # region.
90
+ # If all_manager_uuids_scope is used with :all_manager_uuids => nil, it will do delete_complement of the
91
+ # scope itself. E.g. sending a list of all active regions, we will delete complement entities not
92
+ # belonging to those regions.
93
+ # Example:
94
+ # :all_manager_uuids => [{:source_ref => x}, {:source_ref => y}],
95
+ # :all_manager_uuids_scope => [{:region => regions.lazy_find(X)}, {:region => regions.lazy_find(Y)}]
96
+ #
97
+ # Will cause deletion/archival or all entities that don't have source_ref "x" or "y", but only under
98
+ # regions X and Y.
99
+ # @param all_manager_uuids_timestamp [String] A timestamp in UTC marking a time before we collected all of the
100
+ # all_manager_uuids. Meaning we won't be archiving any newer entities.
101
+ def init_all_manager_uuids(all_manager_uuids, all_manager_uuids_scope, all_manager_uuids_timestamp)
102
+ # TODO(lsmola) Should we refactor this to use references too?
103
+ @all_manager_uuids = all_manager_uuids
104
+ @all_manager_uuids_scope = all_manager_uuids_scope
105
+ @all_manager_uuids_timestamp = all_manager_uuids_timestamp
106
+ end
107
+
108
+ # @param dependency_attributes [Hash] Manually defined dependencies of this InventoryCollection. We can use this
109
+ # by manually place the InventoryCollection into the graph, to make sure the saving is invoked after the
110
+ # dependencies were saved. The dependencies itself are InventoryCollection objects. For a common use-cases
111
+ # we do not need to define dependencies manually, since those are inferred automatically by scanning of the
112
+ # data.
113
+ #
114
+ # Example:
115
+ # :dependency_attributes => {
116
+ # :orchestration_stacks => [collections[:orchestration_stacks]],
117
+ # :orchestration_stacks_resources => [collections[:orchestration_stacks_resources]]
118
+ # }
119
+ # This example is used in Example2 of the <param custom_save_block> and it means that our :custom_save_block
120
+ # will be invoked after the InventoryCollection :orchestration_stacks and :orchestration_stacks_resources
121
+ # are saved.
122
+ # @param parent_inventory_collections [Array] Array of symbols having a name pointing to the
123
+ # InventoryRefresh::InventoryCollection objects, that serve as parents to this InventoryCollection. There are
124
+ # several scenarios to consider, when deciding if InventoryCollection has parent collections, see the example.
125
+ #
126
+ # Example:
127
+ # taking inventory collections :vms and :disks (local disks), if we write that:
128
+ # inventory_collection = InventoryCollection.new({
129
+ # :model_class => ::Disk,
130
+ # :association => :disks,
131
+ # :manager_ref => [:vm, :location]
132
+ # :parent_inventory_collection => [:vms],
133
+ # })
134
+ #
135
+ # Then the decision for having :parent_inventory_collection => [:vms] was probably driven by these
136
+ # points:
137
+ # 1. We can get list of all disks only by doing SQL query through the parent object (so there will be join
138
+ # from vms to disks table).
139
+ # 2. There is no API query for getting all disks from the provider API, we get them inside VM data, or as
140
+ # a Vm subquery
141
+ # 3. Part of the manager_ref of the IC is the VM object (foreign key), so the disk's location is unique
142
+ # only under 1 Vm. (In current models, this modeled going through Hardware model)
143
+ # 4. In targeted refresh, we always expect that each Vm will be saved with all its disks.
144
+ #
145
+ # Then having the above points, adding :parent_inventory_collection => [:vms], will bring these
146
+ # implications:
147
+ # 1. By archiving/deleting Vm, we can no longer see the disk, because those were owned by the Vm. Any
148
+ # archival/deletion of the Disk model, must be then done by cascade delete/hooks logic.
149
+ # 2. Having Vm as a parent ensures we always process it first. So e.g. when providing no Vms for saving
150
+ # we would have no graph dependency (no data --> no edges --> no dependencies) and Disk could be
151
+ # archived/removed before the Vm, while we always want to archive the VM first.
152
+ # 3. For targeted refresh, we always expect that all disks are saved with a VM. So for targeting :disks,
153
+ # we are not using #manager_uuids attribute, since the scope is "all disks of all targeted VMs", so we
154
+ # always use #manager_uuids of the parent. (that is why :parent_inventory_collections and
155
+ # :manager_uuids are mutually exclusive attributes)
156
+ # 4. For automatically building the #targeted_arel query, we need the parent to know what is the root node.
157
+ # While this information can be introspected from the data, it creates a scope for create&update&delete,
158
+ # which means it has to work with no data provided (causing delete all). So with no data we cannot
159
+ # introspect anything.
160
+ def init_ic_relations(dependency_attributes, parent_inventory_collections = nil)
161
+ @dependency_attributes = dependency_attributes || {}
162
+ @dependees = Set.new
163
+ @parent_inventory_collections = parent_inventory_collections
164
+ end
165
+
166
+ # @param complete [Boolean] By default true, :complete is marking we are sending a complete dataset and therefore
167
+ # we can create/update/delete the InventoryObject objects. If :complete is false we will only do
168
+ # create/update without delete.
169
+ # @param create_only [Boolean] TODO
170
+ # @param check_changed [Boolean] By default true. If true, before updating the InventoryObject, we call Rails
171
+ # 'changed?' method. This can optimize speed of updates heavily, but it can fail to recognize the change for
172
+ # e.g. Ancestry and Relationship based columns. If false, we always update the InventoryObject.
173
+ # @param update_only [Boolean] By default false. If true we only update the InventoryObject objects, if false we do
174
+ # create/update/delete.
175
+ # @param use_ar_object [Boolean] True or False. Whether we need to initialize AR object as part of the saving
176
+ # it's needed if the model have special setters, serialize of columns, etc. This setting is relevant only
177
+ # for the batch saver strategy.
178
+ # @param targeted [Boolean] True if the collection is targeted, in that case it will be leveraging :manager_uuids
179
+ # :parent_inventory_collections and :targeted_arel to save a subgraph of a data.
180
+ def init_flags(complete, create_only, check_changed,
181
+ update_only, use_ar_object, targeted,
182
+ assert_graph_integrity)
183
+ @complete = complete.nil? ? true : complete
184
+ @create_only = create_only.nil? ? false : create_only
185
+ @check_changed = check_changed.nil? ? true : check_changed
186
+ @saved = false
187
+ @update_only = update_only.nil? ? false : update_only
188
+ @use_ar_object = use_ar_object || false
189
+ @targeted = !!targeted
190
+ @assert_graph_integrity = assert_graph_integrity.nil? ? true : assert_graph_integrity
191
+ end
192
+
193
+ # @param attributes_blacklist [Array] Attributes we do not want to include into saving. We cannot blacklist an
194
+ # attribute that is needed for saving of the object.
195
+ # Note: attributes_blacklist is also used for internal resolving of the cycles in the graph.
196
+ #
197
+ # In the Example2 of the <param custom_save_block>, we have a custom saving code, that saves a :parent
198
+ # attribute of the OrchestrationStack. That means we don't want that attribute saved as a part of
199
+ # InventoryCollection for OrchestrationStack, so we would set :attributes_blacklist => [:parent]. Then the
200
+ # :parent will be ignored while saving.
201
+ # @param attributes_whitelist [Array] Same usage as the :attributes_blacklist, but defining full set of attributes
202
+ # that should be saved. Attributes that are part of :manager_ref and needed validations are automatically
203
+ # added.
204
+ # @param inventory_object_attributes [Array] Array of attribute names that will be exposed as readers/writers on the
205
+ # InventoryObject objects inside.
206
+ #
207
+ # Example: Given
208
+ # inventory_collection = InventoryCollection.new({
209
+ # :model_class => ::Vm,
210
+ # :arel => @ems.vms,
211
+ # :inventory_object_attributes => [:name, :label]
212
+ # })
213
+ # And building the inventory_object like:
214
+ # inventory_object = inventory_collection.build(:ems_ref => "vm1", :name => "vm1")
215
+ # We can use inventory_object_attributes as setters and getters:
216
+ # inventory_object.name = "Name"
217
+ # inventory_object.label = inventory_object.name
218
+ # Which would be equivalent to less nicer way:
219
+ # inventory_object[:name] = "Name"
220
+ # inventory_object[:label] = inventory_object[:name]
221
+ # So by using inventory_object_attributes, we will be guarding the allowed attributes and will have an
222
+ # explicit list of allowed attributes, that can be used also for documentation purposes.
223
+ # @param batch_extra_attributes [Array] Array of symbols marking which extra attributes we want to store into the
224
+ # db. These extra attributes might be a product of :use_ar_object assignment and we need to specify them
225
+ # manually, if we want to use a batch saving strategy and we have models that populate attributes as a side
226
+ # effect.
227
+ def init_model_attributes(attributes_blacklist, attributes_whitelist,
228
+ inventory_object_attributes, batch_extra_attributes)
229
+ @attributes_blacklist = Set.new
230
+ @attributes_whitelist = Set.new
231
+ @batch_extra_attributes = batch_extra_attributes || []
232
+ @inventory_object_attributes = inventory_object_attributes
233
+ @internal_attributes = %i(__feedback_edge_set_parent __parent_inventory_collections __all_manager_uuids_scope)
234
+ @transitive_dependency_attributes = Set.new
235
+
236
+ blacklist_attributes!(attributes_blacklist) if attributes_blacklist.present?
237
+ whitelist_attributes!(attributes_whitelist) if attributes_whitelist.present?
238
+ end
239
+
240
+ def init_storages
241
+ @data_storage = ::InventoryRefresh::InventoryCollection::DataStorage.new(self, @secondary_refs)
242
+ @references_storage = ::InventoryRefresh::InventoryCollection::ReferencesStorage.new(index_proxy)
243
+ @targeted_scope = ::InventoryRefresh::InventoryCollection::ReferencesStorage.new(index_proxy).merge!(@manager_uuids)
244
+ end
245
+
246
+ # @param arel [ActiveRecord::Associations::CollectionProxy|Arel::SelectManager] Instead of :parent and :association
247
+ # we can provide Arel directly to say what records should be compared to check if InventoryObject will be
248
+ # doing create/update/delete.
249
+ #
250
+ # Example:
251
+ # for a targeted refresh, we want to delete/update/create only a list of vms specified with a list of
252
+ # ems_refs:
253
+ # :arel => manager.vms.where(:ems_ref => manager_refs)
254
+ # Then we want to do the same for the hardwares of only those vms:
255
+ # :arel => manager.hardwares.joins(:vm_or_template).where(
256
+ # 'vms' => {:ems_ref => manager_refs}
257
+ # )
258
+ # And etc. for the other Vm related records.
259
+ # @param targeted_arel [Proc] A callable block that receives this InventoryCollection as a first argument. In there
260
+ # we can leverage a :parent_inventory_collections or :manager_uuids to limit the query based on the
261
+ # manager_uuids available.
262
+ # Example:
263
+ # targeted_arel = lambda do |inventory_collection|
264
+ # # Getting ems_refs of parent :vms and :miq_templates
265
+ # manager_uuids = inventory_collection.parent_inventory_collections.collect(&:manager_uuids).flatten
266
+ # inventory_collection.db_collection_for_comparison.hardwares.joins(:vm_or_template).where(
267
+ # 'vms' => {:ems_ref => manager_uuids}
268
+ # )
269
+ # end
270
+ #
271
+ # inventory_collection = InventoryCollection.new({
272
+ # :model_class => ::Hardware,
273
+ # :association => :hardwares,
274
+ # :parent_inventory_collection => [:vms, :miq_templates],
275
+ # :targeted_arel => targeted_arel,
276
+ # })
277
+ def init_arels(arel, targeted_arel)
278
+ @arel = arel
279
+ @targeted_arel = targeted_arel
280
+ end
281
+
282
+ # @param custom_save_block [Proc] A custom lambda/proc for persisting in the DB, for cases where it's not enough
283
+ # to just save every InventoryObject inside by the defined rules and default saving algorithm.
284
+ #
285
+ # Example1 - saving SomeModel in my own ineffective way :-) :
286
+ #
287
+ # custom_save = lambda do |_ems, inventory_collection|
288
+ # inventory_collection.each |inventory_object| do
289
+ # hash = inventory_object.attributes # Loads possible dependencies into saveable hash
290
+ # obj = SomeModel.find_by(:attr => hash[:attr]) # Note: doing find_by for many models produces N+1
291
+ # # queries, avoid this, this is just a simple example :-)
292
+ # obj.update_attributes(hash) if obj
293
+ # obj ||= SomeModel.create(hash)
294
+ # inventory_object.id = obj.id # If this InventoryObject is referenced elsewhere, we need to store its
295
+ # primary key back to the InventoryObject
296
+ # end
297
+ #
298
+ # Example2 - saving parent OrchestrationStack in a more effective way, than the default saving algorithm can
299
+ # achieve. Ancestry gem requires an ActiveRecord object for association and is not defined as a proper
300
+ # ActiveRecord association. That leads in N+1 queries in the default saving algorithm, so we can do better
301
+ # with custom saving for now. The InventoryCollection is defined as a custom dependencies processor,
302
+ # without its own :model_class and InventoryObjects inside:
303
+ #
304
+ # InventoryRefresh::InventoryCollection.new({
305
+ # :association => :orchestration_stack_ancestry,
306
+ # :custom_save_block => orchestration_stack_ancestry_save_block,
307
+ # :dependency_attributes => {
308
+ # :orchestration_stacks => [collections[:orchestration_stacks]],
309
+ # :orchestration_stacks_resources => [collections[:orchestration_stacks_resources]]
310
+ # }
311
+ # })
312
+ #
313
+ # And the lambda is defined as:
314
+ #
315
+ # orchestration_stack_ancestry_save_block = lambda do |_ems, inventory_collection|
316
+ # stacks_inventory_collection = inventory_collection.dependency_attributes[:orchestration_stacks].try(:first)
317
+ #
318
+ # return if stacks_inventory_collection.blank?
319
+ #
320
+ # stacks_parents = stacks_inventory_collection.data.each_with_object({}) do |x, obj|
321
+ # parent_id = x.data[:parent].load.try(:id)
322
+ # obj[x.id] = parent_id if parent_id
323
+ # end
324
+ #
325
+ # model_class = stacks_inventory_collection.model_class
326
+ #
327
+ # stacks_parents_indexed = model_class
328
+ # .select([:id, :ancestry])
329
+ # .where(:id => stacks_parents.values).find_each.index_by(&:id)
330
+ #
331
+ # model_class
332
+ # .select([:id, :ancestry])
333
+ # .where(:id => stacks_parents.keys).find_each do |stack|
334
+ # parent = stacks_parents_indexed[stacks_parents[stack.id]]
335
+ # stack.update_attribute(:parent, parent)
336
+ # end
337
+ # end
338
+ # @param custom_reconnect_block [Proc] A custom lambda for reconnect logic of previously disconnected records
339
+ #
340
+ # Example - Reconnect disconnected Vms
341
+ # InventoryRefresh::InventoryCollection.new({
342
+ # :association => :orchestration_stack_ancestry,
343
+ # :custom_reconnect_block => vms_custom_reconnect_block,
344
+ # })
345
+ #
346
+ # And the lambda is defined as:
347
+ #
348
+ # vms_custom_reconnect_block = lambda do |inventory_collection, inventory_objects_index, attributes_index|
349
+ # inventory_objects_index.each_slice(1000) do |batch|
350
+ # Vm.where(:ems_ref => batch.map(&:second).map(&:manager_uuid)).each do |record|
351
+ # index = inventory_collection.object_index_with_keys(inventory_collection.manager_ref_to_cols, record)
352
+ #
353
+ # # We need to delete the record from the inventory_objects_index and attributes_index, otherwise it
354
+ # # would be sent for create.
355
+ # inventory_object = inventory_objects_index.delete(index)
356
+ # hash = attributes_index.delete(index)
357
+ #
358
+ # record.assign_attributes(hash.except(:id, :type))
359
+ # if !inventory_collection.check_changed? || record.changed?
360
+ # record.save!
361
+ # inventory_collection.store_updated_records(record)
362
+ # end
363
+ #
364
+ # inventory_object.id = record.id
365
+ # end
366
+ # end
367
+ def init_custom_procs(custom_save_block, custom_reconnect_block)
368
+ @custom_save_block = custom_save_block
369
+ @custom_reconnect_block = custom_reconnect_block
370
+ end
371
+
372
+ # @param default_values [Hash] A hash of an attributes that will be added to every inventory object created by
373
+ # inventory_collection.build(hash)
374
+ #
375
+ # Example: Given
376
+ # inventory_collection = InventoryCollection.new({
377
+ # :model_class => ::Vm,
378
+ # :arel => @ems.vms,
379
+ # :default_values => {:ems_id => 10}
380
+ # })
381
+ # And building the inventory_object like:
382
+ # inventory_object = inventory_collection.build(:ems_ref => "vm_1", :name => "vm1")
383
+ # The inventory_object.data will look like:
384
+ # {:ems_ref => "vm_1", :name => "vm1", :ems_id => 10}
385
+ def init_data(default_values)
386
+ @default_values = default_values || {}
387
+ end
388
+
389
+ def init_changed_records_stats
390
+ @created_records = []
391
+ @updated_records = []
392
+ @deleted_records = []
393
+ end
394
+
395
+ # Processes passed saver strategy
396
+ #
397
+ # @param saver_strategy [Symbol] Passed saver strategy
398
+ # @return [Symbol] Returns back the passed strategy if supported, or raises exception
399
+ def process_saver_strategy(saver_strategy)
400
+ return :default unless saver_strategy
401
+
402
+ saver_strategy = saver_strategy.to_sym
403
+ case saver_strategy
404
+ when :default, :batch, :concurrent_safe_batch
405
+ saver_strategy
406
+ else
407
+ raise "Unknown InventoryCollection saver strategy: :#{saver_strategy}, allowed strategies are "\
408
+ ":default, :batch and :concurrent_safe_batch"
409
+ end
410
+ end
411
+
412
+ # Processes passed strategy, modifies :data_collection_finalized and :saved attributes for db only strategies
413
+ #
414
+ # @param strategy_name [Symbol] Passed saver strategy
415
+ # @return [Symbol] Returns back the passed strategy if supported, or raises exception
416
+ def process_strategy(strategy_name)
417
+ self.data_collection_finalized = false
418
+
419
+ return unless strategy_name
420
+
421
+ strategy_name = strategy_name.to_sym
422
+ case strategy_name
423
+ when :local_db_cache_all
424
+ self.data_collection_finalized = true
425
+ self.saved = true
426
+ when :local_db_find_references
427
+ self.saved = true
428
+ when :local_db_find_missing_references
429
+ nil
430
+ else
431
+ raise "Unknown InventoryCollection strategy: :#{strategy_name}, allowed strategies are :local_db_cache_all, "\
432
+ ":local_db_find_references and :local_db_find_missing_references."
433
+ end
434
+ strategy_name
435
+ end
436
+
437
+ # Processes passed retention strategy
438
+ #
439
+ # @param retention_strategy [Symbol] Passed retention strategy
440
+ # @return [Symbol] Returns back the passed strategy if supported, or raises exception
441
+ def process_retention_strategy(retention_strategy)
442
+ return unless retention_strategy
443
+
444
+ retention_strategy = retention_strategy.to_sym
445
+ case retention_strategy
446
+ when :destroy, :archive
447
+ retention_strategy
448
+ else
449
+ raise "Unknown InventoryCollection retention strategy: :#{retention_strategy}, allowed strategies are "\
450
+ ":destroy and :archive"
451
+ end
452
+ end
453
+ end
454
+ end
455
+ end
456
+ end