inventory_refresh 0.1.2 → 0.2.3
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.codeclimate.yml +1 -0
- data/.gitignore +6 -0
- data/.travis.yml +3 -3
- data/Gemfile +4 -0
- data/inventory_refresh.gemspec +7 -5
- data/lib/inventory_refresh.rb +1 -0
- data/lib/inventory_refresh/inventory_collection.rb +114 -649
- data/lib/inventory_refresh/inventory_collection/builder.rb +249 -0
- data/lib/inventory_refresh/inventory_collection/graph.rb +0 -15
- data/lib/inventory_refresh/inventory_collection/helpers.rb +6 -0
- data/lib/inventory_refresh/inventory_collection/helpers/associations_helper.rb +80 -0
- data/lib/inventory_refresh/inventory_collection/helpers/initialize_helper.rb +456 -0
- data/lib/inventory_refresh/inventory_collection/helpers/questions_helper.rb +132 -0
- data/lib/inventory_refresh/inventory_collection/index/type/skeletal.rb +5 -5
- data/lib/inventory_refresh/inventory_collection/reference.rb +4 -0
- data/lib/inventory_refresh/inventory_collection/scanner.rb +111 -18
- data/lib/inventory_refresh/inventory_collection/serialization.rb +7 -7
- data/lib/inventory_refresh/inventory_collection/unconnected_edge.rb +19 -0
- data/lib/inventory_refresh/inventory_object.rb +17 -11
- data/lib/inventory_refresh/inventory_object_lazy.rb +20 -10
- data/lib/inventory_refresh/persister.rb +212 -0
- data/lib/inventory_refresh/save_collection/base.rb +18 -3
- data/lib/inventory_refresh/save_collection/saver/base.rb +25 -62
- data/lib/inventory_refresh/save_collection/saver/concurrent_safe_batch.rb +73 -225
- data/lib/inventory_refresh/save_collection/saver/partial_upsert_helper.rb +226 -0
- data/lib/inventory_refresh/save_collection/saver/retention_helper.rb +115 -0
- data/lib/inventory_refresh/save_collection/saver/sql_helper.rb +122 -0
- data/lib/inventory_refresh/save_collection/saver/sql_helper_update.rb +24 -5
- data/lib/inventory_refresh/save_collection/saver/sql_helper_upsert.rb +6 -6
- data/lib/inventory_refresh/save_collection/sweeper.rb +69 -0
- data/lib/inventory_refresh/save_inventory.rb +18 -8
- data/lib/inventory_refresh/target_collection.rb +12 -0
- data/lib/inventory_refresh/version.rb +1 -1
- metadata +73 -25
- data/lib/inventory_refresh/save_collection/recursive.rb +0 -52
- data/lib/inventory_refresh/save_collection/saver/concurrent_safe.rb +0 -71
@@ -0,0 +1,115 @@
|
|
1
|
+
module InventoryRefresh::SaveCollection
|
2
|
+
module Saver
|
3
|
+
module RetentionHelper
|
4
|
+
private
|
5
|
+
|
6
|
+
# Deletes a complement of referenced data
|
7
|
+
def delete_complement
|
8
|
+
return unless inventory_collection.delete_allowed?
|
9
|
+
|
10
|
+
all_manager_uuids_size = inventory_collection.all_manager_uuids.size
|
11
|
+
|
12
|
+
logger.debug("Processing :delete_complement of #{inventory_collection} of size "\
|
13
|
+
"#{all_manager_uuids_size}...")
|
14
|
+
|
15
|
+
query = complement_of!(inventory_collection.all_manager_uuids,
|
16
|
+
inventory_collection.all_manager_uuids_scope,
|
17
|
+
inventory_collection.all_manager_uuids_timestamp)
|
18
|
+
|
19
|
+
ids_of_non_active_entities = ActiveRecord::Base.connection.execute(query.to_sql).to_a
|
20
|
+
ids_of_non_active_entities.each_slice(10_000) do |batch|
|
21
|
+
destroy_records!(batch)
|
22
|
+
end
|
23
|
+
|
24
|
+
logger.debug("Processing :delete_complement of #{inventory_collection} of size "\
|
25
|
+
"#{all_manager_uuids_size}, deleted=#{inventory_collection.deleted_records.size}...Complete")
|
26
|
+
end
|
27
|
+
|
28
|
+
# Applies strategy based on :retention_strategy parameter, or fallbacks to legacy_destroy_records.
|
29
|
+
#
|
30
|
+
# @param records [Array<ApplicationRecord, Hash, Array>] Records we want to delete or archive
|
31
|
+
def destroy_records!(records)
|
32
|
+
# TODO(lsmola) the output of this can still grow in a memory a lot, if we would delete a huge chunk of
|
33
|
+
# records. Will we just stream it out? Or maybe give a max amount of deleted records here?
|
34
|
+
|
35
|
+
return false unless inventory_collection.delete_allowed?
|
36
|
+
return if records.blank?
|
37
|
+
|
38
|
+
if inventory_collection.retention_strategy
|
39
|
+
ids = ids_array(records)
|
40
|
+
inventory_collection.store_deleted_records(ids)
|
41
|
+
send("#{inventory_collection.retention_strategy}_all_records!", ids)
|
42
|
+
else
|
43
|
+
legacy_destroy_records!(records)
|
44
|
+
end
|
45
|
+
end
|
46
|
+
|
47
|
+
# Convert records to list of ids in format [{:id => X}, {:id => Y}...]
|
48
|
+
#
|
49
|
+
# @param records [Array<ApplicationRecord, Hash, Array>] Records we want to delete or archive
|
50
|
+
# @return [Array<Hash>] Primary keys in standardized format
|
51
|
+
def ids_array(records)
|
52
|
+
if records.first.kind_of?(Hash)
|
53
|
+
records.map { |x| {:id => x[primary_key]} }
|
54
|
+
elsif records.first.kind_of?(Array)
|
55
|
+
records.map { |x| {:id => x[select_keys_indexes[primary_key]]} }
|
56
|
+
else
|
57
|
+
records.map { |x| {:id => x.public_send(primary_key)} }
|
58
|
+
end
|
59
|
+
end
|
60
|
+
|
61
|
+
# Archives all records
|
62
|
+
#
|
63
|
+
# @param records [Array<Hash>] Records we want to archive.
|
64
|
+
def archive_all_records!(records)
|
65
|
+
inventory_collection.model_class.where(:id => records.map { |x| x[:id] }).update_all(:archived_at => Time.now.utc)
|
66
|
+
end
|
67
|
+
|
68
|
+
# Destroys all records
|
69
|
+
#
|
70
|
+
# @param records [Array<Hash>] Records we want to delete.
|
71
|
+
def destroy_all_records!(records)
|
72
|
+
inventory_collection.model_class.where(:id => records.map { |x| x[:id] }).delete_all
|
73
|
+
end
|
74
|
+
|
75
|
+
# Deletes or sof-deletes records. If the model_class supports a custom class delete method, we will use it for
|
76
|
+
# batch soft-delete. This is the legacy method doing either ineffective deletion/archiving or requiring a method
|
77
|
+
# on a class.
|
78
|
+
#
|
79
|
+
# @param records [Array<ApplicationRecord, Hash>] Records we want to delete. If we have only hashes, we need to
|
80
|
+
# to fetch ApplicationRecord objects from the DB
|
81
|
+
def legacy_destroy_records!(records)
|
82
|
+
# Is the delete_method rails standard deleting method?
|
83
|
+
rails_delete = %i(destroy delete).include?(inventory_collection.delete_method)
|
84
|
+
if !rails_delete && inventory_collection.model_class.respond_to?(inventory_collection.delete_method)
|
85
|
+
# We have custom delete method defined on a class, that means it supports batch destroy
|
86
|
+
inventory_collection.store_deleted_records(records.map { |x| {:id => record_key(x, primary_key)} })
|
87
|
+
inventory_collection.model_class.public_send(inventory_collection.delete_method, records.map { |x| record_key(x, primary_key) })
|
88
|
+
else
|
89
|
+
legacy_ineffective_destroy_records(records)
|
90
|
+
end
|
91
|
+
end
|
92
|
+
|
93
|
+
# Very ineffective way of deleting records, but is needed if we want to invoke hooks.
|
94
|
+
#
|
95
|
+
# @param records [Array<ApplicationRecord, Hash>] Records we want to delete. If we have only hashes, we need to
|
96
|
+
# to fetch ApplicationRecord objects from the DB
|
97
|
+
def legacy_ineffective_destroy_records(records)
|
98
|
+
# We have either standard :destroy and :delete rails method, or custom instance level delete method
|
99
|
+
# Note: The standard :destroy and :delete rails method can't be batched because of the hooks and cascade destroy
|
100
|
+
ActiveRecord::Base.transaction do
|
101
|
+
if pure_sql_records_fetching
|
102
|
+
# For pure SQL fetching, we need to get the AR objects again, so we can call destroy
|
103
|
+
inventory_collection.model_class.where(:id => records.map { |x| record_key(x, primary_key) }).find_each do |record|
|
104
|
+
delete_record!(record)
|
105
|
+
end
|
106
|
+
else
|
107
|
+
records.each do |record|
|
108
|
+
delete_record!(record)
|
109
|
+
end
|
110
|
+
end
|
111
|
+
end
|
112
|
+
end
|
113
|
+
end
|
114
|
+
end
|
115
|
+
end
|
@@ -83,6 +83,128 @@ module InventoryRefresh::SaveCollection
|
|
83
83
|
"#{value}::#{sql_type}"
|
84
84
|
end
|
85
85
|
end
|
86
|
+
|
87
|
+
# Effective way of doing multiselect
|
88
|
+
#
|
89
|
+
# If we use "(col1, col2) IN [(a,e), (b,f), (b,e)]" it's not great, just with 10k batch, we see
|
90
|
+
# *** ActiveRecord::StatementInvalid Exception: PG::StatementTooComplex: ERROR: stack depth limit exceeded
|
91
|
+
# HINT: Increase the configuration parameter "max_stack_depth" (currently 2048kB), after ensuring the
|
92
|
+
# platform's stack depth limit is adequate.
|
93
|
+
#
|
94
|
+
# If we use "(col1 = a AND col2 = e) OR (col1 = b AND col2 = f) OR (col1 = b AND col2 = e)" with 10k batch, it
|
95
|
+
# takes about 6s and consumes 300MB, with 100k it takes ~1h and consume 3GB in Postgre process
|
96
|
+
#
|
97
|
+
# The best way seems to be using CTE, where the list of values we want to map is turned to 'table' and we just
|
98
|
+
# do RIGHT OUTER JOIN to get the complement of given identifiers. Tested on getting complement of 100k items,
|
99
|
+
# using 2 cols (:ems_ref and :uid_ems) from total 150k rows. It takes ~1s and 350MB in Postgre process
|
100
|
+
#
|
101
|
+
# @param manager_uuids [Array<String>, Array[Hash]] Array with manager_uuids of entities. The keys have to match
|
102
|
+
# inventory_collection.manager_ref. We allow passing just array of strings, if manager_ref.size ==1, to
|
103
|
+
# spare some memory
|
104
|
+
# @return [Arel::SelectManager] Arel for getting complement of uuids. This method modifies the passed
|
105
|
+
# manager_uuids to spare some memory
|
106
|
+
def complement_of!(manager_uuids, all_manager_uuids_scope, all_manager_uuids_timestamp)
|
107
|
+
all_attribute_keys = inventory_collection.manager_ref
|
108
|
+
all_attribute_keys_array = inventory_collection.manager_ref.map(&:to_s)
|
109
|
+
|
110
|
+
active_entities = Arel::Table.new(:active_entities)
|
111
|
+
active_entities_cte = Arel::Nodes::As.new(
|
112
|
+
active_entities,
|
113
|
+
Arel.sql("(#{active_entities_query(all_attribute_keys_array, manager_uuids)})")
|
114
|
+
)
|
115
|
+
|
116
|
+
all_entities = Arel::Table.new(:all_entities)
|
117
|
+
all_entities_cte = Arel::Nodes::As.new(
|
118
|
+
all_entities,
|
119
|
+
Arel.sql("(#{all_entities_query(all_manager_uuids_scope, all_manager_uuids_timestamp).select(:id, *all_attribute_keys_array).to_sql})")
|
120
|
+
)
|
121
|
+
join_condition = all_attribute_keys.map { |key| active_entities[key].eq(all_entities[key]) }.inject(:and)
|
122
|
+
where_condition = all_attribute_keys.map { |key| active_entities[key].eq(nil) }.inject(:and)
|
123
|
+
|
124
|
+
active_entities
|
125
|
+
.project(all_entities[:id])
|
126
|
+
.join(all_entities, Arel::Nodes::RightOuterJoin)
|
127
|
+
.on(join_condition)
|
128
|
+
.with(active_entities_cte, all_entities_cte)
|
129
|
+
.where(where_condition)
|
130
|
+
end
|
131
|
+
|
132
|
+
private
|
133
|
+
|
134
|
+
def all_entities_query(all_manager_uuids_scope, all_manager_uuids_timestamp)
|
135
|
+
all_entities_query = inventory_collection.full_collection_for_comparison
|
136
|
+
all_entities_query = all_entities_query.active if inventory_collection.retention_strategy == :archive
|
137
|
+
|
138
|
+
if all_manager_uuids_scope
|
139
|
+
scope_keys = all_manager_uuids_scope.first.keys.map { |x| association_to_foreign_key_mapping[x.to_sym] }.map(&:to_s)
|
140
|
+
scope = load_scope(all_manager_uuids_scope)
|
141
|
+
condition = inventory_collection.build_multi_selection_condition(scope, scope_keys)
|
142
|
+
all_entities_query = all_entities_query.where(condition)
|
143
|
+
end
|
144
|
+
|
145
|
+
if all_manager_uuids_timestamp && supports_column?(:resource_timestamp)
|
146
|
+
all_manager_uuids_timestamp = Time.parse(all_manager_uuids_timestamp).utc
|
147
|
+
|
148
|
+
date_field = model_class.arel_table[:resource_timestamp]
|
149
|
+
all_entities_query = all_entities_query.where(date_field.lt(all_manager_uuids_timestamp))
|
150
|
+
end
|
151
|
+
all_entities_query
|
152
|
+
end
|
153
|
+
|
154
|
+
def load_scope(all_manager_uuids_scope)
|
155
|
+
scope_keys = all_manager_uuids_scope.first.keys.to_set
|
156
|
+
|
157
|
+
all_manager_uuids_scope.map do |cond|
|
158
|
+
assert_scope!(scope_keys, cond)
|
159
|
+
|
160
|
+
cond.map do |key, value|
|
161
|
+
foreign_key = association_to_foreign_key_mapping[key.to_sym]
|
162
|
+
foreign_key_value = value.load&.id
|
163
|
+
|
164
|
+
assert_foreign_keys!(key, value, foreign_key, foreign_key_value)
|
165
|
+
|
166
|
+
[foreign_key, foreign_key_value]
|
167
|
+
end.to_h
|
168
|
+
end
|
169
|
+
end
|
170
|
+
|
171
|
+
def assert_scope!(scope_keys, cond)
|
172
|
+
if cond.keys.to_set != scope_keys
|
173
|
+
raise "'#{inventory_collection}' expected keys for :all_manager_uuids_scope are #{scope_keys.to_a}, got"\
|
174
|
+
" #{cond.keys}. Keys must be the same for all scopes provided."
|
175
|
+
end
|
176
|
+
end
|
177
|
+
|
178
|
+
def assert_foreign_keys!(key, value, foreign_key, foreign_key_value)
|
179
|
+
unless foreign_key
|
180
|
+
raise "'#{inventory_collection}' doesn't have relation :#{key} provided in :all_manager_uuids_scope."
|
181
|
+
end
|
182
|
+
|
183
|
+
unless foreign_key_value
|
184
|
+
raise "'#{inventory_collection}' couldn't load scope value :#{key} => #{value.inspect} provided in :all_manager_uuids_scope"
|
185
|
+
end
|
186
|
+
end
|
187
|
+
|
188
|
+
def active_entities_query(all_attribute_keys_array, manager_uuids)
|
189
|
+
connection = ActiveRecord::Base.connection
|
190
|
+
|
191
|
+
all_attribute_keys_array_q = all_attribute_keys_array.map { |x| quote_column_name(x) }
|
192
|
+
# For Postgre, only first set of values should contain the type casts
|
193
|
+
first_value = manager_uuids.shift.to_h
|
194
|
+
first_value = "(#{all_attribute_keys_array.map { |x| quote(connection, first_value[x], x, true) }.join(",")})"
|
195
|
+
|
196
|
+
# Rest of the values, without the type cast
|
197
|
+
values = manager_uuids.map! do |hash|
|
198
|
+
"(#{all_attribute_keys_array.map { |x| quote(connection, hash[x], x, false) }.join(",")})"
|
199
|
+
end.join(",")
|
200
|
+
|
201
|
+
values = values.blank? ? first_value : [first_value, values].join(",")
|
202
|
+
|
203
|
+
<<-SQL
|
204
|
+
SELECT *
|
205
|
+
FROM (VALUES #{values}) AS active_entities_table(#{all_attribute_keys_array_q.join(",")})
|
206
|
+
SQL
|
207
|
+
end
|
86
208
|
end
|
87
209
|
end
|
88
210
|
end
|
@@ -30,7 +30,7 @@ module InventoryRefresh::SaveCollection
|
|
30
30
|
version_attribute = if inventory_collection.parallel_safe? && supports_remote_data_timestamp?(all_attribute_keys)
|
31
31
|
:resource_timestamp
|
32
32
|
elsif inventory_collection.parallel_safe? && supports_remote_data_version?(all_attribute_keys)
|
33
|
-
:
|
33
|
+
:resource_counter
|
34
34
|
end
|
35
35
|
|
36
36
|
update_query = update_query_beginning(all_attribute_keys_array)
|
@@ -44,6 +44,21 @@ module InventoryRefresh::SaveCollection
|
|
44
44
|
update_query
|
45
45
|
end
|
46
46
|
|
47
|
+
# Build batch update query only for passed all_attribute_keys
|
48
|
+
#
|
49
|
+
# @param all_attribute_keys [Array<Symbol>] Array of all columns we will be saving into each table row
|
50
|
+
# @param hashes [Array<Hash>] data used for building a batch update sql query
|
51
|
+
def build_partial_update_query(all_attribute_keys, hashes)
|
52
|
+
# Cache the connection for the batch
|
53
|
+
connection = get_connection
|
54
|
+
|
55
|
+
all_attribute_keys = (all_attribute_keys + unique_index_columns).uniq
|
56
|
+
|
57
|
+
update_query = update_query_beginning(all_attribute_keys)
|
58
|
+
update_query += update_query_from_values(hashes, all_attribute_keys, connection, unique_index_columns)
|
59
|
+
update_query
|
60
|
+
end
|
61
|
+
|
47
62
|
private
|
48
63
|
|
49
64
|
def update_query_beginning(all_attribute_keys_array)
|
@@ -56,7 +71,7 @@ module InventoryRefresh::SaveCollection
|
|
56
71
|
|
57
72
|
def update_query_reset_version_columns(version_attribute)
|
58
73
|
if version_attribute
|
59
|
-
attr_partial = version_attribute.to_s.pluralize # Changes
|
74
|
+
attr_partial = version_attribute.to_s.pluralize # Changes resource_counter/timestamp to resource_counters/timestamps
|
60
75
|
attr_partial_max = "#{attr_partial}_max"
|
61
76
|
|
62
77
|
# Quote the column names
|
@@ -72,17 +87,21 @@ module InventoryRefresh::SaveCollection
|
|
72
87
|
end
|
73
88
|
end
|
74
89
|
|
75
|
-
def update_query_from_values(hashes, all_attribute_keys_array, connection)
|
90
|
+
def update_query_from_values(hashes, all_attribute_keys_array, connection, matching = [:id])
|
76
91
|
values = hashes.map! do |hash|
|
77
92
|
"(#{all_attribute_keys_array.map { |x| quote(connection, hash[x], x, true) }.join(",")})"
|
78
93
|
end.join(",")
|
79
94
|
|
95
|
+
where_cond = matching.map do |x|
|
96
|
+
"updated_values.#{quote_column_name(x)} = #{q_table_name}.#{quote_column_name(x)}"
|
97
|
+
end.join(" AND ")
|
98
|
+
|
80
99
|
<<-SQL
|
81
100
|
FROM (
|
82
101
|
VALUES
|
83
102
|
#{values}
|
84
103
|
) AS updated_values (#{all_attribute_keys_array.map { |x| quote_column_name(x) }.join(",")})
|
85
|
-
WHERE
|
104
|
+
WHERE #{where_cond}
|
86
105
|
SQL
|
87
106
|
end
|
88
107
|
|
@@ -90,7 +109,7 @@ module InventoryRefresh::SaveCollection
|
|
90
109
|
if version_attribute
|
91
110
|
# This conditional will avoid rewriting new data by old data. But we want it only when version_attribute is
|
92
111
|
# a part of the data, since for the fake records, we just want to update ems_ref.
|
93
|
-
attr_partial = version_attribute.to_s.pluralize # Changes
|
112
|
+
attr_partial = version_attribute.to_s.pluralize # Changes resource_counter/timestamp to resource_counters/timestamps
|
94
113
|
attr_partial_max = "#{attr_partial}_max"
|
95
114
|
|
96
115
|
# Quote the column names
|
@@ -25,7 +25,7 @@ module InventoryRefresh::SaveCollection
|
|
25
25
|
# Cache the connection for the batch
|
26
26
|
connection = get_connection
|
27
27
|
# Ignore versioning columns that are set separately
|
28
|
-
ignore_cols = mode == :partial ? [:resource_timestamp, :
|
28
|
+
ignore_cols = mode == :partial ? [:resource_timestamp, :resource_counter] : []
|
29
29
|
# Make sure we don't send a primary_key for INSERT in any form, it could break PG sequencer
|
30
30
|
all_attribute_keys_array = all_attribute_keys.to_a - [primary_key.to_s, primary_key.to_sym] - ignore_cols
|
31
31
|
|
@@ -83,7 +83,7 @@ module InventoryRefresh::SaveCollection
|
|
83
83
|
|
84
84
|
def insert_query_on_conflict_update(all_attribute_keys, mode, ignore_cols, column_name)
|
85
85
|
if mode == :partial
|
86
|
-
ignore_cols += [:resource_timestamps, :resource_timestamps_max, :
|
86
|
+
ignore_cols += [:resource_timestamps, :resource_timestamps_max, :resource_counters, :resource_counters_max]
|
87
87
|
end
|
88
88
|
ignore_cols += [:created_on, :created_at] # Lets not change created for the update clause
|
89
89
|
|
@@ -91,7 +91,7 @@ module InventoryRefresh::SaveCollection
|
|
91
91
|
version_attribute = if supports_remote_data_timestamp?(all_attribute_keys)
|
92
92
|
:resource_timestamp
|
93
93
|
elsif supports_remote_data_version?(all_attribute_keys)
|
94
|
-
:
|
94
|
+
:resource_counter
|
95
95
|
end
|
96
96
|
|
97
97
|
# TODO(lsmola) should we add :deleted => false to the update clause? That should handle a reconnect, without a
|
@@ -116,7 +116,7 @@ module InventoryRefresh::SaveCollection
|
|
116
116
|
end
|
117
117
|
|
118
118
|
def full_update_condition(attr_full)
|
119
|
-
attr_partial = attr_full.to_s.pluralize # Changes
|
119
|
+
attr_partial = attr_full.to_s.pluralize # Changes resource_counter/timestamp to resource_counters/timestamps
|
120
120
|
attr_partial_max = "#{attr_partial}_max"
|
121
121
|
|
122
122
|
# Quote the column names
|
@@ -135,11 +135,11 @@ module InventoryRefresh::SaveCollection
|
|
135
135
|
end
|
136
136
|
|
137
137
|
def partial_update_condition(attr_full, column_name)
|
138
|
-
attr_partial = attr_full.to_s.pluralize # Changes
|
138
|
+
attr_partial = attr_full.to_s.pluralize # Changes resource_counter/timestamp to resource_counters/timestamps
|
139
139
|
attr_partial_max = "#{attr_partial}_max"
|
140
140
|
cast = if attr_full == :resource_timestamp
|
141
141
|
"timestamp"
|
142
|
-
elsif attr_full == :
|
142
|
+
elsif attr_full == :resource_counter
|
143
143
|
"integer"
|
144
144
|
end
|
145
145
|
|
@@ -0,0 +1,69 @@
|
|
1
|
+
require "inventory_refresh/logging"
|
2
|
+
require "inventory_refresh/save_collection/saver/retention_helper"
|
3
|
+
|
4
|
+
module InventoryRefresh::SaveCollection
|
5
|
+
class Sweeper < InventoryRefresh::SaveCollection::Base
|
6
|
+
class << self
|
7
|
+
# Sweeps inactive records based on :last_seen_on and :refresh_start timestamps. All records having :last_seen_on
|
8
|
+
# lower than :refresh_start or nil will be archived/deleted.
|
9
|
+
#
|
10
|
+
# @param _ems [ActiveRecord] Manager owning the inventory_collections
|
11
|
+
# @param inventory_collections [Array<InventoryRefresh::InventoryCollection>] Array of InventoryCollection objects
|
12
|
+
# for sweeping
|
13
|
+
# @param refresh_state [ActiveRecord] Record of :refresh_states
|
14
|
+
def sweep(_ems, inventory_collections, refresh_state)
|
15
|
+
inventory_collections.each do |inventory_collection|
|
16
|
+
next unless sweep_possible?(inventory_collection, refresh_state)
|
17
|
+
|
18
|
+
new(inventory_collection, refresh_state).sweep
|
19
|
+
end
|
20
|
+
end
|
21
|
+
|
22
|
+
def sweep_possible?(inventory_collection, refresh_state)
|
23
|
+
inventory_collection.supports_column?(:last_seen_at) && inventory_collection.parallel_safe? &&
|
24
|
+
inventory_collection.strategy == :local_db_find_missing_references &&
|
25
|
+
in_scope?(inventory_collection, refresh_state.sweep_scope)
|
26
|
+
end
|
27
|
+
|
28
|
+
def in_scope?(inventory_collection, sweep_scope)
|
29
|
+
return true unless sweep_scope
|
30
|
+
|
31
|
+
if sweep_scope.kind_of?(Array)
|
32
|
+
return true if sweep_scope.include?(inventory_collection&.name&.to_s)
|
33
|
+
end
|
34
|
+
|
35
|
+
false
|
36
|
+
end
|
37
|
+
end
|
38
|
+
|
39
|
+
include InventoryRefresh::SaveCollection::Saver::RetentionHelper
|
40
|
+
|
41
|
+
attr_reader :inventory_collection, :refresh_state, :model_class, :primary_key
|
42
|
+
|
43
|
+
def initialize(inventory_collection, refresh_state)
|
44
|
+
@inventory_collection = inventory_collection
|
45
|
+
@refresh_state = refresh_state
|
46
|
+
|
47
|
+
@model_class = inventory_collection.model_class
|
48
|
+
@primary_key = @model_class.primary_key
|
49
|
+
end
|
50
|
+
|
51
|
+
def sweep
|
52
|
+
refresh_start = refresh_state.created_at
|
53
|
+
raise "Couldn't load :created_at out of RefreshState record: #{refresh_state}" unless refresh_start
|
54
|
+
|
55
|
+
table = model_class.arel_table
|
56
|
+
date_field = table[:last_seen_at]
|
57
|
+
all_entities_query = inventory_collection.full_collection_for_comparison
|
58
|
+
all_entities_query.active if inventory_collection.retention_strategy == :archive
|
59
|
+
|
60
|
+
query = all_entities_query
|
61
|
+
.where(date_field.lt(refresh_start)).or(all_entities_query.where(:last_seen_at => nil))
|
62
|
+
.select(table[:id])
|
63
|
+
|
64
|
+
query.find_in_batches do |batch|
|
65
|
+
destroy_records!(batch)
|
66
|
+
end
|
67
|
+
end
|
68
|
+
end
|
69
|
+
end
|
@@ -1,5 +1,5 @@
|
|
1
|
-
require "inventory_refresh/save_collection/recursive"
|
2
1
|
require "inventory_refresh/save_collection/topological_sort"
|
2
|
+
require "inventory_refresh/save_collection/sweeper"
|
3
3
|
|
4
4
|
module InventoryRefresh
|
5
5
|
class SaveInventory
|
@@ -11,20 +11,30 @@ module InventoryRefresh
|
|
11
11
|
# @param ems [ExtManagementSystem] manager owning the inventory_collections
|
12
12
|
# @param inventory_collections [Array<InventoryRefresh::InventoryCollection>] array of InventoryCollection objects
|
13
13
|
# for saving
|
14
|
-
def save_inventory(ems, inventory_collections
|
14
|
+
def save_inventory(ems, inventory_collections)
|
15
15
|
logger.debug("#{log_header(ems)} Scanning Inventory Collections...Start")
|
16
16
|
InventoryRefresh::InventoryCollection::Scanner.scan!(inventory_collections)
|
17
17
|
logger.debug("#{log_header(ems)} Scanning Inventory Collections...Complete")
|
18
18
|
|
19
19
|
logger.info("#{log_header(ems)} Saving EMS Inventory...")
|
20
|
+
InventoryRefresh::SaveCollection::TopologicalSort.save_collections(ems, inventory_collections)
|
21
|
+
logger.info("#{log_header(ems)} Saving EMS Inventory...Complete")
|
20
22
|
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
23
|
+
ems
|
24
|
+
end
|
25
|
+
|
26
|
+
# Sweeps inactive records based on :last_seen_at and :refresh_start timestamps. All records having :last_seen_at
|
27
|
+
# lower than :refresh_start or nil will be archived/deleted.
|
28
|
+
#
|
29
|
+
# @param ems [ExtManagementSystem] manager owning the inventory_collections
|
30
|
+
# @param inventory_collections [Array<InventoryRefresh::InventoryCollection>] array of InventoryCollection objects
|
31
|
+
# for sweeping
|
32
|
+
# @param refresh_state [ActiveRecord] Record of :refresh_states
|
33
|
+
def sweep_inactive_records(ems, inventory_collections, refresh_state)
|
34
|
+
logger.info("#{log_header(ems)} Sweeping EMS Inventory...")
|
35
|
+
InventoryRefresh::SaveCollection::Sweeper.sweep(ems, inventory_collections, refresh_state)
|
36
|
+
logger.info("#{log_header(ems)} Sweeping EMS Inventory...Complete")
|
26
37
|
|
27
|
-
logger.info("#{log_header(ems)} Saving EMS Inventory...Complete")
|
28
38
|
ems
|
29
39
|
end
|
30
40
|
|