inventory_refresh 0.1.0 → 0.1.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.rspec +1 -1
- data/.rspec_ci +1 -0
- data/.travis.yml +5 -1
- data/Rakefile +5 -11
- data/config/database.yml +12 -0
- data/lib/inventory_refresh.rb +0 -10
- data/lib/inventory_refresh/graph/topological_sort.rb +2 -2
- data/lib/inventory_refresh/logging.rb +11 -4
- data/lib/inventory_refresh/save_collection/base.rb +4 -4
- data/lib/inventory_refresh/save_collection/recursive.rb +1 -1
- data/lib/inventory_refresh/save_collection/saver/base.rb +13 -13
- data/lib/inventory_refresh/save_collection/saver/concurrent_safe_batch.rb +6 -6
- data/lib/inventory_refresh/save_collection/saver/default.rb +2 -2
- data/lib/inventory_refresh/save_collection/saver/sql_helper.rb +1 -1
- data/lib/inventory_refresh/save_collection/saver/sql_helper_update.rb +2 -2
- data/lib/inventory_refresh/save_collection/saver/sql_helper_upsert.rb +2 -2
- data/lib/inventory_refresh/save_collection/topological_sort.rb +5 -5
- data/lib/inventory_refresh/save_inventory.rb +4 -4
- data/lib/inventory_refresh/version.rb +1 -1
- metadata +2 -1
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 6ade3208f2b15b5f3ddd1eb2ba062bc011ba4ce8fbd4bbc838ba9ccef62fb19f
|
4
|
+
data.tar.gz: 6aecb2c6ea69ba72b48819eb8a4af6cbffa9f3c9d475bcca8c88e6e5f5fa0cf1
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: e2d87b6841db8fd728269e4234ae05ca0c80a5612980ee08b83815d626b2945bca34644a972133eedb8f76bce19c627cc93a3d812a41c13edd8d74c9ff5d7175
|
7
|
+
data.tar.gz: c811d7fad848a207228595ede6cdb251a7096192ae5984bf3be0af667956553035c22c45ece7b7962807339b9e45cf06ed4f823561a7c3ccd40fa9a631db1c50
|
data/.rspec
CHANGED
data/.rspec_ci
CHANGED
data/.travis.yml
CHANGED
@@ -3,9 +3,13 @@ sudo: false
|
|
3
3
|
language: ruby
|
4
4
|
cache: bundler
|
5
5
|
rvm:
|
6
|
-
|
6
|
+
- 2.4.4
|
7
|
+
- 2.3.6
|
7
8
|
addons:
|
8
9
|
postgresql: '9.5'
|
10
|
+
env:
|
11
|
+
global:
|
12
|
+
- CC_TEST_REPORTER_ID=1ef1a3a3d007395b11083d634a6fdac1e3d979b6428c39d2cd8d58556cdd68f7
|
9
13
|
before_install:
|
10
14
|
- gem install bundler -v 1.16.4
|
11
15
|
- psql -c "CREATE USER root SUPERUSER PASSWORD 'smartvm';" -U postgres
|
data/Rakefile
CHANGED
@@ -18,24 +18,18 @@ namespace :spec do
|
|
18
18
|
desc "Setup test database"
|
19
19
|
task :setup => [:db_drop, :db_create, :db_load_schema]
|
20
20
|
|
21
|
-
def
|
22
|
-
|
23
|
-
|
24
|
-
:encoding => "utf8",
|
25
|
-
:username => "root",
|
26
|
-
:pool => 5,
|
27
|
-
:wait_timeout => 5,
|
28
|
-
:min_messages => "warning",
|
29
|
-
}
|
21
|
+
def connection_spec
|
22
|
+
require 'yaml'
|
23
|
+
@connection_spec ||= YAML.load_file(File.join(__dir__, %w(config database.yml)))
|
30
24
|
end
|
31
25
|
|
32
26
|
def test_database_name
|
33
|
-
"
|
27
|
+
connection_spec["test"]["database"]
|
34
28
|
end
|
35
29
|
|
36
30
|
def with_connection(database_name)
|
37
31
|
require "active_record"
|
38
|
-
pool = ActiveRecord::Base.establish_connection
|
32
|
+
pool = ActiveRecord::Base.establish_connection(connection_spec["test"].merge("database" => database_name))
|
39
33
|
yield ActiveRecord::Base.connection
|
40
34
|
ensure
|
41
35
|
pool&.disconnect!
|
data/config/database.yml
ADDED
data/lib/inventory_refresh.rb
CHANGED
@@ -8,13 +8,3 @@ require "inventory_refresh/save_inventory"
|
|
8
8
|
require "inventory_refresh/target"
|
9
9
|
require "inventory_refresh/target_collection"
|
10
10
|
require "inventory_refresh/version"
|
11
|
-
|
12
|
-
module InventoryRefresh
|
13
|
-
class << self
|
14
|
-
attr_accessor :log
|
15
|
-
end
|
16
|
-
|
17
|
-
def self.log
|
18
|
-
@log ||= NullLogger.new
|
19
|
-
end
|
20
|
-
end
|
@@ -45,14 +45,14 @@ module InventoryRefresh
|
|
45
45
|
max_depth -= 1
|
46
46
|
if max_depth <= 0
|
47
47
|
message = "Max depth reached while doing topological sort, your graph probably contains a cycle"
|
48
|
-
|
48
|
+
#logger.error("#{message}:\n#{graph.to_graphviz}")
|
49
49
|
raise "#{message} (see log)"
|
50
50
|
end
|
51
51
|
|
52
52
|
set, nodes = nodes.partition { |v| edges.select { |e| e.second == v }.all? { |e| sets.flatten.include?(e.first) } }
|
53
53
|
if set.blank?
|
54
54
|
message = "Blank dependency set while doing topological sort, your graph probably contains a cycle"
|
55
|
-
|
55
|
+
#logger.error("#{message}:\n#{graph.to_graphviz}")
|
56
56
|
raise "#{message} (see log)"
|
57
57
|
end
|
58
58
|
|
@@ -1,8 +1,15 @@
|
|
1
|
-
require "forwardable"
|
2
|
-
|
3
1
|
module InventoryRefresh
|
2
|
+
class << self
|
3
|
+
attr_writer :logger
|
4
|
+
end
|
5
|
+
|
6
|
+
def self.logger
|
7
|
+
@logger ||= NullLogger.new
|
8
|
+
end
|
9
|
+
|
4
10
|
module Logging
|
5
|
-
|
6
|
-
|
11
|
+
def logger
|
12
|
+
InventoryRefresh.logger
|
13
|
+
end
|
7
14
|
end
|
8
15
|
end
|
@@ -14,16 +14,16 @@ module InventoryRefresh::SaveCollection
|
|
14
14
|
# @param ems [ExtManagementSystem] manger owning the InventoryCollection object
|
15
15
|
# @param inventory_collection [InventoryRefresh::InventoryCollection] InventoryCollection object we want to save
|
16
16
|
def save_inventory_object_inventory(ems, inventory_collection)
|
17
|
-
|
18
|
-
|
17
|
+
logger.debug("Saving collection #{inventory_collection} of size #{inventory_collection.size} to"\
|
18
|
+
" the database, for the manager: '#{ems.name}'...")
|
19
19
|
|
20
20
|
if inventory_collection.custom_save_block.present?
|
21
|
-
|
21
|
+
logger.debug("Saving collection #{inventory_collection} using a custom save block")
|
22
22
|
inventory_collection.custom_save_block.call(ems, inventory_collection)
|
23
23
|
else
|
24
24
|
save_inventory(inventory_collection)
|
25
25
|
end
|
26
|
-
|
26
|
+
logger.debug("Saving collection #{inventory_collection}, for the manager: '#{ems.name}'...Complete")
|
27
27
|
inventory_collection.saved = true
|
28
28
|
end
|
29
29
|
|
@@ -44,7 +44,7 @@ module InventoryRefresh::SaveCollection
|
|
44
44
|
end
|
45
45
|
end
|
46
46
|
|
47
|
-
|
47
|
+
logger.debug("Saving #{inventory_collection} of size #{inventory_collection.size}")
|
48
48
|
save_inventory_object_inventory(ems, inventory_collection)
|
49
49
|
end
|
50
50
|
end
|
@@ -140,7 +140,7 @@ module InventoryRefresh::SaveCollection
|
|
140
140
|
inventory_objects_index[index] = inventory_object
|
141
141
|
end
|
142
142
|
|
143
|
-
|
143
|
+
logger.debug("Processing #{inventory_collection} of size #{inventory_collection.size}...")
|
144
144
|
# Records that are in the DB, we will be updating or deleting them.
|
145
145
|
ActiveRecord::Base.transaction do
|
146
146
|
association.find_each do |record|
|
@@ -177,12 +177,12 @@ module InventoryRefresh::SaveCollection
|
|
177
177
|
end
|
178
178
|
end
|
179
179
|
end
|
180
|
-
|
181
|
-
|
182
|
-
|
183
|
-
|
180
|
+
logger.debug("Processing #{inventory_collection}, "\
|
181
|
+
"created=#{inventory_collection.created_records.count}, "\
|
182
|
+
"updated=#{inventory_collection.updated_records.count}, "\
|
183
|
+
"deleted=#{inventory_collection.deleted_records.count}...Complete")
|
184
184
|
rescue => e
|
185
|
-
|
185
|
+
logger.error("Error when saving #{inventory_collection} with #{inventory_collection_details}. Message: #{e.message}")
|
186
186
|
raise e
|
187
187
|
end
|
188
188
|
|
@@ -204,8 +204,8 @@ module InventoryRefresh::SaveCollection
|
|
204
204
|
|
205
205
|
all_manager_uuids_size = inventory_collection.all_manager_uuids.size
|
206
206
|
|
207
|
-
|
208
|
-
|
207
|
+
logger.debug("Processing :delete_complement of #{inventory_collection} of size "\
|
208
|
+
"#{all_manager_uuids_size}...")
|
209
209
|
deleted_counter = 0
|
210
210
|
|
211
211
|
inventory_collection.db_collection_for_comparison_for_complement_of(
|
@@ -219,8 +219,8 @@ module InventoryRefresh::SaveCollection
|
|
219
219
|
end
|
220
220
|
end
|
221
221
|
|
222
|
-
|
223
|
-
|
222
|
+
logger.debug("Processing :delete_complement of #{inventory_collection} of size "\
|
223
|
+
"#{all_manager_uuids_size}, deleted=#{deleted_counter}...Complete")
|
224
224
|
end
|
225
225
|
|
226
226
|
# Deletes/soft-deletes a given record
|
@@ -248,8 +248,8 @@ module InventoryRefresh::SaveCollection
|
|
248
248
|
# relations can return the same record multiple times. We don't want to do SELECT DISTINCT by default, since
|
249
249
|
# it can be very slow.
|
250
250
|
if false # TODO: Rails.env.production?
|
251
|
-
|
252
|
-
|
251
|
+
logger.warn("Please update :association or :arel for #{inventory_collection} to return a DISTINCT result. "\
|
252
|
+
" The duplicate value is being ignored.")
|
253
253
|
return false
|
254
254
|
else
|
255
255
|
raise("Please update :association or :arel for #{inventory_collection} to return a DISTINCT result. ")
|
@@ -274,7 +274,7 @@ module InventoryRefresh::SaveCollection
|
|
274
274
|
"#{inventory_collection.parent.class.name}:"\
|
275
275
|
"#{inventory_collection.parent.try(:id)}"
|
276
276
|
if false # TODO: Rails.env.production?
|
277
|
-
|
277
|
+
logger.warn("Referential integrity check violated, ignoring #{subject}")
|
278
278
|
return false
|
279
279
|
else
|
280
280
|
raise("Referential integrity check violated for #{subject}")
|
@@ -97,7 +97,7 @@ module InventoryRefresh::SaveCollection
|
|
97
97
|
all_attribute_keys << :updated_on if supports_updated_on?
|
98
98
|
all_attribute_keys << :type if supports_sti?
|
99
99
|
|
100
|
-
|
100
|
+
logger.debug("Processing #{inventory_collection} of size #{inventory_collection.size}...")
|
101
101
|
|
102
102
|
unless inventory_collection.create_only?
|
103
103
|
update_or_destroy_records!(batch_iterator(association), inventory_objects_index, attributes_index, all_attribute_keys)
|
@@ -123,12 +123,12 @@ module InventoryRefresh::SaveCollection
|
|
123
123
|
create_or_update_partial_records(all_attribute_keys)
|
124
124
|
end
|
125
125
|
end
|
126
|
-
|
127
|
-
|
128
|
-
|
129
|
-
|
126
|
+
logger.debug("Processing #{inventory_collection}, "\
|
127
|
+
"created=#{inventory_collection.created_records.count}, "\
|
128
|
+
"updated=#{inventory_collection.updated_records.count}, "\
|
129
|
+
"deleted=#{inventory_collection.deleted_records.count}...Complete")
|
130
130
|
rescue => e
|
131
|
-
|
131
|
+
logger.error("Error when saving #{inventory_collection} with #{inventory_collection_details}. Message: #{e.message}")
|
132
132
|
raise e
|
133
133
|
end
|
134
134
|
|
@@ -43,8 +43,8 @@ module InventoryRefresh::SaveCollection
|
|
43
43
|
if unique_db_indexes.include?(index) # Include on Set is O(1)
|
44
44
|
# We have a duplicate in the DB, destroy it. A find_each method does automatically .order(:id => :asc)
|
45
45
|
# so we always keep the oldest record in the case of duplicates.
|
46
|
-
|
47
|
-
|
46
|
+
logger.warn("A duplicate record was detected and destroyed, inventory_collection: "\
|
47
|
+
"'#{inventory_collection}', record: '#{record}', duplicate_index: '#{index}'")
|
48
48
|
record.destroy
|
49
49
|
return false
|
50
50
|
else
|
@@ -54,7 +54,7 @@ module InventoryRefresh::SaveCollection
|
|
54
54
|
connection.quote(value)
|
55
55
|
end
|
56
56
|
rescue TypeError => e
|
57
|
-
|
57
|
+
logger.error("Can't quote value: #{value}, of :#{name} and #{inventory_collection}")
|
58
58
|
raise e
|
59
59
|
end
|
60
60
|
|
@@ -18,7 +18,7 @@ module InventoryRefresh::SaveCollection
|
|
18
18
|
# @param all_attribute_keys [Array<Symbol>] Array of all columns we will be saving into each table row
|
19
19
|
# @param hashes [Array<Hash>] data used for building a batch update sql query
|
20
20
|
def build_update_query(all_attribute_keys, hashes)
|
21
|
-
|
21
|
+
logger.debug("Building update query for #{inventory_collection} of size #{inventory_collection.size}...")
|
22
22
|
# Cache the connection for the batch
|
23
23
|
connection = get_connection
|
24
24
|
|
@@ -39,7 +39,7 @@ module InventoryRefresh::SaveCollection
|
|
39
39
|
update_query += update_query_version_conditions(version_attribute)
|
40
40
|
update_query += update_query_returning
|
41
41
|
|
42
|
-
|
42
|
+
logger.debug("Building update query for #{inventory_collection} of size #{inventory_collection.size}...Complete")
|
43
43
|
|
44
44
|
update_query
|
45
45
|
end
|
@@ -20,7 +20,7 @@ module InventoryRefresh::SaveCollection
|
|
20
20
|
# @param on_conflict [Symbol, NilClass] defines behavior on conflict with unique index constraint, allowed values
|
21
21
|
# are :do_update, :do_nothing, nil
|
22
22
|
def build_insert_query(all_attribute_keys, hashes, on_conflict: nil, mode:, column_name: nil)
|
23
|
-
|
23
|
+
logger.debug("Building insert query for #{inventory_collection} of size #{inventory_collection.size}...")
|
24
24
|
|
25
25
|
# Cache the connection for the batch
|
26
26
|
connection = get_connection
|
@@ -33,7 +33,7 @@ module InventoryRefresh::SaveCollection
|
|
33
33
|
insert_query += insert_query_on_conflict_behavior(all_attribute_keys, on_conflict, mode, ignore_cols, column_name)
|
34
34
|
insert_query += insert_query_returning
|
35
35
|
|
36
|
-
|
36
|
+
logger.debug("Building insert query for #{inventory_collection} of size #{inventory_collection.size}...Complete")
|
37
37
|
|
38
38
|
insert_query
|
39
39
|
end
|
@@ -17,21 +17,21 @@ module InventoryRefresh::SaveCollection
|
|
17
17
|
|
18
18
|
layers = InventoryRefresh::Graph::TopologicalSort.new(graph).topological_sort
|
19
19
|
|
20
|
-
|
20
|
+
logger.debug("Saving manager #{ems.name}...")
|
21
21
|
|
22
22
|
sorted_graph_log = "Topological sorting of manager #{ems.name} resulted in these layers processable in parallel:\n"
|
23
23
|
sorted_graph_log += graph.to_graphviz(:layers => layers)
|
24
|
-
|
24
|
+
logger.debug(sorted_graph_log)
|
25
25
|
|
26
26
|
layers.each_with_index do |layer, index|
|
27
|
-
|
27
|
+
logger.debug("Saving manager #{ems.name} | Layer #{index}")
|
28
28
|
layer.each do |inventory_collection|
|
29
29
|
save_inventory_object_inventory(ems, inventory_collection) unless inventory_collection.saved?
|
30
30
|
end
|
31
|
-
|
31
|
+
logger.debug("Saved manager #{ems.name} | Layer #{index}")
|
32
32
|
end
|
33
33
|
|
34
|
-
|
34
|
+
logger.debug("Saving manager #{ems.name}...Complete")
|
35
35
|
end
|
36
36
|
end
|
37
37
|
end
|
@@ -12,11 +12,11 @@ module InventoryRefresh
|
|
12
12
|
# @param inventory_collections [Array<InventoryRefresh::InventoryCollection>] array of InventoryCollection objects
|
13
13
|
# for saving
|
14
14
|
def save_inventory(ems, inventory_collections, strategy = nil)
|
15
|
-
|
15
|
+
logger.debug("#{log_header(ems)} Scanning Inventory Collections...Start")
|
16
16
|
InventoryRefresh::InventoryCollection::Scanner.scan!(inventory_collections)
|
17
|
-
|
17
|
+
logger.debug("#{log_header(ems)} Scanning Inventory Collections...Complete")
|
18
18
|
|
19
|
-
|
19
|
+
logger.info("#{log_header(ems)} Saving EMS Inventory...")
|
20
20
|
|
21
21
|
if strategy.try(:to_sym) == :recursive
|
22
22
|
InventoryRefresh::SaveCollection::Recursive.save_collections(ems, inventory_collections)
|
@@ -24,7 +24,7 @@ module InventoryRefresh
|
|
24
24
|
InventoryRefresh::SaveCollection::TopologicalSort.save_collections(ems, inventory_collections)
|
25
25
|
end
|
26
26
|
|
27
|
-
|
27
|
+
logger.info("#{log_header(ems)} Saving EMS Inventory...Complete")
|
28
28
|
ems
|
29
29
|
end
|
30
30
|
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: inventory_refresh
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.
|
4
|
+
version: 0.1.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- ManageIQ Developers
|
@@ -158,6 +158,7 @@ files:
|
|
158
158
|
- Rakefile
|
159
159
|
- bin/console
|
160
160
|
- bin/setup
|
161
|
+
- config/database.yml
|
161
162
|
- inventory_refresh.gemspec
|
162
163
|
- lib/inventory_refresh.rb
|
163
164
|
- lib/inventory_refresh/application_record_iterator.rb
|