topological_inventory-core 1.0.0 → 1.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 63b1ec93fd5f7ebb021670fe8184db0945eed00385bafd718d3587aa25a171a7
-  data.tar.gz: 37df3df19aa75c724464eb04b2639d07caa8078543435c9edee5e5fc3dc0aa24
+  metadata.gz: 14973202e680d9bf035a3c85d5ef856140bf11d934bfc488cbade6932c8239c1
+  data.tar.gz: 7493112f3e945f0d0e20cb86385add9f45f623e2825c08b8a3a4ac2a62994822
 SHA512:
-  metadata.gz: 9bc2bd484466880aed1fc578bc5dc1f6b1578e288d430e386a089f505caa51d7585ba668dfcee0d4db5f04f58ec58bad2502db460f258ca2b6127284f05a08c0
-  data.tar.gz: aa83f7119b8b1e2570c965a01fed2abda0d1218b14a0cba6b5f6b66ef97e8cfff6c1161902da72e6ee28aba2a8d66740bea61c0a3fb66f3494ae099401786e73
+  metadata.gz: '08de615e242bc7dd1ec6f4349386064855b947602986736a8095c3b4802adc0fbeabd5aeb03dcc3d965d3a882f8a989c646624482c45de5235a5457797cef871'
+  data.tar.gz: fb1e4895ac6ecb98b6564ed411f5107417b5b2f68ee5a1346312f59b32598065d46d300ed158750fa0df2e85868958e7e347b2f82818e00d39db01cf1a414869

data/app/models/source.rb

@@ -77,6 +77,9 @@ class Source < ApplicationRecord
   has_many :volume_attachments, :through => :volumes
   has_many :volume_types
 
+  # Tasks
+  has_many :tasks
+
   ALLOWED_REFRESH_STATUS_VALUES = ["deployed", "quota_limited"].freeze
   validates :refresh_status, :allow_nil => true, :inclusion => {:in => ALLOWED_REFRESH_STATUS_VALUES, :message => "%{value} is not included in #{ALLOWED_REFRESH_STATUS_VALUES}"}
 end

data/app/models/task.rb

@@ -1,5 +1,6 @@
 class Task < ApplicationRecord
   belongs_to :tenant
+  belongs_to :source, :optional => true
 
   validates :state,  :inclusion => {:in => %w(pending queued running completed)}
   validates :status, :inclusion => {:in => %w(ok warn error)}

data/config/database.yml

@@ -0,0 +1,20 @@
+default: &default
+  adapter: postgresql
+  encoding: utf8
+  username: root
+  pool: 5
+  wait_timeout: 5
+  min_messages: warning
+
+development:
+  <<: *default
+  database: topological_inventory_core_development
+  min_messages: notice
+
+test:
+  <<: *default
+  database: topological_inventory_core_test
+
+production:
+  <<: *default
+  database: topological_inventory_core_production

data/db/migrate/20200403114310_add_target_and_source_to_task.rb

@@ -0,0 +1,10 @@
+class AddTargetAndSourceToTask < ActiveRecord::Migration[5.2]
+  def change
+    add_column :tasks, :target_source_ref, :string, :null => true
+    add_column :tasks, :target_type, :string, :null => true
+
+    add_index :tasks, %i[target_type target_source_ref]
+
+    add_reference :tasks, :source, :index => true, :null => true, :foreign_key => { :on_delete => :nullify }
+  end
+end

data/db/migrate/20200414123737_extract_tasks_source_ref_from_context.rb

@@ -0,0 +1,19 @@
+class ExtractTasksSourceRefFromContext < ActiveRecord::Migration[5.2]
+  def up
+    Task.find_each do |task|
+      source_ref = task.context&.dig('service_instance', 'source_ref')
+      next if source_ref.nil?
+
+      say "Task: #{task.id}, Source ref: #{source_ref}"
+      source_id = task.context&.dig('service_instance', 'source_id')
+
+      task.update_attributes(:target_source_ref => source_ref.to_s,
+                             :target_type       => 'ServiceInstance',
+                             :source_id         => source_id)
+    end
+  end
+
+  def down
+    Task.update_all(:source_id => nil, :target_type => nil, :target_source_ref => nil)
+  end
+end

data/db/schema.rb

@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 2020_03_17_082640) do
+ActiveRecord::Schema.define(version: 2020_04_14_123737) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
@@ -1187,6 +1187,11 @@ ActiveRecord::Schema.define(version: 2020_03_17_082640) do
     t.datetime "completed_at"
     t.datetime "created_at", null: false
     t.datetime "updated_at", null: false
+    t.string "target_source_ref"
+    t.string "target_type"
+    t.bigint "source_id"
+    t.index ["source_id"], name: "index_tasks_on_source_id"
+    t.index ["target_type", "target_source_ref"], name: "index_tasks_on_target_type_and_target_source_ref"
     t.index ["tenant_id"], name: "index_tasks_on_tenant_id"
   end
 
@@ -1561,6 +1566,7 @@ ActiveRecord::Schema.define(version: 2020_03_17_082640) do
   add_foreign_key "subscriptions", "tenants", on_delete: :cascade
   add_foreign_key "tags", "refresh_state_parts", on_delete: :nullify
   add_foreign_key "tags", "tenants", on_delete: :cascade
+  add_foreign_key "tasks", "sources", on_delete: :nullify
   add_foreign_key "tasks", "tenants", on_delete: :cascade
   add_foreign_key "vm_security_groups", "refresh_state_parts", on_delete: :nullify
   add_foreign_key "vm_security_groups", "security_groups", on_delete: :cascade

data/lib/topological_inventory/core/version.rb

@@ -1,5 +1,5 @@
 module TopologicalInventory
   module Core
-    VERSION = '1.0.0'
+    VERSION = '1.1.0'
   end
 end

data/lib/topological_inventory/schema/default.rb

@@ -65,6 +65,7 @@ module TopologicalInventory
         add_volume_attachments
         add_cross_link_vms
         add_vm_security_groups
+        add_service_instance_tasks
       end
 
       def targeted?
@@ -123,6 +124,106 @@ module TopologicalInventory
         end
       end
 
+      def add_service_instance_tasks
+        add_collection(:service_instance_tasks, inventory_collection_builder, {}, {:without_model_class => true}) do |builder|
+          add_default_properties(builder)
+          add_default_values(builder)
+
+          builder.add_dependency_attributes(
+            :service_instances => ->(persister) { [persister.collections[:service_instances]] }
+          )
+
+          save_block = lambda do |source, tasks_collection|
+            service_instance_tasks_custom_save(source, tasks_collection)
+          end
+
+          builder.add_properties(:custom_save_block => save_block)
+        end
+      end
+
+      def service_instance_tasks_custom_save(source, tasks_collection)
+        service_instance_collection = tasks_collection.dependency_attributes[:service_instances]&.first
+        src_refs = service_instance_collection&.data.to_a.collect { |inventory_object| inventory_object.source_ref }
+        return if src_refs.blank?
+
+        # Updating Tasks
+        # service_instance_tasks_update_effective()
+        service_instance_tasks_update_ineffective(source, src_refs)
+      end
+
+      def task_update_values(svc_instance_id, external_url, status, finished_timestamp)
+        {
+          :state  => finished_timestamp.blank? ? 'running' : 'completed',
+          :status => %w[error failed].include?(status) ? 'error' : 'ok', # TODO: ansible-tower specific, normalize in collector
+          :context => {
+            :remote_status => status,
+            :service_instance => {
+              :id => svc_instance_id,
+              :url => external_url
+            }
+          }
+        }
+      end
+
+      # This method is updating one by one using ActiveRecord
+      def service_instance_tasks_update_ineffective(source, svc_instances_source_ref)
+        service_instances = ServiceInstance.where(:source_id => source.id, :source_ref => svc_instances_source_ref)
+        tasks_by_source_ref = Task.where(:state => 'running', :target_type => 'ServiceInstance', :source_id => source.id, :target_source_ref => service_instances.pluck(:source_ref)).index_by(&:target_source_ref)
+
+        service_instances.select(:id, :external_url, :source_ref, :extra).find_in_batches do |group|
+          ActiveRecord::Base.transaction do
+            group.each do |svc_instance|
+              next if tasks_by_source_ref[svc_instance.source_ref].nil?
+
+              values = task_update_values(svc_instance.id, svc_instance.external_url, svc_instance.extra['status'], svc_instance.extra['finished'])
+              tasks_by_source_ref[svc_instance.source_ref].update(values)
+            end
+          end
+        end
+      end
+
+      # This method is bulk updating by raw SQL query
+      def service_instance_tasks_update_effective
+        # Get running tasks
+        tasks_values = Task.where(:state => 'running', :target_type => 'ServiceInstance', :source_id => source.id)
+                         .pluck(:id, :target_source_ref)
+        tasks_id, tasks_source_ref = [], []
+        tasks_values.each do |attrs|
+          tasks_id << attrs[0]
+          tasks_source_ref << attrs[1]
+        end
+
+        # Load saved service instances (IDs needed)
+        service_instances_values = ServiceInstance.where(:source_ref => tasks_source_ref).pluck(:id, :external_url, :source_ref, Arel.sql("extra->'finished'"), Arel.sql("extra->'status'"))
+        return if service_instances_values.blank?
+
+        sql_update_values = []
+
+        # Preparing SQL update values from loaded ServiceInstances
+        service_instances_values.each do |attrs|
+          id, external_url, source_ref, finished_timestamp, status = attrs[0], attrs[1], attrs[2], attrs[3], attrs[4]
+
+          values = task_update_values(id, external_url, status, finished_timestamp)
+          sql_update_values << "('#{source_ref}', '#{values[:state]}', '#{values[:status]}', '#{values[:context].to_json}'::json)"
+        end
+
+        # Update query.
+        # Pairs records by `Task.target_type` and `Task.target_source_ref`
+        sql = <<SQL
+              UPDATE tasks AS t SET
+                state = c.state,
+                status = c.status,
+                context = c.context
+              FROM (VALUES :values
+              ) AS c(source_ref, state, status, context)
+              WHERE t.target_source_ref = c.source_ref
+                AND t.target_type = 'ServiceInstance';
+SQL
+        sql.sub!(':values', sql_update_values.join(','))
+
+        ActiveRecord::Base.connection.execute(sql)
+      end
+
       def add_vm_security_groups
         add_collection(:vm_security_groups) do |builder|
           add_default_properties(builder, manager_ref: [:vm, :security_group])

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: topological_inventory-core
 version: !ruby/object:Gem::Version
-  version: 1.0.0
+  version: 1.1.0
 platform: ruby
 authors:
 - Adam Grare
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-04-16 00:00:00.000000000 Z
+date: 2020-04-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: acts_as_tenant
@@ -30,14 +30,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.3.4
+        version: 0.3.5
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.3.4
+        version: 0.3.5
 - !ruby/object:Gem::Dependency
   name: manageiq-messaging
   requirement: !ruby/object:Gem::Requirement
@@ -249,6 +249,7 @@ files:
 - app/models/volume_type.rb
 - app/views/layouts/topological_inventory/application.html.erb
 - config/database.dev.yml
+- config/database.yml
 - db/migrate/20180919170553_add_sources_and_endpoints.rb
 - db/migrate/20180927171624_add_container_models.rb
 - db/migrate/20180927205317_add_service_catalog_models.rb
@@ -351,6 +352,8 @@ files:
 - db/migrate/20200313124806_remove_extraneous_index_from_service_credential_join_tables.rb
 - db/migrate/20200313141215_add_refresh_state_part_to_persisted_models.rb
 - db/migrate/20200317082640_add_tracking_timestamps_to_refresh_state.rb
+- db/migrate/20200403114310_add_target_and_source_to_task.rb
+- db/migrate/20200414123737_extract_tasks_source_ref_from_context.rb
 - db/schema.rb
 - db/seeds.rb
 - lib/tasks/topological_inventory_tasks.rake