topological_inventory-core 1.1.0 → 1.1.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 14973202e680d9bf035a3c85d5ef856140bf11d934bfc488cbade6932c8239c1
-  data.tar.gz: 7493112f3e945f0d0e20cb86385add9f45f623e2825c08b8a3a4ac2a62994822
+  metadata.gz: fc2335e206b9eba3ab955c62d29422156bb4053e737cd9aece9a5b91748b76ea
+  data.tar.gz: 163c7d564a74402c23b5001b885c2e83795263547592f83032c7514f771300a0
 SHA512:
-  metadata.gz: '08de615e242bc7dd1ec6f4349386064855b947602986736a8095c3b4802adc0fbeabd5aeb03dcc3d965d3a882f8a989c646624482c45de5235a5457797cef871'
-  data.tar.gz: fb1e4895ac6ecb98b6564ed411f5107417b5b2f68ee5a1346312f59b32598065d46d300ed158750fa0df2e85868958e7e347b2f82818e00d39db01cf1a414869
+  metadata.gz: c16a6361df474d38fc189a159f27a33ee3dd4b21d17cc0ae256128ae8ac6db0eb67d64b4e4103fff64fc8905169b7b86e1448e84df6d3ebf1391960fa721b8e0
+  data.tar.gz: 35e182d151981fa96c604c630c87686d3c0a1a65fc061789be277602c0b1b91e73d74e2fc3f5171b3754af0bf2cfba81a45bba20cbc6b9bd01579a222744fca2

data/README.md

@@ -1,4 +1,4 @@
-# TopologicalInventory
+# Topological Inventory Core
 
 [![Build Status](https://travis-ci.org/RedHatInsights/topological_inventory-core.svg)](https://travis-ci.org/RedHatInsights/topological_inventory-core)
 [![Maintainability](https://api.codeclimate.com/v1/badges/34f9bd9412e35c1a36fd/maintainability)](https://codeclimate.com/github/RedHatInsights/topological_inventory-core/maintainability)

data/config/database.yml

@@ -1,6 +1,8 @@
 default: &default
   adapter: postgresql
   encoding: utf8
+  host: localhost
+  port: 5432
   username: root
   pool: 5
   wait_timeout: 5
@@ -8,13 +10,13 @@ default: &default
 
 development:
   <<: *default
-  database: topological_inventory_core_development
+  database: topological_inventory_development
   min_messages: notice
 
 test:
   <<: *default
-  database: topological_inventory_core_test
+  database: topological_inventory_test
 
 production:
   <<: *default
-  database: topological_inventory_core_production
+  database: topological_inventory_production

data/db/migrate/20200414123737_extract_tasks_source_ref_from_context.rb

@@ -1,6 +1,11 @@
 class ExtractTasksSourceRefFromContext < ActiveRecord::Migration[5.2]
   def up
     Task.find_each do |task|
+      if task.context.is_a?(String)
+        say "Found legacy data in Task #{task.id}, converting String to Hash..."
+        task.context = JSON.parse(task.context)
+      end
+
       source_ref = task.context&.dig('service_instance', 'source_ref')
       next if source_ref.nil?
 

data/db/migrate/20200421141934_add_source_updated_at.rb

@@ -0,0 +1,5 @@
+class AddSourceUpdatedAt < ActiveRecord::Migration[5.2]
+  def change
+    add_column :volumes, :source_updated_at, :timestamp
+  end
+end

data/db/migrate/20200428090420_add_request_id_to_task.rb

@@ -0,0 +1,5 @@
+class AddRequestIdToTask < ActiveRecord::Migration[5.2]
+  def change
+    add_column :tasks, :x_rh_insights_request, :string
+  end
+end

data/db/migrate/20200505101547_add_forwardable_headers_remove_request_id_on_task.rb

@@ -0,0 +1,11 @@
+class AddForwardableHeadersRemoveRequestIdOnTask < ActiveRecord::Migration[5.2]
+  def up
+    add_column :tasks, :forwardable_headers, :jsonb
+    remove_column :tasks, :x_rh_insights_request
+  end
+
+  def down
+    add_column :tasks, :x_rh_insights_request, :string
+    remove_column :tasks, :forwardable_headers
+  end
+end

data/db/schema.rb

@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 2020_04_14_123737) do
+ActiveRecord::Schema.define(version: 2020_05_05_101547) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
@@ -1190,6 +1190,7 @@ ActiveRecord::Schema.define(version: 2020_04_14_123737) do
     t.string "target_source_ref"
     t.string "target_type"
     t.bigint "source_id"
+    t.jsonb "forwardable_headers"
     t.index ["source_id"], name: "index_tasks_on_source_id"
     t.index ["target_type", "target_source_ref"], name: "index_tasks_on_target_type_and_target_source_ref"
     t.index ["tenant_id"], name: "index_tasks_on_tenant_id"
@@ -1327,6 +1328,7 @@ ActiveRecord::Schema.define(version: 2020_04_14_123737) do
     t.bigint "orchestration_stack_id"
     t.bigint "subscription_id"
     t.bigint "refresh_state_part_id"
+    t.datetime "source_updated_at"
     t.index ["archived_at"], name: "index_volumes_on_archived_at"
     t.index ["last_seen_at"], name: "index_volumes_on_last_seen_at"
     t.index ["orchestration_stack_id"], name: "index_volumes_on_orchestration_stack_id"

data/lib/topological_inventory/core/version.rb

@@ -1,5 +1,5 @@
 module TopologicalInventory
   module Core
-    VERSION = '1.1.0'
+    VERSION = '1.1.5'
   end
 end

data/lib/topological_inventory/schema/default.rb

@@ -147,63 +147,70 @@ module TopologicalInventory
         return if src_refs.blank?
 
         # Updating Tasks
-        # service_instance_tasks_update_effective()
-        service_instance_tasks_update_ineffective(source, src_refs)
+        # service_instance_tasks_update_by_raw_sql(source)
+        service_instance_tasks_update_by_activerecord(tasks_collection, source, src_refs)
       end
 
-      def task_update_values(svc_instance_id, external_url, status, finished_timestamp)
+      def task_update_values(svc_instance_id, source_ref, external_url, status, task_status, finished_timestamp, source_id)
         {
           :state  => finished_timestamp.blank? ? 'running' : 'completed',
-          :status => %w[error failed].include?(status) ? 'error' : 'ok', # TODO: ansible-tower specific, normalize in collector
+          :status => task_status,
           :context => {
-            :remote_status => status,
             :service_instance => {
-              :id => svc_instance_id,
-              :url => external_url
+              :id         => svc_instance_id,
+              :job_status => status,
+              :source_id  => source_id,
+              :source_ref => source_ref,
+              :url        => external_url
             }
           }
         }
       end
 
       # This method is updating one by one using ActiveRecord
-      def service_instance_tasks_update_ineffective(source, svc_instances_source_ref)
+      def service_instance_tasks_update_by_activerecord(tasks_collection, source, svc_instances_source_ref)
         service_instances = ServiceInstance.where(:source_id => source.id, :source_ref => svc_instances_source_ref)
         tasks_by_source_ref = Task.where(:state => 'running', :target_type => 'ServiceInstance', :source_id => source.id, :target_source_ref => service_instances.pluck(:source_ref)).index_by(&:target_source_ref)
 
         service_instances.select(:id, :external_url, :source_ref, :extra).find_in_batches do |group|
           ActiveRecord::Base.transaction do
             group.each do |svc_instance|
-              next if tasks_by_source_ref[svc_instance.source_ref].nil?
+              next if (task = tasks_by_source_ref[svc_instance.source_ref]).nil?
 
-              values = task_update_values(svc_instance.id, svc_instance.external_url, svc_instance.extra['status'], svc_instance.extra['finished'])
-              tasks_by_source_ref[svc_instance.source_ref].update(values)
+              values = task_update_values(svc_instance.id, svc_instance.source_ref, svc_instance.external_url, svc_instance.extra['status'], svc_instance.extra['task_status'], svc_instance.extra['finished'], source.id)
+              # 1) Updating Task
+              task.update(values)
+
+              # 2) Saving to updated records (will be published in Kafka)
+              # - see topological_inventory-persister:Workflow.send_task_updates_to_queue!
+              tasks_collection.updated_records << values.merge(:id => task.id, :forwardable_headers => task.forwardable_headers)
             end
           end
         end
       end
 
       # This method is bulk updating by raw SQL query
-      def service_instance_tasks_update_effective
+      def service_instance_tasks_update_by_raw_sql(source)
         # Get running tasks
-        tasks_values = Task.where(:state => 'running', :target_type => 'ServiceInstance', :source_id => source.id)
-                         .pluck(:id, :target_source_ref)
-        tasks_id, tasks_source_ref = [], []
-        tasks_values.each do |attrs|
-          tasks_id << attrs[0]
-          tasks_source_ref << attrs[1]
-        end
+        tasks_source_ref = Task.where(:state => 'running', :target_type => 'ServiceInstance', :source_id => source.id)
+                         .pluck(:target_source_ref)
+        return if tasks_source_ref.blank?
 
         # Load saved service instances (IDs needed)
-        service_instances_values = ServiceInstance.where(:source_ref => tasks_source_ref).pluck(:id, :external_url, :source_ref, Arel.sql("extra->'finished'"), Arel.sql("extra->'status'"))
-        return if service_instances_values.blank?
+        svc_instances_values = ServiceInstance.where(:source_ref => tasks_source_ref)
+                                 .pluck(:id, :external_url, :source_ref,
+                                        Arel.sql("extra->'finished'"),
+                                        Arel.sql("extra->'status'"),
+                                        Arel.sql("extra->'task_status'"))
+        return if svc_instances_values.blank?
 
         sql_update_values = []
 
         # Preparing SQL update values from loaded ServiceInstances
-        service_instances_values.each do |attrs|
-          id, external_url, source_ref, finished_timestamp, status = attrs[0], attrs[1], attrs[2], attrs[3], attrs[4]
+        svc_instances_values.each do |attrs|
+          id, external_url, source_ref, finished_timestamp, status, task_status = attrs[0], attrs[1], attrs[2], attrs[3], attrs[4], attrs[5]
 
-          values = task_update_values(id, external_url, status, finished_timestamp)
+          values = task_update_values(id, external_url, status, task_status, finished_timestamp)
           sql_update_values << "('#{source_ref}', '#{values[:state]}', '#{values[:status]}', '#{values[:context].to_json}'::json)"
         end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: topological_inventory-core
 version: !ruby/object:Gem::Version
-  version: 1.1.0
+  version: 1.1.5
 platform: ruby
 authors:
 - Adam Grare
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-04-22 00:00:00.000000000 Z
+date: 2020-06-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: acts_as_tenant
@@ -136,20 +136,48 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '3.8'
+- !ruby/object:Gem::Dependency
+  name: rubocop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.69.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.69.0
+- !ruby/object:Gem::Dependency
+  name: rubocop-performance
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.3'
 - !ruby/object:Gem::Dependency
   name: simplecov
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.17.1
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.17.1
 - !ruby/object:Gem::Dependency
   name: webmock
   requirement: !ruby/object:Gem::Requirement
@@ -354,6 +382,9 @@ files:
 - db/migrate/20200317082640_add_tracking_timestamps_to_refresh_state.rb
 - db/migrate/20200403114310_add_target_and_source_to_task.rb
 - db/migrate/20200414123737_extract_tasks_source_ref_from_context.rb
+- db/migrate/20200421141934_add_source_updated_at.rb
+- db/migrate/20200428090420_add_request_id_to_task.rb
+- db/migrate/20200505101547_add_forwardable_headers_remove_request_id_on_task.rb
 - db/schema.rb
 - db/seeds.rb
 - lib/tasks/topological_inventory_tasks.rake
@@ -385,7 +416,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.2
+rubygems_version: 3.0.3
 signing_key: 
 specification_version: 4
 summary: Core Models and Schema for the Topological Inventory Service.