moose-inventory 2.0 → 2.1

data/lib/moose_inventory/db/schema_migrations.rb

@@ -0,0 +1,248 @@
+# frozen_string_literal: true
+
+module Moose
+  module Inventory
+    module DB
+      # Schema definitions, ordered migrations, and schema-artifact helpers for Moose Inventory DB.
+      # rubocop:disable Metrics/ModuleLength
+      module SchemaMigrations
+        SCHEMA_VERSION = 4
+
+        TABLE_DEFINITIONS = {
+          hosts: lambda do |db|
+            db.create_table(:hosts) do
+              primary_key :id
+              column :name, :text, unique: true
+            end
+          end,
+          hostvars: lambda do |db|
+            db.create_table(:hostvars) do
+              primary_key :id
+              foreign_key :host_id
+              column :name, :text
+              column :value, :text
+            end
+          end,
+          groups: lambda do |db|
+            db.create_table(:groups) do
+              primary_key :id
+              column :name, :text, unique: true
+            end
+          end,
+          groups_groups: lambda do |db|
+            db.create_table(:groups_groups) do
+              primary_key :id
+              foreign_key :parent_id, :groups
+              foreign_key :child_id, :groups
+            end
+          end,
+          groupvars: lambda do |db|
+            db.create_table(:groupvars) do
+              primary_key :id
+              foreign_key :group_id
+              column :name, :text
+              column :value, :text
+            end
+          end,
+          groups_hosts: lambda do |db|
+            db.create_table(:groups_hosts) do
+              primary_key :id
+              foreign_key :host_id, :hosts
+              foreign_key :group_id, :groups
+            end
+          end,
+          schema_info: lambda do |db|
+            db.create_table(:schema_info) do
+              primary_key :id
+              column :version, :integer, null: false
+            end
+          end,
+          audit_events: lambda do |db|
+            db.create_table(:audit_events) do
+              primary_key :id
+              column :created_at, :text, null: false
+              column :actor, :text
+              column :command, :text, null: false
+              column :action, :text, null: false
+              column :entity_type, :text
+              column :entity_name, :text
+              column :details, :text
+            end
+          end,
+          tags: lambda do |db|
+            db.create_table(:tags) do
+              primary_key :id
+              column :name, :text, unique: true, null: false
+            end
+          end,
+          hosts_tags: lambda do |db|
+            db.create_table(:hosts_tags) do
+              primary_key :id
+              foreign_key :host_id, :hosts
+              foreign_key :tag_id, :tags
+            end
+          end,
+          groups_tags: lambda do |db|
+            db.create_table(:groups_tags) do
+              primary_key :id
+              foreign_key :group_id, :groups
+              foreign_key :tag_id, :tags
+            end
+          end
+        }.freeze
+
+        SCHEMA_MIGRATIONS = {
+          1 => %i[hosts hostvars groups groups_groups groupvars groups_hosts schema_info],
+          2 => %i[audit_events],
+          3 => %i[tags hosts_tags groups_tags],
+          4 => []
+        }.freeze
+
+        INDEX_DEFINITIONS = [
+          { table: :hostvars, columns: %i[host_id name], unique: true, name: :idx_hostvars_host_id_name },
+          { table: :groupvars, columns: %i[group_id name], unique: true, name: :idx_groupvars_group_id_name },
+          { table: :groups_hosts, columns: %i[host_id group_id], unique: true,
+            name: :idx_groups_hosts_host_id_group_id },
+          { table: :groups_groups, columns: %i[parent_id child_id], unique: true,
+            name: :idx_groups_groups_parent_id_child_id },
+          { table: :hosts_tags, columns: %i[host_id tag_id], unique: true, name: :idx_hosts_tags_host_id_tag_id },
+          { table: :groups_tags, columns: %i[group_id tag_id], unique: true, name: :idx_groups_tags_group_id_tag_id },
+          { table: :groups_hosts, columns: %i[group_id host_id], unique: false,
+            name: :idx_groups_hosts_group_id_host_id },
+          { table: :groups_groups, columns: %i[child_id parent_id], unique: false,
+            name: :idx_groups_groups_child_id_parent_id },
+          { table: :hosts_tags, columns: %i[tag_id host_id], unique: false, name: :idx_hosts_tags_tag_id_host_id },
+          { table: :groups_tags, columns: %i[tag_id group_id], unique: false, name: :idx_groups_tags_tag_id_group_id }
+        ].freeze
+
+        def migration_versions
+          SCHEMA_MIGRATIONS.keys.sort
+        end
+
+        def schema_version
+          return nil unless @db.table_exists?(:schema_info)
+
+          @db[:schema_info].order(:id).last&.fetch(:version)
+        end
+
+        def migrate_schema!
+          reject_future_schema!
+          migration_versions.each do |version|
+            next if schema_version.to_i >= version && !schema_migration_artifacts_missing?(version)
+
+            apply_schema_migration!(version)
+          end
+        end
+
+        def schema_migration_artifacts_missing?(version)
+          schema_migration_tables_missing?(version) || (version == 4 && schema_indexes_missing?)
+        end
+
+        def schema_migration_tables_missing?(version)
+          SCHEMA_MIGRATIONS.fetch(version).any? { |table_name| !@db.table_exists?(table_name) }
+        end
+
+        def schema_indexes_missing?
+          INDEX_DEFINITIONS.any? do |definition|
+            !@db.table_exists?(definition.fetch(:table)) || !index_exists?(definition.fetch(:table),
+                                                                           definition.fetch(:name))
+          end
+        end
+
+        def apply_schema_migration!(version)
+          tables = SCHEMA_MIGRATIONS.fetch(version)
+          tables.each { |table_name| create_table(table_name) }
+          apply_schema_indexes! if version == 4
+          record_schema_version!(version)
+        end
+
+        def apply_schema_indexes!
+          clean_duplicate_index_rows!
+          INDEX_DEFINITIONS.each { |definition| add_index(definition) }
+        end
+
+        def clean_duplicate_index_rows!
+          dedupe_duplicate_rows!(:hostvars, %i[host_id name], value_columns: [:value])
+          dedupe_duplicate_rows!(:groupvars, %i[group_id name], value_columns: [:value])
+          dedupe_duplicate_rows!(:groups_hosts, %i[host_id group_id])
+          dedupe_duplicate_rows!(:groups_groups, %i[parent_id child_id])
+          dedupe_duplicate_rows!(:hosts_tags, %i[host_id tag_id])
+          dedupe_duplicate_rows!(:groups_tags, %i[group_id tag_id])
+        end
+
+        def dedupe_duplicate_rows!(table_name, columns, value_columns: [])
+          duplicate_keys(table_name, columns).each do |key|
+            rows = @db[table_name].where(key).order(:id).all
+            reject_conflicting_duplicates!(table_name, key, rows, value_columns)
+            @db[table_name].where(id: rows.drop(1).map { |row| row.fetch(:id) }).delete
+          end
+        end
+
+        def duplicate_keys(table_name, columns)
+          @db[table_name]
+            .select(*columns)
+            .group(*columns)
+            .having { count(id) > 1 }
+            .all
+        end
+
+        def reject_conflicting_duplicates!(table_name, key, rows, value_columns)
+          conflicts = value_columns.any? do |column|
+            rows.map { |row| row[column] }.uniq.length > 1
+          end
+          return unless conflicts
+
+          raise @exceptions[:moose], "Cannot add unique indexes because #{table_name} has conflicting duplicates " \
+                                     "for #{key}. Resolve duplicate values before migrating."
+        end
+
+        def add_index(definition)
+          return if index_exists?(definition.fetch(:table), definition.fetch(:name))
+
+          @db.add_index(definition.fetch(:table), definition.fetch(:columns), unique: definition.fetch(:unique),
+                                                                              name: definition.fetch(:name))
+        end
+
+        def index_exists?(table_name, index_name)
+          @db.indexes(table_name).key?(index_name)
+        end
+
+        def record_schema_version!(version)
+          unless @db.table_exists?(:schema_info)
+            raise @exceptions[:moose],
+                  'Cannot record schema version before schema_info exists.'
+          end
+
+          if @db[:schema_info].empty?
+            @db[:schema_info].insert(version: version)
+          else
+            @db[:schema_info].update(version: version)
+          end
+        end
+
+        def reject_future_schema!
+          return unless @db.table_exists?(:schema_info)
+
+          current_version = schema_version
+          return if current_version.nil? || current_version <= SCHEMA_VERSION
+
+          raise @exceptions[:moose], "Database schema version #{current_version} is newer than supported version " \
+                                     "#{SCHEMA_VERSION}. Upgrade moose-inventory before using this database."
+        end
+
+        def create_tables
+          TABLE_DEFINITIONS.each do |table_name, definition|
+            create_table(table_name, definition)
+          end
+        end
+
+        def create_table(table_name, definition = TABLE_DEFINITIONS.fetch(table_name))
+          return if @db.table_exists?(table_name)
+
+          definition.call(@db)
+        end
+      end
+      # rubocop:enable Metrics/ModuleLength
+    end
+  end
+end

data/lib/moose_inventory/inventory_context.rb

@@ -1,16 +1,18 @@
 # frozen_string_literal: true
 
+require 'time'
+
 module Moose
   module Inventory
     ##
-    # Thin facade over the current DB singleton.
+    # Thin facade over an explicitly supplied DB implementation.
     #
     # This gives new operation/service objects a small inventory-facing seam
     # without forcing the legacy CLI to stop using the DB singleton all at once.
     class InventoryContext
       AUTOMATIC_GROUP = 'ungrouped'
 
-      def initialize(db: Moose::Inventory::DB)
+      def initialize(db:)
         @db = db
       end
 
@@ -38,12 +40,76 @@ module Moose
         db.models[:group].find_or_create(name: name)
       end
 
+      def find_tag(name)
+        db.models[:tag].find(name: normalize_tag_name(name))
+      end
+
+      def find_or_create_tag(name)
+        db.models[:tag].find_or_create(name: normalize_tag_name(name))
+      end
+
+      def normalize_tag_name(name)
+        name.to_s.downcase.strip
+      end
+
+      def normalize_tag_names(names)
+        names.map { |name| normalize_tag_name(name) }.reject(&:empty?).uniq
+      end
+
+      def hosts_dataset
+        db.models[:host].dataset
+      end
+
+      def db_dataset(table_name)
+        db.db[table_name]
+      end
+
       def automatic_group
         find_or_create_group(AUTOMATIC_GROUP)
       end
 
+      def find_variable(entity_type, id)
+        db.models[variable_model_key(entity_type)].find(id: id)
+      end
+
+      def create_variable(entity_type, name:, value:)
+        db.models[variable_model_key(entity_type)].create(name: name, value: value)
+      end
+
+      def all_hosts
+        db.models[:host].all
+      end
+
+      def all_groups
+        db.models[:group].all
+      end
+
+      def record_audit_event(attributes)
+        db.models[:audit_event].create(
+          created_at: Time.now.utc.iso8601,
+          actor: attributes[:actor],
+          command: attributes.fetch(:command),
+          action: attributes.fetch(:action),
+          entity_type: attributes[:entity_type],
+          entity_name: attributes[:entity_name],
+          details: attributes[:details]
+        )
+      end
+
+      def audit_events(limit: 20)
+        db.models[:audit_event].reverse_order(:id).limit(limit).all
+      end
+
+      def moose_exception_class
+        db.exceptions[:moose]
+      end
+
       private
 
+      def variable_model_key(entity_type)
+        :"#{entity_type}var"
+      end
+
       attr_reader :db
     end
   end

data/lib/moose_inventory/operations/add_associations.rb

@@ -1,21 +1,23 @@
 # frozen_string_literal: true
 
+require_relative 'operation_event_support'
+
 module Moose
   module Inventory
     module Operations
       # Adds host/group associations for existing primary entities.
       class AddAssociations
         AUTOMATIC_GROUP = 'ungrouped'
-        Event = Struct.new(:type, :payload, keyword_init: true)
-        Result = Struct.new(:events, :warning_count, keyword_init: true)
+        include OperationEventSupport
 
         def initialize(context:)
           @context = context
         end
 
-        def host_to_groups(host:, host_name:, group_names:)
+        def host_to_groups(host:, host_name:, group_names:, dry_run: false)
           events = []
           warning_count = 0
+          @dry_run = dry_run
 
           group_names.each do |group_name|
             next if group_name.nil? || group_name.empty?
@@ -24,13 +26,15 @@ module Moose
           end
 
           remove_automatic_group_from_host(host, host_name, events)
+          emit(events, :dry_run_summary) if dry_run
 
-          Result.new(events: events, warning_count: warning_count)
+          operation_result(events: events, warning_count: warning_count)
         end
 
-        def group_to_hosts(group:, group_name:, host_names:)
+        def group_to_hosts(group:, group_name:, host_names:, dry_run: false)
           events = []
           warning_count = 0
+          @dry_run = dry_run
           hosts_dataset = group.hosts_dataset
 
           host_names.each do |host_name|
@@ -45,12 +49,14 @@ module Moose
             )
           end
 
-          Result.new(events: events, warning_count: warning_count)
+          emit(events, :dry_run_summary) if dry_run
+
+          operation_result(events: events, warning_count: warning_count)
         end
 
         private
 
-        attr_reader :context
+        attr_reader :context, :dry_run
 
         def add_group_to_host(host, host_name, group_name, events)
           warning_count = 0
@@ -69,12 +75,12 @@ module Moose
           if group.nil?
             emit(events, :group_missing_created, name: group_name)
             emit(events, :group_creating_now, name: group_name)
-            group = context.create_group(group_name)
+            group = context.create_group(group_name) unless dry_run
             emit(events, :ok, indent: 6)
             warning_count += 1
           end
 
-          host.add_group(group)
+          host.add_group(group) unless dry_run
           emit(events, :ok, indent: 4)
           warning_count
         end
@@ -94,33 +100,31 @@ module Moose
           if host.nil?
             emit(events, :host_missing_created, name: host_name)
             emit(events, :host_creating_now, name: host_name)
-            host = context.create_host(host_name)
+            host = context.create_host(host_name) unless dry_run
             emit(events, :ok, indent: 6)
             warning_count += 1
           end
 
-          group.add_host(host)
+          group.add_host(host) unless dry_run
           emit(events, :ok, indent: 4)
           remove_automatic_group_from_host(host, host_name, events)
           warning_count
         end
 
         def remove_automatic_group_from_host(host, host_name, events)
+          return if host.nil?
+
           ungrouped = host.groups_dataset[name: AUTOMATIC_GROUP]
           return if ungrouped.nil?
 
           emit(events, :removing_automatic_group, host: host_name)
-          host.remove_group(ungrouped)
+          host.remove_group(ungrouped) unless dry_run
           emit(events, :ok, indent: 4)
         end
 
         def association_exists?(dataset, name)
           !dataset.nil? && !dataset[name: name].nil?
         end
-
-        def emit(events, type, payload = {})
-          events << Event.new(type: type, payload: payload)
-        end
       end
     end
   end

data/lib/moose_inventory/operations/add_groups.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require_relative 'operation_event_support'
+
 module Moose
   module Inventory
     module Operations
@@ -7,16 +9,24 @@ module Moose
       # Adds groups and their optional host associations.
       class AddGroups
         AUTOMATIC_GROUP = 'ungrouped'
-        Event = Struct.new(:type, :payload, keyword_init: true)
-        Result = Struct.new(:events, :warning_count, keyword_init: true)
+        include OperationEventSupport
 
         def initialize(context:)
           @context = context
         end
 
-        def call(names:, hosts:)
+        def call(names:, hosts:, dry_run: false)
           events = []
           warning_count = 0
+          @dry_run = dry_run
+
+          if dry_run
+            names.each do |name|
+              warning_count += add_group(name, hosts, events)
+            end
+            emit(events, :dry_run_summary)
+            return operation_result(events: events, warning_count: warning_count)
+          end
 
           context.transaction do
             names.each do |name|
@@ -24,12 +34,12 @@ module Moose
             end
           end
 
-          Result.new(events: events, warning_count: warning_count)
+          operation_result(events: events, warning_count: warning_count)
         end
 
         private
 
-        attr_reader :context
+        attr_reader :context, :dry_run
 
         def add_group(name, hosts, events)
           warning_count = 0
@@ -52,7 +62,7 @@ module Moose
           group = context.find_group(name)
 
           if group.nil?
-            group = context.create_group(name)
+            group = context.create_group(name) unless dry_run
             emit(events, :ok, indent: 4)
             [group, nil, true]
           else
@@ -73,7 +83,7 @@ module Moose
             emit(events, :association_exists, group: group_name, host: host_name)
             emit(events, :already_exists_skipping, indent: 4)
             warning_count += 1
-          else
+          elsif !dry_run
             group.add_host(host)
           end
           emit(events, :ok, indent: 4)
@@ -88,27 +98,25 @@ module Moose
 
           emit(events, :host_missing_created, name: name)
           emit(events, :host_creating_now, name: name)
-          host = context.create_host(name)
+          host = context.create_host(name) unless dry_run
           emit(events, :ok, indent: 6)
           [host, :warned_create]
         end
 
         def remove_automatic_group_from_host(host, host_name, events)
+          return if host.nil?
+
           ungrouped = host.groups_dataset[name: AUTOMATIC_GROUP]
           return if ungrouped.nil?
 
           emit(events, :removing_automatic_group, host: host_name)
-          host.remove_group(ungrouped)
+          host.remove_group(ungrouped) unless dry_run
           emit(events, :ok, indent: 4)
         end
 
         def association_exists?(dataset, name)
           !dataset.nil? && !dataset[name: name].nil?
         end
-
-        def emit(events, type, payload = {})
-          events << Event.new(type: type, payload: payload)
-        end
       end
     end
   end

data/lib/moose_inventory/operations/add_hosts.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require_relative 'operation_event_support'
+
 module Moose
   module Inventory
     module Operations
@@ -12,26 +14,35 @@ module Moose
       # to progress text.
       class AddHosts
         AUTOMATIC_GROUP = 'ungrouped'
-        Event = Struct.new(:type, :payload, keyword_init: true)
-        Result = Struct.new(:events, keyword_init: true)
+        include OperationEventSupport
 
         def initialize(context:)
           @context = context
         end
 
-        def call(names:, groups:)
+        def call(names:, groups:, dry_run: false)
           events = []
+          @dry_run = dry_run
+
+          if dry_run
+            names.each do |name|
+              add_host(name, groups, events)
+            end
+            emit(events, :dry_run_summary)
+            return operation_result(events: events)
+          end
+
           context.transaction do
             names.each do |name|
               add_host(name, groups, events)
             end
           end
-          Result.new(events: events)
+          operation_result(events: events)
         end
 
         private
 
-        attr_reader :context
+        attr_reader :context, :dry_run
 
         def add_host(name, groups, events)
           emit(events, :host_started, name: name)
@@ -41,7 +52,7 @@ module Moose
             add_group_association(host, name, group_name, groups_dataset, events)
           end
 
-          add_automatic_group_if_needed(host, name, events)
+          add_automatic_group_if_needed(host, name, groups, groups_dataset, events)
           emit(events, :host_complete)
         end
 
@@ -51,7 +62,7 @@ module Moose
           groups_dataset = nil
 
           if host.nil?
-            host = context.create_host(name)
+            host = context.create_host(name) unless dry_run
           else
             emit(events, :host_exists, name: name)
             groups_dataset = host.groups_dataset
@@ -69,7 +80,7 @@ module Moose
 
           if association_exists?(groups_dataset, group_name)
             emit(events, :association_exists, host: host_name, group: group_name)
-          else
+          elsif !dry_run
             host.add_group(group)
           end
 
@@ -81,18 +92,24 @@ module Moose
           return group unless group.nil?
 
           emit(events, :group_missing_created, name: name)
-          context.create_group(name)
+          context.create_group(name) unless dry_run
         end
 
-        def add_automatic_group_if_needed(host, host_name, events)
-          groups_dataset = host.groups_dataset
-          return if groups_dataset.nil? || groups_dataset.any?
+        def add_automatic_group_if_needed(host, host_name, requested_groups, groups_dataset, events)
+          return unless automatic_group_needed?(host, requested_groups, groups_dataset)
 
           emit(events, :adding_automatic_group, host: host_name, group: AUTOMATIC_GROUP)
-          host.add_group(automatic_group)
+          host.add_group(automatic_group) unless dry_run
           emit(events, :ok, indent: 4)
         end
 
+        def automatic_group_needed?(host, requested_groups, groups_dataset)
+          return requested_groups.empty? && (host.nil? || groups_dataset.nil? || groups_dataset.none?) if dry_run
+
+          groups_dataset = host.groups_dataset
+          !groups_dataset.nil? && groups_dataset.none?
+        end
+
         def automatic_group
           context.automatic_group
         end
@@ -100,10 +117,6 @@ module Moose
         def association_exists?(dataset, name)
           !dataset.nil? && !dataset[name: name].nil?
         end
-
-        def emit(events, type, payload = {})
-          events << Event.new(type: type, payload: payload)
-        end
       end
     end
   end