has_helpers 5.0.2

data/app/importers/has_helpers/cards/importer.rb

@@ -0,0 +1,1223 @@
+# frozen_string_literal: true
+
+# HasHelpers::Cards::Importer
+#
+# Description:
+#
+#   Implemented service responsible for persisting all card
+#   definitions declared via the DSL (HasHelpers::Cards::CardBuilder::Base subclasses)
+#   into the database tables:
+#
+#     - card_definitions
+#     - card_fields
+#     - card_definition_resources
+#
+#   It must:
+#     * Upsert records deterministically (idempotent)
+#     * Support selective import by resource or all
+#     * Perform auditing (checksum + diffing) to detect drift
+#       between code and DB definitions
+#     * Fail safely when structural changes occur
+#       (e.g. field removals / relationship removals / data_type changes)
+#     * Persist a definition_json and definition_checksum
+#       for each record for future audits
+#
+# This class implements that contract.
+#
+# Important notes:
+#   - The importer operates at "card" level (one transaction per card).
+#   - It uses the normalized DSL definitions (from HasHelpers::Cards::CardBuilder::Base subclasses).
+#   - It computes checksums based on a normalized JSON representation.
+#   - It supports "dry_run" mode (audit + diff only, no DB writes).
+#   - It supports resource filtering (import only cards that involve specific resources).
+
+require "digest"
+require "json"
+
+module HasHelpers
+  module Cards
+    class Importer
+      # Raised when auditing detects a breaking drift between DSL (code) and DB.
+      # We use this error to stop the import and force devs to write a data migration.
+      class AuditError < StandardError; end
+
+      # Struct used to return a summary of the import operation.
+      Result = Struct.new(
+        :created_cards,
+        :updated_cards,
+        :created_fields,
+        :updated_fields,
+        :created_resources,
+        :updated_resources,
+        :skipped_cards,
+        keyword_init: true
+      )
+
+      # Struct representing the diff between DSL and DB for a single card.
+      #  Auditing / drift:
+      #  - card_changes: :new, :same, :changed
+      #  - fields_added / fields_removed / fields_changed
+      #  - resources_added / resources_removed
+      Diff = Struct.new(
+        :card_changes,
+        :fields_added,
+        :fields_removed,
+        :fields_changed,
+        :resources_added,
+        :resources_removed,
+        keyword_init: true
+      )
+
+      class << self
+        # Public API entry point.
+        #   # Import all resources declared in the DSL
+        #   HasHelpers::Cards::Importer.run
+        #
+        #   # Import only selected resource types
+        #   HasHelpers::Cards::Importer.run([Advisor, Carrier]) # or symbols: [:advisor, :carrier]
+        #
+        #   # Options (keyword args)
+        #   HasHelpers::Cards::Importer.run(resources: [:advisor], mode: :apply, audit: true)
+        #
+        # Supported modes (ClickUp):
+        #   - mode: :dry_run → compute and print diffs, no DB writes,
+        #     raise error if audit fails.
+        #   - mode: :apply   → default. Always audit first. If drift is breaking,
+        #     raise and do not persist.
+        def run(*args, **opts)
+          ::HasHelpers::Cards::Registry.cards
+          resources, mode, audit = normalize_run_arguments(args, opts)
+
+          application = opts[:application] || opts[:application_name]
+          raise ArgumentError, "application is required (e.g. application: 'AgenciesHQ')" if application.blank?
+
+          new(application: application, resources: resources, mode: mode, audit: audit).run
+        end
+
+        private
+
+        # Normalizes positional + keyword arguments into:
+        #   - resources (filter) → Array<Class|Symbol> or nil
+        #   - mode               → :apply or :dry_run
+        #   - audit              → boolean
+        #
+        # This is compatible with:
+        #   HasHelpers::Cards::Importer.run
+        #   HasHelpers::Cards::Importer.run([Advisor, Carrier])
+        #   HasHelpers::Cards::Importer.run(resources: [:advisor], mode: :apply, audit: true)
+        #
+        def normalize_run_arguments(args, opts)
+          # default mode is :apply
+          mode = (opts[:mode] || :apply).to_sym
+          # default audit is true (we always audit before writing)
+          audit = opts.key?(:audit) ? opts[:audit] : true
+
+          resources =
+            if opts.key?(:resources)
+              # explicit keyword arg takes precedence
+              opts[:resources]
+            else
+              # otherwise, optional first positional arg may be the resources filter
+              args.first
+            end
+
+          [resources, mode, audit]
+        end
+      end
+
+      # Constructor is internal; external callers use .run.
+      #
+      #   resources: Class|Symbol|Array or nil (no filter)
+      #   mode: :apply or :dry_run
+      #   audit: true/false (whether to run drift detection)
+      #
+      def initialize(application:, resources:, mode:, audit:)
+        # Normalize resource filter into a compact set of symbols representing resource names (e.g. [:advisor, :carrier]).
+        @application_name = application.to_s
+        @application = ::HasHelpers::Application.find_by!("LOWER(name) = ?", @application_name.downcase)
+        @resource_filter = normalize_resource_filter(resources)
+        @mode            = (mode || :apply).to_sym
+        @audit           = audit
+
+        # This struct accumulates counts while importing.
+        @result = Result.new(
+          created_cards:     0,
+          updated_cards:     0,
+          created_fields:    0,
+          updated_fields:    0,
+          created_resources: 0,
+          updated_resources: 0,
+          skipped_cards:     0
+        )
+      end
+
+      # Main execution entry.
+      #
+      # "Importer Flow"
+      #
+      # Steps:
+      #   1. (Optionally) run migrations for cards (hook only for now).
+      #   2. Discover card definitions from the DSL (CardBuilder::Base descendants).
+      #   3. For each card:
+      #       - Normalize + compute checksum (code view).
+      #       - Build DB view (if card exists).
+      #       - Diff + classify changes.
+      #       - Audit: if breaking drift, raise AuditError.
+      #       - If mode = :apply, upsert card_definitions + card_fields +
+      #         card_definition_resources inside a transaction.
+      #       - If mode = :dry_run, just log diffs and don't write.
+      #   4. Optional global audit: detect cards existing only in DB but
+      #      not in code, when there is no resource filter (breaking).
+      #   5. Log summary of created/updated/skipped records.
+      #
+      def run
+        # "We may add a migrations section just like we have for reports and webhooks"
+        run_migrations_if_any!
+
+        # Discover all card definitions exposed by the DSL (HasHelpers::Cards::CardBuilder::Base subclasses).
+        card_pairs = discover_card_definitions
+        # Each element is: [CardClass, raw_hash_definition]
+        validate_hover_cards!(card_pairs)
+
+        # Hover-only cards live entirely in-memory (DSL registry). They are NOT persisted.
+        # Any pre-existing rows for hover-only identifiers (whether currently flagged as
+        # hover_only in DB or not) are dropped here, then we skip the import for them below.
+        purge_persisted_hover_only_cards!(card_pairs)
+
+        # Import each card definition (or audit, depending on mode), skipping hover-only definitions.
+        card_pairs.each do |klass, raw_def|
+          next if hover_only_definition?(raw_def)
+          import_card!(klass, raw_def)
+        end
+
+        # Global audit for "cards present in DB but missing in code".
+        # This is only meaningful when we are importing "all resources" (i.e. no resource filter is applied).
+        audit_removed_cards!(card_pairs) if audit && resource_filter.nil?
+
+        # Log structured summary for debugging / future dashboards.
+        log_summary
+
+        # Return Result struct (created/updated counts, etc.)
+        result
+      end
+
+      # Definitions where `hover_only: true` are skipped at import — they live in
+      # memory only and are assembled from the registry at request time.
+      def hover_only_definition?(raw_def)
+        defn = raw_def.is_a?(Hash) ? raw_def.deep_symbolize_keys : raw_def.to_h.deep_symbolize_keys
+        !!defn[:hover_only]
+      end
+
+      # Removes any persisted CardDefinition rows that match a hover-only DSL definition
+      # (by identifier). Hover-only cards are in-memory only; this purges any rows left
+      # over from prior runs before we made them in-memory. Safe to call repeatedly.
+      # Dependent rows cascade via :dependent => :destroy associations.
+      def purge_persisted_hover_only_cards!(card_pairs)
+        return if mode == :dry_run
+
+        hover_only_ids =
+          card_pairs.
+            select { |_klass, defn| hover_only_definition?(defn) }.
+            map { |_klass, defn| normalize_identifier((defn.deep_symbolize_keys)[:identifier]) }.
+            compact.
+            uniq
+        return if hover_only_ids.empty?
+
+        identifier_ids = ::HasHelpers::Cards::CardIdentifier.where(name: hover_only_ids).pluck(:id)
+        return if identifier_ids.empty?
+
+        ::HasHelpers::Cards::CardDefinition.where(identifier_id: identifier_ids).find_each(&:destroy)
+      end
+
+      private
+
+      attr_reader :resource_filter, :mode, :audit, :result, :application_name, :application
+
+      # Placeholder to run idempotent migrations before importing.
+      def run_migrations_if_any!
+        # Example: HasHelpers::Cards::Migrations.run_all if defined?(HasHelpers::Cards::Migrations)
+      end
+
+      # 1) Discover Cards
+      # Enumerate all classes inheriting from HasHelpers::Cards::CardBuilder::Base.
+      # Each subclass has already evaluated the DSL and stored its normalized definitions in .definitions.
+      # We also apply resource filtering here ( "Support selective import by resource or all").
+      def discover_card_definitions
+        ::HasHelpers::Cards::CardBuilder::Base.
+          descendants.
+          flat_map do |klass|
+            # Each card builder class can define one or many card definitions.
+            Array(klass.definitions).
+              flat_map do |defn|
+                defn = defn.deep_symbolize_keys
+                defn[:application] ||= application_name if application_name.present?
+                expand_per_resource(defn)
+              end.
+              map { |defn| [klass, defn] }
+          end.
+          select { |_klass, defn| passes_resource_filter?(defn) }
+      end
+
+      # Splits a normalized card definition into one per resource.
+      # Original DSL allows multiple resources per card; we materialize a separate
+      # definition for each resource to keep a 1:1 relationship.
+      def expand_per_resource(definition)
+        resources = Array(definition[:resources])
+        return [definition] if resources.empty?
+
+        resources.map do |res|
+          dup = definition.deep_dup
+          dup[:resources] = [res]
+          dup
+        end
+      end
+
+      # Returns true if this card definition should be imported for the current resource_filter.
+      # Resource filter semantics:
+      #   - Accept classes (Advisor) or symbols (:advisor).
+      #   - If filter is nil/empty, we import all cards.
+      # We map each DSL resource (class_name) to a symbol (advisor, client, etc.), then intersect with the filter.
+      def passes_resource_filter?(definition)
+        return true if resource_filter.blank?
+
+        dsl_resources =
+          Array(definition[:resources]).
+            map { |r| r[:class_name] || r["class_name"] }.
+            compact.
+            map(&:to_s)
+
+        (dsl_resources.map { |cn| normalize_resource_name(cn) } & resource_filter).any?
+      end
+
+      def validate_hover_cards!(card_pairs)
+        definitions =
+          card_pairs.map do |_klass, defn|
+            defn.deep_symbolize_keys
+          end
+
+        definitions_by_identifier =
+          definitions.index_by { |defn| normalize_identifier(defn[:identifier]) }
+
+        definitions.each do |definition|
+          parent_identifier = normalize_identifier(definition[:identifier])
+          parent_name = definition[:name].to_s
+
+          Array(definition[:fields]).each do |field|
+            field = field.deep_symbolize_keys
+            next unless normalize_data_type_name(field[:data_type]) == :hover_card
+
+            field_key = hover_field_key(field)
+            raw_hover_card = field[:hover_card]
+            if raw_hover_card.is_a?(Hash)
+              raise AuditError,
+                    "Card '#{parent_name}' (identifier=#{parent_identifier}) field '#{field_key}' passed a Hash to `hover_card:` (#{raw_hover_card.inspect}). " \
+                    "Use `hover_card_source:` for dynamic identifiers resolved from a data_source."
+            end
+            hover_card_identifier = raw_hover_card.is_a?(String) ? raw_hover_card.presence : nil
+            dynamic_hover_card = field[:hover_card_source].present?
+
+            unless hover_card_identifier || dynamic_hover_card
+              raise AuditError,
+                    "Card '#{parent_name}' (identifier=#{parent_identifier}) field '#{field_key}' declares data_type HoverCard but is missing hover_card (or hover_card_source)."
+            end
+
+            # hover_record_source is optional for :resource-kind fields: the field's own
+            # resolved value (e.g. `policy.carrier_id`) is used as the hover target id.
+            if field[:hover_record_source].blank? && field[:kind].to_s.to_sym != :resource
+              raise AuditError,
+                    "Card '#{parent_name}' (identifier=#{parent_identifier}) field '#{field_key}' declares hover_card '#{hover_card_identifier || '(dynamic)'}' but is missing hover_record_source."
+            end
+
+            # Dynamic hover_card targets are resolved at request time; we can't audit
+            # their target shape statically.
+            next if dynamic_hover_card && hover_card_identifier.nil?
+
+            target = definitions_by_identifier[hover_card_identifier]
+            unless target
+              raise AuditError,
+                    "Card '#{parent_name}' (identifier=#{parent_identifier}) field '#{field_key}' references hover_card '#{hover_card_identifier}', but no card with that identifier exists in this import."
+            end
+
+            unless target[:hover_only]
+              raise AuditError,
+                    "Card '#{parent_name}' (identifier=#{parent_identifier}) field '#{field_key}' references hover_card '#{hover_card_identifier}', but that card is not hover_only: true."
+            end
+
+            nested_field = Array(target[:fields]).find do |target_field|
+              normalize_data_type_name(target_field.deep_symbolize_keys[:data_type]) == :hover_card
+            end
+
+            next unless nested_field
+
+            nested_key = hover_field_key(nested_field.deep_symbolize_keys)
+            raise AuditError,
+                  "Card '#{parent_name}' (identifier=#{parent_identifier}) field '#{field_key}' references hover_card '#{hover_card_identifier}', but that hover-only card contains nested HoverCard field '#{nested_key}'."
+          end
+        end
+      end
+
+      def hover_field_key(field)
+        (field[:key] || field[:ref] || field[:name]).to_s
+      end
+
+      # Converts "Advisor" → :advisor, "HasContactInfo::Contact" → :contact, etc.
+      def normalize_resource_name(class_name)
+        class_name.to_s.demodulize.underscore.to_sym
+      end
+
+      # Normalizes the resource filter argument into an array of symbols.
+      # Examples:
+      #   normalize_resource_filter(nil) → nil  (no filter)
+      #   normalize_resource_filter([Advisor, Carrier]) → [:advisor, :carrier]
+      #   normalize_resource_filter(:advisor) → [:advisor]
+      def normalize_resource_filter(resources)
+        return nil if resources.nil?
+
+        list = resources.is_a?(Array) ? resources : [resources]
+        list.map { |r| r.is_a?(Class) ? normalize_resource_name(r.name) : r.to_sym }.uniq
+      end
+
+      # 2) Normalize + checksum
+      # Normalizes a raw DSL hash definition into a deterministic structure.
+      # This is critical because:
+      #   - We will compute a checksum = SHA256(JSON(definition_json)).
+      #   - We want the JSON to be stable across runs (idempotency).
+      #   - We want to exclude transient / DB-specific attributes (timestamps, ids).
+      # It also aligns with the "definition_json" we persist in card_definitions.
+      def normalize_definition(raw)
+        card = raw.deep_dup
+
+        # icon_name / icon_color are non-persisted, card-level presentation hints.
+        # They live in the DSL registry only, so we drop them before building the
+        # definition_json / checksum to keep persistence (and drift audits) stable.
+        card.delete(:icon_name)
+        card.delete(:icon_color)
+
+        card[:name]            = card[:name].to_s
+        card[:identifier]      = normalize_identifier(card[:identifier])
+        # relationship is currently not persisted in DB, but we keep it in JSON
+        card[:relationship]    = card[:relationship].to_s if card[:relationship]
+        card[:layout_type]     = normalize_enum_name(card[:layout_type])
+        card[:variant]         = normalize_enum_name(card[:variant])
+        card[:description]     = card[:description].to_s if card.key?(:description)
+        card[:is_customizable] = !!card[:is_customizable]
+        card[:default_visibility] = card[:default_visibility] # rubocop:disable Lint/SelfAssignment
+        card[:virtual] = !!card[:virtual]
+        card[:default_position] = card[:default_position] # rubocop:disable Lint/SelfAssignment
+        card[:expansion_scope] = nil # runtime-only
+        card[:data_sources] =
+          Array(card[:data_sources]).
+            map { |ds| normalize_data_source(ds) }.
+            compact
+        # Normalize resources: array of hashes { class_name, is_mandatory }
+        card[:resources] =
+          Array(card[:resources]).
+            map do |r|
+              { class_name: r[:class_name].to_s, is_mandatory: !!r[:is_mandatory] }
+            end.
+            sort_by { |r| r[:class_name] } # deterministic ordering
+        # Normalize fields: we standardize keys + values and then sort.
+        card[:fields] =
+          Array(card[:fields]).
+            map { |f| normalize_field_hash(f) }.
+            sort_by { |f| [f[:default_position] || 0, field_identity(f)] }
+
+        card
+      end
+
+      # Normalizes identifier (option) into a string name.
+      # e.g. HasHelpers::Cards::CardIdentifier::DEMOGRAPHICS -> "Demographics"
+      def normalize_identifier(value)
+        return nil if value.nil?
+        value.respond_to?(:name) && !value.is_a?(String) ? value.name.to_s : value.to_s
+      end
+
+      # Normalizes enums (layout_type, variant) to their name strings.
+      # e.g. HasHelpers::Cards::CardLayoutType::SUMMARY → "Summary".
+      def normalize_enum_name(value)
+        return nil if value.nil?
+
+        if value.respond_to?(:name) && !value.is_a?(String)
+          value.name.to_s
+        else
+          value.to_s
+        end
+      end
+
+      # Normalizes a field hash (resource or custom) for checksum + diff.
+      # Fields (with kind, data_type, etc.)
+      # We unify:
+      #   - kind: :resource vs :custom
+      #   - data_type: normalized symbol (e.g., :numeric, :string)
+      #   - default_visibility, is_required: booleans
+      #   - default_position: numeric (used for ordering but not breaking if changed)
+      def normalize_field_hash(f)
+        kind = (f[:kind] || f["kind"]).to_sym
+
+        base = {
+          kind:               kind,
+          data_type:          normalize_data_type_name(f[:data_type] || f["data_type"]),
+          default_visibility: !!(f[:default_visibility] || f["default_visibility"]),
+          default_position:   f[:default_position] || f["default_position"],
+          is_required:        !!(f[:is_required] || f["is_required"])
+        }
+
+        case kind
+        when :resource
+          base.merge(
+            ref: (f[:ref] || f["ref"]).to_s,
+            name: (f.key?(:name) ? f[:name] : f["name"])
+          )
+        when :custom
+          base.merge(
+            key:  (f[:key] || f["key"]).to_s,
+            name: f[:name] || f["name"]
+            # note: we intentionally exclude mapper from checksum, it's considered implementation detail, not structure.
+          )
+        when :virtual
+          base.merge(
+            ref:  (f[:ref] || f["ref"]).to_s,
+            name: f[:name] || f["name"],
+            custom_label: !!(f[:custom_label] || f["custom_label"]),
+            custom_label_key: (f[:custom_label_key] || f["custom_label_key"])&.to_s
+          )
+        else
+          base
+        end
+      end
+
+      # Normalizes data_type for checksum purposes.
+      # Example:
+      #   HasHelpers::Cards::CardDataType::Numeric → :numeric
+      #   "numeric" → :numeric
+      #   :numeric → :numeric
+      def normalize_data_type_name(value)
+        return nil if value.nil?
+
+        if value.respond_to?(:name) && !value.is_a?(String)
+          value.name.to_s.underscore.to_sym
+        elsif value.is_a?(Symbol)
+          value
+        else
+          value.to_s.underscore.to_sym
+        end
+      end
+
+      def normalize_data_source(ds)
+        return nil if ds.nil?
+
+        { name: ds[:name].to_s }
+      end
+
+      # Identity of a field, used for:
+      #   - indexing
+      #   - diff / drift detection
+      # For resource fields: [:resource, "age"]
+      # For custom fields: [:custom, "DEMOGRAPHICS_FULL_NAME"]
+      def field_identity(field_hash)
+        case field_hash[:kind]
+        when :custom
+          [:custom, field_hash[:key].to_s]
+        when :virtual
+          [:virtual, field_hash[:ref].to_s]
+        else
+          [:resource, field_hash[:ref].to_s]
+        end
+      end
+
+      # "Compute a checksum"
+      # Uses SHA256 over the JSON-encoded normalized definition.
+      # This is the value we store in definition_checksum.
+      def checksum_for(normalized_definition)
+        Digest::SHA256.hexdigest(JSON.generate(normalized_definition))
+      end
+
+      # 3) Import one card (audit + diff + upsert)
+      # Imports (or audits) a single card.
+      def import_card!(klass, raw_definition)
+        # Normalize DSL definition (this will be stored as definition_json).
+        normalized = normalize_definition(raw_definition)
+        checksum   = checksum_for(normalized)
+
+        identifier_name  = normalized[:identifier]
+        identifier_opt   = identifier_option_for(identifier_name)
+        resource_hash    = Array(normalized[:resources]).first
+        application_name = self.application_name
+        resource_opt     = resource_hash && find_resource_class(resource_hash[:class_name], application_name)
+
+        # Concurrency: lock card rows by identifier/resource before writing to avoid race conditions
+        # We lock by identifier_id using SELECT ... FOR UPDATE.
+        db_card =
+          identifier_opt &&
+          resource_opt &&
+          ::HasHelpers::Cards::CardDefinition.
+            joins(:card_definition_resources).
+            lock("FOR UPDATE").
+            where(identifier_id: identifier_opt.id, card_definition_resources: { resource_id: resource_opt.id }).
+            first
+        # If we already have a definition_checksum in DB and it matches the computed checksum, we consider there is no drift for this card.
+        if db_card &&
+           db_card.respond_to?(:definition_checksum) &&
+           db_card.definition_checksum.present? &&
+           db_card.definition_checksum == checksum
+          diffs = Diff.new(
+            card_changes:      :same,
+            fields_added:      [],
+            fields_removed:    [],
+            fields_changed:    [],
+            resources_added:   [],
+            resources_removed: []
+          )
+        else
+          # Build "DB view" (normalized structure) from existing DB records, if any.
+          db_view = db_card && build_db_view_for(db_card)
+          # Compute structural diff using normalized DSL view vs DB view.
+          diffs = compute_diff(normalized, db_view)
+        end
+
+        # Audit / drift classification.
+        # Fail safely when structural changes occur.
+        classify_and_raise_if_breaking!(klass, normalized, diffs) if audit
+
+        # If we are in dry_run mode:
+        #   - we only log diffs
+        #   - we do not write anything to DB
+        if mode == :dry_run
+          log_diffs(normalized[:identifier], diffs)
+          result.skipped_cards += 1
+          return
+        end
+
+        # Transactions: one transaction per card to ensure consistency.
+        ::ActiveRecord::Base.transaction do
+          # Upsert card_definitions (by identifier) + definition_json + definition_checksum
+          card_def = upsert_card_definition!(normalized, checksum, db_card)
+          # Upsert card_definition_resources (links between card_definition and resource)
+          upsert_card_resources!(card_def, normalized[:resources], application_name)
+          # Upsert card_fields (one per resource/custom fiexld)
+          upsert_card_fields!(card_def, normalized[:fields])
+        end
+      end
+
+      # Build a "normalized DB view" for an existing card
+      # Builds a normalized hash representing how this card currently looks in the DB (card_definitions + card_fields + card_definition_resources).
+      # We reuse normalize_definition to align it with the DSL view so that:
+      #   - compute_diff(code_view, db_view) becomes simple.
+      def build_db_view_for(card_def)
+        db_app = card_def.card_definition_resources.includes(resource: :application).first&.resource&.application&.name
+        {
+          application:        db_app,
+          name:               card_def.name,
+          identifier:         card_def.identifier&.name,
+          relationship:       nil,
+          layout_type:        card_def.layout_type&.name,
+          variant:            card_def.variant&.name,
+          description:        card_def.description,
+          is_customizable:    card_def.is_customizable,
+          virtual:            card_def.respond_to?(:is_virtual) ? card_def.is_virtual : card_def.try(:virtual),
+          default_position:   card_def.position,
+          resources:          build_db_resources(card_def),
+          fields:             build_db_fields(card_def)
+        }.then { |h| normalize_definition(h) }
+      end
+
+      # Returns normalized array of resources for the DB view.
+      # For each CardDefinitionResource we extract:
+      #   - resource.name  → class_name
+      #   - is_mandatory   → boolean
+      def build_db_resources(card_def)
+        card_def.card_definition_resources.includes(:resource).map do |link|
+          {
+            class_name:   link.resource.name, # HasHelpers::Resource.name
+            is_mandatory: !!link.is_mandatory
+          }
+        end.sort_by { |r| r[:class_name] }
+      end
+
+      # Returns normalized array of fields for the DB view.
+      # We map them back to the same shape used in normalize_definition.
+      def build_db_fields(card_def)
+        card_def.card_fields.
+                includes(:field_resource, :data_type).
+                map do |field|
+          if field.virtual_ref.present?
+            {
+              kind:               :virtual,
+              ref:                field.virtual_ref.to_s,
+              name:               field.field_custom_name,
+              data_type:          normalize_data_type_name(field.data_type),
+              default_visibility: !!field.default_visibility,
+              default_position:   field.default_position,
+              is_required:        !!field.is_required,
+              custom_label:       !!field.try(:custom_label),
+              custom_label_key:   field.try(:custom_label_key).presence
+            }
+          elsif field.field_resource.present?
+            {
+              kind:               :resource,
+              ref:                field.field_resource&.name.to_s,
+              name:               field.field_custom_name,
+              data_type:          normalize_data_type_name(field.data_type),
+              default_visibility: !!field.default_visibility,
+              default_position:   field.default_position,
+              is_required:        !!field.is_required
+            }
+          else
+            {
+              kind:               :custom,
+              key:                field.field_custom_name.to_s,
+              name:               field.field_custom_name,
+              data_type:          normalize_data_type_name(field.data_type),
+              default_visibility: !!field.default_visibility,
+              default_position:   field.default_position,
+              is_required:        !!field.is_required
+            }
+          end
+        end.sort_by { |f| [f[:default_position] || 0, field_identity(f)] }
+      end
+
+      # Diff logic (code vs DB)
+      # Computes a Diff object given the DSL (code_view) and DB (db_view).
+      # "How to detect drift" + "Auditing".
+      # Cases:
+      #   - db_view is nil → this is a brand new card (additive).
+      #   - otherwise → we compute card attribute changes + fields/resources changes.
+      def compute_diff(code_view, db_view)
+        if db_view.nil?
+          return Diff.new(
+            card_changes:      :new,
+            fields_added:      code_view[:fields],
+            fields_removed:    [],
+            fields_changed:    [],
+            resources_added:   code_view[:resources],
+            resources_removed: []
+          )
+        end
+
+        Diff.new(
+          card_changes:      card_changes(code_view, db_view),
+          fields_added:      diff_fields_added(code_view, db_view),
+          fields_removed:    diff_fields_removed(code_view, db_view),
+          fields_changed:    diff_fields_changed(code_view, db_view),
+          resources_added:   diff_resources_added(code_view, db_view),
+          resources_removed: diff_resources_removed(code_view, db_view)
+        )
+      end
+
+      # Compares high-level card attributes (excluding fields/resources).
+      # Card attribute changes (e.g., layout_type, name, is_customizable) → breaking
+      # We mark card_attributes as :changed or :same.
+      def card_changes(code, db)
+        important_keys = %i[
+          name
+          layout_type
+          variant
+          virtual
+          is_customizable
+          description
+        ]
+
+        changed = important_keys.any? { |key| code[key] != db[key] }
+        changed ? :changed : :same
+      end
+
+      # Helper: builds an index of fields by their identity (kind + ref/key).
+      def index_fields_by_identity(fields)
+        fields.index_by { |f| field_identity(f) }
+      end
+
+      # Code fields that are missing in DB → additive.
+      def diff_fields_added(code, db)
+        code_index = index_fields_by_identity(code[:fields])
+        db_index   = index_fields_by_identity(db[:fields])
+
+        (code_index.keys - db_index.keys).map { |k| code_index[k] }
+      end
+
+      # DB fields that are missing in code → breaking (removal).
+      def diff_fields_removed(code, db)
+        code_index = index_fields_by_identity(code[:fields])
+        db_index   = index_fields_by_identity(db[:fields])
+
+        (db_index.keys - code_index.keys).map { |k| db_index[k] }
+      end
+
+      # Fields present in both code and DB but with changed structural attributes.
+      # Example: Changed field attributes (e.g., data_type, required...) → breaking.
+      # Reordering is non-breaking, so position differences are ignored here.
+      def diff_fields_changed(code, db)
+        code_index = index_fields_by_identity(code[:fields])
+        db_index   = index_fields_by_identity(db[:fields])
+        common     = code_index.keys & db_index.keys
+
+        common.map do |key|
+          c = code_index[key]
+          d = db_index[key]
+
+          # We consider changes in these attributes as potentially breaking.
+          attrs = %i[
+            data_type
+            is_required
+            kind
+          ]
+
+          if attrs.any? { |a| c[a] != d[a] }
+            { identity: key, code: c, db: d }
+          end
+        end.compact
+      end
+
+      # Resources present in code but missing in DB → additive.
+      def diff_resources_added(code, db)
+        code_set = Set.new(code[:resources].map { |r| r[:class_name].to_s })
+        db_set   = Set.new(db[:resources].map   { |r| r[:class_name].to_s })
+
+        (code_set - db_set).map do |cn|
+          code[:resources].find { |r| r[:class_name].to_s == cn }
+        end
+      end
+
+      # Resources present in DB but missing in code → breaking (relationship removal).
+      # Exmaple: Removed relationship → breaking
+      def diff_resources_removed(code, db)
+        code_set = Set.new(code[:resources].map { |r| r[:class_name].to_s })
+        db_set   = Set.new(db[:resources].map   { |r| r[:class_name].to_s })
+
+        (db_set - code_set).map do |cn|
+          db[:resources].find { |r| r[:class_name].to_s == cn }
+        end
+      end
+
+      # Audit classification: raise if breaking drift.
+      # Given the diff, determine if there is any breaking drift.
+      # If yes, raise HasHelpers::Cards::Importer::AuditError with detailed messages,in line with the examples (Removed field, Changed data_type, Removed relationship).
+      def classify_and_raise_if_breaking!(_klass, code_view, diff)
+        card_id = code_view[:identifier]
+        name    = code_view[:name]
+
+        msgs = []
+
+        # Card-level attribute changes
+        if diff.card_changes == :changed
+          msgs << "Card '#{name}' (identifier=#{card_id}) attributes changed (name/layout/variant/is_customizable/position/description). "\
+                  "This is a breaking change. Create a data migration to update/remove it and then try again."
+        end
+
+        # Fields removed from code but still in DB
+        diff.fields_removed.each do |field|
+          label = field_identity(field).join(":")
+          msgs << "Card '#{name}' (identifier=#{card_id}) drift: field '#{label}' is persisted but not in code. "\
+                  "This is a breaking change. Create a data migration to remove it and then try again."
+        end
+
+        # Fields with changed attributes (e.g. data_type, required...)
+        diff.fields_changed.each do |change|
+          label = change[:identity].join(":")
+          msgs << "Card '#{name}' (identifier=#{card_id}) field '#{label}' changed from #{change[:db].inspect} to #{change[:code].inspect}. "\
+                  "This is a breaking change. Create a data migration and then try again."
+        end
+
+        # Relationships removed (card_definition_resources removed in code)
+        diff.resources_removed.each do |res|
+          msgs << "Card '#{name}' (identifier=#{card_id}) drift: resource '#{res[:class_name]}' is persisted but not present in code. "\
+                  "This is a breaking change. Create a data migration to remove it and then try again."
+        end
+
+        raise AuditError, msgs.join("\n") if msgs.any?
+      end
+
+      # Logs diffs for dry_run mode.
+      # Example: mode: :dry_run → compute and print diffs, no DB writes.
+      def log_diffs(identifier, diff)
+        Rails.logger.info(
+          event:                 "cards_importer.dry_run",
+          identifier:            identifier,
+          card_changes:          diff.card_changes,
+          fields_added_count:    diff.fields_added.size,
+          fields_removed_count:  diff.fields_removed.size,
+          fields_changed_count:  diff.fields_changed.size,
+          resources_added_count: diff.resources_added.size,
+          resources_removed_count: diff.resources_removed.size
+        )
+      end
+
+      # Global audit: cards present only in DB
+      # When there is no resource filter, we also check for cards that:
+      #   - exist in card_definitions (DB)
+      #   - but do not appear in any DSL definition.
+      #
+      def audit_removed_cards!(card_pairs)
+        # Track definitions by [identifier, resource_name] so we detect per-resource removals
+        defs_in_code =
+          card_pairs.map do |_klass, defn|
+            norm = normalize_definition(defn.deep_symbolize_keys)
+            res  = Array(norm[:resources]).first
+            [norm[:identifier], res&.[](:class_name)]
+          end.compact.to_set
+
+        app_id = application.id
+
+        ::HasHelpers::Cards::CardDefinition.
+          joins(card_definition_resources: { resource: :application }).
+          where(applications: { id: app_id }).
+          includes(:identifier, card_definition_resources: :resource).
+          find_each do |card|
+            identifier_name = card.identifier&.name
+            resource_name   = card.card_definition_resources.first&.resource&.name
+            next if identifier_name.blank?
+            next if defs_in_code.include?([identifier_name, resource_name])
+
+            raise AuditError,
+              "Card '#{card.name}' (identifier=#{identifier_name}, resource=#{resource_name}) exists in DB but not in the DSL. "\
+              "This is a breaking removal. Create a data migration to remove it and then try again."
+          end
+      end
+
+      # 4) Persist (Upsert)
+      # Upserts into card_definitions using identifier as the logical key.
+      # Example attributes:
+      #   - "Upsert records deterministically (idempotent)"
+      #   - "Persist a definition_json and definition_checksum for each record"
+      # If db_card is nil → we create a new CardDefinition.
+      # Otherwise → we update the existing one.
+      def upsert_card_definition!(code, checksum, db_card)
+        card =
+          if db_card
+            result.updated_cards += 1
+            db_card
+          else
+            result.created_cards += 1
+            ::HasHelpers::Cards::CardDefinition.new
+          end
+
+        card.name           = code[:name]
+        card.identifier     = identifier_option_for(code[:identifier])
+        card.layout_type    = layout_type_option_for(code[:layout_type])
+        card.variant        = variant_option_for(code[:variant])
+        card.description    = code[:description]
+        card.is_customizable = code[:is_customizable]
+        card.default_visibility = code[:default_visibility] if card.respond_to?(:default_visibility=)
+        card.position = code[:default_position]
+        card.relationship = code[:relationship]
+        card.relationship_klass = code[:relationship_klass]
+        if card.has_attribute?(:is_virtual)
+          card.is_virtual = code[:virtual] ? true : nil
+        elsif card.respond_to?(:virtual=)
+          card.virtual = code[:virtual] ? true : nil
+        end
+
+        # These 2 attributes must exist in card_definitions table:
+        #   - definition_json     (jsonb or text)
+        #   - definition_checksum (string)
+        # They are used for auditing future runs.
+        if card.has_attribute?(:definition_json)
+          card.definition_json = code
+        end
+
+        if card.has_attribute?(:definition_checksum)
+          card.definition_checksum = checksum
+        end
+
+        card.save!
+        card
+      end
+
+      # Upserts card_definition_resources for a given card.
+      # Matching key: (card_definition_id, resource_id).
+      # Note: we do not delete DB relationships that are not present in code, because that would be a breaking removal and is handled at audit time.
+      def upsert_card_resources!(card_def, resources_from_code, application_name = nil)
+        # indexing the existing links by resource name ("Advisor", "Client", etc.)
+        existing_links_by_name =
+          card_def.card_definition_resources.includes(:resource).index_by { |link| link.resource.name }
+
+        resources_from_code.each_with_index do |res, index|
+          class_name = res[:class_name].to_s
+
+          # 1) if a link already exists for that resource name, we reuse it
+          link = existing_links_by_name[class_name]
+          resource = link&.resource
+
+          # 2) if not existing, we lookup the resource by name (may fail if DB is dirty)
+          resource ||= find_resource_class(class_name, application_name)
+
+          # 3) we ensure there is a link for (card_definition, resource)
+          link ||= ::HasHelpers::Cards::CardDefinitionResource.find_or_initialize_by(
+            card_definition_id: card_def.id,
+            resource_id:        resource.id
+          )
+
+          if link.new_record?
+            result.created_resources += 1
+          else
+            result.updated_resources += 1
+          end
+
+          link.is_mandatory = res[:is_mandatory]
+          # If this is the first time we see the link, we set a default position.
+          link.default_position ||= (index + 1) * 10
+          link.save!
+        end
+      end
+
+      # Upserts card_fields for a given card.
+      # Matching keys:
+      #   - (card_definition_id, field_id) for resource fields
+      #   - (card_definition_id, field_custom_key) for custom fields
+      # We use find_or_initialize_field to encapsulate that logic.
+      def upsert_card_fields!(card_def, fields_from_code)
+        # All resources (HasHelpers::Resource) associated with this card
+        parent_ids = card_def.card_definition_resources.pluck(:resource_id)
+
+        fields_from_code.each do |f|
+          field = find_or_initialize_field(card_def, f, parent_ids)
+
+          if field.new_record?
+            result.created_fields += 1
+          else
+            result.updated_fields += 1
+          end
+
+          apply_field_attributes!(field, f, parent_ids, card_def)
+          field.save!
+        end
+        # As with resources, we do not delete existing DB fields
+        # that are missing from code; that drift is handled by the audit.
+      end
+
+      # Finds or initializes a CardField given the card_def and field hash.
+      # Custom fields:
+      #   - identity: (card_definition_id, field_custom_name)
+      # Resource fields:
+      #   - identity: (card_definition_id, field_resource)
+      def find_or_initialize_field(card_def, field_hash, parent_ids)
+        identity = field_identity(field_hash)
+
+        case identity.first
+        when :custom
+          ::HasHelpers::Cards::CardField.find_or_initialize_by(
+            card_definition_id: card_def.id,
+            field_custom_name:  field_hash[:name]
+          )
+        when :virtual
+          scope = ::HasHelpers::Cards::CardField.where(card_definition_id: card_def.id, virtual_ref: identity.last)
+          scope.first || ::HasHelpers::Cards::CardField.new(card_definition_id: card_def.id, virtual_ref: identity.last)
+        else
+          resource = find_field_resource(identity.last, parent_ids, card_def.relationship_klass)
+          ::HasHelpers::Cards::CardField.find_or_initialize_by(
+            card_definition_id: card_def.id,
+            field_resource:     resource
+          )
+        end
+      end
+
+      # Applies attributes from the normalized field hash onto a CardField model.
+      #   - Sets field_custom_name for custom fields.
+      #   - Sets field_resource for resource fields.
+      #   - Sets data_type, default_position, default_visibility, is_required.
+      def apply_field_attributes!(field, f, parent_ids, card_def)
+        case f[:kind]
+        when :custom
+          field.field_custom_name = f[:name]
+          field.field_id          = nil
+          field.virtual_ref       = nil if field.respond_to?(:virtual_ref=)
+        when :virtual
+          field.field_custom_name = f[:name]
+          field.field_id          = nil
+          field.virtual_ref       = f[:ref] if field.respond_to?(:virtual_ref=)
+          field.field_resource    = nil if field.respond_to?(:field_resource=)
+          field.custom_label      = !!f[:custom_label]      if field.respond_to?(:custom_label=)
+          field.custom_label_key  = f[:custom_label_key].presence if field.respond_to?(:custom_label_key=)
+        else
+          resource                = find_field_resource(f[:ref], parent_ids, card_def.relationship_klass)
+          field.field_resource    = resource
+
+          # Persist static label override (DSL name:)
+          name_override = (f[:name] || f["name"]).presence
+          field.field_custom_name = name_override if name_override
+
+          field.virtual_ref = nil if field.respond_to?(:virtual_ref=)
+
+          field.custom_label = false if field.respond_to?(:custom_label=)
+          field.custom_label_key = nil if field.respond_to?(:custom_label_key=)
+        end
+
+        field.data_type          = lookup_card_data_type(f[:data_type])
+        field.default_position   = f[:default_position]
+        field.default_visibility = f[:default_visibility]
+        field.is_required        = f[:is_required]
+      end
+
+      # Lookups (Options + Resources)
+      # Finds the HasHelpers::Cards::CardIdentifier option by its name.
+      def identifier_option_for(identifier_name)
+        return nil if identifier_name.blank?
+
+        if identifier_name.is_a?(::HasHelpers::Cards::CardIdentifier)
+          identifier_name
+        else
+          ::HasHelpers::Cards::CardIdentifier.find_by!(name: identifier_name.to_s)
+        end
+      end
+
+      # Finds the HasHelpers::Cards::CardLayoutType option by its name.
+      def layout_type_option_for(layout_name)
+        return nil if layout_name.blank?
+
+        if layout_name.is_a?(::HasHelpers::Cards::CardLayoutType)
+          layout_name
+        else
+          ::HasHelpers::Cards::CardLayoutType.find_by!(name: layout_name.to_s)
+        end
+      end
+
+      # Finds the HasHelpers::Cards::CardVariantType option by its name.
+      def variant_option_for(variant_name)
+        return nil if variant_name.blank?
+
+        if variant_name.is_a?(::HasHelpers::Cards::CardVariantType)
+          variant_name
+        else
+          ::HasHelpers::Cards::CardVariantType.find_by!(name: variant_name.to_s)
+        end
+      end
+
+      # Resolves data_type (symbol or option) to HasHelpers::Cards::CardDataType.
+      # Example:
+      #   :numeric → HasHelpers::Cards::CardDataType.find_by!(name: "Numeric")
+      def lookup_card_data_type(data_type_ident)
+        return nil if data_type_ident.nil?
+
+        if data_type_ident.is_a?(::HasHelpers::Cards::CardDataType)
+          data_type_ident
+        else
+          ::HasHelpers::Cards::CardDataType.find_by!(name: data_type_ident.to_s.camelize)
+        end
+      end
+
+      # Search for the correct field (HasHelpers::Resource) restricting by:
+      # - name (field_name)
+      # - parent_id in the card's resources (when present)
+      # - relationship_klass is the relationship class for the relationship from parent resource this card is using
+      # - application sanity between this row and its field_resource (if any)
+      #
+      # If after filtering there is more than one match, we treat it as a real
+      # data duplication problem and raise AuditError (so devs clean it up).
+      def find_field_resource(field_name, parent_ids, relationship_klass = nil)
+        base_resource = ::HasHelpers::Resource.find(parent_ids.first)
+        application = base_resource.application
+        parent_resource = (relationship_klass && ::HasHelpers::Resource.find_by(name: relationship_klass, parent_id: nil, application: application)) || base_resource
+
+        parent_klass =
+          parent_resource.name.safe_constantize ||
+          ::HasHelpers::ApplicationHelper.translate_klass(parent_resource.name)
+        association_info = inspect_field_and_association(parent_klass, field_name)
+
+        raise AuditError, "The field #{field_name} is not a valid column from the class #{parent_resource.name}" if association_info[0].nil?
+
+        field_class_name = association_info[1]
+
+        scope = ::HasHelpers::Resource.where(name: field_name.to_s, parent: parent_resource, field_class_name: field_class_name)
+        # We eager load field_resource to avoid N+1 queries when we filter in Ruby.
+        matches =
+          scope.
+            includes(:field_resource). # assumes belongs_to :field_resource, class_name: "HasHelpers::Resource", optional: true
+            order(:created_at, :id).
+            to_a
+
+        # Keep only rows where:
+        #   - there is no field_resource (nil), or
+        #   - the field_resource belongs to the same application as the resource itself.
+        filtered = matches.select do |resource|
+          fr = resource.field_resource
+          fr.nil? || fr.application_id == resource.application_id
+        end
+
+        case filtered.size
+        when 1
+          # Ideal case: a single candidate after sanity filtering
+          filtered.first
+        when 0
+          # Nothing found with these constraints: real error in DSL/DB alignment
+          raise ::ActiveRecord::RecordNotFound,
+                "No field found with name=#{field_name} for parent_ids=#{parent_ids.inspect}"
+        else
+          # With all filters applied we still have multiple rows → duplicated data
+          ids = filtered.map(&:id).join(",")
+          raise AuditError,
+                "Ambiguous field '#{field_name}' for relationship_klass=#{relationship_klass} parent_ids=#{parent_ids.inspect}. " \
+                "Found #{filtered.size} candidates (ids=#{ids}) even after filtering by application. " \
+                "Please clean up duplicated HasHelpers::Resource rows so (name + parent_id + application_id) is unique."
+        end
+      end
+
+      # Resolves a resource class name to HasHelpers::Resource.
+      # We assume there is a HasHelpers::Resource record with this name.
+      def find_resource_class(class_name, application_name = nil)
+        scope = ::HasHelpers::Resource.where(name: class_name.to_s, parent_id: nil)
+
+        if application_name.present?
+          app = ::HasHelpers::Application.where("LOWER(name) = ?", application_name.to_s.downcase).first!
+          scope = scope.where(application_id: app.id)
+        end
+
+        scope.first!
+      end
+
+      # Returns [field, association_class_name]
+      # - If `field` maps to a belongs_to association (e.g. created_by_id -> :created_by),
+      #   returns the association's class name.
+      # - If it's a normal column, returns [field, nil]
+      # - If it's neither a column nor a belongs_to foreign key, returns [nil, nil]
+      #
+      # Examples:
+      #   inspect_field_and_association(Policy, "created_by_id") #=> ["created_by_id", "HasHelpers::User"]
+      #   inspect_field_and_association(Policy, "carrier_id")    #=> ["carrier_id", "Carrier"]
+      #   inspect_field_and_association(Policy, "effective_date")#=> ["effective_date", nil]
+      #   inspect_field_and_association(Policy, "nope")          #=> [nil, nil]
+      def inspect_field_and_association(klass, field)
+        return [nil, nil] unless klass.is_a?(Class) && field.present?
+
+        field = field.to_s
+
+        # 1) Must be a real column, otherwise it's invalid.
+        return [nil, nil] unless klass.column_names.include?(field)
+
+        # 2) If it's the foreign_key of a belongs_to association, return that association's klass.
+        refl =
+          klass.reflect_on_all_associations(:belongs_to).find do |r|
+            r.foreign_key.to_s == field
+          end
+
+        assoc_class_name =
+          if refl
+            # Prefer ActiveRecord's resolved class if possible, fallback to metadata.
+            begin
+              refl.klass.name
+            rescue NameError
+              refl.class_name
+            end
+          end
+
+        [field, assoc_class_name]
+      end
+
+      # Summary logging
+      # Logs a concise summary of the import operation.
+      # "Logging: use structured logs for audit diff outputs."
+      def log_summary
+        Rails.logger.info(
+          event:             "cards_importer.summary",
+          created_cards:     result.created_cards,
+          updated_cards:     result.updated_cards,
+          created_fields:    result.created_fields,
+          updated_fields:    result.updated_fields,
+          created_resources: result.created_resources,
+          updated_resources: result.updated_resources,
+          skipped_cards:     result.skipped_cards,
+          mode:              mode,
+          resource_filter:   resource_filter
+        )
+      end
+    end
+  end
+end