structured_data_to_sql 0.1.0

data/lib/structured_data_to_sql/json/profiles/khoros_api_export.rb

@@ -0,0 +1,1960 @@
+# frozen_string_literal: true
+
+require "digest"
+
+require_relative "../../version"
+require_relative "../../diagnostic"
+require_relative "../../format"
+
+module StructuredDataToSql
+  module Json
+    module Profiles
+      # Appends converter-ready Khoros API helper tables without changing the
+      # append-only raw tables. An occurrence is identified by its envelope id
+      # and every flattened context field; the greatest raw _sid wins as a
+      # whole (the exporter contract: append-only, last occurrence wins).
+      class KhorosApiExport
+        NAME = :khoros_api_export
+        # Bumped in lockstep with PROFILE_VERSION in the Khoros converter's
+        # KhorosApiExportContentStaging whenever a canonical table's shape or
+        # semantics change. Version 2 adds normalized root-message read-only state.
+        VERSION = 2
+        PREFIX = "discourse_khoros_api"
+        METADATA_TABLE = "_structured_data_to_sql_profiles"
+        # Explicit storage options on every canonical table so they never
+        # inherit database defaults: utf8mb4 VARCHAR(255) keys need DYNAMIC
+        # row format (a 1020-byte index fails on COMPACT/REDUNDANT servers).
+        TABLE_OPTIONS =
+          "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci ROW_FORMAT=DYNAMIC"
+        REQUIRED_SCHEMA = {
+          "users" => %w[_sid id data_id data_login],
+          "messages" => %w[
+            _sid
+            id
+            data_id
+            data_author_id
+            data_board_id
+            data_body
+            data_conversation_id
+            data_depth
+            data_post_time
+            data_read_only
+            data_subject
+          ],
+          "nodes" => %w[_sid id data_depth data_id data_node_type data_title]
+        }.freeze
+        TABLES =
+          %w[
+            profiles
+            avatars
+            users
+            nodes
+            ranks
+            roles
+            memberships
+            badge_definitions
+            badge_grants
+            engagement
+            floated_messages
+            tags
+            images
+            attachments
+            files
+            source_messages
+          ].map { |name| "#{PREFIX}_#{name}" }.freeze
+        USER_STRING_FIELDS = {
+          id: %w[data_id],
+          login: %w[data_login],
+          email: %w[data_email data_email_address],
+          sso_id: %w[data_sso_id data_ssoid data_sso_uid],
+          rank_id: %w[data_rank_id],
+          first_name: %w[data_first_name],
+          last_name: %w[data_last_name],
+          biography: %w[data_biography],
+          location: %w[data_location],
+          web_page_url: %w[data_web_page_url data_url],
+          language: %w[data_language],
+          title: %w[data_title],
+          signature: %w[data_signature],
+          inline_avatar_url: %w[
+            data_avatar_profile
+            data_avatar_inline
+            data_avatar_message
+          ],
+          registration_status: %w[data_registration_data_status],
+          timezone: %w[data_config_timezone data_timezone],
+          profile_privacy: %w[
+            data_personal_info_privacy
+            data_profile_privacy
+            data_profile_properties_personal_info_privacy
+          ],
+          online_status_privacy: %w[
+            data_online_status_privacy
+            data_profile_properties_online_status_privacy
+          ]
+        }.freeze
+        USER_SCALAR_FIELDS = {
+          registration_time: %w[data_registration_data_registration_time],
+          last_visit_time: %w[data_last_visit_time],
+          banned: %w[data_banned],
+          approved: %w[data_email_verified],
+          deleted: %w[data_deleted]
+        }.freeze
+        USER_CANONICAL_COLUMNS =
+          (USER_STRING_FIELDS.values + USER_SCALAR_FIELDS.values)
+            .flatten
+            .to_set
+            .freeze
+        USER_DELEGATED_COLUMN_PATTERNS = [
+          /\Adata_user_badges(?:_|\z)/,
+          /\Adata_registration_data_sso_registration_fields(?:_|\z)/
+        ].freeze
+        # Envelope, transport, link, and type columns plus v2 presentation
+        # fields that carry no member attribute. Retained raw, never warned.
+        USER_INTENTIONAL_RAW_ONLY_COLUMNS = %w[
+          _sid
+          id
+          api
+          exported_at
+          export_reason
+          url
+          data_href
+          data_type
+          data_view_href
+          data_avatar_favicon
+          data_avatar_print
+          data_avatar_type
+          data_date_pattern
+          data_friendly_date_enabled
+          data_friendly_date_max_age
+          data_mailbox_type
+          data_metrics_type
+          data_rank_type
+          data_registration_data_confirm_email_status
+          data_registration_data_type
+        ].to_set.freeze
+        # Lazy v2 collection stubs ({"query": "SELECT ..."}) and the rank
+        # definition denormalized onto the user row (canonical via ranks).
+        USER_INTENTIONAL_RAW_ONLY_COLUMN_PATTERNS = [
+          /\Adata_[a-z0-9_]+_query\z/,
+          /\Adata_rank_(?!id\z)/
+        ].freeze
+        # Real per-member values the converter does not consume yet; warned
+        # once per run so a migration decision stays visible.
+        USER_REVIEW_ONLY_COLUMNS = %w[
+          data_email_address_privacy
+          data_online_status
+          data_registration_data_registration_access_level
+          data_remember_password
+          data_show_user_signatures
+        ].to_set.freeze
+        USER_REVIEW_ONLY_COLUMN_PATTERNS = [
+          /\Adata_metrics_/,
+          /\Adata_notes(?:_|\z)/,
+          /\Adata_c_/
+        ].freeze
+
+        # Validates the converter-facing key contract while records are already
+        # streaming through the JSON emitter, so an atomic conversion fails
+        # before publishing an unusable dump. Each raw identity (envelope id
+        # plus every flattened context field) keeps only its greatest-_sid
+        # outcome: an identity digest, the winning canonical id, and any
+        # violation message (users add five richness flags). Memory therefore
+        # scales with distinct identities at a few dozen bytes each — for
+        # identity-dense collections such as messages that is per raw row, so
+        # a multi-million-row export costs hundreds of megabytes, not
+        # gigabytes.
+        class InputValidator
+          MAX_KEY_BYTES = 255
+          USER_EMAIL_FIELDS = USER_STRING_FIELDS.fetch(:email)
+          USER_BANNED_FIELD = USER_SCALAR_FIELDS.fetch(:banned).first
+          # Exporter quality signals (contract §9.7): rows that explain what
+          # is knowingly missing. Tallied by the first present label field and
+          # surfaced as warnings so an empty helper reads as "withheld", not
+          # "none".
+          QUALITY_SIGNAL_TABLES = {
+            "errors" => %w[collection phase],
+            "gaps" => %w[collection phase]
+          }.freeze
+          DIRECT = {
+            "avatars" => {
+              key: %w[id],
+              indexed: {
+                user_id: %w[id]
+              }
+            },
+            "users" => {
+              key: %w[data_id],
+              indexed: {
+                id: %w[data_id],
+                email: %w[data_email data_email_address],
+                rank_id: %w[data_rank_id]
+              }
+            },
+            "nodes" => {
+              key: %w[data_id],
+              indexed: {
+                id: %w[data_id],
+                parent_id: %w[data_parent_id]
+              }
+            },
+            "ranks" => {
+              key: %w[data_id],
+              indexed: {
+                id: %w[data_id]
+              }
+            },
+            "roles" => {
+              key: %w[data_id],
+              indexed: {
+                id: %w[data_id],
+                node_id: %w[data_node_id]
+              }
+            },
+            "messages" => {
+              key: %w[data_id],
+              indexed: {
+                id: %w[data_id],
+                conversation_id: %w[data_conversation_id],
+                board_id: %w[data_board_id]
+              }
+            }
+          }.freeze
+          OPTIONAL = {
+            "user_profiles" => {
+              required: {
+                user_id: %w[id]
+              },
+              indexed: {
+                user_id: %w[id]
+              }
+            },
+            "user_profiles_data_profile" => {
+              required: {
+                profile_name: %w[name],
+                profile_value: %w[unnamed]
+              },
+              indexed: {
+                profile_name: %w[name]
+              },
+              child_of: "user_profiles",
+              normalized_profile_key: true
+            },
+            "role_users" => {
+              required: {
+                container_id: %w[context_role_id],
+                user_id: %w[id]
+              },
+              indexed: {
+                container_id: %w[context_role_id],
+                user_id: %w[id]
+              }
+            },
+            "grouphub_members" => {
+              required: {
+                container_id: %w[context_grouphub_id],
+                user_id: %w[id]
+              },
+              indexed: {
+                container_id: %w[context_grouphub_id],
+                user_id: %w[id]
+              }
+            },
+            "badges" => {
+              required: {
+                badge_id: %w[data_id_unnamed data_id]
+              },
+              indexed: {
+                badge_id: %w[data_id_unnamed data_id]
+              }
+            },
+            "user_badges" => {
+              required: {
+                user_id: %w[context_user_id],
+                badge_id: %w[
+                  data_badge_id_unnamed
+                  data_badge_id
+                  data_id_unnamed
+                  data_id
+                ]
+              },
+              indexed: {
+                user_id: %w[context_user_id],
+                badge_id: %w[
+                  data_badge_id_unnamed
+                  data_badge_id
+                  data_id_unnamed
+                  data_id
+                ]
+              }
+            },
+            "kudos" => {
+              required: {
+                message_id: %w[context_message_id],
+                engagement_id: %w[id]
+              },
+              indexed: {
+                message_id: %w[context_message_id],
+                engagement_id: %w[id],
+                user_id: %w[data_user_id]
+              }
+            },
+            "floated_messages" => {
+              required: {
+                board_id: %w[context_board_id],
+                message_id: %w[data_message_id id]
+              },
+              indexed: {
+                board_id: %w[context_board_id],
+                message_id: %w[data_message_id id]
+              }
+            },
+            "labels" => {
+              required: {
+                source_id: %w[id],
+                message_id: %w[context_message_id]
+              },
+              indexed: {
+                source_id: %w[id],
+                message_id: %w[context_message_id]
+              }
+            },
+            "tags" => {
+              required: {
+                source_id: %w[id],
+                message_id: %w[context_message_id]
+              },
+              indexed: {
+                source_id: %w[id],
+                message_id: %w[context_message_id]
+              }
+            },
+            "images" => {
+              required: {
+                message_id: %w[context_message_id],
+                image_id: %w[id]
+              },
+              indexed: {
+                message_id: %w[context_message_id],
+                image_id: %w[id]
+              }
+            },
+            "uploaded_images" => {
+              required: {
+                message_id: %w[context_message_id],
+                image_id: %w[data_id_unnamed data_id]
+              },
+              indexed: {
+                message_id: %w[context_message_id],
+                image_id: %w[data_id_unnamed data_id]
+              }
+            },
+            "attachments" => {
+              required: {
+                message_id: %w[context_message_id],
+                attachment_id: %w[data_id id]
+              },
+              indexed: {
+                message_id: %w[context_message_id],
+                attachment_id: %w[data_id id]
+              }
+            },
+            "files" => {
+              required: {
+                url: %w[url id]
+              },
+              indexed: {
+                source_collection: %w[data_source_collection],
+                source_message_id: %w[data_source_message_id]
+              }
+            },
+            "users_data_user_badges_items" => {
+              required: {
+                badge_id: %w[badge_id]
+              },
+              indexed: {
+                badge_id: %w[badge_id]
+              },
+              child_of: "users"
+            }
+          }.freeze
+
+          # +warnings+ are the one-line strings; +diagnostics+ carry the same
+          # facts structured (lists, run scoping, actions) for the terminal
+          # blocks and the diagnostics report. One diagnostic per warning.
+          attr_reader :warnings, :diagnostics
+          # Optional run scoping: { manifest: Json::ExporterManifest or nil,
+          # source_paths: { "errors" => path, "gaps" => path } }.
+          attr_accessor :run_context
+
+          MAX_LISTED_IDS = 5000
+
+          def initialize
+            @winners = Hash.new { |hash, table| hash[table] = {} }
+            @child_issues = Hash.new { |hash, table| hash[table] = [] }
+            @warnings = []
+            @diagnostics = []
+            @run_context = nil
+            @observed_user_columns = Set.new
+            @user_richness = {}
+            @quality_signals =
+              Hash.new { |hash, table| hash[table] = Hash.new(0) }
+            @quality_signal_rows = Hash.new { |hash, table| hash[table] = [] }
+            @validated = false
+          end
+
+          # +context_columns+ must be pre-sorted; the emitter computes the
+          # sorted list once per table definition.
+          def observe(table, columns, sid, parent_sid: nil, context_columns: [])
+            if QUALITY_SIGNAL_TABLES.key?(table)
+              tally_quality_signal(table, columns)
+            end
+            spec = DIRECT[table] || OPTIONAL[table]
+            return if spec.nil?
+            if spec.key?(:child_of)
+              observe_child(table, spec, columns, sid, parent_sid)
+              return
+            end
+
+            winners = @winners[table]
+            identity = identity_digest(columns, context_columns)
+            if table == "users"
+              @observed_user_columns.merge(columns.keys)
+              observe_user_richness(identity, columns, sid)
+            end
+            previous = winners[identity]
+            return if previous && previous[0] > sid
+
+            winners[identity] = if DIRECT.key?(table)
+              examine_direct(table, spec, columns, sid)
+            else
+              examine_optional(table, spec, columns, sid)
+            end
+          end
+
+          def validate!
+            return if @validated
+
+            DIRECT.each_key { |table| validate_direct(table) }
+            OPTIONAL.each { |table, spec| validate_optional(table, spec) }
+            validate_user_columns
+            warn_credential_degraded_users
+            report_quality_signals
+            @validated = true
+          end
+
+          private
+
+          # Winner tuples are [sid, canonical_id, error] for DIRECT tables and
+          # [sid, :omitted or nil, error] for OPTIONAL tables. Every direct
+          # collection is a complete observation: the last occurrence wins as
+          # a whole, including blank, false, and zero values. Violations are
+          # examined while the row is in hand but raised only in validate!, so
+          # a superseding occurrence can still repair an earlier bad row.
+          def examine_direct(table, spec, columns, sid)
+            key_value = first_value(columns, spec[:key])
+            if blank?(key_value)
+              return [
+                sid,
+                nil,
+                "Khoros API profile requires #{table}.#{spec[:key].first} for the row at #{table}._sid=#{sid}"
+              ]
+            end
+
+            [
+              sid,
+              key_value.to_s,
+              oversized_key_error(table, spec[:indexed], columns, sid)
+            ]
+          end
+
+          def examine_optional(table, spec, columns, sid)
+            missing =
+              spec[:required].any? do |_label, fields|
+                blank?(first_value(columns, fields))
+              end
+            return sid, :omitted, nil if missing
+
+            error =
+              oversized_key_error(table, spec.fetch(:indexed, {}), columns, sid)
+            [sid, nil, error]
+          end
+
+          def oversized_key_error(table, fields_by_label, columns, sid)
+            fields_by_label.each do |label, fields|
+              value = first_value(columns, fields)
+              next if blank?(value)
+              next if value.to_s.bytesize <= MAX_KEY_BYTES
+
+              return(
+                "Khoros API profile key #{table}.#{label} exceeds #{MAX_KEY_BYTES} bytes " \
+                  "for the row at #{table}._sid=#{sid}: #{value.inspect}"
+              )
+            end
+            nil
+          end
+
+          def validate_direct(table)
+            projections = {}
+            @winners[table].each_value do |sid, canonical_id, error|
+              raise UsageError, error if error
+
+              if (other_sid = projections[canonical_id])
+                raise UsageError,
+                      "Khoros API profile cannot map distinct #{table} identities " \
+                        "(rows #{table}._sid=#{other_sid} and #{table}._sid=#{sid}) " \
+                        "to the same canonical id #{canonical_id.inspect}. Re-export or repair #{table}."
+              end
+              projections[canonical_id] = sid
+            end
+          end
+
+          def validate_optional(table, spec)
+            return validate_child(table, spec) if spec[:child_of]
+
+            omitted = 0
+            @winners[table].each_value do |_sid, outcome, error|
+              raise UsageError, error if error
+
+              omitted += 1 if outcome == :omitted
+            end
+            return if omitted.zero?
+
+            warn_omitted(table, omitted)
+          end
+
+          def observe_child(table, spec, columns, sid, parent_sid)
+            identity = [parent_sid, sid]
+            missing =
+              spec[:required].find do |_label, fields|
+                blank?(first_value(columns, fields))
+              end
+            if missing
+              @child_issues[table] << { parent_sid:, kind: :missing, identity: }
+              return
+            end
+
+            spec
+              .fetch(:indexed, {})
+              .each do |label, fields|
+                value = first_value(columns, fields)
+                if spec[:normalized_profile_key] && label == :profile_name
+                  value = normalized_profile_key(value)
+                end
+                next if value.to_s.bytesize <= MAX_KEY_BYTES
+
+                @child_issues[table] << {
+                  parent_sid:,
+                  kind: :oversized,
+                  label:,
+                  value:,
+                  identity:
+                }
+              end
+          end
+
+          def validate_child(table, spec)
+            winning_parent_sids =
+              @winners[spec[:child_of]].values.map(&:first).to_set
+            issues =
+              @child_issues[table].select do |issue|
+                winning_parent_sids.include?(issue[:parent_sid])
+              end
+            if (
+                 issue =
+                   issues.find { |candidate| candidate[:kind] == :oversized }
+               )
+              raise UsageError,
+                    "Khoros API profile key #{table}.#{issue[:label]} exceeds #{MAX_KEY_BYTES} bytes " \
+                      "for identity #{issue[:identity].inspect}: #{issue[:value].inspect}"
+            end
+
+            omitted = issues.count { |candidate| candidate[:kind] == :missing }
+            return if omitted.zero?
+
+            warn_omitted(table, omitted)
+          end
+
+          def first_value(columns, fields)
+            fields.each do |field|
+              value = columns[field]
+              return value unless value.nil?
+            end
+            nil
+          end
+
+          # Credential-degradation detection. The exporter contract is
+          # whole-occurrence last-wins, so a final occurrence written by a
+          # weaker credential (blank email, no ban state) silently replaces a
+          # richer earlier one. The profile does not compensate — the exporter
+          # re-emits such users — but it must not stay quiet about it either.
+          def observe_user_richness(identity, columns, sid)
+            email_blank = blank?(first_value(columns, USER_EMAIL_FIELDS))
+            banned_missing = columns[USER_BANNED_FIELD].nil?
+            state =
+              @user_richness[identity] ||= [0, false, false, false, false, nil]
+            state[1] ||= !email_blank
+            state[2] ||= !banned_missing
+            return if sid < state[0]
+
+            state[0] = sid
+            state[3] = email_blank
+            state[4] = banned_missing
+            state[5] = first_value(columns, DIRECT.fetch("users")[:key]).to_s
+          end
+
+          def warn_credential_degraded_users
+            degraded_ids =
+              @user_richness
+                .each_value
+                .filter_map do |_sid, had_email, had_banned, email_blank, banned_missing, id|
+                if (had_email && email_blank) || (had_banned && banned_missing)
+                  id
+                end
+              end
+            return if degraded_ids.empty?
+
+            degraded = degraded_ids.length
+            add_diagnostic(
+              code: :users_credential_degraded,
+              title: "Users — credential degradation",
+              count: degraded,
+              summary:
+                "The winning (last) occurrence of #{Format.format_count(degraded)} user(s) has a blank email " \
+                  "or no banned state although an earlier occurrence carried it. Last occurrence wins as-is; " \
+                  "the canonical rows for these users are missing those values.",
+              items: degraded_ids.sort.first(MAX_LISTED_IDS),
+              items_label: "Affected user ids",
+              terminal_items: false,
+              details:
+                (
+                  if degraded > MAX_LISTED_IDS
+                    [
+                      "#{Format.format_count(degraded - MAX_LISTED_IDS)} more ids not listed; query raw users for winners with blank data_email"
+                    ]
+                  else
+                    []
+                  end
+                ),
+              action:
+                "Re-export these users with a privileged (administrator) credential; the repaired rows append to users.ndjson and last-wins picks them up on the next conversion.",
+              message:
+                "users: #{degraded} winning occurrence(s) look credential-degraded " \
+                  "(blank email or missing banned state while an earlier occurrence carried it); " \
+                  "last occurrence wins as-is — re-export those users with a privileged credential"
+            )
+          end
+
+          def tally_quality_signal(table, columns)
+            fields = QUALITY_SIGNAL_TABLES.fetch(table)
+            label = first_value(columns, fields)
+            label = "unspecified" if blank?(label)
+            label = label.to_s
+            @quality_signals[table][label] += 1
+            @quality_signal_rows[table] << label
+          end
+
+          QUALITY_SIGNAL_TITLES = {
+            "errors" => ["Exporter errors", "exporter error row(s)"],
+            "gaps" => ["Withheld content", "withheld-content row(s)"]
+          }.freeze
+
+          def report_quality_signals
+            QUALITY_SIGNAL_TABLES.each_key do |table|
+              tallies = @quality_signals[table]
+              next if tallies.empty?
+
+              total = tallies.values.sum
+              breakdown = breakdown_text(tallies)
+              title, noun = QUALITY_SIGNAL_TITLES.fetch(table)
+              scoped = run_scoped_signal(table, total)
+              add_diagnostic(
+                code: :"exporter_#{table}",
+                title: title,
+                count: total,
+                summary:
+                  (
+                    if table == "errors"
+                      "Rows in errors.ndjson: exporter requests that failed (a per-user detail fetch, a page, or a run). What they name is knowingly missing from this dump."
+                    else
+                      "Rows in gaps.ndjson: content the community counts but would not serve, or capabilities a run lacked. An empty canonical helper may be \"withheld\", not \"none\"."
+                    end
+                  ),
+                details:
+                  scoped +
+                    [
+                      "History (all runs): #{Format.format_count(total)} — #{breakdown}"
+                    ],
+                action:
+                  (
+                    if table == "errors"
+                      "Rerun the exporter with a privileged credential; retried rows append to the same files and last-wins heals them. Inspect individual rows in errors.ndjson."
+                    else
+                      "Read the gap text in gaps.ndjson; capability gaps mean a run was made as an ordinary member."
+                    end
+                  ),
+                sources: [source_path_for(table)].compact,
+                message: "#{table}: #{total} #{noun} — #{breakdown}"
+              )
+            end
+          end
+
+          # Attributes the last N rows of the append-only file to the latest
+          # completed manifest run when the manifest's per-run counts fit
+          # inside the file; otherwise says why lifetime totals are shown.
+          def run_scoped_signal(table, total)
+            manifest = @run_context&.dig(:manifest)
+            if manifest.nil?
+              return [
+                "Latest run: not scoped — no manifest.json beside the export; showing lifetime totals"
+              ]
+            end
+            if manifest.problem
+              return [
+                "Latest run: not scoped — #{manifest.problem}; showing lifetime totals"
+              ]
+            end
+            latest = manifest.latest_completed
+            if latest.nil?
+              return [
+                "Latest run: not scoped — no completed run in manifest.json; showing lifetime totals"
+              ]
+            end
+
+            counter = table == "errors" ? :errors : :gaps
+            declared = manifest.runs.sum(&counter)
+            if declared > total
+              return [
+                "Latest run: not scoped — manifest declares #{Format.format_count(declared)} row(s) but the file holds #{Format.format_count(total)}; showing lifetime totals"
+              ]
+            end
+
+            latest_count = latest.public_send(counter)
+            latest_rows = @quality_signal_rows[table].last(latest_count)
+            latest_tallies = Hash.new(0)
+            latest_rows.each { |label| latest_tallies[label] += 1 }
+            window = [latest.started_at, latest.finished_at].compact.join(" → ")
+            lines = [
+              "Latest completed run ##{latest.index}#{window.empty? ? "" : " (#{window})"}: " \
+                "#{Format.format_count(latest_count)}#{latest_count.positive? ? " — #{breakdown_text(latest_tallies)}" : ""}"
+            ]
+            unattributed = total - declared
+            if unattributed.positive?
+              lines << "Unattributed: #{Format.format_count(unattributed)} row(s) written by runs that did not finalize their manifest entry"
+            end
+            lines
+          end
+
+          def breakdown_text(tallies)
+            tallies
+              .sort_by { |label, count| [-count, label] }
+              .map { |label, count| "#{label} #{count}" }
+              .join(", ")
+          end
+
+          def source_path_for(table)
+            @run_context&.dig(:source_paths, table)&.to_s
+          end
+
+          def warn_omitted(table, omitted)
+            add_diagnostic(
+              code: :canonical_rows_omitted,
+              title: "#{table} — rows omitted from canonical helpers",
+              count: omitted,
+              summary:
+                "#{Format.format_count(omitted)} winning row(s) of #{table} lack a required key component (blank id or context) and were left out of the canonical helpers. They remain in the raw table.",
+              action:
+                "Inspect the raw #{table} rows; if the exporter should have supplied the key, re-export that collection.",
+              message:
+                "#{table}: omitted #{omitted} winning row(s) from canonical helpers because a required key component was blank"
+            )
+          end
+
+          def add_diagnostic(**attributes)
+            diagnostic = Diagnostic.new(**attributes)
+            @warnings << diagnostic.message
+            @diagnostics << diagnostic
+            diagnostic
+          end
+
+          def validate_user_columns
+            review_only = []
+            unclassified = []
+            @observed_user_columns.sort.each do |column|
+              next if KhorosApiExport::USER_CANONICAL_COLUMNS.include?(column)
+              if KhorosApiExport::USER_INTENTIONAL_RAW_ONLY_COLUMNS.include?(
+                   column
+                 )
+                next
+              end
+              next if column.start_with?("context_")
+              if match_any?(
+                   KhorosApiExport::USER_INTENTIONAL_RAW_ONLY_COLUMN_PATTERNS,
+                   column
+                 )
+                next
+              end
+              if match_any?(
+                   KhorosApiExport::USER_DELEGATED_COLUMN_PATTERNS,
+                   column
+                 )
+                next
+              end
+
+              if KhorosApiExport::USER_REVIEW_ONLY_COLUMNS.include?(column) ||
+                   match_any?(
+                     KhorosApiExport::USER_REVIEW_ONLY_COLUMN_PATTERNS,
+                     column
+                   )
+                review_only << column
+              else
+                unclassified << column
+              end
+            end
+            if review_only.any?
+              add_diagnostic(
+                code: :users_review_only_fields,
+                title: "Users — review-only fields",
+                count: review_only.length,
+                summary:
+                  "Per-member values the converter does not consume yet. Retained in the raw users table; not mapped to discourse_khoros_api_users.",
+                items: review_only,
+                items_label: "Fields",
+                action:
+                  "None unless these fields need a migration mapping; then add them to the canonical users contract (profile version bump).",
+                message:
+                  "users: review-only scalar field(s) retained only in the raw table: #{review_only.join(", ")}"
+              )
+            end
+            if unclassified.any?
+              add_diagnostic(
+                code: :users_unclassified_fields,
+                title: "Users — unclassified fields",
+                count: unclassified.length,
+                summary:
+                  "Source fields this profile has never seen. Retained in the raw users table only; nothing canonical reads them.",
+                items: unclassified,
+                items_label: "Fields",
+                action:
+                  "Classify each field in the profile (canonical alias, intentional raw-only, or review-only) so it stops appearing here.",
+                message:
+                  "users: unclassified scalar field(s) retained only in the raw table: #{unclassified.join(", ")}"
+              )
+            end
+          end
+
+          def match_any?(patterns, column)
+            patterns.any? { |pattern| pattern.match?(column) }
+          end
+
+          def blank?(value)
+            value.nil? || value.to_s.strip.empty?
+          end
+
+          def normalized_profile_key(value)
+            name = value.to_s.strip
+            name.include?(".") ? name : "profile.#{name}"
+          end
+
+          # Missing and explicit-null context values share one marker so both
+          # collapse to the same identity, matching the SQL PARTITION BY.
+          NIL_VALUE = "\u0000"
+
+          def identity_digest(columns, context_columns)
+            parts = [identity_value(columns["id"])]
+            context_columns.each do |name|
+              parts << name << identity_value(columns[name])
+            end
+            Digest::MD5.digest(parts.join("\u001F"))
+          end
+
+          def identity_value(value)
+            value.nil? ? NIL_VALUE : value.to_s
+          end
+        end
+
+        def self.write_preamble(out)
+          out.write("DROP TABLE IF EXISTS `#{METADATA_TABLE}`;\n")
+        end
+
+        def initialize(table_columns)
+          @table_columns =
+            table_columns.transform_values { |value| Array(value).map(&:to_s) }
+        end
+
+        def write(out)
+          validate!
+          @out = out
+          out.write(
+            "\n-- structured_data_to_sql profile: khoros_api_export v#{VERSION}\n"
+          )
+          TABLES.each { |table| statement("DROP TABLE IF EXISTS `#{table}`") }
+          create_profiles
+          create_avatars
+          create_users
+          create_nodes
+          create_ranks
+          create_roles
+          create_memberships
+          create_badges
+          create_engagement
+          create_floated_messages
+          create_tags
+          create_images
+          create_attachments
+          create_files
+          create_source_messages
+          create_metadata
+        ensure
+          @out = nil
+        end
+
+        def validate!
+          problems =
+            REQUIRED_SCHEMA.filter_map do |table, required|
+              next "#{table}=missing" unless table_exists?(table)
+
+              missing = required - columns(table)
+              "#{table}.#{missing.join(", #{table}.")}=missing" if missing.any?
+            end
+          return if problems.empty?
+
+          raise UsageError,
+                "Unsupported khoros_api_export schema: #{problems.join("; ")}. " \
+                  "Required collections are users, messages, and nodes; a message_authors-only export is not supported. " \
+                  "Regenerate the API export with the current khoros-exporter and prepare it again."
+        end
+
+        private
+
+        def statement(sql)
+          @out.write("#{sql.rstrip};\n")
+        end
+
+        def table_exists?(table)
+          @table_columns.key?(table)
+        end
+
+        def columns(table)
+          @table_columns.fetch(table, [])
+        end
+
+        def has_column?(table, column)
+          columns(table).include?(column)
+        end
+
+        # An optional collection can only contribute canonical rows when it
+        # carries the envelope id its occurrence identity and keys are built
+        # from; without it the helper table is created empty instead of
+        # emitting SQL that references a nonexistent column.
+        def canonical_source?(table, *required_columns)
+          table_exists?(table) && has_column?(table, "id") &&
+            required_columns.all? { |column| has_column?(table, column) }
+        end
+
+        # Returns the column expression, falling back per row (not just when
+        # the column is absent from the schema): mixed v1/v2 exports can carry
+        # the value in either shape on different rows of the same file.
+        def optional(table, column, fallback = nil, table_alias: nil)
+          unless has_column?(table, column)
+            return fallback || "CAST(NULL AS CHAR(255))"
+          end
+
+          expression =
+            table_alias ? "#{table_alias}.`#{column}`" : "`#{column}`"
+          fallback ? "COALESCE(#{expression}, #{fallback})" : expression
+        end
+
+        def key(expression)
+          "CAST(#{expression} AS CHAR(255)) COLLATE utf8mb4_unicode_ci"
+        end
+
+        def text(expression)
+          "CONVERT(#{expression} USING utf8mb4) COLLATE utf8mb4_unicode_ci"
+        end
+
+        def present(*expressions)
+          expressions
+            .map do |expression|
+              "#{expression} IS NOT NULL AND TRIM(CAST(#{expression} AS CHAR)) <> ''"
+            end
+            .join(" AND ")
+        end
+
+        def nonblank(expression)
+          "NULLIF(TRIM(#{expression}), '')"
+        end
+
+        def identity_columns(table, table_alias: "source_row")
+          (["id"] + columns(table).grep(/\Acontext_/).sort).map do |column|
+            "#{table_alias}.`#{column}`"
+          end
+        end
+
+        def canonical_cte(table)
+          <<~SQL
+            RankedRows AS (
+              SELECT source_row.*,
+                ROW_NUMBER() OVER (
+                  PARTITION BY #{identity_columns(table).join(", ")}
+                  ORDER BY source_row._sid DESC
+                ) AS occurrence_rn
+              FROM `#{table}` source_row
+            )
+          SQL
+        end
+
+        def winning_source(table, table_alias)
+          <<~SQL.gsub(/\s+/, " ").strip
+            (SELECT ranked_source.*
+             FROM (
+               SELECT source_row.*,
+                 ROW_NUMBER() OVER (
+                   PARTITION BY #{identity_columns(table).join(", ")}
+                   ORDER BY source_row._sid DESC
+                 ) AS occurrence_rn
+               FROM `#{table}` source_row
+             ) ranked_source
+             WHERE ranked_source.occurrence_rn = 1) #{table_alias}
+          SQL
+        end
+
+        def create_empty(table, columns_sql, indexes: [])
+          lines = columns_sql + indexes
+          statement(<<~SQL)
+            CREATE TABLE `#{table}` (
+              #{lines.join(",\n  ")}
+            ) #{TABLE_OPTIONS}
+          SQL
+        end
+
+        def add_primary_key(table, columns)
+          statement(
+            "ALTER TABLE `#{table}` ADD PRIMARY KEY (#{columns.map { |c| "`#{c}`" }.join(", ")})"
+          )
+        end
+
+        def add_index(table, name, columns)
+          statement(
+            "ALTER TABLE `#{table}` ADD INDEX `#{name}` (#{columns.map { |c| "`#{c}`" }.join(", ")})"
+          )
+        end
+
+        def v1_profile_value_columns
+          return [] unless canonical_source?("user_profiles")
+
+          columns("user_profiles").grep(/\Adata_.+_unnamed\z/)
+        end
+
+        def profile_key_for(column)
+          "profile.#{column.delete_prefix("data_").delete_suffix("_unnamed")}"
+        end
+
+        def create_profiles
+          table = "#{PREFIX}_profiles"
+          value_columns = v1_profile_value_columns
+          sources =
+            value_columns.map do |column|
+              profile_key = profile_key_for(column).gsub("'", "''")
+              <<~SQL.strip
+              SELECT 1 AS source_priority, #{key("p.id")} AS user_id,
+                '#{profile_key}' AS profile_key, p.`#{column}` AS profile_value,
+                p._sid AS source_sid, p._sid AS value_sid
+              FROM #{winning_source("user_profiles", "p")}
+              WHERE #{present("p.id", "p.`#{column}`")}
+            SQL
+            end
+
+          if table_exists?("user_profiles_data_profile") &&
+               canonical_source?("user_profiles") &&
+               %w[_parent_sid name unnamed].all? { |column|
+                 has_column?("user_profiles_data_profile", column)
+               }
+            normalized_key = <<~SQL.gsub(/\s+/, " ").strip
+              CASE
+                WHEN INSTR(TRIM(pv.name), '.') > 0 THEN TRIM(pv.name)
+                ELSE CONCAT('profile.', TRIM(pv.name))
+              END
+            SQL
+            sources << <<~SQL.strip
+              SELECT 2 AS source_priority, #{key("p.id")} AS user_id,
+                #{key(normalized_key)} AS profile_key, pv.unnamed AS profile_value,
+                p._sid AS source_sid, pv._sid AS value_sid
+              FROM `user_profiles_data_profile` pv
+              JOIN #{winning_source("user_profiles", "p")} ON p._sid = pv._parent_sid
+              WHERE #{present("p.id", "pv.name", "pv.unnamed")}
+            SQL
+          end
+
+          if sources.empty?
+            return(
+              create_empty(
+                table,
+                [
+                  "`user_id` VARCHAR(255) NOT NULL",
+                  "`profile_key` VARCHAR(255) NOT NULL",
+                  "`profile_value` LONGTEXT NULL",
+                  "`source_sid` BIGINT NOT NULL"
+                ],
+                indexes: ["PRIMARY KEY (`user_id`, `profile_key`)"]
+              )
+            )
+          end
+
+          statement(<<~SQL)
+            CREATE TABLE `#{table}` #{TABLE_OPTIONS} AS
+            WITH ProfileValues AS (
+              #{sources.join("\nUNION ALL\n")}
+            ), RankedValues AS (
+              SELECT ProfileValues.*,
+                ROW_NUMBER() OVER (
+                  PARTITION BY user_id, profile_key
+                  ORDER BY source_priority DESC, source_sid DESC, value_sid DESC
+                ) AS value_rn
+              FROM ProfileValues
+            )
+            SELECT user_id, profile_key, profile_value, source_sid
+            FROM RankedValues
+            WHERE value_rn = 1
+          SQL
+          add_primary_key(table, %w[user_id profile_key])
+        end
+
+        def create_avatars
+          table = "#{PREFIX}_avatars"
+          unless canonical_source?("avatars")
+            return(
+              create_empty(
+                table,
+                [
+                  "`user_id` VARCHAR(255) NOT NULL",
+                  "`url` TEXT NULL",
+                  "`source_sid` BIGINT NOT NULL"
+                ],
+                indexes: ["PRIMARY KEY (`user_id`)"]
+              )
+            )
+          end
+
+          url =
+            optional(
+              "avatars",
+              "data_url_unnamed",
+              optional("avatars", "data_url"),
+              table_alias: "a"
+            )
+          statement(<<~SQL)
+            CREATE TABLE `#{table}` #{TABLE_OPTIONS} AS
+            WITH #{canonical_cte("avatars")}
+            SELECT #{key("a.id")} AS user_id, #{url} AS url, a._sid AS source_sid
+            FROM RankedRows a
+            WHERE a.occurrence_rn = 1
+          SQL
+          add_primary_key(table, %w[user_id])
+        end
+
+        def profile_lookup(key)
+          key = "profile.#{key}" unless key.include?(".")
+          escaped = key.gsub("'", "''")
+          <<~SQL.gsub(/\s+/, " ").strip
+            (SELECT p.profile_value
+             FROM #{PREFIX}_profiles p
+             WHERE p.user_id = #{key("u.data_id")} AND p.profile_key = '#{escaped}')
+          SQL
+        end
+
+        # Reads a canonical user string from the winning occurrence: the first
+        # nonblank alias present in that row, then the profile fallback. Blank
+        # strings are NULL. Nothing is borrowed from superseded occurrences —
+        # the exporter's last occurrence is the complete observation.
+        def user_value(fields, profile_key: nil)
+          expressions =
+            fields.filter_map do |candidate|
+              nonblank("u.`#{candidate}`") if has_column?("users", candidate)
+            end
+          expressions << nonblank(profile_lookup(profile_key)) if profile_key
+          return "CAST(NULL AS CHAR(255))" if expressions.empty?
+          return expressions.first if expressions.one?
+
+          "COALESCE(#{expressions.join(", ")})"
+        end
+
+        # Same as user_value for scalars (booleans, datetimes): raw values are
+        # kept as-is so explicit false and zero survive; only NULL falls
+        # through to the next alias.
+        def user_scalar(fields, null_type:, profile_key: nil)
+          expressions =
+            fields.filter_map do |candidate|
+              "u.`#{candidate}`" if has_column?("users", candidate)
+            end
+          expressions << nonblank(profile_lookup(profile_key)) if profile_key
+          return "CAST(NULL AS #{null_type})" if expressions.empty?
+          return expressions.first if expressions.one?
+
+          "COALESCE(#{expressions.join(", ")})"
+        end
+
+        # Direct booleans arrive as TINYINT while profile fallbacks are text
+        # ('true'/'false'); normalize to 1/0/NULL so one column has one shape.
+        def boolean_value(expression)
+          <<~SQL.gsub(/\s+/, " ").strip
+            CASE
+              WHEN LOWER(TRIM(CAST(#{expression} AS CHAR))) IN ('1', 'true', 'yes', 'y') THEN 1
+              WHEN LOWER(TRIM(CAST(#{expression} AS CHAR))) IN ('0', 'false', 'no', 'n') THEN 0
+              ELSE NULL
+            END
+          SQL
+        end
+
+        def create_users
+          table = "#{PREFIX}_users"
+          s = USER_STRING_FIELDS
+          n = USER_SCALAR_FIELDS
+          avatar = <<~SQL.gsub(/\s+/, " ").strip
+            COALESCE(
+              NULLIF(TRIM(a.url), ''),
+              #{user_value(s.fetch(:inline_avatar_url))}
+            )
+          SQL
+          statement(<<~SQL)
+            CREATE TABLE `#{table}` #{TABLE_OPTIONS} AS
+            WITH #{canonical_cte("users")}
+            SELECT
+              u._sid AS source_sid,
+              #{key("u.data_id")} AS id,
+              u.data_login AS login,
+              #{key(user_value(s.fetch(:email)))} AS email,
+              #{user_value(s.fetch(:sso_id))} AS sso_id,
+              #{user_scalar(n.fetch(:registration_time), null_type: "DATETIME")} AS registration_time,
+              #{user_scalar(n.fetch(:last_visit_time), null_type: "DATETIME")} AS last_visit_time,
+              #{key(user_value(s.fetch(:rank_id)))} AS rank_id,
+              #{user_value(s.fetch(:first_name), profile_key: "name_first")} AS first_name,
+              #{user_value(s.fetch(:last_name), profile_key: "name_last")} AS last_name,
+              #{user_value(s.fetch(:biography), profile_key: "biography")} AS biography,
+              #{user_value(s.fetch(:location), profile_key: "location")} AS location,
+              #{user_value(s.fetch(:web_page_url), profile_key: "url_homepage")} AS web_page_url,
+              #{user_value(s.fetch(:language), profile_key: "language")} AS language,
+              #{user_value(s.fetch(:title), profile_key: "title")} AS title,
+              #{user_value(s.fetch(:signature), profile_key: "signature")} AS signature,
+              #{avatar} AS avatar_url,
+              #{user_scalar(n.fetch(:banned), null_type: "SIGNED")} AS banned,
+              #{user_value(s.fetch(:registration_status))} AS registration_status,
+              #{user_value(s.fetch(:timezone), profile_key: "config.timezone")} AS timezone,
+              #{user_value(s.fetch(:profile_privacy), profile_key: "profile.privacy")} AS profile_privacy,
+              #{user_value(s.fetch(:online_status_privacy), profile_key: "profile.show_online_status")} AS online_status_privacy,
+              #{boolean_value(user_scalar(n.fetch(:approved), null_type: "SIGNED", profile_key: "user.email_verified"))} AS approved,
+              COALESCE(#{user_scalar(n.fetch(:deleted), null_type: "SIGNED")}, 0) AS deleted
+            FROM RankedRows u
+            LEFT JOIN #{PREFIX}_avatars a ON a.user_id = #{key("u.data_id")}
+            WHERE u.occurrence_rn = 1
+          SQL
+          add_primary_key(table, %w[id])
+          add_index(table, "idx_email", %w[email])
+        end
+
+        def create_nodes
+          table = "#{PREFIX}_nodes"
+          statement(<<~SQL)
+            CREATE TABLE `#{table}` #{TABLE_OPTIONS} AS
+            WITH #{canonical_cte("nodes")}
+            SELECT
+              n._sid AS source_sid,
+              #{key("n.data_id")} AS id,
+              n.data_node_type AS node_type,
+              n.data_title AS title,
+              #{optional("nodes", "data_description", table_alias: "n")} AS description,
+              n.data_depth AS depth,
+              #{optional("nodes", "data_position", "0", table_alias: "n")} AS position,
+              #{optional("nodes", "data_creation_date", table_alias: "n")} AS created_at,
+              #{key(optional("nodes", "data_parent_id", table_alias: "n"))} AS parent_id,
+              COALESCE(#{optional("nodes", "data_hidden", "0", table_alias: "n")}, 0) AS hidden
+            FROM RankedRows n
+            WHERE n.occurrence_rn = 1
+          SQL
+          add_primary_key(table, %w[id])
+          add_index(table, "idx_parent_id", %w[parent_id])
+        end
+
+        def create_ranks
+          table = "#{PREFIX}_ranks"
+          unless canonical_source?("ranks")
+            return(
+              create_empty(
+                table,
+                [
+                  "`id` VARCHAR(255) NOT NULL",
+                  "`name` VARCHAR(255) NULL",
+                  "`position` BIGINT NULL",
+                  "`source_sid` BIGINT NOT NULL"
+                ],
+                indexes: ["PRIMARY KEY (`id`)"]
+              )
+            )
+          end
+          statement(<<~SQL)
+            CREATE TABLE `#{table}` #{TABLE_OPTIONS} AS
+            WITH #{canonical_cte("ranks")}
+            SELECT #{key("r.data_id")} AS id,
+              #{optional("ranks", "data_name", table_alias: "r")} AS name,
+              #{optional("ranks", "data_position", table_alias: "r")} AS position,
+              r._sid AS source_sid
+            FROM RankedRows r
+            WHERE r.occurrence_rn = 1
+          SQL
+          add_primary_key(table, %w[id])
+        end
+
+        def create_roles
+          table = "#{PREFIX}_roles"
+          unless canonical_source?("roles")
+            return(
+              create_empty(
+                table,
+                [
+                  "`id` VARCHAR(255) NOT NULL",
+                  "`name` VARCHAR(255) NULL",
+                  "`description` TEXT NULL",
+                  "`node_id` VARCHAR(255) NULL",
+                  "`source_sid` BIGINT NOT NULL"
+                ],
+                indexes: ["PRIMARY KEY (`id`)"]
+              )
+            )
+          end
+          statement(<<~SQL)
+            CREATE TABLE `#{table}` #{TABLE_OPTIONS} AS
+            WITH #{canonical_cte("roles")}
+            SELECT #{key("r.data_id")} AS id,
+              #{optional("roles", "data_name", table_alias: "r")} AS name,
+              #{optional("roles", "data_description", table_alias: "r")} AS description,
+              #{key(optional("roles", "data_node_id", table_alias: "r"))} AS node_id,
+              r._sid AS source_sid
+            FROM RankedRows r
+            WHERE r.occurrence_rn = 1
+          SQL
+          add_primary_key(table, %w[id])
+        end
+
+        def create_memberships
+          table = "#{PREFIX}_memberships"
+          sources = []
+          if canonical_source?("role_users", "context_role_id")
+            sources << <<~SQL.strip
+              SELECT 'role' AS membership_type, #{key("ru.context_role_id")} AS container_id,
+                #{key("ru.id")} AS user_id, #{key("NULL")} AS membership_role, ru._sid AS source_sid
+              FROM #{winning_source("role_users", "ru")}
+              WHERE #{present("ru.context_role_id", "ru.id")}
+            SQL
+          end
+          if canonical_source?("grouphub_members", "context_grouphub_id")
+            role =
+              optional("grouphub_members", "context_role", table_alias: "gm")
+            sources << <<~SQL.strip
+              SELECT 'grouphub' AS membership_type, #{key("gm.context_grouphub_id")} AS container_id,
+                #{key("gm.id")} AS user_id, #{text(role)} AS membership_role, gm._sid AS source_sid
+              FROM #{winning_source("grouphub_members", "gm")}
+              WHERE #{present("gm.context_grouphub_id", "gm.id")}
+            SQL
+          end
+          if sources.empty?
+            return(
+              create_empty(
+                table,
+                [
+                  "`membership_type` VARCHAR(32) NOT NULL",
+                  "`container_id` VARCHAR(255) NOT NULL",
+                  "`user_id` VARCHAR(255) NOT NULL",
+                  "`membership_role` VARCHAR(255) NULL",
+                  "`source_sid` BIGINT NOT NULL"
+                ],
+                indexes: [
+                  "PRIMARY KEY (`membership_type`, `container_id`, `user_id`)",
+                  "KEY `idx_user_id` (`user_id`)"
+                ]
+              )
+            )
+          end
+          statement(<<~SQL)
+            CREATE TABLE `#{table}` #{TABLE_OPTIONS} AS
+            WITH MembershipRows AS (
+              #{sources.join("\nUNION ALL\n")}
+            ), RankedMemberships AS (
+              SELECT MembershipRows.*,
+                ROW_NUMBER() OVER (
+                  PARTITION BY membership_type, container_id, user_id
+                  ORDER BY source_sid DESC
+                ) AS membership_rn
+              FROM MembershipRows
+            )
+            SELECT membership_type, container_id, user_id, membership_role, source_sid
+            FROM RankedMemberships
+            WHERE membership_rn = 1
+          SQL
+          add_primary_key(table, %w[membership_type container_id user_id])
+          add_index(table, "idx_user_id", %w[user_id])
+        end
+
+        def inline_badge_source
+          return nil unless table_exists?("users_data_user_badges_items")
+
+          <<~SQL.strip
+            SELECT 1 AS source_priority, #{key("u.data_id")} AS user_id, #{key("bg.badge_id")} AS badge_id,
+              #{text(optional("users_data_user_badges_items", "badge_title", table_alias: "bg"))} AS title,
+              #{text(optional("users_data_user_badges_items", "badge_description", table_alias: "bg"))} AS description,
+              #{text(optional("users_data_user_badges_items", "badge_icon_url", table_alias: "bg"))} AS icon_url,
+              #{optional("users_data_user_badges_items", "earned_date", table_alias: "bg")} AS earned_date,
+              u._sid AS source_sid
+            FROM users_data_user_badges_items bg
+            JOIN #{winning_source("users", "u")} ON u._sid = bg._parent_sid
+            WHERE #{present("u.data_id", "bg.badge_id")}
+          SQL
+        end
+
+        def contextual_badge_source
+          return nil unless canonical_source?("user_badges", "context_user_id")
+
+          badge_id =
+            optional(
+              "user_badges",
+              "data_badge_id_unnamed",
+              optional(
+                "user_badges",
+                "data_badge_id",
+                optional(
+                  "user_badges",
+                  "data_id_unnamed",
+                  optional("user_badges", "data_id")
+                )
+              ),
+              table_alias: "ub"
+            )
+          <<~SQL.strip
+            SELECT 2 AS source_priority, #{key("ub.context_user_id")} AS user_id, #{key(badge_id)} AS badge_id,
+              #{key("NULL")} AS title, #{key("NULL")} AS description, #{key("NULL")} AS icon_url,
+              #{optional("user_badges", "data_earned_date_unnamed", optional("user_badges", "data_earned_date"), table_alias: "ub")} AS earned_date,
+              ub._sid AS source_sid
+            FROM #{winning_source("user_badges", "ub")}
+            WHERE #{present("ub.context_user_id", badge_id)}
+          SQL
+        end
+
+        def badge_definition_source
+          return nil unless canonical_source?("badges")
+
+          badge_id =
+            optional(
+              "badges",
+              "data_id_unnamed",
+              optional("badges", "data_id"),
+              table_alias: "b"
+            )
+          <<~SQL.strip
+            SELECT 2 AS priority, #{key(badge_id)} AS badge_id,
+              #{text(optional("badges", "data_title_unnamed", optional("badges", "data_title"), table_alias: "b"))} AS title,
+              #{text(optional("badges", "data_description_unnamed", optional("badges", "data_description"), table_alias: "b"))} AS description,
+              #{text(optional("badges", "data_icon_url_unnamed", optional("badges", "data_icon_url"), table_alias: "b"))} AS icon_url,
+              b._sid AS source_sid
+            FROM #{winning_source("badges", "b")}
+            WHERE #{present(badge_id)}
+          SQL
+        end
+
+        def create_badges
+          definitions = "#{PREFIX}_badge_definitions"
+          grants = "#{PREFIX}_badge_grants"
+          inline = inline_badge_source
+          contextual = contextual_badge_source
+          explicit = badge_definition_source
+          grant_sources = [inline, contextual].compact
+
+          if grant_sources.empty?
+            create_empty(
+              grants,
+              [
+                "`user_id` VARCHAR(255) NOT NULL",
+                "`badge_id` VARCHAR(255) NOT NULL",
+                "`earned_date` DATETIME NULL",
+                "`source_sid` BIGINT NOT NULL"
+              ],
+              indexes: ["PRIMARY KEY (`user_id`, `badge_id`)"]
+            )
+          else
+            statement(<<~SQL)
+              CREATE TABLE `#{grants}` #{TABLE_OPTIONS} AS
+              WITH BadgeGrantRows AS (
+                #{grant_sources.join("\nUNION ALL\n")}
+              ), RankedBadgeGrants AS (
+                SELECT BadgeGrantRows.*,
+                  ROW_NUMBER() OVER (
+                    PARTITION BY user_id, badge_id
+                    -- Prefer an earned timestamp, then the explicit per-user
+                    -- v1 observation over an inline v2 observation. _sid is
+                    -- the final precedence within the same source.
+                    ORDER BY (earned_date IS NOT NULL) DESC, earned_date DESC,
+                      source_priority DESC, source_sid DESC
+                  ) AS grant_rn
+                FROM BadgeGrantRows
+                WHERE user_id IS NOT NULL AND badge_id IS NOT NULL
+              )
+              SELECT user_id, badge_id, earned_date, source_sid
+              FROM RankedBadgeGrants
+              WHERE grant_rn = 1
+            SQL
+            add_primary_key(grants, %w[user_id badge_id])
+          end
+
+          definition_sources = []
+          definition_sources << explicit if explicit
+          definition_sources << <<~SQL.strip if inline
+              SELECT 1 AS priority, badge_id, title, description, icon_url, source_sid
+              FROM (#{inline}) inline_badges
+            SQL
+          if definition_sources.empty?
+            return(
+              create_empty(
+                definitions,
+                [
+                  "`badge_id` VARCHAR(255) NOT NULL",
+                  "`title` VARCHAR(255) NULL",
+                  "`description` TEXT NULL",
+                  "`icon_url` TEXT NULL",
+                  "`source_sid` BIGINT NOT NULL"
+                ],
+                indexes: ["PRIMARY KEY (`badge_id`)"]
+              )
+            )
+          end
+          statement(<<~SQL)
+            CREATE TABLE `#{definitions}` #{TABLE_OPTIONS} AS
+            WITH BadgeDefinitionRows AS (
+              #{definition_sources.join("\nUNION ALL\n")}
+            ), RankedBadgeDefinitions AS (
+              SELECT BadgeDefinitionRows.*,
+                ROW_NUMBER() OVER (
+                  PARTITION BY badge_id
+                  ORDER BY priority DESC, source_sid DESC
+                ) AS definition_rn
+              FROM BadgeDefinitionRows
+              WHERE badge_id IS NOT NULL
+            )
+            SELECT badge_id, title, description, icon_url, source_sid
+            FROM RankedBadgeDefinitions
+            WHERE definition_rn = 1
+          SQL
+          add_primary_key(definitions, %w[badge_id])
+        end
+
+        def create_engagement
+          table = "#{PREFIX}_engagement"
+          unless canonical_source?("kudos", "context_message_id")
+            return(
+              create_empty(
+                table,
+                [
+                  "`message_id` VARCHAR(255) NOT NULL",
+                  "`engagement_id` VARCHAR(255) NOT NULL",
+                  "`user_id` VARCHAR(255) NULL",
+                  "`created_at` DATETIME NULL",
+                  "`weight` BIGINT NULL",
+                  "`source_sid` BIGINT NOT NULL"
+                ],
+                indexes: [
+                  "PRIMARY KEY (`message_id`, `engagement_id`)",
+                  "KEY `idx_user_id` (`user_id`)"
+                ]
+              )
+            )
+          end
+          statement(<<~SQL)
+            CREATE TABLE `#{table}` #{TABLE_OPTIONS} AS
+            WITH #{canonical_cte("kudos")}
+            SELECT #{key("k.context_message_id")} AS message_id, #{key("k.id")} AS engagement_id,
+              #{key(optional("kudos", "data_user_id", table_alias: "k"))} AS user_id,
+              #{optional("kudos", "data_time", table_alias: "k")} AS created_at,
+              #{optional("kudos", "data_weight", "1", table_alias: "k")} AS weight,
+              k._sid AS source_sid
+            FROM RankedRows k
+            WHERE k.occurrence_rn = 1
+              AND #{present("k.context_message_id", "k.id")}
+          SQL
+          add_primary_key(table, %w[message_id engagement_id])
+          add_index(table, "idx_user_id", %w[user_id])
+        end
+
+        def create_floated_messages
+          table = "#{PREFIX}_floated_messages"
+          unless canonical_source?("floated_messages", "context_board_id")
+            return(
+              create_empty(
+                table,
+                [
+                  "`board_id` VARCHAR(255) NOT NULL",
+                  "`message_id` VARCHAR(255) NOT NULL",
+                  "`source_sid` BIGINT NOT NULL"
+                ],
+                indexes: ["PRIMARY KEY (`board_id`, `message_id`)"]
+              )
+            )
+          end
+          message_id =
+            (
+              if has_column?("floated_messages", "data_message_id")
+                "f.data_message_id"
+              else
+                "f.id"
+              end
+            )
+          statement(<<~SQL)
+            CREATE TABLE `#{table}` #{TABLE_OPTIONS} AS
+            WITH #{canonical_cte("floated_messages")}
+            SELECT #{key("f.context_board_id")} AS board_id, #{key(message_id)} AS message_id,
+              f._sid AS source_sid
+            FROM RankedRows f
+            WHERE f.occurrence_rn = 1
+              AND #{present("f.context_board_id", message_id)}
+          SQL
+          add_primary_key(table, %w[board_id message_id])
+        end
+
+        def create_tags
+          table = "#{PREFIX}_tags"
+          sources =
+            %w[labels tags].filter_map do |source|
+              next unless canonical_source?(source, "context_message_id")
+
+              text_column =
+                has_column?(source, "data_text") ? "data_text" : "data_name"
+              next unless has_column?(source, text_column)
+
+              <<~SQL.strip
+              SELECT '#{source == "labels" ? "label" : "freeform_tag"}' AS source_type,
+                #{key("t.id")} AS source_id, #{key("t.context_message_id")} AS message_id,
+                t.`#{text_column}` AS text, t._sid AS source_sid
+              FROM #{winning_source(source, "t")}
+              WHERE #{present("t.id", "t.context_message_id")}
+            SQL
+            end
+          if sources.empty?
+            return(
+              create_empty(
+                table,
+                [
+                  "`source_type` VARCHAR(32) NOT NULL",
+                  "`source_id` VARCHAR(255) NOT NULL",
+                  "`message_id` VARCHAR(255) NOT NULL",
+                  "`text` VARCHAR(255) NULL",
+                  "`source_sid` BIGINT NOT NULL"
+                ],
+                indexes: [
+                  "PRIMARY KEY (`source_type`, `source_id`, `message_id`)",
+                  "KEY `idx_message_id` (`message_id`)"
+                ]
+              )
+            )
+          end
+          statement(<<~SQL)
+            CREATE TABLE `#{table}` #{TABLE_OPTIONS} AS
+            WITH TagRows AS (
+              #{sources.join("\nUNION ALL\n")}
+            ), RankedTags AS (
+              SELECT TagRows.*,
+                ROW_NUMBER() OVER (
+                  PARTITION BY source_type, source_id, message_id
+                  ORDER BY source_sid DESC
+                ) AS tag_rn
+              FROM TagRows
+            )
+            SELECT source_type, source_id, message_id, text, source_sid
+            FROM RankedTags
+            WHERE tag_rn = 1
+          SQL
+          add_primary_key(table, %w[source_type source_id message_id])
+          add_index(table, "idx_message_id", %w[message_id])
+        end
+
+        def create_images
+          table = "#{PREFIX}_images"
+          sources = []
+          if canonical_source?("images", "context_message_id")
+            sources << <<~SQL.strip
+              SELECT 1 AS priority, #{key("i.context_message_id")} AS message_id, #{key("i.id")} AS image_id,
+                #{text("COALESCE(#{nonblank(optional("images", "data_original_href", table_alias: "i"))}, #{nonblank(optional("images", "data_large_href", table_alias: "i"))}, #{nonblank(optional("images", "data_view_href", table_alias: "i"))})")} AS url,
+                #{text(optional("images", "data_title", table_alias: "i"))} AS title,
+                #{text(optional("images", "data_description", table_alias: "i"))} AS description,
+                i._sid AS source_sid
+              FROM #{winning_source("images", "i")}
+              WHERE #{present("i.context_message_id", "i.id")}
+            SQL
+          end
+          if canonical_source?("uploaded_images", "context_message_id")
+            image_id =
+              optional(
+                "uploaded_images",
+                "data_id_unnamed",
+                optional("uploaded_images", "data_id"),
+                table_alias: "i"
+              )
+            url =
+              optional(
+                "uploaded_images",
+                "data_url_unnamed",
+                optional("uploaded_images", "data_url"),
+                table_alias: "i"
+              )
+            title =
+              optional(
+                "uploaded_images",
+                "data_title_unnamed",
+                optional("uploaded_images", "data_title"),
+                table_alias: "i"
+              )
+            description =
+              optional(
+                "uploaded_images",
+                "data_description_unnamed",
+                optional("uploaded_images", "data_description"),
+                table_alias: "i"
+              )
+            usable_url = "CASE WHEN #{present(url)} THEN #{url} ELSE NULL END"
+            sources << <<~SQL.strip
+              SELECT 2 AS priority, #{key("i.context_message_id")} AS message_id, #{key(image_id)} AS image_id,
+                #{text(usable_url)} AS url, #{text(title)} AS title,
+                #{text(description)} AS description, i._sid AS source_sid
+              FROM #{winning_source("uploaded_images", "i")}
+              WHERE #{present("i.context_message_id", image_id)}
+            SQL
+          end
+          if sources.empty?
+            return(
+              create_empty(
+                table,
+                [
+                  "`message_id` VARCHAR(255) NOT NULL",
+                  "`image_id` VARCHAR(255) NOT NULL",
+                  "`url` TEXT NULL",
+                  "`title` VARCHAR(255) NULL",
+                  "`description` TEXT NULL",
+                  "`source_sid` BIGINT NOT NULL"
+                ],
+                indexes: ["PRIMARY KEY (`message_id`, `image_id`)"]
+              )
+            )
+          end
+          statement(<<~SQL)
+            CREATE TABLE `#{table}` #{TABLE_OPTIONS} AS
+            WITH ImageRows AS (
+              #{sources.join("\nUNION ALL\n")}
+            ), RankedImageSources AS (
+              SELECT ImageRows.*,
+                ROW_NUMBER() OVER (
+                  PARTITION BY message_id, image_id, priority
+                  ORDER BY source_sid DESC
+                ) AS source_rn
+              FROM ImageRows
+              WHERE message_id IS NOT NULL AND image_id IS NOT NULL
+            )
+            SELECT message_id, image_id,
+              COALESCE(
+                MAX(CASE WHEN priority = 2 THEN url END),
+                MAX(CASE WHEN priority = 1 THEN url END)
+              ) AS url,
+              COALESCE(
+                MAX(CASE WHEN priority = 1 THEN title END),
+                MAX(CASE WHEN priority = 2 THEN title END)
+              ) AS title,
+              COALESCE(
+                MAX(CASE WHEN priority = 1 THEN description END),
+                MAX(CASE WHEN priority = 2 THEN description END)
+              ) AS description,
+              COALESCE(
+                MAX(CASE WHEN priority = 2 THEN source_sid END),
+                MAX(CASE WHEN priority = 1 THEN source_sid END)
+              ) AS source_sid
+            FROM RankedImageSources
+            WHERE source_rn = 1
+            GROUP BY message_id, image_id
+          SQL
+          add_primary_key(table, %w[message_id image_id])
+        end
+
+        def create_attachments
+          table = "#{PREFIX}_attachments"
+          unless canonical_source?("attachments", "context_message_id")
+            return(
+              create_empty(
+                table,
+                [
+                  "`message_id` VARCHAR(255) NOT NULL",
+                  "`attachment_id` VARCHAR(255) NOT NULL",
+                  "`file_name` TEXT NULL",
+                  "`file_size` BIGINT NULL",
+                  "`content_type` VARCHAR(255) NULL",
+                  "`url` TEXT NULL",
+                  "`description` TEXT NULL",
+                  "`position` BIGINT NULL",
+                  "`source_sid` BIGINT NOT NULL"
+                ],
+                indexes: [
+                  "PRIMARY KEY (`message_id`, `attachment_id`)",
+                  "KEY `idx_message_id` (`message_id`)"
+                ]
+              )
+            )
+          end
+
+          attachment_id =
+            optional("attachments", "data_id", "a.id", table_alias: "a")
+          file_name =
+            optional(
+              "attachments",
+              "data_filename",
+              optional("attachments", "data_file_name", table_alias: "a"),
+              table_alias: "a"
+            )
+          file_size =
+            optional(
+              "attachments",
+              "data_filesize",
+              optional("attachments", "data_file_size", table_alias: "a"),
+              table_alias: "a"
+            )
+          statement(<<~SQL)
+            CREATE TABLE `#{table}` #{TABLE_OPTIONS} AS
+            WITH #{canonical_cte("attachments")}, AttachmentRows AS (
+              SELECT #{key("a.context_message_id")} AS message_id,
+                #{key(attachment_id)} AS attachment_id,
+                #{text(file_name)} AS file_name,
+                #{file_size} AS file_size,
+                #{text(optional("attachments", "data_content_type", table_alias: "a"))} AS content_type,
+                #{text(optional("attachments", "data_url", table_alias: "a"))} AS url,
+                #{text(optional("attachments", "data_description", table_alias: "a"))} AS description,
+                #{optional("attachments", "data_position", table_alias: "a")} AS position,
+                a._sid AS source_sid
+              FROM RankedRows a
+              WHERE a.occurrence_rn = 1
+                AND #{present("a.context_message_id", attachment_id)}
+            ), RankedAttachments AS (
+              SELECT AttachmentRows.*,
+                ROW_NUMBER() OVER (
+                  PARTITION BY message_id, attachment_id
+                  ORDER BY source_sid DESC
+                ) AS attachment_rn
+              FROM AttachmentRows
+            )
+            SELECT message_id, attachment_id, file_name, file_size, content_type,
+              url, description, position, source_sid
+            FROM RankedAttachments
+            WHERE attachment_rn = 1
+          SQL
+          add_primary_key(table, %w[message_id attachment_id])
+          add_index(table, "idx_message_id", %w[message_id])
+        end
+
+        # The exporter's files.ndjson maps every downloaded URL to a
+        # content-addressed blob (`files/<xx>/<sha256>`, no extension). Rows
+        # with status `duplicate` still carry the sha and path (same blob,
+        # different URL). Rows without a path (errors, bot checks, missing)
+        # are retained so the converter can report them; consumers filter
+        # `path IS NOT NULL`. `url_key` is SHA1(url) because URLs exceed the
+        # 255-character key budget.
+        def create_files
+          table = "#{PREFIX}_files"
+          unless canonical_source?("files") && has_column?("files", "url")
+            return(
+              create_empty(
+                table,
+                [
+                  "`url_key` CHAR(40) NOT NULL",
+                  "`url` TEXT NOT NULL",
+                  "`sha256` CHAR(64) NULL",
+                  "`path` VARCHAR(255) NULL",
+                  "`bytes` BIGINT NULL",
+                  "`content_type` VARCHAR(255) NULL",
+                  "`status` VARCHAR(64) NULL",
+                  "`source_collection` VARCHAR(64) NULL",
+                  "`source_id` VARCHAR(255) NULL",
+                  "`source_message_id` VARCHAR(255) NULL",
+                  "`source_sid` BIGINT NOT NULL"
+                ],
+                indexes: [
+                  "PRIMARY KEY (`url_key`)",
+                  "KEY `idx_source` (`source_collection`, `source_message_id`)"
+                ]
+              )
+            )
+          end
+
+          statement(<<~SQL)
+            CREATE TABLE `#{table}` #{TABLE_OPTIONS} AS
+            WITH #{canonical_cte("files")}
+            SELECT
+              SHA1(f.url) AS url_key,
+              #{text("f.url")} AS url,
+              #{key(optional("files", "data_sha256", table_alias: "f"))} AS sha256,
+              #{key(optional("files", "data_path", table_alias: "f"))} AS path,
+              #{optional("files", "data_bytes", table_alias: "f")} AS bytes,
+              #{key(optional("files", "data_content_type", table_alias: "f"))} AS content_type,
+              #{key(optional("files", "data_status", table_alias: "f"))} AS status,
+              #{key(optional("files", "data_source_collection", table_alias: "f"))} AS source_collection,
+              #{key(optional("files", "data_source_id", table_alias: "f"))} AS source_id,
+              #{key(optional("files", "data_source_message_id", table_alias: "f"))} AS source_message_id,
+              f._sid AS source_sid
+            FROM RankedRows f
+            WHERE f.occurrence_rn = 1
+              AND #{present("f.url")}
+          SQL
+          add_primary_key(table, %w[url_key])
+          add_index(
+            table,
+            "idx_source",
+            %w[source_collection source_message_id]
+          )
+        end
+
+        def create_source_messages
+          table = "#{PREFIX}_source_messages"
+          statement(<<~SQL)
+            CREATE TABLE `#{table}` #{TABLE_OPTIONS} AS
+            WITH #{canonical_cte("messages")}
+            SELECT
+              m._sid AS source_sid,
+              #{key("m.data_id")} AS id,
+              m.data_author_id AS author_id,
+              #{key("m.data_board_id")} AS board_id,
+              #{key("m.data_conversation_id")} AS conversation_id,
+              #{optional("messages", "data_parent_id", table_alias: "m")} AS parent_id,
+              m.data_depth AS depth,
+              m.data_subject AS subject,
+              m.data_body AS body,
+              m.data_post_time AS post_time,
+              #{boolean_value("m.data_read_only")} AS read_only,
+              COALESCE(#{optional("messages", "data_is_solution", "0", table_alias: "m")}, 0) AS is_solution,
+              #{optional("messages", "data_solution_data_message_id", table_alias: "m")} AS solution_message_id,
+              #{optional("messages", "data_solution_data_accepter_id", table_alias: "m")} AS solution_accepter_id,
+              #{optional("messages", "data_solution_data_time", table_alias: "m")} AS solution_time
+            FROM RankedRows m
+            WHERE m.occurrence_rn = 1
+          SQL
+          add_primary_key(table, %w[id])
+          add_index(table, "idx_conversation_id", %w[conversation_id])
+          add_index(table, "idx_board_id", %w[board_id])
+        end
+
+        def create_metadata
+          statement(<<~SQL)
+            CREATE TABLE `#{METADATA_TABLE}` (
+              `profile` VARCHAR(64) NOT NULL,
+              `profile_version` BIGINT NOT NULL,
+              `tool_version` VARCHAR(64) NOT NULL,
+              PRIMARY KEY (`profile`)
+            ) #{TABLE_OPTIONS}
+          SQL
+          statement(<<~SQL)
+            INSERT INTO `#{METADATA_TABLE}` (`profile`, `profile_version`, `tool_version`)
+            VALUES ('khoros_api_export', #{VERSION}, '#{StructuredDataToSql::VERSION.gsub("'", "''")}')
+          SQL
+        end
+      end
+    end
+  end
+end