structured_data_to_sql 0.2.0 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ff2d9500aa62ada917f7f65d973d3c9e5495506cfcc6cf06011328482fb50288
-  data.tar.gz: f5440b2010078d1b011106ba82c4256d44098501f91131706c80ad45d259de5f
+  metadata.gz: 0c7a2d7a571374df01b77f059c243d1ddf43ab4ac51fe85f21c3b5a8ba7d1ed4
+  data.tar.gz: bca5b0b6e07ff5c7de2c254bcfa336b27f9ef35c949a6b1e3db4618a61820030
 SHA512:
-  metadata.gz: b443a51fdabf4dc975562fae2ed7e6cde39b616597cb640134dec3d470cc17f9ec27a46289c746b5ce3375d83861e86f0769e84266ad274f4cc7c95f52076b13
-  data.tar.gz: ebe2b6d53b667d71d8d8260eaaf39990cb56ffe6911117a9f2513e2da49da343b10fd1e848a2571a44d65bed1edc06fecd872ce6d962d0d34a800cbdd1ade0bf
+  metadata.gz: 7673f7ac81093a5c3e533b300e71f422a1026ac6c814270f400720f571b09d6918b6badffe99c7c8419983071a68cba51aac951bf7c1348c78c9a48c6afc8624
+  data.tar.gz: de4c881ae651cb0ff5ef6e9973a44d3e90d9763a6ea7521e34a7174c661bd361bb30c0aa97333ff387aa6dd1d2428a8f9958e12d4de8b75ea6ec9d1007bbbd5d

data/lib/structured_data_to_sql/diagnostics_report.rb

@@ -150,8 +150,10 @@ module StructuredDataToSql
         return lines
       end
 
-      lines << "Quality signals are scoped to the latest run whose manifest entry is `completed`, " \
-        "by attributing the last _N_ rows of the append-only errors/gaps files to it (_N_ = that run's declared count). " \
+      lines << "Quality signals are scoped to the latest content run whose manifest entry is `completed`, " \
+        "by attributing the last _N_ rows of the append-only errors/gaps files to it (_N_ = that run's declared count), " \
+        "less whatever later runs declared. A `structure_repair` run that follows it is reported on its own line; " \
+        "a later run that never finalized its entry leaves the breakdown unattributable and it is withheld. " \
         "Rows beyond the sum of declared counts belong to runs that never finalized their entry."
       lines << ""
       lines << "| Run | Status | Started | Finished | Mode | Credential | Emails | Errors | Gaps |"

data/lib/structured_data_to_sql/json/exporter_manifest.rb

@@ -26,6 +26,23 @@ module StructuredDataToSql
           def completed?
             status == "completed"
           end
+
+          # A structure repair appends nodes and its own error/gap rows to an
+          # existing export; it is a run, but not the content export the rows
+          # were converted from.
+          def repair?
+            mode == "structure_repair"
+          end
+
+          def content?
+            !repair?
+          end
+
+          # An entry the exporter closed, whatever the outcome. A run killed
+          # outright stays `running` with no counts, so its rows are undeclared.
+          def finalized?
+            status != "running" && !finished_at.nil?
+          end
         end
 
       FILENAME = "manifest.json"
@@ -64,8 +81,24 @@ module StructuredDataToSql
         parse
       end
 
+      # The latest completed content run: what the quality signals are scoped to.
       def latest_completed
-        @runs.reverse.find(&:completed?)
+        @runs.reverse.find { |run| run.completed? && run.content? }
+      end
+
+      def latest_completed_repair
+        @runs.reverse.find { |run| run.completed? && run.repair? }
+      end
+
+      # Rows declared by the runs after +run+. The files are append-only, so a
+      # run's rows end where the next run's begin.
+      def rows_after(run, counter)
+        @runs.drop(run.index + 1).sum(&counter)
+      end
+
+      # Later runs whose rows are not declared, so their position is unknown.
+      def unfinalized_after(run)
+        @runs.drop(run.index + 1).reject(&:finalized?)
       end
 
       def errors_total

data/lib/structured_data_to_sql/json/profiles/khoros_api_export.rb

@@ -17,8 +17,8 @@ module StructuredDataToSql
         NAME = :khoros_api_export
         # Bumped in lockstep with PROFILE_VERSION in the Khoros converter's
         # KhorosApiExportContentStaging whenever a canonical table's shape or
-        # semantics change. Version 3 adds source-message view counts.
-        VERSION = 3
+        # semantics change. Version 4 adds optional API subscriptions.
+        VERSION = 4
         PREFIX = "discourse_khoros_api"
         METADATA_TABLE = "_structured_data_to_sql_profiles"
         # Explicit storage options on every canonical table so they never
@@ -62,6 +62,7 @@ module StructuredDataToSql
             attachments
             files
             source_messages
+            subscriptions
           ].map { |name| "#{PREFIX}_#{name}" }.freeze
         USER_STRING_FIELDS = {
           id: %w[data_id],
@@ -222,6 +223,21 @@ module StructuredDataToSql
             }
           }.freeze
           OPTIONAL = {
+            "subscriptions" => {
+              required: {
+                subscription_id: %w[id],
+                context_user_id: %w[context_user_id],
+                subscriber_id: %w[data_subscriber_id],
+                target_type: %w[data_target_type],
+                target_id: %w[data_target_id]
+              },
+              indexed: {
+                subscription_id: %w[id],
+                context_user_id: %w[context_user_id],
+                subscriber_id: %w[data_subscriber_id],
+                target_id: %w[data_target_id]
+              }
+            },
             "user_profiles" => {
               required: {
                 user_id: %w[id]
@@ -718,7 +734,7 @@ module StructuredDataToSql
             latest = manifest.latest_completed
             if latest.nil?
               return [
-                "Latest run: not scoped — no completed run in manifest.json; showing lifetime totals"
+                "Latest run: not scoped — no completed content run in manifest.json; showing lifetime totals"
               ]
             end
 
@@ -730,22 +746,77 @@ module StructuredDataToSql
               ]
             end
 
-            latest_count = latest.public_send(counter)
-            latest_rows = @quality_signal_rows[table].last(latest_count)
-            latest_tallies = Hash.new(0)
-            latest_rows.each { |label| latest_tallies[label] += 1 }
-            window = [latest.started_at, latest.finished_at].compact.join(" → ")
+            unattributed = total - declared
             lines = [
-              "Latest completed run ##{latest.index}#{window.empty? ? "" : " (#{window})"}: " \
-                "#{Format.format_count(latest_count)}#{latest_count.positive? ? " — #{breakdown_text(latest_tallies)}" : ""}"
+              attributed_line(
+                "Latest completed run",
+                latest,
+                manifest,
+                table,
+                counter,
+                total,
+                unattributed
+              )
             ]
-            unattributed = total - declared
+            repair = manifest.latest_completed_repair
+            if repair && repair.index > latest.index
+              lines << attributed_line(
+                "Latest structure repair",
+                repair,
+                manifest,
+                table,
+                counter,
+                total,
+                unattributed
+              )
+            end
             if unattributed.positive?
               lines << "Unattributed: #{Format.format_count(unattributed)} row(s) written by runs that did not finalize their manifest entry"
             end
             lines
           end
 
+          # A run's rows sit before whatever later runs appended, so its slice
+          # ends where theirs begin — provided every later run declared its
+          # rows. Undeclared rows that may sit after this run's make the slice a
+          # guess, and a guess is not shown as a breakdown.
+          def attributed_line(
+            label,
+            run,
+            manifest,
+            table,
+            counter,
+            total,
+            unattributed
+          )
+            count = run.public_send(counter)
+            window = [run.started_at, run.finished_at].compact.join(" → ")
+            head =
+              "#{label} ##{run.index}#{window.empty? ? "" : " (#{window})"}: " \
+                "#{Format.format_count(count)}"
+            undeclared =
+              unattributed.positive? ? manifest.unfinalized_after(run) : []
+            unless undeclared.empty?
+              blamed =
+                undeclared
+                  .map { |later| "##{later.index} (#{later.status})" }
+                  .join(", ")
+              return(
+                "#{head} — breakdown not attributable: run #{blamed} did not finalize its " \
+                  "entry, so undeclared rows may sit after this run's"
+              )
+            end
+            return head unless count.positive?
+
+            after = manifest.rows_after(run, counter)
+            tallies = Hash.new(0)
+            @quality_signal_rows[table]
+              .first(total - after)
+              .last(count)
+              .each { |row_label| tallies[row_label] += 1 }
+            "#{head} — #{breakdown_text(tallies)}"
+          end
+
           def breakdown_text(tallies)
             tallies
               .sort_by { |label, count| [-count, label] }
@@ -878,9 +949,10 @@ module StructuredDataToSql
           out.write("DROP TABLE IF EXISTS `#{METADATA_TABLE}`;\n")
         end
 
-        def initialize(table_columns)
+        def initialize(table_columns, supplied_collections: [])
           @table_columns =
             table_columns.transform_values { |value| Array(value).map(&:to_s) }
+          @supplied_collections = supplied_collections
         end
 
         def write(out)
@@ -905,6 +977,7 @@ module StructuredDataToSql
           create_attachments
           create_files
           create_source_messages
+          create_subscriptions
           create_metadata
         ensure
           @out = nil
@@ -1942,18 +2015,70 @@ module StructuredDataToSql
           add_index(table, "idx_board_id", %w[board_id])
         end
 
+        def create_subscriptions
+          table = "#{PREFIX}_subscriptions"
+          unless canonical_source?(
+                   "subscriptions",
+                   "context_user_id",
+                   "data_subscriber_id",
+                   "data_target_type",
+                   "data_target_id"
+                 )
+            create_empty(
+              table,
+              [
+                "`id` VARCHAR(255) NOT NULL",
+                "`context_user_id` VARCHAR(255) NOT NULL",
+                "`subscriber_id` VARCHAR(255) NOT NULL",
+                "`target_type` VARCHAR(255) NOT NULL",
+                "`target_id` VARCHAR(255) NOT NULL",
+                "`source_sid` BIGINT NOT NULL"
+              ],
+              indexes: [
+                "PRIMARY KEY (`id`, `context_user_id`)",
+                "KEY `idx_subscriber_id` (`subscriber_id`)"
+              ]
+            )
+            return
+          end
+
+          statement(<<~SQL)
+            CREATE TABLE `#{table}` #{TABLE_OPTIONS} AS
+            SELECT
+              #{key("s.id")} AS id,
+              #{key("s.context_user_id")} AS context_user_id,
+              #{key("s.data_subscriber_id")} AS subscriber_id,
+              #{key("s.data_target_type")} AS target_type,
+              #{key("s.data_target_id")} AS target_id,
+              s._sid AS source_sid
+            FROM (
+              SELECT source_row.*,
+                ROW_NUMBER() OVER (
+                  PARTITION BY source_row.id, source_row.context_user_id
+                  ORDER BY source_row._sid DESC
+                ) AS occurrence_rn
+              FROM `subscriptions` source_row
+            ) s
+            WHERE s.occurrence_rn = 1
+              AND #{present("s.id", "s.context_user_id", "s.data_subscriber_id", "s.data_target_type", "s.data_target_id")}
+          SQL
+          add_primary_key(table, %w[id context_user_id])
+          add_index(table, "idx_subscriber_id", %w[subscriber_id])
+        end
+
         def create_metadata
           statement(<<~SQL)
             CREATE TABLE `#{METADATA_TABLE}` (
               `profile` VARCHAR(64) NOT NULL,
               `profile_version` BIGINT NOT NULL,
               `tool_version` VARCHAR(64) NOT NULL,
+              `subscriptions_supplied` TINYINT(1) NOT NULL,
               PRIMARY KEY (`profile`)
             ) #{TABLE_OPTIONS}
           SQL
           statement(<<~SQL)
-            INSERT INTO `#{METADATA_TABLE}` (`profile`, `profile_version`, `tool_version`)
-            VALUES ('khoros_api_export', #{VERSION}, '#{StructuredDataToSql::VERSION.gsub("'", "''")}')
+            INSERT INTO `#{METADATA_TABLE}` (`profile`, `profile_version`, `tool_version`, `subscriptions_supplied`)
+            VALUES ('khoros_api_export', #{VERSION}, '#{StructuredDataToSql::VERSION.gsub("'", "''")}', #{@supplied_collections.include?("subscriptions") ? 1 : 0})
           SQL
         end
       end

data/lib/structured_data_to_sql/json_converter.rb

@@ -192,6 +192,8 @@ module StructuredDataToSql
       raise UsageError, "No JSON files found in #{source}" if files.empty?
 
       files_to_process, skipped = files.partition { |file| process_file?(file) }
+      @profile_supplied_collections =
+        files_to_process.map { |file| json_basename(file) }.to_set if @profile
       @stats[:files_skipped] += skipped.length
       if files_to_process.empty?
         raise UsageError,
@@ -693,7 +695,15 @@ module StructuredDataToSql
         ).each_record(
           counting,
           recover: @recover_truncated,
-          ndjson: @ndjson,
+          ndjson:
+            (
+              if @ndjson == :auto &&
+                   path.to_s.match?(/\.(?:ndjson|jsonl)(?:\.gz)?\z/)
+                true
+              else
+                @ndjson
+              end
+            ),
           &block
         )
       end
@@ -784,7 +794,10 @@ module StructuredDataToSql
           @profile_validator.warnings
         end
       warnings.each { |warning| emit_warning(warning) }
-      profile_class.new(@profile_tables).write(out)
+      profile_class.new(
+        @profile_tables,
+        supplied_collections: @profile_supplied_collections
+      ).write(out)
     end
 
     def write_profile_preamble(out)

data/lib/structured_data_to_sql/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module StructuredDataToSql
-  VERSION = "0.2.0"
+  VERSION = "0.3.0"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: structured_data_to_sql
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.3.0
 platform: ruby
 authors:
 - Discourse Team