structured_data_to_sql 0.1.0 → 0.2.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 0ccb8d7b174afed7679305fa1e49d6892bbe7000d3816e01694aad2390773bc3
-  data.tar.gz: 522028f6c1f0935a9e58e52730c46c7682863fd3431b2ffd78bca9f74f6a6718
+  metadata.gz: 9a50afe6c834dcd44e699032caa0cdc7edd95b6bb041ccdd24ff2a4e9baf95ea
+  data.tar.gz: 2ed49f9f4dedd7be4e163006c75cccaca849e8cab58a487526b00c3326f0c184
 SHA512:
-  metadata.gz: 9c3c997b1c817e2536deb6bdd0a86c945101b95a6d46f37cd8dbd875b3f1010f6f7538fe631edbb999fc56c105a632634fd67dd8fcccb761c3777b8eccd8ea4f
-  data.tar.gz: ca3f01ad1ac74c4720e4991f720f377c533076f1d6b54c0d4f2f0e2b3797e67e668e6fff26e249bae28a5c3b5df507b9c9b988fcda39ca8f49fe23644589f8bc
+  metadata.gz: 69ec834c19d138f85dc1182d6b2b057b9e6e07dd52a4f6850312216e63c9567603dd0aa8cfd3b817a8e2c14b89c5bd6f5d770eb6f294db85c60cb3f160cfe0e7
+  data.tar.gz: 2f671632b40604564a48a55650cd4b07ffd95ddeb8e2046cb2972f37c89af283b9f0cf5b01a7cfd1e05d19b875183e48f225bff7752959f4bdae03f3820a217f

data/lib/structured_data_to_sql/diagnostics_report.rb

@@ -150,8 +150,10 @@ module StructuredDataToSql
         return lines
       end
 
-      lines << "Quality signals are scoped to the latest run whose manifest entry is `completed`, " \
-        "by attributing the last _N_ rows of the append-only errors/gaps files to it (_N_ = that run's declared count). " \
+      lines << "Quality signals are scoped to the latest content run whose manifest entry is `completed`, " \
+        "by attributing the last _N_ rows of the append-only errors/gaps files to it (_N_ = that run's declared count), " \
+        "less whatever later runs declared. A `structure_repair` run that follows it is reported on its own line; " \
+        "a later run that never finalized its entry leaves the breakdown unattributable and it is withheld. " \
         "Rows beyond the sum of declared counts belong to runs that never finalized their entry."
       lines << ""
       lines << "| Run | Status | Started | Finished | Mode | Credential | Emails | Errors | Gaps |"

data/lib/structured_data_to_sql/json/exporter_manifest.rb

@@ -26,6 +26,23 @@ module StructuredDataToSql
           def completed?
             status == "completed"
           end
+
+          # A structure repair appends nodes and its own error/gap rows to an
+          # existing export; it is a run, but not the content export the rows
+          # were converted from.
+          def repair?
+            mode == "structure_repair"
+          end
+
+          def content?
+            !repair?
+          end
+
+          # An entry the exporter closed, whatever the outcome. A run killed
+          # outright stays `running` with no counts, so its rows are undeclared.
+          def finalized?
+            status != "running" && !finished_at.nil?
+          end
         end
 
       FILENAME = "manifest.json"
@@ -64,8 +81,24 @@ module StructuredDataToSql
         parse
       end
 
+      # The latest completed content run: what the quality signals are scoped to.
       def latest_completed
-        @runs.reverse.find(&:completed?)
+        @runs.reverse.find { |run| run.completed? && run.content? }
+      end
+
+      def latest_completed_repair
+        @runs.reverse.find { |run| run.completed? && run.repair? }
+      end
+
+      # Rows declared by the runs after +run+. The files are append-only, so a
+      # run's rows end where the next run's begin.
+      def rows_after(run, counter)
+        @runs.drop(run.index + 1).sum(&counter)
+      end
+
+      # Later runs whose rows are not declared, so their position is unknown.
+      def unfinalized_after(run)
+        @runs.drop(run.index + 1).reject(&:finalized?)
       end
 
       def errors_total

data/lib/structured_data_to_sql/json/profiles/khoros_api_export.rb

@@ -17,8 +17,8 @@ module StructuredDataToSql
         NAME = :khoros_api_export
         # Bumped in lockstep with PROFILE_VERSION in the Khoros converter's
         # KhorosApiExportContentStaging whenever a canonical table's shape or
-        # semantics change. Version 2 adds normalized root-message read-only state.
-        VERSION = 2
+        # semantics change. Version 3 adds source-message view counts.
+        VERSION = 3
         PREFIX = "discourse_khoros_api"
         METADATA_TABLE = "_structured_data_to_sql_profiles"
         # Explicit storage options on every canonical table so they never
@@ -37,6 +37,7 @@ module StructuredDataToSql
             data_body
             data_conversation_id
             data_depth
+            data_metrics_views
             data_post_time
             data_read_only
             data_subject
@@ -717,7 +718,7 @@ module StructuredDataToSql
             latest = manifest.latest_completed
             if latest.nil?
               return [
-                "Latest run: not scoped — no completed run in manifest.json; showing lifetime totals"
+                "Latest run: not scoped — no completed content run in manifest.json; showing lifetime totals"
               ]
             end
 
@@ -729,22 +730,77 @@ module StructuredDataToSql
               ]
             end
 
-            latest_count = latest.public_send(counter)
-            latest_rows = @quality_signal_rows[table].last(latest_count)
-            latest_tallies = Hash.new(0)
-            latest_rows.each { |label| latest_tallies[label] += 1 }
-            window = [latest.started_at, latest.finished_at].compact.join(" → ")
+            unattributed = total - declared
             lines = [
-              "Latest completed run ##{latest.index}#{window.empty? ? "" : " (#{window})"}: " \
-                "#{Format.format_count(latest_count)}#{latest_count.positive? ? " — #{breakdown_text(latest_tallies)}" : ""}"
+              attributed_line(
+                "Latest completed run",
+                latest,
+                manifest,
+                table,
+                counter,
+                total,
+                unattributed
+              )
             ]
-            unattributed = total - declared
+            repair = manifest.latest_completed_repair
+            if repair && repair.index > latest.index
+              lines << attributed_line(
+                "Latest structure repair",
+                repair,
+                manifest,
+                table,
+                counter,
+                total,
+                unattributed
+              )
+            end
             if unattributed.positive?
               lines << "Unattributed: #{Format.format_count(unattributed)} row(s) written by runs that did not finalize their manifest entry"
             end
             lines
           end
 
+          # A run's rows sit before whatever later runs appended, so its slice
+          # ends where theirs begin — provided every later run declared its
+          # rows. Undeclared rows that may sit after this run's make the slice a
+          # guess, and a guess is not shown as a breakdown.
+          def attributed_line(
+            label,
+            run,
+            manifest,
+            table,
+            counter,
+            total,
+            unattributed
+          )
+            count = run.public_send(counter)
+            window = [run.started_at, run.finished_at].compact.join(" → ")
+            head =
+              "#{label} ##{run.index}#{window.empty? ? "" : " (#{window})"}: " \
+                "#{Format.format_count(count)}"
+            undeclared =
+              unattributed.positive? ? manifest.unfinalized_after(run) : []
+            unless undeclared.empty?
+              blamed =
+                undeclared
+                  .map { |later| "##{later.index} (#{later.status})" }
+                  .join(", ")
+              return(
+                "#{head} — breakdown not attributable: run #{blamed} did not finalize its " \
+                  "entry, so undeclared rows may sit after this run's"
+              )
+            end
+            return head unless count.positive?
+
+            after = manifest.rows_after(run, counter)
+            tallies = Hash.new(0)
+            @quality_signal_rows[table]
+              .first(total - after)
+              .last(count)
+              .each { |row_label| tallies[row_label] += 1 }
+            "#{head} — #{breakdown_text(tallies)}"
+          end
+
           def breakdown_text(tallies)
             tallies
               .sort_by { |label, count| [-count, label] }
@@ -1927,6 +1983,7 @@ module StructuredDataToSql
               m.data_subject AS subject,
               m.data_body AS body,
               m.data_post_time AS post_time,
+              m.data_metrics_views AS views,
               #{boolean_value("m.data_read_only")} AS read_only,
               COALESCE(#{optional("messages", "data_is_solution", "0", table_alias: "m")}, 0) AS is_solution,
               #{optional("messages", "data_solution_data_message_id", table_alias: "m")} AS solution_message_id,

data/lib/structured_data_to_sql/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module StructuredDataToSql
-  VERSION = "0.1.0"
+  VERSION = "0.2.1"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: structured_data_to_sql
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.1
 platform: ruby
 authors:
 - Discourse Team