structured_data_to_sql 0.2.0 → 0.2.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ff2d9500aa62ada917f7f65d973d3c9e5495506cfcc6cf06011328482fb50288
-  data.tar.gz: f5440b2010078d1b011106ba82c4256d44098501f91131706c80ad45d259de5f
+  metadata.gz: 9a50afe6c834dcd44e699032caa0cdc7edd95b6bb041ccdd24ff2a4e9baf95ea
+  data.tar.gz: 2ed49f9f4dedd7be4e163006c75cccaca849e8cab58a487526b00c3326f0c184
 SHA512:
-  metadata.gz: b443a51fdabf4dc975562fae2ed7e6cde39b616597cb640134dec3d470cc17f9ec27a46289c746b5ce3375d83861e86f0769e84266ad274f4cc7c95f52076b13
-  data.tar.gz: ebe2b6d53b667d71d8d8260eaaf39990cb56ffe6911117a9f2513e2da49da343b10fd1e848a2571a44d65bed1edc06fecd872ce6d962d0d34a800cbdd1ade0bf
+  metadata.gz: 69ec834c19d138f85dc1182d6b2b057b9e6e07dd52a4f6850312216e63c9567603dd0aa8cfd3b817a8e2c14b89c5bd6f5d770eb6f294db85c60cb3f160cfe0e7
+  data.tar.gz: 2f671632b40604564a48a55650cd4b07ffd95ddeb8e2046cb2972f37c89af283b9f0cf5b01a7cfd1e05d19b875183e48f225bff7752959f4bdae03f3820a217f

data/lib/structured_data_to_sql/diagnostics_report.rb

@@ -150,8 +150,10 @@ module StructuredDataToSql
         return lines
       end
 
-      lines << "Quality signals are scoped to the latest run whose manifest entry is `completed`, " \
-        "by attributing the last _N_ rows of the append-only errors/gaps files to it (_N_ = that run's declared count). " \
+      lines << "Quality signals are scoped to the latest content run whose manifest entry is `completed`, " \
+        "by attributing the last _N_ rows of the append-only errors/gaps files to it (_N_ = that run's declared count), " \
+        "less whatever later runs declared. A `structure_repair` run that follows it is reported on its own line; " \
+        "a later run that never finalized its entry leaves the breakdown unattributable and it is withheld. " \
         "Rows beyond the sum of declared counts belong to runs that never finalized their entry."
       lines << ""
       lines << "| Run | Status | Started | Finished | Mode | Credential | Emails | Errors | Gaps |"

data/lib/structured_data_to_sql/json/exporter_manifest.rb

@@ -26,6 +26,23 @@ module StructuredDataToSql
           def completed?
             status == "completed"
           end
+
+          # A structure repair appends nodes and its own error/gap rows to an
+          # existing export; it is a run, but not the content export the rows
+          # were converted from.
+          def repair?
+            mode == "structure_repair"
+          end
+
+          def content?
+            !repair?
+          end
+
+          # An entry the exporter closed, whatever the outcome. A run killed
+          # outright stays `running` with no counts, so its rows are undeclared.
+          def finalized?
+            status != "running" && !finished_at.nil?
+          end
         end
 
       FILENAME = "manifest.json"
@@ -64,8 +81,24 @@ module StructuredDataToSql
         parse
       end
 
+      # The latest completed content run: what the quality signals are scoped to.
       def latest_completed
-        @runs.reverse.find(&:completed?)
+        @runs.reverse.find { |run| run.completed? && run.content? }
+      end
+
+      def latest_completed_repair
+        @runs.reverse.find { |run| run.completed? && run.repair? }
+      end
+
+      # Rows declared by the runs after +run+. The files are append-only, so a
+      # run's rows end where the next run's begin.
+      def rows_after(run, counter)
+        @runs.drop(run.index + 1).sum(&counter)
+      end
+
+      # Later runs whose rows are not declared, so their position is unknown.
+      def unfinalized_after(run)
+        @runs.drop(run.index + 1).reject(&:finalized?)
       end
 
       def errors_total

data/lib/structured_data_to_sql/json/profiles/khoros_api_export.rb

@@ -718,7 +718,7 @@ module StructuredDataToSql
             latest = manifest.latest_completed
             if latest.nil?
               return [
-                "Latest run: not scoped — no completed run in manifest.json; showing lifetime totals"
+                "Latest run: not scoped — no completed content run in manifest.json; showing lifetime totals"
               ]
             end
 
@@ -730,22 +730,77 @@ module StructuredDataToSql
               ]
             end
 
-            latest_count = latest.public_send(counter)
-            latest_rows = @quality_signal_rows[table].last(latest_count)
-            latest_tallies = Hash.new(0)
-            latest_rows.each { |label| latest_tallies[label] += 1 }
-            window = [latest.started_at, latest.finished_at].compact.join(" → ")
+            unattributed = total - declared
             lines = [
-              "Latest completed run ##{latest.index}#{window.empty? ? "" : " (#{window})"}: " \
-                "#{Format.format_count(latest_count)}#{latest_count.positive? ? " — #{breakdown_text(latest_tallies)}" : ""}"
+              attributed_line(
+                "Latest completed run",
+                latest,
+                manifest,
+                table,
+                counter,
+                total,
+                unattributed
+              )
             ]
-            unattributed = total - declared
+            repair = manifest.latest_completed_repair
+            if repair && repair.index > latest.index
+              lines << attributed_line(
+                "Latest structure repair",
+                repair,
+                manifest,
+                table,
+                counter,
+                total,
+                unattributed
+              )
+            end
             if unattributed.positive?
               lines << "Unattributed: #{Format.format_count(unattributed)} row(s) written by runs that did not finalize their manifest entry"
             end
             lines
           end
 
+          # A run's rows sit before whatever later runs appended, so its slice
+          # ends where theirs begin — provided every later run declared its
+          # rows. Undeclared rows that may sit after this run's make the slice a
+          # guess, and a guess is not shown as a breakdown.
+          def attributed_line(
+            label,
+            run,
+            manifest,
+            table,
+            counter,
+            total,
+            unattributed
+          )
+            count = run.public_send(counter)
+            window = [run.started_at, run.finished_at].compact.join(" → ")
+            head =
+              "#{label} ##{run.index}#{window.empty? ? "" : " (#{window})"}: " \
+                "#{Format.format_count(count)}"
+            undeclared =
+              unattributed.positive? ? manifest.unfinalized_after(run) : []
+            unless undeclared.empty?
+              blamed =
+                undeclared
+                  .map { |later| "##{later.index} (#{later.status})" }
+                  .join(", ")
+              return(
+                "#{head} — breakdown not attributable: run #{blamed} did not finalize its " \
+                  "entry, so undeclared rows may sit after this run's"
+              )
+            end
+            return head unless count.positive?
+
+            after = manifest.rows_after(run, counter)
+            tallies = Hash.new(0)
+            @quality_signal_rows[table]
+              .first(total - after)
+              .last(count)
+              .each { |row_label| tallies[row_label] += 1 }
+            "#{head} — #{breakdown_text(tallies)}"
+          end
+
           def breakdown_text(tallies)
             tallies
               .sort_by { |label, count| [-count, label] }

data/lib/structured_data_to_sql/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module StructuredDataToSql
-  VERSION = "0.2.0"
+  VERSION = "0.2.1"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: structured_data_to_sql
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.2.1
 platform: ruby
 authors:
 - Discourse Team