structured_data_to_sql 0.2.1 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9a50afe6c834dcd44e699032caa0cdc7edd95b6bb041ccdd24ff2a4e9baf95ea
-  data.tar.gz: 2ed49f9f4dedd7be4e163006c75cccaca849e8cab58a487526b00c3326f0c184
+  metadata.gz: 0c7a2d7a571374df01b77f059c243d1ddf43ab4ac51fe85f21c3b5a8ba7d1ed4
+  data.tar.gz: bca5b0b6e07ff5c7de2c254bcfa336b27f9ef35c949a6b1e3db4618a61820030
 SHA512:
-  metadata.gz: 69ec834c19d138f85dc1182d6b2b057b9e6e07dd52a4f6850312216e63c9567603dd0aa8cfd3b817a8e2c14b89c5bd6f5d770eb6f294db85c60cb3f160cfe0e7
-  data.tar.gz: 2f671632b40604564a48a55650cd4b07ffd95ddeb8e2046cb2972f37c89af283b9f0cf5b01a7cfd1e05d19b875183e48f225bff7752959f4bdae03f3820a217f
+  metadata.gz: 7673f7ac81093a5c3e533b300e71f422a1026ac6c814270f400720f571b09d6918b6badffe99c7c8419983071a68cba51aac951bf7c1348c78c9a48c6afc8624
+  data.tar.gz: de4c881ae651cb0ff5ef6e9973a44d3e90d9763a6ea7521e34a7174c661bd361bb30c0aa97333ff387aa6dd1d2428a8f9958e12d4de8b75ea6ec9d1007bbbd5d

data/lib/structured_data_to_sql/json/profiles/khoros_api_export.rb

@@ -17,8 +17,8 @@ module StructuredDataToSql
         NAME = :khoros_api_export
         # Bumped in lockstep with PROFILE_VERSION in the Khoros converter's
         # KhorosApiExportContentStaging whenever a canonical table's shape or
-        # semantics change. Version 3 adds source-message view counts.
-        VERSION = 3
+        # semantics change. Version 4 adds optional API subscriptions.
+        VERSION = 4
         PREFIX = "discourse_khoros_api"
         METADATA_TABLE = "_structured_data_to_sql_profiles"
         # Explicit storage options on every canonical table so they never
@@ -62,6 +62,7 @@ module StructuredDataToSql
             attachments
             files
             source_messages
+            subscriptions
           ].map { |name| "#{PREFIX}_#{name}" }.freeze
         USER_STRING_FIELDS = {
           id: %w[data_id],
@@ -222,6 +223,21 @@ module StructuredDataToSql
             }
           }.freeze
           OPTIONAL = {
+            "subscriptions" => {
+              required: {
+                subscription_id: %w[id],
+                context_user_id: %w[context_user_id],
+                subscriber_id: %w[data_subscriber_id],
+                target_type: %w[data_target_type],
+                target_id: %w[data_target_id]
+              },
+              indexed: {
+                subscription_id: %w[id],
+                context_user_id: %w[context_user_id],
+                subscriber_id: %w[data_subscriber_id],
+                target_id: %w[data_target_id]
+              }
+            },
             "user_profiles" => {
               required: {
                 user_id: %w[id]
@@ -933,9 +949,10 @@ module StructuredDataToSql
           out.write("DROP TABLE IF EXISTS `#{METADATA_TABLE}`;\n")
         end
 
-        def initialize(table_columns)
+        def initialize(table_columns, supplied_collections: [])
           @table_columns =
             table_columns.transform_values { |value| Array(value).map(&:to_s) }
+          @supplied_collections = supplied_collections
         end
 
         def write(out)
@@ -960,6 +977,7 @@ module StructuredDataToSql
           create_attachments
           create_files
           create_source_messages
+          create_subscriptions
           create_metadata
         ensure
           @out = nil
@@ -1997,18 +2015,70 @@ module StructuredDataToSql
           add_index(table, "idx_board_id", %w[board_id])
         end
 
+        def create_subscriptions
+          table = "#{PREFIX}_subscriptions"
+          unless canonical_source?(
+                   "subscriptions",
+                   "context_user_id",
+                   "data_subscriber_id",
+                   "data_target_type",
+                   "data_target_id"
+                 )
+            create_empty(
+              table,
+              [
+                "`id` VARCHAR(255) NOT NULL",
+                "`context_user_id` VARCHAR(255) NOT NULL",
+                "`subscriber_id` VARCHAR(255) NOT NULL",
+                "`target_type` VARCHAR(255) NOT NULL",
+                "`target_id` VARCHAR(255) NOT NULL",
+                "`source_sid` BIGINT NOT NULL"
+              ],
+              indexes: [
+                "PRIMARY KEY (`id`, `context_user_id`)",
+                "KEY `idx_subscriber_id` (`subscriber_id`)"
+              ]
+            )
+            return
+          end
+
+          statement(<<~SQL)
+            CREATE TABLE `#{table}` #{TABLE_OPTIONS} AS
+            SELECT
+              #{key("s.id")} AS id,
+              #{key("s.context_user_id")} AS context_user_id,
+              #{key("s.data_subscriber_id")} AS subscriber_id,
+              #{key("s.data_target_type")} AS target_type,
+              #{key("s.data_target_id")} AS target_id,
+              s._sid AS source_sid
+            FROM (
+              SELECT source_row.*,
+                ROW_NUMBER() OVER (
+                  PARTITION BY source_row.id, source_row.context_user_id
+                  ORDER BY source_row._sid DESC
+                ) AS occurrence_rn
+              FROM `subscriptions` source_row
+            ) s
+            WHERE s.occurrence_rn = 1
+              AND #{present("s.id", "s.context_user_id", "s.data_subscriber_id", "s.data_target_type", "s.data_target_id")}
+          SQL
+          add_primary_key(table, %w[id context_user_id])
+          add_index(table, "idx_subscriber_id", %w[subscriber_id])
+        end
+
         def create_metadata
           statement(<<~SQL)
             CREATE TABLE `#{METADATA_TABLE}` (
               `profile` VARCHAR(64) NOT NULL,
               `profile_version` BIGINT NOT NULL,
               `tool_version` VARCHAR(64) NOT NULL,
+              `subscriptions_supplied` TINYINT(1) NOT NULL,
               PRIMARY KEY (`profile`)
             ) #{TABLE_OPTIONS}
           SQL
           statement(<<~SQL)
-            INSERT INTO `#{METADATA_TABLE}` (`profile`, `profile_version`, `tool_version`)
-            VALUES ('khoros_api_export', #{VERSION}, '#{StructuredDataToSql::VERSION.gsub("'", "''")}')
+            INSERT INTO `#{METADATA_TABLE}` (`profile`, `profile_version`, `tool_version`, `subscriptions_supplied`)
+            VALUES ('khoros_api_export', #{VERSION}, '#{StructuredDataToSql::VERSION.gsub("'", "''")}', #{@supplied_collections.include?("subscriptions") ? 1 : 0})
           SQL
         end
       end

data/lib/structured_data_to_sql/json_converter.rb

@@ -192,6 +192,8 @@ module StructuredDataToSql
       raise UsageError, "No JSON files found in #{source}" if files.empty?
 
       files_to_process, skipped = files.partition { |file| process_file?(file) }
+      @profile_supplied_collections =
+        files_to_process.map { |file| json_basename(file) }.to_set if @profile
       @stats[:files_skipped] += skipped.length
       if files_to_process.empty?
         raise UsageError,
@@ -693,7 +695,15 @@ module StructuredDataToSql
         ).each_record(
           counting,
           recover: @recover_truncated,
-          ndjson: @ndjson,
+          ndjson:
+            (
+              if @ndjson == :auto &&
+                   path.to_s.match?(/\.(?:ndjson|jsonl)(?:\.gz)?\z/)
+                true
+              else
+                @ndjson
+              end
+            ),
           &block
         )
       end
@@ -784,7 +794,10 @@ module StructuredDataToSql
           @profile_validator.warnings
         end
       warnings.each { |warning| emit_warning(warning) }
-      profile_class.new(@profile_tables).write(out)
+      profile_class.new(
+        @profile_tables,
+        supplied_collections: @profile_supplied_collections
+      ).write(out)
     end
 
     def write_profile_preamble(out)

data/lib/structured_data_to_sql/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module StructuredDataToSql
-  VERSION = "0.2.1"
+  VERSION = "0.3.0"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: structured_data_to_sql
 version: !ruby/object:Gem::Version
-  version: 0.2.1
+  version: 0.3.0
 platform: ruby
 authors:
 - Discourse Team