e621_export_downloader 0.0.11 → 0.0.13

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 570f83543711b092f8e72709bcc1c3a085ae7b865618551cf6fc4e3a9c749ec6
-  data.tar.gz: 81af136bb9ed74d51284480bed0ae38c3dcc25865e503f19bc651ff6694ec174
+  metadata.gz: a133742764b3b599b2cfd4c3494391538730be19daae10e9df43956aa1913709
+  data.tar.gz: ec28fc8889a2ad4bef174f365985e89868068fe2f206b7325c65a3c881716942
 SHA512:
-  metadata.gz: b149322e32f32367c823377da011d5676227e7e4e454ff09eadabca03fdad57255bb72f5c10472a40e91594ec00e3bed8a47855dbc9deb7fd303c25a2fb941c3
-  data.tar.gz: 4edfafff035bd495ed6a9822b401853c5cc74c822cdea909403f051e8e03d55fd159d950d719023117c56fa48c9d40632e378d9dc464d554311d51992464f2c3
+  metadata.gz: 74262c9fc4b22df8847f75e80abb3ac9951fa7e265a306b168dcf087c64946849c2df5432d75e77a1ae8b0cc5d5869addb2348bd3b217abc7dcf02395e316521
+  data.tar.gz: 61dfc91a23e7c34c69562534f9a42583f362558391264d12c4761b3f2368b9633f8d0f356939efcb78a3a27d71d5a2cefbf8047d536526e02dc8d41ec2b04853

data/lib/e621/csv_importable.rb

@@ -1,21 +1,38 @@
 # frozen_string_literal: true
 # typed: true
 
+require("csv")
+
 module E621
   module CsvImportable
     extend(T::Sig)
 
-    sig { params(csv_path: String).returns(T.untyped) }
+    sig { returns(Integer) }
+    def row_count
+      E621::RowCount[T.unsafe(self).table_name.split(".").last]
+    end
+
+    sig { params(count: Integer).returns(T.untyped) }
+    def row_count=(count)
+      E621::RowCount.set(T.unsafe(self).table_name.split(".").last, count)
+    end
+
+    sig { params(csv_path: String).returns(Integer) }
     def import_from_csv(csv_path)
       model = T.unsafe(self)
-      headers = File.open(csv_path, "rb", &:readline).chomp
-      columns = headers.split(",").map { |h| model.connection.quote_column_name(h.strip) }.join(", ")
+      csv_headers = File.open(csv_path, "rb", &:readline).chomp.split(",").map(&:strip)
+      columns = csv_headers.map { |h| model.connection.quote_column_name(h) }.join(", ")
+
+      count = 0
       raw = model.connection.raw_connection
-      raw.copy_data("COPY #{model.quoted_table_name} (#{columns}) FROM STDIN WITH (FORMAT CSV, HEADER TRUE)") do
-        File.open(csv_path, "rb") do |f|
-          raw.put_copy_data(f.read(65_536)) until f.eof?
+      raw.copy_data("COPY #{model.quoted_table_name} (#{columns}) FROM STDIN WITH (FORMAT CSV)") do
+        CSV.foreach(csv_path, headers: true) do |row|
+          raw.put_copy_data(CSV.generate_line(T.cast(row, CSV::Row).fields))
+          count += 1
         end
       end
+      self.row_count = count
+      count
     end
   end
 end

data/lib/e621/post_version.rb

@@ -25,6 +25,7 @@ module E621
     private_class_method(def self.attributes_from_export(record)
       {
         id:                  record.id,
+        post_id:             record.post_id,
         added_locked_tags:   record.added_locked_tags,
         added_tags:          record.added_tags,
         description:         record.description,

data/lib/e621/row_count.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+# typed: true
+
+require("active_record")
+
+module E621
+  class RowCount < ActiveRecord::Base
+    extend(T::Sig)
+
+    self.table_name = "e621.row_counts"
+    self.primary_key = "table_name"
+    self.record_timestamps = false
+
+    sig { params(table_name: String).returns(Integer) }
+    def self.[](table_name)
+      find_by(table_name: table_name)&.count || 0
+    end
+
+    sig { params(table_name: String, count: Integer).returns(T.untyped) }
+    def self.set(table_name, count)
+      upsert({ table_name: table_name, count: count }, unique_by: :table_name)
+    end
+
+    sig { params(table_name: String, by: Integer).returns(T.untyped) }
+    def self.increment(table_name, by)
+      where(table_name: table_name).update_all("count = count + #{by.to_i}")
+    end
+  end
+end

data/lib/e621_export_downloader/active_record_models.rb

@@ -6,6 +6,7 @@ require_relative("../e621/pool")
 require_relative("../e621/post")
 require_relative("../e621/post_replacement")
 require_relative("../e621/post_version")
+require_relative("../e621/row_count")
 require_relative("../e621/tag")
 require_relative("../e621/tag_alias")
 require_relative("../e621/tag_implication")

data/lib/e621_export_downloader/models/post_version.rb

@@ -30,6 +30,9 @@ module E621ExportDownloader
       sig { returns(T.nilable(Integer)) }
       attr_reader(:parent_id)
 
+      sig { returns(Integer) }
+      attr_reader(:post_id)
+
       sig { returns(T.nilable(String)) }
       attr_reader(:rating)
 
@@ -76,6 +79,7 @@ module E621ExportDownloader
         @locked_tags         = T.let(T.must(record["locked_tags"]).empty? ? nil : record["locked_tags"], T.nilable(String))
         @parent_changed      = T.let(record["parent_changed"] == "t", T::Boolean)
         @parent_id           = T.let(T.must(record["parent_id"]).empty? ? nil : record["parent_id"].to_i, T.nilable(Integer))
+        @post_id             = T.let(record["post_id"].to_i, Integer)
         @rating              = T.let(T.must(record["rating"]).empty? ? nil : record["rating"], T.nilable(String))
         @rating_changed      = T.let(record["rating_changed"] == "t", T::Boolean)
         @reason              = T.let(T.must(record["reason"]).empty? ? nil : record["reason"], T.nilable(String))
@@ -102,6 +106,7 @@ module E621ExportDownloader
           locked_tags:         @locked_tags,
           parent_changed:      @parent_changed,
           parent_id:           @parent_id,
+          post_id:             @post_id,
           rating:              @rating,
           rating_changed:      @rating_changed,
           reason:              @reason,

data/lib/e621_export_downloader/version.rb

@@ -4,7 +4,7 @@
 # loaded by bundler
 module E621ExportDownloader
   module Constants
-    VERSION = "0.0.11"
+    VERSION = "0.0.13"
     WEBSITE = "https://github.com/DonovanDMC/E621ExportDownloader.rb"
   end
 end

data/lib/generators/e621_export_downloader/templates/migrations/create_e621_tables.rb.erb

@@ -2,6 +2,8 @@
 
 class CreateE621Tables < ActiveRecord::Migration<%= migration_version %>
   def change
+    enable_extension("pg_trgm")
+
 <% if use_schema? -%>
     reversible do |r|
       r.up { execute("CREATE SCHEMA <%= schema %>") }
@@ -15,22 +17,27 @@ class CreateE621Tables < ActiveRecord::Migration<%= migration_version %>
       t.string(:group_name)
       t.boolean(:is_active, null: false)
       t.boolean(:is_locked, null: false)
-      t.bigint(:linked_user_id)
+      t.bigint(:linked_user_id, index: true)
       t.string(:name, null: false, index: { unique: true })
       t.text(:other_names, array: true, null: false, default: [])
       t.datetime(:updated_at, null: false)
-      t.text(:urls, array: true, null: false, default: [])
+      t.text(:urls)
+      t.index("group_name gin_trgm_ops", using: :gin)
+      t.index("name gin_trgm_ops", using: :gin)
     end
 
     create_table(<%= table_ref("bulk_update_requests") %>, id: :bigint, default: nil, force: :cascade) do |t|
-      t.bigint(:approver_id)
+      t.bigint(:approver_id, index: true)
       t.datetime(:created_at, null: false)
-      t.bigint(:forum_topic_id)
+      t.bigint(:forum_topic_id, index: true)
+      t.bigint(:forum_post_id, index: true)
       t.text(:script, null: false)
       t.string(:status, null: false, index: true)
       t.string(:title)
       t.datetime(:updated_at, null: false)
       t.bigint(:user_id, null: false, index: true)
+      t.index("script gin_trgm_ops", using: :gin)
+      t.index("title gin_trgm_ops", using: :gin)
     end
 
     create_table(<%= table_ref("pools") %>, id: :bigint, default: nil, force: :cascade) do |t|
@@ -40,12 +47,14 @@ class CreateE621Tables < ActiveRecord::Migration<%= migration_version %>
       t.text(:description, null: false)
       t.boolean(:is_active, null: false)
       t.string(:name, null: false, index: true)
-      t.bigint(:post_ids, array: true, null: false, default: [])
+      t.bigint(:post_ids, array: true, null: false, default: [], index: { using: :gin })
       t.datetime(:updated_at)
+      t.index("description gin_trgm_ops", using: :gin)
+      t.index("name gin_trgm_ops", using: :gin)
     end
 
     create_table(<%= table_ref("posts") %>, id: :bigint, default: nil, force: :cascade) do |t|
-      t.bigint(:approver_id)
+      t.bigint(:approver_id, index: true)
       t.bigint(:change_seq, null: false)
       t.integer(:comment_count, null: false)
       t.datetime(:created_at, null: false)
@@ -58,59 +67,70 @@ class CreateE621Tables < ActiveRecord::Migration<%= migration_version %>
       t.integer(:image_height, null: false)
       t.integer(:image_width, null: false)
       t.boolean(:is_deleted, null: false, index: true)
-      t.boolean(:is_flagged, null: false)
-      t.boolean(:is_note_locked, null: false)
-      t.boolean(:is_pending, null: false)
-      t.boolean(:is_rating_locked, null: false)
-      t.boolean(:is_status_locked, null: false)
+      t.boolean(:is_flagged, null: false, index: true)
+      t.boolean(:is_note_locked, null: false, index: true)
+      t.boolean(:is_pending, null: false, index: true)
+      t.boolean(:is_rating_locked, null: false, index: true)
+      t.boolean(:is_status_locked, null: false, index: true)
       t.text(:locked_tags, null: false)
       t.string(:md5, index: { unique: true }, null: false)
-      t.bigint(:parent_id)
+      t.bigint(:parent_id, index: true)
       t.string(:rating, null: false, index: true)
       t.integer(:score, null: false)
-      t.text(:sources, array: true, null: false, default: [])
-      t.text(:tags, array: true, null: false, default: [])
+      t.text(:source, null: false)
+      t.text(:tag_string, null: false)
       t.integer(:up_score, null: false)
-      t.datetime(:updated_at)
+      t.datetime(:updated_at, index: true)
       t.bigint(:uploader_id, index: true)
+      t.index("description gin_trgm_ops", using: :gin)
+      t.index("source gin_trgm_ops", using: :gin)
+      t.index("tag_string gin_trgm_ops", using: :gin)
     end
 
     create_table(<%= table_ref("post_replacements") %>, id: :bigint, default: nil, force: :cascade) do |t|
-      t.bigint(:approver_id)
+      t.bigint(:approver_id, index: true)
       t.datetime(:created_at, null: false)
       t.bigint(:creator_id, null: false, index: true)
-      t.string(:file_ext, null: false)
-      t.string(:file_name, null: false)
-      t.bigint(:file_size, null: false)
-      t.integer(:image_height, null: false)
-      t.integer(:image_width, null: false)
-      t.string(:md5, null: false)
+      t.string(:file_ext, null: false, index: true)
+      t.string(:file_name, null: false, index: true)
+      t.bigint(:file_size, null: false, index: true)
+      t.integer(:image_height, null: false, index: true)
+      t.integer(:image_width, null: false, index: true)
+      t.string(:md5, null: false, index: true)
       t.bigint(:post_id, null: false, index: true)
       t.text(:reason, null: false)
       t.text(:source)
       t.string(:status, null: false, index: true)
       t.datetime(:updated_at, null: false)
+      t.index("reason gin_trgm_ops", using: :gin)
+      t.index("source gin_trgm_ops", using: :gin)
     end
 
     create_table(<%= table_ref("post_versions") %>, id: :bigint, default: nil, force: :cascade) do |t|
       t.text(:added_locked_tags, array: true, null: false, default: [])
       t.text(:added_tags, array: true, null: false, default: [])
       t.text(:description)
-      t.boolean(:description_changed, null: false)
+      t.boolean(:description_changed, null: false, index: true)
       t.text(:locked_tags)
-      t.boolean(:parent_changed, null: false)
+      t.boolean(:parent_changed, null: false, index: true)
       t.bigint(:parent_id)
+      t.bigint(:post_id, null: false, index: true)
       t.string(:rating)
-      t.boolean(:rating_changed, null: false)
+      t.boolean(:rating_changed, null: false, index: true)
       t.text(:reason)
       t.text(:removed_locked_tags, array: true, null: false, default: [])
       t.text(:removed_tags, array: true, null: false, default: [])
       t.text(:source)
-      t.boolean(:source_changed, null: false)
+      t.boolean(:source_changed, null: false, index: true)
       t.text(:tags)
       t.datetime(:updated_at, null: false, index: true)
-      t.bigint(:updater_id, null: false, index: true)
-      t.integer(:version, null: false)
+      t.bigint(:updater_id, index: true)
+      t.integer(:version, null: false, index: true)
+      t.index("description gin_trgm_ops", using: :gin)
+      t.index("locked_tags gin_trgm_ops", using: :gin)
+      t.index("reason gin_trgm_ops", using: :gin)
+      t.index("source gin_trgm_ops", using: :gin)
+      t.index("tags gin_trgm_ops", using: :gin)
     end
 
     create_table(<%= table_ref("tag_aliases") %>, id: :bigint, default: nil, force: :cascade) do |t|
@@ -118,6 +138,9 @@ class CreateE621Tables < ActiveRecord::Migration<%= migration_version %>
       t.string(:consequent_name, null: false, index: true)
       t.datetime(:created_at)
       t.string(:status, null: false, index: true)
+      t.index(%i[status antecedent_name])
+      t.index("antecedent_name gin_trgm_ops", using: :gin)
+      t.index("consequent_name gin_trgm_ops", using: :gin)
     end
 
     create_table(<%= table_ref("tag_implications") %>, id: :bigint, default: nil, force: :cascade) do |t|
@@ -125,22 +148,34 @@ class CreateE621Tables < ActiveRecord::Migration<%= migration_version %>
       t.string(:consequent_name, null: false, index: true)
       t.datetime(:created_at)
       t.string(:status, null: false, index: true)
+      t.index(%i[status consequent_name])
+      t.index(%i[status antecedent_name])
+      t.index("antecedent_name gin_trgm_ops", using: :gin)
+      t.index("consequent_name gin_trgm_ops", using: :gin)
     end
 
     create_table(<%= table_ref("tags") %>, id: :bigint, default: nil, force: :cascade) do |t|
       t.string(:category, null: false, index: true)
       t.string(:name, null: false, index: { unique: true })
-      t.integer(:post_count, null: false)
+      t.integer(:post_count, null: false, index: { order: { post_count: :desc } })
+      t.index(%i[category post_count], order: { post_count: :desc })
+      t.index("name gin_trgm_ops", using: :gin)
     end
 
     create_table(<%= table_ref("wiki_pages") %>, id: :bigint, default: nil, force: :cascade) do |t|
-      t.text(:body, null: false)
+      t.text(:body, null: false, index: true)
       t.datetime(:created_at, null: false)
-      t.bigint(:creator_id, index: true)
-      t.boolean(:is_locked, null: false)
+      t.bigint(:creator_id, null: false, index: true)
+      t.boolean(:is_locked, null: false, index: true)
       t.string(:title, null: false, index: { unique: true })
-      t.datetime(:updated_at)
-      t.bigint(:uploader_id)
+      t.datetime(:updated_at, null: false)
+      t.bigint(:updater_id, index: true)
+      t.index("body gin_trgm_ops", using: :gin)
+      t.index("title gin_trgm_ops", using: :gin)
+    end
+
+    create_table(<%= table_ref("row_counts") %>, primary_key: :table_name, id: :string, force: :cascade) do |t|
+      t.bigint(:count, null: false, default: 0)
     end
   end
 end

data/sorbet/rbi/shims/active_record.rbi

@@ -13,5 +13,14 @@ module ActiveRecord
 
     sig { params(attributes: T::Array[T::Hash[Symbol, T.untyped]], kwargs: T.untyped).returns(T.untyped) }
     def self.upsert_all(attributes, **kwargs); end
+
+    sig { params(value: T.untyped).void }
+    def self.primary_key=(value); end
+
+    sig { params(kwargs: T.untyped).returns(T.untyped) }
+    def self.find_by(**kwargs); end
+
+    sig { params(kwargs: T.untyped).returns(T.untyped) }
+    def self.where(**kwargs); end
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: e621_export_downloader
 version: !ruby/object:Gem::Version
-  version: 0.0.11
+  version: 0.0.13
 platform: ruby
 authors:
 - Donovan_DMC
@@ -86,6 +86,7 @@ files:
 - lib/e621/post.rb
 - lib/e621/post_replacement.rb
 - lib/e621/post_version.rb
+- lib/e621/row_count.rb
 - lib/e621/tag.rb
 - lib/e621/tag_alias.rb
 - lib/e621/tag_implication.rb