e621_export_downloader 0.0.12 → 0.0.13

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 71464892548f5aea175d642767ec9c081992971fa05cfa591786e30f05849fc2
-  data.tar.gz: 6dab31bc8938e5289981d952e2e674bf32e407e10f83d4fc5bef754174e21750
+  metadata.gz: a133742764b3b599b2cfd4c3494391538730be19daae10e9df43956aa1913709
+  data.tar.gz: ec28fc8889a2ad4bef174f365985e89868068fe2f206b7325c65a3c881716942
 SHA512:
-  metadata.gz: e5f44050f050ef991ac4ab7fecef628c665f449c46208a6d9e3734dbec95e9f088169adad6f3b81ab9133aef4afecce9f36109c25c7f007a3e2e8d2cb2d4807b
-  data.tar.gz: 453c7a2885eebc6681ec5954ae86430147f336806c2f34052cecf862cb6926626ec039df7a1f798b2f467e50a184883f4dc7c0bd28e3dc0fe37ca3d927ab57ad
+  metadata.gz: 74262c9fc4b22df8847f75e80abb3ac9951fa7e265a306b168dcf087c64946849c2df5432d75e77a1ae8b0cc5d5869addb2348bd3b217abc7dcf02395e316521
+  data.tar.gz: 61dfc91a23e7c34c69562534f9a42583f362558391264d12c4761b3f2368b9633f8d0f356939efcb78a3a27d71d5a2cefbf8047d536526e02dc8d41ec2b04853

data/lib/e621/csv_importable.rb

@@ -1,21 +1,38 @@
 # frozen_string_literal: true
 # typed: true
 
+require("csv")
+
 module E621
   module CsvImportable
     extend(T::Sig)
 
-    sig { params(csv_path: String).returns(T.untyped) }
+    sig { returns(Integer) }
+    def row_count
+      E621::RowCount[T.unsafe(self).table_name.split(".").last]
+    end
+
+    sig { params(count: Integer).returns(T.untyped) }
+    def row_count=(count)
+      E621::RowCount.set(T.unsafe(self).table_name.split(".").last, count)
+    end
+
+    sig { params(csv_path: String).returns(Integer) }
     def import_from_csv(csv_path)
       model = T.unsafe(self)
-      headers = File.open(csv_path, "rb", &:readline).chomp
-      columns = headers.split(",").map { |h| model.connection.quote_column_name(h.strip) }.join(", ")
+      csv_headers = File.open(csv_path, "rb", &:readline).chomp.split(",").map(&:strip)
+      columns = csv_headers.map { |h| model.connection.quote_column_name(h) }.join(", ")
+
+      count = 0
       raw = model.connection.raw_connection
-      raw.copy_data("COPY #{model.quoted_table_name} (#{columns}) FROM STDIN WITH (FORMAT CSV, HEADER TRUE)") do
-        File.open(csv_path, "rb") do |f|
-          raw.put_copy_data(f.read(65_536)) until f.eof?
+      raw.copy_data("COPY #{model.quoted_table_name} (#{columns}) FROM STDIN WITH (FORMAT CSV)") do
+        CSV.foreach(csv_path, headers: true) do |row|
+          raw.put_copy_data(CSV.generate_line(T.cast(row, CSV::Row).fields))
+          count += 1
         end
       end
+      self.row_count = count
+      count
     end
   end
 end

data/lib/e621/row_count.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+# typed: true
+
+require("active_record")
+
+module E621
+  class RowCount < ActiveRecord::Base
+    extend(T::Sig)
+
+    self.table_name = "e621.row_counts"
+    self.primary_key = "table_name"
+    self.record_timestamps = false
+
+    sig { params(table_name: String).returns(Integer) }
+    def self.[](table_name)
+      find_by(table_name: table_name)&.count || 0
+    end
+
+    sig { params(table_name: String, count: Integer).returns(T.untyped) }
+    def self.set(table_name, count)
+      upsert({ table_name: table_name, count: count }, unique_by: :table_name)
+    end
+
+    sig { params(table_name: String, by: Integer).returns(T.untyped) }
+    def self.increment(table_name, by)
+      where(table_name: table_name).update_all("count = count + #{by.to_i}")
+    end
+  end
+end

data/lib/e621_export_downloader/active_record_models.rb

@@ -6,6 +6,7 @@ require_relative("../e621/pool")
 require_relative("../e621/post")
 require_relative("../e621/post_replacement")
 require_relative("../e621/post_version")
+require_relative("../e621/row_count")
 require_relative("../e621/tag")
 require_relative("../e621/tag_alias")
 require_relative("../e621/tag_implication")

data/lib/e621_export_downloader/version.rb

@@ -4,7 +4,7 @@
 # loaded by bundler
 module E621ExportDownloader
   module Constants
-    VERSION = "0.0.12"
+    VERSION = "0.0.13"
     WEBSITE = "https://github.com/DonovanDMC/E621ExportDownloader.rb"
   end
 end

data/lib/generators/e621_export_downloader/templates/migrations/create_e621_tables.rb.erb

@@ -2,6 +2,8 @@
 
 class CreateE621Tables < ActiveRecord::Migration<%= migration_version %>
   def change
+    enable_extension("pg_trgm")
+
 <% if use_schema? -%>
     reversible do |r|
       r.up { execute("CREATE SCHEMA <%= schema %>") }
@@ -15,22 +17,27 @@ class CreateE621Tables < ActiveRecord::Migration<%= migration_version %>
       t.string(:group_name)
       t.boolean(:is_active, null: false)
       t.boolean(:is_locked, null: false)
-      t.bigint(:linked_user_id)
+      t.bigint(:linked_user_id, index: true)
       t.string(:name, null: false, index: { unique: true })
       t.text(:other_names, array: true, null: false, default: [])
       t.datetime(:updated_at, null: false)
-      t.text(:urls, null: false)
+      t.text(:urls)
+      t.index("group_name gin_trgm_ops", using: :gin)
+      t.index("name gin_trgm_ops", using: :gin)
     end
 
     create_table(<%= table_ref("bulk_update_requests") %>, id: :bigint, default: nil, force: :cascade) do |t|
-      t.bigint(:approver_id)
+      t.bigint(:approver_id, index: true)
       t.datetime(:created_at, null: false)
-      t.bigint(:forum_topic_id)
+      t.bigint(:forum_topic_id, index: true)
+      t.bigint(:forum_post_id, index: true)
       t.text(:script, null: false)
       t.string(:status, null: false, index: true)
       t.string(:title)
       t.datetime(:updated_at, null: false)
       t.bigint(:user_id, null: false, index: true)
+      t.index("script gin_trgm_ops", using: :gin)
+      t.index("title gin_trgm_ops", using: :gin)
     end
 
     create_table(<%= table_ref("pools") %>, id: :bigint, default: nil, force: :cascade) do |t|
@@ -40,12 +47,14 @@ class CreateE621Tables < ActiveRecord::Migration<%= migration_version %>
       t.text(:description, null: false)
       t.boolean(:is_active, null: false)
       t.string(:name, null: false, index: true)
-      t.bigint(:post_ids, array: true, null: false, default: [])
+      t.bigint(:post_ids, array: true, null: false, default: [], index: { using: :gin })
       t.datetime(:updated_at)
+      t.index("description gin_trgm_ops", using: :gin)
+      t.index("name gin_trgm_ops", using: :gin)
     end
 
     create_table(<%= table_ref("posts") %>, id: :bigint, default: nil, force: :cascade) do |t|
-      t.bigint(:approver_id)
+      t.bigint(:approver_id, index: true)
       t.bigint(:change_seq, null: false)
       t.integer(:comment_count, null: false)
       t.datetime(:created_at, null: false)
@@ -58,60 +67,70 @@ class CreateE621Tables < ActiveRecord::Migration<%= migration_version %>
       t.integer(:image_height, null: false)
       t.integer(:image_width, null: false)
       t.boolean(:is_deleted, null: false, index: true)
-      t.boolean(:is_flagged, null: false)
-      t.boolean(:is_note_locked, null: false)
-      t.boolean(:is_pending, null: false)
-      t.boolean(:is_rating_locked, null: false)
-      t.boolean(:is_status_locked, null: false)
+      t.boolean(:is_flagged, null: false, index: true)
+      t.boolean(:is_note_locked, null: false, index: true)
+      t.boolean(:is_pending, null: false, index: true)
+      t.boolean(:is_rating_locked, null: false, index: true)
+      t.boolean(:is_status_locked, null: false, index: true)
       t.text(:locked_tags, null: false)
       t.string(:md5, index: { unique: true }, null: false)
-      t.bigint(:parent_id)
+      t.bigint(:parent_id, index: true)
       t.string(:rating, null: false, index: true)
       t.integer(:score, null: false)
-      t.text(:sources, array: true, null: false, default: [])
-      t.text(:tags, array: true, null: false, default: [])
+      t.text(:source, null: false)
+      t.text(:tag_string, null: false)
       t.integer(:up_score, null: false)
-      t.datetime(:updated_at)
+      t.datetime(:updated_at, index: true)
       t.bigint(:uploader_id, index: true)
+      t.index("description gin_trgm_ops", using: :gin)
+      t.index("source gin_trgm_ops", using: :gin)
+      t.index("tag_string gin_trgm_ops", using: :gin)
     end
 
     create_table(<%= table_ref("post_replacements") %>, id: :bigint, default: nil, force: :cascade) do |t|
-      t.bigint(:approver_id)
+      t.bigint(:approver_id, index: true)
       t.datetime(:created_at, null: false)
       t.bigint(:creator_id, null: false, index: true)
-      t.string(:file_ext, null: false)
-      t.string(:file_name, null: false)
-      t.bigint(:file_size, null: false)
-      t.integer(:image_height, null: false)
-      t.integer(:image_width, null: false)
-      t.string(:md5, null: false)
+      t.string(:file_ext, null: false, index: true)
+      t.string(:file_name, null: false, index: true)
+      t.bigint(:file_size, null: false, index: true)
+      t.integer(:image_height, null: false, index: true)
+      t.integer(:image_width, null: false, index: true)
+      t.string(:md5, null: false, index: true)
       t.bigint(:post_id, null: false, index: true)
       t.text(:reason, null: false)
       t.text(:source)
       t.string(:status, null: false, index: true)
       t.datetime(:updated_at, null: false)
+      t.index("reason gin_trgm_ops", using: :gin)
+      t.index("source gin_trgm_ops", using: :gin)
     end
 
     create_table(<%= table_ref("post_versions") %>, id: :bigint, default: nil, force: :cascade) do |t|
       t.text(:added_locked_tags, array: true, null: false, default: [])
       t.text(:added_tags, array: true, null: false, default: [])
       t.text(:description)
-      t.boolean(:description_changed, null: false)
+      t.boolean(:description_changed, null: false, index: true)
       t.text(:locked_tags)
-      t.boolean(:parent_changed, null: false)
+      t.boolean(:parent_changed, null: false, index: true)
       t.bigint(:parent_id)
-      t.bigint(:post_id, null: false)
+      t.bigint(:post_id, null: false, index: true)
       t.string(:rating)
-      t.boolean(:rating_changed, null: false)
+      t.boolean(:rating_changed, null: false, index: true)
       t.text(:reason)
       t.text(:removed_locked_tags, array: true, null: false, default: [])
       t.text(:removed_tags, array: true, null: false, default: [])
       t.text(:source)
-      t.boolean(:source_changed, null: false)
+      t.boolean(:source_changed, null: false, index: true)
       t.text(:tags)
       t.datetime(:updated_at, null: false, index: true)
-      t.bigint(:updater_id, null: false, index: true)
-      t.integer(:version, null: false)
+      t.bigint(:updater_id, index: true)
+      t.integer(:version, null: false, index: true)
+      t.index("description gin_trgm_ops", using: :gin)
+      t.index("locked_tags gin_trgm_ops", using: :gin)
+      t.index("reason gin_trgm_ops", using: :gin)
+      t.index("source gin_trgm_ops", using: :gin)
+      t.index("tags gin_trgm_ops", using: :gin)
     end
 
     create_table(<%= table_ref("tag_aliases") %>, id: :bigint, default: nil, force: :cascade) do |t|
@@ -119,6 +138,9 @@ class CreateE621Tables < ActiveRecord::Migration<%= migration_version %>
       t.string(:consequent_name, null: false, index: true)
       t.datetime(:created_at)
       t.string(:status, null: false, index: true)
+      t.index(%i[status antecedent_name])
+      t.index("antecedent_name gin_trgm_ops", using: :gin)
+      t.index("consequent_name gin_trgm_ops", using: :gin)
     end
 
     create_table(<%= table_ref("tag_implications") %>, id: :bigint, default: nil, force: :cascade) do |t|
@@ -126,22 +148,34 @@ class CreateE621Tables < ActiveRecord::Migration<%= migration_version %>
       t.string(:consequent_name, null: false, index: true)
       t.datetime(:created_at)
       t.string(:status, null: false, index: true)
+      t.index(%i[status consequent_name])
+      t.index(%i[status antecedent_name])
+      t.index("antecedent_name gin_trgm_ops", using: :gin)
+      t.index("consequent_name gin_trgm_ops", using: :gin)
     end
 
     create_table(<%= table_ref("tags") %>, id: :bigint, default: nil, force: :cascade) do |t|
       t.string(:category, null: false, index: true)
       t.string(:name, null: false, index: { unique: true })
-      t.integer(:post_count, null: false)
+      t.integer(:post_count, null: false, index: { order: { post_count: :desc } })
+      t.index(%i[category post_count], order: { post_count: :desc })
+      t.index("name gin_trgm_ops", using: :gin)
     end
 
     create_table(<%= table_ref("wiki_pages") %>, id: :bigint, default: nil, force: :cascade) do |t|
-      t.text(:body, null: false)
+      t.text(:body, null: false, index: true)
       t.datetime(:created_at, null: false)
-      t.bigint(:creator_id, index: true)
-      t.boolean(:is_locked, null: false)
+      t.bigint(:creator_id, null: false, index: true)
+      t.boolean(:is_locked, null: false, index: true)
       t.string(:title, null: false, index: { unique: true })
-      t.datetime(:updated_at)
-      t.bigint(:uploader_id)
+      t.datetime(:updated_at, null: false)
+      t.bigint(:updater_id, index: true)
+      t.index("body gin_trgm_ops", using: :gin)
+      t.index("title gin_trgm_ops", using: :gin)
+    end
+
+    create_table(<%= table_ref("row_counts") %>, primary_key: :table_name, id: :string, force: :cascade) do |t|
+      t.bigint(:count, null: false, default: 0)
     end
   end
 end

data/sorbet/rbi/shims/active_record.rbi

@@ -13,5 +13,14 @@ module ActiveRecord
 
     sig { params(attributes: T::Array[T::Hash[Symbol, T.untyped]], kwargs: T.untyped).returns(T.untyped) }
     def self.upsert_all(attributes, **kwargs); end
+
+    sig { params(value: T.untyped).void }
+    def self.primary_key=(value); end
+
+    sig { params(kwargs: T.untyped).returns(T.untyped) }
+    def self.find_by(**kwargs); end
+
+    sig { params(kwargs: T.untyped).returns(T.untyped) }
+    def self.where(**kwargs); end
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: e621_export_downloader
 version: !ruby/object:Gem::Version
-  version: 0.0.12
+  version: 0.0.13
 platform: ruby
 authors:
 - Donovan_DMC
@@ -86,6 +86,7 @@ files:
 - lib/e621/post.rb
 - lib/e621/post_replacement.rb
 - lib/e621/post_version.rb
+- lib/e621/row_count.rb
 - lib/e621/tag.rb
 - lib/e621/tag_alias.rb
 - lib/e621/tag_implication.rb