e621_export_downloader 0.0.12 → 0.0.14

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 71464892548f5aea175d642767ec9c081992971fa05cfa591786e30f05849fc2
-  data.tar.gz: 6dab31bc8938e5289981d952e2e674bf32e407e10f83d4fc5bef754174e21750
+  metadata.gz: 89d525162aa1da43623f66a8bab4618ac17910c2c3e0db7a272e8149be91efb8
+  data.tar.gz: 6152f2a56d9ad995dd5868f2762d4a95c38e2b4d95a289b8806ef74870b50ea7
 SHA512:
-  metadata.gz: e5f44050f050ef991ac4ab7fecef628c665f449c46208a6d9e3734dbec95e9f088169adad6f3b81ab9133aef4afecce9f36109c25c7f007a3e2e8d2cb2d4807b
-  data.tar.gz: 453c7a2885eebc6681ec5954ae86430147f336806c2f34052cecf862cb6926626ec039df7a1f798b2f467e50a184883f4dc7c0bd28e3dc0fe37ca3d927ab57ad
+  metadata.gz: 494ccb09c9727311ec327cdc8926816bd0957f9d0a9455b6b5789364b30d4728d95763a99707b5861fb1d858092acaed1c220abd97dfb22f3f66e9de05fc6eaf
+  data.tar.gz: a0dbdb4a693365f88b5986ffcdef1a23190a3c6ce85e98297d344bf74b91bf4794700d5f0cdf4ab087cb695e0017f6464d21dfe5013419a29930d438e1786f43

data/lib/e621/csv_importable.rb

@@ -5,17 +5,94 @@ module E621
   module CsvImportable
     extend(T::Sig)
 
-    sig { params(csv_path: String).returns(T.untyped) }
-    def import_from_csv(csv_path)
+    sig { returns(Integer) }
+    def row_count
+      E621::RowCount[T.unsafe(self).table_name.split(".").last]
+    end
+
+    sig { params(count: Integer).returns(T.untyped) }
+    def row_count=(count)
+      E621::RowCount.set(T.unsafe(self).table_name.split(".").last, count)
+    end
+
+    # Loads a CSV export into this model's table via PostgreSQL COPY.
+    #
+    # The file's raw bytes are streamed straight into COPY's native CSV parser
+    # (only the header line is read, for the column list) rather than parsing
+    # each row into a CSV::Row and re-serializing it in Ruby. That avoids a
+    # full single-threaded parse + re-encode of every row and preserves the
+    # exact bytes of fields containing embedded newlines/quotes. The row count
+    # is taken from COPY itself (PG::Result#cmd_tuples).
+    #
+    # truncate:         empty the table before loading (full reload).
+    # recreate_indexes: drop every secondary (non-PK, non-constraint) index
+    #                   before the COPY and rebuild it afterward. Building the
+    #                   GIN/btree indexes once over the finished table is far
+    #                   cheaper than maintaining them row-by-row during COPY,
+    #                   and CREATE INDEX can use parallel workers.
+    #
+    # Physical-storage policy (UNLOGGED) and session tuning
+    # (maintenance_work_mem, max_parallel_maintenance_workers, ...) are the
+    # caller's responsibility — set them around this call.
+    sig do
+      params(
+        csv_path:         String,
+        truncate:         T::Boolean,
+        recreate_indexes: T::Boolean,
+        chunk_bytes:      Integer,
+      ).returns(Integer)
+    end
+    def import_from_csv(csv_path, truncate: false, recreate_indexes: false, chunk_bytes: 1 << 20)
+      model = T.unsafe(self)
+      indexes = recreate_indexes ? secondary_index_definitions : {}
+
+      model.connection.execute("TRUNCATE #{model.quoted_table_name}") if truncate
+      indexes.each_key { |name| model.connection.execute("DROP INDEX IF EXISTS #{name}") }
+
+      count = copy_csv(csv_path, chunk_bytes: chunk_bytes)
+
+      indexes.each_value { |ddl| model.connection.execute(ddl) }
+      self.row_count = count
+      count
+    end
+
+    private
+
+    sig { params(csv_path: String, chunk_bytes: Integer).returns(Integer) }
+    def copy_csv(csv_path, chunk_bytes:)
       model = T.unsafe(self)
-      headers = File.open(csv_path, "rb", &:readline).chomp
-      columns = headers.split(",").map { |h| model.connection.quote_column_name(h.strip) }.join(", ")
+      header = File.open(csv_path, "rb", &:readline)
+      columns = header.chomp.split(",").map { |h| model.connection.quote_column_name(h.strip) }.join(", ")
+
       raw = model.connection.raw_connection
-      raw.copy_data("COPY #{model.quoted_table_name} (#{columns}) FROM STDIN WITH (FORMAT CSV, HEADER TRUE)") do
-        File.open(csv_path, "rb") do |f|
-          raw.put_copy_data(f.read(65_536)) until f.eof?
+      result = raw.copy_data("COPY #{model.quoted_table_name} (#{columns}) FROM STDIN WITH (FORMAT CSV)") do
+        File.open(csv_path, "rb") do |io|
+          io.readline # skip header row
+          while (chunk = io.read(chunk_bytes))
+            raw.put_copy_data(chunk)
+          end
         end
       end
+      result.cmd_tuples
+    end
+
+    # Every non-primary-key, non-constraint-backed index on the table, paired
+    # with the DDL needed to recreate it.
+    sig { returns(T::Hash[String, String]) }
+    def secondary_index_definitions
+      model = T.unsafe(self)
+      conn = model.connection
+      sql = <<~SQL
+        SELECT i.indexrelid::regclass::text AS name,
+               pg_get_indexdef(i.indexrelid)  AS ddl
+        FROM pg_index i
+        WHERE i.indrelid = #{conn.quote(model.quoted_table_name)}::regclass
+          AND NOT i.indisprimary
+          AND NOT EXISTS (
+            SELECT 1 FROM pg_constraint c WHERE c.conindid = i.indexrelid
+          )
+      SQL
+      conn.exec_query(sql).to_h { |r| [r["name"], r["ddl"]] }
     end
   end
 end

data/lib/e621/row_count.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+# typed: true
+
+require("active_record")
+
+module E621
+  class RowCount < ActiveRecord::Base
+    extend(T::Sig)
+
+    self.table_name = "e621.row_counts"
+    self.primary_key = "table_name"
+    self.record_timestamps = false
+
+    sig { params(table_name: String).returns(Integer) }
+    def self.[](table_name)
+      find_by(table_name: table_name)&.count || 0
+    end
+
+    sig { params(table_name: String, count: Integer).returns(T.untyped) }
+    def self.set(table_name, count)
+      upsert({ table_name: table_name, count: count }, unique_by: :table_name)
+    end
+
+    sig { params(table_name: String, by: Integer).returns(T.untyped) }
+    def self.increment(table_name, by)
+      where(table_name: table_name).update_all("count = count + #{by.to_i}")
+    end
+  end
+end

data/lib/e621_export_downloader/active_record_models.rb

@@ -6,6 +6,7 @@ require_relative("../e621/pool")
 require_relative("../e621/post")
 require_relative("../e621/post_replacement")
 require_relative("../e621/post_version")
+require_relative("../e621/row_count")
 require_relative("../e621/tag")
 require_relative("../e621/tag_alias")
 require_relative("../e621/tag_implication")

data/lib/e621_export_downloader/version.rb

@@ -4,7 +4,7 @@
 # loaded by bundler
 module E621ExportDownloader
   module Constants
-    VERSION = "0.0.12"
+    VERSION = "0.0.14"
     WEBSITE = "https://github.com/DonovanDMC/E621ExportDownloader.rb"
   end
 end

data/lib/generators/e621_export_downloader/templates/migrations/create_e621_tables.rb.erb

@@ -2,6 +2,8 @@
 
 class CreateE621Tables < ActiveRecord::Migration<%= migration_version %>
   def change
+    enable_extension("pg_trgm")
+
 <% if use_schema? -%>
     reversible do |r|
       r.up { execute("CREATE SCHEMA <%= schema %>") }
@@ -15,22 +17,27 @@ class CreateE621Tables < ActiveRecord::Migration<%= migration_version %>
       t.string(:group_name)
       t.boolean(:is_active, null: false)
       t.boolean(:is_locked, null: false)
-      t.bigint(:linked_user_id)
+      t.bigint(:linked_user_id, index: true)
       t.string(:name, null: false, index: { unique: true })
       t.text(:other_names, array: true, null: false, default: [])
       t.datetime(:updated_at, null: false)
-      t.text(:urls, null: false)
+      t.text(:urls)
+      t.index("group_name gin_trgm_ops", using: :gin)
+      t.index("name gin_trgm_ops", using: :gin)
     end
 
     create_table(<%= table_ref("bulk_update_requests") %>, id: :bigint, default: nil, force: :cascade) do |t|
-      t.bigint(:approver_id)
+      t.bigint(:approver_id, index: true)
       t.datetime(:created_at, null: false)
-      t.bigint(:forum_topic_id)
+      t.bigint(:forum_topic_id, index: true)
+      t.bigint(:forum_post_id, index: true)
       t.text(:script, null: false)
       t.string(:status, null: false, index: true)
       t.string(:title)
       t.datetime(:updated_at, null: false)
       t.bigint(:user_id, null: false, index: true)
+      t.index("script gin_trgm_ops", using: :gin)
+      t.index("title gin_trgm_ops", using: :gin)
     end
 
     create_table(<%= table_ref("pools") %>, id: :bigint, default: nil, force: :cascade) do |t|
@@ -40,12 +47,14 @@ class CreateE621Tables < ActiveRecord::Migration<%= migration_version %>
       t.text(:description, null: false)
       t.boolean(:is_active, null: false)
       t.string(:name, null: false, index: true)
-      t.bigint(:post_ids, array: true, null: false, default: [])
+      t.bigint(:post_ids, array: true, null: false, default: [], index: { using: :gin })
       t.datetime(:updated_at)
+      t.index("description gin_trgm_ops", using: :gin)
+      t.index("name gin_trgm_ops", using: :gin)
     end
 
     create_table(<%= table_ref("posts") %>, id: :bigint, default: nil, force: :cascade) do |t|
-      t.bigint(:approver_id)
+      t.bigint(:approver_id, index: true)
       t.bigint(:change_seq, null: false)
       t.integer(:comment_count, null: false)
       t.datetime(:created_at, null: false)
@@ -58,60 +67,70 @@ class CreateE621Tables < ActiveRecord::Migration<%= migration_version %>
       t.integer(:image_height, null: false)
       t.integer(:image_width, null: false)
       t.boolean(:is_deleted, null: false, index: true)
-      t.boolean(:is_flagged, null: false)
-      t.boolean(:is_note_locked, null: false)
-      t.boolean(:is_pending, null: false)
-      t.boolean(:is_rating_locked, null: false)
-      t.boolean(:is_status_locked, null: false)
+      t.boolean(:is_flagged, null: false, index: true)
+      t.boolean(:is_note_locked, null: false, index: true)
+      t.boolean(:is_pending, null: false, index: true)
+      t.boolean(:is_rating_locked, null: false, index: true)
+      t.boolean(:is_status_locked, null: false, index: true)
       t.text(:locked_tags, null: false)
       t.string(:md5, index: { unique: true }, null: false)
-      t.bigint(:parent_id)
+      t.bigint(:parent_id, index: true)
       t.string(:rating, null: false, index: true)
       t.integer(:score, null: false)
-      t.text(:sources, array: true, null: false, default: [])
-      t.text(:tags, array: true, null: false, default: [])
+      t.text(:source, null: false)
+      t.text(:tag_string, null: false)
       t.integer(:up_score, null: false)
-      t.datetime(:updated_at)
+      t.datetime(:updated_at, index: true)
       t.bigint(:uploader_id, index: true)
+      t.index("description gin_trgm_ops", using: :gin)
+      t.index("source gin_trgm_ops", using: :gin)
+      t.index("tag_string gin_trgm_ops", using: :gin)
     end
 
     create_table(<%= table_ref("post_replacements") %>, id: :bigint, default: nil, force: :cascade) do |t|
-      t.bigint(:approver_id)
+      t.bigint(:approver_id, index: true)
       t.datetime(:created_at, null: false)
       t.bigint(:creator_id, null: false, index: true)
-      t.string(:file_ext, null: false)
-      t.string(:file_name, null: false)
-      t.bigint(:file_size, null: false)
-      t.integer(:image_height, null: false)
-      t.integer(:image_width, null: false)
-      t.string(:md5, null: false)
+      t.string(:file_ext, null: false, index: true)
+      t.string(:file_name, null: false, index: true)
+      t.bigint(:file_size, null: false, index: true)
+      t.integer(:image_height, null: false, index: true)
+      t.integer(:image_width, null: false, index: true)
+      t.string(:md5, null: false, index: true)
       t.bigint(:post_id, null: false, index: true)
       t.text(:reason, null: false)
       t.text(:source)
       t.string(:status, null: false, index: true)
       t.datetime(:updated_at, null: false)
+      t.index("reason gin_trgm_ops", using: :gin)
+      t.index("source gin_trgm_ops", using: :gin)
     end
 
     create_table(<%= table_ref("post_versions") %>, id: :bigint, default: nil, force: :cascade) do |t|
       t.text(:added_locked_tags, array: true, null: false, default: [])
       t.text(:added_tags, array: true, null: false, default: [])
       t.text(:description)
-      t.boolean(:description_changed, null: false)
+      t.boolean(:description_changed, null: false, index: true)
       t.text(:locked_tags)
-      t.boolean(:parent_changed, null: false)
+      t.boolean(:parent_changed, null: false, index: true)
       t.bigint(:parent_id)
-      t.bigint(:post_id, null: false)
+      t.bigint(:post_id, null: false, index: true)
       t.string(:rating)
-      t.boolean(:rating_changed, null: false)
+      t.boolean(:rating_changed, null: false, index: true)
       t.text(:reason)
       t.text(:removed_locked_tags, array: true, null: false, default: [])
       t.text(:removed_tags, array: true, null: false, default: [])
       t.text(:source)
-      t.boolean(:source_changed, null: false)
+      t.boolean(:source_changed, null: false, index: true)
       t.text(:tags)
       t.datetime(:updated_at, null: false, index: true)
-      t.bigint(:updater_id, null: false, index: true)
-      t.integer(:version, null: false)
+      t.bigint(:updater_id, index: true)
+      t.integer(:version, null: false, index: true)
+      t.index("description gin_trgm_ops", using: :gin)
+      t.index("locked_tags gin_trgm_ops", using: :gin)
+      t.index("reason gin_trgm_ops", using: :gin)
+      t.index("source gin_trgm_ops", using: :gin)
+      t.index("tags gin_trgm_ops", using: :gin)
     end
 
     create_table(<%= table_ref("tag_aliases") %>, id: :bigint, default: nil, force: :cascade) do |t|
@@ -119,6 +138,9 @@ class CreateE621Tables < ActiveRecord::Migration<%= migration_version %>
       t.string(:consequent_name, null: false, index: true)
       t.datetime(:created_at)
       t.string(:status, null: false, index: true)
+      t.index(%i[status antecedent_name])
+      t.index("antecedent_name gin_trgm_ops", using: :gin)
+      t.index("consequent_name gin_trgm_ops", using: :gin)
     end
 
     create_table(<%= table_ref("tag_implications") %>, id: :bigint, default: nil, force: :cascade) do |t|
@@ -126,22 +148,34 @@ class CreateE621Tables < ActiveRecord::Migration<%= migration_version %>
       t.string(:consequent_name, null: false, index: true)
       t.datetime(:created_at)
       t.string(:status, null: false, index: true)
+      t.index(%i[status consequent_name])
+      t.index(%i[status antecedent_name])
+      t.index("antecedent_name gin_trgm_ops", using: :gin)
+      t.index("consequent_name gin_trgm_ops", using: :gin)
     end
 
     create_table(<%= table_ref("tags") %>, id: :bigint, default: nil, force: :cascade) do |t|
       t.string(:category, null: false, index: true)
       t.string(:name, null: false, index: { unique: true })
-      t.integer(:post_count, null: false)
+      t.integer(:post_count, null: false, index: { order: { post_count: :desc } })
+      t.index(%i[category post_count], order: { post_count: :desc })
+      t.index("name gin_trgm_ops", using: :gin)
     end
 
     create_table(<%= table_ref("wiki_pages") %>, id: :bigint, default: nil, force: :cascade) do |t|
-      t.text(:body, null: false)
+      t.text(:body, null: false, index: true)
       t.datetime(:created_at, null: false)
-      t.bigint(:creator_id, index: true)
-      t.boolean(:is_locked, null: false)
+      t.bigint(:creator_id, null: false, index: true)
+      t.boolean(:is_locked, null: false, index: true)
       t.string(:title, null: false, index: { unique: true })
-      t.datetime(:updated_at)
-      t.bigint(:uploader_id)
+      t.datetime(:updated_at, null: false)
+      t.bigint(:updater_id, index: true)
+      t.index("body gin_trgm_ops", using: :gin)
+      t.index("title gin_trgm_ops", using: :gin)
+    end
+
+    create_table(<%= table_ref("row_counts") %>, primary_key: :table_name, id: :string, force: :cascade) do |t|
+      t.bigint(:count, null: false, default: 0)
     end
   end
 end

data/sorbet/rbi/shims/active_record.rbi

@@ -13,5 +13,14 @@ module ActiveRecord
 
     sig { params(attributes: T::Array[T::Hash[Symbol, T.untyped]], kwargs: T.untyped).returns(T.untyped) }
     def self.upsert_all(attributes, **kwargs); end
+
+    sig { params(value: T.untyped).void }
+    def self.primary_key=(value); end
+
+    sig { params(kwargs: T.untyped).returns(T.untyped) }
+    def self.find_by(**kwargs); end
+
+    sig { params(kwargs: T.untyped).returns(T.untyped) }
+    def self.where(**kwargs); end
   end
 end

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: e621_export_downloader
 version: !ruby/object:Gem::Version
-  version: 0.0.12
+  version: 0.0.14
 platform: ruby
 authors:
 - Donovan_DMC
 bindir: exe
 cert_chain: []
-date: 2026-06-13 00:00:00.000000000 Z
+date: 2026-06-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: csv
@@ -86,6 +86,7 @@ files:
 - lib/e621/post.rb
 - lib/e621/post_replacement.rb
 - lib/e621/post_version.rb
+- lib/e621/row_count.rb
 - lib/e621/tag.rb
 - lib/e621/tag_alias.rb
 - lib/e621/tag_implication.rb