e621_export_downloader 0.0.7 → 0.0.9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 558ca84b3ec58a0d998ca3c6157e89689990ba3b4671464caecf0bcc88a8fcab
-  data.tar.gz: f22fa283a5363b9933cac89e21058bd99567491d1e86ab782d33dcffe829e5c0
+  metadata.gz: fa5f20a7d6eb9809d86762f16ece735222bf6cf4fe2010255acc5864dfa3a745
+  data.tar.gz: ac6cfb70f5f697cb8fb74548e7b5d05bf53dfc05777588c9f3f33f1a650c50ae
 SHA512:
-  metadata.gz: 3ffc4daf675019aa8ce063d6c6e331669287e18dc042e255071eb60f5f8017e51401b20e4d048bf65070a6ed4c54850349ce1b7642470f6a28e3c4a37f70f9f9
-  data.tar.gz: 997cb28c4fd865a66bdc642742c8f8130962dd24aacc5238bc7313cf06040639a564a76a5a2207160ddca13a8fd2a908e08ae0fe6a7f96c3e2a0db75071bfe8e
+  metadata.gz: 505c8121b28a88a3e0acd10b0feb9d4e304942acce0684063fec0b3effda2a162970dd1efeeafb4d5554384bcd8ebec16d5d0a2ed4fd080d26cd53c6572fdef9
+  data.tar.gz: f70902af72e584a6f834977039d488e1f3b451dc97fe15a755b05293f62bfb8d802e466cbfd030ebb33437ee2d47e74ba6faa98c62a4717f8bae0aa2548665bf

data/lib/e621/artist.rb

@@ -2,10 +2,12 @@
 # typed: true
 
 require("active_record")
+require_relative("csv_importable")
 
 module E621
   class Artist < ActiveRecord::Base
     extend(T::Sig)
+    extend(E621::CsvImportable)
 
     self.table_name = "e621.artists"
     self.record_timestamps = false

data/lib/e621/bulk_update_request.rb

@@ -2,10 +2,12 @@
 # typed: true
 
 require("active_record")
+require_relative("csv_importable")
 
 module E621
   class BulkUpdateRequest < ActiveRecord::Base
     extend(T::Sig)
+    extend(E621::CsvImportable)
 
     self.table_name = "e621.bulk_update_requests"
     self.record_timestamps = false

data/lib/e621/csv_importable.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+# typed: true
+
+module E621
+  module CsvImportable
+    extend(T::Sig)
+
+    sig { params(csv_path: String).returns(T.untyped) }
+    def import_from_csv(csv_path)
+      raw = T.unsafe(self).connection.raw_connection
+      raw.copy_data("COPY #{T.unsafe(self).quoted_table_name} FROM STDIN WITH (FORMAT CSV, HEADER TRUE)") do
+        File.open(csv_path, "rb") do |f|
+          raw.put_copy_data(f.read(65_536)) until f.eof?
+        end
+      end
+    end
+  end
+end

data/lib/e621/pool.rb

@@ -2,10 +2,12 @@
 # typed: true
 
 require("active_record")
+require_relative("csv_importable")
 
 module E621
   class Pool < ActiveRecord::Base
     extend(T::Sig)
+    extend(E621::CsvImportable)
 
     self.table_name = "e621.pools"
     self.record_timestamps = false

data/lib/e621/post.rb

@@ -2,10 +2,12 @@
 # typed: true
 
 require("active_record")
+require_relative("csv_importable")
 
 module E621
   class Post < ActiveRecord::Base
     extend(T::Sig)
+    extend(E621::CsvImportable)
 
     self.table_name = "e621.posts"
     self.record_timestamps = false

data/lib/e621/post_replacement.rb

@@ -2,10 +2,12 @@
 # typed: true
 
 require("active_record")
+require_relative("csv_importable")
 
 module E621
   class PostReplacement < ActiveRecord::Base
     extend(T::Sig)
+    extend(E621::CsvImportable)
 
     self.table_name = "e621.post_replacements"
     self.record_timestamps = false

data/lib/e621/post_version.rb

@@ -2,10 +2,12 @@
 # typed: true
 
 require("active_record")
+require_relative("csv_importable")
 
 module E621
   class PostVersion < ActiveRecord::Base
     extend(T::Sig)
+    extend(E621::CsvImportable)
 
     self.table_name = "e621.post_versions"
     self.record_timestamps = false

data/lib/e621/tag.rb

@@ -2,10 +2,12 @@
 # typed: true
 
 require("active_record")
+require_relative("csv_importable")
 
 module E621
   class Tag < ActiveRecord::Base
     extend(T::Sig)
+    extend(E621::CsvImportable)
 
     self.table_name = "e621.tags"
     self.record_timestamps = false

data/lib/e621/tag_alias.rb

@@ -2,10 +2,12 @@
 # typed: true
 
 require("active_record")
+require_relative("csv_importable")
 
 module E621
   class TagAlias < ActiveRecord::Base
     extend(T::Sig)
+    extend(E621::CsvImportable)
 
     self.table_name = "e621.tag_aliases"
     self.record_timestamps = false

data/lib/e621/tag_implication.rb

@@ -2,10 +2,12 @@
 # typed: true
 
 require("active_record")
+require_relative("csv_importable")
 
 module E621
   class TagImplication < ActiveRecord::Base
     extend(T::Sig)
+    extend(E621::CsvImportable)
 
     self.table_name = "e621.tag_implications"
     self.record_timestamps = false

data/lib/e621/wiki_page.rb

@@ -2,10 +2,12 @@
 # typed: true
 
 require("active_record")
+require_relative("csv_importable")
 
 module E621
   class WikiPage < ActiveRecord::Base
     extend(T::Sig)
+    extend(E621::CsvImportable)
 
     self.table_name = "e621.wiki_pages"
     self.record_timestamps = false

data/lib/e621_export_downloader/client/options.rb

@@ -23,16 +23,16 @@ module E621ExportDownloader
         sig { returns(Parsers) }
         def self.defaults
           Parsers.new(
-            artists:              ->(record) { Models::Artist.new(record) },
-            bulk_update_requests: ->(record) { Models::BulkUpdateRequest.new(record) },
-            pools:                ->(record) { Models::Pool.new(record) },
-            posts:                ->(record) { Models::Post.new(record) },
-            post_replacements:    ->(record) { Models::PostReplacement.new(record) },
-            post_versions:        ->(record) { Models::PostVersion.new(record) },
-            tag_aliases:          ->(record) { Models::TagAlias.new(record) },
-            tag_implications:     ->(record) { Models::TagImplication.new(record) },
-            tags:                 ->(record) { Models::Tag.new(record) },
-            wiki_pages:           ->(record) { Models::WikiPage.new(record) },
+            artists:              ->(record) { E621ExportDownloader::Models::Artist.new(record) },
+            bulk_update_requests: ->(record) { E621ExportDownloader::Models::BulkUpdateRequest.new(record) },
+            pools:                ->(record) { E621ExportDownloader::Models::Pool.new(record) },
+            posts:                ->(record) { E621ExportDownloader::Models::Post.new(record) },
+            post_replacements:    ->(record) { E621ExportDownloader::Models::PostReplacement.new(record) },
+            post_versions:        ->(record) { E621ExportDownloader::Models::PostVersion.new(record) },
+            tag_aliases:          ->(record) { E621ExportDownloader::Models::TagAlias.new(record) },
+            tag_implications:     ->(record) { E621ExportDownloader::Models::TagImplication.new(record) },
+            tags:                 ->(record) { E621ExportDownloader::Models::Tag.new(record) },
+            wiki_pages:           ->(record) { E621ExportDownloader::Models::WikiPage.new(record) },
           )
         end
       end

data/lib/e621_export_downloader/export.rb

@@ -90,7 +90,7 @@ module E621ExportDownloader
       download unless check_downloaded
       client.debug("reading export", header: ["export:#{type.serialize}"])
       total = line_count
-      CSV.foreach(file_path, headers: true) do |row|
+      CSV.foreach(file_path, headers: true, converters: ->(f) { f.nil? ? "" : f }) do |row|
         args = [@parser.call(T.cast(row, CSV::Row).to_hash), total]
         args = args.slice(0, block.arity) if block.arity != -1
         block.call(*T.unsafe(args))

data/lib/e621_export_downloader/version.rb

@@ -4,7 +4,7 @@
 # loaded by bundler
 module E621ExportDownloader
   module Constants
-    VERSION = "0.0.7"
+    VERSION = "0.0.9"
     WEBSITE = "https://github.com/DonovanDMC/E621ExportDownloader.rb"
   end
 end

data/lib/generators/e621_export_downloader/templates/migrations/create_e621_tables.rb.erb

@@ -5,7 +5,7 @@ class CreateE621Tables < ActiveRecord::Migration<%= migration_version %>
 <% if use_schema? -%>
     reversible do |r|
       r.up { execute("CREATE SCHEMA <%= schema %>") }
-      r.down { execute("DROP SCHEMA <%= schema %> CASCADE") }
+      r.down { execute("DROP SCHEMA <%= schema %>") }
     end
 
 <% end -%>

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: e621_export_downloader
 version: !ruby/object:Gem::Version
-  version: 0.0.7
+  version: 0.0.9
 platform: ruby
 authors:
 - Donovan_DMC
@@ -78,10 +78,10 @@ files:
 - LICENSE
 - README.md
 - Rakefile
-- db/migrate/create_e621_tables.rb
 - exe/e621-export-downloader
 - lib/e621/artist.rb
 - lib/e621/bulk_update_request.rb
+- lib/e621/csv_importable.rb
 - lib/e621/pool.rb
 - lib/e621/post.rb
 - lib/e621/post_replacement.rb

data/db/migrate/create_e621_tables.rb

@@ -1,179 +0,0 @@
-# frozen_string_literal: true
-
-class CreateE621Tables < ActiveRecord::Migration[7.1]
-  def change
-    reversible do |r|
-      r.up { execute("CREATE SCHEMA e621") }
-      r.down { execute("DROP SCHEMA e621 CASCADE") }
-    end
-
-    create_table(:e621_artists, id: :bigint, default: nil, force: :cascade) do |t|
-      t.datetime(:created_at, null: false)
-      t.bigint(:creator_id, null: false)
-      t.string(:group_name)
-      t.boolean(:is_active, null: false)
-      t.boolean(:is_locked, null: false)
-      t.bigint(:linked_user_id)
-      t.string(:name, null: false)
-      t.text(:other_names, array: true, null: false, default: [])
-      t.datetime(:updated_at, null: false)
-      t.text(:urls, array: true, null: false, default: [])
-    end
-
-    add_index(:e621_artists, :name, unique: true)
-    add_index(:e621_artists, :creator_id)
-
-    create_table(:e621_bulk_update_requests, id: :bigint, default: nil, force: :cascade) do |t|
-      t.bigint(:approver_id)
-      t.datetime(:created_at, null: false)
-      t.bigint(:forum_topic_id)
-      t.text(:script, null: false)
-      t.string(:status, null: false)
-      t.string(:title)
-      t.datetime(:updated_at, null: false)
-      t.bigint(:user_id, null: false)
-    end
-
-    add_index(:e621_bulk_update_requests, :user_id)
-    add_index(:e621_bulk_update_requests, :status)
-
-    create_table(:e621_pools, id: :bigint, default: nil, force: :cascade) do |t|
-      t.string(:category, null: false)
-      t.datetime(:created_at, null: false)
-      t.bigint(:creator_id, null: false)
-      t.text(:description, null: false)
-      t.boolean(:is_active, null: false)
-      t.string(:name, null: false)
-      t.bigint(:post_ids, array: true, null: false, default: [])
-      t.datetime(:updated_at)
-    end
-
-    add_index(:e621_pools, :creator_id)
-    add_index(:e621_pools, :name)
-
-    create_table(:e621_posts, id: :bigint, default: nil, force: :cascade) do |t|
-      t.bigint(:approver_id)
-      t.bigint(:change_seq, null: false)
-      t.integer(:comment_count, null: false)
-      t.datetime(:created_at, null: false)
-      t.text(:description, null: false)
-      t.integer(:down_score, null: false)
-      t.float(:duration)
-      t.integer(:fav_count, null: false)
-      t.string(:file_ext, null: false)
-      t.bigint(:file_size, null: false)
-      t.integer(:image_height, null: false)
-      t.integer(:image_width, null: false)
-      t.boolean(:is_deleted, null: false)
-      t.boolean(:is_flagged, null: false)
-      t.boolean(:is_note_locked, null: false)
-      t.boolean(:is_pending, null: false)
-      t.boolean(:is_rating_locked, null: false)
-      t.boolean(:is_status_locked, null: false)
-      t.text(:locked_tags, null: false)
-      t.string(:md5)
-      t.bigint(:parent_id)
-      t.string(:rating, null: false)
-      t.integer(:score, null: false)
-      t.text(:sources, array: true, null: false, default: [])
-      t.text(:tags, array: true, null: false, default: [])
-      t.integer(:up_score, null: false)
-      t.datetime(:updated_at)
-      t.bigint(:uploader_id)
-    end
-
-    add_index(:e621_posts, :md5, unique: true, where: "md5 IS NOT NULL")
-    add_index(:e621_posts, :uploader_id)
-    add_index(:e621_posts, :rating)
-    add_index(:e621_posts, :is_deleted)
-
-    create_table(:e621_post_replacements, id: :bigint, default: nil, force: :cascade) do |t|
-      t.bigint(:approver_id)
-      t.datetime(:created_at, null: false)
-      t.bigint(:creator_id, null: false)
-      t.string(:file_ext, null: false)
-      t.string(:file_name, null: false)
-      t.bigint(:file_size, null: false)
-      t.integer(:image_height, null: false)
-      t.integer(:image_width, null: false)
-      t.string(:md5, null: false)
-      t.bigint(:post_id, null: false)
-      t.text(:reason, null: false)
-      t.text(:source)
-      t.string(:status, null: false)
-      t.datetime(:updated_at, null: false)
-    end
-
-    add_index(:e621_post_replacements, :post_id)
-    add_index(:e621_post_replacements, :creator_id)
-    add_index(:e621_post_replacements, :status)
-
-    create_table(:e621_post_versions, id: :bigint, default: nil, force: :cascade) do |t|
-      t.text(:added_locked_tags, array: true, null: false, default: [])
-      t.text(:added_tags, array: true, null: false, default: [])
-      t.text(:description)
-      t.boolean(:description_changed, null: false)
-      t.text(:locked_tags)
-      t.boolean(:parent_changed, null: false)
-      t.bigint(:parent_id)
-      t.string(:rating)
-      t.boolean(:rating_changed, null: false)
-      t.text(:reason)
-      t.text(:removed_locked_tags, array: true, null: false, default: [])
-      t.text(:removed_tags, array: true, null: false, default: [])
-      t.text(:source)
-      t.boolean(:source_changed, null: false)
-      t.text(:tags)
-      t.datetime(:updated_at, null: false)
-      t.bigint(:updater_id, null: false)
-      t.integer(:version, null: false)
-    end
-
-    add_index(:e621_post_versions, :updater_id)
-    add_index(:e621_post_versions, :updated_at)
-
-    create_table(:e621_tag_aliases, id: :bigint, default: nil, force: :cascade) do |t|
-      t.string(:antecedent_name, null: false)
-      t.string(:consequent_name, null: false)
-      t.datetime(:created_at)
-      t.string(:status, null: false)
-    end
-
-    add_index(:e621_tag_aliases, :antecedent_name)
-    add_index(:e621_tag_aliases, :consequent_name)
-    add_index(:e621_tag_aliases, :status)
-
-    create_table(:e621_tag_implications, id: :bigint, default: nil, force: :cascade) do |t|
-      t.string(:antecedent_name, null: false)
-      t.string(:consequent_name, null: false)
-      t.datetime(:created_at)
-      t.string(:status, null: false)
-    end
-
-    add_index(:e621_tag_implications, :antecedent_name)
-    add_index(:e621_tag_implications, :consequent_name)
-    add_index(:e621_tag_implications, :status)
-
-    create_table(:e621_tags, id: :bigint, default: nil, force: :cascade) do |t|
-      t.string(:category, null: false)
-      t.string(:name, null: false)
-      t.integer(:post_count, null: false)
-    end
-
-    add_index(:e621_tags, :name, unique: true)
-    add_index(:e621_tags, :category)
-
-    create_table(:e621_wiki_pages, id: :bigint, default: nil, force: :cascade) do |t|
-      t.text(:body, null: false)
-      t.datetime(:created_at, null: false)
-      t.bigint(:creator_id)
-      t.boolean(:is_locked, null: false)
-      t.string(:title, null: false)
-      t.datetime(:updated_at)
-      t.bigint(:uploader_id)
-    end
-
-    add_index(:e621_wiki_pages, :title, unique: true)
-    add_index(:e621_wiki_pages, :creator_id)
-  end
-end