e621_export_downloader 0.0.16 → 0.0.18

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 0ca5e1aa72caa7ce565b4887222579025187c23a13f86ec2b818367991ee6cc5
-  data.tar.gz: 393166c4a129496f7ecdd409899b644750b98a623d6501ca72c1f937d0645b62
+  metadata.gz: 7259f8077bf1df8805620f4efa3fbeac43c29616e496bea576518275794c5fef
+  data.tar.gz: 4b23216298e535aeed4ceaeb7f38f581bf6fc6fb68c1f7fa94975e9157f7bf71
 SHA512:
-  metadata.gz: 6a2ea533d91b0f8dc403d9f4a287880815e1c955d76b28897cb80b55593c570524099760dfa37f8924c80e340cc83800955da0a5e14a042e270a9bea64f8047c
-  data.tar.gz: 7a38db5401d422e48ae51470a5363bd5bfee65c4e2f0a6095e2fb3803123ee38a7dee90afaf4904f3f3390f593f33b6064ec7f99fcb94ba79f8ea5e6d21dde25
+  metadata.gz: 50436457d2de9e91e8f938631484653a71d8102122ec58f77c8c6287454092c38ef6719215a4aa20cb676978d659ce936750338667d892045027b4ee0a0fa0ed
+  data.tar.gz: b8a0c19646cd5b7ed2076b023adabc016962700dcc4c755ef2b66d3f3b018349be45aafca1c4be1dc8c0be9c8f2dd6de5f1529f6ea27f45a04df5660d6f6f1a9

data/CHANGELOG.md

@@ -0,0 +1,85 @@
+# Changelog
+
+## [0.0.18] - 2026-07-02
+
+_0.0.18_
+
+## [0.0.17] - 2026-06-29
+
+_0.0.17_
+
+## [0.0.16] - 2026-06-24
+
+### Added
+- Add new dbexport fields (from: e621ng/e621ng#2181, pending: e621ng/e621ng#2222)
+
+## [0.0.15] - 2026-06-14
+
+### Added
+- Add method to track download progress
+
+## [0.0.14] - 2026-06-14
+
+### Changed
+- optimize csv import
+
+## [0.0.13] - 2026-06-13
+
+### Added
+- Add stored row count & text indexes
+
+## [0.0.12] - 2026-06-12
+
+### Fixed
+- Fix post versions missing post_id
+
+## [0.0.11] - 2026-06-12
+
+### Fixed
+- Fix csv import
+
+## [0.0.10] - 2026-06-12
+
+### Fixed
+- Fix models
+
+## [0.0.9] - 2026-06-12
+
+### Fixed
+- Fix parsing
+
+## [0.0.8] - 2026-06-12
+
+### Added
+- Add csv import function to models
+
+## [0.0.7] - 2026-06-12
+
+### Fixed
+- Fix railtie
+
+## [0.0.6] - 2026-06-12
+
+_Rails Migration & models_
+
+## [0.0.5] - 2026-06-12
+
+_force republish_
+
+## [0.0.4] - 2026-06-12
+
+_db export changes_
+
+## [0.0.3] - 2026-05-19
+
+### Added
+- Add activejob serializer
+
+## [0.0.2] - 2026-05-05
+
+### Fixed
+- Fix Types serialization
+
+## [0.0.1] - 2026-05-05
+
+_Initial Commit_

data/README.md

@@ -23,12 +23,13 @@ client = E621ExportDownloader::Client.new
 
 # configure options after creation
 client.config do |c|
-  c.cache = true  # keep export files after reading, defaults to false
+  c.cache = true              # keep export files after reading, defaults to false
+  c.validate_checksum = true  # verify the downloaded file against the API-provided checksum, defaults to false
 end
 
 # or pass an Options struct directly
 client = E621ExportDownloader::Client.new(
-  E621ExportDownloader::Client::Options.new(cache: true)
+  E621ExportDownloader::Client::Options.new(cache: true, validate_checksum: true)
 )
 
 # get the export for a type — resolves the latest available export from the e621 API
@@ -39,10 +40,10 @@ export = client.get("posts")
 deferred = client.get_deferred("posts")
 
 # get raw metadata for all exports from the e621 API
-# returns an Array of E621ExportDownloader::APIExportData: { file_name, file_size, name, updated_at, url }
+# returns an Array of E621ExportDownloader::APIExportData: { file_name, file_size, name, checksum, updated_at, url }
 data = client.get_data
 
-# access the metadata for this specific export: { file_name, file_size, name, updated_at, url }
+# access the metadata for this specific export: { file_name, file_size, name, checksum, updated_at, url }
 export.data
 
 # check if the export exists
@@ -50,6 +51,7 @@ exists = export.exists?
 
 # download the export, returns the file path — not required before reading
 # if you move or remove the file do not reuse the export object
+# raises ResolveError if validate_checksum is enabled and the downloaded file's checksum does not match the API
 file = export.download
 
 # delete the downloaded file, if it exists

data/Rakefile

@@ -8,11 +8,21 @@ Rake::TestTask.new(:test) do |t|
   t.pattern = "test/**/*_test.rb"
 end
 
-namespace(:test) do
-  desc("Test Zeitwerk Auto-Importing")
-  task(:zeitwerk) do
-    exec(%(bundle exec ruby -e "require 'e621_export_downloader'; Zeitwerk::Loader.eager_load_all"))
-  end
+desc("Test Zeitwerk Auto-Importing")
+task(:zeitwerk) do
+  ruby(%(-e "require 'e621_export_downloader'; Zeitwerk::Loader.eager_load_all"))
 end
 
-task(default: :rubocop)
+desc("Run all tests")
+task(:all) do
+  puts("=== rubocop ===")
+  sh("bundle exec rubocop")
+  puts("=== zeitwerk ===")
+  sh("bundle exec rake zeitwerk")
+  puts("=== test ===")
+  sh("bundle exec rake test")
+  puts("=== srb ===")
+  sh("bundle exec srb tc")
+end
+
+task(default: :all)

data/lib/e621/tag_alias.rb

@@ -29,7 +29,7 @@ module E621
         approver_id:     record.approver_id,
         consequent_name: record.consequent_name,
         created_at:      record.created_at,
-        # creator_id:      record.creator_id,
+        creator_id:      record.creator_id,
         down_votes:      record.down_votes,
         forum_post_id:   record.forum_post_id,
         forum_topic_id:  record.forum_topic_id,

data/lib/e621/tag_implication.rb

@@ -29,7 +29,7 @@ module E621
         approver_id:      record.approver_id,
         consequent_name:  record.consequent_name,
         created_at:       record.created_at,
-        # creator_id:       record.creator_id,
+        creator_id:       record.creator_id,
         descendant_names: record.descendant_names,
         down_votes:       record.down_votes,
         forum_post_id:    record.forum_post_id,

data/lib/e621_export_downloader/api_export_data.rb

@@ -3,9 +3,10 @@
 
 module E621ExportDownloader
   class APIExportData < T::Struct
+    const(:name, Types)
     const(:file_name, String)
     const(:file_size, Integer)
-    const(:name, Types)
+    const(:checksum, String)
     const(:updated_at, String)
     const(:url, String)
   end

data/lib/e621_export_downloader/client/options/builder.rb

@@ -10,13 +10,17 @@ module E621ExportDownloader
         sig { returns(T::Boolean) }
         attr_accessor(:cache)
 
+        sig { returns(T::Boolean) }
+        attr_accessor(:validate_checksum)
+
         sig { params(parsers: Options::Parsers).returns(Options::Parsers) }
         attr_writer(:parsers)
 
         sig { params(defaults: Options).void }
         def initialize(defaults = Options.new)
-          @cache = T.let(defaults.cache, T::Boolean)
-          @parsers = T.let(defaults.parsers, Options::Parsers)
+          @cache             = T.let(defaults.cache, T::Boolean)
+          @validate_checksum = T.let(defaults.validate_checksum, T::Boolean)
+          @parsers           = T.let(defaults.parsers, Options::Parsers)
         end
 
         sig { params(block: T.nilable(T.proc.params(arg0: Builder::Parsers).void)).returns(Options::Parsers) }

data/lib/e621_export_downloader/client/options.rb

@@ -38,6 +38,7 @@ module E621ExportDownloader
       end
 
       const(:cache, T::Boolean, default: false)
+      const(:validate_checksum, T::Boolean, default: false)
       const(:parsers, Parsers, default: Parsers.defaults)
     end
   end

data/lib/e621_export_downloader/client.rb

@@ -21,7 +21,9 @@ module E621ExportDownloader
 
     sig { params(block: T.proc.params(arg0: Options::Builder).void).void }
     def config(&block)
-      block.call(Options::Builder.new(@options))
+      builder = Options::Builder.new(@options)
+      block.call(builder)
+      @options = Options.new(cache: builder.cache, validate_checksum: builder.validate_checksum, parsers: builder.parsers)
     end
 
     sig { returns(Faraday::Connection) }
@@ -61,9 +63,10 @@ module E621ExportDownloader
       data = JSON.parse(T.unsafe(res).body)
       result = T.let(data.map do |d|
         APIExportData.new(
+          name:       Types.deserialize(d["name"]),
           file_name:  d["file_name"],
           file_size:  d["file_size"].to_i,
-          name:       Types.deserialize(d["name"]),
+          checksum:   d["checksum"],
           updated_at: d["updated_at"],
           url:        d["url"],
         )

data/lib/e621_export_downloader/export.rb

@@ -2,6 +2,10 @@
 # typed: strict
 
 require("csv")
+require("date")
+require("digest")
+require("fileutils")
+require("zlib")
 
 module E621ExportDownloader
   class Export
@@ -51,11 +55,13 @@ module E621ExportDownloader
       client.debug("downloading export", header: ["export:#{type.serialize}"])
 
       FileUtils.mkdir_p(Constants::TEMP_DIR)
+      digest = T.let(client.options.validate_checksum ? Digest::MD5.new : nil, T.nilable(Digest::MD5))
       File.open(file_path, "wb") do |file|
         inflater = Zlib::Inflate.new(Zlib::MAX_WBITS + 16)
 
         res = client.connection.get(data.url) do |req|
           req.options.on_data = proc do |chunk, received|
+            digest&.update(chunk)
             decompressed = inflater.inflate(chunk)
             file.write(decompressed) if decompressed && !decompressed.empty?
             block.call(received, data.file_size) if block
@@ -69,6 +75,10 @@ module E621ExportDownloader
         unless res.success?
           raise(ResolveError, "Failed to download export #{type.serialize}: #{res.status} #{res.reason_phrase}")
         end
+
+        if digest && digest.hexdigest != data.checksum
+          raise(ResolveError, "Checksum mismatch for export #{type.serialize}")
+        end
       rescue # rubocop:disable Style/RescueStandardError
         FileUtils.rm_f(file_path)
         raise

data/lib/e621_export_downloader/models/tag_alias.rb

@@ -18,8 +18,8 @@ module E621ExportDownloader
       sig { returns(T.nilable(DateTime)) }
       attr_reader(:created_at)
 
-      # sig { returns(T.nilable(Integer)) }
-      # attr_reader(:creator_id)
+      sig { returns(T.nilable(Integer)) }
+      attr_reader(:creator_id)
 
       sig { returns(Integer) }
       attr_reader(:down_votes)
@@ -58,7 +58,7 @@ module E621ExportDownloader
         @approver_id     = T.let(T.must(record["approver_id"]).empty? ? nil : record["approver_id"].to_i, T.nilable(Integer))
         @consequent_name = T.let(T.must(record["consequent_name"]), String)
         @created_at      = T.let(T.must(record["created_at"]).empty? ? nil : DateTime.parse(record["created_at"]), T.nilable(DateTime))
-        # @creator_id      = T.let(T.must(record["creator_id"]).empty? ? nil : record["creator_id"].to_i, T.nilable(Integer))
+        @creator_id      = T.let(T.must(record["creator_id"]).empty? ? nil : record["creator_id"].to_i, T.nilable(Integer))
         @down_votes      = T.let(record["down_votes"].to_i, Integer)
         @forum_post_id   = T.let(T.must(record["forum_post_id"]).empty? ? nil : record["forum_post_id"].to_i, T.nilable(Integer))
         @forum_topic_id  = T.let(T.must(record["forum_topic_id"]).empty? ? nil : record["forum_topic_id"].to_i, T.nilable(Integer))
@@ -78,7 +78,7 @@ module E621ExportDownloader
           approver_id:     @approver_id,
           consequent_name: @consequent_name,
           created_at:      @created_at,
-          # creator_id:      @creator_id,
+          creator_id:      @creator_id,
           down_votes:      @down_votes,
           forum_post_id:   @forum_post_id,
           forum_topic_id:  @forum_topic_id,

data/lib/e621_export_downloader/models/tag_implication.rb

@@ -18,8 +18,8 @@ module E621ExportDownloader
       sig { returns(T.nilable(DateTime)) }
       attr_reader(:created_at)
 
-      # sig { returns(T.nilable(Integer)) }
-      # attr_reader(:creator_id)
+      sig { returns(T.nilable(Integer)) }
+      attr_reader(:creator_id)
 
       sig { returns(T::Array[String]) }
       attr_reader(:descendant_names)
@@ -58,7 +58,7 @@ module E621ExportDownloader
         @approver_id     = T.let(T.must(record["approver_id"]).empty? ? nil : record["approver_id"].to_i, T.nilable(Integer))
         @consequent_name = T.let(T.must(record["consequent_name"]), String)
         @created_at      = T.let(T.must(record["created_at"]).empty? ? nil : DateTime.parse(record["created_at"]), T.nilable(DateTime))
-        # @creator_id      = T.let(T.must(record["creator_id"]).empty? ? nil : record["creator_id"].to_i, T.nilable(Integer))
+        @creator_id      = T.let(T.must(record["creator_id"]).empty? ? nil : record["creator_id"].to_i, T.nilable(Integer))
         inner            = T.must(T.must(record["descendant_names"])[1..-2])
         @descendant_names = T.let(inner.empty? ? [] : inner.split(","), T::Array[String])
         @down_votes      = T.let(record["down_votes"].to_i, Integer)
@@ -79,7 +79,7 @@ module E621ExportDownloader
           approver_id:      @approver_id,
           consequent_name:  @consequent_name,
           created_at:       @created_at,
-          # creator_id:       @creator_id,
+          creator_id:       @creator_id,
           descendant_names: @descendant_names,
           down_votes:       @down_votes,
           forum_post_id:    @forum_post_id,

data/lib/e621_export_downloader/version.rb

@@ -4,7 +4,7 @@
 # loaded by bundler
 module E621ExportDownloader
   module Constants
-    VERSION = "0.0.16"
+    VERSION = "0.0.18"
     WEBSITE = "https://github.com/DonovanDMC/E621ExportDownloader.rb"
   end
 end

data/lib/e621_export_downloader.rb

@@ -3,9 +3,16 @@
 require("zeitwerk")
 require("sorbet-runtime")
 
+# Loaded eagerly (rather than left to Zeitwerk) because Bundler's gemspec evaluation already
+# requires version.rb directly to read the gem version, which predefines E621ExportDownloader::Constants
+# before Zeitwerk::Loader#setup runs below. That premature definition makes Zeitwerk skip autoloading
+# constants.rb entirely, since it only autoloads namespaces it hasn't already seen defined.
+require_relative("e621_export_downloader/constants")
+
 loader = Zeitwerk::Loader.for_gem
 loader.inflector.inflect({ "api_export_data" => "APIExportData" })
 loader.ignore("#{__dir__}/e621_export_downloader/version.rb")
+loader.ignore("#{__dir__}/e621_export_downloader/constants.rb")
 loader.ignore("#{__dir__}/e621_export_downloader/railtie.rb")
 loader.ignore("#{__dir__}/e621_export_downloader/serializers/active_job.rb")
 loader.ignore("#{__dir__}/e621_export_downloader/active_record_models.rb")

data/lib/generators/e621_export_downloader/templates/migrations/create_e621_tables.rb.erb

@@ -2,8 +2,6 @@
 
 class CreateE621Tables < ActiveRecord::Migration<%= migration_version %>
   def change
-    enable_extension("pg_trgm")
-
 <% if use_schema? -%>
     reversible do |r|
       r.up { execute("CREATE SCHEMA <%= schema %>") }
@@ -13,48 +11,46 @@ class CreateE621Tables < ActiveRecord::Migration<%= migration_version %>
 <% end -%>
     create_table(<%= table_ref("artists") %>, id: :bigint, default: nil, force: :cascade) do |t|
       t.datetime(:created_at, null: false)
-      t.bigint(:creator_id, null: false, index: true)
+      t.bigint(:creator_id, null: false)
       t.string(:group_name)
       t.boolean(:is_active, null: false)
       t.boolean(:is_locked, null: false)
-      t.bigint(:linked_user_id, index: true)
-      t.string(:name, null: false, index: { unique: true })
+      t.bigint(:linked_user_id)
+      t.string(:name, null: false)
       t.text(:other_names, array: true, null: false, default: [])
       t.datetime(:updated_at, null: false)
-      t.text(:urls)
-      t.index("group_name gin_trgm_ops", using: :gin)
-      t.index("name gin_trgm_ops", using: :gin)
+      t.text(:active_urls, null: false)
+      t.text(:inactive_urls, null: false)
     end
 
     create_table(<%= table_ref("bulk_update_requests") %>, id: :bigint, default: nil, force: :cascade) do |t|
-      t.bigint(:approver_id, index: true)
+      t.bigint(:approver_id)
       t.datetime(:created_at, null: false)
-      t.bigint(:forum_topic_id, index: true)
-      t.bigint(:forum_post_id, index: true)
+      t.bigint(:forum_topic_id)
+      t.bigint(:forum_post_id)
       t.text(:script, null: false)
-      t.string(:status, null: false, index: true)
+      t.string(:status, null: false)
       t.string(:title)
       t.datetime(:updated_at, null: false)
-      t.bigint(:user_id, null: false, index: true)
-      t.index("script gin_trgm_ops", using: :gin)
-      t.index("title gin_trgm_ops", using: :gin)
+      t.bigint(:user_id, null: false)
+      t.integer(:down_votes, null: false)
+      t.integer(:meh_votes, null: false)
+      t.integer(:up_votes, null: false)
     end
 
     create_table(<%= table_ref("pools") %>, id: :bigint, default: nil, force: :cascade) do |t|
       t.string(:category, null: false)
       t.datetime(:created_at, null: false)
-      t.bigint(:creator_id, null: false, index: true)
+      t.bigint(:creator_id, null: false)
       t.text(:description, null: false)
       t.boolean(:is_active, null: false)
-      t.string(:name, null: false, index: true)
-      t.bigint(:post_ids, array: true, null: false, default: [], index: { using: :gin })
+      t.string(:name, null: false)
+      t.bigint(:post_ids, array: true, null: false)
       t.datetime(:updated_at)
-      t.index("description gin_trgm_ops", using: :gin)
-      t.index("name gin_trgm_ops", using: :gin)
     end
 
     create_table(<%= table_ref("posts") %>, id: :bigint, default: nil, force: :cascade) do |t|
-      t.bigint(:approver_id, index: true)
+      t.bigint(:approver_id)
       t.bigint(:change_seq, null: false)
       t.integer(:comment_count, null: false)
       t.datetime(:created_at, null: false)
@@ -66,112 +62,118 @@ class CreateE621Tables < ActiveRecord::Migration<%= migration_version %>
       t.bigint(:file_size, null: false)
       t.integer(:image_height, null: false)
       t.integer(:image_width, null: false)
-      t.boolean(:is_deleted, null: false, index: true)
-      t.boolean(:is_flagged, null: false, index: true)
-      t.boolean(:is_note_locked, null: false, index: true)
-      t.boolean(:is_pending, null: false, index: true)
-      t.boolean(:is_rating_locked, null: false, index: true)
-      t.boolean(:is_status_locked, null: false, index: true)
-      t.text(:locked_tags, null: false)
-      t.string(:md5, index: { unique: true }, null: false)
-      t.bigint(:parent_id, index: true)
-      t.string(:rating, null: false, index: true)
+      t.boolean(:is_deleted, null: false)
+      t.boolean(:is_flagged, null: false)
+      t.boolean(:is_note_locked, null: false)
+      t.boolean(:is_pending, null: false)
+      t.boolean(:is_rating_locked, null: false)
+      t.boolean(:is_status_locked, null: false)
+      t.text(:locked_tags)
+      t.string(:md5, null: false)
+      t.bigint(:parent_id)
+      t.string(:rating, null: false)
       t.integer(:score, null: false)
       t.text(:source, null: false)
       t.text(:tag_string, null: false)
       t.integer(:up_score, null: false)
-      t.datetime(:updated_at, index: true)
-      t.bigint(:uploader_id, index: true)
-      t.index("description gin_trgm_ops", using: :gin)
-      t.index("source gin_trgm_ops", using: :gin)
-      t.index("tag_string gin_trgm_ops", using: :gin)
+      t.datetime(:updated_at)
+      t.bigint(:uploader_id)
+      t.string(:bg_color)
+      t.datetime(:last_noted_at)
+      t.datetime(:last_commented_at)
     end
 
     create_table(<%= table_ref("post_replacements") %>, id: :bigint, default: nil, force: :cascade) do |t|
-      t.bigint(:approver_id, index: true)
+      t.bigint(:approver_id)
       t.datetime(:created_at, null: false)
-      t.bigint(:creator_id, null: false, index: true)
-      t.string(:file_ext, null: false, index: true)
-      t.string(:file_name, null: false, index: true)
-      t.bigint(:file_size, null: false, index: true)
-      t.integer(:image_height, null: false, index: true)
-      t.integer(:image_width, null: false, index: true)
-      t.string(:md5, null: false, index: true)
-      t.bigint(:post_id, null: false, index: true)
+      t.bigint(:creator_id, null: false)
+      t.string(:file_ext, null: false)
+      t.string(:file_name, null: false)
+      t.bigint(:file_size, null: false)
+      t.integer(:image_height, null: false)
+      t.integer(:image_width, null: false)
+      t.string(:md5, null: false)
+      t.bigint(:post_id, null: false)
       t.text(:reason, null: false)
       t.text(:source)
-      t.string(:status, null: false, index: true)
+      t.string(:status, null: false)
       t.datetime(:updated_at, null: false)
-      t.index("reason gin_trgm_ops", using: :gin)
-      t.index("source gin_trgm_ops", using: :gin)
     end
 
     create_table(<%= table_ref("post_versions") %>, id: :bigint, default: nil, force: :cascade) do |t|
       t.text(:added_locked_tags, array: true, null: false, default: [])
       t.text(:added_tags, array: true, null: false, default: [])
       t.text(:description)
-      t.boolean(:description_changed, null: false, index: true)
+      t.boolean(:description_changed, null: false)
       t.text(:locked_tags)
-      t.boolean(:parent_changed, null: false, index: true)
+      t.boolean(:parent_changed, null: false)
       t.bigint(:parent_id)
-      t.bigint(:post_id, null: false, index: true)
+      t.bigint(:post_id, null: false)
       t.string(:rating)
-      t.boolean(:rating_changed, null: false, index: true)
+      t.boolean(:rating_changed, null: false)
       t.text(:reason)
       t.text(:removed_locked_tags, array: true, null: false, default: [])
       t.text(:removed_tags, array: true, null: false, default: [])
       t.text(:source)
-      t.boolean(:source_changed, null: false, index: true)
+      t.boolean(:source_changed, null: false)
       t.text(:tags)
-      t.datetime(:updated_at, null: false, index: true)
-      t.bigint(:updater_id, index: true)
-      t.integer(:version, null: false, index: true)
-      t.index("description gin_trgm_ops", using: :gin)
-      t.index("locked_tags gin_trgm_ops", using: :gin)
-      t.index("reason gin_trgm_ops", using: :gin)
-      t.index("source gin_trgm_ops", using: :gin)
-      t.index("tags gin_trgm_ops", using: :gin)
+      t.datetime(:updated_at, null: false)
+      t.bigint(:updater_id)
+      t.integer(:version, null: false)
     end
 
     create_table(<%= table_ref("tag_aliases") %>, id: :bigint, default: nil, force: :cascade) do |t|
-      t.string(:antecedent_name, null: false, index: true)
-      t.string(:consequent_name, null: false, index: true)
+      t.string(:antecedent_name, null: false)
+      t.string(:consequent_name, null: false)
       t.datetime(:created_at)
-      t.string(:status, null: false, index: true)
-      t.index(%i[status antecedent_name])
-      t.index("antecedent_name gin_trgm_ops", using: :gin)
-      t.index("consequent_name gin_trgm_ops", using: :gin)
+      t.string(:status, null: false)
+      t.integer(:forum_post_id, null: false)
+      t.integer(:forum_topic_id, null: false)
+      t.string(:reason, null: false)
+      t.datetime(:updated_at)
+      t.integer(:approver_id)
+      t.integer(:post_count, null: false)
+      t.integer(:down_votes, null: false)
+      t.integer(:meh_votes, null: false)
+      t.integer(:up_votes, null: false)
+      t.integer(:creator_id, null: false)
     end
 
     create_table(<%= table_ref("tag_implications") %>, id: :bigint, default: nil, force: :cascade) do |t|
-      t.string(:antecedent_name, null: false, index: true)
-      t.string(:consequent_name, null: false, index: true)
+      t.string(:antecedent_name, null: false)
+      t.string(:consequent_name, null: false)
       t.datetime(:created_at)
-      t.string(:status, null: false, index: true)
-      t.index(%i[status consequent_name])
-      t.index(%i[status antecedent_name])
-      t.index("antecedent_name gin_trgm_ops", using: :gin)
-      t.index("consequent_name gin_trgm_ops", using: :gin)
+      t.string(:status, null: false)
+      t.integer(:forum_post_id, null: false)
+      t.integer(:forum_topic_id, null: false)
+      t.string(:reason, null: false)
+      t.datetime(:updated_at)
+      t.integer(:approver_id)
+      t.string(:descendant_names, array: true)
+      t.integer(:down_votes, null: false)
+      t.integer(:meh_votes, null: false)
+      t.integer(:up_votes, null: false)
+      t.integer(:creator_id, null: false)
     end
 
     create_table(<%= table_ref("tags") %>, id: :bigint, default: nil, force: :cascade) do |t|
-      t.string(:category, null: false, index: true)
-      t.string(:name, null: false, index: { unique: true })
-      t.integer(:post_count, null: false, index: { order: { post_count: :desc } })
-      t.index(%i[category post_count], order: { post_count: :desc })
-      t.index("name gin_trgm_ops", using: :gin)
+      t.string(:category, null: false)
+      t.string(:name, null: false)
+      t.integer(:post_count, null: false)
+      t.datetime(:created_at, null: false)
+      t.datetime(:updated_at, null: false)
+      t.boolean(:is_locked, null: false)
     end
 
     create_table(<%= table_ref("wiki_pages") %>, id: :bigint, default: nil, force: :cascade) do |t|
-      t.text(:body, null: false, index: true)
+      t.text(:body, null: false)
       t.datetime(:created_at, null: false)
-      t.bigint(:creator_id, null: false, index: true)
-      t.boolean(:is_locked, null: false, index: true)
-      t.string(:title, null: false, index: { unique: true })
+      t.bigint(:creator_id, null: false)
+      t.boolean(:is_locked, null: false)
+      t.string(:title, null: false)
       t.datetime(:updated_at, null: false)
-      t.bigint(:updater_id, index: true)
-      t.index("body gin_trgm_ops", using: :gin)
-      t.index("title gin_trgm_ops", using: :gin)
+      t.bigint(:updater_id)
+      t.string(:parent)
     end
 
     create_table(<%= table_ref("row_counts") %>, primary_key: :table_name, id: :string, force: :cascade) do |t|

data/scripts/generate_changelog.rb

@@ -0,0 +1,50 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require("date")
+
+def categorize(subject)
+  case subject
+  when /\Aadd /i    then "Added"
+  when /\Afix /i    then "Fixed"
+  when /\Aoptimize /i, /\Arefactor /i, /\Aupdate /i then "Changed"
+  end
+end
+
+shas = `git log --follow --format="%H" -- lib/e621_export_downloader/version.rb`.strip.split("\n")
+
+entries = shas.map do |sha|
+  subject      = `git log -1 --format="%s" #{sha}`.strip
+  date_str     = `git log -1 --format="%ai" #{sha}`.strip
+  body         = `git log -1 --format="%b" #{sha}`.strip
+  version_file = `git show #{sha}:lib/e621_export_downloader/version.rb`
+  version      = version_file.match(/VERSION = "([^"]+)"/)[1]
+  date         = Date.parse(date_str).strftime("%Y-%m-%d")
+  section      = categorize(subject)
+
+  { version: version, date: date, section: section, subject: subject, body: body }
+end
+
+changelog = +"# Changelog\n"
+
+entries.each do |e|
+  changelog << "\n## [#{e[:version]}] - #{e[:date]}\n"
+
+  section = e[:section]
+  if section
+    changelog << "\n### #{section}\n"
+    changelog << "- #{e[:subject]}"
+
+    unless e[:body].empty?
+      refs = e[:body].scan(/(?:from|pending):\s*\S+/).join(", ")
+      changelog << " (#{refs})" unless refs.empty?
+    end
+
+    changelog << "\n"
+  else
+    changelog << "\n_#{e[:subject]}_\n"
+  end
+end
+
+File.write("CHANGELOG.md", changelog)
+puts("Generated CHANGELOG.md")

data/test/export_test.rb

@@ -0,0 +1,93 @@
+# frozen_string_literal: true
+
+require("minitest/autorun")
+require("digest")
+require("fileutils")
+require("zlib")
+require("e621_export_downloader")
+
+module ExportTestHelpers
+  # Stands in for a Faraday::Connection, driving the same #get(url) { |req| req.options.on_data = ... }
+  # streaming contract that Export#download relies on, without touching the network.
+  class FakeConnection
+    def initialize(chunks)
+      @chunks = chunks
+    end
+
+    def get(_url)
+      req = Struct.new(:options).new(Struct.new(:on_data).new)
+      yield(req)
+
+      received = 0
+      @chunks.each do |chunk|
+        received += chunk.bytesize
+        req.options.on_data.call(chunk, received)
+      end
+
+      Struct.new(:status, :reason_phrase, :success?).new(200, "OK", true)
+    end
+
+    def head(_url)
+      Struct.new(:success?).new(true)
+    end
+  end
+end
+
+class ExportChecksumTest < Minitest::Test
+  include(ExportTestHelpers)
+
+  def setup
+    @csv_content = "id,name\n1,foo\n"
+    @compressed = Zlib.gzip(@csv_content)
+    @checksum = Digest::MD5.hexdigest(@compressed)
+    # split across chunks to exercise streaming, matching how a real download arrives
+    midpoint = @compressed.bytesize / 2
+    @chunks = [@compressed.byteslice(0, midpoint), @compressed.byteslice(midpoint..)]
+  end
+
+  def teardown
+    FileUtils.rm_rf(E621ExportDownloader::Constants::TEMP_DIR)
+  end
+
+  def build_export(checksum:, validate_checksum:)
+    data = E621ExportDownloader::APIExportData.new(
+      name:       E621ExportDownloader::Types::Posts,
+      file_name:  "posts-checksum-test.csv",
+      file_size:  @compressed.bytesize,
+      checksum:   checksum,
+      updated_at: "2026-01-01",
+      url:        "https://example.com/posts.csv.gz",
+    )
+    client = E621ExportDownloader::Client.new(
+      E621ExportDownloader::Client::Options.new(validate_checksum: validate_checksum),
+    )
+    chunks = @chunks
+    client.define_singleton_method(:connection) { FakeConnection.new(chunks) }
+    E621ExportDownloader::Export.new(
+      data:   data,
+      client: client,
+      type:   E621ExportDownloader::Types::Posts,
+      parser: ->(record) { record },
+    )
+  end
+
+  def test_download_succeeds_when_checksum_matches_and_validation_is_enabled
+    export = build_export(checksum: @checksum, validate_checksum: true)
+    path = export.download
+    assert(File.exist?(path))
+    assert_equal(@csv_content, File.read(path))
+  end
+
+  def test_download_raises_and_removes_the_file_when_checksum_does_not_match
+    export = build_export(checksum: "0" * 32, validate_checksum: true)
+    error = assert_raises(E621ExportDownloader::ResolveError) { export.download }
+    assert_match(/Checksum mismatch/, error.message)
+    assert_empty(Dir.children(E621ExportDownloader::Constants::TEMP_DIR))
+  end
+
+  def test_download_ignores_checksum_mismatch_when_validation_is_disabled
+    export = build_export(checksum: "0" * 32, validate_checksum: false)
+    path = export.download
+    assert(File.exist?(path))
+  end
+end

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: e621_export_downloader
 version: !ruby/object:Gem::Version
-  version: 0.0.16
+  version: 0.0.18
 platform: ruby
 authors:
 - Donovan_DMC
 bindir: exe
 cert_chain: []
-date: 2026-06-24 00:00:00.000000000 Z
+date: 2026-07-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: csv
@@ -75,6 +75,7 @@ extra_rdoc_files: []
 files:
 - ".irbrc"
 - ".ruby-version"
+- CHANGELOG.md
 - LICENSE
 - README.md
 - Rakefile
@@ -118,6 +119,7 @@ files:
 - lib/generators/e621_export_downloader/install_generator.rb
 - lib/generators/e621_export_downloader/templates/initializers/e621_models.rb.erb
 - lib/generators/e621_export_downloader/templates/migrations/create_e621_tables.rb.erb
+- scripts/generate_changelog.rb
 - sorbet/config
 - sorbet/rbi/annotations/.gitattributes
 - sorbet/rbi/annotations/faraday.rbi
@@ -221,6 +223,7 @@ files:
 - sorbet/rbi/todo.rbi
 - sorbet/tapioca/config.yml
 - sorbet/tapioca/require.rb
+- test/export_test.rb
 - test/generators/install_generator_test.rb
 homepage: https://github.com/DonovanDMC/E621ExportDownloader.rb
 licenses: