e621_export_downloader 0.0.17 → 0.0.18

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6d58e880077e657ef28542f3873a57f6b2759c2dc5cc3ace7b39b0b016134b1d
-  data.tar.gz: 93ba667cb1a85c203cf861aeedfc36ae4ba8dffeb9bc50c08df275a8ccd27844
+  metadata.gz: 7259f8077bf1df8805620f4efa3fbeac43c29616e496bea576518275794c5fef
+  data.tar.gz: 4b23216298e535aeed4ceaeb7f38f581bf6fc6fb68c1f7fa94975e9157f7bf71
 SHA512:
-  metadata.gz: af0d7f73fc67b9c2a291ca5c8de1e200dcd09e42ddf31d4c40fb07938a1f25a13b428dff8ad6e3a44190b6b2c3a672991d8f5b9a40aed2e808286512bf60812e
-  data.tar.gz: c012e57c2fc91e8d3991f47eb59842947a81acad6137a52f1ad768b8dcaeeb9990b13ffdad5d8d6c398a9dd6ba768928183163af14670398ce78b7874ccdf99b
+  metadata.gz: 50436457d2de9e91e8f938631484653a71d8102122ec58f77c8c6287454092c38ef6719215a4aa20cb676978d659ce936750338667d892045027b4ee0a0fa0ed
+  data.tar.gz: b8a0c19646cd5b7ed2076b023adabc016962700dcc4c755ef2b66d3f3b018349be45aafca1c4be1dc8c0be9c8f2dd6de5f1529f6ea27f45a04df5660d6f6f1a9

data/CHANGELOG.md

@@ -1,5 +1,9 @@
 # Changelog
 
+## [0.0.18] - 2026-07-02
+
+_0.0.18_
+
 ## [0.0.17] - 2026-06-29
 
 _0.0.17_

data/README.md

@@ -23,12 +23,13 @@ client = E621ExportDownloader::Client.new
 
 # configure options after creation
 client.config do |c|
-  c.cache = true  # keep export files after reading, defaults to false
+  c.cache = true              # keep export files after reading, defaults to false
+  c.validate_checksum = true  # verify the downloaded file against the API-provided checksum, defaults to false
 end
 
 # or pass an Options struct directly
 client = E621ExportDownloader::Client.new(
-  E621ExportDownloader::Client::Options.new(cache: true)
+  E621ExportDownloader::Client::Options.new(cache: true, validate_checksum: true)
 )
 
 # get the export for a type — resolves the latest available export from the e621 API
@@ -39,10 +40,10 @@ export = client.get("posts")
 deferred = client.get_deferred("posts")
 
 # get raw metadata for all exports from the e621 API
-# returns an Array of E621ExportDownloader::APIExportData: { file_name, file_size, name, updated_at, url }
+# returns an Array of E621ExportDownloader::APIExportData: { file_name, file_size, name, checksum, updated_at, url }
 data = client.get_data
 
-# access the metadata for this specific export: { file_name, file_size, name, updated_at, url }
+# access the metadata for this specific export: { file_name, file_size, name, checksum, updated_at, url }
 export.data
 
 # check if the export exists
@@ -50,6 +51,7 @@ exists = export.exists?
 
 # download the export, returns the file path — not required before reading
 # if you move or remove the file do not reuse the export object
+# raises ResolveError if validate_checksum is enabled and the downloaded file's checksum does not match the API
 file = export.download
 
 # delete the downloaded file, if it exists

data/Rakefile

@@ -8,11 +8,21 @@ Rake::TestTask.new(:test) do |t|
   t.pattern = "test/**/*_test.rb"
 end
 
-namespace(:test) do
-  desc("Test Zeitwerk Auto-Importing")
-  task(:zeitwerk) do
-    exec(%(bundle exec ruby -e "require 'e621_export_downloader'; Zeitwerk::Loader.eager_load_all"))
-  end
+desc("Test Zeitwerk Auto-Importing")
+task(:zeitwerk) do
+  ruby(%(-e "require 'e621_export_downloader'; Zeitwerk::Loader.eager_load_all"))
 end
 
-task(default: :rubocop)
+desc("Run all tests")
+task(:all) do
+  puts("=== rubocop ===")
+  sh("bundle exec rubocop")
+  puts("=== zeitwerk ===")
+  sh("bundle exec rake zeitwerk")
+  puts("=== test ===")
+  sh("bundle exec rake test")
+  puts("=== srb ===")
+  sh("bundle exec srb tc")
+end
+
+task(default: :all)

data/lib/e621/tag_alias.rb

@@ -29,7 +29,7 @@ module E621
         approver_id:     record.approver_id,
         consequent_name: record.consequent_name,
         created_at:      record.created_at,
-        # creator_id:      record.creator_id,
+        creator_id:      record.creator_id,
         down_votes:      record.down_votes,
         forum_post_id:   record.forum_post_id,
         forum_topic_id:  record.forum_topic_id,

data/lib/e621/tag_implication.rb

@@ -29,7 +29,7 @@ module E621
         approver_id:      record.approver_id,
         consequent_name:  record.consequent_name,
         created_at:       record.created_at,
-        # creator_id:       record.creator_id,
+        creator_id:       record.creator_id,
         descendant_names: record.descendant_names,
         down_votes:       record.down_votes,
         forum_post_id:    record.forum_post_id,

data/lib/e621_export_downloader/api_export_data.rb

@@ -3,9 +3,10 @@
 
 module E621ExportDownloader
   class APIExportData < T::Struct
+    const(:name, Types)
     const(:file_name, String)
     const(:file_size, Integer)
-    const(:name, Types)
+    const(:checksum, String)
     const(:updated_at, String)
     const(:url, String)
   end

data/lib/e621_export_downloader/client/options/builder.rb

@@ -10,13 +10,17 @@ module E621ExportDownloader
         sig { returns(T::Boolean) }
         attr_accessor(:cache)
 
+        sig { returns(T::Boolean) }
+        attr_accessor(:validate_checksum)
+
         sig { params(parsers: Options::Parsers).returns(Options::Parsers) }
         attr_writer(:parsers)
 
         sig { params(defaults: Options).void }
         def initialize(defaults = Options.new)
-          @cache = T.let(defaults.cache, T::Boolean)
-          @parsers = T.let(defaults.parsers, Options::Parsers)
+          @cache             = T.let(defaults.cache, T::Boolean)
+          @validate_checksum = T.let(defaults.validate_checksum, T::Boolean)
+          @parsers           = T.let(defaults.parsers, Options::Parsers)
         end
 
         sig { params(block: T.nilable(T.proc.params(arg0: Builder::Parsers).void)).returns(Options::Parsers) }

data/lib/e621_export_downloader/client/options.rb

@@ -38,6 +38,7 @@ module E621ExportDownloader
       end
 
       const(:cache, T::Boolean, default: false)
+      const(:validate_checksum, T::Boolean, default: false)
       const(:parsers, Parsers, default: Parsers.defaults)
     end
   end

data/lib/e621_export_downloader/client.rb

@@ -21,7 +21,9 @@ module E621ExportDownloader
 
     sig { params(block: T.proc.params(arg0: Options::Builder).void).void }
     def config(&block)
-      block.call(Options::Builder.new(@options))
+      builder = Options::Builder.new(@options)
+      block.call(builder)
+      @options = Options.new(cache: builder.cache, validate_checksum: builder.validate_checksum, parsers: builder.parsers)
     end
 
     sig { returns(Faraday::Connection) }
@@ -61,9 +63,10 @@ module E621ExportDownloader
       data = JSON.parse(T.unsafe(res).body)
       result = T.let(data.map do |d|
         APIExportData.new(
+          name:       Types.deserialize(d["name"]),
           file_name:  d["file_name"],
           file_size:  d["file_size"].to_i,
-          name:       Types.deserialize(d["name"]),
+          checksum:   d["checksum"],
           updated_at: d["updated_at"],
           url:        d["url"],
         )

data/lib/e621_export_downloader/export.rb

@@ -2,6 +2,10 @@
 # typed: strict
 
 require("csv")
+require("date")
+require("digest")
+require("fileutils")
+require("zlib")
 
 module E621ExportDownloader
   class Export
@@ -51,11 +55,13 @@ module E621ExportDownloader
       client.debug("downloading export", header: ["export:#{type.serialize}"])
 
       FileUtils.mkdir_p(Constants::TEMP_DIR)
+      digest = T.let(client.options.validate_checksum ? Digest::MD5.new : nil, T.nilable(Digest::MD5))
       File.open(file_path, "wb") do |file|
         inflater = Zlib::Inflate.new(Zlib::MAX_WBITS + 16)
 
         res = client.connection.get(data.url) do |req|
           req.options.on_data = proc do |chunk, received|
+            digest&.update(chunk)
             decompressed = inflater.inflate(chunk)
             file.write(decompressed) if decompressed && !decompressed.empty?
             block.call(received, data.file_size) if block
@@ -69,6 +75,10 @@ module E621ExportDownloader
         unless res.success?
           raise(ResolveError, "Failed to download export #{type.serialize}: #{res.status} #{res.reason_phrase}")
         end
+
+        if digest && digest.hexdigest != data.checksum
+          raise(ResolveError, "Checksum mismatch for export #{type.serialize}")
+        end
       rescue # rubocop:disable Style/RescueStandardError
         FileUtils.rm_f(file_path)
         raise

data/lib/e621_export_downloader/models/tag_alias.rb

@@ -18,8 +18,8 @@ module E621ExportDownloader
       sig { returns(T.nilable(DateTime)) }
       attr_reader(:created_at)
 
-      # sig { returns(T.nilable(Integer)) }
-      # attr_reader(:creator_id)
+      sig { returns(T.nilable(Integer)) }
+      attr_reader(:creator_id)
 
       sig { returns(Integer) }
       attr_reader(:down_votes)
@@ -58,7 +58,7 @@ module E621ExportDownloader
         @approver_id     = T.let(T.must(record["approver_id"]).empty? ? nil : record["approver_id"].to_i, T.nilable(Integer))
         @consequent_name = T.let(T.must(record["consequent_name"]), String)
         @created_at      = T.let(T.must(record["created_at"]).empty? ? nil : DateTime.parse(record["created_at"]), T.nilable(DateTime))
-        # @creator_id      = T.let(T.must(record["creator_id"]).empty? ? nil : record["creator_id"].to_i, T.nilable(Integer))
+        @creator_id      = T.let(T.must(record["creator_id"]).empty? ? nil : record["creator_id"].to_i, T.nilable(Integer))
         @down_votes      = T.let(record["down_votes"].to_i, Integer)
         @forum_post_id   = T.let(T.must(record["forum_post_id"]).empty? ? nil : record["forum_post_id"].to_i, T.nilable(Integer))
         @forum_topic_id  = T.let(T.must(record["forum_topic_id"]).empty? ? nil : record["forum_topic_id"].to_i, T.nilable(Integer))
@@ -78,7 +78,7 @@ module E621ExportDownloader
           approver_id:     @approver_id,
           consequent_name: @consequent_name,
           created_at:      @created_at,
-          # creator_id:      @creator_id,
+          creator_id:      @creator_id,
           down_votes:      @down_votes,
           forum_post_id:   @forum_post_id,
           forum_topic_id:  @forum_topic_id,

data/lib/e621_export_downloader/models/tag_implication.rb

@@ -18,8 +18,8 @@ module E621ExportDownloader
       sig { returns(T.nilable(DateTime)) }
       attr_reader(:created_at)
 
-      # sig { returns(T.nilable(Integer)) }
-      # attr_reader(:creator_id)
+      sig { returns(T.nilable(Integer)) }
+      attr_reader(:creator_id)
 
       sig { returns(T::Array[String]) }
       attr_reader(:descendant_names)
@@ -58,7 +58,7 @@ module E621ExportDownloader
         @approver_id     = T.let(T.must(record["approver_id"]).empty? ? nil : record["approver_id"].to_i, T.nilable(Integer))
         @consequent_name = T.let(T.must(record["consequent_name"]), String)
         @created_at      = T.let(T.must(record["created_at"]).empty? ? nil : DateTime.parse(record["created_at"]), T.nilable(DateTime))
-        # @creator_id      = T.let(T.must(record["creator_id"]).empty? ? nil : record["creator_id"].to_i, T.nilable(Integer))
+        @creator_id      = T.let(T.must(record["creator_id"]).empty? ? nil : record["creator_id"].to_i, T.nilable(Integer))
         inner            = T.must(T.must(record["descendant_names"])[1..-2])
         @descendant_names = T.let(inner.empty? ? [] : inner.split(","), T::Array[String])
         @down_votes      = T.let(record["down_votes"].to_i, Integer)
@@ -79,7 +79,7 @@ module E621ExportDownloader
           approver_id:      @approver_id,
           consequent_name:  @consequent_name,
           created_at:       @created_at,
-          # creator_id:       @creator_id,
+          creator_id:       @creator_id,
           descendant_names: @descendant_names,
           down_votes:       @down_votes,
           forum_post_id:    @forum_post_id,

data/lib/e621_export_downloader/version.rb

@@ -4,7 +4,7 @@
 # loaded by bundler
 module E621ExportDownloader
   module Constants
-    VERSION = "0.0.17"
+    VERSION = "0.0.18"
     WEBSITE = "https://github.com/DonovanDMC/E621ExportDownloader.rb"
   end
 end

data/lib/e621_export_downloader.rb

@@ -3,9 +3,16 @@
 require("zeitwerk")
 require("sorbet-runtime")
 
+# Loaded eagerly (rather than left to Zeitwerk) because Bundler's gemspec evaluation already
+# requires version.rb directly to read the gem version, which predefines E621ExportDownloader::Constants
+# before Zeitwerk::Loader#setup runs below. That premature definition makes Zeitwerk skip autoloading
+# constants.rb entirely, since it only autoloads namespaces it hasn't already seen defined.
+require_relative("e621_export_downloader/constants")
+
 loader = Zeitwerk::Loader.for_gem
 loader.inflector.inflect({ "api_export_data" => "APIExportData" })
 loader.ignore("#{__dir__}/e621_export_downloader/version.rb")
+loader.ignore("#{__dir__}/e621_export_downloader/constants.rb")
 loader.ignore("#{__dir__}/e621_export_downloader/railtie.rb")
 loader.ignore("#{__dir__}/e621_export_downloader/serializers/active_job.rb")
 loader.ignore("#{__dir__}/e621_export_downloader/active_record_models.rb")

data/lib/generators/e621_export_downloader/templates/migrations/create_e621_tables.rb.erb

@@ -68,7 +68,7 @@ class CreateE621Tables < ActiveRecord::Migration<%= migration_version %>
       t.boolean(:is_pending, null: false)
       t.boolean(:is_rating_locked, null: false)
       t.boolean(:is_status_locked, null: false)
-      t.text(:locked_tags, null: false)
+      t.text(:locked_tags)
       t.string(:md5, null: false)
       t.bigint(:parent_id)
       t.string(:rating, null: false)
@@ -136,6 +136,7 @@ class CreateE621Tables < ActiveRecord::Migration<%= migration_version %>
       t.integer(:down_votes, null: false)
       t.integer(:meh_votes, null: false)
       t.integer(:up_votes, null: false)
+      t.integer(:creator_id, null: false)
     end
 
     create_table(<%= table_ref("tag_implications") %>, id: :bigint, default: nil, force: :cascade) do |t|
@@ -152,6 +153,7 @@ class CreateE621Tables < ActiveRecord::Migration<%= migration_version %>
       t.integer(:down_votes, null: false)
       t.integer(:meh_votes, null: false)
       t.integer(:up_votes, null: false)
+      t.integer(:creator_id, null: false)
     end
 
     create_table(<%= table_ref("tags") %>, id: :bigint, default: nil, force: :cascade) do |t|

data/test/export_test.rb

@@ -0,0 +1,93 @@
+# frozen_string_literal: true
+
+require("minitest/autorun")
+require("digest")
+require("fileutils")
+require("zlib")
+require("e621_export_downloader")
+
+module ExportTestHelpers
+  # Stands in for a Faraday::Connection, driving the same #get(url) { |req| req.options.on_data = ... }
+  # streaming contract that Export#download relies on, without touching the network.
+  class FakeConnection
+    def initialize(chunks)
+      @chunks = chunks
+    end
+
+    def get(_url)
+      req = Struct.new(:options).new(Struct.new(:on_data).new)
+      yield(req)
+
+      received = 0
+      @chunks.each do |chunk|
+        received += chunk.bytesize
+        req.options.on_data.call(chunk, received)
+      end
+
+      Struct.new(:status, :reason_phrase, :success?).new(200, "OK", true)
+    end
+
+    def head(_url)
+      Struct.new(:success?).new(true)
+    end
+  end
+end
+
+class ExportChecksumTest < Minitest::Test
+  include(ExportTestHelpers)
+
+  def setup
+    @csv_content = "id,name\n1,foo\n"
+    @compressed = Zlib.gzip(@csv_content)
+    @checksum = Digest::MD5.hexdigest(@compressed)
+    # split across chunks to exercise streaming, matching how a real download arrives
+    midpoint = @compressed.bytesize / 2
+    @chunks = [@compressed.byteslice(0, midpoint), @compressed.byteslice(midpoint..)]
+  end
+
+  def teardown
+    FileUtils.rm_rf(E621ExportDownloader::Constants::TEMP_DIR)
+  end
+
+  def build_export(checksum:, validate_checksum:)
+    data = E621ExportDownloader::APIExportData.new(
+      name:       E621ExportDownloader::Types::Posts,
+      file_name:  "posts-checksum-test.csv",
+      file_size:  @compressed.bytesize,
+      checksum:   checksum,
+      updated_at: "2026-01-01",
+      url:        "https://example.com/posts.csv.gz",
+    )
+    client = E621ExportDownloader::Client.new(
+      E621ExportDownloader::Client::Options.new(validate_checksum: validate_checksum),
+    )
+    chunks = @chunks
+    client.define_singleton_method(:connection) { FakeConnection.new(chunks) }
+    E621ExportDownloader::Export.new(
+      data:   data,
+      client: client,
+      type:   E621ExportDownloader::Types::Posts,
+      parser: ->(record) { record },
+    )
+  end
+
+  def test_download_succeeds_when_checksum_matches_and_validation_is_enabled
+    export = build_export(checksum: @checksum, validate_checksum: true)
+    path = export.download
+    assert(File.exist?(path))
+    assert_equal(@csv_content, File.read(path))
+  end
+
+  def test_download_raises_and_removes_the_file_when_checksum_does_not_match
+    export = build_export(checksum: "0" * 32, validate_checksum: true)
+    error = assert_raises(E621ExportDownloader::ResolveError) { export.download }
+    assert_match(/Checksum mismatch/, error.message)
+    assert_empty(Dir.children(E621ExportDownloader::Constants::TEMP_DIR))
+  end
+
+  def test_download_ignores_checksum_mismatch_when_validation_is_disabled
+    export = build_export(checksum: "0" * 32, validate_checksum: false)
+    path = export.download
+    assert(File.exist?(path))
+  end
+end

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: e621_export_downloader
 version: !ruby/object:Gem::Version
-  version: 0.0.17
+  version: 0.0.18
 platform: ruby
 authors:
 - Donovan_DMC
 bindir: exe
 cert_chain: []
-date: 2026-06-29 00:00:00.000000000 Z
+date: 2026-07-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: csv
@@ -223,6 +223,7 @@ files:
 - sorbet/rbi/todo.rbi
 - sorbet/tapioca/config.yml
 - sorbet/tapioca/require.rb
+- test/export_test.rb
 - test/generators/install_generator_test.rb
 homepage: https://github.com/DonovanDMC/E621ExportDownloader.rb
 licenses: