e621_export_downloader 0.0.18 → 0.0.20

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7259f8077bf1df8805620f4efa3fbeac43c29616e496bea576518275794c5fef
-  data.tar.gz: 4b23216298e535aeed4ceaeb7f38f581bf6fc6fb68c1f7fa94975e9157f7bf71
+  metadata.gz: 34abf6e64f79dc2cac589e527e3fcba32308ed0820f874bde23db2cc33b3eed0
+  data.tar.gz: 68fb4568601f98b9a597cb429937d9c11703158c92f32034302db9adf8b1d4fa
 SHA512:
-  metadata.gz: 50436457d2de9e91e8f938631484653a71d8102122ec58f77c8c6287454092c38ef6719215a4aa20cb676978d659ce936750338667d892045027b4ee0a0fa0ed
-  data.tar.gz: b8a0c19646cd5b7ed2076b023adabc016962700dcc4c755ef2b66d3f3b018349be45aafca1c4be1dc8c0be9c8f2dd6de5f1529f6ea27f45a04df5660d6f6f1a9
+  metadata.gz: 69abf12cfa989fd2ea8196b6a471c6d872363922a1d8f308335d27374bd4f022c23c460663d52fa33f26643e0b41b0f0aeb106167e347c644e31ddb0df888146
+  data.tar.gz: 0afe87bb30bd8403e2402010ad329f3746e3b83b93a419f30ecabef6255e6f66c1188f00a95bbc65189ec866cd16efbdbf397d58ded81cab9fd16dff2314f83d

data/CHANGELOG.md

@@ -1,5 +1,13 @@
 # Changelog
 
+## [0.0.20] - 2026-08-21
+
+_0.0.20_
+
+## [0.0.19] - 2026-08-03
+
+_0.0.19_
+
 ## [0.0.18] - 2026-07-02
 
 _0.0.18_

data/README.md

@@ -25,11 +25,13 @@ client = E621ExportDownloader::Client.new
 client.config do |c|
   c.cache = true              # keep export files after reading, defaults to false
   c.validate_checksum = true  # verify the downloaded file against the API-provided checksum, defaults to false
+  c.retries = 3               # number of times to retry a request after a transient network error (DNS failure, connection reset, timeout, ...), defaults to 3
+  c.retry_delay = 1           # base delay in seconds between retries, multiplied by the attempt number, defaults to 1
 end
 
 # or pass an Options struct directly
 client = E621ExportDownloader::Client.new(
-  E621ExportDownloader::Client::Options.new(cache: true, validate_checksum: true)
+  E621ExportDownloader::Client::Options.new(cache: true, validate_checksum: true, retries: 3, retry_delay: 1)
 )
 
 # get the export for a type — resolves the latest available export from the e621 API
@@ -212,6 +214,29 @@ require("e621_export_downloader/serializers/active_job")
 ActiveJob::Serializers.add_serializers(E621ExportDownloader::Serializers::ActiveJob)
 ```
 
+## Notifications
+
+The gem instruments [`ActiveSupport::Notifications`](https://api.rubyonrails.org/classes/ActiveSupport/Notifications.html) events around its major operations, so you can observe/track progress without passing callback params through every call site:
+
+| Event                                | Fired around                                          | Payload                                              |
+| ------------------------------------- | ------------------------------------------------------ | ----------------------------------------------------- |
+| `download.e621_export_downloader`     | `Export#download`'s network fetch (not a cache hit)     | `type`, `file_size`, `validate_checksum`; on failure, `exception`/`exception_object` (standard `ActiveSupport::Notifications` behavior) are set |
+| `retry.e621_export_downloader`        | Each retried attempt in `Client#with_retries`           | `header`, `attempt`, `error`, `message`, `delay` — not fired for the final, re-raised failure |
+| `truncate.e621_export_downloader`     | `import_from_csv`'s `TRUNCATE`, if `truncate: true`     | `table`                                                |
+| `drop_index.e621_export_downloader`   | Each dropped index, if `recreate_indexes: true`         | `table`, `index`                                       |
+| `copy.e621_export_downloader`         | The `COPY` itself                                       | `table`; `rows` once it finishes                       |
+| `create_index.e621_export_downloader` | Each rebuilt index, if `recreate_indexes: true`          | `table`, `index`                                       |
+
+```ruby
+require("active_support/notifications")
+
+ActiveSupport::Notifications.subscribe("create_index.e621_export_downloader") do |*, payload|
+  puts("rebuilt #{payload[:index]} on #{payload[:table]}")
+end
+```
+
+`create_index`/`drop_index` are what you want for a live "N of M indexes done" progress counter — Postgres itself only exposes progress for whichever *one* index is currently mid-build (`pg_stat_progress_create_index`), not how many of a batch have finished.
+
 ## Contributing
 
 Bug reports and pull requests are welcome on GitHub at https://github.com/DonovanDMC/E621ExportDownloader.rb.

data/lib/e621/csv_importable.rb

@@ -1,6 +1,8 @@
 # frozen_string_literal: true
 # typed: true
 
+require("active_support/notifications")
+
 module E621
   module CsvImportable
     extend(T::Sig)
@@ -34,6 +36,21 @@ module E621
     # Physical-storage policy (UNLOGGED) and session tuning
     # (maintenance_work_mem, max_parallel_maintenance_workers, ...) are the
     # caller's responsibility — set them around this call.
+    #
+    # Every phase below instruments an ActiveSupport::Notifications event
+    # (payload always includes at least table:) rather than accepting
+    # callback params directly, so any number of callers can subscribe
+    # without this method's signature having to grow one param per phase
+    # someone wants to observe:
+    #   "truncate.e621_export_downloader"     — around the TRUNCATE, if truncate: true
+    #   "drop_index.e621_export_downloader"   — once per dropped index, payload also has index:
+    #   "copy.e621_export_downloader"         — around the COPY itself, payload also gets rows: once it finishes
+    #   "create_index.e621_export_downloader" — once per rebuilt index, payload also has index:
+    # A subscriber wanting a live "N of M indexes done" progress counter
+    # should use create_index (or drop_index, for the teardown half) —
+    # Postgres itself only exposes progress for whichever *one* index is
+    # currently mid-build, via pg_stat_progress_create_index, not how many
+    # of a batch have finished.
     sig do
       params(
         csv_path:         String,
@@ -46,12 +63,28 @@ module E621
       model = T.unsafe(self)
       indexes = recreate_indexes ? secondary_index_definitions : {}
 
-      model.connection.execute("TRUNCATE #{model.quoted_table_name}") if truncate
-      indexes.each_key { |name| model.connection.execute("DROP INDEX IF EXISTS #{name}") }
-
-      count = copy_csv(csv_path, chunk_bytes: chunk_bytes)
+      if truncate
+        ActiveSupport::Notifications.instrument("truncate.e621_export_downloader", table: model.table_name) do
+          model.connection.execute("TRUNCATE #{model.quoted_table_name}")
+        end
+      end
+      indexes.each_key do |name|
+        ActiveSupport::Notifications.instrument("drop_index.e621_export_downloader", table: model.table_name, index: name) do
+          model.connection.execute("DROP INDEX IF EXISTS #{name}")
+        end
+      end
 
-      indexes.each_value { |ddl| model.connection.execute(ddl) }
+      count = begin
+        ActiveSupport::Notifications.instrument("copy.e621_export_downloader", table: model.table_name) do |payload|
+          copy_csv(csv_path, chunk_bytes: chunk_bytes).tap { |rows| payload[:rows] = rows }
+        end
+      ensure
+        indexes.each do |name, ddl|
+          ActiveSupport::Notifications.instrument("create_index.e621_export_downloader", table: model.table_name, index: name) do
+            model.connection.execute(ddl)
+          end
+        end
+      end
       self.row_count = count
       count
     end

data/lib/e621_export_downloader/client/options/builder.rb

@@ -13,6 +13,12 @@ module E621ExportDownloader
         sig { returns(T::Boolean) }
         attr_accessor(:validate_checksum)
 
+        sig { returns(Integer) }
+        attr_accessor(:retries)
+
+        sig { returns(Numeric) }
+        attr_accessor(:retry_delay)
+
         sig { params(parsers: Options::Parsers).returns(Options::Parsers) }
         attr_writer(:parsers)
 
@@ -21,6 +27,8 @@ module E621ExportDownloader
           @cache             = T.let(defaults.cache, T::Boolean)
           @validate_checksum = T.let(defaults.validate_checksum, T::Boolean)
           @parsers           = T.let(defaults.parsers, Options::Parsers)
+          @retries           = T.let(defaults.retries, Integer)
+          @retry_delay       = T.let(defaults.retry_delay, Numeric)
         end
 
         sig { params(block: T.nilable(T.proc.params(arg0: Builder::Parsers).void)).returns(Options::Parsers) }

data/lib/e621_export_downloader/client/options.rb

@@ -40,6 +40,8 @@ module E621ExportDownloader
       const(:cache, T::Boolean, default: false)
       const(:validate_checksum, T::Boolean, default: false)
       const(:parsers, Parsers, default: Parsers.defaults)
+      const(:retries, Integer, default: 3)
+      const(:retry_delay, Numeric, default: 1)
     end
   end
 end

data/lib/e621_export_downloader/client.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 # typed: strict
 
+require("active_support/notifications")
 require("faraday")
 require("json")
 
@@ -10,6 +11,9 @@ module E621ExportDownloader
 
     Logger = ::Logger.new($stdout)
 
+    # transient network failures (DNS resolution, connection resets, timeouts, ...) worth retrying
+    RETRYABLE_ERRORS = T.let([Faraday::ConnectionFailed, Faraday::TimeoutError, Faraday::SSLError].freeze, T::Array[T.class_of(Faraday::Error)])
+
     sig { returns(Options) }
     attr_reader(:options)
 
@@ -23,7 +27,13 @@ module E621ExportDownloader
     def config(&block)
       builder = Options::Builder.new(@options)
       block.call(builder)
-      @options = Options.new(cache: builder.cache, validate_checksum: builder.validate_checksum, parsers: builder.parsers)
+      @options = Options.new(
+        cache:             builder.cache,
+        validate_checksum: builder.validate_checksum,
+        parsers:           builder.parsers,
+        retries:           builder.retries,
+        retry_delay:       builder.retry_delay,
+      )
     end
 
     sig { returns(Faraday::Connection) }
@@ -36,6 +46,28 @@ module E621ExportDownloader
       Logger.debug("[e621_export_downloader#{":#{header.join(':')}" unless header.empty?}] #{msg}")
     end
 
+    # Fires "retry.e621_export_downloader" (payload: {header:, attempt:, error:,
+    # message:, delay:}) once per retried attempt — nothing else in the gem
+    # otherwise surfaces transient network failures beyond a debug log line,
+    # so a caller wanting to alert/count on flaky-connection retries has
+    # somewhere to hook in. Not fired on the final, re-raised failure — by
+    # that point it's an ordinary exception the caller already sees directly.
+    sig { params(header: T::Array[String], block: T.proc.returns(T.untyped)).returns(T.untyped) }
+    def with_retries(header: [], &block)
+      attempts = 0
+      begin
+        attempts += 1
+        block.call
+      rescue *RETRYABLE_ERRORS => e
+        raise if attempts > options.retries
+        delay = options.retry_delay * attempts
+        debug("attempt #{attempts} failed (#{e.class}: #{e.message}), retrying in #{delay}s", header: header)
+        ActiveSupport::Notifications.instrument("retry.e621_export_downloader", header: header, attempt: attempts, error: e.class.name, message: e.message, delay: delay)
+        sleep(delay)
+        retry
+      end
+    end
+
     sig { params(type: T.any(Types, String)).returns(Export[T.untyped]) }
     def get(type)
       type = string_to_type(type) if type.is_a?(String)
@@ -58,7 +90,7 @@ module E621ExportDownloader
     def get_data
       return @export_cache unless @export_cache.nil?
       debug("fetching export data from api")
-      res = connection.get(Constants::API_URL)
+      res = with_retries(header: ["export_data"]) { connection.get(Constants::API_URL) }
       raise(ResolveError, "Failed to fetch exports: #{res.status} #{res.reason_phrase}") unless res.success?
       data = JSON.parse(T.unsafe(res).body)
       result = T.let(data.map do |d|

data/lib/e621_export_downloader/export.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 # typed: strict
 
+require("active_support/notifications")
 require("csv")
 require("date")
 require("digest")
@@ -44,6 +45,14 @@ module E621ExportDownloader
       true
     end
 
+    # Instruments "download.e621_export_downloader" (payload: {type:, file_size:,
+    # validate_checksum:}) around the actual network fetch — not the cached-file
+    # early return above, so a subscriber only ever sees this fire for a real
+    # download. Any error (a failed response, a checksum mismatch, exhausted
+    # retries, ...) propagates normally; ActiveSupport::Notifications still
+    # publishes the event when the block raises, with the exception attached
+    # to the payload, so a subscriber doesn't need its own rescue to notice a
+    # failed download.
     sig { params(block: T.nilable(T.proc.params(received: Integer, total: Integer).void)).returns(String) }
     def download(&block)
       raise(ResolveError, "Export #{type.serialize} does not exist") unless exists?
@@ -55,33 +64,9 @@ module E621ExportDownloader
       client.debug("downloading export", header: ["export:#{type.serialize}"])
 
       FileUtils.mkdir_p(Constants::TEMP_DIR)
-      digest = T.let(client.options.validate_checksum ? Digest::MD5.new : nil, T.nilable(Digest::MD5))
-      File.open(file_path, "wb") do |file|
-        inflater = Zlib::Inflate.new(Zlib::MAX_WBITS + 16)
-
-        res = client.connection.get(data.url) do |req|
-          req.options.on_data = proc do |chunk, received|
-            digest&.update(chunk)
-            decompressed = inflater.inflate(chunk)
-            file.write(decompressed) if decompressed && !decompressed.empty?
-            block.call(received, data.file_size) if block
-          end
-        end
-
-        file.write(inflater.finish)
-        inflater.close
-        file.close
-
-        unless res.success?
-          raise(ResolveError, "Failed to download export #{type.serialize}: #{res.status} #{res.reason_phrase}")
-        end
-
-        if digest && digest.hexdigest != data.checksum
-          raise(ResolveError, "Checksum mismatch for export #{type.serialize}")
-        end
-      rescue # rubocop:disable Style/RescueStandardError
-        FileUtils.rm_f(file_path)
-        raise
+      payload = { type: type.serialize, file_size: data.file_size, validate_checksum: client.options.validate_checksum }
+      ActiveSupport::Notifications.instrument("download.e621_export_downloader", payload) do
+        client.with_retries(header: ["export:#{type.serialize}"]) { perform_download(&block) }
       end
 
       @downloaded = true
@@ -91,7 +76,8 @@ module E621ExportDownloader
 
     sig { returns(T::Boolean) }
     def exists?
-      result = client.connection.head(data.url).success?
+      res = client.with_retries(header: ["export:#{type.serialize}"]) { client.connection.head(data.url) }
+      result = res.success?
       client.debug("checked export existence: #{result}", header: ["export:#{type.serialize}"])
       result
     end
@@ -122,6 +108,38 @@ module E621ExportDownloader
 
     private
 
+    sig { params(block: T.nilable(T.proc.params(received: Integer, total: Integer).void)).void }
+    def perform_download(&block)
+      digest = T.let(client.options.validate_checksum ? Digest::SHA256.new : nil, T.nilable(Digest::SHA256))
+      File.open(file_path, "wb") do |file|
+        inflater = Zlib::Inflate.new(Zlib::MAX_WBITS + 16)
+
+        res = client.connection.get(data.url) do |req|
+          req.options.on_data = proc do |chunk, received|
+            digest&.update(chunk)
+            decompressed = inflater.inflate(chunk)
+            file.write(decompressed) if decompressed && !decompressed.empty?
+            block.call(received, data.file_size) if block
+          end
+        end
+
+        file.write(inflater.finish)
+        inflater.close
+        file.close
+
+        unless res.success?
+          raise(ResolveError, "Failed to download export #{type.serialize}: #{res.status} #{res.reason_phrase}")
+        end
+
+        if digest && digest.hexdigest != data.checksum
+          raise(ResolveError, "Checksum mismatch for export #{type.serialize}")
+        end
+      rescue # rubocop:disable Style/RescueStandardError
+        FileUtils.rm_f(file_path)
+        raise
+      end
+    end
+
     sig { returns(String) }
     def format_date
       DateTime.parse(data.updated_at).strftime("%Y-%m-%d")

data/lib/e621_export_downloader/version.rb

@@ -4,7 +4,7 @@
 # loaded by bundler
 module E621ExportDownloader
   module Constants
-    VERSION = "0.0.18"
+    VERSION = "0.0.20"
     WEBSITE = "https://github.com/DonovanDMC/E621ExportDownloader.rb"
   end
 end

data/sorbet/rbi/shims/faraday.rbi

@@ -1,6 +1,12 @@
 # typed: true
 
 module Faraday
+  class Error < StandardError; end
+  class ServerError < Error; end
+  class ConnectionFailed < Error; end
+  class TimeoutError < ServerError; end
+  class SSLError < Error; end
+
   class Request
     sig { returns(Symbol) }
     def http_method; end

data/test/csv_importable_postgres_test.rb

@@ -0,0 +1,154 @@
+# frozen_string_literal: true
+
+require("minitest/autorun")
+require("tempfile")
+require("active_record")
+require("active_support/notifications")
+require("e621_export_downloader")
+require("e621/csv_importable")
+require("e621/row_count")
+
+class CsvImportablePostgresTestModel < ActiveRecord::Base
+  extend(E621::CsvImportable)
+
+  self.table_name = "import_index_test_tags"
+  self.record_timestamps = false
+end
+
+class CsvImportablePostgresTest < Minitest::Test
+  INDEX_NAME = "index_import_index_test_tags_on_name"
+
+  def setup
+    skip("set E621_TEST_DATABASE_URL to run PostgreSQL import tests") unless ENV["E621_TEST_DATABASE_URL"]
+
+    ActiveRecord::Base.establish_connection(ENV.fetch("E621_TEST_DATABASE_URL"))
+    @tempfiles = []
+    setup_schema
+  end
+
+  def teardown
+    return unless ENV["E621_TEST_DATABASE_URL"]
+
+    connection.execute("DROP TABLE IF EXISTS import_index_test_tags")
+    connection.execute("DROP TABLE IF EXISTS e621.row_counts")
+    @tempfiles&.each(&:unlink)
+  end
+
+  def test_recreate_indexes_restores_postgres_indexes_after_successful_import
+    count = CsvImportablePostgresTestModel.import_from_csv(csv_file(successful_csv), truncate: true, recreate_indexes: true, chunk_bytes: 8)
+
+    assert_equal(2, count)
+    assert_equal(2, CsvImportablePostgresTestModel.count)
+    assert_index_exists
+  end
+
+  def test_recreate_indexes_restores_postgres_indexes_after_import_error
+    error = assert_raises(StandardError) do
+      CsvImportablePostgresTestModel.import_from_csv(csv_file(invalid_csv), truncate: true, recreate_indexes: true, chunk_bytes: 8)
+    end
+
+    assert_match(/COPY|missing data|extra data|invalid input/i, error.message)
+    assert_index_exists
+  end
+
+  def test_recreate_indexes_instruments_create_index_notification_per_index
+    events = []
+    callback = ->(_name, _start, _finish, _id, payload) { events << payload }
+
+    ActiveSupport::Notifications.subscribed(callback, "create_index.e621_export_downloader") do
+      CsvImportablePostgresTestModel.import_from_csv(csv_file(successful_csv), truncate: true, recreate_indexes: true, chunk_bytes: 8)
+    end
+
+    assert_equal(1, events.size)
+    assert_equal("import_index_test_tags", events.first[:table])
+    assert_equal(INDEX_NAME, events.first[:index])
+  end
+
+  def test_import_from_csv_instruments_truncate_drop_index_and_copy_notifications
+    truncate_events = []
+    drop_index_events = []
+    copy_events = []
+    subscriptions = [
+      ActiveSupport::Notifications.subscribe("truncate.e621_export_downloader") { |*, payload| truncate_events << payload },
+      ActiveSupport::Notifications.subscribe("drop_index.e621_export_downloader") { |*, payload| drop_index_events << payload },
+      ActiveSupport::Notifications.subscribe("copy.e621_export_downloader") { |*, payload| copy_events << payload },
+    ]
+
+    CsvImportablePostgresTestModel.import_from_csv(csv_file(successful_csv), truncate: true, recreate_indexes: true, chunk_bytes: 8)
+
+    assert_equal(1, truncate_events.size)
+    assert_equal("import_index_test_tags", truncate_events.first[:table])
+
+    assert_equal(1, drop_index_events.size)
+    assert_equal(INDEX_NAME, drop_index_events.first[:index])
+
+    assert_equal(1, copy_events.size)
+    assert_equal(2, copy_events.first[:rows])
+  ensure
+    subscriptions&.each { |s| ActiveSupport::Notifications.unsubscribe(s) }
+  end
+
+  private
+
+  def setup_schema
+    connection.execute("CREATE SCHEMA IF NOT EXISTS e621")
+    connection.execute(<<~SQL)
+      CREATE TABLE IF NOT EXISTS e621.row_counts (
+        table_name varchar PRIMARY KEY,
+        count bigint NOT NULL DEFAULT 0
+      )
+    SQL
+    connection.execute("DROP TABLE IF EXISTS import_index_test_tags")
+    connection.execute(<<~SQL)
+      CREATE TABLE import_index_test_tags (
+        id bigint PRIMARY KEY,
+        name text NOT NULL,
+        post_count integer NOT NULL
+      )
+    SQL
+    connection.execute("CREATE INDEX #{INDEX_NAME} ON import_index_test_tags (name)")
+  end
+
+  def successful_csv
+    <<~CSV
+      id,name,post_count
+      1,alpha,10
+      2,beta,20
+    CSV
+  end
+
+  def invalid_csv
+    <<~CSV
+      id,name,post_count
+      1,alpha
+    CSV
+  end
+
+  def csv_file(content)
+    file = Tempfile.new(["e621-import", ".csv"])
+    file.write(content)
+    file.close
+    @tempfiles << file
+    file.path
+  end
+
+  def assert_index_exists
+    assert(index_exists?, "expected PostgreSQL index #{INDEX_NAME} to exist")
+  end
+
+  def index_exists?
+    connection.select_value(<<~SQL)
+      SELECT EXISTS (
+        SELECT 1
+        FROM pg_class c
+        JOIN pg_namespace n ON n.oid = c.relnamespace
+        WHERE c.relkind = 'i'
+          AND c.relname = #{connection.quote(INDEX_NAME)}
+      )
+    SQL
+  end
+
+  def connection
+    ActiveRecord::Base.connection
+  end
+end

data/test/export_test.rb

@@ -39,7 +39,7 @@ class ExportChecksumTest < Minitest::Test
   def setup
     @csv_content = "id,name\n1,foo\n"
     @compressed = Zlib.gzip(@csv_content)
-    @checksum = Digest::MD5.hexdigest(@compressed)
+    @checksum = Digest::SHA256.hexdigest(@compressed)
     # split across chunks to exercise streaming, matching how a real download arrives
     midpoint = @compressed.bytesize / 2
     @chunks = [@compressed.byteslice(0, midpoint), @compressed.byteslice(midpoint..)]
@@ -79,14 +79,14 @@ class ExportChecksumTest < Minitest::Test
   end
 
   def test_download_raises_and_removes_the_file_when_checksum_does_not_match
-    export = build_export(checksum: "0" * 32, validate_checksum: true)
+    export = build_export(checksum: "0" * 64, validate_checksum: true)
     error = assert_raises(E621ExportDownloader::ResolveError) { export.download }
     assert_match(/Checksum mismatch/, error.message)
     assert_empty(Dir.children(E621ExportDownloader::Constants::TEMP_DIR))
   end
 
   def test_download_ignores_checksum_mismatch_when_validation_is_disabled
-    export = build_export(checksum: "0" * 32, validate_checksum: false)
+    export = build_export(checksum: "0" * 64, validate_checksum: false)
     path = export.download
     assert(File.exist?(path))
   end

data/test/retry_test.rb

@@ -0,0 +1,178 @@
+# frozen_string_literal: true
+
+require("minitest/autorun")
+require("active_support/notifications")
+require("digest")
+require("fileutils")
+require("zlib")
+require("e621_export_downloader")
+
+# A connection double whose #get fails with a transient Faraday error a configurable
+# number of times before succeeding, to exercise the download retry/reset behavior.
+# #head always succeeds so failures are isolated to the streaming GET path.
+class FlakyConnection
+  def initialize(fail_times:, chunks: [])
+    @fail_times = fail_times
+    @get_attempts = 0
+    @chunks = chunks
+  end
+
+  def head(_url)
+    Struct.new(:success?).new(true)
+  end
+
+  def get(_url)
+    @get_attempts += 1
+    raise(Faraday::ConnectionFailed, "transient dns failure") if @get_attempts <= @fail_times
+
+    req = Struct.new(:options).new(Struct.new(:on_data).new)
+    yield(req)
+
+    received = 0
+    @chunks.each do |chunk|
+      received += chunk.bytesize
+      req.options.on_data.call(chunk, received)
+    end
+
+    Struct.new(:status, :reason_phrase, :success?).new(200, "OK", true)
+  end
+end
+
+class WithRetriesTest < Minitest::Test
+  def build_client(retries: 2)
+    E621ExportDownloader::Client.new(
+      E621ExportDownloader::Client::Options.new(retries: retries, retry_delay: 0),
+    )
+  end
+
+  def test_with_retries_recovers_after_transient_failures_within_the_retry_budget
+    client = build_client(retries: 2)
+    attempts = 0
+    result = client.with_retries do
+      attempts += 1
+      raise(Faraday::ConnectionFailed, "boom") if attempts < 3
+      :ok
+    end
+    assert_equal(:ok, result)
+    assert_equal(3, attempts)
+  end
+
+  def test_with_retries_raises_once_the_retry_budget_is_exhausted
+    client = build_client(retries: 1)
+    attempts = 0
+    assert_raises(Faraday::ConnectionFailed) do
+      client.with_retries do
+        attempts += 1
+        raise(Faraday::ConnectionFailed, "boom")
+      end
+    end
+    assert_equal(2, attempts)
+  end
+
+  def test_with_retries_does_not_retry_non_transient_errors
+    client = build_client(retries: 5)
+    attempts = 0
+    assert_raises(ArgumentError) do
+      client.with_retries do
+        attempts += 1
+        raise(ArgumentError, "not a network error")
+      end
+    end
+    assert_equal(1, attempts)
+  end
+
+  def test_with_retries_instruments_a_retry_notification_per_retried_attempt_only
+    client = build_client(retries: 2)
+    events = []
+    callback = ->(_name, _start, _finish, _id, payload) { events << payload }
+
+    attempts = 0
+    ActiveSupport::Notifications.subscribed(callback, "retry.e621_export_downloader") do
+      client.with_retries(header: ["test"]) do
+        attempts += 1
+        raise(Faraday::ConnectionFailed, "boom") if attempts < 3
+        :ok
+      end
+    end
+
+    assert_equal(2, events.size)
+    assert_equal([1, 2], events.map { |e| e[:attempt] })
+    assert_equal(["test"], events.first[:header])
+    assert_equal("Faraday::ConnectionFailed", events.first[:error])
+  end
+end
+
+class ExportRetryTest < Minitest::Test
+  def setup
+    @csv_content = "id,name\n1,foo\n"
+    @compressed = Zlib.gzip(@csv_content)
+  end
+
+  def teardown
+    FileUtils.rm_rf(E621ExportDownloader::Constants::TEMP_DIR)
+  end
+
+  def build_export(connection, retries: 2)
+    data = E621ExportDownloader::APIExportData.new(
+      name:       E621ExportDownloader::Types::Posts,
+      file_name:  "posts-retry-test.csv",
+      file_size:  @compressed.bytesize,
+      checksum:   Digest::SHA256.hexdigest(@compressed),
+      updated_at: "2026-01-01",
+      url:        "https://example.com/posts.csv.gz",
+    )
+    client = E621ExportDownloader::Client.new(
+      E621ExportDownloader::Client::Options.new(retries: retries, retry_delay: 0),
+    )
+    client.define_singleton_method(:connection) { connection }
+    E621ExportDownloader::Export.new(
+      data:   data,
+      client: client,
+      type:   E621ExportDownloader::Types::Posts,
+      parser: ->(record) { record },
+    )
+  end
+
+  def test_download_recovers_from_a_transient_failure_and_produces_a_correct_file
+    connection = FlakyConnection.new(fail_times: 1, chunks: [@compressed])
+    export = build_export(connection, retries: 2)
+    path = export.download
+    assert(File.exist?(path))
+    assert_equal(@csv_content, File.read(path))
+  end
+
+  def test_download_gives_up_and_raises_after_the_retry_budget_is_exhausted
+    connection = FlakyConnection.new(fail_times: 5, chunks: [@compressed])
+    export = build_export(connection, retries: 1)
+    assert_raises(Faraday::ConnectionFailed) { export.download }
+    assert_empty(Dir.children(E621ExportDownloader::Constants::TEMP_DIR))
+  end
+
+  def test_download_instruments_a_download_notification_on_success
+    connection = FlakyConnection.new(fail_times: 0, chunks: [@compressed])
+    export = build_export(connection, retries: 2)
+    events = []
+    callback = ->(_name, _start, _finish, _id, payload) { events << payload }
+
+    ActiveSupport::Notifications.subscribed(callback, "download.e621_export_downloader") { export.download }
+
+    assert_equal(1, events.size)
+    assert_equal("posts", events.first[:type])
+    assert_equal(@compressed.bytesize, events.first[:file_size])
+    assert_nil(events.first[:exception])
+  end
+
+  def test_download_instruments_a_download_notification_with_the_exception_on_failure
+    connection = FlakyConnection.new(fail_times: 5, chunks: [@compressed])
+    export = build_export(connection, retries: 1)
+    events = []
+    callback = ->(_name, _start, _finish, _id, payload) { events << payload }
+
+    ActiveSupport::Notifications.subscribed(callback, "download.e621_export_downloader") do
+      assert_raises(Faraday::ConnectionFailed) { export.download }
+    end
+
+    assert_equal(1, events.size)
+    assert_equal("Faraday::ConnectionFailed", events.first[:exception]&.first)
+  end
+end

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: e621_export_downloader
 version: !ruby/object:Gem::Version
-  version: 0.0.18
+  version: 0.0.20
 platform: ruby
 authors:
 - Donovan_DMC
 bindir: exe
 cert_chain: []
-date: 2026-07-05 00:00:00.000000000 Z
+date: 2026-08-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: csv
@@ -223,8 +223,10 @@ files:
 - sorbet/rbi/todo.rbi
 - sorbet/tapioca/config.yml
 - sorbet/tapioca/require.rb
+- test/csv_importable_postgres_test.rb
 - test/export_test.rb
 - test/generators/install_generator_test.rb
+- test/retry_test.rb
 homepage: https://github.com/DonovanDMC/E621ExportDownloader.rb
 licenses:
 - MIT