e621_export_downloader 0.0.3 → 0.0.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ef31343295188d4e90bc9c826cc0c1df810b9a93ee5a1ff99b616c45a08741c6
-  data.tar.gz: 000ddd8a6db8c17687703730643a64eda21d8562a3fcef10ad2b4aaf7a90f383
+  metadata.gz: a0319ab366f0396847426a4d510dfe49c7a6a5beb49175001b3e4a56fe2e6aa9
+  data.tar.gz: f40878e3cb2253774caaaabb83aa0b0aa014ed648782835946b212f24a32a87f
 SHA512:
-  metadata.gz: fdf9d2410b89b5420ca23cd7df1f35d124bccb86628e3a5c26afbdd863679f159172332f8f20131e279506c22ede1fce2452bc04514ce159ec3dc924c559ca27
-  data.tar.gz: fd2d7e0d94eb0bc2c2f3ec4c9ff12e23185a58facc12fa01d2589315c27d5b1d0e05f90b2243ba6e5664d56901cbbafe76a29520bc478165e5997a0ef501c99d
+  metadata.gz: 1dbb3fea2fa77735294a73049b9684632e5fb96d1b03891199556d6fb5501999ec1b5e53b0d6a31bcc3ba60399fbba347c81a669faed08c1f59b031325297d1e
+  data.tar.gz: 8585b196a8cdb998a07444e4306e293ebde1f2f4ddb2bb08f26043f7d942cce3d1a041d523dccd15c0d9ca51d85902e3b91456e71e47821103754257e2d786c2

data/README.md

@@ -23,51 +23,39 @@ client = E621ExportDownloader::Client.new
 
 # configure options after creation
 client.config do |c|
-  c.cache = true              # keep export files after reading, defaults to true
-  c.rewind_on_not_found = 2  # decrease date by one day if no export is found for that date,
-                              # provide an integer to limit how many days can be rewound,
-                              # `true` is equivalent to `2`, defaults to false
+  c.cache = true  # keep export files after reading, defaults to false
 end
 
 # or pass an Options struct directly
 client = E621ExportDownloader::Client.new(
-  E621ExportDownloader::Client::Options.new(
-    cache:              true,
-    rewind_on_not_found: 2,
-  )
+  E621ExportDownloader::Client::Options.new(cache: true)
 )
 
-# get the helper for interacting with a type of export
-# types: pools, posts, tag_aliases, tag_implications, tags, wiki_pages
-helper = client.get("posts")
-# or use the named shorthand:
-helper = client.posts
+# get the export for a type — resolves the latest available export from the e621 API
+# types: artists, bulk_update_requests, pools, posts, post_replacements, post_versions, tag_aliases, tag_implications, tags, wiki_pages
+export = client.get("posts")
 
-# get a wrapper for a specific date (time components are ignored)
-# using this directly will not trigger rewinding regardless of rewind_on_not_found
-export = helper.get(Date.today)
+# get a deferred export — the API call is made lazily on first use
+deferred = client.get_deferred("posts")
 
-# convenience shorthand on the client — also skips rewinding
-export = client.get_posts(Date.today)
+# get raw metadata for all exports from the e621 API
+# returns an Array of E621ExportDownloader::APIExportData: { file_name, file_size, name, updated_at, url }
+data = client.get_data
 
-# all of the following methods can also be called on the helper with a date argument;
-# if rewind_on_not_found is enabled the helper decrements the date by one day until it
-# finds an export or exhausts the rewind limit, at which point it raises ResolveError
+# access the metadata for this specific export: { file_name, file_size, name, updated_at, url }
+export.data
 
-# check whether the export exists for the date
+# check if the export exists
 exists = export.exists?
-exists = helper.exists?(Date.today)
 
 # download the export, returns the file path — not required before reading
 # if you move or remove the file do not reuse the export object
 file = export.download
-file = helper.download(Date.today)
 
 # delete the downloaded file, if it exists
 export.delete
-helper.delete(Date.today)
 
-# get all of the records as a single array, DO NOT use this for large exports, arrays will millions of items do not perform well and will likely crash your process!
+# get all of the records as a single array, DO NOT use this for large exports, arrays with millions of items do not perform well and will likely crash your process!
 # (not to mention that the posts export is more than 5 gigabytes)
 records = export.read_all
 
@@ -98,45 +86,52 @@ client.config do |c|
 end
 ```
 
+Available parser keys: `artists`, `bulk_update_requests`, `pools`, `posts`, `post_replacements`, `post_versions`, `tag_aliases`, `tag_implications`, `tags`, `wiki_pages`
+
 ## Models
 
 Each export type is parsed into a corresponding model class:
 
-| Export type       | Model class                                       |
-|-------------------|---------------------------------------------------|
-| `pools`           | `E621ExportDownloader::Models::Pool`              |
-| `posts`           | `E621ExportDownloader::Models::Post`              |
-| `tag_aliases`     | `E621ExportDownloader::Models::TagAlias`          |
-| `tag_implications`| `E621ExportDownloader::Models::TagImplication`    |
-| `tags`            | `E621ExportDownloader::Models::Tag`               |
-| `wiki_pages`      | `E621ExportDownloader::Models::WikiPage`          |
+| Export type            | Model class                                           |
+|------------------------|-------------------------------------------------------|
+| `artists`              | `E621ExportDownloader::Models::Artist`                |
+| `bulk_update_requests` | `E621ExportDownloader::Models::BulkUpdateRequest`     |
+| `pools`                | `E621ExportDownloader::Models::Pool`                  |
+| `posts`                | `E621ExportDownloader::Models::Post`                  |
+| `post_replacements`    | `E621ExportDownloader::Models::PostReplacement`       |
+| `post_versions`        | `E621ExportDownloader::Models::PostVersion`           |
+| `tag_aliases`          | `E621ExportDownloader::Models::TagAlias`              |
+| `tag_implications`     | `E621ExportDownloader::Models::TagImplication`        |
+| `tags`                 | `E621ExportDownloader::Models::Tag`                   |
+| `wiki_pages`           | `E621ExportDownloader::Models::WikiPage`              |
 
 ## CLI
 
 ```bash
-# check if an export exists for a given date; date is optional and defaults to today
+# get export metadata from the e621 API as a JSON array
+# outputs { file_name, file_size, name, updated_at, url }[] with no trailing newline
+e621-export-downloader data
+
+# check if an export exists
 # outputs "true" or "false" with no trailing newline
-e621-export-downloader exists posts 2024-01-01
+e621-export-downloader exists posts
 
-# download an export for a given date; date is optional and defaults to today
+# download an export
 # outputs the path to the downloaded file with no trailing newline
-e621-export-downloader download posts 2024-01-01
+e621-export-downloader download posts
 
-# read an export as individual JSON lines; date is optional and defaults to today
+# read an export as individual JSON lines
 # outputs each record as a JSON object on its own line
-e621-export-downloader read posts 2024-01-01
+e621-export-downloader read posts
 
-# read an export as a JSON array; date is optional and defaults to today
+# read an export as a JSON array
 # streams the CSV internally, so it is safe to use with large exports
-e621-export-downloader read-all posts 2024-01-01
+e621-export-downloader read-all posts
 
 e621-export-downloader --help
 e621-export-downloader --version
-e621-export-downloader --cache                    # enable caching (default)
-e621-export-downloader --no-cache                 # disable caching
-e621-export-downloader --rewind-on-not-found      # enable rewinding (up to 2 days)
-e621-export-downloader --rewind-on-not-found 5    # enable rewinding (up to 5 days)
-e621-export-downloader --no-rewind-on-not-found   # disable rewinding (default)
+e621-export-downloader --cache     # enable caching
+e621-export-downloader --no-cache  # disable caching (default)
 ```
 
 ## ActiveJob Integration

data/lib/e621_export_downloader/api_export_data.rb

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+# typed: strict
+
+module E621ExportDownloader
+  class APIExportData < T::Struct
+    const(:file_name, String)
+    const(:file_size, Integer)
+    const(:name, Types)
+    const(:updated_at, String)
+    const(:url, String)
+  end
+end

data/lib/e621_export_downloader/client/options/builder/parsers.rb

@@ -8,12 +8,24 @@ module E621ExportDownloader
         class Parsers
           extend(T::Sig)
 
+          sig { returns(Parser) }
+          attr_accessor(:artists)
+
+          sig { returns(Parser) }
+          attr_accessor(:bulk_update_requests)
+
           sig { returns(Parser) }
           attr_accessor(:pools)
 
           sig { returns(Parser) }
           attr_accessor(:posts)
 
+          sig { returns(Parser) }
+          attr_accessor(:post_replacements)
+
+          sig { returns(Parser) }
+          attr_accessor(:post_versions)
+
           sig { returns(Parser) }
           attr_accessor(:tag_aliases)
 
@@ -28,12 +40,16 @@ module E621ExportDownloader
 
           sig { params(defaults: Options::Parsers).void }
           def initialize(defaults = Options::Parsers.defaults)
-            @pools = T.let(T.must(defaults.pools), Parser)
-            @posts = T.let(T.must(defaults.posts), Parser)
-            @tag_aliases = T.let(T.must(defaults.tag_aliases), Parser)
-            @tag_implications = T.let(T.must(defaults.tag_implications), Parser)
-            @tags = T.let(T.must(defaults.tags), Parser)
-            @wiki_pages = T.let(T.must(defaults.wiki_pages), Parser)
+            @artists              = T.let(T.must(defaults.artists), Parser)
+            @bulk_update_requests = T.let(T.must(defaults.bulk_update_requests), Parser)
+            @pools                = T.let(T.must(defaults.pools), Parser)
+            @posts                = T.let(T.must(defaults.posts), Parser)
+            @post_replacements    = T.let(T.must(defaults.post_replacements), Parser)
+            @post_versions        = T.let(T.must(defaults.post_versions), Parser)
+            @tag_aliases          = T.let(T.must(defaults.tag_aliases), Parser)
+            @tag_implications     = T.let(T.must(defaults.tag_implications), Parser)
+            @tags                 = T.let(T.must(defaults.tags), Parser)
+            @wiki_pages           = T.let(T.must(defaults.wiki_pages), Parser)
           end
         end
       end

data/lib/e621_export_downloader/client/options/builder.rb

@@ -10,16 +10,12 @@ module E621ExportDownloader
         sig { returns(T::Boolean) }
         attr_accessor(:cache)
 
-        sig { returns(T.any(T::Boolean, Integer)) }
-        attr_accessor(:rewind_on_not_found)
-
         sig { params(parsers: Options::Parsers).returns(Options::Parsers) }
         attr_writer(:parsers)
 
         sig { params(defaults: Options).void }
         def initialize(defaults = Options.new)
           @cache = T.let(defaults.cache, T::Boolean)
-          @rewind_on_not_found = T.let(defaults.rewind_on_not_found, T.any(T::Boolean, Integer))
           @parsers = T.let(defaults.parsers, Options::Parsers)
         end
 
@@ -30,12 +26,16 @@ module E621ExportDownloader
           block.call(parsers)
 
           @parsers = Options::Parsers.new(
-            pools:            parsers.pools,
-            posts:            parsers.posts,
-            tag_aliases:      parsers.tag_aliases,
-            tag_implications: parsers.tag_implications,
-            tags:             parsers.tags,
-            wiki_pages:       parsers.wiki_pages,
+            artists:              parsers.artists,
+            bulk_update_requests: parsers.bulk_update_requests,
+            pools:                parsers.pools,
+            posts:                parsers.posts,
+            post_replacements:    parsers.post_replacements,
+            post_versions:        parsers.post_versions,
+            tag_aliases:          parsers.tag_aliases,
+            tag_implications:     parsers.tag_implications,
+            tags:                 parsers.tags,
+            wiki_pages:           parsers.wiki_pages,
           )
         end
       end

data/lib/e621_export_downloader/client/options.rb

@@ -9,8 +9,12 @@ module E621ExportDownloader
       class Parsers < T::Struct
         extend(T::Sig)
 
+        const(:artists, T.nilable(Parser), default: nil)
+        const(:bulk_update_requests, T.nilable(Parser), default: nil)
         const(:pools, T.nilable(Parser), default: nil)
         const(:posts, T.nilable(Parser), default: nil)
+        const(:post_replacements, T.nilable(Parser), default: nil)
+        const(:post_versions, T.nilable(Parser), default: nil)
         const(:tag_aliases, T.nilable(Parser), default: nil)
         const(:tag_implications, T.nilable(Parser), default: nil)
         const(:tags, T.nilable(Parser), default: nil)
@@ -19,18 +23,21 @@ module E621ExportDownloader
         sig { returns(Parsers) }
         def self.defaults
           Parsers.new(
-            pools:            ->(record) { Models::Pool.new(record) },
-            posts:            ->(record) { Models::Post.new(record) },
-            tag_aliases:      ->(record) { Models::TagAlias.new(record) },
-            tag_implications: ->(record) { Models::TagImplication.new(record) },
-            tags:             ->(record) { Models::Tag.new(record) },
-            wiki_pages:       ->(record) { Models::WikiPage.new(record) },
+            artists:              ->(record) { Models::Artist.new(record) },
+            bulk_update_requests: ->(record) { Models::BulkUpdateRequest.new(record) },
+            pools:                ->(record) { Models::Pool.new(record) },
+            posts:                ->(record) { Models::Post.new(record) },
+            post_replacements:    ->(record) { Models::PostReplacement.new(record) },
+            post_versions:        ->(record) { Models::PostVersion.new(record) },
+            tag_aliases:          ->(record) { Models::TagAlias.new(record) },
+            tag_implications:     ->(record) { Models::TagImplication.new(record) },
+            tags:                 ->(record) { Models::Tag.new(record) },
+            wiki_pages:           ->(record) { Models::WikiPage.new(record) },
           )
         end
       end
 
-      const(:cache, T::Boolean, default: true)
-      const(:rewind_on_not_found, T.any(T::Boolean, Integer), default: false)
+      const(:cache, T::Boolean, default: false)
       const(:parsers, Parsers, default: Parsers.defaults)
     end
   end

data/lib/e621_export_downloader/client.rb

@@ -2,6 +2,7 @@
 # typed: strict
 
 require("faraday")
+require("json")
 
 module E621ExportDownloader
   class Client
@@ -15,6 +16,7 @@ module E621ExportDownloader
     sig { params(options: T.nilable(Options)).void }
     def initialize(options = nil)
       @options = T.let(options || Options.new, Options)
+      @export_cache = T.let(nil, T.nilable(T::Array[APIExportData]))
     end
 
     sig { params(block: T.proc.params(arg0: Options::Builder).void).void }
@@ -24,7 +26,7 @@ module E621ExportDownloader
 
     sig { returns(Faraday::Connection) }
     def connection
-      Faraday.new(url: Constants::BASE_URL, headers: { "User-Agent" => Constants::USER_AGENT })
+      Faraday.new(headers: { "User-Agent" => Constants::USER_AGENT })
     end
 
     sig { params(msg: String, header: T::Array[String]).void }
@@ -32,89 +34,60 @@ module E621ExportDownloader
       Logger.debug("[e621_export_downloader#{":#{header.join(':')}" unless header.empty?}] #{msg}")
     end
 
-    sig { params(type: T.any(Types, String)).returns(ExportHelper[T.untyped]) }
+    sig { params(type: T.any(Types, String)).returns(Export[T.untyped]) }
     def get(type)
-      if type.is_a?(String)
-        case type
-        when "pools"
-          type = Types::Pools
-        when "posts"
-          type = Types::Posts
-        when "tag_aliases"
-          type = Types::TagAliases
-        when "tag_implications"
-          type = Types::TagImplications
-        when "tags"
-          type = Types::Tags
-        when "wiki_pages"
-          type = Types::WikiPages
-        else
-          raise(ArgumentError, "invalid type: #{type}")
-        end
-      end
-
+      type = string_to_type(type) if type.is_a?(String)
       T.assert_type!(type, Types)
-      ExportHelper.new(client: self, type: type, parser: options.parsers.public_send(type.serialize))
-    end
-
-    sig { returns(ExportHelper[Models::Pool]) }
-    def pools
-      get(Types::Pools)
-    end
-
-    sig { params(date: T.any(Date, DateTime)).returns(Export[Models::Pool]) }
-    def get_pools(date = Date.today)
-      pools.get(date)
-    end
-
-    sig { returns(ExportHelper[Models::Post]) }
-    def posts
-      get(Types::Posts)
-    end
-
-    sig { params(date: T.any(Date, DateTime)).returns(Export[Models::Post]) }
-    def get_posts(date = Date.today)
-      posts.get(date)
-    end
-
-    sig { returns(ExportHelper[Models::TagAlias]) }
-    def tag_aliases
-      get(Types::TagAliases)
-    end
-
-    sig { params(date: T.any(Date, DateTime)).returns(Export[Models::TagAlias]) }
-    def get_tag_aliases(date = Date.today)
-      tag_aliases.get(date)
-    end
-
-    sig { returns(ExportHelper[Models::TagImplication]) }
-    def tag_implications
-      get(Types::TagImplications)
-    end
-
-    sig { params(date: T.any(Date, DateTime)).returns(Export[Models::TagImplication]) }
-    def get_tag_implications(date = Date.today)
-      tag_implications.get(date)
-    end
-
-    sig { returns(ExportHelper[Models::Tag]) }
-    def tags
-      get(Types::Tags)
+      data = get_data.find { |d| d.name == type }
+      raise(ResolveError, "Export data for \"#{type.serialize}\" not found") if data.nil?
+      debug("creating export for #{type.serialize}")
+      Export.new(data: data, client: self, type: type, parser: options.parsers.public_send(type.serialize))
     end
 
-    sig { params(date: T.any(Date, DateTime)).returns(Export[Models::Tag]) }
-    def get_tags(date = Date.today)
-      tags.get(date)
-    end
-
-    sig { returns(ExportHelper[Models::WikiPage]) }
-    def wiki_pages
-      get(Types::WikiPages)
-    end
-
-    sig { params(date: T.any(Date, DateTime)).returns(Export[Models::WikiPage]) }
-    def get_wiki_pages(date = Date.today)
-      wiki_pages.get(date)
+    sig { params(type: T.any(Types, String)).returns(ExportHelper[T.untyped]) }
+    def get_deferred(type)
+      type = string_to_type(type) if type.is_a?(String)
+      T.assert_type!(type, Types)
+      debug("creating deferred export for #{type.serialize}")
+      ExportHelper.new(type: type, client: self)
+    end
+
+    sig { returns(T::Array[APIExportData]) }
+    def get_data
+      return @export_cache unless @export_cache.nil?
+      debug("fetching export data from api")
+      res = connection.get(Constants::API_URL)
+      raise(ResolveError, "Failed to fetch exports: #{res.status} #{res.reason_phrase}") unless res.success?
+      data = JSON.parse(T.unsafe(res).body)
+      result = T.let(data.map do |d|
+        APIExportData.new(
+          file_name:  d["file_name"],
+          file_size:  d["file_size"].to_i,
+          name:       Types.deserialize(d["name"]),
+          updated_at: d["updated_at"],
+          url:        d["url"],
+        )
+      end, T::Array[APIExportData])
+      @export_cache = result
+    end
+
+    private
+
+    sig { params(type: String).returns(Types) }
+    def string_to_type(type)
+      case type
+      when "artists"              then Types::Artists
+      when "bulk_update_requests" then Types::BulkUpdateRequests
+      when "pools"                then Types::Pools
+      when "posts"                then Types::Posts
+      when "post_replacements"    then Types::PostReplacements
+      when "post_versions"        then Types::PostVersions
+      when "tag_aliases"          then Types::TagAliases
+      when "tag_implications"     then Types::TagImplications
+      when "tags"                 then Types::Tags
+      when "wiki_pages"           then Types::WikiPages
+      else raise(ArgumentError, "invalid type: #{type}")
+      end
     end
   end
 end

data/lib/e621_export_downloader/constants.rb

@@ -9,8 +9,9 @@ module E621ExportDownloader
   module Constants
     extend(T::Sig)
 
+    API_URL = "https://e621.net/db_exports.json"
     BASE_URL = "https://e621.net/db_export/"
-    EXPORT_NAMES = %w[pools posts tag_aliases tag_implications tags wiki_pages].freeze
+    EXPORT_NAMES = %w[artists bulk_update_requests pools posts post_replacements post_versions tag_aliases tag_implications tags wiki_pages].freeze
     TEMP_DIR = T.let(File.join(Dir.tmpdir, "e621-export-downloader"), String)
     USER_AGENT = T.let("E621ExportDownloader.rb/#{VERSION} (#{WEBSITE})", String)
   end

data/lib/e621_export_downloader/export.rb

@@ -10,26 +10,31 @@ module E621ExportDownloader
 
     Model = type_member
 
-    sig { returns(Date) }
-    attr_accessor(:date)
+    sig { returns(APIExportData) }
+    attr_accessor(:data)
 
-    sig { returns(ExportHelper[Model]) }
-    attr_accessor(:helper)
+    sig { returns(Client) }
+    attr_accessor(:client)
+
+    sig { returns(Types) }
+    attr_accessor(:type)
 
     sig { returns(T.nilable(T::Boolean)) }
-    # If nil, no check has been performed yet
     attr_accessor(:downloaded)
 
-    sig { params(date: T.any(Date, DateTime), helper: ExportHelper[Model]).void }
-    def initialize(date:, helper:)
-      @date = T.let(date.is_a?(DateTime) ? date.to_date : date, Date)
-      @helper = helper
+    sig { params(data: APIExportData, client: Client, type: Types, parser: Client::Options::Parser).void }
+    def initialize(data:, client:, type:, parser:)
+      @data = T.let(data, APIExportData)
+      @client = T.let(client, Client)
+      @type = T.let(type, Types)
+      @parser = T.let(parser, Client::Options::Parser)
       @downloaded = T.let(nil, T.nilable(T::Boolean))
     end
 
     sig { returns(T::Boolean) }
     def delete
       return false unless check_downloaded
+      client.debug("deleting cached export", header: ["export:#{type.serialize}"])
       FileUtils.rm(file_path)
       @downloaded = false
       true
@@ -37,19 +42,19 @@ module E621ExportDownloader
 
     sig { returns(String) }
     def download
-      raise(ResolveError, "Export #{helper.type} for #{helper.format_date(date)} does not exist") unless exists?
+      raise(ResolveError, "Export #{type.serialize} does not exist") unless exists?
       if check_downloaded
-        helper.client.debug("using cached export for #{helper.type.serialize}", header: %W[export #{helper.format_date(date)}])
+        client.debug("using cached export", header: ["export:#{type.serialize}"])
         return file_path
       end
 
-      helper.client.debug("downloading export for #{helper.type.serialize}", header: %W[export #{helper.format_date(date)}])
+      client.debug("downloading export", header: ["export:#{type.serialize}"])
 
       FileUtils.mkdir_p(Constants::TEMP_DIR)
       File.open(file_path, "wb") do |file|
         inflater = Zlib::Inflate.new(Zlib::MAX_WBITS + 16)
 
-        res = helper.client.connection.get("#{file_name}.gz") do |req|
+        res = client.connection.get(data.url) do |req|
           req.options.on_data = proc do |chunk, _total|
             decompressed = inflater.inflate(chunk)
             file.write(decompressed) if decompressed && !decompressed.empty?
@@ -61,7 +66,7 @@ module E621ExportDownloader
         file.close
 
         unless res.success?
-          raise(ResolveError, "Failed to download export #{helper.type.serialize} for #{helper.format_date(date)}: #{res.status} #{res.reason_phrase}")
+          raise(ResolveError, "Failed to download export #{type.serialize}: #{res.status} #{res.reason_phrase}")
         end
       rescue # rubocop:disable Style/RescueStandardError
         FileUtils.rm_f(file_path)
@@ -69,30 +74,34 @@ module E621ExportDownloader
       end
 
       @downloaded = true
+      client.debug("download complete: #{file_path}", header: ["export:#{type.serialize}"])
       file_path
     end
 
     sig { returns(T::Boolean) }
     def exists?
-      helper.client.connection.head("#{file_name}.gz").success?
+      result = client.connection.head(data.url).success?
+      client.debug("checked export existence: #{result}", header: ["export:#{type.serialize}"])
+      result
     end
 
     sig { params(block: T.proc.params(arg0: Model, arg1: Integer).void).void }
     def read(&block)
       download unless check_downloaded
-      helper.client.debug("reading export for #{helper.type.serialize}", header: %W[export #{helper.format_date(date)}])
+      client.debug("reading export", header: ["export:#{type.serialize}"])
       total = line_count
       CSV.foreach(file_path, headers: true) do |row|
-        args = [helper.parser.call(T.cast(row, CSV::Row).to_hash), total]
+        args = [@parser.call(T.cast(row, CSV::Row).to_hash), total]
         args = args.slice(0, block.arity) if block.arity != -1
         block.call(*T.unsafe(args))
       end
+      client.debug("finished reading export", header: ["export:#{type.serialize}"])
+      delete unless client.options.cache
     end
 
     sig { returns(T::Array[Model]) }
     def read_all
-      download unless check_downloaded
-      helper.client.debug("reading all records for #{helper.type.serialize}", header: %W[export #{helper.format_date(date)}])
+      client.debug("reading all records", header: ["export:#{type.serialize}"])
       results = []
       read do |record|
         results << record
@@ -102,6 +111,11 @@ module E621ExportDownloader
 
     private
 
+    sig { returns(String) }
+    def format_date
+      DateTime.parse(data.updated_at).strftime("%Y-%m-%d")
+    end
+
     sig { returns(Integer) }
     def line_count
       total = 0
@@ -113,6 +127,8 @@ module E621ExportDownloader
     def check_downloaded
       return @downloaded unless @downloaded.nil?
       @downloaded = File.exist?(file_path)
+      client.debug("checked downloaded state: #{@downloaded}", header: ["export:#{type.serialize}"])
+      @downloaded
     end
 
     sig { returns(String) }
@@ -122,7 +138,7 @@ module E621ExportDownloader
 
     sig { returns(String) }
     def file_name
-      "#{helper.type.serialize}-#{helper.format_date(date)}.csv"
+      "#{type.serialize}-#{format_date}.csv"
     end
   end
 end

data/lib/e621_export_downloader/export_helper.rb

@@ -11,73 +11,51 @@ module E621ExportDownloader
     sig { returns(Types) }
     attr_reader(:type)
 
-    sig { returns(Client::Options::Parser) }
-    attr_reader(:parser)
-
     sig { returns(Client) }
     attr_reader(:client)
 
-    sig { params(type: Types, parser: Client::Options::Parser, client: Client).void }
-    def initialize(type:, parser:, client:)
+    sig { params(type: Types, client: Client).void }
+    def initialize(type:, client:)
       @type = type
-      @parser = parser
       @client = client
-      @rewind_count = T.let(0, Integer)
+      @export = T.let(nil, T.nilable(Export[Model]))
     end
 
-    sig { params(date: T.any(Date, DateTime)).returns(String) }
-    def format_date(date)
-      date.strftime("%Y-%m-%d")
+    sig { returns(T::Boolean) }
+    def delete
+      _get.delete
     end
 
-    sig { params(date: T.any(Date, DateTime)).returns(T::Boolean) }
-    def delete(date)
-      client.debug("deleting export for #{type.serialize}", header: %W[helper #{format_date(date)}])
-      _get(date).delete
+    sig { returns(String) }
+    def download
+      _get.download
     end
 
-    sig { params(date: T.any(Date, DateTime)).returns(String) }
-    def download(date)
-      client.debug("downloading export for #{type.serialize}", header: %W[helper #{format_date(date)}])
-      _get(date).download
+    sig { returns(T::Boolean) }
+    def exists?
+      _get.exists?
     end
 
-    sig { params(date: T.any(Date, DateTime)).returns(T::Boolean) }
-    def exists?(date)
-      client.debug("checking export existence for #{type.serialize}", header: %W[helper #{format_date(date)}])
-      _get(date).exists?
+    sig { returns(Export[Model]) }
+    def get
+      _get
     end
 
-    sig { params(date: T.any(Date, DateTime)).returns(Export[Model]) }
-    def get(date)
-      client.debug("creating export handle for #{type.serialize}", header: %W[helper #{format_date(date)}])
-      Export.new(date: date, helper: self)
+    sig { params(block: T.proc.params(arg0: Model, arg1: Integer).void).void }
+    def read(&block)
+      _get.read(&block)
     end
 
-    private
-
-    sig { returns(Integer) }
-    def max_rewind_count
-      return 0 if client.options.rewind_on_not_found == false
-      return 2 if client.options.rewind_on_not_found == true
-      T.cast(client.options.rewind_on_not_found, Integer)
+    sig { returns(T::Array[Model]) }
+    def read_all
+      _get.read_all
     end
 
-    sig { params(date: T.any(Date, DateTime), original_date: T.any(Date, DateTime)).returns(Export[Model]) }
-    def _get(date, original_date = date)
-      export = get(date)
-      if export.exists?
-        client.debug("resolved export for #{type.serialize}", header: %w[helper])
-        return export
-      end
-
-      if @rewind_count < max_rewind_count
-        @rewind_count += 1
-        client.debug("rewinding export lookup for #{type.serialize} from #{format_date(original_date)} (attempt #{@rewind_count}/#{max_rewind_count})", header: %w[helper])
-        return _get(date.to_date + 1, original_date)
-      end
+    private
 
-      raise(ResolveError, "Export #{type.serialize} for #{format_date(original_date)} does not exist, and either rewinding is not allowed or the maximum rewind limit has been reached")
+    sig { returns(Export[Model]) }
+    def _get
+      @export ||= client.get(type)
     end
   end
 end

data/lib/e621_export_downloader/models/artist.rb

@@ -0,0 +1,77 @@
+# frozen_string_literal: true
+# typed: strict
+
+module E621ExportDownloader
+  module Models
+    class Artist
+      extend(T::Sig)
+
+      sig { returns(DateTime) }
+      attr_reader(:created_at)
+
+      sig { returns(Integer) }
+      attr_reader(:creator_id)
+
+      sig { returns(T.nilable(String)) }
+      attr_reader(:group_name)
+
+      sig { returns(Integer) }
+      attr_reader(:id)
+
+      sig { returns(T::Boolean) }
+      attr_reader(:is_active)
+
+      sig { returns(T::Boolean) }
+      attr_reader(:is_locked)
+
+      sig { returns(T.nilable(Integer)) }
+      attr_reader(:linked_user_id)
+
+      sig { returns(String) }
+      attr_reader(:name)
+
+      sig { returns(T::Array[String]) }
+      attr_reader(:other_names)
+
+      sig { returns(DateTime) }
+      attr_reader(:updated_at)
+
+      sig { returns(T::Array[String]) }
+      attr_reader(:urls)
+
+      sig { params(record: T::Hash[String, String]).void }
+      def initialize(record)
+        @record = T.let(record, T::Hash[String, String])
+        @created_at     = T.let(DateTime.parse(record["created_at"]), DateTime)
+        @creator_id     = T.let(record["creator_id"].to_i, Integer)
+        @group_name     = T.let(T.must(record["group_name"]).empty? ? nil : record["group_name"], T.nilable(String))
+        @id             = T.let(record["id"].to_i, Integer)
+        @is_active      = T.let(record["is_active"] == "t", T::Boolean)
+        @is_locked      = T.let(record["is_locked"] == "t", T::Boolean)
+        @linked_user_id = T.let(T.must(record["linked_user_id"]).empty? ? nil : record["linked_user_id"].to_i, T.nilable(Integer))
+        @name           = T.let(T.must(record["name"]), String)
+        inner           = T.must(T.must(record["other_names"])[1..-2])
+        @other_names    = T.let(inner.empty? ? [] : inner.split(","), T::Array[String])
+        @updated_at     = T.let(DateTime.parse(record["updated_at"]), DateTime)
+        @urls           = T.let(T.must(record["urls"]).split("\n"), T::Array[String])
+      end
+
+      sig { params(_args: T.untyped).returns(String) }
+      def to_json(*_args)
+        {
+          created_at:     @created_at,
+          creator_id:     @creator_id,
+          group_name:     @group_name,
+          id:             @id,
+          is_active:      @is_active,
+          is_locked:      @is_locked,
+          linked_user_id: @linked_user_id,
+          name:           @name,
+          other_names:    @other_names,
+          updated_at:     @updated_at,
+          urls:           @urls,
+        }.to_json
+      end
+    end
+  end
+end

data/lib/e621_export_downloader/models/bulk_update_request.rb

@@ -0,0 +1,66 @@
+# frozen_string_literal: true
+# typed: strict
+
+module E621ExportDownloader
+  module Models
+    class BulkUpdateRequest
+      extend(T::Sig)
+
+      sig { returns(T.nilable(Integer)) }
+      attr_reader(:approver_id)
+
+      sig { returns(DateTime) }
+      attr_reader(:created_at)
+
+      sig { returns(T.nilable(Integer)) }
+      attr_reader(:forum_topic_id)
+
+      sig { returns(Integer) }
+      attr_reader(:id)
+
+      sig { returns(String) }
+      attr_reader(:script)
+
+      sig { returns(String) }
+      attr_reader(:status)
+
+      sig { returns(T.nilable(String)) }
+      attr_reader(:title)
+
+      sig { returns(DateTime) }
+      attr_reader(:updated_at)
+
+      sig { returns(Integer) }
+      attr_reader(:user_id)
+
+      sig { params(record: T::Hash[String, String]).void }
+      def initialize(record)
+        @record         = T.let(record, T::Hash[String, String])
+        @approver_id    = T.let(T.must(record["approver_id"]).empty? ? nil : record["approver_id"].to_i, T.nilable(Integer))
+        @created_at     = T.let(DateTime.parse(record["created_at"]), DateTime)
+        @forum_topic_id = T.let(T.must(record["forum_topic_id"]).empty? ? nil : record["forum_topic_id"].to_i, T.nilable(Integer))
+        @id             = T.let(record["id"].to_i, Integer)
+        @script         = T.let(T.must(record["script"]), String)
+        @status         = T.let(T.must(record["status"]), String)
+        @title          = T.let(T.must(record["title"]).empty? ? nil : record["title"], T.nilable(String))
+        @updated_at     = T.let(DateTime.parse(record["updated_at"]), DateTime)
+        @user_id        = T.let(record["user_id"].to_i, Integer)
+      end
+
+      sig { params(_args: T.untyped).returns(String) }
+      def to_json(*_args)
+        {
+          approver_id:    @approver_id,
+          created_at:     @created_at,
+          forum_topic_id: @forum_topic_id,
+          id:             @id,
+          script:         @script,
+          status:         @status,
+          title:          @title,
+          updated_at:     @updated_at,
+          user_id:        @user_id,
+        }.to_json
+      end
+    end
+  end
+end

data/lib/e621_export_downloader/models/post_replacement.rb

@@ -0,0 +1,96 @@
+# frozen_string_literal: true
+# typed: strict
+
+module E621ExportDownloader
+  module Models
+    class PostReplacement
+      extend(T::Sig)
+
+      sig { returns(T.nilable(Integer)) }
+      attr_reader(:approver_id)
+
+      sig { returns(DateTime) }
+      attr_reader(:created_at)
+
+      sig { returns(Integer) }
+      attr_reader(:creator_id)
+
+      sig { returns(String) }
+      attr_reader(:file_ext)
+
+      sig { returns(String) }
+      attr_reader(:file_name)
+
+      sig { returns(Integer) }
+      attr_reader(:file_size)
+
+      sig { returns(Integer) }
+      attr_reader(:id)
+
+      sig { returns(Integer) }
+      attr_reader(:image_height)
+
+      sig { returns(Integer) }
+      attr_reader(:image_width)
+
+      sig { returns(String) }
+      attr_reader(:md5)
+
+      sig { returns(Integer) }
+      attr_reader(:post_id)
+
+      sig { returns(String) }
+      attr_reader(:reason)
+
+      sig { returns(T.nilable(String)) }
+      attr_reader(:source)
+
+      sig { returns(String) }
+      attr_reader(:status)
+
+      sig { returns(DateTime) }
+      attr_reader(:updated_at)
+
+      sig { params(record: T::Hash[String, String]).void }
+      def initialize(record)
+        @record       = T.let(record, T::Hash[String, String])
+        @approver_id  = T.let(T.must(record["approver_id"]).empty? ? nil : record["approver_id"].to_i, T.nilable(Integer))
+        @created_at   = T.let(DateTime.parse(record["created_at"]), DateTime)
+        @creator_id   = T.let(record["creator_id"].to_i, Integer)
+        @file_ext     = T.let(T.must(record["file_ext"]), String)
+        @file_name    = T.let(T.must(record["file_name"]), String)
+        @file_size    = T.let(record["file_size"].to_i, Integer)
+        @id           = T.let(record["id"].to_i, Integer)
+        @image_height = T.let(record["image_height"].to_i, Integer)
+        @image_width  = T.let(record["image_width"].to_i, Integer)
+        @md5          = T.let(T.must(record["md5"]), String)
+        @post_id      = T.let(record["post_id"].to_i, Integer)
+        @reason       = T.let(T.must(record["reason"]), String)
+        @source       = T.let(T.must(record["source"]).empty? ? nil : record["source"], T.nilable(String))
+        @status       = T.let(T.must(record["status"]), String)
+        @updated_at   = T.let(DateTime.parse(record["updated_at"]), DateTime)
+      end
+
+      sig { params(_args: T.untyped).returns(String) }
+      def to_json(*_args)
+        {
+          approver_id:  @approver_id,
+          created_at:   @created_at,
+          creator_id:   @creator_id,
+          file_ext:     @file_ext,
+          file_name:    @file_name,
+          file_size:    @file_size,
+          id:           @id,
+          image_height: @image_height,
+          image_width:  @image_width,
+          md5:          @md5,
+          post_id:      @post_id,
+          reason:       @reason,
+          source:       @source,
+          status:       @status,
+          updated_at:   @updated_at,
+        }.to_json
+      end
+    end
+  end
+end

data/lib/e621_export_downloader/models/post_version.rb

@@ -0,0 +1,120 @@
+# frozen_string_literal: true
+# typed: strict
+
+module E621ExportDownloader
+  module Models
+    class PostVersion
+      extend(T::Sig)
+
+      sig { returns(T::Array[String]) }
+      attr_reader(:added_locked_tags)
+
+      sig { returns(T::Array[String]) }
+      attr_reader(:added_tags)
+
+      sig { returns(T.nilable(String)) }
+      attr_reader(:description)
+
+      sig { returns(T::Boolean) }
+      attr_reader(:description_changed)
+
+      sig { returns(Integer) }
+      attr_reader(:id)
+
+      sig { returns(T.nilable(String)) }
+      attr_reader(:locked_tags)
+
+      sig { returns(T::Boolean) }
+      attr_reader(:parent_changed)
+
+      sig { returns(T.nilable(Integer)) }
+      attr_reader(:parent_id)
+
+      sig { returns(T.nilable(String)) }
+      attr_reader(:rating)
+
+      sig { returns(T::Boolean) }
+      attr_reader(:rating_changed)
+
+      sig { returns(T.nilable(String)) }
+      attr_reader(:reason)
+
+      sig { returns(T::Array[String]) }
+      attr_reader(:removed_locked_tags)
+
+      sig { returns(T::Array[String]) }
+      attr_reader(:removed_tags)
+
+      sig { returns(T.nilable(String)) }
+      attr_reader(:source)
+
+      sig { returns(T::Boolean) }
+      attr_reader(:source_changed)
+
+      sig { returns(T.nilable(String)) }
+      attr_reader(:tags)
+
+      sig { returns(DateTime) }
+      attr_reader(:updated_at)
+
+      sig { returns(Integer) }
+      attr_reader(:updater_id)
+
+      sig { returns(Integer) }
+      attr_reader(:version)
+
+      sig { params(record: T::Hash[String, String]).void }
+      def initialize(record)
+        @record              = T.let(record, T::Hash[String, String])
+        inner                = T.must(T.must(record["added_locked_tags"])[1..-2])
+        @added_locked_tags   = T.let(inner.empty? ? [] : inner.split(","), T::Array[String])
+        inner                = T.must(T.must(record["added_tags"])[1..-2])
+        @added_tags          = T.let(inner.empty? ? [] : inner.split(","), T::Array[String])
+        @description         = T.let(T.must(record["description"]).empty? ? nil : record["description"], T.nilable(String))
+        @description_changed = T.let(record["description_changed"] == "t", T::Boolean)
+        @id                  = T.let(record["id"].to_i, Integer)
+        @locked_tags         = T.let(T.must(record["locked_tags"]).empty? ? nil : record["locked_tags"], T.nilable(String))
+        @parent_changed      = T.let(record["parent_changed"] == "t", T::Boolean)
+        @parent_id           = T.let(T.must(record["parent_id"]).empty? ? nil : record["parent_id"].to_i, T.nilable(Integer))
+        @rating              = T.let(T.must(record["rating"]).empty? ? nil : record["rating"], T.nilable(String))
+        @rating_changed      = T.let(record["rating_changed"] == "t", T::Boolean)
+        @reason              = T.let(T.must(record["reason"]).empty? ? nil : record["reason"], T.nilable(String))
+        inner                = T.must(T.must(record["removed_locked_tags"])[1..-2])
+        @removed_locked_tags = T.let(inner.empty? ? [] : inner.split(","), T::Array[String])
+        inner                = T.must(T.must(record["removed_tags"])[1..-2])
+        @removed_tags        = T.let(inner.empty? ? [] : inner.split(","), T::Array[String])
+        @source              = T.let(T.must(record["source"]).empty? ? nil : record["source"], T.nilable(String))
+        @source_changed      = T.let(record["source_changed"] == "t", T::Boolean)
+        @tags                = T.let(T.must(record["tags"]).empty? ? nil : record["tags"], T.nilable(String))
+        @updated_at          = T.let(DateTime.parse(record["updated_at"]), DateTime)
+        @updater_id          = T.let(record["updater_id"].to_i, Integer)
+        @version             = T.let(record["version"].to_i, Integer)
+      end
+
+      sig { params(_args: T.untyped).returns(String) }
+      def to_json(*_args)
+        {
+          added_locked_tags:   @added_locked_tags,
+          added_tags:          @added_tags,
+          description:         @description,
+          description_changed: @description_changed,
+          id:                  @id,
+          locked_tags:         @locked_tags,
+          parent_changed:      @parent_changed,
+          parent_id:           @parent_id,
+          rating:              @rating,
+          rating_changed:      @rating_changed,
+          reason:              @reason,
+          removed_locked_tags: @removed_locked_tags,
+          removed_tags:        @removed_tags,
+          source:              @source,
+          source_changed:      @source_changed,
+          tags:                @tags,
+          updated_at:          @updated_at,
+          updater_id:          @updater_id,
+          version:             @version,
+        }.to_json
+      end
+    end
+  end
+end

data/lib/e621_export_downloader/types.rb

@@ -3,8 +3,12 @@
 module E621ExportDownloader
   class Types < T::Enum
     enums do
+      Artists = new("artists")
+      BulkUpdateRequests = new("bulk_update_requests")
       Pools = new("pools")
       Posts = new("posts")
+      PostReplacements = new("post_replacements")
+      PostVersions = new("post_versions")
       TagAliases = new("tag_aliases")
       TagImplications = new("tag_implications")
       Tags = new("tags")

data/lib/e621_export_downloader/version.rb

@@ -4,7 +4,7 @@
 # loaded by bundler
 module E621ExportDownloader
   module Constants
-    VERSION = "0.0.3"
+    VERSION = "0.0.4"
     WEBSITE = "https://github.com/DonovanDMC/E621ExportDownloader.rb"
   end
 end

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: e621_export_downloader
 version: !ruby/object:Gem::Version
-  version: 0.0.3
+  version: 0.0.4
 platform: ruby
 authors:
 - Donovan_DMC
 bindir: exe
 cert_chain: []
-date: 2026-05-19 00:00:00.000000000 Z
+date: 2026-06-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: csv
@@ -80,6 +80,7 @@ files:
 - Rakefile
 - exe/e621-export-downloader
 - lib/e621_export_downloader.rb
+- lib/e621_export_downloader/api_export_data.rb
 - lib/e621_export_downloader/client.rb
 - lib/e621_export_downloader/client/options.rb
 - lib/e621_export_downloader/client/options/builder.rb
@@ -87,8 +88,12 @@ files:
 - lib/e621_export_downloader/constants.rb
 - lib/e621_export_downloader/export.rb
 - lib/e621_export_downloader/export_helper.rb
+- lib/e621_export_downloader/models/artist.rb
+- lib/e621_export_downloader/models/bulk_update_request.rb
 - lib/e621_export_downloader/models/pool.rb
 - lib/e621_export_downloader/models/post.rb
+- lib/e621_export_downloader/models/post_replacement.rb
+- lib/e621_export_downloader/models/post_version.rb
 - lib/e621_export_downloader/models/tag.rb
 - lib/e621_export_downloader/models/tag_alias.rb
 - lib/e621_export_downloader/models/tag_implication.rb