e621_export_downloader 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 07a77d7c815326f0a51b7026c42f89c187fc526cc7caf20af35839e628e5243e
+  data.tar.gz: 23e2cd461ce89e9be5ff79d4a398e84762560e19a6edc9afffb323954ec37ba9
+SHA512:
+  metadata.gz: 25ac18b83938e29298b3221b162594c6112a679b11d510ac70c46aff148f5fb08ede1b26466ccc942aa74a898a3680d7eb2087fb4374f49387b133ecf3974dc2
+  data.tar.gz: 316c7856999ce0ef76a24ba7dcaa7870969a0c1c41a8efe9a4ba30d63447bccc55d172a6c797359b916da2f76d89d2433f0047810798f23a859726e3ed21052b

data/.irbrc

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+# typed: true
+
+require_relative("lib/e621_export_downloader")
+require("sorbet-runtime")
+
+extend(T::Sig) # rubocop:disable Style/MixinUsage
+
+E621ExportDownloader::Client::Logger.level = Logger::DEBUG
+
+sig { returns(E621ExportDownloader::Client) }
+def client
+  E621ExportDownloader::Client.new
+end

data/.ruby-version

@@ -0,0 +1 @@
+3.4.1

data/LICENSE

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2026 Donovan_DMC
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,148 @@
+# E621 Export Downloader
+
+A utility for downloading and parsing e621's exports.
+
+## Installation
+
+```bash
+gem install e621_export_downloader
+```
+
+Or add it to your Gemfile:
+
+```bash
+bundle add e621_export_downloader
+```
+
+## Usage
+
+```ruby
+require("e621_export_downloader")
+
+client = E621ExportDownloader::Client.new
+
+# configure options after creation
+client.config do |c|
+  c.cache = true              # keep export files after reading, defaults to true
+  c.rewind_on_not_found = 2  # decrease date by one day if no export is found for that date,
+                              # provide an integer to limit how many days can be rewound,
+                              # `true` is equivalent to `2`, defaults to false
+end
+
+# or pass an Options struct directly
+client = E621ExportDownloader::Client.new(
+  E621ExportDownloader::Client::Options.new(
+    cache:              true,
+    rewind_on_not_found: 2,
+  )
+)
+
+# get the helper for interacting with a type of export
+# types: pools, posts, tag_aliases, tag_implications, tags, wiki_pages
+helper = client.get("posts")
+# or use the named shorthand:
+helper = client.posts
+
+# get a wrapper for a specific date (time components are ignored)
+# using this directly will not trigger rewinding regardless of rewind_on_not_found
+export = helper.get(Date.today)
+
+# convenience shorthand on the client — also skips rewinding
+export = client.get_posts(Date.today)
+
+# all of the following methods can also be called on the helper with a date argument;
+# if rewind_on_not_found is enabled the helper decrements the date by one day until it
+# finds an export or exhausts the rewind limit, at which point it raises ResolveError
+
+# check whether the export exists for the date
+exists = export.exists?
+exists = helper.exists?(Date.today)
+
+# download the export, returns the file path — not required before reading
+# if you move or remove the file do not reuse the export object
+file = export.download
+file = helper.download(Date.today)
+
+# delete the downloaded file, if it exists
+export.delete
+helper.delete(Date.today)
+
+# get all of the records as a single array, DO NOT use this for large exports, arrays will millions of items do not perform well and will likely crash your process!
+# (not to mention that the posts export is more than 5 gigabytes)
+records = export.read_all
+
+# read streams the CSV and yields each parsed record together with the total row count
+# this is the recommended approach for large exports (the posts export exceeds 5 GB)
+export.read do |record, total|
+  # record is an E621ExportDownloader::Models::Post instance
+end
+```
+
+## Replace or extend a parser
+
+```ruby
+require("e621_export_downloader")
+
+client = E621ExportDownloader::Client.new
+
+client.config do |c|
+  c.parsers do |p|
+    # replace a parser with a custom proc that receives the raw CSV row hash
+    # return nil to skip a record
+    p.posts = proc do |record|
+      post = E621ExportDownloader::Models::Post.new(record)
+      # attach extra data or wrap in your own class
+      post
+    end
+  end
+end
+```
+
+## Models
+
+Each export type is parsed into a corresponding model class:
+
+| Export type       | Model class                                       |
+|-------------------|---------------------------------------------------|
+| `pools`           | `E621ExportDownloader::Models::Pool`              |
+| `posts`           | `E621ExportDownloader::Models::Post`              |
+| `tag_aliases`     | `E621ExportDownloader::Models::TagAlias`          |
+| `tag_implications`| `E621ExportDownloader::Models::TagImplication`    |
+| `tags`            | `E621ExportDownloader::Models::Tag`               |
+| `wiki_pages`      | `E621ExportDownloader::Models::WikiPage`          |
+
+## CLI
+
+```bash
+# check if an export exists for a given date; date is optional and defaults to today
+# outputs "true" or "false" with no trailing newline
+e621-export-downloader exists posts 2024-01-01
+
+# download an export for a given date; date is optional and defaults to today
+# outputs the path to the downloaded file with no trailing newline
+e621-export-downloader download posts 2024-01-01
+
+# read an export as individual JSON lines; date is optional and defaults to today
+# outputs each record as a JSON object on its own line
+e621-export-downloader read posts 2024-01-01
+
+# read an export as a JSON array; date is optional and defaults to today
+# streams the CSV internally, so it is safe to use with large exports
+e621-export-downloader read-all posts 2024-01-01
+
+e621-export-downloader --help
+e621-export-downloader --version
+e621-export-downloader --cache                    # enable caching (default)
+e621-export-downloader --no-cache                 # disable caching
+e621-export-downloader --rewind-on-not-found      # enable rewinding (up to 2 days)
+e621-export-downloader --rewind-on-not-found 5    # enable rewinding (up to 5 days)
+e621-export-downloader --no-rewind-on-not-found   # disable rewinding (default)
+```
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/DonovanDMC/E621ExportDownloader.rb.
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).

data/Rakefile

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+require("bundler/gem_tasks")
+require("rubocop/rake_task")
+
+RuboCop::RakeTask.new
+
+task(default: :rubocop)

data/exe/e621-export-downloader

@@ -0,0 +1,112 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require("optparse")
+require("date")
+require("e621_export_downloader")
+
+options = {
+  cache:               true,
+  rewind_on_not_found: false,
+}
+
+parser = OptionParser.new do |opts| # rubocop:disable Metrics/BlockLength
+  opts.program_name = "e621-export-downloader"
+  opts.banner = <<~BANNER
+    Usage: e621-export-downloader [options] <command> [args]
+
+    Commands:
+      exists <name> [date]      Check if an export exists for a given date
+      download <name> [date]    Download an export for a given date
+      read <name> [date]        Read an export as individual JSON lines
+      read-all <name> [date]    Read an export as a JSON array
+
+    Options:
+  BANNER
+
+  opts.on("--[no-]cache", "Cache downloaded exports (default: enabled)") { |v| options[:cache] = v }
+
+  opts.on("--rewind-on-not-found [DAYS]", Integer,
+          "Rewind date if export not found; optionally limit how many days (true = 2)") do |v|
+    options[:rewind_on_not_found] = v || true
+  end
+  opts.on("--no-rewind-on-not-found", "Disable rewinding of export dates (default)") do
+    options[:rewind_on_not_found] = false
+  end
+
+  opts.on("-v", "--version", "Print the version") do
+    puts(E621ExportDownloader::Constants::VERSION)
+    exit
+  end
+
+  opts.on_tail("-h", "--help", "Print this help") do
+    puts(opts)
+    exit
+  end
+end
+
+parser.order!(ARGV)
+
+command = ARGV.shift
+if command.nil?
+  puts(parser)
+  exit(1)
+end
+
+name = ARGV.shift
+if name.nil?
+  warn("Error: export name is required")
+  exit(1)
+end
+
+date_str = ARGV.shift
+date = date_str ? Date.parse(date_str) : Date.today
+
+client = E621ExportDownloader::Client.new(
+  E621ExportDownloader::Client::Options.new(
+    cache:               options[:cache],
+    rewind_on_not_found: options[:rewind_on_not_found],
+  ),
+)
+
+# Downloads via the helper (respects rewind_on_not_found) and returns an Export
+# handle that is already marked as downloaded so read won't re-fetch.
+def download_and_open(helper, date)
+  path = helper.download(date)
+  resolved_date = Date.parse(File.basename(path, ".csv")[-10..])
+  E621ExportDownloader::Export.new(date: resolved_date, helper: helper)
+end
+
+helper = client.get(name)
+
+case command
+when "exists"
+  print(helper.exists?(date))
+
+when "download"
+  print(helper.download(date))
+
+when "read"
+  export = download_and_open(helper, date)
+  first = true
+  export.read do |record, _total|
+    $stdout.write("\n") unless first
+    first = false
+    $stdout.write(record.to_json)
+  end
+
+when "read-all"
+  export = download_and_open(helper, date)
+  $stdout.write("[")
+  first = true
+  export.read do |record, _total|
+    $stdout.write(",") unless first
+    first = false
+    $stdout.write(record.to_json)
+  end
+  $stdout.write("]")
+
+else
+  warn("Error: unknown command '#{command}'. Run with --help for usage.")
+  exit(1)
+end

data/lib/e621_export_downloader/client/options/builder/parsers.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+# typed: strict
+
+module E621ExportDownloader
+  class Client
+    class Options
+      class Builder
+        class Parsers
+          extend(T::Sig)
+
+          sig { returns(Parser) }
+          attr_accessor(:pools)
+
+          sig { returns(Parser) }
+          attr_accessor(:posts)
+
+          sig { returns(Parser) }
+          attr_accessor(:tag_aliases)
+
+          sig { returns(Parser) }
+          attr_accessor(:tag_implications)
+
+          sig { returns(Parser) }
+          attr_accessor(:tags)
+
+          sig { returns(Parser) }
+          attr_accessor(:wiki_pages)
+
+          sig { params(defaults: Options::Parsers).void }
+          def initialize(defaults = Options::Parsers.defaults)
+            @pools = T.let(T.must(defaults.pools), Parser)
+            @posts = T.let(T.must(defaults.posts), Parser)
+            @tag_aliases = T.let(T.must(defaults.tag_aliases), Parser)
+            @tag_implications = T.let(T.must(defaults.tag_implications), Parser)
+            @tags = T.let(T.must(defaults.tags), Parser)
+            @wiki_pages = T.let(T.must(defaults.wiki_pages), Parser)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/e621_export_downloader/client/options/builder.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+# typed: strict
+
+module E621ExportDownloader
+  class Client
+    class Options
+      class Builder
+        extend(T::Sig)
+
+        sig { returns(T::Boolean) }
+        attr_accessor(:cache)
+
+        sig { returns(T.any(T::Boolean, Integer)) }
+        attr_accessor(:rewind_on_not_found)
+
+        sig { params(parsers: Options::Parsers).returns(Options::Parsers) }
+        attr_writer(:parsers)
+
+        sig { params(defaults: Options).void }
+        def initialize(defaults = Options.new)
+          @cache = T.let(defaults.cache, T::Boolean)
+          @rewind_on_not_found = T.let(defaults.rewind_on_not_found, T.any(T::Boolean, Integer))
+          @parsers = T.let(defaults.parsers, Options::Parsers)
+        end
+
+        sig { params(block: T.nilable(T.proc.params(arg0: Builder::Parsers).void)).returns(Options::Parsers) }
+        def parsers(&block)
+          return @parsers if block.nil?
+          parsers = Builder::Parsers.new(@parsers)
+          block.call(parsers)
+
+          @parsers = Options::Parsers.new(
+            pools:            parsers.pools,
+            posts:            parsers.posts,
+            tag_aliases:      parsers.tag_aliases,
+            tag_implications: parsers.tag_implications,
+            tags:             parsers.tags,
+            wiki_pages:       parsers.wiki_pages,
+          )
+        end
+      end
+    end
+  end
+end

data/lib/e621_export_downloader/client/options.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+# typed: strict
+
+module E621ExportDownloader
+  class Client
+    class Options < T::Struct
+      Parser = T.type_alias { T.proc.params(arg0: T::Hash[String, String]).returns(T.untyped) }
+
+      class Parsers < T::Struct
+        extend(T::Sig)
+
+        const(:pools, T.nilable(Parser), default: nil)
+        const(:posts, T.nilable(Parser), default: nil)
+        const(:tag_aliases, T.nilable(Parser), default: nil)
+        const(:tag_implications, T.nilable(Parser), default: nil)
+        const(:tags, T.nilable(Parser), default: nil)
+        const(:wiki_pages, T.nilable(Parser), default: nil)
+
+        sig { returns(Parsers) }
+        def self.defaults
+          Parsers.new(
+            pools:            ->(record) { Models::Pool.new(record) },
+            posts:            ->(record) { Models::Post.new(record) },
+            tag_aliases:      ->(record) { Models::TagAlias.new(record) },
+            tag_implications: ->(record) { Models::TagImplication.new(record) },
+            tags:             ->(record) { Models::Tag.new(record) },
+            wiki_pages:       ->(record) { Models::WikiPage.new(record) },
+          )
+        end
+      end
+
+      const(:cache, T::Boolean, default: true)
+      const(:rewind_on_not_found, T.any(T::Boolean, Integer), default: false)
+      const(:parsers, Parsers, default: Parsers.defaults)
+    end
+  end
+end

data/lib/e621_export_downloader/client.rb

@@ -0,0 +1,120 @@
+# frozen_string_literal: true
+# typed: strict
+
+require("faraday")
+
+module E621ExportDownloader
+  class Client
+    extend(T::Sig)
+
+    Logger = ::Logger.new($stdout)
+
+    sig { returns(Options) }
+    attr_reader(:options)
+
+    sig { params(options: T.nilable(Options)).void }
+    def initialize(options = nil)
+      @options = T.let(options || Options.new, Options)
+    end
+
+    sig { params(block: T.proc.params(arg0: Options::Builder).void).void }
+    def config(&block)
+      block.call(Options::Builder.new(@options))
+    end
+
+    sig { returns(Faraday::Connection) }
+    def connection
+      Faraday.new(url: Constants::BASE_URL, headers: { "User-Agent" => Constants::USER_AGENT })
+    end
+
+    sig { params(msg: String, header: T::Array[String]).void }
+    def debug(msg, header: [])
+      Logger.debug("[e621_export_downloader#{":#{header.join(':')}" unless header.empty?}] #{msg}")
+    end
+
+    sig { params(type: T.any(Types, String)).returns(ExportHelper[T.untyped]) }
+    def get(type)
+      if type.is_a?(String)
+        case type
+        when "pools"
+          type = Types::Pools
+        when "posts"
+          type = Types::Posts
+        when "tag_aliases"
+          type = Types::TagAliases
+        when "tag_implications"
+          type = Types::TagImplications
+        when "tags"
+          type = Types::Tags
+        when "wiki_pages"
+          type = Types::WikiPages
+        else
+          raise(ArgumentError, "invalid type: #{type}")
+        end
+      end
+
+      T.assert_type!(type, Types)
+      ExportHelper.new(client: self, type: type, parser: options.parsers.public_send(type.serialize))
+    end
+
+    sig { returns(ExportHelper[Models::Pool]) }
+    def pools
+      get(Types::Pools)
+    end
+
+    sig { params(date: T.any(Date, DateTime)).returns(Export[Models::Pool]) }
+    def get_pools(date = Date.today)
+      pools.get(date)
+    end
+
+    sig { returns(ExportHelper[Models::Post]) }
+    def posts
+      get(Types::Posts)
+    end
+
+    sig { params(date: T.any(Date, DateTime)).returns(Export[Models::Post]) }
+    def get_posts(date = Date.today)
+      posts.get(date)
+    end
+
+    sig { returns(ExportHelper[Models::TagAlias]) }
+    def tag_aliases
+      get(Types::TagAliases)
+    end
+
+    sig { params(date: T.any(Date, DateTime)).returns(Export[Models::TagAlias]) }
+    def get_tag_aliases(date = Date.today)
+      tag_aliases.get(date)
+    end
+
+    sig { returns(ExportHelper[Models::TagImplication]) }
+    def tag_implications
+      get(Types::TagImplications)
+    end
+
+    sig { params(date: T.any(Date, DateTime)).returns(Export[Models::TagImplication]) }
+    def get_tag_implications(date = Date.today)
+      tag_implications.get(date)
+    end
+
+    sig { returns(ExportHelper[Models::Tag]) }
+    def tags
+      get(Types::Tags)
+    end
+
+    sig { params(date: T.any(Date, DateTime)).returns(Export[Models::Tag]) }
+    def get_tags(date = Date.today)
+      tags.get(date)
+    end
+
+    sig { returns(ExportHelper[Models::WikiPage]) }
+    def wiki_pages
+      get(Types::WikiPages)
+    end
+
+    sig { params(date: T.any(Date, DateTime)).returns(Export[Models::WikiPage]) }
+    def get_wiki_pages(date = Date.today)
+      wiki_pages.get(date)
+    end
+  end
+end

data/lib/e621_export_downloader/constants.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+# typed: strict
+
+require("tmpdir")
+require("sorbet-runtime")
+require_relative("version")
+
+module E621ExportDownloader
+  module Constants
+    extend(T::Sig)
+
+    BASE_URL = "https://e621.net/db_export/"
+    EXPORT_NAMES = %w[pools posts tag_aliases tag_implications tags wiki_pages].freeze
+    TEMP_DIR = T.let(File.join(Dir.tmpdir, "e621-export-downloader"), String)
+    USER_AGENT = T.let("E621ExportDownloader.rb/#{VERSION} (#{WEBSITE})", String)
+  end
+end