tsumanne 0.0.3 → 0.0.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 354ad61001070464826ad804ca96878fe654058272d07b4b267fc71a6edbb952
-  data.tar.gz: 9ad780cbc6869459adc4533709237166d1e6c5e750631e0d0f7209810d96cdd6
+  metadata.gz: 364315dfc400885784277475e8cfe19af407d5b1d27daee8adeb9bfc942d18cd
+  data.tar.gz: 4e4aba0c9982cb4b8c93979abba3a144c7a8eed87a7852d0d07050596200e0ca
 SHA512:
-  metadata.gz: 372c56ad00aace2a151d3853eb9554810bd5cab30a1c0b3b7efc2b1f931f31649c8faee9cddabd8f2659bcabceffe1a770de80f096798e559125129d4049a31b
-  data.tar.gz: 74cd78607206bb737c79527537d135712dcca8f5126297b566a92d6a341e8ad951aeda1a66bc96662670927615f9c1923feccdb8282dbaed91b2466599742a32
+  metadata.gz: d3fd6ad688af2126ed9f44cce4a03dd21c0448266326ce338117794bf285967d2d0f8fd671e6e26d2bb8d8d33cd86ff2cb80360d89bcff3873c8881d51a6c554
+  data.tar.gz: 9fb34ceda1df2f7f3af6d28b96811ff1326cba98c0d4de947d8221d58e66948237ee9dc76d6968d007c4f69f1958e27273fbcf67fe297f8fa9577f4ad5f34cd1

data/.dockerignore

@@ -0,0 +1,8 @@
+# `.git` is deliberately kept: the gemspec lists the files to package with `git ls-files`.
+# For the same reason, nothing tracked by git may be excluded here.
+.bundle/
+coverage/
+pkg/
+tmp/
+.rspec_status
+*.gem

data/.mdlrc

@@ -0,0 +1,6 @@
+# This file is a Mixlib::Config file, not a style file: `all` / `exclude_rule` are
+# only understood inside a style file passed with `style`. Prefix a rule with `~`
+# to exclude it here.
+#
+# MD013 (line length): badge definitions and links do not wrap.
+rules "~MD013"

data/.rubocop.yml

@@ -1,14 +1,21 @@
-inherit_gem:
-  rubocop-sorbet: config/rbi.yml
-
-require:
- - rubocop-rake
- - rubocop-rspec
- - rubocop-performance
+plugins:
+  - rubocop-performance
+  - rubocop-rake
+  - rubocop-rspec
+  - rubocop-sorbet
 
 AllCops:
-  TargetRubyVersion: 3.2
+  TargetRubyVersion: 3.3
   NewCops: enable
+  Exclude:
+    # Generated by `tapioca gem`; linting them only creates churn on regeneration.
+    - 'sorbet/rbi/gems/**/*.rbi'
+    # Bundler binstub.
+    - 'bin/tapioca'
+
+# This gem keeps its development dependencies in the gemspec, not the Gemfile.
+Gemspec/DevelopmentDependencies:
+  EnforcedStyle: gemspec
 
 Style/StringLiterals:
   Enabled: true
@@ -24,9 +31,10 @@ Layout/LineLength:
 Metrics/MethodLength:
   Max: 15
 
-Lint/EmptyFile:
-  Exclude:
-    - 'sorbet/rbi/gems/*.rbi'
-
 Gemspec:
   Enabled: true
+
+# `CLI` is a flat dispatcher: one short method per subcommand, which reads best in a single class.
+Metrics/ClassLength:
+  Exclude:
+    - 'lib/tsumanne/cli.rb'

data/Dockerfile

@@ -0,0 +1,24 @@
+FROM --platform=$BUILDPLATFORM ruby:4.0@sha256:c400c1e41e8ad2276c82529ff9fd552cdb339a84e761c9d57d2d7f582122fa6e AS builder
+
+WORKDIR /src
+COPY . .
+RUN gem build tsumanne.gemspec --output /tmp/tsumanne.gem
+
+FROM ruby:4.0-slim@sha256:607bf92fa7ecebb4a0c6654b62cb44c48d94b36b6f5a754611ddbbe3dc5b6135
+
+LABEL org.opencontainers.image.title="tsumanne" \
+      org.opencontainers.image.description="Unofficial API wrapper and CLI for tsumanne.net" \
+      org.opencontainers.image.source="https://github.com/eggplants/tsumanne" \
+      org.opencontainers.image.licenses="MIT"
+
+RUN --mount=type=bind,from=builder,source=/tmp/tsumanne.gem,target=/tmp/tsumanne.gem \
+    gem install /tmp/tsumanne.gem --no-document
+
+RUN tsumanne --version
+
+RUN useradd --create-home --user-group tsumanne
+USER tsumanne
+WORKDIR /home/tsumanne
+
+ENTRYPOINT ["tsumanne"]
+CMD ["--help"]

data/README.md

@@ -3,14 +3,10 @@
 [![Rake]](https://github.com/eggplants/tsumanne/actions/workflows/rake.yml)
 [![Release Gem]](https://github.com/eggplants/tsumanne/actions/workflows/release.yml)
 [![Gem Version]](https://badge.fury.io/rb/tsumanne)
-[![Maintainability]](https://codeclimate.com/github/eggplants/tsumanne/maintainability)
-[![Test Coverage]](https://codeclimate.com/github/eggplants/tsumanne/test_coverage)
 
 [Rake]: <https://github.com/eggplants/tsumanne/actions/workflows/rake.yml/badge.svg>
 [Release Gem]: <https://github.com/eggplants/tsumanne/actions/workflows/release.yml/badge.svg>
 [Gem Version]: <https://badge.fury.io/rb/tsumanne.svg>
-[Maintainability]: <https://api.codeclimate.com/v1/badges/673df4b2c0e15b80f06c/maintainability>
-[Test Coverage]: <https://api.codeclimate.com/v1/badges/673df4b2c0e15b80f06c/test_coverage>
 
 API Wrapper for tsumanne.net
 

data/Rakefile

@@ -2,6 +2,8 @@
 
 # frozen_string_literal: true
 
+require "English"
+
 require "bundler/gem_tasks"
 require "rspec/core/rake_task"
 require "rubocop/rake_task"
@@ -12,48 +14,49 @@ require "rubocop/rake_task"
 RSpec::Core::RakeTask.new(:spec)
 
 RuboCop::RakeTask.new do |task|
-  task.requires << 'rubocop-sorbet'
+  task.requires << "rubocop-sorbet"
 end
 
 namespace :mdl do
-  desc 'Format markdown files with markdownlint'
+  desc "Format markdown files with markdownlint"
   task :format do
-    system("bundle exec mdl #{Dir.glob("**/*.md") * ?\s}")
-    puts "#{$?.success? ? :no : :some} errors found"
+    system("bundle exec mdl -c .mdlrc #{Dir.glob("**/*.md") * "\s"}")
+    puts "#{$CHILD_STATUS.success? ? :no : :some} errors found"
   end
 end
 
 namespace :sorbet do
-  desc 'Type-checking with sorbet'
+  desc "Type-checking with sorbet"
   task :check do
     system("bundle exec srb tc")
   end
 
-  desc 'Type-checking and auto-fix correctable errors with sorbet'
+  desc "Type-checking and auto-fix correctable errors with sorbet"
   task :autocorrect do
     system("bundle exec srb tc --autocorrect")
   end
 end
 
 namespace :tapioca do
-  desc 'Generate RBI from gems with tapioca'
+  desc "Generate RBI from gems with tapioca"
   task :gen do
     system("bundle exec tapioca gem --all")
   end
 
-  desc 'Check duplicated definitions exists in shim RBIs with tapioca'
+  desc "Check duplicated definitions exists in shim RBIs with tapioca"
   task :check_shims do
     system("bundle exec tapioca check-shims")
   end
 end
 
 task :default do
-   %i[mdl:format rubocop:autocorrect_all sorbet:autocorrect tapioca:check_shims spec].each do
-     puts <<~EOS
-       .｡*ﾟ+.*.｡　　#{padding = ?= * _1.size}　　ﾟ+..｡*ﾟ+
-       .｡*ﾟ+.*.｡　　#{_1}　　ﾟ+..｡*ﾟ+
-       .｡*ﾟ+.*.｡　　#{padding}　　ﾟ+..｡*ﾟ+
-     EOS
-     Rake::Task[_1].execute
-   end
+  %i[mdl:format rubocop:autocorrect_all sorbet:autocorrect tapioca:check_shims spec].each do |task_name|
+    padding = "=" * task_name.size
+    puts <<~BANNER
+      .｡*ﾟ+.*.｡　　#{padding}　　ﾟ+..｡*ﾟ+
+      .｡*ﾟ+.*.｡　　#{task_name}　　ﾟ+..｡*ﾟ+
+      .｡*ﾟ+.*.｡　　#{padding}　　ﾟ+..｡*ﾟ+
+    BANNER
+    Rake::Task[task_name].execute
+  end
 end

data/exe/tsumanne

@@ -1,8 +1,9 @@
 #! /usr/bin/env ruby
 # typed: strict
+# frozen_string_literal: true
 
 require "sorbet-runtime"
 
 require_relative "../lib/tsumanne/cli"
 
-Tsumanne::CLI.start(ARGV)
+exit(Tsumanne::CLI.start(ARGV))

data/lib/tsumanne/api.rb

@@ -5,9 +5,7 @@
 require "net/http"
 require "uri"
 require "json"
-require "zlib"
 
-require "mhtml"
 require "sorbet-runtime"
 
 require_relative "models/get_threads"
@@ -17,10 +15,11 @@ require_relative "models/register_thread"
 
 # API module for tsumanne.net includes knowledge as const.
 module Tsumanne
+  # Client for the tsumanne.net endpoints of a single board.
   class API
     extend T::Sig
 
-    sig{ returns(String) }
+    sig { returns(String) }
     attr_reader :board_id
 
     sig { params(board_id: Symbol).void }
@@ -35,25 +34,27 @@ module Tsumanne
       GetThreadsResponse.from_hash(fetch_json(paths: [index, page.to_s]))
     end
 
-    sig { params(thread_id: String).returns(Mhtml::RootDocument) }
+    # Returns the merged HTML export of an archived thread, as served by the site (Shift_JIS encoded).
+    #
+    # `mht.php` used to answer with JSON pointing at a gzipped MHTML file; it now redirects straight
+    # to a single self-contained HTML document, so no MHTML parsing is involved anymore.
+    sig { params(thread_id: String).returns(String) }
     def get_thread_mht(thread_id)
-      # https://tsumanne.net/si/mht.php?id=129691
-      res = fetch_json(paths: ["mht.php"], query: { id: thread_id }, method: :get_response)
-      mht_gz = fetch(paths: [T.let(res["path"], String)])
-      # https://ksef-3go.hatenadiary.org/entry/20070924/1190563143
-      # https://docs.ruby-lang.org/ja/latest/method/Zlib=3a=3aInflate/s/new.html
-      zstream = Zlib::Inflate.new(Zlib::MAX_WBITS + 32)
-      buf = zstream.inflate(mht_gz)
-      zstream.finish
-      zstream.close
-      Mhtml::RootDocument.new(buf)
+      # https://tsumanne.net/si/mht.php?id=129691 -> 302 ./mhttmp/86279902.html
+      uri = join_paths(BASE_URL, [@board_id, "mht.php"])
+      uri.query = URI.encode_www_form({ id: thread_id })
+      res = Net::HTTP.get_response(uri)
+      raise Error, "thread #{thread_id} is not archived (HTTP #{res.code})" unless res.is_a?(Net::HTTPRedirection)
+
+      T.must(Net::HTTP.get(URI.join(uri, res["location"])))
     end
 
-    sig { params(thread_path: String).returns(T.nilable(Mhtml::RootDocument)) }
+    sig { params(thread_path: String).returns(T.nilable(String)) }
     def get_thread_from_path(thread_path)
       # https://tsumanne.net/si/data/2023/08/30/8883354/
       match_data = %r{^\d{4}/\d{2}/\d{2}/(?<thread_id>\d+)$}.match(thread_path)
       return if match_data.nil?
+
       get_thread_mht(T.must(match_data[:thread_id]))
     end
 
@@ -78,7 +79,8 @@ module Tsumanne
     def register_thread(uri, indexes: nil)
       # post, https://tsumanne.net/si/input.php?format=json&url=...&category=...
       RegisterThreadResponse.from_hash(
-        fetch_json(paths: ["input.php?format=json"], query: { url: uri, category: (indexes || []).join(",") }, method: :post)
+        fetch_json(paths: ["input.php?format=json"], query: { url: uri, category: (indexes || []).join(",") },
+                   method: :post)
       )
     end
 
@@ -97,11 +99,10 @@ module Tsumanne
       when :get
         uri.query = query
         Net::HTTP.get(uri)
-      when :get_response
-        uri.query = query
-        Net::HTTP.get_response(uri).body
       when :post
-        Net::HTTP.post(uri, query).body
+        # Without an explicit form content type the server does not populate `$_POST`,
+        # and answers with a PHP warning followed by a "not permitted URL" error.
+        Net::HTTP.post(uri, query, "Content-Type" => "application/x-www-form-urlencoded").body
       end
     end
 
@@ -118,7 +119,7 @@ module Tsumanne
 
     sig { params(base: String, paths: T::Array[String]).returns(URI::Generic) }
     def join_paths(base, *paths)
-      URI.parse(T.must(([base] + paths).reduce { File.join(_1, _2) }))
+      URI.parse(T.must(([base] + paths).reduce { |joined, path| File.join(joined, path) }))
     end
   end
 end

data/lib/tsumanne/cli.rb

@@ -2,12 +2,163 @@
 
 # frozen_string_literal: true
 
+require "json"
+require "uri"
+
+require "optimist"
+require "sorbet-runtime"
+
+require_relative "../tsumanne"
+
 module Tsumanne
+  # Command line entry point for the `tsumanne` executable.
   class CLI
     extend T::Sig
-    sig { params(args: BasicObject).returns(NilClass) }
+
+    # Subcommands, in the order `--help` lists them.
+    COMMANDS = T.let({
+      "threads" => "List the archived threads of a board",
+      "thread" => "Print an archived thread, by id or by archive path",
+      "search" => "Find the archived thread of a 2chan.net thread URI",
+      "indexes" => "List the indexes, or search them by keyword",
+      "register" => "Ask the site to archive a 2chan.net thread"
+    }.freeze, T::Hash[String, String])
+
+    # Runs `args` as a command line and returns the exit status the executable should exit with.
+    sig { params(args: T::Array[String]).returns(Integer) }
     def self.start(args)
-      p args
+      new(args).run
+    end
+
+    sig { params(args: T::Array[String]).void }
+    def initialize(args)
+      @args = T.let(args.dup, T::Array[String])
+      # Reassigned to the subcommand parser, so that `--help` educates about the command at hand.
+      @parser = T.let(global_parser, Optimist::Parser)
+    end
+
+    sig { returns(Integer) }
+    def run
+      execute
+      0
+    rescue Optimist::HelpNeeded
+      @parser.educate($stdout)
+      0
+    rescue Optimist::VersionNeeded
+      $stdout.puts(@parser.version)
+      0
+    rescue Optimist::CommandlineError, URI::InvalidURIError, Error => e
+      warn("Error: #{e.message}")
+      1
+    end
+
+    private
+
+    sig { void }
+    def execute
+      options = @parser.parse(@args)
+      # `stop_on` leaves the command and everything after it in `@args`.
+      command = @args.shift
+      raise Optimist::HelpNeeded if command.nil?
+      raise Error, "unknown command `#{command}`" unless COMMANDS.key?(command)
+
+      run_command(command, API.new(board_id: options[:board].to_sym))
+    end
+
+    sig { params(command: String, api: API).void }
+    def run_command(command, api)
+      case command
+      when "threads" then threads(api)
+      when "thread" then thread(api)
+      when "search" then search(api)
+      when "indexes" then indexes(api)
+      else register(api)
+      end
+    end
+
+    sig { params(api: API).void }
+    def threads(api)
+      options = parse_command("threads [options]") do |parser|
+        parser.opt(:index, "Index (category) to list", type: :string, default: "all")
+        parser.opt(:page, "Page to list", type: :integer, default: 1)
+      end
+      print_json(api.get_threads(index: options[:index], page: options[:page]))
+    end
+
+    sig { params(api: API).void }
+    def thread(api)
+      parse_command("thread [options] <thread id | archive path>")
+      target = operand("`thread` needs a thread id or an archive path")
+      $stdout.puts(/\A\d+\z/.match?(target) ? api.get_thread_mht(target) : thread_from_path(api, target))
+    end
+
+    sig { params(api: API, target: String).returns(String) }
+    def thread_from_path(api, target)
+      api.get_thread_from_path(target) ||
+        raise(Error, "`#{target}` is not an archive path (expected `YYYY/MM/DD/<thread id>`)")
+    end
+
+    sig { params(api: API).void }
+    def search(api)
+      parse_command("search [options] <thread uri>")
+      print_json(api.search_thread_from_uri(URI.parse(operand("`search` needs a 2chan.net thread URI"))))
+    end
+
+    sig { params(api: API).void }
+    def indexes(api)
+      options = parse_command("indexes [options]") do |parser|
+        parser.opt(:keyword, "Keyword to search the indexes for", type: :string)
+        parser.opt(:order, "Order to list the indexes in",
+                   type: :string, default: "newer", permitted: INDEXES_ORDERS.keys.map(&:to_s))
+        parser.opt(:page, "Page to list", type: :integer, default: 1)
+      end
+      print_json(api.search_indexes(keyword: options[:keyword], order: options[:order].to_sym, page: options[:page]))
+    end
+
+    sig { params(api: API).void }
+    def register(api)
+      options = parse_command("register [options] <thread uri>") do |parser|
+        parser.opt(:index, "Index (category) to file the thread under, repeatable", type: :string, multi: true)
+      end
+      uri = URI.parse(operand("`register` needs a 2chan.net thread URI"))
+      print_json(api.register_thread(uri, indexes: options[:index]))
+    end
+
+    sig { returns(Optimist::Parser) }
+    def global_parser
+      parser = Optimist::Parser.new
+      parser.usage("[global options] <command> [options] [arguments]")
+      parser.version("tsumanne #{VERSION}")
+      parser.opt(:board, "Board to talk to",
+                 type: :string, default: "img", permitted: BOARD_IDS.keys.map(&:to_s))
+      # Declared here so that `--help` lists them above the command list, instead of below it.
+      parser.opt(:version, "Print version and exit")
+      parser.opt(:help, "Show this message")
+      parser.stop_on(COMMANDS.keys)
+      parser.text("")
+      parser.text("Commands:")
+      COMMANDS.each { |name, description| parser.text(format("  %-8<name>s  %<description>s", name:, description:)) }
+      parser
+    end
+
+    # Parses the options of the command at hand, leaving its operands in `@args`.
+    sig { params(usage: String, blk: T.nilable(T.proc.params(parser: Optimist::Parser).void)).returns(T.untyped) }
+    def parse_command(usage, &blk)
+      parser = Optimist::Parser.new
+      parser.usage("[global options] #{usage}")
+      blk&.call(parser)
+      @parser = parser
+      parser.parse(@args)
+    end
+
+    sig { params(message: String).returns(String) }
+    def operand(message)
+      @args.shift || raise(Error, message)
+    end
+
+    sig { params(response: T::Struct).void }
+    def print_json(response)
+      $stdout.puts(JSON.generate(response.serialize))
     end
   end
 end

data/lib/tsumanne/models/get_threads.rb

@@ -1,10 +1,13 @@
 # typed: strict
+# frozen_string_literal: true
 
 require "sorbet-runtime"
 
+# Response of `GET /<board_id>/<index>/<page>`, a page of archived threads.
 class GetThreadsResponse < T::Struct
   extend T::Sig
 
+  # A single archived thread listed on the page.
   class Log < T::Struct
     extend T::Sig
 
@@ -14,11 +17,13 @@ class GetThreadsResponse < T::Struct
     const :close, T::Boolean
     const :res, Integer
     const :files, Integer
-    const :access, Integer
-    const :public, T::Boolean
+    # Only some boards report this one.
+    const :access, T.nilable(Integer)
     const :text, String
     const :thumb, String
-    const :del, Integer
+    # `public` and `del` are no longer part of the response; kept for older archives.
+    const :public, T.nilable(T::Boolean)
+    const :del, T.nilable(Integer)
     const :atid, T::Boolean
     const :last, String # Time
     const :path, String

data/lib/tsumanne/models/register_thread.rb

@@ -1,10 +1,13 @@
 # typed: strict
+# frozen_string_literal: true
 
 require "sorbet-runtime"
 
+# Response of `POST /<board_id>/input.php`, the thread registration endpoint.
 class RegisterThreadResponse < T::Struct
   extend T::Sig
 
+  # Result of attaching the requested categories to the registered thread.
   class Tags < T::Struct
     extend T::Sig
 
@@ -14,5 +17,6 @@ class RegisterThreadResponse < T::Struct
 
   const :success, T::Boolean
   const :messages, T::Array[String]
-  const :tags, Tags
+  # Only returned when the request actually attached categories.
+  const :tags, T.nilable(Tags)
 end

data/lib/tsumanne/models/search_indexes.rb

@@ -1,10 +1,13 @@
 # typed: strict
+# frozen_string_literal: true
 
 require "sorbet-runtime"
 
+# Response of `GET /<board_id>/indexes.php`, a keyword search over the indexes.
 class SearchIndexesResponse < T::Struct
   extend T::Sig
 
+  # A single index (category) tag matched by the search.
   class Tag < T::Struct
     extend T::Sig
 
@@ -12,7 +15,9 @@ class SearchIndexesResponse < T::Struct
   end
 
   const :success, T::Boolean
-  const :messages, T::Array[String]
-  const :count, Integer
+  const :messages, T.nilable(T::Array[String])
+  # `null` when listing every index instead of searching for one.
+  const :count, T.nilable(Integer)
+  const :lastpage, Integer
   const :tags, T::Array[Tag]
 end

data/lib/tsumanne/models/search_thread_from_uri.rb

@@ -1,11 +1,23 @@
 # typed: strict
+# frozen_string_literal: true
 
 require "sorbet-runtime"
 
+# Response of `GET /<board_id>/indexes.php?sbmt=URL`, a lookup of archived threads by their source URI.
 class SearchThreadFromUriResponse < T::Struct
   extend T::Sig
 
+  # An archived thread whose source URI matched the query.
+  class Log < T::Struct
+    extend T::Sig
+
+    const :id, Integer
+    const :url, String # URI
+    const :path, String
+    const :category, T::Array[String]
+  end
+
   const :success, T::Boolean
-  const :messages, T::Array[String]
-  const :path, String
+  const :logs, T::Array[Log]
+  const :messages, T.nilable(T::Array[String])
 end

data/lib/tsumanne/version.rb

@@ -3,5 +3,5 @@
 # frozen_string_literal: true
 
 module Tsumanne
-  VERSION = "0.0.3"
+  VERSION = "0.0.4"
 end

data/mise.toml

@@ -0,0 +1,42 @@
+[tools]
+"aqua:azu/dockerfile-pin" = "latest"
+claude = "latest"
+pinact = "latest"
+ruby = "latest"
+
+[tasks.install]
+description = "Install gem dependencies"
+run = "bundle install"
+
+[tasks.lint]
+description = "Run all linters and the type checker (no auto-fix)"
+depends = ["install"]
+run = [
+  "bundle exec rubocop",
+  "bundle exec mdl -c .mdlrc $(git ls-files '*.md')",
+  "bundle exec srb tc",
+  "bundle exec tapioca check-shims",
+]
+
+[tasks.test]
+description = "Run the RSpec suite"
+depends = ["install"]
+run = "bundle exec rspec"
+
+[tasks."lint:fix"]
+description = "Auto-fix what the linters and the type checker can fix"
+depends = ["install"]
+run = [
+  "bundle exec rubocop --autocorrect-all",
+  "bundle exec srb tc --autocorrect",
+]
+
+[tasks.pre-commit]
+description = "Auto-fix, then verify everything before committing"
+depends = ["lint:fix"]
+run = ["mise run lint", "mise run test"]
+
+[tasks.release]
+description = "Build and push the gem to RubyGems"
+depends = ["install"]
+run = "bundle exec rake release"

data/sorbet/rbi/gems/.gitattributes

@@ -0,0 +1 @@
+**/*.rbi linguist-generated=true