discord_store 0.1.0

data/lib/discord_store/transport/bucket.rb

@@ -0,0 +1,141 @@
+# frozen_string_literal: true
+
+module DiscordStore
+  module Transport
+    # One Discord rate-limit bucket.
+    #
+    # Discord does not publish its per-route limits; it reports them, per
+    # response, in the X-RateLimit-* headers, and groups routes into opaque
+    # buckets identified by X-RateLimit-Bucket. Several routes can share one
+    # bucket, so the limiter learns the mapping at runtime rather than assuming
+    # a table of constants that will be wrong by next quarter.
+    #
+    # The limit that dominates this library is five messages per five seconds
+    # per channel. That is the number that caps a naive Discord-backed store,
+    # and the only real answer to it is to write to more than one channel —
+    # which is why {ChannelShard} exists.
+    class Bucket
+      UNKNOWN = nil
+
+      attr_reader :hash_key
+
+      def initialize(hash_key: nil, clock: -> { Process.clock_gettime(Process::CLOCK_MONOTONIC) })
+        @hash_key = hash_key
+        @clock = clock
+        @limit = UNKNOWN
+        @remaining = UNKNOWN
+        @reset_at = nil
+        @mutex = Mutex.new
+        @condition = ConditionVariable.new
+      end
+
+      # Blocks until this bucket has room, then claims a slot optimistically.
+      # The claim is provisional: {#update} reconciles it against whatever the
+      # server actually reports, and the server's number always wins.
+      #
+      # @param timeout [Numeric, nil]
+      # @return [void]
+      # @raise [QuotaTimeoutError]
+      def acquire(timeout: nil)
+        deadline = timeout && (@clock.call + timeout)
+
+        @mutex.synchronize do
+          loop do
+            expire_window
+
+            # Nothing learned yet: let one request through so there is something
+            # to learn from.
+            if @remaining == UNKNOWN
+              @remaining = UNKNOWN
+              return
+            end
+
+            if @remaining.positive?
+              @remaining -= 1
+              return
+            end
+
+            wait = [@reset_at.to_f - @clock.call, 0.0].max
+
+            if deadline
+              remaining_time = deadline - @clock.call
+              if remaining_time <= 0
+                raise QuotaTimeoutError,
+                      "waited #{timeout}s for rate-limit bucket #{@hash_key || "(unlearned)"} to reset"
+              end
+
+              wait = [wait, remaining_time].min
+            end
+
+            # A zero wait with no deadline would spin; give the scheduler a tick.
+            @condition.wait(@mutex, wait.positive? ? wait : 0.01)
+          end
+        end
+      end
+
+      # Reconciles local state with the headers on a response.
+      #
+      # @param limit [Integer, nil]
+      # @param remaining [Integer, nil]
+      # @param reset_after [Float, nil] seconds until the window resets
+      # @return [void]
+      def update(limit: nil, remaining: nil, reset_after: nil)
+        @mutex.synchronize do
+          @limit = Integer(limit) if limit
+          @reset_at = @clock.call + Float(reset_after) if reset_after
+
+          if remaining
+            reported = Integer(remaining)
+            # Requests we have already let through but whose responses have not
+            # come back yet are not reflected in the server's count, so take the
+            # pessimistic view.
+            @remaining = @remaining == UNKNOWN ? reported : [@remaining, reported].min
+          end
+
+          @condition.broadcast
+        end
+      end
+
+      # Records the bucket hash the server assigned to this route.
+      #
+      # @param hash_key [String]
+      # @return [void]
+      def hash_key=(hash_key)
+        @mutex.synchronize { @hash_key = hash_key }
+      end
+
+      # Closes the bucket for +seconds+ after a 429.
+      #
+      # @param seconds [Numeric]
+      # @return [void]
+      def penalize(seconds)
+        @mutex.synchronize do
+          @remaining = 0
+          @reset_at = @clock.call + seconds.to_f
+        end
+      end
+
+      # @return [Hash] a snapshot, for instrumentation and tests
+      def state
+        @mutex.synchronize do
+          {
+            hash_key: @hash_key,
+            limit: @limit,
+            remaining: @remaining,
+            reset_in: @reset_at ? [@reset_at - @clock.call, 0.0].max : nil
+          }
+        end
+      end
+
+      private
+
+      # Assumes the mutex is held.
+      def expire_window
+        return unless @reset_at && @clock.call >= @reset_at
+
+        @remaining = @limit == UNKNOWN ? UNKNOWN : @limit
+        @reset_at = nil
+      end
+    end
+  end
+end

data/lib/discord_store/transport/fake.rb

@@ -0,0 +1,394 @@
+# frozen_string_literal: true
+
+require "json"
+require "time"
+require "uri"
+require "securerandom"
+require_relative "http"
+
+module DiscordStore
+  module Transport
+    # An in-memory Discord, good enough to develop and test the whole stack
+    # against without a bot token.
+    #
+    # It models the behaviour that actually shapes this library's design, not
+    # just the happy path:
+    #
+    #   * snowflake IDs that really are monotonic and time-encoded
+    #   * CDN links that carry an +ex+ expiry and stop working when it passes,
+    #     so any code that caches a URL instead of re-resolving it fails here
+    #     rather than in production a day after deploy
+    #   * rate-limit headers, and 429s on demand
+    #   * a two-week bulk-delete window
+    #   * messages from other authors, so the own-messages guard is exercised
+    #
+    # Not a general-purpose Discord mock. It implements exactly the endpoints
+    # {REST} calls.
+    class Fake
+      CDN_HOST = "https://cdn.discordapp.test"
+      CDN_TTL = 24 * 60 * 60
+
+      attr_reader :application_id, :channels, :requests
+
+      # @param application_id [String] the id the fake attributes our writes to
+      # @param clock [#call] returns a Time; move it forward to test expiry
+      # @param rate_limit [Hash] the window the fake advertises in its headers.
+      #   Permissive by default so that tests exercise logic rather than sleep;
+      #   pass Discord's real per-channel window ({limit: 5, reset_after: 5.0})
+      #   when the point of the test is the limiter itself.
+      def initialize(application_id: "111111111111111111", clock: -> { Time.now },
+                     rate_limit: { limit: 1000, reset_after: 0.05 })
+        @application_id = application_id.to_s
+        @clock = clock
+        @rate_limit = rate_limit
+        @channels = Hash.new { |hash, key| hash[key] = [] }
+        @attachments = {}
+        @guilds = {}
+        @requests = []
+        @pending_rate_limits = 0
+        @sequence = 0
+        @mutex = Mutex.new
+      end
+
+      # @param request [Request]
+      # @return [Response]
+      def call(request)
+        @mutex.synchronize do
+          @requests << request
+          uri = URI.parse(request.url)
+
+          return cdn_response(uri) if uri.host == URI.parse(CDN_HOST).host
+
+          if @pending_rate_limits.positive?
+            @pending_rate_limits -= 1
+            return rate_limited_response
+          end
+
+          dispatch(request, uri)
+        end
+      end
+
+      def close; end
+
+      # --- Test helpers ------------------------------------------------------
+
+      # Registers a guild so chunk-size discovery has something to read.
+      #
+      # @param id [String]
+      # @param premium_tier [Integer] 0..3
+      # @return [void]
+      def seed_guild(id, premium_tier: 0)
+        @guilds[id.to_s] = { "id" => id.to_s, "premium_tier" => premium_tier, "name" => "fake" }
+      end
+
+      # Adds a message this library did not write, to prove it gets skipped.
+      #
+      # @return [Hash] the message
+      def seed_foreign_message(channel_id, content:, author_id: "999999999999999999")
+        message = build_message(channel_id, content: content, author_id: author_id)
+        @channels[channel_id.to_s] << message
+        message
+      end
+
+      # Makes the next +count+ API calls answer 429.
+      #
+      # @return [void]
+      def inject_rate_limits(count: 1)
+        @pending_rate_limits = count
+      end
+
+      # @param channel_id [String]
+      # @return [Array<Hash>]
+      def messages_in(channel_id) = @channels[channel_id.to_s]
+
+      # @return [Integer] total API calls seen, for asserting on request counts
+      def request_count = @requests.size
+
+      # @return [Integer] bytes currently held across every attachment
+      def stored_bytes = @attachments.values.sum { |a| a[:content].bytesize }
+
+      private
+
+      def dispatch(request, uri)
+        method = request.verb.to_s.upcase
+        path = uri.path
+        query = URI.decode_www_form(uri.query.to_s).to_h
+
+        case method
+        when "POST"
+          case path
+          when %r{/channels/(\d+)/messages/bulk-delete\z} then bulk_delete(::Regexp.last_match(1), request)
+          when %r{/channels/(\d+)/messages\z} then create_message(::Regexp.last_match(1), request)
+          else not_found
+          end
+        when "PATCH"
+          if path =~ %r{/channels/(\d+)/messages/(\d+)\z}
+            edit_message(::Regexp.last_match(1), ::Regexp.last_match(2), request)
+          else
+            not_found
+          end
+        when "DELETE"
+          if path =~ %r{/channels/(\d+)/messages/(\d+)\z}
+            delete_message(::Regexp.last_match(1), ::Regexp.last_match(2))
+          else
+            not_found
+          end
+        when "GET"
+          dispatch_get(path, query)
+        else
+          not_found
+        end
+      end
+
+      def dispatch_get(path, query)
+        case path
+        when %r{/channels/(\d+)/messages/(\d+)\z}
+          get_message(::Regexp.last_match(1), ::Regexp.last_match(2))
+        when %r{/channels/(\d+)/messages\z}
+          list_messages(::Regexp.last_match(1), query)
+        when %r{/channels/(\d+)\z}
+          ok({ "id" => ::Regexp.last_match(1), "type" => 0 })
+        when %r{/guilds/(\d+)\z}
+          guild = @guilds[::Regexp.last_match(1)]
+          guild ? ok(guild) : not_found
+        when %r{/users/@me\z}
+          ok({ "id" => @application_id, "bot" => true })
+        else
+          not_found
+        end
+      end
+
+      # --- Endpoint implementations -----------------------------------------
+
+      def create_message(channel_id, request)
+        payload, files = parse_body(request)
+
+        nonce = payload["nonce"]
+        if nonce
+          existing = @channels[channel_id].find { |m| m["nonce"] == nonce }
+          # Discord deduplicates on nonce within a short window. Modelling it is
+          # what makes retry-after-timeout safe to test.
+          return ok(existing) if existing
+        end
+
+        message = build_message(channel_id, content: payload["content"].to_s, author_id: @application_id)
+        message["nonce"] = nonce if nonce
+
+        message["attachments"] = files.map.with_index do |file, index|
+          store_attachment(channel_id, message["id"], index, file)
+        end
+
+        @channels[channel_id] << message
+        ok(message)
+      end
+
+      def edit_message(channel_id, message_id, request)
+        message = find_message(channel_id, message_id)
+        return not_found unless message
+
+        payload, = parse_body(request)
+        message["content"] = payload["content"].to_s
+        message["edited_timestamp"] = @clock.call.utc.iso8601
+        ok(message)
+      end
+
+      def get_message(channel_id, message_id)
+        message = find_message(channel_id, message_id)
+        return not_found unless message
+
+        ok(refresh_attachment_urls(message))
+      end
+
+      def list_messages(channel_id, query)
+        limit = (query["limit"] || 50).to_i
+        messages = @channels[channel_id].sort_by { |m| m["id"].to_i }
+
+        if (after = query["after"])
+          messages = messages.select { |m| m["id"].to_i > after.to_i }
+          messages = messages.first(limit)
+        elsif (before = query["before"])
+          messages = messages.select { |m| m["id"].to_i < before.to_i }
+          messages = messages.last(limit)
+        else
+          messages = messages.last(limit)
+        end
+
+        # Discord returns newest first.
+        ok(messages.reverse.map { |m| refresh_attachment_urls(m) })
+      end
+
+      def delete_message(channel_id, message_id)
+        removed = @channels[channel_id].reject! { |m| m["id"] == message_id.to_s }
+        return not_found if removed.nil?
+
+        Response.new(status: 204, headers: rate_limit_headers, body: nil)
+      end
+
+      def bulk_delete(channel_id, request)
+        payload, = parse_body(request)
+        ids = Array(payload["messages"]).map(&:to_s)
+
+        cutoff = @clock.call - REST::BULK_DELETE_MAX_AGE
+        if ids.any? { |id| Snowflake.at(id) <= cutoff }
+          return Response.new(
+            status: 400,
+            headers: rate_limit_headers,
+            body: JSON.generate({
+                                  "code" => 50_034,
+                                  "message" => "You can only bulk delete messages that are " \
+                                               "under 14 days old."
+                                })
+          )
+        end
+
+        @channels[channel_id].reject! { |m| ids.include?(m["id"]) }
+        Response.new(status: 204, headers: rate_limit_headers, body: nil)
+      end
+
+      def cdn_response(uri)
+        segments = uri.path.split("/").reject(&:empty?)
+        attachment_id = segments[2]
+        record = @attachments[attachment_id]
+        return not_found unless record
+
+        query = URI.decode_www_form(uri.query.to_s).to_h
+        expires_at = query["ex"].to_s.to_i(16)
+
+        # The whole point of the fake: an expired link 404s, exactly as Discord's
+        # has since it started signing them.
+        if expires_at.positive? && @clock.call.to_i > expires_at
+          return Response.new(status: 404, headers: {},
+                              body: "This content is no longer available.")
+        end
+
+        Response.new(status: 200, headers: { "content-type" => "application/octet-stream" },
+                     body: record[:content])
+      end
+
+      # --- Helpers -----------------------------------------------------------
+
+      def build_message(channel_id, content:, author_id:)
+        {
+          "id" => next_snowflake,
+          "channel_id" => channel_id.to_s,
+          "content" => content,
+          "author" => { "id" => author_id.to_s, "bot" => author_id.to_s == @application_id },
+          "timestamp" => @clock.call.utc.iso8601,
+          "attachments" => [],
+          "pinned" => false
+        }
+      end
+
+      def store_attachment(channel_id, _message_id, index, file)
+        id = next_snowflake
+        @attachments[id] = { content: file[:content].to_s.dup.force_encoding(Encoding::BINARY),
+                             filename: file[:filename] }
+
+        {
+          "id" => id,
+          "filename" => file[:filename],
+          "size" => file[:content].to_s.bytesize,
+          "url" => signed_url(channel_id, id, file[:filename]),
+          "_index" => index
+        }
+      end
+
+      # Every read re-signs, exactly as Discord does. Code that stores the URL
+      # from a previous read instead of re-fetching the message will pass the
+      # first test and fail once the clock moves.
+      def refresh_attachment_urls(message)
+        copy = message.dup
+        copy["attachments"] = message["attachments"].map do |attachment|
+          url = signed_url(message["channel_id"], attachment["id"], attachment["filename"])
+          attachment.merge("url" => url)
+        end
+        copy
+      end
+
+      def signed_url(channel_id, attachment_id, filename)
+        issued = @clock.call.to_i
+        expires = issued + CDN_TTL
+        # A real HMAC over Discord's private key; here, any opaque value.
+        hmac = SecureRandom.hex(16)
+        "#{CDN_HOST}/attachments/#{channel_id}/#{attachment_id}/#{filename}" \
+          "?ex=#{expires.to_s(16)}&is=#{issued.to_s(16)}&hm=#{hmac}"
+      end
+
+      def find_message(channel_id, message_id)
+        @channels[channel_id.to_s].find { |m| m["id"] == message_id.to_s }
+      end
+
+      def next_snowflake
+        @sequence += 1
+        (Snowflake.from_time(@clock.call) | (@sequence & Snowflake::INCREMENT_MASK)).to_s
+      end
+
+      def parse_body(request)
+        content_type = (request.headers || {}).fetch("Content-Type", "")
+
+        if content_type.start_with?("multipart/form-data")
+          parse_multipart(request.body, content_type)
+        else
+          [JSON.parse(request.body.to_s), []]
+        end
+      rescue JSON::ParserError
+        [{}, []]
+      end
+
+      def parse_multipart(body, content_type)
+        boundary = content_type[/boundary=(.+)\z/, 1]
+        return [{}, []] unless boundary
+
+        payload = {}
+        files = []
+
+        body.to_s.split("--#{boundary}").each do |part|
+          headers, content = part.split("\r\n\r\n", 2)
+          next unless headers && content
+
+          content = content.sub(/\r\n\z/, "")
+
+          if headers.include?('name="payload_json"')
+            payload = JSON.parse(content)
+          elsif (filename = headers[/filename="([^"]*)"/, 1])
+            files << { filename: filename, content: content }
+          end
+        end
+
+        [payload, files]
+      end
+
+      def ok(payload)
+        Response.new(status: 200, headers: rate_limit_headers, body: JSON.generate(payload))
+      end
+
+      def not_found
+        Response.new(status: 404, headers: rate_limit_headers,
+                     body: JSON.generate({ "code" => 10_008, "message" => "Unknown Message" }))
+      end
+
+      def rate_limited_response
+        Response.new(
+          status: 429,
+          headers: rate_limit_headers.merge(
+            "x-ratelimit-remaining" => "0",
+            "x-ratelimit-reset-after" => "0.01",
+            "retry-after" => "0.01",
+            "x-ratelimit-scope" => "user"
+          ),
+          body: JSON.generate({ "message" => "You are being rate limited.", "retry_after" => 0.01 })
+        )
+      end
+
+      def rate_limit_headers
+        limit = @rate_limit[:limit]
+        {
+          "x-ratelimit-bucket" => "fake-bucket",
+          "x-ratelimit-limit" => limit.to_s,
+          "x-ratelimit-remaining" => (limit - 1).to_s,
+          "x-ratelimit-reset-after" => @rate_limit[:reset_after].to_s
+        }
+      end
+    end
+  end
+end

data/lib/discord_store/transport/http.rb

@@ -0,0 +1,174 @@
+# frozen_string_literal: true
+
+require "net/http"
+require "uri"
+require "securerandom"
+
+module DiscordStore
+  module Transport
+    # What every HTTP backend returns.
+    Response = Struct.new(:status, :headers, :body, keyword_init: true) do
+      def success? = status.between?(200, 299)
+      def rate_limited? = status == 429
+      def server_error? = status >= 500
+
+      def json
+        return nil if body.nil? || body.empty?
+
+        require "json"
+        JSON.parse(body)
+      rescue JSON::ParserError
+        nil
+      end
+    end
+
+    # A request that has not been sent yet. Kept as a value object so the fake
+    # backend can assert against exactly what the real one would have sent.
+    #
+    # The HTTP verb is +verb+, not +method+: a Struct member called method would
+    # shadow Object#method, and losing that on a value object passed through
+    # several layers is not worth the nicer name.
+    Request = Struct.new(:verb, :url, :headers, :body, keyword_init: true)
+
+    # Net::HTTP backend.
+    #
+    # Connections are cached per (host, port) in fiber-local storage rather than
+    # thread-local. Under a fiber scheduler two fibers on one thread run
+    # concurrently, and sharing one Net::HTTP object between them interleaves
+    # bytes on the socket and corrupts both responses — the same failure that
+    # forced Rails to grow a fiber-aware connection pool, arriving here through
+    # a completely different door.
+    class NetHTTP
+      # @param open_timeout [Numeric]
+      # @param read_timeout [Numeric]
+      # @param write_timeout [Numeric]
+      def initialize(open_timeout: 5.0, read_timeout: 30.0, write_timeout: 30.0)
+        @open_timeout = open_timeout
+        @read_timeout = read_timeout
+        @write_timeout = write_timeout
+      end
+
+      # @param request [Request]
+      # @return [Response]
+      def call(request)
+        uri = URI.parse(request.url)
+        http = connection_for(uri)
+
+        net_request = build_request(request, uri)
+        response = http.request(net_request)
+
+        Response.new(
+          status: response.code.to_i,
+          headers: flatten_headers(response),
+          body: response.body
+        )
+      rescue Net::OpenTimeout, Net::ReadTimeout, Net::WriteTimeout => e
+        # A dead route that silently drops packets is indistinguishable from a
+        # slow one, so every phase is bounded. Surfaced as a retryable 599.
+        Response.new(status: 599, headers: {}, body: "#{e.class}: #{e.message}")
+      rescue SystemCallError, OpenSSL::SSL::SSLError, IOError => e
+        drop_connection(uri)
+        Response.new(status: 599, headers: {}, body: "#{e.class}: #{e.message}")
+      end
+
+      # Closes every cached connection held by the current fiber.
+      #
+      # @return [void]
+      def close
+        connections.each_value { |http| http.finish if http.started? }
+        connections.clear
+      end
+
+      private
+
+      def build_request(request, uri)
+        klass = case request.verb.to_s.upcase
+                when "GET" then Net::HTTP::Get
+                when "POST" then Net::HTTP::Post
+                when "PATCH" then Net::HTTP::Patch
+                when "PUT" then Net::HTTP::Put
+                when "DELETE" then Net::HTTP::Delete
+                else raise ArgumentError, "unsupported method #{request.verb}"
+                end
+
+        net_request = klass.new(uri.request_uri)
+        (request.headers || {}).each { |key, value| net_request[key] = value }
+        net_request.body = request.body if request.body
+        net_request
+      end
+
+      def connection_for(uri)
+        key = "#{uri.scheme}://#{uri.host}:#{uri.port}"
+
+        cached = connections[key]
+        return cached if cached&.started?
+
+        http = Net::HTTP.new(uri.host, uri.port)
+        http.use_ssl = uri.scheme == "https"
+        http.open_timeout = @open_timeout
+        http.read_timeout = @read_timeout
+        http.write_timeout = @write_timeout
+        http.keep_alive_timeout = 30
+        http.start
+
+        connections[key] = http
+      end
+
+      def drop_connection(uri)
+        key = "#{uri.scheme}://#{uri.host}:#{uri.port}"
+        http = connections.delete(key)
+        http.finish if http&.started?
+      rescue IOError
+        # Already closed; nothing to do.
+      end
+
+      # Fiber-local, and therefore also thread-local: every thread has a root
+      # fiber, so this is strictly narrower than Thread.current storage.
+      def connections
+        Fiber[:discord_store_connections] ||= {}
+      end
+
+      def flatten_headers(response)
+        response.each_header.to_h { |key, value| [key.downcase, value] }
+      end
+    end
+
+    # Builds a multipart/form-data body the way Discord's attachment endpoints
+    # expect it: a +payload_json+ part carrying the message, plus one part per
+    # file named files[n].
+    module Multipart
+      module_function
+
+      # @param payload [String] the JSON message payload
+      # @param files [Array<Hash>] each {filename:, content:, content_type:}
+      # @return [Array(String, String)] content type and body
+      def encode(payload, files)
+        boundary = "----discordstore#{SecureRandom.hex(16)}"
+        body = +""
+
+        body << "--#{boundary}\r\n"
+        body << "Content-Disposition: form-data; name=\"payload_json\"\r\n"
+        body << "Content-Type: application/json\r\n\r\n"
+        body << payload
+        body << "\r\n"
+
+        files.each_with_index do |file, index|
+          body << "--#{boundary}\r\n"
+          body << "Content-Disposition: form-data; name=\"files[#{index}]\"; " \
+                  "filename=\"#{sanitize(file[:filename])}\"\r\n"
+          body << "Content-Type: #{file[:content_type] || "application/octet-stream"}\r\n\r\n"
+          body << file[:content].to_s.dup.force_encoding(Encoding::BINARY)
+          body << "\r\n"
+        end
+
+        body << "--#{boundary}--\r\n"
+
+        ["multipart/form-data; boundary=#{boundary}", body]
+      end
+
+      def sanitize(filename)
+        filename.to_s.gsub(/["\r\n\\]/, "_")
+      end
+    end
+  end
+end