discord_store 0.1.0

data/lib/discord_store/cipher.rb

@@ -0,0 +1,195 @@
+# frozen_string_literal: true
+
+require "openssl"
+require "base64"
+require "securerandom"
+
+module DiscordStore
+  # Payload encryption.
+  #
+  # Discord's own Developer Terms require that end-user data stored off-platform
+  # be encrypted at rest, so a library that stores data *on* the platform has no
+  # excuse for shipping plaintext as the default. Encryption is also the only
+  # thing standing between your records and every member of the guild who can
+  # scroll up.
+  #
+  # AES-256-GCM, not the AES-256-CBC that earlier Discord filesystems used: GCM
+  # authenticates, so a member with Manage Messages who edits one of our messages
+  # produces a decryption failure rather than silently corrupted data.
+  module Cipher
+    # Envelope prefix, so the format can change without ambiguity later.
+    VERSION_TAG = "ds1"
+    SEPARATOR = "."
+    IV_BYTES = 12
+    TAG_BYTES = 16
+    ALGORITHM = "aes-256-gcm"
+
+    # Builds the cipher named by +config+.
+    #
+    # @param config [DiscordStore::Configuration]
+    # @return [#seal, #open]
+    def self.build(config)
+      case config.cipher
+      when :none then Null.new
+      when :aes_256_gcm then AES256GCM.new(config.secret_key_bytes)
+      else
+        raise ConfigurationError, "unknown cipher #{config.cipher.inspect}"
+      end
+    end
+
+    # Generates a fresh key, base64-encoded and ready to paste into an
+    # environment variable.
+    #
+    # @return [String]
+    def self.generate_key
+      Base64.strict_encode64(SecureRandom.bytes(32))
+    end
+
+    # Passthrough. Writes readable JSON into the channel: good for a log you
+    # want humans to read in the Discord client, wrong for anything else.
+    class Null
+      def seal(plaintext, aad: nil) = plaintext.to_s
+      def open(ciphertext, aad: nil) = ciphertext.to_s
+      def seal_binary(plaintext, aad: nil) = plaintext.to_s
+      def open_binary(ciphertext, aad: nil) = ciphertext.to_s
+      def encrypting? = false
+      def overhead = 0
+      def binary_overhead = 0
+    end
+
+    # Authenticated encryption over a compact, markdown-safe envelope:
+    #
+    #   ds1.<iv>.<tag>.<ciphertext>     (each field urlsafe base64, unpadded)
+    #
+    # Base64url is used rather than standard base64 because Discord renders
+    # message content as markdown, and the urlsafe alphabet contains no
+    # characters the renderer will try to interpret.
+    class AES256GCM
+      def initialize(key)
+        raise ConfigurationError, "encryption key is missing" if key.nil?
+        raise ConfigurationError, "encryption key must be 32 bytes" unless key.bytesize == 32
+
+        @key = key
+      end
+
+      def encrypting? = true
+
+      # Worst-case added length for a given plaintext size, so the packer can
+      # decide whether a record still fits in a message before encrypting it.
+      #
+      # @return [Integer]
+      def overhead
+        # version tag + 3 separators + base64(iv) + base64(tag), plus the ~4/3
+        # expansion of the ciphertext itself, which the packer accounts for.
+        VERSION_TAG.bytesize + 3 + b64_len(IV_BYTES) + b64_len(TAG_BYTES)
+      end
+
+      # @param plaintext [String]
+      # @param aad [String, nil] additional authenticated data; bound into the
+      #   tag but not stored. Pass the channel ID to make a record undecryptable
+      #   if it is moved to another channel.
+      # @return [String]
+      def seal(plaintext, aad: nil)
+        cipher = OpenSSL::Cipher.new(ALGORITHM).encrypt
+        cipher.key = @key
+        iv = cipher.random_iv
+        cipher.auth_data = aad.to_s
+        ciphertext = cipher.update(plaintext.to_s) + cipher.final
+
+        [VERSION_TAG, encode(iv), encode(cipher.auth_tag), encode(ciphertext)].join(SEPARATOR)
+      end
+
+      # @param ciphertext [String] an envelope produced by {#seal}
+      # @param aad [String, nil] must match the value passed to {#seal}
+      # @return [String]
+      # @raise [DecryptionError] if the envelope is malformed, was written with
+      #   a different key, or has been tampered with
+      def open(ciphertext, aad: nil)
+        version, iv, tag, body = ciphertext.to_s.split(SEPARATOR, 4)
+
+        unless version == VERSION_TAG && iv && tag && body
+          raise DecryptionError, "not a #{VERSION_TAG} envelope"
+        end
+
+        cipher = OpenSSL::Cipher.new(ALGORITHM).decrypt
+        cipher.key = @key
+        cipher.iv = decode(iv)
+        cipher.auth_tag = decode(tag)
+        cipher.auth_data = aad.to_s
+        cipher.update(decode(body)) + cipher.final
+      rescue OpenSSL::Cipher::CipherError
+        raise DecryptionError,
+              "payload failed authentication: wrong key, wrong channel, or the " \
+              "message was edited by someone with Manage Messages"
+      rescue ArgumentError => e
+        raise DecryptionError, "malformed envelope: #{e.message}"
+      end
+
+      # Attachments are binary, so they skip base64 entirely.
+      #
+      #   DS1B | iv (12 bytes) | tag (16 bytes) | ciphertext
+      #
+      # Text envelopes have to survive being message content, which is why they
+      # are base64. Paying that 33% on an attachment would mean uploading four
+      # bytes for every three stored, against a rate limit that is already the
+      # binding constraint.
+      BINARY_MAGIC = "DS1B"
+
+      # @param plaintext [String]
+      # @param aad [String, nil]
+      # @return [String] binary
+      def seal_binary(plaintext, aad: nil)
+        cipher = OpenSSL::Cipher.new(ALGORITHM).encrypt
+        cipher.key = @key
+        iv = cipher.random_iv
+        cipher.auth_data = aad.to_s
+        ciphertext = cipher.update(plaintext.to_s) + cipher.final
+
+        (+"").force_encoding(Encoding::BINARY) << BINARY_MAGIC << iv << cipher.auth_tag << ciphertext
+      end
+
+      # @param blob [String] binary, as produced by {#seal_binary}
+      # @param aad [String, nil]
+      # @return [String]
+      # @raise [DecryptionError]
+      def open_binary(blob, aad: nil)
+        iv, tag, body = split_binary(blob)
+
+        cipher = OpenSSL::Cipher.new(ALGORITHM).decrypt
+        cipher.key = @key
+        cipher.iv = iv
+        cipher.auth_tag = tag
+        cipher.auth_data = aad.to_s
+        cipher.update(body) + cipher.final
+      rescue OpenSSL::Cipher::CipherError
+        raise DecryptionError, "attachment failed authentication: wrong key, or the bytes were altered"
+      end
+
+      # @return [Integer] bytes added to a binary payload
+      def binary_overhead = BINARY_MAGIC.bytesize + IV_BYTES + TAG_BYTES
+
+      private
+
+      # @return [Array(String, String, String)] iv, tag and ciphertext
+      def split_binary(blob)
+        blob = blob.to_s.dup.force_encoding(Encoding::BINARY)
+        raise DecryptionError, "not a #{BINARY_MAGIC} payload" unless blob.start_with?(BINARY_MAGIC)
+
+        offset = BINARY_MAGIC.bytesize
+        iv = blob.byteslice(offset, IV_BYTES)
+        tag = blob.byteslice(offset + IV_BYTES, TAG_BYTES)
+        body = blob.byteslice(offset + IV_BYTES + TAG_BYTES, blob.bytesize) || +""
+
+        raise DecryptionError, "payload is truncated" if iv.nil? || tag.nil? || tag.bytesize < TAG_BYTES
+
+        [iv, tag, body]
+      end
+
+      def encode(bytes) = Base64.urlsafe_encode64(bytes, padding: false)
+
+      def decode(string) = Base64.urlsafe_decode64(string)
+
+      def b64_len(bytes) = ((bytes * 4) / 3.0).ceil
+    end
+  end
+end

data/lib/discord_store/codec.rb

@@ -0,0 +1,231 @@
+# frozen_string_literal: true
+
+require "json"
+require "base64"
+
+module DiscordStore
+  # Turns records into message payloads and back.
+  #
+  # A Discord message holds 2000 characters, and a request that carries one
+  # 80-byte record is a request that spent the same rate-limit permit as one
+  # carrying twenty. Since permits are the scarce resource, packing is not an
+  # optimisation here — it is most of the throughput.
+  #
+  # Wire format of a message:
+  #
+  #   DS1 i 3            <- header: version, kind, record count
+  #   <envelope>         <- one record per line, base64url so a line break
+  #   <envelope>            can never appear inside one
+  #   <envelope>
+  #
+  # A record too large to fit alongside anything else spills to an attachment
+  # and the message body becomes a stub:
+  #
+  #   DS1 s 1
+  #
+  # Bigger than an attachment is the blob store's problem, not the log's.
+  class Codec
+    HEADER_PREFIX = "DS1"
+    KIND_INLINE = "i"
+    KIND_SPILL = "s"
+    SPILL_FILENAME = "records.ds1"
+    SEPARATOR = "\n"
+
+    # @param cipher [#seal, #open]
+    # @param content_budget [Integer] characters we are willing to use
+    # @param spill_limit [Integer, nil] max attachment bytes; nil means unlimited
+    def initialize(cipher:, content_budget: 1900, spill_limit: nil)
+      @cipher = cipher
+      @content_budget = content_budget
+      @spill_limit = spill_limit
+    end
+
+    # Packs records into as few messages as will hold them.
+    #
+    # @param records [Array<Log::Record>]
+    # @param aad [String, nil] bound into each envelope's authentication tag
+    # @param atomic [Boolean] when true, refuse to split the batch across
+    #   messages: either it fits in one message or it raises. One message is one
+    #   atomic append, so this is how a transaction gets all-or-nothing
+    #   durability on a platform with no transactions.
+    # @return [Array<Hash>] each {content:, files:, records:}
+    # @raise [PayloadTooLargeError]
+    def pack(records, aad: nil, atomic: false)
+      messages = []
+      batch = Batch.new(inline_budget)
+
+      records.each do |record|
+        envelope = seal(record, aad)
+
+        if envelope.length > inline_budget
+          raise_if_atomic(atomic, "a single record exceeds one message")
+          messages << current_message(batch.drain) if batch.any?
+          messages << spill_message(record, envelope)
+          next
+        end
+
+        unless batch.fits?(envelope)
+          raise_if_atomic(atomic, "the batch does not fit in one message")
+          messages << current_message(batch.drain)
+        end
+
+        batch.add(record, envelope)
+      end
+
+      messages << current_message(batch.drain) if batch.any?
+      messages
+    end
+
+    # Reverses {#pack} for one Discord message.
+    #
+    # @param message [Hash] a Discord message object
+    # @param attachment_body [String, nil] the spilled bytes, if the message has
+    #   a spill attachment the caller has already fetched
+    # @param aad [String, nil]
+    # @return [Array<Log::Record>]
+    def unpack(message, attachment_body: nil, aad: nil)
+      content = message["content"].to_s
+      header, body = split_header(content)
+      return [] if header.nil?
+
+      raw = if header[:kind] == KIND_SPILL
+              unless attachment_body
+                raise CorruptRecordError,
+                      "message #{message["id"]} spilled to an attachment that was not fetched"
+              end
+
+              attachment_body
+            else
+              body
+            end
+
+      lines = raw.split(SEPARATOR).reject(&:empty?)
+
+      if header[:count] != lines.size
+        raise CorruptRecordError,
+              "message #{message["id"]} claims #{header[:count]} records but carries #{lines.size}"
+      end
+
+      lines.each_with_index.map do |envelope, position|
+        payload = JSON.parse(@cipher.open(envelope, aad: aad))
+        Log::Record.from_wire(payload, lsn: message["id"], position: position,
+                                       channel_id: message["channel_id"])
+      end
+    rescue JSON::ParserError => e
+      raise CorruptRecordError, "message #{message["id"]} holds invalid JSON: #{e.message}"
+    end
+
+    # Whether a message looks like something this library wrote. Cheap enough to
+    # run before doing any crypto.
+    #
+    # @param message [Hash]
+    # @return [Boolean]
+    def ours?(message)
+      message.is_a?(Hash) && message["content"].to_s.start_with?("#{HEADER_PREFIX} ")
+    end
+
+    # @param message [Hash]
+    # @return [Boolean] whether the payload lives in an attachment
+    def spilled?(message)
+      header, = split_header(message["content"].to_s)
+      header && header[:kind] == KIND_SPILL
+    end
+
+    # Accumulates records until the next one would overflow a message.
+    class Batch
+      def initialize(budget)
+        @budget = budget
+        @pairs = []
+        @size = 0
+      end
+
+      def any? = @pairs.any?
+
+      # @return [Boolean] whether +envelope+ still fits alongside what is held
+      def fits?(envelope)
+        return true if @pairs.empty?
+
+        @size + envelope.length + SEPARATOR.length <= @budget
+      end
+
+      def add(record, envelope)
+        @size += envelope.length + (@pairs.empty? ? 0 : SEPARATOR.length)
+        @pairs << [record, envelope]
+      end
+
+      # @return [Array] the accumulated pairs, resetting the batch
+      def drain
+        pairs = @pairs
+        @pairs = []
+        @size = 0
+        pairs
+      end
+    end
+
+    private
+
+    def seal(record, aad)
+      envelope = @cipher.seal(JSON.generate(record.to_wire), aad: aad)
+
+      if envelope.include?(SEPARATOR)
+        raise CorruptRecordError, "cipher produced a newline; envelopes must be single-line"
+      end
+
+      envelope
+    end
+
+    # Header plus the newline that follows it, plus one separator per record
+    # after the first. Computed against the worst case so a full batch can never
+    # overflow the hard 2000-character limit.
+    def inline_budget
+      @content_budget - (HEADER_PREFIX.length + 8)
+    end
+
+    def current_message(pairs)
+      records = pairs.map(&:first)
+      envelopes = pairs.map(&:last)
+
+      {
+        content: [header(KIND_INLINE, records.size), *envelopes].join(SEPARATOR),
+        files: [],
+        records: records
+      }
+    end
+
+    def spill_message(record, envelope)
+      if @spill_limit && envelope.bytesize > @spill_limit
+        raise PayloadTooLargeError,
+              "record is #{envelope.bytesize} bytes, over the #{@spill_limit}-byte attachment " \
+              "ceiling. Store it as a blob instead of a log record."
+      end
+
+      {
+        content: header(KIND_SPILL, 1),
+        files: [{ filename: SPILL_FILENAME, content: envelope, content_type: "application/octet-stream" }],
+        records: [record]
+      }
+    end
+
+    def header(kind, count) = "#{HEADER_PREFIX} #{kind} #{count}"
+
+    def split_header(content)
+      header_line, body = content.split(SEPARATOR, 2)
+      return [nil, nil] if header_line.nil?
+
+      prefix, kind, count = header_line.split(" ", 3)
+      return [nil, nil] unless prefix == HEADER_PREFIX
+      return [nil, nil] unless [KIND_INLINE, KIND_SPILL].include?(kind)
+
+      [{ kind: kind, count: count.to_i }, body.to_s]
+    end
+
+    def raise_if_atomic(atomic, reason)
+      return unless atomic
+
+      raise PayloadTooLargeError,
+            "#{reason}, and this batch was requested atomic. One Discord message is the " \
+            "largest thing that commits all-or-nothing; split the transaction or accept " \
+            "that replay may observe it partially applied."
+    end
+  end
+end

data/lib/discord_store/configuration.rb

@@ -0,0 +1,197 @@
+# frozen_string_literal: true
+
+module DiscordStore
+  # Process-wide settings. Most applications configure this once at boot:
+  #
+  #   DiscordStore.configure do |c|
+  #     c.i_understand_this_violates_discord_tos = true
+  #     c.token       = ENV.fetch("DISCORD_BOT_TOKEN")
+  #     c.application_id = ENV.fetch("DISCORD_APPLICATION_ID")
+  #     c.guild_id    = ENV.fetch("DISCORD_GUILD_ID")
+  #     c.secret_key  = ENV.fetch("DISCORD_STORE_KEY") # 32 raw bytes, base64
+  #   end
+  class Configuration
+    # Discord's documented global ceiling for a bot token, in requests/second.
+    # We aim below it by default; see #global_rate_limit.
+    DISCORD_GLOBAL_LIMIT = 50
+
+    # Maximum characters in a normal message's +content+ field.
+    MESSAGE_CONTENT_LIMIT = 2000
+
+    # Every setting that has a sensible default, and its default.
+    DEFAULTS = {
+      global_rate_limit: 45,
+      quota_timeout: 15.0,
+      max_retries: 5,
+      open_timeout: 5.0,
+      read_timeout: 30.0,
+      write_timeout: 30.0,
+      user_agent: nil,
+      api_base: "https://discord.com/api/v10",
+      cipher: :aes_256_gcm,
+      content_budget: 1900,
+      max_concurrent_transfers: 24,
+      chunk_size: nil,
+      delete_policy: :tombstone,
+      own_messages_only: true,
+      logger: nil
+    }.freeze
+
+    # --- Acknowledgement -----------------------------------------------------
+
+    # Storing application data in Discord messages violates the Discord
+    # Developer Terms of Service. Nothing in this library will make a network
+    # request until this is explicitly set to true.
+    attr_accessor :i_understand_this_violates_discord_tos
+
+    # --- Credentials and placement -------------------------------------------
+
+    # Bot token, without the "Bot " prefix.
+    attr_accessor :token
+
+    # The bot's application (user) ID. Used to enforce that we only ever read
+    # messages our own bot wrote. Strongly recommended; see #own_messages_only.
+    attr_accessor :application_id
+
+    # The guild that owns the channels we write to.
+    attr_accessor :guild_id
+
+    # Channels used for the append-only log. More than one channel multiplies
+    # write throughput, because Discord's harshest message limit is per-channel.
+    attr_accessor :log_channel_ids
+
+    # Channel used for mutable documents (the KV layer).
+    attr_accessor :document_channel_id
+
+    # Channels used for blob chunks (ActiveStorage). Sharded like the log.
+    attr_accessor :blob_channel_ids
+
+    # Channel holding blob manifests. Kept separate from chunks so that a
+    # manifest scan does not have to page past gigabytes of chunk messages.
+    attr_accessor :manifest_channel_id
+
+    # --- Crypto --------------------------------------------------------------
+
+    # 32 raw bytes, base64-encoded. Required unless +cipher+ is :none.
+    attr_accessor :secret_key
+
+    # :aes_256_gcm (default) or :none. :none writes readable JSON into the
+    # channel, which is useful when you want humans to read the data in the
+    # Discord client, and which forfeits the encryption-at-rest that Discord's
+    # own developer terms require of anyone storing end-user data.
+    attr_accessor :cipher
+
+    # --- Transport -----------------------------------------------------------
+
+    attr_accessor :global_rate_limit, :quota_timeout, :max_retries,
+                  :open_timeout, :read_timeout, :write_timeout,
+                  :user_agent, :api_base, :logger
+
+    # Refuse to read any message not authored by +application_id+. This is the
+    # line between a storage backend and a scraper, and it is enforced in code
+    # rather than in documentation. Leave it on.
+    attr_accessor :own_messages_only
+
+    # --- Encoding ------------------------------------------------------------
+
+    # How many characters of a message's content we are willing to fill. Kept
+    # under MESSAGE_CONTENT_LIMIT so that framing overhead can never push a
+    # message over the hard limit.
+    attr_accessor :content_budget
+
+    # Concurrency ceiling for blob chunk transfers.
+    attr_accessor :max_concurrent_transfers
+
+    # Bytes per blob chunk. Left nil to be discovered at runtime from the
+    # guild's boost tier, because Discord's attachment ceiling has moved
+    # several times and hardcoding it is how these libraries break.
+    attr_accessor :chunk_size
+
+    # What to do when a blob or record is deleted:
+    #
+    #   :tombstone  — edit the message to a tombstone marker, never delete.
+    #                 Cheapest, and survives the 14-day bulk-delete window.
+    #   :bulk       — bulk-delete where possible, tombstone the rest.
+    #   :aggressive — delete everything, one request at a time if we must.
+    attr_accessor :delete_policy
+
+    def initialize
+      DEFAULTS.each { |name, value| public_send(:"#{name}=", value) }
+      @i_understand_this_violates_discord_tos = false
+      @log_channel_ids = []
+      @blob_channel_ids = []
+    end
+
+    # @raise [UnacknowledgedError] if the ToS acknowledgement is missing
+    # @raise [ConfigurationError] if a required setting is missing or invalid
+    # @return [void]
+    def validate!
+      raise UnacknowledgedError unless i_understand_this_violates_discord_tos
+      raise ConfigurationError, "token is required" if blank?(token)
+
+      if own_messages_only && blank?(application_id)
+        raise ConfigurationError,
+              "application_id is required while own_messages_only is enabled " \
+              "(it is what makes the check possible). Set it, or explicitly " \
+              "set own_messages_only = false and accept what that means."
+      end
+
+      validate_cipher!
+      validate_budget!
+      self
+    end
+
+    # The raw 32-byte encryption key.
+    #
+    # @return [String, nil]
+    def secret_key_bytes
+      return nil if cipher == :none
+      return nil if blank?(secret_key)
+
+      require "base64"
+      Base64.strict_decode64(secret_key.to_s)
+    rescue ArgumentError
+      raise ConfigurationError, "secret_key is not valid base64"
+    end
+
+    # Every channel this configuration knows about, deduplicated.
+    #
+    # @return [Array<String>]
+    def all_channel_ids
+      [
+        *log_channel_ids,
+        *blob_channel_ids,
+        document_channel_id,
+        manifest_channel_id
+      ].compact.map(&:to_s).uniq
+    end
+
+    private
+
+    def validate_cipher!
+      unless %i[aes_256_gcm none].include?(cipher)
+        raise ConfigurationError, "cipher must be :aes_256_gcm or :none, got #{cipher.inspect}"
+      end
+      return if cipher == :none
+
+      raise ConfigurationError, "secret_key is required unless cipher is :none" if blank?(secret_key)
+
+      bytes = secret_key_bytes
+      return if bytes && bytes.bytesize == 32
+
+      raise ConfigurationError,
+            "secret_key must decode to exactly 32 bytes, got #{bytes ? bytes.bytesize : 0}"
+    end
+
+    def validate_budget!
+      return if content_budget.positive? && content_budget < MESSAGE_CONTENT_LIMIT
+
+      raise ConfigurationError,
+            "content_budget must be between 1 and #{MESSAGE_CONTENT_LIMIT - 1}"
+    end
+
+    def blank?(value)
+      value.nil? || value.to_s.strip.empty?
+    end
+  end
+end