oxygene 0.0.1 → 0.1.0

data/lib/oxygene/car_archive.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 
+require_relative 'car_section'
 require_relative 'cid'
 require_relative 'errors'
 require_relative 'extensions'
@@ -8,60 +9,128 @@ require 'base64'
 require 'cbor'
 require 'stringio'
 
-# CAR v1: https://ipld.io/specs/transport/car/carv1/
-# multicodec codes: https://github.com/multiformats/multicodec/blob/master/table.csv
-
 module Oxygene
-  class CARSection
-    attr_reader :cid
 
-    def initialize(cid, body_data)
-      @cid = cid
-      @body_data = body_data
-    end
-
-    def body
-      @body ||= CARArchive.convert_data(CBOR.decode(@body_data))
-    end
-  end
+  #
+  # Parses a Content Addressable Archive (CAR) bundle loaded e.g. from an ATProto firehose
+  # message or from a repo .car exported from a PDS. The header part is decoded immediately,
+  # while the subsequent body/data sections are lazily decoded only as requested.
+  #
+  # Only CAR version v1 is supported, and only limited to the features or variants used in ATProto
+  # (e.g. CIDs only in the v1 version as defined in [DASL](https://dasl.ing)).
+  #
+  # Related specifications:
+  #
+  # - [IPLD CAR v1 spec](https://ipld.io/specs/transport/car/carv1/)
+  # - [DASL CAR](https://dasl.ing/car.html)
+  # - [ATProto CAR file serialization](https://atproto.com/specs/repository#car-file-serialization)
+  # - [ATProto data model](https://atproto.com/specs/data-model)
+  # - [multicodec](https://github.com/multiformats/multicodec),
+  #   [multihash](https://github.com/multiformats/multihash)
+  #   and [multibase](https://github.com/multiformats/multibase)
+  #
 
   class CARArchive
     using Oxygene::Extensions
 
-    attr_reader :roots, :sections
+    # @return [Array<CID>] array of root CIDs listed in the archive header
+    attr_reader :roots
+
 
+    # Creates a CAR archive reader from an in-memory CAR file.
+    #
+    # @param data [String] CAR archive file as a binary string
+    # @raise [DecodeError] if the archive header has missing or invalid fields
+    # @raise [UnsupportedError] if the archive uses an unsupported CAR version
+    #
     def initialize(data)
       @sections = []
+      @section_map = {}
       @buffer = StringIO.new(data)
+      @map_needs_update = false
 
       read_header(@buffer)
     end
 
-    def section_with_cid(cid)
-      if section = @sections.detect { |s| s.cid == cid }
-        return section.body
+    # Looks up a section with a given CID in the archive, decoding sections as needed.
+    #
+    # Sections are parsed lazily – if the requested section has already been loaded,
+    # it's returned without parsing any more parts of the archive, otherwise unread
+    # sections are parsed only until a match is found.
+    #
+    # When making repeated lookups for sections in one archive, e.g. when walking
+    # the MST tree of a CAR repo, pass the `use_map: true` option, which tells
+    # `CARArchive` to build and use an index that maps CIDs to sections for quicker lookup
+    # (otherwise, parsed sections are searched sequentially). For performance, this isn't
+    # enabled by default, since the common case of processing firehose commit messages only
+    # does a single lookup to find the record data.
+    #
+    # The CID may be passed as either an {Oxygene::CID} object, or its {CID#cbor_form} string.
+    #
+    # @param cid [CID, String] a CID object or its binary CBOR representation, including the leading null byte
+    # @param use_map [Boolean] whether to use and update the lookup index that maps CIDs to sections
+    # @param return_body [Boolean] whether to return the section's JSON body directly instead of an {Oxygene::CARSection} object
+    #
+    # @return [Hash, Array, CARSection, nil] the requested section or its decoded body converted to ATProto JSON, or nil if not found
+    # @raise [DecodeError] if a section is truncated or malformed
+    # @raise [UnsupportedError] if a section uses an unsupported CID encoding
+
+    def section_with_cid(cid, use_map: false, return_body: true)
+      if found_section = find_parsed_section(cid, use_map)
+        return (return_body ? found_section.json_body : found_section)
       end
 
-      if @buffer
-        while !@buffer.eof?
-          section = read_section(@buffer)
-          return section.body if section.cid == cid
-        end
+      if found_section = parse_sections_until_match(cid, use_map)
+        return (return_body ? found_section.json_body : found_section)
       end
 
-      @buffer = nil
       nil
     end
 
+    # Returns the list of archive sections that have been parsed so far,
+    # without parsing any more sections.
+    #
+    # @return [Array<CARSection>] sections parsed so far, in archive order
+
+    def parsed_sections
+      @sections.dup.freeze
+    end
+
+    # Parses all sections in the archive if they haven't been loaded yet, and
+    # returns the list of all sections in the order as listed in the archive.
+    #
+    # Once all sections have been read, the original archive data buffer is
+    # released so it can be garbage-collected.
+    #
+    # @return [Array<CARSection>] all sections in the archive
+    # @raise [DecodeError] if a section is truncated or malformed
+    # @raise [UnsupportedError] if a section uses an unsupported CID encoding
+
     def sections
       if @buffer
-        read_section(@buffer) while !@buffer.eof?
+        if !@buffer.eof?
+          read_section(@buffer) while !@buffer.eof?
+          @map_needs_update = true
+        end
+
         @buffer = nil
       end
 
       @sections
     end
 
+    # @api private
+    #
+    # Converts decoded CBOR values to their ATProto-specific JSON representation.
+    #
+    # The passed hash or array is converted recursively in place. The conversion involves:
+    # - replacing binary strings with `$bytes` objects with Base64-encoded data
+    # - converting CBOR CID tags to `$link` objects holding an {Oxygene::CID}
+    #
+    # @param object [Hash, Array] decoded CBOR object to convert
+    # @return [Hash, Array] the same object, updated in place
+    # @raise [DecodeError] if the top-level value is not a hash or array
+
     def self.convert_data(object)
       if object.is_a?(Hash)
         object.each do |k, v|
@@ -88,16 +157,39 @@ module Oxygene
       end
     end
 
+    # @api private
+    #
+    # Converts a CBOR CID tag to an ATProto JSON
+    # [$link object](https://atproto.com/specs/data-model#json-representation).
+    #
+    # @param cid [CBOR::Tagged] CBOR tag containing the CID binary data
+    # @return [Hash{String => CID}] a `$link` JSON object
+    # @raise [DecodeError] if the value is not a supported CID
+
     def self.make_cid_link(cid)
       { '$link' => CID.from_cbor_tag(cid) }
     end
 
+    # @api private
+    #
+    # Converts a binary string to an ATProto JSON
+    # [$bytes object](https://atproto.com/specs/data-model#json-representation).
+    #
+    # @param data [String] binary data to encode
+    # @return [Hash{String => String}] a `$bytes` JSON object
+
     def self.make_bytes(data)
-      { '$bytes' => Base64.encode64(data).chomp.gsub(/=+$/, '') }
+      string = Base64.strict_encode64(data)
+      string.chomp!('=') while string.getbyte(-1) == 61
+
+      { '$bytes' => string }
     end
 
+    # Returns a string with a representation of the object for debugging purposes.
+    # @return [String]
+
     def inspect
-      vars = instance_variables.map { |v|
+      vars = (instance_variables - [:@section_map]).map { |v|
         if v == :@sections && @buffer
           "#{v}=[...]"
         else
@@ -108,50 +200,88 @@ module Oxygene
       "#<#{self.class}:0x#{object_id} #{vars.join(", ")}>"
     end
 
+
     private
 
+    def find_parsed_section(cid, use_map)
+      if use_map
+        if @map_needs_update
+          @sections.each { |s| @section_map[s.cid.cbor_form] ||= s }
+          @map_needs_update = false
+        end
+
+        key = cid.is_a?(CID) ? cid.cbor_form : cid
+        @section_map[key]
+      else
+        if cid.is_a?(CID)
+          @sections.detect { |s| s.cid == cid }
+        else
+          @sections.detect { |s| s.cid.cbor_form == cid }
+        end
+      end
+    end
+
+    def parse_sections_until_match(cid, use_map)
+      return if @buffer.nil?
+
+      is_cid = cid.is_a?(CID)
+
+      while !@buffer.eof?
+        section = read_section(@buffer)
+
+        if use_map
+          @section_map[section.cid.cbor_form] = section
+        else
+          @map_needs_update = true
+        end
+
+        match = is_cid ? section.cid == cid : section.cid.cbor_form == cid
+        return section if match
+      end
+
+      @buffer = nil
+    end
+
     def read_header(buffer)
       len = buffer.read_varint
+      raise DecodeError.new("Header length cannot be 0") if len == 0
 
       header_data = buffer.read(len)
       raise DecodeError.new("Header too short: #{header_data}") unless header_data.length == len
 
       header = CBOR.decode(header_data)
+      raise DecodeError.new("Metadata object should be a hash") unless header.is_a?(Hash)
       raise UnsupportedError.new("Unexpected CAR version: #{header['version']}") unless header['version'] == 1
-      @roots = header['roots'].map { |x| CID.from_cbor_tag(x) }
+
+      roots = header['roots']
+      raise DecodeError.new("Missing 'roots' field") if roots.nil?
+      raise DecodeError.new("Invalid 'roots' field: #{roots.inspect}") unless roots.is_a?(Array)
+
+      @roots = header['roots'].map { |x|
+        if x.is_a?(CBOR::Tagged) && x.tag == 42
+          CID.from_cbor_tag(x)
+        else
+          raise DecodeError.new("Unexpected value in the roots array: #{x.inspect}")
+        end
+      }.freeze
     end
 
     def read_section(buffer)
       len = buffer.read_varint
 
+      # TODO: verify content CIDs
       section_data = buffer.read(len)
       raise DecodeError.new("Section too short: #{section_data}") unless section_data.length == len
 
-      sbuffer = StringIO.new(section_data)
-
-      version = sbuffer.read_varint
-      raise UnsupportedError.new("Unexpected CID version: #{version}") unless version == 1
-
-      codec = sbuffer.read_varint
-      raise UnsupportedError.new("Unexpected CID codec: #{codec}") unless codec == 0x71  # dag-cbor
+      cid_data = section_data.byteslice(0, 36)
+      body_data = section_data.byteslice(36..)
+      cid_data.prepend(CID::CBOR_TAG_PREFIX)
+      cid = CID.new(cid_data, binary: true, cbor_prefix: true, codec: :drisl)
 
-      hash = sbuffer.read_varint
-      raise UnsupportedError.new("Unexpected CID hash: #{hash}") unless hash == 0x12  # sha2-256
-
-      clen = sbuffer.read_varint
-      raise UnsupportedError.new("Unexpected CID length: #{clen}") unless clen == 32
-
-      prefix = section_data[0...sbuffer.pos]
-
-      cid_data = sbuffer.read(clen)
-      raise DecodeError.new("CID too short: #{cid_data}") unless cid_data.length == clen
-
-      cid = CID.new(prefix + cid_data)
-
-      body_data = sbuffer.read
       new_section = CARSection.new(cid, body_data)
 
       @sections << new_section
+
       new_section
     end
   end

data/lib/oxygene/car_repo.rb

@@ -0,0 +1,121 @@
+# frozen_string_literal: true
+
+require_relative 'car_archive'
+require_relative 'errors'
+
+module Oxygene
+
+  #
+  # A CAR archive containing an account's ATProto repository, as returned by the
+  # `com.atproto.sync.getRepo` PDS endpoint.
+  #
+  # Oxygene currently supports repositories in the v3 format version.
+  #
+  # Related specifications:
+  # - [ATProto repository](https://atproto.com/specs/repository)
+  #
+
+  class CARRepo < CARArchive
+
+    # Returns the CAR archive section containing the repository's root commit.
+    # @return [CARSection] section identified by the first CAR root CID
+    attr_reader :commit_section
+
+    # Creates an ATProto repository reader from an in-memory CAR file.
+    #
+    # @param data [String] CAR archive file as a binary string
+    # @raise [DecodeError] if the CAR header has missing or invalid fields, the root commit is missing, or a section has invalid data
+    # @raise [UnsupportedError] if the archive, section CID or repository is in an unsupported version
+    #
+    def initialize(data)
+      super
+
+      raise DecodeError, "CAR repository has no root commit" if roots.empty?
+
+      @commit_section = section_with_cid(roots.first, use_map: true, return_body: false)
+      raise DecodeError, "Root commit not found in the archive: #{roots.first.inspect}" if @commit_section.nil?
+
+      commit_body = @commit_section.decoded_body
+      raise DecodeError, "Commit object should be a hash" unless commit_body.is_a?(Hash)
+
+      repo_version = commit_body['version']
+      raise UnsupportedError, "Unexpected repository version: #{repo_version.inspect}" unless repo_version == 3
+    end
+
+    # Returns the repository commit data.
+    #
+    # The commit is decoded from the body of the {#commit_section}. The data is
+    # returned in the ATProto JSON representation – CID links and binary strings
+    # are represented using `$link` and `$bytes` objects respectively. Use
+    # {#commit_section} to access the original CBOR bytes or decoded CBOR values
+    # without conversion.
+    #
+    # See the [ATProto repository spec](https://atproto.com/specs/repository#commit-objects)
+    # for what fields the commit object is expected to contain.
+    #
+    # @return [Hash] root commit data
+
+    def commit
+      commit_section.json_body
+    end
+
+    # Walks through the repository MST tree, running the passed block for each record
+    # in the repo in alphabetical key order.
+    #
+    # Records are stored in the Merkle Search Tree structure in nodes with assigned keys,
+    # where each key is a record path (NSID collection + rkey, e.g.
+    # `app.bsky.feed.post/3juhznhw65225`). The iterator returns all records one by one
+    # sorted alphabetically, so the list is sorted by collection first and then by rkey
+    # within a collection. The callback is passed the path key and the {CID} of the section
+    # which contains the actual record data, The CID can be used to extract the data
+    # through a call to {#section_with_cid} (you almost certainly want to pass
+    # `use_map: true`).
+    #
+    # Normally you can skip the `starting_node_cid` parameter, in which case the tree
+    # traversal begins at the repository root referenced by the CAR header and covers the
+    # entire tree. If a starting CID is passed, the traversal will start from a given node
+    # covering only a subtree.
+    #
+    # @param starting_node_cid [CID, String, nil] CID of the tree node to start from, or `nil` to start at the repository root
+    #
+    # @yield [key, cid] path key of a given record and the CID of its value block
+    # @yieldparam key [String] record path key, i.e. collection + / + rkey
+    # @yieldparam cid [CID] content identifier of the section containing the record value
+    # @raise [DecodeError] if a requested section is missing, a section is truncated or malformed, or a CID is invalid
+    # @raise [UnsupportedError] if a section uses an unsupported CID encoding
+
+    def walk_all_nodes(starting_node_cid = nil, &block)
+      if starting_node_cid.nil?
+        commit = commit_section.decoded_body
+        tree_top_cid = commit['data'].value
+        return walk_all_nodes(tree_top_cid, &block)
+      end
+
+      data = section_with_cid(starting_node_cid, use_map: true, return_body: false)&.decoded_body
+      raise DecodeError, "MST node not found in the archive: #{starting_node_cid.inspect}" if data.nil?
+
+      if data['l']
+        walk_all_nodes(data['l'].value, &block)
+      end
+
+      previous = nil
+
+      (data['e'] || []).each do |e|
+        if previous
+          key = previous[0...e['p']]
+          key << e['k']
+        else
+          key = e['k']
+        end
+
+        previous = key
+
+        block.call(key, CID.from_cbor_tag(e['v']))
+
+        if e['t']
+          walk_all_nodes(e['t'].value, &block)
+        end
+      end
+    end
+  end
+end

data/lib/oxygene/car_section.rb

@@ -0,0 +1,59 @@
+# frozen_string_literal: true
+
+require_relative 'car_archive'
+require 'cbor'
+
+module Oxygene
+
+  #
+  # Represents a single CID-addressed block from a {CARArchive}.
+  #
+
+  class CARSection
+
+    # @return [CID] CID of this section
+    attr_reader :cid
+
+    # @return [String] raw CBOR body binary data
+    attr_reader :data
+
+    # Creates a section from its CID and encoded body.
+    #
+    # @param cid [CID] content identifier of the section
+    # @param data [String] raw CBOR body data
+    #
+    def initialize(cid, data)
+      @cid = cid
+      @data = data
+    end
+
+    # Decodes the body from CBOR, without performing ATProto JSON specific conversions.
+    #
+    # The result is decoded once and memoized. CID links are represented as {CBOR::Tagged}
+    # objects, and byte strings remain binary strings.
+    #
+    # @return [Object] section data decoded from CBOR
+
+    def decoded_body
+      @decoded_body ||= CBOR.decode(@data)
+    end
+
+    # Decodes the body and converts it to an ATProto JSON compatible format.
+    #
+    # The conversion involves:
+    # - replacing binary strings with `$bytes` objects with Base64-encoded data
+    # - converting CBOR CID tags to `$link` objects holding an {Oxygene::CID}
+    #
+    # The result is decoded and stored separately from {#decoded_body}, so calling this
+    # method does not mutate the value returned by {#decoded_body}.
+    #
+    # @return [Hash, Array] decoded ATProto JSON compatible body
+    # @raise [DecodeError] if the decoded top-level value is not a hash or array
+
+    def json_body
+      @json_body ||= CARArchive.convert_data(CBOR.decode(@data))
+    end
+
+    alias body json_body
+  end
+end