ciborium 0.1.0

data/ext/ciborium/src/scan.rs

@@ -0,0 +1,223 @@
+enum Pending {
+    Items(usize),
+    Indefinite { map: bool, items: usize },
+    Tag,
+    // Chunks of an indefinite-length byte (false) or text (true) string.
+    Chunks { text: bool },
+}
+
+/// Where a scan of an incomplete item stopped: the offset of the first head it could not read,
+/// relative to the start of the item, and the containers open at that point.
+#[derive(Default)]
+struct Progress {
+    resume: usize,
+    stack: Vec<Pending>,
+}
+
+/// The progress of scanning one item across calls, so a streaming decoder fed in parts scans
+/// each byte once rather than once per part.
+#[magnus::wrap(class = "Ciborium::Native::ScanState", free_immediately)]
+pub(crate) struct ScanState(std::cell::RefCell<Progress>);
+
+impl ScanState {
+    pub(crate) fn new() -> Self {
+        Self(std::cell::RefCell::new(Progress::default()))
+    }
+
+    /// Forgets the progress, for when the next scan starts at another item.
+    pub(crate) fn clear(&self) {
+        *self.0.borrow_mut() = Progress::default();
+    }
+}
+
+pub(crate) enum Scan {
+    Complete(usize),
+    Incomplete,
+}
+
+/// Finds where the data item at `pos` ends without building Ruby objects, checking the same
+/// well-formedness rules as the decoder except those that need the decoded values.
+pub(crate) struct Scanner<'a> {
+    cursor: crate::cursor::Cursor<'a>,
+    max_depth: usize,
+    stack: Vec<Pending>,
+}
+
+impl<'a> Scanner<'a> {
+    pub(crate) fn new(input: &'a [u8], pos: usize, max_depth: usize) -> Self {
+        Self {
+            cursor: crate::cursor::Cursor::new(input, pos),
+            max_depth,
+            stack: Vec::new(),
+        }
+    }
+
+    /// Scans the item at `pos`, continuing from where `state` stopped, and records where this
+    /// scan stops when the item is still incomplete.
+    pub(crate) fn scan_resuming(
+        input: &'a [u8],
+        pos: usize,
+        max_depth: usize,
+        state: &ScanState,
+    ) -> Result<Scan, crate::error::Error> {
+        let mut progress = state.0.borrow_mut();
+        let mut scanner = crate::scan::Scanner::new(input, pos, max_depth);
+        if pos + progress.resume <= input.len() {
+            scanner.cursor = crate::cursor::Cursor::new(input, pos + progress.resume);
+            scanner.stack = std::mem::take(&mut progress.stack);
+        }
+        let result = scanner.run();
+        *progress = match result {
+            Ok(Scan::Incomplete) => Progress {
+                resume: scanner.cursor.pos() - pos,
+                stack: scanner.stack,
+            },
+            _ => Progress::default(),
+        };
+        result
+    }
+
+    pub(crate) fn scan(&mut self) -> Result<usize, crate::error::Error> {
+        match self.run()? {
+            Scan::Complete(end) => Ok(end),
+            Scan::Incomplete => Err(crate::error::Error::Incomplete),
+        }
+    }
+
+    /// Scans until the item ends or the input does. On running out of input, the cursor and
+    /// stack are left at the start of the head that could not be read in full.
+    fn run(&mut self) -> Result<Scan, crate::error::Error> {
+        loop {
+            let start = self.cursor.pos();
+            match self.step(start) {
+                Ok(true) => return Ok(Scan::Complete(self.cursor.pos())),
+                Ok(false) => {}
+                Err(crate::error::Error::Incomplete) => {
+                    self.cursor = crate::cursor::Cursor::new(self.cursor.input(), start);
+                    return Ok(Scan::Incomplete);
+                }
+                Err(error) => return Err(error),
+            }
+        }
+    }
+
+    /// Reads one head with its body, if any. Returns true once the top-level item is finished.
+    /// Leaves the stack untouched when the input ends before the head and body do.
+    fn step(&mut self, start: usize) -> Result<bool, crate::error::Error> {
+        let header = self.cursor.head()?;
+        if let Some(Pending::Chunks { text }) = self.stack.last() {
+            let text = *text;
+            match header {
+                ciborium_ll::Header::Break => {
+                    self.stack.pop();
+                    return Ok(self.complete());
+                }
+                ciborium_ll::Header::Bytes(Some(len)) if !text => {
+                    self.cursor.string(Some(len), false)?;
+                }
+                ciborium_ll::Header::Text(Some(len)) if text => {
+                    self.cursor.string(Some(len), true)?;
+                }
+                _ => {
+                    return Err(crate::error::Error::malformed(format!(
+                        "invalid chunk in indefinite-length string at offset {start}"
+                    )));
+                }
+            }
+            return Ok(false);
+        }
+        match header {
+            ciborium_ll::Header::Positive(_)
+            | ciborium_ll::Header::Negative(_)
+            | ciborium_ll::Header::Float(_)
+            | ciborium_ll::Header::Simple(_)
+            | ciborium_ll::Header::Array(Some(0))
+            | ciborium_ll::Header::Map(Some(0)) => {}
+            ciborium_ll::Header::Bytes(Some(len)) => {
+                self.cursor.string(Some(len), false)?;
+            }
+            ciborium_ll::Header::Text(Some(len)) => {
+                self.cursor.string(Some(len), true)?;
+            }
+            ciborium_ll::Header::Bytes(None) => {
+                self.open(Pending::Chunks { text: false })?;
+                return Ok(false);
+            }
+            ciborium_ll::Header::Text(None) => {
+                self.open(Pending::Chunks { text: true })?;
+                return Ok(false);
+            }
+            ciborium_ll::Header::Array(Some(n)) => {
+                self.open(Pending::Items(n))?;
+                return Ok(false);
+            }
+            ciborium_ll::Header::Map(Some(n)) => {
+                self.open(Pending::Items(n.saturating_mul(2)))?;
+                return Ok(false);
+            }
+            ciborium_ll::Header::Array(None) => {
+                self.open(Pending::Indefinite {
+                    map: false,
+                    items: 0,
+                })?;
+                return Ok(false);
+            }
+            ciborium_ll::Header::Map(None) => {
+                self.open(Pending::Indefinite {
+                    map: true,
+                    items: 0,
+                })?;
+                return Ok(false);
+            }
+            ciborium_ll::Header::Tag(_) => {
+                self.open(Pending::Tag)?;
+                return Ok(false);
+            }
+            ciborium_ll::Header::Break => match self.stack.last() {
+                Some(Pending::Indefinite { map, items }) if !map || items % 2 == 0 => {
+                    self.stack.pop();
+                }
+                _ => {
+                    return Err(crate::error::Error::malformed(format!(
+                        "unexpected break at offset {start}"
+                    )));
+                }
+            },
+        }
+        Ok(self.complete())
+    }
+
+    fn open(&mut self, pending: Pending) -> Result<(), crate::error::Error> {
+        // An indefinite-length string adds no nesting the decoder counts, and holds only chunks,
+        // so it is always the innermost entry.
+        if !matches!(pending, Pending::Chunks { .. }) && self.stack.len() >= self.max_depth {
+            return Err(crate::error::Error::Depth(self.max_depth));
+        }
+        self.stack.push(pending);
+        Ok(())
+    }
+
+    /// Counts a finished item against its enclosing containers. Returns true once the top-level
+    /// item is finished.
+    fn complete(&mut self) -> bool {
+        loop {
+            match self.stack.last_mut() {
+                None => return true,
+                Some(Pending::Items(n)) => {
+                    *n -= 1;
+                    if *n > 0 {
+                        return false;
+                    }
+                }
+                Some(Pending::Indefinite { items, .. }) => {
+                    *items += 1;
+                    return false;
+                }
+                Some(Pending::Tag) => {}
+                // Popped before its string is counted, so never the innermost entry here.
+                Some(Pending::Chunks { .. }) => return false,
+            }
+            self.stack.pop();
+        }
+    }
+}

data/lib/ciborium/compat/buffer.rb

@@ -0,0 +1,190 @@
+# frozen_string_literal: true
+# rbs_inline: enabled
+
+module Ciborium
+  module Compat
+    # A byte queue in front of an optional IO, as CBOR::Buffer of the cbor gem: reads refill from
+    # the IO when the queue runs dry, and writes go out to it once the queue fills.
+    class Buffer
+      DEFAULT_IO_BUFFER_SIZE = 32 * 1024 #: Integer
+
+      # @rbs @io: untyped
+      # @rbs @io_buffer_size: Integer
+      # @rbs @data: String
+      # @rbs @pos: Integer
+
+      attr_reader :io #: untyped
+
+      # The cbor gem's read_reference_threshold and write_reference_threshold tune its chunk
+      # sharing, which this buffer does not do, so they are accepted and ignored.
+      #
+      #: (?untyped io, ?Hash[Symbol, untyped]? options) -> void
+      def initialize(io = nil, options = nil)
+        if options.nil? && io.is_a?(Hash)
+          options = io
+          io = nil
+        end
+        @io = io
+        @io_buffer_size = Integer(options ? options.fetch(:io_buffer_size, DEFAULT_IO_BUFFER_SIZE) : DEFAULT_IO_BUFFER_SIZE)
+        @data = "".b
+        @pos = 0
+      end
+
+      #: () -> nil
+      def clear
+        @data = "".b
+        @pos = 0
+        nil
+      end
+
+      #: () -> Integer
+      def size
+        @data.bytesize - @pos
+      end
+
+      #: () -> bool
+      def empty?
+        size.zero?
+      end
+
+      #: (String data) -> Integer
+      def write(data)
+        @data << data.b
+        flush if @io && size >= @io_buffer_size
+        data.bytesize
+      end
+
+      #: (String data) -> self
+      def <<(data)
+        write(data)
+        self
+      end
+
+      # Consumes up to n bytes, or everything without n, refilling from the IO until it ends.
+      # Returns nil when n is positive and nothing is left.
+      #
+      #: (?Integer? n, ?String? out) -> String?
+      def read(n = nil, out = nil)
+        return read_all(nil, out) if n.nil?
+        return replace(out, "".b) if n.zero?
+
+        fill(n)
+        return if empty?
+
+        replace(out, take(n))
+      end
+
+      # Consumes exactly n bytes, or everything without n. Raises EOFError, consuming nothing,
+      # when fewer than n bytes are left.
+      #
+      #: (?Integer? n, ?String? out) -> String
+      def read_all(n = nil, out = nil)
+        if n.nil?
+          fill(nil)
+          return replace(out, take(size))
+        end
+        return replace(out, "".b) if n.zero?
+
+        fill(n)
+        raise ::EOFError, "end of buffer reached" if size < n
+
+        replace(out, take(n))
+      end
+
+      # Consumes up to n bytes and returns how many it consumed.
+      #
+      #: (Integer n) -> Integer
+      def skip(n)
+        fill(n)
+        take(n).bytesize
+      end
+
+      # Consumes exactly n bytes, or raises EOFError consuming nothing.
+      #
+      #: (Integer n) -> self
+      def skip_all(n)
+        fill(n)
+        raise ::EOFError, "end of buffer reached" if size < n
+
+        take(n)
+        self
+      end
+
+      #: () -> String
+      def to_str
+        @data.byteslice(@pos..) || "".b
+      end
+
+      alias_method :to_s, :to_str
+
+      #: () -> Array[String]
+      def to_a
+        [to_str]
+      end
+
+      # Writes the queued bytes to the IO, if there is one.
+      #
+      #: () -> self
+      def flush
+        write_to(@io) if @io
+        self
+      end
+
+      #: () -> nil
+      def close
+        @io&.close
+        nil
+      end
+
+      # Writes the queued bytes to io and consumes them.
+      #
+      #: (untyped io) -> Integer
+      def write_to(io)
+        data = take(size)
+        io.write(data)
+        data.bytesize
+      end
+
+      #: (Integer? n) -> void
+      private def fill(n)
+        return unless @io
+
+        while n.nil? || size < n
+          chunk = read_io
+          break unless chunk
+
+          @data << chunk.b
+        end
+      end
+
+      #: () -> String?
+      private def read_io
+        if @io.respond_to?(:readpartial)
+          @io.readpartial(@io_buffer_size)
+        else
+          @io.read(@io_buffer_size)
+        end
+      rescue ::EOFError
+        nil
+      end
+
+      #: (Integer n) -> String
+      private def take(n)
+        data = @data.byteslice(@pos, n) || "".b
+        @pos += data.bytesize
+        if @pos == @data.bytesize
+          @data = "".b
+          @pos = 0
+        end
+        data
+      end
+
+      #: (String? out, String data) -> String
+      private def replace(out, data)
+        return data unless out
+
+        out.replace(data)
+      end
+    end
+  end
+end

data/lib/ciborium/compat/errors.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+# rbs_inline: enabled
+
+module Ciborium
+  module Compat
+    class UnpackError < StandardError; end
+
+    class MalformedFormatError < UnpackError; end
+
+    class StackError < UnpackError; end
+
+    # Raised by Unpacker#read_array_header and #read_map_header on an item of another type.
+    class TypeError < StandardError; end
+
+    # Raises the cbor gem's exception for a Ciborium decoding failure, keeping it as the cause.
+    #
+    #: [T] () { () -> T } -> T
+    def self.translate_errors
+      yield
+    rescue IncompleteError
+      raise ::EOFError, "end of buffer reached"
+    rescue DepthError
+      raise StackError, "nesting too deep"
+    rescue UnexpectedTypeError
+      raise TypeError, "unexpected type"
+    rescue DecodeError
+      raise MalformedFormatError, "malformed CBOR"
+    end
+  end
+end

data/lib/ciborium/compat/packer.rb

@@ -0,0 +1,90 @@
+# frozen_string_literal: true
+# rbs_inline: enabled
+
+module Ciborium
+  module Compat
+    # CBOR::Packer of the cbor gem, writing through Ciborium::Encoder.
+    class Packer
+      # @rbs @buffer: Buffer
+      # @rbs @encoder: Ciborium::Encoder
+
+      attr_reader :buffer #: Buffer
+
+      #: (?untyped io, ?Hash[Symbol, untyped]? options) -> void
+      def initialize(io = nil, options = nil)
+        @buffer = Buffer.new(io, options)
+        @encoder = Ciborium::Encoder.new(**Compat.encode_options)
+      end
+
+      #: (untyped object) -> self
+      def write(object)
+        @encoder.write(object)
+        drain
+      end
+
+      alias_method :pack, :write
+
+      #: () -> self
+      def write_nil
+        write(nil)
+      end
+
+      #: (Integer length) -> self
+      def write_array_header(length)
+        @encoder.write_array_header(length)
+        drain
+      end
+
+      #: (Integer length) -> self
+      def write_map_header(length)
+        @encoder.write_map_header(length)
+        drain
+      end
+
+      #: () -> self
+      def flush
+        @buffer.flush
+        self
+      end
+
+      #: () -> nil
+      def clear
+        @buffer.clear
+      end
+
+      #: () -> Integer
+      def size
+        @buffer.size
+      end
+
+      #: () -> bool
+      def empty?
+        @buffer.empty?
+      end
+
+      #: (untyped io) -> Integer
+      def write_to(io)
+        @buffer.write_to(io)
+      end
+
+      #: () -> String
+      def to_str
+        @buffer.to_str
+      end
+
+      alias_method :to_s, :to_str
+
+      #: () -> Array[String]
+      def to_a
+        @buffer.to_a
+      end
+
+      #: () -> self
+      private def drain
+        @buffer << @encoder.bytes
+        @encoder.clear
+        self
+      end
+    end
+  end
+end

data/lib/ciborium/compat/unpacker.rb

@@ -0,0 +1,111 @@
+# frozen_string_literal: true
+# rbs_inline: enabled
+
+module Ciborium
+  module Compat
+    # CBOR::Unpacker of the cbor gem, reading through Ciborium::Decoder. Input passes through
+    # buffer, which also reads from io when more is needed.
+    class Unpacker
+      # @rbs @buffer: Buffer
+      # @rbs @decoder: Ciborium::Decoder
+
+      attr_reader :buffer #: Buffer
+
+      # options takes symbolize_keys, and the Buffer options.
+      #
+      #: (?untyped io, ?Hash[Symbol, untyped]? options) -> void
+      def initialize(io = nil, options = nil)
+        if options.nil? && io.is_a?(Hash)
+          options = io
+          io = nil
+        end
+        @buffer = Buffer.new(io, options)
+        @decoder = Ciborium::Decoder.new(**Compat.decode_options, symbolize_keys: !!options&.[](:symbolize_keys))
+      end
+
+      #: (String data) -> self
+      def feed(data)
+        @buffer << data
+        self
+      end
+
+      #: () -> untyped
+      def read
+        with_input { @decoder.read }
+      end
+
+      alias_method :unpack, :read
+
+      #: () -> nil
+      def skip
+        with_input { @decoder.skip }
+        nil
+      end
+
+      # Whether the next item is null. Consumes nothing.
+      #
+      #: () -> bool
+      def skip_nil
+        with_input { @decoder.next_nil? }
+      end
+
+      #: () -> Integer
+      def read_array_header
+        with_input { @decoder.read_array_header }
+      end
+
+      #: () -> Integer
+      def read_map_header
+        with_input { @decoder.read_map_header }
+      end
+
+      # Yields each item until the input runs out, leaving a partial item buffered.
+      #
+      #: () { (untyped) -> void } -> nil
+      #: () -> Enumerator[untyped, nil]
+      def each
+        return enum_for(:each) unless block_given?
+
+        loop do
+          item = begin
+            read
+          rescue ::EOFError
+            break
+          end
+          yield item
+        end
+        nil
+      end
+
+      #: (String data) { (untyped) -> void } -> nil
+      def feed_each(data, &block)
+        feed(data)
+        each(&block)
+      end
+
+      #: () -> nil
+      def reset
+        @buffer.clear
+        @decoder.reset
+        nil
+      end
+
+      # Runs the block on the decoder, moving more input into it until the block stops raising
+      # IncompleteError or the input ends.
+      #
+      #: [T] () { () -> T } -> T
+      private def with_input
+        Compat.translate_errors do
+          @decoder.feed(@buffer.read_all(@buffer.size)) unless @buffer.empty?
+          yield
+        rescue IncompleteError
+          chunk = @buffer.read(Buffer::DEFAULT_IO_BUFFER_SIZE)
+          raise unless chunk
+
+          @decoder.feed(chunk)
+          retry
+        end
+      end
+    end
+  end
+end