RubyGems - pdfrb - Versions diffs - 0.1.0 - Mend

pdfrb 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (822) hide show

data/lib/pdfrb/source/parser.rb ADDED Viewed

@@ -0,0 +1,281 @@
+# frozen_string_literal: true
+require "stringio"
+module Pdfrb
+  module Source
+    # Recursive-descent parser. Consumes tokens from a +Tokenizer+
+    # and produces Pdfrb::Model values.
+    #
+    # Recognised productions:
+    #   value      := scalar | array | dict | stream_indirect
+    #   scalar     := number | name | string | hex_string | boolean | null | ref
+    #   ref        := integer integer 'R'
+    #   array      := '[' value* ']'
+    #   dict       := '<<' (name value)* '>>'
+    #   obj_decl   := integer integer 'obj'
+    #   stream_obj := obj_decl dict 'stream' <bytes> 'endstream' 'endobj'
+    #
+    # References are emitted as +Model::Reference+ values (deferred
+    # resolution). The caller resolves them via the owning Document.
+    class Parser
+      attr_reader :tokenizer, :document
+      def initialize(tokenizer, document: nil)
+        @tokenizer = tokenizer
+        @document = document
+      end
+      def parse_value
+        tok = tokenizer.next_token
+        return nil if tok.nil?
+        case tok.type
+        when :integer, :real then maybe_reference(tok) || tok.value
+        when :name then Pdfrb::Model::Cos::NameEncoding.decode(tok.value)
+        when :string, :hex_string then tok.value
+        when :true then true
+        when :false then false
+        when :null then nil
+        when :array_open then parse_array_body
+        when :dict_open then parse_dict_body
+        else
+          raise syntax_error("unexpected token #{tok.inspect}")
+        end
+      end
+      # Parse `<N> <G> obj <body> endobj` (with optional stream).
+      # Returns a Model::Object subclass (Dictionary / Stream / etc.).
+      def parse_indirect_object
+        oid = expect_type(:integer).value
+        gen = expect_type(:integer).value
+        expect_keyword("obj")
+        value = parse_value
+        tok = tokenizer.peek
+        if tok&.type == :keyword && tok.value == "stream"
+          tokenizer.next_token
+          stream = read_stream_payload(value)
+          expect_keyword("endstream")
+          expect_keyword("endobj")
+          wrap_stream(value, stream, oid: oid, gen: gen)
+        else
+          expect_keyword("endobj")
+          wrap_object(value, oid: oid, gen: gen)
+        end
+      end
+      def parse_dict
+        tok = tokenizer.peek
+        return nil if tok.nil?
+        raise syntax_error("expected <<, got #{tok.inspect}") unless tok.type == :dict_open
+        tokenizer.next_token
+        parse_dict_body
+      end
+      private
+      def parse_array_body
+        arr = []
+        loop do
+          tok = tokenizer.peek
+          raise syntax_error("unterminated array") if tok.nil?
+          break if tok.type == :array_close
+          arr << parse_value
+        end
+        tokenizer.next_token # consume ]
+        arr
+      end
+      def parse_dict_body
+        hash = {}
+        loop do
+          tok = tokenizer.peek
+          raise syntax_error("unterminated dict") if tok.nil?
+          break if tok.type == :dict_close
+          key_tok = tokenizer.next_token
+          unless key_tok.type == :name
+            raise syntax_error("dict key must be a name, got #{key_tok.inspect}")
+          end
+          key = Pdfrb::Model::Cos::NameEncoding.decode(key_tok.value)
+          hash[key] = parse_value
+        end
+        tokenizer.next_token # consume >>
+        hash
+      end
+      # Lookahead: if next two tokens are <int> <'R'>, this is a Reference.
+      def maybe_reference(first_tok)
+        n1 = tokenizer.peek(0)
+        n2 = tokenizer.peek(1)
+        return nil unless n1&.type == :integer
+        return nil unless n2&.type == :keyword && n2.value == "R"
+        tokenizer.next_token # consume n1
+        tokenizer.next_token # consume R
+        Pdfrb::Model::Reference.new(first_tok.value.to_i, n1.value.to_i)
+      end
+      def read_stream_payload(dict_value)
+        skip_single_eol
+        # Always scan for "endstream" from the current position.
+        # /Length is unreliable in real-world PDFs (generator bugs,
+        # CRLF→LF conversion). Scanning is correct in all cases.
+        scan_to_endstream
+      end
+      def endstream_next?
+        cur = io.pos
+        b = io.getbyte
+        return false unless b
+        io.ungetbyte(b)
+        # Skip whitespace (max 2 bytes) then check for "endstream".
+        skipped = 0
+        while skipped < 2
+          b = io.getbyte
+          break unless b && Pdfrb::Source::Tokenizer::WHITESPACE_BYTES.include?(b)
+          skipped += 1
+        end
+        io.ungetbyte(b) if b && !Pdfrb::Source::Tokenizer::WHITESPACE_BYTES.include?(b)
+        peek_str = io.read(9)
+        io.seek(cur, IO::SEEK_SET)
+        peek_str == "endstream"
+      end
+      def scan_to_endstream
+        start = io.pos
+        chunk = io.read
+        idx = chunk.index("endstream")
+        if idx
+          # Position IO at the start of "endstream" so the caller's
+          # expect_keyword("endstream") can consume it normally.
+          io.seek(start + idx, IO::SEEK_SET)
+          content = chunk.byteslice(0, idx)
+          # Strip one trailing EOL before "endstream" (spec: it's
+          # not part of the stream data).
+          content = content.sub(/\r?\n\z/, "".b) if content.end_with?("\n")
+          content
+        else
+          io.seek(start, IO::SEEK_SET)
+          +""
+        end
+      end
+      def length_of(dict_value)
+        return nil unless dict_value.is_a?(::Hash)
+        case dict_value[:Length]
+        when Integer then dict_value[:Length]
+        when Pdfrb::Model::Reference
+          @document ? (@document.object(dict_value[:Length])&.value rescue nil) : nil
+        end
+      end
+      def read_bytes(n)
+        return +"".b if n.nil? || n.negative?
+        bytes = io.read(n) || +"".b
+        bytes.force_encoding(Encoding::BINARY)
+      end
+      def scan_to_endstream_length
+        # No /Length — scan from current pos to next "endstream".
+        # Returns the byte count up to (but not including) "endstream".
+        start = io.pos
+        chunk_size = 4096
+        window = +""
+        offset = 0
+        loop do
+          io.seek(start + offset, IO::SEEK_SET)
+          piece = io.read(chunk_size)
+          break unless piece
+          window << piece
+          idx = window.index("endstream")
+          if idx
+            io.seek(start + idx + "endstream".length, IO::SEEK_SET)
+            return idx
+          end
+          offset += [piece.length - "endstream".length + 1, 1].max
+        end
+        io.seek(start, IO::SEEK_SET)
+        0
+      end
+      def skip_single_eol
+        b = io.getbyte
+        return unless b
+        if b == 13 # \r
+          b2 = io.getbyte
+          io.ungetbyte(b2) unless b2.nil? || b2 == 10
+        elsif b != 10
+          io.ungetbyte(b)
+        end
+      end
+      def skip_whitespace
+        loop do
+          b = io.getbyte
+          break if b.nil?
+          if Pdfrb::Source::Tokenizer::WHITESPACE_BYTES.include?(b)
+            next
+          else
+            io.ungetbyte(b)
+            break
+          end
+        end
+      rescue EOFError
+        nil
+      end
+      def io
+        tokenizer.io
+      end
+      def wrap_object(value, oid:, gen:)
+        case value
+        when ::Hash
+          Pdfrb::Model::Cos::Dictionary.new(value, oid: oid, gen: gen, document: @document)
+        when ::Array
+          Pdfrb::Model::PdfArray.new(value, oid: oid, gen: gen, document: @document)
+        else
+          Pdfrb::Model::Object.new(value, oid: oid, gen: gen, document: @document)
+        end
+      end
+      def wrap_stream(dict_value, bytes, oid:, gen:)
+        dict = dict_value.is_a?(::Hash) ? dict_value : {}
+        Pdfrb::Model::Cos::Stream.new(dict, stream: bytes, oid: oid, gen: gen, document: @document)
+      end
+      def expect_type(type)
+        tok = tokenizer.next_token
+        raise syntax_error("expected #{type}, got #{tok.inspect}") unless tok&.type == type
+        tok
+      end
+      def expect_keyword(name)
+        tok = tokenizer.next_token
+        unless tok&.type == :keyword && tok.value == name
+          raise syntax_error("expected keyword #{name.inspect}, got #{tok.inspect}")
+        end
+        tok
+      end
+      def syntax_error(message)
+        Pdfrb::SyntaxError.new(message, source_position: tokenizer.pos)
+      end
+    end
+  end
+end

data/lib/pdfrb/source/recovery.rb ADDED Viewed

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+module Pdfrb
+  module Source
+    # Last-resort recovery: scan the entire file for `\d+ \d+ obj`
+    # patterns and synthesise an XrefSection from the offsets found.
+    # Triggered when the xref table is missing or points to a wrong
+    # offset.
+    module Recovery
+      OBJ_PATTERN = /(\d+)\s+(\d+)\s+obj\b/.freeze
+      private_constant :OBJ_PATTERN
+      module_function
+      def rebuild_xref(io, recover: true)
+        return nil unless recover
+        io.seek(0, IO::SEEK_SET)
+        section = XrefSection.new
+        data = io.read
+        data.force_encoding(Encoding::BINARY)
+        offset = 0
+        while (m = data.match(OBJ_PATTERN, offset))
+          oid = m[1].to_i
+          gen = m[2].to_i
+          entry_offset = m.begin(0)
+          # Take the last-seen gen per oid (PDF spec: highest gen wins).
+          section.add_in_use(oid, gen, entry_offset)
+          offset = m.end(0)
+        end
+        section
+      end
+    end
+  end
+end

data/lib/pdfrb/source/token.rb ADDED Viewed

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+module Pdfrb
+  module Source
+    # One lexed token. +type+ is a Symbol; +value+ is the literal
+    # bytes/string/number as parsed from source; +position+ is the
+    # byte offset in the IO where the token started (used for
+    # diagnostics and xref).
+    Token = Struct.new(:type, :value, :position, keyword_init: true) do
+      def inspect
+        "#<Pdfrb::Source::Token #{type} value=#{value.inspect} pos=#{position}>"
+      end
+    end
+  end
+end

data/lib/pdfrb/source/tokenizer.rb ADDED Viewed

@@ -0,0 +1,320 @@
+# frozen_string_literal: true
+require "set"
+module Pdfrb
+  module Source
+    # Byte-level PDF lexer (s7.2). State machine that emits Token
+    # values via +next_token+ / +peek+. Pull-based so the Parser can
+    # stream through arbitrarily large PDFs without materialising the
+    # whole token stream.
+    #
+    # States:
+    #   :top       — between tokens; dispatch by first byte.
+    #   :string    — inside (...) with depth tracking and \ escapes.
+    #   :hexstring — inside <...> (after distinguishing << as dict open).
+    #   :comment   — inside %... until EOL.
+    #
+    # The keyword set per s7.2: obj/endobj/stream/endstream/xref/
+    # startxref/trailer/true/false/null. The Tokenizer emits them with
+    # type :keyword and value equal to the keyword string; the Parser
+    # dispatches on the value.
+    class Tokenizer
+      WHITESPACE_BYTES = PdfConstants::WHITESPACE.bytes.to_set.freeze
+      DELIMITER_BYTES  = PdfConstants::DELIMITERS.bytes.to_set.freeze
+      # Public so the Parser can recognise the same set when scanning
+      # post-stream whitespace.
+      public_constant :WHITESPACE_BYTES
+      public_constant :DELIMITER_BYTES
+      attr_reader :pos, :io
+      def initialize(io)
+        @io = io
+        @pos = 0
+        @pushback = []
+        @lookahead = []
+      end
+      def next_token
+        return @pushback.pop unless @pushback.empty?
+        fill_lookahead(1) if @lookahead.empty?
+        @lookahead.shift
+      end
+      def peek(offset = 0)
+        fill_lookahead(offset + 1)
+        @lookahead[offset]
+      end
+      def pushback(token)
+        @pushback << token
+        self
+      end
+      private
+      def fill_lookahead(n)
+        while @lookahead.length < n
+          tok = lex_one
+          break if tok.nil?
+          @lookahead << tok
+        end
+      end
+      def lex_one
+        skip_whitespace_and_comments
+        return nil if eos?
+        start_pos = @pos
+        b = peek_byte
+        case b
+        when 40  then read_string(start_pos)
+        when 60
+          peek_byte(1) == 60 ? read_two_byte_token(:dict_open, "<<", start_pos) : read_hex_string(start_pos)
+        when 62
+          peek_byte(1) == 62 ? read_two_byte_token(:dict_close, ">>", start_pos) : raise(lex_error("lone '>' at #{start_pos}"))
+        when 91  then advance_byte && emit(:array_open, "[", start_pos)
+        when 93  then advance_byte && emit(:array_close, "]", start_pos)
+        when 47  then read_name(start_pos)
+        when 43, 45, 46, *digit_bytes then read_number_or_keyword(start_pos)
+        else read_keyword(start_pos)
+        end
+      end
+      def digit_bytes
+        (48..57).to_a
+      end
+      private :digit_bytes
+      def eos?
+        @io.eof?
+      end
+      def peek_byte(offset = 0)
+        cur = @io.pos
+        if offset.zero?
+          b = @io.getbyte
+          @io.seek(cur, IO::SEEK_SET) if b
+          return b
+        end
+        @io.seek(cur + offset, IO::SEEK_SET)
+        b = @io.getbyte
+        @io.seek(cur, IO::SEEK_SET)
+        b
+      end
+      def advance_byte
+        @pos += 1
+        @io.getbyte
+      end
+      # Append an Integer byte to +buf+ without re-encoding through
+      # UTF-8 (which is the default for `+""` in a UTF-8 source file).
+      def append_byte(buf, b)
+        buf << b.chr(Encoding::BINARY)
+      end
+      def skip_whitespace_and_comments
+        loop do
+          break if eos?
+          b = peek_byte
+          if WHITESPACE_BYTES.include?(b)
+            advance_byte
+          elsif b == 37 # %
+            skip_comment
+          else
+            break
+          end
+        end
+      end
+      def skip_comment
+        until eos? || peek_byte == 10 # \n
+          advance_byte
+        end
+        advance_byte if !eos? && peek_byte == 10
+        nil
+      end
+      def read_two_byte_token(type, literal, start_pos)
+        advance_byte
+        advance_byte
+        emit(type, literal, start_pos)
+      end
+      def read_string(start_pos)
+        advance_byte # consume (
+        depth = 1
+        bytes = +""
+        until eos?
+          b = advance_byte
+          case b
+          when 92 # backslash
+            esc = read_string_escape
+            append_byte(bytes, esc) if esc >= 0
+          when 40 # (
+            depth += 1
+            append_byte(bytes, 40)
+          when 41 # )
+            depth -= 1
+            break if depth.zero?
+            append_byte(bytes, 41)
+          else
+            append_byte(bytes, b)
+          end
+        end
+        raise lex_error("unterminated string at #{start_pos}") if depth.positive?
+        emit(:string, bytes.force_encoding(Encoding::BINARY), start_pos)
+      end
+      def read_string_escape
+        b = advance_byte
+        case b
+        when 110 then 10    # \n
+        when 114 then 13    # \r
+        when 116 then 9     # \t
+        when 98  then 8     # \b
+        when 102 then 12    # \f
+        when 40  then 40    # \(
+        when 41  then 41    # \)
+        when 92  then 92    # \\
+        when 48..55 then read_octal_escape(b)
+        when 10, 13
+          # Line continuation — eat CRLF/LF/CR; emit no byte.
+          advance_byte if b == 13 && peek_byte == 10
+          -1
+        else b
+        end
+      end
+      def read_octal_escape(first)
+        digits = [first]
+        2.times do
+          break if eos?
+          b = peek_byte
+          break unless (48..55).cover?(b)
+          digits << advance_byte
+        end
+        digits.map(&:chr).join.to_i(8) & 0xFF
+      end
+      def read_hex_string(start_pos)
+        advance_byte # consume <
+        bytes = +""
+        hi = nil
+        until eos?
+          b = advance_byte
+          break if b == 62 # >
+          next if WHITESPACE_BYTES.include?(b)
+          nibble = hex_value(b)
+          raise lex_error("bad hex digit #{b.chr.inspect} at #{@pos}") unless nibble
+          if hi.nil?
+            hi = nibble
+          else
+            append_byte(bytes, hi * 16 + nibble)
+            hi = nil
+          end
+        end
+        append_byte(bytes, hi * 16 + 0) if hi # odd nibble padded with 0
+        emit(:hex_string, bytes.force_encoding(Encoding::BINARY), start_pos)
+      end
+      def hex_value(b)
+        return (b - 48) if (48..57).cover?(b)
+        return (b - 55) if (65..70).cover?(b)
+        return (b - 87) if (97..102).cover?(b)
+        nil
+      end
+      def read_name(start_pos)
+        advance_byte # consume /
+        bytes = +""
+        until eos?
+          b = peek_byte
+          break if WHITESPACE_BYTES.include?(b) || DELIMITER_BYTES.include?(b)
+          advance_byte
+          if b == 35 # # — expect two hex digits
+            hi = peek_byte
+            lo = peek_byte(1)
+            if hi && lo && (n1 = hex_value(hi)) && (n2 = hex_value(lo))
+              advance_byte; advance_byte
+              append_byte(bytes, n1 * 16 + n2)
+            else
+              append_byte(bytes, 35)
+            end
+          else
+            append_byte(bytes, b)
+          end
+        end
+        emit(:name, bytes.force_encoding(Encoding::BINARY), start_pos)
+      end
+      def read_number_or_keyword(start_pos)
+        bytes = +""
+        append_byte(bytes, advance_byte) # first char
+        until eos?
+          b = peek_byte
+          break if WHITESPACE_BYTES.include?(b) || DELIMITER_BYTES.include?(b)
+          advance_byte
+          append_byte(bytes, b)
+        end
+        emit_number_or_keyword(bytes, start_pos)
+      end
+      def read_keyword(start_pos)
+        bytes = +""
+        until eos?
+          b = peek_byte
+          break if WHITESPACE_BYTES.include?(b) || DELIMITER_BYTES.include?(b)
+          advance_byte
+          append_byte(bytes, b)
+        end
+        emit_keyword(bytes, start_pos)
+      end
+      def emit_number_or_keyword(bytes, start_pos)
+        if bytes.match?(/\A[-+]?\d+\z/)
+          emit(:integer, bytes.to_i, start_pos)
+        elsif bytes.match?(/\A[-+]?(\d*\.\d+|\d+\.?\d*)(?:[eE][-+]?\d+)?\z/) && (bytes.include?(".") || bytes =~ /[eE]/)
+          emit(:real, bytes.to_f, start_pos)
+        else
+          emit_keyword(bytes, start_pos)
+        end
+      end
+      def emit_keyword(bytes, start_pos)
+        case bytes
+        when "true"  then emit(:true, true, start_pos)
+        when "false" then emit(:false, false, start_pos)
+        when "null"  then emit(:null, nil, start_pos)
+        else emit(:keyword, bytes, start_pos)
+        end
+      end
+      def emit(type, value, position)
+        Token.new(type: type, value: value, position: position)
+      end
+      def lex_error(message)
+        Pdfrb::LexError.new(message, source_position: @pos)
+      end
+    end
+  end
+end