pdfrb 0.1.0

data/lib/pdfrb/source/tokenizer.rb.bak

@@ -0,0 +1,320 @@
+# frozen_string_literal: true
+
+require "set"
+
+module Pdfrb
+  module Source
+    # Byte-level PDF lexer (s7.2). State machine that emits Token
+    # values via +next_token+ / +peek+. Pull-based so the Parser can
+    # stream through arbitrarily large PDFs without materialising the
+    # whole token stream.
+    #
+    # States:
+    #   :top       — between tokens; dispatch by first byte.
+    #   :string    — inside (...) with depth tracking and \ escapes.
+    #   :hexstring — inside <...> (after distinguishing << as dict open).
+    #   :comment   — inside %... until EOL.
+    #
+    # The keyword set per s7.2: obj/endobj/stream/endstream/xref/
+    # startxref/trailer/true/false/null. The Tokenizer emits them with
+    # type :keyword and value equal to the keyword string; the Parser
+    # dispatches on the value.
+    class Tokenizer
+      WHITESPACE_BYTES = PdfConstants::WHITESPACE.bytes.to_set.freeze
+      DELIMITER_BYTES  = PdfConstants::DELIMITERS.bytes.to_set.freeze
+
+      # Public so the Parser can recognise the same set when scanning
+      # post-stream whitespace.
+      public_constant :WHITESPACE_BYTES
+      public_constant :DELIMITER_BYTES
+
+      attr_reader :pos, :io
+
+      def initialize(io)
+        @io = io
+        @pos = 0
+        @pushback = []
+        @lookahead = []
+      end
+
+      def next_token
+        return @pushback.pop unless @pushback.empty?
+
+        fill_lookahead(1) if @lookahead.empty?
+        @lookahead.shift
+      end
+
+      def peek(offset = 0)
+        fill_lookahead(offset + 1)
+        @lookahead[offset]
+      end
+
+      def pushback(token)
+        @pushback << token
+        self
+      end
+
+      private
+
+      def fill_lookahead(n)
+        while @lookahead.length < n
+          tok = lex_one
+          break if tok.nil?
+
+          @lookahead << tok
+        end
+      end
+
+      def lex_one
+        skip_whitespace_and_comments
+        return nil if eos?
+
+        start_pos = @pos
+        b = peek_byte
+        case b
+        when 40  then read_string(start_pos)
+        when 60
+          peek_byte(1) == 60 ? read_two_byte_token(:dict_open, "<<", start_pos) : read_hex_string(start_pos)
+        when 62
+          peek_byte(1) == 62 ? read_two_byte_token(:dict_close, ">>", start_pos) : raise(lex_error("lone '>' at #{start_pos}"))
+        when 91  then advance_byte && emit(:array_open, "[", start_pos)
+        when 93  then advance_byte && emit(:array_close, "]", start_pos)
+        when 47  then read_name(start_pos)
+        when 43, 45, 46, *digit_bytes then read_number_or_keyword(start_pos)
+        else read_keyword(start_pos)
+        end
+      end
+
+      def digit_bytes
+        (48..57).to_a
+      end
+      private :digit_bytes
+
+      def eos?
+        @io.eof?
+      end
+
+      def peek_byte(offset = 0)
+        cur = @io.pos
+        if offset.zero?
+          b = @io.getbyte
+          @io.seek(cur, IO::SEEK_SET) if b
+          return b
+        end
+
+        @io.seek(cur + offset, IO::SEEK_SET)
+        b = @io.getbyte
+        @io.seek(cur, IO::SEEK_SET)
+        b
+      end
+
+      def advance_byte
+        @pos += 1
+        @io.getbyte
+      end
+
+      # Append an Integer byte to +buf+ without re-encoding through
+      # UTF-8 (which is the default for `+""` in a UTF-8 source file).
+      def append_byte(buf, b)
+        buf << b.chr(Encoding::BINARY)
+      end
+
+      def skip_whitespace_and_comments
+        loop do
+          break if eos?
+
+          b = peek_byte
+          if WHITESPACE_BYTES.include?(b)
+            advance_byte
+          elsif b == 37 # %
+            skip_comment
+          else
+            break
+          end
+        end
+      end
+
+      def skip_comment
+        until eos? || peek_byte == 10 # \n
+          advance_byte
+        end
+        advance_byte if !eos? && peek_byte == 10
+        nil
+      end
+
+      def read_two_byte_token(type, literal, start_pos)
+        advance_byte
+        advance_byte
+        emit(type, literal, start_pos)
+      end
+
+      def read_string(start_pos)
+        advance_byte # consume (
+        depth = 1
+        bytes = +""
+        until eos?
+          b = advance_byte
+          case b
+          when 92 # backslash
+            esc = read_string_escape
+            bytes << esc if esc >= 0
+          when 40 # (
+            depth += 1
+            bytes << 40
+          when 41 # )
+            depth -= 1
+            break if depth.zero?
+
+            bytes << 41
+          else
+            bytes << b
+          end
+        end
+        raise lex_error("unterminated string at #{start_pos}") if depth.positive?
+
+        emit(:string, bytes.force_encoding(Encoding::BINARY), start_pos)
+      end
+
+      def read_string_escape
+        b = advance_byte
+        case b
+        when 110 then 10    # \n
+        when 114 then 13    # \r
+        when 116 then 9     # \t
+        when 98  then 8     # \b
+        when 102 then 12    # \f
+        when 40  then 40    # \(
+        when 41  then 41    # \)
+        when 92  then 92    # \\
+        when 48..55 then read_octal_escape(b)
+        when 10, 13
+          # Line continuation — eat CRLF/LF/CR; emit no byte.
+          advance_byte if b == 13 && peek_byte == 10
+          -1
+        else b
+        end
+      end
+
+      def read_octal_escape(first)
+        digits = [first]
+        2.times do
+          break if eos?
+
+          b = peek_byte
+          break unless (48..55).cover?(b)
+
+          digits << advance_byte
+        end
+        digits.map(&:chr).join.to_i(8) & 0xFF
+      end
+
+      def read_hex_string(start_pos)
+        advance_byte # consume <
+        bytes = +""
+        hi = nil
+        until eos?
+          b = advance_byte
+          break if b == 62 # >
+
+          next if WHITESPACE_BYTES.include?(b)
+
+          nibble = hex_value(b)
+          raise lex_error("bad hex digit #{b.chr.inspect} at #{@pos}") unless nibble
+
+          if hi.nil?
+            hi = nibble
+          else
+            bytes << (hi * 16 + nibble)
+            hi = nil
+          end
+        end
+        bytes << (hi * 16 + 0) if hi # odd nibble padded with 0
+        emit(:hex_string, bytes.force_encoding(Encoding::BINARY), start_pos)
+      end
+
+      def hex_value(b)
+        return (b - 48) if (48..57).cover?(b)
+        return (b - 55) if (65..70).cover?(b)
+        return (b - 87) if (97..102).cover?(b)
+
+        nil
+      end
+
+      def read_name(start_pos)
+        advance_byte # consume /
+        bytes = +""
+        until eos?
+          b = peek_byte
+          break if WHITESPACE_BYTES.include?(b) || DELIMITER_BYTES.include?(b)
+
+          advance_byte
+          if b == 35 # # — expect two hex digits
+            hi = peek_byte
+            lo = peek_byte(1)
+            if hi && lo && (n1 = hex_value(hi)) && (n2 = hex_value(lo))
+              advance_byte; advance_byte
+              bytes << (n1 * 16 + n2)
+            else
+              bytes << 35
+            end
+          else
+            bytes << b
+          end
+        end
+        emit(:name, bytes.force_encoding(Encoding::BINARY), start_pos)
+      end
+
+      def read_number_or_keyword(start_pos)
+        bytes = +""
+        bytes << advance_byte # first char
+        until eos?
+          b = peek_byte
+          break if WHITESPACE_BYTES.include?(b) || DELIMITER_BYTES.include?(b)
+
+          advance_byte
+          bytes << b
+        end
+        emit_number_or_keyword(bytes, start_pos)
+      end
+
+      def read_keyword(start_pos)
+        bytes = +""
+        until eos?
+          b = peek_byte
+          break if WHITESPACE_BYTES.include?(b) || DELIMITER_BYTES.include?(b)
+
+          advance_byte
+          bytes << b
+        end
+        emit_keyword(bytes, start_pos)
+      end
+
+      def emit_number_or_keyword(bytes, start_pos)
+        if bytes.match?(/\A[-+]?\d+\z/)
+          emit(:integer, bytes.to_i, start_pos)
+        elsif bytes.match?(/\A[-+]?(\d*\.\d+|\d+\.?\d*)(?:[eE][-+]?\d+)?\z/) && (bytes.include?(".") || bytes =~ /[eE]/)
+          emit(:real, bytes.to_f, start_pos)
+        else
+          emit_keyword(bytes, start_pos)
+        end
+      end
+
+      def emit_keyword(bytes, start_pos)
+        case bytes
+        when "true"  then emit(:true, true, start_pos)
+        when "false" then emit(:false, false, start_pos)
+        when "null"  then emit(:null, nil, start_pos)
+        else emit(:keyword, bytes, start_pos)
+        end
+      end
+
+      def emit(type, value, position)
+        Token.new(type: type, value: value, position: position)
+      end
+
+      def lex_error(message)
+        Pdfrb::LexError.new(message, source_position: @pos)
+      end
+    end
+  end
+end

data/lib/pdfrb/source/trailer_reader.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+module Pdfrb
+  module Source
+    # Locates `startxref` near EOF and returns its offset. Per s7.5.5,
+    # the last `startxref` keyword in the file is authoritative.
+    module TrailerReader
+      SCAN_BYTES = 1024
+
+      module_function
+
+      # Returns the byte offset of the xref section (the value of the
+      # last `startxref` keyword in the file), or nil if not found.
+      def startxref_offset(io)
+        size = io.size
+        back = [SCAN_BYTES, size].min
+        io.seek(size - back, IO::SEEK_SET)
+        tail = io.read(back)
+        return nil unless tail
+
+        idx = tail.rindex("startxref")
+        return nil unless idx
+
+        rest = tail[(idx + "startxref".length)..]
+        m = rest.match(/(\d+)/)
+        return nil unless m
+
+        m[1].to_i
+      end
+    end
+  end
+end

data/lib/pdfrb/source/xref_stream_reader.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+module Pdfrb
+  module Source
+    # Reads an XRef stream (s7.5.8, PDF 1.5+). The stream's decoded
+    # bytes contain the xref entries; /W gives the byte widths of the
+    # three fields (type, oid-or-next-free, gen-or-index).
+    module XrefStreamReader
+      module_function
+
+      def read(stream_object, document)
+        dict = stream_object
+        decoded = stream_object.decoded_stream
+        w = dict[:W].to_a
+        raise Pdfrb::ParseError, "XRef stream missing /W" if w.nil? || w.length != 3
+
+        index = (dict[:Index] || [0, dict[:Size]]).to_a
+        section = XrefSection.new(size: dict[:Size] || 0)
+        pos = 0
+        bytes = decoded.bytes
+        (index.length / 2).times do |seg|
+          start_oid = index[seg * 2]
+          count = index[seg * 2 + 1]
+          count.times do |i|
+            type = read_field(bytes, pos, w[0]) || 1
+            pos += w[0]
+            f2 = read_field(bytes, pos, w[1])
+            pos += w[1]
+            f3 = read_field(bytes, pos, w[2])
+            pos += w[2]
+            oid = start_oid + i
+            case type
+            when 0 then section.add_free(oid, f3 || 0, f2 || 0)
+            when 1 then section.add_in_use(oid, f3 || 0, f2 || 0)
+            when 2 then section.add_compressed(oid, f2 || 0, f3 || 0)
+            end
+          end
+        end
+        section
+      end
+
+      def read_field(bytes, pos, width)
+        return nil if width.zero?
+
+        val = 0
+        width.times { |i| val = (val << 8) | (bytes[pos + i] || 0) }
+        val
+      end
+      private_class_method :read_field
+    end
+  end
+end

data/lib/pdfrb/source/xref_table_reader.rb

@@ -0,0 +1,72 @@
+# frozen_string_literal: true
+
+module Pdfrb
+  module Source
+    # Classical xref section parser (s7.5.4). Reads the table starting
+    # at the given byte offset and returns an +XrefSection+.
+    #
+    # Format:
+    #   xref
+    #   <start_oid> <count>
+    #   <10-digit offset> <5-digit gen> <n|f>
+    #   ...
+    #   trailer
+    module XrefTableReader
+      ENTRY_RE = /\A(\d{1,10})\s+(\d{1,5})\s+([nf])\z/.freeze
+      SUBHEADER_RE = /\A(\d+)\s+(\d+)\z/.freeze
+      private_constant :ENTRY_RE
+      private_constant :SUBHEADER_RE
+
+      module_function
+
+      def read(io, offset)
+        io.seek(offset, IO::SEEK_SET)
+        line = io.gets&.strip
+        raise Pdfrb::ParseError, "missing xref keyword at #{offset}" unless line == "xref"
+
+        section = XrefSection.new
+        next_oid = 0
+        while (raw = io.gets)
+          line = raw.strip
+          next if line.empty?
+          break if line == "trailer"
+
+          if (m = SUBHEADER_RE.match(line))
+            next_oid = m[1].to_i
+            count = m[2].to_i
+            count.times do |i|
+              row = io.gets&.strip
+              break unless row
+
+              add_entry(section, next_oid + i, row)
+            end
+            next_oid += 0 # subsection header advances oid implicitly above
+          elsif (m = ENTRY_RE.match(line))
+            # Lone entry without a subheader (some PDFs do this).
+            add_entry(section, next_oid, line)
+            next_oid += 1
+          else
+            # Unknown line — stop or skip. Be tolerant and stop.
+            break
+          end
+        end
+        section
+      end
+
+      def add_entry(section, oid, row)
+        m = ENTRY_RE.match(row)
+        return unless m
+
+        offset_val = m[1].to_i
+        gen = m[2].to_i
+        flag = m[3]
+        if flag == "n"
+          section.add_in_use(oid, gen, offset_val)
+        else
+          section.add_free(oid, gen, offset_val)
+        end
+      end
+      private_class_method :add_entry
+    end
+  end
+end

data/lib/pdfrb/source.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+module Pdfrb
+  # Reading layer: PDF bytes -> Tokenizer -> Parser -> COS object
+  # graph -> ObjectReader resolves References via xref -> Document.
+  #
+  # Same layering concept as `postscript/source/`: a byte-level
+  # state-machine Tokenizer feeds a recursive-descent Parser that
+  # emits Model values. References are stored as +Model::Reference+
+  # values and resolved lazily via the owning Document's ObjectReader.
+  module Source
+    autoload :Token, "pdfrb/source/token"
+    autoload :Tokenizer, "pdfrb/source/tokenizer"
+    autoload :Parser, "pdfrb/source/parser"
+    autoload :HeaderReader, "pdfrb/source/header_reader"
+    autoload :TrailerReader, "pdfrb/source/trailer_reader"
+    autoload :XrefTableReader, "pdfrb/source/xref_table_reader"
+    autoload :XrefStreamReader, "pdfrb/source/xref_stream_reader"
+    autoload :ObjectReader, "pdfrb/source/object_reader"
+    autoload :ObjectStreamReader, "pdfrb/source/object_stream_reader"
+    autoload :Recovery, "pdfrb/source/recovery"
+    autoload :LinearizationDetection, "pdfrb/source/linearization_detection"
+  end
+end

data/lib/pdfrb/task/extract_images.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+module Pdfrb
+  module Task
+    # Walk each page's /Resources/XObject for Image XObjects and yield
+    # (page, name, image_stream). Returns a flat list when no block.
+    module ExtractImages
+      module_function
+
+      ImageInfo = Struct.new(:page_index, :name, :stream,
+                             :width, :height, :filter, keyword_init: true)
+
+      def call(document)
+        results = []
+        document.pages.each_with_index do |page, idx|
+          xobjects = xobject_map(page, document)
+          xobjects&.each do |name, ref|
+            stream = ref.is_a?(Pdfrb::Model::Reference) ?
+                       document.object(ref) : ref
+            next unless stream.is_a?(Pdfrb::Model::Cos::Stream)
+            next unless stream.value[:Subtype] == :Image
+
+            info = ImageInfo.new(
+              page_index: idx,
+              name: name.to_s,
+              stream: stream,
+              width: stream.value[:Width],
+              height: stream.value[:Height],
+              filter: stream.value[:Filter]
+            )
+            block_given? ? yield(info) : (results << info)
+          end
+        end
+        block_given? ? document : results
+      end
+
+      def xobject_map(page, document)
+        # /Resources can be inheritable from a parent Pages node.
+        resources = page.value[:Resources]
+        if resources.nil?
+          parent_ref = page.value[:Parent]
+          parent = parent_ref && document.object(parent_ref)
+          resources = parent&.value&.dig(:Resources) if parent
+        end
+        return nil unless resources
+
+        resources = resources.value if resources.is_a?(Pdfrb::Model::Cos::Dictionary)
+        resources[:XObject]
+      end
+      module_function :xobject_map
+    end
+  end
+end