pdfrb 0.1.0

data/lib/pdfrb/task/extract_text.rb

@@ -0,0 +1,232 @@
+# frozen_string_literal: true
+
+module Pdfrb
+  module Task
+    # Extract plain text from a document by walking each page's
+    # content stream. Decodes glyph codes back to Unicode using:
+    #
+    #   1. The font's /ToUnicode CMap (preferred — gives proper
+    #      Unicode for CIDFont / embedded TTF fonts).
+    #   2. The font's /Encoding + AdobeStandardEncoding glyph names
+    #      (fallback for Type1 standard fonts).
+    #   3. ASCII passthrough (last resort).
+    #
+    # Tracks the text matrix to insert line breaks at vertical moves
+    # and approximate paragraph breaks.
+    module ExtractText
+      module_function
+
+      def call(document)
+        results = []
+        document.pages.each do |page|
+          extractor = TextCollector.new(document, page)
+          begin
+            extractor.process(page.decoded_content)
+          rescue StandardError
+            # Malformed content stream — best effort, skip.
+          end
+          text = extractor.text
+          block_given? ? yield(page, text) : (results << text)
+        end
+        block_given? ? document : results
+      end
+
+      # Internal: collects bytes from +Tj+ / +TJ+ / +'+ / +"'.
+      # Tracks text-matrix y to insert line breaks at vertical moves.
+      # Resolves glyph codes to Unicode via the active font's
+      # /ToUnicode CMap or /Encoding.
+      class TextCollector < Pdfrb::Content::Processor
+        attr_reader :text
+
+        def initialize(document, page)
+          super()
+          @document = document
+          @page = page
+          @text = +""
+          @last_y = nil
+          @font_resolver = FontResolver.new(document, page)
+        end
+
+        def show_text(str)
+          check_line_break
+          @text << decode(str)
+        end
+
+        def show_text_array(arr)
+          check_line_break
+          arr.each do |e|
+            case e
+            when ::String then @text << decode(e)
+            when Numeric then next # kerning adjustment; ignore
+            end
+          end
+        end
+
+        def move_text(tx, ty, **)
+          super
+          check_line_break
+        end
+
+        def update_text_state(**overrides)
+          super
+          @font_resolver.activate(graphics_state.text_state.font_name) if overrides.key?(:font_name)
+        end
+
+        private
+
+        def check_line_break
+          cur_y = graphics_state.text_state.text_matrix.f
+          if @last_y && (cur_y - @last_y).abs > 0.001
+            @text << "\n"
+          end
+          @last_y = cur_y
+        end
+
+        def decode(bytes)
+          @font_resolver.decode(bytes)
+        end
+      end
+
+      # Resolves glyph codes to Unicode strings for the active font.
+      class FontResolver
+        def initialize(document, page)
+          @document = document
+          @page = page
+          @cache = {} # font_name -> decoder (CMap or Encoding)
+          @active = nil
+        end
+
+        def activate(font_name)
+          return if font_name.nil? || font_name == @active_name
+
+          @active_name = font_name
+          @active = loader_for(font_name)
+        end
+
+        def decode(bytes)
+          return bytes.to_s.dup.force_encoding("UTF-8") unless @active
+
+          @active.decode(bytes.to_s)
+        end
+
+        private
+
+        def loader_for(font_name)
+          return @cache[font_name] if @cache.key?(font_name)
+
+          @cache[font_name] = build_decoder(font_name)
+        end
+
+        def build_decoder(font_name)
+          font_dict = find_font(font_name)
+          return IdentityDecoder.new unless font_dict
+
+          ToUnicodeDecoder.new(font_dict) || EncodingDecoder.new(font_dict) || IdentityDecoder.new
+        end
+
+        def find_font(name)
+          resources = @page[:Resources]
+          return nil unless resources
+
+          resources = @document.object(resources) if resources.is_a?(Pdfrb::Model::Reference)
+          fonts = resources[:Font]
+          return nil unless fonts
+
+          fonts = @document.object(fonts) if fonts.is_a?(Pdfrb::Model::Reference)
+          return nil unless fonts.is_a?(::Hash)
+
+          entry = fonts[name] || fonts[name.to_sym] || fonts[name.to_s]
+          entry.is_a?(Pdfrb::Model::Reference) ? @document.object(entry) : entry
+        end
+      end
+      private_constant :FontResolver
+
+      # Uses the font's /ToUnicode CMap to decode glyph codes.
+      class ToUnicodeDecoder
+        def initialize(font_dict)
+          @font_dict = font_dict
+          @cmap = parse_tounicode(font_dict[:ToUnicode])
+        end
+
+        def self.for(font_dict); new(font_dict); end
+
+        def decode(bytes)
+          return bytes unless @cmap
+
+          bytes_to_unicode(bytes)
+        end
+
+        private
+
+        def parse_tounicode(ref)
+          return nil unless ref
+
+          doc = @font_dict.respond_to?(:document) ? @font_dict.document : nil
+          stream = if ref.is_a?(Pdfrb::Model::Reference) && doc
+                     doc.object(ref)
+                   else
+                     ref
+                   end
+          return nil unless stream.is_a?(Pdfrb::Model::Cos::Stream)
+
+          data = stream.decoded_stream
+          Pdfrb::Font::CMap::Parser.parse(data)
+        rescue StandardError
+          nil
+        end
+
+        def bytes_to_unicode(bytes)
+          result = +""
+          i = 0
+          while i < bytes.length
+            # Try 2-byte code first (most CIDFonts use 2-byte codes).
+            code2 = bytes.getbyte(i) * 256
+            code2 += bytes.getbyte(i + 1) if i + 1 < bytes.length
+            decoded = @cmap.decode(code2)
+            if decoded
+              result << decoded
+              i += 2
+              next
+            end
+
+            code1 = bytes.getbyte(i)
+            decoded1 = @cmap.decode(code1)
+            result << (decoded1 || fallback_char(code1))
+            i += 1
+          end
+          result
+        end
+
+        def fallback_char(byte)
+          byte < 128 ? byte.chr : "?"
+        end
+      end
+      private_constant :ToUnicodeDecoder
+
+      # Uses the font's /Encoding (WinAnsiEncoding etc.) to decode bytes.
+      class EncodingDecoder
+        def initialize(font_dict)
+          @font_dict = font_dict
+        end
+
+        def decode(bytes)
+          encoding_name = @font_dict[:Encoding]
+          case encoding_name
+          when Symbol, ::String
+            Pdfrb::Font::Encoding.decode(encoding_name, bytes)
+          else
+            bytes
+          end
+        end
+      end
+      private_constant :EncodingDecoder
+
+      # Identity passthrough — assumes bytes are already UTF-8 (or
+      # just gives back raw bytes for the caller to handle).
+      class IdentityDecoder
+        def decode(bytes); bytes.to_s; end
+      end
+      private_constant :IdentityDecoder
+    end
+  end
+end

data/lib/pdfrb/task/merge.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+module Pdfrb
+  module Task
+    # Merge multiple source PDFs into a target Document. Each source
+    # page is deep-copied via +Importer+ (which preserves resources,
+    # fonts, content streams). Source documents remain unchanged.
+    module Merge
+      module_function
+
+      def call(target, *sources)
+        sources.each { |src| import_all_pages(target, src) }
+        target
+      end
+
+      def import_all_pages(target, source)
+        importer = Pdfrb::Importer.new(target)
+        source.pages.each do |page|
+          new_value = importer.import(page.value, source)
+          new_value[:Parent] = parent_ref(target)
+          new_page = target.add(new_value, type: Pdfrb::Model::Type::Page)
+          kids = pages_root(target).value[:Kids]
+          kids << Pdfrb::Model::Reference.new(new_page.oid, new_page.gen)
+          pages_root(target).value[:Count] = (pages_root(target).value[:Count] || 0) + 1
+        end
+      end
+      private_class_method :import_all_pages
+
+      def pages_root(target)
+        ref = target.catalog.value[:Pages] || begin
+          root = target.add({ Type: :Pages, Kids: [], Count: 0 },
+                            type: Pdfrb::Model::Type::PageTreeNode)
+          target.catalog.value[:Pages] = Pdfrb::Model::Reference.new(root.oid, 0)
+          target.catalog.value[:Pages]
+        end
+        target.object(ref)
+      end
+      private_class_method :pages_root
+
+      def parent_ref(target)
+        root = pages_root(target)
+        Pdfrb::Model::Reference.new(root.oid, root.gen)
+      end
+      private_class_method :parent_ref
+    end
+  end
+end

data/lib/pdfrb/task/optimize.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+module Pdfrb
+  module Task
+    # Placeholder for the optimise task. Real implementation needs
+    # TODO 31 (object-stream packing) + dedup hash + xref-stream
+    # conversion. For now: no-op pass-through so the CLI can wire
+    # without raising.
+    module Optimize
+      module_function
+
+      def call(document, **_opts)
+        # TODO: dedup, pack into ObjStm, switch to xref stream.
+        document
+      end
+    end
+  end
+end

data/lib/pdfrb/task.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+module Pdfrb
+  # Bulk operations on a Document. Each task is a module under Task::*
+  # with a `.call(document, **opts)` class method. Adding a new task
+  # = adding one module file (open/closed via the CLI wiring).
+  module Task
+    autoload :ExtractText, "pdfrb/task/extract_text"
+    autoload :ExtractImages, "pdfrb/task/extract_images"
+    autoload :Merge, "pdfrb/task/merge"
+    autoload :Optimize, "pdfrb/task/optimize"
+  end
+end

data/lib/pdfrb/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module Pdfrb
+  VERSION = "0.1.0"
+end

data/lib/pdfrb/writer.rb

@@ -0,0 +1,144 @@
+# frozen_string_literal: true
+
+module Pdfrb
+  # Writes a Document to an IO as a complete PDF file: header,
+  # indirect objects, xref section, trailer.
+  #
+  # Two modes:
+  #   * +write+ (default) — full rewrite; assign oids to new objects,
+  #     emit every object the document references.
+  #   * +write_incremental+ — append a new revision to an existing
+  #     byte stream. Only valid when the document was opened from an
+  #     IO; the original bytes are preserved verbatim, and a new
+  #     xref + trailer are appended with /Prev pointing back.
+  class Writer
+    DEFAULT_VERSION = "1.4"
+
+    attr_reader :document, :io, :serializer
+
+    def initialize(document, io)
+      @document = document
+      @io = io
+      @serializer = Serializer.new(
+        compress_streams: document.config["writer.compress_streams"],
+        compress_min_size: document.config["writer.compress_min_size"]
+      )
+      @xref_offsets = {} # oid -> byte offset
+    end
+
+    def self.write(document, io)
+      new(document, io).write
+    end
+
+    def self.write_incremental(document, io)
+      new(document, io).write_incremental
+    end
+
+    def write
+      @io.binmode
+      @io.truncate(0) if @io.respond_to?(:truncate)
+      write_header
+      dispatch_before_write
+      each_indirect_object do |obj|
+        @xref_offsets[obj.oid] = @io.pos
+        @io << @serializer.serialize_indirect(obj)
+      end
+      xref_pos = write_xref
+      write_trailer(xref_pos)
+      @io.flush
+      self
+    end
+
+    # Append a new revision to the original byte stream. Requires the
+    # document to have been opened from an IO (so we can copy its
+    # original bytes verbatim) and the IO the writer is given is the
+    # target output.
+    def write_incremental
+      raise Pdfrb::Error, "incremental write requires a source IO" unless document.io
+
+      @io.binmode
+      # Copy the original bytes verbatim.
+      document.io.seek(0, IO::SEEK_SET)
+      @io << document.io.read
+      dispatch_before_write
+      # Emit only the modified objects.
+      document.instance_variable_get(:@objects).each_value do |obj|
+        next unless obj.indirect?
+
+        @xref_offsets[obj.oid] = @io.pos
+        @io << @serializer.serialize_indirect(obj)
+      end
+      xref_pos = write_xref(prev: previous_xref_offset)
+      write_trailer(xref_pos, prev: previous_xref_offset)
+      @io.flush
+      self
+    end
+
+    private
+
+    def write_header
+      @io << "%PDF-#{version}\n"
+      @io << "%\xE2\xE3\xCF\xD3\n"
+    end
+
+    def write_xref(prev: nil)
+      pos = @io.pos
+      oids = @xref_offsets.keys.sort
+      max_oid = oids.max || 0
+      @io << "xref\n"
+      @io << "0 #{max_oid + 1}\n"
+      @io << sprintf("0000000000 65535 f \r\n")
+      (1..max_oid).each do |oid|
+        offset = @xref_offsets[oid]
+        if offset
+          @io << sprintf("%010d %05d n \r\n", offset, 0)
+        else
+          @io << sprintf("0000000000 00000 f \r\n")
+        end
+      end
+      pos
+    end
+
+    def write_trailer(xref_pos, prev: nil)
+      root = document.catalog
+      root_ref = root && root.respond_to?(:indirect?) && root.indirect? ?
+                   Pdfrb::Model::Reference.new(root.oid, root.gen) : nil
+      size = [(@xref_offsets.keys.max || 0) + 1, document.instance_variable_get(:@next_oid) || 1].max
+
+      existing = document.trailer || {}
+      trailer_hash = {}
+      existing.each do |k, v|
+        next if %i[Size Root Prev XRefStm].include?(k)
+
+        trailer_hash[k] = v
+      end
+      trailer_hash[:Size] = size
+      trailer_hash[:Root] = root_ref if root_ref
+      trailer_hash[:Prev] = prev if prev
+
+      @io << "trailer\n"
+      @io << @serializer.serialize(trailer_hash)
+      @io << "\nstartxref\n#{xref_pos}\n%%EOF\n"
+    end
+
+    def previous_xref_offset
+      @previous_xref_offset ||= begin
+        # Re-scan the source's startxref. Same logic as TrailerReader
+        # but the source IO is the document's.
+        Pdfrb::Source::TrailerReader.startxref_offset(document.io)
+      end
+    end
+
+    def dispatch_before_write
+      document.dispatch_message(:before_write)
+    end
+
+    def version
+      document.version || DEFAULT_VERSION
+    end
+
+    def each_indirect_object
+      document.each_indirect_object { |obj| yield obj }
+    end
+  end
+end

data/lib/pdfrb/xref_section.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+module Pdfrb
+  # In-memory cross-reference section (s7.5.4). Maps oid -> Entry
+  # where each Entry is one of:
+  #   * :in_use   — at +offset+ in the file, generation +gen+.
+  #   * :free     — head of free list (with next-free oid +gen+).
+  #   * :compressed — inside ObjStm +obj_stm_oid+ at index +index+.
+  class XrefSection
+    Entry = Struct.new(:type, :offset, :gen, :obj_stm_oid, :index, keyword_init: true) do
+      def in_use?; type == :in_use; end
+      def free?; type == :free; end
+      def compressed?; type == :compressed; end
+    end
+
+    attr_reader :entries, :size
+
+    def initialize(size: 0)
+      @entries = {}
+      @size = size
+    end
+
+    def add_in_use(oid, gen, offset)
+      @entries[oid] = Entry.new(type: :in_use, offset: offset, gen: gen)
+      @size = [@size, oid + 1].max
+    end
+
+    def add_free(oid, gen, next_free)
+      @entries[oid] = Entry.new(type: :free, offset: next_free, gen: gen)
+    end
+
+    def add_compressed(oid, obj_stm_oid, index)
+      @entries[oid] = Entry.new(type: :compressed, obj_stm_oid: obj_stm_oid, index: index)
+      @size = [@size, oid + 1].max
+    end
+
+    def [](oid)
+      @entries[oid]
+    end
+
+    def each_entry(&block)
+      @entries.each(&block)
+    end
+
+    def merge!(other)
+      other.entries.each { |oid, e| @entries[oid] = e unless @entries.key?(oid) }
+      @size = [@size, other.size].max
+      self
+    end
+  end
+end

data/lib/pdfrb.rb

@@ -0,0 +1,97 @@
+# frozen_string_literal: true
+
+# Pdfrb — pure-Ruby PDF library.
+#
+# Two-direction contract:
+#   * "PDF file <=> Model" — parse PDF bytes into a typed Model, serialize
+#     a Model back to PDF bytes. Round-trip is the correctness backbone.
+#   * "API Builder Input => Model" — Document / Canvas build a typed
+#     Model from user intent, which is then serialized.
+#
+# Pdfrb is deliberately scoped to the PDF byte format. Layout concerns
+# (page templates, flows, line breaking, tables, etc.) live in the
+# sibling `arroolio` gem.
+#
+# The PDF object model is sourced directly from the vendored Arlington
+# PDF Model TSVs (data/pdfrb/arlington/*.tsv). Field metadata, version
+# predicates, and validators stay aligned with ISO 32000-2:2020 by data,
+# not by hand-coded mimicry.
+#
+# Layering (mirrors the sibling postscript gem):
+#   Source      — bytes → tokens → COS graph (read direction)
+#   Model       — typed domain model (COS values + Type::* semantics)
+#   Arlington   — TSV loader + predicate evaluator (drives Model field defs)
+#   Filter      — stream filter pipeline (Flate, ASCII-85, LZW, ...)
+#   Content     — content-stream operators + Canvas drawing API
+#   Serializer  — Model → bytes
+#   Writer      — Document → file (xref + trailer assembly)
+#   Document    — top-level facade
+module Pdfrb
+  autoload :VERSION, "pdfrb/version"
+
+  # Errors — every failure mode in the gem subclasses Pdfrb::Error.
+  autoload :Error, "pdfrb/error"
+  autoload :ParseError, "pdfrb/error"
+  autoload :LexError, "pdfrb/error"
+  autoload :SyntaxError, "pdfrb/error"
+  autoload :MalformedPdfError, "pdfrb/error"
+  autoload :SerializeError, "pdfrb/error"
+  autoload :FilterError, "pdfrb/error"
+  autoload :EncryptionError, "pdfrb/error"
+  autoload :UnsupportedVersionError, "pdfrb/error"
+  autoload :ValidationError, "pdfrb/error"
+  autoload :ObjectReferenceError, "pdfrb/error"
+
+  # Cross-cutting helpers.
+  autoload :Configuration, "pdfrb/configuration"
+  autoload :DataDir, "pdfrb/data_dir"
+  autoload :PdfConstants, "pdfrb/pdf_constants"
+
+  # Sub-namespaces. Each lives in its own file which holds the autoloads
+  # for that namespace's children (the "immediate parent" rule).
+  autoload :Source, "pdfrb/source"
+  autoload :Model, "pdfrb/model"
+  autoload :Arlington, "pdfrb/arlington"
+  autoload :Filter, "pdfrb/filter"
+  autoload :Content, "pdfrb/content"
+  autoload :Encryption, "pdfrb/encryption"
+  autoload :DigitalSignature, "pdfrb/digital_signature"
+
+  # Eager-load Type::* subclasses so they self-register in the
+  # /Type-symbol type_map before any wrap call happens. Without
+  # this, the first read of a Catalog wouldn't upgrade to
+  # Type::Catalog (because the constant hadn't been autoloaded yet
+  # and therefore hadn't called register_type).
+  require "pdfrb/model/type/file_trailer"
+  require "pdfrb/model/type/catalog"
+  require "pdfrb/model/type/info"
+  require "pdfrb/model/type/page_tree_node"
+  require "pdfrb/model/type/page"
+  require "pdfrb/model/type/resources"
+  require "pdfrb/model/type/metadata"
+  require "pdfrb/model/type/object_stream"
+  require "pdfrb/model/type/xref_stream"
+  require "pdfrb/model/type/graphics_state_parameter"
+  require "pdfrb/model/type/optional_content_group"
+  require "pdfrb/model/type/struct_tree_root"
+  require "pdfrb/model/type/font_type1"
+  require "pdfrb/model/type/annotation"
+  require "pdfrb/model/type/action"
+  autoload :Font, "pdfrb/font"
+  autoload :FontLoader, "pdfrb/font_loader"
+  autoload :ImageLoader, "pdfrb/image_loader"
+  autoload :Task, "pdfrb/task"
+
+  # Top-level orchestrators.
+  autoload :Serializer, "pdfrb/serializer"
+  autoload :Writer, "pdfrb/writer"
+  autoload :Importer, "pdfrb/importer"
+  autoload :Revision, "pdfrb/revision"
+  autoload :Revisions, "pdfrb/revisions"
+  autoload :XrefSection, "pdfrb/xref_section"
+  autoload :Document, "pdfrb/document"
+
+  # CLI (only loaded when invoked via the +pdfrb+ executable; avoids
+  # pulling in thor for library-only users).
+  autoload :CLI, "pdfrb/cli"
+end