hexapdf 0.26.2 → 0.27.0

data/lib/hexapdf/document/signatures.rb

@@ -35,7 +35,9 @@
 #++
 
 require 'openssl'
+require 'net/http'
 require 'hexapdf/error'
+require 'stringio'
 
 module HexaPDF
   class Document
@@ -43,19 +45,59 @@ module HexaPDF
     # This class provides methods for interacting with digital signatures of a PDF file.
     class Signatures
 
-      # This is the default signing handler which provides the ability to sign a document with a
-      # provided certificate using the adb.pkcs7.detached algorithm.
+      # This is the default signing handler which provides the ability to sign a document with the
+      # adbe.pkcs7.detached or ETSI.CAdES.detached algorithms. It is registered under the :default
+      # name.
+      #
+      # == Usage
+      #
+      # The signing handler is used by default by all methods that need a signing handler. Therefore
+      # it is usually only necessary to provide the actual attribute values.
+      #
+      # This handler provides two ways to create the PKCS#7 signed-data structure required by
+      # Signatures#add:
+      #
+      # * By providing the signing certificate together with the signing key and the certificate
+      #   chain. This way HexaPDF itself does the signing. It is the preferred way if all the needed
+      #   information is available.
+      #
+      #   Assign the respective data to the #certificate, #key and #certificate_chain attributes.
+      #
+      # * By using an external signing mechanism. Here the actual signing happens "outside" of
+      #   HexaPDF, for example, in custom code or even asynchronously. This is needed in case the
+      #   signing certificate plus key are not directly available but only an interface to them
+      #   (e.g. when dealing with a HSM).
+      #
+      #   Assign a callable object to #external_signing. If the signing process needs to be
+      #   asynchronous, make sure to set the #signature_size appropriately, return an empty string
+      #   during signing and later use Signatures.embed_signature to embed the actual signature.
       #
       # Additional functionality:
       #
       # * Optionally setting the reason, location and contact information.
       # * Making the signature a certification signature by applying the DocMDP transform method.
       #
+      # Example:
+      #
+      #   # Signing using certificate + key
+      #   document.sign("output.pdf", certificate: my_cert, key: my_key,
+      #                 certificate_chain: my_chain)
+      #
+      #   # Signing using an external mechanism:
+      #   signing_proc = lambda do |io, byte_range|
+      #     io.pos = byte_range[0]
+      #     data = io.read(byte_range[1])
+      #     io.pos = byte_range[2]
+      #     data << io.read(byte_range[3])
+      #     signing_service.pkcs7_sign(data)
+      #   end
+      #   document.sign("output.pdf", signature_size: 10_000, external_signing: signing_proc)
+      #
       # == Implementing a Signing Handler
       #
       # This class also serves as an example on how to create a custom handler: The public methods
-      # #filter_name, #sub_filter_name, #signature_size, #finalize_objects and #sign are used by the
-      # digital signature algorithm.
+      # #signature_size, #finalize_objects and #sign are used by the digital signature algorithm.
+      # See their descriptions for details.
       #
       # Once a custom signing handler has been created, it can be registered under the
       # 'signature.signing_handler' configuration option for easy use. It has to take keyword
@@ -72,6 +114,13 @@ module HexaPDF
         # certificates up to the root certificate.
         attr_accessor :certificate_chain
 
+        # A callable object fulfilling the same role as the #sign method that is used instead of the
+        # default mechanism for signing.
+        #
+        # If this attribute is set, the attributes #certificate, #key and #certificate_chain are not
+        # used.
+        attr_accessor :external_signing
+
         # The reason for signing. If used, will be set on the signature object.
         attr_accessor :reason
 
@@ -81,6 +130,21 @@ module HexaPDF
         # The contact information. If used, will be set on the signature object.
         attr_accessor :contact_info
 
+        # The size of the serialized signature that should be reserved.
+        #
+        # If this attribute has not been set, an empty string will be signed using #sign to
+        # determine the signature size.
+        #
+        # The size needs to be at least as big as the final signature, otherwise signing results in
+        # an error.
+        attr_writer :signature_size
+
+        # The type of signature to be written (i.e. the value of the /SubFilter key).
+        #
+        # The value can either be :adobe (the default; uses a detached PKCS7 signature) or :etsi
+        # (uses an ETSI CAdES compatible signature).
+        attr_accessor :signature_type
+
         # The DocMDP permissions that should be set on the document.
         #
         # See #doc_mdp_permissions=
@@ -88,19 +152,10 @@ module HexaPDF
 
         # Creates a new DefaultHandler with the given attributes.
         def initialize(**arguments)
+          @signature_size = nil
           arguments.each {|name, value| send("#{name}=", value) }
         end
 
-        # Returns the name to be set on the /Filter key when using this signing handler.
-        def filter_name
-          :'Adobe.PPKLite'
-        end
-
-        # Returns the name to be set on the /SubFilter key when using this signing handler.
-        def sub_filter_name
-          :'adbe.pkcs7.detached'
-        end
-
         # Sets the DocMDP permissions that should be applied to the document.
         #
         # Valid values for +permissions+ are:
@@ -128,13 +183,17 @@ module HexaPDF
           end
         end
 
-        # Returns the size of the signature that would be created.
+        # Returns the size of the serialized signature that should be reserved.
+        #
+        # If a custom size is set using #signature_size=, it used. Otherwise the size is determined
+        # by using #sign to sign an empty string.
         def signature_size
-          sign("").size
+          @signature_size || sign(StringIO.new, [0, 0, 0, 0]).size
         end
 
         # Finalizes the signature field as well as the signature dictionary before writing.
         def finalize_objects(_signature_field, signature)
+          signature[:SubFilter] = :'ETSI.CAdES.detached' if signature_type == :etsi
           signature[:Reason] = reason if reason
           signature[:Location] = location if location
           signature[:ContactInfo] = contact_info if contact_info
@@ -153,14 +212,173 @@ module HexaPDF
         end
 
         # Returns the DER serialized OpenSSL::PKCS7 structure containing the signature for the given
-        # data.
-        def sign(data)
-          OpenSSL::PKCS7.sign(@certificate, @key, data, @certificate_chain,
-                              OpenSSL::PKCS7::DETACHED | OpenSSL::PKCS7::BINARY).to_der
+        # IO byte ranges.
+        #
+        # The +byte_range+ argument is an array containing four numbers [offset1, length1, offset2,
+        # length2]. The offset numbers are byte positions in the +io+ argument and the to-be-signed
+        # data can be determined by reading length bytes at the offsets.
+        def sign(io, byte_range)
+          if external_signing
+            external_signing.call(io, byte_range)
+          else
+            io.pos = byte_range[0]
+            data = io.read(byte_range[1])
+            io.pos = byte_range[2]
+            data << io.read(byte_range[3])
+            OpenSSL::PKCS7.sign(@certificate, @key, data, @certificate_chain,
+                                OpenSSL::PKCS7::DETACHED | OpenSSL::PKCS7::BINARY).to_der
+          end
+        end
+
+      end
+
+      # This is a signing handler for adding a timestamp signature (a PDF2.0 feature) to a PDF
+      # document. It is registered under the :timestamp name.
+      #
+      # The timestamp is provided by a timestamp authority and establishes the document contents at
+      # the time indicated in the timestamp. Timestamping a PDF document is usually done in context
+      # of long term validation but can also be done standalone.
+      #
+      # == Usage
+      #
+      # It is necessary to provide at least the URL of the timestamp authority server (TSA) via
+      # #tsa_url, everything else is optional and uses default values. The TSA server must not use
+      # authentication to be usable.
+      #
+      # Example:
+      #
+      #   document.sign("output.pdf", handler: :timestamp, tsa_url: 'https://freetsa.org/tsr')
+      class TimestampHandler
+
+        # The URL of the timestamp authority server.
+        #
+        # This value is required.
+        attr_accessor :tsa_url
+
+        # The hash algorithm to use for timestamping. Defaults to SHA512.
+        attr_accessor :tsa_hash_algorithm
+
+        # The policy OID to use for timestamping. Defaults to +nil+.
+        attr_accessor :tsa_policy_id
+
+        # The size of the serialized signature that should be reserved.
+        #
+        # If this attribute has not been set, an empty string will be signed using #sign to
+        # determine the signature size which will contact the TSA server
+        #
+        # The size needs to be at least as big as the final signature, otherwise signing results in
+        # an error.
+        attr_writer :signature_size
+
+        # The reason for timestamping. If used, will be set on the signature object.
+        attr_accessor :reason
+
+        # The timestamping location. If used, will be set on the signature object.
+        attr_accessor :location
+
+        # The contact information. If used, will be set on the signature object.
+        attr_accessor :contact_info
+
+        # Creates a new TimestampHandler with the given attributes.
+        def initialize(**arguments)
+          @signature_size = nil
+          arguments.each {|name, value| send("#{name}=", value) }
+        end
+
+        # Returns the size of the serialized signature that should be reserved.
+        def signature_size
+          @signature_size || (sign(StringIO.new, [0, 0, 0, 0]).size * 1.5).to_i
+        end
+
+        # Finalizes the signature field as well as the signature dictionary before writing.
+        def finalize_objects(_signature_field, signature)
+          signature.document.version = '2.0'
+          signature[:Type] = :DocTimeStamp
+          signature[:SubFilter] = :'ETSI.RFC3161'
+          signature[:Reason] = reason if reason
+          signature[:Location] = location if location
+          signature[:ContactInfo] = contact_info if contact_info
+        end
+
+        # Returns the DER serialized OpenSSL::PKCS7 structure containing the timestamp token for the
+        # given IO byte ranges.
+        def sign(io, byte_range)
+          hash_algorithm = tsa_hash_algorithm || 'SHA512'
+          digest = OpenSSL::Digest.new(hash_algorithm)
+          io.pos = byte_range[0]
+          digest << io.read(byte_range[1])
+          io.pos = byte_range[2]
+          digest << io.read(byte_range[3])
+
+          req = OpenSSL::Timestamp::Request.new
+          req.algorithm = hash_algorithm
+          req.message_imprint = digest.digest
+          req.policy_id = tsa_policy_id if tsa_policy_id
+
+          http_response = Net::HTTP.post(URI(tsa_url), req.to_der,
+                                         'content-type' => 'application/timestamp-query')
+          if http_response.kind_of?(Net::HTTPOK)
+            response = OpenSSL::Timestamp::Response.new(http_response.body)
+            if response.status == 0
+              response.token.to_der
+            else
+              raise HexaPDF::Error, "Timestamp token could not be created: #{response.failure_info}"
+            end
+          else
+            raise HexaPDF::Error, "Invalid TSA server response: #{http_response.body}"
+          end
         end
 
       end
 
+      # Embeds the given +signature+ into the /Contents value of the newest signature dictionary of
+      # the PDF document given by the +io+ argument.
+      #
+      # This functionality can be used together with the support for external signing (see
+      # DefaultHandler and DefaultHandler#external_signing) to implement asynchronous signing.
+      #
+      # Note: This will, most probably, only work on documents prepared for external signing by
+      # HexaPDF and not by other libraries.
+      def self.embed_signature(io, signature)
+        doc = HexaPDF::Document.new(io: io)
+        signature_dict = doc.signatures.find {|sig| doc.revisions.current.object(sig) == sig }
+        signature_dict_offset, signature_dict_length = locate_signature_dict(
+          doc.revisions.current.xref_section,
+          doc.revisions.parser.startxref_offset,
+          signature_dict.oid
+        )
+        io.pos = signature_dict_offset
+        signature_data = io.read(signature_dict_length)
+        replace_signature_contents(signature_data, signature)
+        io.pos = signature_dict_offset
+        io.write(signature_data)
+      end
+
+      # Uses the information in the given cross-reference section as well as the byte offset of the
+      # cross-reference section to calculate the offset and length of the signature dictionary with
+      # the given object id.
+      def self.locate_signature_dict(xref_section, start_xref_position, signature_oid)
+        data = xref_section.map {|oid, _gen, entry| [entry.pos, oid] if entry.in_use? }.compact.sort <<
+          [start_xref_position, nil]
+        index = data.index {|_pos, oid| oid == signature_oid }
+        [data[index][0], data[index + 1][0] - data[index][0]]
+      end
+
+      # Replaces the value of the /Contents key in the serialized +signature_data+ with the value of
+      # +contents+.
+      def self.replace_signature_contents(signature_data, contents)
+        signature_data.sub!(/Contents(?:\(.*?\)|<.*?>)/) do |match|
+          length = match.size
+          result = "Contents<#{contents.unpack1('H*')}"
+          if length < result.size
+            raise HexaPDF::Error, "The reserved space for the signature was too small " \
+              "(#{(length - 10) / 2} vs #{(result.size - 10) / 2}) - use the handlers " \
+              "#signature_size method to increase the reserved space"
+          end
+          "#{result.ljust(length - 1, '0')}>"
+        end
+      end
+
       include Enumerable
 
       # Creates a new Signatures object for the given PDF document.
@@ -168,15 +386,15 @@ module HexaPDF
         @document = document
       end
 
-      # Creates a signing handler with the given options and returns it.
+      # Creates a signing handler with the given attributes and returns it.
       #
       # A signing handler name is mapped to a class via the 'signature.signing_handler'
       # configuration option. The default signing handler is DefaultHandler.
-      def handler(name: :default, **options)
+      def handler(name: :default, **attributes)
         handler = @document.config.constantize('signature.signing_handler', name) do
           raise HexaPDF::Error, "No signing handler named '#{name}' is available"
         end
-        handler.new(**options)
+        handler.new(**attributes)
       end
 
       # Adds a signature to the document and returns the corresponding signature object.
@@ -209,8 +427,15 @@ module HexaPDF
       #
       # +write_options+::
       #     The key-value pairs of this hash will be passed on to the HexaPDF::Document#write
-      #     command. Note that +incremental+ will be automatically set if signing an already
-      #     existing file.
+      #     method. Note that +incremental+ will be automatically set to ensure proper behaviour.
+      #
+      # The used signature object will have the following default values set:
+      #
+      # /Filter::    /Adobe.PPKLite
+      # /SubFilter:: /adbe.pkcs7.detached
+      # /M::         The current time.
+      #
+      # These values can be overridden in the #finalize_objects method of the signature handler.
       def add(file_or_io, handler, signature: nil, write_options: {})
         if signature && signature.type != :Sig
           signature_field = signature
@@ -232,11 +457,12 @@ module HexaPDF
         end
 
         # Prepare signature object
-        signature[:Filter] = handler.filter_name
-        signature[:SubFilter] = handler.sub_filter_name
+        signature[:Filter] = :'Adobe.PPKLite'
+        signature[:SubFilter] = :'adbe.pkcs7.detached'
+        signature[:M] = Time.now
+        handler.finalize_objects(signature_field, signature)
         signature[:ByteRange] = [0, 1_000_000_000_000, 1_000_000_000_000, 1_000_000_000_000]
         signature[:Contents] = '00' * handler.signature_size # twice the size due to hex encoding
-        signature[:M] = Time.now
 
         io = if file_or_io.kind_of?(String)
                File.open(file_or_io, 'wb+')
@@ -246,23 +472,19 @@ module HexaPDF
 
         # Save the current state so that we can determine the correct /ByteRange value and set the
         # values
-        handler.finalize_objects(signature_field, signature)
-        start_xref_position, section = @document.write(io, incremental: true, **write_options)
-        data = section.map {|oid, _gen, entry| [entry.pos, oid] if entry.in_use? }.compact.sort <<
-          [start_xref_position, nil]
-        index = data.index {|_pos, oid| oid == signature.oid }
-        signature_offset = data[index][0]
-        signature_length = data[index + 1][0] - data[index][0]
+        start_xref, section = @document.write(io, incremental: true, **write_options)
+        signature_offset, signature_length = self.class.locate_signature_dict(section, start_xref,
+                                                                              signature.oid)
         io.pos = signature_offset
         signature_data = io.read(signature_length)
 
-        io.rewind
-        file_data = io.read
+        io.seek(0, IO::SEEK_END)
+        file_size = io.pos
 
         # Calculate the offsets for the /ByteRange
         contents_offset = signature_offset + signature_data.index('Contents(') + 8
         offset2 = contents_offset + signature[:Contents].size + 2 # +2 because of the needed < and >
-        length2 = file_data.size - offset2
+        length2 = file_size - offset2
         signature[:ByteRange] = [0, contents_offset, offset2, length2]
 
         # Set the correct /ByteRange value
@@ -274,17 +496,12 @@ module HexaPDF
 
         # Now everything besides the /Contents value is correct, so we can read the contents for
         # signing
-        file_data[signature_offset, signature_length] = signature_data
-        signed_contents = file_data[0, contents_offset] << file_data[offset2, length2]
-        signature[:Contents] = handler.sign(signed_contents)
-
-        # Set the correct /Contents value as hexstring
-        signature_data.sub!(/Contents\(0+\)/) do |match|
-          length = match.size
-          result = "Contents<#{signature[:Contents].unpack1('H*')}"
-          "#{result.ljust(length - 1, '0')}>"
-        end
+        io.pos = signature_offset
+        io.write(signature_data)
+        signature[:Contents] = handler.sign(io, signature[:ByteRange].value)
 
+        # And now replace the /Contents value
+        self.class.replace_signature_contents(signature_data, signature[:Contents])
         io.pos = signature_offset
         io.write(signature_data)
 

data/lib/hexapdf/importer.rb

@@ -94,6 +94,9 @@ module HexaPDF
       if object.kind_of?(HexaPDF::Object) && object.document? && @source != object.document
         raise HexaPDF::Error, "Import error: Incorrect document object for importer"
       elsif mapped_object && !mapped_object.null?
+        if object.class != mapped_object.class
+          mapped_object = @destination.wrap(mapped_object, type: object.class)
+        end
         mapped_object
       else
         duplicate(object)

data/lib/hexapdf/parser.rb

@@ -137,6 +137,7 @@ module HexaPDF
             @tokenizer.pos -= 6
           else
             maybe_raise("Invalid value after '#{oid} #{gen} obj', treating as null", pos: @tokenizer.pos)
+            return [nil, oid, gen, nil]
           end
         end
       end

data/lib/hexapdf/revisions.rb

@@ -95,13 +95,15 @@ module HexaPDF
 
             if merge_revision == offset
               xref_section.merge!(revisions.first.xref_section)
+              offset = trailer[:Prev] # Get possible next offset before overwriting trailer
               trailer = revisions.first.trailer
               revisions.shift
+            else
+              offset = trailer[:Prev]
             end
 
             revisions.unshift(Revision.new(document.wrap(trailer, type: :XXTrailer),
                                            xref_section: xref_section, loader: object_loader))
-            offset = trailer[:Prev]
           end
         rescue HexaPDF::MalformedPDFError
           raise unless (reconstructed_revision = parser.reconstructed_revision)

data/lib/hexapdf/tokenizer.rb

@@ -274,7 +274,7 @@ module HexaPDF
       TOKEN_CACHE[str]
     end
 
-    REFERENCE_RE = /[#{WHITESPACE}]+([+-]?\d+)[#{WHITESPACE}]+R#{WHITESPACE_OR_DELIMITER_RE}/ # :nodoc:
+    REFERENCE_RE = /[#{WHITESPACE}]+([+]?\d+)[#{WHITESPACE}]+R#{WHITESPACE_OR_DELIMITER_RE}/ # :nodoc:
 
     # Parses the number (integer or real) at the current position.
     #
@@ -285,7 +285,7 @@ module HexaPDF
         tmp = val.to_i
         # Handle object references, see PDF1.7 s7.3.10
         prepare_string_scanner(10)
-        tmp = Reference.new(tmp, @ss[1].to_i) if @ss.scan(REFERENCE_RE)
+        tmp = Reference.new(tmp, @ss[1].to_i) if tmp > 0 && @ss.scan(REFERENCE_RE)
         tmp
       elsif val.match?(/\A[+-]?(?:\d+\.\d*|\.\d+)\z/)
         val << '0' if val.getbyte(-1) == 46 # dot '.'

data/lib/hexapdf/type/acro_form/form.rb

@@ -395,7 +395,8 @@ module HexaPDF
 
           not_flattened = fields.map {|field| field.each_widget.to_a }.flatten
           document.pages.each {|page| not_flattened = page.flatten_annotations(not_flattened) }
-          fields -= not_flattened.map(&:form_field)
+          not_flattened.map!(&:form_field)
+          fields -= not_flattened
 
           fields.each do |field|
             (field[:Parent]&.[](:Kids) || self[:Fields]).delete(field)
@@ -448,6 +449,32 @@ module HexaPDF
         def perform_validation # :nodoc:
           super
 
+          validate_array = lambda do |parent, container|
+            container.reject! do |field|
+              if !field.kind_of?(HexaPDF::Object) || !field.kind_of?(HexaPDF::Dictionary) || field.null?
+                yield("Invalid object in AcroForm field hierarchy", true)
+                next true
+              end
+              next false unless field.key?(:T) # Skip widgets
+
+              field = document.wrap(field, type: :XXAcroFormField,
+                                    subtype: Field.inherited_value(field, :FT))
+              reject = false
+              if field[:Parent] != parent
+                yield("Parent entry of field (#{field.oid},#{field.gen}) invalid", true)
+                if field[:Parent].nil?
+                  root_fields << field
+                  reject = true
+                else
+                  field[:Parent] = parent
+                end
+              end
+              validate_array.call(field, field[:Kids]) if field.key?(:Kids)
+              reject
+            end
+          end
+          validate_array.call(nil, root_fields)
+
           if (da = self[:DA])
             unless self[:DR]
               yield("When the field /DA is present, the field /DR must also be present")

data/lib/hexapdf/type/catalog.rb

@@ -65,7 +65,7 @@ module HexaPDF
                          :TwoPageLeft, :TwoPageRight]
       define_field :PageMode,          type: Symbol,     default: :UseNone,
         allowed_values: [:UseNone, :UseOutlines, :UseThumbs, :FullScreen, :UseOC, :UseAttachments]
-      define_field :Outlines,          type: Dictionary, indirect: true
+      define_field :Outlines,          type: :Outlines,  indirect: true
       define_field :Threads,           type: PDFArray,   version: '1.1'
       define_field :OpenAction,        type: [Dictionary, PDFArray], version: '1.1'
       define_field :AA,                type: Dictionary, version: '1.4'

data/lib/hexapdf/type/outline.rb

@@ -50,6 +50,8 @@ module HexaPDF
     # The outline dictionary is linked via the /Outlines entry from the Type::Catalog and can
     # directly be accessed via HexaPDF::Document#outline.
     #
+    # == Examples
+    #
     # Here is an example for creating an outline:
     #
     #   doc = HexaPDF::Document.new
@@ -62,6 +64,22 @@ module HexaPDF
     #     end
     #   end
     #
+    # Here is one for copying the complete outline from one PDF to another:
+    #
+    #   doc = HexaPDF::Document.open(ARGV[0])
+    #   target = HexaPDF::Document.new
+    #   stack = [target.outline]
+    #   doc.outline.each_item do |item, level|
+    #     if stack.size < level
+    #       stack << stack.last[:Last]
+    #     elsif stack.size > level
+    #       (stack.size - level).times { stack.pop }
+    #     end
+    #     stack.last.add_item(target.import(item))
+    #   end
+    #   # Copying all the pages so that the references work.
+    #   doc.pages.each {|page| target.pages << target.import(page) }
+    #
     # See: PDF1.7 s12.3.3
     class Outline < Dictionary