pdf-reader-extract-images 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: ae5af6eccd6fb11766f5a3ede1ccbc689ad2ce72c570f481a9a6635c3622df9d
+  data.tar.gz: 3d319014cc564e6ce848853ec81cff50f43039d303d194e6d60dc036e508dce5
+SHA512:
+  metadata.gz: e5db5d318b64a06e3c2270e3efd23aa44aedea20e68cf32cf6a2c6bbac4861d9ab0e4d9ff85dba998e0f0f75c3810ac8e7cf7c3643bd9754ad925b7170d5a1f4
+  data.tar.gz: 550f9a3778412f308043d979365d712170533cb02c65afbdbce5ec33b0f5f6ebd4685c2f4830ab7a1b2b00cff71de5e4a16410d3ebb4458fe662a91c40aaaf72

data/.gitignore

@@ -0,0 +1,8 @@
+/.bundle/
+/.yardoc
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/

data/.tool-versions

@@ -0,0 +1 @@
+ruby 3.0.1

data/Gemfile

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+source "https://rubygems.org"
+
+# Specify your gem's dependencies in pdf-reader-extract-images.gemspec
+gemspec
+
+gem "rake", "~> 13.0"

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2021 Stefan Wienert
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,60 @@
+# Pdf::Reader::Extract::Images
+
+ExtractImages
+
+Based upon the [Example from Pdf::Reader](https://github.com/yob/pdf-reader/blob/main/examples/extract_images.rb), battle hardened in our applicant tracking system with tens of thousands of PDFs.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'pdf-reader-extract-images'
+```
+
+And then execute:
+
+    $ bundle install
+
+Or install it yourself as:
+
+    $ gem install pdf-reader-extract-images
+
+## Usage
+
+```ruby
+require 'pdf-reader-extract-images'
+
+reader = PDF::Reader.new(pdf)
+images = Pdf::Reader::ExtractImages.extract_all(reader)
+
+# pass an image limit to ignore gigantic image-only pdfs
+images = Pdf::Reader::ExtractImages.extract_all(reader, limit: 50)
+
+# [
+#  {
+#    :filename => "1-1-Im1.jpg",
+#    :width => 1772,
+#    :height => 591
+#    :blob => "....",
+#  }
+# ]
+
+# OR you can just scan a single Pdf::Reader Page
+
+reader.pages.each do |page|
+  images = Pdf::Reader::ExtractImages.extract_from_pdf_page(page)
+end
+```
+
+## Limitations
+
+There are some PDFs which have tons of images. Make sure to limit the timeout of an extraction somehow.
+
+Also some PDFs product hundreds of images. Make sure to limit further processing down the line
+
+Unfortunately, there is no public test suite. We have a private test suite that tests live pdfs which we cannot share. If you'd like to contribute problematic PDFs, feel free to open PR!
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).

data/Rakefile

@@ -0,0 +1,4 @@
+# frozen_string_literal: true
+
+require "bundler/gem_tasks"
+task default: %i[]

data/bin/console

@@ -0,0 +1,15 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require "bundler/setup"
+require "pdf/reader/extract/images"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start(__FILE__)

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/lib/pdf/reader/extract_images/extractor.rb

@@ -0,0 +1,73 @@
+module Pdf::Reader::ExtractImages
+  class Extractor
+    def initialize(limit = Float::INFINITY)
+      @images = []
+      @limit = limit
+    end
+
+    def page(page)
+      process_page(page, 0)
+      @images.compact!
+      @images
+    end
+
+    private
+
+    def complete_refs
+      @complete_refs ||= {}
+    end
+
+    def process_page(page, count)
+      xobjects = page.xobjects
+      return count if xobjects.empty?
+
+      xobjects.each do |name, stream|
+        return if @images.length > @limit
+
+        case stream.hash[:Subtype]
+        when :Image then
+          count += 1
+          number = page.respond_to?(:number) ? page.number : 1
+
+          @images << extract_image_from_stream(stream, filename: "#{number}-#{count}-#{name}")
+        when :Form then
+          if page.respond_to?(:objects)
+            count = process_page(PDF::Reader::FormXObject.new(page, stream), count)
+          end
+        end
+      end
+      count
+    end
+
+    def extract_image_from_stream(stream, filename:)
+      case stream.hash[:Filter]
+      when :CCITTFaxDecode
+        begin
+          Tiff.new(stream).save("#{filename}.tif")
+        rescue PDF::Reader::MalformedPDFError
+          nil
+        end
+      when :DCTDecode
+        Jpg.new(stream).save("#{filename}.jpg")
+      when [:FlateDecode, :DCTDecode], :FlateDecode
+        unzipped = Zlib::Inflate.inflate(stream.data)
+        if stream.hash[:ColorSpace]
+          Raw.new(stream, unzipped).save("#{filename}.tif")
+        else
+          {
+            blob: unzipped,
+            width: stream.hash[:Width],
+            height: stream.hash[:Height],
+            filename: "#{filename}.jpg"
+          }
+        end
+      else
+        begin
+          Raw.new(stream).save("#{filename}.tif")
+        rescue PDF::Reader::MalformedPDFError
+          nil
+        end
+      end
+    end
+  end
+end

data/lib/pdf/reader/extract_images/jpg.rb

@@ -0,0 +1,26 @@
+require 'image_processing'
+
+module Pdf::Reader::ExtractImages
+  class Jpg
+    attr_reader :stream
+
+    def initialize(stream)
+      @stream = stream
+    end
+
+    def save(filename)
+      w = stream.hash[:Width]
+      h = stream.hash[:Height]
+      blob = stream.data
+      if stream.hash[:ColorSpace] == :DeviceCMYK && stream.data['Adobe']
+        blob = Tempfile.open(['extract', filename]) { |tf|
+          tf.binmode
+          tf.write stream.data
+          tf.flush
+          ImageProcessing::MiniMagick.source(tf.path).negate.call.read
+        }
+      end
+      { filename: filename, blob: blob, width: w, height: h }
+    end
+  end
+end

data/lib/pdf/reader/extract_images/raw.rb

@@ -0,0 +1,118 @@
+module Pdf::Reader::ExtractImages
+  class Raw
+    attr_reader :stream
+
+    def initialize(stream, data = stream.unfiltered_data)
+      @stream = stream
+      @data = data
+    end
+
+    def save(filename)
+      case @stream.hash[:ColorSpace]
+      when :DeviceCMYK then save_cmyk(filename)
+      when :DeviceGray then save_gray(filename)
+      when :DeviceRGB  then save_rgb(filename)
+      else
+        if @stream.hash[:ColorSpace].is_a?(Array)
+          if @stream.hash[:ColorSpace].include?(:DeviceCMYK)
+            return save_cmyk(filename)
+          elsif @stream.hash[:ColorSpace].include?(:DeviceRGB)
+            return save_rgb(filename)
+          elsif @stream.hash[:ColorSpace].include?(:DeviceGray)
+            return save_gray(filename)
+          end
+        end
+        warn "unsupport color depth #{@stream.hash[:ColorSpace]} #{filename}"
+      end
+    end
+
+    private
+
+    def save_cmyk(filename)
+      h    = stream.hash[:Height]
+      w    = stream.hash[:Width]
+      bpc  = stream.hash[:BitsPerComponent]
+      len  = stream.hash[:Length]
+
+      # Synthesize a TIFF header
+      long_tag  = ->(tag, count, value) { [tag, 4, count, value].pack("ssII") }
+      short_tag = ->(tag, count, value) { [tag, 3, count, value].pack("ssII") }
+      # header = byte order, version magic, offset of directory, directory count,
+      # followed by a series of tags containing metadata.
+      tag_count = 10
+      header = [73, 73, 42, 8, tag_count].pack("ccsIs")
+      tiff = header.dup
+      tiff << short_tag.call(256, 1, w) # image width
+      tiff << short_tag.call(257, 1, h) # image height
+      tiff << long_tag.call(258, 4, (header.size + (tag_count * 12) + 4)) # bits per pixel
+      tiff << short_tag.call(259, 1, 1) # compression
+      tiff << short_tag.call(262, 1, 5) # colorspace - separation
+      tiff << long_tag.call(273, 1, (10 + (tag_count * 12) + 20)) # data offset
+      tiff << short_tag.call(277, 1, 4) # samples per pixel
+      tiff << long_tag.call(279, 1, @data.size) # data byte size
+      tiff << short_tag.call(284, 1, 1) # planer config
+      tiff << long_tag.call(332, 1, 1) # inkset - CMYK
+      tiff << [0].pack("I") # next IFD pointer
+      tiff << [bpc, bpc, bpc, bpc].pack("IIII")
+      tiff << @data
+      { filename: filename, blob: tiff, width: w, height: h }
+    end
+
+    def save_gray(filename)
+      h    = stream.hash[:Height]
+      w    = stream.hash[:Width]
+      bpc  = stream.hash[:BitsPerComponent]
+      len  = stream.hash[:Length]
+
+      # Synthesize a TIFF header
+      long_tag  = ->(tag, count, value) { [tag, 4, count, value].pack("ssII") }
+      short_tag = ->(tag, count, value) { [tag, 3, count, value].pack("ssII") }
+      # header = byte order, version magic, offset of directory, directory count,
+      # followed by a series of tags containing metadata.
+      tag_count = 9
+      header = [73, 73, 42, 8, tag_count].pack("ccsIs")
+      tiff = header.dup
+      tiff << short_tag.call(256, 1, w) # image width
+      tiff << short_tag.call(257, 1, h) # image height
+      tiff << short_tag.call(258, 1, 8) # bits per pixel
+      tiff << short_tag.call(259, 1, 1) # compression
+      tiff << short_tag.call(262, 1, 1) # colorspace - grayscale
+      tiff << long_tag.call(273, 1, (10 + (tag_count * 12) + 4)) # data offset
+      tiff << short_tag.call(277, 1, 1) # samples per pixel
+      tiff << long_tag.call(279, 1, stream.unfiltered_data.size) # data byte size
+      tiff << short_tag.call(284, 1, 1) # planer config
+      tiff << [0].pack("I") # next IFD pointer
+      tiff << stream.unfiltered_data
+      { filename: filename, blob: tiff, width: w, height: h }
+    end
+
+    def save_rgb(filename)
+      h    = stream.hash[:Height]
+      w    = stream.hash[:Width]
+      bpc  = stream.hash[:BitsPerComponent]
+      len  = stream.hash[:Length]
+
+      # Synthesize a TIFF header
+      long_tag  = ->(tag, count, value) { [tag, 4, count, value].pack("ssII") }
+      short_tag = ->(tag, count, value) { [tag, 3, count, value].pack("ssII") }
+      # header = byte order, version magic, offset of directory, directory count,
+      # followed by a series of tags containing metadata.
+      tag_count = 8
+      header = [73, 73, 42, 8, tag_count].pack("ccsIs")
+      tiff = header.dup
+      tiff << short_tag.call(256, 1, w) # image width
+      tiff << short_tag.call(257, 1, h) # image height
+      tiff << long_tag.call(258, 3, (header.size + (tag_count * 12) + 4)) # bits per pixel
+      tiff << short_tag.call(259, 1, 1) # compression
+      tiff << short_tag.call(262, 1, 2) # colorspace - RGB
+      tiff << long_tag.call(273, 1, (header.size + (tag_count * 12) + 16)) # data offset
+      tiff << short_tag.call(277, 1, 3) # samples per pixel
+      tiff << long_tag.call(279, 1, stream.unfiltered_data.size) # data byte size
+      tiff << [0].pack("I") # next IFD pointer
+      tiff << [bpc, bpc, bpc].pack("III")
+      tiff << stream.unfiltered_data
+      { filename: filename, blob: tiff }
+    end
+  end
+end
+

data/lib/pdf/reader/extract_images/tiff.rb

@@ -0,0 +1,50 @@
+module Pdf::Reader::ExtractImages
+  class Tiff
+    attr_reader :stream
+
+    def initialize(stream)
+      @stream = stream
+    end
+
+    def save(filename)
+      k = stream.hash[:DecodeParms][:K]
+      if !k.nil? && stream.hash[:DecodeParms][:K] <= 0
+        save_group_four(filename)
+      else
+        warn "#{filename}: CCITT non-group 4/2D image."
+      end
+    end
+
+    private
+
+    # Group 4, 2D
+    def save_group_four(filename)
+      k    = stream.hash[:DecodeParms][:K]
+      h    = stream.hash[:Height]
+      w    = stream.hash[:Width]
+      bpc  = stream.hash[:BitsPerComponent]
+      mask = stream.hash[:ImageMask]
+      len  = stream.hash[:Length]
+      cols = stream.hash[:DecodeParms][:Columns]
+
+      # Synthesize a TIFF header
+      long_tag  = ->(tag, value) { [tag, 4, 1, value].pack("ssII") }
+      short_tag = ->(tag, value) { [tag, 3, 1, value].pack("ssII") }
+      # header = byte order, version magic, offset of directory, directory count,
+      # followed by a series of tags containing metadata: 259 is a magic number for
+      # the compression type; 273 is the offset of the image data.
+      tiff = [73, 73, 42, 8, 5].pack("ccsIs") \
+        + short_tag.call(256, cols) \
+        + short_tag.call(257, h) \
+        + short_tag.call(259, 4) \
+        + long_tag.call(273, (10 + (5 * 12) + 4)) \
+        + long_tag.call(279, len) \
+        + [0].pack("I") \
+        + stream.data
+      { filename: filename, blob: tiff, width: w, height: h }
+    end
+  end
+end
+end
+
+

data/lib/pdf/reader/extract_images/version.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+module Pdf
+  module Reader
+    module ExtractImages
+      VERSION = "0.1.0"
+    end
+  end
+end

data/lib/pdf-reader-extract-images.rb

@@ -0,0 +1,17 @@
+require "zeitwerk"
+loader = Zeitwerk::Loader.for_gem
+loader.ignore(__FILE__)
+loader.setup
+
+module Pdf::Reader
+  module ExtractImages
+    def self.extract_from_pdf_page(page, limit: Float::INFINITY)
+      Extractor(limit).new.page(page)
+    end
+
+    def self.extract_all(pdf_reader, limit: Float::INFINITY)
+      pdf_reader.pages.flat_map { |page| Extractor.new(limit).page(page) }.compact
+    end
+  end
+end
+

data/pdf-reader-extract-images.gemspec

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+require_relative "lib/pdf/reader/extract_images/version"
+
+Gem::Specification.new do |spec|
+  spec.name          = "pdf-reader-extract-images"
+  spec.version       = Pdf::Reader::ExtractImages::VERSION
+  spec.authors       = ["Stefan Wienert"]
+  spec.email         = ["info@stefanwienert.de"]
+
+  spec.summary       = "Extract all images with format conversions based upon Pdf::Reader library"
+  spec.description   = spec.summary
+  spec.homepage      = "https://github.com/pludoni/pdf-reader-extract-images"
+  spec.license       = "MIT"
+  spec.required_ruby_version = Gem::Requirement.new(">= 2.4.0")
+
+  spec.metadata["homepage_uri"] = spec.homepage
+
+  # Specify which files should be added to the gem when it is released.
+  # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
+  spec.files = Dir.chdir(File.expand_path(__dir__)) do
+    `git ls-files -z`.split("\x0").reject { |f| f.match(%r{\A(?:test|spec|features)/}) }
+  end
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{\Aexe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency "pdf-reader", ">= 2.5.0"
+  spec.add_dependency "image_processing"
+  spec.add_dependency "zeitwerk"
+end

metadata

@@ -0,0 +1,101 @@
+--- !ruby/object:Gem::Specification
+name: pdf-reader-extract-images
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Stefan Wienert
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2021-08-26 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: pdf-reader
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.5.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.5.0
+- !ruby/object:Gem::Dependency
+  name: image_processing
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: zeitwerk
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Extract all images with format conversions based upon Pdf::Reader library
+email:
+- info@stefanwienert.de
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".tool-versions"
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/console
+- bin/setup
+- lib/pdf-reader-extract-images.rb
+- lib/pdf/reader/extract_images/extractor.rb
+- lib/pdf/reader/extract_images/jpg.rb
+- lib/pdf/reader/extract_images/raw.rb
+- lib/pdf/reader/extract_images/tiff.rb
+- lib/pdf/reader/extract_images/version.rb
+- pdf-reader-extract-images.gemspec
+homepage: https://github.com/pludoni/pdf-reader-extract-images
+licenses:
+- MIT
+metadata:
+  homepage_uri: https://github.com/pludoni/pdf-reader-extract-images
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 2.4.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.2.15
+signing_key: 
+specification_version: 4
+summary: Extract all images with format conversions based upon Pdf::Reader library
+test_files: []