plaintext 0.2.0 → 0.3.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: c04f6f06ab6a16b423c2a74ddb38685a49a23372
-  data.tar.gz: 8dc49ce59bde01f25983b982323889a512fa900f
+SHA256:
+  metadata.gz: fde4677879a9d3ce30844e9a5b0439ad2f708f2f532927f96b16d962bd4deb07
+  data.tar.gz: 900cf36c77f25e98876e9838f2e21a11244f37f059c6c272386fd8102cfd500d
 SHA512:
-  metadata.gz: 13876311c960330abc0596ccc09a5eb172feca1101fa154386182bb04c7cfe6ca5f2b8fb4a408ffa658fbe0cbc616e29c8770c2a0680ec11064e950a75eae19d
-  data.tar.gz: 50f4d40bcb4bfb257b4e280e081cea291a633203746c322c028e6dcacd53b68f0e5c50da6ede80bdeb0ae5a7c300c0ddb38694b4ee97d6ed8343ae81e5c8a69e
+  metadata.gz: c744870cc385445ae9a8ad68cfbc5e9489356f23750acdf7a0e8b54ec1ae9c674457c1f14e226aa8aee3bf542ba5fabeac524fda6ad0d89601414c0b77196e5a
+  data.tar.gz: e6c899c622b42856fd2671b0a7961a938e89389801904d70069b1b1735b8e6e374610f7f2b4d72fd08b16db1f24dcaa994e22b59f9e2a73585206f173b28a234

data/.travis.yml

@@ -1,7 +1,7 @@
 language: ruby
 rvm:
-  - 2.2.3
+  - 2.6.4
 before_install:
   - sudo apt-get -qq update
   - sudo apt-get install -y catdoc unrtf poppler-utils tesseract-ocr
-  - gem install bundler -v 1.10.6
+  - gem install bundler -v 2.0.1

data/CHANGELOG

@@ -6,9 +6,50 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [0.3.4] - 2021-04-21
+- Further relax the rubyzip version requirement to allow 2.x versions
+- Fix: Removed --quiet from unrtf arguments. In newer versions this seems to
+  suppress the header line which we used as a marker to strip unrtf comments and
+  meta data from the command output.
+
+## [0.3.3] - 2019-10-01
+- Relax rubyzip dependency. Our usage of rubyzip does not require adressing
+  CVE-2019-16892 and thus is fine with 1.2, but applications using this Gem
+  might use RubyZIP elsewhere too and thus will want to upgrade to >= 1.3.
+
+## [0.3.2] - 2019-09-02
+- Set minimum Nokogiri version to 1.10.4. See CVE-2019-5477.
+- Fix encoding issues for PDFs.
+- Bump development dependencies to bundler version 2 and
+  rake version 12
+- Update travis file to use ruby 2.6.4 and bundler 2.0.1
+
+## [0.3.1] - 2019-01-16
+
+### Added
+- The max_plaintext_bytes limit introduced in 0.3.0 is now also enforced in the
+  zipped XML handlers responsible for office document parsing.
+
+## [0.3.0] - 2019-01-09
+
+### Added
+- `:max_plaintext_bytes` option to place an upper limit on the number of bytes
+  returned. Also limits the amount of data that is actually read from plain text
+  files and external command output to limit memory usage. Set this on the
+  resolver object.
+
+### Changed
+- the unrtf handler now strips the preamble which unrtf adds to its output.
+- text from external command handlers is now converted to UTF-8 before it's returned.
+- the FileHandler `text` method signature has been changed by adding an options
+  hash to support passing the max_plaintext_bytes limit.
+
+
 ## [0.2.0] - 2018-12-22
+
 ### Changed
 - relaxed Nokogiri dependency to '~> 1.8' for Redmine 4 (which uses 1.9)
 
+
 ## [0.1.0] - 2018-02-15
 - Initial release

data/Gemfile

@@ -1,4 +1,6 @@
 source 'https://rubygems.org'
 
+gem "byebug"
+
 # Specify your gem's dependencies in plaintext.gemspec
-gemspec
+gemspec

data/README.md

@@ -109,6 +109,9 @@ catdoc:
 fulltext = Plaintext::Resolver.new(file, content_type).text
 ```
 
+To limit the number of bytes returned (default is 4MB), set the
+`max_plaintext_bytes` property on the resolver instance before calling `text`.
+
 ## License
 
 The `plaintext` gem is free software; you can redistribute it and/or modify it under the terms of the GNU General 

data/lib/plaintext/file_handler.rb

@@ -11,5 +11,27 @@ module Plaintext
         false
       end
     end
+
+    # use `#set(max_size: 1.megabyte)` to give an upper limit of data to be read.
+    #
+    # By default, all data (whole file / command output) will be read which can
+    # be a problem with huge text files (eg SQL dumps)
+    def set(args = {})
+      options.update args
+      self
+    end
+
+    private
+
+    # maximum number of bytes to read from external command output or text
+    # files
+    def max_size
+      options[:max_size]
+    end
+
+    def options
+      @options ||= {}
+    end
+
   end
-end
+end

data/lib/plaintext/file_handler/external_command_handler.rb

@@ -11,23 +11,26 @@ module Plaintext
     # Due to how popen works the command will be executed directly without
     # involving the shell if cmd is an array.
     require 'fileutils'
+
+    FILE_PLACEHOLDER = '__FILE__'.freeze
+    DEFAULT_STREAM_ENCODING = 'ASCII-8BIT'.freeze
+
     def shellout(cmd, options = {}, &block)
       mode = "r+"
       IO.popen(cmd, mode) do |io|
-        io.set_encoding("ASCII-8BIT") if io.respond_to?(:set_encoding)
+        set_stream_encoding(io)
         io.close_write unless options[:write_stdin]
         block.call(io) if block_given?
       end
     end
 
-    FILE_PLACEHOLDER = '__FILE__'.freeze
-
-    def text(file)
+    def text(file, options = {})
       cmd = @command.dup
       cmd[cmd.index(FILE_PLACEHOLDER)] = Pathname(file).to_s
-      shellout(cmd){ |io| io.read }.to_s
+      shellout(cmd) { |io| read io, options[:max_size] }.to_s
     end
 
+
     def accept?(content_type)
       super and available?
     end
@@ -39,5 +42,33 @@ module Plaintext
     def self.available?
       new.available?
     end
+
+    protected
+
+    def utf8_stream?
+      false
+    end
+
+    private
+
+    def set_stream_encoding(io)
+      return unless io.respond_to?(:set_encoding)
+
+      if utf8_stream?
+        io.set_encoding('UTF-8'.freeze)
+      else
+        io.set_encoding(DEFAULT_STREAM_ENCODING)
+      end
+    end
+
+    def read(io, max_size = nil)
+      piece = io.read(max_size)
+
+      if utf8_stream?
+        piece
+      else
+        Plaintext::CodesetUtil.to_utf8 piece, DEFAULT_STREAM_ENCODING
+      end
+    end
   end
 end

data/lib/plaintext/file_handler/external_command_handler/pdf_handler.rb

@@ -5,9 +5,16 @@ module Plaintext
     DEFAULT = [
         '/usr/bin/pdftotext', '-enc', 'UTF-8', '__FILE__', '-'
     ].freeze
+
     def initialize
       @content_type = 'application/pdf'
       @command = Plaintext::Configuration['pdftotext'] || DEFAULT
     end
+
+    protected
+
+    def utf8_stream?
+      true
+    end
   end
 end

data/lib/plaintext/file_handler/external_command_handler/rtf_handler.rb

@@ -3,11 +3,34 @@
 module Plaintext
   class RtfHandler < ExternalCommandHandler
     DEFAULT = [
-        '/usr/bin/unrtf', '--text', '__FILE__'
+        '/usr/bin/unrtf', '--nopict', '--text', '__FILE__'
     ].freeze
     def initialize
       @content_type = 'application/rtf'
       @command = Plaintext::Configuration['unrtf'] || DEFAULT
     end
+
+    private
+
+    UNRTF_HEADER = "###  Translation from RTF performed by UnRTF"
+    END_MARKER   = "-----------------\n"
+
+    def read(io, max_size = nil)
+      if line = io.read(UNRTF_HEADER.length)
+        string = if line.starts_with? UNRTF_HEADER
+          io.gets while $_ != END_MARKER
+          io.read max_size
+        else
+          if max_size.nil?
+            line + io.read
+          elsif max_size > UNRTF_HEADER.length
+            line + io.read(max_size - UNRTF_HEADER.length)
+          else
+            line[0,max_size]
+          end
+        end
+        Plaintext::CodesetUtil.to_utf8 string, "ASCII-8BIT"
+      end
+    end
   end
 end

data/lib/plaintext/file_handler/plaintext_handler.rb

@@ -1,14 +1,16 @@
 # frozen_string_literal: true
 
 module Plaintext
+
   class PlaintextHandler < FileHandler
     CONTENT_TYPES = %w(text/csv text/plain)
     def initialize
       @content_types = CONTENT_TYPES
     end
 
-    def text(file)
-      Plaintext::CodesetUtil.to_utf8 IO.read(file), 'UTF-8'
+    def text(file, options = {})
+      max_size = options[:max_size]
+      Plaintext::CodesetUtil.to_utf8 IO.read(file, max_size), 'UTF-8'
     end
   end
-end
+end

data/lib/plaintext/file_handler/zipped_xml_handler.rb

@@ -9,16 +9,26 @@ module Plaintext
     class SaxDocument < Nokogiri::XML::SAX::Document
       attr_reader :text
 
-      def initialize(text_element, text_namespace)
+      def initialize(text_element, text_namespace, max_size = nil)
         @element = text_element
         @namespace_uri = text_namespace
+        @max_size = max_size
+
         @text = ''.dup
         @is_text = false
       end
 
+      def text_length_exceeded?
+        @max_size && (@text.length > @max_size)
+      end
+
+
       # Handle each element, expecting the name and any attributes
       def start_element_namespace(name, attrs = [], prefix = nil, uri = nil, ns = [])
-        if name == @element and uri == @namespace_uri
+        if name == @element and
+            uri == @namespace_uri and
+            !text_length_exceeded?
+
           @is_text = true
         end
       end
@@ -30,18 +40,22 @@ module Plaintext
 
       # Given the name of an element once its closing tag is reached
       def end_element_namespace(name, prefix = nil, uri = nil)
-        if name == @element and uri == @namespace_uri
+        if name == @element and
+            uri == @namespace_uri and
+            @is_text
+
           @text << ' '
           @is_text = false
         end
       end
     end
 
-    def text(file)
+    def text(file, options = {})
+      max_size = options[:max_size]
       Zip::File.open(file) do |zip_file|
         zip_file.each do |entry|
           if entry.name == @file_name
-            return xml_to_text entry.get_input_stream
+            return xml_to_text entry.get_input_stream, max_size
           end
         end
       end
@@ -49,10 +63,11 @@ module Plaintext
 
     private
 
-    def xml_to_text(io)
-      sax_doc = SaxDocument.new @element, @namespace_uri
+    def xml_to_text(io, max_size)
+      sax_doc = SaxDocument.new @element, @namespace_uri, max_size
       Nokogiri::XML::SAX::Parser.new(sax_doc).parse(io)
-      sax_doc.text
+      text = sax_doc.text
+      max_size.present? ? text[0, max_size] : text
     end
   end
-end
+end

data/lib/plaintext/file_handler/zipped_xml_handler/office_document_handler/pptx_handler.rb

@@ -14,17 +14,24 @@ module Plaintext
       @namespace_uri = 'http://schemas.openxmlformats.org/drawingml/2006/main'
     end
 
-    def text(file)
+    def text(file, options = {})
+      max_size = options[:max_size]
       slides = []
+      result = ''.dup
       Zip::File.open(file) do |zip_file|
         zip_file.each do |entry|
           if entry.name =~ /slide(\d+)\.xml/
-            slides << [$1, xml_to_text(entry.get_input_stream)]
+            slides << [$1, entry]
           end
         end
+
+        slides.sort!{|a, b| a.first <=> b.first}
+        slides.each do |id, entry|
+          result << xml_to_text(entry.get_input_stream, max_size)
+          break if max_size and result.length >= max_size
+        end
       end
-      slides.sort!{|a, b| a.first <=> b.first}
-      slides.map(&:last).join ' '
+      return result
     end
   end
-end
+end

data/lib/plaintext/resolver.rb

@@ -2,7 +2,9 @@
 
 module Plaintext
   class Resolver
-    MAX_FULLTEXT_LENGTH = 4_194_304 # 4 megabytes
+
+    # maximum length of returned plain text in bytes. Default: 4MB
+    attr_accessor :max_plaintext_bytes
 
     class << self
       attr_accessor :cached_file_handlers
@@ -26,15 +28,19 @@ module Plaintext
     def initialize(file, content_type = nil)
       @file = file
       @content_type = content_type
+      @max_plaintext_bytes = 4_194_304 # 4 megabytes
     end
 
+
     # Returns the extracted fulltext or nil if no matching handler was found
     # for the file type.
     def text
-      if handler = find_handler and text = handler.text(@file)
-        text.gsub! /\s+/m, ' '
+      if handler = find_handler and
+          text = handler.text(@file, max_size: max_plaintext_bytes)
+
+        text.gsub!(/\s+/m, ' ')
         text.strip!
-        text.mb_chars.compose.limit(MAX_FULLTEXT_LENGTH).to_s
+        text.mb_chars.compose.limit(max_plaintext_bytes).to_s
       end
     end
 

data/lib/plaintext/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Plaintext
-  VERSION = "0.2.0"
+  VERSION = "0.3.4"
 end

data/plaintext.gemspec

@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 require 'plaintext/version'
 
 Gem::Specification.new do |spec|
-  spec.name          = "plaintext"
+  spec.name          = 'plaintext'
   spec.version       = Plaintext::VERSION
   spec.authors       = ['Jens Krämer', 'Planio GmbH', 'OpenProject GmbH']
   spec.email         = ['info@openproject.com']
@@ -12,17 +12,18 @@ Gem::Specification.new do |spec|
   spec.summary       = 'Extract plain text from most common office documents.'
   spec.description   = "Extract text from common office files. Based on the file's content type a command line tool is selected to do the job."
   spec.homepage      = 'https://github.com/planio-gmbh/plaintext'
-  
+  spec.license       = 'GPL-2.0'
+
   spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
-  spec.bindir        = "exe"
+  spec.bindir        = 'exe'
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
-  spec.require_paths = ["lib"]
+  spec.require_paths = ['lib']
 
-  spec.add_dependency 'rubyzip', '~> 1.2.1'
-  spec.add_dependency 'nokogiri', '~> 1.8'
   spec.add_dependency 'activesupport', '>2.2.1 '
+  spec.add_dependency 'nokogiri', '~> 1.10', '>= 1.10.4'
+  spec.add_dependency 'rubyzip', '>= 1.2.0'
 
-  spec.add_development_dependency "bundler", "~> 1.10"
-  spec.add_development_dependency "rake", "~> 10.0"
-  spec.add_development_dependency "rspec"
+  spec.add_development_dependency 'bundler', '~> 2.0'
+  spec.add_development_dependency 'rake', '~> 12.0'
+  spec.add_development_dependency 'rspec'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: plaintext
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.3.4
 platform: ruby
 authors:
 - Jens Krämer
@@ -10,78 +10,84 @@ authors:
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2018-12-22 00:00:00.000000000 Z
+date: 2021-04-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: rubyzip
+  name: activesupport
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">"
       - !ruby/object:Gem::Version
-        version: 1.2.1
+        version: 2.2.1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">"
       - !ruby/object:Gem::Version
-        version: 1.2.1
+        version: 2.2.1
 - !ruby/object:Gem::Dependency
   name: nokogiri
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.8'
+        version: '1.10'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.10.4
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.8'
+        version: '1.10'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.10.4
 - !ruby/object:Gem::Dependency
-  name: activesupport
+  name: rubyzip
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 2.2.1
+        version: 1.2.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 2.2.1
+        version: 1.2.0
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.10'
+        version: '2.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.10'
+        version: '2.0'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '12.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '12.0'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
@@ -137,7 +143,8 @@ files:
 - plaintext.gemspec
 - plaintext.yml.example
 homepage: https://github.com/planio-gmbh/plaintext
-licenses: []
+licenses:
+- GPL-2.0
 metadata: {}
 post_install_message: 
 rdoc_options: []
@@ -154,8 +161,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.4.5.5
+rubygems_version: 3.0.3
 signing_key: 
 specification_version: 4
 summary: Extract plain text from most common office documents.