plaintext 0.2.0 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: c04f6f06ab6a16b423c2a74ddb38685a49a23372
-  data.tar.gz: 8dc49ce59bde01f25983b982323889a512fa900f
+  metadata.gz: eb87c916a0f34e20251b5a97cbcf1e354fc046b0
+  data.tar.gz: e6dff85b7b0cec1ae3fdebbaabb950be1b563215
 SHA512:
-  metadata.gz: 13876311c960330abc0596ccc09a5eb172feca1101fa154386182bb04c7cfe6ca5f2b8fb4a408ffa658fbe0cbc616e29c8770c2a0680ec11064e950a75eae19d
-  data.tar.gz: 50f4d40bcb4bfb257b4e280e081cea291a633203746c322c028e6dcacd53b68f0e5c50da6ede80bdeb0ae5a7c300c0ddb38694b4ee97d6ed8343ae81e5c8a69e
+  metadata.gz: 48be94f65a07c3e400bacfae575758494afbf3d6004b19a99f7838a032b26d4503ae3ea47b46548de0dd34b4892c551d5d7af3620a72375ae9a771afc0601cc2
+  data.tar.gz: 0b5dc9aa9abae205083d3dd4dcc89572bf36361fee098c6145472735ac6d10c810b07595be095565baa18b99534b61bbbc63b4e26c445ccdac94a3468b6c962c

data/CHANGELOG

@@ -6,9 +6,26 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [0.3.0] - 2019-01-09
+
+### Added
+- `:max_plaintext_bytes` option to place an upper limit on the number of bytes
+  returned. Also limits the amount of data that is actually read from plain text
+  files and external command output to limit memory usage. Set this on the
+  resolver object.
+
+### Changed
+- the unrtf handler now strips the preamble which unrtf adds to its output.
+- text from external command handlers is now converted to UTF-8 before it's returned.
+- the FileHandler `text` method signature has been changed by adding an options
+  hash to support passing the max_plaintext_bytes limit.
+
+
 ## [0.2.0] - 2018-12-22
+
 ### Changed
 - relaxed Nokogiri dependency to '~> 1.8' for Redmine 4 (which uses 1.9)
 
+
 ## [0.1.0] - 2018-02-15
 - Initial release

data/Gemfile

@@ -1,4 +1,6 @@
 source 'https://rubygems.org'
 
+gem "byebug"
+
 # Specify your gem's dependencies in plaintext.gemspec
-gemspec
+gemspec

data/README.md

@@ -109,6 +109,9 @@ catdoc:
 fulltext = Plaintext::Resolver.new(file, content_type).text
 ```
 
+To limit the number of bytes returned (default is 4MB), set the
+`max_plaintext_bytes` property on the resolver instance before calling `text`.
+
 ## License
 
 The `plaintext` gem is free software; you can redistribute it and/or modify it under the terms of the GNU General 

data/lib/plaintext/file_handler.rb

@@ -11,5 +11,27 @@ module Plaintext
         false
       end
     end
+
+    # use `#set(max_size: 1.megabyte)` to give an upper limit of data to be read.
+    #
+    # By default, all data (whole file / command output) will be read which can
+    # be a problem with huge text files (eg SQL dumps)
+    def set(args = {})
+      options.update args
+      self
+    end
+
+    private
+
+    # maximum number of bytes to read from external command output or text
+    # files
+    def max_size
+      options[:max_size]
+    end
+
+    def options
+      @options ||= {}
+    end
+
   end
-end
+end

data/lib/plaintext/file_handler/external_command_handler.rb

@@ -22,12 +22,13 @@ module Plaintext
 
     FILE_PLACEHOLDER = '__FILE__'.freeze
 
-    def text(file)
+    def text(file, options = {})
       cmd = @command.dup
       cmd[cmd.index(FILE_PLACEHOLDER)] = Pathname(file).to_s
-      shellout(cmd){ |io| io.read }.to_s
+      shellout(cmd){ |io| read io, options[:max_size] }.to_s
     end
 
+
     def accept?(content_type)
       super and available?
     end
@@ -39,5 +40,11 @@ module Plaintext
     def self.available?
       new.available?
     end
+
+    private
+
+    def read(io, max_size = nil)
+      Plaintext::CodesetUtil.to_utf8 io.read(max_size), "ASCII-8BIT"
+    end
   end
 end

data/lib/plaintext/file_handler/external_command_handler/rtf_handler.rb

@@ -3,11 +3,34 @@
 module Plaintext
   class RtfHandler < ExternalCommandHandler
     DEFAULT = [
-        '/usr/bin/unrtf', '--text', '__FILE__'
+        '/usr/bin/unrtf', '--nopict', '--quiet', '--text', '__FILE__'
     ].freeze
     def initialize
       @content_type = 'application/rtf'
       @command = Plaintext::Configuration['unrtf'] || DEFAULT
     end
+
+    private
+
+    UNRTF_HEADER = "###  Translation from RTF performed by UnRTF"
+    END_MARKER   = "-----------------\n"
+
+    def read(io, max_size = nil)
+      if line = io.read(UNRTF_HEADER.length)
+        string = if line.starts_with? UNRTF_HEADER
+          io.gets while $_ != END_MARKER
+          io.read max_size
+        else
+          if max_size.nil?
+            line + io.read
+          elsif max_size > UNRTF_HEADER.length
+            line + io.read(max_size - UNRTF_HEADER.length)
+          else
+            line[0,max_size]
+          end
+        end
+        Plaintext::CodesetUtil.to_utf8 string, "ASCII-8BIT"
+      end
+    end
   end
 end

data/lib/plaintext/file_handler/plaintext_handler.rb

@@ -1,14 +1,16 @@
 # frozen_string_literal: true
 
 module Plaintext
+
   class PlaintextHandler < FileHandler
     CONTENT_TYPES = %w(text/csv text/plain)
     def initialize
       @content_types = CONTENT_TYPES
     end
 
-    def text(file)
-      Plaintext::CodesetUtil.to_utf8 IO.read(file), 'UTF-8'
+    def text(file, options = {})
+      max_size = options[:max_size]
+      Plaintext::CodesetUtil.to_utf8 IO.read(file, max_size), 'UTF-8'
     end
   end
-end
+end

data/lib/plaintext/file_handler/zipped_xml_handler.rb

@@ -37,7 +37,7 @@ module Plaintext
       end
     end
 
-    def text(file)
+    def text(file, options = {})
       Zip::File.open(file) do |zip_file|
         zip_file.each do |entry|
           if entry.name == @file_name
@@ -55,4 +55,4 @@ module Plaintext
       sax_doc.text
     end
   end
-end
+end

data/lib/plaintext/file_handler/zipped_xml_handler/office_document_handler/pptx_handler.rb

@@ -14,7 +14,7 @@ module Plaintext
       @namespace_uri = 'http://schemas.openxmlformats.org/drawingml/2006/main'
     end
 
-    def text(file)
+    def text(file, options = {})
       slides = []
       Zip::File.open(file) do |zip_file|
         zip_file.each do |entry|
@@ -27,4 +27,4 @@ module Plaintext
       slides.map(&:last).join ' '
     end
   end
-end
+end

data/lib/plaintext/resolver.rb

@@ -2,7 +2,9 @@
 
 module Plaintext
   class Resolver
-    MAX_FULLTEXT_LENGTH = 4_194_304 # 4 megabytes
+
+    # maximum length of returned plain text in bytes. Default: 4MB
+    attr_accessor :max_plaintext_bytes
 
     class << self
       attr_accessor :cached_file_handlers
@@ -26,15 +28,19 @@ module Plaintext
     def initialize(file, content_type = nil)
       @file = file
       @content_type = content_type
+      @max_plaintext_bytes = 4_194_304 # 4 megabytes
     end
 
+
     # Returns the extracted fulltext or nil if no matching handler was found
     # for the file type.
     def text
-      if handler = find_handler and text = handler.text(@file)
-        text.gsub! /\s+/m, ' '
+      if handler = find_handler and
+          text = handler.text(@file, max_size: max_plaintext_bytes)
+
+        text.gsub!(/\s+/m, ' ')
         text.strip!
-        text.mb_chars.compose.limit(MAX_FULLTEXT_LENGTH).to_s
+        text.mb_chars.compose.limit(max_plaintext_bytes).to_s
       end
     end
 

data/lib/plaintext/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Plaintext
-  VERSION = "0.2.0"
+  VERSION = "0.3.0"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: plaintext
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.3.0
 platform: ruby
 authors:
 - Jens Krämer
@@ -10,7 +10,7 @@ authors:
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2018-12-22 00:00:00.000000000 Z
+date: 2019-01-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rubyzip
@@ -155,7 +155,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.5.5
+rubygems_version: 2.5.2.1
 signing_key: 
 specification_version: 4
 summary: Extract plain text from most common office documents.