simple_text_extract 0.2.0 → 1.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: fa34a31195f18156df695d2fb860a2beb9562f6a8ae76d6f2b6a7d4585e2e306
-  data.tar.gz: 3af31eaa54ed98bf8d0355cdf2bdf29f525a97e1ae66573b1c7317cbfcd3c951
+  metadata.gz: 35eee7886cc545693f85facc69bbe8b99ea141f0af3ad520f6f75d2bc61eddd7
+  data.tar.gz: 5a4b8e4a4dfe54535805f56e986fe36b5470889a96960b2bf8acd1ee4c94b084
 SHA512:
-  metadata.gz: 3d37e232dd959b4c0897439a29b46f64d598f024b61bba2edc2cdad1f0d14461db77b93a227326697d4913e83ab87b4393234943aef964285b253c652849436d
-  data.tar.gz: 44255a841598321a97559ef3a77c3db3e6f6a7d22cb78fc78819aaecef643b8b67c85484b4f10ac979923f28c8f727fcaca69ebec694b3df9fe5f6a9c37cc1f9
+  metadata.gz: 9ace8965b9567c8d9e85f2d52912c8081da83a2dc0ff9c463f6ac28450e39ca874125e501196f546930615b4d6271abad88dc3fe8b91f0ade1253c0468652eca
+  data.tar.gz: 05d9819fd5835b8307ae6ece9b0f4efa8b00d615eaf6fcc054dc38959780761ed4cf40711d7b91de38a9e3390904dead968c0a4a0ccc9221a8ec5122a6f9f0b8

data/.ruby-version

@@ -1 +1 @@
-2.5.3
+2.6.5

data/Gemfile

@@ -4,6 +4,4 @@ source "https://rubygems.org"
 
 git_source(:github) { |repo_name| "https://github.com/#{repo_name}" }
 
-gem "pry"
-
 gemspec

data/Gemfile.lock

@@ -1,44 +1,35 @@
 PATH
   remote: .
   specs:
-    simple_text_extract (0.1.3)
-      roo (~> 2.8)
+    simple_text_extract (1.0.0)
+      roo (~> 2.8.2)
       spreadsheet (~> 1.1.8)
 
 GEM
   remote: https://rubygems.org/
   specs:
-    coderay (1.1.2)
-    metaclass (0.0.4)
-    method_source (0.9.2)
     mini_portile2 (2.4.0)
-    minitest (5.11.3)
-    mocha (1.8.0)
-      metaclass (~> 0.0.1)
-    nokogiri (1.10.1)
+    minitest (5.14.0)
+    mocha (1.11.2)
+    nokogiri (1.10.9)
       mini_portile2 (~> 2.4.0)
-    pry (0.12.2)
-      coderay (~> 1.1.0)
-      method_source (~> 0.9.0)
-    rake (10.5.0)
-    roo (2.8.1)
+    rake (13.0.1)
+    roo (2.8.3)
       nokogiri (~> 1)
-      rubyzip (>= 1.2.1, < 2.0.0)
-    ruby-ole (1.2.12.1)
-    rubyzip (1.2.2)
-    spreadsheet (1.1.8)
+      rubyzip (>= 1.3.0, < 3.0.0)
+    ruby-ole (1.2.12.2)
+    rubyzip (2.3.0)
+    spreadsheet (1.1.9)
       ruby-ole (>= 1.0)
 
 PLATFORMS
   ruby
 
 DEPENDENCIES
-  bundler (~> 1.17)
   minitest (~> 5.0)
   mocha
-  pry
-  rake (~> 10.0)
+  rake (~> 13.0)
   simple_text_extract!
 
 BUNDLED WITH
-   1.17.2
+   2.0.2

data/README.md

@@ -9,6 +9,7 @@ SimpleTextExtract handles parsing text from:
 - `.doc`
 - `.xlsx`
 - `.xls`
+- `.csv`
 - `.txt` 😜
 
 If no text is parsed (for `pdf`), or a file format is not supported (like images), then `nil` is returned and you can move on to the heavy-duty tools like [Henkei](https://github.com/abrom/henkei) 💪.
@@ -34,11 +35,13 @@ Or install it yourself as:
 Text can be parsed from raw file content or files in the filesystem t by calling `SimpleTextExtract.extract`:
 
 ```ruby
-# raw file content using ActiveStorage
-SimpleTextExtract.extract(filename: attachment.blob.filename, raw: attachment.download)
+# using ActiveStorage >= 6
+extract = attachment.open { |tmp| SimpleTextExtract.extract(tempfile: tmp) }
+# raw file content or when ActiveStorage < 6
+extract = SimpleTextExtract.extract(filename: attachment.blob.filename, raw: attachment.download)
 
 # filesystem
-SimpleTextExtract.extract(filepath: "path_to_file.pdf")
+extract = SimpleTextExtract.extract(filepath: "path_to_file.pdf")
 ```
 
 ### Usage Dependencies
@@ -51,6 +54,9 @@ You can choose to use SimpleTextExtract without the following dependencies, but
 `doc` parsing requires `antiword`
 - `brew install antiword`
 
+`xlsx` and `xls` parsing requires `ssconvert` which is part of `gnumeric`
+- `brew install gnumeric`
+
 ### Usage on Heroku
 
 To use on Heroku you'll have to add some custom buildpacks.
@@ -64,13 +70,16 @@ If not, you can either add that buildpack, or add `poppler-utils` to your `Aptfi
 
 ##### heroku-buildpack-apt
 
-To add `antiword` as a dependency on Heroku, install the [heroku-buildpack-apt](https://elements.heroku.com/buildpacks/heroku/heroku-buildpack-apt) buildpack and follow the install instructions.
+To add `antiword` and/or `gnumeric`* as a dependency on Heroku, install the [heroku-buildpack-apt](https://elements.heroku.com/buildpacks/heroku/heroku-buildpack-apt) buildpack and follow the install instructions.
 
 In your `Aptfile`, add:
 ```
 antiword
+gnumeric
 ```
 
+* There is currently an [issue](https://github.com/heroku/heroku-buildpack-google-chrome/issues/59) with the heroku-18 stack that requires additional dependencies added to the Aptfile to get `gnumeric` to work properly.  You can reference the linked issue above to figure out those dependencies, or downgrade to heroku-16 until it is fixed.
+
 ## Benchmarks
 
 *Benchmarks test extracting text from the same file 50 times (Macbook pro)*
@@ -80,8 +89,9 @@ antiword
 | .doc        | 1.40s             | 74.27s                         |
 | .docx       | 0.78s             | 71.44s                         |
 | .pdf*       | 1.73s             | 82.86s                         |
-| .xlsx       | 21.99s            | 51.89s                         |
-| .txt        | 0.036s            | 39.25s                         |
+| .xlsx       | 1.16s             | 51.89s                         |
+| .xls        | 0.80s             | 67.88s                         |
+| .txt        | 0.04s             | 39.25s                         |
 
 * SimpleTextExtract is limited in its text extraction from pdfs, as Tika can also perform OCR on pdfs with Tesseract
 

data/Rakefile

@@ -7,4 +7,4 @@ Rake::TestTask.new(:test) do |t|
   t.test_files = FileList["test/**/*_test.rb"]
 end
 
-task :default => :test
+task default: :test

data/bin/console

@@ -6,8 +6,5 @@ require "simple_text_extract"
 # You can add fixtures and/or initialization code here to make experimenting
 # with your gem easier. You can also use a different console, if you like.
 
-require "pry"
-Pry.start
-
 require "irb"
 IRB.start(__FILE__)

data/lib/simple_text_extract.rb

@@ -2,14 +2,18 @@
 
 require "simple_text_extract/version"
 require "simple_text_extract/text_extractor"
-require "simple_text_extract/file_extractor"
-require "simple_text_extract/tempfile_extractor"
 require "simple_text_extract/format_extractor_factory"
 
 module SimpleTextExtract
+  SUPPORTED_FILETYPES = ["xls", "xlsx", "doc", "docx", "txt", "pdf", "csv"].freeze
+
   class Error < StandardError; end
 
-  def self.extract(filename: nil, raw: nil, filepath: nil)
-    TextExtractor.call(filename: filename, raw: raw, filepath: filepath).to_s
+  def self.extract(filename: nil, raw: nil, filepath: nil, tempfile: nil)
+    TextExtractor.new(filename: filename, raw: raw, filepath: filepath, tempfile: tempfile).to_s
+  end
+
+  def self.supports?(filename: nil)
+    SUPPORTED_FILETYPES.include?(filename.split(".").last)
   end
 end

data/lib/simple_text_extract/format_extractor/doc.rb

@@ -4,7 +4,7 @@ module SimpleTextExtract
   module FormatExtractor
     class Doc < Base
       def extract
-        return nil if missing_dependency?('antiword')
+        return nil if missing_dependency?("antiword")
 
         `antiword #{Shellwords.escape(file.path)}`
       end

data/lib/simple_text_extract/format_extractor/xls.rb

@@ -8,7 +8,6 @@ module SimpleTextExtract
 
         spreadsheet = Spreadsheet.open(file)
         text = []
-
         spreadsheet.worksheets.each do |sheet|
           text << sheet.name
           text << sheet.rows

data/lib/simple_text_extract/format_extractor/xls_x.rb

@@ -6,7 +6,7 @@ module SimpleTextExtract
       def extract
         require "roo"
 
-        spreadsheet = Roo::Spreadsheet.open(file)
+        spreadsheet = Roo::Spreadsheet.open(file, only_visible_sheets: true)
 
         text = []
 

data/lib/simple_text_extract/format_extractor_factory.rb

@@ -10,9 +10,9 @@ require "simple_text_extract/format_extractor/doc"
 
 module SimpleTextExtract
   class FormatExtractorFactory
-    def self.call(file) # rubocop:disable Metrics/MethodLength
+    def self.call(file) # rubocop:disable Metrics/MethodLength, Metrics/CyclomaticComplexity
       case file.path
-      when /.txt$/i
+      when /(.txt$|.csv$)/i
         FormatExtractor::PlainText.new(file)
       when /.pdf$/i
         FormatExtractor::PDF.new(file)

data/lib/simple_text_extract/text_extractor.rb

@@ -2,24 +2,55 @@
 
 module SimpleTextExtract
   class TextExtractor
-    def self.call(filename: nil, raw: nil, filepath: nil)
-      if !filename.nil? && !raw.nil?
-        TempfileExtractor.new(filename: filename.to_s, raw: raw).extract
-      elsif !filepath.nil? && File.exist?(filepath)
-        FileExtractor.new(filepath: filepath).extract
-      end
-    end
+    attr_reader :file
 
-    def extract
-      text = FormatExtractorFactory.call(file).extract
-      cleanup
+    def initialize(filename: nil, raw: nil, filepath: nil, tempfile: nil)
+      @file = get_file(filename: filename, raw: raw, filepath: filepath, tempfile: tempfile)
+    end
 
-      text
+    def to_s
+      @to_s ||= extract.to_s
     end
 
     private
 
+      def get_file(filename:, raw:, filepath:, tempfile:)
+        if tempfile&.class == Tempfile
+          tempfile
+        elsif !filename.nil? && !raw.nil?
+          write_tempfile(filename: filename.to_s, raw: raw)
+        elsif !filepath.nil? && File.exist?(filepath)
+          File.new(filepath)
+        end
+      end
+
+      def extract
+        return unless file
+        return unless file
+
+        begin
+          FormatExtractorFactory.call(file).extract
+        rescue StandardError
+          nil
+        ensure
+          cleanup
+        end
+      end
+
       def cleanup
+        return unless file.class == Tempfile
+
+        file.close
+        file.unlink
+      end
+
+      def write_tempfile(filename:, raw:)
+        filename = filename.split(".").yield_self { |parts| [parts[0], ".#{parts[1]}"] }
+        file = Tempfile.new(filename)
+        raw = String.new(raw, encoding: Encoding::UTF_8)
+
+        file.write(raw)
+        file.tap(&:rewind)
       end
   end
 end

data/lib/simple_text_extract/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module SimpleTextExtract
-  VERSION = "0.2.0"
+  VERSION = "1.1.0"
 end

data/simple_text_extract.gemspec

@@ -28,11 +28,10 @@ Gem::Specification.new do |spec|
   spec.requirements << "pdftotext/poppler"
   spec.required_ruby_version = ">= 2.5"
 
-  spec.add_runtime_dependency "roo", "~> 2.8"
+  spec.add_runtime_dependency "roo", "~> 2.8.2"
   spec.add_runtime_dependency "spreadsheet", "~> 1.1.8"
 
-  spec.add_development_dependency "bundler", "~> 1.17"
-  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "rake", "~> 13.0"
   spec.add_development_dependency "minitest", "~> 5.0"
   spec.add_development_dependency "mocha"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: simple_text_extract
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 1.1.0
 platform: ruby
 authors:
 - Nick Weiland
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2019-01-25 00:00:00.000000000 Z
+date: 2020-07-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: roo
@@ -16,14 +16,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.8'
+        version: 2.8.2
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.8'
+        version: 2.8.2
 - !ruby/object:Gem::Dependency
   name: spreadsheet
   requirement: !ruby/object:Gem::Requirement
@@ -38,34 +38,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 1.1.8
-- !ruby/object:Gem::Dependency
-  name: bundler
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1.17'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1.17'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '13.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '13.0'
 - !ruby/object:Gem::Dependency
   name: minitest
   requirement: !ruby/object:Gem::Requirement
@@ -114,7 +100,6 @@ files:
 - bin/console
 - bin/setup
 - lib/simple_text_extract.rb
-- lib/simple_text_extract/file_extractor.rb
 - lib/simple_text_extract/format_extractor/base.rb
 - lib/simple_text_extract/format_extractor/doc.rb
 - lib/simple_text_extract/format_extractor/doc_x.rb
@@ -123,9 +108,9 @@ files:
 - lib/simple_text_extract/format_extractor/xls.rb
 - lib/simple_text_extract/format_extractor/xls_x.rb
 - lib/simple_text_extract/format_extractor_factory.rb
-- lib/simple_text_extract/tempfile_extractor.rb
 - lib/simple_text_extract/text_extractor.rb
 - lib/simple_text_extract/version.rb
+- simple_text_extract-1.0.2.gem
 - simple_text_extract.gemspec
 - tags
 homepage: https://github.com/weilandia/simple_text_extract
@@ -149,8 +134,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
 requirements:
 - antiword
 - pdftotext/poppler
-rubyforge_project: 
-rubygems_version: 2.7.6
+rubygems_version: 3.0.3
 signing_key: 
 specification_version: 4
 summary: Attempts to quickly extract text from various file types before resorting

data/lib/simple_text_extract/file_extractor.rb

@@ -1,17 +0,0 @@
-# frozen_string_literal: true
-
-module SimpleTextExtract
-  class FileExtractor < TextExtractor
-    attr_reader :filepath
-
-    def initialize(filepath:)
-      @filepath = filepath
-    end
-
-    private
-
-      def file
-        @file ||= File.new(filepath)
-      end
-  end
-end

data/lib/simple_text_extract/tempfile_extractor.rb

@@ -1,34 +0,0 @@
-# frozen_string_literal: true
-
-module SimpleTextExtract
-  class TempfileExtractor < TextExtractor
-    attr_reader :filename, :raw
-
-    def initialize(filename:, raw:)
-      @filename = filename
-      @raw = String.new(raw, encoding: Encoding::UTF_8)
-
-      write_raw
-    end
-
-    private
-
-      def file
-        @file ||= Tempfile.new(filepath)
-      end
-
-      def write_raw
-        file.write(raw)
-        file.rewind
-      end
-
-      def cleanup
-        file.close
-        file.unlink
-      end
-
-      def filepath
-        @filepath ||= filename.split(".").yield_self { |parts| [parts[0], ".#{parts[1]}"] }
-      end
-  end
-end