simple_text_extract 0.1.3 → 1.0.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: bd06e0bd11dd74c71adb01918b474b714ebd3762785931d14d294052aa3301e5
-  data.tar.gz: 8fbfff63e6403e4abfc980abd03fce52a1653010c6d401c2507ae253d9391916
+  metadata.gz: 67a6daab9ba3d33ea757384fda1407875c1451cb2be0bb636ffea9b32384c12d
+  data.tar.gz: e5077817daf69f20d5ad54ae82b55465cf3727f5acb20df7382cc54403ca3e43
 SHA512:
-  metadata.gz: 6d2cc814d3b419e9540800752f097b2f9d860e4756e7bd6b8e62f6178ad6cd7967c0f0dd9912a6f900564a686e78564c2c2acb7cee412c9e857ae6ed48cc906e
-  data.tar.gz: dbb00c6da2de38f9d254486adb98bbce2c607a2c54c8637e2b8c9e08efae746ff8bd2ef384a9b1f7ffc2d4edaabda7527498c63726525aa29ba005f24db03770
+  metadata.gz: 7ef181da803d55ba917a5051402a3ac8527deb8886c68c417b4eabc677523a87fd011840944018c3c485a48c8dd2098b60960fc639b716481310c3ccc30f87a3
+  data.tar.gz: f73297a615714bbf29b48b87f0437f10c565078dd97d7ab0a07171f422233e5170f54e97eb19ea81f1e85b4777d52482807cc6efcc8c1737d31e52c41c59d778

data/.ruby-version

@@ -1 +1 @@
-2.5.3
+2.6.5

data/Gemfile

@@ -4,6 +4,4 @@ source "https://rubygems.org"
 
 git_source(:github) { |repo_name| "https://github.com/#{repo_name}" }
 
-gem "pry"
-
 gemspec

data/Gemfile.lock

@@ -1,32 +1,35 @@
 PATH
   remote: .
   specs:
-    simple_text_extract (0.1.1)
+    simple_text_extract (1.0.0)
+      roo (~> 2.8.2)
+      spreadsheet (~> 1.1.8)
 
 GEM
   remote: https://rubygems.org/
   specs:
-    coderay (1.1.2)
-    metaclass (0.0.4)
-    method_source (0.9.2)
-    minitest (5.11.3)
-    mocha (1.8.0)
-      metaclass (~> 0.0.1)
-    pry (0.12.2)
-      coderay (~> 1.1.0)
-      method_source (~> 0.9.0)
-    rake (10.5.0)
+    mini_portile2 (2.4.0)
+    minitest (5.14.0)
+    mocha (1.11.2)
+    nokogiri (1.10.9)
+      mini_portile2 (~> 2.4.0)
+    rake (13.0.1)
+    roo (2.8.3)
+      nokogiri (~> 1)
+      rubyzip (>= 1.3.0, < 3.0.0)
+    ruby-ole (1.2.12.2)
+    rubyzip (2.3.0)
+    spreadsheet (1.1.9)
+      ruby-ole (>= 1.0)
 
 PLATFORMS
   ruby
 
 DEPENDENCIES
-  bundler (~> 1.17)
   minitest (~> 5.0)
   mocha
-  pry
-  rake (~> 10.0)
+  rake (~> 13.0)
   simple_text_extract!
 
 BUNDLED WITH
-   1.17.2
+   2.0.2

data/README.md

@@ -9,6 +9,7 @@ SimpleTextExtract handles parsing text from:
 - `.doc`
 - `.xlsx`
 - `.xls`
+- `.csv`
 - `.txt` 😜
 
 If no text is parsed (for `pdf`), or a file format is not supported (like images), then `nil` is returned and you can move on to the heavy-duty tools like [Henkei](https://github.com/abrom/henkei) 💪.
@@ -34,11 +35,13 @@ Or install it yourself as:
 Text can be parsed from raw file content or files in the filesystem t by calling `SimpleTextExtract.extract`:
 
 ```ruby
-# raw file content using ActiveStorage
-SimpleTextExtract.extract(filename: attachment.blob.filename, raw: attachment.download)
+# using ActiveStorage >= 6
+extract = attachment.open { |tmp| SimpleTextExtract.extract(tempfile: tmp) }
+# raw file content or when ActiveStorage < 6
+extract = SimpleTextExtract.extract(filename: attachment.blob.filename, raw: attachment.download)
 
 # filesystem
-SimpleTextExtract.extract(filepath: "path_to_file.pdf")
+extract = SimpleTextExtract.extract(filepath: "path_to_file.pdf")
 ```
 
 ### Usage Dependencies
@@ -67,7 +70,7 @@ If not, you can either add that buildpack, or add `poppler-utils` to your `Aptfi
 
 ##### heroku-buildpack-apt
 
-To add `antiword` as a dependency on Heroku, install the [heroku-buildpack-apt](https://elements.heroku.com/buildpacks/heroku/heroku-buildpack-apt) buildpack and follow the install instructions.
+To add `antiword` and/or `gnumeric`* as a dependency on Heroku, install the [heroku-buildpack-apt](https://elements.heroku.com/buildpacks/heroku/heroku-buildpack-apt) buildpack and follow the install instructions.
 
 In your `Aptfile`, add:
 ```
@@ -75,6 +78,8 @@ antiword
 gnumeric
 ```
 
+* There is currently an [issue](https://github.com/heroku/heroku-buildpack-google-chrome/issues/59) with the heroku-18 stack that requires additional dependencies added to the Aptfile to get `gnumeric` to work properly.  You can reference the linked issue above to figure out those dependencies, or downgrade to heroku-16 until it is fixed.
+
 ## Benchmarks
 
 *Benchmarks test extracting text from the same file 50 times (Macbook pro)*
@@ -84,8 +89,9 @@ gnumeric
 | .doc        | 1.40s             | 74.27s                         |
 | .docx       | 0.78s             | 71.44s                         |
 | .pdf*       | 1.73s             | 82.86s                         |
-| .xlsx       | 21.99s            | 51.89s                         |
-| .txt        | 0.036s            | 39.25s                         |
+| .xlsx       | 1.16s             | 51.89s                         |
+| .xls        | 0.80s             | 67.88s                         |
+| .txt        | 0.04s             | 39.25s                         |
 
 * SimpleTextExtract is limited in its text extraction from pdfs, as Tika can also perform OCR on pdfs with Tesseract
 

data/Rakefile

@@ -7,4 +7,4 @@ Rake::TestTask.new(:test) do |t|
   t.test_files = FileList["test/**/*_test.rb"]
 end
 
-task :default => :test
+task default: :test

data/bin/console

@@ -6,8 +6,5 @@ require "simple_text_extract"
 # You can add fixtures and/or initialization code here to make experimenting
 # with your gem easier. You can also use a different console, if you like.
 
-require "pry"
-Pry.start
-
 require "irb"
 IRB.start(__FILE__)

data/lib/simple_text_extract.rb

@@ -2,14 +2,18 @@
 
 require "simple_text_extract/version"
 require "simple_text_extract/text_extractor"
-require "simple_text_extract/file_extractor"
-require "simple_text_extract/tempfile_extractor"
 require "simple_text_extract/format_extractor_factory"
 
 module SimpleTextExtract
+  SUPPORTED_FILETYPES = ["xls", "xlsx", "doc", "docx", "txt", "pdf", "csv"].freeze
+
   class Error < StandardError; end
 
-  def self.extract(filename: nil, raw: nil, filepath: nil)
-    TextExtractor.call(filename: filename, raw: raw, filepath: filepath).to_s
+  def self.extract(filename: nil, raw: nil, filepath: nil, tempfile: nil)
+    TextExtractor.new(filename: filename, raw: raw, filepath: filepath, tempfile: tempfile).to_s
+  end
+
+  def self.supports?(filename: nil)
+    SUPPORTED_FILETYPES.include?(filename.split(".").last)
   end
 end

data/lib/simple_text_extract/format_extractor/xls.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+module SimpleTextExtract
+  module FormatExtractor
+    class Xls < Base
+      def extract
+        require "spreadsheet"
+
+        spreadsheet = Spreadsheet.open(file)
+        text = []
+        spreadsheet.worksheets.each do |sheet|
+          text << sheet.name
+          text << sheet.rows
+        end
+
+        text.flatten.join(" ")
+      end
+    end
+  end
+end

data/lib/simple_text_extract/format_extractor/xls_x.rb

@@ -4,16 +4,18 @@ module SimpleTextExtract
   module FormatExtractor
     class XlsX < Base
       def extract
-        return nil if missing_dependency?("ssconvert")
+        require "roo"
 
-        extract_filepath = "#{file.path.split(".")[0]}.txt"
+        spreadsheet = Roo::Spreadsheet.open(file, only_visible_sheets: true)
 
-        `ssconvert -O 'separator=" "' #{Shellwords.escape(file.path)} #{extract_filepath}`
+        text = []
 
-        text = File.read(extract_filepath)
-        File.unlink(extract_filepath)
+        spreadsheet.each_with_pagename do |name, sheet|
+          text << name
+          1.upto(sheet.last_row.to_i) { |row| text << sheet.row(row) }
+        end
 
-        text
+        text.flatten.join(" ")
       end
     end
   end

data/lib/simple_text_extract/format_extractor_factory.rb

@@ -4,14 +4,15 @@ require "simple_text_extract/format_extractor/base"
 require "simple_text_extract/format_extractor/plain_text"
 require "simple_text_extract/format_extractor/pdf"
 require "simple_text_extract/format_extractor/xls_x"
+require "simple_text_extract/format_extractor/xls"
 require "simple_text_extract/format_extractor/doc_x"
 require "simple_text_extract/format_extractor/doc"
 
 module SimpleTextExtract
   class FormatExtractorFactory
-    def self.call(file) # rubocop:disable Metrics/MethodLength
+    def self.call(file) # rubocop:disable Metrics/MethodLength, Metrics/CyclomaticComplexity
       case file.path
-      when /.txt$/i
+      when /(.txt$|.csv$)/i
         FormatExtractor::PlainText.new(file)
       when /.pdf$/i
         FormatExtractor::PDF.new(file)
@@ -19,8 +20,10 @@ module SimpleTextExtract
         FormatExtractor::DocX.new(file)
       when /.doc$/i
         FormatExtractor::Doc.new(file)
-      when /(.xlsx$|.xls$)/i
+      when /.xlsx$/i
         FormatExtractor::XlsX.new(file)
+      when /.xls$/i
+        FormatExtractor::Xls.new(file)
       else
         FormatExtractor::Base.new(file)
       end

data/lib/simple_text_extract/text_extractor.rb

@@ -2,24 +2,52 @@
 
 module SimpleTextExtract
   class TextExtractor
-    def self.call(filename: nil, raw: nil, filepath: nil)
-      if !filename.nil? && !raw.nil?
-        TempfileExtractor.new(filename: filename.to_s, raw: raw).extract
-      elsif !filepath.nil? && File.exist?(filepath)
-        FileExtractor.new(filepath: filepath).extract
-      end
-    end
+    attr_reader :file
 
-    def extract
-      text = FormatExtractorFactory.call(file).extract
-      cleanup
+    def initialize(filename: nil, raw: nil, filepath: nil, tempfile: nil)
+      @file = get_file(filename: filename, raw: raw, filepath: filepath, tempfile: tempfile)
+    end
 
-      text
+    def to_s
+      @to_s ||= extract.to_s
     end
 
     private
 
+      def get_file(filename:, raw:, filepath:, tempfile:)
+        if tempfile&.class == Tempfile
+          tempfile
+        elsif !filename.nil? && !raw.nil?
+          write_tempfile(filename: filename.to_s, raw: raw)
+        elsif !filepath.nil? && File.exist?(filepath)
+          File.new(filepath)
+        end
+      end
+
+      def extract
+        return unless file
+
+        begin
+          FormatExtractorFactory.call(file).extract
+        ensure
+          cleanup
+        end
+      end
+
       def cleanup
+        return unless file.class == Tempfile
+
+        file.close
+        file.unlink
+      end
+
+      def write_tempfile(filename:, raw:)
+        filename = filename.split(".").yield_self { |parts| [parts[0], ".#{parts[1]}"] }
+        file = Tempfile.new(filename)
+        raw = String.new(raw, encoding: Encoding::UTF_8)
+
+        file.write(raw)
+        file.tap(&:rewind)
       end
   end
 end

data/lib/simple_text_extract/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module SimpleTextExtract
-  VERSION = "0.1.3"
+  VERSION = "1.0.2"
 end

data/simple_text_extract.gemspec

@@ -24,12 +24,14 @@ Gem::Specification.new do |spec|
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
 
-  spec.requirements << "Antiword"
+  spec.requirements << "antiword"
   spec.requirements << "pdftotext/poppler"
   spec.required_ruby_version = ">= 2.5"
 
-  spec.add_development_dependency "bundler", "~> 1.17"
-  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_runtime_dependency "roo", "~> 2.8.2"
+  spec.add_runtime_dependency "spreadsheet", "~> 1.1.8"
+
+  spec.add_development_dependency "rake", "~> 13.0"
   spec.add_development_dependency "minitest", "~> 5.0"
   spec.add_development_dependency "mocha"
 end

metadata

@@ -1,43 +1,57 @@
 --- !ruby/object:Gem::Specification
 name: simple_text_extract
 version: !ruby/object:Gem::Version
-  version: 0.1.3
+  version: 1.0.2
 platform: ruby
 authors:
 - Nick Weiland
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2019-01-24 00:00:00.000000000 Z
+date: 2020-07-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: bundler
+  name: roo
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.17'
-  type: :development
+        version: 2.8.2
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.8.2
+- !ruby/object:Gem::Dependency
+  name: spreadsheet
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.1.8
+  type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.17'
+        version: 1.1.8
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '13.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '13.0'
 - !ruby/object:Gem::Dependency
   name: minitest
   requirement: !ruby/object:Gem::Requirement
@@ -86,15 +100,14 @@ files:
 - bin/console
 - bin/setup
 - lib/simple_text_extract.rb
-- lib/simple_text_extract/file_extractor.rb
 - lib/simple_text_extract/format_extractor/base.rb
 - lib/simple_text_extract/format_extractor/doc.rb
 - lib/simple_text_extract/format_extractor/doc_x.rb
 - lib/simple_text_extract/format_extractor/pdf.rb
 - lib/simple_text_extract/format_extractor/plain_text.rb
+- lib/simple_text_extract/format_extractor/xls.rb
 - lib/simple_text_extract/format_extractor/xls_x.rb
 - lib/simple_text_extract/format_extractor_factory.rb
-- lib/simple_text_extract/tempfile_extractor.rb
 - lib/simple_text_extract/text_extractor.rb
 - lib/simple_text_extract/version.rb
 - simple_text_extract.gemspec
@@ -118,10 +131,9 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements:
-- Antiword
+- antiword
 - pdftotext/poppler
-rubyforge_project: 
-rubygems_version: 2.7.6
+rubygems_version: 3.0.3
 signing_key: 
 specification_version: 4
 summary: Attempts to quickly extract text from various file types before resorting

data/lib/simple_text_extract/file_extractor.rb

@@ -1,17 +0,0 @@
-# frozen_string_literal: true
-
-module SimpleTextExtract
-  class FileExtractor < TextExtractor
-    attr_reader :filepath
-
-    def initialize(filepath:)
-      @filepath = filepath
-    end
-
-    private
-
-      def file
-        @file ||= File.new(filepath)
-      end
-  end
-end

data/lib/simple_text_extract/tempfile_extractor.rb

@@ -1,34 +0,0 @@
-# frozen_string_literal: true
-
-module SimpleTextExtract
-  class TempfileExtractor < TextExtractor
-    attr_reader :filename, :raw
-
-    def initialize(filename:, raw:)
-      @filename = filename
-      @raw = String.new(raw, encoding: Encoding::UTF_8)
-
-      write_raw
-    end
-
-    private
-
-      def file
-        @file ||= Tempfile.new(filepath)
-      end
-
-      def write_raw
-        file.write(raw)
-        file.rewind
-      end
-
-      def cleanup
-        file.close
-        file.unlink
-      end
-
-      def filepath
-        @filepath ||= filename.split(".").yield_self { |parts| [parts[0], ".#{parts[1]}"] }
-      end
-  end
-end