simple_text_extract 0.2.1 → 1.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 83da9d28803f321b9a13aeaad4972211d40733b96f6b5fd085e52ab293a19d30
-  data.tar.gz: 99769610f1adef1d8fbe46647c7253af7859029a362854f4c8d73ec45fa9d8da
+  metadata.gz: 93414a5f260468367e16054bfcbcebbd93962c90a2d6903622a622aa567f754d
+  data.tar.gz: af3fc14fcc1b3c532f5f54c7dfebdb8d4fd4da5de691fe4561b5e39bd6c04fe0
 SHA512:
-  metadata.gz: 6f8dc568cf35fe6519d24dfc9a97a2b3c4d68770d5d489a1a1c4f813307ff7cc2fb973a663656893b448fb2532198f36373827ef202887edb1ad73b0ef53d3e7
-  data.tar.gz: d334282c216656d91cb038d020e4c1da67ca563b708bf4356f0005ff8d1ec2f1dae1ea58c5427828a2a593b2ef238325caab4fdbdf9d3575ca8ca5e14b1791ca
+  metadata.gz: 1219e1404ea893772da0905b7bd808f703240cdc9e7f875a287c4e7886e67ea8fbc4c07a2a55a95a8db3f828d7f7ad87a67832e7834618c62537df3d42e013da
+  data.tar.gz: c052e8f2640f8aa9ac452b9cfcad32695add2c1b374daf4dee712f34592a9d1decaae453dbeff1f473fdcd12df01f1a1f9e7751a03925514e91976bb9d654f10

data/.ruby-version

@@ -1 +1 @@
-2.5.3
+3.0.0

data/Gemfile

@@ -4,6 +4,4 @@ source "https://rubygems.org"
 
 git_source(:github) { |repo_name| "https://github.com/#{repo_name}" }
 
-gem "pry"
-
 gemspec

data/Gemfile.lock

@@ -1,44 +1,38 @@
 PATH
   remote: .
   specs:
-    simple_text_extract (0.2.0)
-      roo (~> 2.8)
+    simple_text_extract (1.2.0)
+      roo (~> 2.8.2)
+      rubyzip (>= 1.0.0)
       spreadsheet (~> 1.1.8)
 
 GEM
   remote: https://rubygems.org/
   specs:
-    coderay (1.1.2)
-    metaclass (0.0.4)
-    method_source (0.9.2)
-    mini_portile2 (2.4.0)
-    minitest (5.11.3)
-    mocha (1.8.0)
-      metaclass (~> 0.0.1)
-    nokogiri (1.10.1)
-      mini_portile2 (~> 2.4.0)
-    pry (0.12.2)
-      coderay (~> 1.1.0)
-      method_source (~> 0.9.0)
-    rake (10.5.0)
-    roo (2.8.1)
+    mini_portile2 (2.5.1)
+    minitest (5.14.4)
+    mocha (1.12.0)
+    nokogiri (1.11.6)
+      mini_portile2 (~> 2.5.0)
+      racc (~> 1.4)
+    racc (1.5.2)
+    rake (13.0.3)
+    roo (2.8.3)
       nokogiri (~> 1)
-      rubyzip (>= 1.2.1, < 2.0.0)
-    ruby-ole (1.2.12.1)
-    rubyzip (1.2.2)
-    spreadsheet (1.1.8)
+      rubyzip (>= 1.3.0, < 3.0.0)
+    ruby-ole (1.2.12.2)
+    rubyzip (2.3.0)
+    spreadsheet (1.1.9)
       ruby-ole (>= 1.0)
 
 PLATFORMS
   ruby
 
 DEPENDENCIES
-  bundler (~> 1.17)
   minitest (~> 5.0)
   mocha
-  pry
-  rake (~> 10.0)
+  rake (~> 13.0)
   simple_text_extract!
 
 BUNDLED WITH
-   1.17.2
+   2.2.3

data/README.md

@@ -9,6 +9,7 @@ SimpleTextExtract handles parsing text from:
 - `.doc`
 - `.xlsx`
 - `.xls`
+- `.csv`
 - `.txt` 😜
 
 If no text is parsed (for `pdf`), or a file format is not supported (like images), then `nil` is returned and you can move on to the heavy-duty tools like [Henkei](https://github.com/abrom/henkei) 💪.
@@ -34,11 +35,13 @@ Or install it yourself as:
 Text can be parsed from raw file content or files in the filesystem t by calling `SimpleTextExtract.extract`:
 
 ```ruby
-# raw file content using ActiveStorage
-SimpleTextExtract.extract(filename: attachment.blob.filename, raw: attachment.download)
+# using ActiveStorage >= 6
+extract = attachment.open { |tmp| SimpleTextExtract.extract(tempfile: tmp) }
+# raw file content or when ActiveStorage < 6
+extract = SimpleTextExtract.extract(filename: attachment.blob.filename, raw: attachment.download)
 
 # filesystem
-SimpleTextExtract.extract(filepath: "path_to_file.pdf")
+extract = SimpleTextExtract.extract(filepath: "path_to_file.pdf")
 ```
 
 ### Usage Dependencies
@@ -48,7 +51,7 @@ You can choose to use SimpleTextExtract without the following dependencies, but
 `pdf` parsing requires `poppler-utils`
 - `brew install poppler`
 
-`doc` parsing requires `antiword`
+`doc` parsing requires `antiword` and `unzip`
 - `brew install antiword`
 
 `xlsx` and `xls` parsing requires `ssconvert` which is part of `gnumeric`
@@ -73,6 +76,7 @@ In your `Aptfile`, add:
 ```
 antiword
 gnumeric
+unzip
 ```
 
 * There is currently an [issue](https://github.com/heroku/heroku-buildpack-google-chrome/issues/59) with the heroku-18 stack that requires additional dependencies added to the Aptfile to get `gnumeric` to work properly.  You can reference the linked issue above to figure out those dependencies, or downgrade to heroku-16 until it is fixed.

data/Rakefile

@@ -7,4 +7,4 @@ Rake::TestTask.new(:test) do |t|
   t.test_files = FileList["test/**/*_test.rb"]
 end
 
-task :default => :test
+task default: :test

data/bin/console

@@ -6,8 +6,5 @@ require "simple_text_extract"
 # You can add fixtures and/or initialization code here to make experimenting
 # with your gem easier. You can also use a different console, if you like.
 
-require "pry"
-Pry.start
-
 require "irb"
 IRB.start(__FILE__)

data/lib/simple_text_extract.rb

@@ -2,20 +2,18 @@
 
 require "simple_text_extract/version"
 require "simple_text_extract/text_extractor"
-require "simple_text_extract/file_extractor"
-require "simple_text_extract/tempfile_extractor"
 require "simple_text_extract/format_extractor_factory"
 
 module SimpleTextExtract
-  SUPPORTED_FILETYPES = ["xls", "xlsx", "doc", "docx", "txt", "pdf"]
+  SUPPORTED_FILETYPES = ["xls", "xlsx", "doc", "docx", "txt", "pdf", "csv", "zip"].freeze
 
   class Error < StandardError; end
 
-  def self.extract(filename: nil, raw: nil, filepath: nil)
-    TextExtractor.call(filename: filename, raw: raw, filepath: filepath).to_s
+  def self.extract(filename: nil, raw: nil, filepath: nil, tempfile: nil)
+    TextExtractor.new(filename: filename, raw: raw, filepath: filepath, tempfile: tempfile).to_s
   end
 
   def self.supports?(filename: nil)
-    SUPPORTED_FILETYPES.include?(filename.split(".")[1])
+    SUPPORTED_FILETYPES.include?(filename.split(".").last)
   end
 end

data/lib/simple_text_extract/format_extractor/doc.rb

@@ -4,7 +4,7 @@ module SimpleTextExtract
   module FormatExtractor
     class Doc < Base
       def extract
-        return nil if missing_dependency?('antiword')
+        return nil if missing_dependency?("antiword")
 
         `antiword #{Shellwords.escape(file.path)}`
       end

data/lib/simple_text_extract/format_extractor/xls.rb

@@ -8,7 +8,6 @@ module SimpleTextExtract
 
         spreadsheet = Spreadsheet.open(file)
         text = []
-
         spreadsheet.worksheets.each do |sheet|
           text << sheet.name
           text << sheet.rows

data/lib/simple_text_extract/format_extractor/xls_x.rb

@@ -6,7 +6,7 @@ module SimpleTextExtract
       def extract
         require "roo"
 
-        spreadsheet = Roo::Spreadsheet.open(file)
+        spreadsheet = Roo::Spreadsheet.open(file, only_visible_sheets: true)
 
         text = []
 

data/lib/simple_text_extract/format_extractor/zip_extract.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+module SimpleTextExtract
+  module FormatExtractor
+    class ZipExtract < Base
+      def extract
+        require "zip"
+
+        result = []
+        Zip::File.open(file) do |zip_file|
+          zip_file.each do |entry|
+            result << entry.name
+            result << SimpleTextExtract.extract(
+              raw: entry.get_input_stream.read,
+              filename: entry.name
+            )
+          end
+        end
+
+        result.join(" ")
+      end
+    end
+  end
+end

data/lib/simple_text_extract/format_extractor_factory.rb

@@ -7,12 +7,15 @@ require "simple_text_extract/format_extractor/xls_x"
 require "simple_text_extract/format_extractor/xls"
 require "simple_text_extract/format_extractor/doc_x"
 require "simple_text_extract/format_extractor/doc"
+require "simple_text_extract/format_extractor/zip_extract"
 
 module SimpleTextExtract
   class FormatExtractorFactory
-    def self.call(file) # rubocop:disable Metrics/MethodLength
+    def self.call(file)
       case file.path
-      when /.txt$/i
+      when /.zip$/i
+        FormatExtractor::ZipExtract.new(file)
+      when /(.txt$|.csv$)/i
         FormatExtractor::PlainText.new(file)
       when /.pdf$/i
         FormatExtractor::PDF.new(file)

data/lib/simple_text_extract/text_extractor.rb

@@ -2,24 +2,54 @@
 
 module SimpleTextExtract
   class TextExtractor
-    def self.call(filename: nil, raw: nil, filepath: nil)
-      if !filename.nil? && !raw.nil?
-        TempfileExtractor.new(filename: filename.to_s, raw: raw).extract
-      elsif !filepath.nil? && File.exist?(filepath)
-        FileExtractor.new(filepath: filepath).extract
-      end
-    end
+    attr_reader :file
 
-    def extract
-      text = FormatExtractorFactory.call(file).extract
-      cleanup
+    def initialize(filename: nil, raw: nil, filepath: nil, tempfile: nil)
+      @file = get_file(filename: filename, raw: raw, filepath: filepath, tempfile: tempfile)
+    end
 
-      text
+    def to_s
+      @to_s ||= extract.to_s
     end
 
     private
 
+      def get_file(filename:, raw:, filepath:, tempfile:)
+        if tempfile&.class == Tempfile
+          tempfile
+        elsif !filename.nil? && !raw.nil?
+          write_tempfile(filename: filename.to_s, raw: raw)
+        elsif !filepath.nil? && File.exist?(filepath)
+          File.new(filepath)
+        end
+      end
+
+      def extract
+        return unless file
+
+        begin
+          FormatExtractorFactory.call(file).extract
+        rescue StandardError
+          nil
+        ensure
+          cleanup
+        end
+      end
+
       def cleanup
+        return unless file.instance_of?(Tempfile)
+
+        file.close
+        file.unlink
+      end
+
+      def write_tempfile(filename:, raw:)
+        filename = filename.split(".").yield_self { |parts| [parts[0], ".#{parts[1]}"] }
+        file = Tempfile.new(filename)
+        raw = String.new(raw, encoding: Encoding::UTF_8)
+
+        file.write(raw)
+        file.tap(&:rewind)
       end
   end
 end

data/lib/simple_text_extract/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module SimpleTextExtract
-  VERSION = "0.2.1"
+  VERSION = "1.2.0"
 end

data/simple_text_extract.gemspec

@@ -28,11 +28,11 @@ Gem::Specification.new do |spec|
   spec.requirements << "pdftotext/poppler"
   spec.required_ruby_version = ">= 2.5"
 
-  spec.add_runtime_dependency "roo", "~> 2.8"
+  spec.add_runtime_dependency "roo", "~> 2.8.2"
   spec.add_runtime_dependency "spreadsheet", "~> 1.1.8"
+  spec.add_runtime_dependency "rubyzip", ">= 1.0.0"
 
-  spec.add_development_dependency "bundler", "~> 1.17"
-  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "rake", "~> 13.0"
   spec.add_development_dependency "minitest", "~> 5.0"
   spec.add_development_dependency "mocha"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: simple_text_extract
 version: !ruby/object:Gem::Version
-  version: 0.2.1
+  version: 1.2.0
 platform: ruby
 authors:
 - Nick Weiland
-autorequire: 
+autorequire:
 bindir: exe
 cert_chain: []
-date: 2019-01-28 00:00:00.000000000 Z
+date: 2021-05-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: roo
@@ -16,14 +16,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.8'
+        version: 2.8.2
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.8'
+        version: 2.8.2
 - !ruby/object:Gem::Dependency
   name: spreadsheet
   requirement: !ruby/object:Gem::Requirement
@@ -39,33 +39,33 @@ dependencies:
       - !ruby/object:Gem::Version
         version: 1.1.8
 - !ruby/object:Gem::Dependency
-  name: bundler
+  name: rubyzip
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1.17'
-  type: :development
+        version: 1.0.0
+  type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1.17'
+        version: 1.0.0
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '13.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '13.0'
 - !ruby/object:Gem::Dependency
   name: minitest
   requirement: !ruby/object:Gem::Requirement
@@ -114,7 +114,6 @@ files:
 - bin/console
 - bin/setup
 - lib/simple_text_extract.rb
-- lib/simple_text_extract/file_extractor.rb
 - lib/simple_text_extract/format_extractor/base.rb
 - lib/simple_text_extract/format_extractor/doc.rb
 - lib/simple_text_extract/format_extractor/doc_x.rb
@@ -122,17 +121,18 @@ files:
 - lib/simple_text_extract/format_extractor/plain_text.rb
 - lib/simple_text_extract/format_extractor/xls.rb
 - lib/simple_text_extract/format_extractor/xls_x.rb
+- lib/simple_text_extract/format_extractor/zip_extract.rb
 - lib/simple_text_extract/format_extractor_factory.rb
-- lib/simple_text_extract/tempfile_extractor.rb
 - lib/simple_text_extract/text_extractor.rb
 - lib/simple_text_extract/version.rb
+- simple_text_extract-1.0.2.gem
 - simple_text_extract.gemspec
 - tags
 homepage: https://github.com/weilandia/simple_text_extract
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -149,9 +149,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
 requirements:
 - antiword
 - pdftotext/poppler
-rubyforge_project: 
-rubygems_version: 2.7.6
-signing_key: 
+rubygems_version: 3.2.3
+signing_key:
 specification_version: 4
 summary: Attempts to quickly extract text from various file types before resorting
   to something more extreme like Apache Tika.

data/lib/simple_text_extract/file_extractor.rb

@@ -1,17 +0,0 @@
-# frozen_string_literal: true
-
-module SimpleTextExtract
-  class FileExtractor < TextExtractor
-    attr_reader :filepath
-
-    def initialize(filepath:)
-      @filepath = filepath
-    end
-
-    private
-
-      def file
-        @file ||= File.new(filepath)
-      end
-  end
-end

data/lib/simple_text_extract/tempfile_extractor.rb

@@ -1,34 +0,0 @@
-# frozen_string_literal: true
-
-module SimpleTextExtract
-  class TempfileExtractor < TextExtractor
-    attr_reader :filename, :raw
-
-    def initialize(filename:, raw:)
-      @filename = filename
-      @raw = String.new(raw, encoding: Encoding::UTF_8)
-
-      write_raw
-    end
-
-    private
-
-      def file
-        @file ||= Tempfile.new(filepath)
-      end
-
-      def write_raw
-        file.write(raw)
-        file.rewind
-      end
-
-      def cleanup
-        file.close
-        file.unlink
-      end
-
-      def filepath
-        @filepath ||= filename.split(".").yield_self { |parts| [parts[0], ".#{parts[1]}"] }
-      end
-  end
-end