slaw 0.5.0 → 0.5.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 5d293949a3ac2383cd254f0efd2268ef24286f1d
-  data.tar.gz: 2439db865559b2f9b494b7755004d3f7a7d9b4ba
+  metadata.gz: 26e7c12c3e421410a6be3b18c19f589915e920fb
+  data.tar.gz: e8c1e77994e30c194b42bcca11bbc3b4eace6f76
 SHA512:
-  metadata.gz: 2a7af42dc109723ea0908e8bb3ba9cbf4269f923dbd8a68e6cabd87f53e98dec59884cb7d4daf0df8de98b3d209302a9ba214e45d04fadffd372fabf91c9db26
-  data.tar.gz: 41bac579c874b9ef29aaf5a87e243fd9ba268efe46c60e94cdc728e0e4d0f01b37b60b40fb0fa80a82889a43ac44174c6d8385386fb72a98c37dd0c5868c3b97
+  metadata.gz: 3e229100f80879b9646135b9c9f6fab2c543a20cd89ed6b021a74164fb92874b876451571bc9552d3150a2953a00be8d927b63c3e7ad780dec800751bff086c3
+  data.tar.gz: c8d2d73f08b67a2535816d9b5ae8f8abfb7cd2627f3aef1688a986bd4e6ba1e71a01cb1c53c0a5b66d9f836191abef6c89e8f43767a12d8b949d329e635c1043

data/bin/slaw

@@ -6,11 +6,15 @@ require 'slaw'
 class SlawCLI < Thor
   # TODO: support different grammars and locales
 
+  class_option :verbose, type: :boolean, desc: "Display log output on stderr"
+
   desc "parse FILE", "parse FILE into Akoma Ntoso XML"
   option :input, enum: ['text', 'pdf'], desc: "Type of input if it can't be determined automatically"
   option :pdftotext, desc: "Location of the pdftotext binary if not in PATH"
   option :definitions, type: :boolean, desc: "Find and link definitions (this can be slow). Default: false"
   def parse(name)
+    logging
+
     Slaw::Extract::Extractor.pdftotext_path = options[:pdftotext] if options[:pdftotext]
     extractor = Slaw::Extract::Extractor.new
 
@@ -31,6 +35,14 @@ class SlawCLI < Thor
 
     puts act.to_xml(indent: 2)
   end
+
+  no_commands do
+    def logging
+      logger = Log4r::Logger.new('Slaw')
+      logger.outputters = Log4r::Outputter.stderr
+      logger.outputters[0].level = options[:verbose] ? Log4r::DEBUG : Log4r::ERROR
+    end
+  end
 end
 
 SlawCLI.start(ARGV)

data/lib/slaw/extract/extractor.rb

@@ -30,8 +30,6 @@ module Slaw
       #
       # @return [String] extracted text
       def extract_from_file(filename)
-        ext = filename[-4..-1].downcase
-
         mimetype = get_mimetype(filename)
 
         case mimetype && mimetype.type
@@ -40,11 +38,11 @@ module Slaw
         when 'text/plain', nil
           extract_from_text(filename)
         else
-          if mimetype.text?
-            extract_from_text(filename)
-          else
-            raise ArgumentError.new("Unsupported file type #{ext} (#{mimetype || unknown})")
+          text = extract_via_tika(filename)
+          if text.empty? or text.nil?
+            raise ArgumentError.new("Unsupported file type #{mimetype || 'unknown'}")
           end
+          text
         end
       end
 
@@ -87,6 +85,20 @@ module Slaw
         cleanup(File.read(filename))
       end
 
+      # Extract text from +filename+ by sending it to apache tika
+      # http://tika.apache.org/
+      def extract_via_tika(filename)
+        # the Yomu gem falls over when trying to write large amounts of data
+        # the JVM stdin, so we manually call java ourselves, relying on yomu
+        # to supply the gem
+        require 'slaw/extract/yomu_patch'
+        logger.info("Using Tika to get text from #{filename}. You need a JVM installed for this.")
+
+        text = Yomu.text_from_file(filename)
+        logger.info("Tika returned #{text.length} bytes")
+        text
+      end
+
       # Run general once-off cleanup of extracted text.
       def cleanup(text)
         text = @cleanser.cleanup(text)

data/lib/slaw/extract/yomu_patch.rb

@@ -0,0 +1,9 @@
+require 'yomu'
+
+class Yomu
+  def self.text_from_file(filename)
+    IO.popen("#{java} -Djava.awt.headless=true -jar #{Yomu::JARPATH} -t '#{filename}'", 'r') do |io|
+      io.read
+    end
+  end
+end

data/lib/slaw/version.rb

@@ -1,3 +1,3 @@
 module Slaw
-  VERSION = "0.5.0"
+  VERSION = "0.5.1"
 end

data/slaw.gemspec

@@ -28,4 +28,5 @@ Gem::Specification.new do |spec|
   spec.add_runtime_dependency "log4r", "~> 1.1.10"
   spec.add_runtime_dependency "thor", "~> 0.19.1"
   spec.add_runtime_dependency "mimemagic", "~> 0.2.1"
+  spec.add_runtime_dependency 'yomu', '~> 0.2.2'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: slaw
 version: !ruby/object:Gem::Version
-  version: 0.5.0
+  version: 0.5.1
 platform: ruby
 authors:
 - Greg Kempe
@@ -136,6 +136,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 0.2.1
+- !ruby/object:Gem::Dependency
+  name: yomu
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.2.2
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.2.2
 description: Slaw is a lightweight library for rendering and generating Akoma Ntoso
   acts from plain text and PDF documents.
 email:
@@ -157,6 +171,7 @@ files:
 - lib/slaw/bylaw.rb
 - lib/slaw/collection.rb
 - lib/slaw/extract/extractor.rb
+- lib/slaw/extract/yomu_patch.rb
 - lib/slaw/generator.rb
 - lib/slaw/lifecycle_event.rb
 - lib/slaw/logging.rb