henkei 1.27.1 → 2.2.0.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7f17479851871389aaeaa73798025c62d10cabc40a6ac703c2b25385258ae179
-  data.tar.gz: 3e1ff327a0ff55ca55b8ce2d44ffce9bd493dcf2894129ecfc143b965d765e84
+  metadata.gz: 64f2ec330c97bf77b16e2f7e14e08f90c405ae42c7aeddce733c0d889eeb4782
+  data.tar.gz: c3b3b91c569c7093bf22a5c751153426f1fdaa62da61f8c40f8b8cabc6ce072c
 SHA512:
-  metadata.gz: ab3a4254edd84f35c990013f684b1f3fbfb98f19c8d6e74aa3f27ffd4e3280c8e84b1679508b751af5178f3cd827cc5fb717d5d8819819fcc1cf77057272bae5
-  data.tar.gz: 8c6d9fcc938653550877fdce23de4a15d0427e41de023a113fd27c3e0bf213765df445b9d14e45496302148a1dc8a7120591505ed68e55b50f4927bdcafb940b
+  metadata.gz: b9f2263c057cb9e958039930aa3c0244c31b5bcc5e636515a757e3b1c5d43a6b52c932a76a8c132af73707453df1e2652e6f02d9926276a75bceb4deb69ffa59
+  data.tar.gz: f8ea0d87ad2bc75213483824a30edba37507110fc721434f0a40477500dc4674d8ce8d1c2a47b06624249b48228bbe9c9a9f9c86bb19e5cb2dcc18a8dec23dbe

data/README.md

@@ -21,6 +21,15 @@ Here are some of the formats supported:
 For the complete list of supported formats, please visit the Apache Tika
 [Supported Document Formats](http://tika.apache.org/0.9/formats.html) page.
 
+## Upgrading from v1.x to v2.x
+
+Apache Tika v2.x brings with it some changes. One key change is that the Tika client and server applications have
+been split up. To keep the gem size down Henkei will only include the client app. That is to say, each time you
+call to Henkei, a new Java process will be started, run your command, then terminate.
+
+Another change is the metadata keys. A lot of duplicate keys have been removed in favour of a more standards
+based approach. A list of the old vs new key names can be found [here](https://cwiki.apache.org/confluence/display/TIKA/Migrating+to+Tika+2.0.0#MigratingtoTika2.0.0-Metadata) 
+
 ## Usage
 
 Text, metadata and MIME type information can be extracted by calling `Henkei.read` directly:
@@ -69,6 +78,20 @@ post '/:name/:filename' do
 end
 ```
 
+### Reading text from inside images (OCR)
+
+You can enable OCR by specifying the optional `include_ocr: true` when calling to the `text` or `html` instance methods,
+as well as the `read` class method. Note that Tika does indicate this will greatly increase processing time.
+
+```ruby
+henkei = Henkei.new 'sample.pages'
+text_with_ocr = henkei.text(include_ocr: true)
+html_with_ocr = henkei.html(include_ocr: true)
+
+data = File.read 'sample.pages'
+text_with_ocr = Henkei.read :text, data, include_ocr: true
+```
+
 ### Reading metadata
 
 Metadata is returned as a hash.

data/henkei.gemspec

@@ -26,6 +26,7 @@ Gem::Specification.new do |spec|
   spec.add_runtime_dependency 'mini_mime', '>= 0.1.1', '< 2'
 
   spec.add_development_dependency 'bundler', '~> 2.0'
+  spec.add_development_dependency 'nokogiri', '~> 1.12'
   spec.add_development_dependency 'rails', '~> 5.0'
   spec.add_development_dependency 'rake', '~> 12.3'
   spec.add_development_dependency 'rspec', '~> 3.7'

data/jar/tika-config-without-ocr.xml

@@ -0,0 +1,9 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<properties>
+  <service-loader initializableProblemHandler="ignore"/>
+  <parsers>
+    <parser class="org.apache.tika.parser.DefaultParser">
+      <parser-exclude class="org.apache.tika.parser.ocr.TesseractOCRParser"/>
+    </parser>
+  </parsers>
+</properties>

data/jar/tika-config.xml

@@ -1,3 +1,4 @@
+<?xml version="1.0" encoding="UTF-8"?>
 <properties>
   <service-loader initializableProblemHandler="ignore"/>
 </properties>

data/lib/henkei/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 class Henkei
-  VERSION = '1.27.1'
+  VERSION = '2.2.0.1'
 end

data/lib/henkei.rb

@@ -25,12 +25,9 @@ require 'open3'
 # Read text and metadata from files and documents using Apache Tika toolkit
 class Henkei # rubocop:disable Metrics/ClassLength
   GEM_PATH = File.dirname(File.dirname(__FILE__))
-  JAR_PATH = File.join(Henkei::GEM_PATH, 'jar', 'tika-app-1.27.jar')
+  JAR_PATH = File.join(Henkei::GEM_PATH, 'jar', 'tika-app-2.2.0.jar')
   CONFIG_PATH = File.join(Henkei::GEM_PATH, 'jar', 'tika-config.xml')
-  DEFAULT_SERVER_PORT = 9293 # an arbitrary, but perfectly cromulent, port
-
-  @@server_port = nil
-  @@server_pid = nil
+  CONFIG_WITHOUT_OCR_PATH = File.join(Henkei::GEM_PATH, 'jar', 'tika-config-without-ocr.xml')
 
   def self.mimetype(content_type)
     if Henkei.configuration.mime_library == 'mime/types' && defined?(MIME::Types)
@@ -50,8 +47,8 @@ class Henkei # rubocop:disable Metrics/ClassLength
   #   text = Henkei.read :text, data
   #   metadata = Henkei.read :metadata, data
   #
-  def self.read(type, data)
-    result = @@server_pid ? server_read(data) : client_read(type, data)
+  def self.read(type, data, include_ocr: false)
+    result = client_read(type, data, include_ocr: include_ocr)
 
     case type
     when :text then result
@@ -96,10 +93,14 @@ class Henkei # rubocop:disable Metrics/ClassLength
   #   henkei = Henkei.new 'sample.pages'
   #   henkei.text
   #
-  def text
+  # Include OCR results from images (includes embedded images in pages/docx/pdf etc)
+  #
+  #   henkei.text(include_ocr: true)
+  #
+  def text(include_ocr: false)
     return @text if defined? @text
 
-    @text = Henkei.read :text, data
+    @text = Henkei.read :text, data, include_ocr: include_ocr
   end
 
   # Returns the text content of the Henkei document in HTML.
@@ -107,10 +108,14 @@ class Henkei # rubocop:disable Metrics/ClassLength
   #   henkei = Henkei.new 'sample.pages'
   #   henkei.html
   #
-  def html
+  # Include OCR results from images (includes embedded images in pages/docx/pdf etc)
+  #
+  #   henkei.html(include_ocr: true)
+  #
+  def html(include_ocr: false)
     return @html if defined? @html
 
-    @html = Henkei.read :html, data
+    @html = Henkei.read :html, data, include_ocr: include_ocr
   end
 
   # Returns the metadata hash of the Henkei document.
@@ -144,9 +149,9 @@ class Henkei # rubocop:disable Metrics/ClassLength
   #
   def creation_date
     return @creation_date if defined? @creation_date
-    return unless metadata['Creation-Date']
+    return unless metadata['dcterms:created']
 
-    @creation_date = Time.parse(metadata['Creation-Date'])
+    @creation_date = Time.parse(metadata['dcterms:created'])
   end
 
   # Returns +true+ if the Henkei document was specified using a file path.
@@ -196,44 +201,6 @@ class Henkei # rubocop:disable Metrics/ClassLength
     @data
   end
 
-  # Returns pid of Tika server, started as a new spawned process.
-  #
-  #  type :html, :text or :metadata
-  #  custom_port e.g. 9293
-  #
-  #  Henkei.server(:text, 9294)
-  #
-  def self.server(type, custom_port = nil)
-    @@server_port = custom_port || DEFAULT_SERVER_PORT
-
-    @@server_pid = Process.spawn(*tika_command(type, server: true))
-    sleep(2) # Give the server 2 seconds to spin up.
-    @@server_pid
-  end
-
-  # Kills server started by Henkei.server
-  #
-  #  Always run this when you're done, or else Tika might run until you kill it manually
-  #  You might try putting your extraction in a begin..rescue...ensure...end block and
-  #    putting this method in the ensure block.
-  #
-  #  Henkei.server(:text)
-  #  reports = ["report1.docx", "report2.doc", "report3.pdf"]
-  #  begin
-  #    my_texts = reports.map{ |report_path| Henkei.new(report_path).text }
-  #  rescue
-  #  ensure
-  #    Henkei.kill_server!
-  #  end
-  #
-  def self.kill_server!
-    return unless @@server_pid
-
-    Process.kill('INT', @@server_pid)
-    @@server_pid = nil
-    @@server_port = nil
-  end
-
   ### Private class methods
 
   # Provide the path to the Java binary
@@ -245,44 +212,21 @@ class Henkei # rubocop:disable Metrics/ClassLength
 
   # Internal helper for calling to Tika library directly
   #
-  def self.client_read(type, data)
-    Open3.capture2(*tika_command(type), stdin_data: data, binmode: true).first
+  def self.client_read(type, data, include_ocr: false)
+    Open3.capture2(*tika_command(type, include_ocr: include_ocr), stdin_data: data, binmode: true).first
   end
   private_class_method :client_read
 
-  # Internal helper for calling to running Tika server
-  #
-  def self.server_read(data)
-    s = TCPSocket.new('localhost', @@server_port)
-    file = StringIO.new(data, 'r')
-
-    loop do
-      chunk = file.read(65_536)
-      break unless chunk
-
-      s.write(chunk)
-    end
-
-    # tell Tika that we're done sending data
-    s.shutdown(Socket::SHUT_WR)
-
-    resp = String.new ''
-    loop do
-      chunk = s.recv(65_536)
-      break if chunk.empty? || !chunk
-
-      resp << chunk
-    end
-    resp
-  end
-  private_class_method :server_read
-
   # Internal helper for building the Java command to call Tika
   #
-  def self.tika_command(type, server: false)
-    command = [java_path, '-Djava.awt.headless=true', '-jar', Henkei::JAR_PATH, "--config=#{Henkei::CONFIG_PATH}"]
-    command += ['--server', '--port', @@server_port.to_s] if server
-    command + switch_for_type(type)
+  def self.tika_command(type, include_ocr: false)
+    [
+      java_path,
+      '-Djava.awt.headless=true',
+      '-jar',
+      Henkei::JAR_PATH,
+      "--config=#{include_ocr ? Henkei::CONFIG_PATH : Henkei::CONFIG_WITHOUT_OCR_PATH}"
+    ] + switch_for_type(type)
   end
   private_class_method :tika_command
 

data/spec/henkei_spec.rb

@@ -2,10 +2,15 @@
 
 require 'helper'
 require 'henkei'
+require 'nokogiri'
 
 # Some of the tests have been known to fail in weird and wonderful ways when `rails` is included
 require 'rails' if ENV['INCLUDE_RAILS'] == 'true'
 
+def travis_ci?
+  ENV['CI'] == 'true' && ENV['TRAVIS'] == 'true'
+end
+
 describe Henkei do
   let(:data) { File.read 'spec/samples/sample.docx' }
 
@@ -52,6 +57,23 @@ describe Henkei do
 
         expect(text).to eq ''
       end
+
+      unless travis_ci?
+        context 'when `include_ocr` is enabled' do
+          it 'returns parsed plain text in the image' do
+            text = Henkei.read :text, data, include_ocr: true
+
+            expect(text).to include <<~TEXT
+              West Side
+    
+              Sea Island
+              PP
+    
+              Richmond
+            TEXT
+          end
+        end
+      end
     end
   end
 
@@ -115,6 +137,7 @@ describe Henkei do
 
   describe '.creation_date' do
     let(:henkei) { Henkei.new 'spec/samples/sample.pages' }
+
     it 'should return Time' do
       expect(henkei.creation_date).to be_a Time
     end
@@ -158,6 +181,30 @@ describe Henkei do
       it '#mimetype returns `image/png`' do
         expect(henkei.mimetype.content_type).to eq 'image/png'
       end
+
+      unless travis_ci?
+        context 'when `include_ocr` is enabled' do
+          it '#text returns plain text of parsed text in the image' do
+            expect(henkei.text(include_ocr: true)).to include <<~TEXT
+              West Side
+    
+              Sea Island
+              PP
+    
+              Richmond
+            TEXT
+          end
+
+          it '#html returns HTML of parsed text in the image' do
+            expect(henkei.html(include_ocr: true)).to include '<meta name="tiff:ImageWidth" content="792"/>'
+
+            html_body = Nokogiri::HTML(henkei.html(include_ocr: true)).at_xpath('//body')
+            ['Anmore', 'Coquitlam', 'West Side', 'Sea Island', 'Richmond', 'Steveston'].each do |location|
+              expect(html_body.text).to include location
+            end
+          end
+        end
+      end
     end
   end
 
@@ -198,40 +245,4 @@ describe Henkei do
       expect(henkei.metadata['Content-Type']).to eq 'application/pdf'
     end
   end
-
-  context 'working as server mode' do
-    specify '#starts and kills server' do
-      begin
-        Henkei.server(:text)
-        expect(Henkei.class_variable_get(:@@server_pid)).not_to be_nil
-        expect(Henkei.class_variable_get(:@@server_port)).not_to be_nil
-
-        s = TCPSocket.new('localhost', Henkei.class_variable_get(:@@server_port))
-        expect(s).to be_a TCPSocket
-        s.close
-      ensure
-        port = Henkei.class_variable_get(:@@server_port)
-        Henkei.kill_server!
-        sleep 2
-        expect { TCPSocket.new('localhost', port) }.to raise_error Errno::ECONNREFUSED
-      end
-    end
-
-    specify '#runs samples through server mode' do
-      begin
-        Henkei.server(:text)
-        expect(Henkei.new('spec/samples/sample.pages').text).to(
-          include 'The quick brown fox jumped over the lazy cat.'
-        )
-        expect(Henkei.new('spec/samples/sample filename with spaces.pages').text).to(
-          include 'The quick brown fox jumped over the lazy cat.'
-        )
-        expect(Henkei.new('spec/samples/sample.docx').text).to(
-          include 'The quick brown fox jumped over the lazy cat.'
-        )
-      ensure
-        Henkei.kill_server!
-      end
-    end
-  end
 end

metadata

@@ -1,15 +1,15 @@
 --- !ruby/object:Gem::Specification
 name: henkei
 version: !ruby/object:Gem::Version
-  version: 1.27.1
+  version: 2.2.0.1
 platform: ruby
 authors:
 - Erol Fornoles
 - Andrew Bromwich
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-07-19 00:00:00.000000000 Z
+date: 2021-12-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: json
@@ -65,6 +65,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '2.0'
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.12'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.12'
 - !ruby/object:Gem::Dependency
   name: rails
   requirement: !ruby/object:Gem::Requirement
@@ -155,7 +169,8 @@ files:
 - Rakefile
 - bin/console
 - henkei.gemspec
-- jar/tika-app-1.27.jar
+- jar/tika-app-2.2.0.jar
+- jar/tika-config-without-ocr.xml
 - jar/tika-config.xml
 - lib/henkei.rb
 - lib/henkei/configuration.rb
@@ -172,7 +187,7 @@ homepage: http://github.com/abrom/henkei
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -190,8 +205,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.6
-signing_key: 
+rubygems_version: 3.0.9
+signing_key:
 specification_version: 4
 summary: Read text and metadata from files and documents (.doc, .docx, .pages, .odt,
   .rtf, .pdf) using Apache Tika toolkit