henkei 2.9.2.1 → 2.9.2.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: db0832d80d7445a7cfd495055c4b9081af1a6aa987adccfd162985dccd4bb2f2
-  data.tar.gz: 746bee81b787098e87b2c684a1cc33c48e3d34995b47c1d6848dd8e8f3d90991
+  metadata.gz: 3ea707f5995f2ff0c9036cb992b991e91ccf9f62b9298222cd81d24876678f96
+  data.tar.gz: 4dcd7c3394b3971a58ea11478147320388212bb67622602c795743a7b9d545f6
 SHA512:
-  metadata.gz: df61c967907e043f3ebfb1e60a76d097e6e565b1d0d0cebf71da021af295d8c3cfc0d897c7d49f5301cd1356b8e9a13ce9ec482174a1dbdad0aa3a021f267afe
-  data.tar.gz: 410971738b61f572796a652d260658025fe8c64c15813774a617485d98882af210daaf8fc91e64946a9844e9a61feb6e9c6f16042647b7cf83fe561f80163781
+  metadata.gz: 03a2c7e3be4f4065385351ddb1694ae9e5c53eae1ddfc8826a3b137868ded1c3a30efb95f515e97cf779fed09ef321565e4ad39c7c232cf315df3ba19c443625
+  data.tar.gz: cbccc2cad4ca82935aee2ffb2c60404264da5dc876f4a3cf05768fb70859dfebfbb436d2d62284e0d5b16d8b246920b0f38e7349d633cabb42facd8bd287ccca

data/README.md

@@ -28,7 +28,7 @@ been split up. To keep the gem size down Henkei will only include the client app
 call to Henkei, a new Java process will be started, run your command, then terminate.
 
 Another change is the metadata keys. A lot of duplicate keys have been removed in favour of a more standards
-based approach. A list of the old vs new key names can be found [here](https://cwiki.apache.org/confluence/display/TIKA/Migrating+to+Tika+2.0.0#MigratingtoTika2.0.0-Metadata) 
+based approach. A list of the old vs new key names can be found [here](https://cwiki.apache.org/confluence/display/TIKA/Migrating+to+Tika+2.0.0#MigratingtoTika2.0.0-Metadata)
 
 ## Usage
 
@@ -111,12 +111,26 @@ henkei.mimetype.content_type #=> "application/vnd.openxmlformats-officedocument.
 henkei.mimetype.extensions #=> ['docx']
 ```
 
+### Output text in a specific character encoding
+
+You can specify the output character encoding by passing in the optional `encoding` argument when calling to the
+`text` or `html` instance methods, as well as the `read` class method.
+
+```ruby
+henkei = Henkei.new 'sample.pages'
+utf_8_text = henkei.text(encoding: 'UTF-8')
+utf_16_html = henkei.html(encoding: 'UTF-16')
+
+data = File.read 'sample.pages'
+utf_32_text = Henkei.read :text, data, encoding: 'UTF-32'
+```
+
 ## Installation and Dependencies
 
 ### Java Runtime
 
 Henkei packages the Apache Tika application jar and requires a working JRE for it to work.
-Check that you either have the `JAVA_HOME` environment variable set, or that `java` is in your path. 
+Check that you either have the `JAVA_HOME` environment variable set, or that `java` is in your path.
 
 ### Gem
 
@@ -131,7 +145,7 @@ And then execute:
 Or install it yourself as:
 
     $ gem install henkei
-    
+
 ### Heroku
 
 Add the JVM Buildpack to your Heroku project:

data/lib/henkei/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 class Henkei
-  VERSION = '2.9.2.1'
+  VERSION = '2.9.2.2'
 end

data/lib/henkei.rb

@@ -47,8 +47,8 @@ class Henkei # rubocop:disable Metrics/ClassLength
   #   text = Henkei.read :text, data
   #   metadata = Henkei.read :metadata, data
   #
-  def self.read(type, data, include_ocr: false)
-    result = client_read(type, data, include_ocr: include_ocr)
+  def self.read(type, data, include_ocr: false, encoding: nil)
+    result = client_read(type, data, include_ocr: include_ocr, encoding: encoding)
 
     case type
     when :text, :html then result
@@ -96,10 +96,14 @@ class Henkei # rubocop:disable Metrics/ClassLength
   #
   #   henkei.text(include_ocr: true)
   #
-  def text(include_ocr: false)
+  # Set the output character encoding (e.g. 'UTF-8')
+  #
+  #   henkei.text(encoding: 'UTF-8')
+  #
+  def text(include_ocr: false, encoding: nil)
     return @text if defined? @text
 
-    @text = Henkei.read :text, data, include_ocr: include_ocr
+    @text = Henkei.read :text, data, include_ocr: include_ocr, encoding: encoding
   end
 
   # Returns the text content of the Henkei document in HTML.
@@ -111,10 +115,14 @@ class Henkei # rubocop:disable Metrics/ClassLength
   #
   #   henkei.html(include_ocr: true)
   #
-  def html(include_ocr: false)
+  # Set the output character encoding (e.g. 'UTF-8')
+  #
+  #   henkei.text(encoding: 'UTF-8')
+  #
+  def html(include_ocr: false, encoding: nil)
     return @html if defined? @html
 
-    @html = Henkei.read :html, data, include_ocr: include_ocr
+    @html = Henkei.read :html, data, include_ocr: include_ocr, encoding: encoding
   end
 
   # Returns the metadata hash of the Henkei document.
@@ -211,20 +219,37 @@ class Henkei # rubocop:disable Metrics/ClassLength
 
   # Internal helper for calling to Tika library directly
   #
-  def self.client_read(type, data, include_ocr: false)
-    Open3.capture2(*tika_command(type, include_ocr: include_ocr), stdin_data: data, binmode: true).first
+  def self.client_read(type, data, include_ocr: false, encoding: nil)
+    unless encoding.nil? || Encoding.name_list.include?(encoding)
+      raise ArgumentError, "unsupported encoding - #{encoding}"
+    end
+
+    Open3.popen2(*tika_command(type, include_ocr: include_ocr, encoding: encoding)) do |stdin, stdout|
+      stdin.binmode
+      stdout.binmode
+      stdout.set_encoding encoding unless encoding.nil?
+
+      out_reader = Thread.new { stdout.read }
+
+      write_data_to_stdin(data, stdin)
+
+      stdin.close
+
+      out_reader.value
+    end
   end
   private_class_method :client_read
 
   # Internal helper for building the Java command to call Tika
   #
-  def self.tika_command(type, include_ocr: false)
+  def self.tika_command(type, include_ocr: false, encoding: nil)
     [
       java_path,
       '-Djava.awt.headless=true',
       '-jar',
       Henkei::JAR_PATH,
-      "--config=#{include_ocr ? Henkei::CONFIG_PATH : Henkei::CONFIG_WITHOUT_OCR_PATH}"
+      "--config=#{include_ocr ? Henkei::CONFIG_PATH : Henkei::CONFIG_WITHOUT_OCR_PATH}",
+      *("--encoding=#{encoding}" unless encoding.nil?)
     ] + switch_for_type(type)
   end
   private_class_method :tika_command
@@ -240,4 +265,21 @@ class Henkei # rubocop:disable Metrics/ClassLength
     }[type]
   end
   private_class_method :switch_for_type
+
+  # Internal helper for writing the input data to stdin when calling Tika
+  #
+  def self.write_data_to_stdin(data, stdin)
+    return unless data
+
+    begin
+      if data.respond_to? :readpartial
+        IO.copy_stream(data, stdin)
+      else
+        stdin.write data
+      end
+    rescue Errno::EPIPE
+      # Catch broken pipe.
+    end
+  end
+  private_class_method :write_data_to_stdin
 end

data/spec/henkei_spec.rb

@@ -72,6 +72,26 @@ describe Henkei do
         end
       end
     end
+
+    context 'when a valid `encoding` value is provided' do
+      let(:encoding) { 'UTF-32' }
+
+      it 'returns the parsed text in the specified encoding' do
+        text = described_class.read :text, data, encoding: encoding
+
+        expect(text.encoding.name).to eq encoding
+      end
+    end
+
+    context 'when an invalid `encoding` value is provided' do
+      let(:encoding) { 'Beef' }
+
+      it 'raises an error' do
+        expect do
+          described_class.read :text, data, encoding: encoding
+        end.to raise_error(ArgumentError, "unsupported encoding - #{encoding}")
+      end
+    end
   end
 
   describe '.new' do

metadata

@@ -1,15 +1,15 @@
 --- !ruby/object:Gem::Specification
 name: henkei
 version: !ruby/object:Gem::Version
-  version: 2.9.2.1
+  version: 2.9.2.2
 platform: ruby
 authors:
 - Erol Fornoles
 - Andrew Bromwich
-autorequire:
+autorequire: 
 bindir: bin
 cert_chain: []
-date: 2024-05-01 00:00:00.000000000 Z
+date: 2024-06-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: json
@@ -91,7 +91,7 @@ licenses:
 metadata:
   allowed_push_host: https://rubygems.org
   rubygems_mfa_required: 'true'
-post_install_message:
+post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
@@ -109,8 +109,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.3.7
-signing_key:
+rubygems_version: 3.0.0
+signing_key: 
 specification_version: 4
 summary: Read text and metadata from files and documents (.doc, .docx, .pages, .odt,
   .rtf, .pdf) using Apache Tika toolkit