rika 1.11.1-java → 2.0.0-java

data/lib/rika/cli/rika_command.rb

@@ -0,0 +1,129 @@
+# frozen_string_literal: true
+
+require 'awesome_print'
+require 'optparse'
+require 'rika'
+require 'rika/formatters'
+require 'rika/cli/args_parser'
+
+# This command line application enables the parsing of documents on the command line.
+# Syntax is:
+#   rika [options] <file or url> [...file or url...]
+# Run with -h or --help option for more details.
+#
+# Defaults to outputting both content (text) and metadata,
+# but the -t and -m flags can be used to enable or suppress either.
+# Supports output formats of JSON, Pretty JSON, YAML, Awesome Print, to_s, and inspect (see Formatters class).
+class RikaCommand
+  attr_reader :args, :help_text, :metadata_formatter, :options, :targets, :text_formatter
+
+  # @param [Array<String>] args command line arguments; default to ARGV but may be overridden for testing
+  def initialize(args = ARGV)
+    # Dup the array in case it has been frozen. The array will be modified later when options are parsed
+    # and removed, and when directories are removed, so this array should not be frozen.
+    @args = args.dup
+  end
+
+  # Main method and entry point for this class' work.
+  def call
+    prepare
+    report_and_exit_if_no_targets_specified
+    if options[:as_array]
+      puts result_array_output
+    else
+      targets.each do |target|
+        result = Rika.parse(target, max_content_length: max_content_length, key_sort: options[:key_sort])
+        puts single_document_output(target, result)
+      end
+    end
+    nil
+  end
+
+  # Prepares to run the parse. This method is separate from #call so that it can be called from tests.
+  # @return [void]
+  private def prepare
+    @options, @targets, @help_text = ArgsParser.call(args)
+    set_output_formats
+  end
+
+  # Sets the output format(s) based on the command line options.
+  # Exits with error message if format is invalid.
+  # @return [void]
+  private def set_output_formats
+    format = options[:format]
+    @metadata_formatter = Rika::Formatters.get(format[0])
+    @text_formatter     = Rika::Formatters.get(format[1])
+    nil
+  rescue KeyError
+    $stderr.puts "Invalid format: #{format}\n\n"
+    $stderr.puts help_text
+    exit 1
+  end
+
+  # Converts a ParseResult to a hash containing the selected pieces of data.
+  # @param [ParseResult] result the parse result
+  # @return [Hash] the hash containing the selected pieces of data
+  private def result_hash(result)
+    h = {}
+    h['source']   = result.metadata['rika:data-source'] if options[:source]
+    h['metadata'] = result.metadata                     if options[:metadata]
+    h['text']     = result.content                      if options[:text]
+    h
+  end
+
+  # Builds the string representation of the result of parsing a single document
+  # @param [String] target the target document
+  # @param [ParseResult] result the parse result
+  # @return [String] the string representation of the result of parsing a single document
+  private def single_document_output(target, result)
+    if options[:metadata] && options[:text] && %w[jj JJ yy].include?(options[:format])
+      metadata_formatter.(result_hash(result))
+    else
+      sio = StringIO.new
+      sio << "Source: #{target}\n"                        if options[:source]
+      sio << metadata_formatter.(result.metadata) << "\n" if options[:metadata]
+      sio << text_formatter.(result.content) << "\n"      if options[:text]
+      sio.string
+    end
+  end
+
+  # Parses the documents and outputs the result of the parse to stdout as an array of hashes.
+  # Outputting as an array necessitates that the metadata and text formatters be the same
+  # (otherwise the output would be invalid, especially with JSON or YAML).
+  # Therefore, the metadata formatter is arbitrarily selected to be used by both.
+  # @return [String] the string representation of the result of parsing the documents
+  private def result_array_output
+    output_hashes = targets.map do |target|
+      result = Rika.parse(target, max_content_length: max_content_length, key_sort: options[:key_sort])
+      result_hash(result)
+    end
+
+    # Either the metadata or text formatter will do, since they will necessarily be the same formatter.
+    metadata_formatter.call(output_hashes)
+  end
+
+  # Tika offers a max_content_length option, but it is not exposed in Rika.
+  # Instead it is used only to enable or disable the entire text output.
+  private def max_content_length
+    options[:text] ? -1 : 0
+  end
+
+  # Prints message and help and exits if no targets are specified.
+  # The exit code is zero because this may not necessarily be an error, and we wouldn't want to
+  # be the cause of aborting a script. The documents specified as input to this command may be
+  # dynamically generated by a script, and the script may not want to abort if no documents are
+  # generated.
+  # @return [void] or exits
+  private def report_and_exit_if_no_targets_specified
+    if targets.empty?
+      $stderr.puts <<~MESSAGE
+
+        No targets specified.
+
+        #{help_text}
+      MESSAGE
+      exit 0
+    end
+    nil
+  end
+end

data/lib/rika/formatters.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+require 'json'
+require 'yaml'
+require 'awesome_print'
+
+module Rika
+  # This module manages the formatters used to format the output of the Rika command line application.
+  class Formatters
+    AWESOME_PRINT_FORMATTER = ->(object) { object.ai }
+    INSPECT_FORMATTER       = ->(object) { object.inspect }
+    JSON_FORMATTER          = ->(object) { object.to_json }
+    PRETTY_JSON_FORMATTER   = ->(object) { JSON.pretty_generate(object) }
+    TO_S_FORMATTER          = ->(object) { object.to_s }
+    YAML_FORMATTER          = ->(object) { object.to_yaml }
+
+    # A hash of formatters, keyed by the format character.
+    # The value is a lambda that takes the object to be formatted as a parameter.
+    # @return [Hash] the hash of formatters
+    FORMATTER_LOOKUP_TABLE = {
+      'a' => AWESOME_PRINT_FORMATTER,
+      'i' => INSPECT_FORMATTER,
+      'j' => JSON_FORMATTER,
+      'J' => PRETTY_JSON_FORMATTER,
+      't' => TO_S_FORMATTER,
+      'y' => YAML_FORMATTER
+    }.freeze
+
+    VALID_OPTION_CHARS = FORMATTER_LOOKUP_TABLE.keys
+
+    # Gets the formatter lambda for the given option character.
+    # @param [String] option_char the option character
+    # @return [Lambda] the formatter lambda
+    # @raise [KeyError] if any option character is invalid
+    def self.get(option_char)
+      FORMATTER_LOOKUP_TABLE.fetch(option_char)
+    end
+  end
+end

data/lib/rika/parse_result.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+module Rika
+  # Encapsulates all results of parsing a document.
+  ParseResult = Struct.new(
+    :content,
+    :metadata,
+    :metadata_java,
+    :content_type,
+    :language,
+    :input_type,
+    :data_source,
+    :max_content_length,
+    keyword_init: true
+  ) do
+    # Support using 'text' instead of 'content'; this often makes more sense.
+    alias_method :text, :content
+
+    # @return [Boolean] true if, and only if, input is a file
+    def file?
+      input_type == :file
+    end
+
+    # @return [Boolean] true if, and only if, input is HTTP
+    def http?
+      input_type == :http
+    end
+
+    # @return [Hash] content and metadata of ParseResult instance as hash
+    def content_and_metadata_hash
+      { content: content, metadata: metadata }
+    end
+  end
+end

data/lib/rika/parser.rb

@@ -1,90 +1,84 @@
-module Rika
-  class Parser
+# frozen_string_literal: true
 
-    attr_reader :data_source, :tika, :metadata_java, :metadata_ruby, :input_type
+require 'uri'
+require_relative 'parse_result'
 
-    def initialize(data_source, max_content_length = -1, detector = DefaultDetector.new)
+module Rika
+  # Parses a document and returns a ParseResult.
+  # This class is intended to be used only by the Rika module, not by users of the gem,
+  # who should instead call `Rika.parse`.
+  class Parser
+    # @param [String] data_source file path or HTTP(s) URL
+    # @param [Boolean] key_sort whether to sort the keys in the metadata hash, defaults to true
+    # @param [Integer] max_content_length maximum content length to return, defaults to all
+    # @param [Detector] detector Tika detector, defaults to DefaultDetector
+    def initialize(data_source, key_sort: true, max_content_length: -1, detector: DefaultDetector.new)
       @data_source = data_source
-      @tika = Tika.new(detector)
-      @tika.set_max_string_length(max_content_length)
-      @metadata_java = nil
-      @metadata_ruby = nil
-      @input_type = get_input_type
-    end
-
-    def content
-      parse
-      @content
-    end
-
-    def metadata
-      unless @metadata_ruby
-        parse
-        @metadata_ruby = metadata_java.names.each_with_object({}) do |name, m_ruby|
-          m_ruby[name] = metadata_java.get(name)
-        end
-      end
-      @metadata_ruby
-    end
-
-    def media_type
-      @media_type ||= file? \
-          ? tika.detect(java.io.File.new(data_source)) \
-          : tika.detect(input_stream)
+      @key_sort = key_sort
+      @max_content_length = max_content_length
+      @detector = detector
+      @input_type = data_source_input_type
+      @tika = Tika.new(@detector)
     end
 
-    # @deprecated
-    def available_metadata
-      metadata.keys
-    end
-
-    # @deprecated
-    def metadata_exists?(name)
-      metadata[name] != nil
-    end
-
-    def language
-      @lang ||= LanguageIdentifier.new(content)
-      @lang.language
-    end
+    # Entry point method for parsing a document
+    # @return [ParseResult] parse result
+    def parse
+      metadata_java = Metadata.new
+      @tika.set_max_string_length(@max_content_length)
+      content = with_input_stream { |stream| @tika.parse_to_string(stream, metadata_java) }
+      language = Rika.language(content)
+      metadata_java.set('rika:language', language)
+      metadata_java.set('rika:data-source', @data_source)
+      metadata = metadata_java_to_ruby(metadata_java)
+      metadata = metadata.sort_by { |key, _value| key.downcase }.to_h if @key_sort
 
-    # @deprecated
-    # https://tika.apache.org/1.9/api/org/apache/tika/language/LanguageIdentifier.html#isReasonablyCertain()
-    # says: WARNING: Will never return true for small amount of input texts.
-    # https://tika.apache.org/1.19/api/org/apache/tika/language/LanguageIdentifier.html
-    # indicated that the LanguageIdentifier class used in this implementation is deprecated.
-    # TODO: More research needed to see if an alternate implementation can be used.
-    def language_is_reasonably_certain?
-      @lang ||= LanguageIdentifier.new(content)
-      @lang.is_reasonably_certain
+      ParseResult.new(
+        content:            content,
+        metadata:           metadata,
+        metadata_java:      metadata_java,
+        content_type:       metadata['Content-Type'],
+        language:           language,
+        input_type:         @input_type,
+        data_source:        @data_source,
+        max_content_length: @max_content_length
+      )
     end
 
-
-    def parse
-      unless @content
-        @metadata_java = Metadata.new
-        @content = tika.parse_to_string(input_stream, @metadata_java).to_s.strip
+    # @param [Metadata] metadata_java a Tika Java metadata instance populated by the parse and added to by this class
+    # @return [Hash] a Ruby hash containing the data of the Java Metadata instance
+    private def metadata_java_to_ruby(metadata_java)
+      metadata_java.names.each_with_object({}) do |name, m_ruby|
+        m_ruby[name] = metadata_java.get(name)
       end
     end
 
-    private def get_input_type
-      if File.file?(data_source)
+    # @return [Symbol] input type (currently only :file and :http are supported)
+    # @raise [IOError] if input is not a file or HTTP resource
+    private def data_source_input_type
+      if File.file?(@data_source)
         :file
-      elsif URI(data_source).is_a?(URI::HTTP) && URI.open(data_source)
+      elsif URI(@data_source).is_a?(URI::HTTP)
         :http
       else
-        raise IOError, "Input (#{data_source}) is not an available file or HTTP resource."
+        raise IOError, "Input (#{@data_source}) is not an available file or HTTP resource."
       end
     end
 
-    private def input_stream
-      file? \
-          ? FileInputStream.new(java.io.File.new(data_source)) \
-          : URL.new(data_source).open_stream
-    end
-
-    private def file?
-      input_type == :file
+    # * Creates and opens an input stream from the configured resource.
+    # * Yields that stream to the passed code block.
+    # * Then closes the stream.
+    # @return [Object] the value returned by the passed code block
+    private def with_input_stream
+      input_stream =
+        if @input_type == :file
+          FileInputStream.new(java.io.File.new(@data_source))
+        else
+          URL.new(@data_source).open_stream
+        end
+      yield input_stream
+    ensure
+      input_stream.close if input_stream.respond_to?(:close)
     end
   end
 end

data/lib/rika/tika_loader.rb

@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+# Requires the Tika jar file, either from the default location (packaged with this gem)
+# or from an override specified in the TIKA_JAR_FILESPEC environment variable.
+
+module Rika
+  # This class handles the loading of the Apache Tika Java jar file.
+  # It is not intended to be instantiated. Instead, call the only public class method, `require_tika`.
+  class TikaLoader
+    # @return [String] absolute filespec of loaded Tika jar file
+    # @raise [TikaLoadError] if the Tika jar file cannot be loaded
+    def self.require_tika
+      tika_jar_filespec = specified_tika_filespec
+
+      begin
+        abs_tika_jar_filespec = File.absolute_path(tika_jar_filespec)
+        require abs_tika_jar_filespec
+        abs_tika_jar_filespec
+      rescue LoadError
+        message = "Unable to load Tika jar file from '#{tika_jar_filespec}'."
+        if tika_jar_filespec != abs_tika_jar_filespec
+          message += "\nAbsolute filespec is '#{abs_tika_jar_filespec}'."
+        end
+        raise TikaLoadError, message
+      end
+    end
+
+    # Gets the Tika jar filespec from the TIKA_JAR_FILESPEC environment variable,
+    # and prints an error message and exits if it is not set.
+    #
+    # @return [String] Tika jar filespec from env var TIKA_JAR_FILESPEC
+    # @raise [TikaLoadError] if the Tika jar file was not specified
+    private_class_method def self.specified_tika_filespec
+      tika_jar_filespec = ENV['TIKA_JAR_FILESPEC']
+      not_specified = tika_jar_filespec.nil? || tika_jar_filespec.strip.empty?
+      raise(TikaLoadError, 'Environment variable TIKA_JAR_FILESPEC is not set.') if not_specified
+
+      tika_jar_filespec
+    end
+
+    # Formats an error message for printing to stderr.
+    #
+    # @param [String] message the error message
+    # @return [String] the formatted error message
+    private_class_method def self.formatted_error_message(message)
+      banner = '!' * 79 # message.length
+      <<~MESSAGE
+
+        #{banner}
+        #{message}
+        #{banner}
+
+      MESSAGE
+    end
+
+    # Prints an error message to stderr and exits with a non-zero exit code.
+    private_class_method def self.print_message_and_exit(message)
+      warn formatted_error_message(message)
+      exit 1
+    end
+  end
+
+  # This error class reports the inability to load the Tika jar file.
+  class TikaLoadError < RuntimeError; end
+end

data/lib/rika/version.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Rika
-  VERSION = "1.11.1"
+  VERSION = '2.0.0'
 end

data/lib/rika.rb

@@ -1,43 +1,114 @@
-# encoding: utf-8
+# frozen_string_literal: true
 
-raise "You need to run JRuby to use Rika" unless RUBY_PLATFORM =~ /java/
-
-require "rika/version"
-require 'uri'
-require 'open-uri'
+# This file is the top level file for the Rika gem.
+# It requires the other files in the gem and provides the top level API.
+# It also provides the top level module for the gem.
+require 'rika/version'
 require_relative 'rika/parser'
-require_relative '../java-lib/tika-app-1.24.1.jar'
+require_relative 'rika/tika_loader'
 
+# The top level module for the Rika gem.
 module Rika
-  import org.apache.tika.metadata.Metadata
-  import org.apache.tika.Tika
-  import org.apache.tika.language.LanguageIdentifier
-  import org.apache.tika.detect.DefaultDetector
-  import java.io.FileInputStream
-  import java.net.URL
+  PROJECT_URL = 'https://github.com/keithrbennett/rika'
+
+  # Loads the Tika jar file and imports the needed Java classes.
+  # @return [Module] the Rika module, for chaining
+  def self.init
+    return if @initialized
+
+    Rika.raise_unless_jruby
+
+    Rika::TikaLoader.require_tika
+    import java.io.FileInputStream
+    import java.net.URL
+    import org.apache.tika.Tika
+    import org.apache.tika.detect.DefaultDetector
+    import org.apache.tika.io.TikaInputStream
+    import org.apache.tika.langdetect.optimaize.OptimaizeLangDetector
+    import org.apache.tika.language.detect.LanguageDetector
+    import org.apache.tika.language.detect.LanguageResult
+    import org.apache.tika.metadata.Metadata
+
+    @initialized = true
+    self
+  end
+
+  # Gets a ParseResult from parsing a document.
+  #
+  # @param [String] data_source file path or HTTP(s) URL
+  # @param [Boolean] key_sort whether to sort the keys in the metadata hash, defaults to true
+  # @param [Integer] max_content_length maximum content length to return, defaults to all
+  # @param [Detector] detector Tika detector, defaults to DefaultDetector
+  # @return [ParseResult]
+  def self.parse(data_source, key_sort: true, max_content_length: -1, detector: DefaultDetector.new)
+    init
+    parser = Parser.new(data_source, key_sort: key_sort, max_content_length: max_content_length, detector: detector)
+    parser.parse
+  end
 
-  def self.parse_content_and_metadata(file_location, max_content_length = -1)
-    parser = Parser.new(file_location, max_content_length)
-    [parser.content, parser.metadata]
+  # @return [String] version of loaded Tika jar file
+  def self.tika_version
+    init
+    Tika.java_class.package.implementation_version
   end
 
-  def self.parse_content_and_metadata_as_hash(file_location, max_content_length = -1)
-    content, metadata = parse_content_and_metadata(file_location, max_content_length)
-    { content: content, metadata: metadata }
+  # @param [String] text text to detect language of
+  # @return [String] language of passed text, as 2-character ISO 639-1 code
+  def self.language(text)
+    init
+    tika_language_detector.detect(text.to_java_string).get_language
   end
 
-  def self.parse_content(file_location, max_content_length = -1)
-    Parser.new(file_location, max_content_length).content
+  # @param [String] data_source file path or HTTP URL
+  # @return [Array<String,Hash>] content and metadata of file at specified location
+  #
+  # @deprecated Instead, get a ParseResult and access the content and metadata fields.
+  def self.parse_content_and_metadata(data_source, max_content_length: -1)
+    init
+    result = parse(data_source, max_content_length: max_content_length)
+    [result.content, result.metadata]
+  end
+
+  # @param [String] data_source file path or HTTP URL
+  # @return [Hash] content and metadata of file at specified location
+  #
+  # @deprecated Instead, use a ParseResult or its to_h method.
+  def self.parse_content_and_metadata_as_hash(data_source, max_content_length: -1)
+    init
+    result = parse(data_source, max_content_length: max_content_length)
+    { content: result.content, metadata: result.metadata }
+  end
+
+  # @param [String] data_source file path or HTTP URL
+  # @return [Parser] parser for resource at specified location
+  #
+  # @deprecated Instead, get a ParseResult and access the content field
+  def self.parse_content(data_source, max_content_length: -1)
+    init
+    parse(data_source, max_content_length: max_content_length).content
   end
 
   # Regarding max_content_length, the default is set at 0 to save unnecessary processing,
   # since the content is being ignored. However, the PDF metadata "pdf:unmappedUnicodeCharsPerPage"
-  # and "pdf:charsPerPage" will be absent if the max_content_length is 0, and will be
-  # ]may differ depending on
-  # the number of characters read.
-  def self.parse_metadata(file_location, max_content_length = 0)
-    Parser.new(file_location, max_content_length).metadata
+  # and "pdf:charsPerPage" will be absent if the max_content_length is 0, and otherwise may differ
+  # depending on the number of characters read.
+  #
+  # @deprecated Instead, get a ParseResult and access the metadata field
+  def self.parse_metadata(data_source, max_content_length: -1)
+    init
+    parse(data_source, max_content_length: max_content_length).metadata
   end
-end
 
+  # @return [Detector] Tika detector
+  def self.tika_language_detector
+    init
+    @tika_language_detector ||= OptimaizeLangDetector.new.loadModels
+  end
 
+  # Raise an error if not running under JRuby.
+  def self.raise_unless_jruby
+    unless RUBY_PLATFORM.match(/java/)
+      raise "\n\n\nRika can only be run with JRuby! It needs access to the Java Virtual Machine.\n\n\n"
+    end
+  end
+end

data/rika.gemspec

@@ -1,23 +1,36 @@
-# -*- encoding: utf-8 -*-
-lib = File.expand_path('../lib', __FILE__)
+# frozen_string_literal: true
+
+require 'English'
+
+lib = File.expand_path('lib', __dir__)
 $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 require 'rika/version'
 
 Gem::Specification.new do |gem|
-  gem.name          = "rika"
+  gem.name          = 'rika'
   gem.version       = Rika::VERSION
-  gem.authors       = ["Richard Nyström", "Keith Bennett"]
-  gem.email         = ["ricny046@gmail.com", "keithrbennett@gmail.com"]
-  gem.description   = %q{ A JRuby wrapper for Apache Tika to extract text and metadata from files of various formats. }
-  gem.summary       = %q{ A JRuby wrapper for Apache Tika to extract text and metadata from files of various formats. }
-  gem.homepage      = "https://github.com/keithrbennett/rika"
-  gem.files         = `git ls-files`.split($/)
-  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
-  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
-  gem.require_paths = ["lib"]
-  gem.add_development_dependency "rspec", "~> 3.9"
-  gem.add_development_dependency "rake", "~> 13.0"
-  gem.platform = "java"
-  gem.license = "Apache-2.0"
-end
+  gem.authors       = ['Richard Nyström', 'Keith Bennett']
+  gem.email         = ['ricny046@gmail.com', 'keithrbennett@gmail.com']
+  gem.description   = 'A JRuby wrapper for Apache Tika to extract text and metadata from files of various formats.'
+  gem.summary       = 'A JRuby wrapper for Apache Tika to extract text and metadata from files of various formats.'
+  gem.homepage      = 'https://github.com/keithrbennett/rika'
+  gem.files         = `git ls-files`.split($INPUT_RECORD_SEPARATOR)
+  gem.executables   = gem.files.grep(%r{^bin/}).map { |f| File.basename(f) }
+  gem.require_paths = ['lib']
+  gem.add_dependency 'awesome_print'
+  gem.platform = 'java'
+  gem.license = 'Apache-2.0'
+  gem.metadata['rubygems_mfa_required'] = 'true'
+
+  # NOTE: I am excluding the Ruby version constraint because this gem runs only in JRuby, and I don't know the
+  # minimum version requirement, and don't want to exclude use of any versions that might work.
 
+  gem.post_install_message = <<~MESSAGE
+
+    Using the rika gem requires that you:
+      1) download the Apache Tika tika-app jar file from https://tika.apache.org/download.html
+      2) place it somewhere accessible to the running application
+      3) specify its location in the TIKA_JAR_FILESPEC environment variable
+
+  MESSAGE
+end