rika 1.6.0-java → 2.0.0-java

data/spec/rika/parser_spec.rb

@@ -0,0 +1,304 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+require 'rika/parser'
+require 'rika/parse_result'
+require 'webrick'
+
+describe Rika::Parser do
+  port = 50515
+
+  let(:text_parse_result)    { Rika.parse(fixture_path('document.txt')) }
+  let(:docx_parse_result)    { Rika.parse(fixture_path('document.docx')) }
+  let(:doc_parse_result)     { Rika.parse(fixture_path('document.doc'))  }
+  let(:pdf_parse_result)     { Rika.parse(fixture_path('document.pdf'))  }
+  let(:image_parse_result)   { Rika.parse(fixture_path('image.jpg'))     }
+  let(:unknown_parse_result) { Rika.parse(fixture_path('unknown.bin'))   }
+  let(:fixtures_dir)         { File.expand_path(File.join(File.dirname(__FILE__), '../fixtures')) }
+  let(:quote_first_line) { 'Stopping by Woods on a Snowy Evening' }
+  let(:url) { "http://#{Socket.gethostname}:#{port}" }
+  let(:sample_pdf_filespec) { fixture_path('document.pdf') }
+  let(:first_line) { ->(string) { string.split("\n").first.strip } }
+
+  # returns a lambda that, when passed an action, will wrap it in an HTTP server
+  let(:server_runner) do
+    ->(action) do
+      server = nil
+      server_thread = Thread.new do
+        server = WEBrick::HTTPServer.new(
+          Port:         port,
+          DocumentRoot: fixtures_dir,
+          AccessLog:    [],
+          Logger:       WEBrick::Log.new('/dev/null')
+        )
+        server.start
+      end
+
+      # Wait for server to become ready on its new thread
+      sleep 0.01 while server.nil?
+      begin
+        action.call
+      ensure
+        server.shutdown
+        server_thread.exit
+      end
+    end
+  end
+
+  context 'when initialized with a content string and metadata' do
+    let(:content) { 'Magnifique' }
+    let(:metadata) { { 'author' => 'John Doe' } }
+    let(:result) { Rika::ParseResult.new(content: content, metadata: metadata) }
+
+    specify '#content_and_metadata_hash returns a hash with content and metadata' do
+      expect(result.content_and_metadata_hash).to eq({ content: content, metadata: metadata })
+    end
+  end
+
+  describe '#parse' do
+    let(:parser) { described_class.new('spec/fixtures/document.pdf') }
+    let(:parse_result) { parser.parse }
+    let(:metadata) { parse_result.metadata }
+
+    specify 'returns an instance of ParseResult' do
+      expect(parse_result).to be_a(Rika::ParseResult)
+    end
+
+    specify 'returns a ParseResult with the expected access methods' do
+      expect(parse_result).to respond_to(
+        :content,
+        :metadata,
+        :metadata_java,
+        :content_type,
+        :language,
+        :input_type,
+        :data_source,
+        :max_content_length
+      )
+    end
+
+    specify 'returns a ParseResult with the expected content' do
+      expect(parse_result.content).to include('Stopping by Woods on a Snowy Evening')
+    end
+
+    specify 'returns a ParseResult with the expected metadata' do
+      expect(parse_result.metadata).to include(
+        'dc:creator' => 'Robert Frost',
+        'dc:format' => 'application/pdf; version=1.3',
+        'dc:title' => 'Stopping by Woods on a Snowy Evening',
+        'rika:data-source' => 'spec/fixtures/document.pdf',
+        'rika:language' => 'en'
+      )
+    end
+
+    specify 'returns a ParseResult with the expected metadata_java' do
+      expect(parse_result.metadata_java).to be_a(Java::OrgApacheTikaMetadata::Metadata)
+    end
+
+    specify 'returns a ParseResult with the expected content_type' do
+      expect(parse_result.content_type).to eq('application/pdf')
+    end
+
+    specify 'returns a ParseResult with the expected language' do
+      expect(parse_result.language).to eq('en')
+    end
+
+    specify 'returns a ParseResult with the expected input_type' do
+      expect(parse_result.input_type).to eq(:file)
+    end
+
+    specify 'returns a ParseResult with the expected data_source' do
+      expect(parse_result.data_source).to eq('spec/fixtures/document.pdf')
+    end
+
+    describe 'metadata key sorting' do
+      RSpec.shared_examples('metadata key sorting') do |caption, key_sort|
+        specify "Metadata keys are #{caption} case insensitively when key_sort is #{key_sort}" do
+          parser = described_class.new('spec/fixtures/document.pdf', key_sort: key_sort)
+          keys = parser.parse.metadata.keys
+          expect(keys == keys.sort_by(&:downcase)).to eq(key_sort)
+          expect(keys).not_to eq(keys.map(&:downcase)) # Above test only valid if both upper and lower case occur.
+        end
+      end
+
+      include_examples 'metadata key sorting', 'sorted', true
+      include_examples 'metadata key sorting', 'not sorted', false
+    end
+
+    specify 'returns a ParseResult with the expected max_content_length' do
+      expect(parse_result.max_content_length).to eq(-1)
+    end
+  end
+
+  it 'raises an error if the file does not exist' do
+    expect { Rika.parse(fixture_path('nonexistent_file.txt')) }.to raise_error(IOError)
+  end
+
+  it 'raises an error if the URL does not exist' do
+    unavailable_server = 'http://k6075sd0dfkr8nvfw0zvwfwckucf2aba.com'
+    unavailable_file_on_web = File.join(unavailable_server, 'x.pdf')
+    expect { Rika.parse(unavailable_file_on_web) }.to raise_error(Java::JavaNet::UnknownHostException)
+  end
+
+  it 'detects a file type without a file extension' do
+    parse_result = Rika.parse(fixture_path('image_jpg_without_extension'))
+    expect(parse_result.metadata['Content-Type']).to eq('image/jpeg')
+  end
+
+  describe '#content' do
+    it 'returns the content in a text file' do
+      expect(first_line.(text_parse_result.content)).to eq(quote_first_line)
+    end
+
+    it 'returns the content in a docx file' do
+      expect(first_line.(docx_parse_result.content)).to eq(quote_first_line)
+    end
+
+    it 'returns the content in a pdf file' do
+      # For some reason, the generated PDF file has a newline at the beginning
+      # and trailing spaces on the lines, so we use the second line, and
+      # use `include` to do the text match.
+      expect(pdf_parse_result.content.lines[1]).to include(quote_first_line)
+    end
+
+    it 'returns no content for an image' do
+      expect(image_parse_result.content).to be_empty
+    end
+
+    it 'only returns max content length from a text file' do
+      expect(Rika.parse(fixture_path('document.txt'), max_content_length: 8).content).to eq('Stopping')
+    end
+
+    it 'only returns max content length from a PDF' do
+      expect(Rika.parse(fixture_path('document.pdf'), max_content_length: 9).content).to eq("\nStopping")
+    end
+
+    it 'only returns max content length for file over http' do
+      server_runner.call(-> do
+        content = Rika.parse(File.join(url, 'document.txt'), max_content_length: 8).content
+        expect(content).to eq('Stopping')
+      end)
+    end
+
+    it 'returns the content from a file over http' do
+      content = server_runner.call(-> do
+        Rika.parse(File.join(url, 'document.txt')).content
+      end)
+      expect(first_line.(content)).to eq(quote_first_line)
+    end
+
+    it 'return empty string for unknown file' do
+      expect(unknown_parse_result.content).to be_empty
+    end
+  end
+
+  # We just test a few of the metadata fields for some common file formats
+  # to make sure the integration with Apache Tika works. Apache Tika already
+  # have tests for all file formats it supports so we won't retest that
+  describe '#metadata' do
+    it 'returns nil if metadata field does not exist' do
+      expect(text_parse_result.metadata['nonsense']).to be_nil
+    end
+
+    it 'returns metadata from a docx file' do
+      expect(docx_parse_result.metadata['meta:page-count']).to eq('1')
+    end
+
+    it 'returns metadata from a pdf file' do
+      expect(pdf_parse_result.metadata['pdf:docinfo:creator']).to eq('Robert Frost')
+    end
+
+    it 'returns metadata from a file over http' do
+      server_runner.call(-> do
+        parser = Rika.parse(File.join(url, 'document.pdf'))
+        expect(parser.metadata['pdf:docinfo:creator']).to eq('Robert Frost')
+      end)
+    end
+
+    it 'returns metadata from an image' do
+      expect(image_parse_result.metadata['Image Height']).to eq('72 pixels')
+      expect(image_parse_result.metadata['Image Width']).to  eq('72 pixels')
+    end
+  end
+
+  describe '#content_type' do
+    it 'returns application/pdf for a pdf file' do
+      expect(pdf_parse_result.content_type).to eq('application/pdf')
+    end
+
+    it 'returns text/plain for a txt file' do
+      expect(text_parse_result.content_type).to eq('text/plain; charset=UTF-8')
+    end
+
+    it 'returns application/pdf for a pdf over http' do
+      server_runner.call(-> do
+        parse_result = Rika.parse(File.join(url, 'document.pdf'))
+        expect(parse_result.content_type).to eq('application/pdf')
+      end)
+    end
+
+    it 'returns application/octet-stream for unknown file' do
+      expect(unknown_parse_result.content_type).to eq('application/octet-stream')
+    end
+
+    it 'returns msword for a doc file' do
+      # There seem to be two permissible content types for a doc file.
+      expect(%w{application/msword application/x-tika-msoffice}.include?(doc_parse_result.content_type)).to be true
+    end
+
+    it 'returns wordprocessingml for a docx file' do
+      expect(docx_parse_result.content_type).to eq(
+        'application/vnd.openxmlformats-officedocument.wordprocessingml.document'
+      )
+    end
+  end
+
+  describe '#language' do
+    it 'returns the language of the content' do
+      %w(en de fr ru es).each do |lang|
+        parse_result = Rika.parse(fixture_path("#{lang}.txt"))
+        expect(parse_result.language).to eq(lang)
+      end
+    end
+  end
+
+  it 'returns valid content using Rika.parse_content' do
+    content = Rika.parse_content(sample_pdf_filespec)
+    expect(content).to be_a(String)
+    expect(content).not_to be_empty
+  end
+
+  it 'returns valid metadata using Rika.parse_metadata' do
+    metadata = Rika.parse_metadata(sample_pdf_filespec)
+    expect(metadata).to be_a(Hash)
+    expect(metadata).not_to be_empty
+  end
+
+  it 'returns valid content and metadata using Rika.parse_content_and_metadata' do
+    content, metadata = Rika.parse_content_and_metadata(sample_pdf_filespec)
+    expect(content).to be_a(String)
+    expect(content).not_to be_empty
+    expect(metadata).to be_a(Hash)
+    expect(metadata).not_to be_empty
+  end
+
+  specify 'both means of getting both content and metadata return the same values' do
+    content1, metadata1 = Rika.parse_content_and_metadata(sample_pdf_filespec)
+
+    h = Rika.parse_content_and_metadata_as_hash(sample_pdf_filespec)
+    content2  = h[:content]
+    metadata2 = h[:metadata]
+
+    expect(content1).to eq(content2)
+    expect(metadata1).to eq(metadata2)
+  end
+
+  specify 'getting content and metadata individually and together return the same values' do
+    content1, metadata1 = Rika.parse_content_and_metadata(sample_pdf_filespec)
+    content2             = Rika.parse_content(sample_pdf_filespec)
+    metadata2            = Rika.parse_metadata(sample_pdf_filespec)
+
+    expect(content1).to eq(content2)
+    expect(metadata1).to eq(metadata2)
+  end
+end

data/spec/rika/rika_spec.rb

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+require 'rika'
+
+describe Rika do
+  it 'has a version number' do
+    expect(Rika::VERSION).not_to be_nil
+  end
+end

data/spec/rika/tika_loader_spec.rb

@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+require 'rika/tika_loader'
+
+describe Rika::TikaLoader do
+  describe '.require_tika' do
+    it 'returns the correct Tika jar file path' do
+      expect(described_class.require_tika).to match(/tika-app-.*\.jar/)
+    end
+
+    it 'calls print_message_and_exit if the Tika jar file cannot be loaded' do
+      allow(ENV).to receive(:[]).with('TIKA_JAR_FILESPEC').and_return('nonexistent_file')
+      expect { described_class.require_tika }.to raise_error(Rika::TikaLoadError) \
+        .with_message(/Unable to load Tika jar file from 'nonexistent_file'./)
+    end
+  end
+
+  describe '.specified_tika_filespec' do
+    it 'returns the correct Tika jar file path' do
+      expect(described_class.send(:specified_tika_filespec)).to match(/tika-app-.*\.jar/)
+    end
+
+    it 'raises a TikaLoadError if the Tika jar filespec is not specified at all in TIKA_JAR_FILESPEC' do
+      allow(ENV).to receive(:[]).with('TIKA_JAR_FILESPEC').and_return(nil)
+      expect { described_class.send(:specified_tika_filespec) }.to raise_error(Rika::TikaLoadError) \
+        .with_message(/Environment variable TIKA_JAR_FILESPEC is not set./)
+    end
+  end
+
+  describe '.print_message_and_exit' do
+    it 'prints the correct message and exits with an exit code of 1' do
+      stderr_orig = $stderr
+      $stderr = StringIO.new
+
+      begin
+        expect { described_class.send(:print_message_and_exit, 'message') }.to raise_error(SystemExit) do |error|
+          expect(error.status).to eq(1)
+        end
+        expect($stderr.string).to match(/message/)
+      ensure
+        $stderr = stderr_orig
+      end
+    end
+  end
+
+  describe '.formatted_error_message' do
+    it 'returns the correct message' do
+      message = 'This is a test message.'
+      expect(described_class.send(:formatted_error_message, message)).to match(/#{message}/)
+    end
+
+    it 'returns the correct banner' do
+      expect(described_class.send(:formatted_error_message, 'message').lines.grep(/!{79}/).size).to be >= 2
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -1,14 +1,22 @@
-require "rika"
+# frozen_string_literal: true
 
-def file_path( *paths )
+require 'simplecov'
+SimpleCov.start { add_filter '/spec/' }
+
+require 'rika'
+
+def fixture_path(*paths)
   File.expand_path(File.join(File.dirname(__FILE__), 'fixtures', *paths))
 end
 
 # See http://rubydoc.info/gems/rspec-core/RSpec/Core/Configuration
 RSpec.configure do |config|
-  config.treat_symbols_as_metadata_keys_with_true_values = true
-  config.run_all_when_everything_filtered = true
-  config.filter_run :focus
+  # Enable the line below if you want ", focus: true" after a test declaration to
+  # denote the only tests that will be run:
+  # config.filter_run :focus
 
   config.order = 'random'
+  config.example_status_persistence_file_path = 'spec/rspec-failed-tests-control-file.txt'
 end
+
+Rika.init

metadata

@@ -1,155 +1,111 @@
 --- !ruby/object:Gem::Specification
 name: rika
 version: !ruby/object:Gem::Version
-  version: 1.6.0
+  version: 2.0.0
 platform: java
 authors:
 - Richard Nyström
+- Keith Bennett
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2014-10-06 00:00:00.000000000 Z
+date: 2023-09-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '='
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 2.14.1
-  name: rspec
+        version: '0'
+  name: awesome_print
   prerelease: false
-  type: :development
+  type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '='
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 2.14.1
-- !ruby/object:Gem::Dependency
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - '='
-      - !ruby/object:Gem::Version
-        version: 10.3.1
-  name: rake
-  prerelease: false
-  type: :development
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - '='
-      - !ruby/object:Gem::Version
-        version: 10.3.1
-description: ' A JRuby wrapper for Apache Tika to extract text and metadata from various
-  file formats. '
+        version: '0'
+description: A JRuby wrapper for Apache Tika to extract text and metadata from files
+  of various formats.
 email:
 - ricny046@gmail.com
-executables: []
+- keithrbennett@gmail.com
+executables:
+- rika
 extensions: []
 extra_rdoc_files: []
 files:
-- .gitignore
-- .rspec
-- .travis.yml
+- ".gitignore"
+- ".rspec"
+- ".rubocop.yml"
 - Gemfile
 - LICENSE.txt
 - README.md
+- RELEASE_NOTES.md
 - Rakefile
+- bin/rika
 - lib/rika.rb
+- lib/rika/cli/args_parser.rb
+- lib/rika/cli/rika_command.rb
+- lib/rika/formatters.rb
+- lib/rika/parse_result.rb
+- lib/rika/parser.rb
+- lib/rika/tika_loader.rb
 - lib/rika/version.rb
 - pom.xml
 - rika.gemspec
+- rika_helper.rb
 - spec/fixtures/de.txt
 - spec/fixtures/document.doc
 - spec/fixtures/document.docx
 - spec/fixtures/document.pdf
+- spec/fixtures/document.txt
 - spec/fixtures/en.txt
 - spec/fixtures/es.txt
 - spec/fixtures/fr.txt
 - spec/fixtures/image.jpg
+- spec/fixtures/image_jpg_without_extension
 - spec/fixtures/lang_cant_be_determined.txt
-- spec/fixtures/over_100k_file.txt
 - spec/fixtures/ru.txt
-- spec/fixtures/text_file.txt
-- spec/fixtures/text_file_without_extension
+- spec/fixtures/tiny.txt
 - spec/fixtures/unknown.bin
-- spec/rika_spec.rb
+- spec/rika/cli/args_parser_spec.rb
+- spec/rika/cli/rika_command_spec.rb
+- spec/rika/formatters_spec.rb
+- spec/rika/parse_result_spec.rb
+- spec/rika/parser_spec.rb
+- spec/rika/rika_spec.rb
+- spec/rika/tika_loader_spec.rb
 - spec/spec_helper.rb
-- target/dependency/apache-mime4j-core-0.7.2.jar
-- target/dependency/apache-mime4j-dom-0.7.2.jar
-- target/dependency/asm-debug-all-4.1.jar
-- target/dependency/aspectjrt-1.8.0.jar
-- target/dependency/bcmail-jdk15-1.45.jar
-- target/dependency/bcprov-jdk15-1.45.jar
-- target/dependency/boilerpipe-1.1.0.jar
-- target/dependency/commons-codec-1.9.jar
-- target/dependency/commons-compress-1.8.1.jar
-- target/dependency/commons-httpclient-3.1.jar
-- target/dependency/commons-logging-1.1.1.jar
-- target/dependency/fontbox-1.8.6.jar
-- target/dependency/isoparser-1.0.2.jar
-- target/dependency/java-libpst-0.8.1.jar
-- target/dependency/jcip-annotations-1.0.jar
-- target/dependency/jdom-1.0.jar
-- target/dependency/jempbox-1.8.6.jar
-- target/dependency/jhighlight-1.0.jar
-- target/dependency/jmatio-1.0.jar
-- target/dependency/juniversalchardet-1.0.3.jar
-- target/dependency/metadata-extractor-2.6.2.jar
-- target/dependency/netcdf-4.2.20.jar
-- target/dependency/pdfbox-1.8.6.jar
-- target/dependency/poi-3.11-beta2.jar
-- target/dependency/poi-ooxml-3.11-beta2.jar
-- target/dependency/poi-ooxml-schemas-3.11-beta2.jar
-- target/dependency/poi-scratchpad-3.11-beta2.jar
-- target/dependency/rome-1.0.jar
-- target/dependency/slf4j-api-1.6.1.jar
-- target/dependency/tagsoup-1.2.1.jar
-- target/dependency/tika-core-1.6.jar
-- target/dependency/tika-parsers-1.6.jar
-- target/dependency/unidataCommon-4.2.20.jar
-- target/dependency/vorbis-java-core-0.6.jar
-- target/dependency/vorbis-java-tika-0.6.jar
-- target/dependency/xercesImpl-2.8.1.jar
-- target/dependency/xml-apis-1.3.03.jar
-- target/dependency/xmlbeans-2.6.0.jar
-- target/dependency/xmpcore-5.1.2.jar
-- target/dependency/xz-1.5.jar
-homepage: https://github.com/ricn/rika
-licenses: []
-metadata: {}
-post_install_message:
+homepage: https://github.com/keithrbennett/rika
+licenses:
+- Apache-2.0
+metadata:
+  rubygems_mfa_required: 'true'
+post_install_message: |2+
+
+  Using the rika gem requires that you:
+    1) download the Apache Tika tika-app jar file from https://tika.apache.org/download.html
+    2) place it somewhere accessible to the running application
+    3) specify its location in the TIKA_JAR_FILESPEC environment variable
+
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project:
-rubygems_version: 2.1.9
+rubygems_version: 3.3.26
 signing_key:
 specification_version: 4
-summary: A JRuby wrapper for Apache Tika to extract text and metadata from various file formats.
-test_files:
-- spec/fixtures/de.txt
-- spec/fixtures/document.doc
-- spec/fixtures/document.docx
-- spec/fixtures/document.pdf
-- spec/fixtures/en.txt
-- spec/fixtures/es.txt
-- spec/fixtures/fr.txt
-- spec/fixtures/image.jpg
-- spec/fixtures/lang_cant_be_determined.txt
-- spec/fixtures/over_100k_file.txt
-- spec/fixtures/ru.txt
-- spec/fixtures/text_file.txt
-- spec/fixtures/text_file_without_extension
-- spec/fixtures/unknown.bin
-- spec/rika_spec.rb
-- spec/spec_helper.rb
+summary: A JRuby wrapper for Apache Tika to extract text and metadata from files of
+  various formats.
+test_files: []

data/.travis.yml

@@ -1,7 +0,0 @@
-language: ruby
-rvm:
-  - jruby-19mode
-  - jruby-head
-notifications:
-  recipients:
-    - ricny046@gmail.com