ddr-filetools 0.4.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: cd1c3088ed857301a2108f8921c4784538374cc4
+  data.tar.gz: 4004b38451534990699d74f034729cab559f7cd2
+SHA512:
+  metadata.gz: ef1570db43cc860bdc3abf6e739f2eabdfece85fc6f5fdbce36f15cdf145eb857303743a527b00422f381558eb9b9104d53264485443a51e2480577b0ef53645
+  data.tar.gz: d79b1d0f67a1af8c769a45bdba0b69b603826703d5d89962e9010327c0beae03176efe108f04e8df86f139e9c270b96f719c918228517c65d795adcfae25b7d1

data/.gitignore

@@ -0,0 +1,6 @@
+Gemfile.lock
+coverage
+bin/tika-*
+bin/fits
+pkg
+tmp

data/.rspec

@@ -0,0 +1,3 @@
+--color
+--warnings
+--require spec_helper

data/.travis.yml

@@ -0,0 +1,9 @@
+language: ruby
+rvm:
+  - 2.1
+before_script: "bundle exec rake tika:download fits:download"
+cache: 
+  - bundler
+notifications:
+  email:
+    - lib-drs@duke.edu

data/Gemfile

@@ -0,0 +1,5 @@
+source 'https://rubygems.org'
+
+gemspec
+
+gem "coveralls", require: false

data/LICENSE.txt

@@ -0,0 +1,12 @@
+Copyright (c) 2014, Duke University
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

data/README.md

@@ -0,0 +1,44 @@
+# Ddr::FileTools
+
+File extraction and analysis tools.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'ddr-filetools'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install ddr-filetools
+
+## Dependencies
+
+TODO
+
+## Configuration
+
+TODO
+
+There are rake tasks for downloading Tika and FITS to expected locations.
+
+```sh
+rake tika:download
+rake fits:download
+```
+
+## Usage
+
+TODO
+
+## Contributing
+
+1. Fork it ( https://github.com/[my-github-username]/ddr_extractor/fork )
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create a new Pull Request

data/Rakefile

@@ -0,0 +1,9 @@
+require "bundler/gem_tasks"
+require 'rspec/core/rake_task'
+
+load "tasks/ddr_filetools.rake"
+
+desc "Run all specs in spec directory"
+RSpec::Core::RakeTask.new(:spec)
+
+task :default => :spec

data/ddr-filetools.gemspec

@@ -0,0 +1,24 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'ddr/filetools/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "ddr-filetools"
+  spec.version       = Ddr::FileTools::VERSION
+  spec.authors       = ["David Chandek-Stark"]
+  spec.email         = ["dchandekstark@gmail.com"]
+  spec.summary       = "Pluggable text and metadata extraction service."
+  spec.description   = "Pluggable text and metadata extraction service."
+  spec.homepage      = "https://github.com/duke-libraries/ddr-filetools"
+  spec.license       = "BSD-3-Clause"
+
+  spec.files         = `git ls-files -z`.split("\x0")
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.6"
+  spec.add_development_dependency "rake"
+  spec.add_development_dependency "rspec", "~> 3.0"
+end

data/lib/ddr-extraction.rb

@@ -0,0 +1 @@
+require "ddr/extraction"

data/lib/ddr/filetools.rb

@@ -0,0 +1,29 @@
+require_relative "filetools/version"
+require_relative "filetools/client"
+require_relative "filetools/tool"
+require_relative "filetools/tika"
+require_relative "filetools/fits"
+require_relative "filetools/text_command"
+require_relative "filetools/metadata_command"
+require_relative "filetools/ocr_command"
+
+module Ddr
+  module FileTools
+
+    class << self
+      def tools
+        @tools ||= {}
+      end
+
+      def register(tool_name, opts)
+        tools[tool_name] = Tool.new(opts)
+      end
+    end
+
+    register :text, command: TextCommand, provider: Tika
+    register :metadata, command: MetadataCommand, provider: Fits
+    register :ocr, command: OcrCommand, provider: Tika
+
+  end
+end
+

data/lib/ddr/filetools/client.rb

@@ -0,0 +1,12 @@
+module Ddr
+  module FileTools
+    class Client
+
+      def run_tool(tool_name, *args)
+        tool = Ddr::FileTools.tools.fetch(tool_name)
+        tool.call(*args)
+      end
+
+    end
+  end
+end

data/lib/ddr/filetools/command.rb

@@ -0,0 +1,17 @@
+module Ddr
+  module FileTools
+    class Command
+
+      attr_reader :provider
+
+      def initialize(provider)
+        @provider = provider
+      end
+
+      def call(file_path)
+        raise NotImplementedError, "Subclasses must implement `call`."
+      end
+
+    end
+  end
+end

data/lib/ddr/filetools/defaults.rb

@@ -0,0 +1,10 @@
+require "ddr-extraction"
+
+bin_dir = File.expand_path("../../../../bin", __FILE__)
+
+Ddr::Extraction.configure do |config|
+  config.adapters.default = :tika
+  config.adapters.tika.path = File.join(bin_dir, "tika-app.jar")
+  config.adapters.fits.path = File.join(bin_dir, "fits", "fits.sh")
+end
+

data/lib/ddr/filetools/fits.rb

@@ -0,0 +1,27 @@
+require_relative "provider"
+
+module Ddr
+  module FileTools
+    class Fits < Provider
+
+      class << self
+        # Path to FITS executable (fits.sh or fits.bat)
+        attr_accessor :fits_path
+      end
+
+      self.fits_path = File.join(File.expand_path("../../../../bin", __FILE__), "fits", "fits.sh")
+
+      def metadata(file_path)
+        call command(file_path)
+      end
+
+      private
+
+      def command(file_path)
+        [self.class.fits_path, "-i", file_path]
+      end
+
+    end
+  end
+end
+

data/lib/ddr/filetools/metadata_command.rb

@@ -0,0 +1,15 @@
+require_relative "command"
+
+module Ddr
+  module FileTools
+    class MetadataCommand < Command
+
+      def call(file_path)
+        provider.metadata(file_path)
+      end
+
+    end
+  end
+end
+
+      

data/lib/ddr/filetools/ocr_command.rb

@@ -0,0 +1,15 @@
+require_relative "command"
+
+module Ddr
+  module FileTools
+    class OcrCommand < Command
+
+      def call(file_path)
+        provider.ocr(file_path)
+      end
+
+    end
+  end
+end
+
+      

data/lib/ddr/filetools/provider.rb

@@ -0,0 +1,15 @@
+require 'open3'
+require_relative "result"
+
+module Ddr
+  module FileTools
+    class Provider
+
+      def call(cmd)
+        out, err, s = Open3.capture3(*cmd)
+        Result.new(out, err, s)
+      end
+
+    end
+  end
+end

data/lib/ddr/filetools/result.rb

@@ -0,0 +1,33 @@
+module Ddr
+  module FileTools
+    class Result
+
+      attr_reader :output, :error, :status
+
+      def initialize(out, err, s)
+        @output, @error, @status = out, err, s
+      end
+
+      def inspect
+        "#<Ddr::FileTools::Result #{success? ? 'SUCCESS' : 'ERROR'}>"
+      end
+
+      def to_s
+        output
+      end
+
+      def read
+        output
+      end
+
+      def success?
+        status.success?
+      end
+
+      def error?
+        !success?
+      end
+
+    end
+  end
+end

data/lib/ddr/filetools/text_command.rb

@@ -0,0 +1,15 @@
+require_relative "command"
+
+module Ddr
+  module FileTools
+    class TextCommand < Command
+
+      def call(file_path)
+        provider.text(file_path)
+      end
+
+    end
+  end
+end
+
+      

data/lib/ddr/filetools/tika.rb

@@ -0,0 +1,30 @@
+require_relative "provider"
+
+module Ddr
+  module FileTools
+    class Tika < Provider
+
+      class << self
+        # Path to tika-app.jar
+        attr_accessor :tika_path
+      end
+
+      self.tika_path = File.join(File.expand_path("../../../../bin", __FILE__), "tika-app.jar")
+
+      def text(file_path)
+        call command(file_path, "--text")
+      end
+
+      def metadata(file_path)
+        call command(file_path, "--metadata", "--xml")
+      end
+
+      private
+
+      def command(file_path, *options)
+        ["java", "-jar", self.class.tika_path, options, file_path].flatten
+      end
+
+    end
+  end
+end

data/lib/ddr/filetools/tool.rb

@@ -0,0 +1,24 @@
+module Ddr
+  module FileTools
+    class Tool
+
+      attr_reader :command, :provider
+
+      def initialize(opts)
+        @command = opts.fetch(:command)
+        @provider = opts.fetch(:provider)
+      end
+
+      def call(*args)
+        build_command.call(*args)
+      end
+
+      private
+
+      def build_command
+        command.new(provider.new)
+      end
+
+    end
+  end
+end

data/lib/ddr/filetools/version.rb

@@ -0,0 +1,5 @@
+module Ddr
+  module FileTools
+    VERSION = "0.4.0"
+  end
+end

data/lib/tasks/ddr_filetools.rake

@@ -0,0 +1,56 @@
+require "openssl"
+require "net/http"
+
+DOWNLOAD_DIR = File.absolute_path("tmp")
+BIN_DIR = File.absolute_path("bin")
+TIKA_VERSION = "1.7"
+FITS_VERSION = "0.8.3"
+
+tika_version = ENV["TIKA_VERSION"] || TIKA_VERSION
+tika_path = File.join(BIN_DIR, "tika-app.jar")
+tika_app = File.basename(tika_path)
+tika_download_url = "http://archive.apache.org/dist/tika/tika-app-#{tika_version}.jar"
+tika_checksum_url = "#{tika_download_url}.sha"
+tika_checksum_type = :SHA1
+
+fits_version = ENV["FITS_VERSION"] || FITS_VERSION
+fits_path = File.join(BIN_DIR, "fits", "fits.sh")
+fits_download_url = "http://projects.iq.harvard.edu/files/fits/files/fits-#{fits_version}.zip"
+
+namespace :tika do
+  desc "Download Tika app"
+  task :download => [:download_dir] do
+    FileUtils.cd(DOWNLOAD_DIR) do
+      puts "Downloading Tika app ... "
+      system "curl -L #{tika_download_url} -o #{tika_app}"
+      checksum = Net::HTTP.get(URI(tika_checksum_url)).chomp
+      puts "Verifiying checksum ... "
+      digest = OpenSSL::Digest.const_get(tika_checksum_type).new
+      digest << File.read(tika_app)
+      if digest.to_s != checksum
+        puts "Checksums do not match -- aborting!"
+        FileUtils.remove_entry_secure(tika_app)
+        abort
+      end
+      FileUtils.mv(tika_app, tika_path)
+    end
+  end
+end
+
+namespace :fits do
+  desc "Download FITS tool"
+  task :download => :download_dir do
+    FileUtils.cd(DOWNLOAD_DIR) do
+      puts "Downloading FITS tool ... "
+      system "curl -L #{fits_download_url} -o fits.zip"
+      system "unzip -a -o -q fits.zip"
+      FileUtils.mv("fits-#{fits_version}", File.dirname(fits_path))
+    end
+    FileUtils.chmod(0755, fits_path)
+  end
+end
+
+task :download_dir do
+  FileUtils.mkdir(DOWNLOAD_DIR) unless Dir.exists?(DOWNLOAD_DIR)
+end
+

data/spec/spec_helper.rb

@@ -0,0 +1,83 @@
+require "coveralls"
+Coveralls.wear!
+
+require "ddr/filetools"
+
+# This file was generated by the `rspec --init` command. Conventionally, all
+# specs live under a `spec` directory, which RSpec adds to the `$LOAD_PATH`.
+# The generated `.rspec` file contains `--require spec_helper` which will cause this
+# file to always be loaded, without a need to explicitly require it in any files.
+#
+# Given that it is always loaded, you are encouraged to keep this file as
+# light-weight as possible. Requiring heavyweight dependencies from this file
+# will add to the boot time of your test suite on EVERY test run, even for an
+# individual file that may not need all of that loaded. Instead, make a
+# separate helper file that requires this one and then use it only in the specs
+# that actually need it.
+#
+# The `.rspec` file also contains a few flags that are not defaults but that
+# users commonly want.
+#
+# See http://rubydoc.info/gems/rspec-core/RSpec/Core/Configuration
+RSpec.configure do |config|
+# The settings below are suggested to provide a good initial experience
+# with RSpec, but feel free to customize to your heart's content.
+
+  # These two settings work together to allow you to limit a spec run
+  # to individual examples or groups you care about by tagging them with
+  # `:focus` metadata. When nothing is tagged with `:focus`, all examples
+  # get run.
+  config.filter_run :focus
+  config.run_all_when_everything_filtered = true
+
+  # Many RSpec users commonly either run the entire suite or an individual
+  # file, and it's useful to allow more verbose output when running an
+  # individual spec file.
+  if config.files_to_run.one?
+    # Use the documentation formatter for detailed output,
+    # unless a formatter has already been configured
+    # (e.g. via a command-line flag).
+    config.default_formatter = 'doc'
+  end
+
+  # Print the 10 slowest examples and example groups at the
+  # end of the spec run, to help surface which specs are running
+  # particularly slow.
+  config.profile_examples = 10
+
+  # Run specs in random order to surface order dependencies. If you find an
+  # order dependency and want to debug it, you can fix the order by providing
+  # the seed, which is printed after each run.
+  #     --seed 1234
+  config.order = :random
+
+  # Seed global randomization in this process using the `--seed` CLI option.
+  # Setting this allows you to use `--seed` to deterministically reproduce
+  # test failures related to randomization by passing the same `--seed` value
+  # as the one that triggered the failure.
+  Kernel.srand config.seed
+
+  # rspec-expectations config goes here. You can use an alternate
+  # assertion/expectation library such as wrong or the stdlib/minitest
+  # assertions if you prefer.
+  config.expect_with :rspec do |expectations|
+    # Enable only the newer, non-monkey-patching expect syntax.
+    # For more details, see:
+    #   - http://myronmars.to/n/dev-blog/2012/06/rspecs-new-expectation-syntax
+    expectations.syntax = :expect
+  end
+
+  # rspec-mocks config goes here. You can use an alternate test double
+  # library (such as bogus or mocha) by changing the `mock_with` option here.
+  config.mock_with :rspec do |mocks|
+    # Enable only the newer, non-monkey-patching expect syntax.
+    # For more details, see:
+    #   - http://teaisaweso.me/blog/2013/05/27/rspecs-new-message-expectation-syntax/
+    mocks.syntax = :expect
+
+    # Prevents you from mocking or stubbing a method that does not exist on
+    # a real object. This is generally recommended.
+    mocks.verify_partial_doubles = true
+  end
+
+end

data/spec/unit/client_spec.rb

@@ -0,0 +1,21 @@
+module Ddr
+  module FileTools
+    RSpec.describe Client do
+  
+      describe "extracting text" do
+        let(:file_path) { File.expand_path("../../fixtures/sample.docx", __FILE__) }
+        it "should extract the text content of the file" do
+          expect(subject.run_tool(:text, file_path).output).to match(/This is a sample document./)
+        end
+      end
+
+      describe "extracting metadata" do
+        let(:file_path) { File.expand_path("../../fixtures/blue-devil.png", __FILE__) }
+        it "should extract technical metadata from the file" do
+          expect(subject.run_tool(:metadata, file_path).output.length).to_not eq(0)
+        end
+      end
+
+    end
+  end    
+end

metadata

@@ -0,0 +1,121 @@
+--- !ruby/object:Gem::Specification
+name: ddr-filetools
+version: !ruby/object:Gem::Version
+  version: 0.4.0
+platform: ruby
+authors:
+- David Chandek-Stark
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2015-02-13 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.6'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.6'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+description: Pluggable text and metadata extraction service.
+email:
+- dchandekstark@gmail.com
+executables:
+- ".keep"
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rspec"
+- ".travis.yml"
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/.keep
+- ddr-filetools.gemspec
+- lib/ddr-extraction.rb
+- lib/ddr/filetools.rb
+- lib/ddr/filetools/client.rb
+- lib/ddr/filetools/command.rb
+- lib/ddr/filetools/defaults.rb
+- lib/ddr/filetools/fits.rb
+- lib/ddr/filetools/metadata_command.rb
+- lib/ddr/filetools/ocr_command.rb
+- lib/ddr/filetools/provider.rb
+- lib/ddr/filetools/result.rb
+- lib/ddr/filetools/text_command.rb
+- lib/ddr/filetools/tika.rb
+- lib/ddr/filetools/tool.rb
+- lib/ddr/filetools/version.rb
+- lib/tasks/ddr_filetools.rake
+- spec/fixtures/blue-devil.png
+- spec/fixtures/sample.docx
+- spec/fixtures/sample.pdf
+- spec/spec_helper.rb
+- spec/unit/client_spec.rb
+homepage: https://github.com/duke-libraries/ddr-filetools
+licenses:
+- BSD-3-Clause
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.2.2
+signing_key: 
+specification_version: 4
+summary: Pluggable text and metadata extraction service.
+test_files:
+- spec/fixtures/blue-devil.png
+- spec/fixtures/sample.docx
+- spec/fixtures/sample.pdf
+- spec/spec_helper.rb
+- spec/unit/client_spec.rb