ddr-extraction 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: c5403a9bee4dc10e433bca20f0a8ee11178a99dc
+  data.tar.gz: d04210fc66bfaa0e8e368fb82f3236c7c2510d38
+SHA512:
+  metadata.gz: 7e300ee401d7cb26eb2cc259723b3a94009f9c2b4296dfa98870a401a67c4ba4c185fe8a8c5a85444c3db0d55c5a4916ef0d5d91cbdfa00356a904ef1cb2bdd0
+  data.tar.gz: 6ad6b2c009439ea8b378df9cfc6accd45c3f169b7967bf8047b3864cc7ae91caa1eefc87779aaff6128e1b15b86e2dec7a8d69b4099d2ed278c747a8bce863d4

data/.gitignore

@@ -0,0 +1,4 @@
+Gemfile.lock
+bin/tika-*
+bin/fits-*
+tmp

data/.rspec

@@ -0,0 +1,3 @@
+--color
+--warnings
+--require spec_helper

data/Gemfile

@@ -0,0 +1,3 @@
+source 'https://rubygems.org'
+
+gemspec

data/LICENSE.txt

@@ -0,0 +1,12 @@
+Copyright (c) 2014, Duke University
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

data/README.md

@@ -0,0 +1,29 @@
+# Ddr::Extractor
+
+Generic file text and metadata extraction service.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'ddr-extractor'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install ddr-extractor
+
+## Usage
+
+TODO
+
+## Contributing
+
+1. Fork it ( https://github.com/[my-github-username]/ddr_extractor/fork )
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create a new Pull Request

data/Rakefile

@@ -0,0 +1,3 @@
+require "bundler/gem_tasks"
+
+load "tasks/ddr_extraction.rake"

data/ddr-extraction.gemspec

@@ -0,0 +1,24 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'ddr/extraction/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "ddr-extraction"
+  spec.version       = Ddr::Extraction::VERSION
+  spec.authors       = ["David Chandek-Stark"]
+  spec.email         = ["dchandekstark@gmail.com"]
+  spec.summary       = "File text and metadata extraction service."
+  spec.description   = "File text and metadata extraction service."
+  spec.homepage      = "https://github.com/duke-libraries/ddr-extraction"
+  spec.license       = "BSD-3-Clause"
+
+  spec.files         = `git ls-files -z`.split("\x0")
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.6"
+  spec.add_development_dependency "rake"
+  spec.add_development_dependency "rspec", "~> 3.0"
+end

data/lib/ddr-extraction.rb

@@ -0,0 +1 @@
+require "ddr/extraction"

data/lib/ddr/extraction.rb

@@ -0,0 +1,52 @@
+require "ddr/extraction/version"
+require "ddr/extraction/adapters"
+require "ddr/extraction/extractor"
+
+module Ddr
+  #
+  # Ddr::Extraction - A file text and metadata extraction service.
+  #
+  module Extraction
+
+    class << self
+
+      attr_accessor :text_adapter, :metadata_adapter
+
+      # Yields a configurable object for the named adapter.
+      #
+      # @param adapter [Symbol] the name of the adapter - e.g., `:tika`, `:fits`
+      def configure_adapter(adapter, &block)
+        yield Adapters.get_adapter(adapter)
+      end
+
+      def adapters
+        Adapters.config
+      end
+
+      def set_defaults
+        bin_dir = File.expand_path("../../../bin", __FILE__)
+
+        configure_adapter :tika do |tika|
+          tika.version = "1.6"
+          tika.path = File.join(bin_dir, "tika-app.jar")
+          tika.download_url = "http://archive.apache.org/dist/tika/tika-app-#{tika.version}.jar"
+          tika.checksum = "99df0d8c3f6a2be498d275053e611fb5afdf0a9d"
+          tika.checksum_type = :SHA1
+        end
+
+        configure_adapter :fits do |fits|
+          fits.version = "0.8.3"
+          fits.path = File.join(bin_dir, "fits-#{fits.version}", "fits.sh")
+          fits.download_url = "http://projects.iq.harvard.edu/files/fits/files/fits-#{fits.version}.zip"
+        end
+
+        adapters.text = :tika
+        adapters.metadata = :fits
+      end
+
+    end
+
+  end
+end
+
+Ddr::Extraction.set_defaults

data/lib/ddr/extraction/adapters.rb

@@ -0,0 +1,25 @@
+module Ddr
+  module Extraction
+    module Adapters
+
+      def self.get_adapter(adapter_name)
+        require_relative "adapters/#{adapter_name}_adapter"
+        class_name = "#{adapter_name.to_s.capitalize}Adapter"
+        const_get(class_name.to_sym, false)
+      end      
+
+      def self.build_adapter(type)
+        adapter_name = config.send(type)
+        adapter = get_adapter(adapter_name)
+        adapter.new
+      end
+
+      Config = Struct.new(:text, :metadata)
+
+      def self.config
+        @@config ||= Config.new
+      end
+
+    end
+  end
+end

data/lib/ddr/extraction/adapters/adapter.rb

@@ -0,0 +1,15 @@
+module Ddr
+  module Extraction
+    module Adapters
+      class Adapter
+
+        class << self
+          def config
+            yield self
+          end
+        end
+
+      end
+    end
+  end
+end

data/lib/ddr/extraction/adapters/fits_adapter.rb

@@ -0,0 +1,30 @@
+require_relative "adapter"
+
+module Ddr
+  module Extraction 
+    module Adapters
+      class FitsAdapter < Adapter
+
+        # Return metadata extracted from file
+        #
+        # @param file [String] the file from which to extract metadata.
+        # @return [IO]
+        def extract_metadata(file)
+          IO.popen([self.class.path, "-i", file])
+        end
+
+        class << self
+          # FITS version
+          attr_accessor :version
+
+          # Path to FITS executable (fits.sh or fits.bat)
+          attr_accessor :path
+
+          # URL to download distribution
+          attr_accessor :download_url
+        end
+
+      end
+    end
+  end
+end

data/lib/ddr/extraction/adapters/metadata_extraction_adapter.rb

@@ -0,0 +1,13 @@
+module Ddr
+  module Extraction
+    module Adapters
+      class MetadataExtractionAdapter
+
+        def extract_metadata(file)
+          NotImplemented
+        end
+
+      end
+    end
+  end
+end

data/lib/ddr/extraction/adapters/text_extraction_adapter.rb

@@ -0,0 +1,13 @@
+module Ddr
+  module Extraction
+    module Adapters
+      class TextExtractionAdapter
+
+        def extract_text(file)
+          NotImplemented
+        end
+
+      end
+    end
+  end
+end

data/lib/ddr/extraction/adapters/tika_adapter.rb

@@ -0,0 +1,42 @@
+require_relative "adapter"
+
+module Ddr
+  module Extraction
+    module Adapters
+      class TikaAdapter < Adapter
+
+        # Extract text from file
+        #
+        # @param file [String] path to file from which to extract text
+        # @return [IO] 
+        def extract_text(file)
+          IO.popen(["java", "-jar", self.class.path, "--text", file])
+        end
+
+        class << self
+          # Tika version
+          attr_accessor :version
+
+          # Path to tika-app.jar
+          attr_accessor :path
+
+          # Base command
+          attr_accessor :command
+
+          # URL to download distribution
+          attr_accessor :download_url
+
+          # Tika distribution checksum
+          attr_accessor :checksum
+
+          # Tika distribution checksum type
+          attr_accessor :checksum_type
+
+          # Tika server port (optional, required for server)
+          attr_accessor :port
+        end          
+
+      end
+    end
+  end
+end

data/lib/ddr/extraction/extractor.rb

@@ -0,0 +1,32 @@
+require_relative "adapters"
+
+module Ddr
+  module Extraction
+    class Extractor
+      extend Forwardable
+
+      def_delegator :text_adapter, :extract_text
+      def_delegator :metadata_adapter, :extract_metadata
+
+      # Extracts a type of content from a file
+      #
+      # @param type [Symbol] the type of content to extract, `:text` or `:metadata`.
+      # @param file [String] path to file from which to extract content.
+      # @return 
+      def extract(type, file)
+        send("extract_#{type}", file)
+      end      
+
+      private
+
+      def text_adapter
+        @text_adapter ||= Adapters.build_adapter(:text)
+      end
+
+      def metadata_adapter
+        @metadata_adapter ||= Adapters.build_adapter(:metadata)
+      end
+
+    end
+  end
+end

data/lib/ddr/extraction/metadata_extractor.rb

@@ -0,0 +1,16 @@
+require "ddr/extraction/adapters"
+
+module Ddr
+  module Extraction
+    class MetadataExtractor
+      extend Forwardable
+
+      def_delegator :@adapter, :extract_metadata
+
+      def initialize
+        @adapter = Ddr::Extraction::Adapters.get_metadata_extraction_adapter
+      end
+
+    end
+  end
+end

data/lib/ddr/extraction/version.rb

@@ -0,0 +1,5 @@
+module Ddr
+  module Extraction
+    VERSION = "0.1.0"
+  end
+end

data/lib/tasks/ddr_extraction.rake

@@ -0,0 +1,48 @@
+require "ddr-extraction"
+require "openssl"
+
+DOWNLOAD_DIR = File.absolute_path("tmp")
+
+namespace :tika do
+  desc "Download Tika app"
+  task :download => :download_dir do
+    tika_app = File.join(DOWNLOAD_DIR, "tika-app.jar")
+    Ddr::Extraction::Adapters::TikaAdapter.config do |tika|
+      puts "Downloading Tika app ... "
+      system "curl -L #{tika.download_url} -o #{tika_app}"
+      puts "Verifiying checksum ... "
+      digest = OpenSSL::Digest.const_get(tika.checksum_type).new
+      digest << File.read(tika.path)
+      if digest.to_s == tika.checksum
+        FileUtils.mv(tika_app, tika.path)
+      else
+        puts "Checksums do not match!"
+      end
+    end
+  end
+
+  # namespace :server do
+  #   desc "Start the Tika server"
+  #   task :start => :environment do      
+  #   end
+  # end
+end
+
+namespace :fits do
+  desc "Download FITS tool"
+  task :download => :download_dir do
+    fits_tool = File.join(DOWNLOAD_DIR, "fits.zip")
+    Ddr::Extraction::Adapters::FitsAdapter.config do |fits|
+      puts "Downloading FITS tool ... "
+      system "curl -L #{fits.download_url} -o #{fits_tool}"
+      # Unzip options: convert text files, force overwrite, extra quiet
+      system "unzip -a -o -qq -d bin #{fits_tool}"
+      FileUtils.chmod(0755, fits.path)
+    end
+  end
+end
+
+task :download_dir do
+  FileUtils.mkdir(DOWNLOAD_DIR) unless Dir.exists?(DOWNLOAD_DIR)
+end
+

data/spec/spec_helper.rb

@@ -0,0 +1,80 @@
+require "ddr-extraction"
+
+# This file was generated by the `rspec --init` command. Conventionally, all
+# specs live under a `spec` directory, which RSpec adds to the `$LOAD_PATH`.
+# The generated `.rspec` file contains `--require spec_helper` which will cause this
+# file to always be loaded, without a need to explicitly require it in any files.
+#
+# Given that it is always loaded, you are encouraged to keep this file as
+# light-weight as possible. Requiring heavyweight dependencies from this file
+# will add to the boot time of your test suite on EVERY test run, even for an
+# individual file that may not need all of that loaded. Instead, make a
+# separate helper file that requires this one and then use it only in the specs
+# that actually need it.
+#
+# The `.rspec` file also contains a few flags that are not defaults but that
+# users commonly want.
+#
+# See http://rubydoc.info/gems/rspec-core/RSpec/Core/Configuration
+RSpec.configure do |config|
+# The settings below are suggested to provide a good initial experience
+# with RSpec, but feel free to customize to your heart's content.
+
+  # These two settings work together to allow you to limit a spec run
+  # to individual examples or groups you care about by tagging them with
+  # `:focus` metadata. When nothing is tagged with `:focus`, all examples
+  # get run.
+  config.filter_run :focus
+  config.run_all_when_everything_filtered = true
+
+  # Many RSpec users commonly either run the entire suite or an individual
+  # file, and it's useful to allow more verbose output when running an
+  # individual spec file.
+  if config.files_to_run.one?
+    # Use the documentation formatter for detailed output,
+    # unless a formatter has already been configured
+    # (e.g. via a command-line flag).
+    config.default_formatter = 'doc'
+  end
+
+  # Print the 10 slowest examples and example groups at the
+  # end of the spec run, to help surface which specs are running
+  # particularly slow.
+  config.profile_examples = 10
+
+  # Run specs in random order to surface order dependencies. If you find an
+  # order dependency and want to debug it, you can fix the order by providing
+  # the seed, which is printed after each run.
+  #     --seed 1234
+  config.order = :random
+
+  # Seed global randomization in this process using the `--seed` CLI option.
+  # Setting this allows you to use `--seed` to deterministically reproduce
+  # test failures related to randomization by passing the same `--seed` value
+  # as the one that triggered the failure.
+  Kernel.srand config.seed
+
+  # rspec-expectations config goes here. You can use an alternate
+  # assertion/expectation library such as wrong or the stdlib/minitest
+  # assertions if you prefer.
+  config.expect_with :rspec do |expectations|
+    # Enable only the newer, non-monkey-patching expect syntax.
+    # For more details, see:
+    #   - http://myronmars.to/n/dev-blog/2012/06/rspecs-new-expectation-syntax
+    expectations.syntax = :expect
+  end
+
+  # rspec-mocks config goes here. You can use an alternate test double
+  # library (such as bogus or mocha) by changing the `mock_with` option here.
+  config.mock_with :rspec do |mocks|
+    # Enable only the newer, non-monkey-patching expect syntax.
+    # For more details, see:
+    #   - http://teaisaweso.me/blog/2013/05/27/rspecs-new-message-expectation-syntax/
+    mocks.syntax = :expect
+
+    # Prevents you from mocking or stubbing a method that does not exist on
+    # a real object. This is generally recommended.
+    mocks.verify_partial_doubles = true
+  end
+
+end

data/spec/unit/extractor_spec.rb

@@ -0,0 +1,21 @@
+module Ddr
+  module Extraction
+    RSpec.describe Extractor do
+
+      describe "extracting text" do
+        let(:file) { File.expand_path("../../fixtures/sample.docx", __FILE__) }
+        it "should extract the text content of the file" do
+          expect(subject.extract(:text, file).read).to match(/This is a sample document./)
+        end
+      end
+
+      describe "extracting metadata" do
+        let(:file) { File.expand_path("../../fixtures/blue-devil.png", __FILE__) }
+        it "should extract technical metadata from the file" do
+          expect(subject.extract(:metadata, file).read.length).to_not eq(0)
+        end
+      end
+    
+    end
+  end
+end

metadata

@@ -0,0 +1,117 @@
+--- !ruby/object:Gem::Specification
+name: ddr-extraction
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- David Chandek-Stark
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-11-12 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.6'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.6'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+description: File text and metadata extraction service.
+email:
+- dchandekstark@gmail.com
+executables:
+- ".keep"
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rspec"
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/.keep
+- ddr-extraction.gemspec
+- lib/ddr-extraction.rb
+- lib/ddr/extraction.rb
+- lib/ddr/extraction/adapters.rb
+- lib/ddr/extraction/adapters/adapter.rb
+- lib/ddr/extraction/adapters/fits_adapter.rb
+- lib/ddr/extraction/adapters/metadata_extraction_adapter.rb
+- lib/ddr/extraction/adapters/text_extraction_adapter.rb
+- lib/ddr/extraction/adapters/tika_adapter.rb
+- lib/ddr/extraction/extractor.rb
+- lib/ddr/extraction/metadata_extractor.rb
+- lib/ddr/extraction/version.rb
+- lib/tasks/ddr_extraction.rake
+- spec/fixtures/blue-devil.png
+- spec/fixtures/sample.docx
+- spec/fixtures/sample.pdf
+- spec/spec_helper.rb
+- spec/unit/extractor_spec.rb
+homepage: https://github.com/duke-libraries/ddr-extraction
+licenses:
+- BSD-3-Clause
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.2.2
+signing_key: 
+specification_version: 4
+summary: File text and metadata extraction service.
+test_files:
+- spec/fixtures/blue-devil.png
+- spec/fixtures/sample.docx
+- spec/fixtures/sample.pdf
+- spec/spec_helper.rb
+- spec/unit/extractor_spec.rb