ddr-extraction 0.2.1 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 1e6c32f2c82cc25e4a8c7b65b98becae648925ad
-  data.tar.gz: 9f8f4da7e5e4c7db8fc66dcddda8bff671642572
+  metadata.gz: 2975444782fb458e450a8acb9ec54a690d457241
+  data.tar.gz: 5d2cedbf73284f32b7c5497d6a49d56aaaa1c0ee
 SHA512:
-  metadata.gz: 8411bd09f0cb81d7cb16827fd3098f474c21fcf407da1ec38e2936fc3fc9982e892de08cf6ed747901fce92b6aef3d8cfa22c4a6f4bb2a4269fbb659adca539c
-  data.tar.gz: 30967c3ecb2bd79fa5895e7198cb15d29b45be5eceb876ce2a60dd94371b4387296daac635b1fea7a0f38270dd59e62fdc11ed0da2a9be087a4067a2b64cffb9
+  metadata.gz: f72b6e7224081f193cbf4b8977167d84f995e467346d60590c48133746e74cd29622ee8d1d6fb3dbc426ec37339ec5e7985c08d5e3755268b1a46dac02bb6f07
+  data.tar.gz: ec401d6d43201348ac80bf5ce13dfcfd854918ed39cb49bdc990c86fd3b682ced86db425387e481f57e9f39bdbd6e7623ee88b44d8f5674052e5e8d07cabb8d5

data/README.md

@@ -6,7 +6,7 @@ Pluggable file text and metadata extraction service.
 
 Add this line to your application's Gemfile:
 
-    gem 'ddr-extractor'
+    gem 'ddr-extraction'
 
 And then execute:
 
@@ -14,68 +14,45 @@ And then execute:
 
 Or install it yourself as:
 
-    $ gem install ddr-extractor
+    $ gem install ddr-extraction
 
-## Usage
+## Dependencies
+
+The gem has no external dependencies of its own. Consult the documentation for each extraction tool used by your configuration.
+
+## Configuration
+
+`Ddr::Extraction` includes default configurations for [Aapche Tika](http://tika.apache.org/) (text and metadata extraction) and [FITS](http://fitstool.org/) (metadata only).  Tika is set as the default adapter when one is not specified to the builder.
 
+```ruby
+require "ddr-extraction
+Ddr::Extraction.load_defaults!
 ```
->> extractor = Ddr::Extraction::Extractor.new
-=> #<Ddr::Extraction::Extractor:0x007fc2851dcfa0>
 
->> text = extractor.extract(:text, "spec/fixtures/sample.docx")
-=> #<IO:fd 11>
+There are rake tasks for downloading Tika and FITS to expected locations.
+
+```sh
+rake tika:download
+rake fits:download
+```
+
+Configuration Example
 
+```ruby
+Ddr::Extraction.configure do |config|
+  config.adapters.default = :tika # Use Tika as the default adapter
+  config.adapters.tika.path = "/path/to/tika-app.jar"
+  config.adapters.fits.path = "/path/to/fits.sh"
+end
+```
+
+## Usage
+
+```
+>> extractor = Ddr::Extraction.build_extractor
+>> text = extractor.extract(:text, "spec/fixtures/sample.docx")
 >> puts text.read
 This is a sample document.
-
->> metadata = extractor.extract(:metadata, "spec/fixtures/blue-devil.png")
-=> #<IO:fd 12>
-
->> puts metadata.read
-<?xml version="1.0" encoding="UTF-8"?>
-<fits xmlns="http://hul.harvard.edu/ois/xml/ns/fits/fits_output" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://hul.harvard.edu/ois/xml/ns/fits/fits_output http://hul.harvard.edu/ois/xml/xsd/fits/fits_output.xsd" version="0.8.3" timestamp="11/12/14 12:36 PM">
-  <identification>
-    <identity format="Portable Network Graphics" mimetype="image/png" toolname="FITS" toolversion="0.8.3">
-      <tool toolname="Exiftool" toolversion="9.13" />
-      <tool toolname="Droid" toolversion="6.1.3" />
-      <tool toolname="ffident" toolversion="0.2" />
-      <tool toolname="Tika" toolversion="1.3" />
-      <version toolname="Droid" toolversion="6.1.3">1.0</version>
-      <externalIdentifier toolname="Droid" toolversion="6.1.3" type="puid">fmt/11</externalIdentifier>
-    </identity>
-  </identification>
-  <fileinfo>
-    <lastmodified toolname="Exiftool" toolversion="9.13" status="SINGLE_RESULT">2014:11:12 12:24:18-05:00</lastmodified>
-    <filepath toolname="OIS File Information" toolversion="0.2" status="SINGLE_RESULT">/path/to/spec/fixtures/blue-devil.png</filepath>
-    <filename toolname="OIS File Information" toolversion="0.2" status="SINGLE_RESULT">blue-devil.png</filename>
-    <size toolname="OIS File Information" toolversion="0.2" status="SINGLE_RESULT">75005</size>
-    <md5checksum toolname="OIS File Information" toolversion="0.2" status="SINGLE_RESULT">e6a5d16da2fbe65311952e2d8b04f069</md5checksum>
-    <fslastmodified toolname="OIS File Information" toolversion="0.2" status="SINGLE_RESULT">1415813058000</fslastmodified>
-  </fileinfo>
-  <filestatus />
-  <metadata>
-    <image>
-      <compressionScheme toolname="Exiftool" toolversion="9.13" status="CONFLICT">Deflate/Inflate</compressionScheme>
-      <compressionScheme toolname="Tika" toolversion="1.3" status="CONFLICT">Deflate</compressionScheme>
-      <imageWidth toolname="Exiftool" toolversion="9.13">200</imageWidth>
-      <imageHeight toolname="Exiftool" toolversion="9.13">200</imageHeight>
-      <orientation toolname="Tika" toolversion="1.3" status="SINGLE_RESULT">normal*</orientation>
-    </image>
-  </metadata>
-  <statistics fitsExecutionTime="791">
-    <tool toolname="OIS Audio Information" toolversion="0.1" status="did not run" />
-    <tool toolname="ADL Tool" toolversion="0.1" status="did not run" />
-    <tool toolname="Jhove" toolversion="1.5" executionTime="556" />
-    <tool toolname="file utility" toolversion="5.04" executionTime="623" />
-    <tool toolname="Exiftool" toolversion="9.13" executionTime="664" />
-    <tool toolname="Droid" toolversion="6.1.3" executionTime="147" />
-    <tool toolname="NLNZ Metadata Extractor" toolversion="3.4GA" executionTime="366" />
-    <tool toolname="OIS File Information" toolversion="0.2" executionTime="142" />
-    <tool toolname="OIS XML Metadata" toolversion="0.2" status="did not run" />
-    <tool toolname="ffident" toolversion="0.2" executionTime="369" />
-    <tool toolname="Tika" toolversion="1.3" executionTime="356" />
-  </statistics>
-</fits>
 ```
 
 ## Contributing

data/lib/ddr/extraction.rb

@@ -1,26 +1,38 @@
 require_relative "extraction/version"
 require_relative "extraction/configuration"
 require_relative "extraction/extractor"
+require_relative "extraction/adapters"
 
 module Ddr
   #
-  # Ddr::Extraction - A file text and metadata extraction service.
+  # Ddr::Extraction - A pluggable content extraction service.
   #
   module Extraction
 
     class << self
 
+      # Returns the service configuration
       def config
         @config ||= Configuration.new
       end
 
-      # Yields a configuration object for the service
+      # Yields the service configuration to a block
       def configure
         yield config
       end
-      
+
+      # Loads default configuration settings
+      def load_defaults!
+        require_relative "extraction/defaults"
+      end
+
+      def build_extractor(adapter_name = nil)
+        Extractor.build(adapter_name)
+      end
+
     end
 
   end
 end
 
+Dir[File.join(__dir__, "extraction", "adapters", "*_adapter.rb")].each { |adapter| require(adapter) }

data/lib/ddr/extraction/adapters.rb

@@ -1,25 +1,59 @@
+require_relative "adapters/registry"
+
 module Ddr
   module Extraction
     module Adapters
 
-      KNOWN_ADAPTERS = [:fits, :tika]
-
       class << self
+        
+        # Accessor for the name of the default adapter
+        attr_accessor :default
+
+        # Return the requested adapter by name.
+        #   If a name is not supplied, return the default adapter.
+        # @see .get_default_adapter
+        #
+        # @param adapter_name [Symbol] the name of the requested adapter.
+        # @return [Class] the adapter class requested.
+        def get_adapter(adapter_name = nil)
+          if adapter_name
+            Registry.instance.adapters[adapter_name.to_sym]
+          else
+            get_default_adapter
+          end
+        end
 
-        def get_adapter(adapter_name)
-          require_relative "adapters/#{adapter_name}_adapter"
-          class_name = "#{adapter_name.to_s.capitalize}Adapter"
-          const_get(class_name.to_sym, false)
+        # Return the default adapter.
+        # Raises an exception if the default adapter has not been configured.
+        def get_default_adapter
+          raise "The default adapter has not been configured." unless default
+          get_adapter(default)
         end
 
-        KNOWN_ADAPTERS.each do |adapter|
-          define_method(adapter) do
-            get_adapter(adapter)
+        # Registers an adapter.
+        # @see Registry#register
+        #
+        # @param name [Symbol] the name of the adapter.
+        # @param adapter [Class] the adapter class to register.
+        def register(name, adapter)
+          Registry.instance.register(name, adapter)
+        end
+
+        # Creates methods to access each adapter.
+        Registry.instance.adapters.each do |name, adapter|
+          define_method(name) do
+            adapter
           end
         end
 
+        def method_missing(name, *args)
+          return get_adapter(name) if Registry.instance.adapters.key?(name.to_sym)
+          super
+        end
       end
 
     end
   end
 end
+
+Dir[File.join(__dir__, "adapters", "*_adapter.rb")].each { |adapter| require(adapter) }

data/lib/ddr/extraction/adapters/adapter.rb

@@ -0,0 +1,55 @@
+module Ddr
+  module Extraction
+    module Adapters
+      class Adapter
+
+        # Supported extraction output types
+        OUTPUT_TYPES = [:text, :metadata]
+
+        class << self
+          # Register the adapter
+          def register(adapter_name)
+            Ddr::Extraction::Adapters.register(adapter_name, self)
+          end
+        end
+
+        # Extract a kind of output from the file path
+        #
+        # @param output [Symbol] the kind of output, `:text` or `:metadata`
+        # @param file_path [String] path to the file to be processed
+        # @return [IO] the result of the extraction
+        # @api public
+        def extract(output, file_path)
+          raise ArgumentError, "Output type must be one of #{OUTPUT_TYPES}." unless OUTPUT_TYPES.include?(output)
+          raise IOError, "File not found: #{file_path}" unless File.exist?(file_path)
+          execute(command(output, file_path))
+        end
+
+        private
+
+        # Returns the command to be executed
+        #
+        # @param output [Symbol] the kind of output.
+        # @param file_path [String] path to the file to be processed.
+        # @return [String, Array] the command as a String or Array
+        # @see #extract
+        # @see #execute
+        # @api private
+        def command(output, file_path)
+          raise NotImplementedError, "The `command' instance method must be implemented by the adapter."
+        end
+
+        # Executes the command in a subprocess.
+        #
+        # @param cmd [String, Array] the command as a String or Array
+        # @see Ruby documentation for IO.popen
+        # @return [IO] the output of the command.
+        # @api private
+        def execute(cmd)
+          IO.popen(cmd)
+        end
+        
+      end
+    end
+  end
+end

data/lib/ddr/extraction/adapters/fits_adapter.rb

@@ -1,22 +1,26 @@
+require_relative "adapter"
+
 module Ddr
   module Extraction 
     module Adapters
-      class FitsAdapter
+      class FitsAdapter < Adapter
 
-        # Return metadata extracted from file
-        #
-        # @param file [String] the file from which to extract metadata.
-        # @return [IO] the output
-        def extract_metadata(file)
-          IO.popen([self.class.path, "-i", file])
-        end
+        register :fits
 
         class << self
           # Path to FITS executable (fits.sh or fits.bat)
           attr_accessor :path
         end
 
+        private
+
+        def command(output, file_path)
+          raise "This adapter only supports :metadata output." unless output == :metadata
+          [self.class.path, "-i", file_path]
+        end
+
       end
     end
   end
 end
+

data/lib/ddr/extraction/adapters/null_adapter.rb

@@ -0,0 +1,21 @@
+require_relative "adapter"
+
+module Ddr
+  module Extraction
+    module Adapters
+      class NullAdapter < Adapter
+
+        register :null
+
+        private
+
+        def command(output, file_path)
+          "echo"
+        end
+
+      end
+
+    end
+  end
+end
+

data/lib/ddr/extraction/adapters/registry.rb

@@ -0,0 +1,42 @@
+require "singleton"
+require_relative "adapter"
+
+module Ddr
+  module Extraction
+    module Adapters
+      #
+      # Registry of adapter names and classes
+      #
+      class Registry
+        include Singleton
+
+        attr_reader :adapters
+
+        def initialize
+          @adapters = {}
+        end
+
+        # Registers an adapter
+        #
+        # @param name [Symbol] the name of the adapter.
+        # @param adapter [Class] the adapter to be registered.
+        def register(name, adapter)
+          name = name.to_sym
+          validate!(name, adapter)
+          adapters[name] = adapter
+        end
+
+        private
+
+        def validate!(name, adapter)
+          raise "Another adapter is registered under the name :#{name}." if adapters.key?(name)
+          unless adapter < Adapter
+            raise ArgumentError, "Only subclasses of Ddr::Extraction::Adapters::Adapter may be registered." 
+          end
+          raise "The adapter #{adapter.to_s} is already registered." if adapters.value?(adapter)
+        end
+      end
+
+    end
+  end
+end

data/lib/ddr/extraction/adapters/tika_adapter.rb

@@ -1,15 +1,11 @@
+require_relative "adapter"
+
 module Ddr
   module Extraction
     module Adapters
-      class TikaAdapter
-
-        # Extract text from file
-        #
-        # @param file [String] path to file from which to extract text
-        # @return [IO] the output
-        def extract_text(file)
-          IO.popen(["java", "-jar", self.class.path, "--text", file])
-        end
+      class TikaAdapter < Adapter
+
+        register :tika
 
         class << self
           # Path to tika-app.jar
@@ -17,9 +13,25 @@ module Ddr
 
           # Tika server port (optional, required for server)
           attr_accessor :port
-        end          
+        end  
+
+        private
+
+        def command(output, file_path)
+          ["java", "-jar", self.class.path, output_options(output), file_path].flatten
+        end
+
+        def output_options(output)
+          case output
+          when :text
+            "--text"
+          when :metadata
+            ["--metadata", "--xml"]
+          end
+        end
 
       end
+
     end
   end
 end

data/lib/ddr/extraction/configuration.rb

@@ -1,22 +1,16 @@
-require_relative "adapter"
 require_relative "adapters"
 
 module Ddr
   module Extraction
     class Configuration
 
+      # Returns an object have settable attributes for adapters.
       def adapters
         config = Adapters
         yield config if block_given?
         config
       end
 
-      def adapter
-        config = Adapter
-        yield config if block_given?
-        config
-      end
-
     end
   end
 end

data/lib/ddr/extraction/defaults.rb

@@ -3,8 +3,7 @@ require "ddr-extraction"
 bin_dir = File.expand_path("../../../../bin", __FILE__)
 
 Ddr::Extraction.configure do |config|
-  config.adapter.text = :tika
-  config.adapter.metadata = :fits
+  config.adapters.default = :tika
   config.adapters.tika.path = File.join(bin_dir, "tika-app.jar")
   config.adapters.fits.path = File.join(bin_dir, "fits", "fits.sh")
 end

data/lib/ddr/extraction/extractor.rb

@@ -1,22 +1,36 @@
+require "delegate"
 require_relative "adapters"
 
 module Ddr
   module Extraction
-    class Extractor
+    #
+    # The Extractor is the main public class.
+    #
+    # It works by delegating to an adapter that does the real work.
+    #
+    #     extractor = Ddr::Extraction::Extractor.build(:tika)
+    #     text = extractor.extract(:text, "/path/to/text/file")
+    #     puts text.read
+    #     ...
+    #
+    class Extractor < ::SimpleDelegator
 
-      # Extracts a type of content from a file
-      #
-      # @param type [Symbol] the type of content to extract, `:text` or `:metadata`.
-      # @param file [String] path to file from which to extract content.
-      # @return [IO] the output
-      def extract(type, file)
-        adapter(type).send("extract_#{type}", file)
-      end
+      class << self
 
-      private
+        # Returns/yields an extractor instance
+        #
+        # @param adapter_name [Symbol] the name of the adapter to plug in.
+        #        If not given, a default adapter will be used, if
+        #        Ddr::Extraction::Adapters.default has been set with 
+        #        the name of the default adapter.
+        # 
+        def build(adapter_name = nil)
+          adapter = Adapters.get_adapter(adapter_name)
+          extractor = new(adapter.new)
+          yield extractor if block_given?
+          extractor
+        end
 
-      def adapter(type)
-        Adapter.build_adapter(type)
       end
 
     end

data/lib/ddr/extraction/version.rb

@@ -1,5 +1,5 @@
 module Ddr
   module Extraction
-    VERSION = "0.2.1"
+    VERSION = "0.3.0"
   end
 end

data/spec/unit/extractor_spec.rb

@@ -3,16 +3,18 @@ module Ddr
     RSpec.describe Extractor do
 
       describe "extracting text" do
-        let(:file) { File.expand_path("../../fixtures/sample.docx", __FILE__) }
+        subject { described_class.build(:tika) }
+        let(:file_path) { File.expand_path("../../fixtures/sample.docx", __FILE__) }
         it "should extract the text content of the file" do
-          expect(subject.extract(:text, file).read).to match(/This is a sample document./)
+          expect(subject.extract(:text, file_path).read).to match(/This is a sample document./)
         end
       end
 
       describe "extracting metadata" do
-        let(:file) { File.expand_path("../../fixtures/blue-devil.png", __FILE__) }
+        subject { described_class.build(:tika) }
+        let(:file_path) { File.expand_path("../../fixtures/blue-devil.png", __FILE__) }
         it "should extract technical metadata from the file" do
-          expect(subject.extract(:metadata, file).read.length).to_not eq(0)
+          expect(subject.extract(:metadata, file_path).read.length).to_not eq(0)
         end
       end
     

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ddr-extraction
 version: !ruby/object:Gem::Version
-  version: 0.2.1
+  version: 0.3.0
 platform: ruby
 authors:
 - David Chandek-Stark
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-11-13 00:00:00.000000000 Z
+date: 2014-11-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -71,9 +71,11 @@ files:
 - ddr-extraction.gemspec
 - lib/ddr-extraction.rb
 - lib/ddr/extraction.rb
-- lib/ddr/extraction/adapter.rb
 - lib/ddr/extraction/adapters.rb
+- lib/ddr/extraction/adapters/adapter.rb
 - lib/ddr/extraction/adapters/fits_adapter.rb
+- lib/ddr/extraction/adapters/null_adapter.rb
+- lib/ddr/extraction/adapters/registry.rb
 - lib/ddr/extraction/adapters/tika_adapter.rb
 - lib/ddr/extraction/configuration.rb
 - lib/ddr/extraction/defaults.rb

data/lib/ddr/extraction/adapter.rb

@@ -1,21 +0,0 @@
-require "delegate"
-require_relative "adapters"
-
-module Ddr
-  module Extraction
-    class Adapter < ::SimpleDelegator
-
-      class << self
-        # Accessors for adapter types
-        attr_accessor :text, :metadata
-
-        def build_adapter(type)
-          adapter_name = send(type)
-          adapter = Adapters.get_adapter(adapter_name)
-          new(adapter.new)
-        end
-      end
-
-    end
-  end
-end