ddr-extraction 0.1.0 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: c5403a9bee4dc10e433bca20f0a8ee11178a99dc
-  data.tar.gz: d04210fc66bfaa0e8e368fb82f3236c7c2510d38
+  metadata.gz: 23d86fadee99408637cfc774cdd62efdaff04e7d
+  data.tar.gz: 03b9386486aa83632f5d96fcc65f209b7ad32952
 SHA512:
-  metadata.gz: 7e300ee401d7cb26eb2cc259723b3a94009f9c2b4296dfa98870a401a67c4ba4c185fe8a8c5a85444c3db0d55c5a4916ef0d5d91cbdfa00356a904ef1cb2bdd0
-  data.tar.gz: 6ad6b2c009439ea8b378df9cfc6accd45c3f169b7967bf8047b3864cc7ae91caa1eefc87779aaff6128e1b15b86e2dec7a8d69b4099d2ed278c747a8bce863d4
+  metadata.gz: afd68b70ea7ee45bed52c1debfabd88c4ebef23d40d997232c4452061f19ac64d0d2791fc90990807d456e468c660ed553538aaaddf59de731643126e3ab89bb
+  data.tar.gz: df8032a81665c2a4592df5a3ff96d3ff8f490dc1c8f16e4f1de482895fe119f48a5dc796e51d0b98aefea78080e1418d2c685a08ba73e94d7df2752322d07098

data/.gitignore

@@ -1,4 +1,5 @@
 Gemfile.lock
 bin/tika-*
 bin/fits-*
+pkg
 tmp

data/README.md

@@ -18,7 +18,65 @@ Or install it yourself as:
 
 ## Usage
 
-TODO
+```
+>> extractor = Ddr::Extraction::Extractor.new
+=> #<Ddr::Extraction::Extractor:0x007fc2851dcfa0>
+
+>> text = extractor.extract(:text, "spec/fixtures/sample.docx")
+=> #<IO:fd 11>
+
+>> puts text.read
+This is a sample document.
+
+>> metadata = extractor.extract(:metadata, "spec/fixtures/blue-devil.png")
+=> #<IO:fd 12>
+
+>> puts metadata.read
+<?xml version="1.0" encoding="UTF-8"?>
+<fits xmlns="http://hul.harvard.edu/ois/xml/ns/fits/fits_output" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://hul.harvard.edu/ois/xml/ns/fits/fits_output http://hul.harvard.edu/ois/xml/xsd/fits/fits_output.xsd" version="0.8.3" timestamp="11/12/14 12:36 PM">
+  <identification>
+    <identity format="Portable Network Graphics" mimetype="image/png" toolname="FITS" toolversion="0.8.3">
+      <tool toolname="Exiftool" toolversion="9.13" />
+      <tool toolname="Droid" toolversion="6.1.3" />
+      <tool toolname="ffident" toolversion="0.2" />
+      <tool toolname="Tika" toolversion="1.3" />
+      <version toolname="Droid" toolversion="6.1.3">1.0</version>
+      <externalIdentifier toolname="Droid" toolversion="6.1.3" type="puid">fmt/11</externalIdentifier>
+    </identity>
+  </identification>
+  <fileinfo>
+    <lastmodified toolname="Exiftool" toolversion="9.13" status="SINGLE_RESULT">2014:11:12 12:24:18-05:00</lastmodified>
+    <filepath toolname="OIS File Information" toolversion="0.2" status="SINGLE_RESULT">/path/to/spec/fixtures/blue-devil.png</filepath>
+    <filename toolname="OIS File Information" toolversion="0.2" status="SINGLE_RESULT">blue-devil.png</filename>
+    <size toolname="OIS File Information" toolversion="0.2" status="SINGLE_RESULT">75005</size>
+    <md5checksum toolname="OIS File Information" toolversion="0.2" status="SINGLE_RESULT">e6a5d16da2fbe65311952e2d8b04f069</md5checksum>
+    <fslastmodified toolname="OIS File Information" toolversion="0.2" status="SINGLE_RESULT">1415813058000</fslastmodified>
+  </fileinfo>
+  <filestatus />
+  <metadata>
+    <image>
+      <compressionScheme toolname="Exiftool" toolversion="9.13" status="CONFLICT">Deflate/Inflate</compressionScheme>
+      <compressionScheme toolname="Tika" toolversion="1.3" status="CONFLICT">Deflate</compressionScheme>
+      <imageWidth toolname="Exiftool" toolversion="9.13">200</imageWidth>
+      <imageHeight toolname="Exiftool" toolversion="9.13">200</imageHeight>
+      <orientation toolname="Tika" toolversion="1.3" status="SINGLE_RESULT">normal*</orientation>
+    </image>
+  </metadata>
+  <statistics fitsExecutionTime="791">
+    <tool toolname="OIS Audio Information" toolversion="0.1" status="did not run" />
+    <tool toolname="ADL Tool" toolversion="0.1" status="did not run" />
+    <tool toolname="Jhove" toolversion="1.5" executionTime="556" />
+    <tool toolname="file utility" toolversion="5.04" executionTime="623" />
+    <tool toolname="Exiftool" toolversion="9.13" executionTime="664" />
+    <tool toolname="Droid" toolversion="6.1.3" executionTime="147" />
+    <tool toolname="NLNZ Metadata Extractor" toolversion="3.4GA" executionTime="366" />
+    <tool toolname="OIS File Information" toolversion="0.2" executionTime="142" />
+    <tool toolname="OIS XML Metadata" toolversion="0.2" status="did not run" />
+    <tool toolname="ffident" toolversion="0.2" executionTime="369" />
+    <tool toolname="Tika" toolversion="1.3" executionTime="356" />
+  </statistics>
+</fits>
+```
 
 ## Contributing
 

data/lib/ddr/extraction.rb

@@ -1,5 +1,5 @@
 require "ddr/extraction/version"
-require "ddr/extraction/adapters"
+require "ddr/extraction/configuration"
 require "ddr/extraction/extractor"
 
 module Ddr
@@ -10,38 +10,12 @@ module Ddr
 
     class << self
 
-      attr_accessor :text_adapter, :metadata_adapter
+      attr_reader :config
 
-      # Yields a configurable object for the named adapter.
-      #
-      # @param adapter [Symbol] the name of the adapter - e.g., `:tika`, `:fits`
-      def configure_adapter(adapter, &block)
-        yield Adapters.get_adapter(adapter)
-      end
-
-      def adapters
-        Adapters.config
-      end
-
-      def set_defaults
-        bin_dir = File.expand_path("../../../bin", __FILE__)
-
-        configure_adapter :tika do |tika|
-          tika.version = "1.6"
-          tika.path = File.join(bin_dir, "tika-app.jar")
-          tika.download_url = "http://archive.apache.org/dist/tika/tika-app-#{tika.version}.jar"
-          tika.checksum = "99df0d8c3f6a2be498d275053e611fb5afdf0a9d"
-          tika.checksum_type = :SHA1
-        end
-
-        configure_adapter :fits do |fits|
-          fits.version = "0.8.3"
-          fits.path = File.join(bin_dir, "fits-#{fits.version}", "fits.sh")
-          fits.download_url = "http://projects.iq.harvard.edu/files/fits/files/fits-#{fits.version}.zip"
-        end
-
-        adapters.text = :tika
-        adapters.metadata = :fits
+      # Yields a configuration object for the service
+      def configure
+        @config ||= Configuration.new
+        yield @config
       end
 
     end
@@ -49,4 +23,3 @@ module Ddr
   end
 end
 
-Ddr::Extraction.set_defaults

data/lib/ddr/extraction/adapter.rb

@@ -0,0 +1,21 @@
+require "delegate"
+require_relative "adapters"
+
+module Ddr
+  module Extraction
+    class Adapter < ::SimpleDelegator
+
+      class << self
+        # Accessors for adapter types
+        attr_accessor :text, :metadata
+
+        def build_adapter(type)
+          adapter_name = send(type)
+          adapter = Adapters.get_adapter(adapter_name)
+          new(adapter.new)
+        end
+      end
+
+    end
+  end
+end

data/lib/ddr/extraction/adapters.rb

@@ -8,18 +8,6 @@ module Ddr
         const_get(class_name.to_sym, false)
       end      
 
-      def self.build_adapter(type)
-        adapter_name = config.send(type)
-        adapter = get_adapter(adapter_name)
-        adapter.new
-      end
-
-      Config = Struct.new(:text, :metadata)
-
-      def self.config
-        @@config ||= Config.new
-      end
-
     end
   end
 end

data/lib/ddr/extraction/adapters/fits_adapter.rb

@@ -1,14 +1,12 @@
-require_relative "adapter"
-
 module Ddr
   module Extraction 
     module Adapters
-      class FitsAdapter < Adapter
+      class FitsAdapter
 
         # Return metadata extracted from file
         #
         # @param file [String] the file from which to extract metadata.
-        # @return [IO]
+        # @return [IO] the output
         def extract_metadata(file)
           IO.popen([self.class.path, "-i", file])
         end

data/lib/ddr/extraction/adapters/tika_adapter.rb

@@ -1,14 +1,12 @@
-require_relative "adapter"
-
 module Ddr
   module Extraction
     module Adapters
-      class TikaAdapter < Adapter
+      class TikaAdapter
 
         # Extract text from file
         #
         # @param file [String] path to file from which to extract text
-        # @return [IO] 
+        # @return [IO] the output
         def extract_text(file)
           IO.popen(["java", "-jar", self.class.path, "--text", file])
         end
@@ -26,8 +24,11 @@ module Ddr
           # URL to download distribution
           attr_accessor :download_url
 
+          # Verify checksum?
+          attr_accessor :verify_checksum
+
           # Tika distribution checksum
-          attr_accessor :checksum
+          attr_accessor :checksum_value
 
           # Tika distribution checksum type
           attr_accessor :checksum_type

data/lib/ddr/extraction/configuration.rb

@@ -0,0 +1,22 @@
+require_relative "adapter"
+require_relative "adapters"
+
+module Ddr
+  module Extraction
+    class Configuration
+
+      def adapters(name)
+        config = Adapters.get_adapter(name)
+        yield config if block_given?
+        config
+      end
+
+      def adapter
+        config = Adapter
+        yield config if block_given?
+        config
+      end
+
+    end
+  end
+end

data/lib/ddr/extraction/defaults.rb

@@ -0,0 +1,26 @@
+require "ddr-extraction"
+
+BIN_DIR = File.expand_path("../../../../bin", __FILE__)
+TIKA_VERSION = "1.6"
+FITS_VERSION = "0.8.3"
+
+Ddr::Extraction.configure do |config|
+  config.adapter.text = :tika
+  config.adapter.metadata = :fits
+
+  config.adapters(:tika) do |tika|
+    tika.version = TIKA_VERSION
+    tika.path = File.join(BIN_DIR, "tika-app.jar")
+    tika.download_url = "http://archive.apache.org/dist/tika/tika-app-#{TIKA_VERSION}.jar"
+    tika.verify_checksum = true
+    tika.checksum_value = "99df0d8c3f6a2be498d275053e611fb5afdf0a9d"
+    tika.checksum_type = :SHA1
+  end
+
+  config.adapters(:fits) do |fits|
+    fits.version = "0.8.3"
+    fits.path = File.join(BIN_DIR, "fits-#{FITS_VERSION}", "fits.sh")
+    fits.download_url = "http://projects.iq.harvard.edu/files/fits/files/fits-#{FITS_VERSION}.zip"
+  end
+end
+

data/lib/ddr/extraction/extractor.rb

@@ -3,28 +3,20 @@ require_relative "adapters"
 module Ddr
   module Extraction
     class Extractor
-      extend Forwardable
-
-      def_delegator :text_adapter, :extract_text
-      def_delegator :metadata_adapter, :extract_metadata
 
       # Extracts a type of content from a file
       #
       # @param type [Symbol] the type of content to extract, `:text` or `:metadata`.
       # @param file [String] path to file from which to extract content.
-      # @return 
+      # @return [IO] the output
       def extract(type, file)
-        send("extract_#{type}", file)
-      end      
+        adapter(type).send("extract_#{type}", file)
+      end
 
       private
 
-      def text_adapter
-        @text_adapter ||= Adapters.build_adapter(:text)
-      end
-
-      def metadata_adapter
-        @metadata_adapter ||= Adapters.build_adapter(:metadata)
+      def adapter(type)
+        Adapter.build_adapter(type)
       end
 
     end

data/lib/ddr/extraction/version.rb

@@ -1,5 +1,5 @@
 module Ddr
   module Extraction
-    VERSION = "0.1.0"
+    VERSION = "0.2.0"
   end
 end

data/lib/tasks/ddr_extraction.rake

@@ -1,4 +1,4 @@
-require "ddr-extraction"
+require "ddr/extraction/defaults"
 require "openssl"
 
 DOWNLOAD_DIR = File.absolute_path("tmp")
@@ -7,18 +7,20 @@ namespace :tika do
   desc "Download Tika app"
   task :download => :download_dir do
     tika_app = File.join(DOWNLOAD_DIR, "tika-app.jar")
-    Ddr::Extraction::Adapters::TikaAdapter.config do |tika|
-      puts "Downloading Tika app ... "
-      system "curl -L #{tika.download_url} -o #{tika_app}"
+    tika_config = Ddr::Extraction.config.adapters(:tika)
+    puts "Downloading Tika app ... "
+    system "curl -L #{tika_config.download_url} -o #{tika_app}"
+    if tika_config.verify_checksum
       puts "Verifiying checksum ... "
-      digest = OpenSSL::Digest.const_get(tika.checksum_type).new
-      digest << File.read(tika.path)
-      if digest.to_s == tika.checksum
-        FileUtils.mv(tika_app, tika.path)
-      else
-        puts "Checksums do not match!"
+      digest = OpenSSL::Digest.const_get(tika_config.checksum_type).new
+      digest << File.read(tika_config.path)
+      if digest.to_s != tika_config.checksum_value
+        puts "Checksums do not match -- aborting!"
+        FileUtils.remove_entry_secure(tika_app)
+        abort
       end
     end
+    FileUtils.mv(tika_app, tika_config.path)
   end
 
   # namespace :server do
@@ -32,13 +34,12 @@ namespace :fits do
   desc "Download FITS tool"
   task :download => :download_dir do
     fits_tool = File.join(DOWNLOAD_DIR, "fits.zip")
-    Ddr::Extraction::Adapters::FitsAdapter.config do |fits|
-      puts "Downloading FITS tool ... "
-      system "curl -L #{fits.download_url} -o #{fits_tool}"
-      # Unzip options: convert text files, force overwrite, extra quiet
-      system "unzip -a -o -qq -d bin #{fits_tool}"
-      FileUtils.chmod(0755, fits.path)
-    end
+    fits_config = Ddr::Extraction.config.adapters(:fits)
+    puts "Downloading FITS tool ... "
+    system "curl -L #{fits_config.download_url} -o #{fits_tool}"
+    # Unzip options: convert text files, force overwrite, extra quiet
+    system "unzip -a -o -qq -d bin #{fits_tool}"
+    FileUtils.chmod(0755, fits_config.path)
   end
 end
 

data/spec/spec_helper.rb

@@ -1,4 +1,4 @@
-require "ddr-extraction"
+require "ddr/extraction/defaults"
 
 # This file was generated by the `rspec --init` command. Conventionally, all
 # specs live under a `spec` directory, which RSpec adds to the `$LOAD_PATH`.

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: ddr-extraction
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.0
 platform: ruby
 authors:
 - David Chandek-Stark
@@ -70,14 +70,13 @@ files:
 - ddr-extraction.gemspec
 - lib/ddr-extraction.rb
 - lib/ddr/extraction.rb
+- lib/ddr/extraction/adapter.rb
 - lib/ddr/extraction/adapters.rb
-- lib/ddr/extraction/adapters/adapter.rb
 - lib/ddr/extraction/adapters/fits_adapter.rb
-- lib/ddr/extraction/adapters/metadata_extraction_adapter.rb
-- lib/ddr/extraction/adapters/text_extraction_adapter.rb
 - lib/ddr/extraction/adapters/tika_adapter.rb
+- lib/ddr/extraction/configuration.rb
+- lib/ddr/extraction/defaults.rb
 - lib/ddr/extraction/extractor.rb
-- lib/ddr/extraction/metadata_extractor.rb
 - lib/ddr/extraction/version.rb
 - lib/tasks/ddr_extraction.rake
 - spec/fixtures/blue-devil.png

data/lib/ddr/extraction/adapters/adapter.rb

@@ -1,15 +0,0 @@
-module Ddr
-  module Extraction
-    module Adapters
-      class Adapter
-
-        class << self
-          def config
-            yield self
-          end
-        end
-
-      end
-    end
-  end
-end

data/lib/ddr/extraction/adapters/metadata_extraction_adapter.rb

@@ -1,13 +0,0 @@
-module Ddr
-  module Extraction
-    module Adapters
-      class MetadataExtractionAdapter
-
-        def extract_metadata(file)
-          NotImplemented
-        end
-
-      end
-    end
-  end
-end

data/lib/ddr/extraction/adapters/text_extraction_adapter.rb

@@ -1,13 +0,0 @@
-module Ddr
-  module Extraction
-    module Adapters
-      class TextExtractionAdapter
-
-        def extract_text(file)
-          NotImplemented
-        end
-
-      end
-    end
-  end
-end

data/lib/ddr/extraction/metadata_extractor.rb

@@ -1,16 +0,0 @@
-require "ddr/extraction/adapters"
-
-module Ddr
-  module Extraction
-    class MetadataExtractor
-      extend Forwardable
-
-      def_delegator :@adapter, :extract_metadata
-
-      def initialize
-        @adapter = Ddr::Extraction::Adapters.get_metadata_extraction_adapter
-      end
-
-    end
-  end
-end