duracloud-client 0.5.0 → 0.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 60ab1ee918d1618f1f9fe31e0bad7461246a1f9c
-  data.tar.gz: 4430b99f0f2a1d98845139e26113179610f7ac57
+  metadata.gz: 7ea05804282811db4ddae6b49a04e87f30f3eec7
+  data.tar.gz: 8ab80ee402b9ff8f17dde4e6aeb445af13af77d0
 SHA512:
-  metadata.gz: 62eeb59e57a5898783fc310aa848819779aa33eaf7c29b88296d01ff76210f459a4ad37c2c4c872a7871ab0027a757537e24e78fa2aacf6f45d27427d25e8bf3
-  data.tar.gz: 24006ad94935c937a5e3a9188b4c42946e9cf485ceb3e643fb4ca14531805e70ff77b341ca4f2ef90c97cac06094dd4727e01ca6e1064ecb8dca546714e45f98
+  metadata.gz: 0b67d6ac687c02438b8814b54ba3b2eec6e84ce1db77370aacd76789e8f22298af0370eda6a666dfc392533515db25608d4490d99819fa0ac8afc1203e823030
+  data.tar.gz: b832eed79711148ef5a0ab4be82d6f7d29186138ef66c403b3b99b71ca504e7f55975f840ee9fc64451848f6876629a182058ee155e165270eace05feb3da711

data/Gemfile

@@ -1,5 +1,4 @@
 source 'https://rubygems.org'
-ruby '2.3.1'
 
 # Specify your gem's dependencies in duracloud.gemspec
 gemspec

data/README.md

@@ -121,6 +121,25 @@ foo8
  => ["ark:/99999/fk4zzzz", "foo", "foo2", "foo22", "foo3", "foo5", "foo7", "foo8"] 
 ```
 
+#### Sync Validation
+
+*New in version 0.6.0*
+
+Sync validation is the process of comparing the files in a local content directory with content in a DuraCloud space in order to confirm that all local content has been successfully sync.  This functionality is NOT part of the DuraCloud REST API and is provided AS IS (per the license terms).
+
+Assumptions:
+- The external program `md5deep` is installed and available on the user's path. See http://md5deep.sourceforge.net/ for `md5deep` documentation and download packages.
+- The content IDs in the target DuraCloud space match the relative paths of files in the source content directory -- i.e., no support for "prefixes" (this may change in the future).
+
+Process:
+- The space manifest is downloaded from DuraCloud and converted to the expected input format for `md5deep` (two columns: md5 hash and file path, separated by two spaces).
+- `md5deep` is run against the content directory in "non-matching" mode (-X) with the converted manifest as the list of "known hashes".
+- Non-matching files from the `md5deep` run are re-checked individually by calling `Duracloud::Content.exist?`. This pass will account for content sync after `md5deep` started as well as files that have been chunked in DuraCloud.
+
+```ruby
+Duracloud::SyncValidation.call(space_id: 'foo', content_dir: '/var/foo/bar')
+```
+
 ### Content
 
 #### Create a new content item and store it in DuraCloud
@@ -299,6 +318,12 @@ D, [2016-05-19T15:39:33.538448 #29974] DEBUG -- : Duracloud::Client GET https://
  => #<CSV::Table mode:col_or_row row_count:8> 
 ```
 
+### Command Line Interface
+
+*New in version 0.6.0*
+
+The `bin/` directory of the gem now includes an executable `duracloud`.  Use `-h/--help` to display usage.  If the gem was installed with `bundler` you may need to run `bundle exec bin/duracloud`.
+
 ## Versioning
 
 We endeavor to follow semantic versioning.  In particular, versions < 1.0 may introduce backward-incompatible changes without notice.  Use at your own risk.  Version 1.0 signals a stable API.

data/bin/duracloud

@@ -0,0 +1,5 @@
+#!/usr/bin/env ruby
+
+require 'duracloud'
+
+Duracloud::Command.call(*ARGV)

data/lib/duracloud/chunked_content.rb

@@ -17,6 +17,10 @@ module Duracloud
       @manifest
     end
 
+    def chunked?
+      true
+    end
+
     private
 
     def do_load_properties
@@ -29,6 +33,7 @@ module Duracloud
       end
       self.properties = manifest.properties.dup
       self.content_type = manifest.source.content_type
+      self.size = manifest.source.size
     end
 
   end

data/lib/duracloud/command.rb

@@ -0,0 +1,128 @@
+require 'optparse'
+require 'active_model'
+
+module Duracloud
+  class Command
+    include ActiveModel::Model
+    include Commands
+
+    COMMANDS = Commands.public_instance_methods.map(&:to_s)
+    USAGE = "Usage: duracloud [#{COMMANDS.join('|')}] [options]"
+    HELP = "Type 'duracloud --help' for usage."
+
+    attr_accessor :command, :user, :password, :host, :port,
+                  :space_id, :store_id, :content_id,
+                  :content_type, :md5,
+                  :content_dir, :format,
+                  :logging
+
+    def self.error!(reason)
+      STDERR.puts reason
+      STDERR.puts HELP
+      exit(false)
+    end
+
+    def self.call(*args)
+      options = {}
+
+      parser = OptionParser.new do |opts|
+        opts.banner = USAGE
+
+        opts.on("-h", "--help",
+                "Prints help") do
+          puts opts
+          exit
+        end
+
+        opts.on("-H", "--host HOST",
+                "DuraCloud host") do |v|
+          options[:host] = v
+        end
+
+        opts.on("-P", "--port PORT",
+                "DuraCloud port") do |v|
+          options[:port] = v
+        end
+
+        opts.on("-u", "--user USER",
+                "DuraCloud user") do |v|
+          options[:user] = v
+        end
+
+        opts.on("-p", "--password PASSWORD",
+                "DuraCloud password") do |v|
+          options[:password] = v
+        end
+
+        opts.on("-l", "--[no-]logging",
+                "Enable/disable logging to STDERR") do |v|
+          options[:logging] = v
+        end
+
+        opts.on("-s", "--space-id SPACE_ID",
+                "DuraCloud space ID") do |v|
+          options[:space_id] = v
+        end
+
+        opts.on("-i", "--store-id STORE_ID",
+                "DuraCloud store ID") do |v|
+          options[:store_id] = v
+        end
+
+        opts.on("-c", "--content-id CONTENT_ID",
+                "DuraCloud content ID") do |v|
+          options[:content_id] = v
+        end
+
+        opts.on("-m", "--md5 MD5",
+                "MD5 digest of content to store or retrieve") do |v|
+          options[:md5] = v
+        end
+
+        opts.on("-b", "--bagit",
+                "Get manifest in BAGIT format (default is TSV)") do
+          options[:format] = Manifest::BAGIT_FORMAT
+        end
+
+        opts.on("-d", "--content-dir CONTENT_DIR",
+                "Local content directory") do |v|
+          options[:content_dir] = v
+        end
+      end
+
+      command = args.shift if COMMANDS.include?(args.first)
+      parser.parse!(args)
+
+      new(options).execute(command)
+    rescue CommandError, OptionParser::ParseError => e
+      error!(e.message)
+    end
+
+    def execute(command)
+      unless COMMANDS.include?(command)
+        raise CommandError, "Invalid command: #{command}."
+      end
+      begin
+        configure_client
+        send(command)
+      rescue Error => e
+        STDERR.puts e.message
+        exit(false)
+      end
+    end
+
+    private
+
+    def configure_client
+      Client.configure do |config|
+        config.user     = user     if user
+        config.password = password if password
+        config.host     = host     if host
+        config.port     = port     if port
+
+        config.silence_logging! unless logging
+      end
+    end
+
+  end
+end

data/lib/duracloud/commands.rb

@@ -0,0 +1,35 @@
+module Duracloud
+  module Commands
+
+    def validate
+      SyncValidation.call(space_id: space_id, store_id: store_id, content_dir: content_dir)
+    end
+
+    def manifest
+      Manifest.download(space_id, store_id, format: format) do |chunk|
+        print chunk
+      end
+    end
+
+    def properties
+      proplist = content_id ? content_properties : space_properties
+      STDOUT.puts proplist
+    end
+
+    private
+
+    def content_properties
+      content = Content.find(space_id: space_id, store_id: store_id, content_id: content_id, md5: md5)
+      proplist = content.properties.map { |k, v| "#{k}: #{v}" }
+      proplist << "MD5: #{content.md5}"
+      proplist << "Size: #{content.size} (#{content.human_size})"
+      proplist << "Chunked?: #{content.chunked?}"
+    end
+
+    def space_properties
+      space = Space.find(space_id, store_id)
+      space.properties.map { |k, v| "#{k}: #{v}" }
+    end
+
+  end
+end

data/lib/duracloud/content.rb

@@ -46,7 +46,7 @@ module Duracloud
     end
 
     attr_accessor :space_id, :content_id, :store_id,
-                  :body, :md5, :content_type
+                  :body, :md5, :content_type, :size
     alias_method :id, :content_id
     validates_presence_of :space_id, :content_id
 
@@ -100,6 +100,14 @@ module Duracloud
       copied
     end
 
+    def chunked?
+      false
+    end
+
+    def human_size
+      ActiveSupport::NumberHelper.number_to_human_size(size, prefix: :si)
+    end
+
     private
 
     def store
@@ -159,6 +167,7 @@ module Duracloud
       end
       self.properties = response.headers
       self.content_type = response.content_type
+      self.size = response.size
     end
 
     def do_delete

data/lib/duracloud/error.rb

@@ -5,4 +5,5 @@ module Duracloud
   class BadRequestError < Error; end
   class ConflictError < Error; end
   class MessageDigestError < Error; end
+  class CommandError < Error; end
 end

data/lib/duracloud/manifest.rb

@@ -13,6 +13,14 @@ module Duracloud
 
     attr_reader :space_id, :store_id
 
+    def self.download(*args, **kwargs, &block)
+      new(*args).download(**kwargs, &block)
+    end
+
+    def self.download_generated(*args, **kwargs, &block)
+      new(*args).download_generated(**kwargs, &block)
+    end
+
     def initialize(space_id, store_id = nil)
       @space_id = space_id
       @store_id = store_id

data/lib/duracloud/response.rb

@@ -8,7 +8,7 @@ module Duracloud
 
     delegate [:header, :body, :code, :ok?, :redirect?, :status, :reason] => :original_response,
              :content_type => :header,
-             [:size, :empty?] => :body
+             :empty? => :body
 
     def_delegator :header, :request_uri, :url
     def_delegator :header, :request_method
@@ -39,5 +39,9 @@ module Duracloud
     def md5
       header["content-md5"].first
     end
+
+    def size
+      header["content-length"].first.to_i rescue nil
+    end
   end
 end

data/lib/duracloud/sync_validation.rb

@@ -0,0 +1,66 @@
+require 'active_model'
+require 'tempfile'
+require 'csv'
+
+module Duracloud
+  class SyncValidation
+    include ActiveModel::Model
+
+    TWO_SPACES = '  '
+    MD5_CSV_OPTS = { col_sep: TWO_SPACES }.freeze
+    MANIFEST_CSV_OPTS = { col_sep: "\t", headers: true, return_headers: false }.freeze
+
+    attr_accessor :space_id, :content_dir, :store_id
+
+    def self.call(*args)
+      new(*args).call
+    end
+
+    def call
+      Tempfile.open("#{space_id}-manifest") do |manifest|
+        Manifest.download(space_id, store_id) do |chunk|
+          manifest.write(chunk)
+        end
+        manifest.close
+
+        # convert manifest into md5deep format
+        Tempfile.open("#{space_id}-md5") do |md5_list|
+          CSV.foreach(manifest.path, MANIFEST_CSV_OPTS) do |row|
+            md5_list.puts [ row[2], row[1] ].join(TWO_SPACES)
+          end
+          md5_list.close
+
+          # run md5deep to find files not listed in the manifest
+          Tempfile.open("#{space_id}-audit") do |audit|
+            audit.close
+            pid = spawn("md5deep", "-X", md5_list.path, "-l", "-r", ".", chdir: content_dir, out: audit.path)
+            Process.wait(pid)
+            case $?.exitstatus
+            when 0
+              true
+            when 1, 2
+              failures = []
+              CSV.foreach(audit.path, MD5_CSV_OPTS) do |md5, path|
+                content_id = path.sub(/^\.\//, "")
+                begin
+                  if !Duracloud::Content.exist?(space_id: space_id, store_id: store_id, content_id: content_id, md5: md5)
+                    failures << [ "MISSING", md5, content_id ].join("\t")
+                  end
+                rescue MessageDigestError => e
+                  failures << [ "CHANGED", md5, content_id ].join("\t")
+                end
+              end
+              STDOUT.puts failures
+              failures.empty?
+            when 64
+              raise Error, "md5deep user error."
+            when 128
+              raise Error, "md5deep internal error."
+            end
+          end
+        end
+      end
+    end
+
+  end
+end

data/lib/duracloud/version.rb

@@ -1,3 +1,3 @@
 module Duracloud
-  VERSION = "0.5.0"
+  VERSION = "0.6.0"
 end

data/lib/duracloud.rb

@@ -7,6 +7,8 @@ module Duracloud
   autoload :BitIntegrityReport, "duracloud/bit_integrity_report"
   autoload :ChunkedContent, "duracloud/chunked_content"
   autoload :Client, "duracloud/client"
+  autoload :Command, "duracloud/command"
+  autoload :Commands, "duracloud/commands"
   autoload :Configuration, "duracloud/configuration"
   autoload :Connection, "duracloud/connection"
   autoload :Content, "duracloud/content"
@@ -25,5 +27,6 @@ module Duracloud
   autoload :SpaceAcls, "duracloud/space_acls"
   autoload :SpaceProperties, "duracloud/space_properties"
   autoload :Store, "duracloud/store"
+  autoload :SyncValidation, "duracloud/sync_validation"
   autoload :TSV, "duracloud/tsv"
 end

data/spec/unit/content_spec.rb

@@ -10,6 +10,7 @@ module Duracloud
         describe "and it is not chunked" do
           before { stub_request(:head, url) }
           it { is_expected.to be_a described_class }
+          it { is_expected.to_not be_chunked }
         end
         describe "and it is chunked" do
           let(:manifest_xml) { File.read(File.expand_path("../../fixtures/content_manifest.xml", __FILE__)) }
@@ -20,7 +21,9 @@ module Duracloud
           end
           it { is_expected.to be_a described_class }
           its(:md5) { is_expected.to eq "164e9aee34c0c42915716e11d5d539b5" }
+          its(:size) { is_expected.to eq 4227858432 }
           its(:content_type) { is_expected.to eq "application/octet-stream" }
+          it { is_expected.to be_chunked }
         end
       end
       describe "when it does not exist" do

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: duracloud-client
 version: !ruby/object:Gem::Version
-  version: 0.5.0
+  version: 0.6.0
 platform: ruby
 authors:
 - David Chandek-Stark
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-06-07 00:00:00.000000000 Z
+date: 2017-06-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: hashie
@@ -145,7 +145,8 @@ dependencies:
 description: Ruby client for communicating with DuraCloud
 email:
 - dchandekstark@gmail.com
-executables: []
+executables:
+- duracloud
 extensions: []
 extra_rdoc_files: []
 files:
@@ -156,6 +157,7 @@ files:
 - LICENSE
 - README.md
 - Rakefile
+- bin/duracloud
 - duracloud.gemspec
 - gemfiles/Gemfile.activemodel-4.2
 - gemfiles/Gemfile.activemodel-5.0
@@ -166,6 +168,8 @@ files:
 - lib/duracloud/bit_integrity_report.rb
 - lib/duracloud/chunked_content.rb
 - lib/duracloud/client.rb
+- lib/duracloud/command.rb
+- lib/duracloud/commands.rb
 - lib/duracloud/configuration.rb
 - lib/duracloud/connection.rb
 - lib/duracloud/content.rb
@@ -183,6 +187,7 @@ files:
 - lib/duracloud/space_acls.rb
 - lib/duracloud/space_properties.rb
 - lib/duracloud/store.rb
+- lib/duracloud/sync_validation.rb
 - lib/duracloud/tsv.rb
 - lib/duracloud/version.rb
 - spec/fixtures/audit_log.tsv