duracloud-client 0.8.0 → 0.9.0

data/lib/duracloud/commands/find_item.rb

@@ -0,0 +1,16 @@
+module Duracloud::Commands
+  class FindItem < Command
+
+    def call
+      content = Duracloud::Content.find(space_id: space_id, store_id: store_id, content_id: content_id, md5: md5)
+      props = content.properties.dup
+      props.merge!("MD5" => content.md5,
+                   "Size" => content.size,
+                   "Chunked" => content.chunked?)
+      props.each do |k, v|
+        puts "#{k}: #{v}"
+      end
+    end
+
+  end
+end

data/lib/duracloud/commands/find_items.rb

@@ -0,0 +1,22 @@
+require 'csv'
+
+module Duracloud::Commands
+  class FindItems < Command
+
+    HEADERS = %i( content_id md5 size content_type modified )
+
+    def call
+      CSV.instance($stdout, headers: HEADERS, write_headers: true) do |csv|
+        CSV.foreach(infile, headers: false) do |row|
+          begin
+            item = Duracloud::Content.find(space_id: space_id, store_id: store_id, content_id: row[0], md5: row[1])
+            csv << HEADERS.map { |header| item.send(header) }
+          rescue Duracloud::NotFoundError, Duracloud::MessageDigestError => e
+            $stderr.puts "ERROR: Content ID #{row[0]} -- #{e.message}"
+          end
+        end
+      end
+    end
+
+  end
+end

data/lib/duracloud/commands/find_missing_items.rb

@@ -0,0 +1,15 @@
+module Duracloud::Commands
+  class FindMissingItems < Command
+
+    def call
+      CSV.instance($stdout, headers: false) do |csv|
+        CSV.foreach(infile, headers: false) do |row|
+          unless Duracloud::Content.exist?(space_id: space_id, store_id: store_id, content_id: row[0], md5: row[1])
+            csv << row
+          end
+        end
+      end
+    end
+
+  end
+end

data/lib/duracloud/commands/find_space.rb

@@ -0,0 +1,12 @@
+module Duracloud::Commands
+  class FindSpace < Command
+
+    def call
+      space = Duracloud::Space.find(space_id, store_id)
+      space.properties.each do |k, v|
+        puts "#{k}: #{v}"
+      end
+    end
+
+  end
+end

data/lib/duracloud/commands/get_storage_report.rb

@@ -0,0 +1,16 @@
+module Duracloud::Commands
+  class GetStorageReport < Command
+
+    def call
+      delegate_to = if space_id
+                      GetStorageReportForSpace
+                    elsif all_spaces
+                      GetStorageReportForAllSpaces
+                    else
+                      GetStorageReportForStore
+                    end
+      delegate_to.call(cli)
+    end
+
+  end
+end

data/lib/duracloud/commands/get_storage_report_for_all_spaces.rb

@@ -0,0 +1,12 @@
+module Duracloud::Commands
+  class GetStorageReportForAllSpaces < Command
+
+    def call
+      Duracloud::StorageReports.for_all_spaces_in_a_store(store_id: store_id).each do |report|
+        puts "-"*40
+        puts report.to_s
+      end
+    end
+
+  end
+end

data/lib/duracloud/commands/get_storage_report_for_space.rb

@@ -0,0 +1,10 @@
+module Duracloud::Commands
+  class GetStorageReportForSpace < Command
+
+    def call
+      reports = Duracloud::StorageReports.by_space(space_id, store_id: store_id)
+      puts reports.last.to_s
+    end
+
+  end
+end

data/lib/duracloud/commands/get_storage_report_for_store.rb

@@ -0,0 +1,10 @@
+module Duracloud::Commands
+  class GetStorageReportForStore < Command
+
+    def call
+      reports = Duracloud::StorageReports.by_store(store_id: store_id)
+      puts reports.last.to_s
+    end
+
+  end
+end

data/lib/duracloud/commands/list_content_ids.rb

@@ -0,0 +1,11 @@
+module Duracloud::Commands
+  class ListContentIds < Command
+
+    def call
+      Duracloud::Space.content_ids(space_id, store_id: store_id, prefix: prefix).each do |id|
+        puts id
+      end
+    end
+
+  end
+end

data/lib/duracloud/commands/list_items.rb

@@ -0,0 +1,17 @@
+require "csv"
+
+module Duracloud::Commands
+  class ListItems < Command
+
+    HEADERS = %i( content_id md5 size content_type modified )
+
+    def call
+      CSV.instance($stdout, headers: HEADERS, write_headers: true) do |csv|
+        Duracloud::Space.items(space_id, store_id: store_id, prefix: prefix).each do |item|
+          csv << HEADERS.map { |header| item.send(header) }
+        end
+      end
+    end
+
+  end
+end

data/lib/duracloud/commands/sync.rb

@@ -1,5 +1,3 @@
-require_relative 'command'
-
 module Duracloud::Commands
   class Sync < Command
 

data/lib/duracloud/commands/validate.rb

@@ -1,10 +1,9 @@
-require_relative "command"
-
 module Duracloud::Commands
   class Validate < Command
 
     def call
-      Duracloud::SyncValidation.call(space_id: space_id, store_id: store_id, content_dir: content_dir)
+      klass = fast ? Duracloud::FastSyncValidation : DuracloudSyncValidation
+      klass.call(space_id: space_id, store_id: store_id, content_dir: content_dir, work_dir: work_dir)
     end
 
   end

data/lib/duracloud/content.rb

@@ -79,21 +79,30 @@ module Duracloud
 
     # @return [Duracloud::Content] the copied content
     #   The current instance still represents the original content.
+    # @raise [Duracloud::Error]
     def copy(**args)
       dest = args.except(:force)
       dest[:space_id]   ||= space_id
+      dest[:store_id]   ||= store_id
       dest[:content_id] ||= content_id
-      raise CopyError, "Destination is the same as the source." if dest == copy_source
+      if dest == copy_source
+        raise CopyError, "Destination is the same as the source."
+      end
       if !args[:force] && Content.exist?(**dest)
-        raise CopyError, "Destination exists and :false option is false."
+        raise CopyError, "Destination exists and `:force' option is false."
       end
       options = { storeID: dest[:store_id], headers: copy_headers }
-      Client.copy_content(dest[:space_id], dest[:content_id], **options)
+      response = Client.copy_content(dest[:space_id], dest[:content_id], **options)
+      if md5 != response.md5
+        raise CopyError, "Message digest of copy does not match source " \
+                         "(source: #{md5}; destination: #{response.md5})"
+      end
       Content.new(dest.merge(md5: md5))
     end
 
     # @return [Duracloud::Content] the moved content
     #   The current instance still represents the deleted content.
+    # @raise [Duracloud::Error]
     def move(**args)
       copied = copy(**args)
       delete

data/lib/duracloud/fast_sync_validation.rb

@@ -0,0 +1,42 @@
+module Duracloud
+  class FastSyncValidation < SyncValidation
+
+    def convert_manifest
+      # content-id is the 2nd column of the manifest
+      system("cut -f 2 #{manifest_filename} | sort", out: converted_manifest_filename)
+    end
+
+    def audit
+      find_files
+      if system("comm", "-23", find_filename, converted_manifest_filename, out: audit_filename)
+        File.empty?(audit_filename) || recheck
+      else
+        raise Error, "Error comparing #{find_filename} with #{converted_manifest_filename}."
+      end
+    end
+
+    def find_files
+      # TODO handle exclude file?
+      outfile = File.join(FileUtils.pwd, find_filename)
+      # Using a separate command for sort so we get find results incrementally
+      system("find -L . -type f | sed -e 's|^\./||'", chdir: content_dir, out: outfile) &&
+        system("sort", "-o", find_filename, find_filename)
+    end
+
+    private
+
+    def do_recheck
+      Enumerator.new do |e|
+        File.foreach(audit_filename) do |line|
+          content_id = line.chomp
+          e << check(content_id)
+        end
+      end
+    end
+
+    def find_filename
+      filename("find.txt")
+    end
+
+  end
+end

data/lib/duracloud/rest_methods.rb

@@ -108,6 +108,21 @@ module Duracloud
             "The API method 'Perform Task' has not been implemented."
     end
 
+    # @see https://wiki.duraspace.org/display/DURACLOUDDOC/DuraCloud+REST+API#DuraCloudRESTAPI-GetStorageReportsbySpace
+    def get_storage_reports_by_space(space_id, **query)
+      durastore(:get, "report/space/#{space_id}", **query)
+    end
+
+    # @see https://wiki.duraspace.org/display/DURACLOUDDOC/DuraCloud+REST+API#DuraCloudRESTAPI-GetStorageReportsbyStore
+    def get_storage_reports_by_store(**query)
+      durastore(:get, "report/store", **query)
+    end
+
+    # @see https://wiki.duraspace.org/display/DURACLOUDDOC/DuraCloud+REST+API#DuraCloudRESTAPI-GetStorageReportsforallSpacesinaStore(inasingleday)
+    def get_storage_reports_for_all_spaces_in_a_store(epoch_ms, **query)
+      durastore(:get, "report/store/#{epoch_ms}", **query)
+    end
+
     private
 
     def durastore(*args, &block)

data/lib/duracloud/storage_report.rb

@@ -0,0 +1,33 @@
+require 'hashie'
+require 'active_support'
+
+module Duracloud
+  class StorageReport < Hashie::Trash
+
+    property "space_id",     from: "spaceId"
+    property "store_id",     from: "storeId"
+    property "byte_count",   from: "byteCount"
+    property "object_count", from: "objectCount"
+    property "account_id",   from: "accountId"
+    property "timestamp"
+
+    def time
+      @time ||= Time.at(timestamp / 1000.0).utc
+    end
+
+    def human_size
+      ActiveSupport::NumberHelper.number_to_human_size(byte_count, prefix: :si)
+    end
+
+    def to_s
+      <<-EOS
+Date:       #{time}
+Space ID:   #{space_id || "(all)"}
+Store ID:   #{store_id}
+Objects:    #{object_count}
+Total size: #{human_size} (#{byte_count} bytes)
+      EOS
+    end
+
+  end
+end

data/lib/duracloud/storage_reports.rb

@@ -0,0 +1,52 @@
+require 'json'
+require 'hashie'
+
+module Duracloud
+  class StorageReports
+    include Enumerable
+    extend Forwardable
+
+    delegate :last => :to_a
+
+    attr_reader :data
+
+    def self.by_space(space_id, **query)
+      params = Params.new(query)
+      response = Client.get_storage_reports_by_space(space_id, **params)
+      new(response)
+    end
+
+    def self.by_store(**query)
+      params = Params.new(query)
+      response = Client.get_storage_reports_by_store(**params)
+      new(response)
+    end
+
+    def self.for_all_spaces_in_a_store(epoch_ms = nil, **query)
+      epoch_ms ||= (Time.now - (3600 * 24)).to_i * 1000
+      params = Params.new(query)
+      response = Client.get_storage_reports_for_all_spaces_in_a_store(epoch_ms, **params)
+      new(response)
+    end
+
+    def initialize(response)
+      @data = JSON.parse(response.body)
+    end
+
+    def each
+      data.each do |report|
+        yield StorageReport.new(report)
+      end
+    end
+
+    private
+
+    class Params < Hashie::Trash
+      property :storeID, from: :store_id
+      property :groupBy, from: :group_by
+      property :start
+      property :end
+    end
+
+  end
+end

data/lib/duracloud/sync_validation.rb

@@ -1,6 +1,7 @@
 require 'active_model'
 require 'tempfile'
 require 'csv'
+require 'fileutils'
 
 module Duracloud
   class SyncValidation
@@ -10,57 +11,139 @@ module Duracloud
     MD5_CSV_OPTS = { col_sep: TWO_SPACES }.freeze
     MANIFEST_CSV_OPTS = { col_sep: "\t", headers: true, return_headers: false }.freeze
 
-    attr_accessor :space_id, :content_dir, :store_id
+    MISSING = "MISSING"
+    CHANGED = "CHANGED"
+    FOUND   = "FOUND"
+
+    attr_accessor :space_id, :content_dir, :store_id, :work_dir, :fast
 
     def self.call(*args)
       new(*args).call
     end
 
+    def in_work_dir
+      if work_dir
+        FileUtils.cd(work_dir) { yield }
+      else
+        Dir.mktmpdir("#{space_id}-validation-") do |tmpdir|
+          FileUtils.cd(tmpdir) { yield }
+        end
+      end
+    end
+
     def call
-      Tempfile.open("#{space_id}-manifest") do |manifest|
+      in_work_dir do
+        download_manifest
+        convert_manifest
+        audit
+      end
+    end
+
+    def download_manifest
+      File.open(manifest_filename, "w") do |manifest|
         Manifest.download(space_id, store_id) do |chunk|
           manifest.write(chunk)
         end
-        manifest.close
-
-        # convert manifest into md5deep format
-        Tempfile.open("#{space_id}-md5") do |md5_list|
-          CSV.foreach(manifest.path, MANIFEST_CSV_OPTS) do |row|
-            md5_list.puts [ row[2], row[1] ].join(TWO_SPACES)
-          end
-          md5_list.close
-
-          # run md5deep to find files not listed in the manifest
-          Tempfile.open("#{space_id}-audit") do |audit|
-            audit.close
-            pid = spawn("md5deep", "-X", md5_list.path, "-l", "-r", ".", chdir: content_dir, out: audit.path)
-            Process.wait(pid)
-            case $?.exitstatus
-            when 0
-              true
-            when 1, 2
-              failures = []
-              CSV.foreach(audit.path, MD5_CSV_OPTS) do |md5, path|
-                content_id = path.sub(/^\.\//, "")
-                begin
-                  if !Duracloud::Content.exist?(space_id: space_id, store_id: store_id, content_id: content_id, md5: md5)
-                    failures << [ "MISSING", md5, content_id ].join("\t")
-                  end
-                rescue MessageDigestError => e
-                  failures << [ "CHANGED", md5, content_id ].join("\t")
-                end
-              end
-              STDOUT.puts failures
-              failures.empty?
-            when 64
-              raise Error, "md5deep user error."
-            when 128
-              raise Error, "md5deep internal error."
-            end
-          end
+      end
+    end
+
+    def convert_manifest
+      File.open(converted_manifest_filename, "w") do |f|
+        CSV.foreach(manifest_filename, MANIFEST_CSV_OPTS) do |row|
+          f.puts [ row[2], row[1] ].join(TWO_SPACES)
         end
       end
     end
 
+    def audit
+      outfile = File.join(FileUtils.pwd, audit_filename)
+      infile = File.join(FileUtils.pwd, converted_manifest_filename)
+      pid = spawn("md5deep", "-X", infile, "-l", "-r", ".", chdir: content_dir, out: outfile)
+      Process.wait(pid)
+      case $?.exitstatus
+      when 0
+        true
+      when 1, 2
+        recheck
+      when 64, 128
+        raise Error, "md5deep error."
+      else
+        raise Error, "Unknown error."
+      end
+    end
+
+    def recheck
+      success = true
+      recheck_file do |csv|
+        do_recheck.each do |result|
+          csv << result.to_a
+          success &&= result.found?
+        end
+      end
+      success
+    end
+
+    private
+
+    CheckResult = Struct.new(:status, :md5, :content_id) do
+      def found?
+        status == FOUND
+      end
+    end
+
+    def recheck_file
+      if work_dir
+        CSV.open(recheck_filename, "w", col_sep: "\t") { |csv| yield(csv) }
+      else
+        CSV($stdout, col_sep: "\t") { |csv| yield(csv) }
+      end
+    end
+
+    def check(content_id, md5 = nil)
+      status = begin
+                 exist?(content_id, md5) ? FOUND : MISSING
+               rescue MessageDigestError => e
+                 CHANGED
+               end
+      CheckResult.new(status, md5 || "-", content_id)
+    end
+
+    def exist?(content_id, md5 = nil)
+      Duracloud::Content.exist?(space_id: space_id, store_id: store_id, content_id: content_id, md5: md5)
+    end
+
+    def do_recheck
+      Enumerator.new do |e|
+        CSV.foreach(audit_filename, MD5_CSV_OPTS) do |md5, path|
+          content_id = path.sub(/^\.\//, "")
+          e << check(content_id, md5)
+        end
+      end
+    end
+
+    def prefix
+      space_id
+    end
+
+    def filename(suffix)
+      [ prefix, suffix ].join("-")
+    end
+
+    def manifest_filename
+      filename("manifest.tsv")
+    end
+
+    def converted_manifest_filename
+      filename("converted-manifest.txt")
+    end
+
+    def audit_filename
+      filename("audit.txt")
+    end
+
+    def recheck_filename
+      filename("recheck.txt")
+    end
+
   end
 end