filecluster 0.3.3 → 0.3.4

data/lib/fc/db.rb

@@ -62,6 +62,7 @@ module FC
           policy_id int NOT NULL,
           dir tinyint(1) NOT NULL DEFAULT 0,
           size bigint NOT NULL DEFAULT 0,
+          md5 varchar(32) DEFAULT NULL,
           status ENUM('new', 'ready', 'error', 'delete') NOT NULL DEFAULT 'new',
           time int DEFAULT NULL,
           copies int NOT NULL DEFAULT 0,

data/lib/fc/item.rb

@@ -3,7 +3,7 @@ require 'shellwords'
 
 module FC
   class Item < DbBase
-    set_table :items, 'name, tag, outer_id, policy_id, dir, size, status, time, copies'
+    set_table :items, 'name, tag, outer_id, policy_id, dir, size, md5, status, time, copies'
     
     # Create item by local path.
     # Additional options: 
@@ -16,20 +16,21 @@ module FC
         :name => item_name.to_s.gsub('//', '/').sub(/\/$/, '').sub(/^\//, '').strip,
         :policy_id => policy.id,
         :dir => File.directory?(local_path),
-        :size => `du -sb #{local_path.shellescape}`.to_i
+        :size => FC::Storage.new(:host => FC::Storage.curr_host).file_size(local_path),
+        :md5 => FC::Storage.new(:host => FC::Storage.curr_host).md5_sum(local_path)
       })
       item_params.delete(:replace)
       item_params.delete(:inplace)
       raise 'Name is empty' if item_params[:name].empty?
       raise 'Zero size path' if item_params[:size] == 0
-      
+
       if local_path.include?(item_name)
-        storage = policy.get_create_storages.detect do |s| 
+        storage = policy.get_create_storages.detect do |s|
           s.host == FC::Storage.curr_host && local_path.index(s.path) == 0 && local_path.sub(s.path, '') == item_params[:name]
         end
         FC::Error.raise "local_path #{local_path} is not valid path for policy ##{policy.id}" unless storage
       end
-      
+
       # new item?
       item = FC::Item.where('name=? AND policy_id=?', item_params[:name], policy.id).first
       if item
@@ -78,7 +79,7 @@ module FC
         else
           storage.copy_path(src, name)
         end
-        size_on_storage = storage.file_size(name)
+        md5_on_storage = storage.md5_sum(name)
       rescue Exception => e
         item_storage.status = 'error'
         item_storage.save
@@ -89,10 +90,10 @@ module FC
         rescue Exception => e
           FC::Error.raise "After copy error: #{e.message}", :item_id => id, :item_storage_id => item_storage.id
         else
-          if size_on_storage != size
+          if md5_on_storage != md5
             item_storage.status = 'error'
             item_storage.save
-            FC::Error.raise "Check size after copy error", :item_id => id, :item_storage_id => item_storage.id
+            FC::Error.raise "Check md5 after copy error", :item_id => id, :item_storage_id => item_storage.id
           else
             item_storage.status = 'ready'
             item_storage.save

data/lib/fc/storage.rb

@@ -53,14 +53,14 @@ module FC
     # copy local_path to storage
     def copy_path(local_path, file_name)
       dst_path = "#{self.path}#{file_name}"
-      
+
       cmd = "rm -rf #{dst_path.shellescape}; mkdir -p #{File.dirname(dst_path).shellescape}"
       cmd = self.class.curr_host == host ? cmd : "ssh -oBatchMode=yes -oStrictHostKeyChecking=no #{self.host} \"#{cmd}\""
       r = `#{cmd} 2>&1`
       raise r if $?.exitstatus != 0
-      
-      cmd = self.class.curr_host == host ? 
-        "cp -r #{local_path.shellescape} #{dst_path.shellescape}" : 
+
+      cmd = self.class.curr_host == host ?
+        "cp -r #{local_path.shellescape} #{dst_path.shellescape}" :
         "scp -r -oBatchMode=yes -oStrictHostKeyChecking=no #{local_path.shellescape} #{self.host}:\"#{dst_path.shellescape}\""
       r = `#{cmd} 2>&1`
       raise r if $?.exitstatus != 0
@@ -108,6 +108,16 @@ module FC
       r.to_i
     end
     
+    # return object md5_sum on storage
+    def md5_sum(file_name)
+      dst_path = "#{self.path}#{file_name}"
+      cmd = "find #{dst_path} -type f -exec md5sum {} \\; | awk '{print $1}' | sort | md5sum"
+      cmd = "ssh -oBatchMode=yes -oStrictHostKeyChecking=no #{self.host} \"#{cmd}\"" if self.class.curr_host != host
+      r = `#{cmd} 2>&1`
+      raise r if $?.exitstatus != 0
+      r.to_s[0..31]
+    end
+    
     # get available storage for copy by size
     def get_proper_storage_for_copy(size, exclude = [])
       get_copy_storages.select do |storage|

data/lib/fc/version.rb

@@ -1,3 +1,3 @@
 module FC
-  VERSION = "0.3.3"
+  VERSION = "0.3.4"
 end

data/test/functional_test.rb

@@ -103,11 +103,14 @@ class FunctionalTest < Test::Unit::TestCase
     assert_raise(RuntimeError) { item_storage.reload }
   end
   
-  should "item create_from_local check size" do
-    FC::Storage.any_instance.stubs(:copy_path => true, :file_size => 10)
+  should "item create_from_local check md5" do
     errors_count = FC::Error.where.count
-    assert_raise(RuntimeError) { FC::Item.create_from_local(@@test_file_path, 'test5', @@policies[1], {:tag => 'test'}) }
-    assert_equal errors_count+1, FC::Error.where.count, "Error not saved after check size" 
+    @item = FC::Item.create_from_local(@@test_file_path, 'test5', @@policies[0], {:tag => 'test'})
+    item_storage = @item.make_item_storage(@@storages[0], status = 'copy')
+    `dd if=/dev/urandom of=#{@@storages[0].path}#{@item.name} bs=100K count=1 2>&1`
+    #`dd if=/dev/urandom of=#{@@storages[0].path}#{@item.name} bs=100K count=1 2>&1`
+    assert_raise(RuntimeError) { @item.copy_item_storage(@@storages[0], @@storages[1], item_storage) }
+    assert_equal errors_count+1, FC::Error.where.count, "Error not saved after check md5"
   end
   
   should "item create_from_local inplace" do

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: filecluster
 version: !ruby/object:Gem::Version
-  version: 0.3.3
+  version: 0.3.4
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-03-18 00:00:00.000000000 Z
+date: 2014-03-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rb-readline
@@ -193,7 +193,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: -312355588801708268
+      hash: -1977585421026350925
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:
@@ -202,7 +202,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: -312355588801708268
+      hash: -1977585421026350925
 requirements: []
 rubyforge_project: 
 rubygems_version: 1.8.24