content_server 1.5.0 → 1.6.0

data/lib/file_utils/file_utils.rb

@@ -5,6 +5,8 @@
 # Such as automatic file generation, symlink, etc.
 # Run from bbfs> ruby -Ilib bin/file_utils --command=generate_files --log_write_to_console=true --log_debug_level=3
 
+# TODO this module is deprecated and should be revised
+
 require 'content_data'
 require 'file_indexing'
 require 'file_utils'
@@ -15,6 +17,7 @@ require 'params'
 
 module FileUtils
   Params.string 'command', nil ,'path'
+  Params.string 'help', nil ,'get a help'
   Params.string 'ref_cd', nil ,'reference path'
   Params.string 'base_cd', nil ,'base path'
   Params.string 'dest', nil ,'destination path'
@@ -28,7 +31,8 @@ module FileUtils
   Params.string 'exist_cd', nil, 'exist_path'
   #Params.string 'config_file', 'path' ,'configuration file path'
   class FileUtils
-    def FileUtils.run
+
+    def self.run
       if Params['command'] == 'mksymlink'
         if Params['ref_cd'].nil?
           Log.error ("--ref_cd is not set")
@@ -59,45 +63,16 @@ module FileUtils
 
         not_found = nil
         begin
-          not_found = FileUtil.mksymlink(ref_cd, base_cd, Params['dest'])
+          not_found = FileUtils.mksymlink(ref_cd, base_cd, Params['dest'])
         rescue NotImplementedError
           Log.error ("symlinks are unimplemented on this machine")
           return nil
         end
         return not_found
-      elsif (Params['command'] == "merge" or
-          Params['command'] == "intersect" or
-          Params['command'] == "minus")
-
-        if Params['cd_a'].nil?
-          Log.error ("--cd_a is not set")
-          return
-        end
-        cd_a = ContentData.new()
-        cd_a.from_file(Params['cd_a'])
-        if cd_a.nil?
-          Log.error ("Error loading content data cd_a=%s" % Params['cd_a'])
-          return
-        end
-
-        if Params['cd_b'].nil?
-          Log.error ("--cd_b is not set")
-          return
-        end
-        cd_b = ContentData.new()
-        cd_b.from_file(Params['cd_b'])
-        if cd_b.nil?
-          Log.error ("Error loading content data cd_b=%s" % Params['cd_b'])
-          return
-        end
-
-        if Params['cd_b'].nil?
-          Log.error ("--dest is not set")
-          return
-        end
-
-        output = FileUtil.contet_data_command(Params['command'], cd_a, cd_b, Params['dest'])
-
+      elsif (Params['command'] == "merge" ||
+             Params['command'] == "intersect" ||
+             Params['command'] == "minus")
+        content_data_command
       elsif Params['command'] == 'unify_time'
         if  Params['cd'].nil?
           Log.error ("--cd is not set")
@@ -176,20 +151,6 @@ module FileUtils
       end
     end
 
-    def self.contet_data_command(command, cd_a, cd_b, dest_path)
-      dest = nil
-      if command == "merge"
-        dest = ContentData.merge(cd_a, cd_b)
-      elsif command == "intersect"
-        dest = ContentData.intersect(cd_a, cd_b)
-      elsif command == "minus"
-        dest = ContentData.remove(cd_b, cd_a)
-      end
-      if dest
-        dest.to_file(dest_path)
-      end
-    end
-
     # Unify modification/first_appearance time according to input DB
     # Input: ContentData
     # Output: ContentData with unified times

data/lib/networking/tcp.rb

@@ -135,14 +135,14 @@ module Networking
       @tcp_socket = nil
       @obj_clb = obj_clb
       @reconnected_clb = reconnected_clb
-      Log.debug3("Start TCPClient initialize  with @obj_clb: %s", @obj_clb)
+      # Variable to signal when remote server is ready.
+      @remote_server_available = ConditionVariable.new
+      @remote_server_available_mutex = Mutex.new
+      Log.debug3("Start TCPClient initialize with @obj_clb: %s", @obj_clb)
       if @obj_clb != nil
         @tcp_thread = start_reading
         @tcp_thread.abort_on_exception = true
       end
-      # Variable to signal when remote server is ready.
-      @remote_server_available = ConditionVariable.new
-      @remote_server_available_mutex = Mutex.new
       open_socket unless socket_good?
       Log.debug3("End TCPClient initialize.")
     end

data/spec/content_data/content_data_spec.rb

@@ -0,0 +1,331 @@
+# NOTE Code Coverage block must be issued before any of your application code is required
+if ENV['BBFS_COVERAGE']
+  require_relative '../spec_helper.rb'
+  SimpleCov.command_name 'content_data'
+end
+require 'rspec'
+require 'tempfile'
+require 'time.rb'
+require 'content_server/server'
+require_relative '../../lib/content_data/content_data.rb'
+
+describe 'Content Data Test' do
+
+  before :all do
+	Params.init Array.new
+	# must preced Log.init, otherwise log containing default values will be created
+	Params['log_write_to_file'] = false
+	Log.init
+  end
+
+  it 'test cloning db 1' do
+    content_data = ContentData::ContentData.new
+    content_data.add_instance("A1", 1242, "server_1",
+			      "/home/file_1", 2222222222)
+
+    content_data_cloned = ContentData::ContentData.new(content_data)
+    #check that DBs are equal
+    content_data_cloned.should == content_data
+
+    content_data_cloned.add_instance("A1", 1242, "server_1",
+				     "/home/file_2", 3333333333)
+    #change orig DB - size
+     content_data_cloned.should_not == content_data
+  end
+
+  it 'test cloning db 2' do
+    content_data = ContentData::ContentData.new
+    content_data.add_instance("A1", 1242, "server_1",
+			      "/home/file_1", 2222222222)
+
+    content_data_cloned = ContentData::ContentData.new(content_data)
+    #check that DBs are equal
+    content_data_cloned.should == content_data
+
+    content_data_cloned.add_instance("A2", 1243, "server_1",
+				     "/home/file_2", 3333333333)
+    #change orig DB - size
+     content_data_cloned.should_not == content_data
+  end
+
+
+  it 'test add instance' do
+    #create first content with instance
+    content_data = ContentData::ContentData.new
+    content_data.add_instance("A1", 50, "server_1",
+			      "/home/file_1", 2222222222)
+
+    #Add new instance - different size
+    # size would be overriden should not be added
+    content_data.add_instance("A1", 60, "server_1",
+			      "/home/file_2", 2222222222)
+    expected = "1\nA1,60,2222222222\n2\nA1,60,server_1,/home/file_1,2222222222\nA1,60,server_1,/home/file_2,2222222222\n"
+    content_data.to_s.should == expected
+
+    #Add new instance - new content
+    #both new content and new instances are created in DB
+    content_data.add_instance("A2", 60, "server_1",
+			      "/home/file_2", 3333333333)
+    expected = "2\nA1,60,2222222222\nA2,60,3333333333\n2\n" +
+	"A1,60,server_1,/home/file_1,2222222222\n" +
+	"A2,60,server_1,/home/file_2,3333333333\n"
+    content_data.to_s.should == expected
+
+	#Add new instance - same content
+    content_data.add_instance("A2", 60, "server_1",
+			      "/home/file_3", 4444444444)
+    expected = "2\nA1,60,2222222222\nA2,60,3333333333\n3\n" +
+	"A1,60,server_1,/home/file_1,2222222222\n" +
+	"A2,60,server_1,/home/file_2,3333333333\n" +
+	"A2,60,server_1,/home/file_3,4444444444\n"
+    content_data.to_s.should == expected
+  end
+
+  it 'test instance exists' do
+    content_data = ContentData::ContentData.new
+    content_data.add_instance("A129", 50, "server_1",
+			      "/home/file_1", 2222222222)
+    content_data.instance_exists('/home/file_1', 'server_1').should == true
+    content_data.instance_exists('/home/file_1', 'stum').should == false
+  end
+
+  it 'test remove instance' do
+    #remove instance also removes content
+    content_data = ContentData::ContentData.new
+    content_data.add_instance("A1", 50, "server_1",
+			      "/home/file_1", 2222222222)
+    content_data.content_exists('A1').should == true
+    content_data.instance_exists('/home/file_1', 'server_1').should == true
+    content_data.remove_instance('server_1', '/home/file_1')
+    content_data.instance_exists('/home/file_1', 'server_1').should == false
+    content_data.content_exists('A1').should == false
+
+    #remove instance does not remove content
+    content_data.add_instance("A1", 50, "server_1",
+			      "/home/file_1", 2222222222)
+    content_data.add_instance("A1", 50, "server_1",
+			      "/home/file_2", 3333333333)
+    content_data.content_exists('A1').should == true
+    content_data.instance_exists('/home/file_1', 'server_1').should == true
+    content_data.remove_instance('server_1', '/home/file_1')
+    content_data.instance_exists('/home/file_1', 'server_1').should == false
+    content_data.content_exists('A1').should == true
+
+    #remove also removes content
+    content_data.remove_instance('server_1', '/home/file_2')
+    content_data.instance_exists('/home/file_1', 'server_1').should == false
+    content_data.content_exists('A1').should == false
+  end
+
+  it 'test to file from file' do
+    content_data = ContentData::ContentData.new
+    content_data.add_instance("A1", 50, "server_1",
+			      "/home/file_1", 22222222222)
+    content_data.add_instance("B1", 60, "server_1",
+			      "/home/file_2", 44444444444)
+    content_data.add_instance("B1", 60, "server_1",
+			      "/home/file_3", 55555555555)
+    file_moc_object = StringIO.new
+    file_moc_object.write(content_data.to_s)
+    test_file = Tempfile.new('content_data_spec.test')
+    content_data.to_file(test_file)
+    content_data_2 = ContentData::ContentData.new
+    content_data_2.from_file(test_file)
+    (content_data == content_data_2).should == true
+  end
+
+  it 'test merge' do
+    content_data_a = ContentData::ContentData.new
+    content_data_a.add_instance("A1", 50, "server_1",
+				"/home/file_1", 22222222222)
+
+    content_data_b = ContentData::ContentData.new
+    content_data_b.add_instance("B1", 60, "server_1",
+				"/home/file_2", 44444444444)
+    content_data_b.add_instance("B1", 60, "server_1",
+				"/home/file_3", 55555555555)
+    content_data_merged = ContentData.merge(content_data_a, content_data_b)
+    expected =  "2\nB1,60,44444444444\nA1,50,22222222222\n" + 
+	"3\nB1,60,server_1,/home/file_2,44444444444\nB1,60,server_1,/home/file_3,55555555555\n" + 
+	"A1,50,server_1,/home/file_1,22222222222\n"
+    content_data_merged.to_s.should == expected
+    content_data_a.remove_instance('server_1', '/home/file_1')
+    expected = "2\nB1,60,44444444444\nA1,50,22222222222\n" + 
+	"3\nB1,60,server_1,/home/file_2,44444444444\nB1,60,server_1,/home/file_3,55555555555\n" + 
+	"A1,50,server_1,/home/file_1,22222222222\n"
+    content_data_merged.to_s.should == expected
+    content_data_b.remove_instance('server_1', '/home/file_2')
+    expected = "2\nB1,60,44444444444\nA1,50,22222222222\n" + 
+	"3\nB1,60,server_1,/home/file_2,44444444444\nB1,60,server_1,/home/file_3,55555555555\n" + 
+	"A1,50,server_1,/home/file_1,22222222222\n"
+    content_data_merged.to_s.should == expected
+  end
+
+  it 'test merge override b' do
+    content_data_a = ContentData::ContentData.new
+    content_data_a.add_instance("A1", 50, "server_1",
+				"/home/file_1", 22222222222)
+
+    content_data_b = ContentData::ContentData.new
+    content_data_b.add_instance("A1", 50, "server_1",
+				"/home/file_1", 22222222222)
+    content_data_b.add_instance("B1", 60, "server_1",
+				"/home/file_2", 44444444444)
+    content_data_b.add_instance("B1", 60, "server_1",
+				"/home/file_3", 55555555555)
+    content_data_merged = ContentData.merge_override_b(content_data_a, content_data_b)
+    expected = "2\nA1,50,22222222222\nB1,60,44444444444\n" + 
+	"3\nA1,50,server_1,/home/file_1,22222222222\nB1,60,server_1,/home/file_2,44444444444\n" + 
+	"B1,60,server_1,/home/file_3,55555555555\n"
+    content_data_merged.to_s.should == expected
+
+    content_data_a.remove_instance('server_1', '/home/file_1')
+    expected = "2\nA1,50,22222222222\nB1,60,44444444444\n" + 
+	"3\nA1,50,server_1,/home/file_1,22222222222\n" + 
+	"B1,60,server_1,/home/file_2,44444444444\nB1,60,server_1,/home/file_3,55555555555\n"
+    content_data_merged.to_s.should == expected
+    content_data_b.remove_instance('server_1', '/home/file_2')
+    expected = "2\nA1,50,22222222222\nB1,60,44444444444\n" + 
+	"2\nA1,50,server_1,/home/file_1,22222222222\nB1,60,server_1,/home/file_3,55555555555\n"
+    content_data_b.to_s.should == expected
+
+    (content_data_merged == content_data_b).should == true
+  end
+
+  it 'test remove' do
+    content_data_a = ContentData::ContentData.new
+    content_data_a.add_instance("A1", 50, "server_1",
+				"/home/file_1", 22222222222)
+
+    content_data_b = ContentData::ContentData.new
+    content_data_b.add_instance("A1", 50, "server_1",
+				"/home/file_1", 22222222222)
+    content_data_b.add_instance("A1", 50, "server_1",
+				"extra_inst", 66666666666)
+    content_data_b.add_instance("B1", 60, "server_1",
+				"/home/file_2", 44444444444)
+    content_data_b.add_instance("B1", 60, "server_1",
+				"/home/file_3", 55555555555)
+    content_data_removed = ContentData.remove(content_data_a, content_data_b)
+    expected =  "1\nB1,60,44444444444\n2\nB1,60,server_1,/home/file_2,44444444444\nB1,60,server_1,/home/file_3,55555555555\n"
+    content_data_removed.to_s.should == expected
+    content_data_a.remove_instance('server_1', '/home/file_1')
+    expected =  "1\nB1,60,44444444444\n2\nB1,60,server_1,/home/file_2,44444444444\nB1,60,server_1,/home/file_3,55555555555\n"
+    content_data_removed.to_s.should == expected
+    content_data_b.remove_instance('server_1', '/home/file_2')
+    expected =  "1\nB1,60,44444444444\n2\nB1,60,server_1,/home/file_2,44444444444\nB1,60,server_1,/home/file_3,55555555555\n"
+    content_data_removed.to_s.should == expected
+
+    #check nil
+    content_data_removed = ContentData.remove(nil, content_data_b)
+    content_data_b.to_s.should == content_data_removed.to_s
+    content_data_removed = ContentData.remove(content_data_b, nil)
+    nil.should == content_data_removed
+  end
+
+  it 'test remove instances' do
+    content_data_a = ContentData::ContentData.new
+    content_data_a.add_instance("A1", 50, "server_1",
+				"/home/file_1", 22222222222)
+
+    content_data_b = ContentData::ContentData.new
+    content_data_b.add_instance("A1", 50, "server_1",
+				"/home/file_1", 22222222222)
+    content_data_b.add_instance("A1", 50, "server_1",
+				"extra_inst", 66666666666)
+    content_data_b.add_instance("B1", 60, "server_1",
+				"/home/file_2", 44444444444)
+    content_data_b.add_instance("B1", 60, "server_1",
+				"/home/file_3", 55555555555)
+    content_data_removed = ContentData.remove_instances(content_data_a, content_data_b)
+    expected = "2\nA1,50,22222222222\nB1,60,44444444444\n" + 
+	"3\nA1,50,server_1,extra_inst,66666666666\nB1,60,server_1,/home/file_2,44444444444\n" + 
+	"B1,60,server_1,/home/file_3,55555555555\n"
+    content_data_removed.to_s.should == expected
+    content_data_a.remove_instance('server_1', '/home/file_1')
+    expected = "2\nA1,50,22222222222\nB1,60,44444444444\n" + 
+	"3\nA1,50,server_1,extra_inst,66666666666\nB1,60,server_1,/home/file_2,44444444444\n" + 
+	"B1,60,server_1,/home/file_3,55555555555\n"
+    content_data_removed.to_s.should == expected
+    content_data_b.remove_instance('server_1', '/home/file_2')
+    expected = "2\nA1,50,22222222222\nB1,60,44444444444\n" + 
+	"3\nA1,50,server_1,extra_inst,66666666666\nB1,60,server_1,/home/file_2,44444444444\n" + 
+	"B1,60,server_1,/home/file_3,55555555555\n"
+    content_data_removed.to_s.should == expected
+
+    #nil.should == content_data_b
+
+    #check nil
+    content_data_removed = ContentData.remove_instances(nil, content_data_b)
+    content_data_b.to_s.should == content_data_removed.to_s
+    content_data_removed = ContentData.remove_instances(content_data_b, nil)
+    content_data_removed.should == nil
+  end
+
+  it 'test remove directory' do
+
+    content_data_b = ContentData::ContentData.new
+    content_data_b.add_instance("A1", 50, "server_1",
+				"/home/file_1", 22222222222)
+    content_data_b.add_instance("A1", 50, "server_1",
+				"extra_inst", 66666666666)
+    content_data_b.add_instance("B1", 60, "server_1",
+				"/home/file_2", 44444444444)
+    content_data_b.add_instance("B1", 60, "server_1",
+				"/home/file_3", 55555555555)
+    content_data_removed = ContentData.remove_directory(content_data_b, 'home', "server_1")
+    expected = "1\nA1,50,22222222222\n1\nA1,50,server_1,extra_inst,66666666666\n"
+    content_data_removed.to_s.should == expected
+
+    content_data_b.remove_instance('server_1', '/home/file_2')
+    expected = "1\nA1,50,22222222222\n1\nA1,50,server_1,extra_inst,66666666666\n"
+    content_data_removed.to_s.should == expected
+
+    expected = "2\nA1,50,22222222222\nB1,60,44444444444\n" + 
+	"3\nA1,50,server_1,/home/file_1,22222222222\nA1,50,server_1,extra_inst,66666666666\n" + 
+	"B1,60,server_1,/home/file_3,55555555555\n"
+    content_data_b.to_s.should == expected
+
+    content_data_b = ContentData::ContentData.new
+    content_data_removed = ContentData.remove_directory(content_data_b, 'home', "server_1")
+    content_data_removed.contents_size.should == 0
+    content_data_removed = ContentData.remove_directory(nil, 'home', "server_1")
+    content_data_removed.should == nil
+  end
+
+  it 'test intersect' do
+    content_data_a = ContentData::ContentData.new
+    content_data_a.add_instance("A1", 50, "server_1",
+				"/home/file_1", 22222222222)
+    content_data_a.add_instance("C1", 70, "server_1",
+				"/home/file_4", 77777777777)
+    content_data_b = ContentData::ContentData.new
+    content_data_b.add_instance("A1", 50, "server_1",
+				"/home/file_1", 22222222222)
+    content_data_b.add_instance("A1", 50, "server_1",
+				"/home/file_5", 55555555555)
+    content_data_b.add_instance("B1", 60, "server_1",
+				"/home/file_2", 44444444444)
+
+    content_data_intersect = ContentData.intersect(content_data_a, content_data_b)
+    expected =  "1\nA1,50,22222222222\n" + 
+	"2\nA1,50,server_1,/home/file_1,22222222222\nA1,50,server_1,/home/file_5,55555555555\n"
+    content_data_intersect.to_s.should == expected
+  end
+
+  it 'test unify time' do
+    content_data_a = ContentData::ContentData.new
+    content_data_a.add_instance("A1", 50, "server_1",
+				"/home/file_1", 22222222222)
+    content_data_a.add_instance("C1", 70, "server_1",
+				"/home/file_4", 77777777777)
+    content_data_a.add_instance("C1", 70, "server_1",
+				"/home/file_5", 33333333333)
+    content_data_a.unify_time
+    expected =  "2\nA1,50,22222222222\nC1,70,33333333333\n" + 
+	"3\nA1,50,server_1,/home/file_1,22222222222\nC1,70,server_1,/home/file_4,33333333333\n" + 
+	"C1,70,server_1,/home/file_5,33333333333\n"
+    content_data_a.to_s.should == expected
+  end
+end
+

data/spec/content_data/validations_spec.rb

@@ -1,3 +1,8 @@
+# NOTE Code Coverage block must be issued before any of your application code is required
+if ENV['BBFS_COVERAGE']
+  require_relative '../spec_helper.rb'
+  SimpleCov.command_name 'content_data'
+end
 require 'rspec'
 require 'file_indexing/index_agent'
 require 'params'