content_server 1.6.0 → 1.6.1

checksums.yaml

@@ -1,15 +1,7 @@
 ---
-!binary "U0hBMQ==":
-  metadata.gz: !binary |-
-    ODk2ZWZlZGIwODU5Y2I1YTg3YzIyZWZmNDQyOGRiOTJhMWMwODJiZg==
-  data.tar.gz: !binary |-
-    NWQxY2E1NGE2ZmQ3YzRlYzFjN2QwNTRlMWMxYzFjMmZhODlhNDVlNA==
+SHA1:
+  metadata.gz: 40813241e0bc388830199783e2c5b6d5e042ce6e
+  data.tar.gz: 59340d51c6c59f5d73460796f790c39de77444be
 SHA512:
-  metadata.gz: !binary |-
-    MmViNDAxZDU0NTk4ZDY0YzNiYTkxZWVkZjAxYzMyNjZhM2Y0ZDE2MmE3NDNl
-    YTE3MTJlZGU0NzBjZjJjYzNmM2Q5YWM1NzEyNzgxZDM4MmRmZDIyOWUxMGY3
-    MjU4YjMxMDFlOThkNDM1M2ZiZTJhN2RjNzFkZmZhYWZkZWE4MTM=
-  data.tar.gz: !binary |-
-    YmQxOWYzZGFhZDY5YjNmOTBkN2VhMDBiOTQwZTdjMmQxNzhmZDgxMzE3YzVl
-    OTg4NDRmYmE5ZDU3ZWJhMGU2YzNiYTQ2Y2M4OTEyMGQzYzNmOWJkYzc0YmMw
-    OTJhNmQ3YTY3MDIzZDI5NmYzOThhNjA0MjQ4MTI0ZGY5OGI1Nzc=
+  metadata.gz: 7741efd82d5a28a6c93bdf067fb3d53310155799ed8d3bb3803e5a299b7f78e15aa36b2583674311e8bda4813f4aadc5efac5930728bd068dcdc88b9a5271494
+  data.tar.gz: 12192e8cbce686cef6c473cc1668da1c887c22444de5d27b39e76af8e6d344c92d1dd43226be721654b07f3d02c8daa8f1d8beb40f0b0a8d153ed6f85d8a8c29

data/lib/content_data/content_data.rb

@@ -1,6 +1,8 @@
+require 'csv'
 require 'content_server/server'
 require 'log'
 require 'params'
+require 'zlib'
 
 module ContentData
   Params.string('instance_check_level', 'shallow', 'Defines check level. Supported levels are: ' \
@@ -31,6 +33,8 @@ module ContentData
 
     CHUNK_SIZE = 5000
 
+    # NOTE Cloning is time/memory expensive operation.
+    # It is highly recomended to avoid it.
     def initialize(other = nil)
       if other.nil?
         @contents_info = {}  # Checksum --> [size, paths-->time(instance), time(content)]
@@ -292,6 +296,7 @@ module ContentData
       end
     end
 
+    # Don't use in production, only for testing.
     def to_s
       return_str = ""
       contents_str = ""
@@ -306,14 +311,92 @@ module ContentData
       return_str << contents_str
       return_str << "%d\n" % [@instances_info.length]
       return_str << instances_str
+
+
+      symlinks_str = ""
+      each_symlink { |server, path, target|
+        symlinks_str << "%s,%s,%s\n" % [server, path, target]
+      }
+      return_str << symlinks_str
+
       return_str
     end
 
+    # Write content data to file.
+    def to_file(filename)
+      content_data_dir = File.dirname(filename)
+      FileUtils.makedirs(content_data_dir) unless File.directory?(content_data_dir)
+      Zlib::GzipWriter.open(filename) do |gz| 
+        gz.write [@instances_info.length].to_csv
+        each_instance { |checksum, size, content_mod_time, instance_mod_time, server, path, inst_index_time|
+          gz.write [checksum, size, server, path, instance_mod_time, inst_index_time].to_csv
+        }
+        gz.write [@symlinks_info.length].to_csv
+        each_symlink { |file, path, target|
+          gz.write [file, path, target].to_csv
+        }
+      end
+    end
+
+    # Imports content data from file.
+    # This method will throw an exception if the file is not in correct format.
+    def from_file(filename)
+      unless File.exists? filename
+        raise ArgumentError.new "No such a file #{filename}"
+      end
+
+      number_of_instances = nil
+      number_of_symlinks = nil
+      Zlib::GzipReader.open(filename) { |gz|
+        gz.each_line do |line|
+          row = line.parse_csv
+          if number_of_instances == nil
+            begin
+              # get number of instances
+              number_of_instances = row[0].to_i
+            rescue ArgumentError
+              raise("Parse error of content data file:#{filename}  line ##{$.}\n" +
+                    "number of instances should be a Number. We got:#{number_of_instances}")
+            end
+          elsif number_of_instances > 0
+            if (6 != row.length)
+              raise("Parse error of content data file:#{filename}  line ##{$.}\n" +
+                        "Expected to read 6 fields ('<' separated) but got #{row.length}.\nLine:#{instance_line}")
+            end
+            add_instance(row[0],        #checksum
+                         row[1].to_i,   # size
+                         row[2],        # server
+                         row[3],        # path
+                         row[4].to_i,   # mod time
+                         row[5].to_i)   # index time
+            number_of_instances -= 1
+          elsif number_of_symlinks == nil
+            begin
+              # get number of symlinks
+              number_of_symlinks = row[0].to_i
+            rescue ArgumentError
+              raise("Parse error of content data file:#{filename}  line ##{$.}\n" +
+                    "number of symlinks should be a Number. We got:#{number_of_symlinks}")
+            end
+          elsif number_of_symlinks > 0
+            if (3 != row.length)
+              raise("Parse error of content data file:#{filename}  line ##{$.}\n" +
+                    "Expected to read 3 fields ('<' separated) but got #{row.length}.\nLine:#{symlinks_line}")
+            end
+            @symlinks_info[[row[0], row[1]]] = row[2]
+            number_of_symlinks -= 1 
+          end
+        end
+      }
+    end
+
+    ############## DEPRECATED: Old deprecated from/to file methods still needed for migration purposes
     # Write content data to file.
     # Write is using chunks (for both content chunks and instances chunks)
     # Chunk is used to maximize GC affect. The temporary memory of each chunk is GCed.
     # Without the chunks used in a dipper stack level, GC keeps the temporary objects as part of the stack context.
-    def to_file(filename)
+    # @deprecated
+    def to_file_old(filename)
       content_data_dir = File.dirname(filename)
       FileUtils.makedirs(content_data_dir) unless File.directory?(content_data_dir)
       File.open(filename, 'w') { |file|
@@ -323,7 +406,7 @@ module ContentData
         content_chunks = @contents_info.length / CHUNK_SIZE + 1
         chunks_counter = 0
         while chunks_counter < content_chunks
-          to_file_contents_chunk(file,contents_enum, CHUNK_SIZE)
+          to_old_file_contents_chunk(file,contents_enum, CHUNK_SIZE)
           GC.start
           chunks_counter += 1
         end
@@ -333,7 +416,7 @@ module ContentData
         contents_enum = @contents_info.each_key
         chunks_counter = 0
         while chunks_counter < content_chunks
-          to_file_instances_chunk(file,contents_enum, CHUNK_SIZE)
+          to_old_file_instances_chunk(file,contents_enum, CHUNK_SIZE)
           GC.start
           chunks_counter += 1
         end
@@ -343,22 +426,22 @@ module ContentData
         file.write("#{@symlinks_info.length}\n")
         loop {
           symlink_key = symlinks_info_enum.next rescue break
-          file.write("#{symlink_key[0]}<#{symlink_key[1]}<#{@symlinks_info[symlink_key]}\n")
+          file.write("#{symlink_key[0]},#{symlink_key[1]},#{@symlinks_info[symlink_key]}\n")
         }
       }
     end
 
-    def to_file_contents_chunk(file, contents_enum, chunk_size)
+    def to_old_file_contents_chunk(file, contents_enum, chunk_size)
       chunk_counter = 0
       while chunk_counter < chunk_size
         checksum = contents_enum.next rescue return
         content_info = @contents_info[checksum]
-        file.write("#{checksum}<#{content_info[0]}<#{content_info[2]}\n")
+        file.write("#{checksum},#{content_info[0]},#{content_info[2]}\n")
         chunk_counter += 1
       end
     end
 
-    def to_file_instances_chunk(file, contents_enum, chunk_size)
+    def to_old_file_instances_chunk(file, contents_enum, chunk_size)
       chunk_counter = 0
       while chunk_counter < chunk_size
         checksum = contents_enum.next rescue return
@@ -369,8 +452,8 @@ module ContentData
           # provide the block with: checksum, size, content modification time,instance modification time,
           #   server and path.
           instance_modification_time,instance_index_time = content_info[1][location]
-          file.write("#{checksum}<#{content_info[0]}<#{location[0]}<#{location[1]}<" +
-                     "#{instance_modification_time}<#{instance_index_time}\n")
+          file.write("#{checksum},#{content_info[0]},#{location[0]},#{location[1]}," +
+                     "#{instance_modification_time},#{instance_index_time}\n")
         }
         chunk_counter += 1
         break if chunk_counter == chunk_size
@@ -380,7 +463,7 @@ module ContentData
     # TODO validation that file indeed contains ContentData missing
     # TODO class level method?
     # Loading db from file using chunks for better memory performance
-    def from_file(filename)
+    def from_file_old(filename)
       # read first line (number of contents)
       # calculate line number (number of instances)
       # read number of instances.
@@ -409,7 +492,7 @@ module ContentData
             # update last chunk size
             chunk_size = number_of_contents - (chunk_index * CHUNK_SIZE)
           end
-          return unless read_contents_chunk(filename, file, chunk_size)
+          return unless read_old_contents_chunk(filename, file, chunk_size)
           GC.start
           chunk_index += 1
         end
@@ -431,7 +514,7 @@ module ContentData
             # update last chunk size
             chunk_size = number_of_instances - (chunk_index * CHUNK_SIZE)
           end
-          return unless read_instances_chunk(filename, file, chunk_size)
+          return unless read_old_instances_chunk(filename, file, chunk_size)
           GC.start
           chunk_index += 1
         end
@@ -443,15 +526,15 @@ module ContentData
                     "number of symlinks should be a Number. We got:#{number_of_symlinks}")
         end
         number_of_symlinks.to_i.times {
-          symlinks_line = file.gets
+          symlinks_line = file.gets.strip
           unless symlinks_line
             raise("Parse error of content data file:#{filename}  line ##{$.}\n" +
                    "Expected to read symlink line but reached EOF")
           end
-          parameters = symlinks_line.split('<')
+          parameters = symlinks_line.split(',')
           if (3 != parameters.length)
             raise("Parse error of content data file:#{filename}  line ##{$.}\n" +
-                  "Expected to read 3 fields ('<' separated) but got #{parameters.length}.\nLine:#{symlinks_line}")
+                  "Expected to read 3 fields (comma separated) but got #{parameters.length}.\nLine:#{symlinks_line}")
           end
 
           @symlinks_info[[parameters[0],parameters[1]]] = parameters[2]
@@ -459,7 +542,7 @@ module ContentData
       }
     end
 
-    def read_contents_chunk(filename, file, chunk_size)
+    def read_old_contents_chunk(filename, file, chunk_size)
       chunk_index = 0
       while chunk_index < chunk_size
         unless file.gets
@@ -471,7 +554,7 @@ module ContentData
       true
     end
 
-    def read_instances_chunk(filename, file, chunk_size)
+    def read_old_instances_chunk(filename, file, chunk_size)
       chunk_index = 0
       while chunk_index < chunk_size
         instance_line = file.gets
@@ -480,11 +563,17 @@ module ContentData
                 "Expected to read Instance line but reached EOF")
         end
 
-        parameters = instance_line.split('<')
-        if (6 != parameters.length)
-          raise("Parse error of content data file:#{filename}  line ##{$.}\n" +
-                    "Expected to read 6 fields ('<' separated) but got #{parameters.length}.\nLine:#{instance_line}")
+        parameters = instance_line.split(',')
+        # bugfix: if file name consist a comma then parsing based on comma separating fails
+        if (parameters.size > 6)
+          (4..parameters.size-3).each do |i|
+            parameters[3] = [parameters[3], parameters[i]].join(",")
+          end
+          (4..parameters.size-3).each do |i|
+            parameters.delete_at(4)
+          end
         end
+
         add_instance(parameters[0],        #checksum
                      parameters[1].to_i,   # size
                      parameters[2],        # server
@@ -495,6 +584,7 @@ module ContentData
       end
       true
     end
+    ########################## END OF DEPRECATED PART #######################
 
     # for each content, all time fields (content and instances) are replaced with the
     # min time found, while going through all time fields.
@@ -763,11 +853,17 @@ module ContentData
   def self.remove(a, b)
     return nil if b.nil?
     return ContentData.new(b) if a.nil?
-    c = ContentData.new(b)  # create new cloned content C from B
-    # remove contents of A from newly cloned content A
-    a.each_content { |checksum, size, content_mod_time|
-      c.remove_content(checksum)
-    }
+    c = ContentData.new
+    b.each_instance do |checksum, size, _, instance_mtime, server, path, index_time|
+      unless (a.content_exists(checksum))
+              c.add_instance(checksum,
+                             size,
+                             server,
+                             path,
+                             instance_mtime,
+                             index_time)
+      end
+    end
     c
   end
 

data/lib/content_server/version.rb

@@ -1,3 +1,3 @@
   module ContentServer
-    VERSION = "1.6.0"
+    VERSION = "1.6.1"
   end

data/lib/file_indexing/index_agent.rb

@@ -7,7 +7,6 @@ require 'content_data'
 require 'file_indexing/indexer_patterns'
 require 'log'
 
-
 module FileIndexing
 ####################
 # Index Agent

data/spec/content_data/content_data_performance_spec.rb

@@ -0,0 +1,321 @@
+# NOTE Code Coverage block must be issued before any of your application code is required
+if ENV['BBFS_COVERAGE']
+  require_relative '../spec_helper.rb'
+  SimpleCov.command_name 'content_data'
+end
+
+require 'rspec'
+require 'tempfile'
+require_relative '../../lib/content_data/content_data.rb'
+
+# NOTE the results are not exact cause they do not run in a clean environment and influenced from
+# monitoring/testing code, but they give a good approximation
+# Supposition: monitoring code penalty is insignificant against time/memory usage of the tested code
+describe 'Content Data Performance Test', :perf =>true do
+
+  NUMBER_INSTANCES = 350_000
+  MAX_CHECKSUM = NUMBER_INSTANCES
+  INSTANCE_SIZE = 1000
+  SERVER = "server"
+  PATH = "file_"
+  MTIME = 1000
+
+  # in kilobytes
+  LIMIT_MEMORY = 250*(1024)  # 250 MB
+  # in seconds
+  LIMIT_TIME = 5*60;  # 5 minutes
+
+  before :all do
+    Params.init Array.new
+    Params['print_params_to_stdout'] = false
+    # must preced Log.init, otherwise log containing default values will be created
+    Params['log_write_to_file'] = false
+    Params['log_write_to_console'] = true
+    Params['log_debug_level'] = 1  # set it > 0 to enable print-outs of used time/memory
+    Log.init
+
+    # module variable contains an initialized ContentData
+    @test_cd = get_initialized
+  end
+
+  let (:terminator) { Limit.new(LIMIT_TIME, LIMIT_MEMORY) }
+
+  # Print-out status messages
+  after :each do
+    unless (terminator.nil?)
+      Log.debug1("#{self.class.description} #{example.description}: #{terminator.msg}")
+    end
+    GC.start
+  end
+
+  # Initialize ContentData object with generated instances
+  # @return [ContentData] an initialized object
+  def get_initialized
+    initialized_cd = ContentData::ContentData.new
+    NUMBER_INSTANCES.times do |i|
+      initialized_cd.add_instance(i.to_s, INSTANCE_SIZE+i, SERVER, PATH + i.to_s, MTIME+i)
+    end
+    initialized_cd
+  end
+
+  # TODO consider to separate it to 2 derived classes: one for memory, one for time monitoring
+  class Limit
+    attr_reader :elapsed_time, :memory_usage, :msg
+
+    def initialize(time_limit, memory_limit)
+      @elapsed_time = 0
+      @memory_usage = 0
+      @msg = String.new
+      @time_limit = time_limit
+      @memory_limit = memory_limit
+    end
+
+    def get_timer_thread(watched_thread)
+      Thread.new do
+        while (@elapsed_time < @time_limit && watched_thread.alive?)
+          @elapsed_time += 1
+          sleep 1
+        end
+        is_succeeded = true
+        if (watched_thread.alive?)
+          Thread.kill(watched_thread)
+          is_succeeded = false
+        end
+        @msg = (is_succeeded ? "" : "do not ") + "finished in #{@elapsed_time} seconds"
+      end
+    end
+
+    def get_memory_limit_thread(watched_thread)
+      Thread.new do
+        init_memory_usage = Process.get_memory_usage
+        while (@memory_usage < @memory_limit && watched_thread.alive?)
+          cur_memory_usage = Process.get_memory_usage - init_memory_usage
+          if (cur_memory_usage > @memory_usage)
+            @memory_usage = cur_memory_usage
+          end
+          sleep 1
+        end
+        if (watched_thread.alive?)
+          Thread.kill(watched_thread)
+        end
+        @msg = "memory usage: #{@memory_usage}"
+      end
+    end
+  end
+
+  # TODO consider more general public method call_with_limit
+  class Proc
+    # Run a procedure.
+    # Terminate it if it is running more then a time limit
+    # TODO consider usage of Process::setrlimit
+    # @param [Limit] limit object
+    def call_with_timer(limit)
+      call_thread = Thread.new { self.call }
+      limit_thread = limit.get_timer_thread(call_thread)
+      [call_thread, limit_thread].each { |th| th.join }
+    end
+
+    # Run a procedure.
+    # Terminate it if it is running more then a memory limit
+    # TODO consider usage of Process::setrlimit
+    # @param [Limit] limit object
+    def call_with_memory_limit(limit)
+      call_thread = Thread.new { self.call }
+      limit_thread = limit.get_memory_limit_thread(call_thread)
+      [call_thread, limit_thread].each { |th| th.join }
+    end
+  end
+
+  module Process
+    # @return [boolean] true when process alive, false otherwise
+    # @raise [Errno::EPERM] if process owned by other user, and there is no permissions to check
+    #   (not our case)
+    def Process.alive?(pid)
+      begin
+        Process.kill(0, pid)
+        true
+      rescue Errno::ESRCH
+        false
+      end
+    end
+
+    # Get memory consumed my the process
+    # @param [Integer] pid, default is current pid
+    # @return [Integer] memory usage in kilobytes
+    def Process.get_memory_usage(pid = Process.pid)
+      if Gem::win_platform?
+        `tasklist /FI \"PID eq #{pid}\" /NH /FO \"CSV\"`.split(',')[4]
+      else
+        `ps -o rss= -p #{pid}`.to_i
+      end
+    end
+  end
+
+  context 'Object initialization' do
+    context 'Init one object' do
+      it "#{NUMBER_INSTANCES} instances in less then #{LIMIT_TIME} seconds" do
+        cd1 = nil
+        init_proc = Proc.new { cd1 = get_initialized }
+        init_proc.call_with_timer(terminator)
+        terminator.elapsed_time.should < LIMIT_TIME
+
+        # checks that test was correct
+        cd1.should be
+        cd1.instances_size.should == NUMBER_INSTANCES
+      end
+
+      it "#{NUMBER_INSTANCES} instances consumes less then #{LIMIT_MEMORY} KB" do
+        cd1 = nil
+        init_proc = Proc.new { cd1 = get_initialized }
+        init_proc.call_with_memory_limit(terminator)
+        terminator.memory_usage.should < LIMIT_MEMORY
+
+        # checks that test was correct
+        cd1.should be
+        cd1.instances_size.should == NUMBER_INSTANCES
+      end
+
+      it "clone of #{NUMBER_INSTANCES} in less then #{LIMIT_TIME} seconds" do
+        cd2 = nil
+        clone_proc = Proc.new { cd2 = ContentData::ContentData.new(@test_cd) }
+        clone_proc.call_with_timer(terminator)
+        terminator.elapsed_time.should < LIMIT_TIME
+
+        # checks that test
+        cd2.should be
+        cd2.instances_size.should == @test_cd.instances_size
+      end
+
+      it "clone of #{NUMBER_INSTANCES} consumes less then #{LIMIT_MEMORY} KB" do
+        cd2 = nil
+        clone_proc = Proc.new { cd2 = ContentData::ContentData.new(@test_cd) }
+        clone_proc.call_with_memory_limit(terminator)
+        terminator.memory_usage.should < LIMIT_MEMORY
+
+        # checks that test was correct
+        cd2.should be
+        cd2.instances_size.should == @test_cd.instances_size
+      end
+    end
+
+
+    context 'Init more then one object' do
+      it "two object of #{NUMBER_INSTANCES} instances each in less then #{2*LIMIT_TIME} seconds" do
+        cd1 = nil
+        cd2 = nil
+        build_proc = Proc.new do
+          cd1 = get_initialized
+          cd2 = get_initialized
+        end
+        build_proc.call_with_timer(terminator)
+        terminator.elapsed_time.should < LIMIT_TIME
+
+        # checks that test was correct
+        cd1.should be
+        cd1.instances_size.should == NUMBER_INSTANCES
+        cd2.should be
+        cd2.instances_size.should == NUMBER_INSTANCES
+      end
+
+      it "three object of #{NUMBER_INSTANCES} instances each in less then #{3*LIMIT_TIME} seconds" do
+        cd1 = nil
+        cd2 = nil
+        cd3 = nil
+        build_proc = Proc.new do
+          cd1 = get_initialized
+          cd2 = get_initialized
+          cd3 = get_initialized
+        end
+        build_proc.call_with_timer(terminator)
+        terminator.elapsed_time.should < LIMIT_TIME
+
+        # checks that test was correct
+        cd1.should be
+        cd1.instances_size.should == NUMBER_INSTANCES
+        cd2.should be
+        cd2.instances_size.should == NUMBER_INSTANCES
+        cd3.should be
+        cd3.instances_size.should == NUMBER_INSTANCES
+      end
+    end
+  end
+
+  context 'Iteration' do
+    it "each instance on #{NUMBER_INSTANCES} instances in less then #{LIMIT_TIME}" do
+      each_thread = Proc.new { @test_cd.each_instance { |ch,_,_,_,_,_,_| ch } }
+      each_thread.call_with_timer(terminator)
+      terminator.elapsed_time.should < LIMIT_TIME
+    end
+  end
+
+  context 'File operations' do
+    before :all do
+      @file = Tempfile.new('to_file_test')
+      @path = @file.path
+    end
+
+    after :all do
+      @file.close!
+    end
+
+    it "save/load on #{NUMBER_INSTANCES} instances in less then #{LIMIT_TIME} seconds" do
+      # Checking to_file
+      to_file_proc = Proc.new do
+        begin
+          @test_cd.to_file(@path)
+        ensure
+          @file.close
+        end
+      end
+      to_file_proc.call_with_timer(terminator)
+      terminator.elapsed_time.should < LIMIT_TIME
+
+      # checking from_file
+      cd2 = ContentData::ContentData.new
+      from_file_proc = Proc.new do
+        begin
+          cd2.from_file(@path)
+        ensure
+          @file.close
+        end
+      end
+      terminator = Limit.new(LIMIT_TIME, LIMIT_MEMORY)
+      from_file_proc.call_with_timer(terminator)
+      terminator.elapsed_time.should < LIMIT_TIME
+
+      # checks that test was correct
+      cd2.instances_size.should == @test_cd.instances_size
+    end
+  end
+
+  describe 'Set operations' do
+    before :all do
+      @cd2 = get_initialized
+      @cd2.add_instance((MAX_CHECKSUM+1).to_s, INSTANCE_SIZE, SERVER, PATH + "new", MTIME)
+    end
+
+    context "minus of two objects with #{NUMBER_INSTANCES} instances each" do
+      it "finish in less then #{LIMIT_TIME} seconds" do
+        res_cd = nil
+        minus_proc = Proc.new { res_cd = ContentData.remove(@test_cd, @cd2) }
+        minus_proc.call_with_timer(terminator)
+        terminator.elapsed_time.should < LIMIT_TIME
+
+        # checks that test was correct
+        res_cd.should be
+        res_cd.instances_size.should == 1
+      end
+
+      it "consume less then #{LIMIT_MEMORY} KB" do
+        res_cd = nil
+        minus_proc = Proc.new { res_cd = ContentData.remove(@test_cd, @cd2) }
+        minus_proc.call_with_memory_limit(terminator)
+        terminator.memory_usage.should < LIMIT_MEMORY
+
+        # checks that test was correct
+        res_cd.should be
+        res_cd.instances_size.should == 1
+      end
+    end
+  end
+end

data/spec/content_data/content_data_spec.rb

@@ -1,3 +1,4 @@
+# coding: UTF-8
 # NOTE Code Coverage block must be issued before any of your application code is required
 if ENV['BBFS_COVERAGE']
   require_relative '../spec_helper.rb'
@@ -125,6 +126,9 @@ describe 'Content Data Test' do
 			      "/home/file_2", 44444444444)
     content_data.add_instance("B1", 60, "server_1",
 			      "/home/file_3", 55555555555)
+    content_data.add_symlink("A1", "/home/symlink_1", "home/file_1")
+    content_data.add_symlink("B1", "/home/symlink_2", "home/file_xxx")
+    content_data.add_symlink("B1", "/home/symlink_1", "home/file_3")
     file_moc_object = StringIO.new
     file_moc_object.write(content_data.to_s)
     test_file = Tempfile.new('content_data_spec.test')
@@ -134,6 +138,26 @@ describe 'Content Data Test' do
     (content_data == content_data_2).should == true
   end
 
+  it 'test old format with comma' do
+    content_data = ContentData::ContentData.new
+    content_data.add_instance("A1", 50, "server_1",
+  			      "/home/file,<><,ласкдфй_1", 22222222222)
+    content_data.add_instance("B1", 60, "server_1",
+  			      "/home/filךלחת:,!גדכשe_2", 44444444444)
+    content_data.add_instance("B1", 60, "server_1",
+  			      "/home/filкакакаe_3", 55555555555)
+    content_data.add_symlink("A1", "/home/syласдфйmlink_1", "home/file_1")
+    content_data.add_symlink("B1", "/home/symlinkדגכע_2", "home/file_xxx")
+    content_data.add_symlink("B1", "/home/symlinkדכע_1", "home/filкакакаe_3")
+    file_moc_object = StringIO.new
+    file_moc_object.write(content_data.to_s)
+    test_file = Tempfile.new('content_data_spec.test')
+    content_data.to_file_old(test_file)
+    content_data_2 = ContentData::ContentData.new
+    content_data_2.from_file_old(test_file)
+    (content_data == content_data_2).should == true
+  end
+
   it 'test merge' do
     content_data_a = ContentData::ContentData.new
     content_data_a.add_instance("A1", 50, "server_1",

data/spec/content_server/file_streamer_spec.rb

@@ -15,7 +15,7 @@ Params['log_write_to_console'] = false
 Params['log_write_to_file'] = false
 Params['log_debug_level'] = 0
 Params['streaming_chunk_size'] = 5
-Params.init ARGV
+Params.init Array.new
 Params['log_write_to_file'] = false
 Params['log_write_to_console'] = false
 Params['enable_monitoring'] = false

metadata

@@ -1,209 +1,97 @@
 --- !ruby/object:Gem::Specification
 name: content_server
 version: !ruby/object:Gem::Version
-  version: 1.6.0
+  version: 1.6.1
 platform: ruby
 authors:
 - BBFS Team
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-03-15 00:00:00.000000000 Z
+date: 2014-05-08 00:00:00.000000000 Z
 dependencies:
-- !ruby/object:Gem::Dependency
-  name: rake
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - '='
-      - !ruby/object:Gem::Version
-        version: 0.9.2.2
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - '='
-      - !ruby/object:Gem::Version
-        version: 0.9.2.2
-- !ruby/object:Gem::Dependency
-  name: algorithms
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: log4r
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: eventmachine
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: json
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: sinatra
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: thin
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: rake
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - '='
-      - !ruby/object:Gem::Version
-        version: 0.9.2.2
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - '='
-      - !ruby/object:Gem::Version
-        version: 0.9.2.2
 - !ruby/object:Gem::Dependency
   name: algorithms
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: log4r
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: eventmachine
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: json
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: sinatra
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: thin
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 description: Monitor and Index a directory and back it up to backup server.
@@ -218,18 +106,24 @@ extensions:
 - ext/run_in_background/mkrf_conf.rb
 extra_rdoc_files: []
 files:
+- bin/backup_server
+- bin/content_server
+- bin/file_utils
+- bin/testing_memory
+- bin/testing_server
+- ext/run_in_background/mkrf_conf.rb
 - lib/content_data.rb
 - lib/content_data/content_data.rb
 - lib/content_data/version.rb
 - lib/content_server.rb
+- lib/content_server/backup_server.rb
+- lib/content_server/content_receiver.rb
+- lib/content_server/content_server.rb
 - lib/content_server/file_streamer.rb
 - lib/content_server/queue_copy.rb
-- lib/content_server/server.rb
-- lib/content_server/content_server.rb
-- lib/content_server/content_receiver.rb
 - lib/content_server/remote_content.rb
+- lib/content_server/server.rb
 - lib/content_server/version.rb
-- lib/content_server/backup_server.rb
 - lib/email.rb
 - lib/email/email.rb
 - lib/email/version.rb
@@ -237,13 +131,13 @@ files:
 - lib/file_copy/copy.rb
 - lib/file_copy/version.rb
 - lib/file_indexing.rb
-- lib/file_indexing/indexer_patterns.rb
 - lib/file_indexing/index_agent.rb
+- lib/file_indexing/indexer_patterns.rb
 - lib/file_indexing/version.rb
 - lib/file_monitoring.rb
+- lib/file_monitoring/file_monitoring.rb
 - lib/file_monitoring/monitor_path.rb
 - lib/file_monitoring/version.rb
-- lib/file_monitoring/file_monitoring.rb
 - lib/file_utils.rb
 - lib/file_utils/file_generator/README
 - lib/file_utils/file_generator/file_generator.rb
@@ -257,20 +151,21 @@ files:
 - lib/params.rb
 - lib/params/version.rb
 - lib/process_monitoring.rb
-- lib/process_monitoring/thread_safe_hash.rb
+- lib/process_monitoring/monitoring.rb
 - lib/process_monitoring/monitoring_info.rb
 - lib/process_monitoring/send_email.rb
-- lib/process_monitoring/monitoring.rb
+- lib/process_monitoring/thread_safe_hash.rb
 - lib/process_monitoring/version.rb
 - lib/run_in_background.rb
 - lib/run_in_background/version.rb
+- lib/testing_memory/testing_memory.rb
 - lib/testing_server.rb
 - lib/testing_server/testing_server.rb
 - lib/testing_server/version.rb
-- lib/testing_memory/testing_memory.rb
 - lib/validations.rb
 - lib/validations/index_validations.rb
 - lib/validations/version.rb
+- spec/content_data/content_data_performance_spec.rb
 - spec/content_data/content_data_spec.rb
 - spec/content_data/validations_spec.rb
 - spec/content_server/content_server_spec.rb
@@ -279,12 +174,6 @@ files:
 - spec/file_indexing/index_agent_spec.rb
 - spec/networking/tcp_spec.rb
 - spec/validations/index_validations_spec.rb
-- bin/content_server
-- bin/backup_server
-- bin/file_utils
-- bin/testing_server
-- bin/testing_memory
-- ext/run_in_background/mkrf_conf.rb
 homepage: http://github.com/bbfsdev/bbfs
 licenses: []
 metadata: {}
@@ -294,23 +183,24 @@ require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ! '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ! '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.1.11
+rubygems_version: 2.2.2
 signing_key: 
 specification_version: 4
 summary: Servers for backing up content.
 test_files:
 - spec/content_data/content_data_spec.rb
 - spec/content_data/validations_spec.rb
+- spec/content_data/content_data_performance_spec.rb
 - spec/content_server/content_server_spec.rb
 - spec/content_server/file_streamer_spec.rb
 - spec/file_copy/copy_spec.rb