content_server 1.5.0 → 1.6.0

checksums.yaml

@@ -0,0 +1,15 @@
+---
+!binary "U0hBMQ==":
+  metadata.gz: !binary |-
+    ODk2ZWZlZGIwODU5Y2I1YTg3YzIyZWZmNDQyOGRiOTJhMWMwODJiZg==
+  data.tar.gz: !binary |-
+    NWQxY2E1NGE2ZmQ3YzRlYzFjN2QwNTRlMWMxYzFjMmZhODlhNDVlNA==
+SHA512:
+  metadata.gz: !binary |-
+    MmViNDAxZDU0NTk4ZDY0YzNiYTkxZWVkZjAxYzMyNjZhM2Y0ZDE2MmE3NDNl
+    YTE3MTJlZGU0NzBjZjJjYzNmM2Q5YWM1NzEyNzgxZDM4MmRmZDIyOWUxMGY3
+    MjU4YjMxMDFlOThkNDM1M2ZiZTJhN2RjNzFkZmZhYWZkZWE4MTM=
+  data.tar.gz: !binary |-
+    YmQxOWYzZGFhZDY5YjNmOTBkN2VhMDBiOTQwZTdjMmQxNzhmZDgxMzE3YzVl
+    OTg4NDRmYmE5ZDU3ZWJhMGU2YzNiYTQ2Y2M4OTEyMGQzYzNmOWJkYzc0YmMw
+    OTJhNmQ3YTY3MDIzZDI5NmYzOThhNjA0MjQ4MTI0ZGY5OGI1Nzc=

data/bin/file_utils

@@ -0,0 +1,118 @@
+#!/usr/bin/env ruby
+
+# This utility executable is used to easily execute BBFS operations on files.
+
+require 'content_data'
+require 'log'
+require 'params'
+
+#############################################
+# Definitions
+#############################################
+Params.string 'command', nil ,'supported commands are: merge, intersect, minus'
+Params.string 'dest', nil ,'destination path'
+Params.string 'cd_a', nil ,'a path'
+Params.string 'cd_b', nil ,'b path'
+
+HELP_MSG = <<EOF
+This is an utility functionality for BBFS,
+such as algebra of sets operations on ContentData files.
+Usage:
+  To get merge of two ContentData files,
+  i.e. contents and appropreate instances that exist in at least one ContentData file:
+    file_utils --command=merge --cd_a=<content_data_file1> --cd_b=<content_data_file_2> [--dest=<result_content_data_file>]
+  To get intersection of two ContentData files,
+  i.e. contents and appropreate instances that exist in the both ContentData files:
+    file_utils --command=intersect --cd_a=<content_data_file1> --cd_b=<content_data_file_2> [--dest=<result_content_data_file>]
+  To get content_data_2 minus content_data_1,
+  i.e. contents and appropreate instances that exist in content_data_file2 and absent in content_data_file1:
+    file_utils --command=minus --cd_a=<content_data_file1> --cd_b=<content_data_file_2> [--dest=<result_content_data_file>]
+EOF
+#############################################
+# Init
+#############################################
+Params.init ARGV
+Log.init
+
+#############################################
+# Methods
+#############################################
+
+# Algebra of sets operations (merge, intersect, minus) on ContentData files
+def content_data_command()
+
+  ['cd_a', 'cd_b'].each do |param|
+    if Params[param].nil?
+      err_msg = "--#{param} is not set"
+      puts err_msg
+      Log.error(err_msg)
+      return
+    end
+  end
+
+  cd_a = ContentData::ContentData.new()
+  begin
+    cd_a.from_file(Params['cd_a'])
+  rescue Exception => e
+    err_msg = "Error loading content data cd_a = %s: %s" % [Params['cd_a'], e.message]
+    puts err_msg
+    Log.error(err_msg)
+    Log.flush
+    return
+  end
+
+  cd_b = ContentData::ContentData.new()
+  begin
+    cd_b.from_file(Params['cd_b'])
+  rescue Exception => e
+    err_msg = "Error loading content data cd_b = %s: %s" % [Params['cd_b'], e.message]
+    puts err_msg
+    Log.error(err_msg)
+    Log.flush
+    return
+  end
+
+  if Params['command'] == "merge"
+    res = ContentData.merge(cd_a, cd_b)
+    preface = '# ' + "Merge of #{Params['cd_a']} and #{Params['cd_b']}"
+  elsif Params['command'] == "intersect"
+    res = ContentData.intersect(cd_a, cd_b)
+    preface = '# ' + "Intersection of #{Params['cd_a']} and #{Params['cd_b']}"
+  elsif Params['command'] == "minus"
+    res = ContentData.remove(cd_a, cd_b)  # cd_b - cd_a
+    preface = '# ' + "Contents and appropreate instances that exist in #{Params['cd_b']}" +
+      " and absent in #{Params['cd_a']}"
+  end
+
+  unless res.nil?
+    puts preface
+    puts res.to_s
+    unless Params['dest'].nil?
+      res.to_file Params['dest']
+    end
+  end
+end
+
+# Main method
+def run
+  if ARGV.empty?
+    puts HELP_MSG
+    exit
+  end
+
+  if (Params['command'] == "merge" ||
+      Params['command'] == "intersect" ||
+      Params['command'] == "minus")
+    content_data_command
+  else
+    err_msg = "Unsupported command: #{Params['command']}"
+    puts err_msg
+    puts HELP_MSG
+    Log.error(err_msg)
+  end
+end
+
+#############################################
+# Main
+#############################################
+run

data/lib/content_data/content_data.rb

@@ -35,16 +35,18 @@ module ContentData
       if other.nil?
         @contents_info = {}  # Checksum --> [size, paths-->time(instance), time(content)]
         @instances_info = {}  # location --> checksum to optimize instances query
+        @symlinks_info = {}  # [server,symlink path] -> target
       else
         @contents_info = other.clone_contents_info
         @instances_info = other.clone_instances_info  # location --> checksum to optimize instances query
+        @symlinks_info =  other.clone_symlinks_info
       end
     end
 
     # Content Data unique identification
     # @return [ID] hash identification
     def unique_id
-      @instances_info.hash
+      [@contents_info.hash,@symlinks_info.hash]
     end
 
     def clone_instances_info
@@ -83,6 +85,16 @@ module ContentData
       clone_contents_info
     end
 
+    def clone_symlinks_info
+      symlinks_info_enum = @symlinks_info.each_key
+      cloned_symlinks = {}
+      loop {
+        symlink_key = symlinks_info_enum.next rescue break
+        cloned_symlinks[[symlink_key[0].clone, symlink_key[0].clone]] = @symlinks_info[symlink_key].clone
+      }
+      cloned_symlinks
+    end
+
     # iterator over @contents_info data structure (not including instances)
     # block is provided with: checksum, size and content modification time
     def each_content(&block)
@@ -131,6 +143,17 @@ module ContentData
       }
     end
 
+    # iterator over @symlinks_info data structure
+    # block is provided with: server, file path and target
+    def each_symlink(&block)
+      symlink_enum = @symlinks_info.each_key
+      loop {
+        symlink_key = symlink_enum.next rescue break
+        symlink_target = @symlinks_info[symlink_key]
+        block.call(symlink_key[0], symlink_key[1], symlink_target)
+      }
+    end
+
     def contents_size()
       @contents_info.length
     end
@@ -139,6 +162,10 @@ module ContentData
       @instances_info.length
     end
 
+    def symlinks_size()
+      @symlinks_info.length
+    end
+
     def checksum_instances_size(checksum)
       content_info = @contents_info[checksum]
       return 0 if content_info.nil?
@@ -181,8 +208,16 @@ module ContentData
       @instances_info[location] = checksum
     end
 
+    def add_symlink(server, path, target)
+      @symlinks_info[[server,path]] = target
+    end
+
+    def remove_symlink(server, path)
+      @symlinks_info.delete([server,path])
+    end
+
     def empty?
-      @contents_info.empty?
+      @contents_info.empty? and @symlinks_info.empty?
     end
 
     def content_exists(checksum)
@@ -193,6 +228,11 @@ module ContentData
       @instances_info.has_key?([server, path])
     end
 
+    def symlink_exists(path, server)
+      @symlinks_info.has_key?([server, path])
+    end
+
+
     # removes an instance record both in @instances_info and @instances_info.
     # input params: server & path - are the instance unique key (called location)
     # removes also the content, if content becomes empty after removing the instance
@@ -208,7 +248,7 @@ module ContentData
     end
 
     # removes all instances records which are located under input param: dir_to_remove.
-    # found records are removed from both @instances_info and @instances_info.
+    # found records are removed from @contents_info , @instances_info and @symlinks_info
     # input params: server & dir_to_remove - are used to check each instance unique key (called location)
     # removes also content\s, if a content\s become\s empty after removing instance\s
     def remove_directory(dir_to_remove, server)
@@ -216,7 +256,9 @@ module ContentData
       loop {
         checksum = contents_enum.next rescue break
         instances =  @contents_info[checksum][1]
-        instances.each_key { |location|
+        instances_enum = instances.each_key
+        loop {
+          location = instances_enum.next rescue break
           if location[0] == server and location[1].scan(dir_to_remove).size > 0
             instances.delete(location)
             @instances_info.delete(location)
@@ -224,27 +266,20 @@ module ContentData
         }
         @contents_info.delete(checksum) if instances.empty?
       }
-    end
 
+      # handle symlinks
+      symlinks_enum = @symlinks_info.each_key
+      loop {
+        symlink_key = symlinks_enum.next rescue break
+        if symlink_key[0] == server and symlink_key[1].scan(dir_to_remove).size > 0
+          @symlinks_info.delete(symlink_key)
+        end
+      }
+    end
 
     def ==(other)
-      return false if other.nil?
-      return false if @contents_info.length != other.contents_size
-      other.each_instance { |checksum, size, content_mod_time, instance_mod_time, server, path|
-        return false if instance_exists(path, server) != other.instance_exists(path, server)
-        local_content_info = @contents_info[checksum]
-        return false if local_content_info.nil?
-        return false if local_content_info[0] != size
-        return false if local_content_info[2] != content_mod_time
-        #check instances
-        local_instances =  local_content_info[1]
-        return false if other.checksum_instances_size(checksum) != local_instances.length
-        location = [server, path]
-        local_instance_mod_time, _ = local_instances[location]
-        return false if local_instance_mod_time.nil?
-        return false if local_instance_mod_time != instance_mod_time
-      }
-      true
+      return nil if other.nil?  # for this case: content_data == nil
+      unique_id  == other.unique_id
     end
 
     def remove_content(checksum)
@@ -282,6 +317,7 @@ module ContentData
       content_data_dir = File.dirname(filename)
       FileUtils.makedirs(content_data_dir) unless File.directory?(content_data_dir)
       File.open(filename, 'w') { |file|
+        # Write contents
         file.write("#{@contents_info.length}\n")
         contents_enum = @contents_info.each_key
         content_chunks = @contents_info.length / CHUNK_SIZE + 1
@@ -291,6 +327,8 @@ module ContentData
           GC.start
           chunks_counter += 1
         end
+
+        # Write instances
         file.write("#{@instances_info.length}\n")
         contents_enum = @contents_info.each_key
         chunks_counter = 0
@@ -299,6 +337,14 @@ module ContentData
           GC.start
           chunks_counter += 1
         end
+
+        # Write symlinks
+        symlinks_info_enum = @symlinks_info.each_key
+        file.write("#{@symlinks_info.length}\n")
+        loop {
+          symlink_key = symlinks_info_enum.next rescue break
+          file.write("#{symlink_key[0]}<#{symlink_key[1]}<#{@symlinks_info[symlink_key]}\n")
+        }
       }
     end
 
@@ -307,7 +353,7 @@ module ContentData
       while chunk_counter < chunk_size
         checksum = contents_enum.next rescue return
         content_info = @contents_info[checksum]
-        file.write("#{checksum},#{content_info[0]},#{content_info[2]}\n")
+        file.write("#{checksum}<#{content_info[0]}<#{content_info[2]}\n")
         chunk_counter += 1
       end
     end
@@ -323,8 +369,8 @@ module ContentData
           # provide the block with: checksum, size, content modification time,instance modification time,
           #   server and path.
           instance_modification_time,instance_index_time = content_info[1][location]
-          file.write("#{checksum},#{content_info[0]},#{location[0]},#{location[1]}," +
-                     "#{instance_modification_time},#{instance_index_time}\n")
+          file.write("#{checksum}<#{content_info[0]}<#{location[0]}<#{location[1]}<" +
+                     "#{instance_modification_time}<#{instance_index_time}\n")
         }
         chunk_counter += 1
         break if chunk_counter == chunk_size
@@ -332,6 +378,7 @@ module ContentData
     end
 
     # TODO validation that file indeed contains ContentData missing
+    # TODO class level method?
     # Loading db from file using chunks for better memory performance
     def from_file(filename)
       # read first line (number of contents)
@@ -339,11 +386,16 @@ module ContentData
       # read number of instances.
       # loop over instances lines (using chunks) and add instances
 
+      unless File.exists? filename
+        raise ArgumentError.new "No such a file #{filename}"
+      end
+
       File.open(filename, 'r') { |file|
         # Get number of contents (at first line)
         number_of_contents = file.gets  # this gets the next line or return nil at EOF
         unless (number_of_contents and number_of_contents.match(/^[\d]+$/))  # check that line is of Number format
-          return reset_load_from_file(filename, file, "number of contents should be a number. We got:#{number_of_contents}")
+          raise("Parse error of content data file:#{filename}  line ##{$.}\n" +
+                "number of contents should be a number. We got:#{number_of_contents}")
         end
         number_of_contents = number_of_contents.to_i
         # advance file lines over all contents. We need only the instances data to build the content data object
@@ -365,7 +417,8 @@ module ContentData
         # get number of instances
         number_of_instances = file.gets
         unless (number_of_instances and number_of_instances.match(/^[\d]+$/))  # check that line is of Number format
-          return reset_load_from_file(filename, file, "number of instances should be a Number. We got:#{number_of_instances}")
+          raise("Parse error of content data file:#{filename}  line ##{$.}\n" +
+                "number of instances should be a Number. We got:#{number_of_instances}")
         end
         number_of_instances = number_of_instances.to_i
         # read in instances chunks and GC
@@ -382,35 +435,56 @@ module ContentData
           GC.start
           chunk_index += 1
         end
+
+        # get number of symlinks
+        number_of_symlinks = file.gets
+        unless (number_of_symlinks and number_of_symlinks.match(/^[\d]+$/))  # check that line is of Number format
+          raise("Parse error of content data file:#{filename}  line ##{$.}\n" +
+                    "number of symlinks should be a Number. We got:#{number_of_symlinks}")
+        end
+        number_of_symlinks.to_i.times {
+          symlinks_line = file.gets
+          unless symlinks_line
+            raise("Parse error of content data file:#{filename}  line ##{$.}\n" +
+                   "Expected to read symlink line but reached EOF")
+          end
+          parameters = symlinks_line.split('<')
+          if (3 != parameters.length)
+            raise("Parse error of content data file:#{filename}  line ##{$.}\n" +
+                  "Expected to read 3 fields ('<' separated) but got #{parameters.length}.\nLine:#{symlinks_line}")
+          end
+
+          @symlinks_info[[parameters[0],parameters[1]]] = parameters[2]
+        }
       }
     end
 
     def read_contents_chunk(filename, file, chunk_size)
       chunk_index = 0
       while chunk_index < chunk_size
-        return reset_load_from_file(filename, file, "Expecting content line but " +
-            "reached end of file after line #{$.}") unless file.gets
+        unless file.gets
+          raise("Parse error of content data file:#{filename}  line ##{$.}\n" +
+                "Expecting content line but reached end of file")
+        end
         chunk_index += 1
       end
       true
     end
 
-    def  read_instances_chunk(filename, file, chunk_size)
+    def read_instances_chunk(filename, file, chunk_size)
       chunk_index = 0
       while chunk_index < chunk_size
         instance_line = file.gets
-        return reset_load_from_file(filename, file, "Expected to read Instance line but reached EOF") unless instance_line
-        parameters = instance_line.split(',')
-        # bugfix: if file name consist a comma then parsing based on comma separating fails
-        if (parameters.size > 6)
-          (4..parameters.size-3).each do |i|
-            parameters[3] = [parameters[3], parameters[i]].join(",")
-          end
-          (4..parameters.size-3).each do |i|
-            parameters.delete_at(4)
-          end
+        unless instance_line
+          raise("Parse error of content data file:#{filename}  line ##{$.}\n" +
+                "Expected to read Instance line but reached EOF")
         end
 
+        parameters = instance_line.split('<')
+        if (6 != parameters.length)
+          raise("Parse error of content data file:#{filename}  line ##{$.}\n" +
+                    "Expected to read 6 fields ('<' separated) but got #{parameters.length}.\nLine:#{instance_line}")
+        end
         add_instance(parameters[0],        #checksum
                      parameters[1].to_i,   # size
                      parameters[2],        # server
@@ -422,14 +496,6 @@ module ContentData
       true
     end
 
-    def reset_load_from_file(file_name, file_io, err_msg)
-      Log.error("unexpected error reading file:#{file_name}\nError message:#{err_msg}")
-      @contents_info = {}  # Checksum --> [size, paths-->time(instance), time(content)]
-      @instances_info = {}  # location --> checksum to optimize instances query
-      file_io.close
-      nil
-    end
-
     # for each content, all time fields (content and instances) are replaced with the
     # min time found, while going through all time fields.
     def unify_time()