content_server 1.1.0 → 1.2.0

data/lib/email/email.rb

@@ -0,0 +1,87 @@
+require 'net/smtp'
+
+module Email
+
+  def Email.send_raw_email(from, password, to, mail_text)
+    smtp = Net::SMTP.new('smtp.gmail.com', 587)
+    smtp.enable_starttls
+    smtp.start(from.split('@')[-1], from, password, :login) do
+      smtp.send_message(mail_text, from, to)
+    end
+  end
+
+  def Email.send_email(from, password, to, subject, body)
+    marker = (0...50).map{ ('a'..'z').to_a[rand(26)] }.join
+    parts = []
+    parts << <<EOF
+From: #{from}
+To: #{to}
+Subject: #{subject}
+MIME-Version: 1.0
+Content-Type: multipart/mixed; boundary=#{marker}
+--#{marker}
+EOF
+
+    # Define the message action
+    parts << <<EOF
+Content-Type: text/plain
+Content-Transfer-Encoding:8bit
+
+#{body}
+--#{marker}
+EOF
+
+    mailtext = parts.join('')
+    send_raw_email(from, password, to, mailtext)
+  end
+
+  def Email.send_attachments_email(from, password, to, subject, body, attachments)
+    # Read a file and encode it into base64 format
+    begin
+      encoded_attachments = []
+      attachments.each do |attachment|
+        file_content = File.read(attachment)
+        encoded_content = [file_content].pack("m")   # base64
+        encoded_attachments << [attachment, encoded_content]
+      end
+    rescue
+      raise "Could not read file #{attachment}"
+    end
+
+    marker = (0...50).map{ ('a'..'z').to_a[rand(26)] }.join
+    parts = []
+    parts << <<EOF
+From: #{from}
+To: #{to}
+Subject: #{subject}
+MIME-Version: 1.0
+Content-Type: multipart/mixed; boundary=#{marker}
+--#{marker}
+EOF
+
+    # Define the message action
+    parts << <<EOF
+Content-Type: text/plain
+Content-Transfer-Encoding:8bit
+
+#{body}
+--#{marker}
+EOF
+
+    encoded_attachments.each do |attachment, encoded_content|
+    # Define the attachment section
+    parts << <<EOF
+Content-Type: multipart/mixed; name=\"#{File.basename(attachment)}\"
+Content-Transfer-Encoding:base64
+Content-Disposition: attachment; filename="#{File.basename(attachment)}"
+
+#{encoded_content}
+--#{marker}--
+EOF
+    end
+
+    mailtext = parts.join('')
+    send_raw_email(from, password, to, mailtext)
+  end
+
+end

data/lib/email/version.rb

@@ -0,0 +1,3 @@
+module SendEmail
+  VERSION = "1.1.0"
+end

data/lib/email.rb

@@ -0,0 +1,4 @@
+require 'email/email'
+
+module Email
+end

data/lib/file_copy/copy.rb

@@ -0,0 +1,68 @@
+require 'net/ssh'
+require 'net/sftp'
+
+require 'log'
+require 'params'
+
+module FileCopy
+
+  # Creates ssh connection, assumes username and password
+  # or without password but with ssh keys.
+  def ssh_connect(username, password, server)
+    username = (username and username.length > 0) ? username : ENV['USER']
+    password = (password and password.length > 0) ? password : nil
+    port = 22 # 22 is a standart SSH port
+    raise "Undefined server" unless server
+    Log.debug1 "Trying to connect(ssh): #{username}, #{password}, #{server}, #{port}."
+    if (username and password)
+      Net::SSH.start(server, username,
+                     :password => password,
+                     :port => port)
+    elsif (username)
+      Net::SSH.start(server, username,
+                     :port => port)
+    else
+      raise "Undefined username"
+    end
+  end
+  module_function :ssh_connect
+
+  # Simply copy map files using sftp server on dest_server.
+  # Note: stfp.upload support parallel uploads - default is 2.
+  # TODO(kolman): packing files, all, not all, determine by part of file.
+  def sftp_copy(username, password, server, files_map)
+    ssh = FileCopy.ssh_connect(username, password, server)
+    ssh.sftp.connect do |sftp|
+      uploads = files_map.map { |from,to|
+        remote_dir = File.dirname(to)
+        sftp_mkdir_recursive(sftp, File.dirname(to))
+        Log.debug1 "Copying #{from} to #{to}"
+        sftp.upload(from, to)
+      }
+      uploads.each { |u| u.wait }
+      Log.debug1 "Done."
+    end # ssh.sftp.connect
+  end # def initialize
+  module_function :sftp_copy
+
+  def sftp_mkdir_recursive(sftp, path)
+    dir_stat = nil
+    begin
+      Log.debug1 "Stat remote dir: #{path}."
+      dir_stat = sftp.stat!(path).directory?
+      Log.debug1 "Stat result #{dir_stat}."
+    rescue Net::SFTP::StatusException
+    end
+    if !dir_stat
+      Log.debug1 "Directory does not exists: #{path}."
+      sftp_mkdir_recursive sftp, File.dirname(path)
+      Log.debug1 "Making dir #{path}."
+      response = sftp.mkdir!(path)
+      Log.debug1 "Making dir ok:#{response.ok?}."
+    end
+  end
+  module_function :sftp_mkdir_recursive
+
+end
+
+

data/lib/file_copy/version.rb

@@ -0,0 +1,4 @@
+module FileCopy
+  VERSION = "1.1.0"
+end
+

data/lib/file_copy.rb

@@ -0,0 +1,4 @@
+require 'file_copy/copy'
+
+module FileCopy
+end

data/lib/file_indexing/index_agent.rb

@@ -0,0 +1,170 @@
+require 'digest/sha1'
+require 'pp'
+require 'set'
+require 'time'
+
+require 'content_data'
+require 'file_indexing/indexer_patterns'
+require 'log'
+
+
+module FileIndexing
+####################
+# Index Agent
+####################
+
+  class IndexAgent
+    attr_reader :indexed_content, :failed_files
+
+    # Why are those lines needed?
+    LOCALTZ = Time.now.zone
+    ENV['TZ'] = 'UTC'
+
+    def initialize
+      @indexed_content = ContentData::ContentData.new
+      @failed_files = Set.new
+    end
+
+    # Calculate file checksum (SHA1)
+    def self.get_checksum(filename)
+      digest = Digest::SHA1.new
+      begin
+        File.open(filename, 'rb') { |f|
+          while buffer = f.read(65536) do
+            digest << buffer
+          end
+        }
+        Log.debug1("#{filename} sha1 #{digest.hexdigest.downcase}")
+        digest.hexdigest.downcase
+      rescue Errno::EACCES, Errno::ETXTBSY => exp
+        Log.warning("#{exp.message}")
+        false
+      end
+    end
+
+    def IndexAgent.get_content_checksum(content)
+      # Calculate checksum.
+      digest = Digest::SHA1.new
+      digest << content
+      digest.hexdigest.downcase
+    end
+
+    # get all files
+    # satisfying the pattern
+    def collect(pattern)
+      Dir.glob(pattern.to_s)
+    end
+
+    # TODO(kolman): Replace this with File.lstat(file).mtime when new version of Ruby comes out.
+    # http://bugs.ruby-lang.org/issues/6385
+    def IndexAgent.get_correct_mtime(file)
+      begin
+        File.open(file, 'r') { |f| f.mtime }
+      rescue Errno::EACCES => e
+        Log.warning("Could not open file #{file} to get mtime. #{e}")
+        return 0
+      end
+    end
+
+    # index device according to the pattern
+    # store the result
+    # does not adds automatically otherDB to stored result
+    # TODO device support
+    def index(patterns, otherDB = nil)
+      abort "#{self.class}: DB not empty. Current implementation permits only one running of index" \
+            unless @indexed_content.empty?
+      local_server_name = `hostname`.strip
+      permit_patterns = []
+      forbid_patterns = []
+      otherDB_updated = ContentData::ContentData.new
+      #otherDB_table = Hash.new   # contains instances from given DB while full path name is a key and instance is a value
+      #otherDB_contents = Hash.new  # given DB contents
+
+      # if there is a given DB then populate table with files
+      # that was already indexed on this server/device
+      if !otherDB.nil?
+        otherDB.each_instance { |checksum, size, content_mod_time, instance_mod_time, server, path|
+          if (server == local_server_name)
+            # add instance
+            otherDB_updated.add_instance(checksum, size, server, path, instance_mod_time)
+          end
+        }
+      end
+
+      permit_patterns = patterns.positive_patterns
+      forbid_patterns = patterns.negative_patterns
+
+      # add files found by positive patterns
+      files = Array.new
+      permit_patterns.each_index do |i|
+        files = files | (collect(permit_patterns[i]));
+      end
+
+      Log.debug1 "Files: #{files}."
+
+      # expand to absolute pathes
+      files.map! {|f| File.expand_path(f)}
+
+      # remove files found by negative patterns
+      forbid_patterns.each_index do |i|
+        forbid_files = Array.new(collect(forbid_patterns[i]));
+        forbid_files.each do |f|
+          files.delete(File.expand_path(f))
+        end
+      end
+
+      # create and add contents and instances
+      files.each do |file|
+        file_stats = File.lstat(file)
+        file_mtime = IndexAgent.get_correct_mtime(file)
+        device = file_stats.dev.to_s
+
+        # index only files
+        next if file_stats.directory?
+
+        # add files present in the given DB to the DB and remove these files
+        # from further processing (save checksum calculation)
+        file_match = false
+        otherDB_updated.each_instance { |checksum, size, content_mod_time, instance_mod_time, server, path|
+          if otherDB_updated.instance_exists(file, local_server_name, checksum)
+            if size == file_stats.size and instance_mod_time == file_mtime.to_i
+              @indexed_content.add_instance(checksum, size, server, file, instance_mod_time)
+              file_match = true
+              break
+            else
+              Log.warning("File (#{file}) size or modification file is different. size=#{size}  actual size=#{file_stats.size}" + \
+                          "   instance_mod_time=#{Time.at(instance_mod_time)}  actual=#{file_mtime}")
+            end
+          end
+        }
+        next if file_match
+        # calculate a checksum
+        unless (checksum = self.class.get_checksum(file))
+          Log.warning("Cheksum failure: " + file)
+          @failed_files.add(file)
+          next
+        end
+
+        @indexed_content.add_instance(checksum, file_stats.size, local_server_name,
+                                      File.expand_path(file), file_mtime.to_i)
+      end
+    end
+
+    def IndexAgent.create_shallow_instance(filename)
+      return nil unless File.exists?(filename)
+      file_stats = File.lstat(filename)
+      file_mtime = IndexAgent.get_correct_mtime(filename)
+      # return instance shallow representation (no server)
+      [file_stats.size,
+       "%s,%s,%s" % [`hostname`.strip , file_stats.dev.to_s , File.expand_path(filename)],
+       file_mtime.to_i]
+    end
+
+    def IndexAgent.global_path(filename)
+      server_name = `hostname`.strip
+      file_stats = File.lstat(filename)
+      return "%s,%s,%s" % [server_name, file_stats.dev.to_s,filename]
+    end
+  end
+end
+

data/lib/file_indexing/indexer_patterns.rb

@@ -0,0 +1,72 @@
+require 'log'
+require 'params'
+
+module FileIndexing
+
+  class IndexerPatterns
+    attr_reader :positive_patterns, :negative_patterns
+
+    # @param indexer_patterns_str [String]
+    def initialize (indexer_patterns = nil)
+      Log.debug1 "Initialize index patterns #{indexer_patterns}."
+      @positive_patterns = Array.new
+      @negative_patterns = Array.new
+      # TODO add a test (including empty collections)
+      if indexer_patterns
+        indexer_patterns.positive_patterns.each do |pattern|
+          add_pattern(pattern)
+        end
+        indexer_patterns.negative_patterns.each do |pattern|
+          add_pattern(pattern, false)
+        end
+      end
+    end
+
+    def serialize
+      # TODO add a test (including empty collections)
+      indexer_patterns = IndexerPatternsMessage.new
+      positive_patterns.each do |pattern|
+        indexer_patterns.positive_patterns << pattern
+      end
+      negative_patterns.each do |pattern|
+        indexer_patterns.negative_patterns << pattern
+      end
+      indexer_patterns
+    end
+
+    # @param pattern [String]
+    # @param is_positive [true]
+    # @param is_positive [false]
+    def add_pattern(pattern, is_positive = true)
+      pattern.gsub!(/\\/,'/')
+      if (is_positive)
+        @positive_patterns << pattern
+      else
+        @negative_patterns << pattern
+      end
+    end
+
+    def parse_from_file(file)
+      input_patterns = IO.readlines(file)
+      begin
+        Log.debug1 "Error loading patterns=%s" % file
+        raise IOError("Error loading patterns=%s" % file)
+      end unless not input_patterns.nil?
+
+      input_patterns.each do |pattern|
+        if (m = /^\s*([+-]):(.*)/.match(pattern))
+          add_pattern(m[2], m[1].eql?('+') ? true : false)
+        elsif (not /^\s*[\/\/|#]/.match(pattern))   # not a comment
+          Log.debug1 "pattern in incorrect format: #{pattern}"
+          raise RuntimeError("pattern in incorrect format: #{pattern}")
+        end
+      end
+    end
+
+    def size
+      return @positive_patterns.size
+    end
+  end
+
+end
+

data/lib/file_indexing/version.rb

@@ -0,0 +1,3 @@
+module FileIndexing
+  VERSION = "1.1.0"
+end

data/lib/file_indexing.rb

@@ -0,0 +1,9 @@
+require 'content_data'
+
+require 'file_indexing/index_agent'
+require 'file_indexing/indexer_patterns'
+
+# Data structure for an abstract layer over files.
+# Each binary sequence is a content, each file is content instance.
+module FileIndexing
+end

data/lib/file_monitoring/file_monitoring.rb

@@ -0,0 +1,105 @@
+require 'algorithms'
+require 'fileutils'
+require 'log4r'
+require 'params'
+require 'content_server/server'
+require 'file_monitoring/monitor_path'
+
+module FileMonitoring
+  # Manages file monitoring of number of file system locations
+  class FileMonitoring
+
+    def initialize ()
+      @content_data_cache = Set.new
+      $local_content_data_lock.synchronize {
+        $local_content_data.each_instance(){
+            |_, _, _, _, _, file_path|
+          # save files to cache
+          Log.info("File in cache: #{file_path.clone}")
+          @content_data_cache.add(file_path.clone)
+        }
+      }
+    end
+
+
+    # Set event queue used for communication between different proceses.
+    # @param queue [Queue]
+    def set_event_queue(queue)
+      @event_queue = queue
+    end
+
+    # The main method. Loops on all paths, each time span and monitors them.
+    #
+    # =Algorithm:
+    # There is a loop that performs at every iteration:
+    #   1.Pull entry with a minimal time of check from queue
+    #   2.Recursively check path taken from entry for changes
+    #     a.Notify subscribed processes on changes
+    #   3.Push entry to the queue with new time of next check
+    #
+    # This methods controlled by <tt>monitoring_paths</tt> configuration parameter,
+    # that provides path and file monitoring configuration data
+    def monitor_files
+      conf_array = Params['monitoring_paths']
+      # Directories states stored in the priority queue,
+      # where the key (priority) is a time when it should be checked next time.
+      # Priority queue means that all entries arranged by key (time to check) in increasing order.
+      pq = Containers::PriorityQueue.new
+      conf_array.each { |elem|
+        priority = (Time.now + elem['scan_period']).to_i
+        dir_stat = DirStat.new(File.expand_path(elem['path']), elem['stable_state'], @content_data_cache, FileStatEnum::NON_EXISTING)
+        dir_stat.set_event_queue(@event_queue) if @event_queue
+        Log.debug1("File monitoring started for: #{elem}")
+        pq.push([priority, elem, dir_stat], -priority)
+      }
+
+      #init log4r
+      monitoring_log_path = Params['default_monitoring_log_path']
+      Log.debug1 'File monitoring log: ' + Params['default_monitoring_log_path']
+      monitoring_log_dir = File.dirname(monitoring_log_path)
+      FileUtils.mkdir_p(monitoring_log_dir) unless File.exists?(monitoring_log_dir)
+
+      @log4r = Log4r::Logger.new 'BBFS monitoring log'
+      @log4r.trace = true
+      formatter = Log4r::PatternFormatter.new(:pattern => "[%d] [%m]")
+      #file setup
+      file_config = {
+          "filename" => Params['default_monitoring_log_path'],
+          "maxsize" => Params['log_rotation_size'],
+          "trunc" => true
+      }
+      file_outputter = Log4r::RollingFileOutputter.new("monitor_log", file_config)
+      file_outputter.level = Log4r::INFO
+      file_outputter.formatter = formatter
+      @log4r.outputters << file_outputter
+      FileStat.set_log(@log4r)
+
+      while true do
+        # pull entry that should be checked next,
+        # according to it's scan_period
+        time, conf, dir_stat = pq.pop
+        # time remains to wait before directory should be checked
+        time_span = time - Time.now.to_i
+        if (time_span > 0)
+          sleep(time_span)
+        end
+
+        unless $testing_memory_active
+          dir_stat.monitor
+        else
+          $testing_memory_log.info("Start monitor at :#{Time.now}")
+          puts "Start monitor at :#{Time.now}"
+          dir_stat.monitor
+          $testing_memory_log.info("End monitor at :#{Time.now}")
+          puts "End monitor at :#{Time.now}"
+        end
+
+        # push entry with new a next time it should be checked as a priority key
+        priority = (Time.now + conf['scan_period']).to_i
+        pq.push([priority, conf, dir_stat], -priority)
+      end
+    end
+  end
+
+end
+