file_indexing 0.0.1

data/lib/file_indexing/index_agent.rb

@@ -0,0 +1,140 @@
+require 'digest/sha1'
+require 'logger'
+require 'pp'
+require 'time'
+require './content_data'
+require './indexer_patterns'
+
+####################
+# Index Agent
+####################
+
+class IndexAgent
+  attr_reader :indexed_content
+
+  LOCALTZ = Time.now.zone
+  ENV['TZ'] = 'UTC'
+
+  def initialize
+    init_log()
+    init_db()
+  end
+
+  def init_db()
+    @indexed_content = ContentData.new
+  end
+
+  def init_log()
+    @log = Logger.new(STDERR)
+    @log.level = Logger::WARN
+    @log.datetime_format = "%Y-%m-%d %H:%M:%S"
+  end
+
+  def set_log(log_path, log_level)
+    @log = Logger.new(log_path) if log_path
+    @log.level = log_level
+  end
+
+  # Calculate file checksum (SHA1)
+  def self.get_checksum(filename)
+    digest = Digest::SHA1.new
+    begin
+      file = File.new(filename)
+      while buffer = file.read(65536)
+        digest << buffer
+      end
+      #@log.info { digest.hexdigest.downcase + ' ' + filename }
+      digest.hexdigest.downcase
+    rescue Errno::EACCES, Errno::ETXTBSY => exp
+      @log.warn { "#{exp.message}" }
+      false
+    ensure
+      file.close if file != nil
+    end
+  end
+
+  # get all files
+  # satisfying the pattern
+  def collect(pattern)
+    Dir.glob(pattern.to_s)
+  end
+
+  # index device according to the pattern
+  # store the result
+  # TODO device support
+  def index(patterns, otherDB = nil)
+    abort "#{self.class}: DB not empty. Current implementation permits only one running of index" unless db.contents.empty?
+
+    server_name = `hostname`
+    permit_patterns = Array.new
+    forbid_patterns = Array.new
+    otherDB_table = Hash.new   # contains instances from given DB while full path name is a key and instance is a value
+    otherDB_contents = Hash.new  # given DB contents
+
+    # if there is a given DB then populate table with files
+    # that was already indexed on this server/device
+    if (otherDB != nil)
+      otherDB_contents.update(otherDB.contents)
+      otherDB.instances.each_value do |i|
+        next unless i.server_name == server_name #and i.device == @device
+        otherDB_table[i.full_path] = i
+      end
+    end
+
+    permit_patterns = patterns.positive_patterns
+    forbid_patterns = patterns.negative_patterns
+
+    # add files found by positive patterns
+    files = Array.new
+    permit_patterns.each_index do |i|
+      files = files | (collect(permit_patterns[i]));
+    end
+
+    # expand to absolute pathes
+    files.map! {|f| File.expand_path(f)}
+
+    # remove files found by negative patterns
+    forbid_patterns.each_index do |i|
+      forbid_files = Array.new(collect(forbid_patterns[i]));
+      forbid_files.each do |f|
+        files.delete(File.expand_path(f))
+      end
+    end
+
+    # create and add contents and instances
+    files.each do |file|
+      file_stats = File.lstat(file)
+
+      # index only files
+      next if (file_stats.directory?)
+
+      # keep only files with names in UTF-8
+      unless file.force_encoding("UTF-8").valid_encoding?
+        @log.warn { "Non-UTF8 file name \"#{file}\"" }
+        next
+      end
+
+      # add files present in the given DB to the DB and remove these files
+      # from further processing (save checksum calculation)
+      if otherDB_table.has_key?(file)
+        instance = otherDB_table[file]
+        if instance.size == file_stats.size and instance.modification_time == file_stats.mtime.utc
+          @indexed_content.add_content(otherDB_contents[instance.checksum])
+          @indexed_content.add_instance(instance)
+          next
+        end
+      end
+
+      # calculate a checksum
+      unless (checksum = self.class.get_checksum(file))
+        @log.warn { "Cheksum failure: " + file }
+        next
+      end
+
+      @indexed_content.add_content(Content.new(checksum, file_stats.size, Time.now.utc)) unless (@indexed_content.content_exists(checksum))
+
+      instance = ContentInstance.new(checksum, file_stats.size, server_name, file_stats.dev.to_s, File.expand_path(file), file_stats.mtime.utc)
+      @indexed_content.add_instance(instance)
+    end
+  end
+end

data/lib/file_indexing/indexer_patterns.rb

@@ -0,0 +1,61 @@
+require './agent.pb'
+
+class IndexerPatterns
+  attr_reader :positive_patterns, :negative_patterns
+
+  # @param indexer_patterns_str [String]
+  def initialize (indexer_patterns = nil)
+    @positive_patterns = Array.new
+    @negative_patterns = Array.new
+    # TODO add a test (including empty collections)
+    unless  indexer_patterns.nil?
+      indexer_patterns.positive_patterns.each do |pattern|
+        add_pattern(pattern)
+      end
+      indexer_patterns.negative_patterns.each do |pattern|
+        add_pattern(pattern, false)
+      end
+    end
+  end
+
+  def serialize
+    # TODO add a test (including empty collections)
+    indexer_patterns = IndexerPatternsMessage.new
+    positive_patterns.each do |pattern|
+      indexer_patterns.positive_patterns << pattern
+    end
+    negative_patterns.each do |pattern|
+      indexer_patterns.negative_patterns << pattern
+    end
+    indexer_patterns
+  end
+
+  # @param pattern [String]
+  # @param is_positive [true]
+  # @param is_positive [false]
+  def add_pattern(pattern, is_positive = true)
+    pattern.gsub!(/\\/,'/')
+    if (is_positive)
+      @positive_patterns << pattern
+    else
+      @negative_patterns << pattern
+    end
+  end
+
+  def parse_from_file(file)
+    input_patterns = IO.readlines(file)
+    begin
+      puts "Error loading patterns=%s" % file
+      raise IOError("Error loading patterns=%s" % file)
+    end unless not input_patterns.nil?
+
+    input_patterns.each do |pattern|
+      if (m = /^\s*([+-]):(.*)/.match(pattern))
+        add_pattern(m[2], m[1].eql?('+') ? true : false)
+      elsif (not /^\s*[\/\/|#]/.match(pattern))   # not a comment
+        puts "pattern in incorrect format: #{pattern}"
+        raise RuntimeError("pattern in incorrect format: #{pattern}")
+      end
+    end
+  end
+end

data/lib/file_indexing.rb

@@ -0,0 +1,10 @@
+require_relative 'file_indexing/index_agent'
+require_relative 'file_indexing/indexer_patterns'
+
+# Data structure for an abstract layer over files.
+# Each binary sequence is a content, each file is content instance.
+module BBFS
+  module FileIndexing
+    VERSION = "0.0.1"
+  end
+end

metadata

@@ -0,0 +1,47 @@
+--- !ruby/object:Gem::Specification
+name: file_indexing
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+  prerelease: 
+platform: ruby
+authors:
+- Gena Petelko, Kolman Vornovitsky
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2012-04-08 00:00:00.000000000Z
+dependencies: []
+description: Indexes files, treats files with same binary sequence as one content.
+email: kolmanv@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/file_indexing.rb
+- lib/file_indexing/index_agent.rb
+- lib/file_indexing/indexer_patterns.rb
+homepage: http://github.com/kolmanv/bbfs
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.15
+signing_key: 
+specification_version: 3
+summary: Indexes files.
+test_files: []