file_indexing 0.0.1 → 0.0.2

data/lib/file_indexing.rb

@@ -1,3 +1,5 @@
+require 'content_data'
+
 require_relative 'file_indexing/index_agent'
 require_relative 'file_indexing/indexer_patterns'
 

data/lib/file_indexing/index_agent.rb

@@ -2,139 +2,153 @@ require 'digest/sha1'
 require 'logger'
 require 'pp'
 require 'time'
-require './content_data'
-require './indexer_patterns'
+
+require 'content_data'
+
+require_relative 'indexer_patterns'
+
+module BBFS
+  module FileIndexing
 
 ####################
 # Index Agent
 ####################
 
-class IndexAgent
-  attr_reader :indexed_content
-
-  LOCALTZ = Time.now.zone
-  ENV['TZ'] = 'UTC'
+    class IndexAgent
+      attr_reader :indexed_content
 
-  def initialize
-    init_log()
-    init_db()
-  end
-
-  def init_db()
-    @indexed_content = ContentData.new
-  end
+      LOCALTZ = Time.now.zone
+      ENV['TZ'] = 'UTC'
 
-  def init_log()
-    @log = Logger.new(STDERR)
-    @log.level = Logger::WARN
-    @log.datetime_format = "%Y-%m-%d %H:%M:%S"
-  end
+      def initialize
+        init_log()
+        init_db()
+      end
 
-  def set_log(log_path, log_level)
-    @log = Logger.new(log_path) if log_path
-    @log.level = log_level
-  end
+      def init_db()
+        @indexed_content = ContentData::ContentData.new
+      end
 
-  # Calculate file checksum (SHA1)
-  def self.get_checksum(filename)
-    digest = Digest::SHA1.new
-    begin
-      file = File.new(filename)
-      while buffer = file.read(65536)
-        digest << buffer
+      def init_log()
+        @log = Logger.new(STDERR)
+        @log.level = Logger::WARN
+        @log.datetime_format = "%Y-%m-%d %H:%M:%S"
       end
-      #@log.info { digest.hexdigest.downcase + ' ' + filename }
-      digest.hexdigest.downcase
-    rescue Errno::EACCES, Errno::ETXTBSY => exp
-      @log.warn { "#{exp.message}" }
-      false
-    ensure
-      file.close if file != nil
-    end
-  end
 
-  # get all files
-  # satisfying the pattern
-  def collect(pattern)
-    Dir.glob(pattern.to_s)
-  end
+      def set_log(log_path, log_level)
+        @log = Logger.new(log_path) if log_path
+        @log.level = log_level
+      end
 
-  # index device according to the pattern
-  # store the result
-  # TODO device support
-  def index(patterns, otherDB = nil)
-    abort "#{self.class}: DB not empty. Current implementation permits only one running of index" unless db.contents.empty?
-
-    server_name = `hostname`
-    permit_patterns = Array.new
-    forbid_patterns = Array.new
-    otherDB_table = Hash.new   # contains instances from given DB while full path name is a key and instance is a value
-    otherDB_contents = Hash.new  # given DB contents
-
-    # if there is a given DB then populate table with files
-    # that was already indexed on this server/device
-    if (otherDB != nil)
-      otherDB_contents.update(otherDB.contents)
-      otherDB.instances.each_value do |i|
-        next unless i.server_name == server_name #and i.device == @device
-        otherDB_table[i.full_path] = i
+      # Calculate file checksum (SHA1)
+      def self.get_checksum(filename)
+        digest = Digest::SHA1.new
+        begin
+          file = File.new(filename)
+          while buffer = file.read(65536)
+            digest << buffer
+          end
+          #@log.info { digest.hexdigest.downcase + ' ' + filename }
+          digest.hexdigest.downcase
+        rescue Errno::EACCES, Errno::ETXTBSY => exp
+          @log.warn { "#{exp.message}" }
+          false
+        ensure
+          file.close if file != nil
+        end
       end
-    end
 
-    permit_patterns = patterns.positive_patterns
-    forbid_patterns = patterns.negative_patterns
+      # get all files
+      # satisfying the pattern
+      def collect(pattern)
+        Dir.glob(pattern.to_s)
+      end
 
-    # add files found by positive patterns
-    files = Array.new
-    permit_patterns.each_index do |i|
-      files = files | (collect(permit_patterns[i]));
-    end
+      # index device according to the pattern
+      # store the result
+      # does not adds automatically otherDB to stored result
+      # TODO device support
+      def index(patterns, otherDB = nil)
+        abort "#{self.class}: DB not empty. Current implementation permits only one running of index" \
+            unless @indexed_content.contents.empty?
+
+        server_name = `hostname`
+        permit_patterns = Array.new
+        forbid_patterns = Array.new
+        otherDB_table = Hash.new   # contains instances from given DB while full path name is a key and instance is a value
+        otherDB_contents = Hash.new  # given DB contents
+
+        # if there is a given DB then populate table with files
+        # that was already indexed on this server/device
+        if (otherDB != nil)
+          otherDB_contents.update(otherDB.contents)
+          otherDB.instances.each_value do |i|
+            next unless i.server_name == server_name #and i.device == @device
+            otherDB_table[i.full_path] = i
+          end
+        end
 
-    # expand to absolute pathes
-    files.map! {|f| File.expand_path(f)}
+        permit_patterns = patterns.positive_patterns
+        forbid_patterns = patterns.negative_patterns
 
-    # remove files found by negative patterns
-    forbid_patterns.each_index do |i|
-      forbid_files = Array.new(collect(forbid_patterns[i]));
-      forbid_files.each do |f|
-        files.delete(File.expand_path(f))
-      end
-    end
+        # add files found by positive patterns
+        files = Array.new
+        permit_patterns.each_index do |i|
+          files = files | (collect(permit_patterns[i]));
+        end
 
-    # create and add contents and instances
-    files.each do |file|
-      file_stats = File.lstat(file)
+        p "Files: #{files}."
 
-      # index only files
-      next if (file_stats.directory?)
+        # expand to absolute pathes
+        files.map! {|f| File.expand_path(f)}
 
-      # keep only files with names in UTF-8
-      unless file.force_encoding("UTF-8").valid_encoding?
-        @log.warn { "Non-UTF8 file name \"#{file}\"" }
-        next
-      end
+        # remove files found by negative patterns
+        forbid_patterns.each_index do |i|
+          forbid_files = Array.new(collect(forbid_patterns[i]));
+          forbid_files.each do |f|
+            files.delete(File.expand_path(f))
+          end
+        end
 
-      # add files present in the given DB to the DB and remove these files
-      # from further processing (save checksum calculation)
-      if otherDB_table.has_key?(file)
-        instance = otherDB_table[file]
-        if instance.size == file_stats.size and instance.modification_time == file_stats.mtime.utc
-          @indexed_content.add_content(otherDB_contents[instance.checksum])
+        # create and add contents and instances
+        files.each do |file|
+          file_stats = File.lstat(file)
+
+          # index only files
+          next if (file_stats.directory?)
+
+          # keep only files with names in UTF-8
+          unless file.force_encoding("UTF-8").valid_encoding?
+            @log.warn { "Non-UTF8 file name \"#{file}\"" }
+            next
+          end
+
+          # add files present in the given DB to the DB and remove these files
+          # from further processing (save checksum calculation)
+          if otherDB_table.has_key?(file)
+            instance = otherDB_table[file]
+            if instance.size == file_stats.size and instance.modification_time == file_stats.mtime.utc
+              @indexed_content.add_content(otherDB_contents[instance.checksum])
+              @indexed_content.add_instance(instance)
+              next
+            end
+          end
+
+          # calculate a checksum
+          unless (checksum = self.class.get_checksum(file))
+            @log.warn { "Cheksum failure: " + file }
+            next
+          end
+
+          @indexed_content.add_content(ContentData::Content.new(checksum, file_stats.size, Time.now.utc)) \
+              unless @indexed_content.content_exists(checksum)
+
+          instance = ContentData::ContentInstance.new(checksum, file_stats.size, server_name, file_stats.dev.to_s,
+                                                      File.expand_path(file), file_stats.mtime.utc)
           @indexed_content.add_instance(instance)
-          next
         end
       end
-
-      # calculate a checksum
-      unless (checksum = self.class.get_checksum(file))
-        @log.warn { "Cheksum failure: " + file }
-        next
-      end
-
-      @indexed_content.add_content(Content.new(checksum, file_stats.size, Time.now.utc)) unless (@indexed_content.content_exists(checksum))
-
-      instance = ContentInstance.new(checksum, file_stats.size, server_name, file_stats.dev.to_s, File.expand_path(file), file_stats.mtime.utc)
-      @indexed_content.add_instance(instance)
     end
+
   end
 end

data/lib/file_indexing/indexer_patterns.rb

@@ -1,61 +1,66 @@
-require './agent.pb'
+module BBFS
+  module FileIndexing
 
-class IndexerPatterns
-  attr_reader :positive_patterns, :negative_patterns
+    class IndexerPatterns
+      attr_reader :positive_patterns, :negative_patterns
 
-  # @param indexer_patterns_str [String]
-  def initialize (indexer_patterns = nil)
-    @positive_patterns = Array.new
-    @negative_patterns = Array.new
-    # TODO add a test (including empty collections)
-    unless  indexer_patterns.nil?
-      indexer_patterns.positive_patterns.each do |pattern|
-        add_pattern(pattern)
+      # @param indexer_patterns_str [String]
+      def initialize (indexer_patterns = nil)
+        p "Initialize index patterns #{indexer_patterns}."
+        @positive_patterns = Array.new
+        @negative_patterns = Array.new
+        # TODO add a test (including empty collections)
+        if indexer_patterns
+          indexer_patterns.positive_patterns.each do |pattern|
+            add_pattern(pattern)
+          end
+          indexer_patterns.negative_patterns.each do |pattern|
+            add_pattern(pattern, false)
+          end
+        end
       end
-      indexer_patterns.negative_patterns.each do |pattern|
-        add_pattern(pattern, false)
-      end
-    end
-  end
 
-  def serialize
-    # TODO add a test (including empty collections)
-    indexer_patterns = IndexerPatternsMessage.new
-    positive_patterns.each do |pattern|
-      indexer_patterns.positive_patterns << pattern
-    end
-    negative_patterns.each do |pattern|
-      indexer_patterns.negative_patterns << pattern
-    end
-    indexer_patterns
-  end
+      def serialize
+        # TODO add a test (including empty collections)
+        indexer_patterns = IndexerPatternsMessage.new
+        positive_patterns.each do |pattern|
+          indexer_patterns.positive_patterns << pattern
+        end
+        negative_patterns.each do |pattern|
+          indexer_patterns.negative_patterns << pattern
+        end
+        indexer_patterns
+      end
 
-  # @param pattern [String]
-  # @param is_positive [true]
-  # @param is_positive [false]
-  def add_pattern(pattern, is_positive = true)
-    pattern.gsub!(/\\/,'/')
-    if (is_positive)
-      @positive_patterns << pattern
-    else
-      @negative_patterns << pattern
-    end
-  end
+      # @param pattern [String]
+      # @param is_positive [true]
+      # @param is_positive [false]
+      def add_pattern(pattern, is_positive = true)
+        pattern.gsub!(/\\/,'/')
+        if (is_positive)
+          @positive_patterns << pattern
+        else
+          @negative_patterns << pattern
+        end
+      end
 
-  def parse_from_file(file)
-    input_patterns = IO.readlines(file)
-    begin
-      puts "Error loading patterns=%s" % file
-      raise IOError("Error loading patterns=%s" % file)
-    end unless not input_patterns.nil?
+      def parse_from_file(file)
+        input_patterns = IO.readlines(file)
+        begin
+          puts "Error loading patterns=%s" % file
+          raise IOError("Error loading patterns=%s" % file)
+        end unless not input_patterns.nil?
 
-    input_patterns.each do |pattern|
-      if (m = /^\s*([+-]):(.*)/.match(pattern))
-        add_pattern(m[2], m[1].eql?('+') ? true : false)
-      elsif (not /^\s*[\/\/|#]/.match(pattern))   # not a comment
-        puts "pattern in incorrect format: #{pattern}"
-        raise RuntimeError("pattern in incorrect format: #{pattern}")
+        input_patterns.each do |pattern|
+          if (m = /^\s*([+-]):(.*)/.match(pattern))
+            add_pattern(m[2], m[1].eql?('+') ? true : false)
+          elsif (not /^\s*[\/\/|#]/.match(pattern))   # not a comment
+            puts "pattern in incorrect format: #{pattern}"
+            raise RuntimeError("pattern in incorrect format: #{pattern}")
+          end
+        end
       end
     end
+
   end
-end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: file_indexing
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.0.2
   prerelease: 
 platform: ruby
 authors:
@@ -9,8 +9,19 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-04-08 00:00:00.000000000Z
-dependencies: []
+date: 2012-04-11 00:00:00.000000000Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: content_data
+  requirement: &70262612434200 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *70262612434200
 description: Indexes files, treats files with same binary sequence as one content.
 email: kolmanv@gmail.com
 executables: []