file_indexing 0.0.2 → 0.0.8

data/lib/file_indexing/index_agent.rb

@@ -1,11 +1,11 @@
 require 'digest/sha1'
-require 'logger'
 require 'pp'
+require 'set'
 require 'time'
 
 require 'content_data'
-
-require_relative 'indexer_patterns'
+require 'file_indexing/indexer_patterns'
+require 'log'
 
 module BBFS
   module FileIndexing
@@ -15,55 +15,58 @@ module BBFS
 ####################
 
     class IndexAgent
-      attr_reader :indexed_content
+      attr_reader :indexed_content, :failed_files
 
+      # Why are those lines needed?
       LOCALTZ = Time.now.zone
       ENV['TZ'] = 'UTC'
 
       def initialize
-        init_log()
-        init_db()
-      end
-
-      def init_db()
         @indexed_content = ContentData::ContentData.new
-      end
-
-      def init_log()
-        @log = Logger.new(STDERR)
-        @log.level = Logger::WARN
-        @log.datetime_format = "%Y-%m-%d %H:%M:%S"
-      end
-
-      def set_log(log_path, log_level)
-        @log = Logger.new(log_path) if log_path
-        @log.level = log_level
+        @failed_files = Set.new
       end
 
       # Calculate file checksum (SHA1)
       def self.get_checksum(filename)
         digest = Digest::SHA1.new
         begin
-          file = File.new(filename)
-          while buffer = file.read(65536)
-            digest << buffer
-          end
-          #@log.info { digest.hexdigest.downcase + ' ' + filename }
+          File.open(filename, 'rb') { |f|
+            while buffer = f.read(65536) do
+              digest << buffer
+            end
+          }
+          Log.debug1("#{filename} sha1 #{digest.hexdigest.downcase}")
           digest.hexdigest.downcase
         rescue Errno::EACCES, Errno::ETXTBSY => exp
-          @log.warn { "#{exp.message}" }
+          Log.warning("#{exp.message}")
           false
-        ensure
-          file.close if file != nil
         end
       end
 
+      def IndexAgent.get_content_checksum(content)
+        # Calculate checksum.
+        digest = Digest::SHA1.new
+        digest << content
+        digest.hexdigest.downcase
+      end
+
       # get all files
       # satisfying the pattern
       def collect(pattern)
         Dir.glob(pattern.to_s)
       end
 
+      # TODO(kolman): Replace this with File.lstat(file).mtime when new version of Ruby comes out.
+      # http://bugs.ruby-lang.org/issues/6385
+      def IndexAgent.get_correct_mtime(file)
+        begin
+        File.open(file, 'r') { |f| f.mtime }
+        rescue Errno::EACCES => e
+          Log.warning("Could not open file #{file} to get mtime. #{e}")
+          return 0
+        end
+      end
+
       # index device according to the pattern
       # store the result
       # does not adds automatically otherDB to stored result
@@ -72,7 +75,7 @@ module BBFS
         abort "#{self.class}: DB not empty. Current implementation permits only one running of index" \
             unless @indexed_content.contents.empty?
 
-        server_name = `hostname`
+        server_name = `hostname`.strip
         permit_patterns = Array.new
         forbid_patterns = Array.new
         otherDB_table = Hash.new   # contains instances from given DB while full path name is a key and instance is a value
@@ -97,7 +100,7 @@ module BBFS
           files = files | (collect(permit_patterns[i]));
         end
 
-        p "Files: #{files}."
+        Log.info "Files: #{files}."
 
         # expand to absolute pathes
         files.map! {|f| File.expand_path(f)}
@@ -113,13 +116,14 @@ module BBFS
         # create and add contents and instances
         files.each do |file|
           file_stats = File.lstat(file)
+          file_mtime = IndexAgent.get_correct_mtime(file)
 
           # index only files
-          next if (file_stats.directory?)
+          next if file_stats.directory?
 
           # keep only files with names in UTF-8
           unless file.force_encoding("UTF-8").valid_encoding?
-            @log.warn { "Non-UTF8 file name \"#{file}\"" }
+            Log.warning("Non UTF-8 file name \"#{file}\", skipping.")
             next
           end
 
@@ -127,27 +131,46 @@ module BBFS
           # from further processing (save checksum calculation)
           if otherDB_table.has_key?(file)
             instance = otherDB_table[file]
-            if instance.size == file_stats.size and instance.modification_time == file_stats.mtime.utc
+            if instance.size == file_stats.size and instance.modification_time == file_mtime
               @indexed_content.add_content(otherDB_contents[instance.checksum])
               @indexed_content.add_instance(instance)
               next
+            else
+              Log.warning("File (#{file}) size or modification file is different.")
             end
           end
 
           # calculate a checksum
           unless (checksum = self.class.get_checksum(file))
-            @log.warn { "Cheksum failure: " + file }
+            Log.warning("Cheksum failure: " + file)
+            @failed_files.add(file)
             next
           end
 
-          @indexed_content.add_content(ContentData::Content.new(checksum, file_stats.size, Time.now.utc)) \
-              unless @indexed_content.content_exists(checksum)
+          if !@indexed_content.content_exists(checksum)
+            @indexed_content.add_content ContentData::Content.new(checksum, file_stats.size,
+                                                                  Time.now.utc)
+          end
 
-          instance = ContentData::ContentInstance.new(checksum, file_stats.size, server_name, file_stats.dev.to_s,
-                                                      File.expand_path(file), file_stats.mtime.utc)
+          instance = ContentData::ContentInstance.new(
+              checksum, file_stats.size, server_name, file_stats.dev.to_s,
+              File.expand_path(file), file_mtime)
           @indexed_content.add_instance(instance)
         end
       end
+
+      def IndexAgent.create_shallow_instance(filename)
+        return nil unless File.exists?(filename)
+        file_stats = File.lstat(filename)
+        file_mtime = IndexAgent.get_correct_mtime(filename)
+        ContentData::ContentInstance.new(nil, file_stats.size, nil, file_stats.dev.to_s,
+                                         File.expand_path(filename), file_mtime)
+      end
+
+      def IndexAgent.global_path(filename)
+        server_name = `hostname`.strip
+        return ContentData::ContentInstance.instance_global_path(server_name, filename)
+      end
     end
 
   end

data/lib/file_indexing/indexer_patterns.rb

@@ -1,3 +1,6 @@
+require 'log'
+require 'params'
+
 module BBFS
   module FileIndexing
 
@@ -6,7 +9,7 @@ module BBFS
 
       # @param indexer_patterns_str [String]
       def initialize (indexer_patterns = nil)
-        p "Initialize index patterns #{indexer_patterns}."
+        Log.info "Initialize index patterns #{indexer_patterns}."
         @positive_patterns = Array.new
         @negative_patterns = Array.new
         # TODO add a test (including empty collections)
@@ -47,7 +50,7 @@ module BBFS
       def parse_from_file(file)
         input_patterns = IO.readlines(file)
         begin
-          puts "Error loading patterns=%s" % file
+          Log.info "Error loading patterns=%s" % file
           raise IOError("Error loading patterns=%s" % file)
         end unless not input_patterns.nil?
 
@@ -55,11 +58,15 @@ module BBFS
           if (m = /^\s*([+-]):(.*)/.match(pattern))
             add_pattern(m[2], m[1].eql?('+') ? true : false)
           elsif (not /^\s*[\/\/|#]/.match(pattern))   # not a comment
-            puts "pattern in incorrect format: #{pattern}"
+            Log.info "pattern in incorrect format: #{pattern}"
             raise RuntimeError("pattern in incorrect format: #{pattern}")
           end
         end
       end
+
+      def size
+        return @positive_patterns.size
+      end
     end
 
   end

data/lib/file_indexing/version.rb

@@ -0,0 +1,5 @@
+module BBFS
+  module FileIndexing
+    VERSION = "0.0.8"
+  end
+end

data/lib/file_indexing.rb

@@ -1,12 +1,11 @@
 require 'content_data'
 
-require_relative 'file_indexing/index_agent'
-require_relative 'file_indexing/indexer_patterns'
+require 'file_indexing/index_agent'
+require 'file_indexing/indexer_patterns'
 
 # Data structure for an abstract layer over files.
 # Each binary sequence is a content, each file is content instance.
 module BBFS
   module FileIndexing
-    VERSION = "0.0.1"
   end
 end

data/spec/file_indexing/index_agent_spec.rb

@@ -0,0 +1,54 @@
+require 'rspec'
+require 'tempfile'
+
+require_relative '../../lib/file_indexing/index_agent.rb'
+
+module BBFS
+  module FileCopy
+    module Spec
+
+      describe 'checksum' do
+        it 'should generate correct checksum' do
+          # The test does not checks the problem the problem is when reading from File
+          # class which handles read(num) different from read()
+          content = ''
+          100000.times { content << 'abagadavazahatikalamansapazkareshet' }
+          content_checksum = FileIndexing::IndexAgent.get_content_checksum(content)
+
+          stream = StringIO.new(content)
+          File.stub(:open).and_yield(stream)
+          file_checksum = FileIndexing::IndexAgent.get_checksum('kuku')
+
+          content_checksum.should == file_checksum
+          content_checksum.should == '381e99eb0e2dfcaf45c9a367a04a4197ef3039a6'
+        end
+
+        it 'should generate correct checksum for temp file' do
+          # A hack to get tmp file name
+          tmp_file = Tempfile.new('foo')
+          path = tmp_file .path
+          tmp_file .close()
+
+          # Open file in binary mode.
+          file = File.open(path, 'wb')
+          100000.times { file.write('abagadavazahatikalamansapazkareshet') }
+          file.close()
+
+          file_checksum = FileIndexing::IndexAgent.get_checksum(path)
+          file_checksum.should == '381e99eb0e2dfcaf45c9a367a04a4197ef3039a6'
+
+          File.open(path, 'rb') { |f|
+            content = f.read()
+            content_checksum = FileIndexing::IndexAgent.get_content_checksum(content)
+            content_checksum.should == '381e99eb0e2dfcaf45c9a367a04a4197ef3039a6'
+            file_checksum.should == content_checksum
+          }
+
+          # Delete tmp file.
+          tmp_file.unlink
+        end
+
+      end
+    end
+  end
+end

data/test/file_indexing/index_agent_test/libxslt-1.1.26.win32/include/libexslt/exslt.h

@@ -0,0 +1,102 @@
+
+#ifndef __EXSLT_H__
+#define __EXSLT_H__
+
+#include <libxml/tree.h>
+#include <libxml/xpath.h>
+#include "exsltexports.h"
+#include <libexslt/exsltconfig.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+EXSLTPUBVAR const char *exsltLibraryVersion;
+EXSLTPUBVAR const int exsltLibexsltVersion;
+EXSLTPUBVAR const int exsltLibxsltVersion;
+EXSLTPUBVAR const int exsltLibxmlVersion;
+
+/**
+ * EXSLT_COMMON_NAMESPACE:
+ *
+ * Namespace for EXSLT common functions
+ */
+#define EXSLT_COMMON_NAMESPACE ((const xmlChar *) "http://exslt.org/common")
+/**
+ * EXSLT_CRYPTO_NAMESPACE:
+ *
+ * Namespace for EXSLT crypto functions
+ */
+#define EXSLT_CRYPTO_NAMESPACE ((const xmlChar *) "http://exslt.org/crypto")
+/**
+ * EXSLT_MATH_NAMESPACE:
+ *
+ * Namespace for EXSLT math functions
+ */
+#define EXSLT_MATH_NAMESPACE ((const xmlChar *) "http://exslt.org/math")
+/**
+ * EXSLT_SETS_NAMESPACE:
+ *
+ * Namespace for EXSLT set functions
+ */
+#define EXSLT_SETS_NAMESPACE ((const xmlChar *) "http://exslt.org/sets")
+/**
+ * EXSLT_FUNCTIONS_NAMESPACE:
+ *
+ * Namespace for EXSLT functions extension functions
+ */
+#define EXSLT_FUNCTIONS_NAMESPACE ((const xmlChar *) "http://exslt.org/functions")
+/**
+ * EXSLT_STRINGS_NAMESPACE:
+ *
+ * Namespace for EXSLT strings functions
+ */
+#define EXSLT_STRINGS_NAMESPACE ((const xmlChar *) "http://exslt.org/strings")
+/**
+ * EXSLT_DATE_NAMESPACE:
+ *
+ * Namespace for EXSLT date functions
+ */
+#define EXSLT_DATE_NAMESPACE ((const xmlChar *) "http://exslt.org/dates-and-times")
+/**
+ * EXSLT_DYNAMIC_NAMESPACE:
+ *
+ * Namespace for EXSLT dynamic functions
+ */
+#define EXSLT_DYNAMIC_NAMESPACE ((const xmlChar *) "http://exslt.org/dynamic")
+
+/**
+ * SAXON_NAMESPACE:
+ *
+ * Namespace for SAXON extensions functions
+ */
+#define SAXON_NAMESPACE ((const xmlChar *) "http://icl.com/saxon")
+
+EXSLTPUBFUN void EXSLTCALL exsltCommonRegister (void);
+#ifdef EXSLT_CRYPTO_ENABLED
+EXSLTPUBFUN void EXSLTCALL exsltCryptoRegister (void);
+#endif
+EXSLTPUBFUN void EXSLTCALL exsltMathRegister (void);
+EXSLTPUBFUN void EXSLTCALL exsltSetsRegister (void);
+EXSLTPUBFUN void EXSLTCALL exsltFuncRegister (void);
+EXSLTPUBFUN void EXSLTCALL exsltStrRegister (void);
+EXSLTPUBFUN void EXSLTCALL exsltDateRegister (void);
+EXSLTPUBFUN void EXSLTCALL exsltSaxonRegister (void);
+EXSLTPUBFUN void EXSLTCALL exsltDynRegister(void);
+
+EXSLTPUBFUN void EXSLTCALL exsltRegisterAll (void);
+
+EXSLTPUBFUN int EXSLTCALL exsltDateXpathCtxtRegister (xmlXPathContextPtr ctxt,
+                                                      const xmlChar *prefix);
+EXSLTPUBFUN int EXSLTCALL exsltMathXpathCtxtRegister (xmlXPathContextPtr ctxt,
+                                                      const xmlChar *prefix);
+EXSLTPUBFUN int EXSLTCALL exsltSetsXpathCtxtRegister (xmlXPathContextPtr ctxt,
+                                                      const xmlChar *prefix);
+EXSLTPUBFUN int EXSLTCALL exsltStrXpathCtxtRegister (xmlXPathContextPtr ctxt,
+                                                     const xmlChar *prefix);
+
+#ifdef __cplusplus
+}
+#endif
+#endif /* __EXSLT_H__ */
+

data/test/file_indexing/index_agent_test/libxslt-1.1.26.win32/include/libexslt/exsltconfig.h

@@ -0,0 +1,73 @@
+/*
+ * exsltconfig.h: compile-time version informations for the EXSLT library
+ *
+ * See Copyright for the status of this software.
+ *
+ * daniel@veillard.com
+ */
+
+#ifndef __XML_EXSLTCONFIG_H__
+#define __XML_EXSLTCONFIG_H__
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/**
+ * LIBEXSLT_DOTTED_VERSION:
+ *
+ * the version string like "1.2.3"
+ */
+#define LIBEXSLT_DOTTED_VERSION "0.8.15"
+
+/**
+ * LIBEXSLT_VERSION:
+ *
+ * the version number: 1.2.3 value is 10203
+ */
+#define LIBEXSLT_VERSION 815
+
+/**
+ * LIBEXSLT_VERSION_STRING:
+ *
+ * the version number string, 1.2.3 value is "10203"
+ */
+#define LIBEXSLT_VERSION_STRING "815"
+
+/**
+ * LIBEXSLT_VERSION_EXTRA:
+ *
+ * extra version information, used to show a CVS compilation
+ */
+#define	LIBEXSLT_VERSION_EXTRA ""
+
+/**
+ * WITH_CRYPTO:
+ *
+ * Whether crypto support is configured into exslt
+ */
+#if 1
+#define EXSLT_CRYPTO_ENABLED
+#endif
+
+/**
+ * ATTRIBUTE_UNUSED:
+ *
+ * This macro is used to flag unused function parameters to GCC
+ */
+#ifdef __GNUC__
+#ifdef HAVE_ANSIDECL_H
+#include <ansidecl.h>
+#endif
+#ifndef ATTRIBUTE_UNUSED
+#define ATTRIBUTE_UNUSED __attribute__((unused))
+#endif
+#else
+#define ATTRIBUTE_UNUSED
+#endif
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* __XML_EXSLTCONFIG_H__ */