nearline 0.0.2 → 0.0.3

data/lib/nearline/archived_file.rb

@@ -4,12 +4,13 @@ module Nearline
     # Represents file metadata and possible related FileContent
     # for a single file on a single system
     class ArchivedFile < ActiveRecord::Base
+      require 'fileutils'
+      
       belongs_to :file_content
       has_and_belongs_to_many :manifests
             
-      def self.create_for(system_name, file_path, manifest)
-        
-        file_information = FileInformation.new(system_name, file_path, manifest)
+      def self.create_for(file_path, manifest)        
+        file_information = FileInformation.new(file_path, manifest)
 
         # The path doesn't actually exist and fails a File.stat
         return nil if file_information.path_hash.nil?
@@ -41,28 +42,29 @@ module Nearline
       
       class FileInformation
         attr_reader :path_hash, :stat, :is_directory, :archived_file_parameters
-        def initialize(system_name, file_path, manifest)
+        def initialize(file_path, manifest)
           @manifest = manifest
-          @stat = read_stat(file_path)
+          @file_path = file_path
+          @stat = read_stat
           @is_directory = File.directory?(file_path)
-          @path_hash = generate_path_hash(system_name, file_path)
-          @archived_file_parameters = build_parameters(system_name, file_path)
+          @path_hash = generate_path_hash
+          @archived_file_parameters = build_parameters
         end
 
-        def read_stat(file_path)
+        def read_stat
           stat = nil
           begin
-            stat = File.stat(file_path)
+            stat = File.stat(@file_path)
           rescue
-            @manifest.add_log("File not found on stat: #{file_path}")
+            @manifest.add_log("File not found on stat: #{@file_path}")
           end
           stat
         end
 
-        def generate_path_hash(system_name, file_path)
+        def generate_path_hash
           return nil if @stat.nil?          
-          target = [system_name, 
-            file_path,
+          target = [@manifest.system_name, 
+            @file_path,
             @stat.uid,
             @stat.gid,
             @stat.mtime.to_i,
@@ -75,11 +77,11 @@ module Nearline
           return nil
         end
 
-        def build_parameters(system_name, file_path)
+        def build_parameters
           return nil if @stat.nil?
           {
-            :system_name => system_name,
-            :path => file_path,
+            :system_name => @manifest.system_name,
+            :path => @file_path,
             :path_hash => @path_hash,
             :file_content => file_content_entry_for_files_only,
             :uid => @stat.uid,
@@ -177,8 +179,7 @@ module Nearline
             io.read(Block::MAX_SIZE, buffer)
             file_size += buffer.size
             whole_file_hash.update(buffer)
-            block = Block.for_content(buffer)              
-            sequencer.preserve_block(block)
+            sequencer.preserve_content(buffer)
           end
         end
         return file_size

data/lib/nearline/block.rb

@@ -29,12 +29,15 @@ module Nearline
       end
     
       def content
+        if !@content.nil?
+          return @content
+        end
         if (self.is_compressed)
-          return Zlib::Inflate.inflate(self.bulk_content)
+          return @content = Zlib::Inflate.inflate(self.bulk_content)
         end
-        self.bulk_content
+        @content = self.bulk_content
       end
-          
+
       def self.for_content(x, old_block = nil)
         unless old_block.nil?
           if x == old_block.content

data/lib/nearline/file_content.rb

@@ -34,7 +34,7 @@ module Nearline
         sequences.each do |seq|
           block = Block.find(seq.block_id)
           io.write(block.content)
-    end
+        end
       end
     
       def verified?
@@ -70,18 +70,19 @@ module Nearline
         @file_content = file_content
       end
       
-      def preserve_block(block)
+      def preserve_content(content)
         @inc += 1
+        block = Block.for_content(content)
         sequence = Sequence.new(
           :sequence => @inc,
           :file_content_id => @file_content.id,
           :block_id => block.id
         )
         sequence.save!
-        sequence
+        sequence        
       end
+            
+    end
       
-        end
-      
-        end
-      end
+  end
+end

data/lib/nearline/manifest.rb

@@ -34,10 +34,11 @@ module Nearline
         manifest.save!
         
         FileFinder.recurse(backup_paths, backup_exclusions) do |file_name|
-          af = ArchivedFile.create_for(system_name, file_name, manifest)
+          $stdout.write file_name + " "
+          af = ArchivedFile.create_for(file_name, manifest)
           if (!af.nil?)
             manifest.archived_files << af
-            $stdout.write "#{af.path}  #{Time.at(af.mtime).asctime}"
+            $stdout.write "#{Time.at(af.mtime).asctime}"
             if (!af.file_content.nil?)
               $stdout.write" (#{af.file_content.file_size} bytes)"
             end
@@ -100,6 +101,23 @@ module Nearline
         end
       end
       
+      def total_size
+        size = 0
+        archived_files.each do |af|
+          unless af.file_content.nil?
+            size += af.file_content.file_size.to_i
+          end
+        end
+        size
+      end
+
+      # A simple string reporting the performance of the manifest
+      def summary
+        completed = (completed_at.nil?) ? "DNF" : completed_at
+        "#{system_name}; started: #{created_at}; finished: #{completed}; " +
+          "#{archived_files.size} files; #{logs.size} Errors reported"
+      end
+      
     end
     
   end

data/lib/nearline/module_methods.rb

@@ -1,5 +1,15 @@
 module Nearline  
   module_function
+
+  # Every model using an ActiveRecord connection
+  AR_MODELS = [
+    Nearline::Models::ArchivedFile,
+    Nearline::Models::Block,
+    Nearline::Models::FileContent,
+    Nearline::Models::Manifest,
+    Nearline::Models::Sequence,
+    Nearline::Models::Log
+  ]
   
   
   # Establishes the ActiveRecord connection
@@ -18,11 +28,13 @@ module Nearline
   # Nearline.connect! 'production'
   # 
   def connect!(config="development")
-    if (config.class.to_s == 'String')
-      ActiveRecord::Base.establish_connection(YAML.load_file("config/database.yml")[config])
+    if (config.is_a? String)
+      ActiveRecord::Base.establish_connection(
+        YAML.load_file("config/database.yml")[config]
+      )
     end
     
-    if (config.class.to_s == 'Hash')
+    if (config.is_a? Hash)
       ActiveRecord::Base.establish_connection(config)      
     end
     
@@ -49,39 +61,42 @@ module Nearline
     # These are the ActiveRecord models in place
     # Each one needs an explicit establish_connection
     # if you don't want it running though ActiveRecord::Base  
-    models = [
-      Nearline::Models::ArchivedFile,
-      Nearline::Models::Block,
-      Nearline::Models::FileContent,
-      Nearline::Models::Manifest,
-      Nearline::Models::Sequence,
-      Nearline::Models::Log
-    ]
-    if (config.class.to_s == 'String')
+    if (config.is_a? String)
       hash = YAML.load_file("config/database.yml")[config]
     else
       hash = config
     end
     
-    models.each do |m|
+    AR_MODELS.each do |m|
       m.establish_connection(hash)
     end
     Nearline::Models::Block.connected?
   end
   
   # Performs a backup labeled for system_name,
-  # Recursing through an array of backup_paths,
+  # Recursing through a single string or an array of backup_paths,
   # Excluding any path matching any of the regular
-  # expressions in the backup_exclusions array.
+  # expressions in the backup_exclusions array or single string.
   # 
   # Expects the Nearline database connection has already
   # been established
   # 
   # Returns a Manifest for the backup
   def backup(system_name, backup_paths,backup_exclusions= [])
-    Nearline::Models::Manifest.backup(system_name, backup_paths, backup_exclusions)
+    Nearline::Models::Manifest.backup(
+      system_name,
+      string_to_array(backup_paths),
+      string_to_array(backup_exclusions)
+    )
   end
-  
+     
+  def string_to_array(x)
+    if x.is_a? String
+      return [x]
+    end
+    x
+  end
+
   # Restore all missing files from the latest backup
   # for system_name
   # 

data/lib/nearline.rb

@@ -3,7 +3,6 @@
 # Nearline
 # Copyright (C) 2008 Robert J. Osborne
 
-require 'nearline/module_methods'
 
 # ActiveRecord database definitions
 require 'nearline/schema'
@@ -13,4 +12,7 @@ require 'nearline/block'
 require 'nearline/file_content'
 require 'nearline/archived_file'
 require 'nearline/log'
-require 'nearline/manifest'
+require 'nearline/manifest'
+
+# Static methods on Nearline
+require 'nearline/module_methods'

data/tasks/gemspec.rake

@@ -3,7 +3,7 @@ require 'rake/gempackagetask'
 
 SPEC = Gem::Specification.new do |s|
   s.name = "nearline"
-  s.version = "0.0.2"
+  s.version = "0.0.3"
   s.author = "Robert J. Osborne"
   s.email = "rjo1970@gmail.com"
   s.summary = "Nearline is a near-line backup and recovery solution"

metadata

@@ -3,8 +3,8 @@ rubygems_version: 0.9.4
 specification_version: 1
 name: nearline
 version: !ruby/object:Gem::Version 
-  version: 0.0.2
-date: 2008-04-05 00:00:00 -04:00
+  version: 0.0.3
+date: 2008-04-07 00:00:00 -04:00
 summary: Nearline is a near-line backup and recovery solution
 require_paths: 
 - lib