nearline 0.0.3 → 0.0.4

data/lib/nearline/archived_file.rb

@@ -7,7 +7,9 @@ module Nearline
       require 'fileutils'
       
       belongs_to :file_content
+      belongs_to :system
       has_and_belongs_to_many :manifests
+      
             
       def self.create_for(file_path, manifest)        
         file_information = FileInformation.new(file_path, manifest)
@@ -16,8 +18,14 @@ module Nearline
         return nil if file_information.path_hash.nil?
 
         # If we find an exising entry, use it
-        hit = self.find_by_path_hash(file_information.path_hash)
-        return hit unless hit.nil?
+        hash = manifest.system.archived_file_lookup_hash
+        hit = hash[file_information.path_hash]
+
+        unless hit.nil?
+          af = ArchivedFile.find(hit)
+          manifest.archived_files << af
+          return af
+        end
         
         # We need to create a record for either a directory or file
         archived_file = ArchivedFile.new(
@@ -27,14 +35,16 @@ module Nearline
         # Find a new directory
         if (file_information.is_directory)
           archived_file.save!
+          manifest.archived_files << archived_file
           return archived_file
         end
         
         # Find a new file that needs persisted
         archived_file.file_content.file_size = 
           [file_information.stat.size].pack('Q').unpack('L').first # HACK for Windows
-        archived_file.persist(manifest)
-        archived_file.save!
+        archived_file = archived_file.persist(manifest)
+        archived_file.save! unless archived_file.nil?
+        manifest.archived_files << archived_file
         archived_file
         
         # TODO: Symbolic links, block devices, ...?
@@ -63,7 +73,7 @@ module Nearline
 
         def generate_path_hash
           return nil if @stat.nil?          
-          target = [@manifest.system_name, 
+          target = [@manifest.system.name, 
             @file_path,
             @stat.uid,
             @stat.gid,
@@ -73,14 +83,14 @@ module Nearline
         end
         
         def file_content_entry_for_files_only
-          return FileContent.fresh_entry unless @is_directory
+          return FileContent.new unless @is_directory
           return nil
         end
 
         def build_parameters
           return nil if @stat.nil?
           {
-            :system_name => @manifest.system_name,
+            :system => @manifest.system,
             :path => @file_path,
             :path_hash => @path_hash,
             :file_content => file_content_entry_for_files_only,
@@ -190,19 +200,21 @@ module Nearline
           manifest.add_log "recorded file length #{file_size} " +
             "does not match #{self.file_content.file_size} " +
             "reported by the file system on path: #{self.path}"
+          self.file_content.file_size = file_size
         end        
       end
       
       def verify_content(manifest)
         unless (self.file_content.verified?)
-          manifest.add_log "failed verification on path: #{self.path}"
+          manifest.add_log "file dropped on failed verification on path: #{self.path}"
+          self.file_content.orphan_check
+          self.destroy
         end        
       end
       
       def unique_sequence_processed?(key,manifest)
         if self.file_content.unique_fingerprint?(key)
           self.file_content.fingerprint = key
-          self.file_content.save!
           self.save!
           verify_content(manifest)
           return true
@@ -211,7 +223,6 @@ module Nearline
       end
       
       def clean_up_duplicate_content
-        Sequence.delete_all("file_content_id=#{self.file_content.id}")
         self.file_content.orphan_check
       end
       

data/lib/nearline/block.rb

@@ -4,9 +4,9 @@ module Nearline
   module Models
 
     # Represents a unit of file content which may be
-    # freely shared across the repository
+    # freely shared across the repository.
     # Its sole responsibility is to preserve and provide
-    # content access
+    # content access.
     class Block < ActiveRecord::Base
       require "zlib"
     
@@ -37,13 +37,22 @@ module Nearline
         end
         @content = self.bulk_content
       end
-
-      def self.for_content(x, old_block = nil)
-        unless old_block.nil?
-          if x == old_block.content
-            return old_block
-          end
+      
+      def self.id_for_content(x)
+        block = Block.new(:bulk_content => x)
+        block.calculate_fingerprint
+        hit = Block.connection.select_one(
+          "select id from blocks where fingerprint='#{block.fingerprint}'"
+        )
+        unless hit.nil?
+          return hit['id']
         end
+        block.attempt_compression
+        block.save!
+        block.id
+      end
+
+      def self.for_content(x)
         block = Models::Block.new(:bulk_content => x)
         block.calculate_fingerprint
         found = find_by_fingerprint(block.fingerprint)

data/lib/nearline/file_content.rb

@@ -1,23 +1,16 @@
 module Nearline
   module Models
 
-    # Has the responsibility of identifying and
+    # Has the responsibility of identifying, restoring and
     # verifying content
     class FileContent < ActiveRecord::Base
       has_many :sequences
       has_many :archived_files
-
-      def self.fresh_entry
-        file_content = FileContent.new
-        file_content.save!
-        file_content
-      end
       
       def orphan_check
-        if (self.archived_files.size == 1)
+        if (self.archived_files.size < 2)
           sequences.each do |s|
             s.destroy
-            s.block.orphan_check
           end
           self.destroy
         end
@@ -62,21 +55,26 @@ module Nearline
     class Sequence < ActiveRecord::Base
       belongs_to :block
       belongs_to :file_content
+      
+      def after_destroy
+        block.orphan_check
+      end
     end
     
     class FileSequencer
       def initialize(file_content)
         @inc = 0
         @file_content = file_content
+        @file_content.save!
       end
       
       def preserve_content(content)
         @inc += 1
-        block = Block.for_content(content)
+        block_id = Block.id_for_content(content)
         sequence = Sequence.new(
           :sequence => @inc,
           :file_content_id => @file_content.id,
-          :block_id => block.id
+          :block_id => block_id
         )
         sequence.save!
         sequence        

data/lib/nearline/manifest.rb

@@ -5,10 +5,14 @@ module Nearline
     class FileFinder
       require 'find'
       def self.recurse(paths, exclusions)
+        regex_exclusions = []
+        for exclusion in exclusions
+          regex_exclusions << /#{exclusion}/
+        end
         paths.each do |path|
           Find.find(path) do |f|
-            exclusions.each do |exclusion|
-              Find.prune if f =~ /#{exclusion}/
+            regex_exclusions.each do |ex|
+              Find.prune if ex.match(f)
             end
             yield f
           end
@@ -22,22 +26,28 @@ module Nearline
       
       has_and_belongs_to_many :archived_files
       has_many :logs
+      belongs_to :system
       
       # Just needed when you create a manifest
       attr_accessor :backup_paths
       # Just needed when you create a manifest
       attr_accessor :backup_exclusions
+            
+      def self.new_for_name(system_name)
+        system = System.for_name(system_name)
+        system.manifests << m = Nearline::Models::Manifest.new
+        system.save!
+        m        
+      end
       
-      # Underlying implementation of Nearline.backup
-      def self.backup(system_name, backup_paths, backup_exclusions)
-        manifest = self.new(:system_name => system_name)
+      def self.backup(system, backup_paths, backup_exclusions)
+        manifest = self.new(:system => system)
         manifest.save!
         
         FileFinder.recurse(backup_paths, backup_exclusions) do |file_name|
           $stdout.write file_name + " "
           af = ArchivedFile.create_for(file_name, manifest)
           if (!af.nil?)
-            manifest.archived_files << af
             $stdout.write "#{Time.at(af.mtime).asctime}"
             if (!af.file_content.nil?)
               $stdout.write" (#{af.file_content.file_size} bytes)"
@@ -51,14 +61,8 @@ module Nearline
         manifest
       end
       
-      # Find the latest Manifest for a system
-      def self.latest_for(system_name)
-        m_result = self.connection.select_one("select id from manifests where system_name='#{system_name}' order by created_at desc")
-        raise "No manifest found" if m_result.nil?
-        self.find(m_result["id"])        
-      end
       
-      # Find all Manifest entries which have never finished.
+      # Find all Manifest entries (across all Systems) which have never finished.
       # 
       # They are:
       # * Currently under-way
@@ -67,8 +71,8 @@ module Nearline
         self.find_all_by_completed_at(nil)
       end
       
-      def self.restore_all_missing(system_name)
-        manifest = latest_for(system_name)
+      def self.restore_all_missing(system, latest_date_time = Time.now)
+        manifest = system.latest_manifest_as_of(latest_date_time)
         manifest.restore_all_missing
       end
       
@@ -114,7 +118,7 @@ module Nearline
       # A simple string reporting the performance of the manifest
       def summary
         completed = (completed_at.nil?) ? "DNF" : completed_at
-        "#{system_name}; started: #{created_at}; finished: #{completed}; " +
+        "#{system.name} started: #{created_at}; finished: #{completed}; " +
           "#{archived_files.size} files; #{logs.size} Errors reported"
       end
       

data/lib/nearline/module_methods.rb

@@ -1,16 +1,15 @@
 module Nearline  
   module_function
 
-  # Every model using an ActiveRecord connection
-  AR_MODELS = [
-    Nearline::Models::ArchivedFile,
-    Nearline::Models::Block,
-    Nearline::Models::FileContent,
-    Nearline::Models::Manifest,
-    Nearline::Models::Sequence,
-    Nearline::Models::Log
-  ]
+  # VERSION of the software
+  VERSION = "0.0.4"
   
+  # Array of every Nearline Model using an ActiveRecord connection
+  AR_MODELS = Nearline::Models.constants.map do |m|
+    Nearline::Models.const_get(m)
+  end.select do |c|
+    c.superclass == ActiveRecord::Base
+  end
   
   # Establishes the ActiveRecord connection
   # 
@@ -32,16 +31,14 @@ module Nearline
       ActiveRecord::Base.establish_connection(
         YAML.load_file("config/database.yml")[config]
       )
-    end
-    
-    if (config.is_a? Hash)
+    elsif (config.is_a? Hash)
       ActiveRecord::Base.establish_connection(config)      
     end
     
     unless Nearline::Models::Block.table_exists?
       Nearline::Models.generate_schema
     end
-    Nearline::Models::Block.connected?
+    nil
   end
   
   # Establishes a connection only to the Nearline ActiveDirectory models
@@ -70,7 +67,7 @@ module Nearline
     AR_MODELS.each do |m|
       m.establish_connection(hash)
     end
-    Nearline::Models::Block.connected?
+    nil
   end
   
   # Performs a backup labeled for system_name,
@@ -82,32 +79,75 @@ module Nearline
   # been established
   # 
   # Returns a Manifest for the backup
+  #
+  # === Examples
+  # Backup my laptop, recursing my home folder, skipping .svn folders
+  # 
+  # Nearline.backup('my_laptop','/home/me', '/\\.svn/')
+  #
+  # Backup my laptop, recurse /home/me and /var/svn
+  # 
+  # Nearline.backup('my_laptop', ['/home/me', '/var/svn']
+  #
   def backup(system_name, backup_paths,backup_exclusions= [])
-    Nearline::Models::Manifest.backup(
+    unless version_check?
+      raise SchemaVersionException.for_version(schema_version)
+    end    
+    Nearline::Models::System.backup(
       system_name,
-      string_to_array(backup_paths),
-      string_to_array(backup_exclusions)
+      Utilities.string_to_array(backup_paths),
+      Utilities.string_to_array(backup_exclusions)
     )
   end
-     
-  def string_to_array(x)
-    if x.is_a? String
-      return [x]
+  
+  module Utilities
+    module_function
+    def self.string_to_array(x)
+      if x.is_a? String
+        return [x]
+      end
+      x      
     end
-    x
   end
-
+  
   # Restore all missing files from the latest backup
-  # for system_name
+  # for system_name and backed up no later than latest_date_time
   # 
-  # All updated or existing files are left alone
+  # All modified or existing files are left alone
   # 
   # Expects the Nearline database connection has already
   # been established
   # 
   # Returns an Array of paths restored
-  def restore(system_name)
-    Nearline::Models::Manifest.restore_all_missing(system_name)
+  def restore(system_name, latest_date_time = Time.now)
+    unless version_check?
+      raise SchemaVersionException.for_version(schema_version)
+    end    
+    Nearline::Models::System.restore_all_missing(system_name, latest_date_time)
+  end
+  
+  
+  # Returns the nearline version of the database
+  def schema_version
+    begin
+      return Nearline::Models::Block.connection.select_value(
+        "select version from nearline_version"
+      )
+    rescue
+      return ""
+    end    
+  end
+  
+  # Returns true only if the Nearline version matches the schema
+  def version_check?
+    Nearline::VERSION == schema_version()
+  end
+  
+  class SchemaVersionException < Exception
+    def self.for_version(v)
+      SchemaVersionException.new("Schema #{v} is not the same "+
+          "version as nearline #{Nearline::VERSION}!")
+    end
   end
   
 end

data/lib/nearline/schema.rb

@@ -12,6 +12,8 @@ module Nearline
         drop_table :manifests
         drop_table :archived_files_manifests
         drop_table :logs
+        drop_table :systems
+        drop_table :nearline_version
       end
     end
     
@@ -46,9 +48,15 @@ module Nearline
           :name => "sequence_jn_index"
 
         add_index :sequences, [:block_id]
+        
+        create_table :systems do |t|
+          t.column :name, :string, :null => false
+        end
+        
+        add_index :systems, [:name], :unique => true
 
         create_table :archived_files do |t|
-          t.column :system_name, :string, :null => false
+          t.column :system_id, :integer, :null => false
           t.column :path, :text, :null => false
           t.column :path_hash, :string, :null => false, :length => 40
           t.column :file_content_id, :integer
@@ -63,7 +71,7 @@ module Nearline
                         
         # Manifests are the reference to a collection of archived files
         create_table :manifests do |t|
-          t.column :system_name, :string
+          t.column :system_id, :integer
           t.column :created_at, :datetime
           t.column :completed_at, :datetime
         end
@@ -86,6 +94,12 @@ module Nearline
           t.column :message, :text
           t.column :created_at, :datetime
         end
+        
+        create_table :nearline_version, :id => false do |t|
+          t.column :version, :string
+        end
+        
+        execute "insert into nearline_version (version) values ('#{Nearline::VERSION}')"        
       end
     end
 

data/lib/nearline/system.rb

@@ -0,0 +1,73 @@
+module Nearline
+  module Models
+    
+    # The System has the responsibility of identifying
+    # what the target was for a backup and relating all
+    # Manifests and ArchivedFiles associated with the
+    # target system.
+    class System < ActiveRecord::Base
+
+      has_many :manifests
+      has_many :archived_files
+      
+      
+      def self.for_name(system_name)
+        system = self.find_by_name(system_name)
+        return system unless system.nil?
+        system = self.new(:name => system_name)
+        system.save!
+        system
+      end
+            
+      # Find the latest Manifest for a system
+      # given the latest_date_time as an upper limit
+      def latest_manifest_as_of(latest_date_time = Time.now)
+        m_result = Manifest.find(:first,
+          :conditions => 
+            ["system_id = ? and created_at <= ?",
+            self.id, latest_date_time],
+          :order => "created_at desc"
+        )
+        raise "No manifest found" if m_result.nil?
+        m_result        
+      end
+      
+      # Method used by the Nearline module to backup the system
+      def self.backup(system_name, backup_paths, backup_exclusions)
+        system = self.for_name(system_name)
+        system.backup(backup_paths, backup_exclusions)
+      end
+      
+      def backup(backup_paths, backup_exclusions)
+        Manifest.backup(self, backup_paths, backup_exclusions)
+      end
+      
+      # Method used by the Nearline module to restore the system
+      def self.restore_all_missing(system_name, latest_date_time)
+        system = self.for_name(system_name)
+        system.restore_all_missing(latest_date_time)
+      end
+      
+      def restore_all_missing(latest_date_time)
+        Manifest.restore_all_missing(self, latest_date_time)
+      end
+      
+      def archived_file_lookup_hash
+        return @lookup_hash if !@lookup_hash.nil?
+        @lookup_hash = {}
+        for af in self.archived_files
+          @lookup_hash[af.path_hash] = af.id
+        end
+        @lookup_hash
+      end
+      
+      def before_destroy
+        for manifest in self.manifests
+          manifest.destroy
+        end
+      end
+      
+    end
+    
+  end
+end

data/lib/nearline.rb

@@ -10,6 +10,7 @@ require 'nearline/schema'
 # ActiveRecord models
 require 'nearline/block'
 require 'nearline/file_content'
+require 'nearline/system'
 require 'nearline/archived_file'
 require 'nearline/log'
 require 'nearline/manifest'

data/tasks/gemspec.rake

@@ -1,15 +1,18 @@
 require 'rake'
 require 'rake/gempackagetask'
 
+$:.unshift File.join(File.dirname(__FILE__), "..", "lib")
+require 'nearline'
+
 SPEC = Gem::Specification.new do |s|
   s.name = "nearline"
-  s.version = "0.0.3"
+  s.version = Nearline::VERSION
   s.author = "Robert J. Osborne"
   s.email = "rjo1970@gmail.com"
   s.summary = "Nearline is a near-line backup and recovery solution"
   s.description = %{
     Nearline is a library to make managing near-line file repositories
-    simple and eleant in pure Ruby.
+    simple and elegant in pure Ruby.
   }
   s.rubyforge_project = "nearline"
   s.files = FileList["{tests,lib,doc,tasks}/**/*"].exclude("rdoc").to_a

metadata

@@ -3,15 +3,15 @@ rubygems_version: 0.9.4
 specification_version: 1
 name: nearline
 version: !ruby/object:Gem::Version 
-  version: 0.0.3
-date: 2008-04-07 00:00:00 -04:00
+  version: 0.0.4
+date: 2008-04-18 00:00:00 -04:00
 summary: Nearline is a near-line backup and recovery solution
 require_paths: 
 - lib
 email: rjo1970@gmail.com
 homepage: 
 rubyforge_project: nearline
-description: Nearline is a library to make managing near-line file repositories simple and eleant in pure Ruby.
+description: Nearline is a library to make managing near-line file repositories simple and elegant in pure Ruby.
 autorequire: nearline
 default_executable: 
 bindir: bin
@@ -37,6 +37,7 @@ files:
 - lib/nearline/manifest.rb
 - lib/nearline/module_methods.rb
 - lib/nearline/schema.rb
+- lib/nearline/system.rb
 - lib/nearline.rb
 - tasks/clean.rake
 - tasks/gemspec.rake