nearline 0.0.1

data/lib/nearline/archived_file.rb

@@ -0,0 +1,224 @@
+module Nearline
+  module Models
+    
+    # Represents file metadata and possible related FileContent
+    # for a single file on a single system
+    class ArchivedFile < ActiveRecord::Base
+      belongs_to :file_content
+      has_and_belongs_to_many :manifests
+            
+      def self.create_for(system_name, file_path, manifest)
+        
+        file_information = FileInformation.new(system_name, file_path, manifest)
+
+        # The path doesn't actually exist and fails a File.stat
+        return nil if file_information.path_hash.nil?
+
+        # If we find an exising entry, use it
+        hit = self.find_by_path_hash(file_information.path_hash)
+        return hit unless hit.nil?
+        
+        # We need to create a record for either a directory or file
+        archived_file = ArchivedFile.new(
+          file_information.archived_file_parameters
+        )
+        
+        # Find a new directory
+        if (file_information.is_directory)
+          archived_file.save!
+          return archived_file
+        end
+        
+        # Find a new file that needs persisted
+        archived_file.file_content.file_size = 
+          [file_information.stat.size].pack('Q').unpack('L').first # HACK for Windows
+        archived_file.persist(manifest)
+        archived_file.save!
+        archived_file
+        
+        # TODO: Symbolic links, block devices, ...?
+      end
+      
+      class FileInformation
+        attr_reader :path_hash, :stat, :is_directory, :archived_file_parameters
+        def initialize(system_name, file_path, manifest)
+          @manifest = manifest
+          @stat = read_stat(file_path)
+          @is_directory = File.directory?(file_path)
+          @path_hash = generate_path_hash(system_name, file_path)
+          @archived_file_parameters = build_parameters(system_name, file_path)
+        end
+
+        def read_stat(file_path)
+          stat = nil
+          begin
+            stat = File.stat(file_path)
+          rescue
+            @manifest.add_log("File not found on stat: #{file_path}")
+          end
+          stat
+        end
+
+        def generate_path_hash(system_name, file_path)
+          return nil if @stat.nil?          
+          target = [system_name, 
+            file_path,
+            @stat.uid,
+            @stat.gid,
+            @stat.mtime.to_i,
+            @stat.mode].join(':')
+          Digest::SHA1.hexdigest(target)
+        end
+        
+        def file_content_entry_for_files_only
+          return FileContent.fresh_entry unless @is_directory
+          return nil
+        end
+
+        def build_parameters(system_name, file_path)
+          return nil if @stat.nil?
+          {
+            :system_name => system_name,
+            :path => file_path,
+            :path_hash => @path_hash,
+            :file_content => file_content_entry_for_files_only,
+            :uid => @stat.uid,
+            :gid => @stat.gid,
+            :mtime => @stat.mtime.to_i,
+            :mode => @stat.mode,
+            :is_directory => @is_directory    
+          }
+        end
+
+      end
+      
+      def restore(*args)
+        @options = args.extract_options!
+        if (self.is_directory)
+          FileUtils.mkdir_p option_override(:path)
+          restore_metadata
+          return
+        end
+        target_path = File.dirname(option_override(:path))
+        if (!File.exist? target_path)
+          FileUtils.mkdir_p target_path
+        end
+        f = File.open(option_override(:path), "wb")
+        self.file_content.restore_to(f)
+        f.close
+        restore_metadata
+        return
+      end
+      
+      def option_override(key)
+        if (@options.has_key?(key))
+          return @options[key]
+        end
+        return self.send(key.to_s)
+      end
+      
+      def restore_metadata
+        path = option_override(:path)
+        mtime = option_override(:mtime)
+        uid = option_override(:uid)
+        gid = option_override(:gid)
+        mode = option_override(:mode)
+        File.utime(0,Time.at(mtime),path)
+        File.chown(uid, gid, path)
+        File.chmod(mode, path)
+      end
+      
+      def before_destroy
+        self.file_content.orphan_check if !self.file_content.nil?
+      end
+      
+      def orphan_check
+        if self.manifests.size == 1
+          self.destroy
+        end
+      end
+      
+      # Actually persist the file to the repository
+      # It has already been determined that a new ArchivedFile record is
+      # necessary and the file requires persisting
+      # 
+      # But, the content may be identical to something else, and we
+      # won't know that until we complete the process and have to
+      # clean up our mess.
+      def persist(manifest)
+        whole_file_hash = Digest::SHA1.new
+        file_size = 0
+        begin
+          file_size = read_file_counting_bytes(whole_file_hash)
+        rescue
+          manifest.add_log "Got error '#{$!}' on path: #{self.path}"
+          self.orphan_check
+          return nil
+        end
+        
+        size_check(file_size, manifest)
+        
+        # Do we have a unique sequence?
+        key = whole_file_hash.hexdigest
+        return self if unique_sequence_processed?(key, manifest)
+                
+        # Handle the case where the sequence is not unique...
+        clean_up_duplicate_content
+        replace_content(key)
+        self
+      end
+      
+      def read_file_counting_bytes(whole_file_hash)
+        sequencer = FileSequencer.new(self.file_content)
+        file_size = 0
+        buffer = ""
+        File.open(self.path, "rb") do |io|
+          while (!io.eof) do
+            io.read(Block::MAX_SIZE, buffer)
+            file_size += buffer.size
+            whole_file_hash.update(buffer)
+            block = Block.for_content(buffer)              
+            sequencer.preserve_block(block)
+          end
+        end
+        return file_size
+      end
+            
+      def size_check(file_size, manifest)
+        if file_size != self.file_content.file_size
+          manifest.add_log "recorded file length #{file_size} " +
+            "does not match #{self.file_content.file_size} " +
+            "reported by the file system on path: #{self.path}"
+        end        
+      end
+      
+      def verify_content(manifest)
+        unless (self.file_content.verified?)
+          manifest.add_log "failed verification on path: #{self.path}"
+        end        
+      end
+      
+      def unique_sequence_processed?(key,manifest)
+        if self.file_content.unique_fingerprint?(key)
+          self.file_content.fingerprint = key
+          self.file_content.save!
+          verify_content(manifest)
+          return true
+        end
+        false
+      end
+      
+      def clean_up_duplicate_content
+        Sequence.delete_all("file_content_id=#{self.file_content.id}")
+        self.file_content.orphan_check
+      end
+      
+      def replace_content(key)
+        self.file_content = FileContent.find_by_fingerprint(key)
+        self.save!                
+      end
+      
+    end
+
+  end
+end

data/lib/nearline/block.rb

@@ -0,0 +1,56 @@
+require 'active_record'
+
+module Nearline
+  module Models
+
+    # Represents a unit of file content which may be
+    # freely shared across the repository
+    # Its sole responsibility is to preserve and provide
+    # content access
+    class Block < ActiveRecord::Base
+      require "zlib"
+    
+      has_many :sequences
+      
+      MAX_SIZE = (64 * 1024)-1
+      
+      def attempt_compression
+        return if (self.is_compressed)
+        # TODO: Have a bump-the-compression option, here?
+        candidate_content = Zlib::Deflate.deflate(self.bulk_content)
+        if candidate_content.length < self.bulk_content.length
+          self.is_compressed = true
+          self.bulk_content = candidate_content
+        end
+      end
+      
+      def calculate_fingerprint
+        self.fingerprint = Digest::SHA1.hexdigest(content)        
+      end
+    
+      def content
+        if (self.is_compressed)
+          return Zlib::Inflate.inflate(self.bulk_content)
+        end
+        self.bulk_content
+      end
+          
+      def self.for_content(x)
+        block = Models::Block.new(:bulk_content => x)
+        block.calculate_fingerprint
+        found = find_by_fingerprint(block.fingerprint)
+        return found if !found.nil?
+        block.attempt_compression
+        block.save!
+        block
+      end
+      
+      def orphan_check
+        if self.sequences.size == 0
+          self.destroy
+        end
+      end
+            
+    end
+  end
+end

data/lib/nearline/file_content.rb

@@ -0,0 +1,85 @@
+module Nearline
+  module Models
+
+    # Has the responsibility of identifying and
+    # verifying content
+    class FileContent < ActiveRecord::Base
+      has_many :sequences
+      has_many :archived_files
+
+      def self.fresh_entry
+        file_content = FileContent.new
+        file_content.save!
+        file_content
+      end
+      
+      def restore_to(io)
+        sequencer = FileSequencer.new(self)
+        sequencer.restore_blocks(io)
+      end
+      
+      def verified?
+        sequencer = FileSequencer.new(self)
+        sequencer.verified?
+      end
+      
+      def orphan_check
+        if (self.archived_files.size == 1)
+          sequences.each do |s|
+            s.destroy
+            s.block.orphan_check
+          end
+          self.destroy
+        end
+      end
+            
+      def unique_fingerprint?(key)
+        hit = FileContent.connection.select_one(
+          "select id from file_contents where fingerprint='#{key}'"
+        )
+        return hit.nil?
+      end
+      
+    end
+    
+    # Has the responsibility of preserving
+    # cardinality of stored blocks
+    class Sequence < ActiveRecord::Base
+      belongs_to :block
+      belongs_to :file_content
+    end
+    
+    class FileSequencer
+      def initialize(file_content)
+        @inc = 0
+        @file_content = file_content
+      end
+      
+      def preserve_block(block)
+        @inc += 1
+        sequence = Sequence.new(
+          :sequence => @inc,
+          :file_content_id => @file_content.id,
+          :block_id => block.id
+        )
+        sequence.save!
+        sequence
+      end
+      
+      def restore_blocks(io)
+        @file_content.sequences.each do |seq|
+          io.write(seq.block.content)
+        end
+      end
+      
+      def verified?
+        whole_file_hash = Digest::SHA1.new
+        @file_content.sequences.each do |seq|
+          whole_file_hash.update(seq.block.content)
+        end
+        @file_content.fingerprint == whole_file_hash.hexdigest
+      end
+    end
+
+  end
+end

data/lib/nearline/log.rb

@@ -0,0 +1,11 @@
+module Nearline
+  module Models
+    
+    # A simple message log for reporting problems during the creation
+    # of a Manifest
+    class Log < ActiveRecord::Base
+      
+    end
+    
+  end
+end

data/lib/nearline/manifest.rb

@@ -0,0 +1,106 @@
+module Nearline
+  module Models
+    
+    # Recuses paths and finds the files to back up
+    class FileFinder
+      require 'find'
+      def self.recurse(paths, exclusions)
+        paths.each do |path|
+          Find.find(path) do |f|
+            exclusions.each do |exclusion|
+              Find.prune if f =~ /#{exclusion}/
+            end
+            yield f
+          end
+        end
+      end
+    end
+    
+    # A Manifest represents the corpus of ArchivedFiles and
+    # set of Log messages resulting from a backup attempt
+    class Manifest < ActiveRecord::Base
+      
+      has_and_belongs_to_many :archived_files
+      has_many :logs
+      
+      # Just needed when you create a manifest
+      attr_accessor :backup_paths
+      # Just needed when you create a manifest
+      attr_accessor :backup_exclusions
+      
+      # Underlying implementation of Nearline.backup
+      def self.backup(system_name, backup_paths, backup_exclusions)
+        manifest = self.new(:system_name => system_name)
+        manifest.save!
+        
+        FileFinder.recurse(backup_paths, backup_exclusions) do |file_name|
+          af = ArchivedFile.create_for(system_name, file_name, manifest)
+          if (!af.nil?)
+            manifest.archived_files << af
+            $stdout.write "#{af.path}  #{Time.at(af.mtime).asctime}"
+            if (!af.file_content.nil?)
+              $stdout.write" (#{af.file_content.file_size} bytes)"
+            end
+            $stdout.write("\n")
+          end
+        end
+
+        manifest.completed_at = Time.now
+        manifest.save!
+        manifest
+      end
+      
+      # Find the latest Manifest for a system
+      def self.latest_for(system_name)
+        m_result = self.connection.select_one("select id from manifests where system_name='#{system_name}' order by created_at desc")
+        raise "No manifest found" if m_result.nil?
+        self.find(m_result["id"])        
+      end
+      
+      # Find all Manifest entries which have never finished.
+      # 
+      # They are:
+      # * Currently under-way
+      # * Have failed in some untimely way
+      def self.incomplete_manifests
+        self.find_all_by_completed_at(nil)
+      end
+      
+      def self.restore_all_missing(system_name)
+        manifest = latest_for(system_name)
+        manifest.restore_all_missing
+      end
+      
+      # Restore all missing files from this manifest back to the filesystem
+      def restore_all_missing
+        files_restored = []
+        self.archived_files.each do |af|
+          begin
+            File.stat(af.path)
+          rescue
+            af.restore
+            files_restored << af.path
+          end
+        end
+        return files_restored
+      end
+      
+      def add_log(message)
+        puts message
+        log = Nearline::Models::Log.new({:message => message, :manifest_id => self.id})
+        log.save!
+      end
+      
+      def before_destroy
+        archived_files.each do |af|
+          af.orphan_check
+        end
+        logs.each do |log|
+          log.destroy
+        end
+      end
+      
+    end
+    
+  end
+end

data/lib/nearline/module_methods.rb

@@ -0,0 +1,98 @@
+module Nearline  
+  module_function
+  
+  
+  # Establishes the ActiveRecord connection
+  # 
+  # Accepts a Hash to establish the connection or
+  # a String referring to an entry in config/database.yml.
+  # 
+  # Will establish the Nearline database tables if they are absent.
+  # 
+  # Stomps on any ActiveRecord::Base.establish_connection you might
+  # have already established.
+  # 
+  # === Examples
+  # Nearline.connect!({:adapter => 'sqlite3', :database => 'data/sqlite.db'})
+  # 
+  # Nearline.connect! 'production'
+  # 
+  def connect!(config="development")
+    if (config.class.to_s == 'String')
+      ActiveRecord::Base.establish_connection(YAML.load_file("config/database.yml")[config])
+    end
+    
+    if (config.class.to_s == 'Hash')
+      ActiveRecord::Base.establish_connection(config)      
+    end
+    
+    unless Nearline::Models::Block.table_exists?
+      Nearline::Models.generate_schema
+    end
+    Nearline::Models::Block.connected?
+  end
+  
+  # Establishes a connection only to the Nearline ActiveDirectory models
+  # 
+  # Will not change the ActiveRecord::Base connection
+  # 
+  # Will not establish Nearline tables in the database
+  # 
+  # Accepts a Hash to establish the connection or
+  # a String referring to an entry in config/database.yml.
+  # === Examples
+  # Nearline.connect({:adapter => 'sqlite3', :database => 'data/sqlite.db'})
+  # 
+  # Nearline.connect 'production'
+  # 
+  def connect(config="development")
+    # These are the ActiveRecord models in place
+    # Each one needs an explicit establish_connection
+    # if you don't want it running though ActiveRecord::Base  
+    models = [
+      Nearline::Models::ArchivedFile,
+      Nearline::Models::Block,
+      Nearline::Models::FileContent,
+      Nearline::Models::Manifest,
+      Nearline::Models::Sequence,
+      Nearline::Models::Log
+    ]
+    if (config.class.to_s == 'String')
+      hash = YAML.load_file("config/database.yml")[config]
+    else
+      hash = config
+    end
+    
+    models.each do |m|
+      m.establish_connection(hash)
+    end
+    Nearline::Models::Block.connected?
+  end
+  
+  # Performs a backup labeled for system_name,
+  # Recursing through an array of backup_paths,
+  # Excluding any path matching any of the regular
+  # expressions in the backup_exclusions array.
+  # 
+  # Expects the Nearline database connection has already
+  # been established
+  # 
+  # Returns a Manifest for the backup
+  def backup(system_name, backup_paths,backup_exclusions= [])
+    Nearline::Models::Manifest.backup(system_name, backup_paths, backup_exclusions)
+  end
+  
+  # Restore all missing files from the latest backup
+  # for system_name
+  # 
+  # All updated or existing files are left alone
+  # 
+  # Expects the Nearline database connection has already
+  # been established
+  # 
+  # Returns an Array of paths restored
+  def restore(system_name)
+    Nearline::Models::Manifest.restore_all_missing(system_name)
+  end
+  
+end

data/lib/nearline/schema.rb

@@ -0,0 +1,90 @@
+module Nearline
+  module Models
+
+    module_function
+    
+    def destroy_schema
+      ActiveRecord::Schema.define do
+        drop_table :blocks
+        drop_table :file_contents
+        drop_table :sequences
+        drop_table :archived_files
+        drop_table :manifests
+        drop_table :archived_files_manifests
+        drop_table :logs
+      end
+    end
+    
+    def empty_schema
+      Nearline::Models::Manifest.destroy_all
+    end
+    
+    def generate_schema
+      ActiveRecord::Schema.define do
+
+        create_table :blocks do |t|
+          t.column :fingerprint, :string, :length => 40, :null => false
+          t.column :bulk_content, :binary
+          t.column :is_compressed, :boolean, :default => false
+        end
+        
+        add_index :blocks, [:fingerprint], :unique => true
+
+        create_table :file_contents do |t|
+          t.column :fingerprint, :string, :length => 40
+          t.column :file_size, :integer, :default => 0
+        end
+
+        create_table :sequences do |t|
+          t.column :sequence, :integer, :null => false
+          t.column :block_id, :integer, :null => false
+          t.column :file_content_id, :integer, :null => false
+        end
+        
+        add_index :sequences, [:sequence, :file_content_id], :unique => true,
+          :name => "sequence_jn_index"
+
+        create_table :archived_files do |t|
+          t.column :system_name, :string, :null => false
+          t.column :path, :text, :null => false
+          t.column :path_hash, :string, :null => false, :length => 40
+          t.column :file_content_id, :integer
+          t.column :uid, :integer, :default => -1
+          t.column :gid, :integer, :default => -1
+          t.column :mtime, :integer, :default => 0
+          t.column :mode, :integer, :default => 33206  # "chmod 100666"
+          t.column :is_directory, :boolean
+        end
+        
+        add_index :archived_files, [:path_hash], :unique => true
+                        
+        # Manifests are the reference to a collection of archived files
+        create_table :manifests do |t|
+          t.column :system_name, :string
+          t.column :created_at, :datetime
+          t.column :completed_at, :datetime
+        end
+        
+        # Joins archived files across manifests so file references may be recycled
+        create_table :archived_files_manifests, :id => false do |t|
+          t.column :archived_file_id, :integer
+          t.column :manifest_id, :integer
+        end
+
+        add_index :archived_files_manifests,
+          [:archived_file_id, :manifest_id], {
+          :unique => true,
+          :name => "manifest_jn_index"
+        }
+        
+        # Keeps a record of problems during backup related to a manifest
+        create_table :logs do |t|
+          t.column :manifest_id, :integer, :null => false
+          t.column :message, :text
+          t.column :created_at, :datetime
+        end
+      end
+    end
+
+  end
+end

data/lib/nearline.rb

@@ -0,0 +1,16 @@
+#!/usr/bin/env ruby
+
+# Nearline
+# Copyright (C) 2008 Robert J. Osborne
+
+require 'nearline/module_methods'
+
+# ActiveRecord database definitions
+require 'nearline/schema'
+
+# ActiveRecord models
+require 'nearline/block'
+require 'nearline/file_content'
+require 'nearline/archived_file'
+require 'nearline/log'
+require 'nearline/manifest'

data/tasks/clean.rake

@@ -0,0 +1,3 @@
+require 'rake/clean'
+
+CLEAN.include("data","html","pkg","coverage", "temp")

data/tasks/gemspec.rake

@@ -0,0 +1,25 @@
+require 'rake'
+require 'rake/gempackagetask'
+
+SPEC = Gem::Specification.new do |s|
+  s.name = "nearline"
+  s.version = "0.0.1"
+  s.author = "Robert J. Osborne"
+  s.email = "rjo1970@gmail.com"
+  s.summary = "Nearline is a near-line backup and recovery solution"
+  s.description = %{
+    Nearline is a library to make managing near-line file repositories
+    simple and eleant in pure Ruby.
+  }
+  s.rubyforge_project = "nearline"
+  s.files = FileList["{tests,lib,doc,tasks}/**/*"].exclude("rdoc").to_a
+  s.add_dependency("activerecord", '>= 2.0.2')
+  s.require_path = "lib"
+  s.autorequire = "nearline"
+  s.test_file = "test/nearline_test.rb"
+  s.has_rdoc = true
+end
+
+Rake::GemPackageTask.new(SPEC) do |pkg|
+  pkg.need_tar = true
+end

data/tasks/rcov.rake

@@ -0,0 +1,13 @@
+task :rcov => [:clean] do
+  begin
+    require 'rcov/rcovtask'
+  
+    Rcov::RcovTask.new do |t|
+      t.libs << "test"
+      t.rcov_opts = ['--text-report']
+      t.test_files = FileList['test/nearline_test.rb']
+      t.verbose = true
+    end
+  rescue LoadError => no_rcov
+  end
+end

data/tasks/test.rake

@@ -0,0 +1,4 @@
+desc "Test nearline"
+task :test => [:clean] do
+  require 'test/nearline_test'
+end

data/test/nearline_test.rb

@@ -0,0 +1,17 @@
+$:.unshift File.join(File.dirname(__FILE__), "..", "test")
+# This is the suite of tests to run against Nearline
+require 'test/unit'
+
+require 'utilities'
+
+$data_path = File.join(File.dirname(__FILE__), "..", "data")
+unless File.exist?($data_path)
+  FileUtils.mkdir $data_path 
+end
+
+require 'schema_test'
+require 'block_test'
+require 'nearline_module_test'
+require 'file_content_test'
+require 'archived_file_test'
+require 'manifest_test'

metadata

@@ -0,0 +1,66 @@
+--- !ruby/object:Gem::Specification 
+rubygems_version: 0.9.4
+specification_version: 1
+name: nearline
+version: !ruby/object:Gem::Version 
+  version: 0.0.1
+date: 2008-04-03 00:00:00 -04:00
+summary: Nearline is a near-line backup and recovery solution
+require_paths: 
+- lib
+email: rjo1970@gmail.com
+homepage: 
+rubyforge_project: nearline
+description: Nearline is a library to make managing near-line file repositories simple and eleant in pure Ruby.
+autorequire: nearline
+default_executable: 
+bindir: bin
+has_rdoc: true
+required_ruby_version: !ruby/object:Gem::Version::Requirement 
+  requirements: 
+  - - ">"
+    - !ruby/object:Gem::Version 
+      version: 0.0.0
+  version: 
+platform: ruby
+signing_key: 
+cert_chain: 
+post_install_message: 
+authors: 
+- Robert J. Osborne
+files: 
+- lib/nearline
+- lib/nearline/archived_file.rb
+- lib/nearline/block.rb
+- lib/nearline/file_content.rb
+- lib/nearline/log.rb
+- lib/nearline/manifest.rb
+- lib/nearline/module_methods.rb
+- lib/nearline/schema.rb
+- lib/nearline.rb
+- tasks/clean.rake
+- tasks/gemspec.rake
+- tasks/rcov.rake
+- tasks/test.rake
+test_files: 
+- test/nearline_test.rb
+rdoc_options: []
+
+extra_rdoc_files: []
+
+executables: []
+
+extensions: []
+
+requirements: []
+
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: activerecord
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Version::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: 2.0.2
+    version: