nearline 0.0.6 → 0.1.0

data/lib/nearline.rb

@@ -1,7 +1,7 @@
 #!/usr/bin/env ruby
 
 # Nearline
-# Copyright (C) 2008 Robert J. Osborne
+# Copyright (C) 2008-2009 Robert J. Osborne
 
 
 # ActiveRecord database definitions

data/lib/nearline/archived_file.rb

@@ -21,32 +21,39 @@ module Nearline
           file_information.archived_file_parameters
         )
         
-        # Find a new link
-        if(file_information.ftype == "link")
-          archived_file.ftype_data = file_information.link_target
-          archived_file.save!
-          file_information.manifest.archived_files << archived_file
-          return archived_file          
-        end
+        begin
+          # Find a new link
+          if(file_information.ftype == "link")
+            archived_file.ftype_data = file_information.link_target
+            archived_file.save!
+            file_information.manifest.archived_files << archived_file
+            return archived_file          
+          end
 
-        # Find a new directory
-        if (file_information.ftype=="directory")
-          archived_file.save!
-          file_information.manifest.archived_files << archived_file
-          return archived_file
-        end
+          # Find a new directory
+          if (file_information.ftype=="directory")
+            archived_file.save!
+            file_information.manifest.archived_files << archived_file
+            return archived_file
+          end
         
-        # Find a new file that needs persisted        
-        archived_file.file_content.file_size = 
-          [file_information.stat.size].pack('Q').unpack('L').first # HACK for Windows
-        archived_file = archived_file.persist(file_information.manifest)
-        unless archived_file.nil? || archived_file.frozen?
-          archived_file.save!
-          file_information.manifest.archived_files << archived_file
+          # Find a new file that needs persisted        
+          archived_file.file_content.file_size = 
+            [file_information.stat.size].pack('Q').unpack('L').first # HACK for Windows
+          archived_file = archived_file.persist(file_information.manifest)
+          unless archived_file.nil? || archived_file.frozen?
+            archived_file.save!
+            file_information.manifest.archived_files << archived_file
+          end
+          return archived_file
+          # TODO: block devices, ...?
+
+        rescue
+          msg = "#{file_information.file_path} failed to persist"
+          puts msg
+          file_information.manifest.add_log(msg)
+          return nil
         end
-        archived_file
-        
-        # TODO: block devices, ...?
       end
             
       def restore(*args)
@@ -56,7 +63,7 @@ module Nearline
           guarantee_path
           File.symlink(self.ftype_data,
             option_override(:path))
-          #restore_metadata
+          # do not restore_metadata on a link
           return
         end
 

data/lib/nearline/block.rb

@@ -9,11 +9,12 @@ module Nearline
     # content access.
     class Block < ActiveRecord::Base
       require "zlib"
+      require "base64"
     
       has_many :sequences
       
       # Maximum block size in bytes
-      @@max_block_size = (64 * 1024)-1
+      @@max_block_size = (32 * 1024)-1
       cattr_accessor :max_block_size
       
       # Level of block compression attempted
@@ -23,36 +24,38 @@ module Nearline
       
       def attempt_compression
         return if (self.is_compressed || @@block_compression_level == 0)
-        candidate_content = Zlib::Deflate.deflate(
-          self.bulk_content,
+        candidate_content = Base64.encode64(Zlib::Deflate.deflate(
+          Base64.decode64(self.bulk_content),
           @@block_compression_level
-        )
+        ))
         if candidate_content.length < self.bulk_content.length
           self.is_compressed = true
           self.bulk_content = candidate_content
         end
       end
-      
-      def calculate_fingerprint
-        self.fingerprint = Digest::SHA1.hexdigest(content)        
-      end
-    
+          
+      #  This is the actual content in unencoded, uncompressed form
       def content
         if !@content.nil?
           return @content
         end
         if (self.is_compressed)
-          return @content = Zlib::Inflate.inflate(self.bulk_content)
+          return @content = Zlib::Inflate.inflate(Base64.decode64(self.bulk_content))
         end
-        @content = self.bulk_content
+        @content = Base64.decode64(self.bulk_content)
       end
-            
+      
+      def content=(content)
+        self.fingerprint = Digest::SHA1.hexdigest(content)
+        self.bulk_content = Base64.encode64(content)
+      end
+                  
       def orphan_check
         if self.sequences.size == 0
           self.destroy
         end
       end
-            
+
     end
   end
-end
+end

data/lib/nearline/file_information.rb

@@ -24,7 +24,8 @@ module Nearline
       end
       
       def cleaned_file_path
-        if RUBY_PLATFORM =~/win/
+        # TODO:  handle \\system\$x style Windows file references.
+        if RUBY_PLATFORM =~/windows/
           return @file_path[2..-1]
         end
         @file_path

data/lib/nearline/file_sequencer.rb

@@ -15,10 +15,11 @@ module Nearline
         if (@file_content.id.nil?)
           @file_content.save!
         end
-        @s = []
-        @b = []
+        @s = []  # sequence array
+        @b = []  # blocks read and fingerprinted
         @file_size = 0
         @offset = 0
+        # TODO: split out SHA1 into its own file read
         @whole_file_hash = Digest::SHA1.new
       end
       
@@ -109,17 +110,18 @@ module Nearline
         while (!@io.eof && count < @@max_blocks)
           count += 1
           
-          # Move to Block
+          # Read block to buffer
           buffer = @io.read(Block.max_block_size)
-          
+                    
           @file_size += buffer.size
           
-          # Move to Block
-          blk = Block.new(:bulk_content => buffer)
+          # Create Block
+          blk = Block.new(:content => buffer)
+          
+          # SHA1 update
           @whole_file_hash.update(buffer)
           
-          # Move to Block
-          blk.calculate_fingerprint
+          # Add block to block array
           @b << blk          
         end
       end

data/lib/nearline/module_methods.rb

@@ -2,9 +2,9 @@ module Nearline
   module_function
 
   # Version of the software
-  VERSION = "0.0.6"
+  VERSION = "0.1.0"
   # Last version that changed the database structure
-  DB_VERSION = "0.0.6"
+  DB_VERSION = "0.1.0"
   
   # Array of every Nearline Model using an ActiveRecord connection
   AR_MODELS = Nearline::Models.constants.map do |m|
@@ -23,7 +23,6 @@ module Nearline
   # Stomps on any ActiveRecord::Base.establish_connection you might
   # have already established.
   # 
-  # ***NOTE: MYSQL is the only recommended database at this time.***
   # 
   # === Examples
   # Nearline.connect!({:adapter => 'sqlite3', :database => 'data/sqlite.db'})
@@ -54,8 +53,6 @@ module Nearline
   # Accepts a Hash to establish the connection or
   # a String referring to an entry in config/database.yml.
   # 
-  # ***NOTE: MYSQL is the only recommended database at this time.***
-  # 
   # === Examples
   # Nearline.connect({:adapter => 'sqlite3', :database => 'data/sqlite.db'})
   # 
@@ -94,7 +91,7 @@ module Nearline
   #
   # Backup my laptop, recurse /home/me and /var/svn
   # 
-  # Nearline.backup('my_laptop', ['/home/me', '/var/svn']
+  # Nearline.backup('my_laptop', ['/home/me', '/var/svn'])
   #
   def backup(system_name, backup_paths,backup_exclusions= [])
     raise_failing_version_check

data/lib/nearline/schema.rb

@@ -26,7 +26,7 @@ module Nearline
 
         create_table :blocks do |t|
           t.column :fingerprint, :string, :length => 40, :null => false
-          t.column :bulk_content, :binary
+          t.column :bulk_content, :text
           t.column :is_compressed, :boolean, :default => false
         end
         

data/tasks/gemspec.rake

@@ -12,8 +12,8 @@ SPEC = Gem::Specification.new do |s|
   s.homepage = "http://rubyforge.org/projects/nearline"
   s.summary = "Nearline is a near-line backup and recovery solution"
   s.description = %{
-    Nearline is a library to make managing near-line file repositories
-    simple and elegant in pure Ruby.
+    Nearline is a library to make database-backed backup file
+    repositories simple and elegant in pure Ruby.
   }
   s.rubyforge_project = "nearline"
   s.files = FileList["{tests,lib,doc,tasks}/**/*"].exclude("rdoc").to_a

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: nearline
 version: !ruby/object:Gem::Version 
-  version: 0.0.6
+  version: 0.1.0
 platform: ruby
 authors: 
 - Robert J. Osborne
@@ -9,11 +9,12 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2008-07-20 00:00:00 -04:00
+date: 2009-12-04 00:00:00 -05:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
   name: activerecord
+  type: :runtime
   version_requirement: 
   version_requirements: !ruby/object:Gem::Requirement 
     requirements: 
@@ -21,7 +22,7 @@ dependencies:
       - !ruby/object:Gem::Version 
         version: 2.0.2
     version: 
-description: Nearline is a library to make managing near-line file repositories simple and elegant in pure Ruby.
+description: "\n    Nearline is a library to make database-backed backup file\n    repositories simple and elegant in pure Ruby.\n  "
 email: rjo1970@gmail.com
 executables: []
 
@@ -30,24 +31,25 @@ extensions: []
 extra_rdoc_files: []
 
 files: 
-- lib/nearline
-- lib/nearline/schema.rb
-- lib/nearline/system.rb
-- lib/nearline/module_methods.rb
-- lib/nearline/file_sequencer.rb
-- lib/nearline/block.rb
-- lib/nearline/log.rb
 - lib/nearline/archived_file.rb
+- lib/nearline/block.rb
 - lib/nearline/file_content.rb
 - lib/nearline/file_information.rb
+- lib/nearline/file_sequencer.rb
+- lib/nearline/log.rb
 - lib/nearline/manifest.rb
+- lib/nearline/module_methods.rb
+- lib/nearline/schema.rb
+- lib/nearline/system.rb
 - lib/nearline.rb
-- tasks/gemspec.rake
 - tasks/clean.rake
-- tasks/test.rake
+- tasks/gemspec.rake
 - tasks/rcov.rake
+- tasks/test.rake
 has_rdoc: true
 homepage: http://rubyforge.org/projects/nearline
+licenses: []
+
 post_install_message: 
 rdoc_options: []
 
@@ -68,9 +70,9 @@ required_rubygems_version: !ruby/object:Gem::Requirement
 requirements: []
 
 rubyforge_project: nearline
-rubygems_version: 1.1.1
+rubygems_version: 1.3.5
 signing_key: 
-specification_version: 2
+specification_version: 3
 summary: Nearline is a near-line backup and recovery solution
 test_files: 
 - test/nearline_test.rb