nearline 0.0.6 → 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,7 +1,7 @@
1
1
  #!/usr/bin/env ruby
2
2
 
3
3
  # Nearline
4
- # Copyright (C) 2008 Robert J. Osborne
4
+ # Copyright (C) 2008-2009 Robert J. Osborne
5
5
 
6
6
 
7
7
  # ActiveRecord database definitions
@@ -21,32 +21,39 @@ module Nearline
21
21
  file_information.archived_file_parameters
22
22
  )
23
23
 
24
- # Find a new link
25
- if(file_information.ftype == "link")
26
- archived_file.ftype_data = file_information.link_target
27
- archived_file.save!
28
- file_information.manifest.archived_files << archived_file
29
- return archived_file
30
- end
24
+ begin
25
+ # Find a new link
26
+ if(file_information.ftype == "link")
27
+ archived_file.ftype_data = file_information.link_target
28
+ archived_file.save!
29
+ file_information.manifest.archived_files << archived_file
30
+ return archived_file
31
+ end
31
32
 
32
- # Find a new directory
33
- if (file_information.ftype=="directory")
34
- archived_file.save!
35
- file_information.manifest.archived_files << archived_file
36
- return archived_file
37
- end
33
+ # Find a new directory
34
+ if (file_information.ftype=="directory")
35
+ archived_file.save!
36
+ file_information.manifest.archived_files << archived_file
37
+ return archived_file
38
+ end
38
39
 
39
- # Find a new file that needs persisted
40
- archived_file.file_content.file_size =
41
- [file_information.stat.size].pack('Q').unpack('L').first # HACK for Windows
42
- archived_file = archived_file.persist(file_information.manifest)
43
- unless archived_file.nil? || archived_file.frozen?
44
- archived_file.save!
45
- file_information.manifest.archived_files << archived_file
40
+ # Find a new file that needs persisted
41
+ archived_file.file_content.file_size =
42
+ [file_information.stat.size].pack('Q').unpack('L').first # HACK for Windows
43
+ archived_file = archived_file.persist(file_information.manifest)
44
+ unless archived_file.nil? || archived_file.frozen?
45
+ archived_file.save!
46
+ file_information.manifest.archived_files << archived_file
47
+ end
48
+ return archived_file
49
+ # TODO: block devices, ...?
50
+
51
+ rescue
52
+ msg = "#{file_information.file_path} failed to persist"
53
+ puts msg
54
+ file_information.manifest.add_log(msg)
55
+ return nil
46
56
  end
47
- archived_file
48
-
49
- # TODO: block devices, ...?
50
57
  end
51
58
 
52
59
  def restore(*args)
@@ -56,7 +63,7 @@ module Nearline
56
63
  guarantee_path
57
64
  File.symlink(self.ftype_data,
58
65
  option_override(:path))
59
- #restore_metadata
66
+ # do not restore_metadata on a link
60
67
  return
61
68
  end
62
69
 
@@ -9,11 +9,12 @@ module Nearline
9
9
  # content access.
10
10
  class Block < ActiveRecord::Base
11
11
  require "zlib"
12
+ require "base64"
12
13
 
13
14
  has_many :sequences
14
15
 
15
16
  # Maximum block size in bytes
16
- @@max_block_size = (64 * 1024)-1
17
+ @@max_block_size = (32 * 1024)-1
17
18
  cattr_accessor :max_block_size
18
19
 
19
20
  # Level of block compression attempted
@@ -23,36 +24,38 @@ module Nearline
23
24
 
24
25
  def attempt_compression
25
26
  return if (self.is_compressed || @@block_compression_level == 0)
26
- candidate_content = Zlib::Deflate.deflate(
27
- self.bulk_content,
27
+ candidate_content = Base64.encode64(Zlib::Deflate.deflate(
28
+ Base64.decode64(self.bulk_content),
28
29
  @@block_compression_level
29
- )
30
+ ))
30
31
  if candidate_content.length < self.bulk_content.length
31
32
  self.is_compressed = true
32
33
  self.bulk_content = candidate_content
33
34
  end
34
35
  end
35
-
36
- def calculate_fingerprint
37
- self.fingerprint = Digest::SHA1.hexdigest(content)
38
- end
39
-
36
+
37
+ # This is the actual content in unencoded, uncompressed form
40
38
  def content
41
39
  if !@content.nil?
42
40
  return @content
43
41
  end
44
42
  if (self.is_compressed)
45
- return @content = Zlib::Inflate.inflate(self.bulk_content)
43
+ return @content = Zlib::Inflate.inflate(Base64.decode64(self.bulk_content))
46
44
  end
47
- @content = self.bulk_content
45
+ @content = Base64.decode64(self.bulk_content)
48
46
  end
49
-
47
+
48
+ def content=(content)
49
+ self.fingerprint = Digest::SHA1.hexdigest(content)
50
+ self.bulk_content = Base64.encode64(content)
51
+ end
52
+
50
53
  def orphan_check
51
54
  if self.sequences.size == 0
52
55
  self.destroy
53
56
  end
54
57
  end
55
-
58
+
56
59
  end
57
60
  end
58
- end
61
+ end
@@ -24,7 +24,8 @@ module Nearline
24
24
  end
25
25
 
26
26
  def cleaned_file_path
27
- if RUBY_PLATFORM =~/win/
27
+ # TODO: handle \\system\$x style Windows file references.
28
+ if RUBY_PLATFORM =~/windows/
28
29
  return @file_path[2..-1]
29
30
  end
30
31
  @file_path
@@ -15,10 +15,11 @@ module Nearline
15
15
  if (@file_content.id.nil?)
16
16
  @file_content.save!
17
17
  end
18
- @s = []
19
- @b = []
18
+ @s = [] # sequence array
19
+ @b = [] # blocks read and fingerprinted
20
20
  @file_size = 0
21
21
  @offset = 0
22
+ # TODO: split out SHA1 into its own file read
22
23
  @whole_file_hash = Digest::SHA1.new
23
24
  end
24
25
 
@@ -109,17 +110,18 @@ module Nearline
109
110
  while (!@io.eof && count < @@max_blocks)
110
111
  count += 1
111
112
 
112
- # Move to Block
113
+ # Read block to buffer
113
114
  buffer = @io.read(Block.max_block_size)
114
-
115
+
115
116
  @file_size += buffer.size
116
117
 
117
- # Move to Block
118
- blk = Block.new(:bulk_content => buffer)
118
+ # Create Block
119
+ blk = Block.new(:content => buffer)
120
+
121
+ # SHA1 update
119
122
  @whole_file_hash.update(buffer)
120
123
 
121
- # Move to Block
122
- blk.calculate_fingerprint
124
+ # Add block to block array
123
125
  @b << blk
124
126
  end
125
127
  end
@@ -2,9 +2,9 @@ module Nearline
2
2
  module_function
3
3
 
4
4
  # Version of the software
5
- VERSION = "0.0.6"
5
+ VERSION = "0.1.0"
6
6
  # Last version that changed the database structure
7
- DB_VERSION = "0.0.6"
7
+ DB_VERSION = "0.1.0"
8
8
 
9
9
  # Array of every Nearline Model using an ActiveRecord connection
10
10
  AR_MODELS = Nearline::Models.constants.map do |m|
@@ -23,7 +23,6 @@ module Nearline
23
23
  # Stomps on any ActiveRecord::Base.establish_connection you might
24
24
  # have already established.
25
25
  #
26
- # ***NOTE: MYSQL is the only recommended database at this time.***
27
26
  #
28
27
  # === Examples
29
28
  # Nearline.connect!({:adapter => 'sqlite3', :database => 'data/sqlite.db'})
@@ -54,8 +53,6 @@ module Nearline
54
53
  # Accepts a Hash to establish the connection or
55
54
  # a String referring to an entry in config/database.yml.
56
55
  #
57
- # ***NOTE: MYSQL is the only recommended database at this time.***
58
- #
59
56
  # === Examples
60
57
  # Nearline.connect({:adapter => 'sqlite3', :database => 'data/sqlite.db'})
61
58
  #
@@ -94,7 +91,7 @@ module Nearline
94
91
  #
95
92
  # Backup my laptop, recurse /home/me and /var/svn
96
93
  #
97
- # Nearline.backup('my_laptop', ['/home/me', '/var/svn']
94
+ # Nearline.backup('my_laptop', ['/home/me', '/var/svn'])
98
95
  #
99
96
  def backup(system_name, backup_paths,backup_exclusions= [])
100
97
  raise_failing_version_check
@@ -26,7 +26,7 @@ module Nearline
26
26
 
27
27
  create_table :blocks do |t|
28
28
  t.column :fingerprint, :string, :length => 40, :null => false
29
- t.column :bulk_content, :binary
29
+ t.column :bulk_content, :text
30
30
  t.column :is_compressed, :boolean, :default => false
31
31
  end
32
32
 
@@ -12,8 +12,8 @@ SPEC = Gem::Specification.new do |s|
12
12
  s.homepage = "http://rubyforge.org/projects/nearline"
13
13
  s.summary = "Nearline is a near-line backup and recovery solution"
14
14
  s.description = %{
15
- Nearline is a library to make managing near-line file repositories
16
- simple and elegant in pure Ruby.
15
+ Nearline is a library to make database-backed backup file
16
+ repositories simple and elegant in pure Ruby.
17
17
  }
18
18
  s.rubyforge_project = "nearline"
19
19
  s.files = FileList["{tests,lib,doc,tasks}/**/*"].exclude("rdoc").to_a
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: nearline
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.0.6
4
+ version: 0.1.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Robert J. Osborne
@@ -9,11 +9,12 @@ autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
11
 
12
- date: 2008-07-20 00:00:00 -04:00
12
+ date: 2009-12-04 00:00:00 -05:00
13
13
  default_executable:
14
14
  dependencies:
15
15
  - !ruby/object:Gem::Dependency
16
16
  name: activerecord
17
+ type: :runtime
17
18
  version_requirement:
18
19
  version_requirements: !ruby/object:Gem::Requirement
19
20
  requirements:
@@ -21,7 +22,7 @@ dependencies:
21
22
  - !ruby/object:Gem::Version
22
23
  version: 2.0.2
23
24
  version:
24
- description: Nearline is a library to make managing near-line file repositories simple and elegant in pure Ruby.
25
+ description: "\n Nearline is a library to make database-backed backup file\n repositories simple and elegant in pure Ruby.\n "
25
26
  email: rjo1970@gmail.com
26
27
  executables: []
27
28
 
@@ -30,24 +31,25 @@ extensions: []
30
31
  extra_rdoc_files: []
31
32
 
32
33
  files:
33
- - lib/nearline
34
- - lib/nearline/schema.rb
35
- - lib/nearline/system.rb
36
- - lib/nearline/module_methods.rb
37
- - lib/nearline/file_sequencer.rb
38
- - lib/nearline/block.rb
39
- - lib/nearline/log.rb
40
34
  - lib/nearline/archived_file.rb
35
+ - lib/nearline/block.rb
41
36
  - lib/nearline/file_content.rb
42
37
  - lib/nearline/file_information.rb
38
+ - lib/nearline/file_sequencer.rb
39
+ - lib/nearline/log.rb
43
40
  - lib/nearline/manifest.rb
41
+ - lib/nearline/module_methods.rb
42
+ - lib/nearline/schema.rb
43
+ - lib/nearline/system.rb
44
44
  - lib/nearline.rb
45
- - tasks/gemspec.rake
46
45
  - tasks/clean.rake
47
- - tasks/test.rake
46
+ - tasks/gemspec.rake
48
47
  - tasks/rcov.rake
48
+ - tasks/test.rake
49
49
  has_rdoc: true
50
50
  homepage: http://rubyforge.org/projects/nearline
51
+ licenses: []
52
+
51
53
  post_install_message:
52
54
  rdoc_options: []
53
55
 
@@ -68,9 +70,9 @@ required_rubygems_version: !ruby/object:Gem::Requirement
68
70
  requirements: []
69
71
 
70
72
  rubyforge_project: nearline
71
- rubygems_version: 1.1.1
73
+ rubygems_version: 1.3.5
72
74
  signing_key:
73
- specification_version: 2
75
+ specification_version: 3
74
76
  summary: Nearline is a near-line backup and recovery solution
75
77
  test_files:
76
78
  - test/nearline_test.rb