nearline 0.0.6 → 0.1.0
Sign up to get free protection for your applications and to get access to all the features.
- data/lib/nearline.rb +1 -1
- data/lib/nearline/archived_file.rb +31 -24
- data/lib/nearline/block.rb +17 -14
- data/lib/nearline/file_information.rb +2 -1
- data/lib/nearline/file_sequencer.rb +10 -8
- data/lib/nearline/module_methods.rb +3 -6
- data/lib/nearline/schema.rb +1 -1
- data/tasks/gemspec.rake +2 -2
- metadata +16 -14
data/lib/nearline.rb
CHANGED
@@ -21,32 +21,39 @@ module Nearline
|
|
21
21
|
file_information.archived_file_parameters
|
22
22
|
)
|
23
23
|
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
24
|
+
begin
|
25
|
+
# Find a new link
|
26
|
+
if(file_information.ftype == "link")
|
27
|
+
archived_file.ftype_data = file_information.link_target
|
28
|
+
archived_file.save!
|
29
|
+
file_information.manifest.archived_files << archived_file
|
30
|
+
return archived_file
|
31
|
+
end
|
31
32
|
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
33
|
+
# Find a new directory
|
34
|
+
if (file_information.ftype=="directory")
|
35
|
+
archived_file.save!
|
36
|
+
file_information.manifest.archived_files << archived_file
|
37
|
+
return archived_file
|
38
|
+
end
|
38
39
|
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
40
|
+
# Find a new file that needs persisted
|
41
|
+
archived_file.file_content.file_size =
|
42
|
+
[file_information.stat.size].pack('Q').unpack('L').first # HACK for Windows
|
43
|
+
archived_file = archived_file.persist(file_information.manifest)
|
44
|
+
unless archived_file.nil? || archived_file.frozen?
|
45
|
+
archived_file.save!
|
46
|
+
file_information.manifest.archived_files << archived_file
|
47
|
+
end
|
48
|
+
return archived_file
|
49
|
+
# TODO: block devices, ...?
|
50
|
+
|
51
|
+
rescue
|
52
|
+
msg = "#{file_information.file_path} failed to persist"
|
53
|
+
puts msg
|
54
|
+
file_information.manifest.add_log(msg)
|
55
|
+
return nil
|
46
56
|
end
|
47
|
-
archived_file
|
48
|
-
|
49
|
-
# TODO: block devices, ...?
|
50
57
|
end
|
51
58
|
|
52
59
|
def restore(*args)
|
@@ -56,7 +63,7 @@ module Nearline
|
|
56
63
|
guarantee_path
|
57
64
|
File.symlink(self.ftype_data,
|
58
65
|
option_override(:path))
|
59
|
-
#restore_metadata
|
66
|
+
# do not restore_metadata on a link
|
60
67
|
return
|
61
68
|
end
|
62
69
|
|
data/lib/nearline/block.rb
CHANGED
@@ -9,11 +9,12 @@ module Nearline
|
|
9
9
|
# content access.
|
10
10
|
class Block < ActiveRecord::Base
|
11
11
|
require "zlib"
|
12
|
+
require "base64"
|
12
13
|
|
13
14
|
has_many :sequences
|
14
15
|
|
15
16
|
# Maximum block size in bytes
|
16
|
-
@@max_block_size = (
|
17
|
+
@@max_block_size = (32 * 1024)-1
|
17
18
|
cattr_accessor :max_block_size
|
18
19
|
|
19
20
|
# Level of block compression attempted
|
@@ -23,36 +24,38 @@ module Nearline
|
|
23
24
|
|
24
25
|
def attempt_compression
|
25
26
|
return if (self.is_compressed || @@block_compression_level == 0)
|
26
|
-
candidate_content = Zlib::Deflate.deflate(
|
27
|
-
self.bulk_content,
|
27
|
+
candidate_content = Base64.encode64(Zlib::Deflate.deflate(
|
28
|
+
Base64.decode64(self.bulk_content),
|
28
29
|
@@block_compression_level
|
29
|
-
)
|
30
|
+
))
|
30
31
|
if candidate_content.length < self.bulk_content.length
|
31
32
|
self.is_compressed = true
|
32
33
|
self.bulk_content = candidate_content
|
33
34
|
end
|
34
35
|
end
|
35
|
-
|
36
|
-
|
37
|
-
self.fingerprint = Digest::SHA1.hexdigest(content)
|
38
|
-
end
|
39
|
-
|
36
|
+
|
37
|
+
# This is the actual content in unencoded, uncompressed form
|
40
38
|
def content
|
41
39
|
if !@content.nil?
|
42
40
|
return @content
|
43
41
|
end
|
44
42
|
if (self.is_compressed)
|
45
|
-
return @content = Zlib::Inflate.inflate(self.bulk_content)
|
43
|
+
return @content = Zlib::Inflate.inflate(Base64.decode64(self.bulk_content))
|
46
44
|
end
|
47
|
-
@content = self.bulk_content
|
45
|
+
@content = Base64.decode64(self.bulk_content)
|
48
46
|
end
|
49
|
-
|
47
|
+
|
48
|
+
def content=(content)
|
49
|
+
self.fingerprint = Digest::SHA1.hexdigest(content)
|
50
|
+
self.bulk_content = Base64.encode64(content)
|
51
|
+
end
|
52
|
+
|
50
53
|
def orphan_check
|
51
54
|
if self.sequences.size == 0
|
52
55
|
self.destroy
|
53
56
|
end
|
54
57
|
end
|
55
|
-
|
58
|
+
|
56
59
|
end
|
57
60
|
end
|
58
|
-
end
|
61
|
+
end
|
@@ -15,10 +15,11 @@ module Nearline
|
|
15
15
|
if (@file_content.id.nil?)
|
16
16
|
@file_content.save!
|
17
17
|
end
|
18
|
-
@s = []
|
19
|
-
@b = []
|
18
|
+
@s = [] # sequence array
|
19
|
+
@b = [] # blocks read and fingerprinted
|
20
20
|
@file_size = 0
|
21
21
|
@offset = 0
|
22
|
+
# TODO: split out SHA1 into its own file read
|
22
23
|
@whole_file_hash = Digest::SHA1.new
|
23
24
|
end
|
24
25
|
|
@@ -109,17 +110,18 @@ module Nearline
|
|
109
110
|
while (!@io.eof && count < @@max_blocks)
|
110
111
|
count += 1
|
111
112
|
|
112
|
-
#
|
113
|
+
# Read block to buffer
|
113
114
|
buffer = @io.read(Block.max_block_size)
|
114
|
-
|
115
|
+
|
115
116
|
@file_size += buffer.size
|
116
117
|
|
117
|
-
#
|
118
|
-
blk = Block.new(:
|
118
|
+
# Create Block
|
119
|
+
blk = Block.new(:content => buffer)
|
120
|
+
|
121
|
+
# SHA1 update
|
119
122
|
@whole_file_hash.update(buffer)
|
120
123
|
|
121
|
-
#
|
122
|
-
blk.calculate_fingerprint
|
124
|
+
# Add block to block array
|
123
125
|
@b << blk
|
124
126
|
end
|
125
127
|
end
|
@@ -2,9 +2,9 @@ module Nearline
|
|
2
2
|
module_function
|
3
3
|
|
4
4
|
# Version of the software
|
5
|
-
VERSION = "0.0
|
5
|
+
VERSION = "0.1.0"
|
6
6
|
# Last version that changed the database structure
|
7
|
-
DB_VERSION = "0.0
|
7
|
+
DB_VERSION = "0.1.0"
|
8
8
|
|
9
9
|
# Array of every Nearline Model using an ActiveRecord connection
|
10
10
|
AR_MODELS = Nearline::Models.constants.map do |m|
|
@@ -23,7 +23,6 @@ module Nearline
|
|
23
23
|
# Stomps on any ActiveRecord::Base.establish_connection you might
|
24
24
|
# have already established.
|
25
25
|
#
|
26
|
-
# ***NOTE: MYSQL is the only recommended database at this time.***
|
27
26
|
#
|
28
27
|
# === Examples
|
29
28
|
# Nearline.connect!({:adapter => 'sqlite3', :database => 'data/sqlite.db'})
|
@@ -54,8 +53,6 @@ module Nearline
|
|
54
53
|
# Accepts a Hash to establish the connection or
|
55
54
|
# a String referring to an entry in config/database.yml.
|
56
55
|
#
|
57
|
-
# ***NOTE: MYSQL is the only recommended database at this time.***
|
58
|
-
#
|
59
56
|
# === Examples
|
60
57
|
# Nearline.connect({:adapter => 'sqlite3', :database => 'data/sqlite.db'})
|
61
58
|
#
|
@@ -94,7 +91,7 @@ module Nearline
|
|
94
91
|
#
|
95
92
|
# Backup my laptop, recurse /home/me and /var/svn
|
96
93
|
#
|
97
|
-
# Nearline.backup('my_laptop', ['/home/me', '/var/svn']
|
94
|
+
# Nearline.backup('my_laptop', ['/home/me', '/var/svn'])
|
98
95
|
#
|
99
96
|
def backup(system_name, backup_paths,backup_exclusions= [])
|
100
97
|
raise_failing_version_check
|
data/lib/nearline/schema.rb
CHANGED
data/tasks/gemspec.rake
CHANGED
@@ -12,8 +12,8 @@ SPEC = Gem::Specification.new do |s|
|
|
12
12
|
s.homepage = "http://rubyforge.org/projects/nearline"
|
13
13
|
s.summary = "Nearline is a near-line backup and recovery solution"
|
14
14
|
s.description = %{
|
15
|
-
Nearline is a library to make
|
16
|
-
simple and elegant in pure Ruby.
|
15
|
+
Nearline is a library to make database-backed backup file
|
16
|
+
repositories simple and elegant in pure Ruby.
|
17
17
|
}
|
18
18
|
s.rubyforge_project = "nearline"
|
19
19
|
s.files = FileList["{tests,lib,doc,tasks}/**/*"].exclude("rdoc").to_a
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: nearline
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0
|
4
|
+
version: 0.1.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Robert J. Osborne
|
@@ -9,11 +9,12 @@ autorequire:
|
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
11
|
|
12
|
-
date:
|
12
|
+
date: 2009-12-04 00:00:00 -05:00
|
13
13
|
default_executable:
|
14
14
|
dependencies:
|
15
15
|
- !ruby/object:Gem::Dependency
|
16
16
|
name: activerecord
|
17
|
+
type: :runtime
|
17
18
|
version_requirement:
|
18
19
|
version_requirements: !ruby/object:Gem::Requirement
|
19
20
|
requirements:
|
@@ -21,7 +22,7 @@ dependencies:
|
|
21
22
|
- !ruby/object:Gem::Version
|
22
23
|
version: 2.0.2
|
23
24
|
version:
|
24
|
-
description: Nearline is a library to make
|
25
|
+
description: "\n Nearline is a library to make database-backed backup file\n repositories simple and elegant in pure Ruby.\n "
|
25
26
|
email: rjo1970@gmail.com
|
26
27
|
executables: []
|
27
28
|
|
@@ -30,24 +31,25 @@ extensions: []
|
|
30
31
|
extra_rdoc_files: []
|
31
32
|
|
32
33
|
files:
|
33
|
-
- lib/nearline
|
34
|
-
- lib/nearline/schema.rb
|
35
|
-
- lib/nearline/system.rb
|
36
|
-
- lib/nearline/module_methods.rb
|
37
|
-
- lib/nearline/file_sequencer.rb
|
38
|
-
- lib/nearline/block.rb
|
39
|
-
- lib/nearline/log.rb
|
40
34
|
- lib/nearline/archived_file.rb
|
35
|
+
- lib/nearline/block.rb
|
41
36
|
- lib/nearline/file_content.rb
|
42
37
|
- lib/nearline/file_information.rb
|
38
|
+
- lib/nearline/file_sequencer.rb
|
39
|
+
- lib/nearline/log.rb
|
43
40
|
- lib/nearline/manifest.rb
|
41
|
+
- lib/nearline/module_methods.rb
|
42
|
+
- lib/nearline/schema.rb
|
43
|
+
- lib/nearline/system.rb
|
44
44
|
- lib/nearline.rb
|
45
|
-
- tasks/gemspec.rake
|
46
45
|
- tasks/clean.rake
|
47
|
-
- tasks/
|
46
|
+
- tasks/gemspec.rake
|
48
47
|
- tasks/rcov.rake
|
48
|
+
- tasks/test.rake
|
49
49
|
has_rdoc: true
|
50
50
|
homepage: http://rubyforge.org/projects/nearline
|
51
|
+
licenses: []
|
52
|
+
|
51
53
|
post_install_message:
|
52
54
|
rdoc_options: []
|
53
55
|
|
@@ -68,9 +70,9 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
68
70
|
requirements: []
|
69
71
|
|
70
72
|
rubyforge_project: nearline
|
71
|
-
rubygems_version: 1.
|
73
|
+
rubygems_version: 1.3.5
|
72
74
|
signing_key:
|
73
|
-
specification_version:
|
75
|
+
specification_version: 3
|
74
76
|
summary: Nearline is a near-line backup and recovery solution
|
75
77
|
test_files:
|
76
78
|
- test/nearline_test.rb
|