nearline 0.0.6 → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/lib/nearline.rb +1 -1
- data/lib/nearline/archived_file.rb +31 -24
- data/lib/nearline/block.rb +17 -14
- data/lib/nearline/file_information.rb +2 -1
- data/lib/nearline/file_sequencer.rb +10 -8
- data/lib/nearline/module_methods.rb +3 -6
- data/lib/nearline/schema.rb +1 -1
- data/tasks/gemspec.rake +2 -2
- metadata +16 -14
data/lib/nearline.rb
CHANGED
@@ -21,32 +21,39 @@ module Nearline
|
|
21
21
|
file_information.archived_file_parameters
|
22
22
|
)
|
23
23
|
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
24
|
+
begin
|
25
|
+
# Find a new link
|
26
|
+
if(file_information.ftype == "link")
|
27
|
+
archived_file.ftype_data = file_information.link_target
|
28
|
+
archived_file.save!
|
29
|
+
file_information.manifest.archived_files << archived_file
|
30
|
+
return archived_file
|
31
|
+
end
|
31
32
|
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
33
|
+
# Find a new directory
|
34
|
+
if (file_information.ftype=="directory")
|
35
|
+
archived_file.save!
|
36
|
+
file_information.manifest.archived_files << archived_file
|
37
|
+
return archived_file
|
38
|
+
end
|
38
39
|
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
40
|
+
# Find a new file that needs persisted
|
41
|
+
archived_file.file_content.file_size =
|
42
|
+
[file_information.stat.size].pack('Q').unpack('L').first # HACK for Windows
|
43
|
+
archived_file = archived_file.persist(file_information.manifest)
|
44
|
+
unless archived_file.nil? || archived_file.frozen?
|
45
|
+
archived_file.save!
|
46
|
+
file_information.manifest.archived_files << archived_file
|
47
|
+
end
|
48
|
+
return archived_file
|
49
|
+
# TODO: block devices, ...?
|
50
|
+
|
51
|
+
rescue
|
52
|
+
msg = "#{file_information.file_path} failed to persist"
|
53
|
+
puts msg
|
54
|
+
file_information.manifest.add_log(msg)
|
55
|
+
return nil
|
46
56
|
end
|
47
|
-
archived_file
|
48
|
-
|
49
|
-
# TODO: block devices, ...?
|
50
57
|
end
|
51
58
|
|
52
59
|
def restore(*args)
|
@@ -56,7 +63,7 @@ module Nearline
|
|
56
63
|
guarantee_path
|
57
64
|
File.symlink(self.ftype_data,
|
58
65
|
option_override(:path))
|
59
|
-
#restore_metadata
|
66
|
+
# do not restore_metadata on a link
|
60
67
|
return
|
61
68
|
end
|
62
69
|
|
data/lib/nearline/block.rb
CHANGED
@@ -9,11 +9,12 @@ module Nearline
|
|
9
9
|
# content access.
|
10
10
|
class Block < ActiveRecord::Base
|
11
11
|
require "zlib"
|
12
|
+
require "base64"
|
12
13
|
|
13
14
|
has_many :sequences
|
14
15
|
|
15
16
|
# Maximum block size in bytes
|
16
|
-
@@max_block_size = (
|
17
|
+
@@max_block_size = (32 * 1024)-1
|
17
18
|
cattr_accessor :max_block_size
|
18
19
|
|
19
20
|
# Level of block compression attempted
|
@@ -23,36 +24,38 @@ module Nearline
|
|
23
24
|
|
24
25
|
def attempt_compression
|
25
26
|
return if (self.is_compressed || @@block_compression_level == 0)
|
26
|
-
candidate_content = Zlib::Deflate.deflate(
|
27
|
-
self.bulk_content,
|
27
|
+
candidate_content = Base64.encode64(Zlib::Deflate.deflate(
|
28
|
+
Base64.decode64(self.bulk_content),
|
28
29
|
@@block_compression_level
|
29
|
-
)
|
30
|
+
))
|
30
31
|
if candidate_content.length < self.bulk_content.length
|
31
32
|
self.is_compressed = true
|
32
33
|
self.bulk_content = candidate_content
|
33
34
|
end
|
34
35
|
end
|
35
|
-
|
36
|
-
|
37
|
-
self.fingerprint = Digest::SHA1.hexdigest(content)
|
38
|
-
end
|
39
|
-
|
36
|
+
|
37
|
+
# This is the actual content in unencoded, uncompressed form
|
40
38
|
def content
|
41
39
|
if !@content.nil?
|
42
40
|
return @content
|
43
41
|
end
|
44
42
|
if (self.is_compressed)
|
45
|
-
return @content = Zlib::Inflate.inflate(self.bulk_content)
|
43
|
+
return @content = Zlib::Inflate.inflate(Base64.decode64(self.bulk_content))
|
46
44
|
end
|
47
|
-
@content = self.bulk_content
|
45
|
+
@content = Base64.decode64(self.bulk_content)
|
48
46
|
end
|
49
|
-
|
47
|
+
|
48
|
+
def content=(content)
|
49
|
+
self.fingerprint = Digest::SHA1.hexdigest(content)
|
50
|
+
self.bulk_content = Base64.encode64(content)
|
51
|
+
end
|
52
|
+
|
50
53
|
def orphan_check
|
51
54
|
if self.sequences.size == 0
|
52
55
|
self.destroy
|
53
56
|
end
|
54
57
|
end
|
55
|
-
|
58
|
+
|
56
59
|
end
|
57
60
|
end
|
58
|
-
end
|
61
|
+
end
|
@@ -15,10 +15,11 @@ module Nearline
|
|
15
15
|
if (@file_content.id.nil?)
|
16
16
|
@file_content.save!
|
17
17
|
end
|
18
|
-
@s = []
|
19
|
-
@b = []
|
18
|
+
@s = [] # sequence array
|
19
|
+
@b = [] # blocks read and fingerprinted
|
20
20
|
@file_size = 0
|
21
21
|
@offset = 0
|
22
|
+
# TODO: split out SHA1 into its own file read
|
22
23
|
@whole_file_hash = Digest::SHA1.new
|
23
24
|
end
|
24
25
|
|
@@ -109,17 +110,18 @@ module Nearline
|
|
109
110
|
while (!@io.eof && count < @@max_blocks)
|
110
111
|
count += 1
|
111
112
|
|
112
|
-
#
|
113
|
+
# Read block to buffer
|
113
114
|
buffer = @io.read(Block.max_block_size)
|
114
|
-
|
115
|
+
|
115
116
|
@file_size += buffer.size
|
116
117
|
|
117
|
-
#
|
118
|
-
blk = Block.new(:
|
118
|
+
# Create Block
|
119
|
+
blk = Block.new(:content => buffer)
|
120
|
+
|
121
|
+
# SHA1 update
|
119
122
|
@whole_file_hash.update(buffer)
|
120
123
|
|
121
|
-
#
|
122
|
-
blk.calculate_fingerprint
|
124
|
+
# Add block to block array
|
123
125
|
@b << blk
|
124
126
|
end
|
125
127
|
end
|
@@ -2,9 +2,9 @@ module Nearline
|
|
2
2
|
module_function
|
3
3
|
|
4
4
|
# Version of the software
|
5
|
-
VERSION = "0.0
|
5
|
+
VERSION = "0.1.0"
|
6
6
|
# Last version that changed the database structure
|
7
|
-
DB_VERSION = "0.0
|
7
|
+
DB_VERSION = "0.1.0"
|
8
8
|
|
9
9
|
# Array of every Nearline Model using an ActiveRecord connection
|
10
10
|
AR_MODELS = Nearline::Models.constants.map do |m|
|
@@ -23,7 +23,6 @@ module Nearline
|
|
23
23
|
# Stomps on any ActiveRecord::Base.establish_connection you might
|
24
24
|
# have already established.
|
25
25
|
#
|
26
|
-
# ***NOTE: MYSQL is the only recommended database at this time.***
|
27
26
|
#
|
28
27
|
# === Examples
|
29
28
|
# Nearline.connect!({:adapter => 'sqlite3', :database => 'data/sqlite.db'})
|
@@ -54,8 +53,6 @@ module Nearline
|
|
54
53
|
# Accepts a Hash to establish the connection or
|
55
54
|
# a String referring to an entry in config/database.yml.
|
56
55
|
#
|
57
|
-
# ***NOTE: MYSQL is the only recommended database at this time.***
|
58
|
-
#
|
59
56
|
# === Examples
|
60
57
|
# Nearline.connect({:adapter => 'sqlite3', :database => 'data/sqlite.db'})
|
61
58
|
#
|
@@ -94,7 +91,7 @@ module Nearline
|
|
94
91
|
#
|
95
92
|
# Backup my laptop, recurse /home/me and /var/svn
|
96
93
|
#
|
97
|
-
# Nearline.backup('my_laptop', ['/home/me', '/var/svn']
|
94
|
+
# Nearline.backup('my_laptop', ['/home/me', '/var/svn'])
|
98
95
|
#
|
99
96
|
def backup(system_name, backup_paths,backup_exclusions= [])
|
100
97
|
raise_failing_version_check
|
data/lib/nearline/schema.rb
CHANGED
data/tasks/gemspec.rake
CHANGED
@@ -12,8 +12,8 @@ SPEC = Gem::Specification.new do |s|
|
|
12
12
|
s.homepage = "http://rubyforge.org/projects/nearline"
|
13
13
|
s.summary = "Nearline is a near-line backup and recovery solution"
|
14
14
|
s.description = %{
|
15
|
-
Nearline is a library to make
|
16
|
-
simple and elegant in pure Ruby.
|
15
|
+
Nearline is a library to make database-backed backup file
|
16
|
+
repositories simple and elegant in pure Ruby.
|
17
17
|
}
|
18
18
|
s.rubyforge_project = "nearline"
|
19
19
|
s.files = FileList["{tests,lib,doc,tasks}/**/*"].exclude("rdoc").to_a
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: nearline
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0
|
4
|
+
version: 0.1.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Robert J. Osborne
|
@@ -9,11 +9,12 @@ autorequire:
|
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
11
|
|
12
|
-
date:
|
12
|
+
date: 2009-12-04 00:00:00 -05:00
|
13
13
|
default_executable:
|
14
14
|
dependencies:
|
15
15
|
- !ruby/object:Gem::Dependency
|
16
16
|
name: activerecord
|
17
|
+
type: :runtime
|
17
18
|
version_requirement:
|
18
19
|
version_requirements: !ruby/object:Gem::Requirement
|
19
20
|
requirements:
|
@@ -21,7 +22,7 @@ dependencies:
|
|
21
22
|
- !ruby/object:Gem::Version
|
22
23
|
version: 2.0.2
|
23
24
|
version:
|
24
|
-
description: Nearline is a library to make
|
25
|
+
description: "\n Nearline is a library to make database-backed backup file\n repositories simple and elegant in pure Ruby.\n "
|
25
26
|
email: rjo1970@gmail.com
|
26
27
|
executables: []
|
27
28
|
|
@@ -30,24 +31,25 @@ extensions: []
|
|
30
31
|
extra_rdoc_files: []
|
31
32
|
|
32
33
|
files:
|
33
|
-
- lib/nearline
|
34
|
-
- lib/nearline/schema.rb
|
35
|
-
- lib/nearline/system.rb
|
36
|
-
- lib/nearline/module_methods.rb
|
37
|
-
- lib/nearline/file_sequencer.rb
|
38
|
-
- lib/nearline/block.rb
|
39
|
-
- lib/nearline/log.rb
|
40
34
|
- lib/nearline/archived_file.rb
|
35
|
+
- lib/nearline/block.rb
|
41
36
|
- lib/nearline/file_content.rb
|
42
37
|
- lib/nearline/file_information.rb
|
38
|
+
- lib/nearline/file_sequencer.rb
|
39
|
+
- lib/nearline/log.rb
|
43
40
|
- lib/nearline/manifest.rb
|
41
|
+
- lib/nearline/module_methods.rb
|
42
|
+
- lib/nearline/schema.rb
|
43
|
+
- lib/nearline/system.rb
|
44
44
|
- lib/nearline.rb
|
45
|
-
- tasks/gemspec.rake
|
46
45
|
- tasks/clean.rake
|
47
|
-
- tasks/
|
46
|
+
- tasks/gemspec.rake
|
48
47
|
- tasks/rcov.rake
|
48
|
+
- tasks/test.rake
|
49
49
|
has_rdoc: true
|
50
50
|
homepage: http://rubyforge.org/projects/nearline
|
51
|
+
licenses: []
|
52
|
+
|
51
53
|
post_install_message:
|
52
54
|
rdoc_options: []
|
53
55
|
|
@@ -68,9 +70,9 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
68
70
|
requirements: []
|
69
71
|
|
70
72
|
rubyforge_project: nearline
|
71
|
-
rubygems_version: 1.
|
73
|
+
rubygems_version: 1.3.5
|
72
74
|
signing_key:
|
73
|
-
specification_version:
|
75
|
+
specification_version: 3
|
74
76
|
summary: Nearline is a near-line backup and recovery solution
|
75
77
|
test_files:
|
76
78
|
- test/nearline_test.rb
|