nearline 0.0.2 → 0.0.3
Sign up to get free protection for your applications and to get access to all the features.
- data/lib/nearline/archived_file.rb +19 -18
- data/lib/nearline/block.rb +6 -3
- data/lib/nearline/file_content.rb +8 -7
- data/lib/nearline/manifest.rb +20 -2
- data/lib/nearline/module_methods.rb +32 -17
- data/lib/nearline.rb +4 -2
- data/tasks/gemspec.rake +1 -1
- metadata +2 -2
@@ -4,12 +4,13 @@ module Nearline
|
|
4
4
|
# Represents file metadata and possible related FileContent
|
5
5
|
# for a single file on a single system
|
6
6
|
class ArchivedFile < ActiveRecord::Base
|
7
|
+
require 'fileutils'
|
8
|
+
|
7
9
|
belongs_to :file_content
|
8
10
|
has_and_belongs_to_many :manifests
|
9
11
|
|
10
|
-
def self.create_for(
|
11
|
-
|
12
|
-
file_information = FileInformation.new(system_name, file_path, manifest)
|
12
|
+
def self.create_for(file_path, manifest)
|
13
|
+
file_information = FileInformation.new(file_path, manifest)
|
13
14
|
|
14
15
|
# The path doesn't actually exist and fails a File.stat
|
15
16
|
return nil if file_information.path_hash.nil?
|
@@ -41,28 +42,29 @@ module Nearline
|
|
41
42
|
|
42
43
|
class FileInformation
|
43
44
|
attr_reader :path_hash, :stat, :is_directory, :archived_file_parameters
|
44
|
-
def initialize(
|
45
|
+
def initialize(file_path, manifest)
|
45
46
|
@manifest = manifest
|
46
|
-
@
|
47
|
+
@file_path = file_path
|
48
|
+
@stat = read_stat
|
47
49
|
@is_directory = File.directory?(file_path)
|
48
|
-
@path_hash = generate_path_hash
|
49
|
-
@archived_file_parameters = build_parameters
|
50
|
+
@path_hash = generate_path_hash
|
51
|
+
@archived_file_parameters = build_parameters
|
50
52
|
end
|
51
53
|
|
52
|
-
def read_stat
|
54
|
+
def read_stat
|
53
55
|
stat = nil
|
54
56
|
begin
|
55
|
-
stat = File.stat(file_path)
|
57
|
+
stat = File.stat(@file_path)
|
56
58
|
rescue
|
57
|
-
@manifest.add_log("File not found on stat: #{file_path}")
|
59
|
+
@manifest.add_log("File not found on stat: #{@file_path}")
|
58
60
|
end
|
59
61
|
stat
|
60
62
|
end
|
61
63
|
|
62
|
-
def generate_path_hash
|
64
|
+
def generate_path_hash
|
63
65
|
return nil if @stat.nil?
|
64
|
-
target = [system_name,
|
65
|
-
file_path,
|
66
|
+
target = [@manifest.system_name,
|
67
|
+
@file_path,
|
66
68
|
@stat.uid,
|
67
69
|
@stat.gid,
|
68
70
|
@stat.mtime.to_i,
|
@@ -75,11 +77,11 @@ module Nearline
|
|
75
77
|
return nil
|
76
78
|
end
|
77
79
|
|
78
|
-
def build_parameters
|
80
|
+
def build_parameters
|
79
81
|
return nil if @stat.nil?
|
80
82
|
{
|
81
|
-
:system_name => system_name,
|
82
|
-
:path => file_path,
|
83
|
+
:system_name => @manifest.system_name,
|
84
|
+
:path => @file_path,
|
83
85
|
:path_hash => @path_hash,
|
84
86
|
:file_content => file_content_entry_for_files_only,
|
85
87
|
:uid => @stat.uid,
|
@@ -177,8 +179,7 @@ module Nearline
|
|
177
179
|
io.read(Block::MAX_SIZE, buffer)
|
178
180
|
file_size += buffer.size
|
179
181
|
whole_file_hash.update(buffer)
|
180
|
-
|
181
|
-
sequencer.preserve_block(block)
|
182
|
+
sequencer.preserve_content(buffer)
|
182
183
|
end
|
183
184
|
end
|
184
185
|
return file_size
|
data/lib/nearline/block.rb
CHANGED
@@ -29,12 +29,15 @@ module Nearline
|
|
29
29
|
end
|
30
30
|
|
31
31
|
def content
|
32
|
+
if !@content.nil?
|
33
|
+
return @content
|
34
|
+
end
|
32
35
|
if (self.is_compressed)
|
33
|
-
return Zlib::Inflate.inflate(self.bulk_content)
|
36
|
+
return @content = Zlib::Inflate.inflate(self.bulk_content)
|
34
37
|
end
|
35
|
-
self.bulk_content
|
38
|
+
@content = self.bulk_content
|
36
39
|
end
|
37
|
-
|
40
|
+
|
38
41
|
def self.for_content(x, old_block = nil)
|
39
42
|
unless old_block.nil?
|
40
43
|
if x == old_block.content
|
@@ -34,7 +34,7 @@ module Nearline
|
|
34
34
|
sequences.each do |seq|
|
35
35
|
block = Block.find(seq.block_id)
|
36
36
|
io.write(block.content)
|
37
|
-
|
37
|
+
end
|
38
38
|
end
|
39
39
|
|
40
40
|
def verified?
|
@@ -70,18 +70,19 @@ module Nearline
|
|
70
70
|
@file_content = file_content
|
71
71
|
end
|
72
72
|
|
73
|
-
def
|
73
|
+
def preserve_content(content)
|
74
74
|
@inc += 1
|
75
|
+
block = Block.for_content(content)
|
75
76
|
sequence = Sequence.new(
|
76
77
|
:sequence => @inc,
|
77
78
|
:file_content_id => @file_content.id,
|
78
79
|
:block_id => block.id
|
79
80
|
)
|
80
81
|
sequence.save!
|
81
|
-
sequence
|
82
|
+
sequence
|
82
83
|
end
|
84
|
+
|
85
|
+
end
|
83
86
|
|
84
|
-
|
85
|
-
|
86
|
-
end
|
87
|
-
end
|
87
|
+
end
|
88
|
+
end
|
data/lib/nearline/manifest.rb
CHANGED
@@ -34,10 +34,11 @@ module Nearline
|
|
34
34
|
manifest.save!
|
35
35
|
|
36
36
|
FileFinder.recurse(backup_paths, backup_exclusions) do |file_name|
|
37
|
-
|
37
|
+
$stdout.write file_name + " "
|
38
|
+
af = ArchivedFile.create_for(file_name, manifest)
|
38
39
|
if (!af.nil?)
|
39
40
|
manifest.archived_files << af
|
40
|
-
$stdout.write "#{
|
41
|
+
$stdout.write "#{Time.at(af.mtime).asctime}"
|
41
42
|
if (!af.file_content.nil?)
|
42
43
|
$stdout.write" (#{af.file_content.file_size} bytes)"
|
43
44
|
end
|
@@ -100,6 +101,23 @@ module Nearline
|
|
100
101
|
end
|
101
102
|
end
|
102
103
|
|
104
|
+
def total_size
|
105
|
+
size = 0
|
106
|
+
archived_files.each do |af|
|
107
|
+
unless af.file_content.nil?
|
108
|
+
size += af.file_content.file_size.to_i
|
109
|
+
end
|
110
|
+
end
|
111
|
+
size
|
112
|
+
end
|
113
|
+
|
114
|
+
# A simple string reporting the performance of the manifest
|
115
|
+
def summary
|
116
|
+
completed = (completed_at.nil?) ? "DNF" : completed_at
|
117
|
+
"#{system_name}; started: #{created_at}; finished: #{completed}; " +
|
118
|
+
"#{archived_files.size} files; #{logs.size} Errors reported"
|
119
|
+
end
|
120
|
+
|
103
121
|
end
|
104
122
|
|
105
123
|
end
|
@@ -1,5 +1,15 @@
|
|
1
1
|
module Nearline
|
2
2
|
module_function
|
3
|
+
|
4
|
+
# Every model using an ActiveRecord connection
|
5
|
+
AR_MODELS = [
|
6
|
+
Nearline::Models::ArchivedFile,
|
7
|
+
Nearline::Models::Block,
|
8
|
+
Nearline::Models::FileContent,
|
9
|
+
Nearline::Models::Manifest,
|
10
|
+
Nearline::Models::Sequence,
|
11
|
+
Nearline::Models::Log
|
12
|
+
]
|
3
13
|
|
4
14
|
|
5
15
|
# Establishes the ActiveRecord connection
|
@@ -18,11 +28,13 @@ module Nearline
|
|
18
28
|
# Nearline.connect! 'production'
|
19
29
|
#
|
20
30
|
def connect!(config="development")
|
21
|
-
if (config.
|
22
|
-
ActiveRecord::Base.establish_connection(
|
31
|
+
if (config.is_a? String)
|
32
|
+
ActiveRecord::Base.establish_connection(
|
33
|
+
YAML.load_file("config/database.yml")[config]
|
34
|
+
)
|
23
35
|
end
|
24
36
|
|
25
|
-
if (config.
|
37
|
+
if (config.is_a? Hash)
|
26
38
|
ActiveRecord::Base.establish_connection(config)
|
27
39
|
end
|
28
40
|
|
@@ -49,39 +61,42 @@ module Nearline
|
|
49
61
|
# These are the ActiveRecord models in place
|
50
62
|
# Each one needs an explicit establish_connection
|
51
63
|
# if you don't want it running though ActiveRecord::Base
|
52
|
-
|
53
|
-
Nearline::Models::ArchivedFile,
|
54
|
-
Nearline::Models::Block,
|
55
|
-
Nearline::Models::FileContent,
|
56
|
-
Nearline::Models::Manifest,
|
57
|
-
Nearline::Models::Sequence,
|
58
|
-
Nearline::Models::Log
|
59
|
-
]
|
60
|
-
if (config.class.to_s == 'String')
|
64
|
+
if (config.is_a? String)
|
61
65
|
hash = YAML.load_file("config/database.yml")[config]
|
62
66
|
else
|
63
67
|
hash = config
|
64
68
|
end
|
65
69
|
|
66
|
-
|
70
|
+
AR_MODELS.each do |m|
|
67
71
|
m.establish_connection(hash)
|
68
72
|
end
|
69
73
|
Nearline::Models::Block.connected?
|
70
74
|
end
|
71
75
|
|
72
76
|
# Performs a backup labeled for system_name,
|
73
|
-
# Recursing through an array of backup_paths,
|
77
|
+
# Recursing through a single string or an array of backup_paths,
|
74
78
|
# Excluding any path matching any of the regular
|
75
|
-
# expressions in the backup_exclusions array.
|
79
|
+
# expressions in the backup_exclusions array or single string.
|
76
80
|
#
|
77
81
|
# Expects the Nearline database connection has already
|
78
82
|
# been established
|
79
83
|
#
|
80
84
|
# Returns a Manifest for the backup
|
81
85
|
def backup(system_name, backup_paths,backup_exclusions= [])
|
82
|
-
Nearline::Models::Manifest.backup(
|
86
|
+
Nearline::Models::Manifest.backup(
|
87
|
+
system_name,
|
88
|
+
string_to_array(backup_paths),
|
89
|
+
string_to_array(backup_exclusions)
|
90
|
+
)
|
83
91
|
end
|
84
|
-
|
92
|
+
|
93
|
+
def string_to_array(x)
|
94
|
+
if x.is_a? String
|
95
|
+
return [x]
|
96
|
+
end
|
97
|
+
x
|
98
|
+
end
|
99
|
+
|
85
100
|
# Restore all missing files from the latest backup
|
86
101
|
# for system_name
|
87
102
|
#
|
data/lib/nearline.rb
CHANGED
@@ -3,7 +3,6 @@
|
|
3
3
|
# Nearline
|
4
4
|
# Copyright (C) 2008 Robert J. Osborne
|
5
5
|
|
6
|
-
require 'nearline/module_methods'
|
7
6
|
|
8
7
|
# ActiveRecord database definitions
|
9
8
|
require 'nearline/schema'
|
@@ -13,4 +12,7 @@ require 'nearline/block'
|
|
13
12
|
require 'nearline/file_content'
|
14
13
|
require 'nearline/archived_file'
|
15
14
|
require 'nearline/log'
|
16
|
-
require 'nearline/manifest'
|
15
|
+
require 'nearline/manifest'
|
16
|
+
|
17
|
+
# Static methods on Nearline
|
18
|
+
require 'nearline/module_methods'
|
data/tasks/gemspec.rake
CHANGED
@@ -3,7 +3,7 @@ require 'rake/gempackagetask'
|
|
3
3
|
|
4
4
|
SPEC = Gem::Specification.new do |s|
|
5
5
|
s.name = "nearline"
|
6
|
-
s.version = "0.0.
|
6
|
+
s.version = "0.0.3"
|
7
7
|
s.author = "Robert J. Osborne"
|
8
8
|
s.email = "rjo1970@gmail.com"
|
9
9
|
s.summary = "Nearline is a near-line backup and recovery solution"
|
metadata
CHANGED
@@ -3,8 +3,8 @@ rubygems_version: 0.9.4
|
|
3
3
|
specification_version: 1
|
4
4
|
name: nearline
|
5
5
|
version: !ruby/object:Gem::Version
|
6
|
-
version: 0.0.
|
7
|
-
date: 2008-04-
|
6
|
+
version: 0.0.3
|
7
|
+
date: 2008-04-07 00:00:00 -04:00
|
8
8
|
summary: Nearline is a near-line backup and recovery solution
|
9
9
|
require_paths:
|
10
10
|
- lib
|