nearline 0.0.5 → 0.0.6
Sign up to get free protection for your applications and to get access to all the features.
- data/lib/nearline/archived_file.rb +31 -8
- data/lib/nearline/file_information.rb +88 -0
- data/lib/nearline/manifest.rb +12 -70
- data/lib/nearline/module_methods.rb +2 -2
- data/lib/nearline/schema.rb +2 -12
- data/lib/nearline.rb +1 -0
- data/test/nearline_test.rb +0 -1
- metadata +3 -2
@@ -16,13 +16,21 @@ module Nearline
|
|
16
16
|
# The path doesn't actually exist and fails a File.lstat
|
17
17
|
return nil if file_information.path_hash.nil?
|
18
18
|
|
19
|
-
# We need to create a record for
|
19
|
+
# We need to create a record for a link, directory or file
|
20
20
|
archived_file = ArchivedFile.new(
|
21
21
|
file_information.archived_file_parameters
|
22
22
|
)
|
23
|
+
|
24
|
+
# Find a new link
|
25
|
+
if(file_information.ftype == "link")
|
26
|
+
archived_file.ftype_data = file_information.link_target
|
27
|
+
archived_file.save!
|
28
|
+
file_information.manifest.archived_files << archived_file
|
29
|
+
return archived_file
|
30
|
+
end
|
23
31
|
|
24
32
|
# Find a new directory
|
25
|
-
if (file_information.
|
33
|
+
if (file_information.ftype=="directory")
|
26
34
|
archived_file.save!
|
27
35
|
file_information.manifest.archived_files << archived_file
|
28
36
|
return archived_file
|
@@ -38,20 +46,28 @@ module Nearline
|
|
38
46
|
end
|
39
47
|
archived_file
|
40
48
|
|
41
|
-
# TODO:
|
49
|
+
# TODO: block devices, ...?
|
42
50
|
end
|
43
51
|
|
44
52
|
def restore(*args)
|
45
53
|
@options = args.extract_options!
|
46
|
-
|
54
|
+
|
55
|
+
if (self.ftype == "link")
|
56
|
+
guarantee_path
|
57
|
+
File.symlink(self.ftype_data,
|
58
|
+
option_override(:path))
|
59
|
+
#restore_metadata
|
60
|
+
return
|
61
|
+
end
|
62
|
+
|
63
|
+
if (self.ftype == "directory")
|
47
64
|
FileUtils.mkdir_p option_override(:path)
|
48
65
|
restore_metadata
|
49
66
|
return
|
50
67
|
end
|
51
|
-
|
52
|
-
|
53
|
-
|
54
|
-
end
|
68
|
+
|
69
|
+
# ftype == "file"
|
70
|
+
guarantee_path
|
55
71
|
f = File.open(option_override(:path), "wb")
|
56
72
|
self.file_content.restore_to(f)
|
57
73
|
f.close
|
@@ -59,6 +75,13 @@ module Nearline
|
|
59
75
|
return
|
60
76
|
end
|
61
77
|
|
78
|
+
def guarantee_path
|
79
|
+
target_path = File.dirname(option_override(:path))
|
80
|
+
if (!File.exist? target_path)
|
81
|
+
FileUtils.mkdir_p target_path
|
82
|
+
end
|
83
|
+
end
|
84
|
+
|
62
85
|
def option_override(key)
|
63
86
|
if (@options.has_key?(key))
|
64
87
|
return @options[key]
|
@@ -0,0 +1,88 @@
|
|
1
|
+
module Nearline
|
2
|
+
module Models
|
3
|
+
|
4
|
+
# Handles file paths and metadata for a file in a manifest
|
5
|
+
# Acts as a builder for archived_file by providing the construction parameters
|
6
|
+
class FileInformation
|
7
|
+
attr_reader :path_hash,
|
8
|
+
:stat, # lstat result
|
9
|
+
:ftype, # file type, either "link", "directory", or "file"
|
10
|
+
:archived_file_parameters, # construction parameters for archived_file
|
11
|
+
:manifest, # reference to the associated manifest, so logging can happen
|
12
|
+
:file_path, # the full, expanded file path of the file
|
13
|
+
:archived_path, # the path as stored to the repository (less drive letter in Windows)
|
14
|
+
:link_target
|
15
|
+
|
16
|
+
def initialize(file_path, manifest)
|
17
|
+
@manifest = manifest
|
18
|
+
@file_path = File.expand_path(file_path)
|
19
|
+
@archived_path = cleaned_file_path
|
20
|
+
@stat = read_stat
|
21
|
+
@ftype = ftype_lookup(file_path)
|
22
|
+
@path_hash = generate_path_hash
|
23
|
+
@archived_file_parameters = build_parameters
|
24
|
+
end
|
25
|
+
|
26
|
+
def cleaned_file_path
|
27
|
+
if RUBY_PLATFORM =~/win/
|
28
|
+
return @file_path[2..-1]
|
29
|
+
end
|
30
|
+
@file_path
|
31
|
+
end
|
32
|
+
|
33
|
+
def read_stat
|
34
|
+
stat = nil
|
35
|
+
begin
|
36
|
+
stat = File.lstat(@file_path)
|
37
|
+
rescue
|
38
|
+
@manifest.add_log("File not found on lstat: #{@file_path}")
|
39
|
+
end
|
40
|
+
stat
|
41
|
+
end
|
42
|
+
|
43
|
+
def generate_path_hash
|
44
|
+
return nil if @stat.nil?
|
45
|
+
target = [@manifest.system.name,
|
46
|
+
@archived_path,
|
47
|
+
@stat.uid,
|
48
|
+
@stat.gid,
|
49
|
+
@stat.mtime.to_i,
|
50
|
+
@stat.mode].join(':')
|
51
|
+
Digest::SHA1.hexdigest(target)
|
52
|
+
end
|
53
|
+
|
54
|
+
def file_content_value
|
55
|
+
return FileContent.new unless @ftype == "directory"
|
56
|
+
return nil
|
57
|
+
end
|
58
|
+
|
59
|
+
def ftype_lookup(file_path)
|
60
|
+
if File.symlink?(file_path)
|
61
|
+
@link_target = File.readlink file_path
|
62
|
+
return "link"
|
63
|
+
end
|
64
|
+
if File.directory?(file_path)
|
65
|
+
return "directory"
|
66
|
+
end
|
67
|
+
return "file"
|
68
|
+
end
|
69
|
+
|
70
|
+
def build_parameters
|
71
|
+
return nil if @stat.nil?
|
72
|
+
{
|
73
|
+
:system => @manifest.system,
|
74
|
+
:path => @archived_path,
|
75
|
+
:path_hash => @path_hash,
|
76
|
+
:file_content => file_content_value,
|
77
|
+
:uid => @stat.uid,
|
78
|
+
:gid => @stat.gid,
|
79
|
+
:mtime => @stat.mtime.to_i,
|
80
|
+
:mode => @stat.mode,
|
81
|
+
:ftype => @ftype
|
82
|
+
}
|
83
|
+
end
|
84
|
+
|
85
|
+
end
|
86
|
+
|
87
|
+
end
|
88
|
+
end
|
data/lib/nearline/manifest.rb
CHANGED
@@ -25,64 +25,6 @@ module Nearline
|
|
25
25
|
end
|
26
26
|
end
|
27
27
|
|
28
|
-
# Handles file paths and metadata for a file in a manifest
|
29
|
-
class FileInformation
|
30
|
-
attr_reader :path_hash, :stat, :is_directory,
|
31
|
-
:archived_file_parameters, :manifest, :file_path
|
32
|
-
|
33
|
-
def initialize(file_path, manifest)
|
34
|
-
@manifest = manifest
|
35
|
-
@file_path = file_path
|
36
|
-
@stat = read_stat
|
37
|
-
@is_directory = File.directory?(file_path)
|
38
|
-
@path_hash = generate_path_hash
|
39
|
-
@archived_file_parameters = build_parameters
|
40
|
-
end
|
41
|
-
|
42
|
-
def read_stat
|
43
|
-
stat = nil
|
44
|
-
begin
|
45
|
-
# TODO: change to lstat when we handle links
|
46
|
-
stat = File.stat(@file_path)
|
47
|
-
rescue
|
48
|
-
@manifest.add_log("File not found on stat: #{@file_path}")
|
49
|
-
end
|
50
|
-
stat
|
51
|
-
end
|
52
|
-
|
53
|
-
def generate_path_hash
|
54
|
-
return nil if @stat.nil?
|
55
|
-
target = [@manifest.system.name,
|
56
|
-
@file_path,
|
57
|
-
@stat.uid,
|
58
|
-
@stat.gid,
|
59
|
-
@stat.mtime.to_i,
|
60
|
-
@stat.mode].join(':')
|
61
|
-
Digest::SHA1.hexdigest(target)
|
62
|
-
end
|
63
|
-
|
64
|
-
def file_content_entry_for_files_only
|
65
|
-
return FileContent.new unless @is_directory
|
66
|
-
return nil
|
67
|
-
end
|
68
|
-
|
69
|
-
def build_parameters
|
70
|
-
return nil if @stat.nil?
|
71
|
-
{
|
72
|
-
:system => @manifest.system,
|
73
|
-
:path => @file_path,
|
74
|
-
:path_hash => @path_hash,
|
75
|
-
:file_content => file_content_entry_for_files_only,
|
76
|
-
:uid => @stat.uid,
|
77
|
-
:gid => @stat.gid,
|
78
|
-
:mtime => @stat.mtime.to_i,
|
79
|
-
:mode => @stat.mode,
|
80
|
-
:is_directory => @is_directory
|
81
|
-
}
|
82
|
-
end
|
83
|
-
|
84
|
-
end
|
85
|
-
|
86
28
|
# A Manifest represents the corpus of ArchivedFiles and
|
87
29
|
# set of Log messages resulting from a backup attempt
|
88
30
|
class Manifest < ActiveRecord::Base
|
@@ -192,16 +134,16 @@ module Nearline
|
|
192
134
|
|
193
135
|
# Iterate all missing files from this manifest, yielding each
|
194
136
|
def iterate_all_missing
|
195
|
-
|
137
|
+
files_iterated = []
|
196
138
|
self.archived_files.each do |af|
|
197
139
|
begin
|
198
|
-
File.
|
140
|
+
File.lstat(af.path)
|
199
141
|
rescue
|
200
142
|
yield af
|
201
|
-
|
143
|
+
files_iterated << af.path
|
202
144
|
end
|
203
145
|
end
|
204
|
-
return
|
146
|
+
return files_iterated
|
205
147
|
end
|
206
148
|
|
207
149
|
def add_log(message)
|
@@ -223,12 +165,12 @@ module Nearline
|
|
223
165
|
def archived_file_content_query(op)
|
224
166
|
<<-END_SQL
|
225
167
|
select distinct fc.id
|
226
|
-
|
227
|
-
|
228
|
-
|
229
|
-
|
230
|
-
|
231
|
-
|
168
|
+
from archived_files af,
|
169
|
+
archived_files_manifests afm, file_contents fc
|
170
|
+
where
|
171
|
+
afm.manifest_id #{op} #{self.id} and
|
172
|
+
afm.archived_file_id = af.id and
|
173
|
+
af.file_content_id = fc.id
|
232
174
|
END_SQL
|
233
175
|
end
|
234
176
|
|
@@ -301,8 +243,8 @@ select distinct af.id
|
|
301
243
|
# A simple string reporting the performance of the manifest
|
302
244
|
def summary
|
303
245
|
completed = (completed_at.nil?) ? "DNF" : completed_at
|
304
|
-
"#{system.name} started: #{created_at}
|
305
|
-
"#{archived_files.size} files
|
246
|
+
"#{system.name} started: #{created_at}\nfinished: #{completed}\n" +
|
247
|
+
"#{archived_files.size} files\n#{logs.size} Error#{(logs.size != 1) ? 's' : ''} reported"
|
306
248
|
end
|
307
249
|
|
308
250
|
end
|
@@ -2,9 +2,9 @@ module Nearline
|
|
2
2
|
module_function
|
3
3
|
|
4
4
|
# Version of the software
|
5
|
-
VERSION = "0.0.
|
5
|
+
VERSION = "0.0.6"
|
6
6
|
# Last version that changed the database structure
|
7
|
-
DB_VERSION = "0.0.
|
7
|
+
DB_VERSION = "0.0.6"
|
8
8
|
|
9
9
|
# Array of every Nearline Model using an ActiveRecord connection
|
10
10
|
AR_MODELS = Nearline::Models.constants.map do |m|
|
data/lib/nearline/schema.rb
CHANGED
@@ -44,11 +44,6 @@ module Nearline
|
|
44
44
|
t.column :file_content_id, :integer, :null => false
|
45
45
|
end
|
46
46
|
|
47
|
-
add_index :sequences, [:sequence, :file_content_id], :unique => true,
|
48
|
-
:name => "sequence_jn_index"
|
49
|
-
|
50
|
-
add_index :sequences, [:block_id]
|
51
|
-
|
52
47
|
create_table :systems do |t|
|
53
48
|
t.column :name, :string, :null => false
|
54
49
|
end
|
@@ -64,7 +59,8 @@ module Nearline
|
|
64
59
|
t.column :gid, :integer, :default => -1
|
65
60
|
t.column :mtime, :integer, :default => 0
|
66
61
|
t.column :mode, :integer, :default => 33206 # "chmod 100666"
|
67
|
-
t.column :
|
62
|
+
t.column :ftype, :string, :null => false
|
63
|
+
t.column :ftype_data, :text
|
68
64
|
end
|
69
65
|
|
70
66
|
add_index :archived_files, [:path_hash], :unique => true
|
@@ -81,12 +77,6 @@ module Nearline
|
|
81
77
|
t.column :archived_file_id, :integer
|
82
78
|
t.column :manifest_id, :integer
|
83
79
|
end
|
84
|
-
|
85
|
-
add_index :archived_files_manifests,
|
86
|
-
[:archived_file_id, :manifest_id], {
|
87
|
-
:unique => true,
|
88
|
-
:name => "manifest_jn_index"
|
89
|
-
}
|
90
80
|
|
91
81
|
# Keeps a record of problems during backup related to a manifest
|
92
82
|
create_table :logs do |t|
|
data/lib/nearline.rb
CHANGED
data/test/nearline_test.rb
CHANGED
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: nearline
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.6
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Robert J. Osborne
|
@@ -9,7 +9,7 @@ autorequire:
|
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
11
|
|
12
|
-
date: 2008-
|
12
|
+
date: 2008-07-20 00:00:00 -04:00
|
13
13
|
default_executable:
|
14
14
|
dependencies:
|
15
15
|
- !ruby/object:Gem::Dependency
|
@@ -39,6 +39,7 @@ files:
|
|
39
39
|
- lib/nearline/log.rb
|
40
40
|
- lib/nearline/archived_file.rb
|
41
41
|
- lib/nearline/file_content.rb
|
42
|
+
- lib/nearline/file_information.rb
|
42
43
|
- lib/nearline/manifest.rb
|
43
44
|
- lib/nearline.rb
|
44
45
|
- tasks/gemspec.rake
|