d8a 0.0.1
Sign up to get free protection for your applications and to get access to all the features.
- data/lib/d8a/cached8a.rb +124 -0
- data/lib/d8a/d8a.rb +205 -0
- data/lib/d8a/digestd8a.rb +83 -0
- data/lib/d8a/filed8a.rb +71 -0
- data/lib/d8a/ftpd8a.rb +132 -0
- data/tests/d8atest.rb +58 -0
- data/tests/tc_cached8a.rb +49 -0
- data/tests/tc_digestd8a.rb +55 -0
- data/tests/tc_filed8a.rb +98 -0
- data/tests/ts_d8a.rb +9 -0
- metadata +64 -0
data/lib/d8a/cached8a.rb
ADDED
@@ -0,0 +1,124 @@
|
|
1
|
+
require 'yaml'
|
2
|
+
|
3
|
+
module CacheD8a
|
4
|
+
# <d8m> -> { <attr> => <value>... }
|
5
|
+
attr_accessor :cache
|
6
|
+
|
7
|
+
|
8
|
+
def CacheD8a.extended(d8a)
|
9
|
+
# Set up new attributes
|
10
|
+
#d8a.attrs.push(:cached)
|
11
|
+
|
12
|
+
# load existing .d8acache
|
13
|
+
begin
|
14
|
+
yaml = ""
|
15
|
+
d8a.read(".d8acache") { |f|
|
16
|
+
while s = f.read(10000)
|
17
|
+
yaml += s
|
18
|
+
end
|
19
|
+
}
|
20
|
+
d8a.cache = YAML.load(yaml)
|
21
|
+
rescue Errno::ENOENT
|
22
|
+
d8a.cache = {}
|
23
|
+
end
|
24
|
+
|
25
|
+
|
26
|
+
# wrap methods
|
27
|
+
class << d8a
|
28
|
+
# use cached attrs when possible
|
29
|
+
alias_method :__cached8a_ref, :[]
|
30
|
+
def [](d8m, attr = nil)
|
31
|
+
cached = cache_entry(d8m)
|
32
|
+
|
33
|
+
if cached
|
34
|
+
attr ? cached[attr] : cached
|
35
|
+
|
36
|
+
elsif attr
|
37
|
+
__cached8a_ref(d8m, attr)
|
38
|
+
|
39
|
+
elsif d8m =~ /.d8acache$/
|
40
|
+
__cached8a_ref(d8m)
|
41
|
+
|
42
|
+
else
|
43
|
+
@cache[d8m] = __cached8a_ref(d8m)
|
44
|
+
end
|
45
|
+
end
|
46
|
+
|
47
|
+
|
48
|
+
# update cache entry after []=
|
49
|
+
alias_method :__cached8a_refeq, :[]=
|
50
|
+
def []=(d8m, attr, val = nil)
|
51
|
+
result = __cached8a_refeq(d8m, attr, val)
|
52
|
+
@cache[d8m][attr] = val if @cache.has_key?(d8m) && ! attr.is_a?(Hash)
|
53
|
+
result
|
54
|
+
end
|
55
|
+
|
56
|
+
|
57
|
+
# don't show .d8acache on each
|
58
|
+
alias_method :__cached8a_each, :each
|
59
|
+
def each
|
60
|
+
__cached8a_each do |d8m|
|
61
|
+
yield d8m unless d8m =~ /.d8acache$/
|
62
|
+
end
|
63
|
+
end
|
64
|
+
|
65
|
+
|
66
|
+
# refresh cache entry after write
|
67
|
+
alias_method :__cached8a_write, :write
|
68
|
+
def write(d8m, &block)
|
69
|
+
@cache.delete(d8m)
|
70
|
+
result = __cached8a_write(d8m, &block)
|
71
|
+
self[d8m] # re-cache values
|
72
|
+
result
|
73
|
+
end
|
74
|
+
|
75
|
+
|
76
|
+
# clear cache entry after delete
|
77
|
+
alias_method :__cached8a_delete, :delete
|
78
|
+
def delete(d8m)
|
79
|
+
@cache.delete(d8m)
|
80
|
+
__cached8a_delete(d8m)
|
81
|
+
end
|
82
|
+
|
83
|
+
|
84
|
+
# rewrite .d8acache on flush
|
85
|
+
alias_method :__cached8a_flush, :flush
|
86
|
+
def flush
|
87
|
+
__cached8a_write(".d8acache") do |f|
|
88
|
+
f.write(YAML.dump(@cache))
|
89
|
+
end
|
90
|
+
__cached8a_flush
|
91
|
+
end
|
92
|
+
end
|
93
|
+
end
|
94
|
+
|
95
|
+
|
96
|
+
# return cached data, deleting any invalid entries and optionally
|
97
|
+
# creating missing entries
|
98
|
+
def cache_entry(d8m)
|
99
|
+
@cache.delete(d8m) if @cache[d8m] &&
|
100
|
+
@id_attrs.any? { |attr| @cache[d8m][attr] != __send__(attr, d8m) }
|
101
|
+
@cache[d8m]
|
102
|
+
end
|
103
|
+
private :cache_entry
|
104
|
+
|
105
|
+
|
106
|
+
# defines a new synthetic attribute that exists only in the cache
|
107
|
+
def synthetic_attr(*new_attrs)
|
108
|
+
new_attrs.each do |new_attr|
|
109
|
+
instance_eval %Q!
|
110
|
+
class << self
|
111
|
+
def #{new_attr.to_s}(d8m, d8m_attrs = nil)
|
112
|
+
cached = cache_entry(d8m)
|
113
|
+
cached ? cached[#{new_attr.to_s}] : nil
|
114
|
+
end
|
115
|
+
|
116
|
+
def #{new_attr.to_s}=(d8m, val)
|
117
|
+
self[d8m] # populate cache
|
118
|
+
@cache[d8m][#{new_attr.inspect}] = val
|
119
|
+
end
|
120
|
+
end!
|
121
|
+
@attrs.push(new_attr)
|
122
|
+
end
|
123
|
+
end
|
124
|
+
end
|
data/lib/d8a/d8a.rb
ADDED
@@ -0,0 +1,205 @@
|
|
1
|
+
# An abstract collection of named data. In addition
|
2
|
+
# to a name, each datum in the collection has
|
3
|
+
# a set of attributes associated with it. This base
|
4
|
+
# class is abstract; users should instantiate one of
|
5
|
+
# the concrete subclasses of this class.
|
6
|
+
#
|
7
|
+
# Subclasses should provide the following methods:
|
8
|
+
#
|
9
|
+
# * an each method that yields the name of each datum
|
10
|
+
#
|
11
|
+
# * a read(d8m) {|io|} method. The block will be
|
12
|
+
# invoked with an IO-like object that minimally supports
|
13
|
+
# a read method with the semantics of IO.read. io is
|
14
|
+
# automatically closed when the block returns. read()
|
15
|
+
# returns the value of the block.
|
16
|
+
#
|
17
|
+
# * a write(d8m) {|io|} method. The block will be
|
18
|
+
# invoked with an IO-like object that minimally supports
|
19
|
+
# a write method with the semantics of IO.write. io is
|
20
|
+
# automatically closed when the block returns. write()
|
21
|
+
# returns the value of the block.
|
22
|
+
#
|
23
|
+
# * a delete(d8m) method
|
24
|
+
#
|
25
|
+
# * a flush method that callers can invok to indicate they
|
26
|
+
# are finished
|
27
|
+
#
|
28
|
+
# Attributes of datum can be defined by subclasses or modules
|
29
|
+
# by appending the attribute's symbol to @attrs and supplying
|
30
|
+
# the following methods:
|
31
|
+
#
|
32
|
+
# * <attr>(d8m, d8mattrs=nil) that returns the
|
33
|
+
# value of the requested attribute for the given d8m.
|
34
|
+
# If available, other attributes for the d8m will be
|
35
|
+
# supplied in the d8mattrs hash.
|
36
|
+
#
|
37
|
+
|
38
|
+
# * <attr>=(d8m, val) that sets the value of the attribute. This is
|
39
|
+
# optional; the attribute will be read-only if this method is not
|
40
|
+
# provided.
|
41
|
+
|
42
|
+
class D8a
|
43
|
+
include Enumerable
|
44
|
+
|
45
|
+
# D8a name
|
46
|
+
attr :d8aname
|
47
|
+
|
48
|
+
|
49
|
+
# Attrs for data
|
50
|
+
attr :attrs
|
51
|
+
|
52
|
+
|
53
|
+
# Identity attrs for data. Users of this D8a can assume that a
|
54
|
+
# datum has not changed if all of these attributes remain constant.
|
55
|
+
attr :id_attrs
|
56
|
+
|
57
|
+
|
58
|
+
# Initialization
|
59
|
+
def initialize(d8aname)
|
60
|
+
@d8aname = d8aname
|
61
|
+
@attrs = [:name]
|
62
|
+
@id_attrs = []
|
63
|
+
end
|
64
|
+
|
65
|
+
|
66
|
+
# :name attribute
|
67
|
+
def name(d8m, *d8mattrs)
|
68
|
+
d8m
|
69
|
+
end
|
70
|
+
|
71
|
+
|
72
|
+
# Returns information about a named datum.
|
73
|
+
def [](d8m, attr = nil)
|
74
|
+
if attr
|
75
|
+
@attrs.include?(attr) ? __send__(attr, d8m) : nil
|
76
|
+
|
77
|
+
else
|
78
|
+
@attrs.inject({:name => d8m}) { |d8mattrs, attr|
|
79
|
+
val = __send__(attr, d8m, d8mattrs)
|
80
|
+
d8mattrs[attr] = val if val
|
81
|
+
d8mattrs
|
82
|
+
}
|
83
|
+
end
|
84
|
+
end
|
85
|
+
|
86
|
+
|
87
|
+
# Sets information about a named datum.
|
88
|
+
def []=(d8m, attr, val = nil)
|
89
|
+
if attr.is_a?(Hash)
|
90
|
+
attr.each { |k, v| self[d8m, k] = v }
|
91
|
+
|
92
|
+
elsif @attrs.include?(attr)
|
93
|
+
begin
|
94
|
+
__send__((attr.to_s + "=").to_sym, d8m, val)
|
95
|
+
rescue NoMethodError
|
96
|
+
nil
|
97
|
+
end
|
98
|
+
end
|
99
|
+
end
|
100
|
+
|
101
|
+
|
102
|
+
# dummy implementation
|
103
|
+
def each
|
104
|
+
raise "each not implemented by this D8a"
|
105
|
+
end
|
106
|
+
|
107
|
+
|
108
|
+
# dummy implementation
|
109
|
+
def read(d8m)
|
110
|
+
raise "read(d8m) not implemented by this D8a"
|
111
|
+
end
|
112
|
+
|
113
|
+
|
114
|
+
# dummy implementation
|
115
|
+
def write(d8m)
|
116
|
+
raise "write(d8m) not implemented by this D8a"
|
117
|
+
end
|
118
|
+
|
119
|
+
|
120
|
+
# dummy implementation
|
121
|
+
def delete(d8m)
|
122
|
+
raise "delete(d8m) not implemented by this D8a"
|
123
|
+
end
|
124
|
+
|
125
|
+
|
126
|
+
# flush any cached data
|
127
|
+
def flush
|
128
|
+
end
|
129
|
+
|
130
|
+
|
131
|
+
# returns the different attributes for two d8m
|
132
|
+
def diff_d8m(d8m1, d8m2, d8a1, d8a2)
|
133
|
+
name = (d8m1||d8m2)[:name]
|
134
|
+
|
135
|
+
d8m1 ||= d8a1.attrs.inject({}) { |attrs,attr| attrs[attr] = nil; attrs }
|
136
|
+
d8m2 ||= d8a2.attrs.inject({}) { |attrs,attr| attrs[attr] = nil; attrs }
|
137
|
+
|
138
|
+
(d8m1.keys & d8m2.keys).find_all { |attr|
|
139
|
+
begin
|
140
|
+
d8m1[attr] != d8m2[attr]
|
141
|
+
rescue
|
142
|
+
true
|
143
|
+
end
|
144
|
+
}
|
145
|
+
end
|
146
|
+
|
147
|
+
|
148
|
+
# invokes block for each diff, returns entries in this not in o
|
149
|
+
def diff(o)
|
150
|
+
each do |d8m|
|
151
|
+
begin
|
152
|
+
myd8m = self[d8m]
|
153
|
+
od8m = o[d8m]
|
154
|
+
yield(myd8m, od8m) unless diff_d8m(myd8m, od8m, self, o).empty?
|
155
|
+
rescue Errno::ENOENT
|
156
|
+
yield(myd8m, nil)
|
157
|
+
end
|
158
|
+
end
|
159
|
+
|
160
|
+
o.each do |d8m|
|
161
|
+
begin
|
162
|
+
myd8m = self[d8m]
|
163
|
+
rescue Errno::ENOENT
|
164
|
+
yield(nil, o[d8m])
|
165
|
+
end
|
166
|
+
end
|
167
|
+
end
|
168
|
+
|
169
|
+
|
170
|
+
# Generates a report of differences as {name -> [:attr1,:attr2...]}
|
171
|
+
def diffreport(o)
|
172
|
+
result = {}
|
173
|
+
|
174
|
+
diff(o) do |a,b|
|
175
|
+
result[(a||b)[:name]] = diff_d8m(a, b, self, o)
|
176
|
+
end
|
177
|
+
|
178
|
+
result
|
179
|
+
end
|
180
|
+
|
181
|
+
|
182
|
+
# copy one datum to another
|
183
|
+
def copy(fromd8a, fromd8m, tod8a, tod8m = fromd8m)
|
184
|
+
fromd8a.read(fromd8m) do |r|
|
185
|
+
tod8a.write(tod8m) do |w|
|
186
|
+
while s = r.read(65536)
|
187
|
+
w.write(s)
|
188
|
+
end
|
189
|
+
end
|
190
|
+
end
|
191
|
+
end
|
192
|
+
|
193
|
+
|
194
|
+
# clones this D8a to another
|
195
|
+
def sync(to)
|
196
|
+
diff(to) do |myd8m,tod8m|
|
197
|
+
if myd8m
|
198
|
+
copy(self, myd8m[:name], to)
|
199
|
+
to[myd8m[:name]] = myd8m
|
200
|
+
else
|
201
|
+
to.delete(tod8m[:name])
|
202
|
+
end
|
203
|
+
end
|
204
|
+
end
|
205
|
+
end
|
@@ -0,0 +1,83 @@
|
|
1
|
+
require 'digest/md5'
|
2
|
+
require 'd8a/cached8a'
|
3
|
+
|
4
|
+
# Add digesting to an IO object
|
5
|
+
module DigestIO
|
6
|
+
attr_accessor :digest
|
7
|
+
|
8
|
+
def DigestIO.extended(o)
|
9
|
+
class << o
|
10
|
+
alias_method :__digestio_read, :read
|
11
|
+
def read(*args)
|
12
|
+
s = __digestio_read(*args)
|
13
|
+
@digest << s if s
|
14
|
+
s
|
15
|
+
end
|
16
|
+
|
17
|
+
alias_method :__digestio_write, :write
|
18
|
+
def write(s)
|
19
|
+
i = __digestio_write(s)
|
20
|
+
@digest << s[0,i]
|
21
|
+
i
|
22
|
+
end
|
23
|
+
end
|
24
|
+
|
25
|
+
o.digest = Digest::MD5.new
|
26
|
+
end
|
27
|
+
end
|
28
|
+
|
29
|
+
|
30
|
+
# Add MD5 digest attributes to data.
|
31
|
+
module DigestD8a
|
32
|
+
# d8m -> [mod time, digest] info for datum that we've seen
|
33
|
+
attr_accessor :digests
|
34
|
+
|
35
|
+
|
36
|
+
def DigestD8a.extended(d8a)
|
37
|
+
d8a.extend CacheD8a
|
38
|
+
d8a.synthetic_attr :md5
|
39
|
+
|
40
|
+
|
41
|
+
# redefine read/write to capture digests
|
42
|
+
class << d8a
|
43
|
+
alias_method :__digestd8a_read, :read
|
44
|
+
def read(d8m)
|
45
|
+
__digestd8a_read(d8m) do |f|
|
46
|
+
f.extend(DigestIO)
|
47
|
+
result = yield f
|
48
|
+
self[d8m, :md5] = f.digest.to_s
|
49
|
+
result
|
50
|
+
end
|
51
|
+
end
|
52
|
+
|
53
|
+
|
54
|
+
alias_method :__digestd8a_write, :write
|
55
|
+
def write(d8m)
|
56
|
+
digest = nil
|
57
|
+
__digestd8a_write(d8m) do |f|
|
58
|
+
f.extend(DigestIO)
|
59
|
+
digest = f.digest
|
60
|
+
yield f
|
61
|
+
end
|
62
|
+
|
63
|
+
# have to set this after the block since the id_attrs check
|
64
|
+
# in the caching logic will blow away anything we set before
|
65
|
+
# write returns
|
66
|
+
|
67
|
+
self[d8m, :md5] = digest.to_s
|
68
|
+
end
|
69
|
+
end
|
70
|
+
end
|
71
|
+
|
72
|
+
|
73
|
+
# forces recomputation of digest for a d8m
|
74
|
+
def compute_digest(d8m)
|
75
|
+
read(d8m) {|f|}
|
76
|
+
end
|
77
|
+
|
78
|
+
|
79
|
+
# force recomputation of all digests
|
80
|
+
def compute_digests
|
81
|
+
self.each { |d8m| compute_md5(d8m) }
|
82
|
+
end
|
83
|
+
end
|
data/lib/d8a/filed8a.rb
ADDED
@@ -0,0 +1,71 @@
|
|
1
|
+
require 'd8a/d8a'
|
2
|
+
require 'find'
|
3
|
+
|
4
|
+
# D8a based on a directory tree.
|
5
|
+
class FileD8a < D8a
|
6
|
+
# base directory
|
7
|
+
attr :basedir
|
8
|
+
|
9
|
+
|
10
|
+
def initialize(basedir, name = File.basename(basedir))
|
11
|
+
super(name)
|
12
|
+
|
13
|
+
File.directory?(basedir) || Dir.mkdir(basedir)
|
14
|
+
@basedir = basedir
|
15
|
+
|
16
|
+
@attrs.push(:size, :mtime, :mode)
|
17
|
+
@id_attrs.push(:size, :mtime)
|
18
|
+
end
|
19
|
+
|
20
|
+
|
21
|
+
# :size attribute
|
22
|
+
def size(d8m, d8mattrs = nil)
|
23
|
+
File.size(File.join(@basedir, d8m))
|
24
|
+
end
|
25
|
+
|
26
|
+
|
27
|
+
# :mtime attribute
|
28
|
+
def mtime(d8m, d8mattrs = nil)
|
29
|
+
File.mtime(File.join(@basedir, d8m))
|
30
|
+
end
|
31
|
+
|
32
|
+
def mtime=(d8m, mtime)
|
33
|
+
File.utime(mtime, mtime, File.join(@basedir, d8m))
|
34
|
+
end
|
35
|
+
|
36
|
+
|
37
|
+
# :mode attribute
|
38
|
+
def mode(d8m, d8mattrs = nil)
|
39
|
+
File.stat(File.join(@basedir, d8m)).mode & 0777
|
40
|
+
end
|
41
|
+
|
42
|
+
def mode=(d8m, mode)
|
43
|
+
File.chmod(mode, File.join(@basedir, d8m))
|
44
|
+
end
|
45
|
+
|
46
|
+
|
47
|
+
|
48
|
+
def each
|
49
|
+
Find.find(@basedir) do |f|
|
50
|
+
File.file?(f) && yield(f[(@basedir.length + 1)..-1])
|
51
|
+
end
|
52
|
+
end
|
53
|
+
|
54
|
+
|
55
|
+
# Reads a d8m
|
56
|
+
def read(d8m, &block)
|
57
|
+
File.open(File.join(@basedir, d8m), "r", &block)
|
58
|
+
end
|
59
|
+
|
60
|
+
|
61
|
+
# Writes a d8m
|
62
|
+
def write(d8m, &block)
|
63
|
+
File.open(File.join(@basedir, d8m), "w", &block)
|
64
|
+
end
|
65
|
+
|
66
|
+
|
67
|
+
# Deletes a d8m
|
68
|
+
def delete(d8m)
|
69
|
+
File.delete(File.join(@basedir, d8m))
|
70
|
+
end
|
71
|
+
end
|
data/lib/d8a/ftpd8a.rb
ADDED
@@ -0,0 +1,132 @@
|
|
1
|
+
require 'd8a/d8a'
|
2
|
+
require 'date'
|
3
|
+
require 'net/ftp'
|
4
|
+
|
5
|
+
# D8a based on an ftp site.
|
6
|
+
class FtpD8a < D8a
|
7
|
+
# Proc(line, state_array) -> {} to process ls -R lines from server
|
8
|
+
attr :ls_parser
|
9
|
+
|
10
|
+
|
11
|
+
# set up a new Net::FTP connection
|
12
|
+
def FtpD8a.open(host, dir = nil, user = nil, passwd = nil, &block)
|
13
|
+
FtpD8a.new("ftp://#{user || ''}#{user ? '@' : ''}#{host}/#{dir || ''}") do |ftp|
|
14
|
+
ftp.close unless !ftp || ftp.closed?
|
15
|
+
ftp = Net::FTP.new(host, user, passwd)
|
16
|
+
ftp.chdir(dir) if dir
|
17
|
+
ftp
|
18
|
+
end
|
19
|
+
end
|
20
|
+
|
21
|
+
|
22
|
+
# initializer
|
23
|
+
def initialize(name, ftp = nil, &block)
|
24
|
+
super(name)
|
25
|
+
|
26
|
+
@ftp_gen = block
|
27
|
+
|
28
|
+
@ftp = ftp
|
29
|
+
ensure_ftp
|
30
|
+
|
31
|
+
@attrs.push(:size, :ftpmtime, :mode)
|
32
|
+
@id_attrs.push(:size, :ftpmtime)
|
33
|
+
|
34
|
+
|
35
|
+
@ls_parser = Proc.new { |line,state|
|
36
|
+
case line
|
37
|
+
when /^$/
|
38
|
+
nil
|
39
|
+
|
40
|
+
when /^([-d])([-r][-w][-x][-r][-w][-x][-r][-w][-x])\s+\d+\s+\d+\s+\d+\s+(\d+)\s+(.{12})\s+(.*)$/
|
41
|
+
next nil if $1 == 'd'
|
42
|
+
|
43
|
+
name = state[0] + $5
|
44
|
+
size = $3.to_i
|
45
|
+
mode = eval("0b#{$2.tr('-rwx', '01')}")
|
46
|
+
ftpmtime =
|
47
|
+
case $4
|
48
|
+
when /^((Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)\s+\d{1,2}\s+\d{4})$/
|
49
|
+
Date.strptime($2, '%b %e %Y')
|
50
|
+
|
51
|
+
when /^(Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)\s+(\d{1,2})\s+(\d{1,2}:\d{2})$/
|
52
|
+
# TODO - this seems kind of clunky...
|
53
|
+
now = DateTime.now
|
54
|
+
Date.strptime("#{$1} #{$2} #{now.year}", '%b %e %Y') > now ?
|
55
|
+
Date.strptime("#{$1} #{$2} #{now.year-1} #{$3}", '%b %e %Y %R') :
|
56
|
+
Date.strptime("#{$1} #{$2} #{now.year} #{$3}", '%b %e %Y %R')
|
57
|
+
end
|
58
|
+
|
59
|
+
name.sub!(%r{^./}, '')
|
60
|
+
|
61
|
+
{ :name => name, :size => size, :mode => mode, :ftpmtime => ftpmtime }
|
62
|
+
|
63
|
+
when /^(.*):$/
|
64
|
+
state[0] = $1 + "/"
|
65
|
+
end
|
66
|
+
}
|
67
|
+
end
|
68
|
+
|
69
|
+
|
70
|
+
private
|
71
|
+
|
72
|
+
# (re)sets up @ftp with an FTP connection to the host
|
73
|
+
def ensure_ftp
|
74
|
+
begin
|
75
|
+
@ftp.pwd
|
76
|
+
rescue
|
77
|
+
@ftp = @ftp_gen.call(@ftp) if @ftp_gen
|
78
|
+
@ftp.pwd
|
79
|
+
end
|
80
|
+
end
|
81
|
+
|
82
|
+
|
83
|
+
public
|
84
|
+
|
85
|
+
# size attr
|
86
|
+
def size(d8m, d8mattrs = nil)
|
87
|
+
@ftpcache[d8m] && @ftpcache[d8m][:size]
|
88
|
+
end
|
89
|
+
|
90
|
+
|
91
|
+
# ftpmtime attr
|
92
|
+
def ftpmtime(d8m, d8mattrs = nil)
|
93
|
+
@ftpcache[d8m] && @ftpcache[d8m][:ftpmtime]
|
94
|
+
end
|
95
|
+
|
96
|
+
|
97
|
+
# mode attr
|
98
|
+
def mode(d8m, d8mattrs = nil)
|
99
|
+
@ftpcache[d8m] && @ftpcache[d8m][:mode]
|
100
|
+
end
|
101
|
+
|
102
|
+
|
103
|
+
def each
|
104
|
+
ensure_ftp
|
105
|
+
dir = [""]
|
106
|
+
@ftpcache = {}
|
107
|
+
@ftp.list("-R") do |line|
|
108
|
+
if (x = @ls_parser.call(line, dir)).is_a?(Hash)
|
109
|
+
@ftpcache[x[:name]] = x
|
110
|
+
yield x[:name]
|
111
|
+
end
|
112
|
+
end
|
113
|
+
end
|
114
|
+
|
115
|
+
|
116
|
+
def read(d8m)
|
117
|
+
end
|
118
|
+
|
119
|
+
|
120
|
+
def write(d8m)
|
121
|
+
end
|
122
|
+
|
123
|
+
|
124
|
+
def delete(d8m)
|
125
|
+
end
|
126
|
+
|
127
|
+
|
128
|
+
def flush
|
129
|
+
#@ftp.close unless !@ftp || @ftp.closed?
|
130
|
+
#@ftp = nil
|
131
|
+
end
|
132
|
+
end
|
data/tests/d8atest.rb
ADDED
@@ -0,0 +1,58 @@
|
|
1
|
+
require 'fileutils'
|
2
|
+
require 'test/unit'
|
3
|
+
require 'tmpdir'
|
4
|
+
|
5
|
+
$:.unshift File.join(File.dirname(__FILE__), "..", "lib") unless
|
6
|
+
$:.include?(File.join(File.dirname(__FILE__), "..", "lib"))
|
7
|
+
|
8
|
+
|
9
|
+
module TestD8a
|
10
|
+
include FileUtils
|
11
|
+
|
12
|
+
def setup_dirname(name)
|
13
|
+
File.join(Dir.tmpdir, "TestFileD8a", name)
|
14
|
+
end
|
15
|
+
|
16
|
+
|
17
|
+
def setup_filed8a(name, files)
|
18
|
+
dir = setup_dirname(name)
|
19
|
+
Dir.mkdir(dir)
|
20
|
+
|
21
|
+
files.each do |file|
|
22
|
+
fname = File.join(dir, file[0])
|
23
|
+
File.open(fname, "w") { |f| f.write("x" * file[1]) }
|
24
|
+
File.utime(file[2], file[2], fname)
|
25
|
+
end
|
26
|
+
|
27
|
+
@d8as[name] = { :dir => dir, :files => files }
|
28
|
+
end
|
29
|
+
|
30
|
+
|
31
|
+
def setup
|
32
|
+
@now = Time.at(Time.now.to_i)
|
33
|
+
|
34
|
+
@rootdir = File.join(Dir.tmpdir, "TestFileD8a")
|
35
|
+
File.directory?(@rootdir) || Dir.mkdir(@rootdir)
|
36
|
+
|
37
|
+
@d8as = {}
|
38
|
+
|
39
|
+
setup_filed8a("b", [["t0-s0", 0, Time.at(0)], ["tnow-s1000", 1000, @now]])
|
40
|
+
setup_filed8a("a", [])
|
41
|
+
end
|
42
|
+
|
43
|
+
|
44
|
+
def teardown
|
45
|
+
rm_rf(@rootdir) unless @skip_teardown
|
46
|
+
@d8as.clear
|
47
|
+
end
|
48
|
+
|
49
|
+
|
50
|
+
def assert_d8aequal(d8a1, d8a2)
|
51
|
+
errors = []
|
52
|
+
d8a1.diff(d8a2) do |d8m1,d8m2|
|
53
|
+
errors << [d8m1,d8m2]
|
54
|
+
end
|
55
|
+
|
56
|
+
assert_equal([], errors)
|
57
|
+
end
|
58
|
+
end
|
@@ -0,0 +1,49 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
require 'test/unit'
|
4
|
+
|
5
|
+
$:.unshift File.join(File.dirname(__FILE__), "..", "lib")
|
6
|
+
$:.unshift File.join(File.dirname(__FILE__))
|
7
|
+
|
8
|
+
require 'd8atest'
|
9
|
+
require 'd8a/filed8a'
|
10
|
+
require 'd8a/cached8a'
|
11
|
+
|
12
|
+
|
13
|
+
class TestCacheD8a < Test::Unit::TestCase
|
14
|
+
include TestD8a
|
15
|
+
|
16
|
+
def test_cache
|
17
|
+
# add caching to a D8a
|
18
|
+
bd8a = FileD8a.new(@d8as["b"][:dir])
|
19
|
+
bd8a.extend CacheD8a
|
20
|
+
|
21
|
+
# create a phantom attr
|
22
|
+
bd8a.synthetic_attr :foo
|
23
|
+
|
24
|
+
assert(bd8a.respond_to?(:foo))
|
25
|
+
assert_equal(-2, bd8a.method(:foo).arity)
|
26
|
+
assert(bd8a.respond_to?(:foo=))
|
27
|
+
assert_equal(2, bd8a.method(:foo=).arity)
|
28
|
+
|
29
|
+
# set new attr on the files
|
30
|
+
bd8a["t0-s0", :foo] = 0
|
31
|
+
bd8a["tnow-s1000", :foo] = @now.to_i
|
32
|
+
|
33
|
+
# flush cache
|
34
|
+
bd8a.flush
|
35
|
+
|
36
|
+
# change timestamp on t0-s0 to invalidate cache entry
|
37
|
+
File.utime(1000, 1000, File.join(@d8as["b"][:dir], "t0-s0"))
|
38
|
+
|
39
|
+
# new D8a for same dir
|
40
|
+
xd8a = FileD8a.new(@d8as["b"][:dir])
|
41
|
+
xd8a.extend CacheD8a
|
42
|
+
|
43
|
+
# verify attr is still there
|
44
|
+
xd8a.synthetic_attr :foo
|
45
|
+
|
46
|
+
assert_nil(xd8a["t0-s0", :foo])
|
47
|
+
assert_equal(@now.to_i, xd8a["tnow-s1000", :foo])
|
48
|
+
end
|
49
|
+
end
|
@@ -0,0 +1,55 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
require 'test/unit'
|
4
|
+
|
5
|
+
$:.unshift File.join(File.dirname(__FILE__), "..", "lib")
|
6
|
+
$:.unshift File.join(File.dirname(__FILE__))
|
7
|
+
|
8
|
+
require 'd8atest'
|
9
|
+
require 'd8a/filed8a'
|
10
|
+
require 'd8a/digestd8a'
|
11
|
+
|
12
|
+
|
13
|
+
class TestDigestD8a < Test::Unit::TestCase
|
14
|
+
include TestD8a
|
15
|
+
|
16
|
+
def test_digest
|
17
|
+
# to compute our own digests
|
18
|
+
require "digest/md5"
|
19
|
+
|
20
|
+
# add digests to a D8a
|
21
|
+
bd8a = FileD8a.new(@d8as["b"][:dir])
|
22
|
+
bd8a.extend DigestD8a
|
23
|
+
|
24
|
+
assert(bd8a.respond_to?(:md5))
|
25
|
+
assert(bd8a.respond_to?(:md5=))
|
26
|
+
|
27
|
+
# should not be any digest to start with
|
28
|
+
assert_nil(bd8a["t0-s0", :md5])
|
29
|
+
|
30
|
+
# verify forced computation of digest
|
31
|
+
bd8a.compute_digest("t0-s0")
|
32
|
+
assert_equal(Digest::MD5.new.to_s, bd8a["t0-s0", :md5])
|
33
|
+
|
34
|
+
# verify digest is computed on read
|
35
|
+
assert_nil(bd8a["tnow-s1000", :md5])
|
36
|
+
bd8a.read("tnow-s1000") { |f|
|
37
|
+
while s = f.read(10000)
|
38
|
+
end
|
39
|
+
}
|
40
|
+
assert_equal((Digest::MD5.new << "x" * 1000).to_s,
|
41
|
+
bd8a["tnow-s1000", :md5])
|
42
|
+
|
43
|
+
# verify digest is computed on write
|
44
|
+
bd8a.write("tnow-s1000") { |f| f.write("z" * 1000) }
|
45
|
+
assert_equal(d = (Digest::MD5.new << "z" * 1000).to_s,
|
46
|
+
bd8a["tnow-s1000", :md5])
|
47
|
+
|
48
|
+
# verifiy digest is cached correctly
|
49
|
+
bd8a.flush
|
50
|
+
xd8a = FileD8a.new(@d8as["b"][:dir])
|
51
|
+
xd8a.extend DigestD8a
|
52
|
+
|
53
|
+
assert_equal(d, xd8a["tnow-s1000", :md5])
|
54
|
+
end
|
55
|
+
end
|
data/tests/tc_filed8a.rb
ADDED
@@ -0,0 +1,98 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
require 'test/unit'
|
4
|
+
|
5
|
+
$:.unshift File.join(File.dirname(__FILE__)) unless
|
6
|
+
$:.include?(File.join(File.dirname(__FILE__)))
|
7
|
+
|
8
|
+
require 'd8atest'
|
9
|
+
require 'd8a/filed8a'
|
10
|
+
require 'd8a/cached8a'
|
11
|
+
|
12
|
+
|
13
|
+
class TestFileD8a < Test::Unit::TestCase
|
14
|
+
include TestD8a
|
15
|
+
|
16
|
+
def assert_contents(d8a, name = d8a.d8aname)
|
17
|
+
files = @d8as[name][:files].clone
|
18
|
+
errors = []
|
19
|
+
|
20
|
+
d8a.each do |d8m|
|
21
|
+
info = [d8a[d8m,:name], d8a[d8m,:size], d8a[d8m,:mtime]]
|
22
|
+
files.delete(info) || errors << "unrecognized d8m: #{info.inspect}"
|
23
|
+
end
|
24
|
+
|
25
|
+
files.each do |notfound|
|
26
|
+
errors << "missing d8m: #{notfound.inspect}"
|
27
|
+
end
|
28
|
+
|
29
|
+
assert_equal([], errors)
|
30
|
+
end
|
31
|
+
|
32
|
+
|
33
|
+
def assert_diffreportequal(dr1, dr2)
|
34
|
+
assert(dr1.size == dr2.size &&
|
35
|
+
dr1.all? { |d8m,attrs| (attrs - dr2[d8m]).empty? },
|
36
|
+
"#{dr1.inspect}\n#{dr2.inspect}")
|
37
|
+
end
|
38
|
+
|
39
|
+
|
40
|
+
|
41
|
+
def test_file
|
42
|
+
# validate known set of files
|
43
|
+
ad8a = FileD8a.new(@d8as["a"][:dir])
|
44
|
+
assert_contents(ad8a)
|
45
|
+
|
46
|
+
bd8a = FileD8a.new(@d8as["b"][:dir])
|
47
|
+
assert_contents(bd8a)
|
48
|
+
|
49
|
+
|
50
|
+
# create one from scratch
|
51
|
+
cdir = setup_dirname("c")
|
52
|
+
cd8a = FileD8a.new(cdir)
|
53
|
+
|
54
|
+
files = @d8as["b"][:files]
|
55
|
+
files.each do |f|
|
56
|
+
cd8a.write(f[0]) { |io| io.write('x' * f[1]) }
|
57
|
+
cd8a[f[0], :mtime] = f[2]
|
58
|
+
end
|
59
|
+
assert_equal(files.size + 2, Dir.entries(cdir).size)
|
60
|
+
files.each do |f|
|
61
|
+
assert_equal(f[1], cd8a[f[0], :size])
|
62
|
+
assert_equal(f[2], cd8a[f[0], :mtime])
|
63
|
+
end
|
64
|
+
|
65
|
+
|
66
|
+
# b and c should be identical
|
67
|
+
assert_d8aequal(cd8a, bd8a)
|
68
|
+
|
69
|
+
|
70
|
+
# create some diffs
|
71
|
+
File.open(File.join(@d8as["b"][:dir], "justb"), "w") do |f|
|
72
|
+
f.write("This file exists only in b.\n")
|
73
|
+
end
|
74
|
+
|
75
|
+
File.open(File.join(cdir, "justc"), "w") do |f|
|
76
|
+
f.write("This file exists only in c.\n")
|
77
|
+
end
|
78
|
+
|
79
|
+
File.open(File.join(@d8as["b"][:dir], "diffsize"), "w") do |f|
|
80
|
+
f.write("This file has different sizes in b & c.\n")
|
81
|
+
end
|
82
|
+
File.open(File.join(cdir, "diffsize"), "w") do |f|
|
83
|
+
f.write("This file has different sizes in b & c. C is longer.\n")
|
84
|
+
end
|
85
|
+
t = File.mtime(File.join(@d8as["b"][:dir], "diffsize"))
|
86
|
+
File.utime(t, t, File.join(cdir, "diffsize"))
|
87
|
+
|
88
|
+
File.open(File.join(@d8as["b"][:dir], "diffmtime"), "w") do |f|
|
89
|
+
f.write("This file has different mtimes in b & c.\n")
|
90
|
+
end
|
91
|
+
File.open(File.join(cdir, "diffmtime"), "w") do |f|
|
92
|
+
f.write("This file has different mtimes in b & c.\n")
|
93
|
+
end
|
94
|
+
File.utime(0, 0, File.join(cdir, "diffmtime"))
|
95
|
+
|
96
|
+
assert_diffreportequal({"justb" => bd8a.attrs, "justc" => cd8a.attrs, "diffmtime" => [:mtime], "diffsize" => [:size]}, bd8a.diffreport(cd8a))
|
97
|
+
end
|
98
|
+
end
|
data/tests/ts_d8a.rb
ADDED
metadata
ADDED
@@ -0,0 +1,64 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
rubygems_version: 0.9.0
|
3
|
+
specification_version: 1
|
4
|
+
name: d8a
|
5
|
+
version: !ruby/object:Gem::Version
|
6
|
+
version: 0.0.1
|
7
|
+
date: 2007-02-19 00:00:00 -05:00
|
8
|
+
summary: A uniform data library and associated tools
|
9
|
+
require_paths:
|
10
|
+
- lib
|
11
|
+
email: mike.j.burr@gmail.com
|
12
|
+
homepage: http://d8a.rubyforge.org
|
13
|
+
rubyforge_project:
|
14
|
+
description:
|
15
|
+
autorequire:
|
16
|
+
default_executable:
|
17
|
+
bindir: bin
|
18
|
+
has_rdoc: false
|
19
|
+
required_ruby_version: !ruby/object:Gem::Version::Requirement
|
20
|
+
requirements:
|
21
|
+
- - ">"
|
22
|
+
- !ruby/object:Gem::Version
|
23
|
+
version: 0.0.0
|
24
|
+
version:
|
25
|
+
platform: ruby
|
26
|
+
signing_key:
|
27
|
+
cert_chain:
|
28
|
+
post_install_message:
|
29
|
+
authors:
|
30
|
+
- Mike Burr
|
31
|
+
files:
|
32
|
+
- lib/d8a
|
33
|
+
- lib/d8a/cached8a.rb
|
34
|
+
- lib/d8a/ftpd8a.rb
|
35
|
+
- lib/d8a/d8a.rb
|
36
|
+
- lib/d8a/digestd8a.rb
|
37
|
+
- lib/d8a/filed8a.rb
|
38
|
+
- tests/tc_digestd8a.rb
|
39
|
+
- tests/tc_filed8a.rb
|
40
|
+
- tests/ts_d8a.rb
|
41
|
+
- tests/tc_cached8a.rb
|
42
|
+
- tests/d8atest.rb
|
43
|
+
test_files:
|
44
|
+
- tests/ts_d8a.rb
|
45
|
+
rdoc_options: []
|
46
|
+
|
47
|
+
extra_rdoc_files: []
|
48
|
+
|
49
|
+
executables: []
|
50
|
+
|
51
|
+
extensions: []
|
52
|
+
|
53
|
+
requirements: []
|
54
|
+
|
55
|
+
dependencies:
|
56
|
+
- !ruby/object:Gem::Dependency
|
57
|
+
name: urirequire
|
58
|
+
version_requirement:
|
59
|
+
version_requirements: !ruby/object:Gem::Version::Requirement
|
60
|
+
requirements:
|
61
|
+
- - ">="
|
62
|
+
- !ruby/object:Gem::Version
|
63
|
+
version: 0.1.0
|
64
|
+
version:
|