d8a 0.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/lib/d8a/cached8a.rb +124 -0
- data/lib/d8a/d8a.rb +205 -0
- data/lib/d8a/digestd8a.rb +83 -0
- data/lib/d8a/filed8a.rb +71 -0
- data/lib/d8a/ftpd8a.rb +132 -0
- data/tests/d8atest.rb +58 -0
- data/tests/tc_cached8a.rb +49 -0
- data/tests/tc_digestd8a.rb +55 -0
- data/tests/tc_filed8a.rb +98 -0
- data/tests/ts_d8a.rb +9 -0
- metadata +64 -0
data/lib/d8a/cached8a.rb
ADDED
@@ -0,0 +1,124 @@
|
|
1
|
+
require 'yaml'
|
2
|
+
|
3
|
+
module CacheD8a
|
4
|
+
# <d8m> -> { <attr> => <value>... }
|
5
|
+
attr_accessor :cache
|
6
|
+
|
7
|
+
|
8
|
+
def CacheD8a.extended(d8a)
|
9
|
+
# Set up new attributes
|
10
|
+
#d8a.attrs.push(:cached)
|
11
|
+
|
12
|
+
# load existing .d8acache
|
13
|
+
begin
|
14
|
+
yaml = ""
|
15
|
+
d8a.read(".d8acache") { |f|
|
16
|
+
while s = f.read(10000)
|
17
|
+
yaml += s
|
18
|
+
end
|
19
|
+
}
|
20
|
+
d8a.cache = YAML.load(yaml)
|
21
|
+
rescue Errno::ENOENT
|
22
|
+
d8a.cache = {}
|
23
|
+
end
|
24
|
+
|
25
|
+
|
26
|
+
# wrap methods
|
27
|
+
class << d8a
|
28
|
+
# use cached attrs when possible
|
29
|
+
alias_method :__cached8a_ref, :[]
|
30
|
+
def [](d8m, attr = nil)
|
31
|
+
cached = cache_entry(d8m)
|
32
|
+
|
33
|
+
if cached
|
34
|
+
attr ? cached[attr] : cached
|
35
|
+
|
36
|
+
elsif attr
|
37
|
+
__cached8a_ref(d8m, attr)
|
38
|
+
|
39
|
+
elsif d8m =~ /.d8acache$/
|
40
|
+
__cached8a_ref(d8m)
|
41
|
+
|
42
|
+
else
|
43
|
+
@cache[d8m] = __cached8a_ref(d8m)
|
44
|
+
end
|
45
|
+
end
|
46
|
+
|
47
|
+
|
48
|
+
# update cache entry after []=
|
49
|
+
alias_method :__cached8a_refeq, :[]=
|
50
|
+
def []=(d8m, attr, val = nil)
|
51
|
+
result = __cached8a_refeq(d8m, attr, val)
|
52
|
+
@cache[d8m][attr] = val if @cache.has_key?(d8m) && ! attr.is_a?(Hash)
|
53
|
+
result
|
54
|
+
end
|
55
|
+
|
56
|
+
|
57
|
+
# don't show .d8acache on each
|
58
|
+
alias_method :__cached8a_each, :each
|
59
|
+
def each
|
60
|
+
__cached8a_each do |d8m|
|
61
|
+
yield d8m unless d8m =~ /.d8acache$/
|
62
|
+
end
|
63
|
+
end
|
64
|
+
|
65
|
+
|
66
|
+
# refresh cache entry after write
|
67
|
+
alias_method :__cached8a_write, :write
|
68
|
+
def write(d8m, &block)
|
69
|
+
@cache.delete(d8m)
|
70
|
+
result = __cached8a_write(d8m, &block)
|
71
|
+
self[d8m] # re-cache values
|
72
|
+
result
|
73
|
+
end
|
74
|
+
|
75
|
+
|
76
|
+
# clear cache entry after delete
|
77
|
+
alias_method :__cached8a_delete, :delete
|
78
|
+
def delete(d8m)
|
79
|
+
@cache.delete(d8m)
|
80
|
+
__cached8a_delete(d8m)
|
81
|
+
end
|
82
|
+
|
83
|
+
|
84
|
+
# rewrite .d8acache on flush
|
85
|
+
alias_method :__cached8a_flush, :flush
|
86
|
+
def flush
|
87
|
+
__cached8a_write(".d8acache") do |f|
|
88
|
+
f.write(YAML.dump(@cache))
|
89
|
+
end
|
90
|
+
__cached8a_flush
|
91
|
+
end
|
92
|
+
end
|
93
|
+
end
|
94
|
+
|
95
|
+
|
96
|
+
# return cached data, deleting any invalid entries and optionally
|
97
|
+
# creating missing entries
|
98
|
+
def cache_entry(d8m)
|
99
|
+
@cache.delete(d8m) if @cache[d8m] &&
|
100
|
+
@id_attrs.any? { |attr| @cache[d8m][attr] != __send__(attr, d8m) }
|
101
|
+
@cache[d8m]
|
102
|
+
end
|
103
|
+
private :cache_entry
|
104
|
+
|
105
|
+
|
106
|
+
# defines a new synthetic attribute that exists only in the cache
|
107
|
+
def synthetic_attr(*new_attrs)
|
108
|
+
new_attrs.each do |new_attr|
|
109
|
+
instance_eval %Q!
|
110
|
+
class << self
|
111
|
+
def #{new_attr.to_s}(d8m, d8m_attrs = nil)
|
112
|
+
cached = cache_entry(d8m)
|
113
|
+
cached ? cached[#{new_attr.to_s}] : nil
|
114
|
+
end
|
115
|
+
|
116
|
+
def #{new_attr.to_s}=(d8m, val)
|
117
|
+
self[d8m] # populate cache
|
118
|
+
@cache[d8m][#{new_attr.inspect}] = val
|
119
|
+
end
|
120
|
+
end!
|
121
|
+
@attrs.push(new_attr)
|
122
|
+
end
|
123
|
+
end
|
124
|
+
end
|
data/lib/d8a/d8a.rb
ADDED
@@ -0,0 +1,205 @@
|
|
1
|
+
# An abstract collection of named data. In addition
|
2
|
+
# to a name, each datum in the collection has
|
3
|
+
# a set of attributes associated with it. This base
|
4
|
+
# class is abstract; users should instantiate one of
|
5
|
+
# the concrete subclasses of this class.
|
6
|
+
#
|
7
|
+
# Subclasses should provide the following methods:
|
8
|
+
#
|
9
|
+
# * an each method that yields the name of each datum
|
10
|
+
#
|
11
|
+
# * a read(d8m) {|io|} method. The block will be
|
12
|
+
# invoked with an IO-like object that minimally supports
|
13
|
+
# a read method with the semantics of IO.read. io is
|
14
|
+
# automatically closed when the block returns. read()
|
15
|
+
# returns the value of the block.
|
16
|
+
#
|
17
|
+
# * a write(d8m) {|io|} method. The block will be
|
18
|
+
# invoked with an IO-like object that minimally supports
|
19
|
+
# a write method with the semantics of IO.write. io is
|
20
|
+
# automatically closed when the block returns. write()
|
21
|
+
# returns the value of the block.
|
22
|
+
#
|
23
|
+
# * a delete(d8m) method
|
24
|
+
#
|
25
|
+
# * a flush method that callers can invok to indicate they
|
26
|
+
# are finished
|
27
|
+
#
|
28
|
+
# Attributes of datum can be defined by subclasses or modules
|
29
|
+
# by appending the attribute's symbol to @attrs and supplying
|
30
|
+
# the following methods:
|
31
|
+
#
|
32
|
+
# * <attr>(d8m, d8mattrs=nil) that returns the
|
33
|
+
# value of the requested attribute for the given d8m.
|
34
|
+
# If available, other attributes for the d8m will be
|
35
|
+
# supplied in the d8mattrs hash.
|
36
|
+
#
|
37
|
+
|
38
|
+
# * <attr>=(d8m, val) that sets the value of the attribute. This is
|
39
|
+
# optional; the attribute will be read-only if this method is not
|
40
|
+
# provided.
|
41
|
+
|
42
|
+
class D8a
|
43
|
+
include Enumerable
|
44
|
+
|
45
|
+
# D8a name
|
46
|
+
attr :d8aname
|
47
|
+
|
48
|
+
|
49
|
+
# Attrs for data
|
50
|
+
attr :attrs
|
51
|
+
|
52
|
+
|
53
|
+
# Identity attrs for data. Users of this D8a can assume that a
|
54
|
+
# datum has not changed if all of these attributes remain constant.
|
55
|
+
attr :id_attrs
|
56
|
+
|
57
|
+
|
58
|
+
# Initialization
|
59
|
+
def initialize(d8aname)
|
60
|
+
@d8aname = d8aname
|
61
|
+
@attrs = [:name]
|
62
|
+
@id_attrs = []
|
63
|
+
end
|
64
|
+
|
65
|
+
|
66
|
+
# :name attribute
|
67
|
+
def name(d8m, *d8mattrs)
|
68
|
+
d8m
|
69
|
+
end
|
70
|
+
|
71
|
+
|
72
|
+
# Returns information about a named datum.
|
73
|
+
def [](d8m, attr = nil)
|
74
|
+
if attr
|
75
|
+
@attrs.include?(attr) ? __send__(attr, d8m) : nil
|
76
|
+
|
77
|
+
else
|
78
|
+
@attrs.inject({:name => d8m}) { |d8mattrs, attr|
|
79
|
+
val = __send__(attr, d8m, d8mattrs)
|
80
|
+
d8mattrs[attr] = val if val
|
81
|
+
d8mattrs
|
82
|
+
}
|
83
|
+
end
|
84
|
+
end
|
85
|
+
|
86
|
+
|
87
|
+
# Sets information about a named datum.
|
88
|
+
def []=(d8m, attr, val = nil)
|
89
|
+
if attr.is_a?(Hash)
|
90
|
+
attr.each { |k, v| self[d8m, k] = v }
|
91
|
+
|
92
|
+
elsif @attrs.include?(attr)
|
93
|
+
begin
|
94
|
+
__send__((attr.to_s + "=").to_sym, d8m, val)
|
95
|
+
rescue NoMethodError
|
96
|
+
nil
|
97
|
+
end
|
98
|
+
end
|
99
|
+
end
|
100
|
+
|
101
|
+
|
102
|
+
# dummy implementation
|
103
|
+
def each
|
104
|
+
raise "each not implemented by this D8a"
|
105
|
+
end
|
106
|
+
|
107
|
+
|
108
|
+
# dummy implementation
|
109
|
+
def read(d8m)
|
110
|
+
raise "read(d8m) not implemented by this D8a"
|
111
|
+
end
|
112
|
+
|
113
|
+
|
114
|
+
# dummy implementation
|
115
|
+
def write(d8m)
|
116
|
+
raise "write(d8m) not implemented by this D8a"
|
117
|
+
end
|
118
|
+
|
119
|
+
|
120
|
+
# dummy implementation
|
121
|
+
def delete(d8m)
|
122
|
+
raise "delete(d8m) not implemented by this D8a"
|
123
|
+
end
|
124
|
+
|
125
|
+
|
126
|
+
# flush any cached data
|
127
|
+
def flush
|
128
|
+
end
|
129
|
+
|
130
|
+
|
131
|
+
# returns the different attributes for two d8m
|
132
|
+
def diff_d8m(d8m1, d8m2, d8a1, d8a2)
|
133
|
+
name = (d8m1||d8m2)[:name]
|
134
|
+
|
135
|
+
d8m1 ||= d8a1.attrs.inject({}) { |attrs,attr| attrs[attr] = nil; attrs }
|
136
|
+
d8m2 ||= d8a2.attrs.inject({}) { |attrs,attr| attrs[attr] = nil; attrs }
|
137
|
+
|
138
|
+
(d8m1.keys & d8m2.keys).find_all { |attr|
|
139
|
+
begin
|
140
|
+
d8m1[attr] != d8m2[attr]
|
141
|
+
rescue
|
142
|
+
true
|
143
|
+
end
|
144
|
+
}
|
145
|
+
end
|
146
|
+
|
147
|
+
|
148
|
+
# invokes block for each diff, returns entries in this not in o
|
149
|
+
def diff(o)
|
150
|
+
each do |d8m|
|
151
|
+
begin
|
152
|
+
myd8m = self[d8m]
|
153
|
+
od8m = o[d8m]
|
154
|
+
yield(myd8m, od8m) unless diff_d8m(myd8m, od8m, self, o).empty?
|
155
|
+
rescue Errno::ENOENT
|
156
|
+
yield(myd8m, nil)
|
157
|
+
end
|
158
|
+
end
|
159
|
+
|
160
|
+
o.each do |d8m|
|
161
|
+
begin
|
162
|
+
myd8m = self[d8m]
|
163
|
+
rescue Errno::ENOENT
|
164
|
+
yield(nil, o[d8m])
|
165
|
+
end
|
166
|
+
end
|
167
|
+
end
|
168
|
+
|
169
|
+
|
170
|
+
# Generates a report of differences as {name -> [:attr1,:attr2...]}
|
171
|
+
def diffreport(o)
|
172
|
+
result = {}
|
173
|
+
|
174
|
+
diff(o) do |a,b|
|
175
|
+
result[(a||b)[:name]] = diff_d8m(a, b, self, o)
|
176
|
+
end
|
177
|
+
|
178
|
+
result
|
179
|
+
end
|
180
|
+
|
181
|
+
|
182
|
+
# copy one datum to another
|
183
|
+
def copy(fromd8a, fromd8m, tod8a, tod8m = fromd8m)
|
184
|
+
fromd8a.read(fromd8m) do |r|
|
185
|
+
tod8a.write(tod8m) do |w|
|
186
|
+
while s = r.read(65536)
|
187
|
+
w.write(s)
|
188
|
+
end
|
189
|
+
end
|
190
|
+
end
|
191
|
+
end
|
192
|
+
|
193
|
+
|
194
|
+
# clones this D8a to another
|
195
|
+
def sync(to)
|
196
|
+
diff(to) do |myd8m,tod8m|
|
197
|
+
if myd8m
|
198
|
+
copy(self, myd8m[:name], to)
|
199
|
+
to[myd8m[:name]] = myd8m
|
200
|
+
else
|
201
|
+
to.delete(tod8m[:name])
|
202
|
+
end
|
203
|
+
end
|
204
|
+
end
|
205
|
+
end
|
@@ -0,0 +1,83 @@
|
|
1
|
+
require 'digest/md5'
|
2
|
+
require 'd8a/cached8a'
|
3
|
+
|
4
|
+
# Add digesting to an IO object
|
5
|
+
module DigestIO
|
6
|
+
attr_accessor :digest
|
7
|
+
|
8
|
+
def DigestIO.extended(o)
|
9
|
+
class << o
|
10
|
+
alias_method :__digestio_read, :read
|
11
|
+
def read(*args)
|
12
|
+
s = __digestio_read(*args)
|
13
|
+
@digest << s if s
|
14
|
+
s
|
15
|
+
end
|
16
|
+
|
17
|
+
alias_method :__digestio_write, :write
|
18
|
+
def write(s)
|
19
|
+
i = __digestio_write(s)
|
20
|
+
@digest << s[0,i]
|
21
|
+
i
|
22
|
+
end
|
23
|
+
end
|
24
|
+
|
25
|
+
o.digest = Digest::MD5.new
|
26
|
+
end
|
27
|
+
end
|
28
|
+
|
29
|
+
|
30
|
+
# Add MD5 digest attributes to data.
|
31
|
+
module DigestD8a
|
32
|
+
# d8m -> [mod time, digest] info for datum that we've seen
|
33
|
+
attr_accessor :digests
|
34
|
+
|
35
|
+
|
36
|
+
def DigestD8a.extended(d8a)
|
37
|
+
d8a.extend CacheD8a
|
38
|
+
d8a.synthetic_attr :md5
|
39
|
+
|
40
|
+
|
41
|
+
# redefine read/write to capture digests
|
42
|
+
class << d8a
|
43
|
+
alias_method :__digestd8a_read, :read
|
44
|
+
def read(d8m)
|
45
|
+
__digestd8a_read(d8m) do |f|
|
46
|
+
f.extend(DigestIO)
|
47
|
+
result = yield f
|
48
|
+
self[d8m, :md5] = f.digest.to_s
|
49
|
+
result
|
50
|
+
end
|
51
|
+
end
|
52
|
+
|
53
|
+
|
54
|
+
alias_method :__digestd8a_write, :write
|
55
|
+
def write(d8m)
|
56
|
+
digest = nil
|
57
|
+
__digestd8a_write(d8m) do |f|
|
58
|
+
f.extend(DigestIO)
|
59
|
+
digest = f.digest
|
60
|
+
yield f
|
61
|
+
end
|
62
|
+
|
63
|
+
# have to set this after the block since the id_attrs check
|
64
|
+
# in the caching logic will blow away anything we set before
|
65
|
+
# write returns
|
66
|
+
|
67
|
+
self[d8m, :md5] = digest.to_s
|
68
|
+
end
|
69
|
+
end
|
70
|
+
end
|
71
|
+
|
72
|
+
|
73
|
+
# forces recomputation of digest for a d8m
|
74
|
+
def compute_digest(d8m)
|
75
|
+
read(d8m) {|f|}
|
76
|
+
end
|
77
|
+
|
78
|
+
|
79
|
+
# force recomputation of all digests
|
80
|
+
def compute_digests
|
81
|
+
self.each { |d8m| compute_md5(d8m) }
|
82
|
+
end
|
83
|
+
end
|
data/lib/d8a/filed8a.rb
ADDED
@@ -0,0 +1,71 @@
|
|
1
|
+
require 'd8a/d8a'
|
2
|
+
require 'find'
|
3
|
+
|
4
|
+
# D8a based on a directory tree.
|
5
|
+
class FileD8a < D8a
|
6
|
+
# base directory
|
7
|
+
attr :basedir
|
8
|
+
|
9
|
+
|
10
|
+
def initialize(basedir, name = File.basename(basedir))
|
11
|
+
super(name)
|
12
|
+
|
13
|
+
File.directory?(basedir) || Dir.mkdir(basedir)
|
14
|
+
@basedir = basedir
|
15
|
+
|
16
|
+
@attrs.push(:size, :mtime, :mode)
|
17
|
+
@id_attrs.push(:size, :mtime)
|
18
|
+
end
|
19
|
+
|
20
|
+
|
21
|
+
# :size attribute
|
22
|
+
def size(d8m, d8mattrs = nil)
|
23
|
+
File.size(File.join(@basedir, d8m))
|
24
|
+
end
|
25
|
+
|
26
|
+
|
27
|
+
# :mtime attribute
|
28
|
+
def mtime(d8m, d8mattrs = nil)
|
29
|
+
File.mtime(File.join(@basedir, d8m))
|
30
|
+
end
|
31
|
+
|
32
|
+
def mtime=(d8m, mtime)
|
33
|
+
File.utime(mtime, mtime, File.join(@basedir, d8m))
|
34
|
+
end
|
35
|
+
|
36
|
+
|
37
|
+
# :mode attribute
|
38
|
+
def mode(d8m, d8mattrs = nil)
|
39
|
+
File.stat(File.join(@basedir, d8m)).mode & 0777
|
40
|
+
end
|
41
|
+
|
42
|
+
def mode=(d8m, mode)
|
43
|
+
File.chmod(mode, File.join(@basedir, d8m))
|
44
|
+
end
|
45
|
+
|
46
|
+
|
47
|
+
|
48
|
+
def each
|
49
|
+
Find.find(@basedir) do |f|
|
50
|
+
File.file?(f) && yield(f[(@basedir.length + 1)..-1])
|
51
|
+
end
|
52
|
+
end
|
53
|
+
|
54
|
+
|
55
|
+
# Reads a d8m
|
56
|
+
def read(d8m, &block)
|
57
|
+
File.open(File.join(@basedir, d8m), "r", &block)
|
58
|
+
end
|
59
|
+
|
60
|
+
|
61
|
+
# Writes a d8m
|
62
|
+
def write(d8m, &block)
|
63
|
+
File.open(File.join(@basedir, d8m), "w", &block)
|
64
|
+
end
|
65
|
+
|
66
|
+
|
67
|
+
# Deletes a d8m
|
68
|
+
def delete(d8m)
|
69
|
+
File.delete(File.join(@basedir, d8m))
|
70
|
+
end
|
71
|
+
end
|
data/lib/d8a/ftpd8a.rb
ADDED
@@ -0,0 +1,132 @@
|
|
1
|
+
require 'd8a/d8a'
|
2
|
+
require 'date'
|
3
|
+
require 'net/ftp'
|
4
|
+
|
5
|
+
# D8a based on an ftp site.
|
6
|
+
class FtpD8a < D8a
|
7
|
+
# Proc(line, state_array) -> {} to process ls -R lines from server
|
8
|
+
attr :ls_parser
|
9
|
+
|
10
|
+
|
11
|
+
# set up a new Net::FTP connection
|
12
|
+
def FtpD8a.open(host, dir = nil, user = nil, passwd = nil, &block)
|
13
|
+
FtpD8a.new("ftp://#{user || ''}#{user ? '@' : ''}#{host}/#{dir || ''}") do |ftp|
|
14
|
+
ftp.close unless !ftp || ftp.closed?
|
15
|
+
ftp = Net::FTP.new(host, user, passwd)
|
16
|
+
ftp.chdir(dir) if dir
|
17
|
+
ftp
|
18
|
+
end
|
19
|
+
end
|
20
|
+
|
21
|
+
|
22
|
+
# initializer
|
23
|
+
def initialize(name, ftp = nil, &block)
|
24
|
+
super(name)
|
25
|
+
|
26
|
+
@ftp_gen = block
|
27
|
+
|
28
|
+
@ftp = ftp
|
29
|
+
ensure_ftp
|
30
|
+
|
31
|
+
@attrs.push(:size, :ftpmtime, :mode)
|
32
|
+
@id_attrs.push(:size, :ftpmtime)
|
33
|
+
|
34
|
+
|
35
|
+
@ls_parser = Proc.new { |line,state|
|
36
|
+
case line
|
37
|
+
when /^$/
|
38
|
+
nil
|
39
|
+
|
40
|
+
when /^([-d])([-r][-w][-x][-r][-w][-x][-r][-w][-x])\s+\d+\s+\d+\s+\d+\s+(\d+)\s+(.{12})\s+(.*)$/
|
41
|
+
next nil if $1 == 'd'
|
42
|
+
|
43
|
+
name = state[0] + $5
|
44
|
+
size = $3.to_i
|
45
|
+
mode = eval("0b#{$2.tr('-rwx', '01')}")
|
46
|
+
ftpmtime =
|
47
|
+
case $4
|
48
|
+
when /^((Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)\s+\d{1,2}\s+\d{4})$/
|
49
|
+
Date.strptime($2, '%b %e %Y')
|
50
|
+
|
51
|
+
when /^(Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)\s+(\d{1,2})\s+(\d{1,2}:\d{2})$/
|
52
|
+
# TODO - this seems kind of clunky...
|
53
|
+
now = DateTime.now
|
54
|
+
Date.strptime("#{$1} #{$2} #{now.year}", '%b %e %Y') > now ?
|
55
|
+
Date.strptime("#{$1} #{$2} #{now.year-1} #{$3}", '%b %e %Y %R') :
|
56
|
+
Date.strptime("#{$1} #{$2} #{now.year} #{$3}", '%b %e %Y %R')
|
57
|
+
end
|
58
|
+
|
59
|
+
name.sub!(%r{^./}, '')
|
60
|
+
|
61
|
+
{ :name => name, :size => size, :mode => mode, :ftpmtime => ftpmtime }
|
62
|
+
|
63
|
+
when /^(.*):$/
|
64
|
+
state[0] = $1 + "/"
|
65
|
+
end
|
66
|
+
}
|
67
|
+
end
|
68
|
+
|
69
|
+
|
70
|
+
private
|
71
|
+
|
72
|
+
# (re)sets up @ftp with an FTP connection to the host
|
73
|
+
def ensure_ftp
|
74
|
+
begin
|
75
|
+
@ftp.pwd
|
76
|
+
rescue
|
77
|
+
@ftp = @ftp_gen.call(@ftp) if @ftp_gen
|
78
|
+
@ftp.pwd
|
79
|
+
end
|
80
|
+
end
|
81
|
+
|
82
|
+
|
83
|
+
public
|
84
|
+
|
85
|
+
# size attr
|
86
|
+
def size(d8m, d8mattrs = nil)
|
87
|
+
@ftpcache[d8m] && @ftpcache[d8m][:size]
|
88
|
+
end
|
89
|
+
|
90
|
+
|
91
|
+
# ftpmtime attr
|
92
|
+
def ftpmtime(d8m, d8mattrs = nil)
|
93
|
+
@ftpcache[d8m] && @ftpcache[d8m][:ftpmtime]
|
94
|
+
end
|
95
|
+
|
96
|
+
|
97
|
+
# mode attr
|
98
|
+
def mode(d8m, d8mattrs = nil)
|
99
|
+
@ftpcache[d8m] && @ftpcache[d8m][:mode]
|
100
|
+
end
|
101
|
+
|
102
|
+
|
103
|
+
def each
|
104
|
+
ensure_ftp
|
105
|
+
dir = [""]
|
106
|
+
@ftpcache = {}
|
107
|
+
@ftp.list("-R") do |line|
|
108
|
+
if (x = @ls_parser.call(line, dir)).is_a?(Hash)
|
109
|
+
@ftpcache[x[:name]] = x
|
110
|
+
yield x[:name]
|
111
|
+
end
|
112
|
+
end
|
113
|
+
end
|
114
|
+
|
115
|
+
|
116
|
+
def read(d8m)
|
117
|
+
end
|
118
|
+
|
119
|
+
|
120
|
+
def write(d8m)
|
121
|
+
end
|
122
|
+
|
123
|
+
|
124
|
+
def delete(d8m)
|
125
|
+
end
|
126
|
+
|
127
|
+
|
128
|
+
def flush
|
129
|
+
#@ftp.close unless !@ftp || @ftp.closed?
|
130
|
+
#@ftp = nil
|
131
|
+
end
|
132
|
+
end
|
data/tests/d8atest.rb
ADDED
@@ -0,0 +1,58 @@
|
|
1
|
+
require 'fileutils'
|
2
|
+
require 'test/unit'
|
3
|
+
require 'tmpdir'
|
4
|
+
|
5
|
+
$:.unshift File.join(File.dirname(__FILE__), "..", "lib") unless
|
6
|
+
$:.include?(File.join(File.dirname(__FILE__), "..", "lib"))
|
7
|
+
|
8
|
+
|
9
|
+
module TestD8a
|
10
|
+
include FileUtils
|
11
|
+
|
12
|
+
def setup_dirname(name)
|
13
|
+
File.join(Dir.tmpdir, "TestFileD8a", name)
|
14
|
+
end
|
15
|
+
|
16
|
+
|
17
|
+
def setup_filed8a(name, files)
|
18
|
+
dir = setup_dirname(name)
|
19
|
+
Dir.mkdir(dir)
|
20
|
+
|
21
|
+
files.each do |file|
|
22
|
+
fname = File.join(dir, file[0])
|
23
|
+
File.open(fname, "w") { |f| f.write("x" * file[1]) }
|
24
|
+
File.utime(file[2], file[2], fname)
|
25
|
+
end
|
26
|
+
|
27
|
+
@d8as[name] = { :dir => dir, :files => files }
|
28
|
+
end
|
29
|
+
|
30
|
+
|
31
|
+
def setup
|
32
|
+
@now = Time.at(Time.now.to_i)
|
33
|
+
|
34
|
+
@rootdir = File.join(Dir.tmpdir, "TestFileD8a")
|
35
|
+
File.directory?(@rootdir) || Dir.mkdir(@rootdir)
|
36
|
+
|
37
|
+
@d8as = {}
|
38
|
+
|
39
|
+
setup_filed8a("b", [["t0-s0", 0, Time.at(0)], ["tnow-s1000", 1000, @now]])
|
40
|
+
setup_filed8a("a", [])
|
41
|
+
end
|
42
|
+
|
43
|
+
|
44
|
+
def teardown
|
45
|
+
rm_rf(@rootdir) unless @skip_teardown
|
46
|
+
@d8as.clear
|
47
|
+
end
|
48
|
+
|
49
|
+
|
50
|
+
def assert_d8aequal(d8a1, d8a2)
|
51
|
+
errors = []
|
52
|
+
d8a1.diff(d8a2) do |d8m1,d8m2|
|
53
|
+
errors << [d8m1,d8m2]
|
54
|
+
end
|
55
|
+
|
56
|
+
assert_equal([], errors)
|
57
|
+
end
|
58
|
+
end
|
@@ -0,0 +1,49 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
require 'test/unit'
|
4
|
+
|
5
|
+
$:.unshift File.join(File.dirname(__FILE__), "..", "lib")
|
6
|
+
$:.unshift File.join(File.dirname(__FILE__))
|
7
|
+
|
8
|
+
require 'd8atest'
|
9
|
+
require 'd8a/filed8a'
|
10
|
+
require 'd8a/cached8a'
|
11
|
+
|
12
|
+
|
13
|
+
class TestCacheD8a < Test::Unit::TestCase
|
14
|
+
include TestD8a
|
15
|
+
|
16
|
+
def test_cache
|
17
|
+
# add caching to a D8a
|
18
|
+
bd8a = FileD8a.new(@d8as["b"][:dir])
|
19
|
+
bd8a.extend CacheD8a
|
20
|
+
|
21
|
+
# create a phantom attr
|
22
|
+
bd8a.synthetic_attr :foo
|
23
|
+
|
24
|
+
assert(bd8a.respond_to?(:foo))
|
25
|
+
assert_equal(-2, bd8a.method(:foo).arity)
|
26
|
+
assert(bd8a.respond_to?(:foo=))
|
27
|
+
assert_equal(2, bd8a.method(:foo=).arity)
|
28
|
+
|
29
|
+
# set new attr on the files
|
30
|
+
bd8a["t0-s0", :foo] = 0
|
31
|
+
bd8a["tnow-s1000", :foo] = @now.to_i
|
32
|
+
|
33
|
+
# flush cache
|
34
|
+
bd8a.flush
|
35
|
+
|
36
|
+
# change timestamp on t0-s0 to invalidate cache entry
|
37
|
+
File.utime(1000, 1000, File.join(@d8as["b"][:dir], "t0-s0"))
|
38
|
+
|
39
|
+
# new D8a for same dir
|
40
|
+
xd8a = FileD8a.new(@d8as["b"][:dir])
|
41
|
+
xd8a.extend CacheD8a
|
42
|
+
|
43
|
+
# verify attr is still there
|
44
|
+
xd8a.synthetic_attr :foo
|
45
|
+
|
46
|
+
assert_nil(xd8a["t0-s0", :foo])
|
47
|
+
assert_equal(@now.to_i, xd8a["tnow-s1000", :foo])
|
48
|
+
end
|
49
|
+
end
|
@@ -0,0 +1,55 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
require 'test/unit'
|
4
|
+
|
5
|
+
$:.unshift File.join(File.dirname(__FILE__), "..", "lib")
|
6
|
+
$:.unshift File.join(File.dirname(__FILE__))
|
7
|
+
|
8
|
+
require 'd8atest'
|
9
|
+
require 'd8a/filed8a'
|
10
|
+
require 'd8a/digestd8a'
|
11
|
+
|
12
|
+
|
13
|
+
class TestDigestD8a < Test::Unit::TestCase
|
14
|
+
include TestD8a
|
15
|
+
|
16
|
+
def test_digest
|
17
|
+
# to compute our own digests
|
18
|
+
require "digest/md5"
|
19
|
+
|
20
|
+
# add digests to a D8a
|
21
|
+
bd8a = FileD8a.new(@d8as["b"][:dir])
|
22
|
+
bd8a.extend DigestD8a
|
23
|
+
|
24
|
+
assert(bd8a.respond_to?(:md5))
|
25
|
+
assert(bd8a.respond_to?(:md5=))
|
26
|
+
|
27
|
+
# should not be any digest to start with
|
28
|
+
assert_nil(bd8a["t0-s0", :md5])
|
29
|
+
|
30
|
+
# verify forced computation of digest
|
31
|
+
bd8a.compute_digest("t0-s0")
|
32
|
+
assert_equal(Digest::MD5.new.to_s, bd8a["t0-s0", :md5])
|
33
|
+
|
34
|
+
# verify digest is computed on read
|
35
|
+
assert_nil(bd8a["tnow-s1000", :md5])
|
36
|
+
bd8a.read("tnow-s1000") { |f|
|
37
|
+
while s = f.read(10000)
|
38
|
+
end
|
39
|
+
}
|
40
|
+
assert_equal((Digest::MD5.new << "x" * 1000).to_s,
|
41
|
+
bd8a["tnow-s1000", :md5])
|
42
|
+
|
43
|
+
# verify digest is computed on write
|
44
|
+
bd8a.write("tnow-s1000") { |f| f.write("z" * 1000) }
|
45
|
+
assert_equal(d = (Digest::MD5.new << "z" * 1000).to_s,
|
46
|
+
bd8a["tnow-s1000", :md5])
|
47
|
+
|
48
|
+
# verifiy digest is cached correctly
|
49
|
+
bd8a.flush
|
50
|
+
xd8a = FileD8a.new(@d8as["b"][:dir])
|
51
|
+
xd8a.extend DigestD8a
|
52
|
+
|
53
|
+
assert_equal(d, xd8a["tnow-s1000", :md5])
|
54
|
+
end
|
55
|
+
end
|
data/tests/tc_filed8a.rb
ADDED
@@ -0,0 +1,98 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
require 'test/unit'
|
4
|
+
|
5
|
+
$:.unshift File.join(File.dirname(__FILE__)) unless
|
6
|
+
$:.include?(File.join(File.dirname(__FILE__)))
|
7
|
+
|
8
|
+
require 'd8atest'
|
9
|
+
require 'd8a/filed8a'
|
10
|
+
require 'd8a/cached8a'
|
11
|
+
|
12
|
+
|
13
|
+
class TestFileD8a < Test::Unit::TestCase
|
14
|
+
include TestD8a
|
15
|
+
|
16
|
+
def assert_contents(d8a, name = d8a.d8aname)
|
17
|
+
files = @d8as[name][:files].clone
|
18
|
+
errors = []
|
19
|
+
|
20
|
+
d8a.each do |d8m|
|
21
|
+
info = [d8a[d8m,:name], d8a[d8m,:size], d8a[d8m,:mtime]]
|
22
|
+
files.delete(info) || errors << "unrecognized d8m: #{info.inspect}"
|
23
|
+
end
|
24
|
+
|
25
|
+
files.each do |notfound|
|
26
|
+
errors << "missing d8m: #{notfound.inspect}"
|
27
|
+
end
|
28
|
+
|
29
|
+
assert_equal([], errors)
|
30
|
+
end
|
31
|
+
|
32
|
+
|
33
|
+
def assert_diffreportequal(dr1, dr2)
|
34
|
+
assert(dr1.size == dr2.size &&
|
35
|
+
dr1.all? { |d8m,attrs| (attrs - dr2[d8m]).empty? },
|
36
|
+
"#{dr1.inspect}\n#{dr2.inspect}")
|
37
|
+
end
|
38
|
+
|
39
|
+
|
40
|
+
|
41
|
+
def test_file
|
42
|
+
# validate known set of files
|
43
|
+
ad8a = FileD8a.new(@d8as["a"][:dir])
|
44
|
+
assert_contents(ad8a)
|
45
|
+
|
46
|
+
bd8a = FileD8a.new(@d8as["b"][:dir])
|
47
|
+
assert_contents(bd8a)
|
48
|
+
|
49
|
+
|
50
|
+
# create one from scratch
|
51
|
+
cdir = setup_dirname("c")
|
52
|
+
cd8a = FileD8a.new(cdir)
|
53
|
+
|
54
|
+
files = @d8as["b"][:files]
|
55
|
+
files.each do |f|
|
56
|
+
cd8a.write(f[0]) { |io| io.write('x' * f[1]) }
|
57
|
+
cd8a[f[0], :mtime] = f[2]
|
58
|
+
end
|
59
|
+
assert_equal(files.size + 2, Dir.entries(cdir).size)
|
60
|
+
files.each do |f|
|
61
|
+
assert_equal(f[1], cd8a[f[0], :size])
|
62
|
+
assert_equal(f[2], cd8a[f[0], :mtime])
|
63
|
+
end
|
64
|
+
|
65
|
+
|
66
|
+
# b and c should be identical
|
67
|
+
assert_d8aequal(cd8a, bd8a)
|
68
|
+
|
69
|
+
|
70
|
+
# create some diffs
|
71
|
+
File.open(File.join(@d8as["b"][:dir], "justb"), "w") do |f|
|
72
|
+
f.write("This file exists only in b.\n")
|
73
|
+
end
|
74
|
+
|
75
|
+
File.open(File.join(cdir, "justc"), "w") do |f|
|
76
|
+
f.write("This file exists only in c.\n")
|
77
|
+
end
|
78
|
+
|
79
|
+
File.open(File.join(@d8as["b"][:dir], "diffsize"), "w") do |f|
|
80
|
+
f.write("This file has different sizes in b & c.\n")
|
81
|
+
end
|
82
|
+
File.open(File.join(cdir, "diffsize"), "w") do |f|
|
83
|
+
f.write("This file has different sizes in b & c. C is longer.\n")
|
84
|
+
end
|
85
|
+
t = File.mtime(File.join(@d8as["b"][:dir], "diffsize"))
|
86
|
+
File.utime(t, t, File.join(cdir, "diffsize"))
|
87
|
+
|
88
|
+
File.open(File.join(@d8as["b"][:dir], "diffmtime"), "w") do |f|
|
89
|
+
f.write("This file has different mtimes in b & c.\n")
|
90
|
+
end
|
91
|
+
File.open(File.join(cdir, "diffmtime"), "w") do |f|
|
92
|
+
f.write("This file has different mtimes in b & c.\n")
|
93
|
+
end
|
94
|
+
File.utime(0, 0, File.join(cdir, "diffmtime"))
|
95
|
+
|
96
|
+
assert_diffreportequal({"justb" => bd8a.attrs, "justc" => cd8a.attrs, "diffmtime" => [:mtime], "diffsize" => [:size]}, bd8a.diffreport(cd8a))
|
97
|
+
end
|
98
|
+
end
|
data/tests/ts_d8a.rb
ADDED
metadata
ADDED
@@ -0,0 +1,64 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
rubygems_version: 0.9.0
|
3
|
+
specification_version: 1
|
4
|
+
name: d8a
|
5
|
+
version: !ruby/object:Gem::Version
|
6
|
+
version: 0.0.1
|
7
|
+
date: 2007-02-19 00:00:00 -05:00
|
8
|
+
summary: A uniform data library and associated tools
|
9
|
+
require_paths:
|
10
|
+
- lib
|
11
|
+
email: mike.j.burr@gmail.com
|
12
|
+
homepage: http://d8a.rubyforge.org
|
13
|
+
rubyforge_project:
|
14
|
+
description:
|
15
|
+
autorequire:
|
16
|
+
default_executable:
|
17
|
+
bindir: bin
|
18
|
+
has_rdoc: false
|
19
|
+
required_ruby_version: !ruby/object:Gem::Version::Requirement
|
20
|
+
requirements:
|
21
|
+
- - ">"
|
22
|
+
- !ruby/object:Gem::Version
|
23
|
+
version: 0.0.0
|
24
|
+
version:
|
25
|
+
platform: ruby
|
26
|
+
signing_key:
|
27
|
+
cert_chain:
|
28
|
+
post_install_message:
|
29
|
+
authors:
|
30
|
+
- Mike Burr
|
31
|
+
files:
|
32
|
+
- lib/d8a
|
33
|
+
- lib/d8a/cached8a.rb
|
34
|
+
- lib/d8a/ftpd8a.rb
|
35
|
+
- lib/d8a/d8a.rb
|
36
|
+
- lib/d8a/digestd8a.rb
|
37
|
+
- lib/d8a/filed8a.rb
|
38
|
+
- tests/tc_digestd8a.rb
|
39
|
+
- tests/tc_filed8a.rb
|
40
|
+
- tests/ts_d8a.rb
|
41
|
+
- tests/tc_cached8a.rb
|
42
|
+
- tests/d8atest.rb
|
43
|
+
test_files:
|
44
|
+
- tests/ts_d8a.rb
|
45
|
+
rdoc_options: []
|
46
|
+
|
47
|
+
extra_rdoc_files: []
|
48
|
+
|
49
|
+
executables: []
|
50
|
+
|
51
|
+
extensions: []
|
52
|
+
|
53
|
+
requirements: []
|
54
|
+
|
55
|
+
dependencies:
|
56
|
+
- !ruby/object:Gem::Dependency
|
57
|
+
name: urirequire
|
58
|
+
version_requirement:
|
59
|
+
version_requirements: !ruby/object:Gem::Version::Requirement
|
60
|
+
requirements:
|
61
|
+
- - ">="
|
62
|
+
- !ruby/object:Gem::Version
|
63
|
+
version: 0.1.0
|
64
|
+
version:
|