excavate 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,199 @@
1
+ # code is obtained from https://github.com/jordansissel/ruby-arr-pm/blob/8071591173ebb90dea27d5acfdde69a37dcb2744/cpio.rb
2
+ # rubocop:disable all
3
+ class BoundedIO
4
+ attr_reader :length
5
+ attr_reader :remaining
6
+
7
+ def initialize(io, length, &eof_callback)
8
+ @io = io
9
+ @length = length
10
+ @remaining = length
11
+
12
+ @eof_callback = eof_callback
13
+ @eof = false
14
+ end
15
+
16
+ def read(size=nil)
17
+ return nil if eof?
18
+ size = @remaining if size.nil?
19
+ data = @io.read(size)
20
+ @remaining -= data.bytesize
21
+ eof?
22
+ data
23
+ end
24
+
25
+ def sysread(size)
26
+ raise EOFError, "end of file reached" if eof?
27
+ read(size)
28
+ end
29
+
30
+ def eof?
31
+ return false if @remaining > 0
32
+ return @eof if @eof
33
+
34
+ @eof_callback.call
35
+ @eof = true
36
+ end
37
+ end
38
+
39
+ module CPIO
40
+ FIELDS = [
41
+ :magic, :ino, :mode, :uid, :gid, :nlink, :mtime, :filesize, :devmajor,
42
+ :devminor, :rdevmajor, :rdevminor, :namesize, :check
43
+ ]
44
+ end
45
+
46
+ class CPIO::ASCIIReader
47
+ FIELD_SIZES = {
48
+ :magic => 6,
49
+ :ino => 8,
50
+ :mode => 8,
51
+ :uid => 8,
52
+ :gid => 8,
53
+ :nlink => 8,
54
+ :mtime => 8,
55
+ :filesize => 8,
56
+ :devmajor => 8,
57
+ :devminor => 8,
58
+ :rdevmajor => 8,
59
+ :rdevminor => 8,
60
+ :namesize => 8,
61
+ :check => 8
62
+ }
63
+ HEADER_LENGTH = FIELD_SIZES.reduce(0) { |m, (_, v)| m + v }
64
+ HEADER_PACK = FIELD_SIZES.collect { |_, v| "A#{v}" }.join
65
+
66
+ FIELD_ORDER = [
67
+ :magic, :ino, :mode, :uid, :gid, :nlink, :mtime, :filesize, :devmajor,
68
+ :devminor, :rdevmajor, :rdevminor, :namesize, :check
69
+ ]
70
+
71
+ def initialize(io)
72
+ @io = io
73
+ end
74
+
75
+ private
76
+
77
+ def io
78
+ @io
79
+ end
80
+
81
+ def each(&block)
82
+ while true
83
+ entry = read
84
+ break if entry.nil?
85
+ # The CPIO format has the end-of-stream marker as a file called "TRAILER!!!"
86
+ break if entry.name == "TRAILER!!!"
87
+ block.call(entry, entry.file)
88
+ verify_correct_read(entry) unless entry.directory?
89
+ end
90
+ end
91
+
92
+ def verify_correct_read(entry)
93
+ # Read and throw away the whole file if not read at all.
94
+ entry.file.tap do |file|
95
+ if file.nil? || file.remaining == 0
96
+ # All OK! :)
97
+ elsif file.remaining == file.length
98
+ file.read(16384) while !file.eof?
99
+ else
100
+ # The file was only partially read? This should be an error by the
101
+ # user.
102
+ consumed = file.length - file.remaining
103
+ raise BadState, "Only #{consumed} bytes were read of the #{file.length} byte file: #{entry.name}"
104
+ end
105
+ end
106
+ end
107
+
108
+ def read
109
+ entry = CPIOEntry.new
110
+ header = io.read(HEADER_LENGTH)
111
+ return nil if header.nil?
112
+ FIELD_ORDER.zip(header.unpack(HEADER_PACK)).each do |field, value|
113
+ entry.send("#{field}=", value.to_i(16))
114
+ end
115
+
116
+ entry.validate
117
+ entry.mtime = Time.at(entry.mtime)
118
+ read_name(entry, @io)
119
+ read_file(entry, @io)
120
+ entry
121
+ end
122
+
123
+ def read_name(entry, io)
124
+ entry.name = io.read(entry.namesize - 1) # - 1 for null terminator
125
+ nul = io.read(1)
126
+ raise ArgumentError, "Corrupt CPIO or bug? Name null terminator was not null: #{nul.inspect}" if nul != "\0"
127
+ padding_data = io.read(padding_name(entry))
128
+ # Padding should be all null bytes
129
+ if padding_data != ("\0" * padding_data.bytesize)
130
+ raise ArgumentError, "Corrupt CPIO or bug? Name null padding was #{padding_name(entry)} bytes: #{padding_data.inspect}"
131
+ end
132
+ end
133
+
134
+ def read_file(entry, io)
135
+ if entry.directory?
136
+ entry.file = nil
137
+ #read_file_padding(entry, io)
138
+ nil
139
+ else
140
+ entry.file = BoundedIO.new(io, entry.filesize) do
141
+ read_file_padding(entry, io)
142
+ end
143
+ end
144
+ end
145
+
146
+ def read_file_padding(entry, io)
147
+ padding_data = io.read(padding_file(entry))
148
+ if padding_data != ("\0" * padding_data.bytesize)
149
+ raise ArgumentError, "Corrupt CPIO or bug? File null padding was #{padding_file(entry)} bytes: #{padding_data.inspect}"
150
+ end
151
+ end
152
+
153
+ def padding_name(entry)
154
+ # name padding is padding up to a multiple of 4 after header+namesize
155
+ -(HEADER_LENGTH + entry.namesize) % 4
156
+ end
157
+
158
+ def padding_file(entry)
159
+ (-(HEADER_LENGTH + entry.filesize + 2) % 4)
160
+ end
161
+ public(:each)
162
+ end
163
+
164
+ class CPIOEntry
165
+ CPIO::FIELDS.each do |field|
166
+ attr_accessor field
167
+ end
168
+
169
+ attr_accessor :name
170
+ attr_accessor :file
171
+
172
+ DIRECTORY_FLAG = 0040000
173
+
174
+ def validate
175
+ raise "Invalid magic #{magic.inspect}" if magic != 0x070701
176
+ raise "Invalid ino #{ino.inspect}" if ino < 0
177
+ raise "Invalid mode #{mode.inspect}" if mode < 0
178
+ raise "Invalid uid #{uid.inspect}" if uid < 0
179
+ raise "Invalid gid #{gid.inspect}" if gid < 0
180
+ raise "Invalid nlink #{nlink.inspect}" if nlink < 0
181
+ raise "Invalid mtime #{mtime.inspect}" if mtime < 0
182
+ raise "Invalid filesize #{filesize.inspect}" if filesize < 0
183
+ raise "Invalid devmajor #{devmajor.inspect}" if devmajor < 0
184
+ raise "Invalid devminor #{devminor.inspect}" if devminor < 0
185
+ raise "Invalid rdevmajor #{rdevmajor.inspect}" if rdevmajor < 0
186
+ raise "Invalid rdevminor #{rdevminor.inspect}" if rdevminor < 0
187
+ raise "Invalid namesize #{namesize.inspect}" if namesize < 0
188
+ raise "Invalid check #{check.inspect}" if check < 0
189
+ end # def validate
190
+
191
+ def read(*args)
192
+ return nil if directory?
193
+ file.read(*args)
194
+ end
195
+
196
+ def directory?
197
+ mode & DIRECTORY_FLAG > 0
198
+ end
199
+ end
@@ -0,0 +1,436 @@
1
+ # code is obtained from https://github.com/thumblemonks/cpio/blob/bad40c293280bb3c1678251c66f0f1f6fb1cc03e/cpio.rb
2
+ # rubocop:disable all
3
+
4
+ require 'stringio'
5
+
6
+ module CPIO
7
+ class ArchiveFormatError < IOError; end
8
+
9
+ class ArchiveHeader
10
+ Magic = '070707'
11
+ Fields = [[6, :magic ],
12
+ [6, :dev ],
13
+ [6, :inode ],
14
+ [6, :mode ],
15
+ [6, :uid ],
16
+ [6, :gid ],
17
+ [6, :numlinks],
18
+ [6, :rdev ],
19
+ [11, :mtime ],
20
+ [6, :namesize],
21
+ [11, :filesize]]
22
+
23
+ FieldDefaults = {:magic => Integer(Magic),
24
+ :dev => 0777777,
25
+ :inode => 0,
26
+ :mode => 0100444,
27
+ :uid => 0,
28
+ :gid => 0,
29
+ :numlinks => 1,
30
+ :rdev => 0,
31
+ :mtime => lambda { Time.now.to_i }}
32
+
33
+ FieldMaxValues = Fields.inject({}) do |map,(width,name)|
34
+ map[name] = Integer("0#{'7' * width}")
35
+ map
36
+ end
37
+
38
+ HeaderSize = Fields.inject(0) do |sum,(size,name)|
39
+ sum + size
40
+ end
41
+
42
+ HeaderUnpackFormat = Fields.collect do |size,name|
43
+ "a%s" % size
44
+ end.join('')
45
+
46
+ Fields.each do |(size,name)|
47
+ define_method(name) { @attrs[name.to_sym] }
48
+ end
49
+
50
+ class << self
51
+ private :new
52
+ end
53
+
54
+ def initialize(attrs)
55
+ @attrs = attrs
56
+ check_attrs
57
+ end
58
+
59
+ def self.from(io)
60
+ data = io.read(HeaderSize)
61
+ verify_size(data)
62
+ verify_magic(data)
63
+ new(unpack_data(data))
64
+ end
65
+
66
+ def self.with_defaults(opts)
67
+ name = opts[:name]
68
+ defaults = FieldDefaults.merge(:mode => opts[:mode], :filesize => opts[:filesize], :namesize => name.size + 1)
69
+ new(defaults)
70
+ end
71
+
72
+ def to_data
73
+ Fields.collect do |(width,name)|
74
+ raise ArchiveFormatError, "Expected header to have key #{name}" unless @attrs.has_key?(name)
75
+ val = @attrs[name].respond_to?(:to_proc) ? @attrs[name].call : @attrs[name]
76
+ raise ArchiveFormatError, "Header value for #{name} exceeds max length of #{FieldMaxValues[name]}" if val > FieldMaxValues[name]
77
+ sprintf("%0*o", Fields.rassoc(name).first, val)
78
+ end.join('')
79
+ end
80
+
81
+ private
82
+
83
+ def check_attrs
84
+ [:mode, :namesize, :filesize].each do |attr|
85
+ raise ArgumentError, "#{attr.inspect} must be given" if !@attrs.has_key?(attr)
86
+ end
87
+ end
88
+
89
+ def self.verify_size(data)
90
+ unless data.size == HeaderSize
91
+ raise ArchiveFormatError, "Header is not long enough to be a valid CPIO archive with ASCII headers."
92
+ end
93
+ end
94
+
95
+ def self.verify_magic(data)
96
+ unless data[0..Magic.size - 1] == Magic
97
+ raise ArchiveFormatError, "Archive does not seem to be a valid CPIO archive with ASCII headers."
98
+ end
99
+ end
100
+
101
+ def self.unpack_data(data)
102
+ contents = {}
103
+ data.unpack(HeaderUnpackFormat).zip(Fields) do |(chunk,(size,name))|
104
+ contents[name] = Integer("0#{chunk}")
105
+ end
106
+ contents
107
+ end
108
+
109
+ end
110
+
111
+ class ArchiveEntry
112
+ TrailerMagic = "TRAILER!!!"
113
+ S_IFMT = 0170000 # bitmask for the file type bitfields
114
+ S_IFREG = 0100000 # regular file
115
+ S_IFDIR = 0040000 # directory
116
+
117
+ ExecutableMask = (0100 | # Owner executable
118
+ 0010 | # Group executable
119
+ 0001) # Other executable
120
+
121
+ attr_reader :filename, :data
122
+
123
+ class << self
124
+ private :new
125
+ end
126
+
127
+ def self.from(io)
128
+ header = ArchiveHeader.from(io)
129
+ filename = read_filename(header, io)
130
+ data = read_data(header, io)
131
+ new(header, filename, data)
132
+ end
133
+
134
+ def self.new_directory(opts)
135
+ mode = S_IFDIR | opts[:mode]
136
+ header = ArchiveHeader.with_defaults(:mode => mode, :name => opts[:name], :filesize => 0)
137
+ new(header, opts[:name], '')
138
+ end
139
+
140
+ def self.new_file(opts)
141
+ mode = S_IFREG | opts[:mode]
142
+ header = ArchiveHeader.with_defaults(:mode => mode, :name => opts[:name], :filesize => opts[:io].size)
143
+ opts[:io].rewind
144
+ new(header, opts[:name], opts[:io].read)
145
+ end
146
+
147
+ def self.new_trailer
148
+ header = ArchiveHeader.with_defaults(:mode => S_IFREG, :name => TrailerMagic, :filesize => 0)
149
+ new(header, TrailerMagic, '')
150
+ end
151
+
152
+ def initialize(header, filename, data)
153
+ @header = header
154
+ @filename = filename
155
+ @data = data
156
+ end
157
+
158
+ def trailer?
159
+ @filename == TrailerMagic && @data.size == 0
160
+ end
161
+
162
+ def directory?
163
+ mode & S_IFMT == S_IFDIR
164
+ end
165
+
166
+ def file?
167
+ mode & S_IFMT == S_IFREG
168
+ end
169
+
170
+ def executable?
171
+ (mode & ExecutableMask) != 0
172
+ end
173
+
174
+ def mode
175
+ @mode ||= sprintf('%o', @header.mode).to_s.oct
176
+ end
177
+
178
+ def to_data
179
+ sprintf("%s%s\000%s", @header.to_data, filename, data)
180
+ end
181
+
182
+ private
183
+
184
+ def self.read_filename(header, io)
185
+ fname = io.read(header.namesize)
186
+ if fname.size != header.namesize
187
+ raise ArchiveFormatError, "Archive header seems to innacurately contain length of filename"
188
+ end
189
+ fname.chomp("\000")
190
+ end
191
+
192
+ def self.read_data(header, io)
193
+ data = io.read(header.filesize)
194
+ if data.size != header.filesize
195
+ raise ArchiveFormatError, "Archive header seems to inaccurately contain length of the entry"
196
+ end
197
+ data
198
+ end
199
+
200
+ end
201
+
202
+ class ArchiveWriter
203
+ class ArchiveFinalizedError < RuntimeError; end
204
+
205
+ def initialize(io)
206
+ @io = io
207
+ @open = false
208
+ end
209
+
210
+ def open?
211
+ @open
212
+ end
213
+
214
+ def open
215
+ raise ArchiveFinalizedError, "This archive has already been finalized" if @finalized
216
+ @open = true
217
+ yield(self)
218
+ ensure
219
+ close
220
+ finalize
221
+ end
222
+
223
+ def mkdir(name, mode = 0555)
224
+ entry = ArchiveEntry.new_directory(:name => name, :mode => mode)
225
+ @io.write(entry.to_data)
226
+ end
227
+
228
+ def add_file(name, mode = 0444)
229
+ file = StringIO.new
230
+ yield(file)
231
+ entry = ArchiveEntry.new_file(:name => name, :mode => mode, :io => file)
232
+ @io.write(entry.to_data)
233
+ end
234
+
235
+ private
236
+
237
+ def add_entry(opts)
238
+ end
239
+
240
+ def write_trailer
241
+ entry = ArchiveEntry.new_trailer
242
+ @io.write(entry.to_data)
243
+ end
244
+
245
+ def finalize
246
+ write_trailer
247
+ @finalized = true
248
+ end
249
+
250
+ def check_open
251
+ raise "#{self.class.name} not open for writing" unless open?
252
+ end
253
+
254
+ def close
255
+ @open = false
256
+ end
257
+
258
+ end # ArchiveWriter
259
+
260
+ class ArchiveReader
261
+
262
+ def initialize(io)
263
+ @io = io
264
+ end
265
+
266
+ def each_entry
267
+ @io.rewind
268
+ while (entry = ArchiveEntry.from(@io)) && !entry.trailer?
269
+ yield(entry)
270
+ end
271
+ end
272
+
273
+ end # ArchiveReader
274
+
275
+ end # CPIO
276
+
277
+ if $PROGRAM_NAME == __FILE__
278
+ require 'stringio'
279
+ require 'test/unit'
280
+ require 'digest/sha1'
281
+
282
+ class CPIOArchiveReaderTest < Test::Unit::TestCase
283
+ CPIOFixture = StringIO.new(DATA.read)
284
+ # These are SHA1 hashes
285
+ ExpectedFixtureHashes = { 'cpio_test/test_executable' => '97bd38305a81f2d89b5f3aa44500ec964b87cf8a',
286
+ 'cpio_test/test_dir/test_file' => 'e7f1aa55a7f83dc99c9978b91072d01a3f5c812e' }
287
+
288
+ def test_given_a_archive_with_a_bad_magic_number_should_raise
289
+ assert_raises(CPIO::ArchiveFormatError) do
290
+ CPIO::ArchiveReader.new(StringIO.new('foo')).each_entry { }
291
+ end
292
+ end
293
+
294
+ def test_given_a_archive_with_a_valid_magic_number_should_not_raise
295
+ archive = CPIO::ArchiveReader.new(CPIOFixture)
296
+ assert_nil archive.each_entry { }
297
+ end
298
+
299
+ def test_given_a_valid_archive_should_have_the_expected_number_of_entries
300
+ archive = CPIO::ArchiveReader.new(CPIOFixture)
301
+ entries = 4
302
+ archive.each_entry { |ent| entries -= 1 }
303
+ assert_equal 0, entries, "Expected #{entries} in the archive."
304
+ end
305
+
306
+ def test_given_a_valid_archive_should_have_the_expected_entry_filenames
307
+ expected = %w[cpio_test cpio_test/test_dir cpio_test/test_dir/test_file cpio_test/test_executable]
308
+ archive = CPIO::ArchiveReader.new(CPIOFixture)
309
+ archive.each_entry { |ent| expected.delete(ent.filename) }
310
+ assert_equal 0, expected.size, "The expected array should be empty but we still have: #{expected.inspect}"
311
+ end
312
+
313
+ def test_given_a_valid_archive_should_have_the_expected_number_of_directories
314
+ expected = 2
315
+ archive = CPIO::ArchiveReader.new(CPIOFixture)
316
+ archive.each_entry { |ent| expected -= 1 if ent.directory? }
317
+ assert_equal 0, expected
318
+ end
319
+
320
+ def test_given_a_valid_archive_should_have_the_expected_number_of_regular_files
321
+ expected = 1
322
+ archive = CPIO::ArchiveReader.new(CPIOFixture)
323
+ archive.each_entry { |ent| expected -= 1 if ent.file? && !ent.executable? }
324
+ assert_equal 0, expected
325
+ end
326
+
327
+ def test_given_a_valid_archive_should_have_the_expected_number_of_executable_files
328
+ expected = 1
329
+ archive = CPIO::ArchiveReader.new(CPIOFixture)
330
+ archive.each_entry { |ent| expected -= 1 if ent.file? && ent.executable? }
331
+ assert_equal 0, expected
332
+ end
333
+
334
+ def test_given_a_valid_archive_should_have_correct_file_contents
335
+ expected = ExpectedFixtureHashes.size
336
+ archive = CPIO::ArchiveReader.new(CPIOFixture)
337
+ archive.each_entry do |ent|
338
+ if (sha1_hash = ExpectedFixtureHashes[ent.filename]) && Digest::SHA1.hexdigest(ent.data) == sha1_hash
339
+ expected -= 1
340
+ end
341
+ end
342
+ assert_equal 0, expected, "Expected all files in the archive to hash correctly."
343
+ end
344
+
345
+ end
346
+
347
+ class CPIOArchiveWriterTest < Test::Unit::TestCase
348
+
349
+ def test_making_directories_should_work
350
+ expected = 2
351
+ io = StringIO.new
352
+ archiver = CPIO::ArchiveWriter.new(io)
353
+ archiver.open do |arch|
354
+ arch.mkdir "foo"
355
+ arch.mkdir "bar"
356
+ end
357
+ CPIO::ArchiveReader.new(io).each_entry { |ent| expected -= 1 if ent.directory? }
358
+ assert_equal 0, expected
359
+ end
360
+
361
+ def test_making_files_should_work
362
+ expected = 2
363
+ io = StringIO.new
364
+ archiver = CPIO::ArchiveWriter.new(io)
365
+ archiver.open do |arch|
366
+ arch.add_file("foo") { |sio| sio.write("foobar") }
367
+ arch.add_file("barfoo") { |sio| sio.write("barfoo") }
368
+ end
369
+ CPIO::ArchiveReader.new(io).each_entry { |ent| expected -= 1 if ent.file? }
370
+ assert_equal 0, expected
371
+ end
372
+
373
+ def test_making_files_and_directories_should_work
374
+ expected = 4
375
+ io = StringIO.new
376
+ archiver = CPIO::ArchiveWriter.new(io)
377
+ archiver.open do |arch|
378
+ arch.mkdir "blah"
379
+ arch.add_file("foo") { |sio| sio.write("foobar") }
380
+ arch.add_file("barfoo") { |sio| sio.write("barfoo") }
381
+ arch.add_file("barfoobaz", 0111) { |sio| sio.write("wee") }
382
+ end
383
+ CPIO::ArchiveReader.new(io).each_entry { |ent| expected -= 1 }
384
+ assert_equal 0, expected
385
+ end
386
+
387
+ def test_adding_empty_files_should_work
388
+ expected = 1
389
+ io = StringIO.new
390
+ archiver = CPIO::ArchiveWriter.new(io)
391
+ archiver.open do |arch|
392
+ arch.add_file("barfoo", 0111) { |sio| }
393
+ end
394
+ CPIO::ArchiveReader.new(io).each_entry { |ent| expected -= 1 if ent.file? }
395
+ assert_equal 0, expected
396
+ end
397
+
398
+ def test_adding_a_file_with_an_excessively_long_name_should_raise
399
+ archiver = CPIO::ArchiveWriter.new(StringIO.new)
400
+ assert_raise(CPIO::ArchiveFormatError) do
401
+ archiver.open do |arch|
402
+ name = "fffff" * (CPIO::ArchiveHeader::FieldMaxValues[:namesize])
403
+ arch.add_file(name) { |sio| }
404
+ end
405
+ end
406
+ end
407
+
408
+ def test_adding_a_non_executable_file_should_preserve_said_mode
409
+ io = StringIO.new
410
+ archiver = CPIO::ArchiveWriter.new(io)
411
+ archiver.open do |arch|
412
+ arch.add_file("barfoo", 0444) { |sio| }
413
+ end
414
+ CPIO::ArchiveReader.new(io).each_entry do |ent|
415
+ assert !ent.executable? && ent.file?
416
+ end
417
+ end
418
+
419
+ def test_adding_an_executable_file_should_preserve_said_mode
420
+ io = StringIO.new
421
+ archiver = CPIO::ArchiveWriter.new(io)
422
+ archiver.open do |arch|
423
+ arch.add_file("barfoo", 0500) { |sio| }
424
+ end
425
+ CPIO::ArchiveReader.new(io).each_entry do |ent|
426
+ assert ent.executable? && ent.file?
427
+ end
428
+ end
429
+ end
430
+
431
+ end
432
+
433
+ __END__
434
+ 0707077777770465470407550007650000240000040000001130242405100001200000000000cpio_test0707077777770465520407550007650000240000030000001130242404300002300000000000cpio_test/test_dir0707077777770465531006440007650000240000010000001130242637200003500000000016cpio_test/test_dir/test_filefoobarbazbeep
435
+ 0707077777770465541007550007650000240000010000001130242636000003200000000012cpio_test/test_executablefoobarbaz
436
+ 0707070000000000000000000000000000000000010000000000000000000001300000000000TRAILER!!!