external 0.1.0 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (49) hide show
  1. data/History +7 -0
  2. data/MIT-LICENSE +1 -3
  3. data/README +162 -127
  4. data/lib/external.rb +2 -3
  5. data/lib/external/base.rb +174 -47
  6. data/lib/external/chunkable.rb +131 -105
  7. data/lib/external/enumerable.rb +78 -33
  8. data/lib/external/io.rb +163 -398
  9. data/lib/external/patches/ruby_1_8_io.rb +31 -0
  10. data/lib/external/patches/windows_io.rb +53 -0
  11. data/lib/external/patches/windows_utils.rb +27 -0
  12. data/lib/external/utils.rb +148 -0
  13. data/lib/external_archive.rb +840 -0
  14. data/lib/external_array.rb +57 -0
  15. data/lib/external_index.rb +1053 -0
  16. metadata +42 -58
  17. data/lib/ext_arc.rb +0 -108
  18. data/lib/ext_arr.rb +0 -727
  19. data/lib/ext_ind.rb +0 -1120
  20. data/test/benchmarks/benchmarks_20070918.txt +0 -45
  21. data/test/benchmarks/benchmarks_20070921.txt +0 -91
  22. data/test/benchmarks/benchmarks_20071006.txt +0 -147
  23. data/test/benchmarks/test_copy_file.rb +0 -80
  24. data/test/benchmarks/test_pos_speed.rb +0 -47
  25. data/test/benchmarks/test_read_time.rb +0 -55
  26. data/test/cached_ext_ind_test.rb +0 -219
  27. data/test/check/benchmark_check.rb +0 -441
  28. data/test/check/namespace_conflicts_check.rb +0 -23
  29. data/test/check/pack_check.rb +0 -90
  30. data/test/ext_arc_test.rb +0 -286
  31. data/test/ext_arr/alt_sep.txt +0 -3
  32. data/test/ext_arr/cr_lf_input.txt +0 -3
  33. data/test/ext_arr/input.index +0 -0
  34. data/test/ext_arr/input.txt +0 -1
  35. data/test/ext_arr/inputb.index +0 -0
  36. data/test/ext_arr/inputb.txt +0 -1
  37. data/test/ext_arr/lf_input.txt +0 -3
  38. data/test/ext_arr/lines.txt +0 -19
  39. data/test/ext_arr/without_index.txt +0 -1
  40. data/test/ext_arr_test.rb +0 -534
  41. data/test/ext_ind_test.rb +0 -1472
  42. data/test/external/base_test.rb +0 -74
  43. data/test/external/chunkable_test.rb +0 -182
  44. data/test/external/index/input.index +0 -0
  45. data/test/external/index/inputb.index +0 -0
  46. data/test/external/io_test.rb +0 -414
  47. data/test/external_test_helper.rb +0 -31
  48. data/test/external_test_suite.rb +0 -4
  49. data/test/test_array.rb +0 -1192
@@ -1,219 +0,0 @@
1
- require File.join(File.dirname(__FILE__), 'ext_ind_test.rb')
2
-
3
- class CachedExtIndTest < ExtIndTest
4
- class CachedExtInd < ExtInd
5
- def initialize(io=nil, options={})
6
- options = {
7
- :cached => true
8
- }.merge(options)
9
-
10
- super(io, options)
11
- end
12
- end
13
-
14
- def setup
15
- # cls represents an array
16
- @cls = CachedExtInd
17
-
18
- @index = CachedExtInd.new
19
- @index.cache.concat(framed_array)
20
- end
21
-
22
- def teardown
23
- end
24
-
25
- #
26
- # setup tests
27
- #
28
-
29
- def test_setup
30
- assert_equal CachedExtInd, @cls
31
-
32
- assert_equal 0, index.pos
33
- assert_equal 5, index.length
34
- assert_equal 4, index.frame_size
35
- assert index.cached?
36
- assert_equal framed_array, index.cache
37
- end
38
-
39
-
40
- #
41
- # initialize tests
42
- #
43
-
44
- undef_method :test_index_initialized_to_single_int_format_by_default
45
-
46
- def test_cache_initialized_to_empty_array_in_cached_mode
47
- index = @cls.new
48
-
49
- assert_equal 'I*', index.format
50
- assert_equal 1, index.frame
51
- assert_equal 4, index.frame_size
52
- assert_equal [0], index.nil_value
53
- assert index.cached?
54
- assert_equal [], index.cache
55
- end
56
-
57
- def test_cache_initialized_to_empty_array_in_cached_mode_regardless_of_format
58
- index = @cls.new nil, :format => "IQS"
59
-
60
- assert index.cached?
61
- assert_equal [], index.cache
62
- end
63
-
64
- #
65
- # class read tests
66
- #
67
-
68
- undef_method :test_read_returns_the_index_file_in_frame
69
-
70
- #
71
- # class directive size tests
72
- #
73
-
74
- undef_method :test_directive_size_returns_the_number_of_bytes_to_pack_a_directive
75
-
76
- #
77
- # length test
78
- #
79
-
80
- undef_method :test_length_returns_io_length_divided_by_frame_size
81
-
82
- def test_length_returns_cache_length
83
- assert_equal index.cache.length, index.length
84
-
85
- index.cache.clear
86
- assert_equal 0, index.length
87
-
88
- index.cache.concat([[1],[2],[3]])
89
- assert_equal 3, index.length
90
- end
91
-
92
- def test_length_is_separate_from_io_length
93
- index.cache.clear
94
- assert_equal 0, index.length
95
-
96
- index.io.length = 10
97
- assert_equal 0, index.length
98
- end
99
-
100
- #
101
- # pos test
102
- #
103
-
104
- undef_method :test_pos_returns_io_pos_divided_by_frame_size
105
-
106
- def test_pos_is_kept_separate_from_io_pos_in_cached_mode
107
- assert_equal 0, index.io.pos
108
- assert_equal 0, index.pos
109
-
110
- index.io.pos = 4
111
- assert_equal 4, index.io.pos
112
- assert_equal 0, index.pos
113
- end
114
-
115
- #
116
- # pos= test
117
- #
118
-
119
- undef_method :test_pos_set_sets_io_pos_to_index_value_of_input_times_frame_size
120
-
121
- def test_pos_set_sets_pos_to_index_value_of_input
122
- index.pos = 1
123
- assert_equal 1, index.pos
124
-
125
- index.pos = -1
126
- assert_equal 4, index.pos
127
- end
128
-
129
- #
130
- # readbytes test
131
- #
132
-
133
- undef_method :test_readbytes_behavior_is_like_io_behavior
134
-
135
- #
136
- # unframed_write tests
137
- #
138
-
139
- undef_method :test_unframed_write_unframed_writes_packed_array_to_io_at_pos_and_adjusts_io_length
140
-
141
- def test_unframed_write_unframed_writes_packed_array_to_cache_in_frame_and_does_not_affect_io
142
- index = @cls.new
143
- index.unframed_write([1,2,3])
144
- assert_equal 0, index.io.length
145
- assert_equal 0, index.io.pos
146
-
147
- assert_equal [[1],[2],[3]], index.cache
148
-
149
- index.unframed_write([-2], 1)
150
- assert_equal 0, index.io.length
151
- assert_equal 0, index.io.pos
152
-
153
- assert_equal [[1],[-2],[3]], index.cache
154
- end
155
-
156
- def test_unframed_write_pads_with_nil_value_if_position_is_past_length
157
- index = @cls.new nil, :nil_value => [8]
158
- assert_equal 0, index.length
159
-
160
- index.unframed_write([1,2,3], 2)
161
- assert_equal [[8],[8],[1],[2],[3]], index.cache
162
- end
163
-
164
- def test_unframed_write_unframed_writes_nothing_with_empty_array
165
- assert_equal framed_array, index.cache
166
-
167
- index.unframed_write([])
168
- assert_equal framed_array, index.cache
169
-
170
- index.unframed_write([], 0)
171
- assert_equal framed_array, index.cache
172
- end
173
-
174
- #
175
- # mixed formats test
176
- #
177
-
178
- def test_read_handles_mixed_formats
179
- index = @cls.new nil, :format => "IQS"
180
-
181
- index.cache << [1,2,3]
182
- index.cache << [4,5,6]
183
- index.cache << [7,8,9]
184
-
185
- assert_equal [[1,2,3],[4,5,6],[7,8,9]], index.read
186
-
187
- index.pos = 1
188
- assert_equal [4,5,6], index.read(1)
189
- end
190
-
191
- def test_unframed_write_handles_mixed_formats
192
- index = @cls.new nil, :format => "IQS"
193
- a = [1,2,3]
194
- b = [4,5,6]
195
- c = [7,8,9]
196
- d = [-4,-5,-6]
197
-
198
- index.unframed_write([1,2,3,4,5,6,7,8,9])
199
- assert_equal [a,b,c], index.cache
200
-
201
- index.pos = 1
202
- index.unframed_write([-4,-5,-6])
203
- assert_equal [a,d,c], index.cache
204
- end
205
-
206
- #
207
- # numeric format range tests
208
- #
209
-
210
- undef_method :test_read_and_unframed_write_handles_full_numeric_range_for_numeric_formats
211
- undef_method :test_read_and_unframed_write_cycle_numerics_beyond_natural_range
212
- undef_method :test_numerics_cycle_up_to_the_unsigned_max_in_either_sign
213
-
214
- def test_unframed_write_does_NOT_check_formatting_of_input_in_cached_mode
215
- index = @cls.new nil, :format => "S"
216
- assert_raise(RangeError) { [ExtIndTest::ULLONG_MAX].pack("S") }
217
- assert_nothing_raised { index.unframed_write([ExtIndTest::ULLONG_MAX]) }
218
- end
219
- end
@@ -1,441 +0,0 @@
1
- require 'test/unit'
2
- require 'benchmark'
3
- require 'tap/test/subset_methods'
4
-
5
- ENV['benchmark'] = 'true'
6
-
7
- class BenchmarkCheck < Test::Unit::TestCase
8
- include Tap::Test::SubsetMethods
9
- include Benchmark
10
-
11
- if match_platform?('darwin')
12
- require 'inline'
13
-
14
- inline do |builder|
15
- builder.c %Q{
16
- int read_in_chunk(VALUE str, int n, int times) {
17
- char *filepath = RSTRING(str)->ptr;
18
- FILE *fp = fopen(filepath, "r");
19
- char input[(n*times)];
20
- int len = (n*times)+1; // ADD ONE to the read length because a null is appended as well
21
-
22
- if (fp != NULL)
23
- {
24
- fgets(input, len, fp);
25
- //printf(input);
26
- fclose(fp);
27
- return 1;
28
- }
29
- else
30
- return 0;
31
- }}
32
- end
33
-
34
- inline do |builder|
35
- builder.c %Q{
36
- int read_in_pieces(VALUE str, int n, int times) {
37
- char *filepath = RSTRING(str)->ptr;
38
- FILE *fp = fopen(filepath, "r");
39
- char input[n];
40
- int len = n+1; // ADD ONE to the read length because a null is appended as well
41
- int i = 0;
42
-
43
- if (fp != NULL)
44
- {
45
- while(i < times)
46
- {
47
- fgets(input, len, fp);
48
- //printf(input);
49
- ++i;
50
- }
51
- fclose(fp);
52
- return 1;
53
- }
54
- else
55
- return 0;
56
- }}
57
- end
58
-
59
- inline do |builder|
60
- builder.c %Q{
61
- int read_in_one_block(VALUE str, int len, int times) {
62
- char *filepath = RSTRING(str)->ptr;
63
- FILE *fp = fopen(filepath, "r");
64
- char input[len*times];
65
- int n_read;
66
-
67
- if (fp != NULL)
68
- {
69
- n_read = fread(input, len, times, fp);
70
- input[n_read*len] = NULL;
71
- //printf(input);
72
- fclose(fp);
73
- return 1;
74
- }
75
- else
76
- return 0;
77
- }}
78
- end
79
-
80
- inline do |builder|
81
- builder.c %Q{
82
- int read_in_blocks(VALUE str, int len, int times) {
83
- char *filepath = RSTRING(str)->ptr;
84
- FILE *fp = fopen(filepath, "r");
85
- char input[len];
86
- int n_read;
87
- int i = 0;
88
-
89
- if (fp != NULL)
90
- {
91
- while(i < times)
92
- {
93
- fread(input, len, 1, fp);
94
- input[len] = NULL;
95
- //printf(input);
96
- ++i;
97
- }
98
-
99
- fclose(fp);
100
- return 1;
101
- }
102
- else
103
- return 0;
104
- }}
105
- end
106
- end
107
-
108
- def test_read_in_chunk_vs_read_in_pieces
109
- platform_test("darwin") do
110
- begin
111
- filepath = File.expand_path("background_test.txt")
112
- File.open(filepath, "w") do |file|
113
- 10000.times do
114
- file << "0123456789"
115
- end
116
- end
117
- assert_equal 10000*10, File.size(filepath)
118
-
119
- benchmark_test(20) do |x|
120
- x.report("1kx read in chunk") { 1000.times { assert read_in_chunk(filepath, 10, 10000) }}
121
- x.report("1kx read in pieces") { 1000.times { assert read_in_pieces(filepath, 10, 10000) }}
122
- x.report("1kx read in one block") { 1000.times { assert read_in_one_block(filepath, 10, 10000) }}
123
- x.report("1kx read in blocks") { 1000.times { assert read_in_blocks(filepath, 10, 10000) }}
124
- x.report("1kx File.read") { 1000.times { File.read(filepath) }}
125
- end
126
-
127
- ensure
128
- FileUtils.rm(filepath) if File.exists?(filepath)
129
- end
130
- end
131
- end
132
-
133
- if match_platform?('darwin')
134
- require 'inline'
135
-
136
- inline do |builder|
137
- builder.c %Q{
138
-
139
- VALUE unpack_to_array(VALUE str, int frame, int size, int times) {
140
- char *filepath = RSTRING(str)->ptr;
141
- FILE *fp = fopen(filepath, "r");
142
- char input[frame*size*times];
143
- char *p = input;
144
- int i, j;
145
- VALUE results, arr;
146
-
147
- if (fp == NULL)
148
- rb_raise(rb_eArgError, "couldn't open file");
149
-
150
- times = fread(input, frame*size, times, fp);
151
- results = rb_ary_new();
152
-
153
- // convert to Fixnums
154
- i = 0;
155
- while(i < times)
156
- {
157
- j = 0;
158
- arr = rb_ary_new();
159
- while(j < frame)
160
- {
161
- // no need to copy the data at *p,
162
- // apparently the conversion can
163
- // happen directly from the pointer
164
- rb_ary_push(arr, UINT2NUM(*p));
165
- p += size;
166
- ++j;
167
- }
168
-
169
- rb_ary_push(results, arr);
170
- ++i;
171
- }
172
-
173
- fclose(fp);
174
- return results;
175
- }}
176
- end
177
- end
178
-
179
- require 'enumerator'
180
-
181
- def test_read_into_arrays
182
- platform_test("darwin") do
183
- begin
184
- filepath = File.expand_path("background_test.txt")
185
-
186
- times = 5000
187
- frame = 5
188
- size = 4
189
- format = "I*"
190
-
191
- array = Array.new(times) { (1..frame).to_a }
192
- File.open(filepath, "w") do |file|
193
- file << array.flatten.pack(format)
194
- end
195
- assert_equal 10000*10, File.size(filepath)
196
- assert_equal array, unpack_to_array(filepath, frame, size, times)
197
-
198
- benchmark_test(20) do |x|
199
- x.report("100x unpack to array") { 100.times { unpack_to_array(filepath, frame, size, times) }}
200
-
201
- results = []
202
- File.read(filepath).unpack(format).each_slice(frame) do |arr|
203
- results << arr
204
- end
205
- assert_equal array, results
206
- x.report("100x File.read.unpack") do
207
- 100.times do
208
- results = []
209
- File.read(filepath).unpack(format).each_slice(frame) do |arr|
210
- results << arr
211
- end
212
- end
213
- end
214
- end
215
-
216
- ensure
217
- FileUtils.rm(filepath) if File.exists?(filepath)
218
- end
219
- end
220
- end
221
-
222
- if match_platform?('darwin')
223
- require 'inline'
224
-
225
- inline do |builder|
226
- builder.c %Q{
227
-
228
- VALUE unpack_str(VALUE str, int frame, int size, int times) {
229
- char *p = RSTRING(str)->ptr;
230
- int i, j;
231
- VALUE results, arr;
232
- char directive = 'I';
233
- results = rb_ary_new();
234
-
235
- i = 0;
236
- while(i < times)
237
- {
238
- j = 0;
239
- arr = rb_ary_new();
240
- while(j < frame)
241
- {
242
- switch(directive)
243
- {
244
- case 'I':
245
- {// no need to copy the data at *p,
246
- // apparently the conversion can
247
- // happen directly from the pointer
248
- rb_ary_push(arr, UINT2NUM(*p));
249
- p += size;
250
- ++j;}
251
- break;
252
- }
253
- }
254
-
255
- rb_ary_push(results, arr);
256
- ++i;
257
- }
258
-
259
- return results;
260
- }}
261
- end
262
- end
263
-
264
- require 'enumerator'
265
-
266
- def test_unpack_speed
267
- platform_test("darwin") do
268
- begin
269
- filepath = File.expand_path("background_test.txt")
270
-
271
- times = 5000
272
- frame = 5
273
- size = 4
274
- format = "I*"
275
-
276
- array = Array.new(times) { (1..frame).to_a }
277
- File.open(filepath, "w") do |file|
278
- file << array.flatten.pack(format)
279
- end
280
- assert_equal 10000*10, File.size(filepath)
281
-
282
- str = File.read(filepath)
283
- assert_equal array, unpack_str(str, frame, size, times)
284
-
285
- benchmark_test(20) do |x|
286
- x.report("100x unpack") { 100.times { unpack_str(str, frame, size, times) }}
287
-
288
- results = []
289
- File.read(filepath).unpack(format).each_slice(frame) do |arr|
290
- results << arr
291
- end
292
- assert_equal array, results
293
- x.report("100x str.unpack") do
294
- 100.times do
295
- results = []
296
- str.unpack(format).each_slice(frame) do |arr|
297
- results << arr
298
- end
299
- end
300
- end
301
- end
302
-
303
- ensure
304
- FileUtils.rm(filepath) if File.exists?(filepath)
305
- end
306
- end
307
- end
308
-
309
- if match_platform?('darwin')
310
- require 'inline'
311
-
312
- inline do |builder|
313
- builder.c %Q{
314
-
315
- int work_with_values() {
316
- int a = NUM2INT(rb_iv_get(self, "@a"));
317
- int b = NUM2INT(rb_iv_get(self, "@b"));
318
-
319
- return a + b;
320
- }}
321
- end
322
- end
323
-
324
- attr_accessor :a, :b
325
-
326
- def test_work_with_values
327
- platform_test("darwin") do
328
- @a = 10
329
- @b = 2
330
-
331
- assert_equal 12, work_with_values
332
- end
333
- end
334
-
335
- if match_platform?('darwin')
336
- require 'inline'
337
-
338
- module FileExt
339
- inline do |builder|
340
- builder.include "<rubyio.h>"
341
- builder.c %Q{
342
-
343
- int read_from_file(int n) {
344
- FILE *fp = RFILE(self)->fptr->f;
345
-
346
- char input[n];
347
- int len = n+1; // ADD ONE to the read length because a null is appended as well
348
-
349
- if (fp != NULL)
350
- {
351
- fgets(input, len, fp);
352
- //printf(input);
353
- fclose(fp);
354
- return 1;
355
- }
356
- else
357
- return 0;
358
- }}
359
- end
360
- end
361
- end
362
-
363
- def test_get_file_pointer
364
- platform_test("darwin") do
365
- begin
366
- filepath = File.expand_path("background_test.txt")
367
- File.open(filepath, 'w+') do |file|
368
- file.extend FileExt
369
- file << "hello world"
370
- file.pos = 0
371
- assert file.read_from_file(5)
372
- end
373
- ensure
374
- FileUtils.rm(filepath) if File.exists?(filepath)
375
- end
376
- end
377
- end
378
-
379
- if match_platform?('darwin')
380
- require 'inline'
381
-
382
- inline do |builder|
383
- builder.include "<rubyio.h>"
384
- builder.c %Q{
385
- int read_from_file(int len, int times) {
386
- FILE *fp = RFILE(rb_iv_get(self, "@file"))->fptr->f;
387
- char input[len*times];
388
-
389
- if (fp == NULL)
390
- return 0;
391
-
392
- fread(input, len, times, fp);
393
- input[len*times] = NULL;
394
- // printf(input);
395
-
396
- return 1;
397
- }}
398
- end
399
- end
400
-
401
- attr_reader :file
402
-
403
- def test_read_from_open_file
404
- platform_test("darwin") do
405
- begin
406
- filepath = File.expand_path("background_test.txt")
407
- File.open(filepath, "w") do |file|
408
- 10000.times do
409
- file << "0123456789"
410
- end
411
- end
412
- assert_equal 10000*10, File.size(filepath)
413
-
414
-
415
- File.open(filepath) do |file|
416
- @file = file
417
-
418
- file.pos = 0
419
- assert read_from_file(10, 2)
420
-
421
- benchmark_test(20) do |x|
422
- x.report("1kx read from file") { 1000.times { file.pos = 0; read_from_file(10, 10000) }}
423
- x.report("1kx file.read") { 1000.times { file.pos = 0; file.read }}
424
- end
425
-
426
- end
427
- ensure
428
- FileUtils.rm(filepath) if File.exists?(filepath)
429
- end
430
- end
431
- end
432
-
433
- def test_array_methods
434
- benchmark_test(20) do |x|
435
- a = []
436
- x.report("1M <<") { (1*1000000).times { a << 1 } }
437
- a.clear
438
- x.report("1M []") { (1*1000000).times { a[1] = 1 } }
439
- end
440
- end
441
- end