external 0.1.0 → 0.3.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (49) hide show
  1. data/History +7 -0
  2. data/MIT-LICENSE +1 -3
  3. data/README +162 -127
  4. data/lib/external.rb +2 -3
  5. data/lib/external/base.rb +174 -47
  6. data/lib/external/chunkable.rb +131 -105
  7. data/lib/external/enumerable.rb +78 -33
  8. data/lib/external/io.rb +163 -398
  9. data/lib/external/patches/ruby_1_8_io.rb +31 -0
  10. data/lib/external/patches/windows_io.rb +53 -0
  11. data/lib/external/patches/windows_utils.rb +27 -0
  12. data/lib/external/utils.rb +148 -0
  13. data/lib/external_archive.rb +840 -0
  14. data/lib/external_array.rb +57 -0
  15. data/lib/external_index.rb +1053 -0
  16. metadata +42 -58
  17. data/lib/ext_arc.rb +0 -108
  18. data/lib/ext_arr.rb +0 -727
  19. data/lib/ext_ind.rb +0 -1120
  20. data/test/benchmarks/benchmarks_20070918.txt +0 -45
  21. data/test/benchmarks/benchmarks_20070921.txt +0 -91
  22. data/test/benchmarks/benchmarks_20071006.txt +0 -147
  23. data/test/benchmarks/test_copy_file.rb +0 -80
  24. data/test/benchmarks/test_pos_speed.rb +0 -47
  25. data/test/benchmarks/test_read_time.rb +0 -55
  26. data/test/cached_ext_ind_test.rb +0 -219
  27. data/test/check/benchmark_check.rb +0 -441
  28. data/test/check/namespace_conflicts_check.rb +0 -23
  29. data/test/check/pack_check.rb +0 -90
  30. data/test/ext_arc_test.rb +0 -286
  31. data/test/ext_arr/alt_sep.txt +0 -3
  32. data/test/ext_arr/cr_lf_input.txt +0 -3
  33. data/test/ext_arr/input.index +0 -0
  34. data/test/ext_arr/input.txt +0 -1
  35. data/test/ext_arr/inputb.index +0 -0
  36. data/test/ext_arr/inputb.txt +0 -1
  37. data/test/ext_arr/lf_input.txt +0 -3
  38. data/test/ext_arr/lines.txt +0 -19
  39. data/test/ext_arr/without_index.txt +0 -1
  40. data/test/ext_arr_test.rb +0 -534
  41. data/test/ext_ind_test.rb +0 -1472
  42. data/test/external/base_test.rb +0 -74
  43. data/test/external/chunkable_test.rb +0 -182
  44. data/test/external/index/input.index +0 -0
  45. data/test/external/index/inputb.index +0 -0
  46. data/test/external/io_test.rb +0 -414
  47. data/test/external_test_helper.rb +0 -31
  48. data/test/external_test_suite.rb +0 -4
  49. data/test/test_array.rb +0 -1192
@@ -1,219 +0,0 @@
1
- require File.join(File.dirname(__FILE__), 'ext_ind_test.rb')
2
-
3
- class CachedExtIndTest < ExtIndTest
4
- class CachedExtInd < ExtInd
5
- def initialize(io=nil, options={})
6
- options = {
7
- :cached => true
8
- }.merge(options)
9
-
10
- super(io, options)
11
- end
12
- end
13
-
14
- def setup
15
- # cls represents an array
16
- @cls = CachedExtInd
17
-
18
- @index = CachedExtInd.new
19
- @index.cache.concat(framed_array)
20
- end
21
-
22
- def teardown
23
- end
24
-
25
- #
26
- # setup tests
27
- #
28
-
29
- def test_setup
30
- assert_equal CachedExtInd, @cls
31
-
32
- assert_equal 0, index.pos
33
- assert_equal 5, index.length
34
- assert_equal 4, index.frame_size
35
- assert index.cached?
36
- assert_equal framed_array, index.cache
37
- end
38
-
39
-
40
- #
41
- # initialize tests
42
- #
43
-
44
- undef_method :test_index_initialized_to_single_int_format_by_default
45
-
46
- def test_cache_initialized_to_empty_array_in_cached_mode
47
- index = @cls.new
48
-
49
- assert_equal 'I*', index.format
50
- assert_equal 1, index.frame
51
- assert_equal 4, index.frame_size
52
- assert_equal [0], index.nil_value
53
- assert index.cached?
54
- assert_equal [], index.cache
55
- end
56
-
57
- def test_cache_initialized_to_empty_array_in_cached_mode_regardless_of_format
58
- index = @cls.new nil, :format => "IQS"
59
-
60
- assert index.cached?
61
- assert_equal [], index.cache
62
- end
63
-
64
- #
65
- # class read tests
66
- #
67
-
68
- undef_method :test_read_returns_the_index_file_in_frame
69
-
70
- #
71
- # class directive size tests
72
- #
73
-
74
- undef_method :test_directive_size_returns_the_number_of_bytes_to_pack_a_directive
75
-
76
- #
77
- # length test
78
- #
79
-
80
- undef_method :test_length_returns_io_length_divided_by_frame_size
81
-
82
- def test_length_returns_cache_length
83
- assert_equal index.cache.length, index.length
84
-
85
- index.cache.clear
86
- assert_equal 0, index.length
87
-
88
- index.cache.concat([[1],[2],[3]])
89
- assert_equal 3, index.length
90
- end
91
-
92
- def test_length_is_separate_from_io_length
93
- index.cache.clear
94
- assert_equal 0, index.length
95
-
96
- index.io.length = 10
97
- assert_equal 0, index.length
98
- end
99
-
100
- #
101
- # pos test
102
- #
103
-
104
- undef_method :test_pos_returns_io_pos_divided_by_frame_size
105
-
106
- def test_pos_is_kept_separate_from_io_pos_in_cached_mode
107
- assert_equal 0, index.io.pos
108
- assert_equal 0, index.pos
109
-
110
- index.io.pos = 4
111
- assert_equal 4, index.io.pos
112
- assert_equal 0, index.pos
113
- end
114
-
115
- #
116
- # pos= test
117
- #
118
-
119
- undef_method :test_pos_set_sets_io_pos_to_index_value_of_input_times_frame_size
120
-
121
- def test_pos_set_sets_pos_to_index_value_of_input
122
- index.pos = 1
123
- assert_equal 1, index.pos
124
-
125
- index.pos = -1
126
- assert_equal 4, index.pos
127
- end
128
-
129
- #
130
- # readbytes test
131
- #
132
-
133
- undef_method :test_readbytes_behavior_is_like_io_behavior
134
-
135
- #
136
- # unframed_write tests
137
- #
138
-
139
- undef_method :test_unframed_write_unframed_writes_packed_array_to_io_at_pos_and_adjusts_io_length
140
-
141
- def test_unframed_write_unframed_writes_packed_array_to_cache_in_frame_and_does_not_affect_io
142
- index = @cls.new
143
- index.unframed_write([1,2,3])
144
- assert_equal 0, index.io.length
145
- assert_equal 0, index.io.pos
146
-
147
- assert_equal [[1],[2],[3]], index.cache
148
-
149
- index.unframed_write([-2], 1)
150
- assert_equal 0, index.io.length
151
- assert_equal 0, index.io.pos
152
-
153
- assert_equal [[1],[-2],[3]], index.cache
154
- end
155
-
156
- def test_unframed_write_pads_with_nil_value_if_position_is_past_length
157
- index = @cls.new nil, :nil_value => [8]
158
- assert_equal 0, index.length
159
-
160
- index.unframed_write([1,2,3], 2)
161
- assert_equal [[8],[8],[1],[2],[3]], index.cache
162
- end
163
-
164
- def test_unframed_write_unframed_writes_nothing_with_empty_array
165
- assert_equal framed_array, index.cache
166
-
167
- index.unframed_write([])
168
- assert_equal framed_array, index.cache
169
-
170
- index.unframed_write([], 0)
171
- assert_equal framed_array, index.cache
172
- end
173
-
174
- #
175
- # mixed formats test
176
- #
177
-
178
- def test_read_handles_mixed_formats
179
- index = @cls.new nil, :format => "IQS"
180
-
181
- index.cache << [1,2,3]
182
- index.cache << [4,5,6]
183
- index.cache << [7,8,9]
184
-
185
- assert_equal [[1,2,3],[4,5,6],[7,8,9]], index.read
186
-
187
- index.pos = 1
188
- assert_equal [4,5,6], index.read(1)
189
- end
190
-
191
- def test_unframed_write_handles_mixed_formats
192
- index = @cls.new nil, :format => "IQS"
193
- a = [1,2,3]
194
- b = [4,5,6]
195
- c = [7,8,9]
196
- d = [-4,-5,-6]
197
-
198
- index.unframed_write([1,2,3,4,5,6,7,8,9])
199
- assert_equal [a,b,c], index.cache
200
-
201
- index.pos = 1
202
- index.unframed_write([-4,-5,-6])
203
- assert_equal [a,d,c], index.cache
204
- end
205
-
206
- #
207
- # numeric format range tests
208
- #
209
-
210
- undef_method :test_read_and_unframed_write_handles_full_numeric_range_for_numeric_formats
211
- undef_method :test_read_and_unframed_write_cycle_numerics_beyond_natural_range
212
- undef_method :test_numerics_cycle_up_to_the_unsigned_max_in_either_sign
213
-
214
- def test_unframed_write_does_NOT_check_formatting_of_input_in_cached_mode
215
- index = @cls.new nil, :format => "S"
216
- assert_raise(RangeError) { [ExtIndTest::ULLONG_MAX].pack("S") }
217
- assert_nothing_raised { index.unframed_write([ExtIndTest::ULLONG_MAX]) }
218
- end
219
- end
@@ -1,441 +0,0 @@
1
- require 'test/unit'
2
- require 'benchmark'
3
- require 'tap/test/subset_methods'
4
-
5
- ENV['benchmark'] = 'true'
6
-
7
- class BenchmarkCheck < Test::Unit::TestCase
8
- include Tap::Test::SubsetMethods
9
- include Benchmark
10
-
11
- if match_platform?('darwin')
12
- require 'inline'
13
-
14
- inline do |builder|
15
- builder.c %Q{
16
- int read_in_chunk(VALUE str, int n, int times) {
17
- char *filepath = RSTRING(str)->ptr;
18
- FILE *fp = fopen(filepath, "r");
19
- char input[(n*times)];
20
- int len = (n*times)+1; // ADD ONE to the read length because a null is appended as well
21
-
22
- if (fp != NULL)
23
- {
24
- fgets(input, len, fp);
25
- //printf(input);
26
- fclose(fp);
27
- return 1;
28
- }
29
- else
30
- return 0;
31
- }}
32
- end
33
-
34
- inline do |builder|
35
- builder.c %Q{
36
- int read_in_pieces(VALUE str, int n, int times) {
37
- char *filepath = RSTRING(str)->ptr;
38
- FILE *fp = fopen(filepath, "r");
39
- char input[n];
40
- int len = n+1; // ADD ONE to the read length because a null is appended as well
41
- int i = 0;
42
-
43
- if (fp != NULL)
44
- {
45
- while(i < times)
46
- {
47
- fgets(input, len, fp);
48
- //printf(input);
49
- ++i;
50
- }
51
- fclose(fp);
52
- return 1;
53
- }
54
- else
55
- return 0;
56
- }}
57
- end
58
-
59
- inline do |builder|
60
- builder.c %Q{
61
- int read_in_one_block(VALUE str, int len, int times) {
62
- char *filepath = RSTRING(str)->ptr;
63
- FILE *fp = fopen(filepath, "r");
64
- char input[len*times];
65
- int n_read;
66
-
67
- if (fp != NULL)
68
- {
69
- n_read = fread(input, len, times, fp);
70
- input[n_read*len] = NULL;
71
- //printf(input);
72
- fclose(fp);
73
- return 1;
74
- }
75
- else
76
- return 0;
77
- }}
78
- end
79
-
80
- inline do |builder|
81
- builder.c %Q{
82
- int read_in_blocks(VALUE str, int len, int times) {
83
- char *filepath = RSTRING(str)->ptr;
84
- FILE *fp = fopen(filepath, "r");
85
- char input[len];
86
- int n_read;
87
- int i = 0;
88
-
89
- if (fp != NULL)
90
- {
91
- while(i < times)
92
- {
93
- fread(input, len, 1, fp);
94
- input[len] = NULL;
95
- //printf(input);
96
- ++i;
97
- }
98
-
99
- fclose(fp);
100
- return 1;
101
- }
102
- else
103
- return 0;
104
- }}
105
- end
106
- end
107
-
108
- def test_read_in_chunk_vs_read_in_pieces
109
- platform_test("darwin") do
110
- begin
111
- filepath = File.expand_path("background_test.txt")
112
- File.open(filepath, "w") do |file|
113
- 10000.times do
114
- file << "0123456789"
115
- end
116
- end
117
- assert_equal 10000*10, File.size(filepath)
118
-
119
- benchmark_test(20) do |x|
120
- x.report("1kx read in chunk") { 1000.times { assert read_in_chunk(filepath, 10, 10000) }}
121
- x.report("1kx read in pieces") { 1000.times { assert read_in_pieces(filepath, 10, 10000) }}
122
- x.report("1kx read in one block") { 1000.times { assert read_in_one_block(filepath, 10, 10000) }}
123
- x.report("1kx read in blocks") { 1000.times { assert read_in_blocks(filepath, 10, 10000) }}
124
- x.report("1kx File.read") { 1000.times { File.read(filepath) }}
125
- end
126
-
127
- ensure
128
- FileUtils.rm(filepath) if File.exists?(filepath)
129
- end
130
- end
131
- end
132
-
133
- if match_platform?('darwin')
134
- require 'inline'
135
-
136
- inline do |builder|
137
- builder.c %Q{
138
-
139
- VALUE unpack_to_array(VALUE str, int frame, int size, int times) {
140
- char *filepath = RSTRING(str)->ptr;
141
- FILE *fp = fopen(filepath, "r");
142
- char input[frame*size*times];
143
- char *p = input;
144
- int i, j;
145
- VALUE results, arr;
146
-
147
- if (fp == NULL)
148
- rb_raise(rb_eArgError, "couldn't open file");
149
-
150
- times = fread(input, frame*size, times, fp);
151
- results = rb_ary_new();
152
-
153
- // convert to Fixnums
154
- i = 0;
155
- while(i < times)
156
- {
157
- j = 0;
158
- arr = rb_ary_new();
159
- while(j < frame)
160
- {
161
- // no need to copy the data at *p,
162
- // apparently the conversion can
163
- // happen directly from the pointer
164
- rb_ary_push(arr, UINT2NUM(*p));
165
- p += size;
166
- ++j;
167
- }
168
-
169
- rb_ary_push(results, arr);
170
- ++i;
171
- }
172
-
173
- fclose(fp);
174
- return results;
175
- }}
176
- end
177
- end
178
-
179
- require 'enumerator'
180
-
181
- def test_read_into_arrays
182
- platform_test("darwin") do
183
- begin
184
- filepath = File.expand_path("background_test.txt")
185
-
186
- times = 5000
187
- frame = 5
188
- size = 4
189
- format = "I*"
190
-
191
- array = Array.new(times) { (1..frame).to_a }
192
- File.open(filepath, "w") do |file|
193
- file << array.flatten.pack(format)
194
- end
195
- assert_equal 10000*10, File.size(filepath)
196
- assert_equal array, unpack_to_array(filepath, frame, size, times)
197
-
198
- benchmark_test(20) do |x|
199
- x.report("100x unpack to array") { 100.times { unpack_to_array(filepath, frame, size, times) }}
200
-
201
- results = []
202
- File.read(filepath).unpack(format).each_slice(frame) do |arr|
203
- results << arr
204
- end
205
- assert_equal array, results
206
- x.report("100x File.read.unpack") do
207
- 100.times do
208
- results = []
209
- File.read(filepath).unpack(format).each_slice(frame) do |arr|
210
- results << arr
211
- end
212
- end
213
- end
214
- end
215
-
216
- ensure
217
- FileUtils.rm(filepath) if File.exists?(filepath)
218
- end
219
- end
220
- end
221
-
222
- if match_platform?('darwin')
223
- require 'inline'
224
-
225
- inline do |builder|
226
- builder.c %Q{
227
-
228
- VALUE unpack_str(VALUE str, int frame, int size, int times) {
229
- char *p = RSTRING(str)->ptr;
230
- int i, j;
231
- VALUE results, arr;
232
- char directive = 'I';
233
- results = rb_ary_new();
234
-
235
- i = 0;
236
- while(i < times)
237
- {
238
- j = 0;
239
- arr = rb_ary_new();
240
- while(j < frame)
241
- {
242
- switch(directive)
243
- {
244
- case 'I':
245
- {// no need to copy the data at *p,
246
- // apparently the conversion can
247
- // happen directly from the pointer
248
- rb_ary_push(arr, UINT2NUM(*p));
249
- p += size;
250
- ++j;}
251
- break;
252
- }
253
- }
254
-
255
- rb_ary_push(results, arr);
256
- ++i;
257
- }
258
-
259
- return results;
260
- }}
261
- end
262
- end
263
-
264
- require 'enumerator'
265
-
266
- def test_unpack_speed
267
- platform_test("darwin") do
268
- begin
269
- filepath = File.expand_path("background_test.txt")
270
-
271
- times = 5000
272
- frame = 5
273
- size = 4
274
- format = "I*"
275
-
276
- array = Array.new(times) { (1..frame).to_a }
277
- File.open(filepath, "w") do |file|
278
- file << array.flatten.pack(format)
279
- end
280
- assert_equal 10000*10, File.size(filepath)
281
-
282
- str = File.read(filepath)
283
- assert_equal array, unpack_str(str, frame, size, times)
284
-
285
- benchmark_test(20) do |x|
286
- x.report("100x unpack") { 100.times { unpack_str(str, frame, size, times) }}
287
-
288
- results = []
289
- File.read(filepath).unpack(format).each_slice(frame) do |arr|
290
- results << arr
291
- end
292
- assert_equal array, results
293
- x.report("100x str.unpack") do
294
- 100.times do
295
- results = []
296
- str.unpack(format).each_slice(frame) do |arr|
297
- results << arr
298
- end
299
- end
300
- end
301
- end
302
-
303
- ensure
304
- FileUtils.rm(filepath) if File.exists?(filepath)
305
- end
306
- end
307
- end
308
-
309
- if match_platform?('darwin')
310
- require 'inline'
311
-
312
- inline do |builder|
313
- builder.c %Q{
314
-
315
- int work_with_values() {
316
- int a = NUM2INT(rb_iv_get(self, "@a"));
317
- int b = NUM2INT(rb_iv_get(self, "@b"));
318
-
319
- return a + b;
320
- }}
321
- end
322
- end
323
-
324
- attr_accessor :a, :b
325
-
326
- def test_work_with_values
327
- platform_test("darwin") do
328
- @a = 10
329
- @b = 2
330
-
331
- assert_equal 12, work_with_values
332
- end
333
- end
334
-
335
- if match_platform?('darwin')
336
- require 'inline'
337
-
338
- module FileExt
339
- inline do |builder|
340
- builder.include "<rubyio.h>"
341
- builder.c %Q{
342
-
343
- int read_from_file(int n) {
344
- FILE *fp = RFILE(self)->fptr->f;
345
-
346
- char input[n];
347
- int len = n+1; // ADD ONE to the read length because a null is appended as well
348
-
349
- if (fp != NULL)
350
- {
351
- fgets(input, len, fp);
352
- //printf(input);
353
- fclose(fp);
354
- return 1;
355
- }
356
- else
357
- return 0;
358
- }}
359
- end
360
- end
361
- end
362
-
363
- def test_get_file_pointer
364
- platform_test("darwin") do
365
- begin
366
- filepath = File.expand_path("background_test.txt")
367
- File.open(filepath, 'w+') do |file|
368
- file.extend FileExt
369
- file << "hello world"
370
- file.pos = 0
371
- assert file.read_from_file(5)
372
- end
373
- ensure
374
- FileUtils.rm(filepath) if File.exists?(filepath)
375
- end
376
- end
377
- end
378
-
379
- if match_platform?('darwin')
380
- require 'inline'
381
-
382
- inline do |builder|
383
- builder.include "<rubyio.h>"
384
- builder.c %Q{
385
- int read_from_file(int len, int times) {
386
- FILE *fp = RFILE(rb_iv_get(self, "@file"))->fptr->f;
387
- char input[len*times];
388
-
389
- if (fp == NULL)
390
- return 0;
391
-
392
- fread(input, len, times, fp);
393
- input[len*times] = NULL;
394
- // printf(input);
395
-
396
- return 1;
397
- }}
398
- end
399
- end
400
-
401
- attr_reader :file
402
-
403
- def test_read_from_open_file
404
- platform_test("darwin") do
405
- begin
406
- filepath = File.expand_path("background_test.txt")
407
- File.open(filepath, "w") do |file|
408
- 10000.times do
409
- file << "0123456789"
410
- end
411
- end
412
- assert_equal 10000*10, File.size(filepath)
413
-
414
-
415
- File.open(filepath) do |file|
416
- @file = file
417
-
418
- file.pos = 0
419
- assert read_from_file(10, 2)
420
-
421
- benchmark_test(20) do |x|
422
- x.report("1kx read from file") { 1000.times { file.pos = 0; read_from_file(10, 10000) }}
423
- x.report("1kx file.read") { 1000.times { file.pos = 0; file.read }}
424
- end
425
-
426
- end
427
- ensure
428
- FileUtils.rm(filepath) if File.exists?(filepath)
429
- end
430
- end
431
- end
432
-
433
- def test_array_methods
434
- benchmark_test(20) do |x|
435
- a = []
436
- x.report("1M <<") { (1*1000000).times { a << 1 } }
437
- a.clear
438
- x.report("1M []") { (1*1000000).times { a[1] = 1 } }
439
- end
440
- end
441
- end