external 0.1.0 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (49) hide show
  1. data/History +7 -0
  2. data/MIT-LICENSE +1 -3
  3. data/README +162 -127
  4. data/lib/external.rb +2 -3
  5. data/lib/external/base.rb +174 -47
  6. data/lib/external/chunkable.rb +131 -105
  7. data/lib/external/enumerable.rb +78 -33
  8. data/lib/external/io.rb +163 -398
  9. data/lib/external/patches/ruby_1_8_io.rb +31 -0
  10. data/lib/external/patches/windows_io.rb +53 -0
  11. data/lib/external/patches/windows_utils.rb +27 -0
  12. data/lib/external/utils.rb +148 -0
  13. data/lib/external_archive.rb +840 -0
  14. data/lib/external_array.rb +57 -0
  15. data/lib/external_index.rb +1053 -0
  16. metadata +42 -58
  17. data/lib/ext_arc.rb +0 -108
  18. data/lib/ext_arr.rb +0 -727
  19. data/lib/ext_ind.rb +0 -1120
  20. data/test/benchmarks/benchmarks_20070918.txt +0 -45
  21. data/test/benchmarks/benchmarks_20070921.txt +0 -91
  22. data/test/benchmarks/benchmarks_20071006.txt +0 -147
  23. data/test/benchmarks/test_copy_file.rb +0 -80
  24. data/test/benchmarks/test_pos_speed.rb +0 -47
  25. data/test/benchmarks/test_read_time.rb +0 -55
  26. data/test/cached_ext_ind_test.rb +0 -219
  27. data/test/check/benchmark_check.rb +0 -441
  28. data/test/check/namespace_conflicts_check.rb +0 -23
  29. data/test/check/pack_check.rb +0 -90
  30. data/test/ext_arc_test.rb +0 -286
  31. data/test/ext_arr/alt_sep.txt +0 -3
  32. data/test/ext_arr/cr_lf_input.txt +0 -3
  33. data/test/ext_arr/input.index +0 -0
  34. data/test/ext_arr/input.txt +0 -1
  35. data/test/ext_arr/inputb.index +0 -0
  36. data/test/ext_arr/inputb.txt +0 -1
  37. data/test/ext_arr/lf_input.txt +0 -3
  38. data/test/ext_arr/lines.txt +0 -19
  39. data/test/ext_arr/without_index.txt +0 -1
  40. data/test/ext_arr_test.rb +0 -534
  41. data/test/ext_ind_test.rb +0 -1472
  42. data/test/external/base_test.rb +0 -74
  43. data/test/external/chunkable_test.rb +0 -182
  44. data/test/external/index/input.index +0 -0
  45. data/test/external/index/inputb.index +0 -0
  46. data/test/external/io_test.rb +0 -414
  47. data/test/external_test_helper.rb +0 -31
  48. data/test/external_test_suite.rb +0 -4
  49. data/test/test_array.rb +0 -1192
@@ -1,45 +0,0 @@
1
- ############################################
2
- # Index with batch processing
3
- ############################################
4
-
5
- ............
6
- test_element_assignment_speed
7
- user system total real
8
- I
9
- 10kx [index]= 0.160000 0.050000 0.210000 ( 0.208374)
10
- 10kx [range]= 11.510000 13.240000 24.750000 ( 28.489626)
11
- 10kx [s,1]= 11.490000 13.010000 24.500000 ( 27.930727)
12
-
13
- IIIIIIIIII
14
- 10kx [index]= 0.220000 0.050000 0.270000 ( 0.263851)
15
- 10kx [range]= 11.390000 13.790000 25.180000 ( 28.857814)
16
- 10kx [s,1]= 11.590000 13.920000 25.510000 ( 28.947041)
17
-
18
- array reference
19
- 10kx [index]= 0.000000 0.000000 0.000000 ( 0.003049)
20
- 10kx [range]= 0.010000 0.000000 0.010000 ( 0.006701)
21
- 10kx [s,1]= 0.000000 0.000000 0.000000 ( 0.005084)
22
-
23
- .
24
- test_element_reference_speed
25
- user system total real
26
- I
27
- 100kx [index] 1.590000 0.640000 2.230000 ( 2.226855)
28
- 100kx [range] 0.670000 0.000000 0.670000 ( 0.674267)
29
- 100kx [s,1] 2.140000 0.710000 2.850000 ( 3.660551)
30
- 100kx [s,100] 11.110000 0.680000 11.790000 ( 11.797169)
31
-
32
- IIIIIIIIII
33
- 100kx [index] 0.380000 0.000000 0.380000 ( 0.381269)
34
- 100kx [range] 0.590000 0.000000 0.590000 ( 0.596402)
35
- 100kx [s,1] 0.460000 0.000000 0.460000 ( 0.456065)
36
- 100kx [s,100] 0.440000 0.000000 0.440000 ( 0.438470)
37
-
38
- array reference
39
- 100kx [index] 0.020000 0.000000 0.020000 ( 0.026224)
40
- 100kx [range] 0.080000 0.000000 0.080000 ( 0.079051)
41
- 100kx [s,1] 0.050000 0.000000 0.050000 ( 0.049507)
42
- 100kx [s,100] 0.050000 0.000000 0.050000 ( 0.050872)
43
-
44
- ..............
45
- Finished in 135.398082 seconds.
@@ -1,91 +0,0 @@
1
- #######################################
2
- # reworked osx
3
- #######################################
4
- test_element_reference_speed
5
- user system total real
6
- I
7
- 100kx [index] 1.950000 0.660000 2.610000 ( 2.658425)
8
- 100kx [range] 2.880000 0.650000 3.530000 ( 3.573627)
9
- 100kx [s,1] 2.100000 0.640000 2.740000 ( 2.774918)
10
- 100kx [s,100] 20.810000 0.700000 21.510000 ( 21.615324)
11
-
12
- IIIIIIIIII
13
- 100kx [index] 1.630000 0.360000 1.990000 ( 1.995421)
14
- 100kx [range] 1.140000 0.000000 1.140000 ( 1.145692)
15
- 100kx [s,1] 0.470000 0.000000 0.470000 ( 0.472543)
16
- 100kx [s,100] 0.470000 0.000000 0.470000 ( 0.472698)
17
-
18
- array reference
19
- 100kx [index] 0.030000 0.000000 0.030000 ( 0.026341)
20
- 100kx [range] 0.060000 0.000000 0.060000 ( 0.060080)
21
- 100kx [s,1] 0.050000 0.000000 0.050000 ( 0.052237)
22
- 100kx [s,100] 0.040000 0.000000 0.040000 ( 0.042142)
23
-
24
- #######################################
25
- # reworked osx accelerated
26
- #######################################
27
- test_element_reference_speed
28
- user system total real
29
- I
30
- 100kx [index] 1.970000 0.650000 2.620000 ( 2.677882)
31
- 100kx [range] 2.870000 0.650000 3.520000 ( 3.530610)
32
- 100kx [s,1] 2.130000 0.640000 2.770000 ( 2.777566)
33
- 100kx [s,100] 8.410000 0.670000 9.080000 ( 9.118457)
34
-
35
- IIIIIIIIII
36
- 100kx [index] 1.630000 0.360000 1.990000 ( 2.001829)
37
- 100kx [range] 1.150000 0.010000 1.160000 ( 1.157869)
38
- 100kx [s,1] 0.470000 0.000000 0.470000 ( 0.477857)
39
- 100kx [s,100] 0.470000 0.000000 0.470000 ( 0.476172)
40
-
41
- array reference
42
- 100kx [index] 0.020000 0.000000 0.020000 ( 0.026329)
43
- 100kx [range] 0.060000 0.000000 0.060000 ( 0.060740)
44
- 100kx [s,1] 0.050000 0.000000 0.050000 ( 0.051872)
45
- 100kx [s,100] 0.040000 0.000000 0.040000 ( 0.043028)
46
-
47
- ..............................................................p. ..
48
-
49
- #####################################
50
- # with bulk processing only
51
- #####################################
52
- ...........F..
53
- test_element_assignment_speed
54
- user system total real
55
- I
56
- 10kx [index]= 0.190000 0.060000 0.250000 ( 0.278135)
57
- 10kx [range]= 12.760000 12.470000 25.230000 ( 40.200132)
58
- 10kx [s,1]= 12.740000 12.360000 25.100000 ( 40.450334)
59
-
60
- IIIIIIIIII
61
- 10kx [index]= 0.240000 0.050000 0.290000 ( 0.287169)
62
- 10kx [range]= 12.410000 13.020000 25.430000 ( 40.168660)
63
- 10kx [s,1]= 12.510000 13.070000 25.580000 ( 40.690157)
64
-
65
- array reference
66
- 10kx [index]= 0.000000 0.000000 0.000000 ( 0.003034)
67
- 10kx [range]= 0.010000 0.000000 0.010000 ( 0.007231)
68
- 10kx [s,1]= 0.000000 0.000000 0.000000 ( 0.005260)
69
-
70
- .
71
- test_element_reference_speed
72
- user system total real
73
- I
74
- 100kx [index] 1.740000 0.670000 2.410000 ( 2.414568)
75
- 100kx [range] 2.510000 0.660000 3.170000 ( 3.183513)
76
- 100kx [s,1] 1.920000 0.680000 2.600000 ( 3.314087)
77
- 100kx [s,100] 11.460000 0.730000 12.190000 ( 12.384289)
78
-
79
- IIIIIIIIII
80
- 100kx [index] 1.420000 0.360000 1.780000 ( 1.785599)
81
- 100kx [range] 0.870000 0.000000 0.870000 ( 0.875749)
82
- 100kx [s,1] 0.330000 0.000000 0.330000 ( 0.326366)
83
- 100kx [s,100] 0.300000 0.000000 0.300000 ( 0.304128)
84
-
85
- array reference
86
- 100kx [index] 0.030000 0.000000 0.030000 ( 0.025789)
87
- 100kx [range] 0.060000 0.000000 0.060000 ( 0.060912)
88
- 100kx [s,1] 0.040000 0.000000 0.040000 ( 0.042461)
89
- 100kx [s,100] 0.050000 0.000000 0.050000 ( 0.051094)
90
-
91
- ..................................F...
@@ -1,147 +0,0 @@
1
- ##########################################
2
- # OSX before ASET optimizations
3
- ##########################################
4
-
5
- test_array_methods
6
- user system total real
7
- 1M << 0.290000 0.010000 0.300000 ( 0.300562)
8
- 1M [] 0.310000 0.000000 0.310000 ( 0.302214)
9
- ....
10
- test_read_from_open_file
11
- user system total real
12
- 1kx read from file 0.040000 0.200000 0.240000 ( 0.519244)
13
- 1kx file.read 0.140000 0.250000 0.390000 ( 0.395224)
14
- .
15
- test_read_in_chunk_vs_read_in_pieces
16
- user system total real
17
- 1kx read in chunk 0.220000 0.160000 0.380000 ( 0.515190)
18
- 1kx read in pieces 0.620000 0.110000 0.730000 ( 0.728748)
19
- 1kx read in one block 0.050000 0.110000 0.160000 ( 0.160819)
20
- 1kx read in blocks 0.390000 0.110000 0.500000 ( 0.495172)
21
- 1kx File.read 0.160000 0.270000 0.430000 ( 0.438735)
22
- .
23
- test_read_into_arrays
24
- user system total real
25
- 100x unpack to array 0.310000 0.030000 0.340000 ( 0.346243)
26
- 100x File.read.unpack 0.940000 0.060000 1.000000 ( 1.022879)
27
- ...
28
- test_unpack_speed
29
- user system total real
30
- 100x unpack 0.300000 0.000000 0.300000 ( 0.313970)
31
- 100x str.unpack 0.930000 0.020000 0.950000 ( 0.960535)
32
- ............................
33
- test_element_assignment_speed_for_index
34
- user system total real
35
- I
36
- 1kx [index]= 0.010000 0.000000 0.010000 ( 0.011112)
37
- 1kx [range]= 0.020000 0.000000 0.020000 ( 0.014436)
38
- 1kx [s,1]= 0.000000 0.000000 0.000000 ( 0.007936)
39
-
40
- IIIIIIIIII
41
- 1kx [index]= 0.010000 0.000000 0.010000 ( 0.012760)
42
- 1kx [range]= 0.020000 0.000000 0.020000 ( 0.014060)
43
- 1kx [s,1]= 0.000000 0.000000 0.000000 ( 0.007683)
44
-
45
- array reference
46
- 1kx [index]= 0.000000 0.000000 0.000000 ( 0.000359)
47
- 1kx [range]= 0.010000 0.000000 0.010000 ( 0.001003)
48
- 1kx [s,1]= 0.000000 0.000000 0.000000 ( 0.000532)
49
-
50
- .
51
- test_element_reference_speed_for_index
52
- user system total real
53
- I
54
- 100kx [index] 0.170000 0.000000 0.170000 ( 0.170148)
55
- 100kx [range] 0.200000 0.000000 0.200000 ( 0.207506)
56
- 100kx [s,1] 0.210000 0.000000 0.210000 ( 0.206175)
57
- 100kx [s,10] 0.190000 0.000000 0.190000 ( 0.196211)
58
- 100kx [s,100] 0.210000 0.000000 0.210000 ( 0.207395)
59
-
60
- IIIIIIIIII
61
- 100kx [index] 0.170000 0.000000 0.170000 ( 0.171383)
62
- 100kx [range] 0.220000 0.000000 0.220000 ( 0.218904)
63
- 100kx [s,1] 0.220000 0.000000 0.220000 ( 0.217726)
64
- 100kx [s,10] 0.220000 0.000000 0.220000 ( 0.227541)
65
- 100kx [s,100] 0.230000 0.010000 0.240000 ( 0.239500)
66
-
67
- array reference
68
- 100kx [index] 0.030000 0.000000 0.030000 ( 0.026727)
69
- 100kx [range] 0.060000 0.000000 0.060000 ( 0.062518)
70
- 100kx [s,1] 0.040000 0.000000 0.040000 ( 0.043396)
71
- 100kx [s,10] 0.040000 0.000000 0.040000 ( 0.043224)
72
- 100kx [s,100] 0.040000 0.000000 0.040000 ( 0.043834)
73
-
74
- ................................................................
75
- test_element_assignment_speed_for_ext_arr
76
- user system total real
77
-
78
- 1kx [index]= 0.120000 0.010000 0.130000 ( 0.120187)
79
- 1kx [range]= 0.080000 0.000000 0.080000 ( 0.082323)
80
- 1kx [s,1]= 0.110000 0.010000 0.120000 ( 0.117434)
81
-
82
- array reference
83
- 1kx [index]= 0.110000 0.000000 0.110000 ( 0.118859)
84
- 1kx [range]= 0.080000 0.010000 0.090000 ( 0.082343)
85
- 1kx [s,1]= 0.110000 0.010000 0.120000 ( 0.116168)
86
-
87
- .
88
- test_element_reference_speed_for_ext_arr
89
- user system total real
90
-
91
- 10kx [index] 0.320000 0.130000 0.450000 ( 0.445001)
92
- 10kx [range] 0.450000 0.110000 0.560000 ( 0.567785)
93
- 10kx [s,1] 0.310000 0.070000 0.380000 ( 0.391895)
94
- 10kx [s,10] 2.490000 0.150000 2.640000 ( 2.669212)
95
-
96
- array reference
97
- 10kx [index] 0.000000 0.000000 0.000000 ( 0.002583)
98
- 10kx [range] 0.010000 0.000000 0.010000 ( 0.004749)
99
- 10kx [s,1] 0.000000 0.000000 0.000000 ( 0.006787)
100
- 10kx [s,10] 0.000000 0.000000 0.000000 ( 0.003268)
101
-
102
- ....................................
103
- test_element_assignment_speed_for_index
104
- user system total real
105
- I
106
- 1kx [index]= 0.020000 0.010000 0.030000 ( 0.026291)
107
- 1kx [range]= 5.400000 1.410000 6.810000 ( 8.220439)
108
- 1kx [s,1]= 5.620000 1.410000 7.030000 ( 8.293661)
109
-
110
- IIIIIIIIII
111
- 1kx [index]= 0.020000 0.010000 0.030000 ( 0.029998)
112
- 1kx [range]= 5.880000 1.480000 7.360000 ( 8.588676)
113
- 1kx [s,1]= 5.580000 1.470000 7.050000 ( 8.308494)
114
-
115
- array reference
116
- 1kx [index]= 0.000000 0.000000 0.000000 ( 0.000329)
117
- 1kx [range]= 0.000000 0.000000 0.000000 ( 0.000700)
118
- 1kx [s,1]= 0.000000 0.000000 0.000000 ( 0.000570)
119
-
120
- .
121
- test_element_reference_speed_for_index
122
- user system total real
123
- I
124
- 100kx [index] 1.950000 0.640000 2.590000 ( 2.598391)
125
- 100kx [range] 3.130000 0.650000 3.780000 ( 3.786401)
126
- 100kx [s,1] 2.390000 0.640000 3.030000 ( 3.029791)
127
- 100kx [s,10] 3.410000 0.650000 4.060000 ( 4.059696)
128
- 100kx [s,100] 11.250000 0.670000 11.920000 ( 11.970536)
129
-
130
- IIIIIIIIII
131
- 100kx [index] 1.610000 0.360000 1.970000 ( 1.972972)
132
- 100kx [range] 1.070000 0.010000 1.080000 ( 1.080418)
133
- 100kx [s,1] 0.380000 0.000000 0.380000 ( 0.382648)
134
- 100kx [s,10] 0.380000 0.000000 0.380000 ( 0.383150)
135
- 100kx [s,100] 0.440000 0.000000 0.440000 ( 0.439358)
136
-
137
- array reference
138
- 100kx [index] 0.020000 0.000000 0.020000 ( 0.026210)
139
- 100kx [range] 0.050000 0.000000 0.050000 ( 0.050129)
140
- 100kx [s,1] 0.040000 0.000000 0.040000 ( 0.033032)
141
- 100kx [s,10] 0.050000 0.000000 0.050000 ( 0.055461)
142
- 100kx [s,100] 0.040000 0.000000 0.040000 ( 0.032775)
143
-
144
- .........................................................................p. ..
145
- Finished in 82.428089 seconds.
146
-
147
- 216 tests, 5189 assertions, 0 failures, 0 errors
@@ -1,80 +0,0 @@
1
- require File.join(File.dirname(__FILE__), '../external_test_helper.rb')
2
- require 'tempfile'
3
- require 'fileutils'
4
-
5
- ENV['ALL'] = 'true'
6
-
7
- class TestCopyFile < Test::Unit::TestCase
8
- include Benchmark
9
- include Tap::Test::SubsetMethods
10
-
11
-
12
- # The lesson from this test is that chunky copy can be
13
- # faster than FileUtils.cp It's interesting to note
14
- # that if you look up the code, you can see that a
15
- # chunky copy is exactly what cp is... with chunks of
16
- # 1024 bytes.
17
- def test_copy_vs_chunky_rewrite
18
- t = Tempfile.new "large_file"
19
- t.close
20
- puts t.path
21
-
22
- prompt_test(:path_to_large_file) do |config|
23
- path = config[:path_to_large_file]
24
-
25
- benchmark_test do |x|
26
- x.report("copy") { FileUtils.cp(path, t.path) }
27
- assert FileUtils.cmp(path, t.path)
28
-
29
- # tenmb = 1 * 2097152
30
- # x.report("chunk 2MB") do
31
- # File.open(path, "r") do |src|
32
- # File.open(t.path, "w") do |target|
33
- # str = ""
34
- # while src.read(tenmb, str)
35
- # target << str
36
- # end
37
- # end
38
- # end
39
- # end
40
-
41
- tenmb = 4 * 2097152
42
- x.report("chunk 8MB") do
43
- File.open(path, "r") do |src|
44
- File.open(t.path, "w") do |target|
45
- str = ""
46
- while src.read(tenmb, str)
47
- target << str
48
- end
49
- end
50
- end
51
- end
52
- assert FileUtils.cmp(path, t.path)
53
-
54
- # tenmb = 4 * 2097152
55
- # x.report("chunk 8MB - no buf") do
56
- # File.open(path, "r") do |src|
57
- # File.open(t.path, "w") do |target|
58
- # while str = src.read(tenmb)
59
- # target << str
60
- # end
61
- # end
62
- # end
63
- # end
64
-
65
- # tenmb = 10 * 2097152
66
- # x.report("chunk 20MB") do
67
- # File.open(path, "r") do |src|
68
- # File.open(t.path, "w") do |target|
69
- # str = ""
70
- # while src.read(tenmb, str)
71
- # target << str
72
- # end
73
- # end
74
- # end
75
- # end
76
-
77
- end
78
- end
79
- end
80
- end
@@ -1,47 +0,0 @@
1
- require File.join(File.dirname(__FILE__), '../external_test_helper.rb')
2
- require 'tempfile'
3
- require 'enumerator'
4
- require 'scanf'
5
-
6
- class TestPosSpeed < Test::Unit::TestCase
7
- include Benchmark
8
-
9
- attr_reader :path
10
-
11
- def setup
12
- t = Tempfile.new('integers')
13
- ints = [*(1..100000).to_a]
14
- t << ints.pack("I*")
15
- t.close
16
- @path = t.path
17
- end
18
-
19
- def test_pos_speed
20
- file = File.open(path, "r")
21
- length = file.stat.size
22
- positions = Array.new(1000) do
23
- p = rand(length)
24
- end
25
-
26
- benchmark_test(20) do |x|
27
- x.report("100kx pos=") do
28
- (1000).times do
29
- positions.each do |p|
30
- file.pos = p
31
- file.pos = p
32
- end
33
- end
34
- end
35
-
36
- x.report("100kx pos= with check") do
37
- (1000).times do
38
- positions.each do |p|
39
- file.pos = p
40
- file.pos = p unless file.pos = p
41
- end
42
- end
43
- end
44
- end
45
- end
46
-
47
- end
@@ -1,55 +0,0 @@
1
- require File.join(File.dirname(__FILE__), '../external_test_helper.rb')
2
- require 'tempfile'
3
- require 'enumerator'
4
- require 'scanf'
5
-
6
- class TestReadTime < Test::Unit::TestCase
7
- include Benchmark
8
-
9
- attr_reader :path, :ints
10
-
11
- def setup
12
- t = Tempfile.new('integers')
13
- ints = [*(1..100000).to_a]
14
- t << ints.pack("I*")
15
- t.close
16
- @path = t.path
17
-
18
- @ints = []
19
- ints.each_slice(1) do |s|
20
- @ints << s
21
- end
22
- end
23
-
24
- def test_read_time
25
- file = File.open(path, "r")
26
- results = nil
27
-
28
- benchmark_test(20) do |x|
29
- x.report("10x full (1)") do
30
- (10).times do
31
- file.pos = 0
32
- str = file.read
33
- results = []
34
- str.unpack("I*").each_slice(1) do |s|
35
- results << s
36
- end
37
- end
38
- end
39
- assert_equal ints, results
40
-
41
- x.report("10x singles (1)") do
42
- (10).times do
43
- file.pos = 0
44
-
45
- results = []
46
- ints.length.times do
47
- results << file.read(4).unpack("I")
48
- end
49
- end
50
- end
51
- assert_equal ints, results
52
- end
53
- end
54
-
55
- end