external 0.1.0 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (49) hide show
  1. data/History +7 -0
  2. data/MIT-LICENSE +1 -3
  3. data/README +162 -127
  4. data/lib/external.rb +2 -3
  5. data/lib/external/base.rb +174 -47
  6. data/lib/external/chunkable.rb +131 -105
  7. data/lib/external/enumerable.rb +78 -33
  8. data/lib/external/io.rb +163 -398
  9. data/lib/external/patches/ruby_1_8_io.rb +31 -0
  10. data/lib/external/patches/windows_io.rb +53 -0
  11. data/lib/external/patches/windows_utils.rb +27 -0
  12. data/lib/external/utils.rb +148 -0
  13. data/lib/external_archive.rb +840 -0
  14. data/lib/external_array.rb +57 -0
  15. data/lib/external_index.rb +1053 -0
  16. metadata +42 -58
  17. data/lib/ext_arc.rb +0 -108
  18. data/lib/ext_arr.rb +0 -727
  19. data/lib/ext_ind.rb +0 -1120
  20. data/test/benchmarks/benchmarks_20070918.txt +0 -45
  21. data/test/benchmarks/benchmarks_20070921.txt +0 -91
  22. data/test/benchmarks/benchmarks_20071006.txt +0 -147
  23. data/test/benchmarks/test_copy_file.rb +0 -80
  24. data/test/benchmarks/test_pos_speed.rb +0 -47
  25. data/test/benchmarks/test_read_time.rb +0 -55
  26. data/test/cached_ext_ind_test.rb +0 -219
  27. data/test/check/benchmark_check.rb +0 -441
  28. data/test/check/namespace_conflicts_check.rb +0 -23
  29. data/test/check/pack_check.rb +0 -90
  30. data/test/ext_arc_test.rb +0 -286
  31. data/test/ext_arr/alt_sep.txt +0 -3
  32. data/test/ext_arr/cr_lf_input.txt +0 -3
  33. data/test/ext_arr/input.index +0 -0
  34. data/test/ext_arr/input.txt +0 -1
  35. data/test/ext_arr/inputb.index +0 -0
  36. data/test/ext_arr/inputb.txt +0 -1
  37. data/test/ext_arr/lf_input.txt +0 -3
  38. data/test/ext_arr/lines.txt +0 -19
  39. data/test/ext_arr/without_index.txt +0 -1
  40. data/test/ext_arr_test.rb +0 -534
  41. data/test/ext_ind_test.rb +0 -1472
  42. data/test/external/base_test.rb +0 -74
  43. data/test/external/chunkable_test.rb +0 -182
  44. data/test/external/index/input.index +0 -0
  45. data/test/external/index/inputb.index +0 -0
  46. data/test/external/io_test.rb +0 -414
  47. data/test/external_test_helper.rb +0 -31
  48. data/test/external_test_suite.rb +0 -4
  49. data/test/test_array.rb +0 -1192
@@ -1,23 +0,0 @@
1
- module External
2
- class Array
3
- end
4
- end
5
-
6
- require 'test/unit'
7
-
8
- class NamespaceConfilctsCheck < Test::Unit::TestCase
9
- include External
10
-
11
- # this is why it's problematic to use a naming scheme like:
12
- # External::Index
13
- # External::Array
14
- # External::Archive
15
- #
16
- # even if you alias to the top level, within External itself
17
- # there is some ambiguity about whether you're using Array
18
- # or External::Array
19
-
20
- def test_array_now_refers_to_external_array
21
- assert_equal Array, External::Array
22
- end
23
- end
@@ -1,90 +0,0 @@
1
- require 'test/unit'
2
-
3
- # a variety of tests that establish some basic facts/assumptions
4
- # that get leveraged in somewhere in the library
5
- class PackCheck < Test::Unit::TestCase
6
-
7
- # NOTE: upon pack:
8
- # unsigned values throw an error if > MAX or < -MAX
9
- # negative values are the same as positive values counting back from MAX
10
-
11
- LONG_MIN = -2147483648
12
- LONG_MAX = 2147483647
13
-
14
- ULONG_MIN = 0
15
- ULONG_MAX = 4294967295
16
-
17
- LLONG_MIN = -9223372036854775808
18
- LLONG_MAX = 9223372036854775807
19
-
20
- ULLONG_MIN = 0
21
- ULLONG_MAX = 18446744073709551615
22
-
23
- def test_negative_unsigned_values_count_back_from_max_in_pack_and_unpack
24
- assert_equal [ULONG_MAX], [-1].pack('I').unpack('I')
25
- assert_equal [ULONG_MAX], [-1].pack('L').unpack('L')
26
- assert_equal [ULLONG_MAX], [-1].pack('Q').unpack('Q')
27
- end
28
-
29
- def test_signed_values_beyond_min_count_back_from_max_in_pack_and_unpack
30
- assert_equal [LONG_MAX], [LONG_MIN-1].pack('i').unpack('i')
31
- assert_equal [LONG_MAX], [LONG_MIN-1].pack('l').unpack('l')
32
- assert_equal [LLONG_MAX], [LLONG_MIN-1].pack('q').unpack('q')
33
- end
34
-
35
- def test_signed_values_beyond_max_count_up_from_min_in_pack_and_unpack
36
- assert_equal [LONG_MIN], [LONG_MAX+1].pack('i').unpack('i')
37
- assert_equal [LONG_MIN], [LONG_MAX+1].pack('l').unpack('l')
38
- assert_equal [LLONG_MIN], [LLONG_MAX+1].pack('q').unpack('q')
39
- end
40
-
41
- def test_numeric_ranges_for_pack_and_unpack
42
- # I,L handle an unsigned long
43
- ['I', 'L'].each do |format|
44
- assert_equal [ULONG_MIN], [ULONG_MIN].pack(format).unpack(format)
45
- assert_equal [ULONG_MAX], [ULONG_MAX].pack(format).unpack(format)
46
-
47
- #assert_equal [ULONG_MIN], [ULONG_MAX+1].pack(format).unpack(format)
48
- assert_equal [ULONG_MAX], [ULONG_MIN-1].pack(format).unpack(format)
49
-
50
- assert_raise(RangeError) { [-(ULONG_MAX+1)].pack(format) }
51
- assert_raise(RangeError) { [(ULONG_MAX+1)].pack(format) }
52
- end
53
-
54
- # i,l handle an signed long
55
- ['i', 'l'].each do |format|
56
- assert_equal [LONG_MIN], [LONG_MIN].pack(format).unpack(format)
57
- assert_equal [LONG_MAX], [LONG_MAX].pack(format).unpack(format)
58
-
59
- assert_equal [LONG_MIN], [LONG_MAX+1].pack(format).unpack(format)
60
- assert_equal [LONG_MAX], [LONG_MIN-1].pack(format).unpack(format)
61
-
62
- assert_raise(RangeError) { [-2*(LONG_MAX+1)].pack(format) }
63
- assert_raise(RangeError) { [2*(LONG_MAX+1)].pack(format) }
64
- end
65
-
66
- # Q handles an unsigned long long
67
- ['Q'].each do |format|
68
- assert_equal [ULLONG_MIN], [ULLONG_MIN].pack(format).unpack(format)
69
- assert_equal [ULLONG_MAX], [ULLONG_MAX].pack(format).unpack(format)
70
-
71
- #assert_equal [ULLONG_MIN], [ULLONG_MAX+1].pack(format).unpack(format)
72
- assert_equal [ULLONG_MAX], [ULLONG_MIN-1].pack(format).unpack(format)
73
-
74
- assert_raise(RangeError) { [-(ULLONG_MAX+1)].pack(format) }
75
- assert_raise(RangeError) { [(ULLONG_MAX+1)].pack(format) }
76
- end
77
-
78
- # q handles an signed long long
79
- ['q'].each do |format|
80
- assert_equal [LLONG_MIN], [LLONG_MIN].pack(format).unpack(format)
81
- assert_equal [LLONG_MAX], [LLONG_MAX].pack(format).unpack(format)
82
-
83
- assert_equal [LLONG_MIN], [LLONG_MAX+1].pack(format).unpack(format)
84
- assert_equal [LLONG_MAX], [LLONG_MIN-1].pack(format).unpack(format)
85
-
86
- assert_raise(RangeError) { [-2*(LLONG_MAX+1)].pack(format) }
87
- assert_raise(RangeError) { [2*(LLONG_MAX+1)].pack(format) }
88
- end
89
- end
90
- end
data/test/ext_arc_test.rb DELETED
@@ -1,286 +0,0 @@
1
- require File.join(File.dirname(__FILE__), 'external_test_helper.rb')
2
- require 'ext_arc'
3
-
4
- class ExtArcTest < Test::Unit::TestCase
5
- include Benchmark
6
-
7
- attr_reader :ea
8
-
9
- def setup
10
- @ea = ExtArc.new
11
- end
12
-
13
- #
14
- # readme doc test
15
- #
16
-
17
- def test_readme_doc_for_ext_arc
18
- arc = ExtArc[">swift", ">brown", ">fox"]
19
- assert_equal ">fox", arc[2]
20
- assert_equal [">swift", ">brown", ">fox"], arc.to_a
21
-
22
- assert_equal Tempfile, arc.io.class
23
- arc.io.rewind
24
- assert_equal ">swift>brown>fox", arc.io.read
25
-
26
- Tempfile.open('test_readme_doc_for_ext_arc') do |file|
27
- file << ">swift>brown>fox"
28
- file.flush
29
-
30
- arc = ExtArc.new(file)
31
- assert_equal [], arc.to_a
32
- arc.reindex_by_sep(:sep_string => ">", :entry_follows_sep => true)
33
- assert_equal [">swift", ">brown", ">fox"], arc.to_a
34
-
35
- arc = ExtArc.new(file)
36
- assert_equal [], arc.to_a
37
- arc.reindex_by_scan(/>\w*/)
38
- assert_equal [">swift", ">brown", ">fox"], arc.to_a
39
- end
40
- end
41
-
42
- #
43
- # entry_to_str, str_to_entry test
44
- #
45
-
46
- def test_entry_to_str_simply_stringifies_entry
47
- obj = "abc"
48
- assert_equal obj.to_s, ea.entry_to_str(obj)
49
-
50
- obj = 1
51
- assert_equal obj.to_s, ea.entry_to_str(obj)
52
- end
53
-
54
- def test_entry_to_str_simply_return_input
55
- obj = "abc"
56
- assert_equal obj.object_id, ea.str_to_entry(obj).object_id
57
- end
58
-
59
- #####################################
60
- # indexing tests
61
- #####################################
62
-
63
- def reindex_by_line_test(expected, options={}, &block)
64
- cases = {
65
- :end_midline => "012\n\n56\n\n9",
66
- :end_on_line => "012\n\n56\n\n9\n",
67
- :end_on_break => "012\n\n56\n\n9\n\n",
68
- :no_break => "0123456789",
69
- :backing_breaks => "012\n\n\n\n\n\n9",
70
- :cr_lf => "012\r\n\r\n56\r\n\r\n9"
71
- }
72
-
73
- cases.each_pair do |key, string|
74
- next unless expected.has_key?(key)
75
-
76
- Tempfile.open("reindex_by_line") do |tempfile|
77
- # MUST SET binmode so that these tests work properly on Windows
78
- tempfile.binmode
79
- tempfile << string
80
- tempfile.flush
81
-
82
- begin
83
- ea = ExtArc.new(tempfile)
84
- ea.reindex_by_line(options, &block)
85
- assert_equal expected[key], ea.to_a, key
86
- ensure
87
- ea.close
88
- end
89
- end
90
- end
91
- end
92
-
93
- def reindex_by_scan_test(expected, carryover_limit=nil, options={}, &block)
94
- cases = {
95
- :end_midline => "012\n\n56\n\n9",
96
- :end_on_line => "012\n\n56\n\n9\n",
97
- :end_on_break => "012\n\n56\n\n9\n\n",
98
- :no_break => "0123456789",
99
- :backing_breaks => "012\n\n\n\n\n\n9",
100
- :cr_lf => "012\r\n\r\n56\r\n\r\n9"}
101
-
102
- cases.each_pair do |key, string|
103
- next unless expected.has_key?(key)
104
-
105
- Tempfile.open("reindex_by_scan") do |tempfile|
106
- tempfile.binmode
107
- tempfile << string
108
- tempfile.flush
109
-
110
- begin
111
- ea = ExtArc.new(tempfile)
112
- ea.reindex_by_scan
113
- assert_equal expected[key].length, ea.length, key
114
- assert_equal expected[key], ea.to_a, key
115
- ensure
116
- ea.close
117
- end
118
- end
119
- end
120
- end
121
-
122
- def test_reindex_treats_each_line_as_break_by_default
123
- reindex_by_line_test(
124
- :end_midline => ["012\n", "\n", "56\n", "\n", "9"],
125
- :end_on_line => ["012\n", "\n", "56\n", "\n", "9\n"],
126
- :end_on_break => ["012\n", "\n", "56\n", "\n", "9\n", "\n"],
127
- :no_break => ["0123456789"],
128
- :backing_breaks => ["012\n", "\n", "\n", "\n", "\n", "\n", "9"],
129
- :cr_lf => ["012\r\n", "\r\n", "56\r\n", "\r\n", "9"])
130
-
131
- reindex_by_scan_test(
132
- :end_midline => ["012\n", "\n", "56\n", "\n", "9"],
133
- :end_on_line => ["012\n", "\n", "56\n", "\n", "9\n"],
134
- :end_on_break => ["012\n", "\n", "56\n", "\n", "9\n", "\n"],
135
- :no_break => ["0123456789"],
136
- :backing_breaks => ["012\n", "\n", "\n", "\n", "\n", "\n", "9"],
137
- :cr_lf => ["012\r\n", "\r\n", "56\r\n", "\r\n", "9"])
138
- end
139
-
140
- def BROKEN_test_reindex_by_scan_with_chunk_size_less_than_full_length
141
- reindex_by_scan_test({
142
- :end_midline => ["012\n", "\n", "56\n", "\n", "9"],
143
- :end_on_line => ["012\n", "\n", "56\n", "\n", "9\n"],
144
- :end_on_break => ["012\n", "\n", "56\n", "\n", "9\n", "\n"],
145
- :no_break => ["0123456789"],
146
- :backing_breaks => ["012\n", "\n", "\n", "\n", "\n", "\n", "9"],
147
- :cr_lf => ["012\r\n", "\r\n", "56\r\n", "\r\n", "9"]},
148
- nil,
149
- :buffer_size => 3)
150
- end
151
-
152
- def test_reindex_block_determines_if_line_is_a_break
153
- reindex_by_line_test(
154
- :end_midline => ["012\n\n", "56\n\n", "9"],
155
- :end_on_line => ["012\n\n", "56\n\n", "9\n"],
156
- :end_on_break => ["012\n\n", "56\n\n", "9\n\n"],
157
- :no_break => ["0123456789"],
158
- :backing_breaks => ["012\n\n", "\n", "\n", "\n", "\n", "9"],
159
- :cr_lf => ["012\r\n\r\n", "56\r\n\r\n", "9"]) do |line|
160
- line.strip.empty?
161
- end
162
- end
163
-
164
- def test_reindex_breaking_before
165
- reindex_by_line_test({
166
- :end_midline => ["012\n", "\n56\n", "\n9"],
167
- :end_on_line => ["012\n", "\n56\n", "\n9\n"],
168
- :end_on_break => ["012\n", "\n56\n", "\n9\n", "\n"],
169
- :no_break => ["0123456789"],
170
- :backing_breaks => ["012\n", "\n", "\n", "\n", "\n", "\n9"],
171
- :cr_lf => ["012\r\n", "\r\n56\r\n", "\r\n9"]},
172
- :break_before => true) do |line|
173
- line.strip.empty?
174
- end
175
- end
176
-
177
- def test_reindex_excluding_break
178
- reindex_by_line_test({
179
- :end_midline => ["012\n", "56\n", "9"],
180
- :end_on_line => ["012\n", "56\n", "9\n"],
181
- :end_on_break => ["012\n", "56\n", "9\n"],
182
- :no_break => ["0123456789"],
183
- :backing_breaks => ["012\n", "9"],
184
- :cr_lf => ["012\r\n", "56\r\n", "9"]},
185
- :exclude_break => true) do |line|
186
- line.strip.empty?
187
- end
188
- end
189
-
190
- def test_reindex_breaking_before_and_excluding_break
191
- # note this is the same as simply excluding the break
192
- reindex_by_line_test({
193
- :end_midline => ["012\n", "56\n", "9"],
194
- :end_on_line => ["012\n", "56\n", "9\n"],
195
- :end_on_break => ["012\n", "56\n", "9\n"],
196
- :no_break => ["0123456789"],
197
- :backing_breaks => ["012\n", "9"],
198
- :cr_lf => ["012\r\n", "56\r\n", "9"]},
199
- :exclude_break => true,
200
- :break_before => true) do |line|
201
- line.strip.empty?
202
- end
203
- end
204
-
205
- def test_reindex_with_alt_sep_string
206
- reindex_by_line_test({
207
- :end_midline => ["012\n\n", "56\n\n", "9"],
208
- :end_on_line => ["012\n\n", "56\n\n", "9\n"],
209
- :end_on_break => ["012\n\n", "56\n\n", "9\n\n"],
210
- :no_break => ["0123456789"],
211
- :backing_breaks => ["012\n\n", "\n\n", "\n\n", "9"],
212
- :cr_lf => ["012\r\n\r\n56\r\n\r\n9"]},
213
- :sep_string => "\n\n")
214
-
215
- reindex_by_line_test({
216
- :end_midline => ["012\n\n56", "\n\n9"],
217
- :end_on_line => ["012\n\n56", "\n\n9\n"],
218
- :end_on_break => ["012\n\n56", "\n\n9\n\n"],
219
- :no_break => ["0123456", "789"],
220
- :backing_breaks => ["012\n\n\n\n\n\n9"],
221
- :cr_lf => ["012\r\n\r\n56", "\r\n\r\n9"]},
222
- :sep_string => "56")
223
- end
224
-
225
- # #
226
- # # file format tests
227
- # #
228
- #
229
- # def file_format_test(path, &block)
230
- # begin
231
- # filepath = tempfile('format')
232
- #
233
- # FileUtils.cp(ifs.filepath(:root, path), filepath + ".txt")
234
- # FileUtils.cp(input_base + '.index', filepath + ".index")
235
- #
236
- # uaio = ExtArr.open(filepath + ".txt", 'ru')
237
- # yield(uaio)
238
- # ensure
239
- # uaio.close
240
- # end
241
- # end
242
- #
243
- # def test_cr_lf_file
244
- # file_format_test('cr_lf_input.txt') do |aio|
245
- # aio.reindex_by_line
246
- #
247
- # assert_equal "012\r\n", aio[0]
248
- # assert_equal "56\r\n", aio[1]
249
- # assert_equal "9", aio[2]
250
- # end
251
- # end
252
- #
253
- # def test_parse_from_lf_file
254
- # file_format_test('lf_input.txt') do |aio|
255
- # aio.reindex_by_line
256
- #
257
- # assert_equal "012\n", aio[0]
258
- # assert_equal "56\n", aio[1]
259
- # assert_equal "9", aio[2]
260
- # end
261
- # end
262
- #
263
- # def test_parse_from_alt_sep
264
- # file_format_test('alt_sep.txt') do |aio|
265
- # aio.reindex_by_line do |line|
266
- # line =~ /^>/
267
- # end
268
- #
269
- # assert_equal ">abc\r\n", aio[0]
270
- # assert_equal ">def\r\n", aio[1]
271
- # assert_equal ">gh", aio[2]
272
- # end
273
- # end
274
-
275
- #######################
276
- # Benchmarks
277
- #######################
278
-
279
- def break_test_reindex_speed
280
- aiobm_test('r') do |x, type, aio|
281
-
282
- x.report("#{type} by_line") { aio.reindex_by_line }
283
- x.report("#{type} by_scan") { aio.reindex_by_scan }
284
- end
285
- end
286
- end
@@ -1,3 +0,0 @@
1
- >abc
2
- >def
3
- >gh
@@ -1,3 +0,0 @@
1
- 012
2
- 56
3
- 9
Binary file
@@ -1 +0,0 @@
1
- abcdefgh
Binary file
@@ -1 +0,0 @@
1
- abcdefgh
@@ -1,3 +0,0 @@
1
- 012
2
- 56
3
- 9
@@ -1,19 +0,0 @@
1
- line 1
2
-
3
- line 2
4
-
5
- line 3
6
-
7
- line 4
8
-
9
- line 5
10
-
11
- line 6
12
-
13
- line 7
14
-
15
- line 8
16
-
17
- line 9
18
-
19
- line 10