hflr 1.1.0 → 1.2.0
Sign up to get free protection for your applications and to get access to all the features.
- data/hflr.gemspec +1 -1
- data/lib/hflr/fl_record_file.rb +55 -7
- data/test/flrfile_test.rb +1 -1
- metadata +1 -1
data/hflr.gemspec
CHANGED
data/lib/hflr/fl_record_file.rb
CHANGED
@@ -26,11 +26,18 @@ class FLRFile
|
|
26
26
|
|
27
27
|
def set_fast
|
28
28
|
@fast = !@record_type_labels.is_a?(Hash)
|
29
|
-
unless @fast
|
30
|
-
|
29
|
+
unless @fast
|
30
|
+
raise "Cannot set fast mode with more than one record type."
|
31
31
|
end
|
32
32
|
if @fast
|
33
33
|
@width = get_record_width_from_file
|
34
|
+
puts "record width #{@width.to_s}"
|
35
|
+
records_to_take = 50000001 / @width
|
36
|
+
puts "records_to_take: #{records_to_take}"
|
37
|
+
@buffer_size = @width * records_to_take
|
38
|
+
puts "Buffer size: #{@buffer_size}"
|
39
|
+
@position=0
|
40
|
+
@current_buffer=nil
|
34
41
|
end
|
35
42
|
end
|
36
43
|
|
@@ -41,6 +48,7 @@ class FLRFile
|
|
41
48
|
end
|
42
49
|
|
43
50
|
@offsets =offsets_to_read(ranges, @width)
|
51
|
+
puts "Chunks of data #{@offsets.inspect}"
|
44
52
|
@ranges = ranges
|
45
53
|
end
|
46
54
|
|
@@ -50,7 +58,11 @@ class FLRFile
|
|
50
58
|
end
|
51
59
|
|
52
60
|
def finished?
|
53
|
-
|
61
|
+
if @fast
|
62
|
+
@offsets.empty? && @current_buffer.nil?
|
63
|
+
else
|
64
|
+
@file.eof?
|
65
|
+
end
|
54
66
|
end
|
55
67
|
|
56
68
|
def close
|
@@ -85,10 +97,31 @@ def get_next_known_line_type
|
|
85
97
|
end
|
86
98
|
|
87
99
|
def fast_get_next_known_line_type
|
88
|
-
unless @offsets.nil? || @offsets.empty?
|
89
|
-
|
90
|
-
|
100
|
+
unless @current_buffer.nil? && (@offsets.nil? || @offsets.empty?)
|
101
|
+
if @current_buffer.nil?
|
102
|
+
chunk = @offsets.shift
|
103
|
+
puts "shifting first chunk #{chunk.inspect}"
|
104
|
+
@file.pos = chunk.pos
|
105
|
+
@current_buffer=@file.read(chunk.width)
|
106
|
+
record= @current_buffer.slice(@position,@width)
|
107
|
+
puts "Record was from #{@position} width of #{@width}"
|
108
|
+
puts "Getting record: #{record}"
|
109
|
+
@position += @width
|
110
|
+
puts "Position incremented to #{@position}"
|
111
|
+
puts "Current buffer size: #{@current_buffer.size}"
|
112
|
+
@current_buffer = nil if @position >= @current_buffer.size
|
113
|
+
puts "current_buffer changed to #{@current_buffer}"
|
114
|
+
return record
|
115
|
+
else
|
116
|
+
record= @current_buffer.slice(@position,@width)
|
117
|
+
puts "Getting subsequent record: #{record}"
|
118
|
+
@position += @width
|
119
|
+
@current_buffer = nil if @position >= @current_buffer.size
|
120
|
+
return record
|
121
|
+
end
|
122
|
+
|
91
123
|
else
|
124
|
+
puts "At end of all chunks"
|
92
125
|
nil
|
93
126
|
end
|
94
127
|
end
|
@@ -151,7 +184,22 @@ end
|
|
151
184
|
private
|
152
185
|
|
153
186
|
def offsets_to_read(ranges, width)
|
154
|
-
ranges.map{|r| r.map{|o| o * width}}.flatten.uniq
|
187
|
+
#ranges.map{|r| r.map{|o| o * width}}.flatten.uniq
|
188
|
+
chunk = Struct.new(:pos,:width)
|
189
|
+
chunks = []
|
190
|
+
ranges.each do |range|
|
191
|
+
offsets = range.map{|offset| offset}
|
192
|
+
taken = []
|
193
|
+
|
194
|
+
while !offsets.empty?
|
195
|
+
taken << offsets.shift
|
196
|
+
if taken.size * width == @buffer_size || offsets.empty?
|
197
|
+
chunks << chunk.new(taken.first, taken.size * width)
|
198
|
+
taken = []
|
199
|
+
end # if
|
200
|
+
end # while
|
201
|
+
end # each
|
202
|
+
chunks
|
155
203
|
end
|
156
204
|
|
157
205
|
def get_record_width_from_file
|
data/test/flrfile_test.rb
CHANGED