carray 3.0.5 → 3.0.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +233 -0
- data/README.md +1 -1
- data/carray.gemspec +3 -2
- data/ext/ca_axis_dispatch.c +6 -0
- data/ext/ca_axis_group.c +1 -1
- data/ext/ca_categorical_iterator.c +7 -7
- data/ext/ca_kernel_iterator.c +116 -231
- data/ext/ca_kernel_iterator.h +410 -254
- data/ext/ca_obj_bincmp.c +9 -1
- data/ext/ca_obj_binop.c +10 -1
- data/ext/ca_obj_bitfield.c +2 -0
- data/ext/ca_obj_byte_swap.c +2 -0
- data/ext/ca_obj_face.c +2 -0
- data/ext/ca_obj_fake.c +3 -1
- data/ext/ca_obj_grid.c +167 -12
- data/ext/ca_obj_meld.c +69 -113
- data/ext/ca_obj_moncmp.c +9 -1
- data/ext/ca_obj_monop.c +40 -2
- data/ext/ca_obj_object.c +54 -20
- data/ext/ca_obj_reduce.c +15 -1
- data/ext/ca_obj_refer.c +65 -20
- data/ext/ca_obj_remap.c +69 -10
- data/ext/ca_obj_roll.c +1 -0
- data/ext/ca_obj_select.c +4 -1
- data/ext/ca_obj_select_axis.c +1 -0
- data/ext/ca_obj_shift.c +1 -0
- data/ext/ca_obj_stack.c +310 -111
- data/ext/ca_obj_stride.c +131 -45
- data/ext/ca_obj_string.c +18 -2
- data/ext/ca_obj_tile.c +1 -0
- data/ext/ca_obj_triop.c +9 -1
- data/ext/ca_obj_window.c +1 -0
- data/ext/ca_op_ipower.c +8 -8
- data/ext/caframe_csv_reader.c +1 -1
- data/ext/carray.h +2 -1
- data/ext/carray_access.c +27 -64
- data/ext/carray_address_basis.c +134 -0
- data/ext/carray_attribute.c +8 -0
- data/ext/carray_broadcast.c +3 -4
- data/ext/carray_build_flags.h +1 -1
- data/ext/carray_cast.c +8 -11
- data/ext/carray_conversion.c +9 -5
- data/ext/carray_core.c +811 -72
- data/ext/carray_element.c +13 -9
- data/ext/carray_histogram.c +25 -3
- data/ext/carray_internal.h +147 -12
- data/ext/carray_kernels_bincmp.c +2007 -3
- data/ext/carray_kernels_binop.c +25365 -5199
- data/ext/carray_kernels_init.c +8 -8
- data/ext/carray_kernels_monop.c +10696 -3678
- data/ext/carray_kernels_reduce_aggregate.c +4429 -702
- data/ext/carray_kernels_reduce_boolean.c +1533 -15
- data/ext/carray_kernels_reduce_cumulative.c +1808 -248
- data/ext/carray_kernels_reduce_extreme.c +4508 -511
- data/ext/carray_kernels_reduce_variance.c +2284 -152
- data/ext/carray_kernels_scan.c +240 -240
- data/ext/carray_kernels_search.c +4211 -264
- data/ext/carray_kernels_sort.c +367 -450
- data/ext/carray_lazy.c +47 -0
- data/ext/carray_loop.c +1 -1
- data/ext/carray_mask.c +130 -139
- data/ext/carray_median_percentile.c +4 -4
- data/ext/carray_memory_view.c +13 -7
- data/ext/carray_operator.c +147 -44
- data/ext/carray_order.c +3 -2
- data/ext/carray_random.c +4 -1
- data/ext/carray_slab.c +97 -21
- data/ext/carray_sort.c +1 -1
- data/ext/carray_test.c +74 -19
- data/ext/carray_time_text.c +2 -2
- data/ext/extconf.rb +11 -0
- data/ext/mkkernel.rb +346 -94
- data/ext/ruby_carray.c +4 -1
- data/ext/version.h +4 -4
- data/lib/carray/attribute.rb +25 -1
- data/lib/carray/autoload_carray.rb +3 -3
- data/lib/carray/autoload_method_extension.rb +5 -3
- data/lib/carray/axis_group.rb +13 -7
- data/lib/carray/basics.rb +1 -1
- data/lib/carray/block_iterator.rb +17 -3
- data/lib/carray/categorical.rb +6 -8
- data/lib/carray/categorical_iterator.rb +2 -2
- data/lib/carray/complex.rb +2 -2
- data/lib/carray/conditional.rb +3 -2
- data/lib/carray/frame/column_verbs.rb +45 -14
- data/lib/carray/frame/convert.rb +19 -0
- data/lib/carray/frame/csv_parser.rb +32 -32
- data/lib/carray/frame/describe.rb +4 -2
- data/lib/carray/frame/frame.rb +399 -78
- data/lib/carray/frame/group.rb +679 -53
- data/lib/carray/frame/io.rb +209 -71
- data/lib/carray/frame/join.rb +146 -37
- data/lib/carray/frame/records.rb +3 -3
- data/lib/carray/frame/reshape.rb +89 -11
- data/lib/carray/frame/sort.rb +57 -9
- data/lib/carray/frame/verbs.rb +384 -49
- data/lib/carray/frame/windows.rb +44 -6
- data/lib/carray/inspect.rb +7 -2
- data/lib/carray/iterator.rb +32 -0
- data/lib/carray/lazy.rb +17 -8
- data/lib/carray/mask_gap_fill.rb +37 -17
- data/lib/carray/meld_reduce.rb +1 -1
- data/lib/carray/methods/choose.rb +3 -2
- data/lib/carray/methods/meshgrid.rb +3 -1
- data/lib/carray/runtime.rb +47 -8
- data/lib/carray/segment_iterator.rb +23 -8
- data/lib/carray/serialize.rb +34 -17
- data/lib/carray/stack.rb +18 -6
- data/lib/carray/time.rb +5 -0
- data/lib/carray/window_iterator.rb +16 -4
- data/yard-stubs/ca_obj_array.rb +26 -1
- data/yard-stubs/carray_cast.rb +8 -5
- data/yard-stubs/carray_conversion.rb +8 -2
- data/yard-stubs/carray_mask.rb +20 -1
- data/yard-stubs/carray_math.rb +31 -16
- data/yard-stubs/carray_memory_view.rb +7 -0
- data/yard-stubs/carray_test.rb +7 -4
- metadata +18 -4
data/lib/carray/frame/io.rb
CHANGED
|
@@ -9,26 +9,32 @@ class CAFrame
|
|
|
9
9
|
# over one object array (§4.2). Casting is a separate step: +types:+ takes
|
|
10
10
|
# anything +cast+ takes as a map -- { "temp" => :float64 }, :infer, or a
|
|
11
11
|
# map with +default:+ -- and casts the frame with it on load, which is the
|
|
12
|
-
# same as calling +cast+ afterwards.
|
|
13
|
-
#
|
|
12
|
+
# same as calling +cast+ afterwards. The default is :infer, which gives
|
|
13
|
+
# each column the type all of its cells read as (see +infer_types+);
|
|
14
|
+
# +types: :string+ leaves every column text. A Hash names the types of some
|
|
15
|
+
# columns and infers the rest, unless its +default:+ says otherwise
|
|
16
|
+
# (+default: :string+ keeps the rest text); +nil+ for a column keeps it
|
|
17
|
+
# text. Broken cells become UNDEF automatically (parse-mask, §6-2).
|
|
14
18
|
#
|
|
15
|
-
# CAFrame.
|
|
16
|
-
# CAFrame.
|
|
17
|
-
# CAFrame.
|
|
18
|
-
# CAFrame.
|
|
19
|
+
# CAFrame.read_csv("obs.csv") # types inferred
|
|
20
|
+
# CAFrame.read_csv("obs.csv", types: :string) # every column text
|
|
21
|
+
# CAFrame.read_csv("obs.csv", types: { "code" => :int32, "id" => nil })
|
|
22
|
+
# CAFrame.read_csv("obs.csv", types: { default: :string, "temp" => :float64 })
|
|
23
|
+
# CAFrame.read_csv("obs.csv", types: { default: :infer, "code" => :int32, "id" => nil })
|
|
24
|
+
# CAFrame.read_csv("obs.csv", types: { default: :float64, "station" => nil })
|
|
25
|
+
# CAFrame.read_csv("obs.csv", types: { "station" => :categorical })
|
|
19
26
|
#
|
|
20
|
-
# +source+ is a path, or an open IO -- anything answering +gets
|
|
21
|
-
# StringIO is. So CSV that is already in memory does not have to go to a
|
|
22
|
-
# temporary file first:
|
|
27
|
+
# +source+ is a path, or an open IO -- anything answering +gets+:
|
|
23
28
|
#
|
|
24
|
-
# CAFrame.
|
|
25
|
-
#
|
|
26
|
-
#
|
|
29
|
+
# CAFrame.read_csv("obs.csv") # a path
|
|
30
|
+
# File.open("obs.csv") { |io| CAFrame.read_csv(io) }
|
|
31
|
+
# CAFrame.read_csv($stdin)
|
|
27
32
|
#
|
|
28
33
|
# A String is always read as a path, never as CSV text: guessing between the
|
|
29
34
|
# two by looking for a newline is the kind of guess that is right until it is
|
|
30
|
-
# not
|
|
31
|
-
# from where it is and left open -- the caller opened
|
|
35
|
+
# not. CSV text already in hand goes to +from_csv+, which takes the same
|
|
36
|
+
# options. An IO is read from where it is and left open -- the caller opened
|
|
37
|
+
# it and closes it.
|
|
32
38
|
#
|
|
33
39
|
# Parsing uses the built-in fast tokenizer (CSVParser). Options:
|
|
34
40
|
# sep: field separator (default ",")
|
|
@@ -70,32 +76,35 @@ class CAFrame
|
|
|
70
76
|
# fields are read as the data fields are, so strip: trims them
|
|
71
77
|
# too
|
|
72
78
|
#
|
|
73
|
-
# CAFrame.
|
|
74
|
-
# CAFrame.
|
|
75
|
-
# CAFrame.
|
|
76
|
-
# CAFrame.
|
|
77
|
-
# CAFrame.
|
|
78
|
-
# CAFrame.
|
|
79
|
-
# CAFrame.
|
|
79
|
+
# CAFrame.read_csv("obs.csv", header: 2) # a title above
|
|
80
|
+
# CAFrame.read_csv("obs.csv", header: 0, data: 3) # units on lines 1-2
|
|
81
|
+
# CAFrame.read_csv("big.csv", data: 1..100) # the first 100 lines
|
|
82
|
+
# CAFrame.read_csv("raw.csv", column_names: %w[date temp rh])
|
|
83
|
+
# CAFrame.read_csv("wide.csv", columns: %w[date temp])
|
|
84
|
+
# CAFrame.read_csv("raw.csv", header: false, columns: { 0 => "date", 3 => "temp" })
|
|
85
|
+
# CAFrame.read_csv("obs.csv", attrs: [:units]) # units on line 1
|
|
80
86
|
#
|
|
81
87
|
# A block, given the reader, reads in any other order with +skip+ /
|
|
82
88
|
# +header+ / +header(name)+ / +column_names+ / +columns+ / +data+ (see
|
|
83
89
|
# CSVReader):
|
|
84
90
|
#
|
|
85
|
-
# CAFrame.
|
|
91
|
+
# CAFrame.read_csv("obs.csv") { |r| r.skip 2; r.header; r.skip 1; r.data }
|
|
86
92
|
#
|
|
87
93
|
# A missing field -- an unquoted empty one, or a cell a short row never
|
|
88
94
|
# reached -- is UNDEF in the frame, whether or not the column is cast by
|
|
89
95
|
# +types:+. A quoted empty field ("") is the empty string, which is a value.
|
|
90
|
-
# So the mask +to_csv+ writes comes back as a mask.
|
|
96
|
+
# So the mask +to_csv+ writes comes back as a mask. A blank value (a quoted
|
|
97
|
+
# "", or spaces kept by strip: false) is no number, truth value or time, so
|
|
98
|
+
# a column holding one is not inferred and stays text; name its type in
|
|
99
|
+
# +types:+ to read it as numbers with the blank cells UNDEF.
|
|
91
100
|
#
|
|
92
101
|
# Files that spell "missing" some other way say so with +missing:+. A field
|
|
93
102
|
# whose text is one of the given tokens is UNDEF too, before +types:+
|
|
94
103
|
# casts, so a -999 sentinel never reaches the column as a number:
|
|
95
104
|
#
|
|
96
|
-
# CAFrame.
|
|
97
|
-
# CAFrame.
|
|
98
|
-
# CAFrame.
|
|
105
|
+
# CAFrame.read_csv("obs.csv", missing: ["-999", "///"]) # every column
|
|
106
|
+
# CAFrame.read_csv("obs.csv", missing: { "temp" => "-999" }) # one column
|
|
107
|
+
# CAFrame.read_csv("obs.csv", missing: { default: "-999", "id" => [] })
|
|
99
108
|
#
|
|
100
109
|
# Tokens are Strings and are compared with the field's text as read (after
|
|
101
110
|
# +strip:+, quoted or not), so "-999" does not match "-999.0". In a Hash,
|
|
@@ -104,35 +113,42 @@ class CAFrame
|
|
|
104
113
|
# +:default+ the columns it does not name keep only the empty field. As in
|
|
105
114
|
# +to_csv+, "" means the empty field, so a quoted "" is still a value.
|
|
106
115
|
#
|
|
107
|
-
#
|
|
108
|
-
# over one backing object array (§3.6 view-by-default).
|
|
116
|
+
# With +types: :string+, the columns are handed to the frame as CABlock views
|
|
117
|
+
# over one backing object array (§3.6 view-by-default). Otherwise, the
|
|
109
118
|
# body is read without making a String per cell: a column cast to a number
|
|
110
119
|
# is read from the text directly, and a column left as text becomes its own
|
|
111
120
|
# CAString. Casting a column materializes it, and +copy+ gives an
|
|
112
121
|
# independent frame.
|
|
113
|
-
def self.
|
|
122
|
+
def self.read_csv(source, types: :infer, on_error: :mask, missing: nil,
|
|
114
123
|
header: nil, data: nil, column_names: nil, columns: nil,
|
|
115
124
|
attrs: nil, sep: ",", quote: '"', strip: false,
|
|
116
125
|
encoding: "bom|utf-8", parser: nil, &block)
|
|
117
126
|
layout = { header: header, data: data, column_names: column_names,
|
|
118
127
|
columns: columns, attrs: attrs }.compact
|
|
119
128
|
if block && !layout.empty?
|
|
120
|
-
raise ArgumentError, "
|
|
129
|
+
raise ArgumentError, "read_csv: give #{layout.keys.map { |k| "#{k}:" }.join(', ')} " \
|
|
121
130
|
"or a reading block, not both"
|
|
122
131
|
end
|
|
123
132
|
# The parser reads the header, the data and the columns itself.
|
|
124
133
|
if parser && !layout.empty?
|
|
125
|
-
raise ArgumentError, "
|
|
134
|
+
raise ArgumentError, "read_csv: give #{layout.keys.map { |k| "#{k}:" }.join(', ')} " \
|
|
126
135
|
"or parser:, not both"
|
|
127
136
|
end
|
|
128
137
|
if block && block.arity != 1
|
|
129
|
-
raise ArgumentError, "
|
|
130
|
-
"parameter, as in
|
|
138
|
+
raise ArgumentError, "read_csv: the reading block takes the reader as its " \
|
|
139
|
+
"parameter, as in read_csv(path) { |r| r.skip 2; r.header; r.data }"
|
|
140
|
+
end
|
|
141
|
+
# A String is a path, always. One with a line break in it that names no
|
|
142
|
+
# file is likely CSV text; the missing file says how to give text.
|
|
143
|
+
if source.is_a?(String) && source.include?("\n") && !File.exist?(source)
|
|
144
|
+
raise Errno::ENOENT, "#{source.inspect}; a String is a path, so to read CSV text " \
|
|
145
|
+
"give it to from_csv(text)"
|
|
131
146
|
end
|
|
132
147
|
missing = missing_tokens(missing) if missing
|
|
148
|
+
types = csv_types(types)
|
|
133
149
|
# With types:, the body is read into CAConstString columns, so a column
|
|
134
150
|
# cast to a number never becomes Strings; the text columns left over
|
|
135
|
-
# become CAString below.
|
|
151
|
+
# become CAString below. With types: :string every column stays text, so the
|
|
136
152
|
# body goes straight to Strings. missing: masks text in place, which
|
|
137
153
|
# wants the Strings first, so it takes that path too.
|
|
138
154
|
read_text_as_const_string = !types.nil? && missing.nil?
|
|
@@ -142,7 +158,7 @@ class CAFrame
|
|
|
142
158
|
elsif source.respond_to?(:gets)
|
|
143
159
|
with_encoding_hint("open the IO in the file's encoding, as in " \
|
|
144
160
|
"File.open(path, \"r:<file encoding>:UTF-8\")") do
|
|
145
|
-
|
|
161
|
+
parse_csv_io(source, sep: sep, quote: quote, strip: strip, layout: layout,
|
|
146
162
|
read_text_as_const_string: read_text_as_const_string, &block)
|
|
147
163
|
end
|
|
148
164
|
else
|
|
@@ -150,7 +166,7 @@ class CAFrame
|
|
|
150
166
|
"file's encoding, as in " \
|
|
151
167
|
"encoding: \"<file encoding>:UTF-8\"") do
|
|
152
168
|
File.open(source, "r:#{encoding}") do |io|
|
|
153
|
-
|
|
169
|
+
parse_csv_io(io, sep: sep, quote: quote, strip: strip, layout: layout,
|
|
154
170
|
read_text_as_const_string: read_text_as_const_string, &block)
|
|
155
171
|
end
|
|
156
172
|
end
|
|
@@ -158,7 +174,8 @@ class CAFrame
|
|
|
158
174
|
|
|
159
175
|
frame = build_frame(names, rows)
|
|
160
176
|
mask_missing_tokens(frame, missing) if missing
|
|
161
|
-
|
|
177
|
+
types = keep_blank_valued_columns_text(frame, types) unless types.nil?
|
|
178
|
+
frame.__send__(:cast_in_place, types, on_error: on_error, option_name: :types) unless types.nil?
|
|
162
179
|
convert_const_string_columns_to_castring(frame)
|
|
163
180
|
column_attrs&.each_with_index do |attrs, j|
|
|
164
181
|
frame[frame.column_names[j]].set_attrs(attrs) unless attrs.empty?
|
|
@@ -166,7 +183,82 @@ class CAFrame
|
|
|
166
183
|
frame
|
|
167
184
|
end
|
|
168
185
|
|
|
169
|
-
#
|
|
186
|
+
# Read CSV text already in hand into a frame -- the String is the CSV, not
|
|
187
|
+
# a path. The options and the reading block are +read_csv+'s; +encoding:+
|
|
188
|
+
# has nothing to open and is refused, the text being in its String's
|
|
189
|
+
# encoding. A BOM at the start of UTF-8 or binary text (as File.binread
|
|
190
|
+
# and many HTTP bodies give) is skipped, as +read_csv+ skips one at the
|
|
191
|
+
# start of a file.
|
|
192
|
+
#
|
|
193
|
+
# CAFrame.from_csv("date,temp\n2026-01-01,1.5\n")
|
|
194
|
+
# CAFrame.from_csv(response.body, types: :string)
|
|
195
|
+
# CAFrame.from_csv(df.to_csv) # the round trip
|
|
196
|
+
def self.from_csv(text, **opts, &block)
|
|
197
|
+
unless text.is_a?(String)
|
|
198
|
+
raise TypeError, "from_csv takes CSV text as a String (got #{text.class}); " \
|
|
199
|
+
"read a path or an IO with read_csv"
|
|
200
|
+
end
|
|
201
|
+
if opts.key?(:encoding)
|
|
202
|
+
raise ArgumentError, "from_csv: encoding: is the open mode of a path; " \
|
|
203
|
+
"text in hand is read in its String's encoding"
|
|
204
|
+
end
|
|
205
|
+
if text.encoding == Encoding::UTF_8
|
|
206
|
+
text = text.delete_prefix("\uFEFF")
|
|
207
|
+
elsif text.encoding == Encoding::BINARY && text.start_with?("\xEF\xBB\xBF".b)
|
|
208
|
+
text = text.byteslice(3..) # bytes read from a file or the network
|
|
209
|
+
end
|
|
210
|
+
read_csv(StringIO.new(text), **opts, &block)
|
|
211
|
+
end
|
|
212
|
+
|
|
213
|
+
# read_csv's types: as the cast map it stands for: inference unless said
|
|
214
|
+
# otherwise. :string reads every column as text (no cast); a Hash without
|
|
215
|
+
# default: infers the columns it does not name, default: :string keeps
|
|
216
|
+
# them text, and :string (or nil) for a column keeps that one text.
|
|
217
|
+
def self.csv_types(types)
|
|
218
|
+
return nil if types == :string
|
|
219
|
+
return types unless types.is_a?(Hash)
|
|
220
|
+
return { default: :infer }.merge(types) unless types.key?(:default)
|
|
221
|
+
return types
|
|
222
|
+
end
|
|
223
|
+
private_class_method :csv_types
|
|
224
|
+
|
|
225
|
+
# The reader masks the fields that are missing (an unquoted empty field, a
|
|
226
|
+
# cell a short row never reaches), so a blank cell it leaves present was
|
|
227
|
+
# written as a value: a quoted "", or spaces kept by strip: false. Such a
|
|
228
|
+
# cell is no number, truth value or time, so a column holding one is kept
|
|
229
|
+
# as text rather than inferred, where a blank cell would count as missing.
|
|
230
|
+
def self.keep_blank_valued_columns_text(frame, types)
|
|
231
|
+
map = types == :infer ? { default: :infer } : types
|
|
232
|
+
return types unless map.is_a?(Hash)
|
|
233
|
+
frame.column_names.each do |name|
|
|
234
|
+
given = [name, name.to_sym].find { |k| map.key?(k) }
|
|
235
|
+
next unless (given ? map[given] : map[:default]) == :infer
|
|
236
|
+
map = map.merge(name => :string) if blank_value_in?(frame[name])
|
|
237
|
+
end
|
|
238
|
+
return map
|
|
239
|
+
end
|
|
240
|
+
|
|
241
|
+
BLANK_LEADS = [" ", "\t", "\n", "\v", "\f", "\r", "\0"].freeze
|
|
242
|
+
Ractor.make_shareable(BLANK_LEADS)
|
|
243
|
+
private_constant :BLANK_LEADS
|
|
244
|
+
|
|
245
|
+
# Whether a text column holds a present cell of nothing but whitespace. A
|
|
246
|
+
# CAConstString is narrowed in C to the cells that are empty or start with
|
|
247
|
+
# whitespace, and only those are read as Strings.
|
|
248
|
+
def self.blank_value_in?(col)
|
|
249
|
+
if col.is_a?(CAConstString)
|
|
250
|
+
lead = col.byte_length.eq(0)
|
|
251
|
+
BLANK_LEADS.each { |c| lead = lead.or(col.start_with?(c)) }
|
|
252
|
+
addrs = lead.strip_mask(false).where
|
|
253
|
+
return addrs.to_a.any? { |i| col[i].strip.empty? }
|
|
254
|
+
end
|
|
255
|
+
return false unless col.is_a?(CAString) || (col.data_type == CA_OBJECT && !col.face?)
|
|
256
|
+
col.each { |v| return true if v.is_a?(String) && v.strip.empty? }
|
|
257
|
+
return false
|
|
258
|
+
end
|
|
259
|
+
private_class_method :keep_blank_valued_columns_text, :blank_value_in?
|
|
260
|
+
|
|
261
|
+
# A text column read_csv reads is a CAString, whichever way the body was
|
|
170
262
|
# read: the string operations (strip!, gsub!, ...) write through to it.
|
|
171
263
|
def self.convert_const_string_columns_to_castring(frame)
|
|
172
264
|
frame.column_names.each do |name|
|
|
@@ -177,7 +269,7 @@ class CAFrame
|
|
|
177
269
|
end
|
|
178
270
|
private_class_method :convert_const_string_columns_to_castring
|
|
179
271
|
|
|
180
|
-
# Split a +missing:+ argument, as
|
|
272
|
+
# Split a +missing:+ argument, as read_csv and to_csv both take it, into the
|
|
181
273
|
# setting for every column and a Hash of per-column settings that replace
|
|
182
274
|
# it. A Hash says :default for every column and a column name for one;
|
|
183
275
|
# column names are Strings, so the Symbol cannot collide with one. A
|
|
@@ -200,7 +292,7 @@ class CAFrame
|
|
|
200
292
|
# The read side of missing:, checked before the file is read: each setting
|
|
201
293
|
# becomes an Array of token Strings.
|
|
202
294
|
def self.missing_tokens(missing)
|
|
203
|
-
default, per_column = split_missing(missing, "
|
|
295
|
+
default, per_column = split_missing(missing, "read_csv")
|
|
204
296
|
tokens = lambda do |value|
|
|
205
297
|
list = Array(value)
|
|
206
298
|
bad = list.reject { |t| t.is_a?(String) }
|
|
@@ -252,7 +344,7 @@ class CAFrame
|
|
|
252
344
|
# Drive the reading-control DSL over one open IO and hand back
|
|
253
345
|
# [names, rows, column_attrs]. Shared by the path and the IO source, so the
|
|
254
346
|
# two cannot come to read a file differently.
|
|
255
|
-
def self.
|
|
347
|
+
def self.parse_csv_io (io, sep:, quote:, strip:, layout:, read_text_as_const_string: false, &block)
|
|
256
348
|
reader = CSVReader.new(io, sep: sep, quote: quote, strip: strip,
|
|
257
349
|
read_text_as_const_string: read_text_as_const_string)
|
|
258
350
|
reader.reporting do
|
|
@@ -262,7 +354,7 @@ class CAFrame
|
|
|
262
354
|
[names, rows, reader.column_attrs(reader.file_ncol)]
|
|
263
355
|
end
|
|
264
356
|
|
|
265
|
-
private_class_method :
|
|
357
|
+
private_class_method :parse_csv_io
|
|
266
358
|
|
|
267
359
|
# Re-raise a failure to decode the input with a line saying how to name the
|
|
268
360
|
# file's encoding. The error and its class are kept; only the message grows.
|
|
@@ -293,47 +385,75 @@ class CAFrame
|
|
|
293
385
|
# no flat CSV cell and raises; export it per column, or use +to_records+ +
|
|
294
386
|
# JSON for the structured shape (memo §11.9, the N-D escape).
|
|
295
387
|
#
|
|
296
|
-
#
|
|
297
|
-
#
|
|
298
|
-
#
|
|
299
|
-
# +
|
|
388
|
+
# +target+ is a path, or an open IO -- anything answering +write+, which is
|
|
389
|
+
# written to where it is and left open. Returns self; +to_csv+ gives the
|
|
390
|
+
# same text as a String. The index (if any) is written as the first column
|
|
391
|
+
# under +axis_name+ unless +index: false+. A masked cell (UNDEF) becomes an
|
|
392
|
+
# empty field, which +read_csv+ reads back as UNDEF (parse-mask, §6-2) -- so mask round-trips. A
|
|
300
393
|
# genuine empty string is written quoted (+""+) to stay distinct from missing,
|
|
301
394
|
# matching the tokenizer's own unquoted-empty vs quoted-empty distinction.
|
|
302
395
|
#
|
|
303
|
-
# df.
|
|
304
|
-
#
|
|
396
|
+
# df.write_csv("out.csv")
|
|
397
|
+
# df.write_csv($stdout)
|
|
305
398
|
#
|
|
306
|
-
# Options: +sep+ / +quote+ mirror +
|
|
399
|
+
# Options: +sep+ / +quote+ mirror +read_csv+; +header+ writes the name row
|
|
307
400
|
# (default true); +index+ writes the index column (default true);
|
|
308
|
-
# +encoding+ transcodes the text before it is written
|
|
401
|
+
# +encoding+ transcodes the text before it is written (default
|
|
309
402
|
# nil leaves it as built, UTF-8). A character the encoding cannot hold
|
|
310
403
|
# raises Encoding::UndefinedConversionError rather than being dropped. A
|
|
311
404
|
# String cell in another encoding is transcoded to UTF-8; one with bytes
|
|
312
405
|
# that are not valid in it, or with no encoding (ASCII-8BIT), raises. Each
|
|
313
406
|
# names the row and the column of the cell.
|
|
314
407
|
#
|
|
315
|
-
# df.
|
|
408
|
+
# df.write_csv("out.csv", encoding: "CP932") # for Excel in Japanese
|
|
316
409
|
#
|
|
317
410
|
# +missing:+ writes a masked cell as that String instead of an empty field,
|
|
318
|
-
# for a reader that expects a sentinel; +
|
|
411
|
+
# for a reader that expects a sentinel; +read_csv(missing:)+ with the same
|
|
319
412
|
# String reads it back as UNDEF. A Hash sets it per column the way
|
|
320
|
-
# +
|
|
413
|
+
# +read_csv+ does: +:default+ for every column, and a column name (or the
|
|
321
414
|
# index's axis name) for one, replacing the default; "" is the empty field.
|
|
322
415
|
# A cell whose value would be written as the same text raises, since the
|
|
323
416
|
# file could not tell the two apart.
|
|
324
417
|
#
|
|
325
|
-
# df.
|
|
326
|
-
# df.
|
|
418
|
+
# df.write_csv("out.csv", missing: "-999")
|
|
419
|
+
# df.write_csv("out.csv", missing: { default: "-999", "comment" => "" })
|
|
327
420
|
#
|
|
328
421
|
# +attrs:+ writes, below the name row, one line per attribute key given,
|
|
329
422
|
# in that order: field j is +attr(key)+ of column j (the index's for the
|
|
330
423
|
# index column). A column without the attribute gets an empty field, an
|
|
331
424
|
# empty String a quoted "", and a number its +to_s+.
|
|
332
|
-
# +
|
|
425
|
+
# +read_csv(attrs:)+ with the same keys reads them back, as Strings.
|
|
426
|
+
#
|
|
427
|
+
# df.write_csv("out.csv", attrs: [:units, :long_name])
|
|
428
|
+
#
|
|
429
|
+
# +precision:+ writes each float cell rounded to that many decimal places,
|
|
430
|
+
# still as a float (precision: 0 writes 3.0), and each part of a complex
|
|
431
|
+
# cell likewise; NaN and infinities are written as they are, and a value
|
|
432
|
+
# that rounds to zero keeps its sign (-0.4 at 0 places is -0.0). Without
|
|
433
|
+
# it a float is written in full, so that read_csv reads back the same
|
|
434
|
+
# value.
|
|
333
435
|
#
|
|
334
|
-
# df.
|
|
335
|
-
def
|
|
336
|
-
|
|
436
|
+
# df.write_csv("out.csv", precision: 2)
|
|
437
|
+
def write_csv(target, **opts)
|
|
438
|
+
out = to_csv(**opts)
|
|
439
|
+
if target.respond_to?(:write)
|
|
440
|
+
target.write(out)
|
|
441
|
+
else
|
|
442
|
+
File.binwrite(target, out)
|
|
443
|
+
end
|
|
444
|
+
self
|
|
445
|
+
end
|
|
446
|
+
|
|
447
|
+
# The frame as CSV text: what +write_csv+ writes, as a String, with the same
|
|
448
|
+
# options. +from_csv+ reads it back.
|
|
449
|
+
#
|
|
450
|
+
# csv = df.to_csv
|
|
451
|
+
# csv = df.to_csv(sep: "\t", index: false)
|
|
452
|
+
def to_csv(sep: ",", quote: '"', header: true, index: true,
|
|
453
|
+
encoding: nil, missing: nil, attrs: [], precision: nil)
|
|
454
|
+
unless precision.nil? || (precision.is_a?(Integer) && precision >= 0)
|
|
455
|
+
raise ArgumentError, "to_csv: precision: is a number of decimal places, 0 or more (got #{precision.inspect})"
|
|
456
|
+
end
|
|
337
457
|
default, per_column = CAFrame.send(:split_missing, missing, "to_csv")
|
|
338
458
|
[default, *per_column.values].each do |text|
|
|
339
459
|
next if text.nil? || text.is_a?(String)
|
|
@@ -367,14 +487,21 @@ class CAFrame
|
|
|
367
487
|
written = []
|
|
368
488
|
if index && @index
|
|
369
489
|
names << @axis_name
|
|
370
|
-
formatted << format_csv_column(@index)
|
|
490
|
+
formatted << format_csv_column(@index, precision)
|
|
371
491
|
written << @index
|
|
372
492
|
end
|
|
373
493
|
@columns.each do |name, col|
|
|
374
494
|
names << name
|
|
375
|
-
formatted << format_csv_column(col)
|
|
495
|
+
formatted << format_csv_column(col, precision)
|
|
376
496
|
written << col
|
|
377
497
|
end
|
|
498
|
+
# A line with no fields reads back as a blank line, which a reader passes
|
|
499
|
+
# over, so the rows would be lost.
|
|
500
|
+
if names.empty? && @nrow > 0
|
|
501
|
+
raise ArgumentError,
|
|
502
|
+
"to_csv: the frame has #{@nrow} rows but no field to write " \
|
|
503
|
+
"(no columns#{@index ? ', and index: false' : ''})"
|
|
504
|
+
end
|
|
378
505
|
attr_rows = attr_keys.map do |key|
|
|
379
506
|
written.each_with_index.map { |col, j| csv_attr_text(col, key, names[j]) }
|
|
380
507
|
end
|
|
@@ -421,12 +548,7 @@ class CAFrame
|
|
|
421
548
|
end
|
|
422
549
|
end
|
|
423
550
|
|
|
424
|
-
|
|
425
|
-
File.binwrite(path, out)
|
|
426
|
-
self
|
|
427
|
-
else
|
|
428
|
-
out
|
|
429
|
-
end
|
|
551
|
+
out
|
|
430
552
|
end
|
|
431
553
|
|
|
432
554
|
# Render the frame as an aligned text table for reading:
|
|
@@ -614,15 +736,15 @@ class CAFrame
|
|
|
614
736
|
end
|
|
615
737
|
end
|
|
616
738
|
|
|
617
|
-
private def format_csv_column(col)
|
|
739
|
+
private def format_csv_column(col, precision = nil)
|
|
618
740
|
# A float32 value goes out as its own shortest decimal ("0.1"), which
|
|
619
|
-
#
|
|
741
|
+
# read_csv reads back as the same value, not as the double it widens
|
|
620
742
|
# to ("0.10000000149011612"); cmplx64 has that in each part.
|
|
621
743
|
if !col.face? && (col.data_type == CA_FLOAT32 || col.data_type == CA_CMPLX64)
|
|
622
744
|
col = col.__widen_keeping_shortest_decimal__
|
|
623
745
|
end
|
|
624
746
|
# A boolean goes out as 0 / 1, as it does in every other serialization;
|
|
625
|
-
#
|
|
747
|
+
# read_csv reads it back with types: { name => :boolean }.
|
|
626
748
|
col = col.to_type(:uint8) if !col.face? && col.data_type == CA_BOOLEAN
|
|
627
749
|
col.to_a.map do |e|
|
|
628
750
|
if UNDEF.equal?(e) || e.nil?
|
|
@@ -631,12 +753,28 @@ class CAFrame
|
|
|
631
753
|
e
|
|
632
754
|
elsif e.respond_to?(:iso8601)
|
|
633
755
|
e.iso8601
|
|
756
|
+
elsif precision && e.is_a?(Float)
|
|
757
|
+
# Still a float, so precision: 0 writes 3.0 and the column reads
|
|
758
|
+
# back as floats.
|
|
759
|
+
csv_rounded(e, precision).to_s
|
|
760
|
+
elsif precision && e.is_a?(Complex) && e.real.is_a?(Float)
|
|
761
|
+
Complex(*[e.real, e.imaginary].map { |x| csv_rounded(x, precision) }).to_s
|
|
634
762
|
else
|
|
635
763
|
e.to_s
|
|
636
764
|
end
|
|
637
765
|
end
|
|
638
766
|
end
|
|
639
767
|
|
|
768
|
+
# A float rounded to +precision+ places, as a Float. A zero keeps the sign
|
|
769
|
+
# of the value it came from at every precision: Float#round gives an
|
|
770
|
+
# Integer at 0 places, which has no -0, where it gives -0.0 at 1 or more.
|
|
771
|
+
private def csv_rounded(x, precision)
|
|
772
|
+
return x unless x.finite?
|
|
773
|
+
r = x.round(precision).to_f
|
|
774
|
+
r = -0.0 if r == 0 && (x < 0 || (x == 0 && (1.0 / x) < 0))
|
|
775
|
+
r
|
|
776
|
+
end
|
|
777
|
+
|
|
640
778
|
private def csv_text(names, attr_rows, formatted, header, sep, quote)
|
|
641
779
|
out = +""
|
|
642
780
|
if header
|
|
@@ -747,7 +885,7 @@ class CAFrame
|
|
|
747
885
|
names ||= Array.new(ncol) { |j| "c#{j}" }
|
|
748
886
|
# A frame holds one column per name, so a repeated one would keep only
|
|
749
887
|
# the last of its columns.
|
|
750
|
-
CAFrame.__send__(:refuse_repeated_names, names, "
|
|
888
|
+
CAFrame.__send__(:refuse_repeated_names, names, "read_csv", "the header row names",
|
|
751
889
|
hint: "pass header: false for the default column names " \
|
|
752
890
|
"(c0, c1, ...), or name the columns with column_names:")
|
|
753
891
|
|