carray 3.0.4 → 3.0.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +438 -0
- data/README.md +1 -1
- data/carray.gemspec +3 -2
- data/ext/ca_array_pool.c +25 -1
- data/ext/ca_axis_dispatch.c +19 -20
- data/ext/ca_axis_group.c +1 -1
- data/ext/ca_categorical_iterator.c +34 -27
- data/ext/ca_compare.c +4 -4
- data/ext/ca_composite_dispatch.c +9 -9
- data/ext/ca_kernel_iterator.c +122 -237
- data/ext/ca_kernel_iterator.h +522 -337
- data/ext/ca_obj_array.c +69 -19
- data/ext/ca_obj_bincmp.c +10 -1
- data/ext/ca_obj_binop.c +11 -1
- data/ext/ca_obj_bitarray.c +5 -1
- data/ext/ca_obj_bitfield.c +60 -43
- data/ext/ca_obj_byte_swap.c +4 -1
- data/ext/ca_obj_const_string.c +15 -3
- data/ext/ca_obj_face.c +44 -1
- data/ext/ca_obj_fake.c +3 -1
- data/ext/ca_obj_field.c +13 -3
- data/ext/ca_obj_grid.c +219 -17
- data/ext/ca_obj_meld.c +69 -113
- data/ext/ca_obj_moncmp.c +9 -1
- data/ext/ca_obj_monop.c +86 -12
- data/ext/ca_obj_object.c +118 -32
- data/ext/ca_obj_record.c +3 -0
- data/ext/ca_obj_reduce.c +19 -1
- data/ext/ca_obj_refer.c +80 -26
- data/ext/ca_obj_remap.c +69 -10
- data/ext/ca_obj_roll.c +27 -6
- data/ext/ca_obj_select.c +38 -6
- data/ext/ca_obj_select_axis.c +1 -0
- data/ext/ca_obj_shift.c +32 -7
- data/ext/ca_obj_stack.c +311 -111
- data/ext/ca_obj_stride.c +159 -64
- data/ext/ca_obj_string.c +27 -2
- data/ext/ca_obj_tile.c +28 -7
- data/ext/ca_obj_transpose.c +8 -4
- data/ext/ca_obj_triop.c +12 -1
- data/ext/ca_obj_window.c +80 -50
- data/ext/ca_op_ipower.c +12 -12
- data/ext/ca_sort_kernels.h +6 -3
- data/ext/caframe_csv_reader.c +1 -1
- data/ext/carray.h +35 -1
- data/ext/carray_access.c +43 -67
- data/ext/carray_address_basis.c +134 -0
- data/ext/carray_attribute.c +32 -0
- data/ext/carray_broadcast.c +3 -4
- data/ext/carray_build_flags.h +1 -1
- data/ext/carray_cast.c +251 -93
- data/ext/carray_cast_func.rb +56 -26
- data/ext/carray_conversion.c +11 -7
- data/ext/carray_core.c +814 -72
- data/ext/carray_count.c +6 -0
- data/ext/carray_element.c +22 -13
- data/ext/carray_factorize.c +20 -20
- data/ext/carray_generate.c +8 -2
- data/ext/carray_histogram.c +26 -4
- data/ext/carray_index_classifier.c +4 -4
- data/ext/carray_internal.h +159 -12
- data/ext/carray_kernels_bincmp.c +2174 -170
- data/ext/carray_kernels_binop.c +25653 -5487
- data/ext/carray_kernels_init.c +8 -8
- data/ext/carray_kernels_moncmp.c +96 -96
- data/ext/carray_kernels_monop.c +12253 -5235
- data/ext/carray_kernels_reduce_aggregate.c +6150 -2423
- data/ext/carray_kernels_reduce_boolean.c +1533 -15
- data/ext/carray_kernels_reduce_cumulative.c +2442 -882
- data/ext/carray_kernels_reduce_extreme.c +5658 -1661
- data/ext/carray_kernels_reduce_variance.c +2534 -402
- data/ext/carray_kernels_scan.c +368 -368
- data/ext/carray_kernels_search.c +4551 -604
- data/ext/carray_kernels_sort.c +659 -751
- data/ext/carray_kernels_triop.c +114 -114
- data/ext/carray_key_windows.c +4 -0
- data/ext/carray_lazy.c +47 -0
- data/ext/carray_loop.c +1 -1
- data/ext/carray_mask.c +175 -161
- data/ext/carray_median_percentile.c +4 -4
- data/ext/carray_memory_view.c +15 -9
- data/ext/carray_operator.c +152 -49
- data/ext/carray_order.c +230 -10
- data/ext/carray_random.c +22 -19
- data/ext/carray_scatter.c +24 -7
- data/ext/carray_slab.c +128 -29
- data/ext/carray_sort.c +224 -16
- data/ext/carray_sort_kernel.c +5 -7
- data/ext/carray_struct.c +35 -43
- data/ext/carray_test.c +74 -19
- data/ext/carray_time_text.c +6 -4
- data/ext/carray_utils.c +63 -21
- data/ext/extconf.rb +11 -0
- data/ext/mkkernel.rb +546 -199
- data/ext/ruby_carray.c +4 -1
- data/ext/version.h +4 -4
- data/lib/carray/attribute.rb +42 -5
- data/lib/carray/autoload_carray.rb +3 -3
- data/lib/carray/autoload_method_extension.rb +5 -3
- data/lib/carray/axis_group.rb +13 -7
- data/lib/carray/basics.rb +28 -8
- data/lib/carray/block_iterator.rb +17 -3
- data/lib/carray/categorical.rb +35 -23
- data/lib/carray/categorical_iterator.rb +8 -8
- data/lib/carray/complex.rb +2 -2
- data/lib/carray/conditional.rb +3 -2
- data/lib/carray/const_string.rb +10 -0
- data/lib/carray/data_type_extension.rb +8 -1
- data/lib/carray/fixlen_string.rb +41 -2
- data/lib/carray/frame/column_verbs.rb +48 -16
- data/lib/carray/frame/concat.rb +17 -3
- data/lib/carray/frame/convert.rb +19 -0
- data/lib/carray/frame/csv_parser.rb +32 -32
- data/lib/carray/frame/describe.rb +4 -2
- data/lib/carray/frame/frame.rb +405 -85
- data/lib/carray/frame/group.rb +777 -64
- data/lib/carray/frame/io.rb +212 -72
- data/lib/carray/frame/join.rb +146 -37
- data/lib/carray/frame/records.rb +3 -3
- data/lib/carray/frame/reshape.rb +89 -11
- data/lib/carray/frame/sort.rb +57 -9
- data/lib/carray/frame/verbs.rb +384 -49
- data/lib/carray/frame/windows.rb +44 -6
- data/lib/carray/fuse_source.rb +31 -2
- data/lib/carray/inspect.rb +7 -2
- data/lib/carray/iterator.rb +32 -0
- data/lib/carray/lazy.rb +52 -8
- data/lib/carray/mask_gap_fill.rb +37 -17
- data/lib/carray/meld_reduce.rb +1 -1
- data/lib/carray/methods/choose.rb +3 -2
- data/lib/carray/methods/is_in.rb +8 -1
- data/lib/carray/methods/meshgrid.rb +3 -1
- data/lib/carray/runtime.rb +47 -8
- data/lib/carray/segment_iterator.rb +35 -16
- data/lib/carray/serialize.rb +53 -19
- data/lib/carray/stack.rb +33 -18
- data/lib/carray/string.rb +8 -0
- data/lib/carray/string_operation_extension.rb +13 -5
- data/lib/carray/struct_builder.rb +5 -3
- data/lib/carray/time.rb +112 -68
- data/lib/carray/window_iterator.rb +16 -4
- data/yard-stubs/ca_obj_array.rb +26 -1
- data/yard-stubs/ca_obj_transpose.rb +5 -4
- data/yard-stubs/carray_attribute.rb +3 -1
- data/yard-stubs/carray_cast.rb +10 -5
- data/yard-stubs/carray_conversion.rb +13 -3
- data/yard-stubs/carray_element.rb +7 -6
- data/yard-stubs/carray_mask.rb +28 -4
- data/yard-stubs/carray_math.rb +45 -20
- data/yard-stubs/carray_memory_view.rb +7 -0
- data/yard-stubs/carray_order.rb +100 -4
- data/yard-stubs/carray_scatter.rb +7 -5
- data/yard-stubs/carray_test.rb +7 -4
- metadata +18 -4
data/lib/carray/frame/io.rb
CHANGED
|
@@ -9,26 +9,32 @@ class CAFrame
|
|
|
9
9
|
# over one object array (§4.2). Casting is a separate step: +types:+ takes
|
|
10
10
|
# anything +cast+ takes as a map -- { "temp" => :float64 }, :infer, or a
|
|
11
11
|
# map with +default:+ -- and casts the frame with it on load, which is the
|
|
12
|
-
# same as calling +cast+ afterwards.
|
|
13
|
-
#
|
|
12
|
+
# same as calling +cast+ afterwards. The default is :infer, which gives
|
|
13
|
+
# each column the type all of its cells read as (see +infer_types+);
|
|
14
|
+
# +types: :string+ leaves every column text. A Hash names the types of some
|
|
15
|
+
# columns and infers the rest, unless its +default:+ says otherwise
|
|
16
|
+
# (+default: :string+ keeps the rest text); +nil+ for a column keeps it
|
|
17
|
+
# text. Broken cells become UNDEF automatically (parse-mask, §6-2).
|
|
14
18
|
#
|
|
15
|
-
# CAFrame.
|
|
16
|
-
# CAFrame.
|
|
17
|
-
# CAFrame.
|
|
18
|
-
# CAFrame.
|
|
19
|
+
# CAFrame.read_csv("obs.csv") # types inferred
|
|
20
|
+
# CAFrame.read_csv("obs.csv", types: :string) # every column text
|
|
21
|
+
# CAFrame.read_csv("obs.csv", types: { "code" => :int32, "id" => nil })
|
|
22
|
+
# CAFrame.read_csv("obs.csv", types: { default: :string, "temp" => :float64 })
|
|
23
|
+
# CAFrame.read_csv("obs.csv", types: { default: :infer, "code" => :int32, "id" => nil })
|
|
24
|
+
# CAFrame.read_csv("obs.csv", types: { default: :float64, "station" => nil })
|
|
25
|
+
# CAFrame.read_csv("obs.csv", types: { "station" => :categorical })
|
|
19
26
|
#
|
|
20
|
-
# +source+ is a path, or an open IO -- anything answering +gets
|
|
21
|
-
# StringIO is. So CSV that is already in memory does not have to go to a
|
|
22
|
-
# temporary file first:
|
|
27
|
+
# +source+ is a path, or an open IO -- anything answering +gets+:
|
|
23
28
|
#
|
|
24
|
-
# CAFrame.
|
|
25
|
-
#
|
|
26
|
-
#
|
|
29
|
+
# CAFrame.read_csv("obs.csv") # a path
|
|
30
|
+
# File.open("obs.csv") { |io| CAFrame.read_csv(io) }
|
|
31
|
+
# CAFrame.read_csv($stdin)
|
|
27
32
|
#
|
|
28
33
|
# A String is always read as a path, never as CSV text: guessing between the
|
|
29
34
|
# two by looking for a newline is the kind of guess that is right until it is
|
|
30
|
-
# not
|
|
31
|
-
# from where it is and left open -- the caller opened
|
|
35
|
+
# not. CSV text already in hand goes to +from_csv+, which takes the same
|
|
36
|
+
# options. An IO is read from where it is and left open -- the caller opened
|
|
37
|
+
# it and closes it.
|
|
32
38
|
#
|
|
33
39
|
# Parsing uses the built-in fast tokenizer (CSVParser). Options:
|
|
34
40
|
# sep: field separator (default ",")
|
|
@@ -66,34 +72,39 @@ class CAFrame
|
|
|
66
72
|
# attrs: attribute keys for the lines right below the header, one
|
|
67
73
|
# key per line: field j of a line becomes that attribute of
|
|
68
74
|
# column j (an empty field gives none). The data then starts
|
|
69
|
-
# below them. What to_csv(attrs:) writes, it reads back
|
|
75
|
+
# below them. What to_csv(attrs:) writes, it reads back. The
|
|
76
|
+
# fields are read as the data fields are, so strip: trims them
|
|
77
|
+
# too
|
|
70
78
|
#
|
|
71
|
-
# CAFrame.
|
|
72
|
-
# CAFrame.
|
|
73
|
-
# CAFrame.
|
|
74
|
-
# CAFrame.
|
|
75
|
-
# CAFrame.
|
|
76
|
-
# CAFrame.
|
|
77
|
-
# CAFrame.
|
|
79
|
+
# CAFrame.read_csv("obs.csv", header: 2) # a title above
|
|
80
|
+
# CAFrame.read_csv("obs.csv", header: 0, data: 3) # units on lines 1-2
|
|
81
|
+
# CAFrame.read_csv("big.csv", data: 1..100) # the first 100 lines
|
|
82
|
+
# CAFrame.read_csv("raw.csv", column_names: %w[date temp rh])
|
|
83
|
+
# CAFrame.read_csv("wide.csv", columns: %w[date temp])
|
|
84
|
+
# CAFrame.read_csv("raw.csv", header: false, columns: { 0 => "date", 3 => "temp" })
|
|
85
|
+
# CAFrame.read_csv("obs.csv", attrs: [:units]) # units on line 1
|
|
78
86
|
#
|
|
79
87
|
# A block, given the reader, reads in any other order with +skip+ /
|
|
80
88
|
# +header+ / +header(name)+ / +column_names+ / +columns+ / +data+ (see
|
|
81
89
|
# CSVReader):
|
|
82
90
|
#
|
|
83
|
-
# CAFrame.
|
|
91
|
+
# CAFrame.read_csv("obs.csv") { |r| r.skip 2; r.header; r.skip 1; r.data }
|
|
84
92
|
#
|
|
85
93
|
# A missing field -- an unquoted empty one, or a cell a short row never
|
|
86
94
|
# reached -- is UNDEF in the frame, whether or not the column is cast by
|
|
87
95
|
# +types:+. A quoted empty field ("") is the empty string, which is a value.
|
|
88
|
-
# So the mask +to_csv+ writes comes back as a mask.
|
|
96
|
+
# So the mask +to_csv+ writes comes back as a mask. A blank value (a quoted
|
|
97
|
+
# "", or spaces kept by strip: false) is no number, truth value or time, so
|
|
98
|
+
# a column holding one is not inferred and stays text; name its type in
|
|
99
|
+
# +types:+ to read it as numbers with the blank cells UNDEF.
|
|
89
100
|
#
|
|
90
101
|
# Files that spell "missing" some other way say so with +missing:+. A field
|
|
91
102
|
# whose text is one of the given tokens is UNDEF too, before +types:+
|
|
92
103
|
# casts, so a -999 sentinel never reaches the column as a number:
|
|
93
104
|
#
|
|
94
|
-
# CAFrame.
|
|
95
|
-
# CAFrame.
|
|
96
|
-
# CAFrame.
|
|
105
|
+
# CAFrame.read_csv("obs.csv", missing: ["-999", "///"]) # every column
|
|
106
|
+
# CAFrame.read_csv("obs.csv", missing: { "temp" => "-999" }) # one column
|
|
107
|
+
# CAFrame.read_csv("obs.csv", missing: { default: "-999", "id" => [] })
|
|
97
108
|
#
|
|
98
109
|
# Tokens are Strings and are compared with the field's text as read (after
|
|
99
110
|
# +strip:+, quoted or not), so "-999" does not match "-999.0". In a Hash,
|
|
@@ -102,35 +113,42 @@ class CAFrame
|
|
|
102
113
|
# +:default+ the columns it does not name keep only the empty field. As in
|
|
103
114
|
# +to_csv+, "" means the empty field, so a quoted "" is still a value.
|
|
104
115
|
#
|
|
105
|
-
#
|
|
106
|
-
# over one backing object array (§3.6 view-by-default).
|
|
116
|
+
# With +types: :string+, the columns are handed to the frame as CABlock views
|
|
117
|
+
# over one backing object array (§3.6 view-by-default). Otherwise, the
|
|
107
118
|
# body is read without making a String per cell: a column cast to a number
|
|
108
119
|
# is read from the text directly, and a column left as text becomes its own
|
|
109
120
|
# CAString. Casting a column materializes it, and +copy+ gives an
|
|
110
121
|
# independent frame.
|
|
111
|
-
def self.
|
|
122
|
+
def self.read_csv(source, types: :infer, on_error: :mask, missing: nil,
|
|
112
123
|
header: nil, data: nil, column_names: nil, columns: nil,
|
|
113
124
|
attrs: nil, sep: ",", quote: '"', strip: false,
|
|
114
125
|
encoding: "bom|utf-8", parser: nil, &block)
|
|
115
126
|
layout = { header: header, data: data, column_names: column_names,
|
|
116
127
|
columns: columns, attrs: attrs }.compact
|
|
117
128
|
if block && !layout.empty?
|
|
118
|
-
raise ArgumentError, "
|
|
129
|
+
raise ArgumentError, "read_csv: give #{layout.keys.map { |k| "#{k}:" }.join(', ')} " \
|
|
119
130
|
"or a reading block, not both"
|
|
120
131
|
end
|
|
121
132
|
# The parser reads the header, the data and the columns itself.
|
|
122
133
|
if parser && !layout.empty?
|
|
123
|
-
raise ArgumentError, "
|
|
134
|
+
raise ArgumentError, "read_csv: give #{layout.keys.map { |k| "#{k}:" }.join(', ')} " \
|
|
124
135
|
"or parser:, not both"
|
|
125
136
|
end
|
|
126
137
|
if block && block.arity != 1
|
|
127
|
-
raise ArgumentError, "
|
|
128
|
-
"parameter, as in
|
|
138
|
+
raise ArgumentError, "read_csv: the reading block takes the reader as its " \
|
|
139
|
+
"parameter, as in read_csv(path) { |r| r.skip 2; r.header; r.data }"
|
|
140
|
+
end
|
|
141
|
+
# A String is a path, always. One with a line break in it that names no
|
|
142
|
+
# file is likely CSV text; the missing file says how to give text.
|
|
143
|
+
if source.is_a?(String) && source.include?("\n") && !File.exist?(source)
|
|
144
|
+
raise Errno::ENOENT, "#{source.inspect}; a String is a path, so to read CSV text " \
|
|
145
|
+
"give it to from_csv(text)"
|
|
129
146
|
end
|
|
130
147
|
missing = missing_tokens(missing) if missing
|
|
148
|
+
types = csv_types(types)
|
|
131
149
|
# With types:, the body is read into CAConstString columns, so a column
|
|
132
150
|
# cast to a number never becomes Strings; the text columns left over
|
|
133
|
-
# become CAString below.
|
|
151
|
+
# become CAString below. With types: :string every column stays text, so the
|
|
134
152
|
# body goes straight to Strings. missing: masks text in place, which
|
|
135
153
|
# wants the Strings first, so it takes that path too.
|
|
136
154
|
read_text_as_const_string = !types.nil? && missing.nil?
|
|
@@ -140,7 +158,7 @@ class CAFrame
|
|
|
140
158
|
elsif source.respond_to?(:gets)
|
|
141
159
|
with_encoding_hint("open the IO in the file's encoding, as in " \
|
|
142
160
|
"File.open(path, \"r:<file encoding>:UTF-8\")") do
|
|
143
|
-
|
|
161
|
+
parse_csv_io(source, sep: sep, quote: quote, strip: strip, layout: layout,
|
|
144
162
|
read_text_as_const_string: read_text_as_const_string, &block)
|
|
145
163
|
end
|
|
146
164
|
else
|
|
@@ -148,7 +166,7 @@ class CAFrame
|
|
|
148
166
|
"file's encoding, as in " \
|
|
149
167
|
"encoding: \"<file encoding>:UTF-8\"") do
|
|
150
168
|
File.open(source, "r:#{encoding}") do |io|
|
|
151
|
-
|
|
169
|
+
parse_csv_io(io, sep: sep, quote: quote, strip: strip, layout: layout,
|
|
152
170
|
read_text_as_const_string: read_text_as_const_string, &block)
|
|
153
171
|
end
|
|
154
172
|
end
|
|
@@ -156,7 +174,8 @@ class CAFrame
|
|
|
156
174
|
|
|
157
175
|
frame = build_frame(names, rows)
|
|
158
176
|
mask_missing_tokens(frame, missing) if missing
|
|
159
|
-
|
|
177
|
+
types = keep_blank_valued_columns_text(frame, types) unless types.nil?
|
|
178
|
+
frame.__send__(:cast_in_place, types, on_error: on_error, option_name: :types) unless types.nil?
|
|
160
179
|
convert_const_string_columns_to_castring(frame)
|
|
161
180
|
column_attrs&.each_with_index do |attrs, j|
|
|
162
181
|
frame[frame.column_names[j]].set_attrs(attrs) unless attrs.empty?
|
|
@@ -164,7 +183,82 @@ class CAFrame
|
|
|
164
183
|
frame
|
|
165
184
|
end
|
|
166
185
|
|
|
167
|
-
#
|
|
186
|
+
# Read CSV text already in hand into a frame -- the String is the CSV, not
|
|
187
|
+
# a path. The options and the reading block are +read_csv+'s; +encoding:+
|
|
188
|
+
# has nothing to open and is refused, the text being in its String's
|
|
189
|
+
# encoding. A BOM at the start of UTF-8 or binary text (as File.binread
|
|
190
|
+
# and many HTTP bodies give) is skipped, as +read_csv+ skips one at the
|
|
191
|
+
# start of a file.
|
|
192
|
+
#
|
|
193
|
+
# CAFrame.from_csv("date,temp\n2026-01-01,1.5\n")
|
|
194
|
+
# CAFrame.from_csv(response.body, types: :string)
|
|
195
|
+
# CAFrame.from_csv(df.to_csv) # the round trip
|
|
196
|
+
def self.from_csv(text, **opts, &block)
|
|
197
|
+
unless text.is_a?(String)
|
|
198
|
+
raise TypeError, "from_csv takes CSV text as a String (got #{text.class}); " \
|
|
199
|
+
"read a path or an IO with read_csv"
|
|
200
|
+
end
|
|
201
|
+
if opts.key?(:encoding)
|
|
202
|
+
raise ArgumentError, "from_csv: encoding: is the open mode of a path; " \
|
|
203
|
+
"text in hand is read in its String's encoding"
|
|
204
|
+
end
|
|
205
|
+
if text.encoding == Encoding::UTF_8
|
|
206
|
+
text = text.delete_prefix("\uFEFF")
|
|
207
|
+
elsif text.encoding == Encoding::BINARY && text.start_with?("\xEF\xBB\xBF".b)
|
|
208
|
+
text = text.byteslice(3..) # bytes read from a file or the network
|
|
209
|
+
end
|
|
210
|
+
read_csv(StringIO.new(text), **opts, &block)
|
|
211
|
+
end
|
|
212
|
+
|
|
213
|
+
# read_csv's types: as the cast map it stands for: inference unless said
|
|
214
|
+
# otherwise. :string reads every column as text (no cast); a Hash without
|
|
215
|
+
# default: infers the columns it does not name, default: :string keeps
|
|
216
|
+
# them text, and :string (or nil) for a column keeps that one text.
|
|
217
|
+
def self.csv_types(types)
|
|
218
|
+
return nil if types == :string
|
|
219
|
+
return types unless types.is_a?(Hash)
|
|
220
|
+
return { default: :infer }.merge(types) unless types.key?(:default)
|
|
221
|
+
return types
|
|
222
|
+
end
|
|
223
|
+
private_class_method :csv_types
|
|
224
|
+
|
|
225
|
+
# The reader masks the fields that are missing (an unquoted empty field, a
|
|
226
|
+
# cell a short row never reaches), so a blank cell it leaves present was
|
|
227
|
+
# written as a value: a quoted "", or spaces kept by strip: false. Such a
|
|
228
|
+
# cell is no number, truth value or time, so a column holding one is kept
|
|
229
|
+
# as text rather than inferred, where a blank cell would count as missing.
|
|
230
|
+
def self.keep_blank_valued_columns_text(frame, types)
|
|
231
|
+
map = types == :infer ? { default: :infer } : types
|
|
232
|
+
return types unless map.is_a?(Hash)
|
|
233
|
+
frame.column_names.each do |name|
|
|
234
|
+
given = [name, name.to_sym].find { |k| map.key?(k) }
|
|
235
|
+
next unless (given ? map[given] : map[:default]) == :infer
|
|
236
|
+
map = map.merge(name => :string) if blank_value_in?(frame[name])
|
|
237
|
+
end
|
|
238
|
+
return map
|
|
239
|
+
end
|
|
240
|
+
|
|
241
|
+
BLANK_LEADS = [" ", "\t", "\n", "\v", "\f", "\r", "\0"].freeze
|
|
242
|
+
Ractor.make_shareable(BLANK_LEADS)
|
|
243
|
+
private_constant :BLANK_LEADS
|
|
244
|
+
|
|
245
|
+
# Whether a text column holds a present cell of nothing but whitespace. A
|
|
246
|
+
# CAConstString is narrowed in C to the cells that are empty or start with
|
|
247
|
+
# whitespace, and only those are read as Strings.
|
|
248
|
+
def self.blank_value_in?(col)
|
|
249
|
+
if col.is_a?(CAConstString)
|
|
250
|
+
lead = col.byte_length.eq(0)
|
|
251
|
+
BLANK_LEADS.each { |c| lead = lead.or(col.start_with?(c)) }
|
|
252
|
+
addrs = lead.strip_mask(false).where
|
|
253
|
+
return addrs.to_a.any? { |i| col[i].strip.empty? }
|
|
254
|
+
end
|
|
255
|
+
return false unless col.is_a?(CAString) || (col.data_type == CA_OBJECT && !col.face?)
|
|
256
|
+
col.each { |v| return true if v.is_a?(String) && v.strip.empty? }
|
|
257
|
+
return false
|
|
258
|
+
end
|
|
259
|
+
private_class_method :keep_blank_valued_columns_text, :blank_value_in?
|
|
260
|
+
|
|
261
|
+
# A text column read_csv reads is a CAString, whichever way the body was
|
|
168
262
|
# read: the string operations (strip!, gsub!, ...) write through to it.
|
|
169
263
|
def self.convert_const_string_columns_to_castring(frame)
|
|
170
264
|
frame.column_names.each do |name|
|
|
@@ -175,7 +269,7 @@ class CAFrame
|
|
|
175
269
|
end
|
|
176
270
|
private_class_method :convert_const_string_columns_to_castring
|
|
177
271
|
|
|
178
|
-
# Split a +missing:+ argument, as
|
|
272
|
+
# Split a +missing:+ argument, as read_csv and to_csv both take it, into the
|
|
179
273
|
# setting for every column and a Hash of per-column settings that replace
|
|
180
274
|
# it. A Hash says :default for every column and a column name for one;
|
|
181
275
|
# column names are Strings, so the Symbol cannot collide with one. A
|
|
@@ -198,7 +292,7 @@ class CAFrame
|
|
|
198
292
|
# The read side of missing:, checked before the file is read: each setting
|
|
199
293
|
# becomes an Array of token Strings.
|
|
200
294
|
def self.missing_tokens(missing)
|
|
201
|
-
default, per_column = split_missing(missing, "
|
|
295
|
+
default, per_column = split_missing(missing, "read_csv")
|
|
202
296
|
tokens = lambda do |value|
|
|
203
297
|
list = Array(value)
|
|
204
298
|
bad = list.reject { |t| t.is_a?(String) }
|
|
@@ -250,7 +344,7 @@ class CAFrame
|
|
|
250
344
|
# Drive the reading-control DSL over one open IO and hand back
|
|
251
345
|
# [names, rows, column_attrs]. Shared by the path and the IO source, so the
|
|
252
346
|
# two cannot come to read a file differently.
|
|
253
|
-
def self.
|
|
347
|
+
def self.parse_csv_io (io, sep:, quote:, strip:, layout:, read_text_as_const_string: false, &block)
|
|
254
348
|
reader = CSVReader.new(io, sep: sep, quote: quote, strip: strip,
|
|
255
349
|
read_text_as_const_string: read_text_as_const_string)
|
|
256
350
|
reader.reporting do
|
|
@@ -260,7 +354,7 @@ class CAFrame
|
|
|
260
354
|
[names, rows, reader.column_attrs(reader.file_ncol)]
|
|
261
355
|
end
|
|
262
356
|
|
|
263
|
-
private_class_method :
|
|
357
|
+
private_class_method :parse_csv_io
|
|
264
358
|
|
|
265
359
|
# Re-raise a failure to decode the input with a line saying how to name the
|
|
266
360
|
# file's encoding. The error and its class are kept; only the message grows.
|
|
@@ -291,47 +385,75 @@ class CAFrame
|
|
|
291
385
|
# no flat CSV cell and raises; export it per column, or use +to_records+ +
|
|
292
386
|
# JSON for the structured shape (memo §11.9, the N-D escape).
|
|
293
387
|
#
|
|
294
|
-
#
|
|
295
|
-
#
|
|
296
|
-
#
|
|
297
|
-
# +
|
|
388
|
+
# +target+ is a path, or an open IO -- anything answering +write+, which is
|
|
389
|
+
# written to where it is and left open. Returns self; +to_csv+ gives the
|
|
390
|
+
# same text as a String. The index (if any) is written as the first column
|
|
391
|
+
# under +axis_name+ unless +index: false+. A masked cell (UNDEF) becomes an
|
|
392
|
+
# empty field, which +read_csv+ reads back as UNDEF (parse-mask, §6-2) -- so mask round-trips. A
|
|
298
393
|
# genuine empty string is written quoted (+""+) to stay distinct from missing,
|
|
299
394
|
# matching the tokenizer's own unquoted-empty vs quoted-empty distinction.
|
|
300
395
|
#
|
|
301
|
-
# df.
|
|
302
|
-
#
|
|
396
|
+
# df.write_csv("out.csv")
|
|
397
|
+
# df.write_csv($stdout)
|
|
303
398
|
#
|
|
304
|
-
# Options: +sep+ / +quote+ mirror +
|
|
399
|
+
# Options: +sep+ / +quote+ mirror +read_csv+; +header+ writes the name row
|
|
305
400
|
# (default true); +index+ writes the index column (default true);
|
|
306
|
-
# +encoding+ transcodes the text before it is written
|
|
401
|
+
# +encoding+ transcodes the text before it is written (default
|
|
307
402
|
# nil leaves it as built, UTF-8). A character the encoding cannot hold
|
|
308
403
|
# raises Encoding::UndefinedConversionError rather than being dropped. A
|
|
309
404
|
# String cell in another encoding is transcoded to UTF-8; one with bytes
|
|
310
405
|
# that are not valid in it, or with no encoding (ASCII-8BIT), raises. Each
|
|
311
406
|
# names the row and the column of the cell.
|
|
312
407
|
#
|
|
313
|
-
# df.
|
|
408
|
+
# df.write_csv("out.csv", encoding: "CP932") # for Excel in Japanese
|
|
314
409
|
#
|
|
315
410
|
# +missing:+ writes a masked cell as that String instead of an empty field,
|
|
316
|
-
# for a reader that expects a sentinel; +
|
|
411
|
+
# for a reader that expects a sentinel; +read_csv(missing:)+ with the same
|
|
317
412
|
# String reads it back as UNDEF. A Hash sets it per column the way
|
|
318
|
-
# +
|
|
413
|
+
# +read_csv+ does: +:default+ for every column, and a column name (or the
|
|
319
414
|
# index's axis name) for one, replacing the default; "" is the empty field.
|
|
320
415
|
# A cell whose value would be written as the same text raises, since the
|
|
321
416
|
# file could not tell the two apart.
|
|
322
417
|
#
|
|
323
|
-
# df.
|
|
324
|
-
# df.
|
|
418
|
+
# df.write_csv("out.csv", missing: "-999")
|
|
419
|
+
# df.write_csv("out.csv", missing: { default: "-999", "comment" => "" })
|
|
325
420
|
#
|
|
326
421
|
# +attrs:+ writes, below the name row, one line per attribute key given,
|
|
327
422
|
# in that order: field j is +attr(key)+ of column j (the index's for the
|
|
328
423
|
# index column). A column without the attribute gets an empty field, an
|
|
329
424
|
# empty String a quoted "", and a number its +to_s+.
|
|
330
|
-
# +
|
|
425
|
+
# +read_csv(attrs:)+ with the same keys reads them back, as Strings.
|
|
426
|
+
#
|
|
427
|
+
# df.write_csv("out.csv", attrs: [:units, :long_name])
|
|
428
|
+
#
|
|
429
|
+
# +precision:+ writes each float cell rounded to that many decimal places,
|
|
430
|
+
# still as a float (precision: 0 writes 3.0), and each part of a complex
|
|
431
|
+
# cell likewise; NaN and infinities are written as they are, and a value
|
|
432
|
+
# that rounds to zero keeps its sign (-0.4 at 0 places is -0.0). Without
|
|
433
|
+
# it a float is written in full, so that read_csv reads back the same
|
|
434
|
+
# value.
|
|
331
435
|
#
|
|
332
|
-
# df.
|
|
333
|
-
def
|
|
334
|
-
|
|
436
|
+
# df.write_csv("out.csv", precision: 2)
|
|
437
|
+
def write_csv(target, **opts)
|
|
438
|
+
out = to_csv(**opts)
|
|
439
|
+
if target.respond_to?(:write)
|
|
440
|
+
target.write(out)
|
|
441
|
+
else
|
|
442
|
+
File.binwrite(target, out)
|
|
443
|
+
end
|
|
444
|
+
self
|
|
445
|
+
end
|
|
446
|
+
|
|
447
|
+
# The frame as CSV text: what +write_csv+ writes, as a String, with the same
|
|
448
|
+
# options. +from_csv+ reads it back.
|
|
449
|
+
#
|
|
450
|
+
# csv = df.to_csv
|
|
451
|
+
# csv = df.to_csv(sep: "\t", index: false)
|
|
452
|
+
def to_csv(sep: ",", quote: '"', header: true, index: true,
|
|
453
|
+
encoding: nil, missing: nil, attrs: [], precision: nil)
|
|
454
|
+
unless precision.nil? || (precision.is_a?(Integer) && precision >= 0)
|
|
455
|
+
raise ArgumentError, "to_csv: precision: is a number of decimal places, 0 or more (got #{precision.inspect})"
|
|
456
|
+
end
|
|
335
457
|
default, per_column = CAFrame.send(:split_missing, missing, "to_csv")
|
|
336
458
|
[default, *per_column.values].each do |text|
|
|
337
459
|
next if text.nil? || text.is_a?(String)
|
|
@@ -365,14 +487,21 @@ class CAFrame
|
|
|
365
487
|
written = []
|
|
366
488
|
if index && @index
|
|
367
489
|
names << @axis_name
|
|
368
|
-
formatted << format_csv_column(@index)
|
|
490
|
+
formatted << format_csv_column(@index, precision)
|
|
369
491
|
written << @index
|
|
370
492
|
end
|
|
371
493
|
@columns.each do |name, col|
|
|
372
494
|
names << name
|
|
373
|
-
formatted << format_csv_column(col)
|
|
495
|
+
formatted << format_csv_column(col, precision)
|
|
374
496
|
written << col
|
|
375
497
|
end
|
|
498
|
+
# A line with no fields reads back as a blank line, which a reader passes
|
|
499
|
+
# over, so the rows would be lost.
|
|
500
|
+
if names.empty? && @nrow > 0
|
|
501
|
+
raise ArgumentError,
|
|
502
|
+
"to_csv: the frame has #{@nrow} rows but no field to write " \
|
|
503
|
+
"(no columns#{@index ? ', and index: false' : ''})"
|
|
504
|
+
end
|
|
376
505
|
attr_rows = attr_keys.map do |key|
|
|
377
506
|
written.each_with_index.map { |col, j| csv_attr_text(col, key, names[j]) }
|
|
378
507
|
end
|
|
@@ -419,12 +548,7 @@ class CAFrame
|
|
|
419
548
|
end
|
|
420
549
|
end
|
|
421
550
|
|
|
422
|
-
|
|
423
|
-
File.binwrite(path, out)
|
|
424
|
-
self
|
|
425
|
-
else
|
|
426
|
-
out
|
|
427
|
-
end
|
|
551
|
+
out
|
|
428
552
|
end
|
|
429
553
|
|
|
430
554
|
# Render the frame as an aligned text table for reading:
|
|
@@ -612,15 +736,15 @@ class CAFrame
|
|
|
612
736
|
end
|
|
613
737
|
end
|
|
614
738
|
|
|
615
|
-
private def format_csv_column(col)
|
|
739
|
+
private def format_csv_column(col, precision = nil)
|
|
616
740
|
# A float32 value goes out as its own shortest decimal ("0.1"), which
|
|
617
|
-
#
|
|
741
|
+
# read_csv reads back as the same value, not as the double it widens
|
|
618
742
|
# to ("0.10000000149011612"); cmplx64 has that in each part.
|
|
619
743
|
if !col.face? && (col.data_type == CA_FLOAT32 || col.data_type == CA_CMPLX64)
|
|
620
744
|
col = col.__widen_keeping_shortest_decimal__
|
|
621
745
|
end
|
|
622
746
|
# A boolean goes out as 0 / 1, as it does in every other serialization;
|
|
623
|
-
#
|
|
747
|
+
# read_csv reads it back with types: { name => :boolean }.
|
|
624
748
|
col = col.to_type(:uint8) if !col.face? && col.data_type == CA_BOOLEAN
|
|
625
749
|
col.to_a.map do |e|
|
|
626
750
|
if UNDEF.equal?(e) || e.nil?
|
|
@@ -629,12 +753,28 @@ class CAFrame
|
|
|
629
753
|
e
|
|
630
754
|
elsif e.respond_to?(:iso8601)
|
|
631
755
|
e.iso8601
|
|
756
|
+
elsif precision && e.is_a?(Float)
|
|
757
|
+
# Still a float, so precision: 0 writes 3.0 and the column reads
|
|
758
|
+
# back as floats.
|
|
759
|
+
csv_rounded(e, precision).to_s
|
|
760
|
+
elsif precision && e.is_a?(Complex) && e.real.is_a?(Float)
|
|
761
|
+
Complex(*[e.real, e.imaginary].map { |x| csv_rounded(x, precision) }).to_s
|
|
632
762
|
else
|
|
633
763
|
e.to_s
|
|
634
764
|
end
|
|
635
765
|
end
|
|
636
766
|
end
|
|
637
767
|
|
|
768
|
+
# A float rounded to +precision+ places, as a Float. A zero keeps the sign
|
|
769
|
+
# of the value it came from at every precision: Float#round gives an
|
|
770
|
+
# Integer at 0 places, which has no -0, where it gives -0.0 at 1 or more.
|
|
771
|
+
private def csv_rounded(x, precision)
|
|
772
|
+
return x unless x.finite?
|
|
773
|
+
r = x.round(precision).to_f
|
|
774
|
+
r = -0.0 if r == 0 && (x < 0 || (x == 0 && (1.0 / x) < 0))
|
|
775
|
+
r
|
|
776
|
+
end
|
|
777
|
+
|
|
638
778
|
private def csv_text(names, attr_rows, formatted, header, sep, quote)
|
|
639
779
|
out = +""
|
|
640
780
|
if header
|
|
@@ -745,7 +885,7 @@ class CAFrame
|
|
|
745
885
|
names ||= Array.new(ncol) { |j| "c#{j}" }
|
|
746
886
|
# A frame holds one column per name, so a repeated one would keep only
|
|
747
887
|
# the last of its columns.
|
|
748
|
-
CAFrame.__send__(:refuse_repeated_names, names, "
|
|
888
|
+
CAFrame.__send__(:refuse_repeated_names, names, "read_csv", "the header row names",
|
|
749
889
|
hint: "pass header: false for the default column names " \
|
|
750
890
|
"(c0, c1, ...), or name the columns with column_names:")
|
|
751
891
|
|