carray 3.0.4 → 3.0.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (155) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +438 -0
  3. data/README.md +1 -1
  4. data/carray.gemspec +3 -2
  5. data/ext/ca_array_pool.c +25 -1
  6. data/ext/ca_axis_dispatch.c +19 -20
  7. data/ext/ca_axis_group.c +1 -1
  8. data/ext/ca_categorical_iterator.c +34 -27
  9. data/ext/ca_compare.c +4 -4
  10. data/ext/ca_composite_dispatch.c +9 -9
  11. data/ext/ca_kernel_iterator.c +122 -237
  12. data/ext/ca_kernel_iterator.h +522 -337
  13. data/ext/ca_obj_array.c +69 -19
  14. data/ext/ca_obj_bincmp.c +10 -1
  15. data/ext/ca_obj_binop.c +11 -1
  16. data/ext/ca_obj_bitarray.c +5 -1
  17. data/ext/ca_obj_bitfield.c +60 -43
  18. data/ext/ca_obj_byte_swap.c +4 -1
  19. data/ext/ca_obj_const_string.c +15 -3
  20. data/ext/ca_obj_face.c +44 -1
  21. data/ext/ca_obj_fake.c +3 -1
  22. data/ext/ca_obj_field.c +13 -3
  23. data/ext/ca_obj_grid.c +219 -17
  24. data/ext/ca_obj_meld.c +69 -113
  25. data/ext/ca_obj_moncmp.c +9 -1
  26. data/ext/ca_obj_monop.c +86 -12
  27. data/ext/ca_obj_object.c +118 -32
  28. data/ext/ca_obj_record.c +3 -0
  29. data/ext/ca_obj_reduce.c +19 -1
  30. data/ext/ca_obj_refer.c +80 -26
  31. data/ext/ca_obj_remap.c +69 -10
  32. data/ext/ca_obj_roll.c +27 -6
  33. data/ext/ca_obj_select.c +38 -6
  34. data/ext/ca_obj_select_axis.c +1 -0
  35. data/ext/ca_obj_shift.c +32 -7
  36. data/ext/ca_obj_stack.c +311 -111
  37. data/ext/ca_obj_stride.c +159 -64
  38. data/ext/ca_obj_string.c +27 -2
  39. data/ext/ca_obj_tile.c +28 -7
  40. data/ext/ca_obj_transpose.c +8 -4
  41. data/ext/ca_obj_triop.c +12 -1
  42. data/ext/ca_obj_window.c +80 -50
  43. data/ext/ca_op_ipower.c +12 -12
  44. data/ext/ca_sort_kernels.h +6 -3
  45. data/ext/caframe_csv_reader.c +1 -1
  46. data/ext/carray.h +35 -1
  47. data/ext/carray_access.c +43 -67
  48. data/ext/carray_address_basis.c +134 -0
  49. data/ext/carray_attribute.c +32 -0
  50. data/ext/carray_broadcast.c +3 -4
  51. data/ext/carray_build_flags.h +1 -1
  52. data/ext/carray_cast.c +251 -93
  53. data/ext/carray_cast_func.rb +56 -26
  54. data/ext/carray_conversion.c +11 -7
  55. data/ext/carray_core.c +814 -72
  56. data/ext/carray_count.c +6 -0
  57. data/ext/carray_element.c +22 -13
  58. data/ext/carray_factorize.c +20 -20
  59. data/ext/carray_generate.c +8 -2
  60. data/ext/carray_histogram.c +26 -4
  61. data/ext/carray_index_classifier.c +4 -4
  62. data/ext/carray_internal.h +159 -12
  63. data/ext/carray_kernels_bincmp.c +2174 -170
  64. data/ext/carray_kernels_binop.c +25653 -5487
  65. data/ext/carray_kernels_init.c +8 -8
  66. data/ext/carray_kernels_moncmp.c +96 -96
  67. data/ext/carray_kernels_monop.c +12253 -5235
  68. data/ext/carray_kernels_reduce_aggregate.c +6150 -2423
  69. data/ext/carray_kernels_reduce_boolean.c +1533 -15
  70. data/ext/carray_kernels_reduce_cumulative.c +2442 -882
  71. data/ext/carray_kernels_reduce_extreme.c +5658 -1661
  72. data/ext/carray_kernels_reduce_variance.c +2534 -402
  73. data/ext/carray_kernels_scan.c +368 -368
  74. data/ext/carray_kernels_search.c +4551 -604
  75. data/ext/carray_kernels_sort.c +659 -751
  76. data/ext/carray_kernels_triop.c +114 -114
  77. data/ext/carray_key_windows.c +4 -0
  78. data/ext/carray_lazy.c +47 -0
  79. data/ext/carray_loop.c +1 -1
  80. data/ext/carray_mask.c +175 -161
  81. data/ext/carray_median_percentile.c +4 -4
  82. data/ext/carray_memory_view.c +15 -9
  83. data/ext/carray_operator.c +152 -49
  84. data/ext/carray_order.c +230 -10
  85. data/ext/carray_random.c +22 -19
  86. data/ext/carray_scatter.c +24 -7
  87. data/ext/carray_slab.c +128 -29
  88. data/ext/carray_sort.c +224 -16
  89. data/ext/carray_sort_kernel.c +5 -7
  90. data/ext/carray_struct.c +35 -43
  91. data/ext/carray_test.c +74 -19
  92. data/ext/carray_time_text.c +6 -4
  93. data/ext/carray_utils.c +63 -21
  94. data/ext/extconf.rb +11 -0
  95. data/ext/mkkernel.rb +546 -199
  96. data/ext/ruby_carray.c +4 -1
  97. data/ext/version.h +4 -4
  98. data/lib/carray/attribute.rb +42 -5
  99. data/lib/carray/autoload_carray.rb +3 -3
  100. data/lib/carray/autoload_method_extension.rb +5 -3
  101. data/lib/carray/axis_group.rb +13 -7
  102. data/lib/carray/basics.rb +28 -8
  103. data/lib/carray/block_iterator.rb +17 -3
  104. data/lib/carray/categorical.rb +35 -23
  105. data/lib/carray/categorical_iterator.rb +8 -8
  106. data/lib/carray/complex.rb +2 -2
  107. data/lib/carray/conditional.rb +3 -2
  108. data/lib/carray/const_string.rb +10 -0
  109. data/lib/carray/data_type_extension.rb +8 -1
  110. data/lib/carray/fixlen_string.rb +41 -2
  111. data/lib/carray/frame/column_verbs.rb +48 -16
  112. data/lib/carray/frame/concat.rb +17 -3
  113. data/lib/carray/frame/convert.rb +19 -0
  114. data/lib/carray/frame/csv_parser.rb +32 -32
  115. data/lib/carray/frame/describe.rb +4 -2
  116. data/lib/carray/frame/frame.rb +405 -85
  117. data/lib/carray/frame/group.rb +777 -64
  118. data/lib/carray/frame/io.rb +212 -72
  119. data/lib/carray/frame/join.rb +146 -37
  120. data/lib/carray/frame/records.rb +3 -3
  121. data/lib/carray/frame/reshape.rb +89 -11
  122. data/lib/carray/frame/sort.rb +57 -9
  123. data/lib/carray/frame/verbs.rb +384 -49
  124. data/lib/carray/frame/windows.rb +44 -6
  125. data/lib/carray/fuse_source.rb +31 -2
  126. data/lib/carray/inspect.rb +7 -2
  127. data/lib/carray/iterator.rb +32 -0
  128. data/lib/carray/lazy.rb +52 -8
  129. data/lib/carray/mask_gap_fill.rb +37 -17
  130. data/lib/carray/meld_reduce.rb +1 -1
  131. data/lib/carray/methods/choose.rb +3 -2
  132. data/lib/carray/methods/is_in.rb +8 -1
  133. data/lib/carray/methods/meshgrid.rb +3 -1
  134. data/lib/carray/runtime.rb +47 -8
  135. data/lib/carray/segment_iterator.rb +35 -16
  136. data/lib/carray/serialize.rb +53 -19
  137. data/lib/carray/stack.rb +33 -18
  138. data/lib/carray/string.rb +8 -0
  139. data/lib/carray/string_operation_extension.rb +13 -5
  140. data/lib/carray/struct_builder.rb +5 -3
  141. data/lib/carray/time.rb +112 -68
  142. data/lib/carray/window_iterator.rb +16 -4
  143. data/yard-stubs/ca_obj_array.rb +26 -1
  144. data/yard-stubs/ca_obj_transpose.rb +5 -4
  145. data/yard-stubs/carray_attribute.rb +3 -1
  146. data/yard-stubs/carray_cast.rb +10 -5
  147. data/yard-stubs/carray_conversion.rb +13 -3
  148. data/yard-stubs/carray_element.rb +7 -6
  149. data/yard-stubs/carray_mask.rb +28 -4
  150. data/yard-stubs/carray_math.rb +45 -20
  151. data/yard-stubs/carray_memory_view.rb +7 -0
  152. data/yard-stubs/carray_order.rb +100 -4
  153. data/yard-stubs/carray_scatter.rb +7 -5
  154. data/yard-stubs/carray_test.rb +7 -4
  155. metadata +18 -4
@@ -9,26 +9,32 @@ class CAFrame
9
9
  # over one object array (§4.2). Casting is a separate step: +types:+ takes
10
10
  # anything +cast+ takes as a map -- { "temp" => :float64 }, :infer, or a
11
11
  # map with +default:+ -- and casts the frame with it on load, which is the
12
- # same as calling +cast+ afterwards. Broken cells become UNDEF
13
- # automatically (parse-mask, §6-2).
12
+ # same as calling +cast+ afterwards. The default is :infer, which gives
13
+ # each column the type all of its cells read as (see +infer_types+);
14
+ # +types: :string+ leaves every column text. A Hash names the types of some
15
+ # columns and infers the rest, unless its +default:+ says otherwise
16
+ # (+default: :string+ keeps the rest text); +nil+ for a column keeps it
17
+ # text. Broken cells become UNDEF automatically (parse-mask, §6-2).
14
18
  #
15
- # CAFrame.from_csv("obs.csv", types: :infer)
16
- # CAFrame.from_csv("obs.csv", types: { default: :infer, "code" => :int32, "id" => nil })
17
- # CAFrame.from_csv("obs.csv", types: { default: :float64, "station" => nil })
18
- # CAFrame.from_csv("obs.csv", types: { "station" => :categorical })
19
+ # CAFrame.read_csv("obs.csv") # types inferred
20
+ # CAFrame.read_csv("obs.csv", types: :string) # every column text
21
+ # CAFrame.read_csv("obs.csv", types: { "code" => :int32, "id" => nil })
22
+ # CAFrame.read_csv("obs.csv", types: { default: :string, "temp" => :float64 })
23
+ # CAFrame.read_csv("obs.csv", types: { default: :infer, "code" => :int32, "id" => nil })
24
+ # CAFrame.read_csv("obs.csv", types: { default: :float64, "station" => nil })
25
+ # CAFrame.read_csv("obs.csv", types: { "station" => :categorical })
19
26
  #
20
- # +source+ is a path, or an open IO -- anything answering +gets+, which a
21
- # StringIO is. So CSV that is already in memory does not have to go to a
22
- # temporary file first:
27
+ # +source+ is a path, or an open IO -- anything answering +gets+:
23
28
  #
24
- # CAFrame.from_csv("obs.csv") # a path
25
- # CAFrame.from_csv(StringIO.new(body)) # text already in hand
26
- # File.open("obs.csv") { |io| CAFrame.from_csv(io) }
29
+ # CAFrame.read_csv("obs.csv") # a path
30
+ # File.open("obs.csv") { |io| CAFrame.read_csv(io) }
31
+ # CAFrame.read_csv($stdin)
27
32
  #
28
33
  # A String is always read as a path, never as CSV text: guessing between the
29
34
  # two by looking for a newline is the kind of guess that is right until it is
30
- # not, and StringIO says which one you meant in six characters. An IO is read
31
- # from where it is and left open -- the caller opened it and closes it.
35
+ # not. CSV text already in hand goes to +from_csv+, which takes the same
36
+ # options. An IO is read from where it is and left open -- the caller opened
37
+ # it and closes it.
32
38
  #
33
39
  # Parsing uses the built-in fast tokenizer (CSVParser). Options:
34
40
  # sep: field separator (default ",")
@@ -66,34 +72,39 @@ class CAFrame
66
72
  # attrs: attribute keys for the lines right below the header, one
67
73
  # key per line: field j of a line becomes that attribute of
68
74
  # column j (an empty field gives none). The data then starts
69
- # below them. What to_csv(attrs:) writes, it reads back
75
+ # below them. What to_csv(attrs:) writes, it reads back. The
76
+ # fields are read as the data fields are, so strip: trims them
77
+ # too
70
78
  #
71
- # CAFrame.from_csv("obs.csv", header: 2) # a title above
72
- # CAFrame.from_csv("obs.csv", header: 0, data: 3) # units on lines 1-2
73
- # CAFrame.from_csv("big.csv", data: 1..100) # the first 100 lines
74
- # CAFrame.from_csv("raw.csv", column_names: %w[date temp rh])
75
- # CAFrame.from_csv("wide.csv", columns: %w[date temp])
76
- # CAFrame.from_csv("raw.csv", header: false, columns: { 0 => "date", 3 => "temp" })
77
- # CAFrame.from_csv("obs.csv", attrs: [:units]) # units on line 1
79
+ # CAFrame.read_csv("obs.csv", header: 2) # a title above
80
+ # CAFrame.read_csv("obs.csv", header: 0, data: 3) # units on lines 1-2
81
+ # CAFrame.read_csv("big.csv", data: 1..100) # the first 100 lines
82
+ # CAFrame.read_csv("raw.csv", column_names: %w[date temp rh])
83
+ # CAFrame.read_csv("wide.csv", columns: %w[date temp])
84
+ # CAFrame.read_csv("raw.csv", header: false, columns: { 0 => "date", 3 => "temp" })
85
+ # CAFrame.read_csv("obs.csv", attrs: [:units]) # units on line 1
78
86
  #
79
87
  # A block, given the reader, reads in any other order with +skip+ /
80
88
  # +header+ / +header(name)+ / +column_names+ / +columns+ / +data+ (see
81
89
  # CSVReader):
82
90
  #
83
- # CAFrame.from_csv("obs.csv") { |r| r.skip 2; r.header; r.skip 1; r.data }
91
+ # CAFrame.read_csv("obs.csv") { |r| r.skip 2; r.header; r.skip 1; r.data }
84
92
  #
85
93
  # A missing field -- an unquoted empty one, or a cell a short row never
86
94
  # reached -- is UNDEF in the frame, whether or not the column is cast by
87
95
  # +types:+. A quoted empty field ("") is the empty string, which is a value.
88
- # So the mask +to_csv+ writes comes back as a mask.
96
+ # So the mask +to_csv+ writes comes back as a mask. A blank value (a quoted
97
+ # "", or spaces kept by strip: false) is no number, truth value or time, so
98
+ # a column holding one is not inferred and stays text; name its type in
99
+ # +types:+ to read it as numbers with the blank cells UNDEF.
89
100
  #
90
101
  # Files that spell "missing" some other way say so with +missing:+. A field
91
102
  # whose text is one of the given tokens is UNDEF too, before +types:+
92
103
  # casts, so a -999 sentinel never reaches the column as a number:
93
104
  #
94
- # CAFrame.from_csv("obs.csv", missing: ["-999", "///"]) # every column
95
- # CAFrame.from_csv("obs.csv", missing: { "temp" => "-999" }) # one column
96
- # CAFrame.from_csv("obs.csv", missing: { default: "-999", "id" => [] })
105
+ # CAFrame.read_csv("obs.csv", missing: ["-999", "///"]) # every column
106
+ # CAFrame.read_csv("obs.csv", missing: { "temp" => "-999" }) # one column
107
+ # CAFrame.read_csv("obs.csv", missing: { default: "-999", "id" => [] })
97
108
  #
98
109
  # Tokens are Strings and are compared with the field's text as read (after
99
110
  # +strip:+, quoted or not), so "-999" does not match "-999.0". In a Hash,
@@ -102,35 +113,42 @@ class CAFrame
102
113
  # +:default+ the columns it does not name keep only the empty field. As in
103
114
  # +to_csv+, "" means the empty field, so a quoted "" is still a value.
104
115
  #
105
- # Without +types:+, the columns are handed to the frame as CABlock views
106
- # over one backing object array (§3.6 view-by-default). With +types:+, the
116
+ # With +types: :string+, the columns are handed to the frame as CABlock views
117
+ # over one backing object array (§3.6 view-by-default). Otherwise, the
107
118
  # body is read without making a String per cell: a column cast to a number
108
119
  # is read from the text directly, and a column left as text becomes its own
109
120
  # CAString. Casting a column materializes it, and +copy+ gives an
110
121
  # independent frame.
111
- def self.from_csv(source, types: nil, on_error: :mask, missing: nil,
122
+ def self.read_csv(source, types: :infer, on_error: :mask, missing: nil,
112
123
  header: nil, data: nil, column_names: nil, columns: nil,
113
124
  attrs: nil, sep: ",", quote: '"', strip: false,
114
125
  encoding: "bom|utf-8", parser: nil, &block)
115
126
  layout = { header: header, data: data, column_names: column_names,
116
127
  columns: columns, attrs: attrs }.compact
117
128
  if block && !layout.empty?
118
- raise ArgumentError, "from_csv: give #{layout.keys.map { |k| "#{k}:" }.join(', ')} " \
129
+ raise ArgumentError, "read_csv: give #{layout.keys.map { |k| "#{k}:" }.join(', ')} " \
119
130
  "or a reading block, not both"
120
131
  end
121
132
  # The parser reads the header, the data and the columns itself.
122
133
  if parser && !layout.empty?
123
- raise ArgumentError, "from_csv: give #{layout.keys.map { |k| "#{k}:" }.join(', ')} " \
134
+ raise ArgumentError, "read_csv: give #{layout.keys.map { |k| "#{k}:" }.join(', ')} " \
124
135
  "or parser:, not both"
125
136
  end
126
137
  if block && block.arity != 1
127
- raise ArgumentError, "from_csv: the reading block takes the reader as its " \
128
- "parameter, as in from_csv(path) { |r| r.skip 2; r.header; r.data }"
138
+ raise ArgumentError, "read_csv: the reading block takes the reader as its " \
139
+ "parameter, as in read_csv(path) { |r| r.skip 2; r.header; r.data }"
140
+ end
141
+ # A String is a path, always. One with a line break in it that names no
142
+ # file is likely CSV text; the missing file says how to give text.
143
+ if source.is_a?(String) && source.include?("\n") && !File.exist?(source)
144
+ raise Errno::ENOENT, "#{source.inspect}; a String is a path, so to read CSV text " \
145
+ "give it to from_csv(text)"
129
146
  end
130
147
  missing = missing_tokens(missing) if missing
148
+ types = csv_types(types)
131
149
  # With types:, the body is read into CAConstString columns, so a column
132
150
  # cast to a number never becomes Strings; the text columns left over
133
- # become CAString below. Without types: every column stays text, so the
151
+ # become CAString below. With types: :string every column stays text, so the
134
152
  # body goes straight to Strings. missing: masks text in place, which
135
153
  # wants the Strings first, so it takes that path too.
136
154
  read_text_as_const_string = !types.nil? && missing.nil?
@@ -140,7 +158,7 @@ class CAFrame
140
158
  elsif source.respond_to?(:gets)
141
159
  with_encoding_hint("open the IO in the file's encoding, as in " \
142
160
  "File.open(path, \"r:<file encoding>:UTF-8\")") do
143
- read_csv(source, sep: sep, quote: quote, strip: strip, layout: layout,
161
+ parse_csv_io(source, sep: sep, quote: quote, strip: strip, layout: layout,
144
162
  read_text_as_const_string: read_text_as_const_string, &block)
145
163
  end
146
164
  else
@@ -148,7 +166,7 @@ class CAFrame
148
166
  "file's encoding, as in " \
149
167
  "encoding: \"<file encoding>:UTF-8\"") do
150
168
  File.open(source, "r:#{encoding}") do |io|
151
- read_csv(io, sep: sep, quote: quote, strip: strip, layout: layout,
169
+ parse_csv_io(io, sep: sep, quote: quote, strip: strip, layout: layout,
152
170
  read_text_as_const_string: read_text_as_const_string, &block)
153
171
  end
154
172
  end
@@ -156,7 +174,8 @@ class CAFrame
156
174
 
157
175
  frame = build_frame(names, rows)
158
176
  mask_missing_tokens(frame, missing) if missing
159
- frame.cast(types, on_error: on_error, option_name: :types) unless types.nil?
177
+ types = keep_blank_valued_columns_text(frame, types) unless types.nil?
178
+ frame.__send__(:cast_in_place, types, on_error: on_error, option_name: :types) unless types.nil?
160
179
  convert_const_string_columns_to_castring(frame)
161
180
  column_attrs&.each_with_index do |attrs, j|
162
181
  frame[frame.column_names[j]].set_attrs(attrs) unless attrs.empty?
@@ -164,7 +183,82 @@ class CAFrame
164
183
  frame
165
184
  end
166
185
 
167
- # A text column from_csv reads is a CAString, whichever way the body was
186
+ # Read CSV text already in hand into a frame -- the String is the CSV, not
187
+ # a path. The options and the reading block are +read_csv+'s; +encoding:+
188
+ # has nothing to open and is refused, the text being in its String's
189
+ # encoding. A BOM at the start of UTF-8 or binary text (as File.binread
190
+ # and many HTTP bodies give) is skipped, as +read_csv+ skips one at the
191
+ # start of a file.
192
+ #
193
+ # CAFrame.from_csv("date,temp\n2026-01-01,1.5\n")
194
+ # CAFrame.from_csv(response.body, types: :string)
195
+ # CAFrame.from_csv(df.to_csv) # the round trip
196
+ def self.from_csv(text, **opts, &block)
197
+ unless text.is_a?(String)
198
+ raise TypeError, "from_csv takes CSV text as a String (got #{text.class}); " \
199
+ "read a path or an IO with read_csv"
200
+ end
201
+ if opts.key?(:encoding)
202
+ raise ArgumentError, "from_csv: encoding: is the open mode of a path; " \
203
+ "text in hand is read in its String's encoding"
204
+ end
205
+ if text.encoding == Encoding::UTF_8
206
+ text = text.delete_prefix("\uFEFF")
207
+ elsif text.encoding == Encoding::BINARY && text.start_with?("\xEF\xBB\xBF".b)
208
+ text = text.byteslice(3..) # bytes read from a file or the network
209
+ end
210
+ read_csv(StringIO.new(text), **opts, &block)
211
+ end
212
+
213
+ # read_csv's types: as the cast map it stands for: inference unless said
214
+ # otherwise. :string reads every column as text (no cast); a Hash without
215
+ # default: infers the columns it does not name, default: :string keeps
216
+ # them text, and :string (or nil) for a column keeps that one text.
217
+ def self.csv_types(types)
218
+ return nil if types == :string
219
+ return types unless types.is_a?(Hash)
220
+ return { default: :infer }.merge(types) unless types.key?(:default)
221
+ return types
222
+ end
223
+ private_class_method :csv_types
224
+
225
+ # The reader masks the fields that are missing (an unquoted empty field, a
226
+ # cell a short row never reaches), so a blank cell it leaves present was
227
+ # written as a value: a quoted "", or spaces kept by strip: false. Such a
228
+ # cell is no number, truth value or time, so a column holding one is kept
229
+ # as text rather than inferred, where a blank cell would count as missing.
230
+ def self.keep_blank_valued_columns_text(frame, types)
231
+ map = types == :infer ? { default: :infer } : types
232
+ return types unless map.is_a?(Hash)
233
+ frame.column_names.each do |name|
234
+ given = [name, name.to_sym].find { |k| map.key?(k) }
235
+ next unless (given ? map[given] : map[:default]) == :infer
236
+ map = map.merge(name => :string) if blank_value_in?(frame[name])
237
+ end
238
+ return map
239
+ end
240
+
241
+ BLANK_LEADS = [" ", "\t", "\n", "\v", "\f", "\r", "\0"].freeze
242
+ Ractor.make_shareable(BLANK_LEADS)
243
+ private_constant :BLANK_LEADS
244
+
245
+ # Whether a text column holds a present cell of nothing but whitespace. A
246
+ # CAConstString is narrowed in C to the cells that are empty or start with
247
+ # whitespace, and only those are read as Strings.
248
+ def self.blank_value_in?(col)
249
+ if col.is_a?(CAConstString)
250
+ lead = col.byte_length.eq(0)
251
+ BLANK_LEADS.each { |c| lead = lead.or(col.start_with?(c)) }
252
+ addrs = lead.strip_mask(false).where
253
+ return addrs.to_a.any? { |i| col[i].strip.empty? }
254
+ end
255
+ return false unless col.is_a?(CAString) || (col.data_type == CA_OBJECT && !col.face?)
256
+ col.each { |v| return true if v.is_a?(String) && v.strip.empty? }
257
+ return false
258
+ end
259
+ private_class_method :keep_blank_valued_columns_text, :blank_value_in?
260
+
261
+ # A text column read_csv reads is a CAString, whichever way the body was
168
262
  # read: the string operations (strip!, gsub!, ...) write through to it.
169
263
  def self.convert_const_string_columns_to_castring(frame)
170
264
  frame.column_names.each do |name|
@@ -175,7 +269,7 @@ class CAFrame
175
269
  end
176
270
  private_class_method :convert_const_string_columns_to_castring
177
271
 
178
- # Split a +missing:+ argument, as from_csv and to_csv both take it, into the
272
+ # Split a +missing:+ argument, as read_csv and to_csv both take it, into the
179
273
  # setting for every column and a Hash of per-column settings that replace
180
274
  # it. A Hash says :default for every column and a column name for one;
181
275
  # column names are Strings, so the Symbol cannot collide with one. A
@@ -198,7 +292,7 @@ class CAFrame
198
292
  # The read side of missing:, checked before the file is read: each setting
199
293
  # becomes an Array of token Strings.
200
294
  def self.missing_tokens(missing)
201
- default, per_column = split_missing(missing, "from_csv")
295
+ default, per_column = split_missing(missing, "read_csv")
202
296
  tokens = lambda do |value|
203
297
  list = Array(value)
204
298
  bad = list.reject { |t| t.is_a?(String) }
@@ -250,7 +344,7 @@ class CAFrame
250
344
  # Drive the reading-control DSL over one open IO and hand back
251
345
  # [names, rows, column_attrs]. Shared by the path and the IO source, so the
252
346
  # two cannot come to read a file differently.
253
- def self.read_csv (io, sep:, quote:, strip:, layout:, read_text_as_const_string: false, &block)
347
+ def self.parse_csv_io (io, sep:, quote:, strip:, layout:, read_text_as_const_string: false, &block)
254
348
  reader = CSVReader.new(io, sep: sep, quote: quote, strip: strip,
255
349
  read_text_as_const_string: read_text_as_const_string)
256
350
  reader.reporting do
@@ -260,7 +354,7 @@ class CAFrame
260
354
  [names, rows, reader.column_attrs(reader.file_ncol)]
261
355
  end
262
356
 
263
- private_class_method :read_csv
357
+ private_class_method :parse_csv_io
264
358
 
265
359
  # Re-raise a failure to decode the input with a line saying how to name the
266
360
  # file's encoding. The error and its class are kept; only the message grows.
@@ -291,47 +385,75 @@ class CAFrame
291
385
  # no flat CSV cell and raises; export it per column, or use +to_records+ +
292
386
  # JSON for the structured shape (memo §11.9, the N-D escape).
293
387
  #
294
- # With +path+, writes the file and returns self; without it, returns the CSV
295
- # String. The index (if any) is written as the first column under +axis_name+
296
- # unless +index: false+. A masked cell (UNDEF) becomes an empty field, which
297
- # +from_csv+ reads back as UNDEF (parse-mask, §6-2) -- so mask round-trips. A
388
+ # +target+ is a path, or an open IO -- anything answering +write+, which is
389
+ # written to where it is and left open. Returns self; +to_csv+ gives the
390
+ # same text as a String. The index (if any) is written as the first column
391
+ # under +axis_name+ unless +index: false+. A masked cell (UNDEF) becomes an
392
+ # empty field, which +read_csv+ reads back as UNDEF (parse-mask, §6-2) -- so mask round-trips. A
298
393
  # genuine empty string is written quoted (+""+) to stay distinct from missing,
299
394
  # matching the tokenizer's own unquoted-empty vs quoted-empty distinction.
300
395
  #
301
- # df.to_csv("out.csv") # write file
302
- # csv = df.to_csv # get a String
396
+ # df.write_csv("out.csv")
397
+ # df.write_csv($stdout)
303
398
  #
304
- # Options: +sep+ / +quote+ mirror +from_csv+; +header+ writes the name row
399
+ # Options: +sep+ / +quote+ mirror +read_csv+; +header+ writes the name row
305
400
  # (default true); +index+ writes the index column (default true);
306
- # +encoding+ transcodes the text before it is written or returned (default
401
+ # +encoding+ transcodes the text before it is written (default
307
402
  # nil leaves it as built, UTF-8). A character the encoding cannot hold
308
403
  # raises Encoding::UndefinedConversionError rather than being dropped. A
309
404
  # String cell in another encoding is transcoded to UTF-8; one with bytes
310
405
  # that are not valid in it, or with no encoding (ASCII-8BIT), raises. Each
311
406
  # names the row and the column of the cell.
312
407
  #
313
- # df.to_csv("out.csv", encoding: "CP932") # for Excel in Japanese
408
+ # df.write_csv("out.csv", encoding: "CP932") # for Excel in Japanese
314
409
  #
315
410
  # +missing:+ writes a masked cell as that String instead of an empty field,
316
- # for a reader that expects a sentinel; +from_csv(missing:)+ with the same
411
+ # for a reader that expects a sentinel; +read_csv(missing:)+ with the same
317
412
  # String reads it back as UNDEF. A Hash sets it per column the way
318
- # +from_csv+ does: +:default+ for every column, and a column name (or the
413
+ # +read_csv+ does: +:default+ for every column, and a column name (or the
319
414
  # index's axis name) for one, replacing the default; "" is the empty field.
320
415
  # A cell whose value would be written as the same text raises, since the
321
416
  # file could not tell the two apart.
322
417
  #
323
- # df.to_csv("out.csv", missing: "-999")
324
- # df.to_csv("out.csv", missing: { default: "-999", "comment" => "" })
418
+ # df.write_csv("out.csv", missing: "-999")
419
+ # df.write_csv("out.csv", missing: { default: "-999", "comment" => "" })
325
420
  #
326
421
  # +attrs:+ writes, below the name row, one line per attribute key given,
327
422
  # in that order: field j is +attr(key)+ of column j (the index's for the
328
423
  # index column). A column without the attribute gets an empty field, an
329
424
  # empty String a quoted "", and a number its +to_s+.
330
- # +from_csv(attrs:)+ with the same keys reads them back, as Strings.
425
+ # +read_csv(attrs:)+ with the same keys reads them back, as Strings.
426
+ #
427
+ # df.write_csv("out.csv", attrs: [:units, :long_name])
428
+ #
429
+ # +precision:+ writes each float cell rounded to that many decimal places,
430
+ # still as a float (precision: 0 writes 3.0), and each part of a complex
431
+ # cell likewise; NaN and infinities are written as they are, and a value
432
+ # that rounds to zero keeps its sign (-0.4 at 0 places is -0.0). Without
433
+ # it a float is written in full, so that read_csv reads back the same
434
+ # value.
331
435
  #
332
- # df.to_csv("out.csv", attrs: [:units, :long_name])
333
- def to_csv(path = nil, sep: ",", quote: '"', header: true, index: true,
334
- encoding: nil, missing: nil, attrs: [])
436
+ # df.write_csv("out.csv", precision: 2)
437
+ def write_csv(target, **opts)
438
+ out = to_csv(**opts)
439
+ if target.respond_to?(:write)
440
+ target.write(out)
441
+ else
442
+ File.binwrite(target, out)
443
+ end
444
+ self
445
+ end
446
+
447
+ # The frame as CSV text: what +write_csv+ writes, as a String, with the same
448
+ # options. +from_csv+ reads it back.
449
+ #
450
+ # csv = df.to_csv
451
+ # csv = df.to_csv(sep: "\t", index: false)
452
+ def to_csv(sep: ",", quote: '"', header: true, index: true,
453
+ encoding: nil, missing: nil, attrs: [], precision: nil)
454
+ unless precision.nil? || (precision.is_a?(Integer) && precision >= 0)
455
+ raise ArgumentError, "to_csv: precision: is a number of decimal places, 0 or more (got #{precision.inspect})"
456
+ end
335
457
  default, per_column = CAFrame.send(:split_missing, missing, "to_csv")
336
458
  [default, *per_column.values].each do |text|
337
459
  next if text.nil? || text.is_a?(String)
@@ -365,14 +487,21 @@ class CAFrame
365
487
  written = []
366
488
  if index && @index
367
489
  names << @axis_name
368
- formatted << format_csv_column(@index)
490
+ formatted << format_csv_column(@index, precision)
369
491
  written << @index
370
492
  end
371
493
  @columns.each do |name, col|
372
494
  names << name
373
- formatted << format_csv_column(col)
495
+ formatted << format_csv_column(col, precision)
374
496
  written << col
375
497
  end
498
+ # A line with no fields reads back as a blank line, which a reader passes
499
+ # over, so the rows would be lost.
500
+ if names.empty? && @nrow > 0
501
+ raise ArgumentError,
502
+ "to_csv: the frame has #{@nrow} rows but no field to write " \
503
+ "(no columns#{@index ? ', and index: false' : ''})"
504
+ end
376
505
  attr_rows = attr_keys.map do |key|
377
506
  written.each_with_index.map { |col, j| csv_attr_text(col, key, names[j]) }
378
507
  end
@@ -419,12 +548,7 @@ class CAFrame
419
548
  end
420
549
  end
421
550
 
422
- if path
423
- File.binwrite(path, out)
424
- self
425
- else
426
- out
427
- end
551
+ out
428
552
  end
429
553
 
430
554
  # Render the frame as an aligned text table for reading:
@@ -612,15 +736,15 @@ class CAFrame
612
736
  end
613
737
  end
614
738
 
615
- private def format_csv_column(col)
739
+ private def format_csv_column(col, precision = nil)
616
740
  # A float32 value goes out as its own shortest decimal ("0.1"), which
617
- # from_csv reads back as the same value, not as the double it widens
741
+ # read_csv reads back as the same value, not as the double it widens
618
742
  # to ("0.10000000149011612"); cmplx64 has that in each part.
619
743
  if !col.face? && (col.data_type == CA_FLOAT32 || col.data_type == CA_CMPLX64)
620
744
  col = col.__widen_keeping_shortest_decimal__
621
745
  end
622
746
  # A boolean goes out as 0 / 1, as it does in every other serialization;
623
- # from_csv reads it back with types: { name => :boolean }.
747
+ # read_csv reads it back with types: { name => :boolean }.
624
748
  col = col.to_type(:uint8) if !col.face? && col.data_type == CA_BOOLEAN
625
749
  col.to_a.map do |e|
626
750
  if UNDEF.equal?(e) || e.nil?
@@ -629,12 +753,28 @@ class CAFrame
629
753
  e
630
754
  elsif e.respond_to?(:iso8601)
631
755
  e.iso8601
756
+ elsif precision && e.is_a?(Float)
757
+ # Still a float, so precision: 0 writes 3.0 and the column reads
758
+ # back as floats.
759
+ csv_rounded(e, precision).to_s
760
+ elsif precision && e.is_a?(Complex) && e.real.is_a?(Float)
761
+ Complex(*[e.real, e.imaginary].map { |x| csv_rounded(x, precision) }).to_s
632
762
  else
633
763
  e.to_s
634
764
  end
635
765
  end
636
766
  end
637
767
 
768
+ # A float rounded to +precision+ places, as a Float. A zero keeps the sign
769
+ # of the value it came from at every precision: Float#round gives an
770
+ # Integer at 0 places, which has no -0, where it gives -0.0 at 1 or more.
771
+ private def csv_rounded(x, precision)
772
+ return x unless x.finite?
773
+ r = x.round(precision).to_f
774
+ r = -0.0 if r == 0 && (x < 0 || (x == 0 && (1.0 / x) < 0))
775
+ r
776
+ end
777
+
638
778
  private def csv_text(names, attr_rows, formatted, header, sep, quote)
639
779
  out = +""
640
780
  if header
@@ -745,7 +885,7 @@ class CAFrame
745
885
  names ||= Array.new(ncol) { |j| "c#{j}" }
746
886
  # A frame holds one column per name, so a repeated one would keep only
747
887
  # the last of its columns.
748
- CAFrame.__send__(:refuse_repeated_names, names, "from_csv", "the header row names",
888
+ CAFrame.__send__(:refuse_repeated_names, names, "read_csv", "the header row names",
749
889
  hint: "pass header: false for the default column names " \
750
890
  "(c0, c1, ...), or name the columns with column_names:")
751
891