smarter_csv 1.16.4 → 1.17.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (46) hide show
  1. checksums.yaml +4 -4
  2. data/.rubocop.yml +10 -1
  3. data/CHANGELOG.md +54 -0
  4. data/Gemfile +10 -5
  5. data/README.md +98 -14
  6. data/TO_DO.md +109 -0
  7. data/docs/_introduction.md +1 -0
  8. data/docs/bad_row_quarantine.md +2 -1
  9. data/docs/basic_read_api.md +6 -1
  10. data/docs/basic_write_api.md +30 -0
  11. data/docs/batch_processing.md +25 -0
  12. data/docs/column_selection.md +1 -0
  13. data/docs/data_transformations.md +1 -0
  14. data/docs/examples.md +126 -0
  15. data/docs/header_transformations.md +23 -0
  16. data/docs/header_validations.md +1 -0
  17. data/docs/history.md +1 -0
  18. data/docs/instrumentation.md +2 -1
  19. data/docs/migrating_from_csv.md +1 -0
  20. data/docs/options.md +20 -18
  21. data/docs/parsing_strategy.md +1 -0
  22. data/docs/real_world_csv.md +51 -1
  23. data/docs/releases/1.16.0/performance_notes.md +15 -15
  24. data/docs/releases/1.17.0/benchmarks.md +121 -0
  25. data/docs/releases/1.17.0/changes.md +161 -0
  26. data/docs/releases/1.17.0/performance_notes.md +126 -0
  27. data/docs/row_col_sep.md +21 -1
  28. data/docs/ruby_csv_pitfalls.md +1 -0
  29. data/docs/value_converters.md +24 -0
  30. data/docs/warnings.md +141 -0
  31. data/ext/smarter_csv/smarter_csv.c +98 -32
  32. data/images/SmarterCSV_1.17.0_vs_RubyCSV_3.3.5_speedup.svg +106 -0
  33. data/images/SmarterCSV_1.17.0_vs_previous_C-speedup.svg +181 -0
  34. data/images/SmarterCSV_1.17.0_vs_previous_Rb-speedup.svg +179 -0
  35. data/lib/smarter_csv/auto_detection.rb +215 -30
  36. data/lib/smarter_csv/file_io.rb +2 -2
  37. data/lib/smarter_csv/hash_transformations.rb +29 -13
  38. data/lib/smarter_csv/parser.rb +42 -33
  39. data/lib/smarter_csv/peekable_io.rb +453 -0
  40. data/lib/smarter_csv/reader.rb +119 -23
  41. data/lib/smarter_csv/reader_options.rb +61 -1
  42. data/lib/smarter_csv/version.rb +1 -1
  43. data/lib/smarter_csv.rb +40 -12
  44. metadata +12 -5
  45. data/TO_DO_v2.md +0 -14
  46. data/ext/smarter_csv/Makefile +0 -270
@@ -1,270 +0,0 @@
1
-
2
- SHELL = /bin/sh
3
-
4
- # V=0 quiet, V=1 verbose. other values don't work.
5
- V = 0
6
- V0 = $(V:0=)
7
- Q1 = $(V:1=)
8
- Q = $(Q1:0=@)
9
- ECHO1 = $(V:1=@ :)
10
- ECHO = $(ECHO1:0=@ echo)
11
- NULLCMD = :
12
-
13
- #### Start of system configuration section. ####
14
-
15
- srcdir = .
16
- topdir = /Users/tilo/.rvm/rubies/ruby-3.2.2/include/ruby-3.2.0
17
- hdrdir = $(topdir)
18
- arch_hdrdir = /Users/tilo/.rvm/rubies/ruby-3.2.2/include/ruby-3.2.0/arm64-darwin23
19
- PATH_SEPARATOR = :
20
- VPATH = $(srcdir):$(arch_hdrdir)/ruby:$(hdrdir)/ruby
21
- prefix = $(DESTDIR)/Users/tilo/.rvm/rubies/ruby-3.2.2
22
- rubysitearchprefix = $(rubylibprefix)/$(sitearch)
23
- rubyarchprefix = $(rubylibprefix)/$(arch)
24
- rubylibprefix = $(libdir)/$(RUBY_BASE_NAME)
25
- exec_prefix = $(prefix)
26
- vendorarchhdrdir = $(vendorhdrdir)/$(sitearch)
27
- sitearchhdrdir = $(sitehdrdir)/$(sitearch)
28
- rubyarchhdrdir = $(rubyhdrdir)/$(arch)
29
- vendorhdrdir = $(rubyhdrdir)/vendor_ruby
30
- sitehdrdir = $(rubyhdrdir)/site_ruby
31
- rubyhdrdir = $(includedir)/$(RUBY_VERSION_NAME)
32
- vendorarchdir = $(vendorlibdir)/$(sitearch)
33
- vendorlibdir = $(vendordir)/$(ruby_version)
34
- vendordir = $(rubylibprefix)/vendor_ruby
35
- sitearchdir = $(sitelibdir)/$(sitearch)
36
- sitelibdir = $(sitedir)/$(ruby_version)
37
- sitedir = $(rubylibprefix)/site_ruby
38
- rubyarchdir = $(rubylibdir)/$(arch)
39
- rubylibdir = $(rubylibprefix)/$(ruby_version)
40
- sitearchincludedir = $(includedir)/$(sitearch)
41
- archincludedir = $(includedir)/$(arch)
42
- sitearchlibdir = $(libdir)/$(sitearch)
43
- archlibdir = $(libdir)/$(arch)
44
- ridir = $(datarootdir)/$(RI_BASE_NAME)
45
- mandir = $(datarootdir)/man
46
- localedir = $(datarootdir)/locale
47
- libdir = $(exec_prefix)/lib
48
- psdir = $(docdir)
49
- pdfdir = $(docdir)
50
- dvidir = $(docdir)
51
- htmldir = $(docdir)
52
- infodir = $(datarootdir)/info
53
- docdir = $(datarootdir)/doc/$(PACKAGE)
54
- oldincludedir = $(DESTDIR)/usr/include
55
- includedir = $(SDKROOT)$(prefix)/include
56
- runstatedir = $(localstatedir)/run
57
- localstatedir = $(prefix)/var
58
- sharedstatedir = $(prefix)/com
59
- sysconfdir = $(prefix)/etc
60
- datadir = $(datarootdir)
61
- datarootdir = $(prefix)/share
62
- libexecdir = $(exec_prefix)/libexec
63
- sbindir = $(exec_prefix)/sbin
64
- bindir = $(exec_prefix)/bin
65
- archdir = $(rubyarchdir)
66
-
67
-
68
- CC_WRAPPER =
69
- CC = gcc
70
- CXX = g++
71
- LIBRUBY = $(LIBRUBY_SO)
72
- LIBRUBY_A = lib$(RUBY_SO_NAME)-static.a
73
- LIBRUBYARG_SHARED = -l$(RUBY_SO_NAME)
74
- LIBRUBYARG_STATIC = -l$(RUBY_SO_NAME)-static -framework CoreFoundation $(MAINLIBS)
75
- empty =
76
- OUTFLAG = -o $(empty)
77
- COUTFLAG = -o $(empty)
78
- CSRCFLAG = $(empty)
79
-
80
- RUBY_EXTCONF_H =
81
- cflags = -fdeclspec $(optflags) $(debugflags) $(warnflags)
82
- cxxflags =
83
- optflags = -O3
84
- debugflags = -ggdb3
85
- warnflags = -Wall -Wextra -Wextra-tokens -Wdeprecated-declarations -Wdivision-by-zero -Wdiv-by-zero -Wimplicit-function-declaration -Wimplicit-int -Wmisleading-indentation -Wpointer-arith -Wshorten-64-to-32 -Wwrite-strings -Wold-style-definition -Wmissing-noreturn -Wno-cast-function-type -Wno-constant-logical-operand -Wno-long-long -Wno-missing-field-initializers -Wno-overlength-strings -Wno-parentheses-equality -Wno-self-assign -Wno-tautological-compare -Wno-unused-parameter -Wno-unused-value -Wunused-variable -Wundef
86
- cppflags =
87
- CCDLFLAGS = -fno-common
88
- CFLAGS = $(CCDLFLAGS) -O3 -I/opt/homebrew/opt/libyaml/include -I/opt/homebrew/opt/libksba/include -I/opt/homebrew/opt/readline/include -I/opt/homebrew/opt/zlib/include -I/opt/homebrew/opt/openssl@1.1/include $(cflags) -fno-common -pipe $(ARCH_FLAG)
89
- INCFLAGS = -I. -I$(arch_hdrdir) -I$(hdrdir)/ruby/backward -I$(hdrdir) -I$(srcdir)
90
- DEFS =
91
- CPPFLAGS = -D_XOPEN_SOURCE -D_DARWIN_C_SOURCE -D_DARWIN_UNLIMITED_SELECT -D_REENTRANT $(DEFS) $(cppflags)
92
- CXXFLAGS = $(CCDLFLAGS) -fdeclspec $(ARCH_FLAG)
93
- ldflags = -L. -L/opt/homebrew/opt/libyaml/lib -L/opt/homebrew/opt/libksba/lib -L/opt/homebrew/opt/readline/lib -L/opt/homebrew/opt/zlib/lib -L/opt/homebrew/opt/openssl@1.1/lib -fstack-protector-strong
94
- dldflags = -L/opt/homebrew/opt/libyaml/lib -L/opt/homebrew/opt/libksba/lib -L/opt/homebrew/opt/readline/lib -L/opt/homebrew/opt/zlib/lib -L/opt/homebrew/opt/openssl@1.1/lib -Wl,-undefined,dynamic_lookup $(LIBRUBYARG_SHARED)
95
- ARCH_FLAG =
96
- DLDFLAGS = $(ldflags) $(dldflags) $(ARCH_FLAG)
97
- LDSHARED = $(CC) -dynamic -bundle
98
- LDSHAREDXX = $(CXX) -dynamic -bundle
99
- AR = ar
100
- EXEEXT =
101
-
102
- RUBY_INSTALL_NAME = $(RUBY_BASE_NAME)
103
- RUBY_SO_NAME = ruby.3.2
104
- RUBYW_INSTALL_NAME =
105
- RUBY_VERSION_NAME = $(RUBY_BASE_NAME)-$(ruby_version)
106
- RUBYW_BASE_NAME = rubyw
107
- RUBY_BASE_NAME = ruby
108
-
109
- arch = arm64-darwin23
110
- sitearch = $(arch)
111
- ruby_version = 3.2.0
112
- ruby = $(bindir)/$(RUBY_BASE_NAME)
113
- RUBY = $(ruby)
114
- BUILTRUBY = $(bindir)/$(RUBY_BASE_NAME)
115
- ruby_headers = $(hdrdir)/ruby.h $(hdrdir)/ruby/backward.h $(hdrdir)/ruby/ruby.h $(hdrdir)/ruby/defines.h $(hdrdir)/ruby/missing.h $(hdrdir)/ruby/intern.h $(hdrdir)/ruby/st.h $(hdrdir)/ruby/subst.h $(arch_hdrdir)/ruby/config.h
116
-
117
- RM = rm -f
118
- RM_RF = rm -fr
119
- RMDIRS = rmdir -p
120
- MAKEDIRS = /opt/homebrew/opt/coreutils/bin/gmkdir -p
121
- INSTALL = /opt/homebrew/opt/coreutils/bin/ginstall -c
122
- INSTALL_PROG = $(INSTALL) -m 0755
123
- INSTALL_DATA = $(INSTALL) -m 644
124
- COPY = cp
125
- TOUCH = exit >
126
-
127
- #### End of system configuration section. ####
128
-
129
- preload =
130
- libpath = . $(libdir)
131
- LIBPATH = -L. -L$(libdir)
132
- DEFFILE =
133
-
134
- CLEANFILES = mkmf.log
135
- DISTCLEANFILES =
136
- DISTCLEANDIRS =
137
-
138
- extout =
139
- extout_prefix =
140
- target_prefix = /smarter_csv
141
- LOCAL_LIBS =
142
- LIBS = $(LIBRUBYARG_SHARED) -lpthread
143
- ORIG_SRCS = smarter_csv.c
144
- SRCS = $(ORIG_SRCS)
145
- OBJS = smarter_csv.o
146
- HDRS =
147
- LOCAL_HDRS =
148
- TARGET = smarter_csv
149
- TARGET_NAME = smarter_csv
150
- TARGET_ENTRY = Init_$(TARGET_NAME)
151
- DLLIB = $(TARGET).bundle
152
- EXTSTATIC =
153
- STATIC_LIB =
154
-
155
- TIMESTAMP_DIR = .
156
- BINDIR = $(bindir)
157
- RUBYCOMMONDIR = $(sitedir)$(target_prefix)
158
- RUBYLIBDIR = $(sitelibdir)$(target_prefix)
159
- RUBYARCHDIR = $(sitearchdir)$(target_prefix)
160
- HDRDIR = $(sitehdrdir)$(target_prefix)
161
- ARCHHDRDIR = $(sitearchhdrdir)$(target_prefix)
162
- TARGET_SO_DIR =
163
- TARGET_SO = $(TARGET_SO_DIR)$(DLLIB)
164
- CLEANLIBS = $(TARGET_SO) $(TARGET_SO).dSYM
165
- CLEANOBJS = $(OBJS) *.bak
166
- TARGET_SO_DIR_TIMESTAMP = $(TIMESTAMP_DIR)/.sitearchdir.-.smarter_csv.time
167
-
168
- all: $(DLLIB)
169
- static: $(STATIC_LIB)
170
- .PHONY: all install static install-so install-rb
171
- .PHONY: clean clean-so clean-static clean-rb
172
-
173
- clean-static::
174
- clean-rb-default::
175
- clean-rb::
176
- clean-so::
177
- clean: clean-so clean-static clean-rb-default clean-rb
178
- -$(Q)$(RM_RF) $(CLEANLIBS) $(CLEANOBJS) $(CLEANFILES) .*.time
179
-
180
- distclean-rb-default::
181
- distclean-rb::
182
- distclean-so::
183
- distclean-static::
184
- distclean: clean distclean-so distclean-static distclean-rb-default distclean-rb
185
- -$(Q)$(RM) Makefile $(RUBY_EXTCONF_H) conftest.* mkmf.log
186
- -$(Q)$(RM) core ruby$(EXEEXT) *~ $(DISTCLEANFILES)
187
- -$(Q)$(RMDIRS) $(DISTCLEANDIRS) 2> /dev/null || true
188
-
189
- realclean: distclean
190
- install: install-so install-rb
191
-
192
- install-so: $(DLLIB) $(TARGET_SO_DIR_TIMESTAMP)
193
- $(INSTALL_PROG) $(DLLIB) $(RUBYARCHDIR)
194
- clean-static::
195
- -$(Q)$(RM) $(STATIC_LIB)
196
- install-rb: pre-install-rb do-install-rb install-rb-default
197
- install-rb-default: pre-install-rb-default do-install-rb-default
198
- pre-install-rb: Makefile
199
- pre-install-rb-default: Makefile
200
- do-install-rb:
201
- do-install-rb-default:
202
- pre-install-rb-default:
203
- @$(NULLCMD)
204
- $(TARGET_SO_DIR_TIMESTAMP):
205
- $(Q) $(MAKEDIRS) $(@D) $(RUBYARCHDIR)
206
- $(Q) $(TOUCH) $@
207
-
208
- site-install: site-install-so site-install-rb
209
- site-install-so: install-so
210
- site-install-rb: install-rb
211
-
212
- .SUFFIXES: .c .m .cc .mm .cxx .cpp .o .S
213
-
214
- .cc.o:
215
- $(ECHO) compiling $(<)
216
- $(Q) $(CXX) $(INCFLAGS) $(CPPFLAGS) $(CXXFLAGS) $(COUTFLAG)$@ -c $(CSRCFLAG)$<
217
-
218
- .cc.S:
219
- $(ECHO) translating $(<)
220
- $(Q) $(CXX) $(INCFLAGS) $(CPPFLAGS) $(CXXFLAGS) $(COUTFLAG)$@ -S $(CSRCFLAG)$<
221
-
222
- .mm.o:
223
- $(ECHO) compiling $(<)
224
- $(Q) $(CXX) $(INCFLAGS) $(CPPFLAGS) $(CXXFLAGS) $(COUTFLAG)$@ -c $(CSRCFLAG)$<
225
-
226
- .mm.S:
227
- $(ECHO) translating $(<)
228
- $(Q) $(CXX) $(INCFLAGS) $(CPPFLAGS) $(CXXFLAGS) $(COUTFLAG)$@ -S $(CSRCFLAG)$<
229
-
230
- .cxx.o:
231
- $(ECHO) compiling $(<)
232
- $(Q) $(CXX) $(INCFLAGS) $(CPPFLAGS) $(CXXFLAGS) $(COUTFLAG)$@ -c $(CSRCFLAG)$<
233
-
234
- .cxx.S:
235
- $(ECHO) translating $(<)
236
- $(Q) $(CXX) $(INCFLAGS) $(CPPFLAGS) $(CXXFLAGS) $(COUTFLAG)$@ -S $(CSRCFLAG)$<
237
-
238
- .cpp.o:
239
- $(ECHO) compiling $(<)
240
- $(Q) $(CXX) $(INCFLAGS) $(CPPFLAGS) $(CXXFLAGS) $(COUTFLAG)$@ -c $(CSRCFLAG)$<
241
-
242
- .cpp.S:
243
- $(ECHO) translating $(<)
244
- $(Q) $(CXX) $(INCFLAGS) $(CPPFLAGS) $(CXXFLAGS) $(COUTFLAG)$@ -S $(CSRCFLAG)$<
245
-
246
- .c.o:
247
- $(ECHO) compiling $(<)
248
- $(Q) $(CC) $(INCFLAGS) $(CPPFLAGS) $(CFLAGS) $(COUTFLAG)$@ -c $(CSRCFLAG)$<
249
-
250
- .c.S:
251
- $(ECHO) translating $(<)
252
- $(Q) $(CC) $(INCFLAGS) $(CPPFLAGS) $(CFLAGS) $(COUTFLAG)$@ -S $(CSRCFLAG)$<
253
-
254
- .m.o:
255
- $(ECHO) compiling $(<)
256
- $(Q) $(CC) $(INCFLAGS) $(CPPFLAGS) $(CFLAGS) $(COUTFLAG)$@ -c $(CSRCFLAG)$<
257
-
258
- .m.S:
259
- $(ECHO) translating $(<)
260
- $(Q) $(CC) $(INCFLAGS) $(CPPFLAGS) $(CFLAGS) $(COUTFLAG)$@ -S $(CSRCFLAG)$<
261
-
262
- $(TARGET_SO): $(OBJS) Makefile
263
- $(ECHO) linking shared-object smarter_csv/$(DLLIB)
264
- -$(Q)$(RM) $(@)
265
- $(Q) $(LDSHARED) -o $@ $(OBJS) $(LIBPATH) $(DLDFLAGS) $(LOCAL_LIBS) $(LIBS)
266
- $(Q) $(POSTLINK)
267
-
268
-
269
-
270
- $(OBJS): $(HDRS) $(ruby_headers)