herringbone 0.1.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +198 -74
- data/bin/herringbone +38 -26
- data/lib/herringbone/active_record.rb +48 -19
- data/lib/herringbone/bloom_filter.rb +370 -0
- data/lib/herringbone/byte_values.rb +29 -0
- data/lib/herringbone/codecs/lz4.rb +66 -3
- data/lib/herringbone/codecs/snappy.rb +132 -3
- data/lib/herringbone/compression.rb +121 -16
- data/lib/herringbone/encodings/delta.rb +76 -19
- data/lib/herringbone/encodings/plain.rb +29 -7
- data/lib/herringbone/encodings/rle.rb +55 -3
- data/lib/herringbone/format.rb +203 -0
- data/lib/herringbone/inspector.rb +1784 -0
- data/lib/herringbone/io_buffer_support.rb +3 -1
- data/lib/herringbone/reader/column_chunk_reader.rb +388 -0
- data/lib/herringbone/reader/column_cursor.rb +225 -0
- data/lib/herringbone/reader/numo.rb +365 -0
- data/lib/herringbone/reader/page_stream.rb +615 -0
- data/lib/herringbone/reader/scan.rb +350 -0
- data/lib/herringbone/reader.rb +585 -272
- data/lib/herringbone/schema.rb +326 -90
- data/lib/herringbone/thrift.rb +155 -4
- data/lib/herringbone/types.rb +177 -42
- data/lib/herringbone/version.rb +2 -1
- data/lib/herringbone/visualizer.rb +1136 -0
- data/lib/herringbone/writer.rb +550 -105
- data/lib/herringbone/xxhash.rb +425 -0
- data/lib/herringbone.rb +64 -9
- metadata +14 -33
data/lib/herringbone/types.rb
CHANGED
|
@@ -21,19 +21,44 @@ module Herringbone
|
|
|
21
21
|
module Types
|
|
22
22
|
module_function
|
|
23
23
|
|
|
24
|
+
# Shorthand for Format::Type
|
|
24
25
|
T = Format::Type
|
|
26
|
+
# Shorthand for Format::ConvertedType
|
|
25
27
|
C = Format::ConvertedType
|
|
26
28
|
|
|
29
|
+
# Julian day number of 1970-01-01, the origin of DATE columns
|
|
27
30
|
EPOCH_JD = Date.new(1970, 1, 1).jd
|
|
31
|
+
# Julian day number of 1970-01-01, the origin of the day half of an INT96 timestamp
|
|
28
32
|
JULIAN_EPOCH_DAY = 2_440_588 # Julian day number of 1970-01-01, used by INT96
|
|
33
|
+
# Nanoseconds in a day, used to split INT96 timestamps into day and nanoseconds of day
|
|
29
34
|
NANOS_PER_DAY = 86_400 * 1_000_000_000
|
|
30
35
|
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
36
|
+
# Ticks per second for each TimeUnit
|
|
37
|
+
UNIT_DIVISORS = {millis: 1_000, micros: 1_000_000, nanos: 1_000_000_000}.freeze
|
|
38
|
+
# Sub-second unit names understood by +Time.at+, for each TimeUnit
|
|
39
|
+
UNIT_NAMES = {millis: :millisecond, micros: :microsecond, nanos: :nanosecond}.freeze
|
|
40
|
+
|
|
41
|
+
# Shorthand for building a LogicalType union.
|
|
42
|
+
# @param kw [Hash{Symbol => Thrift::Struct}] the single union member to set; any LogicalType
|
|
43
|
+
# field is accepted, those used in this module are listed below
|
|
44
|
+
# @option kw [Format::StringType] :string
|
|
45
|
+
# @option kw [Format::EnumType] :enum
|
|
46
|
+
# @option kw [Format::JsonType] :json
|
|
47
|
+
# @option kw [Format::BsonType] :bson
|
|
48
|
+
# @option kw [Format::UUIDType] :uuid
|
|
49
|
+
# @option kw [Format::DateType] :date
|
|
50
|
+
# @option kw [Format::Float16Type] :float16
|
|
51
|
+
# @option kw [Format::IntType] :integer bit width and signedness
|
|
52
|
+
# @option kw [Format::DecimalType] :decimal scale and precision
|
|
53
|
+
# @option kw [Format::TimeType] :time unit and UTC adjustment
|
|
54
|
+
# @option kw [Format::TimestampType] :timestamp unit and UTC adjustment
|
|
55
|
+
# @return [Format::LogicalType]
|
|
35
56
|
def lt(**kw) = Format::LogicalType.new(**kw)
|
|
36
57
|
|
|
58
|
+
# TimeUnit union for a unit name.
|
|
59
|
+
# @param unit [Symbol, String] +:millis+/+:ms+, +:micros+/+:us+ or +:nanos+/+:ns+
|
|
60
|
+
# @return [Format::TimeUnit]
|
|
61
|
+
# @raise [ArgumentError] for any other unit
|
|
37
62
|
def time_unit(unit)
|
|
38
63
|
case unit.to_sym
|
|
39
64
|
when :millis, :ms then Format::TimeUnit.millis
|
|
@@ -43,64 +68,84 @@ module Herringbone
|
|
|
43
68
|
end
|
|
44
69
|
end
|
|
45
70
|
|
|
71
|
+
# Physical attributes of an annotated integer column, with both the INTEGER logical type
|
|
72
|
+
# and the matching INT_n/UINT_n converted type.
|
|
73
|
+
# @param bits [Integer] bit width: 8, 16, 32 or 64 (64 is stored as INT64, the rest as INT32)
|
|
74
|
+
# @param signed [Boolean] whether the integers are signed
|
|
75
|
+
# @return [Hash{Symbol => Object}] +:type+, +:converted_type+ and +:logical_type+
|
|
46
76
|
def int_type(bits, signed)
|
|
47
|
-
physical = bits == 64 ? T::INT64 : T::INT32
|
|
77
|
+
physical = (bits == 64) ? T::INT64 : T::INT32
|
|
48
78
|
converted = signed ? C.const_get("INT_#{bits}") : C.const_get("UINT_#{bits}")
|
|
49
|
-
{
|
|
50
|
-
|
|
79
|
+
{type: physical, converted_type: converted,
|
|
80
|
+
logical_type: lt(integer: Format::IntType.new(bit_width: bits, is_signed: signed))}
|
|
51
81
|
end
|
|
52
82
|
|
|
53
83
|
# Physical attributes (type, logical type etc.) for a DSL type name
|
|
84
|
+
# @param type [Symbol, String] column type name, e.g. +:string+, +:int64+, +:decimal+, +:timestamp+
|
|
85
|
+
# @param opts [Hash{Symbol => Object}] type-specific options
|
|
86
|
+
# @option opts [Boolean] :parquet_enum for +:enum+, annotate as ENUM instead of STRING
|
|
87
|
+
# @option opts [Integer] :length byte length for +:fixed+ (required)
|
|
88
|
+
# @option opts [Symbol] :unit (:micros) for +:time+ and +:timestamp+, see {time_unit}
|
|
89
|
+
# @option opts [Boolean] :utc (true) for +:time+ and +:timestamp+, sets +is_adjusted_to_utc+;
|
|
90
|
+
# the legacy converted type is only written when true
|
|
91
|
+
# @option opts [Integer] :precision number of decimal digits for +:decimal+ (required)
|
|
92
|
+
# @option opts [Integer] :scale (0) digits after the decimal point for +:decimal+
|
|
93
|
+
# @option opts [Symbol] :physical for +:decimal+, force +:int32+, +:int64+, +:binary+ or
|
|
94
|
+
# +:fixed+ storage instead of picking the smallest by precision
|
|
95
|
+
# @return [Hash{Symbol => Object}] keyword arguments for Schema::Node.new: +:type+ and,
|
|
96
|
+
# where applicable, +:type_length+, +:converted_type+, +:logical_type+, +:scale+, +:precision+
|
|
97
|
+
# @raise [ArgumentError] for an unknown type, unit, or invalid decimal precision/scale
|
|
98
|
+
# @raise [KeyError] when a required option is missing or +:physical+ is not recognized
|
|
54
99
|
def physical_attributes(type, **opts)
|
|
55
100
|
case type.to_sym
|
|
56
|
-
when :boolean
|
|
101
|
+
when :boolean then {type: T::BOOLEAN}
|
|
57
102
|
when :int8 then int_type(8, true)
|
|
58
103
|
when :int16 then int_type(16, true)
|
|
59
|
-
when :int32 then {
|
|
60
|
-
when :int64 then {
|
|
104
|
+
when :int32 then {type: T::INT32}
|
|
105
|
+
when :int64 then {type: T::INT64}
|
|
61
106
|
when :uint8 then int_type(8, false)
|
|
62
107
|
when :uint16 then int_type(16, false)
|
|
63
108
|
when :uint32 then int_type(32, false)
|
|
64
109
|
when :uint64 then int_type(64, false)
|
|
65
|
-
when :float then {
|
|
66
|
-
when :double then {
|
|
67
|
-
when :float16 then {
|
|
68
|
-
when :string
|
|
69
|
-
when :binary
|
|
70
|
-
when :json then {
|
|
71
|
-
when :bson then {
|
|
110
|
+
when :float then {type: T::FLOAT}
|
|
111
|
+
when :double then {type: T::DOUBLE}
|
|
112
|
+
when :float16 then {type: T::FIXED_LEN_BYTE_ARRAY, type_length: 2, logical_type: lt(float16: Format::Float16Type.new)}
|
|
113
|
+
when :string then {type: T::BYTE_ARRAY, converted_type: C::UTF8, logical_type: lt(string: Format::StringType.new)}
|
|
114
|
+
when :binary then {type: T::BYTE_ARRAY}
|
|
115
|
+
when :json then {type: T::BYTE_ARRAY, converted_type: C::JSON, logical_type: lt(json: Format::JsonType.new)}
|
|
116
|
+
when :bson then {type: T::BYTE_ARRAY, converted_type: C::BSON, logical_type: lt(bson: Format::BsonType.new)}
|
|
72
117
|
when :enum
|
|
73
118
|
if opts[:parquet_enum]
|
|
74
|
-
{
|
|
119
|
+
{type: T::BYTE_ARRAY, converted_type: C::ENUM, logical_type: lt(enum: Format::EnumType.new)}
|
|
75
120
|
else
|
|
76
121
|
physical_attributes(:string)
|
|
77
122
|
end
|
|
78
|
-
when :uuid then {
|
|
79
|
-
when :date then {
|
|
80
|
-
when :int96 then {
|
|
123
|
+
when :uuid then {type: T::FIXED_LEN_BYTE_ARRAY, type_length: 16, logical_type: lt(uuid: Format::UUIDType.new)}
|
|
124
|
+
when :date then {type: T::INT32, converted_type: C::DATE, logical_type: lt(date: Format::DateType.new)}
|
|
125
|
+
when :int96 then {type: T::INT96}
|
|
81
126
|
when :fixed
|
|
82
|
-
{
|
|
127
|
+
{type: T::FIXED_LEN_BYTE_ARRAY, type_length: Integer(opts.fetch(:length))}
|
|
83
128
|
when :time
|
|
84
129
|
unit = time_unit(opts.fetch(:unit, :micros))
|
|
85
|
-
converted = {
|
|
86
|
-
{
|
|
87
|
-
|
|
130
|
+
converted = {millis: C::TIME_MILLIS, micros: C::TIME_MICROS}[unit.to_sym]
|
|
131
|
+
{type: unit.millis ? T::INT32 : T::INT64, converted_type: opts.fetch(:utc, true) ? converted : nil,
|
|
132
|
+
logical_type: lt(time: Format::TimeType.new(is_adjusted_to_utc: opts.fetch(:utc, true), unit: unit))}
|
|
88
133
|
when :timestamp
|
|
89
134
|
unit = time_unit(opts.fetch(:unit, :micros))
|
|
90
|
-
converted = {
|
|
91
|
-
{
|
|
92
|
-
|
|
135
|
+
converted = {millis: C::TIMESTAMP_MILLIS, micros: C::TIMESTAMP_MICROS}[unit.to_sym]
|
|
136
|
+
{type: T::INT64, converted_type: opts.fetch(:utc, true) ? converted : nil,
|
|
137
|
+
logical_type: lt(timestamp: Format::TimestampType.new(is_adjusted_to_utc: opts.fetch(:utc, true), unit: unit))}
|
|
93
138
|
when :decimal
|
|
94
139
|
precision = Integer(opts.fetch(:precision))
|
|
95
140
|
scale = Integer(opts.fetch(:scale, 0))
|
|
96
141
|
raise ArgumentError, "Decimal precision must be positive" unless precision.positive?
|
|
97
142
|
raise ArgumentError, "Decimal scale must be between 0 and precision" unless scale.between?(0, precision)
|
|
98
143
|
physical = if opts[:physical]
|
|
99
|
-
{
|
|
100
|
-
|
|
101
|
-
elsif precision <= 9 then {
|
|
102
|
-
elsif precision <= 18 then {
|
|
103
|
-
else {
|
|
144
|
+
{int32: {type: T::INT32}, int64: {type: T::INT64}, binary: {type: T::BYTE_ARRAY},
|
|
145
|
+
fixed: {type: T::FIXED_LEN_BYTE_ARRAY, type_length: decimal_bytes(precision)}}.fetch(opts[:physical])
|
|
146
|
+
elsif precision <= 9 then {type: T::INT32}
|
|
147
|
+
elsif precision <= 18 then {type: T::INT64}
|
|
148
|
+
else {type: T::FIXED_LEN_BYTE_ARRAY, type_length: decimal_bytes(precision)}
|
|
104
149
|
end
|
|
105
150
|
physical.merge(converted_type: C::DECIMAL, scale: scale, precision: precision,
|
|
106
151
|
logical_type: lt(decimal: Format::DecimalType.new(scale: scale, precision: precision)))
|
|
@@ -110,6 +155,8 @@ module Herringbone
|
|
|
110
155
|
end
|
|
111
156
|
|
|
112
157
|
# Minimal number of bytes to hold a signed integer of +precision+ decimal digits
|
|
158
|
+
# @param precision [Integer] number of decimal digits
|
|
159
|
+
# @return [Integer] byte length for a FIXED_LEN_BYTE_ARRAY decimal
|
|
113
160
|
def decimal_bytes(precision)
|
|
114
161
|
max = 10**precision
|
|
115
162
|
n = 1
|
|
@@ -118,6 +165,17 @@ module Herringbone
|
|
|
118
165
|
end
|
|
119
166
|
|
|
120
167
|
# Normalized logical kind of a node: [symbol, details]
|
|
168
|
+
#
|
|
169
|
+
# The LogicalType wins when present; otherwise the legacy converted type is mapped onto
|
|
170
|
+
# the same shapes. Details depend on the kind:
|
|
171
|
+
# [:integer, bit_width, signed]
|
|
172
|
+
# [:decimal, scale, precision]
|
|
173
|
+
# [:timestamp, unit, adjusted_to_utc] (unit is :millis, :micros or :nanos)
|
|
174
|
+
# [:time, unit, adjusted_to_utc]
|
|
175
|
+
# [kind] for other annotations (:string, :date, :uuid ...)
|
|
176
|
+
# [nil] for unannotated columns
|
|
177
|
+
# @param node [Schema::Node] leaf node of the physical schema
|
|
178
|
+
# @return [Array] kind Symbol (or nil) followed by its details
|
|
121
179
|
def logical_of(node)
|
|
122
180
|
if (kind = node.logical_type&.kind)
|
|
123
181
|
name, payload = kind
|
|
@@ -153,6 +211,8 @@ module Herringbone
|
|
|
153
211
|
end
|
|
154
212
|
|
|
155
213
|
# Returns a lambda converting a physical value into a Ruby value, or nil when no conversion is needed.
|
|
214
|
+
# @param node [Schema::Node] leaf node of the physical schema
|
|
215
|
+
# @return [Proc, nil] one-argument converter, or nil when decoded values are used as-is
|
|
156
216
|
def reader_for(node)
|
|
157
217
|
kind, a, b = logical_of(node)
|
|
158
218
|
type = node.type
|
|
@@ -161,7 +221,7 @@ module Herringbone
|
|
|
161
221
|
return ->(v) { v.force_encoding(Encoding::UTF_8) } if type == T::BYTE_ARRAY || type == T::FIXED_LEN_BYTE_ARRAY
|
|
162
222
|
when :integer
|
|
163
223
|
if !b && (type == T::INT32 || type == T::INT64)
|
|
164
|
-
mask = type == T::INT32 ? 0xFFFF_FFFF : 0xFFFF_FFFF_FFFF_FFFF
|
|
224
|
+
mask = (type == T::INT32) ? 0xFFFF_FFFF : 0xFFFF_FFFF_FFFF_FFFF
|
|
165
225
|
return ->(v) { v & mask }
|
|
166
226
|
end
|
|
167
227
|
when :date
|
|
@@ -179,12 +239,18 @@ module Herringbone
|
|
|
179
239
|
nil
|
|
180
240
|
end
|
|
181
241
|
|
|
242
|
+
# Converter from an INT64 timestamp to a UTC Time.
|
|
243
|
+
# @param unit [Symbol] +:millis+, +:micros+ or +:nanos+
|
|
244
|
+
# @return [Proc] lambda taking an Integer count of +unit+ since the epoch
|
|
245
|
+
# @raise [KeyError] for an unknown unit
|
|
182
246
|
def timestamp_reader(unit)
|
|
183
247
|
div = UNIT_DIVISORS.fetch(unit)
|
|
184
248
|
name = UNIT_NAMES.fetch(unit)
|
|
185
249
|
->(v) { Time.at(v / div, v % div, name).utc }
|
|
186
250
|
end
|
|
187
251
|
|
|
252
|
+
# Converter from a decoded INT96 value to a UTC Time.
|
|
253
|
+
# @return [Proc] lambda taking a +[nanoseconds_of_day, julian_day]+ pair
|
|
188
254
|
def int96_reader
|
|
189
255
|
lambda do |(nanos, day)|
|
|
190
256
|
secs = (day - JULIAN_EPOCH_DAY) * 86_400
|
|
@@ -192,6 +258,11 @@ module Herringbone
|
|
|
192
258
|
end
|
|
193
259
|
end
|
|
194
260
|
|
|
261
|
+
# Converter from a stored unscaled decimal to a BigDecimal.
|
|
262
|
+
# @param type [Integer] physical type (Format::Type) of the column
|
|
263
|
+
# @param scale [Integer] digits after the decimal point
|
|
264
|
+
# @return [Proc, nil] lambda taking an Integer (INT32/INT64) or big-endian two's complement
|
|
265
|
+
# bytes (BYTE_ARRAY/FIXED_LEN_BYTE_ARRAY), or nil for any other physical type
|
|
195
266
|
def decimal_reader(type, scale)
|
|
196
267
|
to_decimal = scale.zero? ? ->(i) { BigDecimal(i) } : ->(i) { BigDecimal("#{i}e-#{scale}") }
|
|
197
268
|
case type
|
|
@@ -202,13 +273,20 @@ module Herringbone
|
|
|
202
273
|
end
|
|
203
274
|
|
|
204
275
|
# Big-endian two's complement bytes to Integer
|
|
276
|
+
# @param bytes [String] binary string; empty decodes as 0
|
|
277
|
+
# @return [Integer]
|
|
205
278
|
def be_to_int(bytes)
|
|
206
279
|
return 0 if bytes.empty?
|
|
207
280
|
i = bytes.unpack1("H*").to_i(16)
|
|
208
281
|
bits = bytes.bytesize * 8
|
|
209
|
-
i >= (1 << (bits - 1)) ? i - (1 << bits) : i
|
|
282
|
+
(i >= (1 << (bits - 1))) ? i - (1 << bits) : i
|
|
210
283
|
end
|
|
211
284
|
|
|
285
|
+
# Integer to big-endian two's complement bytes of a fixed width
|
|
286
|
+
# @param i [Integer] value to encode
|
|
287
|
+
# @param nbytes [Integer] output width in bytes
|
|
288
|
+
# @return [String] binary string of +nbytes+ bytes
|
|
289
|
+
# @raise [EncodeError] when +i+ does not fit in +nbytes+ bytes
|
|
212
290
|
def int_to_be(i, nbytes)
|
|
213
291
|
bits = nbytes * 8
|
|
214
292
|
raise EncodeError, "Decimal value #{i} does not fit in #{nbytes} bytes" unless i.bit_length < bits
|
|
@@ -216,6 +294,9 @@ module Herringbone
|
|
|
216
294
|
[i.to_s(16).rjust(nbytes * 2, "0")].pack("H*")
|
|
217
295
|
end
|
|
218
296
|
|
|
297
|
+
# Decodes an IEEE 754 half-precision value, including subnormals, infinities and NaN.
|
|
298
|
+
# @param h [Integer] 16-bit pattern
|
|
299
|
+
# @return [Float]
|
|
219
300
|
def half_to_float(h)
|
|
220
301
|
sign = (h >> 15).zero? ? 1.0 : -1.0
|
|
221
302
|
exp = (h >> 10) & 0x1F
|
|
@@ -231,9 +312,11 @@ module Herringbone
|
|
|
231
312
|
|
|
232
313
|
# Rounds a Float to the nearest half-precision value (ties to even). Every step is exact
|
|
233
314
|
# in double arithmetic, so there is no double rounding.
|
|
315
|
+
# @param f [Float] value to round; overflows to infinity, NaN becomes the canonical quiet NaN
|
|
316
|
+
# @return [Integer] 16-bit pattern
|
|
234
317
|
def float_to_half(f)
|
|
235
318
|
return 0x7E00 if f.nan?
|
|
236
|
-
sign = f.negative? || (f.zero? && (1.0 / f).negative?) ? 0x8000 : 0
|
|
319
|
+
sign = (f.negative? || (f.zero? && (1.0 / f).negative?)) ? 0x8000 : 0
|
|
237
320
|
a = f.abs
|
|
238
321
|
return sign | 0x7C00 if a >= 65_520.0
|
|
239
322
|
return sign | (a * 2.0**24).round(half: :even) if a < 2.0**-14
|
|
@@ -252,13 +335,17 @@ module Herringbone
|
|
|
252
335
|
# date: Date, Time/DateTime (its calendar date), ISO-8601 String, Integer days since epoch
|
|
253
336
|
# timestamp: Time, DateTime, ActiveSupport::TimeWithZone, Date (midnight UTC), ISO-8601 String,
|
|
254
337
|
# Integer in the column's unit
|
|
255
|
-
# time: Time (its time of day), "HH:MM:SS[.fraction]" String, Integer in the column's unit
|
|
338
|
+
# time: Time (its time of day), "HH:MM[:SS[.fraction]]" String, Integer in the column's unit
|
|
256
339
|
# json: String (used as-is) or any other object (serialized with JSON.generate)
|
|
257
340
|
# string: String, Symbol or anything responding to to_s
|
|
258
341
|
# boolean: true/false, 1/0, "true"/"false", "t"/"f", "1"/"0", "yes"/"no"
|
|
259
342
|
# integers: Integer, or a Float/BigDecimal/Rational/String holding a whole number
|
|
260
343
|
# decimal: BigDecimal, Integer, Rational, Float or numeric String
|
|
261
344
|
# uuid: String with or without dashes, or 16 raw bytes
|
|
345
|
+
# The returned lambda raises ArgumentError or RangeError for values it cannot convert.
|
|
346
|
+
# @param node [Schema::Node] leaf node of the physical schema
|
|
347
|
+
# @return [Proc, nil] one-argument converter; nil only for a column with no recognized
|
|
348
|
+
# physical type
|
|
262
349
|
def writer_for(node)
|
|
263
350
|
kind, a, b = logical_of(node)
|
|
264
351
|
type = node.type
|
|
@@ -279,15 +366,15 @@ module Herringbone
|
|
|
279
366
|
return decimal_writer(node, a)
|
|
280
367
|
when :uuid
|
|
281
368
|
return fixed_checker(16) do |v|
|
|
282
|
-
v.bytesize == 16 && v.encoding == Encoding::BINARY ? v : [v.to_s.delete("-")].pack("H*")
|
|
369
|
+
(v.bytesize == 16 && v.encoding == Encoding::BINARY) ? v : [v.to_s.delete("-")].pack("H*")
|
|
283
370
|
end
|
|
284
371
|
when :float16
|
|
285
372
|
return ->(v) { [float_to_half(Float(v))].pack("v") }
|
|
286
373
|
when :integer
|
|
287
374
|
if !b && (type == T::INT32 || type == T::INT64)
|
|
288
|
-
bits = type == T::INT32 ? 32 : 64
|
|
375
|
+
bits = (type == T::INT32) ? 32 : 64
|
|
289
376
|
check = int_checker(0, (1 << a) - 1)
|
|
290
|
-
return ->(v) { Delta.wrap(check.call(v), bits) }
|
|
377
|
+
return ->(v) { Encodings::Delta.wrap(check.call(v), bits) }
|
|
291
378
|
elsif type == T::INT32 || type == T::INT64
|
|
292
379
|
return int_checker(-(1 << (a - 1)), (1 << (a - 1)) - 1)
|
|
293
380
|
end
|
|
@@ -311,19 +398,29 @@ module Herringbone
|
|
|
311
398
|
end
|
|
312
399
|
end
|
|
313
400
|
|
|
401
|
+
# Values accepted for a boolean column (strings are also matched case-insensitively)
|
|
314
402
|
BOOLEANS = {
|
|
315
403
|
true => true, false => false, 1 => true, 0 => false,
|
|
316
404
|
"true" => true, "false" => false, "t" => true, "f" => false, "1" => true, "0" => false,
|
|
317
405
|
"yes" => true, "no" => false, "TRUE" => true, "FALSE" => false, "T" => true, "F" => false
|
|
318
406
|
}.freeze
|
|
319
407
|
|
|
408
|
+
# Coerces a value to true/false using BOOLEANS.
|
|
409
|
+
# @param v [Object] true/false, 1/0, or a String/Symbol such as "yes" or :false
|
|
410
|
+
# @return [Boolean]
|
|
411
|
+
# @raise [ArgumentError] when +v+ is not recognized as a boolean
|
|
320
412
|
def to_boolean(v)
|
|
321
413
|
BOOLEANS.fetch(v) do
|
|
322
|
-
s = v.is_a?(String) || v.is_a?(Symbol) ? v.to_s.downcase : nil
|
|
414
|
+
s = (v.is_a?(String) || v.is_a?(Symbol)) ? v.to_s.downcase : nil
|
|
323
415
|
BOOLEANS.fetch(s) { raise ArgumentError, "expected a boolean, got #{v.inspect}" }
|
|
324
416
|
end
|
|
325
417
|
end
|
|
326
418
|
|
|
419
|
+
# Days since the Unix epoch in the proleptic Gregorian calendar, as stored in DATE columns.
|
|
420
|
+
# @param v [Date, String, Integer, #to_date] a Date, an ISO-8601 date String, an Integer
|
|
421
|
+
# (returned as-is) or anything responding to +to_date+ (Time, DateTime)
|
|
422
|
+
# @return [Integer]
|
|
423
|
+
# @raise [ArgumentError] when +v+ cannot be turned into a Date
|
|
327
424
|
def date_to_days(v)
|
|
328
425
|
return v if v.is_a?(Integer)
|
|
329
426
|
d = case v
|
|
@@ -338,6 +435,10 @@ module Herringbone
|
|
|
338
435
|
end
|
|
339
436
|
|
|
340
437
|
# Converts the values a timestamp column accepts into a Time (or Time-like) object
|
|
438
|
+
# @param v [Time, DateTime, Date, String, #to_i] a Time, DateTime, Date (taken as midnight UTC),
|
|
439
|
+
# ISO-8601 (or +Time.parse+-able) String, or a Time-like object responding to +to_i+ and +nsec+
|
|
440
|
+
# @return [Time, Object] a Time, or +v+ itself when it is Time-like
|
|
441
|
+
# @raise [ArgumentError] when +v+ is none of the above or the String cannot be parsed
|
|
341
442
|
def to_time(v)
|
|
342
443
|
case v
|
|
343
444
|
when Time then v
|
|
@@ -356,6 +457,13 @@ module Herringbone
|
|
|
356
457
|
end
|
|
357
458
|
end
|
|
358
459
|
|
|
460
|
+
# Converter from a timestamp-like value to an INT64 count of +unit+ since the epoch.
|
|
461
|
+
# Integers pass through unchanged; sub-unit precision is truncated.
|
|
462
|
+
# @param unit [Symbol] +:millis+, +:micros+ or +:nanos+
|
|
463
|
+
# @param utc [Boolean] whether the column is adjusted to UTC; when false the local wall
|
|
464
|
+
# clock time of the value (its UTC offset added) is stored
|
|
465
|
+
# @return [Proc] one-argument lambda, see {to_time} for accepted values
|
|
466
|
+
# @raise [KeyError] for an unknown unit
|
|
359
467
|
def timestamp_writer(unit, utc)
|
|
360
468
|
mult = UNIT_DIVISORS.fetch(unit)
|
|
361
469
|
lambda do |v|
|
|
@@ -368,6 +476,13 @@ module Herringbone
|
|
|
368
476
|
end
|
|
369
477
|
end
|
|
370
478
|
|
|
479
|
+
# Converter from a time of day to a count of +unit+ since midnight.
|
|
480
|
+
# The lambda accepts an Integer (returned as-is), an "HH:MM[:SS[.fraction]]" String or
|
|
481
|
+
# anything responding to +hour+ and +nsec+ (Time, DateTime), and raises ArgumentError or
|
|
482
|
+
# RangeError for anything else or a time past 23:59:59.
|
|
483
|
+
# @param unit [Symbol] +:millis+, +:micros+ or +:nanos+
|
|
484
|
+
# @return [Proc] one-argument lambda
|
|
485
|
+
# @raise [KeyError] for an unknown unit
|
|
371
486
|
def time_writer(unit)
|
|
372
487
|
mult = UNIT_DIVISORS.fetch(unit)
|
|
373
488
|
lambda do |v|
|
|
@@ -389,6 +504,9 @@ module Herringbone
|
|
|
389
504
|
|
|
390
505
|
# String column restricted to a set of values. +values+ is an Array of labels, or a Hash of
|
|
391
506
|
# label => stored value like Rails' `Model.statuses`, in which case either is accepted.
|
|
507
|
+
# Symbols are accepted in place of String labels.
|
|
508
|
+
# @param values [Array<String, Symbol>, Hash{String, Symbol => Object}] allowed labels
|
|
509
|
+
# @return [Proc] lambda returning the String label, raising ArgumentError for any other value
|
|
392
510
|
def enum_writer(values)
|
|
393
511
|
labels = {}
|
|
394
512
|
if values.is_a?(Hash)
|
|
@@ -407,15 +525,24 @@ module Herringbone
|
|
|
407
525
|
end
|
|
408
526
|
|
|
409
527
|
# Converts to Integer, rejecting fractional numbers and values outside min..max
|
|
528
|
+
# @param min [Integer] smallest accepted value
|
|
529
|
+
# @param max [Integer] largest accepted value
|
|
530
|
+
# @return [Proc] lambda raising ArgumentError for non-integers and RangeError when out of range
|
|
410
531
|
def int_checker(min, max)
|
|
411
532
|
lambda do |v|
|
|
412
533
|
i = Integer(v)
|
|
413
534
|
raise ArgumentError, "#{v.inspect} is not an integer" unless v.is_a?(Integer) || !v.is_a?(Numeric) || v == i
|
|
414
|
-
raise RangeError, "#{i} is outside #{min}..#{max}" unless i
|
|
535
|
+
raise RangeError, "#{i} is outside #{min}..#{max}" unless i.between?(min, max)
|
|
415
536
|
i
|
|
416
537
|
end
|
|
417
538
|
end
|
|
418
539
|
|
|
540
|
+
# Wraps a conversion to a String and checks the result has exactly +length+ bytes.
|
|
541
|
+
# @param length [Integer] required byte length
|
|
542
|
+
# @yield [v] converts the incoming value
|
|
543
|
+
# @yieldparam v [Object] value handed to the writer
|
|
544
|
+
# @yieldreturn [String] bytes to store
|
|
545
|
+
# @return [Proc] lambda raising ArgumentError when the converted String has another length
|
|
419
546
|
def fixed_checker(length, &convert)
|
|
420
547
|
lambda do |v|
|
|
421
548
|
s = convert.call(v)
|
|
@@ -424,6 +551,14 @@ module Herringbone
|
|
|
424
551
|
end
|
|
425
552
|
end
|
|
426
553
|
|
|
554
|
+
# Converter from a numeric value to the stored unscaled decimal. Values are rounded
|
|
555
|
+
# (half away from zero) to +scale+ digits; the lambda raises RangeError when the result
|
|
556
|
+
# exceeds the column's precision.
|
|
557
|
+
# @param node [Schema::Node] DECIMAL leaf node, for its physical type, precision and type length
|
|
558
|
+
# @param scale [Integer] digits after the decimal point
|
|
559
|
+
# @return [Proc, nil] lambda returning an Integer (INT32/INT64) or big-endian two's complement
|
|
560
|
+
# bytes (FIXED_LEN_BYTE_ARRAY, BYTE_ARRAY with minimal length), or nil for any other
|
|
561
|
+
# physical type
|
|
427
562
|
def decimal_writer(node, scale)
|
|
428
563
|
mult = 10**scale
|
|
429
564
|
limit = node.precision ? 10**node.precision : nil
|