questdb 0.1.0-x64-mingw-ucrt
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/CHANGELOG.md +22 -0
- data/LICENSE.txt +201 -0
- data/README.md +170 -0
- data/SECURITY.md +19 -0
- data/THIRD_PARTY_NOTICES.md +242 -0
- data/docs/arrow.md +89 -0
- data/docs/configuration.md +61 -0
- data/docs/design.md +46 -0
- data/docs/ingestion.md +96 -0
- data/docs/performance.md +36 -0
- data/docs/python-parity.md +34 -0
- data/docs/querying.md +54 -0
- data/docs/release-infrastructure.md +70 -0
- data/docs/releasing.md +47 -0
- data/docs/troubleshooting.md +41 -0
- data/examples/basic.rb +32 -0
- data/examples/callbacks.rb +46 -0
- data/examples/columnar.rb +41 -0
- data/examples/independent_buffer.rb +39 -0
- data/examples/query.rb +28 -0
- data/examples/standalone.rb +33 -0
- data/examples/tls_auth.rb +33 -0
- data/lib/questdb/arrow.rb +250 -0
- data/lib/questdb/client.rb +308 -0
- data/lib/questdb/column_vector.rb +91 -0
- data/lib/questdb/config.rb +161 -0
- data/lib/questdb/connection_event.rb +60 -0
- data/lib/questdb/direct_ingestion.rb +440 -0
- data/lib/questdb/event_inbox.rb +60 -0
- data/lib/questdb/ingress/buffer.rb +104 -0
- data/lib/questdb/ingress/error.rb +43 -0
- data/lib/questdb/ingress/float64_array.rb +60 -0
- data/lib/questdb/ingress/native.rb +40 -0
- data/lib/questdb/ingress/timestamp.rb +53 -0
- data/lib/questdb/ingress.rb +77 -0
- data/lib/questdb/native/3.1/questdb_ingress.so +0 -0
- data/lib/questdb/native/3.2/questdb_ingress.so +0 -0
- data/lib/questdb/native/3.3/questdb_ingress.so +0 -0
- data/lib/questdb/native/3.4/questdb_ingress.so +0 -0
- data/lib/questdb/pooled_sender.rb +215 -0
- data/lib/questdb/query.rb +429 -0
- data/lib/questdb/sender.rb +436 -0
- data/lib/questdb/sender_error.rb +49 -0
- data/lib/questdb/types.rb +198 -0
- data/lib/questdb/version.rb +27 -0
- data/lib/questdb.rb +22 -0
- data/questdb.gemspec +69 -0
- data/sig/questdb-arrow.rbs +46 -0
- data/sig/questdb.rbs +337 -0
- metadata +188 -0
|
@@ -0,0 +1,440 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
# ___ _ ____ ____
|
|
4
|
+
# / _ \ _ _ ___ ___| |_| _ \| __ )
|
|
5
|
+
# | | | | | | |/ _ \/ __| __| | | | _ \
|
|
6
|
+
# | |_| | |_| | __/\__ \ |_| |_| | |_) |
|
|
7
|
+
# \__\_\\__,_|\___||___/\__|____/|____/
|
|
8
|
+
#
|
|
9
|
+
# Copyright (c) 2014-2019 Appsicle
|
|
10
|
+
# Copyright (c) 2019-2026 QuestDB
|
|
11
|
+
#
|
|
12
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
13
|
+
# you may not use this file except in compliance with the License.
|
|
14
|
+
# You may obtain a copy of the License at
|
|
15
|
+
#
|
|
16
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
17
|
+
#
|
|
18
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
19
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
20
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
21
|
+
# See the License for the specific language governing permissions and
|
|
22
|
+
# limitations under the License.
|
|
23
|
+
#
|
|
24
|
+
|
|
25
|
+
module QuestDB
|
|
26
|
+
module DirectIngestion
|
|
27
|
+
module_function
|
|
28
|
+
|
|
29
|
+
KIND_CODES = {
|
|
30
|
+
byte: 0, short: 1, int: 2, long: 3, float: 4, double: 5,
|
|
31
|
+
boolean: 6, uuid: 7, long256: 8, ipv4: 9, timestamp: 10,
|
|
32
|
+
timestamp_nanos: 11, date: 12, varchar: 13, binary: 14,
|
|
33
|
+
symbol: 15, char: 16, geohash: 17, decimal64: 18,
|
|
34
|
+
decimal128: 19, decimal256: 20, double_array: 21
|
|
35
|
+
}.freeze
|
|
36
|
+
AT_NOW = 0
|
|
37
|
+
AT_SCALAR_NANOS = 1
|
|
38
|
+
AT_MICROS = 2
|
|
39
|
+
AT_NANOS = 3
|
|
40
|
+
INTEGER_RANGES = {
|
|
41
|
+
byte: (-(1 << 7))..((1 << 7) - 1), short: (-(1 << 15))..((1 << 15) - 1),
|
|
42
|
+
int: (-(1 << 31))..((1 << 31) - 1), long: (-(1 << 63))..((1 << 63) - 1)
|
|
43
|
+
}.freeze
|
|
44
|
+
InferredVector = Struct.new(:kind, :values, :scale, :precision_bits, :shape)
|
|
45
|
+
private_constant :InferredVector
|
|
46
|
+
|
|
47
|
+
# Returns the compact, native-facing representation. It contains only
|
|
48
|
+
# private frozen strings, so mutating the caller's Arrays after this method
|
|
49
|
+
# returns cannot affect an in-flight write.
|
|
50
|
+
def prepare(table, columns:, symbols:, at:, max_rows_per_batch:, ack:)
|
|
51
|
+
table = utf8_string(table.is_a?(Symbol) ? table.to_s : table, "table name").freeze
|
|
52
|
+
raise ArgumentError, "table name must not be empty" if table.empty?
|
|
53
|
+
raise TypeError, "columns must be a Hash" unless columns.is_a?(Hash)
|
|
54
|
+
raise TypeError, "symbols must be a Hash" unless symbols.is_a?(Hash)
|
|
55
|
+
batch_size = Integer(max_rows_per_batch)
|
|
56
|
+
raise ArgumentError, "max_rows_per_batch must be positive" unless batch_size.positive?
|
|
57
|
+
ack_level = { ok: 0, durable: 1 }.fetch(ack.respond_to?(:to_sym) ? ack.to_sym : ack) do
|
|
58
|
+
raise ArgumentError, "ack must be :ok or :durable"
|
|
59
|
+
end
|
|
60
|
+
|
|
61
|
+
names = {}
|
|
62
|
+
vectors = []
|
|
63
|
+
row_count = nil
|
|
64
|
+
columns.each do |name, input|
|
|
65
|
+
normalized_name = column_name(name, names)
|
|
66
|
+
vector = input.is_a?(ColumnVector) ? input : infer(input, normalized_name)
|
|
67
|
+
row_count = establish_row_count(row_count, vector.values.length, normalized_name)
|
|
68
|
+
vectors << normalize_column(normalized_name, vector)
|
|
69
|
+
end
|
|
70
|
+
symbols.each do |name, input|
|
|
71
|
+
normalized_name = column_name(name, names)
|
|
72
|
+
values = input.is_a?(ColumnVector) ? input.values : array!(input, normalized_name)
|
|
73
|
+
if input.is_a?(ColumnVector) && input.kind != :varchar
|
|
74
|
+
raise ArgumentError, "symbol #{normalized_name.inspect} descriptor must use :varchar"
|
|
75
|
+
end
|
|
76
|
+
row_count = establish_row_count(row_count, values.length, normalized_name)
|
|
77
|
+
vectors << normalize_symbol(normalized_name, values)
|
|
78
|
+
end
|
|
79
|
+
|
|
80
|
+
row_count ||= 0
|
|
81
|
+
at_kind, at_data, at_count = normalize_at(at, row_count)
|
|
82
|
+
if row_count.zero?
|
|
83
|
+
if at_count && at_count.positive?
|
|
84
|
+
raise ArgumentError, "designated timestamp has #{at_count} rows but columns are empty"
|
|
85
|
+
end
|
|
86
|
+
return [table, [].freeze, AT_NOW, nil, 0, batch_size, ack_level].freeze
|
|
87
|
+
end
|
|
88
|
+
raise ArgumentError, "at vector has #{at_count} rows, expected #{row_count}" if at_count && at_count != row_count
|
|
89
|
+
|
|
90
|
+
[table, vectors.freeze, at_kind, at_data, row_count, batch_size, ack_level].freeze
|
|
91
|
+
end
|
|
92
|
+
|
|
93
|
+
def infer(input, name)
|
|
94
|
+
values = array!(input, name)
|
|
95
|
+
first = values.find { |value| !value.nil? }
|
|
96
|
+
if first.nil?
|
|
97
|
+
raise ArgumentError, "column #{name.inspect} is all nil; use QuestDB::ColumnVector with an explicit kind" unless values.empty?
|
|
98
|
+
return InferredVector.new(:long, values, nil, nil, nil)
|
|
99
|
+
end
|
|
100
|
+
|
|
101
|
+
kind = case first
|
|
102
|
+
when Integer then :long
|
|
103
|
+
when Float then :double
|
|
104
|
+
when TrueClass, FalseClass then :boolean
|
|
105
|
+
when String then :varchar
|
|
106
|
+
when Binary then :binary
|
|
107
|
+
when UUID then :uuid
|
|
108
|
+
when Long256 then :long256
|
|
109
|
+
when IPv4 then :ipv4
|
|
110
|
+
when TimestampNanos then :timestamp_nanos
|
|
111
|
+
when TimestampMicros then :timestamp
|
|
112
|
+
when DateMillis then :date
|
|
113
|
+
when Char then :char
|
|
114
|
+
when Geohash then :geohash
|
|
115
|
+
when DecimalValue then :decimal
|
|
116
|
+
when Float64Array then :double_array
|
|
117
|
+
else
|
|
118
|
+
raise TypeError, "cannot infer direct column type for #{name.inspect} from #{first.class}"
|
|
119
|
+
end
|
|
120
|
+
unless homogeneous_for_kind?(kind, values, first.class)
|
|
121
|
+
raise TypeError, "column #{name.inspect} is heterogeneous; use a homogeneous Array or explicit ColumnVector"
|
|
122
|
+
end
|
|
123
|
+
kind = decimal_kind(values) if kind == :decimal
|
|
124
|
+
metadata = case kind
|
|
125
|
+
when :geohash
|
|
126
|
+
bits = first.precision_bits
|
|
127
|
+
raise ArgumentError, "all geohashes in #{name.inspect} must have the same precision" unless values.all? { |v| v.nil? || v.precision_bits == bits }
|
|
128
|
+
{ precision_bits: bits }
|
|
129
|
+
when :decimal64, :decimal128, :decimal256
|
|
130
|
+
scale = first.scale
|
|
131
|
+
raise ArgumentError, "all decimals in #{name.inspect} must have the same non-negative scale" unless values.all? { |v| v.nil? || v.scale == scale }
|
|
132
|
+
{ scale: scale }
|
|
133
|
+
when :double_array
|
|
134
|
+
shape = first.shape
|
|
135
|
+
raise ArgumentError, "all arrays in #{name.inspect} must have the same shape" unless values.all? { |v| v.nil? || v.shape == shape }
|
|
136
|
+
{ shape: shape }
|
|
137
|
+
else {}
|
|
138
|
+
end
|
|
139
|
+
InferredVector.new(kind, values, metadata[:scale], metadata[:precision_bits], metadata[:shape])
|
|
140
|
+
end
|
|
141
|
+
|
|
142
|
+
def normalize_column(name, vector)
|
|
143
|
+
kind = vector.kind
|
|
144
|
+
values = vector.values
|
|
145
|
+
if %i[byte short boolean char].include?(kind) && values.any?(&:nil?)
|
|
146
|
+
raise ArgumentError, "#{kind} cannot represent NULL in QuestDB; use :int, :varchar, or another nullable kind"
|
|
147
|
+
end
|
|
148
|
+
validity = validity_for(values)
|
|
149
|
+
data = case kind
|
|
150
|
+
when :byte, :short, :int, :long then pack_integers(values, kind)
|
|
151
|
+
when :float then pack_floats(values, "f")
|
|
152
|
+
when :double then pack_floats(values, "d")
|
|
153
|
+
when :boolean then pack_booleans(values)
|
|
154
|
+
when :varchar then pack_variable(values) { |v| utf8_string(v, "VARCHAR value", copy: false) }
|
|
155
|
+
when :binary then pack_variable(values) { |v| v.is_a?(Binary) ? v.bytes : String(v) }
|
|
156
|
+
when :uuid then values.map { |v| v.nil? ? ("\0" * 16) : (v.is_a?(UUID) ? v : UUID.new(v)).bytes }.join.b
|
|
157
|
+
when :long256 then values.map { |v| v.nil? ? ("\0" * 32) : (v.is_a?(Long256) ? v : Long256.new(v)).bytes }.join.b
|
|
158
|
+
when :ipv4 then values.map { |v| v.nil? ? 0 : (v.is_a?(IPv4) ? v : IPv4.new(v)).value }.pack("L*")
|
|
159
|
+
when :timestamp then values.map { |v| v.nil? ? 0 : timestamp_value(v, :micros) }.pack("q*")
|
|
160
|
+
when :timestamp_nanos then values.map { |v| v.nil? ? 0 : timestamp_value(v, :nanos) }.pack("q*")
|
|
161
|
+
when :date then values.map { |v| v.nil? ? 0 : date_value(v) }.pack("q*")
|
|
162
|
+
when :char then values.map { |v| v.nil? ? 0 : (v.is_a?(Char) ? v : Char.new(v)).value }.pack("S*")
|
|
163
|
+
when :geohash then pack_geohashes(values, vector.precision_bits)
|
|
164
|
+
when :decimal64, :decimal128, :decimal256 then pack_decimals(values, kind, vector.scale)
|
|
165
|
+
when :double_array then pack_double_arrays(values, vector.shape)
|
|
166
|
+
else raise ArgumentError, "unsupported direct column kind #{kind.inspect}"
|
|
167
|
+
end
|
|
168
|
+
|
|
169
|
+
descriptor = if %i[varchar binary].include?(kind)
|
|
170
|
+
offsets, bytes = data
|
|
171
|
+
[KIND_CODES.fetch(kind), name, offsets.freeze, validity, bytes.freeze, nil, 0, 0]
|
|
172
|
+
elsif kind == :geohash
|
|
173
|
+
[KIND_CODES.fetch(kind), name, data.freeze, validity, nil, nil, vector.precision_bits, 0]
|
|
174
|
+
elsif %i[decimal64 decimal128 decimal256].include?(kind)
|
|
175
|
+
[KIND_CODES.fetch(kind), name, data.freeze, validity, nil, nil, vector.scale, 0]
|
|
176
|
+
elsif kind == :double_array
|
|
177
|
+
shape = vector.shape.pack("L*").freeze
|
|
178
|
+
[KIND_CODES.fetch(kind), name, data.freeze, validity, shape, nil, vector.shape.length, 0]
|
|
179
|
+
else
|
|
180
|
+
[KIND_CODES.fetch(kind), name, data.freeze, validity, nil, nil, 0, 0]
|
|
181
|
+
end
|
|
182
|
+
descriptor.freeze
|
|
183
|
+
end
|
|
184
|
+
|
|
185
|
+
def normalize_symbol(name, values)
|
|
186
|
+
values = array!(values, name)
|
|
187
|
+
dictionary = {}
|
|
188
|
+
source_codes = {}
|
|
189
|
+
entries = []
|
|
190
|
+
codes = values.map do |value|
|
|
191
|
+
next 0 if value.nil?
|
|
192
|
+
source_codes.fetch(value) do
|
|
193
|
+
text = utf8_string(value, "SYMBOL value", copy: false)
|
|
194
|
+
index = dictionary.fetch(text) do
|
|
195
|
+
owned = text.dup.freeze
|
|
196
|
+
entries << owned
|
|
197
|
+
dictionary[owned] = entries.length - 1
|
|
198
|
+
end
|
|
199
|
+
source_codes[value] = index
|
|
200
|
+
end
|
|
201
|
+
end
|
|
202
|
+
offsets = [0]
|
|
203
|
+
bytes = +"".b
|
|
204
|
+
entries.each { |entry| bytes << entry; offsets << bytes.bytesize }
|
|
205
|
+
[KIND_CODES[:symbol], name, codes.pack("l*").freeze, validity_for(values), offsets.pack("l*").freeze,
|
|
206
|
+
bytes.force_encoding(Encoding::BINARY).freeze, 0, 0].freeze
|
|
207
|
+
end
|
|
208
|
+
|
|
209
|
+
def normalize_at(at, row_count)
|
|
210
|
+
return [AT_NOW, nil, nil] if at.equal?(SERVER_TIMESTAMP)
|
|
211
|
+
if at.is_a?(Array) || at.is_a?(ColumnVector)
|
|
212
|
+
vector = at.is_a?(ColumnVector) ? at : nil
|
|
213
|
+
values = vector ? vector.values : at
|
|
214
|
+
if values.any?(&:nil?)
|
|
215
|
+
raise ArgumentError, "designated timestamp vector cannot contain nil"
|
|
216
|
+
end
|
|
217
|
+
kind = if vector
|
|
218
|
+
raise ArgumentError, "at ColumnVector must be :timestamp or :timestamp_nanos" unless %i[timestamp timestamp_nanos].include?(vector.kind)
|
|
219
|
+
vector.kind
|
|
220
|
+
else
|
|
221
|
+
nonempty = values.first
|
|
222
|
+
case nonempty
|
|
223
|
+
when TimestampNanos, Time then :timestamp_nanos
|
|
224
|
+
when TimestampMicros then :timestamp
|
|
225
|
+
when nil then :timestamp
|
|
226
|
+
else raise TypeError, "at Array values must be TimestampNanos, TimestampMicros, or Time"
|
|
227
|
+
end
|
|
228
|
+
end
|
|
229
|
+
expected = kind == :timestamp_nanos ? [TimestampNanos, Time] : [TimestampMicros]
|
|
230
|
+
unless values.all? { |value| expected.any? { |klass| value.is_a?(klass) } }
|
|
231
|
+
raise TypeError, "designated timestamp vector is heterogeneous"
|
|
232
|
+
end
|
|
233
|
+
unit = kind == :timestamp_nanos ? :nanos : :micros
|
|
234
|
+
packed = values.map { |value| non_negative_timestamp(timestamp_value(value, unit)) }.pack("q*").freeze
|
|
235
|
+
[kind == :timestamp_nanos ? AT_NANOS : AT_MICROS, packed, values.length]
|
|
236
|
+
else
|
|
237
|
+
nanos = case at
|
|
238
|
+
when TimestampNanos then at.value
|
|
239
|
+
when TimestampMicros then checked_multiply(at.value, 1_000, "designated timestamp")
|
|
240
|
+
when Time then time_nanos(at)
|
|
241
|
+
else raise TypeError, "at must be a timestamp vector, TimestampNanos, TimestampMicros, Time, or SERVER_TIMESTAMP"
|
|
242
|
+
end
|
|
243
|
+
[AT_SCALAR_NANOS, non_negative_timestamp(nanos), nil]
|
|
244
|
+
end
|
|
245
|
+
end
|
|
246
|
+
|
|
247
|
+
def validity_for(values)
|
|
248
|
+
return nil unless values.any?(&:nil?)
|
|
249
|
+
bits = Array.new((values.length + 7) / 8, 0)
|
|
250
|
+
values.each_with_index { |value, index| bits[index / 8] |= 1 << (index % 8) unless value.nil? }
|
|
251
|
+
bits.pack("C*").freeze
|
|
252
|
+
end
|
|
253
|
+
|
|
254
|
+
def pack_integers(values, kind)
|
|
255
|
+
range = INTEGER_RANGES.fetch(kind)
|
|
256
|
+
converted = values.map do |value|
|
|
257
|
+
next 0 if value.nil?
|
|
258
|
+
number = Integer(value)
|
|
259
|
+
raise RangeError, "#{kind} value #{number} is out of range" unless range.cover?(number)
|
|
260
|
+
if (kind == :int || kind == :long) && number == range.begin
|
|
261
|
+
raise RangeError, "#{kind} minimum value is reserved as QuestDB NULL"
|
|
262
|
+
end
|
|
263
|
+
number
|
|
264
|
+
end
|
|
265
|
+
converted.pack({ byte: "c*", short: "s*", int: "l*", long: "q*" }.fetch(kind))
|
|
266
|
+
end
|
|
267
|
+
|
|
268
|
+
def pack_floats(values, directive)
|
|
269
|
+
values.map do |value|
|
|
270
|
+
next 0.0 if value.nil?
|
|
271
|
+
raise TypeError, "floating-point columns accept only Numeric values" unless value.is_a?(Numeric)
|
|
272
|
+
number = Float(value)
|
|
273
|
+
raise RangeError, "NaN is reserved as QuestDB NULL" if number.nan?
|
|
274
|
+
if directive == "f" && number.finite? && number.abs > 3.4028234663852886e38
|
|
275
|
+
raise RangeError, "float value is outside IEEE-754 binary32 range"
|
|
276
|
+
end
|
|
277
|
+
number
|
|
278
|
+
end.pack("#{directive}*")
|
|
279
|
+
end
|
|
280
|
+
|
|
281
|
+
def pack_booleans(values)
|
|
282
|
+
bits = Array.new((values.length + 7) / 8, 0)
|
|
283
|
+
values.each_with_index do |value, index|
|
|
284
|
+
next if value.nil? || value == false
|
|
285
|
+
raise TypeError, "boolean columns accept only true, false, or nil" unless value == true
|
|
286
|
+
bits[index / 8] |= 1 << (index % 8)
|
|
287
|
+
end
|
|
288
|
+
bits.pack("C*")
|
|
289
|
+
end
|
|
290
|
+
|
|
291
|
+
def pack_variable(values)
|
|
292
|
+
offsets = [0]
|
|
293
|
+
bytes = +"".b
|
|
294
|
+
values.each do |value|
|
|
295
|
+
bytes << yield(value) unless value.nil?
|
|
296
|
+
raise RangeError, "variable-width column exceeds 2 GiB" if bytes.bytesize > 0x7fff_ffff
|
|
297
|
+
offsets << bytes.bytesize
|
|
298
|
+
end
|
|
299
|
+
[offsets.pack("l*"), bytes.force_encoding(Encoding::BINARY)]
|
|
300
|
+
end
|
|
301
|
+
|
|
302
|
+
def pack_geohashes(values, bits)
|
|
303
|
+
limit = (1 << bits) - 1
|
|
304
|
+
numbers = values.map do |value|
|
|
305
|
+
next 0 if value.nil?
|
|
306
|
+
geohash = value.is_a?(Geohash) ? value : Geohash.new(value, precision_bits: bits)
|
|
307
|
+
raise ArgumentError, "geohash precision mismatch" unless geohash.precision_bits == bits
|
|
308
|
+
raise RangeError, "geohash value exceeds precision" unless geohash.value.between?(0, limit)
|
|
309
|
+
geohash.value
|
|
310
|
+
end
|
|
311
|
+
numbers.pack(bits <= 8 ? "C*" : bits <= 16 ? "S*" : bits <= 32 ? "L*" : "Q*")
|
|
312
|
+
end
|
|
313
|
+
|
|
314
|
+
def pack_decimals(values, kind, scale)
|
|
315
|
+
width = { decimal64: 8, decimal128: 16, decimal256: 32 }.fetch(kind)
|
|
316
|
+
values.map do |value|
|
|
317
|
+
mantissa = if value.nil?
|
|
318
|
+
0
|
|
319
|
+
elsif value.is_a?(DecimalValue)
|
|
320
|
+
raise ArgumentError, "decimal scale mismatch" unless value.scale == scale
|
|
321
|
+
value.mantissa
|
|
322
|
+
else Integer(value)
|
|
323
|
+
end
|
|
324
|
+
min = -(1 << (width * 8 - 1)); max = (1 << (width * 8 - 1)) - 1
|
|
325
|
+
raise RangeError, "#{kind} mantissa is out of range" unless mantissa.between?(min, max)
|
|
326
|
+
twos_complement_bytes(mantissa, width)
|
|
327
|
+
end.join.b
|
|
328
|
+
end
|
|
329
|
+
|
|
330
|
+
def pack_double_arrays(values, shape)
|
|
331
|
+
size = shape.inject(1, :*)
|
|
332
|
+
values.map do |value|
|
|
333
|
+
next Array.new(size, 0.0).pack("d*") if value.nil?
|
|
334
|
+
array = value.is_a?(Float64Array) ? value : Float64Array.from_nested(value)
|
|
335
|
+
raise ArgumentError, "double_array shape mismatch" unless array.shape == shape
|
|
336
|
+
raise ArgumentError, "double_array value count mismatch" unless array.values.length == size
|
|
337
|
+
array.values.map { |item| Float(item) }.pack("d*")
|
|
338
|
+
end.join.b
|
|
339
|
+
end
|
|
340
|
+
|
|
341
|
+
def decimal_kind(values)
|
|
342
|
+
first = values.find { |value| !value.nil? }
|
|
343
|
+
scale = first.scale
|
|
344
|
+
raise RangeError, "decimal inference requires a non-negative scale" if scale.negative?
|
|
345
|
+
bits = values.reduce(0) do |maximum, value|
|
|
346
|
+
next maximum unless value
|
|
347
|
+
candidate = value.mantissa.bit_length + 1
|
|
348
|
+
candidate > maximum ? candidate : maximum
|
|
349
|
+
end
|
|
350
|
+
kind = bits <= 63 && scale <= 18 ? :decimal64 : bits <= 127 && scale <= 38 ? :decimal128 : :decimal256
|
|
351
|
+
cap = { decimal64: 18, decimal128: 38, decimal256: 76 }.fetch(kind)
|
|
352
|
+
raise RangeError, "decimal cannot fit a direct QuestDB decimal type" if bits > 255 || scale > cap
|
|
353
|
+
kind
|
|
354
|
+
end
|
|
355
|
+
|
|
356
|
+
def homogeneous_for_kind?(kind, values, expected_class)
|
|
357
|
+
case kind
|
|
358
|
+
when :boolean then values.all? { |value| value.nil? || value == true || value == false }
|
|
359
|
+
when :varchar then values.all? { |value| value.nil? || value.class == String }
|
|
360
|
+
else values.all? { |value| value.nil? || value.class == expected_class }
|
|
361
|
+
end
|
|
362
|
+
end
|
|
363
|
+
|
|
364
|
+
def timestamp_value(value, unit)
|
|
365
|
+
result = case value
|
|
366
|
+
when TimestampNanos
|
|
367
|
+
unit == :nanos ? value.value : value.value.div(1_000)
|
|
368
|
+
when TimestampMicros
|
|
369
|
+
unit == :micros ? value.value : checked_multiply(value.value, 1_000, "timestamp")
|
|
370
|
+
when Time
|
|
371
|
+
unit == :nanos ? time_nanos(value) : time_nanos(value).div(1_000)
|
|
372
|
+
else Integer(value)
|
|
373
|
+
end
|
|
374
|
+
check_i64(result, "timestamp")
|
|
375
|
+
end
|
|
376
|
+
|
|
377
|
+
def date_value(value)
|
|
378
|
+
check_i64(value.is_a?(DateMillis) ? value.value : Integer(value), "date")
|
|
379
|
+
end
|
|
380
|
+
|
|
381
|
+
def time_nanos(time)
|
|
382
|
+
check_i64(time.to_i * 1_000_000_000 + time.nsec, "timestamp")
|
|
383
|
+
end
|
|
384
|
+
|
|
385
|
+
def non_negative_timestamp(value)
|
|
386
|
+
raise RangeError, "designated timestamp cannot be before the Unix epoch" if value.negative?
|
|
387
|
+
value
|
|
388
|
+
end
|
|
389
|
+
|
|
390
|
+
def checked_multiply(value, multiplier, label)
|
|
391
|
+
check_i64(Integer(value) * multiplier, label)
|
|
392
|
+
end
|
|
393
|
+
|
|
394
|
+
def check_i64(value, label)
|
|
395
|
+
raise RangeError, "#{label} is outside signed 64-bit range" unless value.between?(-(1 << 63), (1 << 63) - 1)
|
|
396
|
+
value
|
|
397
|
+
end
|
|
398
|
+
|
|
399
|
+
def twos_complement_bytes(value, width)
|
|
400
|
+
unsigned = value.negative? ? (1 << (width * 8)) + value : value
|
|
401
|
+
bytes = Array.new(width) { |index| (unsigned >> (index * 8)) & 0xff }
|
|
402
|
+
bytes.reverse! unless [1].pack("S") == "\x01\x00".b
|
|
403
|
+
bytes.pack("C*")
|
|
404
|
+
end
|
|
405
|
+
|
|
406
|
+
def establish_row_count(current, candidate, name)
|
|
407
|
+
return candidate if current.nil?
|
|
408
|
+
raise ArgumentError, "column #{name.inspect} has #{candidate} rows, expected #{current}" unless candidate == current
|
|
409
|
+
current
|
|
410
|
+
end
|
|
411
|
+
|
|
412
|
+
def column_name(name, names)
|
|
413
|
+
normalized = utf8_string(name.is_a?(Symbol) ? name.to_s : name, "column name").freeze
|
|
414
|
+
raise ArgumentError, "column name must not be empty" if normalized.empty?
|
|
415
|
+
raise ArgumentError, "duplicate column #{normalized.inspect}" if names.key?(normalized)
|
|
416
|
+
names[normalized] = true
|
|
417
|
+
normalized
|
|
418
|
+
end
|
|
419
|
+
|
|
420
|
+
def array!(value, name)
|
|
421
|
+
raise TypeError, "column #{name.inspect} must be an Array or ColumnVector" unless value.is_a?(Array)
|
|
422
|
+
value
|
|
423
|
+
end
|
|
424
|
+
|
|
425
|
+
def utf8_string(value, label, copy: true)
|
|
426
|
+
string = String(value)
|
|
427
|
+
if string.encoding == Encoding::UTF_8
|
|
428
|
+
raise EncodingError, "#{label} is not valid UTF-8" unless string.valid_encoding?
|
|
429
|
+
return copy ? string.dup : string
|
|
430
|
+
end
|
|
431
|
+
if string.ascii_only?
|
|
432
|
+
return string unless copy
|
|
433
|
+
return string.dup.force_encoding(Encoding::UTF_8)
|
|
434
|
+
end
|
|
435
|
+
string.encode(Encoding::UTF_8)
|
|
436
|
+
rescue Encoding::UndefinedConversionError, Encoding::InvalidByteSequenceError => error
|
|
437
|
+
raise EncodingError, "#{label} is not valid UTF-8: #{error.message}"
|
|
438
|
+
end
|
|
439
|
+
end
|
|
440
|
+
end
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
# ___ _ ____ ____
|
|
4
|
+
# / _ \ _ _ ___ ___| |_| _ \| __ )
|
|
5
|
+
# | | | | | | |/ _ \/ __| __| | | | _ \
|
|
6
|
+
# | |_| | |_| | __/\__ \ |_| |_| | |_) |
|
|
7
|
+
# \__\_\\__,_|\___||___/\__|____/|____/
|
|
8
|
+
#
|
|
9
|
+
# Copyright (c) 2014-2019 Appsicle
|
|
10
|
+
# Copyright (c) 2019-2026 QuestDB
|
|
11
|
+
#
|
|
12
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
13
|
+
# you may not use this file except in compliance with the License.
|
|
14
|
+
# You may obtain a copy of the License at
|
|
15
|
+
#
|
|
16
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
17
|
+
#
|
|
18
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
19
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
20
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
21
|
+
# See the License for the specific language governing permissions and
|
|
22
|
+
# limitations under the License.
|
|
23
|
+
#
|
|
24
|
+
require "thread"
|
|
25
|
+
|
|
26
|
+
module QuestDB
|
|
27
|
+
# Internal bounded, drop-oldest queue used after native callbacks have been
|
|
28
|
+
# copied into Ruby-owned values. Callback dispatch never waits on consumers.
|
|
29
|
+
class EventInbox
|
|
30
|
+
def initialize(capacity)
|
|
31
|
+
@capacity = capacity.zero? ? 64 : capacity
|
|
32
|
+
@items = []
|
|
33
|
+
@mutex = Mutex.new
|
|
34
|
+
@delivered = 0
|
|
35
|
+
@dropped = 0
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
def push(value)
|
|
39
|
+
@mutex.synchronize do
|
|
40
|
+
if @items.length == @capacity
|
|
41
|
+
@items.shift
|
|
42
|
+
@dropped += 1
|
|
43
|
+
end
|
|
44
|
+
@items << value
|
|
45
|
+
end
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
def poll
|
|
49
|
+
@mutex.synchronize do
|
|
50
|
+
value = @items.shift
|
|
51
|
+
@delivered += 1 if value
|
|
52
|
+
value
|
|
53
|
+
end
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
def delivered = @mutex.synchronize { @delivered }
|
|
57
|
+
def dropped = @mutex.synchronize { @dropped }
|
|
58
|
+
end
|
|
59
|
+
private_constant :EventInbox
|
|
60
|
+
end
|
|
@@ -0,0 +1,104 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
# ___ _ ____ ____
|
|
4
|
+
# / _ \ _ _ ___ ___| |_| _ \| __ )
|
|
5
|
+
# | | | | | | |/ _ \/ __| __| | | | _ \
|
|
6
|
+
# | |_| | |_| | __/\__ \ |_| |_| | |_) |
|
|
7
|
+
# \__\_\\__,_|\___||___/\__|____/|____/
|
|
8
|
+
#
|
|
9
|
+
# Copyright (c) 2014-2019 Appsicle
|
|
10
|
+
# Copyright (c) 2019-2026 QuestDB
|
|
11
|
+
#
|
|
12
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
13
|
+
# you may not use this file except in compliance with the License.
|
|
14
|
+
# You may obtain a copy of the License at
|
|
15
|
+
#
|
|
16
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
17
|
+
#
|
|
18
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
19
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
20
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
21
|
+
# See the License for the specific language governing permissions and
|
|
22
|
+
# limitations under the License.
|
|
23
|
+
#
|
|
24
|
+
|
|
25
|
+
module QuestDB
|
|
26
|
+
module Ingress
|
|
27
|
+
class Buffer
|
|
28
|
+
MAX_PREALLOCATE_BYTES = 100 * 1024 * 1024
|
|
29
|
+
EMPTY_HASH = {}.freeze
|
|
30
|
+
private_constant :EMPTY_HASH
|
|
31
|
+
|
|
32
|
+
class << self
|
|
33
|
+
private
|
|
34
|
+
|
|
35
|
+
def from_native(native)
|
|
36
|
+
object = allocate
|
|
37
|
+
object.send(:initialize_native, native)
|
|
38
|
+
object
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
private :new
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
def row(table, symbols: EMPTY_HASH, columns: EMPTY_HASH, at:)
|
|
45
|
+
append(table, symbols, columns, at)
|
|
46
|
+
self
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
def row_count = @native.row_count
|
|
50
|
+
alias length row_count
|
|
51
|
+
def empty? = row_count.zero?
|
|
52
|
+
|
|
53
|
+
# QWP encodes at publication time. These values are only memory hints.
|
|
54
|
+
def bytesize = @native.bytesize
|
|
55
|
+
def capacity = @native.capacity
|
|
56
|
+
|
|
57
|
+
def reserve(additional)
|
|
58
|
+
additional = Integer(additional)
|
|
59
|
+
raise ArgumentError, "additional must be non-negative" if additional.negative?
|
|
60
|
+
raise RangeError, "buffer preallocation exceeds 100 MiB" if additional > MAX_PREALLOCATE_BYTES
|
|
61
|
+
@native.reserve(additional)
|
|
62
|
+
self
|
|
63
|
+
end
|
|
64
|
+
|
|
65
|
+
def clear = (@native.clear; self)
|
|
66
|
+
def initialize_copy(*) = raise(TypeError, "QuestDB buffers cannot be copied")
|
|
67
|
+
|
|
68
|
+
protected
|
|
69
|
+
|
|
70
|
+
attr_reader :native
|
|
71
|
+
|
|
72
|
+
private
|
|
73
|
+
|
|
74
|
+
def initialize_native(native)
|
|
75
|
+
@native = native
|
|
76
|
+
end
|
|
77
|
+
|
|
78
|
+
# Positional internal entry point avoids constructing a second keyword
|
|
79
|
+
# Hash when Sender delegates its ergonomic keyword API to the buffer.
|
|
80
|
+
def append(table, symbols, columns, at)
|
|
81
|
+
raise TypeError, "symbols must be a Hash" unless symbols.is_a?(Hash)
|
|
82
|
+
raise TypeError, "columns must be a Hash" unless columns.is_a?(Hash)
|
|
83
|
+
return false unless hash_has_value?(symbols) || hash_has_value?(columns)
|
|
84
|
+
@native.append_row(table, symbols, columns, normalize_at(at))
|
|
85
|
+
rescue EncodingError => error
|
|
86
|
+
raise IngressError.new(error.message, code: :invalid_utf8), cause: error
|
|
87
|
+
end
|
|
88
|
+
|
|
89
|
+
def hash_has_value?(hash)
|
|
90
|
+
hash.each_value { |value| return true unless value.nil? }
|
|
91
|
+
false
|
|
92
|
+
end
|
|
93
|
+
|
|
94
|
+
def normalize_at(value)
|
|
95
|
+
case value
|
|
96
|
+
when TimestampNanos, TimestampMicros, Time then value
|
|
97
|
+
else
|
|
98
|
+
return value if value.equal?(SERVER_TIMESTAMP)
|
|
99
|
+
raise IngressError.new("at must be TimestampNanos, TimestampMicros, Time, or SERVER_TIMESTAMP", code: :invalid_timestamp)
|
|
100
|
+
end
|
|
101
|
+
end
|
|
102
|
+
end
|
|
103
|
+
end
|
|
104
|
+
end
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
# ___ _ ____ ____
|
|
2
|
+
# / _ \ _ _ ___ ___| |_| _ \| __ )
|
|
3
|
+
# | | | | | | |/ _ \/ __| __| | | | _ \
|
|
4
|
+
# | |_| | |_| | __/\__ \ |_| |_| | |_) |
|
|
5
|
+
# \__\_\\__,_|\___||___/\__|____/|____/
|
|
6
|
+
#
|
|
7
|
+
# Copyright (c) 2014-2019 Appsicle
|
|
8
|
+
# Copyright (c) 2019-2026 QuestDB
|
|
9
|
+
#
|
|
10
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
11
|
+
# you may not use this file except in compliance with the License.
|
|
12
|
+
# You may obtain a copy of the License at
|
|
13
|
+
#
|
|
14
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
15
|
+
#
|
|
16
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
17
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
18
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
19
|
+
# See the License for the specific language governing permissions and
|
|
20
|
+
# limitations under the License.
|
|
21
|
+
#
|
|
22
|
+
module QuestDB
|
|
23
|
+
module Ingress
|
|
24
|
+
class IngressError < StandardError
|
|
25
|
+
attr_reader :code, :sender_error
|
|
26
|
+
|
|
27
|
+
def initialize(message, code: :invalid_api_call, in_doubt: false, sender_error: nil)
|
|
28
|
+
@code = code.to_sym
|
|
29
|
+
@in_doubt = !!in_doubt
|
|
30
|
+
@sender_error = if sender_error.is_a?(Array) && defined?(QuestDB::SenderError)
|
|
31
|
+
QuestDB::SenderError.from_native(sender_error)
|
|
32
|
+
else
|
|
33
|
+
sender_error
|
|
34
|
+
end
|
|
35
|
+
super(message)
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
def in_doubt? = @in_doubt
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
class ServerRejectionError < IngressError; end
|
|
42
|
+
end
|
|
43
|
+
end
|