questdb 0.1.0-x64-mingw-ucrt

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (51) hide show
  1. checksums.yaml +7 -0
  2. data/CHANGELOG.md +22 -0
  3. data/LICENSE.txt +201 -0
  4. data/README.md +170 -0
  5. data/SECURITY.md +19 -0
  6. data/THIRD_PARTY_NOTICES.md +242 -0
  7. data/docs/arrow.md +89 -0
  8. data/docs/configuration.md +61 -0
  9. data/docs/design.md +46 -0
  10. data/docs/ingestion.md +96 -0
  11. data/docs/performance.md +36 -0
  12. data/docs/python-parity.md +34 -0
  13. data/docs/querying.md +54 -0
  14. data/docs/release-infrastructure.md +70 -0
  15. data/docs/releasing.md +47 -0
  16. data/docs/troubleshooting.md +41 -0
  17. data/examples/basic.rb +32 -0
  18. data/examples/callbacks.rb +46 -0
  19. data/examples/columnar.rb +41 -0
  20. data/examples/independent_buffer.rb +39 -0
  21. data/examples/query.rb +28 -0
  22. data/examples/standalone.rb +33 -0
  23. data/examples/tls_auth.rb +33 -0
  24. data/lib/questdb/arrow.rb +250 -0
  25. data/lib/questdb/client.rb +308 -0
  26. data/lib/questdb/column_vector.rb +91 -0
  27. data/lib/questdb/config.rb +161 -0
  28. data/lib/questdb/connection_event.rb +60 -0
  29. data/lib/questdb/direct_ingestion.rb +440 -0
  30. data/lib/questdb/event_inbox.rb +60 -0
  31. data/lib/questdb/ingress/buffer.rb +104 -0
  32. data/lib/questdb/ingress/error.rb +43 -0
  33. data/lib/questdb/ingress/float64_array.rb +60 -0
  34. data/lib/questdb/ingress/native.rb +40 -0
  35. data/lib/questdb/ingress/timestamp.rb +53 -0
  36. data/lib/questdb/ingress.rb +77 -0
  37. data/lib/questdb/native/3.1/questdb_ingress.so +0 -0
  38. data/lib/questdb/native/3.2/questdb_ingress.so +0 -0
  39. data/lib/questdb/native/3.3/questdb_ingress.so +0 -0
  40. data/lib/questdb/native/3.4/questdb_ingress.so +0 -0
  41. data/lib/questdb/pooled_sender.rb +215 -0
  42. data/lib/questdb/query.rb +429 -0
  43. data/lib/questdb/sender.rb +436 -0
  44. data/lib/questdb/sender_error.rb +49 -0
  45. data/lib/questdb/types.rb +198 -0
  46. data/lib/questdb/version.rb +27 -0
  47. data/lib/questdb.rb +22 -0
  48. data/questdb.gemspec +69 -0
  49. data/sig/questdb-arrow.rbs +46 -0
  50. data/sig/questdb.rbs +337 -0
  51. metadata +188 -0
@@ -0,0 +1,440 @@
1
+ # frozen_string_literal: true
2
+
3
+ # ___ _ ____ ____
4
+ # / _ \ _ _ ___ ___| |_| _ \| __ )
5
+ # | | | | | | |/ _ \/ __| __| | | | _ \
6
+ # | |_| | |_| | __/\__ \ |_| |_| | |_) |
7
+ # \__\_\\__,_|\___||___/\__|____/|____/
8
+ #
9
+ # Copyright (c) 2014-2019 Appsicle
10
+ # Copyright (c) 2019-2026 QuestDB
11
+ #
12
+ # Licensed under the Apache License, Version 2.0 (the "License");
13
+ # you may not use this file except in compliance with the License.
14
+ # You may obtain a copy of the License at
15
+ #
16
+ # http://www.apache.org/licenses/LICENSE-2.0
17
+ #
18
+ # Unless required by applicable law or agreed to in writing, software
19
+ # distributed under the License is distributed on an "AS IS" BASIS,
20
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
21
+ # See the License for the specific language governing permissions and
22
+ # limitations under the License.
23
+ #
24
+
25
+ module QuestDB
26
+ module DirectIngestion
27
+ module_function
28
+
29
+ KIND_CODES = {
30
+ byte: 0, short: 1, int: 2, long: 3, float: 4, double: 5,
31
+ boolean: 6, uuid: 7, long256: 8, ipv4: 9, timestamp: 10,
32
+ timestamp_nanos: 11, date: 12, varchar: 13, binary: 14,
33
+ symbol: 15, char: 16, geohash: 17, decimal64: 18,
34
+ decimal128: 19, decimal256: 20, double_array: 21
35
+ }.freeze
36
+ AT_NOW = 0
37
+ AT_SCALAR_NANOS = 1
38
+ AT_MICROS = 2
39
+ AT_NANOS = 3
40
+ INTEGER_RANGES = {
41
+ byte: (-(1 << 7))..((1 << 7) - 1), short: (-(1 << 15))..((1 << 15) - 1),
42
+ int: (-(1 << 31))..((1 << 31) - 1), long: (-(1 << 63))..((1 << 63) - 1)
43
+ }.freeze
44
+ InferredVector = Struct.new(:kind, :values, :scale, :precision_bits, :shape)
45
+ private_constant :InferredVector
46
+
47
+ # Returns the compact, native-facing representation. It contains only
48
+ # private frozen strings, so mutating the caller's Arrays after this method
49
+ # returns cannot affect an in-flight write.
50
+ def prepare(table, columns:, symbols:, at:, max_rows_per_batch:, ack:)
51
+ table = utf8_string(table.is_a?(Symbol) ? table.to_s : table, "table name").freeze
52
+ raise ArgumentError, "table name must not be empty" if table.empty?
53
+ raise TypeError, "columns must be a Hash" unless columns.is_a?(Hash)
54
+ raise TypeError, "symbols must be a Hash" unless symbols.is_a?(Hash)
55
+ batch_size = Integer(max_rows_per_batch)
56
+ raise ArgumentError, "max_rows_per_batch must be positive" unless batch_size.positive?
57
+ ack_level = { ok: 0, durable: 1 }.fetch(ack.respond_to?(:to_sym) ? ack.to_sym : ack) do
58
+ raise ArgumentError, "ack must be :ok or :durable"
59
+ end
60
+
61
+ names = {}
62
+ vectors = []
63
+ row_count = nil
64
+ columns.each do |name, input|
65
+ normalized_name = column_name(name, names)
66
+ vector = input.is_a?(ColumnVector) ? input : infer(input, normalized_name)
67
+ row_count = establish_row_count(row_count, vector.values.length, normalized_name)
68
+ vectors << normalize_column(normalized_name, vector)
69
+ end
70
+ symbols.each do |name, input|
71
+ normalized_name = column_name(name, names)
72
+ values = input.is_a?(ColumnVector) ? input.values : array!(input, normalized_name)
73
+ if input.is_a?(ColumnVector) && input.kind != :varchar
74
+ raise ArgumentError, "symbol #{normalized_name.inspect} descriptor must use :varchar"
75
+ end
76
+ row_count = establish_row_count(row_count, values.length, normalized_name)
77
+ vectors << normalize_symbol(normalized_name, values)
78
+ end
79
+
80
+ row_count ||= 0
81
+ at_kind, at_data, at_count = normalize_at(at, row_count)
82
+ if row_count.zero?
83
+ if at_count && at_count.positive?
84
+ raise ArgumentError, "designated timestamp has #{at_count} rows but columns are empty"
85
+ end
86
+ return [table, [].freeze, AT_NOW, nil, 0, batch_size, ack_level].freeze
87
+ end
88
+ raise ArgumentError, "at vector has #{at_count} rows, expected #{row_count}" if at_count && at_count != row_count
89
+
90
+ [table, vectors.freeze, at_kind, at_data, row_count, batch_size, ack_level].freeze
91
+ end
92
+
93
+ def infer(input, name)
94
+ values = array!(input, name)
95
+ first = values.find { |value| !value.nil? }
96
+ if first.nil?
97
+ raise ArgumentError, "column #{name.inspect} is all nil; use QuestDB::ColumnVector with an explicit kind" unless values.empty?
98
+ return InferredVector.new(:long, values, nil, nil, nil)
99
+ end
100
+
101
+ kind = case first
102
+ when Integer then :long
103
+ when Float then :double
104
+ when TrueClass, FalseClass then :boolean
105
+ when String then :varchar
106
+ when Binary then :binary
107
+ when UUID then :uuid
108
+ when Long256 then :long256
109
+ when IPv4 then :ipv4
110
+ when TimestampNanos then :timestamp_nanos
111
+ when TimestampMicros then :timestamp
112
+ when DateMillis then :date
113
+ when Char then :char
114
+ when Geohash then :geohash
115
+ when DecimalValue then :decimal
116
+ when Float64Array then :double_array
117
+ else
118
+ raise TypeError, "cannot infer direct column type for #{name.inspect} from #{first.class}"
119
+ end
120
+ unless homogeneous_for_kind?(kind, values, first.class)
121
+ raise TypeError, "column #{name.inspect} is heterogeneous; use a homogeneous Array or explicit ColumnVector"
122
+ end
123
+ kind = decimal_kind(values) if kind == :decimal
124
+ metadata = case kind
125
+ when :geohash
126
+ bits = first.precision_bits
127
+ raise ArgumentError, "all geohashes in #{name.inspect} must have the same precision" unless values.all? { |v| v.nil? || v.precision_bits == bits }
128
+ { precision_bits: bits }
129
+ when :decimal64, :decimal128, :decimal256
130
+ scale = first.scale
131
+ raise ArgumentError, "all decimals in #{name.inspect} must have the same non-negative scale" unless values.all? { |v| v.nil? || v.scale == scale }
132
+ { scale: scale }
133
+ when :double_array
134
+ shape = first.shape
135
+ raise ArgumentError, "all arrays in #{name.inspect} must have the same shape" unless values.all? { |v| v.nil? || v.shape == shape }
136
+ { shape: shape }
137
+ else {}
138
+ end
139
+ InferredVector.new(kind, values, metadata[:scale], metadata[:precision_bits], metadata[:shape])
140
+ end
141
+
142
+ def normalize_column(name, vector)
143
+ kind = vector.kind
144
+ values = vector.values
145
+ if %i[byte short boolean char].include?(kind) && values.any?(&:nil?)
146
+ raise ArgumentError, "#{kind} cannot represent NULL in QuestDB; use :int, :varchar, or another nullable kind"
147
+ end
148
+ validity = validity_for(values)
149
+ data = case kind
150
+ when :byte, :short, :int, :long then pack_integers(values, kind)
151
+ when :float then pack_floats(values, "f")
152
+ when :double then pack_floats(values, "d")
153
+ when :boolean then pack_booleans(values)
154
+ when :varchar then pack_variable(values) { |v| utf8_string(v, "VARCHAR value", copy: false) }
155
+ when :binary then pack_variable(values) { |v| v.is_a?(Binary) ? v.bytes : String(v) }
156
+ when :uuid then values.map { |v| v.nil? ? ("\0" * 16) : (v.is_a?(UUID) ? v : UUID.new(v)).bytes }.join.b
157
+ when :long256 then values.map { |v| v.nil? ? ("\0" * 32) : (v.is_a?(Long256) ? v : Long256.new(v)).bytes }.join.b
158
+ when :ipv4 then values.map { |v| v.nil? ? 0 : (v.is_a?(IPv4) ? v : IPv4.new(v)).value }.pack("L*")
159
+ when :timestamp then values.map { |v| v.nil? ? 0 : timestamp_value(v, :micros) }.pack("q*")
160
+ when :timestamp_nanos then values.map { |v| v.nil? ? 0 : timestamp_value(v, :nanos) }.pack("q*")
161
+ when :date then values.map { |v| v.nil? ? 0 : date_value(v) }.pack("q*")
162
+ when :char then values.map { |v| v.nil? ? 0 : (v.is_a?(Char) ? v : Char.new(v)).value }.pack("S*")
163
+ when :geohash then pack_geohashes(values, vector.precision_bits)
164
+ when :decimal64, :decimal128, :decimal256 then pack_decimals(values, kind, vector.scale)
165
+ when :double_array then pack_double_arrays(values, vector.shape)
166
+ else raise ArgumentError, "unsupported direct column kind #{kind.inspect}"
167
+ end
168
+
169
+ descriptor = if %i[varchar binary].include?(kind)
170
+ offsets, bytes = data
171
+ [KIND_CODES.fetch(kind), name, offsets.freeze, validity, bytes.freeze, nil, 0, 0]
172
+ elsif kind == :geohash
173
+ [KIND_CODES.fetch(kind), name, data.freeze, validity, nil, nil, vector.precision_bits, 0]
174
+ elsif %i[decimal64 decimal128 decimal256].include?(kind)
175
+ [KIND_CODES.fetch(kind), name, data.freeze, validity, nil, nil, vector.scale, 0]
176
+ elsif kind == :double_array
177
+ shape = vector.shape.pack("L*").freeze
178
+ [KIND_CODES.fetch(kind), name, data.freeze, validity, shape, nil, vector.shape.length, 0]
179
+ else
180
+ [KIND_CODES.fetch(kind), name, data.freeze, validity, nil, nil, 0, 0]
181
+ end
182
+ descriptor.freeze
183
+ end
184
+
185
+ def normalize_symbol(name, values)
186
+ values = array!(values, name)
187
+ dictionary = {}
188
+ source_codes = {}
189
+ entries = []
190
+ codes = values.map do |value|
191
+ next 0 if value.nil?
192
+ source_codes.fetch(value) do
193
+ text = utf8_string(value, "SYMBOL value", copy: false)
194
+ index = dictionary.fetch(text) do
195
+ owned = text.dup.freeze
196
+ entries << owned
197
+ dictionary[owned] = entries.length - 1
198
+ end
199
+ source_codes[value] = index
200
+ end
201
+ end
202
+ offsets = [0]
203
+ bytes = +"".b
204
+ entries.each { |entry| bytes << entry; offsets << bytes.bytesize }
205
+ [KIND_CODES[:symbol], name, codes.pack("l*").freeze, validity_for(values), offsets.pack("l*").freeze,
206
+ bytes.force_encoding(Encoding::BINARY).freeze, 0, 0].freeze
207
+ end
208
+
209
+ def normalize_at(at, row_count)
210
+ return [AT_NOW, nil, nil] if at.equal?(SERVER_TIMESTAMP)
211
+ if at.is_a?(Array) || at.is_a?(ColumnVector)
212
+ vector = at.is_a?(ColumnVector) ? at : nil
213
+ values = vector ? vector.values : at
214
+ if values.any?(&:nil?)
215
+ raise ArgumentError, "designated timestamp vector cannot contain nil"
216
+ end
217
+ kind = if vector
218
+ raise ArgumentError, "at ColumnVector must be :timestamp or :timestamp_nanos" unless %i[timestamp timestamp_nanos].include?(vector.kind)
219
+ vector.kind
220
+ else
221
+ nonempty = values.first
222
+ case nonempty
223
+ when TimestampNanos, Time then :timestamp_nanos
224
+ when TimestampMicros then :timestamp
225
+ when nil then :timestamp
226
+ else raise TypeError, "at Array values must be TimestampNanos, TimestampMicros, or Time"
227
+ end
228
+ end
229
+ expected = kind == :timestamp_nanos ? [TimestampNanos, Time] : [TimestampMicros]
230
+ unless values.all? { |value| expected.any? { |klass| value.is_a?(klass) } }
231
+ raise TypeError, "designated timestamp vector is heterogeneous"
232
+ end
233
+ unit = kind == :timestamp_nanos ? :nanos : :micros
234
+ packed = values.map { |value| non_negative_timestamp(timestamp_value(value, unit)) }.pack("q*").freeze
235
+ [kind == :timestamp_nanos ? AT_NANOS : AT_MICROS, packed, values.length]
236
+ else
237
+ nanos = case at
238
+ when TimestampNanos then at.value
239
+ when TimestampMicros then checked_multiply(at.value, 1_000, "designated timestamp")
240
+ when Time then time_nanos(at)
241
+ else raise TypeError, "at must be a timestamp vector, TimestampNanos, TimestampMicros, Time, or SERVER_TIMESTAMP"
242
+ end
243
+ [AT_SCALAR_NANOS, non_negative_timestamp(nanos), nil]
244
+ end
245
+ end
246
+
247
+ def validity_for(values)
248
+ return nil unless values.any?(&:nil?)
249
+ bits = Array.new((values.length + 7) / 8, 0)
250
+ values.each_with_index { |value, index| bits[index / 8] |= 1 << (index % 8) unless value.nil? }
251
+ bits.pack("C*").freeze
252
+ end
253
+
254
+ def pack_integers(values, kind)
255
+ range = INTEGER_RANGES.fetch(kind)
256
+ converted = values.map do |value|
257
+ next 0 if value.nil?
258
+ number = Integer(value)
259
+ raise RangeError, "#{kind} value #{number} is out of range" unless range.cover?(number)
260
+ if (kind == :int || kind == :long) && number == range.begin
261
+ raise RangeError, "#{kind} minimum value is reserved as QuestDB NULL"
262
+ end
263
+ number
264
+ end
265
+ converted.pack({ byte: "c*", short: "s*", int: "l*", long: "q*" }.fetch(kind))
266
+ end
267
+
268
+ def pack_floats(values, directive)
269
+ values.map do |value|
270
+ next 0.0 if value.nil?
271
+ raise TypeError, "floating-point columns accept only Numeric values" unless value.is_a?(Numeric)
272
+ number = Float(value)
273
+ raise RangeError, "NaN is reserved as QuestDB NULL" if number.nan?
274
+ if directive == "f" && number.finite? && number.abs > 3.4028234663852886e38
275
+ raise RangeError, "float value is outside IEEE-754 binary32 range"
276
+ end
277
+ number
278
+ end.pack("#{directive}*")
279
+ end
280
+
281
+ def pack_booleans(values)
282
+ bits = Array.new((values.length + 7) / 8, 0)
283
+ values.each_with_index do |value, index|
284
+ next if value.nil? || value == false
285
+ raise TypeError, "boolean columns accept only true, false, or nil" unless value == true
286
+ bits[index / 8] |= 1 << (index % 8)
287
+ end
288
+ bits.pack("C*")
289
+ end
290
+
291
+ def pack_variable(values)
292
+ offsets = [0]
293
+ bytes = +"".b
294
+ values.each do |value|
295
+ bytes << yield(value) unless value.nil?
296
+ raise RangeError, "variable-width column exceeds 2 GiB" if bytes.bytesize > 0x7fff_ffff
297
+ offsets << bytes.bytesize
298
+ end
299
+ [offsets.pack("l*"), bytes.force_encoding(Encoding::BINARY)]
300
+ end
301
+
302
+ def pack_geohashes(values, bits)
303
+ limit = (1 << bits) - 1
304
+ numbers = values.map do |value|
305
+ next 0 if value.nil?
306
+ geohash = value.is_a?(Geohash) ? value : Geohash.new(value, precision_bits: bits)
307
+ raise ArgumentError, "geohash precision mismatch" unless geohash.precision_bits == bits
308
+ raise RangeError, "geohash value exceeds precision" unless geohash.value.between?(0, limit)
309
+ geohash.value
310
+ end
311
+ numbers.pack(bits <= 8 ? "C*" : bits <= 16 ? "S*" : bits <= 32 ? "L*" : "Q*")
312
+ end
313
+
314
+ def pack_decimals(values, kind, scale)
315
+ width = { decimal64: 8, decimal128: 16, decimal256: 32 }.fetch(kind)
316
+ values.map do |value|
317
+ mantissa = if value.nil?
318
+ 0
319
+ elsif value.is_a?(DecimalValue)
320
+ raise ArgumentError, "decimal scale mismatch" unless value.scale == scale
321
+ value.mantissa
322
+ else Integer(value)
323
+ end
324
+ min = -(1 << (width * 8 - 1)); max = (1 << (width * 8 - 1)) - 1
325
+ raise RangeError, "#{kind} mantissa is out of range" unless mantissa.between?(min, max)
326
+ twos_complement_bytes(mantissa, width)
327
+ end.join.b
328
+ end
329
+
330
+ def pack_double_arrays(values, shape)
331
+ size = shape.inject(1, :*)
332
+ values.map do |value|
333
+ next Array.new(size, 0.0).pack("d*") if value.nil?
334
+ array = value.is_a?(Float64Array) ? value : Float64Array.from_nested(value)
335
+ raise ArgumentError, "double_array shape mismatch" unless array.shape == shape
336
+ raise ArgumentError, "double_array value count mismatch" unless array.values.length == size
337
+ array.values.map { |item| Float(item) }.pack("d*")
338
+ end.join.b
339
+ end
340
+
341
+ def decimal_kind(values)
342
+ first = values.find { |value| !value.nil? }
343
+ scale = first.scale
344
+ raise RangeError, "decimal inference requires a non-negative scale" if scale.negative?
345
+ bits = values.reduce(0) do |maximum, value|
346
+ next maximum unless value
347
+ candidate = value.mantissa.bit_length + 1
348
+ candidate > maximum ? candidate : maximum
349
+ end
350
+ kind = bits <= 63 && scale <= 18 ? :decimal64 : bits <= 127 && scale <= 38 ? :decimal128 : :decimal256
351
+ cap = { decimal64: 18, decimal128: 38, decimal256: 76 }.fetch(kind)
352
+ raise RangeError, "decimal cannot fit a direct QuestDB decimal type" if bits > 255 || scale > cap
353
+ kind
354
+ end
355
+
356
+ def homogeneous_for_kind?(kind, values, expected_class)
357
+ case kind
358
+ when :boolean then values.all? { |value| value.nil? || value == true || value == false }
359
+ when :varchar then values.all? { |value| value.nil? || value.class == String }
360
+ else values.all? { |value| value.nil? || value.class == expected_class }
361
+ end
362
+ end
363
+
364
+ def timestamp_value(value, unit)
365
+ result = case value
366
+ when TimestampNanos
367
+ unit == :nanos ? value.value : value.value.div(1_000)
368
+ when TimestampMicros
369
+ unit == :micros ? value.value : checked_multiply(value.value, 1_000, "timestamp")
370
+ when Time
371
+ unit == :nanos ? time_nanos(value) : time_nanos(value).div(1_000)
372
+ else Integer(value)
373
+ end
374
+ check_i64(result, "timestamp")
375
+ end
376
+
377
+ def date_value(value)
378
+ check_i64(value.is_a?(DateMillis) ? value.value : Integer(value), "date")
379
+ end
380
+
381
+ def time_nanos(time)
382
+ check_i64(time.to_i * 1_000_000_000 + time.nsec, "timestamp")
383
+ end
384
+
385
+ def non_negative_timestamp(value)
386
+ raise RangeError, "designated timestamp cannot be before the Unix epoch" if value.negative?
387
+ value
388
+ end
389
+
390
+ def checked_multiply(value, multiplier, label)
391
+ check_i64(Integer(value) * multiplier, label)
392
+ end
393
+
394
+ def check_i64(value, label)
395
+ raise RangeError, "#{label} is outside signed 64-bit range" unless value.between?(-(1 << 63), (1 << 63) - 1)
396
+ value
397
+ end
398
+
399
+ def twos_complement_bytes(value, width)
400
+ unsigned = value.negative? ? (1 << (width * 8)) + value : value
401
+ bytes = Array.new(width) { |index| (unsigned >> (index * 8)) & 0xff }
402
+ bytes.reverse! unless [1].pack("S") == "\x01\x00".b
403
+ bytes.pack("C*")
404
+ end
405
+
406
+ def establish_row_count(current, candidate, name)
407
+ return candidate if current.nil?
408
+ raise ArgumentError, "column #{name.inspect} has #{candidate} rows, expected #{current}" unless candidate == current
409
+ current
410
+ end
411
+
412
+ def column_name(name, names)
413
+ normalized = utf8_string(name.is_a?(Symbol) ? name.to_s : name, "column name").freeze
414
+ raise ArgumentError, "column name must not be empty" if normalized.empty?
415
+ raise ArgumentError, "duplicate column #{normalized.inspect}" if names.key?(normalized)
416
+ names[normalized] = true
417
+ normalized
418
+ end
419
+
420
+ def array!(value, name)
421
+ raise TypeError, "column #{name.inspect} must be an Array or ColumnVector" unless value.is_a?(Array)
422
+ value
423
+ end
424
+
425
+ def utf8_string(value, label, copy: true)
426
+ string = String(value)
427
+ if string.encoding == Encoding::UTF_8
428
+ raise EncodingError, "#{label} is not valid UTF-8" unless string.valid_encoding?
429
+ return copy ? string.dup : string
430
+ end
431
+ if string.ascii_only?
432
+ return string unless copy
433
+ return string.dup.force_encoding(Encoding::UTF_8)
434
+ end
435
+ string.encode(Encoding::UTF_8)
436
+ rescue Encoding::UndefinedConversionError, Encoding::InvalidByteSequenceError => error
437
+ raise EncodingError, "#{label} is not valid UTF-8: #{error.message}"
438
+ end
439
+ end
440
+ end
@@ -0,0 +1,60 @@
1
+ # frozen_string_literal: true
2
+
3
+ # ___ _ ____ ____
4
+ # / _ \ _ _ ___ ___| |_| _ \| __ )
5
+ # | | | | | | |/ _ \/ __| __| | | | _ \
6
+ # | |_| | |_| | __/\__ \ |_| |_| | |_) |
7
+ # \__\_\\__,_|\___||___/\__|____/|____/
8
+ #
9
+ # Copyright (c) 2014-2019 Appsicle
10
+ # Copyright (c) 2019-2026 QuestDB
11
+ #
12
+ # Licensed under the Apache License, Version 2.0 (the "License");
13
+ # you may not use this file except in compliance with the License.
14
+ # You may obtain a copy of the License at
15
+ #
16
+ # http://www.apache.org/licenses/LICENSE-2.0
17
+ #
18
+ # Unless required by applicable law or agreed to in writing, software
19
+ # distributed under the License is distributed on an "AS IS" BASIS,
20
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
21
+ # See the License for the specific language governing permissions and
22
+ # limitations under the License.
23
+ #
24
+ require "thread"
25
+
26
+ module QuestDB
27
+ # Internal bounded, drop-oldest queue used after native callbacks have been
28
+ # copied into Ruby-owned values. Callback dispatch never waits on consumers.
29
+ class EventInbox
30
+ def initialize(capacity)
31
+ @capacity = capacity.zero? ? 64 : capacity
32
+ @items = []
33
+ @mutex = Mutex.new
34
+ @delivered = 0
35
+ @dropped = 0
36
+ end
37
+
38
+ def push(value)
39
+ @mutex.synchronize do
40
+ if @items.length == @capacity
41
+ @items.shift
42
+ @dropped += 1
43
+ end
44
+ @items << value
45
+ end
46
+ end
47
+
48
+ def poll
49
+ @mutex.synchronize do
50
+ value = @items.shift
51
+ @delivered += 1 if value
52
+ value
53
+ end
54
+ end
55
+
56
+ def delivered = @mutex.synchronize { @delivered }
57
+ def dropped = @mutex.synchronize { @dropped }
58
+ end
59
+ private_constant :EventInbox
60
+ end
@@ -0,0 +1,104 @@
1
+ # frozen_string_literal: true
2
+
3
+ # ___ _ ____ ____
4
+ # / _ \ _ _ ___ ___| |_| _ \| __ )
5
+ # | | | | | | |/ _ \/ __| __| | | | _ \
6
+ # | |_| | |_| | __/\__ \ |_| |_| | |_) |
7
+ # \__\_\\__,_|\___||___/\__|____/|____/
8
+ #
9
+ # Copyright (c) 2014-2019 Appsicle
10
+ # Copyright (c) 2019-2026 QuestDB
11
+ #
12
+ # Licensed under the Apache License, Version 2.0 (the "License");
13
+ # you may not use this file except in compliance with the License.
14
+ # You may obtain a copy of the License at
15
+ #
16
+ # http://www.apache.org/licenses/LICENSE-2.0
17
+ #
18
+ # Unless required by applicable law or agreed to in writing, software
19
+ # distributed under the License is distributed on an "AS IS" BASIS,
20
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
21
+ # See the License for the specific language governing permissions and
22
+ # limitations under the License.
23
+ #
24
+
25
+ module QuestDB
26
+ module Ingress
27
+ class Buffer
28
+ MAX_PREALLOCATE_BYTES = 100 * 1024 * 1024
29
+ EMPTY_HASH = {}.freeze
30
+ private_constant :EMPTY_HASH
31
+
32
+ class << self
33
+ private
34
+
35
+ def from_native(native)
36
+ object = allocate
37
+ object.send(:initialize_native, native)
38
+ object
39
+ end
40
+
41
+ private :new
42
+ end
43
+
44
+ def row(table, symbols: EMPTY_HASH, columns: EMPTY_HASH, at:)
45
+ append(table, symbols, columns, at)
46
+ self
47
+ end
48
+
49
+ def row_count = @native.row_count
50
+ alias length row_count
51
+ def empty? = row_count.zero?
52
+
53
+ # QWP encodes at publication time. These values are only memory hints.
54
+ def bytesize = @native.bytesize
55
+ def capacity = @native.capacity
56
+
57
+ def reserve(additional)
58
+ additional = Integer(additional)
59
+ raise ArgumentError, "additional must be non-negative" if additional.negative?
60
+ raise RangeError, "buffer preallocation exceeds 100 MiB" if additional > MAX_PREALLOCATE_BYTES
61
+ @native.reserve(additional)
62
+ self
63
+ end
64
+
65
+ def clear = (@native.clear; self)
66
+ def initialize_copy(*) = raise(TypeError, "QuestDB buffers cannot be copied")
67
+
68
+ protected
69
+
70
+ attr_reader :native
71
+
72
+ private
73
+
74
+ def initialize_native(native)
75
+ @native = native
76
+ end
77
+
78
+ # Positional internal entry point avoids constructing a second keyword
79
+ # Hash when Sender delegates its ergonomic keyword API to the buffer.
80
+ def append(table, symbols, columns, at)
81
+ raise TypeError, "symbols must be a Hash" unless symbols.is_a?(Hash)
82
+ raise TypeError, "columns must be a Hash" unless columns.is_a?(Hash)
83
+ return false unless hash_has_value?(symbols) || hash_has_value?(columns)
84
+ @native.append_row(table, symbols, columns, normalize_at(at))
85
+ rescue EncodingError => error
86
+ raise IngressError.new(error.message, code: :invalid_utf8), cause: error
87
+ end
88
+
89
+ def hash_has_value?(hash)
90
+ hash.each_value { |value| return true unless value.nil? }
91
+ false
92
+ end
93
+
94
+ def normalize_at(value)
95
+ case value
96
+ when TimestampNanos, TimestampMicros, Time then value
97
+ else
98
+ return value if value.equal?(SERVER_TIMESTAMP)
99
+ raise IngressError.new("at must be TimestampNanos, TimestampMicros, Time, or SERVER_TIMESTAMP", code: :invalid_timestamp)
100
+ end
101
+ end
102
+ end
103
+ end
104
+ end
@@ -0,0 +1,43 @@
1
+ # ___ _ ____ ____
2
+ # / _ \ _ _ ___ ___| |_| _ \| __ )
3
+ # | | | | | | |/ _ \/ __| __| | | | _ \
4
+ # | |_| | |_| | __/\__ \ |_| |_| | |_) |
5
+ # \__\_\\__,_|\___||___/\__|____/|____/
6
+ #
7
+ # Copyright (c) 2014-2019 Appsicle
8
+ # Copyright (c) 2019-2026 QuestDB
9
+ #
10
+ # Licensed under the Apache License, Version 2.0 (the "License");
11
+ # you may not use this file except in compliance with the License.
12
+ # You may obtain a copy of the License at
13
+ #
14
+ # http://www.apache.org/licenses/LICENSE-2.0
15
+ #
16
+ # Unless required by applicable law or agreed to in writing, software
17
+ # distributed under the License is distributed on an "AS IS" BASIS,
18
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
19
+ # See the License for the specific language governing permissions and
20
+ # limitations under the License.
21
+ #
22
+ module QuestDB
23
+ module Ingress
24
+ class IngressError < StandardError
25
+ attr_reader :code, :sender_error
26
+
27
+ def initialize(message, code: :invalid_api_call, in_doubt: false, sender_error: nil)
28
+ @code = code.to_sym
29
+ @in_doubt = !!in_doubt
30
+ @sender_error = if sender_error.is_a?(Array) && defined?(QuestDB::SenderError)
31
+ QuestDB::SenderError.from_native(sender_error)
32
+ else
33
+ sender_error
34
+ end
35
+ super(message)
36
+ end
37
+
38
+ def in_doubt? = @in_doubt
39
+ end
40
+
41
+ class ServerRejectionError < IngressError; end
42
+ end
43
+ end