questdb 0.1.0-aarch64-linux-gnu

data/lib/questdb/ingress/float64_array.rb

@@ -0,0 +1,60 @@
+#      ___                  _   ____  ____
+#     / _ \ _   _  ___  ___| |_|  _ \| __ )
+#    | | | | | | |/ _ \/ __| __| | | |  _ \
+#    | |_| | |_| |  __/\__ \ |_| |_| | |_) |
+#     \__\_\\__,_|\___||___/\__|____/|____/
+#
+#   Copyright (c) 2014-2019 Appsicle
+#   Copyright (c) 2019-2026 QuestDB
+#
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+#
+module QuestDB
+  module Ingress
+    class Float64Array
+      attr_reader :values, :shape
+      def initialize(values, shape:)
+        raise TypeError, "values and shape must be Arrays" unless values.is_a?(Array) && shape.is_a?(Array)
+        @shape = shape.map { |d| Integer(d) }
+        raise ArgumentError, "dimensions must be non-negative" unless @shape.all? { |dimension| dimension >= 0 }
+        expected = if @shape.empty?
+                     0
+                   else
+                     @shape.reduce(1) do |n, d|
+                       next 0 if d.zero? || n.zero?
+                       raise RangeError, "shape is too large" if n > ((1 << 63) - 1) / d
+                       n * d
+                     end
+                   end
+        raise ArgumentError, "shape requires #{expected} values, got #{values.length}" unless expected == values.length
+        @values = values.map { |v| Float(v) }.freeze
+        @shape.freeze
+        freeze
+      end
+      def self.from_native(values, shape) = new(values, shape: shape)
+      def self.from_nested(value)
+        raise TypeError, "nested value must be an Array" unless value.is_a?(Array)
+        walk = lambda do |node|
+          return [[], [Float(node)]] unless node.is_a?(Array)
+          return [[0], []] if node.empty?
+          children = node.map { |child| walk.call(child) }
+          child_shape = children.first.first
+          raise ArgumentError, "ragged nested array" unless children.all? { |shape, _| shape == child_shape }
+          [[node.length, *child_shape], children.flat_map(&:last)]
+        end
+        shape, values = walk.call(value)
+        new(values, shape: shape)
+      end
+    end
+  end
+end

data/lib/questdb/ingress/native.rb

@@ -0,0 +1,40 @@
+#      ___                  _   ____  ____
+#     / _ \ _   _  ___  ___| |_|  _ \| __ )
+#    | | | | | | |/ _ \/ __| __| | | |  _ \
+#    | |_| | |_| |  __/\__ \ |_| |_| | |_) |
+#     \__\_\\__,_|\___||___/\__|____/|____/
+#
+#   Copyright (c) 2014-2019 Appsicle
+#   Copyright (c) 2019-2026 QuestDB
+#
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+#
+# Source installs place the extension on RubyGems' extension load path.
+# Precompiled gems carry one binary per supported Ruby minor because MRI's C
+# ABI and libruby soname are not stable across minor releases.
+begin
+  require "questdb_ingress"
+rescue LoadError => source_error
+  require "rbconfig"
+  ruby_abi = RUBY_VERSION[/\A\d+\.\d+/]
+  native = File.expand_path("../native/#{ruby_abi}/questdb_ingress", __dir__)
+  if File.file?("#{native}.#{RbConfig::CONFIG.fetch('DLEXT')}")
+    require native
+  else
+    begin
+      require_relative "../../../ext/questdb_ingress/questdb_ingress"
+    rescue LoadError
+      raise source_error
+    end
+  end
+end

data/lib/questdb/ingress/timestamp.rb

@@ -0,0 +1,53 @@
+#      ___                  _   ____  ____
+#     / _ \ _   _  ___  ___| |_|  _ \| __ )
+#    | | | | | | |/ _ \/ __| __| | | |  _ \
+#    | |_| | |_| |  __/\__ \ |_| |_| | |_) |
+#     \__\_\\__,_|\___||___/\__|____/|____/
+#
+#   Copyright (c) 2014-2019 Appsicle
+#   Copyright (c) 2019-2026 QuestDB
+#
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+#
+module QuestDB
+  module Ingress
+    class Timestamp
+      MIN = -(1 << 63)
+      MAX = (1 << 63) - 1
+      attr_reader :value
+      def initialize(value)
+        raise TypeError, "timestamp must be an Integer" unless value.is_a?(Integer)
+        raise RangeError, "timestamp must fit signed 64-bit" unless value.between?(MIN, MAX)
+        @value = value
+        freeze
+      end
+      def ==(other) = other.class == self.class && other.value == value
+    end
+    class TimestampNanos < Timestamp
+      def self.now = new(Process.clock_gettime(Process::CLOCK_REALTIME, :nanosecond))
+      def self.from_time(time)
+        raise TypeError, "expected Time" unless time.is_a?(Time)
+        new(time.to_i * 1_000_000_000 + time.nsec)
+      end
+    end
+    class TimestampMicros < Timestamp
+      def self.now = new(Process.clock_gettime(Process::CLOCK_REALTIME, :microsecond))
+      def self.from_time(time)
+        raise TypeError, "expected Time" unless time.is_a?(Time)
+        new(time.to_i * 1_000_000 + time.usec)
+      end
+    end
+    SERVER_TIMESTAMP = Object.new.freeze
+    ServerTimestamp = SERVER_TIMESTAMP
+  end
+end

data/lib/questdb/ingress.rb

@@ -0,0 +1,77 @@
+# frozen_string_literal: true
+
+#      ___                  _   ____  ____
+#     / _ \ _   _  ___  ___| |_|  _ \| __ )
+#    | | | | | | |/ _ \/ __| __| | | |  _ \
+#    | |_| | |_| |  __/\__ \ |_| |_| | |_) |
+#     \__\_\\__,_|\___||___/\__|____/|____/
+#
+#   Copyright (c) 2014-2019 Appsicle
+#   Copyright (c) 2019-2026 QuestDB
+#
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+#
+require "bigdecimal"
+require_relative "version"
+require_relative "ingress/error"
+require_relative "ingress/timestamp"
+require_relative "ingress/float64_array"
+require_relative "types"
+require_relative "column_vector"
+require_relative "direct_ingestion"
+require_relative "query"
+require_relative "ingress/native"
+require_relative "config"
+require_relative "sender_error"
+require_relative "connection_event"
+require_relative "event_inbox"
+require_relative "ingress/buffer"
+require_relative "pooled_sender"
+require_relative "sender"
+require_relative "client"
+
+module QuestDB
+  VERSION = Ingress::VERSION unless const_defined?(:VERSION, false)
+  C_CLIENT_VERSION = Ingress::C_CLIENT_VERSION unless const_defined?(:C_CLIENT_VERSION, false)
+  Error = Ingress::IngressError unless const_defined?(:Error, false)
+  QuestDBError = Ingress::IngressError unless const_defined?(:QuestDBError, false)
+  ServerRejectionError = Ingress::ServerRejectionError unless const_defined?(:ServerRejectionError, false)
+  QuestDBServerRejectionError = Ingress::ServerRejectionError unless const_defined?(:QuestDBServerRejectionError, false)
+  Timestamp = Ingress::Timestamp unless const_defined?(:Timestamp, false)
+  TimestampNanos = Ingress::TimestampNanos unless const_defined?(:TimestampNanos, false)
+  TimestampMicros = Ingress::TimestampMicros unless const_defined?(:TimestampMicros, false)
+  Float64Array = Ingress::Float64Array unless const_defined?(:Float64Array, false)
+  SERVER_TIMESTAMP = Ingress::SERVER_TIMESTAMP unless const_defined?(:SERVER_TIMESTAMP, false)
+
+  module_function
+
+  def connect(conf = nil, host: nil, port: nil, tls: nil, connection_listener: nil,
+              error_handler: nil,
+              connection_event_inbox_capacity: Client::DEFAULT_EVENT_INBOX_CAPACITY,
+              error_event_inbox_capacity: Client::DEFAULT_EVENT_INBOX_CAPACITY, **settings)
+    configuration, effective = Config.build(conf, host: host, port: port, tls: tls, settings: settings)
+    client = Client.new(
+      configuration, effective_settings: effective,
+      connection_listener: connection_listener, error_handler: error_handler,
+      connection_event_inbox_capacity: connection_event_inbox_capacity,
+      error_event_inbox_capacity: error_event_inbox_capacity
+    )
+    return client unless block_given?
+
+    begin
+      yield client
+    ensure
+      client.close
+    end
+  end
+end

data/lib/questdb/pooled_sender.rb

@@ -0,0 +1,215 @@
+# frozen_string_literal: true
+
+#      ___                  _   ____  ____
+#     / _ \ _   _  ___  ___| |_|  _ \| __ )
+#    | | | | | | |/ _ \/ __| __| | | |  _ \
+#    | |_| | |_| |  __/\__ \ |_| |_| | |_) |
+#     \__\_\\__,_|\___||___/\__|____/|____/
+#
+#   Copyright (c) 2014-2019 Appsicle
+#   Copyright (c) 2019-2026 QuestDB
+#
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+#
+
+module QuestDB
+  class PooledSender
+    ACK_LEVELS = { ok: 0, durable: 1 }.freeze
+    EMPTY_HASH = {}.freeze
+    private_constant :EMPTY_HASH
+
+    attr_reader :auto_flush_rows, :auto_flush_bytes, :auto_flush_interval
+
+    def initialize(client, native, auto_flush:)
+      @client = client
+      @native = native
+      @owner_thread = Thread.current
+      @closed = false
+      @buffer = Ingress::Buffer.send(:from_native, @native.new_buffer)
+      @auto_flush_rows = auto_flush.fetch(:rows)
+      @auto_flush_bytes = auto_flush.fetch(:bytes)
+      @auto_flush_interval = auto_flush.fetch(:interval)
+      @dynamic_auto_flush_bytes = auto_flush.fetch(:dynamic_bytes)
+      @batch_started = nil
+    end
+
+    def row(table, symbols: EMPTY_HASH, columns: EMPTY_HASH, at:)
+      ensure_open!
+      appended = @buffer.__send__(:append, table, symbols, columns, at)
+      return self unless appended
+      @batch_started ||= monotonic
+      flush if should_auto_flush?
+      self
+    end
+
+    def row_count = (@buffer&.row_count || 0)
+    alias length row_count
+    def empty? = row_count.zero?
+    def closed? = @closed || @native.closed?
+
+    def flush(wait: false, ack: :ok)
+      ensure_open!
+      level = normalize_ack(ack)
+      raise ArgumentError, "ack: :durable requires wait: true" if level == ACK_LEVELS[:durable] && !wait
+      return self if empty? && !wait
+      @native.flush(@buffer.send(:native), !!wait, level)
+      @batch_started = nil if @buffer.empty?
+      self
+    end
+
+    def wait(timeout_millis: 0, ack: :ok)
+      ensure_open!
+      timeout = non_negative_integer(timeout_millis, "timeout_millis")
+      @native.wait(normalize_ack(ack), timeout)
+      self
+    end
+
+    def flush_and_get_fsn
+      ensure_open!
+      fsn = @native.flush_fsn(@buffer.send(:native), false)
+      @batch_started = nil if @buffer.empty?
+      fsn
+    end
+
+    def flush_and_keep_and_get_fsn
+      ensure_open!
+      @native.flush_fsn(@buffer.send(:native), true)
+    end
+
+    def published_fsn
+      ensure_open!
+      @native.published_fsn
+    end
+
+    def acked_fsn
+      ensure_open!
+      @native.acked_fsn
+    end
+
+    def await_acked_fsn(fsn, timeout_millis: 0)
+      ensure_open!
+      fsn = non_negative_integer(fsn, "fsn")
+      timeout = non_negative_integer(timeout_millis, "timeout_millis")
+      current = @native.acked_fsn
+      return true if current && current >= fsn
+      begin
+        @native.wait(ACK_LEVELS[:ok], timeout)
+      rescue Ingress::IngressError => error
+        raise unless error.code == :failover_retry
+      end
+      current = @native.acked_fsn
+      !!(current && current >= fsn)
+    end
+
+    # Convenience parity with Client#ingest_columns. This uses a separate
+    # direct connection and therefore has no ordering relationship with this
+    # lease's buffered row stream.
+    def ingest_columns(table, columns:, symbols: EMPTY_HASH, at:, max_rows_per_batch: 16_384, ack: :ok)
+      ensure_open!
+      @client.ingest_columns(table, columns: columns, symbols: symbols, at: at,
+                            max_rows_per_batch: max_rows_per_batch, ack: ack)
+      self
+    end
+    alias write_columns ingest_columns
+
+    def poll_error
+      ensure_open!
+      values = @native.poll_error
+      values && SenderError.from_native(values)
+    end
+
+    def error_events_dropped
+      ensure_open!
+      @native.error_events_dropped
+    end
+
+    def close(flush: true, wait: false)
+      return self if @closed
+      ensure_thread!
+      begin
+        if flush && !empty?
+          self.flush(wait: wait)
+        elsif wait
+          self.wait
+        end
+      ensure
+        @buffer&.clear
+        @native.close(false)
+        @closed = true
+        @client.send(:sender_released, self)
+      end
+      self
+    end
+
+    def open
+      raise ArgumentError, "a block is required" unless block_given?
+      begin
+        result = yield self
+        close
+        result
+      rescue Exception
+        close(flush: false)
+        raise
+      end
+    end
+
+    def initialize_copy(*) = raise(TypeError, "QuestDB pooled senders cannot be copied")
+
+    private
+
+    def ensure_thread!
+      return if Thread.current.equal?(@owner_thread)
+      raise Ingress::IngressError.new("pooled sender must be used on the thread that borrowed it", code: :invalid_api_call)
+    end
+
+    def ensure_open!
+      ensure_thread!
+      raise Ingress::IngressError.new("pooled sender is closed", code: :invalid_api_call) if closed?
+    end
+
+    def normalize_ack(value)
+      value = value.to_sym if value.respond_to?(:to_sym)
+      ACK_LEVELS.fetch(value) do
+        raise ArgumentError, "ack must be :ok or :durable"
+      end
+    end
+
+    def non_negative_integer(value, name)
+      value = Integer(value)
+      raise ArgumentError, "#{name} must be non-negative" if value.negative?
+      value
+    end
+
+    def should_auto_flush?
+      return false unless @batch_started
+      return true if @auto_flush_rows && row_count >= @auto_flush_rows
+      if @dynamic_auto_flush_bytes || @auto_flush_bytes
+        encoded_cap = @native.effective_frame_cap
+        server_cap_known = encoded_cap.positive?
+        hard_cap = encoded_cap.abs
+        soft_cap = (hard_cap / 10) * 9 + ((hard_cap % 10) * 9) / 10
+        byte_limit = if @dynamic_auto_flush_bytes
+                       server_cap_known ? soft_cap : [soft_cap, 8 * 1024 * 1024].min
+                     else
+                       [@auto_flush_bytes, soft_cap].min
+                     end
+        return true if @buffer.bytesize >= byte_limit
+      end
+      @auto_flush_interval && monotonic - @batch_started >= @auto_flush_interval
+    end
+
+    def monotonic
+      Process.clock_gettime(Process::CLOCK_MONOTONIC, :millisecond)
+    end
+  end
+end