visionary-sdk 0.4.0

data/lib/visionary/schema.rb

@@ -0,0 +1,163 @@
+# frozen_string_literal: true
+
+module Visionary
+  # Table definitions shared by every store: the shipped CREATE scripts (copies
+  # of sdks/go/schema), column orders and the upgrades for older tables
+  # (sdks/go/migrate.go).
+  module Schema
+    DIR = File.expand_path("schema", __dir__)
+
+    REQUEST_COLUMNS = %w[
+      request_id created_at method path route query status duration_ms ip user_agent
+      user_id user_handle user_role request_headers request_content_type request_body
+      request_body_truncated request_size response_content_type response_body
+      response_body_truncated response_size error_code error_key
+    ].freeze
+
+    # api_logs columns added after the first release, in insert order.
+    ADDED_REQUEST_COLUMNS = [
+      %w[service text], %w[version text], %w[db_count int], %w[db_ms real], %w[http_count int], %w[http_ms real]
+    ].freeze
+
+    SYSTEM_BASE_COLUMNS = %w[
+      created_at instance interval_s
+      host_cpu_percent load_1 load_5 load_15
+      host_mem_used_mb host_mem_total_mb host_mem_percent
+      disk_used_percent disk_free_gb net_rx_kbps net_tx_kbps
+      proc_cpu_percent proc_rss_mb proc_threads proc_open_fds
+      goroutines heap_alloc_mb heap_inuse_mb heap_objects
+      gc_count gc_pause_ms gc_cpu_percent
+      db_total_conns db_idle_conns db_acquired_conns db_max_conns
+      db_acquires db_empty_acquires db_acquire_wait_ms
+      http_requests http_errors http_rps http_in_flight
+      uptime_s
+    ].freeze
+
+    # system_metrics columns added after the first release, in insert order.
+    ADDED_SYSTEM_COLUMNS = [
+      %w[cpu_count int], %w[cpu_user_percent real], %w[cpu_system_percent real],
+      %w[cpu_iowait_percent real], %w[cpu_steal_percent real],
+      %w[host_mem_available_mb real], %w[host_mem_cached_mb real],
+      %w[swap_used_mb real], %w[swap_total_mb real],
+      %w[disk_total_gb real], %w[disk_inodes_percent real],
+      %w[disk_read_kbps real], %w[disk_write_kbps real], %w[disk_read_iops real],
+      %w[disk_write_iops real], %w[disk_busy_percent real], %w[disk_await_ms real],
+      %w[net_rx_pps real], %w[net_tx_pps real], %w[net_errors int], %w[net_drops int],
+      %w[tcp_established int], %w[tcp_retrans int],
+      %w[host_procs int], %w[procs_running int], %w[procs_blocked int],
+      %w[ctx_switches_ps real], %w[proc_read_kbps real], %w[proc_write_kbps real],
+      %w[proc_max_fds int], %w[service text], %w[version text],
+      %w[sdk text], %w[runtime text]
+    ].freeze
+
+    SPAN_COLUMNS = %w[
+      created_at request_id service instance kind name target detail offset_ms duration_ms status error row_count
+    ].freeze
+
+    ERROR_COLUMNS = %w[
+      created_at service instance version request_id method route path user_id
+      type message stack fingerprint handled runtime
+    ].freeze
+
+    RELEASE_COLUMNS = %w[created_at service instance version sdk sdk_version runtime].freeze
+
+    TABLES = {
+      "api_logs" => REQUEST_COLUMNS + ADDED_REQUEST_COLUMNS.map(&:first),
+      "system_metrics" => SYSTEM_BASE_COLUMNS + ADDED_SYSTEM_COLUMNS.map(&:first),
+      "spans" => SPAN_COLUMNS,
+      "errors" => ERROR_COLUMNS,
+      "releases" => RELEASE_COLUMNS
+    }.freeze
+
+    # Which script creates a table, and the columns older copies may lack.
+    GROUPS = {
+      requests: { file: "requests.sql", table: "api_logs", added: ADDED_REQUEST_COLUMNS },
+      system: { file: "system.sql", table: "system_metrics", added: ADDED_SYSTEM_COLUMNS },
+      telemetry: { file: "telemetry.sql", table: nil, added: [] }
+    }.freeze
+
+    PURGE_CHUNK = 5000
+
+    module_function
+
+    def folder(dialect)
+      %w[postgres mysql].include?(dialect) ? dialect : "sqlite"
+    end
+
+    def statements(dialect, group)
+      script = File.read(File.join(DIR, folder(dialect), GROUPS.fetch(group)[:file]), encoding: "UTF-8")
+      script.split(";").map(&:strip).reject(&:empty?)
+    end
+
+    def columns_query(dialect, table)
+      case dialect
+      when "postgres"
+        "SELECT column_name FROM information_schema.columns WHERE table_schema = current_schema() AND table_name = '#{table}'"
+      when "mysql"
+        "SELECT column_name FROM information_schema.columns WHERE table_schema = DATABASE() AND table_name = '#{table}'"
+      else
+        "SELECT name FROM pragma_table_info('#{table}')"
+      end
+    end
+
+    def sql_type(dialect, kind)
+      case kind
+      when "text" then dialect == "mysql" ? "VARCHAR(255)" : "TEXT"
+      when "int" then %w[postgres mysql].include?(dialect) ? "BIGINT" : "INTEGER"
+      else { "postgres" => "DOUBLE PRECISION", "mysql" => "DOUBLE" }.fetch(dialect, "REAL")
+      end
+    end
+
+    # ALTER statements for the added columns missing from existing names.
+    def missing_columns(dialect, table, added, existing)
+      have = existing.map { |c| c.to_s.downcase }
+      added.reject { |name, _| have.include?(name) }
+           .map { |name, kind| "ALTER TABLE #{table} ADD COLUMN #{name} #{sql_type(dialect, kind)}" }
+    end
+
+    # An ALTER that lost a race with another instance.
+    def duplicate_column?(error)
+      msg = error.message.downcase
+      msg.include?("duplicate column") || msg.include?("already exists") || msg.include?("1060")
+    end
+
+    # MySQL rejects LIMIT inside IN (subquery) but allows DELETE ... LIMIT;
+    # the others are the other way round.
+    def purge_sql(dialect, table)
+      if dialect == "mysql"
+        "DELETE FROM #{table} WHERE created_at < ? ORDER BY id LIMIT #{PURGE_CHUNK}"
+      else
+        "DELETE FROM #{table} WHERE id IN (SELECT id FROM #{table} WHERE created_at < ? LIMIT #{PURGE_CHUNK})"
+      end
+    end
+
+    # D1 takes string params only, so there each placeholder becomes
+    # NULLIF(?, '') and nulls travel as empty strings.
+    def insert_sql(table, string_params)
+      cols = TABLES.fetch(table)
+      mark = string_params ? "NULLIF(?, '')" : "?"
+      "INSERT INTO #{table} (#{cols.join(', ')}) VALUES (#{([mark] * cols.size).join(', ')})"
+    end
+
+    # MySQL in strict mode rejects over-long values, which would fail the
+    # whole batch, so values are cut to the column size first: characters for
+    # VARCHAR(n), bytes for TEXT and MEDIUMTEXT. Read from the shipped schema.
+    def mysql_limits
+      @mysql_limits ||= begin
+        limits = Hash.new { |h, k| h[k] = {} }
+        GROUPS.each_value do |g|
+          table = nil
+          File.foreach(File.join(DIR, "mysql", g[:file]), encoding: "UTF-8") do |line|
+            if (m = line.match(/CREATE TABLE IF NOT EXISTS (\w+)/))
+              table = m[1]
+            elsif table && (m = line.match(/\A\s*(\w+)\s+(VARCHAR\((\d+)\)|MEDIUMTEXT|TEXT)/))
+              limits[table][m[1]] = m[3] ? [:chars, m[3].to_i] : [:bytes, m[2] == "TEXT" ? 65_535 : 16_777_215]
+            end
+          end
+        end
+        limits.default_proc = nil
+        limits.freeze
+      end
+    end
+  end
+end

data/lib/visionary/schemas.rb

@@ -0,0 +1,64 @@
+# frozen_string_literal: true
+
+module Visionary
+  # Creates tables on first use and retries on later writes until it works,
+  # so a database that is down at startup doesn't leave tables missing.
+  # Older tables gain the columns added since (ALTER TABLE ... ADD COLUMN).
+  class Schemas
+    def initialize(store, logger)
+      @store = store
+      @logger = logger
+      @ready = {}
+      @reported = {}
+      @mutex = Mutex.new
+    end
+
+    # A fork can happen while a parent thread holds the lock.
+    def after_fork
+      @mutex = Mutex.new
+    end
+
+    def ensure(group)
+      @mutex.synchronize do
+        return true if @ready[group]
+        attempt(group)
+        @ready[group] = true
+      rescue StandardError => e
+        unless @reported[group]
+          @reported[group] = true
+          @logger.error("visionary: create #{group} tables failed (will retry): #{e.message}")
+        end
+        false
+      end
+    end
+
+    private
+
+    # CREATE ... IF NOT EXISTS is not race-safe in PostgreSQL: when several
+    # processes start at once, one may fail while another creates the table.
+    def attempt(group)
+      tries = 0
+      begin
+        create(group)
+      rescue StandardError
+        raise if (tries += 1) >= 3
+        sleep(0.25 * tries)
+        retry
+      end
+    end
+
+    def create(group)
+      dialect = @store.dialect
+      @store.exec_script(Schema.statements(dialect, group))
+      g = Schema::GROUPS.fetch(group)
+      return if g[:added].empty?
+
+      existing = @store.column_names(Schema.columns_query(dialect, g[:table]))
+      Schema.missing_columns(dialect, g[:table], g[:added], existing).each do |stmt|
+        @store.exec(stmt)
+      rescue StandardError => e
+        raise unless Schema.duplicate_column?(e)
+      end
+    end
+  end
+end

data/lib/visionary/store/active_record.rb

@@ -0,0 +1,103 @@
+# frozen_string_literal: true
+
+module Visionary
+  module Store
+    # Writes through the application's ActiveRecord pool. Each operation
+    # checks a connection out for the background thread doing it and returns
+    # it right after, so Visionary holds at most one pool connection at a time
+    # (the mutex) and only while writing. Values are quoted by the adapter,
+    # exactly as ActiveRecord does when prepared statements are off.
+    class ActiveRecord
+      attr_reader :dialect
+
+      def initialize(connection_class)
+        @klass = connection_class
+        @mutex = Mutex.new
+        adapter = adapter_name.downcase
+        @dialect = if adapter.include?("postg") then "postgres"
+                   elsif adapter.include?("mysql") || adapter.include?("trilogy") then "mysql"
+                   elsif adapter.include?("sqlite") then "sqlite"
+                   else raise ConfigError, "visionary: unsupported ActiveRecord adapter #{adapter.inspect}"
+                   end
+      end
+
+      def string_params? = false
+
+      def exec_script(statements)
+        statements.each do |s|
+          with { |c| c.execute(s) }
+        rescue StandardError => e
+          next if dialect == "mysql" && Store.duplicate_index?(e)
+          raise
+        end
+      end
+
+      def exec(sql)
+        with { |c| c.execute(sql) }
+      end
+
+      def column_names(sql)
+        with { |c| c.select_values(sql).map(&:to_s) }
+      end
+
+      def batch(stmts)
+        with do |c|
+          c.transaction(requires_new: true) do
+            stmts.each { |sql, args| c.execute(bind(c, sql, args)) }
+          end
+        end
+      end
+
+      def changes(sql, args)
+        with { |c| c.exec_delete(bind(c, sql, args)) }
+      end
+
+      # ActiveRecord discards inherited pools itself after fork.
+      def after_fork
+        @mutex = Mutex.new
+      end
+
+      def close; end
+
+      private
+
+      def adapter_name
+        if @klass.respond_to?(:connection_db_config)
+          @klass.connection_db_config.adapter.to_s
+        else
+          @klass.connection_pool.with_connection(&:adapter_name).to_s
+        end
+      end
+
+      # Our own statements stay out of the app's log.
+      def with(&block)
+        @mutex.synchronize do
+          logger = @klass.logger
+          if logger.respond_to?(:silence)
+            logger.silence(::Logger::WARN) { @klass.connection_pool.with_connection(&block) }
+          else
+            @klass.connection_pool.with_connection(&block)
+          end
+        end
+      end
+
+      def bind(conn, sql, args)
+        i = -1
+        quote = nil
+        out = +""
+        sql.each_char do |ch|
+          if quote
+            quote = nil if ch == quote
+          elsif ch == "'"
+            quote = ch
+          elsif ch == "?"
+            out << conn.quote(args[i += 1])
+            next
+          end
+          out << ch
+        end
+        out
+      end
+    end
+  end
+end

data/lib/visionary/store/d1.rb

@@ -0,0 +1,129 @@
+# frozen_string_literal: true
+
+require "json"
+require "net/http"
+require "uri"
+
+module Visionary
+  module Store
+    # Cloudflare D1 over its REST API. Every param travels as a string; the
+    # inserts wrap placeholders in NULLIF(?, '') so "" means NULL.
+    class D1
+      DEFAULT_API_BASE = "https://api.cloudflare.com/client/v4"
+      MAX_BATCH = 50
+      RESPONSE_LIMIT = 1 << 20
+
+      class Error < StandardError; end
+
+      def initialize(account_id:, database_id:, api_token:, api_base: "", timeout: 20)
+        base = (api_base.to_s.strip.empty? ? DEFAULT_API_BASE : api_base.strip).chomp("/")
+        db = "#{base}/accounts/#{account_id.strip}/d1/database/#{database_id.strip}"
+        @query_uri = URI("#{db}/query")
+        @raw_uri = URI("#{db}/raw")
+        @token = api_token.strip
+        @timeout = timeout
+        @mutex = Mutex.new
+      end
+
+      def dialect = "d1"
+      def string_params? = true
+
+      def exec_script(statements)
+        statements.each_slice(MAX_BATCH) { |chunk| post(@query_uri, { batch: chunk.map { |s| { sql: s, params: [] } } }) }
+      end
+
+      def exec(sql)
+        post(@query_uri, { sql: sql })
+      end
+
+      def column_names(sql)
+        result = post(@raw_uri, { sql: sql })
+        rows = result.is_a?(Array) && result.dig(0, "results", "rows")
+        Array(rows).map { |r| r.is_a?(Array) ? r[0].to_s : nil }.compact
+      end
+
+      def batch(stmts)
+        stmts.each_slice(MAX_BATCH) do |chunk|
+          post(@query_uri, { batch: chunk.map { |sql, args| { sql: sql, params: args.map { |a| self.class.param(a) } } } })
+        end
+      end
+
+      def changes(sql, args)
+        result = post(@query_uri, { sql: sql, params: args.map { |a| self.class.param(a) } })
+        Integer(result.dig(0, "meta", "changes") || 0)
+      rescue TypeError, ArgumentError
+        0
+      end
+
+      def after_fork
+        @http = nil
+        @mutex = Mutex.new
+      end
+
+      def close
+        @mutex.synchronize { finish }
+      end
+
+      def self.param(v)
+        case v
+        when nil then ""
+        when true then "1"
+        when false then "0"
+        when Float
+          s = v.to_s
+          s.include?("e") ? format("%.6f", v) : s
+        else v.to_s
+        end
+      end
+
+      private
+
+      def post(uri, body)
+        @mutex.synchronize do
+          req = Net::HTTP::Post.new(uri)
+          req["Authorization"] = "Bearer #{@token}"
+          req["Content-Type"] = "application/json"
+          req.body = JSON.generate(body)
+          res = begin
+            connection(uri).request(req)
+          rescue StandardError => e
+            finish
+            raise Error, "d1: request: #{e.message}"
+          end
+          raw = res.body.to_s
+          raise Error, "d1: response larger than #{RESPONSE_LIMIT} bytes" if raw.bytesize > RESPONSE_LIMIT
+          env = begin
+            JSON.parse(raw)
+          rescue JSON::ParserError
+            raise Error, "d1: status #{res.code}: unreadable response"
+          end
+          if !env.is_a?(Hash) || env["success"] != true || res.code.to_i >= 300
+            msgs = (env.is_a?(Hash) ? Array(env["errors"]) : []).map { |e| e.is_a?(Hash) ? e["message"].to_s : nil }.compact
+            raise Error, "d1: status #{res.code}: #{msgs.empty? ? 'unknown D1 error' : msgs.join('; ')}"
+          end
+          env["result"]
+        end
+      end
+
+      # One keep-alive connection, reopened after errors.
+      def connection(uri)
+        return @http if @http&.started?
+        http = Net::HTTP.new(uri.host, uri.port)
+        http.use_ssl = uri.scheme == "https"
+        http.open_timeout = @timeout
+        http.read_timeout = @timeout
+        http.write_timeout = @timeout if http.respond_to?(:write_timeout=)
+        http.keep_alive_timeout = 30
+        @http = http.start
+      end
+
+      def finish
+        @http&.finish if @http&.started?
+      rescue StandardError
+        nil
+      ensure
+        @http = nil
+      end
+    end
+  end
+end