visionary-sdk 0.4.0

data/lib/visionary/writer.rb

@@ -0,0 +1,185 @@
+# frozen_string_literal: true
+
+module Visionary
+  # The raw capture of one request. Redaction, error-field extraction and
+  # user lookup happen on the writer thread, off the request path.
+  Entry = Struct.new(
+    :request_id, :started_at, :method, :path, :route, :query, :status, :duration_ms, :ip, :user_agent,
+    :user_id, :user_handle, :user_role, :headers, :request_content_type, :request_body, :request_truncated,
+    :request_size, :response_content_type, :response_body, :response_truncated, :response_size,
+    :totals, :span_rows, :error_rows,
+    keyword_init: true
+  )
+
+  # The single background thread that batches api_logs, spans, errors and
+  # releases rows. Queue items are [:request, Entry], [:span, row],
+  # [:error, row] or [:release, row].
+  class Writer
+    INSERTS = { request: "api_logs", span: "spans", error: "errors", release: "releases" }.freeze
+
+    def initialize(cfg, store, schemas, logger)
+      @cfg = cfg
+      @store = store
+      @schemas = schemas
+      @logger = logger
+      @redactor = Redactor.new(cfg.redact_headers, cfg.redact_fields, cfg.redact_request_fields)
+      @batch_size = cfg.batch_size.between?(1, 50) ? cfg.batch_size : 50
+      @flush_interval = cfg.flush_interval.to_f.positive? ? cfg.flush_interval.to_f : 3.0
+      @queue_size = cfg.queue_size.positive? ? cfg.queue_size : 5000
+      @sql = INSERTS.transform_values { |t| Schema.insert_sql(t, store.string_params?) }
+      @groups = []
+      @groups << :requests if cfg.log_requests
+      @groups << :telemetry if cfg.spans || cfg.errors || cfg.version
+      reset
+    end
+
+    def start
+      @thread = Thread.new { run }
+      @thread.name = "visionary-writer" if @thread.respond_to?(:name=)
+      self
+    end
+
+    # Threads don't survive fork; the child starts over with an empty queue.
+    def after_fork
+      reset
+      start
+    end
+
+    # Never blocks; a full queue drops and counts the item.
+    def enqueue(item)
+      @queue.push(item)
+    end
+
+    # Stops the thread after it has drained the queue, waiting at most
+    # timeout seconds. False if it was still busy then.
+    def close(timeout)
+      @queue.close
+      return true unless @thread
+      return true if @thread.join(timeout)
+      @logger.warn("visionary: log flush timed out; #{@queue.size} pending")
+      false
+    end
+
+    private
+
+    def reset
+      @queue = BoundedQueue.new(@queue_size)
+      @thread = nil
+    end
+
+    def run
+      Telemetry.internal!
+      @groups.each { |g| @schemas.ensure(g) }
+      next_flush = mono + @flush_interval
+      batch = []
+      loop do
+        items = @queue.take(@batch_size - batch.size, next_flush)
+        batch.concat(items)
+        closed = @queue.closed? && @queue.size.zero?
+        if batch.size >= @batch_size || mono >= next_flush || closed
+          write(batch) unless batch.empty?
+          batch = []
+          report_drops
+          next_flush = mono + @flush_interval
+        end
+        break if closed
+      end
+    rescue StandardError => e
+      @logger.error("visionary: writer stopped: #{e.class}: #{e.message}")
+    end
+
+    def report_drops
+      n = @queue.take_dropped
+      @logger.warn("visionary: queue full; #{n} entries dropped") if n.positive?
+    end
+
+    def write(batch)
+      stmts = begin
+        users = lookup_users(batch)
+        batch.flat_map { |kind, item| statements(kind, item, users) }
+      rescue StandardError => e
+        @logger.error("visionary: building rows failed; #{batch.size} entries dropped: #{e.class}: #{e.message}")
+        return
+      end
+      err = nil
+      3.times do |attempt|
+        @groups.each { |g| @schemas.ensure(g) }
+        begin
+          @store.batch(stmts)
+          return
+        rescue StandardError => e
+          err = e
+        end
+        sleep(attempt + 1) if attempt < 2
+      end
+      @logger.error("visionary: batch write failed; #{batch.size} entries dropped: #{err.message}")
+    end
+
+    def statements(kind, item, users)
+      if kind == :request
+        out = [stmt(:request, request_row(item, users))]
+        item.span_rows&.each { |r| out << stmt(:span, r) }
+        item.error_rows&.each { |r| out << stmt(:error, r) }
+        out
+      else
+        [stmt(kind, item)]
+      end
+    end
+
+    def stmt(kind, row)
+      [@sql.fetch(kind), Store.fit(@store.dialect, INSERTS.fetch(kind), row)]
+    end
+
+    # Resolves each distinct user missing a handle or role once per batch.
+    def lookup_users(batch)
+      out = {}
+      lookup = @cfg.lookup_user
+      return out unless lookup
+      batch.each do |kind, e|
+        next unless kind == :request
+        id = e.user_id
+        next if id.empty? || (!e.user_handle.empty? && !e.user_role.empty?) || out.key?(id)
+        found = begin
+          lookup.call(id)
+        rescue StandardError => ex
+          @logger.warn("visionary: lookup_user(#{id.inspect}) failed: #{ex.message}")
+          nil
+        end
+        out[id] = found ? [found[0].to_s, found[1].to_s] : ["", ""]
+      end
+      out
+    end
+
+    def request_row(e, users)
+      handle = e.user_handle
+      role = e.user_role
+      if (found = users[e.user_id])
+        handle = found[0] if handle.empty?
+        role = found[1] if role.empty?
+      end
+      req_raw = Visionary.utf8(e.request_body)
+      resp_raw = Visionary.utf8(e.response_body)
+      code = key = ""
+      code, key = Redactor.error_fields(resp_raw, @cfg.error_code_field, @cfg.error_key_field) if e.status >= 400
+      req_body = resp_body = ""
+      if @cfg.capture_bodies
+        req_body = @redactor.request_body(req_raw, e.request_content_type)
+        resp_body = @redactor.response_body(resp_raw)
+      end
+      db_count, db_ms, http_count, http_ms = e.totals
+      [
+        e.request_id, Visionary.timestamp(e.started_at), e.method, e.path, nz(e.route), nz(e.query),
+        e.status, e.duration_ms, nz(e.ip), nz(e.user_agent), nz(e.user_id), nz(handle), nz(role),
+        nz(@redactor.headers(e.headers)), nz(e.request_content_type), nz(req_body), e.request_truncated ? 1 : 0,
+        e.request_size, nz(e.response_content_type), nz(resp_body), e.response_truncated ? 1 : 0, e.response_size,
+        nz(code), nz(key), @cfg.service, @cfg.version, db_count, db_ms, http_count, http_ms
+      ]
+    end
+
+    def nz(s)
+      s.nil? || s.empty? ? nil : s
+    end
+
+    def mono = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+  end
+end

data/lib/visionary.rb

@@ -0,0 +1,157 @@
+# frozen_string_literal: true
+
+require_relative "visionary/version"
+require_relative "visionary/config"
+require_relative "visionary/schema"
+require_relative "visionary/store"
+require_relative "visionary/schemas"
+require_relative "visionary/redactor"
+require_relative "visionary/context"
+require_relative "visionary/queue"
+require_relative "visionary/writer"
+require_relative "visionary/retention"
+require_relative "visionary/system/sources"
+require_relative "visionary/system/recorder"
+require_relative "visionary/telemetry"
+require_relative "visionary/telemetry/hooks"
+require_relative "visionary/body"
+require_relative "visionary/middleware"
+require_relative "visionary/agent"
+require_relative "visionary/fork"
+
+# Visionary collects what the Visionary server analyses, straight from a Ruby
+# service: one api_logs row per HTTP request, spans for database and outbound
+# HTTP calls, errors, deploy markers and periodic system_metrics samples.
+#
+#   # config.ru (Sinatra, Roda, Hanami, any Rack app)
+#   require "visionary"
+#   use Visionary::Middleware
+#
+# In Rails the Railtie inserts the middleware. Everything is configured
+# through VISIONARY_* environment variables, or Visionary.configure.
+module Visionary
+  @lock = Mutex.new
+  @agent = nil
+  @config = nil
+
+  class << self
+    # The configuration the agent starts with (from the environment unless
+    # set). Change it before the first request, e.g. in an initializer.
+    def config
+      @config ||= Config.from_env
+    end
+
+    attr_writer :config
+
+    def configure
+      yield config
+      config
+    end
+
+    # Builds and starts the process-wide agent; later calls return it. Never
+    # raises: on any problem the agent is a no-op.
+    def start(cfg = nil)
+      return @agent if @agent
+      @lock.synchronize do
+        return @agent if @agent
+        @config = cfg if cfg
+        agent = begin
+          Agent.new(config).start
+        rescue StandardError => e
+          warn("visionary: start failed (#{e.class}: #{e.message}); collection is off")
+          Agent.new(Config.new.tap { |c| c.log_requests = c.system_metrics = c.spans = c.errors = false })
+        end
+        at_exit { agent.close }
+        @agent = agent
+      end
+    end
+
+    # The running agent, started on first use.
+    def agent = @agent || start
+
+    # The agent's span and error recorder, or nil when collection is off.
+    def telemetry = @agent&.telemetry
+
+    # Flushes and stops collection, waiting at most timeout seconds.
+    def close(timeout = 10)
+      @agent&.close(timeout)
+    end
+
+    # Attaches the signed-in user to the request being logged. Call it from
+    # authentication code (a before_action, a Warden callback, a Rack
+    # middleware) once the user is known. Only id is required. No-op outside a
+    # request seen by Visionary::Middleware.
+    def set_user(id, handle: nil, role: nil)
+      RequestContext.current&.set_user(id, handle, role)
+      nil
+    end
+
+    # Records the route pattern (e.g. "/users/:id") for routers Visionary
+    # can't see. No-op outside a request.
+    def set_route(pattern)
+      ctx = RequestContext.current
+      ctx.route = pattern if ctx
+      nil
+    end
+
+    # Measures the block as a span and returns its value. kind is "custom",
+    # "cache", "queue", "db" or "http"; name should be low-cardinality (the
+    # operation, not its arguments).
+    #
+    #   Visionary.span("charge card", kind: "http", target: "api.stripe.com") { stripe.charge(...) }
+    def span(name, kind: "custom", target: nil, detail: nil)
+      rec = telemetry
+      return yield unless rec&.spans_on
+
+      t0 = Telemetry.now_ms
+      status = "ok"
+      error = nil
+      begin
+        yield
+      rescue Exception => e # re-raised; only observed here
+        status = "error"
+        error = "#{e.class}: #{e.message}"
+        raise
+      ensure
+        rec.span(kind: kind, name: name, target: target, detail: detail, start_mono: t0,
+                 duration_ms: Telemetry.now_ms - t0, status: status, error: error)
+      end
+    end
+
+    # Records a handled error (handled = 1). Inside a request it links to it.
+    #
+    #   rescue PaymentError => e
+    #     Visionary.capture_exception(e)
+    def capture_exception(exception = $!)
+      telemetry&.exception(exception, handled: true)
+      nil
+    end
+
+    # Restarts the background threads in a forked child. Called
+    # automatically through Process._fork; Puma's on_worker_boot may call it
+    # too (it is idempotent per process).
+    def after_fork
+      agent = @agent
+      return unless agent
+      @lock = Mutex.new
+      agent.after_fork
+    rescue StandardError => e
+      warn("visionary: restart after fork failed: #{e.message}")
+    end
+
+    # created_at format for every table: UTC, milliseconds, literal Z.
+    def timestamp(time)
+      time.getutc.strftime("%Y-%m-%dT%H:%M:%S.%LZ")
+    end
+
+    # Bytes as UTF-8 text, invalid sequences replaced.
+    def utf8(s)
+      return "" if s.nil?
+      s = s.to_s
+      return s if s.encoding == Encoding::UTF_8 && s.valid_encoding?
+      s.dup.force_encoding(Encoding::UTF_8).scrub
+    end
+  end
+end
+
+require_relative "visionary/railtie" if defined?(Rails::Railtie)

metadata

@@ -0,0 +1,90 @@
+--- !ruby/object:Gem::Specification
+name: visionary-sdk
+version: !ruby/object:Gem::Version
+  version: 0.4.0
+platform: ruby
+authors:
+- styropyr0
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2026-10-18 00:00:00.000000000 Z
+dependencies: []
+description: |
+  The Ruby collector for Visionary. A Rack middleware (with a Railtie for Rails)
+  writes one row per HTTP request, database and outbound HTTP spans, errors,
+  deploy markers and periodic host/process samples to PostgreSQL, MySQL,
+  SQLite or Cloudflare D1, on background threads.
+email:
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- CHANGELOG.md
+- LICENSE
+- README.md
+- lib/visionary.rb
+- lib/visionary/agent.rb
+- lib/visionary/body.rb
+- lib/visionary/config.rb
+- lib/visionary/context.rb
+- lib/visionary/fork.rb
+- lib/visionary/middleware.rb
+- lib/visionary/queue.rb
+- lib/visionary/railtie.rb
+- lib/visionary/redactor.rb
+- lib/visionary/retention.rb
+- lib/visionary/schema.rb
+- lib/visionary/schema/mysql/requests.sql
+- lib/visionary/schema/mysql/system.sql
+- lib/visionary/schema/mysql/telemetry.sql
+- lib/visionary/schema/postgres/requests.sql
+- lib/visionary/schema/postgres/system.sql
+- lib/visionary/schema/postgres/telemetry.sql
+- lib/visionary/schema/sqlite/requests.sql
+- lib/visionary/schema/sqlite/system.sql
+- lib/visionary/schema/sqlite/telemetry.sql
+- lib/visionary/schemas.rb
+- lib/visionary/store.rb
+- lib/visionary/store/active_record.rb
+- lib/visionary/store/d1.rb
+- lib/visionary/store/sql.rb
+- lib/visionary/system/recorder.rb
+- lib/visionary/system/sources.rb
+- lib/visionary/telemetry.rb
+- lib/visionary/telemetry/errors.rb
+- lib/visionary/telemetry/hooks.rb
+- lib/visionary/telemetry/sql_text.rb
+- lib/visionary/version.rb
+- lib/visionary/writer.rb
+homepage: https://github.com/styropyr0/Visionary
+licenses:
+- Apache-2.0
+metadata:
+  homepage_uri: https://github.com/styropyr0/Visionary
+  source_code_uri: https://github.com/styropyr0/Visionary/tree/main/sdks/ruby
+  changelog_uri: https://github.com/styropyr0/Visionary/blob/main/sdks/ruby/CHANGELOG.md
+  bug_tracker_uri: https://github.com/styropyr0/Visionary/issues
+  documentation_uri: https://github.com/styropyr0/Visionary/tree/main/sdks/ruby#readme
+  rubygems_mfa_required: 'true'
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '3.1'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.5.22
+signing_key:
+specification_version: 4
+summary: Request logs, spans, errors and system metrics for Visionary, from Rack and
+  Rails apps
+test_files: []