fluent-plugin-oceanbase-logs 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 3a3230feaa77c86a454f2733f9097f05a2e57f43dd418c1fed7af21d5345a875
+  data.tar.gz: d2a7f628ddb81778b10c6b9a14ffc1d6fa1fe6cbc4b1e7771da21840a42ef93e
+SHA512:
+  metadata.gz: f2f8cbbf267cdf26e929677364f9a0db2cfdc1256fa231bbdf85f347cd301d8fdbd26c1b219d2faed9a94cacfd177e5aad665f291f9aafc809ab8406f4e25d32
+  data.tar.gz: 9f1725a4250c1988960db905493ce797d66b096a6f586b3af6c3642956b3c5fb7194516d82c828f5c728b086dfe0fbc61f01ea39927a22dcf7742acad6c81a0c

data/lib/fluent/plugin/in_oceanbase_logs.rb

@@ -0,0 +1,371 @@
+require 'net/http'
+require 'uri'
+require 'json'
+require 'openssl'
+require 'time'
+require 'digest'
+require 'securerandom'
+require 'fluent/plugin/input'
+
+module Fluent::Plugin
+  class OceanBaseLogsInput < Input
+    Fluent::Plugin.register_input('oceanbase_logs', self)
+
+    helpers :thread, :storage
+
+    DEFAULT_STORAGE_TYPE = 'local'
+
+    LOG_TYPE_PATHS = {
+      'slow_sql' => 'slowSql',
+      'top_sql'  => 'topSql',
+    }.freeze
+
+    config_param :log_type, :enum, list: LOG_TYPE_PATHS.keys.map(&:to_sym), default: :slow_sql,
+      desc: "Type of SQL diagnostics to collect: slow_sql or top_sql."
+    config_param :fetch_samples, :bool, default: false,
+      desc: "When true, fetch per-execution sample details for each SQL (each execution = one event)."
+
+    config_param :access_key_id, :string, secret: true,
+      desc: "OceanBase Cloud AccessKey ID."
+    config_param :access_key_secret, :string, secret: true,
+      desc: "OceanBase Cloud AccessKey Secret."
+
+    config_param :instance_id, :string,
+      desc: "OceanBase cluster instance ID."
+    config_param :tenant_id, :string,
+      desc: "OceanBase tenant ID."
+    config_param :project_id, :string, default: nil,
+      desc: "OceanBase Cloud project ID (X-Ob-Project-Id header)."
+
+    config_param :db_name, :string, default: nil,
+      desc: "Filter by database name."
+    config_param :search_keyword, :string, default: nil,
+      desc: "Search keyword for SQL text."
+    config_param :node_ip, :string, default: nil,
+      desc: "Filter by database node IP."
+    config_param :filter_condition, :string, default: nil,
+      desc: "Advanced filter (e.g. '@avgCpuTime > 20 and @executions > 100')."
+    config_param :sql_text_length, :integer, default: 65535,
+      desc: "Max length of SQL text returned."
+
+    config_param :tag, :string,
+      desc: "Fluentd tag for emitted events."
+    config_param :fetch_interval, :time, default: 300,
+      desc: "Seconds between each API poll (default 5 min)."
+    config_param :lookback_seconds, :integer, default: 600,
+      desc: "How far back each query window looks (default 10 min)."
+    config_param :endpoint, :string, default: 'api-cloud-cn.oceanbase.com',
+      desc: "API endpoint."
+    config_param :http_proxy, :string, default: nil,
+      desc: "HTTP proxy URL."
+    config_param :ssl_verify_peer, :bool, default: true,
+      desc: "Verify SSL certificates."
+
+    config_param :deduplicate, :bool, default: true,
+      desc: "Enable deduplication."
+    config_param :include_metadata, :bool, default: true,
+      desc: "Attach instance_id / tenant_id / log_type to each record."
+
+    config_section :storage do
+      config_set_default :usage, 'seen_traces'
+      config_set_default :@type, DEFAULT_STORAGE_TYPE
+      config_set_default :persistent, false
+    end
+
+    def configure(conf)
+      super
+      @endpoint = @endpoint.to_s.strip
+      @endpoint = 'api-cloud-cn.oceanbase.com' if @endpoint.empty?
+      @api_path_segment = LOG_TYPE_PATHS[@log_type.to_s]
+      if @deduplicate
+        @seen_storage = storage_create(
+          usage: 'seen_traces',
+          conf: config,
+          default_type: DEFAULT_STORAGE_TYPE
+        )
+      end
+    end
+
+    def start
+      super
+      @finished = false
+      thread_create(:in_oceanbase_logs_runner, &method(:run))
+    end
+
+    def shutdown
+      @finished = true
+      super
+    end
+
+    private
+
+    def run
+      until @finished
+        begin
+          fetch_and_emit
+        rescue => e
+          log.error "Failed to fetch OceanBase #{@log_type} data",
+                    error: e.message, error_class: e.class.to_s
+          log.debug_backtrace(e.backtrace)
+        end
+        sleep_interruptible(@fetch_interval)
+      end
+    end
+
+    def sleep_interruptible(seconds)
+      seconds.to_i.times do
+        break if @finished
+        sleep 1
+      end
+    end
+
+    def fetch_and_emit
+      now = Time.now.utc
+      start_time = (now - @lookback_seconds).strftime('%Y-%m-%dT%H:%M:%SZ')
+      end_time   = now.strftime('%Y-%m-%dT%H:%M:%SZ')
+
+      if @fetch_samples
+        fetch_and_emit_samples(start_time, end_time)
+      else
+        fetch_and_emit_list(start_time, end_time)
+      end
+    end
+
+    # ---- Mode 1: emit aggregated list (one record per sqlId) ----
+    def fetch_and_emit_list(start_time, end_time)
+      response = call_list_api(start_time, end_time)
+      return unless response
+
+      records = extract_records(response)
+      return if records.nil? || records.empty?
+
+      es = Fluent::MultiEventStream.new
+
+      records.each do |record|
+        sql_id = record['sqlId']
+        next if sql_id.nil? || sql_id.empty?
+
+        if @deduplicate
+          dedup_key = :"list_#{sql_id}_#{start_time}"
+          next if @seen_storage.get(dedup_key)
+          @seen_storage.put(dedup_key, Time.now.to_i.to_s)
+        end
+
+        record = attach_metadata(record, start_time, end_time) if @include_metadata
+        es.add(Fluent::EventTime.now, record)
+      end
+
+      router.emit_stream(@tag, es) unless es.empty?
+      log.info "Emitted #{es.size} #{@log_type} events (#{start_time} ~ #{end_time})" if es.size > 0
+    end
+
+    # ---- Mode 2: emit per-execution samples (one record per trace) ----
+    def fetch_and_emit_samples(start_time, end_time)
+      list_response = call_list_api(start_time, end_time)
+      return unless list_response
+
+      sql_records = extract_records(list_response)
+      return if sql_records.nil? || sql_records.empty?
+
+      sql_ids = sql_records.map { |r| r['sqlId'] }.compact.uniq
+      log.debug "Found #{sql_ids.size} unique SQL IDs, fetching samples..."
+
+      total_emitted = 0
+
+      sql_ids.each do |sql_id|
+        samples = fetch_samples_for_sql(sql_id, start_time, end_time)
+        next if samples.nil? || samples.empty?
+
+        es = Fluent::MultiEventStream.new
+
+        samples.each do |sample|
+          trace_id = sample['traceId']
+          dedup_id = trace_id || "#{sql_id}_#{sample['requestTime']}"
+
+          if @deduplicate
+            dedup_key = :"trace_#{dedup_id}"
+            next if @seen_storage.get(dedup_key)
+            @seen_storage.put(dedup_key, Time.now.to_i.to_s)
+          end
+
+          sample = attach_metadata(sample, start_time, end_time) if @include_metadata
+
+          event_time = if sample['requestTime']
+                         begin
+                           Fluent::EventTime.from_time(Time.parse(sample['requestTime']))
+                         rescue
+                           Fluent::EventTime.now
+                         end
+                       else
+                         Fluent::EventTime.now
+                       end
+
+          es.add(event_time, sample)
+        end
+
+        unless es.empty?
+          router.emit_stream(@tag, es)
+          total_emitted += es.size
+        end
+      end
+
+      log.info "Emitted #{total_emitted} #{@log_type} sample events (#{start_time} ~ #{end_time})" if total_emitted > 0
+    end
+
+    def fetch_samples_for_sql(sql_id, start_time, end_time)
+      path = "/api/v2/instances/#{@instance_id}/tenants/#{@tenant_id}/sqls/#{sql_id}/samples"
+      params = {
+        'startTime' => start_time,
+        'endTime'   => end_time,
+      }
+      params['dbName'] = @db_name if @db_name
+
+      response = call_api_raw(path, params)
+      return nil unless response
+      extract_records(response)
+    end
+
+    def attach_metadata(record, start_time, end_time)
+      record.merge(
+        'ob_instance_id'   => @instance_id,
+        'ob_tenant_id'     => @tenant_id,
+        'ob_log_type'      => @log_type.to_s,
+        'query_start_time' => start_time,
+        'query_end_time'   => end_time
+      )
+    end
+
+    def extract_records(response)
+      if response['data'].is_a?(Hash) && response['data']['dataList'].is_a?(Array)
+        response['data']['dataList']
+      elsif response['Data'].is_a?(Array)
+        response['Data']
+      elsif response['data'].is_a?(Array)
+        response['data']
+      else
+        log.warn "Unexpected API response structure", keys: response.keys
+        nil
+      end
+    end
+
+    # ---- API calls ----
+
+    def call_list_api(start_time, end_time)
+      path = "/api/v2/instances/#{@instance_id}/tenants/#{@tenant_id}/#{@api_path_segment}"
+      params = {
+        'startTime' => start_time,
+        'endTime'   => end_time,
+      }
+      params['dbName']          = @db_name          if @db_name
+      params['searchKeyWord']   = @search_keyword   if @search_keyword
+      params['nodeIp']          = @node_ip           if @node_ip
+      params['filterCondition'] = @filter_condition  if @filter_condition
+      params['sqlTextLength']   = @sql_text_length.to_s
+
+      call_api_raw(path, params)
+    end
+
+    def call_api_raw(path, params)
+      query = params.map { |k, v| "#{URI.encode_www_form_component(k)}=#{URI.encode_www_form_component(v)}" }.join('&')
+      uri = URI("https://#{@endpoint}#{path}?#{query}")
+
+      http = build_http(uri)
+      resp = nil
+
+      http.start do |session|
+        resp = request_with_digest_auth(session, uri)
+      end
+
+      unless resp && resp.code.to_i == 200
+        log.error "OceanBase API HTTP #{resp&.code}", body: resp&.body, path: path
+        return nil
+      end
+
+      body = JSON.parse(resp.body)
+      unless body['success'] == true
+        log.error "OceanBase API error",
+                  code: body['errorCode'], message: body['errorMessage'], path: path
+        return nil
+      end
+      body
+    rescue JSON::ParserError => e
+      log.error "Failed to parse API response", error: e.message, path: path
+      nil
+    end
+
+    # ---- HTTP Digest Auth ----
+
+    def request_with_digest_auth(session, uri)
+      req = Net::HTTP::Get.new(uri)
+      req['X-Ob-Project-Id'] = @project_id if @project_id
+
+      initial_resp = session.request(req)
+      return initial_resp unless initial_resp.code.to_i == 401
+
+      auth_header = initial_resp['www-authenticate']
+      return initial_resp unless auth_header && auth_header.start_with?('Digest')
+
+      digest = build_digest_header(auth_header, uri, 'GET')
+      retry_req = Net::HTTP::Get.new(uri)
+      retry_req['X-Ob-Project-Id'] = @project_id if @project_id
+      retry_req['Authorization'] = digest
+
+      session.request(retry_req)
+    end
+
+    def build_digest_header(www_auth, uri, method)
+      params = parse_digest_challenge(www_auth)
+      realm  = params['realm']
+      nonce  = params['nonce']
+      qop    = params['qop']
+      opaque = params['opaque']
+
+      nc = '00000001'
+      cnonce = SecureRandom.hex(8)
+
+      ha1 = md5("#{@access_key_id}:#{realm}:#{@access_key_secret}")
+      ha2 = md5("#{method}:#{uri.request_uri}")
+
+      if qop
+        response = md5("#{ha1}:#{nonce}:#{nc}:#{cnonce}:#{qop}:#{ha2}")
+      else
+        response = md5("#{ha1}:#{nonce}:#{ha2}")
+      end
+
+      header = %Q(Digest username="#{@access_key_id}", realm="#{realm}", nonce="#{nonce}", uri="#{uri.request_uri}", response="#{response}")
+      header += %Q(, qop=#{qop}, nc=#{nc}, cnonce="#{cnonce}") if qop
+      header += %Q(, opaque="#{opaque}") if opaque
+      header
+    end
+
+    def parse_digest_challenge(header)
+      params = {}
+      header.sub(/^Digest\s+/, '').scan(/(\w+)="([^"]*)"/) do |key, value|
+        params[key] = value
+      end
+      header.sub(/^Digest\s+/, '').scan(/(\w+)=([^",\s]+)/) do |key, value|
+        params[key] ||= value
+      end
+      params
+    end
+
+    def md5(str)
+      Digest::MD5.hexdigest(str)
+    end
+
+    def build_http(uri)
+      if @http_proxy
+        proxy = URI(@http_proxy)
+        http = Net::HTTP.new(uri.host, uri.port,
+                             proxy.host, proxy.port, proxy.user, proxy.password)
+      else
+        http = Net::HTTP.new(uri.host, uri.port)
+      end
+      http.use_ssl     = true
+      http.verify_mode = @ssl_verify_peer ? OpenSSL::SSL::VERIFY_PEER : OpenSSL::SSL::VERIFY_NONE
+      http.open_timeout = 30
+      http.read_timeout = 60
+      http
+    end
+  end
+end

data/lib/fluent/plugin/oceanbase/logs/version.rb

@@ -0,0 +1,9 @@
+module Fluent
+  module Plugin
+    module OceanBase
+      module Logs
+        VERSION = "0.1.0"
+      end
+    end
+  end
+end

metadata

@@ -0,0 +1,130 @@
+--- !ruby/object:Gem::Specification
+name: fluent-plugin-oceanbase-logs
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- OceanBase Integrations
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2026-03-06 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.8.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.8.0
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: test-unit
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: test-unit-rr
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: webmock
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+description: Periodically fetches slow SQL information from OceanBase Cloud via the
+  DescribeOasSlowSQLList API and emits them as Fluentd events.
+email:
+- integrations@example.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/fluent/plugin/in_oceanbase_logs.rb
+- lib/fluent/plugin/oceanbase/logs/version.rb
+homepage: https://github.com/your-org/fluent-plugin-oceanbase-logs
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '2.4'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.3.5
+signing_key: 
+specification_version: 4
+summary: Fluentd input plugin for OceanBase Cloud Logs
+test_files: []