fluent-plugin-oceanbase-logs 0.1.3 → 0.1.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ff677af2c1882b27e86cde4cbfef8ef79039f45eae20e2a7c8eebd4c2fdbd98e
-  data.tar.gz: 2fbe0571f3c1e1b840ab6714faff44e5a7396cad011c4845fb710964db9a490b
+  metadata.gz: 8b21a8c2fad6676c5f9b5b8f00cd849091b8c4077c745b2fa7a462995f9b7b07
+  data.tar.gz: 0f1a70d45968dc28815fb3c507e85b7115cc6b3682f0a5a762a0bdc5467f9019
 SHA512:
-  metadata.gz: 380099690cb075af74c720a6eb147fdb5a1234b428bac1d54e1ea3a36d191f2b47b2a7962ed97ee7f5f2c9f576aad55a4cf025bba39bd8152747fdc6439242bf
-  data.tar.gz: 36cd5b13ca5fd22159451bc2c15c45c7861ad1122a62d9e95a4184c2e5e0a5f49c08183609e7767125d65330b69adf506060b9c59b228f785b641c56c41add20
+  metadata.gz: 02b0aaf81f118afdcc215f377f7c13ca2ca63d2a1cd28008f778b902e9277b614f28464973fbec4d5ce4a7550fbc3eca9a9f43833367379f58a7f967398f1a05
+  data.tar.gz: 2dfdd6d0bbe2ac20532ef99e1ef1e764d7d8fa9244043865370e9b1c77378766503ae19680f50266085e39a0ad8af0784ee00eeb19f904e4ee6523de902692f8

data/README.md

@@ -44,6 +44,47 @@ tenant_id         "#{ENV['OCEANBASE_TENANT_ID']}"
 
 Optional: `OCEANBASE_ENDPOINT`, `OCEANBASE_FETCH_INTERVAL`, `OCEANBASE_LOOKBACK_SECONDS`, `OCEANBASE_DB_NAME`, `OCEANBASE_SEARCH_KEYWORD`, `OCEANBASE_PROJECT_ID` — see `.env.example` and the Docker table below.
 
+### Multiple instances, tenants, and databases
+
+Use one or more `<target>` blocks. Each block sets **`instance_id`** and **`tenant_id`** (required). Optional **`db_name`** applies the same `dbName` filter as the top-level parameter, for that scope only.
+
+When any `<target>` is present, the plugin **only** uses those scopes; top-level `instance_id` / `tenant_id` / `db_name` are ignored (you may leave them empty). Global options such as `access_key_id`, `log_type`, `search_keyword`, `endpoint`, and `fetch_interval` still apply to every target.
+
+Example: two clusters, one tenant each, and a second scope that filters one database on another tenant:
+
+```conf
+<source>
+  @type oceanbase_logs
+  tag  oceanbase.slow_sql
+  log_type slow_sql
+  access_key_id     "#{ENV['OCEANBASE_ACCESS_KEY_ID']}"
+  access_key_secret "#{ENV['OCEANBASE_ACCESS_KEY_SECRET']}"
+  instance_id  ""
+  tenant_id    ""
+  endpoint     api-cloud-cn.oceanbase.com
+  fetch_interval   60
+  lookback_seconds 600
+  deduplicate      true
+  include_metadata true
+  <target>
+    instance_id "#{ENV['OCEANBASE_INSTANCE_1']}"
+    tenant_id   "#{ENV['OCEANBASE_TENANT_A']}"
+  </target>
+  <target>
+    instance_id "#{ENV['OCEANBASE_INSTANCE_2']}"
+    tenant_id   "#{ENV['OCEANBASE_TENANT_B']}"
+    db_name     "#{ENV['OCEANBASE_DB_ANALYTICS']}"
+  </target>
+  <storage>
+    @type local
+    persistent true
+    path /var/log/fluentd/slow_sql_seen
+  </storage>
+</source>
+```
+
+The same tenant with multiple databases needs one `<target>` per database (or a target without `db_name` to pull all DBs for that tenant, if your API usage allows omitting `dbName`).
+
 ### Example: Slow SQL → JSON file
 
 Full sample: [`example/fluentd.conf`](example/fluentd.conf).

data/lib/fluent/plugin/in_oceanbase_logs.rb

@@ -13,6 +13,8 @@ module Fluent::Plugin
 
     helpers :thread, :storage
 
+    FetchScope = Struct.new(:instance_id, :tenant_id, :db_name)
+
     DEFAULT_STORAGE_TYPE = 'local'
 
     LOG_TYPE_PATHS = {
@@ -28,10 +30,19 @@ module Fluent::Plugin
     config_param :access_key_secret, :string, secret: true,
       desc: "OceanBase Cloud AccessKey Secret."
 
-    config_param :instance_id, :string,
-      desc: "OceanBase cluster instance ID."
-    config_param :tenant_id, :string,
-      desc: "OceanBase tenant ID."
+    config_param :instance_id, :string, default: '',
+      desc: "OceanBase cluster instance ID (legacy single scope; ignored when <target> sections are present)."
+    config_param :tenant_id, :string, default: '',
+      desc: "OceanBase tenant ID (legacy single scope; ignored when <target> sections are present)."
+
+    config_section :target, param_name: :targets, multi: true, required: false do
+      config_param :instance_id, :string,
+        desc: "Instance ID for this scope."
+      config_param :tenant_id, :string,
+        desc: "Tenant ID for this scope."
+      config_param :db_name, :string, default: nil,
+        desc: "Optional database name filter for this scope."
+    end
     config_param :project_id, :string, default: nil,
       desc: "OceanBase Cloud project ID (X-Ob-Project-Id header)."
 
@@ -82,13 +93,14 @@ module Fluent::Plugin
       end
       raise Fluent::ConfigError, 'access_key_id is required and cannot be empty' if @access_key_id.empty?
       raise Fluent::ConfigError, 'access_key_secret is required and cannot be empty' if @access_key_secret.empty?
-      raise Fluent::ConfigError, 'instance_id is required and cannot be empty (e.g. set OCEANBASE_INSTANCE_ID)' if @instance_id.empty?
-      raise Fluent::ConfigError, 'tenant_id is required and cannot be empty (e.g. set OCEANBASE_TENANT_ID)' if @tenant_id.empty?
 
       %i[@db_name @search_keyword @node_ip @filter_condition @project_id].each do |iv|
         v = instance_variable_get(iv)
         instance_variable_set(iv, nil) if v.is_a?(String) && v.strip.empty?
       end
+
+      @fetch_scopes = build_fetch_scopes
+
       @api_path_segment = LOG_TYPE_PATHS[@log_type.to_s]
       if @deduplicate
         @seen_storage = storage_create(
@@ -112,6 +124,36 @@ module Fluent::Plugin
 
     private
 
+    def build_fetch_scopes
+      scopes = []
+      if @targets && !@targets.empty?
+        @targets.each_with_index do |t, idx|
+          iid = t.instance_id.to_s.strip
+          tid = t.tenant_id.to_s.strip
+          raise Fluent::ConfigError, "target[#{idx}]: instance_id is required and cannot be empty" if iid.empty?
+          raise Fluent::ConfigError, "target[#{idx}]: tenant_id is required and cannot be empty" if tid.empty?
+          db = normalize_optional_db_name(t.db_name)
+          scopes << FetchScope.new(iid, tid, db)
+        end
+      else
+        raise Fluent::ConfigError, 'instance_id is required and cannot be empty (e.g. set OCEANBASE_INSTANCE_ID), or define one or more <target> sections' if @instance_id.empty?
+        raise Fluent::ConfigError, 'tenant_id is required and cannot be empty (e.g. set OCEANBASE_TENANT_ID), or define one or more <target> sections' if @tenant_id.empty?
+        scopes << FetchScope.new(@instance_id, @tenant_id, @db_name)
+      end
+      scopes
+    end
+
+    def normalize_optional_db_name(value)
+      return nil if value.nil?
+      s = value.to_s.strip
+      s.empty? ? nil : s
+    end
+
+    def dedup_storage_key(scope, dedup_id)
+      db = scope.db_name.to_s
+      "trace:#{scope.instance_id}:#{scope.tenant_id}:#{db}:#{dedup_id}"
+    end
+
     def run
       until @finished
         begin
@@ -136,24 +178,26 @@ module Fluent::Plugin
       now = Time.now.utc
       start_time = (now - @lookback_seconds).strftime('%Y-%m-%dT%H:%M:%SZ')
       end_time   = now.strftime('%Y-%m-%dT%H:%M:%SZ')
-      fetch_and_emit_samples(start_time, end_time)
+      @fetch_scopes.each do |scope|
+        fetch_and_emit_samples(start_time, end_time, scope)
+      end
     end
 
     # Fetch list then per-execution samples (one record per trace)
-    def fetch_and_emit_samples(start_time, end_time)
-      list_response = call_list_api(start_time, end_time)
+    def fetch_and_emit_samples(start_time, end_time, scope)
+      list_response = call_list_api(start_time, end_time, scope)
       return unless list_response
 
       sql_records = extract_records(list_response)
       return if sql_records.nil? || sql_records.empty?
 
       sql_ids = sql_records.map { |r| r['sqlId'] }.compact.uniq
-      log.debug "Found #{sql_ids.size} unique SQL IDs, fetching samples..."
+      log.debug "Found #{sql_ids.size} unique SQL IDs for instance=#{scope.instance_id} tenant=#{scope.tenant_id} db=#{scope.db_name.inspect}, fetching samples..."
 
       total_emitted = 0
 
       sql_ids.each do |sql_id|
-        samples = fetch_samples_for_sql(sql_id, start_time, end_time)
+        samples = fetch_samples_for_sql(sql_id, start_time, end_time, scope)
         next if samples.nil? || samples.empty?
 
         es = Fluent::MultiEventStream.new
@@ -163,13 +207,13 @@ module Fluent::Plugin
           dedup_id = trace_id || "#{sql_id}_#{sample['requestTime']}"
 
           if @deduplicate
-            dedup_key = :"trace_#{dedup_id}"
+            dedup_key = dedup_storage_key(scope, dedup_id)
             next if @seen_storage.get(dedup_key)
             @seen_storage.put(dedup_key, Time.now.to_i.to_s)
           end
 
           sample['ob_log_type'] = @log_type.to_s
-          sample = attach_metadata(sample, start_time, end_time) if @include_metadata
+          sample = attach_metadata(sample, start_time, end_time, scope) if @include_metadata
 
           event_time = if sample['requestTime']
                          begin
@@ -190,26 +234,26 @@ module Fluent::Plugin
         end
       end
 
-      log.info "Emitted #{total_emitted} #{@log_type} sample events (#{start_time} ~ #{end_time})" if total_emitted > 0
+      log.info "Emitted #{total_emitted} #{@log_type} sample events for instance=#{scope.instance_id} tenant=#{scope.tenant_id} (#{start_time} ~ #{end_time})" if total_emitted > 0
     end
 
-    def fetch_samples_for_sql(sql_id, start_time, end_time)
-      path = "/api/v2/instances/#{@instance_id}/tenants/#{@tenant_id}/sqls/#{sql_id}/samples"
+    def fetch_samples_for_sql(sql_id, start_time, end_time, scope)
+      path = "/api/v2/instances/#{scope.instance_id}/tenants/#{scope.tenant_id}/sqls/#{sql_id}/samples"
       params = {
         'startTime' => start_time,
         'endTime'   => end_time,
       }
-      params['dbName'] = @db_name if @db_name
+      params['dbName'] = scope.db_name if scope.db_name
 
       response = call_api_raw(path, params)
       return nil unless response
       extract_records(response)
     end
 
-    def attach_metadata(record, start_time, end_time)
+    def attach_metadata(record, start_time, end_time, scope)
       record.merge(
-        'ob_instance_id'   => @instance_id,
-        'ob_tenant_id'     => @tenant_id,
+        'ob_instance_id'   => scope.instance_id,
+        'ob_tenant_id'     => scope.tenant_id,
         'query_start_time' => start_time,
         'query_end_time'   => end_time
       )
@@ -235,13 +279,13 @@ module Fluent::Plugin
 
     # ---- API calls ----
 
-    def call_list_api(start_time, end_time)
-      path = "/api/v2/instances/#{@instance_id}/tenants/#{@tenant_id}/#{@api_path_segment}"
+    def call_list_api(start_time, end_time, scope)
+      path = "/api/v2/instances/#{scope.instance_id}/tenants/#{scope.tenant_id}/#{@api_path_segment}"
       params = {
         'startTime' => start_time,
         'endTime'   => end_time,
       }
-      params['dbName']          = @db_name          if @db_name
+      params['dbName']          = scope.db_name       if scope.db_name
       params['searchKeyWord']   = @search_keyword   if @search_keyword
       params['nodeIp']          = @node_ip           if @node_ip
       params['filterCondition'] = @filter_condition  if @filter_condition

data/lib/fluent/plugin/oceanbase/logs/version.rb

@@ -2,7 +2,7 @@ module Fluent
   module Plugin
     module OceanBase
       module Logs
-        VERSION = "0.1.3"
+        VERSION = "0.1.4"
       end
     end
   end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-oceanbase-logs
 version: !ruby/object:Gem::Version
-  version: 0.1.3
+  version: 0.1.4
 platform: ruby
 authors:
 - OceanBase Integrations