fluent-plugin-oceanbase-logs 0.1.0 → 0.1.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 3a3230feaa77c86a454f2733f9097f05a2e57f43dd418c1fed7af21d5345a875
-  data.tar.gz: d2a7f628ddb81778b10c6b9a14ffc1d6fa1fe6cbc4b1e7771da21840a42ef93e
+  metadata.gz: ff677af2c1882b27e86cde4cbfef8ef79039f45eae20e2a7c8eebd4c2fdbd98e
+  data.tar.gz: 2fbe0571f3c1e1b840ab6714faff44e5a7396cad011c4845fb710964db9a490b
 SHA512:
-  metadata.gz: f2f8cbbf267cdf26e929677364f9a0db2cfdc1256fa231bbdf85f347cd301d8fdbd26c1b219d2faed9a94cacfd177e5aad665f291f9aafc809ab8406f4e25d32
-  data.tar.gz: 9f1725a4250c1988960db905493ce797d66b096a6f586b3af6c3642956b3c5fb7194516d82c828f5c728b086dfe0fbc61f01ea39927a22dcf7742acad6c81a0c
+  metadata.gz: 380099690cb075af74c720a6eb147fdb5a1234b428bac1d54e1ea3a36d191f2b47b2a7962ed97ee7f5f2c9f576aad55a4cf025bba39bd8152747fdc6439242bf
+  data.tar.gz: 36cd5b13ca5fd22159451bc2c15c45c7861ad1122a62d9e95a4184c2e5e0a5f49c08183609e7767125d65330b69adf506060b9c59b228f785b641c56c41add20

data/LICENSE

@@ -0,0 +1,201 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

data/README.md

@@ -0,0 +1,107 @@
+# fluent-plugin-oceanbase-logs
+
+Fluentd **input** plugin: periodically pulls SQL diagnostics from [OceanBase Cloud](https://www.oceanbase.com/) . Each event is **one execution sample**
+
+| `log_type` | API | Meaning |
+| --- | --- | --- |
+| `slow_sql` (default) | `…/slowSql` + per-`sqlId` samples | Slow SQL |
+| `top_sql` | `…/topSql` + samples | Top SQL |
+
+Every record includes **`ob_log_type`** (`slow_sql` or `top_sql`). With `include_metadata true` (default), records also get `ob_instance_id`, `ob_tenant_id`, and the query time window.
+
+## Requirements
+
+| Gem | Fluentd | Ruby |
+| --- | --- | --- |
+| >= 0.1.2 | >= 1.8.0 | >= 2.4 |
+
+For **Grafana Loki** output you additionally need [fluent-plugin-grafana-loki](https://github.com/grafana/fluent-plugin-grafana-loki).
+
+## Installation
+
+```bash
+gem install fluent-plugin-oceanbase-logs
+```
+
+## Preparation
+
+1. Create an AccessKey in [OceanBase Cloud — AccessKey](https://console-cn.oceanbase.com/account/accessKey).
+2. In the console, copy **Instance ID** and **Tenant ID** 
+
+
+## Configuration
+
+### Environment variables
+
+Typical pattern in Fluentd:
+
+```text
+access_key_id     "#{ENV['OCEANBASE_ACCESS_KEY_ID']}"
+access_key_secret "#{ENV['OCEANBASE_ACCESS_KEY_SECRET']}"
+instance_id       "#{ENV['OCEANBASE_INSTANCE_ID']}"
+tenant_id         "#{ENV['OCEANBASE_TENANT_ID']}"
+```
+
+Optional: `OCEANBASE_ENDPOINT`, `OCEANBASE_FETCH_INTERVAL`, `OCEANBASE_LOOKBACK_SECONDS`, `OCEANBASE_DB_NAME`, `OCEANBASE_SEARCH_KEYWORD`, `OCEANBASE_PROJECT_ID` — see `.env.example` and the Docker table below.
+
+### Example: Slow SQL → JSON file
+
+Full sample: [`example/fluentd.conf`](example/fluentd.conf).
+
+```conf
+<source>
+  @type oceanbase_logs
+  tag  oceanbase.slow_sql
+  log_type slow_sql
+  access_key_id     "#{ENV['OCEANBASE_ACCESS_KEY_ID']}"
+  access_key_secret "#{ENV['OCEANBASE_ACCESS_KEY_SECRET']}"
+  instance_id       "#{ENV['OCEANBASE_INSTANCE_ID']}"
+  tenant_id         "#{ENV['OCEANBASE_TENANT_ID']}"
+  endpoint          api-cloud-cn.oceanbase.com
+  fetch_interval    60
+  lookback_seconds  600
+  deduplicate       true
+  include_metadata  true
+  <storage>
+    @type local
+    persistent true
+    path /var/log/fluentd/slow_sql_seen
+  </storage>
+</source>
+
+<match oceanbase.slow_sql>
+  @type file
+  path /var/log/fluentd/slow_sql
+  append true
+  <format>
+    @type json
+  </format>
+</match>
+```
+
+### Example: Loki + Docker Compose
+
+Ready-made stack (Loki, Fluentd, Grafana) and a **slow_sql + top_sql**：
+
+```bash
+cp .env.example .env   # fill in secrets
+cd example/oceanbase2loki-docker && docker compose up -d
+```
+
+**Compose-related environment** (host `.env` or exports):
+
+| Variable | Required | Default |
+| --- | --- | --- |
+| `LOKI_URL` | no | `http://loki:3100` |
+| `OCEANBASE_ACCESS_KEY_ID` | **yes** | — |
+| `OCEANBASE_ACCESS_KEY_SECRET` | **yes** | — |
+| `OCEANBASE_INSTANCE_ID` | **yes** | — |
+| `OCEANBASE_TENANT_ID` | **yes** | — |
+| `OCEANBASE_ENDPOINT` | no | `api-cloud-cn.oceanbase.com` |
+| `OCEANBASE_FETCH_INTERVAL` | no | `60` |
+| `OCEANBASE_LOOKBACK_SECONDS` | no | `600` |
+| `OCEANBASE_DB_NAME` | no | `test` |
+| `OCEANBASE_SEARCH_KEYWORD` | no | `SELECT` |
+| `OCEANBASE_PROJECT_ID` | no | *(unset)* |
+
+> The bundled `fluentd-to-loki.conf` uses `record.dig` and avoids `";` / `]` patterns that break Fluentd’s config parser and `record_transformer` — keep that style if you edit it.

data/lib/fluent/plugin/in_oceanbase_logs.rb

@@ -22,8 +22,6 @@ module Fluent::Plugin
 
     config_param :log_type, :enum, list: LOG_TYPE_PATHS.keys.map(&:to_sym), default: :slow_sql,
       desc: "Type of SQL diagnostics to collect: slow_sql or top_sql."
-    config_param :fetch_samples, :bool, default: false,
-      desc: "When true, fetch per-execution sample details for each SQL (each execution = one event)."
 
     config_param :access_key_id, :string, secret: true,
       desc: "OceanBase Cloud AccessKey ID."
@@ -62,7 +60,7 @@ module Fluent::Plugin
       desc: "Verify SSL certificates."
 
     config_param :deduplicate, :bool, default: true,
-      desc: "Enable deduplication."
+      desc: "Enable deduplication (by traceId)."
     config_param :include_metadata, :bool, default: true,
       desc: "Attach instance_id / tenant_id / log_type to each record."
 
@@ -76,6 +74,21 @@ module Fluent::Plugin
       super
       @endpoint = @endpoint.to_s.strip
       @endpoint = 'api-cloud-cn.oceanbase.com' if @endpoint.empty?
+
+      %i[@access_key_id @access_key_secret @instance_id @tenant_id].each do |iv|
+        v = instance_variable_get(iv)
+        next unless v.is_a?(String)
+        instance_variable_set(iv, v.strip)
+      end
+      raise Fluent::ConfigError, 'access_key_id is required and cannot be empty' if @access_key_id.empty?
+      raise Fluent::ConfigError, 'access_key_secret is required and cannot be empty' if @access_key_secret.empty?
+      raise Fluent::ConfigError, 'instance_id is required and cannot be empty (e.g. set OCEANBASE_INSTANCE_ID)' if @instance_id.empty?
+      raise Fluent::ConfigError, 'tenant_id is required and cannot be empty (e.g. set OCEANBASE_TENANT_ID)' if @tenant_id.empty?
+
+      %i[@db_name @search_keyword @node_ip @filter_condition @project_id].each do |iv|
+        v = instance_variable_get(iv)
+        instance_variable_set(iv, nil) if v.is_a?(String) && v.strip.empty?
+      end
       @api_path_segment = LOG_TYPE_PATHS[@log_type.to_s]
       if @deduplicate
         @seen_storage = storage_create(
@@ -123,43 +136,10 @@ module Fluent::Plugin
       now = Time.now.utc
       start_time = (now - @lookback_seconds).strftime('%Y-%m-%dT%H:%M:%SZ')
       end_time   = now.strftime('%Y-%m-%dT%H:%M:%SZ')
-
-      if @fetch_samples
-        fetch_and_emit_samples(start_time, end_time)
-      else
-        fetch_and_emit_list(start_time, end_time)
-      end
-    end
-
-    # ---- Mode 1: emit aggregated list (one record per sqlId) ----
-    def fetch_and_emit_list(start_time, end_time)
-      response = call_list_api(start_time, end_time)
-      return unless response
-
-      records = extract_records(response)
-      return if records.nil? || records.empty?
-
-      es = Fluent::MultiEventStream.new
-
-      records.each do |record|
-        sql_id = record['sqlId']
-        next if sql_id.nil? || sql_id.empty?
-
-        if @deduplicate
-          dedup_key = :"list_#{sql_id}_#{start_time}"
-          next if @seen_storage.get(dedup_key)
-          @seen_storage.put(dedup_key, Time.now.to_i.to_s)
-        end
-
-        record = attach_metadata(record, start_time, end_time) if @include_metadata
-        es.add(Fluent::EventTime.now, record)
-      end
-
-      router.emit_stream(@tag, es) unless es.empty?
-      log.info "Emitted #{es.size} #{@log_type} events (#{start_time} ~ #{end_time})" if es.size > 0
+      fetch_and_emit_samples(start_time, end_time)
     end
 
-    # ---- Mode 2: emit per-execution samples (one record per trace) ----
+    # Fetch list then per-execution samples (one record per trace)
     def fetch_and_emit_samples(start_time, end_time)
       list_response = call_list_api(start_time, end_time)
       return unless list_response
@@ -188,6 +168,7 @@ module Fluent::Plugin
             @seen_storage.put(dedup_key, Time.now.to_i.to_s)
           end
 
+          sample['ob_log_type'] = @log_type.to_s
           sample = attach_metadata(sample, start_time, end_time) if @include_metadata
 
           event_time = if sample['requestTime']
@@ -229,23 +210,27 @@ module Fluent::Plugin
       record.merge(
         'ob_instance_id'   => @instance_id,
         'ob_tenant_id'     => @tenant_id,
-        'ob_log_type'      => @log_type.to_s,
         'query_start_time' => start_time,
         'query_end_time'   => end_time
       )
     end
 
     def extract_records(response)
-      if response['data'].is_a?(Hash) && response['data']['dataList'].is_a?(Array)
-        response['data']['dataList']
-      elsif response['Data'].is_a?(Array)
-        response['Data']
-      elsif response['data'].is_a?(Array)
-        response['data']
-      else
-        log.warn "Unexpected API response structure", keys: response.keys
-        nil
+      data = response['data']
+      return [] if data.nil?
+
+      if data.is_a?(Hash)
+        list = data['dataList']
+        return list if list.is_a?(Array)
+        return [] if list.nil? || list == []
+      elsif data.is_a?(Array)
+        return data
       end
+
+      log.warn "Unexpected API response structure",
+               keys: response.keys, data_class: data.class.name,
+               data_keys: (data.respond_to?(:keys) ? data.keys : nil)
+      []
     end
 
     # ---- API calls ----
@@ -277,7 +262,16 @@ module Fluent::Plugin
       end
 
       unless resp && resp.code.to_i == 200
-        log.error "OceanBase API HTTP #{resp&.code}", body: resp&.body, path: path
+        code = resp&.code
+        raw  = resp&.body.to_s
+        detail = nil
+        begin
+          j = JSON.parse(raw)
+          detail = j['message'] || j['errorMessage'] || j['msg']
+        rescue JSON::ParserError
+        end
+        log.error "OceanBase API HTTP #{code}",
+                  message: detail, body: (raw.bytesize > 512 ? raw.byteslice(0, 512) + '...' : raw), path: path
         return nil
       end
 

data/lib/fluent/plugin/oceanbase/logs/version.rb

@@ -2,7 +2,7 @@ module Fluent
   module Plugin
     module OceanBase
       module Logs
-        VERSION = "0.1.0"
+        VERSION = "0.1.3"
       end
     end
   end

metadata

@@ -1,14 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-oceanbase-logs
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.3
 platform: ruby
 authors:
 - OceanBase Integrations
-autorequire: 
 bindir: bin
 cert_chain: []
-date: 2026-03-06 00:00:00.000000000 Z
+date: 1980-01-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -66,20 +65,6 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '3.0'
-- !ruby/object:Gem::Dependency
-  name: test-unit-rr
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
 - !ruby/object:Gem::Dependency
   name: webmock
   requirement: !ruby/object:Gem::Requirement
@@ -94,21 +79,22 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '3.0'
-description: Periodically fetches slow SQL information from OceanBase Cloud via the
-  DescribeOasSlowSQLList API and emits them as Fluentd events.
+description: Fetches Slow SQL and Top SQL per-execution samples from OceanBase Cloud
+  and emits them as Fluentd events (one record per trace, dedup by traceId).
 email:
 - integrations@example.com
 executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- LICENSE
+- README.md
 - lib/fluent/plugin/in_oceanbase_logs.rb
 - lib/fluent/plugin/oceanbase/logs/version.rb
-homepage: https://github.com/your-org/fluent-plugin-oceanbase-logs
+homepage: https://github.com/sc-source/fluent-plugin-oceanbase-logs
 licenses:
-- MIT
+- Apache-2.0
 metadata: {}
-post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
@@ -123,8 +109,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.3.5
-signing_key: 
+rubygems_version: 3.6.9
 specification_version: 4
 summary: Fluentd input plugin for OceanBase Cloud Logs
 test_files: []