fluent-plugin-oceanbase-logs 0.1.0 → 0.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 3a3230feaa77c86a454f2733f9097f05a2e57f43dd418c1fed7af21d5345a875
-  data.tar.gz: d2a7f628ddb81778b10c6b9a14ffc1d6fa1fe6cbc4b1e7771da21840a42ef93e
+  metadata.gz: 562c1a532751f82c0aef77e9cbb283e074ad46d0252f85220be1c53edb332a00
+  data.tar.gz: b5a4b85ac83aa4b7bd980728b1ed2735faeff4d4d71d4b1744c98c92d9ff8521
 SHA512:
-  metadata.gz: f2f8cbbf267cdf26e929677364f9a0db2cfdc1256fa231bbdf85f347cd301d8fdbd26c1b219d2faed9a94cacfd177e5aad665f291f9aafc809ab8406f4e25d32
-  data.tar.gz: 9f1725a4250c1988960db905493ce797d66b096a6f586b3af6c3642956b3c5fb7194516d82c828f5c728b086dfe0fbc61f01ea39927a22dcf7742acad6c81a0c
+  metadata.gz: c6f100568de25bdafb0d8dcab796fd9d29a7a118e4383fee0331c3c110931eec4f0b95a3c10048f7f38024fcc338cdfb78ad0c4273657c57caa82d4ab78462ed
+  data.tar.gz: d4954f6b6b2af17fa111f22378f5dcb047f20e9bce6981c65ec8575898ea9c873d42ecc61154a6c116a5813e4c21ce5736c33c95ef2fb074e9ddb7f8f568db61

data/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 OceanBase Integrations
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/README.md

@@ -0,0 +1,165 @@
+# fluent-plugin-oceanbase-logs
+
+Fluentd input plugin that periodically fetches SQL diagnostics data from [OceanBase Cloud](https://www.oceanbase.com/). Each event is one SQL execution (sample), with deduplication by `traceId`.
+
+| `log_type` | API Path | Description |
+| --- | --- | --- |
+| `slow_sql` (default) | `/api/v2/.../slowSql` + samples | Slow SQL (per-execution) |
+| `top_sql` | `/api/v2/.../topSql` + samples | Top SQL (per-execution) |
+
+## Requirements
+
+| fluent-plugin-oceanbase-logs | fluentd   | ruby   |
+| ---------------------------- | --------- | ------ |
+| >= 0.1.0                     | >= 1.8.0  | >= 2.4 |
+
+## Installation
+
+```bash
+gem install fluent-plugin-oceanbase-logs
+```
+
+## Preparation
+
+1. Create an AccessKey pair at [OceanBase Cloud AccessKey Management](https://console-cn.oceanbase.com/account/accessKey)
+2. Find your **Instance ID** and **Tenant ID** from the OceanBase Cloud console
+
+## Configuration
+
+### Slow SQL → file (one record per execution)
+
+```xml
+<source>
+  @type oceanbase_logs
+  tag  oceanbase.slow_sql
+  log_type slow_sql
+
+  access_key_id     "#{ENV['OCEANBASE_ACCESS_KEY_ID']}"
+  access_key_secret "#{ENV['OCEANBASE_ACCESS_KEY_SECRET']}"
+  instance_id       "#{ENV['OCEANBASE_INSTANCE_ID']}"
+  tenant_id         "#{ENV['OCEANBASE_TENANT_ID']}"
+
+  # 服务接入点（API 地址）
+  endpoint          api-cloud-cn.oceanbase.com
+  # 采集间隔（秒）与采集时间范围（秒）
+  fetch_interval    60
+  lookback_seconds  600
+
+  deduplicate       true
+
+  <storage>
+    @type local
+    persistent true
+    path /var/log/fluentd/slow_sql_seen
+  </storage>
+</source>
+
+<match oceanbase.slow_sql>
+  @type file
+  path /var/log/fluentd/slow_sql
+  append true
+  <format>
+    @type json
+  </format>
+  <buffer>
+    @type file
+    path /var/log/fluentd/buffer/slow_sql
+    flush_mode interval
+    flush_interval 5s
+  </buffer>
+</match>
+```
+
+### Parameters
+
+#### Core
+
+| Parameter | Type | Required | Default | Description |
+| --- | --- | --- | --- | --- |
+| `log_type` | enum | no | `slow_sql` | `slow_sql` or `top_sql` |
+| `tag` | string | **yes** | — | Fluentd tag for emitted events |
+
+#### Authentication
+
+| Parameter | Type | Required | Default | Description |
+| --- | --- | --- | --- | --- |
+| `access_key_id` | string | **yes** | — | OceanBase Cloud AccessKey ID |
+| `access_key_secret` | string | **yes** | — | OceanBase Cloud AccessKey Secret |
+
+#### Cluster
+
+| Parameter | Type | Required | Default | Description |
+| --- | --- | --- | --- | --- |
+| `instance_id` | string | **yes** | — | OceanBase cluster instance ID |
+| `tenant_id` | string | **yes** | — | OceanBase tenant ID |
+| `project_id` | string | no | nil | Project ID (`X-Ob-Project-Id` header) |
+
+#### Filters
+
+| Parameter | Type | Required | Default | Description |
+| --- | --- | --- | --- | --- |
+| `db_name` | string | no | nil | Filter by database name |
+| `search_keyword` | string | no | nil | Search keyword in SQL text |
+| `node_ip` | string | no | nil | Database node IP |
+| `filter_condition` | string | no | nil | Advanced filter (e.g. `@avgCpuTime > 20`) |
+| `sql_text_length` | integer | no | 65535 | Max SQL text length returned |
+
+#### 服务接入点与采集时间（均在配置文件中配置）
+
+| Parameter | Type | Required | Default | Description |
+| --- | --- | --- | --- | --- |
+| `endpoint` | string | no | `api-cloud-cn.oceanbase.com` | 服务接入点（API 地址；国际站可用 `api-cloud.oceanbase.com`） |
+| `fetch_interval` | time | no | 300 (5 min) | 采集间隔（秒）：每隔多久请求一次 API |
+| `lookback_seconds` | integer | no | 600 (10 min) | 采集时间范围（秒）：每次请求查询最近多长时间的数据 |
+
+可通过环境变量覆盖：`OCEANBASE_ENDPOINT`、`OCEANBASE_FETCH_INTERVAL`、`OCEANBASE_LOOKBACK_SECONDS`（参见 `example/` 与 `.env.example`）。
+
+#### Behaviour
+
+| Parameter | Type | Required | Default | Description |
+| --- | --- | --- | --- | --- |
+| `deduplicate` | bool | no | true | Skip already-seen records (by traceId) |
+| `include_metadata` | bool | no | true | Attach `ob_instance_id` / `ob_tenant_id` / `ob_log_type` to records |
+| `ssl_verify_peer` | bool | no | true | Verify SSL certificates |
+| `http_proxy` | string | no | nil | HTTP proxy URL |
+
+（`endpoint` 见上表「服务接入点与采集时间」。）
+
+## Output record fields
+
+Each record is one SQL execution (sample). Main fields:
+
+| Field | Type | Description |
+| --- | --- | --- |
+| `sqlId` | string | SQL identifier |
+| `fullSqlText` | string | Complete SQL text |
+| `sqlTextShort` | string | SQL text (truncated) |
+| `sqlType` | string | `SELECT`, `UPDATE`, etc. |
+| `dbName` | string | Database name |
+| `userName` | string | User |
+| `requestTime` | string | Execution timestamp (UTC) |
+| `traceId` | string | Unique trace ID (used for dedup) |
+| `elapsedTime` | double | Response time (ms) |
+| `cpuTime` | double | CPU time (ms) |
+| `executeTime` | double | Plan execution time (ms) |
+| `returnRows` | long | Returned rows |
+| `affectedRows` | long | Affected rows |
+
+## Examples
+
+See the `example/` directory:
+
+- `fluentd.conf` — Slow SQL to file
+- `fluentd_to_file.conf` — Slow SQL + Top SQL to file
+- `fluentd_to_loki.conf` — Slow SQL + Top SQL to Grafana Loki
+
+## Test
+
+```bash
+bundle install
+bundle exec rake test
+```
+
+## License
+
+MIT

data/lib/fluent/plugin/in_oceanbase_logs.rb

@@ -22,8 +22,6 @@ module Fluent::Plugin
 
     config_param :log_type, :enum, list: LOG_TYPE_PATHS.keys.map(&:to_sym), default: :slow_sql,
       desc: "Type of SQL diagnostics to collect: slow_sql or top_sql."
-    config_param :fetch_samples, :bool, default: false,
-      desc: "When true, fetch per-execution sample details for each SQL (each execution = one event)."
 
     config_param :access_key_id, :string, secret: true,
       desc: "OceanBase Cloud AccessKey ID."
@@ -62,7 +60,7 @@ module Fluent::Plugin
       desc: "Verify SSL certificates."
 
     config_param :deduplicate, :bool, default: true,
-      desc: "Enable deduplication."
+      desc: "Enable deduplication (by traceId)."
     config_param :include_metadata, :bool, default: true,
       desc: "Attach instance_id / tenant_id / log_type to each record."
 
@@ -123,43 +121,10 @@ module Fluent::Plugin
       now = Time.now.utc
       start_time = (now - @lookback_seconds).strftime('%Y-%m-%dT%H:%M:%SZ')
       end_time   = now.strftime('%Y-%m-%dT%H:%M:%SZ')
-
-      if @fetch_samples
-        fetch_and_emit_samples(start_time, end_time)
-      else
-        fetch_and_emit_list(start_time, end_time)
-      end
+      fetch_and_emit_samples(start_time, end_time)
     end
 
-    # ---- Mode 1: emit aggregated list (one record per sqlId) ----
-    def fetch_and_emit_list(start_time, end_time)
-      response = call_list_api(start_time, end_time)
-      return unless response
-
-      records = extract_records(response)
-      return if records.nil? || records.empty?
-
-      es = Fluent::MultiEventStream.new
-
-      records.each do |record|
-        sql_id = record['sqlId']
-        next if sql_id.nil? || sql_id.empty?
-
-        if @deduplicate
-          dedup_key = :"list_#{sql_id}_#{start_time}"
-          next if @seen_storage.get(dedup_key)
-          @seen_storage.put(dedup_key, Time.now.to_i.to_s)
-        end
-
-        record = attach_metadata(record, start_time, end_time) if @include_metadata
-        es.add(Fluent::EventTime.now, record)
-      end
-
-      router.emit_stream(@tag, es) unless es.empty?
-      log.info "Emitted #{es.size} #{@log_type} events (#{start_time} ~ #{end_time})" if es.size > 0
-    end
-
-    # ---- Mode 2: emit per-execution samples (one record per trace) ----
+    # Fetch list then per-execution samples (one record per trace)
     def fetch_and_emit_samples(start_time, end_time)
       list_response = call_list_api(start_time, end_time)
       return unless list_response
@@ -236,16 +201,21 @@ module Fluent::Plugin
     end
 
     def extract_records(response)
-      if response['data'].is_a?(Hash) && response['data']['dataList'].is_a?(Array)
-        response['data']['dataList']
-      elsif response['Data'].is_a?(Array)
-        response['Data']
-      elsif response['data'].is_a?(Array)
-        response['data']
-      else
-        log.warn "Unexpected API response structure", keys: response.keys
-        nil
+      data = response['data']
+      return [] if data.nil?
+
+      if data.is_a?(Hash)
+        list = data['dataList']
+        return list if list.is_a?(Array)
+        return [] if list.nil? || list == []
+      elsif data.is_a?(Array)
+        return data
       end
+
+      log.warn "Unexpected API response structure",
+               keys: response.keys, data_class: data.class.name,
+               data_keys: (data.respond_to?(:keys) ? data.keys : nil)
+      []
     end
 
     # ---- API calls ----

data/lib/fluent/plugin/oceanbase/logs/version.rb

@@ -2,7 +2,7 @@ module Fluent
   module Plugin
     module OceanBase
       module Logs
-        VERSION = "0.1.0"
+        VERSION = "0.1.2"
       end
     end
   end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-oceanbase-logs
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.2
 platform: ruby
 authors:
 - OceanBase Integrations
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2026-03-06 00:00:00.000000000 Z
+date: 2026-03-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -94,14 +94,16 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '3.0'
-description: Periodically fetches slow SQL information from OceanBase Cloud via the
-  DescribeOasSlowSQLList API and emits them as Fluentd events.
+description: Fetches Slow SQL and Top SQL per-execution samples from OceanBase Cloud
+  and emits them as Fluentd events (one record per trace, dedup by traceId).
 email:
 - integrations@example.com
 executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- LICENSE
+- README.md
 - lib/fluent/plugin/in_oceanbase_logs.rb
 - lib/fluent/plugin/oceanbase/logs/version.rb
 homepage: https://github.com/your-org/fluent-plugin-oceanbase-logs
@@ -123,7 +125,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.3.5
+rubygems_version: 3.0.3.1
 signing_key: 
 specification_version: 4
 summary: Fluentd input plugin for OceanBase Cloud Logs