fluent-plugin-elb-log 1.3.2 → 1.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 70cc28816877d7449177587fc80f76cb571ff1f2a73cf87725835baceacd1fa1
-  data.tar.gz: ed8073d78f9433d8fdfbb5a499725fb55751b2f14710c48e9b02a7edfc5d2f6a
+  metadata.gz: b90a7ba2cb9c1a065697787d2634625df53bb3f28b2f4a3d88c940071255b848
+  data.tar.gz: 899bbcf97b460fbf709f4aee85a6ab418c8d58cd5fb53f7aeabaf54fcb729511
 SHA512:
-  metadata.gz: '038e446b92d1034613eb244e4b540b2daa0668942bd662f8a807b9f01c49fe273247e22d4a321f1eac007a19f999244c86e933fb2c8c686a406b83e9b35b350d'
-  data.tar.gz: 2355ddaec49a2b56e1fd69ae7703659ac7d9852d7e3a6ea10d44832de406dd2486690cb5fd7aa083d393c3054e0858fd9c21fbe35f1e264e1effca95ffe0a284
+  metadata.gz: dabbbdc5f79f83ba6003057126d62e492da600584840d72d2240eaea1bcc532cf56d9177762b0aa1cdc055e88bfb1d7840f7a11c6b95861905df3dd4e28eedbe
+  data.tar.gz: 48d929caa219fc1b8ef6aa2c5e971cfe2ea1c1f1f6a556ff431b3a7e338eb0c8f9a8701f284aec9ec5e71a7c1304f6cf0d0e403edccfe9d6361324bc362b76a4

data/README.md

@@ -58,6 +58,9 @@ SSL_CERT_FILE=/etc/ssl/certs/ca-bundle.crt (If you using amazon linux)
   tag               <tag name(default: elb.access)>
   delete            <boolean delete processed log files from S3(default: false)>
   include_all_message <boolean (default:false)>
+  start_time        <last timestamp to start from>
+  exclude_pattern_logfile_elb_name <exclude pattern for logfile_elb_name key>
+  use_sqs           <boolean use SQS polling instead of iteratively processing the whole bucket>
 
   # following attibutes are required if you don't use IAM Role
   access_key_id     <access_key>
@@ -65,6 +68,44 @@ SSL_CERT_FILE=/etc/ssl/certs/ca-bundle.crt (If you using amazon linux)
 </source>
 ```
 
+`use_sqs` automatically creates an SQS queue named `fluent-plugin-elb-log-<current_timestamp>` 
+and sets up the `all object create event` S3 event notification for the chosen S3 bucket. Stopping fluentd deletes both autmatically.
+To make it work, the following IAM policy should be attached to the instance IAM role or the user:
+```
+{
+    "Version": "2012-10-17",
+    "Statement": [
+        {
+            "Sid": "FluentdPermissions",
+            "Effect": "Allow",
+            "Action": [
+                "sqs:DeleteMessage",
+                "s3:GetObject",
+                "sqs:ReceiveMessage",
+                "sqs:DeleteQueue",
+                "sqs:GetQueueAttributes",
+                "s3:ListBucket",
+                "s3:PutBucketNotification",
+                "sqs:CreateQueue"
+            ],
+            "Resource": [
+                "arn:aws:sqs:*:123456789012:fluent-plugin-elb-log-*",
+                "arn:aws:s3:::alb-logs-bucket/*",
+                "arn:aws:s3:::alb-logs-bucket"
+            ]
+        }
+    ]
+}
+```
+When `use_sqs` is false:
+  - 300 seconds is a good value for the `refresh_interval`
+  - the plugin executes processing the whole S3 bucket (with the respect of `s3_prefix` and `start_time`/`timestamp_file`) every `refresh_interval`
+
+When `use_sqs` is true:
+  - `refresh_interval` of 30-60 seconds should be fine.
+  - the plugin executes processing the whole S3 bucket (with the respect of `s3_prefix` and `start_time`/`timestamp_file`) only once (at start) and then 
+polls the SQS queue every `refresh_interval`.
+
 ### Example setting
 ```config
 <source>
@@ -74,16 +115,39 @@ SSL_CERT_FILE=/etc/ssl/certs/ca-bundle.crt (If you using amazon linux)
   s3_prefix         prodcution/web
   timestamp_file    /tmp/elb_last_at.dat
   buf_file          /tmp/fluentd-elblog.tmpfile
-  refresh_interval  300
+  refresh_interval  30
   tag               elb.access
   delete            false
   include_all_message false
+  exclude_pattern_logfile_elb_name "^app\.(uat|qa)\."
+  start_time        2025-02-27T10:45:00
+  use_sqs           true
   access_key_id     XXXXXXXXXXXXXXXXXXXX
   secret_access_key xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
 </source>
 
-<match **>
-  @type stdout
+<filter elb.access>
+  @type record_transformer
+  <record>
+    timestamp ${record["request_creation_time"]}
+    logfile_name ${record["key"]}
+  </record>
+  remove_keys prefix,logfile_date,logfile_elb_name,logfile_hash,logfile_timestamp,logfile_timestamp_unixtime,key,time,s3_last_modified_unixtime
+</filter>
+
+<match elb.access>
+#  @type stdout
+  @type opensearch
+  hosts node-1,node-2,node-3
+  port 9200
+  scheme https
+  logstash_format true
+  logstash_prefix alb-logs
+  user fluentd
+  password secret
+  ssl_verify true
+  ca_file /etc/fluentd/root-ca.pem
+  flush_interval 300s
 </match>
 ```
 
@@ -96,7 +160,7 @@ SSL_CERT_FILE=/etc/ssl/certs/ca-bundle.crt (If you using amazon linux)
     "logfile_elb_name":"my-elb-name",
     "elb_ip_address":"52.0.0.0",
     "logfile_hash":"12squv5w",
-    "elb_timestamp":"20150615T0400Z",
+    "logfile_timestamp":"20150615T0400Z",
     "key":"TEST/AWSLogs/123456789012/elasticloadbalancing/ap-northeast-1/2015/06/15/123456789012_elasticloadbalancing_ap-northeast-1_my-elb-name_20150615T0400Z_52.68.215.138_69squv5w.log",
     "prefix":"TEST",
     "elb_timestamp_unixtime":1434340800,
@@ -104,13 +168,13 @@ SSL_CERT_FILE=/etc/ssl/certs/ca-bundle.crt (If you using amazon linux)
     "elb":"my-elb-name",
     "client":"54.1.1.1",
     "client_port":"43759",
-    "backend":"10.0.0.1",
-    "backend_port":"80",
+    "target":"10.0.0.1",
+    "target_port":"80",
     "request_processing_time":4.0e-05,
-    "backend_processing_time":0.105048,
+    "target_processing_time":0.105048,
     "response_processing_time":2.4e-05,
     "elb_status_code":"200",
-    "backend_status_code":"200",
+    "target_status_code":"200",
     "received_bytes":0,
     "sent_bytes":4622,
     "request_method":"GET",
@@ -129,8 +193,10 @@ SSL_CERT_FILE=/etc/ssl/certs/ca-bundle.crt (If you using amazon linux)
     "actions_executed": "forward",
     "redirect_url": "-",
     "error_reason": "-",
-    "option1": "\"192.168.0.1:443\"",
-    "option2": "\"301\"",
-    "option3": null
+    "target_port_list": "\"192.168.0.1:443\"",
+    "target_status_code_list": "\"301\"",
+    "classification": "-",
+    "classification_reason": "-",
+    "conn_trace_id": "TID_xxxxxxxxxxxxxxxxxxxxxxx"
 }
 ```

data/fluent-plugin-elb-log.gemspec

@@ -4,12 +4,12 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name          = "fluent-plugin-elb-log"
-  spec.version       = "1.3.2"
-  spec.authors       = ["shinsaka"]
+  spec.version       = "1.4.0"
+  spec.authors       = ["shinsaka","jazzl0ver"]
   spec.email         = ["shinx1265@gmail.com"]
   spec.summary       = "Amazon ELB log input plugin"
   spec.description   = "Amazon ELB log input plugin for fluentd"
-  spec.homepage      = "https://github.com/shinsaka/fluent-plugin-elb-log"
+  spec.homepage      = "https://github.com/jazzl0ver/fluent-plugin-elb-log"
   spec.license       = "MIT"
 
   spec.files         = `git ls-files -z`.split("\x0")

data/fluent.conf.sample

@@ -5,12 +5,22 @@
   s3_prefix         prodcution/web
   timestamp_file    /tmp/elb_last_at.dat
   buf_file          /tmp/fluentd-elblog.tmpfile
-  refresh_interval  300
+  refresh_interval  30
   tag               elb.access
+  use_sqs           true
   access_key_id     XXXXXXXXXXXXXXXXXXXX
   secret_access_key xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
 </source>
 
+<filter elb.access>
+  @type record_transformer
+  <record>
+    timestamp ${record["request_creation_time"]}
+    logfile_name ${record["key"]}
+  </record>
+  remove_keys prefix,logfile_date,logfile_elb_name,logfile_hash,logfile_timestamp,logfile_timestamp_unixtime,key,time,s3_last_modified_unixtime
+</filter>
+
 <match elb.access>
-  @type stdout
+    @type stdout
 </match>

data/lib/fluent/plugin/in_elb_log.rb

@@ -3,6 +3,7 @@ require 'zlib'
 require 'fileutils'
 require 'aws-sdk-s3'
 require 'aws-sdk-ec2'
+require 'aws-sdk-sqs'
 require 'fluent/input'
 require 'digest/sha1'
 
@@ -11,8 +12,8 @@ class Fluent::Plugin::Elb_LogInput < Fluent::Plugin::Input
 
   helpers :timer
 
-  LOGFILE_REGEXP = /^((?<prefix>.+?)\/|)AWSLogs\/(?<account_id>[0-9]{12})\/elasticloadbalancing\/(?<region>.+?)\/(?<logfile_date>[0-9]{4}\/[0-9]{2}\/[0-9]{2})\/[0-9]{12}_elasticloadbalancing_.+?_(?<logfile_elb_name>[^_]+)_(?<elb_timestamp>[0-9]{8}T[0-9]{4}Z)_(?<elb_ip_address>.+?)_(?<logfile_hash>.+)\.log(.gz)?$/
-  ACCESSLOG_REGEXP = /^((?<type>[a-z0-9]+) )?(?<time>\d{4}-\d{2}-\d{2}T\d{2}\:\d{2}\:\d{2}\.\d{6}Z) (?<elb>.+?) (?<client>[^ ]+)\:(?<client_port>.+?) (?<backend>.+?)(\:(?<backend_port>.+?))? (?<request_processing_time>.+?) (?<backend_processing_time>.+?) (?<response_processing_time>.+?) (?<elb_status_code>.+?) (?<backend_status_code>.+?) (?<received_bytes>.+?) (?<sent_bytes>.+?) \"(?<request_method>.+?) (?<request_uri>.+?) (?<request_protocol>.+?)\"(\s+\"(?<user_agent>.+?)\" (?<ssl_cipher>.+?) (?<ssl_protocol>[^\s]+)( (?<target_group_arn>arn:\S+) (?<trace_id>[^\s]+))?( \"(?<domain_name>.+?)\" \"(?<chosen_cert_arn>.+?)\" (?<matched_rule_priority>.+?) (?<request_creation_time>.+?) \"(?<actions_executed>.+?)\" \"(?<redirect_url>.+?)\" \"(?<error_reason>[^\s]+)\"( |$))?((?<option1>[^\s]+)( |$))?((?<option2>[^\s]+)( |$))?( (?<option3>.*))?)?/
+  LOGFILE_REGEXP = /^((?<prefix>.+?)\/|)AWSLogs\/(?<account_id>[0-9]{12})\/elasticloadbalancing\/(?<region>.+?)\/(?<logfile_date>[0-9]{4}\/[0-9]{2}\/[0-9]{2})\/[0-9]{12}_elasticloadbalancing_.+?_(?<logfile_elb_name>[^_]+)_(?<logfile_timestamp>[0-9]{8}T[0-9]{4}Z)_(?<elb_ip_address>.+?)_(?<logfile_hash>.+)\.log(.gz)?$/
+  ACCESSLOG_REGEXP = /^((?<type>[a-z0-9]+) )?(?<time>\d{4}-\d{2}-\d{2}T\d{2}\:\d{2}\:\d{2}\.\d{6}Z) (?<elb>\S+) (?<client>\S+)\:(?<client_port>\S+) (?<target>[^:\s]+)(?::(?<target_port>\S+))? (?<request_processing_time>\S+) (?<target_processing_time>\S+) (?<response_processing_time>\S+) (?<elb_status_code>\S+) (?<target_status_code>\S+) (?<received_bytes>\S+) (?<sent_bytes>\S+) \"(?<request_method>\S+) (?<request_uri>\S+) (?<request_protocol>\S+)\" \"(?<user_agent>.+?)\" (?<ssl_cipher>\S+) (?<ssl_protocol>\S+) (?<target_group_arn>\S+) \"(?<trace_id>\S+)\" \"(?<domain_name>\S+)\" \"(?<chosen_cert_arn>\S+)\" (?<matched_rule_priority>\S+) (?<request_creation_time>\S+) \"(?<actions_executed>\S+)\" \"(?<redirect_url>\S+)\" \"(?<error_reason>\S+)\" \"(?<target_port_list>\S+)\" \"(?<target_status_code_list>\S+)\" \"(?<classification>\S+)\" \"(?<classification_reason>\S+)\" (?<conn_trace_id>\S+)/
   config_param :access_key_id, :string, default: nil, secret: true
   config_param :secret_access_key, :string, default: nil, secret: true
   config_param :region, :string
@@ -28,6 +29,8 @@ class Fluent::Plugin::Elb_LogInput < Fluent::Plugin::Input
   config_param :num_nodes, :integer, default: 1
   config_param :node_no, :integer, default: 0
   config_param :include_all_message, :bool, default: false
+  config_param :exclude_pattern_logfile_elb_name, :string, default: nil
+  config_param :use_sqs, :bool, default: true
 
   def configure(conf)
     super
@@ -38,9 +41,16 @@ class Fluent::Plugin::Elb_LogInput < Fluent::Plugin::Input
     end
     raise Fluent::ConfigError.new("s3_bucketname is required") unless @s3_bucketname
     raise Fluent::ConfigError.new("timestamp_file is required") unless @timestamp_file
+
+    @s3_client = s3_client
     raise Fluent::ConfigError.new("s3 bucket not found #{@s3_bucketname}") unless s3bucket_is_ok?
   end
 
+  def initialize
+    super
+    @running = true
+  end
+
   def start
     super
 
@@ -48,11 +58,62 @@ class Fluent::Plugin::Elb_LogInput < Fluent::Plugin::Input
     File.open(@timestamp_file, File::RDWR|File::CREAT).close
     File.open(@buf_file, File::RDWR|File::CREAT).close
 
-    timer_execute(:in_elb_log, @refresh_interval, &method(:input))
+    @exclude_pattern_logfile_elb_name_re = Regexp.new(@exclude_pattern_logfile_elb_name)
+
+    Signal.trap('INT') { shutdown }
+
+    if @use_sqs
+      input
+      @sqs_client = sqs_client
+      setup_sqs_timer
+    else
+      setup_input_timer
+    end
   end
 
   private
 
+  def shutdown
+    log.debug "shutdown"
+    if @running
+	@running = false
+        if @use_sqs
+          log.debug "pausing shutdown for 2x#{@refresh_interval}"
+          sleep 2*@refresh_interval
+          unset_sqs if @queue_url && @queue_url != ""
+        else
+          log.debug "pausing shutdown for 30 seconds"
+          sleep 30
+        end
+    end
+  end
+
+  def setup_sqs_timer
+    if @running
+      setup_sqs
+      timer_execute(:in_elb_log, @refresh_interval) do
+	if @running
+    	    process_sqs
+    	    log.debug "sleeping for #{@refresh_interval}"
+    	else
+    	    log.debug "Unsetting SQS"
+	    unset_sqs if @queue_url && @queue_url != ""
+	end
+      end
+    end
+  end
+
+  def setup_input_timer
+    if @running
+      timer_execute(:in_elb_log, @refresh_interval) do
+	if @running
+    	    input
+	    log.debug "sleeping input for #{@refresh_interval}"
+	end
+      end
+    end
+  end
+
   def has_iam_role?
     begin
       ec2 = Aws::EC2::Client.new(region: @region)
@@ -62,6 +123,99 @@ class Fluent::Plugin::Elb_LogInput < Fluent::Plugin::Input
     end
   end
 
+  def setup_sqs
+    begin
+    timestamp = Time.now.to_i
+    queue_name = "fluent-plugin-elb-log-#{timestamp}"
+
+    sts_client = Aws::STS::Client.new
+    account_id = sts_client.get_caller_identity.account
+
+    queue_policy = {
+      "Version": "2012-10-17",
+      "Id": "__default_policy_ID",
+      "Statement": [
+        {
+          "Sid": "S3_service_publish",
+          "Effect": "Allow",
+          "Principal": {
+            "Service": "s3.amazonaws.com"
+          },
+          "Action": "SQS:SendMessage",
+          "Resource": "arn:aws:sqs:us-east-1:#{account_id}:#{queue_name}",
+          "Condition": {
+            "StringEquals": {
+              "aws:SourceAccount": "#{account_id}"
+	    },
+            "ArnLike": {
+    	      "aws:SourceArn": "arn:aws:s3:*:*:#{@s3_bucket}"
+	    }
+          }
+        }
+      ]
+    }.to_json
+    create_queue_response = @sqs_client.create_queue(queue_name: queue_name,
+      attributes: {
+        'Policy' => queue_policy
+      }
+    )
+    @queue_url = create_queue_response.queue_url
+    queue_attributes = @sqs_client.get_queue_attributes(
+      queue_url: @queue_url,
+      attribute_names: ['QueueArn']
+    )
+    queue_arn = queue_attributes.attributes['QueueArn']
+    log.debug "New SQS queue created: #{queue_arn}"
+
+    notification_configuration = {
+      queue_configurations: [
+	{
+	  id: "fluent-plugin-elb-log",
+          events: ['s3:ObjectCreated:*'],
+          queue_arn: "#{queue_arn}",
+          filter: {
+            key: {
+              filter_rules: [
+                {
+                  name: 'Prefix',
+                  value: "#{@s3_prefix}"
+                }
+              ]
+            }
+          }
+        }
+      ]
+    }
+
+    @s3_client.put_bucket_notification_configuration(
+      bucket: @s3_bucketname,
+      notification_configuration: notification_configuration
+    )
+    log.debug "S3 notification events has been set for #{@s3_bucketname}/#{@s3_prefix}"
+
+    rescue Aws::SQS::Errors::InvalidAttributeValue => e
+      log.debug "SQS error: #{e.message}"
+
+    rescue Aws::S3::Errors::InvalidArgument => e
+      log.debug "S3 Event error: #{e.message}"
+      sqs_client.delete_queue(queue_url: @queue_url)
+      log.debug "#{queue_arn} deleted"
+    end
+  end
+
+  def unset_sqs
+    return if @queue_url.nil? || @queue_url == ""
+    @s3_client.put_bucket_notification_configuration(
+      bucket: @s3_bucketname,
+      notification_configuration: {}
+    )
+    log.debug "S3 notification events has been removed"
+
+    @sqs_client.delete_queue(queue_url: @queue_url)
+    log.debug "SQS queue #{@queue_url} has been removed"
+    @queue_url = ""
+  end
+
   def get_timestamp_file
     begin
       # get timestamp last proc
@@ -125,15 +279,34 @@ class Fluent::Plugin::Elb_LogInput < Fluent::Plugin::Input
     end
   end
 
+  def sqs_client
+    begin
+      options = {
+        region: @region,
+      }
+      if @access_key_id && @secret_access_key
+        options[:access_key_id] = @access_key_id
+        options[:secret_access_key] = @secret_access_key
+      end
+      if @http_proxy
+        options[:http_proxy] = @http_proxy
+      end
+      log.debug "SQS client connect"
+      Aws::SQS::Client.new(options)
+    rescue => e
+      log.warn "SQS Client error occurred: #{e.message}"
+    end
+  end
+
   def input
     begin
-      log.debug "start"
+      log.debug "input start"
       timestamp = get_timestamp_file()
 
       object_keys = get_object_keys(timestamp)
       object_keys = sort_object_key(object_keys)
 
-      log.info "processing #{object_keys.count} object(s)."
+      log.info "found #{object_keys.count} new object(s)."
 
       object_keys.each do |object_key|
         record_common = {
@@ -143,10 +316,10 @@ class Fluent::Plugin::Elb_LogInput < Fluent::Plugin::Input
           "logfile_elb_name" => object_key[:logfile_elb_name],
           "elb_ip_address" => object_key[:elb_ip_address],
           "logfile_hash" => object_key[:logfile_hash],
-          "elb_timestamp" => object_key[:elb_timestamp],
+          "logfile_timestamp" => object_key[:logfile_timestamp],
           "key" => object_key[:key],
           "prefix" => object_key[:prefix],
-          "elb_timestamp_unixtime" => object_key[:elb_timestamp_unixtime],
+          "logfile_timestamp_unixtime" => object_key[:logfile_timestamp_unixtime],
           "s3_last_modified_unixtime" => object_key[:s3_last_modified_unixtime],
         }
 
@@ -164,6 +337,89 @@ class Fluent::Plugin::Elb_LogInput < Fluent::Plugin::Input
     end
   end
 
+  def process_sqs
+    begin
+      if @running
+        number_of_messages = @sqs_client.get_queue_attributes(
+          queue_url: @queue_url,
+          attribute_names: ["ApproximateNumberOfMessages"]
+        ).attributes["ApproximateNumberOfMessages"].to_i
+
+	count = 0
+	while count < number_of_messages
+	    messages = @sqs_client.receive_message(
+    	      queue_url: @queue_url,
+              max_number_of_messages: 10,
+              wait_time_seconds: 10
+    	    ).messages
+    	    log.debug "SQS queue has ApproximateNumberOfMessages=#{number_of_messages}" if number_of_messages > 0
+
+	    break if messages.empty?
+
+    	    messages.each do |message|
+    	      count += 1
+    	      s3_event = JSON.parse(message.body)
+              unless s3_event.key?('Records')
+        	@sqs_client.delete_message(
+                 queue_url: @queue_url,
+                 receipt_handle: message.receipt_handle
+	        )
+	        next
+	      end
+              #log.debug "S3 event: #{s3_event}"
+              bucket = s3_event['Records'][0]['s3']['bucket']['name']
+              key = s3_event['Records'][0]['s3']['object']['key']
+              event_time = Time.parse(s3_event['Records'][0]['eventTime']).to_i
+              log.debug "S3 event received for #{key} at #{s3_event['Records'][0]['eventTime']}"
+
+              object_key = get_object_key(key, event_time, 0)
+              if object_key.nil?
+        	@sqs_client.delete_message(
+                 queue_url: @queue_url,
+                 receipt_handle: message.receipt_handle
+	        )
+	        next
+	      end
+
+              record_common = {
+                "account_id" => object_key[:account_id],
+                "region" => object_key[:region],
+                "logfile_date" => object_key[:logfile_date],
+                "logfile_elb_name" => object_key[:logfile_elb_name],
+                "elb_ip_address" => object_key[:elb_ip_address],
+                "logfile_hash" => object_key[:logfile_hash],
+                "logfile_timestamp" => object_key[:logfile_timestamp],
+                "key" => object_key[:key],
+                "prefix" => object_key[:prefix],
+                "logfile_timestamp_unixtime" => object_key[:logfile_timestamp_unixtime],
+                "s3_last_modified_unixtime" => object_key[:s3_last_modified_unixtime],
+              }
+
+              get_file_from_s3(object_key[:key])
+              emit_lines_from_buffer_file(record_common)
+
+              put_timestamp_file(object_key[:s3_last_modified_unixtime])
+
+              if @delete
+                delete_file_from_s3(object_key[:key])
+              end
+
+              @sqs_client.delete_message(
+               queue_url: @queue_url,
+               receipt_handle: message.receipt_handle
+              )
+    	    end
+    	    if count == 0
+    		log.debug "No messages out of #{number_of_messages} were processed - should never happen"
+    		break
+    	    end
+        end
+      end
+      rescue => e
+        log.warn "error occurred: #{e.message}"
+    end
+  end
+
   def sort_object_key(src_object_keys)
     begin
       src_object_keys.sort do |a, b|
@@ -175,52 +431,66 @@ class Fluent::Plugin::Elb_LogInput < Fluent::Plugin::Input
   end
 
   def get_object_list(max_num)
-    s3_client.list_objects(
+    @s3_client.list_objects(
       bucket: @s3_bucketname,
       max_keys: max_num,
       prefix: @s3_prefix
     )
   end
 
+  def get_object_key(key, last_modified, timestamp)
+    node_no = Digest::SHA1.hexdigest(key).to_i(16) % @num_nodes
+    return nil unless node_no == @node_no
+
+    matches = LOGFILE_REGEXP.match(key)
+
+    s3_last_modified_unixtime = last_modified
+    if s3_last_modified_unixtime > timestamp and matches
+      exclude_pattern_matches = @exclude_pattern_logfile_elb_name_re.match(matches[:logfile_elb_name])
+      if exclude_pattern_matches
+        log.debug "Skipping object #{key} b/c it matches exclude_pattern_logfile_elb_name"
+        return nil
+      end
+
+      object_key = {
+        key: key,
+        prefix: matches[:prefix],
+        account_id: matches[:account_id],
+        region: matches[:region],
+        logfile_date: matches[:logfile_date],
+        logfile_elb_name: matches[:logfile_elb_name],
+        logfile_timestamp: matches[:logfile_timestamp],
+        elb_ip_address: matches[:elb_ip_address],
+        logfile_hash: matches[:logfile_hash],
+        logfile_timestamp_unixtime: Time.parse(matches[:logfile_timestamp]).to_i,
+        s3_last_modified_unixtime: s3_last_modified_unixtime,
+      }
+    end
+
+    return object_key
+  end
+
   def get_object_keys(timestamp)
     object_keys = []
 
-    resp = s3_client.list_objects_v2(
+    resp = @s3_client.list_objects_v2(
       bucket: @s3_bucketname,
       prefix: @s3_prefix
     )
 
-    loop do
+    while @running do
       resp.contents.each do |content|
-        s3_last_modified_unixtime = content.last_modified.to_i
-
-        object_key = content.key
-        node_no = Digest::SHA1.hexdigest(object_key).to_i(16) % @num_nodes
-        next unless node_no == @node_no
-
-        matches = LOGFILE_REGEXP.match(object_key)
-        if s3_last_modified_unixtime > timestamp and matches
-          object_keys << {
-            key: object_key,
-            prefix: matches[:prefix],
-            account_id: matches[:account_id],
-            region: matches[:region],
-            logfile_date: matches[:logfile_date],
-            logfile_elb_name: matches[:logfile_elb_name],
-            elb_timestamp: matches[:elb_timestamp],
-            elb_ip_address: matches[:elb_ip_address],
-            logfile_hash: matches[:logfile_hash],
-            elb_timestamp_unixtime: Time.parse(matches[:elb_timestamp]).to_i,
-            s3_last_modified_unixtime: s3_last_modified_unixtime,
-          }
-        end
+        log.debug "Getting #{content.key}"
+        object_key = get_object_key(content.key, content.last_modified.to_i, timestamp)
+        next if object_key.nil?
+        object_keys << object_key
       end
 
       if !resp.is_truncated
         return object_keys
       end
 
-      resp = s3_client.list_objects_v2(
+      resp = @s3_client.list_objects_v2(
         bucket: @s3_bucketname,
         prefix: @s3_prefix,
         continuation_token: resp.next_continuation_token
@@ -248,10 +518,10 @@ class Fluent::Plugin::Elb_LogInput < Fluent::Plugin::Input
 
   def get_file_from_s3(object_name)
     begin
-      log.debug "getting object from s3 name is #{object_name}"
+      log.debug "retrieving #{object_name}"
 
       Tempfile.create('fluent-elblog') do |tfile|
-        s3_client.get_object(bucket: @s3_bucketname, key: object_name, response_target: tfile.path)
+        @s3_client.get_object(bucket: @s3_bucketname, key: object_name, response_target: tfile.path)
 
         if File.extname(object_name) != '.gz'
           FileUtils.cp(tfile.path, @buf_file)
@@ -268,7 +538,7 @@ class Fluent::Plugin::Elb_LogInput < Fluent::Plugin::Input
     begin
       log.debug "deleting object from s3 name is #{object_name}"
 
-      s3_client.delete_object(bucket: @s3_bucketname, key: object_name)
+      @s3_client.delete_object(bucket: @s3_bucketname, key: object_name)
     rescue => e
       log.warn "error occurred: #{e.message}, #{e.backtrace}"
     end
@@ -308,13 +578,13 @@ class Fluent::Plugin::Elb_LogInput < Fluent::Plugin::Input
       "elb" => item[:elb],
       "client" => item[:client],
       "client_port" => item[:client_port],
-      "backend" => item[:backend],
-      "backend_port" => item[:backend_port],
+      "target" => item[:target],
+      "target_port" => item[:target_port],
       "request_processing_time" => item[:request_processing_time].to_f,
-      "backend_processing_time" => item[:backend_processing_time].to_f,
+      "target_processing_time" => item[:target_processing_time].to_f,
       "response_processing_time" => item[:response_processing_time].to_f,
       "elb_status_code" => item[:elb_status_code],
-      "backend_status_code" => item[:backend_status_code],
+      "target_status_code" => item[:target_status_code],
       "received_bytes" => item[:received_bytes].to_i,
       "sent_bytes" => item[:sent_bytes].to_i,
       "request_method" => item[:request_method],
@@ -333,9 +603,11 @@ class Fluent::Plugin::Elb_LogInput < Fluent::Plugin::Input
       "actions_executed" => item[:actions_executed],
       "redirect_url" => item[:redirect_url],
       "error_reason" => item[:error_reason],
-      "option1" => item[:option1],
-      "option2" => item[:option2],
-      "option3" => item[:option3]
+      "target_port_list" => item[:target_port_list],
+      "target_status_code_list" => item[:target_status_code_list],
+      "classification" => item[:classification],
+      "classification_reason" => item[:classification_reason],
+      "conn_trace_id" => item[:conn_trace_id]
     }
   end
 end

data/test/plugin/in_elb_log.rb

@@ -164,7 +164,7 @@ class Elb_LogInputTest < Test::Unit::TestCase
     assert_equal('ap-northeast-1', m[:region])
     assert_equal('2017/05/03', m[:logfile_date])
     assert_equal('app.elbname.59bfa19e900030c2', m[:logfile_elb_name])
-    assert_equal('20170503T1310Z', m[:elb_timestamp])
+    assert_equal('20170503T1310Z', m[:logfile_timestamp])
     assert_equal('10.0.0.1', m[:elb_ip_address])
     assert_equal('2tko12gv', m[:logfile_hash])
   end
@@ -177,13 +177,13 @@ class Elb_LogInputTest < Test::Unit::TestCase
     assert_equal('elbname', m[:elb])
     assert_equal('10.11.12.13', m[:client])
     assert_equal('37852', m[:client_port])
-    assert_equal('192.168.30.186', m[:backend])
-    assert_equal('443', m[:backend_port])
+    assert_equal('192.168.30.186', m[:target])
+    assert_equal('443', m[:target_port])
     assert_equal('0.00004', m[:request_processing_time])
-    assert_equal('0.085372', m[:backend_processing_time])
+    assert_equal('0.085372', m[:target_processing_time])
     assert_equal('0.000039', m[:response_processing_time])
     assert_equal('301', m[:elb_status_code])
-    assert_equal('301', m[:backend_status_code])
+    assert_equal('301', m[:target_status_code])
     assert_equal('0', m[:received_bytes])
     assert_equal('0', m[:sent_bytes])
     assert_equal('GET', m[:request_method])
@@ -195,7 +195,7 @@ class Elb_LogInputTest < Test::Unit::TestCase
     assert_equal(nil, m[:type])
     assert_equal(nil, m[:target_group_arn])
     assert_equal(nil, m[:trace_id])
-    assert_equal(nil, m[:option3])
+    assert_equal(nil, m[:conn_trace_id])
   end
 
   def test_log_application_lb_parse
@@ -206,13 +206,13 @@ class Elb_LogInputTest < Test::Unit::TestCase
     assert_equal('app/elbname/59bfa19e900030c2', m[:elb])
     assert_equal('10.20.30.40', m[:client])
     assert_equal('52730', m[:client_port])
-    assert_equal('192.168.30.186', m[:backend])
-    assert_equal('443', m[:backend_port])
+    assert_equal('192.168.30.186', m[:target])
+    assert_equal('443', m[:target_port])
     assert_equal('0.006', m[:request_processing_time])
-    assert_equal('0.000', m[:backend_processing_time])
+    assert_equal('0.000', m[:target_processing_time])
     assert_equal('0.086', m[:response_processing_time])
     assert_equal('301', m[:elb_status_code])
-    assert_equal('301', m[:backend_status_code])
+    assert_equal('301', m[:target_status_code])
     assert_equal('117', m[:received_bytes])
     assert_equal('507', m[:sent_bytes])
     assert_equal('GET', m[:request_method])
@@ -224,7 +224,7 @@ class Elb_LogInputTest < Test::Unit::TestCase
     assert_equal('https', m[:type])
     assert_equal('arn:aws:elasticloadbalancing:ap-northeast-1:123456789012:targetgroup/lbgrp1/605122a4e4ee9f2d', m[:target_group_arn])
     assert_equal('"Root=1-590c7929-4eb4cb393d46a01d22db8473"', m[:trace_id])
-    assert_equal(nil, m[:option3])
+    assert_equal(nil, m[:conn_trace_id])
   end
 
   def test_grp_and_trace_fileld
@@ -235,13 +235,13 @@ class Elb_LogInputTest < Test::Unit::TestCase
     assert_equal('app/elbname/aabbccdd9988', m[:elb])
     assert_equal('10.248.9.92', m[:client])
     assert_equal('54000', m[:client_port])
-    assert_equal('10.248.9.77', m[:backend])
-    assert_equal('80', m[:backend_port])
+    assert_equal('10.248.9.77', m[:target])
+    assert_equal('80', m[:target_port])
     assert_equal('0.001', m[:request_processing_time])
-    assert_equal('0.002', m[:backend_processing_time])
+    assert_equal('0.002', m[:target_processing_time])
     assert_equal('0.003', m[:response_processing_time])
     assert_equal('200', m[:elb_status_code])
-    assert_equal('200', m[:backend_status_code])
+    assert_equal('200', m[:target_status_code])
     assert_equal('686', m[:received_bytes])
     assert_equal('6476', m[:sent_bytes])
     assert_equal('GET', m[:request_method])
@@ -252,7 +252,9 @@ class Elb_LogInputTest < Test::Unit::TestCase
     assert_equal('-', m[:ssl_protocol])
     assert_equal('arn:aws:elasticloadbalancing:us-east-1:123456789123:targetgroup/example-service/1234abcd1234abcd', m[:target_group_arn])
     assert_equal('"Root=1-xxxxxxxx-yyyyyyyyyyyyyyyyyyyzzzzz"', m[:trace_id])
-    assert_equal('"-" "-" 3', m[:option3])
+    assert_equal('domain_name', m[:domain_name])
+    assert_equal('chosen_cert_arn', m[:chosen_cert_arn])
+    assert_equal('matched_rule_priority', m[:matched_rule_priority])
   end
 
   def test_alb_all_field
@@ -264,14 +266,14 @@ class Elb_LogInputTest < Test::Unit::TestCase
     assert_equal('app/my-alb/520e61ffffffffff', m[:elb])
     assert_equal('60.11.22.33', m[:client])
     assert_equal('51306', m[:client_port])
-    assert_equal('192.168.30.111', m[:backend])
-    assert_equal('443', m[:backend_port])
+    assert_equal('192.168.30.111', m[:target])
+    assert_equal('443', m[:target_port])
 
     assert_equal('0.010', m[:request_processing_time])
-    assert_equal('0.097', m[:backend_processing_time])
+    assert_equal('0.097', m[:target_processing_time])
     assert_equal('0.001', m[:response_processing_time])
     assert_equal('301', m[:elb_status_code])
-    assert_equal('301', m[:backend_status_code])
+    assert_equal('301', m[:target_status_code])
     assert_equal('414', m[:received_bytes])
     assert_equal('507', m[:sent_bytes])
     assert_equal('GET', m[:request_method])
@@ -289,8 +291,8 @@ class Elb_LogInputTest < Test::Unit::TestCase
     assert_equal('forward', m[:actions_executed])
     assert_equal('redirect://url-something.com/', m[:redirect_url])
     assert_equal('error_reason', m[:error_reason])
-    assert_equal('"192.168.30.186:443"', m[:option1])
-    assert_equal('"301"', m[:option2])
-    assert_equal(nil, m[:option3])
+    assert_equal('"192.168.30.186:443"', m[:target_port_list])
+    assert_equal('"301"', m[:target_status_code_list])
+    assert_equal(nil, m[:classification])
   end
 end

metadata

@@ -1,14 +1,15 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-elb-log
 version: !ruby/object:Gem::Version
-  version: 1.3.2
+  version: 1.4.0
 platform: ruby
 authors:
 - shinsaka
+- jazzl0ver
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-11-10 00:00:00.000000000 Z
+date: 2025-03-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -146,7 +147,7 @@ files:
 - lib/fluent/plugin/in_elb_log.rb
 - test/helper.rb
 - test/plugin/in_elb_log.rb
-homepage: https://github.com/shinsaka/fluent-plugin-elb-log
+homepage: https://github.com/jazzl0ver/fluent-plugin-elb-log
 licenses:
 - MIT
 metadata: {}
@@ -165,7 +166,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
+rubygems_version: 3.3.5
 signing_key: 
 specification_version: 4
 summary: Amazon ELB log input plugin