fluent-plugin-containiq 0.0.7

data/k8s/conf/kubernetes/startupscript.conf

@@ -0,0 +1,10 @@
+<source>
+  @type tail
+  @id in_tail_startupscript
+  path /var/log/startupscript.log
+  pos_file /var/log/fluentd-startupscript.log.pos
+  tag startupscript
+  <parse>
+    @type syslog
+  </parse>
+</source>

data/k8s/conf/kubernetes.conf

@@ -0,0 +1,36 @@
+<label @FLUENT_LOG>
+  <match fluent.**>
+    @type null
+    @id ignore_fluent_logs
+  </match>
+</label>
+
+@include kubernetes/cluster-autoscaler.conf
+@include kubernetes/containers.conf
+@include kubernetes/docker.conf
+@include kubernetes/etcd.conf
+@include kubernetes/glbc.conf
+@include kubernetes/kube-apiserver-audit.conf
+@include kubernetes/kube-apiserver.conf
+@include kubernetes/kube-controller-manager.conf
+@include kubernetes/kube-proxy.conf
+@include kubernetes/kube-scheduler.conf
+@include kubernetes/kubelet.conf
+@include kubernetes/rescheduler.conf
+@include kubernetes/salt.conf
+@include kubernetes/startupscript.conf
+
+
+<filter kubernetes.**>
+  @type kubernetes_metadata
+  @id filter_kube_metadata
+  kubernetes_url "#{ENV['FLUENT_FILTER_KUBERNETES_URL'] || 'https://' + ENV.fetch('KUBERNETES_SERVICE_HOST') + ':' + ENV.fetch('KUBERNETES_SERVICE_PORT') + '/api'}"
+  verify_ssl "#{ENV['KUBERNETES_VERIFY_SSL'] || true}"
+  ca_file "#{ENV['KUBERNETES_CA_FILE']}"
+  skip_labels "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_LABELS'] || 'false'}"
+  skip_container_metadata "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_CONTAINER_METADATA'] || 'false'}"
+  skip_master_url "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_MASTER_URL'] || 'false'}"
+  skip_namespace_metadata "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_NAMESPACE_METADATA'] || 'false'}"
+  watch "#{ENV['FLUENT_KUBERNETES_WATCH'] || 'true'}"
+</filter>
+

data/k8s/conf/prometheus.conf

@@ -0,0 +1,13 @@
+# Prometheus metric exposed on 0.0.0.0:24231/metrics
+<source>
+  @type prometheus
+  @id in_prometheus
+  bind "#{ENV['FLUENTD_PROMETHEUS_BIND'] || '0.0.0.0'}"
+  port "#{ENV['FLUENTD_PROMETHEUS_PORT'] || '24231'}"
+  metrics_path "#{ENV['FLUENTD_PROMETHEUS_PATH'] || '/metrics'}"
+</source>
+
+<source>
+  @type prometheus_output_monitor
+  @id in_prometheus_output_monitor
+</source>

data/k8s/conf/systemd.conf

@@ -0,0 +1,43 @@
+
+# Logs from systemd-journal for interesting services.
+<source>
+  @type systemd
+  @id in_systemd_kubelet
+  matches [{ "_SYSTEMD_UNIT": "kubelet.service" }]
+  <storage>
+    @type local
+    persistent true
+    path /var/log/fluentd-journald-kubelet-cursor.json
+  </storage>
+  read_from_head true
+  tag kubelet
+</source>
+
+# Logs from docker-systemd
+<source>
+  @type systemd
+  @id in_systemd_docker
+  matches [{ "_SYSTEMD_UNIT": "docker.service" }]
+  <storage>
+    @type local
+    persistent true
+    path /var/log/fluentd-journald-docker-cursor.json
+  </storage>
+  read_from_head true
+  tag docker.systemd
+</source>
+
+# Logs from systemd-journal for interesting services.
+<source>
+  @type systemd
+  @id in_systemd_bootkube
+  matches [{ "_SYSTEMD_UNIT": "bootkube.service" }]
+  <storage>
+    @type local
+    persistent true
+    path /var/log/fluentd-journald-bootkube-cursor.json
+  </storage>
+  read_from_head true
+  tag bootkube
+</source>
+

data/k8s/conf/tail_container_parse.conf

@@ -0,0 +1,4 @@
+<parse>
+  @type "#{ENV['FLUENT_CONTAINER_TAIL_PARSER_TYPE'] || 'json'}"
+  time_format "#{ENV['FLUENT_CONTAINER_TAIL_PARSER_TIME_FORMAT'] || '%Y-%m-%dT%H:%M:%S.%NZ'}"
+</parse>

data/k8s/entrypoint.sh

@@ -0,0 +1,4 @@
+#!/usr/bin/env sh
+
+# run with -vv to show debugging messages
+exec fluentd -vv -c /fluentd/etc/${FLUENTD_CONF} -p /fluentd/plugins --gemfile /fluentd/Gemfile ${FLUENTD_OPT}

data/k8s/fluentd-daemonset.yaml

@@ -0,0 +1,111 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: containiq
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: fluentd
+  namespace: containiq
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: fluentd
+  namespace: containiq
+rules:
+  - apiGroups:
+      - ''
+    resources:
+      - pods
+      - namespaces
+    verbs:
+      - get
+      - list
+      - watch
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: fluentd
+roleRef:
+  kind: ClusterRole
+  name: fluentd
+  apiGroup: rbac.authorization.k8s.io
+subjects:
+  - kind: ServiceAccount
+    name: fluentd
+    namespace: containiq
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: fluentd
+  namespace: containiq
+  labels:
+    name: fluentd
+spec:
+  selector:
+    matchLabels:
+      name: fluentd
+  template:
+    metadata:
+      labels:
+        name: fluentd
+    spec:
+      serviceAccount: fluentd
+      serviceAccountName: fluentd
+      tolerations:
+        - key: node-role.kubernetes.io/master
+          effect: NoSchedule
+      containers:
+        - name: fluentd
+          imagePullPolicy: Always
+          image: containiq/logging-agent
+          env:
+            # https://github.com/fabric8io/fluent-plugin-kubernetes_metadata_filter#environment-variables-for-kubernetes
+            - name: K8S_NODE_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: spec.nodeName
+            # TODO: should we set this here or another k8s resource?
+            - name: INGEST_LOGS_ENDPOINT_URL
+              value: https://localhost
+            - name: NOTIFICATION_FILE_LOCATION
+              value: /containiq/notification-config.yaml
+            - name: FLUENT_KUBERNETES_METADATA_SKIP_LABELS
+              value: 'true'
+            - name: FLUENT_KUBERNETES_METADATA_SKIP_CONTAINER_METADATA
+              value: 'true'
+            - name: FLUENT_KUBERNETES_METADATA_SKIP_NAMESPACE_METADATA
+              value: 'true'
+            - name: FLUENT_KUBERNETES_METADATA_SKIP_MASTER_URL
+              value: 'true'
+          resources:
+            limits:
+              memory: 200Mi
+            requests:
+              cpu: 100m
+              memory: 200Mi
+          volumeMounts:
+            - name: varlog
+              mountPath: /var/log
+            - name: varlibdockercontainers
+              mountPath: /var/lib/docker/containers
+              readOnly: true
+            - name: containiqconfig
+              mountPath: /containiq
+      volumes:
+        - name: varlog
+          hostPath:
+            path: /var/log
+        - name: varlibdockercontainers
+          hostPath:
+            path: /var/lib/docker/containers
+
+        - name: containiqconfig
+          projected:
+            sources:
+              - secret:
+                  name: containiq-notification

data/k8s/plugins/parser_kubernetes.rb

@@ -0,0 +1,68 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+# The following Fluentd parser plugin, aims to simplify the parsing of multiline
+# logs found in Kubernetes nodes. Since many log files shared the same format and
+# in order to simplify the configuration, this plugin provides a 'kubernetes' format
+# parser (built on top of MultilineParser).
+#
+# When tailing files, this 'kubernetes' format should be applied to the following
+# log file sources:
+#
+#  - /var/log/kubelet.log
+#  - /var/log/kube-proxy.log
+#  - /var/log/kube-apiserver.log
+#  - /var/log/kube-controller-manager.log
+#  - /var/log/kube-scheduler.log
+#  - /var/log/rescheduler.log
+#  - /var/log/glbc.log
+#  - /var/log/cluster-autoscaler.log
+#
+# Usage:
+#
+# ---- fluentd.conf ----
+#
+# <source>
+#   @type tail
+#   path ./kubelet.log
+#   read_from_head yes
+#   tag kubelet
+#   <parse>
+#     @type kubernetes
+#   </parse>
+# </source>
+#
+# ----   EOF       ---
+
+require 'fluent/plugin/parser_regexp'
+
+module Fluent
+  module Plugin
+    class KubernetesParser < RegexpParser
+      Fluent::Plugin.register_parser("kubernetes", self)
+
+      CONF_FORMAT_FIRSTLINE = %q{/^\w\d{4}/}
+      CONF_FORMAT1 = %q{/^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<message>.*)/m}
+      CONF_TIME_FORMAT = "%m%d %H:%M:%S.%N"
+
+      def configure(conf)
+        conf['expression'] = CONF_FORMAT1
+        conf['time_format'] = CONF_TIME_FORMAT
+        super
+      end
+    end
+  end
+end

data/k8s/plugins/parser_multiline_kubernetes.rb

@@ -0,0 +1,69 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+# The following Fluentd parser plugin, aims to simplify the parsing of multiline
+# logs found in Kubernetes nodes. Since many log files shared the same format and
+# in order to simplify the configuration, this plugin provides a 'kubernetes' format
+# parser (built on top of MultilineParser).
+#
+# When tailing files, this 'kubernetes' format should be applied to the following
+# log file sources:
+#
+#  - /var/log/kubelet.log
+#  - /var/log/kube-proxy.log
+#  - /var/log/kube-apiserver.log
+#  - /var/log/kube-controller-manager.log
+#  - /var/log/kube-scheduler.log
+#  - /var/log/rescheduler.log
+#  - /var/log/glbc.log
+#  - /var/log/cluster-autoscaler.log
+#
+# Usage:
+#
+# ---- fluentd.conf ----
+#
+# <source>
+#   @type tail
+#   path ./kubelet.log
+#   read_from_head yes
+#   tag kubelet
+#   <parse>
+#     @type multiline_kubernetes
+#   </parse>
+# </source>
+#
+# ----   EOF       ---
+
+require 'fluent/plugin/parser_multiline'
+
+module Fluent
+  module Plugin
+    class MultilineKubernetesParser < MultilineParser
+      Fluent::Plugin.register_parser("multiline_kubernetes", self)
+
+      CONF_FORMAT_FIRSTLINE = %q{/^\w\d{4}/}
+      CONF_FORMAT1 = %q{/^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<message>.*)/}
+      CONF_TIME_FORMAT = "%m%d %H:%M:%S.%N"
+
+      def configure(conf)
+        conf['format_firstline'] = CONF_FORMAT_FIRSTLINE
+        conf['format1'] = CONF_FORMAT1
+        conf['time_format'] = CONF_TIME_FORMAT
+        super
+      end
+    end
+  end
+end

data/lib/fluent/plugin/out_containiq.rb

@@ -0,0 +1,173 @@
+require "fluent/plugin/output"
+
+module Fluent::Plugin
+  class ContainiqOutput < Fluent::Plugin::Output
+    Fluent::Plugin.register_output("containiq", self)
+
+    config_param :bulk_limit, :integer, default: 1000000 # Logz.io has a 1MB limit and recommends leaving some overhead
+    config_param :bulk_limit_warning_limit, :integer, default: nil # If fluent warnings are sent to the output, truncating is necessary to prevent a recursion
+    config_param :http_idle_timeout, :integer, default: 5
+    config_param :gzip, :bool, default: false # False for backward compatibility
+
+    def start
+      super
+      require 'net/http/persistent'
+
+      endpoint_url = ENV["INGEST_LOGS_ENDPOINT_URL"]
+      raise 'missing environment variable: INGEST_LOGS_ENDPOINT_URL' if endpoint_url.nil?
+
+      @uri = URI endpoint_url
+      log.debug "ContainIQ URL #{endpoint_url}"
+
+      @http = Net::HTTP::Persistent.new name: 'fluent-plugin-containiq'
+
+      api_key = get_api_key()
+      @http.headers['Authorization'] = "Bearer #{api_key}"
+
+      @http.headers['Content-Type'] = 'text/plain'
+      if @gzip
+        @http.headers['Content-Encoding'] = 'gzip'
+      end
+      @http.idle_timeout = @http_idle_timeout
+      @http.socket_options << [Socket::SOL_SOCKET, Socket::SO_KEEPALIVE, 1]
+
+      log.debug "Started ContainIQ shipper.."
+    end
+
+    def shutdown
+      super
+    end
+
+    def formatted_to_msgpack_binary?
+      true
+    end
+
+    def multi_workers_ready?
+      true
+    end
+
+    def format(tag, time, record)
+      if time.is_a?(Fluent::EventTime)
+        sec_frac = time.to_f
+      else
+        sec_frac = time * 1.0
+      end
+      [tag, sec_frac, record].to_msgpack
+    end
+
+    def write(chunk)
+      encode_chunk(chunk) { |bulk_records, bulk_size|
+        send_bulk(bulk_records, bulk_size)
+      }
+    end
+
+    def encode_chunk(chunk)
+      records = []
+      bulk_size = 0
+      chunk.each { |tag, time, record|
+        record['timestamp'] ||= Time.at(time).iso8601(3)
+
+        begin
+          json_record = Yajl.dump(record)
+          record_size = json_record.size + (1 if !records.empty?).to_i # Accounting for trailing "\n"
+        rescue
+          log.error "Adding record #{record} to buffer failed. Exception: #{$!}"
+          next
+        end
+
+        if record_size > @bulk_limit
+          if @bulk_limit_warning_limit.is_a?(Integer)
+            log.warn "Record with size #{record_size} exceeds #{@bulk_limit} and can't be sent to ContainIQ. Record starts with (truncated at #{@bulk_limit_warning_limit} characters): #{json_record[0,@bulk_limit_warning_limit]}"
+            # Send the full message to debug facility
+            log.debug "Record with size #{record_size} exceeds #{@bulk_limit} and can't be sent to ContainIQ. Record is: #{json_record}"
+          else
+            log.warn "Record with size #{record_size} exceeds #{@bulk_limit} and can't be sent to ContainIQ. Record is: #{json_record}"
+          end
+          next
+        end
+        if bulk_size + record_size > @bulk_limit
+          yield(records, bulk_size)
+          records = []
+          bulk_size = 0
+        end
+        records.push(json_record)
+        bulk_size += record_size
+      }
+      if records
+        yield(records, bulk_size)
+      end
+    end
+
+    def send_bulk(bulk_records, bulk_size)
+      log.debug "Sending a bulk of #{bulk_records.size} records, size #{bulk_size}B to ContainIQ"
+
+      # Setting our request
+      post = Net::HTTP::Post.new @uri.request_uri
+
+      # TODO: not sure we need this; Logz.io included it with the following comment:
+      # Logz.io bulk http endpoint expecting log line with \n delimiter
+      post.body = bulk_records.join("\n")
+      if gzip
+        post.body = compress(post.body)
+      end
+
+      retry_count = 4     # How many times to resend failed bulks
+      sleep_interval = 2  # How long to sleep initially between retries
+
+      begin
+        retry_count.times do |counter|
+          should_retry = true
+          begin
+            response = @http.request @uri, post
+            if response.code != '200'
+              if response.code == '401'
+                log.error "You are not authorized with ContainIQ! Token OK? dropping logs..."
+                should_retry = false
+              elsif response.code == '400'
+                log.info "Got 400 code from ContainIQ. This means that some of your logs are too big, or badly formatted. Response: #{response.body}"
+                should_retry = false
+              else
+                log.warn "Got HTTP #{response.code} from ContainIQ, not giving up just yet (Try #{counter + 1}/#{retry_count})"
+              end
+            else
+              log.debug "Successfully sent bulk of #{bulk_records.size} records, size #{bulk_size}B to ContainIQ"
+              should_retry = false
+            end
+          rescue StandardError => e
+            log.warn "Error connecting to ContainIQ. Got exception: #{e} (Try #{counter + 1}/#{retry_count})"
+          end
+
+          if should_retry
+            if counter == retry_count - 1
+              log.error "Could not send your bulk after #{retry_count} tries Sorry! Your bulk is: #{post.body}"
+              break
+            end
+            sleep(sleep_interval)
+            sleep_interval *= 2
+          else
+            return
+          end
+        end
+      rescue Exception => e
+        log.error "Got unexpected exception! Here: #{e}"
+      end
+    end
+
+    def compress(string)
+      wio = StringIO.new("w")
+      w_gz = Zlib::GzipWriter.new(wio)
+      w_gz.write(string)
+      w_gz.close
+      wio.string
+    end
+
+    def get_api_key
+      file = ENV["NOTIFICATION_FILE_LOCATION"]
+      raise 'missing environment variable: NOTIFICATION_FILE_LOCATION' if file.nil?
+      fileFirstLine = File.open(file, &:readline)
+      scan = fileFirstLine.gsub("\n",'').scan(/key: (.+)/i)
+      raise 'unable to parse secret key' if scan.empty?
+      api_key = scan.first.first
+    end
+  end
+end

data/test/helper.rb

@@ -0,0 +1,8 @@
+$LOAD_PATH.unshift(File.expand_path("../../", __FILE__))
+require "test-unit"
+require "fluent/test"
+require "fluent/test/driver/output"
+require "fluent/test/helpers"
+
+Test::Unit::TestCase.include(Fluent::Test::Helpers)
+Test::Unit::TestCase.extend(Fluent::Test::Helpers)

data/test/plugin/test_out_containiq.rb

@@ -0,0 +1,18 @@
+require "helper"
+require "fluent/plugin/out_containiq.rb"
+
+class ContainiqOutputTest < Test::Unit::TestCase
+  setup do
+    Fluent::Test.setup
+  end
+
+  test "failure" do
+    flunk
+  end
+
+  private
+
+  def create_driver(conf)
+    Fluent::Test::Driver::Output.new(Fluent::Plugin::ContainiqOutput).configure(conf)
+  end
+end