fluent-plugin-vmware-loginsight 0.1.10 → 1.3.0

data/examples/k8s-log-collector-ds.yaml

@@ -23,30 +23,32 @@ data:
   myapp-fluent.conf: |
     # Input sources
     @include general.conf
-    @include systemd-input.conf
-    @include kubernetes-input.conf
-
-    # Parsing/Filtering
-    @include kubernetes-filter.conf
+    @include systemd.conf
+    @include kubernetes.conf
+    @include kube-audit.conf
 
     # Forwading - Be vigilant of the order in which these plugins are specified. Order matters!
-    @include myapp-loginsight-output.conf
+    @include vmw-li.conf
 
   general.conf: |
     <system>
       log_level info
     </system>
     # Prevent fluentd from handling records containing its own logs to handle cycles.
-    <match fluent.**>
-      @type null
-    </match>
+    <label @FLUENT_LOG>
+      <match fluent.**>
+        @type null
+      </match>
+    </label>
 
-  systemd-input.conf: |
+  systemd.conf: |
+    # Journal logs
     <source>
       @type systemd
+      @id in_systemd_logs
       path /run/log/journal
       # Can filter logs if we want, e.g.
-      # filters [{ "_SYSTEMD_UNIT": "kubelet.service" }]
+      #filters [{ "_SYSTEMD_UNIT": "kubelet.service" }]
       <storage>
         @type local
         persistent true
@@ -57,70 +59,114 @@ data:
       strip_underscores true
     </source>
 
-  kubernetes-input.conf: |
+  kubernetes.conf: |
+    # Container logs
+    # Kubernetes docker logs are stored under /var/lib/docker/containers for
+    # which kubernetes creates a symlink at /var/log/containers
     <source>
       @type tail
+      @id in_tail_container_logs
       path /var/log/containers/*.log
       # One could exclude certain logs like:
-      # exclude_path ["/var/log/containers/log-collector*.log"]
+      #exclude_path ["/var/log/containers/log-collector*.log"]
       pos_file /var/log/fluentd-docker.pos
-      time_format %Y-%m-%dT%H:%M:%S
-      tag kubernetes.*
-      format json
       read_from_head true
+      # Set this watcher to false if you have many files to tail
+      enable_stat_watcher false
+      refresh_interval 5
+      tag kubernetes.*
+      <parse>
+        @type json
+        time_key time
+        keep_time_key true
+        time_format %Y-%m-%dT%H:%M:%S.%NZ
+      </parse>
     </source>
-
-  kubernetes-filter.conf: |
+    # Kubernetes metadata filter that tags additional meta data for each container event
     <filter kubernetes.**>
       @type kubernetes_metadata
-      merge_json_log true
-      preserve_json_log true
+      @id filter_kube_metadata
+      kubernetes_url "#{ENV['FLUENT_FILTER_KUBERNETES_URL'] || 'https://' + ENV.fetch('KUBERNETES_SERVICE_HOST') + ':' + ENV.                  fetch('KUBERNETES_SERVICE_PORT') + '/api'}"
+      verify_ssl "#{ENV['KUBERNETES_VERIFY_SSL'] || true}"
+      ca_file "#{ENV['KUBERNETES_CA_FILE']}"
+      skip_labels "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_LABELS'] || 'false'}"
+      skip_container_metadata "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_CONTAINER_METADATA'] || 'false'}"
+      skip_master_url "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_MASTER_URL'] || 'false'}"
+      skip_namespace_metadata "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_NAMESPACE_METADATA'] || 'false'}"
+    </filter>
+
+    # Prefix the tag by namespace. This would make it easy to match logs by namespaces
+    <match kubernetes.**>
+      @type rewrite_tag_filter
+      <rule>
+        key $.kubernetes.namespace_name
+        pattern ^(.+)$
+        tag $1.${tag}
+      </rule>
+    </match>
+
+  kube-audit.conf: |
+    # Kube-apiserver audit logs
+    <source>
+      @type tail
+      @id in_tail_kube_audit_logs
+      # path to audit logs for kube-apiserver
+      path "/var/log/kube-audit/audit.log"
+      pos_file /var/log/kube-audit.pos
+      tag kube-audit
+      <parse>
+        @type json
+        time_key timestamp
+        keep_time_key false
+        time_format %Y-%m-%dT%H:%M:%SZ
+      </parse>
+    </source>
+    # Loginsight doesn't support ingesting `source` as a field name, get rid of it
+    <filter kube-audit>
+      @type record_transformer
+      @id filter_kube_audit_logs
+      enable_ruby
+      remove_keys source
+      <record>
+        log ${record}
+      </record>
     </filter>
 
-  myapp-loginsight-output.conf: |
+  vmw-li.conf: |
+    # Match everything
     # We are capturing all log messages and redirecting them to endpoints mentioned in each <store> tag.
     # One may redirect these logs to muliple endpoints (including multiple LI instances).
     # Or one may chose to tag their specific logs and add their own config to capture those tagged logs and redirect
-    # them to appropriate endpoint. This specific config needs to preceed this generic one.
+    # them to appropriate endpoint. That specific config needs to preceed this generic one.
     <match **>
       @type copy
       <store>
         @type vmware_loginsight
+        @id out_vmw_li_all_container_logs
         scheme https
         ssl_verify true
         # Loginsight host: One may use IP address or cname
-        # host X.X.X.X
-        host my-loginsight.mycompany.com
-        port 9000
-        path api/v1/events/ingest
+        #host X.X.X.X
+        host MY_LOGINSIGHT_HOST
+        port 9543
         agent_id XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX
-        http_method post
-        serializer json
-        rate_limit_msec 0
-        raise_on_error false
-        include_tag_key true
-        tag_key tag
+        # Keys from log event whose values should be added as log message/text to
+        # Loginsight. Note these key/value pairs  won't be added as metadata/fields
+        log_text_keys ["log","msg","message"]
+        # Use this flag if you want to enable http debug logs
+        http_conn_debug false
       </store>
-      # If we want to debug and send logs to stdout as well
-      # <store>
-      #   @type stdout
-      # </store>
+      # copy plugin supports sending/copying logs to multiple plugins
+      # One may choose to send them to multiple LIs
+      # Or one may want send a copy to stdout for debugging
+      # Please note, if you use stdout along with LI, catch the logger's log to make
+      # sure they're not cyclic
+      #<store>
+      #  @type stdout
+      #</store>
     </match>
 
 
-  extra.conf: |
-    # If we want to transform events we could use:
-    #<filter **>
-    #  @type record_transformer
-    #  enable_ruby
-    #  auto_typecast
-    #  <record>
-    #    hostname "#{Socket.gethostname}"
-    #    mykey ${["message"=>record.to_json]}
-    #  </record>
-    #</filter>
-
-
 ---
 kind: DaemonSet
 apiVersion: extensions/v1beta1
@@ -131,8 +177,21 @@ metadata:
     app: "log-collector"
     version: v1
 spec:
+  selector:
+    matchLabels:
+      app: "log-collector"
+  revisionHistoryLimit: 3
+  minReadySeconds: 10
+  updateStrategy:
+    type: RollingUpdate
+    rollingUpdate:
+      # How many pods can be unavailable during the rolling update.
+      maxUnavailable: 3
   template:
     metadata:
+      annotations:
+        # One may use this annotation to trigger rollout whenever fluentd config changes
+        configHash: GENERATED_HASH
       labels:
         app: "log-collector"
         version: v1

data/fluent-plugin-vmware-loginsight.gemspec

@@ -14,7 +14,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name    = "fluent-plugin-vmware-loginsight"
-  spec.version = "0.1.10"
+  spec.version = File.read("VERSION").strip
   spec.authors = ["Vishal Mohite", "Chris Todd"]
   spec.email   = ["vmohite@vmware.com", "toddc@vmware.com"]
 
@@ -31,7 +31,7 @@ Gem::Specification.new do |spec|
   spec.test_files    = test_files
   spec.require_paths = ["lib"]
 
-  spec.add_development_dependency "bundler", "~> 1.14"
+  spec.add_development_dependency "bundler", ">= 2.1.0"
   spec.add_development_dependency "rake", "~> 12.0"
   spec.add_development_dependency "test-unit", "~> 3.0"
   spec.add_runtime_dependency "fluentd", [">= 0.14.10", "< 2"]