fluent-plugin-juniper-telemetry 0.2.8

data/lib/firewall.pb.rb

@@ -0,0 +1,83 @@
+# encoding: utf-8
+
+##
+# This file is auto-generated. DO NOT EDIT!
+#
+require 'protobuf/message'
+
+
+##
+# Imports
+#
+require 'jvision_top.pb'
+
+
+##
+# Message Classes
+#
+class Firewall < ::Protobuf::Message; end
+class FirewallStats < ::Protobuf::Message; end
+class MemoryUsage < ::Protobuf::Message; end
+class CounterStats < ::Protobuf::Message; end
+class PolicerStats < ::Protobuf::Message; end
+class ExtendedPolicerStats < ::Protobuf::Message; end
+class HierarchicalPolicerStats < ::Protobuf::Message; end
+
+
+##
+# Message Fields
+#
+class Firewall
+  repeated ::FirewallStats, :firewall_stats, 1
+end
+
+class FirewallStats
+  required :string, :filter_name, 1
+  optional :uint64, :timestamp, 2
+  repeated ::MemoryUsage, :memory_usage, 3
+  repeated ::CounterStats, :counter_stats, 4
+  repeated ::PolicerStats, :policer_stats, 5
+  repeated ::HierarchicalPolicerStats, :hierarchical_policer_stats, 6
+end
+
+class MemoryUsage
+  required :string, :name, 1
+  optional :uint64, :allocated, 2
+end
+
+class CounterStats
+  required :string, :name, 1
+  optional :uint64, :packets, 2
+  optional :uint64, :bytes, 3
+end
+
+class PolicerStats
+  required :string, :name, 1
+  optional :uint64, :out_of_spec_packets, 2
+  optional :uint64, :out_of_spec_bytes, 3
+  optional ::ExtendedPolicerStats, :extended_policer_stats, 4
+end
+
+class ExtendedPolicerStats
+  optional :uint64, :offered_packets, 1
+  optional :uint64, :offered_bytes, 2
+  optional :uint64, :transmitted_packets, 3
+  optional :uint64, :transmitted_bytes, 4
+end
+
+class HierarchicalPolicerStats
+  required :string, :name, 1
+  optional :uint64, :premium_packets, 2
+  optional :uint64, :premium_bytes, 3
+  optional :uint64, :aggregate_packets, 4
+  optional :uint64, :aggregate_bytes, 5
+end
+
+
+##
+# Extended Message Fields
+#
+class ::JuniperNetworksSensors < ::Protobuf::Message
+  optional ::Firewall, :jnpr_firewall_ext, 6, :extension => true
+end
+

data/lib/fluent/plugin/parser_juniper_analyticsd.rb

@@ -0,0 +1,125 @@
+module Fluent
+    class TextParser
+        class JuniperAnalyticsdParser < Parser
+
+            Plugin.register_parser("juniper_analyticsd", self)
+
+            config_param :output_format, :string, :default => 'structured'
+
+            # This method is called after config_params have read configuration parameters
+            def configure(conf)
+                super
+
+                ## Check if "output_format" has a valid value
+                unless  @output_format.to_s == "structured" ||
+                        @output_format.to_s == "flat" ||
+                        @output_format.to_s == "statsd"
+
+                    raise ConfigError, "output_format value '#{@output_format}' is not valid. Must be : structured, flat or statsd"
+                end
+            end
+
+            def parse(text)
+
+                payload = JSON.parse(text)
+                time = Engine.now
+
+                ## Extract contextual info
+                record_type = payload["record-type"]
+                record_time = payload["time"]
+                device_name = payload["router-id"]
+                port_name = payload["port"]
+
+                ## Record time is in microsecond and until 0.14 Fluentd do not support lower than 1s
+                ## We need to trim record time for now to fit fluentd
+                json_time = (record_time/1000000).to_i
+
+                if record_type == 'traffic-stats'
+
+                    ## Delete contextual info
+                    payload.delete("record-type")
+                    payload.delete("time")
+                    payload.delete("router-id")
+                    payload.delete("port")
+
+                    payload.each do |key, value|
+                        record = {}
+
+                        if output_format.to_s == 'flat'
+                            full_name = "device.#{clean_up_name(device_name)}.interface.#{clean_up_name(port_name)}.type.#{key}"
+
+                            record[full_name.downcase]= value
+
+                        elsif output_format.to_s == 'structured'
+                            record['device'] = device_name
+                            record['type'] = record_type + '.' + key
+                            record['interface'] = port_name
+                            record['value'] = value
+
+                        elsif output_format.to_s == 'statsd'
+
+                            full_name = "interface.#{clean_up_name(port_name)}.type.#{key}"
+                            record[:statsd_type] = 'gauge'
+                            record[:statsd_key] = full_name.downcase
+                            record[:statsd_gauge] = value
+                        else
+                            $log.warn "Output_format '#{output_format.to_s}' not supported for #{record_type}"
+                        end
+
+                        yield json_time, record
+                    end
+                elsif record_type == 'queue-stats'
+
+                    ## Delete contextual info
+                    payload.delete("record-type")
+                    payload.delete("time")
+                    payload.delete("router-id")
+                    payload.delete("port")
+
+                    payload.each do |key, value|
+                        record = {}
+
+                        if output_format.to_s == 'flat'
+
+                            full_name = "device.#{clean_up_name(device_name)}.interface.#{clean_up_name(port_name)}.queue.#{key}"
+
+                            record[full_name.downcase]= value
+
+                        elsif output_format.to_s == 'structured'
+                            record['device'] = device_name
+                            record['type'] = record_type + '.' + key
+                            record['interface'] = port_name
+                            record['value'] = value
+
+                        elsif output_format.to_s == 'statsd'
+                            full_name = "interface.#{clean_up_name(port_name)}.queue.#{key}"
+                            record[:statsd_type] = 'gauge'
+                            record[:statsd_key] = full_name.downcase
+                            record[:statsd_gauge] = value
+
+                        else
+                            $log.warn "Output_format '#{output_format.to_s}' not supported for #{record_type}"
+                        end
+
+                        yield json_time, record
+                    end
+                else
+                    $log.warn "Recard type '#{record_type}' not supported"
+                end
+            end
+
+            ## Clean up device name and interface name to remove restricted caracter
+            ## Used for flat and statsd format
+            def clean_up_name(name)
+
+                tmp = name
+
+                tmp.gsub!('/', '_')
+                tmp.gsub!(':', '_')
+                tmp.gsub!('.', '_')
+
+                return tmp.to_s
+            end
+        end
+    end
+end

data/lib/fluent/plugin/parser_juniper_jti.rb

@@ -0,0 +1,282 @@
+require 'protobuf'
+require 'jvision_top.pb.rb'
+require 'port.pb.rb'
+require 'logical_port.pb.rb'
+require 'firewall.pb.rb'
+
+module Fluent
+    class TextParser
+        class JuniperJtiParser < Parser
+
+            Plugin.register_parser("juniper_jti", self)
+
+            config_param :output_format, :string, :default => 'structured'
+
+            # This method is called after config_params have read configuration parameters
+            def configure(conf)
+                super
+
+                ## Check if "output_format" has a valid value
+                unless  @output_format.to_s == "structured" ||
+                        @output_format.to_s == "flat" ||
+                        @output_format.to_s == "statsd"
+
+                    raise ConfigError, "output_format value '#{@output_format}' is not valid. Must be : structured, flat or statsd"
+                end
+            end
+
+            def parse(text)
+
+                ## Decode GBP packet
+                jti_msg =  TelemetryStream.decode(text)
+
+                resource = ""
+
+                ## Extract device name & Timestamp
+                device_name = jti_msg.system_id
+                gpb_time = jti_msg.timestamp
+
+                ## Extract sensor
+                begin
+                  jnpr_sensor = jti_msg.enterprise.juniperNetworks
+                  datas_sensors = JSON.parse(jnpr_sensor.to_json)
+                  $log.debug  "Extract sensor data from " + device_name
+                rescue
+                  $log.warn   "Unable to extract sensor data sensor from jti_msg.enterprise.juniperNetworks, something went wrong"
+                  $log.debug  "Unable to extract sensor data sensor from jti_msg.enterprise.juniperNetworks, Data Dump : " + jti_msg.inspect.to_s
+                  return
+                end
+
+                ## Go over each Sensor
+                datas_sensors.each do |sensor, s_data|
+
+                    ##############################################################
+                    ### Support for resource /junos/system/linecard/interface/  ##
+                    ##############################################################
+                    if sensor == "jnpr_interface_ext"
+
+                      resource = "/junos/system/linecard/interface/"
+
+                      datas_sensors[sensor]['interface_stats'].each do |datas|
+
+                        # Save all info extracted on a list
+                        sensor_data = []
+
+                        # Catch Exception during parsing
+                        begin
+                          ## Extract interface name and clean up
+                          # interface_name = datas['if_name']
+                          sensor_data.push({ 'device' => device_name  })
+                          sensor_data.push({ 'interface' => datas['if_name']  })
+
+                          # Check if the interface has a parent
+                          if datas.key?('parent_ae_name')
+                            sensor_data.push({ 'interface_parent' =>  datas['parent_ae_name']  })
+                            datas.delete("parent_ae_name")
+                          end
+
+                          ## Clean up Current object
+                          datas.delete("if_name")
+                          datas.delete("init_time")
+                          datas.delete("snmp_if_index")
+
+                          datas.each do |section, data|
+
+                            ## egress_queue_info is an Array
+                            if data.kind_of?(Array)
+                              data.each do |queue|
+
+                                ## Create local copy to avoid variable sharing
+                                local_sensor_data = sensor_data.dup
+
+                                ## Save and Cleanup Queue number
+                                local_sensor_data.push({ 'egress_queue' => queue['queue_number']  })
+                                queue.delete("queue_number")
+
+                                queue.each do |type,value|
+                                  local_sensor_data.push({ 'type' => section + '.' + type  })
+                                  local_sensor_data.push({ 'value' => value  })
+
+                                  record = build_record(output_format, local_sensor_data)
+                                  yield gpb_time, record
+                                end
+                              end
+                            else
+                              data.each do |type,value|
+
+                                ## Create local copy to avoid using some variable
+                                local_sensor_data = sensor_data.dup
+
+                                local_sensor_data.push({ 'type' => section + '.' + type  })
+                                local_sensor_data.push({ 'value' => value  })
+
+                                record = build_record(output_format, local_sensor_data)
+                                yield gpb_time, record
+                              end
+                            end
+                          end
+                        rescue
+                          $log.warn   "Unable to parse " + sensor + " sensor, an error occured.."
+                          $log.debug  "Unable to parse " + sensor + " sensor, Data Dump : " + datas.inspect.to_s
+                        end
+                      end
+                    ##############################################################
+                    ### Support for resource /junos/system/linecard/firewall/  ##
+                    ##############################################################
+                    # elsif sensor == "jnpr_firewall_ext"
+                    #
+                    #   resource = "/junos/system/linecard/firewall"
+                    #
+                    #   datas_sensors[sensor]['firewall_stats'].each do |datas|
+                    #
+                    #     begin
+                    #       ## Extract interface name and clean up
+                    #       sensor_data.push({ 'device' => device_name  })
+                    #       sensor_data.push({ 'filter_name' => datas['filter_name']  })
+                    #
+                    #       ## Clean up Current object
+                    #       # datas.delete("filter_name")
+                    #
+                    #       $log.warn  "Sensor Filter : " + datas['filter_name']
+                    #
+                    #     rescue
+                    #       $log.warn   "Unable to parse " + sensor + " sensor, an error occured.."
+                    #       $log.debug  "Unable to parse " + sensor + " sensor, Data Dump : " + datas.inspect.to_s
+                    #     end
+                    #   end
+
+                    ##############################################################
+                    ### Support for resource /junos/system/linecard/interface/logical/usage ##
+                    ##############################################################
+                    elsif sensor == "jnprLogicalInterfaceExt"
+
+                      resource = "/junos/system/linecard/interface/logical/usage"
+
+                      datas_sensors[sensor]['interface_info'].each do |datas|
+
+                        # Save all info extracted on a list
+                        sensor_data = []
+
+                        begin
+                          ## Extract interface name and clean up
+                          sensor_data.push({ 'device' => device_name  })
+                          sensor_data.push({ 'interface' => datas['if_name']  })
+
+                          ## Clean up Current object
+                          datas.delete("if_name")
+                          datas.delete("init_time")
+                          datas.delete("snmp_if_index")
+                          datas.delete("op_state")
+
+                          datas.each do |section, data|
+                            data.each do |type, value|
+
+                              sensor_data.push({ 'type' => section + '.' + type  })
+                              sensor_data.push({ 'value' => value  })
+
+                              record = build_record(output_format, sensor_data)
+                              yield gpb_time, record
+
+                            end
+                          end
+                        rescue
+                          $log.warn   "Unable to parse " + sensor + " sensor, an error occured.."
+                          $log.debug  "Unable to parse " + sensor + " sensor, Data Dump : " + datas.inspect.to_s
+                        end
+                      end
+                    else
+                        $log.warn  "Unsupported sensor : " + sensor
+                        # puts datas_sensors[sensor].inspect.to_s
+                    end
+                end
+            end
+
+            def clean_up_name(name)
+
+                tmp = name
+                ## Clean up device name and interface name to remove restricted caracter
+                tmp.gsub!('/', '_')
+                tmp.gsub!(':', '_')
+                tmp.gsub!('.', '_')
+
+                return tmp
+            end
+
+            def build_record(type, data_to_build)
+
+              if type.to_s == 'flat'
+
+                # initialize variables
+                name = ""
+                sensor_value = ""
+
+                ## Concatene all key/value into a string and stop at "value"
+                data_to_build.each do |entry|
+                  entry.each do |key, value|
+
+                    if key == "value"
+                      sensor_value = value
+                      next
+                    end
+
+                    if name == ""
+                      name = key + "." + clean_up_name(value).to_s
+                    else
+                      name = name + "." + key + "." + clean_up_name(value).to_s
+                    end
+                  end
+                end
+
+                record = { name => sensor_value }
+                return record
+              elsif output_format.to_s == 'structured'
+                record = {}
+                ## Convert list into Hash
+                ## Each entry on the list is a hash with 1 key/value
+                data_to_build.each do |entry|
+                  entry.each do |key, value|
+                    record[key] = value
+                  end
+                end
+
+                return record
+
+              elsif output_format.to_s == 'statsd'
+
+                  record = {}
+
+                  # initialize variables
+                  name = ""
+                  sensor_value = ""
+
+                  ## Concatene all key/value into a string, exclude device & stop at "value"
+                  data_to_build.each do |entry|
+                    entry.each do |key, value|
+
+                      if key == "value"
+                        sensor_value = value
+                        next
+                      elsif key == "device"
+                        next
+                      else
+                        if name == ""
+                          name = key + "." + clean_up_name(value).to_s
+                        else
+                          name = name + "." + key + "." + clean_up_name(value).to_s
+                        end
+                      end
+                    end
+                  end
+
+                  record[:statsd_type] = 'gauge'
+                  record[:statsd_key] = name.downcase
+                  record[:statsd_gauge] = sensor_value
+
+                  return record
+              else
+                $log.warn "Output_format '#{type.to_s}' not supported"
+              end
+            end
+        end
+    end
+end