fluent-plugin-telemetry-iosxe 0.0.2 → 0.0.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 62be67bb9c7c1ef02a5bf9e260b571ddfe3abfe0
-  data.tar.gz: 8548dd316fd7999eac791f7ab5cdd02bda013f13
+  metadata.gz: c069a9e9a4437641954f364633f3b247c8cadcd6
+  data.tar.gz: 4f30e86445e81c7c5d644522d1ba07763bdc3e11
 SHA512:
-  metadata.gz: 1e8d6d34402473d30097a88f190ec4e956eccf0cfe2c766cfd7124e6986d4dd65e588e063421113f6f5be1371628ce8e6b7e17e4492fc135d9ac32845c4cd722
-  data.tar.gz: f3d64da56e125b5282f794195a4e9cc48ea3504fdb5331de7e6138bb9373effdfbb9c8aa50edd5a3ceba41d6822eff8dc3510d0f1ed178d0034c3ce266d5a31d
+  metadata.gz: 18533bfd476b6056e937abb63f0532bb7051884e747ebf193671677eca6fd810db6facf551f2b6349be13f2a1d69e1398fc224665d8f9854193f5345a11921af
+  data.tar.gz: 86833af049566697d894f064b9c2d6db1718d08c416925c96dd4f2e3c8d3c39ced6d765567f578033cfad952b180e3d3eeebf1dabb10823d0084c9a0b6d5546c

data/Gemfile.lock

@@ -1,58 +1,58 @@
 PATH
   remote: .
   specs:
-    fluent-plugin-telemetry-iosxe (0.0.2)
+    fluent-plugin-telemetry-iosxe (0.0.3)
       fluentd (>= 0.14.10, < 2)
-      net-ssh (~> 4.2.0)
-      nokogiri (~> 1.8.1)
+      net-ssh (~> 5.1.0)
+      nokogiri (~> 1.10.0)
       nori (~> 2.6.0)
 
 GEM
   remote: https://rubygems.org/
   specs:
     cool.io (1.5.3)
-    fluentd (1.0.1)
+    dig_rb (1.0.1)
+    fluentd (1.3.3)
       cool.io (>= 1.4.5, < 2.0.0)
+      dig_rb (~> 1.0.0)
       http_parser.rb (>= 0.5.1, < 0.7.0)
       msgpack (>= 0.7.0, < 2.0.0)
-      ruby_dig (~> 0.0.2)
       serverengine (>= 2.0.4, < 3.0.0)
       sigdump (~> 0.2.2)
-      strptime (~> 0.1)
+      strptime (>= 0.2.2, < 1.0.0)
       tzinfo (~> 1.0)
       tzinfo-data (~> 1.0)
       yajl-ruby (~> 1.0)
     http_parser.rb (0.6.0)
-    mini_portile2 (2.3.0)
-    msgpack (1.2.0)
-    net-ssh (4.2.0)
-    nokogiri (1.8.1)
-      mini_portile2 (~> 2.3.0)
+    mini_portile2 (2.4.0)
+    msgpack (1.2.6)
+    net-ssh (5.1.0)
+    nokogiri (1.10.0)
+      mini_portile2 (~> 2.4.0)
     nori (2.6.0)
     power_assert (1.1.1)
     rake (12.3.0)
-    ruby_dig (0.0.2)
-    serverengine (2.0.5)
+    serverengine (2.1.0)
       sigdump (~> 0.2.2)
     sigdump (0.2.4)
-    strptime (0.1.9)
+    strptime (0.2.3)
     test-unit (3.2.6)
       power_assert
     thread_safe (0.3.6)
-    tzinfo (1.2.4)
+    tzinfo (1.2.5)
       thread_safe (~> 0.1)
-    tzinfo-data (1.2017.3)
+    tzinfo-data (1.2018.9)
       tzinfo (>= 1.0.0)
-    yajl-ruby (1.3.1)
+    yajl-ruby (1.4.1)
 
 PLATFORMS
   ruby
 
 DEPENDENCIES
-  bundler (~> 1.14)
+  bundler (~> 2.0.1)
   fluent-plugin-telemetry-iosxe!
   rake (~> 12.0)
   test-unit (~> 3.0)
 
 BUNDLED WITH
-   1.16.0
+   2.0.1

data/README.md

@@ -42,7 +42,7 @@ Collect telemetry input and then output to stdout.
   port 830
   user admin
   password admin
-  xpath_filter /ios-emul-oper-db:ios-emul-oper-db/cpu-usage/five-seconds
+  xpath_filters /process-cpu-ios-xe-oper:cpu-usage/cpu-utilization/five-seconds
   tag cpu-usage-five-seconds
   period 500
   @label @telemetry
@@ -54,6 +54,15 @@ Collect telemetry input and then output to stdout.
 </label>
 ```
 
+The output is as below.
+
+```
+2018-12-14 03:30:54.000000000 +0000 cpu-usage-five-seconds: {"cpu_usage":{"cpu_utilization":{"five_seconds":1}}}
+2018-12-14 03:30:59.000000000 +0000 cpu-usage-five-seconds: {"cpu_usage":{"cpu_utilization":{"five_seconds":2}}}
+2018-12-14 03:31:04.000000000 +0000 cpu-usage-five-seconds: {"cpu_usage":{"cpu_utilization":{"five_seconds":1}}}
+...
+```
+
 ### Configuration Example 2
 
 Collect telemetry input and then output to InfluxDB with flattening input.
@@ -63,7 +72,7 @@ Collect telemetry input and then output to InfluxDB with flattening input.
   @type telemetry_iosxe
   server 192.0.2.1
   port 830
-  xpath_filter /ios-emul-oper-db:ios-emul-oper-db/cpu-usage/five-seconds
+  xpath_filters /process-cpu-ios-xe-oper:cpu-usage/cpu-utilization/five-seconds
   user admin
   password admin
   tag cpu-usage-five-seconds
@@ -87,6 +96,19 @@ Collect telemetry input and then output to InfluxDB with flattening input.
   </match>
 </label>
 ```
+The inserted date on InfluxDB is as below.
+
+```
+> select * from "cpu-usage-five-seconds" limit 5
+name: cpu-usage-five-seconds
+time                cpu_usage.cpu_utilization.five_seconds
+----                --------------------------------------
+1544759262000000000 1
+1544759267000000000 2
+1544759272000000000 1
+1544759277000000000 2
+1544759282000000000 1
+```
 
 **server**
 
@@ -99,7 +121,11 @@ TCP port number to subscribe to Telemetry publisher.
 
 **xpath_filter**
 
-XPath filter to specify the information element to subscribe to.
+This parameter is deprecated. Use 'xpath_filters' instead.
+
+**xpath_filters**
+
+XPath filters to specify the information element to subscribe to.
 
 **user**
 

data/fluent-plugin-telemetry-iosxe.gemspec

@@ -3,7 +3,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name    = "fluent-plugin-telemetry-iosxe"
-  spec.version = "0.0.2"
+  spec.version = "0.0.3"
   spec.authors = ["Tetsuhiro Sato"]
   spec.email   = ["tetsusat@cisco.com"]
 
@@ -20,11 +20,11 @@ Gem::Specification.new do |spec|
   spec.test_files    = test_files
   spec.require_paths = ["lib"]
 
-  spec.add_development_dependency "bundler", "~> 1.14"
+  spec.add_development_dependency "bundler", "~> 2.0.1"
   spec.add_development_dependency "rake", "~> 12.0"
   spec.add_development_dependency "test-unit", "~> 3.0"
   spec.add_runtime_dependency "fluentd", [">= 0.14.10", "< 2"]
-  spec.add_runtime_dependency "net-ssh", ["~> 4.2.0"]
-  spec.add_runtime_dependency "nokogiri", ["~> 1.8.1"]
+  spec.add_runtime_dependency "net-ssh", ["~> 5.1.0"]
+  spec.add_runtime_dependency "nokogiri", ["~> 1.10.0"]
   spec.add_runtime_dependency "nori", ["~> 2.6.0"]
 end

data/lib/fluent/plugin/in_telemetry_iosxe.rb

@@ -21,7 +21,8 @@ module Fluent
     config_param :user, :string, default: "admin"
     config_param :password, :string, default: "admin", secret: true
     config_param :parser, :enum,list: [:rexml, :nokogiri], default: :nokogiri
-    config_param :xpath_filter, :string
+    config_param :xpath_filter, :string, default: nil, deprecated: "Use 'xpath_filters' instead"
+    config_param :xpath_filters, :array, value_type: :string
     config_param :tag, :string
     config_param :period, :integer
     config_param :strip_namespaces, :bool, default: true
@@ -40,6 +41,8 @@ module Fluent
         @sigint = true
       end
       @hello_done = false
+      @subscription_index = 0
+      @subscription_ids = []
       @buffer = ""
       @parser = Nori.new(:parser => @parser, :advanced_typecasting => false)
       hello = <<-EOS
@@ -47,21 +50,11 @@ module Fluent
 <hello xmlns="urn:ietf:params:xml:ns:netconf:base:1.0">
   <capabilities>
     <capability>urn:ietf:params:netconf:base:1.0</capability>
+  <capability>urn:ietf:params:netconf:base:1.1</capability>
   </capabilities>
 </hello>]]>]]>
       EOS
 
-      subscription = <<-EOS
-<?xml version="1.0" encoding="utf-8"?>
-<rpc message-id="101" xmlns="urn:ietf:params:xml:ns:netconf:base:1.0">
-  <establish-subscription xmlns="urn:ietf:params:xml:ns:yang:ietf-event-notifications" xmlns:yp="urn:ietf:params:xml:ns:yang:ietf-yang-push">
-    <stream>yp:yang-push</stream>
-    <yp:xpath-filter>#{@xpath_filter}</yp:xpath-filter>
-    <yp:period>#{@period}</yp:period>
-  </establish-subscription>
-</rpc>
-      EOS
-      
       @ssh = Net::SSH.start(@server, @user, :port => @port, :password => @password, :timeout => 10)
       @channel = @ssh.open_channel do |channel|
         channel.subsystem("netconf") do |ch, success|
@@ -79,91 +72,157 @@ module Fluent
           end
           log.info "send hello"
           ch.send_data(hello)
-          log.info "send subscription"
-          ch.send_data(subscription)
         end
       end
-      @ssh.loop(1) { not @sigint } # if we get sigint, we need to end loop 
+      @ssh.loop(1) { not @sigint } # if we get sigint, we need to end loop
     end
 
-    def shutdown
-      log.info "shutdown ..."
+    protected
+    def receive_data(data)
+      log.debug "receive data ..."
+      if @hello_done  # Chunked Framing
+        if data.include?('##')
+          if data == '##'
+            parse()
+          else
+            data.each_line do |line|
+              if not line =~ /^(#\d+|##)/
+                @buffer << line
+              end
+            end
+            parse()
+          end
+        else
+          data.each_line do |line|
+            if not line =~ /^#\d+/
+              @buffer << line
+            end
+          end
+        end
+      else    # End-of-Message Framing
+        if data.include?(']]>]]>')
+          if data == ']]>]]>'
+            parse()
+          else
+            data.each_line do |line|
+              if line != ']]>]]>'
+                @buffer << line
+              end
+            end
+            parse()
+          end
+        else
+          @buffer << data
+        end
+      end
+    end
+
+    def parse
+      log.debug "parse!"
+      log.debug @buffer
+      d = @parser.parse(@buffer)
+      if d['hello']
+        handle_hello(d)
+        @hello_done = true
+        subscribe()
+      elsif d['notification']
+        handle_notification(d)
+      elsif d["rpc_reply"]
+        if d["rpc_reply"]["subscription_id"]
+          handle_subscription_reply(d)
+          if @subscription_index < @xpath_filters.size
+            subscribe()
+          end
+        else
+          log.fatal d["rpc_reply"]["subscription_result"]
+        end
+      else
+        log.warn "got other messages"
+        log.debug d
+      end
+      @buffer = ''
+    end
+
+    def handle_hello(data)
+      log.info "got hello session_id=#{data["hello"]["session_id"]}"
+      log.debug data
+      @session_id = data["hello"]["session_id"]
+    end
+
+    def subscribe
+      subscription = <<-EOS
+<?xml version="1.0" encoding="utf-8"?>
+<rpc message-id="#{@subscription_index+1}" xmlns="urn:ietf:params:xml:ns:netconf:base:1.0">
+  <establish-subscription xmlns="urn:ietf:params:xml:ns:yang:ietf-event-notifications" xmlns:yp="urn:ietf:params:xml:ns:yang:ietf-yang-push">
+    <stream>yp:yang-push</stream>
+    <yp:xpath-filter>#{@xpath_filters[@subscription_index]}</yp:xpath-filter>
+    <yp:period>#{@period}</yp:period>
+  </establish-subscription>
+</rpc>
+      EOS
+      log.info "subscribe [#{@subscription_index+1}] #{@xpath_filters[@subscription_index]}"
+      log.debug chunk_frame(subscription.chomp)
+      @channel.send_data(chunk_frame(subscription.chomp))
+      @channel.send_data("\n##\n")
+      @subscription_index += 1
+    end
+
+    def unsubscribe(id)
       delete_subscription = <<-EOS
 <?xml version="1.0" encoding="utf-8"?>
 <rpc message-id="101" xmlns="urn:ietf:params:xml:ns:netconf:base:1.0">
   <delete-subscription xmlns="urn:ietf:params:xml:ns:yang:ietf-event-notifications" xmlns:netconf="urn:ietf:params:xml:ns:netconf:base:1.0">
-    <subscription-id>#{@subscription_id}</subscription-id>
+    <subscription-id>#{id}</subscription-id>
   </delete-subscription>
-</rpc>]]>]]>
+</rpc>
       EOS
+      log.info "unubscribe subscription_id=#{id}"
+      log.debug chunk_frame(delete_subscription.chomp)
+      @channel.send_data(chunk_frame(delete_subscription.chomp))
+      @channel.send_data("\n##\n")
+    end
+
+    def shutdown
+      log.info "shutdown ..."
+      for id in @subscription_ids
+        unsubscribe(id)
+      end
       close_session = <<-EOS
 <?xml version="1.0" encoding="utf-8"?>
 <rpc message-id="101" xmlns="urn:ietf:params:xml:ns:netconf:base:1.0">
   <close-session/>
-</rpc>]]>]]>
+</rpc>
       EOS
-      @channel.send_data(delete_subscription)
-      @channel.send_data(close_session)
+      @channel.send_data(chunk_frame(close_session.chomp))
+      @channel.send_data("\n##\n")
       @ssh.close
       super
     end
 
-    protected
-    def receive_data(data)
-      log.info "receive data ..."
-      if data.include?("]]>]]>")
-        msg = data.split("]]>]]>")
-        if msg.empty?  # this could happen when data is exactly "]]>]]>"
-          if @hello_done
-            parse_msg()
-          else
-            parse_hello()
-            @hello_done = true
-          end
-        else
-          for m in msg
-            @buffer << m
-            if @hello_done
-              parse_msg()
-            else
-              parse_hello()
-              @hello_done = true
-            end
-          end
-        end
-      else
-        @buffer << data
-      end
-    end
-
-    def parse_hello
-      hello = @parser.parse(@buffer)
-      log.debug hello
-      @session_id = hello["hello"]["session_id"]
-      log.info "Session ID is #{@session_id}"
-      @buffer = ""
+    def chunk_frame(msg)
+      "\n##{msg.length}\n" << msg
     end
 
-    def parse_msg
-      h = @parser.parse(@buffer)
-      log.debug h
-      if h["notification"]
-        iso8601_time = h["notification"]["eventTime"]
-        unix_time = Time.iso8601(iso8601_time).to_i
-        content = h["notification"]["push_update"]["datastore_contents_xml"]
-        traverse(content) do |node, key, parent|
-          if @typecast_integer && node.is_i?
-            parent[key] = node.to_i unless parent.nil?
-          elsif @typecast_float && node.is_f?
-            parent[key] = node.to_f unless parent.nil?
-          end
+    def handle_notification(data)
+      log.info "got notification"
+      log.debug data
+      iso8601_time = data["notification"]["eventTime"]
+      unix_time = Time.iso8601(iso8601_time).to_i
+      content = data["notification"]["push_update"]["datastore_contents_xml"]
+      traverse(content) do |node, key, parent|
+        if @typecast_integer && node.is_i?
+          parent[key] = node.to_i unless parent.nil?
+        elsif @typecast_float && node.is_f?
+          parent[key] = node.to_f unless parent.nil?
         end
-        router.emit(@tag, unix_time, content)
-      elsif h["rpc_reply"] && h["rpc_reply"]["subscription_id"]
-        @subscription_id = h["rpc_reply"]["subscription_id"]
-        log.info "Subscription ID is #{@subscription_id}"
       end
-      @buffer = ""
+      router.emit(@tag, unix_time, content)
+    end
+
+    def handle_subscription_reply(data)
+      log.info "got subscription reply subscription_id=#{data['rpc_reply']['subscription_id']}"
+      log.debug data
+      @subscription_ids << data["rpc_reply"]["subscription_id"]
     end
 
     def traverse(obj, key=nil, parent=nil, &blk)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-telemetry-iosxe
 version: !ruby/object:Gem::Version
-  version: 0.0.2
+  version: 0.0.3
 platform: ruby
 authors:
 - Tetsuhiro Sato
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-12-20 00:00:00.000000000 Z
+date: 2019-01-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -16,14 +16,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.14'
+        version: 2.0.1
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.14'
+        version: 2.0.1
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -78,28 +78,28 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 4.2.0
+        version: 5.1.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 4.2.0
+        version: 5.1.0
 - !ruby/object:Gem::Dependency
   name: nokogiri
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.8.1
+        version: 1.10.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.8.1
+        version: 1.10.0
 - !ruby/object:Gem::Dependency
   name: nori
   requirement: !ruby/object:Gem::Requirement
@@ -150,7 +150,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.5.2.1
+rubygems_version: 2.6.14.3
 signing_key: 
 specification_version: 4
 summary: Fluentd input plugin to collect IOS-XE telemetry.