fluent-plugin-machinist 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 02f43bc747c5bd41cad3df6747d1b622ddf94649ec5026f0921976ad1fa0229c
+  data.tar.gz: 6db613f11cf3223fdb011bdd7d839262ea17c4f02154245cb12b03e41e5a8fc2
+SHA512:
+  metadata.gz: 80f598d5491ee42fe2239945b1db42faa398a60bf3667e6c9cddc42f786d6694b887a00d343d8475c815d424c696a7e6b8c2de490a94159839e99b644eedaf41
+  data.tar.gz: 1fe1c7d18c5c1465d584e9189e78d003fd346de65cb6ec24f0a783af5a5ae9c9c87721693e1d7cd775da131e1ac22475f94f0aea932fd1905df14b75d7300983

data/.gitignore

@@ -0,0 +1,17 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp

data/Gemfile

@@ -0,0 +1,3 @@
+source "https://rubygems.org"
+
+gemspec

data/LICENSE

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2019 Naoya Kaneko
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,184 @@
+# Fluent::Plugin::Machinist
+
+## Overview
+
+
+[Fluentd](http://fluentd.org/) input plugin that forwards record to Machinist
+
+- [Machinist](https://machinist.iij.jp/)
+
+
+## Installation
+
+install from rubygems
+
+```
+gem install fluent-plugin-machinist
+or
+fluent-gem install fluent-plugin-machinist
+```
+
+or build and install
+
+```
+git clone https://github.com/iij/fluent-plugin-machinist.git
+cd fluent-plugin-machinist
+rake build
+gem install pkg/fluent-plugin-machinist-0.1.0.gem
+```
+
+## Configuration
+
+```
+<match test.***>
+    @type machinist
+
+    endpoint_url https://gw.machinist.iij.jp/gateway
+    agent_id WHATEVER_AGENT_ID_HERE
+    api_key WHATEVER_API_KEY_HERE
+    use_ssl true
+    verify_ssl true
+
+    value_key temp
+
+    # optional parameters
+    namespace mydesk
+    tags floor:17F,sensor:wionode
+</match>
+```
+
+```
+<match test.***>
+    @type machinist
+
+    endpoint_url https://gw.machinist.iij.jp/gateway
+    agent_id WHATEVER_AGENT_ID_HERE
+    api_key WHATEVER_API_KEY_HERE
+    use_ssl true
+    verify_ssl true
+
+    value_keys temp,humid
+
+    # optional parameters
+    namespace_key location
+    tag_keys floor,sensor
+</match>
+```
+
+
+**endpoint\_url**
+
+URL where the plugin does POST record (required).
+
+**agent\_id**
+
+Agent ID is used to authenticate to endpoint\_url (required).
+
+**api_key**
+
+Key used to authenticate to endpoint\_url (required).
+
+**use\_ssl**
+
+Whether the plugin use ssl to communicate endpoint\_url. Required to be true if
+endpoint\_url is "https://".
+(Default: false)
+
+**value\_key**
+
+Specifying which key and value to be used as Machinist metric.
+The plugin extracts entry with matching key in record and expand key,value into "name","value" hash.
+
+```
+Example: 
+if
+  recored is  {"hoge":"fuga", "temp": "28.0", "humid": "50.0" }
+  "value_key" is "temp"
+then
+  generates metrics as [{"name": "temp", "value": 28.0}]
+```
+
+Either value\_key or value\_keys is mutually required.
+
+**value\_keys**
+
+Specifying which keys and values to be used as Machinist metric.
+Unlike value\_key, this accepts array of keys.
+
+```
+Example: 
+if
+  recored is  {"hoge":"fuga", "temp": "28.0", "humid": "50.0" }
+  "value_keys" is "temp,humid"
+then
+  generates metrics as [{"name": "temp", "value": 28.0}, {"name": "humid", "value": 50.0}]
+```
+
+value\_keys is mutually exclusive with value\_key.
+
+**namespace**
+
+Specifying static namespace to be used in metric (string).
+
+
+```
+if
+  namespace is "mydesk"
+  recored is  {"hoge":"fuga", "temp": "28.0", "humid": "50.0" }
+  "value_key" is "temp"
+then
+  generates metrics as [{"name":"temp", "value":28.0, "namespace":"mydesk"]
+```
+
+**namespace\_key**
+
+Extracts namespace from value for specified key in record (array).
+
+```
+if
+  namespace_key is "location"
+  recored is  {"hoge":"fuga", "temp": "28.0", "humid": "50.0", "location":"myhome" }
+  "value_key" is "temp"
+then
+  generates metrics as [{"name":"temp", "value":28.0, "namespace":"myhome"]
+```
+
+**tags**
+
+Specifying static tags to be used in metric (hash).
+
+```
+if
+  tags is "floor:17F,sensor:wionode"
+  recored is  {"hoge":"fuga", "temp": "28.0", "humid": "50.0", "location":"myhome" }
+  "value_key" is "temp"
+then
+  generates metrics as [{"name":"temp", "value":28.0, "tags":{"floor":"17F", "sensor":"wionode"}}]
+```
+
+**tag\_keys**
+
+Extracts tags from specified keys in record (array).
+
+```
+if
+  tag_keys is "floor,sensor"
+  recored is  {"hoge":"fuga", "temp": "28.0", "humid": "50.0", "location":"myhome", "floor":"18F", "sensor":"wionode" }
+  "value\_key" is "temp"
+then
+  generates metrics as [{"name":"temp", "value":28.0, "tags":{"floor":"18F", "sensor":"wionode"}}]
+```
+
+## TODO
+
+- "meta" support
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/iij/fluent-plugin-machinist
+
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT).
+

data/Rakefile

@@ -0,0 +1,13 @@
+require "bundler"
+Bundler::GemHelper.install_tasks
+
+require "rake/testtask"
+
+Rake::TestTask.new(:test) do |t|
+  t.libs.push("lib", "test")
+  t.test_files = FileList["test/**/test_*.rb"]
+  t.verbose = true
+  t.warning = true
+end
+
+task default: [:test]

data/example/test-fluentd.conf

@@ -0,0 +1,27 @@
+<source>
+  @type exec
+  tag airport
+  command /bin/sh test.sh
+  format json
+  run_interval 5s
+</source>
+
+<match airport.**>
+  @type copy
+  <store>
+    @type stdout
+  </store>
+  <store>
+    @type machinist
+
+    endpoint_url https://gw.machinist.iij.jp/gateway
+    agent_id XXXXXXXXXX
+    api_key XXXXXX
+
+    value_keys rssi,noise
+
+    namespace macbookpro01
+
+    tag_keys ssid
+  </store>
+</match>

data/example/test.sh

@@ -0,0 +1,4 @@
+#!/bin/sh
+# This is Wi-Fi scanning script for testing
+# Example {"rssi": -47, "noise": -96, "ssid": "test_ssid"}
+airport -I | awk '{gsub(/ /,"", $0);print $1}' | awk -F ':' '/^agrCtlRSSI/{rssi = int($2)} /^agrCtlNoise/{noise = int($2)} /^SSID/{ssid = $2} END {printf "{\"rssi\": %d, \"noise\": %d, \"ssid\": \"%s\"}", rssi, noise, ssid}'

data/fluent-plugin-machinist.gemspec

@@ -0,0 +1,27 @@
+lib = File.expand_path("../lib", __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+
+Gem::Specification.new do |spec|
+  spec.name    = "fluent-plugin-machinist"
+  spec.version = "0.1.0"
+  spec.authors = ["Naoya Kaneko"]
+  spec.email   = ["enukane@glenda9.org"]
+
+  spec.summary       = %q{Fluent output plugin for IIJ Machinist}
+  spec.description   = %q{Fluent output plubin for IIJ Machinist}
+  spec.homepage      = "https://github.com/iij/fluent-plugin-machinist"
+  spec.license       = "MIT"
+
+  test_files, files  = `git ls-files -z`.split("\x0").partition do |f|
+    f.match(%r{^(test|spec|features)/})
+  end
+  spec.files         = files
+  spec.executables   = files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = test_files
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.14"
+  spec.add_development_dependency "rake", "~> 12.0"
+  spec.add_development_dependency "test-unit", "~> 3.0"
+  spec.add_runtime_dependency "fluentd", [">= 0.14.10", "< 2"]
+end

data/lib/fluent/plugin/out_machinist.rb

@@ -0,0 +1,127 @@
+#
+# Copyright 2019- Naoya Kaneko
+#
+
+require "fluent/plugin/output"
+
+module Fluent
+  module Plugin
+    class MachinistOutput < Fluent::Plugin::Output
+      Fluent::Plugin.register_output("machinist", self)
+
+      def initialize
+        super
+        require "net/http"
+        require "uri"
+      end
+
+      # authentication and access requirements
+      config_param :endpoint_url, :string
+      config_param :agent_id, :string
+      config_param :api_key, :string
+      config_param :use_ssl, :bool, :default => true
+      config_param :verify_ssl, :bool, :default => true
+
+      # name & value selection
+      config_param :value_key, :string, :default => nil
+      config_param :value_keys, :array, value_type: :string, :default => nil
+
+      # specifying statistically
+      ## namespace
+      config_param :namespace, :string, :default => nil # namespace string (static)
+      config_param :namespace_key, :string, :default => nil #TBD: use value for specified hash entry for namespace
+      config_param :tags, :hash, :default => nil # specify tags hash (static)
+      config_param :tag_keys, :array, value_type: :string, :default => nil # 
+
+      def start
+        super
+      end
+
+      def shutdown
+        super
+      end
+
+      def process(tag, es)
+        es.each do |time, record|
+          handle_record(tag, time, record)
+        end
+      end
+
+      def handle_record(tag, time, record)
+        data = create_payload(tag, time, record)
+
+        send_request(data)
+      end
+
+      def create_payload(tag, time, record)
+        metrics = []
+        @value_keys = [@value_key] if @value_key
+        ns = if @namespace
+               @namespace
+             elsif @namespace_key
+               record[@namespace_key]
+             else
+               nil
+             end
+        tags = if @tags
+                 @tags
+               elsif @tag_keys
+                 @tag_keys.map{|key|
+                   [key, record[key].to_s]
+                 }.to_h
+               else
+                 nil
+               end
+
+        metrics += @value_keys.map{|key|
+          metric = {
+            :name => key,
+            :value => record[key].to_f
+          }
+          metric[:namespace] = ns if ns
+          metric[:tags] = tags if tags
+          metric
+        }
+
+        json = {
+          :agent_id => @agent_id,
+          :api_key => @api_key,
+          :metrics => metrics
+        }
+
+        return JSON.dump(json)
+      end
+
+      def send_request(data)
+        url = URI(@endpoint_url)
+        http = Net::HTTP.new(url.host, url.port)
+        if @use_ssl
+          http.use_ssl = true
+          http.ca_file = OpenSSL::X509::DEFAULT_CERT_FILE
+          unless @verify_ssl
+            http.verify_mode = OpenSSL::SSL::VERIFY_NONE
+          end
+        end
+
+        req = Net::HTTP::Post.new(url)
+        req["content-type"] = 'application/json'
+
+        req.body = data
+        res = http.request(req)
+
+        unless res and res.is_a?(Net::HTTPSuccess)
+          summary = if res
+                      "#{res.code} #{res.message} #{res.body}"
+                    else
+                      "res=nil"
+                    end
+          $log.warn "failed to #{req.method} #{url} #{summary}"
+        end
+      rescue => e
+        $log.warn "Net::HTTP.#{req.method.capitalize} raises exception : #{e.class}, '#{e.message}'"
+        raise e
+      end
+
+    end
+  end
+end

data/test/helper.rb

@@ -0,0 +1,8 @@
+$LOAD_PATH.unshift(File.expand_path("../../", __FILE__))
+require "test-unit"
+require "fluent/test"
+require "fluent/test/driver/output"
+require "fluent/test/helpers"
+
+Test::Unit::TestCase.include(Fluent::Test::Helpers)
+Test::Unit::TestCase.extend(Fluent::Test::Helpers)

data/test/plugin/test_out_machinist.rb

@@ -0,0 +1,46 @@
+require "helper"
+require "fluent/plugin/out_machinist.rb"
+
+class MachinistOutputTest < Test::Unit::TestCase
+  setup do
+    Fluent::Test.setup
+  end
+
+  CONFIG = %[
+    endpoint_url https://example.org/gateway
+    agent_id THIS_IS_AGENTID
+    api_key THIS_IS_API_KEY
+    use_ssl true
+    verify_ssl true
+
+    value_key VALUE_KEY
+    value_keys VALUE_KEY0,VALUE_KEY1
+
+    namespace NAMESPACE
+    namespace_key NAMESPACE_KEY
+
+    tags TAG_KEY0:TAG_VALUE0,TAG_KEY1:TAG_VALUE1
+    tag_keys TAG_KEY2,TAG_KEY3
+  ]
+
+  test "configure" do
+    d = create_driver(CONFIG)
+
+    assert_equal "https://example.org/gateway", d.instance.endpoint_url
+    assert_equal "THIS_IS_AGENTID", d.instance.agent_id
+    assert_equal "THIS_IS_API_KEY", d.instance.api_key
+    assert_true d.instance.use_ssl
+    assert_true d.instance.verify_ssl
+    assert_equal "VALUE_KEY", d.instance.value_key
+    assert_equal ["VALUE_KEY0","VALUE_KEY1"], d.instance.value_keys
+    assert_equal "NAMESPACE", d.instance.namespace
+    assert_equal({"TAG_KEY0"=>"TAG_VALUE0", "TAG_KEY1"=>"TAG_VALUE1"}, d.instance.tags)
+    assert_equal ["TAG_KEY2", "TAG_KEY3"], d.instance.tag_keys
+  end
+
+  private
+
+  def create_driver(conf)
+    Fluent::Test::Driver::Output.new(Fluent::Plugin::MachinistOutput).configure(conf)
+  end
+end

metadata

@@ -0,0 +1,118 @@
+--- !ruby/object:Gem::Specification
+name: fluent-plugin-machinist
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Naoya Kaneko
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2019-12-09 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.14'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.14'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '12.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '12.0'
+- !ruby/object:Gem::Dependency
+  name: test-unit
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.14.10
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.14.10
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2'
+description: Fluent output plubin for IIJ Machinist
+email:
+- enukane@glenda9.org
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- Gemfile
+- LICENSE
+- README.md
+- Rakefile
+- example/test-fluentd.conf
+- example/test.sh
+- fluent-plugin-machinist.gemspec
+- lib/fluent/plugin/out_machinist.rb
+- test/helper.rb
+- test/plugin/test_out_machinist.rb
+homepage: https://github.com/iij/fluent-plugin-machinist
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.0.6
+signing_key: 
+specification_version: 4
+summary: Fluent output plugin for IIJ Machinist
+test_files:
+- test/helper.rb
+- test/plugin/test_out_machinist.rb