fluent-plugin-everysense 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: b65915adad0fd2a9922936d6902098e070d9a4a5
+  data.tar.gz: e1cbc37c6759f0ee1ce2f3c0f0ce7c0544bf0e8e
+SHA512:
+  metadata.gz: 3df7f3708cfc8f66f89e9c0fc6d71b43599599fa67740870a935921fc2f4ac6415595818ea4bc0b0d5b869d7553fe0d436d356d00ea88a1282cac57aaca96f26
+  data.tar.gz: ac200dd0e86b6d0b2563ca31606a836d2f427f7cf9936b0371293dabc633fa0eb07162ca2dd13cc591daa0c91bb8dc5258db4a797d4374698d9ecc637ffc5fc6

data/.gitignore

@@ -0,0 +1,36 @@
+*.gem
+*.rbc
+/.config
+/coverage/
+/InstalledFiles
+/pkg/
+/spec/reports/
+/spec/examples.txt
+/test/tmp/
+/test/version_tmp/
+/tmp/
+
+## Specific to RubyMotion:
+.dat*
+.repl_history
+build/
+
+## Documentation cache and generated files:
+/.yardoc/
+/_yardoc/
+/doc/
+/rdoc/
+
+## Environment normalization:
+/.bundle/
+/vendor/bundle
+/lib/bundler/man/
+
+# for a library or gem, you might want to ignore these files since the code is
+# intended to run in multiple environments; otherwise, check them in:
+# Gemfile.lock
+# .ruby-version
+# .ruby-gemset
+
+# unless supporting rvm < 1.11.0 or doing something fancy, ignore this:
+.rvmrc

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in fluent-plugin-mqtt-io.gemspec
+gemspec

data/LICENSE

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2016 Toyokazu Akiyama
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/README.md

@@ -0,0 +1,84 @@
+# fluent-plugin-everysense
+Fluent Input/Output Plugin for EverySense platform
+
+## What is EverySense?
+
+Collecting personal data is one of the key points in IoT applications. However, it is basically difficult to search people who can provide own sensor data and to motivate people for providing own data to the world. [EverySense](http://every-sense.com/) is a platform to exchange sensor data which can define own recipe to search the data provider matching the condition. It also provide "point" mechanism to motivate people for providing own data. The "point" can be exchanged with the other sensor data or goods like general creadit card point.
+
+This plugin provides input/output function to [EverySense platform](https://service.every-sense.com/). Data from fluentd event can be uploaded to EverySense and the data provided from EverySense can be posted into fluentd network.
+
+
+## Installation
+
+```
+gem install fluent-plugin-everysense
+```
+
+
+## Usage
+
+fluent-plugin-everysense has Input and Output Plugins for EverySense platform.
+
+
+### Input Plugin (Fluent::EverySenseInput)
+
+Input Plugin can be used via source directive in the configuration.
+
+```
+<source>
+  @type everysense
+  format json
+  tag tag_name
+  login_name your_login_name
+  password your_password
+  device_id xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxx
+  polling_interval 15
+</source>
+```
+
+- **format** (required): type of parser can be specified to parse input data
+- **tag** (required): tag name appended to the input data inside fluentd network
+- **login_name** (required): login_name to login https://service.every-sense.com/
+- **password** (required): password for the login_name
+- **device_id** (required): the target device id to obtain input data
+- **polling_interval**: interval to poll EverySense JSON over HTTP API
+
+
+### Output Plugin (Fluent::EverySenseOutput)
+
+Output Plugin can be used via match directive.
+
+```
+<match tag_name>
+  @type everysense
+  login_name your_login_name
+  password your_password
+  format json
+  device_id xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxx
+</match>
+```
+
+- **format** (required): type of formatter can be specified to parse input data
+- **login_name** (required): login_name to login https://service.every-sense.com/
+- **password** (required): password for the login_name
+- **device_id** (required): the target device id to submit sensor data
+
+
+## TODO
+
+Recipe related function is not implemented yet.
+
+
+## Contributing
+
+1. Fork it ( http://github.com/toyokazu/fluent-plugin-everysense/fork )
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request
+
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT).
+

data/Rakefile

@@ -0,0 +1,10 @@
+require "bundler/gem_tasks"
+require "rake/testtask"
+
+Rake::TestTask.new(:test) do |t|
+  t.libs << "test"
+  t.libs << "lib"
+  t.test_files = FileList['test/**/*_test.rb']
+end
+
+task :default => :test

data/fluent-plugin-everysense.gemspec

@@ -0,0 +1,27 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+
+Gem::Specification.new do |spec|
+  spec.name          = "fluent-plugin-everysense"
+  spec.version       = "0.0.1"
+  spec.authors       = ["Toyokazu Akiyama"]
+  spec.email         = ["toyokazu@gmail.com"]
+
+  spec.summary       = %q{Fluent Input/Output plugin for EverySense Framework}
+  spec.description   = %q{Fluent Input/Output plugin for EverySense Framework}
+  spec.homepage      = "https://github.com/toyokazu/fluent-plugin-everysense"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files`.gsub(/images\/[\w\.]+\n/, "").split($/)
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.required_ruby_version = '>= 2.0.0'
+
+  spec.add_dependency 'fluentd', '>= 0.10.0'
+
+  spec.add_development_dependency "bundler", "~> 1.10"
+  spec.add_development_dependency "rake", "~> 10.0"
+end

data/lib/fluent/plugin/everysense_proxy.rb

@@ -0,0 +1,136 @@
+module Fluent
+  module EverySenseProxy
+    require 'uri'
+    require 'net/http'
+    require 'json'
+    require 'time'
+
+    def self.included(base)
+      base.desc 'EverySense API URI'
+      base.config_param :url, :string, :default => 'https://api.every-sense.com:8001/'
+      base.desc 'login_name for EverySense API'
+      base.config_param :login_name, :string, :default => nil  # TODO: mandatory option
+      base.desc 'password for EverySense API'
+      base.config_param :password, :string, :default => nil  # TODO: mandatory option
+      base.config_param :device_id, :string, :default => nil  # TODO: mandatory option
+      base.config_param :recipe_id, :string, :default => nil
+      base.config_param :format, :string, :default => 'json'
+      #base.config_param :keep_alive, :integer, :default => 2
+      base.config_param :from, :string, :default => Time.now.iso8601
+    end
+
+    def start_proxy
+      $log.debug "start everysense proxy #{@uri}"
+
+      @uri = URI.parse(@url)
+      @https = Net::HTTP.new(@uri.host, @uri.port)
+      @https.use_ssl = (@uri.scheme == 'https')
+      @session_key = nil
+    end
+
+    def shutdown_proxy
+      delete_session
+    end
+
+    def error_handler(response, message)
+      if response.code != "200"
+        $log.error :error => message
+        $log.debug "code: #{response.code}"
+        $log.debug "message: #{response.message}"
+        $log.debug "body: #{response.body}"
+        return false
+      end
+      return true
+    end
+
+    def valid_session?
+      !@session_key.nil? # TODO validate @session_key using EverySense API
+    end
+
+    def validate_device
+      if @device_id.nil?
+        $log.error :error => 'Invalid device id.'
+        return false
+      end
+      return true
+    end
+
+    def validate_recipe
+      if @recipe_id.nil?
+        $log.error :error => 'Invalid recipe id.'
+        return false
+      end
+      return true
+    end
+
+    def create_session_request
+      session_req = Net::HTTP::Post.new(@uri + '/session')
+      session_req.body = {login_name: @login_name, password: @password}.to_json
+      session_req.content_type = 'application/json'
+      session_req
+    end
+
+    def create_session
+      return @session_key if valid_session?
+      @session_req ||= create_session_request
+      session_res = @https.request(@session_req)
+      return nil if !error_handler(session_res, 'create_session failed.')
+      @session_key = JSON.parse(session_res.body)["session_key"]
+    end
+
+    def delete_session_request
+      Net::HTTP::Delete.new(@uri + "/session/#{@session_key}")
+    end
+
+    def delete_session
+      return if !valid_session?
+      del_session_res = @https.request(delete_session_request)
+      error_handler(del_session_res, 'delete_session failed.')
+    end
+
+    def put_message_request(message)
+      put_message_req = Net::HTTP::Post.new(@uri + "/device_data/#{@device_id}")
+      put_message_req.body = message
+      put_message_req.content_type = "application/#{@format}"
+      put_message_req
+    end
+
+    def put_message(message)
+      return if !validate_device
+      put_message_res = @https.request(put_message_request(message))
+      error_handler(put_message_res, "put_message: '#{message}' failed.")
+    end
+
+    def get_message_request
+      get_message_req = @uri + "/device_data/#{@device_id}?session_key=#{@session_key}&from=#{URI.encode_www_form_component(@from)}&to=#{URI.encode_www_form_component(Time.now.iso8601)}"
+      @from = (Time.now + 1).iso8601
+      get_message_req
+    end
+
+    def get_message
+      if !valid_session?
+        return nil if create_session.nil?
+        $log.debug "session #{@session_key} created."
+      end
+      return nil if !validate_device
+      get_message_res = @https.get(get_message_request)
+      return nil if !error_handler(get_message_res,"get_message failed.")
+      get_message_res.body
+    end
+
+    def get_recipe_request
+      @uri + "/recipe_data/#{@recipe_id}"
+    end
+
+    def get_recipe
+      if !valid_session?
+        return nil if create_session.nil?
+        $log.debug "session #{@session_key} created."
+      end
+      return nil if !validate_recipe
+      get_recipe_res = @https.get(get_recipe_request)
+      return nil if !error_handler(get_recipe_res, "get_recipe failed.")
+      get_recipe_res.body
+    end
+  end
+end

data/lib/fluent/plugin/in_everysense.rb

@@ -0,0 +1,86 @@
+module Fluent
+  class EverySenseInput < Input
+    require 'fluent/plugin/everysense_proxy'
+    include EverySenseProxy
+
+    Plugin.register_input('everysense', self)
+
+    desc 'Tag name assigned to inputs'
+    config_param :tag, :string, :default => nil # TODO: mandatory option
+    desc 'Polling interval to get message from EverySense API'
+    config_param :polling_interval, :integer, :default => 15
+    config_param :recv_time, :bool, :default => false
+    config_param :recv_time_key, :string, :default => "recv_time"
+
+    # Define `router` method of v0.12 to support v0.10 or earlier
+    unless method_defined?(:router)
+      define_method("router") { Fluent::Engine }
+    end
+
+    def configure(conf)
+      super
+      configure_parser(conf)
+    end
+
+    def configure_parser(conf)
+      @parser = Plugin.new_parser(@format)
+      @parser.configure(conf)
+    end
+
+    def start
+      raise StandardError.new if @tag.nil?
+      super
+      start_proxy
+      @proxy_thread = Thread.new do
+        while (true)
+          begin
+            message = get_message
+            $log.debug "get_message: #{message}"
+            emit(message) if !message.nil?
+            sleep @polling_interval
+          rescue Exception => e
+            $log.error :error => e.to_s
+            $log.debug(e.backtrace.join("\n"))
+            #$log.debug_backtrace(e.backtrace)
+            sleep @polling_interval
+          end
+        end
+      end
+    end
+
+    def add_recv_time(record)
+      if @recv_time
+        # recv_time is recorded in ms
+        record.merge({@recv_time_key => Time.now.instance_eval { self.to_i * 1000 + (usec/1000) }})
+      else
+        record
+      end
+    end
+
+    def parse(message)
+      @parser.parse(message) do |time, record|
+        if time.nil?
+          $log.debug "Since time_key field is nil, Fluent::Engine.now is used."
+          time = Fluent::Engine.now
+        end
+        $log.debug "#{time}, #{add_recv_time(record)}"
+        return [time, add_recv_time(record)]
+      end
+    end
+
+    def emit(message)
+      begin
+        router.emit(@tag, *parse(message))
+      rescue Exception => e
+        $log.error :error => e.to_s
+        $log.debug_backtrace(e.backtrace)
+      end
+    end
+
+    def shutdown
+      super
+      @proxy_thread.kill
+      shutdown_proxy
+    end
+  end
+end

data/lib/fluent/plugin/out_everysense.rb

@@ -0,0 +1,58 @@
+module Fluent
+  class EverySenseOutput < Output
+    require 'fluent/plugin/everysense_proxy'
+    include EverySenseProxy
+
+    Plugin.register_output('everysense', self)
+
+    # config_param defines a parameter. You can refer a parameter via @path instance variable
+    # Without :default, a parameter is required.
+    #desc 'Flush interval to put message to EverySense API'
+    #config_param :flush_interval, :integer, :default => 15
+
+    # This method is called before starting.
+    # 'conf' is a Hash that includes configuration parameters.
+    # If the configuration is invalid, raise Fluent::ConfigError.
+    def configure(conf)
+      super
+      configure_formatter(conf)
+    end
+
+    def configure_formatter(conf)
+      @formatter = Plugin.new_formatter(@format)
+      @formatter.configure(conf)
+    end
+
+    # This method is called when starting.
+    # Open sockets or files here.
+    def start
+      super
+      start_proxy
+    end
+
+    def add_send_time(record)
+      if @send_time
+        # send_time is recorded in ms
+        record.merge({@send_time_key => Time.now.instance_eval { self.to_i * 1000 + (usec/1000) }})
+      else
+        record
+      end
+    end
+
+    def emit(tag, es, chain)
+      chain.next
+      es.each {|time,record|
+        $log.debug "#{tag}, #{@formatter.format_record(add_send_time(record)).chomp}\n"
+        put_message(@formatter.format_record(add_send_time(record)))
+      }
+      $log.flush
+    end
+
+    # This method is called when shutting down.
+    # Shutdown the thread and close sockets or files here.
+    def shutdown
+      super
+      shutdown_proxy
+    end
+  end
+end

data/tutorial/ja/json_over_http_api_tutorial.md

@@ -0,0 +1,119 @@
+# Ruby を使って EverySense の API で遊んでみよう！
+
+Ruby のインタラクティブシェル irb あるいは pry などを利用して EverySense の API で遊んでみましょう。以下では最も標準的な API である JSON over HTTPゲートウェイについて触れています。irb を起動して以下のコードを貼り付けてみましょう。ただし、login_name および password は以下のURIからご自身で登録したものに書き換えて下さい。
+
+https://service.every-sense.com/
+
+以下ではまずデバイスのデータをアップロード、ダウンロードする方法から触れますので、ファームオーナー（「センサー情報を提供される方」）でアカウント登録をしてください。
+
+```ruby
+require 'uri'
+require 'net/http'
+require 'json'
+
+uri = URI.parse('https://api.every-sense.com:8001/')
+https = Net::HTTP.new(uri.host, uri.port)
+https.use_ssl = true
+session_req = Net::HTTP::Post.new(uri + '/session')
+session_req.body = {login_name: 'your_name', password: 'your_password'}.to_json
+session_req.content_type = 'application/json'
+session_res = https.request(session_req)
+
+session_res.code
+session_res.message
+session_res.body
+```
+
+最後の3行では以下の様な結果が出力されましたか？
+
+```
+irb(main):013:0* session_res.code
+=> "200"
+irb(main):014:0> session_res.message
+=> "OK"
+irb(main):015:0> session_res.body
+=> "{\"code\":0,\"session_key\":\"YYYYYYYY-YYYY-YYYY-YYYY-YYYYYYYYYYYYY\"}"
+```
+
+ここで取得したsession_keyは、デバイスのデータ（/device_data）およびレシピのデータ（/recipe_data）を読み出す際に必要となります。
+
+```ruby
+session_key = JSON.parse(session_res.body)["session_key"]
+```
+
+次にデバイスのデータのアップロード、ダウンロードについて試してみるため、テスト用のデバイス Test_Device を登録してください。デバイスの登録は以下から行えます。
+
+https://service.every-sense.com/ja/devices
+
+デバイスクラスとしてTest_Deviceを選択し、「追加」します。
+登録できると、UUIDが表示されます。
+XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX
+
+デバイスのデータのアップロード、ダウンロードをする際にUUIDが必要になります。
+
+まずはアップロードから試してみましょう。なんと、アップロードには認証は不要です。ということは、デバイスのUUIDを知っている人は誰でもアップロードできてしまう？？皆さんUUIDの取り扱いには注意しましょう。
+
+URIにデバイスのUUIDを指定して、アップロードしたいデータをPOSTします。ここでは content_type に 'application/json' を指定して、JSON形式のデータをアップロードしてみます。
+
+```ruby
+upload_req = Net::HTTP::Post.new(uri + '/device_data/XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX')
+upload_req.body = {test_data: {data1: "hoge", data2: 123}}.to_json
+upload_req.content_type = 'application/json'
+upload_res = https.request(upload_req)
+upload_res.body
+```
+
+最後の応答に以下のように 'code: 0' が返されていればアップロード成功です。
+
+```
+irb(main):023:0> upload_res.body
+=> "{\"code\":0}"
+```
+
+データの取得にはログインID (UUID) + パスワードあるいは、session_keyが要求されます。以下では session_key を利用しています。
+
+```ruby
+get_res = https.get(uri + "/device_data/XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX?session_key=#{session_key}")
+get_res.body
+```
+
+しばらくはキャッシュされているので、getすると同じデータがダウンロードされます。そのため、
+実際には from, to でデータが登録された期間を指定して必要なデータをすることになります。
+
+```ruby
+get_res = https.get(uri + "/device_data/XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX?session_key=#{session_key}&from=2016-03-06-06:20&to=2016-03-06-07:00")
+get_res.body
+```
+
+from, to の時刻はUTCで指定してください。
+
+次に「センサー情報収集」のため、レストランオーナーとしてレシピを登録、確認する方法について見ていきます。レストランオーナーになるためには、service.every-sense.com の右上の設定メニューから「レストランオーナー」の登録を行った上で、「オーナー切替」で「レストランオーナー」として作業してください。
+
+https://service.every-sense.com/
+
+ZZZZZZZZ-ZZZZ-ZZZZ-ZZZZ-ZZZZZZZZZZZZ
+
+
+```ruby
+recipe_data = https.get(uri + "/recipe_data/ZZZZZZZZ-ZZZZ-ZZZZ-ZZZZ-ZZZZZZZZZZZZ.xml?session_key=#{session_key}&from=2016-03-06-07:00&to=2016-03-06-07:50")
+recipe_data.body
+```
+
+# session_keyについて
+
+device_dataにはlogin_name、passwordでもアクセスできますが、毎回提示すると login_name、passwordが奪われる可能性が高くなるため、代わりにsession_keyを用いてアクセスします。ただし、session_keyが他人に奪われると不正にアクセスされる可能性があるため、利用が終わったらsession_keyを無効化するようにしましょう。以下の手順でsession_keyを無効化できます。
+
+```ruby
+del_session_req = Net::HTTP::Delete.new(uri + "/session/#{session_key}")
+del_session_res = https.request(del_session_req)
+del_session_res.body
+```
+
+以下のように 'code: 0' が返されていれば削除成功です。
+
+```
+irb(main):037:0> del_session_res.body
+=> "{\"code\":0}"
+```
+
+現状はsession_keyにはかなり長い時間の有効期間が与えられているようで、1日たってもアクセスできました (^^;;

metadata

@@ -0,0 +1,96 @@
+--- !ruby/object:Gem::Specification
+name: fluent-plugin-everysense
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Toyokazu Akiyama
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2016-03-08 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.10.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.10.0
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.10'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.10'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+description: Fluent Input/Output plugin for EverySense Framework
+email:
+- toyokazu@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- Gemfile
+- LICENSE
+- README.md
+- Rakefile
+- fluent-plugin-everysense.gemspec
+- lib/fluent/plugin/everysense_proxy.rb
+- lib/fluent/plugin/in_everysense.rb
+- lib/fluent/plugin/out_everysense.rb
+- tutorial/ja/json_over_http_api_tutorial.md
+homepage: https://github.com/toyokazu/fluent-plugin-everysense
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 2.0.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.5.1
+signing_key: 
+specification_version: 4
+summary: Fluent Input/Output plugin for EverySense Framework
+test_files: []