fluent-plugin-azuresearch 0.1.2 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 5e676e6a3c79df3f6064dc64e0a36c6f75909d23
-  data.tar.gz: a5c371a85bff5fab8abf3e80cd5e9eea35146de3
+  metadata.gz: 76f573ed22cba02133e8bebf6c70a5f70710c636
+  data.tar.gz: 8d5e301c6c5bf5f0ea36b890c21029cbfee95182
 SHA512:
-  metadata.gz: b9c867f9cca43eb788cfc9182f239c5d3dbcdfac863fcb800fbd604ec5194e02a286e1b9b5bc67ff5d87567631a03ec04693c6afd013ce9fa05af549500225f9
-  data.tar.gz: 1cebebbcfaf3fccda57d184f8d9fcaa0e2f1ea21dab0d59dc53d4a5728d3a29471e7d83b0d61cb1cceec3159677ac8d26eaa991c8234fd0e68b64a96f409153a
+  metadata.gz: 02b7facbaaf3dd9694fd146953c91638326d67c4c41508db7b4571f490145c201027c8e44f2b0d98e2b76cbe85c3148b8364879ece9cc0209906f3dfd264677c
+  data.tar.gz: 7478e66492a848353d8e3b0dce39a4e77b19ab5ed4dc212c8f5b003e67706fe99534a06be961454ec2d69f79c126e17abdafb289cb43d94b74c12f71d17ab2dc

data/ChangeLog.md

@@ -0,0 +1,17 @@
+## 0.2.0
+
+* Migrate to use fluentd v0.14 API - [PR#3](https://github.com/yokawasa/fluent-plugin-azuresearch/pull/3)
+* Support plugin specific log level - [PR#2](https://github.com/yokawasa/fluent-plugin-azuresearch/pull/2)
+* Added example script and configuration
+
+## 0.1.2
+
+* Change gem package dependency option for rest-client from add_development_dependency to add_dependency
+
+## 0.1.1
+
+* Security enhanced option: Added secret option to api_key
+
+## 0.1.0
+
+* Inital Release

data/README.md

@@ -20,6 +20,7 @@ To use Microsoft Azure Search, you must create an Azure Search service in the Az
 
     <match azuresearch.*>
         @type azuresearch
+        @log_level info
         endpoint   https://AZURE_SEARCH_ACCOUNT.search.windows.net
         api_key    AZURE_SEARCH_API_KEY
         search_index  messages
@@ -33,6 +34,8 @@ To use Microsoft Azure Search, you must create an Azure Search service in the Az
  * **column\_names (required)** - Column names in a target Azure search index. Each column needs to be separated by a comma.
  * **key\_names (optional)** - Default:nil. Key names in incomming record to insert. Each key needs to be separated by a comma. ${time} is placeholder for Time.at(time).strftime("%Y-%m-%dT%H:%M:%SZ"), and ${tag} is placeholder for tag. By default, **key\_names** is as same as **column\_names**
 
+[note] @log_level is a fluentd built-in parameter (optional) that controls verbosity of logging: fatal|error|warn|info|debug|trace (See also [Logging of Fluentd](http://docs.fluentd.org/articles/logging#log-level)) 
+
 ## Sample Configurations
 ### Case1 - column_names is as same as key_names 
 
@@ -197,6 +200,14 @@ Please don't forget that you need forward input configuration to receive the mes
 ## TODOs
  * Input validation for Azure Search - check total size of columns to add 
 
+## Change log
+* [Changelog](ChangeLog.md)
+
+## Links
+
+* http://yokawasa.github.io/fluent-plugin-azuresearch
+* https://rubygems.org/gems/fluent-plugin-azuresearch
+
 ## Contributing
 
 Bug reports and pull requests are welcome on GitHub at https://github.com/yokawasa/fluent-plugin-azuresearch.

data/VERSION

@@ -1 +1 @@
-0.1.2
+0.2.0

data/examples/createindex-messages.sh

@@ -0,0 +1,20 @@
+#!/bin/sh
+
+SERVICE_NAME='Azure Search Service Name'
+ADMIN_KEY='Azure Search API Admin Key'
+API_VER='2015-02-28-Preview'
+CONTENT_TYPE='application/json'
+URL="https://${SERVICE_NAME}.search.windows.net/indexes?api-version=${API_VER}"
+
+curl -s\
+ -H "Content-Type: ${CONTENT_TYPE}"\
+ -H "api-key: ${ADMIN_KEY}"\
+ -XPOST ${URL} -d'{
+    "name": "messages",
+    "fields": [
+        { "name":"id", "type":"Edm.String", "key": true, "searchable": false },
+        { "name":"user_name", "type":"Edm.String" },
+        { "name":"message", "type":"Edm.String", "filterable":false, "sortable":false, "facetable":false, "analyzer":"en.lucene" },
+        { "name":"created_at", "type":"Edm.DateTimeOffset", "facetable":false}
+    ]
+}'

data/examples/fluent.conf

@@ -0,0 +1,14 @@
+<source>
+    @type forward
+    @id forward_input
+</source>
+
+<match azuresearch.*>
+    @type azuresearch
+    @log_level info
+    endpoint   https://<AzureSearchServiceName>.search.windows.net
+    api_key    <Azure Search API Admin Key>
+    search_index  messages
+    column_names id,user_name,message,created_at
+    key_names postid,user,content,posttime
+</match>

data/examples/send-event.sh

@@ -0,0 +1,4 @@
+#!/bin/sh
+
+echo ' { "postid":"1", "user":"ladygaga", "content":"post by ladygaga", "posttime":"2017-05-20T00:00:00Z"}' | fluent-cat azuresearch.msg
+

data/fluent-plugin-azuresearch.gemspec

@@ -18,10 +18,9 @@ Gem::Specification.new do |gem|
   gem.test_files    = gem.files.grep(%r{^(test|gem|features)/})
   gem.require_paths = ["lib"]
 
-  gem.add_dependency "fluentd", [">= 0.10.58", "< 2"]
+  gem.add_dependency "fluentd", [">= 0.14.15", "< 2"]
   gem.add_dependency "rest-client"
   gem.add_development_dependency "bundler", "~> 1.11"
   gem.add_development_dependency "rake", "~> 10.0"
   gem.add_development_dependency "test-unit"
 end
-

data/lib/fluent/plugin/azuresearch/client.rb

@@ -1,4 +1,4 @@
-module Fluent
+module Fluent::Plugin
   module AzureSearch
     class Client
 
@@ -16,8 +16,8 @@ module Fluent
         end 
 
         def add_documents(index_name, documents, merge=true)
-            raise ConfigError, 'no index_name' if index_name.empty?
-            raise ConfigError, 'no documents' if documents.empty?
+            raise Fluent::ConfigError, 'no index_name' if index_name.empty?
+            raise Fluent::ConfigError, 'no documents' if documents.empty?
             action = merge ? 'mergeOrUpload' : 'upload' 
             for document in documents
                 document['@search.action'] = action

data/lib/fluent/plugin/out_azuresearch.rb

@@ -1,15 +1,17 @@
 # -*- coding: utf-8 -*-
 
-module Fluent
-  class AzureSearchOutput < BufferedOutput
-    Plugin.register_output('azuresearch', self)
+require 'fluent/plugin/output'
+require 'msgpack'
+require 'time'
+require 'fluent/plugin/azuresearch/client'
 
-    def initialize
-        super
-        require 'msgpack'
-        require 'time'
-        require 'fluent/plugin/azuresearch/client'
-    end
+module Fluent::Plugin
+  class AzureSearchOutput < Output
+    Fluent::Plugin.register_output('azuresearch', self)
+
+    helpers :compat_parameters
+
+    DEFAULT_BUFFER_TYPE = "memory"
 
     config_param :endpoint, :string,
                  :desc => "Azure Search Endpoint URL"
@@ -26,23 +28,29 @@ ${time} is placeholder for Time.at(time).strftime("%Y-%m-%dT%H:%M:%SZ"),
 and ${tag} is placeholder for tag
 DESC
 
+    config_section :buffer do
+        config_set_default :@type, DEFAULT_BUFFER_TYPE
+        config_set_default :chunk_keys, ['tag']
+    end
+
     def configure(conf)
+        compat_parameters_convert(conf, :buffer)
         super
-        raise ConfigError, 'no endpoint' if @endpoint.empty?
-        raise ConfigError, 'no api_key' if @api_key.empty?
-        raise ConfigError, 'no search_index' if @search_index.empty?
-        raise ConfigError, 'no column_names' if @column_names.empty?
-       
+        raise Fluent::ConfigError, 'no endpoint' if @endpoint.empty?
+        raise Fluent::ConfigError, 'no api_key' if @api_key.empty?
+        raise Fluent::ConfigError, 'no search_index' if @search_index.empty?
+        raise Fluent::ConfigError, 'no column_names' if @column_names.empty?
+
         @column_names = @column_names.split(',')
         @key_names = @key_names.nil? ? @column_names : @key_names.split(',')
-        raise ConfigError, 'NOT match keys number: column_names and key_names' \
+        raise Fluent::ConfigError, 'NOT match keys number: column_names and key_names' \
                 if @key_names.length != @column_names.length
     end
 
     def start
         super
         # start
-        @client=Fluent::AzureSearch::Client::new( @endpoint, @api_key )
+        @client=Fluent::Plugin::AzureSearch::Client::new( @endpoint, @api_key )
     end
 
     def shutdown
@@ -65,6 +73,14 @@ DESC
         [tag, time, values].to_msgpack
     end
 
+    def formatted_to_msgpack_binary?
+        true
+    end
+
+    def multi_workers_ready?
+        true
+    end
+
     def write(chunk)
         documents = []
         chunk.msgpack_each do |tag, time, values|
@@ -79,7 +95,7 @@ DESC
             res = @client.add_documents(@search_index, documents)
             puts res
         rescue Exception => ex
-            $log.fatal "UnknownError: '#{ex}'"
+            log.fatal "UnknownError: '#{ex}'"
                           + ", data=>" + (documents.to_json).to_s
         end
     end

data/test/helper.rb

@@ -23,7 +23,10 @@ unless ENV.has_key?('VERBOSE')
   $log = nulllogger
 end
 
+require 'fluent/test/driver/output'
+require 'fluent/test/helpers'
 require 'fluent/plugin/out_azuresearch'
 
 class Test::Unit::TestCase
+  include Fluent::Test::Helpers
 end

data/test/plugin/test_azuresearch.rb

@@ -18,57 +18,57 @@ class AzureSearchOutputTest < Test::Unit::TestCase
   #   utc
   # ]
 
-  def create_driver(conf = CONFIG, tag='azuresearch.test')
-    Fluent::Test::BufferedOutputTestDriver.new(Fluent::AzureSearchOutput, tag).configure(conf)
+  def create_driver(conf = CONFIG)
+    Fluent::Test::Driver::Output.new(Fluent::Plugin::AzureSearchOutput).configure(conf)
   end
 
   def test_configure
-    #### set configurations
-    # d = create_driver %[
-    #   path test_path
-    #   compress gz
-    # ]
-    #### check configurations
-    # assert_equal 'test_path', d.instance.path
-    # assert_equal :gz, d.instance.compress
+    d = create_driver
+
+    assert_equal 'https://AZURE_SEARCH_ACCOUNT.search.windows.net', d.instance.endpoint
+    assert_equal 'AZURE_SEARCH_API_KEY', d.instance.api_key
+    assert_equal 'messages', d.instance.search_index
+    assert_equal ["id", "user_name", "message", "tag", "created_at"], d.instance.column_names
   end
 
   def test_format
     d = create_driver
 
-    # time = Time.parse("2011-01-02 13:14:15 UTC").to_i
-    # d.emit({"a"=>1}, time)
-    # d.emit({"a"=>2}, time)
-
-    # d.expect_format %[2011-01-02T13:14:15Z\ttest\t{"a":1}\n]
-    # d.expect_format %[2011-01-02T13:14:15Z\ttest\t{"a":2}\n]
+    time = event_time("2011-01-02 13:14:15 UTC")
+    d.run(default_tag: 'documentdb.test') do
+      d.feed(time, {"a"=>1})
+      d.feed(time, {"a"=>2})
+    end
 
-    # d.run
+    # assert_equal EXPECTED1, d.formatted[0]
+    # assert_equal EXPECTED2, d.formatted[1]
   end
 
   def test_write
     d = create_driver
 
-    time = Time.parse("2016-01-28 13:14:15 UTC").to_i
-    d.emit(
-        {
-            "postid" => "10001",
-            "user"=> "ladygaga",
-            "content" => "post by ladygaga",
-            "tag" => "azuresearch.msg",
-            "posttime" =>"2016-01-31T00:00:00Z"
-        }, time)
+    time = event_time("2016-01-28 13:14:15 UTC")
+    data = d.run(default_tag: 'azuresearch.test') do
+      d.feed(
+          time,
+          {
+              "postid" => "10001",
+              "user"=> "ladygaga",
+              "content" => "post by ladygaga",
+              "tag" => "azuresearch.msg",
+              "posttime" =>"2016-01-31T00:00:00Z"
+          })
 
-    d.emit(
-        {
-            "postid" => "10002",
-            "user"=> "katyperry",
-            "content" => "post by katyperry",
-            "tag" => "azuresearch.msg",
-            "posttime" => "2016-01-31T00:00:00Z"
-        }, time)
-
-    data = d.run
+      d.feed(
+          time,
+          {
+              "postid" => "10002",
+              "user"=> "katyperry",
+              "content" => "post by katyperry",
+              "tag" => "azuresearch.msg",
+              "posttime" => "2016-01-31T00:00:00Z"
+          })
+    end
     puts data
     # ### FileOutput#write returns path
     # path = d.run
@@ -76,4 +76,3 @@ class AzureSearchOutputTest < Test::Unit::TestCase
     # assert_equal expect_path, path
   end
 end
-

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-azuresearch
 version: !ruby/object:Gem::Version
-  version: 0.1.2
+  version: 0.2.0
 platform: ruby
 authors:
 - Yoichi Kawasaki
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-02-20 00:00:00.000000000 Z
+date: 2017-05-31 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -16,7 +16,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.10.58
+        version: 0.14.15
     - - "<"
       - !ruby/object:Gem::Version
         version: '2'
@@ -26,7 +26,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.10.58
+        version: 0.14.15
     - - "<"
       - !ruby/object:Gem::Version
         version: '2'
@@ -94,11 +94,14 @@ extensions: []
 extra_rdoc_files: []
 files:
 - ".gitignore"
-- ChangeLog
+- ChangeLog.md
 - Gemfile
 - README.md
 - Rakefile
 - VERSION
+- examples/createindex-messages.sh
+- examples/fluent.conf
+- examples/send-event.sh
 - fluent-plugin-azuresearch.gemspec
 - lib/fluent/plugin/azuresearch/client.rb
 - lib/fluent/plugin/out_azuresearch.rb
@@ -124,7 +127,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.5.1
+rubygems_version: 2.5.2
 signing_key: 
 specification_version: 4
 summary: Azure Search output plugin for Fluentd

data/ChangeLog

@@ -1,11 +0,0 @@
-Release 0.1.2 - 2016/02/20
-
-	* Change gem package dependency option for rest-client from add_development_dependency to add_dependency
-
-Release 0.1.1 - 2016/02/20
-
-	* Security enhanced option: Added secret option to api_key
-
-Release 0.1.0 - 2016/02/11
-
-	* Inital Release