fluent-plugin-jubatus 0.0.1 → 0.0.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 483f2c695c7f2a74eb89c1e6274b7d17234d5dfc
-  data.tar.gz: 33367a26b845d3c9373da9351819da2a81192c31
+  metadata.gz: 6b3fcff4427e5a827c447714310649fc457706d1
+  data.tar.gz: ff03960d8a31f7fe5b28cdb747a8f186072cd7df
 SHA512:
-  metadata.gz: 01a003c9b9a64c50bdb80909f04daf90832026395075e38af683ccfcda44e53dff4527f82e966a88b29ab8edd7a7baec62d15df62e1e03525b194744287de794
-  data.tar.gz: af137d2b712e41d8dda0610bb8c262a4a997b6c6128da152ef63dfe3552dd299f435d4ebf9b6c62b9bf2403479f4adc6c2ef46dd64d96254a81eae54b3c0835f
+  metadata.gz: ccadad217cf603b02ceb91db22b33c040faa8ebcade2a9411b73cfa5047d74364c6a8fc720c39b4fb65177e5ad210752a7fa943b567f302c8986a15152674191
+  data.tar.gz: 1a4cdfb2aaab487b6779ea7e22d6b4478d77e30b3de19667065bca54767f85ab76e8cbd699c071c6b10a39db232bb05d339012aeb4a1f73627daf1f1bec175cb

data/README.md

@@ -1,4 +1,4 @@
-# Fluent::Plugin::Jubatus
+# Fluent::Plugin::Jubatus, a plugin for [Fluentd](http://fluentd.org)
 
 fluentd pluing for jubatus
 

data/Rakefile

@@ -1 +1,6 @@
 require "bundler/gem_tasks"
+
+require 'rspec/core/rake_task'
+
+RSpec::Core::RakeTask.new(:spec)
+

data/fluent-plugin-jubatus.gemspec

@@ -3,7 +3,7 @@ $:.push File.expand_path('../lib', __FILE__)
 
 Gem::Specification.new do |gem|
   gem.name          = "fluent-plugin-jubatus"
-  gem.version       = "0.0.1"
+  gem.version       = "0.0.2"
   gem.authors       = ["MATSUMOTO Katsuyoshi"]
   gem.email         = ["matsumoto.katsuyoshi+rubygems@gmail.com"]
   gem.description   = %q{Jubatus output plugin for fluentd}
@@ -19,4 +19,6 @@ Gem::Specification.new do |gem|
   gem.add_dependency 'jubatus'
   gem.add_development_dependency "simplecov"
   gem.add_development_dependency "rake"
+  gem.add_development_dependency "pry"
+  gem.add_development_dependency "rspec"
 end

data/lib/fluent/plugin/jubatus.rb

@@ -0,0 +1,99 @@
+require 'jubatus/anomaly/client'
+require 'jubatus/anomaly/types'
+require 'jubatus/classifier/client'
+require 'jubatus/classifier/types'
+require 'jubatus/clustering/client'
+require 'jubatus/clustering/types'
+require 'jubatus/recommender/client'
+require 'jubatus/recommender/types'
+
+class FluentdJubatus
+  def initialize(type, host, port, name='')
+    @jubatus = case type
+      when /anomaly/i
+        Jubatus::Anomaly::Client::Anomaly.new(host, port, name)
+      when /classifier/i
+        Jubatus::Classifier::Client::Classifier.new(host, port, name)
+      when /clustering/i
+        Jubatus::Clustering::Client::Clustering.new(host, port, name)
+      when /recommender/i
+        Jubatus::Recommender::Client::Recommender.new(host, port, name)
+      end
+  end
+
+  def set_datum(data, keys)
+    datum = {}
+    data.each do |k,v|
+      datum[k.to_s] = v.to_f if keys[:num].include?(k.to_s)
+      datum[k.to_s] = v.to_s if keys[:str].include?(k.to_s)
+    end
+    Jubatus::Common::Datum.new(datum)
+  end
+
+  def analyze(type, datum, num = 10)
+    case type
+    when /anomaly/i
+      @jubatus.calc_score(datum)
+    when /classifier/i
+      @jubatus.classify([datum])
+    when /clustering/i
+      @jubatus.get_nearest_members(datum)
+    when /recommender/i
+      @jubatus.similar_row_from_datum(datum, num)
+    end
+  end
+
+  def close
+    @jubatus.get_client.close
+  end
+
+  def learn(type, datum, key = nil)
+    # todo
+  end
+
+  def self.fix_result(type, result)
+    case type
+    when /anomaly/i
+      fix_anomaly(result)
+    when /classifier/i
+      fix_classifier(result)
+    when /clustering/i
+      fix_clustering(result)
+    when /recommender/i
+      fix_clustering(result)
+    end
+  end
+
+  private
+  def self.fix_anomaly(result)
+    result
+  end
+
+  def self.fix_classifier(results)
+    r = []
+    results.each do |result|
+      est = {}
+      result.each do |res|
+        est[res.label] = res.score
+      end
+      r << est
+    end
+    r
+  end
+
+  def self.fix_clustering(results)
+    r = {}
+    results.each do |result|
+      r[result.id] = result.score
+    end
+    r
+  end
+
+  def self.fix_recommender(results)
+    result = {}
+    results.each do |r|
+      result[r.id] = r.score
+    end
+    result
+  end
+end

data/lib/fluent/plugin/out_jubatus.rb

@@ -12,18 +12,14 @@ class JubatusOutput < Output
 
   def initialize
     super
-    require 'jubatus/classifier/client'
-    require 'jubatus/classifier/types'
-    require 'jubatus/anomaly/client'
-    require 'jubatus/anomaly/types'
-    require 'jubatus/recommender/client'
-    require 'jubatus/recommender/types'
+    require 'fluent/plugin/jubatus'
   end
 
   def configure(conf)
     super
-    @str = @str_keys.split(/,/).map{|str| str.strip}
-    @num = @num_keys.split(/,/).map{|num| num.strip}
+    str = @str_keys.split(/,/).map{|key| key.strip }
+    num = @num_keys.split(/,/).map{|key| key.strip }
+    @keys = {str: str, num: num}
   end
 
   def start
@@ -36,7 +32,7 @@ class JubatusOutput < Output
 
   def emit(tag, es, chain)
     es.each do |time, record|
-      result = result_format(jubatus_run(record))
+      result = result_format(@client_api, jubatus_run(record))
       Engine.emit(@tag, time, result)
     end
 
@@ -45,91 +41,30 @@ class JubatusOutput < Output
 
   private
   def jubatus_run(data)
-    datum = set_datum(data)
-    if @learn_analyze =~ /^analyze$/i
-      analyze(datum)
-    elsif @learn_analyze =~ /^train$/i
-      update(datum)
-    end
-  end
-
-  def analyze(datum)
-    case
-    when @client_api =~ /^classif(y|ier)$/i
-      classify(datum)
-    when @client_api =~ /^anomaly$/i
-      anomaly(datum)
-    when @client_api =~ /^recommender/i
-      recommend(datum)
-    end
-  rescue => e
-    e
-  end
-
-  def update(datum)
-    jubatus = Jubatus::Classifier::Client::Classifier.new(@host, @port.to_i)
-    jubatus.train(@name, [datum])
-    jubatus.get_client.close
-  rescue => e
-    e
-  end
-
-  def set_datum(data)
-    str = []
-    num = []
-    data.each do |key, value|
-      str << [key, value.to_s] if @str.include?(key)
-      num << [key, value.to_f] if @num.include?(key)
-    end
-    case
-    when @client_api =~ /^classif(y|ier)$/i
-      Jubatus::Classifier::Datum.new(str, num)
-    when @client_api =~ /^anomaly$/i
-      Jubatus::Anomaly::Datum.new(str, num)
-    end
-  end
-
-  def result_format(data)
-    case
-    when @client_api =~ /^classifier$/i
-      result_classify(data)
-    when @client_api =~ /^anomaly$/i
-      result_anomaly(data)
-    end
-  end
-
-  def classify(datum)
-    jubatus = Jubatus::Classifier::Client::Classifier.new(@host, @port.to_i)
-    result = jubatus.classify(@name, [datum])
-    jubatus.get_client.close()
-    result
-  rescue => e
-    e
-  end
-
-  def anomaly(datum)
-    jubatus = Jubatus::Anomaly::Client::Anomaly.new(@host, @port.to_i)
-    result = jubatus.add(@name, datum)
-    jubatus.get_client.close()
-    result
-  rescue => e
-    e
-  end
-
-  def result_classify(data)
-    result = {}
-    data.map do |datum|
-      datum.map do |est|
-        result[est[0]] = est[1]
+    count = 0
+    jubatus = FluentdJubatus.new(@client_api, @host, @port, @name)
+    begin
+      datum = jubatus.set_datum(data, @keys)
+      case @learn_analyze
+      when /^analyze$/i
+        jubatus.analyze(@client_api, datum)
+      when /^learn$/i
+        # todo
+        # jubatus.learn(@client_api, datum)
       end
+    rescue MessagePack::RPC::ConnectionTimeoutError => e
+      jubatus.close
+      count += 1
+      raise e if count > 10
+      sleep 0.1
+      retry
+    rescue => e
+      e
     end
-    result
   end
 
-  def result_anomaly(data)
-    value = data[1]
-    value = data[1].to_s if data[1].to_s == "Infinity"
-    { id: data[0], value: value }
+  def result_format(type, result)
+    FluentdJubatus.fix_result(type, result)
   end
 end
 end

data/spec/jubatus_spec.rb

@@ -0,0 +1,117 @@
+require File.expand_path(__dir__ + '/spec_helper')
+
+describe FluentdJubatus do
+  let(:fluent_conf) { {host: '127.0.0.1', port: 9199, name: ''} }
+  let(:data) { {a: 'a', b: '1', c: '1.0', d: '-1.0', e: 'f', f: '1,', g: 'str'} }
+  let(:keys) { {str:['a','e'], num:['b','c','d']} }
+  let(:label) { 'fluentd' }
+  let(:jubatus) {
+    described_class.new(type, fluent_conf[:host], fluent_conf[:port], fluent_conf[:name])
+  }
+  let(:datum) { jubatus.set_datum(data, keys) }
+  let(:raw_jubatus) { jubatus.instance_variable_get(:@jubatus) }
+  let(:result) { described_class.fix_result(type, jubatus.analyze(type, datum)) }
+  let(:log) { File.expand_path(__dir__) }
+  # This test run only in ubuntu or debian using deb package
+  let(:ubuntu) { "/opt/jubatus/share/jubatus/example/config/#{type}" }
+
+  def startup(path, jubatus_type: type, config_path: path, jubatus_log: log, jubatus_port: fluent_conf[:port])
+    @pid = spawn("juba#{jubatus_type} -f #{config_path} -l #{jubatus_log} -p #{jubatus_port}")
+    sleep 0.1
+  end
+
+  def stop_jubatus
+    Process.kill(9, @pid)
+    Process.wait(@pid)
+    Dir.glob("./spec/juba#{type}.*").each do |f|
+      File.delete(File.expand_path(f))
+    end
+  end
+
+  context 'anomaly' do
+    let(:type) { 'anomaly' }
+
+    before do
+      startup(ubuntu+'/lof.json')
+      raw_jubatus.update(label, datum)
+    end
+
+    after do
+      stop_jubatus
+    end
+
+    it 'analyze' do
+      expect(jubatus.analyze(type, datum)).to eq(1.to_f)
+    end
+
+    it 'fix results' do
+      expect(result).to eq(1.to_f)
+    end
+  end
+
+  context 'classifier' do
+    let(:type) { 'classifier' }
+
+    before do
+      startup(ubuntu+'/arow.json')
+      raw_jubatus.train([[label,datum]])
+    end
+
+    after do
+      stop_jubatus
+    end
+
+    it 'analyze' do
+      expect(jubatus.analyze(type, datum).first.first.label).to eq(label)
+    end
+
+    it 'fix results' do
+      expect(result.size).to eq(1)
+    end
+  end
+
+  context 'clustering' do
+    let(:type) { 'clustering' }
+
+    before do
+      startup(ubuntu+'/kmeans.json')
+      1000.times do
+        raw_jubatus.push([datum])
+      end
+    end
+
+    after do
+      stop_jubatus
+    end
+
+    it 'analyze' do
+      expect(jubatus.analyze(type, datum)).to be_true
+    end
+
+    it 'fix results' do
+      pending('which do you need results in nearest center point, or nearest cluster members?')
+      expect(result.size).to eq(1)
+    end
+  end
+
+  context 'recommender' do
+    let(:type) { 'recommender' }
+
+    before do
+      startup(ubuntu+'/inverted_index.json')
+      raw_jubatus.update_row(label,datum)
+    end
+
+    after do
+      stop_jubatus
+    end
+
+    it 'analyze' do
+      expect(jubatus.analyze(type, datum).first.score).to eq(1)
+    end
+
+    it 'fix results' do
+      expect(result.size).to eq(1)
+    end
+  end
+end

data/spec/out_jubatus_spec.rb

@@ -0,0 +1,53 @@
+require File.expand_path(__dir__ + '/spec_helper')
+
+describe Fluent::JubatusOutput do
+  let(:fluentd){ Fluent::Test::TestDriver.new(described_class) }
+  let(:config){
+    %[
+      type jubatus
+      client_api classifier
+      host 127.0.0.1
+      port 9199
+      str_keys first, second, third
+      num_keys ichi, ni, san
+      learn_analyze analyze
+    ]
+  }
+
+  context 'set config params' do
+    let(:conf){ fluentd.configure(config).instance }
+    it 'string keys' do
+      expect(conf.instance_variable_get(:@keys)[:str]).to eq(['first','second','third'])
+    end
+
+    it 'number keys' do
+      expect(conf.instance_variable_get(:@keys)[:num]).to eq(['ichi','ni','san'])
+    end
+
+
+    it 'client api' do
+      expect(conf.client_api).to eq('classifier')
+    end
+
+    it 'host' do
+      expect(conf.host).to eq('127.0.0.1')
+    end
+
+    it 'port' do
+      expect(conf.port).to eq('9199')
+    end
+
+    it 'learn or analyze' do
+      expect(conf.learn_analyze).to eq('analyze')
+    end
+  end
+
+  context 'fluentd' do
+    let(:classifier){ fluend.configure(config) }
+    let(:data){ {first: '10', second: 10, third: "abcd", ichi: 1, ni: "-2", san: '3.0'} }
+    it 'emit' do
+      
+    end
+  end
+end
+

data/spec/spec_helper.rb

@@ -0,0 +1,10 @@
+require 'simplecov'
+
+SimpleCov.start do
+  add_filter 'spec'
+end
+
+$:.unshift File.expand_path(File.join(__dir__, '..', 'lib'))
+require 'fluent/test'
+require 'fluent/plugin/out_jubatus'
+require 'fluent/plugin/jubatus'

metadata

@@ -1,69 +1,97 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-jubatus
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.0.2
 platform: ruby
 authors:
 - MATSUMOTO Katsuyoshi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-10-08 00:00:00.000000000 Z
+date: 2014-04-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: jubatus
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: simplecov
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 description: Jubatus output plugin for fluentd
@@ -73,13 +101,17 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- .gitignore
+- ".gitignore"
 - Gemfile
 - LICENSE.txt
 - README.md
 - Rakefile
 - fluent-plugin-jubatus.gemspec
+- lib/fluent/plugin/jubatus.rb
 - lib/fluent/plugin/out_jubatus.rb
+- spec/jubatus_spec.rb
+- spec/out_jubatus_spec.rb
+- spec/spec_helper.rb
 homepage: https://github.com/katsyoshi/fluent-plugin-jubatus
 licenses: []
 metadata: {}
@@ -89,18 +121,21 @@ require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.0.3
+rubygems_version: 2.2.2
 signing_key: 
 specification_version: 4
 summary: Jubatus output plugin for fluentd
-test_files: []
+test_files:
+- spec/jubatus_spec.rb
+- spec/out_jubatus_spec.rb
+- spec/spec_helper.rb