jubatus 0.4.1 → 0.5.0

data/.gitignore

@@ -1,4 +1,6 @@
-lib/
+lib/jubatus/*
+!lib/jubatus/common
+!lib/jubatus/common.rb
 *.gem
 *.rbc
 .bundle
@@ -14,6 +16,7 @@ test/version_tmp
 tmp
 *.model.js
 *.json
+*~
 
 # YARD artifacts
 .yardoc

data/README.rst

@@ -4,7 +4,7 @@ Jubatus Ruby Client
 Template of Jubatus client in Ruby.
 See `RubyGems <http://rubygems.org/gems/jubatus>`_ for the latest release.
 
-All codes are generated by `MessagePack IDL <https://github.com/msgpack/msgpack-haskell/tree/master/msgpack-idl>`_ .
+All codes are generated by ``jenerator`` .
 
 
 Install

data/Rakefile

@@ -14,6 +14,14 @@ task :test do |test|
   end
 end
 
+task :integration_test do |test|
+  Rake::TestTask.new(:integration_test) do |t|
+    t.libs << 'lib' << 'integration_test'
+    t.test_files = FileList['integration_test/**/*.rb']
+    t.verbose = true
+  end
+end
+
 # for Ruby 1.8
 task :coverage do |coverage|
   system("rcov -o test/coverage -I lib:test  --exclude . --include-file lib/jubatus --aggregate coverage.info test/**/*.rb")

data/VERSION

@@ -1 +1 @@
-0.4.1
+0.5.0

data/{test → integration_test}/jubatus_test/anomaly/test.rb

@@ -5,7 +5,6 @@ require 'test/unit'
 require 'json'
 
 require 'jubatus/anomaly/client'
-require 'jubatus/anomaly/types'
 require 'jubatus_test/test_util'
 
 class AnomalyTest < Test::Unit::TestCase
@@ -31,7 +30,7 @@ class AnomalyTest < Test::Unit::TestCase
        "reverse_nearest_neighbor_num" => 30,
        "method" => "euclid_lsh",
        "parameter" => {
-         "lsh_num" => 8,
+         "hash_num" => 8,
          "table_num" => 16,
          "probe_num" => 64,
          "bin_width" => 10.0,
@@ -43,7 +42,7 @@ class AnomalyTest < Test::Unit::TestCase
 
     TestUtil.write_file("config_anomaly.json", @config.to_json)
     @srv = TestUtil.fork_process("anomaly", PORT, "config_anomaly.json")
-    @cli = Jubatus::Anomaly::Client::Anomaly.new(HOST, PORT)
+    @cli = Jubatus::Anomaly::Client::Anomaly.new(HOST, PORT, "name")
   end
 
   def teardown
@@ -55,55 +54,55 @@ class AnomalyTest < Test::Unit::TestCase
   end
 
   def test_clear_row
-    d = Jubatus::Anomaly::Datum.new([], [])
-    (row_id, score) = @cli.add("name", d)
-    assert_equal(true, @cli.clear_row("name", row_id))
+    d = Jubatus::Common::Datum.new
+    res = @cli.add(d)
+    assert_equal(true, @cli.clear_row(res.id))
   end
 
   def test_add
-    d = Jubatus::Anomaly::Datum.new([], [])
-    (row_id, score) = @cli.add("name", d)
+    d = Jubatus::Common::Datum.new
+    res = @cli.add(d)
   end
 
   def test_update
-    d = Jubatus::Anomaly::Datum.new([], [])
-    (row_id, score) = @cli.add("name", d)
-    d = Jubatus::Anomaly::Datum.new([], [['val', 3.1]])
-    score = @cli.update("name", row_id, d)
+    d = Jubatus::Common::Datum.new
+    res = @cli.add(d)
+    d = Jubatus::Common::Datum.new('val' => 3.1)
+    score = @cli.update(res.id, d)
   end
 
   def test_clear
-    assert_equal(true, @cli.clear("name"))
+    assert_equal(true, @cli.clear)
   end
 
   def test_calc_score
-    d = Jubatus::Anomaly::Datum.new([], [['val', 1.1]])
-    (row_id, score) = @cli.add("name", d)
-    d = Jubatus::Anomaly::Datum.new([], [['val', 3.1]])
-    score = @cli.calc_score("name", d)
+    d = Jubatus::Common::Datum.new('val' => 1.1)
+    res = @cli.add(d)
+    d = Jubatus::Common::Datum.new('val' => 3.1)
+    score = @cli.calc_score(d)
   end
 
   def test_get_all_rows
-    @cli.get_all_rows("name")
+    @cli.get_all_rows
   end
 
   def test_get_config
-    config = @cli.get_config("name")
+    config = @cli.get_config
     assert_equal(JSON.parse(config), @config)
   end
 
   def test_save
-    assert_equal(true, @cli.save("name", "anomaly.save_test.model"))
+    assert_equal(true, @cli.save("anomaly.save_test.model"))
   end
 
   def test_load
     model_name = "anomaly.load_test.model"
-    @cli.save("name", model_name)
-    assert_equal(true, @cli.load("name", model_name))
+    @cli.save(model_name)
+    assert_equal(true, @cli.load(model_name))
   end
 
   def test_get_status
-    @cli.get_status("name")
+    @cli.get_status
   end
 
 end

data/{test → integration_test}/jubatus_test/classifier/test.rb

@@ -5,7 +5,6 @@ require 'test/unit'
 require 'json'
 
 require 'jubatus/classifier/client'
-require 'jubatus/classifier/types'
 require 'jubatus_test/test_util'
 
 class ClassifierTest < Test::Unit::TestCase
@@ -33,7 +32,7 @@ class ClassifierTest < Test::Unit::TestCase
 
     TestUtil.write_file("config_classifier.json", @config.to_json)
     @srv = TestUtil.fork_process("classifier", PORT, "config_classifier.json")
-    @cli = Jubatus::Classifier::Client::Classifier.new(HOST, PORT)
+    @cli = Jubatus::Classifier::Client::Classifier.new(HOST, PORT, "name")
   end
 
   def teardown
@@ -45,38 +44,34 @@ class ClassifierTest < Test::Unit::TestCase
   end
 
   def test_get_config
-    config = @cli.get_config("name")
+    config = @cli.get_config
     assert_equal(JSON.parse(config), @config)
   end
 
   def test_train
-    string_values = [["key1", "val1"], ["key2", "val2"]]
-    num_values = [["key1", 1.0], ["key2", 2.0]]
-    d = Jubatus::Classifier::Datum.new(string_values, num_values)
+    d = Jubatus::Common::Datum.new("skey1" => "val1", "skey2" => "val2", "nkey1" => 1.0, "nkey2" => 2.0)
     data = [["label", d]]
-    assert_equal(@cli.train("name", data), 1)
+    assert_equal(@cli.train(data), 1)
   end
 
   def test_classify
-    string_values = [["key1", "val1"], ["key2", "val2"]]
-    num_values = [["key1", 1.0], ["key2", 2.0]]
-    d = Jubatus::Classifier::Datum.new(string_values, num_values)
+    d = Jubatus::Common::Datum.new("skey1" => "val1", "skey2" => "val2", "nkey1" => 1.0, "nkey2" => 2.0)
     data = [d]
-    result = @cli.classify("name", data)
+    result = @cli.classify(data)
   end
 
   def test_save
-    assert_equal(@cli.save("name", "classifier.save_test.model"), true)
+    assert_equal(@cli.save("classifier.save_test.model"), true)
   end
 
   def test_load
     model_name = "classifier.load_test.model"
-    @cli.save("name", model_name)
-    assert_equal(@cli.load("name", model_name), true)
+    @cli.save(model_name)
+    assert_equal(@cli.load(model_name), true)
   end
 
   def test_get_status
-    @cli.get_status("name")
+    @cli.get_status
   end
 
 end

data/integration_test/jubatus_test/clustering/test.rb

@@ -0,0 +1,133 @@
+#!/usr/bin/env ruby
+
+require 'test/unit'
+
+require 'json'
+
+require 'jubatus/clustering/client'
+require 'jubatus_test/test_util'
+
+class ClusteringTest < Test::Unit::TestCase
+  HOST = "127.0.0.1"
+  PORT = 23008
+  TIMEOUT = 10
+
+  def setup
+    @config = {
+      "method" => "kmeans",
+      "converter" => {
+        "string_filter_types" => {},
+        "string_filter_rules" => [],
+        "num_filter_types" => {},
+        "num_filter_rules" => [],
+        "string_types" => {},
+        "string_rules" => [
+                           { "key" => "*", "type" => "str", "sample_weight" => "bin", "global_weight" => "bin" }
+                          ],
+        "num_types" => {},
+        "num_rules" => [
+                        { "key" => "*", "type" => "num" }
+                       ]
+      },
+      "parameter" => {
+        "k" => 10,
+        "compressor_method" => "simple",
+        "bucket_size" => 1,
+        "compressed_bucket_size" => 1,
+        "bicriteria_base_size" => 10,
+        "bucket_length" => 1,
+        "forgetting_factor" => 0,
+        "forgetting_threshold" => 0.5
+      }
+    }
+
+    TestUtil.write_file("config_clustering.json", @config.to_json)
+    @srv = TestUtil.fork_process("clustering", PORT, "config_clustering.json")
+    @cli = Jubatus::Clustering::Client::Clustering.new(HOST, PORT, "name")
+  end
+
+  def teardown
+    TestUtil.kill_process(@srv)
+  end
+
+  def test_get_client
+    assert_instance_of( MessagePack::RPC::Client, @cli.get_client )
+  end
+
+  def test_get_config
+    config = @cli.get_config
+    assert_equal(JSON.parse(config), @config)
+  end
+
+  def test_save
+    assert_equal(true, @cli.save("clustering.save_test.model"))
+  end
+
+  def test_load
+    model_name = "clustering.load_test.model"
+    @cli.save(model_name)
+    assert_equal(true, @cli.load(model_name))
+  end
+
+  def test_get_status
+    @cli.get_status
+  end
+
+  def test_push
+    d = Jubatus::Common::Datum.new
+    res = @cli.push([d])
+  end
+
+  def test_get_revision
+    r = @cli.get_revision
+    assert_instance_of(Fixnum, r)
+  end
+
+  def test_get_core_members
+    d = Jubatus::Common::Datum.new
+    @cli.push([d])
+    res = @cli.get_core_members()
+    assert_equal(10, res.length)
+    assert_equal(1, res[0].length)
+    assert_instance_of(Jubatus::Clustering::WeightedDatum, res[0][0])
+  end
+
+  def test_k_center
+    for i in 0..99
+      d = Jubatus::Common::Datum.new({"nkey1" => i, "nkey2" => -i})
+      @cli.push([d])
+    end
+    res = @cli.get_k_center()
+    assert_equal(10, res.length)
+    assert_instance_of(Jubatus::Common::Datum, res[0])
+  end
+
+  def test_nearest_center
+    for i in 0..99
+      d = Jubatus::Common::Datum.new({"nkey1" => i, "nkey2" => -i})
+      @cli.push([d])
+    end
+    q = Jubatus::Common::Datum.new({"nkey1" => 2.0, "nkey2" => 1.0})
+    res = @cli.get_nearest_center(q)
+    assert_instance_of(Jubatus::Common::Datum, res)
+  end
+
+  def test_nearest_center
+    for i in 0..99
+      d = Jubatus::Common::Datum.new({"nkey1" => i, "nkey2" => -i})
+      @cli.push([d])
+    end
+    q = Jubatus::Common::Datum.new({"nkey1" => 2.0, "nkey2" => 1.0})
+    res = @cli.get_nearest_center(q)
+    assert_instance_of(Jubatus::Common::Datum, res)
+  end
+
+  def test_nearest_members
+    d = Jubatus::Common::Datum.new({"nkey1" => 1.0, "nkey2" => 1.0})
+    q = Jubatus::Common::Datum.new({"nkey1" => 2.0, "nkey2" => 1.0})
+    @cli.push([d])
+    res = @cli.get_nearest_members(q)
+    assert_instance_of(Jubatus::Clustering::WeightedDatum, res[0])
+  end
+
+end

data/integration_test/jubatus_test/error/test.rb

@@ -0,0 +1,51 @@
+require 'json'
+require 'test/unit'
+require 'jubatus/common'
+require 'jubatus_test/test_util'
+
+class ErrorTest < Test::Unit::TestCase
+  HOST = "127.0.0.1"
+  PORT = 23000
+  TIMEOUT = 10
+
+  def setup
+    @config = {
+        "method" => "AROW",
+        "converter" => {
+            "string_filter_types" => {},
+            "string_filter_rules" => [],
+            "num_filter_types" => {},
+            "num_filter_rules" => [],
+            "string_types" => {},
+            "string_rules" => [],
+            "num_types" => {},
+            "num_rules" => []
+        },
+        "parameter" => {
+            "regularization_weight" => 1.001
+        }
+    }
+
+    TestUtil.write_file("config_for_error.json", @config.to_json)
+    @srv = TestUtil.fork_process("classifier", PORT, "config_for_error.json")
+    cli = MessagePack::RPC::Client.new(HOST, PORT)
+    @cli = Jubatus::Common::Client.new(cli, "name")
+  end
+
+  def teardown
+    TestUtil.kill_process(@srv)
+  end
+
+  def test_unknownmethod
+    assert_raise(Jubatus::Common::UnknownMethod) {
+      @cli.call("unknown_method", [], nil, [])
+    }
+  end
+
+  def test_typemismatch
+    # 'train' method requires list of labeled datum, but this test sends a string.
+    assert_raise(Jubatus::Common::TypeMismatch) {
+      @cli.call("train", [""], nil, [Jubatus::Common::TString.new])
+    }
+  end
+end

data/{test → integration_test}/jubatus_test/graph/test.rb

@@ -6,7 +6,6 @@ require 'jubatus_test/test_util'
 require 'json'
 
 require 'jubatus/graph/client'
-require 'jubatus/graph/types'
 
 class GraphTest < Test::Unit::TestCase
   HOST = "127.0.0.1"
@@ -24,7 +23,7 @@ class GraphTest < Test::Unit::TestCase
 
     TestUtil.write_file("config_graph.json", @config.to_json)
     @srv = TestUtil.fork_process("graph", PORT, "config_graph.json")
-    @cli = Jubatus::Graph::Client::Graph.new(HOST, PORT)
+    @cli = Jubatus::Graph::Client::Graph.new(HOST, PORT, "name")
   end
 
   def teardown
@@ -38,56 +37,42 @@ class GraphTest < Test::Unit::TestCase
   def test_node_info
     edge_query = [["a", "b"], ["c", "d"], ["e", "f"]]
     node_query = [["0", "1"], ["2", "3"]]
-    p = Jubatus::Graph::Preset_query.new(edge_query, node_query)
+    p = Jubatus::Graph::PresetQuery.new(edge_query, node_query)
     in_edges = [0, 0]
     out_edges = [0, 0]
     Jubatus::Graph::Node.new(p, in_edges, out_edges)
   end
 
   def test_create_node
-    name = "name"
-    @cli.clear(name)
-    nid = @cli.create_node("sample_node")
+    @cli.clear
+    nid = @cli.create_node
     assert_equal(nid.to_i.to_s, nid)
   end
 
   def test_remove_node
-    name = "name"
-    @cli.clear(name)
-    nid = @cli.create_node(name)
-    assert_equal(@cli.remove_node(name, nid), true)
+    @cli.clear
+    nid = @cli.create_node
+    assert_equal(@cli.remove_node(nid), true)
   end
 
   def test_update_node
-    name = "name"
-    @cli.clear(name)
-    nid = @cli.create_node(name)
+    @cli.clear
+    nid = @cli.create_node
     prop = {"key1" => "val1", "key2" => "val2"}
-    assert_equal(@cli.update_node(name, nid, prop), true)
+    assert_equal(@cli.update_node(nid, prop), true)
   end
 
   def test_create_edge
-    name = "name"
-    @cli.clear(name)
-    src = @cli.create_node(name)
-    tgt = @cli.create_node(name)
+    @cli.clear
+    src = @cli.create_node
+    tgt = @cli.create_node
     prop = {"key1" => "val1", "key2" => "val2"}
     ei = Jubatus::Graph::Edge.new(prop, src, tgt)
-    eid = @cli.create_edge("name", tgt, ei)
-  end
-
-  def test_create_edge
-    name = "name"
-    @cli.clear(name)
-    src = @cli.create_node(name)
-    tgt = @cli.create_node(name)
-    prop = {"key1" => "val1", "key2" => "val2"}
-    ei = Jubatus::Graph::Edge.new(prop, src, tgt)
-    eid = @cli.create_edge(src, tgt, ei)
+    eid = @cli.create_edge(tgt, ei)
   end
 
   def test_get_config
-    config = @cli.get_config("name")
+    config = @cli.get_config
     assert_equal(JSON.parse(config), @config)
   end
 

data/integration_test/jubatus_test/nearest_neighbor/test.rb

@@ -0,0 +1,86 @@
+#!/usr/bin/env ruby
+
+require 'test/unit'
+
+require 'json'
+
+require 'jubatus/nearest_neighbor/client'
+require 'jubatus_test/test_util'
+
+class NearestNeighborTest < Test::Unit::TestCase
+  HOST = "127.0.0.1"
+  PORT = 23007
+  TIMEOUT = 10
+
+  def setup
+    @config = {
+      "method" => "lsh",
+      "converter" => {
+        "string_filter_types" => {},
+        "string_filter_rules" => [],
+        "num_filter_types" => {},
+        "num_filter_rules" => [],
+        "string_types" => {},
+        "string_rules" => [{"key" => "*", "type" => "str",  "sample_weight" => "bin", "global_weight" => "bin"}],
+        "num_types" => {},
+        "num_rules" => [{"key" => "*", "type" => "num"}]
+      },
+      "parameter" => {
+        "hash_num" => 64
+      }
+    }
+
+    TestUtil.write_file("config_nearest_neighbor.json", @config.to_json)
+    @srv = TestUtil.fork_process("nearest_neighbor", PORT, "config_nearest_neighbor.json")
+    @cli = Jubatus::NearestNeighbor::Client::NearestNeighbor.new(HOST, PORT, "name")
+  end
+
+  def teardown
+    TestUtil.kill_process(@srv)
+  end
+
+  def test_get_client
+    assert_instance_of( MessagePack::RPC::Client, @cli.get_client )
+  end
+
+  def test_get_config
+    config = @cli.get_config
+    assert_equal(JSON.parse(config), @config)
+  end
+
+  def test_neighbor_row
+    @cli.clear
+    d = Jubatus::Common::Datum.new("skey1" => "val1", "skey2" => "val2", "nkey1" => 1.0, "nkey2" => 2.0)
+    @cli.set_row("neighbor_row", d)
+    d1 = @cli.neighbor_row_from_id("neighbor_row", 10)
+    d2 = @cli.neighbor_row_from_data(d, 10)
+  end
+
+  def test_similar_row
+    @cli.clear
+    d = Jubatus::Common::Datum.new("skey1" => "val1", "skey2" => "val2", "nkey1" => 1.0, "nkey2" => 2.0)
+    @cli.set_row("similar_row", d)
+    s1 = @cli.similar_row_from_id("similar_row", 10)
+    s2 = @cli.similar_row_from_data(d, 10)
+  end
+
+  def test_clear
+    @cli.clear
+  end
+
+  def test_save
+    assert_equal(@cli.save("nearest_neighbor.save_test.model"), true)
+  end
+
+  def test_load
+    model_name = "nearest_neighbor.load_test.model"
+    @cli.save(model_name)
+    assert_equal(@cli.load(model_name), true)
+  end
+
+  def test_get_status
+    @cli.get_status
+  end
+
+end
+

data/integration_test/jubatus_test/recommender/test.rb

@@ -0,0 +1,111 @@
+#!/usr/bin/env ruby
+
+require 'test/unit'
+
+require 'json'
+
+require 'jubatus/recommender/client'
+require 'jubatus_test/test_util'
+
+class RecommenderTest < Test::Unit::TestCase
+  HOST = "127.0.0.1"
+  PORT = 23003
+  TIMEOUT = 10
+
+  def setup
+    @config = {
+        "method" => "inverted_index",
+        "converter" => {
+            "string_filter_types" => {},
+            "string_filter_rules" => [],
+            "num_filter_types" => {},
+            "num_filter_rules" => [],
+            "string_types" => {},
+            "string_rules" => [{"key" => "*", "type" => "str",  "sample_weight" => "bin", "global_weight" => "bin"}],
+            "num_types" => {},
+            "num_rules" => [{"key" => "*", "type" => "num"}]
+        }
+    }
+
+    TestUtil.write_file("config_recommender.json", @config.to_json)
+    @srv = TestUtil.fork_process("recommender", PORT, "config_recommender.json")
+    @cli = Jubatus::Recommender::Client::Recommender.new(HOST, PORT, "name")
+  end
+
+  def teardown
+    TestUtil.kill_process(@srv)
+  end
+
+  def test_get_client
+    assert_instance_of( MessagePack::RPC::Client, @cli.get_client )
+  end
+
+  def test_get_config
+    config = @cli.get_config
+    assert_equal(JSON.parse(config), @config)
+  end
+
+  def test_complete_row
+    @cli.clear_row("complete_row")
+    d = Jubatus::Common::Datum.new("skey1" => "val1", "skey2" => "val2", "nkey1" => 1.0, "nkey2" => 2.0)
+    @cli.update_row("complete_row", d)
+    d1 = @cli.complete_row_from_id("complete_row")
+    d2 = @cli.complete_row_from_datum(d)
+  end
+
+  def test_get_similar_row
+    @cli.clear_row("similar_row")
+    d = Jubatus::Common::Datum.new("skey1" => "val1", "skey2" => "val2", "nkey1" => 1.0, "nkey2" => 2.0)
+    @cli.update_row("similar_row", d)
+    s1 = @cli.similar_row_from_id("similar_row", 10)
+    s2 = @cli.similar_row_from_datum(d, 10)
+  end
+
+  def test_decode_row
+    @cli.clear_row("decode_row")
+    d = Jubatus::Common::Datum.new("skey1" => "val1", "skey2" => "val2", "nkey1" => 1.0, "nkey2" => 2.0)
+    @cli.update_row("decode_row", d)
+    decoded_row = @cli.decode_row("decode_row")
+    assert_equal(d.string_values, decoded_row.string_values)
+    assert_equal(d.num_values, decoded_row.num_values)
+  end
+
+  def test_get_row
+    @cli.clear
+    d = Jubatus::Common::Datum.new("skey1" => "val1", "skey2" => "val2", "nkey1" => 1.0, "nkey2" => 2.0)
+    @cli.update_row("get_row", d)
+    row_names = @cli.get_all_rows
+    assert_equal(row_names, ["get_row"])
+  end
+
+
+  def test_clear
+    @cli.clear
+  end
+
+  def test_calcs
+    d = Jubatus::Common::Datum.new("skey1" => "val1", "skey2" => "val2", "nkey1" => 1.0, "nkey2" => 2.0)
+    assert_in_delta(@cli.calc_similarity(d, d), 1, 0.000001)
+    assert_in_delta(@cli.calc_l2norm(d), Math.sqrt(1*1 + 1*1+ 1*1 + 2*2), 0.000001)
+  end
+
+  def test_clear
+    @cli.clear
+  end
+
+  def test_save
+    assert_equal(@cli.save("recommender.save_test.model"), true)
+  end
+
+  def test_load
+    model_name = "recommender.load_test.model"
+    @cli.save(model_name)
+    assert_equal(@cli.load(model_name), true)
+  end
+
+  def test_get_status
+    @cli.get_status
+  end
+
+end
+