jubatus 0.4.1 → 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (43) hide show
  1. data/.gitignore +4 -1
  2. data/README.rst +1 -1
  3. data/Rakefile +8 -0
  4. data/VERSION +1 -1
  5. data/{test → integration_test}/jubatus_test/anomaly/test.rb +22 -23
  6. data/{test → integration_test}/jubatus_test/classifier/test.rb +10 -15
  7. data/integration_test/jubatus_test/clustering/test.rb +133 -0
  8. data/integration_test/jubatus_test/error/test.rb +51 -0
  9. data/{test → integration_test}/jubatus_test/graph/test.rb +15 -30
  10. data/integration_test/jubatus_test/nearest_neighbor/test.rb +86 -0
  11. data/integration_test/jubatus_test/recommender/test.rb +111 -0
  12. data/{test → integration_test}/jubatus_test/regression/test.rb +10 -15
  13. data/integration_test/jubatus_test/stat/test.rb +108 -0
  14. data/{test → integration_test}/jubatus_test/test_util.rb +1 -0
  15. data/jubatus.gemspec +1 -1
  16. data/lib/jubatus/anomaly/client.rb +34 -33
  17. data/lib/jubatus/anomaly/types.rb +30 -18
  18. data/lib/jubatus/classifier/client.rb +22 -28
  19. data/lib/jubatus/classifier/types.rb +56 -33
  20. data/lib/jubatus/clustering/client.rb +51 -0
  21. data/lib/jubatus/clustering/types.rb +44 -0
  22. data/lib/jubatus/common/client.rb +83 -0
  23. data/lib/jubatus/common/datum.rb +85 -0
  24. data/lib/jubatus/common/message_string_generator.rb +41 -0
  25. data/lib/jubatus/common/types.rb +247 -0
  26. data/lib/jubatus/common.rb +4 -0
  27. data/lib/jubatus/graph/client.rb +83 -60
  28. data/lib/jubatus/graph/types.rb +146 -85
  29. data/lib/jubatus/nearest_neighbor/client.rb +53 -0
  30. data/lib/jubatus/nearest_neighbor/types.rb +44 -0
  31. data/lib/jubatus/recommender/client.rb +50 -44
  32. data/lib/jubatus/recommender/types.rb +29 -26
  33. data/lib/jubatus/regression/client.rb +22 -28
  34. data/lib/jubatus/regression/types.rb +30 -18
  35. data/lib/jubatus/stat/client.rb +37 -38
  36. data/lib/jubatus/stat/types.rb +5 -4
  37. data/test/jubatus_test/common/client_test.rb +104 -0
  38. data/test/jubatus_test/common/datum_test.rb +43 -0
  39. data/test/jubatus_test/common/message_string_generator_test.rb +50 -0
  40. data/test/jubatus_test/common/types_test.rb +73 -0
  41. metadata +29 -16
  42. data/test/jubatus_test/recommender/test.rb +0 -122
  43. data/test/jubatus_test/stat/test.rb +0 -109
@@ -5,7 +5,6 @@ require 'test/unit'
5
5
  require 'json'
6
6
 
7
7
  require 'jubatus/regression/client'
8
- require 'jubatus/regression/types'
9
8
  require 'jubatus_test/test_util'
10
9
 
11
10
  class RegressionTest < Test::Unit::TestCase
@@ -34,7 +33,7 @@ class RegressionTest < Test::Unit::TestCase
34
33
 
35
34
  TestUtil.write_file("config_regression.json", @config.to_json)
36
35
  @srv = TestUtil.fork_process("regression", PORT, "config_regression.json")
37
- @cli = Jubatus::Regression::Client::Regression.new(HOST, PORT)
36
+ @cli = Jubatus::Regression::Client::Regression.new(HOST, PORT, "name")
38
37
  end
39
38
 
40
39
  def teardown
@@ -46,38 +45,34 @@ class RegressionTest < Test::Unit::TestCase
46
45
  end
47
46
 
48
47
  def test_get_config
49
- config = @cli.get_config("name")
48
+ config = @cli.get_config
50
49
  assert_equal(JSON.parse(config), @config)
51
50
  end
52
51
 
53
52
  def test_train
54
- string_values = [["key1", "val1"], ["key2", "val2"]]
55
- num_values = [["key1", 1.0], ["key2", 2.0]]
56
- d = Jubatus::Regression::Datum.new(string_values, num_values)
53
+ d = Jubatus::Common::Datum.new("skey1" => "val1", "skey2" => "val2", "nkey1" => 1.0, "nkey2" => 2.0)
57
54
  data = [[1.0, d]]
58
- assert_equal(@cli.train("name", data), 1)
55
+ assert_equal(@cli.train(data), 1)
59
56
  end
60
57
 
61
58
  def test_estimate
62
- string_values = [["key1", "val1"], ["key2", "val2"]]
63
- num_values = [["key1", 1.0], ["key2", 2.0]]
64
- d = Jubatus::Regression::Datum.new(string_values, num_values)
59
+ d = Jubatus::Common::Datum.new("skey1" => "val1", "skey2" => "val2", "nkey1" => 1.0, "nkey2" => 2.0)
65
60
  data = [d]
66
- result = @cli.estimate("name", data)
61
+ result = @cli.estimate(data)
67
62
  end
68
63
 
69
64
  def test_save
70
- assert_equal(@cli.save("name", "regression.save_test.model"), true)
65
+ assert_equal(@cli.save("regression.save_test.model"), true)
71
66
  end
72
67
 
73
68
  def test_load
74
69
  model_name = "regression.load_test.model"
75
- @cli.save("name", model_name)
76
- assert_equal(@cli.load("name", model_name), true)
70
+ @cli.save(model_name)
71
+ assert_equal(@cli.load(model_name), true)
77
72
  end
78
73
 
79
74
  def test_get_status
80
- @cli.get_status("name")
75
+ @cli.get_status
81
76
  end
82
77
 
83
78
  end
@@ -0,0 +1,108 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ require 'test/unit'
4
+
5
+ require 'json'
6
+
7
+ require 'jubatus/stat/client'
8
+ require 'jubatus_test/test_util'
9
+
10
+ class StatTest < Test::Unit::TestCase
11
+ HOST = "127.0.0.1"
12
+ PORT = 23004
13
+ TIMEOUT = 10
14
+
15
+ def setup
16
+ @config = {
17
+ "window_size" => 10
18
+ }
19
+
20
+ TestUtil.write_file("config_stat.json", @config.to_json)
21
+ @srv = TestUtil.fork_process("stat", PORT, "config_stat.json")
22
+ @cli = Jubatus::Stat::Client::Stat.new(HOST, PORT, "name")
23
+
24
+ end
25
+
26
+ def teardown
27
+ TestUtil.kill_process(@srv)
28
+ end
29
+
30
+ def test_get_client
31
+ assert_instance_of( MessagePack::RPC::Client, @cli.get_client )
32
+ end
33
+
34
+ def test_get_config
35
+ config = @cli.get_config
36
+ assert_equal(JSON.parse(config), @config)
37
+ end
38
+
39
+ def test_stddev
40
+ @cli.push("stddev", 1.0)
41
+ @cli.push("stddev", 2.0)
42
+ @cli.push("stddev", 3.0)
43
+ @cli.push("stddev", 4.0)
44
+ @cli.push("stddev", 5.0)
45
+ assert_equal(@cli.stddev("stddev"), Math::sqrt(2.0))
46
+ end
47
+
48
+ def test_sum
49
+ @cli.push("sum", 1.0)
50
+ @cli.push("sum", 2.0)
51
+ @cli.push("sum", 3.0)
52
+ @cli.push("sum", 4.0)
53
+ @cli.push("sum", 5.0)
54
+ assert_equal(@cli.sum("sum"), 15.0)
55
+ end
56
+
57
+ def test_max
58
+ @cli.push("max", 1.0)
59
+ @cli.push("max", 2.0)
60
+ @cli.push("max", 3.0)
61
+ @cli.push("max", 4.0)
62
+ @cli.push("max", 5.0)
63
+ assert_equal(@cli.max("max"), 5.0)
64
+ end
65
+
66
+ def test_min
67
+ @cli.push("min", 1.0)
68
+ @cli.push("min", 2.0)
69
+ @cli.push("min", 3.0)
70
+ @cli.push("min", 4.0)
71
+ @cli.push("min", 5.0)
72
+ assert_equal(@cli.min("min"), 1.0)
73
+ end
74
+
75
+ def test_entropy
76
+ @cli.push("entropy", 1.0)
77
+ @cli.push("entropy", 2.0)
78
+ @cli.push("entropy", 3.0)
79
+ @cli.push("entropy", 4.0)
80
+ @cli.push("entropy", 5.0)
81
+ assert_equal(@cli.entropy("entropy"), 0.0)
82
+ end
83
+
84
+ def test_moment
85
+ @cli.push("moment", 1.0)
86
+ @cli.push("moment", 2.0)
87
+ @cli.push("moment", 3.0)
88
+ @cli.push("moment", 4.0)
89
+ @cli.push("moment", 5.0)
90
+ assert_equal(@cli.moment("moment", 3, 0.0), 45.0)
91
+ end
92
+
93
+ def test_save
94
+ assert_equal(@cli.save("stat.save_test.model"), true)
95
+ end
96
+
97
+ def test_load
98
+ model_name = "stat.load_test.model"
99
+ @cli.save(model_name)
100
+ assert_equal(@cli.load(model_name), true)
101
+ end
102
+
103
+ def test_get_status
104
+ @cli.get_status
105
+ end
106
+
107
+ end
108
+
@@ -26,6 +26,7 @@ class TestUtil
26
26
  child = Process.fork()
27
27
  if child.nil? then
28
28
  begin
29
+ STDERR.reopen File.open('/dev/null', 'w')
29
30
  exec(cmd, cmd, "--rpc-port", port.to_s, "--configpath", config, "--thread", "100", "--datadir", ".")
30
31
  rescue
31
32
  puts $!
data/jubatus.gemspec CHANGED
@@ -12,7 +12,7 @@ Gem::Specification.new do |gem|
12
12
  gem.has_rdoc = false
13
13
  gem.license = "MIT"
14
14
 
15
- files = `git ls-files`.split("\n")
15
+ files = `git ls-files`.split("\n") + Dir.glob("lib/jubatus/**/*.rb")
16
16
  excludes = ["patch/*", "generate.sh"]
17
17
 
18
18
  gem.files = files.reject { |f| excludes.any? { |e| File.fnmatch(e, f) } }
@@ -1,53 +1,54 @@
1
- # This file is auto-generated from anomaly.idl
1
+ # This file is auto-generated from anomaly.idl(0.4.5-347-g86989a6) with jenerator version 0.4.5-418-gd2d5f04/develop
2
2
  # *** DO NOT EDIT ***
3
3
 
4
4
  require 'rubygems'
5
5
  require 'msgpack/rpc'
6
+ require 'jubatus/common'
6
7
  require File.join(File.dirname(__FILE__), 'types')
7
8
 
8
9
  module Jubatus
10
+
9
11
  module Anomaly
10
12
  module Client
11
13
 
12
- class Anomaly
13
- def initialize(host, port)
14
- @cli = MessagePack::RPC::Client.new(host, port)
15
- end
16
- def get_client
17
- @cli
18
- end
19
- def get_config(name)
20
- @cli.call(:get_config, name)
21
- end
22
- def clear_row(name, id)
23
- @cli.call(:clear_row, name, id)
14
+ class Anomaly < Jubatus::Common::ClientBase
15
+ include Jubatus::Common
16
+ def initialize(host, port, name, timeout_sec=10)
17
+ super
24
18
  end
25
- def add(name, row)
26
- @cli.call(:add, name, row)
27
- end
28
- def update(name, id, row)
29
- @cli.call(:update, name, id, row)
19
+
20
+ def clear_row(id)
21
+ @jubatus_client.call("clear_row", [id], TBool.new, [TString.new])
30
22
  end
31
- def clear(name)
32
- @cli.call(:clear, name)
23
+
24
+ def add(row)
25
+ @jubatus_client.call("add", [row], TUserDef.new(IdWithScore), [TDatum.new])
33
26
  end
34
- def calc_score(name, row)
35
- @cli.call(:calc_score, name, row)
27
+
28
+ def update(id, row)
29
+ @jubatus_client.call("update", [id, row], TFloat.new, [TString.new,
30
+ TDatum.new])
36
31
  end
37
- def get_all_rows(name)
38
- @cli.call(:get_all_rows, name)
32
+
33
+ def overwrite(id, row)
34
+ @jubatus_client.call("overwrite", [id, row], TFloat.new, [TString.new,
35
+ TDatum.new])
39
36
  end
40
- def save(name, id)
41
- @cli.call(:save, name, id)
37
+
38
+ def clear
39
+ @jubatus_client.call("clear", [], TBool.new, [])
42
40
  end
43
- def load(name, id)
44
- @cli.call(:load, name, id)
41
+
42
+ def calc_score(row)
43
+ @jubatus_client.call("calc_score", [row], TFloat.new, [TDatum.new])
45
44
  end
46
- def get_status(name)
47
- @cli.call(:get_status, name)
45
+
46
+ def get_all_rows
47
+ @jubatus_client.call("get_all_rows", [], TList.new(TString.new), [])
48
48
  end
49
49
  end
50
50
 
51
- end
52
- end
53
- end
51
+ end # Client
52
+ end # Anomaly
53
+
54
+ end # Jubatus
@@ -1,32 +1,44 @@
1
- # This file is auto-generated from anomaly.idl
1
+ # This file is auto-generated from anomaly.idl(0.4.5-347-g86989a6) with jenerator version 0.4.5-418-gd2d5f04/develop
2
2
  # *** DO NOT EDIT ***
3
3
 
4
4
  require 'rubygems'
5
5
  require 'msgpack/rpc'
6
+ require 'jubatus/common'
7
+
6
8
  module Jubatus
7
9
  module Anomaly
8
10
 
9
- class Datum
10
- def initialize(string_values, num_values)
11
- @string_values = string_values
12
- @num_values = num_values
13
- end
14
- def to_tuple
15
- [@string_values.map {|x| [x[0], x[1], ] },
16
- @num_values.map {|x| [x[0], x[1], ] }]
11
+ class IdWithScore
12
+ include Jubatus::Common
13
+ TYPE = TTuple.new(TString.new, TFloat.new)
14
+
15
+ def initialize(id, score)
16
+ @id = id
17
+ @score = score
17
18
  end
19
+
18
20
  def to_msgpack(out = '')
19
- to_tuple.to_msgpack(out)
21
+ t = [@id, @score]
22
+ return TYPE.to_msgpack(t)
23
+ end
24
+
25
+ def IdWithScore.from_msgpack(m)
26
+ val = TYPE.from_msgpack(m)
27
+ IdWithScore.new(*val)
20
28
  end
21
- def Datum.from_tuple(tuple)
22
- Datum.new(
23
- tuple[0].map { |x| [x[0] , x[1] ] },
24
- tuple[1].map { |x| [x[0] , x[1] ] }
25
- )
29
+
30
+ def to_s
31
+ gen = Jubatus::Common::MessageStringGenerator.new
32
+ gen.open("id_with_score")
33
+ gen.add("id", @id)
34
+ gen.add("score", @score)
35
+ gen.close()
36
+ return gen.to_s
26
37
  end
27
- attr_accessor :string_values, :num_values
28
- end
29
38
 
30
- end
39
+ attr_reader :id, :score
40
+
31
41
  end
32
42
 
43
+ end # Anomaly
44
+ end # Jubatus
@@ -1,44 +1,38 @@
1
- # This file is auto-generated from classifier.idl
1
+ # This file is auto-generated from classifier.idl(0.4.5-347-g86989a6) with jenerator version 0.4.5-418-gd2d5f04/develop
2
2
  # *** DO NOT EDIT ***
3
3
 
4
4
  require 'rubygems'
5
5
  require 'msgpack/rpc'
6
+ require 'jubatus/common'
6
7
  require File.join(File.dirname(__FILE__), 'types')
7
8
 
8
9
  module Jubatus
10
+
9
11
  module Classifier
10
12
  module Client
11
13
 
12
- class Classifier
13
- def initialize(host, port)
14
- @cli = MessagePack::RPC::Client.new(host, port)
15
- end
16
- def get_client
17
- @cli
18
- end
19
- def get_config(name)
20
- @cli.call(:get_config, name)
21
- end
22
- def train(name, data)
23
- @cli.call(:train, name, data)
14
+ class Classifier < Jubatus::Common::ClientBase
15
+ include Jubatus::Common
16
+ def initialize(host, port, name, timeout_sec=10)
17
+ super
24
18
  end
25
- def classify(name, data)
26
- @cli.call(:classify, name, data)
27
- end
28
- def clear(name)
29
- @cli.call(:clear, name)
30
- end
31
- def save(name, id)
32
- @cli.call(:save, name, id)
19
+
20
+ def train(data)
21
+ @jubatus_client.call("train", [data], TInt.new(true, 4), [TList.new(
22
+ TUserDef.new(LabeledDatum))])
33
23
  end
34
- def load(name, id)
35
- @cli.call(:load, name, id)
24
+
25
+ def classify(data)
26
+ @jubatus_client.call("classify", [data], TList.new(TList.new(TUserDef.new(
27
+ EstimateResult))), [TList.new(TDatum.new)])
36
28
  end
37
- def get_status(name)
38
- @cli.call(:get_status, name)
29
+
30
+ def clear
31
+ @jubatus_client.call("clear", [], TBool.new, [])
39
32
  end
40
33
  end
41
34
 
42
- end
43
- end
44
- end
35
+ end # Client
36
+ end # Classifier
37
+
38
+ end # Jubatus
@@ -1,53 +1,76 @@
1
- # This file is auto-generated from classifier.idl
1
+ # This file is auto-generated from classifier.idl(0.4.5-347-g86989a6) with jenerator version 0.4.5-418-gd2d5f04/develop
2
2
  # *** DO NOT EDIT ***
3
3
 
4
4
  require 'rubygems'
5
5
  require 'msgpack/rpc'
6
+ require 'jubatus/common'
7
+
6
8
  module Jubatus
7
9
  module Classifier
8
10
 
9
- class Datum
10
- def initialize(string_values, num_values)
11
- @string_values = string_values
12
- @num_values = num_values
13
- end
14
- def to_tuple
15
- [@string_values.map {|x| [x[0], x[1], ] },
16
- @num_values.map {|x| [x[0], x[1], ] }]
11
+ class EstimateResult
12
+ include Jubatus::Common
13
+ TYPE = TTuple.new(TString.new, TFloat.new)
14
+
15
+ def initialize(label, score)
16
+ @label = label
17
+ @score = score
17
18
  end
19
+
18
20
  def to_msgpack(out = '')
19
- to_tuple.to_msgpack(out)
21
+ t = [@label, @score]
22
+ return TYPE.to_msgpack(t)
20
23
  end
21
- def Datum.from_tuple(tuple)
22
- Datum.new(
23
- tuple[0].map { |x| [x[0] , x[1] ] },
24
- tuple[1].map { |x| [x[0] , x[1] ] }
25
- )
24
+
25
+ def EstimateResult.from_msgpack(m)
26
+ val = TYPE.from_msgpack(m)
27
+ EstimateResult.new(*val)
26
28
  end
27
- attr_accessor :string_values, :num_values
28
- end
29
29
 
30
- class Estimate_result
31
- def initialize(label, score)
32
- @label = label
33
- @score = score
30
+ def to_s
31
+ gen = Jubatus::Common::MessageStringGenerator.new
32
+ gen.open("estimate_result")
33
+ gen.add("label", @label)
34
+ gen.add("score", @score)
35
+ gen.close()
36
+ return gen.to_s
34
37
  end
35
- def to_tuple
36
- [@label,
37
- @score]
38
+
39
+ attr_reader :label, :score
40
+
41
+ end
42
+
43
+ class LabeledDatum
44
+ include Jubatus::Common
45
+ TYPE = TTuple.new(TString.new, TDatum.new)
46
+
47
+ def initialize(label, data)
48
+ @label = label
49
+ @data = data
38
50
  end
51
+
39
52
  def to_msgpack(out = '')
40
- to_tuple.to_msgpack(out)
53
+ t = [@label, @data]
54
+ return TYPE.to_msgpack(t)
55
+ end
56
+
57
+ def LabeledDatum.from_msgpack(m)
58
+ val = TYPE.from_msgpack(m)
59
+ LabeledDatum.new(*val)
41
60
  end
42
- def Estimate_result.from_tuple(tuple)
43
- Estimate_result.new(
44
- tuple[0],
45
- tuple[1]
46
- )
61
+
62
+ def to_s
63
+ gen = Jubatus::Common::MessageStringGenerator.new
64
+ gen.open("labeled_datum")
65
+ gen.add("label", @label)
66
+ gen.add("data", @data)
67
+ gen.close()
68
+ return gen.to_s
47
69
  end
48
- attr_accessor :label, :score
49
- end
50
70
 
51
- end
71
+ attr_reader :label, :data
72
+
52
73
  end
53
74
 
75
+ end # Classifier
76
+ end # Jubatus
@@ -0,0 +1,51 @@
1
+ # This file is auto-generated from clustering.idl(0.4.5-350-g9c67807) with jenerator version 0.4.5-418-gd2d5f04/develop
2
+ # *** DO NOT EDIT ***
3
+
4
+ require 'rubygems'
5
+ require 'msgpack/rpc'
6
+ require 'jubatus/common'
7
+ require File.join(File.dirname(__FILE__), 'types')
8
+
9
+ module Jubatus
10
+
11
+ module Clustering
12
+ module Client
13
+
14
+ class Clustering < Jubatus::Common::ClientBase
15
+ include Jubatus::Common
16
+ def initialize(host, port, name, timeout_sec=10)
17
+ super
18
+ end
19
+
20
+ def push(points)
21
+ @jubatus_client.call("push", [points], TBool.new, [TList.new(TDatum.new)])
22
+ end
23
+
24
+ def get_revision
25
+ @jubatus_client.call("get_revision", [], TInt.new(false, 4), [])
26
+ end
27
+
28
+ def get_core_members
29
+ @jubatus_client.call("get_core_members", [], TList.new(TList.new(
30
+ TUserDef.new(WeightedDatum))), [])
31
+ end
32
+
33
+ def get_k_center
34
+ @jubatus_client.call("get_k_center", [], TList.new(TDatum.new), [])
35
+ end
36
+
37
+ def get_nearest_center(point)
38
+ @jubatus_client.call("get_nearest_center", [point], TDatum.new,
39
+ [TDatum.new])
40
+ end
41
+
42
+ def get_nearest_members(point)
43
+ @jubatus_client.call("get_nearest_members", [point], TList.new(TUserDef.new(
44
+ WeightedDatum)), [TDatum.new])
45
+ end
46
+ end
47
+
48
+ end # Client
49
+ end # Clustering
50
+
51
+ end # Jubatus
@@ -0,0 +1,44 @@
1
+ # This file is auto-generated from clustering.idl(0.4.5-350-g9c67807) with jenerator version 0.4.5-418-gd2d5f04/develop
2
+ # *** DO NOT EDIT ***
3
+
4
+ require 'rubygems'
5
+ require 'msgpack/rpc'
6
+ require 'jubatus/common'
7
+
8
+ module Jubatus
9
+ module Clustering
10
+
11
+ class WeightedDatum
12
+ include Jubatus::Common
13
+ TYPE = TTuple.new(TFloat.new, TDatum.new)
14
+
15
+ def initialize(weight, point)
16
+ @weight = weight
17
+ @point = point
18
+ end
19
+
20
+ def to_msgpack(out = '')
21
+ t = [@weight, @point]
22
+ return TYPE.to_msgpack(t)
23
+ end
24
+
25
+ def WeightedDatum.from_msgpack(m)
26
+ val = TYPE.from_msgpack(m)
27
+ WeightedDatum.new(*val)
28
+ end
29
+
30
+ def to_s
31
+ gen = Jubatus::Common::MessageStringGenerator.new
32
+ gen.open("weighted_datum")
33
+ gen.add("weight", @weight)
34
+ gen.add("point", @point)
35
+ gen.close()
36
+ return gen.to_s
37
+ end
38
+
39
+ attr_reader :weight, :point
40
+
41
+ end
42
+
43
+ end # Clustering
44
+ end # Jubatus