cellect-client 0.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (39) hide show
  1. checksums.yaml +7 -0
  2. data/.gitignore +15 -0
  3. data/.rspec +2 -0
  4. data/Gemfile +2 -0
  5. data/README.md +19 -0
  6. data/Rakefile +9 -0
  7. data/cellect-client.gemspec +35 -0
  8. data/cellect.gemspec +33 -0
  9. data/data/.gitkeep +0 -0
  10. data/lib/cellect/client/connection.rb +58 -0
  11. data/lib/cellect/client/node_set.rb +35 -0
  12. data/lib/cellect/client.rb +28 -0
  13. data/lib/cellect/node_set.rb +38 -0
  14. data/lib/cellect/version.rb +3 -0
  15. data/lib/cellect.rb +7 -0
  16. data/log/.gitkeep +0 -0
  17. data/spec/client/connection_spec.rb +64 -0
  18. data/spec/client/node_set_spec.rb +24 -0
  19. data/spec/fixtures/project_data/grouped_pairwise_priority.json +109 -0
  20. data/spec/fixtures/project_data/grouped_pairwise_random.json +89 -0
  21. data/spec/fixtures/project_data/grouped_priority.json +59 -0
  22. data/spec/fixtures/project_data/grouped_random.json +49 -0
  23. data/spec/fixtures/project_data/pairwise_priority.json +49 -0
  24. data/spec/fixtures/project_data/pairwise_random.json +39 -0
  25. data/spec/fixtures/project_data/priority.json +49 -0
  26. data/spec/fixtures/project_data/random.json +39 -0
  27. data/spec/fixtures/user_data/complete_user.json +118 -0
  28. data/spec/fixtures/user_data/new_user.json +26 -0
  29. data/spec/fixtures/user_data/partial_user.json +58 -0
  30. data/spec/spec_helper.rb +43 -0
  31. data/spec/support/cellect_helper.rb +12 -0
  32. data/spec/support/shared_api_context.rb +11 -0
  33. data/spec/support/shared_examples_for_node_set.rb +27 -0
  34. data/spec/support/shared_examples_for_project.rb +26 -0
  35. data/spec/support/shared_examples_for_set.rb +34 -0
  36. data/spec/support/spec_adapter.rb +43 -0
  37. data/spec/support/zk_setup.rb +26 -0
  38. data/tmp/.gitkeep +0 -0
  39. metadata +242 -0
checksums.yaml ADDED
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA1:
3
+ metadata.gz: a945054d21ad4dccae1e6a4447fef9dcbf7f30d4
4
+ data.tar.gz: fee2b2070cff140fe90e41cce096e1f80d13e1d6
5
+ SHA512:
6
+ metadata.gz: dc92be103ff6b6027bcd2b915430a9fe7877ed03077e71a8b00ac3f99e4f63c5b8c04db104a8abf1ab019453ed7f1fecd3ca8f5acd7032025f1f8c423e060a02
7
+ data.tar.gz: a7b7cfb49ff7b3c196d1d3e6711c70af644b828aa732a6dd1021c777a3d28d5900c09abe2413858995d359723dc094eae9dc654feccdb6fcae2a6768c66a562d
data/.gitignore ADDED
@@ -0,0 +1,15 @@
1
+ .bundle
2
+ .DS_Store
3
+ .rvmrc
4
+ .ruby-version
5
+ /data/*
6
+ /ext/*.o
7
+ /ext/Makefile
8
+ /ext/*.bundle
9
+ /scratch
10
+ *.so
11
+ .vagrant
12
+ log/*
13
+ tmp/*
14
+ *.gem
15
+ pkg/*
data/.rspec ADDED
@@ -0,0 +1,2 @@
1
+ --color
2
+ --format progress
data/Gemfile ADDED
@@ -0,0 +1,2 @@
1
+ source 'http://rubygems.org'
2
+ gemspec name: 'cellect'
data/README.md ADDED
@@ -0,0 +1,19 @@
1
+ # Cellect
2
+
3
+ This is a work in progress
4
+
5
+ ## Building
6
+
7
+ 1. Install [Boost](http://www.boost.org/): OS X: `brew update && brew install boost`, Ubuntu: `sudo apt-get update && sudo apt-get install libboost-all-dev`
8
+ 2. Install gem dependencies: `bundle` (See Note)
9
+ 3. Build extension: `cd ext; ruby extconf.rb; make; cd ..`
10
+
11
+ ### Note
12
+ To install rice your Ruby must be compiled with shared libraries enabled, from the rice docs:
13
+ * rvm: `rvm reinstall [version] -- --enable-shared`
14
+ * rbenv: `CONFIGURE_OPTS="--enable-shared" rbenv install [version]`
15
+
16
+
17
+ ## Testing
18
+
19
+ Run the specs with `rake`
data/Rakefile ADDED
@@ -0,0 +1,9 @@
1
+ require 'bundler/gem_helper'
2
+ require 'rspec/core/rake_task'
3
+
4
+ Bundler::GemHelper.install_tasks name: 'cellect'
5
+ Bundler::GemHelper.install_tasks name: 'cellect-server'
6
+ Bundler::GemHelper.install_tasks name: 'cellect-client'
7
+
8
+ RSpec::Core::RakeTask.new :spec
9
+ task default: :spec
@@ -0,0 +1,35 @@
1
+ # coding: utf-8
2
+ lib = File.expand_path('../lib', __FILE__)
3
+ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
4
+ require 'cellect/version'
5
+
6
+ Gem::Specification.new do |spec|
7
+ spec.name = 'cellect-client'
8
+ spec.version = Cellect::VERSION
9
+ spec.authors = ['Michael Parrish']
10
+ spec.email = ['michael@zooniverse.org']
11
+ spec.summary = ''
12
+ spec.description = ''
13
+ spec.homepage = 'https://github.com/parrish/Cellect'
14
+ spec.license = 'MIT'
15
+
16
+ ignored_paths = %w(config data log script tmp).collect{ |path| Dir["#{ path }/**/*"] }.flatten
17
+ ignored_files = %w(Dockerfile Vagrantfile Gemfile.lock config.ru) + ignored_paths
18
+
19
+ spec.files = `git ls-files -z`.split("\x0").reject{ |f| f =~ /server/ } - ignored_files
20
+ spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
21
+ spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
22
+ spec.require_paths = ['lib']
23
+
24
+ spec.add_development_dependency 'bundler', '~> 1.5'
25
+ spec.add_development_dependency 'rake'
26
+ spec.add_development_dependency 'oj'
27
+ spec.add_development_dependency 'rspec'
28
+ spec.add_development_dependency 'rack-test'
29
+ spec.add_development_dependency 'pry'
30
+
31
+ spec.add_runtime_dependency 'celluloid', '0.16.0.pre'
32
+ spec.add_runtime_dependency 'celluloid-io', '0.16.0.pre'
33
+ spec.add_runtime_dependency 'http', '~> 0.6'
34
+ spec.add_runtime_dependency 'zk', '~> 1.9'
35
+ end
data/cellect.gemspec ADDED
@@ -0,0 +1,33 @@
1
+ # coding: utf-8
2
+ lib = File.expand_path('../lib', __FILE__)
3
+ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
4
+ require 'cellect/version'
5
+
6
+ Gem::Specification.new do |spec|
7
+ spec.name = 'cellect'
8
+ spec.version = Cellect::VERSION
9
+ spec.authors = ['Michael Parrish']
10
+ spec.email = ['michael@zooniverse.org']
11
+ spec.summary = ''
12
+ spec.description = ''
13
+ spec.homepage = 'https://github.com/parrish/Cellect'
14
+ spec.license = 'MIT'
15
+
16
+ spec.files = ['lib/cellect.rb', 'lib/cellect/version.rb']
17
+ spec.executables = []
18
+ spec.test_files = []
19
+ spec.require_paths = ['lib']
20
+
21
+ spec.add_development_dependency 'bundler', '~> 1.5'
22
+ spec.add_development_dependency 'rake'
23
+ spec.add_development_dependency 'oj'
24
+ spec.add_development_dependency 'rspec'
25
+ spec.add_development_dependency 'rack-test'
26
+ spec.add_development_dependency 'pry'
27
+ spec.add_development_dependency 'puma', '~> 2.8'
28
+ spec.add_development_dependency 'pg', '~> 0.17'
29
+ spec.add_development_dependency 'connection_pool', '~> 2.0'
30
+
31
+ spec.add_runtime_dependency 'cellect-server', Cellect::VERSION
32
+ spec.add_runtime_dependency 'cellect-client', Cellect::VERSION
33
+ end
data/data/.gitkeep ADDED
File without changes
@@ -0,0 +1,58 @@
1
+ require 'http'
2
+
3
+ module Cellect
4
+ module Client
5
+ class Connection
6
+ include Celluloid
7
+ include Celluloid::IO
8
+
9
+ def reload_project(id)
10
+ broadcast :post, "/projects/#{ id }/reload"
11
+ end
12
+
13
+ def delete_project(id)
14
+ broadcast :delete, "/projects/#{ id }"
15
+ end
16
+
17
+ def add_subject(id, project_id: project_id, group_id: nil, priority: nil)
18
+ broadcast :put, "/projects/#{ project_id }/add", querystring(subject_id: id, group_id: group_id, priority: priority)
19
+ end
20
+
21
+ def remove_subject(id, project_id: project_id, group_id: nil)
22
+ broadcast :put, "/projects/#{ project_id }/remove", querystring(subject_id: id, group_id: group_id)
23
+ end
24
+
25
+ def load_user(id, host: host, project_id: project_id)
26
+ send_http host, :post, "/projects/#{ project_id }/users/#{ id }/load"
27
+ end
28
+
29
+ def add_seen(id, user_id: user_id, host: host, project_id: project_id)
30
+ send_http host, :put, "/projects/#{ project_id }/users/#{ user_id }/add_seen", querystring(subject_id: id)
31
+ end
32
+
33
+ protected
34
+
35
+ def broadcast(action, path, query = '')
36
+ Cellect::Client.node_set.nodes.each_pair do |node, host|
37
+ send_http host, action, path, query
38
+ end
39
+ end
40
+
41
+ def send_http(host, action, path, query = '')
42
+ params = { host: host, path: path }
43
+ params[:query] = query if query && !query.empty?
44
+ uri = URI::HTTP.build params
45
+ HTTP.send action, uri.to_s, socket_class: Celluloid::IO::TCPSocket
46
+ end
47
+
48
+ def querystring(hash = { })
49
+ [].tap do |list|
50
+ hash.each_pair do |key, value|
51
+ next unless value
52
+ list << "#{ key }=#{ value }"
53
+ end
54
+ end.join('&')
55
+ end
56
+ end
57
+ end
58
+ end
@@ -0,0 +1,35 @@
1
+ require 'cellect/node_set'
2
+
3
+ module Cellect
4
+ module Client
5
+ class NodeSet < Cellect::NodeSet
6
+ attr_accessor :nodes
7
+
8
+ def initialize
9
+ self.nodes = { }
10
+ super
11
+ end
12
+
13
+ protected
14
+
15
+ def nodes_changed(nodes)
16
+ self.nodes = { }
17
+ nodes.each do |node|
18
+ self.nodes[node] = zk.get("/nodes/#{ node }").first
19
+ end
20
+ end
21
+
22
+ def setup
23
+ watch_nodes
24
+ zk.mkdir_p '/nodes'
25
+ nodes_changed zk.children('/nodes', watch: true)
26
+ end
27
+
28
+ def watch_nodes
29
+ zk.register('/nodes') do |event|
30
+ nodes_changed zk.children('/nodes', watch: true)
31
+ end
32
+ end
33
+ end
34
+ end
35
+ end
@@ -0,0 +1,28 @@
1
+ require 'cellect'
2
+
3
+ module Cellect
4
+ module Client
5
+ require 'cellect/client/node_set'
6
+ require 'cellect/client/connection'
7
+
8
+ class << self
9
+ attr_accessor :connection, :_node_set
10
+ end
11
+
12
+ def self.node_set
13
+ self._node_set ||= NodeSet.supervise
14
+ _node_set.actors.first
15
+ end
16
+
17
+ def self.ready?
18
+ node_set.ready?
19
+ end
20
+
21
+ def self.choose_host
22
+ node_set.nodes.values.sample
23
+ end
24
+
25
+ Client.node_set
26
+ Client.connection = Connection.pool size: ENV.fetch('CELLECT_POOL_SIZE', 100).to_i
27
+ end
28
+ end
@@ -0,0 +1,38 @@
1
+ require 'zk'
2
+ require 'timeout'
3
+
4
+ module Cellect
5
+ class NodeSet
6
+ include Celluloid
7
+
8
+ attr_accessor :zk, :state
9
+
10
+ def initialize
11
+ self.state = :initializing
12
+ after(0.001){ async.initialize_zk } # don't block waiting for ZK to connect
13
+ end
14
+
15
+ def initialize_zk
16
+ # don't let ZK hang the thread, just retry connection on restart
17
+ Timeout::timeout(5) do
18
+ self.zk = ZK.new zk_url, chroot: '/cellect'
19
+ end
20
+ setup
21
+ self.state = :ready
22
+ end
23
+
24
+ def ready?
25
+ state == :ready
26
+ end
27
+
28
+ protected
29
+
30
+ def zk_url
31
+ ENV.fetch 'ZK_URL', 'localhost:2181'
32
+ end
33
+
34
+ def setup
35
+
36
+ end
37
+ end
38
+ end
@@ -0,0 +1,3 @@
1
+ module Cellect
2
+ VERSION = '0.0.1'
3
+ end
data/lib/cellect.rb ADDED
@@ -0,0 +1,7 @@
1
+ require 'celluloid'
2
+ require 'celluloid/io'
3
+ require 'cellect/version'
4
+
5
+ module Cellect
6
+
7
+ end
data/log/.gitkeep ADDED
File without changes
@@ -0,0 +1,64 @@
1
+ require 'spec_helper'
2
+
3
+ module Cellect::Client
4
+ describe Connection do
5
+ let(:connection){ Cellect::Client.connection }
6
+
7
+ before(:each) do
8
+ Cellect::Client.node_set.stub(:nodes).and_return 'a' => '1', 'b' => '2'
9
+ end
10
+
11
+ def should_send(action: action, url: url, to: to)
12
+ HTTP.should_receive(:send).with action, "http://#{ to }/#{ url }", socket_class: Celluloid::IO::TCPSocket
13
+ end
14
+
15
+ def should_broadcast(action: action, url: url)
16
+ [1, 2].each{ |i| should_send action: action, url: url, to: i }
17
+ end
18
+
19
+ it 'should reload projects' do
20
+ should_broadcast action: :post, url: 'projects/random/reload'
21
+ connection.reload_project 'random'
22
+ end
23
+
24
+ it 'should delete projects' do
25
+ should_broadcast action: :delete, url: 'projects/random'
26
+ connection.delete_project 'random'
27
+ end
28
+
29
+ it 'should add subjects' do
30
+ should_broadcast action: :put, url: 'projects/random/add?subject_id=123'
31
+ connection.add_subject 123, project_id: 'random'
32
+ end
33
+
34
+ it 'should add grouped subjects' do
35
+ should_broadcast action: :put, url: 'projects/random/add?subject_id=123&group_id=321'
36
+ connection.add_subject 123, project_id: 'random', group_id: 321
37
+ end
38
+
39
+ it 'should add prioritized grouped subjects' do
40
+ should_broadcast action: :put, url: 'projects/random/add?subject_id=123&group_id=321&priority=0.123'
41
+ connection.add_subject 123, project_id: 'random', group_id: 321, priority: 0.123
42
+ end
43
+
44
+ it 'should remove subjects' do
45
+ should_broadcast action: :put, url: 'projects/random/remove?subject_id=123'
46
+ connection.remove_subject 123, project_id: 'random'
47
+ end
48
+
49
+ it 'should remove grouped subjects' do
50
+ should_broadcast action: :put, url: 'projects/random/remove?subject_id=123&group_id=321'
51
+ connection.remove_subject 123, project_id: 'random', group_id: 321
52
+ end
53
+
54
+ it 'should load users' do
55
+ should_send action: :post, url: 'projects/random/users/123/load', to: 1
56
+ connection.load_user 123, host: '1', project_id: 'random'
57
+ end
58
+
59
+ it 'should add seen subjects' do
60
+ should_send action: :put, url: 'projects/random/users/123/add_seen?subject_id=456', to: 1
61
+ connection.add_seen 456, host: '1', user_id: 123, project_id: 'random'
62
+ end
63
+ end
64
+ end
@@ -0,0 +1,24 @@
1
+ require 'spec_helper'
2
+
3
+ module Cellect::Client
4
+ describe NodeSet do
5
+ it_behaves_like 'node set'
6
+ let(:node_set){ Cellect::Client.node_set }
7
+
8
+ it 'should update the node list when changing' do
9
+ begin
10
+ pass_until node_set, is: :ready
11
+ node_set.zk.create '/nodes/node', data: 'foo', mode: :ephemeral_sequential
12
+
13
+ 10.times do
14
+ break if node_set.nodes['node0000000001']
15
+ Thread.pass
16
+ end
17
+
18
+ node_set.nodes['node0000000001'].should == 'foo'
19
+ ensure
20
+ node_set.zk.delete '/nodes/node0000000001'
21
+ end
22
+ end
23
+ end
24
+ end
@@ -0,0 +1,109 @@
1
+ {
2
+ "id": 8,
3
+ "name": "grouped_pairwise_priority",
4
+ "prioritized": true,
5
+ "pairwise": true,
6
+ "grouped": true,
7
+ "entries": [
8
+ {
9
+ "id": 16,
10
+ "priority": -6,
11
+ "group_id": 1
12
+ },
13
+ {
14
+ "id": 17,
15
+ "priority": -7,
16
+ "group_id": 2
17
+ },
18
+ {
19
+ "id": 3,
20
+ "priority": 7,
21
+ "group_id": 3
22
+ },
23
+ {
24
+ "id": 8,
25
+ "priority": 2,
26
+ "group_id": 2
27
+ },
28
+ {
29
+ "id": 7,
30
+ "priority": 3,
31
+ "group_id": 1
32
+ },
33
+ {
34
+ "id": 13,
35
+ "priority": -3,
36
+ "group_id": 1
37
+ },
38
+ {
39
+ "id": 14,
40
+ "priority": -4,
41
+ "group_id": 2
42
+ },
43
+ {
44
+ "id": 4,
45
+ "priority": 6,
46
+ "group_id": 1
47
+ },
48
+ {
49
+ "id": 6,
50
+ "priority": 4,
51
+ "group_id": 3
52
+ },
53
+ {
54
+ "id": 18,
55
+ "priority": -8,
56
+ "group_id": 3
57
+ },
58
+ {
59
+ "id": 12,
60
+ "priority": -2,
61
+ "group_id": 3
62
+ },
63
+ {
64
+ "id": 1,
65
+ "priority": 9,
66
+ "group_id": 1
67
+ },
68
+ {
69
+ "id": 11,
70
+ "priority": -1,
71
+ "group_id": 2
72
+ },
73
+ {
74
+ "id": 10,
75
+ "priority": 0,
76
+ "group_id": 1
77
+ },
78
+ {
79
+ "id": 9,
80
+ "priority": 1,
81
+ "group_id": 3
82
+ },
83
+ {
84
+ "id": 20,
85
+ "priority": -10,
86
+ "group_id": 2
87
+ },
88
+ {
89
+ "id": 2,
90
+ "priority": 8,
91
+ "group_id": 2
92
+ },
93
+ {
94
+ "id": 15,
95
+ "priority": -5,
96
+ "group_id": 3
97
+ },
98
+ {
99
+ "id": 5,
100
+ "priority": 5,
101
+ "group_id": 2
102
+ },
103
+ {
104
+ "id": 19,
105
+ "priority": -9,
106
+ "group_id": 1
107
+ }
108
+ ]
109
+ }
@@ -0,0 +1,89 @@
1
+ {
2
+ "id": 7,
3
+ "name": "grouped_pairwise_random",
4
+ "prioritized": false,
5
+ "pairwise": true,
6
+ "grouped": true,
7
+ "entries": [
8
+ {
9
+ "id": 1,
10
+ "group_id": 1
11
+ },
12
+ {
13
+ "id": 2,
14
+ "group_id": 2
15
+ },
16
+ {
17
+ "id": 3,
18
+ "group_id": 3
19
+ },
20
+ {
21
+ "id": 4,
22
+ "group_id": 1
23
+ },
24
+ {
25
+ "id": 5,
26
+ "group_id": 2
27
+ },
28
+ {
29
+ "id": 6,
30
+ "group_id": 3
31
+ },
32
+ {
33
+ "id": 7,
34
+ "group_id": 1
35
+ },
36
+ {
37
+ "id": 8,
38
+ "group_id": 2
39
+ },
40
+ {
41
+ "id": 9,
42
+ "group_id": 3
43
+ },
44
+ {
45
+ "id": 10,
46
+ "group_id": 1
47
+ },
48
+ {
49
+ "id": 11,
50
+ "group_id": 2
51
+ },
52
+ {
53
+ "id": 12,
54
+ "group_id": 3
55
+ },
56
+ {
57
+ "id": 13,
58
+ "group_id": 1
59
+ },
60
+ {
61
+ "id": 14,
62
+ "group_id": 2
63
+ },
64
+ {
65
+ "id": 15,
66
+ "group_id": 3
67
+ },
68
+ {
69
+ "id": 16,
70
+ "group_id": 1
71
+ },
72
+ {
73
+ "id": 17,
74
+ "group_id": 2
75
+ },
76
+ {
77
+ "id": 18,
78
+ "group_id": 3
79
+ },
80
+ {
81
+ "id": 19,
82
+ "group_id": 1
83
+ },
84
+ {
85
+ "id": 20,
86
+ "group_id": 2
87
+ }
88
+ ]
89
+ }
@@ -0,0 +1,59 @@
1
+ {
2
+ "id": 4,
3
+ "name": "grouped_priority",
4
+ "prioritized": true,
5
+ "pairwise": false,
6
+ "grouped": true,
7
+ "entries": [
8
+ {
9
+ "id": 8,
10
+ "priority": 2,
11
+ "group_id": 2
12
+ },
13
+ {
14
+ "id": 7,
15
+ "priority": 3,
16
+ "group_id": 1
17
+ },
18
+ {
19
+ "id": 10,
20
+ "priority": 0,
21
+ "group_id": 1
22
+ },
23
+ {
24
+ "id": 5,
25
+ "priority": 5,
26
+ "group_id": 2
27
+ },
28
+ {
29
+ "id": 6,
30
+ "priority": 4,
31
+ "group_id": 3
32
+ },
33
+ {
34
+ "id": 2,
35
+ "priority": 8,
36
+ "group_id": 2
37
+ },
38
+ {
39
+ "id": 3,
40
+ "priority": 7,
41
+ "group_id": 3
42
+ },
43
+ {
44
+ "id": 1,
45
+ "priority": 9,
46
+ "group_id": 1
47
+ },
48
+ {
49
+ "id": 9,
50
+ "priority": 1,
51
+ "group_id": 3
52
+ },
53
+ {
54
+ "id": 4,
55
+ "priority": 6,
56
+ "group_id": 1
57
+ }
58
+ ]
59
+ }