cellect-server 0.1.3 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.gitignore +1 -0
- data/.travis.yml +2 -8
- data/cellect-server.gemspec +4 -4
- data/cellect.gemspec +4 -3
- data/lib/cellect.rb +1 -1
- data/lib/cellect/node_set.rb +12 -12
- data/lib/cellect/server.rb +4 -4
- data/lib/cellect/server/adapters.rb +2 -2
- data/lib/cellect/server/adapters/default.rb +6 -6
- data/lib/cellect/server/adapters/postgres.rb +5 -5
- data/lib/cellect/server/api.rb +9 -9
- data/lib/cellect/server/api/helpers.rb +5 -5
- data/lib/cellect/server/api/sets.rb +2 -2
- data/lib/cellect/server/api/users.rb +5 -5
- data/lib/cellect/server/grouped_workflow.rb +10 -10
- data/lib/cellect/server/node_set.rb +2 -2
- data/lib/cellect/server/user.rb +10 -10
- data/lib/cellect/server/workflow.rb +21 -21
- data/lib/cellect/version.rb +1 -1
- data/spec/server/api/add_seen_spec.rb +2 -2
- data/spec/server/api/add_spec.rb +4 -4
- data/spec/server/api/remove_spec.rb +4 -4
- data/spec/server/api/sample_spec.rb +3 -3
- data/spec/server/api/user_load_spec.rb +2 -2
- data/spec/server/grouped_workflow_spec.rb +10 -10
- data/spec/server/node_set_spec.rb +1 -1
- data/spec/server/server_spec.rb +2 -2
- data/spec/server/user_spec.rb +5 -5
- data/spec/server/workflow_spec.rb +9 -9
- data/spec/spec_helper.rb +3 -4
- data/spec/support/shared_api_context.rb +2 -2
- data/spec/support/shared_examples_for_node_set.rb +3 -3
- data/spec/support/shared_examples_for_set.rb +6 -6
- data/spec/support/shared_examples_for_workflow.rb +5 -5
- data/spec/support/spec_adapter.rb +6 -6
- data/spec/support/zk_setup.rb +48 -25
- metadata +3 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: efa9bdf71d7f86007ecd6f45dbe0f5cf0893f924
|
4
|
+
data.tar.gz: 07fa08cfb0ff77ad4c099f9ada1a4bebedabb7e1
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 2733ecff41bcb63e5d5e135e0e68701356e32bce5d02e13110fe009a0c7e00ef28899d83c63a0d4304e145ba4889b648e73f323ba8fe38405daa206ad24ce349
|
7
|
+
data.tar.gz: b8ec41a5c735f5af7fbc2feb767b023f56d8a35e6a400d037b5a35b376673da538c537ba65fa95bcb606080fc2d8537871c6ff2b536cc3a47912231c36768b75
|
data/.gitignore
CHANGED
data/.travis.yml
CHANGED
@@ -5,16 +5,10 @@ before_install:
|
|
5
5
|
- sudo add-apt-repository ppa:boost-latest/ppa -y
|
6
6
|
- sudo apt-get update -q
|
7
7
|
- sudo apt-get install -y autoconf automake boost1.55 libffi-dev
|
8
|
-
|
9
|
-
before_script:
|
10
|
-
- test `which zkServer.sh` && zkServer.sh stop && rm -rf /tmp/zookeeper/*; true
|
11
|
-
- wget -q -O - "http://mirrors.koehn.com/apache/zookeeper/zookeeper-3.4.6/zookeeper-3.4.6.tar.gz" | tar zx -C "$HOME/"
|
12
|
-
- export PATH="$PATH:/$HOME/zookeeper-3.4.6/bin"
|
13
|
-
- cp "$HOME/zookeeper-3.4.6/conf/zoo_sample.cfg" "$HOME/zookeeper-3.4.6/conf/zoo.cfg"
|
14
|
-
- zkServer.sh start
|
8
|
+
- gem update bundler
|
15
9
|
|
16
10
|
rvm:
|
17
11
|
- 2.1.5
|
18
12
|
- 2.2.1
|
19
13
|
|
20
|
-
script:
|
14
|
+
script: bundle exec rake spec
|
data/cellect-server.gemspec
CHANGED
@@ -12,15 +12,15 @@ Gem::Specification.new do |spec|
|
|
12
12
|
spec.description = ''
|
13
13
|
spec.homepage = 'https://github.com/parrish/Cellect'
|
14
14
|
spec.license = 'MIT'
|
15
|
-
|
15
|
+
|
16
16
|
ignored_paths = %w(config data log script tmp).collect{ |path| Dir["#{ path }/**/*"] }.flatten
|
17
17
|
ignored_files = %w(Dockerfile Vagrantfile Gemfile.lock config.ru) + ignored_paths
|
18
|
-
|
18
|
+
|
19
19
|
spec.files = `git ls-files -z`.split("\x0").reject{ |f| f =~ /client/ } - ignored_files
|
20
20
|
spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
|
21
21
|
spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
|
22
22
|
spec.require_paths = ['lib']
|
23
|
-
|
23
|
+
|
24
24
|
spec.add_development_dependency 'bundler', '~> 1.5'
|
25
25
|
spec.add_development_dependency 'rake'
|
26
26
|
spec.add_development_dependency 'oj'
|
@@ -30,7 +30,7 @@ Gem::Specification.new do |spec|
|
|
30
30
|
spec.add_development_dependency 'puma', '~> 2.8'
|
31
31
|
spec.add_development_dependency 'pg', '~> 0.17'
|
32
32
|
spec.add_development_dependency 'connection_pool', '~> 2.0'
|
33
|
-
|
33
|
+
|
34
34
|
spec.add_runtime_dependency 'diff_set', '~> 0.0.3'
|
35
35
|
spec.add_runtime_dependency 'celluloid', '0.16.0'
|
36
36
|
spec.add_runtime_dependency 'celluloid-io', '0.16.0'
|
data/cellect.gemspec
CHANGED
@@ -12,23 +12,24 @@ Gem::Specification.new do |spec|
|
|
12
12
|
spec.description = ''
|
13
13
|
spec.homepage = 'https://github.com/parrish/Cellect'
|
14
14
|
spec.license = 'MIT'
|
15
|
-
|
15
|
+
|
16
16
|
spec.files = ['lib/cellect.rb', 'lib/cellect/version.rb']
|
17
17
|
spec.executables = []
|
18
18
|
spec.test_files = []
|
19
19
|
spec.require_paths = ['lib']
|
20
|
-
|
20
|
+
|
21
21
|
spec.add_development_dependency 'bundler', '~> 1.5'
|
22
22
|
spec.add_development_dependency 'rake'
|
23
23
|
spec.add_development_dependency 'oj'
|
24
24
|
spec.add_development_dependency 'rspec'
|
25
25
|
spec.add_development_dependency 'rack-test'
|
26
|
+
spec.add_development_dependency 'zk-server'
|
26
27
|
spec.add_development_dependency 'pry'
|
27
28
|
spec.add_development_dependency 'puma', '~> 2.8'
|
28
29
|
spec.add_development_dependency 'pg', '~> 0.17'
|
29
30
|
spec.add_development_dependency 'connection_pool', '~> 2.0'
|
30
31
|
spec.add_development_dependency 'codeclimate-test-reporter'
|
31
|
-
|
32
|
+
|
32
33
|
spec.add_runtime_dependency 'cellect-server', Cellect::VERSION
|
33
34
|
spec.add_runtime_dependency 'cellect-client', Cellect::VERSION
|
34
35
|
end
|
data/lib/cellect.rb
CHANGED
data/lib/cellect/node_set.rb
CHANGED
@@ -1,39 +1,39 @@
|
|
1
1
|
require 'zk'
|
2
|
-
require 'timeout'
|
3
2
|
|
4
3
|
module Cellect
|
5
4
|
class NodeSet
|
6
5
|
include Celluloid
|
7
|
-
|
6
|
+
ConnectionError = Class.new(StandardError)
|
7
|
+
|
8
8
|
attr_accessor :zk, :state
|
9
|
-
|
9
|
+
|
10
10
|
# Sets up the node set and starts connecting to ZooKeeper
|
11
11
|
def initialize(zk_url = nil)
|
12
12
|
@zk_url = zk_url
|
13
13
|
self.state = :initializing
|
14
14
|
after(0.001){ async.initialize_zk } # don't block waiting for ZK to connect
|
15
15
|
end
|
16
|
-
|
16
|
+
|
17
17
|
# Connect to ZooKeeper, setup this node, and change state
|
18
18
|
def initialize_zk
|
19
|
-
# don't let ZK hang the thread,
|
20
|
-
|
21
|
-
|
22
|
-
|
19
|
+
# don't let ZK hang the thread, timeout and check connection status
|
20
|
+
zk = ZK::Client.new zk_url, timeout: 0.5, chroot: '/cellect'
|
21
|
+
raise ConnectionError.new("Can't connect to ZK server.") unless zk.connected?
|
22
|
+
self.zk = zk
|
23
23
|
setup
|
24
24
|
self.state = :ready
|
25
25
|
end
|
26
|
-
|
26
|
+
|
27
27
|
def ready?
|
28
28
|
state == :ready
|
29
29
|
end
|
30
|
-
|
30
|
+
|
31
31
|
protected
|
32
|
-
|
32
|
+
|
33
33
|
def zk_url
|
34
34
|
@zk_url || ENV.fetch('ZK_URL', 'localhost:2181')
|
35
35
|
end
|
36
|
-
|
36
|
+
|
37
37
|
def setup
|
38
38
|
# Specialized in subclasses
|
39
39
|
end
|
data/lib/cellect/server.rb
CHANGED
@@ -10,22 +10,22 @@ module Cellect
|
|
10
10
|
require 'cellect/server/grouped_workflow'
|
11
11
|
require 'cellect/server/user'
|
12
12
|
require 'cellect/server/api'
|
13
|
-
|
13
|
+
|
14
14
|
class << self
|
15
15
|
attr_accessor :node_set
|
16
16
|
end
|
17
|
-
|
17
|
+
|
18
18
|
# The server is ready when all workflows have finished loading
|
19
19
|
def self.ready?
|
20
20
|
Workflow.all.each do |workflow|
|
21
21
|
return false unless workflow.ready?
|
22
22
|
end
|
23
|
-
|
23
|
+
|
24
24
|
true
|
25
25
|
rescue
|
26
26
|
false
|
27
27
|
end
|
28
|
-
|
28
|
+
|
29
29
|
Server.node_set = NodeSet.supervise
|
30
30
|
end
|
31
31
|
end
|
@@ -13,7 +13,7 @@ module Cellect
|
|
13
13
|
def workflow_list(*names)
|
14
14
|
raise NotImplementedError
|
15
15
|
end
|
16
|
-
|
16
|
+
|
17
17
|
# Load the data for a workflow, this method:
|
18
18
|
# Accepts a workflow
|
19
19
|
# Returns an array of hashes in the form:
|
@@ -25,7 +25,7 @@ module Cellect
|
|
25
25
|
def load_data_for(workflow_name)
|
26
26
|
raise NotImplementedError
|
27
27
|
end
|
28
|
-
|
28
|
+
|
29
29
|
# Load seen ids for a user, this method:
|
30
30
|
# Accepts a workflow_name, and a user id
|
31
31
|
# Returns an array in the form:
|
@@ -33,11 +33,11 @@ module Cellect
|
|
33
33
|
def load_user(workflow_name, id)
|
34
34
|
raise NotImplementedError
|
35
35
|
end
|
36
|
-
|
36
|
+
|
37
37
|
def load_workflows(*names)
|
38
38
|
workflow_list(*names).each{ |workflow_info| load_workflow workflow_info }
|
39
39
|
end
|
40
|
-
|
40
|
+
|
41
41
|
def load_workflow(args)
|
42
42
|
info = if args.is_a?(Hash)
|
43
43
|
args
|
@@ -46,10 +46,10 @@ module Cellect
|
|
46
46
|
else
|
47
47
|
raise ArgumentError
|
48
48
|
end
|
49
|
-
|
49
|
+
|
50
50
|
workflow_for info
|
51
51
|
end
|
52
|
-
|
52
|
+
|
53
53
|
def workflow_for(opts = { })
|
54
54
|
workflow_klass = opts.fetch('grouped', false) ? GroupedWorkflow : Workflow
|
55
55
|
workflow_klass[opts['name']] = opts
|
@@ -10,7 +10,7 @@ module Cellect
|
|
10
10
|
PG.connect connection_options
|
11
11
|
end
|
12
12
|
end
|
13
|
-
|
13
|
+
|
14
14
|
def workflow_list(*names)
|
15
15
|
with_pg do |pg|
|
16
16
|
statement = 'SELECT * FROM workflows'
|
@@ -33,7 +33,7 @@ module Cellect
|
|
33
33
|
end
|
34
34
|
end
|
35
35
|
end
|
36
|
-
|
36
|
+
|
37
37
|
def load_data_for(workflow_name)
|
38
38
|
with_pg do |pg|
|
39
39
|
statement = <<-SQL
|
@@ -52,7 +52,7 @@ module Cellect
|
|
52
52
|
end
|
53
53
|
end
|
54
54
|
end
|
55
|
-
|
55
|
+
|
56
56
|
def load_user(workflow_name, id)
|
57
57
|
with_pg do |pg|
|
58
58
|
statement = <<-SQL
|
@@ -64,11 +64,11 @@ module Cellect
|
|
64
64
|
end
|
65
65
|
end
|
66
66
|
end
|
67
|
-
|
67
|
+
|
68
68
|
def with_pg
|
69
69
|
@pg.with{ |pg| yield pg }
|
70
70
|
end
|
71
|
-
|
71
|
+
|
72
72
|
def connection_options
|
73
73
|
{
|
74
74
|
host: ENV.fetch('PG_HOST', '127.0.0.1'),
|
data/lib/cellect/server/api.rb
CHANGED
@@ -4,11 +4,11 @@ module Cellect
|
|
4
4
|
module Server
|
5
5
|
class API < Grape::API
|
6
6
|
format :json
|
7
|
-
|
7
|
+
|
8
8
|
require 'cellect/server/api/helpers'
|
9
9
|
require 'cellect/server/api/sets'
|
10
10
|
require 'cellect/server/api/users'
|
11
|
-
|
11
|
+
|
12
12
|
# GET /stats
|
13
13
|
#
|
14
14
|
# Provides system load information
|
@@ -16,41 +16,41 @@ module Cellect
|
|
16
16
|
usage = ->(keyword) do
|
17
17
|
`ps axo #{ keyword }`.chomp.split("\n")[1..-1].collect(&:to_f).inject :+
|
18
18
|
end
|
19
|
-
|
19
|
+
|
20
20
|
{
|
21
21
|
memory: usage.call('%mem'),
|
22
22
|
cpu: usage.call('%cpu')
|
23
23
|
}
|
24
24
|
end
|
25
|
-
|
25
|
+
|
26
26
|
resources :workflows do
|
27
|
-
|
27
|
+
|
28
28
|
# GET /workflows
|
29
29
|
#
|
30
30
|
# Returns a list of available workflows
|
31
31
|
get do
|
32
32
|
Cellect::Server.adapter.workflow_list
|
33
33
|
end
|
34
|
-
|
34
|
+
|
35
35
|
segment '/:workflow_id' do
|
36
36
|
helpers Helpers
|
37
37
|
mount Sets
|
38
38
|
mount Users
|
39
|
-
|
39
|
+
|
40
40
|
# GET /workflows/:workflow_id/status
|
41
41
|
#
|
42
42
|
# Returns the workflow's status
|
43
43
|
get :status do
|
44
44
|
workflow.status
|
45
45
|
end
|
46
|
-
|
46
|
+
|
47
47
|
# POST /workflows/:workflow_id/reload
|
48
48
|
#
|
49
49
|
# Reloads the workflow from the adapter
|
50
50
|
post :reload do
|
51
51
|
workflow.async.load_data
|
52
52
|
end
|
53
|
-
|
53
|
+
|
54
54
|
# DELETE /workflows/:workflow_id
|
55
55
|
#
|
56
56
|
# Not implemented
|
@@ -5,7 +5,7 @@ module Cellect
|
|
5
5
|
def workflow
|
6
6
|
@workflow ||= Workflow[params[:workflow_id]]
|
7
7
|
end
|
8
|
-
|
8
|
+
|
9
9
|
def selector_params
|
10
10
|
{
|
11
11
|
limit: param_to_int(:limit, default: 5),
|
@@ -13,7 +13,7 @@ module Cellect
|
|
13
13
|
group_id: param_to_int(:group_id)
|
14
14
|
}
|
15
15
|
end
|
16
|
-
|
16
|
+
|
17
17
|
def update_params
|
18
18
|
{
|
19
19
|
subject_id: param_to_int(:subject_id),
|
@@ -21,15 +21,15 @@ module Cellect
|
|
21
21
|
priority: param_to_float(:priority)
|
22
22
|
}
|
23
23
|
end
|
24
|
-
|
24
|
+
|
25
25
|
def param_to_int(param, default: nil)
|
26
26
|
_param_to param, :to_i, default
|
27
27
|
end
|
28
|
-
|
28
|
+
|
29
29
|
def param_to_float(param, default: nil)
|
30
30
|
_param_to param, :to_f, default
|
31
31
|
end
|
32
|
-
|
32
|
+
|
33
33
|
def _param_to(param, conversion, default)
|
34
34
|
val = params[param].try conversion
|
35
35
|
params[param] && val && val > 0 ? val : default
|
@@ -12,7 +12,7 @@ module Cellect
|
|
12
12
|
get do
|
13
13
|
workflow.sample selector_params
|
14
14
|
end
|
15
|
-
|
15
|
+
|
16
16
|
# PUT /workflows/:workflow_id/add
|
17
17
|
#
|
18
18
|
# Adds a subject to a workflow or updates the priority
|
@@ -24,7 +24,7 @@ module Cellect
|
|
24
24
|
workflow.add update_params
|
25
25
|
nil
|
26
26
|
end
|
27
|
-
|
27
|
+
|
28
28
|
# PUT /workflows/:workflow_id/remove
|
29
29
|
#
|
30
30
|
# Removes a subject from a workflow
|
@@ -12,24 +12,24 @@ module Cellect
|
|
12
12
|
put :add_seen do
|
13
13
|
user_id = param_to_int :user_id
|
14
14
|
subject_id = param_to_int :subject_id
|
15
|
-
|
15
|
+
|
16
16
|
if user_id && user_id > 0 && subject_id && subject_id > 0
|
17
17
|
workflow.async.add_seen_for user_id, subject_id
|
18
18
|
end
|
19
|
-
|
19
|
+
|
20
20
|
nil
|
21
21
|
end
|
22
|
-
|
22
|
+
|
23
23
|
# POST /workflows/:workflow_id/users/:user_id/load
|
24
24
|
#
|
25
25
|
# Preloads a user for a workflow
|
26
26
|
post :load do
|
27
27
|
user_id = param_to_int :user_id
|
28
|
-
|
28
|
+
|
29
29
|
if user_id && user_id > 0
|
30
30
|
workflow.async.user user_id
|
31
31
|
end
|
32
|
-
|
32
|
+
|
33
33
|
nil
|
34
34
|
end
|
35
35
|
end
|
@@ -2,13 +2,13 @@ module Cellect
|
|
2
2
|
module Server
|
3
3
|
class GroupedWorkflow < Workflow
|
4
4
|
attr_accessor :groups
|
5
|
-
|
5
|
+
|
6
6
|
# Sets up the new workflow
|
7
7
|
def initialize(name, pairwise: false, prioritized: false)
|
8
8
|
self.groups = { }
|
9
9
|
super
|
10
10
|
end
|
11
|
-
|
11
|
+
|
12
12
|
# Load subjects from the adapter into their groups
|
13
13
|
def load_data
|
14
14
|
self.state = :initializing
|
@@ -20,17 +20,17 @@ module Cellect
|
|
20
20
|
end
|
21
21
|
self.state = :ready
|
22
22
|
end
|
23
|
-
|
23
|
+
|
24
24
|
# Returns a group by id or samples one randomly
|
25
25
|
def group(group_id = nil)
|
26
26
|
groups[group_id] || groups.values.sample
|
27
27
|
end
|
28
|
-
|
28
|
+
|
29
29
|
# Get unseen subjects from a group for a user
|
30
30
|
def unseen_for(user_name, group_id: nil, limit: 5)
|
31
31
|
group(group_id).subtract user(user_name).seen, limit
|
32
32
|
end
|
33
|
-
|
33
|
+
|
34
34
|
# Get a sample of subjects from a group for a user
|
35
35
|
#
|
36
36
|
# Accepts a hash in the form:
|
@@ -46,7 +46,7 @@ module Cellect
|
|
46
46
|
group(opts[:group_id]).sample opts[:limit]
|
47
47
|
end
|
48
48
|
end
|
49
|
-
|
49
|
+
|
50
50
|
# Adds or updates a subject in a group
|
51
51
|
#
|
52
52
|
# Accepts a hash in the form:
|
@@ -62,7 +62,7 @@ module Cellect
|
|
62
62
|
groups[opts[:group_id]].add opts[:subject_id]
|
63
63
|
end
|
64
64
|
end
|
65
|
-
|
65
|
+
|
66
66
|
# Removes a subject from a group
|
67
67
|
#
|
68
68
|
# Accepts a hash in the form:
|
@@ -73,19 +73,19 @@ module Cellect
|
|
73
73
|
def remove(opts = { })
|
74
74
|
groups[opts[:group_id]].remove opts[:subject_id]
|
75
75
|
end
|
76
|
-
|
76
|
+
|
77
77
|
# General information about this workflow
|
78
78
|
def status
|
79
79
|
# Get the number of subjects in each group
|
80
80
|
group_counts = Hash[*groups.collect{ |id, set| [id, set.size] }.flatten]
|
81
|
-
|
81
|
+
|
82
82
|
super.merge({
|
83
83
|
grouped: true,
|
84
84
|
subjects: group_counts.values.inject(:+),
|
85
85
|
groups: group_counts
|
86
86
|
})
|
87
87
|
end
|
88
|
-
|
88
|
+
|
89
89
|
def grouped?
|
90
90
|
true
|
91
91
|
end
|