cellect-server 0.1.3 → 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (38) hide show
  1. checksums.yaml +4 -4
  2. data/.gitignore +1 -0
  3. data/.travis.yml +2 -8
  4. data/cellect-server.gemspec +4 -4
  5. data/cellect.gemspec +4 -3
  6. data/lib/cellect.rb +1 -1
  7. data/lib/cellect/node_set.rb +12 -12
  8. data/lib/cellect/server.rb +4 -4
  9. data/lib/cellect/server/adapters.rb +2 -2
  10. data/lib/cellect/server/adapters/default.rb +6 -6
  11. data/lib/cellect/server/adapters/postgres.rb +5 -5
  12. data/lib/cellect/server/api.rb +9 -9
  13. data/lib/cellect/server/api/helpers.rb +5 -5
  14. data/lib/cellect/server/api/sets.rb +2 -2
  15. data/lib/cellect/server/api/users.rb +5 -5
  16. data/lib/cellect/server/grouped_workflow.rb +10 -10
  17. data/lib/cellect/server/node_set.rb +2 -2
  18. data/lib/cellect/server/user.rb +10 -10
  19. data/lib/cellect/server/workflow.rb +21 -21
  20. data/lib/cellect/version.rb +1 -1
  21. data/spec/server/api/add_seen_spec.rb +2 -2
  22. data/spec/server/api/add_spec.rb +4 -4
  23. data/spec/server/api/remove_spec.rb +4 -4
  24. data/spec/server/api/sample_spec.rb +3 -3
  25. data/spec/server/api/user_load_spec.rb +2 -2
  26. data/spec/server/grouped_workflow_spec.rb +10 -10
  27. data/spec/server/node_set_spec.rb +1 -1
  28. data/spec/server/server_spec.rb +2 -2
  29. data/spec/server/user_spec.rb +5 -5
  30. data/spec/server/workflow_spec.rb +9 -9
  31. data/spec/spec_helper.rb +3 -4
  32. data/spec/support/shared_api_context.rb +2 -2
  33. data/spec/support/shared_examples_for_node_set.rb +3 -3
  34. data/spec/support/shared_examples_for_set.rb +6 -6
  35. data/spec/support/shared_examples_for_workflow.rb +5 -5
  36. data/spec/support/spec_adapter.rb +6 -6
  37. data/spec/support/zk_setup.rb +48 -25
  38. metadata +3 -3
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: ce5ed6155919dbcb28b9deed2f937b67dbe726e0
4
- data.tar.gz: b08a7bf06a958740ff668b927a970be4f6b17b9d
3
+ metadata.gz: efa9bdf71d7f86007ecd6f45dbe0f5cf0893f924
4
+ data.tar.gz: 07fa08cfb0ff77ad4c099f9ada1a4bebedabb7e1
5
5
  SHA512:
6
- metadata.gz: 278e910a81b231c326267c2a8be769c7759f3ec815a1fd95e9ced36a26be3e838642746832a5d6c646d3ee78b7e8f0337af860d55e2d297b054b5a3102e8f579
7
- data.tar.gz: 86e06b066a4d8a308949cc23356f3dac6f06d2acd5341196cbd1007c9bb964dae696fa23539b3951e10c55c050b8c1c0aebd8b381199b80d5c63c74bb770d01a
6
+ metadata.gz: 2733ecff41bcb63e5d5e135e0e68701356e32bce5d02e13110fe009a0c7e00ef28899d83c63a0d4304e145ba4889b648e73f323ba8fe38405daa206ad24ce349
7
+ data.tar.gz: b8ec41a5c735f5af7fbc2feb767b023f56d8a35e6a400d037b5a35b376673da538c537ba65fa95bcb606080fc2d8537871c6ff2b536cc3a47912231c36768b75
data/.gitignore CHANGED
@@ -16,3 +16,4 @@ pkg/*
16
16
  *.swp
17
17
  ext/
18
18
  /coverage
19
+ /Gemfile.lock
data/.travis.yml CHANGED
@@ -5,16 +5,10 @@ before_install:
5
5
  - sudo add-apt-repository ppa:boost-latest/ppa -y
6
6
  - sudo apt-get update -q
7
7
  - sudo apt-get install -y autoconf automake boost1.55 libffi-dev
8
-
9
- before_script:
10
- - test `which zkServer.sh` && zkServer.sh stop && rm -rf /tmp/zookeeper/*; true
11
- - wget -q -O - "http://mirrors.koehn.com/apache/zookeeper/zookeeper-3.4.6/zookeeper-3.4.6.tar.gz" | tar zx -C "$HOME/"
12
- - export PATH="$PATH:/$HOME/zookeeper-3.4.6/bin"
13
- - cp "$HOME/zookeeper-3.4.6/conf/zoo_sample.cfg" "$HOME/zookeeper-3.4.6/conf/zoo.cfg"
14
- - zkServer.sh start
8
+ - gem update bundler
15
9
 
16
10
  rvm:
17
11
  - 2.1.5
18
12
  - 2.2.1
19
13
 
20
- script: ZK_URL=localhost:2181 bundle exec rake
14
+ script: bundle exec rake spec
@@ -12,15 +12,15 @@ Gem::Specification.new do |spec|
12
12
  spec.description = ''
13
13
  spec.homepage = 'https://github.com/parrish/Cellect'
14
14
  spec.license = 'MIT'
15
-
15
+
16
16
  ignored_paths = %w(config data log script tmp).collect{ |path| Dir["#{ path }/**/*"] }.flatten
17
17
  ignored_files = %w(Dockerfile Vagrantfile Gemfile.lock config.ru) + ignored_paths
18
-
18
+
19
19
  spec.files = `git ls-files -z`.split("\x0").reject{ |f| f =~ /client/ } - ignored_files
20
20
  spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
21
21
  spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
22
22
  spec.require_paths = ['lib']
23
-
23
+
24
24
  spec.add_development_dependency 'bundler', '~> 1.5'
25
25
  spec.add_development_dependency 'rake'
26
26
  spec.add_development_dependency 'oj'
@@ -30,7 +30,7 @@ Gem::Specification.new do |spec|
30
30
  spec.add_development_dependency 'puma', '~> 2.8'
31
31
  spec.add_development_dependency 'pg', '~> 0.17'
32
32
  spec.add_development_dependency 'connection_pool', '~> 2.0'
33
-
33
+
34
34
  spec.add_runtime_dependency 'diff_set', '~> 0.0.3'
35
35
  spec.add_runtime_dependency 'celluloid', '0.16.0'
36
36
  spec.add_runtime_dependency 'celluloid-io', '0.16.0'
data/cellect.gemspec CHANGED
@@ -12,23 +12,24 @@ Gem::Specification.new do |spec|
12
12
  spec.description = ''
13
13
  spec.homepage = 'https://github.com/parrish/Cellect'
14
14
  spec.license = 'MIT'
15
-
15
+
16
16
  spec.files = ['lib/cellect.rb', 'lib/cellect/version.rb']
17
17
  spec.executables = []
18
18
  spec.test_files = []
19
19
  spec.require_paths = ['lib']
20
-
20
+
21
21
  spec.add_development_dependency 'bundler', '~> 1.5'
22
22
  spec.add_development_dependency 'rake'
23
23
  spec.add_development_dependency 'oj'
24
24
  spec.add_development_dependency 'rspec'
25
25
  spec.add_development_dependency 'rack-test'
26
+ spec.add_development_dependency 'zk-server'
26
27
  spec.add_development_dependency 'pry'
27
28
  spec.add_development_dependency 'puma', '~> 2.8'
28
29
  spec.add_development_dependency 'pg', '~> 0.17'
29
30
  spec.add_development_dependency 'connection_pool', '~> 2.0'
30
31
  spec.add_development_dependency 'codeclimate-test-reporter'
31
-
32
+
32
33
  spec.add_runtime_dependency 'cellect-server', Cellect::VERSION
33
34
  spec.add_runtime_dependency 'cellect-client', Cellect::VERSION
34
35
  end
data/lib/cellect.rb CHANGED
@@ -3,5 +3,5 @@ require 'celluloid/io'
3
3
  require 'cellect/version'
4
4
 
5
5
  module Cellect
6
-
6
+
7
7
  end
@@ -1,39 +1,39 @@
1
1
  require 'zk'
2
- require 'timeout'
3
2
 
4
3
  module Cellect
5
4
  class NodeSet
6
5
  include Celluloid
7
-
6
+ ConnectionError = Class.new(StandardError)
7
+
8
8
  attr_accessor :zk, :state
9
-
9
+
10
10
  # Sets up the node set and starts connecting to ZooKeeper
11
11
  def initialize(zk_url = nil)
12
12
  @zk_url = zk_url
13
13
  self.state = :initializing
14
14
  after(0.001){ async.initialize_zk } # don't block waiting for ZK to connect
15
15
  end
16
-
16
+
17
17
  # Connect to ZooKeeper, setup this node, and change state
18
18
  def initialize_zk
19
- # don't let ZK hang the thread, just retry connection on restart
20
- Timeout::timeout(5) do
21
- self.zk = ZK.new zk_url, chroot: '/cellect'
22
- end
19
+ # don't let ZK hang the thread, timeout and check connection status
20
+ zk = ZK::Client.new zk_url, timeout: 0.5, chroot: '/cellect'
21
+ raise ConnectionError.new("Can't connect to ZK server.") unless zk.connected?
22
+ self.zk = zk
23
23
  setup
24
24
  self.state = :ready
25
25
  end
26
-
26
+
27
27
  def ready?
28
28
  state == :ready
29
29
  end
30
-
30
+
31
31
  protected
32
-
32
+
33
33
  def zk_url
34
34
  @zk_url || ENV.fetch('ZK_URL', 'localhost:2181')
35
35
  end
36
-
36
+
37
37
  def setup
38
38
  # Specialized in subclasses
39
39
  end
@@ -10,22 +10,22 @@ module Cellect
10
10
  require 'cellect/server/grouped_workflow'
11
11
  require 'cellect/server/user'
12
12
  require 'cellect/server/api'
13
-
13
+
14
14
  class << self
15
15
  attr_accessor :node_set
16
16
  end
17
-
17
+
18
18
  # The server is ready when all workflows have finished loading
19
19
  def self.ready?
20
20
  Workflow.all.each do |workflow|
21
21
  return false unless workflow.ready?
22
22
  end
23
-
23
+
24
24
  true
25
25
  rescue
26
26
  false
27
27
  end
28
-
28
+
29
29
  Server.node_set = NodeSet.supervise
30
30
  end
31
31
  end
@@ -3,11 +3,11 @@ module Cellect
3
3
  class << self
4
4
  attr_accessor :adapter
5
5
  end
6
-
6
+
7
7
  module Adapters
8
8
  require 'cellect/server/adapters/default'
9
9
  end
10
-
10
+
11
11
  Cellect::Server.adapter = Adapters::Default.new
12
12
  end
13
13
  end
@@ -13,7 +13,7 @@ module Cellect
13
13
  def workflow_list(*names)
14
14
  raise NotImplementedError
15
15
  end
16
-
16
+
17
17
  # Load the data for a workflow, this method:
18
18
  # Accepts a workflow
19
19
  # Returns an array of hashes in the form:
@@ -25,7 +25,7 @@ module Cellect
25
25
  def load_data_for(workflow_name)
26
26
  raise NotImplementedError
27
27
  end
28
-
28
+
29
29
  # Load seen ids for a user, this method:
30
30
  # Accepts a workflow_name, and a user id
31
31
  # Returns an array in the form:
@@ -33,11 +33,11 @@ module Cellect
33
33
  def load_user(workflow_name, id)
34
34
  raise NotImplementedError
35
35
  end
36
-
36
+
37
37
  def load_workflows(*names)
38
38
  workflow_list(*names).each{ |workflow_info| load_workflow workflow_info }
39
39
  end
40
-
40
+
41
41
  def load_workflow(args)
42
42
  info = if args.is_a?(Hash)
43
43
  args
@@ -46,10 +46,10 @@ module Cellect
46
46
  else
47
47
  raise ArgumentError
48
48
  end
49
-
49
+
50
50
  workflow_for info
51
51
  end
52
-
52
+
53
53
  def workflow_for(opts = { })
54
54
  workflow_klass = opts.fetch('grouped', false) ? GroupedWorkflow : Workflow
55
55
  workflow_klass[opts['name']] = opts
@@ -10,7 +10,7 @@ module Cellect
10
10
  PG.connect connection_options
11
11
  end
12
12
  end
13
-
13
+
14
14
  def workflow_list(*names)
15
15
  with_pg do |pg|
16
16
  statement = 'SELECT * FROM workflows'
@@ -33,7 +33,7 @@ module Cellect
33
33
  end
34
34
  end
35
35
  end
36
-
36
+
37
37
  def load_data_for(workflow_name)
38
38
  with_pg do |pg|
39
39
  statement = <<-SQL
@@ -52,7 +52,7 @@ module Cellect
52
52
  end
53
53
  end
54
54
  end
55
-
55
+
56
56
  def load_user(workflow_name, id)
57
57
  with_pg do |pg|
58
58
  statement = <<-SQL
@@ -64,11 +64,11 @@ module Cellect
64
64
  end
65
65
  end
66
66
  end
67
-
67
+
68
68
  def with_pg
69
69
  @pg.with{ |pg| yield pg }
70
70
  end
71
-
71
+
72
72
  def connection_options
73
73
  {
74
74
  host: ENV.fetch('PG_HOST', '127.0.0.1'),
@@ -4,11 +4,11 @@ module Cellect
4
4
  module Server
5
5
  class API < Grape::API
6
6
  format :json
7
-
7
+
8
8
  require 'cellect/server/api/helpers'
9
9
  require 'cellect/server/api/sets'
10
10
  require 'cellect/server/api/users'
11
-
11
+
12
12
  # GET /stats
13
13
  #
14
14
  # Provides system load information
@@ -16,41 +16,41 @@ module Cellect
16
16
  usage = ->(keyword) do
17
17
  `ps axo #{ keyword }`.chomp.split("\n")[1..-1].collect(&:to_f).inject :+
18
18
  end
19
-
19
+
20
20
  {
21
21
  memory: usage.call('%mem'),
22
22
  cpu: usage.call('%cpu')
23
23
  }
24
24
  end
25
-
25
+
26
26
  resources :workflows do
27
-
27
+
28
28
  # GET /workflows
29
29
  #
30
30
  # Returns a list of available workflows
31
31
  get do
32
32
  Cellect::Server.adapter.workflow_list
33
33
  end
34
-
34
+
35
35
  segment '/:workflow_id' do
36
36
  helpers Helpers
37
37
  mount Sets
38
38
  mount Users
39
-
39
+
40
40
  # GET /workflows/:workflow_id/status
41
41
  #
42
42
  # Returns the workflow's status
43
43
  get :status do
44
44
  workflow.status
45
45
  end
46
-
46
+
47
47
  # POST /workflows/:workflow_id/reload
48
48
  #
49
49
  # Reloads the workflow from the adapter
50
50
  post :reload do
51
51
  workflow.async.load_data
52
52
  end
53
-
53
+
54
54
  # DELETE /workflows/:workflow_id
55
55
  #
56
56
  # Not implemented
@@ -5,7 +5,7 @@ module Cellect
5
5
  def workflow
6
6
  @workflow ||= Workflow[params[:workflow_id]]
7
7
  end
8
-
8
+
9
9
  def selector_params
10
10
  {
11
11
  limit: param_to_int(:limit, default: 5),
@@ -13,7 +13,7 @@ module Cellect
13
13
  group_id: param_to_int(:group_id)
14
14
  }
15
15
  end
16
-
16
+
17
17
  def update_params
18
18
  {
19
19
  subject_id: param_to_int(:subject_id),
@@ -21,15 +21,15 @@ module Cellect
21
21
  priority: param_to_float(:priority)
22
22
  }
23
23
  end
24
-
24
+
25
25
  def param_to_int(param, default: nil)
26
26
  _param_to param, :to_i, default
27
27
  end
28
-
28
+
29
29
  def param_to_float(param, default: nil)
30
30
  _param_to param, :to_f, default
31
31
  end
32
-
32
+
33
33
  def _param_to(param, conversion, default)
34
34
  val = params[param].try conversion
35
35
  params[param] && val && val > 0 ? val : default
@@ -12,7 +12,7 @@ module Cellect
12
12
  get do
13
13
  workflow.sample selector_params
14
14
  end
15
-
15
+
16
16
  # PUT /workflows/:workflow_id/add
17
17
  #
18
18
  # Adds a subject to a workflow or updates the priority
@@ -24,7 +24,7 @@ module Cellect
24
24
  workflow.add update_params
25
25
  nil
26
26
  end
27
-
27
+
28
28
  # PUT /workflows/:workflow_id/remove
29
29
  #
30
30
  # Removes a subject from a workflow
@@ -12,24 +12,24 @@ module Cellect
12
12
  put :add_seen do
13
13
  user_id = param_to_int :user_id
14
14
  subject_id = param_to_int :subject_id
15
-
15
+
16
16
  if user_id && user_id > 0 && subject_id && subject_id > 0
17
17
  workflow.async.add_seen_for user_id, subject_id
18
18
  end
19
-
19
+
20
20
  nil
21
21
  end
22
-
22
+
23
23
  # POST /workflows/:workflow_id/users/:user_id/load
24
24
  #
25
25
  # Preloads a user for a workflow
26
26
  post :load do
27
27
  user_id = param_to_int :user_id
28
-
28
+
29
29
  if user_id && user_id > 0
30
30
  workflow.async.user user_id
31
31
  end
32
-
32
+
33
33
  nil
34
34
  end
35
35
  end
@@ -2,13 +2,13 @@ module Cellect
2
2
  module Server
3
3
  class GroupedWorkflow < Workflow
4
4
  attr_accessor :groups
5
-
5
+
6
6
  # Sets up the new workflow
7
7
  def initialize(name, pairwise: false, prioritized: false)
8
8
  self.groups = { }
9
9
  super
10
10
  end
11
-
11
+
12
12
  # Load subjects from the adapter into their groups
13
13
  def load_data
14
14
  self.state = :initializing
@@ -20,17 +20,17 @@ module Cellect
20
20
  end
21
21
  self.state = :ready
22
22
  end
23
-
23
+
24
24
  # Returns a group by id or samples one randomly
25
25
  def group(group_id = nil)
26
26
  groups[group_id] || groups.values.sample
27
27
  end
28
-
28
+
29
29
  # Get unseen subjects from a group for a user
30
30
  def unseen_for(user_name, group_id: nil, limit: 5)
31
31
  group(group_id).subtract user(user_name).seen, limit
32
32
  end
33
-
33
+
34
34
  # Get a sample of subjects from a group for a user
35
35
  #
36
36
  # Accepts a hash in the form:
@@ -46,7 +46,7 @@ module Cellect
46
46
  group(opts[:group_id]).sample opts[:limit]
47
47
  end
48
48
  end
49
-
49
+
50
50
  # Adds or updates a subject in a group
51
51
  #
52
52
  # Accepts a hash in the form:
@@ -62,7 +62,7 @@ module Cellect
62
62
  groups[opts[:group_id]].add opts[:subject_id]
63
63
  end
64
64
  end
65
-
65
+
66
66
  # Removes a subject from a group
67
67
  #
68
68
  # Accepts a hash in the form:
@@ -73,19 +73,19 @@ module Cellect
73
73
  def remove(opts = { })
74
74
  groups[opts[:group_id]].remove opts[:subject_id]
75
75
  end
76
-
76
+
77
77
  # General information about this workflow
78
78
  def status
79
79
  # Get the number of subjects in each group
80
80
  group_counts = Hash[*groups.collect{ |id, set| [id, set.size] }.flatten]
81
-
81
+
82
82
  super.merge({
83
83
  grouped: true,
84
84
  subjects: group_counts.values.inject(:+),
85
85
  groups: group_counts
86
86
  })
87
87
  end
88
-
88
+
89
89
  def grouped?
90
90
  true
91
91
  end