cellect-client 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: a945054d21ad4dccae1e6a4447fef9dcbf7f30d4
+  data.tar.gz: fee2b2070cff140fe90e41cce096e1f80d13e1d6
+SHA512:
+  metadata.gz: dc92be103ff6b6027bcd2b915430a9fe7877ed03077e71a8b00ac3f99e4f63c5b8c04db104a8abf1ab019453ed7f1fecd3ca8f5acd7032025f1f8c423e060a02
+  data.tar.gz: a7b7cfb49ff7b3c196d1d3e6711c70af644b828aa732a6dd1021c777a3d28d5900c09abe2413858995d359723dc094eae9dc654feccdb6fcae2a6768c66a562d

data/.gitignore

@@ -0,0 +1,15 @@
+.bundle
+.DS_Store
+.rvmrc
+.ruby-version
+/data/*
+/ext/*.o
+/ext/Makefile
+/ext/*.bundle
+/scratch
+*.so
+.vagrant
+log/*
+tmp/*
+*.gem
+pkg/*

data/.rspec

@@ -0,0 +1,2 @@
+--color
+--format progress

data/Gemfile

@@ -0,0 +1,2 @@
+source 'http://rubygems.org'
+gemspec name: 'cellect'

data/README.md

@@ -0,0 +1,19 @@
+# Cellect
+
+This is a work in progress
+
+## Building
+
+1. Install [Boost](http://www.boost.org/): OS X: `brew update && brew install boost`, Ubuntu: `sudo apt-get update && sudo apt-get install libboost-all-dev`
+2. Install gem dependencies: `bundle` (See Note)
+3. Build extension: `cd ext; ruby extconf.rb; make; cd ..`
+
+### Note
+To install rice your Ruby must be compiled with shared libraries enabled, from the rice docs: 
+* rvm:   `rvm reinstall [version] -- --enable-shared`
+* rbenv: `CONFIGURE_OPTS="--enable-shared" rbenv install [version]`
+
+
+## Testing
+
+Run the specs with `rake`

data/Rakefile

@@ -0,0 +1,9 @@
+require 'bundler/gem_helper'
+require 'rspec/core/rake_task'
+
+Bundler::GemHelper.install_tasks name: 'cellect'
+Bundler::GemHelper.install_tasks name: 'cellect-server'
+Bundler::GemHelper.install_tasks name: 'cellect-client'
+
+RSpec::Core::RakeTask.new :spec
+task default: :spec

data/cellect-client.gemspec

@@ -0,0 +1,35 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'cellect/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = 'cellect-client'
+  spec.version       = Cellect::VERSION
+  spec.authors       = ['Michael Parrish']
+  spec.email         = ['michael@zooniverse.org']
+  spec.summary       = ''
+  spec.description   = ''
+  spec.homepage      = 'https://github.com/parrish/Cellect'
+  spec.license       = 'MIT'
+  
+  ignored_paths = %w(config data log script tmp).collect{ |path| Dir["#{ path }/**/*"] }.flatten
+  ignored_files = %w(Dockerfile Vagrantfile Gemfile.lock config.ru) + ignored_paths
+  
+  spec.files         = `git ls-files -z`.split("\x0").reject{ |f| f =~ /server/ } - ignored_files
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ['lib']
+  
+  spec.add_development_dependency 'bundler', '~> 1.5'
+  spec.add_development_dependency 'rake'
+  spec.add_development_dependency 'oj'
+  spec.add_development_dependency 'rspec'
+  spec.add_development_dependency 'rack-test'
+  spec.add_development_dependency 'pry'
+  
+  spec.add_runtime_dependency 'celluloid', '0.16.0.pre'
+  spec.add_runtime_dependency 'celluloid-io', '0.16.0.pre'
+  spec.add_runtime_dependency 'http', '~> 0.6'
+  spec.add_runtime_dependency 'zk', '~> 1.9'
+end

data/cellect.gemspec

@@ -0,0 +1,33 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'cellect/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = 'cellect'
+  spec.version       = Cellect::VERSION
+  spec.authors       = ['Michael Parrish']
+  spec.email         = ['michael@zooniverse.org']
+  spec.summary       = ''
+  spec.description   = ''
+  spec.homepage      = 'https://github.com/parrish/Cellect'
+  spec.license       = 'MIT'
+  
+  spec.files         = ['lib/cellect.rb', 'lib/cellect/version.rb']
+  spec.executables   = []
+  spec.test_files    = []
+  spec.require_paths = ['lib']
+  
+  spec.add_development_dependency 'bundler', '~> 1.5'
+  spec.add_development_dependency 'rake'
+  spec.add_development_dependency 'oj'
+  spec.add_development_dependency 'rspec'
+  spec.add_development_dependency 'rack-test'
+  spec.add_development_dependency 'pry'
+  spec.add_development_dependency 'puma', '~> 2.8'
+  spec.add_development_dependency 'pg', '~> 0.17'
+  spec.add_development_dependency 'connection_pool', '~> 2.0'
+  
+  spec.add_runtime_dependency 'cellect-server', Cellect::VERSION
+  spec.add_runtime_dependency 'cellect-client', Cellect::VERSION
+end

data/lib/cellect/client/connection.rb

@@ -0,0 +1,58 @@
+require 'http'
+
+module Cellect
+  module Client
+    class Connection
+      include Celluloid
+      include Celluloid::IO
+      
+      def reload_project(id)
+        broadcast :post, "/projects/#{ id }/reload"
+      end
+      
+      def delete_project(id)
+        broadcast :delete, "/projects/#{ id }"
+      end
+      
+      def add_subject(id, project_id: project_id, group_id: nil, priority: nil)
+        broadcast :put, "/projects/#{ project_id }/add", querystring(subject_id: id, group_id: group_id, priority: priority)
+      end
+      
+      def remove_subject(id, project_id: project_id, group_id: nil)
+        broadcast :put, "/projects/#{ project_id }/remove", querystring(subject_id: id, group_id: group_id)
+      end
+      
+      def load_user(id, host: host, project_id: project_id)
+        send_http host, :post, "/projects/#{ project_id }/users/#{ id }/load"
+      end
+      
+      def add_seen(id, user_id: user_id, host: host, project_id: project_id)
+        send_http host, :put, "/projects/#{ project_id }/users/#{ user_id }/add_seen", querystring(subject_id: id)
+      end
+      
+      protected
+      
+      def broadcast(action, path, query = '')
+        Cellect::Client.node_set.nodes.each_pair do |node, host|
+          send_http host, action, path, query
+        end
+      end
+      
+      def send_http(host, action, path, query = '')
+        params = { host: host, path: path }
+        params[:query] = query if query && !query.empty?
+        uri = URI::HTTP.build params
+        HTTP.send action, uri.to_s, socket_class: Celluloid::IO::TCPSocket
+      end
+      
+      def querystring(hash = { })
+        [].tap do |list|
+          hash.each_pair do |key, value|
+            next unless value
+            list << "#{ key }=#{ value }"
+          end
+        end.join('&')
+      end
+    end
+  end
+end

data/lib/cellect/client/node_set.rb

@@ -0,0 +1,35 @@
+require 'cellect/node_set'
+
+module Cellect
+  module Client
+    class NodeSet < Cellect::NodeSet
+      attr_accessor :nodes
+      
+      def initialize
+        self.nodes = { }
+        super
+      end
+      
+      protected
+      
+      def nodes_changed(nodes)
+        self.nodes = { }
+        nodes.each do |node|
+          self.nodes[node] = zk.get("/nodes/#{ node }").first
+        end
+      end
+      
+      def setup
+        watch_nodes
+        zk.mkdir_p '/nodes'
+        nodes_changed zk.children('/nodes', watch: true)
+      end
+      
+      def watch_nodes
+        zk.register('/nodes') do |event|
+          nodes_changed zk.children('/nodes', watch: true)
+        end
+      end
+    end
+  end
+end

data/lib/cellect/client.rb

@@ -0,0 +1,28 @@
+require 'cellect'
+
+module Cellect
+  module Client
+    require 'cellect/client/node_set'
+    require 'cellect/client/connection'
+    
+    class << self
+      attr_accessor :connection, :_node_set
+    end
+    
+    def self.node_set
+      self._node_set ||= NodeSet.supervise
+      _node_set.actors.first
+    end
+    
+    def self.ready?
+      node_set.ready?
+    end
+    
+    def self.choose_host
+      node_set.nodes.values.sample
+    end
+    
+    Client.node_set
+    Client.connection = Connection.pool size: ENV.fetch('CELLECT_POOL_SIZE', 100).to_i
+  end
+end

data/lib/cellect/node_set.rb

@@ -0,0 +1,38 @@
+require 'zk'
+require 'timeout'
+
+module Cellect
+  class NodeSet
+    include Celluloid
+    
+    attr_accessor :zk, :state
+    
+    def initialize
+      self.state = :initializing
+      after(0.001){ async.initialize_zk } # don't block waiting for ZK to connect
+    end
+    
+    def initialize_zk
+      # don't let ZK hang the thread, just retry connection on restart
+      Timeout::timeout(5) do
+        self.zk = ZK.new zk_url, chroot: '/cellect'
+      end
+      setup
+      self.state = :ready
+    end
+    
+    def ready?
+      state == :ready
+    end
+    
+    protected
+    
+    def zk_url
+      ENV.fetch 'ZK_URL', 'localhost:2181'
+    end
+    
+    def setup
+      
+    end
+  end
+end

data/lib/cellect/version.rb

@@ -0,0 +1,3 @@
+module Cellect
+  VERSION = '0.0.1'
+end

data/lib/cellect.rb

@@ -0,0 +1,7 @@
+require 'celluloid'
+require 'celluloid/io'
+require 'cellect/version'
+
+module Cellect
+  
+end

data/spec/client/connection_spec.rb

@@ -0,0 +1,64 @@
+require 'spec_helper'
+
+module Cellect::Client
+  describe Connection do
+    let(:connection){ Cellect::Client.connection }
+    
+    before(:each) do
+      Cellect::Client.node_set.stub(:nodes).and_return 'a' => '1', 'b' => '2'
+    end
+    
+    def should_send(action: action, url: url, to: to)
+      HTTP.should_receive(:send).with action, "http://#{ to }/#{ url }", socket_class: Celluloid::IO::TCPSocket
+    end
+    
+    def should_broadcast(action: action, url: url)
+      [1, 2].each{ |i| should_send action: action, url: url, to: i }
+    end
+    
+    it 'should reload projects' do
+      should_broadcast action: :post, url: 'projects/random/reload'
+      connection.reload_project 'random'
+    end
+    
+    it 'should delete projects' do
+      should_broadcast action: :delete, url: 'projects/random'
+      connection.delete_project 'random'
+    end
+    
+    it 'should add subjects' do
+      should_broadcast action: :put, url: 'projects/random/add?subject_id=123'
+      connection.add_subject 123, project_id: 'random'
+    end
+    
+    it 'should add grouped subjects' do
+      should_broadcast action: :put, url: 'projects/random/add?subject_id=123&group_id=321'
+      connection.add_subject 123, project_id: 'random', group_id: 321
+    end
+    
+    it 'should add prioritized grouped subjects' do
+      should_broadcast action: :put, url: 'projects/random/add?subject_id=123&group_id=321&priority=0.123'
+      connection.add_subject 123, project_id: 'random', group_id: 321, priority: 0.123
+    end
+    
+    it 'should remove subjects' do
+      should_broadcast action: :put, url: 'projects/random/remove?subject_id=123'
+      connection.remove_subject 123, project_id: 'random'
+    end
+    
+    it 'should remove grouped subjects' do
+      should_broadcast action: :put, url: 'projects/random/remove?subject_id=123&group_id=321'
+      connection.remove_subject 123, project_id: 'random', group_id: 321
+    end
+    
+    it 'should load users' do
+      should_send action: :post, url: 'projects/random/users/123/load', to: 1
+      connection.load_user 123, host: '1', project_id: 'random'
+    end
+    
+    it 'should add seen subjects' do
+      should_send action: :put, url: 'projects/random/users/123/add_seen?subject_id=456', to: 1
+      connection.add_seen 456, host: '1', user_id: 123, project_id: 'random'
+    end
+  end
+end

data/spec/client/node_set_spec.rb

@@ -0,0 +1,24 @@
+require 'spec_helper'
+
+module Cellect::Client
+  describe NodeSet do
+    it_behaves_like 'node set'
+    let(:node_set){ Cellect::Client.node_set }
+    
+    it 'should update the node list when changing' do
+      begin
+        pass_until node_set, is: :ready
+        node_set.zk.create '/nodes/node', data: 'foo', mode: :ephemeral_sequential
+        
+        10.times do
+          break if node_set.nodes['node0000000001']
+          Thread.pass
+        end
+        
+        node_set.nodes['node0000000001'].should == 'foo'
+      ensure
+        node_set.zk.delete '/nodes/node0000000001'
+      end
+    end
+  end
+end

data/spec/fixtures/project_data/grouped_pairwise_priority.json

@@ -0,0 +1,109 @@
+{
+  "id": 8,
+  "name": "grouped_pairwise_priority",
+  "prioritized": true,
+  "pairwise": true,
+  "grouped": true,
+  "entries": [
+    {
+      "id": 16,
+      "priority": -6,
+      "group_id": 1
+    },
+    {
+      "id": 17,
+      "priority": -7,
+      "group_id": 2
+    },
+    {
+      "id": 3,
+      "priority": 7,
+      "group_id": 3
+    },
+    {
+      "id": 8,
+      "priority": 2,
+      "group_id": 2
+    },
+    {
+      "id": 7,
+      "priority": 3,
+      "group_id": 1
+    },
+    {
+      "id": 13,
+      "priority": -3,
+      "group_id": 1
+    },
+    {
+      "id": 14,
+      "priority": -4,
+      "group_id": 2
+    },
+    {
+      "id": 4,
+      "priority": 6,
+      "group_id": 1
+    },
+    {
+      "id": 6,
+      "priority": 4,
+      "group_id": 3
+    },
+    {
+      "id": 18,
+      "priority": -8,
+      "group_id": 3
+    },
+    {
+      "id": 12,
+      "priority": -2,
+      "group_id": 3
+    },
+    {
+      "id": 1,
+      "priority": 9,
+      "group_id": 1
+    },
+    {
+      "id": 11,
+      "priority": -1,
+      "group_id": 2
+    },
+    {
+      "id": 10,
+      "priority": 0,
+      "group_id": 1
+    },
+    {
+      "id": 9,
+      "priority": 1,
+      "group_id": 3
+    },
+    {
+      "id": 20,
+      "priority": -10,
+      "group_id": 2
+    },
+    {
+      "id": 2,
+      "priority": 8,
+      "group_id": 2
+    },
+    {
+      "id": 15,
+      "priority": -5,
+      "group_id": 3
+    },
+    {
+      "id": 5,
+      "priority": 5,
+      "group_id": 2
+    },
+    {
+      "id": 19,
+      "priority": -9,
+      "group_id": 1
+    }
+  ]
+}

data/spec/fixtures/project_data/grouped_pairwise_random.json

@@ -0,0 +1,89 @@
+{
+  "id": 7,
+  "name": "grouped_pairwise_random",
+  "prioritized": false,
+  "pairwise": true,
+  "grouped": true,
+  "entries": [
+    {
+      "id": 1,
+      "group_id": 1
+    },
+    {
+      "id": 2,
+      "group_id": 2
+    },
+    {
+      "id": 3,
+      "group_id": 3
+    },
+    {
+      "id": 4,
+      "group_id": 1
+    },
+    {
+      "id": 5,
+      "group_id": 2
+    },
+    {
+      "id": 6,
+      "group_id": 3
+    },
+    {
+      "id": 7,
+      "group_id": 1
+    },
+    {
+      "id": 8,
+      "group_id": 2
+    },
+    {
+      "id": 9,
+      "group_id": 3
+    },
+    {
+      "id": 10,
+      "group_id": 1
+    },
+    {
+      "id": 11,
+      "group_id": 2
+    },
+    {
+      "id": 12,
+      "group_id": 3
+    },
+    {
+      "id": 13,
+      "group_id": 1
+    },
+    {
+      "id": 14,
+      "group_id": 2
+    },
+    {
+      "id": 15,
+      "group_id": 3
+    },
+    {
+      "id": 16,
+      "group_id": 1
+    },
+    {
+      "id": 17,
+      "group_id": 2
+    },
+    {
+      "id": 18,
+      "group_id": 3
+    },
+    {
+      "id": 19,
+      "group_id": 1
+    },
+    {
+      "id": 20,
+      "group_id": 2
+    }
+  ]
+}

data/spec/fixtures/project_data/grouped_priority.json

@@ -0,0 +1,59 @@
+{
+  "id": 4,
+  "name": "grouped_priority",
+  "prioritized": true,
+  "pairwise": false,
+  "grouped": true,
+  "entries": [
+    {
+      "id": 8,
+      "priority": 2,
+      "group_id": 2
+    },
+    {
+      "id": 7,
+      "priority": 3,
+      "group_id": 1
+    },
+    {
+      "id": 10,
+      "priority": 0,
+      "group_id": 1
+    },
+    {
+      "id": 5,
+      "priority": 5,
+      "group_id": 2
+    },
+    {
+      "id": 6,
+      "priority": 4,
+      "group_id": 3
+    },
+    {
+      "id": 2,
+      "priority": 8,
+      "group_id": 2
+    },
+    {
+      "id": 3,
+      "priority": 7,
+      "group_id": 3
+    },
+    {
+      "id": 1,
+      "priority": 9,
+      "group_id": 1
+    },
+    {
+      "id": 9,
+      "priority": 1,
+      "group_id": 3
+    },
+    {
+      "id": 4,
+      "priority": 6,
+      "group_id": 1
+    }
+  ]
+}