cellect-server 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 3b7f28e9a20fa75b575fe23b06eb1690f1c78943
+  data.tar.gz: ece4c409106d6bdba0d50e9fead70d0c1ce399d9
+SHA512:
+  metadata.gz: 439ca8ce160576d91933308d882ce74c131d926d8b9e1394e45e37b532f0c36ca122b79e9e4625508712b89eee2cddc4654d55f694a71f21aa0cfcd23837c579
+  data.tar.gz: 90b0afb4aa5ab74af30c3094fb8e103b507ab96c8ad2dbe6b5119844ddbea2b0f451e8b7cfd386373d1959e43c6dd5339b53ec5b2e118a499caf63edcf23e561

data/.gitignore

@@ -0,0 +1,15 @@
+.bundle
+.DS_Store
+.rvmrc
+.ruby-version
+/data/*
+/ext/*.o
+/ext/Makefile
+/ext/*.bundle
+/scratch
+*.so
+.vagrant
+log/*
+tmp/*
+*.gem
+pkg/*

data/.rspec

@@ -0,0 +1,2 @@
+--color
+--format progress

data/Gemfile

@@ -0,0 +1,2 @@
+source 'http://rubygems.org'
+gemspec name: 'cellect'

data/README.md

@@ -0,0 +1,19 @@
+# Cellect
+
+This is a work in progress
+
+## Building
+
+1. Install [Boost](http://www.boost.org/): OS X: `brew update && brew install boost`, Ubuntu: `sudo apt-get update && sudo apt-get install libboost-all-dev`
+2. Install gem dependencies: `bundle` (See Note)
+3. Build extension: `cd ext; ruby extconf.rb; make; cd ..`
+
+### Note
+To install rice your Ruby must be compiled with shared libraries enabled, from the rice docs: 
+* rvm:   `rvm reinstall [version] -- --enable-shared`
+* rbenv: `CONFIGURE_OPTS="--enable-shared" rbenv install [version]`
+
+
+## Testing
+
+Run the specs with `rake`

data/Rakefile

@@ -0,0 +1,9 @@
+require 'bundler/gem_helper'
+require 'rspec/core/rake_task'
+
+Bundler::GemHelper.install_tasks name: 'cellect'
+Bundler::GemHelper.install_tasks name: 'cellect-server'
+Bundler::GemHelper.install_tasks name: 'cellect-client'
+
+RSpec::Core::RakeTask.new :spec
+task default: :spec

data/cellect-server.gemspec

@@ -0,0 +1,40 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'cellect/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = 'cellect-server'
+  spec.version       = Cellect::VERSION
+  spec.authors       = ['Michael Parrish']
+  spec.email         = ['michael@zooniverse.org']
+  spec.summary       = ''
+  spec.description   = ''
+  spec.homepage      = 'https://github.com/parrish/Cellect'
+  spec.license       = 'MIT'
+  
+  ignored_paths = %w(config data log script tmp).collect{ |path| Dir["#{ path }/**/*"] }.flatten
+  ignored_files = %w(Dockerfile Vagrantfile Gemfile.lock config.ru) + ignored_paths
+  
+  spec.files         = `git ls-files -z`.split("\x0").reject{ |f| f =~ /client/ } - ignored_files
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ['lib']
+  
+  spec.add_development_dependency 'bundler', '~> 1.5'
+  spec.add_development_dependency 'rake'
+  spec.add_development_dependency 'oj'
+  spec.add_development_dependency 'rspec'
+  spec.add_development_dependency 'rack-test'
+  spec.add_development_dependency 'pry'
+  spec.add_development_dependency 'puma', '~> 2.8'
+  spec.add_development_dependency 'pg', '~> 0.17'
+  spec.add_development_dependency 'connection_pool', '~> 2.0'
+  
+  spec.add_runtime_dependency 'diff_set'
+  spec.add_runtime_dependency 'celluloid', '0.16.0.pre'
+  spec.add_runtime_dependency 'celluloid-io', '0.16.0.pre'
+  spec.add_runtime_dependency 'http', '~> 0.6'
+  spec.add_runtime_dependency 'zk', '~> 1.9'
+  spec.add_runtime_dependency 'grape', '~> 0.7'
+end

data/cellect.gemspec

@@ -0,0 +1,33 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'cellect/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = 'cellect'
+  spec.version       = Cellect::VERSION
+  spec.authors       = ['Michael Parrish']
+  spec.email         = ['michael@zooniverse.org']
+  spec.summary       = ''
+  spec.description   = ''
+  spec.homepage      = 'https://github.com/parrish/Cellect'
+  spec.license       = 'MIT'
+  
+  spec.files         = ['lib/cellect.rb', 'lib/cellect/version.rb']
+  spec.executables   = []
+  spec.test_files    = []
+  spec.require_paths = ['lib']
+  
+  spec.add_development_dependency 'bundler', '~> 1.5'
+  spec.add_development_dependency 'rake'
+  spec.add_development_dependency 'oj'
+  spec.add_development_dependency 'rspec'
+  spec.add_development_dependency 'rack-test'
+  spec.add_development_dependency 'pry'
+  spec.add_development_dependency 'puma', '~> 2.8'
+  spec.add_development_dependency 'pg', '~> 0.17'
+  spec.add_development_dependency 'connection_pool', '~> 2.0'
+  
+  spec.add_runtime_dependency 'cellect-server', Cellect::VERSION
+  spec.add_runtime_dependency 'cellect-client', Cellect::VERSION
+end

data/lib/cellect.rb

@@ -0,0 +1,7 @@
+require 'celluloid'
+require 'celluloid/io'
+require 'cellect/version'
+
+module Cellect
+  
+end

data/lib/cellect/node_set.rb

@@ -0,0 +1,38 @@
+require 'zk'
+require 'timeout'
+
+module Cellect
+  class NodeSet
+    include Celluloid
+    
+    attr_accessor :zk, :state
+    
+    def initialize
+      self.state = :initializing
+      after(0.001){ async.initialize_zk } # don't block waiting for ZK to connect
+    end
+    
+    def initialize_zk
+      # don't let ZK hang the thread, just retry connection on restart
+      Timeout::timeout(5) do
+        self.zk = ZK.new zk_url, chroot: '/cellect'
+      end
+      setup
+      self.state = :ready
+    end
+    
+    def ready?
+      state == :ready
+    end
+    
+    protected
+    
+    def zk_url
+      ENV.fetch 'ZK_URL', 'localhost:2181'
+    end
+    
+    def setup
+      
+    end
+  end
+end

data/lib/cellect/server.rb

@@ -0,0 +1,30 @@
+require 'diff_set'
+require 'cellect'
+require 'celluloid/autostart'
+
+module Cellect
+  module Server
+    require 'cellect/server/node_set'
+    require 'cellect/server/adapters'
+    require 'cellect/server/project'
+    require 'cellect/server/grouped_project'
+    require 'cellect/server/user'
+    require 'cellect/server/api'
+    
+    class << self
+      attr_accessor :node_set
+    end
+    
+    def self.ready?
+      Project.all.each do |project|
+        return false unless project.ready?
+      end
+      
+      true
+    rescue
+      false
+    end
+    
+    Server.node_set = NodeSet.supervise
+  end
+end

data/lib/cellect/server/adapters.rb

@@ -0,0 +1,13 @@
+module Cellect
+  module Server
+    class << self
+      attr_accessor :adapter
+    end
+    
+    module Adapters
+      require 'cellect/server/adapters/default'
+    end
+    
+    Cellect::Server.adapter = Adapters::Default.new
+  end
+end

data/lib/cellect/server/adapters/default.rb

@@ -0,0 +1,60 @@
+module Cellect
+  module Server
+    module Adapters
+      class Default
+        # Return a list of projects to load in the form:
+        #   [{
+        #     'id' => 123,
+        #     'name' => 'foo',
+        #     'prioritized' => false,
+        #     'pairwise' => false,
+        #     'grouped' => false
+        #   }, ...]
+        def project_list
+          raise NotImplementedError
+        end
+        
+        # Load the data for a project, this method:
+        #   Accepts a project
+        #   Returns an array of hashes in the form:
+        #   {
+        #     'id' => 123,
+        #     'priority' => 0.123,
+        #     'group_id' => 456
+        #   }
+        def load_data_for(project_name)
+          raise NotImplementedError
+        end
+        
+        # Load seen ids for a user, this method:
+        #   Accepts a project_name, and a user id
+        #   Returns an array in the form:
+        #   [1, 2, 3]
+        def load_user(project_name, id)
+          raise NotImplementedError
+        end
+        
+        def load_projects
+          project_list.each{ |project_info| load_project project_info }
+        end
+        
+        def load_project(args)
+          info = if args.is_a?(Hash)
+            args
+          elsif args.is_a?(String)
+            project_list.select{ |h| h['name'] == args }.first
+          else
+            raise ArgumentError
+          end
+          
+          project_for info
+        end
+        
+        def project_for(opts = { })
+          project_klass = opts.fetch('grouped', false) ? GroupedProject : Project
+          project_klass[opts['name'], pairwise: opts['pairwise'], prioritized: opts['prioritized']]
+        end
+      end
+    end
+  end
+end

data/lib/cellect/server/adapters/postgres.rb

@@ -0,0 +1,64 @@
+require 'pg'
+require 'connection_pool'
+
+module Cellect
+  module Server
+    module Adapters
+      class Postgres < Default
+        def initialize
+          @pg ||= ConnectionPool.new(size: ENV.fetch('PG_POOL_SIZE', 25).to_i) do
+            PG.connect connection_options
+          end
+        end
+        
+        def project_list
+          with_pg do |pg|
+            pg.exec('select * from projects').collect do |row|
+              {
+                'id' => row['id'].to_i,
+                'name' => row['id'],
+                'prioritized' => row['prioritized'] == 't',
+                'pairwise' => row['pairwise'] == 't',
+                'grouped' => row['grouped'] == 't'
+              }
+            end
+          end
+        end
+        
+        def load_data_for(project_name)
+          with_pg do |pg|
+            pg.exec("select id, priority, group_id from project_#{ project_name }_subjects").collect do |row|
+              {
+                'id' => row['id'].to_i,
+                'priority' => row['priority'].to_f,
+                'group_id' => row['group_id'].to_i
+              }
+            end
+          end
+        end
+        
+        def load_user(project_name, id)
+          with_pg do |pg|
+            pg.exec("select subject_id from project_#{ project_name }_classifications where user_id=#{ id }").collect do |row|
+              row['subject_id'].to_i
+            end
+          end
+        end
+        
+        def with_pg
+          @pg.with{ |pg| yield pg }
+        end
+        
+        def connection_options
+          {
+            host: ENV.fetch('PG_HOST', '127.0.0.1'),
+            port: ENV.fetch('PG_PORT', '5432'),
+            dbname: ENV.fetch('PG_DB', 'cellect'),
+            user: ENV.fetch('PG_USER', 'cellect'),
+            password: ENV.fetch('PG_PASS', '')
+          }
+        end
+      end
+    end
+  end
+end

data/lib/cellect/server/api.rb

@@ -0,0 +1,48 @@
+require 'grape'
+
+module Cellect
+  module Server
+    class API < Grape::API
+      format :json
+      
+      require 'cellect/server/api/helpers'
+      require 'cellect/server/api/sets'
+      require 'cellect/server/api/users'
+      
+      get :stats do
+        usage = ->(keyword) do
+          `ps axo #{ keyword }`.chomp.split("\n")[1..-1].collect(&:to_f).inject :+
+        end
+        
+        {
+          memory: usage.call('%mem'),
+          cpu: usage.call('%cpu')
+        }
+      end
+      
+      resources :projects do
+        get do
+          Cellect::Server.adapter.project_list
+        end
+        
+        segment '/:project_id' do
+          helpers Helpers
+          mount Sets
+          mount Users
+          
+          get :status do
+            project.status
+          end
+          
+          post :reload do
+            project.async.load_data
+          end
+          
+          delete do
+            # delete a project (maybe?)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/cellect/server/api/helpers.rb

@@ -0,0 +1,44 @@
+module Cellect
+  module Server
+    class API
+      module Helpers
+        def project
+          @project ||= Project[params[:project_id]]
+        end
+        
+        def selector_params
+          {
+            limit: param_to_int(:limit, default: 5),
+            user_id: param_to_int(:user_id),
+            group_id: param_to_int(:group_id)
+          }
+        end
+        
+        def update_params
+          {
+            subject_id: param_to_int(:subject_id),
+            group_id: param_to_int(:group_id),
+            priority: param_to_float(:priority)
+          }
+        end
+        
+        def param_to_int(param, default: nil)
+          _param_to param, :to_i, default
+        end
+        
+        def param_to_float(param, default: nil)
+          _param_to param, :to_f, default
+        end
+        
+        def _param_to(param, conversion, default)
+          val = params[param].try conversion
+          params[param] && val && val > 0 ? val : default
+        end
+        
+        def to_query(hash)
+          hash.select{ |k, v| v }.collect{ |k, v| "#{ k }=#{ v }" }.join '&'
+        end
+      end
+    end
+  end
+end