vayacondios-server 0.2.11 → 0.3.0

data/lib/vayacondios/configuration.rb

@@ -0,0 +1,63 @@
+module Vayacondios
+  class Configuration
+
+    attr_accessor :base_filename, :load_order
+
+    def initialize(base_fname = nil)
+      @base_filename = base_fname || 'vayacondios.yml'
+      @load_order    = %w[ global project ]
+      @settings      = Configliere::Param.new
+    end
+
+    def defaults
+      Hash.new
+    end
+
+    def global
+      File.join('/etc/vayacondios', base_filename)
+    end
+    
+    def project
+      File.join(ENV['PWD'], 'config', base_filename)
+    end
+    
+    def overlay(conf = nil)
+      @overlay = conf unless conf.nil?
+      @overlay
+    end
+    
+    def resolved?
+      !!@resolved
+    end
+    
+    def resolved_settings
+      resolve!
+      @resolved_settings.dup
+    end
+    
+    def [] handle
+      resolved_settings[handle.to_sym]
+    end
+ 
+    def apply_all
+      scopes = load_order.dup.unshift(:defaults).push(:overlay)
+      scopes.each do |scope|
+        conf = send scope
+        if conf.is_a? String
+          @settings.read_yaml File.read(conf) if File.readable?(conf)
+        elsif conf.is_a? Hash
+          @settings.deep_merge! conf
+        end
+      end
+    end
+    
+    def resolve!
+      unless resolved?
+        apply_all
+        @resolved_settings = @settings.to_hash.symbolize_keys
+        @resolved = true
+      end
+    end
+
+  end
+end

data/lib/vayacondios/server/api.rb

@@ -0,0 +1,126 @@
+require 'vayacondios-server'
+
+module Vayacondios::Server
+
+  # Implements the Vayacondios server API.
+  #
+  # ## Setup
+  #
+  # Once the Goliath server has booted, this class is handed control
+  # to process web requests.  It uses a set of Rack-aware and
+  # Goliath-friendly plugins to accomplish some of the edges stuff
+  # like routing, parsing params, validating, &c.
+  #
+  # ## Request Loop
+  #
+  # When handling an actual request, it has to do four things:
+  #
+  # * determine which handler class to instantiate to handle the request
+  # * determine the full set of params contained in the request
+  # * call the appropriate method on the new handler, passing in these params
+  # * handle any errors that bubble up
+  #
+  # ## Configuration
+  #
+  # Goliath is kind of weirdly hard to configure nicely.  This class
+  # is also required to define an #options_parser method which is
+  # momentarily handed control at bootup time to interpret options
+  # passed to the `vcd-server` program.
+  #
+  # It **simultaneously** is required to read a configuration file
+  # from disk.  This configuration file is aware of the Rack
+  # environment the code is running in so it can take
+  # environment-specific actions like creating single-connections in
+  # test/development but using a pool of shared connections in
+  # production mode.  The default file is located in the Vayacondios
+  # source distribution at `config/vcd-server.rb`.
+  #
+  class Api < Goliath::API
+    include ApiOptions
+
+    plugin Goliath::Chimp::Plugin::ActivityMonitor, window: 30
+
+    use Goliath::Rack::Heartbeat
+    use Goliath::Chimp::Rack::Formatters::JSON
+    use Goliath::Chimp::Rack::ForceContentType,           'application/json'
+    use Goliath::Rack::Render
+    use Goliath::Rack::Params
+    use Goliath::Chimp::Rack::ApiVersion,                 Vayacondios::GEM_VERSION, api: 'Vayacondios'
+    use Goliath::Chimp::Rack::ServerMetrics,              env_key: { 'routes' => :type }, default: 'other'
+    use Goliath::Rack::Validation::RequestMethod,         %w[ GET POST PUT PATCH DELETE ]
+    use Goliath::Chimp::Rack::ControlMethods,             'POST'   => :create,
+                                                          'GET'    => :retrieve,
+                                                          'PATCH'  => :update,
+                                                          'PUT'    => :update,
+                                                          'DELETE' => :delete
+    use Goliath::Chimp::Rack::Validation::Routes,         /^
+                                                            \/#{Vayacondios::API_VERSION}
+                                                            \/(?<organization>[a-z][-_\w]+)
+                                                            \/(?<type>[-\.\w]+)
+                                                            (\/(?<topic>[-\.\w]+)
+                                                            (\/(?<id>([-\.\w+]\/?)+))?)?
+                                                          $/ix,
+                                                          "/#{Vayacondios::API_VERSION}/<organization>/<type>/<topic>/<id>"
+    use Goliath::Chimp::Rack::Validation::RouteHandler,   :type, 'stash'   => StashHandler,
+                                                                 'stashes' => StashesHandler,
+                                                                 'event'   => EventHandler,
+                                                                 'events'  => EventsHandler,
+                                                                 'stream'  => StreamHandler
+    use Goliath::Chimp::Rack::Validation::RequiredRoutes, :type, 'stash'     => :topic,
+                                                                 /^events?$/ => :topic,
+                                                                 'stream'    => :topic
+
+    # The document part of the request, e.g. - params that came
+    # directly from its body.
+    #
+    # Goliath::Rack::Params dumps all non-Hash types that were JSON
+    # parsed under this header. By accessing the #document this way
+    # we allow for non-Hash bodies to be sent as requests.
+    #
+    # @return [Hash,Array,String,Fixnum,nil] any native JSON datatype
+    def document
+      params.has_key?('_json') ? params['_json'] : params
+    end
+
+    # Assign a callback to the stream endpoint. Some of the Rack
+    # logic is recreated because of the way streaming data works.
+    def open_stream(env, hndlr)
+      env[:subscription] = hndlr.stream_data{ |data| env.stream_send MultiJson.dump(data).concat("\n") }
+    end
+
+    # Make sure to remove any outstanding streaming connections
+    # when the client disconnects
+    def on_close env
+      return unless env[:subscription]
+      env.delete(:subscription).close_stream!
+    end
+
+    # Deliver a response for the request.
+    #
+    # Uses the method set by Infochimps::Rack::ControlMethods to
+    # determine which action to call on the handler determined by
+    # Infochimps::Rack::Validation::RouteHandler
+    #
+    # Traps Goliath::Validation::Errors by returning the appropriate
+    # response.
+    #
+    # Traps all other errors by responding with a 500.
+    #
+    # @param [Hash] env the current request environment
+    def response env
+      h = handler.new(logger, db)
+      open_stream(env, h) if routes[:type] == 'stream'
+      body = h.call(control_method, routes, document)
+      [200, {}, body]
+    rescue Goliath::Validation::Error => e
+      return [e.status_code, {}, { error: e.message }]
+    rescue Document::Error => e
+      return [400, {}, { error: e.message }]
+    rescue => e
+      logger.error "#{e.class} -- #{e.message}"
+      e.backtrace.each{ |line| logger.error line }
+      return [500, {}, { error: "#{e.class} -- #{e.message}" }]
+    end
+  end
+  
+end

data/lib/vayacondios/server/api_options.rb

@@ -0,0 +1,56 @@
+module Vayacondios::Server
+  module ApiOptions
+
+    def options_parser opts, options
+      opts.banner = <<-BANNER.gsub(/^ {8}/, '').strip
+        usage: vcd-server [--param=value|--param|-p value|-p]
+
+        Vayacondios server lets any system that can speak JSON over HTTP read
+        and write configuration and events.
+
+        It provides the following HTTP endpoints, all of which assume a
+        JSON-encoded request body.
+
+        Events:
+          GET    /v2/ORG/event/TOPIC/ID
+          POST   /v2/ORG/event/TOPIC[/ID]      (announce)
+          DELETE /v2/ORG/event/TOPIC/ID
+          GET    /v2/ORG/events/TOPIC          (events)
+          DELETE /v2/ORG/events/TOPIC
+
+        Stashes:
+          GET    /v2/ORG/stash/TOPIC[/ID]      (get)
+          POST   /v2/ORG/stash/TOPIC[/ID]      (set!)
+          DELETE /v2/ORG/stash/TOPIC[/ID]      (delete)
+          GET    /v2/ORG/stashes               (stashes)
+          DELETE /v2/ORG/stashes               (delete_many)
+      BANNER
+
+      opts.separator ''
+      opts.separator 'Database options:'
+
+      options[:database] = {}
+      db_options = options[:database]
+      defaults = DbConfig.defaults[:development]
+      opts.on('-d', '--database.driver NAME', "Database driver (default: #{defaults[:driver]})") do |name|
+        db_options[:driver] = name
+      end
+      opts.on('-h', '--database.host HOST', "Database host (default: #{defaults[:host]})") do |host|
+        db_options[:host] = host
+      end
+      opts.on('-o', '--database.port PORT', Integer, "Database port (default: #{defaults[:port]})") do |port|
+        db_options[:port] = port
+      end
+      opts.on('-D', '--database.name NAME', "Database name (default: #{defaults[:name]})") do |name|
+        db_options[:name] = name
+      end
+      opts.on('-n', '--database.connections NUM', Integer, "Number of database connections to make (default: #{defaults[:connections]}). Production only") do |num|
+        db_options[:connections] = num
+      end
+
+      options[:config] = File.join(Vayacondios.library_dir, 'config/vcd-server.rb')
+      options[:port]   = Vayacondios::DEFAULT_SERVER_PORT
+    end
+
+  end
+end

data/lib/vayacondios/server/configuration.rb

@@ -0,0 +1,23 @@
+module Vayacondios::Server
+  class Configuration < Vayacondios::Configuration
+
+    def defaults
+      { 
+        development: {
+          driver:      'mongo',
+          host:        'localhost',
+          port:        27017,
+          name:        'vayacondios_development',
+          connections: 20,
+        }
+      }      
+    end
+
+    def env(handle = nil)
+      handle ||= :development
+      resolved_settings[handle.to_sym] || {}
+    end    
+  end
+  
+  DbConfig = Configuration.new('database.yml') unless defined? DbConfig
+end

data/lib/vayacondios/server/driver.rb

@@ -0,0 +1,71 @@
+module Vayacondios::Server
+  module Driver
+
+    Error = Class.new(StandardError) unless defined? Error
+
+    # Factory methods for drivers
+    class << self
+      def drivers
+        @list ||= {
+          mongo: MongoDriver,
+        }
+      end
+
+      def load_driver handle
+        drivers = File.expand_path('../drivers', __FILE__)
+        driver_file = File.join(drivers, handle.to_s + '.rb')
+        load driver_file if File.exist? driver_file
+      end
+
+      def retrieve handle
+        load_driver handle
+        drivers[handle.to_sym]
+      end
+
+      def included base
+        base.class_eval{ attr_reader :location, :log } if base.is_a? Class
+      end
+    end
+
+    # Main api entrance method
+    def call(name, request, *options)
+      send("base_#{name}", request)
+      set_location request.location
+      send(name, request.document, *options)
+    end
+
+    def base_insert request
+      log.debug "  Processing by #{self.class}#insert"
+    end
+
+    def base_retrieve request
+      log.debug "  Processing by #{self.class}#retrieve"
+    end
+
+    def base_update request
+      log.debug "  Processing by #{self.class}#update"
+    end
+
+    def base_search request
+      log.debug "  Processing by #{self.class}#search"
+    end
+
+    def base_remove request
+      log.debug "  Processing by #{self.class}#remove"
+    end
+
+    def set_location loc
+      log.debug "    Location: #{loc}"
+      @location = loc
+    end
+
+    def set_log device
+      @log = device
+    end
+
+    # for testing only
+    def unset_location
+      @location = nil
+    end
+  end
+end

data/lib/vayacondios/server/drivers/mongo.rb

@@ -0,0 +1,126 @@
+module Vayacondios::Server
+  class MongoDriver
+    include Driver
+
+    def self.connect(options = {})
+      new(options)
+    end
+
+    def initialize(options = {})
+      @log = options[:log]
+      mongo = EM::Mongo::Connection.new(options[:host], options[:port], 1, reconnect_in: 1)
+      @connection = mongo.db options[:name]
+    end
+
+    def connection
+      @connection.collection location
+    end
+
+    # Coerce objects into a BSON::ObjectId representation if possible.
+    #
+    # @param [BSON::ObjectId,Hash,#to_s] id the object to be coerced
+    # @return [BSON::ObjectId] the canonical representation of the ID
+    # @raise [Error] if `id` is a Hash and is missing the `$oid` parameter which is expected in this case
+    # @raise [Error] if the String representation of `id` is blank or empty
+    def format_id id
+      case
+      when id.is_a?(BSON::ObjectId)
+        id
+      when id.is_a?(Hash)
+        raise Error.new("When settings the ID of a #{self.class} with a Hash, an '$oid' key is required") if id['$oid'].nil?
+        format_id(id['$oid'])
+      when !id.to_s.empty?
+        id.to_s.match(/^[a-f0-9]{24}$/) ? BSON::ObjectId(id.to_s) : id.to_s
+      else
+        raise Error.new("A #{self} cannot have a blank or empty ID")
+      end
+    end
+
+    def mongo_prepare doc
+      doc[:_id] = format_id(doc[:_id]) if doc[:_id]
+      doc
+    end
+
+    def mongo_unprepare doc
+      doc['_id'] = doc['_id'].to_s
+      doc.symbolize_keys
+    end
+
+    def selector query
+      sel = { }.tap do |sel|
+        time = query.delete(:_t)
+        sel[:_t] = time.inject({}){ |t, (k,v)| t[('$' + k.to_s).to_sym] = v ; t } if time
+        data = query.delete(:_d)
+        sel.merge! to_dotted_hash(_d: data)
+      end
+      query.merge(sel).compact_blank
+    end
+
+    def to_dotted_hash(hsh, key_string = '')
+      hsh.each_with_object({}) do |(k, v), ret|
+        key = key_string + k.to_s
+        if v.is_a? Hash
+          ret.merge! to_dotted_hash(v, key + '.')
+        else
+          ret[key] = v
+        end
+      end
+    end
+    
+    def projector query
+      if query[:sort] == 'time'
+        query[:sort] = '_t'
+      elsif query[:sort].present?
+        query[:sort] = '_d.' + query[:sort]
+      end
+      query[:_reverse] if query.delete(:order) == 'descending'
+      query
+    end
+
+    def search(request, filter, opts)
+      select = selector(filter)
+      log.debug "    Selector doc: #{select}"
+      project = projector(opts)
+      log.debug "    Projector doc: #{project}"
+      res = connection.find(select, project) || []
+      log.debug "      Result: #{res}"
+      res.map{ |res| mongo_unprepare res }
+    end
+
+    def insert request
+      mongo_doc = mongo_prepare request
+      log.debug "    Mongo doc: #{mongo_doc}"
+      res = connection.save mongo_doc
+      log.debug "      Result: #{res}"
+      res = mongo_doc[:_id] if res == true
+      { _id: format_id(res).to_s }
+    end
+
+    def retrieve request
+      mongo_doc = mongo_prepare request
+      log.debug "    Mongo doc: #{mongo_doc}"
+      res = connection.find_one mongo_doc
+      log.debug "      Result: #{res}"
+      return nil if res.nil?
+      mongo_unprepare res
+    end
+
+    def remove(request, filter)
+      mongo_doc = mongo_prepare(request)
+      mongo_doc.merge! selector(filter)
+      log.debug "    Mongo doc: #{mongo_doc}"
+      res = connection.remove mongo_doc
+      log.debug "      Result: #{res}"
+      nil
+    end
+
+    # for testing only
+    def reset!
+      connection.drop
+    end
+
+    def count
+      connection.count
+    end
+  end
+end

data/lib/vayacondios/server/handlers/document_handler.rb

@@ -0,0 +1,81 @@
+module Vayacondios::Server
+
+  # Generic handler for all documents.
+  #
+  # Handlers link HTTP applications to document classes.
+  #
+  # @attr [Logger] log the log to use
+  # @attr [Driver] database the database driver
+  class DocumentHandler
+    include Goliath::Chimp::Handler
+
+    attr_reader :log, :database
+
+    # Create a new DocumentHandler.
+    #
+    # @param [Logger] log
+    def initialize(log, db)
+      @log = log
+      @database = db
+    end
+
+    # Search for matching documents.
+    #
+    # @param [Hash] query the search query
+    # @return [Array<Hash>] the matching documents
+    def base_search(params, query)
+      log.debug("Processing by #{self.class}#search")
+      log.debug("  Parameters: #{params.inspect}")
+      log.debug("  Query:      #{query.inspect}")
+    end
+    
+    # Create a document.
+    #
+    # @param [Hash] params routing information like `organization`, `topic,`, or `id`
+    # @param [Hash] document the body of the document
+    # @return [Hash] the created document
+    def base_create(params, document)
+      log.debug("Processing by #{self.class}#create")
+      log.debug("  Parameters: #{params.inspect}")
+      log.debug("  Document:   #{document.inspect}")
+    end
+
+    # Find and show a particular document.
+    #
+    # @param [Hash] params routing information like `organization`, `topic,`, or `id`
+    # @return [Object] the document (or part of a document) that was found
+    def base_retrieve(params, document)
+      log.debug("Processing by #{self.class}#retrieve")
+      log.debug("  Parameters: #{params.inspect}")
+    end
+    
+    # Update a document.
+    #
+    # @param [Hash] params routing information like `organization`, `topic,`, or `id`
+    # @param [Hash] document the body of the document
+    # @return [Object] the document (or part a document) that was updated
+    def base_update(params, document)
+      log.debug("Processing by #{self.class}#update")
+      log.debug("  Parameters: #{params.inspect}")
+      log.debug("  Document:   #{document.inspect}")
+    end
+    
+    # Delete a document.
+    #
+    # @param [Hash] params routing information like `organization`, `topic,`, or `id`
+    # @return [Hash] details about which documents were deleted
+    def base_delete(params, document)
+      log.debug("Processing by #{self.class}#delete")
+      log.debug("  Parameters: #{params.inspect}")
+    end
+
+    def call(name, params, document)
+      send("base_#{name}", params, document)
+      send(name, params, document)
+    end
+
+    def action_successful
+      { ok: true }
+    end
+  end
+end

data/lib/vayacondios/server/handlers/event_handler.rb

@@ -1,33 +1,38 @@
-# Vayacondios::EventHandler
-#
-# This handler will accept requests to update Event for an organization. All
-# updates will overwrite an existing document.
+module Vayacondios::Server
+  class EventHandler < DocumentHandler
 
-class Vayacondios
-
-  class EventHandler
-
-    def initialize(mongodb)
-      @mongo = mongodb
+    # Create an event.
+    #
+    # @param [Hash] params routing information like `organization`, `topic,`, or `id`
+    # @param [Hash] document the body of the document
+    def create(params, document)
+      Event.create(params, document) do |request|
+        database.call(:insert, request)
+      end
     end
 
-    def update(document, options={})
-      raise Error::BadRequest.new unless options[:topic]
-      raise Error::BadRequest.new if options[:id] && /\W/ =~ options[:id]
-
-      existing_document = EventDocument.find(@mongo, options)
-      if existing_document
-        existing_document.update(document)
-      else
-        existing_document = EventDocument.create(@mongo, document, options)
-      end
-      existing_document.body
+    # Find and show a particular event.
+    #
+    # @param [Hash] params routing information like `organization`, `topic,`, or `id`
+    # @param [Hash] document the body of the document
+    # @raise [Goliath::Validation::Error] if no event is found.  Returns a 404.
+    def retrieve(params, document)
+      Event.find(params) do |request|
+        database.call(:retrieve, request)
+      end or raise Goliath::Validation::NotFoundError.new("Event with topic <#{params[:topic]}> and ID <#{params[:id]}> not found")
     end
 
-    def self.find(mongodb, options)
-      existing_document = EventDocument.find(mongodb, options)
-      raise Error::NotFound.new unless existing_document
-      existing_document
+    # Delete a specific event.
+    #
+    # @param [Hash] params routing information like `organization`, `topic,`, or `id`
+    # @param [Hash] document the body of the document
+    # @raise [Goliath::Validation::Error] if params do not have an id
+    def delete(params, document)
+      raise Goliath::Validation::BadRequestError.new('An <Id> is required to delete an Event') unless params[:id]
+      Event.destroy(params, {}) do |request, options|
+        database.call(:remove, request, options)
+      end
+      action_successful
     end
   end
-end
+end

data/lib/vayacondios/server/handlers/events_handler.rb

@@ -0,0 +1,31 @@
+module Vayacondios::Server
+
+  # Handles requests against multiple Events.
+  class EventsHandler < DocumentHandler
+
+    # Search for events matching a given query.
+    #
+    # @param [Hash] params routing information like `organization`, `topic,`, or `id`
+    # @param [Hash] query the search query
+    def search(params, query)
+      Event.search(params, query) do |request, filter, opts|
+        database.call(:search, request, filter, opts)
+      end
+    end
+
+    # FIXME
+    # Abstract this into method delegation
+    def base_retrieve(params, query)
+      base_search(params, query)
+    end
+    alias_method :retrieve, :search
+
+    def delete(params, query)
+      Event.destroy(params, query) do |request, opts|
+        database.call(:remove, request, opts)
+      end
+      action_successful
+    end
+
+  end
+end