vayacondios-server 0.2.11 → 0.3.0

data/lib/vayacondios/server/handlers/stash_handler.rb

@@ -0,0 +1,69 @@
+module Vayacondios::Server
+  class StashHandler < DocumentHandler
+
+    # Create a stash.
+    #
+    # @param [Hash] params routing information like `organization`, `topic,`, or `id`
+    # @param [Hash] document the body of the stash
+    def create(params, document)      
+      stash = Stash.create(params, document) do |request|
+        database.call(:insert, request)
+      end
+      stash.delete(:topic)
+      stash
+    end
+
+    # Find and show a particular stash.
+    #
+    # @param [Hash] params routing information like `organization`, `topic,`, or `id`
+    # @raise [Goliath::Validation::Error] if no stash is found.  Returns a 404.
+    def retrieve(params, document)
+      stash = Stash.find(params) do |request|
+        database.call(:retrieve, request)
+      end or raise Goliath::Validation::NotFoundError.new("Stash with topic <#{params[:topic]}> not found")
+      stash.delete(:topic)
+      if slice = params[:id]
+        stash[slice.to_sym] or raise Goliath::Validation::NotFoundError.new("Stash with topic <#{params[:topic]}> found, but does not contain Id <#{slice}>")
+      else
+        stash
+      end
+    end
+
+    # Update a stash.
+    #
+    # @param [Hash] params routing information like `organization`, `topic,`, or `id`
+    # @param [Hash] document the body of the stash
+    # def update(params, document)
+    #   id = params.delete(:id)
+    #   raise Goliath::Validation::BadRequestError.new("If not including an id the document must be a Hash") if id.blank? && !document.is_a?(Hash)
+    #   # Not sure if I like this behavior for updating a non-existant document
+    #   original = begin ; retrieve(params.dup, nil) ; rescue Goliath::Validation::NotFoundError ; nil ; end
+    #   if original.is_a? Hash
+    #     if id
+    #       original.delete(id.to_sym)
+    #       original.merge!(id => document) unless document.nil?
+    #     else
+    #       original.merge! document
+    #     end
+    #   else
+    #     params[:id] = id if id && document
+    #     original = document
+    #   end
+    #   Stash.update(params, original) do |request|
+    #     database.call(:update, request)
+    #   end
+    # end
+ 
+    # Delete a single stash.
+    #
+    # @param [Hash] params routing information like `organization`, `topic,`, or `id`
+    # @param [Hash] document presentation or filter information
+    def delete(params, document)
+      raise Goliath::Validation::NotImplementedError.new 'Deleting an Id from a Stash is not supported' if params[:id]
+      Stash.destroy(params, {}) do |request, options|
+        database.call(:remove, request, options)
+      end
+      action_successful
+    end
+  end
+end

data/lib/vayacondios/server/handlers/stashes_handler.rb

@@ -0,0 +1,49 @@
+module Vayacondios::Server
+  class StashesHandler < DocumentHandler
+
+    # Apply a replacement across many stashes.
+    #
+    # @param [Hash] params routing information like `organization`, `topic,`, or `id`
+    # @param [Hash] document the body of the request containing a search query and a replacement
+    # def create(params, document)
+    #   document.symbolize_keys!
+    #   query       = document[:query]  || {}
+    #   replacement = document[:update] || {}
+    #   Stash.replace_many(params, query, replacement)
+    # end
+
+    # Search for stashes matching a given query.
+    #
+    # @param [Hash] params routing information like `organization`, `topic,`, or `id`
+    # @param [Hash] query a search query
+    def retrieve(params, query)
+      stashes = Stash.search(params, query) do |request, filter, options|
+        database.call(:search, request, filter, options)
+      end
+    end
+    
+    # Apply an update across many stashes.
+    #
+    # @param [Hash] params routing information like `organization`, `topic,`, or `id`
+    # @param [Hash] document the body of the request containing a search query and an update
+    # def update(params={}, document={})
+    #   query  = (document[:query]  || document['query']  || {})
+    #   update = (document[:update] || document['update'] || {})
+    #   Stash.update_many(log, database, params, query, update)
+    # end
+
+    # Delete many stashes that match a query.
+    #
+    # @param [Hash] params routing information like `organization`, `topic,`, or `id`
+    # @param [Hash] query a search query
+    # @param [Hash] document the body of the request containing a search query and an update
+    def delete(params, query)
+      raise Goliath::Validation::BadRequestError.new 'Query cannot be empty' if query.empty?
+      Stash.destroy(params, query) do |request, filter|
+        database.call(:remove, request, filter)
+      end
+      action_successful
+    end
+
+  end
+end

data/lib/vayacondios/server/handlers/stream_handler.rb

@@ -0,0 +1,39 @@
+module Vayacondios::Server
+  class StreamHandler < EventsHandler
+
+    attr_reader :cursor, :timer, :on_data
+
+    def retrieve(params, query)
+      @timer = EM::Synchrony.add_periodic_timer(1){ stream_events }
+      @cursor = Event.receive(params).prepare_search(query)
+      Goliath::Response::STREAMING
+    end
+
+    def stream_data(&on_data)
+      @on_data = on_data
+      self
+    end
+
+    def close_stream!
+      timer.cancel
+    end
+
+    def update_cursor latest
+      cursor.filter.delete(:_t)
+      cursor.prepare_search(after: latest)
+    end
+    
+    def stream_events
+      log.debug 'Streaming events'
+      log.debug "  Stream cursor is #{cursor.filter}"
+      available = database.call(:search, cursor, cursor.filter.dup, {})
+      unless available.empty?
+        available.each do |result|
+          event = Event.new.format_response(result)
+          on_data.call event
+          update_cursor event[:time]
+        end
+      end
+    end
+  end
+end

data/lib/vayacondios/server/models/document.rb

@@ -0,0 +1,87 @@
+# The Document model is a base model used by Event, Stash, &c.
+#
+# The `organization` and `topic` properties are defined as instance
+# attributes because they are used for routing the document.
+# Subclasses should continue to store each separate piece of routing
+# information as its own instance variable and store the entire
+# payload/data as a single field.
+#
+#   organization             coca_cola
+#   |---- topic              |---- ad_campaigns
+#   |---- ...     ------>    |---- web_traffic
+#   `---- topic              `---- social_media_buzz
+#
+# @attr [String] organization the organization this document belongs to
+# @attr [String] topic the unit which identifies documents of a given "kind"
+module Vayacondios::Server
+  class Document
+    include Gorillib::Model
+
+    field :organization, String
+    field :topic,        String
+    field :id,           String
+    field :body,         Hash
+    field :filter,       Hash
+
+    def receive_organization name
+      @organization = sanitize_location_name(name).gsub(/^system\./, '_system.')
+    end
+
+    def format_response result
+      from_document(result.symbolize_keys.compact).external_document
+    end
+  
+    # A class for errors that arise within documents due to internal or
+    # IO errors.
+    Error = Class.new(StandardError)
+
+    # Sanitize a string to make a suitable component of a database
+    # location name.
+    #
+    # Replaces all characters that aren't letters, digits, underscores,
+    # periods, or hyphens with underscores.  Also replaces periods at
+    # the beginning or at the end of a collection name with an
+    # underscore.
+    #
+    # @param [String] name
+    # @return [String] the sanitized `name`
+    def sanitize_location_name name
+      name.to_s.gsub(/^\.|[^-\w\.]+|\.$/, '_')
+    end
+
+    class << self
+      def extract_query_options! params
+        params.symbolize_keys!
+        opts = {}
+        [:limit, :order, :sort, :fields].each{ |opt| opts[opt] = params.delete opt }
+        opts.merge default_query_options
+      end
+
+      def search(params, query, &driver)
+        options = extract_query_options! query
+        action  = receive(params).prepare_search(query)
+        result  = driver.call(action, action.filter, options)
+        result.map{ |res| new.format_response res }
+      end
+      
+      def create(params, document, &driver)
+        action = receive(params).prepare_create(document)
+        result = driver.call(action)
+        action.format_response result
+      end
+
+      def find(params, &driver)
+        action = receive(params).prepare_find
+        result = driver.call(action, {})
+        return nil if result.nil?
+        action.format_response result
+      end
+      
+      def destroy(params, document, &driver)
+        action = receive(params).prepare_destroy(document.symbolize_keys)
+        result = driver.call(action, action.filter)
+        return result
+      end
+    end
+  end
+end

data/lib/vayacondios/server/models/event.rb

@@ -0,0 +1,198 @@
+# Events are documents with arbitrary key/value data but which also
+# have
+#
+# * an ID, auto-generated to a random unique value when none is provided
+# * a timestamp, auto-generated to the current time when none is provided
+#
+# An HTTP request which announces a new event
+#
+# ```
+# POST /v2/coca_cola/event/ad_campaigns
+# { "impresions": 23829, "errors": 29 }
+# ```
+# 
+# would result in a document in the `coca_cola.ad_campaigns.events`
+# collection with the following structure:
+#
+# ```
+# {
+#   "_id": ObjectId("51c38ad981bdb34d32000001"),
+#   "t":   ISODate("2013-06-20T23:06:01.846Z"),
+#   "d":   {
+#     "impressions": 23829,
+#     "errors":      29
+#   }
+# }
+# ```
+#
+# When explictly setting the ID and timestamp with a request like:
+#
+# ```
+# POST /v2/coca_cola/event/ad_campaigns/atx-jan-01
+# { "impresions": 23829, "errors": 29, "time": "2013-01-20 18:09:39 -0500" }
+# ```
+#
+# a document would still be generated in the
+# `coca_cola.ad_campaigns.events` collection with a similar structure:
+#
+# ```
+# {
+#   "_id": "atx-jan-01",
+#   "t":   ISODate("2013-01-20T23:09:39Z")
+#   "d":   {
+#     "impressions": 23829,
+#     "errors":      29
+#   }
+# }
+# ```
+#
+# @attr [Time] timestamp the timestamp for which this event records data
+#
+module Vayacondios::Server
+  class Event < Vayacondios::Server::Document
+
+    # The default number of events returned when searching.
+    LIMIT  = 50
+    
+    # The default sort order when searching
+    ORDER  = 'descending'
+
+    # The default sort field when searching.
+    SORT   = 'time'
+
+    # The default time window (measured relative to the current time)
+    # when searching.
+    WINDOW = 3600
+
+    def self.default_query_options
+      { limit: LIMIT, order: ORDER, sort: SORT } 
+    end
+
+    field :time, Time   # assigned or Time.now.utc
+    field :host, String # assigned or read from the client
+
+    # Set the topic of this document, sanitizing it for the database
+    #
+    # @param [String] name
+    # @return [String] the sanitized `name`
+    def receive_topic name
+      @topic = sanitize_location_name name
+    end
+
+    # Normalizes the time
+    def receive_time t
+      @time = format_time to_timestamp(t)
+    end
+
+    # Parses an object into a timestamp.  
+    #
+    # @param [String, Numeric, Time, nil] obj
+    # @param [Time] default the time value to return if none could be found in the `obj`
+    # @return [Time]
+    def to_timestamp(obj, default = Time.now)
+      case obj
+      when String  then Time.parse(obj)        
+      when Date    then obj.to_time
+      when Time    then obj
+      when Numeric then Time.at(obj)
+      else              default
+      end
+    rescue ArgumentError => e
+      default
+    end
+
+    def format_time ts
+      ts.round(3).utc
+    end
+
+    # The name of the collection this event will store its data in.
+    #
+    # @return [String]
+    def location
+      [organization, topic, 'events'].join('.')
+    end
+
+    # An events internal database representation
+    def document
+      { _id: id, _t: time, _d: body }.compact
+    end
+
+    # Populate a new Event from a database representation
+    def from_document doc
+      d = {}.tap do |d|
+        d[:id]   = doc[:_id]
+        d[:time] = doc[:_t]
+        d[:body] = doc[:_d]
+      end.compact
+      receive! d
+      self
+    end
+
+    # An event as presented to a user
+    def external_document
+      { id: id, time: time.iso8601(3) }.merge(body)
+    end
+
+    # Returns a Hash that can be used for selection criteria in a query
+    # to a MongoDB collection.
+    #
+    # For any given `query` object, this method should be run **after**
+    # Event.projector because Event.projector modifies the `query` by
+    # removing certain special options which would otherwise be
+    # interpreted by this method..
+    #
+    # @param [Hash] query
+    # @option query [String, Numeric, Time, nil] from the earliest time for a matched event (>=)
+    # @option query [String, Numeric, Time, nil] upto the latest time for a matched event (<=)
+    # @option query [String, Numeric, Time, nil] after the earliest time for a matched event (>)
+    # @option query [String, Numeric, Time, nil] before the latest time for a matched event (<)
+    # @option query [String, Regexp]] id a regular expression that matches the ID of the event
+    # @return [Hash] the selector Hash
+    # @see Event.projector
+    def event_filter query
+      filter = { _t: {} }.tap do |filter|
+        if query.has_key? :after
+          filter[:_t][:gt]  = to_timestamp query.delete(:after)
+          query.delete(:from)
+        elsif query.has_key? :from
+          filter[:_t][:gte] = to_timestamp query.delete(:from)
+        end
+
+        if query.has_key? :before
+          filter[:_t][:lt]  = to_timestamp query.delete(:before)
+          query.delete(:upto)
+        elsif query.has_key? :upto
+          filter[:_t][:lte] = to_timestamp query.delete(:upto)
+        end
+        # sel['_id'] = Regexp.new(query.delete(:id)) if query[:id]
+        query.each_pair{ |key, val| filter[:_d] ||= {} ; filter[:_d][key] = val }
+      end
+    end
+
+    # Prepare Event search request
+    def prepare_search query
+      receive!(filter: event_filter(query))
+      self
+    end
+
+    # Prepare Event retrieve request
+    def prepare_find
+      raise Error.new('Cannot find an event without an ID') if id.blank?
+      self
+    end
+
+    # Prepare Event create request
+    def prepare_create document
+      raise Error.new('Events must be Hash-like to create') unless document.is_a?(Hash)
+      document.symbolize_keys!
+      receive!(time: document.delete(:time), body: document)
+      self
+    end
+
+    # Prepare Event remove request
+    def prepare_destroy query
+      receive!(filter: event_filter(query))
+      self
+    end  
+  end
+end

data/lib/vayacondios/server/models/stash.rb

@@ -0,0 +1,100 @@
+# Stashes are documents with arbitrary key/value data.
+#
+# An HTTP request which sets a new stash
+#
+# ```
+# POST /v2/coca_cola/stash/campaigns
+# { "creatives": [ "atx-jan-01", "bos-jan-01"], "test": false }
+# ```
+#
+# would result in a document in the `coca_cola.stash` collection with
+# the following structure:
+#
+# ```
+# {
+#   "_id": "campaigns",
+#   "creatives": [ "atx-jan-01", "bos-jan-01"],
+#   "test": false
+# }
+# ```
+#
+module Vayacondios::Server
+  class Stash < Vayacondios::Server::Document
+
+    # The default number of stashes returned when searching.
+    LIMIT = 50
+
+    # The default sort order when searching.
+    SORT  = 'ascending'
+    ORDER = '_id'
+
+    # Returned as an acknowledgement of the request when there is no
+    # better option (#destroy, #update_many, &c.)
+    # OK = {ok: true}
+
+    def self.default_query_options
+      { limit: LIMIT, order: ORDER, sort: SORT } 
+    end
+
+    # The name of the collection this stash will store its data in.
+    #
+    # @return [String]
+    def location
+      [organization.to_s, 'stash'].join('.')
+    end
+
+    def document
+      { _id: topic }.compact.merge(body || {})
+    end
+
+    def from_document doc
+      d = {}.tap do |d|
+        d[:topic] = doc.delete(:_id)
+        doc = nil if doc.empty?
+        if body.nil?
+          new_body = doc 
+        else
+          new_body = body.merge(doc || {})
+        end     
+        d[:body]  =  new_body
+      end
+      receive! d
+      self
+    end
+    
+    def external_document
+      { topic: topic }.merge(body || {})
+    end
+
+    def prepare_search query
+      filter = query.merge(_id: topic).compact
+      receive!(filter: filter)
+      self
+    end
+
+    def prepare_create document
+      if document.is_a? Hash
+        document.symbolize_keys!
+        raise Error.new ':topic is a reserved key and cannot be used in a stash document' if document.has_key?(:topic)
+      end    
+      if id.blank?
+        raise Error.new 'If not including an Id, the document must be a Hash' unless document.is_a? Hash
+        receive!(body: document)
+      else
+        receive!(body: { id => document })
+      end
+      self
+    end
+
+    def prepare_find
+      raise Error.new('Cannot find a stash without a topic') unless topic
+      self
+    end
+
+    def prepare_destroy query
+      filter = query.merge(_id: topic).compact
+      receive!(filter: filter)
+      self
+    end
+  end
+end

data/lib/vayacondios/server.rb

@@ -0,0 +1,35 @@
+require 'goliath'
+require 'goliath/chimp'
+require 'em-mongo'
+require 'em-synchrony/em-http'
+require 'em-synchrony/em-mongo'
+require 'configliere'
+
+require 'gorillib/object/blank'
+require 'gorillib/enumerable/sum'
+require 'gorillib/hash/compact'
+require 'gorillib/hash/deep_merge'
+require 'gorillib/hash/keys'
+require 'gorillib/model'
+require 'gorillib/string/constantize'
+require 'gorillib/string/inflections'
+require 'multi_json'
+
+require 'vayacondios'
+require 'vayacondios/configuration'
+
+require 'vayacondios/server/api_options'
+require 'vayacondios/server/configuration'
+require 'vayacondios/server/driver'
+require 'vayacondios/server/drivers/mongo'
+
+require 'vayacondios/server/models/document'
+require 'vayacondios/server/models/event'
+require 'vayacondios/server/models/stash'
+
+require 'vayacondios/server/handlers/document_handler'
+require 'vayacondios/server/handlers/event_handler'
+require 'vayacondios/server/handlers/events_handler'
+require 'vayacondios/server/handlers/stash_handler'
+require 'vayacondios/server/handlers/stashes_handler'
+require 'vayacondios/server/handlers/stream_handler'

data/lib/vayacondios-server.rb

@@ -1,29 +1,35 @@
 require 'goliath'
+require 'goliath/chimp'
 require 'em-mongo'
 require 'em-synchrony/em-http'
 require 'em-synchrony/em-mongo'
+require 'configliere'
 
 require 'gorillib/object/blank'
 require 'gorillib/enumerable/sum'
+require 'gorillib/hash/compact'
 require 'gorillib/hash/deep_merge'
 require 'gorillib/hash/keys'
+require 'gorillib/model'
 require 'gorillib/string/constantize'
 require 'gorillib/string/inflections'
 require 'multi_json'
 
-require 'vayacondios/server/errors/bad_request'
-require 'vayacondios/server/errors/not_found'
+require 'vayacondios'
+require 'vayacondios/configuration'
 
-require 'vayacondios/server/model/config_document'
-require 'vayacondios/server/model/event_document'
-require 'vayacondios/server/model/itemset_document'
+require 'vayacondios/server/api_options'
+require 'vayacondios/server/configuration'
+require 'vayacondios/server/driver'
+require 'vayacondios/server/drivers/mongo'
 
-require 'vayacondios/server/handlers/config_handler'
-require 'vayacondios/server/handlers/event_handler'
-require 'vayacondios/server/handlers/itemset_handler'
+require 'vayacondios/server/models/document'
+require 'vayacondios/server/models/event'
+require 'vayacondios/server/models/stash'
 
-require 'vayacondios/server/rack/extract_methods'
-require 'vayacondios/server/rack/params'
-require 'vayacondios/server/rack/jsonize'
-require 'vayacondios/server/rack/path'
-require 'vayacondios/server/rack/path_validation'
+require 'vayacondios/server/handlers/document_handler'
+require 'vayacondios/server/handlers/event_handler'
+require 'vayacondios/server/handlers/events_handler'
+require 'vayacondios/server/handlers/stash_handler'
+require 'vayacondios/server/handlers/stashes_handler'
+require 'vayacondios/server/handlers/stream_handler'

data/lib/vayacondios.rb

@@ -0,0 +1,22 @@
+module Vayacondios
+  # Version of the api
+  API_VERSION            = 'v3'
+
+  # Gem version for both client and server
+  GEM_VERSION            = '0.3.0'
+
+  # Default port to find/connect to for the server
+  DEFAULT_SERVER_PORT    = 3467
+
+  # Default address to bind/connect to for the server
+  DEFAULT_SERVER_ADDRESS = 'localhost'
+
+  module_function
+
+  def library_dir
+    File.expand_path('../..', __FILE__)
+  end
+end
+
+# Alias for Vayacondios
+Vcd = Vayacondios