squab 1.3.2

data/LICENSE.md

@@ -0,0 +1,13 @@
+Copyright 2013 Square Inc.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.

data/README.md

@@ -0,0 +1,4 @@
+Squab Human Oriented Event Bus
+=====
+
+This is the Squab Server directory.  Please see the root level directory for information on squab, using squab, and developing squab.

data/bin/squab

@@ -0,0 +1,4 @@
+#!/usr/bin/env ruby
+require 'rubygems'
+require 'squab/web'
+Squab::Web.run!

data/defaults.yaml

@@ -0,0 +1,5 @@
+port: 8082
+environment: development
+# Without a leading slash, this is relative to your lib/squab directory
+root: '../../'
+dbconn: "sqlite:///tmp/squab.sqlite"

data/lib/squab.rb

@@ -0,0 +1,2 @@
+require 'squab/db'
+require 'squab/events'

data/lib/squab/db.rb

@@ -0,0 +1,37 @@
+#!/usr/bin/ruby
+
+require 'rubygems'
+require 'sequel'
+
+module Squab
+  class DB
+    attr_accessor :events, :threadsafe
+
+    def initialize(conn_string)
+      @db = Sequel.connect(conn_string)
+      if not @db.table_exists?('events')
+        bootstrap
+      end
+      # SQLite is not threadsafe
+      @threadsafe = @db.database_type != :sqlite
+      @events = @db.from(:events)
+    end
+
+    def bootstrap
+      @db.create_table :events do
+          primary_key :id
+          String :uid
+          String :value
+          String :url
+          String :source
+          Float :date
+          Boolean :deleted
+          index :uid
+          index :date
+          index :source
+          index :deleted
+      end
+    end
+  end
+end
+

data/lib/squab/events.rb

@@ -0,0 +1,301 @@
+require 'rubygems'
+require 'squab/db'
+require 'json'
+
+module Squab
+  class Events < Squab::DB
+    def initialize(conn_string, opts={})
+      @return_json = opts.include?(:json) ? opts[:json] : true
+      super(conn_string)
+    end
+
+    def all(limit=1000)
+      all_events = nil
+      if limit == 0
+        all_events = @events.all.sort_by { |event| event[:id] }
+        all_events.reverse!
+      else
+        all_events = @events.limit(limit.to_i).reverse_order(:id)
+      end
+      format_rows(all_events)
+    end
+
+    # Wrap calls with this when you don't want JSON to return ever.
+    # TODO: Redo the entire internal-returning-json-thing
+    def with_no_json(&block)
+      # Whatever the situation is on json, store it, and set it to false for
+      # the period of this block call
+      returner_type = @return_json
+      @return_json = false
+
+      ret_val = block.call
+
+      # Put it back to whatever it was
+      @return_json = returner_type
+      ret_val
+    end
+
+    def event_slice(search_params)
+      # Grab all events, or just a slice of events if there's
+      # a time specified
+      # TODO: Build this off the sequel data source so it's not
+      #       a giant if/else
+      with_no_json do
+        if search_params.include?(:from) || search_params.include?(:to)
+          from = search_params.delete(:from)
+          to = search_params.delete(:to)
+          if to.nil?
+            newer_than(from)
+          elsif from.nil?
+            between(0, to)
+          else
+            between(from, to)
+          end
+        elsif search_params.include?(:fromId) || search_params.include?(:toId)
+          from_id = search_params.delete(:fromId)
+          to_id = search_params.delete(:toId)
+          if to_id.nil?
+            starting_at(from_id)
+          elsif from_id.nil?
+            between_ids(1, to_id)
+          else
+            between_ids(from_id, to_id)
+          end
+        else
+          # If all the params nil or empty or 0 then return a limited set
+          if search_params.all? {|_, v| v.to_i == 0}
+            all(1000)
+          else
+            # We have some search params, so return everything
+            all(0)
+          end
+        end
+      end
+    end
+
+    def search_fields(search_params)
+      search_params = keys_to_sym(search_params)
+      ret_limit = search_params.delete(:limit)
+      all_events = event_slice(search_params)
+
+      # Filter out nils and empty strings up front
+      # This allows the catch-all default-all to still work
+      valid_search_params = {}
+      search_params.each do |k,v|
+        next if v.nil?
+        next if v.empty?
+        valid_search_params[k] = v
+      end
+
+      # Short circuit here and give back all events if there's no search
+      # parameters
+      if valid_search_params.empty?
+        if ret_limit
+          return all_events.limit(ret_limit)
+        else
+          return all_events
+        end
+      end
+
+      ret_events = []
+
+      all_events.each do |event|
+        matched = false
+        valid_search_params.each do |k, v|
+          # Silently ignore bad search fields
+          if event[k]
+            if event[k].match(/#{v}/i)
+              matched = true
+            else
+              matched = false
+              # Stop looking, this is an AND search
+              break
+            end
+          end
+        end
+        if matched
+          ret_events.push(event)
+        end
+      end
+
+      if ret_limit
+        ret_events[0..(ret_limit.to_i - 1)]
+      else
+        ret_events
+      end
+    end
+
+    # This now brokers between the new and old style search
+    def search(pattern, fields=['value'])
+      if pattern.kind_of?(Hash)
+        format_rows(search_fields(pattern))
+      elsif pattern.kind_of?(String)
+        format_rows(search_text(pattern, fields))
+      else
+        format_rows(nil)
+      end
+    end
+
+    # Old style search, one pattern, multiple fields
+    def search_text(pattern, fields)
+      all_events = with_no_json do
+        all(0)
+      end
+      ret_events = []
+
+      all_events.each do |event|
+        event.each do |k, v|
+          if fields.member?(k.to_s) or fields.member?('all')
+            if v =~ /#{pattern}/i
+              ret_events.push(event)
+              break
+            end
+          end
+        end
+      end
+      ret_events
+    end
+
+    def by_user(uid, limit=1000)
+      user_events = @events.where(:uid => uid).reverse_order(:id).limit(limit)
+      format_rows(user_events)
+    end
+
+    def by_source(source, limit=1000)
+      source_events = @events.where(:source => source).reverse_order(:id).limit(limit)
+      format_rows(source_events)
+    end
+
+    def by_id(id)
+      single_event = @events.where(:id => id)
+      format_rows(single_event)
+    end
+
+    def recent()
+      all(50)
+    end
+
+    def delete(event_id)
+    end
+
+    def starting_at(event_id)
+      new_events =  @events.where{id >= "#{event_id}"}.reverse_order(:id)
+      format_rows(new_events)
+    end
+
+    def between_ids(start_id, end_id)
+      events = @events.where(:id => start_id..end_id)
+      format_rows(events)
+    end
+
+    def list_distinct(column)
+      column = column.to_sym
+      things = @events.select(column).distinct
+      thing_list = things.map{|t| t[column]}
+      # don't return nil, that's just silly
+      thing_list.delete(nil)
+      if @return_json
+        "#{thing_list.to_json}\n"
+      else
+        thing_list
+      end
+    end
+
+    def url_list()
+      list_distinct(:url)
+    end
+
+    def source_list()
+      list_distinct(:source)
+    end
+
+    def user_list()
+      list_distinct(:uid)
+    end
+
+    def format_rows(rows)
+      if @return_json
+        rows_to_json(rows)
+      else
+        rows
+      end
+    end
+
+    def rows_to_json(rows)
+      ret_events = []
+      unless rows.nil?
+        rows.each do |row|
+          cur_event = Squab::Event.new(row[:value],
+                                       row[:url],
+                                       row[:uid],
+                                       row[:source],
+                                       row[:date],
+                                       row[:id])
+          ret_events.push(cur_event)
+        end
+      end
+      "#{ret_events.to_json}\n"
+    end
+
+    def newer_than(timestamp)
+      new_events = @events.where{date >= "#{timestamp.to_s}"}.reverse_order(:id)
+      format_rows(new_events)
+    end
+
+    def between(from, to)
+      new_events = @events.where{date >= "#{from}"}.where{date <= "#{to}"}.reverse_order(:id)
+      format_rows(new_events)
+    end
+
+    def add_event(event)
+      if event.date.nil?
+        now = Time.now.to_i
+        event.date = now
+      end
+      @events.insert(:date => event.date,
+                     :value => event.value,
+                     :url => event.url,
+                     :source => event.source,
+                     :uid => event.uid,
+                     :deleted => false)
+      event.to_json
+    end
+
+    def keys_to_sym(old_hash)
+      new_hash = {}
+      old_hash.each do |k,v|
+        new_hash[k.to_sym] = old_hash[k]
+      end
+      new_hash
+    end
+  end
+
+  class Event
+    attr_accessor :date, :value, :url, :uid, :id, :source
+   
+    def initialize(value, url, uid, source, date=nil, id=nil)
+      @date = date
+      @value = value
+      @uid = uid
+      @url = url
+      @source = source
+      @id = id
+      @push = false
+    end
+
+    def to_h
+      {
+        date: @date,
+        uid: @uid,
+        value: @value,
+        url: @url,
+        source: @source,
+        id: @id,
+      }
+    end
+
+    def to_json(*a)
+      self.to_h.to_json(*a)
+    end
+  end
+end

data/lib/squab/web.rb

@@ -0,0 +1,231 @@
+require 'json'
+require 'pathname'
+require 'sinatra'
+require 'sinatra/config_file'
+require 'squab'
+
+module Squab
+  class Web < Sinatra::Base
+    register Sinatra::ConfigFile
+    
+    # Set up some defaults we include with the package
+    default_config = File.join(File.dirname(__FILE__), '../../defaults.yaml')
+    config_file default_config
+
+    # Check for user provided defaults
+    config = ENV['SQUAB_CONFIG'] || '/etc/squab.yaml'
+    if File.exists?(config)
+      config_file config
+      set :config, config
+    else
+      set :config, default_config
+    end
+
+    # Take whatever the connect string is from the config and feed it
+    # to Squab::Events and make an object
+    set :dbconn, Squab::Events.new(settings.dbconn)
+
+    # Some database backends are not threadsafe, use a lock if squab
+    # doesn't report they are threadsafe
+    if not settings.dbconn.threadsafe
+      enable :lock
+    end
+
+    unless settings.root.start_with?('/')
+      # Allow relative pathing
+      set :root, 
+        File.expand_path(File.join(File.dirname(__FILE__), settings.root))
+    end
+
+    enable :logging
+    enable :dump_errors
+    disable :raise_errors
+
+    configure :development do
+      # 0 is debug level
+      set :logging, 0
+      enable :show_exceptions
+    end
+
+    helpers do
+      def bad_request
+        status 400
+        redirect "api.html"
+      end
+      def safe_db(&block)
+        begin
+          block.call
+        rescue Sequel::DatabaseConnectionError, Sequel::DatabaseError => e
+          $stderr.puts e
+          exit!
+        end
+      end
+      def get_json_body(request)
+        data = nil
+        begin
+          request.body.rewind
+          data = JSON.parse request.body.read
+          logger.debug(data.to_s)
+        rescue JSON::ParserError
+          request.body.rewind
+          logger.warn("Bad JSON Body: " + request.body.read)
+          logger.debug("Bad Request: " + request.inspect.to_s)
+        end
+        data
+      end
+    end
+
+    get '/' do
+      File.read(File.join(settings.public_folder, 'events.html'))
+    end
+
+    get '/api/v1/events' do
+      safe_db do
+        settings.dbconn.all
+      end
+    end
+
+    get '/api/v1/events/recent' do
+      safe_db do
+        settings.dbconn.recent
+      end
+    end
+
+    get '/api/v1/events/limit/:limit' do |limit|
+      safe_db do
+        settings.dbconn.all(limit)
+      end
+    end
+
+    get '/api/v1/events/:id' do |id|
+      safe_db do
+        settings.dbconn.by_id(id)
+      end
+    end
+
+    get '/api/v1/events/starting/:id' do |id|
+      safe_db do
+        settings.dbconn.starting_at(id)
+      end
+    end
+
+    get '/api/v1/events/starting/:start_id/to/:end_id' do |start_id, end_id|
+      safe_db do
+        settings.dbconn.between_ids(start_id, end_id)
+      end
+    end
+
+    get '/api/v1/events/since/:date' do |date|
+      date = date.to_i
+      now = Time.now.to_i
+      if date > now
+        date = now
+      end
+      safe_db do
+        settings.dbconn.newer_than(date)
+      end
+    end
+
+    get '/api/v1/events/since/:start_date/to/:end_date' do |start_date, end_date|
+      start_date = start_date.to_i
+      end_date = end_date.to_i
+      now = Time.now.to_i
+      if end_date > now
+        end_date = now
+      end
+      if start_date > end_date
+        []
+      else
+        safe_db do
+          settings.dbconn.between(start_date, end_date)
+        end
+      end
+    end
+
+    get '/api/v1/events/user/:user' do |user|
+      safe_db do
+        settings.dbconn.by_user(user)
+      end
+    end
+
+    get '/api/v1/events/source/:source' do |source|
+      safe_db do
+        settings.dbconn.by_source(source)
+      end
+    end
+
+    get '/api/v1/events/search/:field/:pattern' do |field, pattern|
+      safe_db do
+        settings.dbconn.search(pattern, [field])
+      end
+    end
+
+    get '/api/v1/events/search/:field/:pattern/limit/:limit' do |field, pattern, limit|
+      safe_db do
+        settings.dbconn.search({field => pattern, :limit => limit})
+      end
+    end
+
+    get '/api/v1/users' do
+      safe_db do
+        settings.dbconn.user_list
+      end
+    end
+
+    get '/api/v1/urls' do
+      safe_db do
+        settings.dbconn.url_list
+      end
+    end
+
+    get '/api/v1/sources' do
+      safe_db do
+        settings.dbconn.source_list
+      end
+    end
+
+    post '/api/v1/events' do
+      data = get_json_body(request)
+      return 400 unless data.kind_of?(Hash)
+      safe_db do
+        e = settings.dbconn
+
+        new_event = Event.new(data['value'],
+                              data['url'],
+                              data['uid'],
+                              data['source'],
+                              data['date'])
+
+        "#{e.add_event(new_event)}\n"
+      end
+    end
+
+    post '/api/v1/events/search' do
+      data = get_json_body(request)
+      return 400 unless data.kind_of?(Hash)
+      safe_db do
+        if data.has_key?('fields') && data.has_key?('pattern')
+          settings.dbconn.search(data["pattern"], data["fields"])
+        else
+          settings.dbconn.search(data)
+        end
+      end
+    end
+
+    get '/_status' do
+      if File.exists?(File.join(File.dirname(settings.config), ('down')))
+        500
+      else
+        200
+      end
+    end
+
+    get '/api' do
+      redirect "api.html"
+    end
+
+    get '*' do
+      File.read(File.join(settings.public_folder, 'events.html'))
+    end
+  end
+end