overwatch-collection 0.1.1

data/.gitignore

@@ -0,0 +1,4 @@
+log/*.log
+pkg
+.yardoc
+doc

data/.watchr

@@ -0,0 +1,18 @@
+def run_spec(file)
+  unless File.exist?(file)
+    puts "#{file} does not exist"
+    return
+  end
+
+  puts "Running #{file}"
+  system "bundle exec rspec #{file}"
+  puts
+end
+
+watch("spec/.*/*_spec\.rb") do |match|
+  run_spec match[0]
+end
+
+watch("lib/(.*/.*)\.rb") do |match|
+  run_spec %{spec/#{match[1]}_spec.rb}
+end

data/Gemfile

@@ -0,0 +1,33 @@
+source 'http://rubygems.org'
+
+gem 'dm-core', '>= 1.1.0'
+gem 'dm-active_model', '>= 1.1.0'
+gem 'dm-redis-adapter', '>= 0.4.0'
+gem 'dm-serializer', '>= 1.1.0'
+gem 'dm-timestamps', '>= 1.1.0'
+gem 'dm-validations', '>= 1.1.0'
+gem 'dm-types', '>= 1.1.0'
+gem 'yajl-ruby', '>= 0.8.2', :require => 'yajl'
+gem 'hashie', '>= 1.0.0'
+gem 'rest-client', '>= 1.6.3'
+gem 'sinatra', '>= 1.2.6'
+gem 'sinatra-logger', '>= 0.1.1', :require => 'sinatra/logger'
+gem 'activesupport', '>= 3.0.9', :require => 'active_support/all'
+gem 'gli', '>= 1.3.2'
+
+group :development, :test do
+  gem 'rspec', '>= 2.6.0'
+  gem 'rack-test', '>= 0.6.0', :require => 'rack/test'
+  gem 'spork', '>= 0.9.0.rc8'
+  gem 'watchr', '>= 0.7'
+  gem 'factory_girl', '>= 1.3.3'
+  gem 'json_spec', '>= 0.5.0'
+  gem "timecop", ">= 0.3.5"
+end
+
+
+group :doc do
+  gem 'yard'
+  # gem 'rdiscount'
+  gem 'yard-dm', '>= 0'
+end

data/Gemfile.lock

@@ -0,0 +1,104 @@
+GEM
+  remote: http://rubygems.org/
+  specs:
+    activemodel (3.0.9)
+      activesupport (= 3.0.9)
+      builder (~> 2.1.2)
+      i18n (~> 0.5.0)
+    activesupport (3.0.9)
+    addressable (2.2.6)
+    bcrypt-ruby (2.1.4)
+    builder (2.1.2)
+    diff-lcs (1.1.2)
+    dm-active_model (1.1.0)
+      activemodel (~> 3.0.4)
+      dm-core (~> 1.1.0)
+    dm-core (1.1.0)
+      addressable (~> 2.2.4)
+    dm-redis-adapter (0.4.0)
+      dm-core (>= 1.1.0)
+      dm-types (>= 1.1.0)
+      hiredis (~> 0.3.0)
+      redis (~> 2.2)
+    dm-serializer (1.1.0)
+      dm-core (~> 1.1.0)
+      fastercsv (~> 1.5.4)
+      json (~> 1.4.6)
+    dm-timestamps (1.1.0)
+      dm-core (~> 1.1.0)
+    dm-types (1.1.0)
+      bcrypt-ruby (~> 2.1.4)
+      dm-core (~> 1.1.0)
+      fastercsv (~> 1.5.4)
+      json (~> 1.4.6)
+      stringex (~> 1.2.0)
+      uuidtools (~> 2.1.2)
+    dm-validations (1.1.0)
+      dm-core (~> 1.1.0)
+    factory_girl (1.3.3)
+    fastercsv (1.5.4)
+    gli (1.3.2)
+    hashie (1.0.0)
+    hiredis (0.3.2)
+    i18n (0.5.0)
+    json (1.4.6)
+    json_spec (0.5.0)
+      json (~> 1.0)
+      rspec (~> 2.0)
+    mime-types (1.16)
+    rack (1.3.1)
+    rack-test (0.6.0)
+      rack (>= 1.0)
+    redis (2.2.1)
+    rest-client (1.6.3)
+      mime-types (>= 1.16)
+    rspec (2.6.0)
+      rspec-core (~> 2.6.0)
+      rspec-expectations (~> 2.6.0)
+      rspec-mocks (~> 2.6.0)
+    rspec-core (2.6.4)
+    rspec-expectations (2.6.0)
+      diff-lcs (~> 1.1.2)
+    rspec-mocks (2.6.0)
+    sinatra (1.2.6)
+      rack (~> 1.1)
+      tilt (< 2.0, >= 1.2.2)
+    sinatra-logger (0.1.1)
+      sinatra (>= 1.0)
+    spork (0.9.0.rc9)
+    stringex (1.2.2)
+    tilt (1.3.2)
+    timecop (0.3.5)
+    uuidtools (2.1.2)
+    watchr (0.7)
+    yajl-ruby (0.8.2)
+    yard (0.7.2)
+    yard-dm (0.1.1)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  activesupport (>= 3.0.9)
+  dm-active_model (>= 1.1.0)
+  dm-core (>= 1.1.0)
+  dm-redis-adapter (>= 0.4.0)
+  dm-serializer (>= 1.1.0)
+  dm-timestamps (>= 1.1.0)
+  dm-types (>= 1.1.0)
+  dm-validations (>= 1.1.0)
+  factory_girl (>= 1.3.3)
+  gli (>= 1.3.2)
+  hashie (>= 1.0.0)
+  json_spec (>= 0.5.0)
+  rack-test (>= 0.6.0)
+  rest-client (>= 1.6.3)
+  rspec (>= 2.6.0)
+  sinatra (>= 1.2.6)
+  sinatra-logger (>= 0.1.1)
+  spork (>= 0.9.0.rc8)
+  timecop (>= 0.3.5)
+  watchr (>= 0.7)
+  yajl-ruby (>= 0.8.2)
+  yard
+  yard-dm

data/README.md

@@ -0,0 +1,26 @@
+
+
+
+## Features
+
+### Resources
+
+One of the biggest differences
+### Snapshots
+
+When a snapshot is recorded, it's socked away in its raw form so you can come back at a later time and review the exact state of a given resource without having to piece individual metrics together.
+
+### Metrics
+
+Snapshots are also broken up and saved as individual attribute/value pairs, which enables you to track a particular attribute over a given period of time. 
+
+
+## Roadmap
+
+* Documentation. Like, seriously.
+* Log to STDOUT like a proper service.
+* Callbacks! Decide what happens after data gets collected.
+* Taggable resources and snapshots
+## TODO
+
+* Let config file be, er, configurable

data/Rakefile

@@ -0,0 +1,57 @@
+# Add your own tasks in files placed in lib/tasks ending in .rake,
+# for example lib/tasks/capistrano.rake, and they will automatically be available to Rake.
+
+require 'rake'
+
+# require 'resque/tasks'
+# require 'resque_scheduler/tasks'    
+
+require 'bundler/gem_tasks'
+
+require 'rspec/core/rake_task'
+
+desc "Run specs"
+RSpec::Core::RakeTask.new do |task|
+  task.pattern = "spec/**/*_spec.rb"
+end
+  
+desc "Run watchr"
+task :watchr do
+  sh %{bundle exec watchr .watchr}
+end
+
+desc "Run spork"
+task :spork do
+  sh %{bundle exec spork}
+end
+
+
+Bundler.require(:doc)
+desc "Generate documentation"
+YARD::Rake::YardocTask.new do |t|
+  t.files = [ 'lib/**/*.rb' ]
+end
+
+# namespace :resque do
+  # task :setup do
+    # require 'resque'
+    # require 'resque_scheduler'
+    # require 'resque/scheduler'
+    
+    # Resque.redis = 'localhost:6379'
+    
+    # Resque.schedule = YAML.load_file(
+      # File.join(File.expand_path(File.dirname(__FILE__)), 'config/schedule.yml')
+    # )
+  
+  # end
+# end    
+
+namespace :overwatch do
+  namespace :test do 
+    task :snapshot => :environment do
+      a = Asset.first
+      a.snapshots.create(:raw_data => {:one => rand(10), :two => { :three => rand(10) }})
+    end
+  end
+end

data/bin/overwatch-collection

@@ -0,0 +1,38 @@
+#!/usr/bin/env ruby
+
+require 'gli'
+
+$: << File.expand_path(File.join(File.dirname(__FILE__), "../lib"))
+
+require 'overwatch/collection'
+
+include GLI
+
+version Overwatch::Collection::VERSION
+
+desc 'Start overwatch-collection server'
+command :start do |c|
+  c.desc 'Port to which to bind'
+  c.arg_name 'PORT'
+  c.default_value '9001'
+  c.flag [:p, :port]
+  
+  c.desc 'Host on which to run'
+  c.arg_name 'HOST'
+  c.default_value 'localhost'
+  c.flag [:h, :host]
+  
+  c.desc 'Config file'
+  c.arg_name 'CONFIG'
+  c.default_value File.expand_path(File.join(File.dirname(__FILE__), "/../config/overwatch.yml"))
+  c.flag [:c, :config]
+  
+  c.action do |global_options, options, args|
+    Overwatch.config_path = options[:c]
+    Overwatch::Collection::Application.run! :host => options[:h], :port => options[:p].to_i
+  end
+  
+end
+  
+  
+exit run(ARGV)

data/config.ru

@@ -0,0 +1,9 @@
+require 'rubygems'
+require 'bundler'
+Bundler.require(:default)
+
+$: << File.join(File.dirname(__FILE__), "lib")
+
+require 'overwatch/collection'
+
+run Overwatch::Collection::Application

data/config/overwatch.yml

@@ -0,0 +1,6 @@
+collection:
+  storage:
+    adapter: 'redis'
+    db: '10'
+    host: 'localhost'
+    port: '6379'

data/lib/..rb

@@ -0,0 +1,5 @@
+require "./version"
+
+module .
+  # Your code goes here...
+end

data/lib/overwatch/collection.rb

@@ -0,0 +1,42 @@
+require 'bundler'
+Bundler.require(:default)
+
+require File.expand_path(File.join(File.dirname(__FILE__), "collection/version"))
+
+require File.expand_path(File.join(File.dirname(__FILE__), "collection/attributes"))
+require File.expand_path(File.join(File.dirname(__FILE__), "collection/models/resource"))
+require File.expand_path(File.join(File.dirname(__FILE__), "collection/models/snapshot"))
+require File.expand_path(File.join(File.dirname(__FILE__), "collection/application"))
+
+module Overwatch
+  module Collection
+  end  
+  class << self
+    def config_path=(path)
+      @config_path = path
+    end
+    
+    def config_path
+      @config_path ||= File.expand_path(File.dirname(__FILE__)) + "/../../config/overwatch.yml"
+    end
+    
+    def config
+      @config ||= {}
+      @config.merge!(YAML.load_file(config_path))
+    end
+  end
+
+end
+
+$redis = Redis.new(
+  :host => Overwatch.config['collection']['storage']['host'],
+  :port => Overwatch.config['collection']['storage']['port'],
+  :db => Overwatch.config['collection']['storage']['db']
+)
+
+DataMapper.setup(:default, { 
+  :adapter => "redis",
+  :host => Overwatch.config['collection']['storage']['host'],
+  :port => Overwatch.config['collection']['storage']['port'],
+  :db => Overwatch.config['collection']['storage']['db']
+})

data/lib/overwatch/collection/application.rb

@@ -0,0 +1,46 @@
+require 'sinatra/base'
+require 'sinatra/logger'
+# require 'sinatra/reloader' if development?
+
+module Overwatch
+  module Collection
+    class Application < Sinatra::Base
+      register Sinatra::Logger
+      configure do
+        set :app_file, __FILE__
+        set :root, File.expand_path(File.join(File.dirname(__FILE__), "../../../"))
+        set :logging, true
+        set :run, false
+        set :show_exceptions, false
+        set :server, %w[ thin mongrel webrick ]
+        set :raise_errors, false
+        set :logger_level, :info
+      end
+      
+      [ :development, :test].each do |env|
+        configure(env) do
+          set :raise_errors, true
+          set :show_exceptions, true
+          
+        end
+      end
+      
+      error DataMapper::ObjectNotFoundError do
+        halt 404
+      end
+      
+      before do
+        content_type "application/json"
+      end
+      
+      configure(:production) do
+        # TODO: allow this variable to be configured
+        set :redis_url, ENV['REDIS_URL'] || 'redis://localhost:6379/0'
+      end
+
+    end
+  end
+end
+
+require File.expand_path(File.join(File.dirname(__FILE__), "routes/resource"))
+require File.expand_path(File.join(File.dirname(__FILE__), "routes/snapshot"))

data/lib/overwatch/collection/attributes.rb

@@ -0,0 +1,136 @@
+module Overwatch
+  module Collection
+    module Attributes
+
+      def attribute_keys
+        $redis.smembers("overwatch::resource:#{self.id}:attribute_keys").sort
+      end
+
+      def average(attr, options={})
+        function(:average, attr, options)
+      end
+
+      def min(attr, options={})
+        function(:min, attr, options)
+      end
+
+      def max(attr, options={})
+        function(:max, attr, options)
+      end
+
+      def median(attr, options={})
+        function(:median, attr, options)
+      end
+
+      def first(attr, options={})
+        function(:first, attr, options)
+      end
+
+      def last(attr, options={})
+        function(:last, attr, options)
+      end
+
+      def function(func, attr, options={})
+        case func
+        when :max
+          values_for(attr, options)[:data].max
+        when :min
+          values_for(attr, options)[:data].min
+        when :average
+          values = values_for(attr, options)[:data]
+          if is_a_number?(values.first)
+            values.map!(&:to_f)
+            values.inject(:+) / values.size
+          else
+            values.first
+          end
+        when :median
+          values = values_for(attr, options)[:data].sort
+          mid = values.size / 2
+          values[mid]
+        when :first
+          value = $redis.zrangebyscore("resource:#{self.id}:#{attr}", options[:start_at], options[:end_at])[0]
+          value.split(":")[1] rescue nil
+        when :last 
+          value = $redis.zrevrangebyscore("resource:#{self.id}:#{attr}", options[:end_at], options[:start_at])[0]
+          value.split(":")[1] rescue nil
+        end
+      end
+
+      def values_for(attr, options={})
+        raise ArgumentError, "attribute does not exist" unless attribute_keys.include?(attr)
+        start_at = options[:start_at] ||  "-inf" #(Time.now - 1.day).to_i.to_s
+        end_at = options[:end_at] || "+inf"
+        interval = options[:interval]
+        values = $redis.zrangebyscore("overwatch::resource:#{self.id}:#{attr}", start_at, end_at)
+        values.map! do |v|
+          val = v.split(":")[1]
+          is_a_number?(val) ? val.to_f : val
+        end
+        values.compact!
+        values = case interval
+        when 'hour'
+          values
+        when 'day'
+          values.each_slice(60).map { |s| s[0] }
+        when 'week'
+          values.each_slice(100).map { |s| s[0] }
+        when 'month'
+          values.each_slice(432).map { |s| s[0] }
+        else
+          values
+        end
+        { :name => attr, :data => values }#, :start_at => start_at, :end_at => end_at }
+      end
+
+      def is_a_number?(str)
+        str.to_s.match(/\A[+-]?\d+?(\.\d+)?\Z/) == nil ? false : true
+      end
+
+      def values_with_dates_for(attr, options={})
+        raise ArgumentError, "attribute does not exist" unless attribute_keys.include?(attr)
+        start_at = options[:start_at] || "-inf"
+        end_at = options[:end_at] || "+inf"
+        interval = options[:interval]
+        values = $redis.zrangebyscore("overwatch::resource:#{self.id}:#{attr}", start_at, end_at)
+        values.map! do |v|
+          val = v.split(":")
+          [ val[0].to_i * 1000, is_a_number?(val[1]) ? val[1].to_f : val[1] ]
+        end
+        values.compact!
+        values = case interval
+        when 'hour'
+          values
+        when 'day'
+          values.each_slice(60).map { |s| s[0] }
+        when 'week'
+          values.each_slice(100).map { |s| s[0] }
+        when 'month'
+          values.each_slice(432).map { |s| s[0] }
+        else
+          values
+        end
+        { :name => attr, :data => values } #, :start_at => start_at, :end_at => end_at }
+      end
+
+      def from_dotted_hash(source=self.attribute_keys)
+        source.map do |main_value|
+          main_value.to_s.split(".").reverse.inject(main_value) do |value, key|
+            {key.to_sym => value}
+          end
+        end
+      end
+
+
+      def top_level_attributes
+        self.attribute_keys.map do |key|
+          key.split(".")[0]
+        end.uniq
+      end
+
+      def sub_attributes(sub_attr)
+        self.attribute_keys.select {|k| k =~ /^#{sub_attr}/ }
+      end
+    end
+  end
+end