glowworm 0.3.0

data/lib/glowworm/em.rb

@@ -0,0 +1,8 @@
+begin
+  require "em-synchrony"
+  require "em-net-http"
+rescue LoadError
+  raise "You must have em/sinatra-synchrony AND em-net-http installed for the glowworm/em to work"
+end
+require "glowworm"
+Glowworm.send(:_em_)

data/lib/glowworm/no_bg.rb

@@ -0,0 +1,2 @@
+require "glowworm"
+Glowworm.no_bg

data/lib/glowworm/version.rb

@@ -0,0 +1,3 @@
+module Glowworm
+  VERSION = "0.3.0"
+end

data/server/Gemfile

@@ -0,0 +1,27 @@
+source "http://rubygems.org"
+source "http://gems.ooyala.com"
+
+gem "capistrano"
+gem "ecology", "~>0.0.14"
+gem "multi_json", "~>1.3.2"
+gem "mysql"  # Should be mysql2, but sequel doesn't support it right now
+gem "railsless-deploy", :require => nil  # see https://github.com/leehambley/railsless-deploy#bundler-usage
+gem "sequel"
+gem "termite", '~>0.0.14'
+gem "yajl-ruby"   # This is much faster than MultiJson's default OKJson
+
+group :deploy do
+  gem "email-utils"
+end
+
+group :test do
+  gem "sqlite3"
+  gem "trollop"
+  gem "glowworm"
+  gem "minitap",  ">= 0.3.5"
+end
+
+group :development do
+  gem "pry"
+  gem "httparty"
+end

data/server/Gemfile.lock

@@ -0,0 +1,87 @@
+GEM
+  remote: http://rubygems.org/
+  remote: http://gems.ooyala.com/
+  specs:
+    ansi (1.4.3)
+    capistrano (2.14.1)
+      highline
+      net-scp (>= 1.0.0)
+      net-sftp (>= 2.0.0)
+      net-ssh (>= 2.0.14)
+      net-ssh-gateway (>= 1.1.0)
+    chronic (0.9.0)
+    coderay (1.0.8)
+    ecology (0.0.18)
+      erubis
+      multi_json
+    email-utils (0.0.2)
+    erubis (2.7.0)
+    glowworm (0.1.8)
+      ecology (~> 0.0.18)
+      hastur (~> 1.2.0)
+      httparty
+      multi_json
+      termite (~> 0.0.13)
+      trollop
+    hastur (1.2.18)
+      chronic
+      multi_json (~> 1.3.2)
+    highline (1.6.15)
+    httparty (0.10.0)
+      multi_json (~> 1.0)
+      multi_xml
+    json (1.7.6)
+    method_source (0.8.1)
+    minitap (0.3.5)
+      minitest
+      tapout (>= 0.3.0)
+    minitest (4.4.0)
+    multi_json (1.3.7)
+    multi_xml (0.5.2)
+    mysql (2.9.0)
+    net-scp (1.0.4)
+      net-ssh (>= 1.99.1)
+    net-sftp (2.0.5)
+      net-ssh (>= 2.0.9)
+    net-ssh (2.6.3)
+    net-ssh-gateway (1.1.0)
+      net-ssh (>= 1.99.1)
+    pry (0.9.10)
+      coderay (~> 1.0.5)
+      method_source (~> 0.8)
+      slop (~> 3.3.1)
+    railsless-deploy (1.0.2)
+    rainbow (1.1.4)
+    sequel (3.43.0)
+    slop (3.3.3)
+    sqlite3 (1.3.7)
+    tapout (0.4.1)
+      ansi
+      json
+    termite (0.0.20)
+      ecology (~> 0.0.6)
+      multi_json
+      rainbow (~> 1.1.3)
+    trollop (2.0)
+    yajl-ruby (1.1.0)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  capistrano
+  ecology (~> 0.0.14)
+  email-utils
+  glowworm
+  hastur (~> 1.2.0)
+  httparty
+  minitap (>= 0.3.5)
+  multi_json (~> 1.3.2)
+  mysql
+  pry
+  railsless-deploy
+  sequel
+  sqlite3
+  termite (~> 0.0.14)
+  trollop
+  yajl-ruby

data/server/PROTOCOL

@@ -0,0 +1,39 @@
+Glowworm's wire protocol is intentionally very simple, and carried via
+HTTP requests.  Some goals, in order of importance:
+
+* Simplicity, in order to achieve high Reliability and Availability
+* Readability and Debuggability
+* Not too unreasonably high bandwidth
+
+A given request will be a read of a set of accounts and a set of
+features.  Usually, that is one of:
+
+* One feature for one account
+* One feature for all accounts
+* All features for one account
+* All features for all accounts
+
+A request will return data in the following form:
+
+* What account_sets the given account(s) are part of
+* The setting of each feature for each account_set
+* A list of override flags for specific account/feature combinations
+
+That data is serialized as JSON, in the following form:
+
+{
+  "version": 1,
+  "account_sets": {
+    "13742": 1,  // account 13742 is part of account_set 1
+    "19244": [7,8],  // account 19244 is part of account_sets 7 and 8
+    //...
+  },
+  "features": {
+    "turn_foo_widget_orange": [1, 2, 3, 4, 6, 7, 8], // account_sets for which this feature is active by default
+    //...
+  },
+  "overrides": [
+    "turn_foo_widget_orange": {"19244": false, "742": true},
+    //...
+  ]
+}

data/server/check_mk_checks/check_glowworm_server

@@ -0,0 +1,43 @@
+#!/usr/bin/env ruby
+require "time"
+
+SERVICE_NAME="GlowwormServerStatus"
+HOST="127.0.0.1"
+PORT=4999
+URI="/healthz"
+URL="http://#{HOST}:#{PORT}#{URI}"
+
+OUTPUT=`curl -I #{URL} 2> /dev/null`
+if OUTPUT.empty?
+  puts "2 #{SERVICE_NAME} - Glowworm Nginx Server Down"
+  exit
+end
+
+MOD_TIME=Time.parse `echo "#{OUTPUT}" |grep Last-Modified |sed -e 's/^Last-Modified: //'`
+HTTP_STATUS=OUTPUT.split[1].to_i
+
+# TODO(jbhat): We need to check that output's time, which looks like: Thu, 18 Oct 2012 05:56:57 GMT,
+# is within 10 minutes of now. If we get a 404, we should return that the glowworm server is down.
+# If we get a 500, we should report that there is an error (shouldn't happen).
+# If we get a 200, but the file is more than 10 minutes ago, we should return an error of stale data.
+# If we get a 200 with fresh file, but the contents are not OOYALA GLOWWORM OK, we should error its contents.
+# Finally, we can return ok if the file has last been modified within 10 minutes, and contains the string.
+STATUS, OUTPUT_TEXT = case HTTP_STATUS
+when 200
+  AGE=Time.now - MOD_TIME
+  if(AGE > 600)
+    [2, "Data Stale, healthz has not been updated in #{AGE} seconds."]
+  else
+    HEALTHZ_CONTENTS=`curl #{URL} 2> /dev/null`.chomp
+    if HEALTHZ_CONTENTS =~ /OOYALA GLOWWORM OK/
+      [0, HEALTHZ_CONTENTS]
+    else
+      [2, HEALTHZ_CONTENTS]
+    end
+  end
+when 404
+  [2, "404 Glowworm Service Down"]
+else
+  [2, "#{STATUS} Error Retrieving Healthz"]
+end
+puts "#{STATUS} #{SERVICE_NAME} - #{OUTPUT_TEXT}"

data/server/db_migrations/20111004214649_change_feature_accounts_to_string.rb

@@ -0,0 +1,60 @@
+Sequel.migration do
+  up do
+    create_table(:account_sets) do
+      primary_key :id
+      Datetime :created_at, :null => false
+      Datetime :updated_at, :null => false
+      String :name, :null => false
+      String :description
+      index :name, :unique => true
+    end
+
+    create_table(:account_set_accounts) do
+      primary_key :id
+      Datetime :added_at, :null => false
+      Integer :account_set_id, :null => false
+      Integer :provider_id, :null => true
+      String :account, :limit => 10, :null => false
+      index :account_set_id
+      index :provider_id
+      index :account
+      index [:account_set_id, :account], :unique => true
+    end
+
+    create_table(:features) do
+      primary_key :id
+      Datetime :created_at, :null => false
+      Datetime :updated_at, :null => false
+      String :name, :null => false
+      String :description
+      index :name, :unique => true
+    end
+
+    create_table(:account_set_features) do
+      primary_key :id
+      Datetime :added_at, :null => false
+      Integer :account_set_id, :null => false
+      Integer :feature_id, :null => false
+      index :feature_id
+      index [:account_set_id, :feature_id], :unique => true
+    end
+
+    create_table(:feature_account_overrides) do
+      Integer :provider_id, :null => true
+      String :account, :limit => 10, :null => false
+      Integer :feature_id, :null => false
+      Integer :value, :null => false
+      index :provider_id
+      index :feature_id
+      index [:account, :feature_id], :unique => true
+    end
+  end
+
+  down do
+    drop_table :account_sets
+    drop_table :account_set_accounts
+    drop_table :features
+    drop_table :account_set_features
+    drop_table :feature_account_overrides
+  end
+end

data/server/db_migrations/20111028104546_add_value_to_account_set_features.rb

@@ -0,0 +1,12 @@
+require File.join(File.dirname(__FILE__), "migration_helper")
+
+Sequel.migration do
+  up do
+    add_column :account_set_features, :value, Integer, :null => false, :default => 1
+    self[:account_set_features].update(:value => 1)
+  end
+
+  down do
+    drop_column :account_set_features, :value
+  end
+end

data/server/db_migrations/20120217090636_add_fully_active_flag_to_features.rb

@@ -0,0 +1,15 @@
+require File.join(File.dirname(__FILE__), "migration_helper")
+
+Sequel.migration do
+  up do
+    alter_table(:features) do
+      add_column :fully_active, :boolean, :default => false
+    end
+  end
+
+  down do
+    alter_table(:features) do
+      drop_column :fully_active
+    end
+  end
+end

data/server/example_test_data.rb

@@ -0,0 +1,66 @@
+#!/usr/bin/env ruby
+
+require "ecology"
+require "sequel"
+require "termite"
+
+Ecology.read("glowworm_server.ecology")
+
+db_spec = Ecology.property("db") || "sqlite:glowworm_server.sqlite"
+DB = Sequel.connect db_spec
+MyLogger = Termite::Logger.new
+
+if ARGV.include?("--clear")
+  DB[:account_sets].delete
+  DB[:features].delete
+  DB[:account_set_accounts].delete
+  DB[:account_set_features].delete
+  DB[:feature_account_overrides].delete
+end
+
+DB[:account_sets].insert(:name => "Major", :description => "Major accounts", :id => 1,
+                         :created_at => Time.now, :updated_at => Time.now)
+DB[:account_sets].insert(:name => "Minor", :description => "Minor accounts", :id => 2,
+                         :created_at => Time.now, :updated_at => Time.now)
+
+
+DB[:account_set_accounts].insert(:account_set_id => 1, :account => "27366", :provider_id => 27366,
+                                 :added_at => Time.now)
+
+# Add account #35 to major accounts
+DB[:account_set_accounts].insert(:account_set_id => 1, :account => "35", :provider_id => 35,
+                                 :added_at => Time.now)
+
+# Add account #42 to minor accounts
+DB[:account_set_accounts].insert(:account_set_id => 2, :account => "42", :provider_id => 42,
+                                 :added_at => Time.now)
+
+# Add account #49 to minor accounts
+DB[:account_set_accounts].insert(:account_set_id => 2, :account => "49", :provider_id => 49,
+                                 :added_at => Time.now)
+
+DB[:features].insert(:id => 100, :name => "video_rec", :description => "video_rec",
+                     :created_at => Time.now, :updated_at => Time.now)
+
+
+DB[:features].insert(:id => 1, :name => "foo_feature", :description => "foo",
+                     :created_at => Time.now, :updated_at => Time.now)
+
+DB[:features].insert(:id => 2, :name => "new_signup_message", :description => "Use new signup message?",
+                     :created_at => Time.now, :updated_at => Time.now)
+
+# Turn on foo_feature for major accounts, but not minor
+DB[:account_set_features].insert(:account_set_id => 1, :feature_id => 1,
+                                 :added_at => Time.now)
+DB[:account_set_features].insert(:account_set_id => 1, :feature_id => 100,
+                                 :added_at => Time.now)
+
+if ARGV.include?("--new-signup")
+  DB[:account_set_features].insert(:account_set_id => 1, :feature_id => 2,
+                                   :added_at => Time.now)
+end
+
+DB[:feature_account_overrides].insert(:account => "49", :provider_id => 49, :feature_id => 1,
+                                      :value => 1)
+
+puts "Added test data successfully!"

data/server/glowworm_server.ecology.erb

@@ -0,0 +1,16 @@
+<%
+  begin
+    require 'config/pushed_environment'
+  rescue LoadError
+    nil
+  end
+
+  db_user = (DB_USER rescue "root")
+  db_password = (DB_PASSWORD rescue "")
+  db_location = (DB_LOCATION rescue "localhost")
+%>
+{
+  "application": "glowworm_server",
+  "db": "mysql://<%= db_user %>:<%= db_password %>@<%= db_location %>/vstreams",
+  "cache_period": 300
+}

data/server/glowworm_server.rb

@@ -0,0 +1,226 @@
+#!/usr/bin/env ruby
+
+require "sequel"
+require "yajl"
+require "multi_json"
+require "ecology"
+require "termite"
+require "mysql"
+
+require_relative "config/pushed_environment"
+require "digest/md5"
+require "fileutils"
+
+$LOAD_PATH << File.dirname(__FILE__)
+# Try to load the symlinked Glowworm version file, if it rsync'd
+(require "version") rescue nil
+
+GLOWWORM_VERSION = (Glowworm::VERSION rescue "Unknown")
+
+# Be explicit about engine
+MultiJson.use :yajl
+
+# Calculate the md5 from the data passed in.  Assumes a hash with no md5 unless told otherwise
+def md5_from_hash(json_hash, options = { :json => false, :md5 => false })
+  if options[:json]
+    json_hash = MultiJson.load json_hash rescue {} # Assume empty hash if there is a problem
+  end
+
+  json_hash.delete "md5" if options[:md5]
+
+  dumped_json = MultiJson.dump json_hash
+  md5 = Digest::MD5.hexdigest(dumped_json)
+  md5
+end
+
+# accounts - hash of account names to account set IDs
+# features - hash of feature names to arrays of account sets that set it
+# overrides - hash of feature names to hashes of {account => value}
+def format_json(accounts, features, overrides, fully_active)
+  json_hash = {
+    "version" => 1,
+    "account_sets" => accounts,
+    "features" => features,
+    "overrides" => overrides,
+    "fully_active" => fully_active,
+  }
+
+  return json_hash
+end
+
+class DBException < Exception
+end
+
+def try_db
+  yield
+rescue => e
+  MyLogger.error "Error talking to database. Message: #{e.message}. Backtrace: #{e.backtrace}"
+  raise DBException
+end
+
+def json_for_account_and_feature
+  MyLogger.info "Querying DB for all accounts and features"
+
+  # If there has previously been a DB failure, try to reconnect right away
+  try_db do
+    DB.connect DB_SPEC
+    @failures = 0
+  end if @failures > 0
+
+  # Right now, account_sets with no accounts won't be returned.  I think that's fine.
+  accounts = {}
+  try_db do
+    DB[:account_set_accounts].each do |row|
+      accounts[row[:account]] ||= []
+      accounts[row[:account]] << row[:account_set_id].to_s
+    end
+  end
+
+  features = {}
+  fully_active = {}
+  try_db do
+    DB[:features].left_join(:account_set_features, :feature_id => :id).each do |row|
+      features[row[:name]] ||= {}
+      features[row[:name]][row[:account_set_id].to_s] = row[:value] if row[:account_set_id]
+      fully_active[row[:name]] = row[:fully_active]
+    end
+  end
+
+  overrides = {}
+  try_db do
+    DB[:feature_account_overrides].join(:features, :id => :feature_id).each do |row|
+      overrides[row[:name]] ||= {}
+      overrides[row[:name]][row[:account]] = row[:value]
+    end
+  end
+
+  json = format_json accounts, features, overrides, fully_active
+
+  MyLogger.debug "Returning JSON: #{json}"
+
+  [200, json]
+rescue DBException
+  [500, ""]
+end
+
+Ecology.read("glowworm_server.ecology")
+SERVING_DIRECTORY = Ecology.property("nginx_directory") || "#{SHARED_PATH}/www"
+LIVE_FILENAME = "#{SHARED_PATH}/server_is_live" unless defined? LIVE_FILENAME
+FileUtils.mkdir_p SERVING_DIRECTORY
+AIM_FILE = ENV['AIM_FILE'] || "#{SERVING_DIRECTORY}/all_if_modified"
+STATUSZ_FILE = "#{SERVING_DIRECTORY}/statusz"
+HEALTHZ_FILE = "#{SERVING_DIRECTORY}/healthz"
+DB_SPEC = Ecology.property("db") || "sqlite:glowworm_server.sqlite"
+cache_period = Ecology.property("cache_period") || 60
+MyLogger = Termite::Logger.new nil, nil, nil, :use_logger_prefix => true, :stderr_level => "info"
+@failures = 0
+begin
+  DB = try_db { Sequel.connect DB_SPEC }
+rescue DBException
+  @failures += 1
+end
+md5_hash = if File.exists? AIM_FILE
+  contents = File.open(AIM_FILE).read
+  md5_from_hash(contents, :json => true, :md5 => true)
+else
+  nil
+end
+
+def write_statusz_file(status)
+  str = "Glowworm server from gem version '#{GLOWWORM_VERSION}' Running on Ruby #{RUBY_VERSION}.\n"
+  str += if status == 200 && @failures.zero?
+    "Last modified #{@last_modified || "in previous run period"}."
+  else
+    "Last update failed due to database connection error at #{Time.now}.\n" +
+    "#{@failures} consecutive connection errors encountered."
+  end
+  File.open(STATUSZ_FILE, "w") do |file|
+    file.puts str
+  end
+end
+
+def write_healthz_file(status)
+  # Return 404 if LIVE_FILENAME is not present
+  if !File.exists?(LIVE_FILENAME)
+    File.delete HEALTHZ_FILE if File.exists?(HEALTHZ_FILE)
+    return
+  end
+
+  str = if status == 200 && @failures.zero?
+    "OOYALA GLOWWORM OK"
+  else
+    "Error reconnecting to database. #{@failures} consecutive connection errors encountered."
+  end
+  File.open(HEALTHZ_FILE, "w") do |file|
+    file.puts str
+  end
+end
+
+unless File.exists? STATUSZ_FILE
+  # Write statusz file. Parameter is status code, but will output the right thing unless passed 500.
+  write_statusz_file nil
+end
+
+if File.exists?(HEALTHZ_FILE) && !File.exists?(LIVE_FILENAME)
+  # Write statusz file. Parameter is status code, but will output the right thing unless passed 500
+  write_healthz_file nil
+end
+
+# This is an implementation of "do while": run once if ENV["RUN_ONCE"] is set, loop otherwise
+run_repeatedly = !ENV.has_key?("RUN_ONCE")
+begin
+  status, json_hash = json_for_account_and_feature
+
+  new_md5_hash = md5_from_hash(json_hash)
+
+  if status == 200 && @failures.zero?
+    @failures = 0
+    @last_success = Time.now
+    if md5_hash != new_md5_hash || !File.exists?(AIM_FILE)
+      @last_modified = Time.now
+
+      json_hash[:md5] = new_md5_hash
+      # Yes, this is reencoding all the JSON.  We *could* just do hideous string
+      # surgery and replace the final close-brace with a comma, the MD5 bit, and
+      # a new close-brace.  For now, clarity beats efficiency since this runs rarely.
+      json_string = MultiJson.dump(json_hash)
+      # If we get to this point, then the file should have been modified, otherwise the hashes would match
+      File.open(AIM_FILE, "w") do |file|
+        file.write json_string
+      end
+      md5_hash = new_md5_hash
+      MyLogger.info "NEW_DATA - #{json_string[0..40]}..., hash #{md5_hash}"
+    else
+      MyLogger.info "No new data - checksum #{md5_hash}"
+    end
+  else # status == 500
+    @failures += 1
+    @retry_time = case @failures
+    when 0..1
+      0 # retry immediately on first failure
+    when 2..7
+      2 ** (@failures - 2) # 1, 2, 4, 8, 16, 32
+    else
+      55 # sleep for 55 seconds after 7th failure (because cmk retry is 60 seconds)
+    end
+    MyLogger.error "#{@failures} consecutive database failure(s). Retrying in #{@retry_time} seconds."
+  end
+  write_statusz_file status
+  write_healthz_file status if status != 500 || (Time.now - @last_success) > 60 * 10
+  if run_repeatedly
+    # Randomize the cache period so we don't get a thundering herd of
+    # Glowworm servers all updating on the same five-minute
+    # boundary every five minutes.  This is a big DB query.
+    #
+    # We continue to randomize as we go forward instead of starting
+    # with a random delay.  This is because delays in service will
+    # otherwise tend to clump servers up over time, causing an
+    # emergent thundering herd.
+    sleep_period = if status == 500
+      @retry_time
+    else
+      (cache_period / 2) + Random.rand(cache_period + 1)
+    end
+    sleep sleep_period
+  end
+end while run_repeatedly