glowworm 0.3.0

data/bin/load_tester

@@ -0,0 +1,84 @@
+#!/usr/bin/env ruby
+
+require "glowworm"
+require "trollop"
+
+OPTS = Trollop::options do
+  opt :concurrent, "How many threads", :type => :int, :default => 1
+  opt :runs, "How many times to run", :type => :int, :default => 1
+  opt :account, "An account to query", :type => String, :required => true, :multi => true
+  opt :feature, "A feature to query", :type => String, :required => true, :multi => true
+  opt :server, "Server to query", :type => String, :default => "http://localhost:4999"
+  opt :fork, "Fork instead of threading", :type => :boolean, :default => false
+  opt :delay, "Seconds to sleep between requests (e.g. 0.001)", :type => :float, :default => 0.0
+  opt :header, "Print header information", :type => :boolean, :default => true
+  opt :log_too_much, "Log too much", :type => :boolean
+  opt :show_backtrace, "Show error traces", :type => :boolean
+  opt :timeout, "Timeout per request", :type => Float, :default => 0.1
+end
+
+if OPTS[:header]
+  puts "We print dot for correct responses, F for incorrect responses, and E for exceptions."
+  puts "#{OPTS[:concurrent]} #{OPTS[:fork] ? "processes" : "threads"}, #{OPTS[:server]}"
+  puts "Client version #{Glowworm::VERSION} go!"
+end
+
+CHAR_LIST = (0..9).to_a + ("a".."z").to_a + ("A".."Z").to_a
+def num_to_char(num)
+  n = num % 62
+  CHAR_LIST[n]
+end
+
+Glowworm.server = OPTS[:server]
+
+if ENV["TEST_GLOWWORM_VERSION"] && ENV["TEST_GLOWWORM_VERSION"] != "latest" && Glowworm::VERSION != ENV["TEST_GLOWWORM_VERSION"]
+  fail "Should be testing under #{ENV["TEST_GLOWWORM_VERSION"]}"
+end
+
+$stderr.puts "Glowworm version: #{Glowworm::VERSION}"
+
+# Silence all logging
+fake_logger = Object.new
+def fake_logger.method_missing(*args); end
+Glowworm.termite_logger = fake_logger
+
+threads = []
+forks = []
+OPTS[:concurrent].times do
+  action = proc do
+    if OPTS[:log_too_much]
+      # TODO(noah): Use thread ID not PID for threads
+      log_path = "/tmp/gw_multi_test_#{Process.pid}_#{num_to_char(Process.pid)}"
+      logger = Termite::Logger.new(log_path, nil, nil, :level => "debug", :console_print => false)
+      Glowworm.termite_logger = logger
+    end
+    OPTS[:runs].times do
+      begin
+        x = Glowworm.feature_flag(OPTS[:account].sample, OPTS[:feature].sample, :default => false,
+                                  :timeout => OPTS[:timeout], :ttl => 0.0)
+        if x == true
+          print "."
+        else
+          print "F(#{num_to_char(Process.pid)})"
+        end
+      rescue
+        print "E"
+        puts "Backtrace: #{$!.message}\n #{$!.backtrace.join("\n")}" if OPTS[:show_backtrace]
+      end
+      sleep(OPTS[:delay]) if OPTS[:delay] > 0
+    end
+  end
+
+  if OPTS[:fork]
+    forks << fork(&action)
+  else
+    threads << Thread.new(&action)
+  end
+end
+
+if OPTS[:fork]
+  forks.each { |f| Process.wait f }
+else
+  threads.each{ |t| t.join }
+end
+puts ""

data/ci_jobs/glowworm-continuous-deploy-next-staging/run.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+export PATH=~/.rbenv/bin/:${PATH}
+eval "$(rbenv init - --no-rehash)"
+
+export OO_DEPLOY_DIR=/home/role-c3/ooyala-security/deploy_configs
+cd glowworm/server/
+gem install --no-rdoc --no-ri bundler
+bundle install --without test
+bundle exec cap to_next_staging deploy:full

data/ci_jobs/glowworm-integrations/run.sh

@@ -0,0 +1,15 @@
+#!/bin/bash -l
+
+if [[ -z $WORKSPACE ]] ; then
+  # Support execution from the shell
+  export PROJECT_DIR=$(pwd)
+else
+  export PROJECT_DIR="$WORKSPACE/glowworm"
+fi
+
+source "$PROJECT_DIR/ci_jobs/setup.sh"
+
+echo START TASK: tests
+exec_rake_task test:integration
+echo END TASK
+

data/ci_jobs/glowworm-performance/run.sh

@@ -0,0 +1,2 @@
+#!/usr/bin/env bash -l
+exit 1

data/ci_jobs/glowworm-robustness/run.sh

@@ -0,0 +1,2 @@
+#!/usr/bin/env bash -l
+exit 1

data/ci_jobs/glowworm-units/run.sh

@@ -0,0 +1,13 @@
+#!/bin/bash -l
+
+if [[ -z $WORKSPACE ]] ; then
+  # Support execution from the shell
+  export PROJECT_DIR=$(pwd);
+else
+  export PROJECT_DIR=$WORKSPACE/glowworm;
+fi
+
+source $PROJECT_DIR/ci_jobs/setup.sh
+
+echo START TASK: tests
+exec_rake_task test:units

data/ci_jobs/setup.sh

@@ -0,0 +1,119 @@
+#!/usr/bin/env bash -l
+set -e
+
+echo START TASK: Determining test config
+
+export TAP_OUTPUT=true;
+
+if [[ -z $WORKSPACE ]] ; then
+  export RESULTS_DIR="$(pwd)/results"
+else
+  export RESULTS_DIR="$WORKSPACE/results"
+fi
+
+rm -rf "$RESULTS_DIR"
+mkdir -p "$RESULTS_DIR"
+
+cd "$PROJECT_DIR"
+
+# The ruby versions to test are without the gemset specifier.
+# Default to testing 1.9.3-p194. Which is the approved 1.9.3 ruby version.
+: ${TEST_RUBY_VERSION:=1.9.3-p194}
+
+# Default the server and client ruby versions to TEST_RUBY_VERSION
+: ${TEST_SERVER_RUBY_VERSION:=$TEST_RUBY_VERSION}
+: ${TEST_CLIENT_RUBY_VERSION:=$TEST_RUBY_VERSION}
+
+export TEST_RUBY_VERSION TEST_SERVER_RUBY_VERSION TEST_CLIENT_RUBY_VERSION
+
+# default to verifying the glowworm code contained in the repo
+# Other valid values are the glowworm gem version that should be used.
+: ${TEST_GLOWWORM_VERSION:=latest}
+
+export TEST_GLOWWORM_VERSION
+
+# The ruby distribution to test is with gemset specifier
+export TEST_CLIENT_RUBY_DIST=${TEST_CLIENT_RUBY_VERSION}
+export TEST_SERVER_RUBY_DIST=${TEST_SERVER_RUBY_VERSION}
+
+echo END TASK
+
+echo START TASK: Load test harness ruby dist
+
+if [[ -z ${TEST_HARNESS_RUBY_VERSION} ]]; then
+  export TEST_HARNESS_RUBY_DIST=1.9.3-p125
+else
+  export TEST_HARNESS_RUBY_DIST=${TEST_HARNESS_RUBY_VERSION}
+fi
+
+echo END TASK
+
+echo START TASK: init rbenv
+
+eval "$(rbenv init -)"
+
+echo END TASK
+
+function setup_ruby_dist
+{
+  rbenv shell $1
+
+  gem install --no-rdoc --no-ri bundler
+
+  bundle install
+}
+
+echo START TASK: Init ruby distribution ${TEST_SERVER_RUBY_DIST}
+
+# Setup the server ruby distribution
+( cd server ; setup_ruby_dist ${TEST_SERVER_RUBY_DIST} );
+
+echo END TASK
+
+echo START TASK: Init ruby distribution ${TEST_CLIENT_RUBY_DIST}
+
+# If the glowworm version is 'latest' then just setup the client using bundle install
+if [[ $TEST_GLOWWORM_VERSION == latest ]]; then
+  echo using latest;
+
+  setup_ruby_dist ${TEST_CLIENT_RUBY_DIST}
+else
+  echo using ${TEST_GLOWWORM_VERSION}
+
+  rbenv shell ${TEST_CLIENT_RUBY_DIST}
+
+  gem install --no-rdoc --no-ri bundler
+
+  bundle install --gemfile ${PROJECT_DIR}/test/integration/gemfile_for_specific_glowworm_version
+fi
+
+echo Running test harness under ${TEST_HARNESS_RUBY_DIST};
+
+# setup the test harness ruby distribution as a superset of the client and server distributions
+# along with additional test harness specific packages.
+setup_ruby_dist ${TEST_HARNESS_RUBY_DIST};
+(cd server ; setup_ruby_dist ${TEST_HARNESS_RUBY_DIST});
+
+# TODO(corey): install via a Gemfile
+gem install --no-rdoc --no-ri tapout
+
+echo END TASK
+
+function translate_TAP_results()
+{
+  actual_exit_code=$?
+  find "$RESULTS_DIR" -name '*.yaml' -exec bash -c 'tapout tap < {} > {}.tap' \;
+  return $actual_exit_code
+}
+
+function exec_rake_task()
+{
+  # TODO(corey): Have the rake task output each unit test to an individual YAMLish TAP file
+  # Always try to translate the YAMLish TAP to PERLish TAP for jenkins
+  trap translate_TAP_results EXIT
+
+  # Also output to the current STDOUT so the console log has all data.
+  # this is last to assure the exit code of the script is the result of rake
+  rake "$1" "$@" | tee $RESULTS_DIR/results.yaml
+  exit ${PIPESTATUS[0]}
+}

data/example/example_server.ecology

@@ -0,0 +1,6 @@
+{
+  "application": "example of using Glowworm",
+  "features": {
+    "server": "http://localhost:4999"
+  }
+}

data/example/example_server.rb

@@ -0,0 +1,32 @@
+#!/usr/bin/env ruby
+
+require "rubygems"
+require "sinatra"
+require "glowworm"
+
+Ecology.read("example_server.ecology")
+
+configure do
+  set :port, "8087"
+end
+
+get "/" do
+  if Glowworm.feature_flag(35, "new_signup_message", :ttl => 15)
+    button_text = "join now for free?"
+  else
+    button_text = "sign up?"
+  end
+
+<<STATIC_PAGE
+  <html>
+    <head>
+      <title>Feature Test App</title>
+      <meta http-equiv="refresh" content="5">
+    </head>
+    <body>
+      <h1> Feature Test App </h1>
+      Why don't you <button>#{button_text}</button>
+    </body>
+  </html>
+STATIC_PAGE
+end

data/glowworm.gemspec

@@ -0,0 +1,54 @@
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
+
+require "glowworm/version"
+
+Gem::Specification.new do |s|
+  s.name        = "glowworm"
+  s.version     = Glowworm::VERSION
+  s.platform    = Gem::Platform::RUBY
+  s.authors     = ["Noah Gibbs"]
+  s.email       = ["noah@ooyala.com"]
+  s.homepage    = "http://www.ooyala.com"
+  s.summary     = %q{Ruby feature management for dark deploys}
+  s.description = <<EOS
+Glowworm queries and caches feature flags from a central
+REST server, while balancing quick changes and good performance.
+EOS
+
+  s.rubyforge_project = "glowworm"
+
+  ignores = File.readlines(".gitignore").grep(/\S+/).map {|pattern| pattern.chomp }
+  ignores += ["server/config/*", "server/Capfile", "server/shared_lib/*",
+              "server/etc/*", "server/tools/*"]
+  dotfiles = Dir[".*"]
+  s.files = Dir["**/*"].reject {|f| File.directory?(f) || ignores.any? {|i| File.fnmatch(i, f) } } + dotfiles
+  s.test_files = s.files.grep(/^test\//)
+  s.executables << "glowworm"
+
+  s.bindir = 'bin'
+  s.require_paths = ["lib"]
+
+  s.add_dependency "ecology", "~>0.0.18"
+  s.add_dependency "termite", "~>0.0.13"
+  s.add_dependency "httparty"
+  s.add_dependency "multi_json"
+  s.add_dependency "trollop"
+
+  s.add_development_dependency "bundler"
+  s.add_development_dependency "scope", "~>0.2.1"
+  s.add_development_dependency "nodule", "~>0.0.25"
+  s.add_development_dependency "mocha"
+  s.add_development_dependency "sqlite3"
+  s.add_development_dependency "rake"
+  s.add_development_dependency "minitap", ">= 0.3.5"
+  s.add_development_dependency "sinatra"
+  s.add_development_dependency "sequel"
+  s.add_development_dependency "mysql"
+  s.add_development_dependency "rack-ooyala-headers"
+  s.add_development_dependency "yajl-ruby"
+  s.add_development_dependency "tapout"
+  s.add_development_dependency "sinatra-synchrony"
+  s.add_development_dependency "thin"
+  s.add_development_dependency "em-net-http"
+end

data/lib/glowworm.rb

@@ -0,0 +1,501 @@
+require "rubygems"
+require "glowworm/version"
+
+require "ecology"
+require "termite"
+require "httparty"
+require "uri"
+
+# TODO(noah): convert log warnings to Hastur calls when they're actually stats.
+# Right now our console output is *way* too chatty, but we don't want to lose
+# this information when syslog picks what to save.
+
+module Glowworm
+  class << self
+    attr_accessor :termite_logger
+    attr_accessor :thread_mutex
+    attr_accessor :cache_mutex
+    attr_accessor :ttl
+    attr_accessor :timeout
+    attr_reader :server
+    attr_reader :threadless
+
+    # Obsoleted on Feb 15, 2012
+    attr_reader :warm_on_boot
+    attr_reader :silent_failure
+
+    Glowworm.thread_mutex = Mutex.new
+    Glowworm.cache_mutex = Mutex.new
+
+    def warm_on_boot=(value)
+      @warm_on_boot = value
+      puts "Glowworm.warm_on_boot is obsolete as of Feb 15, 2012."
+      puts "You can just remove the line that assigns it."
+      Glowworm.termite_logger.warn("Used obsolete GW.warm_on_boot") if Glowworm.termite_logger
+    end
+
+    def silent_failure=(value)
+      @silent_failure = value
+      puts "Glowworm.silent_failure is obsolete as of Feb 15, 2012."
+      puts "You can just remove the line that assigns it."
+      Glowworm.termite_logger.warn("Used obsolete GW.silent_failure") if Glowworm.termite_logger
+    end
+
+    def server=(value)
+      @server = value
+      return if value.nil?
+      unless value[0..4] == "http:"
+        @server = "http://#{value}"
+      end
+    end
+
+    # Get the value of a single feature for a single account
+    # This is equivalent to calling feature_flag with option
+    #   get_value => true
+    #
+    # Options: (see feature_flag)
+    def feature_value(account, feature, options = {})
+      feature_flag(account, feature, options.merge(:get_value => true))
+    end
+
+    # Return the freshness in seconds of this account/feature
+    # combination.  If nil is returned, we don't have actual server
+    # data for this combination.
+    #
+    # Note that just calling freshness won't check for updates.  You
+    # need to call prefetch or feature_flag if you want updates.
+    #
+    def freshness(account, feature, options = {})
+      @last_success
+    end
+
+    # Get whether a single feature is enabled for a single account
+    #
+    # Options:
+    # ttl: how long server data stays fresh before re-fetching
+    # timeout: how long to wait for a response before returning stale data
+    # get_value: true to return actual value rather than true/false
+    #            defaults to false
+    def feature_flag(account, feature, options = {})
+      Ecology.read # Ecology must be initialized by now
+
+      if account.nil? || !account.respond_to?(:to_s)
+        raise "You must provide a string-castable account, not #{account.inspect}!"
+      end
+      if feature.nil? || !feature.respond_to?(:to_s)
+        raise "You must provide a string-castable feature, not #{feature.inspect}!"
+      end
+
+      account = account.to_s
+      feature = feature.to_s
+
+      status = fetch_from_server(account, feature, options)
+      if status.nil?
+        status = 504
+        Glowworm.termite_logger.debug "Server didn't respond within #{options[:timeout] || Glowworm.timeout} seconds"
+      else
+        Glowworm.termite_logger.debug "Received status #{status}"
+      end
+
+      local_account_set_cache = local_feature_cache = local_override_cache = local_fully_active_cache = nil
+
+      Glowworm.cache_mutex.synchronize do
+        # Store references to caches locally within mutex, then use these
+        local_account_set_cache = @account_set_cache
+        local_feature_cache = @feature_cache
+        local_override_cache = @override_cache
+        local_fully_active_cache = @fully_active_cache
+      end
+
+      unless local_override_cache && local_account_set_cache && local_feature_cache
+        Glowworm.termite_logger.debug "Caches aren't instantiated at all..."
+        return options[:default] if options.has_key?(:default)
+        Glowworm.termite_logger.error("You don't seem to have ever received any data from #{Glowworm.server}.  " +
+          "Is Glowworm misconfigured?")
+        return false
+      end
+
+      get_value = options[:get_value] || false
+
+      if local_override_cache[feature] && local_override_cache[feature].has_key?(account)
+        return get_value ? local_override_cache[feature][account][:value] :
+          local_override_cache[feature][account][:flag]
+      end
+
+      account_set = local_account_set_cache[account]
+      if account_set && local_feature_cache[feature]
+        value = nil
+        account_set.sort.each do |single_set|
+          if get_value
+            value ||= local_feature_cache[feature][single_set]
+          else
+            value ||= local_feature_cache[feature].has_key?(single_set)
+          end
+        end
+      else
+        # Server has nothing for this account and feature.
+        Glowworm.termite_logger.debug "Can't find #{feature} for #{account}"
+      end
+
+      return value unless value.nil?
+
+      # At this point, there is no information about this feature for this specific account.
+      # We fall back to default first, and then true or false depending on if the feature is fully active.
+      if local_fully_active_cache && local_fully_active_cache[feature]
+        # Currently, fully_active isn't available for get_value
+        # features.
+        fully_active = local_fully_active_cache[feature]
+      end
+
+      message = "This feature has never been cached for the given account! "
+      unless (local_feature_cache || {}).has_key?(feature)
+        message += "Feature '#{feature[0...1_000]}' seems not to exist.  "
+      end
+      unless (local_account_set_cache || {}).has_key?(account)
+        message += "Account '#{account[0...1_000]}' seems not to exist.  "
+      end
+      (account_set || []).each do |single_set|
+        if local_feature_cache[feature].nil? || !local_feature_cache[feature].has_key?(single_set)
+          unless single_set.nil?
+            message += "There is no data for account set " + single_set.to_s[0...1_000] + " for this feature."
+          end
+        end
+      end
+
+      Glowworm.termite_logger.warn message
+
+      return options[:default] if options.has_key?(:default)
+
+      unless fully_active.nil?
+        # Currently, fully_active isn't available for get_value
+        # features.
+        Glowworm.termite_logger.info("This feature is fully active. Returning true.")
+        fully_active
+      else
+        Glowworm.termite_logger.warn("No default, feature or account never cached: #{message}")
+        false
+      end
+    end
+
+    # Prefetch feature data from the server if stale.
+    #
+    #   account - an account or :all
+    #   feature - a feature or :all
+    #
+    # Options:
+    #   ttl: how long server data stays fresh
+    #   timeout: how long to wait on the server response
+    #   raise: raise an exception if the prefetch fails or times out
+    def prefetch(account, feature, options = {})
+      Ecology.read # Ecology must be initialized by now
+
+      account != nil && account.respond_to?(:to_s) or
+        raise "You must provide a string-castable account, not #{account.inspect}!"
+      feature != nil && feature.respond_to?(:to_s) or
+        raise "You must provide a string-castable feature, not #{feature.inspect}!"
+
+      account = account.to_s
+      feature = feature.to_s
+
+      # By default a prefetch shouldn't wait at all
+      options[:timeout] ||= 0.0
+
+      Glowworm.termite_logger.warn "Glowworm prefetch: '#{account}', '#{feature}'"
+      fetch_from_server(account, feature, options)
+    end
+
+    def reset
+      @account_set_cache = nil
+      @feature_cache = nil
+      @override_cache = nil
+      @fully_active_cache = nil
+      @last_md5 = nil
+      @last_fetch_time = nil
+      @last_success = nil
+      @background_thread = nil
+      @last_modified = nil
+      @threadless = nil
+      @em = nil
+
+      Glowworm.ttl = nil
+      Glowworm.timeout = nil
+      Glowworm.server = nil
+      Glowworm.termite_logger = nil
+    end
+
+    def all_cached_accounts
+      # This accessor already duplicates, so we
+      # can just return it.
+      @account_set_cache ? @account_set_cache.keys : []
+    end
+
+    def all_cached_features
+      # This accessor already duplicates, so we
+      # can just return it.
+      @feature_cache ? @feature_cache.keys : []
+    end
+
+    def account_set_cache
+      # Duplicate the cache so that the caller can't
+      # change cache entries - that would be bad.
+      @account_set_cache ? @account_set_cache.dup : nil
+    end
+
+    def feature_cache
+      # Duplicate the cache so that the caller can't
+      # change cache entries - that would be bad.
+      @feature_cache ? @feature_cache.dup : nil
+    end
+
+    def override_cache
+      # Duplicate the cache so that the caller can't
+      # change cache entries - that would be bad.
+      @override_cache ? @override_cache.dup : nil
+    end
+
+    def fully_active_cache
+      # Duplicate the cache so that the caller can't
+      # change cache entries - that would be bad.
+      @fully_active_cache ? @fully_active_cache.dup : nil
+    end
+
+    def last_md5
+      # Duplicate the md5 so that the caller can't
+      # change it - that would be bad.
+      @last_md5 ? @last_md5.dup : nil
+    end
+
+    # Sychronously update cache.  Raises an error if the update fails.
+    # This will wrap in a EM synchrony block if eventmachine is being used, or update regularly otherwise.
+    def update_cache_in_foreground
+      if @em
+        begin
+          EM.synchrony do
+            _update_cache_in_foreground_
+          end
+        rescue NameError
+          _update_cache_in_foreground_
+        rescue NoMethodError
+          raise 'Eventmachine is being used without requiring "glowworm/em".  Please use this require.'
+        end
+      else
+         _update_cache_in_foreground_
+      end
+    end
+
+    def require_em
+      begin
+        require "em-synchrony"
+        require "em-net-http"
+      rescue LoadError
+        raise "You must have em/sinatra-synchrony AND em-net-http installed for the Glowworm.em to work"
+      end
+    end
+
+    # If you are using EM, but can't call "glowworm/em" at require time, then set Glowworm.em
+    # from within your fiber.
+    def em
+      Ecology.read
+      Glowworm.timeout = 0
+      @em = true
+      @threadless = true
+      require_em
+      update_cache_in_foreground
+    end
+
+    def no_bg
+      Ecology.read
+      Glowworm.timeout = 0
+      @threadless = true
+      # This first call should be the EM-free version, as at require time EM.synchrony causes a hang.
+      _update_cache_in_foreground_
+    end
+
+    private
+
+    # This functions is marked private, as it should only be called at require time.
+    # This is because the internal _update_cache_in_foreground_ is called, which means that we expect
+    # to be in a require-time architecture, and not yet with EM loaded.
+    def _em_
+      no_bg
+      @em = true
+    end
+
+    private
+
+    def _update_cache_in_foreground_
+      account = feature = :all
+      url = url_for_account_and_feature(account, feature)
+      update_cache(account, feature, url, :foreground => true)
+    end
+
+    def url_for_account_and_feature(account, feature)
+      url = Glowworm.server + "/all_if_modified"
+      url += "?md5=#{@last_md5}" if @last_md5
+      url
+    end
+
+    def fetch_from_server(account, feature, options = {})
+      ttl = options[:ttl] || Glowworm.ttl
+      timeout = options[:timeout] || Glowworm.timeout
+
+      # If no background thread, rely on prefetched data
+      return 204 if @threadless
+
+      # Return cached value if it's within TTL
+      time_now = Time.now  # Call Time.now once for consistency
+
+      if @last_fetch_time
+        freshness = time_now - @last_fetch_time
+
+        Glowworm.termite_logger.debug "Freshness: #{freshness} seconds"
+        if freshness < ttl
+          Glowworm.termite_logger.warn "Already fresh: account #{account}, feature: #{feature}," +
+                                         "TTL is #{ttl}"
+          return 200
+        end
+      end
+
+      Glowworm.cache_mutex.synchronize do
+        @account_set_cache ||= {}
+        @feature_cache ||= {}
+        @override_cache ||= {}
+        @fully_active_cache ||= {}
+      end
+      duration = Time.now - time_now
+
+      bg_thread = nil
+
+      Glowworm.thread_mutex.synchronize do
+        bg_thread = @background_thread
+
+        if bg_thread
+          Glowworm.termite_logger.debug "Using existing background thread"
+        else
+          Glowworm.termite_logger.debug "Creating new background thread"
+          bg_thread = @background_thread = Thread.new do
+            situation = "querying server #{Glowworm.server.inspect}, account #{account}, feature #{feature}"
+
+            begin
+              url = url_for_account_and_feature(account, feature)
+              update_cache(account, feature, url)
+              Glowworm.termite_logger.warn "Succeeded in getting updated Glowworm feature: '#{account}', '#{feature}'"
+            rescue Exception # catch everything
+              Glowworm.termite_logger.warn "Exception #{situation}, message '#{$!.message}'"
+              Glowworm.termite_logger.info "Exception #{situation}, message '#{$!.message}'," +
+                "backtrace '#{$!.backtrace.inspect}'"
+
+              raise if options[:raise]
+            end
+          end
+        end
+      end
+
+      if bg_thread.join(timeout)
+        Glowworm.termite_logger.debug "Successfully joined background thread"
+
+        status = bg_thread[:last_status]
+
+        Glowworm.thread_mutex.synchronize { @background_thread = nil if @background_thread == bg_thread }
+
+        status
+      else
+        Glowworm.termite_logger.debug "Join returned false after #{timeout} seconds, returning 504"
+        504
+      end
+    end
+
+    def update_cache(account, feature, url, options = {})
+      situation = "querying server #{url}, all info, checksum #{@last_md5.inspect}"
+
+      start_time = Time.now
+      response = HTTParty.get(url, :headers => { "if-modified-since" => @last_modified || "" })
+
+      latency = Time.now - start_time
+      Glowworm.termite_logger.debug "Request latency: #{latency} seconds"
+
+      @last_modified = response.headers["last-modified"]
+
+      if options[:foreground]
+        raise "Cache update failed with response code #{response.code}!" unless [200, 204, 304].include? response.code
+      else
+        Thread.current[:last_status] = response.code
+      end
+      @last_fetch_time = Time.now
+      if response.code >= 200 && response.code < 300
+        Glowworm.termite_logger.info "Success #{situation}, response code #{response.code}, response size #{response.body.size}",
+                                     {}, :component => "glowworm"
+        refresh_cache_from_response(response.body)
+        @last_success = @last_fetch_time
+      elsif response.code >= 300 && response.code < 400
+        # Normally this is a 304 unmodified - no update
+        @last_success = @last_fetch_time
+        Glowworm.termite_logger.info "Up to date when #{situation}, response code #{response.code}"
+      elsif response.code >= 400 && response.code < 500
+        # This is a 404 or similar - not found.
+        Glowworm.termite_logger.warn("Failure #{situation}, response code #{response.code}, not found",
+                                     {}, :component => "glowworm")
+      else
+        Glowworm.termite_logger.warn("Failure #{situation}, response code #{response.code}",
+                                     {}, :component => "glowworm")
+      end
+    end
+
+    def refresh_cache_from_response(response)
+      # In order to keep the amount of mutex-locked code to a minimum, we will pre-construct the hashes locally,
+      # and then populate the caches with these local variables within a mutex
+      data = MultiJson.load(response)
+
+      raise "Wrong Glowworm protocol version!" unless data["version"] == 1
+
+      local_account_set_cache = {}
+      (data["account_sets"] || {}).each do |account, account_set|
+        local_account_set_cache[account] = account_set.map(&:to_i)
+      end
+
+      local_feature_cache = {}
+      (data["features"] || {}).each do |feature, account_sets|
+        local_feature_cache[feature] = {}
+        account_sets.each do |account_set, value|
+          # Store the value of this account_set for this feature
+          local_feature_cache[feature][account_set.to_i] = value
+        end
+      end
+
+      local_override_cache = {}
+      (data["overrides"] || {}).each do |feature, overrides|
+        local_override_cache[feature] = {}
+        overrides.each do |account, value|
+          local_override_cache[feature][account] = { :flag => value != 0, :value => value }
+        end
+      end
+
+      local_fully_active_cache = {}
+      (data["fully_active"] || {}).each do |feature, fully_active|
+        local_fully_active_cache[feature] = fully_active
+      end
+
+      local_last_md5 = data["md5"]
+      Glowworm.cache_mutex.synchronize do
+        @account_set_cache = local_account_set_cache
+        @feature_cache = local_feature_cache
+        @override_cache = local_override_cache
+        @fully_active_cache = local_fully_active_cache
+        @last_md5 = local_last_md5
+      end
+    end
+
+    Ecology.on_initialize do
+      Glowworm.termite_logger ||= Termite::Logger.new(:component => "glowworm", :use_logger_prefix => true)
+      Glowworm.ttl ||= Ecology.property("features:ttl") || Ecology.property("features:refresh") || 300
+      Glowworm.timeout ||= Ecology.property("features:timeout") || 0.1
+      Glowworm.server ||= ENV["GW_URI"] || (Ecology.property("features:server") || "http://localhost:4999")
+    end
+
+    # When Ecology.reset is called, reset Glowworm too.
+    # Usually this is for testing.
+    Ecology.on_reset do
+      Glowworm.reset
+    end
+  end
+end