slang 0.34.0

data/lib/slang/updater/http_helpers.rb

@@ -0,0 +1,49 @@
+module Slang
+  module Updater
+
+    module HTTPHelpers
+      CHILLAX_RETRY    = 1
+      GONE_RETRY       = 7200
+      TEMPORARY_RETRY  = 60
+      UNEXPECTED_RETRY = 600
+      UNKNOWN_RETRY    = 120
+
+      def get(uri, add_headers={})
+        request = Net::HTTP::Get.new(uri)
+        perform_request(request, add_headers)
+      end
+
+      def post(uri, body, add_headers={})
+        request = Net::HTTP::Post.new(uri)
+        request.body = body
+        perform_request(request, add_headers)
+      end
+
+      def perform_request(request, add_headers)
+        add_headers["User-Agent"] = Slang::USER_AGENT
+        add_headers["Accept-Encoding"] = "gzip" # Akamai doesn't handle Ruby's default Accept-Encoding with q-values
+        add_headers.each { |name, value| request[name] = value }
+        begin
+          uri = request.uri
+          response = Net::HTTP.start(uri.hostname, uri.port, use_ssl: (uri.scheme == "https")) do |http|
+            http.request(request)
+          end
+          response.body = gunzip(response.body) if response["Content-Encoding"] == "gzip" # must uncompress ourselves
+          response
+        rescue SystemCallError => e
+          raise NetworkError.new(TEMPORARY_RETRY), e.message
+        end
+      end
+
+      def gunzip(data)
+        io = StringIO.new(data)
+        gz = Zlib::GzipReader.new(io)
+        uncompressed = gz.read
+        gz.close
+        uncompressed
+      end
+
+    end # module
+
+  end
+end

data/lib/slang/updater/key_reporter.rb

@@ -0,0 +1,92 @@
+require "slang/updater/http_helpers"
+require "slang/updater/squelchable"
+module Slang
+  module Updater
+
+    class KeyReporter
+      include HTTPHelpers
+      include Squelchable
+
+      attr_reader :uri
+      attr_reader :dev_key
+      attr_reader :report_interval
+
+      # Create a key reporter.
+      #
+      # @param [String]  key reporter endpoint
+      # @param [String]  developer key
+      # @param [Numeric] report interval
+      #
+      def initialize(url, dev_key, report_interval=10)
+        @uri = URI(url)
+        @dev_key = dev_key
+        @report_interval = report_interval
+        @pending = {}
+        @sent = Set.new
+        @lock = Mutex.new
+      end
+
+      # Report unknown key.
+      #
+      # @param [String] the unknown key
+      # @param [Hash]   variable map passed (may be empty)
+      #
+      def unknown_key(key, variable_map)
+        var_names = variable_map.keys.sort!
+        key_info = var_names.unshift(key)
+        key_id = key_info.join("|").to_sym
+
+        pid = Process.pid
+        @lock.synchronize do
+          unless @pid == pid
+            @pid = pid
+            @thread = Thread.new { loop { run_loop } }
+            Slang.log_info("Started background key reporter thread (pid=#{pid}).")
+          end
+          unless @sent.include?(key_id)
+            @pending[key_id] = key_info
+          end
+        end
+      end
+
+    private
+
+      def run_loop
+        squelch_thread_activity
+        begin
+          check_for_pending_keys
+          squelch!(report_interval)
+        rescue NetworkError => e
+          squelch!(e.retry_in, "network failure - #{e.message}")
+        rescue => e
+          squelch!(nil, "unexpected error - #{e.message}")
+          Slang.log_error(e.backtrace.join("\n")) unless SlangError === e
+        end
+      end
+
+      def check_for_pending_keys
+        report_pending = @lock.synchronize { @pending.dup }
+        return if report_pending.empty?
+
+        values = report_pending.values # key, [var1], ..., [varN]
+        Slang.log_info("Reporting new key(s): #{values.map { |k| k.first }.join(", ")}")
+        json = Oj.dump(values, mode: :compat)
+        response = post(uri, json, "Content-Type" => "application/json", "X-Slang-Dev-Key" => dev_key)
+        case response
+        when Net::HTTPSuccess #2xx
+          @lock.synchronize do
+            report_pending.each_key do |key_id|
+              @sent << key_id
+              @pending.delete(key_id)
+            end
+          end
+        when Net::HTTPServerError # 5xx
+          raise NetworkError.new(TEMPORARY_ERROR), "HTTP #{response.code}"
+        else
+          raise NetworkError.new(nil), "HTTP #{response.code}"
+        end
+      end
+
+    end # class
+  end
+end

data/lib/slang/updater/production.rb

@@ -0,0 +1,218 @@
+require "slang/updater/abstract"
+module Slang
+  module Updater
+
+    # The production updater is responsible for periodically checking for new snapshots and activating them. State and
+    # snapshots are persisted to permanent storage in the Slang data directory. Multiple Ruby processes in the same
+    # Slang project can (and should) share the same Slang data directory to avoid redundant update checks.
+    #
+    class Production < Abstract
+
+      # Return the latest snapshot. Thread-safe.
+      #
+      def snapshot
+        pid = Process.pid
+        lock.synchronize do
+          unless @pid == pid
+            @pid = pid
+            @thread = Thread.new { loop { run_loop } }
+            Slang.log_info("Started background update thread (pid=#{pid}).")
+          end
+          @snapshot
+        end
+      end
+
+    private
+
+      def run_loop
+        begin
+          sleep_for = shared_state.exclusive_lock do |state|
+            (state[:snapshot_id] == @snapshot.id) ? (state[:check_at].to_f - Time.now.to_f) : 0
+          end
+          if sleep_for > 0
+            sleep_for += rand
+            Slang.log_debug("Resting for a bit ~#{sleep_for.round(2)}s.")
+            sleep(sleep_for)
+            return
+          end
+          Slang.log_debug("Huh? I'm awake!")
+          check_for_new_snapshot
+        rescue => e
+          Slang.log_warn("Snapshot update failed: #{e.message}")
+          Slang.log_warn(e.backtrace.join("\n")) unless SlangError === e
+          shared_state.exclusive_lock do |state|
+            state.delete(:checking_pid) if state[:checking_pid] == @pid
+            state[:check_at] = Time.now.to_f + UNEXPECTED_RETRY
+          end
+          sleep(UNEXPECTED_RETRY)
+        end
+      end
+
+      def check_for_new_snapshot
+        action, arg = shared_state.exclusive_lock do |state|
+          if state[:snapshot_id] != @snapshot.id
+            [ :activate_snapshot, state[:snapshot_id] ]
+          elsif state[:checking_pid]
+            alive = Process.getpgid(state[:checking_pid]) rescue nil
+            if alive
+              [ :checker_alive, state[:checking_pid] ]
+            else
+              state[:checking_pid] = @pid
+              [ :checker_dead, state[:modified_at] ]
+            end
+          else
+            state[:checking_pid] = @pid
+            [ :check, state[:modified_at] ]
+          end
+        end
+
+        case action
+        when :activate_snapshot
+          Slang.log_debug("Sweet. Found new snapshot.")
+          snapshot_path = data_file_path(arg)
+          new_snapshot = Snapshot.from_json(File.read(snapshot_path))
+          activate(new_snapshot)
+          return
+        when :checker_alive
+          Slang.log_debug("Chillin' like a villian (waiting on PID #{arg}).")
+          sleep(CHILLAX_RETRY)
+          return
+        when :checker_dead
+          Slang.log_debug("Checking worker is dead. RIP. My turn.")
+        else # :check
+          Slang.log_debug("Looking for a new snapshot.")
+        end
+
+        begin
+          modified_at, check_in = fetch_meta(arg)
+          shared_state.exclusive_lock do |state|
+            state.delete(:checking_pid)
+            state[:check_at] = Time.now.to_f + check_in
+            state[:modified_at] = modified_at
+          end
+          sleep(check_in)
+        rescue NetworkError => e
+          shared_state.exclusive_lock do |state|
+            state.delete(:checking_pid)
+            state[:check_at] = Time.now.to_f + e.retry_in
+          end
+          sleep(e.retry_in)
+        end
+      end
+
+      def fetch_meta(modified_at)
+        Slang.log_debug("Fetching meta.")
+        response = get(uri, "If-Modified-Since" => Time.at(modified_at).httpdate)
+        case response
+        when Net::HTTPNotModified # 304
+          Slang.log_info("Snapshot up-to-date.")
+        when Net::HTTPOK # 200
+          Slang.log_debug("New snapshot? YES!")
+          parse_meta(response.body.force_encoding(Encoding::UTF_8))
+        when Net::HTTPNotFound # 404
+          raise NetworkError.new(UNKNOWN_RETRY), "Unknown project (404)."
+        when Net::HTTPGone # 410
+          raise NetworkError.new(GONE_RETRY), "Unknown project (410)."
+        when Net::HTTPServerError # 5xx
+          raise NetworkError.new(TEMPORARY_RETRY), "HTTP #{response.code}"
+        else
+          raise NetworkError.new(UNEXPECTED_RETRY), "Unexpected response (HTTP #{response.code})"
+        end
+        last_modified = Time.httpdate(response["Last-Modified"]) rescue Time.now.utc
+        max_age = response["Cache-Control"] =~ /max-age=(\d+)/ ? [$1.to_i, 10].max : UNEXPECTED_RETRY
+        [ last_modified.to_i, max_age ]
+      end
+
+      def parse_meta(json)
+        meta = Oj.strict_load(json, symbol_keys: true)
+
+        if meta[:id] == @snapshot.id
+          Slang.log_debug("Err.. Nope. Already at this snapshot - #{@snapshot.id}")
+          return
+        end
+
+        delta_url = meta[:deltas].each_slice(2) do |snapshot_id, url|
+          break(url) if snapshot_id == @snapshot.id
+        end
+        if delta_url
+          Slang.log_debug("Delta-delta-delta. How can I help-ya? #{delta_url}")
+          begin
+            snapshot_json = patch_snapshot(get_asset(delta_url))
+          rescue => e
+            Slang.log_warn("Unexpected error fetching/applying delta, trying snapshot - #{e.message}")
+          end
+        end
+        snapshot_json ||= get_asset(meta[:url])
+        persist_new_snapshot(snapshot_json)
+      end
+
+      def get_asset(url)
+        response = get(URI(url))
+        case response
+        when Net::HTTPOK # 200
+          Slang.log_debug("Asset retrieved.")
+          response.body.force_encoding(Encoding::UTF_8)
+        when Net::HTTPNotFound # 404
+          raise NetworkError.new(UNKNOWN_RETRY), "Unknown asset (404)."
+        when Net::HTTPGone # 410
+          raise NetworkError.new(GONE_RETRY), "Unknown asset (410)."
+        when Net::HTTPServerError # 5xx
+          raise NetworkError.new(TEMPORARY_RETRY), "HTTP #{response.code}"
+        else
+          raise NetworkError.new(UNEXPECTED_RETRY), "Unexpected response (HTTP #{response.code})"
+        end
+      end
+
+      def persist_new_snapshot(json)
+        new_snapshot = Snapshot.from_json(json)
+
+        File.write(data_file_path(new_snapshot.id), json)
+        shared_state.exclusive_lock { |state| state[:snapshot_id] = new_snapshot.id }
+        FileUtils.rm(data_file_path(@snapshot.id), force: true)
+
+        activate(new_snapshot) # last
+      end
+
+      def patch_snapshot(delta_json)
+        snapshot_json = File.read(data_file_path(@snapshot.id))
+        snapshot_array = Snapshot.array_from_json(snapshot_json)
+        delta_array = Oj.strict_load(delta_json)
+
+        delta_format, snapshot_id, timestamp, default_locale_code, locales_diff, drop, add, change = *delta_array
+        raise UpdaterError, "unknown snapshot delta format - #{delta_format}" unless delta_format == 6
+        locales = patch(snapshot_array[4], locales_diff)
+        translations = snapshot_array[5]
+        drop.each { |i| translations.delete_at(i) }
+        add.each do |array|
+          i = array.shift
+          translations.insert(i, array)
+        end
+        change.each do |array|
+          i = array.shift
+          translations[i] = patch(translations[i], array)
+        end
+        new_snapshot_array = [ delta_format, snapshot_id, timestamp, default_locale_code, locales, translations ]
+        Oj.dump(new_snapshot_array, mode: :strict)
+      end
+
+      def patch(object, transforms)
+        return object if transforms.empty?
+        data = Oj.dump(object, mode: :strict).force_encoding(Encoding::BINARY)
+        i = 0
+        transforms.each do |x|
+          if String === x
+            x.force_encoding(Encoding::BINARY)
+            data.insert(i, x)
+            i += x.length
+          elsif x < 0
+            data.slice!(i, -x)
+          else
+            i += x
+          end
+        end
+        Oj.strict_load(data.force_encoding(Encoding::UTF_8))
+      end
+
+    end
+  end
+end

data/lib/slang/updater/shared_state.rb

@@ -0,0 +1,59 @@
+module Slang
+  module Updater
+
+    # A multi-process-safe shared state file. Uses JSON for serialization, and exclusive file locking.
+    #
+    class SharedState
+
+      attr_reader :path
+      attr_reader :permissions
+
+      # Create a SharedState instance. State file will be created as necessary.
+      #
+      # @param [String] pathname of stale file.
+      # @param [Fixnum] permissions, defaults to 0644.
+      #
+      def initialize(path, permissions = 0644)
+        @path = File.expand_path(path).freeze
+        @permissions = permissions
+      end
+
+      # This method opens the shared state file, obtains an exclusive lock, and reads/deserializes its content. A
+      # state hash (possibly empty) is yielded to the block, *while* the exclusive lock is held. The block may mutate
+      # the state hash as necessary. When the yield returns, any changes made to the state are serialized and written
+      # out the the file. Finally, the file is closed and lock released.
+      #
+      # @yield [Hash] the current state (keys are symbols).
+      # @return result of yielded block
+      #
+      def exclusive_lock
+        File.open(path, File::RDWR | File::CREAT, permissions) do |f|
+          f.flock(File::LOCK_EX)
+          state = deserialize(f.read)
+          state.each_value { |v| v.freeze }
+          original_state = state.dup
+          ret = yield(state)
+          unless original_state == state
+            f.rewind
+            f.write(serialize(state))
+            f.flush
+            f.truncate(f.pos)
+          end
+          ret
+        end
+      end
+
+    private
+
+      def deserialize(json)
+        return {} if json.empty?
+        Oj.strict_load(json.force_encoding(Encoding::UTF_8), symbol_keys: true)
+      end
+
+      def serialize(obj)
+        Oj.dump(obj, mode: :compat)
+      end
+
+    end # class
+  end
+end

data/lib/slang/updater/squelchable.rb

@@ -0,0 +1,45 @@
+module Slang
+  module Updater
+
+    # This module adds the ability to squelch activity for a temporary amount of time (or permanently).
+    #
+    module Squelchable
+
+      def squelch!(interval, reason=nil)
+        if interval
+          @squelch_until = Time.now + interval
+          Slang.log_warn("Slang activity temporarily disabled for ~#{interval}s. #{reason}") if reason
+        else
+          @squelch_until = :forever
+          Slang.log_error("Slang activity permanently disabled - #{reason}")
+        end
+      end
+
+      def squelched?
+        if @squelch_until
+          if @squelch_until == :forever
+            :forever
+          else
+            wait = @squelch_until - Time.now
+            if wait > 0
+              wait
+            else
+              @squelch_until = nil
+            end
+          end
+        end
+      end
+
+      def squelch_thread_activity
+        wait = squelched?
+        case wait
+        when nil;      # continue
+        when :forever; loop { Thread.stop }
+        else           sleep(wait)
+        end
+      end
+
+    end # module
+
+  end
+end