prefab-cloud-ruby 0.0.22 → 0.0.23

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 46899736646bad8d02c9ef4eca66a38fb58a10b3
-  data.tar.gz: faa44f377c1299e333bc1860b65897fd09c9288b
+  metadata.gz: 1174d61400436bdc42b40cf7141729065ffb7c35
+  data.tar.gz: f275ed0da1795f276f950bc956a105d28ed27886
 SHA512:
-  metadata.gz: 2b58f9f0ecfd29dd5a1531058a0dc129d7846585e7b95b36bb70c96b9bcbb57f4489432347c725527b2992318857aef7a6a91a6676d6efeb2a8610dea82d4025
-  data.tar.gz: cc30b8afe888be463eed8035b75b6d737b9071f761380123471ca096b7ee26344d96a850c8db3117073baf770b176e15a117579e75e7cd670afdd5c054fd512d
+  metadata.gz: 68b0c8ee0dfac57878dd13ab41bb31d653b131a2923821f39e1165036b89c31101863aaa872f34443f502d021b69fb54575e6369cb3fde83fc362bfee8dd8b45
+  data.tar.gz: 92eabb20b31054ed1bce771631d893e79f86e1493aaf3375e0a1426aa64449b729a1b84daf1b7ac982a7bc0d545ff8fc98d81ae17e8638f5978ef13f2d750598

data/Gemfile

@@ -2,6 +2,7 @@ source "https://rubygems.org"
 
 gem 'grpc', '~> 1.11.0'
 gem 'concurrent-ruby', '~> 1.0', '>= 1.0.5'
+gem 'aws-sdk-s3', '~> 1'
 
 group :development do
   gem 'grpc-tools', '~> 1.11.0'

data/Gemfile.lock

@@ -8,6 +8,19 @@ GEM
       tzinfo (~> 1.1)
     addressable (2.5.2)
       public_suffix (>= 2.0.2, < 4.0)
+    aws-partitions (1.86.0)
+    aws-sdk-core (3.20.2)
+      aws-partitions (~> 1.0)
+      aws-sigv4 (~> 1.0)
+      jmespath (~> 1.0)
+    aws-sdk-kms (1.5.0)
+      aws-sdk-core (~> 3)
+      aws-sigv4 (~> 1.0)
+    aws-sdk-s3 (1.10.0)
+      aws-sdk-core (~> 3)
+      aws-sdk-kms (~> 1)
+      aws-sigv4 (~> 1.0)
+    aws-sigv4 (1.0.2)
     builder (3.2.3)
     concurrent-ruby (1.0.5)
     descendants_tracker (0.0.4)
@@ -42,6 +55,7 @@ GEM
     highline (1.7.10)
     i18n (1.0.1)
       concurrent-ruby (~> 1.0)
+    jmespath (1.4.0)
     json (1.8.6)
     juwelier (2.4.9)
       builder
@@ -108,6 +122,7 @@ PLATFORMS
   ruby
 
 DEPENDENCIES
+  aws-sdk-s3 (~> 1)
   bundler (~> 1.0)
   concurrent-ruby (~> 1.0, >= 1.0.5)
   grpc (~> 1.11.0)

data/VERSION

@@ -1 +1 @@
-0.0.22
+0.0.23

data/lib/prefab/client.rb

@@ -4,7 +4,7 @@ module Prefab
     MAX_SLEEP_SEC = 10
     BASE_SLEEP_SEC = 0.5
 
-    attr_reader :account_id, :shared_cache, :stats, :namespace, :creds, :interceptor
+    attr_reader :account_id, :shared_cache, :stats, :namespace, :creds, :interceptor, :api_key
 
     def initialize(api_key: ENV['PREFAB_API_KEY'],
                    logdev: nil,
@@ -20,6 +20,7 @@ module Prefab
       @local = local
       @stats = (stats || NoopStats.new)
       @shared_cache = (shared_cache || NoopCache.new)
+      @api_key = api_key
       @account_id = api_key.split("|")[0].to_i
       @namespace = namespace
 

data/lib/prefab/config_client.rb

@@ -1,26 +1,24 @@
 module Prefab
   class ConfigClient
     RECONNECT_WAIT = 5
-    SUSPENDERS_RESET_LOCK_SEC = 20
-    DEFAULT_CHECKPOINT_FREQ_SEC = 10
-    DEFAULT_MAX_SUSPENDERS_AGE_SEC = 60
+    DEFAULT_CHECKPOINT_FREQ_SEC = 60
 
     def initialize(base_client, timeout)
       @base_client = base_client
       @timeout = timeout
       @initialization_lock = Concurrent::ReadWriteLock.new
 
-      @suspenders_max_age_secs = (ENV["PREFAB_SUSPENDERS_MAX_AGE_SEC"] || DEFAULT_MAX_SUSPENDERS_AGE_SEC)
       @checkpoint_freq_secs = (ENV["PREFAB_DEFAULT_CHECKPOINT_FREQ_SEC"] || DEFAULT_CHECKPOINT_FREQ_SEC)
 
       @config_loader = Prefab::ConfigLoader.new(@base_client)
       @config_resolver = Prefab::ConfigResolver.new(@base_client, @config_loader)
 
       @initialization_lock.acquire_write_lock
+      @s3 = Aws::S3::Resource.new(region: 'us-east-1')
 
-      load_or_save_checkpoint
-      ensure_api_connection_started
-      start_checkpointing_thread if has_real_cache?
+      load_checkpoint
+      start_api_connection_thread(@config_loader.highwater_mark)
+      start_checkpointing_thread
     end
 
     def get(prop)
@@ -69,10 +67,9 @@ module Prefab
     # Bootstrap out of the cache
     # returns the high-watermark of what was in the cache
     def load_checkpoint
-      checkpoint = @base_client.shared_cache.read(checkpoint_cache_key)
-
-      if checkpoint
-        deltas = Prefab::ConfigDeltas.decode(checkpoint)
+      obj = @s3.bucket('prefab-cloud-checkpoints-prod').object(@base_client.api_key.gsub("|", "/"))
+      obj.get do |f|
+        deltas = Prefab::ConfigDeltas.decode(f)
         deltas.deltas.each do |delta|
           @config_loader.set(delta)
         end
@@ -80,33 +77,18 @@ module Prefab
         @base_client.stats.increment("prefab.config.checkpoint.load")
         @config_resolver.update
         finish_init!
-      else
-        @base_client.log_internal Logger::INFO, "No checkpoint"
       end
-    end
 
-    # Save off the config to a local cache as a backup
-    #
-    def save_checkpoint
-      begin
-        deltas = @config_resolver.export_api_deltas
-        @base_client.log_internal Logger::DEBUG, "Save Checkpoint #{@config_loader.highwater_mark} Thread #{Thread.current.object_id}"
-        @base_client.shared_cache.write(checkpoint_cache_key, Prefab::ConfigDeltas.encode(deltas))
-        @base_client.shared_cache.write(checkpoint_highwater_cache_key, @config_loader.highwater_mark)
-        @base_client.stats.increment("prefab.config.checkpoint.save")
-      rescue StandardError => exn
-        @base_client.log_internal Logger::INFO, "Issue Saving Checkpoint #{exn.message}"
-      end
+    rescue Aws::S3::Errors::NoSuchKey
+      @base_client.log_internal Logger::INFO, "No checkpoint"
     end
 
-    # A thread that saves current state to the cache, "checkpointing" it
+    # A thread that checks for a checkpoint
     def start_checkpointing_thread
       Thread.new do
         loop do
           begin
-            load_or_save_checkpoint
-
-            suspenders_if_needed
+            load_checkpoint
 
             started_at = Time.now
             delta = @checkpoint_freq_secs - (Time.now - started_at)
@@ -120,72 +102,6 @@ module Prefab
       end
     end
 
-    # check what our shared highwater mark is
-    # if it is higher than our own highwater mark, we must have missed something, so load the checkpoint
-    # if it is lower than our own highwater mark, save a checkpoint
-    # if everything is up to date, but the shared highwater mark is old, coordinate amongst other processes to have
-    #    one process "double check" by restarting the API thread
-    def load_or_save_checkpoint
-      shared_highwater_mark = get_shared_highwater_mark
-      @base_client.log_internal Logger::DEBUG, "Checkpoint_if_needed highwater apx ahead/behind #{(@config_loader.highwater_mark - shared_highwater_mark) / (1000 * 10000)}"
-
-      if shared_highwater_mark > @config_loader.highwater_mark
-        @base_client.log_internal Logger::DEBUG, "We were behind, loading checkpoint"
-        load_checkpoint
-      elsif shared_highwater_mark < @config_loader.highwater_mark
-        @base_client.log_internal Logger::DEBUG, "Saving off checkpoint"
-        save_checkpoint
-      end
-    end
-
-    def suspenders_if_needed
-      if suspenders_age_is_old?
-        if get_shared_lock?
-          @base_client.log_internal Logger::DEBUG, "Suspenders > PREFAB_SUSPENDERS_MAX_AGE_SEC #{@suspenders_max_age_secs}. We have been chosen to run suspenders"
-          reset_api_connection
-        else
-          @base_client.log_internal Logger::DEBUG, "Suspenders > PREFAB_SUSPENDERS_MAX_AGE_SEC #{@suspenders_max_age_secs}. Other process is running suspenders"
-        end
-      end
-    end
-
-    def suspenders_age_is_old?
-      age = Time.now.to_i - get_checkpoint_suspenders
-      @base_client.log_internal Logger::DEBUG, "checkpoint_suspenders_is_old? apx #{age}" if age > @suspenders_max_age_secs
-      age > @suspenders_max_age_secs
-    end
-
-    def get_shared_highwater_mark
-      (@base_client.shared_cache.read(checkpoint_highwater_cache_key) || 0).to_i
-    end
-
-    def get_checkpoint_suspenders
-      (@base_client.shared_cache.read(suspenders_last_run_cache_key) || 0).to_i
-    end
-
-    def set_suspenders
-      @base_client.shared_cache.write(suspenders_last_run_cache_key, Time.now.to_i)
-    end
-
-    def get_shared_lock?
-      in_progess = @base_client.shared_cache.read(suspenders_reset_in_progress_cache_key)
-      if in_progess.nil?
-        @base_client.shared_cache.write(suspenders_reset_in_progress_cache_key, "true", { expires_in: SUSPENDERS_RESET_LOCK_SEC })
-        true
-      else
-        false
-      end
-    end
-
-    def ensure_api_connection_started
-      reset_api_connection if @api_connection_thread.nil?
-    end
-
-    def reset_api_connection
-      @api_connection_thread&.exit
-      start_api_connection_thread(@config_loader.highwater_mark)
-    end
-
     def finish_init!
       if @initialization_lock.write_locked?
         @initialization_lock.release_write_lock
@@ -200,7 +116,6 @@ module Prefab
                                                     start_at_id: start_at_id)
       @base_client.log_internal Logger::DEBUG, "start api connection thread #{start_at_id}"
       @base_client.stats.increment("prefab.config.api.start")
-      set_suspenders
       @api_connection_thread = Thread.new do
         while true do
           begin
@@ -220,26 +135,6 @@ module Prefab
         end
       end
     end
-
-    def has_real_cache?
-      @base_client.shared_cache.class != NoopCache
-    end
-
-    def checkpoint_cache_key
-      @base_client.cache_key "config:checkpoint"
-    end
-
-    def suspenders_reset_in_progress_cache_key
-      @base_client.cache_key "config:checkpoint:updating"
-    end
-
-    def checkpoint_highwater_cache_key
-      @base_client.cache_key "config:checkpoint:highwater"
-    end
-
-    def suspenders_last_run_cache_key
-      @base_client.cache_key "config:checkpoint:suspenders"
-    end
   end
 end
 

data/lib/prefab-cloud-ruby.rb

@@ -14,4 +14,4 @@ require 'prefab/auth_interceptor'
 require 'prefab/noop_cache'
 require 'prefab/noop_stats'
 require 'prefab/murmer3'
-
+require 'aws-sdk-s3'

data/prefab-cloud-ruby.gemspec

@@ -2,16 +2,16 @@
 # DO NOT EDIT THIS FILE DIRECTLY
 # Instead, edit Juwelier::Tasks in Rakefile, and run 'rake gemspec'
 # -*- encoding: utf-8 -*-
-# stub: prefab-cloud-ruby 0.0.22 ruby lib
+# stub: prefab-cloud-ruby 0.0.23 ruby lib
 
 Gem::Specification.new do |s|
   s.name = "prefab-cloud-ruby".freeze
-  s.version = "0.0.22"
+  s.version = "0.0.23"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0".freeze) if s.respond_to? :required_rubygems_version=
   s.require_paths = ["lib".freeze]
   s.authors = ["Jeff Dwyer".freeze]
-  s.date = "2018-05-03"
+  s.date = "2018-05-15"
   s.description = "RateLimits & Config as a service".freeze
   s.email = "jdwyer@prefab.cloud".freeze
   s.extra_rdoc_files = [
@@ -60,6 +60,7 @@ Gem::Specification.new do |s|
     if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
       s.add_runtime_dependency(%q<grpc>.freeze, ["~> 1.11.0"])
       s.add_runtime_dependency(%q<concurrent-ruby>.freeze, [">= 1.0.5", "~> 1.0"])
+      s.add_runtime_dependency(%q<aws-sdk-s3>.freeze, ["~> 1"])
       s.add_development_dependency(%q<grpc-tools>.freeze, ["~> 1.11.0"])
       s.add_development_dependency(%q<shoulda>.freeze, [">= 0"])
       s.add_development_dependency(%q<rdoc>.freeze, ["~> 3.12"])
@@ -69,6 +70,7 @@ Gem::Specification.new do |s|
     else
       s.add_dependency(%q<grpc>.freeze, ["~> 1.11.0"])
       s.add_dependency(%q<concurrent-ruby>.freeze, [">= 1.0.5", "~> 1.0"])
+      s.add_dependency(%q<aws-sdk-s3>.freeze, ["~> 1"])
       s.add_dependency(%q<grpc-tools>.freeze, ["~> 1.11.0"])
       s.add_dependency(%q<shoulda>.freeze, [">= 0"])
       s.add_dependency(%q<rdoc>.freeze, ["~> 3.12"])
@@ -79,6 +81,7 @@ Gem::Specification.new do |s|
   else
     s.add_dependency(%q<grpc>.freeze, ["~> 1.11.0"])
     s.add_dependency(%q<concurrent-ruby>.freeze, [">= 1.0.5", "~> 1.0"])
+    s.add_dependency(%q<aws-sdk-s3>.freeze, ["~> 1"])
     s.add_dependency(%q<grpc-tools>.freeze, ["~> 1.11.0"])
     s.add_dependency(%q<shoulda>.freeze, [">= 0"])
     s.add_dependency(%q<rdoc>.freeze, ["~> 3.12"])

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: prefab-cloud-ruby
 version: !ruby/object:Gem::Version
-  version: 0.0.22
+  version: 0.0.23
 platform: ruby
 authors:
 - Jeff Dwyer
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-05-03 00:00:00.000000000 Z
+date: 2018-05-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: grpc
@@ -44,6 +44,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.0'
+- !ruby/object:Gem::Dependency
+  name: aws-sdk-s3
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1'
 - !ruby/object:Gem::Dependency
   name: grpc-tools
   requirement: !ruby/object:Gem::Requirement