herdst_worker 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: c5b5decf991f20026eb7fe48327f09a7c385c6fad0fdff66e683a4d4d142c8d8
+  data.tar.gz: f3e8a159f0ceb54681f120a69c688ed95a86e78a1f7c008b960f2bd41c3558eb
+SHA512:
+  metadata.gz: 768569b090a702a755a589d353af9c9f67d478c7a8e81bc7a6a92f04d5b3c826a11e546fa237347ed5e8c83284b630803673b749c830a32a2cfb87ff3e4fe5a3
+  data.tar.gz: 84044ceab2c26c95c5338b16d3ef4c97596781b04a5d4a341550aef02b7cd166d2b4eed1013986868ceaeacfe6066719b01e9c9f3089e856ac01aaee4d35fb08

data/Rakefile

@@ -0,0 +1,7 @@
+ENV["TZ"] = "UTC"
+ENV["ENV"] ||= (ARGV[0] == "test" ? "test" : "development")
+
+require_relative 'lib/herdst_worker'
+
+Rake.add_rakelib 'tasks'
+task default: %w[test]

data/bin/herdst_worker

@@ -0,0 +1,16 @@
+#!/usr/bin/env ruby
+
+args = Hash[ ARGV.flat_map { |s| s.scan(/--?([^=\s]+)(?:=(\S+))?/) } ]
+
+application_file = args["application"] || "application.rb"
+application_path = "#{Dir.pwd}/#{application_file}"
+application_env = args["env"] || "production"
+queue_enabled = args["queue_enabled"] == "false" ? false : true
+
+require_relative "../lib/herdst_worker"
+
+application_instance = HerdstWorker::Application.new(application_env, queue_enabled)
+
+require_relative application_path
+
+application_instance.run

data/lib/herdst_worker.rb

@@ -0,0 +1,8 @@
+# Require Lib files
+require 'active_support/all'
+require 'active_record'
+require 'mysql2'
+require "aws-sdk-core"
+
+# Require application
+require_relative 'herdst_worker/application/facade'

data/lib/herdst_worker/adapters/database.rb

@@ -0,0 +1,31 @@
+module HerdstWorker
+    module Adapters
+        class Database
+            
+            
+            def self.setup(app)
+                db_config = app.config_for(:database)
+                
+                if app.config.is_dev?
+                    ActiveRecord::Base.logger = app.logger.activerecord
+                end
+                
+                ActiveRecord::Base.default_timezone = :utc
+                ActiveRecord::Base.establish_connection(
+                    adapter:    db_config[:adapter],
+                    encoding:   db_config[:encoding],
+                    charset:    db_config[:charset],
+                    collation:  db_config[:collation],
+                    pool:       db_config[:pool],
+                    host:       db_config[:host],
+                    username:   db_config[:username],
+                    password:   db_config[:password],
+                    database:   db_config[:database]
+                )
+                ActiveRecord::Base.connection.enable_query_cache!
+            end
+            
+            
+        end
+    end
+end

data/lib/herdst_worker/adapters/facade.rb

@@ -0,0 +1,20 @@
+require "sentry-raven"
+
+require_relative "database"
+require_relative "sentry"
+
+
+module HerdstWorker
+    module Adapters
+        class Facade
+            
+            
+            def self.bootstrap(app)
+                HerdstWorker::Adapters::Database.setup(app)
+                HerdstWorker::Adapters::Sentry.setup(app)
+            end
+            
+            
+        end
+    end
+end

data/lib/herdst_worker/adapters/sentry.rb

@@ -0,0 +1,26 @@
+require "sentry-raven"
+
+
+module HerdstWorker
+    module Adapters
+        class Sentry
+            
+            
+            def self.setup(app)
+                environment = app.config.metadata[:DEPLOYMENT_ENV]
+                release = app.config.metadata[:RELEASE_VERSION]
+                sentry_key = app.config.metadata[:SENTRY_KEY]
+                
+                Raven.configure do |config|
+                    config.current_environment = environment
+                    config.release = release
+                    config.dsn = sentry_key if sentry_key
+                    config.environments = ["preview", "production"]
+                    config.async = lambda { |event| Thread.new { Raven.send_event(event) } }
+                end
+            end
+            
+            
+        end
+    end
+end

data/lib/herdst_worker/application/facade.rb

@@ -0,0 +1,88 @@
+require_relative '../configuration/facade'
+require_relative '../log/facade'
+require_relative '../autoload/facade'
+require_relative '../signals/facade'
+require_relative '../queue/facade'
+require_relative '../adapters/facade'
+
+
+module HerdstWorker
+    class Application
+        
+        
+        attr_accessor :name
+        attr_accessor :logger, :config, :autoload
+        attr_accessor :poller_enabled, :queues, :poller_url, :queue
+        
+        
+        def initialize(env, poller_enabled)
+            self.poller_enabled = poller_enabled
+            self.queues = ActiveSupport::HashWithIndifferentAccess.new
+            
+            HerdstWorker.set_application(self)
+            
+            self.logger = HerdstWorker::Log::Facade.new(get_logger_level(env))
+            self.config = HerdstWorker::Configuration::Facade.new(env)
+            self.autoload = HerdstWorker::Autoload::Facade.new(self)
+            self.set_inflections
+            
+            HerdstWorker::Adapters::Facade.bootstrap(self)
+        end
+        
+        
+        def configure(&block)
+            yield self
+            
+            self.autoload.reload
+        end
+        
+        
+        def add_queue(name, url, poller = false)
+            self.queues[name] = url
+            self.poller_url = url if poller
+        end
+        
+        
+        def run
+            if self.queue == nil
+                self.logger.info "Starting Application (#{$$})"
+                
+                HerdstWorker::Signals::Facade.listen(self.config.paths.temp)
+                
+                self.queue = HerdstWorker::Queue::Facade.new(self, self.poller_enabled, self.poller_url, 20)
+                self.queue.start
+            end
+        end
+        
+        
+        def config_for(name)
+            self.config.config_for(name)
+        end
+        
+        
+        private            
+            def set_inflections
+                ActiveSupport::Inflector.inflections do |inflect|
+                    inflect.irregular "meta", "meta"
+                end
+            end
+            
+            def get_logger_level(env)
+                return "WARN" if env == "production"
+                return "FATAL" if env == "test"
+                return "DEBUG"
+            end
+            
+            
+    end
+    
+    
+    def self.set_application(application)
+        @@application = application
+    end
+    
+    
+    def self.application
+        @@application
+    end
+end

data/lib/herdst_worker/autoload/facade.rb

@@ -0,0 +1,67 @@
+module HerdstWorker
+    module Autoload
+        class Facade
+            
+            
+            attr_accessor :root_path, :loaded, :files, :paths
+            
+            
+            def initialize(app)
+                self.root_path = app.config.paths.root
+                
+                self.loaded = []
+                self.files = []
+                self.paths = [ "lib" ]
+                
+                self.reload
+            end
+            
+            
+            def <<(path)
+                full_path = self.root_path + "/" + path
+                
+                if File.directory?(full_path)
+                    self.paths << path
+                else
+                    self.files << path
+                end
+            end
+            
+            
+            def reload
+                self.files.each do |file|
+                    self.load_file(self.root_path + "/" + file)
+                end
+                
+                self.paths.each do |folder|
+                    concern_files = Dir.glob("#{root_path}/#{folder}/concerns/*.rb").sort
+                    folder_files = Dir.glob("#{root_path}/#{folder}/*.rb").sort
+                    
+                    concern_sub_files = Dir.glob("#{self.root_path}/#{folder}/**/concerns/*.rb").sort
+                    folder_sub_files = Dir.glob("#{self.root_path}/#{folder}/**/*.rb").sort
+                    
+                    files = (concern_files + folder_files + concern_sub_files + folder_sub_files).uniq
+                    
+                    files.each do |file|
+                        self.load_file(file)
+                    end
+                end
+            end
+
+
+            def has_loaded?(file)
+                self.loaded.include?(file)
+            end
+
+            
+            def load_file(file)
+                unless has_loaded?(file)
+                    load file
+                    self.loaded << file
+                end
+            end
+            
+            
+        end
+    end
+end

data/lib/herdst_worker/configuration/facade.rb

@@ -0,0 +1,51 @@
+require 'yaml'
+require 'erb'
+
+require_relative 'metadata'
+require_relative 'paths'
+
+
+module HerdstWorker
+    module Configuration
+        class Facade
+            
+            
+            attr_accessor :env, :metadata, :paths, :actions
+            
+            
+            def initialize(env)
+                self.env = env.downcase
+                
+                self.paths = Paths.new
+                self.metadata = Metadata.new(env, self)
+                self.actions = self.config_for(:actions)
+            end
+            
+            
+            def config_for(name)
+                file = "#{self.paths.config}/#{name}.yml"
+                file_contents = ERB.new(File.new(file).read)
+                yaml = YAML.load(file_contents.result(binding)).with_indifferent_access
+                
+                yaml.include?(self.env) ? yaml[self.env] : yaml
+            end
+            
+            
+            def is_prod?
+                self.env == "production"
+            end
+            
+            
+            def is_test?
+                self.env == "test"
+            end
+            
+
+            def is_dev?
+                self.env == "development"
+            end
+            
+            
+        end
+    end
+end

data/lib/herdst_worker/configuration/metadata.rb

@@ -0,0 +1,183 @@
+require "aws-sdk-core"
+require "aws-sdk-secretsmanager"
+require "aws-sdk-ecs"
+require "net/http"
+
+
+module HerdstWorker
+    module Configuration
+        class Metadata
+    
+            
+            attr_accessor :config, :aws_credentials, :secrets
+            attr_accessor :config_suffix, :secrets_suffix
+            
+            
+            def initialize(env, config)
+                self.config = config
+                self.config_suffix = "-service-config"
+                self.secrets_suffix = "-service-secrets"
+                
+                self.secrets = {}.with_indifferent_access
+                self.secrets["ENV"] = env
+                
+                begin
+                    self.reload!
+                rescue Exception => ex
+                    raise ex unless self.is_prod?
+                end
+            end
+            
+            
+            def reload!
+                self.get_aws_credentials!
+                self.set_task_info!
+                self.get_config_and_secrets!
+                
+                self
+            end
+            
+            
+            def get_secrets
+                self.secrets
+            end
+            
+            
+            def [](key)
+                self.secrets[key.to_s.upcase]
+            end
+            
+            
+            def []=(key, value)
+                self.secrets[key.to_s.upcase] = value
+            end
+            
+            
+            def get_aws_credentials
+                self.get_aws_credentials! rescue nil
+            end
+            
+            
+            def get_aws_credentials!
+                credentials = self.is_prod? ? 
+                     make_request!(get_credentials_uri) :
+                     self.config.config_for(:aws)
+                
+                self.aws_credentials = Aws::Credentials.new(
+                    credentials["AccessKeyId"], 
+                    credentials["SecretAccessKey"], 
+                    credentials["Token"]
+                )
+                
+                Aws.config.update(
+                    region: self.get_aws_region,
+                    credentials: self.aws_credentials
+                )
+                
+                self.aws_credentials
+            end
+            
+            
+            def get_aws_region
+                # "ap-southeast-2"
+                
+                self["AWS_REGION"] = (ENV["AWS_REGION"] || "us-east-1")
+            end
+            
+            
+            protected
+                def is_prod?
+                    self.secrets["ENV"] == "production"
+                end
+                
+                
+                def set_task_info!
+                    return self.secrets["DEPLOYMENT_ENV"] if self.secrets["DEPLOYMENT_ENV"]
+                    
+                    if self.is_prod?
+                        metadata = make_request!(get_metadata_uri)
+                        
+                        ecs_client = Aws::ECS::Client.new(
+                            :region => self.secrets["AWS_REGION"],
+                            :credentials => self.aws_credentials
+                        )
+                        
+                        ecs_response = ecs_client.describe_tasks(
+                            :cluster => metadata["Cluster"],
+                            :tasks => [
+                                metadata["TaskARN"]
+                            ],
+                            :include => ["TAGS"]
+                        )
+                        
+                        ecs_response["tasks"][0]["tags"].each do |i|
+                            self[i["key"]] = i["value"]
+                        end
+                        
+                        report_error("Deployment env not found") unless self.secrets["DEPLOYMENT_ENV"]
+                        
+                        self["RELEASE_VERSION"] = ecs_response["tasks"][0]["containers"][0]["containerArn"] rescue nil
+                    else
+                        self["DEPLOYMENT_ENV"] = "development"
+                    end
+                end
+                
+                
+                def get_config_and_secrets!
+                    values = nil
+                    
+                    if self.is_prod?
+                        secrets_client = Aws::SecretsManager::Client.new(
+                            :region => self.secrets["AWS_REGION"],
+                            :credentials => self.aws_credentials
+                        )
+                        
+                        config = secrets_client.get_secret_value({
+                            :secret_id => (self.secrets["DEPLOYMENT_ENV"].to_s + self.config_suffix)
+                        })
+                        
+                        secrets = secrets_client.get_secret_value({
+                            :secret_id => (self.secrets["DEPLOYMENT_ENV"].to_s + self.secrets_suffix)
+                        })
+                        
+                        values = Hash.new
+                        values.merge!(JSON.parse(config[:secret_string]))
+                        values.merge!(JSON.parse(secrets[:secret_string]))
+                    else
+                        values = self.config.config_for(:metadata)
+                    end
+                    
+                    values.each { |k, v| self[k] = v } if values
+                end
+            
+            
+            private
+                def get_metadata_uri
+                    ENV["ECS_CONTAINER_METADATA_URI"].to_s + "/task"
+                end
+                
+                def get_credentials_uri
+                    "http://169.254.170.2" + ENV["AWS_CONTAINER_CREDENTIALS_RELATIVE_URI"].to_s
+                end
+                
+                def make_request!(request_url)
+                    uri = URI.parse(request_url)
+                    http = Net::HTTP.start(uri.host, uri.port, :use_ssl => uri.scheme == "https")
+                    request = Net::HTTP::Get.new(uri.to_s)
+                    response = http.request(request) rescue nil
+                    
+                    if response && response.code.to_i === 200
+                        return JSON.parse(response.body).with_indifferent_access
+                    else
+                        report_error("Failed to ger request: #{request_url}")
+                    end
+                end
+                
+                def report_error(message)
+                    raise message
+                end
+        
+            
+        end
+    end
+end