shatter-rb 0.0.1

data/lib/shatter/service/base.rb

@@ -0,0 +1,93 @@
+# frozen_string_literal: true
+require 'singleton'
+require 'benchmark'
+require "drb/drb"
+require "zk"
+require "concurrent-ruby"
+
+# Server Side of the drb
+module Shatter
+  module Service
+    class Base
+      include Concurrent::Async
+
+      class ZooKeeperConnection
+        include Singleton
+        attr_reader :client
+        def initialize
+          @client = ZK.new(Shatter::Config.zookeeper_host)
+        end
+      end
+
+      class << self
+        attr_reader :service_definition
+        attr_writer :service_definition
+      end
+
+      def self.response_for(uuid)
+        ResponsePool.instance.pool[uuid]
+      end
+
+      def self.respond_to_missing?(method)
+        @service_definition.instance_methods.include?(method.to_sym)
+      end
+
+      def self.set_static_result_for(uuid, result)
+        puts "Setting static content"
+        populate_pool_with_result(Time.now, {uuid:, result:}, nil)
+      end
+
+      def self.method_missing(method, *args, &)
+        Shatter.logger.info "Mapping #{method}"
+        uuid = args[0].is_a?(Hash) ? args[0][:uuid] : args[0].uuid
+        return {error: 'missing uuid'} if uuid.nil?
+        future = @service_definition.new.async.send(method, *args, &)
+        future.add_observer(self, :populate_pool_with_result)
+      end
+
+      def self.populate_pool_with_result(time, value, err)
+        if err
+          Shatter.logger.info err
+        end
+        ResponsePool.instance.pool[value[:uuid]] = value
+        zk = ZooKeeperConnection.instance.client
+        key = nil
+        key = Util.zookeeper_response_key(value[:uuid])
+        my_ip = ENV["HOST_NAME"] || "localhost"
+        host = "#{my_ip}:#{Shatter::Config.service_port}"
+        Shatter.logger.info "Recording location of #{value[:uuid]} at #{host} #{value}"
+        zk.create(key, host)
+        my_ip
+      rescue Exception => e
+        Shatter.logger.error e
+        raise e
+      end
+
+      def self.close
+        logger = Shatter.logger
+        logger.info "Closing down DRb service"
+        port = Shatter::Config.service_port
+        uri = "localhost:#{port}"
+        logger.info "Removing my existnce at #{port} to zookeeper"
+        Shatter::Service::Discovery.deregister_service(uri)
+        logger.info "Closed DRb service"
+      end
+
+      def self.init
+        logger.info "Initing DRb service"
+        port = Shatter::Config.service_port
+        uri = "localhost:#{port}"
+        logger.info "Logging my existnce at #{uri} to zookeeper"
+        Shatter::Service::Discovery.register_service(uri)
+        logger.info "Starting DRb service"
+        DRb.start_service("druby://#{uri}", self)
+        logger.info "DRb service started"
+        DRb.thread.join
+      end
+
+      def self.logger
+        Shatter.logger
+      end
+    end
+  end
+end

data/lib/shatter/service/discovery.rb

@@ -0,0 +1,49 @@
+module Shatter
+  module Service
+    class Discovery
+      class << self
+
+        def deregister_service(service_url)
+          zk = ZK.new (Shatter::Config.zookeeper_host)
+          if zk.exists?("/shater_service_instances/#{service_url}")
+            zk.delete("/shater_service_instances/#{service_url}")
+          end
+          zk.close
+        end
+
+        def register_service(service_url)
+          Shatter.logger.info "Registering #{service_url} to zookeeper"
+          zk = ZK.new (Shatter::Config.zookeeper_host)
+          unless zk.exists?("/shater_service_instances/#{service_url}")
+            created = zk.create("/shater_service_instances/#{service_url}")
+            Shatter.logger.info "Registered #{created}"
+          end
+          puts zk.children("/shater_service_instances")
+          zk.close
+        end
+
+        def service_instance_url
+          service_instance_urls.sample
+        end
+
+        def service_instance_urls
+          zk = ZK.new (Shatter::Config.zookeeper_host)
+          urls = zk.children("/shater_service_instances")
+          zk.close
+          urls
+        end
+
+        def service_instance_url_for_uuid(uuid)
+          druby_instance_url = nil
+          key = Util.zookeeper_response_key(uuid)
+          zk = ZK.new(Shatter::Config.zookeeper_host)
+          druby_instance_url = zk.get(key)[0] if zk.exists?(key)
+          zk.close
+          Shatter.logger.debug "Service instance url for #{uuid} - #{druby_instance_url}"
+
+          druby_instance_url
+        end
+      end
+    end
+  end
+end

data/lib/shatter/service/function.rb

@@ -0,0 +1,56 @@
+module Shatter
+  module Service
+    class Function
+
+      def self.define_param(name, type:, nullable: true)
+        @@param_meta ||= {}
+        @@param_meta[name] = {name:, type:, nullable:}
+      end
+
+      def self.meta
+        @@param_meta
+      end
+
+      define_param :uuid, nullable: false, type: 'string'
+
+      def initialize(function_params)
+        @params = function_params
+      end
+
+      def params
+        Hash[@@param_meta.keys.map { |k| [k]}].merge(@params)
+      end
+
+      def call
+        raise 'Invalid Args' unless valid_params?
+        valid_keys = @@param_meta.keys
+        {result: nil, error:nil}.merge(self.invoke.merge(uuid: params[:uuid]))
+      rescue Exception => e
+        {result: nil, error: e.message, uuid: params[:uuid]}
+      end
+
+      def valid_params?
+        self.class.meta.keys.each do |arg|
+          meta = self.class.meta[arg]
+          val = @params[arg]
+          puts meta
+          raise "#{arg} cannot be nil" if !meta[:nullable] && val.nil?
+          raise "expected #{arg} to be a string" if meta[:type]  == 'string' && !val.is_a?(String)
+
+          if meta[:type]  == 'integer'
+            if !meta[:nullable] && val.nil?
+              raise 'value cannot be nil'
+            end
+            if !val.nil? && !val.is_a?(Integer)
+              raise "expected #{arg} to be an integer"
+            end
+          end
+        end
+      end
+
+      def self.invoke
+        raise 'cant invoke for base function'
+      end
+    end
+  end
+end

data/lib/shatter/service/function_params.rb

@@ -0,0 +1,21 @@
+module Shatter
+  module Service
+    class FunctionParams < Data
+
+      def self.generate(*args, &block)
+        args << :uuid
+
+        Data.define(*args) do
+          def to_typescript
+            typescript_name = self.class.to_s.split("::").last(2).join
+            out = <<-HEREDOC.gsub(/^\s+/, "")
+              type #{typescript_name} {
+              };
+              export default #{typescript_name}
+            HEREDOC
+          end
+        end
+      end
+    end
+  end
+end

data/lib/shatter/service/response_pool.rb

@@ -0,0 +1,10 @@
+require 'singleton'
+module Shatter
+  class ResponsePool
+    include Singleton
+    attr_reader :pool
+    def initialize
+      @pool = {}
+    end
+  end
+end

data/lib/shatter/service/service_definition.rb

@@ -0,0 +1,21 @@
+require "concurrent-ruby"
+require 'set'
+
+module Shatter
+  module Service
+    class ServiceDefinition
+      include Concurrent::Async
+      class << self
+        attr_reader :function_collection
+      end
+      def self.register_function(identifier, function)
+        Shatter.logger.info "Registering function - #{identifier} for #{self}"
+        @function_collection ||= {}
+        @function_collection[identifier.to_s] = function
+        define_method identifier do |params|
+          function.new(params).call
+        end
+      end
+    end
+  end
+end

data/lib/shatter/util.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+require 'logger'
+require 'singleton'
+
+module Shatter
+  module Util
+    class Logger < ::Logger
+      include Singleton
+      def initialize
+        super(STDOUT, datetime_format: '%Y-%m-%d %H:%M:%S')
+      end
+    end
+
+    def self.zookeeper_response_key(uuid)
+      raise 'Cant produce key without uuid' if uuid.nil?
+      "/shatter::response_data_locations/#{uuid}"
+    end
+  end
+end

data/lib/shatter/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module Shatter
+  VERSION = "0.0.1"
+end

data/lib/shatter/web/application.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+require "drb/drb"
+require "concurrent-ruby"
+
+require_relative "./application"
+
+module Shatter
+  module Web
+    class Application
+      include Concurrent::Async
+
+      def response_for(uuid)
+        druby_instance_url = Shatter::Service::Discovery.service_instance_url_for_uuid(uuid)
+        return unless druby_instance_url
+        DRbObject.new_with_uri("druby://#{druby_instance_url}").response_for(uuid)
+      end
+
+      def route(uuid, path, params)
+        operation = path.scan(/\/(.+)$/).first&.first
+        raise 'failed to route' if operation.nil?
+        Shatter.logger.info Shatter::Service::Base.service_definition.function_collection
+        Shatter.logger.info operation
+        function = Shatter::Service::Base.service_definition.function_collection[operation]
+        if function.nil?
+          DRbObject.new_with_uri("druby://#{Shatter::Service::Discovery.service_instance_url}")
+            .set_static_result_for(uuid, {result: nil, error: :unknown_operation})
+        else
+          begin
+            func_params = params.merge(uuid:)
+            Shatter.logger.info "routing #{path}/#{func_params}"
+            DRbObject.new_with_uri("druby://#{Shatter::Service::Discovery.service_instance_url}")
+              .send(
+                operation.to_sym,
+                func_params
+              )
+          rescue ArgumentError => e
+            DRbObject.new_with_uri("druby://#{Shatter::Service::Discovery.service_instance_url}")
+              .set_static_result_for(uuid, {result: nil, error: e.message })
+          end
+        end
+      rescue Exception => e
+        Shatter.logger.error "caught error: #{e.message}"
+        Shatter.logger.error "#{e.backtrace.join("\n")}"
+        { data:, error: e.message, uuid: }
+      end
+
+    end
+  end
+end

data/lib/shatter/web/server.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+require "securerandom"
+require "json"
+
+module Shatter
+  module Web
+    class Server
+
+      class << self
+        attr_reader :application
+        attr_writer :application
+      end
+
+      def self.call(env)
+        request = Rack::Request.new(env)
+        path = env["PATH_INFO"]
+        if env["PATH_INFO"] == "/callbacks"
+          uuid = env['QUERY_STRING'].split("=")[1]
+          response_for(uuid)
+        elsif env["PATH_INFO"] == "/base_time"
+          [200, {}, [""]]
+        else
+          params = JSON.parse(request.body.read, {symbolize_names: true})
+          uuid = SecureRandom.uuid
+          future = application.new.async.route(uuid, path, params)
+          future.add_observer(:server_call_result, self)
+          [200, { "delay" => "20", "location" => "/callbacks?uuid=#{uuid}" }, []]
+        end
+      end
+
+      def self.server_call_result(time, value, error)
+        return if error.nil?
+        Shatter.logger.error "#{ error }"
+      end
+
+      def self.response_for(uuid)
+        response = application.new.response_for(uuid)
+        return [200, {}, [response.to_json]] unless response.nil?
+
+        [200, { "delay" => "100", "location" => "/callbacks?uuid=#{uuid}" }, []]
+      end
+    end
+  end
+end

data/lib/shatter.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+require_relative "shatter/version"
+require_relative "shatter/web/application"
+require_relative "shatter/config"
+require_relative "shatter/web/server"
+require_relative "shatter/service/base"
+require_relative "shatter/service/discovery"
+require_relative "shatter/service/service_definition"
+require_relative "shatter/service/response_pool"
+require_relative "shatter/util"
+
+$stdout.sync = true
+
+module Shatter
+  class Error < StandardError; end
+  def self.logger
+    Util::Logger.instance
+  end
+  # Your code goes here...
+end

data/shatter.gemspec

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+require_relative "lib/shatter/version"
+
+Gem::Specification.new do |spec|
+  spec.name = "shatter-rb"
+  spec.version = Shatter::VERSION
+  spec.authors = ["Eric Roos"]
+  spec.email = ["ericroos@hey.com"]
+
+  spec.summary = "Write a short summary, because RubyGems requires one."
+  spec.description = "Write a longer description or delete this line."
+  spec.homepage = "https://rubygems.org/gems/shatter"
+  spec.license = "MIT"
+  spec.required_ruby_version = ">= 3.1.0"
+
+  #spec.metadata["allowed_push_host"] = "TODO: Set to your gem server 'https://example.com'"
+
+  spec.metadata["homepage_uri"] = spec.homepage
+  spec.metadata["source_code_uri"] = "https://github.com/EricRoos/shatter"
+  spec.metadata["changelog_uri"] = "https://github.com/EricRoos/shatter/tree/master/CHANGELOG"
+
+  # Specify which files should be added to the gem when it is released.
+  # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
+  spec.files = Dir.chdir(__dir__) do
+    `git ls-files -z`.split("\x0").reject do |f|
+      (f == __FILE__) || f.match(%r{\A(?:(?:javascript|bin|test|spec|features)/|\.(?:git|circleci)|appveyor)})
+    end
+  end
+  spec.bindir = "exe"
+  spec.executables = spec.files.grep(%r{\Aexe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  # Uncomment to register a new dependency of your gem
+  # spec.add_dependency "example-gem", "~> 1.0"
+  spec.add_dependency "rack", "~> 3.0"
+  spec.add_dependency "rackup", "~> 0.2.3"
+  spec.add_dependency "concurrent-ruby", "~> 1.1"
+  spec.add_dependency "pg", "~> 1.4"
+  spec.add_dependency "zk", "~> 1.10"
+  spec.add_dependency "puma", "~> 6.0"
+  spec.add_dependency "thor", "~> 1.2.1"
+  spec.add_dependency "erb", "~> 2.2.0"
+  # spec.add_dependency "zookeeper", "~>1.5.4", github: 'EricRoos/zookeeper'
+
+  spec.add_development_dependency "rake", "~> 13.0"
+  spec.add_development_dependency "rspec", "~> 3.0"
+  spec.add_development_dependency "rubocop", "~> 1.21"
+  spec.add_development_dependency "rubocop-rake"
+  spec.add_development_dependency "rubocop-rspec"
+  # For more information and examples about making a new gem, check out our
+  # guide at: https://bundler.io/guides/creating_gem.html
+  #spec.metadata["rubygems_mfa_required"] = "true"
+end

data/sig/shatter.rbs

@@ -0,0 +1,4 @@
+module Shatter
+  VERSION: String
+  # See the writing guide of rbs: https://github.com/ruby/rbs#guides
+end

data/templates/Gemfile.template

@@ -0,0 +1,5 @@
+source "https://rubygems.org"
+gem 'shatter', path: '..'
+gem "zookeeper", "~>1.5.4", github: 'EricRoos/zookeeper'
+
+gem "foreman", "~> 0.87.2"

data/templates/application.erb

@@ -0,0 +1,6 @@
+module <%= app_name %>
+  class Application < Shatter::Web::Application
+  end
+end
+
+require_relative './app/service_definition'

data/templates/config.ru

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+require "bundler/setup"
+require "shatter"
+
+require_relative "./config/environment"
+
+use Rack::CommonLogger
+run Shatter::Web::Server

data/templates/docker-compose.yml

@@ -0,0 +1,32 @@
+version: '3.1'
+
+services:
+  zoo1:
+    image: zookeeper
+    restart: always
+    hostname: zoo1
+    ports:
+      - 2181:2181
+    environment:
+      ZOO_MY_ID: 1
+      ZOO_SERVERS: server.1=zoo1:2888:3888;2181 server.2=zoo2:2888:3888;2181 server.3=zoo3:2888:3888;2181
+
+  zoo2:
+    image: zookeeper
+    restart: always
+    hostname: zoo2
+    ports:
+      - 2182:2181
+    environment:
+      ZOO_MY_ID: 2
+      ZOO_SERVERS: server.1=zoo1:2888:3888;2181 server.2=zoo2:2888:3888;2181 server.3=zoo3:2888:3888;2181
+
+  zoo3:
+    image: zookeeper
+    restart: always
+    hostname: zoo3
+    ports:
+      - 2183:2181
+    environment:
+      ZOO_MY_ID: 3
+      ZOO_SERVERS: server.1=zoo1:2888:3888;2181 server.2=zoo2:2888:3888;2181 server.3=zoo3:2888:3888;2181

data/templates/docker-compose.yml.example

@@ -0,0 +1,32 @@
+version: '3.1'
+
+services:
+  zoo1:
+    image: zookeeper
+    restart: always
+    hostname: zoo1
+    ports:
+      - 2181:2181
+    environment:
+      ZOO_MY_ID: 1
+      ZOO_SERVERS: server.1=zoo1:2888:3888;2181 server.2=zoo2:2888:3888;2181 server.3=zoo3:2888:3888;2181
+
+  zoo2:
+    image: zookeeper
+    restart: always
+    hostname: zoo2
+    ports:
+      - 2182:2181
+    environment:
+      ZOO_MY_ID: 2
+      ZOO_SERVERS: server.1=zoo1:2888:3888;2181 server.2=zoo2:2888:3888;2181 server.3=zoo3:2888:3888;2181
+
+  zoo3:
+    image: zookeeper
+    restart: always
+    hostname: zoo3
+    ports:
+      - 2183:2181
+    environment:
+      ZOO_MY_ID: 3
+      ZOO_SERVERS: server.1=zoo1:2888:3888;2181 server.2=zoo2:2888:3888;2181 server.3=zoo3:2888:3888;2181

data/templates/environment.rb.erb

@@ -0,0 +1,9 @@
+Shatter::Config.zookeeper_host = "localhost:2181"
+Shatter::Config.initial_delay = 100
+Shatter::Config.missing_result_delay = 100
+Shatter::Config.service_port = ENV.fetch('SHATTER_SERVICE_PORT') { 8787 }
+
+require_relative '../application'
+
+Shatter::Service::Base.service_definition = <%= app_name%>::ServiceDefinition
+Shatter::Web::Server.application = <%= app_name%>::Application

data/templates/hello_world_function.rb.erb

@@ -0,0 +1,17 @@
+require "pg"
+require "shatter/service/function"
+require "shatter/service/function_params"
+
+module <%= app_name %>
+  module Functions
+    class HelloWorldFunction < Shatter::Service::Function
+      define_param :name, nullable: false, type: 'string'
+      define_param :number, nullable: false, type: 'integer'
+
+      def invoke
+        params.to_h => name:, uuid:
+        { result: "Hello #{name}", uuid:, error: nil, uuid: }
+      end
+    end
+  end
+end

data/templates/service_definition.rb.erb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+require_relative './functions/hello_world_function'
+
+module <%= app_name %>
+  class ServiceDefinition < Shatter::Service::ServiceDefinition
+    register_function :hello_world, <%= app_name %>::Functions::HelloWorldFunction
+  end
+end