concord-ruby 0.0.4

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 0fd32ec29dc607471dd30ca833037e545be7d3d0
+  data.tar.gz: 23db22d170657075bc23e99534d6e4f7daf44b33
+SHA512:
+  metadata.gz: ef43347e379410626c26da939574695fef7d5a14afe48e078f0667868d74192fbd3e34398d34219cf68aff8716fc244b36e9e0563acc4ea147ba601e928ab6b6
+  data.tar.gz: 73d5ff237e89ee45953f5dc9a0031f3f648e6ef43b0a9a8e21cfb4a0aab8a17d7f79d7e0e4ccf89ba6718caa70c94ad3e4e1f623e54b9287e4a06899a759f87d

data/lib/concord.rb

@@ -0,0 +1,201 @@
+require 'thrift'
+require 'optparse'
+require 'bolt_proxy_service'
+require 'computation_service'
+require 'bolt_types'
+require 'bolt_constants'
+
+module Concord
+  # Wrapper over the ComputationMetadata object from the thrift interface
+  class Metadata
+    attr_accessor :name, :istreams, :ostreams
+
+    # Initialize a new `Metadata` object
+    # @param name [String] The globally unique identifier of the application
+    # @param istreams [Array] An array of input streams. These can be formatted
+    # as either a `String` identifying the stream or an `Array` of length two,
+    # in which the first parameter is the `String` identifier and the second
+    # parameter is a `StreamGrouping` defining how incoming records should be
+    # routed.
+    # @param ostreams [Array] An array of `String` stream identifiers that this
+    # computation may produce records on.
+    def initialize(name: nil, istreams: [], ostreams: [])
+      self.name = name
+      self.istreams = istreams
+      self.ostreams = ostreams
+    end
+  end
+
+  # Transactional wrapper for proxy <=> client interactions
+  class ComputationContext
+    attr_accessor :transaction, :computation
+
+    # Initialize a new `ComputationContext`
+    # @param computation [Object] The user-defined computation
+    def initialize(computation)
+      self.computation = computation
+      self.transaction = ::Concord::Thrift::ComputationTx.new
+      self.transaction.records = []
+      self.transaction.timers = {}
+    end
+
+    # Produce a record on `stream` with `key` and `value`
+    # @param stream [String] Globally unique name of string
+    # @param key [String] Key to group by on stream (pending grouping)
+    # @param value [String] Binary blob to pass downstream with tuple
+    def produce_record(stream, key, value)
+      r = ::Concord::Thrift::Record.new
+      r.meta = ::Concord::Thrift::RecordMetadata.new
+      r.key = key
+      r.data = value
+      r.userStream = stream
+      transaction.records.push(r)
+    end
+
+    # Set a timer to trigger a callback in the future
+    # @param key [String] Name of the timer. This parameter will be passed to
+    # the `process_timer` callback to identify the specific callback.
+    # @param time [FixNum] Integer representing the time (in milliseconds) at which the callback should be triggered.
+    def set_timer(key, time)
+      transaction.timers[key] = time
+    end
+
+    # Retrieve a binary blob stored in the proxy state
+    # @param key [String] Key to fetch from data store
+    # @return [String] Binary blob of data
+    def get_state(key)
+      computation.get_state(key)
+    end
+
+    # Store a binary blob, identified by a key, in the proxy state
+    # @param key [String] Key to set in data store
+    # @param value [String] Binary blob
+    def set_state(key, value)
+      computation.set_state(key, value)
+    end
+  end
+
+  # Thrift service definition. Wraps a user-defined computation.
+  class Computation
+    attr_accessor :handler, :proxy_host, :proxy_port
+
+    # Initialize a new `Computation` and register it with the proxy
+    # @param handler [Object] The user-defined computation
+    # @param proxy_host [String] The address the proxy is listening on
+    # @param proxy_port [FixNum] The port the proxy is listening on
+    def initialize(handler: nil, proxy_host: nil, proxy_port: nil)
+      self.handler = handler
+      self.proxy_host = proxy_host
+      self.proxy_port = proxy_port
+      proxy.registerWithScheduler(boltMetadata)
+    end
+
+    # Initialize a new `Computation` and start serving it. This is the only
+    # method directly called by users.
+    # @param computation [Object] The user-defined computation
+    def self.serve(computation)
+      listen_address = ENV[::Concord::Thrift::KConcordEnvKeyClientListenAddr]
+      proxy_address = ENV[::Concord::Thrift::KConcordEnvKeyClientProxyAddr]
+      listen_host, listen_port = listen_address.split(':')
+      proxy_host, proxy_port = proxy_address.split(':')
+
+      handler = self.new(handler: computation,
+                         proxy_host: proxy_host,
+                         proxy_port: Integer(proxy_port))
+
+      processor = Thrift::ComputationService::Processor.new(handler)
+      transport = ::Thrift::ServerSocket.new(listen_host, Integer(listen_port))
+      transport_factory = ::Thrift::FramedTransportFactory.new
+      protocol_factory = ::Thrift::BinaryProtocolAcceleratedFactory.new
+      server = ::Thrift::ThreadedServer.new(processor,
+                                            transport,
+                                            transport_factory,
+                                            protocol_factory)
+      server.serve
+    end
+
+    # Process an upstream record. Wraps the user method in a transaction, which
+    # is returned to the proxy upon completion.
+    # @param record [Concord::Thrift::Record] The record to process
+    def boltProcessRecord(record)
+      ctx = ComputationContext.new(self)
+      handler.process_record(ctx, record)
+      ctx.transaction
+    end
+
+    # Process a timer callback from the proxy. Wraps the user method in a
+    # transaction, which is returned to the proxy upon completion.
+    # @param key [String] Callback identifier
+    # @param time [FixNum] Time this callback was scheduled to trigger.
+    def boltProcessTimer(key, time)
+      ctx = ComputationContext.new(self)
+      handler.process_timer(ctx, key, time)
+      ctx.transaction
+    end
+
+    # The initialization function, called when the framework is ready to start
+    # sending the computation records. Wraps the user method in a transaction,
+    # which is returned to the proxy upon completion.
+    def init
+      ctx = ComputationContext.new(self)
+      handler.init(ctx)
+      ctx.transaction
+    end
+
+    # @return [Concord::Thrift::ComputationMetadata] The user-defined computation metadata.
+    def boltMetadata
+      metadata = handler.metadata
+      enrich_metadata(metadata)
+    end
+
+    # Retrieve a binary blob stored in the proxy state
+    # @param key [String] Key to fetch from data store
+    # @return [String] Binary blob of data
+    def get_state(key)
+      proxy.getState(key)
+    end
+
+    # Store a binary blob, identified by a key, in the proxy state
+    # @param key [String] Key to set in data store
+    # @param value [String] Binary blob
+    def set_state(key, value)
+      proxy.setState(key, value)
+    end
+
+    private
+
+    def proxy
+      if @proxy.nil? || !@proxy_socket.open?
+        @proxy_socket = ::Thrift::Socket.new(proxy_host, proxy_port)
+        transport = ::Thrift::FramedTransport.new(@proxy_socket)
+        protocol = ::Thrift::BinaryProtocol.new(transport)
+        @proxy = ::Concord::Thrift::BoltProxyService::Client.new(protocol)
+        transport.open
+      end
+
+      @proxy
+    end
+
+    def enrich_metadata(metadata)
+      enrich_stream = ->(stream) {
+        name = nil
+        grouping = nil
+        if stream.is_a?(Array)
+          name, grouping = stream
+        end
+        sm = ::Concord::Thrift::StreamMetadata.new
+        sm.name = name
+        sm.grouping unless grouping.nil?
+        sm
+      }
+      cm = ::Concord::Thrift::ComputationMetadata.new
+      cm.name = metadata.name
+      cm.istreams = metadata.istreams.map { |x| enrich_stream.call(x) }
+      cm.ostreams = metadata.ostreams.map { |x| enrich_stream.call(x) }
+      cm.proxyEndpoint = ::Concord::Thrift::Endpoint.new
+      cm.proxyEndpoint.ip = proxy_host
+      cm.proxyEndpoint.port = proxy_port
+      cm
+    end
+  end
+end

data/lib/concord/utils.rb

@@ -0,0 +1,13 @@
+module Concord
+  # Some helper functions useful in client computations
+  module Utils
+    extend self
+
+    # Get the time in milliseconds
+    # @param t [Time] The time to convert to milliseconds
+    def time_millis(t = Time.now)
+      (t.to_f * 1000.0).to_i
+    end
+  end
+end
+

data/lib/gen-rb/bolt_constants.rb

@@ -0,0 +1,47 @@
+#
+# Autogenerated by Thrift Compiler (0.9.2)
+#
+# DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
+#
+
+require 'thrift'
+require 'bolt_types'
+
+module Concord
+  module Thrift
+    KBoltEnvKeyBasePath = %q"BOLT_BASE_PATH"
+
+    KBoltDefaultEnvBasePath = %q"/tmp/"
+
+    KBoltEnvKeyPathPrefix = %q"BOLT"
+
+    KDefaultThriftServiceIOThreads = 2
+
+    KConcordEnvKeyClientListenAddr = %q"CONCORD_client_listen_address"
+
+    KConcordEnvKeyClientProxyAddr = %q"CONCORD_client_proxy_address"
+
+    KDatabasePath = %q"/tmp"
+
+    KDatabaseEntryTTL = 43200
+
+    KDefaultBatchSize = 2048
+
+    KDefaultTraceSampleEveryN = 1024
+
+    KIncomingMessageQueueTopic = %q"incoming"
+
+    KOutgoingMessageQueueTopic = %q"outgoing"
+
+    KQueueStreamNameToIdMapTopic = %q"stream_map"
+
+    KMessageQueueWatermarkTopic = %q"watermarks"
+
+    KMessageQueueBatchSize = 1024
+
+    KMessageQueueTTL = 21600
+
+    KBoltTraceHeader = %q"bolt_traces"
+
+  end
+end

data/lib/gen-rb/bolt_manager_service.rb

@@ -0,0 +1,151 @@
+#
+# Autogenerated by Thrift Compiler (0.9.2)
+#
+# DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
+#
+
+require 'thrift'
+require 'bolt_types'
+
+module Concord
+  module Thrift
+    module BoltManagerService
+      class Client
+        include ::Thrift::Client
+
+        def setEnvironmentVar(name, value)
+          send_setEnvironmentVar(name, value)
+          recv_setEnvironmentVar()
+        end
+
+        def send_setEnvironmentVar(name, value)
+          send_message('setEnvironmentVar', SetEnvironmentVar_args, :name => name, :value => value)
+        end
+
+        def recv_setEnvironmentVar()
+          result = receive_message(SetEnvironmentVar_result)
+          raise result.e unless result.e.nil?
+          return
+        end
+
+        def signal(topologyId, s)
+          send_signal(topologyId, s)
+          recv_signal()
+        end
+
+        def send_signal(topologyId, s)
+          send_message('signal', Signal_args, :topologyId => topologyId, :s => s)
+        end
+
+        def recv_signal()
+          result = receive_message(Signal_result)
+          raise result.e unless result.e.nil?
+          return
+        end
+
+      end
+
+      class Processor
+        include ::Thrift::Processor
+
+        def process_setEnvironmentVar(seqid, iprot, oprot)
+          args = read_args(iprot, SetEnvironmentVar_args)
+          result = SetEnvironmentVar_result.new()
+          begin
+            @handler.setEnvironmentVar(args.name, args.value)
+          rescue ::Concord::Thrift::BoltError => e
+            result.e = e
+          end
+          write_result(result, oprot, 'setEnvironmentVar', seqid)
+        end
+
+        def process_signal(seqid, iprot, oprot)
+          args = read_args(iprot, Signal_args)
+          result = Signal_result.new()
+          begin
+            @handler.signal(args.topologyId, args.s)
+          rescue ::Concord::Thrift::BoltError => e
+            result.e = e
+          end
+          write_result(result, oprot, 'signal', seqid)
+        end
+
+      end
+
+      # HELPER FUNCTIONS AND STRUCTURES
+
+      class SetEnvironmentVar_args
+        include ::Thrift::Struct, ::Thrift::Struct_Union
+        NAME = 1
+        VALUE = 2
+
+        FIELDS = {
+          NAME => {:type => ::Thrift::Types::STRING, :name => 'name'},
+          VALUE => {:type => ::Thrift::Types::STRING, :name => 'value'}
+        }
+
+        def struct_fields; FIELDS; end
+
+        def validate
+        end
+
+        ::Thrift::Struct.generate_accessors self
+      end
+
+      class SetEnvironmentVar_result
+        include ::Thrift::Struct, ::Thrift::Struct_Union
+        E = 1
+
+        FIELDS = {
+          E => {:type => ::Thrift::Types::STRUCT, :name => 'e', :class => ::Concord::Thrift::BoltError}
+        }
+
+        def struct_fields; FIELDS; end
+
+        def validate
+        end
+
+        ::Thrift::Struct.generate_accessors self
+      end
+
+      class Signal_args
+        include ::Thrift::Struct, ::Thrift::Struct_Union
+        TOPOLOGYID = 1
+        S = 2
+
+        FIELDS = {
+          TOPOLOGYID => {:type => ::Thrift::Types::I64, :name => 'topologyId'},
+          S => {:type => ::Thrift::Types::I32, :name => 's', :enum_class => ::Concord::Thrift::Signal}
+        }
+
+        def struct_fields; FIELDS; end
+
+        def validate
+          unless @s.nil? || ::Concord::Thrift::Signal::VALID_VALUES.include?(@s)
+            raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Invalid value of field s!')
+          end
+        end
+
+        ::Thrift::Struct.generate_accessors self
+      end
+
+      class Signal_result
+        include ::Thrift::Struct, ::Thrift::Struct_Union
+        E = 1
+
+        FIELDS = {
+          E => {:type => ::Thrift::Types::STRUCT, :name => 'e', :class => ::Concord::Thrift::BoltError}
+        }
+
+        def struct_fields; FIELDS; end
+
+        def validate
+        end
+
+        ::Thrift::Struct.generate_accessors self
+      end
+
+    end
+
+  end
+end

data/lib/gen-rb/bolt_metrics_service.rb

@@ -0,0 +1,268 @@
+#
+# Autogenerated by Thrift Compiler (0.9.2)
+#
+# DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
+#
+
+require 'thrift'
+require 'bolt_types'
+
+module Concord
+  module Thrift
+    module BoltMetricsService
+      class Client
+        include ::Thrift::Client
+
+        def gauge(name, val)
+          send_gauge(name, val)
+          recv_gauge()
+        end
+
+        def send_gauge(name, val)
+          send_message('gauge', Gauge_args, :name => name, :val => val)
+        end
+
+        def recv_gauge()
+          result = receive_message(Gauge_result)
+          raise result.e unless result.e.nil?
+          return
+        end
+
+        def timer(name, duration)
+          send_timer(name, duration)
+          recv_timer()
+        end
+
+        def send_timer(name, duration)
+          send_message('timer', Timer_args, :name => name, :duration => duration)
+        end
+
+        def recv_timer()
+          result = receive_message(Timer_result)
+          raise result.e unless result.e.nil?
+          return
+        end
+
+        def histogram(name, measure)
+          send_histogram(name, measure)
+          recv_histogram()
+        end
+
+        def send_histogram(name, measure)
+          send_message('histogram', Histogram_args, :name => name, :measure => measure)
+        end
+
+        def recv_histogram()
+          result = receive_message(Histogram_result)
+          raise result.e unless result.e.nil?
+          return
+        end
+
+        def sum(name, counter)
+          send_sum(name, counter)
+          recv_sum()
+        end
+
+        def send_sum(name, counter)
+          send_message('sum', Sum_args, :name => name, :counter => counter)
+        end
+
+        def recv_sum()
+          result = receive_message(Sum_result)
+          raise result.e unless result.e.nil?
+          return
+        end
+
+      end
+
+      class Processor
+        include ::Thrift::Processor
+
+        def process_gauge(seqid, iprot, oprot)
+          args = read_args(iprot, Gauge_args)
+          result = Gauge_result.new()
+          begin
+            @handler.gauge(args.name, args.val)
+          rescue ::Concord::Thrift::BoltError => e
+            result.e = e
+          end
+          write_result(result, oprot, 'gauge', seqid)
+        end
+
+        def process_timer(seqid, iprot, oprot)
+          args = read_args(iprot, Timer_args)
+          result = Timer_result.new()
+          begin
+            @handler.timer(args.name, args.duration)
+          rescue ::Concord::Thrift::BoltError => e
+            result.e = e
+          end
+          write_result(result, oprot, 'timer', seqid)
+        end
+
+        def process_histogram(seqid, iprot, oprot)
+          args = read_args(iprot, Histogram_args)
+          result = Histogram_result.new()
+          begin
+            @handler.histogram(args.name, args.measure)
+          rescue ::Concord::Thrift::BoltError => e
+            result.e = e
+          end
+          write_result(result, oprot, 'histogram', seqid)
+        end
+
+        def process_sum(seqid, iprot, oprot)
+          args = read_args(iprot, Sum_args)
+          result = Sum_result.new()
+          begin
+            @handler.sum(args.name, args.counter)
+          rescue ::Concord::Thrift::BoltError => e
+            result.e = e
+          end
+          write_result(result, oprot, 'sum', seqid)
+        end
+
+      end
+
+      # HELPER FUNCTIONS AND STRUCTURES
+
+      class Gauge_args
+        include ::Thrift::Struct, ::Thrift::Struct_Union
+        NAME = 1
+        VAL = 2
+
+        FIELDS = {
+          NAME => {:type => ::Thrift::Types::STRING, :name => 'name'},
+          VAL => {:type => ::Thrift::Types::I64, :name => 'val'}
+        }
+
+        def struct_fields; FIELDS; end
+
+        def validate
+        end
+
+        ::Thrift::Struct.generate_accessors self
+      end
+
+      class Gauge_result
+        include ::Thrift::Struct, ::Thrift::Struct_Union
+        E = 1
+
+        FIELDS = {
+          E => {:type => ::Thrift::Types::STRUCT, :name => 'e', :class => ::Concord::Thrift::BoltError}
+        }
+
+        def struct_fields; FIELDS; end
+
+        def validate
+        end
+
+        ::Thrift::Struct.generate_accessors self
+      end
+
+      class Timer_args
+        include ::Thrift::Struct, ::Thrift::Struct_Union
+        NAME = 1
+        DURATION = 2
+
+        FIELDS = {
+          NAME => {:type => ::Thrift::Types::STRING, :name => 'name'},
+          DURATION => {:type => ::Thrift::Types::I64, :name => 'duration'}
+        }
+
+        def struct_fields; FIELDS; end
+
+        def validate
+        end
+
+        ::Thrift::Struct.generate_accessors self
+      end
+
+      class Timer_result
+        include ::Thrift::Struct, ::Thrift::Struct_Union
+        E = 1
+
+        FIELDS = {
+          E => {:type => ::Thrift::Types::STRUCT, :name => 'e', :class => ::Concord::Thrift::BoltError}
+        }
+
+        def struct_fields; FIELDS; end
+
+        def validate
+        end
+
+        ::Thrift::Struct.generate_accessors self
+      end
+
+      class Histogram_args
+        include ::Thrift::Struct, ::Thrift::Struct_Union
+        NAME = 1
+        MEASURE = 2
+
+        FIELDS = {
+          NAME => {:type => ::Thrift::Types::STRING, :name => 'name'},
+          MEASURE => {:type => ::Thrift::Types::I64, :name => 'measure'}
+        }
+
+        def struct_fields; FIELDS; end
+
+        def validate
+        end
+
+        ::Thrift::Struct.generate_accessors self
+      end
+
+      class Histogram_result
+        include ::Thrift::Struct, ::Thrift::Struct_Union
+        E = 1
+
+        FIELDS = {
+          E => {:type => ::Thrift::Types::STRUCT, :name => 'e', :class => ::Concord::Thrift::BoltError}
+        }
+
+        def struct_fields; FIELDS; end
+
+        def validate
+        end
+
+        ::Thrift::Struct.generate_accessors self
+      end
+
+      class Sum_args
+        include ::Thrift::Struct, ::Thrift::Struct_Union
+        NAME = 1
+        COUNTER = 2
+
+        FIELDS = {
+          NAME => {:type => ::Thrift::Types::STRING, :name => 'name'},
+          COUNTER => {:type => ::Thrift::Types::I64, :name => 'counter'}
+        }
+
+        def struct_fields; FIELDS; end
+
+        def validate
+        end
+
+        ::Thrift::Struct.generate_accessors self
+      end
+
+      class Sum_result
+        include ::Thrift::Struct, ::Thrift::Struct_Union
+        E = 1
+
+        FIELDS = {
+          E => {:type => ::Thrift::Types::STRUCT, :name => 'e', :class => ::Concord::Thrift::BoltError}
+        }
+
+        def struct_fields; FIELDS; end
+
+        def validate
+        end
+
+        ::Thrift::Struct.generate_accessors self
+      end
+
+    end
+
+  end
+end