observed 0.1.1 → 0.2.0.rc1

data/integrations/observed-eventmachine/observed-eventmachine.gemspec

@@ -0,0 +1,28 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'observed/eventmachine/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "observed-eventmachine"
+  spec.version       = Observed::EM::VERSION
+  spec.authors       = ["KUOKA Yusuke"]
+  spec.email         = ["yusuke.kuoka@gree.net"]
+  spec.description   = %q{An integration of Observed with EventMachine}
+  spec.summary       = %q{observed-eventmachine allows running Observed observations on EventMachine to achieve scalability}
+  spec.homepage      = ""
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files`.split($/)
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency 'eventmachine', '~> 1.0.3'
+
+  spec.add_development_dependency "bundler", "~> 1.3"
+  spec.add_development_dependency "rake"
+  spec.add_development_dependency "rspec"
+  spec.add_development_dependency "cucumber"
+  spec.add_development_dependency "aruba"
+end

data/lib/observed/application/oneshot.rb

@@ -3,10 +3,7 @@ require 'optparse'
 require 'pathname'
 
 require 'observed/config'
-require 'observed/config_builder'
-require 'observed/config_dsl'
-require 'observed/observer'
-require 'observed/system'
+require 'observed/context'
 
 module Observed
 
@@ -28,17 +25,12 @@ module Observed
       end
 
       def run(observation_name=nil)
-        system.run(observation_name)
-      end
-
-      def logger
-        @logger ||= Logger.new(STDOUT)
+        @system.run(observation_name)
       end
 
       class << self
-        def from_argv(argv)
-
-          command_line_args = argv.dup
+        def parse_argv!(argv)
+          command_line_args = argv
 
           args = {}
 
@@ -55,38 +47,29 @@ module Observed
 
           opts.parse!(command_line_args)
 
-          if command_line_args.size != 1
+          unless command_line_args.size == 1 || command_line_args.size == 2
             fail InvalidArgumentError, "Invalid number of arguments #{command_line_args.size} where arguments are #{command_line_args}"
           end
 
-          args[:config_file] = command_line_args.first
+          args[:config_file] = command_line_args.shift
+
+          args
+        end
 
+        def from_argv(argv)
+          args = parse_argv!(argv.dup)
           create(args)
         end
 
         # @param [Hash<Symbol,String>] args
         # @option args [Array<String>] :argv The Ruby's `ARGV` like object which is treated as intialization parameters for Oneshoft application.
         def create(args)
-          logger_out = if args[:log_file]
-                         File.open(args[:log_file], 'a')
-                       else
-                         STDOUT
-                       end
-          logger = Logger.new(logger_out)
-          logger.level = if args[:debug]
-                           Logger::DEBUG
-                         else
-                           Logger::INFO
-                         end
-          sys = Observed::System.new(logger: logger)
+          ctx = Observed::Context.new(args)
+          sys = ctx.system
           config = if args[:yaml_file]
                      YAML.load_file(args[:yaml_file])
                    elsif args[:config_file]
-                     path = args[:config_file]
-                     config_builder = Observed::ConfigBuilder.new(system: sys, logger: logger)
-                     config_dsl = Observed::ConfigDSL.new(builder: config_builder, logger: logger)
-                     config_dsl.eval_file(path)
-                     config_dsl.config
+                     sys.config
                    elsif args[:config]
                      c = args[:config]
                      c
@@ -103,12 +86,6 @@ module Observed
         end
       end
 
-      private
-
-      def system
-        @system
-      end
-
     end
   end
 end

data/lib/observed/builtin_plugins/file.rb

@@ -1,16 +1,17 @@
 require 'observed/hash/fetcher'
 require 'observed/reporter'
 require 'observed/reporter/regexp_matching'
+require 'observed/reporter/report_formatting'
 
 module Observed
   module BuiltinPlugins
     class File < Observed::Reporter
 
       include Observed::Reporter::RegexpMatching
+      include Observed::Reporter::ReportFormatting
 
       UNDEFINED = Object.new
 
-      attribute :format, default: -> tag, time, data { "#{Time.at(time)} #{tag} #{data}" }
       attribute :path
       attribute :mode, default: :append
 
@@ -18,15 +19,7 @@ module Observed
       # @param [Time] time
       # @param [Hash] data
       def report(tag, time, data)
-        num_params = format.parameters.size
-        formatted_data = case num_params
-                         when 3
-                           format.call(tag, time, data)
-                         when 4
-                           format.call(tag, time, data, Observed::Hash::Fetcher.new(data))
-                         else
-                           fail "Number of parameters for the function for the key :format must be 3 or 4, but was #{num_params}(#{format.parameters}"
-                         end
+        formatted_data = format_report(tag, time, data)
         mode = case self.mode
                when :append, 'a'
                  'a'
@@ -34,16 +27,14 @@ module Observed
                  'w'
                else
                  fail "Unsupported value for the parameter `:mode`: Supported values are :append, :overwrite, " +
-                     "'a', 'w'. The specified value is #{self.mode.inspect}"
+                          "'a', 'w'. The specified value is #{self.mode.inspect}"
                end
         ::File.open(path, mode) do |f|
           f.puts formatted_data
         end
       end
 
-      def self.plugin_name
-        'file'
-      end
+      plugin_name 'file'
     end
   end
 end

data/lib/observed/builtin_plugins/stdout.rb

@@ -2,34 +2,27 @@ require 'observed/hash/fetcher'
 require 'observed/observer'
 require 'observed/reporter'
 require 'observed/reporter/regexp_matching'
+require 'observed/reporter/report_formatting'
 
 module Observed
   module BuiltinPlugins
     class Stdout < Observed::Reporter
 
       include Observed::Reporter::RegexpMatching
+      include Observed::Reporter::ReportFormatting
 
-      attribute :format, default: -> tag, time, data { "#{Time.at(time)} #{tag} #{data}" }
+      attribute :output, default: STDOUT
 
       # @param [String] tag
       # @param [Time] time
       # @param [Hash] data
+      # @param [Object] format_result
       def report(tag, time, data)
-        num_params = format.parameters.size
-        formatted_data = case num_params
-        when 3
-          format.call(tag, time, data)
-        when 4
-          format.call(tag, time, data, Observed::Hash::Fetcher.new(data))
-        else
-          fail "Number of parameters for the function for the key :format must be 3 or 4, but was #{num_params}(#{format.parameters}"
-        end
-        puts formatted_data
+        formatted_data = format_report(tag, time, data)
+        output.puts formatted_data
       end
 
-      def self.plugin_name
-        'stdout'
-      end
+      plugin_name 'stdout'
     end
   end
 end

data/lib/observed/config.rb

@@ -7,10 +7,6 @@ module Observed
 
     include Observed::Configurable
 
-    # !@attribute [rw] writers
-    #  @return [Array<Observed::Writer>]
-    attribute :writers
-
     # !@attribute [rw] readers
     #  @return [Array<Observed::Reader>]
     attribute :readers
@@ -23,5 +19,9 @@ module Observed
     #  @return [Array<Observed::Observer>]
     attribute :observers
 
+    # !@attribuet [rw] translators
+    #  @return [Array<Observed::Translator]
+    attribute :translators
+
   end
 end

data/lib/observed/config_builder.rb

@@ -1,36 +1,65 @@
+require 'logger'
+require 'thread'
+
 require 'observed/config'
 require 'observed/configurable'
 require 'observed/default'
 require 'observed/hash'
-require 'observed/reader'
-require 'observed/writer'
+require 'observed/translator'
+require 'observed/execution_job_factory'
 
 module Observed
 
+  class ProcObserver < Observed::Observer
+    def initialize(&block)
+      @block = block
+    end
+    def observe(data=nil, options=nil)
+      @block.call data, options
+    end
+  end
+
+  class ProcTranslator < Observed::Translator
+    def initialize(&block)
+      @block = block
+    end
+    def translate(tag, time, data)
+      @block.call data, {tag: tag, time: time}
+    end
+  end
+
+  class ProcReporter < Observed::Reporter
+    def initialize(tag_pattern, &block)
+      @tag_pattern = tag_pattern
+      @block = block
+    end
+    def match(tag)
+      tag.match(@tag_pattern) if tag && @tag_pattern
+    end
+    def report(tag, time, data)
+      @block.call data, {tag: tag, time: time}
+    end
+  end
+
   class ConfigBuilder
     include Observed::Configurable
 
+    attribute :logger, default: Logger.new(STDOUT, Logger::DEBUG)
+
     def initialize(args)
-      @writer_plugins = args[:writer_plugins] if args[:writer_plugins]
-      @reader_plugins = args[:reader_plugins] if args[:reader_plugins]
+      @group_mutex = ::Mutex.new
+      @context = args[:context]
       @observer_plugins = args[:observer_plugins] if args[:observer_plugins]
       @reporter_plugins = args[:reporter_plugins] if args[:reporter_plugins]
+      @translator_plugins = args[:translator_plugins] if args[:translator_plugins]
       @system = args[:system] || fail("The key :system must be in #{args}")
-      @logger = args[:logger] || Logger.new(STDOUT, Logger::DEBUG)
+      configure args
     end
 
     def system
       @system
     end
 
-    def writer_plugins
-      @writer_plugins || select_named_plugins_of(Observed::Writer)
-    end
-
-    def reader_plugins
-      @reader_plugins || select_named_plugins_of(Observed::Reader)
-    end
-
     def observer_plugins
       @observer_plugins || select_named_plugins_of(Observed::Observer)
     end
@@ -39,6 +68,10 @@ module Observed
       @reporter_plugins || select_named_plugins_of(Observed::Reporter)
     end
 
+    def translator_plugins
+      @translator_plugins || select_named_plugins_of(Observed::Translator)
+    end
+
     def select_named_plugins_of(klass)
       plugins = {}
       klass.select_named_plugins.each do |plugin|
@@ -49,8 +82,6 @@ module Observed
 
     def build
       Observed::Config.new(
-          writers: writers,
-          readers: readers,
           observers: observers,
           reporters: reporters
       )
@@ -60,31 +91,53 @@ module Observed
     # @param [Hash] args The configuration for each reporter which may or may not contain (1) which reporter plugin to
     # use or which writer plugin to use (in combination with the default reporter plugin) (2) initialization parameters
     # to instantiate the reporter/writer plugin
-    def report(tag_pattern, args)
-      writer = write(args)
-      tag_pattern || fail("Tag pattern missing: #{tag_pattern} where args: #{args}")
-      reporter = if writer
-                   Observed::Default::Reporter.new.configure(tag_pattern: tag_pattern, writer: writer, system: system)
-                 else
+    def report(tag_pattern=nil, args={}, &block)
+      if tag_pattern.is_a? ::Hash
+        args = tag_pattern
+        tag_pattern = nil
+      end
+      reporter = if args[:via] || args[:using]
                    via = args[:via] || args[:using]
                    with = args[:with] || args[:which] || {}
-                   with = ({logger: @logger}).merge(with).merge({tag_pattern: tag_pattern})
+                   with = ({logger: @logger}).merge(with).merge({tag_pattern: tag_pattern, system: system})
                    plugin = reporter_plugins[via] ||
                        fail(RuntimeError, %Q|The reporter plugin named "#{via}" is not found in "#{reporter_plugins}"|)
                    plugin.new(with)
+                 elsif block_given?
+                   Observed::ProcReporter.new tag_pattern, &block
+                 else
+                   fail "Invalid combination of arguments: #{tag_pattern} #{args}"
                  end
-      begin
-        reporter.match('test')
-      rescue => e
-        fail "A mis-configured reporter plugin found: #{reporter}"
-      rescue NotImplementedError => e
-        builtin_methods = Object.methods
-        info = (reporter.methods - builtin_methods).map {|sym| reporter.method(sym) }.map(&:source_location).compact
-        fail "Incomplete reporter plugin found: #{reporter}, defined in: #{info}"
-      end
 
       reporters << reporter
-      reporter
+      report_it = convert_to_job(reporter)
+      if tag_pattern
+        receive(tag_pattern).then(report_it)
+      end
+      report_it
+    end
+
+    class ObserverCompatibilityAdapter < Observed::Observer
+      include Observed::Configurable
+      attribute :observer
+      attribute :system
+      attribute :tag
+
+      def configure(args)
+        super
+        observer.configure(args)
+      end
+
+      def observe(data=nil, options=nil)
+        case observer.method(:observe).parameters.size
+          when 0
+            observer.observe
+          when 1
+            observer.observe data
+          when 2
+            observer.observe data, options
+        end
+      end
     end
 
     # @param [String] tag The tag which is assigned to data which is generated from this observer, and is sent to
@@ -92,67 +145,74 @@ module Observed
     # @param [Hash] args The configuration for each observer which may or may not contain (1) which observer plugin to
     # use or which reader plugin to use (in combination with the default observer plugin) (2) initialization parameters
     # to instantiate the observer/reader plugin
-    def observe(tag, args)
-      reader = read(args)
-      observer = if reader
-                   Observed::Default::Observer.new.configure(tag: tag, reader: reader, system: system)
-                 else
+    def observe(tag=nil, args={}, &block)
+      if tag.is_a? ::Hash
+        args = tag
+        tag = nil
+      end
+      observer = if args[:via] || args[:using]
                    via = args[:via] || args[:using] ||
                        fail(RuntimeError, %Q|Missing observer plugin name for the tag "#{tag}" in "#{args}"|)
                    with = args[:with] || args[:which] || {}
                    plugin = observer_plugins[via] ||
                        fail(RuntimeError, %Q|The observer plugin named "#{via}" is not found in "#{observer_plugins}"|)
-                   plugin.new(({logger: @logger}).merge(with).merge(tag: tag, system: system))
+                   observer = plugin.new(({logger: logger}).merge(with).merge(tag: tag, system: system))
+                   ObserverCompatibilityAdapter.new(
+                     system: system,
+                     observer: observer,
+                     tag: tag
+                   )
+                 elsif block_given?
+                   Observed::ProcObserver.new &block
+                 else
+                   fail "No args valid args (in args=#{args}) or a block given"
+                 end
+      observe_that = convert_to_job(observer)
+      result = if tag
+        a = observe_that.then(emit(tag))
+        group tag, (group(tag) + [a])
+        a
+      else
+        observe_that
+      end
+      observers << result
+      result
+    end
+
+    def translate(args={}, &block)
+      translator = if args[:via] || args[:using]
+                     #tag_pattern || fail("Tag pattern missing: #{tag_pattern} where args: #{args}")
+                     via = args[:via] || args[:using]
+                     with = args[:with] || args[:which] || {}
+                     with = ({logger: logger}).merge(with).merge({system: system})
+                     plugin = translator_plugins[via] ||
+                         fail(RuntimeError, %Q|The reporter plugin named "#{via}" is not found in "#{translator_plugins}"|)
+                     plugin.new(with)
+                   else
+                     Observed::ProcTranslator.new &block
                  end
-      observers << observer
-      observer
-    end
-
-    def write(args)
-      to = args[:to]
-      with = args[:with] || args[:which]
-      writer = case to
-               when String
-                 plugin = writer_plugins[to] ||
-                     fail(RuntimeError, %Q|The writer plugin named "#{to}" is not found in "#{writer_plugins}"|)
-                 with = ({logger: @logger}).merge(with)
-                 plugin.new(with)
-               when Observed::Writer
-                 to
-               when nil
-                 nil
-               else
-                 fail "Unexpected type of value for the key :to in: #{args}"
-               end
-      writers << writer if writer
-      writer
-    end
-
-    def read(args)
-      from = args[:from]
-      with = args[:with] || [:which]
-      reader = case from
-               when String
-                 plugin = reader_plugins[from] || fail(RuntimeError, %Q|The reader plugin named "#{from}" is not found in "#{reader_plugins}"|)
-                 with = ({logger: @logger}).merge(with)
-                 plugin.new(with)
-               when Observed::Reader
-                 from
-               when nil
-                 nil
-               else
-                 fail "Unexpected type of value for the key :from in: #{args}"
-               end
-      readers << reader if reader
-      reader
-    end
-
-    def writers
-      @writers ||= []
-    end
-
-    def readers
-      @readers ||= []
+      convert_to_job(translator)
+    end
+
+    def emit(tag)
+      @context.jobbed_event_bus.pipe_to_emit(tag)
+    end
+
+    def receive(pattern)
+      @context.jobbed_event_bus.receive(pattern)
+    end
+
+    # Updates or get the observations belongs to the group named `name`
+    def group(name, observations=nil)
+      @group_mutex.synchronize do
+        @observations ||= {}
+        @observations[name] = observations if observations
+        @observations[name] || []
+      end
+    end
+
+    def run_group(name)
+      @context.job_factory.parallel(group(name))
     end
 
     def reporters
@@ -162,6 +222,13 @@ module Observed
     def observers
       @observers ||= []
     end
+
+    private
+
+    def convert_to_job(underlying)
+      @execution_job_factory ||= @context.execution_job_factory
+      @execution_job_factory.convert_to_job(underlying)
+    end
   end
 
 end

data/lib/observed/config_dsl.rb

@@ -15,16 +15,15 @@ module Observed
 
     include Observed::Configurable
 
-    def_delegators :@builder, :observe, :report, :read, :write
+    def_delegators :@builder, :observe, :translate, :report, :read, :write, :emit, :group, :run_group, :receive
 
     attribute :builder
+    attribute :logger, default: Logger.new(STDOUT)
 
     def initialize(args)
       args[:builder] || fail("The key :builder must exist in #{args}")
       @builder = args[:builder]
 
-      @logger = args[:logger] if args[:logger]
-
       configure(args)
     end
 
@@ -68,10 +67,5 @@ module Observed
       eval_file file
     end
 
-    private
-
-    def logger
-      @logger ||= Logger.new(STDOUT)
-    end
   end
 end

data/lib/observed/configurable.rb

@@ -1,4 +1,7 @@
 module Observed
+  # Indicates that classes included this module to have attributes which are configurable.
+  # `configurable` means that the attributes can be configured via named parameters of
+  # the constructor and the `configure` instance method of the class included this module.
   module Configurable
 
     def initialize(args={})
@@ -14,11 +17,25 @@ module Observed
       self
     end
 
+    # @param [String|Symbol] name
+    def has_attribute_value?(name)
+      !! get_attribute_value(name)
+    end
+
+    # @param [String|Symbol] name
+    # @return [Object] In order of precedence, the value of the instance variable named `"@" + name`,
+    #                  or the value `@attributes[name]`, or the default value for the attribute named `name`
+    def get_attribute_value(name)
+      instance_variable_get("@#{name.to_s}") || @attributes[name] || self.class.defaults[name]
+    end
+
     module ClassMethods
       # @param [String|Symbol] name
       def attribute(name, options={})
-        define_method(name) do
-          instance_variable_get("@#{name.to_s}") || @attributes[name] || self.class.defaults[name] || fail_for_not_configured_parameter(name)
+        unless instance_methods.include? name.intern
+          define_method(name) do
+            get_attribute_value(name) || fail_for_not_configured_parameter(name)
+          end
         end
         default_value =  options && options[:default]
         default name => default_value if default_value
@@ -36,13 +53,54 @@ module Observed
         self.new(args)
       end
 
+      # Inherits the default values stored in @defaults to the sub-class
+      def inherited(klass)
+        super if defined? super
+        klass.default defaults
+      end
+
+    end
+
+    module ModuleMethods
+      # @param [String|Symbol] name
+      def attribute(name, options={})
+        @attributes ||= {}
+        @attributes = @attributes.merge(name => options)
+      end
+
+      def attributes
+        @attributes ||
+          fail(<<EOS
+#{self} includes Observed::Configurable. Though, no attributes are configured for #{self}.
+We don't need to include Observed::Configurable, or it might be a bug?
+EOS
+              )
+      end
+
+      def included(klass)
+        ensure_configurable klass
+
+        attributes.each do |name, options|
+          klass.attribute name, options
+        end
+      end
+
+      def ensure_configurable(klass)
+        unless klass.include? Configurable
+          fail "The class #{klass} must include Observed::Configurable to include #{self}"
+        end
+      end
     end
 
     class NotConfiguredError < RuntimeError; end
 
     class << self
       def included(klass)
-        klass.extend ClassMethods
+        if klass.is_a? Class
+          klass.extend ClassMethods
+        else
+          klass.extend ModuleMethods
+        end
       end
     end