fusuma 1.10.1 → 2.0.0

data/lib/fusuma.rb

@@ -2,9 +2,10 @@
 
 require_relative './fusuma/version'
 require_relative './fusuma/multi_logger'
-require_relative './fusuma/config.rb'
-require_relative './fusuma/environment.rb'
-require_relative './fusuma/plugin/manager.rb'
+require_relative './fusuma/config'
+require_relative './fusuma/environment'
+require_relative './fusuma/device'
+require_relative './fusuma/plugin/manager'
 
 # this is top level module
 module Fusuma
@@ -15,6 +16,8 @@ module Fusuma
         set_trap
         read_options(option)
         instance = new
+        ## NOTE: Uncomment following line to measure performance
+        # instance.run_with_lineprof
         instance.run
       end
 
@@ -47,8 +50,6 @@ module Fusuma
       end
 
       def load_custom_config(config_path = nil)
-        return unless config_path
-
         Config.custom_path = config_path
       end
     end
@@ -63,59 +64,120 @@ module Fusuma
     end
 
     def run
-      # TODO: run with multi thread
-      @inputs.first.run do |event|
-        clear_expired_events
-        filtered = filter(event) || next
-        parsed = parse(filtered) || next
-        buffered = buffer(parsed) || next
-        detected = detect(buffered) || next
-        merged = merge(detected) || next
-        execute(merged)
+      loop { pipeline }
+    end
+
+    def pipeline
+      event = input || return
+      clear_expired_events
+      filtered = filter(event) || return
+      parsed = parse(filtered) || return
+      buffered = buffer(parsed) || return
+      detected = detect(buffered) || return
+      condition, context, event = merge(detected) || return
+      execute(condition, context, event)
+    end
+
+    # For performance monitoring
+    def run_with_lineprof(count: 1000)
+      require 'rblineprof'
+      require 'rblineprof-report'
+
+      profile = lineprof(%r{#{Pathname.new(__FILE__).parent}/.}) do
+        count.times { pipeline }
       end
+      LineProf.report(profile)
+      exit 0
+    end
+
+    # @return [Plugin::Events::Event]
+    def input
+      Plugin::Inputs::Input.select(@inputs)
     end
 
+    # @param [Plugin::Events::Event]
+    # @return [Plugin::Events::Event]
     def filter(event)
       event if @filters.any? { |f| f.filter(event) }
     end
 
+    # @param [Plugin::Events::Event]
+    # @return [Plugin::Events::Event]
     def parse(event)
       @parsers.reduce(event) { |e, p| p.parse(e) if e }
     end
 
+    # @param [Plugin::Events::Event]
+    # @return [Array<Plugin::Buffers::Buffer>]
+    # @return [NilClass]
     def buffer(event)
-      @buffers.any? { |b| b.buffer(event) } && @buffers
+      @buffers.select { |b| b.buffer(event) }
     end
 
     # @param buffers [Array<Buffer>]
     # @return [Array<Event>]
     def detect(buffers)
-      @detectors.each_with_object([]) do |detector, detected|
-        if (event = detector.detect(buffers))
-          detected << event
-        end
+      matched_detectors = @detectors.select do |detector|
+        detector.watch? ||
+          buffers.any? { |b| detector.sources.include?(b.type) }
+      end
+
+      events = matched_detectors.each_with_object([]) do |detector, detected|
+        Array(detector.detect(@buffers)).each { |e| detected << e }
       end
+
+      return if events.empty?
+
+      events
     end
 
-    # @param events [Array<Event>]
-    # @return [Event] a Event merged all records from arguments
+    # @param events [Array<Plugin::Events::Event>]
+    # @return [Plugin::Events::Event] Event merged all records from arguments
     # @return [NilClass] when event is NOT given
     def merge(events)
-      main_events, modifiers = events.partition { |event| event.record.mergable? }
-      return nil unless (main_event = main_events.first)
+      index_events, context_events = events.partition { |event| event.record.type == :index }
+      main_events, modifiers = index_events.partition { |event| event.record.mergable? }
+      request_context = context_events.each_with_object({}) do |e, results|
+        results[e.record.name] = e.record.value
+      end
+      main_events.sort_by! { |e| e.record.trigger_priority }
+
+      condition = nil
+      matched_context = nil
+      event = main_events.find do |main_event|
+        matched_context = Config::Searcher.find_context(request_context) do
+          condition, index_record = Config::Searcher.find_condition do
+            main_event.record.merge(records: modifiers.map(&:record))
+          end
+          main_event if index_record
+        end
+      end
+      return if event.nil?
 
-      main_event.record.merge(records: modifiers.map(&:record))
-      main_event
+      [condition, matched_context, event]
     end
 
-    def execute(event)
+    # @param event [Plugin::Events::Event]
+    def execute(condition, context, event)
       return unless event
 
-      executor = @executors.find do |e|
-        e.executable?(event)
+      # Find executable condition and executor
+      executor = Config::Searcher.with_context(context) do
+        Config::Searcher.with_condition(condition) do
+          @executors.find { |e| e.executable?(event) }
+        end
       end
 
-      executor&.execute(event)
+      return if executor.nil?
+
+      # Check interval and execute
+      Config::Searcher.with_context(context) do
+        Config::Searcher.with_condition(condition) do
+          executor.enough_interval?(event) &&
+            executor.update_interval(event) &&
+            executor.execute(event)
+        end
+      end
     end
 
     def clear_expired_events

data/lib/fusuma/config.rb

@@ -1,7 +1,10 @@
 # frozen_string_literal: true
 
-require_relative './multi_logger.rb'
-require_relative './config/index.rb'
+require_relative './multi_logger'
+require_relative './config/index'
+require_relative './config/searcher'
+require_relative './config/yaml_duplication_checker'
+require_relative './hash_support'
 require 'singleton'
 require 'yaml'
 
@@ -9,11 +12,19 @@ require 'yaml'
 module Fusuma
   # read keymap from yaml file
   class Config
+    class NotFoundError < StandardError; end
+
+    class InvalidFileError < StandardError; end
+
     include Singleton
 
     class << self
-      def search(keys)
-        instance.search(keys)
+      def search(index)
+        instance.search(index)
+      end
+
+      def find_execute_key(index)
+        instance.find_execute_key(index)
       end
 
       def custom_path=(new_path)
@@ -21,14 +32,12 @@ module Fusuma
       end
     end
 
-    attr_reader :keymap
-    attr_reader :custom_path
+    attr_reader :keymap, :custom_path, :searcher
 
     def initialize
+      @searcher = Searcher.new
       @custom_path = nil
-      @cache = nil
       @keymap = nil
-      reload
     end
 
     def custom_path=(new_path)
@@ -37,40 +46,61 @@ module Fusuma
     end
 
     def reload
-      @cache  = nil
-      @keymap = YAML.load_file(file_path).deep_symbolize_keys
-      MultiLogger.info "reload config : #{file_path}"
+      @searcher = Searcher.new
+      path = find_filepath
+      MultiLogger.info "reload config: #{path}"
+      @keymap = validate(path)
       self
     end
 
+    # @return [Hash] If check passes
+    # @raise [InvalidFileError] If check does not pass
+    def validate(path)
+      duplicates = []
+      YAMLDuplicationChecker.check(File.read(path), path) do |ignored, duplicate|
+        MultiLogger.error "#{path}: #{ignored.value} is duplicated"
+        duplicates << duplicate.value
+      end
+      raise InvalidFileError, "Detect duplicate keys #{duplicates}" unless duplicates.empty?
+
+      yamls = YAML.load_stream(File.read(path)).compact
+      yamls.map(&:deep_symbolize_keys)
+    rescue StandardError => e
+      MultiLogger.error e.message
+      raise InvalidFileError, e.message
+    end
+
     # @param index [Index]
+    # @param context [Hash]
     def search(index)
-      cache(index.cache_key) do
-        index.keys.reduce(keymap) do |location, key|
-          if location.is_a?(Hash)
-            begin
-              if key.skippable
-                location.fetch(key.symbol, location)
-              else
-                location.fetch(key.symbol, nil)
-              end
-            end
-          else
-            location
-          end
-        end
-      end
+      @searcher.search_with_cache(index, location: keymap)
+    end
+
+    # @param index [Config::Index]
+    # @return Symbol
+    def find_execute_key(index)
+      @execute_keys ||= Plugin::Executors::Executor.plugins.map do |executor|
+        executor.new.execute_keys
+      end.flatten
+
+      execute_params = search(index)
+      return if execute_params.nil?
+
+      @execute_keys.find { |k| execute_params.keys.include?(k) }
     end
 
     private
 
-    def file_path
+    def find_filepath
       filename = 'fusuma/config.yml'
-      if custom_path && File.exist?(expand_custom_path)
-        expand_custom_path
+      if custom_path
+        return expand_custom_path if File.exist?(expand_custom_path)
+
+        raise NotFoundError, "#{expand_custom_path} is NOT FOUND"
       elsif File.exist?(expand_config_path(filename))
         expand_config_path(filename)
       else
+        MultiLogger.warn "config file: #{expand_config_path(filename)} is NOT FOUND"
         expand_default_path(filename)
       end
     end
@@ -86,36 +116,5 @@ module Fusuma
     def expand_default_path(filename)
       File.expand_path "../../#{filename}", __FILE__
     end
-
-    def cache(key)
-      @cache ||= {}
-      key = key.join(',') if key.is_a? Array
-      if @cache.key?(key)
-        @cache[key]
-      else
-        @cache[key] = block_given? ? yield : nil
-      end
-    end
-  end
-end
-
-# activesupport-4.1.1/lib/active_support/core_ext/hash/keys.rb
-class Hash
-  def deep_symbolize_keys
-    deep_transform_keys do |key|
-      begin
-        key.to_sym
-      rescue StandardError
-        key
-      end
-    end
-  end
-
-  def deep_transform_keys(&block)
-    result = {}
-    each do |key, value|
-      result[yield(key)] = value.is_a?(Hash) ? value.deep_transform_keys(&block) : value
-    end
-    result
   end
 end

data/lib/fusuma/config/index.rb

@@ -19,6 +19,11 @@ module Fusuma
                   [Key.new(keys)]
                 end
       end
+
+      def inspect
+        @keys.map(&:inspect)
+      end
+
       attr_reader :keys
 
       def cache_key
@@ -32,17 +37,45 @@ module Fusuma
         end
       end
 
+      # @return [Index]
+      def with_context
+        keys = @keys.map do |key|
+          next if Searcher.skip? && key.skippable
+
+          if Searcher.fallback? && key.fallback
+            key.fallback
+          else
+            key
+          end
+        end
+        self.class.new(keys.compact)
+      end
+
       # Keys in Index
       class Key
-        def initialize(symbol_word, skippable: false)
+        def initialize(symbol_word, skippable: false, fallback: nil)
           @symbol = begin
-                      symbol_word.to_sym
-                    rescue StandardError
-                      symbol_word
-                    end
+            symbol_word.to_sym
+          rescue StandardError
+            symbol_word
+          end
+
           @skippable = skippable
+
+          @fallback = begin
+            fallback.to_sym
+          rescue StandardError
+            fallback
+          end
         end
-        attr_reader :symbol, :skippable
+
+        def inspect
+          skip_marker = @skippable && Searcher.skip? ? '(skip)' : ''
+          fallback_marker = @fallback && Searcher.fallback? ? '(fallback)' : ''
+          "#{@symbol}#{skip_marker}#{fallback_marker}"
+        end
+
+        attr_reader :symbol, :skippable, :fallback
       end
     end
   end

data/lib/fusuma/config/searcher.rb

@@ -0,0 +1,166 @@
+# frozen_string_literal: true
+
+# Index for searching value from config.yml
+module Fusuma
+  class Config
+    # Search config.yml
+    class Searcher
+      def initialize
+        @cache = nil
+      end
+
+      # @param index [Index]
+      # @param location [Hash]
+      # @return [NilClass]
+      # @return [Hash]
+      # @return [Object]
+      def search(index, location:)
+        key = index.keys.first
+        return location if key.nil?
+
+        return nil if location.nil?
+
+        return nil unless location.is_a?(Hash)
+
+        next_index = Index.new(index.keys[1..-1])
+
+        value = nil
+        next_location_cadidates(location, key).find do |next_location|
+          value = search(next_index, location: next_location)
+        end
+        value
+      end
+
+      def search_with_context(index, location:, context:)
+        return nil if location.nil?
+
+        return search(index, location: location[0]) if context == {}
+
+        new_location = location.find do |conf|
+          search(index, location: conf) if conf[:context] == context
+        end
+        search(index, location: new_location)
+      end
+
+      # @param index [Index]
+      # @param location [Hash]
+      # @return [NilClass]
+      # @return [Hash]
+      # @return [Object]
+      def search_with_cache(index, location:)
+        cache([index.cache_key, Searcher.context, Searcher.skip?, Searcher.fallback?]) do
+          search_with_context(index, location: location, context: Searcher.context)
+        end
+      end
+
+      def cache(key)
+        @cache ||= {}
+        key = key.join(',') if key.is_a? Array
+        if @cache.key?(key)
+          @cache[key]
+        else
+          @cache[key] = block_given? ? yield : nil
+        end
+      end
+
+      private
+
+      # next locations' candidates sorted by priority
+      #  1. look up location with key
+      #  2. fallback to other key
+      #  3. skip the key and go to child location
+      def next_location_cadidates(location, key)
+        [
+          location[key.symbol],
+          Searcher.skip? && key.skippable && location
+        ].compact
+      end
+
+      class << self
+        # @return [Hash]
+        def conditions(&block)
+          {
+            nothing: -> { block.call },
+            skip: -> { Config::Searcher.skip { block.call } }
+          }
+        end
+
+        # Execute block with specified conditions
+        # @param conidtion [Symbol]
+        # @return [Object]
+        def with_condition(condition, &block)
+          conditions(&block)[condition].call
+        end
+
+        # Execute block with all conditions
+        # @return [Array<Symbol, Object>]
+        def find_condition(&block)
+          conditions(&block).find do |c, l|
+            result = l.call
+            return [c, result] if result
+
+            nil
+          end
+        end
+
+        # Search with context from load_streamed Config
+        # @param context [Hash]
+        # @return [Object]
+        def with_context(context, &block)
+          @context = context || {}
+          result = block.call
+          @context = {}
+          result
+        end
+
+        # Return a matching context from config
+        # @params request_context [Hash]
+        # @return [Hash]
+        def find_context(request_context, &block)
+          # Search in blocks in the following order.
+          # 1. complete match config[:context] == request_context
+          # 2. partial match config[:context] =~ request_context
+          # 3. no context
+          Config.instance.keymap.each do |config|
+            next unless config[:context] == request_context
+            return config[:context] if with_context(config[:context]) { block.call }
+          end
+          if request_context.keys.size > 1
+            Config.instance.keymap.each do |config|
+              next if config[:context].nil?
+
+              next unless config[:context].all? { |k, v| request_context[k] == v }
+              return config[:context] if with_context(config[:context]) { block.call }
+            end
+          end
+          return {} if with_context({}) { block.call }
+        end
+
+        attr_reader :context
+
+        def fallback?
+          @fallback
+        end
+
+        def skip?
+          @skip
+        end
+
+        # switch context for fallback
+        def fallback(&block)
+          @fallback = true
+          result = block.call
+          @fallback = false
+          result
+        end
+
+        def skip(&block)
+          @skip = true
+          result = block.call
+          @skip = false
+          result
+        end
+      end
+    end
+  end
+end