dh_easy-core 0.2.2

data/lib/dh_easy/core/plugin.rb

@@ -0,0 +1,19 @@
+require 'dh_easy/core/plugin/initialize_hook'
+require 'dh_easy/core/plugin/context_integrator'
+require 'dh_easy/core/plugin/collection_vault'
+require 'dh_easy/core/plugin/config_behavior'
+require 'dh_easy/core/plugin/executor_behavior'
+require 'dh_easy/core/plugin/executor'
+require 'dh_easy/core/plugin/parser_behavior'
+require 'dh_easy/core/plugin/parser'
+require 'dh_easy/core/plugin/seeder_behavior'
+require 'dh_easy/core/plugin/seeder'
+require 'dh_easy/core/plugin/finisher_behavior'
+require 'dh_easy/core/plugin/finisher'
+
+module DhEasy
+  module Core
+    module Plugin
+    end
+  end
+end

data/lib/dh_easy/core/plugin/collection_vault.rb

@@ -0,0 +1,23 @@
+module DhEasy
+  module Core
+    module Plugin
+      module CollectionVault
+        # Stored collections info as hash.
+        def collections
+          @collections ||= {}
+        end
+
+        # Add a new collection
+        #
+        # @param [Symbol] key Collection key used to lookup for collection name.
+        # @param [String] name Collection name used on outputs.
+        def add_collection key, name
+          if collections.has_key? key
+            raise "Can't add \"#{key}\" collection, it already exists!"
+          end
+          collections[key] = name
+        end
+      end
+    end
+  end
+end

data/lib/dh_easy/core/plugin/config_behavior.rb

@@ -0,0 +1,43 @@
+module DhEasy
+  module Core
+    module Plugin
+      module ConfigBehavior
+        include DhEasy::Core::Plugin::ContextIntegrator
+        include DhEasy::Core::Plugin::CollectionVault
+
+        attr_reader :config_collection_key
+
+        # Hook to map config behavior on self
+        #
+        # @param [Hash] opts ({}) Configuration options.
+        # @option opts [Array] :config_collection ([:config, 'config']) Key value pair array to se a custom collection.
+        #
+        # @example
+        #   initialize_hook_core_config_behavior config_collection: [:my_config, 'abc']
+        #   config_collection
+        #   # => 'abc'
+        def initialize_hook_core_config_behavior opts = {}
+          @config_collection_key, collection = opts[:config_collection] || [:config, 'config']
+          add_collection config_collection_key, collection
+        end
+
+        # Get config collection name.
+        # @return [String]
+        def config_collection
+          collections[config_collection_key]
+        end
+
+        # Find a configuration value by item key.
+        #
+        # @param [Symbol] key Item key to find.
+        #
+        # @note Instance must implement:
+        #   * `find_output(collection, query)`
+        def find_config key
+          value = find_output config_collection, '_id' => key
+          value ||= {'_collection' => config_collection, '_id' => key}
+        end
+      end
+    end
+  end
+end

data/lib/dh_easy/core/plugin/context_integrator.rb

@@ -0,0 +1,60 @@
+module DhEasy
+  module Core
+    module Plugin
+      module ContextIntegrator
+        # Last mocked ontext object.
+        attr_reader :context
+
+        # Mock a context methods into self.
+        #
+        # @param origin Object that represents the context to mock.
+        #
+        # @example
+        #   class MyContext
+        #     attr_accessor :message
+        #     def initialize
+        #       message = 'Hello world!'
+        #     end
+        #
+        #     def hello_world
+        #       message
+        #     end
+        #   end
+        #
+        #   class Foo
+        #     include ContextIntegrator
+        #
+        #     def hello_person
+        #       'Hello person!'
+        #     end
+        #   end
+        #
+        #   context = MyContext.new
+        #   my_object = Foo.new
+        #   my_object.mock_context context
+        #
+        #   puts my_object.hello_world
+        #   # => 'Hello world!'
+        #   puts my_object.hello_person
+        #   # => 'Hello person!'
+        #
+        #   context.message = 'Hello world again!'
+        #   puts my_object.hello_world
+        #   # => 'Hello world again!
+        def mock_context origin
+          @context = origin
+          DhEasy::Core.mock_instance_methods context, self
+        end
+
+        # Hook to mock context on initialize.
+        #
+        # @param [Hash] opts ({}) Configuration options.
+        # @option opts :context Object that represents the context to mock.
+        def initialize_hook_core_context_integrator opts = [{}]
+          raise ':context object is required.' if opts[:context].nil?
+          mock_context opts[:context]
+        end
+      end
+    end
+  end
+end

data/lib/dh_easy/core/plugin/executor.rb

@@ -0,0 +1,19 @@
+module DhEasy
+  module Core
+    module Plugin
+      module Executor
+        include DhEasy::Core::Plugin::InitializeHook
+        include DhEasy::Core::Plugin::ExecutorBehavior
+
+        # Initialize hooks.
+        #
+        # @param [Hash] opts ({}) Configuration options.
+        #
+        # @see DhEasy::Core::Plugin::ContextIntegrator#initialize_hook_core_context_integrator
+        def initialize opts = {}
+          initialize_hooks opts
+        end
+      end
+    end
+  end
+end

data/lib/dh_easy/core/plugin/executor_behavior.rb

@@ -0,0 +1,32 @@
+module DhEasy
+  module Core
+    module Plugin
+      module ExecutorBehavior
+        include DhEasy::Core::Plugin::ContextIntegrator
+
+        # Enqueue a single/multiple pages for fetch. Analog to `save_pages`.
+        #
+        # @param [Array,Hash] object Pages to save being Hash when single and
+        #   Array when many.
+        #
+        # @note Instance must implement:
+        #   * `save_pages(pages)`
+        def enqueue object
+          object = [object] unless object.is_a? Array
+          save_pages object
+        end
+
+        # Save a single/multiple outputs. Analog to `save_outputs`.
+        #
+        # @param [Array,Hash] object Outputs to save being Hash when single and Array when many.
+        #
+        # @note Instance must implement:
+        #   * `save_outputs(outputs)`
+        def save object
+          object = [object] unless object.is_a? Array
+          save_outputs object
+        end
+      end
+    end
+  end
+end

data/lib/dh_easy/core/plugin/finisher.rb

@@ -0,0 +1,19 @@
+module DhEasy
+  module Core
+    module Plugin
+      module Finisher
+        include DhEasy::Core::Plugin::InitializeHook
+        include DhEasy::Core::Plugin::FinisherBehavior
+
+        # Initialize finisher and hooks.
+        #
+        # @param [Hash] opts ({}) Configuration options.
+        #
+        # @see DhEasy::Core::Plugin::ContextIntegrator#initialize_hook_core_context_integrator
+        def initialize opts = {}
+          initialize_hooks opts
+        end
+      end
+    end
+  end
+end

data/lib/dh_easy/core/plugin/finisher_behavior.rb

@@ -0,0 +1,9 @@
+module DhEasy
+  module Core
+    module Plugin
+      module FinisherBehavior
+        include DhEasy::Core::Plugin::ExecutorBehavior
+      end
+    end
+  end
+end

data/lib/dh_easy/core/plugin/initialize_hook.rb

@@ -0,0 +1,17 @@
+module DhEasy
+  module Core
+    module Plugin
+      module InitializeHook
+        # Execute all methods with `initilaize_hook_` prefix (hooks).
+        #
+        # @param [Hash] opts ({}) Configuration options sent to all hooks.
+        def initialize_hooks opts = {}
+          initializers = self.methods.select{|i|i.to_s =~ /^initialize_hook_/}
+          initializers.each do |method|
+            self.send method, opts
+          end
+        end
+      end
+    end
+  end
+end

data/lib/dh_easy/core/plugin/parser.rb

@@ -0,0 +1,19 @@
+module DhEasy
+  module Core
+    module Plugin
+      module Parser
+        include DhEasy::Core::Plugin::InitializeHook
+        include DhEasy::Core::Plugin::ParserBehavior
+
+        # Initialize parser and hooks.
+        #
+        # @param [Hash] opts ({}) Configuration options.
+        #
+        # @see DhEasy::Core::Plugin::ContextIntegrator#initialize_hook_core_context_integrator
+        def initialize opts = {}
+          initialize_hooks opts
+        end
+      end
+    end
+  end
+end

data/lib/dh_easy/core/plugin/parser_behavior.rb

@@ -0,0 +1,17 @@
+module DhEasy
+  module Core
+    module Plugin
+      module ParserBehavior
+        include DhEasy::Core::Plugin::ExecutorBehavior
+
+        # Alias to `page['vars']`.
+        #
+        # @note Instance must implement:
+        #   * `page`
+        def vars
+          page['vars']
+        end
+      end
+    end
+  end
+end

data/lib/dh_easy/core/plugin/seeder.rb

@@ -0,0 +1,44 @@
+module DhEasy
+  module Core
+    module Plugin
+      module Seeder
+        include DhEasy::Core::Plugin::InitializeHook
+        include DhEasy::Core::Plugin::SeederBehavior
+
+        # Root input directory path.
+        # @return [String]
+        attr_accessor :root_input_dir
+
+        # Referer to use on page seeding.
+        # @return [String]
+        attr_accessor :referer
+
+        # Cookie to use on page seeing.
+        # @return [String]
+        attr_accessor :cookie
+
+        # Hook to initialize seeder object.
+        #
+        # @param [Hash] opts ({}) Configuration options.
+        # @option opts [String] :root_input_dir (nil) Root directory for inputs.
+        # @option opts [String] :referer (nil) New pages referer, useful to dynamic setups.
+        # @option opts [String] :cookie (nil) Cookie to use on seeded pages fetchs.
+        def initialize_hook_core_seeder opts = {}
+          @root_input_dir = opts[:root_input_dir]
+          @referer = opts[:referer]
+          @cookie = opts[:cookie]
+        end
+
+        # Initialize seeder and hooks.
+        #
+        # @param [Hash] opts ({}) Configuration options.
+        #
+        # @see DhEasy::Core::Plugin::ContextIntegrator#initialize_hook_core_context_integrator
+        # @see #initialize_hook_core_seeder
+        def initialize opts = {}
+          initialize_hooks opts
+        end
+      end
+    end
+  end
+end

data/lib/dh_easy/core/plugin/seeder_behavior.rb

@@ -0,0 +1,9 @@
+module DhEasy
+  module Core
+    module Plugin
+      module SeederBehavior
+        include DhEasy::Core::Plugin::ExecutorBehavior
+      end
+    end
+  end
+end

data/lib/dh_easy/core/smart_collection.rb

@@ -0,0 +1,236 @@
+module DhEasy
+  module Core
+    # Smart collection capable to avoid duplicates on insert by matching id
+    #   defined fields along events.
+    class SmartCollection < Array
+      # Implemented event list.
+      EVENTS = [
+        :before_defaults,
+        :before_match,
+        :before_insert,
+        :after_insert
+      ]
+
+      # Key fields, analog to primary keys.
+      attr_reader :key_fields
+      # Default fields values. Apply to missing fields and null values.
+      attr_reader :defaults
+
+      # Initialize collection
+      #
+      # @param [Array] key_fields Key fields, analog to primary keys.
+      # @param [Hash] opts ({}) Configuration options.
+      # @option opts [Array] :values ([]) Initial values; will avoid duplicates on insert.
+      # @option opts [Hash] :defaults ({}) Default values. `proc` values will be executed to get default value.
+      #
+      # @example With default values.
+      #   count = 0
+      #   defaults = {
+      #     'id' => lambda{|item| count += 1},
+      #     'aaa' => 111,
+      #     'bbb' => proc{|item| item['ccc'].nil? ? 'No ccc' : 'Has ccc'}
+      #   }
+      #   values = [
+      #     {'aaa' => 'Defaults apply on nil values only', 'bbb' => nil},
+      #     {'ccc' => 'ddd'},
+      #     {'id' => 'abc123'}
+      #   ]
+      #   new_item = {'bbb' => 'Look mom! no ccc'}
+      #   collection = SmartCollection.new ['id'], defaults: defaults
+      #   collection << new_item
+      #   collection
+      #   # => [
+      #   #   {'id' => 1, 'aaa' => 'Defaults apply on nil values only', 'bbb' => 'No ccc'},
+      #   #   {'id' => 2, 'aaa' => 111, 'bbb' => 'Has ccc', 'ccc' => 'ddd'},
+      #   #   {'id' => 'abc123', 'aaa' => 111, 'bbb' => 'No ccc'},
+      #   #   {'id' => 3, 'aaa' => 111, 'bbb' => 'Look mom! no ccc'}
+      #   # ]
+      #
+      # @note Defaults will apply to missing fields and null values only.
+      def initialize key_fields, opts = {}
+        @key_fields = key_fields || []
+        @defaults = opts[:defaults] || {}
+        super 0
+        (opts[:values] || []).each{|item| self << item}
+      end
+
+      # Asigned events.
+      # @private
+      def events
+        @events ||= {}
+      end
+
+      # Add event binding by key and block.
+      #
+      # @param [Symbol] key Event name.
+      #
+      # @raise [ArgumentError] When unknown event key.
+      #
+      # @example before_defaults
+      #   defaults = {'aaa' => 111}
+      #   collection = SmartCollection.new [],
+      #     defaults: defaults
+      #   collection.bind_event(:before_defaults) do |collection, item|
+      #     puts collection
+      #     # => []
+      #     puts item
+      #     # => {'bbb' => 222}
+      #
+      #     # Sending the item back is required, or a new one
+      #     #   in case you want to replace item to insert.
+      #     item
+      #   end
+      #   data << {'bbb' => 222}
+      #   data
+      #   # => [{'aaa' => 111, 'bbb' => 222}]
+      #
+      # @example before_match
+      #   keys = ['id']
+      #   defaults = {'aaa' => 111}
+      #   values = [
+      #     {'id' => 1, 'ccc' => 333}
+      #   ]
+      #   collection = SmartCollection.new keys,
+      #     defaults: defaults
+      #     values: values
+      #   collection.bind_event(:before_match) do |collection, item|
+      #     puts collection
+      #     # => [{'id' => 1, 'aaa' => 111, 'ccc' => 333}]
+      #     puts item
+      #     # => {'id' => 1, 'aaa' => 111, 'bbb' => 222}
+      #
+      #     # Sending the item back is required, or a new one
+      #     #   in case you want to replace item to insert.
+      #     item
+      #   end
+      #   data << {'id' => 1, 'bbb' => 222}
+      #   data
+      #   # => [{'id' => 1, 'aaa' => 111, 'bbb' => 222}]
+      #
+      # @example before_insert
+      #   keys = ['id']
+      #   defaults = {'aaa' => 111}
+      #   values = [
+      #     {'id' => 1, 'ccc' => 333}
+      #   ]
+      #   collection = SmartCollection.new keys,
+      #     defaults: defaults
+      #     values: values
+      #   collection.bind_event(:before_insert) do |collection, item, match|
+      #     puts collection
+      #     # => [{'id' => 1, 'aaa' => 111, 'ccc' => 333}]
+      #     puts item
+      #     # => {'id' => 1, 'aaa' => 111, 'bbb' => 222}
+      #     puts match
+      #     # => {'id' => 1, 'aaa' => 111, 'ccc' => 333}
+      #
+      #     # Sending the item back is required, or a new one
+      #     #   in case you want to replace item to insert.
+      #     item
+      #   end
+      #   data << {'id' => 1, 'bbb' => 222}
+      #   data
+      #   # => [{'id' => 1, 'aaa' => 111, 'bbb' => 222}]
+      #
+      # @example after_insert
+      #   keys = ['id']
+      #   defaults = {'aaa' => 111}
+      #   values = [
+      #     {'id' => 1, 'ccc' => 333}
+      #   ]
+      #   collection = SmartCollection.new keys,
+      #     defaults: defaults
+      #     values: values
+      #   collection.bind_event(:after_insert) do |collection, item, match|
+      #     puts collection
+      #     # => [{'id' => 1, 'aaa' => 111, 'bbb' => 222}]
+      #     puts item
+      #     # => {'id' => 1, 'aaa' => 111, 'bbb' => 222}
+      #     puts match
+      #     # => {'id' => 1, 'aaa' => 111, 'ccc' => 333}
+      #     # No need to send item back since it is already inserted
+      #   end
+      #   data << {'id' => 1, 'bbb' => 222}
+      #   data
+      #   # => [{'id' => 1, 'aaa' => 111, 'bbb' => 222}]
+      #
+      # @note Some events will expect a return value to replace item on insertion:
+      #   * `before_match`
+      #   * `before_defaults`
+      #   * `before_insert`
+      def bind_event key, &block
+        unless EVENTS.include? key
+          raise ArgumentError.new("Unknown event '#{key}'")
+        end
+        (events[key] ||= []) << block
+      end
+
+      # Call an event
+      # @private
+      #
+      # @param [Symbol] key Event name.
+      # @param default Detault return value when event's return nil.
+      # @param args event arguments.
+      def call_event key, default = nil, *args
+        return default if events[key].nil?
+        result = nil
+        events[key].each{|event| result = event.call self, *args}
+        result.nil? ? default : result
+      end
+
+      # Check whenever two items keys match.
+      #
+      # @param [Hash] item_a Item to match.
+      # @param [Hash] item_b Item to match.
+      #
+      # @return [Boolean]
+      def match_keys? item_a, item_b
+        return false if key_fields.nil? || key_fields.count < 1
+        return true if item_a.nil? && item_b.nil?
+        return false if item_a.nil? || item_b.nil?
+        key_fields.each do |key|
+          return false if item_a[key] != item_b[key]
+        end
+        true
+      end
+
+      # Apply default values into item.
+      # @private
+      #
+      # @param [Hash] item Item to apply defaults.
+      #
+      # @return [Hash] Item
+      def apply_defaults item
+        defaults.each do |key, value|
+          next unless item[key].nil?
+          item[key] = value.respond_to?(:call) ? value.call(item) : value
+        end
+      end
+
+      # Find an item by matching filter keys
+      #
+      # @param [Hash] filter
+      #
+      # @return [Hash,nil] First existing item match or nil when no match.
+      #
+      # @note _Warning:_ It uses table scan to filter and will be slow.
+      def find_match filter
+        self.find do |item|
+          match_keys? item, filter
+        end
+      end
+
+      # Add/remplace an item avoiding duplicates
+      def << item
+        item = call_event :before_defaults, item, item
+        apply_defaults item
+        item = call_event :before_match, item, item
+        match = find_match item
+        item = call_event :before_insert, item, item, match
+        delete match unless match.nil?
+        result = super(item)
+        call_event :after_insert, result, item, match
+      end
+    end
+  end
+end