RubyGems - ae_easy-core - Versions diffs - 0.2.0 → 0.2.1 - Mend

ae_easy-core 0.2.0 → 0.2.1

Files changed (76) hide show

checksums.yaml +4 -4
data/CODE_OF_CONDUCT.md +1 -1
data/Gemfile +1 -1
data/LICENSE +1 -1
data/README.md +8 -4
data/Rakefile +0 -10
data/ae_easy-core.gemspec +6 -13
data/lib/ae_easy/core.rb +4 -256
metadata +18 -125
data/doc/AeEasy.html +0 -117
data/doc/AeEasy/Core.html +0 -1590
data/doc/AeEasy/Core/Config.html +0 -311
data/doc/AeEasy/Core/Exception.html +0 -117
data/doc/AeEasy/Core/Exception/OutdatedError.html +0 -135
data/doc/AeEasy/Core/Helper.html +0 -117
data/doc/AeEasy/Core/Helper/Cookie.html +0 -1070
data/doc/AeEasy/Core/Mock.html +0 -282
data/doc/AeEasy/Core/Mock/FakeDb.html +0 -3779
data/doc/AeEasy/Core/Mock/FakeExecutor.html +0 -3289
data/doc/AeEasy/Core/Mock/FakeFinisher.html +0 -160
data/doc/AeEasy/Core/Mock/FakeParser.html +0 -160
data/doc/AeEasy/Core/Mock/FakeSeeder.html +0 -160
data/doc/AeEasy/Core/Plugin.html +0 -117
data/doc/AeEasy/Core/Plugin/CollectionVault.html +0 -299
data/doc/AeEasy/Core/Plugin/ConfigBehavior.html +0 -541
data/doc/AeEasy/Core/Plugin/ContextIntegrator.html +0 -445
data/doc/AeEasy/Core/Plugin/Executor.html +0 -259
data/doc/AeEasy/Core/Plugin/ExecutorBehavior.html +0 -344
data/doc/AeEasy/Core/Plugin/Finisher.html +0 -265
data/doc/AeEasy/Core/Plugin/FinisherBehavior.html +0 -142
data/doc/AeEasy/Core/Plugin/InitializeHook.html +0 -220
data/doc/AeEasy/Core/Plugin/Parser.html +0 -270
data/doc/AeEasy/Core/Plugin/ParserBehavior.html +0 -235
data/doc/AeEasy/Core/Plugin/Seeder.html +0 -674
data/doc/AeEasy/Core/Plugin/SeederBehavior.html +0 -142
data/doc/AeEasy/Core/SmartCollection.html +0 -1087
data/doc/_index.html +0 -364
data/doc/class_list.html +0 -51
data/doc/css/common.css +0 -1
data/doc/css/full_list.css +0 -58
data/doc/css/style.css +0 -496
data/doc/file.README.html +0 -91
data/doc/file_list.html +0 -56
data/doc/frames.html +0 -17
data/doc/index.html +0 -91
data/doc/js/app.js +0 -303
data/doc/js/full_list.js +0 -216
data/doc/js/jquery.js +0 -4
data/doc/method_list.html +0 -939
data/doc/top-level-namespace.html +0 -110
data/lib/ae_easy/core/config.rb +0 -27
data/lib/ae_easy/core/exception.rb +0 -8
data/lib/ae_easy/core/exception/outdated_error.rb +0 -9
data/lib/ae_easy/core/helper.rb +0 -8
data/lib/ae_easy/core/helper/cookie.rb +0 -209
data/lib/ae_easy/core/mock.rb +0 -45
data/lib/ae_easy/core/mock/fake_db.rb +0 -561
data/lib/ae_easy/core/mock/fake_executor.rb +0 -373
data/lib/ae_easy/core/mock/fake_finisher.rb +0 -28
data/lib/ae_easy/core/mock/fake_parser.rb +0 -33
data/lib/ae_easy/core/mock/fake_seeder.rb +0 -28
data/lib/ae_easy/core/plugin.rb +0 -19
data/lib/ae_easy/core/plugin/collection_vault.rb +0 -23
data/lib/ae_easy/core/plugin/config_behavior.rb +0 -43
data/lib/ae_easy/core/plugin/context_integrator.rb +0 -60
data/lib/ae_easy/core/plugin/executor.rb +0 -19
data/lib/ae_easy/core/plugin/executor_behavior.rb +0 -32
data/lib/ae_easy/core/plugin/finisher.rb +0 -19
data/lib/ae_easy/core/plugin/finisher_behavior.rb +0 -9
data/lib/ae_easy/core/plugin/initialize_hook.rb +0 -17
data/lib/ae_easy/core/plugin/parser.rb +0 -19
data/lib/ae_easy/core/plugin/parser_behavior.rb +0 -17
data/lib/ae_easy/core/plugin/seeder.rb +0 -44
data/lib/ae_easy/core/plugin/seeder_behavior.rb +0 -9
data/lib/ae_easy/core/smart_collection.rb +0 -236
data/lib/ae_easy/core/version.rb +0 -6

data/doc/top-level-namespace.html DELETED

@@ -1,110 +0,0 @@
-<!DOCTYPE html>
-<html>
-  <head>
-    <meta charset="utf-8">
-<meta name="viewport" content="width=device-width, initial-scale=1.0">
-<title>
-  Top Level Namespace
-    &mdash; Documentation by YARD 0.9.20
-</title>
-  <link rel="stylesheet" href="css/style.css" type="text/css" charset="utf-8" />
-  <link rel="stylesheet" href="css/common.css" type="text/css" charset="utf-8" />
-<script type="text/javascript" charset="utf-8">
-  pathId = "";
-  relpath = '';
-</script>
-  <script type="text/javascript" charset="utf-8" src="js/jquery.js"></script>
-  <script type="text/javascript" charset="utf-8" src="js/app.js"></script>
-  </head>
-  <body>
-    <div class="nav_wrap">
-      <iframe id="nav" src="class_list.html?1"></iframe>
-      <div id="resizer"></div>
-    </div>
-    <div id="main" tabindex="-1">
-      <div id="header">
-        <div id="menu">
-    <a href="_index.html">Index</a> &raquo;
-    <span class="title">Top Level Namespace</span>
-</div>
-        <div id="search">
-    <a class="full_list_link" id="class_list_link"
-        href="class_list.html">
-        <svg width="24" height="24">
-          <rect x="0" y="4" width="24" height="4" rx="1" ry="1"></rect>
-          <rect x="0" y="12" width="24" height="4" rx="1" ry="1"></rect>
-          <rect x="0" y="20" width="24" height="4" rx="1" ry="1"></rect>
-        </svg>
-    </a>
-</div>
-        <div class="clear"></div>
-      </div>
-      <div id="content"><h1>Top Level Namespace
-</h1>
-<div class="box_info">
-</div>
-<h2>Defined Under Namespace</h2>
-<p class="children">
-      <strong class="modules">Modules:</strong> <span class='object_link'><a href="AeEasy.html" title="AeEasy (module)">AeEasy</a></span>
-</p>
-</div>
-      <div id="footer">
-  Generated on Fri Sep 27 02:01:30 2019 by
-  <a href="http://yardoc.org" title="Yay! A Ruby Documentation Tool" target="_parent">yard</a>
-  0.9.20 (ruby-2.5.3).
-</div>
-    </div>
-  </body>
-</html>

data/lib/ae_easy/core/config.rb DELETED

@@ -1,27 +0,0 @@
-module AeEasy
-  module Core
-    # Configuration manager tool useful for global configuration data accross
-    #   the scraping process.
-    class Config
-      include AeEasy::Core::Plugin::InitializeHook
-      include AeEasy::Core::Plugin::ConfigBehavior
-      # {AeEasy::Core::Plugin::ConfigBehavior#config_collection_key}
-      alias :collection_key :config_collection_key
-      # {AeEasy::Core::Plugin::ConfigBehavior#config_collection}
-      alias :collection :config_collection
-      # Initialize config object
-      #
-      # @param [Hash] opts ({}) Configuration options.
-      #
-      # @see AeEasy::Core::Plugin::ConfigBehavior#initialize_hook_core_config_behavior
-      def initialize opts = {}
-        opts = opts.merge(
-          config_collection: opts[:collection]
-        )
-        initialize_hooks opts
-      end
-    end
-  end
-end

data/lib/ae_easy/core/exception.rb DELETED

@@ -1,8 +0,0 @@
-require 'ae_easy/core/exception/outdated_error'
-module AeEasy
-  module Core
-    module Exception
-    end
-  end
-end

data/lib/ae_easy/core/exception/outdated_error.rb DELETED

@@ -1,9 +0,0 @@
-module AeEasy
-  module Core
-    module Exception
-      # Exception that indicates something is outdated error.
-      class OutdatedError < StandardError
-      end
-    end
-  end
-end

data/lib/ae_easy/core/helper.rb DELETED

@@ -1,8 +0,0 @@
-require 'ae_easy/core/helper/cookie'
-module AeEasy
-  module Core
-    module Helper
-    end
-  end
-end

data/lib/ae_easy/core/helper/cookie.rb DELETED

@@ -1,209 +0,0 @@
-module AeEasy
-  module Core
-    module Helper
-      # Helper used for lower level cookie management.
-      class Cookie
-        class << self
-          # Parse request cookies on different formats.
-          #
-          # @param [String,Hash,Array] cookies Cookies to parse.
-          # @param [Hash] cookie_hash ({}) External hash to store parsed cookies.
-          #
-          # @return [Hash]
-          #
-          # @example Parse from string.
-          #   parse_from_request 'aaa=111; bbb=222'
-          #   # => {'aaa' => 111, 'bbb' => 222}
-          #
-          # @example Parse from array.
-          #   cookies = [
-          #     'aaa=111',
-          #     'bbb=222'
-          #   ]
-          #   parse_from_response cookies
-          #   # => {'aaa' => 111, 'bbb' => 222}
-          #
-          # @example Parse with `cookie_hash`.
-          #   cookie_hash = {'ccc' => 333}
-          #   parse_from_request 'aaa=111; bbb=222', cookie_hash
-          #   cookie_hash
-          #   # => {'aaa' => 1, 'bbb' => 2, 'ccc' => 333}
-          def parse_from_request cookies, cookie_hash = {}
-            # Retrieve from hash
-            if cookies.is_a? Hash
-              cookie_hash.merge! cookies
-              return cookie_hash
-            end
-            # Extract from string
-            cookies = cookies.split '; ' if cookies.is_a? String
-            # Extract from array
-            cookies&.each do |raw_cookie|
-              key, value = raw_cookie.split('=', 2)
-              cookie_hash[key] = value
-            end
-            cookie_hash
-          end
-          # Parse response cookies on different formats.
-          #
-          # @param [String,Hash,Array] cookies Cookies to parse.
-          # @param [Hash] cookie_hash ({}) External hash to store parsed cookies.
-          #
-          # @return [Hash]
-          #
-          # @example Parse from string
-          #   parse_from_response 'aaa=111; bbb=222'
-          #   # => {'aaa' => 111, 'bbb' => 222}
-          #
-          # @example Parse from array.
-          #   cookies = [
-          #     'aaa=111; Expires=Thu, Jan 01 1970 00:00:00 UTC; path=/',
-          #     'bbb=222; path=/',
-          #     'ccc=333; path=/; expires=Wed, Jan 01 3000 00:00:00 UTC'
-          #   ]
-          #   parse_from_response cookies
-          #   # => {'bbb' => 222, 'ccc' => 333}
-          #
-          # @example Parse with `cookie_hash`.
-          #   cookie_hash = {'ccc' => 333}
-          #   parse_from_response 'aaa=111; bbb=222', cookie_hash
-          #   cookie_hash
-          #   # => {'aaa' => 111, 'bbb' => 222, 'ccc' => 333}
-          def parse_from_response cookies, cookie_hash = {}
-            # Retrieve from hash
-            if cookies.is_a? Hash
-              cookie_hash.merge! cookies
-              return cookie_hash
-            end
-            # Retrieve from String
-            cookies = cookies.split '; ' if cookies.is_a? String
-            # Extract from array
-            info = cookie = expires = key = value = nil
-            cookies&.each do |raw_cookie|
-              # Extract cookie data
-              key_pair = raw_cookie.scan(/(?:;\s+([^\=]+)=([^;]*))/i) || []
-              cookie = key_pair.inject(Hash.new){|h,i|h[i[0].to_s.downcase] = i[1]; h}
-              cookie[:key], cookie[:value] = raw_cookie.match(/^\s*(?<key>[^\=]+)\=(?<value>[^;]*)/i)&.captures
-              # Check cookie expire
-              expires = cookie['expires'].nil? ? nil : Time.parse(cookie['expires'])
-              if !expires.nil? && Time.now > expires
-                cookie_hash.delete cookie[:key]
-                next
-              end
-              # Save cookie
-              cookie_hash[cookie[:key]] = cookie[:value]
-            end
-            cookie_hash
-          end
-          # Apply request and response cookies as a hash.
-          #
-          # @param [String,Array,Hash] request_cookies Cookies to parse.
-          # @param [String,Array,Hash] response_cookies Cookies to parse.
-          #
-          # @return [Hash]
-          #
-          # @example
-          #   request_cookies = 'aaa=111; ddd=444'
-          #   response_cookies = [
-          #     'aaa=111; Expires=Thu, Jan 01 1970 00:00:00 UTC; path=/',
-          #     'bbb=222; path=/',
-          #     'ccc=333; path=/; expires=Wed, Jan 01 3000 00:00:00 UTC'
-          #   ]
-          #   update_as_hash , response_cookies
-          #   # => {'bbb' => 222, 'ccc' => 333, 'ddd' => 444}
-          def update_as_hash request_cookies, response_cookies
-            cookie_hash = {}
-            parse_from_request request_cookies, cookie_hash
-            parse_from_response response_cookies, cookie_hash
-            cookie_hash
-          end
-          # Encode cookies as request cookie string.
-          #
-          # @param [Hash] cookie_hash Hash with cookies.
-          #
-          # @return [String]
-          #
-          # @example
-          #   cookie_hash = {
-          #     'aaa' => 111,
-          #     'bbb' => 222
-          #   }
-          #   encode_to_header cookie_hash
-          #   # => 'aaa=111; bbb=222'
-          def encode_to_header cookie_hash
-            cookie_hash.map{|k,v| "#{k}=#{v}"}.join '; '
-          end
-          # Apply request and response cookies as a string with request format.
-          #
-          # @param [String,Array,Hash] request_cookies Cookies to parse.
-          # @param [String,Array,Hash] response_cookies Cookies to parse.
-          #
-          # @return [String]
-          #
-          # @example
-          #   request_cookies = 'aaa=111; ddd=444'
-          #   response_cookies = [
-          #     'aaa=111; Expires=Thu, Jan 01 1970 00:00:00 UTC; path=/',
-          #     'bbb=222; path=/',
-          #     'ccc=333; path=/; expires=Wed, Jan 01 3000 00:00:00 UTC'
-          #   ]
-          #   update_as_hash , response_cookies
-          #   # => 'bbb=222; ccc=333; ddd=444'
-          def update request_cookies, response_cookies
-            cookie_hash = update_as_hash request_cookies, response_cookies
-            encode_to_header cookie_hash
-          end
-          # Compare if cookie is included into base cookie
-          #
-          # @param [Hash] base_cookie_hash Hash that represent universe.
-          # @param [Hash] cookie_hash Hash that represents to compare.
-          #
-          # @return [Boolean]
-          #
-          # @example Check a success match.
-          #   base_cookie_hash = {
-          #     'aaa' => 111,
-          #     'bbb' => 222,
-          #     'ccc' => 333,
-          #     'ddd' => 444
-          #   }
-          #   cookie_hash = {
-          #     'bbb' => 222,
-          #     'ddd' => 444
-          #   }
-          #   include? base_cookie_hash, cookie_hash
-          #   # => true
-          #
-          # @example Check with fail match.
-          #   base_cookie_hash = {
-          #     'aaa' => 111,
-          #     'bbb' => 222,
-          #     'ccc' => 333,
-          #     'ddd' => 444
-          #   }
-          #   cookie_hash = {
-          #     'bbb' => 555,
-          #     'ddd' => 444
-          #   }
-          #   include? base_cookie_hash, cookie_hash
-          #   # => false
-          def include? base_cookie_hash, cookie_hash
-            cookie_hash.each do |key, value|
-              return false unless base_cookie_hash.has_key?(key) && base_cookie_hash[key] == value
-            end
-            true
-          end
-        end
-      end
-    end
-  end
-end

data/lib/ae_easy/core/mock.rb DELETED

@@ -1,45 +0,0 @@
-require 'ae_easy/core/mock/fake_db'
-require 'ae_easy/core/mock/fake_executor'
-require 'ae_easy/core/mock/fake_parser'
-require 'ae_easy/core/mock/fake_seeder'
-require 'ae_easy/core/mock/fake_finisher'
-module AeEasy
-  module Core
-    module Mock
-      # Generate a context and message queue from a list of exposed methods.
-      #
-      # @param [Array] exposed_methods List of exposed methods.
-      #
-      # @example
-      #   exposed_methods = [:boo, :bar]
-      #   context, message_queue = AeEasy::Core::Mock.context_vars exposed_methods
-      #   context.boo 1, 2
-      #   context.bar 'A', 'B'
-      #   context.bar '111', '222'
-      #   message_queue
-      #   # => [
-      #   #   [:boo, [1, 2]],
-      #   #   [:bar, ['A', 'B']],
-      #   #   [:bar, ['111', '222']]
-      #   # ]
-      #
-      # @return [Array] `[context, message_queue]` being:
-      #   * `context`: Object implementing exposed methods.
-      #   * `[Array] message_queue`: Array to store messages.
-      def self.context_vars exposed_methods
-        context = Object.new
-        metaclass = class << context; self; end
-        message_queue = [] # Beat reference bug
-        exposed_methods = exposed_methods
-        exposed_methods.each do |key|
-          metaclass.send(:define_method, key) do |*args|
-            # Record all method calls into message queue for easy access
-            message_queue << [key, args]
-          end
-        end
-        [context, message_queue]
-      end
-    end
-  end
-end

data/lib/ae_easy/core/mock/fake_db.rb DELETED

@@ -1,561 +0,0 @@
-module AeEasy
-  module Core
-    module Mock
-      # Fake in memory database that emulates `Answersengine` database objects' black box behavior.
-      class FakeDb
-        # Page id keys, analog to primary keys.
-        PAGE_KEYS = ['gid'].freeze
-        # Output id keys, analog to primary keys.
-        OUTPUT_KEYS = ['_id', '_collection'].freeze
-        # Job id keys, analog to primary keys.
-        JOB_KEYS = ['job_id'].freeze
-        # Job available status.
-        JOB_STATUSES = {
-          active: 'active',
-          done: 'done',
-          cancelled: 'cancelled',
-          paused: 'paused'
-        }
-        # Default collection for saved outputs
-        DEFAULT_COLLECTION = 'default'
-        # Generate a smart collection with keys and initial values.
-        #
-        # @param [Array] keys Analog to primary keys, combination will be uniq.
-        # @param [Hash] opts Configuration options (see AeEasy::Core::SmartCollection#initialize).
-        #
-        # @return [AeEasy::Core::SmartCollection]
-        def self.new_collection keys, opts = {}
-          AeEasy::Core::SmartCollection.new keys, opts
-        end
-        # Generate a fake UUID.
-        #
-        # @param seed (nil) Object to use as seed for uuid.
-        #
-        # @return [String]
-        def self.fake_uuid seed = nil
-          seed ||= (Time.new.to_f + rand)
-          Digest::SHA1.hexdigest seed.to_s
-        end
-        # Generate a fake UUID based on output fields without `_` prefix.
-        #
-        # @param [Hash] data Output data.
-        #
-        # @return [String]
-        def self.output_uuid data
-          seed = data.select{|k,v|k.to_s =~ /^[^_]/}.hash
-          fake_uuid seed
-        end
-        # Build a page with defaults by using FakeDb engine.
-        #
-        # @param [Hash] page Page initial values.
-        # @param [Hash] opts ({}) Configuration options (see #initialize).
-        #
-        # @return [Hash]
-        def self.build_page page, opts = {}
-          opts = {
-            allow_page_gid_override: true,
-            allow_job_id_override: true
-          }.merge opts
-          temp_db = AeEasy::Core::Mock::FakeDb.new opts
-          temp_db.pages << page
-          temp_db.pages.first
-        end
-        # Build a fake page by using FakeDb engine.
-        #
-        # @param [Hash] opts ({}) Configuration options (see #initialize).
-        # @option opts [String] :url ('https://example.com') Page url.
-        #
-        # @return [Hash]
-        def self.build_fake_page opts = {}
-          page = {
-            'url' => (opts[:url] || 'https://example.com')
-          }
-          build_page page, opts
-        end
-        # Clean an URL to remove fragment, lowercase schema and host, and sort
-        #   query string.
-        #
-        # @param [String] raw_url URL to clean.
-        #
-        # @return [String]
-        def self.clean_uri raw_url
-          url = URI.parse(raw_url)
-          url.hostname = url.hostname.downcase
-          url.fragment = nil
-          # Sort query string keys
-          unless url.query.nil?
-            query_string = CGI.parse(url.query)
-            keys = query_string.keys.sort
-            data = []
-            keys.each do |key|
-              query_string[key].each do |value|
-                data << "#{URI.encode key}=#{URI.encode value}"
-              end
-            end
-            url.query = data.join('&')
-          end
-          url.to_s
-        end
-        # Format headers for gid generation.
-        # @private
-        #
-        # @param [Hash,nil] headers Headers hash.
-        #
-        # @return [Hash]
-        def self.format_headers headers
-          return {} if headers.nil?
-          data = {}
-          headers.each do |key, value|
-            unless value.is_a? Array
-              data[key] = value
-              next
-            end
-            data[key] = value.sort
-          end
-          data
-        end
-        # Build a job with defaults by using FakeDb engine.
-        #
-        # @param [Hash] job Job initial values.
-        # @param [Hash] opts ({}) Configuration options (see #initialize).
-        #
-        # @return [Hash]
-        def self.build_job job, opts = {}
-          temp_db = AeEasy::Core::Mock::FakeDb.new opts
-          temp_db.jobs << job
-          temp_db.jobs.last
-        end
-        # Build a fake job by using FakeDb engine.
-        #
-        # @param [Hash] opts ({}) Configuration options (see #initialize).
-        # @option opts [String] :scraper_name (nil) Scraper name.
-        # @option opts [Integer] :job_id (nil) Job id.
-        # @option opts [String] :status ('done').
-        #
-        # @return [Hash]
-        def self.build_fake_job opts = {}
-          job = {
-            'job_id' => opts[:job_id],
-            'scraper_name' => opts[:scraper_name],
-            'status' => (opts[:status] || 'done')
-          }
-          build_job job, opts
-        end
-        # Return a timestamp
-        #
-        # @param [Time] time (nil) Time from which to get time stamp.
-        #
-        # @return [String]
-        def self.time_stamp time = nil
-          time = Time.new if time.nil?
-          time.utc.strftime('%Y-%m-%dT%H:%M:%SZ')
-        end
-        # Get current job or create new one from values.
-        #
-        # @param [Integer] target_job_id (nil) Job id to ensure existance.
-        #
-        # @return [Hash]
-        def ensure_job target_job_id = nil
-          target_job_id = job_id if target_job_id.nil?
-          job = jobs.find{|v|v['job_id'] == target_job_id}
-          return job unless job.nil?
-          job = {
-            'job_id' => target_job_id,
-            'scraper_name' => scraper_name,
-          }
-          job['status'] = 'active' unless target_job_id != job_id
-          jobs << job
-          jobs.last
-        end
-        # Fake scraper_name.
-        # @return [String,nil]
-        def scraper_name
-          @scraper_name ||= 'my_scraper'
-        end
-        # Set fake scraper_name value.
-        def scraper_name= value
-          job = ensure_job
-          @scraper_name = value
-          job['scraper_name'] = scraper_name
-        end
-        # Fake job id.
-        # @return [Integer,nil]
-        def job_id
-          @job_id ||= generate_job_id
-        end
-        # Set fake job id value.
-        def job_id= value
-          @job_id = value
-          ensure_job
-          job_id
-        end
-        # Current fake page gid.
-        # @return [Integer,nil]
-        def page_gid
-          @page_gid ||= self.class.fake_uuid
-        end
-        # Set current fake page gid value.
-        def page_gid= value
-          @page_gid = value
-        end
-        # Enable page gid override on page or output insert.
-        def enable_page_gid_override
-          @allow_page_gid_override = true
-        end
-        # Disable page gid override on page or output insert.
-        def disable_page_gid_override
-          @allow_page_gid_override = false
-        end
-        # Specify whenever page gid overriding by user is allowed on page or
-        #   output insert.
-        #
-        # @return [Boolean] `true` when allowed, else `false`.
-        def allow_page_gid_override?
-          @allow_page_gid_override ||= false
-        end
-        # Enable job id override on page or output insert.
-        def enable_job_id_override
-          @allow_job_id_override = true
-        end
-        # Disable job id override on page or output insert.
-        def disable_job_id_override
-          @allow_job_id_override = false
-        end
-        # Specify whenever job id overriding by user is allowed on page or
-        #   output insert.
-        #
-        # @return [Boolean] `true` when allowed, else `false`.
-        def allow_job_id_override?
-          @allow_job_id_override ||= false
-        end
-        # Initialize fake database.
-        #
-        # @param [Hash] opts ({}) Configuration options.
-        # @option opts [Integer,nil] :job_id Job id default value.
-        # @option opts [String,nil] :scraper_name Scraper name default value.
-        # @option opts [String,nil] :page_gid Page gid default value.
-        # @option opts [Boolean, nil] :allow_page_gid_override (false) Specify
-        #   whenever page gid can be overrided on page or output insert.
-        # @option opts [Boolean, nil] :allow_job_id_override (false) Specify
-        #   whenever job id can be overrided on page or output insert.
-        def initialize opts = {}
-          self.job_id = opts[:job_id]
-          self.scraper_name = opts[:scraper_name]
-          self.page_gid = opts[:page_gid]
-          @allow_page_gid_override = opts[:allow_page_gid_override].nil? ? false : !!opts[:allow_page_gid_override]
-          @allow_job_id_override = opts[:allow_job_id_override].nil? ? false : !!opts[:allow_job_id_override]
-        end
-        # Generate a fake scraper name.
-        #
-        # @return [String]
-        def generate_scraper_name
-          Faker::Internet.unique.slug
-        end
-        # Generate a fake job_id.
-        #
-        # @return [Integer]
-        def generate_job_id
-          jobs.count < 1 ? 1 : (jobs.max{|a,b|a['job_id'] <=> b['job_id']}['job_id'] + 1)
-        end
-        # Get output keys with key generators to emulate saving on db.
-        # @private
-        #
-        # @return [Hash]
-        def job_defaults
-          @job_defaults ||= {
-            'job_id' => lambda{|job| generate_job_id},
-            'scraper_name' => lambda{|job| generate_scraper_name},
-            'status' => 'done',
-            'created_at' => lambda{|job| Time.now}
-          }
-        end
-        # Stored job collection
-        #
-        # @return [AeEasy::Core::SmartCollection]
-        def jobs
-          return @jobs unless @jobs.nil?
-          collection = self.class.new_collection JOB_KEYS,
-            defaults: job_defaults
-          collection.bind_event(:before_defaults) do |collection, raw_item|
-            AeEasy::Core.deep_stringify_keys raw_item
-          end
-          collection.bind_event(:before_insert) do |collection, item, match|
-            item['job_id'] ||= generate_job_id
-            item
-          end
-          @jobs ||= collection
-        end
-        # Generate a fake UUID based on page data:
-        #   * url
-        #   * method
-        #   * headers
-        #   * fetch_type
-        #   * cookie
-        #   * no_redirect
-        #   * body
-        #   * ua_type
-        #
-        # @param [Hash] page_data Page data.
-        #
-        # @return [String]
-        def generate_page_gid page_data
-          fields = [
-            'url',
-            'method',
-            'headers',
-            'fetch_type',
-            'cookie',
-            'no_redirect',
-            'body',
-            'ua_type'
-          ]
-          data = page_data.select{|k,v|fields.include? k}
-          data['url'] = self.class.clean_uri data['url']
-          data['headers'] = self.class.format_headers data['headers']
-          data['cookie'] = AeEasy::Core::Helper::Cookie.parse_from_request data['cookie'] unless data['cookie'].nil?
-          seed = data.select{|k,v|fields.include? k}.hash
-          checksum = self.class.fake_uuid seed
-          "#{URI.parse(data['url']).hostname}-#{checksum}"
-        end
-        # Get page keys with key generators to emulate saving on db.
-        # @private
-        #
-        # @return [Hash]
-        def page_defaults
-          @page_defaults ||= {
-            'url' => nil,
-            'status' => 'to_fetch',
-            'job_id' => lambda{|page| job_id},
-            'method' => 'GET',
-            'headers' => {},
-            'fetch_type' => 'standard',
-            'cookie' => nil,
-            'no_redirect' => false,
-            'body' => nil,
-            'ua_type' => 'desktop',
-            'no_url_encode' => false,
-            'http2' => false,
-            'vars' => {}
-          }
-        end
-        # Stored page collection.
-        #
-        # @return [AeEasy::Core::SmartCollection]
-        #
-        # @note Page gid will be replaced on insert by an auto generated uuid
-        #   unless page gid overriding is enabled
-        #   (see #allow_page_gid_override?)
-        def pages
-          return @pages unless @page.nil?
-          collection = self.class.new_collection PAGE_KEYS,
-            defaults: page_defaults
-          collection.bind_event(:before_defaults) do |collection, raw_item|
-            item = AeEasy::Core.deep_stringify_keys raw_item
-            item.delete 'job_id' unless allow_job_id_override?
-            item
-          end
-          collection.bind_event(:before_insert) do |collection, item, match|
-            if item['gid'].nil? || !allow_page_gid_override?
-              item['gid'] = generate_page_gid item
-            end
-            item
-          end
-          collection.bind_event(:after_insert) do |collection, item|
-            ensure_job item['job_id']
-          end
-          @pages ||= collection
-        end
-        # Generate a fake UUID for outputs.
-        #
-        # @param [Hash] data Output data.
-        #
-        # @return [String]
-        def generate_output_id data
-          # Generate random UUID to match AnswersEngine behavior
-          self.class.fake_uuid
-        end
-        # Get output keys with key generators to emulate saving on db.
-        # @private
-        #
-        # @return [Hash]
-        def output_defaults
-          @output_defaults ||= {
-            '_collection' => DEFAULT_COLLECTION,
-            '_job_id' => lambda{|output| job_id},
-            '_created_at' => lambda{|output| self.class.time_stamp},
-            '_gid' => lambda{|output| page_gid}
-          }
-        end
-        # Stored output collection
-        #
-        # @return [AeEasy::Core::SmartCollection]
-        def outputs
-          return @outputs unless @outputs.nil?
-          collection = self.class.new_collection OUTPUT_KEYS,
-            defaults: output_defaults
-          collection.bind_event(:before_defaults) do |collection, raw_item|
-            item = AeEasy::Core.deep_stringify_keys raw_item
-            item.delete '_job_id' unless allow_job_id_override?
-            item.delete '_gid_id' unless allow_page_gid_override?
-            item
-          end
-          collection.bind_event(:before_insert) do |collection, item, match|
-            item['_id'] ||= generate_output_id item
-            item
-          end
-          collection.bind_event(:after_insert) do |collection, item|
-            ensure_job item['_job_id']
-          end
-          @outputs ||= collection
-        end
-        # Match data to filters.
-        # @private
-        #
-        # @param data Hash containing data.
-        # @param filters Filters to apply on match.
-        #
-        # @return [Boolean]
-        #
-        # @note Missing and `nil` values on `data` will match when `filters`'
-        #   field is `nil`.
-        def match? data, filters
-          filters.each do |key, value|
-            return false if data[key] != value
-          end
-          true
-        end
-        # Search items from a collection.
-        #
-        # @param [Symbol] collection Allowed values: `:outputs`, `:pages`.
-        # @param [Hash] filter Filters to query.
-        # @param [Integer] offset (0) Search results offset.
-        # @param [Integer,nil] limit (nil) Limit search results count. Set to `nil` for unlimited.
-        #
-        # @raise ArgumentError On unknown collection.
-        #
-        # @note _Warning:_ It uses table scan to filter and should be used on test suites only.
-        def query collection, filter, offset = 0, limit = nil
-          return [] unless limit.nil? || limit > 0
-          # Get collection items
-          items = case collection
-          when :outputs
-            outputs
-          when :pages
-            pages
-          when :jobs
-            jobs
-          else
-            raise ArgumentError.new "Unknown collection #{collection}."
-          end
-          # Search items
-          count = 0
-          matches = []
-          items.each do |item|
-            next unless match? item, filter
-            count += 1
-            # Skip until offset
-            next unless offset < count
-            # Break on limit reach
-            break unless limit.nil? || matches.count < limit
-            matches << item
-          end
-          matches
-        end
-        # Refetch a page.
-        #
-        # @param [Integer] job_id Page's job_id to refetch.
-        # @param [String] gid Page's gid to refetch.
-        def refetch job_id, gid
-          page = pages.find_match('gid' => gid, 'job_id' => job_id)
-          raise Exception.new("Page not found with job_id \"#{job_id}\" gid \"#{gid}\"") if page.nil?
-          page['status'] = 'to_fetch'
-          page['freshness'] = self.class.time_stamp
-          page['to_fetch'] = self.class.time_stamp
-          page['fetched_from'] = nil
-          page['fetching_at'] = '2001-01-01T00:00:00Z'
-          page['fetched_at'] = nil
-          page['fetching_try_count'] = 0
-          page['effective_url'] = nil
-          page['parsing_at'] = nil
-          page['parsing_failed_at'] = nil
-          page['parsed_at'] = nil
-          page['parsing_try_count'] = 0
-          page['parsing_fail_count'] = 0
-          page['parsing_updated_at'] = '2001-01-01T00:00:00Z'
-          page['response_checksum'] = nil
-          page['response_status'] = nil
-          page['response_status_code'] = nil
-          page['response_headers'] = nil
-          page['response_cookie'] = nil
-          page['response_proto'] = nil
-          page['content_type'] = nil
-          page['content_size'] = 0
-          page['failed_response_status_code'] = nil
-          page['failed_response_headers'] = nil
-          page['failed_response_cookie'] = nil
-          page['failed_effective_url'] = nil
-          page['failed_at'] = nil
-          page['failed_content_type'] = nil
-        end
-        # Reparse a page.
-        #
-        # @param [Integer] job_id Page's job_id to reparse.
-        # @param [String] gid Page's gid to reparse.
-        def reparse job_id, gid
-          page = pages.find_match('gid' => gid, 'job_id' => job_id)
-          raise Exception.new("Page not found with job_id \"#{job_id}\" gid \"#{gid}\"") if page.nil?
-          page['status'] = 'to_parse'
-          page['parsing_at'] = nil
-          page['parsing_failed_at'] = nil
-          page['parsing_updated_at'] = '2001-01-01T00:00:00Z'
-          page['parsed_at'] = nil
-          page['parsing_try_count'] = 0
-          page['parsing_fail_count'] = 0
-        end
-      end
-    end
-  end
-end