RubyGems - ae_easy-core - Versions diffs - 0.2.0 → 0.2.1 - Mend

ae_easy-core 0.2.0 → 0.2.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (76) hide show

checksums.yaml +4 -4
data/CODE_OF_CONDUCT.md +1 -1
data/Gemfile +1 -1
data/LICENSE +1 -1
data/README.md +8 -4
data/Rakefile +0 -10
data/ae_easy-core.gemspec +6 -13
data/lib/ae_easy/core.rb +4 -256
metadata +18 -125
data/doc/AeEasy.html +0 -117
data/doc/AeEasy/Core.html +0 -1590
data/doc/AeEasy/Core/Config.html +0 -311
data/doc/AeEasy/Core/Exception.html +0 -117
data/doc/AeEasy/Core/Exception/OutdatedError.html +0 -135
data/doc/AeEasy/Core/Helper.html +0 -117
data/doc/AeEasy/Core/Helper/Cookie.html +0 -1070
data/doc/AeEasy/Core/Mock.html +0 -282
data/doc/AeEasy/Core/Mock/FakeDb.html +0 -3779
data/doc/AeEasy/Core/Mock/FakeExecutor.html +0 -3289
data/doc/AeEasy/Core/Mock/FakeFinisher.html +0 -160
data/doc/AeEasy/Core/Mock/FakeParser.html +0 -160
data/doc/AeEasy/Core/Mock/FakeSeeder.html +0 -160
data/doc/AeEasy/Core/Plugin.html +0 -117
data/doc/AeEasy/Core/Plugin/CollectionVault.html +0 -299
data/doc/AeEasy/Core/Plugin/ConfigBehavior.html +0 -541
data/doc/AeEasy/Core/Plugin/ContextIntegrator.html +0 -445
data/doc/AeEasy/Core/Plugin/Executor.html +0 -259
data/doc/AeEasy/Core/Plugin/ExecutorBehavior.html +0 -344
data/doc/AeEasy/Core/Plugin/Finisher.html +0 -265
data/doc/AeEasy/Core/Plugin/FinisherBehavior.html +0 -142
data/doc/AeEasy/Core/Plugin/InitializeHook.html +0 -220
data/doc/AeEasy/Core/Plugin/Parser.html +0 -270
data/doc/AeEasy/Core/Plugin/ParserBehavior.html +0 -235
data/doc/AeEasy/Core/Plugin/Seeder.html +0 -674
data/doc/AeEasy/Core/Plugin/SeederBehavior.html +0 -142
data/doc/AeEasy/Core/SmartCollection.html +0 -1087
data/doc/_index.html +0 -364
data/doc/class_list.html +0 -51
data/doc/css/common.css +0 -1
data/doc/css/full_list.css +0 -58
data/doc/css/style.css +0 -496
data/doc/file.README.html +0 -91
data/doc/file_list.html +0 -56
data/doc/frames.html +0 -17
data/doc/index.html +0 -91
data/doc/js/app.js +0 -303
data/doc/js/full_list.js +0 -216
data/doc/js/jquery.js +0 -4
data/doc/method_list.html +0 -939
data/doc/top-level-namespace.html +0 -110
data/lib/ae_easy/core/config.rb +0 -27
data/lib/ae_easy/core/exception.rb +0 -8
data/lib/ae_easy/core/exception/outdated_error.rb +0 -9
data/lib/ae_easy/core/helper.rb +0 -8
data/lib/ae_easy/core/helper/cookie.rb +0 -209
data/lib/ae_easy/core/mock.rb +0 -45
data/lib/ae_easy/core/mock/fake_db.rb +0 -561
data/lib/ae_easy/core/mock/fake_executor.rb +0 -373
data/lib/ae_easy/core/mock/fake_finisher.rb +0 -28
data/lib/ae_easy/core/mock/fake_parser.rb +0 -33
data/lib/ae_easy/core/mock/fake_seeder.rb +0 -28
data/lib/ae_easy/core/plugin.rb +0 -19
data/lib/ae_easy/core/plugin/collection_vault.rb +0 -23
data/lib/ae_easy/core/plugin/config_behavior.rb +0 -43
data/lib/ae_easy/core/plugin/context_integrator.rb +0 -60
data/lib/ae_easy/core/plugin/executor.rb +0 -19
data/lib/ae_easy/core/plugin/executor_behavior.rb +0 -32
data/lib/ae_easy/core/plugin/finisher.rb +0 -19
data/lib/ae_easy/core/plugin/finisher_behavior.rb +0 -9
data/lib/ae_easy/core/plugin/initialize_hook.rb +0 -17
data/lib/ae_easy/core/plugin/parser.rb +0 -19
data/lib/ae_easy/core/plugin/parser_behavior.rb +0 -17
data/lib/ae_easy/core/plugin/seeder.rb +0 -44
data/lib/ae_easy/core/plugin/seeder_behavior.rb +0 -9
data/lib/ae_easy/core/smart_collection.rb +0 -236
data/lib/ae_easy/core/version.rb +0 -6

data/doc/top-level-namespace.html DELETED

@@ -1,110 +0,0 @@
-<!DOCTYPE html>
-<html>
-  <head>
-    <meta charset="utf-8">
-<meta name="viewport" content="width=device-width, initial-scale=1.0">
-<title>
-  Top Level Namespace
-    &mdash; Documentation by YARD 0.9.20
-</title>
-  <link rel="stylesheet" href="css/style.css" type="text/css" charset="utf-8" />
-  <link rel="stylesheet" href="css/common.css" type="text/css" charset="utf-8" />
-<script type="text/javascript" charset="utf-8">
-  pathId = "";
-  relpath = '';
-</script>
-  <script type="text/javascript" charset="utf-8" src="js/jquery.js"></script>
-  <script type="text/javascript" charset="utf-8" src="js/app.js"></script>
-  </head>
-  <body>
-    <div class="nav_wrap">
-      <iframe id="nav" src="class_list.html?1"></iframe>
-      <div id="resizer"></div>
-    </div>
-    <div id="main" tabindex="-1">
-      <div id="header">
-        <div id="menu">
-    <a href="_index.html">Index</a> &raquo;
-    <span class="title">Top Level Namespace</span>
-</div>
-        <div id="search">
-    <a class="full_list_link" id="class_list_link"
-        href="class_list.html">
-        <svg width="24" height="24">
-          <rect x="0" y="4" width="24" height="4" rx="1" ry="1"></rect>
-          <rect x="0" y="12" width="24" height="4" rx="1" ry="1"></rect>
-          <rect x="0" y="20" width="24" height="4" rx="1" ry="1"></rect>
-        </svg>
-    </a>
-</div>
-        <div class="clear"></div>
-      </div>
-      <div id="content"><h1>Top Level Namespace
-</h1>
-<div class="box_info">
-</div>
-<h2>Defined Under Namespace</h2>
-<p class="children">
-      <strong class="modules">Modules:</strong> <span class='object_link'><a href="AeEasy.html" title="AeEasy (module)">AeEasy</a></span>
-</p>
-</div>
-      <div id="footer">
-  Generated on Fri Sep 27 02:01:30 2019 by
-  <a href="http://yardoc.org" title="Yay! A Ruby Documentation Tool" target="_parent">yard</a>
-  0.9.20 (ruby-2.5.3).
-</div>
-    </div>
-  </body>
-</html>

data/lib/ae_easy/core/config.rb DELETED

@@ -1,27 +0,0 @@
-module AeEasy
-  module Core
-    # Configuration manager tool useful for global configuration data accross
-    #   the scraping process.
-    class Config
-      include AeEasy::Core::Plugin::InitializeHook
-      include AeEasy::Core::Plugin::ConfigBehavior
-      # {AeEasy::Core::Plugin::ConfigBehavior#config_collection_key}
-      alias :collection_key :config_collection_key
-      # {AeEasy::Core::Plugin::ConfigBehavior#config_collection}
-      alias :collection :config_collection
-      # Initialize config object
-      #
-      # @param [Hash] opts ({}) Configuration options.
-      #
-      # @see AeEasy::Core::Plugin::ConfigBehavior#initialize_hook_core_config_behavior
-      def initialize opts = {}
-        opts = opts.merge(
-          config_collection: opts[:collection]
-        )
-        initialize_hooks opts
-      end
-    end
-  end
-end

data/lib/ae_easy/core/exception.rb DELETED

@@ -1,8 +0,0 @@
-require 'ae_easy/core/exception/outdated_error'
-module AeEasy
-  module Core
-    module Exception
-    end
-  end
-end

data/lib/ae_easy/core/exception/outdated_error.rb DELETED

@@ -1,9 +0,0 @@
-module AeEasy
-  module Core
-    module Exception
-      # Exception that indicates something is outdated error.
-      class OutdatedError < StandardError
-      end
-    end
-  end
-end

data/lib/ae_easy/core/helper.rb DELETED

@@ -1,8 +0,0 @@
-require 'ae_easy/core/helper/cookie'
-module AeEasy
-  module Core
-    module Helper
-    end
-  end
-end

data/lib/ae_easy/core/helper/cookie.rb DELETED

@@ -1,209 +0,0 @@
-module AeEasy
-  module Core
-    module Helper
-      # Helper used for lower level cookie management.
-      class Cookie
-        class << self
-          # Parse request cookies on different formats.
-          #
-          # @param [String,Hash,Array] cookies Cookies to parse.
-          # @param [Hash] cookie_hash ({}) External hash to store parsed cookies.
-          #
-          # @return [Hash]
-          #
-          # @example Parse from string.
-          #   parse_from_request 'aaa=111; bbb=222'
-          #   # => {'aaa' => 111, 'bbb' => 222}
-          #
-          # @example Parse from array.
-          #   cookies = [
-          #     'aaa=111',
-          #     'bbb=222'
-          #   ]
-          #   parse_from_response cookies
-          #   # => {'aaa' => 111, 'bbb' => 222}
-          #
-          # @example Parse with `cookie_hash`.
-          #   cookie_hash = {'ccc' => 333}
-          #   parse_from_request 'aaa=111; bbb=222', cookie_hash
-          #   cookie_hash
-          #   # => {'aaa' => 1, 'bbb' => 2, 'ccc' => 333}
-          def parse_from_request cookies, cookie_hash = {}
-            # Retrieve from hash
-            if cookies.is_a? Hash
-              cookie_hash.merge! cookies
-              return cookie_hash
-            end
-            # Extract from string
-            cookies = cookies.split '; ' if cookies.is_a? String
-            # Extract from array
-            cookies&.each do |raw_cookie|
-              key, value = raw_cookie.split('=', 2)
-              cookie_hash[key] = value
-            end
-            cookie_hash
-          end
-          # Parse response cookies on different formats.
-          #
-          # @param [String,Hash,Array] cookies Cookies to parse.
-          # @param [Hash] cookie_hash ({}) External hash to store parsed cookies.
-          #
-          # @return [Hash]
-          #
-          # @example Parse from string
-          #   parse_from_response 'aaa=111; bbb=222'
-          #   # => {'aaa' => 111, 'bbb' => 222}
-          #
-          # @example Parse from array.
-          #   cookies = [
-          #     'aaa=111; Expires=Thu, Jan 01 1970 00:00:00 UTC; path=/',
-          #     'bbb=222; path=/',
-          #     'ccc=333; path=/; expires=Wed, Jan 01 3000 00:00:00 UTC'
-          #   ]
-          #   parse_from_response cookies
-          #   # => {'bbb' => 222, 'ccc' => 333}
-          #
-          # @example Parse with `cookie_hash`.
-          #   cookie_hash = {'ccc' => 333}
-          #   parse_from_response 'aaa=111; bbb=222', cookie_hash
-          #   cookie_hash
-          #   # => {'aaa' => 111, 'bbb' => 222, 'ccc' => 333}
-          def parse_from_response cookies, cookie_hash = {}
-            # Retrieve from hash
-            if cookies.is_a? Hash
-              cookie_hash.merge! cookies
-              return cookie_hash
-            end
-            # Retrieve from String
-            cookies = cookies.split '; ' if cookies.is_a? String
-            # Extract from array
-            info = cookie = expires = key = value = nil
-            cookies&.each do |raw_cookie|
-              # Extract cookie data
-              key_pair = raw_cookie.scan(/(?:;\s+([^\=]+)=([^;]*))/i) || []
-              cookie = key_pair.inject(Hash.new){|h,i|h[i[0].to_s.downcase] = i[1]; h}
-              cookie[:key], cookie[:value] = raw_cookie.match(/^\s*(?<key>[^\=]+)\=(?<value>[^;]*)/i)&.captures
-              # Check cookie expire
-              expires = cookie['expires'].nil? ? nil : Time.parse(cookie['expires'])
-              if !expires.nil? && Time.now > expires
-                cookie_hash.delete cookie[:key]
-                next
-              end
-              # Save cookie
-              cookie_hash[cookie[:key]] = cookie[:value]
-            end
-            cookie_hash
-          end
-          # Apply request and response cookies as a hash.
-          #
-          # @param [String,Array,Hash] request_cookies Cookies to parse.
-          # @param [String,Array,Hash] response_cookies Cookies to parse.
-          #
-          # @return [Hash]
-          #
-          # @example
-          #   request_cookies = 'aaa=111; ddd=444'
-          #   response_cookies = [
-          #     'aaa=111; Expires=Thu, Jan 01 1970 00:00:00 UTC; path=/',
-          #     'bbb=222; path=/',
-          #     'ccc=333; path=/; expires=Wed, Jan 01 3000 00:00:00 UTC'
-          #   ]
-          #   update_as_hash , response_cookies
-          #   # => {'bbb' => 222, 'ccc' => 333, 'ddd' => 444}
-          def update_as_hash request_cookies, response_cookies
-            cookie_hash = {}
-            parse_from_request request_cookies, cookie_hash
-            parse_from_response response_cookies, cookie_hash
-            cookie_hash
-          end
-          # Encode cookies as request cookie string.
-          #
-          # @param [Hash] cookie_hash Hash with cookies.
-          #
-          # @return [String]
-          #
-          # @example
-          #   cookie_hash = {
-          #     'aaa' => 111,
-          #     'bbb' => 222
-          #   }
-          #   encode_to_header cookie_hash
-          #   # => 'aaa=111; bbb=222'
-          def encode_to_header cookie_hash
-            cookie_hash.map{|k,v| "#{k}=#{v}"}.join '; '
-          end
-          # Apply request and response cookies as a string with request format.
-          #
-          # @param [String,Array,Hash] request_cookies Cookies to parse.
-          # @param [String,Array,Hash] response_cookies Cookies to parse.
-          #
-          # @return [String]
-          #
-          # @example
-          #   request_cookies = 'aaa=111; ddd=444'
-          #   response_cookies = [
-          #     'aaa=111; Expires=Thu, Jan 01 1970 00:00:00 UTC; path=/',
-          #     'bbb=222; path=/',
-          #     'ccc=333; path=/; expires=Wed, Jan 01 3000 00:00:00 UTC'
-          #   ]
-          #   update_as_hash , response_cookies
-          #   # => 'bbb=222; ccc=333; ddd=444'
-          def update request_cookies, response_cookies
-            cookie_hash = update_as_hash request_cookies, response_cookies
-            encode_to_header cookie_hash
-          end
-          # Compare if cookie is included into base cookie
-          #
-          # @param [Hash] base_cookie_hash Hash that represent universe.
-          # @param [Hash] cookie_hash Hash that represents to compare.
-          #
-          # @return [Boolean]
-          #
-          # @example Check a success match.
-          #   base_cookie_hash = {
-          #     'aaa' => 111,
-          #     'bbb' => 222,
-          #     'ccc' => 333,
-          #     'ddd' => 444
-          #   }
-          #   cookie_hash = {
-          #     'bbb' => 222,
-          #     'ddd' => 444
-          #   }
-          #   include? base_cookie_hash, cookie_hash
-          #   # => true
-          #
-          # @example Check with fail match.
-          #   base_cookie_hash = {
-          #     'aaa' => 111,
-          #     'bbb' => 222,
-          #     'ccc' => 333,
-          #     'ddd' => 444
-          #   }
-          #   cookie_hash = {
-          #     'bbb' => 555,
-          #     'ddd' => 444
-          #   }
-          #   include? base_cookie_hash, cookie_hash
-          #   # => false
-          def include? base_cookie_hash, cookie_hash
-            cookie_hash.each do |key, value|
-              return false unless base_cookie_hash.has_key?(key) && base_cookie_hash[key] == value
-            end
-            true
-          end
-        end
-      end
-    end
-  end
-end

data/lib/ae_easy/core/mock.rb DELETED

@@ -1,45 +0,0 @@
-require 'ae_easy/core/mock/fake_db'
-require 'ae_easy/core/mock/fake_executor'
-require 'ae_easy/core/mock/fake_parser'
-require 'ae_easy/core/mock/fake_seeder'
-require 'ae_easy/core/mock/fake_finisher'
-module AeEasy
-  module Core
-    module Mock
-      # Generate a context and message queue from a list of exposed methods.
-      #
-      # @param [Array] exposed_methods List of exposed methods.
-      #
-      # @example
-      #   exposed_methods = [:boo, :bar]
-      #   context, message_queue = AeEasy::Core::Mock.context_vars exposed_methods
-      #   context.boo 1, 2
-      #   context.bar 'A', 'B'
-      #   context.bar '111', '222'
-      #   message_queue
-      #   # => [
-      #   #   [:boo, [1, 2]],
-      #   #   [:bar, ['A', 'B']],
-      #   #   [:bar, ['111', '222']]
-      #   # ]
-      #
-      # @return [Array] `[context, message_queue]` being:
-      #   * `context`: Object implementing exposed methods.
-      #   * `[Array] message_queue`: Array to store messages.
-      def self.context_vars exposed_methods
-        context = Object.new
-        metaclass = class << context; self; end
-        message_queue = [] # Beat reference bug
-        exposed_methods = exposed_methods
-        exposed_methods.each do |key|
-          metaclass.send(:define_method, key) do |*args|
-            # Record all method calls into message queue for easy access
-            message_queue << [key, args]
-          end
-        end
-        [context, message_queue]
-      end
-    end
-  end
-end

data/lib/ae_easy/core/mock/fake_db.rb DELETED

@@ -1,561 +0,0 @@
-module AeEasy
-  module Core
-    module Mock
-      # Fake in memory database that emulates `Answersengine` database objects' black box behavior.
-      class FakeDb
-        # Page id keys, analog to primary keys.
-        PAGE_KEYS = ['gid'].freeze
-        # Output id keys, analog to primary keys.
-        OUTPUT_KEYS = ['_id', '_collection'].freeze
-        # Job id keys, analog to primary keys.
-        JOB_KEYS = ['job_id'].freeze
-        # Job available status.
-        JOB_STATUSES = {
-          active: 'active',
-          done: 'done',
-          cancelled: 'cancelled',
-          paused: 'paused'
-        }
-        # Default collection for saved outputs
-        DEFAULT_COLLECTION = 'default'
-        # Generate a smart collection with keys and initial values.
-        #
-        # @param [Array] keys Analog to primary keys, combination will be uniq.
-        # @param [Hash] opts Configuration options (see AeEasy::Core::SmartCollection#initialize).
-        #
-        # @return [AeEasy::Core::SmartCollection]
-        def self.new_collection keys, opts = {}
-          AeEasy::Core::SmartCollection.new keys, opts
-        end
-        # Generate a fake UUID.
-        #
-        # @param seed (nil) Object to use as seed for uuid.
-        #
-        # @return [String]
-        def self.fake_uuid seed = nil
-          seed ||= (Time.new.to_f + rand)
-          Digest::SHA1.hexdigest seed.to_s
-        end
-        # Generate a fake UUID based on output fields without `_` prefix.
-        #
-        # @param [Hash] data Output data.
-        #
-        # @return [String]
-        def self.output_uuid data
-          seed = data.select{|k,v|k.to_s =~ /^[^_]/}.hash
-          fake_uuid seed
-        end
-        # Build a page with defaults by using FakeDb engine.
-        #
-        # @param [Hash] page Page initial values.
-        # @param [Hash] opts ({}) Configuration options (see #initialize).
-        #
-        # @return [Hash]
-        def self.build_page page, opts = {}
-          opts = {
-            allow_page_gid_override: true,
-            allow_job_id_override: true
-          }.merge opts
-          temp_db = AeEasy::Core::Mock::FakeDb.new opts
-          temp_db.pages << page
-          temp_db.pages.first
-        end
-        # Build a fake page by using FakeDb engine.
-        #
-        # @param [Hash] opts ({}) Configuration options (see #initialize).
-        # @option opts [String] :url ('https://example.com') Page url.
-        #
-        # @return [Hash]
-        def self.build_fake_page opts = {}
-          page = {
-            'url' => (opts[:url] || 'https://example.com')
-          }
-          build_page page, opts
-        end
-        # Clean an URL to remove fragment, lowercase schema and host, and sort
-        #   query string.
-        #
-        # @param [String] raw_url URL to clean.
-        #
-        # @return [String]
-        def self.clean_uri raw_url
-          url = URI.parse(raw_url)
-          url.hostname = url.hostname.downcase
-          url.fragment = nil
-          # Sort query string keys
-          unless url.query.nil?
-            query_string = CGI.parse(url.query)
-            keys = query_string.keys.sort
-            data = []
-            keys.each do |key|
-              query_string[key].each do |value|
-                data << "#{URI.encode key}=#{URI.encode value}"
-              end
-            end
-            url.query = data.join('&')
-          end
-          url.to_s
-        end
-        # Format headers for gid generation.
-        # @private
-        #
-        # @param [Hash,nil] headers Headers hash.
-        #
-        # @return [Hash]
-        def self.format_headers headers
-          return {} if headers.nil?
-          data = {}
-          headers.each do |key, value|
-            unless value.is_a? Array
-              data[key] = value
-              next
-            end
-            data[key] = value.sort
-          end
-          data
-        end
-        # Build a job with defaults by using FakeDb engine.
-        #
-        # @param [Hash] job Job initial values.
-        # @param [Hash] opts ({}) Configuration options (see #initialize).
-        #
-        # @return [Hash]
-        def self.build_job job, opts = {}
-          temp_db = AeEasy::Core::Mock::FakeDb.new opts
-          temp_db.jobs << job
-          temp_db.jobs.last
-        end
-        # Build a fake job by using FakeDb engine.
-        #
-        # @param [Hash] opts ({}) Configuration options (see #initialize).
-        # @option opts [String] :scraper_name (nil) Scraper name.
-        # @option opts [Integer] :job_id (nil) Job id.
-        # @option opts [String] :status ('done').
-        #
-        # @return [Hash]
-        def self.build_fake_job opts = {}
-          job = {
-            'job_id' => opts[:job_id],
-            'scraper_name' => opts[:scraper_name],
-            'status' => (opts[:status] || 'done')
-          }
-          build_job job, opts
-        end
-        # Return a timestamp
-        #
-        # @param [Time] time (nil) Time from which to get time stamp.
-        #
-        # @return [String]
-        def self.time_stamp time = nil
-          time = Time.new if time.nil?
-          time.utc.strftime('%Y-%m-%dT%H:%M:%SZ')
-        end
-        # Get current job or create new one from values.
-        #
-        # @param [Integer] target_job_id (nil) Job id to ensure existance.
-        #
-        # @return [Hash]
-        def ensure_job target_job_id = nil
-          target_job_id = job_id if target_job_id.nil?
-          job = jobs.find{|v|v['job_id'] == target_job_id}
-          return job unless job.nil?
-          job = {
-            'job_id' => target_job_id,
-            'scraper_name' => scraper_name,
-          }
-          job['status'] = 'active' unless target_job_id != job_id
-          jobs << job
-          jobs.last
-        end
-        # Fake scraper_name.
-        # @return [String,nil]
-        def scraper_name
-          @scraper_name ||= 'my_scraper'
-        end
-        # Set fake scraper_name value.
-        def scraper_name= value
-          job = ensure_job
-          @scraper_name = value
-          job['scraper_name'] = scraper_name
-        end
-        # Fake job id.
-        # @return [Integer,nil]
-        def job_id
-          @job_id ||= generate_job_id
-        end
-        # Set fake job id value.
-        def job_id= value
-          @job_id = value
-          ensure_job
-          job_id
-        end
-        # Current fake page gid.
-        # @return [Integer,nil]
-        def page_gid
-          @page_gid ||= self.class.fake_uuid
-        end
-        # Set current fake page gid value.
-        def page_gid= value
-          @page_gid = value
-        end
-        # Enable page gid override on page or output insert.
-        def enable_page_gid_override
-          @allow_page_gid_override = true
-        end
-        # Disable page gid override on page or output insert.
-        def disable_page_gid_override
-          @allow_page_gid_override = false
-        end
-        # Specify whenever page gid overriding by user is allowed on page or
-        #   output insert.
-        #
-        # @return [Boolean] `true` when allowed, else `false`.
-        def allow_page_gid_override?
-          @allow_page_gid_override ||= false
-        end
-        # Enable job id override on page or output insert.
-        def enable_job_id_override
-          @allow_job_id_override = true
-        end
-        # Disable job id override on page or output insert.
-        def disable_job_id_override
-          @allow_job_id_override = false
-        end
-        # Specify whenever job id overriding by user is allowed on page or
-        #   output insert.
-        #
-        # @return [Boolean] `true` when allowed, else `false`.
-        def allow_job_id_override?
-          @allow_job_id_override ||= false
-        end
-        # Initialize fake database.
-        #
-        # @param [Hash] opts ({}) Configuration options.
-        # @option opts [Integer,nil] :job_id Job id default value.
-        # @option opts [String,nil] :scraper_name Scraper name default value.
-        # @option opts [String,nil] :page_gid Page gid default value.
-        # @option opts [Boolean, nil] :allow_page_gid_override (false) Specify
-        #   whenever page gid can be overrided on page or output insert.
-        # @option opts [Boolean, nil] :allow_job_id_override (false) Specify
-        #   whenever job id can be overrided on page or output insert.
-        def initialize opts = {}
-          self.job_id = opts[:job_id]
-          self.scraper_name = opts[:scraper_name]
-          self.page_gid = opts[:page_gid]
-          @allow_page_gid_override = opts[:allow_page_gid_override].nil? ? false : !!opts[:allow_page_gid_override]
-          @allow_job_id_override = opts[:allow_job_id_override].nil? ? false : !!opts[:allow_job_id_override]
-        end
-        # Generate a fake scraper name.
-        #
-        # @return [String]
-        def generate_scraper_name
-          Faker::Internet.unique.slug
-        end
-        # Generate a fake job_id.
-        #
-        # @return [Integer]
-        def generate_job_id
-          jobs.count < 1 ? 1 : (jobs.max{|a,b|a['job_id'] <=> b['job_id']}['job_id'] + 1)
-        end
-        # Get output keys with key generators to emulate saving on db.
-        # @private
-        #
-        # @return [Hash]
-        def job_defaults
-          @job_defaults ||= {
-            'job_id' => lambda{|job| generate_job_id},
-            'scraper_name' => lambda{|job| generate_scraper_name},
-            'status' => 'done',
-            'created_at' => lambda{|job| Time.now}
-          }
-        end
-        # Stored job collection
-        #
-        # @return [AeEasy::Core::SmartCollection]
-        def jobs
-          return @jobs unless @jobs.nil?
-          collection = self.class.new_collection JOB_KEYS,
-            defaults: job_defaults
-          collection.bind_event(:before_defaults) do |collection, raw_item|
-            AeEasy::Core.deep_stringify_keys raw_item
-          end
-          collection.bind_event(:before_insert) do |collection, item, match|
-            item['job_id'] ||= generate_job_id
-            item
-          end
-          @jobs ||= collection
-        end
-        # Generate a fake UUID based on page data:
-        #   * url
-        #   * method
-        #   * headers
-        #   * fetch_type
-        #   * cookie
-        #   * no_redirect
-        #   * body
-        #   * ua_type
-        #
-        # @param [Hash] page_data Page data.
-        #
-        # @return [String]
-        def generate_page_gid page_data
-          fields = [
-            'url',
-            'method',
-            'headers',
-            'fetch_type',
-            'cookie',
-            'no_redirect',
-            'body',
-            'ua_type'
-          ]
-          data = page_data.select{|k,v|fields.include? k}
-          data['url'] = self.class.clean_uri data['url']
-          data['headers'] = self.class.format_headers data['headers']
-          data['cookie'] = AeEasy::Core::Helper::Cookie.parse_from_request data['cookie'] unless data['cookie'].nil?
-          seed = data.select{|k,v|fields.include? k}.hash
-          checksum = self.class.fake_uuid seed
-          "#{URI.parse(data['url']).hostname}-#{checksum}"
-        end
-        # Get page keys with key generators to emulate saving on db.
-        # @private
-        #
-        # @return [Hash]
-        def page_defaults
-          @page_defaults ||= {
-            'url' => nil,
-            'status' => 'to_fetch',
-            'job_id' => lambda{|page| job_id},
-            'method' => 'GET',
-            'headers' => {},
-            'fetch_type' => 'standard',
-            'cookie' => nil,
-            'no_redirect' => false,
-            'body' => nil,
-            'ua_type' => 'desktop',
-            'no_url_encode' => false,
-            'http2' => false,
-            'vars' => {}
-          }
-        end
-        # Stored page collection.
-        #
-        # @return [AeEasy::Core::SmartCollection]
-        #
-        # @note Page gid will be replaced on insert by an auto generated uuid
-        #   unless page gid overriding is enabled
-        #   (see #allow_page_gid_override?)
-        def pages
-          return @pages unless @page.nil?
-          collection = self.class.new_collection PAGE_KEYS,
-            defaults: page_defaults
-          collection.bind_event(:before_defaults) do |collection, raw_item|
-            item = AeEasy::Core.deep_stringify_keys raw_item
-            item.delete 'job_id' unless allow_job_id_override?
-            item
-          end
-          collection.bind_event(:before_insert) do |collection, item, match|
-            if item['gid'].nil? || !allow_page_gid_override?
-              item['gid'] = generate_page_gid item
-            end
-            item
-          end
-          collection.bind_event(:after_insert) do |collection, item|
-            ensure_job item['job_id']
-          end
-          @pages ||= collection
-        end
-        # Generate a fake UUID for outputs.
-        #
-        # @param [Hash] data Output data.
-        #
-        # @return [String]
-        def generate_output_id data
-          # Generate random UUID to match AnswersEngine behavior
-          self.class.fake_uuid
-        end
-        # Get output keys with key generators to emulate saving on db.
-        # @private
-        #
-        # @return [Hash]
-        def output_defaults
-          @output_defaults ||= {
-            '_collection' => DEFAULT_COLLECTION,
-            '_job_id' => lambda{|output| job_id},
-            '_created_at' => lambda{|output| self.class.time_stamp},
-            '_gid' => lambda{|output| page_gid}
-          }
-        end
-        # Stored output collection
-        #
-        # @return [AeEasy::Core::SmartCollection]
-        def outputs
-          return @outputs unless @outputs.nil?
-          collection = self.class.new_collection OUTPUT_KEYS,
-            defaults: output_defaults
-          collection.bind_event(:before_defaults) do |collection, raw_item|
-            item = AeEasy::Core.deep_stringify_keys raw_item
-            item.delete '_job_id' unless allow_job_id_override?
-            item.delete '_gid_id' unless allow_page_gid_override?
-            item
-          end
-          collection.bind_event(:before_insert) do |collection, item, match|
-            item['_id'] ||= generate_output_id item
-            item
-          end
-          collection.bind_event(:after_insert) do |collection, item|
-            ensure_job item['_job_id']
-          end
-          @outputs ||= collection
-        end
-        # Match data to filters.
-        # @private
-        #
-        # @param data Hash containing data.
-        # @param filters Filters to apply on match.
-        #
-        # @return [Boolean]
-        #
-        # @note Missing and `nil` values on `data` will match when `filters`'
-        #   field is `nil`.
-        def match? data, filters
-          filters.each do |key, value|
-            return false if data[key] != value
-          end
-          true
-        end
-        # Search items from a collection.
-        #
-        # @param [Symbol] collection Allowed values: `:outputs`, `:pages`.
-        # @param [Hash] filter Filters to query.
-        # @param [Integer] offset (0) Search results offset.
-        # @param [Integer,nil] limit (nil) Limit search results count. Set to `nil` for unlimited.
-        #
-        # @raise ArgumentError On unknown collection.
-        #
-        # @note _Warning:_ It uses table scan to filter and should be used on test suites only.
-        def query collection, filter, offset = 0, limit = nil
-          return [] unless limit.nil? || limit > 0
-          # Get collection items
-          items = case collection
-          when :outputs
-            outputs
-          when :pages
-            pages
-          when :jobs
-            jobs
-          else
-            raise ArgumentError.new "Unknown collection #{collection}."
-          end
-          # Search items
-          count = 0
-          matches = []
-          items.each do |item|
-            next unless match? item, filter
-            count += 1
-            # Skip until offset
-            next unless offset < count
-            # Break on limit reach
-            break unless limit.nil? || matches.count < limit
-            matches << item
-          end
-          matches
-        end
-        # Refetch a page.
-        #
-        # @param [Integer] job_id Page's job_id to refetch.
-        # @param [String] gid Page's gid to refetch.
-        def refetch job_id, gid
-          page = pages.find_match('gid' => gid, 'job_id' => job_id)
-          raise Exception.new("Page not found with job_id \"#{job_id}\" gid \"#{gid}\"") if page.nil?
-          page['status'] = 'to_fetch'
-          page['freshness'] = self.class.time_stamp
-          page['to_fetch'] = self.class.time_stamp
-          page['fetched_from'] = nil
-          page['fetching_at'] = '2001-01-01T00:00:00Z'
-          page['fetched_at'] = nil
-          page['fetching_try_count'] = 0
-          page['effective_url'] = nil
-          page['parsing_at'] = nil
-          page['parsing_failed_at'] = nil
-          page['parsed_at'] = nil
-          page['parsing_try_count'] = 0
-          page['parsing_fail_count'] = 0
-          page['parsing_updated_at'] = '2001-01-01T00:00:00Z'
-          page['response_checksum'] = nil
-          page['response_status'] = nil
-          page['response_status_code'] = nil
-          page['response_headers'] = nil
-          page['response_cookie'] = nil
-          page['response_proto'] = nil
-          page['content_type'] = nil
-          page['content_size'] = 0
-          page['failed_response_status_code'] = nil
-          page['failed_response_headers'] = nil
-          page['failed_response_cookie'] = nil
-          page['failed_effective_url'] = nil
-          page['failed_at'] = nil
-          page['failed_content_type'] = nil
-        end
-        # Reparse a page.
-        #
-        # @param [Integer] job_id Page's job_id to reparse.
-        # @param [String] gid Page's gid to reparse.
-        def reparse job_id, gid
-          page = pages.find_match('gid' => gid, 'job_id' => job_id)
-          raise Exception.new("Page not found with job_id \"#{job_id}\" gid \"#{gid}\"") if page.nil?
-          page['status'] = 'to_parse'
-          page['parsing_at'] = nil
-          page['parsing_failed_at'] = nil
-          page['parsing_updated_at'] = '2001-01-01T00:00:00Z'
-          page['parsed_at'] = nil
-          page['parsing_try_count'] = 0
-          page['parsing_fail_count'] = 0
-        end
-      end
-    end
-  end
-end