ae_easy-core 0.0.4 → 0.0.5

data/doc/top-level-namespace.html

@@ -100,7 +100,7 @@
 </div>
 
       <div id="footer">
-  Generated on Wed Feb 13 21:43:50 2019 by
+  Generated on Wed Feb 20 18:03:17 2019 by
   <a href="http://yardoc.org" title="Yay! A Ruby Documentation Tool" target="_parent">yard</a>
   0.9.18 (ruby-2.5.3).
 </div>

data/lib/ae_easy/core/mock/fake_db.rb

@@ -7,6 +7,15 @@ module AeEasy
         PAGE_KEYS = ['gid'].freeze
         # Output id keys, analog to primary keys.
         OUTPUT_KEYS = ['_id', '_collection'].freeze
+        # Job id keys, analog to primary keys.
+        JOB_KEYS = ['job_id'].freeze
+        # Job available status.
+        JOB_STATUSES = {
+          active: 'active',
+          done: 'done',
+          cancelled: 'cancelled',
+          paused: 'paused'
+        }
         # Default collection for saved outputs
         DEFAULT_COLLECTION = 'default'
 
@@ -37,8 +46,11 @@ module AeEasy
         #
         # @return [Hash]
         def self.build_page page, opts = {}
+          opts = {
+            allow_page_gid_override: true,
+            allow_job_id_override: true
+          }.merge opts
           temp_db = AeEasy::Core::Mock::FakeDb.new opts
-          temp_db.enable_page_gid_override
           temp_db.pages << page
           temp_db.pages.first
         end
@@ -56,15 +68,122 @@ module AeEasy
           build_page page, opts
         end
 
+        # Clean an URL to remove fragment, lowercase schema and host, and sort
+        #   query string.
+        #
+        # @param [String] raw_url URL to clean.
+        #
+        # @return [String]
+        def self.clean_uri raw_url
+          url = URI.parse(raw_url)
+          url.hostname = url.hostname.downcase
+          url.fragment = nil
+
+          # Sort query string keys
+          unless url.query.nil?
+            query_string = CGI.parse(url.query)
+            keys = query_string.keys.sort
+            data = []
+            keys.each do |key|
+              query_string[key].each do |value|
+                data << "#{URI.encode key}=#{URI.encode value}"
+              end
+            end
+            url.query = data.join('&')
+          end
+          url.to_s
+        end
+
+        # Format headers for gid generation.
+        # @private
+        #
+        # @param [Hash|nil] headers Headers hash.
+        #
+        # @return [Hash]
+        def self.format_headers headers
+          return {} if headers.nil?
+          data = {}
+          headers.each do |key, value|
+            unless value.is_a? Array
+              data[key] = value
+              next
+            end
+            data[key] = value.sort
+          end
+          data
+        end
+
+        # Build a job with defaults by using FakeDb engine.
+        #
+        # @param [Hash] job Job initial values.
+        # @param [Hash] opts ({}) Configuration options (see #initialize).
+        #
+        # @return [Hash]
+        def self.build_job job, opts = {}
+          temp_db = AeEasy::Core::Mock::FakeDb.new opts
+          temp_db.jobs << job
+          temp_db.jobs.last
+        end
+
+        # Build a fake job by using FakeDb engine.
+        #
+        # @param [Hash] opts ({}) Configuration options (see #initialize).
+        # @option opts [String] :scraper_name (nil) Scraper name.
+        # @option opts [Integer] :job_id (nil) Job id.
+        # @option opts [String] :status ('done').
+        #
+        # @return [Hash]
+        def self.build_fake_job opts = {}
+          job = {
+            'job_id' => opts[:job_id],
+            'scraper_name' => opts[:scraper_name],
+            'status' => (opts[:status] || 'done')
+          }
+          build_job job, opts
+        end
+
+        # Get current job or create new one from values.
+        #
+        # @param [Integer] target_job_id (nil) Job id to ensure existance.
+        #
+        # @return [Hash]
+        def ensure_job target_job_id = nil
+          target_job_id = job_id if target_job_id.nil?
+          job = jobs.find{|v|v['job_id'] == target_job_id}
+          return job unless job.nil?
+          job = {
+            'job_id' => target_job_id,
+            'scraper_name' => scraper_name,
+          }
+          job['status'] = 'active' unless target_job_id != job_id
+          jobs << job
+          jobs.last
+        end
+
+        # Fake scraper_name.
+        # @return [String,nil]
+        def scraper_name
+          @scraper_name ||= 'my_scraper'
+        end
+
+        # Set fake scraper_name value.
+        def scraper_name= value
+          job = ensure_job
+          @scraper_name = value
+          job['scraper_name'] = scraper_name
+        end
+
         # Fake job id.
         # @return [Integer,nil]
         def job_id
-          @job_id ||= rand(1000) + 1
+          @job_id ||= generate_job_id
         end
 
         # Set fake job id value.
         def job_id= value
           @job_id = value
+          ensure_job
+          job_id
         end
 
         # Current fake page gid.
@@ -78,35 +197,102 @@ module AeEasy
           @page_gid = value
         end
 
-        # Enable page gid override on page insert.
+        # Enable page gid override on page or output insert.
         def enable_page_gid_override
           @allow_page_gid_override = true
         end
 
-        # Disable page gid override on page insert.
+        # Disable page gid override on page or output insert.
         def disable_page_gid_override
           @allow_page_gid_override = false
         end
 
-        # Specify whenever page gid overriding by user is allowed on page
-        #   insert.
+        # Specify whenever page gid overriding by user is allowed on page or
+        #   output insert.
         #
         # @return [Boolean] `true` when allowed, else `false`.
         def allow_page_gid_override?
           @allow_page_gid_override ||= false
         end
 
+        # Enable job id override on page or output insert.
+        def enable_job_id_override
+          @allow_job_id_override = true
+        end
+
+        # Disable job id override on page or output insert.
+        def disable_job_id_override
+          @allow_job_id_override = false
+        end
+
+        # Specify whenever job id overriding by user is allowed on page or
+        #   output insert.
+        #
+        # @return [Boolean] `true` when allowed, else `false`.
+        def allow_job_id_override?
+          @allow_job_id_override ||= false
+        end
+
         # Initialize fake database.
         #
         # @param [Hash] opts ({}) Configuration options.
         # @option opts [Integer,nil] :job_id Job id default value.
+        # @option opts [String,nil] :scraper_name Scraper name default value.
         # @option opts [String,nil] :page_gid Page gid default value.
         # @option opts [Boolean, nil] :allow_page_gid_override (false) Specify
-        #   whenever page gid can be overrided on page insert.
+        #   whenever page gid can be overrided on page or output insert.
+        # @option opts [Boolean, nil] :allow_job_id_override (false) Specify
+        #   whenever job id can be overrided on page or output insert.
         def initialize opts = {}
           self.job_id = opts[:job_id]
+          self.scraper_name = opts[:scraper_name]
           self.page_gid = opts[:page_gid]
           @allow_page_gid_override = opts[:allow_page_gid_override].nil? ? false : !!opts[:allow_page_gid_override]
+          @allow_job_id_override = opts[:allow_job_id_override].nil? ? false : !!opts[:allow_job_id_override]
+        end
+
+        # Generate a fake scraper name.
+        #
+        # @return [String]
+        def generate_scraper_name
+          Faker::Internet.unique.slug
+        end
+
+        # Generate a fake job_id.
+        #
+        # @return [Integer]
+        def generate_job_id
+          jobs.count < 1 ? 1 : (jobs.max{|a,b|a['job_id'] <=> b['job_id']}['job_id'] + 1)
+        end
+
+        # Get output keys with key generators to emulate saving on db.
+        # @private
+        #
+        # @return [Hash]
+        def job_defaults
+          @job_defaults ||= {
+            'job_id' => lambda{|job| generate_job_id},
+            'scraper_name' => lambda{|job| generate_scraper_name},
+            'status' => 'done',
+            'created_at' => lambda{|job| Time.now}
+          }
+        end
+
+        # Stored job collection
+        #
+        # @return [AeEasy::Core::SmartCollection]
+        def jobs
+          return @jobs unless @jobs.nil?
+          collection = self.class.new_collection JOB_KEYS,
+            defaults: job_defaults
+          collection.bind_event(:before_defaults) do |collection, raw_item|
+            AeEasy::Core.deep_stringify_keys raw_item
+          end
+          collection.bind_event(:before_insert) do |collection, item, match|
+            item['job_id'] ||= generate_job_id
+            item
+          end
+          @jobs ||= collection
         end
 
         # Generate a fake UUID based on page data:
@@ -119,10 +305,10 @@ module AeEasy
         #   * body
         #   * ua_type
         #
-        # @param [Hash] data Output data.
+        # @param [Hash] page_data Page data.
         #
         # @return [String]
-        def generate_page_gid data
+        def generate_page_gid page_data
           fields = [
             'url',
             'method',
@@ -133,8 +319,13 @@ module AeEasy
             'body',
             'ua_type'
           ]
+          data = page_data.select{|k,v|fields.include? k}
+          data['url'] = self.class.clean_uri data['url']
+          data['headers'] = self.class.format_headers data['headers']
+          data['cookie'] = AeEasy::Core::Helper::Cookie.parse_from_request data['cookie'] unless data['cookie'].nil?
           seed = data.select{|k,v|fields.include? k}.hash
-          self.class.fake_uuid seed
+          checksum = self.class.fake_uuid seed
+          "#{URI.parse(data['url']).hostname}-#{checksum}"
         end
 
         # Get page keys with key generators to emulate saving on db.
@@ -142,8 +333,9 @@ module AeEasy
         #
         # @return [Hash]
         def page_defaults
-          @page_keys ||= {
+          @page_defaults ||= {
             'url' => nil,
+            'job_id' => lambda{|page| job_id},
             'method' => 'GET',
             'headers' => {},
             'fetch_type' => 'standard',
@@ -168,7 +360,9 @@ module AeEasy
           collection = self.class.new_collection PAGE_KEYS,
             defaults: page_defaults
           collection.bind_event(:before_defaults) do |collection, raw_item|
-            AeEasy::Core.deep_stringify_keys raw_item
+            item = AeEasy::Core.deep_stringify_keys raw_item
+            item.delete 'job_id' unless allow_job_id_override?
+            item
           end
           collection.bind_event(:before_insert) do |collection, item, match|
             if item['gid'].nil? || !allow_page_gid_override?
@@ -194,7 +388,7 @@ module AeEasy
         #
         # @return [Hash]
         def output_defaults
-          @output_keys ||= {
+          @output_defaults ||= {
             '_collection' => DEFAULT_COLLECTION,
             '_job_id' => lambda{|output| job_id},
             '_created_at' => lambda{|output| Time.new.strftime('%Y-%m-%dT%H:%M:%SZ')},
@@ -210,12 +404,18 @@ module AeEasy
           collection = self.class.new_collection OUTPUT_KEYS,
             defaults: output_defaults
           collection.bind_event(:before_defaults) do |collection, raw_item|
-            AeEasy::Core.deep_stringify_keys raw_item
+            item = AeEasy::Core.deep_stringify_keys raw_item
+            item.delete '_job_id' unless allow_job_id_override?
+            item.delete '_gid_id' unless allow_page_gid_override?
+            item
           end
           collection.bind_event(:before_insert) do |collection, item, match|
             item['_id'] ||= generate_output_id item
             item
           end
+          collection.bind_event(:after_insert) do |collection, item|
+            ensure item['job_id']
+          end
           @outputs ||= collection
         end
 
@@ -255,6 +455,8 @@ module AeEasy
             outputs
           when :pages
             pages
+          when :jobs
+            jobs
           else
             raise ArgumentError.new "Unknown collection #{collection}."
           end

data/lib/ae_easy/core/mock/fake_executor.rb

@@ -3,6 +3,9 @@ module AeEasy
     module Mock
       # Fake executor that emulates `AnswersEngine` executor.
       module FakeExecutor
+        # Max allowed page size when query outputs (see #find_outputs).
+        MAX_FIND_OUTPUTS_PER_PAGE = 500
+
         # Page content.
         # @return [String,nil]
         attr_accessor :content
@@ -92,16 +95,28 @@ module AeEasy
           end
           @outputs = opts[:outputs]
           self.job_id = opts[:job_id]
+          self.scraper_name = opts[:scraper_name]
           self.page = opts[:page]
         end
 
+        # Fake scraper name used by executor.
+        # @return [Integer,nil]
+        def scraper_name
+          db.scraper_name
+        end
+
+        # Set fake scraper name value.
+        def scraper_name= value
+          db.scraper_name = value
+        end
+
         # Fake job ID used by executor.
         # @return [Integer,nil]
         def job_id
           db.job_id
         end
 
-        # Set fake job id value.
+        # Set fake job ID value.
         def job_id= value
           db.job_id = value
           page['job_id'] = value
@@ -124,6 +139,11 @@ module AeEasy
           @page = value
         end
 
+        # Retrive a list of saved jobs.
+        def saved_jobs
+          db.jobs
+        end
+
         # Retrive a list of saved pages. Drafted pages can be included.
         def saved_pages
           db.pages
@@ -134,18 +154,29 @@ module AeEasy
           db.outputs
         end
 
-        # Save a page collection on db.
+        # Save a job collection on db and remove all the element from +list+.
+        #
+        # @param [Array] list Collection of jobs to save.
+        def save_jobs list
+          list.each{|job| db.jobs << job}
+          list.clear
+        end
+
+        # Save a page collection on db and remove all the element from +list+.
         #
         # @param [Array] list Collection of pages to save.
         def save_pages list
           list.each{|page| db.pages << page}
+          list.clear
         end
 
-        # Save an output collection on db.
+        # Save an output collection on db and remove all the element from
+        #   +list+.
         #
         # @param [Array] list Collection of outputs to save.
         def save_outputs list
           list.each{|output| db.outputs << output}
+          list.clear
         end
 
         # Save draft pages into db and clear draft queue.
@@ -166,19 +197,71 @@ module AeEasy
           flush_outputs
         end
 
+        # Get latest job by scraper_name.
+        #
+        # @param [String] scraper_name Scraper name.
+        # @param [Hash] filter ({}) Additional_filters.
+        #
+        # @return [Hash|nil] Return nil if no scraper_name or scraper_name is
+        #   nil.
+        def latest_job_by scraper_name, filter = {}
+          return nil if scraper_name.nil?
+          data = db.query :jobs, filter.merge('scraper_name' => scraper_name)
+          data.max{|a,b| a['created_at'] <=> b['created_at']}
+        end
+
         # Find outputs by collection and query with pagination.
         #
         # @param [String] collection ('default') Collection name.
         # @param [Hash] query ({}) Filters to query.
         # @param [Integer] page (1) Page number.
         # @param [Integer] per_page (30) Page size.
+        # @param [Hash] opts ({}) Configuration options.
+        # @option opts [String|nil] :scraper_name (nil) Scraper name to query
+        #   from.
+        # @option opts [Integer|nil] :job_id (nil) Job's id to query from.
+        #
+        # @raise [ArgumentError] +collection+ is not String.
+        # @raise [ArgumentError] +query+ is not a Hash.
+        # @raise [ArgumentError] +page+ is not an Integer greater than 0.
+        # @raise [ArgumentError] +per_page+ is not an Integer between 1 and 500.
         #
         # @return [Array]
-        def find_outputs collection = 'default', query = {}, page = 1, per_page = 30
+        #
+        # @example
+        #   find_outputs
+        # @example
+        #   find_outputs 'my_collection'
+        # @example
+        #   find_outputs 'my_collection', {}
+        # @example
+        #   find_outputs 'my_collection', {}, 1
+        # @example
+        #   find_outputs 'my_collection', {}, 1, 30
+        # @example Find from another scraper by name
+        #   find_outputs 'my_collection', {}, 1, 30, scraper_name: 'my_scraper'
+        # @example Find from another scraper by job_id
+        #   find_outputs 'my_collection', {}, 1, 30, job_id: 123
+        #
+        # @note *opts `:job_id` option is prioritize over `:scraper_name` when
+        #   both exists. If none add provided or nil values, then current job
+        #   will be used to query instead, this is the defaul behavior.
+        def find_outputs collection = 'default', query = {}, page = 1, per_page = 30, opts = {}
+          raise ArgumentError.new("collection needs to be a String.") unless collection.is_a?(String)
+          raise ArgumentError.new("query needs to be a Hash.") unless query.is_a?(Hash)
+          unless page.is_a?(Integer) && page > 0
+            raise ArgumentError.new("page needs to be an Integer greater than 0.")
+          end
+          unless per_page.is_a?(Integer) && per_page > 0 && per_page <= MAX_FIND_OUTPUTS_PER_PAGE
+            raise ArgumentError.new("per_page needs to be an Integer between 1 and #{MAX_FIND_OUTPUTS_PER_PAGE}.")
+          end
+
           count = 0
           offset = (page - 1) * per_page
+          job = latest_job_by(opts[:scraper_name])
           fixed_query = query.merge(
-            '_collection' => collection
+            '_collection' => collection,
+            '_job_id' => opts[:job_id] || (job.nil? ? job_id : job['job_id'])
           )
           db.query :outputs, fixed_query, offset, per_page
         end
@@ -187,10 +270,32 @@ module AeEasy
         #
         # @param [String] collection ('default') Collection name.
         # @param [Hash] query ({}) Filters to query.
+        # @param [Hash] opts ({}) Configuration options.
+        # @option opts [String|nil] :scraper_name (nil) Scraper name to query
+        #   from.
+        # @option opts [Integer|nil] :job_id (nil) Job's id to query from.
+        #
+        # @raise [ArgumentError] +collection+ is not String.
+        # @raise [ArgumentError] +query+ is not a Hash.
         #
         # @return [Hash, nil]
-        def find_output collection = 'default', query = {}
-          result = find_outputs(collection, query, 1, 1)
+        #
+        # @example
+        #   find_output
+        # @example
+        #   find_output 'my_collection'
+        # @example
+        #   find_output 'my_collection', {}
+        # @example Find from another scraper by name
+        #   find_output 'my_collection', {}, scraper_name: 'my_scraper'
+        # @example Find from another scraper by job_id
+        #   find_output 'my_collection', {}, job_id: 123
+        #
+        # @note *opts `:job_id` option is prioritize over `:scraper_name` when
+        #   both exists. If none add provided or nil values, then current job
+        #   will be used to query instead, this is the defaul behavior.
+        def find_output collection = 'default', query = {}, opts = {}
+          result = find_outputs(collection, query, 1, 1, opts)
           result.nil? ? nil : result.first
         end
 

data/lib/ae_easy/core/version.rb

@@ -1,6 +1,6 @@
 module AeEasy
   module Core
     # Gem version
-    VERSION = "0.0.4"
+    VERSION = "0.0.5"
   end
 end

data/lib/ae_easy/core.rb

@@ -1,4 +1,5 @@
 require 'time'
+require 'faker'
 require 'answersengine'
 require 'ae_easy/core/smart_collection'
 require 'ae_easy/core/exception'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ae_easy-core
 version: !ruby/object:Gem::Version
-  version: 0.0.4
+  version: 0.0.5
 platform: ruby
 authors:
 - Eduardo Rosales
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-02-13 00:00:00.000000000 Z
+date: 2019-02-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: answersengine
@@ -16,14 +16,28 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.2.25
+        version: 0.2.32
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.2.25
+        version: 0.2.32
+- !ruby/object:Gem::Dependency
+  name: faker
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -134,7 +148,6 @@ files:
 - ".yardopts"
 - CODE_OF_CONDUCT.md
 - Gemfile
-- Gemfile.lock
 - LICENSE
 - README.md
 - Rakefile

data/Gemfile.lock

@@ -1,57 +0,0 @@
-PATH
-  remote: .
-  specs:
-    ae_easy-core (0.0.4)
-      answersengine (>= 0.2.25)
-
-GEM
-  remote: https://rubygems.org/
-  specs:
-    ansi (1.5.0)
-    answersengine (0.2.31)
-      httparty (~> 0.16.2)
-      nokogiri (~> 1.6, < 1.10)
-      thor (~> 0.20.3)
-    byebug (10.0.2)
-    docile (1.3.1)
-    hirb (0.7.3)
-    httparty (0.16.3)
-      mime-types (~> 3.0)
-      multi_xml (>= 0.5.2)
-    json (2.1.0)
-    mime-types (3.2.2)
-      mime-types-data (~> 3.2015)
-    mime-types-data (3.2018.0812)
-    mini_portile2 (2.4.0)
-    minitest (5.11.3)
-    multi_xml (0.6.0)
-    nokogiri (1.9.1)
-      mini_portile2 (~> 2.4.0)
-    rake (10.5.0)
-    simplecov (0.16.1)
-      docile (~> 1.1)
-      json (>= 1.8, < 3)
-      simplecov-html (~> 0.10.0)
-    simplecov-console (0.4.2)
-      ansi
-      hirb
-      simplecov
-    simplecov-html (0.10.2)
-    thor (0.20.3)
-    timecop (0.9.1)
-
-PLATFORMS
-  ruby
-
-DEPENDENCIES
-  ae_easy-core!
-  bundler (>= 1.16.3)
-  byebug
-  minitest (>= 5.11)
-  rake (>= 10.0)
-  simplecov (>= 0.16.1)
-  simplecov-console (>= 0.4.2)
-  timecop (>= 0.9.1)
-
-BUNDLED WITH
-   1.17.3