answersengine 0.2.33 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: af3d2a9258208b8fa48b641da2c5965551390519
-  data.tar.gz: 0ed05cf695e1237c00f55c4914fdf625d8564a59
+SHA256:
+  metadata.gz: b4c9499d52facc546f3f78c357de5b3a2b625e0668ef77bbaa3053e2f8940861
+  data.tar.gz: ce03196a46bcb2ca53e753de266fa6129c320db8e645e6fd753d833bcdf3a12d
 SHA512:
-  metadata.gz: cd4096df084c799fdb90aeb832918e34e15f1d5e7374b23af7fcbbde443fb1f10087ef5ce0dce41a512d39d516f2fbb2675fc9947a5cfc44be7313218af66119
-  data.tar.gz: a83793d9eceb6ff07bce795f0e35941cd24af87658a32ecdee5e79795d72139ac5cbc506b7d653ebf48f0debd243c3974a3856a06545b61ff9eca3c9b7c134e6
+  metadata.gz: dcd0280c93f52604723c9df3ace160654c8d4bed3e9568d8136046908e8be40b2078b9fe343bb21b58a28966b7954102231c53b53c83c5b79542302ac57e83eb
+  data.tar.gz: 47a80c92802ce98c2c86f9c3c099efd532fed95fd9f640f2c985f26b0a1596f6361eb67c4575eef342904d28a76296bd27bc6aacfeb794a01d7e79923297124c

data/lib/answersengine/client/auth_token.rb

@@ -0,0 +1,50 @@
+module AnswersEngine
+  module Client
+    class AuthToken < AnswersEngine::Client::Base
+
+      def find(token)
+        self.class.get("/auth_tokens/#{token}", @options)
+      end
+
+      def all(opts={})
+        self.class.get("/auth_tokens", @options)
+      end
+
+      def create(role, description, opts={})
+        body = {
+            role: role,
+            description: description}
+
+        @options.merge!({body: body.to_json})
+        self.class.post("/auth_tokens", @options)
+      end
+
+      def create_on_account(account_id, role, description)
+        body = {
+            role: role,
+            description: description}
+
+        @options.merge!({body: body.to_json})
+        self.class.post("/accounts/#{account_id}/auth_tokens", @options)
+      end
+
+      def update(token, role, description="", opts={})
+        body = {}
+
+        body[:role] = role
+        body[:description] = description if description.present?
+        @options.merge!({body: body.to_json})
+
+        self.class.put("/auth_tokens/#{token}", @options)
+      end
+
+      def delete(token, opts={})
+        body = {}
+        @options.merge!({body: body.to_json})
+
+        self.class.delete("/auth_tokens/#{token}", @options)
+      end
+    end
+  end
+end
+

data/lib/answersengine/client/base.rb

@@ -4,12 +4,15 @@ module AnswersEngine
   module Client
     class Base
       include HTTParty
-      base_uri(ENV['ANSWERSENGINE_API_URL'].nil? ? 'https://fetch.answersengine.com/api/v1' : ENV['ANSWERSENGINE_API_URL'])
 
       def self.env_auth_token
         ENV['ANSWERSENGINE_TOKEN']
       end
 
+      def env_api_url
+        ENV['ANSWERSENGINE_API_URL'].nil? ? 'https://fetch.answersengine.com/api/v1' : ENV['ANSWERSENGINE_API_URL']
+      end
+
       def auth_token
         @auth_token ||= self.class.env_auth_token
       end
@@ -19,6 +22,7 @@ module AnswersEngine
       end
 
       def initialize(opts={})
+        self.class.base_uri(env_api_url)
         self.auth_token = opts[:auth_token] unless opts[:auth_token].nil?
         @options = { headers: {
           "Authorization" => "Bearer #{auth_token}",

data/lib/answersengine/client.rb

@@ -1,4 +1,5 @@
 require "answersengine/client/base"
+require "answersengine/client/auth_token"
 require 'answersengine/client/export'
 require "answersengine/client/scraper"
 require "answersengine/client/scraper_deployment"

data/lib/answersengine/scraper/executor.rb

@@ -191,29 +191,88 @@ module AnswersEngine
         result.respond_to?(:first) ? result.first : nil
       end
 
+      # Remove dups by prioritizing the latest dup.
+      #
+      # @param [Array] list List of hashes to dedup.
+      # @param [Hash] key_defaults Key and default value pair hash to use on
+      #   uniq validation.
+      #
+      # @return [Integer] Removed duplicated items count.
+      def remove_old_dups!(list, key_defaults)
+        raw_count = list.count
+        keys = key_defaults.keys
+        force_uniq = 0
+        list.reverse!.uniq! do |item|
+          # Extract stringify keys as hash
+          key_hash = Hash[item.map{|k,v|keys.include?(k.to_s) ? [k.to_s,v] : nil}.select{|i|!i.nil?}]
+
+          # Apply defaults for uniq validation
+          key_defaults.each{|k,v| key_hash[k] = v if key_hash[k].nil?}
+
+          # Don't dedup nil key defaults
+          skip_dedup = !keys.find{|k| key_hash[k].nil?}.nil?
+          skip_dedup ? (force_uniq += 1) : key_hash
+        end
+        list.reverse!
+        dup_count = raw_count - list.count
+        dup_count
+      end
+
+      # Remove page dups by prioritizing the latest dup.
+      #
+      # @param [Array] list List of pages to dedup.
+      #
+      # @return [Integer] Removed duplicated items count.
+      #
+      # @note It will not dedup for now as it is hard to build gid.
+      #   TODO: Build gid so we can dedup
+      def remove_old_page_dups!(list)
+        key_defaults = {
+          'gid' => nil
+        }
+        remove_old_dups! list, key_defaults
+      end
+
+      # Remove dups by prioritizing the latest dup.
+      #
+      # @param [Array] list List of outputs to dedup.
+      #
+      # @return [Integer] Removed duplicated items count.
+      def remove_old_output_dups!(list)
+        key_defaults = {
+          '_id' => nil,
+          '_collection' => 'default'
+        }
+        remove_old_dups! list, key_defaults
+      end
+
       def save_pages_and_outputs(pages = [], outputs = [], status)
         total_pages = pages.count
         total_outputs = outputs.count
         records_per_slice = 100
         until pages.empty? && outputs.empty?
           pages_slice = pages.shift(records_per_slice)
+          pages_dup_count = remove_old_page_dups! pages_slice
           outputs_slice = outputs.shift(records_per_slice)
+          outputs_dup_count = remove_old_output_dups! outputs_slice
 
           log_msgs = []
           unless pages_slice.empty?
-            log_msgs << "#{pages_slice.count} out of #{total_pages} Pages"
+            page_dups_ignored = pages_dup_count > 0 ? " (#{pages_dup_count} dups ignored)" : ''
+            log_msgs << "#{pages_slice.count} out of #{total_pages} Pages#{page_dups_ignored}"
             unless save
               puts '----------------------------------------'
-              puts "Would have saved #{log_msgs.last}"
+              puts "Would have saved #{log_msgs.last}#{page_dups_ignored}"
               puts JSON.pretty_generate pages_slice
             end
           end
 
           unless outputs_slice.empty?
-            log_msgs << "#{outputs_slice.count} out of #{total_outputs} Outputs"
+            output_dups_ignored = outputs_dup_count > 0 ? " (#{outputs_dup_count} dups ignored)" : ''
+            log_msgs << "#{outputs_slice.count} out of #{total_outputs} Outputs#{output_dups_ignored}"
             unless save
               puts '----------------------------------------'
-              puts "Would have saved #{log_msgs.last}"
+              puts "Would have saved #{log_msgs.last}#{output_dups_ignored}"
               puts JSON.pretty_generate outputs_slice
             end
           end
@@ -279,7 +338,7 @@ module AnswersEngine
 
       # Eval a filename with a custom binding
       #
-      # @param [String] filename File path to read.
+      # @param [String] file_path File path to read.
       # @param [Binding] context Context binding to evaluate with.
       #
       # @note Using this method will allow scripts to contain `return` to

data/lib/answersengine/version.rb

@@ -1,3 +1,3 @@
 module AnswersEngine
-  VERSION = "0.2.33"
+  VERSION = "0.3.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: answersengine
 version: !ruby/object:Gem::Version
-  version: 0.2.33
+  version: 0.3.0
 platform: ruby
 authors:
 - Parama Danoesubroto
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2019-03-11 00:00:00.000000000 Z
+date: 2019-03-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: thor
@@ -201,6 +201,7 @@ files:
 - lib/answersengine/cli/scraper_page.rb
 - lib/answersengine/cli/seeder.rb
 - lib/answersengine/client.rb
+- lib/answersengine/client/auth_token.rb
 - lib/answersengine/client/backblaze_content.rb
 - lib/answersengine/client/base.rb
 - lib/answersengine/client/export.rb
@@ -248,7 +249,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.6.14.1
+rubygems_version: 2.7.6
 signing_key: 
 specification_version: 4
 summary: AnswersEngine toolbelt for developers