answersengine 0.10.1 → 0.10.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: df0c5550c9fbe72ceeac38e0d921331972a476f99071e4c2427d9d1b1fe27d50
-  data.tar.gz: 804beea78e0069464f7761022d73d838c2ebf939c2a3e855871dc31810ea6c2b
+  metadata.gz: e7a6b853369f925f1537eb535fc8ebb3dd45fc82195b67bdd68db6452fee5eea
+  data.tar.gz: 84616bf96239a2cf6718e98c046bebdc849a1b527a04ff1363dbc2bfb466804a
 SHA512:
-  metadata.gz: adfac18949a0d32220086b1b244f561021e6ae19bb632a803ac30a2ec2cd396b582907e2eb5080b5d9cdfc68657ac5e3d93479aa142e9b6b1bb7369332dfa510
-  data.tar.gz: 56ffed752b3953ba1c4b94bf99c6989e4bf72852a94736bece31f9ac0e666e42e70b16656d6f6d31e242e0792fe4934d1ede61b8fee570402e40695af20a5e84
+  metadata.gz: a0a10c9b24b90c2224a212fdebb48cef6d20f67cae6e468202c0b91923ac237dc88456e2159276986d180ec0e214b066e1fe9ffbfaed3b040c0a6c7a23723011
+  data.tar.gz: 0f00ab279ae62d1226f2c4db7fc3dee173d2e255ac7abf67d66f4c4a1ccd1dc031e66a1336e92645ad666b04128ea3b08c636c546132918ed2bf161f62ed6ccb

data/CODE_OF_CONDUCT.md

@@ -55,7 +55,7 @@ further defined and clarified by project maintainers.
 ## Enforcement
 
 Instances of abusive, harassing, or otherwise unacceptable behavior may be
-reported by contacting the project team at parama@answersengine.com. All
+reported by contacting the project team at perry@datahen.com. All
 complaints will be reviewed and investigated and will result in a response that
 is deemed necessary and appropriate to the circumstances. The project team is
 obligated to maintain confidentiality with regard to the reporter of an incident.

data/LICENSE.txt

@@ -1,6 +1,6 @@
 The MIT License (MIT)
 
-Copyright (c) 2018 Parama Danoesubroto
+Copyright (c) 2019 Parama Danoesubroto
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

data/README.md

@@ -1,6 +1,6 @@
-# AnswersEngine
+# AnswersEngine (Deprecated)
 
-Welcome to the AnswersEngine gem, this gem includes the client and the CLI to be able to integrate with answersengine.com
+Welcome to the AnswersEngine gem, this gem includes the client and the CLI as an alias to [datahen](https://github.com/DataHenOfficial/datahen-ruby) gem client and CLI, to be able to integrate with datahen.com. It has been deprecated, please use [datahen](https://github.com/DataHenOfficial/datahen-ruby) gem instead.
 
 ## Installation
 
@@ -22,9 +22,8 @@ Or install it yourself as:
 
 ## Contributing
 
-Bug reports and pull requests are welcome on GitHub at https://github.com/[USERNAME]/answersengine. This project is intended to be a safe, welcoming space for collaboration, and contributors are expected to adhere to the [Contributor Covenant](http://contributor-covenant.org) code of conduct.
+Bug reports and pull requests are welcome on GitHub at https://github.com/DataHenOfficial/datahen-ruby. This project is intended to be a safe, welcoming space for collaboration, and contributors are expected to adhere to the [Contributor Covenant](http://contributor-covenant.org) code of conduct.
 
 ## License
 
 The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).
-

data/answersengine.gemspec

@@ -1,17 +1,16 @@
 
 lib = File.expand_path("../lib", __FILE__)
 $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
-require "answersengine/version"
 
 Gem::Specification.new do |spec|
   spec.name          = "answersengine"
-  spec.version       = AnswersEngine::VERSION
+  spec.version       = "0.10.2"
   spec.authors       = ["Parama Danoesubroto"]
-  spec.email         = ["parama@answersengine.com"]
+  spec.email         = ["perry@datahen.com"]
 
-  spec.summary       = %q{AnswersEngine toolbelt for developers}
-  spec.description   = %q{AnswersEngine toolbelt to develop scrapers and other scripts}
-  spec.homepage      = "https://answersengine.com"
+  spec.summary       = %q{(Deprecated: Use datahen gem instead.) Compatibility alias for DataHen toolbelt for developers.}
+  spec.description   = %q{(Deprecated: Use datahen gem instead.) Compatibility alias for DataHen toolbelt to develop scrapers and other scripts.}
+  spec.homepage      = "https://datahen.com"
   spec.license       = "MIT"
 
   # Prevent pushing this gem to RubyGems.org. To allow pushes either set the 'allowed_push_host'
@@ -34,14 +33,9 @@ Gem::Specification.new do |spec|
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
   spec.required_ruby_version = '>= 2.2.2'
-  spec.add_dependency "thor", "~> 0.20.3"
-  spec.add_dependency 'httparty', '~> 0.16.2'
-  spec.add_dependency 'nokogiri', '~> 1.6', '< 1.10'
+  spec.add_dependency "datahen", ">= 0"
   spec.add_development_dependency 'bundler', '>= 1.16'
   spec.add_development_dependency 'rake', '>= 10.0'
   spec.add_development_dependency 'minitest', '>= 5.11'
-  spec.add_development_dependency 'simplecov', '>= 0.16.1'
-  spec.add_development_dependency 'simplecov-console', '>= 0.4.2'
-  spec.add_development_dependency 'timecop', '>= 0.9.1'
   spec.add_development_dependency 'byebug', '>= 0'
 end

data/exe/answersengine

@@ -1,3 +1,4 @@
 #!/usr/bin/env ruby
-require 'answersengine/cli'
-AnswersEngine::CLI.start
+require 'answersengine'
+require 'datahen/cli'
+Datahen::CLI.start

data/lib/answersengine.rb

@@ -1,5 +1,22 @@
-require "answersengine/version"
-require "answersengine/scraper"
+require "datahen"
 
-module AnswersEngine
+# Override env variables methods to include `ANSWERSENGINE_*` env variables
+module Datahen
+  module Client
+    class Base
+      def self.env_auth_token
+        ENV['DATAHEN_TOKEN'].nil? ? ENV['ANSWERSENGINE_TOKEN'] : ENV['DATAHEN_TOKEN']
+      end
+
+      def env_api_url
+        return ENV['DATAHEN_API_URL'] unless ENV['DATAHEN_API_URL'].nil?
+        ENV['ANSWERSENGINE_API_URL'].nil? ? 'https://app.datahen.com/api/v1' : ENV['ANSWERSENGINE_API_URL']
+      end
+    end
+  end
 end
+
+ENV['ANSWERSENGINE_TOKEN'] = ENV['DATAHEN_TOKEN'] if ENV['ANSWERSENGINE_TOKEN'].to_s.strip == ''
+
+# (Deprecated) Alias to Datahen module.
+AnswersEngine = ::Datahen

metadata

@@ -1,63 +1,29 @@
 --- !ruby/object:Gem::Specification
 name: answersengine
 version: !ruby/object:Gem::Version
-  version: 0.10.1
+  version: 0.10.2
 platform: ruby
 authors:
 - Parama Danoesubroto
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2019-11-08 00:00:00.000000000 Z
+date: 2019-12-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: thor
+  name: datahen
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: 0.20.3
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: 0.20.3
-- !ruby/object:Gem::Dependency
-  name: httparty
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: 0.16.2
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: 0.16.2
-- !ruby/object:Gem::Dependency
-  name: nokogiri
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1.6'
-    - - "<"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1.10'
+        version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1.6'
-    - - "<"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1.10'
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -100,48 +66,6 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '5.11'
-- !ruby/object:Gem::Dependency
-  name: simplecov
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 0.16.1
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 0.16.1
-- !ruby/object:Gem::Dependency
-  name: simplecov-console
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 0.4.2
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 0.4.2
-- !ruby/object:Gem::Dependency
-  name: timecop
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 0.9.1
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 0.9.1
 - !ruby/object:Gem::Dependency
   name: byebug
   requirement: !ruby/object:Gem::Requirement
@@ -156,9 +80,10 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-description: AnswersEngine toolbelt to develop scrapers and other scripts
+description: "(Deprecated: Use datahen gem instead.) Compatibility alias for DataHen
+  toolbelt to develop scrapers and other scripts."
 email:
-- parama@answersengine.com
+- perry@datahen.com
 executables:
 - answersengine
 extensions: []
@@ -174,78 +99,14 @@ files:
 - answersengine.gemspec
 - bin/console
 - bin/setup
-- examples/fetchtest/libraries/hello.rb
-- examples/fetchtest/libraries/hello_fail.rb
-- examples/fetchtest/parsers/failed.rb
-- examples/fetchtest/parsers/find_outputs.rb
-- examples/fetchtest/parsers/home.rb
-- examples/fetchtest/parsers/nested_fail.rb
-- examples/fetchtest/parsers/simple.rb
-- examples/fetchtest/seeders/csv_seeder.rb
-- examples/fetchtest/seeders/failed.rb
-- examples/fetchtest/seeders/list_of_urls.csv
-- examples/fetchtest/seeders/seed.rb
-- examples/fetchtest/seeders/test_reset_page.rb
 - exe/answersengine
 - lib/answersengine.rb
-- lib/answersengine/cli.rb
-- lib/answersengine/cli/env_var.rb
-- lib/answersengine/cli/finisher.rb
-- lib/answersengine/cli/global_page.rb
-- lib/answersengine/cli/job.rb
-- lib/answersengine/cli/job_output.rb
-- lib/answersengine/cli/parser.rb
-- lib/answersengine/cli/scraper.rb
-- lib/answersengine/cli/scraper_deployment.rb
-- lib/answersengine/cli/scraper_export.rb
-- lib/answersengine/cli/scraper_exporter.rb
-- lib/answersengine/cli/scraper_finisher.rb
-- lib/answersengine/cli/scraper_job.rb
-- lib/answersengine/cli/scraper_job_var.rb
-- lib/answersengine/cli/scraper_page.rb
-- lib/answersengine/cli/scraper_var.rb
-- lib/answersengine/cli/seeder.rb
-- lib/answersengine/client.rb
-- lib/answersengine/client/auth_token.rb
-- lib/answersengine/client/backblaze_content.rb
-- lib/answersengine/client/base.rb
-- lib/answersengine/client/deploy_key.rb
-- lib/answersengine/client/env_var.rb
-- lib/answersengine/client/export.rb
-- lib/answersengine/client/global_page.rb
-- lib/answersengine/client/job.rb
-- lib/answersengine/client/job_export.rb
-- lib/answersengine/client/job_log.rb
-- lib/answersengine/client/job_output.rb
-- lib/answersengine/client/job_page.rb
-- lib/answersengine/client/job_stat.rb
-- lib/answersengine/client/scraper.rb
-- lib/answersengine/client/scraper_deployment.rb
-- lib/answersengine/client/scraper_export.rb
-- lib/answersengine/client/scraper_exporter.rb
-- lib/answersengine/client/scraper_finisher.rb
-- lib/answersengine/client/scraper_job.rb
-- lib/answersengine/client/scraper_job_output.rb
-- lib/answersengine/client/scraper_job_page.rb
-- lib/answersengine/client/scraper_job_var.rb
-- lib/answersengine/client/scraper_var.rb
-- lib/answersengine/plugin.rb
-- lib/answersengine/plugin/context_exposer.rb
-- lib/answersengine/scraper.rb
-- lib/answersengine/scraper/executor.rb
-- lib/answersengine/scraper/finisher.rb
-- lib/answersengine/scraper/parser.rb
-- lib/answersengine/scraper/ruby_finisher_executor.rb
-- lib/answersengine/scraper/ruby_parser_executor.rb
-- lib/answersengine/scraper/ruby_seeder_executor.rb
-- lib/answersengine/scraper/seeder.rb
-- lib/answersengine/version.rb
-homepage: https://answersengine.com
+homepage: https://datahen.com
 licenses:
 - MIT
 metadata:
   allowed_push_host: https://rubygems.org
-  homepage_uri: https://answersengine.com
+  homepage_uri: https://datahen.com
   source_code_uri: https://github.com/answersengine/answersengine
 post_install_message: 
 rdoc_options: []
@@ -266,5 +127,6 @@ rubyforge_project:
 rubygems_version: 2.7.6
 signing_key: 
 specification_version: 4
-summary: AnswersEngine toolbelt for developers
+summary: "(Deprecated: Use datahen gem instead.) Compatibility alias for DataHen toolbelt
+  for developers."
 test_files: []

data/examples/fetchtest/libraries/hello.rb

@@ -1,9 +0,0 @@
-class Hello
-  def initialize
-    @hi = "Hello"
-  end
-
-  def say
-    @hi
-  end
-end

data/examples/fetchtest/libraries/hello_fail.rb

@@ -1,10 +0,0 @@
-class HelloFail
-  def initialize
-    @hi = "Hello"
-  end
-
-  def say
-    raise "fail from Hello class"
-    @hi
-  end
-end

data/examples/fetchtest/parsers/failed.rb

@@ -1,2 +0,0 @@
-raise "parsing intentionally failed"
-# 1 = 1

data/examples/fetchtest/parsers/find_outputs.rb

@@ -1,18 +0,0 @@
-
-puts
-puts "list all output on a collection"
-puts find_outputs('home', {},2).count
-puts find_outputs('home', {})
-
-puts "find_outputs"
-puts find_outputs('home', "_id": "b3d6f737731842b2be198fc3a85283b7")
-puts
-puts "find_outputs not found"
-puts find_outputs('home', "_id": "b3d6f737731842b2be198fc3a85283b7--").inspect
-# puts nil['_collection']
-puts
-puts "find_output"
-puts find_output('home',"_id": "b3d6f737731842b2be198fc3a85283b7")['_collection']
-puts
-puts "find_output not found"
-puts find_output('home',"_id": "b3d6f737731842b2be198fc3a85283b7--").inspect

data/examples/fetchtest/parsers/home.rb

@@ -1,50 +0,0 @@
-puts `pwd`
-
-require './libraries/hello'
-
-hello = Hello.new
-puts "hello say #{hello.say}"
-puts "page gid:#{page['gid']}"
-puts "page #{page}"
-
-puts "content #{content}"
-
-nokogiri = Nokogiri.HTML(content)
-
-h1 = nokogiri.at('h1')
-heading = h1.nil? ? '' : h1.text 
-text = nokogiri.text
-
-doc1 = {
-    _collection: "home",
-    # _id: "1234",
-    text: text,
-    heading: heading,
-    response_headers: page['response_headers'],
-    some_vars: page['vars']
-    # url: page.url
-}
-doc2 = {
-    _collection: "home",
-    # _id: "12345",
-    text: text,
-    heading: heading,
-    response_headers: page['response_headers'],
-    some_vars: page['vars']
-    # url: page.url
-}
-
-
-outputs << doc1
-outputs << {}
-outputs << doc2
-
-
-pages << {
-  url: "http://fetchtest.datahen.com/statuses/200?q=queuedFromParserWithVars",
-  vars: {"abc":[1,2,3], "def": "defcontent"}
-}
-
-puts "inspect page: #{page}"
-
-puts "inspect vars: #{page['vars']}"

data/examples/fetchtest/parsers/nested_fail.rb

@@ -1,3 +0,0 @@
-require './libraries/hello_fail'
-hello = HelloFail.new
-puts "hello say #{hello.say}"

data/examples/fetchtest/parsers/simple.rb

@@ -1,14 +0,0 @@
-nokogiri = Nokogiri.HTML(content)
-
-outputs << {
-    _collection: "home",
-    _id: "1234",
-    text: nokogiri.text,
-    heading: nokogiri.at('h1').text,
-    response_headers: page['response_headers'],
-}
-
-pages << {
-  url: "http://fetchtest.datahen.com/statuses/200?q=queuedFromParser",
-  vars: {"abc":[1,2,3], "def": "defcontent"}
-}