phantomblaster 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: bc45e1660a2476296084bf99de734f35449a2f8ae4716af296f2b8083d622813
+  data.tar.gz: 77a1e7f85c01d4a4b43f7f359d84fc796afce7cd678b2d59085e6626265d6439
+SHA512:
+  metadata.gz: b567452bed265587c9398a4adfa6b4d0930ba4977e791984078f844cf9f705417b10bc453ecadb9de17aa71432bfbf05ee81adeddc210c519db138724ddec8a9
+  data.tar.gz: 4b0103920b45f9cf73ccc446cd1d872bff0d141673316981c155745f38651a7f560acc54b08db6eeea13854c7bbebd63abcf169d949df0f3fe1ef43317f9c9a7

data/.gitignore

@@ -0,0 +1,13 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+/scripts/
+
+# rspec failure tracking
+.rspec_status

data/.rspec

@@ -0,0 +1,2 @@
+--format documentation
+--color

data/.rubocop.yml

@@ -0,0 +1,4 @@
+Metrics/LineLength:
+  Max: 100
+Style/Documentation:
+  Enabled: false

data/.travis.yml

@@ -0,0 +1,5 @@
+sudo: false
+language: ruby
+rvm:
+  - 2.3.1
+before_install: gem install bundler -v 1.15.1

data/Gemfile

@@ -0,0 +1,4 @@
+source "https://rubygems.org"
+
+# Specify your gem's dependencies in phantomblaster.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2018 Jason Adkison
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,69 @@
+# Phantomblaster
+
+A ruby gem for interacting with the [Phantombuster](https://phantombuster.com) service.
+
+Currently includes a CLI tool to make it easier to develop and work with agent scripts. In the future, it will provide an easy way to interact with agents from within any ruby program.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'phantomblaster'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install phantomblaster
+
+## Usage
+
+```
+  phantomblaster account            # Displays information about the account
+  phantomblaster agents             # Lists all remote agents
+  phantomblaster download FILENAME  # Downloads an agent script
+  phantomblaster generate FILENAME  # Generates an agent script
+  phantomblaster help [COMMAND]     # Describe available commands or one specific command
+  phantomblaster pull               # Fetches all agent scripts
+  phantomblaster push               # Pushes all agent scripts
+  phantomblaster scripts            # Lists all remote scripts
+  phantomblaster upload FILENAME    # Uploads an agent script
+```
+
+## Configuration
+
+There are currently 2 options available.
+
+You can configure options by creating an initializer:
+
+```ruby
+Phantomblaster.configure do |config|
+  config.api_key = 'XXXXXXXXX'
+  config.scripts_dir = '/path/to/scripts'
+end
+```
+
+Or set the following environment variables:
+
+* PHANTOMBUSTER_API_KEY
+* PHANTOMBUSTER_SCRIPTS_DIR
+
+The scripts directory must be an absolute path.
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/jasonadkison/phantomblaster.
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT).

data/Rakefile

@@ -0,0 +1,6 @@
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new(:spec)
+
+task :default => :spec

data/bin/console

@@ -0,0 +1,15 @@
+#!/usr/bin/env ruby
+
+require 'bundler/setup'
+require 'pry'
+require 'phantomblaster'
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require 'irb'
+IRB.start(__FILE__)

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/exe/phantomblaster

@@ -0,0 +1,6 @@
+#!/usr/bin/env ruby
+
+require 'phantomblaster'
+require 'phantomblaster/cli'
+
+Phantomblaster::CLI.start

data/lib/phantomblaster.rb

@@ -0,0 +1,33 @@
+require 'phantomblaster/version'
+require 'phantomblaster/errors'
+require 'phantomblaster/client'
+require 'phantomblaster/models/user'
+require 'phantomblaster/models/script'
+
+# A module for containing phantombuster functionalities.
+module Phantomblaster
+  class << self
+    attr_accessor :configuration
+  end
+
+  def self.configure
+    self.configuration ||= Configuration.new
+    yield(configuration)
+  end
+
+  # Class that holds the config state for the gem.
+  # Define each config option with a getter/setter.
+  class Configuration
+    attr_accessor :api_key, :scripts_dir
+
+    def initialize
+      @api_key = ''
+      @scripts_dir = ''
+    end
+  end
+end
+
+Phantomblaster.configure do |config|
+  config.api_key = ENV['PHANTOMBUSTER_API_KEY']
+  config.scripts_dir = ENV['PHANTOMBUSTER_SCRIPTS_DIR'] || File.join(File.dirname(__FILE__), '../scripts')
+end

data/lib/phantomblaster/cli.rb

@@ -0,0 +1,115 @@
+require 'thor'
+require 'terminal-table'
+require 'pathname'
+require 'phantomblaster/generators/script'
+
+module Phantomblaster
+  class CLI < Thor
+    package_name 'phantomblaster'
+
+    desc 'account', 'Displays information about the account'
+    def account
+      title = 'Phantombuster Account'
+      headings = ['Email', 'API Key']
+      rows = [[current_user.email, Phantomblaster.configuration.api_key]]
+      table = Terminal::Table.new title: title, headings: headings, rows: rows
+      say table
+    end
+
+    desc 'agents', 'Lists all remote agents'
+    def agents
+      title = 'Phantombuster Agents'
+      headings = ['Agent Name', 'Agent ID', 'Script ID', 'Last Status']
+      rows = []
+      current_agents.each do |agent|
+        rows << [agent['name'], agent['id'], agent['scriptId'], agent['lastEndStatus']]
+      end
+      table = Terminal::Table.new title: title, headings: headings, rows: rows
+      say table
+    end
+
+    desc 'scripts', 'Lists all remote scripts'
+    def scripts
+      title = 'Phantombuster Scripts'
+      headings = ['Script Name', 'Script ID', 'Script Source', 'Script Last Saved At']
+      rows = []
+      current_scripts.each do |script|
+        rows << [script.name, script.id, script.source, script.last_saved_at]
+      end
+      table = Terminal::Table.new title: title, headings: headings, rows: rows
+      say table
+    end
+
+    desc 'generate FILENAME', 'Generates an agent script'
+    def generate(name)
+      Phantomblaster::Generators::Script.start([name])
+    end
+
+    desc 'upload FILENAME', 'Uploads an agent script'
+    def upload(name)
+      return unless yes?("Upload #{name} to Phantombuster?")
+
+      res = Phantomblaster::Models::Script.upload(name)
+      say res
+    end
+
+    desc 'download FILENAME', 'Downloads an agent script'
+    def download(name, force = false)
+      script = Phantomblaster::Models::Script.find_by_name(name)
+      folder_pathname = Pathname.new(Phantomblaster.configuration.scripts_dir)
+      file_pathname = Pathname.new(name)
+      full_pathname = folder_pathname.join(file_pathname)
+
+      unless folder_pathname.directory?
+        return unless yes?("Directory #{folder_pathname.realdirpath} does not exist. Create it?")
+
+        folder_pathname.mkpath
+      end
+
+      if !force && full_pathname.exist?
+        return unless yes?("File #{full_pathname.realpath} already exists. Overwrite?")
+      end
+
+      full_pathname.open('w') { |f| f << script.text }
+      say "Wrote #{full_pathname.realpath}"
+    end
+
+    desc 'pull', 'Fetches all agent scripts'
+    def pull
+      return unless yes?("This will pull from Phantombuster and overwrite any existing scripts. " \
+                         "Are you sure you want to continue?")
+
+      current_scripts.each do |script|
+        download(script.name, true)
+      end
+    end
+
+    desc 'push', 'Pushes all agent scripts'
+    def push
+      return unless yes?("This will push all local scripts to Phantombuster and overwrite any " \
+                         "existing scripts. Are you sure you want to continue?")
+
+      folder_pathname = Pathname.new(Phantomblaster.configuration.scripts_dir)
+      raise 'Scripts directory does not exist' unless folder_pathname.directory?
+
+      Pathname.glob("#{folder_pathname.realdirpath}/**/*.js").each do |pn|
+        _dir, file = pn.split
+        upload(file)
+      end
+    end
+
+    private
+
+    def current_user
+      @current_user ||= Phantomblaster::Models::User.find
+    end
+
+    def current_agents
+      @current_agents ||= current_user.agents
+    end
+
+    def current_scripts
+      @current_scripts ||= Phantomblaster::Models::Script.all
+    end
+  end
+end

data/lib/phantomblaster/client.rb

@@ -0,0 +1,48 @@
+require 'uri'
+require 'net/http'
+require 'json'
+
+module Phantomblaster
+  class Client
+    API_URL = 'https://phantombuster.com/api/v1'.freeze
+
+    def self.build_request(path, args = {}, &_block)
+      uri = URI("#{API_URL}#{path}")
+      new_query = URI.decode_www_form(uri.query || '')
+      args.to_a.each { |arg| new_query << arg }
+      new_query << ['key', Phantomblaster.configuration.api_key]
+      uri.query = URI.encode_www_form(new_query)
+      http = Net::HTTP.new(uri.host, uri.port)
+      http.use_ssl = true
+      yield [http, uri]
+    end
+
+    def self.get(path, args = {})
+      response = build_request(path, args) do |http, uri|
+        http.request(Net::HTTP::Get.new(uri))
+      end
+
+      unless response.is_a?(Net::HTTPSuccess)
+        raise APIError, "Unexpected response: #{response.read_body}"
+      end
+
+      JSON.parse(response.read_body)['data']
+    rescue JSON::ParserError => _e
+      raise APIError, "Could not parse response: #{response.read_body}"
+    end
+
+    def self.post(path, args = {})
+      response = build_request(path, args) do |http, uri|
+        http.request(Net::HTTP::Post.new(uri))
+      end
+
+      unless response.is_a?(Net::HTTPSuccess)
+        raise APIError, "Unexpected response: #{response.read_body}"
+      end
+
+      JSON.parse(response.read_body)
+    rescue JSON::ParserError => _e
+      raise APIError, "Could not parse response: #{response.read_body}"
+    end
+  end
+end

data/lib/phantomblaster/errors.rb

@@ -0,0 +1,5 @@
+module Phantomblaster
+  class Error < StandardError; end
+  class APIError < Error; end
+  class MissingFileError < Error; end
+end

data/lib/phantomblaster/generators/script.rb

@@ -0,0 +1,22 @@
+require 'thor/group'
+
+module Phantomblaster
+  module Generators
+    class Script < Thor::Group
+      include Thor::Actions
+      argument :filename, type: :string
+
+      def self.source_root
+        File.dirname(__FILE__) + '/script'
+      end
+
+      def create_scripts_dir
+        empty_directory(Phantomblaster.configuration.scripts_dir)
+      end
+
+      def copy_script
+        template('script.js', "#{Phantomblaster.configuration.scripts_dir}/#{filename}")
+      end
+    end
+  end
+end

data/lib/phantomblaster/generators/script/script.js

@@ -0,0 +1,55 @@
+// Phantombuster configuration {
+
+  "phantombuster command: nodejs"
+  "phantombuster package: 5"
+  "phantombuster flags: save-folder"
+
+  const Buster = require("phantombuster")
+  const buster = new Buster()
+
+  const Nick = require("nickjs")
+  const nick = new Nick()
+
+  // }
+
+  /*
+    This demo script returns two things:
+    1) JSON of all the Hacker News homepage links
+    2) Screenshot of the Hacker News homepage
+
+    Edit it as you wish, have a look at our documentation on https://hub.phantombuster.com/ and do get in touch.
+
+    Most importantly, click LAUNCH to get a taste of Phantombuster's power!! :)
+  */
+
+  nick.newTab().then(async (tab) => {
+
+    await tab.open("news.ycombinator.com")
+    await tab.untilVisible("#hnmain") // Make sure we have loaded the right page
+    await tab.inject("../injectables/jquery-3.0.0.min.js") // We're going to use jQuery to scrape
+
+    // Evaluate a function in the current page DOM context. Execution is sandboxed: page has no access to the Nick context
+    // In other words: Open the browser inspector to execute this function in the console
+    const hackerNewsLinks = await tab.evaluate((arg, callback) => {
+      const data = []
+      $(".athing").each((index, element) => {
+        data.push({
+          title: $(element).find(".storylink").text(),
+          url: $(element).find(".storylink").attr("href")
+        })
+      })
+      callback(null, data)
+    })
+
+    await buster.setResultObject(hackerNewsLinks) // Send the result back to Phantombuster
+    await tab.screenshot("screenshot.png") // Why not take a screenshot while we're at it?
+
+  })
+  .then(() => {
+    console.log("Job done!")
+    nick.exit()
+  })
+  .catch((err) => {
+    console.log(`Something went wrong: ${err}`)
+    nick.exit(1)
+  })

data/lib/phantomblaster/models/script.rb

@@ -0,0 +1,44 @@
+module Phantomblaster
+  module Models
+    class Script
+      def self.find(id)
+        data = Phantomblaster::Client.get("/script/by-id/json/#{id}", withoutText: false)
+        new(data)
+      end
+
+      def self.find_by_name(name)
+        data = Phantomblaster::Client.get("/script/by-name/json/#{name}", withoutText: false)
+        new(data)
+      end
+
+      def self.all
+        data = Phantomblaster::Client.get('/scripts')
+        data.map { |params| new(params) }
+      end
+
+      def self.upload(name)
+        pathname = Pathname.new("#{Phantomblaster.configuration.scripts_dir}/#{name}")
+        raise MissingFileError, "#{pathname.realdirpath} not found" unless pathname.file?
+
+        text = pathname.open(&:read)
+        Phantomblaster::Client.post("/script/#{name}", text: text,
+                                                       insertOnly: false,
+                                                       source: :phantomblaster)
+      end
+
+      attr_reader :id, :name, :source, :last_saved_at
+
+      def initialize(params)
+        @id = params['id']
+        @name = params['name']
+        @source = params['source']
+        @last_saved_at = params['lastSaveDate']
+        @text = params['text'].to_s
+      end
+
+      def text
+        @text ||= self.class.find(id).instance_variable_get(:@text)
+      end
+    end
+  end
+end

data/lib/phantomblaster/models/user.rb

@@ -0,0 +1,17 @@
+module Phantomblaster
+  module Models
+    class User
+      def self.find
+        data = Phantomblaster::Client.get('/user')
+        new(data)
+      end
+
+      attr_reader :email, :agents
+
+      def initialize(params)
+        @email = params['email']
+        @agents = params['agents']
+      end
+    end
+  end
+end

data/lib/phantomblaster/version.rb

@@ -0,0 +1,3 @@
+module Phantomblaster
+  VERSION = '0.1.0'.freeze
+end

data/phantomblaster.gemspec

@@ -0,0 +1,31 @@
+lib = File.expand_path('lib', __dir__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'phantomblaster/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = 'phantomblaster'
+  spec.version       = Phantomblaster::VERSION
+  spec.authors       = ['Jason Adkison']
+  spec.email         = ['jadkison@gmail.com']
+  spec.summary       = 'A gem for interacting with Phantombuster agents.'
+  spec.description   = 'A gem for interacting with Phantombuster agents. Allows you to easily ' \
+                       'manage, sync and consume agents from within your Ruby/Rails based ' \
+                       'applications.'
+  spec.homepage      = 'https://github.com/jasonadkison/phantomblaster'
+  spec.license       = 'MIT'
+  spec.files         = `git ls-files -z`.split("\x0").reject do |f|
+    f.match(%r{^(test|spec|features)/})
+  end
+  spec.bindir        = 'exe'
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ['lib']
+
+  spec.add_development_dependency 'bundler', '~> 1.15'
+  spec.add_development_dependency 'pry', '~> 0.11.3'
+  spec.add_development_dependency 'rake', '~> 10.0'
+  spec.add_development_dependency 'rspec', '~> 3.0'
+  spec.add_development_dependency 'rubocop', '~> 0.59.1'
+
+  spec.add_runtime_dependency 'terminal-table', '~> 1.8.0'
+  spec.add_runtime_dependency 'thor', '~> 0.20.0'
+end

metadata

@@ -0,0 +1,165 @@
+--- !ruby/object:Gem::Specification
+name: phantomblaster
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Jason Adkison
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2018-09-19 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.15'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.15'
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.11.3
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.11.3
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: rubocop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.59.1
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.59.1
+- !ruby/object:Gem::Dependency
+  name: terminal-table
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.8.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.8.0
+- !ruby/object:Gem::Dependency
+  name: thor
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.20.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.20.0
+description: A gem for interacting with Phantombuster agents. Allows you to easily
+  manage, sync and consume agents from within your Ruby/Rails based applications.
+email:
+- jadkison@gmail.com
+executables:
+- phantomblaster
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rspec"
+- ".rubocop.yml"
+- ".travis.yml"
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/console
+- bin/setup
+- exe/phantomblaster
+- lib/phantomblaster.rb
+- lib/phantomblaster/cli.rb
+- lib/phantomblaster/client.rb
+- lib/phantomblaster/errors.rb
+- lib/phantomblaster/generators/script.rb
+- lib/phantomblaster/generators/script/script.js
+- lib/phantomblaster/models/script.rb
+- lib/phantomblaster/models/user.rb
+- lib/phantomblaster/version.rb
+- phantomblaster.gemspec
+homepage: https://github.com/jasonadkison/phantomblaster
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.7.7
+signing_key: 
+specification_version: 4
+summary: A gem for interacting with Phantombuster agents.
+test_files: []