urlscan 0.3.0 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 0e026187891feb001443e0380f839bd4ad5cb08d0601be2c73af45345d8a1f3d
-  data.tar.gz: 8b027008ebdb5c6686c4f1b2cfd841cebe4e933949473163d36f9ac8e3527ba4
+  metadata.gz: 798702a904c93cd2895f683aae28dd0dc3d44dfea5b17d247168a6189bcb9c21
+  data.tar.gz: 3fbca0a7a6210fa2062ea02f84bb83492544bbb53aa5d7d5e0cad09077fa271e
 SHA512:
-  metadata.gz: a8c269b3080f8ec2e8d0785eba8b5fa3531ae169bd2718f02011920d20907d4c19a7c2293e6a146e391c6e5f231bc353732ae7d3f83f7ece50c66a068d8cff56
-  data.tar.gz: 251095794ae5577e33d887497caa82fccea882f89a322eb8b02cac86f8f31e27e975c156131e0bb29ccb966a96b32159bcb0297bee8644edbc3829a5e80c5167
+  metadata.gz: 4cfebe6cd4c4673e3afaa2575b4af09e5b46d357097ce3701c26200dc2ff565124a04b57a1bdccb652e3330a5886e1fbf2550b93f00e73f12f7000d52c61768c
+  data.tar.gz: 6239d851913a5b67fd54c4688106145f3ffe4faea8908faf928d92c6ed8263c8d78c4891719ba1dc0f799d233ea99b8af498773fef8221c016e9e625443921c9

data/README.md

@@ -18,7 +18,7 @@ gem install urlscan
 ## API usage
 
 ```ruby
-require 'urlscan'
+require "urlscan"
 
 # when given nothing, it tries to load your API key from ENV["URLSCAN_API_KEY"]
 api = UrlScan::API.new
@@ -27,32 +27,34 @@ api = UrlScan::API.new(api_key)
 
 # Submit a URL to scan
 res = api.submit("https://wikipedia.org")
-puts res["result"] # => "https://urlscan.io/result/ac04bc14-4efe-439d-b356-8384843daf75/"
 
 # Get a scan result
 res = api.result("ac04bc14-4efe-439d-b356-8384843daf75")
-p res # => See the following URL as an example of the response.
-      #    https://gist.github.com/ninoseki/a974d7e95629813615b380c30e737825#file-result-json
 
 # Get a DOM
 res = api.dom("ac04bc14-4efe-439d-b356-8384843daf75")
-p res
 
 # Search
 res = api.search("wikipedia.org")
-p res  # => See the following URL as an example of the reponse.
-       #    https://gist.github.com/ninoseki/a974d7e95629813615b380c30e737825#file-search-json
 ```
 
-### Supported API endpoints
+## Supported API endpoints
 
-| HTTP Method | URI                     | API method                                                     |
-|-------------|-------------------------|----------------------------------------------------------------|
-| POST        | /scan                   | `UrlScan::API#submit(url, is_public = true)`                   |
-| GET         | /result/`uuid`/         | `UrlScan::API#result(uuid)`                                    |
-| GET         | /dom/`uuid`/            | `UrlScan::API#dom(uuid)`                                       |
-| GET         | /screenshots/`uuid`.png | `UrlScan::API#screenshot(uuid)`                                |
-| GET         | /search                 | `UrlScan::API#search(q, size: 100, offset: 0, sort: "_score")` |
+| HTTP Method | URI                     | API method                                                                    |
+|-------------|-------------------------|-------------------------------------------------------------------------------|
+| POST        | /scan                   | `UrlScan::Clients::Community#submit(url, is_public = true)`                   |
+| GET         | /result/`uuid`/         | `UrlScan::Clients::Community#result(uuid)`                                    |
+| GET         | /dom/`uuid`/            | `UrlScan::Clients::Community#dom(uuid)`                                       |
+| GET         | /screenshots/`uuid`.png | `UrlScan::Clients::Community#screenshot(uuid)`                                |
+| GET         | /search                 | `UrlScan::Clients::Community#search(q, size: 100, offset: 0, sort: "_score")` |
+
+### Pro
+
+| HTTP Method | URI     | API method                                                        |
+|-------------|---------|-------------------------------------------------------------------|
+| GET         | /search | `UrlScan::Clients::Pro#search(query: nil, filter: nil, size: 50)` |
+| GET         | /brands | `UrlScan::Clients::Pro#brands`                                    |
+| GET         | /kits   | `UrlScan::Clients::Pro#kits`                                      |
 
 ## CLI usage
 
@@ -70,3 +72,15 @@ Options:
   [--API-KEY=API_KEY]
 
 ```
+
+### Pro
+
+```bash
+$ urlscan pro
+Commands:
+  urlscan pro help [COMMAND]  # Describe subcommands or one specific subcommand
+  urlscan pro search          # search for scans
+
+Options:
+  [--API-KEY=API_KEY]
+```

data/lib/urlscan.rb

@@ -2,5 +2,14 @@
 
 require "urlscan/version"
 require "urlscan/exceptions"
+
+require "urlscan/clients/base"
+require "urlscan/clients/community"
+require "urlscan/clients/pro"
+
+require "urlscan/commands/base"
+require "urlscan/commands/community"
+require "urlscan/commands/pro"
+
 require "urlscan/api"
 require "urlscan/cli"

data/lib/urlscan/api.rb

@@ -1,104 +1,18 @@
 # frozen_string_literal: true
 
-require "net/https"
-require "json"
+require "forwardable"
 
 module UrlScan
   class API
-    VERSION = 1
-    HOST = "urlscan.io"
-    URL = "https://#{HOST}/api/v#{VERSION}"
+    extend Forwardable
 
-    attr_reader :key
+    attr_reader :pro
 
     def initialize(key = ENV["URLSCAN_API_KEY"])
-      @key = key
+      @community = Clients::Community.new(key)
+      @pro = Clients::Pro.new(key)
     end
 
-    # @return [Hash]
-    def submit(url, is_public = true)
-      raise ArgumentError, "API key is required for this method." if key.nil?
-
-      params = { url: url, public: is_public ? "on" : "off" }
-      post("/scan/", params) { |json| json }
-    end
-
-    # @return [Hash]
-    def result(uuid)
-      get("/result/#{uuid}") { |json| json }
-    end
-
-    # @return [String]
-    def dom(uuid)
-      get("/dom/#{uuid}/") { |dom| dom }
-    end
-
-    def screenshot(uuid)
-      get("/screenshots/#{uuid}.png") { |png| png }
-    end
-
-    # @return [Hash]
-    def search(q, size: 100, offset: 0, sort: "_score")
-      params = { q: q, size: size, offset: offset, sort: sort }
-      query = URI.encode_www_form(params)
-      get("/search/?#{query}") { |json| json }
-    end
-
-    def url_for(path)
-      URI(URL + path)
-    end
-
-    def https_options
-      if proxy = ENV["HTTPS_PROXY"]
-        uri = URI(proxy)
-        {
-          proxy_address: uri.hostname,
-          proxy_port: uri.port,
-          proxy_from_env: false,
-          use_ssl: true
-        }
-      else
-        { use_ssl: true }
-      end
-    end
-
-    def request(req)
-      Net::HTTP.start(HOST, 443, https_options) do |http|
-        response = http.request(req)
-
-        case response.code
-        when "200"
-          if response["Content-Type"].to_s.include? "application/json"
-            yield JSON.parse(response.body)
-          else
-            yield response.body
-          end
-        when "400" then raise ProcessingError, response.body
-        when "401" then raise AuthenticationError, response.body
-        when "404" then raise NotFound, response.body
-        when "429" then raise RateLimited, response.body
-        when "500" then raise InternalServerError, response.body
-        else
-          raise ResponseError, response.body
-        end
-      end
-    end
-
-    def auth_header
-      { "API-KEY": key }
-    end
-
-    def get(path, &block)
-      get = Net::HTTP::Get.new(url_for(path))
-      request(get, &block)
-    end
-
-    def post(path, json, &block)
-      post = Net::HTTP::Post.new(url_for(path), auth_header)
-      post.content_type = "application/json"
-      post.body = json.to_json
-
-      request(post, &block)
-    end
+    def_delegators :@community, :submit, :result, :dom, :screenshot, :search
   end
 end

data/lib/urlscan/cli.rb

@@ -1,68 +1,8 @@
 # frozen_string_literal: true
 
-require "json"
-require "thor"
-
 module UrlScan
-  class CLI < Thor
-    class_option :API_KEY, type: :string
-
-    desc "submit [URL]", "submit a scan to [URL]"
-    method_option :public, type: :boolean, default: true
-    def submit(url)
-      with_error_handling do
-        res = api.submit(url, options[:public])
-        puts JSON.pretty_generate(res)
-      end
-    end
-
-    desc "result [UUID]", "get the result of a scan using the scan id [UUID]"
-    def result(uuid)
-      with_error_handling do
-        res = api.result(uuid)
-        puts JSON.pretty_generate(res)
-      end
-    end
-
-    desc "search [QUERY]", "search for scans by [QUERY]"
-    method_option :size, type: :numeric, default: 100
-    method_option :offset, type: :numeric, default: 0
-    method_option :sort, type: :string, default: "_score"
-    def search(query)
-      with_error_handling do
-        res = api.search(query, size: options["size"], offset: options["offset"], sort: options["sort"])
-        puts JSON.pretty_generate(res)
-      end
-    end
-
-    desc "dom [UUID]", "get the DOM of a scan using the scan id [UUID]"
-    def dom(uuid)
-      with_error_handling do
-        res = api.dom(uuid)
-        puts res
-      end
-    end
-
-    desc "screenshot [UUID]", "get the screenshot(image/png) of a scan using the scan id [UUID]"
-    def screenshot(uuid)
-      with_error_handling do
-        res = api.screenshot(uuid)
-        puts res
-      end
-    end
-
-    no_commands do
-      def api
-        options["API_KEY"] ? API.new(options["API_KEY"]) : API.new
-      end
-
-      def with_error_handling
-        yield
-      rescue ArgumentError => _e
-        puts "Warning: please specify your urlscan.io API key via ENV['URLSCAN_API_KEY'] or --API-KEY"
-      rescue ResponseError => e
-        puts "Warning: #{e}"
-      end
-    end
+  class CLI < Commands::Community
+    desc "pro", "PRO api commands"
+    subcommand "pro", Commands::Pro
   end
 end

data/lib/urlscan/clients/base.rb

@@ -0,0 +1,82 @@
+# frozen_string_literal: true
+
+require "net/https"
+require "json"
+
+module UrlScan
+  module Clients
+    class Base
+      VERSION = 1
+      HOST = "urlscan.io"
+
+      attr_reader :key
+
+      def initialize(key = ENV["URLSCAN_API_KEY"])
+        @key = key
+      end
+
+      private
+
+      def url
+        @url ||= "https://#{self.class::HOST}/api/v#{self.class::VERSION}"
+      end
+
+      def url_for(path)
+        URI(url + path)
+      end
+
+      def https_options
+        if proxy = ENV["HTTPS_PROXY"]
+          uri = URI(proxy)
+          {
+            proxy_address: uri.hostname,
+            proxy_port: uri.port,
+            proxy_from_env: false,
+            use_ssl: true
+          }
+        else
+          { use_ssl: true }
+        end
+      end
+
+      def request(req)
+        Net::HTTP.start(HOST, 443, https_options) do |http|
+          response = http.request(req)
+
+          case response.code
+          when "200"
+            if response["Content-Type"].to_s.include? "application/json"
+              yield JSON.parse(response.body)
+            else
+              yield response.body
+            end
+          when "400" then raise ProcessingError, response.body
+          when "401" then raise AuthenticationError, response.body
+          when "404" then raise NotFound, response.body
+          when "429" then raise RateLimited, response.body
+          when "500" then raise InternalServerError, response.body
+          else
+            raise ResponseError, response.body
+          end
+        end
+      end
+
+      def auth_header
+        { "API-KEY": key }
+      end
+
+      def get(path, &block)
+        get = Net::HTTP::Get.new(url_for(path))
+        request(get, &block)
+      end
+
+      def post(path, json, &block)
+        post = Net::HTTP::Post.new(url_for(path), auth_header)
+        post.content_type = "application/json"
+        post.body = json.to_json
+
+        request(post, &block)
+      end
+    end
+  end
+end

data/lib/urlscan/clients/community.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+module UrlScan
+  module Clients
+    class Community < Base
+      # @return [Hash]
+      def submit(url, is_public = true)
+        raise ArgumentError, "API key is required for this method." if key.nil?
+
+        params = { url: url, public: is_public ? "on" : "off" }
+        post("/scan/", params) { |json| json }
+      end
+
+      # @return [Hash]
+      def result(uuid)
+        get("/result/#{uuid}") { |json| json }
+      end
+
+      # @return [String]
+      def dom(uuid)
+        get("/dom/#{uuid}/") { |dom| dom }
+      end
+
+      def screenshot(uuid)
+        get("/screenshots/#{uuid}.png") { |png| png }
+      end
+
+      # @return [Hash]
+      def search(q, size: 100, offset: 0, sort: "_score")
+        params = { q: q, size: size, offset: offset, sort: sort }
+        query = URI.encode_www_form(params)
+        get("/search/?#{query}") { |json| json }
+      end
+    end
+  end
+end

data/lib/urlscan/clients/pro.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+module UrlScan
+  module Clients
+    class Pro < Base
+      VERSION = 1
+      HOST = "pro.urlscan.com"
+
+      def initialize(key = ENV["URLSCAN_API_KEY"])
+        super key
+
+        raise ArgumentError, "API key is required for this class." if key.nil?
+      end
+
+      # @return [Hash]
+      def search(query: nil, filter: nil, size: 50)
+        filter = build_filter(filter)
+        params = { q: query, size: size }.compact
+        uri_query = URI.encode_www_form(params)
+        uri_query << "&filter=#{filter}" if filter
+
+        get("/search?#{uri_query}") { |json| json }
+      end
+
+      # @return [Hash]
+      def brands
+        get("/brands") { |json| json }
+      end
+
+      # @return [Hash]
+      def kits
+        get("/kits") { |json| json }
+      end
+
+      private
+
+      def build_filter(filter)
+        return nil unless filter
+
+        filter.start_with?("$") ? filter : "$#{filter}"
+      end
+
+      def url
+        @url ||= "https://#{self.class::HOST}/api/v#{self.class::VERSION}/pro"
+      end
+
+      def get(path, &block)
+        get = Net::HTTP::Get.new(url_for(path), auth_header)
+        request(get, &block)
+      end
+    end
+  end
+end

data/lib/urlscan/commands/base.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+require "json"
+require "thor"
+
+module UrlScan
+  module Commands
+    class Base < Thor
+      class_option :API_KEY, type: :string
+
+      no_commands do
+        def api
+          options.key?("API_KEY") ? API.new(options["API_KEY"]) : API.new
+        end
+
+        def with_error_handling
+          yield
+        rescue ArgumentError => _e
+          puts "Warning: please specify your urlscan.io API key via ENV['URLSCAN_API_KEY'] or --API-KEY"
+        rescue ResponseError => e
+          puts "Warning: #{e}"
+        end
+      end
+    end
+  end
+end

data/lib/urlscan/commands/community.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+module UrlScan
+  module Commands
+    class Community < Base
+      desc "submit [URL]", "submit a scan to [URL]"
+      method_option :public, type: :boolean, default: true
+      def submit(url)
+        with_error_handling do
+          res = api.submit(url, options[:public])
+          puts JSON.pretty_generate(res)
+        end
+      end
+
+      desc "result [UUID]", "get the result of a scan using the scan id [UUID]"
+      def result(uuid)
+        with_error_handling do
+          res = api.result(uuid)
+          puts JSON.pretty_generate(res)
+        end
+      end
+
+      desc "search [QUERY]", "search for scans by [QUERY]"
+      method_option :size, type: :numeric, default: 100
+      method_option :offset, type: :numeric, default: 0
+      method_option :sort, type: :string, default: "_score"
+      def search(query)
+        with_error_handling do
+          res = api.search(query, size: options["size"], offset: options["offset"], sort: options["sort"])
+          puts JSON.pretty_generate(res)
+        end
+      end
+
+      desc "dom [UUID]", "get the DOM of a scan using the scan id [UUID]"
+      def dom(uuid)
+        with_error_handling do
+          res = api.dom(uuid)
+          puts res
+        end
+      end
+
+      desc "screenshot [UUID]", "get the screenshot(image/png) of a scan using the scan id [UUID]"
+      def screenshot(uuid)
+        with_error_handling do
+          res = api.screenshot(uuid)
+          puts res
+        end
+      end
+    end
+  end
+end

data/lib/urlscan/commands/pro.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+module UrlScan
+  module Commands
+    class Pro < Base
+      desc "search", "search for scans"
+      method_option :size, type: :numeric, default: 50
+      method_option :query, type: :string
+      method_option :filter, type: :string
+      def search
+        with_error_handling do
+          res = api.pro.search(query: options["query"], filter: options["filter"], size: options["size"])
+          puts JSON.pretty_generate(res)
+        end
+      end
+    end
+  end
+end

data/lib/urlscan/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module UrlScan
-  VERSION = "0.3.0"
+  VERSION = "0.4.0"
 end

data/urlscan.gemspec

@@ -23,11 +23,10 @@ Gem::Specification.new do |spec|
 
   spec.add_development_dependency "bundler", "~> 2.0"
   spec.add_development_dependency "coveralls", "~> 0.8"
-  spec.add_development_dependency "dotenv", "~> 2.5"
   spec.add_development_dependency "rake", "~> 12.3"
   spec.add_development_dependency "rspec", "~> 3.8"
   spec.add_development_dependency "vcr", "~> 5.0"
-  spec.add_development_dependency "webmock", "~> 3.6"
+  spec.add_development_dependency "webmock", "~> 3.7"
 
   spec.add_runtime_dependency "thor", "~> 0.20"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: urlscan
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.4.0
 platform: ruby
 authors:
 - Manabu Niseki
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2019-08-18 00:00:00.000000000 Z
+date: 2019-09-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -38,20 +38,6 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.8'
-- !ruby/object:Gem::Dependency
-  name: dotenv
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '2.5'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '2.5'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -100,14 +86,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.6'
+        version: '3.7'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.6'
+        version: '3.7'
 - !ruby/object:Gem::Dependency
   name: thor
   requirement: !ruby/object:Gem::Requirement
@@ -143,6 +129,12 @@ files:
 - lib/urlscan.rb
 - lib/urlscan/api.rb
 - lib/urlscan/cli.rb
+- lib/urlscan/clients/base.rb
+- lib/urlscan/clients/community.rb
+- lib/urlscan/clients/pro.rb
+- lib/urlscan/commands/base.rb
+- lib/urlscan/commands/community.rb
+- lib/urlscan/commands/pro.rb
 - lib/urlscan/exceptions.rb
 - lib/urlscan/version.rb
 - urlscan.gemspec