urlscan 0.3.0 → 0.7.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 0e026187891feb001443e0380f839bd4ad5cb08d0601be2c73af45345d8a1f3d
-  data.tar.gz: 8b027008ebdb5c6686c4f1b2cfd841cebe4e933949473163d36f9ac8e3527ba4
+  metadata.gz: 0d83b308d78e0bd41123a1a6d37a4cec0d947dcb97dc9a79176a44b142c1c00d
+  data.tar.gz: 6912d1c6cbdc4099d1cd391b364c6e19ee98b9976ce0a6928c22ec333f43d33a
 SHA512:
-  metadata.gz: a8c269b3080f8ec2e8d0785eba8b5fa3531ae169bd2718f02011920d20907d4c19a7c2293e6a146e391c6e5f231bc353732ae7d3f83f7ece50c66a068d8cff56
-  data.tar.gz: 251095794ae5577e33d887497caa82fccea882f89a322eb8b02cac86f8f31e27e975c156131e0bb29ccb966a96b32159bcb0297bee8644edbc3829a5e80c5167
+  metadata.gz: 447e4250c9f685716e981caccde9020c1c3f1f7cf6c00d89db553e541b174046cc3458800962e6fa8c1bcd407d6b5106cd20031864a5fcc63bde4aee30e29a8e
+  data.tar.gz: 910c24d8f9457e0736b6b11604150af4fb644c396d7da9f285e63e6006ec492ae0fb24155c705169bb1e97afd5f6e1df143404cd8eee2783cbf0c7b5a445e1c0

data/.github/workflows/test.yml

@@ -0,0 +1,25 @@
+name: Ruby CI
+
+on: [pull_request]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    strategy:
+      fail-fast: false
+      matrix:
+        ruby: [2.7, "3.0"]
+
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Ruby
+        uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: ${{ matrix.ruby }}
+          bundler-cache: true
+      - name: Build and test with Rake
+        run: |
+          gem install bundler
+          bundle install
+          bundle exec rake

data/README.md

@@ -1,13 +1,13 @@
 # urlscan
 
 [![Gem Version](https://badge.fury.io/rb/urlscan.svg)](https://badge.fury.io/rb/urlscan)
-[![Build Status](https://travis-ci.org/ninoseki/urlscan.svg?branch=master)](https://travis-ci.org/ninoseki/urlscan)
+[![Ruby CI](https://github.com/ninoseki/urlscan/actions/workflows/test.yml/badge.svg)](https://github.com/ninoseki/urlscan/actions/workflows/test.yml)
 [![Maintainability](https://api.codeclimate.com/v1/badges/c6625486f2d57039adef/maintainability)](https://codeclimate.com/github/ninoseki/urlscan/maintainability)
 [![Coverage Status](https://coveralls.io/repos/github/ninoseki/urlscan/badge.svg?branch=master)](https://coveralls.io/github/ninoseki/urlscan?branch=master)
 
 ## Description
 
-Ruby API client for [urlscan.io](https://urlscan.io/).
+[urlscan.io](https://urlscan.io/) API wrapper for Ruby.
 
 ## Installation
 
@@ -18,7 +18,7 @@ gem install urlscan
 ## API usage
 
 ```ruby
-require 'urlscan'
+require "urlscan"
 
 # when given nothing, it tries to load your API key from ENV["URLSCAN_API_KEY"]
 api = UrlScan::API.new
@@ -27,32 +27,35 @@ api = UrlScan::API.new(api_key)
 
 # Submit a URL to scan
 res = api.submit("https://wikipedia.org")
-puts res["result"] # => "https://urlscan.io/result/ac04bc14-4efe-439d-b356-8384843daf75/"
 
 # Get a scan result
 res = api.result("ac04bc14-4efe-439d-b356-8384843daf75")
-p res # => See the following URL as an example of the response.
-      #    https://gist.github.com/ninoseki/a974d7e95629813615b380c30e737825#file-result-json
 
 # Get a DOM
 res = api.dom("ac04bc14-4efe-439d-b356-8384843daf75")
-p res
 
 # Search
 res = api.search("wikipedia.org")
-p res  # => See the following URL as an example of the reponse.
-       #    https://gist.github.com/ninoseki/a974d7e95629813615b380c30e737825#file-search-json
 ```
 
-### Supported API endpoints
+## Supported API endpoints
 
-| HTTP Method | URI                     | API method                                                     |
-|-------------|-------------------------|----------------------------------------------------------------|
-| POST        | /scan                   | `UrlScan::API#submit(url, is_public = true)`                   |
-| GET         | /result/`uuid`/         | `UrlScan::API#result(uuid)`                                    |
-| GET         | /dom/`uuid`/            | `UrlScan::API#dom(uuid)`                                       |
-| GET         | /screenshots/`uuid`.png | `UrlScan::API#screenshot(uuid)`                                |
-| GET         | /search                 | `UrlScan::API#search(q, size: 100, offset: 0, sort: "_score")` |
+| HTTP Method | URI                     | API method                                                            |
+|-------------|-------------------------|-----------------------------------------------------------------------|
+| POST        | /scan                   | `UrlScan::Clients::Community#submit(url, is_public = true)`           |
+| GET         | /result/`uuid`/         | `UrlScan::Clients::Community#result(uuid)`                            |
+| GET         | /dom/`uuid`/            | `UrlScan::Clients::Community#dom(uuid)`                               |
+| GET         | /screenshots/`uuid`.png | `UrlScan::Clients::Community#screenshot(uuid)`                        |
+| GET         | /search                 | `UrlScan::Clients::Community#search(q, size: 100, search_after: nil)` |
+
+### Pro
+
+| HTTP Method | URI                     | API method                            |
+|-------------|-------------------------|---------------------------------------|
+| GET         | /brands                 | `UrlScan::Clients::Pro#brands`        |
+| GET         | /kits                   | `UrlScan::Clients::Pro#kits`          |
+| GET         | /phishfeed              | `UrlScan::Clients::Pro#phishfeed`     |
+| GET         | /result/`uuid`/similar/ | `UrlScan::Clients::Pro#similar(uuid)` |
 
 ## CLI usage
 

data/lib/urlscan.rb

@@ -2,5 +2,13 @@
 
 require "urlscan/version"
 require "urlscan/exceptions"
+
+require "urlscan/clients/base"
+require "urlscan/clients/community"
+require "urlscan/clients/pro"
+
+require "urlscan/commands/base"
+require "urlscan/commands/community"
+
 require "urlscan/api"
 require "urlscan/cli"

data/lib/urlscan/api.rb

@@ -1,104 +1,18 @@
 # frozen_string_literal: true
 
-require "net/https"
-require "json"
+require "forwardable"
 
 module UrlScan
   class API
-    VERSION = 1
-    HOST = "urlscan.io"
-    URL = "https://#{HOST}/api/v#{VERSION}"
+    extend Forwardable
 
-    attr_reader :key
+    attr_reader :pro
 
     def initialize(key = ENV["URLSCAN_API_KEY"])
-      @key = key
+      @community = Clients::Community.new(key)
+      @pro = Clients::Pro.new(key)
     end
 
-    # @return [Hash]
-    def submit(url, is_public = true)
-      raise ArgumentError, "API key is required for this method." if key.nil?
-
-      params = { url: url, public: is_public ? "on" : "off" }
-      post("/scan/", params) { |json| json }
-    end
-
-    # @return [Hash]
-    def result(uuid)
-      get("/result/#{uuid}") { |json| json }
-    end
-
-    # @return [String]
-    def dom(uuid)
-      get("/dom/#{uuid}/") { |dom| dom }
-    end
-
-    def screenshot(uuid)
-      get("/screenshots/#{uuid}.png") { |png| png }
-    end
-
-    # @return [Hash]
-    def search(q, size: 100, offset: 0, sort: "_score")
-      params = { q: q, size: size, offset: offset, sort: sort }
-      query = URI.encode_www_form(params)
-      get("/search/?#{query}") { |json| json }
-    end
-
-    def url_for(path)
-      URI(URL + path)
-    end
-
-    def https_options
-      if proxy = ENV["HTTPS_PROXY"]
-        uri = URI(proxy)
-        {
-          proxy_address: uri.hostname,
-          proxy_port: uri.port,
-          proxy_from_env: false,
-          use_ssl: true
-        }
-      else
-        { use_ssl: true }
-      end
-    end
-
-    def request(req)
-      Net::HTTP.start(HOST, 443, https_options) do |http|
-        response = http.request(req)
-
-        case response.code
-        when "200"
-          if response["Content-Type"].to_s.include? "application/json"
-            yield JSON.parse(response.body)
-          else
-            yield response.body
-          end
-        when "400" then raise ProcessingError, response.body
-        when "401" then raise AuthenticationError, response.body
-        when "404" then raise NotFound, response.body
-        when "429" then raise RateLimited, response.body
-        when "500" then raise InternalServerError, response.body
-        else
-          raise ResponseError, response.body
-        end
-      end
-    end
-
-    def auth_header
-      { "API-KEY": key }
-    end
-
-    def get(path, &block)
-      get = Net::HTTP::Get.new(url_for(path))
-      request(get, &block)
-    end
-
-    def post(path, json, &block)
-      post = Net::HTTP::Post.new(url_for(path), auth_header)
-      post.content_type = "application/json"
-      post.body = json.to_json
-
-      request(post, &block)
-    end
+    def_delegators :@community, :submit, :result, :dom, :screenshot, :search
   end
 end

data/lib/urlscan/cli.rb

@@ -1,68 +1,5 @@
 # frozen_string_literal: true
 
-require "json"
-require "thor"
-
 module UrlScan
-  class CLI < Thor
-    class_option :API_KEY, type: :string
-
-    desc "submit [URL]", "submit a scan to [URL]"
-    method_option :public, type: :boolean, default: true
-    def submit(url)
-      with_error_handling do
-        res = api.submit(url, options[:public])
-        puts JSON.pretty_generate(res)
-      end
-    end
-
-    desc "result [UUID]", "get the result of a scan using the scan id [UUID]"
-    def result(uuid)
-      with_error_handling do
-        res = api.result(uuid)
-        puts JSON.pretty_generate(res)
-      end
-    end
-
-    desc "search [QUERY]", "search for scans by [QUERY]"
-    method_option :size, type: :numeric, default: 100
-    method_option :offset, type: :numeric, default: 0
-    method_option :sort, type: :string, default: "_score"
-    def search(query)
-      with_error_handling do
-        res = api.search(query, size: options["size"], offset: options["offset"], sort: options["sort"])
-        puts JSON.pretty_generate(res)
-      end
-    end
-
-    desc "dom [UUID]", "get the DOM of a scan using the scan id [UUID]"
-    def dom(uuid)
-      with_error_handling do
-        res = api.dom(uuid)
-        puts res
-      end
-    end
-
-    desc "screenshot [UUID]", "get the screenshot(image/png) of a scan using the scan id [UUID]"
-    def screenshot(uuid)
-      with_error_handling do
-        res = api.screenshot(uuid)
-        puts res
-      end
-    end
-
-    no_commands do
-      def api
-        options["API_KEY"] ? API.new(options["API_KEY"]) : API.new
-      end
-
-      def with_error_handling
-        yield
-      rescue ArgumentError => _e
-        puts "Warning: please specify your urlscan.io API key via ENV['URLSCAN_API_KEY'] or --API-KEY"
-      rescue ResponseError => e
-        puts "Warning: #{e}"
-      end
-    end
-  end
+  class CLI < Commands::Community; end
 end

data/lib/urlscan/clients/base.rb

@@ -0,0 +1,85 @@
+# frozen_string_literal: true
+
+require "net/https"
+require "json"
+
+module UrlScan
+  module Clients
+    class Base
+      VERSION = 1
+      HOST = "urlscan.io"
+
+      attr_reader :key
+
+      def initialize(key = ENV["URLSCAN_API_KEY"])
+        @key = key
+      end
+
+      private
+
+      def url
+        @url ||= "https://#{HOST}/api/v#{VERSION}"
+      end
+
+      def url_for(path)
+        URI(url + path)
+      end
+
+      def https_options
+        if proxy = ENV["HTTPS_PROXY"]
+          uri = URI(proxy)
+          {
+            proxy_address: uri.hostname,
+            proxy_port: uri.port,
+            proxy_from_env: false,
+            use_ssl: true
+          }
+        else
+          { use_ssl: true }
+        end
+      end
+
+      def request(req)
+        Net::HTTP.start(HOST, 443, https_options) do |http|
+          response = http.request(req)
+
+          case response.code
+          when "200"
+            if response["Content-Type"].to_s.include? "application/json"
+              yield JSON.parse(response.body)
+            else
+              yield response.body
+            end
+          when "400" then raise ProcessingError, response.body
+          when "401" then raise AuthenticationError, response.body
+          when "404" then raise NotFound, response.body
+          when "429" then raise RateLimited, response.body
+          when "500" then raise InternalServerError, response.body
+          else
+            raise ResponseError, response.body
+          end
+        end
+      end
+
+      def default_headers
+        @default_headers ||= { "API-KEY": key }.compact
+      end
+
+      def get(path, params = {}, &block)
+        uri = url_for(path)
+        uri.query = URI.encode_www_form(params)
+
+        get = Net::HTTP::Get.new(uri, default_headers)
+        request(get, &block)
+      end
+
+      def post(path, json, &block)
+        post = Net::HTTP::Post.new(url_for(path), default_headers)
+        post.content_type = "application/json"
+        post.body = json.to_json
+
+        request(post, &block)
+      end
+    end
+  end
+end

data/lib/urlscan/clients/community.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+module UrlScan
+  module Clients
+    class Community < Base
+      # @return [Hash]
+      def submit(url, is_public = true)
+        params = { url: url, public: is_public ? "on" : "off" }
+        post("/scan/", params) { |json| json }
+      end
+
+      # @return [Hash]
+      def result(uuid)
+        get("/result/#{uuid}") { |json| json }
+      end
+
+      # @return [String]
+      def dom(uuid)
+        get("/dom/#{uuid}/") { |dom| dom }
+      end
+
+      def screenshot(uuid)
+        get("/screenshots/#{uuid}.png") { |png| png }
+      end
+
+      # @return [Hash]
+      def search(q, size: 100, search_after: nil)
+        params = { q: q, size: size, search_after: search_after }.compact
+        get("/search/", params) { |json| json }
+      end
+    end
+  end
+end

data/lib/urlscan/clients/pro.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+module UrlScan
+  module Clients
+    class Pro < Base
+      VERSION = 1
+      HOST = "urlscan.io"
+
+      # @return [Hash]
+      def brands
+        get("/brands") { |json| json }
+      end
+
+      # @return [Hash]
+      def kits
+        get("/kits") { |json| json }
+      end
+
+      # @return [Hash]
+      def phishfeed(q: "result.task.time:>now-24h", format: "json")
+        params = { q: q, format: format }
+        get("/phishfeed", params) { |json| json }
+      end
+
+      # @return [Hash]
+      def similar(uuid)
+        get("/result/#{uuid}/similar/") { |json| json }
+      end
+
+      private
+
+      def build_filter(filter)
+        return nil unless filter
+
+        filter.start_with?("$") ? filter : "$#{filter}"
+      end
+
+      def url
+        @url ||= "https://#{HOST}/api/v#{VERSION}/pro"
+      end
+    end
+  end
+end

data/lib/urlscan/commands/base.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+require "json"
+require "thor"
+
+module UrlScan
+  module Commands
+    class Base < Thor
+      class_option :API_KEY, type: :string
+
+      no_commands do
+        def api
+          options.key?("API_KEY") ? API.new(options["API_KEY"]) : API.new
+        end
+
+        def with_error_handling
+          yield
+        rescue ArgumentError => _e
+          puts "Warning: please specify your urlscan.io API key via ENV['URLSCAN_API_KEY'] or --API-KEY"
+        rescue ResponseError => e
+          puts "Warning: #{e}"
+        end
+      end
+    end
+  end
+end

data/lib/urlscan/commands/community.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+module UrlScan
+  module Commands
+    class Community < Base
+      desc "submit [URL]", "submit a scan to [URL]"
+      method_option :public, type: :boolean, default: true
+      def submit(url)
+        with_error_handling do
+          res = api.submit(url, options[:public])
+          puts JSON.pretty_generate(res)
+        end
+      end
+
+      desc "result [UUID]", "get the result of a scan using the scan id [UUID]"
+      def result(uuid)
+        with_error_handling do
+          res = api.result(uuid)
+          puts JSON.pretty_generate(res)
+        end
+      end
+
+      desc "search [QUERY]", "search for scans by [QUERY]"
+      method_option :size, type: :numeric, default: 100
+      method_option :search_after, type: :string
+      def search(query)
+        with_error_handling do
+          res = api.search(query, size: options["size"], search_after: options["search_after"])
+          puts JSON.pretty_generate(res)
+        end
+      end
+
+      desc "dom [UUID]", "get the DOM of a scan using the scan id [UUID]"
+      def dom(uuid)
+        with_error_handling do
+          res = api.dom(uuid)
+          puts res
+        end
+      end
+
+      desc "screenshot [UUID]", "get the screenshot(image/png) of a scan using the scan id [UUID]"
+      def screenshot(uuid)
+        with_error_handling do
+          res = api.screenshot(uuid)
+          puts res
+        end
+      end
+    end
+  end
+end

data/lib/urlscan/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module UrlScan
-  VERSION = "0.3.0"
+  VERSION = "0.7.0"
 end

data/renovate.json

@@ -0,0 +1,5 @@
+{
+  "extends": [
+    "config:base"
+  ]
+}

data/urlscan.gemspec

@@ -21,13 +21,12 @@ Gem::Specification.new do |spec|
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
 
-  spec.add_development_dependency "bundler", "~> 2.0"
+  spec.add_development_dependency "bundler", "~> 2.2"
   spec.add_development_dependency "coveralls", "~> 0.8"
-  spec.add_development_dependency "dotenv", "~> 2.5"
-  spec.add_development_dependency "rake", "~> 12.3"
-  spec.add_development_dependency "rspec", "~> 3.8"
-  spec.add_development_dependency "vcr", "~> 5.0"
-  spec.add_development_dependency "webmock", "~> 3.6"
+  spec.add_development_dependency "rake", "~> 13.0"
+  spec.add_development_dependency "rspec", "~> 3.10"
+  spec.add_development_dependency "vcr", "~> 6.0"
+  spec.add_development_dependency "webmock", "~> 3.12"
 
-  spec.add_runtime_dependency "thor", "~> 0.20"
+  spec.add_runtime_dependency "thor", "~> 1.1"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: urlscan
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.7.0
 platform: ruby
 authors:
 - Manabu Niseki
-autorequire: 
+autorequire:
 bindir: exe
 cert_chain: []
-date: 2019-08-18 00:00:00.000000000 Z
+date: 2021-04-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -16,14 +16,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.0'
+        version: '2.2'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.0'
+        version: '2.2'
 - !ruby/object:Gem::Dependency
   name: coveralls
   requirement: !ruby/object:Gem::Requirement
@@ -38,91 +38,77 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.8'
-- !ruby/object:Gem::Dependency
-  name: dotenv
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '2.5'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '2.5'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '12.3'
+        version: '13.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '12.3'
+        version: '13.0'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.8'
+        version: '3.10'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.8'
+        version: '3.10'
 - !ruby/object:Gem::Dependency
   name: vcr
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '5.0'
+        version: '6.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '5.0'
+        version: '6.0'
 - !ruby/object:Gem::Dependency
   name: webmock
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.6'
+        version: '3.12'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.6'
+        version: '3.12'
 - !ruby/object:Gem::Dependency
   name: thor
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.20'
+        version: '1.1'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.20'
-description: 
+        version: '1.1'
+description:
 email:
 - manabu.niseki@gmail.com
 executables:
@@ -131,10 +117,10 @@ extensions: []
 extra_rdoc_files: []
 files:
 - ".codeclimate.yml"
+- ".github/workflows/test.yml"
 - ".gitignore"
 - ".rspec"
 - ".rubocop.yml"
-- ".travis.yml"
 - Gemfile
 - LICENSE
 - README.md
@@ -143,14 +129,20 @@ files:
 - lib/urlscan.rb
 - lib/urlscan/api.rb
 - lib/urlscan/cli.rb
+- lib/urlscan/clients/base.rb
+- lib/urlscan/clients/community.rb
+- lib/urlscan/clients/pro.rb
+- lib/urlscan/commands/base.rb
+- lib/urlscan/commands/community.rb
 - lib/urlscan/exceptions.rb
 - lib/urlscan/version.rb
+- renovate.json
 - urlscan.gemspec
 homepage: https://github.com/ninoseki/urlscan
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -165,8 +157,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.4
-signing_key: 
+rubygems_version: 3.2.15
+signing_key:
 specification_version: 4
 summary: Ruby API client for urlscan.io
 test_files: []

data/.travis.yml

@@ -1,9 +0,0 @@
-sudo: false
-language: ruby
-cache: bundler
-rvm:
-  - 2.5
-  - 2.6
-before_install:
-  - gem update --system
-  - gem install bundler