urlscan 0.4.1 → 0.8.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c64c50eef43387533567d006b5a7cc5d91e9756c828d8c2abb0c71fb7cdfd8a0
-  data.tar.gz: 9804b57e3aa1e639af02f6d251572fde399e244a4978fd215de652451c9301b9
+  metadata.gz: 22f4d64dfca8b7ee22806108031b47a532dd3968e60afa641352c4a471647c51
+  data.tar.gz: a5b30512b61221b181e60e6712d327d92de34879b0268ea10b53cb2e0feb2748
 SHA512:
-  metadata.gz: f314f8dc01685bc52b4d4e0546cdf121ad26c82615c7021f2eb4341ed034dc4ee2ec6e00970e2749419baccf00991ae6aea92ead472b1ff335a25492d537ac3d
-  data.tar.gz: 9466cdb9855100e645554851697128a7df854afc1bb1fd34b6561e4ea1eb0d137457b43f694a88dbdd7047cdbcf7b42431774a081357c27717f080ec382c2d7e
+  metadata.gz: f41fc1f52104b46aa979e822bd11618888dcadefd0bdbfab98d8428621dad1583945cbcb3b80271f4db72023307bea6283bacc830963ba2941e7afde81d6e6ea
+  data.tar.gz: 829b96a975ac28e5417594e67644bc9578cd48c24ce92e349007a3f427659de05dbaa8e355da4c6b1f07458062efe06117a5c45e44bc74246fadac762209c54f

data/.github/workflows/test.yml

@@ -0,0 +1,23 @@
+name: Ruby CI
+
+on: [pull_request]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    strategy:
+      fail-fast: false
+      matrix:
+        ruby: [2.7, "3.0"]
+
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Ruby
+        uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: ${{ matrix.ruby }}
+          bundler-cache: true
+      - name: Build and test with Rake
+        run: |
+          bundle exec rake

data/README.md

@@ -1,13 +1,13 @@
 # urlscan
 
 [![Gem Version](https://badge.fury.io/rb/urlscan.svg)](https://badge.fury.io/rb/urlscan)
-[![Build Status](https://travis-ci.org/ninoseki/urlscan.svg?branch=master)](https://travis-ci.org/ninoseki/urlscan)
+[![Ruby CI](https://github.com/ninoseki/urlscan/actions/workflows/test.yml/badge.svg)](https://github.com/ninoseki/urlscan/actions/workflows/test.yml)
 [![Maintainability](https://api.codeclimate.com/v1/badges/c6625486f2d57039adef/maintainability)](https://codeclimate.com/github/ninoseki/urlscan/maintainability)
 [![Coverage Status](https://coveralls.io/repos/github/ninoseki/urlscan/badge.svg?branch=master)](https://coveralls.io/github/ninoseki/urlscan?branch=master)
 
 ## Description
 
-Ruby API client for [urlscan.io](https://urlscan.io/).
+[urlscan.io](https://urlscan.io/) API wrapper for Ruby.
 
 ## Installation
 
@@ -40,21 +40,22 @@ res = api.search("wikipedia.org")
 
 ## Supported API endpoints
 
-| HTTP Method | URI                     | API method                                                                    |
-|-------------|-------------------------|-------------------------------------------------------------------------------|
-| POST        | /scan                   | `UrlScan::Clients::Community#submit(url, is_public = true)`                   |
-| GET         | /result/`uuid`/         | `UrlScan::Clients::Community#result(uuid)`                                    |
-| GET         | /dom/`uuid`/            | `UrlScan::Clients::Community#dom(uuid)`                                       |
-| GET         | /screenshots/`uuid`.png | `UrlScan::Clients::Community#screenshot(uuid)`                                |
-| GET         | /search                 | `UrlScan::Clients::Community#search(q, size: 100, offset: 0, sort: "_score")` |
+| HTTP Method | URI                     | API method                                                                                                                                 |
+|-------------|-------------------------|--------------------------------------------------------------------------------------------------------------------------------------------|
+| POST        | /scan                   | `UrlScan::Clients::Community#submit(url, customagent: nil, referer: nil, visibility: nil, tags: nil, override_safety: nil, country: nil))` |
+| GET         | /result/`uuid`/         | `UrlScan::Clients::Community#result(uuid)`                                                                                                 |
+| GET         | /dom/`uuid`/            | `UrlScan::Clients::Community#dom(uuid)`                                                                                                    |
+| GET         | /screenshots/`uuid`.png | `UrlScan::Clients::Community#screenshot(uuid)`                                                                                             |
+| GET         | /search                 | `UrlScan::Clients::Community#search(q, size: 100, search_after: nil)`                                                                      |
 
 ### Pro
 
-| HTTP Method | URI     | API method                                                        |
-|-------------|---------|-------------------------------------------------------------------|
-| GET         | /search | `UrlScan::Clients::Pro#search(query: nil, filter: nil, size: 50)` |
-| GET         | /brands | `UrlScan::Clients::Pro#brands`                                    |
-| GET         | /kits   | `UrlScan::Clients::Pro#kits`                                      |
+| HTTP Method | URI                     | API method                                                                                                                                    |
+|-------------|-------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------|
+| GET         | /brands                 | `UrlScan::Clients::Pro#brands`                                                                                                                |
+| GET         | /kits                   | `UrlScan::Clients::Pro#kits`                                                                                                                  |
+| GET         | /phishfeed              | `UrlScan::Clients::Pro#phishfeed(q: "result.task.time:>now-24h", format: "json", limit: ni)`                                                  |
+| GET         | /result/`uuid`/similar/ | `UrlScan::Clients::Pro#similar(uuid, q: nil, size: nil, search_after: nil, threshold: nil, min_size: nil, method: nil, resource_types: nil))` |
 
 ## CLI usage
 
@@ -72,15 +73,3 @@ Options:
   [--API-KEY=API_KEY]
 
 ```
-
-### Pro
-
-```bash
-$ urlscan pro
-Commands:
-  urlscan pro help [COMMAND]  # Describe subcommands or one specific subcommand
-  urlscan pro search          # search for scans
-
-Options:
-  [--API-KEY=API_KEY]
-```

data/lib/urlscan/cli.rb

@@ -1,8 +1,5 @@
 # frozen_string_literal: true
 
 module UrlScan
-  class CLI < Commands::Community
-    desc "pro", "PRO api commands"
-    subcommand "pro", Commands::Pro
-  end
+  class CLI < Commands::Community; end
 end

data/lib/urlscan/clients/base.rb

@@ -18,7 +18,7 @@ module UrlScan
       private
 
       def url
-        @url ||= "https://#{self.class::HOST}/api/v#{self.class::VERSION}"
+        @url ||= "https://#{HOST}/api/v#{VERSION}"
       end
 
       def url_for(path)
@@ -61,17 +61,20 @@ module UrlScan
         end
       end
 
-      def auth_header
-        { "API-KEY": key }
+      def default_headers
+        @default_headers ||= { "API-KEY": key }.compact
       end
 
-      def get(path, &block)
-        get = Net::HTTP::Get.new(url_for(path))
+      def get(path, params = {}, &block)
+        uri = url_for(path)
+        uri.query = URI.encode_www_form(params)
+
+        get = Net::HTTP::Get.new(uri, default_headers)
         request(get, &block)
       end
 
       def post(path, json, &block)
-        post = Net::HTTP::Post.new(url_for(path), auth_header)
+        post = Net::HTTP::Post.new(url_for(path), default_headers)
         post.content_type = "application/json"
         post.body = json.to_json
 

data/lib/urlscan/clients/community.rb

@@ -4,10 +4,16 @@ module UrlScan
   module Clients
     class Community < Base
       # @return [Hash]
-      def submit(url, is_public = true)
-        raise ArgumentError, "API key is required for this method." if key.nil?
-
-        params = { url: url, public: is_public ? "on" : "off" }
+      def submit(url, customagent: nil, referer: nil, visibility: nil, tags: nil, override_safety: nil, country: nil)
+        params = {
+          url: url,
+          customagent: customagent,
+          referer: referer,
+          visibility: visibility,
+          tags: tags,
+          overrideSafety: override_safety,
+          country: country
+        }.compact
         post("/scan/", params) { |json| json }
       end
 
@@ -26,10 +32,9 @@ module UrlScan
       end
 
       # @return [Hash]
-      def search(q, size: 100, offset: 0, sort: "_score")
-        params = { q: q, size: size, offset: offset, sort: sort }
-        query = URI.encode_www_form(params)
-        get("/search/?#{query}") { |json| json }
+      def search(q, size: 100, search_after: nil)
+        params = { q: q, size: size, search_after: search_after }.compact
+        get("/search/", params) { |json| json }
       end
     end
   end

data/lib/urlscan/clients/pro.rb

@@ -4,32 +4,41 @@ module UrlScan
   module Clients
     class Pro < Base
       VERSION = 1
-      HOST = "pro.urlscan.com"
+      HOST = "urlscan.io"
 
       # @return [Hash]
-      def search(query: nil, filter: nil, size: 50)
-        raise ArgumentError, "API key is required for this method." if key.nil?
-
-        filter = build_filter(filter)
-        params = { q: query, size: size }.compact
-        uri_query = URI.encode_www_form(params)
-        uri_query << "&filter=#{filter}" if filter
-
-        get("/search?#{uri_query}") { |json| json }
+      def brands
+        get("/brands") { |json| json }
       end
 
       # @return [Hash]
-      def brands
-        raise ArgumentError, "API key is required for this method." if key.nil?
+      def kits
+        get("/kits") { |json| json }
+      end
 
-        get("/brands") { |json| json }
+      # @return [Hash]
+      def phishfeed(q: "result.task.time:>now-24h", format: "json", limit: nil)
+        params = { q: q, format: format, limit: limit }.compact
+        get("/phishfeed", params) { |json| json }
       end
 
       # @return [Hash]
-      def kits
-        raise ArgumentError, "API key is required for this method." if key.nil?
+      def similar(uuid, q: nil, size: nil, search_after: nil, threshold: nil, min_size: nil, method: nil, resource_types: nil)
+        params = {
+          q: q,
+          size: size,
+          search_after: search_after,
+          threshold: threshold,
+          minSize: min_size,
+          method: method,
+          resourceTypes: resource_types
+        }.compact
+        get("/result/#{uuid}/similar/", params) { |json| json }
+      end
 
-        get("/kits") { |json| json }
+      # @return [Hash]
+      def scanners
+        get("/livescan/scanners/") { |json| json }
       end
 
       private
@@ -41,12 +50,7 @@ module UrlScan
       end
 
       def url
-        @url ||= "https://#{self.class::HOST}/api/v#{self.class::VERSION}/pro"
-      end
-
-      def get(path, &block)
-        get = Net::HTTP::Get.new(url_for(path), auth_header)
-        request(get, &block)
+        @url ||= "https://#{HOST}/api/v#{VERSION}/pro"
       end
     end
   end

data/lib/urlscan/commands/community.rb

@@ -4,10 +4,23 @@ module UrlScan
   module Commands
     class Community < Base
       desc "submit [URL]", "submit a scan to [URL]"
-      method_option :public, type: :boolean, default: true
+      method_option :customagent, type: :string
+      method_option :referer, type: :string
+      method_option :visibility, type: :string
+      method_option :tags, type: :array
+      method_option :override_safety, type: :string
+      method_option :country, type: :string
       def submit(url)
         with_error_handling do
-          res = api.submit(url, options[:public])
+          res = api.submit(
+            url,
+            customagent: options[:customagent],
+            referer: options[:referer],
+            visibility: options[:visibility],
+            tags: options[:tags],
+            override_safety: options[:override_safety],
+            country: options[:country]
+          )
           puts JSON.pretty_generate(res)
         end
       end
@@ -22,11 +35,10 @@ module UrlScan
 
       desc "search [QUERY]", "search for scans by [QUERY]"
       method_option :size, type: :numeric, default: 100
-      method_option :offset, type: :numeric, default: 0
-      method_option :sort, type: :string, default: "_score"
+      method_option :search_after, type: :string
       def search(query)
         with_error_handling do
-          res = api.search(query, size: options["size"], offset: options["offset"], sort: options["sort"])
+          res = api.search(query, size: options["size"], search_after: options["search_after"])
           puts JSON.pretty_generate(res)
         end
       end

data/lib/urlscan/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module UrlScan
-  VERSION = "0.4.1"
+  VERSION = "0.8.0"
 end

data/lib/urlscan.rb

@@ -9,7 +9,6 @@ require "urlscan/clients/pro"
 
 require "urlscan/commands/base"
 require "urlscan/commands/community"
-require "urlscan/commands/pro"
 
 require "urlscan/api"
 require "urlscan/cli"

data/renovate.json

@@ -0,0 +1,5 @@
+{
+  "extends": [
+    "config:base"
+  ]
+}

data/urlscan.gemspec

@@ -21,12 +21,12 @@ Gem::Specification.new do |spec|
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
 
-  spec.add_development_dependency "bundler", "~> 2.0"
-  spec.add_development_dependency "coveralls", "~> 0.8"
-  spec.add_development_dependency "rake", "~> 12.3"
-  spec.add_development_dependency "rspec", "~> 3.8"
-  spec.add_development_dependency "vcr", "~> 5.0"
-  spec.add_development_dependency "webmock", "~> 3.7"
+  spec.add_development_dependency "bundler", "~> 2.2"
+  spec.add_development_dependency "coveralls_reborn", "~> 0.23"
+  spec.add_development_dependency "rake", "~> 13.0"
+  spec.add_development_dependency "rspec", "~> 3.10"
+  spec.add_development_dependency "vcr", "~> 6.0"
+  spec.add_development_dependency "webmock", "~> 3.14"
 
-  spec.add_runtime_dependency "thor", "~> 0.20"
+  spec.add_runtime_dependency "thor", "~> 1.1"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: urlscan
 version: !ruby/object:Gem::Version
-  version: 0.4.1
+  version: 0.8.0
 platform: ruby
 authors:
 - Manabu Niseki
-autorequire: 
+autorequire:
 bindir: exe
 cert_chain: []
-date: 2019-09-19 00:00:00.000000000 Z
+date: 2021-12-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -16,99 +16,99 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.0'
+        version: '2.2'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.0'
+        version: '2.2'
 - !ruby/object:Gem::Dependency
-  name: coveralls
+  name: coveralls_reborn
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.8'
+        version: '0.23'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.8'
+        version: '0.23'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '12.3'
+        version: '13.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '12.3'
+        version: '13.0'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.8'
+        version: '3.10'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.8'
+        version: '3.10'
 - !ruby/object:Gem::Dependency
   name: vcr
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '5.0'
+        version: '6.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '5.0'
+        version: '6.0'
 - !ruby/object:Gem::Dependency
   name: webmock
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.7'
+        version: '3.14'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.7'
+        version: '3.14'
 - !ruby/object:Gem::Dependency
   name: thor
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.20'
+        version: '1.1'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.20'
-description: 
+        version: '1.1'
+description:
 email:
 - manabu.niseki@gmail.com
 executables:
@@ -117,10 +117,10 @@ extensions: []
 extra_rdoc_files: []
 files:
 - ".codeclimate.yml"
+- ".github/workflows/test.yml"
 - ".gitignore"
 - ".rspec"
 - ".rubocop.yml"
-- ".travis.yml"
 - Gemfile
 - LICENSE
 - README.md
@@ -134,15 +134,15 @@ files:
 - lib/urlscan/clients/pro.rb
 - lib/urlscan/commands/base.rb
 - lib/urlscan/commands/community.rb
-- lib/urlscan/commands/pro.rb
 - lib/urlscan/exceptions.rb
 - lib/urlscan/version.rb
+- renovate.json
 - urlscan.gemspec
 homepage: https://github.com/ninoseki/urlscan
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -157,8 +157,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.4
-signing_key: 
+rubygems_version: 3.2.14
+signing_key:
 specification_version: 4
 summary: Ruby API client for urlscan.io
 test_files: []

data/.travis.yml

@@ -1,9 +0,0 @@
-sudo: false
-language: ruby
-cache: bundler
-rvm:
-  - 2.5
-  - 2.6
-before_install:
-  - gem update --system
-  - gem install bundler

data/lib/urlscan/commands/pro.rb

@@ -1,18 +0,0 @@
-# frozen_string_literal: true
-
-module UrlScan
-  module Commands
-    class Pro < Base
-      desc "search", "search for scans"
-      method_option :size, type: :numeric, default: 50
-      method_option :query, type: :string
-      method_option :filter, type: :string
-      def search
-        with_error_handling do
-          res = api.pro.search(query: options["query"], filter: options["filter"], size: options["size"])
-          puts JSON.pretty_generate(res)
-        end
-      end
-    end
-  end
-end