urlscan 0.1.1 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 06ecedfa33d35b87c46e2230de22f8871d9e8d933dc232e11fd671d9b66de21a
-  data.tar.gz: 6712e92c91f06afb1122414e50ad484add46ca964880c87de40091a9e2a57489
+  metadata.gz: 042c7671173ca5f51491179a399986f61c075a4d410851017d43260e37c870f1
+  data.tar.gz: 5c090165f7c5d711dddae216934bb35efdf35c26a6864f6f9890841ad69ef7ca
 SHA512:
-  metadata.gz: 3119770d1ba5385836a736b89e2b4029fae5696202ec1c669302193c4b0db6c1b0838f39f644f83bb3c941ed44da5a99876aa744e6d0106e1fac56797037ace0
-  data.tar.gz: 25620724cf36ab95377f67886da10347b717474fdc708799cc00196ad7ad36e97691a9ea1618db83c7f7f92c6c8dad889b9fcf0b88f5163fbb4aef739061ecc4
+  metadata.gz: 65782649fe5229b9bed2090a57de4eed5ed85aa257b35b3cde8d3b732275ce3a39fcef2fcdb75fb54d9402682e0528d8e5b6ea7dbb5bb5ee24c93151be3ad64c
+  data.tar.gz: 106287f92bbea337c90d2f8af16fe1df6d856b1fb3c827388b051eaf0eae9503e76e8da23fea236bc9634c636137e96e51194fa01273a8e46efc01913b2193de

data/.rubocop.yml

@@ -1,6 +1,13 @@
 # Relaxed.Ruby.Style
 ## Version 2.2
 
+require:
+  - rubocop-performance
+  - rubocop-rspec
+
+AllCops:
+  TargetRubyVersion: 2.6
+
 Style/Alias:
   Enabled: false
   StyleGuide: https://relaxed.ruby.style/#stylealias

data/.travis.yml

@@ -1,8 +1,9 @@
+sudo: false
 language: ruby
 cache: bundler
 rvm:
-  - 2.4
   - 2.5
-script: bundle exec rake spec
-gemfile:
-  - Gemfile
+  - 2.6
+before_install:
+  - gem update --system
+  - gem install bundler

data/README.md

@@ -6,54 +6,58 @@
 
 ## Description
 
-Ruby API client for [urlscan.io](https://urlscan.io/)
+Ruby API client for [urlscan.io](https://urlscan.io/).
 
 ## Installation
 
 ```bash
-$ gem install urlscan
+gem install urlscan
 ```
 
 ## API usage
 
-Initialize the API:
-
 ```ruby
 require 'urlscan'
-# initialize the API by passing the API key.
-api = UrlScan::API.new(api_key)
-# or initialize the API by using `$URLSCAN_API_KEY` environment variable
-api = UrlScan::API.new
-```
 
-Submit:
+# when given nothing, it tries to load your API key from ENV["SHODAN_API_KEY"]
+api = UrlScan::API.new
+# or you can set it manually
+api = UrlScan::API.new(api_key)
 
-```ruby
+# Submit a URL to scan
 res = api.submit("https://wikipedia.org")
 puts res["result"] # => "https://urlscan.io/result/ac04bc14-4efe-439d-b356-8384843daf75/"
-```
 
-Result:
-
-```ruby
+# Get a scan result
 res = api.result("ac04bc14-4efe-439d-b356-8384843daf75")
 p res # => See the following URL as an example of the response.
       #    https://gist.github.com/ninoseki/a974d7e95629813615b380c30e737825#file-result-json
-```
 
-Search:
+# Get a DOM
+res = api.dom("ac04bc14-4efe-439d-b356-8384843daf75")
+p res
 
-```ruby
+# Search
 res = api.search("wikipedia.org")
 p res  # => See the following URL as an example of the reponse.
        #    https://gist.github.com/ninoseki/a974d7e95629813615b380c30e737825#file-search-json
 ```
 
+### Supported API endpoints
+
+| HTTP Method | URI            | API method                                                        |
+|-------------|----------------|-------------------------------------------------------------------|
+| POST        | /scan          | `UrlScan::API#submit(url, is_public = true)`                      |
+| GET         | /result/$uuid/ | `UrlScan::API#result(uuid)`                                       |
+| GET         | /dom/$uuid/    | `UrlScan::API#dom(uuid)`                                          |
+| GET         | /search        | `UrlScan::API#search(q, size = 100, offset = 0, sort = "_score")` |
+
 ## CLI usage
 
 ```bash
 $ urlscan
 Commands:
+  urlscan dom [UUID]      # get the DOM of a scan using the [UUID]
   urlscan help [COMMAND]  # Describe available commands or one specific command
   urlscan result [UUID]   # get the result of a scan using the [UUID]
   urlscan search [QUERY]  # search for scans by [QUERY]
@@ -61,4 +65,5 @@ Commands:
 
 Options:
   [--API-KEY=API_KEY]
+
 ```

data/lib/urlscan.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require "urlscan/api"
 require "urlscan/cli"
 require "urlscan/version"

data/lib/urlscan/api.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'net/https'
 require 'json'
 
@@ -6,25 +8,34 @@ require 'urlscan/exceptions'
 module UrlScan
   class API
     VERSION = 1
-    HOST = "urlscan.io".freeze
-    URL = "https://#{HOST}/api/v#{VERSION}".freeze
+    HOST = "urlscan.io"
+    URL = "https://#{HOST}/api/v#{VERSION}"
 
     attr_reader :key
 
     def initialize(key = ENV["URLSCAN_API_KEY"])
       raise ArgumentError, "`key` argument required" if key.nil?
+
       @key = key
     end
 
+    # @return [Hash]
     def submit(url, is_public = true)
       params = { url: url, public: is_public ? "on" : "off" }
       post("/scan/", params) { |json| json }
     end
 
+    # @return [Hash]
     def result(uuid)
       get("/result/#{uuid}") { |json| json }
     end
 
+    # @return [String]
+    def dom(uuid)
+      get("/dom/#{uuid}/") { |dom| dom }
+    end
+
+    # @return [Hash]
     def search(q, size = 100, offset = 0, sort = "_score")
       params = { q: q, size: size, offset: offset, sort: sort }
       query = URI.encode_www_form(params)
@@ -39,8 +50,8 @@ module UrlScan
       if proxy = ENV["HTTPS_PROXY"]
         uri = URI(proxy)
         {
-          proxy_address:  uri.hostname,
-          proxy_port:     uri.port,
+          proxy_address: uri.hostname,
+          proxy_port: uri.port,
           proxy_from_env: false,
           use_ssl: true
         }
@@ -54,7 +65,12 @@ module UrlScan
         response = http.request(req)
 
         case response.code
-        when '200' then yield JSON.parse(response.body)
+        when '200'
+          if response["Content-Type"].to_s.include? "application/json"
+            yield JSON.parse(response.body)
+          else
+            yield response.body
+          end
         when '400' then raise ProcessingError, response.body
         when '401' then raise AuthenticationError, response.body
         when '404' then raise NotFound, response.body

data/lib/urlscan/cli.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'json'
 require 'thor'
 
@@ -33,6 +35,14 @@ module UrlScan
       end
     end
 
+    desc "dom [UUID]", "get the DOM of a scan using the [UUID]"
+    def dom(uuid)
+      with_error_handling do
+        res = api.dom(uuid)
+        puts res
+      end
+    end
+
     no_commands do
       def api
         options[:API_KEY] ? API.new(options[:API_KEY]) : API.new
@@ -40,10 +50,10 @@ module UrlScan
 
       def with_error_handling
         yield
-      rescue ArgumentError => _
+      rescue ArgumentError => _e
         puts "Warning: please specify your urlscan.io API key via ENV['URLSCAN_API_KEY] or --API-KEY"
-      rescue ResponseError => e
-        puts "Warning: #{e}"
+      rescue ResponseError => _e
+        puts "Warning: #{_e}"
       end
     end
   end

data/lib/urlscan/exceptions.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module UrlScan
   class ResponseError < StandardError; end
 

data/lib/urlscan/version.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module UrlScan
-  VERSION = "0.1.1".freeze
+  VERSION = "0.2.0"
 end

data/urlscan.gemspec

@@ -1,4 +1,4 @@
-lib = File.expand_path("../lib", __FILE__)
+lib = File.expand_path('lib', __dir__)
 $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 require "urlscan/version"
 
@@ -19,13 +19,13 @@ Gem::Specification.new do |spec|
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
 
-  spec.add_development_dependency "bundler", "~> 1.16"
+  spec.add_development_dependency "bundler", "~> 2.0"
   spec.add_development_dependency "coveralls", "~> 0.8"
   spec.add_development_dependency "dotenv", "~> 2.5"
   spec.add_development_dependency "rake", "~> 12.3"
   spec.add_development_dependency "rspec", "~> 3.8"
   spec.add_development_dependency "vcr", "~> 4.0"
-  spec.add_development_dependency "webmock", "~> 3.4"
+  spec.add_development_dependency "webmock", "~> 3.5"
 
   spec.add_runtime_dependency "thor", "~> 0.19"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: urlscan
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.2.0
 platform: ruby
 authors:
 - Manabu Niseki
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2018-08-12 00:00:00.000000000 Z
+date: 2019-05-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -16,14 +16,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.16'
+        version: '2.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.16'
+        version: '2.0'
 - !ruby/object:Gem::Dependency
   name: coveralls
   requirement: !ruby/object:Gem::Requirement
@@ -100,14 +100,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.4'
+        version: '3.5'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.4'
+        version: '3.5'
 - !ruby/object:Gem::Dependency
   name: thor
   requirement: !ruby/object:Gem::Requirement
@@ -165,8 +165,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.7.6
+rubygems_version: 3.0.2
 signing_key: 
 specification_version: 4
 summary: Ruby API client for urlscan.io