urlscan 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: afc8c9c3a3da9d6d835e29753009f8ed8eec2f61
+  data.tar.gz: 559e28b2f5abb908384d8ca3167dce2a55ed0c06
+SHA512:
+  metadata.gz: 64119e3a093a5be4b5357170b2dab17398e892ecd3e2de068c2bf4768e6f9d9f3b72cf93559e2553ab7eb520f086e2f2cf383bbbfdbff2bfd336b04e3c300659
+  data.tar.gz: '09c0449187b23e492748f16ee0ae7b5181c355b0c3f97b99ff1b701895967a51575cca0b290ac418169c90584a6bfe28167aebe28a070aa969860e0d487724d9'

data/.codeclimate.yml

@@ -0,0 +1,12 @@
+version: "2"
+prepare:
+  fetch:
+    - url: "https://raw.githubusercontent.com/janlelis/relaxed.ruby.style/master/.rubocop.yml"
+      path: "alternate-rubocop-path.yml"
+plugins:
+  rubocop:
+    enabled: true
+    config:
+      file: "alternate-rubocop-path.yml"
+exclude_patterns:
+  - "spec/**/*"

data/.gitignore

@@ -0,0 +1,38 @@
+*.gem
+*.rbc
+/.config
+/coverage/
+/InstalledFiles
+/pkg/
+/spec/reports/
+/spec/examples.txt
+/test/tmp/
+/test/version_tmp/
+/tmp/
+/.vscode
+
+# Used by dotenv library to load environment variables.
+.env
+
+## Documentation cache and generated files:
+/.yardoc/
+/_yardoc/
+/doc/
+/rdoc/
+
+## Environment normalization:
+/.bundle/
+/vendor/bundle
+/lib/bundler/man/
+
+# for a library or gem, you might want to ignore these files since the code is
+# intended to run in multiple environments; otherwise, check them in:
+Gemfile.lock
+# .ruby-version
+# .ruby-gemset
+
+# unless supporting rvm < 1.11.0 or doing something fancy, ignore this:
+.rvmrc
+
+# rspec failure tracking
+.rspec_status

data/.rspec

@@ -0,0 +1,3 @@
+--format documentation
+--color
+--require spec_helper

data/.rubocop.yml

@@ -0,0 +1,168 @@
+# Relaxed.Ruby.Style
+## Version 2.1
+
+Style/Alias:
+  Enabled: false
+  StyleGuide: http://relaxed.ruby.style/#stylealias
+
+Style/AsciiComments:
+  Enabled: false
+  StyleGuide: http://relaxed.ruby.style/#styleasciicomments
+
+Style/BeginBlock:
+  Enabled: false
+  StyleGuide: http://relaxed.ruby.style/#stylebeginblock
+
+Style/BlockDelimiters:
+  Enabled: false
+  StyleGuide: http://relaxed.ruby.style/#styleblockdelimiters
+
+Style/CommentAnnotation:
+  Enabled: false
+  StyleGuide: http://relaxed.ruby.style/#stylecommentannotation
+
+Style/Documentation:
+  Enabled: false
+  StyleGuide: http://relaxed.ruby.style/#styledocumentation
+
+Layout/DotPosition:
+  Enabled: false
+  StyleGuide: http://relaxed.ruby.style/#layoutdotposition
+
+Style/DoubleNegation:
+  Enabled: false
+  StyleGuide: http://relaxed.ruby.style/#styledoublenegation
+
+Style/EndBlock:
+  Enabled: false
+  StyleGuide: http://relaxed.ruby.style/#styleendblock
+
+Style/FormatString:
+  Enabled: false
+  StyleGuide: http://relaxed.ruby.style/#styleformatstring
+
+Style/IfUnlessModifier:
+  Enabled: false
+  StyleGuide: http://relaxed.ruby.style/#styleifunlessmodifier
+
+Style/Lambda:
+  Enabled: false
+  StyleGuide: http://relaxed.ruby.style/#stylelambda
+
+Style/ModuleFunction:
+  Enabled: false
+  StyleGuide: http://relaxed.ruby.style/#stylemodulefunction
+
+Style/MultilineBlockChain:
+  Enabled: false
+  StyleGuide: http://relaxed.ruby.style/#stylemultilineblockchain
+
+Style/NegatedIf:
+  Enabled: false
+  StyleGuide: http://relaxed.ruby.style/#stylenegatedif
+
+Style/NegatedWhile:
+  Enabled: false
+  StyleGuide: http://relaxed.ruby.style/#stylenegatedwhile
+
+Style/ParallelAssignment:
+  Enabled: false
+  StyleGuide: http://relaxed.ruby.style/#styleparallelassignment
+
+Style/PercentLiteralDelimiters:
+  Enabled: false
+  StyleGuide: http://relaxed.ruby.style/#stylepercentliteraldelimiters
+
+Style/PerlBackrefs:
+  Enabled: false
+  StyleGuide: http://relaxed.ruby.style/#styleperlbackrefs
+
+Style/Semicolon:
+  Enabled: false
+  StyleGuide: http://relaxed.ruby.style/#stylesemicolon
+
+Style/SignalException:
+  Enabled: false
+  StyleGuide: http://relaxed.ruby.style/#stylesignalexception
+
+Style/SingleLineBlockParams:
+  Enabled: false
+  StyleGuide: http://relaxed.ruby.style/#stylesinglelineblockparams
+
+Style/SingleLineMethods:
+  Enabled: false
+  StyleGuide: http://relaxed.ruby.style/#stylesinglelinemethods
+
+Layout/SpaceBeforeBlockBraces:
+  Enabled: false
+  StyleGuide: http://relaxed.ruby.style/#layoutspacebeforeblockbraces
+
+Layout/SpaceInsideParens:
+  Enabled: false
+  StyleGuide: http://relaxed.ruby.style/#layoutspaceinsideparens
+
+Style/SpecialGlobalVars:
+  Enabled: false
+  StyleGuide: http://relaxed.ruby.style/#stylespecialglobalvars
+
+Style/StringLiterals:
+  Enabled: false
+  StyleGuide: http://relaxed.ruby.style/#stylestringliterals
+
+Style/TrailingCommaInArguments:
+  Enabled: false
+  StyleGuide: http://relaxed.ruby.style/#styletrailingcommainarguments
+
+Style/TrailingCommaInLiteral:
+  Enabled: false
+  StyleGuide: http://relaxed.ruby.style/#styletrailingcommainliteral
+
+Style/WhileUntilModifier:
+  Enabled: false
+  StyleGuide: http://relaxed.ruby.style/#stylewhileuntilmodifier
+
+Style/WordArray:
+  Enabled: false
+  StyleGuide: http://relaxed.ruby.style/#stylewordarray
+
+Lint/AmbiguousRegexpLiteral:
+  Enabled: false
+  StyleGuide: http://relaxed.ruby.style/#lintambiguousregexpliteral
+
+Lint/AssignmentInCondition:
+  Enabled: false
+  StyleGuide: http://relaxed.ruby.style/#lintassignmentincondition
+
+Metrics/AbcSize:
+  Enabled: false
+
+Metrics/BlockNesting:
+  Enabled: false
+
+Metrics/BlockLength:
+  Exclude:
+    - 'spec/**/*'
+
+Metrics/ClassLength:
+  Enabled: false
+
+Metrics/ModuleLength:
+  Enabled: false
+
+Metrics/CyclomaticComplexity:
+  Enabled: false
+
+Metrics/LineLength:
+  Enabled: false
+
+Metrics/MethodLength:
+  Enabled: false
+
+Metrics/ParameterLists:
+  Enabled: false
+
+Metrics/PerceivedComplexity:
+  Enabled: false
+
+AllCops:
+  TargetRubyVersion: 2.4

data/.travis.yml

@@ -0,0 +1,7 @@
+language: ruby
+cache: bundler
+rvm:
+  - 2.4
+script: bundle exec rake spec
+gemfile:
+  - Gemfile

data/Gemfile

@@ -0,0 +1,3 @@
+source "https://rubygems.org"
+
+gemspec

data/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2018 Manabu Niseki
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/README.md

@@ -0,0 +1,63 @@
+# UrlScan
+
+[![Build Status](https://travis-ci.org/ninoseki/urlscan.svg?branch=master)](https://travis-ci.org/ninoseki/urlscan)
+[![Maintainability](https://api.codeclimate.com/v1/badges/c6625486f2d57039adef/maintainability)](https://codeclimate.com/github/ninoseki/urlscan/maintainability)
+
+## Description
+
+Ruby API client for [urlscan.io](https://urlscan.io/)
+
+## Installation
+
+```bash
+$ gem install urlscan
+```
+
+## API usage
+
+Initialize the API:
+
+```ruby
+require 'urlscan'
+# initialize the API by passing the API key.
+api = UrlScan::API.new(api_key)
+# or initialize the API by using `$URLSCAN_API_KEY` environment variable
+api = UrlScan::API.new
+```
+
+Submit:
+
+```ruby
+res = api.submit("https://wikipedia.org")
+puts res["result"] # => "https://urlscan.io/result/ac04bc14-4efe-439d-b356-8384843daf75/"
+```
+
+Result:
+
+```ruby
+res = api.result("ac04bc14-4efe-439d-b356-8384843daf75")
+p res # => See the following URL as an example of the response.
+      #    https://gist.github.com/ninoseki/a974d7e95629813615b380c30e737825#file-result-json
+```
+
+Search:
+
+```ruby
+res = api.search("wikipedia.org")
+p res  # => See the following URL as an example of the reponse.
+       #    https://gist.github.com/ninoseki/a974d7e95629813615b380c30e737825#file-search-json
+```
+
+## CLI usage
+
+```bash
+$ urlscan
+Commands:
+  urlscan help [COMMAND]  # Describe available commands or one specific command
+  urlscan result [UUID]   # get the result of a scan using the [UUID]
+  urlscan search [QUERY]  # search for scans by [QUERY]
+  urlscan submit [URL]    # submit a scan to [URL]
+
+Options:
+  [--API-KEY=API_KEY]
+```

data/Rakefile

@@ -0,0 +1,6 @@
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new(:spec)
+
+task default: :spec

data/exe/urlscan

@@ -0,0 +1,5 @@
+#!/usr/bin/env ruby
+
+require "urlscan"
+
+UrlScan::CLI.start

data/lib/urlscan.rb

@@ -0,0 +1,3 @@
+require "urlscan/api"
+require "urlscan/cli"
+require "urlscan/version"

data/lib/urlscan/api.rb

@@ -0,0 +1,86 @@
+require 'net/https'
+require 'json'
+
+require 'urlscan/exceptions'
+
+module UrlScan
+  class API
+    VERSION = 1
+    HOST = "urlscan.io".freeze
+    URL = "https://#{HOST}/api/v#{VERSION}".freeze
+
+    attr_reader :key
+
+    def initialize(key = ENV["URLSCAN_API_KEY"])
+      raise ArgumentError, "`key` argument required" if key.nil?
+      @key = key
+    end
+
+    def submit(url, is_public = true)
+      params = { url: url, public: is_public ? "on" : "off" }
+      post("/scan/", params) { |json| json }
+    end
+
+    def result(uuid)
+      get("/result/#{uuid}") { |json| json }
+    end
+
+    def search(q, size = 100, offset = 0, sort = "_score")
+      params = { q: q, size: size, offset: offset, sort: sort }
+      query = URI.encode_www_form(params)
+      get("/search/?#{query}") { |json| json }
+    end
+
+    def url_for(path)
+      URI(URL + path)
+    end
+
+    def https_options
+      if proxy = ENV["HTTPS_PROXY"]
+        uri = URI(proxy)
+        {
+          proxy_address:  uri.hostname,
+          proxy_port:     uri.port,
+          proxy_from_env: false,
+          use_ssl: true
+        }
+      else
+        { use_ssl: true }
+      end
+    end
+
+    def request(req)
+      Net::HTTP.start(HOST, 443, https_options) do |http|
+        response = http.request(req)
+
+        case response.code
+        when '200' then yield JSON.parse(response.body)
+        when '400' then raise ProcessingError, response.body
+        when '401' then raise AuthenticationError, response.body
+        when '404' then raise NotFound, response.body
+        when '429' then raise RateLimited, response.body
+        when '500' then raise InternalServerError, response.body
+        else
+          raise ResponseError, response.body
+        end
+      end
+    end
+
+    def auth_header
+      { "API-KEY": key }
+    end
+
+    def get(path, &block)
+      get = Net::HTTP::Get.new(url_for(path), auth_header)
+      request(get, &block)
+    end
+
+    def post(path, json, &block)
+      post = Net::HTTP::Post.new(url_for(path), auth_header)
+      post.content_type = 'application/json'
+      post.body = json.to_json
+
+      request(post, &block)
+    end
+  end
+end

data/lib/urlscan/cli.rb

@@ -0,0 +1,50 @@
+require 'json'
+require 'thor'
+
+module UrlScan
+  class CLI < Thor
+    class_option :API_KEY, type: :string
+
+    desc "submit [URL]", "submit a scan to [URL]"
+    method_option :public, type: :boolean, default: true
+    def submit(url)
+      with_error_handling do
+        res = api.submit(url, options[:public])
+        puts JSON.pretty_generate(res)
+      end
+    end
+
+    desc "result [UUID]", "get the result of a scan using the [UUID]"
+    def result(uuid)
+      with_error_handling do
+        res = api.result(uuid)
+        puts JSON.pretty_generate(res)
+      end
+    end
+
+    desc "search [QUERY]", "search for scans by [QUERY]"
+    method_option :size, type: :numeric, default: 100
+    method_option :offset, type: :numeric, default: 0
+    method_option :sort, type: :string, default: "_score"
+    def search(query)
+      with_error_handling do
+        res = api.search(query, options[:size], options[:offset], options[:sort])
+        puts JSON.pretty_generate(res)
+      end
+    end
+
+    no_commands do
+      def api
+        options[:API_KEY] ? API.new(options[:API_KEY]) : API.new
+      end
+
+      def with_error_handling
+        yield
+      rescue ArgumentError => _
+        puts "Warning: please specify your urlscan.io API key via ENV['URLSCAN_API_KEY] or --API-KEY"
+      rescue ResponseError => e
+        puts "Warning: #{e}"
+      end
+    end
+  end
+end

data/lib/urlscan/exceptions.rb

@@ -0,0 +1,13 @@
+module UrlScan
+  class ResponseError < StandardError; end
+
+  class AuthenticationError < ResponseError; end
+
+  class NotFound < ResponseError; end
+
+  class ProcessingError < ResponseError; end
+
+  class RateLimited < ResponseError; end
+
+  class InternalServerError < ResponseError; end
+end

data/lib/urlscan/version.rb

@@ -0,0 +1,3 @@
+module UrlScan
+  VERSION = "0.1.0".freeze
+end

data/urlscan.gemspec

@@ -0,0 +1,30 @@
+lib = File.expand_path("../lib", __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require "urlscan/version"
+
+Gem::Specification.new do |spec|
+  spec.name          = "urlscan"
+  spec.version       = UrlScan::VERSION
+  spec.authors       = ["Manabu Niseki"]
+  spec.email         = ["manabu.niseki@gmail.com"]
+
+  spec.summary       = "Ruby API client for urlscan.io"
+  spec.homepage      = "https://github.com/ninoseki/urlscan"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files -z`.split("\x0").reject do |f|
+    f.match(%r{^(test|spec|features)/})
+  end
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.16"
+  spec.add_development_dependency "dotenv", "~> 2.2"
+  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "rspec", "~> 3.0"
+  spec.add_development_dependency "vcr", "~> 4.0"
+  spec.add_development_dependency "webmock", "~> 3.1"
+
+  spec.add_runtime_dependency "thor", "~> 0.20"
+end

metadata

@@ -0,0 +1,159 @@
+--- !ruby/object:Gem::Specification
+name: urlscan
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Manabu Niseki
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2018-01-19 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.16'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.16'
+- !ruby/object:Gem::Dependency
+  name: dotenv
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.2'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.2'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: vcr
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.0'
+- !ruby/object:Gem::Dependency
+  name: webmock
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.1'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.1'
+- !ruby/object:Gem::Dependency
+  name: thor
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.20'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.20'
+description: 
+email:
+- manabu.niseki@gmail.com
+executables:
+- urlscan
+extensions: []
+extra_rdoc_files: []
+files:
+- ".codeclimate.yml"
+- ".gitignore"
+- ".rspec"
+- ".rubocop.yml"
+- ".travis.yml"
+- Gemfile
+- LICENSE
+- README.md
+- Rakefile
+- exe/urlscan
+- lib/urlscan.rb
+- lib/urlscan/api.rb
+- lib/urlscan/cli.rb
+- lib/urlscan/exceptions.rb
+- lib/urlscan/version.rb
+- urlscan.gemspec
+homepage: https://github.com/ninoseki/urlscan
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.6.11
+signing_key: 
+specification_version: 4
+summary: Ruby API client for urlscan.io
+test_files: []