osakana 0.2.1 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 21f7bc97c893c14d7c702ab8867c7ce63144f5c4cdb54f019e9c8fa99393e3b8
-  data.tar.gz: 3d41bfebf99474b4e9176b8dd2218e3febfa3a106ab70fb1e287de3650683733
+  metadata.gz: e16910b5733fad0a62fd5334ea405cda967abf3a8bce51042785a586340f049d
+  data.tar.gz: 171eff5ae67aa6030407e8ea4d34d2b3f38d592ebdb9992224ac4ce03b96b1d4
 SHA512:
-  metadata.gz: 7582cf02d5e62690d0b5db05aaf2eacc3360b6f9f53380ff4eec99701a00069bc673d0f4a65bfd0eb93fb759237ac820ec3bb576e0ade9362357d9a620f935be
-  data.tar.gz: 3c5d87e8746db988fcc29d67ee72c1907b982f714e8895080b1a0883c7d80f8913e653bd28db3955a0aa0f3aa785ddfa65e7a76bd11ff0ecfc180053d56122fd
+  metadata.gz: f8cc4687b7b665b65bc92ad4bf2506d57c71620b479ed1f62b14d4c5f685baa82763ae2d3f62d335196b5202e214cd1713e3318b0600d86a95f0c90fac0b545a
+  data.tar.gz: 7d883c02582709a28aa762550dd3ec93a4ab3756137f90df3e2b016c13adaab64e0d4997db9a8f989c2479c34650782a03cfb868aa70236ed9e707d8507e8297

data/README.md

@@ -8,10 +8,16 @@ Osakana is a Swiss army knife tool for my phishing research.
 
 ## Features
 
-- [Ayashige](https://github.com/ninoseki/ayashige) lookup
-- Censys lookup
-- Check newly registered domains on DNPedia by keyword
-- Slack integration
+- Lookup supports:
+  - [Ayashige](https://github.com/ninoseki/ayashige) lookup
+  - Censys lookup
+  - DNPedia (newly registered domains) lookup
+  - urlscan.io lookup
+- IoC enrichment:
+  - SecurityTrails integration (an API Key is required)
+  - Robtex integration
+- Notification:
+  - Slack notification
 
 ## Prerequisites
 
@@ -32,6 +38,7 @@ Commands:
   osakana censys_lookup [QUERY]          # lookup on Censys by a given query
   osakana check_newly_domains [KEYWORD]  # check newly registered domains on DNPedia by a given keyword
   osakana help [COMMAND]                 # Describe available commands or one specific command
+  osakana urlscan_lookup [QUERY]         # look up on urlscan.io by a given query
 ```
 
 ## Configuration
@@ -40,15 +47,21 @@ Commands:
 
 Please set the following environmental variables for enabling Censys lookup.
 
-- CENSYS_ID: your Censys API ID
-- CENSYS_SECRET: your Censys secret key
+- `CENSYS_ID`: your Censys API ID
+- `CENSYS_SECRET`: your Censys secret key
+
+### SecurityTrails
+
+Please set the following environmental variable for enabling SecurityTrails integration.
+
+- `SECURITYTRAILS_API_KEY`: your SecurityTrails API key
 
 ### Slack
 
-Please set the following environmental variables for enabling Slack integration.
+Please set the following environmental variables for enabling Slack notification.
 
-- SLACK_WEBHOOK_URL: A Slack webhook URL.
-- SLACK_CHANNEL_NAME: A Slask channel name which will be notified.
+- `SLACK_WEBHOOK_URL`: A Slack webhook URL.
+- `SLACK_CHANNEL_NAME`: A Slask channel name which will be notified.
 
 ## Screenshots
 

data/lib/osakana.rb

@@ -2,11 +2,17 @@
 
 require "osakana/version"
 
+require "osakana/enrichers/base"
+require "osakana/enrichers/robtex"
+require "osakana/enrichers/securitytrails"
+require "osakana/enrichers/enricher"
+
 require "osakana/website"
 
 require "osakana/ayashige"
 require "osakana/censys"
 require "osakana/dnpedia"
+require "osakana/urlscan"
 
 require "osakana/notifier"
 require "osakana/monitor"

data/lib/osakana/cli.rb

@@ -19,6 +19,12 @@ module Osakana
       Monitor.ayashige_lookup(keyword)
     end
 
+    desc "urlscan_lookup [QUERY]", "look up on urlscan.io by a given query"
+    option :size, type: :numeric, default: 100
+    def urlscan_lookup(query)
+      Monitor.urlscan_lookup(query, size: options.dig("size"))
+    end
+
     no_commands do
       def with_error_handling
         yield

data/lib/osakana/enrichers/base.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module Osakana
+  module Enrichers
+    class Base
+      def domain_to_ipv4(_domain)
+        raise NotImplementedError, "You must implement #{self.class}##{__method__}"
+      end
+
+      def ipv4_to_domain(_ipv4)
+        raise NotImplementedError, "You must implement #{self.class}##{__method__}"
+      end
+    end
+  end
+end

data/lib/osakana/enrichers/enricher.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module Osakana
+  module Enrichers
+    class Enricher < Base
+      def initialize
+        @enrichers = [SecurityTrails.new, Robtex.new]
+      end
+
+      def ipv4_to_domain(ipv4)
+        @enrichers.each do |enricher|
+          return enricher.ipv4_to_domain(ipv4)
+        rescue ArgumentError, ::SecurityTrails::Error, ::Robtex::ResponseError, URI::InvalidURIError => _
+          next
+        end
+        nil
+      end
+
+      def domain_to_ipv4(domain)
+        @enrichers.each do |enricher|
+          return enricher.domain_to_ipv4(domain)
+        rescue ArgumentError, ::SecurityTrails::Error, ::Robtex::ResponseError => _
+          next
+        end
+        nil
+      end
+    end
+  end
+end

data/lib/osakana/enrichers/robtex.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+require "robtex"
+
+module Osakana
+  module Enrichers
+    class Robtex < Base
+      def api
+        @api ||= ::Robtex::API.new
+      end
+
+      def domain_to_ipv4(domain)
+        results = api.fpdns(domain)
+        result = results.find do |res|
+          res.dig("rrtype") == "A"
+        end
+        result&.dig("rrdata")
+      end
+
+      def ipv4_to_domain(ipv4)
+        res = api.ip(ipv4)
+        active = res.dig("act")
+        active&.first&.dig("o")
+      end
+    end
+  end
+end

data/lib/osakana/enrichers/securitytrails.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+require "securitytrails"
+
+module Osakana
+  module Enrichers
+    class SecurityTrails < Base
+      def api
+        @api ||= ::SecurityTrails::API.new
+      end
+
+      def domain_to_ipv4(domain)
+        res = api.history.get_dns_history(domain, "a")
+        res&.records&.first&.values&.first&.ip
+      end
+
+      def ipv4_to_domain(ipv4)
+        res = api.domains.search( filter: { ipv4: ipv4 })
+        res&.records&.first&.hostname
+      end
+    end
+  end
+end

data/lib/osakana/monitor.rb

@@ -19,5 +19,11 @@ module Osakana
       attachements = websites.map(&:to_attachement)
       Notifier.notify("Ayashige keyword: #{keyword}", attachements)
     end
+
+    def self.urlscan_lookup(query, size:)
+      websites = Urlscan.lookup(query, size: size)
+      attachements = websites.map(&:to_attachement)
+      Notifier.notify("urlscan.io query: #{query}", attachements)
+    end
   end
 end

data/lib/osakana/urlscan.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+require "http"
+require "json"
+
+module Osakana
+  class Urlscan
+    BASE_URL = "https://urlscan.io/api/v1/search/"
+
+    def lookup(query, size: 100)
+      res = HTTP.get(BASE_URL, params: { q: query, size: size })
+      return [] unless res.code == 200
+
+      websites = []
+      json = JSON.parse(res.body.to_s)
+      results = json.dig("results") || []
+      results.each do |item|
+        domain = item.dig("page", "domain")
+        ipv4 = item.dig("page", "ip")
+        time = item.dig("task", "time")
+
+        websites << Website.new(domain: domain, ipv4: ipv4, date: time)
+      end
+      websites
+    end
+
+    def self.lookup(query, size: 100)
+      new.lookup(query, size: size)
+    end
+  end
+end

data/lib/osakana/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Osakana
-  VERSION = "0.2.1"
+  VERSION = "0.3.0"
 end

data/lib/osakana/website.rb

@@ -1,7 +1,6 @@
 # frozen_string_literal: true
 
 require "date"
-require "robtex"
 
 module Osakana
   class Website
@@ -16,29 +15,18 @@ module Osakana
       @domain = domain
       @date = date ? DateTime.parse(date).to_date.to_s : "N/A"
 
-      @robtex = Robtex::API.new
+      @enricher = Enrichers::Enricher.new
     end
 
     def domain
       @domain ||= [].tap do |out|
-        res = @robtex.ip(ipv4)
-        active = res.dig("act")
-        next unless active
-        next if active.empty?
-
-        out << active.first.dig("o")
+        out << @enricher.ipv4_to_domain(ipv4)
       end.first || "N/A"
     end
 
     def ipv4
       @ipv4 ||= [].tap do |out|
-        results = @robtex.fpdns(domain)
-        result = results.find do |res|
-          res.dig("rrtype") == "A"
-        end
-        next unless result
-
-        out << result.dig("rrdata")
+        out << @enricher.domain_to_ipv4(domain)
       end.first || "N/A"
     end
 

data/osakana.gemspec

@@ -34,6 +34,7 @@ Gem::Specification.new do |spec|
   spec.add_dependency "censu", "~> 0.2"
   spec.add_dependency "http", "~> 4.0"
   spec.add_dependency "robtex", "~> 0.1"
+  spec.add_dependency "securitytrails", "~> 0.1"
   spec.add_dependency "slack-incoming-webhooks", "~> 0.2"
   spec.add_dependency "thor", "~> 0.19"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: osakana
 version: !ruby/object:Gem::Version
-  version: 0.2.1
+  version: 0.3.0
 platform: ruby
 authors:
 - Manabu Niseki
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2019-02-22 00:00:00.000000000 Z
+date: 2019-03-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -136,6 +136,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.1'
+- !ruby/object:Gem::Dependency
+  name: securitytrails
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.1'
 - !ruby/object:Gem::Dependency
   name: slack-incoming-webhooks
   requirement: !ruby/object:Gem::Requirement
@@ -187,8 +201,13 @@ files:
 - lib/osakana/censys.rb
 - lib/osakana/cli.rb
 - lib/osakana/dnpedia.rb
+- lib/osakana/enrichers/base.rb
+- lib/osakana/enrichers/enricher.rb
+- lib/osakana/enrichers/robtex.rb
+- lib/osakana/enrichers/securitytrails.rb
 - lib/osakana/monitor.rb
 - lib/osakana/notifier.rb
+- lib/osakana/urlscan.rb
 - lib/osakana/version.rb
 - lib/osakana/website.rb
 - osakana.gemspec