ukemi 0.1.0 → 0.4.1

data/exe/ukemi

@@ -5,4 +5,6 @@ $LOAD_PATH.unshift("#{__dir__}/../lib")
 
 require "ukemi"
 
-Ukemi::CLI.start
+ARGV.unshift(Ukemi::CLI.default_task) unless Ukemi::CLI.all_tasks.key?(ARGV[0])
+
+Ukemi::CLI.start(ARGV)

data/lib/ukemi.rb

@@ -22,10 +22,14 @@ require "ukemi/record"
 require "ukemi/services/service"
 
 require "ukemi/services/circl"
+require "ukemi/services/dnsdb"
+require "ukemi/services/otx"
 require "ukemi/services/passivetotal"
 require "ukemi/services/securitytrails"
 require "ukemi/services/virustotal"
 
 require "ukemi/moderator"
 
+require "ukemi/configuration"
+
 require "ukemi/cli"

data/lib/ukemi/cli.rb

@@ -6,16 +6,32 @@ require "thor"
 module Ukemi
   class CLI < Thor
     desc "lookup [IP|DOMAIN]", "Lookup passive DNS services"
+    method_option :order_by, type: :string, desc: "Ordering of the passve DNS resolutions (last_seen or first_seen)", default: "-last_seen"
     def lookup(data)
       data = refang(data)
+      set_ordering options["order_by"]
+
       result = Moderator.lookup(data)
       puts JSON.pretty_generate(result)
     end
 
+    default_command :lookup
+
     no_commands do
       def refang(data)
         data.gsub("[.]", ".").gsub("(.)", ".")
       end
+
+      def set_ordering(order_by)
+        parts = order_by.split("-")
+        ordering_key = parts.last
+        sort_order = parts.length == 2 ? "DESC" : "ASC"
+
+        Ukemi.configure do |config|
+          config.ordering_key = ordering_key
+          config.sort_order = sort_order
+        end
+      end
     end
   end
 end

data/lib/ukemi/configuration.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+module Ukemi
+  class Configuration
+    attr_accessor :ordering_key, :sort_order
+
+    def initialize
+      @ordering_key = "last_seen"
+      @sort_order = "DESC"
+    end
+  end
+
+  class << self
+    def configuration
+      @configuration ||= Configuration.new
+    end
+
+    attr_writer :configuration
+
+    def configure
+      yield configuration
+    end
+  end
+end

data/lib/ukemi/moderator.rb

@@ -2,6 +2,7 @@
 
 require "parallel"
 require "time"
+require "date"
 
 module Ukemi
   class Moderator
@@ -12,34 +13,63 @@ module Ukemi
 
         begin
           service.lookup data
-        rescue ::PassiveTotal::Error, ::VirusTotal::Error, ::SecurityTrails::Error, PassiveCIRCL::Error
+        rescue ::PassiveTotal::Error, ::VirusTotal::Error, ::SecurityTrails::Error, PassiveCIRCL::Error, DNSDB::Error, Faraday::Error
           nil
         end
       end.flatten.compact
 
+      format records
+    end
+
+    def format(records)
       memo = Hash.new { |h, k| h[k] = [] }
+
       records.each do |record|
         memo[record.data] << {
-          firtst_seen: record.first_seen,
+          first_seen: record.first_seen,
           last_seen: record.last_seen,
-          source: record.source,
+          source: record.source
         }
       end
+      # Merge first seen last seen and make the sources a list.
+      formatted = memo.map do |key, sources|
+        first_seens = sources.filter_map { |record| convert_to_unixtime record[:first_seen] }
+        last_seens = sources.filter_map { |record| convert_to_unixtime record[:last_seen] }
+        [
+          key,
+          {
+            first_seen: convert_to_date(first_seens.min),
+            last_seen: convert_to_date(last_seens.max),
+            sources: sources
+          }
+        ]
+      end.to_h
 
-      memo.sort_by do |_key, array|
-        last_seens = array.map do |record|
-          parse_to_unixtime record.dig(:last_seen)
+      # Sorting
+      ordering_key = Ukemi.configuration.ordering_key.to_sym
+      sort_order = Ukemi.configuration.sort_order
+      formatted.sort_by do |_key, hash|
+        value = hash[ordering_key]
+        if sort_order == "DESC"
+          value ? -convert_to_unixtime(value) : -1
+        else
+          value ? convert_to_unixtime(value) : Float::MAX.to_i
         end
-        -last_seens.max
       end.to_h
     end
 
-    def parse_to_unixtime(date)
-      return -1 unless date
+    def convert_to_unixtime(date)
+      return nil unless date
 
       Time.parse(date).to_i
     end
 
+    def convert_to_date(time)
+      return nil unless time
+
+      Time.at(time).to_date.to_s
+    end
+
     class << self
       def lookup(data)
         new.lookup data

data/lib/ukemi/record.rb

@@ -2,10 +2,7 @@
 
 module Ukemi
   class Record
-    attr_reader :data
-    attr_reader :first_seen
-    attr_reader :last_seen
-    attr_reader :source
+    attr_reader :data, :first_seen, :last_seen, :source
 
     def initialize(data:, first_seen: nil, last_seen: nil, source: nil)
       @data = data

data/lib/ukemi/services/circl.rb

@@ -8,7 +8,7 @@ module Ukemi
       private
 
       def config_keys
-        %w(CIRCL_PASSIVE_USERNAME CIRCL_PASSIVE_PASSWORD)
+        %w[CIRCL_PASSIVE_USERNAME CIRCL_PASSIVE_PASSWORD]
       end
 
       def api
@@ -26,14 +26,14 @@ module Ukemi
       def passive_dns_lookup(data, key = nil)
         results = api.dns.query(data)
         results = results.select do |result|
-          result.dig("rrtype") == "A"
+          result["rrtype"] == "A"
         end
 
         results.map do |result|
           Record.new(
-            data: result.dig(key),
-            first_seen: Time.at(result.dig("time_first")).to_date.to_s,
-            last_seen: Time.at(result.dig("time_last")).to_date.to_s,
+            data: result[key],
+            first_seen: Time.at(result["time_first"]).to_date.to_s,
+            last_seen: Time.at(result["time_last"]).to_date.to_s,
             source: name
           )
         end

data/lib/ukemi/services/dnsdb.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+require "date"
+require "dnsdb"
+
+module Ukemi
+  module Services
+    class DNSDB < Service
+      private
+
+      def config_keys
+        %w[DNSDB_API_KEY]
+      end
+
+      def api
+        @api ||= ::DNSDB::API.new
+      end
+
+      def lookup_by_ip(data)
+        results = api.lookup.rdata(type: "ip", value: data, rrtype: "A")
+        results.map do |result|
+          rrname = result["rrname"]
+          # Remove the last dot (e.g. "example.com.")
+          data = rrname[0..-2]
+          Record.new(
+            data: data,
+            first_seen: Time.at(result["time_first"]).to_date.to_s,
+            last_seen: Time.at(result["time_last"]).to_date.to_s,
+            source: name
+          )
+        end
+      end
+
+      def lookup_by_domain(data)
+        results = api.lookup.rrset(owner_name: data, rrtype: "A")
+        results.map do |result|
+          first_seen = Time.at(result["time_first"]).to_date.to_s
+          last_seen = Time.at(result["time_last"]).to_date.to_s
+
+          values = result["rdata"] || []
+          values.map do |value|
+            Record.new(
+              data: value,
+              first_seen: first_seen,
+              last_seen: last_seen,
+              source: name
+            )
+          end
+        end.flatten
+      end
+    end
+  end
+end

data/lib/ukemi/services/otx.rb

@@ -0,0 +1,72 @@
+# frozen_string_literal: true
+
+require "date"
+require "otx_ruby"
+
+module Ukemi
+  module Services
+    class OTX < Service
+      private
+
+      def config_keys
+        %w[OTX_API_KEY]
+      end
+
+      def api_key
+        @api_key ||= ENV["OTX_API_KEY"]
+      end
+
+      def domain_client
+        @domain_client ||= ::OTX::Domain.new(api_key)
+      end
+
+      def ip_client
+        @ip_client ||= ::OTX::IP.new(api_key)
+      end
+
+      def lookup_by_ip(data)
+        records = ip_client.get_passive_dns(data)
+        memo = Hash.new { |h, k| h[k] = [] }
+        records.each do |record|
+          next if record.record_type != "A"
+
+          domain = record.hostname
+          memo[domain] << Date.parse(record.last).to_s
+          memo[domain] << Date.parse(record.first).to_s
+        end
+
+        memo.keys.map do |domain|
+          Record.new(
+            data: domain,
+            first_seen: memo[domain].min,
+            last_seen: memo[domain].max,
+            source: name
+          )
+        end
+      end
+
+      def lookup_by_domain(data)
+        records = domain_client.get_passive_dns(data)
+
+        memo = Hash.new { |h, k| h[k] = [] }
+        records.each do |record|
+          next if record.record_type != "A"
+          next if record.hostname != data
+
+          ip = record.address
+          memo[ip] << Date.parse(record.last).to_s
+          memo[ip] << Date.parse(record.first).to_s
+        end
+
+        memo.keys.map do |ip|
+          Record.new(
+            data: ip,
+            first_seen: memo[ip].min,
+            last_seen: memo[ip].max,
+            source: name
+          )
+        end
+      end
+    end
+  end
+end

data/lib/ukemi/services/passivetotal.rb

@@ -12,12 +12,12 @@ module Ukemi
       end
 
       def config_keys
-        %w(PASSIVETOTAL_USERNAME PASSIVETOTAL_API_KEY)
+        %w[PASSIVETOTAL_USERNAME PASSIVETOTAL_API_KEY]
       end
 
       def lookup_by_ip(data)
         res = api.dns.passive(data)
-        results = res.dig("results") || []
+        results = res["results"] || []
         convert_to_records results
       end
 
@@ -27,9 +27,9 @@ module Ukemi
 
       def convert_to_records(results)
         results.map do |result|
-          data = result.dig("resolve")
-          first_seen = result.dig("firstSeen").to_s.split.first
-          last_seen = result.dig("lastSeen").to_s.split.first
+          data = result["resolve"]
+          first_seen = result["firstSeen"].to_s.split.first
+          last_seen = result["lastSeen"].to_s.split.first
           Record.new(
             data: data,
             first_seen: first_seen,

data/lib/ukemi/services/securitytrails.rb

@@ -9,7 +9,7 @@ module Ukemi
       private
 
       def config_keys
-        %w(SECURITYTRAILS_API_KEY)
+        %w[SECURITYTRAILS_API_KEY]
       end
 
       def api
@@ -17,9 +17,9 @@ module Ukemi
       end
 
       def lookup_by_ip(data)
-        result = api.domains.search( filter: { ipv4: data })
-        records = result.dig("records") || []
-        hostnames = records.map { |record| record.dig("hostname") }
+        result = api.domains.search(filter: { ipv4: data })
+        records = result["records"] || []
+        hostnames = records.map { |record| record["hostname"] }
         hostnames.map do |hostname|
           Record.new(
             data: hostname,
@@ -32,24 +32,25 @@ module Ukemi
 
       def lookup_by_domain(data)
         result = api.history.get_all_dns_history(data, type: "a")
-        records = result.dig("records") || []
-        records.map do |record|
-          values = record.dig("values") || []
-          values.map do |value|
-            Record.new(
-              data: value.dig("ip"),
-              first_seen: record.dig("first_seen"),
-              last_seen: record.dig("last_seen"),
-              source: name
-            )
+        records = result["records"] || []
+
+        memo = Hash.new { |h, k| h[k] = [] }
+        records.each do |record|
+          values = record["values"] || []
+          values.each do |value|
+            ip = value["ip"]
+            memo[ip] << record["first_seen"]
+            memo[ip] << record["last_seen"]
           end
-        end.flatten
-      end
+        end
 
-      def extract_attributes(response)
-        data = response.dig("data") || []
-        data.map do |item|
-          item.dig("attributes") || []
+        memo.keys.map do |ip|
+          Record.new(
+            data: ip,
+            first_seen: memo[ip].min,
+            last_seen: memo[ip].max,
+            source: name
+          )
         end
       end
     end

data/lib/ukemi/services/virustotal.rb

@@ -9,7 +9,7 @@ module Ukemi
       private
 
       def config_keys
-        %w(VIRUSTOTAL_API_KEY)
+        %w[VIRUSTOTAL_API_KEY]
       end
 
       def api
@@ -29,9 +29,9 @@ module Ukemi
       end
 
       def extract_attributes(response)
-        data = response.dig("data") || []
+        data = response["data"] || []
         data.map do |item|
-          item.dig("attributes") || []
+          item["attributes"] || []
         end
       end
 
@@ -39,8 +39,8 @@ module Ukemi
         memo = Hash.new { |h, k| h[k] = [] }
 
         attributes.each do |attribute|
-          data = attribute.dig(key)
-          date = Time.at(attribute.dig("date")).to_date.to_s
+          data = attribute[key]
+          date = Time.at(attribute["date"]).to_date.to_s
           memo[data] << date
         end