RubyGems - pdnssoc - Versions diffs - 0.1.3 - Mend

pdnssoc 0.1.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (21) hide show

checksums.yaml +7 -0
data/config/notification_email.html +80 -0
data/config/pdnssoc.conf +32 -0
data/config/td-agent.conf +173 -0
data/lib/alerts.rb +118 -0
data/lib/configalerts.rb +86 -0
data/lib/constants.rb +43 -0
data/lib/email.rb +41 -0
data/lib/inputdata.rb +46 -0
data/lib/lookingback.sh +10 -0
data/lib/misp_refresh.sh +26 -0
data/lib/pdnssoc.rb +7 -0
data/lib/post_install.rb +16 -0
data/lib/trigger.rb +124 -0
data/timers/lookingback.service +6 -0
data/timers/lookingback.timer +9 -0
data/timers/misp_refresh.service +6 -0
data/timers/misp_refresh.timer +9 -0
data/timers/pdnssoc.service +6 -0
data/timers/pdnssoc.timer +9 -0
metadata +70 -0

checksums.yaml ADDED Viewed

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 252758f53648ce58439fc34723eb137fc8cd99225720895cb42dd095ff9d56bc
+  data.tar.gz: 6a41db486378f24aa6a56765109c8c0cdd0aacf855b07a7f9616b7499bc98526
+SHA512:
+  metadata.gz: 0f81edfb60851df94c8c2541bfb71c7d264d5cda518ce1327d1af6051762984ae47eade862ad60defcb14bae1d530ce1a078ba1998a6ae28f901e41474460c29
+  data.tar.gz: ebdf588d2afad5cbec0379d6e50e11c82a29b61449b09867e77aeee30a9e739e203f87c2e80aa4e0da814f2b3f151459fdd5724d775a6d292597bf81effa0ab2

data/config/notification_email.html ADDED Viewed

@@ -0,0 +1,80 @@
+<!DOCTYPE html>
+<html lang="en">
+    <head>
+        <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
+        <style>
+            body {
+                font-family: sans-serif;
+                font-size: 10pt;
+            }
+            h1 {
+                font-size: 16pt;
+            }
+            table th {
+                padding: 10px;
+                border-top: 1px solid #fafafa;
+                border-bottom: 1px solid #e0e0e0;
+                background: #ededed;
+                background: -webkit-gradient(linear, left top, left bottom, from(#ededed), to(#ebebeb));
+                background: -moz-linear-gradient(top, #ededed, #ebebeb);
+            }
+            table {
+                border-collapse: collapse;
+                border-spacing: 0;
+            }
+            table td,
+            table th {
+                border: 1px solid #ccc;
+            }
+            table td {
+                padding: 2px 5px;
+            }
+        </style>
+    </head>
+    <body>
+        <table>
+            <tbody>
+                <tr>
+                    <th>pDNSSOC client</th>
+                    <th>First Occurrence</th>
+                    <th>IoCs detected</th>
+                    <th>MISP event</th>
+                    <th>Total # of IoCs</th>
+                    <th>Publication</th>
+                    <th>Organisation</th>
+                    <th>Comment</th>
+                    <th>Tags</th>
+                </tr>
+                <% all_results.each do |ioc, data_ioc| %>
+                    <tr>
+                        <% if data_ioc["client_name"] == "" %>
+                            <td style="text-align: left;" rowspan="<%= data_ioc["misp"].length %>"><%= data_ioc["client_ip"] %></td>
+                        <% else %>
+                            <td style="text-align: left;" rowspan="<%= data_ioc["misp"].length %>"><%= data_ioc["client_name"] %> (<%= data_ioc["client_ip"] %>)</td>
+                        <% end %>
+                        <td style="text-align: left;" rowspan="<%= data_ioc["misp"].length %>"><%= Time.at(data_ioc["first_occurrence"]).strftime(TIME_FORMAT_YMD) %></td>
+                        <td style="text-align: left;" rowspan="<%= data_ioc["misp"].length %>"><a href="" target="_new"><%= data_ioc["ioc_detected"] %></a></td>
+                        <% data_ioc["misp"].each_with_index do |misp_event, idx_event| %>
+                            <% if idx_event > 0 %>
+                                <tr>
+                            <% end %>
+                            <td style="text-align: left;"><a href="https://<%= misp_event["misp_server"] %>/events/view/<%= misp_event["misp_id"] %>" target="_new"><%= misp_event["misp_info"] %></a></td>
+                            <td style="text-align: left;"><%=  misp_event["num_iocs"] %></td>
+                            <td style="text-align: left;"><%=  misp_event["publication"] %></td>
+                            <td style="text-align: left;"><%=  misp_event["organisation"] %></td>
+                            <td style="text-align: left;"><%=  misp_event["comment"] %></td>
+                            <td style="text-align: left;">
+                                <% for tag in misp_event["tags"] do %>
+                                    <span style="background: <%= tag["colour"] %>;">
+                                        <b><span style="color: #fff; mix-blend-mode: difference; padding: 5px;"><%= tag["name"] %></span></b>
+                                    </span>
+                                <% end %>
+                            </td>
+                        </tr>
+                        <% end %>
+                    </tr>
+                <% end %>
+            </tbody>
+        </table>
+    </body>
+</html>

data/config/pdnssoc.conf ADDED Viewed

@@ -0,0 +1,32 @@
+{
+    "misp_servers": [
+        {
+            "domain":"misp1.myserver.org",
+            "api_key":"API_KEY_1",
+            "parameter_domains":"/attributes/restSearch/returnFormat:text/type:domain/",
+            "parameter_ips":"/attributes/restSearch/returnFormat:text/type:ip-src/type:ip-dst/"
+        },
+        {
+            "domain":"misp2.myotherserver.ch",
+            "api_key":"API_KEY_2",
+            "parameter_domains":"/attributes/restSearch/returnFormat:text/type:domain/from:2022-01-01/",
+            "parameter_ips":"/attributes/restSearch/returnFormat:text/type:ip-src/type:ip-dst/from:2022-01-01/"
+        }
+    ],
+    "alerts_path":"/var/log/td-agent/pdnssoc-alerts/",
+    "pdns_client" : {
+        "127.0.0.1":
+        {
+            "name":"Test host",
+            "email":"email@address.tld"
+        }
+    },
+    "email" : {
+        "from":"pdnssoc-dev@domain.tld",
+        "to":"pdnssoc-dev@domain.tld",
+        "subject":"[pDNSSOC] Community XYZ alert",
+        "server":localhost",
+        "port":25
+    }
+}

data/config/td-agent.conf ADDED Viewed

@@ -0,0 +1,173 @@
+## match tag=debug.** and dump to console
+<match debug.**>
+  @type stdout
+  @id output_stdout
+</match>
+## SYSLOG INGESTION
+# Getting DNS queries from syslog
+<source>
+  @type syslog
+  port 5140
+  protocol_type tcp
+  tag syslog
+</source>
+# Parsing DNS queries from syslog
+<match syslog>
+  @type rewrite_tag_filter
+  <rule>
+   key message
+   pattern /(?<date>.*) client \@.* (?<client>.*)#.*: query: (?<query>\S+) .*/
+   tag pdnssocdata
+  </rule>
+</match>
+## TCP INGESTION
+# The data coming in BIND logs using syslog format
+<source>
+  @type tcp
+  port 5141
+  tag pdnssocdata
+  <parse>
+        @type regexp
+        expression /(?<date>.*) client \@.* (?<client>.*)#.*: query: (?<query>\S+) .*/
+        # BIND9 variant:
+        #expression /(?<date>.*) client (?<client>.*)#.*query: (?<query>\S+) .*/
+  </parse>
+#  delimiter "\n" # optional. "\n" (newline) by default
+</source>
+## dnstap pDNS data INGESTION
+# dnstap must be configured to forward RESOLVER_RESPONSE, for example in BIND: "dnstap { resolver response; };"
+# (?<date>.*) RR (?<client>.*):\w+ <- .* \w+ (?<query>[\.|\w]+)\/.*$
+<source>
+  @type tcp
+  port 5142
+  tag pdnssocdata
+  <parse>
+        @type regexp
+        expression /(?<date>.*) RR (?<client>.*):\w+ <- .* \w+ (?<query>\w+\.\w+)\/.*/
+  </parse>
+</source>
+## HTTP source to ingest JSON log entries
+<source>
+  @type http
+  tag pdnssocdata
+  port 5143
+  bind 0.0.0.0
+  body_size_limit 32m
+  keepalive_timeout 10s
+</source>
+## OTHER PDNSSOC SERVERS
+# Data coming from lower-level pDNSSOC servers
+<source>
+  @type forward
+  port 5555
+  tag pdnssocdata
+  <parse>
+        @type regexp
+    expression /{"date":"(?<date>.*)","client":"(?<client>.*)","query":"(?<query>.*)"}/
+  </parse>
+</source>
+## DATA ROUTING
+# Copying our pdnssocdata into multiple streams
+## DATA ROUTING
+# Copying our pdnssocdata into multiple streams
+<match pdnssocdata>
+        @type copy
+        <store>
+         @type relabel
+         @label @pdnssocdata_allqueries
+        </store>
+        <store>
+         @type relabel
+         @label @pdnssocdata_correlate_query
+        </store>
+        <store>
+         @type relabel
+         @label @pdnssocdata_correlate_response
+        </store>
+        ## Send data to another pDNSSOC server
+        #<store>
+        # @type forward
+        #   send_timeout 60s
+        # recover_wait 10s
+        # hard_timeout 60s
+        # <server>
+        #   host upstream-pdnssoc.domain.edu
+        #   port 5555
+        # </server>
+        #</store>
+        #######################
+</match>
+# Logging all queries (pdnssocdata_allqueries) to disc.
+# This is used by the "looking back in the past" mechanism.
+<label @pdnssocdata_allqueries>
+<match pdnssocdata>
+ @type file
+  path /var/log/td-agent/queries
+  compress gzip
+  time_slice_format %Y%m%d-%H%M
+  <format>
+    @type json
+  </format>
+</match>
+</label>
+# Correlating the pdnssocdata_correlate data flow with malicious domains from MISP
+# Then writing to disc
+# Matching DNS queries with malicious domains
+<label @pdnssocdata_correlate_query>
+<filter pdnssocdata>
+  @type filter_list
+  filter AC
+  key_to_filter query
+  pattern_file_paths ["/etc/td-agent/misp_domains.txt"]
+  filter_empty true
+  action whitelist
+</filter>
+<match pdnssocdata>
+  @type relabel
+  @label @pdnssocdata_correlate
+</match>
+</label>
+# Matching DNS replies with malicious IPs
+<label @pdnssocdata_correlate_response>
+<filter pdnssocdata>
+  @type filter_list
+  filter AC
+  key_to_filter answer
+  pattern_file_paths ["/etc/td-agent/misp_domains.txt", "/etc/td-agent/misp_ips.txt"]
+  filter_empty true
+  action whitelist
+</filter>
+<match pdnssocdata>
+  @type relabel
+  @label @pdnssocdata_correlate
+</match>
+</label>
+<label @pdnssocdata_correlate>
+<match pdnssocdata>
+ @type file
+  path /var/log/td-agent/pdnssoc-alerts/pdnssoc-buffer
+  time_slice_format %Y%m%d-%H%M
+  <format>
+    @type json
+  </format>
+  <buffer>
+    timekey 2m
+    timekey_use_utc true
+    timekey_wait 1m
+  </buffer>
+</match>
+</label>

data/lib/alerts.rb ADDED Viewed

@@ -0,0 +1,118 @@
+require "misp"
+require 'timeout'
+require_relative 'configalerts'
+class Alert
+  include ConfigAlerts
+  include ConstantsErrors
+  include ConstantsAlerts
+  include ConstantsGeneral
+  @@faulty_misp_servers = []
+  def get_faulty_misp()
+    return @@faulty_misp_servers
+  end
+  def query_misp(misp_server, ioc_detected, type_ioc, all_uuids)
+    result_misp = []
+    list_uuids = []
+    misp_url = WEB_URL % [d: misp_server["domain"]]
+    misp_api_key = misp_server["api_key"]
+    api_endpoint = misp_url + misp_server["parameter_%{t}s" % [t:type_ioc]] # parameter_ips
+    # Setup config MISP server
+    MISP.configure do |config|
+      config.api_endpoint = api_endpoint
+      config.api_key = misp_api_key
+    end
+    # Search
+    misp_events = []
+    begin
+      Timeout::timeout(TIMEOUT_MISP_QUERY) {
+        misp_events = MISP::Event.search(value: ioc_detected)
+        # One domain can have multiple events associated
+        for misp_event in misp_events
+          # If there's a valid event and we did not processed it before proceed
+          if not all_uuids.include? misp_event.uuid
+            # Find the attribute where the malicious domain is defined
+            for attribute in misp_event.attributes
+              if attribute.type.include?(type_ioc) and attribute.value == ioc_detected and attribute.to_ids == true
+                $tags = []
+                for tag in attribute.tags do
+                  $tags.append({"colour" => tag.colour, "name" => tag.name})
+                end
+                event = {
+                  'misp_uuid' => misp_event.uuid,
+                  'misp_info' => misp_event.info,
+                  'misp_id' => misp_event.id,
+                  'misp_server' => misp_server["domain"],
+                  'num_iocs' => misp_event.attribute_count,
+                  'publication' => misp_event.date,
+                  'organisation' => misp_event.orgc.name,
+                  'comment' => attribute.comment,
+                  'tags' => $tags
+                }
+                list_uuids.append(misp_event.uuid)
+                result_misp.append(event)
+                break
+              end
+            end
+          end
+        end
+      }
+    rescue Exception => e
+      error_message = MISPQUERY % [u:misp_url]
+      @@log_sys.error(error_message + e.message)
+      @@faulty_misp_servers.append(misp_url)
+    ensure
+      return result_misp, list_uuids
+    end
+  end
+  def get_client_info(ip_client, key_reference)
+    pdns_client = ""
+    if @@pdns_config[ip_client]
+      # If the name is present on the configuration file we will use it on the email. Otherwise we will just use the ip
+      if @@pdns_config[ip_client].keys.include?(key_reference) and ! @@pdns_config[ip_client][key_reference].empty?
+        pdns_client = @@pdns_config[ip_client][key_reference]
+      end
+    else
+      @@log_sys.error(UNKNOWN_CLIENT % [c:ip_client])
+    end
+    return pdns_client
+  end
+  def parse_log(ioc_detected, type_ioc, date, ip_client)
+    result_ioc = {}
+    events = [] # MISP events detected for a particular IOC
+    events_uuid = [] # List of MISP uuid used to avoid repeat events
+    for misp_server in @@misp_config
+      if ! @@faulty_misp_servers.include?(misp_server["url"])
+        events_uuid = events_uuid.flatten
+        events_detected, uuids=query_misp(misp_server, ioc_detected, type_ioc, events_uuid)
+        if events_detected.length > 0
+          events_uuid.append(uuids)
+          events.append(events_detected)
+        end
+      end
+    end
+    events = events.flatten
+    $num_misp_events = events.length()
+    if $num_misp_events > 0
+      result_ioc= {
+        'client_ip' => ip_client,
+        'client_name' => get_client_info(ip_client, "name"),
+        'count' => 1,
+        'first_occurrence' => date,
+        'ioc_detected' => ioc_detected,
+        "misp" => events
+      }
+    end
+    return result_ioc
+  end
+end

data/lib/configalerts.rb ADDED Viewed

@@ -0,0 +1,86 @@
+require "json"
+require 'logger'
+require "time"
+puts $LOAD_PATH
+require 'parseconfig'
+require_relative 'constants'
+module ConfigAlerts
+  include ConstantsConfig
+  include ConstantsErrors
+  def initialize()
+    # Setup Logging
+    @@log_alerts = Logger.new(PATH_LOG + FILENAME_LOG_ALERT, 'daily')
+    @@log_alerts.formatter = proc do |severity, datetime, progname, msg| {message: msg}.to_json + $/ end
+    @@log_sys = Logger.new(PATH_LOG + FILENAME_LOG_SYS, 'daily')
+    @@log_sys.formatter = proc do |severity, datetime, progname, msg| "#{datetime}, #{severity}: #{msg} #{progname} \n" end
+    # Open config files
+    @@misp_config, @@alerts_config, @@email_config, @@pdns_config = init_config()
+    # Get the list of bad domains
+    @@bad_domains = File.read(PATH_MISP_D)
+    @@bad_ips = File.read(PATH_MISP_IP)
+    # Get HTML template for the email
+    @@html_email = init_html()
+  end
+  def init_html()
+    html_data = ""
+    # Template HTML of the email
+    f = File.open(PATH_HTML, "r")
+    f.each_line do |line| html_data += line end
+    raise TypeError, "html_data expected an String, got #{html_data.class.name}" unless html_data.kind_of?(String)
+    return html_data
+  end
+  def init_config()
+    config_data = JSON.parse(File.read(PATH_PDNS_CONF))
+    # Initialize vaiables
+    misp_config = config_data["misp_servers"]
+    alerts_config = config_data["alerts_path"]
+    email_config = config_data["email"]
+    pdns_config = config_data["pdns_client"]
+    # Check if they all have the expected format
+    bool_conf = (misp_config.kind_of?(Array) and alerts_config.kind_of?(String) and email_config.kind_of?(Hash) and pdns_config.kind_of?(Hash))
+    # Check if all the required info is present
+    if bool_conf
+      misp_subconf = params_to_check(misp_config, ['domain', 'api_key', 'parameter'])
+      email_subconf = params_to_check(email_config, ['from', 'to', 'subject', 'server', 'port'])
+    end
+    # If some field is missing or empty the code breaks
+    if ! (bool_conf and misp_subconf and email_subconf)
+      @@log_sys.error(CONFIGFILE) #+ "Backtrace: " + e.backtrace.join(" / "))
+      raise (error_message)
+    end
+    return misp_config, alerts_config, email_config, pdns_config
+  end
+  def params_to_check(configuration, params)
+    # Iterate over all the params to make sure they are on the config file
+    for param in params
+      # If it is an array you want to check the inner maps
+      if configuration.kind_of?(Array)
+        for map in configuration
+          if ! param_in_map(param,map)
+            return false
+          end
+        end
+      else
+        if ! param_in_map(param,configuration)
+          return false
+        end
+      end
+    return true
+    end
+  end
+  def param_in_map(param,map)
+    # If the param is not on the map or if it is empy -> False
+    bool = map.keys.include?(param) and ! map[param].empty?
+    return bool
+  end
+end

data/lib/constants.rb ADDED Viewed

@@ -0,0 +1,43 @@
+module ConstantsConfig
+    file_path = File.expand_path(__FILE__)
+    lib_path = File.dirname(file_path)
+    common_path = File.dirname(lib_path)
+    # If the env variables are not defined, use the default values
+    PATH_LOG = ENV['PATH_LOG'] || "/var/log/td-agent/"
+    PATH_ALERTS = PATH_LOG + 'pdnssoc-alerts/'
+    PATH_TDAGENT = "/etc/td-agent/"
+    PATH_PDNS_CONF = ENV['PATH_PDNS_CONF'] || "/etc/pdnssoc/pdnssoc.conf"
+    PATH_MISP_D = ENV['PATH_MISP_D'] || File.join(PATH_TDAGENT, "misp_domains.txt")
+    PATH_MISP_IP = ENV['PATH_MISP_D'] || File.join(PATH_TDAGENT, "misp_ips.txt")
+    PATH_HTML = ENV['PATH_HTML'] || "/etc/pdnssoc/notification_email.html"
+    FILENAME_LOG_ALERT = ENV['FILENAME_LOG_ALERT'] || "alerts.log"
+    FILENAME_LOG_SYS = ENV['FILENAME_LOG_SYS'] || "pdnssoc_sys.log"
+end
+module ConstantsGeneral
+    WEB_URL="https://%{d}"
+    TIME_FORMAT_YMD = "%Y-%m-%dT%H:%M:%S.%L%z"
+end
+module ConstantsErrors
+    CONFIGFILE = "ConfigFileError. Some parameters of your config file are either missing or have a wrong format. "
+    MISPQUERY = "MISP query failed using %{u} and therefore will be skipped. The error message is -> "
+    UNKNOWN_CLIENT="An unknown client %{c} has been detected. Add it on the configuration file to receive alerts. "
+    TRIGGER_ERROR = "DNS/pDNS queries cannot be read. %{e}"
+    SMTP_ERROR = "The email could not be sent. Check the SMTP configuration."
+    MISSING_KEY_ALERT =	"One of the keys is missing for: %s. This log entry will be skipped"
+end
+module ConstantsAlerts
+    TIMEOUT_MISP_QUERY = ENV['TIMEOUT_MISP_QUERY'] || 20
+end
+module ConstantsData
+    RGX_FILE_TIME = "/\d{8}-\d{4}/"
+    RGX_FILE_REF = 'pdnssoc-buffer.*.log'
+    GROUP_SIZE = 5 * 1024 * 1024
+end

data/lib/email.rb ADDED Viewed

@@ -0,0 +1,41 @@
+require 'net/smtp'
+require_relative 'constants'
+require 'erb'
+require 'time'
+class Email
+  include ConfigAlerts
+  include ConstantsErrors
+  include ConstantsGeneral
+  def send_email(email_to, all_results)
+    begin
+      f = File.open(PATH_HTML, "r")
+      template_html = ERB.new(f.read)
+      html_email = template_html.result(binding)
+      # Compose the message to send
+      message = <<~MESSAGE_END
+      From: #{@@email_config["from"]}
+      To: #{email_to}
+      MIME-Version: 1.0
+      Content-type: text/html
+      Subject: #{@@email_config["subject"]}
+      #{html_email}
+      MESSAGE_END
+      # Send the email
+      Net::SMTP.start(@@email_config["server"], @@email_config["port"]) do |smtp|
+        smtp.send_message message, @@email_config["from"], email_to end
+    rescue Exception => e
+        @@log_sys.error(SMTP_ERROR + e.message) #+ "Backtrace: " + e.backtrace.join(" / "))
+        raise Exception,  SMTP_ERROR + e.message
+    end
+  end
+end

data/lib/inputdata.rb ADDED Viewed

@@ -0,0 +1,46 @@
+require_relative 'constants'
+require 'fileutils'
+module InputData
+  include ConstantsData
+  include ConstantsConfig
+  def get_groups()
+    # Get a list of all files in the directory
+    files = Dir.glob(File.join(PATH_ALERTS, RGX_FILE_REF))
+    # Sort files by date extracted from the filename, newest first
+    sorted_files = files.sort_by do |file|
+      match = File.basename(file).match(RGX_FILE_TIME)
+      match ? match[0] : ''
+    end.reverse
+    # Initialize groups as an empty array
+    groups = []
+    current_group = [] # Initialize current group as an empty array
+    current_group_size = 0
+    # Iterate through the sorted files
+    sorted_files.each do |file|
+      # Get the file size in bytes
+      file_size = File.size(file)
+      # Check if adding the file exceeds the group size limit
+      if current_group_size + file_size > GROUP_SIZE  # Convert 500MB to bytes
+        # Add the current group to the groups array
+        groups << current_group
+        # Create a new current group and reset current group size
+        current_group = []
+        current_group_size = 0
+      end
+      # Add the file to the current group
+      current_group << file
+      current_group_size += file_size
+    end
+    groups << current_group
+    return groups
+  end
+end

data/lib/lookingback.sh ADDED Viewed

@@ -0,0 +1,10 @@
+#!/bin/sh
+temp_file=$(mktemp)
+alert_path=`grep alerts_path /etc/pdnssoc/pdnssoc.conf | awk -F "\"" '{print $4}'`
+zgrep  -F -f /etc/td-agent/misp_domains.txt /var/log/td-agent/queries.*.log* > ${temp_file}
+grep  -w -f /etc/td-agent/misp_domains.txt ${temp_file} >> ${alert_path}pdnssoc-pastlog.log
+rm -f ${temp_file}

data/lib/misp_refresh.sh ADDED Viewed

@@ -0,0 +1,26 @@
+#!/bin/sh
+#
+# CRON file for pDNSSOC
+# Downloading MISP events
+pull_misp_data(){
+jq  --raw-output '.misp_servers[] | "\(.api_key)|\(.domain)|\(.'$1')"' /etc/pdnssoc/pdnssoc.conf |
+while IFS="|" read -r api_key domain parameter; do
+         printf -v mycurl 'curl -k -qsS --header "Authorization: %s" "%s%s"\n' "$api_key" "https://$domain/" "$parameter"
+         raw_data=$( eval ${mycurl})
+         sorted_data=$(printf "$raw_data\n"|sort -u)
+         echo "$sorted_data\n" > /etc/td-agent/$2
+    done
+}
+pull_misp_data "parameter_domains" "misp_domains.txt"
+pull_misp_data "parameter_ips" "misp_ips.txt"
+# Add something below to send an alert to the security contact if misp_domains.txt is empty
+# Reload the configuration file for fluentd
+kill -1 `cat /var/run/td-agent/td-agent.pid`

data/lib/pdnssoc.rb ADDED Viewed

@@ -0,0 +1,7 @@
+require_relative 'trigger'
+$stdout.sync = true
+trigger = Trigger.new()
+trigger.run()

data/lib/post_install.rb ADDED Viewed

@@ -0,0 +1,16 @@
+puts "PostInstallScript loaded successfully."
+class PostInstallScript
+  def self.run
+    puts "PostInstallScript.run method executed."
+    # Your post-installation script logic here
+    # Will read and execute the tasks defined in tasks/tasks_install.rake
+    if ENV['SKIP_POST_INSTALL_HOOK'].nil?
+      puts "Running post-installation setup..."
+      Rake::Task["rake_install:install"].invoke
+      puts "Post-installation setup completed."
+    end
+  end
+end
+load File.join(File.dirname(__FILE__), 'tasks', 'tasks_install.rake')

data/lib/trigger.rb ADDED Viewed

@@ -0,0 +1,124 @@
+require_relative 'configalerts'
+require_relative 'email'
+require_relative 'alerts'
+require_relative 'constants'
+require_relative 'inputdata'
+require "time"
+class Trigger
+  include ConfigAlerts
+  include InputData
+  def initialize()
+    super()
+    @@alerts_found = {}
+  end
+  def delete_logs(processed_logs)
+    for filename in processed_logs
+      @@log_sys.debug("Deleting: " + filename)
+      File.delete(filename) if File.exist?(filename)
+    end
+  end
+  def get_email_client(ip_client)
+    # TODO: Given a boolean, skip any event from clients that are not on the config file
+    # If we have email for that client we will contact them directly
+    if @@pdns_config.keys.include?(ip_client) and @@pdns_config[ip_client].keys.include?("email") and ! @@pdns_config[ip_client]["email"].empty?
+      email_client = @@pdns_config[ip_client]["email"]
+    else
+      # If we don't have the email of the clientb we send the email to the default security contact
+      email_client = @@email_config["to"]
+    end
+    return email_client
+  end
+  def check_alert_keys(json_log_read)
+    required_keys = ["query", "client", "date"]
+    all_required_keys = required_keys.all? { |string| json_log_read.key?(string) }
+    return all_required_keys
+  end
+  def study_ioc(list_iocs, ioc_detected, type_ioc, ip_client, date)
+    skip_iocs = []
+    # Domains that are legit or that do not have information in MISP will be skipped
+    begin
+      if ! skip_iocs.include?(ioc_detected)
+        # If it is a malicious domain
+        if list_iocs.include?(ioc_detected)
+          email_client = get_email_client(ip_client)
+          # If it was already analyzed -> +1
+          if ! @@alerts_found.empty? and @@alerts_found.include?(email_client) and \
+              @@alerts_found[email_client].include?(ioc_detected)
+            @@alerts_found[email_client][ioc_detected]["count"] += 1
+          else
+            # We don't have information so we will query MISP
+            alert = Alert.new()
+            @result_ioc = alert.parse_log(ioc_detected, type_ioc, date, ip_client)
+          end
+          if @result_ioc.empty?
+            # Although it is a malicious domain it doesn't have any data in MISP -> skip next time
+            skip_iocs.append(ioc_detected)
+          else
+            # We have found data in MISP about this domain -> we will report it to the right client
+            if ! @@alerts_found.include?(email_client)
+              @@alerts_found[email_client] = {}
+            end
+            @@alerts_found[email_client][ioc_detected] = @result_ioc
+            @@log_alerts.info(@result_ioc)
+          end
+        else
+          skip_iocs.append(ioc_detected)
+        end
+      end
+    rescue Exception => e
+      raise Exception, TRIGGER_ERROR % [e:e]
+    end
+  end
+  def analyze_all_iocs(group_of_files)
+    all_alerts = {}
+    # Iterate over all files inside the group
+    for filename in group_of_files
+      # Read each line opf the file that represents one log entry
+      File.readlines(filename).each do |line|
+        json_log = JSON.parse(line)
+          # If the data is not complete -> skip log
+          if ! check_alert_keys(json_log)
+            @@log_sys.error(MISSING_KEY_ALERT % line)
+            next
+          end
+          ip_client = json_log["client"]
+          date = Time.parse(json_log["date"]).to_i
+          # If in addition of the domain, the resolved IP is provided, it will be analyzed
+          if json_log.keys.include?("answer")
+            study_ioc(@@bad_ips, json_log["answer"], "ip", ip_client, date)
+          end
+          # Analyze if the domain is malicious
+          study_ioc(@@bad_domains, json_log["query"], "domain", ip_client, date)
+      end
+    end
+  end
+  def run()
+    # The files to process are gouped so in case of failure at least some of them will be processed
+    groups = get_groups()
+    for group_of_files in groups
+      # Analyze all IOCs from the logs
+      analyze_all_iocs(group_of_files)
+      if @@alerts_found.empty?
+        @@log_sys.debug("No alerts found!")
+      else
+        # We will send an alert to each client (with an email on the config file)
+        @@alerts_found.each do |email_client, client_data|
+          email = Email.new()
+          email.send_email(email_client, client_data)
+        end
+      end
+      # If the send_email is not successful the logs will not be deleted
+      delete_logs(group_of_files)
+    end
+  end
+end

data/timers/lookingback.service ADDED Viewed

@@ -0,0 +1,6 @@
+[Unit]
+Description=Run lookingback.sh every day at 12:00
+[Service]
+ExecStart=/bin/bash /usr/local/bin/pdnssoc/lookingback.sh
+User=root

data/timers/lookingback.timer ADDED Viewed

@@ -0,0 +1,9 @@
+[Unit]
+Description=Run lookingback.sh every day at 12:00
+[Timer]
+OnCalendar=*-*-* 12:00:00
+Persistent=true
+[Install]
+WantedBy=timers.target

data/timers/misp_refresh.service ADDED Viewed

@@ -0,0 +1,6 @@
+[Unit]
+Description=Run misp_refresh.sh every 15 minutes
+[Service]
+ExecStart=/bin/bash /usr/local/bin/pdnssoc/misp_refresh.sh
+User=root

data/timers/misp_refresh.timer ADDED Viewed

@@ -0,0 +1,9 @@
+[Unit]
+Description=Run misp_refresh.sh every hour
+[Timer]
+OnCalendar=hourly
+Persistent=true
+[Install]
+WantedBy=timers.target

data/timers/pdnssoc.service ADDED Viewed

@@ -0,0 +1,6 @@
+[Unit]
+Description=Run pdnssoc.rb every 15 minutes
+[Service]
+ExecStart=/opt/td-agent/bin/ruby /usr/local/bin/pdnssoc/pdnssoc.rb
+User=root

data/timers/pdnssoc.timer ADDED Viewed

@@ -0,0 +1,9 @@
+[Unit]
+Description=Run pdnssoc.rb every 15 minutes
+[Timer]
+OnCalendar=*:0/15
+Persistent=true
+[Install]
+WantedBy=timers.target

metadata ADDED Viewed

@@ -0,0 +1,70 @@
+--- !ruby/object:Gem::Specification
+name: pdnssoc
+version: !ruby/object:Gem::Version
+  version: 0.1.3
+platform: ruby
+authors:
+- Pau Cutrina
+- Romain Wartel
+- Christos Arvanitis
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2023-08-07 00:00:00.000000000 Z
+dependencies: []
+description: pDNS correlation with MISP
+email:
+- admin@safer-trust.org
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- config/notification_email.html
+- config/pdnssoc.conf
+- config/td-agent.conf
+- lib/alerts.rb
+- lib/configalerts.rb
+- lib/constants.rb
+- lib/email.rb
+- lib/inputdata.rb
+- lib/lookingback.sh
+- lib/misp_refresh.sh
+- lib/pdnssoc.rb
+- lib/post_install.rb
+- lib/trigger.rb
+- timers/lookingback.service
+- timers/lookingback.timer
+- timers/misp_refresh.service
+- timers/misp_refresh.timer
+- timers/pdnssoc.service
+- timers/pdnssoc.timer
+homepage: https://github.com/CERN-CERT/pDNSSOC/
+licenses:
+- MIT
+metadata:
+  rubygems_mfa_required: 'true'
+  source_code_uri: https://github.com/CERN-CERT/pDNSSOC
+  changelog_uri: https://github.com/CERN-CERT/pDNSSOC/blob/master/CHANGELOG.md
+  homepage_uri: https://github.com/CERN-CERT/pDNSSOC
+  github_repo: ssh://github.com/CERN-CERT/pDNSSOC
+post_install_message: pDNSSOC has been installed successfuly!
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 2.5.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements:
+- Ruby (>= 2.5.0)
+rubygems_version: 3.2.33
+signing_key:
+specification_version: 4
+summary: pDNS correlation with MISP
+test_files: []