pdnssoc 0.1.3

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml ADDED
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA256:
3
+ metadata.gz: 252758f53648ce58439fc34723eb137fc8cd99225720895cb42dd095ff9d56bc
4
+ data.tar.gz: 6a41db486378f24aa6a56765109c8c0cdd0aacf855b07a7f9616b7499bc98526
5
+ SHA512:
6
+ metadata.gz: 0f81edfb60851df94c8c2541bfb71c7d264d5cda518ce1327d1af6051762984ae47eade862ad60defcb14bae1d530ce1a078ba1998a6ae28f901e41474460c29
7
+ data.tar.gz: ebdf588d2afad5cbec0379d6e50e11c82a29b61449b09867e77aeee30a9e739e203f87c2e80aa4e0da814f2b3f151459fdd5724d775a6d292597bf81effa0ab2
@@ -0,0 +1,80 @@
1
+ <!DOCTYPE html>
2
+ <html lang="en">
3
+ <head>
4
+ <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
5
+ <style>
6
+ body {
7
+ font-family: sans-serif;
8
+ font-size: 10pt;
9
+ }
10
+ h1 {
11
+ font-size: 16pt;
12
+ }
13
+ table th {
14
+ padding: 10px;
15
+ border-top: 1px solid #fafafa;
16
+ border-bottom: 1px solid #e0e0e0;
17
+ background: #ededed;
18
+ background: -webkit-gradient(linear, left top, left bottom, from(#ededed), to(#ebebeb));
19
+ background: -moz-linear-gradient(top, #ededed, #ebebeb);
20
+ }
21
+ table {
22
+ border-collapse: collapse;
23
+ border-spacing: 0;
24
+ }
25
+ table td,
26
+ table th {
27
+ border: 1px solid #ccc;
28
+ }
29
+ table td {
30
+ padding: 2px 5px;
31
+ }
32
+ </style>
33
+ </head>
34
+ <body>
35
+ <table>
36
+ <tbody>
37
+ <tr>
38
+ <th>pDNSSOC client</th>
39
+ <th>First Occurrence</th>
40
+ <th>IoCs detected</th>
41
+ <th>MISP event</th>
42
+ <th>Total # of IoCs</th>
43
+ <th>Publication</th>
44
+ <th>Organisation</th>
45
+ <th>Comment</th>
46
+ <th>Tags</th>
47
+ </tr>
48
+ <% all_results.each do |ioc, data_ioc| %>
49
+ <tr>
50
+ <% if data_ioc["client_name"] == "" %>
51
+ <td style="text-align: left;" rowspan="<%= data_ioc["misp"].length %>"><%= data_ioc["client_ip"] %></td>
52
+ <% else %>
53
+ <td style="text-align: left;" rowspan="<%= data_ioc["misp"].length %>"><%= data_ioc["client_name"] %> (<%= data_ioc["client_ip"] %>)</td>
54
+ <% end %>
55
+ <td style="text-align: left;" rowspan="<%= data_ioc["misp"].length %>"><%= Time.at(data_ioc["first_occurrence"]).strftime(TIME_FORMAT_YMD) %></td>
56
+ <td style="text-align: left;" rowspan="<%= data_ioc["misp"].length %>"><a href="" target="_new"><%= data_ioc["ioc_detected"] %></a></td>
57
+ <% data_ioc["misp"].each_with_index do |misp_event, idx_event| %>
58
+ <% if idx_event > 0 %>
59
+ <tr>
60
+ <% end %>
61
+ <td style="text-align: left;"><a href="https://<%= misp_event["misp_server"] %>/events/view/<%= misp_event["misp_id"] %>" target="_new"><%= misp_event["misp_info"] %></a></td>
62
+ <td style="text-align: left;"><%= misp_event["num_iocs"] %></td>
63
+ <td style="text-align: left;"><%= misp_event["publication"] %></td>
64
+ <td style="text-align: left;"><%= misp_event["organisation"] %></td>
65
+ <td style="text-align: left;"><%= misp_event["comment"] %></td>
66
+ <td style="text-align: left;">
67
+ <% for tag in misp_event["tags"] do %>
68
+ <span style="background: <%= tag["colour"] %>;">
69
+ <b><span style="color: #fff; mix-blend-mode: difference; padding: 5px;"><%= tag["name"] %></span></b>
70
+ </span>
71
+ <% end %>
72
+ </td>
73
+ </tr>
74
+ <% end %>
75
+ </tr>
76
+ <% end %>
77
+ </tbody>
78
+ </table>
79
+ </body>
80
+ </html>
@@ -0,0 +1,32 @@
1
+ {
2
+ "misp_servers": [
3
+ {
4
+ "domain":"misp1.myserver.org",
5
+ "api_key":"API_KEY_1",
6
+ "parameter_domains":"/attributes/restSearch/returnFormat:text/type:domain/",
7
+ "parameter_ips":"/attributes/restSearch/returnFormat:text/type:ip-src/type:ip-dst/"
8
+ },
9
+ {
10
+ "domain":"misp2.myotherserver.ch",
11
+ "api_key":"API_KEY_2",
12
+ "parameter_domains":"/attributes/restSearch/returnFormat:text/type:domain/from:2022-01-01/",
13
+ "parameter_ips":"/attributes/restSearch/returnFormat:text/type:ip-src/type:ip-dst/from:2022-01-01/"
14
+ }
15
+ ],
16
+ "alerts_path":"/var/log/td-agent/pdnssoc-alerts/",
17
+ "pdns_client" : {
18
+ "127.0.0.1":
19
+ {
20
+ "name":"Test host",
21
+ "email":"email@address.tld"
22
+ }
23
+ },
24
+ "email" : {
25
+ "from":"pdnssoc-dev@domain.tld",
26
+ "to":"pdnssoc-dev@domain.tld",
27
+ "subject":"[pDNSSOC] Community XYZ alert",
28
+ "server":localhost",
29
+ "port":25
30
+ }
31
+
32
+ }
@@ -0,0 +1,173 @@
1
+ ## match tag=debug.** and dump to console
2
+ <match debug.**>
3
+ @type stdout
4
+ @id output_stdout
5
+ </match>
6
+
7
+ ## SYSLOG INGESTION
8
+ # Getting DNS queries from syslog
9
+ <source>
10
+ @type syslog
11
+ port 5140
12
+ protocol_type tcp
13
+ tag syslog
14
+ </source>
15
+ # Parsing DNS queries from syslog
16
+ <match syslog>
17
+ @type rewrite_tag_filter
18
+ <rule>
19
+ key message
20
+ pattern /(?<date>.*) client \@.* (?<client>.*)#.*: query: (?<query>\S+) .*/
21
+ tag pdnssocdata
22
+ </rule>
23
+ </match>
24
+
25
+ ## TCP INGESTION
26
+ # The data coming in BIND logs using syslog format
27
+ <source>
28
+ @type tcp
29
+ port 5141
30
+ tag pdnssocdata
31
+ <parse>
32
+ @type regexp
33
+ expression /(?<date>.*) client \@.* (?<client>.*)#.*: query: (?<query>\S+) .*/
34
+ # BIND9 variant:
35
+ #expression /(?<date>.*) client (?<client>.*)#.*query: (?<query>\S+) .*/
36
+ </parse>
37
+ # delimiter "\n" # optional. "\n" (newline) by default
38
+ </source>
39
+
40
+ ## dnstap pDNS data INGESTION
41
+ # dnstap must be configured to forward RESOLVER_RESPONSE, for example in BIND: "dnstap { resolver response; };"
42
+ # (?<date>.*) RR (?<client>.*):\w+ <- .* \w+ (?<query>[\.|\w]+)\/.*$
43
+ <source>
44
+ @type tcp
45
+ port 5142
46
+ tag pdnssocdata
47
+ <parse>
48
+ @type regexp
49
+ expression /(?<date>.*) RR (?<client>.*):\w+ <- .* \w+ (?<query>\w+\.\w+)\/.*/
50
+ </parse>
51
+ </source>
52
+
53
+ ## HTTP source to ingest JSON log entries
54
+ <source>
55
+ @type http
56
+ tag pdnssocdata
57
+ port 5143
58
+ bind 0.0.0.0
59
+ body_size_limit 32m
60
+ keepalive_timeout 10s
61
+ </source>
62
+
63
+ ## OTHER PDNSSOC SERVERS
64
+ # Data coming from lower-level pDNSSOC servers
65
+ <source>
66
+ @type forward
67
+ port 5555
68
+ tag pdnssocdata
69
+ <parse>
70
+ @type regexp
71
+ expression /{"date":"(?<date>.*)","client":"(?<client>.*)","query":"(?<query>.*)"}/
72
+ </parse>
73
+ </source>
74
+
75
+
76
+ ## DATA ROUTING
77
+ # Copying our pdnssocdata into multiple streams
78
+ ## DATA ROUTING
79
+ # Copying our pdnssocdata into multiple streams
80
+ <match pdnssocdata>
81
+ @type copy
82
+ <store>
83
+ @type relabel
84
+ @label @pdnssocdata_allqueries
85
+ </store>
86
+ <store>
87
+ @type relabel
88
+ @label @pdnssocdata_correlate_query
89
+ </store>
90
+ <store>
91
+ @type relabel
92
+ @label @pdnssocdata_correlate_response
93
+ </store>
94
+
95
+ ## Send data to another pDNSSOC server
96
+ #<store>
97
+ # @type forward
98
+ # send_timeout 60s
99
+ # recover_wait 10s
100
+ # hard_timeout 60s
101
+ # <server>
102
+ # host upstream-pdnssoc.domain.edu
103
+ # port 5555
104
+ # </server>
105
+ #</store>
106
+ #######################
107
+
108
+ </match>
109
+
110
+ # Logging all queries (pdnssocdata_allqueries) to disc.
111
+ # This is used by the "looking back in the past" mechanism.
112
+ <label @pdnssocdata_allqueries>
113
+ <match pdnssocdata>
114
+ @type file
115
+ path /var/log/td-agent/queries
116
+ compress gzip
117
+ time_slice_format %Y%m%d-%H%M
118
+ <format>
119
+ @type json
120
+ </format>
121
+ </match>
122
+ </label>
123
+
124
+
125
+ # Correlating the pdnssocdata_correlate data flow with malicious domains from MISP
126
+ # Then writing to disc
127
+ # Matching DNS queries with malicious domains
128
+ <label @pdnssocdata_correlate_query>
129
+ <filter pdnssocdata>
130
+ @type filter_list
131
+ filter AC
132
+ key_to_filter query
133
+ pattern_file_paths ["/etc/td-agent/misp_domains.txt"]
134
+ filter_empty true
135
+ action whitelist
136
+ </filter>
137
+ <match pdnssocdata>
138
+ @type relabel
139
+ @label @pdnssocdata_correlate
140
+ </match>
141
+ </label>
142
+
143
+ # Matching DNS replies with malicious IPs
144
+ <label @pdnssocdata_correlate_response>
145
+ <filter pdnssocdata>
146
+ @type filter_list
147
+ filter AC
148
+ key_to_filter answer
149
+ pattern_file_paths ["/etc/td-agent/misp_domains.txt", "/etc/td-agent/misp_ips.txt"]
150
+ filter_empty true
151
+ action whitelist
152
+ </filter>
153
+ <match pdnssocdata>
154
+ @type relabel
155
+ @label @pdnssocdata_correlate
156
+ </match>
157
+ </label>
158
+
159
+ <label @pdnssocdata_correlate>
160
+ <match pdnssocdata>
161
+ @type file
162
+ path /var/log/td-agent/pdnssoc-alerts/pdnssoc-buffer
163
+ time_slice_format %Y%m%d-%H%M
164
+ <format>
165
+ @type json
166
+ </format>
167
+ <buffer>
168
+ timekey 2m
169
+ timekey_use_utc true
170
+ timekey_wait 1m
171
+ </buffer>
172
+ </match>
173
+ </label>
data/lib/alerts.rb ADDED
@@ -0,0 +1,118 @@
1
+ require "misp"
2
+ require 'timeout'
3
+ require_relative 'configalerts'
4
+
5
+ class Alert
6
+ include ConfigAlerts
7
+ include ConstantsErrors
8
+ include ConstantsAlerts
9
+ include ConstantsGeneral
10
+
11
+ @@faulty_misp_servers = []
12
+
13
+ def get_faulty_misp()
14
+ return @@faulty_misp_servers
15
+ end
16
+
17
+ def query_misp(misp_server, ioc_detected, type_ioc, all_uuids)
18
+ result_misp = []
19
+ list_uuids = []
20
+ misp_url = WEB_URL % [d: misp_server["domain"]]
21
+ misp_api_key = misp_server["api_key"]
22
+ api_endpoint = misp_url + misp_server["parameter_%{t}s" % [t:type_ioc]] # parameter_ips
23
+ # Setup config MISP server
24
+ MISP.configure do |config|
25
+ config.api_endpoint = api_endpoint
26
+ config.api_key = misp_api_key
27
+ end
28
+
29
+ # Search
30
+ misp_events = []
31
+ begin
32
+ Timeout::timeout(TIMEOUT_MISP_QUERY) {
33
+ misp_events = MISP::Event.search(value: ioc_detected)
34
+ # One domain can have multiple events associated
35
+ for misp_event in misp_events
36
+ # If there's a valid event and we did not processed it before proceed
37
+ if not all_uuids.include? misp_event.uuid
38
+ # Find the attribute where the malicious domain is defined
39
+ for attribute in misp_event.attributes
40
+ if attribute.type.include?(type_ioc) and attribute.value == ioc_detected and attribute.to_ids == true
41
+ $tags = []
42
+ for tag in attribute.tags do
43
+ $tags.append({"colour" => tag.colour, "name" => tag.name})
44
+ end
45
+ event = {
46
+ 'misp_uuid' => misp_event.uuid,
47
+ 'misp_info' => misp_event.info,
48
+ 'misp_id' => misp_event.id,
49
+ 'misp_server' => misp_server["domain"],
50
+ 'num_iocs' => misp_event.attribute_count,
51
+ 'publication' => misp_event.date,
52
+ 'organisation' => misp_event.orgc.name,
53
+ 'comment' => attribute.comment,
54
+ 'tags' => $tags
55
+ }
56
+ list_uuids.append(misp_event.uuid)
57
+ result_misp.append(event)
58
+ break
59
+ end
60
+ end
61
+ end
62
+ end
63
+ }
64
+ rescue Exception => e
65
+ error_message = MISPQUERY % [u:misp_url]
66
+ @@log_sys.error(error_message + e.message)
67
+ @@faulty_misp_servers.append(misp_url)
68
+ ensure
69
+ return result_misp, list_uuids
70
+ end
71
+ end
72
+
73
+ def get_client_info(ip_client, key_reference)
74
+ pdns_client = ""
75
+ if @@pdns_config[ip_client]
76
+ # If the name is present on the configuration file we will use it on the email. Otherwise we will just use the ip
77
+ if @@pdns_config[ip_client].keys.include?(key_reference) and ! @@pdns_config[ip_client][key_reference].empty?
78
+ pdns_client = @@pdns_config[ip_client][key_reference]
79
+ end
80
+ else
81
+ @@log_sys.error(UNKNOWN_CLIENT % [c:ip_client])
82
+ end
83
+ return pdns_client
84
+ end
85
+
86
+ def parse_log(ioc_detected, type_ioc, date, ip_client)
87
+ result_ioc = {}
88
+ events = [] # MISP events detected for a particular IOC
89
+ events_uuid = [] # List of MISP uuid used to avoid repeat events
90
+
91
+ for misp_server in @@misp_config
92
+ if ! @@faulty_misp_servers.include?(misp_server["url"])
93
+ events_uuid = events_uuid.flatten
94
+ events_detected, uuids=query_misp(misp_server, ioc_detected, type_ioc, events_uuid)
95
+ if events_detected.length > 0
96
+ events_uuid.append(uuids)
97
+ events.append(events_detected)
98
+ end
99
+ end
100
+ end
101
+ events = events.flatten
102
+ $num_misp_events = events.length()
103
+
104
+ if $num_misp_events > 0
105
+ result_ioc= {
106
+ 'client_ip' => ip_client,
107
+ 'client_name' => get_client_info(ip_client, "name"),
108
+ 'count' => 1,
109
+ 'first_occurrence' => date,
110
+ 'ioc_detected' => ioc_detected,
111
+ "misp" => events
112
+ }
113
+ end
114
+ return result_ioc
115
+ end
116
+ end
117
+
118
+
@@ -0,0 +1,86 @@
1
+ require "json"
2
+ require 'logger'
3
+ require "time"
4
+ puts $LOAD_PATH
5
+ require 'parseconfig'
6
+ require_relative 'constants'
7
+
8
+
9
+ module ConfigAlerts
10
+ include ConstantsConfig
11
+ include ConstantsErrors
12
+
13
+ def initialize()
14
+ # Setup Logging
15
+ @@log_alerts = Logger.new(PATH_LOG + FILENAME_LOG_ALERT, 'daily')
16
+ @@log_alerts.formatter = proc do |severity, datetime, progname, msg| {message: msg}.to_json + $/ end
17
+ @@log_sys = Logger.new(PATH_LOG + FILENAME_LOG_SYS, 'daily')
18
+ @@log_sys.formatter = proc do |severity, datetime, progname, msg| "#{datetime}, #{severity}: #{msg} #{progname} \n" end
19
+ # Open config files
20
+ @@misp_config, @@alerts_config, @@email_config, @@pdns_config = init_config()
21
+ # Get the list of bad domains
22
+ @@bad_domains = File.read(PATH_MISP_D)
23
+ @@bad_ips = File.read(PATH_MISP_IP)
24
+ # Get HTML template for the email
25
+ @@html_email = init_html()
26
+ end
27
+
28
+ def init_html()
29
+ html_data = ""
30
+ # Template HTML of the email
31
+ f = File.open(PATH_HTML, "r")
32
+ f.each_line do |line| html_data += line end
33
+ raise TypeError, "html_data expected an String, got #{html_data.class.name}" unless html_data.kind_of?(String)
34
+ return html_data
35
+ end
36
+
37
+ def init_config()
38
+ config_data = JSON.parse(File.read(PATH_PDNS_CONF))
39
+ # Initialize vaiables
40
+ misp_config = config_data["misp_servers"]
41
+ alerts_config = config_data["alerts_path"]
42
+ email_config = config_data["email"]
43
+ pdns_config = config_data["pdns_client"]
44
+ # Check if they all have the expected format
45
+ bool_conf = (misp_config.kind_of?(Array) and alerts_config.kind_of?(String) and email_config.kind_of?(Hash) and pdns_config.kind_of?(Hash))
46
+ # Check if all the required info is present
47
+ if bool_conf
48
+ misp_subconf = params_to_check(misp_config, ['domain', 'api_key', 'parameter'])
49
+ email_subconf = params_to_check(email_config, ['from', 'to', 'subject', 'server', 'port'])
50
+ end
51
+ # If some field is missing or empty the code breaks
52
+ if ! (bool_conf and misp_subconf and email_subconf)
53
+ @@log_sys.error(CONFIGFILE) #+ "Backtrace: " + e.backtrace.join(" / "))
54
+ raise (error_message)
55
+ end
56
+ return misp_config, alerts_config, email_config, pdns_config
57
+ end
58
+
59
+ def params_to_check(configuration, params)
60
+ # Iterate over all the params to make sure they are on the config file
61
+ for param in params
62
+ # If it is an array you want to check the inner maps
63
+ if configuration.kind_of?(Array)
64
+ for map in configuration
65
+ if ! param_in_map(param,map)
66
+ return false
67
+ end
68
+ end
69
+ else
70
+ if ! param_in_map(param,configuration)
71
+ return false
72
+ end
73
+ end
74
+ return true
75
+ end
76
+ end
77
+
78
+ def param_in_map(param,map)
79
+ # If the param is not on the map or if it is empy -> False
80
+ bool = map.keys.include?(param) and ! map[param].empty?
81
+ return bool
82
+ end
83
+
84
+ end
85
+
86
+
data/lib/constants.rb ADDED
@@ -0,0 +1,43 @@
1
+
2
+ module ConstantsConfig
3
+ file_path = File.expand_path(__FILE__)
4
+ lib_path = File.dirname(file_path)
5
+ common_path = File.dirname(lib_path)
6
+ # If the env variables are not defined, use the default values
7
+ PATH_LOG = ENV['PATH_LOG'] || "/var/log/td-agent/"
8
+ PATH_ALERTS = PATH_LOG + 'pdnssoc-alerts/'
9
+ PATH_TDAGENT = "/etc/td-agent/"
10
+ PATH_PDNS_CONF = ENV['PATH_PDNS_CONF'] || "/etc/pdnssoc/pdnssoc.conf"
11
+ PATH_MISP_D = ENV['PATH_MISP_D'] || File.join(PATH_TDAGENT, "misp_domains.txt")
12
+ PATH_MISP_IP = ENV['PATH_MISP_D'] || File.join(PATH_TDAGENT, "misp_ips.txt")
13
+ PATH_HTML = ENV['PATH_HTML'] || "/etc/pdnssoc/notification_email.html"
14
+ FILENAME_LOG_ALERT = ENV['FILENAME_LOG_ALERT'] || "alerts.log"
15
+ FILENAME_LOG_SYS = ENV['FILENAME_LOG_SYS'] || "pdnssoc_sys.log"
16
+ end
17
+
18
+ module ConstantsGeneral
19
+ WEB_URL="https://%{d}"
20
+ TIME_FORMAT_YMD = "%Y-%m-%dT%H:%M:%S.%L%z"
21
+
22
+ end
23
+
24
+ module ConstantsErrors
25
+ CONFIGFILE = "ConfigFileError. Some parameters of your config file are either missing or have a wrong format. "
26
+ MISPQUERY = "MISP query failed using %{u} and therefore will be skipped. The error message is -> "
27
+ UNKNOWN_CLIENT="An unknown client %{c} has been detected. Add it on the configuration file to receive alerts. "
28
+ TRIGGER_ERROR = "DNS/pDNS queries cannot be read. %{e}"
29
+ SMTP_ERROR = "The email could not be sent. Check the SMTP configuration."
30
+ MISSING_KEY_ALERT = "One of the keys is missing for: %s. This log entry will be skipped"
31
+
32
+ end
33
+
34
+ module ConstantsAlerts
35
+ TIMEOUT_MISP_QUERY = ENV['TIMEOUT_MISP_QUERY'] || 20
36
+ end
37
+
38
+ module ConstantsData
39
+ RGX_FILE_TIME = "/\d{8}-\d{4}/"
40
+ RGX_FILE_REF = 'pdnssoc-buffer.*.log'
41
+ GROUP_SIZE = 5 * 1024 * 1024
42
+ end
43
+
data/lib/email.rb ADDED
@@ -0,0 +1,41 @@
1
+ require 'net/smtp'
2
+ require_relative 'constants'
3
+ require 'erb'
4
+ require 'time'
5
+
6
+
7
+ class Email
8
+ include ConfigAlerts
9
+ include ConstantsErrors
10
+ include ConstantsGeneral
11
+
12
+ def send_email(email_to, all_results)
13
+ begin
14
+ f = File.open(PATH_HTML, "r")
15
+ template_html = ERB.new(f.read)
16
+ html_email = template_html.result(binding)
17
+
18
+ # Compose the message to send
19
+ message = <<~MESSAGE_END
20
+ From: #{@@email_config["from"]}
21
+ To: #{email_to}
22
+ MIME-Version: 1.0
23
+ Content-type: text/html
24
+ Subject: #{@@email_config["subject"]}
25
+
26
+ #{html_email}
27
+
28
+ MESSAGE_END
29
+
30
+ # Send the email
31
+ Net::SMTP.start(@@email_config["server"], @@email_config["port"]) do |smtp|
32
+ smtp.send_message message, @@email_config["from"], email_to end
33
+
34
+ rescue Exception => e
35
+ @@log_sys.error(SMTP_ERROR + e.message) #+ "Backtrace: " + e.backtrace.join(" / "))
36
+ raise Exception, SMTP_ERROR + e.message
37
+ end
38
+ end
39
+
40
+ end
41
+
data/lib/inputdata.rb ADDED
@@ -0,0 +1,46 @@
1
+ require_relative 'constants'
2
+ require 'fileutils'
3
+
4
+ module InputData
5
+ include ConstantsData
6
+ include ConstantsConfig
7
+
8
+ def get_groups()
9
+ # Get a list of all files in the directory
10
+ files = Dir.glob(File.join(PATH_ALERTS, RGX_FILE_REF))
11
+
12
+ # Sort files by date extracted from the filename, newest first
13
+ sorted_files = files.sort_by do |file|
14
+ match = File.basename(file).match(RGX_FILE_TIME)
15
+ match ? match[0] : ''
16
+ end.reverse
17
+
18
+ # Initialize groups as an empty array
19
+ groups = []
20
+ current_group = [] # Initialize current group as an empty array
21
+ current_group_size = 0
22
+
23
+ # Iterate through the sorted files
24
+ sorted_files.each do |file|
25
+ # Get the file size in bytes
26
+ file_size = File.size(file)
27
+
28
+ # Check if adding the file exceeds the group size limit
29
+ if current_group_size + file_size > GROUP_SIZE # Convert 500MB to bytes
30
+ # Add the current group to the groups array
31
+ groups << current_group
32
+
33
+ # Create a new current group and reset current group size
34
+ current_group = []
35
+ current_group_size = 0
36
+ end
37
+
38
+ # Add the file to the current group
39
+ current_group << file
40
+ current_group_size += file_size
41
+ end
42
+ groups << current_group
43
+ return groups
44
+ end
45
+ end
46
+
@@ -0,0 +1,10 @@
1
+ #!/bin/sh
2
+
3
+
4
+ temp_file=$(mktemp)
5
+ alert_path=`grep alerts_path /etc/pdnssoc/pdnssoc.conf | awk -F "\"" '{print $4}'`
6
+
7
+ zgrep -F -f /etc/td-agent/misp_domains.txt /var/log/td-agent/queries.*.log* > ${temp_file}
8
+ grep -w -f /etc/td-agent/misp_domains.txt ${temp_file} >> ${alert_path}pdnssoc-pastlog.log
9
+
10
+ rm -f ${temp_file}
@@ -0,0 +1,26 @@
1
+ #!/bin/sh
2
+ #
3
+ # CRON file for pDNSSOC
4
+
5
+ # Downloading MISP events
6
+
7
+ pull_misp_data(){
8
+ jq --raw-output '.misp_servers[] | "\(.api_key)|\(.domain)|\(.'$1')"' /etc/pdnssoc/pdnssoc.conf |
9
+ while IFS="|" read -r api_key domain parameter; do
10
+ printf -v mycurl 'curl -k -qsS --header "Authorization: %s" "%s%s"\n' "$api_key" "https://$domain/" "$parameter"
11
+ raw_data=$( eval ${mycurl})
12
+ sorted_data=$(printf "$raw_data\n"|sort -u)
13
+ echo "$sorted_data\n" > /etc/td-agent/$2
14
+
15
+ done
16
+ }
17
+
18
+ pull_misp_data "parameter_domains" "misp_domains.txt"
19
+ pull_misp_data "parameter_ips" "misp_ips.txt"
20
+
21
+ # Add something below to send an alert to the security contact if misp_domains.txt is empty
22
+
23
+
24
+ # Reload the configuration file for fluentd
25
+
26
+ kill -1 `cat /var/run/td-agent/td-agent.pid`
data/lib/pdnssoc.rb ADDED
@@ -0,0 +1,7 @@
1
+ require_relative 'trigger'
2
+
3
+ $stdout.sync = true
4
+
5
+ trigger = Trigger.new()
6
+ trigger.run()
7
+
@@ -0,0 +1,16 @@
1
+ puts "PostInstallScript loaded successfully."
2
+
3
+ class PostInstallScript
4
+ def self.run
5
+ puts "PostInstallScript.run method executed."
6
+ # Your post-installation script logic here
7
+ # Will read and execute the tasks defined in tasks/tasks_install.rake
8
+ if ENV['SKIP_POST_INSTALL_HOOK'].nil?
9
+ puts "Running post-installation setup..."
10
+ Rake::Task["rake_install:install"].invoke
11
+ puts "Post-installation setup completed."
12
+ end
13
+ end
14
+ end
15
+
16
+ load File.join(File.dirname(__FILE__), 'tasks', 'tasks_install.rake')
data/lib/trigger.rb ADDED
@@ -0,0 +1,124 @@
1
+ require_relative 'configalerts'
2
+ require_relative 'email'
3
+ require_relative 'alerts'
4
+ require_relative 'constants'
5
+ require_relative 'inputdata'
6
+ require "time"
7
+
8
+ class Trigger
9
+ include ConfigAlerts
10
+ include InputData
11
+
12
+ def initialize()
13
+ super()
14
+ @@alerts_found = {}
15
+ end
16
+
17
+ def delete_logs(processed_logs)
18
+ for filename in processed_logs
19
+ @@log_sys.debug("Deleting: " + filename)
20
+ File.delete(filename) if File.exist?(filename)
21
+ end
22
+ end
23
+
24
+ def get_email_client(ip_client)
25
+ # TODO: Given a boolean, skip any event from clients that are not on the config file
26
+
27
+ # If we have email for that client we will contact them directly
28
+ if @@pdns_config.keys.include?(ip_client) and @@pdns_config[ip_client].keys.include?("email") and ! @@pdns_config[ip_client]["email"].empty?
29
+ email_client = @@pdns_config[ip_client]["email"]
30
+ else
31
+ # If we don't have the email of the clientb we send the email to the default security contact
32
+ email_client = @@email_config["to"]
33
+ end
34
+ return email_client
35
+ end
36
+
37
+ def check_alert_keys(json_log_read)
38
+ required_keys = ["query", "client", "date"]
39
+ all_required_keys = required_keys.all? { |string| json_log_read.key?(string) }
40
+ return all_required_keys
41
+ end
42
+
43
+ def study_ioc(list_iocs, ioc_detected, type_ioc, ip_client, date)
44
+ skip_iocs = []
45
+ # Domains that are legit or that do not have information in MISP will be skipped
46
+ begin
47
+ if ! skip_iocs.include?(ioc_detected)
48
+ # If it is a malicious domain
49
+ if list_iocs.include?(ioc_detected)
50
+ email_client = get_email_client(ip_client)
51
+ # If it was already analyzed -> +1
52
+ if ! @@alerts_found.empty? and @@alerts_found.include?(email_client) and \
53
+ @@alerts_found[email_client].include?(ioc_detected)
54
+ @@alerts_found[email_client][ioc_detected]["count"] += 1
55
+ else
56
+ # We don't have information so we will query MISP
57
+ alert = Alert.new()
58
+ @result_ioc = alert.parse_log(ioc_detected, type_ioc, date, ip_client)
59
+ end
60
+ if @result_ioc.empty?
61
+ # Although it is a malicious domain it doesn't have any data in MISP -> skip next time
62
+ skip_iocs.append(ioc_detected)
63
+ else
64
+ # We have found data in MISP about this domain -> we will report it to the right client
65
+ if ! @@alerts_found.include?(email_client)
66
+ @@alerts_found[email_client] = {}
67
+ end
68
+ @@alerts_found[email_client][ioc_detected] = @result_ioc
69
+ @@log_alerts.info(@result_ioc)
70
+ end
71
+ else
72
+ skip_iocs.append(ioc_detected)
73
+ end
74
+ end
75
+ rescue Exception => e
76
+ raise Exception, TRIGGER_ERROR % [e:e]
77
+ end
78
+ end
79
+
80
+ def analyze_all_iocs(group_of_files)
81
+ all_alerts = {}
82
+ # Iterate over all files inside the group
83
+ for filename in group_of_files
84
+ # Read each line opf the file that represents one log entry
85
+ File.readlines(filename).each do |line|
86
+ json_log = JSON.parse(line)
87
+ # If the data is not complete -> skip log
88
+ if ! check_alert_keys(json_log)
89
+ @@log_sys.error(MISSING_KEY_ALERT % line)
90
+ next
91
+ end
92
+ ip_client = json_log["client"]
93
+ date = Time.parse(json_log["date"]).to_i
94
+ # If in addition of the domain, the resolved IP is provided, it will be analyzed
95
+ if json_log.keys.include?("answer")
96
+ study_ioc(@@bad_ips, json_log["answer"], "ip", ip_client, date)
97
+ end
98
+ # Analyze if the domain is malicious
99
+ study_ioc(@@bad_domains, json_log["query"], "domain", ip_client, date)
100
+ end
101
+ end
102
+ end
103
+
104
+ def run()
105
+ # The files to process are gouped so in case of failure at least some of them will be processed
106
+ groups = get_groups()
107
+ for group_of_files in groups
108
+ # Analyze all IOCs from the logs
109
+ analyze_all_iocs(group_of_files)
110
+ if @@alerts_found.empty?
111
+ @@log_sys.debug("No alerts found!")
112
+ else
113
+ # We will send an alert to each client (with an email on the config file)
114
+ @@alerts_found.each do |email_client, client_data|
115
+ email = Email.new()
116
+ email.send_email(email_client, client_data)
117
+ end
118
+ end
119
+ # If the send_email is not successful the logs will not be deleted
120
+ delete_logs(group_of_files)
121
+ end
122
+ end
123
+ end
124
+
@@ -0,0 +1,6 @@
1
+ [Unit]
2
+ Description=Run lookingback.sh every day at 12:00
3
+
4
+ [Service]
5
+ ExecStart=/bin/bash /usr/local/bin/pdnssoc/lookingback.sh
6
+ User=root
@@ -0,0 +1,9 @@
1
+ [Unit]
2
+ Description=Run lookingback.sh every day at 12:00
3
+
4
+ [Timer]
5
+ OnCalendar=*-*-* 12:00:00
6
+ Persistent=true
7
+
8
+ [Install]
9
+ WantedBy=timers.target
@@ -0,0 +1,6 @@
1
+ [Unit]
2
+ Description=Run misp_refresh.sh every 15 minutes
3
+
4
+ [Service]
5
+ ExecStart=/bin/bash /usr/local/bin/pdnssoc/misp_refresh.sh
6
+ User=root
@@ -0,0 +1,9 @@
1
+ [Unit]
2
+ Description=Run misp_refresh.sh every hour
3
+
4
+ [Timer]
5
+ OnCalendar=hourly
6
+ Persistent=true
7
+
8
+ [Install]
9
+ WantedBy=timers.target
@@ -0,0 +1,6 @@
1
+ [Unit]
2
+ Description=Run pdnssoc.rb every 15 minutes
3
+
4
+ [Service]
5
+ ExecStart=/opt/td-agent/bin/ruby /usr/local/bin/pdnssoc/pdnssoc.rb
6
+ User=root
@@ -0,0 +1,9 @@
1
+ [Unit]
2
+ Description=Run pdnssoc.rb every 15 minutes
3
+
4
+ [Timer]
5
+ OnCalendar=*:0/15
6
+ Persistent=true
7
+
8
+ [Install]
9
+ WantedBy=timers.target
metadata ADDED
@@ -0,0 +1,70 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: pdnssoc
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.1.3
5
+ platform: ruby
6
+ authors:
7
+ - Pau Cutrina
8
+ - Romain Wartel
9
+ - Christos Arvanitis
10
+ autorequire:
11
+ bindir: bin
12
+ cert_chain: []
13
+ date: 2023-08-07 00:00:00.000000000 Z
14
+ dependencies: []
15
+ description: pDNS correlation with MISP
16
+ email:
17
+ - admin@safer-trust.org
18
+ executables: []
19
+ extensions: []
20
+ extra_rdoc_files: []
21
+ files:
22
+ - config/notification_email.html
23
+ - config/pdnssoc.conf
24
+ - config/td-agent.conf
25
+ - lib/alerts.rb
26
+ - lib/configalerts.rb
27
+ - lib/constants.rb
28
+ - lib/email.rb
29
+ - lib/inputdata.rb
30
+ - lib/lookingback.sh
31
+ - lib/misp_refresh.sh
32
+ - lib/pdnssoc.rb
33
+ - lib/post_install.rb
34
+ - lib/trigger.rb
35
+ - timers/lookingback.service
36
+ - timers/lookingback.timer
37
+ - timers/misp_refresh.service
38
+ - timers/misp_refresh.timer
39
+ - timers/pdnssoc.service
40
+ - timers/pdnssoc.timer
41
+ homepage: https://github.com/CERN-CERT/pDNSSOC/
42
+ licenses:
43
+ - MIT
44
+ metadata:
45
+ rubygems_mfa_required: 'true'
46
+ source_code_uri: https://github.com/CERN-CERT/pDNSSOC
47
+ changelog_uri: https://github.com/CERN-CERT/pDNSSOC/blob/master/CHANGELOG.md
48
+ homepage_uri: https://github.com/CERN-CERT/pDNSSOC
49
+ github_repo: ssh://github.com/CERN-CERT/pDNSSOC
50
+ post_install_message: pDNSSOC has been installed successfuly!
51
+ rdoc_options: []
52
+ require_paths:
53
+ - lib
54
+ required_ruby_version: !ruby/object:Gem::Requirement
55
+ requirements:
56
+ - - ">="
57
+ - !ruby/object:Gem::Version
58
+ version: 2.5.0
59
+ required_rubygems_version: !ruby/object:Gem::Requirement
60
+ requirements:
61
+ - - ">="
62
+ - !ruby/object:Gem::Version
63
+ version: '0'
64
+ requirements:
65
+ - Ruby (>= 2.5.0)
66
+ rubygems_version: 3.2.33
67
+ signing_key:
68
+ specification_version: 4
69
+ summary: pDNS correlation with MISP
70
+ test_files: []