pdnssoc 0.1.3
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/config/notification_email.html +80 -0
- data/config/pdnssoc.conf +32 -0
- data/config/td-agent.conf +173 -0
- data/lib/alerts.rb +118 -0
- data/lib/configalerts.rb +86 -0
- data/lib/constants.rb +43 -0
- data/lib/email.rb +41 -0
- data/lib/inputdata.rb +46 -0
- data/lib/lookingback.sh +10 -0
- data/lib/misp_refresh.sh +26 -0
- data/lib/pdnssoc.rb +7 -0
- data/lib/post_install.rb +16 -0
- data/lib/trigger.rb +124 -0
- data/timers/lookingback.service +6 -0
- data/timers/lookingback.timer +9 -0
- data/timers/misp_refresh.service +6 -0
- data/timers/misp_refresh.timer +9 -0
- data/timers/pdnssoc.service +6 -0
- data/timers/pdnssoc.timer +9 -0
- metadata +70 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA256:
|
3
|
+
metadata.gz: 252758f53648ce58439fc34723eb137fc8cd99225720895cb42dd095ff9d56bc
|
4
|
+
data.tar.gz: 6a41db486378f24aa6a56765109c8c0cdd0aacf855b07a7f9616b7499bc98526
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: 0f81edfb60851df94c8c2541bfb71c7d264d5cda518ce1327d1af6051762984ae47eade862ad60defcb14bae1d530ce1a078ba1998a6ae28f901e41474460c29
|
7
|
+
data.tar.gz: ebdf588d2afad5cbec0379d6e50e11c82a29b61449b09867e77aeee30a9e739e203f87c2e80aa4e0da814f2b3f151459fdd5724d775a6d292597bf81effa0ab2
|
@@ -0,0 +1,80 @@
|
|
1
|
+
<!DOCTYPE html>
|
2
|
+
<html lang="en">
|
3
|
+
<head>
|
4
|
+
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
|
5
|
+
<style>
|
6
|
+
body {
|
7
|
+
font-family: sans-serif;
|
8
|
+
font-size: 10pt;
|
9
|
+
}
|
10
|
+
h1 {
|
11
|
+
font-size: 16pt;
|
12
|
+
}
|
13
|
+
table th {
|
14
|
+
padding: 10px;
|
15
|
+
border-top: 1px solid #fafafa;
|
16
|
+
border-bottom: 1px solid #e0e0e0;
|
17
|
+
background: #ededed;
|
18
|
+
background: -webkit-gradient(linear, left top, left bottom, from(#ededed), to(#ebebeb));
|
19
|
+
background: -moz-linear-gradient(top, #ededed, #ebebeb);
|
20
|
+
}
|
21
|
+
table {
|
22
|
+
border-collapse: collapse;
|
23
|
+
border-spacing: 0;
|
24
|
+
}
|
25
|
+
table td,
|
26
|
+
table th {
|
27
|
+
border: 1px solid #ccc;
|
28
|
+
}
|
29
|
+
table td {
|
30
|
+
padding: 2px 5px;
|
31
|
+
}
|
32
|
+
</style>
|
33
|
+
</head>
|
34
|
+
<body>
|
35
|
+
<table>
|
36
|
+
<tbody>
|
37
|
+
<tr>
|
38
|
+
<th>pDNSSOC client</th>
|
39
|
+
<th>First Occurrence</th>
|
40
|
+
<th>IoCs detected</th>
|
41
|
+
<th>MISP event</th>
|
42
|
+
<th>Total # of IoCs</th>
|
43
|
+
<th>Publication</th>
|
44
|
+
<th>Organisation</th>
|
45
|
+
<th>Comment</th>
|
46
|
+
<th>Tags</th>
|
47
|
+
</tr>
|
48
|
+
<% all_results.each do |ioc, data_ioc| %>
|
49
|
+
<tr>
|
50
|
+
<% if data_ioc["client_name"] == "" %>
|
51
|
+
<td style="text-align: left;" rowspan="<%= data_ioc["misp"].length %>"><%= data_ioc["client_ip"] %></td>
|
52
|
+
<% else %>
|
53
|
+
<td style="text-align: left;" rowspan="<%= data_ioc["misp"].length %>"><%= data_ioc["client_name"] %> (<%= data_ioc["client_ip"] %>)</td>
|
54
|
+
<% end %>
|
55
|
+
<td style="text-align: left;" rowspan="<%= data_ioc["misp"].length %>"><%= Time.at(data_ioc["first_occurrence"]).strftime(TIME_FORMAT_YMD) %></td>
|
56
|
+
<td style="text-align: left;" rowspan="<%= data_ioc["misp"].length %>"><a href="" target="_new"><%= data_ioc["ioc_detected"] %></a></td>
|
57
|
+
<% data_ioc["misp"].each_with_index do |misp_event, idx_event| %>
|
58
|
+
<% if idx_event > 0 %>
|
59
|
+
<tr>
|
60
|
+
<% end %>
|
61
|
+
<td style="text-align: left;"><a href="https://<%= misp_event["misp_server"] %>/events/view/<%= misp_event["misp_id"] %>" target="_new"><%= misp_event["misp_info"] %></a></td>
|
62
|
+
<td style="text-align: left;"><%= misp_event["num_iocs"] %></td>
|
63
|
+
<td style="text-align: left;"><%= misp_event["publication"] %></td>
|
64
|
+
<td style="text-align: left;"><%= misp_event["organisation"] %></td>
|
65
|
+
<td style="text-align: left;"><%= misp_event["comment"] %></td>
|
66
|
+
<td style="text-align: left;">
|
67
|
+
<% for tag in misp_event["tags"] do %>
|
68
|
+
<span style="background: <%= tag["colour"] %>;">
|
69
|
+
<b><span style="color: #fff; mix-blend-mode: difference; padding: 5px;"><%= tag["name"] %></span></b>
|
70
|
+
</span>
|
71
|
+
<% end %>
|
72
|
+
</td>
|
73
|
+
</tr>
|
74
|
+
<% end %>
|
75
|
+
</tr>
|
76
|
+
<% end %>
|
77
|
+
</tbody>
|
78
|
+
</table>
|
79
|
+
</body>
|
80
|
+
</html>
|
data/config/pdnssoc.conf
ADDED
@@ -0,0 +1,32 @@
|
|
1
|
+
{
|
2
|
+
"misp_servers": [
|
3
|
+
{
|
4
|
+
"domain":"misp1.myserver.org",
|
5
|
+
"api_key":"API_KEY_1",
|
6
|
+
"parameter_domains":"/attributes/restSearch/returnFormat:text/type:domain/",
|
7
|
+
"parameter_ips":"/attributes/restSearch/returnFormat:text/type:ip-src/type:ip-dst/"
|
8
|
+
},
|
9
|
+
{
|
10
|
+
"domain":"misp2.myotherserver.ch",
|
11
|
+
"api_key":"API_KEY_2",
|
12
|
+
"parameter_domains":"/attributes/restSearch/returnFormat:text/type:domain/from:2022-01-01/",
|
13
|
+
"parameter_ips":"/attributes/restSearch/returnFormat:text/type:ip-src/type:ip-dst/from:2022-01-01/"
|
14
|
+
}
|
15
|
+
],
|
16
|
+
"alerts_path":"/var/log/td-agent/pdnssoc-alerts/",
|
17
|
+
"pdns_client" : {
|
18
|
+
"127.0.0.1":
|
19
|
+
{
|
20
|
+
"name":"Test host",
|
21
|
+
"email":"email@address.tld"
|
22
|
+
}
|
23
|
+
},
|
24
|
+
"email" : {
|
25
|
+
"from":"pdnssoc-dev@domain.tld",
|
26
|
+
"to":"pdnssoc-dev@domain.tld",
|
27
|
+
"subject":"[pDNSSOC] Community XYZ alert",
|
28
|
+
"server":localhost",
|
29
|
+
"port":25
|
30
|
+
}
|
31
|
+
|
32
|
+
}
|
@@ -0,0 +1,173 @@
|
|
1
|
+
## match tag=debug.** and dump to console
|
2
|
+
<match debug.**>
|
3
|
+
@type stdout
|
4
|
+
@id output_stdout
|
5
|
+
</match>
|
6
|
+
|
7
|
+
## SYSLOG INGESTION
|
8
|
+
# Getting DNS queries from syslog
|
9
|
+
<source>
|
10
|
+
@type syslog
|
11
|
+
port 5140
|
12
|
+
protocol_type tcp
|
13
|
+
tag syslog
|
14
|
+
</source>
|
15
|
+
# Parsing DNS queries from syslog
|
16
|
+
<match syslog>
|
17
|
+
@type rewrite_tag_filter
|
18
|
+
<rule>
|
19
|
+
key message
|
20
|
+
pattern /(?<date>.*) client \@.* (?<client>.*)#.*: query: (?<query>\S+) .*/
|
21
|
+
tag pdnssocdata
|
22
|
+
</rule>
|
23
|
+
</match>
|
24
|
+
|
25
|
+
## TCP INGESTION
|
26
|
+
# The data coming in BIND logs using syslog format
|
27
|
+
<source>
|
28
|
+
@type tcp
|
29
|
+
port 5141
|
30
|
+
tag pdnssocdata
|
31
|
+
<parse>
|
32
|
+
@type regexp
|
33
|
+
expression /(?<date>.*) client \@.* (?<client>.*)#.*: query: (?<query>\S+) .*/
|
34
|
+
# BIND9 variant:
|
35
|
+
#expression /(?<date>.*) client (?<client>.*)#.*query: (?<query>\S+) .*/
|
36
|
+
</parse>
|
37
|
+
# delimiter "\n" # optional. "\n" (newline) by default
|
38
|
+
</source>
|
39
|
+
|
40
|
+
## dnstap pDNS data INGESTION
|
41
|
+
# dnstap must be configured to forward RESOLVER_RESPONSE, for example in BIND: "dnstap { resolver response; };"
|
42
|
+
# (?<date>.*) RR (?<client>.*):\w+ <- .* \w+ (?<query>[\.|\w]+)\/.*$
|
43
|
+
<source>
|
44
|
+
@type tcp
|
45
|
+
port 5142
|
46
|
+
tag pdnssocdata
|
47
|
+
<parse>
|
48
|
+
@type regexp
|
49
|
+
expression /(?<date>.*) RR (?<client>.*):\w+ <- .* \w+ (?<query>\w+\.\w+)\/.*/
|
50
|
+
</parse>
|
51
|
+
</source>
|
52
|
+
|
53
|
+
## HTTP source to ingest JSON log entries
|
54
|
+
<source>
|
55
|
+
@type http
|
56
|
+
tag pdnssocdata
|
57
|
+
port 5143
|
58
|
+
bind 0.0.0.0
|
59
|
+
body_size_limit 32m
|
60
|
+
keepalive_timeout 10s
|
61
|
+
</source>
|
62
|
+
|
63
|
+
## OTHER PDNSSOC SERVERS
|
64
|
+
# Data coming from lower-level pDNSSOC servers
|
65
|
+
<source>
|
66
|
+
@type forward
|
67
|
+
port 5555
|
68
|
+
tag pdnssocdata
|
69
|
+
<parse>
|
70
|
+
@type regexp
|
71
|
+
expression /{"date":"(?<date>.*)","client":"(?<client>.*)","query":"(?<query>.*)"}/
|
72
|
+
</parse>
|
73
|
+
</source>
|
74
|
+
|
75
|
+
|
76
|
+
## DATA ROUTING
|
77
|
+
# Copying our pdnssocdata into multiple streams
|
78
|
+
## DATA ROUTING
|
79
|
+
# Copying our pdnssocdata into multiple streams
|
80
|
+
<match pdnssocdata>
|
81
|
+
@type copy
|
82
|
+
<store>
|
83
|
+
@type relabel
|
84
|
+
@label @pdnssocdata_allqueries
|
85
|
+
</store>
|
86
|
+
<store>
|
87
|
+
@type relabel
|
88
|
+
@label @pdnssocdata_correlate_query
|
89
|
+
</store>
|
90
|
+
<store>
|
91
|
+
@type relabel
|
92
|
+
@label @pdnssocdata_correlate_response
|
93
|
+
</store>
|
94
|
+
|
95
|
+
## Send data to another pDNSSOC server
|
96
|
+
#<store>
|
97
|
+
# @type forward
|
98
|
+
# send_timeout 60s
|
99
|
+
# recover_wait 10s
|
100
|
+
# hard_timeout 60s
|
101
|
+
# <server>
|
102
|
+
# host upstream-pdnssoc.domain.edu
|
103
|
+
# port 5555
|
104
|
+
# </server>
|
105
|
+
#</store>
|
106
|
+
#######################
|
107
|
+
|
108
|
+
</match>
|
109
|
+
|
110
|
+
# Logging all queries (pdnssocdata_allqueries) to disc.
|
111
|
+
# This is used by the "looking back in the past" mechanism.
|
112
|
+
<label @pdnssocdata_allqueries>
|
113
|
+
<match pdnssocdata>
|
114
|
+
@type file
|
115
|
+
path /var/log/td-agent/queries
|
116
|
+
compress gzip
|
117
|
+
time_slice_format %Y%m%d-%H%M
|
118
|
+
<format>
|
119
|
+
@type json
|
120
|
+
</format>
|
121
|
+
</match>
|
122
|
+
</label>
|
123
|
+
|
124
|
+
|
125
|
+
# Correlating the pdnssocdata_correlate data flow with malicious domains from MISP
|
126
|
+
# Then writing to disc
|
127
|
+
# Matching DNS queries with malicious domains
|
128
|
+
<label @pdnssocdata_correlate_query>
|
129
|
+
<filter pdnssocdata>
|
130
|
+
@type filter_list
|
131
|
+
filter AC
|
132
|
+
key_to_filter query
|
133
|
+
pattern_file_paths ["/etc/td-agent/misp_domains.txt"]
|
134
|
+
filter_empty true
|
135
|
+
action whitelist
|
136
|
+
</filter>
|
137
|
+
<match pdnssocdata>
|
138
|
+
@type relabel
|
139
|
+
@label @pdnssocdata_correlate
|
140
|
+
</match>
|
141
|
+
</label>
|
142
|
+
|
143
|
+
# Matching DNS replies with malicious IPs
|
144
|
+
<label @pdnssocdata_correlate_response>
|
145
|
+
<filter pdnssocdata>
|
146
|
+
@type filter_list
|
147
|
+
filter AC
|
148
|
+
key_to_filter answer
|
149
|
+
pattern_file_paths ["/etc/td-agent/misp_domains.txt", "/etc/td-agent/misp_ips.txt"]
|
150
|
+
filter_empty true
|
151
|
+
action whitelist
|
152
|
+
</filter>
|
153
|
+
<match pdnssocdata>
|
154
|
+
@type relabel
|
155
|
+
@label @pdnssocdata_correlate
|
156
|
+
</match>
|
157
|
+
</label>
|
158
|
+
|
159
|
+
<label @pdnssocdata_correlate>
|
160
|
+
<match pdnssocdata>
|
161
|
+
@type file
|
162
|
+
path /var/log/td-agent/pdnssoc-alerts/pdnssoc-buffer
|
163
|
+
time_slice_format %Y%m%d-%H%M
|
164
|
+
<format>
|
165
|
+
@type json
|
166
|
+
</format>
|
167
|
+
<buffer>
|
168
|
+
timekey 2m
|
169
|
+
timekey_use_utc true
|
170
|
+
timekey_wait 1m
|
171
|
+
</buffer>
|
172
|
+
</match>
|
173
|
+
</label>
|
data/lib/alerts.rb
ADDED
@@ -0,0 +1,118 @@
|
|
1
|
+
require "misp"
|
2
|
+
require 'timeout'
|
3
|
+
require_relative 'configalerts'
|
4
|
+
|
5
|
+
class Alert
|
6
|
+
include ConfigAlerts
|
7
|
+
include ConstantsErrors
|
8
|
+
include ConstantsAlerts
|
9
|
+
include ConstantsGeneral
|
10
|
+
|
11
|
+
@@faulty_misp_servers = []
|
12
|
+
|
13
|
+
def get_faulty_misp()
|
14
|
+
return @@faulty_misp_servers
|
15
|
+
end
|
16
|
+
|
17
|
+
def query_misp(misp_server, ioc_detected, type_ioc, all_uuids)
|
18
|
+
result_misp = []
|
19
|
+
list_uuids = []
|
20
|
+
misp_url = WEB_URL % [d: misp_server["domain"]]
|
21
|
+
misp_api_key = misp_server["api_key"]
|
22
|
+
api_endpoint = misp_url + misp_server["parameter_%{t}s" % [t:type_ioc]] # parameter_ips
|
23
|
+
# Setup config MISP server
|
24
|
+
MISP.configure do |config|
|
25
|
+
config.api_endpoint = api_endpoint
|
26
|
+
config.api_key = misp_api_key
|
27
|
+
end
|
28
|
+
|
29
|
+
# Search
|
30
|
+
misp_events = []
|
31
|
+
begin
|
32
|
+
Timeout::timeout(TIMEOUT_MISP_QUERY) {
|
33
|
+
misp_events = MISP::Event.search(value: ioc_detected)
|
34
|
+
# One domain can have multiple events associated
|
35
|
+
for misp_event in misp_events
|
36
|
+
# If there's a valid event and we did not processed it before proceed
|
37
|
+
if not all_uuids.include? misp_event.uuid
|
38
|
+
# Find the attribute where the malicious domain is defined
|
39
|
+
for attribute in misp_event.attributes
|
40
|
+
if attribute.type.include?(type_ioc) and attribute.value == ioc_detected and attribute.to_ids == true
|
41
|
+
$tags = []
|
42
|
+
for tag in attribute.tags do
|
43
|
+
$tags.append({"colour" => tag.colour, "name" => tag.name})
|
44
|
+
end
|
45
|
+
event = {
|
46
|
+
'misp_uuid' => misp_event.uuid,
|
47
|
+
'misp_info' => misp_event.info,
|
48
|
+
'misp_id' => misp_event.id,
|
49
|
+
'misp_server' => misp_server["domain"],
|
50
|
+
'num_iocs' => misp_event.attribute_count,
|
51
|
+
'publication' => misp_event.date,
|
52
|
+
'organisation' => misp_event.orgc.name,
|
53
|
+
'comment' => attribute.comment,
|
54
|
+
'tags' => $tags
|
55
|
+
}
|
56
|
+
list_uuids.append(misp_event.uuid)
|
57
|
+
result_misp.append(event)
|
58
|
+
break
|
59
|
+
end
|
60
|
+
end
|
61
|
+
end
|
62
|
+
end
|
63
|
+
}
|
64
|
+
rescue Exception => e
|
65
|
+
error_message = MISPQUERY % [u:misp_url]
|
66
|
+
@@log_sys.error(error_message + e.message)
|
67
|
+
@@faulty_misp_servers.append(misp_url)
|
68
|
+
ensure
|
69
|
+
return result_misp, list_uuids
|
70
|
+
end
|
71
|
+
end
|
72
|
+
|
73
|
+
def get_client_info(ip_client, key_reference)
|
74
|
+
pdns_client = ""
|
75
|
+
if @@pdns_config[ip_client]
|
76
|
+
# If the name is present on the configuration file we will use it on the email. Otherwise we will just use the ip
|
77
|
+
if @@pdns_config[ip_client].keys.include?(key_reference) and ! @@pdns_config[ip_client][key_reference].empty?
|
78
|
+
pdns_client = @@pdns_config[ip_client][key_reference]
|
79
|
+
end
|
80
|
+
else
|
81
|
+
@@log_sys.error(UNKNOWN_CLIENT % [c:ip_client])
|
82
|
+
end
|
83
|
+
return pdns_client
|
84
|
+
end
|
85
|
+
|
86
|
+
def parse_log(ioc_detected, type_ioc, date, ip_client)
|
87
|
+
result_ioc = {}
|
88
|
+
events = [] # MISP events detected for a particular IOC
|
89
|
+
events_uuid = [] # List of MISP uuid used to avoid repeat events
|
90
|
+
|
91
|
+
for misp_server in @@misp_config
|
92
|
+
if ! @@faulty_misp_servers.include?(misp_server["url"])
|
93
|
+
events_uuid = events_uuid.flatten
|
94
|
+
events_detected, uuids=query_misp(misp_server, ioc_detected, type_ioc, events_uuid)
|
95
|
+
if events_detected.length > 0
|
96
|
+
events_uuid.append(uuids)
|
97
|
+
events.append(events_detected)
|
98
|
+
end
|
99
|
+
end
|
100
|
+
end
|
101
|
+
events = events.flatten
|
102
|
+
$num_misp_events = events.length()
|
103
|
+
|
104
|
+
if $num_misp_events > 0
|
105
|
+
result_ioc= {
|
106
|
+
'client_ip' => ip_client,
|
107
|
+
'client_name' => get_client_info(ip_client, "name"),
|
108
|
+
'count' => 1,
|
109
|
+
'first_occurrence' => date,
|
110
|
+
'ioc_detected' => ioc_detected,
|
111
|
+
"misp" => events
|
112
|
+
}
|
113
|
+
end
|
114
|
+
return result_ioc
|
115
|
+
end
|
116
|
+
end
|
117
|
+
|
118
|
+
|
data/lib/configalerts.rb
ADDED
@@ -0,0 +1,86 @@
|
|
1
|
+
require "json"
|
2
|
+
require 'logger'
|
3
|
+
require "time"
|
4
|
+
puts $LOAD_PATH
|
5
|
+
require 'parseconfig'
|
6
|
+
require_relative 'constants'
|
7
|
+
|
8
|
+
|
9
|
+
module ConfigAlerts
|
10
|
+
include ConstantsConfig
|
11
|
+
include ConstantsErrors
|
12
|
+
|
13
|
+
def initialize()
|
14
|
+
# Setup Logging
|
15
|
+
@@log_alerts = Logger.new(PATH_LOG + FILENAME_LOG_ALERT, 'daily')
|
16
|
+
@@log_alerts.formatter = proc do |severity, datetime, progname, msg| {message: msg}.to_json + $/ end
|
17
|
+
@@log_sys = Logger.new(PATH_LOG + FILENAME_LOG_SYS, 'daily')
|
18
|
+
@@log_sys.formatter = proc do |severity, datetime, progname, msg| "#{datetime}, #{severity}: #{msg} #{progname} \n" end
|
19
|
+
# Open config files
|
20
|
+
@@misp_config, @@alerts_config, @@email_config, @@pdns_config = init_config()
|
21
|
+
# Get the list of bad domains
|
22
|
+
@@bad_domains = File.read(PATH_MISP_D)
|
23
|
+
@@bad_ips = File.read(PATH_MISP_IP)
|
24
|
+
# Get HTML template for the email
|
25
|
+
@@html_email = init_html()
|
26
|
+
end
|
27
|
+
|
28
|
+
def init_html()
|
29
|
+
html_data = ""
|
30
|
+
# Template HTML of the email
|
31
|
+
f = File.open(PATH_HTML, "r")
|
32
|
+
f.each_line do |line| html_data += line end
|
33
|
+
raise TypeError, "html_data expected an String, got #{html_data.class.name}" unless html_data.kind_of?(String)
|
34
|
+
return html_data
|
35
|
+
end
|
36
|
+
|
37
|
+
def init_config()
|
38
|
+
config_data = JSON.parse(File.read(PATH_PDNS_CONF))
|
39
|
+
# Initialize vaiables
|
40
|
+
misp_config = config_data["misp_servers"]
|
41
|
+
alerts_config = config_data["alerts_path"]
|
42
|
+
email_config = config_data["email"]
|
43
|
+
pdns_config = config_data["pdns_client"]
|
44
|
+
# Check if they all have the expected format
|
45
|
+
bool_conf = (misp_config.kind_of?(Array) and alerts_config.kind_of?(String) and email_config.kind_of?(Hash) and pdns_config.kind_of?(Hash))
|
46
|
+
# Check if all the required info is present
|
47
|
+
if bool_conf
|
48
|
+
misp_subconf = params_to_check(misp_config, ['domain', 'api_key', 'parameter'])
|
49
|
+
email_subconf = params_to_check(email_config, ['from', 'to', 'subject', 'server', 'port'])
|
50
|
+
end
|
51
|
+
# If some field is missing or empty the code breaks
|
52
|
+
if ! (bool_conf and misp_subconf and email_subconf)
|
53
|
+
@@log_sys.error(CONFIGFILE) #+ "Backtrace: " + e.backtrace.join(" / "))
|
54
|
+
raise (error_message)
|
55
|
+
end
|
56
|
+
return misp_config, alerts_config, email_config, pdns_config
|
57
|
+
end
|
58
|
+
|
59
|
+
def params_to_check(configuration, params)
|
60
|
+
# Iterate over all the params to make sure they are on the config file
|
61
|
+
for param in params
|
62
|
+
# If it is an array you want to check the inner maps
|
63
|
+
if configuration.kind_of?(Array)
|
64
|
+
for map in configuration
|
65
|
+
if ! param_in_map(param,map)
|
66
|
+
return false
|
67
|
+
end
|
68
|
+
end
|
69
|
+
else
|
70
|
+
if ! param_in_map(param,configuration)
|
71
|
+
return false
|
72
|
+
end
|
73
|
+
end
|
74
|
+
return true
|
75
|
+
end
|
76
|
+
end
|
77
|
+
|
78
|
+
def param_in_map(param,map)
|
79
|
+
# If the param is not on the map or if it is empy -> False
|
80
|
+
bool = map.keys.include?(param) and ! map[param].empty?
|
81
|
+
return bool
|
82
|
+
end
|
83
|
+
|
84
|
+
end
|
85
|
+
|
86
|
+
|
data/lib/constants.rb
ADDED
@@ -0,0 +1,43 @@
|
|
1
|
+
|
2
|
+
module ConstantsConfig
|
3
|
+
file_path = File.expand_path(__FILE__)
|
4
|
+
lib_path = File.dirname(file_path)
|
5
|
+
common_path = File.dirname(lib_path)
|
6
|
+
# If the env variables are not defined, use the default values
|
7
|
+
PATH_LOG = ENV['PATH_LOG'] || "/var/log/td-agent/"
|
8
|
+
PATH_ALERTS = PATH_LOG + 'pdnssoc-alerts/'
|
9
|
+
PATH_TDAGENT = "/etc/td-agent/"
|
10
|
+
PATH_PDNS_CONF = ENV['PATH_PDNS_CONF'] || "/etc/pdnssoc/pdnssoc.conf"
|
11
|
+
PATH_MISP_D = ENV['PATH_MISP_D'] || File.join(PATH_TDAGENT, "misp_domains.txt")
|
12
|
+
PATH_MISP_IP = ENV['PATH_MISP_D'] || File.join(PATH_TDAGENT, "misp_ips.txt")
|
13
|
+
PATH_HTML = ENV['PATH_HTML'] || "/etc/pdnssoc/notification_email.html"
|
14
|
+
FILENAME_LOG_ALERT = ENV['FILENAME_LOG_ALERT'] || "alerts.log"
|
15
|
+
FILENAME_LOG_SYS = ENV['FILENAME_LOG_SYS'] || "pdnssoc_sys.log"
|
16
|
+
end
|
17
|
+
|
18
|
+
module ConstantsGeneral
|
19
|
+
WEB_URL="https://%{d}"
|
20
|
+
TIME_FORMAT_YMD = "%Y-%m-%dT%H:%M:%S.%L%z"
|
21
|
+
|
22
|
+
end
|
23
|
+
|
24
|
+
module ConstantsErrors
|
25
|
+
CONFIGFILE = "ConfigFileError. Some parameters of your config file are either missing or have a wrong format. "
|
26
|
+
MISPQUERY = "MISP query failed using %{u} and therefore will be skipped. The error message is -> "
|
27
|
+
UNKNOWN_CLIENT="An unknown client %{c} has been detected. Add it on the configuration file to receive alerts. "
|
28
|
+
TRIGGER_ERROR = "DNS/pDNS queries cannot be read. %{e}"
|
29
|
+
SMTP_ERROR = "The email could not be sent. Check the SMTP configuration."
|
30
|
+
MISSING_KEY_ALERT = "One of the keys is missing for: %s. This log entry will be skipped"
|
31
|
+
|
32
|
+
end
|
33
|
+
|
34
|
+
module ConstantsAlerts
|
35
|
+
TIMEOUT_MISP_QUERY = ENV['TIMEOUT_MISP_QUERY'] || 20
|
36
|
+
end
|
37
|
+
|
38
|
+
module ConstantsData
|
39
|
+
RGX_FILE_TIME = "/\d{8}-\d{4}/"
|
40
|
+
RGX_FILE_REF = 'pdnssoc-buffer.*.log'
|
41
|
+
GROUP_SIZE = 5 * 1024 * 1024
|
42
|
+
end
|
43
|
+
|
data/lib/email.rb
ADDED
@@ -0,0 +1,41 @@
|
|
1
|
+
require 'net/smtp'
|
2
|
+
require_relative 'constants'
|
3
|
+
require 'erb'
|
4
|
+
require 'time'
|
5
|
+
|
6
|
+
|
7
|
+
class Email
|
8
|
+
include ConfigAlerts
|
9
|
+
include ConstantsErrors
|
10
|
+
include ConstantsGeneral
|
11
|
+
|
12
|
+
def send_email(email_to, all_results)
|
13
|
+
begin
|
14
|
+
f = File.open(PATH_HTML, "r")
|
15
|
+
template_html = ERB.new(f.read)
|
16
|
+
html_email = template_html.result(binding)
|
17
|
+
|
18
|
+
# Compose the message to send
|
19
|
+
message = <<~MESSAGE_END
|
20
|
+
From: #{@@email_config["from"]}
|
21
|
+
To: #{email_to}
|
22
|
+
MIME-Version: 1.0
|
23
|
+
Content-type: text/html
|
24
|
+
Subject: #{@@email_config["subject"]}
|
25
|
+
|
26
|
+
#{html_email}
|
27
|
+
|
28
|
+
MESSAGE_END
|
29
|
+
|
30
|
+
# Send the email
|
31
|
+
Net::SMTP.start(@@email_config["server"], @@email_config["port"]) do |smtp|
|
32
|
+
smtp.send_message message, @@email_config["from"], email_to end
|
33
|
+
|
34
|
+
rescue Exception => e
|
35
|
+
@@log_sys.error(SMTP_ERROR + e.message) #+ "Backtrace: " + e.backtrace.join(" / "))
|
36
|
+
raise Exception, SMTP_ERROR + e.message
|
37
|
+
end
|
38
|
+
end
|
39
|
+
|
40
|
+
end
|
41
|
+
|
data/lib/inputdata.rb
ADDED
@@ -0,0 +1,46 @@
|
|
1
|
+
require_relative 'constants'
|
2
|
+
require 'fileutils'
|
3
|
+
|
4
|
+
module InputData
|
5
|
+
include ConstantsData
|
6
|
+
include ConstantsConfig
|
7
|
+
|
8
|
+
def get_groups()
|
9
|
+
# Get a list of all files in the directory
|
10
|
+
files = Dir.glob(File.join(PATH_ALERTS, RGX_FILE_REF))
|
11
|
+
|
12
|
+
# Sort files by date extracted from the filename, newest first
|
13
|
+
sorted_files = files.sort_by do |file|
|
14
|
+
match = File.basename(file).match(RGX_FILE_TIME)
|
15
|
+
match ? match[0] : ''
|
16
|
+
end.reverse
|
17
|
+
|
18
|
+
# Initialize groups as an empty array
|
19
|
+
groups = []
|
20
|
+
current_group = [] # Initialize current group as an empty array
|
21
|
+
current_group_size = 0
|
22
|
+
|
23
|
+
# Iterate through the sorted files
|
24
|
+
sorted_files.each do |file|
|
25
|
+
# Get the file size in bytes
|
26
|
+
file_size = File.size(file)
|
27
|
+
|
28
|
+
# Check if adding the file exceeds the group size limit
|
29
|
+
if current_group_size + file_size > GROUP_SIZE # Convert 500MB to bytes
|
30
|
+
# Add the current group to the groups array
|
31
|
+
groups << current_group
|
32
|
+
|
33
|
+
# Create a new current group and reset current group size
|
34
|
+
current_group = []
|
35
|
+
current_group_size = 0
|
36
|
+
end
|
37
|
+
|
38
|
+
# Add the file to the current group
|
39
|
+
current_group << file
|
40
|
+
current_group_size += file_size
|
41
|
+
end
|
42
|
+
groups << current_group
|
43
|
+
return groups
|
44
|
+
end
|
45
|
+
end
|
46
|
+
|
data/lib/lookingback.sh
ADDED
@@ -0,0 +1,10 @@
|
|
1
|
+
#!/bin/sh
|
2
|
+
|
3
|
+
|
4
|
+
temp_file=$(mktemp)
|
5
|
+
alert_path=`grep alerts_path /etc/pdnssoc/pdnssoc.conf | awk -F "\"" '{print $4}'`
|
6
|
+
|
7
|
+
zgrep -F -f /etc/td-agent/misp_domains.txt /var/log/td-agent/queries.*.log* > ${temp_file}
|
8
|
+
grep -w -f /etc/td-agent/misp_domains.txt ${temp_file} >> ${alert_path}pdnssoc-pastlog.log
|
9
|
+
|
10
|
+
rm -f ${temp_file}
|
data/lib/misp_refresh.sh
ADDED
@@ -0,0 +1,26 @@
|
|
1
|
+
#!/bin/sh
|
2
|
+
#
|
3
|
+
# CRON file for pDNSSOC
|
4
|
+
|
5
|
+
# Downloading MISP events
|
6
|
+
|
7
|
+
pull_misp_data(){
|
8
|
+
jq --raw-output '.misp_servers[] | "\(.api_key)|\(.domain)|\(.'$1')"' /etc/pdnssoc/pdnssoc.conf |
|
9
|
+
while IFS="|" read -r api_key domain parameter; do
|
10
|
+
printf -v mycurl 'curl -k -qsS --header "Authorization: %s" "%s%s"\n' "$api_key" "https://$domain/" "$parameter"
|
11
|
+
raw_data=$( eval ${mycurl})
|
12
|
+
sorted_data=$(printf "$raw_data\n"|sort -u)
|
13
|
+
echo "$sorted_data\n" > /etc/td-agent/$2
|
14
|
+
|
15
|
+
done
|
16
|
+
}
|
17
|
+
|
18
|
+
pull_misp_data "parameter_domains" "misp_domains.txt"
|
19
|
+
pull_misp_data "parameter_ips" "misp_ips.txt"
|
20
|
+
|
21
|
+
# Add something below to send an alert to the security contact if misp_domains.txt is empty
|
22
|
+
|
23
|
+
|
24
|
+
# Reload the configuration file for fluentd
|
25
|
+
|
26
|
+
kill -1 `cat /var/run/td-agent/td-agent.pid`
|
data/lib/pdnssoc.rb
ADDED
data/lib/post_install.rb
ADDED
@@ -0,0 +1,16 @@
|
|
1
|
+
puts "PostInstallScript loaded successfully."
|
2
|
+
|
3
|
+
class PostInstallScript
|
4
|
+
def self.run
|
5
|
+
puts "PostInstallScript.run method executed."
|
6
|
+
# Your post-installation script logic here
|
7
|
+
# Will read and execute the tasks defined in tasks/tasks_install.rake
|
8
|
+
if ENV['SKIP_POST_INSTALL_HOOK'].nil?
|
9
|
+
puts "Running post-installation setup..."
|
10
|
+
Rake::Task["rake_install:install"].invoke
|
11
|
+
puts "Post-installation setup completed."
|
12
|
+
end
|
13
|
+
end
|
14
|
+
end
|
15
|
+
|
16
|
+
load File.join(File.dirname(__FILE__), 'tasks', 'tasks_install.rake')
|
data/lib/trigger.rb
ADDED
@@ -0,0 +1,124 @@
|
|
1
|
+
require_relative 'configalerts'
|
2
|
+
require_relative 'email'
|
3
|
+
require_relative 'alerts'
|
4
|
+
require_relative 'constants'
|
5
|
+
require_relative 'inputdata'
|
6
|
+
require "time"
|
7
|
+
|
8
|
+
class Trigger
|
9
|
+
include ConfigAlerts
|
10
|
+
include InputData
|
11
|
+
|
12
|
+
def initialize()
|
13
|
+
super()
|
14
|
+
@@alerts_found = {}
|
15
|
+
end
|
16
|
+
|
17
|
+
def delete_logs(processed_logs)
|
18
|
+
for filename in processed_logs
|
19
|
+
@@log_sys.debug("Deleting: " + filename)
|
20
|
+
File.delete(filename) if File.exist?(filename)
|
21
|
+
end
|
22
|
+
end
|
23
|
+
|
24
|
+
def get_email_client(ip_client)
|
25
|
+
# TODO: Given a boolean, skip any event from clients that are not on the config file
|
26
|
+
|
27
|
+
# If we have email for that client we will contact them directly
|
28
|
+
if @@pdns_config.keys.include?(ip_client) and @@pdns_config[ip_client].keys.include?("email") and ! @@pdns_config[ip_client]["email"].empty?
|
29
|
+
email_client = @@pdns_config[ip_client]["email"]
|
30
|
+
else
|
31
|
+
# If we don't have the email of the clientb we send the email to the default security contact
|
32
|
+
email_client = @@email_config["to"]
|
33
|
+
end
|
34
|
+
return email_client
|
35
|
+
end
|
36
|
+
|
37
|
+
def check_alert_keys(json_log_read)
|
38
|
+
required_keys = ["query", "client", "date"]
|
39
|
+
all_required_keys = required_keys.all? { |string| json_log_read.key?(string) }
|
40
|
+
return all_required_keys
|
41
|
+
end
|
42
|
+
|
43
|
+
def study_ioc(list_iocs, ioc_detected, type_ioc, ip_client, date)
|
44
|
+
skip_iocs = []
|
45
|
+
# Domains that are legit or that do not have information in MISP will be skipped
|
46
|
+
begin
|
47
|
+
if ! skip_iocs.include?(ioc_detected)
|
48
|
+
# If it is a malicious domain
|
49
|
+
if list_iocs.include?(ioc_detected)
|
50
|
+
email_client = get_email_client(ip_client)
|
51
|
+
# If it was already analyzed -> +1
|
52
|
+
if ! @@alerts_found.empty? and @@alerts_found.include?(email_client) and \
|
53
|
+
@@alerts_found[email_client].include?(ioc_detected)
|
54
|
+
@@alerts_found[email_client][ioc_detected]["count"] += 1
|
55
|
+
else
|
56
|
+
# We don't have information so we will query MISP
|
57
|
+
alert = Alert.new()
|
58
|
+
@result_ioc = alert.parse_log(ioc_detected, type_ioc, date, ip_client)
|
59
|
+
end
|
60
|
+
if @result_ioc.empty?
|
61
|
+
# Although it is a malicious domain it doesn't have any data in MISP -> skip next time
|
62
|
+
skip_iocs.append(ioc_detected)
|
63
|
+
else
|
64
|
+
# We have found data in MISP about this domain -> we will report it to the right client
|
65
|
+
if ! @@alerts_found.include?(email_client)
|
66
|
+
@@alerts_found[email_client] = {}
|
67
|
+
end
|
68
|
+
@@alerts_found[email_client][ioc_detected] = @result_ioc
|
69
|
+
@@log_alerts.info(@result_ioc)
|
70
|
+
end
|
71
|
+
else
|
72
|
+
skip_iocs.append(ioc_detected)
|
73
|
+
end
|
74
|
+
end
|
75
|
+
rescue Exception => e
|
76
|
+
raise Exception, TRIGGER_ERROR % [e:e]
|
77
|
+
end
|
78
|
+
end
|
79
|
+
|
80
|
+
def analyze_all_iocs(group_of_files)
|
81
|
+
all_alerts = {}
|
82
|
+
# Iterate over all files inside the group
|
83
|
+
for filename in group_of_files
|
84
|
+
# Read each line opf the file that represents one log entry
|
85
|
+
File.readlines(filename).each do |line|
|
86
|
+
json_log = JSON.parse(line)
|
87
|
+
# If the data is not complete -> skip log
|
88
|
+
if ! check_alert_keys(json_log)
|
89
|
+
@@log_sys.error(MISSING_KEY_ALERT % line)
|
90
|
+
next
|
91
|
+
end
|
92
|
+
ip_client = json_log["client"]
|
93
|
+
date = Time.parse(json_log["date"]).to_i
|
94
|
+
# If in addition of the domain, the resolved IP is provided, it will be analyzed
|
95
|
+
if json_log.keys.include?("answer")
|
96
|
+
study_ioc(@@bad_ips, json_log["answer"], "ip", ip_client, date)
|
97
|
+
end
|
98
|
+
# Analyze if the domain is malicious
|
99
|
+
study_ioc(@@bad_domains, json_log["query"], "domain", ip_client, date)
|
100
|
+
end
|
101
|
+
end
|
102
|
+
end
|
103
|
+
|
104
|
+
def run()
|
105
|
+
# The files to process are gouped so in case of failure at least some of them will be processed
|
106
|
+
groups = get_groups()
|
107
|
+
for group_of_files in groups
|
108
|
+
# Analyze all IOCs from the logs
|
109
|
+
analyze_all_iocs(group_of_files)
|
110
|
+
if @@alerts_found.empty?
|
111
|
+
@@log_sys.debug("No alerts found!")
|
112
|
+
else
|
113
|
+
# We will send an alert to each client (with an email on the config file)
|
114
|
+
@@alerts_found.each do |email_client, client_data|
|
115
|
+
email = Email.new()
|
116
|
+
email.send_email(email_client, client_data)
|
117
|
+
end
|
118
|
+
end
|
119
|
+
# If the send_email is not successful the logs will not be deleted
|
120
|
+
delete_logs(group_of_files)
|
121
|
+
end
|
122
|
+
end
|
123
|
+
end
|
124
|
+
|
metadata
ADDED
@@ -0,0 +1,70 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: pdnssoc
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.1.3
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Pau Cutrina
|
8
|
+
- Romain Wartel
|
9
|
+
- Christos Arvanitis
|
10
|
+
autorequire:
|
11
|
+
bindir: bin
|
12
|
+
cert_chain: []
|
13
|
+
date: 2023-08-07 00:00:00.000000000 Z
|
14
|
+
dependencies: []
|
15
|
+
description: pDNS correlation with MISP
|
16
|
+
email:
|
17
|
+
- admin@safer-trust.org
|
18
|
+
executables: []
|
19
|
+
extensions: []
|
20
|
+
extra_rdoc_files: []
|
21
|
+
files:
|
22
|
+
- config/notification_email.html
|
23
|
+
- config/pdnssoc.conf
|
24
|
+
- config/td-agent.conf
|
25
|
+
- lib/alerts.rb
|
26
|
+
- lib/configalerts.rb
|
27
|
+
- lib/constants.rb
|
28
|
+
- lib/email.rb
|
29
|
+
- lib/inputdata.rb
|
30
|
+
- lib/lookingback.sh
|
31
|
+
- lib/misp_refresh.sh
|
32
|
+
- lib/pdnssoc.rb
|
33
|
+
- lib/post_install.rb
|
34
|
+
- lib/trigger.rb
|
35
|
+
- timers/lookingback.service
|
36
|
+
- timers/lookingback.timer
|
37
|
+
- timers/misp_refresh.service
|
38
|
+
- timers/misp_refresh.timer
|
39
|
+
- timers/pdnssoc.service
|
40
|
+
- timers/pdnssoc.timer
|
41
|
+
homepage: https://github.com/CERN-CERT/pDNSSOC/
|
42
|
+
licenses:
|
43
|
+
- MIT
|
44
|
+
metadata:
|
45
|
+
rubygems_mfa_required: 'true'
|
46
|
+
source_code_uri: https://github.com/CERN-CERT/pDNSSOC
|
47
|
+
changelog_uri: https://github.com/CERN-CERT/pDNSSOC/blob/master/CHANGELOG.md
|
48
|
+
homepage_uri: https://github.com/CERN-CERT/pDNSSOC
|
49
|
+
github_repo: ssh://github.com/CERN-CERT/pDNSSOC
|
50
|
+
post_install_message: pDNSSOC has been installed successfuly!
|
51
|
+
rdoc_options: []
|
52
|
+
require_paths:
|
53
|
+
- lib
|
54
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
55
|
+
requirements:
|
56
|
+
- - ">="
|
57
|
+
- !ruby/object:Gem::Version
|
58
|
+
version: 2.5.0
|
59
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
60
|
+
requirements:
|
61
|
+
- - ">="
|
62
|
+
- !ruby/object:Gem::Version
|
63
|
+
version: '0'
|
64
|
+
requirements:
|
65
|
+
- Ruby (>= 2.5.0)
|
66
|
+
rubygems_version: 3.2.33
|
67
|
+
signing_key:
|
68
|
+
specification_version: 4
|
69
|
+
summary: pDNS correlation with MISP
|
70
|
+
test_files: []
|