ukemi 0.1.0

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml ADDED
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA256:
3
+ metadata.gz: a77579f014e97cc048e95dc5488e228ee19baba62d5074dbc9bfcb5c75f9d568
4
+ data.tar.gz: e156b9c6de521ea49d7a69725c724176a5d1739deee7bba32aef4e662c64ddae
5
+ SHA512:
6
+ metadata.gz: f4e3eb6f8f4dd2223ba2eed1846ff8d95ee8d14913f5f6e682269489ee9348fea3af858c9bf849bac4d612a42a0a3c1c7bae3b27dde3461b1104c3e0e3752964
7
+ data.tar.gz: 34467c49986dc11f2ac4fae646c1d96419ebe199838fe20a8f859c1619137ce2f3d0bcfc11e1f356545e14f63a6b5ff81b7ad9f39b206b70469959a8ca9a5986
data/.gitignore ADDED
@@ -0,0 +1,59 @@
1
+ *.gem
2
+ *.rbc
3
+ /.config
4
+ /coverage/
5
+ /InstalledFiles
6
+ /pkg/
7
+ /spec/reports/
8
+ /spec/examples.txt
9
+ /test/tmp/
10
+ /test/version_tmp/
11
+ /tmp/
12
+
13
+ # Used by dotenv library to load environment variables.
14
+ # .env
15
+
16
+ # Ignore Byebug command history file.
17
+ .byebug_history
18
+
19
+ ## Specific to RubyMotion:
20
+ .dat*
21
+ .repl_history
22
+ build/
23
+ *.bridgesupport
24
+ build-iPhoneOS/
25
+ build-iPhoneSimulator/
26
+
27
+ ## Specific to RubyMotion (use of CocoaPods):
28
+ #
29
+ # We recommend against adding the Pods directory to your .gitignore. However
30
+ # you should judge for yourself, the pros and cons are mentioned at:
31
+ # https://guides.cocoapods.org/using/using-cocoapods.html#should-i-check-the-pods-directory-into-source-control
32
+ #
33
+ # vendor/Pods/
34
+
35
+ ## Documentation cache and generated files:
36
+ /.yardoc/
37
+ /_yardoc/
38
+ /doc/
39
+ /rdoc/
40
+
41
+ ## Environment normalization:
42
+ /.bundle/
43
+ /vendor/bundle
44
+ /lib/bundler/man/
45
+
46
+ # for a library or gem, you might want to ignore these files since the code is
47
+ # intended to run in multiple environments; otherwise, check them in:
48
+ Gemfile.lock
49
+ .ruby-version
50
+ .ruby-gemset
51
+
52
+ # unless supporting rvm < 1.11.0 or doing something fancy, ignore this:
53
+ .rvmrc
54
+
55
+ # Used by RuboCop. Remote config files pulled in from inherit_from directive.
56
+ # .rubocop-https?--*
57
+
58
+ # RSpec
59
+ .rspec_status
data/.rspec ADDED
@@ -0,0 +1,3 @@
1
+ --format documentation
2
+ --color
3
+ --require spec_helper
data/.travis.yml ADDED
@@ -0,0 +1,6 @@
1
+ ---
2
+ language: ruby
3
+ cache: bundler
4
+ rvm:
5
+ - 2.7
6
+ before_install: gem install bundler -v 2.1
data/Gemfile ADDED
@@ -0,0 +1,6 @@
1
+ # frozen_string_literal: true
2
+
3
+ source "https://rubygems.org"
4
+
5
+ # Specify your gem's dependencies in ukemi.gemspec
6
+ gemspec
data/LICENSE ADDED
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2020 Manabu Niseki
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
data/README.md ADDED
@@ -0,0 +1,423 @@
1
+ # ukemi
2
+
3
+ [![Build Status](https://travis-ci.com/ninoseki/ukemi.svg?branch=master)](https://travis-ci.com/ninoseki/ukemi)
4
+ [![Coverage Status](https://coveralls.io/repos/github/ninoseki/ukemi/badge.svg?branch=master)](https://coveralls.io/github/ninoseki/ukemi?branch=master)
5
+ [![CodeFactor](https://www.codefactor.io/repository/github/ninoseki/ukemi/badge)](https://www.codefactor.io/repository/github/ninoseki/ukemi)
6
+
7
+ Ukemi is a CIL tool for querying passive DNS services.
8
+
9
+ It supports the following services.
10
+
11
+ - [CIRCL passive DNS](https://www.circl.lu/services/passive-dns/)
12
+ - [PassiveTotal](https://community.riskiq.com/)
13
+ - [SecurityTrails](https://securitytrails.com/)
14
+ - [VirusTotal](http://virustotal.com)
15
+
16
+ It outputs passive DNS resolutions as JSON.
17
+
18
+ ## Instalattion
19
+
20
+ ```bash
21
+ gem install ukemi
22
+ ```
23
+
24
+ ## Configuration
25
+
26
+ Configuration is done via environment variables.
27
+
28
+ | Key | Desc. |
29
+ |------------------------|----------------------------|
30
+ | CIRCL_PASSIVE_PASSWORD | CIRCL passive DNS password |
31
+ | CIRCL_PASSIVE_USERNAME | CIRCL passive DNS username |
32
+ | PASSIVETOTAL_API_KEY | PassiveTotal API key |
33
+ | PASSIVETOTAL_USERNAME | PassiveTotal username |
34
+ | SECURITYTRAILS_API_KEY | SecurityTrails API key |
35
+ | VIRUSTOTAL_API_KEY | VirusTotal API key |
36
+
37
+ ## Usage
38
+
39
+ ```bash
40
+ $ ukemi
41
+ Commands:
42
+ ukemi help [COMMAND] # Describe available commands or one specific command
43
+ ukemi lookup [IP|DOMAIN] # Lookup passive DNS services
44
+
45
+ $ ukemi help looup
46
+ Usage:
47
+ ukemi lookup [IP|DOMAIN]
48
+
49
+ Lookup passive DNS services
50
+ ```
51
+
52
+ ```bash
53
+ $ ukemi lookup circl.lu
54
+ {
55
+ "149.13.33.14": [
56
+ {
57
+ "firtst_seen": "2016-10-07",
58
+ "last_seen": "2018-10-26",
59
+ "source": "CIRCL"
60
+ },
61
+ {
62
+ "firtst_seen": "2017-05-26",
63
+ "last_seen": "2020-03-15",
64
+ "source": "SecurityTrails"
65
+ },
66
+ {
67
+ "firtst_seen": "2019-12-04",
68
+ "last_seen": "2019-12-04",
69
+ "source": "VirusTotal"
70
+ }
71
+ ],
72
+ "149.13.33.4": [
73
+ {
74
+ "firtst_seen": "2011-03-08",
75
+ "last_seen": "2012-02-13",
76
+ "source": "CIRCL"
77
+ },
78
+ {
79
+ "firtst_seen": "2013-07-30",
80
+ "last_seen": "2013-07-30",
81
+ "source": "VirusTotal"
82
+ }
83
+ ],
84
+ "194.154.205.24": [
85
+ {
86
+ "firtst_seen": "2011-03-03",
87
+ "last_seen": "2011-03-03",
88
+ "source": "CIRCL"
89
+ }
90
+ ]
91
+ }
92
+
93
+ $ ukemi lookup 195.123.226.243
94
+ {
95
+ "liankt.club": [
96
+ {
97
+ "firtst_seen": "2020-02-15",
98
+ "last_seen": "2020-03-13",
99
+ "source": "PassiveTotal"
100
+ },
101
+ {
102
+ "firtst_seen": "2020-02-16",
103
+ "last_seen": "2020-02-16",
104
+ "source": "VirusTotal"
105
+ }
106
+ ],
107
+ "weidt.club": [
108
+ {
109
+ "firtst_seen": "2020-03-12",
110
+ "last_seen": "2020-03-12",
111
+ "source": "PassiveTotal"
112
+ }
113
+ ],
114
+ "jikt.club": [
115
+ {
116
+ "firtst_seen": "2020-03-04",
117
+ "last_seen": "2020-03-12",
118
+ "source": "PassiveTotal"
119
+ },
120
+ {
121
+ "firtst_seen": "2020-03-05",
122
+ "last_seen": "2020-03-05",
123
+ "source": "VirusTotal"
124
+ }
125
+ ],
126
+ "biesi.club": [
127
+ {
128
+ "firtst_seen": "2020-02-15",
129
+ "last_seen": "2020-03-12",
130
+ "source": "PassiveTotal"
131
+ },
132
+ {
133
+ "firtst_seen": "2020-02-20",
134
+ "last_seen": "2020-02-20",
135
+ "source": "VirusTotal"
136
+ }
137
+ ],
138
+ "kaikt.club": [
139
+ {
140
+ "firtst_seen": "2020-02-15",
141
+ "last_seen": "2020-03-12",
142
+ "source": "PassiveTotal"
143
+ },
144
+ {
145
+ "firtst_seen": "2020-02-21",
146
+ "last_seen": "2020-02-21",
147
+ "source": "VirusTotal"
148
+ }
149
+ ],
150
+ "zhaokt.club": [
151
+ {
152
+ "firtst_seen": "2020-02-15",
153
+ "last_seen": "2020-03-11",
154
+ "source": "PassiveTotal"
155
+ },
156
+ {
157
+ "firtst_seen": "2020-02-18",
158
+ "last_seen": "2020-02-18",
159
+ "source": "VirusTotal"
160
+ }
161
+ ],
162
+ "yangdt.club": [
163
+ {
164
+ "firtst_seen": "2020-02-26",
165
+ "last_seen": "2020-03-10",
166
+ "source": "PassiveTotal"
167
+ },
168
+ {
169
+ "firtst_seen": "2020-02-27",
170
+ "last_seen": "2020-02-27",
171
+ "source": "VirusTotal"
172
+ }
173
+ ],
174
+ "jinkt.club": [
175
+ {
176
+ "firtst_seen": "2020-02-21",
177
+ "last_seen": "2020-03-10",
178
+ "source": "PassiveTotal"
179
+ },
180
+ {
181
+ "firtst_seen": "2020-02-22",
182
+ "last_seen": "2020-02-22",
183
+ "source": "VirusTotal"
184
+ }
185
+ ],
186
+ "taokt.club": [
187
+ {
188
+ "firtst_seen": "2020-03-10",
189
+ "last_seen": "2020-03-10",
190
+ "source": "PassiveTotal"
191
+ }
192
+ ],
193
+ "xinkt.club": [
194
+ {
195
+ "firtst_seen": "2020-02-17",
196
+ "last_seen": "2020-03-09",
197
+ "source": "PassiveTotal"
198
+ },
199
+ {
200
+ "firtst_seen": "2020-02-19",
201
+ "last_seen": "2020-02-19",
202
+ "source": "VirusTotal"
203
+ }
204
+ ],
205
+ "mail.realty-advertising.ru": [
206
+ {
207
+ "firtst_seen": "2019-11-08",
208
+ "last_seen": "2020-03-09",
209
+ "source": "PassiveTotal"
210
+ }
211
+ ],
212
+ "realty-advertising.ru": [
213
+ {
214
+ "firtst_seen": "2019-11-08",
215
+ "last_seen": "2020-03-06",
216
+ "source": "PassiveTotal"
217
+ }
218
+ ],
219
+ "ns1.realty-advertising.ru": [
220
+ {
221
+ "firtst_seen": "2019-12-02",
222
+ "last_seen": "2020-03-04",
223
+ "source": "PassiveTotal"
224
+ }
225
+ ],
226
+ "ns2.realty-advertising.ru": [
227
+ {
228
+ "firtst_seen": "2019-12-04",
229
+ "last_seen": "2020-03-04",
230
+ "source": "PassiveTotal"
231
+ }
232
+ ],
233
+ "xiankt.club": [
234
+ {
235
+ "firtst_seen": "2020-02-15",
236
+ "last_seen": "2020-03-03",
237
+ "source": "PassiveTotal"
238
+ },
239
+ {
240
+ "firtst_seen": "2020-02-16",
241
+ "last_seen": "2020-02-16",
242
+ "source": "VirusTotal"
243
+ }
244
+ ],
245
+ "nittsu-si.com": [
246
+ {
247
+ "firtst_seen": "2020-02-15",
248
+ "last_seen": "2020-03-03",
249
+ "source": "PassiveTotal"
250
+ },
251
+ {
252
+ "firtst_seen": "2020-02-21",
253
+ "last_seen": "2020-02-21",
254
+ "source": "VirusTotal"
255
+ }
256
+ ],
257
+ "mailer.realty-advertising.ru": [
258
+ {
259
+ "firtst_seen": "2020-02-23",
260
+ "last_seen": "2020-02-23",
261
+ "source": "PassiveTotal"
262
+ }
263
+ ],
264
+ "mail7.realty-advertising.ru": [
265
+ {
266
+ "firtst_seen": "2020-02-23",
267
+ "last_seen": "2020-02-23",
268
+ "source": "PassiveTotal"
269
+ }
270
+ ],
271
+ "zimbra.realty-advertising.ru": [
272
+ {
273
+ "firtst_seen": "2020-02-23",
274
+ "last_seen": "2020-02-23",
275
+ "source": "PassiveTotal"
276
+ }
277
+ ],
278
+ "relay2.realty-advertising.ru": [
279
+ {
280
+ "firtst_seen": "2020-02-23",
281
+ "last_seen": "2020-02-23",
282
+ "source": "PassiveTotal"
283
+ }
284
+ ],
285
+ "sniper.realty-advertising.ru": [
286
+ {
287
+ "firtst_seen": "2020-02-22",
288
+ "last_seen": "2020-02-22",
289
+ "source": "PassiveTotal"
290
+ }
291
+ ],
292
+ "mailx.realty-advertising.ru": [
293
+ {
294
+ "firtst_seen": "2020-02-22",
295
+ "last_seen": "2020-02-22",
296
+ "source": "PassiveTotal"
297
+ }
298
+ ],
299
+ "send.realty-advertising.ru": [
300
+ {
301
+ "firtst_seen": "2020-02-22",
302
+ "last_seen": "2020-02-22",
303
+ "source": "PassiveTotal"
304
+ }
305
+ ],
306
+ "mta.realty-advertising.ru": [
307
+ {
308
+ "firtst_seen": "2020-02-22",
309
+ "last_seen": "2020-02-22",
310
+ "source": "PassiveTotal"
311
+ }
312
+ ],
313
+ "home.realty-advertising.ru": [
314
+ {
315
+ "firtst_seen": "2020-02-22",
316
+ "last_seen": "2020-02-22",
317
+ "source": "PassiveTotal"
318
+ }
319
+ ],
320
+ "pbrand.realty-advertising.ru": [
321
+ {
322
+ "firtst_seen": "2020-02-22",
323
+ "last_seen": "2020-02-22",
324
+ "source": "PassiveTotal"
325
+ }
326
+ ],
327
+ "smtpauth.realty-advertising.ru": [
328
+ {
329
+ "firtst_seen": "2020-02-22",
330
+ "last_seen": "2020-02-22",
331
+ "source": "PassiveTotal"
332
+ }
333
+ ],
334
+ "gate.realty-advertising.ru": [
335
+ {
336
+ "firtst_seen": "2020-02-21",
337
+ "last_seen": "2020-02-21",
338
+ "source": "PassiveTotal"
339
+ }
340
+ ],
341
+ "mx02.realty-advertising.ru": [
342
+ {
343
+ "firtst_seen": "2020-02-21",
344
+ "last_seen": "2020-02-21",
345
+ "source": "PassiveTotal"
346
+ }
347
+ ],
348
+ "outmail.realty-advertising.ru": [
349
+ {
350
+ "firtst_seen": "2020-02-21",
351
+ "last_seen": "2020-02-21",
352
+ "source": "PassiveTotal"
353
+ }
354
+ ],
355
+ "exchange.realty-advertising.ru": [
356
+ {
357
+ "firtst_seen": "2020-02-21",
358
+ "last_seen": "2020-02-21",
359
+ "source": "PassiveTotal"
360
+ }
361
+ ],
362
+ "ms.realty-advertising.ru": [
363
+ {
364
+ "firtst_seen": "2020-02-21",
365
+ "last_seen": "2020-02-21",
366
+ "source": "PassiveTotal"
367
+ }
368
+ ],
369
+ "owa.realty-advertising.ru": [
370
+ {
371
+ "firtst_seen": "2020-02-20",
372
+ "last_seen": "2020-02-20",
373
+ "source": "PassiveTotal"
374
+ }
375
+ ],
376
+ "mail8.realty-advertising.ru": [
377
+ {
378
+ "firtst_seen": "2020-02-20",
379
+ "last_seen": "2020-02-20",
380
+ "source": "PassiveTotal"
381
+ }
382
+ ],
383
+ "mta-sts.realty-advertising.ru": [
384
+ {
385
+ "firtst_seen": "2019-11-11",
386
+ "last_seen": "2020-02-08",
387
+ "source": "PassiveTotal"
388
+ }
389
+ ],
390
+ "mail02.realty-advertising.ru": [
391
+ {
392
+ "firtst_seen": "2020-01-18",
393
+ "last_seen": "2020-01-18",
394
+ "source": "PassiveTotal"
395
+ }
396
+ ],
397
+ "www.realty-advertising.ru": [
398
+ {
399
+ "firtst_seen": "2019-11-08",
400
+ "last_seen": "2019-11-12",
401
+ "source": "PassiveTotal"
402
+ }
403
+ ],
404
+ "ln-048.rd-00003024.id-11744955.v0.tun.vpnoverdns.com": [
405
+ {
406
+ "firtst_seen": "2017-04-06",
407
+ "last_seen": "2017-04-06",
408
+ "source": "PassiveTotal"
409
+ }
410
+ ],
411
+ "mnen6k7g.info": [
412
+ {
413
+ "firtst_seen": "2010-10-28",
414
+ "last_seen": "2010-10-28",
415
+ "source": "PassiveTotal"
416
+ }
417
+ ]
418
+ }
419
+ ```
420
+
421
+ ## License
422
+
423
+ The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).
data/Rakefile ADDED
@@ -0,0 +1,8 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "bundler/gem_tasks"
4
+ require "rspec/core/rake_task"
5
+
6
+ RSpec::Core::RakeTask.new(:spec)
7
+
8
+ task default: :spec
data/bin/console ADDED
@@ -0,0 +1,14 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ require "bundler/setup"
4
+ require "ukemi"
5
+
6
+ # You can add fixtures and/or initialization code here to make experimenting
7
+ # with your gem easier. You can also use a different console, if you like.
8
+
9
+ # (If you use this, don't forget to add pry to your Gemfile!)
10
+ # require "pry"
11
+ # Pry.start
12
+
13
+ require "irb"
14
+ IRB.start(__FILE__)
data/bin/setup ADDED
@@ -0,0 +1,8 @@
1
+ #!/usr/bin/env bash
2
+ set -euo pipefail
3
+ IFS=$'\n\t'
4
+ set -vx
5
+
6
+ bundle install
7
+
8
+ # Do any other automated setup that you need to do here
data/exe/ukemi ADDED
@@ -0,0 +1,8 @@
1
+ #!/usr/bin/env ruby
2
+ # frozen_string_literal: true
3
+
4
+ $LOAD_PATH.unshift("#{__dir__}/../lib")
5
+
6
+ require "ukemi"
7
+
8
+ Ukemi::CLI.start
data/lib/ukemi/cli.rb ADDED
@@ -0,0 +1,21 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "json"
4
+ require "thor"
5
+
6
+ module Ukemi
7
+ class CLI < Thor
8
+ desc "lookup [IP|DOMAIN]", "Lookup passive DNS services"
9
+ def lookup(data)
10
+ data = refang(data)
11
+ result = Moderator.lookup(data)
12
+ puts JSON.pretty_generate(result)
13
+ end
14
+
15
+ no_commands do
16
+ def refang(data)
17
+ data.gsub("[.]", ".").gsub("(.)", ".")
18
+ end
19
+ end
20
+ end
21
+ end
@@ -0,0 +1,5 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Ukemi
4
+ class Error < StandardError; end
5
+ end
@@ -0,0 +1,49 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "parallel"
4
+ require "time"
5
+
6
+ module Ukemi
7
+ class Moderator
8
+ def lookup(data)
9
+ records = Parallel.map(Ukemi.services) do |klass|
10
+ service = klass.new
11
+ next unless service.configurated?
12
+
13
+ begin
14
+ service.lookup data
15
+ rescue ::PassiveTotal::Error, ::VirusTotal::Error, ::SecurityTrails::Error, PassiveCIRCL::Error
16
+ nil
17
+ end
18
+ end.flatten.compact
19
+
20
+ memo = Hash.new { |h, k| h[k] = [] }
21
+ records.each do |record|
22
+ memo[record.data] << {
23
+ firtst_seen: record.first_seen,
24
+ last_seen: record.last_seen,
25
+ source: record.source,
26
+ }
27
+ end
28
+
29
+ memo.sort_by do |_key, array|
30
+ last_seens = array.map do |record|
31
+ parse_to_unixtime record.dig(:last_seen)
32
+ end
33
+ -last_seens.max
34
+ end.to_h
35
+ end
36
+
37
+ def parse_to_unixtime(date)
38
+ return -1 unless date
39
+
40
+ Time.parse(date).to_i
41
+ end
42
+
43
+ class << self
44
+ def lookup(data)
45
+ new.lookup data
46
+ end
47
+ end
48
+ end
49
+ end
@@ -0,0 +1,26 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Ukemi
4
+ class Record
5
+ attr_reader :data
6
+ attr_reader :first_seen
7
+ attr_reader :last_seen
8
+ attr_reader :source
9
+
10
+ def initialize(data:, first_seen: nil, last_seen: nil, source: nil)
11
+ @data = data
12
+ @first_seen = first_seen
13
+ @last_seen = last_seen
14
+ @source = source
15
+ end
16
+
17
+ def to_h
18
+ {
19
+ data: data,
20
+ first_seen: first_seen,
21
+ last_seen: last_seen,
22
+ source: source
23
+ }
24
+ end
25
+ end
26
+ end
@@ -0,0 +1,43 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "passive_circl"
4
+
5
+ module Ukemi
6
+ module Services
7
+ class CIRCL < Service
8
+ private
9
+
10
+ def config_keys
11
+ %w(CIRCL_PASSIVE_USERNAME CIRCL_PASSIVE_PASSWORD)
12
+ end
13
+
14
+ def api
15
+ @api ||= PassiveCIRCL::API.new
16
+ end
17
+
18
+ def lookup_by_domain(data)
19
+ passive_dns_lookup(data, "rdata")
20
+ end
21
+
22
+ def lookup_by_ip(data)
23
+ passive_dns_lookup(data, "rrname")
24
+ end
25
+
26
+ def passive_dns_lookup(data, key = nil)
27
+ results = api.dns.query(data)
28
+ results = results.select do |result|
29
+ result.dig("rrtype") == "A"
30
+ end
31
+
32
+ results.map do |result|
33
+ Record.new(
34
+ data: result.dig(key),
35
+ first_seen: Time.at(result.dig("time_first")).to_date.to_s,
36
+ last_seen: Time.at(result.dig("time_last")).to_date.to_s,
37
+ source: name
38
+ )
39
+ end
40
+ end
41
+ end
42
+ end
43
+ end
@@ -0,0 +1,43 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "passivetotal"
4
+
5
+ module Ukemi
6
+ module Services
7
+ class PassiveTotal < Service
8
+ private
9
+
10
+ def api
11
+ @api ||= ::PassiveTotal::API.new
12
+ end
13
+
14
+ def config_keys
15
+ %w(PASSIVETOTAL_USERNAME PASSIVETOTAL_API_KEY)
16
+ end
17
+
18
+ def lookup_by_ip(data)
19
+ res = api.dns.passive(data)
20
+ results = res.dig("results") || []
21
+ convert_to_records results
22
+ end
23
+
24
+ def lookup_by_domain(_data)
25
+ []
26
+ end
27
+
28
+ def convert_to_records(results)
29
+ results.map do |result|
30
+ data = result.dig("resolve")
31
+ first_seen = result.dig("firstSeen").to_s.split.first
32
+ last_seen = result.dig("lastSeen").to_s.split.first
33
+ Record.new(
34
+ data: data,
35
+ first_seen: first_seen,
36
+ last_seen: last_seen,
37
+ source: name
38
+ )
39
+ end
40
+ end
41
+ end
42
+ end
43
+ end
@@ -0,0 +1,57 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "date"
4
+ require "securitytrails"
5
+
6
+ module Ukemi
7
+ module Services
8
+ class SecurityTrails < Service
9
+ private
10
+
11
+ def config_keys
12
+ %w(SECURITYTRAILS_API_KEY)
13
+ end
14
+
15
+ def api
16
+ @api ||= ::SecurityTrails::API.new
17
+ end
18
+
19
+ def lookup_by_ip(data)
20
+ result = api.domains.search( filter: { ipv4: data })
21
+ records = result.dig("records") || []
22
+ hostnames = records.map { |record| record.dig("hostname") }
23
+ hostnames.map do |hostname|
24
+ Record.new(
25
+ data: hostname,
26
+ first_seen: nil,
27
+ last_seen: nil,
28
+ source: name
29
+ )
30
+ end
31
+ end
32
+
33
+ def lookup_by_domain(data)
34
+ result = api.history.get_all_dns_history(data, type: "a")
35
+ records = result.dig("records") || []
36
+ records.map do |record|
37
+ values = record.dig("values") || []
38
+ values.map do |value|
39
+ Record.new(
40
+ data: value.dig("ip"),
41
+ first_seen: record.dig("first_seen"),
42
+ last_seen: record.dig("last_seen"),
43
+ source: name
44
+ )
45
+ end
46
+ end.flatten
47
+ end
48
+
49
+ def extract_attributes(response)
50
+ data = response.dig("data") || []
51
+ data.map do |item|
52
+ item.dig("attributes") || []
53
+ end
54
+ end
55
+ end
56
+ end
57
+ end
@@ -0,0 +1,73 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "addressable/uri"
4
+ require "ipaddr"
5
+ require "public_suffix"
6
+
7
+ module Ukemi
8
+ module Services
9
+ class Service
10
+ def name
11
+ @name ||= self.class.to_s.split("::").last
12
+ end
13
+
14
+ def lookup(data)
15
+ @data = data
16
+
17
+ case type
18
+ when "ip"
19
+ lookup_by_ip data
20
+ when "domain"
21
+ lookup_by_domain data
22
+ else
23
+ raise ArgumentError, "#{data} is not a valid input."
24
+ end
25
+ end
26
+
27
+ def configurated?
28
+ config_keys.all? do |key|
29
+ ENV.key? key
30
+ end
31
+ end
32
+
33
+ class << self
34
+ def inherited(child)
35
+ Ukemi.services << child
36
+ end
37
+ end
38
+
39
+ private
40
+
41
+ def config_keys
42
+ raise NotImplementedError, "You must implement #{self.class}##{__method__}"
43
+ end
44
+
45
+ def lookup_by_ip
46
+ raise NotImplementedError, "You must implement #{self.class}##{__method__}"
47
+ end
48
+
49
+ def lookup_by_domain
50
+ raise NotImplementedError, "You must implement #{self.class}##{__method__}"
51
+ end
52
+
53
+ def ip?
54
+ IPAddr.new @data
55
+ true
56
+ rescue IPAddr::InvalidAddressError => _e
57
+ false
58
+ end
59
+
60
+ def domain?
61
+ uri = Addressable::URI.parse("http://#{@data}")
62
+ uri.host == @data && PublicSuffix.valid?(uri.host)
63
+ rescue Addressable::URI::InvalidURIError => _e
64
+ false
65
+ end
66
+
67
+ def type
68
+ return "ip" if ip?
69
+ return "domain" if domain?
70
+ end
71
+ end
72
+ end
73
+ end
@@ -0,0 +1,58 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "date"
4
+ require "virustotal"
5
+
6
+ module Ukemi
7
+ module Services
8
+ class VirusTotal < Service
9
+ private
10
+
11
+ def config_keys
12
+ %w(VIRUSTOTAL_API_KEY)
13
+ end
14
+
15
+ def api
16
+ @api ||= ::VirusTotal::API.new
17
+ end
18
+
19
+ def lookup_by_ip(data)
20
+ res = api.ip_address.resolutions(data)
21
+ attributes = extract_attributes(res)
22
+ convert_to_records attributes, "host_name"
23
+ end
24
+
25
+ def lookup_by_domain(data)
26
+ res = api.domain.resolutions(data)
27
+ attributes = extract_attributes(res)
28
+ convert_to_records attributes, "ip_address"
29
+ end
30
+
31
+ def extract_attributes(response)
32
+ data = response.dig("data") || []
33
+ data.map do |item|
34
+ item.dig("attributes") || []
35
+ end
36
+ end
37
+
38
+ def convert_to_records(attributes, key = nil)
39
+ memo = Hash.new { |h, k| h[k] = [] }
40
+
41
+ attributes.each do |attribute|
42
+ data = attribute.dig(key)
43
+ date = Time.at(attribute.dig("date")).to_date.to_s
44
+ memo[data] << date
45
+ end
46
+
47
+ memo.keys.map do |data|
48
+ Record.new(
49
+ data: data,
50
+ first_seen: memo[data].min,
51
+ last_seen: memo[data].max,
52
+ source: name
53
+ )
54
+ end
55
+ end
56
+ end
57
+ end
58
+ end
@@ -0,0 +1,5 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Ukemi
4
+ VERSION = "0.1.0"
5
+ end
data/lib/ukemi.rb ADDED
@@ -0,0 +1,31 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "mem"
4
+
5
+ module Ukemi
6
+ class << self
7
+ include Mem
8
+
9
+ def services
10
+ []
11
+ end
12
+ memoize :services
13
+ end
14
+ end
15
+
16
+ require "ukemi/version"
17
+
18
+ require "ukemi/error"
19
+
20
+ require "ukemi/record"
21
+
22
+ require "ukemi/services/service"
23
+
24
+ require "ukemi/services/circl"
25
+ require "ukemi/services/passivetotal"
26
+ require "ukemi/services/securitytrails"
27
+ require "ukemi/services/virustotal"
28
+
29
+ require "ukemi/moderator"
30
+
31
+ require "ukemi/cli"
data/ukemi.gemspec ADDED
@@ -0,0 +1,44 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative 'lib/ukemi/version'
4
+
5
+ Gem::Specification.new do |spec|
6
+ spec.name = "ukemi"
7
+ spec.version = Ukemi::VERSION
8
+ spec.authors = ["Manabu Niseki"]
9
+ spec.email = ["manabu.niseki@gmail.com"]
10
+
11
+ spec.summary = "A CLI tool for querying passive DNS services"
12
+ spec.description = "A CLI tool for querying passive DNS services"
13
+ spec.homepage = "https://github.com/ninoseki/ukemi"
14
+ spec.license = "MIT"
15
+ spec.required_ruby_version = Gem::Requirement.new(">= 2.3.0")
16
+
17
+ spec.metadata["homepage_uri"] = spec.homepage
18
+
19
+ # Specify which files should be added to the gem when it is released.
20
+ # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
21
+ spec.files = Dir.chdir(File.expand_path(__dir__)) do
22
+ `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
23
+ end
24
+ spec.bindir = "exe"
25
+ spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
26
+ spec.require_paths = ["lib"]
27
+
28
+ spec.add_development_dependency "bundler", "~> 2.1"
29
+ spec.add_development_dependency "coveralls", "~> 0.8"
30
+ spec.add_development_dependency "rake", "~> 13.0"
31
+ spec.add_development_dependency "rspec", "~> 3.9"
32
+ spec.add_development_dependency "vcr", "~> 5.0"
33
+ spec.add_development_dependency "webmock", "~> 3.8"
34
+
35
+ spec.add_dependency "addressable", "~> 2.7"
36
+ spec.add_dependency "mem", "~> 0.1"
37
+ spec.add_dependency "parallel", "~> 1.19"
38
+ spec.add_dependency "passive_circl", "~> 0.1"
39
+ spec.add_dependency "passivetotalx", "~> 0.1"
40
+ spec.add_dependency "public_suffix", "~> 4.0"
41
+ spec.add_dependency "securitytrails", "~> 1.0"
42
+ spec.add_dependency "thor", "~> 1.0"
43
+ spec.add_dependency "virustotalx", "~> 1.1"
44
+ end
metadata ADDED
@@ -0,0 +1,277 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: ukemi
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.1.0
5
+ platform: ruby
6
+ authors:
7
+ - Manabu Niseki
8
+ autorequire:
9
+ bindir: exe
10
+ cert_chain: []
11
+ date: 2020-03-15 00:00:00.000000000 Z
12
+ dependencies:
13
+ - !ruby/object:Gem::Dependency
14
+ name: bundler
15
+ requirement: !ruby/object:Gem::Requirement
16
+ requirements:
17
+ - - "~>"
18
+ - !ruby/object:Gem::Version
19
+ version: '2.1'
20
+ type: :development
21
+ prerelease: false
22
+ version_requirements: !ruby/object:Gem::Requirement
23
+ requirements:
24
+ - - "~>"
25
+ - !ruby/object:Gem::Version
26
+ version: '2.1'
27
+ - !ruby/object:Gem::Dependency
28
+ name: coveralls
29
+ requirement: !ruby/object:Gem::Requirement
30
+ requirements:
31
+ - - "~>"
32
+ - !ruby/object:Gem::Version
33
+ version: '0.8'
34
+ type: :development
35
+ prerelease: false
36
+ version_requirements: !ruby/object:Gem::Requirement
37
+ requirements:
38
+ - - "~>"
39
+ - !ruby/object:Gem::Version
40
+ version: '0.8'
41
+ - !ruby/object:Gem::Dependency
42
+ name: rake
43
+ requirement: !ruby/object:Gem::Requirement
44
+ requirements:
45
+ - - "~>"
46
+ - !ruby/object:Gem::Version
47
+ version: '13.0'
48
+ type: :development
49
+ prerelease: false
50
+ version_requirements: !ruby/object:Gem::Requirement
51
+ requirements:
52
+ - - "~>"
53
+ - !ruby/object:Gem::Version
54
+ version: '13.0'
55
+ - !ruby/object:Gem::Dependency
56
+ name: rspec
57
+ requirement: !ruby/object:Gem::Requirement
58
+ requirements:
59
+ - - "~>"
60
+ - !ruby/object:Gem::Version
61
+ version: '3.9'
62
+ type: :development
63
+ prerelease: false
64
+ version_requirements: !ruby/object:Gem::Requirement
65
+ requirements:
66
+ - - "~>"
67
+ - !ruby/object:Gem::Version
68
+ version: '3.9'
69
+ - !ruby/object:Gem::Dependency
70
+ name: vcr
71
+ requirement: !ruby/object:Gem::Requirement
72
+ requirements:
73
+ - - "~>"
74
+ - !ruby/object:Gem::Version
75
+ version: '5.0'
76
+ type: :development
77
+ prerelease: false
78
+ version_requirements: !ruby/object:Gem::Requirement
79
+ requirements:
80
+ - - "~>"
81
+ - !ruby/object:Gem::Version
82
+ version: '5.0'
83
+ - !ruby/object:Gem::Dependency
84
+ name: webmock
85
+ requirement: !ruby/object:Gem::Requirement
86
+ requirements:
87
+ - - "~>"
88
+ - !ruby/object:Gem::Version
89
+ version: '3.8'
90
+ type: :development
91
+ prerelease: false
92
+ version_requirements: !ruby/object:Gem::Requirement
93
+ requirements:
94
+ - - "~>"
95
+ - !ruby/object:Gem::Version
96
+ version: '3.8'
97
+ - !ruby/object:Gem::Dependency
98
+ name: addressable
99
+ requirement: !ruby/object:Gem::Requirement
100
+ requirements:
101
+ - - "~>"
102
+ - !ruby/object:Gem::Version
103
+ version: '2.7'
104
+ type: :runtime
105
+ prerelease: false
106
+ version_requirements: !ruby/object:Gem::Requirement
107
+ requirements:
108
+ - - "~>"
109
+ - !ruby/object:Gem::Version
110
+ version: '2.7'
111
+ - !ruby/object:Gem::Dependency
112
+ name: mem
113
+ requirement: !ruby/object:Gem::Requirement
114
+ requirements:
115
+ - - "~>"
116
+ - !ruby/object:Gem::Version
117
+ version: '0.1'
118
+ type: :runtime
119
+ prerelease: false
120
+ version_requirements: !ruby/object:Gem::Requirement
121
+ requirements:
122
+ - - "~>"
123
+ - !ruby/object:Gem::Version
124
+ version: '0.1'
125
+ - !ruby/object:Gem::Dependency
126
+ name: parallel
127
+ requirement: !ruby/object:Gem::Requirement
128
+ requirements:
129
+ - - "~>"
130
+ - !ruby/object:Gem::Version
131
+ version: '1.19'
132
+ type: :runtime
133
+ prerelease: false
134
+ version_requirements: !ruby/object:Gem::Requirement
135
+ requirements:
136
+ - - "~>"
137
+ - !ruby/object:Gem::Version
138
+ version: '1.19'
139
+ - !ruby/object:Gem::Dependency
140
+ name: passive_circl
141
+ requirement: !ruby/object:Gem::Requirement
142
+ requirements:
143
+ - - "~>"
144
+ - !ruby/object:Gem::Version
145
+ version: '0.1'
146
+ type: :runtime
147
+ prerelease: false
148
+ version_requirements: !ruby/object:Gem::Requirement
149
+ requirements:
150
+ - - "~>"
151
+ - !ruby/object:Gem::Version
152
+ version: '0.1'
153
+ - !ruby/object:Gem::Dependency
154
+ name: passivetotalx
155
+ requirement: !ruby/object:Gem::Requirement
156
+ requirements:
157
+ - - "~>"
158
+ - !ruby/object:Gem::Version
159
+ version: '0.1'
160
+ type: :runtime
161
+ prerelease: false
162
+ version_requirements: !ruby/object:Gem::Requirement
163
+ requirements:
164
+ - - "~>"
165
+ - !ruby/object:Gem::Version
166
+ version: '0.1'
167
+ - !ruby/object:Gem::Dependency
168
+ name: public_suffix
169
+ requirement: !ruby/object:Gem::Requirement
170
+ requirements:
171
+ - - "~>"
172
+ - !ruby/object:Gem::Version
173
+ version: '4.0'
174
+ type: :runtime
175
+ prerelease: false
176
+ version_requirements: !ruby/object:Gem::Requirement
177
+ requirements:
178
+ - - "~>"
179
+ - !ruby/object:Gem::Version
180
+ version: '4.0'
181
+ - !ruby/object:Gem::Dependency
182
+ name: securitytrails
183
+ requirement: !ruby/object:Gem::Requirement
184
+ requirements:
185
+ - - "~>"
186
+ - !ruby/object:Gem::Version
187
+ version: '1.0'
188
+ type: :runtime
189
+ prerelease: false
190
+ version_requirements: !ruby/object:Gem::Requirement
191
+ requirements:
192
+ - - "~>"
193
+ - !ruby/object:Gem::Version
194
+ version: '1.0'
195
+ - !ruby/object:Gem::Dependency
196
+ name: thor
197
+ requirement: !ruby/object:Gem::Requirement
198
+ requirements:
199
+ - - "~>"
200
+ - !ruby/object:Gem::Version
201
+ version: '1.0'
202
+ type: :runtime
203
+ prerelease: false
204
+ version_requirements: !ruby/object:Gem::Requirement
205
+ requirements:
206
+ - - "~>"
207
+ - !ruby/object:Gem::Version
208
+ version: '1.0'
209
+ - !ruby/object:Gem::Dependency
210
+ name: virustotalx
211
+ requirement: !ruby/object:Gem::Requirement
212
+ requirements:
213
+ - - "~>"
214
+ - !ruby/object:Gem::Version
215
+ version: '1.1'
216
+ type: :runtime
217
+ prerelease: false
218
+ version_requirements: !ruby/object:Gem::Requirement
219
+ requirements:
220
+ - - "~>"
221
+ - !ruby/object:Gem::Version
222
+ version: '1.1'
223
+ description: A CLI tool for querying passive DNS services
224
+ email:
225
+ - manabu.niseki@gmail.com
226
+ executables:
227
+ - ukemi
228
+ extensions: []
229
+ extra_rdoc_files: []
230
+ files:
231
+ - ".gitignore"
232
+ - ".rspec"
233
+ - ".travis.yml"
234
+ - Gemfile
235
+ - LICENSE
236
+ - README.md
237
+ - Rakefile
238
+ - bin/console
239
+ - bin/setup
240
+ - exe/ukemi
241
+ - lib/ukemi.rb
242
+ - lib/ukemi/cli.rb
243
+ - lib/ukemi/error.rb
244
+ - lib/ukemi/moderator.rb
245
+ - lib/ukemi/record.rb
246
+ - lib/ukemi/services/circl.rb
247
+ - lib/ukemi/services/passivetotal.rb
248
+ - lib/ukemi/services/securitytrails.rb
249
+ - lib/ukemi/services/service.rb
250
+ - lib/ukemi/services/virustotal.rb
251
+ - lib/ukemi/version.rb
252
+ - ukemi.gemspec
253
+ homepage: https://github.com/ninoseki/ukemi
254
+ licenses:
255
+ - MIT
256
+ metadata:
257
+ homepage_uri: https://github.com/ninoseki/ukemi
258
+ post_install_message:
259
+ rdoc_options: []
260
+ require_paths:
261
+ - lib
262
+ required_ruby_version: !ruby/object:Gem::Requirement
263
+ requirements:
264
+ - - ">="
265
+ - !ruby/object:Gem::Version
266
+ version: 2.3.0
267
+ required_rubygems_version: !ruby/object:Gem::Requirement
268
+ requirements:
269
+ - - ">="
270
+ - !ruby/object:Gem::Version
271
+ version: '0'
272
+ requirements: []
273
+ rubygems_version: 3.1.2
274
+ signing_key:
275
+ specification_version: 4
276
+ summary: A CLI tool for querying passive DNS services
277
+ test_files: []