ukemi 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml ADDED
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA256:
3
+ metadata.gz: a77579f014e97cc048e95dc5488e228ee19baba62d5074dbc9bfcb5c75f9d568
4
+ data.tar.gz: e156b9c6de521ea49d7a69725c724176a5d1739deee7bba32aef4e662c64ddae
5
+ SHA512:
6
+ metadata.gz: f4e3eb6f8f4dd2223ba2eed1846ff8d95ee8d14913f5f6e682269489ee9348fea3af858c9bf849bac4d612a42a0a3c1c7bae3b27dde3461b1104c3e0e3752964
7
+ data.tar.gz: 34467c49986dc11f2ac4fae646c1d96419ebe199838fe20a8f859c1619137ce2f3d0bcfc11e1f356545e14f63a6b5ff81b7ad9f39b206b70469959a8ca9a5986
data/.gitignore ADDED
@@ -0,0 +1,59 @@
1
+ *.gem
2
+ *.rbc
3
+ /.config
4
+ /coverage/
5
+ /InstalledFiles
6
+ /pkg/
7
+ /spec/reports/
8
+ /spec/examples.txt
9
+ /test/tmp/
10
+ /test/version_tmp/
11
+ /tmp/
12
+
13
+ # Used by dotenv library to load environment variables.
14
+ # .env
15
+
16
+ # Ignore Byebug command history file.
17
+ .byebug_history
18
+
19
+ ## Specific to RubyMotion:
20
+ .dat*
21
+ .repl_history
22
+ build/
23
+ *.bridgesupport
24
+ build-iPhoneOS/
25
+ build-iPhoneSimulator/
26
+
27
+ ## Specific to RubyMotion (use of CocoaPods):
28
+ #
29
+ # We recommend against adding the Pods directory to your .gitignore. However
30
+ # you should judge for yourself, the pros and cons are mentioned at:
31
+ # https://guides.cocoapods.org/using/using-cocoapods.html#should-i-check-the-pods-directory-into-source-control
32
+ #
33
+ # vendor/Pods/
34
+
35
+ ## Documentation cache and generated files:
36
+ /.yardoc/
37
+ /_yardoc/
38
+ /doc/
39
+ /rdoc/
40
+
41
+ ## Environment normalization:
42
+ /.bundle/
43
+ /vendor/bundle
44
+ /lib/bundler/man/
45
+
46
+ # for a library or gem, you might want to ignore these files since the code is
47
+ # intended to run in multiple environments; otherwise, check them in:
48
+ Gemfile.lock
49
+ .ruby-version
50
+ .ruby-gemset
51
+
52
+ # unless supporting rvm < 1.11.0 or doing something fancy, ignore this:
53
+ .rvmrc
54
+
55
+ # Used by RuboCop. Remote config files pulled in from inherit_from directive.
56
+ # .rubocop-https?--*
57
+
58
+ # RSpec
59
+ .rspec_status
data/.rspec ADDED
@@ -0,0 +1,3 @@
1
+ --format documentation
2
+ --color
3
+ --require spec_helper
data/.travis.yml ADDED
@@ -0,0 +1,6 @@
1
+ ---
2
+ language: ruby
3
+ cache: bundler
4
+ rvm:
5
+ - 2.7
6
+ before_install: gem install bundler -v 2.1
data/Gemfile ADDED
@@ -0,0 +1,6 @@
1
+ # frozen_string_literal: true
2
+
3
+ source "https://rubygems.org"
4
+
5
+ # Specify your gem's dependencies in ukemi.gemspec
6
+ gemspec
data/LICENSE ADDED
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2020 Manabu Niseki
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
data/README.md ADDED
@@ -0,0 +1,423 @@
1
+ # ukemi
2
+
3
+ [![Build Status](https://travis-ci.com/ninoseki/ukemi.svg?branch=master)](https://travis-ci.com/ninoseki/ukemi)
4
+ [![Coverage Status](https://coveralls.io/repos/github/ninoseki/ukemi/badge.svg?branch=master)](https://coveralls.io/github/ninoseki/ukemi?branch=master)
5
+ [![CodeFactor](https://www.codefactor.io/repository/github/ninoseki/ukemi/badge)](https://www.codefactor.io/repository/github/ninoseki/ukemi)
6
+
7
+ Ukemi is a CIL tool for querying passive DNS services.
8
+
9
+ It supports the following services.
10
+
11
+ - [CIRCL passive DNS](https://www.circl.lu/services/passive-dns/)
12
+ - [PassiveTotal](https://community.riskiq.com/)
13
+ - [SecurityTrails](https://securitytrails.com/)
14
+ - [VirusTotal](http://virustotal.com)
15
+
16
+ It outputs passive DNS resolutions as JSON.
17
+
18
+ ## Instalattion
19
+
20
+ ```bash
21
+ gem install ukemi
22
+ ```
23
+
24
+ ## Configuration
25
+
26
+ Configuration is done via environment variables.
27
+
28
+ | Key | Desc. |
29
+ |------------------------|----------------------------|
30
+ | CIRCL_PASSIVE_PASSWORD | CIRCL passive DNS password |
31
+ | CIRCL_PASSIVE_USERNAME | CIRCL passive DNS username |
32
+ | PASSIVETOTAL_API_KEY | PassiveTotal API key |
33
+ | PASSIVETOTAL_USERNAME | PassiveTotal username |
34
+ | SECURITYTRAILS_API_KEY | SecurityTrails API key |
35
+ | VIRUSTOTAL_API_KEY | VirusTotal API key |
36
+
37
+ ## Usage
38
+
39
+ ```bash
40
+ $ ukemi
41
+ Commands:
42
+ ukemi help [COMMAND] # Describe available commands or one specific command
43
+ ukemi lookup [IP|DOMAIN] # Lookup passive DNS services
44
+
45
+ $ ukemi help looup
46
+ Usage:
47
+ ukemi lookup [IP|DOMAIN]
48
+
49
+ Lookup passive DNS services
50
+ ```
51
+
52
+ ```bash
53
+ $ ukemi lookup circl.lu
54
+ {
55
+ "149.13.33.14": [
56
+ {
57
+ "firtst_seen": "2016-10-07",
58
+ "last_seen": "2018-10-26",
59
+ "source": "CIRCL"
60
+ },
61
+ {
62
+ "firtst_seen": "2017-05-26",
63
+ "last_seen": "2020-03-15",
64
+ "source": "SecurityTrails"
65
+ },
66
+ {
67
+ "firtst_seen": "2019-12-04",
68
+ "last_seen": "2019-12-04",
69
+ "source": "VirusTotal"
70
+ }
71
+ ],
72
+ "149.13.33.4": [
73
+ {
74
+ "firtst_seen": "2011-03-08",
75
+ "last_seen": "2012-02-13",
76
+ "source": "CIRCL"
77
+ },
78
+ {
79
+ "firtst_seen": "2013-07-30",
80
+ "last_seen": "2013-07-30",
81
+ "source": "VirusTotal"
82
+ }
83
+ ],
84
+ "194.154.205.24": [
85
+ {
86
+ "firtst_seen": "2011-03-03",
87
+ "last_seen": "2011-03-03",
88
+ "source": "CIRCL"
89
+ }
90
+ ]
91
+ }
92
+
93
+ $ ukemi lookup 195.123.226.243
94
+ {
95
+ "liankt.club": [
96
+ {
97
+ "firtst_seen": "2020-02-15",
98
+ "last_seen": "2020-03-13",
99
+ "source": "PassiveTotal"
100
+ },
101
+ {
102
+ "firtst_seen": "2020-02-16",
103
+ "last_seen": "2020-02-16",
104
+ "source": "VirusTotal"
105
+ }
106
+ ],
107
+ "weidt.club": [
108
+ {
109
+ "firtst_seen": "2020-03-12",
110
+ "last_seen": "2020-03-12",
111
+ "source": "PassiveTotal"
112
+ }
113
+ ],
114
+ "jikt.club": [
115
+ {
116
+ "firtst_seen": "2020-03-04",
117
+ "last_seen": "2020-03-12",
118
+ "source": "PassiveTotal"
119
+ },
120
+ {
121
+ "firtst_seen": "2020-03-05",
122
+ "last_seen": "2020-03-05",
123
+ "source": "VirusTotal"
124
+ }
125
+ ],
126
+ "biesi.club": [
127
+ {
128
+ "firtst_seen": "2020-02-15",
129
+ "last_seen": "2020-03-12",
130
+ "source": "PassiveTotal"
131
+ },
132
+ {
133
+ "firtst_seen": "2020-02-20",
134
+ "last_seen": "2020-02-20",
135
+ "source": "VirusTotal"
136
+ }
137
+ ],
138
+ "kaikt.club": [
139
+ {
140
+ "firtst_seen": "2020-02-15",
141
+ "last_seen": "2020-03-12",
142
+ "source": "PassiveTotal"
143
+ },
144
+ {
145
+ "firtst_seen": "2020-02-21",
146
+ "last_seen": "2020-02-21",
147
+ "source": "VirusTotal"
148
+ }
149
+ ],
150
+ "zhaokt.club": [
151
+ {
152
+ "firtst_seen": "2020-02-15",
153
+ "last_seen": "2020-03-11",
154
+ "source": "PassiveTotal"
155
+ },
156
+ {
157
+ "firtst_seen": "2020-02-18",
158
+ "last_seen": "2020-02-18",
159
+ "source": "VirusTotal"
160
+ }
161
+ ],
162
+ "yangdt.club": [
163
+ {
164
+ "firtst_seen": "2020-02-26",
165
+ "last_seen": "2020-03-10",
166
+ "source": "PassiveTotal"
167
+ },
168
+ {
169
+ "firtst_seen": "2020-02-27",
170
+ "last_seen": "2020-02-27",
171
+ "source": "VirusTotal"
172
+ }
173
+ ],
174
+ "jinkt.club": [
175
+ {
176
+ "firtst_seen": "2020-02-21",
177
+ "last_seen": "2020-03-10",
178
+ "source": "PassiveTotal"
179
+ },
180
+ {
181
+ "firtst_seen": "2020-02-22",
182
+ "last_seen": "2020-02-22",
183
+ "source": "VirusTotal"
184
+ }
185
+ ],
186
+ "taokt.club": [
187
+ {
188
+ "firtst_seen": "2020-03-10",
189
+ "last_seen": "2020-03-10",
190
+ "source": "PassiveTotal"
191
+ }
192
+ ],
193
+ "xinkt.club": [
194
+ {
195
+ "firtst_seen": "2020-02-17",
196
+ "last_seen": "2020-03-09",
197
+ "source": "PassiveTotal"
198
+ },
199
+ {
200
+ "firtst_seen": "2020-02-19",
201
+ "last_seen": "2020-02-19",
202
+ "source": "VirusTotal"
203
+ }
204
+ ],
205
+ "mail.realty-advertising.ru": [
206
+ {
207
+ "firtst_seen": "2019-11-08",
208
+ "last_seen": "2020-03-09",
209
+ "source": "PassiveTotal"
210
+ }
211
+ ],
212
+ "realty-advertising.ru": [
213
+ {
214
+ "firtst_seen": "2019-11-08",
215
+ "last_seen": "2020-03-06",
216
+ "source": "PassiveTotal"
217
+ }
218
+ ],
219
+ "ns1.realty-advertising.ru": [
220
+ {
221
+ "firtst_seen": "2019-12-02",
222
+ "last_seen": "2020-03-04",
223
+ "source": "PassiveTotal"
224
+ }
225
+ ],
226
+ "ns2.realty-advertising.ru": [
227
+ {
228
+ "firtst_seen": "2019-12-04",
229
+ "last_seen": "2020-03-04",
230
+ "source": "PassiveTotal"
231
+ }
232
+ ],
233
+ "xiankt.club": [
234
+ {
235
+ "firtst_seen": "2020-02-15",
236
+ "last_seen": "2020-03-03",
237
+ "source": "PassiveTotal"
238
+ },
239
+ {
240
+ "firtst_seen": "2020-02-16",
241
+ "last_seen": "2020-02-16",
242
+ "source": "VirusTotal"
243
+ }
244
+ ],
245
+ "nittsu-si.com": [
246
+ {
247
+ "firtst_seen": "2020-02-15",
248
+ "last_seen": "2020-03-03",
249
+ "source": "PassiveTotal"
250
+ },
251
+ {
252
+ "firtst_seen": "2020-02-21",
253
+ "last_seen": "2020-02-21",
254
+ "source": "VirusTotal"
255
+ }
256
+ ],
257
+ "mailer.realty-advertising.ru": [
258
+ {
259
+ "firtst_seen": "2020-02-23",
260
+ "last_seen": "2020-02-23",
261
+ "source": "PassiveTotal"
262
+ }
263
+ ],
264
+ "mail7.realty-advertising.ru": [
265
+ {
266
+ "firtst_seen": "2020-02-23",
267
+ "last_seen": "2020-02-23",
268
+ "source": "PassiveTotal"
269
+ }
270
+ ],
271
+ "zimbra.realty-advertising.ru": [
272
+ {
273
+ "firtst_seen": "2020-02-23",
274
+ "last_seen": "2020-02-23",
275
+ "source": "PassiveTotal"
276
+ }
277
+ ],
278
+ "relay2.realty-advertising.ru": [
279
+ {
280
+ "firtst_seen": "2020-02-23",
281
+ "last_seen": "2020-02-23",
282
+ "source": "PassiveTotal"
283
+ }
284
+ ],
285
+ "sniper.realty-advertising.ru": [
286
+ {
287
+ "firtst_seen": "2020-02-22",
288
+ "last_seen": "2020-02-22",
289
+ "source": "PassiveTotal"
290
+ }
291
+ ],
292
+ "mailx.realty-advertising.ru": [
293
+ {
294
+ "firtst_seen": "2020-02-22",
295
+ "last_seen": "2020-02-22",
296
+ "source": "PassiveTotal"
297
+ }
298
+ ],
299
+ "send.realty-advertising.ru": [
300
+ {
301
+ "firtst_seen": "2020-02-22",
302
+ "last_seen": "2020-02-22",
303
+ "source": "PassiveTotal"
304
+ }
305
+ ],
306
+ "mta.realty-advertising.ru": [
307
+ {
308
+ "firtst_seen": "2020-02-22",
309
+ "last_seen": "2020-02-22",
310
+ "source": "PassiveTotal"
311
+ }
312
+ ],
313
+ "home.realty-advertising.ru": [
314
+ {
315
+ "firtst_seen": "2020-02-22",
316
+ "last_seen": "2020-02-22",
317
+ "source": "PassiveTotal"
318
+ }
319
+ ],
320
+ "pbrand.realty-advertising.ru": [
321
+ {
322
+ "firtst_seen": "2020-02-22",
323
+ "last_seen": "2020-02-22",
324
+ "source": "PassiveTotal"
325
+ }
326
+ ],
327
+ "smtpauth.realty-advertising.ru": [
328
+ {
329
+ "firtst_seen": "2020-02-22",
330
+ "last_seen": "2020-02-22",
331
+ "source": "PassiveTotal"
332
+ }
333
+ ],
334
+ "gate.realty-advertising.ru": [
335
+ {
336
+ "firtst_seen": "2020-02-21",
337
+ "last_seen": "2020-02-21",
338
+ "source": "PassiveTotal"
339
+ }
340
+ ],
341
+ "mx02.realty-advertising.ru": [
342
+ {
343
+ "firtst_seen": "2020-02-21",
344
+ "last_seen": "2020-02-21",
345
+ "source": "PassiveTotal"
346
+ }
347
+ ],
348
+ "outmail.realty-advertising.ru": [
349
+ {
350
+ "firtst_seen": "2020-02-21",
351
+ "last_seen": "2020-02-21",
352
+ "source": "PassiveTotal"
353
+ }
354
+ ],
355
+ "exchange.realty-advertising.ru": [
356
+ {
357
+ "firtst_seen": "2020-02-21",
358
+ "last_seen": "2020-02-21",
359
+ "source": "PassiveTotal"
360
+ }
361
+ ],
362
+ "ms.realty-advertising.ru": [
363
+ {
364
+ "firtst_seen": "2020-02-21",
365
+ "last_seen": "2020-02-21",
366
+ "source": "PassiveTotal"
367
+ }
368
+ ],
369
+ "owa.realty-advertising.ru": [
370
+ {
371
+ "firtst_seen": "2020-02-20",
372
+ "last_seen": "2020-02-20",
373
+ "source": "PassiveTotal"
374
+ }
375
+ ],
376
+ "mail8.realty-advertising.ru": [
377
+ {
378
+ "firtst_seen": "2020-02-20",
379
+ "last_seen": "2020-02-20",
380
+ "source": "PassiveTotal"
381
+ }
382
+ ],
383
+ "mta-sts.realty-advertising.ru": [
384
+ {
385
+ "firtst_seen": "2019-11-11",
386
+ "last_seen": "2020-02-08",
387
+ "source": "PassiveTotal"
388
+ }
389
+ ],
390
+ "mail02.realty-advertising.ru": [
391
+ {
392
+ "firtst_seen": "2020-01-18",
393
+ "last_seen": "2020-01-18",
394
+ "source": "PassiveTotal"
395
+ }
396
+ ],
397
+ "www.realty-advertising.ru": [
398
+ {
399
+ "firtst_seen": "2019-11-08",
400
+ "last_seen": "2019-11-12",
401
+ "source": "PassiveTotal"
402
+ }
403
+ ],
404
+ "ln-048.rd-00003024.id-11744955.v0.tun.vpnoverdns.com": [
405
+ {
406
+ "firtst_seen": "2017-04-06",
407
+ "last_seen": "2017-04-06",
408
+ "source": "PassiveTotal"
409
+ }
410
+ ],
411
+ "mnen6k7g.info": [
412
+ {
413
+ "firtst_seen": "2010-10-28",
414
+ "last_seen": "2010-10-28",
415
+ "source": "PassiveTotal"
416
+ }
417
+ ]
418
+ }
419
+ ```
420
+
421
+ ## License
422
+
423
+ The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).
data/Rakefile ADDED
@@ -0,0 +1,8 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "bundler/gem_tasks"
4
+ require "rspec/core/rake_task"
5
+
6
+ RSpec::Core::RakeTask.new(:spec)
7
+
8
+ task default: :spec
data/bin/console ADDED
@@ -0,0 +1,14 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ require "bundler/setup"
4
+ require "ukemi"
5
+
6
+ # You can add fixtures and/or initialization code here to make experimenting
7
+ # with your gem easier. You can also use a different console, if you like.
8
+
9
+ # (If you use this, don't forget to add pry to your Gemfile!)
10
+ # require "pry"
11
+ # Pry.start
12
+
13
+ require "irb"
14
+ IRB.start(__FILE__)
data/bin/setup ADDED
@@ -0,0 +1,8 @@
1
+ #!/usr/bin/env bash
2
+ set -euo pipefail
3
+ IFS=$'\n\t'
4
+ set -vx
5
+
6
+ bundle install
7
+
8
+ # Do any other automated setup that you need to do here
data/exe/ukemi ADDED
@@ -0,0 +1,8 @@
1
+ #!/usr/bin/env ruby
2
+ # frozen_string_literal: true
3
+
4
+ $LOAD_PATH.unshift("#{__dir__}/../lib")
5
+
6
+ require "ukemi"
7
+
8
+ Ukemi::CLI.start
data/lib/ukemi/cli.rb ADDED
@@ -0,0 +1,21 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "json"
4
+ require "thor"
5
+
6
+ module Ukemi
7
+ class CLI < Thor
8
+ desc "lookup [IP|DOMAIN]", "Lookup passive DNS services"
9
+ def lookup(data)
10
+ data = refang(data)
11
+ result = Moderator.lookup(data)
12
+ puts JSON.pretty_generate(result)
13
+ end
14
+
15
+ no_commands do
16
+ def refang(data)
17
+ data.gsub("[.]", ".").gsub("(.)", ".")
18
+ end
19
+ end
20
+ end
21
+ end
@@ -0,0 +1,5 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Ukemi
4
+ class Error < StandardError; end
5
+ end
@@ -0,0 +1,49 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "parallel"
4
+ require "time"
5
+
6
+ module Ukemi
7
+ class Moderator
8
+ def lookup(data)
9
+ records = Parallel.map(Ukemi.services) do |klass|
10
+ service = klass.new
11
+ next unless service.configurated?
12
+
13
+ begin
14
+ service.lookup data
15
+ rescue ::PassiveTotal::Error, ::VirusTotal::Error, ::SecurityTrails::Error, PassiveCIRCL::Error
16
+ nil
17
+ end
18
+ end.flatten.compact
19
+
20
+ memo = Hash.new { |h, k| h[k] = [] }
21
+ records.each do |record|
22
+ memo[record.data] << {
23
+ firtst_seen: record.first_seen,
24
+ last_seen: record.last_seen,
25
+ source: record.source,
26
+ }
27
+ end
28
+
29
+ memo.sort_by do |_key, array|
30
+ last_seens = array.map do |record|
31
+ parse_to_unixtime record.dig(:last_seen)
32
+ end
33
+ -last_seens.max
34
+ end.to_h
35
+ end
36
+
37
+ def parse_to_unixtime(date)
38
+ return -1 unless date
39
+
40
+ Time.parse(date).to_i
41
+ end
42
+
43
+ class << self
44
+ def lookup(data)
45
+ new.lookup data
46
+ end
47
+ end
48
+ end
49
+ end
@@ -0,0 +1,26 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Ukemi
4
+ class Record
5
+ attr_reader :data
6
+ attr_reader :first_seen
7
+ attr_reader :last_seen
8
+ attr_reader :source
9
+
10
+ def initialize(data:, first_seen: nil, last_seen: nil, source: nil)
11
+ @data = data
12
+ @first_seen = first_seen
13
+ @last_seen = last_seen
14
+ @source = source
15
+ end
16
+
17
+ def to_h
18
+ {
19
+ data: data,
20
+ first_seen: first_seen,
21
+ last_seen: last_seen,
22
+ source: source
23
+ }
24
+ end
25
+ end
26
+ end
@@ -0,0 +1,43 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "passive_circl"
4
+
5
+ module Ukemi
6
+ module Services
7
+ class CIRCL < Service
8
+ private
9
+
10
+ def config_keys
11
+ %w(CIRCL_PASSIVE_USERNAME CIRCL_PASSIVE_PASSWORD)
12
+ end
13
+
14
+ def api
15
+ @api ||= PassiveCIRCL::API.new
16
+ end
17
+
18
+ def lookup_by_domain(data)
19
+ passive_dns_lookup(data, "rdata")
20
+ end
21
+
22
+ def lookup_by_ip(data)
23
+ passive_dns_lookup(data, "rrname")
24
+ end
25
+
26
+ def passive_dns_lookup(data, key = nil)
27
+ results = api.dns.query(data)
28
+ results = results.select do |result|
29
+ result.dig("rrtype") == "A"
30
+ end
31
+
32
+ results.map do |result|
33
+ Record.new(
34
+ data: result.dig(key),
35
+ first_seen: Time.at(result.dig("time_first")).to_date.to_s,
36
+ last_seen: Time.at(result.dig("time_last")).to_date.to_s,
37
+ source: name
38
+ )
39
+ end
40
+ end
41
+ end
42
+ end
43
+ end
@@ -0,0 +1,43 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "passivetotal"
4
+
5
+ module Ukemi
6
+ module Services
7
+ class PassiveTotal < Service
8
+ private
9
+
10
+ def api
11
+ @api ||= ::PassiveTotal::API.new
12
+ end
13
+
14
+ def config_keys
15
+ %w(PASSIVETOTAL_USERNAME PASSIVETOTAL_API_KEY)
16
+ end
17
+
18
+ def lookup_by_ip(data)
19
+ res = api.dns.passive(data)
20
+ results = res.dig("results") || []
21
+ convert_to_records results
22
+ end
23
+
24
+ def lookup_by_domain(_data)
25
+ []
26
+ end
27
+
28
+ def convert_to_records(results)
29
+ results.map do |result|
30
+ data = result.dig("resolve")
31
+ first_seen = result.dig("firstSeen").to_s.split.first
32
+ last_seen = result.dig("lastSeen").to_s.split.first
33
+ Record.new(
34
+ data: data,
35
+ first_seen: first_seen,
36
+ last_seen: last_seen,
37
+ source: name
38
+ )
39
+ end
40
+ end
41
+ end
42
+ end
43
+ end
@@ -0,0 +1,57 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "date"
4
+ require "securitytrails"
5
+
6
+ module Ukemi
7
+ module Services
8
+ class SecurityTrails < Service
9
+ private
10
+
11
+ def config_keys
12
+ %w(SECURITYTRAILS_API_KEY)
13
+ end
14
+
15
+ def api
16
+ @api ||= ::SecurityTrails::API.new
17
+ end
18
+
19
+ def lookup_by_ip(data)
20
+ result = api.domains.search( filter: { ipv4: data })
21
+ records = result.dig("records") || []
22
+ hostnames = records.map { |record| record.dig("hostname") }
23
+ hostnames.map do |hostname|
24
+ Record.new(
25
+ data: hostname,
26
+ first_seen: nil,
27
+ last_seen: nil,
28
+ source: name
29
+ )
30
+ end
31
+ end
32
+
33
+ def lookup_by_domain(data)
34
+ result = api.history.get_all_dns_history(data, type: "a")
35
+ records = result.dig("records") || []
36
+ records.map do |record|
37
+ values = record.dig("values") || []
38
+ values.map do |value|
39
+ Record.new(
40
+ data: value.dig("ip"),
41
+ first_seen: record.dig("first_seen"),
42
+ last_seen: record.dig("last_seen"),
43
+ source: name
44
+ )
45
+ end
46
+ end.flatten
47
+ end
48
+
49
+ def extract_attributes(response)
50
+ data = response.dig("data") || []
51
+ data.map do |item|
52
+ item.dig("attributes") || []
53
+ end
54
+ end
55
+ end
56
+ end
57
+ end
@@ -0,0 +1,73 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "addressable/uri"
4
+ require "ipaddr"
5
+ require "public_suffix"
6
+
7
+ module Ukemi
8
+ module Services
9
+ class Service
10
+ def name
11
+ @name ||= self.class.to_s.split("::").last
12
+ end
13
+
14
+ def lookup(data)
15
+ @data = data
16
+
17
+ case type
18
+ when "ip"
19
+ lookup_by_ip data
20
+ when "domain"
21
+ lookup_by_domain data
22
+ else
23
+ raise ArgumentError, "#{data} is not a valid input."
24
+ end
25
+ end
26
+
27
+ def configurated?
28
+ config_keys.all? do |key|
29
+ ENV.key? key
30
+ end
31
+ end
32
+
33
+ class << self
34
+ def inherited(child)
35
+ Ukemi.services << child
36
+ end
37
+ end
38
+
39
+ private
40
+
41
+ def config_keys
42
+ raise NotImplementedError, "You must implement #{self.class}##{__method__}"
43
+ end
44
+
45
+ def lookup_by_ip
46
+ raise NotImplementedError, "You must implement #{self.class}##{__method__}"
47
+ end
48
+
49
+ def lookup_by_domain
50
+ raise NotImplementedError, "You must implement #{self.class}##{__method__}"
51
+ end
52
+
53
+ def ip?
54
+ IPAddr.new @data
55
+ true
56
+ rescue IPAddr::InvalidAddressError => _e
57
+ false
58
+ end
59
+
60
+ def domain?
61
+ uri = Addressable::URI.parse("http://#{@data}")
62
+ uri.host == @data && PublicSuffix.valid?(uri.host)
63
+ rescue Addressable::URI::InvalidURIError => _e
64
+ false
65
+ end
66
+
67
+ def type
68
+ return "ip" if ip?
69
+ return "domain" if domain?
70
+ end
71
+ end
72
+ end
73
+ end
@@ -0,0 +1,58 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "date"
4
+ require "virustotal"
5
+
6
+ module Ukemi
7
+ module Services
8
+ class VirusTotal < Service
9
+ private
10
+
11
+ def config_keys
12
+ %w(VIRUSTOTAL_API_KEY)
13
+ end
14
+
15
+ def api
16
+ @api ||= ::VirusTotal::API.new
17
+ end
18
+
19
+ def lookup_by_ip(data)
20
+ res = api.ip_address.resolutions(data)
21
+ attributes = extract_attributes(res)
22
+ convert_to_records attributes, "host_name"
23
+ end
24
+
25
+ def lookup_by_domain(data)
26
+ res = api.domain.resolutions(data)
27
+ attributes = extract_attributes(res)
28
+ convert_to_records attributes, "ip_address"
29
+ end
30
+
31
+ def extract_attributes(response)
32
+ data = response.dig("data") || []
33
+ data.map do |item|
34
+ item.dig("attributes") || []
35
+ end
36
+ end
37
+
38
+ def convert_to_records(attributes, key = nil)
39
+ memo = Hash.new { |h, k| h[k] = [] }
40
+
41
+ attributes.each do |attribute|
42
+ data = attribute.dig(key)
43
+ date = Time.at(attribute.dig("date")).to_date.to_s
44
+ memo[data] << date
45
+ end
46
+
47
+ memo.keys.map do |data|
48
+ Record.new(
49
+ data: data,
50
+ first_seen: memo[data].min,
51
+ last_seen: memo[data].max,
52
+ source: name
53
+ )
54
+ end
55
+ end
56
+ end
57
+ end
58
+ end
@@ -0,0 +1,5 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Ukemi
4
+ VERSION = "0.1.0"
5
+ end
data/lib/ukemi.rb ADDED
@@ -0,0 +1,31 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "mem"
4
+
5
+ module Ukemi
6
+ class << self
7
+ include Mem
8
+
9
+ def services
10
+ []
11
+ end
12
+ memoize :services
13
+ end
14
+ end
15
+
16
+ require "ukemi/version"
17
+
18
+ require "ukemi/error"
19
+
20
+ require "ukemi/record"
21
+
22
+ require "ukemi/services/service"
23
+
24
+ require "ukemi/services/circl"
25
+ require "ukemi/services/passivetotal"
26
+ require "ukemi/services/securitytrails"
27
+ require "ukemi/services/virustotal"
28
+
29
+ require "ukemi/moderator"
30
+
31
+ require "ukemi/cli"
data/ukemi.gemspec ADDED
@@ -0,0 +1,44 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative 'lib/ukemi/version'
4
+
5
+ Gem::Specification.new do |spec|
6
+ spec.name = "ukemi"
7
+ spec.version = Ukemi::VERSION
8
+ spec.authors = ["Manabu Niseki"]
9
+ spec.email = ["manabu.niseki@gmail.com"]
10
+
11
+ spec.summary = "A CLI tool for querying passive DNS services"
12
+ spec.description = "A CLI tool for querying passive DNS services"
13
+ spec.homepage = "https://github.com/ninoseki/ukemi"
14
+ spec.license = "MIT"
15
+ spec.required_ruby_version = Gem::Requirement.new(">= 2.3.0")
16
+
17
+ spec.metadata["homepage_uri"] = spec.homepage
18
+
19
+ # Specify which files should be added to the gem when it is released.
20
+ # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
21
+ spec.files = Dir.chdir(File.expand_path(__dir__)) do
22
+ `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
23
+ end
24
+ spec.bindir = "exe"
25
+ spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
26
+ spec.require_paths = ["lib"]
27
+
28
+ spec.add_development_dependency "bundler", "~> 2.1"
29
+ spec.add_development_dependency "coveralls", "~> 0.8"
30
+ spec.add_development_dependency "rake", "~> 13.0"
31
+ spec.add_development_dependency "rspec", "~> 3.9"
32
+ spec.add_development_dependency "vcr", "~> 5.0"
33
+ spec.add_development_dependency "webmock", "~> 3.8"
34
+
35
+ spec.add_dependency "addressable", "~> 2.7"
36
+ spec.add_dependency "mem", "~> 0.1"
37
+ spec.add_dependency "parallel", "~> 1.19"
38
+ spec.add_dependency "passive_circl", "~> 0.1"
39
+ spec.add_dependency "passivetotalx", "~> 0.1"
40
+ spec.add_dependency "public_suffix", "~> 4.0"
41
+ spec.add_dependency "securitytrails", "~> 1.0"
42
+ spec.add_dependency "thor", "~> 1.0"
43
+ spec.add_dependency "virustotalx", "~> 1.1"
44
+ end
metadata ADDED
@@ -0,0 +1,277 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: ukemi
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.1.0
5
+ platform: ruby
6
+ authors:
7
+ - Manabu Niseki
8
+ autorequire:
9
+ bindir: exe
10
+ cert_chain: []
11
+ date: 2020-03-15 00:00:00.000000000 Z
12
+ dependencies:
13
+ - !ruby/object:Gem::Dependency
14
+ name: bundler
15
+ requirement: !ruby/object:Gem::Requirement
16
+ requirements:
17
+ - - "~>"
18
+ - !ruby/object:Gem::Version
19
+ version: '2.1'
20
+ type: :development
21
+ prerelease: false
22
+ version_requirements: !ruby/object:Gem::Requirement
23
+ requirements:
24
+ - - "~>"
25
+ - !ruby/object:Gem::Version
26
+ version: '2.1'
27
+ - !ruby/object:Gem::Dependency
28
+ name: coveralls
29
+ requirement: !ruby/object:Gem::Requirement
30
+ requirements:
31
+ - - "~>"
32
+ - !ruby/object:Gem::Version
33
+ version: '0.8'
34
+ type: :development
35
+ prerelease: false
36
+ version_requirements: !ruby/object:Gem::Requirement
37
+ requirements:
38
+ - - "~>"
39
+ - !ruby/object:Gem::Version
40
+ version: '0.8'
41
+ - !ruby/object:Gem::Dependency
42
+ name: rake
43
+ requirement: !ruby/object:Gem::Requirement
44
+ requirements:
45
+ - - "~>"
46
+ - !ruby/object:Gem::Version
47
+ version: '13.0'
48
+ type: :development
49
+ prerelease: false
50
+ version_requirements: !ruby/object:Gem::Requirement
51
+ requirements:
52
+ - - "~>"
53
+ - !ruby/object:Gem::Version
54
+ version: '13.0'
55
+ - !ruby/object:Gem::Dependency
56
+ name: rspec
57
+ requirement: !ruby/object:Gem::Requirement
58
+ requirements:
59
+ - - "~>"
60
+ - !ruby/object:Gem::Version
61
+ version: '3.9'
62
+ type: :development
63
+ prerelease: false
64
+ version_requirements: !ruby/object:Gem::Requirement
65
+ requirements:
66
+ - - "~>"
67
+ - !ruby/object:Gem::Version
68
+ version: '3.9'
69
+ - !ruby/object:Gem::Dependency
70
+ name: vcr
71
+ requirement: !ruby/object:Gem::Requirement
72
+ requirements:
73
+ - - "~>"
74
+ - !ruby/object:Gem::Version
75
+ version: '5.0'
76
+ type: :development
77
+ prerelease: false
78
+ version_requirements: !ruby/object:Gem::Requirement
79
+ requirements:
80
+ - - "~>"
81
+ - !ruby/object:Gem::Version
82
+ version: '5.0'
83
+ - !ruby/object:Gem::Dependency
84
+ name: webmock
85
+ requirement: !ruby/object:Gem::Requirement
86
+ requirements:
87
+ - - "~>"
88
+ - !ruby/object:Gem::Version
89
+ version: '3.8'
90
+ type: :development
91
+ prerelease: false
92
+ version_requirements: !ruby/object:Gem::Requirement
93
+ requirements:
94
+ - - "~>"
95
+ - !ruby/object:Gem::Version
96
+ version: '3.8'
97
+ - !ruby/object:Gem::Dependency
98
+ name: addressable
99
+ requirement: !ruby/object:Gem::Requirement
100
+ requirements:
101
+ - - "~>"
102
+ - !ruby/object:Gem::Version
103
+ version: '2.7'
104
+ type: :runtime
105
+ prerelease: false
106
+ version_requirements: !ruby/object:Gem::Requirement
107
+ requirements:
108
+ - - "~>"
109
+ - !ruby/object:Gem::Version
110
+ version: '2.7'
111
+ - !ruby/object:Gem::Dependency
112
+ name: mem
113
+ requirement: !ruby/object:Gem::Requirement
114
+ requirements:
115
+ - - "~>"
116
+ - !ruby/object:Gem::Version
117
+ version: '0.1'
118
+ type: :runtime
119
+ prerelease: false
120
+ version_requirements: !ruby/object:Gem::Requirement
121
+ requirements:
122
+ - - "~>"
123
+ - !ruby/object:Gem::Version
124
+ version: '0.1'
125
+ - !ruby/object:Gem::Dependency
126
+ name: parallel
127
+ requirement: !ruby/object:Gem::Requirement
128
+ requirements:
129
+ - - "~>"
130
+ - !ruby/object:Gem::Version
131
+ version: '1.19'
132
+ type: :runtime
133
+ prerelease: false
134
+ version_requirements: !ruby/object:Gem::Requirement
135
+ requirements:
136
+ - - "~>"
137
+ - !ruby/object:Gem::Version
138
+ version: '1.19'
139
+ - !ruby/object:Gem::Dependency
140
+ name: passive_circl
141
+ requirement: !ruby/object:Gem::Requirement
142
+ requirements:
143
+ - - "~>"
144
+ - !ruby/object:Gem::Version
145
+ version: '0.1'
146
+ type: :runtime
147
+ prerelease: false
148
+ version_requirements: !ruby/object:Gem::Requirement
149
+ requirements:
150
+ - - "~>"
151
+ - !ruby/object:Gem::Version
152
+ version: '0.1'
153
+ - !ruby/object:Gem::Dependency
154
+ name: passivetotalx
155
+ requirement: !ruby/object:Gem::Requirement
156
+ requirements:
157
+ - - "~>"
158
+ - !ruby/object:Gem::Version
159
+ version: '0.1'
160
+ type: :runtime
161
+ prerelease: false
162
+ version_requirements: !ruby/object:Gem::Requirement
163
+ requirements:
164
+ - - "~>"
165
+ - !ruby/object:Gem::Version
166
+ version: '0.1'
167
+ - !ruby/object:Gem::Dependency
168
+ name: public_suffix
169
+ requirement: !ruby/object:Gem::Requirement
170
+ requirements:
171
+ - - "~>"
172
+ - !ruby/object:Gem::Version
173
+ version: '4.0'
174
+ type: :runtime
175
+ prerelease: false
176
+ version_requirements: !ruby/object:Gem::Requirement
177
+ requirements:
178
+ - - "~>"
179
+ - !ruby/object:Gem::Version
180
+ version: '4.0'
181
+ - !ruby/object:Gem::Dependency
182
+ name: securitytrails
183
+ requirement: !ruby/object:Gem::Requirement
184
+ requirements:
185
+ - - "~>"
186
+ - !ruby/object:Gem::Version
187
+ version: '1.0'
188
+ type: :runtime
189
+ prerelease: false
190
+ version_requirements: !ruby/object:Gem::Requirement
191
+ requirements:
192
+ - - "~>"
193
+ - !ruby/object:Gem::Version
194
+ version: '1.0'
195
+ - !ruby/object:Gem::Dependency
196
+ name: thor
197
+ requirement: !ruby/object:Gem::Requirement
198
+ requirements:
199
+ - - "~>"
200
+ - !ruby/object:Gem::Version
201
+ version: '1.0'
202
+ type: :runtime
203
+ prerelease: false
204
+ version_requirements: !ruby/object:Gem::Requirement
205
+ requirements:
206
+ - - "~>"
207
+ - !ruby/object:Gem::Version
208
+ version: '1.0'
209
+ - !ruby/object:Gem::Dependency
210
+ name: virustotalx
211
+ requirement: !ruby/object:Gem::Requirement
212
+ requirements:
213
+ - - "~>"
214
+ - !ruby/object:Gem::Version
215
+ version: '1.1'
216
+ type: :runtime
217
+ prerelease: false
218
+ version_requirements: !ruby/object:Gem::Requirement
219
+ requirements:
220
+ - - "~>"
221
+ - !ruby/object:Gem::Version
222
+ version: '1.1'
223
+ description: A CLI tool for querying passive DNS services
224
+ email:
225
+ - manabu.niseki@gmail.com
226
+ executables:
227
+ - ukemi
228
+ extensions: []
229
+ extra_rdoc_files: []
230
+ files:
231
+ - ".gitignore"
232
+ - ".rspec"
233
+ - ".travis.yml"
234
+ - Gemfile
235
+ - LICENSE
236
+ - README.md
237
+ - Rakefile
238
+ - bin/console
239
+ - bin/setup
240
+ - exe/ukemi
241
+ - lib/ukemi.rb
242
+ - lib/ukemi/cli.rb
243
+ - lib/ukemi/error.rb
244
+ - lib/ukemi/moderator.rb
245
+ - lib/ukemi/record.rb
246
+ - lib/ukemi/services/circl.rb
247
+ - lib/ukemi/services/passivetotal.rb
248
+ - lib/ukemi/services/securitytrails.rb
249
+ - lib/ukemi/services/service.rb
250
+ - lib/ukemi/services/virustotal.rb
251
+ - lib/ukemi/version.rb
252
+ - ukemi.gemspec
253
+ homepage: https://github.com/ninoseki/ukemi
254
+ licenses:
255
+ - MIT
256
+ metadata:
257
+ homepage_uri: https://github.com/ninoseki/ukemi
258
+ post_install_message:
259
+ rdoc_options: []
260
+ require_paths:
261
+ - lib
262
+ required_ruby_version: !ruby/object:Gem::Requirement
263
+ requirements:
264
+ - - ">="
265
+ - !ruby/object:Gem::Version
266
+ version: 2.3.0
267
+ required_rubygems_version: !ruby/object:Gem::Requirement
268
+ requirements:
269
+ - - ">="
270
+ - !ruby/object:Gem::Version
271
+ version: '0'
272
+ requirements: []
273
+ rubygems_version: 3.1.2
274
+ signing_key:
275
+ specification_version: 4
276
+ summary: A CLI tool for querying passive DNS services
277
+ test_files: []