threatinator 0.1.6 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (197) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +13 -2
  3. data/Gemfile +18 -13
  4. data/Rakefile +1 -1
  5. data/VERSION +1 -1
  6. data/feeds/ET_block-ip_reputation.feed +26 -0
  7. data/feeds/ET_openbadlist-ip_reputation.feed +35 -0
  8. data/feeds/bambenek_c2_masterlist-domain_reputation.feed +15 -0
  9. data/feeds/bambenek_c2_masterlist-ip_reputation.feed +15 -0
  10. data/feeds/bambenek_dga_feed-domain_reputation.feed +15 -0
  11. data/feeds/berkeley-ip_reputation.feed +23 -0
  12. data/feeds/bitcash_cz_blacklist.feed +20 -0
  13. data/feeds/botscout-ip_reputation.feed +24 -0
  14. data/feeds/cert_mxpoison-ip_reputation.feed +21 -0
  15. data/feeds/chaosreigns-ip_reputation.feed +36 -0
  16. data/feeds/cydef_torexit-ip_reputation.feed +24 -0
  17. data/feeds/danger_bruteforce-ip_reputation.feed +23 -0
  18. data/feeds/falconcrest-ip_reputation.feed +18 -0
  19. data/feeds/h3x_asprox.feed +17 -0
  20. data/feeds/hosts-file_hphostspartial-domain_reputation.feed +18 -0
  21. data/feeds/infiltrated_vabl-ip_reputation.feed +29 -0
  22. data/feeds/isc_suspicious_high-domain_reputation.feed +25 -0
  23. data/feeds/isc_suspicious_low-domain_reputation.feed +25 -0
  24. data/feeds/isc_suspicious_medium-domain_reputation.feed +25 -0
  25. data/feeds/malwaredomainlist-url_reputation.feed +16 -0
  26. data/feeds/malwaredomains-domain_reputation.feed +27 -0
  27. data/feeds/malwaredomains_dyndns-domain_reputation.feed +27 -0
  28. data/feeds/malwaredomains_justdomains-domain_reputation.feed +18 -0
  29. data/feeds/multiproxy-ip_reputation.feed +20 -0
  30. data/feeds/openphish-url_reputation.feed +22 -0
  31. data/feeds/packetmail_perimeterbad-ip_reputation.feed +26 -0
  32. data/feeds/phishtank.feed +1 -1
  33. data/feeds/sigmaproject_atma.feed +25 -0
  34. data/feeds/sigmaproject_spyware.feed +24 -0
  35. data/feeds/sigmaproject_webexploit.feed +26 -0
  36. data/feeds/snort_bpf-ip_reputation.feed +19 -0
  37. data/feeds/steeman-ip_reputation.feed +19 -0
  38. data/feeds/trustedsec-ip_reputation.feed +17 -0
  39. data/feeds/virbl-ip_reputation.feed +24 -0
  40. data/feeds/vxvault-url_reputation.feed +22 -0
  41. data/feeds/yoyo_adservers-domain_reputation.feed +16 -0
  42. data/lib/threatinator/actions/run/action.rb +15 -3
  43. data/lib/threatinator/actions/run/coverage_observer.rb +12 -7
  44. data/lib/threatinator/actions/run/status_observer.rb +37 -0
  45. data/lib/threatinator/cli.rb +9 -3
  46. data/lib/threatinator/cli/parser.rb +14 -4
  47. data/lib/threatinator/config.rb +1 -0
  48. data/lib/threatinator/config/logger.rb +14 -0
  49. data/lib/threatinator/event.rb +28 -18
  50. data/lib/threatinator/event_builder.rb +52 -23
  51. data/lib/threatinator/exceptions.rb +3 -6
  52. data/lib/threatinator/feed.rb +1 -1
  53. data/lib/threatinator/feed_runner.rb +63 -7
  54. data/lib/threatinator/logger.rb +66 -0
  55. data/lib/threatinator/logging.rb +20 -0
  56. data/lib/threatinator/model/base.rb +23 -0
  57. data/lib/threatinator/model/collection.rb +64 -0
  58. data/lib/threatinator/model/observables/fqdn_collection.rb +13 -0
  59. data/lib/threatinator/model/observables/ipv4.rb +30 -0
  60. data/lib/threatinator/model/observables/ipv4_collection.rb +14 -0
  61. data/lib/threatinator/model/observables/url_collection.rb +16 -0
  62. data/lib/threatinator/model/validations.rb +1 -0
  63. data/lib/threatinator/model/validations/type.rb +21 -0
  64. data/lib/threatinator/plugins/output/csv.rb +20 -9
  65. data/spec/feeds/ET_block-ip_reputation_spec.rb +50 -0
  66. data/spec/feeds/ET_compromised-ip_reputation_spec.rb +2 -5
  67. data/spec/feeds/ET_openbadlist-ip_reputation_spec.rb +56 -0
  68. data/spec/feeds/alienvault-ip_reputation_spec.rb +2 -5
  69. data/spec/feeds/arbor_fastflux-domain_reputation_spec.rb +0 -3
  70. data/spec/feeds/arbor_ssh-ip_reputation_spec.rb +2 -5
  71. data/spec/feeds/autoshun_shunlist_spec.rb +1 -4
  72. data/spec/feeds/bambenek_c2_masterlist-domain_reputation_spec.rb +39 -0
  73. data/spec/feeds/bambenek_c2_masterlist-ip_reputation_spec.rb +39 -0
  74. data/spec/feeds/bambenek_dga_feed-domain_reputation_spec.rb +39 -0
  75. data/spec/feeds/berkeley-ip_reputation_spec.rb +47 -0
  76. data/spec/feeds/bitcash_cz_blacklist-ip_reputation_spec.rb +50 -0
  77. data/spec/feeds/blocklist_de_apache-ip_reputation_spec.rb +2 -5
  78. data/spec/feeds/blocklist_de_bots-ip_reputation_spec.rb +2 -5
  79. data/spec/feeds/blocklist_de_ftp-ip_reputation_spec.rb +2 -5
  80. data/spec/feeds/blocklist_de_imap-ip_reputation_spec.rb +2 -5
  81. data/spec/feeds/blocklist_de_pop3-ip_reputation_spec.rb +2 -5
  82. data/spec/feeds/blocklist_de_proftpd-ip_reputation_spec.rb +2 -5
  83. data/spec/feeds/blocklist_de_sip-ip_reputation_spec.rb +2 -5
  84. data/spec/feeds/blocklist_de_ssh-ip_reputation_spec.rb +2 -5
  85. data/spec/feeds/blocklist_de_strongips-ip_reputation_spec.rb +2 -5
  86. data/spec/feeds/botscout-ip_reputation_spec.rb +50 -0
  87. data/spec/feeds/cert_mxpoison-ip_reputation_spec.rb +47 -0
  88. data/spec/feeds/chaosreigns-ip_reputation_spec.rb +50 -0
  89. data/spec/feeds/ciarmy-ip_reputation_spec.rb +2 -5
  90. data/spec/feeds/cruzit-ip_reputation_spec.rb +2 -5
  91. data/spec/feeds/cydef_torexit-ip_reputation_spec.rb +47 -0
  92. data/spec/feeds/dan_me_uk_torlist-ip_reputation_spec.rb +2 -5
  93. data/spec/feeds/danger_bruteforce-ip_reputation_spec.rb +47 -0
  94. data/spec/feeds/data/ET_block-ip_reputation.txt +80 -0
  95. data/spec/feeds/data/ET_openbadlist-ip_reputation.txt +62 -0
  96. data/spec/feeds/data/bambenek_c2-dommasterlist.csv +30 -0
  97. data/spec/feeds/data/bambenek_c2-ipmasterlist.csv +27 -0
  98. data/spec/feeds/data/bambenek_dga_feed.csv +42 -0
  99. data/spec/feeds/data/berkeley.txt +29 -0
  100. data/spec/feeds/data/bitcash_cz_blacklist.txt +7 -0
  101. data/spec/feeds/data/botscout-ip-reputation.txt +713 -0
  102. data/spec/feeds/data/cert_mxpoison-ip_reputation.txt +17 -0
  103. data/spec/feeds/data/chaosreigns-ip-reputation.txt +26 -0
  104. data/spec/feeds/data/cydef_torexit-ip_reputation.txt +27 -0
  105. data/spec/feeds/data/danger_bruteforce-ip_reputation.txt +12 -0
  106. data/spec/feeds/data/falconcrest_iplist.txt +345 -0
  107. data/spec/feeds/data/h3x_asprox.txt +20 -0
  108. data/spec/feeds/data/hosts-file_hphostspartial_domainlist.txt +24 -0
  109. data/spec/feeds/data/infiltrated_vabl_iplist.txt +33 -0
  110. data/spec/feeds/data/isc_suspicious_high_domainlist.txt +26 -0
  111. data/spec/feeds/data/isc_suspicious_low_domainlist.txt +34 -0
  112. data/spec/feeds/data/isc_suspicious_medium_domainlist.txt +32 -0
  113. data/spec/feeds/data/malwaredomainlist-url-reputation.txt +8 -0
  114. data/spec/feeds/data/malwaredomains_domainlist.txt +24 -0
  115. data/spec/feeds/data/malwaredomains_dyndns_domainlist.txt +34 -0
  116. data/spec/feeds/data/malwaredomains_justdomains_domainlist.txt +18 -0
  117. data/spec/feeds/data/multiproxy_iplist.txt +15 -0
  118. data/spec/feeds/data/openphish-url-reputation.txt +16 -0
  119. data/spec/feeds/data/packetmail_perimeterbad-ip_reputation.txt +44 -0
  120. data/spec/feeds/data/sigmaproject_atma.return.gz +0 -0
  121. data/spec/feeds/data/sigmaproject_spyware.return.gz +0 -0
  122. data/spec/feeds/data/sigmaproject_webexploit.return.gz +0 -0
  123. data/spec/feeds/data/snort_bpf-ip_reputation.txt +16 -0
  124. data/spec/feeds/data/steeman-ip-reputation.txt +13 -0
  125. data/spec/feeds/data/trustedsec-ip-reputation.txt +12 -0
  126. data/spec/feeds/data/virbl-ip_reputation.txt +14 -0
  127. data/spec/feeds/data/vxvault-url-reputation.txt +15 -0
  128. data/spec/feeds/data/yoyo_adservers.txt +25 -0
  129. data/spec/feeds/dshield_attackers-top1000_spec.rb +1 -4
  130. data/spec/feeds/falconcrest-ip_reputation_spec.rb +37 -0
  131. data/spec/feeds/feodo-domain_reputation_spec.rb +0 -3
  132. data/spec/feeds/feodo-ip_reputation_spec.rb +2 -5
  133. data/spec/feeds/h3x_asprox-ip_reputation_spec.rb +50 -0
  134. data/spec/feeds/hosts-file_hphostspartial-domain_reputation_spec.rb +47 -0
  135. data/spec/feeds/infiltrated-ip_reputation_spec.rb +2 -5
  136. data/spec/feeds/infiltrated_vabl-ip_reputation_spec.rb +47 -0
  137. data/spec/feeds/isc_suspicious_high-domain_reputation_spec.rb +47 -0
  138. data/spec/feeds/isc_suspicious_low-domain_reputation_spec.rb +47 -0
  139. data/spec/feeds/isc_suspicious_medium-domain_reputation_spec.rb +47 -0
  140. data/spec/feeds/malc0de-domain_reputation_spec.rb +0 -3
  141. data/spec/feeds/malc0de-ip_reputation_spec.rb +2 -5
  142. data/spec/feeds/malwaredomainlist_url_reputation_spec.rb +50 -0
  143. data/spec/feeds/malwaredomains-domain_reputation_spec.rb +47 -0
  144. data/spec/feeds/malwaredomains_dyndns-domain_reputation_spec.rb +47 -0
  145. data/spec/feeds/malwaredomains_justdomains-domain_reputation_spec.rb +47 -0
  146. data/spec/feeds/mirc-domain_reputation_spec.rb +0 -3
  147. data/spec/feeds/multiproxy-ip_reputation_spec.rb +47 -0
  148. data/spec/feeds/nothink_irc-ip_reputation_spec.rb +2 -5
  149. data/spec/feeds/nothink_ssh-ip_reputation_spec.rb +2 -5
  150. data/spec/feeds/openbl-ip_reputation_spec.rb +2 -5
  151. data/spec/feeds/openphish_url_reputation_spec.rb +50 -0
  152. data/spec/feeds/packetmail_perimeterbad-ip_reputation_spec.rb +47 -0
  153. data/spec/feeds/palevo-domain_reputation_spec.rb +0 -3
  154. data/spec/feeds/palevo-ip_reputation_spec.rb +2 -5
  155. data/spec/feeds/phishtank_spec.rb +2 -5
  156. data/spec/feeds/sigmaproject_atma_spec.rb +63 -0
  157. data/spec/feeds/sigmaproject_spyware_spec.rb +64 -0
  158. data/spec/feeds/sigmaproject_webexploit_spec.rb +63 -0
  159. data/spec/feeds/snort_bpf-ip_reputation_spec.rb +47 -0
  160. data/spec/feeds/spyeye-domain_reputation_spec.rb +0 -3
  161. data/spec/feeds/spyeye-ip_reputation_spec.rb +2 -5
  162. data/spec/feeds/steeman-ip_reputation_spec.rb +50 -0
  163. data/spec/feeds/t-arend-de_ssh-ip_reputation_spec.rb +2 -5
  164. data/spec/feeds/the_haleys_ssh-ip_reputation_spec.rb +2 -5
  165. data/spec/feeds/trustedsec-ip_reputation_spec.rb +47 -0
  166. data/spec/feeds/virbl-ip_reputation_spec.rb +47 -0
  167. data/spec/feeds/vxvault_url_reputation_spec.rb +50 -0
  168. data/spec/feeds/yourcmc_ssh-ip_reputation_spec.rb +2 -5
  169. data/spec/feeds/yoyo_adservers_spec.rb +47 -0
  170. data/spec/feeds/zeus-domain_reputation_spec.rb +0 -3
  171. data/spec/feeds/zeus-ip_reputation_spec.rb +2 -5
  172. data/spec/spec_helper.rb +2 -0
  173. data/spec/support/factories/event.rb +11 -7
  174. data/spec/support/factories/feed.rb +28 -1
  175. data/spec/support/factories/ipv4.rb +36 -0
  176. data/spec/support/factories/url.rb +34 -0
  177. data/spec/support/shared/feed_runner_observer.rb +136 -0
  178. data/spec/support/shared/feeds.rb +19 -4
  179. data/spec/support/shared/model/collection.rb +164 -0
  180. data/spec/threatinator/actions/run/action_spec.rb +27 -10
  181. data/spec/threatinator/actions/run/coverage_observer_spec.rb +39 -4
  182. data/spec/threatinator/actions/run/status_observer_spec.rb +86 -0
  183. data/spec/threatinator/event_builder_spec.rb +111 -21
  184. data/spec/threatinator/event_spec.rb +237 -13
  185. data/spec/threatinator/event_spec.rb.new +319 -0
  186. data/spec/threatinator/feed_builder_spec.rb +0 -3
  187. data/spec/threatinator/feed_runner_spec.rb +254 -70
  188. data/spec/threatinator/logger_spec.rb +29 -0
  189. data/spec/threatinator/model/observables/fqdn_collection_spec.rb +42 -0
  190. data/spec/threatinator/model/observables/ipv4_collection_spec.rb +36 -0
  191. data/spec/threatinator/model/observables/ipv4_spec.rb +75 -0
  192. data/spec/threatinator/model/observables/url_collection_spec.rb +45 -0
  193. data/spec/threatinator/model/validations/type_spec.rb +37 -0
  194. data/spec/threatinator/plugins/output/csv_spec.rb +4 -3
  195. metadata +216 -19
  196. data/lib/threatinator/property_definer.rb +0 -101
  197. data/spec/threatinator/property_definer_spec.rb +0 -155
@@ -1,101 +0,0 @@
1
-
2
- module Threatinator
3
- # A helper that lets us easily define properties within an object and
4
- # validate them.
5
- module PropertyDefiner
6
- def self.included(base)
7
- base.extend(Threatinator::PropertyDefiner::ClassMethods)
8
- end
9
-
10
- def _properties
11
- @_properties ||= Hash.new
12
- end
13
-
14
- def _prop(name)
15
- self.class._properties[name]
16
- end
17
-
18
- def _get(name)
19
- if _properties.has_key?(name)
20
- return _properties[name]
21
- end
22
- prop = _prop(name)
23
- return nil if prop.nil?
24
- val = prop.default_value
25
- _set(name, val)
26
- val
27
- end
28
-
29
- def _set(name, val)
30
- prop = _prop(name)
31
- return if prop.nil?
32
- prop.validate!(self, val)
33
- _properties[name] = val
34
- end
35
-
36
- def _parse_properties(opts = {})
37
- opts.each { |k, v| _set(k, v) }
38
- end
39
-
40
- class Property
41
- attr_reader :name, :opts
42
- def initialize(name, opts = {})
43
- @name = name.to_sym
44
- @type = opts.delete(:type) || Object
45
- if @validator = opts.delete(:validate)
46
- unless @validator.kind_of?(Proc)
47
- raise ArgumentError.new(":validate must be a proc")
48
- end
49
- end
50
- @default_value = opts.delete(:default)
51
- @opts = opts
52
- end
53
-
54
- def default_value
55
- case @default_value
56
- when Proc
57
- return @default_value.call()
58
- when nil
59
- return nil
60
- else
61
- return @default_value
62
- end
63
- end
64
-
65
- def valid?(obj, val)
66
- return false unless val.kind_of?(@type)
67
- unless @validator.nil?
68
- return false unless @validator.call(obj, val)
69
- end
70
- true
71
- end
72
-
73
- def validate!(obj, val)
74
- unless valid?(obj, val)
75
- raise Threatinator::Exceptions::InvalidAttributeError.new(self.name, val)
76
- end
77
- end
78
- end
79
-
80
- module ClassMethods
81
- def _properties
82
- @@properties ||= {}
83
- end
84
-
85
- def _define_property(prop)
86
- name = prop.name
87
- _properties[name] = prop
88
- define_method("#{prop.name}=".to_sym) do |newval|
89
- _set(name, newval)
90
- end
91
- define_method(name) do
92
- _get(name)
93
- end
94
- end
95
-
96
- def property(name, opts = {})
97
- _define_property(Property.new(name, opts))
98
- end
99
- end
100
- end
101
- end
@@ -1,155 +0,0 @@
1
- require 'spec_helper'
2
- require 'threatinator/property_definer'
3
-
4
- describe Threatinator::PropertyDefiner do
5
- describe "defining a property" do
6
- let(:klass) {
7
- Class.new do
8
- include Threatinator::PropertyDefiner
9
- property :my_prop
10
- end
11
- }
12
- it "should add a setter for the property you define" do
13
- klass = Class.new do
14
- include Threatinator::PropertyDefiner
15
- property :some_prop
16
- end
17
- expect(klass.new).to respond_to(:some_prop=)
18
- end
19
- it "should add a getter for the property you define" do
20
- klass = Class.new do
21
- include Threatinator::PropertyDefiner
22
- property :another_prop
23
- end
24
- expect(klass.new).to respond_to(:another_prop)
25
- end
26
- end
27
-
28
- describe "getting and setting properties" do
29
- let(:klass) {
30
- Class.new do
31
- include Threatinator::PropertyDefiner
32
- property :my_prop
33
- end
34
- }
35
- it "should let me set the value of a property and retrieve it" do
36
- instance = klass.new
37
- instance.my_prop = 1234
38
- expect(instance.my_prop).to eq(1234)
39
- end
40
- it "the getter should respond with nil by default" do
41
- instance = klass.new
42
- expect(instance.my_prop).to be_nil
43
- end
44
- end
45
- describe ":type (type validation)" do
46
- context "when :type is set to a class" do
47
- let(:klass) {
48
- Class.new do
49
- include Threatinator::PropertyDefiner
50
- property :my_prop, type: Integer
51
- end
52
- }
53
- let(:instance) { klass.new }
54
-
55
- it "should not raise an error if the property is set to a instance of the specified class" do
56
- expect{instance.my_prop = 1234}.not_to raise_error
57
- end
58
-
59
- it "should raise an InvalidAttributeError if the property being set is not of its defined type" do
60
- expect{instance.my_prop = "asdf"}.to raise_error do |e|
61
- expect(e).to be_a(Threatinator::Exceptions::InvalidAttributeError)
62
- expect(e.attribute).to eq(:my_prop)
63
- end
64
- end
65
- end
66
- end
67
-
68
- describe ":validate" do
69
- it "should raise an ArgumentError if :validate is not a proc" do
70
- expect {
71
- Class.new do
72
- include Threatinator::PropertyDefiner
73
- property :my_prop, validate: 1234
74
- end
75
- }.to raise_error(ArgumentError)
76
-
77
- end
78
-
79
- it "should call the validate block whenever the associated attribute is set" do
80
- expect { |b|
81
- klass = Class.new do
82
- include Threatinator::PropertyDefiner
83
- property :my_prop, validate: lambda { |obj, val| b.to_proc.call(obj, val); true }
84
- end
85
- instance = klass.new
86
- instance.my_prop = "value1"
87
- instance.my_prop = "value2"
88
- }.to yield_successive_args(
89
- [kind_of(Threatinator::PropertyDefiner), "value1"],
90
- [kind_of(Threatinator::PropertyDefiner), "value2"])
91
- end
92
-
93
- it "should raise an InvalidAttributeError if the validation block returns false" do
94
- klass = Class.new do
95
- include Threatinator::PropertyDefiner
96
- property :my_prop, validate: lambda { |obj, val| false }
97
- end
98
- instance = klass.new
99
- expect { |b|
100
- instance.my_prop = "value1"
101
- }.to raise_error do |e|
102
- expect(e).to be_a(Threatinator::Exceptions::InvalidAttributeError)
103
- expect(e.attribute).to eq(:my_prop)
104
- end
105
- end
106
- it "should not raise an error if the validation block returns true" do
107
- klass = Class.new do
108
- include Threatinator::PropertyDefiner
109
- property :my_prop, validate: lambda { |obj, val| true }
110
- end
111
- instance = klass.new
112
- expect { |b|
113
- instance.my_prop = "value1"
114
- }.not_to raise_error
115
- end
116
- end
117
-
118
- describe ":default" do
119
- context "when :default is set to a value" do
120
- it "should return the object when accessed via a getter" do
121
- expected_object = "my_val"
122
- klass = Class.new do
123
- include Threatinator::PropertyDefiner
124
- property :my_prop, default: expected_object
125
- end
126
- instance = klass.new
127
- expect(instance.my_prop).to be(expected_object)
128
- end
129
- end
130
- context "when :default is set to a proc" do
131
- it "should not call the proc if a value is provided" do
132
- expect { |b|
133
- klass = Class.new do
134
- include Threatinator::PropertyDefiner
135
- property :my_prop, default: b.to_proc
136
- end
137
- instance = klass.new
138
- instance.my_prop = 1234
139
- }.not_to yield_control
140
- end
141
- it "should only call the proc once, and only when it is accessed" do
142
- expect { |b|
143
- klass = Class.new do
144
- include Threatinator::PropertyDefiner
145
- property :my_prop, default: lambda { b.to_proc.call(); }
146
- end
147
- instance = klass.new
148
- val1 = instance.my_prop
149
- val2 = instance.my_prop
150
- val3 = instance.my_prop
151
- }.to yield_control.exactly(1).times
152
- end
153
- end
154
- end
155
- end