threatinator 0.1.6 → 0.2.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (197) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +13 -2
  3. data/Gemfile +18 -13
  4. data/Rakefile +1 -1
  5. data/VERSION +1 -1
  6. data/feeds/ET_block-ip_reputation.feed +26 -0
  7. data/feeds/ET_openbadlist-ip_reputation.feed +35 -0
  8. data/feeds/bambenek_c2_masterlist-domain_reputation.feed +15 -0
  9. data/feeds/bambenek_c2_masterlist-ip_reputation.feed +15 -0
  10. data/feeds/bambenek_dga_feed-domain_reputation.feed +15 -0
  11. data/feeds/berkeley-ip_reputation.feed +23 -0
  12. data/feeds/bitcash_cz_blacklist.feed +20 -0
  13. data/feeds/botscout-ip_reputation.feed +24 -0
  14. data/feeds/cert_mxpoison-ip_reputation.feed +21 -0
  15. data/feeds/chaosreigns-ip_reputation.feed +36 -0
  16. data/feeds/cydef_torexit-ip_reputation.feed +24 -0
  17. data/feeds/danger_bruteforce-ip_reputation.feed +23 -0
  18. data/feeds/falconcrest-ip_reputation.feed +18 -0
  19. data/feeds/h3x_asprox.feed +17 -0
  20. data/feeds/hosts-file_hphostspartial-domain_reputation.feed +18 -0
  21. data/feeds/infiltrated_vabl-ip_reputation.feed +29 -0
  22. data/feeds/isc_suspicious_high-domain_reputation.feed +25 -0
  23. data/feeds/isc_suspicious_low-domain_reputation.feed +25 -0
  24. data/feeds/isc_suspicious_medium-domain_reputation.feed +25 -0
  25. data/feeds/malwaredomainlist-url_reputation.feed +16 -0
  26. data/feeds/malwaredomains-domain_reputation.feed +27 -0
  27. data/feeds/malwaredomains_dyndns-domain_reputation.feed +27 -0
  28. data/feeds/malwaredomains_justdomains-domain_reputation.feed +18 -0
  29. data/feeds/multiproxy-ip_reputation.feed +20 -0
  30. data/feeds/openphish-url_reputation.feed +22 -0
  31. data/feeds/packetmail_perimeterbad-ip_reputation.feed +26 -0
  32. data/feeds/phishtank.feed +1 -1
  33. data/feeds/sigmaproject_atma.feed +25 -0
  34. data/feeds/sigmaproject_spyware.feed +24 -0
  35. data/feeds/sigmaproject_webexploit.feed +26 -0
  36. data/feeds/snort_bpf-ip_reputation.feed +19 -0
  37. data/feeds/steeman-ip_reputation.feed +19 -0
  38. data/feeds/trustedsec-ip_reputation.feed +17 -0
  39. data/feeds/virbl-ip_reputation.feed +24 -0
  40. data/feeds/vxvault-url_reputation.feed +22 -0
  41. data/feeds/yoyo_adservers-domain_reputation.feed +16 -0
  42. data/lib/threatinator/actions/run/action.rb +15 -3
  43. data/lib/threatinator/actions/run/coverage_observer.rb +12 -7
  44. data/lib/threatinator/actions/run/status_observer.rb +37 -0
  45. data/lib/threatinator/cli.rb +9 -3
  46. data/lib/threatinator/cli/parser.rb +14 -4
  47. data/lib/threatinator/config.rb +1 -0
  48. data/lib/threatinator/config/logger.rb +14 -0
  49. data/lib/threatinator/event.rb +28 -18
  50. data/lib/threatinator/event_builder.rb +52 -23
  51. data/lib/threatinator/exceptions.rb +3 -6
  52. data/lib/threatinator/feed.rb +1 -1
  53. data/lib/threatinator/feed_runner.rb +63 -7
  54. data/lib/threatinator/logger.rb +66 -0
  55. data/lib/threatinator/logging.rb +20 -0
  56. data/lib/threatinator/model/base.rb +23 -0
  57. data/lib/threatinator/model/collection.rb +64 -0
  58. data/lib/threatinator/model/observables/fqdn_collection.rb +13 -0
  59. data/lib/threatinator/model/observables/ipv4.rb +30 -0
  60. data/lib/threatinator/model/observables/ipv4_collection.rb +14 -0
  61. data/lib/threatinator/model/observables/url_collection.rb +16 -0
  62. data/lib/threatinator/model/validations.rb +1 -0
  63. data/lib/threatinator/model/validations/type.rb +21 -0
  64. data/lib/threatinator/plugins/output/csv.rb +20 -9
  65. data/spec/feeds/ET_block-ip_reputation_spec.rb +50 -0
  66. data/spec/feeds/ET_compromised-ip_reputation_spec.rb +2 -5
  67. data/spec/feeds/ET_openbadlist-ip_reputation_spec.rb +56 -0
  68. data/spec/feeds/alienvault-ip_reputation_spec.rb +2 -5
  69. data/spec/feeds/arbor_fastflux-domain_reputation_spec.rb +0 -3
  70. data/spec/feeds/arbor_ssh-ip_reputation_spec.rb +2 -5
  71. data/spec/feeds/autoshun_shunlist_spec.rb +1 -4
  72. data/spec/feeds/bambenek_c2_masterlist-domain_reputation_spec.rb +39 -0
  73. data/spec/feeds/bambenek_c2_masterlist-ip_reputation_spec.rb +39 -0
  74. data/spec/feeds/bambenek_dga_feed-domain_reputation_spec.rb +39 -0
  75. data/spec/feeds/berkeley-ip_reputation_spec.rb +47 -0
  76. data/spec/feeds/bitcash_cz_blacklist-ip_reputation_spec.rb +50 -0
  77. data/spec/feeds/blocklist_de_apache-ip_reputation_spec.rb +2 -5
  78. data/spec/feeds/blocklist_de_bots-ip_reputation_spec.rb +2 -5
  79. data/spec/feeds/blocklist_de_ftp-ip_reputation_spec.rb +2 -5
  80. data/spec/feeds/blocklist_de_imap-ip_reputation_spec.rb +2 -5
  81. data/spec/feeds/blocklist_de_pop3-ip_reputation_spec.rb +2 -5
  82. data/spec/feeds/blocklist_de_proftpd-ip_reputation_spec.rb +2 -5
  83. data/spec/feeds/blocklist_de_sip-ip_reputation_spec.rb +2 -5
  84. data/spec/feeds/blocklist_de_ssh-ip_reputation_spec.rb +2 -5
  85. data/spec/feeds/blocklist_de_strongips-ip_reputation_spec.rb +2 -5
  86. data/spec/feeds/botscout-ip_reputation_spec.rb +50 -0
  87. data/spec/feeds/cert_mxpoison-ip_reputation_spec.rb +47 -0
  88. data/spec/feeds/chaosreigns-ip_reputation_spec.rb +50 -0
  89. data/spec/feeds/ciarmy-ip_reputation_spec.rb +2 -5
  90. data/spec/feeds/cruzit-ip_reputation_spec.rb +2 -5
  91. data/spec/feeds/cydef_torexit-ip_reputation_spec.rb +47 -0
  92. data/spec/feeds/dan_me_uk_torlist-ip_reputation_spec.rb +2 -5
  93. data/spec/feeds/danger_bruteforce-ip_reputation_spec.rb +47 -0
  94. data/spec/feeds/data/ET_block-ip_reputation.txt +80 -0
  95. data/spec/feeds/data/ET_openbadlist-ip_reputation.txt +62 -0
  96. data/spec/feeds/data/bambenek_c2-dommasterlist.csv +30 -0
  97. data/spec/feeds/data/bambenek_c2-ipmasterlist.csv +27 -0
  98. data/spec/feeds/data/bambenek_dga_feed.csv +42 -0
  99. data/spec/feeds/data/berkeley.txt +29 -0
  100. data/spec/feeds/data/bitcash_cz_blacklist.txt +7 -0
  101. data/spec/feeds/data/botscout-ip-reputation.txt +713 -0
  102. data/spec/feeds/data/cert_mxpoison-ip_reputation.txt +17 -0
  103. data/spec/feeds/data/chaosreigns-ip-reputation.txt +26 -0
  104. data/spec/feeds/data/cydef_torexit-ip_reputation.txt +27 -0
  105. data/spec/feeds/data/danger_bruteforce-ip_reputation.txt +12 -0
  106. data/spec/feeds/data/falconcrest_iplist.txt +345 -0
  107. data/spec/feeds/data/h3x_asprox.txt +20 -0
  108. data/spec/feeds/data/hosts-file_hphostspartial_domainlist.txt +24 -0
  109. data/spec/feeds/data/infiltrated_vabl_iplist.txt +33 -0
  110. data/spec/feeds/data/isc_suspicious_high_domainlist.txt +26 -0
  111. data/spec/feeds/data/isc_suspicious_low_domainlist.txt +34 -0
  112. data/spec/feeds/data/isc_suspicious_medium_domainlist.txt +32 -0
  113. data/spec/feeds/data/malwaredomainlist-url-reputation.txt +8 -0
  114. data/spec/feeds/data/malwaredomains_domainlist.txt +24 -0
  115. data/spec/feeds/data/malwaredomains_dyndns_domainlist.txt +34 -0
  116. data/spec/feeds/data/malwaredomains_justdomains_domainlist.txt +18 -0
  117. data/spec/feeds/data/multiproxy_iplist.txt +15 -0
  118. data/spec/feeds/data/openphish-url-reputation.txt +16 -0
  119. data/spec/feeds/data/packetmail_perimeterbad-ip_reputation.txt +44 -0
  120. data/spec/feeds/data/sigmaproject_atma.return.gz +0 -0
  121. data/spec/feeds/data/sigmaproject_spyware.return.gz +0 -0
  122. data/spec/feeds/data/sigmaproject_webexploit.return.gz +0 -0
  123. data/spec/feeds/data/snort_bpf-ip_reputation.txt +16 -0
  124. data/spec/feeds/data/steeman-ip-reputation.txt +13 -0
  125. data/spec/feeds/data/trustedsec-ip-reputation.txt +12 -0
  126. data/spec/feeds/data/virbl-ip_reputation.txt +14 -0
  127. data/spec/feeds/data/vxvault-url-reputation.txt +15 -0
  128. data/spec/feeds/data/yoyo_adservers.txt +25 -0
  129. data/spec/feeds/dshield_attackers-top1000_spec.rb +1 -4
  130. data/spec/feeds/falconcrest-ip_reputation_spec.rb +37 -0
  131. data/spec/feeds/feodo-domain_reputation_spec.rb +0 -3
  132. data/spec/feeds/feodo-ip_reputation_spec.rb +2 -5
  133. data/spec/feeds/h3x_asprox-ip_reputation_spec.rb +50 -0
  134. data/spec/feeds/hosts-file_hphostspartial-domain_reputation_spec.rb +47 -0
  135. data/spec/feeds/infiltrated-ip_reputation_spec.rb +2 -5
  136. data/spec/feeds/infiltrated_vabl-ip_reputation_spec.rb +47 -0
  137. data/spec/feeds/isc_suspicious_high-domain_reputation_spec.rb +47 -0
  138. data/spec/feeds/isc_suspicious_low-domain_reputation_spec.rb +47 -0
  139. data/spec/feeds/isc_suspicious_medium-domain_reputation_spec.rb +47 -0
  140. data/spec/feeds/malc0de-domain_reputation_spec.rb +0 -3
  141. data/spec/feeds/malc0de-ip_reputation_spec.rb +2 -5
  142. data/spec/feeds/malwaredomainlist_url_reputation_spec.rb +50 -0
  143. data/spec/feeds/malwaredomains-domain_reputation_spec.rb +47 -0
  144. data/spec/feeds/malwaredomains_dyndns-domain_reputation_spec.rb +47 -0
  145. data/spec/feeds/malwaredomains_justdomains-domain_reputation_spec.rb +47 -0
  146. data/spec/feeds/mirc-domain_reputation_spec.rb +0 -3
  147. data/spec/feeds/multiproxy-ip_reputation_spec.rb +47 -0
  148. data/spec/feeds/nothink_irc-ip_reputation_spec.rb +2 -5
  149. data/spec/feeds/nothink_ssh-ip_reputation_spec.rb +2 -5
  150. data/spec/feeds/openbl-ip_reputation_spec.rb +2 -5
  151. data/spec/feeds/openphish_url_reputation_spec.rb +50 -0
  152. data/spec/feeds/packetmail_perimeterbad-ip_reputation_spec.rb +47 -0
  153. data/spec/feeds/palevo-domain_reputation_spec.rb +0 -3
  154. data/spec/feeds/palevo-ip_reputation_spec.rb +2 -5
  155. data/spec/feeds/phishtank_spec.rb +2 -5
  156. data/spec/feeds/sigmaproject_atma_spec.rb +63 -0
  157. data/spec/feeds/sigmaproject_spyware_spec.rb +64 -0
  158. data/spec/feeds/sigmaproject_webexploit_spec.rb +63 -0
  159. data/spec/feeds/snort_bpf-ip_reputation_spec.rb +47 -0
  160. data/spec/feeds/spyeye-domain_reputation_spec.rb +0 -3
  161. data/spec/feeds/spyeye-ip_reputation_spec.rb +2 -5
  162. data/spec/feeds/steeman-ip_reputation_spec.rb +50 -0
  163. data/spec/feeds/t-arend-de_ssh-ip_reputation_spec.rb +2 -5
  164. data/spec/feeds/the_haleys_ssh-ip_reputation_spec.rb +2 -5
  165. data/spec/feeds/trustedsec-ip_reputation_spec.rb +47 -0
  166. data/spec/feeds/virbl-ip_reputation_spec.rb +47 -0
  167. data/spec/feeds/vxvault_url_reputation_spec.rb +50 -0
  168. data/spec/feeds/yourcmc_ssh-ip_reputation_spec.rb +2 -5
  169. data/spec/feeds/yoyo_adservers_spec.rb +47 -0
  170. data/spec/feeds/zeus-domain_reputation_spec.rb +0 -3
  171. data/spec/feeds/zeus-ip_reputation_spec.rb +2 -5
  172. data/spec/spec_helper.rb +2 -0
  173. data/spec/support/factories/event.rb +11 -7
  174. data/spec/support/factories/feed.rb +28 -1
  175. data/spec/support/factories/ipv4.rb +36 -0
  176. data/spec/support/factories/url.rb +34 -0
  177. data/spec/support/shared/feed_runner_observer.rb +136 -0
  178. data/spec/support/shared/feeds.rb +19 -4
  179. data/spec/support/shared/model/collection.rb +164 -0
  180. data/spec/threatinator/actions/run/action_spec.rb +27 -10
  181. data/spec/threatinator/actions/run/coverage_observer_spec.rb +39 -4
  182. data/spec/threatinator/actions/run/status_observer_spec.rb +86 -0
  183. data/spec/threatinator/event_builder_spec.rb +111 -21
  184. data/spec/threatinator/event_spec.rb +237 -13
  185. data/spec/threatinator/event_spec.rb.new +319 -0
  186. data/spec/threatinator/feed_builder_spec.rb +0 -3
  187. data/spec/threatinator/feed_runner_spec.rb +254 -70
  188. data/spec/threatinator/logger_spec.rb +29 -0
  189. data/spec/threatinator/model/observables/fqdn_collection_spec.rb +42 -0
  190. data/spec/threatinator/model/observables/ipv4_collection_spec.rb +36 -0
  191. data/spec/threatinator/model/observables/ipv4_spec.rb +75 -0
  192. data/spec/threatinator/model/observables/url_collection_spec.rb +45 -0
  193. data/spec/threatinator/model/validations/type_spec.rb +37 -0
  194. data/spec/threatinator/plugins/output/csv_spec.rb +4 -3
  195. metadata +216 -19
  196. data/lib/threatinator/property_definer.rb +0 -101
  197. data/spec/threatinator/property_definer_spec.rb +0 -155
@@ -1,101 +0,0 @@
1
-
2
- module Threatinator
3
- # A helper that lets us easily define properties within an object and
4
- # validate them.
5
- module PropertyDefiner
6
- def self.included(base)
7
- base.extend(Threatinator::PropertyDefiner::ClassMethods)
8
- end
9
-
10
- def _properties
11
- @_properties ||= Hash.new
12
- end
13
-
14
- def _prop(name)
15
- self.class._properties[name]
16
- end
17
-
18
- def _get(name)
19
- if _properties.has_key?(name)
20
- return _properties[name]
21
- end
22
- prop = _prop(name)
23
- return nil if prop.nil?
24
- val = prop.default_value
25
- _set(name, val)
26
- val
27
- end
28
-
29
- def _set(name, val)
30
- prop = _prop(name)
31
- return if prop.nil?
32
- prop.validate!(self, val)
33
- _properties[name] = val
34
- end
35
-
36
- def _parse_properties(opts = {})
37
- opts.each { |k, v| _set(k, v) }
38
- end
39
-
40
- class Property
41
- attr_reader :name, :opts
42
- def initialize(name, opts = {})
43
- @name = name.to_sym
44
- @type = opts.delete(:type) || Object
45
- if @validator = opts.delete(:validate)
46
- unless @validator.kind_of?(Proc)
47
- raise ArgumentError.new(":validate must be a proc")
48
- end
49
- end
50
- @default_value = opts.delete(:default)
51
- @opts = opts
52
- end
53
-
54
- def default_value
55
- case @default_value
56
- when Proc
57
- return @default_value.call()
58
- when nil
59
- return nil
60
- else
61
- return @default_value
62
- end
63
- end
64
-
65
- def valid?(obj, val)
66
- return false unless val.kind_of?(@type)
67
- unless @validator.nil?
68
- return false unless @validator.call(obj, val)
69
- end
70
- true
71
- end
72
-
73
- def validate!(obj, val)
74
- unless valid?(obj, val)
75
- raise Threatinator::Exceptions::InvalidAttributeError.new(self.name, val)
76
- end
77
- end
78
- end
79
-
80
- module ClassMethods
81
- def _properties
82
- @@properties ||= {}
83
- end
84
-
85
- def _define_property(prop)
86
- name = prop.name
87
- _properties[name] = prop
88
- define_method("#{prop.name}=".to_sym) do |newval|
89
- _set(name, newval)
90
- end
91
- define_method(name) do
92
- _get(name)
93
- end
94
- end
95
-
96
- def property(name, opts = {})
97
- _define_property(Property.new(name, opts))
98
- end
99
- end
100
- end
101
- end
@@ -1,155 +0,0 @@
1
- require 'spec_helper'
2
- require 'threatinator/property_definer'
3
-
4
- describe Threatinator::PropertyDefiner do
5
- describe "defining a property" do
6
- let(:klass) {
7
- Class.new do
8
- include Threatinator::PropertyDefiner
9
- property :my_prop
10
- end
11
- }
12
- it "should add a setter for the property you define" do
13
- klass = Class.new do
14
- include Threatinator::PropertyDefiner
15
- property :some_prop
16
- end
17
- expect(klass.new).to respond_to(:some_prop=)
18
- end
19
- it "should add a getter for the property you define" do
20
- klass = Class.new do
21
- include Threatinator::PropertyDefiner
22
- property :another_prop
23
- end
24
- expect(klass.new).to respond_to(:another_prop)
25
- end
26
- end
27
-
28
- describe "getting and setting properties" do
29
- let(:klass) {
30
- Class.new do
31
- include Threatinator::PropertyDefiner
32
- property :my_prop
33
- end
34
- }
35
- it "should let me set the value of a property and retrieve it" do
36
- instance = klass.new
37
- instance.my_prop = 1234
38
- expect(instance.my_prop).to eq(1234)
39
- end
40
- it "the getter should respond with nil by default" do
41
- instance = klass.new
42
- expect(instance.my_prop).to be_nil
43
- end
44
- end
45
- describe ":type (type validation)" do
46
- context "when :type is set to a class" do
47
- let(:klass) {
48
- Class.new do
49
- include Threatinator::PropertyDefiner
50
- property :my_prop, type: Integer
51
- end
52
- }
53
- let(:instance) { klass.new }
54
-
55
- it "should not raise an error if the property is set to a instance of the specified class" do
56
- expect{instance.my_prop = 1234}.not_to raise_error
57
- end
58
-
59
- it "should raise an InvalidAttributeError if the property being set is not of its defined type" do
60
- expect{instance.my_prop = "asdf"}.to raise_error do |e|
61
- expect(e).to be_a(Threatinator::Exceptions::InvalidAttributeError)
62
- expect(e.attribute).to eq(:my_prop)
63
- end
64
- end
65
- end
66
- end
67
-
68
- describe ":validate" do
69
- it "should raise an ArgumentError if :validate is not a proc" do
70
- expect {
71
- Class.new do
72
- include Threatinator::PropertyDefiner
73
- property :my_prop, validate: 1234
74
- end
75
- }.to raise_error(ArgumentError)
76
-
77
- end
78
-
79
- it "should call the validate block whenever the associated attribute is set" do
80
- expect { |b|
81
- klass = Class.new do
82
- include Threatinator::PropertyDefiner
83
- property :my_prop, validate: lambda { |obj, val| b.to_proc.call(obj, val); true }
84
- end
85
- instance = klass.new
86
- instance.my_prop = "value1"
87
- instance.my_prop = "value2"
88
- }.to yield_successive_args(
89
- [kind_of(Threatinator::PropertyDefiner), "value1"],
90
- [kind_of(Threatinator::PropertyDefiner), "value2"])
91
- end
92
-
93
- it "should raise an InvalidAttributeError if the validation block returns false" do
94
- klass = Class.new do
95
- include Threatinator::PropertyDefiner
96
- property :my_prop, validate: lambda { |obj, val| false }
97
- end
98
- instance = klass.new
99
- expect { |b|
100
- instance.my_prop = "value1"
101
- }.to raise_error do |e|
102
- expect(e).to be_a(Threatinator::Exceptions::InvalidAttributeError)
103
- expect(e.attribute).to eq(:my_prop)
104
- end
105
- end
106
- it "should not raise an error if the validation block returns true" do
107
- klass = Class.new do
108
- include Threatinator::PropertyDefiner
109
- property :my_prop, validate: lambda { |obj, val| true }
110
- end
111
- instance = klass.new
112
- expect { |b|
113
- instance.my_prop = "value1"
114
- }.not_to raise_error
115
- end
116
- end
117
-
118
- describe ":default" do
119
- context "when :default is set to a value" do
120
- it "should return the object when accessed via a getter" do
121
- expected_object = "my_val"
122
- klass = Class.new do
123
- include Threatinator::PropertyDefiner
124
- property :my_prop, default: expected_object
125
- end
126
- instance = klass.new
127
- expect(instance.my_prop).to be(expected_object)
128
- end
129
- end
130
- context "when :default is set to a proc" do
131
- it "should not call the proc if a value is provided" do
132
- expect { |b|
133
- klass = Class.new do
134
- include Threatinator::PropertyDefiner
135
- property :my_prop, default: b.to_proc
136
- end
137
- instance = klass.new
138
- instance.my_prop = 1234
139
- }.not_to yield_control
140
- end
141
- it "should only call the proc once, and only when it is accessed" do
142
- expect { |b|
143
- klass = Class.new do
144
- include Threatinator::PropertyDefiner
145
- property :my_prop, default: lambda { b.to_proc.call(); }
146
- end
147
- instance = klass.new
148
- val1 = instance.my_prop
149
- val2 = instance.my_prop
150
- val3 = instance.my_prop
151
- }.to yield_control.exactly(1).times
152
- end
153
- end
154
- end
155
- end