opentpx 2.2.0.17

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA1:
3
+ metadata.gz: 628e72feecaccbf2b9c8e954ffde77943b114994
4
+ data.tar.gz: f25599ac25d14c33ed77f97b40c14e7aa2d71b72
5
+ SHA512:
6
+ metadata.gz: ffaefd096d67a70fd88613d3a569cd41d7caf25dfd8acb51c135f1500194fa8ff08c4ecc42b50f3dc2459b41adc41da131350c58a716e5a7ac685704e36313b1
7
+ data.tar.gz: 555d8910a2f1b02efe9ccaf5f2f832c271e94d6387a6ff725121c714ce6203e7e15e48f497b3de57f9ae81f90188768c85e1c26e90fe36dd5b5d0f6a5c1b6d96
@@ -0,0 +1,15 @@
1
+ --------------------------------------------------------------------------------------------------------------------------------
2
+ Copyright 2015 LookingGlass Cyber Solutions
3
+
4
+ Licensed under the Apache License, Version 2.0 (the "License");
5
+ you may not use this file except in compliance with the License.
6
+ You may obtain a copy of the License at
7
+
8
+ http://www.apache.org/licenses/LICENSE-2.0
9
+
10
+ Unless required by applicable law or agreed to in writing, software
11
+ distributed under the License is distributed on an "AS IS" BASIS,
12
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ See the License for the specific language governing permissions and
14
+ limitations under the License.
15
+ --------------------------------------------------------------------------------------------------------------------------------
@@ -0,0 +1,44 @@
1
+ # OpenTPX - Threat Partner eXchange
2
+
3
+ OpenTPX is an open-source format and tools for exchanging machine-readable threat intelligence and network security operations data. This is a JSON-based format that allows sharing of data between connected systems.
4
+
5
+ This is the open source Ruby gem developed by Lookingglass Cyber Solutions.
6
+
7
+ ## Installation
8
+
9
+ ### Gem
10
+
11
+ You must use an installed gem to use the validator and parser tools.
12
+
13
+ gem install opentpx
14
+
15
+ To validate a file in your own scripts:
16
+
17
+ require 'opentpx'
18
+ TPX_2_2::Validator.validate_file! "path/to/my/tpx.json"
19
+
20
+ Or use the `opentpx_tools` executable:
21
+
22
+ opentpx_tools validate 'path/to/my/tpx.json'
23
+
24
+ Options allow you to quiet warnings and specify the tpx version. By default, it will verify for the latest TPX version. For more help:
25
+
26
+ opentpx_tools help validate
27
+
28
+
29
+
30
+ --------------------------------------------------------------------------------------------------------------------------------
31
+ Copyright 2015 LookingGlass Cyber Solutions
32
+
33
+ Licensed under the Apache License, Version 2.0 (the "License");
34
+ you may not use this file except in compliance with the License.
35
+ You may obtain a copy of the License at
36
+
37
+ http://www.apache.org/licenses/LICENSE-2.0
38
+
39
+ Unless required by applicable law or agreed to in writing, software
40
+ distributed under the License is distributed on an "AS IS" BASIS,
41
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
42
+ See the License for the specific language governing permissions and limitations under the License.
43
+
44
+ --------------------------------------------------------------------------------------------------------------------------------
@@ -0,0 +1,15 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ unless RUBY_VERSION =~ /^2.1/
4
+ print "This script targets Ruby version 2.1. It appears you have a different Ruby version installed. Would you like to execute anyway? [Y/n]: "
5
+ answer = STDIN.gets.strip
6
+ answer = 'y' if answer == ''
7
+ exit unless answer =~ /y/i
8
+ end
9
+
10
+ $:<< File.join(__dir__, '..', 'lib')
11
+
12
+ require 'tpx'
13
+ require 'tpx/tools'
14
+
15
+ exit TPX::Tools.run(ARGV)
@@ -0,0 +1,7 @@
1
+ $LOAD_PATH << __dir__ unless $LOAD_PATH.include?(__dir__)
2
+
3
+ module TPX
4
+ require 'tpx/version'
5
+ # load the current TPX version
6
+ require 'tpx_2_2'
7
+ end
@@ -0,0 +1,34 @@
1
+ module TPX_2_2
2
+ module AttributeAccessors
3
+
4
+ # Overrides default method_missing to alias method names to hash keys.
5
+ #
6
+ # @param [String] meth The name of the called method.
7
+ # @param [Array<Symbol>] args Additional arguments.
8
+ # @param [Proc] block Additional block.
9
+ #
10
+ # @raise [NoMethodError] Error thrown if method does not reference a hash key.
11
+ #
12
+ # @return [Object] Value in the hash corresponding to the given key.
13
+ def method_missing(meth, *args, &block)
14
+ unless self.keys.find {|k| k.to_sym == meth.to_sym }
15
+ raise NoMethodError, "undefined method `#{meth}' for #{self}"
16
+ end
17
+ self[meth.to_sym]
18
+ end
19
+
20
+ # Returns the list of keys for the hash.
21
+ #
22
+ # @return [Array<String>] The list of hash keys.
23
+ def attributes
24
+ self.keys
25
+ end
26
+
27
+ # Returns the object represented as a string.
28
+ #
29
+ # @return [String] The class, id, and the to_s method of the parent class.
30
+ def to_s
31
+ "<##{self.class}:#{self.object_id} #{super}>"
32
+ end
33
+ end
34
+ end
@@ -0,0 +1,11 @@
1
+ require 'tpx/2_2/data_model'
2
+
3
+ module TPX_2_2
4
+
5
+ # An element in a classification list.
6
+ class ClassificationElement < DataModel
7
+ MANDATORY_ATTRIBUTES = [
8
+ :classification_id_s
9
+ ]
10
+ end
11
+ end
@@ -0,0 +1,11 @@
1
+ require 'tpx/2_2/homogeneous_list'
2
+ require 'tpx/2_2/classification_element'
3
+
4
+ module TPX_2_2
5
+
6
+ # A list of classifications of an observable.
7
+ class ClassificationElementList < HomogeneousList
8
+ homogeneous_list_of ClassificationElement
9
+ children_keyed_by :classification_id_s
10
+ end
11
+ end
@@ -0,0 +1,21 @@
1
+ require 'tpx/2_2/homogeneous_list'
2
+ require 'tpx/2_2/collection_element'
3
+
4
+ # "collection_c_array": [
5
+ # { "name_id_s": "Aruba", "iso_3_s": "abw", "iso_2_s": "aw", "region_code_ui": 0, "continent_code_ui": 6, "continent_code_s": "na", "country_code_ui": 533 },
6
+ # { "name_id_s": "Afghanistan", "iso_3_s": "afg", "iso_2_s": "af", "region_code_ui": 1, "continent_code_ui": 4, "continent_code_s": "as", "country_code_ui": 4 },
7
+ # { "name_id_s": "Angola", "iso_3_s": "ago", "iso_2_s": "ao", "region_code_ui": 1, "continent_code_ui": 1, "continent_code_s": "af", "country_code_ui": 24 },
8
+ # { "name_id_s": "Anguilla", "iso_3_s": "aia", "iso_2_s": "ai", "region_code_ui": 0, "continent_code_ui": 6, "continent_code_s": "na", "country_code_ui": 660 },
9
+ # { "name_id_s": "Aland Islands", "iso_3_s": "ala", "iso_2_s": "ax", "region_code_ui": 0, "continent_code_ui": 5, "continent_code_s": "eu", "country_code_ui": 248 },
10
+ # { "name_id_s": "Albania", "iso_3_s": "alb", "iso_2_s": "al", "region_code_ui": 1, "continent_code_ui": 5, "continent_code_s": "eu", "country_code_ui": 8 }
11
+ # ]
12
+
13
+
14
+ module TPX_2_2
15
+
16
+ # A named group of network elements, host elements or observables.
17
+ class Collection < HomogeneousList
18
+ homogeneous_list_of CollectionElement
19
+ children_keyed_by :name_id_s
20
+ end
21
+ end
@@ -0,0 +1,13 @@
1
+ require 'tpx/2_2/data_model'
2
+
3
+ # { "name_id_s": "Albania", "iso_3_s": "alb", "iso_2_s": "al", "region_code_ui": 1, "continent_code_ui": 5, "continent_code_s": "eu", "country_code_ui": 8 }
4
+
5
+ module TPX_2_2
6
+
7
+ # An element in a classification list.
8
+ class CollectionElement < DataModel
9
+ MANDATORY_ATTRIBUTES = [
10
+ :name_id_s
11
+ ]
12
+ end
13
+ end
@@ -0,0 +1,32 @@
1
+ require 'tpx/2_2/exceptions'
2
+ require 'tpx/2_2/mandatory_attributes'
3
+ require 'tpx/2_2/attribute_accessors'
4
+
5
+ module TPX_2_2
6
+
7
+ # The base class for a TPX dictionary/hash.
8
+ class DataModel < ::HashWithIndifferentAccess
9
+ include AttributeAccessors
10
+ include MandatoryAttributes
11
+
12
+ # Overrides the default initialize to validate the input data.
13
+ #
14
+ # @param input_hash [Hash] The input hash.
15
+ #
16
+ # @return [DataModel] The returned object.
17
+ def initialize(input_hash)
18
+ unless input_hash.is_a? Hash
19
+ raise ValidationError, "Parameter `input_hash` supplied to #{self.class} must be of type Hash (#{input_hash.class}: #{input_hash.inspect})!"
20
+ end
21
+ super input_hash
22
+ validate!
23
+ end
24
+
25
+ # Overrides the default to_h method to return a hash with symbolized keys.
26
+ #
27
+ # @return [Object] The returned hash.
28
+ def to_h
29
+ self.symbolize_keys.to_h
30
+ end
31
+ end
32
+ end
@@ -0,0 +1,41 @@
1
+ require 'tpx/2_2/data_model'
2
+ require 'tpx/2_2/threat_observable'
3
+ require 'tpx/2_2/observable'
4
+
5
+ module TPX_2_2
6
+
7
+ # A set of threat observable associations to one or more subjects
8
+ # (i.e. elements) including network, host or user subjects.
9
+ class ElementObservable < DataModel
10
+
11
+ MANDATORY_ATTRIBUTES = [:threat_observable_c_map]
12
+
13
+ MUST_HAVE_ONE_AND_ONLY_ONE_OF_ATTRIBUTES = [
14
+ [
15
+ :subject_ipv4_s,
16
+ :subject_ipv4_i,
17
+ :subject_ipv4_ui,
18
+ :subject_ipv6_s,
19
+ :subject_ipv6_ll,
20
+ :subject_fqdn_s,
21
+ :subject_cidrv4_s,
22
+ :subject_cidrv6_s,
23
+ :subject_asn_s,
24
+ :subject_asn_ui,
25
+ :subject_md5_h,
26
+ :subject_sha1_h,
27
+ :subject_sha256_h,
28
+ :subject_sha512_h,
29
+ :subject_registrykey_s,
30
+ :subject_filename_s,
31
+ :subject_filepath_s,
32
+ :subject_mutex_s,
33
+ :subject_actor_s,
34
+ :subject_email_s
35
+ ]
36
+ ]
37
+
38
+ SUBJECT_ATTRIBUTES = MUST_HAVE_ONE_AND_ONLY_ONE_OF_ATTRIBUTES.first
39
+
40
+ end # class ElementObservable
41
+ end # module TPX_2_2
@@ -0,0 +1,17 @@
1
+ require 'tpx/2_2/merging_heterogeneous_list'
2
+ require 'tpx/2_2/element_observable'
3
+
4
+ module TPX_2_2
5
+
6
+ # A list of element_observables.
7
+ class ElementObservableList < MergingHeterogeneousList
8
+
9
+ children_of_class_and_id(
10
+ ElementObservable::SUBJECT_ATTRIBUTES.map do |subject_attribute_name|
11
+ [ElementObservable, subject_attribute_name]
12
+ end
13
+ )
14
+ on_duplicate_addition_merge_attributes [:threat_observable_c_map]
15
+
16
+ end # class
17
+ end
@@ -0,0 +1,13 @@
1
+ module TPX_2_2
2
+ # Warning thrown when validation pass but there's some information worth passing to the caller
3
+ class ValidationWarning < RuntimeError; end
4
+
5
+ # Error thrown when an instantialized object invalid.
6
+ class ValidationError < RuntimeError; end
7
+
8
+ # Error thrown when a duplicate element is inserted into a list.
9
+ class DuplicateElementInsertError < RuntimeError; end
10
+
11
+ # Error thrown when a required attribute is not found.
12
+ class AttributeDefinitionError < StandardError; end
13
+ end
@@ -0,0 +1,220 @@
1
+ require 'tpx/2_2/validator'
2
+ require 'tpx/2_2/data_model'
3
+ require 'tpx/2_2/observable_dictionary'
4
+ require 'tpx/2_2/element_observable_list'
5
+ require 'tpx/2_2/network_list'
6
+ require 'tpx/2_2/collection'
7
+
8
+ module TPX_2_2
9
+
10
+ # An object representing a TPX file, holding all associated data.
11
+ class Exchange < DataModel
12
+
13
+ DEFAULT_MAX_FILESIZE = 1024*1024*1024
14
+
15
+ ELEMENT_LISTS = {
16
+ observable_dictionary_c_array: [ObservableDictionary, ObservableDefinition, :dictionary_file_manifest, 'data_dictionary'],
17
+ element_observable_c_array: [ElementObservableList, ElementObservable, :observable_element_file_manifest, 'data'],
18
+ asn_c_array: [NetworkList, Network, :asn_file_manifest, 'asn'],
19
+ collection_c_array: [Collection, CollectionElement, :collection_file_manifest, 'collection']
20
+ }
21
+
22
+ MANDATORY_ATTRIBUTES = [
23
+ :schema_version_s,
24
+ :provider_s,
25
+ :source_observable_s,
26
+ :last_updated_t,
27
+ :list_name_s
28
+ ]
29
+
30
+ class << self
31
+ # Initialize a TPX_2_2::Exchange from a given filename.
32
+ #
33
+ # @param [String] filename The filename containing the exchange. This does not call Validator.validate!
34
+ def init_file(filename)
35
+ h = Oj.load_file(filename)
36
+ self.new(h)
37
+ end
38
+
39
+ # Imports a tpx file from a given filename.
40
+ #
41
+ # @param [String] filename The filename to import.
42
+ def import_file(filename)
43
+ h = Oj.load_file(filename)
44
+ import(h)
45
+ end
46
+
47
+ # Imports a tpx file from a given json string.
48
+ #
49
+ # @param [String] str The string of json to import.
50
+ def import_s(str)
51
+ h = Oj.load(str)
52
+ import(h)
53
+ end
54
+
55
+ # Imports a tpx file from a given hash.
56
+ #
57
+ # @param [Hash] input_hash The hash to import.
58
+ def import(input_hash)
59
+ Validator.validate!(input_hash)
60
+ self.new(input_hash)
61
+ end
62
+ end
63
+
64
+ # Overrides the default initialize to validate the input data.
65
+ #
66
+ # @param [Hash] input_hash The input hash.
67
+ #
68
+ # @return [DataModel] The returned object.
69
+ def initialize(input_hash)
70
+ input_hash = ::HashWithIndifferentAccess.new(input_hash)
71
+ input_hash[:schema_version_s] = TPX_2_2::CURRENT_SCHEMA_VERSION
72
+ input_hash[:last_updated_t] ||= Time.now.utc.to_i
73
+ input_hash[:observable_dictionary_c_array] ||= []
74
+ input_hash[:source_description_s] ||= ''
75
+
76
+ has_one_list = false
77
+ ELEMENT_LISTS.keys.each do |list_key|
78
+ has_list = false
79
+ has_manifest = false
80
+ if input_hash.has_key?(list_key) && input_hash.has_key?(ELEMENT_LISTS[list_key][2])
81
+ raise ValidationError, "Only one of #{list_key} or #{ELEMENT_LISTS[list_key][2]} should be supplied to #{self.class}#initialize input_hash."
82
+ elsif input_hash.has_key?(list_key)
83
+ has_one_list = true
84
+ input_hash[list_key] = ELEMENT_LISTS[list_key][0].new(input_hash[list_key])
85
+ elsif input_hash.has_key?(ELEMENT_LISTS[list_key][2])
86
+ has_one_list = true
87
+ end
88
+ end
89
+
90
+ unless has_one_list
91
+ raise ValidationError, "At list one list element (#{ELEMENT_LISTS.keys}) should be supplied to #{self.class}#initialize."
92
+ end
93
+
94
+ super input_hash
95
+ end
96
+
97
+ # Overrides default << method to add data to the exchange. Checks
98
+ # that added elements are of the correct class.
99
+ #
100
+ # @param element [Object] Element to add to the exchange.
101
+ #
102
+ # @return [Object] The updated Exchange.
103
+ def <<(element)
104
+ if element.is_a? Array
105
+ element.each do |e|
106
+ self << e
107
+ end
108
+ return self
109
+ end
110
+
111
+ element_type_supported = false
112
+
113
+ ELEMENT_LISTS.each do |list_key, list_def|
114
+ list_type, list_element, list_manifest_key, list_manifest_file_type = *list_def
115
+ if element.class == list_element
116
+ self[list_key] ||= list_type.new([])
117
+ self[list_key] << element
118
+ element_type_supported = true
119
+ end
120
+ end
121
+
122
+ unless element_type_supported
123
+ raise ValidationError, "Element provided to #{self.class}#<< has invalid object type (#{element.class})!"
124
+ end
125
+
126
+ return self
127
+ end
128
+
129
+ # Returns the exchange with empty elements deleted.
130
+ #
131
+ # @param [Hash] h The hash from which to scrub empty elements.
132
+ #
133
+ # @return [Object] The hash with deleted empty elements.
134
+ def _to_h_scrub(h)
135
+ h_scrub = h.dup
136
+ [:observable_dictionary_c_array, :element_observable_c_array, :asn_c_array, :collection_c_array].each do |key|
137
+ h_scrub.delete(key) if h_scrub.has_key? key && h_scrub[key].blank?
138
+ end
139
+ return h_scrub
140
+ end
141
+
142
+ # Alias for _to_h_scrub.
143
+ def to_h
144
+ _to_h_scrub(super)
145
+ end
146
+
147
+ # Alias for _to_h_scrub.
148
+ def to_hash
149
+ _to_h_scrub(super)
150
+ end
151
+
152
+ # Exports the current exchange to a tpx file.
153
+ #
154
+ # @param [String] filepath The file to be exported.
155
+ # @param [Hash] options Additional options to be passed to the json exporter.
156
+ def to_tpx_file(filepath, options={})
157
+ data = (manifest_files_count > 1) ? self.to_manifest(filepath) : self
158
+ Oj.to_file(filepath, data, options.merge({mode: :compat}))
159
+ @manifest_files_count = nil
160
+ end
161
+
162
+
163
+ private
164
+
165
+ def enumerable?(container)
166
+ container.respond_to? :each
167
+ end
168
+
169
+ def serializable?(element)
170
+ element.respond_to? :to_hash
171
+ end
172
+
173
+ def manifest_files_count
174
+ @manifest_files_count ||= estimated_tpx_file_size / DEFAULT_MAX_FILESIZE
175
+ end
176
+
177
+ def estimated_tpx_file_size
178
+ Oj.dump(self).size
179
+ end
180
+
181
+ def split_section(section)
182
+ split_count = section.size / manifest_files_count
183
+ section.each_slice(split_count > 1 ? split_count : 1).to_a
184
+ end
185
+
186
+ def to_manifest_section(exchange_section, manifest_file_type, path)
187
+ subsections = split_section(exchange_section)
188
+ files = []
189
+
190
+ subsections.each_with_index do |item, index|
191
+ # save subsection as TPX file
192
+ files << "#{manifest_file_type}_#{index + 1}.json"
193
+ Oj.to_file(File.join(path, files.last), item)
194
+ end
195
+ files
196
+ end
197
+
198
+ def to_manifest(manifest_file_path)
199
+ exchange_hash = self.to_hash
200
+ manifest_hash = {}
201
+
202
+ keys = ['observable_dictionary_c_array', 'element_observable_c_array', 'asn_c_array', 'collection_c_array']
203
+ path = File.dirname(manifest_file_path)
204
+
205
+ exchange_hash.each do |key, val|
206
+ manifest_hash[key] = val unless keys.include?(key)
207
+ end
208
+
209
+ ELEMENT_LISTS.each do |list_key, list_def|
210
+ list_type, list_element, list_manifest_key, list_manifest_file_type = *list_def
211
+ if exchange_hash.has_key? list_key.to_s
212
+ manifest_hash[list_manifest_key.to_s] = to_manifest_section(exchange_hash[list_key.to_s], list_manifest_file_type.to_s, path) unless exchange_hash[list_key.to_s].empty?
213
+ end
214
+ end
215
+
216
+ manifest_hash
217
+ end
218
+
219
+ end # class Exchange
220
+ end # module TPX_2_2