opentpx 2.2.0.17

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 628e72feecaccbf2b9c8e954ffde77943b114994
+  data.tar.gz: f25599ac25d14c33ed77f97b40c14e7aa2d71b72
+SHA512:
+  metadata.gz: ffaefd096d67a70fd88613d3a569cd41d7caf25dfd8acb51c135f1500194fa8ff08c4ecc42b50f3dc2459b41adc41da131350c58a716e5a7ac685704e36313b1
+  data.tar.gz: 555d8910a2f1b02efe9ccaf5f2f832c271e94d6387a6ff725121c714ce6203e7e15e48f497b3de57f9ae81f90188768c85e1c26e90fe36dd5b5d0f6a5c1b6d96

data/LICENSE.txt

@@ -0,0 +1,15 @@
+--------------------------------------------------------------------------------------------------------------------------------
+Copyright 2015 LookingGlass Cyber Solutions
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+--------------------------------------------------------------------------------------------------------------------------------

data/README.md

@@ -0,0 +1,44 @@
+# OpenTPX - Threat Partner eXchange
+
+OpenTPX is an open-source format and tools for exchanging machine-readable threat intelligence and network security operations data.  This is a JSON-based format that allows sharing of data between connected systems.
+
+This is the open source Ruby gem developed by Lookingglass Cyber Solutions.
+
+## Installation
+
+### Gem
+
+You must use an installed gem to use the validator and parser tools.
+
+    gem install opentpx
+
+To validate a file in your own scripts:
+
+    require 'opentpx'
+    TPX_2_2::Validator.validate_file! "path/to/my/tpx.json"
+
+Or use the `opentpx_tools` executable:
+
+    opentpx_tools validate 'path/to/my/tpx.json'
+
+Options allow you to quiet warnings and specify the tpx version. By default, it will verify for the latest TPX version. For more help:
+
+    opentpx_tools help validate
+
+
+
+--------------------------------------------------------------------------------------------------------------------------------
+Copyright 2015 LookingGlass Cyber Solutions
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and limitations under the License.
+
+--------------------------------------------------------------------------------------------------------------------------------

data/bin/opentpx_tools

@@ -0,0 +1,15 @@
+#!/usr/bin/env ruby
+
+unless RUBY_VERSION =~ /^2.1/
+  print "This script targets Ruby version 2.1. It appears you have a different Ruby version installed.  Would you like to execute anyway? [Y/n]: "
+  answer = STDIN.gets.strip
+  answer = 'y' if answer == ''
+  exit unless answer =~ /y/i
+end
+
+$:<< File.join(__dir__, '..', 'lib')
+
+require 'tpx'
+require 'tpx/tools'
+
+exit TPX::Tools.run(ARGV)

data/lib/tpx.rb

@@ -0,0 +1,7 @@
+$LOAD_PATH << __dir__ unless $LOAD_PATH.include?(__dir__)
+
+module TPX
+  require 'tpx/version'
+  # load the current TPX version
+  require 'tpx_2_2'
+end

data/lib/tpx/2_2/attribute_accessors.rb

@@ -0,0 +1,34 @@
+module TPX_2_2
+  module AttributeAccessors
+
+    # Overrides default method_missing to alias method names to hash keys.
+    #
+    # @param [String] meth The name of the called method.
+    # @param [Array<Symbol>] args Additional arguments.
+    # @param [Proc] block Additional block.
+    #
+    # @raise [NoMethodError] Error thrown if method does not reference a hash key.
+    #
+    # @return [Object] Value in the hash corresponding to the given key.
+    def method_missing(meth, *args, &block)
+      unless self.keys.find {|k| k.to_sym == meth.to_sym }
+        raise NoMethodError, "undefined method `#{meth}' for #{self}"
+      end
+      self[meth.to_sym]
+    end
+
+    # Returns the list of keys for the hash.
+    #
+    # @return [Array<String>] The list of hash keys.
+    def attributes
+      self.keys
+    end
+
+    # Returns the object represented as a string.
+    #
+    # @return [String] The class, id, and the to_s method of the parent class.
+    def to_s
+      "<##{self.class}:#{self.object_id} #{super}>"
+    end
+  end
+end

data/lib/tpx/2_2/classification_element.rb

@@ -0,0 +1,11 @@
+require 'tpx/2_2/data_model'
+
+module TPX_2_2
+
+  # An element in a classification list.
+  class ClassificationElement < DataModel
+    MANDATORY_ATTRIBUTES = [
+      :classification_id_s
+    ]
+  end
+end

data/lib/tpx/2_2/classification_element_list.rb

@@ -0,0 +1,11 @@
+require 'tpx/2_2/homogeneous_list'
+require 'tpx/2_2/classification_element'
+
+module TPX_2_2
+
+  # A list of classifications of an observable.
+  class ClassificationElementList < HomogeneousList
+    homogeneous_list_of ClassificationElement
+    children_keyed_by :classification_id_s
+  end
+end

data/lib/tpx/2_2/collection.rb

@@ -0,0 +1,21 @@
+require 'tpx/2_2/homogeneous_list'
+require 'tpx/2_2/collection_element'
+
+# "collection_c_array": [
+#     { "name_id_s": "Aruba", "iso_3_s": "abw", "iso_2_s": "aw", "region_code_ui": 0, "continent_code_ui": 6, "continent_code_s": "na", "country_code_ui": 533 },
+#     { "name_id_s": "Afghanistan", "iso_3_s": "afg", "iso_2_s": "af", "region_code_ui": 1, "continent_code_ui": 4, "continent_code_s": "as", "country_code_ui": 4 },
+#     { "name_id_s": "Angola", "iso_3_s": "ago", "iso_2_s": "ao", "region_code_ui": 1, "continent_code_ui": 1, "continent_code_s": "af", "country_code_ui": 24 },
+#     { "name_id_s": "Anguilla", "iso_3_s": "aia", "iso_2_s": "ai", "region_code_ui": 0, "continent_code_ui": 6, "continent_code_s": "na", "country_code_ui": 660 },
+#     { "name_id_s": "Aland Islands", "iso_3_s": "ala", "iso_2_s": "ax", "region_code_ui": 0, "continent_code_ui": 5, "continent_code_s": "eu", "country_code_ui": 248 },
+#     { "name_id_s": "Albania", "iso_3_s": "alb", "iso_2_s": "al", "region_code_ui": 1, "continent_code_ui": 5, "continent_code_s": "eu", "country_code_ui": 8 }
+# ]
+
+
+module TPX_2_2
+
+  # A named group of network elements, host elements or observables.
+  class Collection < HomogeneousList
+    homogeneous_list_of CollectionElement
+    children_keyed_by :name_id_s
+  end
+end

data/lib/tpx/2_2/collection_element.rb

@@ -0,0 +1,13 @@
+require 'tpx/2_2/data_model'
+
+# { "name_id_s": "Albania", "iso_3_s": "alb", "iso_2_s": "al", "region_code_ui": 1, "continent_code_ui": 5, "continent_code_s": "eu", "country_code_ui": 8 }
+
+module TPX_2_2
+
+  # An element in a classification list.
+  class CollectionElement < DataModel
+    MANDATORY_ATTRIBUTES = [
+      :name_id_s
+    ]
+  end
+end

data/lib/tpx/2_2/data_model.rb

@@ -0,0 +1,32 @@
+require 'tpx/2_2/exceptions'
+require 'tpx/2_2/mandatory_attributes'
+require 'tpx/2_2/attribute_accessors'
+
+module TPX_2_2
+
+  # The base class for a TPX dictionary/hash.
+  class DataModel < ::HashWithIndifferentAccess
+    include AttributeAccessors
+    include MandatoryAttributes
+
+    # Overrides the default initialize to validate the input data.
+    #
+    # @param input_hash [Hash] The input hash.
+    #
+    # @return [DataModel] The returned object.
+    def initialize(input_hash)
+      unless input_hash.is_a? Hash
+        raise ValidationError, "Parameter `input_hash` supplied to #{self.class} must be of type Hash (#{input_hash.class}: #{input_hash.inspect})!"
+      end
+      super input_hash
+      validate!
+    end
+
+    # Overrides the default to_h method to return a hash with symbolized keys.
+    #
+    # @return [Object] The returned hash.
+    def to_h
+      self.symbolize_keys.to_h
+    end
+  end
+end

data/lib/tpx/2_2/element_observable.rb

@@ -0,0 +1,41 @@
+require 'tpx/2_2/data_model'
+require 'tpx/2_2/threat_observable'
+require 'tpx/2_2/observable'
+
+module TPX_2_2
+
+  # A set of threat observable associations to one or more subjects
+  # (i.e. elements) including network, host or user subjects.
+  class ElementObservable < DataModel
+
+    MANDATORY_ATTRIBUTES = [:threat_observable_c_map]
+
+    MUST_HAVE_ONE_AND_ONLY_ONE_OF_ATTRIBUTES = [
+      [
+        :subject_ipv4_s,
+        :subject_ipv4_i,
+        :subject_ipv4_ui,
+        :subject_ipv6_s,
+        :subject_ipv6_ll,
+        :subject_fqdn_s,
+        :subject_cidrv4_s,
+        :subject_cidrv6_s,
+        :subject_asn_s,
+        :subject_asn_ui,
+        :subject_md5_h,
+        :subject_sha1_h,
+        :subject_sha256_h,
+        :subject_sha512_h,
+        :subject_registrykey_s,
+        :subject_filename_s,
+        :subject_filepath_s,
+        :subject_mutex_s,
+        :subject_actor_s,
+        :subject_email_s
+      ]
+    ]
+
+    SUBJECT_ATTRIBUTES = MUST_HAVE_ONE_AND_ONLY_ONE_OF_ATTRIBUTES.first
+
+  end # class ElementObservable
+end # module TPX_2_2

data/lib/tpx/2_2/element_observable_list.rb

@@ -0,0 +1,17 @@
+require 'tpx/2_2/merging_heterogeneous_list'
+require 'tpx/2_2/element_observable'
+
+module TPX_2_2
+
+  # A list of element_observables.
+  class ElementObservableList < MergingHeterogeneousList
+
+    children_of_class_and_id(
+      ElementObservable::SUBJECT_ATTRIBUTES.map do |subject_attribute_name|
+        [ElementObservable, subject_attribute_name]
+      end
+    )
+    on_duplicate_addition_merge_attributes [:threat_observable_c_map]
+
+  end # class
+end

data/lib/tpx/2_2/exceptions.rb

@@ -0,0 +1,13 @@
+module TPX_2_2
+  # Warning thrown when validation pass but there's some information worth passing to the caller
+  class ValidationWarning < RuntimeError; end
+
+  # Error thrown when an instantialized object invalid.
+  class ValidationError < RuntimeError; end
+
+  # Error thrown when a duplicate element is inserted into a list.
+  class DuplicateElementInsertError < RuntimeError; end
+
+  # Error thrown when a required attribute is not found.
+  class AttributeDefinitionError < StandardError; end
+end

data/lib/tpx/2_2/exchange.rb

@@ -0,0 +1,220 @@
+require 'tpx/2_2/validator'
+require 'tpx/2_2/data_model'
+require 'tpx/2_2/observable_dictionary'
+require 'tpx/2_2/element_observable_list'
+require 'tpx/2_2/network_list'
+require 'tpx/2_2/collection'
+
+module TPX_2_2
+
+  # An object representing a TPX file, holding all associated data.
+  class Exchange < DataModel
+
+    DEFAULT_MAX_FILESIZE = 1024*1024*1024
+
+    ELEMENT_LISTS = {
+      observable_dictionary_c_array: [ObservableDictionary, ObservableDefinition, :dictionary_file_manifest, 'data_dictionary'],
+      element_observable_c_array: [ElementObservableList, ElementObservable, :observable_element_file_manifest, 'data'],
+      asn_c_array: [NetworkList, Network, :asn_file_manifest, 'asn'],
+      collection_c_array: [Collection, CollectionElement, :collection_file_manifest, 'collection']
+    }
+
+    MANDATORY_ATTRIBUTES = [
+      :schema_version_s,
+      :provider_s,
+      :source_observable_s,
+      :last_updated_t,
+      :list_name_s
+    ]
+
+    class << self
+      # Initialize a TPX_2_2::Exchange from a given filename.
+      #
+      # @param [String] filename The filename containing the exchange.  This does not call Validator.validate!
+      def init_file(filename)
+        h = Oj.load_file(filename)
+        self.new(h)
+      end
+
+      # Imports a tpx file from a given filename.
+      #
+      # @param [String] filename The filename to import.
+      def import_file(filename)
+        h = Oj.load_file(filename)
+        import(h)
+      end
+
+      # Imports a tpx file from a given json string.
+      #
+      # @param [String] str The string of json to import.
+      def import_s(str)
+        h = Oj.load(str)
+        import(h)
+      end
+
+      # Imports a tpx file from a given hash.
+      #
+      # @param [Hash] input_hash The hash to import.
+      def import(input_hash)
+        Validator.validate!(input_hash)
+        self.new(input_hash)
+      end
+    end
+
+    # Overrides the default initialize to validate the input data.
+    #
+    # @param [Hash] input_hash The input hash.
+    #
+    # @return [DataModel] The returned object.
+    def initialize(input_hash)
+      input_hash = ::HashWithIndifferentAccess.new(input_hash)
+      input_hash[:schema_version_s] = TPX_2_2::CURRENT_SCHEMA_VERSION
+      input_hash[:last_updated_t] ||= Time.now.utc.to_i
+      input_hash[:observable_dictionary_c_array] ||= []
+      input_hash[:source_description_s] ||= ''
+
+      has_one_list = false
+      ELEMENT_LISTS.keys.each do |list_key|
+        has_list = false
+        has_manifest = false
+        if input_hash.has_key?(list_key) && input_hash.has_key?(ELEMENT_LISTS[list_key][2])
+          raise ValidationError, "Only one of #{list_key} or #{ELEMENT_LISTS[list_key][2]} should be supplied to #{self.class}#initialize input_hash."
+        elsif input_hash.has_key?(list_key)
+          has_one_list = true
+          input_hash[list_key] = ELEMENT_LISTS[list_key][0].new(input_hash[list_key])
+        elsif input_hash.has_key?(ELEMENT_LISTS[list_key][2])
+          has_one_list = true
+        end
+      end
+
+      unless has_one_list
+        raise ValidationError, "At list one list element (#{ELEMENT_LISTS.keys}) should be supplied to #{self.class}#initialize."
+      end
+
+      super input_hash
+    end
+
+    # Overrides default << method to add data to the exchange. Checks
+    # that added elements are of the correct class.
+    #
+    # @param element [Object] Element to add to the exchange.
+    #
+    # @return [Object] The updated Exchange.
+    def <<(element)
+      if element.is_a? Array
+        element.each do |e|
+          self << e
+        end
+        return self
+      end
+
+      element_type_supported = false
+
+      ELEMENT_LISTS.each do |list_key, list_def|
+        list_type, list_element, list_manifest_key, list_manifest_file_type = *list_def
+        if element.class == list_element
+          self[list_key] ||= list_type.new([])
+          self[list_key] << element
+          element_type_supported = true
+        end
+      end
+
+      unless element_type_supported
+        raise ValidationError, "Element provided to #{self.class}#<< has invalid object type (#{element.class})!"
+      end
+
+      return self
+    end
+
+    # Returns the exchange with empty elements deleted.
+    #
+    # @param [Hash] h The hash from which to scrub empty elements.
+    #
+    # @return [Object] The hash with deleted empty elements.
+    def _to_h_scrub(h)
+      h_scrub = h.dup
+      [:observable_dictionary_c_array, :element_observable_c_array, :asn_c_array, :collection_c_array].each do |key|
+        h_scrub.delete(key) if h_scrub.has_key? key && h_scrub[key].blank?
+      end
+      return h_scrub
+    end
+
+    # Alias for _to_h_scrub.
+    def to_h
+      _to_h_scrub(super)
+    end
+
+    # Alias for _to_h_scrub.
+    def to_hash
+      _to_h_scrub(super)
+    end
+
+    # Exports the current exchange to a tpx file.
+    #
+    # @param [String] filepath The file to be exported.
+    # @param [Hash] options Additional options to be passed to the json exporter.
+    def to_tpx_file(filepath, options={})
+      data = (manifest_files_count > 1) ? self.to_manifest(filepath) : self
+      Oj.to_file(filepath, data, options.merge({mode: :compat}))
+      @manifest_files_count = nil
+    end
+
+
+    private
+
+    def enumerable?(container)
+      container.respond_to? :each
+    end
+
+    def serializable?(element)
+      element.respond_to? :to_hash
+    end
+
+    def manifest_files_count
+      @manifest_files_count ||= estimated_tpx_file_size / DEFAULT_MAX_FILESIZE
+    end
+
+    def estimated_tpx_file_size
+      Oj.dump(self).size
+    end
+
+    def split_section(section)
+      split_count = section.size / manifest_files_count
+      section.each_slice(split_count > 1 ? split_count : 1).to_a
+    end
+
+    def to_manifest_section(exchange_section, manifest_file_type, path)
+      subsections = split_section(exchange_section)
+      files = []
+
+      subsections.each_with_index do |item, index|
+        # save subsection as TPX file
+        files << "#{manifest_file_type}_#{index + 1}.json"
+        Oj.to_file(File.join(path, files.last), item)
+      end
+      files
+    end
+
+    def to_manifest(manifest_file_path)
+      exchange_hash = self.to_hash
+      manifest_hash = {}
+
+      keys = ['observable_dictionary_c_array', 'element_observable_c_array', 'asn_c_array', 'collection_c_array']
+      path = File.dirname(manifest_file_path)
+
+      exchange_hash.each do |key, val|
+        manifest_hash[key] = val unless keys.include?(key)
+      end
+
+      ELEMENT_LISTS.each do |list_key, list_def|
+        list_type, list_element, list_manifest_key, list_manifest_file_type = *list_def
+        if exchange_hash.has_key? list_key.to_s
+          manifest_hash[list_manifest_key.to_s] = to_manifest_section(exchange_hash[list_key.to_s], list_manifest_file_type.to_s, path) unless exchange_hash[list_key.to_s].empty?
+        end
+      end
+
+      manifest_hash
+    end
+
+  end # class Exchange
+end # module TPX_2_2