opentpx 2.2.0.17

data/lib/tpx/2_2/threat_observable.rb

@@ -0,0 +1,14 @@
+require 'tpx/2_2/data_model'
+require 'tpx/2_2/observable'
+
+
+module TPX_2_2
+
+  # The representation of the threat observable (the hash map keyed
+  # by the observable_id within the element observable).
+  class ThreatObservable < Observable
+    MANDATORY_ATTRIBUTES = Observable::MANDATORY_ATTRIBUTES + [
+      :occurred_at_t
+    ]
+  end # class
+end

data/lib/tpx/2_2/validator.rb

@@ -0,0 +1,279 @@
+require 'deep_merge'
+# This uses the TPX JSON Schema to validate an input document.
+module TPX_2_2
+  class Validator
+
+    class << self
+      TPX_KEYS = ['observable_dictionary_c_array', 'element_observable_c_array', 'collection_c_array', 'asn_c_array']
+      MANIFEST_KEYS = ['dictionary_file_manifest', 'observable_element_file_manifest', 'collection_file_manifest', 'network_file_manifest']
+
+      # validate_file! opens json file and validates it. Raises an error if validation fails.
+      #
+      # @param [String] filepath path to file which needs to be validated.
+      # @example
+      #   TPX_2_2::Validator.validate_file!('folder_name/file_name.json')
+      def validate_file!(filepath)
+        unless File.exists? filepath
+          raise ValidationError , "No such file or directory '#{filepath}'"
+        end
+
+        @@current_path = File.dirname(filepath)
+
+        begin
+          h = Oj.load_file(filepath)
+        rescue => e
+          raise ValidationError, "File '#{filepath}' is not a valid JSON file:\n#{e}"
+        end
+
+        if h.kind_of?(Hash) and tpx_manifest?(h)
+          DeepMerge::deep_merge!(load_manifest_files(h), h, {:merge_hash_arrays => true})
+          delete_manifest_keys!(h)
+        end
+
+        validate!(h)
+      end
+
+      # validate_json! validates json string. Raises an error if validation fails.
+      #
+      # @param [String] json_enc_str the json string which need to be validated.
+      # @example
+      #   TPX_2_2::Validator.validate_json!("{\"provider_s\":\"provider\",\"schema_version_s\":\"2.1.6\",\"source_observable_s\":\"source_observable\",\"source_description_s\":\"source_description\",\"distribution_time_t\":1438862400,\"last_updated_t\":1438862400,\"list_name_s\":\"list_name\",\"element_observable_c_array\":[{\"subject_s\":\"192.168.0.1\",\"type_s\":\"ipv4\",\"score_i\":80,\"score_24hr_decay_i\":0,\"threat_observable_c_map\":{\"Conficker\":{\"occurred_at_t\":4355545,\"last_seen_t\":13123}}}],\"observable_dictionary_c_array\":[{\"observable_id_s\":\"Malicious Host\",\"description_s\":\"This network node is a malicious host.\",\"criticality_i\":20,\"classification_c_array\":[{\"classification_id_s\":\"Malicious Host\",\"criticality_i\":20}]}],\"collection_c_array\":[{\"name_id_s\":\"MarketSeg1\",\"occurred_at_t\":1212312323,\"last_updated_t\":1212312323,\"description_s\":\"This collection is related to MarketSeg1\",\"author_s\":\"Allan Thomson\",\"workspace_s\":\"lg-system\"}],\"asn_c_array\":[{\"asn_number_ui\":1,\"occurred_at_t\":1212312323,\"as_owner_s\":\"ABC Corp\"}]}")
+      def validate_json!(json_enc_str)
+        h = Oj.load(json_enc_str)
+        validate!(h)
+      end
+
+      # validate! validates hash. Raises an error if validation fails.
+      #
+      # @param [Hash] input_hash the hash which needs to be validated.
+      # @example
+      #   TPX_2_2::Validator.validate!({
+      #     'provider_s': 'provider',
+      #     'schema_version_s': '2.1.6',
+      #     'source_observable_s': 'source_observable',
+      #     'source_description_s': 'source_description',
+      #     'distribution_time_t': 1438862400,
+      #     'last_updated_t': 1438862400,
+      #     'list_name_s': 'list_name',
+      #     'element_observable_c_array': [
+      #       {
+      #         'subject_s': '192.168.0.1',
+      #         'type_s': 'ipv4',
+      #         'score_i': 80,
+      #         'score_24hr_decay_i': 0,
+      #         'threat_observable_c_map': {
+      #           'Conficker': {
+      #             'occurred_at_t': 4355545,
+      #             'last_seen_t': 13123
+      #           }
+      #         }
+      #       }
+      #     ],
+      #     'observable_dictionary_c_array': [
+      #       {
+      #         'observable_id_s': 'Malicious Host',
+      #         'description_s': 'This network node is a malicious host.',
+      #         'criticality_i': 20,
+      #         'classification_c_array': [
+      #           {
+      #             'classification_id_s': 'Malicious Host',
+      #             'criticality_i': 20
+      #           }
+      #         ]
+      #       }
+      #     ],
+      #     'collection_c_array': [
+      #       {
+      #         'name_id_s': 'MarketSeg1',
+      #         'occurred_at_t': 1212312323,
+      #         'last_updated_t': 1212312323,
+      #         'description_s': 'This collection is related to MarketSeg1',
+      #         'author_s': 'Allan Thomson',
+      #         'workspace_s': 'lg-system'
+      #       }
+      #     ],
+      #     'asn_c_array': [
+      #       {
+      #         'asn_number_ui': 1,
+      #         'occurred_at_t': 1212312323,
+      #         'as_owner_s': 'ABC Corp'
+      #       }
+      #     ]
+      #   })
+      def validate!(input_hash)
+        @@undefined_observables = []
+
+        if input_hash.nil? || input_hash.empty?
+          raise ValidationError, " TPX has no content"
+        end
+
+        validate_schema!(input_hash)
+
+        if (TPX_KEYS & input_hash.keys).length > 0 && tpx_manifest?(input_hash)
+          raise TPX_2_2::ValidationError, "TPX file must either be in single-file or manifest format."
+        end
+
+        validate_element_observables!(input_hash['element_observable_c_array'])
+        validate_observable_dictionary!(input_hash['observable_dictionary_c_array'], input_hash['element_observable_c_array'])
+        validate_collections!(input_hash['collection_c_array'])
+        validate_networks!(input_hash['asn_c_array'])
+        if (input_hash['observable_dictionary_c_array'].nil? || input_hash['observable_dictionary_c_array'].empty?)
+          raise TPX_2_2::ValidationWarning, "observable dictionary is not defined"
+        end
+        unless @@undefined_observables.empty?
+          raise TPX_2_2::ValidationWarning, "observables #{@@undefined_observables.inspect} are not defined in the observable dictionary"
+        end
+      end
+
+
+      private
+
+      def validate_schema!(input_hash)
+        errors = JSON::Validator.fully_validate(TPX_2_2::SCHEMA, input_hash)
+        raise ValidationError, "#{errors.join(' ')}" unless errors.empty?
+      end
+
+      def validate_element_observables!(element_observable_list)
+        return if element_observable_list.nil? #nothing to validate, just return
+
+        schema = {
+          "type" => "object",
+          "required" => ["threat_observable_c_map"],
+          "properties" => {
+            "threat_observable_c_map" => {"type" => "hash"}
+          }
+        }
+        element_observable_list.each do |element_observable|
+          errors = JSON::Validator.fully_validate(schema, element_observable)
+          raise ValidationError, "#{errors.join(' ')}" unless errors.empty?
+        end
+      end
+
+      def validate_observable_dictionary!(observable_dictionaries, element_observable_list)
+        return if observable_dictionaries.nil? #nothing to validate, just return
+
+        schema = {
+          "type" => "object",
+          "required" => [
+            "observable_id_s",
+            "description_s",
+            "classification_c_array"
+          ],
+          "properties" => {
+            "observable_id_s" => {"type" => "string"},
+            "description_s" => {"type" => "string"},
+            "classification_c_array" => {"type" => "hash"}
+          }
+        }
+
+        observables = []
+
+        observable_dictionaries.each do |observable_dictionary|
+          errors = JSON::Validator.fully_validate(schema, observable_dictionary)
+          raise TPX_2_2::ValidationError, "#{errors.join(' ')}" unless errors.empty?
+          observables << observable_dictionary['observable_id_s']
+        end
+
+        return if element_observable_list.nil?
+        element_observable_list.each do |element_observable|
+          element = element_observable['threat_observable_c_map'] || element_observable[:threat_observable_c_map]
+          element.each_key do |observable|
+            @@undefined_observables << """#{observable}""" unless observables.include? observable
+          end
+        end
+
+      end
+
+      def validate_collections!(collections)
+        return if collections.nil? #nothing to validate, just return
+
+        schema = {
+          "type" => "object",
+          "required" => ["name_id_s"],
+          "properties" => {
+            "name_id_s" => {"type" => "string"}
+          }
+        }
+
+        collections.each do |collection|
+          errors = JSON::Validator.fully_validate(schema, collection)
+          raise ValidationError, "#{errors.join(' ')}" unless errors.empty?
+          collection.each do |key, value|
+            if (key == 'collection_c_array') && value.is_a?(Array)
+              validate_collections!(value)
+            end
+          end
+        end
+      end
+
+      def validate_networks!(networks)
+        return if networks.nil? #nothing to validate, just return
+
+        schema = {
+          "type" => "object",
+          "required" => ["asn_number_ui"],
+          "properties" => {
+            "asn_number_ui" => {"type" => "integer"}
+          }
+        }
+
+        networks.each do |net|
+          errors = JSON::Validator.fully_validate(schema, net)
+          raise ValidationError, "#{errors.join(' ')}" unless errors.empty?
+        end
+      end
+
+      # tpx_manifest? returns true if provided hash represents TPX manifest and contains TPX manifest entries
+      #
+      # @param [Hash] Hash with tpx content
+      # @example
+      #   TPX_2_2::Validator.tpx_manifest?({   "schema_version_s"=> "2.2.0",  "provider_s"=> "Lookingglass Cyber Solutions",  "list_name_s"=> "Virus Tracker",   "source_observable_s"=> "Virus Tracker",   "source_file_s"=> "https://virustracker.net/",  "source_description_s"=> "Virus Tracker provides real-time information about virus infections observed through custom sinkholes.",   "distribution_time_t"=> 1443007504,  "last_updated_t"=> 1443007504,   "score_i"=> 90,   "dictionary_file_manifest"=> [  "dictionary.json"  ],   "observable_element_file_manifest"=> [  "data8.json"  ]})
+      def tpx_manifest?(input_hash)
+        (MANIFEST_KEYS & input_hash.keys).length > 0
+      end
+
+      # current_path returns the current path which can be one of: 1) current working directory 2) file directory path being validated
+      def current_path
+        @@current_path || Dir.pwd
+      end
+
+      # load_manifest_files loads files referenced in manifests and returns merged content
+      #
+      # @param [Hash] Hash with tpx content
+      def load_manifest_files(input_hash)
+        res = {}
+        MANIFEST_KEYS.each do |manifest_section|
+          files = input_hash[manifest_section]
+          unless files.blank?
+            files.each do |manifest_file|
+              DeepMerge::deep_merge!(load_manifest_file(manifest_file), res, {:merge_hash_arrays => true})
+            end
+          end
+        end
+        res
+      end
+
+      # delete_manifest_keys! deletes manifest keys from input_hash in order to be valid TPX
+      #
+      # @param [Hash] Hash with tpx content
+      def delete_manifest_keys!(input_hash)
+        MANIFEST_KEYS.each do |key|
+          input_hash.delete key
+        end
+      end
+
+      # load_manifest_file loads TPX content from specified file, in case if file doesn't exists searches for it in current_path. In case if file not found ValidationError is raised
+      #
+      # @param [String] filename
+      def load_manifest_file(filename)
+        return Oj.load_file(filename) if File.exists? filename
+
+        filepath = File.join(current_path, File.basename(filename))
+        return Oj.load_file(filepath) if File.exists? filepath
+
+        raise ValidationError, "Could not find #{filename}"
+      end
+
+    end # class < self
+  end
+end

data/lib/tpx/tools.rb

@@ -0,0 +1,81 @@
+require 'gli'
+require 'tpx'
+
+module TPX
+  class Tools
+    extend ::GLI::App
+
+    TPX_VALID = "Validation succeeded"
+    TPX_INVALID = "The TPX file provided is invalid for the following reasons:"
+    TPX_VERSION_UNKNOWN = "Unknown TPX Version: "
+
+    class << self
+      attr_accessor :quiet
+
+      def msg(message)
+        puts message unless quiet
+      end
+
+      def get_tpx_version_const(tpx_version)
+        underscored_tpx_version = tpx_version.gsub('.', '_')
+        Object.const_get("TPX_#{underscored_tpx_version}")
+      rescue NameError => e
+        msg( e )
+        raise TPX_VERSION_UNKNOWN + "'#{tpx_version}'"
+      end
+
+      def validate(tpx_version_const, filepath)
+        begin
+          tpx_version_const::Validator.validate_file! filepath
+        rescue tpx_version_const::ValidationWarning => w
+          msg( "Warning: #{w}" )
+          puts TPX_VALID + " against " + tpx_version_const.to_s
+        rescue => e
+          puts TPX_INVALID
+          puts e
+        else
+          puts TPX_VALID
+        end
+      end
+
+
+    end
+
+    program_desc 'OpenTPX Tools'
+    version TPX::VERSION
+
+    switch [:q, :quiet], default_value: false,
+                           desc: 'quiet non-essential output and warnings.'
+
+    #--- COMMAND validate
+    desc "validates that a file is of a valid TPX file format"
+    long_desc "validates that a file is of a valid TPX file format\n\n" \
+      "TPX_FILE_PATH - The path to the TPX file. The file may be either a" \
+      " stand-alone TPX file or a tpx manifest file. For details on TPX" \
+      " manifest, please see https://github.com/Lookingglass/tpx"
+
+    arg 'TPX_FILE_PATH'
+
+    command :validate do |c|
+      c.flag [:v, :tpx_version], default_value: "2.2",
+        arg_name: 'VERSION',
+        type: String,
+        desc: "The version of tpx to validate the file against. Possible" \
+        " values: '2.2'"
+
+      c.action do |global_options, options, args|
+        self.quiet = global_options[:quiet]
+        tpx_version = options[:tpx_version]
+        tpx_file_path = args[0]
+
+        raise "Missing option TPX_VERSION" if tpx_version.nil?
+        raise "Missing arg TPX_FILE_PATH" if tpx_file_path.nil?
+
+        tpx_version = tpx_version.strip
+        msg( "Validating '#{tpx_file_path}' against TPX '#{tpx_version}' schema" )
+        tpx_version_const = get_tpx_version_const(tpx_version)
+        validate(tpx_version_const, tpx_file_path)
+      end
+    end
+  end
+end

data/lib/tpx/version.rb

@@ -0,0 +1,3 @@
+module TPX
+  VERSION = '2.2.0.17'
+end

data/lib/tpx_2_2.rb

@@ -0,0 +1,14 @@
+module TPX_2_2
+  CURRENT_SCHEMA_VERSION = '2.2.0'
+  SCHEMA = File.join(File.dirname(File.expand_path(__FILE__)), 'tpx', '2_2', 'schema', 'tpx.2.2.schema.json')
+
+  require 'json-schema'
+  require 'active_support'
+  require 'active_support/core_ext/object/blank'
+  require 'active_support/core_ext/hash'
+  require 'oj'
+
+  require 'tpx/2_2/exceptions'
+  require 'tpx/2_2/validator'
+  require 'tpx/2_2/exchange'
+end

metadata

@@ -0,0 +1,218 @@
+--- !ruby/object:Gem::Specification
+name: opentpx
+version: !ruby/object:Gem::Version
+  version: 2.2.0.17
+platform: ruby
+authors:
+- LookingGlass
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2015-10-17 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: timecop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: yard
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: oj
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: json-schema
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.5.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.5.1
+- !ruby/object:Gem::Dependency
+  name: deep_merge
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: gli
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.13.2
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.13.2
+description: An open-source format and tools for exchanging threat intelligence data.  This
+  is a JSON-based format that allows sharing of data between partner organizations.
+email:
+- support@lgscout.com
+executables:
+- opentpx_tools
+extensions: []
+extra_rdoc_files: []
+files:
+- LICENSE.txt
+- README.md
+- bin/opentpx_tools
+- lib/tpx.rb
+- lib/tpx/2_2/attribute_accessors.rb
+- lib/tpx/2_2/classification_element.rb
+- lib/tpx/2_2/classification_element_list.rb
+- lib/tpx/2_2/collection.rb
+- lib/tpx/2_2/collection_element.rb
+- lib/tpx/2_2/data_model.rb
+- lib/tpx/2_2/element_observable.rb
+- lib/tpx/2_2/element_observable_list.rb
+- lib/tpx/2_2/exceptions.rb
+- lib/tpx/2_2/exchange.rb
+- lib/tpx/2_2/heterogeneous_list.rb
+- lib/tpx/2_2/homogeneous_list.rb
+- lib/tpx/2_2/mandatory_attributes.rb
+- lib/tpx/2_2/merging_heterogeneous_list.rb
+- lib/tpx/2_2/merging_homogeneous_list.rb
+- lib/tpx/2_2/network.rb
+- lib/tpx/2_2/network_list.rb
+- lib/tpx/2_2/observable.rb
+- lib/tpx/2_2/observable_attribute_map.rb
+- lib/tpx/2_2/observable_definition.rb
+- lib/tpx/2_2/observable_dictionary.rb
+- lib/tpx/2_2/schema/tpx.2.2.schema.json
+- lib/tpx/2_2/threat_observable.rb
+- lib/tpx/2_2/validator.rb
+- lib/tpx/tools.rb
+- lib/tpx/version.rb
+- lib/tpx_2_2.rb
+homepage: http://www.opentpx.org
+licenses:
+- The Apache License, Version 2.0. See LICENSE.txt
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.3
+signing_key: 
+specification_version: 4
+summary: Open Threat Partner Exchange (OpenTPX)
+test_files: []
+has_rdoc: