stix_schema_spy 1.0

data/lib/stix_schema_spy/models/schema.rb

@@ -0,0 +1,217 @@
+# A STIX/CybOX/etc XML Schema
+# This is the centerpiece of most of the parsing
+
+require 'json'
+
+module StixSchemaSpy
+  class Schema
+
+    include StixSchemaSpy::HasChildren
+    include StixSchemaSpy::Util::SchemaNaming
+
+    @@schemas = {}
+    @@schemas_by_file = {}
+    @@config = JSON.parse(File.read(File.join(File.dirname(File.expand_path(__FILE__)), '..', '..', '..', 'config', 'mappings.json')))
+    attr_reader :doc, :prefix, :types, :namespace, :filename
+
+    @schema_root = {
+      stix: 'http://stix.mitre.org/XMLSchema',
+      cybox: 'http://cybox.mitre.org/XMLSchema'
+    }
+
+    def initialize(schema_location)
+      # Open the schema
+      @filename = schema_location.split(/[\/\\]/).last
+      @doc = Nokogiri::XML(open(schema_location))
+
+      @elements = {}
+      @attributes = {}
+      @types = {}
+      @special_fields = []
+
+      # Find this document's prefix (if any)
+      @namespace = doc.root.attributes['targetNamespace'].value
+      @prefix = find_prefix(doc)
+      @@schemas[prefix] = self
+
+      # First, process any schemas that this schema imports (unless they're blacklisted)
+      path = schema_location.split('/')[0...-1].join('/')
+      doc.xpath('//xs:import', {'xs' => 'http://www.w3.org/2001/XMLSchema'}).each do |import|
+        if self.class.config['schemas'][import.attributes['namespace'].value] && lp = Schema.config['schemas'][import.attributes['namespace'].value]['localPath']
+          schema_location = "config/schemas/#{lp}"
+        else
+          schema_location = import.attributes['schemaLocation'].value
+          schema_location = "#{path}/#{schema_location}" unless (schema_location =~ /http/)
+        end
+        self.class.build(schema_location) unless self.class.blacklisted?(import.attributes['namespace'].value)
+      end
+
+      doc.xpath('//xs:include', {'xs' => 'http://www.w3.org/2001/XMLSchema'}).each do |include_elem|
+        schema_location = include_elem.attributes['schemaLocation'].value
+        schema_location = "#{path}/#{schema_location}" unless (schema_location =~ /http/)
+        @@schemas_by_file[schema_location.split('/').last] = :imported
+        process_doc(Nokogiri::XML(open(schema_location)))
+      end
+
+      process_doc(@doc)
+    end
+
+    def process_doc(doc)
+      doc.xpath('/xs:schema/*', {'xs' => 'http://www.w3.org/2001/XMLSchema'}).each do |field|
+        process_field(field)
+      end
+    end
+
+    # Get the configuration entry for this schema
+    def config
+      @@config['schemas'][namespace] || {}
+    end
+
+    # This makes the has_children module work a little easier (doesn't have to know the difference)
+    # between schemas and types.
+    def schema
+      self
+    end
+
+    # Again, just for compatibility with Type
+    def extension
+      nil
+    end
+
+    # More compatibility with type
+    def load!
+      # Schemas are automatically loaded, so do nothing
+    end
+
+    # Find the version of this schema by looking at the version attribute
+    def version
+      doc.root.attributes['version'].value || "Unknown"
+    end
+
+    # Find the namespace prefix by searching through the namespaces for the TNS
+    def find_prefix(doc)
+      return config['prefix'] if config && config['prefix']
+
+      # Loop through the attributes until we see one with the same value
+      ns_prefix_attribute = doc.namespaces.find do |prefix, ns|
+        ns.to_s == namespace.to_s && prefix != 'xmlns'
+      end
+
+      # If the attribute was found, return it, otherwise return nil
+      ns_prefix_attribute ? ns_prefix_attribute[0].split(':').last : "Unknown"
+    end
+
+    # Find a type in this schema by name
+    def find_type(name)
+      @types[name]
+    end
+
+    # Returns the path to the documentation, just by looking it up in the config
+    def doc_path
+      config['docs']
+    end
+
+    # Returns whether the schema is a CybOX object
+    def is_cybox_object?
+      namespace =~ /objects#/
+    end
+
+    def complex_types
+      @types.values.select {|t| t.kind_of?(ComplexType)}
+    end
+
+    def simple_types
+      @types.values.select {|t| t.kind_of?(SimpleType)}
+    end
+
+    # Build a new schema, cache it
+    def self.build(schema_location)
+      filename = schema_location.split('/').last
+      if filename == 'generic.xsd'
+        filename = schema_location.split('/')[-2..-1].join('/')
+      end
+      @@schemas_by_file[filename] ||= self.new(schema_location)
+    end
+
+    def self.schemas_by_file
+      @@schemas_by_file
+    end
+
+    # Return the schemas configuration hash
+    def self.config
+      @@config
+    end
+
+    # Get all schemas
+    def self.all
+      @@schemas.values
+    end
+
+    # Find a schema by prefix
+    def self.find(prefix_or_ns)
+      if @@schemas[prefix_or_ns]
+        @@schemas[prefix_or_ns]
+      elsif schema_mapping = self.config['schemas'][prefix_or_ns]
+        @@schemas[schema_mapping['prefix']]
+      end
+    end
+
+    def self.namespaces
+      self.all.inject({'xs' => 'http://www.w3.org/2001/XMLSchema'}) {|coll, schema| coll[schema.prefix] = schema.namespace; coll}
+    end
+
+    def self.schema_dir
+      File.join(File.dirname(File.expand_path(__FILE__)), '..', '..', '..', 'config', self.version)
+    end
+
+    def self.version
+      "1.1"
+    end
+
+    # Don't process non STIX or CybOX schemas
+    def self.blacklisted?(namespace)
+      blacklist_enabled? && (namespace =~ /(stix)|(cybox)|(data-marking)/).nil?
+    end
+
+    def self.blacklist_enabled?
+      !!@blacklist_enabled
+    end
+
+    def self.blacklist_enabled=(val)
+      @blacklist_enabled = val
+    end
+
+    def preload!
+      100.times do
+        schema.elements.map(&:type)
+        schema.attributes.map(&:type)
+        schema.complex_types.each {|t| t.elements.each(&:type)}
+      end
+    end
+
+    def self.preload!
+      return if @preloaded
+      @preloaded = true
+      Dir.glob("#{schema_dir}/stix/cybox/*.xsd").each {|f| self.build(f)}
+      Dir.glob("#{schema_dir}/stix/cybox/objects/*.xsd").each {|f| self.build(f)}
+      Dir.glob("#{schema_dir}/stix/cybox/extensions/*.xsd").each {|f| self.build(f)}
+      Dir.glob("#{schema_dir}/stix/*.xsd").each {|f| self.build(f)}
+      Dir.glob("#{schema_dir}/stix/extensions/**/*.xsd").each {|f| self.build(f)}
+      @uber_schema = Dir.chdir(schema_dir) {Nokogiri::XML::Schema.new(File.read('uber_schema.xsd'))}
+    end
+
+    def self.schema_root
+      @schema_root
+    end
+
+    def self.schema_root=(value)
+      @schema_root = value
+    end
+
+    def blacklisted?
+      self.class.blacklisted?(self.namespace)
+    end
+
+    self.blacklist_enabled = true
+  end
+end

data/lib/stix_schema_spy/models/simple_type.rb

@@ -0,0 +1,47 @@
+# Representation of a simple type
+
+module StixSchemaSpy
+  class SimpleType < StixSchemaSpy::Type
+
+    def self.build(xml, schema)
+      self.new(xml, schema)
+    end
+
+    # Returns whether or not this simple type is an enumeration
+    def enumeration?
+      @is_enumeration ||= @xml.xpath('./xs:restriction/xs:enumeration', {'xs' => 'http://www.w3.org/2001/XMLSchema'}).length > 0
+    end
+
+    # Returns the list of values for this enumeration
+    def enumeration_values
+      enumeration = @xml.xpath('./xs:restriction/xs:enumeration', {'xs' => 'http://www.w3.org/2001/XMLSchema'})
+      if enumeration.length > 0
+        return enumeration.map {|elem| [elem.attributes['value'].value, elem.xpath('./xs:annotation/xs:documentation', {'xs' => 'http://www.w3.org/2001/XMLSchema'}).text]}
+      else
+        raise "Not an enumeration"
+      end    
+    end
+
+    # A shortcut to checking whether there's an example because simple types never have examples
+    def has_example?
+      false
+    end
+
+    # Not a complex type
+    def complex?
+      false
+    end
+
+    def fields
+      []
+    end
+
+    def own_fields
+      []
+    end
+
+    def elements
+      []
+    end
+  end
+end

data/lib/stix_schema_spy/models/special_field.rb

@@ -0,0 +1,21 @@
+# Represents some weird XML construct we don't handle (xs:any, xs:anyAttribute primarily)
+
+module StixSchemaSpy
+  class SpecialField
+
+    attr_accessor :name, :type, :documentation
+
+    def initialize(name)
+      @name = name
+      @type = ExternalType.new("", "")
+    end
+
+    def attribute?
+      false
+    end
+
+    def element?
+      true
+    end
+  end
+end

data/lib/stix_schema_spy/models/type.rb

@@ -0,0 +1,106 @@
+# Generic Type (ComplexType, SimpleType, or ExternalType)
+
+module StixSchemaSpy
+  class Type
+
+    attr_reader :name, :documentation, :extension, :schema, :inline
+
+    def initialize(xml, schema, inline = false)
+      @inline = !!inline
+      @schema = schema
+      @xml = xml
+      @name = xml.attributes['name'] ? xml.attributes['name'].value : "#{inline}InlineType"
+      @documentation = xml.xpath('./xs:annotation/xs:documentation', {'xs' => 'http://www.w3.org/2001/XMLSchema'}).to_a.map {|node| node.text}.join("\n")
+      @extension = get_extension(xml)
+    end
+
+    def full_name
+      @inline ? name : "#{prefix}:#{name}"
+    end
+
+    def get_extension(type)
+      if node = type.xpath('xs:complexContent/xs:extension | xs:simpleContent/xs:extension | xs:complexContent/xs:restriction | xs:simpleContent/xs:restriction', {'xs' => 'http://www.w3.org/2001/XMLSchema'}).first
+        base = node.attributes['base'].value
+        schema.find_type(base) || Type.find(base)
+      else
+        nil
+      end
+    end
+
+    def abstract?
+      !!(@xml.attributes['abstract'] && @xml.attributes['abstract'].value == "true")
+    end
+
+    def parent_type
+      @extension
+    end
+
+    # URL is just the prefix/type
+    # TODO: This should really go in a helper....
+    def url
+      "#{prefix}/#{name}"
+    end
+
+    # This may be overriden by classes that inherit this. Otherwise it just checks to see if a template exists
+    def has_example?
+      File.exists?("views/examples/#{prefix}/#{name}.erb")
+    end
+
+    # This may be overriden by classes that inherit this. Otherwise it just renders the template
+    def example(configuration)
+      ERB.new(File.read("views/examples/#{prefix}/#{name}.erb")).result(configuration.get_binding)
+    end
+
+    def doc_path
+      @schema.doc_path ? @schema.doc_path + "##{name}" : nil
+    end
+
+    # Returns whether or not this type is a vocabulary
+    def vocab?
+      @is_vocab ||= full_name =~ /Vocab-\d\.\d$/
+    end
+
+    def self.inline(xml, schema, name)
+      type = if complex_type = xml.xpath('xs:complexType', {'xs' => 'http://www.w3.org/2001/XMLSchema'}).first
+        ComplexType.new(complex_type, schema, name)
+      elsif simple_type = xml.xpath('xs:simpleType', {'xs' => 'http://www.w3.org/2001/XMLSchema'}).first
+        SimpleType.new(simple_type, schema, name)
+      else
+        $logger.warn "Unable to find type for #{xml.attributes['name'].value}" if defined?($logger)
+        ExternalType.new("", "")
+      end
+      schema.types[type.name] = type
+      type
+    end
+
+    def self.find(prefix, name = nil)
+      if name.nil?
+        if prefix.split(':').length == 2
+          prefix, name = prefix.split(':')
+        else
+          name = prefix
+          prefix = nil
+        end
+      end
+      
+      if schema = Schema.find(prefix)
+        schema.find_type(name)
+      else
+        ExternalType.new(prefix, name)
+      end
+    end
+
+    def prefix
+      @schema.prefix
+    end
+
+    def self.counter
+      @counter ||= 0
+      @counter += 1
+    end
+
+    def inspect
+      "#<#{self.class.to_s}:#{object_id} @name=\"#{full_name}\">"
+    end
+  end
+end

data/lib/stix_schema_spy/util/schema_naming.rb

@@ -0,0 +1,106 @@
+# SchemaIdentifier is a set of methods added to the schema class that help identify
+# the type of schema: is it a STIX component, STIX extension, CybOX object, etc.
+
+module StixSchemaSpy
+  module Util
+    module SchemaNaming
+
+      PROJECT_NAMES = {
+        cybox: 'CybOX',
+        stix: 'STIX'
+      }
+
+      EXTENSION_TYPE = {
+        'ap' => 'attack_pattern',
+        'structuredcoa' => 'structured_coa',
+        'testmechanism' => 'test_mechanism',
+        'markingstructure' => 'marking'
+      }
+
+      # Returns a symbol indicating the type of schema:
+      # One of: core, common, vocabularies, component, object, extension, nil (external)
+      def type
+        case namespace
+        when /stix-\d/, /cybox-\d/
+          :core
+        when /common/
+          :common
+        when /vocabularies/
+          :vocabularies
+        when /cybox\.mitre\.org\/objects#/
+          :object
+        when /\.mitre\.org\/extensions\//
+          :extension
+        when /stix\.mitre\.org/ # If it's STIX and not core, common, vocabs, or extensions, it's a component
+          :component
+        when 'http://data-marking.mitre.org/Marking-1'
+          :marking
+        else
+          nil
+        end
+      end
+
+      # Returns a symbol indicating whether the schema is STIX or CybOX
+      # One of: stix, cybox, nil
+      def project
+        case namespace
+        when /cybox/
+          :cybox
+        when /stix/, /data-marking/
+          :stix
+        else
+          nil
+        end
+      end
+
+      def title
+        return config['title'] if config['title']
+        case type
+        when :core, :common, :vocabularies
+          "#{PROJECT_NAMES[project]} #{type.to_s.capitalize}"
+        when :object, :component
+          name = xpath_name.gsub("STIX ", "").gsub("CybOX ", "").gsub("_", " ")
+          name == "COA" ? "Course of Action" : name
+        when :extension
+          match = (xpath_name || "").match(/- ([\w \.]+)( Instance)?/)
+          match ? match[1] : nil
+        when :marking
+          "Data Markings"
+        else
+          prefix.capitalize
+        end
+      end
+
+      def schema_location
+        return config['schemaLocation'] if config['schemaLocation']
+        root = Schema.schema_root[project]
+        case type
+        when :core, :common
+          "#{root}/#{type}/#{version}/#{project}_#{type}.xsd"
+        when :vocabularies
+          "#{root}/default_#{type}/#{version}/#{project}_default_#{type}.xsd"
+        when :component
+          component = title.downcase.gsub(' ', '_').downcase
+          "#{root}/#{component}/#{version}/#{component}.xsd"
+        when :object
+          object_name = namespace.match(/#(.+)Object/)[1].gsub(/([^ ])([A-Z][a-z])/, '\1_\2')
+          "#{root}/objects/#{object_name}/#{version}/#{object_name}_Object.xsd"
+        when :marking
+          "#{root}/data_marking/#{version}/data_marking.xsd"
+        when :extension
+          extension_type = namespace.match(/\/(\w+)#/)[1].downcase
+          extension_type = EXTENSION_TYPE[extension_type] if EXTENSION_TYPE[extension_type]
+          extension_name = self.filename.gsub("_" + extension_type, '').gsub('.xsd', '')
+          "#{root}/extensions/#{extension_type}/#{extension_name}/#{version}/#{extension_name}_#{extension_type}.xsd"
+        else
+          nil # Fail condition
+        end
+      end
+
+      def xpath_name
+        element = @doc.xpath('/xs:schema/xs:annotation[1]/xs:appinfo/schema', {'xs' => 'http://www.w3.org/2001/XMLSchema'}).first
+        element ? element.text : nil
+      end
+    end
+  end
+end