stix_schema_spy 1.0

data/lib/stix_schema_spy/util/tasks.rb

@@ -0,0 +1,69 @@
+require 'fileutils'
+require 'nokogiri'
+
+namespace :stix_schema_spy do
+
+  desc "Generate the uber-schema that can be used to validate against a single file"
+  task :generate_schemas, :output_dir do |task, args|
+    output_dir = args[:output_dir] ? File.expand_path(args[:output_dir]) : nil
+
+    import_chains = Hash.new([])
+    namespace_map = {}
+    schema_dir = File.join(File.dirname(__FILE__), '..', '..', '..', 'config', 'schemas')
+
+    # Create the output directory if we need one
+    FileUtils.mkdir_p(output_dir) if output_dir
+
+    # Collect each of the necessary schemas
+    Dir.chdir(schema_dir) do
+      namespace_map = Dir.glob('**/*.xsd').each_with_object({}) do |schema_location, coll|
+        next if schema_location =~ /uber_schema/
+        puts "Processing #{schema_location}"
+        filename = schema_location.split('/').last
+
+        File.open(schema_location, 'r:bom|utf-8') do |file| # This discards the UTF BOM, which will choke libxml/nokogiri
+
+          # Get the schema text
+          schema = file.read
+          
+          # Parse the document and grab the namespace
+          src = Nokogiri::XML::Document.parse(schema, nil, nil, Nokogiri::XML::ParseOptions::PEDANTIC)
+          namespace = src.root.attributes['targetNamespace'].value
+
+          # If there are any includes, we need to start tracking them
+          src.root.xpath('xs:include', {'xs' => 'http://www.w3.org/2001/XMLSchema'}).each do |import|
+            import_chains[namespace] << import.attributes['schemaLocation'].value.split('/').last
+          end
+
+          # Add the schema if it's the top level of the include chain
+          coll[namespace] = schema_location unless import_chains[namespace].include?(filename) # Set this namespace's parent to the current schema unless any other includes point to it
+
+          # Write the schema out to the target directory
+          if output_dir
+            directory = schema_location.split('/')[0..-2].join('/')
+            Dir.chdir(output_dir) {
+              FileUtils.mkdir_p(directory)
+              File.open(schema_location, "w") {|f| f.write(schema)}
+            }
+          end
+        end            
+      end
+    end
+
+    # Then build them into a series of import statements
+    doc = Nokogiri::XML::Builder.new  do |xml|
+      xml.schema('xmlns' => 'http://www.w3.org/2001/XMLSchema') do |root|
+        namespace_map.each do |namespace, schema_location|
+          root.import('namespace' => namespace, 'schemaLocation' => "#{schema_location}")
+        end
+      end
+    end
+
+    # Write the schema
+    Dir.chdir(output_dir || 'config/schemas') {
+      File.open("uber_schema.xsd", "w") {|f| f.write(doc.to_xml)}  
+    }
+    
+  end
+
+end

data/lib/stix_schema_spy/version.rb

@@ -0,0 +1,3 @@
+module StixSchemaSpy
+  VERSION = "1.0"
+end

data/spec/lib/schema_naming_spec.rb

@@ -0,0 +1,190 @@
+require 'spec_helper'
+
+describe Util::SchemaNaming do
+
+  before do
+    @class = Class.new
+    @class.send(:include, Util::SchemaNaming)
+
+    @schema = @class.new
+    @schema.stub(:config).and_return({})
+
+    def type_for(namespace)
+      @schema.stub(:namespace).and_return(namespace)
+      @schema.type
+    end
+
+    def project_for(namespace)
+      @schema.stub(:namespace).and_return(namespace)
+      @schema.project
+    end
+
+    def name_for(namespace)
+      @schema.stub(:namespace).and_return(namespace)
+      @schema.title
+    end
+
+    def location_for(namespace, version="1.1")
+      @schema.stub(:namespace).and_return(namespace)
+      @schema.stub(:version).and_return(version)
+      @schema.schema_location
+    end
+  end
+
+  context "#type" do
+    it "should find core" do
+      type_for("http://stix.mitre.org/stix-1").should == :core
+      type_for("http://cybox.mitre.org/cybox-2").should == :core
+    end
+
+    it "should find common" do
+      type_for("http://stix.mitre.org/common-1").should == :common
+      type_for("http://cybox.mitre.org/common-2").should == :common
+    end
+
+    it "should find vocabularies" do
+      type_for("http://stix.mitre.org/default_vocabularies-1").should == :vocabularies
+      type_for("http://cybox.mitre.org/default_vocabularies-2").should == :vocabularies
+    end
+
+    it "should find objects" do
+      type_for("http://cybox.mitre.org/objects#AddressObject-2").should == :object
+      type_for("http://cybox.mitre.org/objects#WinNetworkShareObject-2").should == :object
+      type_for("http://cybox.mitre.org/objects#FileObject-2").should == :object
+      type_for("http://cybox.mitre.org/objects#SocketAddressObject-1").should == :object
+    end
+
+    it "should find extensions" do
+      type_for("http://stix.mitre.org/extensions/TestMechanism#Snort-1").should == :extension
+      type_for("http://cybox.mitre.org/extensions/Address#CIQAddress3.0-1").should == :extension
+      type_for("http://stix.mitre.org/extensions/Address#CIQAddress3.0-1").should == :extension
+      type_for("http://stix.mitre.org/extensions/Malware#MAEC4.1-1").should == :extension
+    end
+
+    it "should find components" do
+      type_for("http://stix.mitre.org/Campaign-1").should == :component
+      type_for("http://stix.mitre.org/TTP-1").should == :component
+      type_for("http://stix.mitre.org/Indicator-2").should == :component
+      type_for("http://stix.mitre.org/CourseOfAction-1").should == :component
+    end
+
+    it "should find data markings" do
+      type_for("http://data-marking.mitre.org/Marking-1").should == :marking
+    end
+
+    it "should not recognize anything else" do
+      type_for("http://google.com/schemas/4").should == nil
+    end
+  end
+
+  context "#project" do
+    it "should detect STIX" do
+      project_for("http://stix.mitre.org/Campaign-1").should == :stix
+      project_for("http://data-marking.mitre.org/Marking-1").should == :stix
+    end
+
+    it "should detect CybOX" do
+      project_for("http://cybox.mitre.org/default_vocabularies-2").should == :cybox
+    end
+
+    it "should detect no known project" do
+      project_for("http://maec.mitre.org/default_vocabularies-2").should == nil
+    end
+  end
+
+  context "#name" do
+    it "should find the name for core, common, and vocabs" do
+      name_for("http://stix.mitre.org/stix-1").should == "STIX Core"
+      name_for("http://cybox.mitre.org/cybox-2").should == "CybOX Core"
+
+      name_for("http://stix.mitre.org/common-1").should == "STIX Common"
+      name_for("http://cybox.mitre.org/common-2").should == "CybOX Common"
+
+      name_for("http://stix.mitre.org/default_vocabularies-1").should == "STIX Vocabularies"
+      name_for("http://cybox.mitre.org/default_vocabularies-2").should == "CybOX Vocabularies"
+    end
+
+    it "should find the name for objects" do
+      @schema.stub(:xpath_name).and_return("Address_Object")
+      name_for("http://cybox.mitre.org/objects#AddressObject-2").should == "Address Object"
+    end
+
+    it "should find the name for components" do
+      @schema.stub(:xpath_name).and_return("STIX Campaign")
+      name_for("http://stix.mitre.org/Campaign-1").should == "Campaign"
+
+      @schema.stub(:xpath_name).and_return("STIX COA")
+      name_for("http://stix.mitre.org/CourseOfAction-1").should == "Course of Action"
+    end
+
+    it "should find the name for extensions" do
+      @schema.stub(:xpath_name).and_return("STIX Extension - Snort Test Mechanism Instance")
+      name_for("http://stix.mitre.org/extensions/TestMechanism#Snort-1").should == "Snort Test Mechanism"
+    end
+
+    it "should find the name for extensions" do
+      name_for("http://data-marking.mitre.org/Marking-1").should == "Data Markings"
+    end
+
+  end
+
+  context "#schema_location" do
+    it "should get schema location for core and common" do
+      location_for("http://stix.mitre.org/stix-1", '1.1').should == "http://stix.mitre.org/XMLSchema/core/1.1/stix_core.xsd"
+      location_for("http://cybox.mitre.org/cybox-2", '2.1').should == "http://cybox.mitre.org/XMLSchema/core/2.1/cybox_core.xsd"
+
+      location_for("http://stix.mitre.org/common-1", '1.1').should == "http://stix.mitre.org/XMLSchema/common/1.1/stix_common.xsd"
+      location_for("http://cybox.mitre.org/common-2", '2.1').should == "http://cybox.mitre.org/XMLSchema/common/2.1/cybox_common.xsd"
+    end
+
+    it "should get schema location for vocabularies" do
+      location_for("http://stix.mitre.org/default_vocabularies-1", '1.1').should == "http://stix.mitre.org/XMLSchema/default_vocabularies/1.1/stix_default_vocabularies.xsd"
+      location_for("http://cybox.mitre.org/default_vocabularies-2", '2.1').should == "http://cybox.mitre.org/XMLSchema/default_vocabularies/2.1/cybox_default_vocabularies.xsd"
+    end
+
+    it "should get schema location for components" do
+      @schema.stub(:title).and_return("Indicator")
+      location_for("http://stix.mitre.org/Indicator-2", '2.1').should == "http://stix.mitre.org/XMLSchema/indicator/2.1/indicator.xsd"
+
+      @schema.stub(:title).and_return("Threat Actor")
+      location_for("http://stix.mitre.org/ThreatActor-2", '1.1').should == "http://stix.mitre.org/XMLSchema/threat_actor/1.1/threat_actor.xsd"
+
+      @schema.stub(:title).and_return("Course of Action")
+      location_for("http://stix.mitre.org/CourseOfAction-1", '1.1').should == "http://stix.mitre.org/XMLSchema/course_of_action/1.1/course_of_action.xsd"
+    end
+
+    it "should get schema location for objects" do
+      location_for("http://cybox.mitre.org/objects#AddressObject-2", '2.1').should == "http://cybox.mitre.org/XMLSchema/objects/Address/2.1/Address_Object.xsd"
+      location_for("http://cybox.mitre.org/objects#WinNetworkShareObject-2", '2.1').should == "http://cybox.mitre.org/XMLSchema/objects/Win_Network_Share/2.1/Win_Network_Share_Object.xsd"
+      location_for("http://cybox.mitre.org/objects#FileObject-2", '2.1').should == "http://cybox.mitre.org/XMLSchema/objects/File/2.1/File_Object.xsd"
+      location_for("http://cybox.mitre.org/objects#SocketAddressObject-1", '1.1').should == "http://cybox.mitre.org/XMLSchema/objects/Socket_Address/1.1/Socket_Address_Object.xsd"
+    end
+
+    it "should get schema location for extensions" do
+      @schema.stub(:filename).and_return("snort_test_mechanism.xsd")
+      location_for("http://stix.mitre.org/extensions/TestMechanism#Snort-1").should ==
+                   "http://stix.mitre.org/XMLSchema/extensions/test_mechanism/snort/1.1/snort_test_mechanism.xsd"
+
+      @schema.stub(:filename).and_return("capec_2.7_attack_pattern.xsd")
+      location_for("http://stix.mitre.org/extensions/AP#CAPEC2.7-1").should ==
+                   "http://stix.mitre.org/XMLSchema/extensions/attack_pattern/capec_2.7/1.1/capec_2.7_attack_pattern.xsd"
+
+      @schema.stub(:filename).and_return("terms_of_use_marking.xsd")
+      location_for("http://data-marking.mitre.org/extensions/MarkingStructure#Terms_Of_Use-1").should ==
+                   "http://stix.mitre.org/XMLSchema/extensions/marking/terms_of_use/1.1/terms_of_use_marking.xsd"
+
+      @schema.stub(:filename).and_return("generic_structured_coa.xsd")
+      location_for("http://stix.mitre.org/extensions/StructuredCOA#Generic-1").should ==
+                   "http://stix.mitre.org/XMLSchema/extensions/structured_coa/generic/1.1/generic_structured_coa.xsd"
+    end
+
+    it "should get schema location for markings" do
+      location_for("http://data-marking.mitre.org/Marking-1").should == "http://stix.mitre.org/XMLSchema/data_marking/1.1/data_marking.xsd"
+    end
+  end
+
+  it "should get the xpath schema name" do
+    Schema.new('config/schemas/stix/campaign.xsd').xpath_name.should == 'STIX Campaign'
+    Schema.new('config/schemas/stix/cybox/objects/Address_Object.xsd').xpath_name.should == 'Address_Object'
+  end
+end

data/spec/models/complex_type_spec.rb

@@ -0,0 +1,68 @@
+require 'spec_helper'
+
+describe ComplexType do
+  context "The class" do
+    it "should be able to create a new complex type from schema" do
+      doc = Nokogiri::XML(open('config/schemas/stix/stix_core.xsd'))
+      type = ComplexType.new(doc.xpath('//xs:complexType').first, self)
+    end
+
+    it "should create appropriate subclasses" do
+      schema = Schema.build('config/schemas/stix/stix_core.xsd')
+      doc = Nokogiri::XML(open('config/schemas/stix/stix_core.xsd'))
+
+      type = ComplexType.build(doc.xpath('//xs:complexType')[1], schema)
+      type.should be_kind_of ComplexType
+    end
+  end
+
+  context "instances" do
+    before do
+      @stix_type             = Schema.build('config/schemas/stix/stix_core.xsd').find_type('STIXType')
+      @ttp_type              = Schema.build('config/schemas/stix/ttp.xsd').find_type('TTPType')
+      @indicator_type_vocab  = Schema.build('config/schemas/stix/stix_default_vocabularies.xsd').find_type('IndicatorTypeVocab-1.0')
+    end
+
+    it "should list its own fields" do
+      @stix_type.own_fields.length.should == 14
+      @ttp_type.own_fields.length.should == 15
+
+      @stix_type.own_fields.first.should be_kind_of(Node)
+    end
+
+    it "should list all of its fields (including inherited)" do
+      @stix_type.fields.length.should == 14
+      @ttp_type.fields.length.should == 18
+
+      @stix_type.fields.first.should be_kind_of(Node)
+    end
+
+    it "should list elements" do
+      @stix_type.elements.length.should == 10
+      @ttp_type.elements.length.should == 14
+
+      @stix_type.elements.first.should be_kind_of(Element)
+    end
+
+    it "should list attributes" do
+      @stix_type.attributes.length.should == 4
+      @ttp_type.attributes.length.should == 4
+
+      @stix_type.attributes.first.should be_kind_of(Attribute)
+    end
+
+    it "should indicate that it's a complex type" do
+      @stix_type.should be_complex
+    end
+
+    it "should be a vocabulary as appropriate" do
+      @stix_type.should_not be_vocab
+      @indicator_type_vocab.should be_vocab
+    end
+
+    it "should find values for a vocabulary" do
+      @indicator_type_vocab.vocab_values.length.should == 10
+      expect {@stix_type.vocab_values}.to raise_error
+    end
+  end
+end

data/spec/models/node_spec.rb

@@ -0,0 +1,47 @@
+require 'spec_helper'
+
+describe Node do
+  context "The Node class" do
+    it "should be able to create itself from schema" do
+      @doc = Nokogiri::XML(open('config/schemas/stix/stix_core.xsd'))
+      node = Node.new(@doc.xpath('//xs:element').first, self)
+
+      node.documentation.should == "The STIX_Package field contains a bundle of information characterized in the Structured Threat Information eXpression (STIX) language."
+    end
+  end
+
+  context "A node with a defined type" do
+    before do
+      @schema = Schema.build('config/schemas/stix/stix_core.xsd')
+      @stix_package = @schema.find_element('STIX_Package')
+    end
+
+    it "should be able to find its type" do
+      @stix_package.type.should == @schema.find_type('STIXType')
+    end
+
+    it "should be an element or attribute as appropriate" do
+      @stix_package.should be_element
+      @stix_package.type.attributes.first.should be_attribute
+    end
+
+    it "should have the given name" do
+      @stix_package.name.should == "STIX_Package"
+    end
+  end
+
+  context "A node with a referenced type" do
+    before do
+      @schema = Schema.build("config/schemas/cybox/cybox_core.xsd")
+      @observable = @schema.find_type("ObservablesType").find_element('Observable')
+    end
+
+    it "should have an anonymous type" do
+      @observable.type.should == @schema.find_type("ObservableType")
+    end
+  end
+
+  context "A node with an inline type" do
+
+  end
+end

data/spec/models/schema_spec.rb

@@ -0,0 +1,63 @@
+require 'spec_helper'
+
+describe Schema do
+  context "The Schema class" do
+    it "should parse a schema" do
+      schema = Schema.build('config/schemas/stix/stix_core.xsd')
+      schema.should_not be_nil
+
+      schema.namespace.should == "http://stix.mitre.org/stix-1"
+      schema.prefix.should == "stix"
+      Schema.find('stix').should == schema
+
+      schema.types.length.should == 11
+      schema.elements.length.should == 1
+      schema.attributes.length.should == 0
+    end
+
+    it "should default the blacklist to true" do
+      Schema.blacklist_enabled?.should == true
+    end
+
+    it "should be able to enable and disable the blacklist (schemas to ignore)" do
+      Schema.blacklist_enabled = false
+      Schema.blacklist_enabled?.should == false
+    end
+
+    it "should indicate whether a schema is blacklisted" do
+      Schema.blacklist_enabled = false
+      Schema.blacklisted?("http://stix.mitre.org").should == false
+      Schema.blacklisted?("http://iodef.ietf.org").should == false
+      Schema.blacklist_enabled = true
+      Schema.blacklisted?("http://stix.mitre.org").should == false
+      Schema.blacklisted?("http://cybox.ietf.org").should == false
+      Schema.blacklisted?("http://data-marking.mitre.org").should == false
+      Schema.blacklisted?("http://iodef.ietf.org").should == true
+    end
+
+    it "should return namespace mappings" do
+      Schema.build('config/schemas/stix/stix_common.xsd')
+
+      Schema.namespaces['xs'].should == 'http://www.w3.org/2001/XMLSchema'
+      Schema.namespaces['stixCommon'].should == 'http://stix.mitre.org/common-1'
+    end
+  end
+
+  context "A schema instance" do
+    before do
+      @schema = Schema.build('config/schemas/stix/stix_core.xsd')
+    end
+
+    it "should be able to find an element" do
+      @schema.find_element('STIX_Package').should_not be_nil
+    end
+
+    it "should be able to list global elements" do
+      @schema.elements.should == [@schema.find_element('STIX_Package')]
+    end
+
+    it "should be able to find the schema location" do
+      @schema.schema_location.should == 'http://stix.mitre.org/XMLSchema/core/1.1/stix_core.xsd'
+    end
+  end
+end