active_stix 0.1.21

data/app/models/active_stix/bundled_object.rb

@@ -0,0 +1,4 @@
+class ActiveStix::BundledObject < ApplicationRecord
+  belongs_to :bundle, foreign_key: "bundle_ref", primary_key: "stix_id"
+  belongs_to :stix_object, polymorphic: true, foreign_key: "object_ref", primary_key: "stix_id", foreign_type: :object_type
+end

data/app/models/active_stix/campaign.rb

@@ -0,0 +1,65 @@
+class ActiveStix::Campaign < ApplicationRecord
+  has_many :source_relationships, class_name: 'ActiveStix::Relationship', primary_key: 'stix_id', foreign_key: 'source_ref' #relationships where this class is the source
+  has_many :target_relationships, class_name: 'ActiveStix::Relationship', primary_key: 'stix_id', foreign_key: 'target_ref' #relationships where this class is the target
+
+  def self.ingest_json(obj)
+    camp = find_or_create_by(stix_id: obj['id'], name: obj['name'], description: obj['description'], first_seen: obj['first_seen'], last_seen: obj['last_seen'])
+    camp
+  end
+
+  before_create do
+    self.stix_id = "#{type}--#{SecureRandom.uuid}" if stix_id.blank?
+  end
+
+
+  def type
+    'campaign'
+  end
+
+
+  def attack_patterns
+    attack_pattern_relationships.map(&:target)
+  end
+
+  def attack_pattern_relationships
+    relationships = []
+    ActiveStix::Relationship.where(source: self, relationship_type: "uses", target_type: "ActiveStix::AttackPattern").each do |rel|
+      relationships << rel
+    end
+    relationships
+  end
+
+  def indicator_relationships
+    relationships = []
+    ActiveStix::Relationship.where(source_type: "ActiveStix::Indicator", relationship_type: "indicates", target: self).each do |rel|
+      relationships << rel
+    end
+    relationships
+  end
+
+  def as_stix
+    as_json(only: []).tap do |hash|
+      hash["id"] = stix_id
+      hash["type"] = type
+      hash["created"] = created_at.rfc3339(3)
+      hash["modified"] = updated_at.rfc3339(3)
+      hash["first_seen"] = created_at.rfc3339(3)
+      hash["last_seen"] = updated_at.rfc3339(3)
+      hash["name"] = name
+      hash["description"] = description
+    end
+  end
+
+  def convert_to_json
+    {
+        :type => "campaign",
+        :id => stix_id,
+        :created => created_at.to_s,
+        :modified => updated_at.to_s,
+        :name => name,
+        :description => description,
+        :first_seen => first_seen,
+        :last_seen => last_seen
+    }
+  end
+end

data/app/models/active_stix/cc_ref.rb

@@ -0,0 +1,6 @@
+module ActiveStix
+  class CcRef < ActiveStix::Recipient
+    belongs_to :email_message
+    belongs_to :email_address
+  end
+end

data/app/models/active_stix/course_of_action.rb

@@ -0,0 +1,54 @@
+module ActiveStix
+  class CourseOfAction < ApplicationRecord
+    has_many :reference_items, as: 'referrer'
+    has_many :external_references, class_name: 'ActiveStix::ExternalReference', through: :reference_items
+
+    has_many :reference_object_marking_courses, class_name: 'ActiveStix::ReferenceObjectMarkingCourse', foreign_key: 'stix_course_of_action_id'
+    has_many :marking_definitions, class_name: 'ActiveStix::MarkingDefinition', through: :reference_object_marking_courses, primary_key: 'id'
+
+    has_many :source_relationships, class_name: 'ActiveStix::Relationship', primary_key: 'stix_id', foreign_key: 'source_ref' #relationships where this class is the source
+    has_many :target_relationships, class_name: 'ActiveStix::Relationship', primary_key: 'stix_id', foreign_key: 'target_ref' #relationships where this class is the target
+
+    def self.ingest_json(obj)
+      course_of_action = find_or_create_by(stix_id: obj['id'], name: obj['name'], description: obj['description'])
+      if obj.has_key?('external_references')
+        obj['external_references'].each do |er|
+          external_reference = ExternalReference.ingest_json(er, obj['id'])
+          course_of_action.external_references << external_reference unless ActiveStix::ReferenceItem.find_by(external_reference:external_reference, referrer: course_of_action)
+        end
+      end
+      if obj.has_key?('object_marking_refs')
+        # TODO
+        # obj['object_marking_refs'].each do |mr|
+        #   marking_definition = MarkingDefinition.create_by_id(mr)
+        #   course_of_action.marking_definitions << marking_definition unless ActiveStix::ReferenceObjectMarkingCourse.find_by(stix_marking_definition_id:marking_definition.id, stix_course_of_action_id:course_of_action.id)
+        # end
+      end
+      course_of_action.save
+      course_of_action
+    end
+
+    def convert_to_json
+      external_refs_arr = []
+      external_references.each do | x |
+        external_refs_arr << x.convert_to_json
+      end
+
+      marking_def_arr = []
+      marking_definitions.each do | x |
+        marking_def_arr << x.convert_to_json
+      end
+
+      {
+          :external_references => external_refs_arr,
+          :object_marking_refs => marking_def_arr,
+          :id => stix_id,
+          :name => name,
+          :type => "course-of-action",
+          :modified => updated_at.to_s,
+          :created => created_at.to_s,
+          :description => description
+      }
+    end
+  end
+end

data/app/models/active_stix/cyber_observable.rb

@@ -0,0 +1,4 @@
+class ActiveStix::CyberObservable < ApplicationRecord
+  belongs_to :observed_datum, :class_name => 'ActiveStix::ObservedDatum', foreign_key: :observed_datum_ref, primary_key: :stix_id
+  belongs_to :cyber_observable_object, polymorphic: true, foreign_key: :observable_object_id
+end

data/app/models/active_stix/email_address.rb

@@ -0,0 +1,27 @@
+module ActiveStix
+  class EmailAddress < ApplicationRecord
+
+    has_many :to_refs, foreign_key: 'stix_email_address_id'
+    has_many :email_messages, foreign_key: :from_ref
+    belongs_to :identity, class_name: 'ActiveStix::Identity'
+
+    # after_create_commit do
+    #   assign_iddentity_to_group
+    # end
+
+    def self.from_eml(eml)
+      eml.correspondences.all.collect do |c|
+        {
+            "type": "email-address",
+            "value": c.email.address,
+            "display_name": c.email.alias,
+            "from_ref": c.agent.id
+        }
+      end
+    end
+
+    def assign_identity_to_group
+
+    end
+  end
+end

data/app/models/active_stix/email_message.rb

@@ -0,0 +1,339 @@
+module ActiveStix
+  class EmailMessage < ApplicationRecord
+    has_many :to_refs, foreign_key: 'email_message_id'
+    has_many :cc_refs, foreign_key: 'email_message_id'
+    has_many :bcc_refs, foreign_key: 'email_message_id'
+    has_many :recipients, foreign_key: 'email_message_id'
+    has_many :cyber_observables, :class_name => 'ActiveStix::CyberObservable', foreign_key: :observable_object_id, as: :cyber_observable_object
+    has_many :observed_data, through: :cyber_observables
+    belongs_to :from, class_name: 'ActiveStix::EmailAddress', foreign_key: 'from_ref'
+    belongs_to :raw_email_artifact, foreign_key: :raw_email_ref, class_name: 'ActiveStix::Artifact', primary_key: :stix_id
+
+
+    before_create do
+      self.stix_id = "#{type}--#{SecureRandom.uuid}" if stix_id.blank?
+    end
+
+    after_create do
+      ActiveStix::ObservedDatum.wrap_object(self)
+    end
+
+    def primary_observed_datum
+      observed_data.first
+    end
+
+    def from_identity
+      from.identity
+    end
+
+    def identities
+      ids = [from.identity]
+      to_refs.each do |tr|
+        ids << tr.email_address.identity
+      end
+
+      cc_refs.each do |ccr|
+        ids << ccr.email_address.identity
+      end
+
+      bcc_refs.each do |bccr|
+        ids << bccr.email_address.identity
+      end
+      ids.uniq
+    end
+
+    def mail_server
+      Mail.new(raw_email_artifact.payload_bin).message_id.split(/[@\.]/)[-3..-1]
+    end
+
+    def readable
+      eml.readable
+    end
+
+    def self.from_eml(eml)
+      eml = eml.encode(Encoding.find('UTF-8'), {invalid: :replace, undef: :replace, replace: ''}).sub("\xEF\xBB\xBF".force_encoding('UTF-8'), "").gsub("\r", "").unicode_normalize(:nfkc)
+      mail = Mail.new(eml)
+
+      #TODO Add details for cc_refs, bcc_refs etc. from the email message object
+
+      is_multipart = mail.multipart?
+      date = mail.date
+      content_type = mail.content_type
+      from = create_from(mail, eml)
+      sender_ref = create_sender(mail, eml)
+      subject = mail.header["Subject"] ? mail.header["Subject"].value : ""
+      received_lines = mail.header["Received"]
+
+      #TODO add the appropriate details here later
+      #TODO late make additional header fields more specific, check stix spec
+      email_message = EmailMessage.create(
+          "is_multipart": is_multipart,
+          "date": date,
+          "content_type": content_type,
+          "from": from,
+          "sender_ref": sender_ref,
+          "subject": subject,
+          "received_lines": received_lines,
+          "additional_header_fields": mail.header,
+          "body": mail.body,
+          "body_multipart": body_multipart(mail),
+          "raw_email_ref": raw_email_artifact(mail).stix_id
+      )
+      create_recipients(mail, eml, email_message)
+
+      email_message
+    end
+
+    def self.raw_email_artifact(eml)
+      artifact = Artifact.create_from_eml(eml)
+    end
+
+    def self.body_multipart(mail)
+      if mail.multipart?
+        mail.parts.collect {|part| part.mime_type}
+      else
+        nil
+      end
+    end
+
+    def type
+      return "email-message"
+    end
+
+    def self.extract_info_from_address_header(descriptive_name)
+      descriptive_name = descriptive_name.to_s
+      descriptive_name.gsub!('\"', '')
+      temp = descriptive_name.split(' via ')
+      fullname = temp[0].to_s
+      if temp.size > 1
+        medium = temp[1]
+      end
+      if not fullname.lstrip().start_with?('<')
+        name_parts = fullname.split(',')
+        if name_parts.size > 1
+          last_name = name_parts[0]
+          first_name = name_parts[1].lstrip()
+        else
+          name_parts = fullname.split(' ')
+          if name_parts.size > 1
+            first_name = name_parts[0]
+            last_name = name_parts[1]
+          end
+        end
+      end
+      return first_name, last_name, medium
+    end
+
+    def self.create_recipients(mail, eml, email_message)
+      if mail.header["To"]
+        get_recipients_by_type("To", mail, eml, email_message)
+      end
+
+      if mail.header["CC"]
+        get_recipients_by_type("CC", mail, eml, email_message)
+      end
+
+      if mail.header["BCC"]
+        get_recipients_by_type("BCC", mail, eml, email_message)
+      end
+    end
+
+    #TODO The two methods below can definitely be consolidated to one. Make sure to do so.
+    def self.create_from(mail, eml)
+      if mail.header["From"]
+        mail.header["From"].address_list.addresses.each do |address|
+          user_account = UserAccount.where("lower(user_id) = ?", address.address.downcase).first
+          fname, lname, medium = extract_info_from_address_header(address.display_name)
+          #TODO Create an identity here so it can be linked to the user account and email address
+          if user_account.nil?
+            identity = create_identity(address.address.downcase, address.name)
+            user_account = UserAccount.create(user_id: address.address.downcase, account_login: get_name_from_address(address.address), display_name: address.name, is_service_account: false, is_privileged: false, can_escalate_privs: false, is_disabled: false, identity_id: identity.id)
+            identity.user_accounts << user_account
+          else
+            identity = user_account.identity
+          end
+          email_address = EmailAddress.where("lower(value) = ?", address.address.downcase).first
+          if email_address.nil?
+            email_address = EmailAddress.create(value: address.address.downcase, display_name: address.name, belongs_to_ref: user_account.id, identity_id: identity.id)
+          end
+
+          return email_address
+        end
+      end
+
+    end
+
+    def self.create_sender(mail, eml)
+      if mail.from.any?
+        address = mail.from.first
+        user_account = UserAccount.where("lower(user_id) = ?", address.downcase).first
+        #TODO Create an identity here so it can be linked to the user account and email address
+        if user_account.nil?
+          identity = create_identity(address, address)
+          user_account = UserAccount.create(user_id: address, account_login: get_name_from_address(address), display_name: address, is_service_account: false, is_privileged: false, can_escalate_privs: false, is_disabled: false, identity_id: identity.id)
+          identity.users_accounts << user_account
+        else
+          identity = user_account.identity
+        end
+        email_address = EmailAddress.where("lower(value) = ?", address.downcase).first
+        if email_address.nil?
+          email_address = EmailAddress.create(value: address, display_name: address, belongs_to_ref: user_account.id, identity_id: identity.id)
+        end
+        return email_address
+      end
+    end
+
+    def self.get_recipients_by_type(recipient_type, mail, eml, email_message)
+      return if mail.header[recipient_type].errors.any? # todo we should throw and catch an exception here
+      mail.header[recipient_type].each do |address|
+        # TODO this will be the user account
+        user_account = UserAccount.where("lower(user_id) = ?", address.address.downcase).first
+
+        fname, lname, medium = extract_info_from_address_header(address.display_name)
+
+        #TODO Create an identity here so it can be linked to the user account and email address
+        if user_account.nil?
+          identity = create_identity(address.address.downcase, address.name)
+          user_account = identity.user_accounts.create(user_id: address.address, account_login: get_name_from_address(address.address), display_name: address.name, is_service_account: false, is_privileged: false, can_escalate_privs: false, is_disabled: false)
+        else
+          identity = user_account.identity
+        end
+
+        # TODO this will be the email address
+        email_address = EmailAddress.where("lower(value) = ?", address.address.downcase).first
+        if email_address.nil?
+          email_address = EmailAddress.create(value: address.address.downcase, display_name: address.name, belongs_to_ref: user_account.id, identity_id: identity.id)
+        end
+
+        # Create header specific refs 
+        create_recipient_type_ref(recipient_type, email_message, email_address)
+      end
+    end
+
+    def self.create_recipient_type_ref(recipient_type, email_message, email_address)
+
+      case recipient_type
+      when "To"
+        ToRef.create(email_message: email_message, email_address: email_address)
+      when "CC"
+        CcRef.create(email_message: email_message, email_address: email_address)
+      when "BCC"
+        BccRef.create(email_message: email_message, email_address: email_address)
+      end
+    end
+
+    def self.create_identity(address, display_name)
+      #TODO figure out where to get the details for whether for identity_class, description etc.
+      identity = Identity.create(name: display_name || Mail::Address.new(address).local, contact_information: {email_addresses: [address]}, identity_class: 'individual')
+
+      organization = from_domain_identity(address)
+      ActiveStix::Identity.employ(identity, organization)
+
+      return identity
+    end
+
+    #NOTE Instance method created for the class to use
+    def self.from_domain_identity(address)
+      org_name = Organization.company_from_email_address(address)
+      public_suffix = Organization.public_suffix_from_email_address(address)
+
+      organization = ActiveStix::Identity.find_or_create_by(name: org_name + public_suffix, identity_class: "organization")
+
+      if organization.contact_information.nil?
+        organization.update_attribute("contact_information", {domain: org_name, public_suffix: public_suffix})
+      else
+        contact_information = organization.contact_information
+        contact_information["public_suffix"] = public_suffix
+        organization.contact_information = contact_information
+        organization.save
+      end
+
+      freemail = ActiveStix::AttackPattern.find_by(name: "Free email")
+      if freemail and freemail.targets?(organization)
+        personal_organization = ActiveStix::Identity.find_or_create_by(name: address, identity_class: "organization")
+        ActiveStix::Identity.employ(identity, personal_organization)
+      end
+      organization
+    end
+
+    def from_domain_identity
+      ActiveStix::EmailMessage.from_domain_identity(from.value)
+    end
+
+    def self.get_name_from_address(address)
+      return address.split('@')[0]
+    end
+
+    def self.find_or_create_user_account(email, aliasname)
+      if email.agent.nil?
+        agent = Agent.create
+        agent.emails << email
+      else
+        agent = email.agent
+        unless agent.emails.where(alias: aliasname).any?
+          agent.emails << email
+        end
+      end
+      return agent
+    end
+
+    def stix_references
+      object_refs = {}
+      object_index = 1
+      [to_refs, cc_refs, bcc_refs].each do |field|
+        field.each do |ref|
+          object_refs[{
+
+                          "type": "email-addr",
+                          "value": ref.email_address.value.to_s,
+                          "display_name": ref.email_address.display_name.to_s
+                      }] = object_index.to_s
+          object_index += 1
+        end
+      end
+      object_refs[{
+
+                      "type": "email-addr",
+                      "value": from.value.to_s,
+                      "display_name": from.display_name.to_s
+                  }] = object_index.to_s
+      object_refs
+    end
+
+
+    def mail
+      Mail.new(eml.raw_source)
+    end
+
+    def as_stix
+
+      as_json(only: []).tap do |hash|
+        # hash["x_panacea_id"] = "<#{id}>".force_encoding("utf-8")
+        hash["type"] = "email-message"
+
+        mail = Mail.new(raw_email_artifact.payload_bin)
+        hash["date"] = mail.date.utc.iso8601(3)
+
+        #  todo fix this when jpl turns on stix
+        hash["is_multipart"] = false
+
+        # if mail.multipart?
+        #   stix_parts = mail.parts.collect do |part|
+        #     {
+        #         "content_type" => part.content_type.to_s,
+        #         "content_disposition" => part.content_transfer_encoding.to_s,
+        #         "body" => Digest::SHA1.hexdigest(part.to_s)
+        #     }
+        #   end.to_a
+        #   hash["body_multipart"] = stix_parts
+        # else
+        #   hash["body"] = eml.readable
+        # end
+
+        hash
+      end
+    end
+
+
+  end
+end