arc_weld 0.2.5

data/README.md

@@ -0,0 +1,73 @@
+# ArcWeld
+
+[![Build Status](https://travis-ci.org/ryanbreed/arc_weld.svg?branch=master)](https://travis-ci.org/ryanbreed/arc_weld)
+
+Toolkit for generating ArcSight resources. Useful for fine-grained control
+over the asset model. Currently supports generating the following resource
+types:
+
+- Asset
+- AssetCategory
+- Customer
+- Location
+- Network
+- Zone
+- Groups of the above resources
+
+Basic support classes included for building resource XML archives and reading
+resources from CSV files. You can also model supported relationships between
+the resources using the relationship extension modules. This lets you do things
+like add a location to zones, add a network to a customer, categorize resources,
+etc.
+
+
+
+## **WARNING**
+
+You will completely trash your ESM if you do not understand *exactly* what
+you are getting into with the archive tool. There's many reasons why not much
+documentation exists on the subject. `$MANAGER_HOME/bin/arcsight archive -h`
+to get started on that journey. Also, there is no guarantee that HP will
+even continue to expose the archive tool interface. It could be deprecated
+at any time in any release with no notice.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'arc_weld'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install arc_weld
+
+## Usage
+
+So far, the spec files contain the most complete description of how to use the
+resource classes and relationship modules. There is a basic cli tool for
+creating a csv-based model project and using that to generate the model xml.
+Bear in mind that you can always break up a modeling effort into multiple smaller
+projects and merge them in the target ESM if that makes more sense.
+
+`ArcWeld::Resource` and `ArcWeld::Relationship` define the core DSL for describing
+resources and relationships. The code for the resources and relationships themselves
+are defined under `lib/arc_weld/resources` and `lib/arc_weld/relationships`.
+
+Be advised - this is also an experiment for me, so some things could be better
+organized. This will happen before I cut a 1.0.0.
+
+## Licensed under GPLv3
+
+## Development
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/ryanbreed/arc_weld.

data/Rakefile

@@ -0,0 +1,6 @@
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new(:spec)
+
+task :default => :spec

data/arc_weld.gemspec

@@ -0,0 +1,34 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'arc_weld/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = 'arc_weld'
+  spec.version       = ArcWeld::VERSION
+  spec.authors       = ['Ryan Breed']
+  spec.email         = ['opensource@breed.org']
+
+  spec.summary       = 'Toolkit for building ArcSight resources'
+  spec.description   = 'Toolkit for building ArcSight resources'
+  spec.homepage      = 'https://github.com/ryanbreed/arc_weld'
+
+  spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  spec.bindir        = 'bin'
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.require_paths = ['lib']
+
+  spec.add_dependency 'gyoku'
+  spec.add_dependency 'nokogiri'
+  spec.add_dependency 'thor'
+
+  spec.add_development_dependency 'pry'
+  spec.add_development_dependency 'pry-doc'
+  spec.add_development_dependency 'bundler', '~> 1.10'
+  spec.add_development_dependency 'rake', '~> 10.0'
+  spec.add_development_dependency 'rspec'
+  spec.add_development_dependency 'guard'
+  spec.add_development_dependency 'guard-rspec'
+  spec.add_development_dependency 'guard-bundler'
+  spec.add_development_dependency 'simplecov'
+end

data/bin/setup

@@ -0,0 +1,7 @@
+#!/bin/bash
+set -euo pipefail
+IFS=$'\n\t'
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/bin/weld

@@ -0,0 +1,8 @@
+#!/usr/bin/env ruby
+$LOAD_PATH.push(File.join(File.expand_path(File.dirname(__FILE__)), '..','lib'))
+require 'arc_weld'
+if Dir.exist?(local_lib=File.join(Dir.pwd,'lib'))
+  Dir[File.join(local_lib,'*.rb')]. each {|local_lib| require local_lib }
+end
+
+ArcWeld::WeldCli.start(ARGV)

data/lib/arc_weld/archive.rb

@@ -0,0 +1,87 @@
+module ArcWeld
+  class Archive
+    attr_accessor :resources, :createTime, :dtd_location
+
+    BUILD_VERSION = '6.5.1.1952.1'
+    BUILD_TIME    = '3-6-2015_17:2:26'
+    TIME_FORMAT   = '%m-%d-%Y_%H:%M:%S'
+
+    PARSE_DEFAULT  = Nokogiri::XML::ParseOptions::DEFAULT_XML
+    PARSE_VALIDATE = Nokogiri::XML::ParseOptions::DEFAULT_XML | Nokogiri::XML::ParseOptions::DTDLOAD
+
+    ARCHIVE_DTD   = "../../schema/xml/archive/arcsight-archive.dtd"
+
+    def initialize(*args)
+      Hash[*args].each {|k,v| self.send(format('%s=',k),v)}
+      @resources    ||= []
+      @createTime   ||= Time.new
+      @dtd_location ||= ARCHIVE_DTD
+      self
+    end
+
+    def include_list
+      resources.map {|g| g.ref.render }.join('')
+    end
+
+    def add_item(item)
+      resources.push(item) unless resources.include?(item)
+    end
+
+    def add(*items)
+      items.each { |item| add_item(item) }
+    end
+
+    def dtd(location=ARCHIVE_DTD)
+      format('<!DOCTYPE archive SYSTEM "%s">', dtd_location)
+    end
+
+    def archive_creation_parameters
+      acp = {
+        ArchiveCreationParameters: {
+          :action    => 'insert',
+          'excludeReferenceIDs/' => '',
+          :format    => 'default',
+          'formatRoot/' => ''
+        }
+      }
+
+      if resources.empty?
+        acp[:ArchiveCreationParameters]['include/'] = ''
+      else
+        acp[:ArchiveCreationParameters][:include] = { :list! => include_list }
+      end
+
+      Gyoku.xml(acp, {key_converter: :none})
+    end
+
+    def create_time
+      @createTime.strftime(TIME_FORMAT)
+    end
+
+    def resource_content
+      resources.map {|res| res.render }.join('')
+    end
+
+    def to_h
+      archive_h = {
+        :archive! => archive_creation_parameters + resource_content,
+        :attributes! => {
+          :archive! => {
+            :buildVersion => BUILD_VERSION,
+            :buildTime    => BUILD_TIME,
+            :createTime   => create_time
+          }
+        }
+      }
+    end
+
+    def render
+      Gyoku.xml(to_h, {key_converter: :none} )
+    end
+
+    def xml(options=PARSE_DEFAULT)
+      Nokogiri.XML(dtd + render, nil, 'UTF-8', options)
+    end
+
+  end
+end

data/lib/arc_weld/cli/project.rb

@@ -0,0 +1,116 @@
+module ArcWeld
+  module Cli
+    class Project
+      include ArcWeld::Helpers
+      attr_accessor :name, :input_dir, :output_dir, :resource_driver,
+        :resource_root, :resource_inputs, :relationships, :base_model
+
+      def initialize(config)
+        config.each {|k,v| self.send(format('%s=',k),v)}
+        self
+      end
+
+      def resource_class_for(resource_type)
+        ArcWeld.const_get(constantize(resource_type))
+      end
+
+      def resource_input_path(resource_type)
+        File.join(self.input_dir, resource_inputs.fetch(resource_type))
+      end
+
+      def resource_reader(resource_type)
+        klass = ArcWeld::Cli::ResourceReaders.const_get(constantize(resource_driver))
+        klass.new( path:         resource_input_path(resource_type),
+                   target_class: resource_class_for(resource_type) )
+      end
+
+      def reader
+        @readers ||= Hash[
+          resource_inputs.keys.map { |type| [type, resource_reader(type)]}
+        ]
+      end
+
+      def resources
+        @resources ||= Hash.new {|h,k| h[k]=Array.new }
+      end
+
+      def archive_resources(resource_type)
+        archive.resources.select {|r| r.resource_type == constantize(resource_type)}
+      end
+
+      def archive
+        @archive ||= Archive.new
+      end
+
+      def load_resources
+        reader.each do |type, type_reader|
+          res=type_reader.to_a
+          resources[type].push(*res)
+        end
+      end
+
+      def resource_group(type)
+        klass = ArcWeld.const_get(constantize(type))
+        group_name = resource_root[type]
+        ArcWeld::Group.new(
+          name:                  group_name,
+          externalID:            format('weld_%s_root', type),
+          description:           format('ArcWeld root %s group',
+                                        constantize(type)),
+          containedResourceType: klass.class_id,
+          parent_ref:            klass.toplevel
+        )
+      end
+
+      def resource_roots
+        @roots ||= Hash[
+          resource_root.map do |type, group_name|
+            grp = resource_group(type)
+            resources['group'].push(grp)
+            [type, grp ]
+          end
+        ]
+      end
+
+      def zone_assets
+        resources['asset'].each do |asset|
+          asset.auto_zone(*resources['zone'])
+        end
+      end
+
+      def assign_model_relations
+        relationships.each do |rel|
+          klass = ArcWeld::Cli::RelationReaders.const_get(constantize(rel['driver']))
+          rel_reader = klass.new(
+            path:               File.join(input_dir,rel['file']),
+            relationship_type: rel['type'] )
+          sources = resources[rel_reader.src_type]
+          dests   = resources[rel_reader.dst_type]
+          rel_reader.relate(sources,dests)
+        end
+      end
+
+      def assign_resource_groups
+        resource_roots['zone'].add_children(*resources['zone'])
+
+        zone_assets
+
+        unzoned = resources['asset'].select {|a| a.in_zone.nil?}
+        resource_roots['asset'].add_children(*unzoned)
+
+        %w{ network customer location asset_category }.each do |type|
+          resource_roots[type].add_children(*resources[type])
+        end
+      end
+
+      def generate_archive
+        resource_roots
+        load_resources
+        assign_resource_groups
+        assign_model_relations
+        archive.add(*resources.values.flatten)
+        archive
+      end
+    end
+  end
+end

data/lib/arc_weld/cli/readers/relation_csv.rb

@@ -0,0 +1,41 @@
+module ArcWeld
+  module Cli
+    module RelationReaders
+      class Csv
+        include ArcWeld::Helpers
+        attr_accessor :path, :relationship_type,
+                      :src_select, :dst_select,
+                      :src_class,  :dst_class
+
+        attr_reader   :csv, :keys, :src_type, :dst_type
+
+        def initialize(*args)
+          Hash[*args].each {|k,v| self.send(format('%s=',k),v)}
+          @csv = CSV.open(path)
+          @keys = csv.readline
+          @src_type, @src_select, @dst_type, @dst_select = keys.flat_map do |k|
+             k.split(':')
+          end
+          @src_class=ArcWeld.const_get(constantize(src_type))
+          @dst_class=ArcWeld.const_get(constantize(dst_type))
+          self
+        end
+
+        def relate(sources,destinations)
+          csv.each do |selected_src, selected_dst|
+            src_instance=sources.find {|s| s.send(src_select)==selected_src}
+            dst_instance=destinations.find {|s| s.send(dst_select)==selected_dst}
+            if (src_instance.nil? || dst_instance.nil?)
+              STDERR.puts format('could not %s %s/%s',
+                                 relationship_type,
+                                 selected_src,
+                                 selected_dst)
+            else
+               src_instance.send(relationship_type,dst_instance)
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/arc_weld/cli/readers/resource_csv.rb

@@ -0,0 +1,64 @@
+require 'csv'
+
+module ArcWeld
+  module Cli
+    module ResourceReaders
+      class Csv
+        attr_accessor :path, :target_class, :key_proc
+        attr_reader   :keys, :csv
+        DEFAULT_OPTIONS = {skip_lines: %r{\A[#;]}}
+  
+        def initialize(path: nil, target_class: nil, options: DEFAULT_OPTIONS)
+          @path, @target_class = path, target_class
+          @csv = CSV.open(path, 'r', options)
+          @key_proc ||= {}
+          @keys = csv.readline.map { |k| k.to_sym }
+        end
+  
+        def read_keys
+          keys - processed_keys
+        end
+  
+        def processed_keys
+          key_proc.keys
+        end
+  
+        def parse(array)
+          row_hash = Hash[keys.zip(array)]
+          instance_hash = row_hash.select {|k,v| read_keys.include?(k)}
+          new_instance = target_class.new(instance_hash)
+  
+          if processed_keys.empty?
+            new_instance
+          else
+            processed_hash = row_hash.select {|k,v| read_keys.include?(k)==false }
+            processed_hash.each do |processed_key, processed_value|
+              key_proc.fetch(processed_key).call(processed_key,processed_value,new_instance)
+            end
+            new_instance
+          end
+        end
+  
+        def not_at_start_record?
+          (csv.lineno!=1)
+        end
+  
+        def each(rewind: not_at_start_record?)
+          reset_csv if rewind
+          csv.each {|arr| yield parse(arr) }
+        end
+  
+        def to_a(rewind: not_at_start_record?)
+          reset_csv if rewind
+          csv.readlines.map {|arr| parse(arr)}
+        end
+  
+        def reset_csv
+          csv.rewind
+          *trash = csv.readline
+          csv.lineno
+        end
+      end
+    end
+  end
+end

data/lib/arc_weld/cli.rb

@@ -0,0 +1,45 @@
+require 'arc_weld'
+require 'arc_weld/cli/project'
+require 'arc_weld/cli/readers/relation_csv'
+require 'arc_weld/cli/readers/resource_csv'
+require 'openssl'
+require 'json'
+module ArcWeld
+  class WeldCli < Thor
+
+   include Thor::Actions
+    def self.source_root
+      @source_root ||= File.join( File.dirname( File.expand_path(__FILE__) ),
+                                  'templates')
+    end
+
+    class_option :config,
+      type: :string,
+      default: 'weld.json',
+      desc: 'config file'
+
+    desc 'new PROJECT_NAME', 'generate new project'
+    option :dir,
+      type: :string,
+      default: Dir.pwd,
+      desc: 'create project directory here'
+    option :driver,
+      type: :string,
+      default: 'csv',
+      desc: 'driver for reading resources and relations'
+    def new(name)
+      self.destination_root = options[:dir]
+      @config = options.merge(name: name)
+      directory(format('project-%s',options[:driver]),name)
+    end
+
+    desc 'generate FILENAME', 'render entire model in FILENAME'
+    def generate(filename='model.xml')
+      config = JSON.parse(File.read(options[:config]))
+      project = ArcWeld::Cli::Project.new(config)
+      outfile = File.join(project.output_dir, filename)
+      archive = project.generate_archive
+      File.open(outfile,'w') {|f| f.puts archive.xml.to_s }
+    end
+  end
+end

data/lib/arc_weld/helpers.rb

@@ -0,0 +1,12 @@
+module ArcWeld
+  module Helpers
+    def constantize(literal)
+      (literal.to_s.split(%r{[ _]}).map &:capitalize).join('')
+    end
+
+    def uri_join(*parts)
+      pieces=parts.flat_map {|p| p.to_s.gsub(%r{\A/},'').gsub(%r{/\z},'')}
+      format('/%s', pieces.join('/'))
+    end
+  end
+end

data/lib/arc_weld/reference.rb

@@ -0,0 +1,51 @@
+module ArcWeld
+  class Reference
+    attr_accessor :type, :uri, :id, :externalID
+
+    def initialize(*args)
+      Hash[*args].each {|k,v| self.send(format('%s=',k),v)}
+      @type ||= 'Group'
+    end
+
+    def ref
+      self
+    end
+
+    def ==(other)
+      [:type, :uri, :id, :externalID].all? do |attr|
+        self.send(attr) == other.send(attr)
+      end
+    end
+
+    def identity_hash
+      if identity=={}
+        {}
+      else
+        Hash[identity.keys.map {|k| format('@%s',k.to_s)}.zip(identity.values)]
+      end
+    end
+
+    def identity
+      if id!=nil
+        {:id => id}
+      elsif externalID!=nil
+        {:externalID => externalID}
+      else
+        {}
+      end
+    end
+
+    def to_h
+     {
+        'ref/' => {
+          :@type => type,
+          :@uri   => uri
+        }.merge(identity_hash)
+      }
+    end
+
+    def render
+      Gyoku.xml(to_h, key_converter: :none)
+    end
+  end
+end

data/lib/arc_weld/relationship.rb

@@ -0,0 +1,96 @@
+require 'arc_weld/relationships/customer_of'
+require 'arc_weld/relationships/location_of'
+require 'arc_weld/relationships/has_alternate_interface'
+require 'arc_weld/relationships/has_child'
+require 'arc_weld/relationships/has_customer'
+require 'arc_weld/relationships/has_location'
+require 'arc_weld/relationships/in_category'
+require 'arc_weld/relationships/in_network'
+require 'arc_weld/relationships/in_zone'
+
+module ArcWeld
+  module Relationship
+
+    def self.included(klass)
+      klass.class_eval '@@RELATIONSHIPS = []'
+      klass.class_eval do
+        extend ClassMethods
+      end
+
+    end
+
+    def relationship_types
+      self.class.class_variable_get :@@RELATIONSHIPS
+    end
+
+    def relationship_hash
+      relationship_types.reduce({}) do |memo, key|
+        meth = format('%s_relationship',key)
+        if self.send(meth).nil?
+          memo
+        else
+          memo.merge(self.send(meth))
+        end
+      end
+    end
+
+    module ClassMethods
+      def has_relationship(*args)
+        #puts format('defining relationship on %s', self.name)
+        options=case args[-1]
+          when Hash
+            args.pop
+          else
+            Hash.new(false)
+        end
+
+        args.each do |sym|
+          register_relationship(sym,options[:multiple])
+        end
+      end
+
+      def class_relationship_types
+        class_variable_get :@@RELATIONSHIPS
+      end
+
+      def register_relationship(name, multiple=false)
+        unless class_relationship_types.include?(name)
+          class_relationship_types << name
+
+          if ArcWeld::Relationships.const_defined? constantize(name)
+            self.class_eval "include ArcWeld::Relationships::#{constantize(name)}"
+          end
+          # TODO: this is a horrible mess
+          unless self.method_defined? name
+            if multiple
+              self.class_eval do
+                define_method("#{name}") do
+                  val = self.instance_variable_get("@#{name}")
+                  if val.nil?
+                    self.instance_variable_set("@#{name}",[])
+                  end
+                  self.instance_variable_get("@#{name}")
+                end
+              end
+            else
+              self.class_eval do
+                define_method("#{name}") do
+                  self.instance_variable_get("@#{name}")
+                end
+              end
+            end
+          end
+          unless self.method_defined? "#{name}="
+            self.class_eval do
+              define_method("#{name}=") do |val|
+                self.instance_variable_set("@#{name}",val)
+              end
+            end
+          end
+
+
+        end
+      end
+    end
+  end
+end

data/lib/arc_weld/relationships/customer_of.rb

@@ -0,0 +1,28 @@
+module ArcWeld
+  module Relationships
+    module CustomerOf
+      def add_customer_resource(res)
+        unless customer_of.include?(res)
+          customer_of.push(res)
+          res.add_customer(self)
+        end
+      end
+
+      def add_customer_resources(*resources)
+        resources.each { |res| add_customer_resource(res) }
+      end
+
+      def related_customer_of_references
+        (customer_of.map { |resource| resource.ref.render }).join
+      end
+
+      def customer_of_relationship
+        unless customer_of.empty?
+          { 'customerOf' => {
+              'list!' => related_customer_of_references
+          }}
+        end
+      end
+    end
+  end
+end