kriterion 0.0.1 → 0.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 42aff9f85dd3bbed3821c4028322c8918b5dc3e0ec7a5ce46691b6eb737aef7b
-  data.tar.gz: 62929aa1fd49a1f16739c0914b4739678e7e551198796635a1b52d092a88ebcb
+  metadata.gz: 2ab8ab52ebad0244b1765403f402b557a447230df58eb131c0dcbe2e57a6c56a
+  data.tar.gz: ad31c92bc460c6ac60b4cfe9040cc2a2df2072c9b1c03938e5898462621fe095
 SHA512:
-  metadata.gz: 3ae4cf132a87e5d2834c9cb13f6e7ff9c352856192ba8efa041eb28a31c21f6b28f12afa5513a2303f3d4ff56925ba80d3226cd0ff049128e87059c534f1e4bc
-  data.tar.gz: 04d59dc56805ce22a40d77cdd7779a17b8baf43db965d93c2d3b6db2740f7e57e11cef86c3b9bb2790a16734d9995769134c40a9e38e45ab1c9754314341c072
+  metadata.gz: 31aef891445eb70aa59053bb9fa198ca8e0a8cbf3e8d540ee752b8282e368643ea03a470122f5a1cf8a75e2ac8457fd9d9d769b5049e65b669ca83a2b7759152
+  data.tar.gz: 9b0ba3e9dd101cd8d91e37ae8a3a40cca2edd3476124e02fb0d4215fbf21458e420c96e046793e433ffc6ef19eba18263e08747bf0f5e74a424f7c05ec784d1c

data/.rubocop.yml

@@ -0,0 +1,3 @@
+
+Style/ClassVars:
+  Enabled: false

data/Dockerfile

@@ -14,5 +14,4 @@ ENV mongo_port 27017
 ENV queue reports
 ENV uri http://restmq:8888
 
-ENTRYPOINT ["bundle", "exec", "kriterion", "worker"]
-CMD ["--debug"]
+ENTRYPOINT ["bundle", "exec", "kriterion"]

data/Gemfile

@@ -9,4 +9,6 @@ group :development do
   gem 'nokogiri'
   gem 'pry'
   gem 'pry-byebug'
+  gem 'kubeclient'
+  gem 'googleauth'
 end

data/Gemfile.lock

@@ -1,57 +1,119 @@
 PATH
   remote: .
   specs:
-    kriterion (0.0.1)
+    kriterion (0.1.0)
       cri (~> 2.10)
       httparty (~> 0.16)
       mongo (~> 2.5)
+      sinatra (~> 2.0)
 
 GEM
   remote: https://rubygems.org/
   specs:
+    addressable (2.5.2)
+      public_suffix (>= 2.0.2, < 4.0)
     bson (4.3.0)
     byebug (10.0.2)
     coderay (1.1.2)
     colored (1.2)
-    cri (2.10.1)
+    cri (2.12.0)
       colored (~> 1.2)
     diff-lcs (1.3)
+    domain_name (0.5.20180417)
+      unf (>= 0.0.5, < 1.0.0)
+    faraday (0.15.2)
+      multipart-post (>= 1.2, < 3)
+    googleauth (0.6.4)
+      faraday (~> 0.12)
+      jwt (>= 1.4, < 3.0)
+      memoist (~> 0.12)
+      multi_json (~> 1.11)
+      os (>= 0.9, < 2.0)
+      signet (~> 0.7)
+    http (3.3.0)
+      addressable (~> 2.3)
+      http-cookie (~> 1.0)
+      http-form_data (~> 2.0)
+      http_parser.rb (~> 0.6.0)
+    http-cookie (1.0.3)
+      domain_name (~> 0.5)
+    http-form_data (2.1.1)
+    http_parser.rb (0.6.0)
     httparty (0.16.2)
       multi_xml (>= 0.5.2)
+    jwt (2.1.0)
+    kubeclient (4.0.0)
+      http (~> 3.0)
+      recursive-open-struct (~> 1.0, >= 1.0.4)
+      rest-client (~> 2.0)
+    memoist (0.16.0)
     method_source (0.9.0)
+    mime-types (3.2.2)
+      mime-types-data (~> 3.2015)
+    mime-types-data (3.2018.0812)
     mini_portile2 (2.3.0)
-    mongo (2.6.1)
+    mongo (2.6.2)
       bson (>= 4.3.0, < 5.0.0)
+    multi_json (1.13.1)
     multi_xml (0.6.0)
+    multipart-post (2.0.0)
+    mustermann (1.0.2)
+    netrc (0.11.0)
     nokogiri (1.8.4)
       mini_portile2 (~> 2.3.0)
+    os (1.0.0)
     pry (0.11.3)
       coderay (~> 1.1.0)
       method_source (~> 0.9.0)
     pry-byebug (3.6.0)
       byebug (~> 10.0)
       pry (~> 0.10)
+    public_suffix (3.0.3)
+    rack (2.0.5)
+    rack-protection (2.0.3)
+      rack
     rake (10.5.0)
-    rspec (3.7.0)
-      rspec-core (~> 3.7.0)
-      rspec-expectations (~> 3.7.0)
-      rspec-mocks (~> 3.7.0)
-    rspec-core (3.7.1)
-      rspec-support (~> 3.7.0)
-    rspec-expectations (3.7.0)
+    recursive-open-struct (1.1.0)
+    rest-client (2.0.2)
+      http-cookie (>= 1.0.2, < 2.0)
+      mime-types (>= 1.16, < 4.0)
+      netrc (~> 0.8)
+    rspec (3.8.0)
+      rspec-core (~> 3.8.0)
+      rspec-expectations (~> 3.8.0)
+      rspec-mocks (~> 3.8.0)
+    rspec-core (3.8.0)
+      rspec-support (~> 3.8.0)
+    rspec-expectations (3.8.1)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.7.0)
-    rspec-mocks (3.7.0)
+      rspec-support (~> 3.8.0)
+    rspec-mocks (3.8.0)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.7.0)
-    rspec-support (3.7.1)
+      rspec-support (~> 3.8.0)
+    rspec-support (3.8.0)
+    signet (0.8.1)
+      addressable (~> 2.3)
+      faraday (~> 0.9)
+      jwt (>= 1.5, < 3.0)
+      multi_json (~> 1.10)
+    sinatra (2.0.3)
+      mustermann (~> 1.0)
+      rack (~> 2.0)
+      rack-protection (= 2.0.3)
+      tilt (~> 2.0)
+    tilt (2.0.8)
+    unf (0.1.4)
+      unf_ext
+    unf_ext (0.0.7.5)
 
 PLATFORMS
   ruby
 
 DEPENDENCIES
   bundler (~> 1.16)
+  googleauth
   kriterion!
+  kubeclient
   nokogiri
   pry
   pry-byebug

data/README.md

@@ -1,9 +1,5 @@
 # Kriterion
 
-Welcome to your new gem! In this directory, you'll find the files you need to be able to package up your Ruby library into a gem. Put your Ruby code in the file `lib/kriterion`. To experiment with that code, run `bin/console` for an interactive prompt.
-
-TODO: Delete this and the text above, and describe your gem
-
 ## Installation
 
 Add this line to your application's Gemfile:
@@ -28,6 +24,8 @@ TODO: Write usage instructions here
 
 This project requires MongoDB and RestMQ to be up and working. You can run them up manually using the commands below, or run `docker-compose up` to spin up everything.
 
+To populate the queue with example reports, run `bundle exec ruby spec/populate_queue.rb http://localhost:8888`
+
 ### Docker Containers
 
 #### `kriterion_worker`

data/bin/build.sh

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+
+docker build -t kriterion-base .
+docker build -t kriterion-worker docker/worker
+docker build -t kriterion-api docker/api

data/bin/kill_pods_in_deployment.rb

@@ -0,0 +1,60 @@
+#! /bin/env ruby
+require 'kubeclient'
+require 'pry'
+
+deployment     = ARGV[0]
+auth_method    = ARGV[1]
+
+namespace = 'kriterion'
+
+case auth_method
+when 'k8s'
+  # Use the token file that exists within K8s
+  auth_options = {
+    bearer_token_file: '/var/run/secrets/kubernetes.io/serviceaccount/token'
+  }
+
+  ssl_options = {}
+  if File.exist?('/var/run/secrets/kubernetes.io/serviceaccount/ca.crt')
+    ssl_options[:ca_file] = '/var/run/secrets/kubernetes.io/serviceaccount/ca.crt'
+  end
+
+  client = Kubeclient::Client.new(
+    'https://kubernetes.default.svc',
+    'v1',
+    auth_options: auth_options,
+    ssl_options:  ssl_options
+  )
+when 'local'
+  # require 'googleauth'
+  config        = Kubeclient::Config.read(ENV['KUBECONFIG'] || '~/.kube/config')
+  context       = config.context
+
+  auth_options = {
+    bearer_token: ENV['KUBETOKEN']
+  }
+
+  client = Kubeclient::Client.new(
+    context.api_endpoint,
+    context.api_version,
+    ssl_options: context.ssl_options,
+    auth_options: auth_options
+  )
+end
+
+puts "Discovering Services..."
+client.discover
+
+puts "Getting Pods..."
+
+client.get_pods(namespace: namespace).each do |pod|
+  # Check if it's one that we want to kill
+  if pod[:metadata][:labels][:deployment] == deployment
+    puts "Found pod #{pod[:metadata][:name]} in deployment #{pod[:metadata][:labels][:deployment]}"
+    puts "Killing #{pod[:metadata][:name]}..."
+    client.delete_pod(pod[:metadata][:name], namespace)
+    puts "Done."
+  end
+end
+
+puts "Done."

data/config.ru

@@ -0,0 +1,2 @@
+require './lib/kriterion/api'
+run Kriterion::API

data/criterion.gemspec

@@ -24,6 +24,7 @@ Gem::Specification.new do |spec|
   spec.add_runtime_dependency 'cri', '~> 2.10'
   spec.add_runtime_dependency 'httparty', '~> 0.16'
   spec.add_runtime_dependency 'mongo', '~> 2.5'
+  spec.add_runtime_dependency 'sinatra', '~> 2.0'
 
   spec.add_development_dependency 'bundler', '~> 1.16'
   spec.add_development_dependency 'rake', '~> 10.0'

data/docker-compose.yml

@@ -2,7 +2,16 @@ version: '3'
 services:
   worker:
     build: .
+    image: kriterion
+    command: worker
     tty: true
+  api:
+    build: .
+    image: kriterion
+    command: api
+    tty: true
+    ports:
+      - "4567:4567"
   mongodb:
     image: mongo:latest
     ports:

data/lib/kriterion/api.rb

@@ -1,27 +1,122 @@
 require 'json'
-require 'mongo'
+require 'sinatra/base'
 require 'kriterion/logs'
-include Kriterion::Logs
+require 'kriterion/connector'
 
 class Kriterion
-  class API
-    attr_reader :mongo
-    attr_reader :standards_dir
+  class API < Sinatra::Application
+    attr_accessor :queue_uri
+    attr_accessor :metrics
+    attr_accessor :backend
 
-    def initialize(opts)
-      if opts[:debug]
-        logger.level = Kriterion::Logs::DEBUG
+    include Kriterion::Logs
+
+    # We only every want to have one instance of this running at a time. This is
+    # required due to the way that Sinatra initialises things This adds the
+    # initialize method etc.
+    @@instance = nil
+
+    def initialize(opts = {})
+      # If there is already an instance, copy the objects from that
+      if @@instance
+        @queue_uri = @@instance.queue_uri
+        @metrics   = @@instance.metrics
+        @backend   = @@instance.backend
+      else
+        @queue_uri, @metrics, @backend = Kriterion::Connector.connections(opts)
+        @@instance = self
+      end
+      super()
+      logger.info "Initialised Kritioner API version #{Kriterion::VERSION}"
+    end
+
+    set :bind, '0.0.0.0'
+    set :environment, :test
+
+    # Set headers and default values
+    before do
+      headers 'Content-Type' => 'application/json'
+    end
+
+    get '/standards' do
+      find 'standards'
+    end
+
+    get '/standards/:name' do |name|
+      find 'standards', name: name
+    end
+
+    get '/sections' do
+      find 'sections', standard: options[:standard]
+    end
+
+    get '/sections/:uuid' do |uuid|
+      find 'sections', uuid: uuid
+    end
+
+    get '/sections/:uuid/sections' do |uuid|
+      # Get the main section
+      parent = backend.find_section(
+        { uuid: uuid },
+        recurse: true
+      )
+
+      # Return all direct children
+      parent.sections.map do |section|
+        section.to_h(options[:mode])
+      end.to_json
+    end
+
+    get '/resources' do
+      find 'resources', parent_uuid: options[:item]
+    end
+
+    get '/resources/:uuid' do |uuid|
+      find 'resources', uuid: uuid
+    end
+
+    private
+
+    # Finds things in the backend and returns them as JSON
+    def find(thing, query = {})
+      result = backend.send("find_#{thing}", query, recurse: options[:recurse])
+
+      if result.is_a? Array
+        result.map do |object|
+          object.to_h(options[:mode])
+        end.to_json
+      else
+        result.to_h(options[:mode]).to_json
+      end
+    end
+
+    # Returns options that are relevant to the queries we will be doing based
+    # on the params that were passed
+    def options
+      # Defualt level should be full
+      level = params['level'] || 'full'
+
+      # Convert all other params to symbols for later use
+      sym_params = params.each_with_object({}) do |memo, (k, v)|
+        memo[k.to_sym] = v
+        memo
       end
 
-      @mongo_hostname = opts[:mongo_hostname]
-      @mongo_port     = opts[:mongo_port]
-      @mongo_database = opts[:mongo_database]
-      @mongo          = Mongo::Client.new([ "#{@mongo_hostname}:#{@mongo_port}" ], :database => @mongo_database)
-      @standards_dir  = opts[:standards_dir]
+      # Return all data
+      sym_params.merge(mode_options[level])
     end
 
-    def run
-      # Find all standards and add them to mongodb
+    def mode_options
+      {
+        'basic' => {
+          recurse: false,
+          mode: :basic
+        },
+        'full' => {
+          recurse: true,
+          mode: :full
+        }
+      }
     end
   end
 end

data/lib/kriterion/backend.rb

@@ -1,6 +1,73 @@
+require 'kriterion/metrics'
+
+
 class Kriterion
   class Backend
-    @@nackend = nil
+    @@backend = nil
+
+    attr_reader :metrics
+
+    def initialize(opts)
+      @metrics = opts[:metrics] || Kriterion::Metrics.new
+    end
+
+    # This is the meat of what a backend is. This section defines the following
+    # methods:
+    #
+    #   - find_standard
+    #   - find_standards
+    #   - find_section
+    #   - find_sections
+    #   - find_item
+    #   - find_items
+    #   - find_resource
+    #   - find_resources
+    #
+    # These are the main methods that Kriterion will use to interact with the
+    # backend. The backend must in turn implement the following methods:
+    #
+    #   - find(object_type, query, options)
+    #   - insert(object)
+    #
+    %i[
+      standard
+      section
+      item
+      resource
+      event
+    ].each do |thing|
+      define_method("find_#{thing}") do |query, opts|
+        validate_opts opts
+
+        # Call out to find_{thing}s
+        results = send("find_#{thing}s", query, opts)
+
+        # Validate that we only have one result and return it
+        return results.first if results.count == 1
+        raise "Found > 1 #{thing} with name: #{name}" if results.count > 1
+        nil
+      end
+
+      define_method("find_#{thing}s") do |query, opts|
+        result = nil
+        metrics[:"backend_find_#{thing}s"] += Benchmark.realtime do
+          result = find(thing, query, opts)
+        end
+        result
+      end
+
+      define_method("ensure_#{thing}") do |object|
+        pk    = object.primary_key
+        query = {
+          pk => object.send(pk)
+        }
+
+        metrics[:backend_insert] += Benchmark.realtime do
+          insert(object) unless send("find_#{thing}", query, recurse: false)
+        end
+        object
+      end
+    end
 
     def self.set(backend)
       @@backend = backend
@@ -9,5 +76,18 @@ class Kriterion
     def self.get
       @@backend
     end
+
+    private
+
+    # Validate options hash
+    def validate_opts(opts)
+      valid_keys = [
+        :recurse
+      ]
+
+      unless opts.keys.all? { |k| valid_keys.include?(k) }
+        raise "Options hash is invalid #{opts}"
+      end
+    end
   end
 end

data/lib/kriterion/backend/mongodb.rb

@@ -23,11 +23,10 @@ class Kriterion
       attr_reader :resources_db
       attr_reader :events_db
       attr_reader :standard_details_db
-      attr_reader :metrics
 
       def initialize(opts)
+        super(opts)
         logger.info 'Initializing MongoDB backend'
-        @metrics             = opts[:metrics] || Kriterion::Metrics.new
         @hostname            = opts[:hostname]
         @port                = opts[:port]
         @database            = opts[:database]
@@ -43,47 +42,23 @@ class Kriterion
         @standard_details_db = @client[:standard_details]
       end
 
-      def get_standard(name, opts = {})
-        standard = nil
-        metrics[:backend_get_standard] += Benchmark.realtime do
-          # Set recursion to false by default
-          opts[:recurse] = opts[:recurse] || false
+      def find(type, query, opts)
+        database_for(type).find(query).map do |result|
+          # Sanitise the data if required
+          params = sanitise_data(type, result)
 
-          standard = sanitise_standard(find_standard(name))
-          return nil if standard.nil?
+          # Create the object and return in an array
+          object = class_for(type).new(params)
 
-          find_children!(standard) if opts[:recurse]
-        end
-
-        standard
-      end
+          # Find children if required
+          find_children!(object) if opts[:recurse]
 
-      def find_sections(query)
-        sections_db.find(
-          query
-        ).map do |section|
-          Kriterion::Section.new(section)
+          object
         end
       end
 
-      def add_standard(standard)
-        insert_into_db(standards_db, standard)
-      end
-
-      def add_section(section)
-        insert_into_db(sections_db, section)
-      end
-
-      def add_item(item)
-        insert_into_db(items_db, item)
-      end
-
-      def add_resource(resource)
-        insert_into_db(resources_db, resource)
-      end
-
-      def add_event(event)
-        insert_into_db(events_db, event)
+      def insert(object)
+        insert_into_db(database_for(object), object)
       end
 
       def add_unchanged_node(resource, certname)
@@ -96,14 +71,7 @@ class Kriterion
       end
 
       def update_compliance!(thing)
-        databases = {
-          Kriterion::Standard => standards_db,
-          Kriterion::Section  => sections_db,
-          Kriterion::Item     => items_db,
-          Kriterion::Resource => resources_db
-        }
-
-        databases[thing.class].update_one(
+        database_for(thing).update_one(
           { uuid: thing.uuid },
           '$set' => {
             compliance: thing.compliance
@@ -128,6 +96,42 @@ class Kriterion
 
       private
 
+      def sanitise_data(type, data)
+        if type == :standard
+          return nil if data.nil?
+          # Compile the regex from a lazy-compiled BSON regex back to a ruby one
+          data['item_syntax'] = data['item_syntax'].compile
+        end
+        data  
+      end
+
+      # Returns the database for a given object type
+      def database_for(object)
+        cls = class_for(object)
+        databases = {
+          Kriterion::Standard => @standards_db,
+          Kriterion::Section  => @sections_db,
+          Kriterion::Item     => @items_db,
+          Kriterion::Resource => @resources_db,
+          Kriterion::Event    => @events_db
+        }
+        databases[cls]
+      end
+
+      def class_for(name)
+        classes = {
+          'standard' => Kriterion::Standard,
+          'section'  => Kriterion::Section,
+          'item'     => Kriterion::Item,
+          'resource' => Kriterion::Resource,
+          'event'    => Kriterion::Event
+        }
+        # If someone has passed in an object, just return the class
+        return name.class if classes.value? name.class
+
+        classes[name.to_s]
+      end
+
       def find_children!(object)
         accepted_objects = [
           Kriterion::Standard,
@@ -209,27 +213,6 @@ class Kriterion
           Kriterion::Section.new(section)
         end
       end
-
-      def find_standard(name)
-        result = standards_db.find(name: name)
-        count  = result.count
-        case count
-        when 0
-          nil
-        when 1
-          result.first
-        else
-          raise "Found > 1 standards with name: #{name}"
-        end
-      end
-
-      # Takes a result and sanities it to Kriterion::Standard object
-      def sanitise_standard(result)
-        return nil if result.nil?
-        # Compile the regex from a lazy-compiled BSON regex back to a ruby one
-        result['item_syntax'] = result['item_syntax'].compile
-        Kriterion::Standard.new(result)
-      end
     end
   end
 end

data/lib/kriterion/cli.rb

@@ -26,3 +26,4 @@ class Kriterion
 end
 
 require 'kriterion/cli/worker'
+require 'kriterion/cli/api'

data/lib/kriterion/cli/api.rb

@@ -14,17 +14,21 @@ class Kriterion
             exit 0
           end
 
-          option   :u, :standards_dir,            'URI of the RestMQ server', argument: :required
-          optional :h, :mongo_hostname, 'Hostname of the MongoDB server to use', default: 'localhost'
-          optional :d, :mongo_database, 'Name of the MongoDB database to use', default: 'kriterion'
-          optional :p, :mongo_port,     'Port for MongoDB', default: 27017
+          optional :u, :uri           , 'URI of the RestMQ server'             , default: ENV['uri'] || 'http://localhost:8888'
+          optional :q, :queue         , 'Queue to subscribe to'                , default: ENV['queue']|| 'reports'
+          optional :h, :mongo_hostname, 'Hostname of the MongoDB server to use', default: ENV['mongo_hostname']|| 'localhost'
+          optional :d, :mongo_database, 'Name of the MongoDB database to use'  , default: ENV['mongo_database']|| 'kriterion'
+          optional :p, :mongo_port    , 'Port for MongoDB'                     , default: ENV['mongo_port']|| 27017
 
 
           run do |opts, args, cmd|
             # TODO: Get log levels working properly
             require 'kriterion/api'
-            worker = Kriterion::API.new(opts)
-            worker.run
+
+            # Create a new worker withe the options we want, run! will detect
+            # this
+            Kriterion::API.new(opts)
+            Kriterion::API.run!
           end
         end
       end

data/lib/kriterion/connector.rb

@@ -0,0 +1,42 @@
+require 'kriterion/logs'
+require 'kriterion/metrics'
+require 'kriterion/backend'
+require 'kriterion/backend/mongodb'
+
+class Kriterion
+  module Connector
+    def self.connections(opts = {})
+      logger.level = if opts[:debug]
+                       Kriterion::Logs::DEBUG
+                     else
+                       Kriterion::Logs::INFO
+                     end
+
+      # Set up connections
+      uri       = opts[:uri]
+      queue     = opts[:queue]
+      queue_uri = URI("#{uri}/q/#{queue}")
+      metrics   = Kriterion::Metrics.new
+
+      # Set up the backend
+      # TODO: Clean this up and make fully dynamic
+      backend_name = opts[:backend] || 'mongodb'
+      case backend_name
+      when 'mongodb'
+        require 'kriterion/backend/mongodb'
+        Kriterion::Backend.set(
+          Kriterion::Backend::MongoDB.new(
+            hostname: opts[:mongo_hostname],
+            port:     opts[:mongo_port],
+            database: opts[:mongo_database],
+            metrics:  metrics
+          )
+        )
+      end
+
+      backend = Kriterion::Backend.get
+
+      [queue_uri, metrics, backend]
+    end
+  end
+end

data/lib/kriterion/event.rb

@@ -31,6 +31,23 @@ class Kriterion
       @corrective_change = data['corrective_change']
       @certname          = data['certname']
       @resource          = data['resource']
+      @full_description  = "#{@certname}/#{@resource}/#{@property}: #{@message}"
+    end
+
+    def full_description
+      # We want to update this when it is called to ensure it is up to date.
+      # This could just be amethod instead of an instance variable but that
+      # would mean that it wouldn't get stored in the database, which we want.
+      @full_description = "#{@certname}/#{@resource}/#{@property}: #{@message}"
+    end
+
+    # Resources don't have compliance so we don't want this to do anything
+    def compliance
+      nil
+    end
+
+    def primary_key
+      :full_description
     end
   end
 end

data/lib/kriterion/item.rb

@@ -13,6 +13,8 @@ class Kriterion
     attr_accessor :resources
 
     def initialize(data)
+      super(data)
+
       @uuid         = data['uuid'] || SecureRandom.uuid
       @id           = data['id']
       @title        = data['title']
@@ -38,5 +40,17 @@ class Kriterion
       parents.delete(id)
       parents.reverse
     end
+
+    def expandable?
+      true
+    end
+
+    def expandable_keys
+      [:resources]
+    end
+
+    def self.primary_key
+      :id
+    end
   end
 end

data/lib/kriterion/metrics.rb

@@ -18,5 +18,9 @@ class Kriterion
         logger.info "  #{name} #{value.round(2)}s"
       end
     end
+
+    def reset!
+      @metrics = {}
+    end
   end
 end

data/lib/kriterion/object.rb

@@ -1,32 +1,60 @@
 class Kriterion
   class Object
+    def initialize(data)
+      @compliance = data['compliance']
+    end
+
     def to_h(mode = :basic)
       raise 'Mode must be :basic or :full' unless %i[basic full].include? mode
       hash = {}
 
+      # Add all instance variables to the hash without the @ sign
       instance_variables.each do |v|
         hash[v.to_s.gsub(/^@/, '')] = instance_variable_get(v.to_s)
       end
 
       if mode == :basic
         hash.reject do |k, _v|
-          %w[
-            sections
-            items
-            resources
-            events
-          ].include? k
+          full_keys.include? k
         end
       elsif mode == :full
+        expandable_keys.each do |key|
+          hash[key.to_s] = send(key).map { |x| x.to_h(:full) }
+        end
         hash
       end
     end
 
+    def full_keys
+      %w[
+        sections
+        items
+        resources
+        events
+      ]
+    end
+
+    # Objects should deflault to not being expandable unless someone has
+    # specifided it
+    def expandable?
+      false
+    end
+
+    def expandable_keys
+      []
+    end
+
     def find_section(name)
       sections ? sections.select { |s| s.name == name }[0] : nil
     end
 
+    # Returns the cahced complicance value or calculates from scratch if
+    # required
     def compliance(objects)
+      # Returns cached value if it exists
+      return @compliance if @compliance
+
+      # Calculate compliance
       total         = objects.count
       compliant     = objects.count { |o| o.compliance['compliant'] }
       non_compliant = total - compliant
@@ -46,5 +74,26 @@ class Kriterion
         }
       }
     end
+
+    def flush_compliance!
+      @compliance = nil
+      # Flush the compliance of all children also
+      expandable_keys.each do |key|
+        send(key).each do |thing|
+          thing.flush_compliance!
+          yield(thing) if block_given?
+        end
+      end
+      yield(self) if block_given?
+      compliance
+    end
+
+    def primary_key
+      self.class.primary_key
+    end
+
+    def self.primary_key
+      :name
+    end
   end
 end

data/lib/kriterion/resource.rb

@@ -30,6 +30,15 @@ class Kriterion
       @events           = hash['events'] || []
       @parent_uuid      = hash['parent_uuid']
       @unchanged_nodes  = hash['unchanged_nodes'] || []
+      @compliance       = hash['compliance']
+    end
+
+    def expandable?
+      true
+    end
+
+    def expandable_keys
+      [:events]
     end
 
     def ==(other)
@@ -37,6 +46,9 @@ class Kriterion
     end
 
     def compliance
+      # Returns cached value if it exists
+      return @compliance if @compliance
+
       compliant     = unchanged_nodes.count
       non_compliant = events.group_by(&:certname).count
       total         = compliant + non_compliant
@@ -56,5 +68,9 @@ class Kriterion
         }
       }
     end
+
+    def self.primary_key
+      :title
+    end
   end
 end

data/lib/kriterion/section.rb

@@ -11,6 +11,8 @@ class Kriterion
     attr_accessor :sections
 
     def initialize(data)
+      super(data)
+
       @uuid        = data['uuid'] || SecureRandom.uuid
       @name        = data['name']
       @standard    = data['standard']
@@ -21,6 +23,17 @@ class Kriterion
       @parent_uuid = data['parent_uuid']
     end
 
+    def expandable?
+      true
+    end
+
+    def expandable_keys
+      %i[
+        sections
+        items
+      ]
+    end
+
     def type
       :section
     end

data/lib/kriterion/standard.rb

@@ -17,6 +17,7 @@ class Kriterion
     attr_accessor :items
 
     def initialize(data)
+      super(data)
       @uuid              = data['uuid'] || SecureRandom.uuid
       @name              = data['name']
       @date              = data['date']
@@ -54,6 +55,17 @@ class Kriterion
       @@standards = backend.standards
     end
 
+    def expandable?
+      true
+    end
+
+    def expandable_keys
+      %i[
+        sections
+        items
+      ]
+    end
+
     def type
       :standard
     end

data/lib/kriterion/version.rb

@@ -1,3 +1,3 @@
 class Kriterion
-  VERSION = "0.0.1"
+  VERSION = '0.1.0'
 end

data/lib/kriterion/worker.rb

@@ -1,67 +1,37 @@
 require 'json'
-require 'logger'
 require 'net/http'
 require 'benchmark'
 require 'kriterion'
-require 'kriterion/logs'
 require 'kriterion/item'
 require 'kriterion/report'
 require 'kriterion/metrics'
 require 'kriterion/section'
 require 'kriterion/standard'
-
-require 'pry'
+require 'kriterion/connector'
 
 class Kriterion
   class Worker
     include Kriterion::Logs
+    include Kriterion::Connector
 
-    attr_reader :uri
-    attr_reader :queue
     attr_reader :queue_uri
     attr_reader :standards
     attr_reader :backend
     attr_reader :metrics
 
     def initialize(opts = {})
-      logger.level = if opts[:debug]
-                       Kriterion::Logs::DEBUG
-                     else
-                       Kriterion::Logs::INFO
-                     end
-
-      # Set up connections
-      @uri       = opts[:uri]
-      @queue     = opts[:queue]
-      @queue_uri = URI("#{@uri}/q/#{@queue}")
-      @metrics   = Kriterion::Metrics.new
-
-      # Set up the backend
-      # TODO: Clean this up and make fully dynamic
-      backend_name = opts[:backend] || 'mongodb'
-      case backend_name
-      when 'mongodb'
-        require 'kriterion/backend/mongodb'
-        Kriterion::Backend.set(
-          Kriterion::Backend::MongoDB.new(
-            hostname: opts[:mongo_hostname],
-            port:     opts[:mongo_port],
-            database: opts[:mongo_database],
-            metrics:  metrics
-          )
-        )
-      end
-
-      @backend = Kriterion::Backend.get
+      @queue_uri, @metrics, @backend = Kriterion::Connector.connections(opts)
 
-      # TODO: Work out how workers are going to get the list of standards frmo the API runner
+      # TODO: Work out how workers are going to get the list of standards frmo
+      # the API runner
       # TODO: Remove placeholder code
       standards_dir   = File.expand_path('standards', Kriterion::ROOT)
       @standards      = Kriterion.standards([standards_dir])
+      logger.info "Initialised Kritioner worker version #{Kriterion::VERSION}"
     end
 
     def process_report(report)
-      report  = Kriterion::Report.new(report)
+      report = Kriterion::Report.new(report)
 
       # Check if the report contains any relevant resources
       standard_names     = standards.keys
@@ -85,16 +55,19 @@ class Kriterion
       end
 
       affected_standards.each do |name, resources|
-        standard = backend.get_standard(name, recurse: true)
+        standard = backend.find_standard({ name: name }, recurse: true)
         unless standard
           # If the standard doesn't yet exist in the backed, add it
           standard = Kriterion::Standard.new(@standards[name])
           logger.debug "Adding starndard #{standard.name} to backend"
-          backend.add_standard(standard)
+          backend.ensure_standard(standard)
           # TODO: See if there is a better way to deal with this, the reason I'm
           # doing this is that I want to make sure that there is not difference
           # between a newly created object and one that came from the database
-          standard = backend.get_standard(name, recurse: true)
+          standard = backend.find_standard(
+            { name: name },
+            recurse: true
+          )
         end
 
         resources.each do |resource|
@@ -124,21 +97,10 @@ class Kriterion
               if previous.find_section(current)
                 previous.find_section(current)
               else
-                # This is a new section that does not yet exist in the database,
-                # we therefore need to get the details and all them all in
-                current_section_name = if previous.is_a? Kriterion::Standard
-                                         current
-                                       elsif previous.is_a? Kriterion::Section
-                                         [
-                                           previous.name,
-                                           current
-                                         ].join(standard.section_separator)
-                                       end
-
                 # Get the details from the standards database (name, description
                 # etc.)
                 current_section = @standards[name]['sections'].select do |s|
-                  s['name'] == current_section_name
+                  s['name'] == current
                 end[0]
 
                 if current_section.nil?
@@ -151,7 +113,7 @@ class Kriterion
                   current_section = Kriterion::Section.new(current_section)
 
                   # Add the section to the backend
-                  backend.add_section(current_section)
+                  backend.ensure_section(current_section)
                   current_section
                 end
               end
@@ -172,7 +134,7 @@ class Kriterion
                    item_details['parent_uuid']  = section.uuid
                    item_details['parent_type']  = section.type
                    item_details['section_path'] = captures
-                   backend.add_item(Kriterion::Item.new(item_details))
+                   backend.ensure_item(Kriterion::Item.new(item_details))
                  else
                    raise "Found muliple sections with the id #{section_tag}"
                  end
@@ -183,7 +145,7 @@ class Kriterion
           # Add the new resource to the backend if it doesn't exist
           unless item.resources.include? resource
             item.resources << resource
-            backend.add_resource(resource)
+            backend.ensure_resource(resource)
           end
 
           # Inform the database that this node is unchanged if we have no events
@@ -196,38 +158,24 @@ class Kriterion
             event          = Kriterion::Event.new(event)
             event.certname = report.certname
             event.resource = resource.resource
-            backend.add_event(event)
+            backend.ensure_event(event)
             event
           end
-
-          metrics[:update_compliance] += Benchmark.realtime do
-            # Finally update the compliance details for this resource and its
-            # parent item
-            backend.update_compliance! resource
-            backend.update_compliance! item
-
-            # Find all of the parent sections and update the compliance on them
-            # Don't recalculate the compliance of the standard yet, wait until
-            # the end.
-            item.parent_names(standard.section_separator).each do |parent|
-              # TODO: Complete this so that it updates the compliance of
-              # everything. It's probably better if we re-query this stuff from
-              # the database to reduce the chances of race conditions
-              result = backend.find_sections(
-                name: parent,
-                standard: standard.name
-              )
-              result.each { |r| backend.update_compliance! r }
-            end
-          end
         end
 
         # Reload the standard as new sections may have been added
-        standard = backend.get_standard(name, recurse: true)
+        standard = backend.find_standard(
+          { name: name },
+          recurse: true
+        )
 
         metrics[:update_compliance] += Benchmark.realtime do
-          # Recalculate the compliance of a given standard once it is done
-          backend.update_compliance! standard
+          # Recalculate the compliance of a given standard once it is done (This
+          # also calculates the compliacne of all children and yeilds them to
+          # block)
+          standard.flush_compliance! do |child|
+            backend.update_compliance! child
+          end
         end
       end
     end
@@ -254,6 +202,7 @@ class Kriterion
             end
 
             metrics.print
+            metrics.reset!
           end
         rescue Timeout::Error, Errno::EINVAL, Errno::ECONNRESET, EOFError,
                Net::HTTPBadResponse, Net::HTTPHeaderSyntaxError,

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: kriterion
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.1.0
 platform: ruby
 authors:
 - Dylan Ratcliffe
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2018-07-27 00:00:00.000000000 Z
+date: 2018-08-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: cri
@@ -52,6 +52,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '2.5'
+- !ruby/object:Gem::Dependency
+  name: sinatra
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -103,6 +117,7 @@ extensions: []
 extra_rdoc_files: []
 files:
 - ".gitignore"
+- ".rubocop.yml"
 - ".ruby-version"
 - ".travis.yml"
 - Dockerfile
@@ -111,8 +126,11 @@ files:
 - LICENSE.txt
 - README.md
 - Rakefile
+- bin/build.sh
+- bin/kill_pods_in_deployment.rb
 - bin/setup
 - bin/update_stigs.rb
+- config.ru
 - criterion.gemspec
 - docker-compose.yml
 - exe/kriterion
@@ -123,6 +141,7 @@ files:
 - lib/kriterion/cli.rb
 - lib/kriterion/cli/api.rb
 - lib/kriterion/cli/worker.rb
+- lib/kriterion/connector.rb
 - lib/kriterion/event.rb
 - lib/kriterion/item.rb
 - lib/kriterion/logs.rb