plan_executor 1.0.2

data/lib/tasks/tasks.rake

@@ -0,0 +1,329 @@
+namespace :crucible do
+
+  FHIR_SERVERS = [
+    # DSTU1
+    # 'http://fhir.healthintersections.com.au/open',
+    # 'http://bonfire.mitre.org:8080/fhir',
+    # 'http://spark.furore.com/fhir',
+    # 'http://nprogram.azurewebsites.net',
+    # 'https://fhir-api.smartplatforms.org',
+    # 'https://fhir-open-api.smartplatforms.org',
+    # 'https://fhir.orionhealth.com/blaze/fhir',
+    # 'http://worden.globalgold.co.uk:8080/FHIR_a/farm/cobalt',
+    # 'http://worden.globalgold.co.uk:8080/FHIR_a/farm/bronze',
+    # 'http://fhirtest.uhn.ca/base'
+
+    # DSTU2
+    'https://fhir-api-dstu2.smarthealthit.org',
+    'http://fhirtest.uhn.ca/baseDstu2',
+    'http://bp.oridashi.com.au',
+    'http://md.oridashi.com.au',
+    'http://zm.oridashi.com.au',
+    'http://wildfhir.aegis.net/fhir2',
+    'http://fhir-dev.healthintersections.com.au/open'
+  ]
+
+  desc 'console'
+  task :console, [] do |t, args|
+    binding.pry
+  end
+
+  desc 'execute all'
+  task :execute_all, [:url, :html_summary] do |t, args|
+    require 'benchmark'
+    b = Benchmark.measure {
+      client = FHIR::Client.new(args.url)
+      options = client.get_oauth2_metadata_from_conformance
+      set_client_secrets(client,options) unless options.empty?
+      results = execute_all(client)
+      if args.html_summary
+        generate_html_summary(args.url, results, "ExecuteAll")
+      end
+    }
+    puts "Execute All completed in #{b.real} seconds."
+  end
+
+  desc 'execute all test scripts'
+  task :execute_all_testscripts, [:url, :html_summary] do |t, args|
+    require 'benchmark'
+    b = Benchmark.measure {
+      client = FHIR::Client.new(args.url)
+      options = client.get_oauth2_metadata_from_conformance
+      set_client_secrets(client,options) unless options.empty?
+      results = Crucible::Tests::TestScriptEngine.new(client).execute_all
+      output_results results
+      if args.html_summary
+        generate_html_summary(args.url, results, "ExecuteAll")
+      end
+    }
+    puts "Execute All completed in #{b.real} seconds."
+  end
+
+  desc 'execute'
+  task :execute, [:url, :test, :resource] do |t, args|
+    require 'benchmark'
+    b = Benchmark.measure {
+      client = FHIR::Client.new(args.url)
+      options = client.get_oauth2_metadata_from_conformance
+      set_client_secrets(client,options) unless options.empty?
+      execute_test(client, args.test, args.resource)
+    }
+    puts "Execute #{args.test} completed in #{b.real} seconds."
+  end
+
+  desc 'metadata'
+  task :metadata, [:url, :test] do |t, args|
+    require 'benchmark'
+    b = Benchmark.measure { collect_metadata(FHIR::Client.new(args.url), args.test) }
+    puts "Metadata #{args.test} completed in #{b.real} seconds."
+  end
+
+  def set_client_secrets(client,options)
+    puts "Using OAuth2 Options: #{options}"
+    print 'Enter client id: '
+    client_id = STDIN.gets.chomp
+    print 'Enter client secret: '
+    client_secret = STDIN.gets.chomp
+    if client_id!="" && client_secret!=""
+      options[:client_id] = client_id
+      options[:client_secret] = client_secret
+      # set_oauth2_auth(client,secret,authorizePath,tokenPath)
+      client.set_oauth2_auth(options[:client_id],options[:client_secret],options[:authorize_url],options[:token_url])
+    else
+      puts "Ignoring OAuth2 credentials: empty id or secret. Using unsecured client..."
+    end
+  end
+
+  def execute_test(client, key, resourceType=nil)
+    executor = Crucible::Tests::Executor.new(client)
+    test = executor.find_test(key)
+    if test.nil?
+      puts "Unable to find test: #{key}"
+      return
+    end
+    results = nil
+    if !resourceType.nil? && test.respond_to?(:resource_class=)
+      fhir_classes = Mongoid.models.select {|c| c.name.include? 'FHIR'}
+      klass = fhir_classes.find{|x|x.to_s.include?(resourceType)}
+      results = test.execute(klass) if !klass.nil?
+    end
+    results = executor.execute(test) if results.nil?
+    output_results results
+  end
+
+  def execute_all(client)
+    results = Crucible::Tests::Executor.new(client).execute_all
+    output_results results
+  end
+
+  def execute_multiserver_test(client, client2, key)
+    executor = Crucible::Tests::Executor.new(client, client2)
+    output_results executor.execute(executor.find_test(key))
+  end
+
+  def collect_metadata(client, test)
+    output_results Crucible::Tests::Executor.new(client).metadata(test), true
+  end
+
+  def output_results(results, metadata_only=false)
+    require 'turn'
+    results.keys.each do |suite_key|
+      puts suite_key
+      results[suite_key].each do |test|
+        puts write_result(test['status'], test[:test_method], test['message'])
+        if test['status'].upcase=='ERROR'
+          puts " "*12 + "-"*40
+          puts " "*12 + "#{test['data'].gsub("\n","\n"+" "*12)}"
+          puts " "*12 + "-"*40
+        end
+        if (metadata_only==true)
+          # warnings
+          puts (test['warnings'].map { |w| "#{(' '*10)}WARNING: #{w}" }).join("\n") if test['warnings']
+          # metadata
+          puts (test['links'].map { |w| "#{(' '*10)}Link: #{w}" }).join("\n") if test['links']
+          puts (test['requires'].map { |w| "#{(' '*10)}Requires: #{w[:resource]}: #{w[:methods]}" }).join("\n") if test['requires']
+          puts (test['validates'].map { |w| "#{(' '*10)}Validates: #{w[:resource]}: #{w[:methods]}" }).join("\n") if test['validates']
+          # data
+          puts (' '*10) + test['data'] if test['data']
+        end
+      end
+    end
+    results
+  end
+
+  def generate_html_summary(url, results, id="summary")
+    require 'erb'
+    require 'tilt'
+    require 'fileutils'
+    totals = Hash.new(0)
+    metadata = Hash.new(0)
+    results.values.each do |suite|
+      suite.map{|t| t["status"]}.each_with_object(totals) { |n, h| h[n] += 1}
+      suite.map{|t| {k: t["id"], v: t["validates"], s: t["status"]}}.each_with_object(metadata) do |n, h|
+        n[:v].each do |val|
+          resource = val[:resource].try(:titleize).try(:downcase)
+          test_key = n[:k]
+          h[resource] = {pass: [], fail: [], error: [], skip: []} unless h.keys.include?(resource)
+          h[resource][n[:s].to_sym] << test_key
+          val[:methods].each do |meth|
+            h[meth] = {pass: [], fail: [], error: [], skip: []} unless h.keys.include?(meth)
+            h[meth][n[:s].to_sym] << test_key
+          end if val[:methods]
+          val[:formats].each do |format|
+            h[format] = {pass: [], fail: [], error: [], skip: []} unless h.keys.include?(format)
+            h[format][n[:s].to_sym] << test_key
+          end if val[:formats]
+        end if n[:v]
+      end
+    end
+    template = Tilt.new(File.join(File.dirname(__FILE__), "templates", "summary.html.erb"))
+    timestamp = Time.now
+    summary = template.render(self, {:results => results, :timestamp => timestamp.strftime("%D %r"), :totals => totals, :url => url, :metadata => metadata})
+    summary_file = "#{id}_#{url.gsub(/[^a-z0-9]/,'-')}_#{timestamp.strftime("%m-%d-%y_%H-%M-%S")}.html"
+    FileUtils::mkdir_p("html_summaries/#{id}")
+    File.write("html_summaries/#{id}/#{summary_file}", summary)
+    system("open html_summaries/#{id}/#{summary_file}")
+  end
+
+  desc 'execute custom'
+  task :execute_custom, [:test, :resource_type, :html_summary] do |t, args|
+    require 'benchmark'
+
+    puts "# #{args.test}"
+    puts
+
+    seconds = 0.0
+
+    FHIR_SERVERS.each do |url|
+      puts "## #{url}"
+      puts "```"
+      b = Benchmark.measure {
+        client = FHIR::Client.new(url)
+        options = client.get_oauth2_metadata_from_conformance
+        set_client_secrets(client,options) unless options.empty?
+        results = execute_test(client, args.test, args.resource_type)
+        if args.html_summary
+          generate_html_summary(url, results, args.test)
+        end
+      }
+      seconds += b.real
+      puts "```"
+      puts
+    end
+    puts "Execute Custom #{args.test} completed for #{FHIR_SERVERS.length} servers in #{seconds} seconds."
+  end
+
+  desc 'execute all custom'
+  task :execute_all_custom, [:html_summary] do |t, args|
+    require 'benchmark'
+
+    puts "# #{args.test}"
+    puts
+
+    seconds = 0.0
+
+    FHIR_SERVERS.each do |url|
+      puts "## #{url}"
+      puts "```"
+      b = Benchmark.measure {
+        client = FHIR::Client.new(url)
+        options = client.get_oauth2_metadata_from_conformance
+        set_client_secrets(client,options) unless options.empty?
+        results = execute_all(client)
+        if args.html_summary
+          generate_html_summary(url, results, "ExecuteAll")
+        end
+      }
+      seconds += b.real
+      puts "```"
+      puts
+    end
+    puts "Execute All Custom #{args.test} completed for #{FHIR_SERVERS.length} servers in #{seconds} seconds."
+  end
+
+  desc 'list all'
+  task :list_all do
+    require 'benchmark'
+    b = Benchmark.measure { puts Crucible::Tests::Executor.list_all }
+    puts "List all tests completed in #{b.real} seconds."
+  end
+
+  desc 'list names of test suites'
+  task :list_suites do
+    require 'benchmark'
+    b = Benchmark.measure do
+      suites = Crucible::Tests::Executor.list_all
+      suite_names = []
+      suites.each do |key,value|
+        suite_names << value['author'].split('::').last if !key.start_with?('TS')
+      end
+      suite_names.uniq!
+      suite_names.each {|x| puts "  #{x}"}
+    end
+    puts "List all suites completed in #{b.real} seconds."
+  end
+
+  desc 'list all test scripts'
+  task :list_all_test_scripts do
+    require 'benchmark'
+    b = Benchmark.measure { puts Crucible::Tests::TestScriptEngine.list_all.keys }
+    puts "List all tests completed in #{b.real} seconds."
+  end
+
+  desc 'execute with requirements'
+  task :execute_w_requirements, [:url, :test] do |t, args|
+    require 'turn'
+
+    module Crucible
+      module Tests
+        class BaseTest
+
+          alias execute_test_method_orig execute_test_method
+
+          def execute_test_method(test_method)
+            r = execute_test_method_orig(test_method)
+            @@requirements ||= {}
+            @@requirements[self.class.name] ||= {}
+            @@requirements[self.class.name][test_method] = @client.requirements
+            @client.clear_requirements
+            r
+          end
+
+        end
+      end
+    end
+
+    client = FHIR::Client.new(args.url)
+    options = client.get_oauth2_metadata_from_conformance
+    set_client_secrets(client,options) unless options.empty?
+    client.monitor_requirements
+    test = args.test.to_sym
+    execute_test(client, test)
+    pp Crucible::Tests::BaseTest.class_variable_get :@@requirements
+  end
+
+
+  def write_result(status, test_name, message)
+    tab_size = 10
+    "#{' '*(tab_size - status.length)}#{Turn::Colorize.method(status.to_sym).call(status.upcase)} #{test_name}: #{message}"
+  end
+
+  namespace :multiserver do
+    desc 'execute'
+    task :execute, [:url1, :url2, :test] do |t, args|
+      require 'turn'
+      require 'benchmark'
+      b = Benchmark.measure {
+        client1 = FHIR::Client.new(args.url1)
+        options = client1.get_oauth2_metadata_from_conformance
+        set_client_secrets(client1,options) unless options.empty?
+        client2 = FHIR::Client.new(args.url2)
+        options = client2.get_oauth2_metadata_from_conformance
+        set_client_secrets(client2,options) unless options.empty?
+        execute_multiserver_test(client1, client2, args.test)
+      }
+      puts "Execute multiserver #{args.test} completed in #{b.real} seconds."
+    end
+  end
+
+end

data/lib/tasks/templates/summary.html.erb

@@ -0,0 +1,163 @@
+<html>
+<head>
+  <title>Execute All Summary :: <%= timestamp %></title>
+  <!-- Latest compiled and minified CSS -->
+  <link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.4/css/bootstrap.min.css">
+
+  <!-- Optional theme -->
+  <link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.4/css/bootstrap-theme.min.css">
+</head>
+<body>
+  <div class="container">
+    <h1>Executed <%= results.length %> Suites, <%= timestamp %></h1>
+    <h2><small><a href="<%= url %>" target="_blank"><%= url %></a></small></h2>
+    <div class="well well-sm">
+      <button class="btn btn-danger">
+        <span class="badge"><%= totals["error"] %></span> Errors
+      </button>
+      <button class="btn btn-warning">
+        <span class="badge"><%= totals["fail"] %></span> Failures
+      </button>
+      <button class="btn btn-success">
+        <span class="badge"><%= totals["pass"] %></span> Passes
+      </button>
+      <button class="btn btn-info">
+        <span class="badge"><%= totals["skip"] %></span> Skips
+      </button>
+    </div>
+
+    <div class="well well-sm">
+    <% metadata.each do |leaf, tags| %>
+      <div class="row">
+        <div class="col-md-2"><strong><%= leaf %></strong></div>
+        <div class="col-md-10">
+          <div class="row">
+          <% tags.each do |leaf, tags| %>
+            <% next unless tags.length>0 %>
+            <div class="col-md-2"><strong><%= tags.length %> <%= leaf.to_s.try(:pluralize) %></strong></div>
+            <div class="col-md-10"><%= tags %></div>
+          <% end %>
+        </div>
+        </div>
+      </div>
+    <% end %>
+    </div>
+
+    <% results.keys.each_with_index do |suite_key, index| %>
+      <% suite = results[suite_key] %>
+      <% errored = suite.detect{|t| t["status"] == "error"} %>
+      <% failed = suite.detect{|t| t["status"] == "fail"} || suite.detect{|t| t["status"] == "pass"}.nil? %>
+      <div class="panel panel-<%= if errored then "danger" elsif failed then "warning" else "success" end %>">
+        <div class="panel-heading" role="tab" id="heading<%= index %>">
+          <h3 class="panel-title">
+            <a data-toggle="collapse" data-parent="#accordion" href="#collapse<%= index %>" aria-expanded="true" aria-controls="collapse<%= index %>">
+              <%= suite_key.to_s %>
+            </a>
+          </h3>
+        </div>
+        <div id="collapse<%= index %>" class="panel-collapse collapse" role="tabpanel" aria-labelledby="heading<%= index %>">
+          <div class="panel-body">
+
+            <% suite.each do |test| %>
+              <% status_map = {"pass" => "success", "fail" => "warning", "error" => "danger", "skip" => "info"} %>
+              <div class="panel panel-<%= status_map[test["status"]]%>">
+                <div class="panel-heading">
+                  <%= test["status"].upcase %> - <%= test["key"] %> - <%= test[:test_method].to_s %>
+                </div>
+                <div class="panel-body">
+                  <% test.each do |key, value| %>
+                    <% next if ["key", "code", :test_method].include?(key) %>
+                    <% next if key=="data" && test["status"]!="error" %>
+                    <div class="row">
+                      <div class="col-md-1"><strong><%= key %></strong></div>
+                      <div class="col-md-11">
+                        <% if key == "data" %><pre><% end %>
+                        <% if value.class == Hash %>
+                          <% value.each do |k, v| %>
+                            <div class="row">
+                              <div class="col-md-1"><%= k %></div>
+                              <div class="col-md-11"><%= v %></div>
+                            </div>
+                          <% end %>
+                        <% elsif value.class == Array %>
+                          <% value.each do |v| %>
+                            <% if v.class == Hash %>
+                              <% v.each do |vk, vv| %>
+                                <div class="row">
+                                  <div class="col-md-1"><%= vk %></div>
+                                  <div class="col-md-11"><%= vv %></div>
+                                </div>
+                              <% end %>
+                            <% else %>
+                              <div class="row">
+                                <div class="col-md-12"><%= v %></div>
+                              </div>
+                            <% end %>
+                          <% end %>
+                        <% else %>
+                          <%= value %>
+                        <% end %>
+                        <% if key == "data" %></pre><% end %>
+                      </div>
+                    </div>
+                  <% end %>
+                </div>
+              </div>
+            <% end %>
+
+          </div>
+        </div>
+      </div>
+    <% end %>
+
+    <div class="text-center">
+      <div class="btn btn-primary elevator-button">
+        <span class="glyphicon glyphicon-triangle-top" aria-hidden="true"></span> Back to Top
+      </div>
+    </div>
+  </div>
+  <script src="https://code.jquery.com/jquery-1.11.2.min.js"></script>
+  <!-- Latest compiled and minified JavaScript -->
+  <script src="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.4/js/bootstrap.min.js"></script>
+  <script>
+    var Elevator=function(){"use strict";function n(n,e){for(var t in e)t in n||(n[t]=e[t]);return n}function e(n,e,t,u){return n/=u/2,1>n?t/2*n*n+e:(n--,-t/2*(n*(n-2)-1)+e)}function t(n){p||(p=n);var u=n-p,o=e(u,v,-v,A);window.scrollTo(0,o),A>u?s=requestAnimationFrame(t):i()}function u(){w||(w=!0,v=document.documentElement.scrollTop||m.scrollTop,f||(A=1.5*v),requestAnimationFrame(t),c&&c.play())}function o(){p=null,v=null,w=!1}function i(){o(),c&&(c.pause(),c.currentTime=0),d&&d.play()}function r(){w&&(cancelAnimationFrame(s),o(),c&&(c.pause(),c.currentTime=0),window.scrollTo(0,0))}function l(n){n.addEventListener("click",u,!1)}function a(n){m=document.body,n.element&&l(n.element),n.duration&&(f=!0,A=n.duration),n.mainAudio&&(c=new Audio(n.mainAudio),c.setAttribute("preload","true"),c.setAttribute("loop","true")),n.endAudio&&(d=new Audio(n.endAudio),d.setAttribute("preload","true")),window.addEventListener("blur",r,!1)}var c,d,m=null,s=null,A=null,f=!1,p=null,v=null,w=!1;return n(a,{elevate:u})}();
+  </script>
+  <script>
+    function toggleErrors() {
+      $(".panel-body > .panel-danger").parents(".panel-danger > .panel-collapse").collapse('toggle');
+      $(".btn-danger").toggleClass("active");
+    }
+
+    function toggleFailures() {
+      $(".panel-body > .panel-warning").parents(".panel-collapse").collapse('toggle');
+      $(".btn-warnings").toggleClass("active");
+    }
+
+    function togglePasses() {
+      $(".panel-body > .panel-success").parents(".panel-collapse").collapse('toggle');
+      $(".btn-success").toggleClass("active");
+    }
+
+    function toggleSkips() {
+      $(".panel-body > .panel-info").parents(".panel-collapse").collapse('toggle');
+      $(".btn-info").toggleClass("active");
+    }
+
+    $(".btn-danger").click(toggleErrors);
+    $(".btn-warning").click(toggleFailures);
+    $(".btn-success").click(togglePasses);
+    $(".btn-info").click(toggleSkips);
+  </script>
+  <script>
+  // Elevator script included on the page, already.
+
+  window.onload = function() {
+    var elevator = new Elevator({
+      element: document.querySelector('.elevator-button'),
+      mainAudio: 'elevator-music.mp3',
+      endAudio: 'ding.mp3'
+    });
+  }
+  </script>
+</body>
+</html>

data/lib/test_result.rb

@@ -0,0 +1,69 @@
+module Crucible
+  module Tests
+    class TestResult
+
+      attr_accessor :key
+      attr_accessor :id
+      attr_accessor :description
+      attr_accessor :status
+      attr_accessor :message
+      attr_accessor :data
+      attr_accessor :warnings
+      attr_accessor :requires
+      attr_accessor :validates
+      attr_accessor :links
+      attr_accessor :code
+
+      def initialize(key, description, status, message, data)
+        @key = key
+        @status = status
+        @description = description
+        @message = message
+        @data = data
+      end
+
+      def update(status, message, data)
+        @status = status
+        @message = message
+        @data = data
+        self
+      end
+
+      def passed?
+        return ( (@status==true) or (@status=='passed') )
+      end
+
+      def failed?
+        !self.passed?
+      end
+
+      def to_hash
+        hash = {}
+        hash['key'] = @key
+        hash['id'] = @id || @key
+        hash['description'] = force_encoding(@description)
+        hash['status'] = force_encoding(@status)
+        if @message.class == Array
+          hash['message'] = @message.map { |m| force_encoding(m) }
+        else
+          hash['message'] = force_encoding(@message)
+        end
+        hash['data'] = force_encoding(@data) unless hash['status'] == 'pass'
+        hash['warnings'] = warnings if warnings
+        hash['requires'] = requires if requires
+        hash['validates'] = validates if validates
+        hash['links'] = links if links
+        hash['code'] = @code
+        hash
+      end
+
+      private
+
+      def force_encoding(value)
+        return nil if value.blank?
+        value.force_encoding("UTF-8")
+      end
+
+    end
+  end
+end