earl-report 0.0.1

data/README.md

@@ -0,0 +1,99 @@
+# earl-report
+============
+
+Ruby gem to consolidate multiple EARL report and generate a rollup conformance report.
+
+## Description
+
+Reads a test manifest in the
+[standard RDF WG format](http://www.w3.org/2011/rdf-wg/wiki/Turtle_Test_Suite)
+and generates a rollup report in RDFa+HTML.
+
+## Test Specifications
+The test manifest is presumed to be of the following form:
+
+### Manifest Header
+
+The manifest header looks like:
+
+    <>  rdf:type mf:Manifest ;
+        rdfs:comment "Turtle tests" ;
+        mf:entries
+        (
+        ....
+        ) .
+
+where .... is a list of links to test descriptions, one per line.
+
+### Test description
+
+This is an example of a synatx test:
+
+    <#turtle-syntax-file-01> rdf:type rdft:TestTurtlePositiveSyntax ;
+       mf:name    "turtle-syntax-file-01" ;
+       rdfs:comment "Further description of the test" ;
+       mf:action  <turtle-syntax-file-01.ttl> ;
+       mf:result  <turtle-eval-struct-01.nt> .
+
+## Individual EARL reports
+
+Results for individual implementations should be specified in Turtle form, but
+may be specified in an any compatible RDF serialization (JSON-LD is presumed to
+be a cached rollup report). The report is composed of `Assertion` declarations
+in the following form:
+
+    [ a earl:Assertion;
+      earl:assertedBy <http://greggkellogg.net/foaf#me>;
+      earl:subject <http://rubygems.org/gems/rdf-turtle>;
+      earl:test <http://dvcs.w3.org/hg/rdf/raw-file/e80b58a1a711/rdf-turtle/tests-ttl/manifest.ttl#turtle-syntax-file-01>;
+      earl:result [
+        a earl:TestResult;
+        earl:outcome earl:passed;
+        dc:date "2012-11-17T15:19:11-05:00"^^xsd:dateTime];
+      earl:mode earl:automatic ] .
+
+Additionally, `earl:subject` is expected to reference a [DOAP]() description
+of the reported software, in the following form:
+
+    <http://rubygems.org/gems/rdf-turtle> a doap:Project, earl:TestSubject, earl:Software ;
+      doap:name          "RDF::Turtle" ;
+      doap:developer     <http://greggkellogg.net/foaf#me> ;
+      doap:homepage      <http://ruby-rdf.github.com/rdf-turtle> ;
+      doap:description   "RDF::Turtle is an Turtle reader/writer for the RDF.rb library suite."@en ;
+      doap:programming-language "Ruby" .
+
+The [DOAP]() description may be included in the [EARL]() report. If not found,
+the IRI identified by `earl:subject` will be dereferenced and is presumed to
+provide a [DOAP]() specification of the test subject.
+
+The `doap:developer` is expected to reference a [FOAF]() profile for the agent
+(user or organization) responsible for the test subject. It is expected to be
+of the following form:
+
+    <http://greggkellogg.net/foaf#me> foaf:name "Gregg Kellogg" .
+
+If not found, the IRI identified by `doap:developer`
+will be dereferenced and is presumed to provide a [FOAF]() profile of the developer.
+
+## Usage
+
+The `earl` command may be used to directly create a report from zero or more input files, which are themselves [EARL][] report.
+
+    gem install earl-report
+    
+    earl \
+      --output FILE     # Location for generated report
+      --tempate [FILE]  # Location of report template file; returns default if not specified
+      --bibRef          # The default ReSpec-formatted bibliographic reference for the report
+      --name            # The name of the software being reported upon
+      manifest          # one or more test manifests used to define test descriptions
+      report*           # one or more EARL report in most RDF formats
+
+## Report generation template
+
+The report template is in ReSpec form using [Haml]() to generate individual elements.
+
+[DOAP]: https://github.com/edumbill/doap/wiki
+[EARL]: http://www.w3.org/TR/EARL10-Schema/
+[FOAF]: http://xmlns.com/foaf/spec/
+[Haml]: http://haml.info/

data/VERSION

@@ -0,0 +1 @@
+0.0.1

data/bin/earl-report

@@ -0,0 +1,68 @@
+#!/usr/bin/env ruby
+require 'rubygems'
+$:.unshift(File.expand_path(File.join(File.dirname(__FILE__), "..", 'lib')))
+
+require 'earl_report'
+require 'getoptlong'
+
+def run(input, options)
+end
+
+OPT_ARGS = [
+  ["--bibRef",            GetoptLong::REQUIRED_ARGUMENT,"ReSpec BibRef of specification being reported upon"],
+  ["--format", "-f",      GetoptLong::REQUIRED_ARGUMENT,"Format of output, one of 'ttl', 'json', or 'html'"],
+  ["--json",              GetoptLong::NO_ARGUMENT,      "Input is a JSON-LD formatted result"],
+  ["--name",              GetoptLong::REQUIRED_ARGUMENT,"Name of specification"],
+  ["--output", "-o",      GetoptLong::REQUIRED_ARGUMENT,"Output report to file"],
+  ["--template",          GetoptLong::OPTIONAL_ARGUMENT,"Specify or return default report template"],
+  ["--verbose",           GetoptLong::NO_ARGUMENT,      "Detail on execution"],
+  ["--help", "-?",        GetoptLong::NO_ARGUMENT,      "This message"]
+]
+def usage
+  STDERR.puts %{
+    Generate EARL report for mutliple test results against a test manifest.
+    
+    Usage: #{$0} [options] test-manifest test-result ...
+  }.gsub(/^    /, '')
+  width = OPT_ARGS.map do |o|
+    l = o.first.length
+    l += o[1].length + 2 if o[1].is_a?(String)
+    l
+  end.max
+  OPT_ARGS.each do |o|
+    s = "  %-*s  " % [width, (o[1].is_a?(String) ? "#{o[0,2].join(', ')}" : o[0])]
+    s += o.last
+    STDERR.puts s
+  end
+  exit(1)
+end
+
+opts = GetoptLong.new(*OPT_ARGS.map {|o| o[0..-2]})
+
+options = {
+  :format => :html,
+  :io => STDOUT}
+
+opts.each do |opt, arg|
+  case opt
+  when '--bibRef'       then options[:bibRef] = arg
+  when '--format'       then options[:format] = arg.to_sym
+  when '--name'         then options[:name] = arg
+  when '--output'       then options[:io] = File.open(arg, "w")
+  when '--template'     then options[:template] = arg
+  when '--verbose'      then options[:verbose] = true
+  when '--help'         then usage
+  else
+    options[opt.to_sym] = arg
+  end
+end
+
+# If requesting template, just return it
+if options.has_key?(:template) && options[:template].empty?
+  File.open(File.expand_path("../../lib/earl_report/views/earl_report.html.haml", __FILE__)) do |f|
+    options[:io].write(f.read)
+  end
+else
+  earl = EarlReport.new(*ARGV, options)
+  earl.generate(options.merge(:source_files => ARGV))
+end

data/lib/earl_report.rb

@@ -0,0 +1,424 @@
+# EARL reporting
+require 'linkeddata'
+require 'sparql'
+require 'haml'
+
+##
+# EARL reporting class.
+# Instantiate a new class using one or more input graphs
+class EarlReport
+  attr_reader :graph
+  TEST_SUBJECT_QUERY = %(
+    PREFIX doap: <http://usefulinc.com/ns/doap#>
+    PREFIX foaf: <http://xmlns.com/foaf/0.1/>
+    
+    SELECT DISTINCT ?uri ?name ?developer ?dev_name ?dev_type ?doap_desc ?homepage ?language
+    WHERE {
+      ?uri a doap:Project; doap:name ?name .
+      OPTIONAL { ?uri doap:developer ?developer .}
+      OPTIONAL { ?uri doap:homepage ?homepage . }
+      OPTIONAL { ?uri doap:description ?doap_desc . }
+      OPTIONAL { ?uri doap:programming-language ?language . }
+      OPTIONAL { ?developer foaf:name ?dev_name .}
+      OPTIONAL { ?developer a ?dev_type . }
+    }
+  ).freeze
+
+  DOAP_QUERY = %(
+    PREFIX earl: <http://www.w3.org/ns/earl#>
+    PREFIX doap: <http://usefulinc.com/ns/doap#>
+    
+    SELECT DISTINCT ?subject ?name
+    WHERE {
+      [ a earl:Assertion; earl:subject ?subject ] .
+      OPTIONAL {
+        ?subject a doap:Project; doap:name ?name
+      }
+    }
+  )
+
+  ASSERTION_QUERY = %(
+    PREFIX earl: <http://www.w3.org/ns/earl#>
+    
+    SELECT ?by ?mode ?outcome ?subject ?test
+    WHERE {
+      [ a earl:Assertion;
+        earl:assertedBy ?by;
+        earl:mode ?mode;
+        earl:result [earl:outcome ?outcome];
+        earl:subject ?subject;
+        earl:test ?test ] .
+    }
+  ).freeze
+
+  # Convenience vocabularies
+  class EARL < RDF::Vocabulary("http://www.w3.org/ns/earl#"); end
+  class MF < RDF::Vocabulary("http://www.w3.org/2001/sw/DataAccess/tests/test-manifest#"); end
+
+  ##
+  # Load test assertions and look for referenced software and developer information
+  # @param [Array<String>] *files Assertions
+  # @param [Hash{Symbol => Object}] options
+  # @option options [Boolean] :verbose (true)
+  def initialize(*files)
+    @options = files.last.is_a?(Hash) ? files.pop.dup : {}
+    @graph = RDF::Graph.new
+    @prefixes = {}
+    files.flatten.each do |file|
+      status "read #{file}"
+      file_graph = case file
+      when /\.jsonld/
+        @json_hash = ::JSON.parse(File.read(file))
+        return
+      else RDF::Graph.load(file)
+      end
+      status "  loaded #{file_graph.count} triples"
+      @graph << file_graph
+    end
+
+    # Find or load DOAP descriptions for all subjects
+    SPARQL.execute(DOAP_QUERY, @graph).each do |solution|
+      subject = solution[:subject]
+
+      # Load DOAP definitions
+      unless solution[:name] # not loaded
+        status "read doap description for #{subject}"
+        begin
+          doap_graph = RDF::Graph.load(subject)
+          status "  loaded #{doap_graph.count} triples"
+          @graph << doap_graph.to_a
+        rescue
+          status "  failed"
+        end
+      end
+    end
+
+    # Load developers referenced from Test Subjects
+    SPARQL.execute(TEST_SUBJECT_QUERY, @graph).each do |solution|
+      # Load DOAP definitions
+      if solution[:developer] && !solution[:dev_name] # not loaded
+        status "read description for #{solution[:developer].inspect}"
+        begin
+          foaf_graph = RDF::Graph.load(solution[:developer])
+          status "  loaded #{foaf_graph.count} triples"
+          @graph << foaf_graph.to_a
+        rescue
+          status "  failed"
+        end
+      end
+    end
+  end
+    
+  ##
+  # Dump the collesced output graph
+  #
+  # If no `io` option is provided, the output is returned as a string
+  #
+  # @param [Hash{Symbol => Object}] options
+  # @option options [Symbol] format (:html)
+  # @option options [String] :bibRef
+  #   ReSpec bibliography reference for specification being tested
+  # @option options [Array<String>] :source_files
+  #   Used for referencing the files used to generate this report
+  # @option options[IO] :io
+  #   Optional `IO` to output results
+  # @return [String] serialized graph, if `io` is nil
+  def generate(options = {})
+    options = {
+      format:       :html,
+      bibRef:       "[[TURTLE]]",
+      name:         "Turtle Test Results",
+    }.merge(options)
+
+    io = options[:io]
+
+    status("generate: #{options[:format]}")
+    ##
+    # Retrieve Hashed information in JSON-LD format
+    hash = json_hash(options)
+    case options[:format]
+    when :jsonld, :json
+      json = hash.to_json(JSON::LD::JSON_STATE)
+      io.write(json) if io
+      json
+    when :turtle, :ttl
+      if io
+        earl_turtle(options.merge(:json_hash => hash))
+      else
+        io = StringIO.new
+        earl_turtle(:json_hash => hash, :io => io)
+        io.rewind
+        io.read
+      end
+    when :html
+      template = options[:template] ||
+        File.read(File.expand_path('../earl_report/views/earl_report.html.haml', __FILE__))
+
+      # Generate HTML report
+      html = Haml::Engine.new(template, :format => :xhtml)
+        .render(self, :tests => hash, :source_files => options.fetch(:source_files, []))
+      io.write(html) if io
+      html
+    else
+      if io
+        RDF::Writer.for(options[:format]).new(io) {|w| w << graph}
+      else
+        graph.dump(options[:format])
+      end
+    end
+  end
+
+  private
+  
+  ##
+  # Return hashed EARL report in JSON-LD form
+  # @return [Hash]
+  def json_hash(options)
+    @json_hash ||= begin
+      # Customized JSON-LD output
+      {
+        "@context" => {
+          dc:           "http://purl.org/dc/terms/",
+          doap:         "http://usefulinc.com/ns/doap#",
+          earl:         "http://www.w3.org/ns/earl#",
+          mf:           "http://www.w3.org/2001/sw/DataAccess/tests/test-manifest#",
+          foaf:         "http://xmlns.com/foaf/0.1/",
+          rdfs:         "http://www.w3.org/2000/01/rdf-schema#",
+          assertedBy:   {"@id" => "earl:assertedBy", "@type" => "@id"},
+          bibRef:       {"@id" => "dc: bibliographicCitation"},
+          description:  {"@id" => "dc:description"},
+          developer:    {"@id" => "doap:developer", "@type" => "@id"},
+          homepage:     {"@id" => "doap:homepage", "@type" => "@id"},
+          doap_desc:    {"@id" => "doap:description"},
+          language:     {"@id" => "doap:programming-language"},
+          testAction:   {"@id" => "mf:action", "@type" => "@id"},
+          testResult:   {"@id" => "mf:result", "@type" => "@id"},
+          label:        {"@id" => "rdfs:label"},
+          mode:         {"@id" => "earl:mode", "@type" => "@id"},
+          name:         {"@id" => "doap:name"},
+          outcome:      {"@id" => "earl:outcome", "@type" => "@id"},
+          result:       {"@id" => "earl:result"},
+          subject:      {"@id" => "earl:subject", "@type" => "@id"},
+          test:         {"@id" => "earl:test", "@type" => "@id"},
+          title:        {"@id" => "dc:title"}
+        },
+        "@id"          => "",
+        "@type"        => %w(earl:Software doap:Project),
+        'name'         => options[:name],
+        'bibRef'       => options[:bibRef],
+        'testSubjects' => json_test_subject_info,
+        'tests'        => json_result_info
+      }
+    end
+  end
+
+  ##
+  # Return array of test subject information
+  # @return [Array]
+  def json_test_subject_info
+    # Get the set of subjects
+    ts_info = {}
+    SPARQL.execute(TEST_SUBJECT_QUERY, @graph).each do |solution|
+      status "solution #{solution.to_hash.inspect}"
+      info = ts_info[solution[:uri].to_s] ||= {}
+      %w(name doap_desc homepage language).each do |prop|
+        info[prop] = solution[prop.to_sym].to_s if solution[prop.to_sym]
+      end
+      if solution[:dev_name]
+        dev_type = solution[:dev_type].to_s =~ /Organization/ ? "foaf:Organization" : "foaf:Person"
+        info['developer'] = Hash.ordered
+        info['developer']['@id'] = solution[:developer].to_s if solution[:developer].uri?
+        info['developer']['@type'] = dev_type
+        info['developer']['foaf:name'] = solution[:dev_name].to_s if solution[:dev_name]
+      end
+    end
+
+    # Map ids and values to array entries
+    ts_info.keys.map do |id|
+      info = ts_info[id]
+      subject = Hash.ordered
+      subject["@id"] = id
+      subject["@type"] = %w(earl:TestSubject doap:Project)
+      %w(name developer doap_desc homepage language).each do |prop|
+        subject[prop] = info[prop] if info[prop]
+      end
+      subject
+    end
+  end
+  
+  ##
+  # Return result information for each test
+  #
+  # @return [Array]
+  def json_result_info
+    test_cases = {}
+
+    @graph.query(:predicate => MF['entries']) do |stmt|
+      # Iterate through the test manifest and write out a TestCase
+      # for each test
+      RDF::List.new(stmt.object, @graph).map do |tc|
+        tc_hash = {}
+        tc_hash['@id'] = tc.to_s
+        tc_hash['@type'] = %w(earl:TestCriterion earl:TestCase)
+
+        # Extract important properties
+        @graph.query(:subject => tc).each do |tc_stmt|
+          case tc_stmt.predicate.to_s
+          when MF['name'].to_s
+            tc_hash['title'] = tc_stmt.object.to_s
+          when RDF::RDFS.comment.to_s
+            tc_hash['description'] = tc_stmt.object.to_s
+          when MF.action.to_s
+            tc_hash['testAction'] = tc_stmt.object.to_s
+          when MF.result.to_s
+            tc_hash['testResult'] = tc_stmt.object.to_s
+          else
+            #STDERR.puts "TC soln: #{tc_stmt.inspect}"
+          end
+        end
+
+        test_cases[tc.to_s] = tc_hash
+      end
+    end
+
+    raise "No test cases found" if test_cases.empty?
+
+    status "Test cases:\n  #{test_cases.keys.join("\n  ")}"
+    # Iterate through assertions and add to appropriate test case
+    SPARQL.execute(ASSERTION_QUERY, @graph).each do |solution|
+      tc = test_cases[solution[:test].to_s]
+      STDERR.puts "No test case found for #{solution[:test]}" unless tc
+      tc ||= {}
+      subject = solution[:subject].to_s
+      ta_hash = {}
+      ta_hash['@type'] = 'earl:Assertion'
+      ta_hash['assertedBy'] = solution[:by].to_s
+      ta_hash['test'] = solution[:test].to_s
+      ta_hash['mode'] = "earl:#{solution[:mode].to_s.split('#').last || 'automatic'}"
+      ta_hash['subject'] = subject
+      ta_hash['result'] = {
+        '@type' => 'earl:TestResult',
+        "outcome" => (solution[:outcome] == EARL.passed ? 'earl:passed' : 'earl:failed')
+      }
+      tc[subject] = ta_hash
+    end
+
+    test_cases.values
+  end
+  
+  ##
+  # Output consoloated EARL report as Turtle
+  # @param [IO, StringIO] io
+  # @return [String]
+  def earl_turtle(options)
+    io = options[:io]
+    json_hash = options[:json_hash]
+    # Write preamble
+    {
+      :dc       => RDF::DC,
+      :doap     => RDF::DOAP,
+      :earl     => EARL,
+      :foaf     => RDF::FOAF,
+      :mf       => MF,
+      :owl      => RDF::OWL,
+      :rdf      => RDF,
+      :rdfs     => RDF::RDFS,
+      :xhv      => RDF::XHV,
+      :xsd      => RDF::XSD
+    }.each do |prefix, vocab|
+      io.puts("@prefix #{prefix}: <#{vocab.to_uri}> .")
+    end
+    io.puts
+
+    # Write earl:Software for the report
+    io.puts %(<#{json_hash['@id']}> a earl:Software, doap:Project;)
+    io.puts %(  doap:homepage <#{json_hash['homepage']}>;)
+    io.puts %(  doap:name "#{json_hash['name']}".)
+
+    # Test Cases
+    # also collect each assertion definition
+    test_cases = {}
+    assertions = []
+
+    # Tests
+    json_hash['tests'].each do |test_case|
+      tc_desc =  test_cases[test_case['test']] ||= test_case.dup
+      test_case.keys.select {|k| k =~ /^http:/}.each do |ts_uri|
+        tc_desc[ts_uri] = test_case[ts_uri]['@id']
+        assertions << test_case[ts_uri]
+      end
+    end
+    
+    # Write out each earl:TestSubject
+    io.puts %(#\n# Subject Definitions\n#)
+    json_hash['testSubjects'].each do |ts_desc|
+      io.write(test_subject_turtle(ts_desc))
+    end
+    
+    # Write out each earl:TestCase
+    io.puts %(#\n# Test Case Definitions\n#)
+    test_cases.keys.sort.each do |num|
+      io.write(tc_turtle(test_cases[num]))
+    end
+    
+    # Write out each earl:Assertion
+    io.puts %(#\n# Assertions\n#)
+    assertions.sort_by {|a| a['@id']}.each do |as_desc|
+      io.write(as_turtle(as_desc))
+    end
+  end
+  
+  ##
+  # Write out Test Subject definition for each earl:TestSubject
+  # @param [Hash] desc
+  # @return [String]
+  def test_subject_turtle(desc)
+    developer = desc['developer']
+    res = %(<#{desc['@id']}> a #{desc['@type'].join(', ')};\n)
+    res += %(  doap:name "#{desc['name']}";\n)
+    res += %(  doap:description """#{desc['doap_desc']}""";\n)     if desc['doap_desc']
+    res += %(  doap:programming-language "#{desc['language']}";\n) if desc['language']
+    if developer && developer['@id']
+      res += %(  doap:developer <#{developer['@id']}> .\n\n)
+      res += %(<#{developer['@id']}> a #{[developer['@type']].flatten.join(', ')};\n)
+      res += %(  foaf:name "#{developer['foaf:name']}" .\n)
+    elsif developer
+      res += %(  doap:developer [ a #{developer['@type'] || "foaf:Person"}; foaf:name "#{developer['foaf:name']}"] .\n)
+    else
+      res += %(  .\n)
+    end
+    res + "\n"
+  end
+  
+  ##
+  # Write out each Test Case definition
+  # @prarm[Hash] desc
+  # @return [String]
+  def tc_turtle(desc)
+    res = %(<#{desc['@id']}> a #{[desc['@type']].flatten.join(', ')};\n)
+    res += %(  dc:title "#{desc['title']}";\n)
+    res += %(  dc:description """#{desc['description']}""";\n)
+    res += %(  mf:action <#{desc['testAction']}>;\n)
+    res += %(  mf:result <#{desc['testResult']}>;\n)
+    res + "\n"
+  end
+
+  ##
+  # Write out each Assertion definition
+  # @prarm[Hash] desc
+  # @return [String]
+  def as_turtle(desc)
+    res =  %([ a earl:Assertion;\n)
+    res += %(  earl:assertedBy <#{desc['assertedBy']}>;\n)
+    res += %(  earl:test <#{desc['test']}>;\n)
+    res += %(  earl:subject <#{desc['subject']}>;\n)
+    res += %(  earl:mode #{desc['mode']};\n)
+    res += %(  earl:result [ a earl:TestResult; earl:outcome #{desc['result']['outcome']}] ] .\n)
+    res += %(\n)
+    res
+  end
+  
+  def status(message)
+    puts message if @options[:verbose]
+  end
+end