polytrix 0.0.1 → 0.1.0.pre

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 8655afabebaa23eba72bb5028b0efb5d22ecc7d7
-  data.tar.gz: 336634653e22e6b814edbb9fc00598fc32c431f5
+  metadata.gz: 1c91cf7e97c088e23aa0134c2e20acfab0640195
+  data.tar.gz: 9bd8c63ae3cfbbc3bc75fc54c3b094ad848dd8c7
 SHA512:
-  metadata.gz: 16ee58b313175b50b7b446a9bbcfc3699b80261ab3502d26c2b785ce51e121b7d50ced60f72edd61396ec75c42167da7ca6ea4049799af9ce9eb531e8c47c39b
-  data.tar.gz: 4e0cdb6e135581bbdaee0eb5fb06d0771335448f4b944eb345cec16772e4a2447a28a3e3bc0d93dd256fd9f123e22e137ed50bd06784b7265e77b9a9ffe198ce
+  metadata.gz: 5361e579064d58b4168e5929f995d1a42506c2faa49b246ddccd83eec3f2adef3d28b2761ab0bfd9b173e6ef690702b7eba56caef2e1a75746045c48c29e97b3
+  data.tar.gz: 61bcc9b099b67a95695cc59ac0d23baf3791cb8eec7c1bed382964c25e03ebc18fac01e6bce5bed6be1666e45a58ada4ef611323121dcf1bdc5c43c3c4f6823c

data/.gitignore

@@ -1,13 +1,13 @@
 Gemfile.lock
 pacto.log
 **/.bundle/*.log
-reports/
+/reports/
 .rbenv-vars
 .packer-creds.json
 **/tmp/
 node_modules/
 .tokens.json
-docs/
 /polytrix/
 pkg/
 coverage/
+.yardoc/

data/.rspec

@@ -1,11 +1,6 @@
--I features
---format MatrixFormatter::Formatters::JSONFormatter
---out reports/matrix<%= ENV['TEST_ENV_NUMBER'].to_i %>.json
---format MatrixFormatter::Formatters::HTMLFormatter
---out reports/matrix.html
+-r spec_helper
 --format html
 --out reports/rspec.html
 --format documentation
 --color
---require features_helper.rb
 

data/.rubocop-todo.yml

@@ -0,0 +1,19 @@
+# This configuration was generated by `rubocop --auto-gen-config`
+# on 2014-06-12 15:05:43 -0400 using RuboCop version 0.18.1.
+# The point is for the user to remove these configuration records
+# one by one as the offences are removed from the code base.
+# Note that changes in the inspected code, or installation of new
+# versions of RuboCop, may require this file to be generated again.
+
+# Offence count: 41
+Documentation:
+  Enabled: false
+
+# Offence count: 57
+LineLength:
+  Max: 165
+
+# Offence count: 6
+# Configuration parameters: CountComments.
+MethodLength:
+  Max: 21

data/.rubocop.yml

@@ -0,0 +1,10 @@
+inherit_from: .rubocop-todo.yml
+
+AllCops:
+  Include:
+    - Guardfile
+    - Rakefile
+    - pacto.gemspec
+  Exclude:
+    - bin/**
+    - tmp/**

data/.travis.yml

@@ -0,0 +1,11 @@
+before_install:
+ - sudo add-apt-repository ppa:cwchien/gradle
+ - sudo apt-get update -qq
+ - sudo apt-get install -qq gradle
+rvm:
+  - 1.9.3
+  - 2.0.0
+  - 2.1.0
+allow_failures:
+  rvm:
+    - 1.9.3 # YAML formatting differs

data/Gemfile

@@ -2,22 +2,6 @@
 source "https://rubygems.org"
 gemspec
 
-gem "pacto", :git => 'https://github.com/thoughtworks/pacto', :branch => 'uri_templates'
-# uriTemplate support merged  into webmock but not yet released - webmock#334
-gem "webmock", :git => 'https://github.com/bblimke/webmock', :branch => 'master'
-# optional param fix merged but not yet released - addressable#140
-gem "addressable", :git => 'https://github.com/sporkmonger/addressable.git', :branch => 'master'
-gem "fog", '~> 1.19'
-gem "vcr"
-gem "goliath"
-gem "excon"
-gem "em-http-request"
-gem "rspec_junit_formatter"
 gem "pry"
-gem "matrix_formatter", :git => 'https://github.com/maxlinc/matrix_formatter', :branch => 'html5'
-gem "highline"
-gem "parallel_tests"
 gem "rake-notes"
 gem "simplecov"
-gem "mixlib-shellout"
-

data/README.md

@@ -1,48 +1,139 @@
-# DRG Tests #
+# Polytrix - the Polyglot Testing Matrix
 
-This repo contains smoke tests for Rackspace SDKs.  In order to run it you'll either need:
-  - Lots of dependencies installed on your laptop
-  - VirtualBox & Vagrant, so you can install the necessary dependencies inside a local VM
-  - Packer and Vagrant-Rackspace, so you can build images and use images on the Rackspace cloud to run tests
+Polytrix is a polyglot test runner and documentation generator. It aims to let you run sample code written in any language. It's especially useful if you want to run similar code samples in multiple languages. Simply put, if a project like [Slate](https://github.com/tripit/slate) looks like an interesting documentation option, then you might be interested in Polytrix for testing.
 
-The last option can also be combined with Jenkins JClouds plugin in order to setup CI builds using the images produced by Packer.
+# Features
 
-## Dependencies
+Polytrix samples defined in a "test manifest" written in YAML. The test manifest is meant to be portable so you can use you can create a "lightweight sample runner" in your preferred build/test tool of choice for your language, and then integrate the samples with Polytrix later to get the extra features. Polytrix can:
+- Run sample code in any language and several platforms
+- Perform compatibility testing checking multiple implementations (in different langauges) against the same set of assertions
+- Generate documentation from sample code and test results
+- Generate compatibility or feature matrix reports
 
-The tests will use package managers for each SDK where appropriate (e.g. bundler for fog, npm for pkgcloud, etc.)  You need to be setup at least to the point where those package managers can run.  You'll also need dnsmasq in order to intercept some of the HTTP transactions for testing purposes.  It is possible to set this all up on your own machine - but the easiest way is to use Vagrant, Packer, or anything else that will let you use the Chef scripts in the project (targeted for Ubuntu).
+Polytrix provides a few built-in assertions, but also has a plugin system that you can use to do more advanced validation, like using [Pacto](https://github.com/thoughtworks/pacto) to intercept and validate the usage of RESTful services.
 
-## Creating a DRG Image
+# Usage preview
 
-### Packer
+Polytrix is currently run via rspec. You can create a script that looks like this and run it with rspec:
 
-The easiest way to create an image is with [Packer](http://www.packer.io).
+```ruby
+require 'polytrix/rspec'
 
-### Rackspace credentials
+Polytrix.implementors = Dir['sdks/*'].map{ |sdk|
+  name = File.basename(sdk)
+  Polytrix::Implementor.new :name => name
+}
 
-You need to create a `.rbenv-vars` file with your Rackspace credentials.  These will be loaded as environment variables when the tests run.  The file should contain:
+Polytrix.load_manifest 'polytrix_tests.yml'
+Polytrix.bootstrap
+Polytrix.run_tests
+```
+
+Polytrix will use the information in the Manifest and Implementors (see the sections below) to build an rspec test suite. It will setup tags for each Implementor, and names corresponding with the tests in the manifest.
+
+So in our Polytrix examples project you can use commands like:
+
+```sh
+$ # Only run tests for the Java implementor
+$ bundle exec rspec -t java
+$ # Run the "hello world" tests in each language
+$ bundle exec rspec -e "hello world"
+```
+
+## Usage Breakdown
+
+### Defining Implementors (SDKs)
+
+Polytrix can run the tests against multiple implementors. This usually means an SDK, but we used the generic term implementor because Polytrix works equally well for testing code katas, coursework, or other items. Perhaps even things like multi-platform plugins for [Calatrava](https://github.com/calatrava/calatrava/wiki/Plugins) or [PhoneGap](http://docs.phonegap.com/en/3.4.0/guide_hybrid_plugins_index.md.html#Plugin%20Development%20Guide).
+
+This snippet defines the implementors:
+```ruby
+Polytrix.implementors = Dir['sdks/*'].map{ |sdk|
+  name = File.basename(sdk)
+  Polytrix::Implementor.new :name => name
+}
+```
+
+See the full Implementor documentation for details on other attributes you can set, like `:language`. Polytrix will try to infer any information you don't pass.
+
+#### Bootstrapping, compiling, wrapper scripts
 
+Polytrix currently uses the [script/bootstrap](http://wynnnetherland.com/linked/2013012801/bootstrapping-consistency) pattern to allow each implementor to hook into dependency management, build tools, or other systems. Polytrix will look for three scripts (on Windows it will look for a *.ps1 version written in PowerShell):
+
+| File             | Purpose                                                        |
+| ---------------- | -------------------------------------------------------------- |
+| script/bootstrap | Prepare the SDK, usually by running depenency management tool. |
+| script/wrapper   | Wrapper script instead of executing code samples as scripts    |
+
+The bootstrap script is called by `Polytrix.bootstrap`. The wrapper script, if it exists, wraps the executino of the code sample. If there is no wrapper script, Polytrix will try to execute the sample code as a script. That works for many non-compiled scripting languages, like Ruby or Python, but won't work for something like Java.
+
+If there is a wrapper script, Polytrix will call it with teh sample source file as the first argument, e.g.:
+```sh
+$ cd my_java_sdk
+$ ./script/wrapper src/samples/HelloWorld.java
 ```
-RAX_USERNAME=<your_rackspace_username>
-RAX_API_KEY=<your_rackspace_api_key>
+
+### Defining tests - the test manifest
+
+Tests are defined in a YAML "test manifest" which defines what sample code should be executed, and what input it should receive. Standardizing the input is important for compliance testing, becaues it is difficult to maintain tests where one example expects "FOO=bar" and another expects "--foo bar".
+
+A simple test manifest looks like this:
+```yaml
+---
+  global_env:                          # global_env defines input available for all scenarios
+    LOCALE: <%= ENV['LANG'] %>         # templating is allowed
+    FAVORITE_NUMBER: 5
+  suites:                              # suites defines the test suites that can be executed
+    Katas:                             # "Katas" is the name of the first suite
+      env:                             # These "env" values are only available within the "Katas" suite
+        NAME: 'Max'
+      samples:                         # samples defines the individual tests in a suite
+        - hello world
+        - quine
+    Tutorials:                         # "Tutorials" is the name of the second suite
+          env:
+          samples:
+            - deploying
+            - documenting
 ```
 
-### Getting a DRG box
+### Test setup
+
+`Polytrix.run_tests` runs the tests. Actually, right now it really just defines them in rspec, you still need to run the whole script via the RSpec command for the tests to run.
+
+### Finding samples
+
+Polytrix finds samples based on a loose naming convention. This makes it easier to use file names that are idiomatic for each implementor, while still allowing Polytrix to find the right file.
+
+Polytrix basically does a case-insensitive search for a file whose name matches the scenario name, ignoring subfolders, spaces, prefixes, puctuation and file extension.  So these files all match a scenario named "hello world":
+- hello_world.rb
+- src/com/world/HelloWorld.java
+- samples/01_hello_world.go
+
+### Reports and documentation
+
+You can use any RSpec formatter with Polytrix, since Polytrix is based on RSpec.
+
+There are also some Polytrix specific documentation and report generators. You can generate:
+- Scenario-level reports: Documentation or reports for a single scenario
+- Global reports: Documentation or reports summarizing all tested scenarios
+
+The documentatio/reports are generated via a template processing system. Polytrix searches the template directory ('doc-src/' by default) for scenario-level samples using the same logic as in the "Finding samples" section above. It looks for a template matching "index" (e.g. index.md, index.rst, index.html) for the global report.
 
-There is a Vagrantfile in the project.  It uses a "DRG" box.  There isn't currently a published DRG box, so you'll need to produce your own.  You can:
+The templates are processed as ERB. In addition to being able to access the top-level Polytrix API, the following variables are bound:
+| Variable   | Description                              |
+| ---------- | ---------------------------------------- |
+| scenario   | The name of the scenario being processed |
+| challenges | One or more Challenge objects containing the scenario configuration and results |
 
-* Comment out `config.vm.box = "drg"` and uncomment the other lines with an alternate box.
-* Run `vagrant up`, `vagrant provision`, and then `vagrant package --output drg.box`.  These steps will take a while.
-* Run `vagrant box add drg drg.box`
-* You now have a drg box, and can restore the Vagrant file to it's original state.
+### Common compliance tests
 
-### Running tests
+Refactoring... documentation coming soon.
 
+### Plugins
 
+Refactoring... documentation coming soon.
 
-### (Tenative) Roadmap
-* Use standard, shared images (travis-images? vagrantcloud?)
-* Integrate with travis-build for .travis.yml support
-* Separate Polytrix framework from reference tests
+# Influences
 
-### BHAG
-* Run tests as an interactive course via a browser
+Several projects have influenced ideas in Polytrix. If you find Polytrix interesting or want to contribute, you may want to look at those projects. See influence.md.

data/Rakefile

@@ -1,30 +1,19 @@
 require 'polytrix'
 require 'bundler/gem_tasks'
-require 'rspec/core/rake_task'
-require 'pacto/rake_task'
 require 'rake/notes/rake_task'
-require 'highline/import'
-require 'json'
-
-NOT_SETUP = "You need to set RAX_USERNAME and RAX_API_KEY env vars in order to run tests"
+require 'rspec/core/rake_task'
+require 'cucumber'
+require 'cucumber/rake/task'
+require 'rubocop/rake_task'
 
-RSpec::Core::RakeTask.new('features') do |t|
-  t.rspec_opts = "-I features"
-  t.pattern = 'features/**/*_spec.rb'
-end
+task :default => [:spec, :features, :self, :rubocop]
 
 RSpec::Core::RakeTask.new('spec') do |t|
   t.rspec_opts = "-f documentation"
 end
 
-task :default => :spec
-
-desc 'Run all the SDK tests'
-task :features => [:check_setup]
-
-desc 'Check pre-requisites'
-task :check_setup do
-  fail NOT_SETUP unless ENV['RAX_USERNAME'] && ENV['RAX_API_KEY']
+Cucumber::Rake::Task.new(:features) do |t|
+  t.cucumber_opts = "features --require features/support --require features/step_definitions"
 end
 
 desc 'Remove reports and other generated artifacts'
@@ -33,111 +22,17 @@ task :clean do
   FileUtils.rm_rf 'reports'
 end
 
-desc 'Fetch dependencies for each SDK'
-task :bootstrap do
-  Bundler.with_clean_env do
-    Dir['sdks/*'].each do |sdk_dir|
-      Dir.chdir sdk_dir do
-        if is_windows?
-          system "PowerShell -NoProfile -ExecutionPolicy Bypass .\\scripts\\bootstrap"
-        else
-          system "scripts/bootstrap"
-        end
-      end
-    end
-  end
-end
-
-desc 'Configure the test framework'
-task :setup do
-  username = ask "Enter your Rackspace Username: "
-  api_key  = ask("Enter your Rackspace API KEY: "){|q| q.echo = "*"}
-  password = ask("Enter your Rackspace Password (for Packer): "){|q| q.echo = "*"}
-
-  puts "Creating .rbenv-vars"
-  File.open(".rbenv-vars", 'w') do |f|
-    f.puts "RAX_USERNAME=#{username}"
-    f.puts "RAX_API_KEY=#{api_key}"
-    f.puts
-  end
-
-  puts "Creating .packer-creds.json"
-  packer_creds = {
-    "RAX_USERNAME" => username,
-    "RAX_PASSWORD" => password
-  }
-  File.open(".packer-creds.json", "w") do |f|
-    f.puts JSON.pretty_generate packer_creds
-  end
-end
-
-namespace :documentation do
-  desc 'Generate annoted source code'
-  task :annotated do
-    system "groc"
-  end
-
-  task :copy_src do
-    sdks = Dir['sdks/*'].each do |sdk|
-      sdk = File.basename sdk
-      FileUtils.mkdir_p "docs/src/#{sdk}"
-      FileUtils.cp_r "sdks/#{sdk}/challenges/", "docs/src/#{sdk}"
-    end
-  end
-
-  desc 'Generate the Feature Matrix dashboard'
-  task :dashboard => [:copy_src, :annotated] do
-    $: << 'features'
-    require  'features_helper'
-    require "matrix_formatter/assets/generator"
-    require "matrix_formatter/formatters/html5_report_writer"
-    require 'fileutils'
-
-    asset_generator_options = {
-      :extra_resource_paths => ['lib/formatter/resources']
-    }
-    asset_generator = MatrixFormatter::Assets::Generator.new asset_generator_options
-    asset_generator.generate
-
-    output = File.open("docs/dashboard.html", 'w')
-    options = {:view => 'angular.html.slim', :layout => 'default_layout.html.slim'}
-    formatter = MatrixFormatter::Formatters::HTML5ReportWriter.new output, options
-    matrix = formatter.parse_results Dir['reports/matrix*.json']
-    # puts MultiJson.encode formatter.matrix
-    formatter.write_report
-    output.close
-
-    fail "Combined results contain failures - check the reports" if formatter.has_failures?
-  end
-end
-
-desc "Run the tests in parallel, split by SDK.  Doesn't work on Windows, but you can use rspec_parallel to split by file instead."
-task :parallel_spec do
-  tags = Dir['sdks/*'].map{|sdk| File.basename sdk}
-  puts "Detected SDKs: #{tags}"
-  threads = []
-  Thread.main[:results] = []
-  tags.each_with_index do | tag, index |
-    threads << Thread.new do
-      puts "Starting #{tag} on process #{index}"
-      Thread.main[:results] << {
-        :tag => tag,
-        :success  => sh("TEST_ENV_NUMBER=#{index} bundle exec rspec --options .rspec_parallel -t #{tag} features")
-      }
-    end
-  end
-  threads.each do |thread|
-    thread.join
-  end
-end
-
-namespace :image do
-  desc 'Build a Rackspace image (using Packer)'
-  task :rackspace do
-    system "cd packer; packer build -var-file=../.packer-creds.json -only openstack packer.json"
-  end
+desc 'Self-test and self-document'
+task :self do
+  sh 'bundle exec polytrix bootstrap'
+  sh 'bundle exec polytrix test'
+  sh 'bundle exec polytrix code2doc samples/sdks/java/challenges/*.java --target-dir=docs/samples/code2doc/java'
+  sh 'bundle exec polytrix code2doc samples/sdks/python/challenges/*.py --target-dir=docs/samples/code2doc/python'
+  sh 'bundle exec polytrix code2doc samples/sdks/ruby/challenges/*.rb --target-dir=docs/samples/code2doc/ruby'
+  # sh 'bundle exec polytrix code2doc samples/*.rb'
 end
 
-def is_windows?
-  RbConfig::CONFIG['host_os'] =~ /mswin(\d+)|mingw/i
+Rubocop::RakeTask.new(:rubocop) do |task|
+  # abort rake on failure
+  task.fail_on_error = true
 end

data/bin/polytrix

@@ -0,0 +1,5 @@
+#!/usr/bin/env ruby
+require 'polytrix/cli'
+
+$LOAD_PATH << 'spec'
+Polytrix::CLI::Main.start

data/doc-src/_markdown.md

@@ -0,0 +1,5 @@
+<% challenges.compact.each do |challenge| %>
+``` <%= challenge.implementor.language %>
+<%= File.read(challenge.source_file) %>
+```
+<% end %>

data/doc-src/default_bootstrap.md

@@ -0,0 +1,13 @@
+This code will cause Polytrix to bootstrap:
+
+<% challenges.compact.each do |challenge| %>
+``` <%= challenge.implementor.language %>
+<%= File.read(challenge.source_file) %>
+```
+
+And the results will look like:
+```
+<%= challenge.result.execution_result.stdout %>
+```
+
+<% end %>

data/docs/influences.md

@@ -0,0 +1,28 @@
+# Influences
+
+This project was influenced by many other projects. The following projects are worth checking out, either as alternatives you might want to use instead of Polytrix, or to understand the domain better before contributing.
+
+# Polyglot Testing
+
+## Travis-CI
+
+I do not know of any project that is as far along as Travis in terms of making it easy to test projects in most common languages. Currently this project is inspired by Travis, but in the future I hope to re-use Travis components and contribute back to the Travis community.
+
+The most important Travis components for Polyglot testing are:
+- [travis-cookbooks](https://github.com/travis-ci/travis-cookbooks) and [travis-images](https://github.com/travis-ci/travis-images) - create environments that are setup for testing any popular language
+- [https://github.com/travis-ci/travis-build](travis-build) - creates scripts that handle setting up the environment for a given platform (Linux, OSX, Windows) and follow language conventions for dependency management and compilation.
+
+## JSON-LD Test Suite and EARL Report
+
+I've seen several compliance test suites, but the [JSON-LD suite](http://json-ld.org/test-suite/) is one of my favorites, including the [earl-report](https://github.com/gkellogg/earl-report) gem it uses to create reports.
+
+The main difference between Polytrix and JSON-LD's test suite (and similar compliance tests for other projects) is that JSON-LD is data-driven and Polytrix is driven by sample code. The JSON-LD approach is good for standards about a data format like JSON-LD, while Polytrix is more for testing the functional completeness of similar SDKs.
+
+## Codecademy
+
+Codecademy's [Course Creation](http://www.codecademy.com/docs/creation) documentation explains how they test code samples written in different languages. You can think of Polytrix as serving a similar function to Codecademy's [Submission Tests](http://www.codecademy.com/docs/submission_tests), with two important differences:
+
+- Polytrix aims for a single submission test for all languages: so could share a "Hello, world!" submission correctness test for Ruby, Java, and PHP.
+- Polytrix is being used for automated test suites where the code submissions are pre-written and repeatedly tested, though creating a course with Polytrix may be possible.
+
+This of course has some limitations. Since Polytrix is polyglot, we generally test only the effect of running the sample and not the syntax. Polytrix is not suitable for testing each variation when [there's more than one way to do it](http://en.wikipedia.org/wiki/There's_more_than_one_way_to_do_it), but you can use it to test that at least one way is possible.

data/docs/samples/code2doc/java/HelloWorld.md

@@ -0,0 +1,13 @@
+This is an example of the standard [Hello World](http://en.wikipedia.org/wiki/Hello_world_program)
+program written in [Java][java].
+```java
+public class HelloWorld {
+```
+A Java CLI application is defined by creating a main method with this signature:
+```java
+  public static void main(String[] args) {
+    System.out.println("Hello, world!");
+  }
+}
+```
+[java]: http://en.wikipedia.org/wiki/Java_(programming_language)

data/docs/samples/code2doc/java/Quine.md

@@ -0,0 +1,33 @@
+```java
+public class Quine
+{
+  public static void main(String[] args)
+  {
+    char q = 34;      // Quotation mark character
+    String[] l = {    // Array of source code
+    "public class Quine",
+    "{",
+    "  public static void main(String[] args)",
+    "  {",
+    "    char q = 34;      // Quotation mark character",
+    "    String[] l = {    // Array of source code",
+    "    ",
+    "    };",
+    "    for(int i = 0; i < 6; i++ )          // Print opening code",
+    "        System.out.println(l[i]);",
+    "    for(int i = 0; i < l.length; i++)    // Print string array",
+    "        System.out.println( l[6] + q + l[i] + q + ',' );",
+    "    for(int i = 7; i < l.length; i++)    // Print this code",
+    "        System.out.println( l[i] );",
+    "  }",
+    "}",
+    };
+    for(int i = 0; i < 6; i++ )          // Print opening code
+        System.out.println(l[i]);
+    for(int i = 0; i < l.length; i++)    // Print string array
+        System.out.println( l[6] + q + l[i] + q + ',' );
+    for(int i = 7; i < l.length; i++)    // Print this code
+        System.out.println( l[i] );
+  }
+}
+```

data/docs/samples/code2doc/python/hello_world.md

@@ -0,0 +1,3 @@
+```py
+print 'Hello, world!'
+```

data/docs/samples/code2doc/python/quine.md

@@ -0,0 +1,4 @@
+```py
+s = 's = %r\nprint(s%%s)'
+print(s%s)
+```

data/docs/samples/code2doc/ruby/hello_world.md

@@ -0,0 +1,7 @@
+```rb
+name = 'world'
+```
+This will print Hello, world!
+```rb
+puts "Hello, #{name}!"
+```

data/features/execution.feature

@@ -0,0 +1,67 @@
+Feature: Running SDKs
+
+  Polytrix uses the [script/bootstrap](http://wynnnetherland.com/linked/2013012801/bootstrapping-consistency) pattern to prepare SDKs for testing. You can hook into any package manager, compiler, build tool, or any other toolchain to prepare to build and run samples.
+
+  Polytrix also sets up tags for the SDKs so you can use the normal rspec `-t` option to select which SDK to run.
+
+  Scenario: Bootstrap an SDK
+    Given the java SDK
+    And the empty polytrix config
+    And the standard rspec setup
+    When I run `bundle exec rspec`
+    Then the output should contain ":compileJava"
+    And the output should contain "BUILD SUCCESSFUL"
+
+Scenario: Running all SDKs
+    Given the ruby SDK
+    And the java SDK
+    And the python SDK
+    And the hello_world polytrix config
+    And the standard rspec setup
+    When I run `bundle exec rspec`
+    And the output should contain "3 examples, 0 failures"
+
+  Scenario: Running a single SDK
+    Given the ruby SDK
+    And the java SDK
+    And the python SDK
+    And the hello_world polytrix config
+    And the standard rspec setup
+    When I run `bundle exec rspec -t ruby`
+    Then the output should contain "Hello, world!"
+    And the output should contain "1 example, 0 failures"
+
+  Scenario: Custom assertions
+    Given the ruby SDK
+    And the java SDK
+    And the python SDK
+    And the hello_world polytrix config
+    And the standard rspec setup
+    And a file named "spec/custom_spec.rb" with:
+    """
+    require 'polytrix/rspec'
+
+    Polytrix.configure do |polytrix|
+      Dir['sdks/*'].each do |sdk|
+        polytrix.implementor sdk
+      end
+      polytrix.test_manifest = 'polytrix_tests.yml'
+    end
+
+    # This is more of an integration test, but makes sure the rspec API is working.
+    # Expect results to all be pending, because there's no implementors in this proj.
+
+    Polytrix.validate(suite: 'Katas', sample: 'hello world') do |challenge|
+      # You can make assertions about the process using the Mixlib::ShellOut API
+      expect(challenge[:result].execution_result.stdout).to include 'Hello, world!'
+      expect(challenge[:result].execution_result.stderr).to be_empty
+      expect(challenge[:result].execution_result.exitstatus).to eq(1) # normally this would be 0
+    end
+
+    Polytrix.validate(suite: 'Katas', sample: 'quine') do |challenge|
+      expect(challenge[:result].execution_result.stdout).to eq File.read(challenge[:result].source)
+    end
+    Polytrix.load_tests
+    """
+    When I run `bundle exec rspec spec/custom_spec.rb`
+    And the output should match /expected: 1\s+got: 0/

data/features/fixtures/configs/empty.yml

@@ -0,0 +1,12 @@
+---
+  global_env:
+    LOCALE: <%= ENV['LANG'] %>
+    FAVORITE_NUMBER: 5
+  suites:
+    Katas:
+      env:
+        NAME: 'Max'
+      samples:
+    Tutorials:
+      env:
+      samples: