polytrix 0.0.1 → 0.1.0.pre

data/lib/polytrix/documentation/helpers/code_helper.rb

@@ -0,0 +1,85 @@
+require 'polytrix/documentation/code_segmenter'
+
+module Polytrix
+  module Documentation
+    module Helpers
+      module CodeHelper
+        class ReStructuredTextHelper
+          def self.code_block(source, language)
+            buffer = StringIO.new
+            buffer.puts ".. code-block:: #{language}"
+            indented_source = source.lines.map do|line|
+              "  #{line}"
+            end.join("\n")
+            buffer.puts indented_source
+            buffer.string
+          end
+        end
+        class MarkdownHelper
+          def self.code_block(source, language)
+            buffer = StringIO.new
+            buffer.puts "```#{language}"
+            buffer.puts source
+            buffer.puts '```'
+            buffer.string
+          end
+        end
+        def initialize(*args)
+          @segmenter = Polytrix::Documentation::CodeSegmenter.new
+          super
+        end
+
+        def source
+          File.read source_file
+        end
+
+        def code_block(source_code, language, opts = { format: :markdown })
+          case opts[:format]
+          when :rst
+            ReStructuredTextHelper.code_block source_code, language
+          when :markdown
+            MarkdownHelper.code_block source_code, language
+          else
+            fail IllegalArgumentError, "Unknown format: #{format}"
+          end
+        end
+
+        # Loses proper indentation on comments
+        def snippet_after(matcher)
+          segments = @segmenter.segment(source)
+          buffer = StringIO.new
+          segment = segments.find do |s|
+            doc_segment_content = s.first.join
+            doc_segment_content.match matcher
+          end
+          buffer.print segment[1].join "\n" if segment # return code segment
+          buffer.string
+        end
+
+        def snippet_between(before_matcher, after_matcher)
+          segments = @segmenter.segment(source)
+          start_segment = find_segment_index segments, before_matcher
+          end_segment   = find_segment_index segments, after_matcher
+          buffer = StringIO.new
+          if start_segment && end_segment
+            segments[start_segment...end_segment].each do |segment|
+              buffer.puts @segmenter.comment(segment[0]) unless segment == segments[start_segment]
+              buffer.puts segment[1].join
+            end
+          end
+          buffer.puts "\n"
+          buffer.string
+        end
+
+        private
+
+        def find_segment_index(segments, matcher)
+          segments.find_index do |s|
+            doc_segment_content = s.first.join
+            doc_segment_content.match matcher
+          end
+        end
+      end
+    end
+  end
+end

data/lib/polytrix/documentation/view_helper.rb

@@ -0,0 +1,21 @@
+module Polytrix
+  module Documentation
+    module ViewHelper
+      def polytrix_toc
+        buffer = StringIO.new
+        buffer.puts '<ul>'
+        Polytrix.manifest.suites.each do |suite_name, suite|
+          buffer.puts "<li>#{suite_name}</li>"
+          buffer.puts '<ul>'
+          suite.samples.each do |challenge_name|
+            buffer.puts "<li>#{challenge_name}</li>"
+          end
+          buffer.puts '</ul>'
+        end
+        buffer.puts '</ul>'
+
+        buffer.string
+      end
+    end
+  end
+end

data/lib/polytrix/documentation_generator.rb

@@ -1,18 +1,67 @@
+require 'tilt' # seems to be a bug where padrino-helpers should require tilt
+require 'padrino-helpers'
+
 module Polytrix
   class DocumentationGenerator
-    include Polytrix::Core::FileFinder
-    attr_reader :template_file
+    [
+      Padrino::Helpers::OutputHelpers,
+      Padrino::Helpers::AssetTagHelpers,
+      Padrino::Helpers::TagHelpers,
+      Polytrix::Documentation::Helpers::CodeHelper
+    ].each do | helper|
+      include helper
+    end
+
+    attr_reader :scenario
+
+    def initialize(template_file = nil, scenario = nil)
+      @scenario = scenario
+      @template_file = template_file
+    end
+
+    def process(challenges)
+      @challenges = challenges
+      if File.readable? @template_file
+        # @template_file ||= find_file @search_path, scenario, ""
+        erb = ERB.new File.read(@template_file)
+        @result = erb.result(binding) || ''
+      end
+    end
 
-    def initialize(search_path)
-      @search_path = search_path
+    def save(target_file)
+      fail 'No results to write, please call process before save' if @result.nil?
+      if @result.empty?
+        # Warn: skip creating empty file
+      else
+        FileUtils.mkdir_p File.dirname(target_file)
+        File.open(target_file, 'wb') do |f|
+          f.write @result
+        end
+      end
     end
 
-    def process(scenario, challenges)
-      @template_file = find_file @search_path, scenario, ""
-      erb = ERB.new File.read(template_file)
-      erb.result binding
-    rescue Polytrix::Core::FileFinder::FileNotFound
-      nil
+    def code2doc(source_file, language)
+      source_code = File.read(source_file)
+      if language.nil?
+        language, comment_style = Documentation::CommentStyles.infer File.extname(source_file)
+        segmenter_language = comment_style[:language] || language
+      else
+        segmenter_language = language
+      end
+
+      buffer = StringIO.new
+      segmenter_options = {
+        language: segmenter_language
+      }
+      segmenter = Polytrix::Documentation::CodeSegmenter.new(segmenter_options)
+      segments = segmenter.segment source_code
+      segments.each do |comment, code|
+        comment = comment.join("\n")
+        code = code.join("\n")
+        buffer.puts comment unless comment.empty?
+        buffer.puts code_block code, language unless code.empty?
+      end
+      buffer.string
     end
   end
 end

data/lib/polytrix/executor.rb

@@ -0,0 +1,89 @@
+require 'hashie/dash'
+require 'thor'
+
+module Polytrix
+  module Executor
+    class OutputDecorator
+      # Reserve :red, :black, :white
+      COLORS = [:green, :yellow, :blue, :magenta, :cyan]
+
+      def self.next_color
+        @next_color ||= 0
+        @next_color += 1
+        COLORS[@next_color % COLORS.size]
+      end
+
+      def initialize(real_io, prefix = nil)
+        @real_io = real_io
+        # @prefix = set_color(prefix, :cyan)
+        @prefix = "#{prefix}: " if prefix
+        @color = self.class.next_color
+        @thor_shell = Thor::Shell::Color.new
+      end
+
+      def puts(line)
+        line = line.gsub(/^/, @prefix) if @prefix
+        @real_io.puts @thor_shell.set_color(line, @color)
+      end
+
+      def <<(line)
+        line = line.gsub(/^/, @prefix) if @prefix
+        @real_io << @thor_shell.set_color(line, @color)
+      end
+
+      def method_missing(meth, *args, &block)
+        @real_io.send meth, *args, &block
+      end
+    end
+
+    class ExecutionError < StandardError
+      attr_accessor :execution_result
+    end
+
+    class ExecutionResult < Hashie::Dash
+      property :exitstatus, require: true
+      property :stdout, required: true
+      property :stderr, required: true
+    end
+
+    class InteractiveExecutor
+      def execute(command, opts)
+        system command
+        # FIXME: This needs to return execution result, if interactive remains supported
+      end
+    end
+
+    class ShellOutExecutor
+      def execute(command, opts)
+        prefix = opts.delete :prefix
+        shell = Mixlib::ShellOut.new(command, opts)
+        shell.live_stream = OutputDecorator.new($stdout, prefix) unless Polytrix.configuration.suppress_output
+        shell.run_command
+        execution_result = ExecutionResult.new exitstatus: shell.exitstatus, stdout: shell.stdout, stderr: shell.stderr
+        begin
+          shell.error!
+        rescue Mixlib::ShellOut::ShellCommandFailed => e
+          execution_error = ExecutionError.new(e)
+          execution_error.execution_result = execution_result
+          raise execution_error
+        end
+
+        execution_result
+      end
+    end
+
+    attr_writer :executor
+
+    def executor
+      @executor ||= if ENV['INTERACTIVE']
+                      InteractiveExecutor.new
+                    else
+                      ShellOutExecutor.new
+                    end
+    end
+
+    def execute(command, opts = {})
+      executor.execute(command, opts)
+    end
+  end
+end

data/lib/polytrix/logger.rb

@@ -0,0 +1,17 @@
+require 'logger'
+
+module Polytrix
+  module ClassMethods
+    def logger
+      @logger ||= Polytrix.configuration.logger
+    end
+  end
+
+  module Logger
+    include ClassMethods
+
+    def self.included(base)
+      base.extend(ClassMethods)
+    end
+  end
+end

data/lib/polytrix/manifest.rb

@@ -2,13 +2,44 @@ require 'yaml'
 require 'hashie/dash'
 require 'hashie/mash'
 require 'hashie/extensions/coercion'
+require 'hashie/extensions/deep_merge'
 
 module Polytrix
+  # Polytrix::Manifest acts as a test manifest. It defines the test scenarios that should be run,
+  # and may be shared across multiple implementors when used for a compliance suite.
+  #
+  # A manifest is generally defined and loaded from YAML. Here's an example manifest:
+  #   ---
+  #   global_env:
+  #     LOCALE: <%= ENV['LANG'] %>
+  #     FAVORITE_NUMBER: 5
+  #   suites:
+  #     Katas:
+  #       env:
+  #         NAME: 'Max'
+  #       samples:
+  #         - hello world
+  #         - quine
+  #     Tutorials:
+  #           env:
+  #           samples:
+  #             - deploying
+  #             - documenting
+  #
+  # The *suites* object defines the tests. Each object, under suites, like *Katas* or *Tutorials* in this
+  # example, represents a test suite. A test suite is subdivided into *samples*, that each act as a scenario.
+  # The *global_env* object and the *env* under each suite define (and standardize) the input for each test.
+  # The *global_env* values will be made available to all tests as environment variables, along with the *env*
+  # values for that specific test.
+  #
   class Manifest < Hashie::Dash
+    include Logger
+    include Hashie::Extensions::DeepMerge
+
     class Environment < Hashie::Mash
-      # Hashie Coercion - automatically treat all values as string
+      # Hashie Coercion - automatically treat all values as String
       def self.coerce(obj)
-        data = obj.inject({}) do |h, (key, value)|
+        data = obj.reduce({}) do |h, (key, value)|
           h[key] = value.to_s
           h
         end
@@ -16,13 +47,17 @@ module Polytrix
       end
     end
 
-    class Suite < Hashie::Mash
+    class Suite < Hashie::Dash
+      include Hashie::Extensions::Coercion
+      property :env, default: {}
+      property :samples, default: []
+      property :results
     end
 
     class Suites < Hashie::Mash
-      # Hashie Coercion - automatically treat all values as string
+      # Hashie Coercion - automatically treat all values as Suite
       def self.coerce(obj)
-        data = obj.inject({}) do |h, (key, value)|
+        data = obj.reduce({}) do |h, (key, value)|
           h[key] = Polytrix::Manifest::Suite.new(value)
           h
         end
@@ -36,11 +71,33 @@ module Polytrix
     property :suites
     coerce_key :suites, Polytrix::Manifest::Suites
 
+    # Parses a YAML file to create a {Manifest} object.
     def self.from_yaml(yaml_file)
+      ENV['POLYTRIX_SEED'] ||= $PROCESS_ID.to_s
+      logger.debug "Loading #{yaml_file}"
       raw_content = File.read(yaml_file)
       processed_content = ERB.new(raw_content).result
-      data = YAML::load processed_content
+      data = YAML.load processed_content
       new data
     end
+
+    def find_suite(suite_name)
+      _, suite = suites.find { |name, _| name.downcase == suite_name.downcase }
+      suite
+    end
+
+    def find_challenge(suite_name, scenario_name)
+      suite = find_suite suite_name
+      return nil if suite.nil?
+
+      if suite.samples.is_a? Array
+        # No results yet
+        suite.samples.find { |name, _| name.downcase == scenario_name.downcase }
+        Challenge.new suite: suite_name, name: scenario_name
+      else
+        _, challenge_data = find_suite('identity').samples.find { |name, challenge| name.downcase == scenario_name.downcase }
+        Challenge.new(suite: suite_name, name: scenario_name, result: challenge_data)
+      end
+    end
   end
-end
+end

data/lib/polytrix/result.rb

@@ -2,8 +2,22 @@ require 'hashie/dash'
 
 module Polytrix
   class Result < Hashie::Dash
-    property :process, required: true
-    property :source
+    extend Forwardable
+    include Hashie::Extensions::Coercion
+    property :execution_result # , required: true
+    def_delegators :execution_result, :stdout, :stderr, :exitstatus
+    property :source_file # , required: true
     property :data
+    property :validations, default: Validations.new
+    coerce_key :validations, Validations
+
+    def status
+      # A feature can be validated by different suites, or manually vs an automated suite.
+      # That's why there's a precedence rather than boolean algebra here...
+      return 'failed' if validations.any? { |v| v.result == 'failed' }
+      return 'passed' if validations.any? { |v| v.result == 'passed' }
+      return 'pending' if validations.any? { |v| v.result == 'pending' }
+      'skipped'
+    end
   end
 end

data/lib/polytrix/rspec/documentation_formatter.rb

@@ -6,36 +6,61 @@ require 'fileutils'
 module Polytrix
   module RSpec
     class DocumentationFormatter < ::RSpec::Core::Formatters::BaseFormatter
+      include Polytrix::Core::FileSystemHelper
+
       def initialize(output)
+        @templates_dir = 'doc-src'
+        @output_dir = 'docs'
         @results = Hashie::Mash.new
+        @summary_files = %w(index)
         super
       end
 
       def example_group_finished(example_group)
-        group_names = example_group.parent_groups.map{|g| g.description}
-        polytrix_challenges = example_group.examples.map { |e| e.metadata[:polytrix] }
-        produce_doc example_group.description, polytrix_challenges
+        polytrix_challenges = example_group.examples.map { |e| e.metadata[:polytrix_challenge] }
+        target_file = target_file_for example_group
+        template_file = template_for example_group
+        produce_doc template_file, target_file, example_group.description, polytrix_challenges if template_file
       end
 
       def dump_summary(duration, example_count, failure_count, pending_count)
-        doc_gen = Polytrix::DocumentationGenerator.new 'doc-src'
-        all_challenges = examples.map{|e| e.metadata[:polytrix]}
+        all_challenges = examples.map { |e| e.metadata[:polytrix_challenge] }
         grouped_challenges = all_challenges.compact.group_by(&:name)
-        produce_doc 'index', grouped_challenges
+        @summary_files.each do |summary_file|
+          template_file = template_for summary_file, use_default: false
+          next if template_file.nil?
+          target_file = target_file_for_summary(template_file)
+          produce_doc template_file, target_file, 'Summary', grouped_challenges
+        end
       end
 
       private
-      def produce_doc(name, data)
-        doc_gen = Polytrix::DocumentationGenerator.new 'doc-src'
-        doc = doc_gen.process(name, data)
-        target_file = doc_gen.template_file.to_s.gsub 'doc-src', 'docs'
-        unless target_file.empty?
-          FileUtils.mkdir_p File.dirname(target_file)
-          File.open(target_file, 'wb') do |f|
-            f.write doc
-          end
+
+      def template_for(name, opts = { use_default: true })
+        name = name.description if name.respond_to? :description
+        begin
+          find_file @templates_dir, name, ''
+        rescue Polytrix::Core::FileSystemHelper::FileNotFound
+          Polytrix.configuration.default_doc_template if opts[:use_default] == true
         end
       end
+
+      def target_file_for(example_group)
+        names = [@output_dir].concat(example_group.parent_groups.reverse.map(&:description))
+        # Markdown format by default, but will be overridden to match the template
+        slugify(names.join File::SEPARATOR) + '.md'
+      end
+
+      def target_file_for_summary(template_file)
+        name = File.basename(template_file)
+        slugify("docs/#{name}")
+      end
+
+      def produce_doc(template_file, target_file, scenario, data)
+        doc_gen = Polytrix::DocumentationGenerator.new template_file, scenario
+        doc_gen.process data
+        doc_gen.save target_file
+      end
     end
   end
-end
+end

data/lib/polytrix/rspec/yaml_report.rb

@@ -0,0 +1,51 @@
+require 'polytrix/rspec'
+require 'hashie/mash'
+require 'yaml'
+require 'fileutils'
+
+module Polytrix
+  module RSpec
+    class YAMLReport < ::RSpec::Core::Formatters::BaseFormatter
+      def example_passed(example)
+        add_implementation_result example, :passed
+      end
+
+      def example_failed(example)
+        add_implementation_result example, :failed
+      end
+
+      def example_pending(example)
+        add_implementation_result example, :pending
+      end
+
+      def dump_summary(duration, example_count, failure_count, pending_count)
+        results = Hashie::Mash.new(Polytrix.manifest.dup.to_hash)
+        all_challenges = examples.map { |e| e.metadata[:polytrix_challenge] }
+        grouped_challenges = all_challenges.compact.group_by(&:name)
+        results.suites.each do |suite_name, suite|
+          suite.samples = suite.samples.each_with_object({}) do |sample_name, sample_results|
+            sample_results[sample_name] ||= {}
+            if grouped_challenges[sample_name]
+              challenge_results = grouped_challenges[sample_name]
+              challenge_results.each do |challenge|
+                sample_results[sample_name][challenge.implementor.name] = challenge.result
+              end
+            end
+          end
+        end
+        @output.puts YAML.dump(results.to_hash)
+      end
+
+      private
+
+      def add_implementation_result(example, state)
+        validation = Validation.new(
+          validated_by: 'polytrix',
+          result: state.to_s
+        )
+        challenge = example.metadata[:polytrix_challenge]
+        challenge.result.validations << validation unless challenge.nil? || challenge.result.nil?
+      end
+    end
+  end
+end

data/lib/polytrix/rspec.rb

@@ -9,67 +9,46 @@ end
 
 module Polytrix
   module RSpec
-    module Helper
-      def challenge_runner
-        @challenge_runner ||= Polytrix::ChallengeRunner.createRunner
-      end
-
-      def execute_challenge(sdk_dir, challenge_name, vars)
-        implementor_name = File.basename(sdk_dir) # Might not be a good assumption
-        implementor = Polytrix.implementors.find { |i| i.name == implementor_name }
-        challenge = ChallengeBuilder.new(implementor).build :name => challenge_name, :vars => vars, :basedir => sdk_dir, :implementor => implementor.name
-        example.metadata[:polytrix] = challenge
-        result = challenge.run
-        yield result
-      end
-    end
-
     class << self
-      def run_manifest(manifest)
-        manifest['suites'].each do |suite_name, suite_config|
-          describe suite_name do
-            suite_config['samples'].each do |scenario|
-              code_sample scenario, '', suite_config['env'].to_hash do |result|
-                instance_exec result, &Polytrix.default_validator_callback
+      def shared_examples(caller) # rubocop:disable MethodLength
+        # FIXME: Long method because it's hard to eval in the right context
+        caller.instance_eval do
+          Polytrix.manifest['suites'].each do |suite_name, suite_config|
+            describe suite_name do
+              samples = suite_config['samples'] || []
+              samples.each do |scenario|
+                describe scenario do
+                  Polytrix.implementors.each do |sdk|
+                    it sdk.name, sdk.name.to_sym => true do
+                      begin
+                        skip "#{sdk.name} is not setup" unless File.directory? sdk.basedir
+                        challenge = sdk.build_challenge suite: suite_name, name: scenario, vars: suite_config['env']
+                        example.metadata[:polytrix_challenge] = challenge
+                        challenge.run
+                        validators = Polytrix::ValidatorRegistry.validators_for challenge
+                        validators.each do |validator|
+                          instance_exec challenge, &validator.callback
+                        end
+                      rescue Polytrix::FeatureNotImplementedError => e
+                        skip e.message
+                      rescue ThreadError => e
+                        # Extra debug info for ThreadError
+                        $stderr.puts "ThreadError detected: #{e.message}"
+                        $stderr.puts "ThreadError backtrace: #{e.backtrace}"
+                        fail e
+                      end
+                    end
+                  end
+                end
               end
             end
           end
         end
       end
-    end
-  end
-end
-
-def code_sample(challenge, description = '', environment = [], services = [], &block)
-  challenge_file = challenge.downcase.gsub(' ', '_')
-  describe challenge, markdown: description,
-    # :environment => redact(environment),
-                      services: services do
-    Polytrix.implementors.each do |sdk|
-      sdk = sdk.name if sdk.respond_to? :name
-      it sdk, sdk.to_sym => true, 'data-challenge' => challenge_file, 'data-sdk' => sdk do
-        Polytrix.results.example_started example
-        begin
-          sdk_dir = Polytrix.sdk_dir sdk
-          pending "#{sdk} is not setup" unless File.directory? sdk_dir
-          challenge_runner.find_challenge! challenge_file, sdk_dir
-          execute_challenge sdk_dir, challenge_file, environment do |result|
-            Polytrix.results.execution_result example, result
-            instance_exec result, &block
-          end
 
-        rescue Polytrix::FeatureNotImplementedError => e
-          pending e.message
-        rescue ThreadError => e
-          puts "ThreadError detected: #{e.message}"
-          puts "ThreadError backtrace: #{e.backtrace}"
-          raise e
-        end
+      def run_manifest(manifest)
+        shared_examples(self)
       end
     end
   end
 end
-
-RSpec.configure do |c|
-  c.include Polytrix::RSpec::Helper
-end

data/lib/polytrix/runners/middleware/feature_executor.rb

@@ -2,6 +2,7 @@ module Polytrix
   module Runners
     module Middleware
       class FeatureExecutor
+        include Polytrix::Core::FileSystemHelper
         def initialize(app)
           @app   = app
         end
@@ -10,10 +11,10 @@ module Polytrix
           challenge_runner = env[:challenge_runner]
           env_file = env[:env_file]
           source_file = env[:source_file]
-          relative_source_file = source_file.relative_path_from env[:basedir]
+          relative_source_file = relativize(source_file, env[:basedir])
           command = challenge_runner.challenge_command(env_file, relative_source_file)
-          process = challenge_runner.run_command command
-          env[:result] = Result.new(process: process, source: env[:source_file])
+          execution_result = challenge_runner.run_command command
+          env[:result] = Result.new(execution_result: execution_result, source_file: env[:source_file].to_s)
           @app.call env
           env[:result]
         end