minimum-term 0.2.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: c8b2676a106025dc8b1940e68c3c711b82c77fda
+  data.tar.gz: 562cea8babc12813ac818aaa140ec5df1d24e865
+SHA512:
+  metadata.gz: 8b3072fecfb406b52f68f26d1085496cb33fc09a07dc64c42ca73d904c6576bf70efb8d92bb50af1807f5d7696fe1400874dac9a830aedc7bf9d137bc1c95167
+  data.tar.gz: d6621d9603a127b7b50169f258f22512f794232e40cd8c6daa751c00d8ad2ed30e2676ccd0164387c5e3aa78587a7228db132bbd122f25089847e9935a1f847f

data/.gitignore

@@ -0,0 +1,9 @@
+contracts/**/*.json
+**/*.schema.json
+spec/support/**.json
+tags
+gems.tags
+**/*blueprint-ast.json
+coverage
+**/*.gem
+Gemfile.lock

data/.rspec

@@ -0,0 +1 @@
+--color

data/.ruby-gemset

@@ -0,0 +1 @@
+minimum-term

data/.ruby-version

@@ -0,0 +1 @@
+2

data/Gemfile

@@ -0,0 +1,2 @@
+source 'https://rubygems.org'
+gemspec

data/Guardfile

@@ -0,0 +1,41 @@
+# A sample Guardfile
+# More info at https://github.com/guard/guard#readme
+
+## Uncomment and set this to only include directories you want to watch
+# directories %w(app lib config test spec features) \
+#  .select{|d| Dir.exists?(d) ? d : UI.warning("Directory #{d} does not exist")}
+
+## Note: if you are using the `directories` clause above and you are not
+## watching the project directory ('.'), then you will want to move
+## the Guardfile to a watched dir and symlink it back, e.g.
+#
+#  $ mkdir config
+#  $ mv Guardfile config/
+#  $ ln -s config/Guardfile .
+#
+# and, you'll have to watch "config/Guardfile" instead of "Guardfile"
+
+guard :bundler do
+  watch('Gemfile')
+  # Uncomment next line if your Gemfile contains the `gemspec' command.
+  watch(/^.+\.gemspec/)
+end
+
+guard 'ctags-bundler', :src_path => ["lib"] do
+  watch(/^(app|lib|spec\/support)\/.*\.rb$/)
+  watch('Gemfile.lock')
+  watch(/^.+\.gemspec/)
+end
+
+guard :rspec, cmd: 'IGNORE_LOW_COVERAGE=1 rspec', all_on_start: true  do
+  watch(%r{^spec/support/.*\.mson$}) { "spec" }
+  watch(%r{^spec/support/.*\.rb$}) { "spec" }
+  watch('spec/spec_helper.rb')  { "spec" }
+
+  # We could run individual specs, sure, but for now I dictate the tests
+  # are only green when we have 100% coverage, so partial runs will never
+  # succeed. Therefore, always run all the things.
+  watch(%r{^(spec/.+_spec\.rb)$}) { "spec" }
+  watch(%r{^lib/(.+)\.rb$})     { "spec" }
+end
+

data/README.markdown

@@ -0,0 +1,58 @@
+# Minimum term
+
+This shall be a framework so that in each of our services one can define what messages it publishes and what messages it consumes. That way when changing what one service publishes or consumes, one can immediately see the effects on our other services.
+
+
+## Prerequesites
+
+  - install drafter via `brew install --HEAD
+    https://raw.github.com/apiaryio/drafter/master/tools/homebrew/drafter.rb`
+  - run `bundle`
+
+## Tests and development
+  - run `guard` in a spare terminal which will run the tests,
+    install gems, and so forth
+  - run `rspec spec` to run all the tests
+  - check out  `open coverage/index.html` or `open coverage/rcov/index.html`
+  - run `bundle console` to play around with a console
+
+## Structure
+- Infrastructure
+  - Service
+    - Contracts
+      - Publish contract
+        - PublishedObjects
+      - Consume contract
+        - ConsumedObjects
+
+
+## Convert MSON to JSON Schema files
+First, check out [this API Blueprint map](https://github.com/apiaryio/api-blueprint/wiki/API-Blueprint-Map) to understand how _API Blueprint_ documents are laid out:
+
+![API Blueprint map](https://raw.githubusercontent.com/apiaryio/api-blueprint/master/assets/map.png)
+
+You can see that their structure covers a full API use case with resource groups, single resources, actions on those resources including requests and responses. All we want, though, is the little red top level branch called `Data structures`.
+
+In theory, we'd use a ruby gem called [RedSnow](https://github.com/apiaryio/redsnow), which has bindings to [SnowCrash](https://github.com/apiaryio/snowcrash) which parses _API Blueprints_ into an AST. Unfortunately, RedSnow ignores that red `Data structures` branch we want (SnowCrash parses it just fine).
+
+So for now, we use a command line tool called [drafter](https://github.com/apiaryio/drafter) to convert MSON into an _API Blueprint_ AST. From that AST we pic the `Data structures` entry and convert it into [JSON Schema]()s
+
+Luckily, a rake task does all that for you. To convert all `*.mson` files in `contracts/` into `*.schema.json` files,
+
+put this in your `Rakefile`:
+
+```ruby
+require "minimum-term/tasks"
+```
+
+and smoke it:
+
+```shell
+/home/dev/minimum-term$ DATA_DIR=contracts/ rake minimum_term:mson_to_json_schema
+Converting 4 files:
+OK /home/dev/minimum-term/contracts/consumer/consume.mson
+OK /home/dev/minimum-term/contracts/invalid_property/consume.mson
+OK /home/dev/minimum-term/contracts/missing_required/consume.mson
+OK /home/dev/minimum-term/contracts/publisher/publish.mson
+/home/dev/minimum-term$
+```

data/Rakefile

@@ -0,0 +1,9 @@
+#encoding: utf-8
+#!/usr/bin/env ruby
+
+require 'bundler/gem_tasks'
+Bundler.require
+
+$:.unshift File.join(File.dirname(__FILE__), "lib")
+
+require 'minimum-term/tasks'

data/circle.yml

@@ -0,0 +1,5 @@
+general:
+  build_dir: ruby
+machine:
+  ruby:
+    version: 2.2.1

data/lib/minimum-term/compare/json_schema.rb

@@ -0,0 +1,123 @@
+module MinimumTerm
+  module Compare
+    class JsonSchema
+      ERRORS = {
+        :ERR_ARRAY_ITEM_MISMATCH  => nil,
+        :ERR_MISSING_DEFINITION   => nil,
+        :ERR_MISSING_POINTER      => nil,
+        :ERR_MISSING_PROPERTY     => nil,
+        :ERR_MISSING_REQUIRED     => nil,
+        :ERR_MISSING_TYPE_AND_REF => nil,
+        :ERR_TYPE_MISMATCH        => nil,
+        :ERR_NOT_SUPPORTED        => nil
+      }
+
+      attr_reader :errors
+
+      def initialize(containing_schema)
+        @containing_schema = containing_schema
+      end
+
+      def contains?(contained_schema, pry = false)
+        @errors = []
+        @contained_schema = contained_schema
+        definitions_contained?
+      end
+
+      private
+
+      def definitions_contained?
+        @contained_schema['definitions'].each do |property, contained_property|
+          containing_property = @containing_schema['definitions'][property]
+          return _e(:ERR_MISSING_DEFINITION, [property]) unless containing_property
+          return false unless schema_contains?(containing_property, contained_property, [property])
+        end
+        true
+      end
+
+      def _e(error, location, extra = nil)
+        message = [ERRORS[error], extra].compact.join(": ")
+        @errors.push(error: error, message: message, location: location.join("/"))
+        false
+      end
+
+      def schema_contains?(publish, consume, location = [])
+
+        # We can only compare types and $refs, so let's make
+        # sure they're there
+        return _e!(:ERR_MISSING_TYPE_AND_REF) unless
+          (consume['type'] or consume['$ref']) and
+          (publish['type'] or publish['$ref'])
+
+        # There's four possibilities here:
+        #
+        # 1) publish and consume have a type defined
+        # 2) publish and consume have a $ref defined
+        # 3) publish has a $ref defined, and consume an inline object
+        # 4) consume has a $ref defined, and publish an inline object
+        #    (we don't support this yet, as otherwise couldn't check for
+        #    missing definitions, because we could never know if something
+        #    specified in the definitions of the consuming schema exists in
+        #    the publishing schema as an inline property somewhere).
+        #    TODO: check if what I just said makes sense. I'm not sure anymore.
+        # Let's go:
+
+        # 1)
+        if (consume['type'] and publish['type'])
+          if consume['type'] != publish['type']
+            return _e(:ERR_TYPE_MISMATCH, location, "#{consume['type']} != #{publish['type']}")
+          end
+
+        # 2)
+        elsif(consume['$ref'] and publish['$ref'])
+         resolved_consume = resolve_pointer(consume['$ref'], @contained_schema)
+         resolved_publish = resolve_pointer(publish['$ref'], @containing_schema)
+         return schema_contains?(resolved_publish, resolved_consume, location)
+
+        # 3)
+        elsif(consume['type'] and publish['$ref'])
+          if resolved_ref = resolve_pointer(publish['$ref'], @containing_schema)
+            return schema_contains?(resolved_ref, consume, location)
+          else
+            return _e(:ERR_MISSING_POINTER, location, publish['$ref'])
+          end
+
+        # 4)
+        elsif(consume['$ref'] and publish['type'])
+          return _e(:ERR_NOT_SUPPORTED, location)
+        end
+
+        # Make sure required properties in consume are required in publish
+        consume_required = consume['required'] || []
+        publish_required = publish['required'] || []
+        missing = (consume_required - publish_required)
+        return _e(:ERR_MISSING_REQUIRED, location, missing.to_json) unless missing.empty?
+
+        # We already know that publish and consume's type are equal
+        # but if they're objects, we need to do some recursion
+        if consume['type'] == 'object'
+          consume['properties'].each do |property, schema|
+            return _e(:ERR_MISSING_PROPERTY, location, property) unless publish['properties'][property]
+            return false unless schema_contains?(publish['properties'][property], schema, location + [property])
+          end
+        end
+
+        if consume['type'] == 'array'
+          sorted_publish = publish['items'].sort
+          consume['items'].sort.each_with_index do |item, i|
+            next if schema_contains?(sorted_publish[i], item)
+            return _e(:ERR_ARRAY_ITEM_MISMATCH, location)
+          end
+        end
+
+        true
+      end
+
+      def resolve_pointer(pointer, schema)
+        type = pointer[/\#\/definitions\/([^\/]+)$/, 1]
+        return false unless type
+        schema['definitions'][type]
+      end
+    end
+  end
+end

data/lib/minimum-term/consume_contract.rb

@@ -0,0 +1,9 @@
+require 'minimum-term/contract'
+
+module MinimumTerm
+  class ConsumeContract < MinimumTerm::Contract
+    def object_description_class
+      MinimumTerm::ConsumedObject
+    end
+  end
+end

data/lib/minimum-term/consumed_object.rb

@@ -0,0 +1,16 @@
+require 'minimum-term/object_description'
+
+module MinimumTerm
+  class ConsumedObject < MinimumTerm::ObjectDescription
+
+    def publisher
+      i = @scoped_name.index(MinimumTerm::SCOPE_SEPARATOR)
+      return @defined_in_service unless i
+      @defined_in_service.infrastructure.services[@scoped_name[0...i].underscore.to_sym]
+    end
+
+    def consumer
+      @defined_in_service
+    end
+  end
+end

data/lib/minimum-term/contract.rb

@@ -0,0 +1,34 @@
+require 'active_support/core_ext/hash/indifferent_access'
+require 'minimum-term/published_object'
+require 'minimum-term/consumed_object'
+
+module MinimumTerm
+  class Contract
+    attr_reader :service, :schema
+
+    def initialize(service, schema_or_file)
+      @service = service
+      load_schema(schema_or_file)
+    end
+
+    def objects
+      return [] unless @schema[:definitions]
+      @schema[:definitions].map do |scoped_name, schema|
+        object_description_class.new(service, scoped_name, schema)
+      end
+    end
+
+    private
+
+    def load_schema(schema_or_file)
+      if schema_or_file.is_a?(Hash)
+        @schema = schema_or_file
+      elsif File.readable?(schema_or_file)
+        @schema = JSON.parse(open(schema_or_file).read)
+      else
+        @schema = {}
+      end
+      @schema = @schema.with_indifferent_access
+    end
+  end
+end

data/lib/minimum-term/conversion/apiary_to_json_schema.rb

@@ -0,0 +1,9 @@
+require 'minimum-term/conversion/data_structure'
+
+module MinimumTerm
+  module Conversion
+    class ApiaryToJsonSchema
+
+    end
+  end
+end

data/lib/minimum-term/conversion/data_structure.rb

@@ -0,0 +1,92 @@
+require 'active_support/core_ext/string'
+
+module MinimumTerm
+  module Conversion
+    class DataStructure
+      PRIMITIVES = %w{boolean string number array enum object}
+
+      def self.scope(scope, string)
+        [scope, string.to_s].compact.join(MinimumTerm::SCOPE_SEPARATOR).underscore
+      end
+
+      def initialize(id, data, scope = nil)
+        @scope = scope
+        @data = data
+        @id = self.class.scope(@scope, id)
+        @schema = json_schema_blueprint
+        @schema['title'] = @id
+        add_description_to_json_schema
+        add_properties_to_json_schema
+      end
+
+      def to_json
+        @schema
+      end
+
+      private
+
+      def add_description_to_json_schema
+        return unless @data['meta']
+        description = @data['meta']['description']
+        return unless description
+        @schema['description'] = description.strip
+      end
+
+      def add_properties_to_json_schema
+        return unless @data['content']
+        members = @data['content'].select{|d| d['element'] == 'member' }
+        members.each do |s|
+          content = s['content']
+          type = content['value']['element']
+
+          spec = {}
+          name = s['content']['key']['content'].underscore
+
+          # This is either type: primimtive or $ref: reference_name
+          spec.merge!(primitive_or_reference(type))
+
+          value_content = content['value']['content']
+
+          # We might have a description
+          spec['description'] = s['meta']['description'] if s['meta']
+
+          # If it's an array, we need to pluck out the item types
+          if type == 'array'
+            nestedTypes = value_content.map{|d| d['element'] }.compact
+            spec['items'] = nestedTypes.map{|t| primitive_or_reference(t) }
+
+          # If it's an object, we need recursion
+          elsif type == 'object'
+            spec['properties'] = {}
+            value_content.select{|d| d['element'] == 'member'}.each do |data|
+              data_structure = DataStructure.new('tmp', content['value'], @scope).to_json
+              spec['properties'].merge!(data_structure['properties'])
+            end
+          end
+
+          @schema['properties'][name] = spec
+          if attributes = s['attributes']
+            @schema['required'] << name if attributes['typeAttributes'].include?('required')
+          end
+        end
+      end
+
+      def primitive_or_reference(type)
+        if PRIMITIVES.include?(type)
+          { 'type' => type }
+        else
+          { '$ref' => "#/definitions/#{self.class.scope(@scope, type)}" }
+        end
+      end
+
+      def json_schema_blueprint
+        {
+          "type" => "object",
+          "properties" => {},
+          "required" => []
+        }
+      end
+
+    end
+  end
+end

data/lib/minimum-term/conversion/error.rb

@@ -0,0 +1,7 @@
+module MinimumTerm
+  module Conversion
+    class Error < StandardError
+
+    end
+  end
+end

data/lib/minimum-term/conversion.rb

@@ -0,0 +1,125 @@
+require 'fileutils'
+require 'open3'
+require 'minimum-term/conversion/apiary_to_json_schema'
+require 'minimum-term/conversion/error'
+
+module MinimumTerm
+  module Conversion
+    def self.mson_to_json_schema(filename:, keep_intermediary_files: false, verbose: false)
+      begin
+        mson_to_json_schema!(filename: filename, keep_intermediary_files: keep_intermediary_files, verbose: verbose)
+        puts "OK ".green + filename if verbose
+        true
+      rescue
+        puts "ERROR ".red + filename if verbose
+        false
+      end
+    end
+
+    def self.mson_to_json_schema!(filename:, keep_intermediary_files: false, verbose: true)
+
+      # For now, we'll use the containing directory's name as a scope
+      service_scope = File.dirname(filename).split(File::SEPARATOR).last.underscore
+
+      # Parse MSON to an apiary blueprint AST
+      # (see https://github.com/apiaryio/api-blueprint/wiki/API-Blueprint-Map)
+      to_ast = mson_to_ast_json(filename)
+      raise Error, "Error: #{to_ast}" unless to_ast[:status] == 0
+
+      # Pluck out Data structures from it
+      data_structures = data_structures_from_blueprint_ast(to_ast[:outfile])
+
+      # Generate json schema from each contained data structure
+      schema = {
+        "$schema"     => "http://json-schema.org/draft-04/schema#",
+        "title"       => service_scope,
+        "definitions" => {},
+        "type"        => "object",
+        "properties"  => {},
+      }
+
+      # The scope for the data structure is the name of the service
+      # publishing the object. So if we're parsing a 'consume' schema,
+      # the containing objects are alredy scoped (because a consume
+      # schema says 'i consume object X from service Y'.
+      basename = File.basename(filename)
+      if basename.end_with?("publish.mson")
+        data_structure_autoscope = service_scope
+      elsif basename.end_with?("consume.mson")
+        data_structure_autoscope = nil
+      else
+        raise Error, "Invalid filename #{basename}, can't tell if it's a publish or consume schema"
+      end
+
+      # The json schema we're constructing contains every known
+      # object type in the 'definitions'. So if we have definitions for
+      # the objects User, Post and Tag, the schema will look like this:
+      #
+      # {
+      #   "$schema": "..."
+      #
+      #   "definitions": {
+      #     "user": { "type": "object", "properties": { ... }}
+      #     "post": { "type": "object", "properties": { ... }}
+      #     "tag":  { "type": "object", "properties": { ... }}
+      #   }
+      #
+      #   "properties": {
+      #     "user": "#/definitions/user"
+      #     "post": "#/definitions/post"
+      #     "tag":  "#/definitions/tag"
+      #   }
+      #
+      # }
+      #
+      # So when testing an object of type `user` against this schema,
+      # we need to wrap it as:
+      #
+      # {
+      #   user: {
+      #     "your": "actual",
+      #     "data": "goes here"
+      #     }
+      # }
+      #
+      data_structures.each do |data|
+        schema_data = data['content'].select{|d| d['element'] == 'object' }.first
+        id = schema_data['meta']['id']
+        json= DataStructure.new(id, schema_data, data_structure_autoscope).to_json
+        member = json.delete('title')
+        schema['definitions'][member] = json
+        schema['properties'][member] = {"$ref" => "#/definitions/#{member}"}
+      end
+
+      # Write it in a file
+      outfile = filename.gsub(/\.\w+$/, '.schema.json')
+      File.open(outfile, 'w'){ |f| f.puts JSON.pretty_generate(schema) }
+
+      # Clean up
+      FileUtils.rm_f(to_ast[:outfile]) unless keep_intermediary_files
+      true
+    end
+
+    def self.data_structures_from_blueprint_ast(filename)
+      c = JSON.parse(open(filename).read)['content'].first
+      return [] unless c
+      c['content']
+    end
+
+    def self.mson_to_ast_json(filename)
+      input = filename
+      output = filename.gsub(/\.\w+$/, '.blueprint-ast.json')
+
+      cmd = "drafter -u -f json -o #{Shellwords.escape(output)} #{Shellwords.escape(input)}"
+      stdin, stdout, status = Open3.capture3(cmd)
+
+      {
+        cmd: cmd,
+        outfile: output,
+        stdin: stdin,
+        stdout: stdout,
+        status: status
+      }
+    end
+  end
+end

data/lib/minimum-term/infrastructure.rb

@@ -0,0 +1,75 @@
+require 'active_support/core_ext/hash/indifferent_access'
+
+module MinimumTerm
+  class Infrastructure
+    attr_reader :services, :errors, :data_dir
+
+    def initialize(data_dir:, verbose: false)
+      @verbose = !!verbose
+      @data_dir = data_dir
+      @mutex = Mutex.new
+      load_services
+    end
+
+    def reload
+      load_services
+    end
+
+    def contracts_fulfilled?
+      load_services
+      @mutex.synchronize do
+        @errors = {}
+        publishers.each do |publisher|
+          publisher.satisfies_consumers?(verbose: @verbose)
+          next if publisher.errors.empty?
+          @errors.merge! publisher.errors
+        end
+        @errors.empty?
+      end
+    end
+
+    def publishers
+      services.values.select do |service|
+        service.published_objects.length > 0
+      end
+    end
+
+    def consumers
+      services.values.select do |service|
+        service.consumed_objects.length > 0
+      end
+    end
+
+    def convert_all!(keep_intermediary_files = false)
+      json_files.each{ |file| FileUtils.rm_f(file) }
+      mson_files.each do |file|
+        MinimumTerm::Conversion.mson_to_json_schema!(
+          filename: file,
+          keep_intermediary_files: keep_intermediary_files,
+          verbose: @verbose)
+      end
+      reload
+    end
+
+    def mson_files
+      Dir.glob(File.join(@data_dir, "/**/*.mson"))
+    end
+
+    def json_files
+      Dir.glob(File.join(@data_dir, "/**/*.schema.json"))
+    end
+
+    private
+
+    def load_services
+      @mutex.synchronize do
+        @services = {}.with_indifferent_access
+        dirs = Dir.glob(File.join(@data_dir, "*/"))
+        dirs.each do |dir|
+          service = MinimumTerm::Service.new(self, dir)
+          @services[service.name] = service
+        end
+      end
+    end
+  end
+end

data/lib/minimum-term/object_description.rb

@@ -0,0 +1,34 @@
+# This represents a description of an Object (as it was in MSON and later
+# JSON Schema). It can come in two flavors:
+#
+#   1) A published object
+#   2) A consumed object
+#
+# A published object only refers to one servce:
+#
+#   - its publisher
+#
+# However, a consumed object is referring to two services:
+#
+#   - its publisher
+#   - its consumer
+#
+#
+module MinimumTerm
+  class ObjectDescription
+    attr_reader :service, :name, :schema
+    def initialize(defined_in_service, scoped_name, schema)
+      @defined_in_service = defined_in_service
+      @scoped_name = scoped_name
+      @name = remove_service_from_scoped_name(scoped_name)
+      @schema = schema
+    end
+
+    private
+
+    def remove_service_from_scoped_name(n)
+      n[n.index(MinimumTerm::SCOPE_SEPARATOR)+1..-1]
+    end
+
+  end
+end