simmer 1.0.0.pre.alpha.6 → 1.0.0.pre.alpha.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d7bb744ed6c7545eff9608e7b4548b5b71afbc219d75cba872f93b6fb0e7db31
-  data.tar.gz: ab9487df40d1d7dd7030c09e9d35398cf45169ec5c1ac322165728ee40bdaa8a
+  metadata.gz: c2f5116d06a3aa1eaafbe7a14cc204f40eeeb1a45408801ecc9f15706b36f2a2
+  data.tar.gz: 49c6a41c862824592fbd31fa9654570d93bb84a6ba70f8e59353b7187dc15e17
 SHA512:
-  metadata.gz: da855ec25b4bccb880dc974abab073cadd22467f36451320c1bf550599f818c72684c5c2b01e61453a8d498e46beb949c21035b1a012009c4ee37f9e36ef59ad
-  data.tar.gz: e906e083ade596051e93e5a67f64faa5c306ac112eaafa0dabb507c2e42d0daec0baa3434645ee39675b0b05ff9657db331ef435ff36417dc4d40294ce3f7023
+  metadata.gz: e2315fec18c186454fd415c4050f183d753a635f5e45e630da3b609adfd1f498930c18a25f2ea61f4a79b0dd59fb6ee9558c2081b7092a8624fe41c4e70d68d1
+  data.tar.gz: c5572b235b2bf109e213c3d428bd7668c97af2967f56a925d3d6b0f522f1689af9e46124ccec2460e1f71a5a04f36eb13478380e633d060e4d7999f32b198739

data/README.md

@@ -2,4 +2,314 @@
 
 ---
 
-Under construction.
+[![Gem Version](https://badge.fury.io/rb/simmer.svg)](https://badge.fury.io/rb/simmer) [![Build Status](https://travis-ci.org/bluemarblepayroll/simmer.svg?branch=master)](https://travis-ci.org/bluemarblepayroll/simmer) [![Maintainability](https://api.codeclimate.com/v1/badges/61996dff817d44efc408/maintainability)](https://codeclimate.com/github/bluemarblepayroll/simmer/maintainability) [![Test Coverage](https://api.codeclimate.com/v1/badges/61996dff817d44efc408/test_coverage)](https://codeclimate.com/github/bluemarblepayroll/simmer/test_coverage) [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
+
+*Note: This is not officially supported by Hitachi Vantara.*
+
+This library provides is a Ruby-based testing suite for [Pentaho Data Integration](https://www.hitachivantara.com/en-us/products/data-management-analytics/pentaho-platform/pentaho-data-integration.html).  You can create specifications for Pentaho transformations and jobs then ensure they always run correctly.
+
+## Compatibility & Limitations
+
+This library was tested against:
+
+* Kettle version 6.1.0.1-196
+* MacOS and Linux
+
+Note that it also is currently limited to:
+
+* MySQL
+* Amazon Simple Storage Service
+
+Future enhancements potentially could include breaking these out and making them plug-ins in order to support other database and cloud storage vendors/systems.
+
+## Installation
+
+To install through Rubygems:
+
+````bash
+gem install simmer
+````
+
+You can also add this to your Gemfile:
+
+````bash
+bundle add simmer
+````
+
+After installation, you will need do to two things:
+
+1. Add simmer configuration file
+2. Add simmer directory
+
+### Simmer Configuration File
+
+The configuration file contains information about external systems, such as:
+
+* Amazon Simple Storage Service
+* Local File System
+* Pentaho Data Integration
+* MySQL Database
+
+Copy this configuration template into your project's root to: `config/simmer.yaml`:
+
+````yaml
+mysql_database:
+  database:
+  username:
+  host:
+  port:
+  flags: MULTI_STATEMENTS
+
+spoon_client:
+  dir: spec/mocks/spoon
+  args: 0
+
+# local_file_system:
+#  dir: tmp/store_test
+
+# aws_file_system:
+#   access_key_id:
+#   bucket:
+#   default_expires_in_seconds: 3600
+#   encryption: AES256
+#   region:
+#   secret_access_key:
+````
+
+Fill out the missing configuration values required for each section.  If you would like to use your local file system then un-comment the `local_file_system` key.  If you would like to use AWS S3 then un-comment out the `aws_file_system` key.
+
+### Simmer Directory
+
+You will also need to create the following folder structure in your project's root folder:
+
+* **simmer/files**: Place any files necessary to stage in this directory.
+* **simmer/fixtures**: Place yaml files, that describe database records, necessary to stage the database.
+* **simmer/specs**: Place specification yaml files here.
+
+It does not matter how each of these directories are internally structured, they can contain folder structure in any arbitrary way.  These directories should be version controlled as they contain the necessary information to execute your tests.  But you may want to ignore the `simmer/results` directory as that will store the results after execution.
+
+## Getting Started
+
+### What is a Specification?
+
+A specification is a blueprint for how to run a transformation or job and contains configuration for:
+
+* File system state before execution
+* Database state before execution
+* Execution command
+* Expected database state after execution
+* Expected execution output
+
+#### Specification Example
+
+The following is an example specification for a transformation:
+
+````yaml
+name: Declassify Users
+stage:
+  files:
+    src: noc_list.csv
+    dest: input/noc_list.csv
+  fixtures:
+    - iron_man
+    - hulk
+act:
+  name: load_noc_list
+  repository: top_secret
+  type: transformation
+  params:
+    files:
+      input_file: noc_list.csv
+    keys:
+      code: 'The secret code is: {codes.the_secret_one}'
+assert:
+  assertions:
+    - type: table
+      name: agents
+      records:
+        - call_sign: iron_man
+          first: tony
+          last: stark
+        - call_sign: hulk
+          first: bruce
+          last: banner
+    - type: table
+      name: agents
+      logic: includes
+      records:
+        - last: stark
+    - type: output
+      value: output to stdout
+````
+
+##### Stage Section
+
+The stage section defines the pre-execution state that needs to exist before PDI execution.  There are two options:
+
+1. Files
+2. Fixtures
+
+###### Files
+
+Each file entry specifies two things:
+
+* **src**: the location of the file (relative to the `simmer/files`)
+* **dest**: where to copy it to (within the configured file system: local or S3)
+
+###### Fixtures
+
+Each fixture entry contains the name of a fixture specified in one of the yaml files within fixture directory.
+
+Fixtures live in yaml files within the `simmer/fixtures` directory.  They can be placed in any arbitrary file, the only restriction is their top-level keys that uniquely identify a fixture.  Here is an example of a fixture file:
+
+````yaml
+hulk:
+  table: agents
+  fields:
+    call_sign: hulk
+    first: CLASSIFIED
+    last: CLASSIFIED
+
+iron_man:
+  table: agents
+  fields:
+    call_sign: iron_man
+    first: CLASSIFIED
+    last: CLASSIFIED
+````
+
+This example specifies two fixtures: `hulk` and `iron_man`.  Each will end up creating a record in the `agents` table with their respective attributes (columns.)
+
+##### Act Section
+
+The act configuration contains the necessary information for invoking Pentaho through its Spoon script.  The options are:
+
+* **name**: The name of the transformation or job
+* **repository**: The name of the Kettle repository
+* **type**: transformation or job
+* **file params**: key-value pairs to send through to Spoon as params.  The values will be joined with and are relative to the `simmer/files` directory.
+* **key params**: key-value pairs to send through to Spoon as params.
+
+##### Assert Section
+
+The assert section contains the expected state of:
+
+* Database table contents
+* Pentaho output contents
+
+Take the assert block from the example above:
+
+````yaml
+assert:
+  assertions:
+    - type: table
+      name: agents
+      records:
+        - call_sign: iron_man
+          first: tony
+          last: stark
+        - call_sign: hulk
+          first: bruce
+          last: banner
+    - type: table
+      name: agents
+      logic: includes
+      records:
+        - last: stark
+    - type: output
+      value: output to stdout
+````
+
+This contains two table and one output assertion.  It explicitly states that:
+
+* The table `agents` should exactly contain two records with the column values as described (iron_man and hulk)
+* The table `agents` should include a record where the last name is `stark`
+* The output should contain the string described in the value somewhere in the log
+
+###### Table Assertion Rules
+
+Currently table assertions operate under a very rudimentary set of rules:
+
+* Record order does not matter
+* Each record being asserted should have the same keys compared
+* All values are asserted against their string coerced value
+* There is no concept of relationships or associations (yet)
+
+### Running Tests
+
+After you have configured simmer and written a specification, you can run it by executing:
+
+````bash
+bundle exec simmer ./simmer/specs/name_of_the_spec.yaml
+````
+
+The passed in path can also be a directory and all specs in the directory (recursively) will be executed:
+
+````bash
+bundle exec simmer ./simmer/specs/some_directory
+````
+
+You can also omit the path altogether to execute all specs:
+
+````bash
+bundle exec simmer
+````
+
+## Contributing
+
+### Development Environment Configuration
+
+Basic steps to take to get this repository compiling:
+
+1. Install [Ruby](https://www.ruby-lang.org/en/documentation/installation/) (check simmer.gemspec for versions supported)
+2. Install bundler (gem install bundler)
+3. Clone the repository (git clone git@github.com:bluemarblepayroll/simmer.git)
+4. Navigate to the root folder (cd simmer)
+5. Install dependencies (bundle)
+
+### Running Tests
+
+To execute the test suite and code-coverage tool, run:
+
+````bash
+bundle exec rspec spec --format documentation
+````
+
+Alternatively, you can have Guard watch for changes:
+
+````bash
+bundle exec guard
+````
+
+Also, do not forget to run Rubocop:
+
+````bash
+bundle exec rubocop
+````
+
+or run all three in one command:
+
+````bash
+bundle exec rake
+````
+
+### Publishing
+
+Note: ensure you have proper authorization before trying to publish new versions.
+
+After code changes have successfully gone through the Pull Request review process then the following steps should be followed for publishing new versions:
+
+1. Merge Pull Request into master
+2. Update `lib/simmer/version.rb` using [semantic versioning](https://semver.org/)
+3. Install dependencies: `bundle`
+4. Update `CHANGELOG.md` with release notes
+5. Commit & push master to remote and ensure CI builds master successfully
+6. Run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Code of Conduct
+
+Everyone interacting in this codebase, issue trackers, chat rooms and mailing lists is expected to follow the [code of conduct](https://github.com/bluemarblepayroll/simmer/blob/master/CODE_OF_CONDUCT.md).
+
+## License
+
+This project is MIT Licensed.

data/lib/simmer/specification/assert/assertions/table.rb

@@ -17,9 +17,23 @@ module Simmer
         class Table
           acts_as_hashable
 
-          attr_reader :name, :record_set
+          module Logic
+            EQUALS   = :equals
+            INCLUDES = :includes
+          end
+          include Logic
+
+          LOGIC_METHODS = {
+            EQUALS => ->(actual_record_set, record_set) { actual_record_set == record_set },
+            INCLUDES => lambda { |actual_record_set, record_set|
+              (actual_record_set & record_set) == record_set
+            }
+          }.freeze
+
+          attr_reader :logic, :name, :record_set
 
-          def initialize(name:, records: [])
+          def initialize(logic: EQUALS, name:, records: [])
+            @logic      = Logic.const_get(logic.to_s.upcase.to_sym)
             @name       = name.to_s
             @record_set = Util::RecordSet.new(records)
 
@@ -31,7 +45,7 @@ module Simmer
             actual_records      = database.records(name, keys)
             actual_record_set   = Util::RecordSet.new(actual_records)
 
-            return nil if actual_record_set == record_set
+            return nil if LOGIC_METHODS[logic].call(actual_record_set, record_set)
 
             BadTableAssertion.new(name, record_set, actual_record_set)
           end

data/lib/simmer/util/record.rb

@@ -16,7 +16,7 @@ module Simmer
 
       attr_reader :data
 
-      def_delegators :data, :to_h
+      def_delegators :data, :to_h, :hash
 
       def initialize(data = {})
         data = data.respond_to?(:to_h) ? data.to_h : data

data/lib/simmer/util/record_set.rb

@@ -41,6 +41,10 @@ module Simmer
         records.flat_map(&:keys)
       end
 
+      def &(other)
+        self.class.new(records & other.records)
+      end
+
       private
 
       def array(val)

data/lib/simmer/version.rb

@@ -8,5 +8,5 @@
 #
 
 module Simmer
-  VERSION = '1.0.0-alpha.6'
+  VERSION = '1.0.0-alpha.7'
 end

data/spec/fixtures/specifications/load_noc_list.yaml

@@ -26,5 +26,12 @@ assert:
         - call_sign: hulk
           first: bruce
           last: banner
+    - type: table
+      name: agents
+      logic: includes
+      records:
+        - call_sign: iron_man
+          first: tony
+          last: stark
     - type: output
       value: output to stdout

data/spec/simmer/specification/assert_spec.rb

@@ -18,9 +18,11 @@ describe Simmer::Specification::Assert do
     subject { described_class.make(config) }
 
     it 'sets assertions' do
-      expect(subject.assertions.length).to                  eq(2)
+      expect(subject.assertions.length).to                  eq(3)
       expect(subject.assertions.first.name).to              eq('agents')
       expect(subject.assertions.first.record_set.length).to eq(2)
+      expect(subject.assertions[1].name).to                 eq('agents')
+      expect(subject.assertions[1].record_set.length).to    eq(1)
       expect(subject.assertions.last.value).to              eq('output to stdout')
     end
   end

data/spec/simmer/util/record_set_spec.rb

@@ -38,6 +38,13 @@ describe Simmer::Util::RecordSet do
     }
   end
 
+  let(:hash3) do
+    {
+      e: 'e',
+      f: 'f'
+    }
+  end
+
   subject { described_class.new([hash1, hash2]) }
 
   describe 'equality' do
@@ -64,4 +71,14 @@ describe Simmer::Util::RecordSet do
       expect(subject.records.length).to eq(1)
     end
   end
+
+  specify '#& returns intersection of both record sets' do
+    subject1 = described_class.new([hash1, hash2])
+    subject2 = described_class.new([hash1, hash3])
+
+    expected = described_class.new([hash1])
+    actual   = subject1 & subject2
+
+    expect(actual).to eq(expected)
+  end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: simmer
 version: !ruby/object:Gem::Version
-  version: 1.0.0.pre.alpha.6
+  version: 1.0.0.pre.alpha.7
 platform: ruby
 authors:
 - Matthew Ruggio