nlu_tools 0.1.0 → 0.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 57668b33bec11c4b6f381cb623b58ecb55d160e4c9b581929c1be7dc8a7c75d5
-  data.tar.gz: 40e5cf7c30e7a193b30eb35734f4d3ae57760bf98b8ae96c23352c4581635a74
+  metadata.gz: 205a7f60e574bca2be39388e5cc4b570649d07593d10b4072cbe4f8a8ee91051
+  data.tar.gz: 2519efe88b93f42abc13408453ee1bac35c608781a6162cfc3e260cee9edb913
 SHA512:
-  metadata.gz: 25075378cd1ae7605043e2df602899f053e76c29cd40beb45b7161c3be147ae40abf43992b28fca2d2f31ebe2e8e37373b165fa501abc82cc8d9a4355cf3c925
-  data.tar.gz: 36191f7979f3dd4c9d9ba8aa3515a017cf417857167ddf47169d45245d08c9eaa2148df58c689f6ce964a686fa8144c1c833e1c47507d3d2bbe9e93e46cc21a0
+  metadata.gz: b437466a2f80dc1a6a3ce032986afceca1073215cd96afb458fec8c7709dcbbbdc6d3dcd487fb3f28a119f931ace1e1a5856dc43a22c2ba442e546c099197fe4
+  data.tar.gz: 11d6bfc0e6e6369638c0a205c3654e616825c8a72ab112f62bc4ae1cfcf5fcdf476e4ce228f20ed508594c3866c315fd660ba17f1768d31f4f17a4ce9ce36fbd

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+0.1.1 (16/Oct/2019)
+---
+* added sample data for training & testing
+* added import file schema validation
+* added testing file schema validation
+
 0.1.0 (09/Oct/2019)
 ---
 * Initial release

data/Gemfile.lock

@@ -1,8 +1,10 @@
 PATH
   remote: .
   specs:
-    nlu_tools (0.1.0)
+    nlu_tools (0.1.1)
+      json_schemer (~> 0.2.7)
       nlu_adapter (~> 0.1.6)
+      thor (~> 0.20.3)
 
 GEM
   remote: https://rubygems.org/
@@ -11,13 +13,13 @@ GEM
       public_suffix (>= 2.0.2, < 5.0)
     ast (2.4.0)
     aws-eventstream (1.0.3)
-    aws-partitions (1.220.0)
+    aws-partitions (1.223.0)
     aws-sdk-core (3.68.1)
       aws-eventstream (~> 1.0, >= 1.0.2)
       aws-partitions (~> 1.0)
       aws-sigv4 (~> 1.1)
       jmespath (~> 1.0)
-    aws-sdk-lex (1.19.0)
+    aws-sdk-lex (1.20.0)
       aws-sdk-core (~> 3, >= 3.61.1)
       aws-sigv4 (~> 1.1)
     aws-sdk-lexmodelbuildingservice (1.21.0)
@@ -29,6 +31,8 @@ GEM
     diff-lcs (1.3)
     domain_name (0.5.20190701)
       unf (>= 0.0.5, < 1.0.0)
+    ecma-re-validator (0.2.0)
+      regexp_parser (~> 1.2)
     eventmachine (1.2.7)
     faraday (0.17.0)
       multipart-post (>= 1.2, < 3)
@@ -37,29 +41,30 @@ GEM
       websocket-driver (>= 0.5.1)
     google-cloud-dialogflow (0.2.3)
       google-gax (~> 1.3)
-    google-gax (1.7.1)
-      google-protobuf (~> 3.2)
-      googleapis-common-protos (>= 1.3.5, < 2.0)
-      googleauth (>= 0.6.2, < 0.10.0)
-      grpc (>= 1.7.2, < 2.0)
+    google-gax (1.8.1)
+      google-protobuf (~> 3.9)
+      googleapis-common-protos (>= 1.3.9, < 2.0)
+      googleauth (~> 0.9)
+      grpc (~> 1.24)
       rly (~> 0.2.3)
-    google-protobuf (3.9.2)
+    google-protobuf (3.9.2-x86_64-linux)
     googleapis-common-protos (1.3.9)
       google-protobuf (~> 3.0)
       googleapis-common-protos-types (~> 1.0)
       grpc (~> 1.0)
     googleapis-common-protos-types (1.0.4)
       google-protobuf (~> 3.0)
-    googleauth (0.9.0)
+    googleauth (0.10.0)
       faraday (~> 0.12)
       jwt (>= 1.4, < 3.0)
       memoist (~> 0.16)
       multi_json (~> 1.11)
       os (>= 0.9, < 2.0)
-      signet (~> 0.7)
-    grpc (1.24.0)
+      signet (~> 0.12)
+    grpc (1.24.0-x86_64-linux)
       google-protobuf (~> 3.8)
       googleapis-common-protos-types (~> 1.0)
+    hana (1.3.5)
     http (3.3.0)
       addressable (~> 2.3)
       http-cookie (~> 1.0)
@@ -76,6 +81,11 @@ GEM
       http (~> 3.3)
     jaro_winkler (1.5.3)
     jmespath (1.4.0)
+    json_schemer (0.2.7)
+      ecma-re-validator (~> 0.2)
+      hana (~> 1.3)
+      regexp_parser (~> 1.5)
+      uri_template (~> 0.7)
     jwt (2.2.1)
     matrix (0.1.0)
     memoist (0.16.0)
@@ -94,6 +104,7 @@ GEM
     public_suffix (4.0.1)
     rainbow (3.0.0)
     rake (13.0.0)
+    regexp_parser (1.6.0)
     rly (0.2.3)
     rspec (3.9.0)
       rspec-core (~> 3.9.0)
@@ -116,15 +127,17 @@ GEM
       ruby-progressbar (~> 1.7)
       unicode-display_width (>= 1.4.0, < 1.7)
     ruby-progressbar (1.10.1)
-    signet (0.11.0)
+    signet (0.12.0)
       addressable (~> 2.3)
       faraday (~> 0.9)
       jwt (>= 1.5, < 3.0)
       multi_json (~> 1.10)
+    thor (0.20.3)
     unf (0.1.4)
       unf_ext
     unf_ext (0.0.7.6)
     unicode-display_width (1.6.0)
+    uri_template (0.7.0)
     websocket-driver (0.7.1)
       websocket-extensions (>= 0.1.0)
     websocket-extensions (0.1.4)

data/README.md

@@ -11,6 +11,7 @@ $ gem install nlu_tools
 
 ## Usage
 
+### get help
 ```bash
 $ nlu_toolset help
 $ nlu_toolset import help
@@ -21,6 +22,43 @@ $ nlu_toolset test help lex
 $ nlu_toolset test help dialogflow
 ```
 
+
+### setup keys
+```bash
+$ export AWS_REGION='AWS_REGION'
+$ export AWS_ACCESS_KEY_ID='AWS_ACCESS_KEY_ID'
+$ export AWS_SECRET_ACCESS_KEY='AWS_SECRET_ACCESS_KEY'
+$ export GOOGLE_APPLICATION_CREDENTIALS='/path/to/google-project-key.json'
+
+```
+
+### importing training data
+```bash
+$ nlu_toolset import lex -f ./data/simple_train.json --botname testbot
+
+$ nlu_toolset import dialogflow -f ./data/simple_train.json -p google-project-id
+```
+
+### running tests
+```bash
+$ nlu_toolset test lex -f ./data/simple_test.json --botname testbot -o output_l.csv -t summary
+
+$ nlu_toolset test dialogflow -f ./data/simple_test.json -p google-project-id -o output_d.csv
+```
+
+### creating training & test data
+The sample training and testing json files in [data](data) directory has two intents taken for simplicity from [Watson Assistant Sample Application repo](https://github.com/watson-developer-cloud/assistant-simple/) / [bank_simple_workspace.json](https://github.com/watson-developer-cloud/assistant-simple/blob/master/training/bank_simple_workspace.json). It is generated using [jq](https://stedolan.github.io/jq/). Please feel free to fetch the file and run the below commands to generate the full training & test set.
+
+#### get all intents and its examples except the last 2 for training:
+```bash
+$ jq '{"training_data": [.intents[] | {"intent":.intent, "utterences":[.examples[].text] | .[:-2]}]}' data/bank_simple_workspace.json > ./data/simple_train.json
+```
+
+#### get the last 2 intents and its examples for testing:
+```bash
+$ jq '{"testing_data": [.intents[] | {"intent":.intent, "utterences":[.examples[].text] | .[-2:]}]}' data/bank_simple_workspace.json > ./data/simple_test.json
+```
+
 ## Development
 
 After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
@@ -29,7 +67,7 @@ To install this gem onto your local machine, run `bundle exec rake install`. To
 
 ## Contributing
 
-Bug reports and pull requests are welcome on GitHub at https://github.com/[USERNAME]/nlu_tools. This project is intended to be a safe, welcoming space for collaboration, and contributors are expected to adhere to the [Contributor Covenant](http://contributor-covenant.org) code of conduct.
+Bug reports and pull requests are welcome on GitHub at https://github.com/technopreneurG/nlu_tools. This project is intended to be a safe, welcoming space for collaboration, and contributors are expected to adhere to the [Contributor Covenant](http://contributor-covenant.org) code of conduct.
 
 ## License
 
@@ -37,4 +75,4 @@ The gem is available as open source under the terms of the [MIT License](https:/
 
 ## Code of Conduct
 
-Everyone interacting in the NluTools project’s codebases, issue trackers, chat rooms and mailing lists is expected to follow the [code of conduct](https://github.com/[USERNAME]/nlu_tools/blob/master/CODE_OF_CONDUCT.md).
+Everyone interacting in the NluTools project’s codebases, issue trackers, chat rooms and mailing lists is expected to follow the [code of conduct](https://github.com/technopreneurG/nlu_tools/blob/master/CODE_OF_CONDUCT.md).

data/data/simple_test.json

@@ -0,0 +1,18 @@
+{
+  "testing_data": [
+    {
+      "intent": "Banking-General_Rate_Query",
+      "utterences": [
+        "APR for personal credit cards",
+        "interest rates for personal credit cards"
+      ]
+    },
+    {
+      "intent": "Loyalty_Program-Loyalty_Status",
+      "utterences": [
+        "Am I still part of the rewards program?",
+        "Am I still in the loyalty program?"
+      ]
+    }
+  ]
+}

data/data/simple_train.json

@@ -0,0 +1,38 @@
+{
+  "training_data": [
+    {
+      "intent": "Banking-General_Rate_Query",
+      "utterences": [
+        "who can i contact to talk about credit card rates",
+        "credit card rates at the moment",
+        "credit cards interest rate information",
+        "What are the interest rates on your credit cards",
+        "how about your credit card interest rates",
+        "annual percentage rate credit cards",
+        "are your card interest rates high",
+        "current interest rate for credit cards",
+        "credit cards rate of interest",
+        "What are your credit card interest rates",
+        "What are your credit card APR rates",
+        "What is the interest rate on your credit cards",
+        "tell me the annual percentage rate on your credit card",
+        "What credit card interest rate do you charge",
+        "are your APR rates low for your credit card",
+        "What is the APR on your credit cards",
+        "credit card interest rate information",
+        "would like to know about credit card rates"
+      ]
+    },
+    {
+      "intent": "Loyalty_Program-Loyalty_Status",
+      "utterences": [
+        "Can I use loyalty points",
+        "Can you tell me where Im at with the rewards points",
+        "status of loyalty program",
+        "Is this account still in the loyalty scheme",
+        "Where am I at with the loyalty program",
+        "Do I still have access to loyalty rewards"
+      ]
+    }
+  ]
+}

data/lib/nlu_tools/cli/import.rb

@@ -1,4 +1,6 @@
 # frozen_string_literal: true
+require 'pathname'
+require 'json_schemer'
 
 module NluTools
   # The subcommand import
@@ -29,12 +31,19 @@ module NluTools
         exit(1)
       end
       intents = JSON.parse(File.read(options[:file]))
-      d = NluAdapter.new(:Dialogflow,
-                         project_id: options[:project_id],
-                         session_id: 'SESSION1')
-      intents.each do |intent|
-        i = d.new_intent(intent['intent'], intent['train'])
-        d.create_intent(i)
+      schema = Pathname.new('schema/nlu_training_data.json')
+      schemer = JSONSchemer.schema(schema)
+      if schemer.valid?(intents)
+        d = NluAdapter.new(:Dialogflow,
+                           project_id: options[:project_id],
+                           session_id: 'SESSION1')
+        intents['training_data'].each do |intent|
+          i = d.new_intent(intent['intent'], intent['utterences'])
+          d.create_intent(i)
+        end
+      else
+        puts "Training data is not in valid format\nPlease check data/simple_train.json for reference"
+        exit(1)
       end
     end
 
@@ -65,21 +74,26 @@ module NluTools
       end
 
       intents = JSON.parse(File.read(options[:file]))
-      puts intents.size
-      l = NluAdapter.new(:Lex,
-                         bot_name: options[:botname],
-                         bot_alias: 'BotAlias',
-                         user_id: 'user-1')
-      lex_intents = []
-      intents.each do |intent|
-        intent_name = intent['intent'].gsub('-', '_')
-        puts '---'
-        i = l.new_intent(intent_name, intent['train'])
-        lex_intents << i
-      end
+      schema = Pathname.new('schema/nlu_training_data.json')
+      schemer = JSONSchemer.schema(schema)
+      if schemer.valid?(intents)
+        l = NluAdapter.new(:Lex,
+                           bot_name: options[:botname],
+                           bot_alias: 'BotAlias',
+                           user_id: 'user-1')
+        lex_intents = []
+        intents['training_data'].each do |intent|
+          intent_name = intent['intent'].gsub('-', '_')
+          i = l.new_intent(intent_name, intent['utterences'])
+          lex_intents << i
+        end
 
-      ic = l.new_intent_collection(options[:botname], lex_intents)
-      l.create_intent_collection(ic)
+        ic = l.new_intent_collection(options[:botname], lex_intents)
+        l.create_intent_collection(ic)
+      else
+        puts "Training data is not in valid format\nPlease check data/simple_train.json for reference"
+        exit(1)
+      end
     end
   end
 end

data/lib/nlu_tools/cli/test.rb

@@ -1,4 +1,6 @@
 # frozen_string_literal: true
+require 'pathname'
+require 'json_schemer'
 
 module NluTools
   # The subcommand for test
@@ -42,15 +44,22 @@ module NluTools
       end
 
       intents = JSON.parse(File.read(options[:file]))
-      na = NluAdapter.new(:Dialogflow,
-                          project_id: options[:project_id],
-                          session_id: 'SESSION1')
-      new_intents = {}
-      intents.each do |intent|
-        new_intents[intent['intent']] = intent['test']
-      end
+      schema = Pathname.new('schema/nlu_testing_data.json')
+      schemer = JSONSchemer.schema(schema)
+      if schemer.valid?(intents)
+        na = NluAdapter.new(:Dialogflow,
+                            project_id: options[:project_id],
+                            session_id: 'SESSION1')
+        new_intents = {}
+        intents['testing_data'].each do |intent|
+          new_intents[intent['intent']] = intent['utterences']
+        end
 
-      run_tests(na, new_intents, options[:output_file], options[:output_type])
+        run_tests(na, new_intents, options[:output_file], options[:output_type])
+      else
+        puts "Testing data is not in valid format\nPlease check data/simple_test.json for reference"
+        exit(1)
+      end
     end
 
     desc 'lex', 'test the data in <file> using AWS Lex'
@@ -95,23 +104,30 @@ module NluTools
       end
 
       intents = JSON.parse(File.read(options[:file]))
-      na = NluAdapter.new(:Lex,
-                          bot_name: options[:botname],
-                          bot_alias: 'BotAlias',
-                          user_id: 'user-1')
-      new_intents = {}
-      intents.each do |intent|
-        intent_name = intent['intent'].gsub('-', '_')
-        new_intents[intent_name] = intent['test']
-      end
+      schema = Pathname.new('schema/nlu_testing_data.json')
+      schemer = JSONSchemer.schema(schema)
+      if schemer.valid?(intents)
+        na = NluAdapter.new(:Lex,
+                            bot_name: options[:botname],
+                            bot_alias: 'BotAlias',
+                            user_id: 'user-1')
+        new_intents = {}
+        intents['testing_data'].each do |intent|
+          intent_name = intent['intent'].gsub('-', '_')
+          new_intents[intent_name] = intent['utterences']
+        end
 
-      run_tests(na, new_intents, options[:output_file], options[:output_type])
+        run_tests(na, new_intents, options[:output_file], options[:output_type])
+      else
+        puts "Testing data is not in valid format\nPlease check data/simple_test.json for reference"
+        exit(1)
+      end
     end
 
     private
 
     def run_tests(nlu_adapter, intents, output_file, output_type)
-      ext = output_file.nil? || output_file.empty? ? :none : output_file.split('.')[-1].intern
+      ext = output_file.nil? || output_file.empty? || output_file == :STDOUT ? :none : output_file.split('.')[-1].intern
       type = output_type.nil? || output_type.empty? ? :raw : output_type.intern
 
       case type

data/lib/nlu_tools/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module NluTools
-  VERSION = '0.1.0'
+  VERSION = '0.1.1'
 end

data/nlu_tools.gemspec

@@ -43,4 +43,6 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency 'rubocop', '~> 0.75.0'
 
   spec.add_dependency('nlu_adapter', '~> 0.1.6')
+  spec.add_dependency('json_schemer', '~> 0.2.7')
+  spec.add_dependency('thor', '~> 0.20.3')
 end

data/schema/nlu_testing_data.json

@@ -0,0 +1,33 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "$id": "https://github.com/technopreneurG/nlu_tools_ruby/schema/nlu_testing_data.json",
+  "title": "NluTestingData",
+  "description": "NLU testing data schema",
+  "type": "object",
+  "properties": {
+    "testing_data": {
+      "type": "array",
+      "description": "array of intents which need to be tested",
+      "items": {
+        "type": "object",
+        "properties": {
+          "intent" : {
+            "description": "known intent associated with the utterences",
+            "type": "string"
+          },
+          "utterences": {
+            "description": "the testing sentences",
+            "type": "array",
+            "items": {
+              "type": "string"
+            },
+            "minItems": 1,
+            "uniqueItems": true
+          }
+        },
+        "required": ["intent", "utterences"]
+      }
+    }
+  },
+  "required": ["testing_data"]
+}

data/schema/nlu_training_data.json

@@ -0,0 +1,33 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "$id": "https://github.com/technopreneurG/nlu_tools_ruby/schema/nlu_training_data.json",
+  "title": "NluTrainingData",
+  "description": "NLU training data schema",
+  "type": "object",
+  "properties": {
+    "training_data": {
+      "type": "array",
+      "description": "array of intents which need to be imported into the NLU system",
+      "items": {
+        "type": "object",
+        "properties": {
+          "intent" : {
+            "description": "intent associated with the utterences",
+            "type": "string"
+          },
+          "utterences": {
+            "description": "the training sentences",
+            "type": "array",
+            "items": {
+              "type": "string"
+            },
+            "minItems": 1,
+            "uniqueItems": true
+          }
+        },
+        "required": ["intent", "utterences"]
+      }
+    }
+  },
+  "required": ["training_data"]
+}

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: nlu_tools
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.1
 platform: ruby
 authors:
 - Girish Nair
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2019-10-08 00:00:00.000000000 Z
+date: 2019-10-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -94,6 +94,34 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 0.1.6
+- !ruby/object:Gem::Dependency
+  name: json_schemer
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.2.7
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.2.7
+- !ruby/object:Gem::Dependency
+  name: thor
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.20.3
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.20.3
 description: A Toolset for NLU evaluation
 email:
 - getgirish@gmail.com
@@ -115,6 +143,8 @@ files:
 - bin/console
 - bin/setup
 - data/.keep
+- data/simple_test.json
+- data/simple_train.json
 - exe/nlu_toolset
 - lib/nlu_tools.rb
 - lib/nlu_tools/cli.rb
@@ -122,6 +152,8 @@ files:
 - lib/nlu_tools/cli/test.rb
 - lib/nlu_tools/version.rb
 - nlu_tools.gemspec
+- schema/nlu_testing_data.json
+- schema/nlu_training_data.json
 homepage: https://rubygems.org/gems/nlu_tools
 licenses:
 - MIT