utterance_parser 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 85acb8695946b4e4b06d92b60739da61e665f9d8
+  data.tar.gz: e5f5470fc49b76ecb9a4982963a6208f96c5a2d6
+SHA512:
+  metadata.gz: d2a03241e5d4122d7a966188e2e1a25bbd89a5868ec24ce0e252df1b13f31c80684639fe461a5b156ed2bb2ae707303ac809aae237905964a78777b6479f787b
+  data.tar.gz: 8da431c80f7ab3122de719c392e84d515674dd784aa1559cf442e32d25164cb19367c899842378b3d3c5e85b7b248a24f57e5329cc9cf72894e0871c3d7596da

data/.gitignore

@@ -0,0 +1,22 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp
+*.bundle
+*.so
+*.o
+*.a
+mkmf.log

data/.travis.yml

@@ -0,0 +1,3 @@
+language: ruby
+rvm:
+  - 2.1.1

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in utterance_parser.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2016 macournoyer
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,75 @@
+# UtteranceParser
+
+A trainable natural language parser that extracts intent and entities from utterances.
+
+It uses a [Naive Bayes classifier](https://en.wikipedia.org/wiki/Naive_Bayes_classifier) to determine intent and [Conditional random fields](https://en.wikipedia.org/wiki/Conditional_random_field) to extract entities.
+
+For example, it can turn this:
+
+> Remind me to pick up the kids in two hours
+
+into ...
+
+```ruby
+[
+  # intent
+  "reminder",
+  # entities
+  {task: "pick up the kids", time: "in two hours"}
+]
+```
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'utterance_parser'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install utterance_parser
+
+## Usage
+
+```ruby
+parser = UtteranceParser.new
+
+parser.train(
+  # Utterance => intent
+  "Hi" => "greeting",
+  "Hello" => "greeting",
+
+  "What time is it" => "time",
+  "What's the weather outside" => "weather",
+
+  # Mark entities using XML tags
+  "Remind me to <task>get stuff done</task> <time>tomorrow</time>" => "reminder",
+  "Remind me to <task>buy milk</task> <time>in one hour</time>" => "reminder",
+  "Remind me to <task>pick up the kids</task> <time>in two hours</time>" => "reminder",
+
+  "Play some <playlist>jazz</playlist>" => "play",
+  "Play some <playlist>blues</playlist>" => "play",
+  "Play some <playlist>rap</playlist>" => "play"
+)
+
+parser.parse "Hello there!"
+# => ["greeting", {}]
+
+parser.parse "Play some rock"
+# => ["play", {playlist: "rock"}]
+
+parser.parse "Remind me to buy stuff in three hours"
+# => ["reminder", {task: "buy stuff", time: "in three hours"}]
+```
+
+## Contributing
+
+1. Fork it ( https://github.com/macournoyer/utterance_parser/fork )
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create a new Pull Request

data/Rakefile

@@ -0,0 +1,10 @@
+require "bundler/gem_tasks"
+require "rake/testtask"
+
+Rake::TestTask.new(:test) do |t|
+  t.libs << "test"
+  t.pattern = 'test/*_test.rb'
+end
+
+task :default => :test
+

data/lib/utterance_parser.rb

@@ -0,0 +1,13 @@
+require_relative 'utterance_parser/version'
+require_relative 'utterance_parser/utils'
+require_relative 'utterance_parser/utterance'
+require_relative 'utterance_parser/example'
+require_relative 'utterance_parser/pos_tagger'
+require_relative 'utterance_parser/parser'
+
+module UtteranceParser
+  def self.new(*args)
+    Parser.new(*args)
+  end
+end
+

data/lib/utterance_parser/example.rb

@@ -0,0 +1,47 @@
+module UtteranceParser
+  # An example utterance for training the parser.
+  # Can be labeled with entities via XML tags: `Play some <category>rap</category>`.
+  class Example < Utterance
+    attr_reader :labeled_text, :intent
+
+    def initialize(labeled_text, intent)
+      super(labeled_text.gsub(TAG_RE, '\2'))
+      @labeled_text = labeled_text
+      @intent = intent
+    end
+
+    def ==(other)
+      other.class == self.class && other.labeled_text == @labeled_text
+    end
+
+    def labeled_tokens
+      labels = tags_with_position(@labeled_text)
+
+      tags_with_position(PosTagger.add_tags(@text)).map do |tag, word, word_position|
+        label = labels.detect do |name, content, label_position|
+          # If the word position intersect with the label's, it's a match
+          if label_position.include? word_position.begin
+            break name
+          end
+        end
+        [word, tag.upcase, label]
+      end
+    end
+
+    private
+      # Return tags, their content, and their position in the `text` *without the tags*.
+      def tags_with_position(text)
+        tags = []
+        tags_offset = 0
+
+        text.scan(TAG_RE) do |tag, content|
+          tags_offset += tag.size + 2 # <tag>
+          index = $~.offset(2)[0] - tags_offset
+          tags << [tag, content, (index..index + content.size)]
+          tags_offset += tag.size + 3 # </tag>
+        end
+
+        tags
+      end
+  end
+end

data/lib/utterance_parser/parser.rb

@@ -0,0 +1,88 @@
+require 'nbayes'
+require 'wapiti'
+
+module UtteranceParser
+  class Parser
+    def initialize(save_path=nil)
+      build_save_paths(save_path) if save_path
+
+      if save_path && File.exist?(@classifier_file)
+        @classifier = NBayes::Base.from(@classifier_file)
+      else
+        @classifier = NBayes::Base.new
+      end
+
+      if save_path && File.exist?(@labeller_file)
+        @labeller = Wapiti::Model.load(@labeller_file)
+      else
+        @labeller = Wapiti::Model.new pattern: "#{__dir__}/pattern.txt"
+      end
+    end
+
+    def train(examples)
+      case examples
+      when Array
+        # All good!
+      when Hash
+        examples = examples.map { |utterance, intent| Example.new(utterance, intent) }
+      else
+        raise ArgumentError, "Expected [<Example>, ...] or { utterance => intent, ... }"
+      end
+
+      examples.each do |example|
+        @classifier.train(example.pos_tokens, example.intent)
+      end
+
+      labeled_examples = examples.map do |example|
+        example.labeled_tokens.map { |word, tag, entity| [word, tag, entity || "_"].join(" ") }
+      end
+
+      @labeller.train labeled_examples
+    end
+
+    def parse(text)
+      utterance = Utterance.new(text)
+      intent = @classifier.classify(utterance.pos_tokens).max_class
+      labeled = @labeller.label([ utterance.pos_tokens.map { |t| t.join(" ") } ]).first
+      [intent, extract_entities(labeled)]
+    end
+
+    def save(path=nil)
+      build_save_paths path if path
+
+      if !defined?(@classifier_file) || !defined?(@labeller_file)
+        raise ArgumentError, "Path to save directory missing"
+      end
+
+      @classifier.dump(@classifier_file)
+      @labeller.compact
+      @labeller.save(@labeller_file)
+    end
+
+    private
+      # Extract entities from tokens.
+      # Eg.::
+      #   [ ["Play NNP", "_"], ["some DET", "_"], ["jazz NN", "category"] ]
+      # Returns:
+      #   { category: "jazz" }
+      def extract_entities(tokens)
+        # FIXME this will not handle duplicated labels, eg.: category being used twice.
+        labeled = tokens.group_by { |tagged_word, label| label }
+        labeled.delete("_")
+        
+        labeled.each_with_object({}) do |(label, words), entities|
+          entities[label.to_sym] = words.map do |word, _|
+            # Remove the POS tag from. Eg.: `word == 'jazz NN'`
+            word[0, word.rindex(" ")]
+          end.join(" ")
+        end
+      end
+
+      def build_save_paths(save_path)
+        raise ArgumentError, "Path to save directory missing" unless File.directory?(save_path)
+
+        @classifier_file = File.join(save_path, "classifier.yml")
+        @labeller_file = File.join(save_path, "labeller.mod")
+      end
+  end
+end

data/lib/utterance_parser/pattern.txt

@@ -0,0 +1,32 @@
+# Wapiti CRF template file
+# Based on example in https://taku910.github.io/crfpp/#templ
+
+# Unigram
+u:%x[-3,0]
+u:%x[-2,0]
+u:%x[-1,0]
+u:%x[0,0]
+u:%x[1,0]
+u:%x[2,0]
+u:%x[3,0]
+u:%x[-1,0]/%x[0,0]
+u:%x[0,0]/%x[1,0]
+
+u:%x[-3,1]
+u:%x[-2,1]
+u:%x[-1,1]
+u:%x[0,1]
+u:%x[1,1]
+u:%x[2,1]
+u:%x[3,1]
+u:%x[-2,1]/%x[-1,1]
+u:%x[-1,1]/%x[0,1]
+u:%x[0,1]/%x[1,1]
+u:%x[1,1]/%x[2,1]
+
+u:%x[-2,1]/%x[-1,1]/%x[0,1]
+u:%x[-1,1]/%x[0,1]/%x[1,1]
+u:%x[0,1]/%x[1,1]/%x[2,1]
+
+# Bigram
+B

data/lib/utterance_parser/pos_tagger.rb

@@ -0,0 +1,13 @@
+# EngTagger generates two tons of warnings (metric however).
+# It also uses globals and monkey patches core classes.
+# But it's easy to install and just works, so ...
+#
+#                 ¯\_(ツ)_/¯
+#
+UtteranceParser::Utils.ignore_warnings do
+  require 'engtagger'
+end
+
+module UtteranceParser
+  PosTagger = Utils.ignore_warnings { EngTagger.new }
+end

data/lib/utterance_parser/utils.rb

@@ -0,0 +1,10 @@
+module UtteranceParser
+  module Utils
+    def self.ignore_warnings
+      old_verbose, $VERBOSE = $VERBOSE, nil
+      yield
+    ensure
+      $VERBOSE = old_verbose
+    end
+  end
+end

data/lib/utterance_parser/utterance.rb

@@ -0,0 +1,19 @@
+module UtteranceParser
+  class Utterance
+    TAG_RE = /<(.+?)>(.*?)<.+?>/
+
+    attr_reader :text
+
+    def initialize(text)
+      @text = text
+    end
+
+    def ==(other)
+      other.class == self.class && other.text == @text
+    end
+
+    def pos_tokens
+      PosTagger.add_tags(@text).scan(TAG_RE).map { |tag, word| [word, tag.upcase] }
+    end
+  end
+end

data/lib/utterance_parser/version.rb

@@ -0,0 +1,3 @@
+module UtteranceParser
+  VERSION = "0.1.0"
+end

data/test/example_test.rb

@@ -0,0 +1,20 @@
+require 'test_helper'
+
+class ExampleTest < Minitest::Test
+  def test_tokens
+    assert_equal [["What", "WP", nil], ["time", "NN", nil], ["is", "VBZ", nil], ["it", "PRP", nil], ["?", "PP", nil]],
+                 Example.new("What time is it?", "time").labeled_tokens
+  end
+
+  def test_single_labeled_tokens
+    assert_equal [["Play", "NNP", nil], ["some", "DET", nil], ["jazz", "NN", "category"]],
+                 Example.new("Play some <category>jazz</category>", "play").labeled_tokens
+  end
+
+  def test_multi_labeled_tokens
+    assert_equal [["Play", "NNP", nil], ["some", "DET", nil],
+                  ["smooth", "JJ", "category"], ["jazz", "NN", "category"],
+                  ["now", "RB", "time"], ["please", "VB", nil]],
+                 Example.new("Play some <category>smooth jazz</category> <time>now</time> please", "play").labeled_tokens
+  end
+end

data/test/parser_test.rb

@@ -0,0 +1,51 @@
+require 'test_helper'
+require 'fileutils'
+
+class ParserTest < Minitest::Test
+  def setup
+    @parser = Parser.new
+    @parser.train(
+      "Hi" => "greeting",
+      "Hello" => "greeting",
+      "What time is it" => "time",
+      "What's the weather outside" => "weather",
+      "Remind me to <task>buy milk</task> <time>tomorrow</time>" => "reminder",
+      "Remind me to <task>pick up the kids</task> <time>in one hour</time>" => "reminder",
+      "Remind me to <task>get stuff done</task> <time>in two hours</time>" => "reminder",
+      "Play some <category>jazz</category>" => "play",
+      "Play some <category>blues</category>" => "play",
+      "Play some <category>rap</category>" => "play",
+      "Play something" => "play",
+      "Play something please" => "play",
+    )
+  end
+
+  def test_save
+    FileUtils.mkdir_p "tmp/parser"
+    FileUtils.rm_rf "tmp/parser/*"
+    @parser.save "tmp/parser"
+    assert File.file? "tmp/parser/classifier.yml"
+    assert File.file? "tmp/parser/classifier.yml"
+    @parser = Parser.new("tmp/parser")
+  end
+
+  def self.test_parses(utterance, intent, entities=nil)
+    define_method "test_parses \"#{utterance}\"" do
+      actual_intent, actual_entities = @parser.parse utterance
+      assert_equal actual_intent, intent, "Unexpected intent"
+      assert_equal(actual_entities, entities, "Unexpected entities") if entities
+    end
+  end
+
+  test_parses "Hello there!", "greeting"
+
+  test_parses "Could you play something nice please", "play"
+
+  test_parses "Play some rock", "play", category: "rock"
+
+  test_parses "Remind me to buy stuff in three hours",
+              "reminder", task: "buy stuff", time: "in three hours"
+
+  test_parses "Remind me to go play outside tomorrow",
+              "reminder", task: "go play outside", time: "tomorrow"
+end

data/test/test_helper.rb

@@ -0,0 +1,8 @@
+$LOAD_PATH.unshift File.expand_path('../../lib', __FILE__)
+require 'utterance_parser'
+
+require 'minitest/autorun'
+
+class Minitest::Test
+  include UtteranceParser
+end

data/utterance_parser.gemspec

@@ -0,0 +1,27 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'utterance_parser/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "utterance_parser"
+  spec.version       = UtteranceParser::VERSION
+  spec.authors       = ["Marc-Andre Cournoyer"]
+  spec.email         = ["macournoyer@gmail.com"]
+  spec.summary       = "Extract intent and entities from natural language utterances"
+  spec.description   = "A trainable natural language parser that extracts intent and entities from utterances."
+  spec.homepage      = "https://github.com/macournoyer/utterance_parser"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files -z`.split("\x0")
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency 'engtagger', '~> 0.2'
+  spec.add_dependency 'nbayes', '~> 0.1'
+  spec.add_dependency 'wapiti', '~> 0.1'
+  spec.add_development_dependency "bundler", "~> 1.6"
+  spec.add_development_dependency "rake"
+  spec.add_development_dependency "minitest"
+end

metadata

@@ -0,0 +1,150 @@
+--- !ruby/object:Gem::Specification
+name: utterance_parser
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Marc-Andre Cournoyer
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2016-05-14 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: engtagger
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.2'
+- !ruby/object:Gem::Dependency
+  name: nbayes
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.1'
+- !ruby/object:Gem::Dependency
+  name: wapiti
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.1'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.6'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.6'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: A trainable natural language parser that extracts intent and entities
+  from utterances.
+email:
+- macournoyer@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".travis.yml"
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- lib/utterance_parser.rb
+- lib/utterance_parser/example.rb
+- lib/utterance_parser/parser.rb
+- lib/utterance_parser/pattern.txt
+- lib/utterance_parser/pos_tagger.rb
+- lib/utterance_parser/utils.rb
+- lib/utterance_parser/utterance.rb
+- lib/utterance_parser/version.rb
+- test/example_test.rb
+- test/parser_test.rb
+- test/test_helper.rb
+- utterance_parser.gemspec
+homepage: https://github.com/macournoyer/utterance_parser
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.2.2
+signing_key: 
+specification_version: 4
+summary: Extract intent and entities from natural language utterances
+test_files:
+- test/example_test.rb
+- test/parser_test.rb
+- test/test_helper.rb