expando 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 2d9e0e4d3a50bb75dea3ac215b6bedc116c5d21a
+  data.tar.gz: 2397629047792d35cc5171cc9808bb514e30157e
+SHA512:
+  metadata.gz: 816a5d9adc5f1a066b5385132276c892d192f15daf50d53b18040e4490b4d3f5ff595058a8d7115c71669af49c94b746c25004a0a67b9ec0f76be203205ca794
+  data.tar.gz: b7e3f739d3be12cad151a26da0106797c81a37d9e8e4d8967bcf06c0fac05642524b2adbd55331da29ed3993611ba22c21df2fe6621b5b40f229a2013380c14b

data/.gitignore

@@ -0,0 +1,9 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/

data/.rspec

@@ -0,0 +1,2 @@
+--format documentation
+--color

data/.ruby-gemset

@@ -0,0 +1 @@
+expando

data/.ruby-version

@@ -0,0 +1 @@
+ruby-2.2.0

data/Gemfile

@@ -0,0 +1,8 @@
+source 'https://rubygems.org'
+
+# gem 'api-ai-ruby', git: 'git@github.com:voxable-labs/api-ai-ruby.git'
+# Uncomment the below when developing on api-ai-ruby locally
+gem 'api-ai-ruby', :path => '../../api-ai-ruby'
+
+# Specify your gem's dependencies in expando.gemspec
+gemspec

data/README.md

@@ -0,0 +1,2 @@
+# Expando
+

data/Rakefile

@@ -0,0 +1 @@
+require "bundler/gem_tasks"

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require 'bundler/setup'
+require 'expando'
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require 'irb'
+IRB.start

data/bin/expando

@@ -0,0 +1,82 @@
+#!/usr/bin/env ruby
+
+$LOAD_PATH << File.expand_path( File.dirname( __FILE__ ) + '/../lib' )
+
+require 'expando'
+require 'gli'
+
+include GLI::App
+
+version Expando::VERSION
+
+program_desc 'A tool for managing files written in the Expando markup language.'
+
+config_file File.join(ENV['HOME'],'.expando.rc.yaml')
+
+flag :client_access_token,    desc: 'Api.ai Client Access Token'
+flag :developer_access_token, desc: 'Api.ai Developer Access Token'
+flag :intents_path,           desc: 'The path to the directory containing intent files'
+flag :entities_path,          desc: 'The path to the directory containing entity files'
+
+# Make credentials globally accessible
+pre do | global_options, command, options, args |
+  # TODO: Why are these being repeated in global_options?
+  global_options[:credentials] = {
+      client_access_token:    global_options[:client_access_token],
+      developer_access_token: global_options[:developer_access_token]
+  }
+end
+
+desc 'Update entities and/or intents'
+long_desc <<-DESC
+  Update Expando's entities and intents on Api.ai based on the contents of the
+  files in the entities and intents directories, respectively.
+DESC
+command [ :update, :u ] do | c |
+  c.desc "Update Expando's entities"
+  c.long_desc <<-DESC
+    Updates the specified entities. The entity names can be specified after this
+    argument in a comma delimited list, and should match the file name of the
+    specific entity in the entities directory. If no entity name is specified,
+    all entities are updated.
+  DESC
+  c.arg 'entity, entity[, entity]*'
+  c.command :entities do | entities |
+    entities.action do | global_options, options, args |
+      # TODO: update all by default
+      args.each do | entity |
+        updater = Expando::EntityUpdater.new( entity,
+                                              client_keys:   global_options[:credentials],
+                                              intents_path:  global_options[:intents_path],
+                                              entities_path: global_options[:entities_path])
+        updater.update!
+      end
+    end
+  end
+
+  c.desc "Update Api.ai intents"
+  c.long_desc <<-DESC
+    Updates the specified intents. The intent names can be specified after this
+    argument in a comma delimited list, and should match the file name of the
+    specific intent in the intents directory. If no intent name is specified,
+    all intents are updated.
+  DESC
+  c.arg 'intent, intent[, intent]*'
+  c.command :intents do | intents |
+    intents.action do | global_options, options, args |
+      # TODO: update all by default
+      args.each do | intent |
+        updater = Expando::IntentUpdater.new( intent,
+                                              client_keys:   global_options[:credentials],
+                                              intents_path:  global_options[:intents_path],
+                                              entities_path: global_options[:entities_path])
+        updater.update!
+      end
+    end
+  end
+
+  # TODO: Default to both in sequence
+  #c.default_command :entities
+end
+
+exit run(ARGV)

data/bin/setup

@@ -0,0 +1,7 @@
+#!/bin/bash
+set -euo pipefail
+IFS=$'\n\t'
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/circle.yml

@@ -0,0 +1,8 @@
+test:
+  override:
+    - RAILS_ENV=test bundle exec rspec -r rspec_junit_formatter --format RspecJunitFormatter -o $CIRCLE_TEST_REPORTS/rspec/junit.xml
+deployment:
+  staging:
+    branch: /.*/
+    commands:
+      - bundle exec ./bin/expando update entities app

data/expando.gemspec

@@ -0,0 +1,31 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'expando/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = 'expando'
+  spec.version       = Expando::VERSION
+  spec.authors       = ['Matt Buck']
+  spec.email         = ['matt@voxable.io']
+
+  spec.summary       = 'The Expando reference implementation.'
+  spec.description   = 'A translation language for defining user utterance examples in conversational interfaces.'
+  spec.homepage      = 'http://voxable.io'
+
+  spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  spec.bindir        = 'bin'
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.require_paths = ['lib']
+
+  spec.add_development_dependency 'bundler', '~> 1.9'
+  spec.add_development_dependency 'rake', '~> 10.0'
+  spec.add_development_dependency 'rspec', '~> 3.4.0'
+  spec.add_development_dependency 'rspec_junit_formatter', '~> 0.2.2'
+  spec.add_development_dependency 'climate_control', '~> 0.0.3'
+
+  spec.add_runtime_dependency 'api-ai-ruby', '~> 1.1.0'
+  spec.add_runtime_dependency 'gli', '~> 2.13.4'
+  spec.add_runtime_dependency 'colorize', '~> 0.7.7'
+  spec.add_runtime_dependency 'awesome_print', '~> 1.6.1'
+end

data/lib/expando.rb

@@ -0,0 +1,12 @@
+require 'expando/version'
+require 'expando/updater'
+require 'expando/entity_updater'
+require 'expando/intent_updater'
+require 'expando/expander'
+require 'api-ai-ruby'
+require 'colorize'
+require 'awesome_print'
+
+module Expando
+  # Your code goes here...
+end

data/lib/expando/entity_updater.rb

@@ -0,0 +1,60 @@
+module Expando
+  # Responsible for updating entity objects on Api.ai based on the contents of
+  # files in `/entities`.
+  class EntityUpdater < Updater
+    # !@attribute name
+    #   @return [String] the name of the entity to be updated
+    attr_accessor :name
+
+    # !@attribute entities_path
+    #   @return [String] the path to the directory containing the entities text files
+    attr_accessor :entities_path
+
+    # Initialize a new `EntityUpdater`.
+    #
+    # @see Updater#initialize
+    def initialize( * )
+      super
+    end
+
+    # Update the named entity on Api.ai.
+    #
+    # @return [Hash] if request successful. This is the response body.
+    # @return [ApiAiRuby::RequestError] if request is in error.
+    def update!
+      entity = [{ name: @name.to_s, entries: expanded_entries }]
+
+      response = @client.update_entities_request( entity )
+
+      handle_response( response, :entity )
+    end
+
+    private
+
+    # @return [Array<String>] The expanded list of entries and their synonyms.
+    def expanded_entries
+      sorted_entries.inject( [] ) do | entries, ( entry_value, synonyms ) |
+        entries << { value: entry_value, synonyms: [ entry_value ] + synonyms }
+      end
+    end
+
+    # @return [Hash] Properly sorted entries. Each key is the entry's name, and
+    #   the value is the list of synonyms for that entry.
+    def sorted_entries
+      sorted = Hash.new( [] )
+
+      expanded_entities.each do | line |
+        entry_value, *synonyms = *line.split(',').collect{ |s| s.strip }
+        sorted[ entry_value ] = sorted[ entry_value ] + synonyms
+      end
+
+      sorted
+    end
+
+    # @return [Array<String>] The expanded list of entities.
+    def expanded_entities
+      entity_file_path = File.join( @entities_path, @name.to_s + '.txt')
+      Expander.expand! file_lines( entity_file_path )
+    end
+  end
+end

data/lib/expando/expander.rb

@@ -0,0 +1,59 @@
+# The `Expander` module is responsible for taking an array of intents or utterances,
+# and expanding any strings containing the following tokens:
+#
+#     {I|we} heard you {love|hate} computers
+#
+# into the following Cartesian product:
+#
+#     I heard you love computers
+#     I heard you hate computers
+#     we heard you love computers
+#     we heard you hate computers
+#
+# This greatly reduces the complexity of creating performant speech interfaces.
+module Expando
+  module Expander
+    TOKEN_REGEX = /(?<!\\)\((.*?)\)/
+
+    module_function
+
+    # Generate a new `Expander`.
+    #
+    # @param [Array<String>] lines The text to scan and expand.
+    # @return [Array] The expanded text.
+    def expand!( lines )
+      expanded_lines = []
+
+      lines.each do |line|
+        tokens = line.scan TOKEN_REGEX
+
+        # Don't perform expansion if no tokens are present.
+        if tokens.empty?
+          expanded_lines << line
+          next
+        end
+
+        expanded_tokens = []
+        tokens.each_with_index do |token, index|
+          expanded_tokens[index] = token[0].split( '|' )
+        end
+
+        # Produce Cartesian product of all tokenized values.
+        token_product = expanded_tokens[ 0 ].product( *expanded_tokens[ 1..-1 ] )
+
+        # Generate new expanded lines.
+        token_product.each do |replacement_values|
+          expanded_line = line
+
+          replacement_values.each do |value|
+            expanded_line = expanded_line.sub( /\{(.*?)\}/, value )
+          end
+
+          expanded_lines << expanded_line
+        end
+      end
+
+      expanded_lines
+    end
+  end
+end

data/lib/expando/intent_updater.rb

@@ -0,0 +1,95 @@
+require 'json'
+
+module Expando
+  # Responsible for updating intent objects on Api.ai based on the contents of
+  # files in `/intents`.
+  class IntentUpdater < Updater
+    # !@attribute name
+    #   @return [String] the name of the intent to be updated
+    attr_accessor :name
+
+    # !@attribute intents_path
+    #   @return [String] the path to the directory containing the intent source files
+    attr_accessor :intents_path
+
+    # Initialize a new `IntentUpdater`.
+    #
+    # @see Updater#initialize
+    def initialize( * )
+      super
+    end
+
+    # Update the named entity on Api.ai.
+    #
+    # @return [Hash] if request successful. This is the response body.
+    # @return [ApiAiRuby::RequestError] if request is in error.
+    def update!
+      response = @client.update_intent_request( intent_json )
+
+      handle_response( response, :intent )
+    end
+
+    private
+
+    # Construct a proper JSON object for updating the intent, based on its current
+    #   state plus the expanded templates
+    #
+    # @return [Hash] The constructed JSON of for the intent.
+    def intent_json
+      @id = intent_id
+
+      log_message "Fetching latest version of #{ @name } intent"
+      json = @client.get_intent_request( @id )
+
+      json[ :userSays ] = expanded_utterances
+
+      # TODO: Make this a separate, tested method
+      responses = File.readlines File.join( @intents_path, '..', 'responses', @name.to_s + '.txt' )
+      responses = responses.collect { |response| response.chomp }
+
+      responsesJson = json[ :responses ]
+      responsesJson[ 0 ][ :speech ] = responses
+      json[:responses] = responsesJson
+
+      # Clean up portions of the JSON response that we don't need in the request
+      %w{templates state priority webhookUsed}.each { |key| json.delete( key.to_sym ) }
+
+      json
+    end
+
+    # Fetch the ID of the intent with this name on Api.ai.
+    #
+    # @return [String] The ID of the intent with this `@name` on Api.ai.
+    def intent_id
+      log_message "Fetching id of #{ @name } intent"
+      intents = @client.get_intents_request
+
+      matching_intent = intents.select { |intent| intent[ :name ] == @name.to_s }
+
+      if matching_intent.empty?
+        # TODO: Consult Exceptional Ruby for a better way to do this
+        raise "There is no intent named #{@name}"
+      else
+        return matching_intent.first[ :id ]
+      end
+    end
+
+    # @return [Array<String>] The expanded list of intent utterances.
+    def expanded_utterances
+      intent_utterance_file_path = File.join( @intents_path, @name.to_s + '.txt')
+      # TODO: Test
+      utterances = Expander.expand! file_lines( intent_utterance_file_path )
+
+      utterances.collect do |utterance|
+        {
+          data: [
+              text: utterance
+          ],
+          # TODO: Make this an aption
+          isTemplate: false
+        }
+      end
+    end
+  end
+end
+

data/lib/expando/updater.rb

@@ -0,0 +1,114 @@
+module Expando
+  class Updater
+    # The default location of intent source files
+    DEFAULT_INTENTS_PATH = File.join( Dir.getwd, '../../intents')
+
+    # The default location of entity source files
+    DEFAULT_ENTITIES_PATH = File.join( Dir.getwd, '../../entities' )
+
+    # Initialize a new `Updater`.
+    #
+    # @param [Symbol] name The name of the intent or entity to update. (default: `nil`)
+    # @param [String] entities_path The path to the directory containing the
+    #   entities text files. (default: `'entities'`)
+    # @param [String] intents_path The path to the directory containing the
+    #   intents source files. (default: 'intents')
+    # @param [Hash] client_keys A hash of Api.ai credentials.
+    # @option client_keys [String] :developer_access_token The Api.ai developer
+    #   access token.
+    # @option client_keys [String] :client_access_token The Api.ai client access
+    #   token.
+    # @return [Updater] The new `Updater`.
+    def initialize( name = nil, intents_path: DEFAULT_INTENTS_PATH, entities_path: DEFAULT_ENTITIES_PATH, client_keys: {})
+      @name = name
+      @intents_path = intents_path
+      @entities_path = entities_path
+
+      @client = ApiAiRuby::Client.new( credentials( client_keys ) )
+    end
+
+    private
+
+    # Generate a credentials hash for Api.ai from environment variables or passed
+    # arguments, whichever is provided.
+    #
+    # @param [Hash] client_keys A hash of Api.ai credentials.
+    # @option client_keys [String] :developer_access_token The Api.ai developer
+    #   access token.
+    # @option client_keys [String] :client_access_token The Api.ai client access
+    #   token.
+    # @return [Hash] The Api.ai client credentials.
+    def credentials(client_keys)
+      developer_access_token = ENV['API_AI_DEVELOPER_ACCESS_TOKEN'] || client_keys[:developer_access_token]
+      client_access_token = ENV['API_AI_CLIENT_ACCESS_TOKEN'] || client_keys[:client_access_token]
+
+      {
+          client_access_token: client_access_token,
+          developer_access_token: developer_access_token,
+      }
+    end
+
+    # Properly handle the response from Api.ai.
+    #
+    # @param [Hash] response The response from `ApiAiRuby::Client`.
+    # @param [Symbol] type Either `:intent` or `:entity`, depending on what is
+    #   being updated.
+    # @return [void]
+    def handle_response( response, type )
+      begin
+        if successful?( response )
+          log_completion_message(type )
+        else
+          puts failed_update_message(type )
+          ap response
+        end
+      rescue Exception => e
+        puts e.message
+        puts e.backtrace.inspect
+
+        abort( failed_update_message )
+      end
+    end
+
+    # Determine if the query was successful.
+    #
+    # @param [Hash] response The raw response from Api.ai
+    # @return [Boolean] `true` if successful, `false` otherwise.
+    def successful?( response )
+      response && response[ :status ] && ( response[ :status ][ :code ] == 200 )
+    end
+
+    # Generate a failed entity update message.
+    #
+    # @param [Symbol] type The type of update (`:entity` or `:intent`).
+    # @return [String] The failed update message.
+    def failed_update_message( type )
+      '• '.colorize( :blue ) + "#{ @name } #{ type.to_s } update failed:".colorize(:red )
+    end
+
+    # Output a log message.
+    #
+    # @param [String] The message.
+    # @return [void]
+    def log_message( message )
+      puts '• '.colorize( :blue ) + message
+    end
+
+    # Output a successful update message.
+    #
+    # @param [Symbol] type The type of update (`:entity` or `:intent`).
+    # @return [void]
+    def log_completion_message( type )
+      puts "• ".colorize( :blue ) + "#{ @name } #{ type.to_s } successfully updated!".colorize( :green )
+      puts "\nExpando:".colorize( :magenta ) + " Api.ai agent updated."
+    end
+
+    # Read a file into an array of strings.
+    #
+    # @param [String] file_path The path to the file to convert.
+    # @return [Array<String>] An array of all of the lines in the file.
+    def file_lines( file_path )
+      File.read( file_path ).lines.collect{ |line| line.chomp }
+    end
+  end
+end

data/lib/expando/version.rb

@@ -0,0 +1,3 @@
+module Expando
+  VERSION = '0.1.0'
+end

metadata

@@ -0,0 +1,192 @@
+--- !ruby/object:Gem::Specification
+name: expando
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Matt Buck
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2016-05-27 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.9'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.9'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.4.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.4.0
+- !ruby/object:Gem::Dependency
+  name: rspec_junit_formatter
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.2.2
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.2.2
+- !ruby/object:Gem::Dependency
+  name: climate_control
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.0.3
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.0.3
+- !ruby/object:Gem::Dependency
+  name: api-ai-ruby
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.1.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.1.0
+- !ruby/object:Gem::Dependency
+  name: gli
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.13.4
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.13.4
+- !ruby/object:Gem::Dependency
+  name: colorize
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.7.7
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.7.7
+- !ruby/object:Gem::Dependency
+  name: awesome_print
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.6.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.6.1
+description: A translation language for defining user utterance examples in conversational
+  interfaces.
+email:
+- matt@voxable.io
+executables:
+- console
+- expando
+- setup
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rspec"
+- ".ruby-gemset"
+- ".ruby-version"
+- Gemfile
+- README.md
+- Rakefile
+- bin/console
+- bin/expando
+- bin/setup
+- circle.yml
+- expando.gemspec
+- lib/expando.rb
+- lib/expando/entity_updater.rb
+- lib/expando/expander.rb
+- lib/expando/intent_updater.rb
+- lib/expando/updater.rb
+- lib/expando/version.rb
+homepage: http://voxable.io
+licenses: []
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.6
+signing_key: 
+specification_version: 4
+summary: The Expando reference implementation.
+test_files: []
+has_rdoc: