lita-markov 0.0.1 → 1.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 80906c8495bb1dcbbc33b970047390e606b429c8
-  data.tar.gz: 85f84a17b45289e6c86334318322eebba6724b8f
+  metadata.gz: 627ba42fa0054e9a6cd2d46bbde5cb6510f1b985
+  data.tar.gz: c53c2b0f565930850bd0a60085d6d02c69128eee
 SHA512:
-  metadata.gz: f7c18adeba628b7e6cc48a22922fd745595104ce6846c919b773ab475e7c22b4d2e852d747055adadaec74b0dc881cab36254b305ab4c2df179c6a77e1b118e3
-  data.tar.gz: e4fce6fb0f3d70d04d2153fb202d1740c223abadc475a18a6629ba5080f211219dc58a0817bd36d58d6529801b8da7a2eaeb700585b7bb69d7ff1c20daf45145
+  metadata.gz: b634fa615bb4960166da153db753ea5c8f05f9dece26a856626fbdd8b73be6ee887ce559a55fdfa17d215085e9d61b9e62c40d65a89630ef813fba33a756c7d5
+  data.tar.gz: 885beb293a6f002273be56e374700e56d2739bd451076ccffc5114d7a261cc9aee76c4e6fabb379dc564d62301fe9ab7c0a15d9c9cf6cf3659ba5fe0c1ea682f

data/.gitignore

@@ -1,2 +1,3 @@
+.DS_Store
 Gemfile.lock
 tmp

data/README.md

@@ -11,6 +11,17 @@ Add `lita-markov` to your Lita instance's Gemfile:
 gem 'lita-markov'
 ```
 
+Configure the database URL for your SQL database
+([Sequel](http://sequel.jeremyevans.net/) is used for
+communicating with databases):
+
+```ruby
+# lita_config.rb
+Lita.configure do |config|
+  config.handlers.markov.database_url = ENV['DATABASE_URL']
+end
+```
+
 ## Usage
 
 The bot will automatically ingest all messages into the Redis-backed Markov

data/lib/lita/handlers/markov/engine.rb

@@ -0,0 +1,172 @@
+require 'sequel'
+
+class Lita::Handlers::Markov
+  class Engine
+    class EmptyDictionaryError < StandardError; end
+
+    # Default development database URL
+    DEFAULT_DATABASE_URL = 'mysql2://root@localhost/lita-markov'
+
+    attr_accessor :handler
+    attr_reader :db
+
+    def initialize(handler = nil)
+      @handler = handler
+      @depth   = 2
+
+      database_url = DEFAULT_DATABASE_URL
+      database_url ||= handler.config.database_url if handler
+
+      @db = Sequel.connect database_url
+
+      @db.create_table?(:dictionary) do
+        column :user,          String,  null: false # The user the states are associated with
+        column :current_state, String,  null: false # Word(s) the user has "said"
+        column :next_state,    String,  null: false # Word that follows that word
+        column :frequency,     Integer, null: false # Frequency that the next word follows the current state/word
+
+        primary_key [:user, :current_state, :next_state]
+      end
+    end
+
+    # user   - Username of the user
+    # string - String of words that the user has just said (ideally a sentence)
+    def ingest user, string
+      string = sanitize_string string
+      words  = separate_string string
+
+      return if words.length == 0
+
+      # Capitalize the first word
+      words = [words[0].capitalize] + words.slice(1..-1)
+
+      # Iterate over it one step at a time in sets of `@depth + 1`
+      words.each_cons(@depth + 1) do |words|
+        current_state = words[0]+' '+words[1]
+        next_state    = words[2]
+
+        add_entry user, current_state, next_state
+      end # words.each_cons
+    end # def ingest
+
+    def add_entry user, current_state, next_state
+      dictionary = @db[:dictionary]
+
+      @db.transaction do
+        entry = {
+          user: user,
+          current_state: current_state,
+          next_state: next_state
+        }
+
+        if dictionary.where(entry).any?
+          # Entry is already present, so increment its frequency
+          frequency = dictionary.where(entry).get(:frequency)
+
+          dictionary.where(entry).update frequency: frequency + 1
+        else
+          dictionary.insert entry.merge(frequency: 1)
+        end
+      end
+    end
+
+    def random_capitalized_word
+      states = @db[:dictionary].map(:current_state)
+
+      capitalized_states = states.select do |state|
+        /^[A-Z]/ =~ state
+      end
+
+      if capitalized_states.length > 0
+        state = capitalized_states.sample
+      else
+        state = states.sample
+      end
+
+      raise EmptyDictionaryError, 'No data for user' if state.nil?
+
+      return state.split(' ').first
+    end
+
+    def random_second_word(first_word)
+      states = @db[:dictionary]
+        .where(Sequel.like(:current_state, first_word+'%'))
+        .map(:current_state)
+
+      state = states.sample
+      state.split(' ').last
+    end
+
+    def is_punctuation?(string)
+      PUNCTUATION.any? { |p| string.end_with? p }
+    end
+
+    def get_next_state(user, current_state)
+      states = @db[:dictionary]
+        .where(user: user, current_state: current_state)
+        .select(:next_state, :frequency)
+        .all
+
+      distribution = states.flat_map do |state|
+        Array.new(state[:frequency]) { state[:next_state] }
+      end
+
+      distribution.sample
+    end
+
+    def generate_sentence_for(user, length = 30)
+      first_word = random_capitalized_word
+      second_word = random_second_word first_word
+
+      sentence = [first_word, second_word]
+
+      while sentence.length < length
+        current_state = sentence.slice(sentence.length - @depth, @depth).join ' '
+
+        next_state = get_next_state user, current_state
+
+        # Stop if we failed to find a next state
+        break if next_state.nil?
+
+        sentence << next_state
+
+        break if is_punctuation? next_state
+      end
+
+      sentence.slice(0..-2).join(' ') + sentence.last
+    end
+
+    def separate_string string
+      # Including the punctuation in group so they'll be included in the
+      # split results
+      string
+        .split(/([.!?])|\s+/)
+        .map { |w| w.strip }
+        .select { |w| !w.empty? }
+    end
+
+    PUNCTUATION = ['.', '!', '?']
+
+    # Don't allow anything besides letters, digits, whitespace, and puncutation
+    ILLEGAL_CHARACTERS = /[^\w\d\s:;,.!?#@]/
+
+    SIMPLE_CODE_BLOCK   = /`[^`]+`/
+    EXTENDED_CODE_BLOCK = /```.+```/m
+
+    def sanitize_string string
+      string = string
+        .strip()
+        .gsub(/http[^\s]+/, '')        # Remove any hyperlinks
+        .gsub(SIMPLE_CODE_BLOCK, '')   # Remove code blocks and illegal characters
+        .gsub(EXTENDED_CODE_BLOCK, '')
+        .gsub(ILLEGAL_CHARACTERS, '')
+        .gsub(/([:;,.!?])/, '\1 ')     # Put whitespace after punctuation for proper separation
+        .strip()
+
+      ends_with_punctuation = PUNCTUATION.any? { |p| string.end_with? p }
+      string = string+'.' unless ends_with_punctuation
+
+      string
+    end
+  end
+end

data/lib/lita/handlers/markov.rb

@@ -1,8 +1,13 @@
+# Forward definition of Markov handler class
+class Lita::Handlers::Markov < Lita::Handler; end
+
+require 'lita/handlers/markov/engine'
+
 module Lita::Handlers
-  class Markov < Lita::Handler
-    Dictionary = MarkyMarkov::PersistentJSONDictionary
+  class Markov
+    attr_reader :engine
 
-    REDIS_KEY_PREFIX = 'lita-markov:'
+    config :database_url
     
     route(/.+/, :ingest, command: false)
 
@@ -10,6 +15,12 @@ module Lita::Handlers
       'markov USER' => 'Generate a markov chain from the given user.'
     })
 
+    def initialize(robot)
+      super(robot)
+
+      @engine = Engine.new self
+    end
+
     def ingest(chat)
       # Don't ingest messages addressed to ourselves
       return if chat.command?
@@ -17,26 +28,20 @@ module Lita::Handlers
       message = chat.matches[0].strip
 
       # Get the mention name (ie. 'dirk') of the user
-      name       = chat.user.mention_name
-      dictionary = dictionary_for_user name
-
-      # Passing `false` to indicate it's a string and not a file name
-      dictionary.parse_source message, false
+      id = chat.user.id
 
-      save_dictionary name, dictionary
+      @engine.ingest id, message
     end
 
     def generate(chat)
       name = chat.matches[0][0].strip
-
-      dictionary = dictionary_for_user name
-      generator  = MarkovSentenceGenerator.new dictionary
+      id   = Lita::User.fuzzy_find(name).id
 
       begin
-        sentence = generator.generate_sentence 1
+        sentence = @engine.generate_sentence_for id
 
         chat.reply sentence
-      rescue EmptyDictionaryError
+      rescue Engine::EmptyDictionaryError
         chat.reply "Looks like #{name} hasn't said anything!"
       end
     end

data/lib/lita-markov.rb

@@ -4,7 +4,6 @@ Lita.load_locales Dir[File.expand_path(
   File.join("..", "..", "locales", "*.yml"), __FILE__
 )]
 
-require 'marky_markov/persistent_json_dictionary'
 require 'lita/handlers/markov'
 
 # Lita::Handlers::Markov.template_root File.expand_path(

data/lita-markov.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |spec|
   spec.name          = "lita-markov"
-  spec.version       = "0.0.1"
+  spec.version       = "1.0.0"
   spec.authors       = ["Dirk Gadsden"]
   spec.email         = ["dirk@dirk.to"]
   spec.description   = "Markov chains for Lita."
@@ -13,9 +13,10 @@ Gem::Specification.new do |spec|
   spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
   spec.require_paths = ["lib"]
 
-  spec.add_runtime_dependency "lita", ">= 4.6"
-  spec.add_runtime_dependency "marky_markov", "~> 0.3.5"
-  spec.add_runtime_dependency "oj", "~> 2.13.1"
+  spec.add_runtime_dependency "lita",         ">= 4.6"
+  spec.add_runtime_dependency "sequel",       "~> 4.28.0"
+  spec.add_runtime_dependency "mysql2",       "~> 0.4.1"
+  spec.add_runtime_dependency "pg",           "~> 0.18.4"
 
   spec.add_development_dependency "bundler", "~> 1.3"
   spec.add_development_dependency "pry-byebug"

data/spec/lita/handlers/markov/engine_spec.rb

@@ -0,0 +1,54 @@
+require 'spec_helper'
+require 'pry'
+
+describe Lita::Handlers::Markov::Engine do
+  before(:each) do
+    subject.db[:dictionary].delete
+  end
+
+  it 'will sanitize links from a message' do
+    message = 'hello https://www.example.com world!'
+
+    expect(subject.sanitize_string(message)).to eql 'hello  world!'
+  end
+
+  it 'will remove code blocks from a message' do
+    message = 'I have `code in` me.'
+
+    expect(subject.sanitize_string(message)).to eql 'I have  me.'
+  end
+
+  it 'will remove illegal characters from a message' do
+    message = 'I have a bad % character.'
+
+    expect(subject.sanitize_string(message)).to eql 'I have a bad  character.'
+  end
+
+  it 'will separate a string into words' do
+    string = "I am\n so  totally\tseparated."
+
+    expect(subject.separate_string(string)).to eql ['I', 'am', 'so', 'totally', 'separated', '.']
+  end
+
+  it 'will ingest messages' do
+    dictionary = subject.db[:dictionary]
+
+    subject.ingest('user', 'hello big, fun world!')
+
+    # Check that the first state made it in and is capitalized
+    expect(dictionary.where(current_state: 'Hello big,').count).to eql 1
+    # Check that the last state made it in
+    expect(dictionary.where(current_state: 'fun world', next_state: '!').count).to eql 1
+
+    subject.ingest('user', 'Hello big, fun planet!')
+
+    # Check that the frequency of the "Hello big," -> "fun" state went up
+    expect(dictionary.where(current_state: 'Hello big,', next_state: 'fun').get(:frequency)).to eql 2
+  end
+
+  it 'will generate a sentence' do
+    subject.ingest('user', 'Hello cruel world.')
+
+    expect(subject.generate_sentence_for 'user').to eql 'Hello cruel world.'
+  end
+end

data/spec/lita/handlers/markov_spec.rb

@@ -3,7 +3,7 @@ require 'pry'
 
 describe Lita::Handlers::Markov, lita_handler: true do
   before(:each) do
-    Lita.redis.flushall
+    subject.engine.db[:dictionary].delete
   end
 
   it "won't call #ingest for non-command messages" do
@@ -13,16 +13,6 @@ describe Lita::Handlers::Markov, lita_handler: true do
     send_command 'bar'
   end
 
-  it "will ingest a message into that person's dictionary" do
-    send_message 'hello markov world'
-    send_message 'hello markov planet'
-
-    dictionary = subject.dictionary_for_user user.mention_name
-
-    # Check that the messages made it into the dictionary
-    expect(dictionary.dictionary[['hello', 'markov']]).to eql ['world', 'planet']
-  end
-
   it 'will build a sentence' do
     send_message 'I love cookies!'
     send_message 'I love pancakes!'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: lita-markov
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 1.0.0
 platform: ruby
 authors:
 - Dirk Gadsden
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-11-23 00:00:00.000000000 Z
+date: 2015-11-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: lita
@@ -25,33 +25,47 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '4.6'
 - !ruby/object:Gem::Dependency
-  name: marky_markov
+  name: sequel
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.3.5
+        version: 4.28.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.3.5
+        version: 4.28.0
 - !ruby/object:Gem::Dependency
-  name: oj
+  name: mysql2
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 2.13.1
+        version: 0.4.1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 2.13.1
+        version: 0.4.1
+- !ruby/object:Gem::Dependency
+  name: pg
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.18.4
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.18.4
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -136,11 +150,11 @@ files:
 - Rakefile
 - lib/lita-markov.rb
 - lib/lita/handlers/markov.rb
-- lib/marky_markov/persistent_json_dictionary.rb
+- lib/lita/handlers/markov/engine.rb
 - lita-markov.gemspec
 - locales/en.yml
+- spec/lita/handlers/markov/engine_spec.rb
 - spec/lita/handlers/markov_spec.rb
-- spec/marky_markov/persistent_json_dictionary_spec.rb
 - spec/spec_helper.rb
 - templates/.gitkeep
 homepage: http://github.com/dirk/lita-markov
@@ -163,11 +177,11 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.5.1
+rubygems_version: 2.4.5
 signing_key: 
 specification_version: 4
 summary: Markov chains for Lita.
 test_files:
+- spec/lita/handlers/markov/engine_spec.rb
 - spec/lita/handlers/markov_spec.rb
-- spec/marky_markov/persistent_json_dictionary_spec.rb
 - spec/spec_helper.rb

data/lib/marky_markov/persistent_json_dictionary.rb

@@ -1,34 +0,0 @@
-require 'marky_markov'
-require 'oj'
-
-module MarkyMarkov
-  class PersistentJSONDictionary < ::PersistentDictionary
-    def initialize(*args)
-      super(*args)
-
-      @dictionary        = {}
-      @capitalized_words = []
-    end
-
-    # No-op instead of reading from the filesystem
-    def open_dictionary
-      nil
-    end
-
-    def load_json(json)
-      data = Oj.load json
-
-      @depth             = data['depth']
-      @dictionary        = data['dictionary']
-      @capitalized_words = data['capitalized_words']
-    end
-
-    def to_json
-      Oj.dump(
-        'depth'             => @depth,
-        'dictionary'        => @dictionary,
-        'capitalized_words' => @capitalized_words
-      )
-    end
-  end
-end

data/spec/marky_markov/persistent_json_dictionary_spec.rb

@@ -1,28 +0,0 @@
-require 'spec_helper'
-
-describe MarkyMarkov::PersistentJSONDictionary do
-  subject { MarkyMarkov::PersistentJSONDictionary.new 'whoa' }
-  
-  it 'initializes with sensible defaults' do
-    expect(subject.dictionary).to eq({})
-  end
-
-  it 'saves a dictionary' do
-    subject.add_word ['a', 'b'], 'c'
-
-    json = subject.to_json
-
-    expect(json).to eql '{"depth":2,"dictionary":{"^#1":[["a","b"],["c"]]},"capitalized_words":[]}'
-  end
-
-  it 'saves and loads a dictionary' do
-    subject.add_word ['a', 'b'], 'c'
-
-    json = subject.to_json
-
-    new_dictionary = MarkyMarkov::PersistentJSONDictionary.new 'whoa-another-one'
-    new_dictionary.load_json json
-
-    expect(new_dictionary.dictionary).to eql(['a', 'b'] => ['c'])
-  end
-end