lita-markov 1.0.2 → 1.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 7b5c291ff7b19be493ce427ed3ef9d49eac2ba4d
-  data.tar.gz: bec9e862f464c5ed4daeeea15adf946bb22335b7
+  metadata.gz: 34357a93d6cd917570538738c72f1bc0d301b78b
+  data.tar.gz: 4fa6a56d9c25baf00d141bb9d593bc11bf3fc469
 SHA512:
-  metadata.gz: 5f82af9abdb8fcd6c6413e6f384cf884034d34b4be5a63a6e6f28357d9465a02b81a215447ea16196730aca584ab7bb229b4e8de03bcdcc4615ab9d94c9ab722
-  data.tar.gz: b7bc2ad646600648117a2013986d56a02ad803cba950e8e81167f429f688742c5fe08625ae53fa294e507c041f9bbb09500a30d9c8ad2f144bf423fd4f0d6ca8
+  metadata.gz: a139b7b1e4618b420addacdaf0732885d02defcbc0c741186c892efb85b7d9dba0ea230b6dffd544903cac599f257f73624158340cf7baf10cf1cb745f84189a
+  data.tar.gz: 7737301aa793730960cc142892586d64d6a8e5310760fe4ad868d792610d04240ba253f2bd69c24148c3c5be2cd842efa4d9bce7fbef8a698c111c3aac2c682a

data/lib/lita/handlers/markov.rb

@@ -1,10 +1,13 @@
 # Forward definition of Markov handler class
 class Lita::Handlers::Markov < Lita::Handler; end
 
+require 'oj'
 require 'lita/handlers/markov/engine'
 
 module Lita::Handlers
   class Markov
+    template_root File.expand_path('../../../../templates', __FILE__)
+
     config :database_url, type: String, required: true
     
     route(/.+/, :ingest, command: false)
@@ -13,8 +16,15 @@ module Lita::Handlers
       'markov USER' => 'Generate a markov chain from the given user.'
     })
 
+    http.get  '/markov/backlog',        :backlog_form
+    http.post '/markov/upload_backlog', :upload_backlog
+
+    # Share the engine instance between all instances of the bot
+    def self.engine(instance)
+      @engine ||= Engine.new(instance.config.database_url)
+    end
     def engine
-      @engine ||= Engine.new config.database_url
+      self.class.engine(self)
     end
 
     def ingest(chat)
@@ -47,22 +57,67 @@ module Lita::Handlers
       end
     end
 
-    def save_dictionary(name, dictionary)
-      redis.set key_for_user(name), dictionary.to_json
+    def backlog_form(request, response)
+      render_backlog_form response
     end
 
-    def dictionary_for_user(name)
-      key        = key_for_user name
-      dictionary = Dictionary.new name
-      json       = redis.get key
+    def upload_backlog(request, response)
+      t0 = Time.now
+
+      response.headers['Content-Type'] = 'text/plain'
+
+      multipart = Rack::Multipart.parse_multipart request.env
+      tempfile  = multipart.values.first[:tempfile]
+
+      begin
+        messages = Oj.load File.read(tempfile.path).strip, :mode => :strict
+      rescue Oj::ParseError => error
+        response.write error.message
+        return
+      end
+
+      messages.select! { |m| m['type'] == 'message' }
 
-      dictionary.load_json(json) if json
+      users = {}
+      find_user = proc do |id|
+        users[id] ||= Lita::User.fuzzy_find id
+      end
+
+      meta_tag_regex = /<(\w|[!|@])+>/
+
+      count = 0
+      messages.each do |message|
+        count += 1
+
+        begin
+          text = message['text'.freeze]
+          next unless text
+
+          user = find_user.call message['user'.freeze]
+          unless user
+            response.write "User not found for message ##{count}: #{message['user']}\n"
+            next
+          end
 
-      dictionary
+          message = text.gsub meta_tag_regex, ''.freeze
+
+          engine.ingest user.id, message
+
+          if count % 1000 == 0
+            response.write "Processed #{count} messages\n"
+          end
+        rescue => error
+          response.write "Error writing message ##{count}: #{error.inspect}\n"
+        end
+      end
+
+      response.write "Processed #{count} total messages in #{Time.now - t0} seconds\n"
     end
 
-    def key_for_user(name)
-      REDIS_KEY_PREFIX+name.downcase
+    private
+
+    def render_backlog_form(response)
+      response.write render_template('backlog_form')
     end
 
     Lita.register_handler self

data/lib/lita/handlers/markov/engine.rb

@@ -146,35 +146,42 @@ class Lita::Handlers::Markov
       chain
     end
 
+    STRING_SEPARATOR = /([.!?])|\s+/
+
     def separate_string string
       # Including the punctuation in group so they'll be included in the
       # split results
       string
-        .split(/([.!?])|\s+/)
-        .map { |w| w.strip }
+        .split(STRING_SEPARATOR)
+        .map { |w| w.strip!; w }
         .select { |w| !w.empty? }
     end
 
-    PUNCTUATION = ['.', '!', '?']
+    PUNCTUATION = [',', '.', '!', '?']
 
     # Don't allow anything besides letters, digits, whitespace, and puncutation
     ILLEGAL_CHARACTERS = /[^\w\d\s:;,.!?#@]/
 
+    HYPERLINKS          = /http[^\s]+/
     SIMPLE_CODE_BLOCK   = /`[^`]+`/
     EXTENDED_CODE_BLOCK = /```.+```/m
 
+    REPEATED_PUNCTUATION = /([.!?])[.!?]+/
+    BASIC_PUNCTUATION    = /([;,.!?])/
+
+
     def sanitize_string string
       string = string
-        .strip()
-        .gsub(/http[^\s]+/, '')        # Remove any hyperlinks
-        .gsub(SIMPLE_CODE_BLOCK, '')   # Remove code blocks and illegal characters
-        .gsub(EXTENDED_CODE_BLOCK, '')
-        .gsub(ILLEGAL_CHARACTERS, '')
-        .gsub(/([:;,.!?])/, '\1 ')     # Put whitespace after punctuation for proper separation
+        .gsub(HYPERLINKS, ''.freeze)             # Remove any hyperlinks
+        .gsub(SIMPLE_CODE_BLOCK, ''.freeze)      # Remove code blocks and illegal characters
+        .gsub(EXTENDED_CODE_BLOCK, ''.freeze)
+        .gsub(ILLEGAL_CHARACTERS, ''.freeze)
+        .gsub(REPEATED_PUNCTUATION, '\1'.freeze) # Trim down repeated punctuation
+        .gsub(BASIC_PUNCTUATION, '\1 '.freeze)   # Put whitespace after punctuation for proper separation
         .strip()
 
       ends_with_punctuation = PUNCTUATION.any? { |p| string.end_with? p }
-      string = string+'.' unless ends_with_punctuation
+      string = string+'.'.freeze unless ends_with_punctuation
 
       string
     end

data/lita-markov.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |spec|
   spec.name          = "lita-markov"
-  spec.version       = "1.0.2"
+  spec.version       = "1.1.0"
   spec.authors       = ["Dirk Gadsden"]
   spec.email         = ["dirk@dirk.to"]
   spec.description   = "Markov chains for Lita."
@@ -17,6 +17,7 @@ Gem::Specification.new do |spec|
   spec.add_runtime_dependency "sequel",       "~> 4.28.0"
   spec.add_runtime_dependency "mysql2",       "~> 0.4.1"
   spec.add_runtime_dependency "pg",           "~> 0.18.4"
+  spec.add_runtime_dependency "oj",           "~> 2.13.1"
 
   spec.add_development_dependency "bundler", "~> 1.3"
   spec.add_development_dependency "pry-byebug"

data/templates/backlog_form.erb

@@ -0,0 +1,8 @@
+<html>
+  <body>
+    <form action="/markov/upload_backlog" method="post" enctype="multipart/form-data">
+      <h3>Select Slack JSON archive to upload:</h3>
+      <p><input type="file" name="backlog" /></p>
+      <p><input type="submit" /></p>
+    </form>
+</html>

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: lita-markov
 version: !ruby/object:Gem::Version
-  version: 1.0.2
+  version: 1.1.0
 platform: ruby
 authors:
 - Dirk Gadsden
@@ -66,6 +66,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 0.18.4
+- !ruby/object:Gem::Dependency
+  name: oj
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.13.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.13.1
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -157,6 +171,7 @@ files:
 - spec/lita/handlers/markov_spec.rb
 - spec/spec_helper.rb
 - templates/.gitkeep
+- templates/backlog_form.erb
 homepage: http://github.com/dirk/lita-markov
 licenses: []
 metadata:
@@ -177,7 +192,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.5
+rubygems_version: 2.4.5.1
 signing_key: 
 specification_version: 4
 summary: Markov chains for Lita.