moo_ebooks 1.0.0

data/lib/moo_ebooks.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+module Ebooks
+  GEM_PATH = File.expand_path(File.join(File.dirname(__FILE__), '..'))
+  DATA_PATH = File.join(GEM_PATH, 'data')
+  SKELETON_PATH = File.join(GEM_PATH, 'skeleton')
+  TEST_PATH = File.join(GEM_PATH, 'test')
+  TEST_CORPUS_PATH = File.join(TEST_PATH, 'corpus/0xabad1dea.tweets')
+  INTERIM = :interim
+end
+
+require 'moo_ebooks/nlp'
+require 'moo_ebooks/suffix'
+require 'moo_ebooks/model'

data/lib/moo_ebooks/model.rb

@@ -0,0 +1,270 @@
+# frozen_string_literal: true
+
+require 'json'
+require 'set'
+require 'digest/md5'
+
+module Ebooks
+  # Main class for Model management. Models are required for text generation.
+  #
+  # @notice Only JSON format is supported.
+  # @notice For corpus files. These are assumed to have a `statuses` key and a
+  # `mentions` key, which hold the different statuses in them.
+  #
+  # @notice Make sure NOT to include reblogs (retweets) into corpus data. Those
+  # will negatively impact text creation
+  class Model
+    # @return [Array<String>]
+    # An array of unique tokens. This is the main source of actual strings
+    # in the model. Manipulation of a token is done using its index
+    # in this array, which we call a "tiki"
+    attr_accessor :tokens
+
+    # @return [Array<Array<Integer>>]
+    # Sentences represented by arrays of tikis
+    attr_accessor :sentences
+
+    # @return [Array<Array<Integer>>]
+    # Sentences derived from Twitter mentions
+    attr_accessor :mentions
+
+    # @return [Array<String>]
+    # The top 200 most important keywords, in descending order
+    attr_accessor :keywords
+    def initialize
+      @tokens = []
+      @sentences = []
+      @mentions = []
+      @keywords = []
+
+      # Reverse lookup tiki by token, for faster generation
+      @tikis = {}
+    end
+
+    # Load a saved model
+    # @param data [Hash]
+    # @return [Ebooks::Model]
+    def self.from_hash(data)
+      model = Model.new
+      model.tokens = data[:tokens]
+      model.sentences = data[:sentences]
+      model.mentions = data[:mentions]
+      model.keywords = data[:keywords]
+      model
+    end
+
+    # Load a saved model
+    # @param data [String]
+    # @reutrn [Ebooks::Model]
+    def self.from_json(data)
+      from_hash(JSON.parse(data, symbolize_names: true))
+    end
+
+    # Turn this model into its JSON representation.
+    def to_json
+      to_hash.to_json
+    end
+
+    # Turn this model into its Hash representation
+    def to_hash
+      { tokens: @tokens, sentences: @sentences, mentions: @mentions,
+        keywords: @keywords }
+    end
+
+    # Consume a corpus into this model
+    # @param content [Hash]
+    def consume(content)
+      model = Ebooks::Model.new
+      model.consume!(content)
+      model
+    end
+
+    # Consume a corpus into this model
+    # @param content [Hash]
+    def consume!(content)
+      unless content.key?(:statuses) || content.key?(:mentions)
+        raise ArgumentError, 'Malformed hash object. At least :statuses and/or'\
+                             ' :mentions must be present as a key'
+      end
+      consume_statuses(content[:statuses]) unless content[:statuses].nil?
+      consume_mentions(content[:mentions]) unless content[:mentions].nil?
+      nil
+    end
+
+    # Generate some text
+    # @param limit [Integer] available characters
+    # @param generator [SuffixGenerator, nil]
+    # @param retry_limit [Integer] how many times to retry on invalid status
+    # @return [String]
+    def update(limit = 140, generator = nil, retry_limit = 10)
+      tikis = gather_tikis(limit, generator, retry_limit)
+
+      status = NLP.reconstruct(tikis, @tokens)
+
+      fix status
+    end
+
+    # Generates a response by looking for related sentences
+    # in the corpus and building a smaller generator from these
+    # @param input [String]
+    # @param limit [Integer] characters available for response
+    # @param sentences [Array<Array<Integer>>]
+    # @return [String]
+    def reply(input, limit = 140, sentences = @mentions)
+      # Prefer mentions
+      relevant, slightly_relevant = find_relevant(sentences, input)
+
+      if relevant.length >= 3
+        generator = SuffixGenerator.build(relevant)
+        update(limit, generator)
+      elsif slightly_relevant.length >= 5
+        generator = SuffixGenerator.build(slightly_relevant)
+        update(limit, generator)
+      else
+        update(limit)
+      end
+    end
+
+    private
+
+    def gather_tikis(limit, generator, retry_limit)
+      responding = !generator.nil?
+      generator ||= SuffixGenerator.build(@sentences)
+
+      @retries = 0
+
+      tikis = make_bigram_tikis(limit, generator, retry_limit, responding)
+
+      if verbatim?(tikis) && tikis.length > 3
+        # We made a verbatim status by accident
+        tikis = make_unigram_tikis(limit, generator, retry_limit)
+      end
+      @retries = nil
+      tikis
+    end
+
+    def make_unigram_tikis(limit, generator, retry_limit)
+      while (tikis = generator.generate(3, :unigrams))
+        break if valid_status?(tikis, limit) && !verbatim?(tikis)
+
+        @retries += 1
+        break if retry_limit_reached?(retry_limit)
+      end
+      tikis
+    end
+
+    def make_bigram_tikis(limit, generator, retry_limit, responding)
+      while (tikis = generator.generate(3, :bigrams))
+        break if (tikis.length > 3 || responding) && valid_status?(tikis, limit)
+
+        @retries += 1
+        break if retry_limit_reached?(retry_limit)
+      end
+      tikis
+    end
+
+    def retry_limit_reached?(retry_limit)
+      @retries >= retry_limit
+    end
+
+    # Reverse lookup a token index from a token
+    # @param token [String]
+    # @return [Integer]
+    def tikify(token)
+      if @tikis.key?(token)
+        @tikis[token]
+      else
+        @tokens << token
+        @tikis[token] = @tokens.length - 1
+      end
+    end
+
+    # Convert a body of text into arrays of tikis
+    # @param text [String]
+    # @return [Array<Array<Integer>>]
+    def mass_tikify(text)
+      sentences = NLP.sentences(text)
+
+      sentences.map do |s|
+        tokens = NLP.tokenize(s).reject do |t|
+          # Don't include usernames/urls as tokens
+          t.include?('@') || t.include?('http')
+        end
+
+        tokens.map { |t| tikify(t) }
+      end
+    end
+
+    # Test if a sentence has been copied verbatim from original
+    # @param tikis [Array<Integer>]
+    # @return [Boolean]
+    def verbatim?(tikis)
+      @sentences.include?(tikis) || @mentions.include?(tikis)
+    end
+
+    # Check if an array of tikis comprises a valid status
+    # @param tikis [Array<Integer>]
+    # @param limit Integer how many chars we have left
+    def valid_status?(tikis, limit)
+      status = NLP.reconstruct(tikis, @tokens)
+      status.length <= limit && !NLP.unmatched_enclosers?(status)
+    end
+
+    # Consume a sequence of statuses (excluding mentions)
+    # @param statuses [Array<String>]
+    def consume_statuses(statuses)
+      statuses.map! do |status|
+        NLP.normalize(status)
+      end
+
+      text = statuses.join("\n").encode('UTF-8', invalid: :replace)
+      @sentences = mass_tikify(text)
+      @keywords = NLP.keywords(text).top(200).map(&:to_s)
+
+      nil
+    end
+
+    # Consume a sequence of mentions
+    # @param mentions [Array<String>]
+    def consume_mentions(mentions)
+      mentions.map! do |mention|
+        NLP.normalize(mention)
+      end
+
+      mention_text = mentions.join("\n").encode('UTF-8', invalid: :replace)
+      @mentions = mass_tikify(mention_text)
+
+      nil
+    end
+
+    # Correct encoding issues in generated text
+    # @param text [String]
+    # @return [String]
+    def fix(text)
+      NLP.htmlentities.decode text
+    end
+
+    # Finds relevant and slightly relevant tokenized sentences to input
+    # comparing non-stopword token overlaps
+    # @param sentences [Array<Array<Integer>>]
+    # @param input [String]
+    # @return [Array<Array<Array<Integer>>, Array<Array<Integer>>>]
+    def find_relevant(sentences, input)
+      relevant = []
+      slightly_relevant = []
+
+      tokenized = NLP.tokenize(input).map(&:downcase)
+
+      sentences.each do |sent|
+        tokenized.each do |token|
+          if sent.map { |tiki| @tokens[tiki].downcase }.include?(token)
+            relevant << sent unless NLP.stopword?(token)
+            slightly_relevant << sent
+          end
+        end
+      end
+
+      [relevant, slightly_relevant]
+    end
+  end
+end

data/lib/moo_ebooks/nlp.rb

@@ -0,0 +1,170 @@
+
+# frozen_string_literal: true
+
+require 'highscore'
+require 'htmlentities'
+
+module Ebooks
+  # @private
+  module NLP
+    # We deliberately limit our punctuation handling to stuff we can do
+    # consistently
+    # It'll just be a part of another token if we don't split it out, and
+    # that's fine
+    PUNCTUATION = '.?!,'
+
+    # Lazy-load NLP libraries and resources
+    # Some of this stuff is pretty heavy and we don't necessarily need
+    # to be using it all of the time
+
+    # Lazily loads an array of stopwords
+    # Stopwords are common words that should often be ignored
+    # @return [Array<String>]
+    def self.stopwords
+      @stopwords ||= File.read(File.join(DATA_PATH, 'stopwords.txt')).split
+    end
+
+    # Lazily load HTML entity decoder
+    # @return [HTMLEntities]
+    def self.htmlentities
+      @htmlentities ||= HTMLEntities.new
+    end
+
+    ### Utility functions
+
+    # Normalize some strange unicode punctuation variants
+    # @param text [String]
+    # @return [String]
+    def self.normalize(text)
+      htmlentities.decode(text.tr('“', '"').tr('”', '"').tr('’', "'")
+        .gsub('…', '...'))
+    end
+
+    # Split text into sentences
+    # We use ad hoc approach because fancy libraries do not deal
+    # especially well with tweet formatting, and we can fake solving
+    # the quote problem during generation
+    # @param text [String]
+    # @return [Array<String>]
+    def self.sentences(text)
+      text.split(/\n+|(?<=[.?!])\s+/)
+    end
+
+    # Split a sentence into word-level tokens
+    # As above, this is ad hoc because tokenization libraries
+    # do not behave well wrt. things like emoticons and timestamps
+    # @param sentence [String]
+    # @return [Array<String>]
+    def self.tokenize(sentence)
+      regex = /\s+|(?<=[#{PUNCTUATION}]\s)(?=[a-zA-Z])|
+        (?<=[a-zA-Z])(?=[#{PUNCTUATION}]+\s)/x
+      sentence.split(regex)
+    end
+
+    # Use highscore gem to find interesting keywords in a corpus
+    # @param text [String]
+    # @return [Highscore::Keywords]
+    def self.keywords(text)
+      # Preprocess to remove stopwords (highscore's blacklist is v. slow)
+      text = NLP.tokenize(text).reject { |t| stopword?(t) }.join(' ')
+
+      text = Highscore::Content.new(text)
+
+      text.configure do
+        # set :multiplier, 2
+        # set :upper_case, 3
+        # set :long_words, 2
+        # set :long_words_threshold, 15
+        # set :vowels, 1                     # => default: 0 = not considered
+        # set :consonants, 5                 # => default: 0 = not considered
+        # set :ignore_case, true             # => default: false
+        set :word_pattern, /(?<!@)(?<=\s)[\p{Word}']+/ # => default: /\w+/
+        # set :stemming, true                # => default: false
+      end
+
+      text.keywords
+    end
+
+    # Builds a proper sentence from a list of tikis
+    # @param tikis [Array<Integer>]
+    # @param tokens [Array<String>]
+    # @return [String]
+    def self.reconstruct(tikis, tokens)
+      text = ''
+      last_token = nil
+      tikis.each do |tiki|
+        next if tiki == INTERIM
+        token = tokens[tiki]
+        text += ' ' if last_token && space_between?(last_token, token)
+        text += token
+        last_token = token
+      end
+      text
+    end
+
+    # Determine if we need to insert a space between two tokens
+    # @param token1 [String]
+    # @param token2 [String]
+    # @return [Boolean]
+    def self.space_between?(token1, token2)
+      p1 = punctuation?(token1)
+      p2 = punctuation?(token2)
+      if (p1 && p2) || (!p1 && p2) # "foo?!" || "foo."
+        false
+      else # "foo rah" || "foo. rah"
+        true
+      end
+    end
+
+    # Is this token comprised of punctuation?
+    # @param token [String]
+    # @return [Boolean]
+    def self.punctuation?(token)
+      (token.chars.to_set - PUNCTUATION.chars.to_set).empty?
+    end
+
+    # Is this token a stopword?
+    # @param token [String]
+    # @return [Boolean]
+    def self.stopword?(token)
+      @stopword_set ||= stopwords.map(&:downcase).to_set
+      @stopword_set.include?(token.downcase)
+    end
+
+    # Determine if a sample of text contains unmatched brackets or quotes
+    # This is one of the more frequent and noticeable failure modes for
+    # the generator; we can just tell it to retry
+    # @param text [String]
+    # @return [Boolean]
+    def self.unmatched_enclosers?(text)
+      enclosers = ['**', '""', '()', '[]', '``', "''"]
+      enclosers.each do |pair|
+        starter = Regexp.new('(\W|^)' + Regexp.escape(pair[0]) + '\S')
+        ender = Regexp.new('\S' + Regexp.escape(pair[1]) + '(\W|$)')
+
+        opened = 0
+
+        tokenize(text).each do |token|
+          opened += 1 if token.match(starter)
+          opened -= 1 if token.match(ender)
+
+          return true if opened.negative? # Too many ends!
+        end
+
+        return true if opened != 0 # Mismatch somewhere.
+      end
+
+      false
+    end
+
+    # Determine if ary2 is a subsequence of ary1
+    # @param ary1 [Array]
+    # @param ary2 [Array]
+    # @return [Boolean]
+    def self.subseq?(ary1, ary2)
+      !ary1.each_index.find do |i|
+        ary1[i...i + ary2.length] == ary2
+      end.nil?
+    end
+  end
+end