ruby_markovify 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 418c85d51a3ff770f15683624daaaf228c4c9e72
+  data.tar.gz: 3abddddeedcf70339a40def10b0ce1f876b6492c
+SHA512:
+  metadata.gz: 65ffb392f740d449c9e45cf4497d3d631aa8d826cd73cb04c13a4f86140e9c28b0cd0f0e5f5886822030a0eb5f55e75b5516f97f36f1875c51926a6707959f52
+  data.tar.gz: ecd84fdc50c2c27993f3e4de937f21c9638e3afb578fc248d97a60ffd2b73f27d72c3dc1833df9d829f90292e1519745a6bbccf3202d35c79d42125b174b7404

data/.gitignore

@@ -0,0 +1,9 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in ruby_markovify.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2016 meew0
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/MARKOVIFY_LICENSE.txt

@@ -0,0 +1,25 @@
+The following text is the license for markovify by Jeremy Singer-Vine, which
+ruby_markovify is based on and uses code from.
+===================
+
+The MIT License (MIT)
+
+Copyright (c) 2015, Jeremy Singer-Vine
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/README.md

@@ -0,0 +1,57 @@
+# RubyMarkovify
+
+A Ruby port of the excellent [markovify](https://github.com/jsvine/markovify) Python library.
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'ruby_markovify'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install ruby_markovify
+
+## Usage
+
+The ruby_markovify method names are identical to the ones in markovify, so if you need specific methods you can just read the markovify docs.
+
+Here's a basic usage example:
+```rb
+require 'ruby_markovify'
+
+# Read the corpus from a file
+corpus = File.read('corpus.txt')
+
+# Make a text from the corpus with state size 3
+text = RubyMarkovify::Text.new(corpus, 3)
+
+puts text.make_sentence  # Generates a random sentence
+puts text.make_sentence_with_start('I have')  # Generates a random sentence starting with 'I have'
+puts text.make_short_sentence(40)  # Generates a random sentence at most 40 characters long
+```
+
+In addition to the default markovify `Text` and `NewlineText` classes, ruby_markovify also includes an `ArrayText` class that uses an array (already split into sentences) as its corpus.
+
+Note that ruby_markovify currently implements none of the JSON methods (`Chain#to_json`, `Chain.from_json` and `Text.from_chain`), so if you need them you'll have to implement them yourself (feel free to make a PR!)
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/meew0/ruby_markovify.
+
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT).
+

data/Rakefile

@@ -0,0 +1,2 @@
+require "bundler/gem_tasks"
+task :default => :spec

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "ruby_markovify"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/lib/ruby_markovify.rb

@@ -0,0 +1,4 @@
+require 'ruby_markovify/version'
+require 'ruby_markovify/text'
+
+module RubyMarkovify; end

data/lib/ruby_markovify/chain.rb

@@ -0,0 +1,57 @@
+module RubyMarkovify
+  def self.cumulative_sum(array)
+    sum = 0
+    array.map { |x| sum += x }
+  end
+
+  class Chain
+    def initialize(corpus, state_size, model = nil)
+      @state_size = state_size
+      @model = model || build(corpus, @state_size)
+    end
+
+    def build(corpus, state_size)
+      fail ArgumentError, '`corpus` must be an Array of Arrays!' unless corpus.is_a?(Array) && corpus[0].is_a?(Array)
+
+      model = {}
+
+      corpus.each do |run|
+        items = [:begin] * state_size + run + [:end]
+
+        0.upto(run.length + 1) do |i|
+          state = items[i...i+state_size]
+          follow = items[i+state_size]
+
+          model[state] ||= {}
+          model[state][follow] ||= 0
+          model[state][follow] += 1
+        end
+      end
+
+      model
+    end
+
+    def move(state)
+      choices, weights = @model[state].keys, @model[state].values
+      cumdist = RubyMarkovify.cumulative_sum(weights)
+      r = rand * cumdist[-1]
+      choices[cumdist.index { |e| e >= r }]
+    end
+
+    def gen(init_state = nil)
+      state = init_state || [:begin] * @state_size
+      result = []
+      loop do
+        next_word = move(state)
+        break if next_word == :end
+        result << next_word
+        state = state[1..-1] + [next_word]
+      end
+      result
+    end
+
+    # As Ruby doesn't have the concept of generators, #gen returns an array itself,
+    # so we don't need to make a separate method for walk
+    alias_method :walk, :gen
+  end
+end

data/lib/ruby_markovify/splitters.rb

@@ -0,0 +1,71 @@
+module RubyMarkovify
+  module Splitters
+    ASCII_LOWERCASE = 'abcdefghijklmnopqrstuvwxyz'
+    ASCII_UPPERCASE = ASCII_LOWERCASE.upcase
+
+    # States w/ with thanks to https://github.com/unitedstates/python-us
+    # Titles w/ thanks to https://github.com/nytimes/emphasis and @donohoe
+    STATES = %w{
+      ala ariz ark calif colo conn del fla ga ill ind kan ky la md mass mich minn miss mo mont neb nev okla
+      ore pa tenn vt va wash wis wyo
+    }
+    UNITED_STATES = %w{u.s}
+    TITLES = %w{mr ms mrs msr dr gov pres sen sens rep reps prof gen messrs col sr jf sgt mgr fr rev jr snr atty supt}
+    STREETS = %w{ave blvd st rd hwy}
+    MONTHS = %w{jan feb mar apr jun jul aug sep sept oct nov dec}
+    INITIALS = ASCII_LOWERCASE.chars
+
+    ABBR_CAPPED = STATES + UNITED_STATES + TITLES + STREETS + MONTHS + INITIALS
+    ABBR_LOWERCASE = %w{etc v vs viz al pct}
+    EXCEPTIONS = %w{U.S. U.N. E.U. F.B.I. C.I.A.}
+
+    PUNCTUATION = %w{? !}
+
+    def is_abbreviation(dotted_word)
+      clipped = dotted_word[0..-2]
+      if ASCII_UPPERCASE.include? clipped[0]
+        ABBR_CAPPED.include? clipped.downcase
+      else
+        ABBR_LOWERCASE.include? clipped
+      end
+    end
+
+    def is_sentence_ender(word)
+      return false if EXCEPTIONS.include? word
+      return true if PUNCTUATION.include? word[-1]
+      return true if word.sub(/[^A-Z]/, '').length > 1
+      return true if word[-1] == '.' && !is_abbreviation(word)
+      false
+    end
+
+    # A word that ends with punctuation
+    # Followed by optional quote/parens/etc
+    # Followed by whitespace + non-(lowercase or dash)
+    END_PATTERN = /([\w\.'’&\]\)]+[\.\?!])([‘’“”'\"\)\]]*)(\s+(?![a-z\-–—]))/
+
+    def split_into_sentences(text)
+      res = []
+      text.scan(END_PATTERN) do |c|
+        res << [c, $~.offset(0)[0]]
+      end
+
+      end_indices = res.select do |e|
+        groups, _ = e
+        is_sentence_ender(groups[0])
+      end.map do |e|
+        groups, index = e
+        index + groups[0].length + groups[1].length
+      end
+
+      spans = ([nil] + end_indices).zip(end_indices + [nil])
+
+      ret = spans.map do |elem|
+        start_idx, end_idx = elem
+        next if end_idx == nil
+        start_idx ||= 0
+        text[start_idx..end_idx].strip
+      end
+      ret.compact
+    end
+  end
+end

data/lib/ruby_markovify/text.rb

@@ -0,0 +1,98 @@
+require 'ruby_markovify/chain'
+require 'ruby_markovify/splitters'
+require 'unidecode'
+
+module RubyMarkovify
+  class Text
+    def initialize(input_text, state_size = nil, chain = nil)
+      runs = generate_corpus(input_text)
+      @rejoined_text = sentence_join(runs.map { |e| word_join(e) })
+      state_size ||= 2
+      @chain = chain || Chain.new(runs, state_size)
+    end
+
+    include RubyMarkovify::Splitters
+    def sentence_split(text)
+      split_into_sentences(text)
+    end
+
+    def sentence_join(sentences)
+      sentences.join ' '
+    end
+
+    WORD_SPLIT_PATTERN = /\s+/
+    def word_split(sentence)
+      sentence.split(WORD_SPLIT_PATTERN)
+    end
+
+    def word_join(words)
+      words.join ' '
+    end
+
+    REJECT_PATTERN = /(^')|('$)|\s'|'\s|["(\(\)\[\])]/
+
+    def test_sentence_input(sentence)
+      !!(sentence.to_ascii =~ REJECT_PATTERN)
+    end
+
+    def generate_corpus(text)
+      sentences = sentence_split text
+      sentences.reject! { |e| test_sentence_input(e) }
+      sentences.map { |e| word_split(e) }
+    end
+
+    def test_sentence_output(words, max_overlap_ratio, max_overlap_total)
+      overlap_ratio = (max_overlap_ratio * words.length).round
+      overlap_max = [max_overlap_total, overlap_ratio].min
+      overlap_over = overlap_max + 1
+      gram_count = [words.length - overlap_max, 1].max
+
+      grams = [*0..gram_count].map { |i| words[i..i+overlap_over] }
+      grams.each do |g|
+        gram_joined = word_join(g)
+        return false if @rejoined_text.include? gram_joined
+      end
+
+      true
+    end
+
+    DEFAULT_MAX_OVERLAP_RATIO = 0.7
+    DEFAULT_MAX_OVERLAP_TOTAL = 15
+    DEFAULT_TRIES = 10
+
+    def make_sentence(init_state = nil, options = {})
+      tries = options[:tries] || DEFAULT_TRIES
+      mor = options[:max_overlap_ratio] || DEFAULT_MAX_OVERLAP_RATIO
+      mot = options[:max_overlap_total] || DEFAULT_MAX_OVERLAP_TOTAL
+
+      tries.times do
+        words = @chain.walk(init_state)
+        return word_join(words) if test_sentence_output(words, mor, mot)
+      end
+      nil
+    end
+
+    def make_short_sentence(char_limit, options = {})
+      loop do
+        sentence = make_sentence(nil, options)
+        return sentence if sentence && sentence.length < char_limit
+      end
+    end
+
+    def make_sentence_with_start(beginning, options = {})
+      make_sentence(word_split(beginning), options)
+    end
+  end
+
+  class NewlineText < Text
+    def sentence_split(text)
+      text.split /\s*\n\s*/
+    end
+  end
+
+  class ArrayText < Text
+    def sentence_split(text)
+      text
+    end
+  end
+end

data/lib/ruby_markovify/version.rb

@@ -0,0 +1,3 @@
+module RubyMarkovify
+  VERSION = '0.1.0'
+end

data/ruby_markovify.gemspec

@@ -0,0 +1,24 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'ruby_markovify/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = 'ruby_markovify'
+  spec.version       = RubyMarkovify::VERSION
+  spec.authors       = ['meew0']
+  spec.email         = ['blactbt@live.de']
+
+  spec.summary       = %q{A Ruby port of the excellent `markovify` Python module.}
+  spec.homepage      = 'https://github.com/meew0/ruby_markovify'
+  spec.license       = 'MIT'
+  spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  spec.bindir        = 'exe'
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ['lib']
+
+  spec.add_dependency 'unidecode'
+
+  spec.add_development_dependency 'bundler', '~> 1.12.a'
+  spec.add_development_dependency 'rake', '~> 10.0'
+end

metadata

@@ -0,0 +1,100 @@
+--- !ruby/object:Gem::Specification
+name: ruby_markovify
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- meew0
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2016-02-21 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: unidecode
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.12.a
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.12.a
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+description: 
+email:
+- blactbt@live.de
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- Gemfile
+- LICENSE.txt
+- MARKOVIFY_LICENSE.txt
+- README.md
+- Rakefile
+- bin/console
+- bin/setup
+- lib/ruby_markovify.rb
+- lib/ruby_markovify/chain.rb
+- lib/ruby_markovify/splitters.rb
+- lib/ruby_markovify/text.rb
+- lib/ruby_markovify/version.rb
+- ruby_markovify.gemspec
+homepage: https://github.com/meew0/ruby_markovify
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.5.1
+signing_key: 
+specification_version: 4
+summary: A Ruby port of the excellent `markovify` Python module.
+test_files: []