markovian 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 86536007f535d2ba077ed8ff3aeabf61d9ee6107
+  data.tar.gz: 61bd78dab63cafa0b0643ddb39793dec563beae5
+SHA512:
+  metadata.gz: e2695ad84f944bbb850e62693f3a96b4678f7d01ef54a557f58804056b6dea5f0431ca9cd21c38365676c61ae505cd931696b125b666672b9c30e42e335ec095
+  data.tar.gz: 4ba6686bba2a42acb3afc5effec994e18b168fab8c286db00afe1b401dc73cf09d9eb3f039e7aa66fa5202796795297ff1fd98d21963fd83f902ce25543b6d62

data/.gitignore

@@ -0,0 +1,18 @@
+# See http://help.github.com/ignore-files/ for more about ignoring files.
+#
+# If you find yourself ignoring temporary files generated by your text editor
+# or operating system, you probably want to add a global ignore instead:
+#   git config --global core.excludesfile '~/.gitignore_global'
+
+# Ignore bundler config.
+/.bundle
+
+# Ignore the default SQLite database.
+/db/*.sqlite3
+/db/*.sqlite3-journal
+
+# Ignore all logfiles and tempfiles.
+/log/*.log
+/tmp
+.DS_Store
+Gemfile.lock

data/.rspec

@@ -0,0 +1,2 @@
+--color
+--require spec_helper

data/.ruby-gemset

@@ -0,0 +1 @@
+markov-ahkoppel

data/.ruby-version

@@ -0,0 +1 @@
+2.2

data/CODE_OF_CONDUCT.md

@@ -0,0 +1,13 @@
+# Contributor Code of Conduct
+
+As contributors and maintainers of this project, we pledge to respect all people who contribute through reporting issues, posting feature requests, updating documentation, submitting pull requests or patches, and other activities.
+
+We are committed to making participation in this project a harassment-free experience for everyone, regardless of level of experience, gender, gender identity and expression, sexual orientation, disability, personal appearance, body size, race, ethnicity, age, or religion.
+
+Examples of unacceptable behavior by participants include the use of sexual language or imagery, derogatory comments or personal attacks, trolling, public or private harassment, insults, or other unprofessional conduct.
+
+Project maintainers have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, issues, and other contributions that are not aligned to this Code of Conduct. Project maintainers who do not follow the Code of Conduct may be removed from the project team.
+
+Instances of abusive, harassing, or otherwise unacceptable behavior may be reported by opening an issue or contacting one or more of the project maintainers.
+
+This Code of Conduct is adapted from the [Contributor Covenant](http://contributor-covenant.org), version 1.0.0, available at [http://contributor-covenant.org/version/1/0/0/](http://contributor-covenant.org/version/1/0/0/)

data/Gemfile

@@ -0,0 +1,15 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in markov-ahkoppel2.gemspec
+gemspec
+
+group :development, :test do
+  gem 'byebug'
+end
+
+group :test do
+  gem "rspec"
+  gem "faker"
+  gem "codeclimate-test-reporter"
+end
+

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2015 Alex Koppel
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,54 @@
+[![Code Climate](https://codeclimate.com/github/arsduo/markovian/badges/gpa.svg)](https://codeclimate.com/github/arsduo/markovian)
+[![Test Coverage](https://codeclimate.com/github/arsduo/markovian/badges/coverage.svg)](https://codeclimate.com/github/arsduo/markovian/coverage)
+[![Build Status](https://travis-ci.org/arsduo/markovian.svg)](https://travis-ci.org/arsduo/markovian)
+
+# A Markov Implementation
+
+## Installation
+
+Clone from Github, and then execute:
+
+    $ bundle
+
+## Usage
+
+Fuller documentation will come shortly. For now, let's see how we can use Markovian to build some tweets from a Twitter archive we've downloaded:
+
+```ruby
+> path = #{path_to_twitter_archive}
+ => path_to_twitter_archive
+> importer = Markovian::Importers::Twitter::CsvImporter.new(path)
+ => #<Markovian::Importers::Twitter::CsvImporter:0x007fd0ca3282a8 @path=path_to_twitter_archive>
+> tweets = importer.texts_for_markov_analysis; puts tweets.count
+14394
+ => nil
+
+# Create a Chainset (the structure holding all the word relations)...
+> chainset = Markovian::ChainSet.new
+ => #<Markovian::ChainSet:0x007fd0ca03df70 ...>
+# And add all the tweets to the Markov dictionary
+> tweets.each {|t| Markovian::Chain::TextCompiler.new(t, chainset).incorporate_into_chain}; puts "done."
+done.
+ => nil
+
+# Now, we can build some text!
+> Markovian::TextBuilder.new("markov", chainset).construct
+=> "markov chains a lot better than a month, i've been here half an hour of night when you can get behind belgium for the offline train journey"
+```
+
+Exactly!
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/arsduo/markovian. This project is intended to be a safe, welcoming space for collaboration, and contributors are expected to adhere to the [Contributor Covenant](contributor-covenant.org) code of conduct.
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT).
+

data/Rakefile

@@ -0,0 +1,8 @@
+require "bundler/gem_tasks"
+
+task :default => :spec
+
+require 'rspec/core/rake_task'
+RSpec::Core::RakeTask.new do |t|
+  t.rspec_opts = ["--color", '--format doc']
+end

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "markovian"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start

data/bin/setup

@@ -0,0 +1,7 @@
+#!/bin/bash
+set -euo pipefail
+IFS=$'\n\t'
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/changelog.md

@@ -0,0 +1,8 @@
+# CHANGELOG
+
+## 0.1.0 and below
+
+* Ability to build bidirectional chainsets (pair of chains) from arrays of texts
+* Ability to import Twitter archives and produce an array of tweets
+* Ability to generate Markovian texts from a chainset
+* Gem framework

data/db/seeds.rb

@@ -0,0 +1,7 @@
+# This file should contain all the record creation needed to seed the database with its default values.
+# The data can then be loaded with the rake db:seed (or created alongside the db with db:setup).
+#
+# Examples:
+#
+#   cities = City.create([{ name: 'Chicago' }, { name: 'Copenhagen' }])
+#   Mayor.create(name: 'Emanuel', city: cities.first)

data/lib/markovian/chain/dictionary.rb

@@ -0,0 +1,24 @@
+# This class represents a dictionary of words or phrases and the various words that can follow
+# them.  Currently it's implemented as a hash of arrays, but more advanced representations may
+# follow.
+#
+# The key is an opaque value, which could represent either a single word or a phrase as desired.
+module Markovian
+  class Chain
+    class Dictionary
+      def push(key, word)
+        dictionary[key] += [word]
+      end
+
+      def next_word(key)
+        dictionary[key].sample
+      end
+
+      protected
+
+      def dictionary
+        @dictionary ||= Hash.new([])
+      end
+    end
+  end
+end

data/lib/markovian/chain/text_compiler.rb

@@ -0,0 +1,66 @@
+require 'markovian/utils/text_splitter'
+
+# Given a text to analyze, this class returns a hash of Markov results: two-word phrases (two by
+# default) pointing to an array of historical next words.
+#
+# So, for instance, the phrase "Cats are cute, cats are annoying" would map to:
+# {
+#   "cats are" => [cute, annoying],
+#   "are cute" => [cats],
+#   "cute cats" => [are],
+# }
+#
+# Notes:
+# * Next words (in v1) are not unique, in order to represent weighting. There are definitely more
+#   space-compact ways to do that, but that's left for future implementation.
+# * Punctuation is for later.
+# * Handling sentences or newlines is later -- I'm not sure the right way to do it.
+# * Capitalization is deferred for later.
+module Markovian
+  class Chain
+    class TextCompiler
+      # Pass in a text, and optionally an existing Markov chain to add data to. In many cases, you
+      # may be building a chain using a set of smaller texts instead of one large texts (dialog,
+      # for instance, or Twitter archives), and so may call this class repeatedly for elements of
+      # the parent corpus.
+      attr_reader :text, :chainset
+      def initialize(text, starter_chainset = ChainSet.new)
+        @text = text
+        @chainset = starter_chainset
+      end
+
+      def incorporate_into_chain
+        add_text_to_chain(interesting_split_text, forward_chain)
+        # to assemble backward text, we just create a chainset with all the texts reversed
+        # that allows us to see what words precede any given word
+        add_text_to_chain(interesting_split_text.reverse, backward_chain)
+        chainset
+      end
+
+      protected
+
+      def forward_chain
+        chainset.forward
+      end
+
+      def backward_chain
+        chainset.backward
+      end
+
+      def add_text_to_chain(text_elements, chain)
+        previous_word = nil
+        text_elements.each_with_index do |word, index|
+          # if we're not at the beginning or the end of the text -- e.g. we have a full triple
+          if next_word = text_elements[index + 1]
+            chain.lengthen(word, next_word: next_word, previous_word: previous_word)
+          end
+          previous_word = word
+        end
+      end
+
+      def interesting_split_text
+        @interesting_split_text ||= Utils::TextSplitter.new(text).components
+      end
+    end
+  end
+end

data/lib/markovian/chain.rb

@@ -0,0 +1,41 @@
+require 'markovian/chain/dictionary'
+
+# The Chain represents Markov info as it's being assembled or expanded from a text. To compensate
+# for small sample sizes, we track multiple chains (derived from both two-word phrases and single
+# word). Phrases are prefered, but if we can't find a match, we'll try with a single word.
+module Markovian
+  class Chain
+    def initialize
+      @one_key_dictionary = Dictionary.new
+      @two_key_dictionary = Dictionary.new
+    end
+
+    attr_reader :one_key_dictionary, :two_key_dictionary
+    def lengthen(word, next_word:, previous_word: nil)
+      @one_key_dictionary.push(word, next_word)
+      @two_key_dictionary.push(two_word_key(previous_word, word), next_word)
+      word
+    end
+
+    def next_word(word, previous_word: nil)
+      result_for_two_words(previous_word, word) || result_for_one_word(word)
+    end
+
+    protected
+
+    def result_for_two_words(previous_word, word)
+      @two_key_dictionary.next_word(two_word_key(previous_word, word)) if previous_word
+    end
+
+    def result_for_one_word(word)
+      @one_key_dictionary.next_word(word)
+    end
+
+    # We represent the two words as a space-delimited phrase for simplicity and speed of access via
+    # hash keys.
+    def two_word_key(word1, word2)
+      "#{word1} #{word2}"
+    end
+  end
+end
+

data/lib/markovian/chain_set.rb

@@ -0,0 +1,22 @@
+require 'markovian/chain'
+
+# This class represents a pair of chains, one going forward and one going backward. With this, we
+# can construct phrases in which the original seed word appears at any point in the text (going
+# backward to create the earlier text, forward to create the rest).
+module Markovian
+  class ChainSet
+    attr_reader :forward, :backward
+    def initialize
+      @forward, @backward = Chain.new, Chain.new
+    end
+
+    def next_word(word, previous_word: nil)
+      forward.next_word(word, previous_word: previous_word)
+    end
+
+    def previous_word(word, following_word: nil)
+      # backward goes in the opposite direction to forward
+      backward.next_word(word, previous_word: following_word)
+    end
+  end
+end

data/lib/markovian/importers/twitter/csv_importer.rb

@@ -0,0 +1,43 @@
+require 'csv'
+require 'markovian/importers/twitter/tweet'
+
+# This class will import a Twitter archive CSV, returning a set of tweets suitable for importation
+# into a Markovian chain.
+module Markovian
+  module Importers
+    module Twitter
+      class CsvImporter
+        attr_reader :path
+        def initialize(path)
+          @path = path
+        end
+
+        def texts_for_markov_analysis
+          # reject any blank tweets -- in our case, those with only a stripped-out URL
+          tweet_enumerator.reject {|t| t.empty?}
+        end
+
+        protected
+
+        def csv_enumerator
+          # returns an iterator object that we can roll through
+          # this does not actually start reading the file
+          @csv_enumerator ||= CSV.open(path, headers: true).each
+        end
+
+        # an iterator over personal tweets (e.g. not RTs)
+        # the lazy iterator allows us to add the condition without having to parse the entire file at
+        # once (which could easily encounter tens of thousands of rows).
+        def personal_tweet_enumerator
+          csv_enumerator.select {|row| row["retweeted_status_id"].empty? }
+        end
+
+        def tweet_enumerator
+          personal_tweet_enumerator.map do |row|
+            Tweet.new(row["text"]).interesting_text
+          end
+        end
+      end
+    end
+  end
+end

data/lib/markovian/importers/twitter/tweet.rb

@@ -0,0 +1,37 @@
+module Markovian
+  module Importers
+    module Twitter
+      # Represents an individual tweet
+      class Tweet
+        attr_reader :text
+        def initialize(text)
+          @text = text
+        end
+
+        # Not currently used, but we might want to weight mentions later.
+        def mentions
+          text.scan(/(\@[a-z0-9_]+)/).flatten
+        end
+
+        def interesting_text
+          without_urls(without_leading_dot(text))
+        end
+
+        protected
+
+        # We don't want URLs to be considered inside our Markov machine.
+        # URL matching is nearly impossible, but this regexp should be good enough: http://stackoverflow.com/questions/17733236/optimize-gruber-url-regex-for-javascript
+        # Nowadays Twitter replaces URLS with their own link shortener, but historically that wasn't
+        # always true.
+        def without_urls(string)
+          string.gsub(/\b((?:[a-z][\w-]+:(?:\/{1,3}|[a-z0-9%])|www\d{0,3}[.]|[a-z0-9.\-]+[.][a-z]{2,4}\/)\S+(?:[^\s`!\[\]{};:'".,?«»“”‘’]))/i, "")
+        end
+
+        # Avoid dots used to trigger mentions
+        def without_leading_dot(string)
+          string.gsub(/^\.\@/, "@")
+        end
+      end
+    end
+  end
+end

data/lib/markovian/text_builder.rb

@@ -0,0 +1,63 @@
+require 'markovian/utils/text_splitter'
+
+# This class, given a seed word and a Markov chain_set, will attempt to construct a new text using
+# the Markov associations.
+module Markovian
+  class TextBuilder
+    attr_reader :seed_text, :chain_set
+    def initialize(seed_text, chain_set)
+      @seed_text = seed_text
+      @chain_set = chain_set
+    end
+
+    def construct(length: 140, seed: default_seed, start_result_with_seed_word: false)
+      # TODO: if we don't hit a result for the first pair, move backward through the original text
+      # until we get something
+      result_with_next_word(
+        previous_pair: seed,
+        result: start_result_with_seed_word ? format_result_array(seed) : nil,
+        length: length
+      )
+    end
+
+    def default_seed
+      if split_seed_text.length >= 2
+        split_seed_text[-2..-1]
+      else
+        # if we only have a one-word seed text, the previous word is nil
+        [nil, split_seed_text.first]
+      end
+    end
+
+    protected
+
+    def result_with_next_word(previous_pair:, result:, length:)
+      previous_word, current_word = previous_pair
+      if next_word = chain_set.next_word(current_word, previous_word: previous_word)
+        # we use join rather than + to avoid leading spaces, and strip to ignore leading nils or
+        # empty strings
+        interim_result = format_result_array([result, next_word])
+        if interim_result.length > length
+          result
+        else
+          result_with_next_word(
+            previous_pair: [current_word, next_word],
+            result: interim_result,
+            length: length
+          )
+        end
+      else
+        result
+      end
+    end
+
+    # Turn an array of words into an ongoing string
+    def format_result_array(array_of_words)
+      array_of_words.compact.map(&:strip).join(" ")
+    end
+
+    def split_seed_text
+      @split_seed_text ||= Utils::TextSplitter.new(seed_text).components
+    end
+  end
+end

data/lib/markovian/utils/text_splitter.rb

@@ -0,0 +1,35 @@
+module Markovian
+  module Utils
+    class TextSplitter
+      attr_reader :text
+      def initialize(text)
+        @text = text
+      end
+
+      # We split on spaces, quotes, (various symbols followed by either another dash, a space,
+      # another dot, or the end of the text), or (colons preceded by space or the beginning of the
+      # text).
+      # We don't want to block things like Jones-Smith, tl;dr, abc.def, or it's.
+      # Any of the following:
+      # [\s\(\)] - a space or parentheses on their own
+      # " - a quote on its own
+      # [\.-:;\?\!]([-\.\s]|$) - a period, dash, ?, or ! followed by a space, period, dash, or the
+      #                          end of the word
+      # [\s^]' - a single ' following a non-letter
+      WORD_DELIMITERS = /([\s\(\)]|"|[\.\-:;\?\!]([\-\.\s]|$)|[\s^]')/
+
+      # anything that doesn't contain any letters is not a word we need to care about
+      MARKERS_OF_INTEREST = /[A-Za-z]/
+
+      def components
+        split_text.select {|t| t.match(MARKERS_OF_INTEREST)}
+      end
+
+      protected
+
+      def split_text
+        text.downcase.split(WORD_DELIMITERS)
+      end
+    end
+  end
+end

data/lib/markovian/version.rb

@@ -0,0 +1,3 @@
+module Markovian
+  VERSION = "0.1.0"
+end

data/lib/markovian.rb

@@ -0,0 +1,11 @@
+require 'oj'
+require 'markovian/chain_set'
+require 'markovian/text_builder'
+require 'markovian/chain/text_compiler'
+# importers
+require 'markovian/importers/twitter/csv_importer'
+
+# The base module.
+module Markovian
+end
+

data/markovian.gemspec

@@ -0,0 +1,25 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'markovian/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "markovian"
+  spec.version       = Markovian::VERSION
+  spec.authors       = ["Alex Koppel"]
+  spec.email         = ["alex@alexkoppel.com"]
+
+  spec.summary       = %q{A simple, hopefully easy-to-use Markov chain generator.}
+  spec.description   = %q{A simple, hopefully easy-to-use Markov chain generator.}
+  spec.homepage      = "https://github.com/arsduo/markov-ahkoppel"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_runtime_dependency "oj"
+  spec.add_development_dependency "bundler", "~> 1.7"
+  spec.add_development_dependency "rake", "~> 10.0"
+end

metadata

@@ -0,0 +1,111 @@
+--- !ruby/object:Gem::Specification
+name: markovian
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Alex Koppel
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2015-09-09 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: oj
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.7'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.7'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+description: A simple, hopefully easy-to-use Markov chain generator.
+email:
+- alex@alexkoppel.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rspec"
+- ".ruby-gemset"
+- ".ruby-version"
+- CODE_OF_CONDUCT.md
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/console
+- bin/setup
+- changelog.md
+- db/seeds.rb
+- lib/.DS_Store
+- lib/markovian.rb
+- lib/markovian/chain.rb
+- lib/markovian/chain/dictionary.rb
+- lib/markovian/chain/text_compiler.rb
+- lib/markovian/chain_set.rb
+- lib/markovian/importers/twitter/csv_importer.rb
+- lib/markovian/importers/twitter/tweet.rb
+- lib/markovian/text_builder.rb
+- lib/markovian/utils/text_splitter.rb
+- lib/markovian/version.rb
+- markovian.gemspec
+homepage: https://github.com/arsduo/markov-ahkoppel
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.6
+signing_key: 
+specification_version: 4
+summary: A simple, hopefully easy-to-use Markov chain generator.
+test_files: []