markov_words 0.2.3 → 1.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 37eb273f25ad18c46a9c3a66c6467222af81817c
-  data.tar.gz: 8ad07cb6d169686909c60d3f492e0bdedd2b4a23
+  metadata.gz: 51ad9f97683566b1d304b1b4a65c9468e5a9ec16
+  data.tar.gz: 83d124fbaa55ec819de036a82f8e02a8401f68b7
 SHA512:
-  metadata.gz: 1c04a14d059551ecff738aee90e93c60f0ec9750922e9c076392cc09f013ec9e7231dcb7b42811a3ba00181638b82e21b3ab758d892d0d3e8dd288602378d408
-  data.tar.gz: 394f7499450531865e92164708c95ef4d0b6c39ecb8f2e7bb680830f428714669f6b2d86977c40f0422fe51ee7ff5edc04f8b7b87f754d88387673d39f700af7
+  metadata.gz: 161f176055d116252078f2435413096097d40337a6891f804e82f99923c79ddaeb689df4fdc27df09bb743339fa1f18fd0c04707c9d89ea521b7ffa456d3b12a
+  data.tar.gz: '0291eed8a0eb463aa20904a1c6376522477eab5cbfd88f7577e91bc97bcf61fc560397a99694e680f0f4345718dc3b31b6575786a98045495d0183ae71e9ac0c'

data/.rubocop.yml

@@ -0,0 +1,4 @@
+AllCops:
+  TargetRubyVersion: 2.3
+Metrics/ClassLength:
+  Max: 150

data/Gemfile

@@ -1,3 +1,5 @@
+# frozen-string-literal: true
+
 source 'https://rubygems.org'
 
 git_source(:github) { |repo_name| "https://github.com/#{repo_name}" }

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    markov_words (0.2.3)
+    markov_words (1.0.0)
 
 GEM
   remote: https://rubygems.org/
@@ -13,6 +13,9 @@ GEM
       coderay (~> 1.1.0)
       method_source (~> 0.9.0)
     rake (10.5.0)
+    sqlite3 (1.3.13)
+    sqlite3-ruby (1.3.3)
+      sqlite3 (>= 1.3.3)
     yard (0.9.12)
 
 PLATFORMS
@@ -24,6 +27,7 @@ DEPENDENCIES
   minitest (~> 5.0)
   pry (~> 0.11)
   rake (~> 10.0)
+  sqlite3-ruby (~> 1.3)
   yard (~> 0.6)
 
 BUNDLED WITH

data/README.md

@@ -7,8 +7,6 @@ As it turns out, [Markov Chains](http://www.thagomizer.com/blog/2017/11/07/marko
 
 While there are [quite](https://github.com/dabrorius/markov-noodles) a [few](https://github.com/dabrorius/markov-noodles) [wonderful](https://github.com/imikimi/literate_randomizer) Ruby libraries that do this, they all focus either on _actual_ English words, or on creating random _sentences_ but not words. We created this library to do the same thing, but with words, hence the name `MarkovWords`.
 
-
-
 ## Installation
 
 Add this line to your application's Gemfile:
@@ -32,19 +30,19 @@ Basic usage is as follows:
 ```ruby
 require 'markov_words'
 
-words = MarkovWords::Words.new
+generator = MarkovWords::Generator.new
 # returns a random word
-puts words.word 
+puts generator.word 
 ```
 
-You might prefer using a number of n-grams (letter combinations being tracked) higher than the default number (which is 2). We've found that the higher you go, the more accurate words tend to sound, at the expense of having to generate a much larger database of n-gram => letter correspondences. In the case of the default `/usr/share/dict` file, `gram_size = 1` yields a roughly `3.2Kb` database size; `gram_size = 2` yields `18Kb`. Once you get up to `gram_size = 8`, you're into ~`30MB` territory, which slows things down a bit. 
+You might prefer using a number of n-grams (letter combinations being tracked) higher than the default number (which is 2). We've found that the higher you go, the more accurate words tend to sound, as the likelihood that you've started with a partial word the entire length of a word from your dictionary goes up. The increased "real-sounding-ness" comes at the expense of having to generate a much larger database of n-gram => letter correspondences, and accordingly slower access times. 
 
 To set gram_size:
 
 ```ruby
-words = MarkovWords::words.new(gram_size: 8)
+generator = MarkovWords::Generator.new(gram_size: 7)
 # Will take a while the first time, while the database is created.
-puts words.word 
+puts generator.word 
 ```
 
 ### Dictionary
@@ -53,48 +51,58 @@ By default, `MarkovWords` will use the system dictionary located (on Macs) in `/
 
 ```ruby
 # eg to generate random proper names instead of English-sounding words.
-words = MarkovWords::Words.new(corpus_file: '/usr/share/dict/propernames')
+generator = MarkovWords::Generator.new(corpus_file: '/usr/share/dict/propernames')
 ```
 
 This is pretty great, because it means that if you have a dictionary to emulate, you can make words that sound like anything!
 
 ### Data Storage
 
-`MarkovWords` stores its database of n-gram concurrences in `Marshal`'ed text files on disk and loads it into memory when necessary. You can control the location of the data file with:
+`MarkovWords` stores its database of n-gram concurrences on disk and loads it into memory when necessary. You can control the location of the data file with:
+
+```ruby
+# eg to store the data in /tmp/markov.data
+generator = MarkovWords::Generator.new(data_file: /tmp/markov.data)
+```
+
+You can also clear out the contents of the data file (because `MarkovWords` will re-use it by default), by passing `flush_data: true`:
 
 ```ruby
 # eg to store the data in /tmp/markov.data
-words = MarkovWords::Words.new(data_file: /tmp/markov.data)
+generator = MarkovWords::Generator.new(data_file: /tmp/markov.data, flush_data: true)
 ```
 
+
 ### Caching
 
-Because calculation can get slow, especially at high n-gram sizess, `MarkovWords` will cache 100 words into a `words_ngramsize.cache` file by default. If you want to control caching, you can adjust caching parameters eg:
+Because calculation can get slow, especially at high n-gram sizes, `MarkovWords` will cache 100 words by default . If you want to control caching, you can adjust caching parameters eg:
 
 ```ruby
 # For no caching whatsoever
-words = MarkovWords::Words.new(cache: false)
+generator = MarkovWords::Generator.new(perform_caching: false)
 
 # To change the number of pre-computed/stored words to 1000:
-words = MarkovWords::Words.new(cache_size: 1000)
-
-# To change the location of the cache file to /tmp/markov.cache:
-words = MarkovWords::Words.new(cache_file: '/tmp/markov.cache')
-
-# Of course, options can be combined:
-words = MarkovWords::Words.new(
-  cache_file: '/tmp/markov.cache',
-  cache_size: 1000
-)
+generator = MarkovWords::Generator.new(cache_size: 1000)
 ```
 
-Lastly, you can "top off" the cache to make sure it's full with:
+You can "top off" the cache to make sure it's full with:
 
 ```ruby
-words = MarkovWords::Words.new
-words.refresh_cache
+generator = MarkovWords::Generator.new
+generator.refresh_cache
 ```
 
+## Change Log
+
+- `1.0.0` introduced a couple of breaking changes:
+    - `Words` class renamed to `Generator`.
+    - `Generator`:
+        - `cache: [boolean]` parameter was re-named to `perform_caching: [boolean]`.
+        - Removed a lot of `attr_accessor` variables such as `data_store`, `min_length`, `max_length` etc., in favor of a leaner + cleaner API.
+        - The cache file is no longer persisted to disk separately (because `FileStore` is using SQLite instead of direct-disk storage).
+- `0.2.x` was all about Rubocop compliance, so it was a few method refactors but nothing major.
+- `0.1.0` initial commit
+
 ## Development
 
 After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake test` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.

data/Rakefile

@@ -1,3 +1,5 @@
+# frozen-string-literal: true
+
 require 'bundler/gem_tasks'
 require 'rake/testtask'
 

data/bin/console

@@ -1,4 +1,5 @@
 #!/usr/bin/env ruby
+# frozen-string-literal: true
 
 require 'bundler/setup'
 require 'markov_words'

data/lib/markov_words/file_store.rb

@@ -1,39 +1,68 @@
+# frozen-string-literal: true
+
 require 'securerandom'
+require 'sqlite3'
 
 module MarkovWords
-  # Utility for persisting arbitrary data to disk as Marshal'ed Ruby objects
+  # Utility for persisting arbitrary data to disk.
   class FileStore
-    attr_reader :file_path
-    attr_reader :data
-
     # @option opts [String] :file_path Path and name for where the file should
     #   be stored.
     # @option opts [Boolean] :flush_data Do you want the file to be cleared on
     #   open?
     def initialize(opts)
       @file_path = opts.fetch :file_path, "/tmp/#{SecureRandom.base64}"
-      delete_if_exists(@file_path) if opts[:flush_data]
+      initialize_db
+      empty_db if opts[:flush_data]
     end
 
-    # Store arbitary data into file storage
+    # Store arbitary data, named with a `key`
+    # @param key [Symbol] Unique key for later data retrieval
     # @param data [Object] Any Marshal-able object
-    def store_data(data)
-      File.open(@file_path, 'wb') { |f| Marshal.dump(data, f) }
+    def store_data(key = :discard, data = nil)
+      key = key.to_s unless key.is_a? String
+      if data_exists(key)
+        @db.execute 'UPDATE data SET value = ? WHERE key = ?',
+                    [Marshal.dump(data), key]
+      else
+        @db.execute 'INSERT INTO data VALUES ( ?, ? )',
+                    [key, Marshal.dump(data)]
+      end
     end
 
-    # Retrieve whatever data is stored in the file + return it
-    def retrieve_data
-      result = nil
-      if File.exist?(@file_path)
-        File.open(@file_path, 'r') { |f| result = Marshal.load(f) }
-      end
-      result
+    # Retrieve whatever data is stored in at `key`, and return it!
+    def retrieve_data(key = '')
+      key = key.to_s unless key.is_a? String
+      data_array = @db.execute 'SELECT value FROM data WHERE key = ?', key
+      Marshal.load(data_array[0][0]) unless data_array[0].nil?
     end
 
     private
 
-    def delete_if_exists(path)
-      File.delete path if File.exist? path
+    def create_initial_tables
+      @db.execute <<~SQL
+        create table data (
+          key varchar(5),
+          value binary
+        );
+      SQL
+    end
+
+    def data_exists(key)
+      query = 'SELECT count(*) FROM data WHERE key = ?'
+      @db.execute(query, key)[0][0].positive?
+    end
+
+    def empty_db
+      @db.execute 'delete from data'
+    end
+
+    def initialize_db
+      @db = SQLite3::Database.new @file_path
+      table = @db.execute("SELECT name FROM sqlite_master
+                          WHERE type='table'
+                          AND name='data';").first
+      create_initial_tables unless table
     end
   end
 end

data/lib/markov_words/{words.rb → generator.rb}

@@ -1,38 +1,41 @@
+# frozen-string-literal: true
+
 module MarkovWords
   # This class takes care of word generation, caching, and data storage.
-  class Words
-    # Perform caching? Defaults to true.
-    attr_reader :cache
-    # File location where you want to store the cache
-    attr_reader :cache_file
-    # How many words you want to store in the cache?
-    attr_reader :cache_size
-    # Object for storing + retrieving cache data from persistent storage
-    attr_reader :cache_store
-    # Your dictionary of words. Defaults to /usr/share/dict/words.
-    attr_reader :corpus_file
-    # Where should your database be stored on disk?
-    attr_reader :data_file
-    # Object for storing + retrieving n-gram data from persistent storage
-    attr_reader :data_store
-    # The database of "grams" (word/count combinations), stored on the disk and
-    # loaded into this variable in memory when generating words.
-    attr_reader :grams
-    # Number of n-grams to compute for your database. Defaults to 2
-    attr_reader :gram_size
-    # Max generated word length. Defaults to 16
-    attr_reader :max_length
-    # Minimum generated word length. Defaults to 3. NOTE: If your corpus size
-    # is very small (<1000 words or so), it's hard to guarantee a min_length
-    # because so many n-grams will have no association, which terminates word
-    # generation.
-    attr_reader :min_length
+  class Generator
+    # The current list of cached words.
+    # @return [Array<String>] All words in the cache.
+    def cache
+      @data_store.retrieve_data(:cache)
+    end
+
+    # The current database of n-gram mappings
+    # @return [Hash] n-gram database
+    def grams
+      @grams = @grams ||
+               @data_store.retrieve_data(:grams) ||
+               markov_corpus(@corpus_file, @gram_size)
+    end
 
     # Create a new "Words" object
-    # @param opts [Hash] options sent to the object. Any of the object
-    #   attributes (eg `:cache_file` or `:gram_size`) are valid parameters to
-    #   add to the `opts` hash.
-    # @return [Words] A `MarkovWords::Words` object.
+    # @param opts [Hash]
+    # @option opts [Integer] :cache_size How many words to pre-calculate +
+    #   store in the cache for quick retrieval
+    # @option opts [String] :corpus_file ('/usr/share/dict/words') Your
+    #   dictionary of words.
+    # @option opts [String] :data_file Location where calculations are
+    #   persisted to disk.
+    # @option opts [String] :flush_data Remove any previously-stored data from
+    #   an existing database file.
+    # @option opts [Integer] :gram_size (2) Number of n-grams to compute for
+    #   your database.
+    # @option opts [Integer] :max_length (16) Max generated word length.
+    # @option opts [Integer] :min_length (3) Minimum generated word length.
+    #   NOTE: If your corpus size is very small (<1000 words or so), it's hard
+    #   to guarantee a min_length because so many n-grams will have no
+    #   association, which terminates word generation.
+    # @option opts [Boolean] :perform_caching (true) Perform caching?
+    # @return [Words] A `MarkovWords::Generator` object.
     def initialize(opts = {})
       @grams = nil
       @gram_size = opts.fetch :gram_size, 2
@@ -44,13 +47,14 @@ module MarkovWords
     end
 
     # "Top off" the cache of stored words, and ensure that it's at
-    # `@cache_size`. If `@cache` is set to `false`, returns an empty array.
+    # `@cache_size`. If `perform_caching` is set to `false`, returns an empty
+    # array.
     # @return [Array<String>] All words in the cache.
     def refresh_cache
-      if @cache
-        words_array = @cache_store.retrieve_data
+      if @perform_caching
+        words_array = @data_store.retrieve_data(:cache) || []
         words_array << generate_word while words_array.length < @cache_size
-        @cache_store.store_data words_array
+        @data_store.store_data(:cache, words_array)
         words_array
       else
         []
@@ -60,7 +64,7 @@ module MarkovWords
     # Generate a new word, or return one from the cache if available.
     # @return [String] The word.
     def word
-      if @cache
+      if @perform_caching
         load_word_from_cache
       else
         generate_word
@@ -69,22 +73,6 @@ module MarkovWords
 
     private
 
-    def initialize_cache(opts)
-      @cache = opts.fetch :cache, true
-      @cache_file = opts.fetch :cache_file,
-                               "tmp/markov_words_#{@gram_size}.cache"
-      @cache_size = opts.fetch :cache_size, 70
-      @cache_store = FileStore.new(file_path: @cache_file)
-    end
-
-    def initialize_data(opts)
-      @corpus_file = opts.fetch :corpus_file,
-                                '/usr/share/dict/words'
-      @data_file = opts.fetch :data_file,
-                              "tmp/markov_words_#{@gram_size}.data"
-      @data_store = FileStore.new(file_path: @data_file)
-    end
-
     def contains_vowel?(ary)
       if ary.length < 2
         true
@@ -93,9 +81,20 @@ module MarkovWords
       end
     end
 
-    # Generates an English (by default) -sounding word.
+    def initialize_cache(opts)
+      @cache_size = opts.fetch :cache_size, 100
+      @perform_caching = opts.fetch :perform_caching, true
+    end
+
+    def initialize_data(opts)
+      @corpus_file = opts.fetch :corpus_file, '/usr/share/dict/words'
+      @data_file = opts.fetch :data_file, 'tmp/markov_words.data'
+      @flush_data = opts.fetch :flush_data, false
+      @data_store = FileStore.new file_path: @data_file,
+                                  flush_data: @flush_data
+    end
+
     def generate_word
-      set_grams if @grams.nil?
       generate_gram_array(generate_word_length).join
     end
 
@@ -109,19 +108,16 @@ module MarkovWords
         current_gram_size = gal >= @gram_size ? @gram_size : gal
         key = gram_array[-current_gram_size..-1].join
 
-        gram = pick_random_char(@grams[key])
+        gram = pick_random_char(grams[key])
         gram_array << gram
       end
       gram_array
     end
 
-    # Set initial array of chars, which is taken from the @grams key list.
-    # must contain a vowel in the first 2 chars (unless @gram_size == 1 in
-    # which case any letter).
     def generate_initial_gram_array
       initial_gram_array = []
 
-      all_grams_array = @grams.to_a
+      all_grams_array = grams.to_a
       gram_min_length = @gram_size < @min_length ? @gram_size : @min_length
       until initial_gram_array.length >= gram_min_length &&
             contains_vowel?(initial_gram_array)
@@ -130,7 +126,6 @@ module MarkovWords
       initial_gram_array
     end
 
-    # The word must be a random length, between @min and @max
     def generate_word_length
       word_length = 0
       until word_length >= @min_length
@@ -139,14 +134,18 @@ module MarkovWords
       word_length
     end
 
+    def line_ending?(word)
+      /[\r\n]/.match? word
+    end
+
     def load_word_from_cache
-      words_array = @cache_store.retrieve_data
+      words_array = @data_store.retrieve_data(:cache)
       if words_array.nil? || words_array.empty?
         words_array = Array.new(@cache_size) { generate_word }
       end
 
       word = words_array.pop
-      cache_store.store_data words_array
+      @data_store.store_data(:cache, words_array)
 
       word
     end
@@ -199,16 +198,5 @@ module MarkovWords
         return char if counter >= pick_num
       end
     end
-
-    def line_ending?(word)
-      /[\r\n]/.match? word
-    end
-
-    def set_grams
-      grams = @data_store.retrieve_data ||
-              markov_corpus(@corpus_file, @gram_size)
-      @data_store.store_data grams unless grams == @grams
-      @grams = grams
-    end
   end
 end

data/lib/markov_words/version.rb

@@ -1,4 +1,6 @@
+# frozen-string-literal: true
+
 module MarkovWords
   # Current version
-  VERSION = '0.2.3'.freeze
+  VERSION = '1.0.0'
 end

data/lib/markov_words.rb

@@ -1,5 +1,7 @@
+# frozen-string-literal: true
+
 require 'markov_words/version'
-require 'markov_words/words'
+require 'markov_words/generator'
 require 'markov_words/file_store'
 require 'securerandom'
 

data/markov_words.gemspec

@@ -1,3 +1,5 @@
+# frozen-string-literal: true
+
 lib = File.expand_path('../lib', __FILE__)
 $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 require 'markov_words/version'
@@ -8,8 +10,14 @@ Gem::Specification.new do |spec|
   spec.authors = ['Donald Merand']
   spec.email = ['dmerand@explo.org']
 
-  spec.summary = %{Generate words (not sentences) using Markov-chain techniques.}
-  spec.description = %{It's often nice to have random English-sounding words, eg. for password generators. This library uses Markov-chain techniques on words, as opposed to many others which focus on sentences.}
+  spec.summary = <<~SUMMARY
+    Generate words (not sentences) using Markov-chain techniques.}
+  SUMMARY
+  spec.description = <<~DESCRIPTION
+    It's often nice to have random English-sounding words, eg. for password
+    generators. This library uses Markov-chain techniques on words, as opposed
+    to many others which focus on sentences.
+  DESCRIPTION
   spec.homepage = 'https://github.com/exploration/markov_words'
   spec.license = 'MIT'
 
@@ -24,5 +32,8 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency 'minitest', '~> 5.0'
   spec.add_development_dependency 'pry', '~> 0.11'
   spec.add_development_dependency 'rake', '~> 10.0'
+  spec.add_development_dependency 'sqlite3-ruby', '~> 1.3'
   spec.add_development_dependency 'yard', '~> 0.6'
+
+  spec.required_ruby_version = '~> 2.3'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: markov_words
 version: !ruby/object:Gem::Version
-  version: 0.2.3
+  version: 1.0.0
 platform: ruby
 authors:
 - Donald Merand
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2017-12-07 00:00:00.000000000 Z
+date: 2017-12-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -66,6 +66,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: sqlite3-ruby
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.3'
 - !ruby/object:Gem::Dependency
   name: yard
   requirement: !ruby/object:Gem::Requirement
@@ -80,9 +94,10 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.6'
-description: It's often nice to have random English-sounding words, eg. for password
-  generators. This library uses Markov-chain techniques on words, as opposed to many
-  others which focus on sentences.
+description: |
+  It's often nice to have random English-sounding words, eg. for password
+  generators. This library uses Markov-chain techniques on words, as opposed
+  to many others which focus on sentences.
 email:
 - dmerand@explo.org
 executables: []
@@ -90,6 +105,7 @@ extensions: []
 extra_rdoc_files: []
 files:
 - ".gitignore"
+- ".rubocop.yml"
 - ".travis.yml"
 - ".yardopts"
 - CODE_OF_CONDUCT.md
@@ -102,8 +118,8 @@ files:
 - bin/setup
 - lib/markov_words.rb
 - lib/markov_words/file_store.rb
+- lib/markov_words/generator.rb
 - lib/markov_words/version.rb
-- lib/markov_words/words.rb
 - markov_words.gemspec
 homepage: https://github.com/exploration/markov_words
 licenses:
@@ -115,9 +131,9 @@ require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - "~>"
     - !ruby/object:Gem::Version
-      version: '0'
+      version: '2.3'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
@@ -128,5 +144,5 @@ rubyforge_project:
 rubygems_version: 2.6.13
 signing_key: 
 specification_version: 4
-summary: Generate words (not sentences) using Markov-chain techniques.
+summary: Generate words (not sentences) using Markov-chain techniques.}
 test_files: []