entropic 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: d35ae5edab8f2a86c92301ba0a35c1c90bfd3d26
+  data.tar.gz: 3463138f664205b707808c9ae8783120258c897f
+SHA512:
+  metadata.gz: 862964de635c81b16806aa8f9acb1a5e347ac399208d688ed4b5b702ddac216ad29a0f5d339c3e12d66a43eaad05c370ffdfa3bba96740a3f1f2c8b7e2aa7f4d
+  data.tar.gz: 2847b9f7d5ae73f36be387594085fd211203e9c365ee25b2cf9655b7ccd1565a9fa2b7d64961ebbb23cccebb9f889607a3b9756309dd0c69efd70f6aa848e055

data/.gitignore

@@ -0,0 +1,14 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+
+# rspec failure tracking
+.rspec_status
+
+tags

data/.rspec

@@ -0,0 +1,2 @@
+--format documentation
+--color

data/.travis.yml

@@ -0,0 +1,5 @@
+sudo: false
+language: ruby
+rvm:
+  - 2.2.5
+before_install: gem install bundler -v 1.14.6

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in entropic.gemspec
+gemspec

data/README.md

@@ -0,0 +1,72 @@
+# Entropic
+
+Entropic trains and predicts entropy of strings based on character n-gram models. For example:
+
+```ruby
+require 'entropic'
+>> m = Entropic::Model.read(open('https://raw.githubusercontent.com/willf/entropy/master/data/google_books_2.tsv')); true
+=> true
+>> m.predict("entropy")
+=> {:log_prob_total=>-37.181802347513745, :log_prob_average=>-6.1969670579189575, :size=>6}
+>> m.predict("yportne")
+=> {:log_prob_total=>-34.25705444264748, :log_prob_average=>-5.70950907377458, :size=>6}
+```
+
+The string 'yportne' is much less likely than the string 'entropy'.
+
+You can also train a model, using strings one per line.
+
+```ruby
+>> n = Entropic::Model.new(2); true
+=> true
+>> File.open('/tmp/training.txt') {|f| n.train(f)}; true
+=> true
+>> n.predict('love')
+=> {:log_prob_total=>-15.396216763909154, :log_prob_average=>-5.132072254636385, :size=>3}
+```
+
+You can also train a model, using strings and a count of the number of times it appers, tab separated.
+
+```ruby
+>> o = Entropic::Model.new(2); true
+=> true
+>> File.open('/tmp/training_with_counts.txt') {|f| o.train_with_multiplier(f)}; true
+=> true
+>> o.predict('love')
+=> {:log_prob_total=>-15.396216763909154, :log_prob_average=>-5.132072254636385, :size=>3}
+```
+
+You can also dump a model, to be read later.
+
+```ruby
+>> File.open('/tmp/save.tsv','w') {|f| o.dump(f)}; true
+=> true
+```
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'entropic'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install entropic
+
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/willf/entropic.
+

data/Rakefile

@@ -0,0 +1,6 @@
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new(:spec)
+
+task :default => :spec

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "entropic"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start(__FILE__)

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/entropic.gemspec

@@ -0,0 +1,35 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'entropic/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "entropic"
+  spec.version       = Entropic::VERSION
+  spec.authors       = ["Will Fitzgerald"]
+  spec.email         = ["willf@github.com"]
+
+  spec.description = %q{Train and predict string entropy based on character n-grams.}
+  spec.summary     = spec.description
+  spec.homepage      = "https://github.com/willf/entropic"
+
+  # Prevent pushing this gem to RubyGems.org. To allow pushes either set the 'allowed_push_host'
+  # to allow pushing to a single host or delete this section to allow pushing to any host.
+  # if spec.respond_to?(:metadata)
+  #   spec.metadata['allowed_push_host'] = "TODO: Set to 'http://mygemserver.com'"
+  # else
+  #   raise "RubyGems 2.0 or newer is required to protect against " \
+  #     "public gem pushes."
+  # end
+
+  spec.files         = `git ls-files -z`.split("\x0").reject do |f|
+    f.match(%r{^(test|spec|features)/})
+  end
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.14"
+  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "rspec", "~> 3.0"
+end

data/lib/entropic/version.rb

@@ -0,0 +1,3 @@
+module Entropic
+  VERSION = "0.1.0"
+end

data/lib/entropic.rb

@@ -0,0 +1,230 @@
+require "entropic/version"
+
+# Public: classes and methods useful for estimating entropy on strings.
+#
+# Examples
+#
+#   model = Entropic.Model.read("ngrams.tsv")
+#   model.predict("the")
+#   # => { log_prob_total: -101.1, log_prob_average: -20.02, size: 5 }
+#
+module Entropic
+
+  # Public: create a sliding window of ngrams from a string
+  #
+  # string: The String to slide over
+  # n: The Integer defining the size of the ngrams
+  #
+  # Examples
+  #
+  #  sliding('01234', 2)  
+  #  # => ['01', '12', '23', '34']
+  #
+  def Entropic.sliding(string, n)
+    (0..string.length - n).map { |i| (string[i, n]).to_s }
+  end
+
+  # Public: a counter for ngrams
+  class NGramCounter
+    attr_accessor :size, :counts, :total
+    def initialize(size)
+      @size = size
+      @counts = Hash.new(0)
+      @total = 0
+    end
+
+    # Public: update a counter with a string, and a multiplier
+    #
+    # Examples
+    # 
+    # counter = NGramCounter.new(2)
+    # counter.update_with_multiplier('01234', 1)
+    #
+    # string: The String to update with
+    # multiplier: The Integer describing how much weight (will often be 1)
+    #
+    def update_with_multiplier(string, multiplier)
+      Entropic.sliding(string, @size).each do |ngram|
+        @counts[ngram] += multiplier
+        @total += multiplier
+      end
+    end
+
+    # Public: update a counter with a string, with a multiplier of 1
+    #
+    # Examples
+    # 
+    # counter = NGramCounter.new(2)
+    # counter.update('01234')
+    #
+    # string: The String to update with
+    #
+    def update(string)
+      update_with_multiplier(string, 1)
+    end
+ 
+    # Public: get count for string, with default
+    #
+    # Examples
+    # 
+    # counter = NGramCounter.new(2)
+    # counter.update('01234')
+    # counter.count('01', 0)
+    # #=> 1
+    # counter.count('bob, 0)
+    # #=> 0
+    #
+    # ngram: The String to check
+    # if_not_found : what to update with
+    #
+    def count(ngram, if_not_found)
+      @counts.fetch(ngram, if_not_found)
+    end
+  end
+
+  # Public; A model for entropy
+  class Model
+    VERSION = '1.0.0'.freeze
+    attr_accessor :size, :map
+
+    def initialize(size)
+      @size = size
+      @map = {}
+      (1..size).each { |key| @map[key] = NGramCounter.new(key) }
+    end
+
+    # Public: update a model with a string, and a multiplier
+    #
+    # Examples
+    # 
+    # model = Model.new(2)
+    # model.update_with_multiplier('01234', 1)
+    #
+    # string: The String to update with
+    # multiplier: The Integer describing how much weight (will often be 1)
+    #
+    def update_with_multiplier(string, multiplier)
+      @map.each do |_, counter|
+        counter.update_with_multiplier(string, multiplier)
+      end
+    end
+
+    # Public: update a model with a string, with mulitplier or 1
+    #
+    # Examples
+    # 
+    # model = Model.new(2)
+    # model.update('01234')
+    #
+    # string: The String to update with
+    #
+    def update(string)
+      update_with_multiplier(string, 1)
+    end
+
+    # Public: log probability of a ngram string in a model
+    # returns value of first suffix of string
+    # or log_prob of a 1-gram appearing once if no suffix found
+    #
+    # Examples
+    # 
+    # model = Model.new(2)
+    # model.update('01234')
+    # model.log_prob('01')
+    # 
+    # string: The String to query
+    #
+    def log_prob(key)
+      last_total = 1
+      if @map.all? { |_, m| m.counts.empty? } || !key || key == ''
+        return Math.log(0, 2.0) # -Infinity
+      end
+
+      (1..key.size).each do |i|
+        k = key[-i..-1]
+        counter = @map.fetch(k.size, nil)
+        next unless counter
+        count = counter.counts.fetch(k, nil)
+        return Math.log(count, 2.0) - Math.log(counter.total, 2.0) if count
+        last_total = counter.total
+      end
+      # found it nowhere. Return '1 count' from last total
+      Math.log(1.0, 2.0) - Math.log(last_total, 2.0)
+    end
+
+    # Public: dump model to some io object
+    # 
+    # io: the IOWriter to write to
+    #
+    def dump(io)
+      @map.each do |k, m|
+        m.counts.each do |ngram, count|
+          io.write("#{k}\t#{ngram}\t#{count}\n")
+        end
+      end
+    end
+
+    # Public: predict the log_prob sum and average over a string
+    #         which will be split into ngrams
+    # 
+    # string: The String to query
+    # 
+    # returns: a dictionary of
+    #  - log_prob_total
+    #  - log_prob_average
+    #  - size (number of ngrams in string)
+    def predict(string)
+      ngrams = Entropic.sliding(string, @size)
+      log_prob_total = ngrams.map { |ngram| log_prob(ngram) }.inject(0.0, :+)
+      log_prob_average = log_prob_total / ngrams.size.to_f
+      { log_prob_total: log_prob_total, log_prob_average: log_prob_average, size: ngrams.size }
+    end
+
+    # Public: create a Model from reading from an IO object
+    #
+    # io: the IOReader
+    #
+    # returns: Model with stats filled in, and size of largest ngram
+    def self.read(io)
+      model = Model.new(0)
+      max_size = 0
+      io.each_line do |string|
+        ngram_size, ngram, count = string.strip.split(/\t/)
+        ngram_size = ngram_size.to_i
+        count = count.to_f
+        model.map[ngram_size] = NGramCounter.new(ngram_size) unless model.map.include?(ngram_size)
+        counter = model.map[ngram_size]
+        counter.total += count
+        counter.counts[ngram] = count
+        max_size = ngram_size if ngram_size > max_size
+      end
+      model.size = max_size
+      model
+    end
+
+    # Public: Train a model on a bunch of data, line by line
+    #
+    # io: the IOReader
+    # 
+    def train(io)
+      io.each_line do |string|
+        update(string)
+      end
+    end
+
+    # Public: Train a model on a bunch of data, line by line,
+    #         with a multiplier
+    #         each data line should be <string><tab><multiplier>
+    #
+    # io: the IOReader
+    # 
+    def train_with_multiplier(io)
+      io.each_line do |string|
+        text, count = string.strip.split(/\t/)
+        count = count.to_i
+        update_with_multiplier(text, count)
+      end
+    end
+  end
+end
+

metadata

@@ -0,0 +1,96 @@
+--- !ruby/object:Gem::Specification
+name: entropic
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Will Fitzgerald
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2018-01-12 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.14'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.14'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+description: Train and predict string entropy based on character n-grams.
+email:
+- willf@github.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rspec"
+- ".travis.yml"
+- Gemfile
+- README.md
+- Rakefile
+- bin/console
+- bin/setup
+- entropic.gemspec
+- lib/entropic.rb
+- lib/entropic/version.rb
+homepage: https://github.com/willf/entropic
+licenses: []
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.5.1
+signing_key: 
+specification_version: 4
+summary: Train and predict string entropy based on character n-grams.
+test_files: []