markov-ruby 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: e7dffa91c3e5e4a258f78dceda0418d5b047cb0e
+  data.tar.gz: e30b7d34af228f58425c9ab616b1a98f6129f0a7
+SHA512:
+  metadata.gz: bfad5c546e6b1dd84b0d1c09eb68e8bb3a341393d8a514e256e846f2793877e6c6be3671433812452362b898bb1364353a2a08e202d46a70edcf556c774161c1
+  data.tar.gz: 987c317d1dedcd637fbfb1f429c854cb6769fd6aed49c093732ff64c00539c0fe3e5025d0594ddfb771799d82b7176de33378d5a9a7cf169c2cad83e99a3513e

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+lib = File.expand_path("../../lib", __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+
+require "markov"
+require "irb"
+
+@source = "../test/fixtures/text_sample.txt"
+@dbname = "markov_test"
+@db = Markov::DB.new(dbname: @dbname, chunk: 4)
+@g = Markov::Generate.new("markov_test")
+
+IRB.start

data/lib/markov.rb

@@ -0,0 +1,15 @@
+require 'markov/generate'
+require 'markov/version'
+require 'markov/parser'
+require 'markov/db'
+require 'fileutils'
+require 'engtagger'
+require 'tmpdir'
+require 'byebug'
+require 'json'
+require 'csv'
+require 'cgi'
+require 'pg'
+
+module Markov
+end

data/lib/markov/db.rb

@@ -0,0 +1,92 @@
+module Markov
+  class DB
+    
+    def initialize options={}
+      @host = options[:host] || "localhost"
+      @user = options[:user] || "postgres"
+      @port = options[:port] || 5432
+      @dbname = options[:dbname] || ""
+      @password = options[:password] || ""
+      @chunk = options[:chunk] || 4
+    end
+    
+    def config
+      {
+        host: @host,
+        port: @port,
+        user: @user, 
+        dbname: @dbname, 
+        password: @password
+      }
+    end
+
+    def word_groups source, options={}
+      @word_groups ||= Parser.new(source).groups(@chunk, options)
+    end
+    
+    def split input
+      [input].flatten.inject(""){ |r,a|
+        r << [CGI.escape(a.split('/')[0]), a.split('/')[1] || "\"\""].join(",")
+      } 
+    end
+
+    def tmp_csv name, source
+      @dir = Dir.mktmpdir
+      @path = [@dir, name].join("/")
+      @tmp_csv = CSV.open(@path, "wb") do |csv|
+        word_groups(source, { tagged: true }).each do |g|
+          csv << [
+            name,
+            "{#{g[:prefix].map { |w| split(w) }.join(",")}}",
+            split(g[:suffix])
+          ]
+        end
+      end
+      @path
+    end
+    
+    def import_csv name, source
+      begin
+        @csv = tmp_csv(name, source)
+        @query = "COPY word_groups FROM '#{@dir}/#{name}' DELIMITER ',' CSV"
+        connection(@query)
+      ensure
+        @csv && FileUtils.remove_entry(File.dirname(@csv))
+        @word_groups = nil
+      end
+    end
+    
+    def lookup word, source
+      @query = "SELECT suffix, count(*) AS count
+                FROM word_groups 
+                WHERE prefix[5] = '#{word}'
+                AND source = '#{source}'
+                GROUP BY suffix"
+      connection(@query).values
+    end
+    
+    def csv_sources
+      @query = "SELECT DISTINCT source
+                FROM word_groups"
+      connection(@query).values.flatten
+    end
+
+    def json_sources
+      @query = "SELECT DISTINCT word_groups->'source'
+                FROM word_groups_jsonb"
+      connection(@query).values.flatten
+    end
+    
+    private
+        
+    def connection query=nil
+      begin
+        conn = PG.connect(config)
+        query && conn.exec(query)
+      ensure
+        conn.close
+      end
+    end
+    
+  end
+end

data/lib/markov/generate.rb

@@ -0,0 +1,43 @@
+module Markov
+  class Generate
+  
+    attr_reader :length, :weight, :start
+  
+    def initialize dbname, options={}
+      @length = options[:length] || 200
+      @weight = options[:weight] || 1.2
+      @start = options[:start] || "The"
+      @table = options[:table]
+      @db = DB.new(dbname: dbname, chunk: options[:chunk] || 4)
+    end
+    
+    def current_word
+      @current_word = @word || @start
+    end
+    
+    def next_word
+      word  = current_word.match(/^,/) ? "," : current_word.split(",")[0]
+      words = lookup(CGI.escape(word), @table)
+      index = words[:probability].sample
+      @word = CGI.unescape(words[:suffices][index])
+    end
+    
+    def lookup word, table
+      @lookup = @db.lookup(word, table)
+      {
+        suffices: @lookup.map { |s| s[0] },
+        probability: @lookup.map.each_with_index { |c,i|
+          [ (c[1].to_i ** @weight).round.times.inject([]){ |r,a| r << i } ]
+        }.flatten
+      }
+    end
+    
+    def text
+      (0..@length).inject("#@start "){ |r,a|
+        next_word
+        r << "#{current_word.match(/^,/) ? "," : current_word.split(",")[0]} "
+      }.strip.squeeze(" ").gsub(/\s(\.|\,|:|;|`|'|\?|!)/,"\\1")
+    end
+   
+  end
+end

data/lib/markov/parser.rb

@@ -0,0 +1,33 @@
+module Markov
+  class Parser
+    
+    attr_reader :source
+
+    def initialize source, options={}
+      @source = source
+    end
+  
+    def raw_text
+      @raw_text ||= File.read(@source).scrub!
+    end
+
+    def tagged_text
+      tagger = EngTagger.new
+      @tagged_text ||= tagger.get_readable(raw_text)
+    end
+    
+    def groups chunk_size, options={}
+      text = options[:tagged] ? tagged_text : raw_text
+      words = text.split
+      (chunk_size - 1).times { |r,a| words.unshift("\"\"") }
+
+      words.each_cons(chunk_size).to_a.inject([]){ |r,a| 
+        r << { 
+          prefix: a[0..(chunk_size - 2)],
+          suffix: a.last 
+        } 
+      }
+    end
+
+  end
+end

data/lib/markov/version.rb

@@ -0,0 +1,3 @@
+module Markov
+  VERSION = "0.0.1".freeze
+end

metadata

@@ -0,0 +1,133 @@
+--- !ruby/object:Gem::Specification
+name: markov-ruby
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- rob allen
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2016-07-05 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: engtagger
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.2.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.2.0
+- !ruby/object:Gem::Dependency
+  name: pg
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.18.4
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.18.4
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.7'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.7'
+- !ruby/object:Gem::Dependency
+  name: guard
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 2.14.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 2.14.0
+- !ruby/object:Gem::Dependency
+  name: guard-minitest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 2.4.5
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 2.4.5
+- !ruby/object:Gem::Dependency
+  name: byebug
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 9.0.5
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 9.0.5
+description: A postgresql backed markov text generator.
+email: rob.all3n@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- bin/console
+- lib/markov.rb
+- lib/markov/db.rb
+- lib/markov/generate.rb
+- lib/markov/parser.rb
+- lib/markov/version.rb
+homepage: ''
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.5.1
+signing_key: 
+specification_version: 4
+summary: A postgresql backed markov text generator.
+test_files: []