ruby_nlp 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: be92ab0a6b600f0f21aa400c53ee1cf110e41f6d
+  data.tar.gz: d661fcb65a06a5772b9805701d6f830997604a49
+SHA512:
+  metadata.gz: a0ebce070e03ffe77a815329f4a36cfb456d2ef640c8c08a8803331a71cf2e4c2d1b4df6c9cbc80047085b410d587b40326a84a956b2b26d704a2ad7a4e7834c
+  data.tar.gz: 4913d4f7711ff42eb7d58ddd5ee340c1f35baaeb768c5f1857765f3dbba9b6da9c98e60361ddb5f06e3576ac09e4cf754cdcaf3e63953ed6f6663a3cc8f3baec

data/README.md

@@ -0,0 +1,88 @@
+# Ruby NLP
+
+This repository is a pure Ruby toolkit for NLP. It's the examples part of an ongoing series I'm writing on SitePoint.com, so watch this repository for updates!
+
+## Installing
+
+Simply install the gem:
+
+```sh
+gem install ruby_nlp
+```
+
+## What's In This Project?
+
+### *n*-grams
+
+A simple class to generate *n*-grams from an input string, either per word, per character, or on some custom splitter.
+
+```ruby
+require 'ruby_nlp/ngram'
+ngram = Ngram.new("foo bar lux win")
+
+ngram.unigrams # => [["foo"], ["bar"], ["lux"], ["win"]]
+ngram.bigrams # => [["foo", "bar"], ["bar", "lux"], ["lux", "win"]]
+ngram.trigrams # => [["foo", "bar", "lux"], ["bar", "lux", "win"]]
+ngram.ngrams(4) # => [["foo", "bar", "lux", "win"]]
+
+ngram = Ngram.new("abcde", regex: //)
+ngram.unigrams # => [["a"], ["b"], ["c"], ["d"], ["e"]]
+```
+
+### Corpus Extractor
+
+There is a basic corpus loader. It takes a glob for the files that are part of the corpus, and a class to delegate the sentence extraction to.
+
+Currently, support for the following corpus types are included:
+
+* Brown Corpus, in the `BrownCorpusFile` class at `ruby_nlp/corpus_files/brown`. This corpus currently strips the tagging to reveal only the raw sentences in the corpus.
+
+An example of this in action:
+
+```ruby
+require 'ruby_nlp/corpus'
+require 'ruby_nlp/corpus_files/brown'
+
+corpus = Corpus.new('brown/c*', BrownCorpusFile)
+corpus.trigrams # => One *really* big array of trigrams!
+
+corpus.files # => `Array` of `BrownCorpusFile` instances, one for each file found.
+corpus.sentences # => Flattened `Array` of sentences.
+```
+
+## Examples
+
+Some examples are in the `examples` directory, one per article from SitePoint.com in this series.
+
+## Future Work
+
+* Support for other corpora.
+* Preservation, filtering and searching of tags in corpora.
+* Markov modelling and chaining.
+* Text generation using Markov modelling.
+* Part of speech tagging.
+* Classification.
+
+## License
+
+The MIT License (MIT)
+
+Copyright (c) 2013 Nathan Kleyn
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/lib/ruby_nlp/corpus.rb

@@ -0,0 +1,38 @@
+require 'ruby_nlp/ngram'
+
+class Corpus
+  def initialize(glob, klass)
+    @glob = glob
+    @klass = klass
+  end
+
+  def files
+    @files ||= Dir[@glob].map do |file|
+      @klass.new(file)
+    end
+  end
+
+  def sentences
+    files.map do |file|
+      file.sentences
+    end.flatten
+  end
+
+  def ngrams(n)
+    sentences.map do |sentence|
+      Ngram.new(sentence).ngrams(n)
+    end.flatten(1)
+  end
+
+  def unigrams
+    ngrams(1)
+  end
+
+  def bigrams
+    ngrams(2)
+  end
+
+  def trigrams
+    ngrams(3)
+  end
+end

data/lib/ruby_nlp/corpus_files/brown.rb

@@ -0,0 +1,19 @@
+class BrownCorpusFile
+  def initialize(path)
+    @path = path
+  end
+
+  def sentences
+    @sentences ||= File.open(@path) do |file|
+      file.each_line.each_with_object([]) do |line, acc|
+        stripped_line = line.strip
+
+        unless stripped_line.nil? || stripped_line.empty?
+          acc << line.split(' ').map do |word|
+            word.split('/').first
+          end.join(' ')
+        end
+      end
+    end
+  end
+end

data/lib/ruby_nlp/ngram.rb

@@ -0,0 +1,24 @@
+class Ngram
+  attr_accessor :options
+
+  def initialize(target, options = { regex: / / })
+    @target = target
+    @options = options
+  end
+
+  def ngrams(n)
+    @target.split(@options[:regex]).each_cons(n).to_a
+  end
+
+  def unigrams
+    ngrams(1)
+  end
+
+  def bigrams
+    ngrams(2)
+  end
+
+  def trigrams
+    ngrams(3)
+  end
+end

metadata

@@ -0,0 +1,63 @@
+--- !ruby/object:Gem::Specification
+name: ruby_nlp
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Nathan Kleyn
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-09-14 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 2.14.1
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 2.14.1
+description: A simple NLP toolkit in pure Ruby. See http://github.com/nathankleyn/ruby_nlp
+  for more information.
+email: nathan@nathankleyn.com
+executables: []
+extensions: []
+extra_rdoc_files:
+- README.md
+files:
+- README.md
+- lib/ruby_nlp/corpus_files/brown.rb
+- lib/ruby_nlp/corpus.rb
+- lib/ruby_nlp/ngram.rb
+homepage: http://github.com/nathankleyn/ruby_nlp
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.0.3
+signing_key: 
+specification_version: 4
+summary: A simple NLP toolkit in pure Ruby.
+test_files: []