crm114 1.0.0

data/CHANGELOG

@@ -0,0 +1,3 @@
+== 1.0.0 / 2006-11-06
+
+* Initial release of CRM114 library for Ruby.

data/LICENSE

@@ -0,0 +1,19 @@
+Copyright (c) 2005-2006 Arto Bendiken <http://bendiken.net/>
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to
+deal in the Software without restriction, including without limitation the
+rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+sell copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+IN THE SOFTWARE.

data/Manifest.txt

@@ -0,0 +1,8 @@
+CHANGELOG
+LICENSE
+Manifest.txt
+README
+Rakefile
+lib/crm114.rb
+test/test_code_or_text.rb
+test/test_crm114.rb

data/README

@@ -0,0 +1,59 @@
+== CRM114 Controllable Regex Mutilator for Ruby
+
+This is a Ruby interface to the CRM114 Controllable Regex Mutilator, an
+advanced and fast text classifier that uses sparse binary polynomial
+matching with a Bayesian Chain Rule evaluator and a hidden Markov model to
+categorize data with up to a 99.87% accuracy.
+
+The Ruby wrapper grew out of this:
+* http://bendiken.net/2006/07/05/spam-filters-alien-technology-and-ruby-on-rails
+
+=== About CRM114
+
+* http://crm114.sourceforge.net
+* http://en.wikipedia.org/wiki/CRM114
+* http://en.wikipedia.org/wiki/Dr_Strangelove
+* http://www.paulgraham.com/wsy.html
+
+== Download
+
+* http://rubyforge.org/projects/crm114
+* gem install crm114
+* svn checkout svn://rubyforge.org/var/svn/crm114
+
+== Dependencies
+
+Requires the CRM114 binaries to be installed. Specifically, the '+crm+'
+binary should be accessible in the current user's PATH environment variable.
+
+== Usage
+
+The CRM114 library interface is very similar to that of the
+Classifier[http://rubyforge.org/projects/classifier/] project.
+
+Here follows a brief example:
+
+  require 'crm114'
+  crm = Classifier::CRM114.new([:interesting, :boring])
+  crm.train! :interesting, 'Some data set with a decent signal to noise ratio.'
+  crm.train! :boring, 'Pig latin, as in lorem ipsum dolor sit amet.'
+  crm.classify 'Lorem ipsum'       => [:boring, 0.99]
+  crm.interesting? 'Lorem ipsum'   => false
+  crm.boring? 'Lorem ipsum'        => true
+
+Have a look at the included unit tests for more comprehensive examples.
+
+== Related Projects
+
+* http://www.elegantchaos.com/node/129 (crm.py)
+* http://rubyforge.org/projects/classifier/
+* http://rubyforge.org/projects/bishop/
+
+== Author
+
+Arto Bendiken (mailto:arto.bendiken@gmail.com) - http://bendiken.net
+
+== License
+
+Released under the terms of the MIT license. See the accompanying LICENSE
+file for more information.

data/Rakefile

@@ -0,0 +1,41 @@
+$:.unshift(File.expand_path(File.dirname(__FILE__) + '/lib'))
+
+require 'rubygems'
+require 'crm114'
+
+PKG_NAME    = 'crm114'
+PKG_VERSION = Classifier::CRM114::VERSION
+PKG_DESC    = 'Ruby interface to the CRM114 Controllable Regex Mutilator text classification engine.'
+PKG_URL     = 'http://crm114.rubyforge.org/'
+
+PKG_AUTHOR  = 'Arto Bendiken'
+PKG_EMAIL   = 'arto.bendiken@gmail.com'
+
+##############################################################################
+
+require 'hoe'
+
+Hoe.new(PKG_NAME, PKG_VERSION) do |p|
+  p.author = PKG_AUTHOR
+  p.email = PKG_EMAIL
+  p.url = PKG_URL
+  p.summary = PKG_DESC
+  p.description = p.paragraphs_of('README', 1).first
+  p.changes = p.paragraphs_of('CHANGELOG', 0..1).join("\n\n")
+  p.spec_extras = { :rdoc_options => ['--main', 'README'] }
+end
+
+##############################################################################
+
+def egrep(pattern, files)
+  Dir[files].each do |file|
+    File.open(file).readlines.each_with_index do |line, lineno|
+      puts "#{file}:#{lineno + 1}:#{line}" if line =~ pattern
+    end
+  end
+end
+
+desc 'Look for TODO and FIXME tags in the code base.'
+task :todo do
+  egrep /#.*(FIXME|TODO)/, '**/*.rb'
+end

data/lib/crm114.rb

@@ -0,0 +1,88 @@
+# Author::    Arto Bendiken (mailto:arto.bendiken@gmail.com)
+# Copyright:: Copyright (c) 2006 Arto Bendiken.
+# License::   MIT
+
+module Classifier
+
+  class CRM114
+
+    VERSION = '1.0.0'
+
+    CLASSIFICATION_TYPE = '<osb unique microgroom>'
+    FILE_EXTENSION = '.css'
+    CMD_CRM = '/usr/bin/env crm'
+    OPT_LEARN = '-{ learn %s ( %s ) }'
+    OPT_CLASSIFY = '-{ isolate (:stats:); classify %s ( %s ) (:stats:); match [:stats:] (:: :best: :prob:) /Best match to file .. \\(%s\\/([[:graph:]]+)\\%s\\) prob: ([0-9.]+)/; output /:*:best:\\t:*:prob:/ }'
+
+    # Returns a string containg the installed CRM114 engine version in a
+    # format such as "20060118-BlameTheReavers".
+    def self.version
+      $1 if IO.popen(CMD_CRM + ' -v', 'r') { |pipe| pipe.readline } =~ /CRM114, version ([\d\w\-\.]+)/
+    end
+
+    # Returns a new CRM114 classifier defined by the given _categories_.
+    def initialize(categories, options = {})
+      @categories = categories.to_a.collect { |category| category.to_s.to_sym }
+      @path = File.expand_path(options[:path] || '.')
+      @debug = options[:debug] || false
+    end
+
+    # Trains the classifier to consider the given _text_ to be a sample from
+    # the set named by _category_.
+    def learn!(category, text, &block)
+      cmd = CMD_CRM + " '" + (OPT_LEARN % [CLASSIFICATION_TYPE, css_file_path(category)]) + "'"
+      puts cmd if @debug
+      IO.popen(cmd, 'w') { |pipe| block_given? ? block.call(pipe) : pipe.write(text) }
+    end
+
+    alias_method :train!, :learn!
+
+    def unlearn!(category, text, &block) # :nodoc:
+      raise 'unlearning not supported at present'
+    end
+
+    alias_method :untrain!, :unlearn! #:nodoc:
+
+    # Returns the classification of the provided _text_ as a tuple
+    # containing the highest-probability category and a confidence indicator
+    # in the range of 0.5..1.0.
+    def classify(text = nil, &block)
+      files = @categories.collect { |category| css_file_path(category) }
+      cmd = CMD_CRM + " '" + (OPT_CLASSIFY % [CLASSIFICATION_TYPE, files.join(' '), @path.gsub(/\//, '\/'), FILE_EXTENSION]) + "'"
+      puts cmd if @debug
+      result = IO.popen(cmd, 'r+') do |pipe|
+        block_given? ? block.call(pipe) : pipe.write(text)
+        pipe.close_write
+        pipe.readline unless pipe.closed? || pipe.eof?
+      end
+      return [nil, 0.0] unless result && result.include?("\t")
+      result = result.split("\t")
+      [result.first.to_sym, result.last.to_f]
+    end
+
+    def method_missing(symbol, *args) # :nodoc:
+      case symbol.to_s[-1]
+        when ?!
+          category = symbol.to_s.chop.to_sym
+          return learn!(category, *args) if @categories.include?(category)
+        when ?? # it's a predicate
+          category = symbol.to_s.chop.to_sym
+          return classify(*args).first == category if @categories.include?(category)
+      end
+      super
+    end
+
+    protected
+
+      def self.create_css_file(file)
+        cmd = CMD_CRM + " '" + (OPT_LEARN % [CLASSIFICATION_TYPE, file]) + "'"
+        IO.popen(cmd, 'w') { |pipe| pipe.close }
+      end
+
+      def css_file_path(category)
+        File.join(@path, category.to_s + FILE_EXTENSION)
+      end
+
+  end
+
+end

data/test/test_code_or_text.rb

@@ -0,0 +1,29 @@
+$:.unshift(File.expand_path(File.dirname(__FILE__) + '/../lib'))
+
+require 'test/unit'
+require 'crm114'
+
+class TestCodeOrText < Test::Unit::TestCase
+
+  def setup
+   @path = File.dirname(__FILE__)
+   @crm = Classifier::CRM114.new([:code, :text], :path => @path)
+   assert_nothing_raised do
+     Dir["#{@path}/../lib/*.rb"].each { |file| @crm.code! File.read(file) }
+     ['CHANGELOG', 'README', 'LICENSE'].each { |file| @crm.text! File.read(file) }
+   end
+  end
+
+  def teardown
+    Dir["#{@path}/*.css"].each { |file| File.delete(file) }
+  end
+
+  def test_code
+    assert @crm.code?('class DrStrangelove; def self.lesson; stop_worrying && love_the_bomb; end; end')
+  end
+
+  def test_text
+    assert @crm.text?('This an interface to the Dr. Strangelove-inspired CRM114 Controllable Regex Mutilator.')
+  end
+
+end

data/test/test_crm114.rb

@@ -0,0 +1,61 @@
+$:.unshift(File.expand_path(File.dirname(__FILE__) + '/../lib'))
+
+require 'test/unit'
+require 'crm114'
+
+class TestCRM114 < Test::Unit::TestCase
+
+  def setup
+    @path = File.dirname(__FILE__)
+    @crm = Classifier::CRM114.new([:interesting, :boring], :path => @path)
+    assert_nothing_raised do
+      @crm.train! :interesting, <<EOT
+        Computational processes are abstract beings that inhabit computers.
+        As they evolve, processes manipulate other abstract things called
+        data. The evolution of a process is directed by a pattern of rules
+        called a program. People create programs to direct processes. In
+        effect, we conjure the spirits of the computer with our spells.
+EOT
+      @crm.train! :boring, <<EOT
+        Lorem ipsum dolor sit amet, consectetuer adipiscing elit. Nam vitae
+        nisi nec sapien congue porttitor. Proin quam risus, pharetra non,
+        lacinia sed, vehicula non, eros. Suspendisse velit augue, aliquet
+        vel, sagittis vitae, porttitor sed, metus. Integer tortor tellus,
+        tempus tincidunt, viverra a, fringilla vitae, sapien. Ut ac eros.
+        Donec molestie nulla sed nibh. Pellentesque quam quam, vehicula sed,
+        venenatis vitae, tristique quis, lectus. Aenean odio purus, pharetra
+        non, facilisis sed, rutrum eu, lectus. Curabitur odio. Ut laoreet
+        dolor vitae nunc. Donec dapibus. Morbi tempor libero et dolor.
+        Aliquam rutrum metus quis nibh. Ut pharetra turpis vel metus. Donec
+        vel arcu. Sed neque orci, accumsan et, faucibus quis, porttitor in,
+        lacus. Lorem ipsum dolor sit amet, consectetuer adipiscing elit.
+        Phasellus et arcu. Mauris nunc.
+EOT
+    end
+  end
+
+  def teardown
+    Dir["#{@path}/*.css"].each { |file| File.delete(file) }
+  end
+
+  def test_version
+    assert_match(/^[\d]+-[\w\d]+$/, Classifier::CRM114.version)
+  end
+
+  def test_unlearning
+    assert_raise(RuntimeError) { @crm.unlearn!(:boring, 'Lorem ipsum') }
+  end
+
+  def test_interesting
+    assert_equal(:interesting, @crm.classify('Thus, programs must be written for people to read,').first)
+    assert_equal(true, @crm.interesting?('and only incidentally for machines to execute.'))
+    assert_equal(false, @crm.boring?('learning to program is considerably less dangerous than learning sorcery'))
+  end
+
+  def test_boring
+    assert_equal(:boring, @crm.classify('Lorem ipsum dolor sit amet, sed neque orci.').first)
+    assert_equal(false, @crm.interesting?('Donec dapibus. Morbi tempor libero et dolor.'))
+    assert_equal(true, @crm.boring?('Aliquam rutrum metus quis nibh. Ut pharetra turpis vel metus.'))
+  end
+
+end

metadata

@@ -0,0 +1,63 @@
+--- !ruby/object:Gem::Specification 
+rubygems_version: 0.9.0
+specification_version: 1
+name: crm114
+version: !ruby/object:Gem::Version 
+  version: 1.0.0
+date: 2006-11-06 00:00:00 +01:00
+summary: Ruby interface to the CRM114 Controllable Regex Mutilator text classification engine.
+require_paths: 
+- lib
+- test
+email: arto.bendiken@gmail.com
+homepage: http://crm114.rubyforge.org/
+rubyforge_project: crm114
+description: This is a Ruby interface to the CRM114 Controllable Regex Mutilator, an advanced and fast text classifier that uses sparse binary polynomial matching with a Bayesian Chain Rule evaluator and a hidden Markov model to categorize data with up to a 99.87% accuracy.
+autorequire: 
+default_executable: 
+bindir: bin
+has_rdoc: true
+required_ruby_version: !ruby/object:Gem::Version::Requirement 
+  requirements: 
+  - - ">"
+    - !ruby/object:Gem::Version 
+      version: 0.0.0
+  version: 
+platform: ruby
+signing_key: 
+cert_chain: 
+post_install_message: 
+authors: 
+- Arto Bendiken
+files: 
+- CHANGELOG
+- LICENSE
+- Manifest.txt
+- README
+- Rakefile
+- lib/crm114.rb
+- test/test_code_or_text.rb
+- test/test_crm114.rb
+test_files: []
+
+rdoc_options: 
+- --main
+- README
+extra_rdoc_files: []
+
+executables: []
+
+extensions: []
+
+requirements: []
+
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: hoe
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Version::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: 1.1.2
+    version: