ots 0.4.2

data/README

@@ -0,0 +1,25 @@
+ots is an interface to libots - The open text summarizer
+
+INSTALL:
+  
+  sudo gem install ots --source http://gems.github.com
+
+REQUIREMENT:
+
+  * Ruby >= 1.8.7 ( >= 1.9.1 recommended)
+  * rubygems >= 1.3.5
+  * ruby development libraries    (debian: ruby1.8-dev, ruby1.9.1-dev)
+  * libxml2 development libraries (debian: libxml2-dev)
+  * libots development libraries  (debian: libots-dev)
+  * glib2.0 development libraries (debian: libglib2.0-dev)
+
+USAGE:
+
+  >> require "rubygems"
+  >> require "ots"
+  >> summarizer = ots.new
+  >> summarizer.parse("I think I need some ice cream to cool me off. It is too hot down under")
+  >> summarizer.title
+  => [ "hot","cool","cream","ice","think" ]
+  >> summarizer.summarize(:lines => 1)
+  => [ { :sentence => "I think I need some ice cream to cool me off", :score => 57 } ]

data/VERSION

@@ -0,0 +1 @@
+0.4.2

data/ext/extconf.rb

@@ -0,0 +1,14 @@
+require 'mkmf'
+
+$CFLAGS  = "-I/usr/include/glib-2.0 -I/usr/lib/glib-2.0/include -Wall"
+$LDFLAGS = "-lglib-2.0"
+
+dir_config("libots", ["/usr/local", "/opt/local", "/usr"])
+
+headers = [ 'stdio.h', 'stdlib.h', 'string.h', 'libots-1/ots/libots.h' ]
+if have_header('libots-1/ots/libots.h') && have_library('ots-1', 'ots_new_article', headers)
+  create_makefile 'ots'
+else
+  puts "Cannot find libots headers or libraries"
+  exit 1
+end

data/ext/ots.c

@@ -0,0 +1,197 @@
+#include <ruby.h>
+
+/* ruby 1.9 only */
+#ifdef RUBY_VM
+  #include <ruby/encoding.h>
+#endif
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include <libots-1/ots/libots.h>
+
+#define ID_CONST_GET rb_intern("const_get")
+#define CONST_GET(scope, constant) (rb_funcall(scope, ID_CONST_GET, 1, rb_str_new2(constant)))
+
+static VALUE rb_cOTS;
+static VALUE eLoadError;
+static VALUE eRuntimeError;
+static VALUE eArgumentError;
+
+typedef struct {
+  gchar *word;    /* the word */
+  gchar *stem;    /*stem of the word*/
+  gint occ;     /* how many times have we seen this word in the text? */
+} OtsWordEntery;
+
+
+/* helpers */
+
+OtsArticle* get_article(VALUE self, gboolean error_on_missing) {
+  VALUE rb_article_object = rb_iv_get(self, "@article");
+  if (rb_article_object == Qnil) {
+    if (error_on_missing)
+      rb_raise(eRuntimeError, "libots document not initialized properly. Did you forget to parse content ?");
+    else
+      return NULL;
+  }
+  return (OtsArticle *)DATA_PTR(rb_article_object);
+}
+
+void rb_ots_free_article(VALUE self) {
+  OtsArticle *article = DATA_PTR(rb_iv_get(self, "@article"));
+  ots_free_article(article);
+}
+
+VALUE rb_string(char *utf8) {
+  VALUE str = rb_str_new(utf8, strlen(utf8));
+
+  /* ruby 1.9 only - force bytestream to utf8 */
+  #ifdef RUBY_VM
+    rb_enc_associate(str, rb_to_encoding(rb_str_new2("UTF-8")));
+    ENC_CODERANGE_CLEAR(str);
+  #endif
+
+  return str;
+}
+
+/* ruby libots methods/wrappers */
+
+VALUE rb_ots_init(VALUE self) {
+  OtsArticle *article = get_article(self, FALSE);
+  VALUE dict = Qnil;
+  if (article != NULL) {
+    dict = rb_iv_get(self, "@dict");
+    ots_free_article(article);
+  }
+  article = ots_new_article();
+  rb_iv_set(self, "@article", Data_Wrap_Struct(rb_cObject, 0, 0, article));
+  rb_iv_set(self, "@dict", dict);
+  return self;
+}
+
+VALUE rb_ots_load_dictionary(VALUE self, VALUE dict) {
+  char *dict_cstr = "en";
+  if (dict != Qnil) dict_cstr = RSTRING_PTR(dict);
+
+  OtsArticle *article = get_article(self, FALSE);
+  if (article == NULL) {
+    rb_ots_init(self);
+    article = get_article(self, TRUE);
+  }
+
+  if (!ots_load_xml_dictionary(article, (unsigned const char *)dict_cstr)) {
+    rb_ots_free_article(self);
+    rb_raise(eLoadError, "Could not find dictionary file: %s", dict_cstr);
+  }
+
+  rb_iv_set(self, "@dict", dict);
+  return Qtrue;
+}
+
+VALUE rb_ots_parse_string(VALUE self, VALUE string) {
+  const unsigned char *string_cstr = (const unsigned char *)RSTRING_PTR(string);
+  size_t string_len = RSTRING_LEN(string);
+
+  rb_ots_init(self);
+  rb_ots_load_dictionary(self, rb_iv_get(self, "@dict"));
+  OtsArticle *article = get_article(self, TRUE);
+  ots_parse_stream(string_cstr, string_len, article);
+  ots_grade_doc(article);
+  return Qtrue;
+}
+
+VALUE rb_ots_highlight_lines(VALUE self, int lines) {
+  OtsArticle *article = get_article(self, TRUE);
+  ots_highlight_doc_lines(article, lines);
+  return Qtrue;
+}
+
+VALUE rb_ots_highlight_percent(VALUE self, int percent) {
+  OtsArticle *article = get_article(self, TRUE);
+  ots_highlight_doc(article, percent);
+  return Qtrue;
+}
+
+VALUE rb_ots_article_title(VALUE self) {
+  OtsArticle *article = get_article(self, TRUE);
+  if (article->title != NULL)
+    return rb_string(article->title);
+  else
+    return Qnil;
+}
+
+VALUE rb_ots_article_keywords(VALUE self) {
+  OtsArticle *article = get_article(self, TRUE);
+  GList* words = article->ImpWords;
+  VALUE iwords = rb_ary_new();
+  while (words != NULL) {
+    OtsWordEntery *data = (OtsWordEntery *)words->data;
+    if (data != NULL && strlen(data->word) > 0)
+      rb_ary_push(iwords, rb_string(data->word));
+    words = words->next;
+  }
+
+  return iwords;
+}
+
+VALUE rb_ots_get_highlighted_lines(VALUE self) {
+  OtsArticle *article = get_article(self, TRUE);
+  OtsSentence *sentence;
+  GList *curr_line = article->lines;
+  VALUE hlt_lines = rb_ary_new();
+
+  while (curr_line != NULL) {
+    sentence = (OtsSentence *)curr_line->data;
+    if (sentence->selected) {
+      size_t len;
+      unsigned char* content = ots_get_line_text(sentence, TRUE, &len);
+      VALUE hlt_line = rb_hash_new();
+      rb_hash_aset(hlt_line, ID2SYM(rb_intern("sentence")), rb_string((char *)content));
+      rb_hash_aset(hlt_line, ID2SYM(rb_intern("score")), LONG2FIX(sentence->score));
+      rb_ary_push(hlt_lines, hlt_line);
+    }
+    curr_line = g_list_next(curr_line);
+  }
+
+  return hlt_lines;
+}
+
+VALUE rb_summarize(VALUE self, VALUE options) {
+
+  VALUE lines = rb_hash_aref(options, ID2SYM(rb_intern("lines")));
+  VALUE percent = rb_hash_aref(options, ID2SYM(rb_intern("percent")));
+
+  if (lines != Qnil && percent != Qnil) {
+    rb_ots_free_article(self);
+    rb_raise(eArgumentError, "Cannot summarize on :lines & :percent, only one is allowed");
+  }
+  else if (lines == Qnil && percent == Qnil) {
+    rb_ots_free_article(self);
+    rb_raise(eArgumentError, "Need either :lines or :percent to summarize");
+  }
+
+  if (lines != Qnil)
+    rb_ots_highlight_lines(self, FIX2INT(lines));
+  else if (percent != Qnil) 
+    rb_ots_highlight_percent(self, FIX2INT(percent));
+  return rb_ots_get_highlighted_lines(self);
+}
+
+/* init */
+
+void Init_ots(void) {
+    eLoadError     = CONST_GET(rb_mKernel, "LoadError");
+    eRuntimeError  = CONST_GET(rb_mKernel, "RuntimeError");
+    eArgumentError = CONST_GET(rb_mKernel, "ArgumentError");
+    rb_cOTS = rb_define_class("OTS", rb_cObject);
+    rb_define_method(rb_cOTS, "load_dictionary", rb_ots_load_dictionary, 1);
+    rb_define_method(rb_cOTS, "parse", rb_ots_parse_string, 1);
+    rb_define_method(rb_cOTS, "highlight_lines", rb_ots_highlight_lines, 1);
+    rb_define_method(rb_cOTS, "highlight_percent", rb_ots_highlight_percent, 1);
+    rb_define_method(rb_cOTS, "highlighted_content", rb_ots_get_highlighted_lines, 0);
+    rb_define_method(rb_cOTS, "summarize", rb_summarize, 1);
+    rb_define_method(rb_cOTS, "title", rb_ots_article_title, 0);
+    rb_define_method(rb_cOTS, "keywords", rb_ots_article_keywords, 0);
+}

data/lib/ots.rb

@@ -0,0 +1 @@
+require File.join(File.dirname(__FILE__), %w(.. ext ots))

data/test/ots_test.rb

@@ -0,0 +1,62 @@
+require 'helper'
+
+class OTSTest < Test::Unit::TestCase
+
+  SAMPLE = <<-TEXT
+    The hawksbill turtle is a critically endangered sea turtle belonging to the family Cheloniidae.
+    It is the only species in its genus. The species has a worldwide distribution, with Atlantic and
+    Pacific subspecies.
+  TEXT
+
+  context 'Title' do
+    should 'extract title from given document' do
+      ots = OTS.new
+      ots.parse SAMPLE
+      assert_equal 'species,turtle,subspecies,pacific,atlantic', ots.title
+    end
+  end
+
+  context 'Keywords' do
+    should 'extract keywords from given document' do
+      ots = OTS.new
+      ots.parse SAMPLE
+      assert_equal %W(
+        species turtle subspecies pacific atlantic distribution worldwide genus cheloniidae family
+        belonging sea endangered critically hawksbill
+      ), ots.keywords
+    end
+  end
+
+  context 'Summary' do
+    should 'extract keywords from given document' do
+      ots = OTS.new
+      ots.parse SAMPLE
+      lines = ots.summarize(:lines => 2).map do |value|
+        { :sentence => value[:sentence].gsub(/\n\s*/, ' ').strip, :score => value[:score] }
+      end
+
+      assert_equal [
+        {
+          :sentence => "The hawksbill turtle is a critically endangered sea turtle belonging to the family Cheloniidae.",
+          :score    => 48
+        },
+        {
+          :sentence => "The species has a worldwide distribution, with Atlantic and Pacific subspecies.",
+          :score    => 20
+        }
+      ], lines
+
+    end
+
+    should 'utf8 encode strings properly' do
+      ots = OTS.new
+      text = "The hawksbill turtle\xE2\x80\x93is critically endangered."
+      text.force_encoding('UTF-8') if RUBY_VERSION >= "1.9"
+
+      ots.parse(text)
+      summary = ots.summarize(:lines => 1).first[:sentence]
+      assert_equal text, summary
+    end
+  end
+
+end

metadata

@@ -0,0 +1,82 @@
+--- !ruby/object:Gem::Specification 
+name: ots
+version: !ruby/object:Gem::Version 
+  prerelease: false
+  segments: 
+  - 0
+  - 4
+  - 2
+  version: 0.4.2
+platform: ruby
+authors: 
+- Bharanee Rathna
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2010-02-16 00:00:00 +11:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: shoulda
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        segments: 
+        - 2
+        - 10
+        version: "2.10"
+  type: :development
+  version_requirements: *id001
+description: Ruby interface to libots libraries for unix.
+email: deepfryed@gmail.com
+executables: []
+
+extensions: 
+- ext/extconf.rb
+extra_rdoc_files: 
+- README
+files: 
+- README
+- VERSION
+- ext/ots.c
+- lib/ots.rb
+- test/ots_test.rb
+- ext/extconf.rb
+has_rdoc: true
+homepage: http://github.com/deepfryed/ots
+licenses: []
+
+post_install_message: 
+rdoc_options: 
+- --charset=UTF-8
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.3.7
+signing_key: 
+specification_version: 3
+summary: Open Text Summarizer interface for Ruby.
+test_files: 
+- test/ots_test.rb