summarize 1.0.0

data/.gitignore

@@ -0,0 +1,11 @@
+Makefile
+lib/ots/ots.bundle
+tmp/*
+*.DS_Store
+coverage/*
+*.swp
+*.swo
+pkg/*
+*.gem
+
+!.keep

data/README.markdown

@@ -0,0 +1,42 @@
+# Summarize
+
+## A Ruby C wrapper for Open Text Summarizer
+
+## Install
+
+### Manual install
+
+    git clone https://github.com/ssoper/summarize.git
+    cd summarize
+    rake build # install the rake-compiler gem if you dont already have it
+    gem build summarize.gemspec
+    gem install summarize-1.0.gem
+
+## Usage
+
+The summarize method is added to File which you can use to summarize the contents of any plain text file
+
+    File.open('path/to/file').summarize
+    
+Or use the String method
+
+    "text to summarize".summarize
+
+By default it uses an English dictionary for summarizing but forty languages are supported. Pass in the valid ISO 639 language code to use one. A ratio (default is 25%) can also be passed in.
+
+    # Parse an article using Portuguese stemming rules with a ratio of 50%
+    "text to summarize".summarize(:language => 'pt', :ratio => 50)
+
+You can also use custom stemming rules
+
+    "text to summarize".summarize(:dictionary => 'path/to/custom/dictionary')
+
+## Dependencies
+
+You must have glib-2.0 and libxml-2.0 installed and properly configured.
+    
+## Author
+
+Gem written by Sean Soper ([@ssoper](http://twitter.com/ssoper))
+
+The Open Text Summarizer library was written by Nadav Rotem and can be found at <http://libots.sourceforge.net/>

data/Rakefile

@@ -0,0 +1,49 @@
+require "rubygems"
+require "rake"
+require "benchmark"
+
+# Built using rake-compiler (https://github.com/luislavena/rake-compiler)
+begin
+  require "rake/extensiontask"
+rescue LoadError
+  abort "\nYou must install the rake-compiler gem\n\n  gem install rake-compiler\n\n"
+end
+
+Rake::ExtensionTask.new("summarize") do |extension|
+  extension.lib_dir = "lib/summarize"
+end
+
+task :build => [:clean, :compile]
+
+task :test => :build do
+  $:.unshift(File.join(File.dirname(__FILE__), "lib"))
+  require "summarize"
+  require "test/unit"
+
+  class TestSummarizer < Test::Unit::TestCase
+    def setup
+      @jupiter = File.read(File.join(File.dirname(__FILE__), 'sample_data/jupiter.txt'))
+    end
+
+    def test_ratios
+      content, ratio_25, ratio_50 = @jupiter.split('||').collect { |text| text.strip }
+      assert content.summarize(:ratio => 25) == ratio_25
+      assert content.summarize(:ratio => 50) == ratio_50
+    end
+
+    def test_unsupported_language
+      content = @jupiter.split('||').first
+      assert_raise RuntimeError do
+        content.summarize(:language => 'nonexistent')
+      end
+    end
+
+    def test_invalid_dictionary_file
+      content = @jupiter.split('||').first
+      assert_raise RuntimeError do
+        content.summarize(:dictionary => '/path/to/nowhere')
+      end
+    end
+  end
+
+end

data/ext/summarize/article.c

@@ -0,0 +1,119 @@
+/*
+ *  article.c
+ *
+ *  Copyright (C) 2003 Nadav Rotem <nadav256@hotmail.com>
+ * 
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 2 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU Library General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program; if not, write to the Free Software
+ *  Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include "libots.h"
+#include "grader-tc.h"
+
+extern void ots_free_TF_wordlist (GList * aList);
+
+#define MAX_WORD_LENGTH 35
+
+/*Data structure related functions*/
+
+OtsSentence *
+ots_new_sentence (void)
+{
+  OtsSentence *aLine = g_new0 (OtsSentence, 1);
+  aLine->words = NULL;
+  aLine->wc = 0;
+  aLine->selected = 0;
+  aLine->score = 0;
+  return aLine;
+}
+
+void
+ots_free_sentence (OtsSentence * sen)
+{
+  if (sen != NULL)
+    {
+      g_list_foreach (sen->words, (GFunc) g_free, NULL);
+      g_list_free (sen->words);
+      g_free (sen);
+    }
+sen=NULL;
+}
+
+OtsArticle *
+ots_new_article (void)
+{
+  OtsArticle *Doc;
+  Doc = g_new0 (OtsArticle, 1);
+  Doc->lineCount = 0;
+  Doc->title = NULL;
+  Doc->stem=new_stem_rule ();
+  Doc->lines=NULL;
+  Doc->dict = NULL;
+  Doc->ImpWords = NULL;
+  Doc->wordStat = NULL;
+  
+  Doc->tf_terms=NULL;
+  return Doc;
+}
+
+void
+ots_free_article (OtsArticle * art)
+{
+  if (NULL != art)
+    {
+      free_stem_rule (art->stem);
+  	   ots_free_wordlist (art->dict);
+  	   ots_free_wordlist (art->ImpWords);
+	   ots_free_wordlist (art->wordStat);
+      
+      ots_free_TF_wordlist(art->tf_terms);
+      
+	   g_list_foreach (art->lines, (GFunc) ots_free_sentence, NULL);
+      g_list_free (art->lines);
+      
+      if (art->title != NULL) g_free (art->title);
+      g_free (art);
+    } 
+art=NULL;
+}
+
+OtsSentence *
+ots_append_line (OtsArticle * Doc)
+{
+  OtsSentence *aLine = ots_new_sentence ();
+  Doc->lineCount++;
+  Doc->lines = g_list_append (Doc->lines, aLine);
+  return aLine;
+}
+
+void
+ots_append_word (OtsSentence * aLine,unsigned const char *aWord)
+{
+  if ((aWord == NULL) || (0==strlen(aWord)) ||(NULL==aLine)) return;
+  aLine->wc++;
+  aLine->words = g_list_append (aLine->words, (gpointer) g_strdup (aWord));
+  return;
+}
+
+
+gboolean
+ots_is_line_selected(const OtsSentence *aLine)
+{
+  if (aLine==NULL) {printf("Warning:Line=NULL\n"); return FALSE;}
+  return (aLine->selected);
+}

data/ext/summarize/dic/bg.xml

@@ -0,0 +1,101 @@
+<?xml version="1.0"?>
+<dictionary lang="bulgarian">
+ <stemmer>
+     <step1_pre> 
+	  <rule>"|</rule> 
+	  <rule>(|</rule> 
+	 </step1_pre>
+	 
+	 
+    <step1_post>
+      <rule>."|</rule> 
+	  <rule>,"|</rule> 
+	  <rule>.|</rule>
+	  <rule>,|</rule> 
+	  <rule>"|</rule>
+	  <rule>)|</rule> 
+	  <rule>?|</rule> 
+	  <rule>:|</rule> 
+	  <rule>;|</rule> 
+	  <rule>!|</rule> 
+	 </step1_post>
+  
+  
+    <manual>
+	  <rule>wrote|write</rule>
+	  <rule>came|come</rule> 
+	  <rule>went|go</rule>
+	 </manual>
+  
+    <post>
+       <rule>before1|1after</rule>
+    </post>
+   <pre>
+       <rule>before1|1after</rule>
+   </pre> 
+  </stemmer>
+      <parser>
+    
+ 	 <linebreak>
+	  <rule>."</rule>
+	  <rule>?"</rule>
+	  <rule>!"</rule>
+	  <rule>,"</rule>
+	  <rule>.</rule>
+	  <rule>?</rule>  
+	  <rule>;</rule>
+	  <rule>|</rule>
+	  <rule>!</rule>  
+	 </linebreak>
+	 
+ 	 <linedontbreak>
+	  <rule>Dr.</rule>
+	  <rule>Mr.</rule>  
+	  <rule>Mrs.</rule>
+	  <rule>U.S.</rule>  
+	  <rule>Rep.</rule>  
+	  <rule>Sen.</rule>  
+	 </linedontbreak>
+   </parser>
+ <grader-tc>
+<word>август</word>
+<word>април</word>
+<word>в</word>
+<word>всеки</word>
+<word>всичко</word>
+<word>вторник</word>
+<word>да</word>
+<word>декември</word>
+<word>за</word>
+<word>и</word>
+<word>или</word>
+<word>има</word>
+<word>което</word>
+<word>към</word>
+<word>май</word>
+<word>март</word>
+<word>на</word>
+<word>не</word>
+<word>неделя</word>
+<word>ноември</word>
+<word>октомври</word>
+<word>от</word>
+<word>петък</word>
+<word>по</word>
+<word>понеделник</word>
+<word>при</word>
+<word>с</word>
+<word>септември</word>
+<word>сряда</word>
+<word>сто</word>
+<word>събота</word>
+<word>трябва</word>
+<word>февруари</word>
+<word>хиляда</word>
+<word>че</word>
+<word>четвъртък</word>
+<word>юли</word>
+<word>юни</word>
+<word>януари</word>
+ </grader-tc>
+</dictionary>

data/ext/summarize/dic/ca.xml

@@ -0,0 +1,141 @@
+<?xml version="1.0"?>
+<dictionary lang="catalan">
+ <stemmer> 
+     <step1_pre> 
+	  <rule>"|</rule> 
+	  <rule>(|</rule> 
+	 </step1_pre>
+	 
+	 
+    <step1_post>
+      <rule>."|</rule> 
+	  <rule>,"|</rule> 
+	  <rule>.|</rule>
+	  <rule>,|</rule> 
+	  <rule>"|</rule>
+	  <rule>)|</rule> 
+	  <rule>?|</rule> 
+	  <rule>:|</rule> 
+	  <rule>;|</rule> 
+	  <rule>!|</rule> 
+	 </step1_post>
+ 
+ 
+    <manual>
+	  <rule>wrote|write</rule>
+	  <rule>came|come</rule> 
+	  <rule>went|go</rule>
+	 </manual>
+	 
+    <post>
+       <rule>before1|1after</rule>
+    </post>
+   <pre>
+       <rule>before1|1after</rule>
+   </pre> 
+  </stemmer>
+   <parser>
+    
+ 	 <linebreak>
+	  <rule>."</rule>
+	  <rule>?"</rule>
+	  <rule>!"</rule>
+	  <rule>,"</rule>
+	  <rule>.</rule>
+	  <rule>?</rule>  
+	  <rule>;</rule>
+	  <rule>|</rule>
+	  <rule>!</rule>  
+	 </linebreak>
+	 
+ 	 <linedontbreak>
+	  <rule>Dr.</rule>
+	  <rule>Mr.</rule>  
+	  <rule>Mrs.</rule>
+	  <rule>U.S.</rule>  
+	  <rule>Rep.</rule>  
+	  <rule>Sen.</rule>  
+	 </linedontbreak>
+   </parser>
+ <grader-tc>
+<word>a</word>
+<word>abans</word>
+<word>al</word>
+<word>amb</word>
+<word>ambdós</word>
+<word>anar</word>
+<word>ara</word>
+<word>baix</word>
+<word>cap</word>
+<word>cert</word>
+<word>com</word>
+<word>cuál</word>
+<word>damunt</word>
+<word>de</word>
+<word>dins</word>
+<word>doble</word>
+<word>dos</word>
+<word>dues</word>
+<word>el</word>
+<word>ell</word>
+<word>ella</word>
+<word>elles</word>
+<word>ells</word>
+<word>els</word>
+<word>en</word>
+<word>ésser</word>
+<word>estar</word>
+<word>excepte</word>
+<word>jo</word>
+<word>la</word>
+<word>les</word>
+<word>lluny</word>
+<word>lo</word>
+<word>los</word>
+<word>mai</word>
+<word>me</word>
+<word>meu</word>
+<word>meus</word>
+<word>meva</word>
+<word>meves</word>
+<word>mí</word>
+<word>na</word>
+<word>nos</word>
+<word>nosaltres</word>
+<word>nostra</word>
+<word>nostre</word>
+<word>nostres</word>
+<word>qual</word>
+<word>quals</word>
+<word>quan</word>
+<word>quelcom</word>
+<word>quin</word>
+<word>quina</word>
+<word>quines</word>
+<word>quins</word>
+<word>se</word>
+<word>ser</word>
+<word>seu</word>
+<word>seus</word>
+<word>seva</word>
+<word>seves</word>
+<word>sí</word>
+<word>tenir</word>
+<word>teu</word>
+<word>teus</word>
+<word>teva</word>
+<word>teves</word>
+<word>tu</word>
+<word>u</word>
+<word>un</word>
+<word>una</word>
+<word>unes</word>
+<word>uns</word>
+<word>vosaltres</word>
+<word>vostè</word>
+<word>vostès</word>
+<word>vostra</word>
+<word>vostre</word>
+<word>vostres</word>
+ </grader-tc>
+</dictionary>