summarize 1.0.0

data/ext/summarize/wordlist.c

@@ -0,0 +1,220 @@
+/*
+ *  wordlist.c
+ *
+ *  Copyright (C) 2003 Nadav Rotem <nadav256@hotmail.com>
+ * 
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 2 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU Library General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program; if not, write to the Free Software
+ *  Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include "libots.h"
+#include "grader-tc.h"
+
+/*word lists manipulations , mainly for grader-tc */
+
+OtsWordEntery *
+ots_new_wordEntery_strip(unsigned const char *wordString,const OtsStemRule *rule) /*for real text use*/
+{
+  OtsWordEntery *aWord = g_new0 (OtsWordEntery, 1);
+  aWord->occ = 1;
+  aWord->word = ots_stem_format(wordString,rule);  
+  aWord->stem = ots_stem_strip(wordString,rule);
+  return aWord;
+}
+
+OtsWordEntery *
+ots_new_wordEntery (unsigned const char *wordString) /*for dictionary use only, no formating here*/
+{
+  OtsWordEntery *aWord = g_new0 (OtsWordEntery, 1);
+  aWord->occ = 1;
+  aWord->word = g_strdup (wordString);  
+  aWord->stem = g_strdup (wordString);
+  return aWord;
+}
+
+
+void
+ots_free_wordEntery (OtsWordEntery * WC)
+{
+  if (WC != NULL)
+    {
+      if (NULL!=WC->word) g_free (WC->word);
+      if (NULL!=WC->stem) g_free (WC->stem);
+      g_free (WC);
+    }
+}
+
+void
+ots_free_wordlist (GList * aList)
+{  
+  if (aList != NULL)
+    {
+  		 	g_list_foreach(aList,(GFunc)ots_free_wordEntery , NULL);
+    		g_list_free(aList);
+    } 
+}
+
+OtsWordEntery *
+ots_copy_wordEntery (OtsWordEntery * obj)
+{
+  OtsWordEntery *aWord;
+  if (obj == NULL) { return NULL;}
+  aWord = g_new (OtsWordEntery, 1);
+  aWord->occ = obj->occ;
+  aWord->word = g_strdup (obj->word);  
+  if (NULL!=obj->stem)
+  {aWord->stem = g_strdup (obj->stem);} else {aWord->stem=NULL;}
+  return aWord;
+}
+
+static int
+ots_sort_handler (OtsWordEntery * node1, OtsWordEntery * node2)
+{
+  if (node1->occ > node2->occ)
+    return -1;
+  if (node1->occ < node2->occ)
+    return 1;
+  return 0;
+}
+
+GList *
+ots_sort_list (GList* aList)
+{ 
+  GList *newList; 
+  newList = g_list_sort (aList, (GCompareFunc) ots_sort_handler);	/* sort article */
+  return newList;
+}
+
+GList *
+ots_union_list (const GList *aLst, const GList * bLst)
+{
+  GList *li;
+  GList *di;
+  int insert;
+  GList *newLst=NULL;
+  
+  for (li = (GList *) aLst; li != NULL; li = li->next)
+    {
+      insert = 1;
+      for (di = (GList *) bLst; di != NULL; di = di->next)
+		{
+		 if(( li->data) && (di->data) && (((OtsWordEntery *) li->data)->word) && (((OtsWordEntery *) di->data)->word)) /*all defined?*/
+	    if (0 == g_strncasecmp ((((OtsWordEntery *) li->data)->word), /*fix me: unicode issue?*/
+				  (((OtsWordEntery *) di->data)->word), 10))
+	    insert = 0;		/* if word in B */
+
+		}
+      if (insert == 1) 
+	if ((li->data))
+      newLst = g_list_append (newLst,ots_copy_wordEntery ((OtsWordEntery *) li->data));
+    }
+    
+    return newLst;
+}
+
+
+char *
+ots_word_in_list (const GList *aList,const int index)	/* return the String value of the n'th word */
+{
+  OtsWordEntery *obj = NULL;
+  
+  GList *item =(GList *)g_list_nth ((GList *)aList, index);
+  if (item != NULL) obj = item->data;
+  if (obj == NULL)
+    {
+      return NULL;
+    }
+  else
+    return obj->word;
+}
+
+char *
+ots_stem_in_list (const GList *aList,const int index)	/* return the String value of stem of the n'th word */
+{
+  OtsWordEntery *obj = NULL;
+
+  GList *item =(GList *)g_list_nth ((GList *)aList, index);
+  if (item != NULL) obj = item->data;
+  if (obj == NULL)
+    {
+      return NULL;
+    }
+  else
+    return obj->stem;
+}
+
+/*Adds a word to the word count of the article*/
+void
+ots_add_wordstat (OtsArticle * Doc,
+		  unsigned const char *wordString)
+{
+  GList *li;
+  OtsWordEntery *stat;
+  OtsStemRule * rule=Doc->stem;
+  char *tmp = NULL;
+
+  if (NULL==wordString) return;
+  if (NULL==Doc) return;
+    
+  if (0==strlen(wordString)) return;
+  if (0==strcmp(wordString," ")) return;
+  if (0==strcmp(wordString,"\n")) return;
+  if (0==strcmp(wordString,"\t")) return;
+  
+  if (wordString)
+  tmp = ots_stem_strip (wordString, rule);
+
+  for (li = (GList *) Doc->wordStat; li != NULL; li = li->next)	/* search the word in current wordlist */
+    {
+      if (li->data)
+      if (0 == strcmp (tmp, ((OtsWordEntery *) li->data)->stem))
+	{
+	  ((OtsWordEntery *) li->data)->occ++;	/* occurred in another place in the text now; */
+	  g_free (tmp);
+	
+		/*printf for debug*/
+	    /* 
+	    if (0!=strcmp(((OtsWordEntery *) li->data)->word,wordString)  )
+	     printf("[%s]==[%s]\n",((OtsWordEntery *) li->data)->word,wordString);
+	    */
+
+	  return;
+	}
+    }
+
+  stat = ots_new_wordEntery_strip (wordString, rule);	/* if not in list , Add  stem  it to the list */
+  if ((stat)) 
+  Doc->wordStat = g_list_prepend (Doc->wordStat, stat);
+  g_free (tmp);
+  return;
+}		  
+
+
+
+
+void
+ots_print_wordlist (FILE * stream, const GList * aList)
+{
+  GList *li;
+  for (li = (GList *) aList; li != NULL; li = li->next)
+    fprintf (stream, "Word[%d][%s]\n", ((OtsWordEntery *) li->data)->occ,
+	    		 			((OtsWordEntery *) li->data)->word);
+}
+
+
+

data/lib/summarize.rb

@@ -0,0 +1,91 @@
+require 'summarize/summarize'
+
+class Hash
+  def symbolize_keys
+    inject({}) do |options, (key, value)|
+      options[(key.to_sym rescue key) || key] = value
+      options
+    end
+  end
+end unless {}.respond_to? 'symbolize_keys'
+
+module Summarize
+  VERSION = "1.0.0"
+
+  LANGUAGES = [
+    'bg', # Bulgarian
+    'ca', # Catalan
+    'cs', # Czech
+    'cy', # Welsh
+    'da', # Danish
+    'de', # German
+    'el', # Greek
+    'en', # English
+    'eo', # Esperanto
+    'es', # Spanish
+    'et', # Creole
+    'eu', # Basque
+    'fi', # Finnish
+    'fr', # French
+    'ga', # Irish
+    'gl', # Galician
+    'he', # Hebrew
+    'hu', # Hungarian
+    'ia', # Interlingua
+    'id', # Indonesian
+    'is', # Icelandic
+    'it', # Italian
+    'lv', # Latvian
+    'mi', # Maori
+    'ms', # Malay
+    'mt', # Maltese
+    'nl', # Dutch
+    'nn', # Norweigan
+    'pl', # Polish
+    'pt', # Portuguese
+    'ro', # Romanian
+    'ru', # Russian
+    'sv', # Swedish
+    'tl', # Tagalog
+    'tr', # Turkish
+    'uk', # Ukrainian
+    'yi'  # Yiddish
+  ]
+
+  def self.parse_options(options = {})
+    default_options = {
+      :ratio => 25,     # percentage
+      :language => 'en' # ISO 639-1 code
+    }
+
+    options = default_options.merge(options.symbolize_keys)
+
+    if options.key? :dictionary
+      dict_file = options[:dictionary]
+    else
+      raise "Language not supported" unless LANGUAGES.index(options[:language])
+      dict_file = File.join(File.expand_path(File.dirname(__FILE__)), "../ext/summarize/dic/#{options[:language]}")
+    end
+
+    return [dict_file, options[:ratio]]
+  end
+
+end
+
+class String
+  extend Summarize
+
+  def summarize(options = {})
+    dict_file, ratio = Summarize.parse_options(options)
+    String.send(:summarize, self, dict_file, ratio)
+  end
+
+end
+
+class File
+
+  def summarize(options = {})
+    self.read.summarize(options)
+  end
+
+end

data/sample_data/jupiter.txt

@@ -0,0 +1,15 @@
+Jupiter is the fifth planet from the Sun and the largest planet within the Solar System. It is a gas giant with a mass  slightly less than one-thousandth of the Sun but is two and a half times the mass of all the other planets in our Solar System combined. Jupiter is classified as a gas giant along with Saturn, Uranus and Neptune. Together, these four planets are sometimes referred to as the Jovian planets.
+
+The planet was known by astronomers of ancient times and was associated with the mythology and religious beliefs of many cultures. The Romans named the planet after the Roman god Jupiter. When viewed from Earth, Jupiter can reach an apparent magnitude of -2.94, making it on average the third-brightest object in the night sky after the Moon and Venus. (Mars can briefly match Jupiter's brightness at certain points in its orbit.)
+
+Jupiter is primarily composed of hydrogen with a quarter of its mass being helium; it may also have a rocky core of heavier elements. Because of its rapid rotation, Jupiter's shape is that of an oblate spheroid (it possesses a slight but noticeable bulge around the equator). The outer atmosphere is visibly segregated into several bands at different latitudes, resulting in turbulence and storms along their interacting boundaries. A prominent result is the Great Red Spot, a giant storm that is known to have existed since at least the 17th century when it was first seen by telescope. Surrounding the planet is a faint planetary ring system and a powerful magnetosphere. There are also at least 63 moons, including the four large moons called the Galilean moons that were first discovered by Galileo Galilei in 1610. Ganymede, the largest of these moons, has a diameter greater than that of the planet Mercury.
+
+Jupiter has been explored on several occasions by robotic spacecraft, most notably during the early Pioneer and Voyager flyby missions and later by the Galileo orbiter. The most recent probe to visit Jupiter was the Pluto-bound New Horizons spacecraft in late February 2007. The probe used the gravity from Jupiter to increase its speed. Future targets for exploration in the Jovian system include the possible ice-covered liquid ocean on the moon Europa.
+||
+Jupiter is the fifth planet from the Sun and the largest planet within the Solar System. It is a gas giant with a mass  slightly less than one-thousandth of the Sun but is two and a half times the mass of all the other planets in our Solar System combined. The Romans named the planet after the Roman god Jupiter. (Mars can briefly match Jupiter's brightness at certain points in its orbit.)
+
+Jupiter is primarily composed of hydrogen with a quarter of its mass being helium; There are also at least 63 moons, including the four large moons called the Galilean moons that were first discovered by Galileo Galilei in 1610.
+||
+Jupiter is the fifth planet from the Sun and the largest planet within the Solar System. It is a gas giant with a mass  slightly less than one-thousandth of the Sun but is two and a half times the mass of all the other planets in our Solar System combined. Together, these four planets are sometimes referred to as the Jovian planets. The Romans named the planet after the Roman god Jupiter. When viewed from Earth, Jupiter can reach an apparent magnitude of -2.94, making it on average the third-brightest object in the night sky after the Moon and Venus. (Mars can briefly match Jupiter's brightness at certain points in its orbit.)
+
+Jupiter is primarily composed of hydrogen with a quarter of its mass being helium; Because of its rapid rotation, Jupiter's shape is that of an oblate spheroid (it possesses a slight but noticeable bulge around the equator). There are also at least 63 moons, including the four large moons called the Galilean moons that were first discovered by Galileo Galilei in 1610. The most recent probe to visit Jupiter was the Pluto-bound New Horizons spacecraft in late February 2007.

data/summarize.gemspec

@@ -0,0 +1,21 @@
+$:.unshift(File.join(File.dirname(__FILE__), "lib"))
+require "summarize"
+
+Gem::Specification.new do |s|
+  s.name             = %q{summarize}
+  s.summary          = %q{Open Text Summarizer}
+  s.description      = %q{Ruby C Extension for Open Text Summarizer}
+  s.homepage         = %q{http://github.com/intridea}
+  s.version          = Summarize::VERSION
+  s.authors          = ["Sean Soper"]
+  s.email            = %q{sean.soper@gmail.com}
+
+  s.rubygems_version = %q{1.3.7}
+  s.date             = %q{2010-11-08}
+
+  s.require_paths = ["lib"]
+  s.extensions = ["ext/summarize/extconf.rb"]
+  s.files      = %x{git ls-files}.split("\n").reject {|file| file =~ /^(features|cucumber)/ }
+
+  s.add_dependency 'rake-compiler'
+end

metadata

@@ -0,0 +1,140 @@
+--- !ruby/object:Gem::Specification 
+name: summarize
+version: !ruby/object:Gem::Version 
+  hash: 23
+  prerelease: false
+  segments: 
+  - 1
+  - 0
+  - 0
+  version: 1.0.0
+platform: ruby
+authors: 
+- Sean Soper
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2010-11-08 00:00:00 -05:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: rake-compiler
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  type: :runtime
+  version_requirements: *id001
+description: Ruby C Extension for Open Text Summarizer
+email: sean.soper@gmail.com
+executables: []
+
+extensions: 
+- ext/summarize/extconf.rb
+extra_rdoc_files: []
+
+files: 
+- .gitignore
+- README.markdown
+- Rakefile
+- ext/summarize/article.c
+- ext/summarize/dic/bg.xml
+- ext/summarize/dic/ca.xml
+- ext/summarize/dic/cs.xml
+- ext/summarize/dic/cy.xml
+- ext/summarize/dic/da.xml
+- ext/summarize/dic/de.xml
+- ext/summarize/dic/el.xml
+- ext/summarize/dic/en.xml
+- ext/summarize/dic/eo.xml
+- ext/summarize/dic/es.xml
+- ext/summarize/dic/et.xml
+- ext/summarize/dic/eu.xml
+- ext/summarize/dic/fi.xml
+- ext/summarize/dic/fr.xml
+- ext/summarize/dic/ga.xml
+- ext/summarize/dic/gl.xml
+- ext/summarize/dic/he.xml
+- ext/summarize/dic/hu.xml
+- ext/summarize/dic/ia.xml
+- ext/summarize/dic/id.xml
+- ext/summarize/dic/is.xml
+- ext/summarize/dic/it.xml
+- ext/summarize/dic/lv.xml
+- ext/summarize/dic/mi.xml
+- ext/summarize/dic/ms.xml
+- ext/summarize/dic/mt.xml
+- ext/summarize/dic/nl.xml
+- ext/summarize/dic/nn.xml
+- ext/summarize/dic/pl.xml
+- ext/summarize/dic/pt.xml
+- ext/summarize/dic/ro.xml
+- ext/summarize/dic/ru.xml
+- ext/summarize/dic/sv.xml
+- ext/summarize/dic/tl.xml
+- ext/summarize/dic/tr.xml
+- ext/summarize/dic/uk.xml
+- ext/summarize/dic/yi.xml
+- ext/summarize/dictionary.c
+- ext/summarize/extconf.rb
+- ext/summarize/grader-tc.c
+- ext/summarize/grader-tc.h
+- ext/summarize/grader-tf.c
+- ext/summarize/grader.c
+- ext/summarize/highlighter.c
+- ext/summarize/html.c
+- ext/summarize/libots.h
+- ext/summarize/parser.c
+- ext/summarize/relations.c
+- ext/summarize/stemmer.c
+- ext/summarize/summarize.c
+- ext/summarize/summarize.h
+- ext/summarize/text.c
+- ext/summarize/wordlist.c
+- lib/summarize.rb
+- lib/summarize/summarize.bundle
+- sample_data/jupiter.txt
+- summarize.gemspec
+has_rdoc: true
+homepage: http://github.com/intridea
+licenses: []
+
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.3.7
+signing_key: 
+specification_version: 3
+summary: Open Text Summarizer
+test_files: []
+