summarize 1.0.0

data/ext/summarize/dictionary.c

@@ -0,0 +1,331 @@
+/*
+ *  dictionary.c
+ *
+ *  Copyright (C) 2003 Nadav Rotem <nadav256@hotmail.com>
+ * 
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 2 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU Library General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program; if not, write to the Free Software
+ *  Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include "libots.h"
+#include "grader-tc.h"
+
+#include <libxml/xmlmemory.h>
+#include <libxml/parser.h>
+
+
+/* loads the xml dictionary file to memory*/
+
+gboolean
+ots_load_xml_dictionary (OtsArticle * Doc,unsigned const char *name)
+{
+
+  xmlDocPtr doc=NULL;
+  xmlNodePtr head=NULL;
+  xmlNodePtr stem=NULL;
+  xmlNodePtr pre=NULL;
+  xmlNodePtr post=NULL;
+  xmlNodePtr syno=NULL;	    		/* synonyms     */
+  xmlNodePtr manual=NULL;	   	/* manual  */
+  xmlNodePtr step1_pre=NULL;		/* step1  */
+  xmlNodePtr step1_post=NULL;		/* step1  */
+  
+  xmlNodePtr parse=NULL;	   	/* parser rules */
+  xmlNodePtr pbreak=NULL;	      
+  xmlNodePtr pdbreak=NULL;		
+  
+  xmlNodePtr tc_words=NULL;	   	/* term count dictionary   */
+  xmlNodePtr tf_words=NULL;	   	/* term frequency dictionary   */
+  
+
+  OtsStemRule * rule=Doc->stem;
+	
+  char *local_dict_name;
+  
+  local_dict_name = g_strdup_printf ("%s.xml", name);
+
+
+	if (g_file_test(local_dict_name,G_FILE_TEST_EXISTS))
+		  doc = xmlParseFile (local_dict_name); /* it warns to the screen so we cant use it; enable for web services only */
+  if (doc == NULL) return (FALSE);
+
+  head = xmlDocGetRootElement (doc);
+  if (head == NULL)
+    {
+      fprintf (stderr, "empty document\n");
+      xmlFreeDoc (doc);
+      return (FALSE);
+    }
+
+  if (xmlStrcmp (head->name, (const xmlChar *) "dictionary"))
+    {
+      fprintf (stderr, "%s", head->name);
+      xmlFreeDoc (doc);
+      return (FALSE);
+    }
+
+  if (head != NULL)
+    stem = head->xmlChildrenNode;
+  while ((stem != NULL)
+	 && (xmlStrcmp (stem->name, (const xmlChar *) "stemmer")))
+    {
+      stem = stem->next;
+    }
+
+  if (head != NULL)
+    parse = head->xmlChildrenNode;
+  while ((parse != NULL)
+	 && (xmlStrcmp (parse->name, (const xmlChar *) "parser")))
+    {
+      parse = parse->next;
+    }
+
+  if (head != NULL)
+    tc_words = head->xmlChildrenNode;
+  while ((tc_words != NULL)
+	 && (xmlStrcmp (tc_words->name, (const xmlChar *) "grader-tc")))
+    {
+      tc_words = tc_words->next;
+    }
+
+
+  if (head != NULL)
+    tf_words = head->xmlChildrenNode;
+  while ((tf_words != NULL)
+	 && (xmlStrcmp (tf_words->name, (const xmlChar *) "grader-tf")))
+    {
+      tf_words = tf_words->next;
+    }
+    
+    
+
+  if (stem != NULL)
+    pre = stem->xmlChildrenNode;
+  while ((pre != NULL) && (xmlStrcmp (pre->name, (const xmlChar *) "pre")))
+    {
+      pre = pre->next;
+    }
+
+  if (stem != NULL)
+    post = stem->xmlChildrenNode;
+  while ((post != NULL) && (xmlStrcmp (post->name, (const xmlChar *) "post")))
+    {
+      post = post->next;
+    }
+
+
+  if (stem != NULL)
+    syno = stem->xmlChildrenNode;
+  while ((syno != NULL)
+	 && (xmlStrcmp (syno->name, (const xmlChar *) "synonyms")))
+    {
+      syno = syno->next;
+    }
+
+  if (stem != NULL)
+    manual = stem->xmlChildrenNode;
+  while ((manual != NULL)
+	 && (xmlStrcmp (manual->name, (const xmlChar *) "manual")))
+    {
+      manual = manual->next;
+    }
+
+
+  if (stem != NULL)
+    step1_pre = stem->xmlChildrenNode;
+  while ((step1_pre != NULL)
+	 && (xmlStrcmp (step1_pre->name, (const xmlChar *) "step1_pre")))
+    {
+      step1_pre = step1_pre->next;
+    }
+    
+    
+    
+  if (stem != NULL)
+    step1_post = stem->xmlChildrenNode;
+  while ((step1_post != NULL)
+	 && (xmlStrcmp (step1_post->name, (const xmlChar *) "step1_post")))
+    {
+      step1_post = step1_post->next;
+    }
+
+
+  if (pre != NULL)
+    pre = pre->xmlChildrenNode;	/*point to first word */
+  while (pre != NULL)
+    {
+      if (0 == xmlStrcmp (pre->name, (const xmlChar *) "rule"))
+	rule->RemovePre =
+	  g_list_append (rule->RemovePre,
+			 (xmlNodeListGetString
+				 (doc, pre->xmlChildrenNode, 1)));
+      pre = pre->next;
+    }
+
+
+  if (post != NULL)
+    post = post->xmlChildrenNode;
+  while (post != NULL)
+    {
+      if (0 == xmlStrcmp (post->name, (const xmlChar *) "rule"))
+	rule->RemovePost =
+	  g_list_append (rule->RemovePost,
+			       (xmlNodeListGetString
+				 (doc, post->xmlChildrenNode, 1)));
+      post = post->next;
+    }
+
+  if (syno != NULL)
+    syno = syno->xmlChildrenNode;
+  while (syno != NULL)
+    {
+      if (0 == xmlStrcmp (syno->name, (const xmlChar *) "rule"))
+	rule->synonyms =
+	  g_list_append (rule->synonyms,
+			     (xmlNodeListGetString
+				 (doc, syno->xmlChildrenNode, 1)));
+      syno = syno->next;
+    }
+
+  if (manual != NULL)
+    manual = manual->xmlChildrenNode;
+  while (manual != NULL)
+    {
+      if (0 == xmlStrcmp (manual->name, (const xmlChar *) "rule"))
+	rule->manual =
+	  g_list_append (rule->manual,
+			      (xmlNodeListGetString
+				 (doc, manual->xmlChildrenNode, 1)));
+      manual = manual->next;
+    }
+
+
+
+
+ if (step1_pre != NULL)
+    step1_pre = step1_pre->xmlChildrenNode;
+  while (step1_pre != NULL)
+    {
+      if (0 == xmlStrcmp (step1_pre->name, (const xmlChar *) "rule"))
+	rule->step1_pre =
+	  g_list_append (rule->step1_pre,
+			      (xmlNodeListGetString
+				 (doc, step1_pre->xmlChildrenNode, 1)));
+      step1_pre = step1_pre->next;
+    }
+
+
+
+ if (step1_post != NULL)
+    step1_post = step1_post->xmlChildrenNode;
+  while (step1_post != NULL)
+    {
+      if (0 == xmlStrcmp (step1_post->name, (const xmlChar *) "rule"))
+	rule->step1_post =
+	  g_list_append (rule->step1_post,
+			        (xmlNodeListGetString
+				 (doc, step1_post->xmlChildrenNode, 1)));
+      step1_post = step1_post->next;
+    }
+
+ if (parse != NULL)
+    pbreak = parse->xmlChildrenNode;
+  while ((pbreak != NULL) && (xmlStrcmp (pbreak->name, (const xmlChar *) "linebreak")))
+    {
+      pbreak = pbreak->next;
+    }
+
+
+
+ if (parse != NULL)
+    pdbreak = parse->xmlChildrenNode;
+  while ((pdbreak != NULL) && (xmlStrcmp (pdbreak->name, (const xmlChar *) "linedontbreak")))
+    {
+      pdbreak = pdbreak->next;
+    }
+    
+    
+  /*Parser break*/
+  if (pbreak != NULL)
+    pbreak = pbreak->xmlChildrenNode;
+  while (pbreak != NULL)
+    {
+      if (0 == xmlStrcmp (pbreak->name, (const xmlChar *) "rule"))
+	rule->ParserBreak =
+	  g_list_append (rule->ParserBreak,
+			        (xmlNodeListGetString
+				 (doc, pbreak->xmlChildrenNode, 1)));
+      pbreak = pbreak->next;
+    }
+
+  /*Parser Don't break*/
+  if (pdbreak != NULL)
+    pdbreak = pdbreak->xmlChildrenNode;
+  while (pdbreak != NULL)
+    {
+      if (0 == xmlStrcmp (pdbreak->name, (const xmlChar *) "rule"))
+	rule->ParserDontBreak =
+	  g_list_append (rule->ParserDontBreak,
+			        (xmlNodeListGetString
+				 (doc, pdbreak->xmlChildrenNode, 1)));
+      pdbreak = pdbreak->next;
+    }
+
+  /*Term Count load dict*/
+
+  if (tc_words != NULL)
+    tc_words = tc_words->xmlChildrenNode;
+  while (tc_words != NULL)
+    {
+      if (0 == xmlStrcmp (tc_words->name, (const xmlChar *) "word"))
+	{
+		xmlChar *key;
+		key=xmlNodeListGetString(doc, tc_words->xmlChildrenNode,1);
+	   Doc->dict = g_list_append (Doc->dict,(gpointer)ots_new_wordEntery(key));
+      xmlFree(key);
+   }
+      tc_words = tc_words->next;
+    }
+      
+  
+  /*Term Frequency load dict*/
+  
+  if (tf_words != NULL)
+    tf_words = tf_words->xmlChildrenNode;
+  while (tf_words != NULL)
+    {
+      if (0 == xmlStrcmp (tf_words->name, (const xmlChar *) "word"))
+	{
+		xmlChar *key;
+		xmlChar *idf_key;
+		key=xmlNodeListGetString(doc, tf_words->xmlChildrenNode,1);
+	   
+	   idf_key=xmlGetProp(tf_words,"idf");
+	   Doc->tf_terms = g_list_append (Doc->tf_terms,ots_new_OtsWordTF(key,atof(idf_key)));
+      xmlFree(key);
+      xmlFree(idf_key);
+   }
+      tf_words = tf_words->next;
+    }
+    
+    
+  xmlFreeDoc(doc);
+  xmlCleanupParser ();
+  g_free(local_dict_name);
+  return (TRUE);
+}

data/ext/summarize/extconf.rb

@@ -0,0 +1,6 @@
+require 'mkmf'
+
+$CFLAGS = ENV["CFLAGS"].to_s + " " + `pkg-config --cflags glib-2.0 libxml-2.0`.chomp
+$LDFLAGS = ENV["LDFLAGS"].to_s  + " " + `pkg-config --libs glib-2.0 libxml-2.0`.chomp
+
+create_makefile('summarize/summarize')

data/ext/summarize/grader-tc.c

@@ -0,0 +1,185 @@
+/*
+ *  grader-tc.c
+ *
+ *  Copyright (C) 2003 Nadav Rotem <nadav256@hotmail.com>
+ * 
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 2 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU Library General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program; if not, write to the Free Software
+ *  Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include "libots.h"
+
+
+#include "grader-tc.h"
+
+
+/*Grader - Term count algorithm*/
+/*This is non-normelized term frequency algorithm without using inverse document frequency database */
+
+#define NUM_KEY_WORDS 100 /* use first n key words only */
+
+int
+ots_get_article_word_count (const OtsArticle * Doc)
+{
+  GList *li;
+  int articleWC;
+  articleWC = 0;
+
+  if (Doc==NULL) return 0;
+
+  for (li = (GList *) Doc->lines; li != NULL; li = li->next)
+    {
+      articleWC += ((OtsSentence *) li->data)->wc;
+    }
+
+  return articleWC;
+}
+
+
+/*take this line and add each word to the "wordStat" list
+* this list will hold all of the words in the article and the number 
+* of times they appeared in the article.
+*/
+
+static void 
+ots_line_add_wordlist(OtsArticle * Doc,const OtsSentence * aLine) 
+{
+  GList *li;
+  if ((aLine==NULL) ||(NULL==Doc)) { return;}
+  
+      for (li = (GList *) aLine->words; li != NULL; li = li->next)	/* for each word in the sentence Do: */
+			if (li->data && strlen (li->data)) ots_add_wordstat (Doc, (char *)li->data);
+		    
+  return;
+}
+
+static void 
+ots_create_wordlist(OtsArticle * Doc)
+{
+GList *line;
+if (Doc==NULL) return;
+
+for (line = (GList *) Doc->lines; line != NULL; line = line->next)	
+	{ 
+	OtsSentence * aLine=line->data;
+	if (aLine)
+	ots_line_add_wordlist(Doc,aLine);
+	}
+}
+
+
+	
+
+static int
+keyVal (const int n)			/* Ugly , I know */
+{
+  if (n == 1) return 3;
+  if (n == 2) return 2;
+  if (n == 3) return 2;
+  if (n == 4) return 2;
+  return 1;
+}
+
+
+static void
+ots_grade_line (GList *impList, OtsSentence * aLine,
+		OtsStemRule * rule)
+{
+  GList *li;
+  GList *di;
+  int n;
+  char *tmp_stem;
+
+  if ((aLine==NULL)||(rule==NULL)||(impList==NULL)) return;
+
+  for (li = (GList *) aLine->words; li != NULL; li = li->next)	/* for each word */
+    {
+      n = 0;
+      tmp_stem = ots_stem_strip ((unsigned char *) li->data, rule);
+
+      for (di = (GList *) impList;
+	   ((di != NULL) && (n < NUM_KEY_WORDS)); di = di->next)
+	{
+	  n++;
+	if ((NULL!=((OtsWordEntery *) di->data)->stem) && (NULL!=tmp_stem))
+	  if (0 == strcmp ((((OtsWordEntery *) di->data)->stem), tmp_stem)) 
+	    {
+	      /* debug:  
+	      if (0!=strcmp((((OtsWordEntery *) di->data)->word),li->data)) 
+	      printf("[%s][%s] stem[%s]\n",(((OtsWordEntery *) di->data)->word),li->data,tmp);*/
+
+	      aLine->score += (((OtsWordEntery *) di->data)->occ) * keyVal (n);
+	    }
+
+	}
+
+      g_free (tmp_stem);
+    }
+
+}
+
+
+void
+ots_create_title_tc(OtsArticle * Doc)
+{
+
+	char *tmp;
+	char *word;
+	int i;
+    	GString *title;
+	if (NULL==Doc) return;
+
+	title=g_string_new(NULL);
+	
+	for (i=0;i<5;i++)
+	{
+	word = ots_word_in_list(Doc->ImpWords,i);
+	if (word) g_string_append(title,word); else break;
+	if (i<4) g_string_append(title,",");
+	}
+	
+	tmp=title->str;
+	if (NULL!=title) g_string_free(title,FALSE);
+	Doc->title=tmp;
+}
+
+
+void
+ots_grade_doc_tc (OtsArticle * Doc)
+{
+
+  GList *li;
+  if (NULL==Doc) return;  
+  ots_create_wordlist(Doc);
+ 
+  
+   Doc->ImpWords=ots_union_list (Doc->wordStat, Doc->dict);	   /* subtract from the Article wordlist all the words in the dic file (on , the , is...) */
+   Doc->ImpWords=ots_sort_list (Doc->ImpWords);			  /* sort the list , top 3 is what the article talks about (SARS , virus , cure ... ) */
+
+   /*to print wordlist: ots_print_wordlist (stdout, Doc->ImpWords);*/
+
+  if (0 == Doc->lineCount) return;
+
+  for (li = (GList *) Doc->lines; li != NULL; li = li->next)
+    {
+    if (li->data)
+      ots_grade_line (Doc->ImpWords, (OtsSentence *) li->data, Doc->stem);
+    }
+
+
+  ots_create_title_tc(Doc);
+}