summarize 1.0.0

data/ext/summarize/libots.h

@@ -0,0 +1,158 @@
+/*
+ *  libots.h
+ *
+ *  Copyright (C) 2003 Nadav Rotem <nadav256@hotmail.com>
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 2 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU Library General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program; if not, write to the Free Software
+ *  Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+ */
+
+#ifndef HAVE_LIBOTS_H
+#define HAVE_LIBOTS_H
+
+#include <glib.h>
+
+G_BEGIN_DECLS 
+
+typedef struct
+{ 			/* the Term Frequency data structure */
+  char* word;			
+  double tf; /*Also used for TF*/  
+} OtsWordTF;
+
+
+typedef struct
+{
+	/*a GList of char*  */
+  GList *RemovePre;   /* (a|b)  replace string a with b */ 
+  GList *RemovePost; 
+  GList *step1_pre;
+  GList *step1_post;
+  
+  GList *synonyms; 
+  GList *manual; 
+	
+  GList *ParserBreak; 	
+  GList *ParserDontBreak; 	
+	
+	
+	/*to be implemented*/ 
+  GList *ReplaceChars; 
+  
+} OtsStemRule;
+
+
+typedef struct
+{
+  GList *words;			 /* a Glist of words (char*) */
+  glong score;				 /*score set by the grader*/
+  gboolean selected;     /*is selected?*/
+  gint wc;               /*word count*/
+  void *user_data;       /*pointer to the original sentence , or serial number maybe*/
+} OtsSentence;
+
+
+typedef struct
+{
+  GList *lines;			   /* a Glist of sentences (struct Sentence) */
+  gint lineCount;          /*lines in the text*/         
+  char *title;              /*title , auto generated*/
+  
+  OtsStemRule *stem;       /*stemming & parsing rules*/
+
+ /*Term Frequency grader*/  
+  GList *tf_terms; 
+  GList *idf_terms; 
+  
+
+  /*Term Count grader*/  
+  GList *dict; 		/* dictionary from xml*/
+  GList *wordStat;	/* a wordlist of all words in the article and their occ */    
+  GList *ImpWords;   /*important words - for term count grader*/
+  
+  
+} OtsArticle;
+
+
+OtsArticle *ots_new_article (void);
+void ots_free_article (OtsArticle *art);
+
+/*parser*/
+void ots_parse_file (FILE * stream, OtsArticle * Doc);	/*file input */
+void ots_parse_stream(const unsigned char *utf8 , size_t len ,OtsArticle *Doc); /*parse unicode stream*/
+
+OtsSentence *ots_append_line (OtsArticle * Doc);
+void ots_append_word (OtsSentence * aLine,unsigned const char *aWord);
+void ots_add_wordstat (OtsArticle * Doc,unsigned const char *wordString);
+
+
+/*dictionary*/
+gboolean ots_load_xml_dictionary (OtsArticle * Doc,unsigned const char *name);
+
+int ots_get_article_word_count (const OtsArticle * Doc);
+
+
+/*grader*/
+void ots_highlight_doc (OtsArticle * Doc, int percent);     /*example: 20%*/
+void ots_highlight_doc_lines (OtsArticle * Doc, int lines); /*example: 10 lines*/
+void ots_highlight_doc_words (OtsArticle * Doc, int words); /*example: 50 words*/
+
+void ots_grade_doc (OtsArticle * Doc);
+
+void  ots_free_OtsWordTF(OtsWordTF *obj); /*todo: put in .h file*/
+OtsWordTF*  ots_new_OtsWordTF(const char* word,const double idf);
+
+
+/*HTML output*/
+void ots_print_HTML (FILE * stream, const OtsArticle * Doc);
+unsigned char *ots_get_doc_HTML (const OtsArticle * Doc, size_t * out_len);
+
+/*TEXT output*/
+void ots_print_doc (FILE * stream, const OtsArticle * Doc);
+unsigned char *ots_get_doc_text (const OtsArticle * Doc, size_t * out_len);
+
+
+/*Plugin writing*/
+unsigned char* ots_get_line_text (const OtsSentence *aLine, gboolean only_if_selected, size_t *out_size);
+gboolean ots_is_line_selected(const OtsSentence *aLine);
+
+/*Stemm support*/
+OtsStemRule *new_stem_rule(void);
+void free_stem_rule (OtsStemRule *rule);
+unsigned char * ots_stem_strip (unsigned const char * aWord, const OtsStemRule *rule); /*returns newly allocated string with the root of the word*/
+unsigned char *ots_stem_format (unsigned const char *aWord, const OtsStemRule * rule); /*Remove leading spaces, comas, colons, etc. */
+
+/*Relations between texts*/
+
+/*Returns the number of topics that two blocks of text share*/
+int ots_text_relations(
+const unsigned char *text1,const unsigned char *lang_code1,
+const unsigned char *text2,const unsigned char *lang_code2,const int topic_num);
+
+/*For a given text, return the list of the topics*/
+char* ots_text_topics(const unsigned char *text,const unsigned char *lang_code,int topic_num);
+
+
+/*For a given text, return the list of the stemmed topics*/
+GList* ots_text_stem_list(const unsigned char *text,const unsigned char *lang_code,int topic_num);
+
+
+/*Gives a score on the relations between two lists of topics; simmilar to the inner product*/
+int ots_topic_list_score(const GList *topic_list1,const GList *topic_list2);
+
+G_END_DECLS
+
+
+
+#endif /* HAVE_LIBOTS_H */

data/ext/summarize/parser.c

@@ -0,0 +1,173 @@
+/*
+ *  parser.c
+ *
+ *  Copyright (C) 2003 Nadav Rotem <nadav256@hotmail.com>
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 2 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU Library General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program; if not, write to the Free Software
+ *  Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <strings.h>
+#include "libots.h"
+
+#define BUFFER_SIZE (1024*8)
+
+int
+ots_match_post (const char *aWord,const char *post)
+{
+  int i, wlen, plen;
+  
+
+  wlen = strlen (aWord);
+  plen = strlen (post);
+
+  if (plen > wlen) return 0;
+
+  for (i = 0; i < plen; i++)
+    if (aWord[wlen - plen + i] != post[i])
+      return 0;		/* no match */
+
+  return 1;		/*word match */
+}
+
+void
+ots_parse_file (FILE * stream, OtsArticle * Doc )
+{
+  unsigned char fread_buffer[BUFFER_SIZE];
+  unsigned char *buffer;
+  size_t nread, total_read, avail_size;
+
+  buffer = g_new0 (unsigned char, BUFFER_SIZE);
+
+  avail_size = BUFFER_SIZE;
+  total_read = nread = 0;
+  while ((nread =
+	  fread (fread_buffer, sizeof (unsigned char), sizeof (fread_buffer),
+		 stream)) > 0)
+    {
+      if (nread + total_read > avail_size)
+	{
+	  avail_size *= 2;
+	  buffer = g_renew (unsigned char, buffer, avail_size);
+	}
+
+      strncpy (buffer + total_read, fread_buffer, nread);
+      total_read += nread;
+    }
+
+  ots_parse_stream (buffer, total_read, Doc);
+  g_free (buffer);
+}
+
+
+
+
+
+int
+ots_parser_should_break(const char *aWord,const OtsStemRule * rule)
+{
+  GList *li;
+  char *postfix;
+  int toBreak=0;
+  
+  for (li = (GList *) rule->ParserBreak; li != NULL; li = li->next)
+    {
+      postfix=li->data;
+      if (ots_match_post (aWord, postfix) )
+      {  
+         toBreak=1;
+     		break; 
+      }
+      
+    }
+
+
+  for (li = (GList *) rule->ParserDontBreak; li != NULL; li = li->next)
+    {
+      postfix=li->data;
+      if (ots_match_post (aWord, postfix) )
+      {
+         toBreak=0;
+     		break; 
+      }
+      
+    }
+return toBreak;
+}
+      
+
+
+void
+ots_parse_stream(const unsigned char *utf8, size_t len, OtsArticle * Doc)	/*parse the unicode stream */
+{
+
+  OtsSentence *tmpLine = ots_append_line (Doc);
+  OtsStemRule * rule=Doc->stem;
+  gunichar uc;
+  int index = 0;
+  char *s = (char *) utf8;
+  GString *word_buffer = g_string_new (NULL);
+
+   
+while ((*s) && (index < len))
+    {
+  uc = g_utf8_get_char (s);      
+     
+      if (!g_unichar_isspace (uc))	/* space is the end of a word */
+		{
+
+		g_string_append_unichar(word_buffer,uc); 
+
+		} 
+		else 
+		{
+		  
+		  if (0<word_buffer->len)
+				{
+				ots_append_word (tmpLine, word_buffer->str);
+  				
+  				if (ots_parser_should_break(word_buffer->str,rule)) {
+  					tmpLine = ots_append_line (Doc);	/* Add a new Line */
+  					}
+  	
+  				g_string_assign (word_buffer, "");
+  				
+  				}
+		
+		if (uc=='\n') {ots_append_word (tmpLine,"\n");}
+		else 
+		{ots_append_word (tmpLine," ");}
+		
+		g_string_assign (word_buffer,"");
+		}
+
+      s = g_utf8_next_char (s);
+
+      index++;
+    }
+
+
+		if (0<word_buffer->len) /*final flush*/
+				{
+				ots_append_word (tmpLine, word_buffer->str);
+				g_string_assign (word_buffer, "");
+  				}
+  				 
+ 
+
+  g_string_free (word_buffer, TRUE);
+}

data/ext/summarize/relations.c

@@ -0,0 +1,163 @@
+/*
+ *  relations.c
+ *
+ *  Copyright (C) 2003 Nadav Rotem <nadav256@hotmail.com>
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 2 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU Library General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program; if not, write to the Free Software
+ *  Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+ */
+ 
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include "grader-tc.h"
+
+#include "libots.h"
+/*
+The Inner product of two texts is defined as the number of topics they
+share. This set of functions implements this relations using the ots
+api.
+
+Application: a relation between a slashdot article and a comment made
+usage: ots_text_relations(story,"en",comment,"en",n);
+where n is the max number of most important topics to consider; safe to give a high number (ex: 20);
+
+returns: 
+	0 - off topic
+	n - number of topics they share
+
+*/
+
+#define OTS_MAX_TOPIC_WORD_SIZE 256
+
+/*Returns the number of topics that two blocks of text share*/
+int ots_text_relations(
+const unsigned char *text1,const unsigned char *lang_code1,
+const unsigned char *text2,const unsigned char *lang_code2,const int topic_num)
+{
+GList* top1;
+GList* top2;
+int score;
+
+top1=ots_text_stem_list(text1,lang_code1,topic_num);
+top2=ots_text_stem_list(text2,lang_code2,topic_num);
+
+score=ots_topic_list_score(top1,top2);
+
+if (top1){g_list_foreach (top1, (GFunc) g_free, NULL);g_list_free (top1);}
+if (top2){g_list_foreach (top2, (GFunc) g_free, NULL);g_list_free (top2);}
+
+return score;
+}
+
+
+
+
+/*For a given text, return the list of the topics*/
+char* ots_text_topics(
+const unsigned char *text,const unsigned char *lang_code,int topic_num)
+{
+int i;
+GString *word;
+unsigned char *str;
+unsigned char *tmp;
+OtsArticle *Art;
+
+if (NULL==text) return NULL;
+word = g_string_new (NULL);
+
+Art = ots_new_article ();
+
+ots_load_xml_dictionary(Art,lang_code); /*Load the dictionary*/
+if (text!=NULL) ots_parse_stream (text,strlen(text), Art);   /* read text , put it in struct Article */
+ots_grade_doc (Art);
+
+
+for (i=0;i<=topic_num;i++)
+{
+tmp=ots_word_in_list(Art->ImpWords,i);
+if ((tmp!=NULL)&&(strlen(tmp)>0)) {g_string_append(word,tmp);
+		g_string_append(word," "); }
+}
+
+
+str=word->str;
+g_string_free (word, FALSE);
+ots_free_article (Art);
+
+return str;
+}
+
+
+
+/*For a given text, return the list of the stemmed topics*/
+GList* ots_text_stem_list(const unsigned char *text, const unsigned char *lang_code, int topic_num)
+{
+int i;
+GList *topics=NULL;
+unsigned char *tmp;
+OtsArticle *Art;
+
+if (NULL==text) return NULL;
+
+Art = ots_new_article ();
+
+ots_load_xml_dictionary(Art,lang_code); 
+if (text!=NULL) ots_parse_stream (text,strlen(text), Art);   
+ots_grade_doc (Art);
+
+
+for (i=0;i<=topic_num;i++)
+{
+tmp=ots_stem_in_list(Art->ImpWords,i);
+if ((tmp)&&(strlen(tmp)>0))
+topics=g_list_append(topics,g_strdup(tmp));
+}
+
+
+ots_free_article (Art);
+return topics;
+}		
+
+/*Gives a score on the relations between two lists of topics; simmilar to the inner product*/
+int ots_topic_list_score(
+const GList *topic_list1,
+const GList *topic_list2)
+{
+	int count=0;
+	GList *tmplist1;
+	GList *tmplist2;
+	
+	if (!(topic_list1)) return 0;
+	if (!(topic_list2)) return 0;
+	
+	tmplist1 = g_list_first(topic_list1);
+	while(tmplist1)
+	{
+	tmplist2 = g_list_first(topic_list2);
+	while(tmplist2)
+		{
+		
+		if ((tmplist1->data)&&(tmplist2->data)&&(strlen(tmplist2->data)>1)) 
+			if (0==strncmp(tmplist1->data,tmplist2->data,OTS_MAX_TOPIC_WORD_SIZE)) 
+							{count++;}
+		
+		tmplist2 = g_list_next(tmplist2);
+		}
+	tmplist1 = g_list_next(tmplist1);
+	}
+
+return count;
+}
+