ots 0.4.3 → 0.4.4

data/ext/extconf.rb

@@ -1,26 +1,25 @@
 require 'mkmf'
 
-$CFLAGS  = %x{pkg-config --cflags glib-2.0}.strip
-$LGFLAGS = %x{pkg-config --libs glib-2.0}.strip
+glib_cflags   = %x{pkg-config --cflags  glib-2.0}.strip
+glib_ldflags  = %x{pkg-config --libs    glib-2.0}.strip
 
-if $CFLAGS.empty?
+if glib_cflags.empty?
   warn %q{WARNING: No pkg-config found for glib-2.0, using defaults. Set GLIB_INCLUDE_DIR env to override.}
   dirs = ENV.fetch('GLIB_INCLUDE_DIR', '/usr/include/glib-2.0 /usr/lib/glib-2.0/include')
-  $CFLAGS = dirs.split(/\s+/).map {|dir| "-I#{dir}"}.join(' ')
+  glib_cflags = dirs.split(/\s+/).map {|dir| "-I#{dir}"}.join(' ')
 end
 
-if $LDFLAGS.empty?
+if glib_ldflags.empty?
   warn %q{WARNING: No pkg-config found for glib-2.0, using defaults. Set GLIB_LIB env to override.}
   libs = ENV.fetch('GLIB_LIB', 'glib-2.0')
-  $LDFLAGS = libs.split(/\s+/).map {|lib| "-l#{lib}"}.join(' ')
+  glib_ldflags = libs.split(/\s+/).map {|lib| "-l#{lib}"}.join(' ')
 end
 
-dir_config("libots", ["/usr/local", "/opt/local", "/usr"])
+dir      = File.expand_path(File.dirname(__FILE__) + '/../dictionaries')
+$CFLAGS  = glib_cflags + %Q{ -I/usr/include/libxml2 -DDICTIONARY_DIR='"#{dir}/"'}
+$LDFLAGS = glib_ldflags
 
-headers = [ 'stdio.h', 'stdlib.h', 'string.h', 'libots-1/ots/libots.h' ]
-if have_header('libots-1/ots/libots.h') && have_library('ots-1', 'ots_new_article', headers)
-  create_makefile 'ots'
-else
-  puts "Cannot find libots headers or libraries"
-  exit 1
-end
+find_library('glib-2.0', 'main')
+find_library('xml2',     'main')
+
+create_makefile 'ots'

data/ext/grader-tc.c

@@ -0,0 +1,185 @@
+/*
+ *  grader-tc.c
+ *
+ *  Copyright (C) 2003 Nadav Rotem <nadav256@hotmail.com>
+ * 
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 2 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU Library General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program; if not, write to the Free Software
+ *  Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include "libots.h"
+
+
+#include "grader-tc.h"
+
+
+/*Grader - Term count algorithm*/
+/*This is non-normelized term frequency algorithm without using inverse document frequency database */
+
+#define NUM_KEY_WORDS 100 /* use first n key words only */
+
+int
+ots_get_article_word_count (const OtsArticle * Doc)
+{
+  GList *li;
+  int articleWC;
+  articleWC = 0;
+
+  if (Doc==NULL) return 0;
+
+  for (li = (GList *) Doc->lines; li != NULL; li = li->next)
+    {
+      articleWC += ((OtsSentence *) li->data)->wc;
+    }
+
+  return articleWC;
+}
+
+
+/*take this line and add each word to the "wordStat" list
+* this list will hold all of the words in the article and the number 
+* of times they appeared in the article.
+*/
+
+static void 
+ots_line_add_wordlist(OtsArticle * Doc,const OtsSentence * aLine) 
+{
+  GList *li;
+  if ((aLine==NULL) ||(NULL==Doc)) { return;}
+  
+      for (li = (GList *) aLine->words; li != NULL; li = li->next)	/* for each word in the sentence Do: */
+			if (li->data && strlen (li->data)) ots_add_wordstat (Doc, (char *)li->data);
+		    
+  return;
+}
+
+static void 
+ots_create_wordlist(OtsArticle * Doc)
+{
+GList *line;
+if (Doc==NULL) return;
+
+for (line = (GList *) Doc->lines; line != NULL; line = line->next)	
+	{ 
+	OtsSentence * aLine=line->data;
+	if (aLine)
+	ots_line_add_wordlist(Doc,aLine);
+	}
+}
+
+
+	
+
+static int
+keyVal (const int n)			/* Ugly , I know */
+{
+  if (n == 1) return 3;
+  if (n == 2) return 2;
+  if (n == 3) return 2;
+  if (n == 4) return 2;
+  return 1;
+}
+
+
+static void
+ots_grade_line (GList *impList, OtsSentence * aLine,
+		OtsStemRule * rule)
+{
+  GList *li;
+  GList *di;
+  int n;
+  char *tmp_stem;
+
+  if ((aLine==NULL)||(rule==NULL)||(impList==NULL)) return;
+
+  for (li = (GList *) aLine->words; li != NULL; li = li->next)	/* for each word */
+    {
+      n = 0;
+      tmp_stem = ots_stem_strip ((unsigned char *) li->data, rule);
+
+      for (di = (GList *) impList;
+	   ((di != NULL) && (n < NUM_KEY_WORDS)); di = di->next)
+	{
+	  n++;
+	if ((NULL!=((OtsWordEntery *) di->data)->stem) && (NULL!=tmp_stem))
+	  if (0 == strcmp ((((OtsWordEntery *) di->data)->stem), tmp_stem)) 
+	    {
+	      /* debug:  
+	      if (0!=strcmp((((OtsWordEntery *) di->data)->word),li->data)) 
+	      printf("[%s][%s] stem[%s]\n",(((OtsWordEntery *) di->data)->word),li->data,tmp);*/
+
+	      aLine->score += (((OtsWordEntery *) di->data)->occ) * keyVal (n);
+	    }
+
+	}
+
+      g_free (tmp_stem);
+    }
+
+}
+
+
+void
+ots_create_title_tc(OtsArticle * Doc)
+{
+
+	char *tmp;
+	char *word;
+	int i;
+    	GString *title;
+	if (NULL==Doc) return;
+
+	title=g_string_new(NULL);
+	
+	for (i=0;i<5;i++)
+	{
+	word = ots_word_in_list(Doc->ImpWords,i);
+	if (word) g_string_append(title,word); else break;
+	if (i<4) g_string_append(title,",");
+	}
+	
+	tmp=title->str;
+	if (NULL!=title) g_string_free(title,FALSE);
+	Doc->title=tmp;
+}
+
+
+void
+ots_grade_doc_tc (OtsArticle * Doc)
+{
+
+  GList *li;
+  if (NULL==Doc) return;  
+  ots_create_wordlist(Doc);
+ 
+  
+   Doc->ImpWords=ots_union_list (Doc->wordStat, Doc->dict);	   /* subtract from the Article wordlist all the words in the dic file (on , the , is...) */
+   Doc->ImpWords=ots_sort_list (Doc->ImpWords);			  /* sort the list , top 3 is what the article talks about (SARS , virus , cure ... ) */
+
+   /*to print wordlist: ots_print_wordlist (stdout, Doc->ImpWords);*/
+
+  if (0 == Doc->lineCount) return;
+
+  for (li = (GList *) Doc->lines; li != NULL; li = li->next)
+    {
+    if (li->data)
+      ots_grade_line (Doc->ImpWords, (OtsSentence *) li->data, Doc->stem);
+    }
+
+
+  ots_create_title_tc(Doc);
+}

data/ext/grader-tc.h

@@ -0,0 +1,64 @@
+/*
+ *  grader-tc.h
+ *
+ *  Copyright (C) 2003 Nadav Rotem <nadav256@hotmail.com>
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 2 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU Library General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program; if not, write to the Free Software
+ *  Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+ */
+
+#ifndef HAVE_GRADERTC_H
+#define HAVE_GRADERTC_H
+
+
+#include <glib.h>
+#include "libots.h"
+
+G_BEGIN_DECLS 
+
+
+typedef struct
+{
+  gchar *word;    /* the word */
+  gchar *stem;    /*stem of the word*/
+  gint occ;			/* how many times have we seen this word in the text? */
+} OtsWordEntery;
+
+/*Word list manipulations*/
+void ots_free_wordlist (GList *aList);
+
+
+
+OtsWordEntery *ots_copy_wordEntery (OtsWordEntery * obj);
+OtsWordEntery *ots_new_wordEntery (unsigned const char *wordString);
+OtsWordEntery *ots_new_wordEntery_strip (unsigned const char *wordString,const OtsStemRule *rule);
+void ots_free_wordEntery (OtsWordEntery * WC);
+
+GList *ots_sort_list (GList* aList);
+GList *ots_union_list (const GList *aLst, const GList * bLst);
+
+char *ots_word_in_list (const GList *aList,const int index);
+char *ots_stem_in_list (const GList *aList,const int index);
+void ots_add_wordstat (OtsArticle * Doc,unsigned const char *wordString);
+
+
+/*grader*/
+
+void ots_grade_doc_tc (OtsArticle * Doc);
+
+G_END_DECLS
+
+
+
+#endif /* HAVE_GRADERTC_H */

data/ext/grader-tf.c

@@ -0,0 +1,116 @@
+/*
+ *  grader-tf.c
+ *
+ *  Copyright (C) 2003 Nadav Rotem <nadav256@hotmail.com>
+ * 
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 2 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU Library General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program; if not, write to the Free Software
+ *  Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include "libots.h"
+
+/*Grader - using the Term frequency algorithm. Will give each line a score*/
+
+
+
+OtsWordTF* 
+ots_new_OtsWordTF(const char* word,const double tf)
+{
+ OtsWordTF* obj=g_new0(OtsWordTF,1);
+ if (word!=NULL) obj->word=g_strdup(word);
+ obj->tf=tf;
+ return obj;
+}
+
+void 
+ots_free_OtsWordTF(OtsWordTF *obj)
+{
+ if (obj!=NULL)
+  {
+   if (obj->word!=NULL) g_free(obj->word);
+   g_free(obj);
+  }
+}
+
+void
+ots_free_TF_wordlist (GList * aList)
+{  
+  if (aList != NULL)
+    {
+  		 	g_list_foreach(aList,(GFunc)ots_free_OtsWordTF, NULL);
+    			g_list_free(aList);
+    }
+}
+
+
+void 
+ots_grade_line_tf (OtsSentence * aLine)
+{
+
+return;
+}
+      
+
+
+void
+ots_grade_doc_tf (OtsArticle * Doc)
+{
+
+ GList *li;
+  
+  /*Load tf list*/
+  /*Load idf list*/
+    
+ if (0 == Doc->lineCount) return;
+
+  for (li = (GList *) Doc->lines; li != NULL; li = li->next)
+    {
+      ots_grade_line_tf ((OtsSentence *) li->data /* , tf list , idf list*/);
+    }
+    
+return; 
+}
+
+
+double
+ots_tf_word_score (const double tf,const double idf)
+/*IDF: how rare is word across the collection
+  TF: how often is word in doc */
+{
+
+return tf*idf; 
+}
+
+/*
+Determine frequency of query words
+n = (num-of-sentences words appears in) 
+N = (total-number-of-sentences)
+f = n/N
+*/
+
+double
+ots_calc_idf (const int term_count,const int doc_word_count)
+{
+return -log(doc_word_count/term_count);
+}
+
+double
+ots_calc_tf (const int term_count,const int doc_word_count)
+{
+if (term_count==0) return 0; else 
+return 0.5+0.5*(doc_word_count/term_count);
+}

data/ext/grader.c

@@ -0,0 +1,85 @@
+/*
+ *  grader.c
+ *
+ *  Copyright (C) 2003 Nadav Rotem <nadav256@hotmail.com>
+ * 
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 2 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU Library General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program; if not, write to the Free Software
+ *  Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include "libots.h"
+
+extern void ots_grade_doc_tc (OtsArticle * Doc);
+
+/*Grader driver - will call one of the grading algorithm*/
+
+
+
+void
+ots_grade_structure (OtsArticle * Doc) /*must be called after the first grader*/
+{
+  GList *li;
+  GList *first;
+  GList *second;
+  OtsSentence *first_line=NULL;
+
+  first = NULL;
+  second = NULL;
+
+ if (Doc==NULL) return;
+  
+ if (Doc->lines!=NULL) 
+  first_line= ((OtsSentence *) (Doc->lines->data));
+  if (NULL!=first_line) first_line->score *= 2;	/*first line/title is very important so we increase its score */
+
+ 	 /*This loop will *1.6 the score of each line that
+ 	 starts with \n \n , in other words a new paragraph*/
+ 
+  for (li = (GList *) Doc->lines; li != NULL; li = li->next)
+  {
+      OtsSentence *aLine = (li->data);
+      if (NULL != aLine)	/*line is there */
+		{
+		  first = aLine->words;	/*first word? */
+		  if (NULL != first)
+	  	  second = first->next;	/*second word? */
+	  		if ((NULL != first) && (NULL != second))	/*have content? */
+	    	if (strcmp (first->data, "\n") && strcmp (second->data, "\n"))	/*new paragraph? */
+	   		  	 aLine->score *= 1.6;
+		}
+
+  }
+
+}
+
+/** 
+Each grader needs to do:
+1.give a ->score to each line 
+2.Set the ->title of the document
+**/
+
+void
+ots_grade_doc (OtsArticle * Doc)
+{
+
+ if (Doc==NULL) return;
+  ots_grade_doc_tc(Doc);  /*Term count*/
+
+	/* or   ots_grade_doc_fc (Doc); Term Frequency  */
+		
+  ots_grade_structure (Doc);
+}