ots 0.4.3 → 0.4.4

data/ext/ots.h

@@ -0,0 +1,15 @@
+#pragma once
+
+#include <ruby.h>
+#include <ruby/encoding.h>
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include <libots.h>
+#include "version.h"
+
+#define TO_S(v)                     rb_funcall(v, rb_intern("to_s"), 0)
+#define CSTRING(v)                  RSTRING_PTR(TO_S(v))
+#define rb_enc_str_new2(text, enc)  rb_enc_str_new(text, strlen(text), enc)

data/ext/parser.c

@@ -0,0 +1,173 @@
+/*
+ *  parser.c
+ *
+ *  Copyright (C) 2003 Nadav Rotem <nadav256@hotmail.com>
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 2 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU Library General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program; if not, write to the Free Software
+ *  Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <strings.h>
+#include "libots.h"
+
+#define BUFFER_SIZE (1024*8)
+
+int
+ots_match_post (const char *aWord,const char *post)
+{
+  int i, wlen, plen;
+  
+
+  wlen = strlen (aWord);
+  plen = strlen (post);
+
+  if (plen > wlen) return 0;
+
+  for (i = 0; i < plen; i++)
+    if (aWord[wlen - plen + i] != post[i])
+      return 0;		/* no match */
+
+  return 1;		/*word match */
+}
+
+void
+ots_parse_file (FILE * stream, OtsArticle * Doc )
+{
+  unsigned char fread_buffer[BUFFER_SIZE];
+  unsigned char *buffer;
+  size_t nread, total_read, avail_size;
+
+  buffer = g_new0 (unsigned char, BUFFER_SIZE);
+
+  avail_size = BUFFER_SIZE;
+  total_read = nread = 0;
+  while ((nread =
+	  fread (fread_buffer, sizeof (unsigned char), sizeof (fread_buffer),
+		 stream)) > 0)
+    {
+      if (nread + total_read > avail_size)
+	{
+	  avail_size *= 2;
+	  buffer = g_renew (unsigned char, buffer, avail_size);
+	}
+
+      strncpy (buffer + total_read, fread_buffer, nread);
+      total_read += nread;
+    }
+
+  ots_parse_stream (buffer, total_read, Doc);
+  g_free (buffer);
+}
+
+
+
+
+
+int
+ots_parser_should_break(const char *aWord,const OtsStemRule * rule)
+{
+  GList *li;
+  char *postfix;
+  int toBreak=0;
+  
+  for (li = (GList *) rule->ParserBreak; li != NULL; li = li->next)
+    {
+      postfix=li->data;
+      if (ots_match_post (aWord, postfix) )
+      {  
+         toBreak=1;
+     		break; 
+      }
+      
+    }
+
+
+  for (li = (GList *) rule->ParserDontBreak; li != NULL; li = li->next)
+    {
+      postfix=li->data;
+      if (ots_match_post (aWord, postfix) )
+      {
+         toBreak=0;
+     		break; 
+      }
+      
+    }
+return toBreak;
+}
+      
+
+
+void
+ots_parse_stream(const unsigned char *utf8, size_t len, OtsArticle * Doc)	/*parse the unicode stream */
+{
+
+  OtsSentence *tmpLine = ots_append_line (Doc);
+  OtsStemRule * rule=Doc->stem;
+  gunichar uc;
+  int index = 0;
+  char *s = (char *) utf8;
+  GString *word_buffer = g_string_new (NULL);
+
+   
+while ((*s) && (index < len))
+    {
+  uc = g_utf8_get_char (s);      
+     
+      if (!g_unichar_isspace (uc))	/* space is the end of a word */
+		{
+
+		g_string_append_unichar(word_buffer,uc); 
+
+		} 
+		else 
+		{
+		  
+		  if (0<word_buffer->len)
+				{
+				ots_append_word (tmpLine, word_buffer->str);
+  				
+  				if (ots_parser_should_break(word_buffer->str,rule)) {
+  					tmpLine = ots_append_line (Doc);	/* Add a new Line */
+  					}
+  	
+  				g_string_assign (word_buffer, "");
+  				
+  				}
+		
+		if (uc=='\n') {ots_append_word (tmpLine,"\n");}
+		else 
+		{ots_append_word (tmpLine," ");}
+		
+		g_string_assign (word_buffer,"");
+		}
+
+      s = g_utf8_next_char (s);
+
+      index++;
+    }
+
+
+		if (0<word_buffer->len) /*final flush*/
+				{
+				ots_append_word (tmpLine, word_buffer->str);
+				g_string_assign (word_buffer, "");
+  				}
+  				 
+ 
+
+  g_string_free (word_buffer, TRUE);
+}

data/ext/relations.c

@@ -0,0 +1,163 @@
+/*
+ *  relations.c
+ *
+ *  Copyright (C) 2003 Nadav Rotem <nadav256@hotmail.com>
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 2 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU Library General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program; if not, write to the Free Software
+ *  Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+ */
+ 
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include "grader-tc.h"
+
+#include "libots.h"
+/*
+The Inner product of two texts is defined as the number of topics they
+share. This set of functions implements this relations using the ots
+api.
+
+Application: a relation between a slashdot article and a comment made
+usage: ots_text_relations(story,"en",comment,"en",n);
+where n is the max number of most important topics to consider; safe to give a high number (ex: 20);
+
+returns: 
+	0 - off topic
+	n - number of topics they share
+
+*/
+
+#define OTS_MAX_TOPIC_WORD_SIZE 256
+
+/*Returns the number of topics that two blocks of text share*/
+int ots_text_relations(
+const unsigned char *text1,const unsigned char *lang_code1,
+const unsigned char *text2,const unsigned char *lang_code2,const int topic_num)
+{
+GList* top1;
+GList* top2;
+int score;
+
+top1=ots_text_stem_list(text1,lang_code1,topic_num);
+top2=ots_text_stem_list(text2,lang_code2,topic_num);
+
+score=ots_topic_list_score(top1,top2);
+
+if (top1){g_list_foreach (top1, (GFunc) g_free, NULL);g_list_free (top1);}
+if (top2){g_list_foreach (top2, (GFunc) g_free, NULL);g_list_free (top2);}
+
+return score;
+}
+
+
+
+
+/*For a given text, return the list of the topics*/
+char* ots_text_topics(
+const unsigned char *text,const unsigned char *lang_code,int topic_num)
+{
+int i;
+GString *word;
+unsigned char *str;
+unsigned char *tmp;
+OtsArticle *Art;
+
+if (NULL==text) return NULL;
+word = g_string_new (NULL);
+
+Art = ots_new_article ();
+
+ots_load_xml_dictionary(Art,lang_code); /*Load the dictionary*/
+if (text!=NULL) ots_parse_stream (text,strlen(text), Art);   /* read text , put it in struct Article */
+ots_grade_doc (Art);
+
+
+for (i=0;i<=topic_num;i++)
+{
+tmp=ots_word_in_list(Art->ImpWords,i);
+if ((tmp!=NULL)&&(strlen(tmp)>0)) {g_string_append(word,tmp);
+		g_string_append(word," "); }
+}
+
+
+str=word->str;
+g_string_free (word, FALSE);
+ots_free_article (Art);
+
+return str;
+}
+
+
+
+/*For a given text, return the list of the stemmed topics*/
+GList* ots_text_stem_list(const unsigned char *text, const unsigned char *lang_code, int topic_num)
+{
+int i;
+GList *topics=NULL;
+unsigned char *tmp;
+OtsArticle *Art;
+
+if (NULL==text) return NULL;
+
+Art = ots_new_article ();
+
+ots_load_xml_dictionary(Art,lang_code); 
+if (text!=NULL) ots_parse_stream (text,strlen(text), Art);   
+ots_grade_doc (Art);
+
+
+for (i=0;i<=topic_num;i++)
+{
+tmp=ots_stem_in_list(Art->ImpWords,i);
+if ((tmp)&&(strlen(tmp)>0))
+topics=g_list_append(topics,g_strdup(tmp));
+}
+
+
+ots_free_article (Art);
+return topics;
+}		
+
+/*Gives a score on the relations between two lists of topics; simmilar to the inner product*/
+int ots_topic_list_score(
+const GList *topic_list1,
+const GList *topic_list2)
+{
+	int count=0;
+	GList *tmplist1;
+	GList *tmplist2;
+	
+	if (!(topic_list1)) return 0;
+	if (!(topic_list2)) return 0;
+	
+	tmplist1 = g_list_first(topic_list1);
+	while(tmplist1)
+	{
+	tmplist2 = g_list_first(topic_list2);
+	while(tmplist2)
+		{
+		
+		if ((tmplist1->data)&&(tmplist2->data)&&(strlen(tmplist2->data)>1)) 
+			if (0==strncmp(tmplist1->data,tmplist2->data,OTS_MAX_TOPIC_WORD_SIZE)) 
+							{count++;}
+		
+		tmplist2 = g_list_next(tmplist2);
+		}
+	tmplist1 = g_list_next(tmplist1);
+	}
+
+return count;
+}
+	

data/ext/stemmer.c

@@ -0,0 +1,332 @@
+/*
+ *  stemmer.c
+ *
+ *  Copyright (C) 2003 Nadav Rotem <nadav256@hotmail.com>
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 2 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU Library General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program; if not, write to the Free Software
+ *  Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+ */
+
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include "libots.h"
+
+#define MAX_PREFIX_SIZE 256
+
+OtsStemRule *
+new_stem_rule ()
+{
+  OtsStemRule *rule = g_new0 (OtsStemRule, 1);
+  return rule;
+}
+
+void
+free_stem_rule (OtsStemRule *rule)
+{
+
+if (rule != NULL)
+    {
+      g_list_foreach (rule->RemovePre, (GFunc) g_free, NULL);
+      g_list_free (rule->RemovePre);
+      g_list_foreach (rule->RemovePost, (GFunc) g_free, NULL);
+      g_list_free (rule->RemovePost);
+
+      g_list_foreach (rule->step1_pre, (GFunc) g_free, NULL);
+      g_list_free (rule->step1_pre);      
+      g_list_foreach (rule->step1_post, (GFunc) g_free, NULL);
+      g_list_free (rule->step1_post);
+
+      g_list_foreach (rule->synonyms, (GFunc) g_free, NULL);
+      g_list_free (rule->synonyms);
+      g_list_foreach (rule->manual, (GFunc) g_free, NULL);
+      g_list_free (rule->manual);
+
+      g_list_foreach (rule->ParserBreak, (GFunc) g_free, NULL);
+      g_list_free (rule->ParserBreak);
+      g_list_foreach (rule->ParserDontBreak, (GFunc) g_free, NULL);
+      g_list_free (rule->ParserDontBreak);
+
+      g_list_foreach (rule->ReplaceChars, (GFunc) g_free, NULL);
+      g_list_free (rule->ReplaceChars);           
+ 
+      g_free (rule);
+    } 
+  return;
+}
+
+static void
+ots_stem_break (unsigned const char *comp,unsigned char *part_a,unsigned char *part_b)	/*given already alocated part_a and b */
+{				/*example "red|blue" */
+  int i, j, clen;
+  i = 0;
+  j = 0;
+  
+  if (comp==NULL) return;
+  if (part_a==NULL) return;
+  if (part_b==NULL) return;
+
+  clen = strlen (comp);
+
+
+  part_a[0] = 0;
+  part_b[0] = 0;
+
+  while ((i < clen) && (i < MAX_PREFIX_SIZE) && (comp[i] != '|'))
+    {
+      part_a[i] = comp[i];
+      i++;
+    }
+  part_a[i] = 0;
+
+  i++;				/*skip the | mark */
+  while (i < clen && (j < MAX_PREFIX_SIZE))
+    {
+      part_b[j] = comp[i];
+      i++;
+      j++;
+    }
+  part_b[j] = 0;
+  return;
+}
+
+
+static unsigned char *
+ots_stem_remove_pre (unsigned const char *aWord,unsigned const char *pre,unsigned const char *new)
+{
+  int i, plen, wlen, nlen;
+  unsigned char *new_str = NULL;
+
+  if (aWord==NULL) return NULL;
+
+  plen = strlen (pre);
+  wlen = strlen (aWord);
+  nlen = strlen (new);
+
+  for (i = 0; i < plen; i++)
+    if (aWord[i] != pre[i])
+      return NULL;		/*no match */
+
+  new_str = g_new0 (char, wlen + nlen +5);
+  for (i = 0; i <= nlen; i++)
+    new_str[i] = new[i];
+
+  for (i = nlen; i <= nlen + wlen - plen; i++)
+    new_str[i] = aWord[i + plen - nlen];
+
+  new_str[i + 1] = 0;
+  return new_str;
+}
+
+
+
+static unsigned char *
+ots_stem_remove_post (unsigned const char *aWord,unsigned const char *post,unsigned const char *new)
+{
+  unsigned int i, wlen, plen, nlen;
+  unsigned char *new_str = NULL;
+
+  if ((NULL==aWord)||(NULL==post)||(NULL==new)) return NULL;
+
+  wlen = strlen (aWord);
+  plen = strlen (post);
+  nlen = strlen (new);
+
+  if (plen>wlen) return NULL;
+
+
+  for (i = 0; i < plen; i++)
+    if (aWord[wlen - plen + i]!= post[i])
+      return NULL;		/* no match */
+
+  new_str = g_new0 (char, wlen + nlen +5);
+
+  for (i = 0; i <= wlen - plen; i++)	/*place word */
+    new_str[i] = aWord[i];
+
+  for (i = 0; i <= nlen; i++)	/*place newfix */
+    new_str[wlen - plen + i] = new[i];
+
+  return new_str;		/*word replaced */
+}  
+  
+
+
+static unsigned char *
+ots_stem_replace_word (unsigned const char *aWord,unsigned const char *old,unsigned const char *new)
+{
+
+  if (aWord==NULL) return NULL;
+  
+if ((aWord)&&(0 == strcmp (aWord, old)))
+    {
+      return g_strdup (new);
+    }
+  else
+    {
+      return NULL;
+    }
+
+}
+
+
+
+
+
+unsigned char *
+ots_stem_format (unsigned const char *aWord, const OtsStemRule * rule)
+{
+  GList *li;
+  unsigned char *rep = NULL;
+  unsigned char *normWord = NULL;
+
+  if (aWord==NULL) return NULL;
+  
+  normWord = g_utf8_strdown (aWord, -1);	/*lowercase the word */
+
+  char *prefix;
+  char *newfix;
+
+  prefix = g_new0 (char, MAX_PREFIX_SIZE);
+  newfix = g_new0 (char, MAX_PREFIX_SIZE);
+
+ for (li = (GList *) rule->step1_pre; li != NULL; li = li->next)
+    {
+      ots_stem_break (li->data, prefix, newfix);
+      rep = ots_stem_remove_pre (normWord, prefix, newfix);
+      if (NULL != rep)
+		{
+	  		g_free (normWord);
+	  		normWord = rep;
+	  		rep = NULL;
+		}
+    }
+
+
+ for (li = (GList *) rule->step1_post; li != NULL; li = li->next)
+    {
+      ots_stem_break (li->data, prefix, newfix);
+      rep = ots_stem_remove_post(normWord, prefix, newfix);
+      if (NULL != rep)
+		{
+	  		g_free (normWord);
+	  		normWord = rep;
+	  		rep = NULL;
+		}
+    }
+
+  g_free (prefix);
+  g_free (newfix);
+
+  return normWord;
+}
+
+
+
+
+
+
+
+
+unsigned char *
+ots_stem_strip (unsigned const char *aWord,const OtsStemRule * rule)
+{
+  GList *li;
+  unsigned char *rep = NULL;
+
+  unsigned char *prefix;
+  unsigned char *newfix;
+  unsigned char *normWord=NULL;
+
+  prefix = g_new0 (char, MAX_PREFIX_SIZE);
+  newfix = g_new0 (char, MAX_PREFIX_SIZE);
+
+  if (aWord==NULL) return NULL;
+
+  normWord = ots_stem_format (aWord,rule);
+
+
+  for (li = (GList *) rule->manual; li != NULL; li = li->next)
+    {
+      ots_stem_break (li->data, prefix, newfix);
+      rep = ots_stem_replace_word (normWord, prefix, newfix);
+      if (NULL != rep)
+	{
+	  g_free (normWord);
+	  normWord = rep;
+	  rep = NULL;
+	  break;
+	}
+    }
+
+
+
+
+  for (li = (GList *) rule->RemovePre; li != NULL; li = li->next)
+    {
+      ots_stem_break (li->data, prefix, newfix);
+      rep = ots_stem_remove_pre (normWord, prefix, newfix);
+      if (NULL != rep)
+	{
+	  g_free (normWord);
+	  normWord = rep;
+	  rep = NULL;
+	  break;
+	}
+    }
+
+
+  for (li = (GList *) rule->RemovePost; li != NULL; li = li->next)
+    {
+      ots_stem_break (li->data, prefix, newfix);
+      rep = ots_stem_remove_post (normWord, prefix, newfix);
+      if (NULL != rep)
+	{
+	  g_free (normWord);
+	  normWord = rep;
+	  rep = NULL;
+	  break;
+	}
+
+    }
+
+
+  for (li = (GList *) rule->synonyms; li != NULL; li = li->next)
+    {
+      ots_stem_break (li->data, prefix, newfix);
+      rep = ots_stem_replace_word (normWord, prefix, newfix);
+      if (NULL != rep)
+	{
+	  g_free (normWord);
+	  normWord = rep;
+	  rep = NULL;
+	  break;
+	}
+    }
+
+
+  g_free (prefix);
+  g_free (newfix);
+
+
+  if (strlen(normWord)<3)  /*stem is two letter long. thats not right. N(eed)==N(ation) ?*/
+		{ 
+		g_free(normWord);  
+		normWord = ots_stem_format (aWord,rule);	/*lowercase the word */	
+		}
+
+
+  return normWord;
+}