ruletagger 0.0.1

data/ext/rule_tagger/darray.h

@@ -0,0 +1,125 @@
+#ifndef _darray_h_
+#define _darray_h_
+
+#include "sysdep.h"
+#include "bool.h"
+
+typedef struct st_Darray *Darray;
+
+#ifdef __STDC__
+extern Darray Darray_create(NOARGS);
+extern NORET Darray_destroy(Darray);
+extern NORET Darray_hint(Darray, unsigned int, unsigned int);
+extern unsigned int Darray_len(Darray);
+extern NORET Darray_addh(Darray, VOIDP);
+extern NORET Darray_addl(Darray, VOIDP);
+extern VOIDP Darray_remh(Darray);
+extern VOIDP Darray_reml(Darray);
+extern Bool Darray_valid_index(Darray, unsigned int);
+extern NORET Darray_set(Darray, unsigned int, VOIDP);
+extern VOIDP Darray_get(Darray, unsigned int);
+extern NORET Darray_values(Darray, VOIDP *);
+extern Darray Darray_copy(Darray);
+extern Darray Darray_remove(Darray, int);
+extern Darray Darray_duplicate(Darray);
+extern Darray Darray_insert(Darray, int, VOIDP);
+extern void   Darray_clear(Darray); 
+#else
+extern Darray Darray_create();
+extern NORET Darray_destroy();
+extern NORET Darray_hint();
+extern unsigned int Darray_len();
+extern NORET Darray_addh();
+extern NORET Darray_addl();
+extern VOIDP Darray_remh();
+extern VOIDP Darray_reml();
+extern Bool Darray_valid_index();
+extern NORET Darray_set();
+extern VOIDP Darray_get();
+extern NORET Darray_values();
+extern Darray Darray_copy();
+extern Darray Darray_remove();
+extern Darray Darray_duplicate();
+extern Darray Darray_insert();
+extern void   Darray_clear(); 
+#endif /* __STDC__ */
+
+
+/*
+ * Dynamic arrays are zero-based (0 <= index < length-of-array)
+ *
+ * Darray_create()
+ * Creates and returns an empty dynamic array.
+ *
+ * Darray_destroy(dynamic_array)
+ * Frees all memory used by the dynamic_array.  Calling this routine
+ * should be the last use of the dynamic_array.  The object held by
+ * the array are not destroyed.
+ * 
+ * Darray_hint(dynamic_array, addl_hint, addh_hint)
+ * The hints specify how many additional (occuring after the hint call) addl
+ * and addh operations are expected.  Large values may improve performance
+ * at the cost of additional memory usage.
+ *
+ * Darray_len(dynamic_array)
+ * Returns the length of the dynamic array passed.
+ *
+ * Darray_addh(dynamic_array, element)
+ * Adds the element to the "high end" of the dynamic array.  
+ * 
+ * Darray_addl(dynamic_array, element)
+ * Adds the element to the "low end" of the dynamic array.  
+ *
+ * Darray_remh(dynamic_array)
+ * Returns element at the "high end" of the dynamic array,
+ * and removes that element from the array.  Darray_len(dynamic_array)
+ * must be > 0.
+ *
+ * Darray_reml(dynamic_array)
+ * Returns element at the "low end" of the dynamic array,
+ * and removes that element from the array.  Darray_len(dynamic_array)
+ * must be > 0.
+ *
+ * Darray_valid_index(dynamic_array, index)
+ * Checks if index is valid for use in a call to Darray_set or Darray_get.
+ * Returns Bool_TRUE if index < Darray_len(dynamic_array), 
+ * Bool_FALSE otherwise.
+ *
+ * Darray_set(dynamic_array, index, new_value)
+ * Sets the index position in the dynamic array to be
+ * the new value.  Darray_valid_index(dynamic_array, index) must be Bool_TRUE.
+ * 
+ * Darray_get(dynamic_array, index)
+ * Returns the value at the index position in the dynamic array.
+ * Darray_valid_index(dynamic_array, index) must be Bool_TRUE.
+ * 
+ * Darray_values(dynamic_array, pointer to c_array)
+ * Stores all the elements (as void pointers) of the dynamic array into 
+ * the c_array.  The
+ * c_array must contain at least Darray_len(dynamic_array) slots for
+ * void pointers (note: not object pointers).
+ *
+ * Darray_copy(dynamic_array)
+ * Creates and returns a new dynamic array, with the same contents as
+ * the dynamic array passed.  The new dynamic array does not inherit 
+ * any hints that may apply to the original dynamic array.
+ * 
+ * Darray_remove(dynamic_array, index)
+ * removes the item indexed by index from the array.  Index is base 0
+ * (i.e. first item in array is 0, last item is Darray_len()-1)
+ * (added by Rich Pito: this is not efficient code at the moment)
+ * 
+ * Darray_insert(dynamic_array, index, data)
+ * inserts the data into the dynamic_array at index.  data is inserted
+ * at position index.  All items with position values greater than or
+ * equal to index are assigned new position values one greater than
+ * their old ones.  If index is greater than the maximum position
+ * value in the dynamic_array, data is added as the topmost element to
+ * the array.  If index is less than zero it is inserted as if index
+ * were zero. 
+ * (added by Rich Pito: this is not efficient code at the moment)
+
+
+ */
+
+#endif /* ifndef _darray_h_ */

data/ext/rule_tagger/darrayP.h

@@ -0,0 +1,50 @@
+#ifndef _darrayP_h_
+#define _darrayP_h_
+
+#include "sysdep.h"
+#include "memory.h"
+
+#include "darray.h"
+
+typedef struct st_Darray {
+  unsigned length;
+  VOIDP *storage;
+  unsigned storage_offset;
+  unsigned storage_length;
+} Darray_rep;
+
+#define draise(p_to_rep) ((Darray)p_to_rep)
+#define dlower(obj) ((Darray_rep *)obj)
+#define dcreate() ((Darray_rep *)Memory_allocate(sizeof(Darray_rep)))
+/*#define destroy(p_to_rep) (Memory_free(p_to_rep))*/
+
+#define MAX_GROW_STEP 100	/* Defines the maximum amount to increase
+				   storage_length when storage is full.
+				   A call to Darray_hint can cause a larger
+				   grow to occur; this only applies to 
+				   "on-demand grows" */
+
+enum grow_direction {GROW_HIGH, GROW_LOW};
+
+#ifdef __STDC__
+static NORET grow(Darray_rep *, enum grow_direction, unsigned);
+#else
+static NORET grow();
+#endif
+
+/* Invariant conditions:
+ * 
+ * Each array component (object) is stored in a void pointer.
+ *
+ * [storage_length] >= 1
+ * 
+ * [storage] is pointer to a a heap object large enough for
+ * [storage_length] pointers to void,
+ *
+ * [storage_length] >= [length]
+ * 
+ * The n-th element (index=n-1) of the dynamic array is located at 
+ * address [storage]+[storage_offset]+n-1.
+ *
+ */ 
+#endif /* ifndef _darrayP_h_ */

data/ext/rule_tagger/extconf.rb

@@ -0,0 +1,14 @@
+require 'mkmf'
+
+dir_config('rule_tagger')
+have_header('stdlib.h')
+have_header('string.h')
+have_library('c', 'main')
+
+if !have_func('snprintf', 'stdio.h')
+  raise "You must have snprintf available to compile this library"
+end
+
+CFLAGS='-Wall -g'
+
+create_makefile('rule_tagger')

data/ext/rule_tagger/lex.c

@@ -0,0 +1,170 @@
+#include <stdio.h>
+#include <string.h>
+#include <stdlib.h>
+#include "useful.h"
+#include "lex.h"
+
+int numspaces(char *buf);
+int numchars(char *buf, char achar);
+
+char *append_with_space(w1,w2)
+     char *w1,*w2;
+{
+  char *result;
+  
+  result = (char *)malloc((strlen(w1) + strlen(w2)+2) * sizeof(char)); 
+  strcpy(result,w1);
+  strcat(result," ");
+  strcat(result,w2);
+  return(result);
+}
+
+char *append_with_char(w1,w2,w3)
+     char *w1,*w2,w3;
+{
+  char *result;
+  result = (char *)malloc((strlen(w1) + strlen(w2)+2) * sizeof(char)); 
+  sprintf(result,"%s%c%s",w1,w3,w2);
+  return(result);
+}
+  
+void perl_split_free( char **split_buf )
+{
+  int i;
+  for( i = 0; split_buf[i] != NULL; ++i ) { 
+    free( split_buf[i] );
+  }
+  free( split_buf );
+}
+
+
+char **perl_split(buf)
+     const char *buf;
+{
+  char **return_buf;
+  int cntr = 0;
+  char *temp,*temp2;
+
+  temp2 = (char *)malloc(sizeof(char)*(1+strlen(buf)));
+  while(*buf == ' ' || *buf == '\t') ++buf;
+  strcpy(temp2,buf);
+  return_buf = (char **) malloc(sizeof(char *) * ((numspaces(temp2)+1) + 2));
+  return_buf[cntr++] = strdup( (char *)strtok(temp2," \t") );
+  while ( (temp = (char *)strtok(NULL," \t")) ) 
+	if (temp != NULL) {
+		return_buf[cntr] = strdup(temp);
+		++cntr;}
+  return_buf[cntr] = NULL;
+  free(temp2);
+  return(return_buf);
+}
+
+
+
+char **perl_split_independent(buf_in)
+     const char *buf_in;
+{
+  char **return_buf;
+  int cntr = 0;
+  char *temp;
+  char *buf = mystrdup(buf_in);
+
+  while(*buf == ' ' || *buf == '\t') ++buf;
+  return_buf = (char **) malloc(sizeof(char *) * (numspaces(buf)+3));
+  return_buf[cntr++] = (char *)mystrdup((char *)strtok(buf," \t"));
+  while ((temp = (char *)strtok(NULL,"\t ")) != NULL) {
+    return_buf[cntr] =(char *)mystrdup(temp);
+    ++cntr;
+  }
+  return_buf[cntr] = NULL;
+  return(return_buf); }
+
+
+
+
+char **perl_split_on_char(buf,achar)
+     char *buf;
+     char achar;
+{
+  char **return_buf;
+  int cntr = 0;
+  char *temp,temp2[2],*temp3;
+
+  temp3 = (char *)malloc(sizeof(char)*(1+strlen(buf)));
+  temp2[0] = achar; temp2[1] = '\0';
+  return_buf = (char **) malloc(sizeof(char *) * ((numchars(temp3,achar)+1) + 2));
+  return_buf[cntr++] = (char *)strtok(temp3,temp2);
+  while ( (temp = (char *)strtok(NULL,temp2)) )
+	if (temp != NULL) {
+		return_buf[cntr] = temp;
+		++cntr;}
+  return_buf[cntr] = NULL;
+  return(return_buf); }
+
+
+
+
+char **perl_split_on_nothing(buf)
+     char *buf;
+{
+  char **return_buf;
+  int cntr;
+  char *temp2;  
+
+  temp2 = (char *)malloc(sizeof(char)*(1+strlen(buf)));
+  strcpy(temp2,buf);
+  
+  return_buf = (char **) malloc(sizeof(char *) * (strlen(buf)+1));
+  for (cntr = 0; cntr < strlen(buf); ++cntr) {
+    return_buf[cntr] = (char *)malloc(sizeof(char)*2);
+    return_buf[cntr][0] = temp2[cntr];
+    return_buf[cntr][1] = '\0'; }
+    return_buf[cntr] = NULL;
+    return(return_buf); 
+}
+
+int numspaces(buf)
+     char *buf;
+{
+  int tot,count;
+  tot = 0;
+  for (count = 0; count < strlen(buf); ++count) 
+    if (buf[count]==' ')
+      ++tot;
+  return(tot); }
+    
+int numchars(buf,achar)
+     char *buf,achar;
+{
+  int tot,count;
+  tot = 0;
+  for (count = 0; count < strlen(buf); ++count) 
+    if (buf[count]== achar)
+      ++tot;
+  return(tot); }
+    
+
+
+char *return_tag(theword)
+     char *theword;
+{
+  char *tempword;
+  tempword = (char *)strchr(theword,'/');
+  if (tempword != NULL)  return (tempword+1); 
+  else return(NULL); }
+
+
+char *before_tag(theword)
+     char *theword;
+{
+  int count = 0;
+  
+  while (theword[count] != '\0' &&
+	 theword[count] != '/')
+    count++;
+  if (theword[count] == '/')
+    theword[count] = '\0';
+  return(theword); }
+
+
+

data/ext/rule_tagger/lex.h

@@ -0,0 +1,49 @@
+#ifndef _LEX_H_
+#define _LEX_H_
+
+
+#ifdef __STDC__
+extern char *append_with_space(char *,char *); 
+    /* takes 2 strings, and appends them with a space btwn them */
+
+extern char *append_with_char(char *,char *,char);
+             /* takes 2 strings, and appends them with character w3 (the */
+	     /* third argument) btwn them */
+
+extern char **perl_split(const char *);
+	      /* takes a string with spaces and does a split, returning */
+	      /* an array of ptrs to strings. */
+              /* like perl - @temp = split(/\s+/,$buf); 
+		 Last ptr is a null. */
+     /* x = perl_split(buf);  then you are responsible for freeing
+	*x and x */
+extern void perl_split_free( char **split_buf );
+
+extern char **perl_split_independent(const char *);
+         /* same as perl_split, but each element in the array is a separate */
+	 /* string of memory. */
+
+extern char **perl_split_on_char(char *,char);
+        /* same as perl_split, but split on the specified character, instead */
+	/* of space */
+
+extern char **perl_split_on_nothing(char *);
+         /* same as perl_split, but split on nothing instead of space*/
+
+
+
+extern char *return_tag(char *);   /* returns a ptr to the tag
+				in a tagged word (the/dt),
+				or NULL if not tagged.*/
+#else
+extern char *append_with_space(); 
+extern char *append_with_char();
+extern char **perl_split();
+extern char **perl_split_independent();
+extern char **perl_split_on_char();
+extern char **perl_split_on_nothing();
+extern char *return_tag(); 
+ 
+#endif /* __STDC__ */
+
+#endif /* _LEX_H_ */

data/ext/rule_tagger/memory.c

@@ -0,0 +1,127 @@
+#include <stddef.h>
+#include <stdlib.h>
+#include <assert.h>
+
+#include <stdio.h>
+#include "sysdep.h"
+
+
+#ifndef MEMORY_LEAK_COUNT
+long Memory_unfreed_bytes(NOARGS)
+{
+  return 0;
+}
+#else
+static long bytes_unfreed;
+long Memory_unfreed_bytes(NOARGS)
+{
+  return bytes_unfreed;
+}
+
+#endif
+
+#ifdef MEMORY_CHECK_POINTERS
+
+#define MEMORY_REGISTRY_HT_SIZE 997
+
+#include "registry.h"
+#include "bool.h"
+static Registry valid_blocks = NULL;
+static int dont_check = 0;
+
+static NORET mem_add_to_rgy(temp, bytes)
+VOIDP temp;
+size_t bytes;
+{
+  dont_check = 1;
+  assert(Registry_add(valid_blocks, temp, (VOIDP)bytes)==Bool_TRUE);
+#ifdef MEMORY_LEAK_COUNT
+  bytes_unfreed += bytes;
+#endif
+  dont_check = 0;
+}
+
+static NORET mem_remove_from_rgy(temp)
+VOIDP temp;
+{
+  long num_bytes;
+
+  dont_check = 1;
+  num_bytes = (long)Registry_get(valid_blocks, temp);
+  assert(num_bytes != 0);
+#ifdef MEMORY_LEAK_COUNT
+  bytes_unfreed -= num_bytes;
+#endif
+  assert(Registry_remove(valid_blocks, temp));
+  dont_check = 0;
+}
+
+static int mem_rgy_initialized(NOARGS)
+{
+  if (valid_blocks == (Registry)NULL) {
+    dont_check = 1;
+    valid_blocks = Registry_create(Registry_ptrcmp,
+				   Registry_ptrhash);
+    Registry_size_hint(valid_blocks, (unsigned int)MEMORY_REGISTRY_HT_SIZE);
+    dont_check = 0;
+  }
+  return 1;
+}
+
+#endif
+
+VOIDP Memory_allocate(n)
+     size_t n;
+{
+  VOIDP temp;
+
+  temp = (VOIDP)malloc(n);
+
+#ifdef MEMORY_CHECK_POINTERS
+  assert(dont_check ||mem_rgy_initialized());
+#endif
+
+
+  if (temp == NULL) {
+    fprintf(stderr, "Fatal error allocating %d bytes", (int)n);
+    abort();
+  }
+#ifdef MEMORY_CHECK_POINTERS
+  assert(dont_check || (mem_add_to_rgy(temp,n),1));
+#endif
+  return temp;
+}
+
+VOIDP Memory_reallocate(ptr, n)
+     VOIDP ptr;
+     size_t n;
+{
+  VOIDP temp;
+
+#ifdef MEMORY_CHECK_POINTERS
+  assert(dont_check ||mem_rgy_initialized());
+  assert(dont_check ||(mem_remove_from_rgy(ptr),1));
+#endif
+
+  temp = (VOIDP)realloc(ptr, n);
+  if (temp == NULL) {
+    fprintf(stderr, "Fatal error allocating %d bytes", (int)n);
+    abort();
+  }
+#ifdef MEMORY_CHECK_POINTERS
+  assert(dont_check ||(mem_add_to_rgy(temp,n),1));
+#endif
+  return temp;
+}
+
+NORET Memory_free(ptr)
+     VOIDP ptr;
+{
+#ifdef MEMORY_CHECK_POINTERS
+  assert(dont_check ||mem_rgy_initialized());
+  assert(dont_check ||(mem_remove_from_rgy(ptr),1));
+#endif
+
+  if (ptr != NULL)
+    free(ptr);
+}

data/ext/rule_tagger/memory.h

@@ -0,0 +1,20 @@
+#ifndef _Memory_h_
+#define _Memory_h_
+
+#include <stddef.h>
+
+#include "sysdep.h"
+
+#ifdef __STDC__
+extern VOIDP Memory_allocate(size_t);
+extern VOIDP Memory_reallocate(VOIDP, size_t);
+extern NORET Memory_free(VOIDP);
+extern long Memory_unfreed_bytes(NOARGS);
+#else
+extern VOIDP Memory_allocate();
+extern VOIDP Memory_reallocate();
+extern NORET Memory_free();
+extern long Memory_unfreed_bytes(NOARGS);
+#endif /* __STDC__ */
+
+#endif