grammar_cop 0.1.0

data/ext/link_grammar/link-grammar/word-utils.h

@@ -0,0 +1,152 @@
+/*************************************************************************/
+/* Copyright (c) 2004                                                    */
+/* Daniel Sleator, David Temperley, and John Lafferty                    */
+/* All rights reserved                                                   */
+/*                                                                       */
+/* Use of the link grammar parsing system is subject to the terms of the */
+/* license set forth in the LICENSE file included with this software,    */ 
+/* and also available at http://www.link.cs.cmu.edu/link/license.html    */
+/* This license allows free redistribution and use in source and binary  */
+/* forms, with or without modification, subject to certain conditions.   */
+/*                                                                       */
+/*************************************************************************/
+
+#ifndef _LINK_GRAMMAR_WORD_UTILS_H_
+#define _LINK_GRAMMAR_WORD_UTILS_H_
+
+//#include <link-grammar/api-types.h>
+#include "api-types.h"
+
+/* Exp utilities ... */
+void free_Exp(Exp *);
+void free_E_list(E_list *);
+int  size_of_expression(Exp *);
+Exp * copy_Exp(Exp *);
+/* int exp_compare(Exp * e1, Exp * e2); */
+/* int exp_contains(Exp * super, Exp * sub); */
+
+
+/* X_node utilities ... */
+X_node *    catenate_X_nodes(X_node *, X_node *);
+void free_X_nodes(X_node *);
+
+
+/* Connector utilities ... */
+Connector * connector_new(void);
+Connector * init_connector(Connector *c);
+void free_connectors(Connector *);
+void exfree_connectors(Connector *);
+Connector * copy_connectors(Connector *);
+Connector * excopy_connectors(Connector * c);
+
+/* Link utilities ... */
+Link *      excopy_link(Link *);
+void        exfree_link(Link *);
+
+
+/* Connector-set utilities ... */
+Connector_set * connector_set_create(Exp *e);
+void connector_set_delete(Connector_set * conset);
+int match_in_connector_set(Sentence, Connector_set *conset, Connector * c, int d);
+int word_has_connector(Dict_node *, const char *, int);
+
+
+/* Dictionary utilities ... */
+int word_contains(Dictionary dict, const char * word, const char * macro);
+Dict_node * list_whole_dictionary(Dict_node *, Dict_node *);
+
+static inline int string_hash(const char *s)
+{
+	unsigned int i;
+
+	/* djb2 hash */
+	i = 5381;
+	while (*s)
+	{
+		i = ((i << 5) + i) + *s;
+		s++;
+	}
+	return i;
+}
+
+/**
+ * This hash function only looks at the leading upper case letters of
+ * the connector string, and the label fields.  This ensures that if two
+ * strings match (formally), then they must hash to the same place.
+ */
+static inline int connector_hash(Connector * c)
+{
+	const char *s;
+	unsigned int i;
+
+	if (-1 != c->hash) return c->hash;
+
+	/* For most situations, both hashes are very nearly equal;
+	 * sdbm seems to be about 5% faster than djb2, hard to say.
+	 * In either case, realize that the connector string is
+	 * very very short - usually one or two letters, so we have
+	 * probably only 8 or 10 bits total entropy coming in!  */
+#if 0
+	/* djb2 hash */
+	i = 5381;
+	i = ((i << 5) + i) + (0xff & c->label);
+	s = c->string;
+	while (isupper((int) *s)) /* connector tables cannot contain UTF8, yet */
+	{
+		i = ((i << 5) + i) + *s;
+		s++;
+	}
+	i += i>>14;
+
+#else
+	/* sdbm hash */
+	i = (0xff & c->label);
+	s = c->string;
+	while (isupper((int) *s)) /* connector tables cannot contain UTF8, yet */
+	{
+		i = *s + (i << 6) + (i << 16) - i;
+		s++;
+	}
+#endif
+
+	c->prune_string = s;
+	c->hash = i;
+	return i;
+}
+
+/**
+ * hash function. Based on some tests, this seems to be an almost
+ * "perfect" hash, in that almost all hash buckets have the same size!
+ */
+static inline int pair_hash(int log2_table_size,
+                            int lw, int rw,
+                            const Connector *le, const Connector *re,
+                            int cost)
+{
+	int table_size = (1 << log2_table_size);
+	unsigned int i = 0;
+
+#if 0
+ 	/* hash function. Based on some tests, this seems to be
+	 * an almost "perfect" hash, in that almost all hash buckets
+	 * have the same size! */
+	i += 1 << cost;
+	i += 1 << (lw % (log2_table_size-1));
+	i += 1 << (rw % (log2_table_size-1));
+	i += ((unsigned int) le) >> 2;
+	i += ((unsigned int) le) >> log2_table_size;
+	i += ((unsigned int) re) >> 2;
+	i += ((unsigned int) re) >> log2_table_size;
+	i += i >> log2_table_size;
+#else
+	/* sdbm-based hash */
+	i = cost;
+	i = lw + (i << 6) + (i << 16) - i;
+	i = rw + (i << 6) + (i << 16) - i;
+	i = ((unsigned long) le) + (i << 6) + (i << 16) - i;
+	i = ((unsigned long) re) + (i << 6) + (i << 16) - i;
+#endif
+
+   return i & (table_size-1);
+}
+#endif /* _LINK_GRAMMAR_WORD_UTILS_H_ */

data/ext/link_grammar/link_grammar.c

@@ -0,0 +1,202 @@
+#include <locale.h>
+#include <stdio.h>
+#include <string.h>
+#include <limits.h>
+#include <math.h>
+
+#include "link_grammar.h"
+
+void Init_link_grammar() {
+	VALUE lg = rb_define_module("LinkGrammar");
+	rb_define_module_function(lg, "create_dictionary", create_dictionary, 1);
+	rb_define_module_function(lg, "delete_dictionary", delete_dictionary, 1);
+	rb_define_module_function(lg, "dictionary_path_set", dictionary_path_set, 1);
+	rb_define_module_function(lg, "create_parse_options", create_parse_options, 0);
+	rb_define_module_function(lg, "delete_parse_options", delete_parse_options, 1);
+	rb_define_module_function(lg, "linkage_limit_set", linkage_limit_set, 2);
+	rb_define_module_function(lg, "linkage_limit_get", linkage_limit_get, 1);
+	rb_define_module_function(lg, "short_length_set", short_length_set, 2);
+	rb_define_module_function(lg, "short_length_get", short_length_get, 1);
+	rb_define_module_function(lg, "create_sentence", create_sentence, 2);
+	rb_define_module_function(lg, "parse_sentence", parse_sentence, 2);
+	rb_define_module_function(lg, "delete_sentence", delete_sentence, 1);
+	rb_define_module_function(lg, "create_linkage", create_linkage, 3);
+	rb_define_module_function(lg, "delete_linkage", delete_linkage, 1);
+	rb_define_module_function(lg, "print_linkage_diagram", print_linkage_diagram, 1);
+	rb_define_module_function(lg, "linkage_count_sublinkages", linkage_count_sublinkages, 1);
+	rb_define_module_function(lg, "linkage_count_num_words", linkage_count_num_words, 1);
+	rb_define_module_function(lg, "linkage_count_num_links", linkage_count_num_links, 1);	
+}
+
+//functions for retrieving pointers from ruby VALUE wrapper
+DictionaryPtr *retrieve_dictionary(VALUE dict) {
+	DictionaryPtr *ptr;
+	Data_Get_Struct(dict, DictionaryPtr, ptr);
+	return ptr;
+}
+
+ParseOptionsPtr *retrieve_parse_options(VALUE opts) {
+	ParseOptionsPtr *ptr;
+	Data_Get_Struct(opts, ParseOptionsPtr, ptr);
+	return ptr;
+}
+
+SentencePtr *retrieve_sentence(VALUE sentence) {
+	SentencePtr *ptr;
+	Data_Get_Struct(sentence, SentencePtr, ptr);
+	return ptr;
+}
+
+LinkagePtr *retrieve_linkage(VALUE link) {
+	LinkagePtr *ptr;
+	Data_Get_Struct(link, LinkagePtr, ptr);
+	return ptr;
+}
+
+//custom routines for freeing memory pointers
+void free_dictionary_ptr(DictionaryPtr *ptr) {
+	dictionary_delete(ptr->dict);
+	free(ptr);
+}
+
+void free_parse_options_ptr(ParseOptionsPtr *ptr) {
+	parse_options_delete(ptr->opts);
+	free(ptr);
+}
+
+void free_sentence_ptr(SentencePtr *ptr) {
+	sentence_delete(ptr->sentence);
+	free(ptr);
+}
+
+void free_linkage_ptr(LinkagePtr *ptr) {
+	linkage_delete(ptr->linkage);
+	free(ptr);
+}
+
+VALUE dictionary_path_set(const VALUE self, VALUE dict_path) {
+	char *pathstr = StringValuePtr(dict_path);
+	dictionary_set_data_dir(pathstr);
+	return Qnil;
+}
+
+VALUE create_dictionary(const VALUE self, volatile VALUE lang) {
+	Dictionary dict = dictionary_create_lang(StringValuePtr(lang));
+	DictionaryPtr *ptr = ALLOC(DictionaryPtr);
+	ptr->dict = NULL;
+	ptr->dict = dict;
+	return Data_Wrap_Struct(self, 0, free_dictionary_ptr, ptr);
+}
+
+VALUE delete_dictionary(const VALUE self, VALUE dict) {
+	DictionaryPtr *ptr = retrieve_dictionary(dict);
+	free_dictionary_ptr(ptr);
+	return Qnil;
+}
+
+VALUE create_parse_options(const VALUE self) {
+	Parse_Options options = parse_options_create();
+	ParseOptionsPtr *ptr = ALLOC(ParseOptionsPtr);
+	ptr->opts = NULL;
+	ptr->opts = options;
+	return Data_Wrap_Struct(self, 0, free_parse_options_ptr, ptr);
+}
+
+VALUE delete_parse_options(const VALUE self, VALUE opts) {
+	ParseOptionsPtr *ptr = retrieve_parse_options(opts);
+	//int a = parse_options_delete(ptr->opts);
+	free_parse_options_ptr(ptr);
+	//return INT2FIX(a);
+	return Qnil;
+}
+
+VALUE linkage_limit_set(const VALUE self, VALUE opts, volatile VALUE limit) {
+	ParseOptionsPtr *ptr = retrieve_parse_options(opts);
+	int l = NUM2INT(limit);
+	parse_options_set_linkage_limit(ptr->opts, l);
+	return Qnil;
+}
+
+VALUE linkage_limit_get(const VALUE self, VALUE opts) {
+	ParseOptionsPtr *ptr = retrieve_parse_options(opts);
+	int l = parse_options_get_linkage_limit(ptr->opts);
+	return INT2FIX(l);
+}
+
+VALUE short_length_set(const VALUE self, VALUE opts, volatile VALUE length) {
+	ParseOptionsPtr *ptr = retrieve_parse_options(opts);
+	int l = NUM2INT(length);
+	parse_options_set_short_length(ptr->opts, l);
+	return Qnil;
+}
+
+VALUE short_length_get(const VALUE self, VALUE opts) {
+	ParseOptionsPtr *ptr = retrieve_parse_options(opts);
+	int l = parse_options_get_short_length(ptr->opts);
+	return INT2FIX(l);
+}
+
+VALUE create_sentence(const VALUE self, volatile VALUE str, VALUE dict) {
+	char *text = StringValuePtr(str);
+	DictionaryPtr *dict_ptr = retrieve_dictionary(dict);
+	Sentence sentence = sentence_create(text, dict_ptr->dict);
+	SentencePtr *sent_ptr = ALLOC(SentencePtr);
+	sent_ptr->sentence = NULL;
+	sent_ptr->sentence = sentence;
+	return Data_Wrap_Struct(self, 0, free, sent_ptr);
+}
+
+VALUE parse_sentence(const VALUE self, VALUE sentence, VALUE opts) {
+	SentencePtr *sent_ptr = retrieve_sentence(sentence);
+	ParseOptionsPtr *opts_ptr = retrieve_parse_options(opts);
+	int num_linkages = sentence_parse(sent_ptr->sentence, opts_ptr->opts);
+	return INT2FIX(num_linkages);
+}
+
+VALUE delete_sentence(const VALUE self, VALUE sentence) {
+	SentencePtr *ptr = retrieve_sentence(sentence);
+	sentence_delete(ptr->sentence);
+	//free_sentence_ptr(ptr);
+	return Qnil;
+}
+
+VALUE create_linkage(const VALUE self, volatile VALUE k, VALUE sentence, VALUE opts) {
+	int ck = NUM2INT(k);
+	SentencePtr *sent_ptr = retrieve_sentence(sentence);
+	ParseOptionsPtr *opts_ptr = retrieve_parse_options(opts);
+	Linkage linkage = linkage_create(ck, sent_ptr->sentence, opts_ptr->opts);
+	LinkagePtr *link_ptr = ALLOC(LinkagePtr);
+	link_ptr->linkage = NULL;
+	link_ptr->linkage = linkage;
+	return Data_Wrap_Struct(self, 0, free_linkage_ptr, link_ptr);
+}
+
+VALUE delete_linkage(const VALUE self, VALUE link) {
+	LinkagePtr *link_ptr = retrieve_linkage(link);
+	free_linkage_ptr(link_ptr);
+	return Qnil;
+}
+
+VALUE print_linkage_diagram(const VALUE self, VALUE link) {
+	LinkagePtr *link_ptr = retrieve_linkage(link);
+	char *diagram = linkage_print_diagram(link_ptr->linkage);
+	return rb_str_new2(diagram);
+}
+
+VALUE linkage_count_sublinkages(const VALUE self, VALUE link) {
+	LinkagePtr *link_ptr = retrieve_linkage(link);
+	int cnt = linkage_get_num_sublinkages(link_ptr->linkage);
+	return INT2FIX(cnt);
+}
+
+VALUE linkage_count_num_words(const VALUE self, VALUE link) {
+	LinkagePtr *link_ptr = retrieve_linkage(link);
+	int cnt = linkage_get_num_words(link_ptr->linkage);
+	return INT2FIX(cnt);
+}
+
+VALUE linkage_count_num_links(const VALUE self, VALUE link) {
+	LinkagePtr *link_ptr = retrieve_linkage(link);
+	int cnt = linkage_get_num_links(link_ptr->linkage);
+	return INT2FIX(cnt);
+}

data/ext/link_grammar/link_grammar.h

@@ -0,0 +1,99 @@
+#include "link-grammar/api.h"
+#include "link-grammar/api.c" //need ifndef
+#include "link-grammar/analyze-linkage.h"
+#include "link-grammar/analyze-linkage.c"
+#include "link-grammar/and.h"
+#include "link-grammar/and.c"
+#include "link-grammar/build-disjuncts.h"
+#include "link-grammar/build-disjuncts.c"
+#include "link-grammar/count.h"
+#include "link-grammar/count.c"
+#include "link-grammar/disjunct-utils.h" //ifndef guard
+#include "link-grammar/disjunct-utils.c"
+#include "link-grammar/disjuncts.h"
+#include "link-grammar/disjuncts.c"
+#include "link-grammar/error.h"
+#include "link-grammar/error.c"
+#include "link-grammar/extract-links.h"
+#include "link-grammar/extract-links.c"
+#include "link-grammar/fast-match.h"
+#include "link-grammar/fast-match.c"
+#include "link-grammar/idiom.h"
+#include "link-grammar/idiom.c" //rename is_number to is_number_idiom line 61 and 82 (same function name as tokenize)
+#include "link-grammar/massage.h"
+#include "link-grammar/massage.c"
+#include "link-grammar/pp_knowledge.h"
+#include "link-grammar/pp_knowledge.c"
+#include "link-grammar/pp_lexer.h"
+#include "link-grammar/pp_lexer.c"
+#include "link-grammar/pp_linkset.h"
+#include "link-grammar/pp_linkset.c"
+#include "link-grammar/post-process.h"
+#include "link-grammar/post-process.c"
+#include "link-grammar/preparation.h"
+#include "link-grammar/preparation.c"
+#include "link-grammar/print-util.h"
+#include "link-grammar/print-util.c"
+#include "link-grammar/print.h"
+#include "link-grammar/print.c"
+#include "link-grammar/prune.h"
+#include "link-grammar/prune.c" //comment out line 20 to 27 (same as disjunct-util line87). Can also use ifndef
+#include "link-grammar/resources.h"
+#include "link-grammar/resources.c"
+#include "link-grammar/read-dict.h"
+#include "link-grammar/read-dict.c"
+#include "link-grammar/read-regex.h"
+#include "link-grammar/read-regex.c"
+#include "link-grammar/regex-morph.h"
+#include "link-grammar/regex-morph.c"
+#include "link-grammar/string-set.h"
+#include "link-grammar/string-set.c"
+#include "link-grammar/tokenize.h"
+#include "link-grammar/tokenize.c" //put ifndef around spellcheck.h
+#include "link-grammar/utilities.h" //define DICTIONARY_DIR "../../../data/"
+#include "link-grammar/utilities.c"
+#include "link-grammar/word-utils.h"
+#include "link-grammar/word-utils.c"
+#include "link-grammar/word-file.h"
+#include "link-grammar/word-file.c"
+
+#include "ruby.h"
+
+//pointer wrappers containing Dictionary, Parse_Options, Sentence objects
+typedef struct dictionary_ptr {
+	Dictionary dict;
+} DictionaryPtr;
+
+typedef struct parse_options_ptr {
+	Parse_Options opts;
+} ParseOptionsPtr;
+
+typedef struct sentence_ptr {
+	Sentence sentence;
+} SentencePtr;
+
+typedef struct linkage_ptr {
+	Linkage linkage;
+} LinkagePtr;
+
+VALUE dictionary_path_set(const VALUE self, volatile VALUE dict_path);
+VALUE create_dictionary(const VALUE self, volatile VALUE lang);
+VALUE delete_dictionary(const VALUE self, VALUE dict);
+
+VALUE create_parse_options(const VALUE self);
+VALUE delete_parse_options(const VALUE self, VALUE opts);
+VALUE linkage_limit_set(const VALUE self, VALUE opts, volatile VALUE limit);
+VALUE linkage_limit_get(const VALUE self, VALUE opts);
+VALUE short_length_set(const VALUE self, VALUE opts, volatile VALUE length);
+VALUE short_length_get(const VALUE self, VALUE opts);
+
+VALUE create_sentence(const VALUE self, volatile VALUE str, VALUE dict);
+VALUE parse_sentence(const VALUE self, VALUE sentence, VALUE opts);
+VALUE delete_sentence(const VALUE self, VALUE sentence);
+
+VALUE create_linkage(const VALUE self, volatile VALUE k, VALUE sentence, VALUE opts);
+VALUE delete_linkage(const VALUE self, VALUE link);
+VALUE print_linkage_diagram(const VALUE self, VALUE link);
+VALUE linkage_count_sublinkages(const VALUE self, VALUE link);
+VALUE linkage_count_num_words(const VALUE self, VALUE link);
+VALUE linkage_count_num_links(const VALUE self, VALUE link);