wapiti 0.0.1

data/ext/wapiti/thread.c

@@ -0,0 +1,171 @@
+/*
+ *      Wapiti - A linear-chain CRF tool
+ *
+ * Copyright (c) 2009-2011  CNRS
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "model.h"
+#include "tools.h"
+#include "thread.h"
+#include "wapiti.h"
+
+/******************************************************************************
+ * Multi-threading code
+ *
+ *   This module handle the thread managment code using POSIX pthreads, on
+ *   non-POSIX systems you will have to rewrite this using your systems threads.
+ *   all code who depend on threads is located here so this process must not be
+ *   too difficult.
+ *   If you don't want to use multithreading on non-POSIX system, just enable
+ *   the definition of MTH_ANSI in wapiti.h. This will disable multithreading.
+ *
+ *   The jobs system is a simple scheduling system, you have to provide the
+ *   number of jobs to be done and the size of each batch, a call to getjob will
+ *   return the index of the first available and the size of the batch, and mark
+ *   these jobs as done. This is usefull if your jobs are numbered but you can't
+ *   do a trivial split as each of them may require different amount of time to
+ *   be completed like gradient computation which depend on the length of the
+ *   sequences.
+ *   If you provide a count of 0, the job system is disabled.
+ ******************************************************************************/
+#ifdef MTH_ANSI
+struct job_s {
+	size_t size;
+};
+
+bool mth_getjob(job_t *job, size_t *cnt, size_t *pos) {
+	if (job->size == 0)
+		return false;
+	*cnt = job->size;
+	*pos = 0;
+	job->size = 0;
+	return true;
+}
+
+void mth_spawn(func_t *f, int W, void *ud[W], size_t size, size_t batch) {
+	unused(batch);
+	if (size == 0) {
+		f(NULL, 0, 1, ud[0]);
+	} else {
+		job_t job = {size};
+		f(&job, 0, 1, ud[0]);
+	}
+}
+
+#else
+
+#include <pthread.h>
+
+struct job_s {
+	size_t size;
+	size_t send;
+	size_t batch;
+	pthread_mutex_t lock;
+};
+
+typedef struct mth_s mth_t;
+struct mth_s {
+	job_t  *job;
+	int     id;
+	int     cnt;
+	func_t *f;
+	void   *ud;
+};
+
+/* mth_getjob:
+ *   Get a new bunch of sequence to process. This function will return a new
+ *   batch of sequence to process starting at position <pos> and with size
+ *   <cnt> and return true. If no more batch are available, return false.
+ *   This function use a lock to ensure thread safety as it will be called by
+ *   the multiple workers threads.
+ */
+bool mth_getjob(job_t *job, size_t *cnt, size_t *pos) {
+	if (job == NULL)
+		return false;
+	if (job->send == job->size)
+		return false;
+	pthread_mutex_lock(&job->lock);
+	*cnt = min(job->batch, job->size - job->send);
+	*pos = job->send;
+	job->send += *cnt;
+	pthread_mutex_unlock(&job->lock);
+	return true;
+}
+
+static void *mth_stub(void *ud) {
+	mth_t *mth = (mth_t *)ud;
+	mth->f(mth->job, mth->id, mth->cnt, mth->ud);
+	return NULL;
+}
+
+/* mth_spawn:
+ *   This function spawn W threads for calling the 'f' function. The function
+ *   will get a unique identifier between 0 and W-1 and a user data from the
+ *   'ud' array.
+ */
+void mth_spawn(func_t *f, int W, void *ud[W], size_t size, size_t batch) {
+	// First prepare the jobs scheduler
+	job_t job, *pjob = NULL;
+	if (size != 0) {
+		pjob = &job;
+		job.size = size;
+		job.send = 0;
+		job.batch = batch;
+		if (pthread_mutex_init(&job.lock, NULL) != 0)
+			fatal("failed to create mutex");
+	}
+	// We handle differently the case where user requested a single thread
+	// for efficiency.
+	if (W == 1) {
+		f(&job, 0, 1, ud[0]);
+		return;
+	}
+	// We prepare the parameters structures that will be send to the threads
+	// with informations for calling the user function.
+	mth_t p[W];
+	for (int w = 0; w < W; w++) {
+		p[w].job = pjob;
+		p[w].id  = w;
+		p[w].cnt = W;
+		p[w].f   = f;
+		p[w].ud  = ud[w];
+	}
+	// We are now ready to spawn the threads and wait for them to finish
+	// their jobs. So we just create all the thread and try to join them
+	// waiting for there return.
+	pthread_attr_t attr;
+	pthread_attr_init(&attr);
+	pthread_attr_setscope(&attr, PTHREAD_SCOPE_SYSTEM);
+	pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_JOINABLE);
+	pthread_t th[W];
+	for (int w = 0; w < W; w++)
+		if (pthread_create(&th[w], &attr, &mth_stub, &p[w]) != 0)
+			fatal("failed to create thread");
+	for (int w = 0; w < W; w++)
+		if (pthread_join(th[w], NULL) != 0)
+			fatal("failed to join thread");
+	pthread_attr_destroy(&attr);
+}
+#endif
+

data/ext/wapiti/thread.h

@@ -0,0 +1,42 @@
+/*
+ *      Wapiti - A linear-chain CRF tool
+ *
+ * Copyright (c) 2009-2011  CNRS
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef thread_h
+#define thread_h
+
+#include <pthread.h>
+
+#include "model.h"
+
+typedef struct job_s job_t;
+
+typedef void (func_t)(job_t *job, int id, int cnt, void *ud);
+
+bool mth_getjob(job_t *job, size_t *cnt, size_t *pos);
+void mth_spawn(func_t *f, int W, void *ud[W], size_t size, size_t batch);
+
+#endif

data/ext/wapiti/tools.c

@@ -0,0 +1,202 @@
+/*
+ *      Wapiti - A linear-chain CRF tool
+ *
+ * Copyright (c) 2009-2011  CNRS
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <errno.h>
+#include <stdarg.h>
+#include <stddef.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+
+#include "tools.h"
+
+/*
+ * Wapiti Ruby Logging
+ *
+ * Wapiti-Ruby delegates all wapiti logging messages to a Ruby logger in the
+ * main Wapiti module.
+ *
+ */
+
+#include "native.h"
+
+
+/*******************************************************************************
+ * Error handling and memory managment
+ *
+ *   Wapiti use a very simple system for error handling: violently fail. Errors
+ *   can occurs in two cases, when user feed Wapiti with bad datas or when there
+ *   is a problem on the system side. In both cases, there is nothing we can do,
+ *   so the best thing is to exit with a meaning full error message.
+ *
+ *   Memory allocation is one of the possible point of failure and its painfull
+ *   to always remeber to check return value of malloc so we provide wrapper
+ *   around it and realloc who check and fail in case of error.
+ ******************************************************************************/
+
+/* fatal:
+ *   This is the main error function, it will print the given message with same
+ *   formating than the printf family and exit program with an error. We let the
+ *   OS care about freeing ressources.
+ */
+void fatal(const char *msg, ...) {
+	va_list args;
+	va_start(args, msg);
+
+	// VALUE message = rb_vsprintf(msg, args);
+	VALUE message = rb_str_new2(msg);
+
+	va_end(args);
+	
+	(void)rb_funcall(cLogger, rb_intern("fatal"), 1, message);
+	rb_raise(cNativeError, StringValuePtr(message));
+}
+
+/* pfatal:
+ *   This one is very similar to the fatal function but print an additional
+ *   system error message depending on the errno. This can be used when a
+ *   function who set the errno fail to print more detailed informations. You
+ *   must be carefull to not call other functino that might reset it before
+ *   calling pfatal.
+ */
+void pfatal(const char *msg, ...) {
+	const char *err = strerror(errno);
+	va_list args;
+	va_start(args, msg);
+
+	// VALUE message = rb_vsprintf(msg, args);
+	// rb_str_catf(message, ": <%s>", err);
+	VALUE message = rb_str_new2(msg);
+
+	va_end(args);
+
+	(void)rb_funcall(cLogger, rb_intern("fatal"), 1, message);
+	rb_raise(cNativeError, StringValuePtr(message));
+}
+
+/* warning:
+ *   This one is less violent as it just print a warning on stderr, but doesn't
+ *   exit the program. It is intended to inform the user that something strange
+ *   have happen and the result might be not what it have expected.
+ */
+void warning(const char *msg, ...) {
+	va_list args;
+	va_start(args, msg);
+
+	// (void)rb_funcall(cLogger, rb_intern("warn"), 1, rb_vsprintf(msg, args));
+	(void)rb_funcall(cLogger, rb_intern("warn"), 1, rb_str_new2(msg));
+
+	va_end(args);
+}
+
+/* info:
+ *   Function used for all progress reports. This is where an eventual verbose
+ *   level can be implemented later or redirection to a logfile. For now, it is
+ *   just a wrapper for printf to stderr. Note that unlike the previous one,
+ *   this function doesn't automatically append a new line character.
+ */
+void info(const char *msg, ...) {
+	va_list args;
+	va_start(args, msg);
+
+	// (void)rb_funcall(cLogger, rb_intern("info"), 1, rb_vsprintf(msg, args));
+	(void)rb_funcall(cLogger, rb_intern("info"), 1, rb_str_new2(msg));
+
+	va_end(args);
+}
+
+/* wapiti_xmalloc:
+ *   A simple wrapper around malloc who violently fail if memory cannot be
+ *   allocated, so it will never return NULL.
+ */
+void *wapiti_xmalloc(size_t size) {
+	void *ptr = malloc(size);
+	if (ptr == NULL)
+		fatal("out of memory");
+	return ptr;
+}
+
+/* wapiti_xrealloc:
+ *   As wapiti_xmalloc, this is a simple wrapper around realloc who fail on memory
+ *   error and so never return NULL.
+ */
+void *wapiti_xrealloc(void *ptr, size_t size) {
+	void *new = realloc(ptr, size);
+	if (new == NULL)
+		fatal("out of memory");
+	return new;
+}
+
+/* xstrdup:
+ *   As the previous one, this is a safe version of xstrdup who fail on
+ *   allocation error.
+ */
+char *xstrdup(const char *str) {
+	const int len = strlen(str) + 1;
+	char *res = wapiti_xmalloc(sizeof(char) * len);
+	memcpy(res, str, len);
+	return res;
+}
+
+/******************************************************************************
+ * Netstring for persistent storage
+ *
+ *   This follow the format proposed by D.J. Bernstein for safe and portable
+ *   storage of string in persistent file and networks. This used for storing
+ *   strings in saved models.
+ *   We just add an additional end-of-line character to make the output files
+ *   more readable.
+ *
+ ******************************************************************************/
+
+/* ns_readstr:
+ *   Read a string from the given file in netstring format. The string is
+ *   returned as a newly allocated bloc of memory 0-terminated.
+ */
+char *ns_readstr(FILE *file) {
+	int len;
+	if (fscanf(file, "%d:", &len) != 1)
+		pfatal("cannot read from file");
+	char *buf = wapiti_xmalloc(len + 1);
+	if (fread(buf, len, 1, file) != 1)
+		pfatal("cannot read from file");
+	if (fgetc(file) != ',')
+		fatal("invalid format");
+	buf[len] = '\0';
+	fgetc(file);
+	return buf;
+}
+
+/* ns_writestr:
+ *   Write a string in the netstring format to the given file.
+ */
+void ns_writestr(FILE *file, const char *str) {
+	const int len = strlen(str);
+	if (fprintf(file, "%d:%s,\n", len, str) < 0)
+		pfatal("cannot write to file");
+}
+

data/ext/wapiti/tools.h

@@ -0,0 +1,54 @@
+/*
+ *      Wapiti - A linear-chain CRF tool
+ *
+ * Copyright (c) 2009-2011  CNRS
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+#ifndef tools_h
+#define tools_h
+
+#include <stdarg.h>
+#include <stddef.h>
+#include <stdio.h>
+
+#include <ruby.h>
+
+#define unused(v) ((void)(v))
+#define none ((size_t)-1)
+
+#define min(a, b) ((a) < (b) ? (a) : (b))
+#define max(a, b) ((a) < (b) ? (b) : (a))
+
+void fatal(const char *msg, ...);
+void pfatal(const char *msg, ...);
+void warning(const char *msg, ...);
+void info(const char *msg, ...);
+
+void *wapiti_xmalloc(size_t size);
+void *wapiti_xrealloc(void *ptr, size_t size);
+char *xstrdup(const char *str);
+
+char *ns_readstr(FILE *file);
+void ns_writestr(FILE *file, const char *str);
+
+#endif