wapiti 0.1.1 → 1.0.0

data/ext/wapiti/native.h

@@ -2,17 +2,12 @@
 #define native_h
 
 #include <ruby.h>
-
-#ifdef HAVE_RUBY_ENCODING_H
 #include <ruby/encoding.h>
-#endif
 
 extern VALUE mWapiti;
 extern VALUE mNative;
-
 extern VALUE cOptions;
 extern VALUE cModel;
-
 extern VALUE cNativeError;
 extern VALUE cLogger;
 

data/ext/wapiti/pattern.c

@@ -340,7 +340,7 @@ char *pat_exec(const pat_t *pat, const tok_t *tok, uint32_t at) {
 			else if (pos >= (int32_t)T)
 				value = eval[min( pos - (int32_t)T, 4)];
 			else if (col >= tok->cnts[pos])
-				fatal("missing tokens, cannot apply pattern");
+				fatal("missing tokens at %d: %d/%d, cannot apply pattern", pos, col, tok->cnts[pos]);
 			else
 				value = tok->toks[pos][col];
 		}

data/ext/wapiti/progress.c

@@ -24,23 +24,18 @@
  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  * POSSIBILITY OF SUCH DAMAGE.
  */
+
+
 #include <inttypes.h>
-#include <signal.h>
-#include <stdbool.h>
 #include <stddef.h>
-#include <stdint.h>
 #include <stdlib.h>
 #include <stdio.h>
-
+#include <signal.h>
 #include <unistd.h>
-#include <sys/time.h>
-#include <sys/resource.h>
 
-#include "wapiti.h"
+#include "progress.h"
 #include "decoder.h"
-#include "model.h"
 #include "options.h"
-#include "progress.h"
 #include "tools.h"
 
 /*******************************************************************************
@@ -74,7 +69,7 @@ bool uit_stop = false;
 /* uit_signal:
  *   Signal handler to catch interupt signal. When a signal is received, the
  *   trainer is aksed to stop as soon as possible leaving the model in a clean
- *   state. We don't reinstall the handler so if user send a second interupt
+ *   state. We don't reinstall the handler so if user send a second interrupt
  *   signal, the program will stop imediatly. (to cope with BSD system, we even
  *   reinstall explicitly the default handler)
  */
@@ -85,15 +80,17 @@ static void uit_signal(int sig) {
 
 /* uit_setup:
  *   Install the signal handler for clean early stop from the user if possible
- *   and start the timer.
  */
 void uit_setup(mdl_t *mdl) {
 	uit_stop = false;
-	if (signal(SIGINT, uit_signal) == SIG_ERR)
+	if (signal(SIGINT, uit_signal) == SIG_ERR) {
 		warning("failed to set signal handler, no clean early stop");
-	gettimeofday(&mdl->timer, NULL);
-	if (mdl->opt->stopwin != 0)
+  }
+
+	if (mdl->opt->stopwin != 0) {
 		mdl->werr = wapiti_xmalloc(sizeof(double) * mdl->opt->stopwin);
+  }
+
 	mdl->wcnt = mdl->wpos = 0;
 }
 
@@ -107,41 +104,23 @@ void uit_cleanup(mdl_t *mdl) {
 		free(mdl->werr);
 		mdl->werr = NULL;
 	}
+
 	signal(SIGINT, SIG_DFL);
 }
 
 /* uit_progress:
- *   Display a progress repport to the user consisting of some informations
+ *   Display a progress report to the user consisting of information
  *   provided by the trainer: iteration count and objective function value, and
- *   some informations computed here on the current model performances.
- *   This function return true if the trainer have to keep training the model
+ *   some information computed here on the current model performance.
+ *   This function returns true if the trainer have to keep training the model
  *   and false if he must stop, so this is were we will implement the trainer
  *   independant stoping criterion.
  */
-bool uit_progress(mdl_t *mdl, uint32_t it, double obj) {
+bool uit_progress(mdl_t *mdl) {
 	// First we just compute the error rate on devel or train data
 	double te, se;
 	tag_eval(mdl, &te, &se);
-	// Next, we compute the number of active features
-	uint64_t act = 0;
-	for (uint64_t f = 0; f < mdl->nftr; f++)
-		if (mdl->theta[f] != 0.0)
-			act++;
-	// Compute timings. As some training algorithms are multi-threaded, we
-	// cannot use ansi/c function and must rely on posix one to sum time
-	// spent in main thread and in child ones.
-	tms_t now; gettimeofday(&now, NULL);
-	double tm = (now.tv_sec        + (double)now.tv_usec        * 1.0e-6)
-	          - (mdl->timer.tv_sec + (double)mdl->timer.tv_usec * 1.0e-6);
-	mdl->total += tm;
-	mdl->timer  = now;
-	// And display progress report
-	info("  [%4"PRIu32"]", it);
-	info(obj >= 0.0 ? " obj=%-10.2f" : " obj=NA", obj);
-	info(" act=%-8"PRIu64, act);
-	info(" err=%5.2f%%/%5.2f%%", te, se);
-	info(" time=%.2fs/%.2fs", tm, mdl->total);
-	info("\n");
+
 	// If requested, check the error rate stoping criterion. We check if the
 	// error rate is stable enought over a few iterations.
 	bool res = true;
@@ -159,10 +138,6 @@ bool uit_progress(mdl_t *mdl, uint32_t it, double obj) {
 				res = false;
 		}
 	}
-	// And return
-	if (uit_stop)
-		return false;
-	return res;
-}
-
 
+	return (uit_stop) ? false : res;
+}

data/ext/wapiti/progress.h

@@ -30,15 +30,12 @@
 
 #include <stdbool.h>
 #include <stdint.h>
-
-#include "wapiti.h"
 #include "model.h"
 
 extern bool uit_stop;
 
 void uit_setup(mdl_t *mdl);
 void uit_cleanup(mdl_t *mdl);
-bool uit_progress(mdl_t *mdl, uint32_t it, double obj);
+bool uit_progress(mdl_t *mdl);
 
 #endif
-

data/ext/wapiti/rprop.c

@@ -176,7 +176,7 @@ void trn_rprop(mdl_t *mdl) {
 		for (uint64_t i = 0; i < nftr; i++) {
 			uint64_t f;
 			double vxp, vstp, vgp;
-			if (fscanf(file, "%"PRIu64" %la %la %la\n", &f, &vxp,
+			if (fscanf(file, "%"PRIu64" %le %le %le\n", &f, &vxp,
 					&vstp, &vgp) != 4)
 				fatal(err);
 			if (wbt && !cut) xp[f] = vxp;
@@ -199,11 +199,10 @@ void trn_rprop(mdl_t *mdl) {
 	// And iterate the gradient computation / weight update process until
 	// convergence or stop request
 	for (uint32_t k = 0; !uit_stop && k < K; k++) {
-		double fx = grd_gradient(grd);
 		if (uit_stop)
 			break;
 		mth_spawn((func_t *)trn_rpropsub, W, (void **)rprop, 0, 0);
-		if (uit_progress(mdl, k + 1, fx) == false)
+		if (uit_progress(mdl) == false)
 			break;
 	}
 	// Save state if user requested it
@@ -216,7 +215,7 @@ void trn_rprop(mdl_t *mdl) {
 			double vxp = xp != NULL ? xp[f] : 0.0;
 			double vstp = stp[f], vgp = gp[f];
 			fprintf(file, "%"PRIu64" ", f);
-			fprintf(file, "%la %la %la\n", vxp, vstp, vgp);
+			fprintf(file, "%le %le %le\n", vxp, vstp, vgp);
 		}
 		fclose(file);
 	}

data/ext/wapiti/sgdl1.c

@@ -104,7 +104,7 @@ void trn_sgdl1(mdl_t *mdl) {
 	// The index is a simple table indexed by sequences number. Each entry
 	// point to two lists of observations terminated by <none>, one for
 	// unigrams obss and one for bigrams obss.
-	info("    - Build the index\n");
+	info("build the index");
 	sgd_idx_t *idx  = wapiti_xmalloc(sizeof(sgd_idx_t) * S);
 	for (uint32_t s = 0; s < S; s++) {
 		const seq_t *seq = mdl->train->seq[s];
@@ -126,7 +126,7 @@ void trn_sgdl1(mdl_t *mdl) {
 		memcpy(idx[s].uobs, uobs, ucnt * sizeof(uint64_t));
 		memcpy(idx[s].bobs, bobs, bcnt * sizeof(uint64_t));
 	}
-	info("      Done\n");
+	info("indexing done");
 	// We will process sequences in random order in each iteration, so we
 	// will have to permute them. The current permutation is stored in a
 	// vector called <perm> shuffled at the start of each iteration. We
@@ -202,7 +202,7 @@ void trn_sgdl1(mdl_t *mdl) {
 		if (uit_stop)
 			break;
 		// Repport progress back to the user
-		if (!uit_progress(mdl, k + 1, -1.0))
+		if (!uit_progress(mdl))
 			break;
 	}
 	grd_stfree(grd_st);

data/ext/wapiti/tools.c

@@ -26,13 +26,11 @@
  */
 
 #include <errno.h>
-#include <stdarg.h>
-#include <stddef.h>
 #include <stdlib.h>
-#include <stdio.h>
 #include <string.h>
 
 #include "tools.h"
+#include "native.h"
 
 /*
  * Wapiti Ruby Logging
@@ -42,8 +40,20 @@
  *
  */
 
-#include "native.h"
+FILE *ufopen(VALUE path, const char *mode) {
+  FILE *file = (FILE*)0;
+  Check_Type(path, T_STRING);
 
+  if (rb_obj_tainted(path)) {
+    fatal("failed to open file from tainted string '%s'", StringValueCStr(path));
+  }
+
+  if (!(file = fopen(StringValueCStr(path), mode))) {
+    pfatal("failed to open file '%s'", StringValueCStr(path));
+  }
+
+  return file;
+}
 
 /*******************************************************************************
  * Error handling and memory managment
@@ -63,13 +73,13 @@
  *   formating than the printf family and exit program with an error. We let the
  *   OS care about freeing ressources.
  */
-void fatal(const char *msg, ...) {
+__attribute__((noreturn)) void fatal(const char *fmt, ...) {
+  VALUE msg;
 	va_list args;
-	va_start(args, msg);
-
-	rb_raise(cNativeError, msg, args);
-
+	va_start(args, fmt);
+  msg = rb_vsprintf(fmt, args);
 	va_end(args);
+	rb_raise(cNativeError, StringValueCStr(msg));
 }
 
 /* pfatal:
@@ -79,17 +89,15 @@ void fatal(const char *msg, ...) {
  *   must be carefull to not call other functino that might reset it before
  *   calling pfatal.
  */
-void pfatal(const char *msg, ...) {
-	// const char *err = strerror(errno);
+__attribute__((noreturn)) void pfatal(const char *fmt, ...) {
+	const char *err = strerror(errno);
+  VALUE msg;
 	va_list args;
-	va_start(args, msg);
-
-	// VALUE message = rb_vsprintf(msg, args);
-	// rb_str_catf(message, ": <%s>", err);
-	rb_raise(cNativeError, msg, args);
-
+	va_start(args, fmt);
+  msg = rb_vsprintf(fmt, args);
 	va_end(args);
-
+	rb_str_catf(msg, ": %s", err);
+	rb_raise(cNativeError, StringValueCStr(msg));
 }
 
 /* warning:
@@ -97,14 +105,13 @@ void pfatal(const char *msg, ...) {
  *   exit the program. It is intended to inform the user that something strange
  *   have happen and the result might be not what it have expected.
  */
-void warning(const char *msg, ...) {
+void warning(const char *fmt, ...) {
+  VALUE msg;
 	va_list args;
-	va_start(args, msg);
-
-	// (void)rb_funcall(cLogger, rb_intern("warn"), 1, rb_vsprintf(msg, args));
-	(void)rb_funcall(cLogger, rb_intern("warn"), 1, rb_str_new2(msg));
-
+	va_start(args, fmt);
+  msg = rb_vsprintf(fmt, args);
 	va_end(args);
+	(void)rb_funcall(cLogger, rb_intern("warn"), 1, msg);
 }
 
 /* info:
@@ -113,14 +120,13 @@ void warning(const char *msg, ...) {
  *   just a wrapper for printf to stderr. Note that unlike the previous one,
  *   this function doesn't automatically append a new line character.
  */
-void info(const char *msg, ...) {
+void info(const char *fmt, ...) {
+  VALUE msg;
 	va_list args;
-	va_start(args, msg);
-
-	// (void)rb_funcall(cLogger, rb_intern("info"), 1, rb_vsprintf(msg, args));
-	(void)rb_funcall(cLogger, rb_intern("info"), 1, rb_str_new2(msg));
-
+	va_start(args, fmt);
+  msg = rb_vsprintf(fmt, args);
 	va_end(args);
+	(void)rb_funcall(cLogger, rb_intern("info"), 1, msg);
 }
 
 /*  wapiti_xmalloc:

data/ext/wapiti/tools.h

@@ -31,17 +31,22 @@
 #include <stddef.h>
 #include <stdint.h>
 #include <stdio.h>
+#include <ruby.h>
 
 #define unused(v) ((void)(v))
 #define none ((uint64_t)-1)
 
+#undef min
 #define min(a, b) ((a) < (b) ? (a) : (b))
+#undef max
 #define max(a, b) ((a) < (b) ? (b) : (a))
 
-void fatal(const char *msg, ...);
-void pfatal(const char *msg, ...);
-void warning(const char *msg, ...);
-void info(const char *msg, ...);
+FILE *ufopen(VALUE path, const char *mode);
+
+void fatal(const char *fmt, ...);
+void pfatal(const char *fmt, ...);
+void warning(const char *fmt, ...);
+void info(const char *fmt, ...);
 
 void *wapiti_xmalloc(size_t size);
 void *wapiti_xrealloc(void *ptr, size_t size);

data/ext/wapiti/trainers.c

@@ -0,0 +1,55 @@
+#include <string.h>
+#include "tools.h"
+#include "trainers.h"
+
+static const struct {
+  const char *name;
+  trn_t train;
+} trn_lst[] = {
+  {"l-bfgs", trn_lbfgs},
+  {"sgd-l1", trn_sgdl1},
+  {"bcd",    trn_bcd  },
+  {"rprop",  trn_rprop},
+  {"rprop+", trn_rprop},
+  {"rprop-", trn_rprop}
+};
+
+static const uint32_t trn_cnt = sizeof(trn_lst) / sizeof(trn_lst[0]);
+
+trn_t trn_get(const char *algo) {
+  uint32_t trn;
+
+  for (trn = 0; trn < trn_cnt; trn++) {
+    if (!strcmp(algo, trn_lst[trn].name)) break;
+  }
+
+  if (trn == trn_cnt) {
+    fatal("unknown algorithm '%s'", algo);
+    trn = 0;
+  }
+
+  return trn_lst[trn].train;
+}
+
+static const char *typ_lst[] = {
+	"maxent",
+	"memm",
+	"crf"
+};
+
+static const uint32_t typ_cnt = sizeof(typ_lst) / sizeof(typ_lst[0]);
+
+uint32_t typ_get(const char* type) {
+  uint32_t typ;
+
+	for (typ = 0; typ < typ_cnt; typ++) {
+		if (!strcmp(type, typ_lst[typ])) break;
+  }
+
+	if (typ == typ_cnt) {
+		fatal("unknown model type '%s'", type);
+  }
+
+  return typ;
+}
+

data/ext/wapiti/trainers.h

@@ -35,5 +35,8 @@ void trn_sgdl1(mdl_t *mdl);
 void trn_bcd(mdl_t *mdl);
 void trn_rprop(mdl_t *mdl);
 
-#endif
+typedef void (*trn_t)(mdl_t*);
+trn_t trn_get(const char *algo);
+uint32_t typ_get(const char *type);
 
+#endif

data/lib/wapiti.rb

@@ -1,30 +1,10 @@
-
-require 'logger'
-require 'tempfile'
-
 require 'wapiti/version'
-
-module Wapiti
-
-  Logger = ::Logger.new(STDOUT)
-  Logger.level = ::Logger::WARN
-
-  class << self
-    def log
-      Logger
-    end
-
-    def debug!
-      log.level == ::Logger::DEBUG
-    end
-  end
-
-end
-
 require 'wapiti/errors'
+require 'wapiti/log'
+require 'wapiti/token'
+require 'wapiti/sequence'
+require 'wapiti/dataset'
 require 'wapiti/native'
-
 require 'wapiti/options'
 require 'wapiti/model'
-
 require 'wapiti/utility'