wapiti 0.0.1

data/ext/wapiti/native.h

@@ -0,0 +1,15 @@
+#ifndef native_h
+#define native_h
+
+#include <ruby.h>
+
+extern VALUE mWapiti;
+extern VALUE mNative;
+
+extern VALUE cOptions;
+extern VALUE cModel;
+
+extern VALUE cNativeError;
+extern VALUE cLogger;
+
+#endif

data/ext/wapiti/options.c

@@ -0,0 +1,278 @@
+/*
+ *      Wapiti - A linear-chain CRF tool
+ *
+ * Copyright (c) 2009-2011  CNRS
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <limits.h>
+#include <stdbool.h>
+#include <stddef.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+
+#include "wapiti.h"
+#include "tools.h"
+#include "options.h"
+#include "vmath.h"
+
+/******************************************************************************
+ * Command line parsing
+ *
+ *   This module handle command line parsing and put all things defined by the
+ *   user in a special structure in order to make them accessible to the
+ *   remaining of the program.
+ ******************************************************************************/
+
+/* opt_help:
+ *   Just display the help message describing modes and switch.
+ */
+static void opt_help(const char *pname) {
+	static const char msg[] =
+		"Global switchs:\n"
+		"\t-h | --help      display this help message\n"
+		"\t   | --version   display version information\n"
+		"\n"
+		"Training mode:\n"
+		"    %1$s train [options] [input data] [model file]\n"
+		"\t   | --me               force maxent mode\n"
+		"\t-a | --algo     STRING  training algorithm to use\n"
+		"\t-p | --pattern  FILE    patterns for extracting features\n"
+		"\t-m | --model    FILE    model file to preload\n"
+		"\t-d | --devel    FILE    development dataset\n"
+		"\t-c | --compact          compact model after training\n"
+		"\t-t | --nthread  INT     number of worker threads\n"
+		"\t-j | --jobsize  INT     job size for worker threads\n"
+		"\t-s | --sparse           enable sparse forward/backward\n"
+		"\t-i | --maxiter  INT     maximum number of iterations\n"
+		"\t-1 | --rho1     FLOAT   l1 penalty parameter\n"
+		"\t-2 | --rho2     FLOAT   l2 penalty parameter\n"
+		"\t-o | --objwin   INT     convergence window size\n"
+		"\t-w | --stopwin  INT     stop window size\n"
+		"\t-e | --stopeps  FLOAT   stop epsilon value\n"
+		"\t   | --clip             (l-bfgs) clip gradient\n"
+		"\t   | --histsz   INT     (l-bfgs) history size\n"
+		"\t   | --maxls    INT     (l-bfgs) max linesearch iters\n"
+		"\t   | --eta0     FLOAT   (sgd-l1) learning rate\n"
+		"\t   | --alpha    FLOAT   (sgd-l1) exp decay parameter\n"
+		"\t   | --kappa    FLOAT   (bcd)    stability parameter\n"
+		"\t   | --stpmin   FLOAT   (rprop)  minimum step size\n"
+		"\t   | --stpmax   FLOAT   (rprop)  maximum step size\n"
+		"\t   | --stpinc   FLOAT   (rprop)  step increment factor\n"
+		"\t   | --stpdec   FLOAT   (rprop)  step decrement factor\n"
+		"\t   | --cutoff           (rprop)  alternate projection\n"
+		"\n"
+		"Labelling mode:\n"
+		"    %1$s label [options] [input data] [output data]\n"
+		"\t   | --me               force maxent mode\n"
+		"\t-m | --model    FILE    model file to load\n"
+		"\t-l | --label            output only labels\n"
+		"\t-c | --check            input is already labeled\n"
+		"\t-s | --score            add scores to output\n"
+		"\t-p | --post             label using posteriors\n"
+		"\t-n | --nbest    INT     output n-best list\n"
+		"\n"
+		"Dumping mode\n"
+		"    %1$s dump [input model] [output text]\n";
+	fprintf(stderr, msg, pname);
+}
+
+/* opt_defaults:
+ *   Default values for all parameters of the model.
+ */
+const opt_t opt_defaults = {
+	.mode    = -1,
+	.input   = NULL,     .output  = NULL,
+	.maxent  = false,
+	.algo    = "l-bfgs", .pattern = NULL,  .model   = NULL, .devel   = NULL,
+	.compact = false,    .sparse  = false,
+	.nthread = 1,        .jobsize = 64,    .maxiter = 0,
+	.rho1    = 0.5,      .rho2    = 0.0001,
+	.objwin  = 5,        .stopwin = 5,     .stopeps = 0.02,
+	.lbfgs = {.clip   = false, .histsz = 5, .maxls = 40},
+	.sgdl1 = {.eta0   = 0.8,   .alpha  = 0.85},
+	.bcd   = {.kappa  = 1.5},
+	.rprop = {.stpmin = 1e-8, .stpmax = 50.0, .stpinc = 1.2, .stpdec = 0.5,
+	          .cutoff = false},
+	.label   = false,    .check   = false, .outsc = false,
+	.lblpost = false,    .nbest = 1
+};
+
+/* opt_switch:
+ *   Define available switchs for the different modes in a readable way for the
+ *   command line argument parser.
+ */
+struct {
+	int     mode;
+	char   *dshort;
+	char   *dlong;
+	char    kind;
+	size_t  offset;
+} opt_switch[] = {
+	{0, "##", "--me",      'B', offsetof(opt_t, maxent      )},
+	{0, "-a", "--algo",    'S', offsetof(opt_t, algo        )},
+	{0, "-p", "--pattern", 'S', offsetof(opt_t, pattern     )},
+	{0, "-m", "--model",   'S', offsetof(opt_t, model       )},
+	{0, "-d", "--devel",   'S', offsetof(opt_t, devel       )},
+	{0, "-c", "--compact", 'B', offsetof(opt_t, compact     )},
+	{0, "-s", "--sparse",  'B', offsetof(opt_t, sparse      )},
+	{0, "-t", "--nthread", 'I', offsetof(opt_t, nthread     )},
+	{0, "-j", "--josize",  'I', offsetof(opt_t, jobsize     )},
+	{0, "-i", "--maxiter", 'I', offsetof(opt_t, maxiter     )},
+	{0, "-1", "--rho1",    'F', offsetof(opt_t, rho1        )},
+	{0, "-2", "--rho2",    'F', offsetof(opt_t, rho2        )},
+	{0, "-o", "--objsz",   'I', offsetof(opt_t, objwin      )},
+	{0, "-w", "--stopwin", 'I', offsetof(opt_t, stopwin     )},
+	{0, "-e", "--stopeps", 'F', offsetof(opt_t, stopeps     )},
+	{0, "##", "--clip",    'B', offsetof(opt_t, lbfgs.clip  )},
+	{0, "##", "--histsz",  'I', offsetof(opt_t, lbfgs.histsz)},
+	{0, "##", "--maxls",   'I', offsetof(opt_t, lbfgs.maxls )},
+	{0, "##", "--eta0",    'F', offsetof(opt_t, sgdl1.eta0  )},
+	{0," ##", "--alpha",   'F', offsetof(opt_t, sgdl1.alpha )},
+	{0, "##", "--kappa",   'F', offsetof(opt_t, bcd.kappa   )},
+	{0, "##", "--stpmin",  'F', offsetof(opt_t, rprop.stpmin)},
+	{0, "##", "--stpmax",  'F', offsetof(opt_t, rprop.stpmax)},
+	{0, "##", "--stpinc",  'F', offsetof(opt_t, rprop.stpinc)},
+	{0, "##", "--stpdec",  'F', offsetof(opt_t, rprop.stpdec)},
+	{0, "##", "--cutoff",  'B', offsetof(opt_t, rprop.cutoff)},
+	{1, "##", "--me",      'B', offsetof(opt_t, maxent      )},
+	{1, "-m", "--model",   'S', offsetof(opt_t, model       )},
+	{1, "-l", "--label",   'B', offsetof(opt_t, label       )},
+	{1, "-c", "--check",   'B', offsetof(opt_t, check       )},
+	{1, "-s", "--score",   'B', offsetof(opt_t, outsc       )},
+	{1, "-p", "--post",    'B', offsetof(opt_t, lblpost     )},
+	{1, "-n", "--nbest",   'I', offsetof(opt_t, nbest       )},
+	{-1, NULL, NULL, '\0', 0}
+};
+
+/* argparse:
+ *   This is the main function for command line parsing. It use the previous
+ *   table to known how to interpret the switchs and store values in the opt_t
+ *   structure.
+ */
+void opt_parse(int argc, char *argv[argc], opt_t *opt) {
+	static const char *err_badval = "invalid value for switch '%s'";
+	const char *pname = argv[0];
+	argc--, argv++;
+	if (argc == 0) {
+		opt_help(pname);
+		fatal("no mode specified");
+	}
+	// First special handling for help and version
+	if (!strcmp(argv[0], "-h") || !strcmp(argv[0], "--help")) {
+		opt_help(pname);
+		exit(EXIT_FAILURE);
+	} else if (!strcmp(argv[0], "--version")) {
+		fprintf(stderr, "Wapiti v" VERSION "\n");
+		fprintf(stderr, "  Optimization mode:   %s\n", xvm_mode());
+		exit(EXIT_SUCCESS);
+	}
+	// Get the mode to use
+	if (!strcmp(argv[0], "t") || !strcmp(argv[0], "train")) {
+		opt->mode = 0;
+	} else if (!strcmp(argv[0], "l") || !strcmp(argv[0], "label")) {
+		opt->mode = 1;
+	} else if (!strcmp(argv[0], "d") || !strcmp(argv[0], "dump")) {
+		opt->mode = 2;
+	} else {
+		fatal("unknown mode <%s>", argv[0]);
+	}
+	argc--, argv++;
+	// Parse remaining arguments
+	opt->input  = NULL;
+	opt->output = NULL;
+	while (argc > 0) {
+		const char *arg = argv[0];
+		int idx;
+		// Check if this argument is a filename or an option
+		if (arg[0] != '-') {
+			if (opt->input == NULL)
+				opt->input = argv[0];
+			else if (opt->output == NULL)
+				opt->output = argv[0];
+			else
+				fatal("too much input files on command line");
+			argc--, argv++;
+			continue;
+		}
+		// Search the current switch in the table or fail if it cannot
+		// be found.
+		for (idx = 0; opt_switch[idx].mode != -1; idx++) {
+			if (opt_switch[idx].mode != opt->mode)
+				continue;
+			if (!strcmp(arg, opt_switch[idx].dshort))
+				break;
+			if (!strcmp(arg, opt_switch[idx].dlong))
+				break;
+		}
+		if (opt_switch[idx].mode == -1)
+			fatal("unknown option '%s'", arg);
+		// Decode the argument and store it in the structure
+		if (opt_switch[idx].kind != 'B' && argc < 2)
+			fatal("missing argument for switch '%s'", arg);
+		void *ptr = (void *)((char *)opt + opt_switch[idx].offset);
+		switch (opt_switch[idx].kind) {
+			case 'S':
+				*((char **)ptr) = argv[1];
+				argc -= 2, argv += 2;
+				break;
+			case 'I':
+				if (sscanf(argv[1], "%d", (int *)ptr) != 1)
+					fatal(err_badval, arg);
+				argc -= 2, argv += 2;
+				break;
+			case 'F': {
+				double tmp;
+				if (sscanf(argv[1], "%lf", &tmp) != 1)
+					fatal(err_badval, arg);
+				*((double *)ptr) = tmp;
+				argc -= 2, argv += 2;
+				break; }
+			case 'B':
+				*((bool *)ptr) = true;
+				argc--, argv++;
+				break;
+		}
+	}
+	// Small trick for the maxiter switch
+	if (opt->maxiter == 0)
+		opt->maxiter = INT_MAX;
+	// Check that all options are valid
+	#define argchecksub(name, test)                      \
+		if (!(test))                                 \
+			fatal("invalid value for <"name">");
+	argchecksub("--thread",  opt->nthread      >  0  );
+	argchecksub("--jobsize", opt->jobsize      >  0  );
+	argchecksub("--rho1",    opt->rho1         >= 0.0);
+	argchecksub("--rho2",    opt->rho2         >= 0.0);
+	argchecksub("--histsz",  opt->lbfgs.histsz >  0  );
+	argchecksub("--maxls",   opt->lbfgs.maxls  >  0  );
+	argchecksub("--eta0",    opt->sgdl1.eta0   >  0.0);
+	argchecksub("--alpha",   opt->sgdl1.alpha  >  0.0);
+	argchecksub("--nbest",   opt->nbest        >  0  );
+	#undef argchecksub
+	if (opt->maxent && !strcmp(opt->algo, "bcd"))
+		fatal("BCD not supported for training maxent models");
+}
+

data/ext/wapiti/options.h

@@ -0,0 +1,91 @@
+/*
+ *      Wapiti - A linear-chain CRF tool
+ *
+ * Copyright (c) 2009-2011  CNRS
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+#ifndef options_h
+#define options_h
+
+#include <stdbool.h>
+
+#include "wapiti.h"
+
+/* opt_t:
+ *   This structure hold all user configurable parameter for Wapiti and is
+ *   filled with parameters from command line.
+ */
+typedef struct opt_s opt_t;
+struct opt_s {
+	int    mode;
+	char  *input,  *output;
+	bool   maxent;
+	// Options for training
+	char  *algo,   *pattern;
+	char  *model,  *devel;
+	bool   compact, sparse;
+	int    nthread;
+	int    jobsize;
+	int    maxiter;
+	double rho1,    rho2;
+	// Window size criterion
+	int    objwin;
+	int    stopwin;
+	double stopeps;
+	// Options specific to L-BFGS
+	struct {
+		bool   clip;
+		int    histsz;
+		int    maxls;
+	} lbfgs;
+	// Options specific to SGD-L1
+	struct {
+		double eta0;
+		double alpha;
+	} sgdl1;
+	// Options specific to BCD
+	struct {
+		double kappa;
+	} bcd;
+	// Options specific to RPROP
+	struct {
+		double stpmin;
+		double stpmax;
+		double stpinc;
+		double stpdec;
+		bool   cutoff;
+	} rprop;
+	// Options for labelling
+	bool   label;
+	bool   check;
+	bool   outsc;
+	bool   lblpost;
+	int    nbest;
+};
+
+extern const opt_t opt_defaults;
+
+void opt_parse(int argc, char *argv[argc], opt_t *opt);
+
+#endif
+

data/ext/wapiti/pattern.c

@@ -0,0 +1,395 @@
+/*
+ *      Wapiti - A linear-chain CRF tool
+ *
+ * Copyright (c) 2009-2011  CNRS
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <ctype.h>
+#include <stdbool.h>
+#include <stddef.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include "pattern.h"
+#include "sequence.h"
+#include "tools.h"
+
+/******************************************************************************
+ * A simple regular expression matcher
+ *
+ *   This module implement a simple regular expression matcher, it implement
+ *   just a subset of the classical regexp simple to implement but sufficient
+ *   for most usages and avoid to add a dependency to a full regexp library.
+ *
+ *   The recognized subset is quite simple. First for matching characters :
+ *       .  -> match any characters
+ *       \x -> match a character class (in uppercase, match the complement)
+ *               \d : digit       \a : alpha      \w : alpha + digit
+ *               \l : lowercase   \u : uppercase  \p : punctuation
+ *               \s : space
+ *             or escape a character
+ *       x  -> any other character match itself
+ *   And the constructs :
+ *       ^  -> at the begining of the regexp, anchor it at start of string
+ *       $  -> at the end of regexp, anchor it at end of string
+ *       *  -> match any number of repetition of the previous character
+ *       ?  -> optionally match the previous character
+ *
+ *   This subset is implemented quite efficiently using recursion. All recursive
+ *   calls are tail-call so they should be optimized by the compiler. As we do
+ *   direct interpretation, we have to backtrack so performance can be very poor
+ *   on specialy designed regexp. This is not a problem as the regexp as well as
+ *   the string is expected to be very simple here. If this is not the case, you
+ *   better have to prepare your data better.
+ ******************************************************************************/
+
+/* rex_matchit:
+ *   Match a single caracter at the start fo the string. The character might be
+ *   a plain char, a dot or char class.
+ */
+static bool rex_matchit(const char *ch, const char *str) {
+	if (str[0] == '\0')
+		return false;
+	if (ch[0] == '.')
+		return true;
+	if (ch[0] == '\\') {
+		switch (ch[1]) {
+			case 'a': return  isalpha(str[0]);
+			case 'd': return  isdigit(str[0]);
+			case 'l': return  islower(str[0]);
+			case 'p': return  ispunct(str[0]);
+			case 's': return  isspace(str[0]);
+			case 'u': return  isupper(str[0]);
+			case 'w': return  isalnum(str[0]);
+			case 'A': return !isalpha(str[0]);
+			case 'D': return !isdigit(str[0]);
+			case 'L': return !islower(str[0]);
+			case 'P': return !ispunct(str[0]);
+			case 'S': return !isspace(str[0]);
+			case 'U': return !isupper(str[0]);
+			case 'W': return !isalnum(str[0]);
+		}
+		return ch[1] == str[0];
+	}
+	return ch[0] == str[0];
+}
+
+/* rex_matchme:
+ *   Match a regular expresion at the start of the string. If a match is found,
+ *   is length is returned in len. The mathing is done through tail-recursion
+ *   for good performances.
+ */
+static bool rex_matchme(const char *re, const char *str, int *len) {
+	// Special check for end of regexp
+	if (re[0] == '\0')
+		return true;
+	if (re[0] == '$' && re[1] == '\0')
+		return (str[0] == '\0');
+	// Get first char of regexp
+	const char *ch  = re;
+	const char *nxt = re + 1 + (ch[0] == '\\');
+	// Special check for the following construct "x**" where the first star
+	// is consumed normally but lead the second (which is wrong) to be
+	// interpreted as a char to mach as if it was escaped (and same for the
+	// optional construct)
+	if (*ch == '*' || *ch == '?')
+		fatal("unescaped * or ? in regexp: %s", re);
+	// Handle star repetition
+	if (nxt[0] == '*') {
+		nxt++;
+		do {
+			const int save = *len;
+			if (rex_matchme(nxt, str, len))
+				return true;
+			*len = save + 1;
+		} while (rex_matchit(ch, str++));
+		return false;
+	}
+	// Handle optional
+	if (nxt[0] == '?') {
+		nxt++;
+		if (rex_matchit(ch, str)) {
+			(*len)++;
+			if (rex_matchme(nxt, str + 1, len))
+				return true;
+			(*len)--;
+		}
+		return rex_matchme(nxt, str, len);
+	}
+	// Classical char matching
+	(*len)++;
+	if (rex_matchit(ch, str))
+		return rex_matchme(nxt, str + 1, len);
+	return false;
+}
+
+/* rex_match:
+ *   Match a regular expresion in the given string. If a match is found, the
+ *   position of the start of the match is returned and is len is returned in
+ *   len, else -1 is returned.
+ */
+static int rex_match(const char *re, const char *str, int *len) {
+	// Special case for anchor at start
+	if (*re == '^') {
+		*len = 0;
+		if (rex_matchme(re + 1, str, len))
+			return 0;
+		return -1;
+	}
+	// And general case for any position
+	int pos = 0;
+	do {
+		*len = 0;
+		if (rex_matchme(re, str + pos, len))
+			return pos;
+	} while (str[pos++] != '\0');
+	// Matching failed
+	return -1;
+}
+
+/*******************************************************************************
+ * Pattern handling
+ *
+ *   Patterns are the heart the data input process, they provide a way to tell
+ *   Wapiti how the interesting information can be extracted from the input
+ *   data. A pattern is simply a string who embed special commands about tokens
+ *   to extract from the input sequence. They are compiled to a special form
+ *   used during data loading.
+ *   For training, each position of a sequence hold a list of observation made
+ *   at this position, pattern give a way to specify these observations.
+ *
+ *   During sequence loading, all patterns are applied at each position to
+ *   produce a list of string representing the observations which will be in
+ *   turn transformed to numerical identifiers. This module take care of
+ *   building the string representation.
+ *
+ *   As said, a patern is a string with specific commands in the forms %c[...]
+ *   where 'c' is the command with arguments between the bracket. All commands
+ *   take at least to numerical arguments which define a token in the input
+ *   sequence. The first one is an offset from the current position and the
+ *   second one is a column number. With these two parameters, we get a string
+ *   in the input sequence on which we apply the command.
+ *
+ *   All command are specified with a character and result in a string which
+ *   will replace the command in the pattern string. If the command character is
+ *   lower case, the result is copied verbatim, if it is uppercase, the result
+ *   is copied with casing removed. The following commands are available:
+ *     'x' -- result is the token itself
+ *     't' -- test if a regular expression match the token. Result will be
+ *            either "true" or "false"
+ *     'm' -- match a regular expression on the token. Result is the first
+ *            substring matched.
+ ******************************************************************************/
+
+/* pat_comp:
+ *   Compile the pattern to a form more suitable to easily apply it on tokens
+ *   list during data reading. The given pattern string is interned in the
+ *   compiled pattern and will be freed with it, so you don't have to take care
+ *   of it and must not modify it after the compilation.
+ */
+pat_t *pat_comp(char *p) {
+	pat_t *pat = NULL;
+	// Allocate memory for the compiled pattern, the allocation is based
+	// on an over-estimation of the number of required item. As compiled
+	// pattern take a neglectible amount of memory, this waste is not
+	// important.
+	int mitems = 0;
+	for (int pos = 0; p[pos] != '\0'; pos++)
+		if (p[pos] == '%')
+			mitems++;
+	mitems = mitems * 2 + 1;
+	pat = wapiti_xmalloc(sizeof(pat_t) + sizeof(pat->items[0]) * mitems);
+	pat->src = p;
+	// Next, we go through the pattern compiling the items as they are
+	// found. Commands are parsed and put in a corresponding item, and
+	// segment of char not in a command are put in a 's' item.
+	int nitems = 0;
+	int ntoks = 0;
+	int pos = 0;
+	while (p[pos] != '\0') {
+		pat_item_t *item = &(pat->items[nitems++]);
+		item->value = NULL;
+		if (p[pos] == '%') {
+			// This is a command, so first parse its type and check
+			// its a valid one. Next prepare the item.
+			const char type = tolower(p[pos + 1]);
+			if (type != 'x' && type != 't' && type != 'm')
+				fatal("unknown command type: '%c'", type);
+			item->type = type;
+			item->caps = (p[pos + 1] != type);
+			pos += 2;
+			// Next we parse the offset and column and store them in
+			// the item.
+			const char *at = p + pos;
+			int off, col, nch;
+			item->absolute = false;
+			if (sscanf(at, "[@%d,%d%n", &off, &col, &nch) == 2)
+				item->absolute = true;
+			else if (sscanf(at, "[%d,%d%n", &off, &col, &nch) != 2)
+				fatal("invalid pattern: %s", p);
+			if (col < 0)
+				fatal("invalid column number: %d", col);
+			item->offset = off;
+			item->column = col;
+			ntoks = max(ntoks, col);
+			pos += nch;
+			// And parse the end of the argument list, for 'x' there
+			// is nothing to read but for 't' and 'm' we have to get
+			// read the regexp.
+			if (type == 't' || type == 'm') {
+				if (p[pos] != ',' && p[pos + 1] != '"')
+					fatal("missing arg in pattern: %s", p);
+				const int start = (pos += 2);
+				while (p[pos] != '\0') {
+					if (p[pos] == '"')
+						break;
+					if (p[pos] == '\\' && p[pos+1] != '\0')
+						pos++;
+					pos++;
+				}
+				if (p[pos] != '"')
+					fatal("unended argument: %s", p);
+				const int len = pos - start;
+				item->value = wapiti_xmalloc(sizeof(char) * (len + 1));
+				memcpy(item->value, p + start, len);
+				item->value[len] = '\0';
+				pos++;
+			}
+			// Just check the end of the arg list and loop.
+			if (p[pos] != ']')
+				fatal("missing end of pattern: %s", p);
+			pos++;
+		} else {
+			// No command here, so build an 's' item with the chars
+			// until end of pattern or next command and put it in
+			// the list.
+			const int start = pos;
+			while (p[pos] != '\0' && p[pos] != '%')
+				pos++;
+			const int len = pos - start;
+			item->type  = 's';
+			item->caps  = false;
+			item->value = wapiti_xmalloc(sizeof(char) * (len + 1));
+			memcpy(item->value, p + start, len);
+			item->value[len] = '\0';
+		}
+	}
+	pat->ntoks = ntoks;
+	pat->nitems = nitems;
+	return pat;
+}
+
+/* pat_exec:
+ *   Execute a compiled pattern at position 'at' in the given tokens sequences
+ *   in order to produce an observation string. The string is returned as a
+ *   newly allocated memory block and the caller is responsible to free it when
+ *   not needed anymore.
+ */
+char *pat_exec(const pat_t *pat, const tok_t *tok, int at) {
+	static char *bval[] = {"_x-1", "_x-2", "_x-3", "_x-4", "_x-#"};
+	static char *eval[] = {"_x+1", "_x+2", "_x+3", "_x+4", "_x+#"};
+	const int T = tok->len;
+	// Prepare the buffer who will hold the result
+	int size = 16, pos = 0;
+	char *buffer = wapiti_xmalloc(sizeof(char) * size);
+	// And loop over the compiled items
+	for (int it = 0; it < pat->nitems; it++) {
+		const pat_item_t *item = &(pat->items[it]);
+		char *value = NULL;
+		int len = 0;
+		// First, if needed, we retrieve the token at the referenced
+		// position in the sequence. We store it in value and let the
+		// command handler do what it need with it.
+		if (item->type != 's') {
+			int pos = item->offset;
+			if (item->absolute) {
+				if (item->offset < 0)
+					pos += T;
+				else
+					pos--;
+			} else {
+				pos += at;
+			}
+			int col = item->column;
+			if (pos < 0)
+				value = bval[min(-pos - 1, 4)];
+			else if (pos >= T)
+				value = eval[min( pos - T, 4)];
+			else if (col >= tok->cnts[pos])
+				fatal("missing tokens, cannot apply pattern");
+			else
+				value = tok->toks[pos][col];
+		}
+		// Next, we handle the command, 's' and 'x' are very simple but
+		// 't' and 'm' require us to call the regexp matcher.
+		if (item->type == 's') {
+			value = item->value;
+			len = strlen(value);
+		} else if (item->type == 'x') {
+			len = strlen(value);
+		} else if (item->type == 't') {
+			if (rex_match(item->value, value, &len) == -1)
+				value = "false";
+			else
+				value = "true";
+			len = strlen(value);
+		} else if (item->type == 'm') {
+			int pos = rex_match(item->value, value, &len);
+			if (pos == -1)
+				len = 0;
+			value += pos;
+		}
+		// And we add it to the buffer, growing it if needed. If the
+		// user requested it, we also remove caps from the string.
+		if (pos + len >= size - 1) {
+			while (pos + len >= size - 1)
+				size = size * 1.4;
+			buffer = wapiti_xrealloc(buffer, sizeof(char) * size);
+		}
+		memcpy(buffer + pos, value, len);
+		if (item->caps)
+			for (int i = pos; i < pos + len; i++)
+				buffer[i] = tolower(buffer[i]);
+		pos += len;
+	}
+	// Adjust the result and return it.
+	buffer[pos++] = '\0';
+	buffer = wapiti_xrealloc(buffer, sizeof(char) * pos);
+	return buffer;
+}
+
+/* pat_free:
+ *   Free all memory used by a compiled pattern object. Note that this will free
+ *   the pointer to the source string given to pat_comp so you must be sure to
+ *   not use this pointer again.
+ */
+void pat_free(pat_t *pat) {
+	for (int it = 0; it < pat->nitems; it++)
+		free(pat->items[it].value);
+	free(pat->src);
+	free(pat);
+}
+
+