ealdent-lda-ruby 0.1.1

data/cokus.c

@@ -0,0 +1,145 @@
+// This is the ``Mersenne Twister'' random number generator MT19937, which
+// generates pseudorandom integers uniformly distributed in 0..(2^32 - 1)
+// starting from any odd seed in 0..(2^32 - 1).  This version is a recode
+// by Shawn Cokus (Cokus@math.washington.edu) on March 8, 1998 of a version by
+// Takuji Nishimura (who had suggestions from Topher Cooper and Marc Rieffel in
+// July-August 1997).
+//
+// Effectiveness of the recoding (on Goedel2.math.washington.edu, a DEC Alpha
+// running OSF/1) using GCC -O3 as a compiler: before recoding: 51.6 sec. to
+// generate 300 million random numbers; after recoding: 24.0 sec. for the same
+// (i.e., 46.5% of original time), so speed is now about 12.5 million random
+// number generations per second on this machine.
+//
+// According to the URL <http://www.math.keio.ac.jp/~matumoto/emt.html>
+// (and paraphrasing a bit in places), the Mersenne Twister is ``designed
+// with consideration of the flaws of various existing generators,'' has
+// a period of 2^19937 - 1, gives a sequence that is 623-dimensionally
+// equidistributed, and ``has passed many stringent tests, including the
+// die-hard test of G. Marsaglia and the load test of P. Hellekalek and
+// S. Wegenkittl.''  It is efficient in memory usage (typically using 2506
+// to 5012 bytes of static data, depending on data type sizes, and the code
+// is quite short as well).  It generates random numbers in batches of 624
+// at a time, so the caching and pipelining of modern systems is exploited.
+// It is also divide- and mod-free.
+//
+// This library is free software; you can redistribute it and/or modify it
+// under the terms of the GNU Library General Public License as published by
+// the Free Software Foundation (either version 2 of the License or, at your
+// option, any later version).  This library is distributed in the hope that
+// it will be useful, but WITHOUT ANY WARRANTY, without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See
+// the GNU Library General Public License for more details.  You should have
+// received a copy of the GNU Library General Public License along with this
+// library; if not, write to the Free Software Foundation, Inc., 59 Temple
+// Place, Suite 330, Boston, MA 02111-1307, USA.
+//
+// The code as Shawn received it included the following notice:
+//
+//   Copyright (C) 1997 Makoto Matsumoto and Takuji Nishimura.  When
+//   you use this, send an e-mail to <matumoto@math.keio.ac.jp> with
+//   an appropriate reference to your work.
+//
+// It would be nice to CC: <Cokus@math.washington.edu> when you write.
+//
+
+#include "cokus.h"
+
+static uint32   state[N+1];     // state vector + 1 extra to not violate ANSI C
+static uint32   *next;          // next random value is computed from here
+static int      left = -1;      // can *next++ this many times before reloading
+
+void seedMT(uint32 seed)
+ {
+    //
+    // We initialize state[0..(N-1)] via the generator
+    //
+    //   x_new = (69069 * x_old) mod 2^32
+    //
+    // from Line 15 of Table 1, p. 106, Sec. 3.3.4 of Knuth's
+    // _The Art of Computer Programming_, Volume 2, 3rd ed.
+    //
+    // Notes (SJC): I do not know what the initial state requirements
+    // of the Mersenne Twister are, but it seems this seeding generator
+    // could be better.  It achieves the maximum period for its modulus
+    // (2^30) iff x_initial is odd (p. 20-21, Sec. 3.2.1.2, Knuth); if
+    // x_initial can be even, you have sequences like 0, 0, 0, ...;
+    // 2^31, 2^31, 2^31, ...; 2^30, 2^30, 2^30, ...; 2^29, 2^29 + 2^31,
+    // 2^29, 2^29 + 2^31, ..., etc. so I force seed to be odd below.
+    //
+    // Even if x_initial is odd, if x_initial is 1 mod 4 then
+    //
+    //   the          lowest bit of x is always 1,
+    //   the  next-to-lowest bit of x is always 0,
+    //   the 2nd-from-lowest bit of x alternates      ... 0 1 0 1 0 1 0 1 ... ,
+    //   the 3rd-from-lowest bit of x 4-cycles        ... 0 1 1 0 0 1 1 0 ... ,
+    //   the 4th-from-lowest bit of x has the 8-cycle ... 0 0 0 1 1 1 1 0 ... ,
+    //    ...
+    //
+    // and if x_initial is 3 mod 4 then
+    //
+    //   the          lowest bit of x is always 1,
+    //   the  next-to-lowest bit of x is always 1,
+    //   the 2nd-from-lowest bit of x alternates      ... 0 1 0 1 0 1 0 1 ... ,
+    //   the 3rd-from-lowest bit of x 4-cycles        ... 0 0 1 1 0 0 1 1 ... ,
+    //   the 4th-from-lowest bit of x has the 8-cycle ... 0 0 1 1 1 1 0 0 ... ,
+    //    ...
+    //
+    // The generator's potency (min. s>=0 with (69069-1)^s = 0 mod 2^32) is
+    // 16, which seems to be alright by p. 25, Sec. 3.2.1.3 of Knuth.  It
+    // also does well in the dimension 2..5 spectral tests, but it could be
+    // better in dimension 6 (Line 15, Table 1, p. 106, Sec. 3.3.4, Knuth).
+    //
+    // Note that the random number user does not see the values generated
+    // here directly since reloadMT() will always munge them first, so maybe
+    // none of all of this matters.  In fact, the seed values made here could
+    // even be extra-special desirable if the Mersenne Twister theory says
+    // so-- that's why the only change I made is to restrict to odd seeds.
+    //
+
+    register uint32 x = (seed | 1U) & 0xFFFFFFFFU, *s = state;
+    register int    j;
+
+    for(left=0, *s++=x, j=N; --j;
+        *s++ = (x*=69069U) & 0xFFFFFFFFU);
+ }
+
+
+uint32 reloadMT(void)
+{
+    register uint32 *p0=state, *p2=state+2, *pM=state+M, s0, s1;
+    register int    j;
+
+    if(left < -1)
+        seedMT(4357U);
+
+    left=N-1, next=state+1;
+
+    for(s0=state[0], s1=state[1], j=N-M+1; --j; s0=s1, s1=*p2++)
+        *p0++ = *pM++ ^ (mixBits(s0, s1) >> 1) ^ (loBit(s1) ? K : 0U);
+
+    for(pM=state, j=M; --j; s0=s1, s1=*p2++)
+        *p0++ = *pM++ ^ (mixBits(s0, s1) >> 1) ^ (loBit(s1) ? K : 0U);
+
+    s1=state[0], *p0 = *pM ^ (mixBits(s0, s1) >> 1) ^ (loBit(s1) ? K : 0U);
+    s1 ^= (s1 >> 11);
+    s1 ^= (s1 <<  7) & 0x9D2C5680U;
+    s1 ^= (s1 << 15) & 0xEFC60000U;
+    return(s1 ^ (s1 >> 18));
+ }
+
+uint32 randomMT(void)
+ {
+    uint32 y;
+
+    if(--left < 0)
+        return(reloadMT());
+
+    y  = *next++;
+    y ^= (y >> 11);
+    y ^= (y <<  7) & 0x9D2C5680U;
+    y ^= (y << 15) & 0xEFC60000U;
+    y ^= (y >> 18);
+    return(y);
+ }
+

data/cokus.h

@@ -0,0 +1,27 @@
+#ifndef COKUS_H
+#define COKUS_H
+
+#include <stdio.h>
+#include <stdlib.h>
+
+//
+// uint32 must be an unsigned integer type capable of holding at least 32
+// bits; exactly 32 should be fastest, but 64 is better on an Alpha with
+// GCC at -O3 optimization so try your options and see what's best for you
+//
+
+typedef unsigned long uint32;
+
+#define N              (624)                 // length of state vector
+#define M              (397)                 // a period parameter
+#define K              (0x9908B0DFU)         // a magic constant
+#define hiBit(u)       ((u) & 0x80000000U)   // mask all but highest   bit of u
+#define loBit(u)       ((u) & 0x00000001U)   // mask all but lowest    bit of u
+#define loBits(u)      ((u) & 0x7FFFFFFFU)   // mask     the highest   bit of u
+#define mixBits(u, v)  (hiBit(u)|loBits(v))  // move hi bit of u to hi bit of v
+
+void seedMT(uint32 seed);
+uint32 reloadMT(void);
+uint32 randomMT(void);
+
+#endif

data/extconf.rb

@@ -0,0 +1,4 @@
+require 'mkmf'
+
+dir_config("lda_ext")
+create_makefile("lda_ext")

data/lda-alpha.c

@@ -0,0 +1,68 @@
+// (C) Copyright 2004, David M. Blei (blei [at] cs [dot] cmu [dot] edu)
+
+// This file is part of LDA-C.
+
+// LDA-C is free software; you can redistribute it and/or modify it under
+// the terms of the GNU General Public License as published by the Free
+// Software Foundation; either version 2 of the License, or (at your
+// option) any later version.
+
+// LDA-C is distributed in the hope that it will be useful, but WITHOUT
+// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+// FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+// for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with this program; if not, write to the Free Software
+// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+// USA
+
+#include "lda-alpha.h"
+
+/*
+ * objective function and its derivatives
+ *
+ */
+
+double alhood(double a, double ss, int D, int K)
+{ return(D * (lgamma(K * a) - K * lgamma(a)) + (a - 1) * ss); }
+
+double d_alhood(double a, double ss, int D, int K)
+{ return(D * (K * digamma(K * a) - K * digamma(a)) + ss); }
+
+double d2_alhood(double a, int D, int K)
+{ return(D * (K * K * trigamma(K * a) - K * trigamma(a))); }
+
+
+/*
+ * newtons method
+ *
+ */
+
+double opt_alpha(double ss, int D, int K)
+{
+    double a, log_a, init_a = 100;
+    double f, df, d2f;
+    int iter = 0;
+
+    log_a = log(init_a);
+    do
+    {
+        iter++;
+        a = exp(log_a);
+        if (isnan(a))
+        {
+            init_a = init_a * 10;
+            printf("warning : alpha is nan; new init = %5.5f\n", init_a);
+            a = init_a;
+            log_a = log(a);
+        }
+        f = alhood(a, ss, D, K);
+        df = d_alhood(a, ss, D, K);
+        d2f = d2_alhood(a, D, K);
+        log_a = log_a - df/(d2f * a + df);
+        printf("alpha maximization : %5.5f   %5.5f\n", f, df);
+    }
+    while ((fabs(df) > NEWTON_THRESH) && (iter < MAX_ALPHA_ITER));
+    return(exp(log_a));
+}

data/lda-alpha.h

@@ -0,0 +1,20 @@
+#ifndef LDA_ALPHA_H
+#define LDA_ALPHA_H
+
+#include <stdlib.h>
+#include <math.h>
+#include <float.h>
+
+#include "lda.h"
+#include "utils.h"
+
+#define NEWTON_THRESH 1e-5
+#define MAX_ALPHA_ITER 1000
+
+double alhood(double a, double ss, int D, int K);
+double d_alhood(double a, double ss, int D, int K);
+double d2_alhood(double a, int D, int K);
+double opt_alpha(double ss, int D, int K);
+void maximize_alpha(double** gamma, lda_model* model, int num_docs);
+
+#endif

data/lda-data.c

@@ -0,0 +1,67 @@
+// (C) Copyright 2004, David M. Blei (blei [at] cs [dot] cmu [dot] edu)
+
+// This file is part of LDA-C.
+
+// LDA-C is free software; you can redistribute it and/or modify it under
+// the terms of the GNU General Public License as published by the Free
+// Software Foundation; either version 2 of the License, or (at your
+// option) any later version.
+
+// LDA-C is distributed in the hope that it will be useful, but WITHOUT
+// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+// FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+// for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with this program; if not, write to the Free Software
+// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+// USA
+
+#include "lda-data.h"
+
+corpus* read_data(char* data_filename)
+{
+	FILE *fileptr;
+	int length, count, word, n, nd, nw;
+	corpus* c;
+
+	printf("reading data from %s\n", data_filename);
+	c = malloc(sizeof(corpus));
+	c->docs = 0;
+	c->num_terms = 0;
+	c->num_docs = 0;
+	fileptr = fopen(data_filename, "r");
+	nd = 0; nw = 0;
+	while ((fscanf(fileptr, "%10d", &length) != EOF))
+	{
+		c->docs = (document*) realloc(c->docs, sizeof(document)*(nd+1));
+		c->docs[nd].length = length;
+		c->docs[nd].total = 0;
+		c->docs[nd].words = malloc(sizeof(int)*length);
+		c->docs[nd].counts = malloc(sizeof(int)*length);
+		for (n = 0; n < length; n++)
+		{
+			fscanf(fileptr, "%10d:%10d", &word, &count);
+			word = word - OFFSET;
+			c->docs[nd].words[n] = word;
+			c->docs[nd].counts[n] = count;
+			c->docs[nd].total += count;
+			if (word >= nw) { nw = word + 1; }
+		}
+		nd++;
+	}
+	fclose(fileptr);
+	c->num_docs = nd;
+	c->num_terms = nw;
+	printf("number of docs    : %d\n", nd);
+	printf("number of terms   : %d\n", nw);
+	return(c);
+}
+
+int max_corpus_length(corpus* c)
+{
+	int n, max = 0;
+	for (n = 0; n < c->num_docs; n++)
+		if (c->docs[n].length > max) max = c->docs[n].length;
+	return(max);
+}

data/lda-data.h

@@ -0,0 +1,14 @@
+#ifndef LDA_DATA_H
+#define LDA_DATA_H
+
+#include <stdio.h>
+#include <stdlib.h>
+
+#include "lda.h"
+
+#define OFFSET 0;                  // offset for reading data
+
+corpus* read_data(char* data_filename);
+int max_corpus_length(corpus* c);
+
+#endif

data/lda-inference.c

@@ -0,0 +1,875 @@
+// (C) Copyright 2004, David M. Blei (blei [at] cs [dot] cmu [dot] edu)
+
+// This file is part of LDA-C.
+
+// LDA-C is free software; you can redistribute it and/or modify it under
+// the terms of the GNU General Public License as published by the Free
+// Software Foundation; either version 2 of the License, or (at your
+// option) any later version.
+
+// LDA-C is distributed in the hope that it will be useful, but WITHOUT
+// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+// FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+// for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with this program; if not, write to the Free Software
+// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+// USA
+
+#ifndef USE_RUBY
+#define USE_RUBY
+#endif
+
+#include <stdlib.h>
+#include <stdio.h>
+#include <math.h>
+#include <float.h>
+#include <string.h>
+#include <time.h>
+
+#include "lda.h"
+#include "lda-data.h"
+#include "lda-inference.h"
+#include "lda-model.h"
+#include "lda-alpha.h"
+#include "utils.h"
+#include "cokus.h"
+
+#ifdef USE_RUBY
+#include "ruby.h"
+
+VALUE rb_mLda;
+VALUE rb_cLda;
+VALUE rb_cLdaCorpus;
+VALUE rb_cLdaDocument;
+#endif
+
+/*
+ * variational inference
+ */
+
+double lda_inference(document* doc, lda_model* model, double* var_gamma, double** phi) {
+	double converged = 1;
+	double phisum = 0, likelihood = 0;
+	double likelihood_old = 0, oldphi[model->num_topics];
+	int k, n, var_iter;
+	double digamma_gam[model->num_topics];
+
+		// compute posterior dirichlet
+
+	for (k = 0; k < model->num_topics; k++)
+	{
+		var_gamma[k] = model->alpha + (doc->total/((double) model->num_topics));
+		digamma_gam[k] = digamma(var_gamma[k]);
+		for (n = 0; n < doc->length; n++)
+			phi[n][k] = 1.0/model->num_topics;
+	}
+	var_iter = 0;
+
+	while ((converged > VAR_CONVERGED) &&
+		((var_iter < VAR_MAX_ITER) || (VAR_MAX_ITER == -1)))
+	{
+		var_iter++;
+		for (n = 0; n < doc->length; n++)
+		{
+			phisum = 0;
+			for (k = 0; k < model->num_topics; k++)
+			{
+				oldphi[k] = phi[n][k];
+				phi[n][k] =
+					digamma_gam[k] +
+					model->log_prob_w[k][doc->words[n]];
+
+				if (k > 0)
+					phisum = log_sum(phisum, phi[n][k]);
+				else
+					phisum = phi[n][k]; // note, phi is in log space
+			}
+
+			for (k = 0; k < model->num_topics; k++)
+			{
+				phi[n][k] = exp(phi[n][k] - phisum);
+				var_gamma[k] =
+					var_gamma[k] + doc->counts[n]*(phi[n][k] - oldphi[k]);
+								// !!! a lot of extra digamma's here because of how we're computing it
+								// !!! but its more automatically updated too.
+				digamma_gam[k] = digamma(var_gamma[k]);
+			}
+		}
+
+		likelihood = compute_likelihood(doc, model, phi, var_gamma);
+		assert(!isnan(likelihood));
+		converged = (likelihood_old - likelihood) / likelihood_old;
+		likelihood_old = likelihood;
+
+				// printf("[LDA INF] %8.5f %1.3e\n", likelihood, converged);
+	}
+	return(likelihood);
+}
+
+
+/*
+ * compute likelihood bound
+ */
+
+double compute_likelihood(document* doc, lda_model* model, double** phi, double* var_gamma) {
+	double likelihood = 0, digsum = 0, var_gamma_sum = 0, dig[model->num_topics];
+	int k, n;
+
+	for (k = 0; k < model->num_topics; k++)
+	{
+		dig[k] = digamma(var_gamma[k]);
+		var_gamma_sum += var_gamma[k];
+	}
+	digsum = digamma(var_gamma_sum);
+
+	likelihood = lgamma(model->alpha * model -> num_topics) - model -> num_topics * lgamma(model->alpha) - (lgamma(var_gamma_sum));
+
+	for (k = 0; k < model->num_topics; k++)
+	{
+		likelihood += (model->alpha - 1)*(dig[k] - digsum) + lgamma(var_gamma[k]) - (var_gamma[k] - 1)*(dig[k] - digsum);
+		
+		for (n = 0; n < doc->length; n++)
+		{
+			if (phi[n][k] > 0)
+			{
+				likelihood += doc->counts[n]*
+					(phi[n][k]*((dig[k] - digsum) - log(phi[n][k])
+					+ model->log_prob_w[k][doc->words[n]]));
+			}
+		}
+	}
+	return(likelihood);
+}
+
+
+double doc_e_step(document* doc, double* gamma, double** phi, lda_model* model, lda_suffstats* ss) {
+	double likelihood;
+	int n, k;
+
+		// posterior inference
+
+	likelihood = lda_inference(doc, model, gamma, phi);
+
+		// update sufficient statistics
+
+	double gamma_sum = 0;
+	for (k = 0; k < model->num_topics; k++)
+	{
+		gamma_sum += gamma[k];
+		ss->alpha_suffstats += digamma(gamma[k]);
+	}
+	ss->alpha_suffstats -= model->num_topics * digamma(gamma_sum);
+
+	for (n = 0; n < doc->length; n++)
+	{
+		for (k = 0; k < model->num_topics; k++)
+		{
+			ss->class_word[k][doc->words[n]] += doc->counts[n]*phi[n][k];
+			ss->class_total[k] += doc->counts[n]*phi[n][k];
+		}
+	}
+
+	ss->num_docs = ss->num_docs + 1;
+
+	return(likelihood);
+}
+
+
+/*
+ * writes the word assignments line for a document to a file
+ */
+
+void write_word_assignment(FILE* f, document* doc, double** phi, lda_model* model) {
+	int n;
+
+	fprintf(f, "%03d", doc->length);
+	for (n = 0; n < doc->length; n++) {
+		fprintf(f, " %04d:%02d", doc->words[n], argmax(phi[n], model->num_topics));
+	}
+	fprintf(f, "\n");
+	fflush(f);
+}
+
+
+/*
+ * saves the gamma parameters of the current dataset
+ */
+
+void save_gamma(char* filename, double** gamma, int num_docs, int num_topics) {
+	FILE* fileptr;
+	int d, k;
+	fileptr = fopen(filename, "w");
+
+	for (d = 0; d < num_docs; d++) {
+		fprintf(fileptr, "%5.10f", gamma[d][0]);
+		for (k = 1; k < num_topics; k++) {
+			fprintf(fileptr, " %5.10f", gamma[d][k]);
+		}
+		fprintf(fileptr, "\n");
+	}
+	fclose(fileptr);
+}
+
+
+void run_em(char* start, char* directory, corpus* corpus) {
+	int d, n;
+	lda_model *model = NULL;
+	double **var_gamma, **phi;
+
+	// allocate variational parameters
+
+	var_gamma = malloc(sizeof(double*)*(corpus->num_docs));
+	for (d = 0; d < corpus->num_docs; d++)
+		var_gamma[d] = malloc(sizeof(double) * NTOPICS);
+
+	int max_length = max_corpus_length(corpus);
+	phi = malloc(sizeof(double*)*max_length);
+	for (n = 0; n < max_length; n++)
+		phi[n] = malloc(sizeof(double) * NTOPICS);
+
+	// initialize model
+
+	char filename[100];
+
+	lda_suffstats* ss = NULL;
+	if (strcmp(start, "seeded")==0) {
+		model = new_lda_model(corpus->num_terms, NTOPICS);
+		ss = new_lda_suffstats(model);
+		corpus_initialize_ss(ss, model, corpus);
+		lda_mle(model, ss, 0);
+		model->alpha = INITIAL_ALPHA;
+	} else if (strcmp(start, "random")==0) {
+		model = new_lda_model(corpus->num_terms, NTOPICS);
+		ss = new_lda_suffstats(model);
+		random_initialize_ss(ss, model);
+		lda_mle(model, ss, 0);
+		model->alpha = INITIAL_ALPHA;
+	} else {
+		model = load_lda_model(start);
+		ss = new_lda_suffstats(model);
+	}
+
+	sprintf(filename,"%s/000",directory);
+	save_lda_model(model, filename);
+
+	// run expectation maximization
+
+	int i = 0;
+	double likelihood, likelihood_old = 0, converged = 1;
+	sprintf(filename, "%s/likelihood.dat", directory);
+	FILE* likelihood_file = fopen(filename, "w");
+
+	while (((converged < 0) || (converged > EM_CONVERGED) || (i <= 2)) && (i <= EM_MAX_ITER)) {
+		i++;
+		printf("**** em iteration %d ****\n", i);
+		likelihood = 0;
+		zero_initialize_ss(ss, model);
+
+		// e-step
+
+		for (d = 0; d < corpus->num_docs; d++) {
+			if ((d % 1000) == 0) printf("document %d\n",d);
+			likelihood += doc_e_step(&(corpus->docs[d]), var_gamma[d], phi, model, ss);
+		}
+
+		// m-step
+
+		lda_mle(model, ss, ESTIMATE_ALPHA);
+
+		// check for convergence
+
+		converged = (likelihood_old - likelihood) / (likelihood_old);
+		if (converged < 0) VAR_MAX_ITER = VAR_MAX_ITER * 2;
+		likelihood_old = likelihood;
+
+		// output model and likelihood
+
+		fprintf(likelihood_file, "%10.10f\t%5.5e\n", likelihood, converged);
+		fflush(likelihood_file);
+		if ((i % LAG) == 0)
+		{
+			sprintf(filename,"%s/%03d",directory, i);
+			save_lda_model(model, filename);
+			sprintf(filename,"%s/%03d.gamma",directory, i);
+			save_gamma(filename, var_gamma, corpus->num_docs, model->num_topics);
+		}
+	}
+
+		// output the final model
+
+	sprintf(filename,"%s/final",directory);
+	save_lda_model(model, filename);
+	sprintf(filename,"%s/final.gamma",directory);
+	save_gamma(filename, var_gamma, corpus->num_docs, model->num_topics);
+
+		// output the word assignments (for visualization)
+
+	sprintf(filename, "%s/word-assignments.dat", directory);
+	FILE* w_asgn_file = fopen(filename, "w");
+	for (d = 0; d < corpus->num_docs; d++)
+	{
+		if ((d % 100) == 0) printf("final e step document %d\n",d);
+		likelihood += lda_inference(&(corpus->docs[d]), model, var_gamma[d], phi);
+		write_word_assignment(w_asgn_file, &(corpus->docs[d]), phi, model);
+	}
+	fclose(w_asgn_file);
+	fclose(likelihood_file);
+}
+
+
+/*
+ * read settings.
+ */
+
+void read_settings(char* filename) {
+	FILE* fileptr;
+	char alpha_action[100];
+	fileptr = fopen(filename, "r");
+	fscanf(fileptr, "var max iter %d\n", &VAR_MAX_ITER);
+	fscanf(fileptr, "var convergence %f\n", &VAR_CONVERGED);
+	fscanf(fileptr, "em max iter %d\n", &EM_MAX_ITER);
+	fscanf(fileptr, "em convergence %f\n", &EM_CONVERGED);
+	fscanf(fileptr, "alpha %s", alpha_action);
+	if (strcmp(alpha_action, "fixed")==0)
+	{
+		ESTIMATE_ALPHA = 0;
+	}
+	else
+	{
+		ESTIMATE_ALPHA = 1;
+	}
+	fclose(fileptr);
+}
+
+
+
+
+/*
+* inference only
+	*
+*/
+
+void infer(char* model_root, char* save, corpus* corpus) {
+	FILE* fileptr;
+	char filename[100];
+	int i, d, n;
+	lda_model *model;
+	double **var_gamma, likelihood, **phi;
+	document* doc;
+
+	model = load_lda_model(model_root);
+	var_gamma = malloc(sizeof(double*)*(corpus->num_docs));
+	for (i = 0; i < corpus->num_docs; i++)
+		var_gamma[i] = malloc(sizeof(double)*model->num_topics);
+	sprintf(filename, "%s-lda-lhood.dat", save);
+	fileptr = fopen(filename, "w");
+	for (d = 0; d < corpus->num_docs; d++) {
+		if (((d % 100) == 0) && (d>0)) printf("document %d\n",d);
+
+		doc = &(corpus->docs[d]);
+		phi = (double**) malloc(sizeof(double*) * doc->length);
+		for (n = 0; n < doc->length; n++)
+			phi[n] = (double*) malloc(sizeof(double) * model->num_topics);
+		likelihood = lda_inference(doc, model, var_gamma[d], phi);
+
+		fprintf(fileptr, "%5.5f\n", likelihood);
+	}
+	fclose(fileptr);
+	sprintf(filename, "%s-gamma.dat", save);
+	save_gamma(filename, var_gamma, corpus->num_docs, model->num_topics);
+}
+
+
+/*
+ * update sufficient statistics
+ *
+ */
+
+
+
+/*
+ * main
+ *
+ */
+
+int main(int argc, char* argv[]) {
+	corpus* corpus;
+
+	long t1;
+	(void) time(&t1);
+	seedMT(t1);
+		// seedMT(4357U);
+
+	if (argc > 1)
+	{
+		if (strcmp(argv[1], "est")==0)
+		{
+			INITIAL_ALPHA = atof(argv[2]);
+			NTOPICS = atoi(argv[3]);
+			read_settings(argv[4]);
+			corpus = read_data(argv[5]);
+			make_directory(argv[7]);
+			run_em(argv[6], argv[7], corpus);
+		}
+		if (strcmp(argv[1], "inf")==0)
+		{
+			read_settings(argv[2]);
+			corpus = read_data(argv[4]);
+			infer(argv[3], argv[5], corpus);
+		}
+	}
+	else
+	{
+		printf("usage : lda est [initial alpha] [k] [settings] [data] [random/seeded/*] [directory]\n");
+		printf("        lda inf [settings] [model] [data] [name]\n");
+	}
+	return(0);
+}
+
+#ifdef USE_RUBY
+
+/* */
+void run_quiet_em(char* start, corpus* corpus) {
+	int d, n;
+	lda_model *model = NULL;
+	double **var_gamma, **phi;
+
+	// allocate variational parameters
+
+	var_gamma = malloc(sizeof(double*)*(corpus->num_docs));
+	for (d = 0; d < corpus->num_docs; d++)
+		var_gamma[d] = malloc(sizeof(double) * NTOPICS);
+
+	int max_length = max_corpus_length(corpus);
+	phi = malloc(sizeof(double*)*max_length);
+	for (n = 0; n < max_length; n++)
+		phi[n] = malloc(sizeof(double) * NTOPICS);
+
+	// initialize model
+
+	lda_suffstats* ss = NULL;
+	if (strcmp(start, "seeded")==0) {
+		model = new_lda_model(corpus->num_terms, NTOPICS);
+		ss = new_lda_suffstats(model);
+		corpus_initialize_ss(ss, model, corpus);
+		lda_mle(model, ss, 0);
+		model->alpha = INITIAL_ALPHA;
+	} else if (strcmp(start, "random")==0) {
+		model = new_lda_model(corpus->num_terms, NTOPICS);
+		ss = new_lda_suffstats(model);
+		random_initialize_ss(ss, model);
+		lda_mle(model, ss, 0);
+		model->alpha = INITIAL_ALPHA;
+	} else {
+		model = load_lda_model(start);
+		ss = new_lda_suffstats(model);
+	}
+
+	// save the model in the last_model global
+	last_model = model;
+	model_loaded = TRUE;
+
+	// run expectation maximization
+
+	int i = 0;
+	double likelihood, likelihood_old = 0, converged = 1;
+
+	while (((converged < 0) || (converged > EM_CONVERGED) || (i <= 2)) && (i <= EM_MAX_ITER)) {
+		i++;
+		printf("**** em iteration %d ****\n", i);
+		likelihood = 0;
+		zero_initialize_ss(ss, model);
+
+		// e-step
+
+		for (d = 0; d < corpus->num_docs; d++) {
+			if ((d % 1000) == 0) printf("document %d\n",d);
+			likelihood += doc_e_step(&(corpus->docs[d]), var_gamma[d], phi, model, ss);
+		}
+
+		// m-step
+
+		lda_mle(model, ss, ESTIMATE_ALPHA);
+
+		// check for convergence
+
+		converged = (likelihood_old - likelihood) / (likelihood_old);
+		if (converged < 0) VAR_MAX_ITER = VAR_MAX_ITER * 2;
+		likelihood_old = likelihood;
+
+		// store model and likelihood
+
+		last_model = model;
+		last_gamma = var_gamma;
+	}
+
+	// output the final model
+
+	last_model = model;
+	last_gamma = var_gamma;
+
+	// output the word assignments (for visualization)
+	/*
+	char filename[100];
+	sprintf(filename, "%s/word-assignments.dat", directory);
+	FILE* w_asgn_file = fopen(filename, "w");
+	for (d = 0; d < corpus->num_docs; d++) {
+		if ((d % 100) == 0)
+			printf("final e step document %d\n",d);
+		likelihood += lda_inference(&(corpus->docs[d]), model, var_gamma[d], phi);
+		write_word_assignment(w_asgn_file, &(corpus->docs[d]), phi, model);
+	}
+	fclose(w_asgn_file);
+	*/
+}
+
+
+/*
+ * Set all of the settings in one command:
+ * 
+ *  * init_alpha
+ *  * num_topics
+ *  * max_iter
+ *  * convergence
+ *  * em_max_iter
+ *  * em_convergence
+ *  * est_alpha
+ */
+static VALUE wrap_set_config(VALUE self, VALUE init_alpha, VALUE num_topics, VALUE max_iter, VALUE convergence, VALUE em_max_iter, VALUE em_convergence, VALUE est_alpha) {
+	INITIAL_ALPHA = NUM2DBL(init_alpha);
+	NTOPICS = NUM2INT(num_topics);
+	VAR_MAX_ITER = NUM2INT(max_iter);
+	VAR_CONVERGED = (float)NUM2DBL(convergence);
+	EM_MAX_ITER = NUM2INT(em_max_iter);
+	EM_CONVERGED = (float)NUM2DBL(em_convergence);
+	ESTIMATE_ALPHA = NUM2INT(est_alpha);
+
+	return Qtrue;
+}
+
+/*
+ * Get the maximum iterations.
+ */
+static VALUE wrap_get_max_iter(VALUE self) {
+	return rb_int_new(VAR_MAX_ITER);
+}
+
+/*
+ * Set the maximum iterations.
+ */
+static VALUE wrap_set_max_iter(VALUE self, VALUE max_iter) {
+	VAR_MAX_ITER = NUM2INT(max_iter);
+
+	return max_iter;
+}
+
+/*
+ * Get the convergence setting.
+ */
+static VALUE wrap_get_converged(VALUE self) {
+	return rb_float_new(VAR_CONVERGED);
+}
+
+/*
+ * Set the convergence setting.
+ */
+static VALUE wrap_set_converged(VALUE self, VALUE converged) {
+	VAR_CONVERGED = (float)NUM2DBL(converged);
+
+	return converged;
+}
+
+/*
+ * Get the max iterations for the EM algorithm.
+ */
+static VALUE wrap_get_em_max_iter(VALUE self) {
+	return rb_int_new(EM_MAX_ITER);
+}
+
+/*
+ * Set the max iterations for the EM algorithm.
+ */
+static VALUE wrap_set_em_max_iter(VALUE self, VALUE em_max_iter) {
+	EM_MAX_ITER = NUM2INT(em_max_iter);
+
+	return em_max_iter;
+}
+
+/*
+ * Get the convergence value for EM.
+ */
+static VALUE wrap_get_em_converged(VALUE self) {
+	return rb_float_new(EM_CONVERGED);
+}
+
+/*
+ * Set the convergence value for EM.
+ */
+static VALUE wrap_set_em_converged(VALUE self, VALUE em_converged) {
+	EM_CONVERGED = (float)NUM2DBL(em_converged);
+
+	return em_converged;
+}
+
+/*
+ * Get the initial alpha value.
+ */
+static VALUE wrap_get_initial_alpha(VALUE self) {
+	return rb_float_new(INITIAL_ALPHA);
+}
+
+/*
+ * Get the number of topics being clustered.
+ */
+static VALUE wrap_get_num_topics(VALUE self) {
+	return rb_int_new(NTOPICS);
+}
+
+/*
+ * Set the initial value of alpha.
+ */
+static VALUE wrap_set_initial_alpha(VALUE self, VALUE initial_alpha) {
+	INITIAL_ALPHA = (float)NUM2DBL(initial_alpha);
+	
+	return initial_alpha;
+}
+
+/*
+ * Set the number of topics to be clustered.
+ */
+static VALUE wrap_set_num_topics(VALUE self, VALUE ntopics) {
+	NTOPICS = NUM2INT(ntopics);
+	
+	return ntopics;
+}
+
+/*
+ * Get the estimate alpha value (fixed = 0).
+ */
+static VALUE wrap_get_estimate_alpha(VALUE self) {
+	return rb_int_new(ESTIMATE_ALPHA);
+}
+
+/*
+ * Set the estimate alpha value (fixed = 0).
+ */
+static VALUE wrap_set_estimate_alpha(VALUE self, VALUE est_alpha) {
+	ESTIMATE_ALPHA = NUM2INT(est_alpha);
+	
+	return est_alpha;
+}
+
+
+
+/*
+ * Run the EM algorithm with the loaded corpus and using the current
+ * configuration settings.  The +start+ parameter can take the following
+ * values:
+ *  * random - starting alpha are randomized
+ *  * seeded - loaded based on the corpus values
+ *  * <filename> - path to the file containing the model
+ */
+static VALUE wrap_em(VALUE self, VALUE start) {
+	if (!corpus_loaded)
+		return Qnil;
+	
+	run_quiet_em(STR2CSTR(start), last_corpus);
+
+	return Qnil;
+}
+
+
+/*
+ * Load settings from the given file.
+ */
+static VALUE wrap_load_settings(VALUE self, VALUE settings_file) {
+	read_settings(STR2CSTR(settings_file));
+
+	return Qtrue;
+}
+
+/*
+ * Load the corpus from the given file.  This will not create
+ * a +Corpus+ object that is accessible, but it will load the corpus
+ * much faster.
+ */
+static VALUE wrap_load_corpus(VALUE self, VALUE filename) {
+	if (!corpus_loaded) {
+		last_corpus = read_data(STR2CSTR(filename));
+		corpus_loaded = TRUE;
+		return Qtrue;
+	} else {
+		return Qtrue;
+	}
+}
+
+/*
+ * Set the corpus.
+ */
+static VALUE wrap_ruby_corpus(VALUE self, VALUE rcorpus) {
+	corpus* c;
+	int i = 0;
+	int j = 0;
+	
+	c = malloc(sizeof(corpus));
+	c->num_terms = NUM2INT(rb_iv_get(rcorpus, "@num_terms"));
+	c->num_docs = NUM2INT(rb_iv_get(rcorpus, "@num_docs"));
+	c->docs = (document*) malloc(sizeof(document) * c->num_docs);
+	VALUE doc_ary = rb_iv_get(rcorpus, "@documents");
+	for (i = 0; i < c->num_docs; i++) {
+		VALUE one_doc = rb_ary_entry(doc_ary, i);
+		VALUE words = rb_iv_get(one_doc, "@words");
+		VALUE counts = rb_iv_get(one_doc, "@counts");
+		
+		c->docs[i].length = NUM2INT(rb_iv_get(one_doc, "@length"));
+		c->docs[i].total = NUM2INT(rb_iv_get(one_doc, "@total"));
+		c->docs[i].words = malloc(sizeof(int) * c->docs[i].length);
+		c->docs[i].counts = malloc(sizeof(int) * c->docs[i].length);
+		for (j = 0; j < c->docs[i].length; j++) {
+			VALUE one_word = NUM2INT(rb_ary_entry(words, j));
+			VALUE one_count = NUM2INT(rb_ary_entry(counts, j));
+			c->docs[i].words[j] = one_word;
+			c->docs[i].counts[j] = one_count;
+		}
+	}
+	
+	last_corpus = c;
+	corpus_loaded = TRUE;
+	
+	rb_iv_set(self, "@corpus", rcorpus);
+	
+	return Qtrue;
+}
+
+
+/*
+ * Get the gamma values after the model has been run.
+ */
+static VALUE wrap_get_gamma(VALUE self) {
+	if (!model_loaded)
+		return Qnil;
+	
+	// last_gamma is a double[num_docs][num_topics]
+	VALUE arr;
+	int i = 0, j = 0;
+	
+	arr = rb_ary_new2(last_corpus->num_docs);
+	for (i = 0; i < last_corpus->num_docs; i++) {
+		VALUE arr2 = rb_ary_new2(last_model->num_topics);
+		for (j = 0; j < last_model->num_topics; j++) {
+			rb_ary_store(arr2, j, rb_float_new(last_gamma[i][j]));
+		}
+		rb_ary_store(arr, i, arr2);
+	}
+	
+	return arr;
+}
+
+/*
+ * Get the beta matrix after the model has been run.
+ */
+static VALUE wrap_get_model_beta(VALUE self) {
+	if (!model_loaded)
+		return Qnil;
+		
+	// beta is a double[num_topics][num_terms]
+	VALUE arr;
+	int i = 0, j = 0;
+	
+	arr = rb_ary_new2(last_model->num_topics);
+	for (i = 0; i < last_model->num_topics; i++) {
+		VALUE arr2 = rb_ary_new2(last_model->num_terms);
+		for (j = 0; j < last_model->num_terms; j++) {
+			rb_ary_store(arr2, j, rb_float_new(last_model->log_prob_w[i][j]));
+		}
+		rb_ary_store(arr, i, arr2);
+	}
+	
+	return arr;
+}
+
+
+/*
+ * Get the settings used for the model.
+ */
+static VALUE wrap_get_model_settings(VALUE self) {
+	if (!model_loaded)
+		return Qnil;
+
+	VALUE arr;
+	
+	arr = rb_ary_new();
+	rb_ary_push(arr, rb_int_new(last_model->num_topics));
+	rb_ary_push(arr, rb_int_new(last_model->num_terms));
+	rb_ary_push(arr, rb_float_new(last_model->alpha));
+	
+	return arr;		//	[num_topics, num_terms, alpha]
+}
+
+
+void Init_lda_ext() {
+	corpus_loaded = FALSE;
+	model_loaded = FALSE;
+	
+	rb_require("lda");
+	
+	/*
+	 * The Latent Dirichlet Allocation algorithm by Blei et al (2003).  Ruby wrapper based on 
+	 * lda-c code by David Blei (available at http://www.cs.princeton.edu/~blei/lda-c).
+	 */
+	rb_mLda = rb_define_module("Lda");
+	
+	/*
+	 * Class that handles most of the functionality of LDA.
+	 */
+	rb_cLda = rb_define_class_under(rb_mLda, "Lda", rb_cObject);
+	
+	/*
+	 * Class that represents a corpus.
+	 */
+	rb_cLdaCorpus = rb_define_class_under(rb_mLda, "Corpus", rb_cObject);
+	
+	/*
+	 * Class that represents a single document.
+	 */
+	rb_cLdaDocument = rb_define_class_under(rb_mLda, "Document", rb_cObject);
+	
+	
+	// method to load the corpus
+	rb_define_method(rb_cLda, "fast_load_corpus_from_file", wrap_load_corpus, 1);
+	rb_define_method(rb_cLda, "corpus=", wrap_ruby_corpus, 1);
+	
+	// method to run em
+	rb_define_method(rb_cLda, "em", wrap_em, 1);
+
+	// method to load settings from file
+	rb_define_method(rb_cLda, "load_settings", wrap_load_settings, 1);
+
+	// method to set all the config options at once
+	rb_define_method(rb_cLda, "set_config", wrap_set_config, 5);
+
+	// accessor stuff for main settings
+	rb_define_method(rb_cLda, "max_iter", wrap_get_max_iter, 0);
+	rb_define_method(rb_cLda, "max_iter=", wrap_set_max_iter, 1);
+	rb_define_method(rb_cLda, "convergence", wrap_get_converged, 0);
+	rb_define_method(rb_cLda, "convergence=", wrap_set_converged, 1);
+	rb_define_method(rb_cLda, "em_max_iter", wrap_get_em_max_iter, 0);
+	rb_define_method(rb_cLda, "em_max_iter=", wrap_set_em_max_iter, 1);
+	rb_define_method(rb_cLda, "em_convergence", wrap_get_em_converged, 0);
+	rb_define_method(rb_cLda, "em_convergence=", wrap_set_em_converged, 1);	
+	rb_define_method(rb_cLda, "init_alpha=", wrap_set_initial_alpha, 1);
+	rb_define_method(rb_cLda, "init_alpha", wrap_get_initial_alpha, 0);
+	rb_define_method(rb_cLda, "est_alpha=", wrap_set_estimate_alpha, 1);
+	rb_define_method(rb_cLda, "est_alpha", wrap_get_estimate_alpha, 0);
+	rb_define_method(rb_cLda, "num_topics", wrap_get_num_topics, 0);
+	rb_define_method(rb_cLda, "num_topics=", wrap_set_num_topics, 1);
+	
+	// retrieve model and gamma
+	rb_define_method(rb_cLda, "beta", wrap_get_model_beta, 0);
+	rb_define_method(rb_cLda, "gamma", wrap_get_gamma, 0);
+	rb_define_method(rb_cLda, "model", wrap_get_model_settings, 0);
+}
+
+#endif