ruby-sfst 0.1.0

data/ext/sfst_machine/compact.C

@@ -0,0 +1,616 @@
+/*******************************************************************/
+/*                                                                 */
+/*  FILE     compact.C                                             */
+/*  MODULE   compact                                               */
+/*  PROGRAM  SFST                                                  */
+/*  AUTHOR   Helmut Schmid, IMS, University of Stuttgart           */
+/*                                                                 */
+/*  PURPOSE  Code needed for analysing data                        */
+/*                                                                 */
+/*******************************************************************/
+
+#include <stdio.h>
+#include <math.h>
+
+#include <limits.h>
+
+#include "compact.h"
+
+using std::equal_range;
+using std::vector;
+using std::pair;
+
+const int BUFFER_SIZE=1000;
+
+
+/*******************************************************************/
+/*                                                                 */
+/*  CompactTransducer::convert                                     */
+/*                                                                 */
+/*******************************************************************/
+
+void CompactTransducer::convert( CAnalysis &cana, Analysis &ana )
+
+{
+  ana.resize(cana.size());
+  for( size_t i=0; i<cana.size(); i++ )
+    ana[i] = label[cana[i]];
+}
+
+
+/*******************************************************************/
+/*                                                                 */
+/*  CompactTransducer::analyze                                     */
+/*                                                                 */
+/*******************************************************************/
+
+void CompactTransducer::analyze(unsigned int n, vector<Character> &input,
+			       size_t ipos, CAnalysis &ca, 
+			       vector<CAnalysis> &analyses )
+{
+  // "n" is the number of the current transducer node/state
+  // "input" is the sequence of input symbols
+  // "ipos" is the input position currently analysed
+  // "ca" stores the incomplete analysis string
+  // "analyses" stores the analyses found so far
+
+  if (analyses.size() > 10000)
+    return; // limit the maximal number of analyses
+
+  // Is the input string fully analyzed and the current node a final node?
+  if (finalp[n] && ipos == input.size())
+    // store the new analysis
+    analyses.push_back(ca);
+
+  // follow the epsilon transitions
+  // first_arc[n] is the number of the first outgoing transition of node n
+  // first_arc[n+1]-1 is the number of the last outgoing transition of node n
+  // first_arc[n+1] is the number of the first outgoing transition of node n+1
+  unsigned int i;
+  for( i=first_arc[n]; 
+       i<first_arc[n+1] && label[i].upper_char() == Label::epsilon; 
+       i++)
+    {
+      ca.push_back(i);
+      analyze(target_node[i], input, ipos, ca, analyses);
+      ca.pop_back();
+    }
+
+  // follow the non-epsilon transitions
+
+  // scan the next input symbol
+  if (ipos < input.size()) {
+    // find the set of arcs with matching upper character in the sorted list
+    pair<Label*,Label*>range = 
+      equal_range(label+i, label+first_arc[n+1], Label(input[ipos]));
+    unsigned int to = (unsigned int)(range.second - label);
+
+    // follow the non-epsilon transitions
+    for( i=range.first-label; i<to; i++) {
+      ca.push_back(i);
+      analyze(target_node[i], input, ipos+1, ca, analyses);
+      ca.pop_back();
+    }
+  }
+}
+
+
+/*******************************************************************/
+/*                                                                 */
+/*  CompactTransducer::analyze_string                              */
+/*                                                                 */
+/*******************************************************************/
+
+void CompactTransducer::analyze_string( char *s, vector<CAnalysis> &analyses )
+
+{
+  // "s" input string to be analyzed
+  // "analyses" is the data structure in which the results are stored
+  // and returned
+
+  vector<Character> input;
+  alphabet.string2symseq( s, input );
+
+  analyses.clear();
+  CAnalysis ca; // data structure where the current incomplete analysis
+                   // is stored
+  analyze(0, input, 0, ca, analyses); // start the analysis
+
+  if (analyses.size() > 10000)
+    fprintf(stderr,"Warning: Only the first 10000 analyses considered for \"%s\"!\n", s);
+  
+  if (simplest_only && analyses.size() > 1)
+    disambiguate( analyses ); // select the simplest analyses
+}
+
+
+
+/*******************************************************************/
+/*                                                                 */
+/*  CompactTransducer::~CompactTransducer                          */
+/*                                                                 */
+/*******************************************************************/
+
+CompactTransducer::~CompactTransducer()
+
+{
+  delete[] finalp;
+  delete[] first_arc;
+  delete[] label;
+  delete[] target_node;
+  delete[] final_logprob;
+  delete[] arc_logprob;
+}
+
+
+/*******************************************************************/
+/*                                                                 */
+/*  CompactTransducer::CompactTransducer                           */
+/*                                                                 */
+/*******************************************************************/
+
+CompactTransducer::CompactTransducer()
+
+{
+  both_layers = false;
+  simplest_only = false;
+  number_of_nodes = 0;
+  number_of_arcs = 0;
+  finalp = NULL;
+  first_arc = NULL;
+  label = NULL;
+  target_node = NULL;
+  arc_logprob = final_logprob = (float*)NULL;
+}
+
+
+/*******************************************************************/
+/*                                                                 */
+/*  CompactTransducer::read_finalp                                 */
+/*                                                                 */
+/*******************************************************************/
+
+void CompactTransducer::read_finalp( FILE *file )
+
+{
+  int k=0;
+  unsigned char n=0;
+  for( size_t i=0; i<number_of_nodes; i++ ) {
+    if (k == 0) {
+      n = (unsigned char)fgetc(file);
+      k = 8;
+    }
+    k--;
+    if (n & (1 << k))
+      finalp[i] = 1;
+    else
+      finalp[i] = 0;
+  }
+}
+
+
+/*******************************************************************/
+/*                                                                 */
+/*  CompactTransducer::read_first_arcs                             */
+/*                                                                 */
+/*******************************************************************/
+
+void CompactTransducer::read_first_arcs( FILE *file )
+
+{
+  int k=0;
+  unsigned int n=0;
+  size_t bits=(size_t)ceil(log(number_of_arcs+1)/log(2));
+
+  for( size_t i=0; i<=number_of_nodes; i++ ) {
+    first_arc[i] = n >> (sizeof(n)*8 - bits);
+    n <<= bits;
+    k -= bits;
+    if (k < 0) {
+      read_num(&n,sizeof(n),file);
+      first_arc[i] |= n >> (sizeof(n)*8 + k);
+      n <<= -k;
+      k += sizeof(n) * 8;
+    }
+  }
+}
+
+
+/*******************************************************************/
+/*                                                                 */
+/*  CompactTransducer::read_target_nodes                           */
+/*                                                                 */
+/*******************************************************************/
+
+void CompactTransducer::read_target_nodes( FILE *file )
+
+{
+  int k=0;
+  unsigned int n=0;
+  size_t bits=(size_t)ceil(log(number_of_nodes)/log(2));
+
+  for( size_t i=0; i<number_of_arcs; i++ ) {
+    target_node[i] = n >> (sizeof(n)*8 - bits);
+    n <<= bits;
+    k -= bits;
+    if (k < 0) {
+      read_num(&n,sizeof(n),file);
+      target_node[i] |= n >> (sizeof(n)*8 + k);
+      n <<= -k;
+      k += sizeof(n) * 8;
+    }
+  }
+}
+
+
+/*******************************************************************/
+/*                                                                 */
+/*  CompactTransducer::read_labels                                 */
+/*                                                                 */
+/*******************************************************************/
+
+void CompactTransducer::read_labels( FILE *file )
+
+{
+  size_t N=0;
+  Label Num2Label[alphabet.size()];
+  for( Alphabet::const_iterator it=alphabet.begin();
+       it != alphabet.end(); it++ )
+    {
+      Label l=*it;
+      Num2Label[N++] = l;
+    }
+
+  int k=0;
+  unsigned int n=0;
+  size_t bits=(size_t)ceil(log(alphabet.size())/log(2));
+
+  for( size_t i=0; i<number_of_arcs; i++ ) {
+    unsigned int l = n >> (sizeof(n)*8 - bits);
+    n <<= bits;
+    k -= bits;
+    if (k < 0) {
+      read_num(&n,sizeof(n),file);
+      l |= n >> (sizeof(n)*8 + k);
+      n <<= -k;
+      k += sizeof(n) * 8;
+    }
+    label[i] = Num2Label[l];
+  }
+}
+
+
+/*******************************************************************/
+/*                                                                 */
+/*  CompactTransducer::read_probs                                  */
+/*                                                                 */
+/*******************************************************************/
+
+void CompactTransducer::read_probs( FILE *file )
+
+{
+  size_t n,m;
+  fread(&n, sizeof(n), 1, file);
+  if (fread(&m, sizeof(n), 1, file) != 1 ||
+      n != node_count() || m != arc_count())
+    {
+      fprintf(stderr,"Error: incompatible probability file!\n");
+      exit(1);
+    }
+  final_logprob = new float[n];
+  arc_logprob = new float[m];
+  fread(final_logprob, sizeof(float), n, file);
+  if (fread(arc_logprob, sizeof(float), n, file) != n) {
+    fprintf(stderr,"Error: in probability file!\n");
+    exit(1);
+  }
+}
+
+
+/*******************************************************************/
+/*                                                                 */
+/*  CompactTransducer::CompactTransducer                           */
+/*                                                                 */
+/*******************************************************************/
+
+CompactTransducer::CompactTransducer( FILE *file, FILE *pfile )
+
+{
+  both_layers = false;
+  simplest_only = false;
+
+  if (fgetc(file) != 'c')
+      throw "Error: wrong file format (not a compact transducer)\n";
+
+  alphabet.read(file);
+
+  read_num(&number_of_nodes,sizeof(number_of_nodes),file);
+  read_num(&number_of_arcs,sizeof(number_of_arcs),file);
+
+  if (!ferror(file)) {
+    // memory allocation
+    finalp = new char[number_of_nodes];
+    first_arc = new unsigned[number_of_nodes+1];
+    label = new Label[number_of_arcs];
+    target_node = new unsigned[number_of_arcs];
+    
+    // reading the data
+    read_finalp(file);
+    read_first_arcs(file);
+    read_labels(file);
+    read_target_nodes(file);
+  }
+
+  if (pfile == NULL)
+    arc_logprob = final_logprob = (float*)NULL;
+  else
+    read_probs(pfile);
+}
+
+
+/*******************************************************************/
+/*                                                                 */
+/*  CompactTransducer::longest_match2                              */
+/*                                                                 */
+/*******************************************************************/
+
+void CompactTransducer::longest_match2(unsigned int n, char *string, int l, 
+				      CAnalysis &ca, int &bl, CAnalysis &ba)
+{
+  // n: transducer state
+  // string: rest string
+  // l: length of current analysis
+  // bl: length of the currently longest match
+  // ca: current analysis
+  // ba: best analysis
+
+  if (finalp[n] && l > bl) {
+    // store the new analysis
+    bl = l;
+    ba = ca; // copy the arc vector
+  }
+
+  // follow the epsilon transitions
+  unsigned int i;
+  for( i=first_arc[n]; 
+       i<first_arc[n+1] && label[i].upper_char() == Label::epsilon; 
+       i++)
+    {
+      ca.push_back(i);
+      longest_match2(target_node[i], string, l, ca, bl, ba);
+      ca.pop_back();
+    }
+
+  // follow the non-epsilon transitions
+  char *end=string;
+  int c=alphabet.next_code(end, false);
+  l += end-string;
+  if (c != EOF) {
+    // find the set of arcs with matching upper character in the sort list
+    pair<Label*,Label*>range = 
+      equal_range(label+i, label+first_arc[n+1], Label((Character)c));
+    unsigned int to = (unsigned int)(range.second - label);
+    for( i=range.first-label; i<to; i++) {
+      ca.push_back(i);
+      longest_match2(target_node[i], end, l, ca, bl, ba);
+      ca.pop_back();
+    }
+  }
+}
+
+
+/*******************************************************************/
+/*                                                                 */
+/*  CompactTransducer::print_analysis                              */
+/*                                                                 */
+/*******************************************************************/
+
+char *CompactTransducer::print_analysis( CAnalysis &cana )
+
+{
+  Analysis ana;
+  convert(cana, ana);
+  return alphabet.print_analysis( ana, both_layers );
+}
+
+
+/*******************************************************************/
+/*                                                                 */
+/*  CompactTransducer::longest_match                               */
+/*                                                                 */
+/*******************************************************************/
+
+const char *CompactTransducer::longest_match( char* &string )
+
+{
+  vector<char> analysis;
+  CAnalysis ca, ba;
+  int l=0;
+  longest_match2(0, string, 0, ca, l, ba);
+
+  // no match? return the next character
+  if (ba.size() == 0) {
+    int c=alphabet.next_code(string, false);
+    return alphabet.code2symbol(c);
+  }
+
+  string += l;
+  return print_analysis( ba );
+}
+
+
+/*******************************************************************/
+/*                                                                 */
+/*  CompactTransducer::disambiguate                                */
+/*                                                                 */
+/*******************************************************************/
+
+void CompactTransducer::disambiguate( vector<CAnalysis> &analyses )
+
+{
+  // compute the scores
+  int bestscore=INT_MIN;
+  vector<int> score;
+  Analysis ana;
+
+  for( size_t i=0; i<analyses.size(); i++ ) {
+    convert(analyses[i], ana);
+    score.push_back(alphabet.compute_score(ana));
+    if (bestscore < score[i])
+      bestscore = score[i];
+  }
+
+  // delete suboptimal analyses
+  size_t k=0;
+  for( size_t i=0; i<analyses.size(); i++ )
+    if (score[i] == bestscore)
+      analyses[k++] = analyses[i];
+  analyses.resize(k);
+}
+
+
+/*******************************************************************/
+/*                                                                 */
+/*  CompactTransducer::train2                                      */
+/*                                                                 */
+/*******************************************************************/
+
+bool CompactTransducer::train2( char *s, vector<double> &arcfreq, 
+			       vector<double> &finalfreq )
+{
+  vector<CAnalysis> analyses;
+  vector<Label> input;
+  alphabet.string2labelseq( s, input );
+
+  CAnalysis ca; // data structure where the analysis is stored
+  unsigned int n=0;
+  bool failure=false;
+  for( size_t i=0; i<input.size(); i++ ) {
+    failure = true;
+    for( unsigned int k=first_arc[n]; k<first_arc[n+1]; k++) {
+      if (label[k] == input[i]) {
+	ca.push_back(k);
+	n = target_node[k];
+	failure = false;
+	break;
+      }
+    }
+    if (failure)
+      break;
+  }
+  if (failure || !finalp[n]) {
+    fprintf(stderr,"Warning: The following input is not covered:\n%s\n", s);
+    return false;
+  }
+
+  for( size_t k=0; k<ca.size(); k++ )
+    arcfreq[ca[k]]++;
+  finalfreq[target_node[ca.back()]]++;
+
+  return true;
+}
+
+
+/*******************************************************************/
+/*                                                                 */
+/*  CompactTransducer::train                                       */
+/*                                                                 */
+/*******************************************************************/
+
+bool CompactTransducer::train( char *s, vector<double> &arcfreq, 
+			      vector<double> &finalfreq )
+{
+  vector<CAnalysis> analyses;
+  vector<Character> input;
+  alphabet.string2symseq( s, input );
+
+  CAnalysis ca; // data structure where the current incomplete analysis
+                   // is stored
+  analyze(0, input, 0, ca, analyses); // start the analysis
+
+  if (analyses.size() > 10000)
+    return true; // ignore inputs with more than 10000 analyses
+  else if (analyses.size() == 0)
+    return false;
+  
+  if (simplest_only && analyses.size() > 1)
+    disambiguate( analyses ); // select the simplest analyses
+
+  if (analyses.size() > 0) {
+    double incr = 1.0 / analyses.size();
+    CAnalysis arcs;
+
+    for( size_t i=0; i<analyses.size(); i++ ) {
+      CAnalysis &arcs=analyses[i];
+      for( size_t k=0; k<arcs.size(); k++ )
+	arcfreq[arcs[k]] += incr;
+      finalfreq[target_node[arcs.back()]] += incr;
+    }
+  }
+  return true;
+}
+
+
+/*******************************************************************/
+/*                                                                 */
+/*  CompactTransducer::estimate_probs                              */
+/*                                                                 */
+/*******************************************************************/
+
+void CompactTransducer::estimate_probs( vector<double> &arcfreq, 
+				       vector<double> &finalfreq )
+{
+  // turn frequencies into probabilities
+  for( size_t n=0; n<finalfreq.size(); n++ ) {
+    double sum = finalfreq[n];
+    for( size_t a=first_arc[n]; a<first_arc[n+1]; a++ )
+      sum += arcfreq[a];
+    if (sum == 0.0)
+      sum = 1.0;
+    finalfreq[n] = finalfreq[n] / sum;
+    for( size_t a=first_arc[n]; a<first_arc[n+1]; a++ )
+      arcfreq[a] = arcfreq[a] / sum;
+  }
+}
+
+
+
+/*******************************************************************/
+/*                                                                 */
+/*  CompactTransducer::compute_probs                               */
+/*                                                                 */
+/*******************************************************************/
+
+void CompactTransducer::compute_probs( vector<CAnalysis> &analyses, 
+				      vector<double> &prob )
+{
+  prob.resize(analyses.size());
+  double sum=0.0;
+  for( size_t i=0; i<analyses.size(); i++ ) {
+    CAnalysis &a=analyses[i];
+
+    // compute the probability
+    double logprob=0.0;
+    for( size_t k=0; k<a.size(); k++ )
+      logprob += arc_logprob[a[k]];
+    logprob += final_logprob[target_node[a.back()]];
+    prob[i] = exp(logprob);
+    sum += prob[i];
+  }
+
+  // sort the analyses
+  vector<CAnalysis> oldanalyses(analyses);
+  vector<double> oldprob(prob);
+  for( size_t i=0; i<analyses.size(); i++ ) {
+    prob[i] = -1.0;
+    int n=0;
+    for( size_t k=0; k<oldanalyses.size(); k++ )
+      if (prob[i] < oldprob[k]) {
+	prob[i] = oldprob[k];
+	n = k;
+      }
+    analyses[i] = oldanalyses[n];
+    oldprob[n] = -1.0;
+    prob[i] /= sum; // normalization
+  }
+}

data/ext/sfst_machine/compact.h

@@ -0,0 +1,98 @@
+/*******************************************************************/
+/*                                                                 */
+/*  FILE     compact.h                                             */
+/*  MODULE   compact                                               */
+/*  PROGRAM  SFST                                                  */
+/*  AUTHOR   Helmut Schmid, IMS, University of Stuttgart           */
+/*                                                                 */
+/*  PURPOSE  finite state tools                                    */
+/*                                                                 */
+/*******************************************************************/
+
+#ifndef _COMPACT_H_
+#define _COMPACT_H_
+
+#include "alphabet.h"
+
+#include <vector>
+
+typedef std::vector<unsigned int> CAnalysis;
+    
+class CompactTransducer {
+    
+protected:
+
+  // the following data structures are used to store the nodes
+
+  unsigned int number_of_nodes; // number of nodes in the transducer
+  char *finalp;  // finalp[i] is 1 if node i is final and 0 otherwise
+  unsigned int *first_arc;  // first_arc[i] is the number of the first
+			    // arc outgoing from node i
+
+  // the following data structures are used to store the transition arcs
+
+  unsigned int number_of_arcs; // total number of arcs in the transducer
+  Label *label;              // the label (character pair) of arc i
+  unsigned int *target_node; // target node of arc i
+
+  // the following data structures are used to store the stochastic parameters
+  float *final_logprob;
+  float *arc_logprob;
+
+  // functions needed to read the transducer from a file
+
+  void read_finalp( FILE *file );
+  void read_first_arcs( FILE *file );
+  void read_target_nodes( FILE *file );
+  void read_labels( FILE *file );
+  void read_probs( FILE *file );
+
+  // functions needed to analyze data with the transducer
+
+  void analyze( unsigned int n, std::vector<Character> &ch, size_t ipos,
+		CAnalysis&, std::vector<CAnalysis>&);
+
+  // function selecting the simplest morphological analysis
+
+  int compute_score( CAnalysis &ana );
+  void disambiguate( std::vector<CAnalysis> &analyses );
+
+  // functions for longest-match analysis of input data
+
+  void longest_match2(unsigned int, char*, int, CAnalysis&, int&, CAnalysis&);
+
+  void convert( CAnalysis &cana, Analysis &ana );
+  
+public:
+  size_t node_count() { return number_of_nodes; };
+  size_t arc_count() { return number_of_arcs; };
+
+  bool both_layers;   // print surface and analysis symbols
+  bool simplest_only; // print only the simplest analyses
+
+  Alphabet alphabet;  // data structure which maps symbols to numeric codes
+  CompactTransducer(); // dummy constructor
+  CompactTransducer( FILE*, FILE *pfile=NULL ); // reads a (stochastic) transducer
+  ~CompactTransducer();  // destroys a transducer
+  
+  // the analysis function returns the set of analyses for the string "s"
+  // in the argument "analyses"
+  void analyze_string( char *s, std::vector<CAnalysis > &analyses );
+
+  void compute_probs( std::vector<CAnalysis> &analyses, std::vector<double> &prob );
+  char *print_analysis( CAnalysis &ana );
+  
+  // longest-match analysis
+  const char *longest_match( char*& );
+
+  // EM training
+  bool train2( char *s, std::vector<double> &arcfreq, std::vector<double> &finalfreq );
+  bool train( char *s, std::vector<double> &arcfreq, std::vector<double> &finalfreq );
+  void estimate_probs( std::vector<double> &arcfreq, std::vector<double> &finalfreq );
+
+  // robust analysis
+  float robust_analyze_string( char *string, std::vector<CAnalysis> &analyses,
+			       float ErrorsAllowed );
+};
+
+#endif