gosu 0.7.16 → 0.7.17

data/dependencies/libvorbis/vq/latticehint.c

@@ -1,430 +0,0 @@
-/********************************************************************
- *                                                                  *
- * THIS FILE IS PART OF THE OggVorbis SOFTWARE CODEC SOURCE CODE.   *
- * USE, DISTRIBUTION AND REPRODUCTION OF THIS LIBRARY SOURCE IS     *
- * GOVERNED BY A BSD-STYLE SOURCE LICENSE INCLUDED WITH THIS SOURCE *
- * IN 'COPYING'. PLEASE READ THESE TERMS BEFORE DISTRIBUTING.       *
- *                                                                  *
- * THE OggVorbis SOURCE CODE IS (C) COPYRIGHT 1994-2001             *
- * by the Xiph.Org Foundation http://www.xiph.org/                  *
- *                                                                  *
- ********************************************************************
-
- function: utility main for building thresh/pigeonhole encode hints
- last mod: $Id: latticehint.c 13293 2007-07-24 00:09:47Z xiphmont $
-
- ********************************************************************/
-
-#include <stdlib.h>
-#include <stdio.h>
-#include <math.h>
-#include <string.h>
-#include <errno.h>
-#include "../lib/scales.h"
-#include "bookutil.h"
-#include "vqgen.h"
-#include "vqsplit.h"
-
-/* The purpose of this util is to build encode hints for lattice
-   codebooks so that brute forcing each codebook entry isn't needed.
-   Threshhold hints are for books in which each scalar in the vector
-   is independant (eg, residue) and pigeonhole lookups provide a
-   minimum error fit for words where the scalars are interdependant
-   (each affecting the fit of the next in sequence) as in an LSP
-   sequential book (or can be used along with a sparse threshhold map,
-   like a splitting tree that need not be trained) 
-
-   If the input book is non-sequential, a threshhold hint is built.
-   If the input book is sequential, a pigeonholing hist is built.
-   If the book is sparse, a pigeonholing hint is built, possibly in addition
-     to the threshhold hint 
-
-   command line:
-   latticehint book.vqh [threshlist]
-
-   latticehint produces book.vqh on stdout */
-
-static int longsort(const void *a, const void *b){
-  return(**((long **)a)-**((long **)b));
-}
-
-static int addtosearch(int entry,long **tempstack,long *tempcount,int add){
-  long *ptr=tempstack[entry];
-  long i=tempcount[entry];
-
-  if(ptr){
-    while(i--)
-      if(*ptr++==add)return(0);
-    tempstack[entry]=_ogg_realloc(tempstack[entry],
-			     (tempcount[entry]+1)*sizeof(long));
-  }else{
-    tempstack[entry]=_ogg_malloc(sizeof(long));
-  }
-
-  tempstack[entry][tempcount[entry]++]=add;
-  return(1);
-}
-
-static void setvals(int dim,encode_aux_pigeonhole *p,
-		    long *temptrack,float *tempmin,float *tempmax,
-		    int seqp){
-  int i;
-  float last=0.f;
-  for(i=0;i<dim;i++){
-    tempmin[i]=(temptrack[i])*p->del+p->min+last;
-    tempmax[i]=tempmin[i]+p->del;
-    if(seqp)last=tempmin[i];
-  }
-}
-
-/* note that things are currently set up such that input fits that
-   quantize outside the pigeonmap are dropped and brute-forced.  So we
-   can ignore the <0 and >=n boundary cases in min/max error */
-
-static float minerror(int dim,float *a,encode_aux_pigeonhole *p,
-		       long *temptrack,float *tempmin,float *tempmax){
-  int i;
-  float err=0.f;
-  for(i=0;i<dim;i++){
-    float eval=0.f;
-    if(a[i]<tempmin[i]){
-      eval=tempmin[i]-a[i];
-    }else if(a[i]>tempmax[i]){
-      eval=a[i]-tempmax[i];
-    }
-    err+=eval*eval;
-  }
-  return(err);
-}
-
-static float maxerror(int dim,float *a,encode_aux_pigeonhole *p,
-		       long *temptrack,float *tempmin,float *tempmax){
-  int i;
-  float err=0.f,eval;
-  for(i=0;i<dim;i++){
-    if(a[i]<tempmin[i]){
-      eval=tempmax[i]-a[i];
-    }else if(a[i]>tempmax[i]){
-      eval=a[i]-tempmin[i];
-    }else{
-      float t1=a[i]-tempmin[i];
-      eval=tempmax[i]-a[i];
-      if(t1>eval)eval=t1;
-    }
-    err+=eval*eval;
-  }
-  return(err);
-}
-
-int main(int argc,char *argv[]){
-  codebook *b;
-  static_codebook *c;
-  int entries=-1,dim=-1;
-  float min,del;
-  char *name;
-  long i,j;
-  float *suggestions;
-  int suggcount=0;
-
-  if(argv[1]==NULL){
-    fprintf(stderr,"Need a lattice book on the command line.\n");
-    exit(1);
-  }
-
-  {
-    char *ptr;
-    char *filename=strdup(argv[1]);
-
-    b=codebook_load(filename);
-    c=(static_codebook *)(b->c);
-    
-    ptr=strrchr(filename,'.');
-    if(ptr){
-      *ptr='\0';
-      name=strdup(filename);
-    }else{
-      name=strdup(filename);
-    }
-  }
-
-  if(c->maptype!=1){
-    fprintf(stderr,"Provided book is not a latticebook.\n");
-    exit(1);
-  }
-
-  entries=b->entries;
-  dim=b->dim;
-  min=_float32_unpack(c->q_min);
-  del=_float32_unpack(c->q_delta);
-
-  /* Do we want to gen a threshold hint? */
-  if(c->q_sequencep==0){
-    /* yes. Discard any preexisting threshhold hint */
-    long quantvals=_book_maptype1_quantvals(c);
-    long **quantsort=alloca(quantvals*sizeof(long *));
-    encode_aux_threshmatch *t=_ogg_calloc(1,sizeof(encode_aux_threshmatch));
-    c->thresh_tree=t;
-
-    fprintf(stderr,"Adding threshold hint to %s...\n",name);
-
-    /* partial/complete suggestions */
-    if(argv[2]){
-      char *ptr=strdup(argv[2]);
-      suggestions=alloca(sizeof(float)*quantvals);
-			 
-      for(suggcount=0;ptr && suggcount<quantvals;suggcount++){
-	char *ptr2=strchr(ptr,',');
-	if(ptr2)*ptr2++='\0';
-	suggestions[suggcount]=atof(ptr);
-	ptr=ptr2;
-      }
-    }
-
-    /* simplest possible threshold hint only */
-    t->quantthresh=_ogg_calloc(quantvals-1,sizeof(float));
-    t->quantmap=_ogg_calloc(quantvals,sizeof(int));
-    t->threshvals=quantvals;
-    t->quantvals=quantvals;
-
-    /* the quantvals may not be in order; sort em first */
-    for(i=0;i<quantvals;i++)quantsort[i]=c->quantlist+i;
-    qsort(quantsort,quantvals,sizeof(long *),longsort);
-
-    /* ok, gen the map and thresholds */
-    for(i=0;i<quantvals;i++)t->quantmap[i]=quantsort[i]-c->quantlist;
-    for(i=0;i<quantvals-1;i++){
-      float v1=*(quantsort[i])*del+min;
-      float v2=*(quantsort[i+1])*del+min;
-      
-      for(j=0;j<suggcount;j++)
-	if(v1<suggestions[j] && suggestions[j]<v2){
-	  t->quantthresh[i]=suggestions[j];
-	  break;
-	}
-      
-      if(j==suggcount){
-	t->quantthresh[i]=(v1+v2)*.5;
-      }
-    }
-  }
-
-  /* Do we want to gen a pigeonhole hint? */
-#if 0
-  for(i=0;i<entries;i++)if(c->lengthlist[i]==0)break;
-  if(c->q_sequencep || i<entries){
-    long **tempstack;
-    long *tempcount;
-    long *temptrack;
-    float *tempmin;
-    float *tempmax;
-    long totalstack=0;
-    long pigeons;
-    long subpigeons;
-    long quantvals=_book_maptype1_quantvals(c);
-    int changep=1,factor;
-
-    encode_aux_pigeonhole *p=_ogg_calloc(1,sizeof(encode_aux_pigeonhole));
-    c->pigeon_tree=p;
-
-    fprintf(stderr,"Adding pigeonhole hint to %s...\n",name);
-    
-    /* the idea is that we quantize uniformly, even in a nonuniform
-       lattice, so that quantization of one scalar has a predictable
-       result on the next sequential scalar in a greedy matching
-       algorithm.  We generate a lookup based on the quantization of
-       the vector (pigeonmap groups quantized entries together) and
-       list the entries that could possible be the best fit for any
-       given member of that pigeonhole.  The encode process then has a
-       much smaller list to brute force */
-
-    /* find our pigeonhole-specific quantization values, fill in the
-       quant value->pigeonhole map */
-    factor=3;
-    p->del=del;
-    p->min=min;
-    p->quantvals=quantvals;
-    {
-      int max=0;
-      for(i=0;i<quantvals;i++)if(max<c->quantlist[i])max=c->quantlist[i];
-      p->mapentries=max;
-    }
-    p->pigeonmap=_ogg_malloc(p->mapentries*sizeof(long));
-    p->quantvals=(quantvals+factor-1)/factor;
-
-    /* pigeonhole roughly on the boundaries of the quantvals; the
-       exact pigeonhole grouping is an optimization issue, not a
-       correctness issue */
-    for(i=0;i<p->mapentries;i++){
-      float thisval=del*i+min; /* middle of the quant zone */
-      int quant=0;
-      float err=fabs(c->quantlist[0]*del+min-thisval);
-      for(j=1;j<quantvals;j++){
-	float thiserr=fabs(c->quantlist[j]*del+min-thisval);
-	if(thiserr<err){
-	  quant=j/factor;
-	  err=thiserr;
-	}
-      }
-      p->pigeonmap[i]=quant;
-    }
-    
-    /* pigeonmap complete.  Now do the grungy business of finding the
-    entries that could possibly be the best fit for a value appearing
-    in the pigeonhole. The trick that allows the below to work is the
-    uniform quantization; even though the scalars may be 'sequential'
-    (each a delta from the last), the uniform quantization means that
-    the error variance is *not* dependant.  Given a pigeonhole and an
-    entry, we can find the minimum and maximum possible errors
-    (relative to the entry) for any point that could appear in the
-    pigeonhole */
-    
-    /* must iterate over both pigeonholes and entries */
-    /* temporarily (in order to avoid thinking hard), we grow each
-       pigeonhole seperately, the build a stack of 'em later */
-    pigeons=1;
-    subpigeons=1;
-    for(i=0;i<dim;i++)subpigeons*=p->mapentries;
-    for(i=0;i<dim;i++)pigeons*=p->quantvals;
-    temptrack=_ogg_calloc(dim,sizeof(long));
-    tempmin=_ogg_calloc(dim,sizeof(float));
-    tempmax=_ogg_calloc(dim,sizeof(float));
-    tempstack=_ogg_calloc(pigeons,sizeof(long *));
-    tempcount=_ogg_calloc(pigeons,sizeof(long));
-
-    while(1){
-      float errorpost=-1;
-      char buffer[80];
-
-      /* map our current pigeonhole to a 'big pigeonhole' so we know
-         what list we're after */
-      int entry=0;
-      for(i=dim-1;i>=0;i--)entry=entry*p->quantvals+p->pigeonmap[temptrack[i]];
-      setvals(dim,p,temptrack,tempmin,tempmax,c->q_sequencep);
-      sprintf(buffer,"Building pigeonhole search list [%ld]...",totalstack);
-
-
-      /* Search all entries to find the one with the minimum possible
-         maximum error.  Record that error */
-      for(i=0;i<entries;i++){
-	if(c->lengthlist[i]>0){
-	  float this=maxerror(dim,b->valuelist+i*dim,p,
-			       temptrack,tempmin,tempmax);
-	  if(errorpost==-1 || this<errorpost)errorpost=this;
-	  spinnit(buffer,subpigeons);
-	}
-      }
-
-      /* Our search list will contain all entries with a minimum
-         possible error <= our errorpost */
-      for(i=0;i<entries;i++)
-	if(c->lengthlist[i]>0){
-	  spinnit(buffer,subpigeons);
-	  if(minerror(dim,b->valuelist+i*dim,p,
-		      temptrack,tempmin,tempmax)<errorpost)
-	    totalstack+=addtosearch(entry,tempstack,tempcount,i);
-	}
-
-      for(i=0;i<dim;i++){
-	temptrack[i]++;
-	if(temptrack[i]<p->mapentries)break;
-	temptrack[i]=0;
-      }
-      if(i==dim)break;
-      subpigeons--;
-    }
-
-    fprintf(stderr,"\r                                                     "
-	    "\rTotal search list size (all entries): %ld\n",totalstack);
-
-    /* pare the index of lists for improbable quantizations (where
-       improbable is determined by c->lengthlist; we assume that
-       pigeonholing is in sync with the codeword cells, which it is */
-    /*for(i=0;i<entries;i++){
-      float probability= 1.f/(1<<c->lengthlist[i]);
-      if(c->lengthlist[i]==0 || probability*entries<cutoff){
-	totalstack-=tempcount[i];
-	tempcount[i]=0;
-      }
-      }*/
-
-    /* pare the list of shortlists; merge contained and similar lists
-       together */
-    p->fitmap=_ogg_malloc(pigeons*sizeof(long));
-    for(i=0;i<pigeons;i++)p->fitmap[i]=-1;
-    while(changep){
-      char buffer[80];
-      changep=0;
-
-      for(i=0;i<pigeons;i++){
-	if(p->fitmap[i]<0 && tempcount[i]){
-	  for(j=i+1;j<pigeons;j++){
-	    if(p->fitmap[j]<0 && tempcount[j]){
-	      /* is one list a superset, or are they sufficiently similar? */
-	      int amiss=0,bmiss=0,ii,jj;
-	      for(ii=0;ii<tempcount[i];ii++){
-		for(jj=0;jj<tempcount[j];jj++)
-		  if(tempstack[i][ii]==tempstack[j][jj])break;
-		if(jj==tempcount[j])amiss++;
-	      }
-	      for(jj=0;jj<tempcount[j];jj++){
-		for(ii=0;ii<tempcount[i];ii++)
-		  if(tempstack[i][ii]==tempstack[j][jj])break;
-		if(ii==tempcount[i])bmiss++;
-	      }
-	      if(amiss==0 ||
-		 bmiss==0 ||
-		 (amiss*2<tempcount[i] && bmiss*2<tempcount[j] &&
-		 tempcount[i]+bmiss<entries/30)){
-
-		/*superset/similar  Add all of one to the other. */
-		for(jj=0;jj<tempcount[j];jj++)
-		  totalstack+=addtosearch(i,tempstack,tempcount,
-					  tempstack[j][jj]);
-		totalstack-=tempcount[j];
-		p->fitmap[j]=i;
-		changep=1;
-	      }
-	    }
-	  }
-	  sprintf(buffer,"Consolidating [%ld total, %s]... ",totalstack,
-		  changep?"reit":"nochange");
-	  spinnit(buffer,pigeons-i);
-	}
-      }
-    }
-
-    /* repack the temp stack in final form */
-    fprintf(stderr,"\r                                                       "
-	    "\rFinal total list size: %ld\n",totalstack);
-    
-
-    p->fittotal=totalstack;
-    p->fitlist=_ogg_malloc((totalstack+1)*sizeof(long));
-    p->fitlength=_ogg_malloc(pigeons*sizeof(long));
-    {
-      long usage=0;
-      for(i=0;i<pigeons;i++){
-	if(p->fitmap[i]==-1){
-	  if(tempcount[i])
-	    memcpy(p->fitlist+usage,tempstack[i],tempcount[i]*sizeof(long));
-	  p->fitmap[i]=usage;
-	  p->fitlength[i]=tempcount[i];
-	  usage+=tempcount[i];
-	  if(usage>totalstack){
-	    fprintf(stderr,"Internal error; usage>totalstack\n");
-	    exit(1);
-	  }
-	}else{
-	  p->fitlength[i]=p->fitlength[p->fitmap[i]];
-	  p->fitmap[i]=p->fitmap[p->fitmap[i]];
-	}
-      }
-    }
-  }
-#endif
-
-  write_codebook(stdout,name,c); 
-  fprintf(stderr,"\r                                                     "
-	  "\nDone.\n");
-  exit(0);
-}

data/dependencies/libvorbis/vq/latticepare.c

@@ -1,595 +0,0 @@
-/********************************************************************
- *                                                                  *
- * THIS FILE IS PART OF THE OggVorbis SOFTWARE CODEC SOURCE CODE.   *
- * USE, DISTRIBUTION AND REPRODUCTION OF THIS LIBRARY SOURCE IS     *
- * GOVERNED BY A BSD-STYLE SOURCE LICENSE INCLUDED WITH THIS SOURCE *
- * IN 'COPYING'. PLEASE READ THESE TERMS BEFORE DISTRIBUTING.       *
- *                                                                  *
- * THE OggVorbis SOURCE CODE IS (C) COPYRIGHT 1994-2001             *
- * by the Xiph.Org Foundation http://www.xiph.org/                  *
- *                                                                  *
- ********************************************************************
-
- function: utility for paring low hit count cells from lattice codebook
- last mod: $Id: latticepare.c 13293 2007-07-24 00:09:47Z xiphmont $
-
- ********************************************************************/
-
-#include <stdlib.h>
-#include <stdio.h>
-#include <math.h>
-#include <string.h>
-#include <errno.h>
-#include "../lib/scales.h"
-#include "bookutil.h"
-#include "vqgen.h"
-#include "vqsplit.h"
-#include "../lib/os.h"
-
-/* Lattice codebooks have two strengths: important fetaures that are
-   poorly modelled by global error minimization training (eg, strong
-   peaks) are not neglected 2) compact quantized representation.
-
-   A fully populated lattice codebook, however, swings point 1 too far
-   in the opposite direction; rare features need not be modelled quite
-   so religiously and as such, we waste bits unless we eliminate the
-   least common cells.  The codebook rep supports unused cells, so we
-   need to tag such cells and build an auxiliary (non-thresh) search
-   mechanism to find the proper match quickly */
-
-/* two basic steps; first is pare the cell for which dispersal creates
-   the least additional error.  This will naturally choose
-   low-population cells and cells that have not taken on points from
-   neighboring paring (but does not result in the lattice collapsing
-   inward and leaving low population ares totally unmodelled).  After
-   paring has removed the desired number of cells, we need to build an
-   auxiliary search for each culled point */
-
-/* Although lattice books (due to threshhold-based matching) do not
-   actually use error to make cell selections (in fact, it need not
-   bear any relation), the 'secondbest' entry finder here is in fact
-   error/distance based, so latticepare is only useful on such books */
-
-/* command line:
-   latticepare latticebook.vqh input_data.vqd <target_cells>
-
-   produces a new output book on stdout 
-*/
-
-static float _dist(int el,float *a, float *b){
-  int i;
-  float acc=0.f;
-  for(i=0;i<el;i++){
-    float val=(a[i]-b[i]);
-    acc+=val*val;
-  }
-  return(acc);
-}
-
-static float *pointlist;
-static long points=0;
-
-void add_vector(codebook *b,float *vec,long n){
-  int dim=b->dim,i,j;
-  int step=n/dim;
-  for(i=0;i<step;i++){
-    for(j=i;j<n;j+=step){
-      pointlist[points++]=vec[j];
-    }
-  }
-}
-
-static int bestm(codebook *b,float *vec){
-  encode_aux_threshmatch *tt=b->c->thresh_tree;
-  int dim=b->dim;
-  int i,k,o;
-  int best=0;
-  
-  /* what would be the closest match if the codebook was fully
-     populated? */
-  
-  for(k=0,o=dim-1;k<dim;k++,o--){
-    int i;
-    for(i=0;i<tt->threshvals-1;i++)
-      if(vec[o]<tt->quantthresh[i])break;
-    best=(best*tt->quantvals)+tt->quantmap[i];
-  }
-  return(best);
-}
-
-static int closest(codebook *b,float *vec,int current){
-  encode_aux_threshmatch *tt=b->c->thresh_tree;
-  int dim=b->dim;
-  int i,k,o;
-
-  float bestmetric=0;
-  int bestentry=-1;
-  int best=bestm(b,vec);
-
-  if(current<0 && b->c->lengthlist[best]>0)return best;
-
-  for(i=0;i<b->entries;i++){
-    if(b->c->lengthlist[i]>0 && i!=best && i!=current){
-      float thismetric=_dist(dim, vec, b->valuelist+i*dim);
-      if(bestentry==-1 || thismetric<bestmetric){
-	bestentry=i;
-	bestmetric=thismetric;
-      }
-    }
-  }
-
-  return(bestentry);
-}
-
-static float _heuristic(codebook *b,float *ppt,int secondbest){
-  float *secondcell=b->valuelist+secondbest*b->dim;
-  int best=bestm(b,ppt);
-  float *firstcell=b->valuelist+best*b->dim;
-  float error=_dist(b->dim,firstcell,secondcell);
-  float *zero=alloca(b->dim*sizeof(float));
-  float fromzero;
-  
-  memset(zero,0,b->dim*sizeof(float));
-  fromzero=sqrt(_dist(b->dim,firstcell,zero));
-
-  return(error/fromzero);
-}
-
-static int longsort(const void *a, const void *b){
-  return **(long **)b-**(long **)a;
-}
-
-void usage(void){
-  fprintf(stderr,"Ogg/Vorbis lattice codebook paring utility\n\n"
-	  "usage: latticepare book.vqh data.vqd <target_cells> <protected_cells> base\n"
-	  "where <target_cells> is the desired number of final cells (or -1\n"
-	  "                     for no change)\n"
-	  "      <protected_cells> is the number of highest-hit count cells\n"
-	  "                     to protect from dispersal\n"
-	  "      base is the base name (not including .vqh) of the new\n"
-	  "                     book\n\n");
-  exit(1);
-}
-
-int main(int argc,char *argv[]){
-  char *basename;
-  codebook *b=NULL;
-  int entries=0;
-  int dim=0;
-  long i,j,target=-1,protect=-1;
-  FILE *out=NULL;
-
-  int argnum=0;
-
-  argv++;
-  if(*argv==NULL){
-    usage();
-    exit(1);
-  }
-
-  while(*argv){
-    if(*argv[0]=='-'){
-
-      argv++;
-	
-    }else{
-      switch (argnum++){
-      case 0:case 1:
-	{
-	  /* yes, this is evil.  However, it's very convenient to parse file
-	     extentions */
-	  
-	  /* input file.  What kind? */
-	  char *dot;
-	  char *ext=NULL;
-	  char *name=strdup(*argv++);
-	  dot=strrchr(name,'.');
-	  if(dot)
-	    ext=dot+1;
-	  else{
-	    ext="";
-	    
-	  }
-	  
-	  
-	  /* codebook */
-	  if(!strcmp(ext,"vqh")){
-	    
-	    basename=strrchr(name,'/');
-	    if(basename)
-	      basename=strdup(basename)+1;
-	    else
-	      basename=strdup(name);
-	    dot=strrchr(basename,'.');
-	    if(dot)*dot='\0';
-	    
-	    b=codebook_load(name);
-	    dim=b->dim;
-	    entries=b->entries;
-	  }
-	  
-	  /* data file; we do actually need to suck it into memory */
-	  /* we're dealing with just one book, so we can de-interleave */ 
-	  if(!strcmp(ext,"vqd") && !points){
-	    int cols;
-	    long lines=0;
-	    char *line;
-	    float *vec;
-	    FILE *in=fopen(name,"r");
-	    if(!in){
-	      fprintf(stderr,"Could not open input file %s\n",name);
-	      exit(1);
-	    }
-	    
-	    reset_next_value();
-	    line=setup_line(in);
-	    /* count cols before we start reading */
-	    {
-	      char *temp=line;
-	      while(*temp==' ')temp++;
-	      for(cols=0;*temp;cols++){
-		while(*temp>32)temp++;
-		while(*temp==' ')temp++;
-	      }
-	    }
-	    vec=alloca(cols*sizeof(float));
-	    /* count, then load, to avoid fragmenting the hell out of
-	       memory */
-	    while(line){
-	      lines++;
-	      for(j=0;j<cols;j++)
-		if(get_line_value(in,vec+j)){
-		  fprintf(stderr,"Too few columns on line %ld in data file\n",lines);
-		  exit(1);
-		}
-	      if((lines&0xff)==0)spinnit("counting samples...",lines*cols);
-	      line=setup_line(in);
-	    }
-	    pointlist=_ogg_malloc((cols*lines+entries*dim)*sizeof(float));
-	    
-	    rewind(in);
-	    line=setup_line(in);
-	    while(line){
-	      lines--;
-	      for(j=0;j<cols;j++)
-		if(get_line_value(in,vec+j)){
-		  fprintf(stderr,"Too few columns on line %ld in data file\n",lines);
-		  exit(1);
-		}
-	      /* deinterleave, add to heap */
-	      add_vector(b,vec,cols);
-	      if((lines&0xff)==0)spinnit("loading samples...",lines*cols);
-	      
-	      line=setup_line(in);
-	    }
-	    fclose(in);
-	  }
-	}
-	break;
-      case 2:
-	target=atol(*argv++);
-	if(target==0)target=entries;
-	break;
-      case 3:
-	protect=atol(*argv++);
-	break;
-      case 4:
-	{
-	  char *buff=alloca(strlen(*argv)+5);
-	  sprintf(buff,"%s.vqh",*argv);
-	  basename=*argv++;
-
-	  out=fopen(buff,"w");
-	  if(!out){
-	    fprintf(stderr,"unable ot open %s for output",buff);
-	    exit(1);
-	  }
-	}
-	break;
-      default:
-	usage();
-      }
-    }
-  }
-  if(!entries || !points || !out)usage();
-  if(target==-1)usage();
-
-  /* add guard points */
-  for(i=0;i<entries;i++)
-    for(j=0;j<dim;j++)
-      pointlist[points++]=b->valuelist[i*dim+j];
-  
-  points/=dim;
-
-  /* set up auxiliary vectors for error tracking */
-  {
-    encode_aux_nearestmatch *nt=NULL;
-    long pointssofar=0;
-    long *pointindex;
-    long indexedpoints=0;
-    long *entryindex;
-    long *reventry;
-    long *membership=_ogg_malloc(points*sizeof(long));
-    long *firsthead=_ogg_malloc(entries*sizeof(long));
-    long *secondary=_ogg_malloc(points*sizeof(long));
-    long *secondhead=_ogg_malloc(entries*sizeof(long));
-
-    long *cellcount=_ogg_calloc(entries,sizeof(long));
-    long *cellcount2=_ogg_calloc(entries,sizeof(long));
-    float *cellerror=_ogg_calloc(entries,sizeof(float));
-    float *cellerrormax=_ogg_calloc(entries,sizeof(float));
-    long cellsleft=entries;
-    for(i=0;i<points;i++)membership[i]=-1;
-    for(i=0;i<entries;i++)firsthead[i]=-1;
-    for(i=0;i<points;i++)secondary[i]=-1;
-    for(i=0;i<entries;i++)secondhead[i]=-1;
-
-    for(i=0;i<points;i++){
-      /* assign vectors to the nearest cell.  Also keep track of second
-	 nearest for error statistics */
-      float *ppt=pointlist+i*dim;
-      int    firstentry=closest(b,ppt,-1);
-      int    secondentry=closest(b,ppt,firstentry);
-      float firstmetric=_dist(dim,b->valuelist+dim*firstentry,ppt);
-      float secondmetric=_dist(dim,b->valuelist+dim*secondentry,ppt);
-      
-      if(!(i&0xff))spinnit("initializing... ",points-i);
-    
-      membership[i]=firsthead[firstentry];
-      firsthead[firstentry]=i;
-      secondary[i]=secondhead[secondentry];
-      secondhead[secondentry]=i;
-
-      if(i<points-entries){
-	cellerror[firstentry]+=secondmetric-firstmetric;
-	cellerrormax[firstentry]=max(cellerrormax[firstentry],
-				     _heuristic(b,ppt,secondentry));
-	cellcount[firstentry]++;
-	cellcount2[secondentry]++;
-      }
-    }
-
-    /* which cells are most heavily populated?  Protect as many from
-       dispersal as the user has requested */
-    {
-      long **countindex=_ogg_calloc(entries,sizeof(long *));
-      for(i=0;i<entries;i++)countindex[i]=cellcount+i;
-      qsort(countindex,entries,sizeof(long *),longsort);
-      for(i=0;i<protect;i++){
-	int ptr=countindex[i]-cellcount;
-	cellerrormax[ptr]=9e50f;
-      }
-    }
-
-    {
-      fprintf(stderr,"\r");
-      for(i=0;i<entries;i++){
-	/* decompose index */
-	int entry=i;
-	for(j=0;j<dim;j++){
-	  fprintf(stderr,"%d:",entry%b->c->thresh_tree->quantvals);
-	  entry/=b->c->thresh_tree->quantvals;
-	}
-	
-	fprintf(stderr,":%ld/%ld, ",cellcount[i],cellcount2[i]);
-      }
-      fprintf(stderr,"\n");
-    }
-
-    /* do the automatic cull request */
-    while(cellsleft>target){
-      int bestcell=-1;
-      float besterror=0;
-      float besterror2=0;
-      long head=-1;
-      char spinbuf[80];
-      sprintf(spinbuf,"cells left to eliminate: %ld : ",cellsleft-target);
-
-      /* find the cell with lowest removal impact */
-      for(i=0;i<entries;i++){
-	if(b->c->lengthlist[i]>0){
-	  if(bestcell==-1 || cellerrormax[i]<=besterror2){
-	    if(bestcell==-1 || cellerrormax[i]<besterror2 || 
-	       besterror>cellerror[i]){
-	      besterror=cellerror[i];
-	      besterror2=cellerrormax[i];
-	      bestcell=i;
-	    }
-	  }
-	}
-      }
-
-      fprintf(stderr,"\reliminating cell %d                              \n"
-	      "     dispersal error of %g max/%g total (%ld hits)\n",
-	      bestcell,besterror2,besterror,cellcount[bestcell]);
-
-      /* disperse it.  move each point out, adding it (properly) to
-         the second best */
-      b->c->lengthlist[bestcell]=0;
-      head=firsthead[bestcell];
-      firsthead[bestcell]=-1;
-      while(head!=-1){
-	/* head is a point number */
-	float *ppt=pointlist+head*dim;
-	int firstentry=closest(b,ppt,-1);
-	int secondentry=closest(b,ppt,firstentry);
-	float firstmetric=_dist(dim,b->valuelist+dim*firstentry,ppt);
-	float secondmetric=_dist(dim,b->valuelist+dim*secondentry,ppt);
-	long next=membership[head];
-
-	if(head<points-entries){
-	  cellcount[firstentry]++;
-	  cellcount[bestcell]--;
-	  cellerror[firstentry]+=secondmetric-firstmetric;
-	  cellerrormax[firstentry]=max(cellerrormax[firstentry],
-				       _heuristic(b,ppt,secondentry));
-	}
-
-	membership[head]=firsthead[firstentry];
-	firsthead[firstentry]=head;
-	head=next;
-	if(cellcount[bestcell]%128==0)
-	  spinnit(spinbuf,cellcount[bestcell]+cellcount2[bestcell]);
-
-      }
-
-      /* now see that all points that had the dispersed cell as second
-         choice have second choice reassigned */
-      head=secondhead[bestcell];
-      secondhead[bestcell]=-1;
-      while(head!=-1){
-	float *ppt=pointlist+head*dim;
-	/* who are we assigned to now? */
-	int firstentry=closest(b,ppt,-1);
-	/* what is the new second closest match? */
-	int secondentry=closest(b,ppt,firstentry);
-	/* old second closest is the cell being disbanded */
-	float oldsecondmetric=_dist(dim,b->valuelist+dim*bestcell,ppt);
-	/* new second closest error */
-	float secondmetric=_dist(dim,b->valuelist+dim*secondentry,ppt);
-	long next=secondary[head];
-
-	if(head<points-entries){
-	  cellcount2[secondentry]++;
-	  cellcount2[bestcell]--;
-	  cellerror[firstentry]+=secondmetric-oldsecondmetric;
-	  cellerrormax[firstentry]=max(cellerrormax[firstentry],
-				       _heuristic(b,ppt,secondentry));
-	}
-	
-	secondary[head]=secondhead[secondentry];
-	secondhead[secondentry]=head;
-	head=next;
-
-	if(cellcount2[bestcell]%128==0)
-	  spinnit(spinbuf,cellcount2[bestcell]);
-      }
-
-      cellsleft--;
-    }
-
-    /* paring is over.  Build decision trees using points that now fall
-       through the thresh matcher. */
-    /* we don't free membership; we flatten it in order to use in lp_split */
-
-    for(i=0;i<entries;i++){
-      long head=firsthead[i];
-      spinnit("rearranging membership cache... ",entries-i);
-      while(head!=-1){
-	long next=membership[head];
-	membership[head]=i;
-	head=next;
-      }
-    }
-
-    free(secondhead);
-    free(firsthead);
-    free(cellerror);
-    free(cellerrormax);
-    free(secondary);
-
-    pointindex=_ogg_malloc(points*sizeof(long));
-    /* make a point index of fall-through points */
-    for(i=0;i<points;i++){
-      int best=_best(b,pointlist+i*dim,1);
-      if(best==-1)
-	pointindex[indexedpoints++]=i;
-      spinnit("finding orphaned points... ",points-i);
-    }
-
-    /* make an entry index */
-    entryindex=_ogg_malloc(entries*sizeof(long));
-    target=0;
-    for(i=0;i<entries;i++){
-      if(b->c->lengthlist[i]>0)
-	entryindex[target++]=i;
-    }
-
-    /* make working space for a reverse entry index */
-    reventry=_ogg_malloc(entries*sizeof(long));
-
-    /* do the split */
-    nt=b->c->nearest_tree=
-      _ogg_calloc(1,sizeof(encode_aux_nearestmatch));
-
-    nt->alloc=4096;
-    nt->ptr0=_ogg_malloc(sizeof(long)*nt->alloc);
-    nt->ptr1=_ogg_malloc(sizeof(long)*nt->alloc);
-    nt->p=_ogg_malloc(sizeof(long)*nt->alloc);
-    nt->q=_ogg_malloc(sizeof(long)*nt->alloc);
-    nt->aux=0;
-
-    fprintf(stderr,"Leaves added: %d              \n",
-            lp_split(pointlist,points,
-                     b,entryindex,target,
-                     pointindex,indexedpoints,
-                     membership,reventry,
-                     0,&pointssofar));
-    free(membership);
-    free(reventry);
-    free(pointindex);
-
-    /* hack alert.  I should just change the damned splitter and
-       codebook writer */
-    for(i=0;i<nt->aux;i++)nt->p[i]*=dim;
-    for(i=0;i<nt->aux;i++)nt->q[i]*=dim;
-    
-    /* recount hits.  Build new lengthlist. reuse entryindex storage */
-    for(i=0;i<entries;i++)entryindex[i]=1;
-    for(i=0;i<points-entries;i++){
-      int best=_best(b,pointlist+i*dim,1);
-      float *a=pointlist+i*dim;
-      if(!(i&0xff))spinnit("counting hits...",i);
-      if(best==-1){
-	fprintf(stderr,"\nINTERNAL ERROR; a point count not be matched to a\n"
-		"codebook entry.  The new decision tree is broken.\n");
-	exit(1);
-      }
-      entryindex[best]++;
-    }
-    for(i=0;i<nt->aux;i++)nt->p[i]/=dim;
-    for(i=0;i<nt->aux;i++)nt->q[i]/=dim;
-    
-    /* the lengthlist builder doesn't actually deal with 0 hit entries.
-       So, we pack the 'sparse' hit list into a dense list, then unpack
-       the lengths after the build */
-    {
-      int upper=0;
-      long *lengthlist=_ogg_calloc(entries,sizeof(long));
-      for(i=0;i<entries;i++){
-	if(b->c->lengthlist[i]>0)
-	  entryindex[upper++]=entryindex[i];
-	else{
-	  if(entryindex[i]>1){
-	    fprintf(stderr,"\nINTERNAL ERROR; _best matched to unused entry\n");
-	    exit(1);
-	  }
-	}
-      }
-      
-      /* sanity check */
-      if(upper != target){
-	fprintf(stderr,"\nINTERNAL ERROR; packed the wrong number of entries\n");
-	exit(1);
-      }
-    
-      build_tree_from_lengths(upper,entryindex,lengthlist);
-      
-      upper=0;
-      for(i=0;i<entries;i++){
-	if(b->c->lengthlist[i]>0)
-	  b->c->lengthlist[i]=lengthlist[upper++];
-      }
-
-    }
-  }
-  /* we're done.  write it out. */
-  write_codebook(out,basename,b->c);
-
-  fprintf(stderr,"\r                                        \nDone.\n");
-  return(0);
-}
-
-
-
-