pdf2json 0.1.0

data/pdf2json-0.52-source/xpdf/JPXStream.h

@@ -0,0 +1,351 @@
+//========================================================================
+//
+// JPXStream.h
+//
+// Copyright 2002-2003 Glyph & Cog, LLC
+//
+//========================================================================
+
+#ifndef JPXSTREAM_H
+#define JPXSTREAM_H
+
+#include <aconf.h>
+
+#ifdef USE_GCC_PRAGMAS
+#pragma interface
+#endif
+
+#include "gtypes.h"
+#include "Object.h"
+#include "Stream.h"
+
+class JArithmeticDecoder;
+class JArithmeticDecoderStats;
+
+//------------------------------------------------------------------------
+
+enum JPXColorSpaceType {
+  jpxCSBiLevel = 0,
+  jpxCSYCbCr1 = 1,
+  jpxCSYCbCr2 = 3,
+  jpxCSYCBCr3 = 4,
+  jpxCSPhotoYCC = 9,
+  jpxCSCMY = 11,
+  jpxCSCMYK = 12,
+  jpxCSYCCK = 13,
+  jpxCSCIELab = 14,
+  jpxCSsRGB = 16,
+  jpxCSGrayscale = 17,
+  jpxCSBiLevel2 = 18,
+  jpxCSCIEJab = 19,
+  jpxCSCISesRGB = 20,
+  jpxCSROMMRGB = 21,
+  jpxCSsRGBYCbCr = 22,
+  jpxCSYPbPr1125 = 23,
+  jpxCSYPbPr1250 = 24
+};
+
+struct JPXColorSpecCIELab {
+  Guint rl, ol, ra, oa, rb, ob, il;
+};
+
+struct JPXColorSpecEnumerated {
+  JPXColorSpaceType type;	// color space type
+  union {
+    JPXColorSpecCIELab cieLab;
+  };
+};
+
+struct JPXColorSpec {
+  Guint meth;			// method
+  int prec;			// precedence
+  union {
+    JPXColorSpecEnumerated enumerated;
+  };
+};
+
+//------------------------------------------------------------------------
+
+struct JPXPalette {
+  Guint nEntries;		// number of entries in the palette
+  Guint nComps;			// number of components in each entry
+  Guint *bpc;			// bits per component, for each component
+  int *c;			// color data:
+				//   c[i*nComps+j] = entry i, component j
+};
+
+//------------------------------------------------------------------------
+
+struct JPXCompMap {
+  Guint nChannels;		// number of channels
+  Guint *comp;			// codestream components mapped to each channel
+  Guint *type;			// 0 for direct use, 1 for palette mapping
+  Guint *pComp;			// palette components to use
+};
+
+//------------------------------------------------------------------------
+
+struct JPXChannelDefn {
+  Guint nChannels;		// number of channels
+  Guint *idx;			// channel indexes
+  Guint *type;			// channel types
+  Guint *assoc;			// channel associations
+};
+
+//------------------------------------------------------------------------
+
+struct JPXTagTreeNode {
+  GBool finished;		// true if this node is finished
+  Guint val;			// current value
+};
+
+//------------------------------------------------------------------------
+
+struct JPXCoeff {
+  Gushort flags;		// flag bits
+  Gushort len;			// number of significant bits in mag
+  Guint mag;			// magnitude value
+};
+
+// coefficient flags
+#define jpxCoeffSignificantB  0
+#define jpxCoeffTouchedB      1
+#define jpxCoeffFirstMagRefB  2
+#define jpxCoeffSignB         7
+#define jpxCoeffSignificant   (1 << jpxCoeffSignificantB)
+#define jpxCoeffTouched       (1 << jpxCoeffTouchedB)
+#define jpxCoeffFirstMagRef   (1 << jpxCoeffFirstMagRefB)
+#define jpxCoeffSign          (1 << jpxCoeffSignB)
+
+//------------------------------------------------------------------------
+
+struct JPXCodeBlock {
+  //----- size
+  Guint x0, y0, x1, y1;		// bounds
+
+  //----- persistent state
+  GBool seen;			// true if this code-block has already
+				//   been seen
+  Guint lBlock;			// base number of bits used for pkt data length
+  Guint nextPass;		// next coding pass
+
+  //---- info from first packet
+  Guint nZeroBitPlanes;		// number of zero bit planes
+
+  //----- info for the current packet
+  Guint included;		// code-block inclusion in this packet:
+				//   0=not included, 1=included
+  Guint nCodingPasses;		// number of coding passes in this pkt
+  Guint dataLen;		// pkt data length
+
+  //----- coefficient data
+  JPXCoeff *coeffs;		// the coefficients
+  JArithmeticDecoder		// arithmetic decoder
+    *arithDecoder;
+  JArithmeticDecoderStats	// arithmetic decoder stats
+    *stats;
+};
+
+//------------------------------------------------------------------------
+
+struct JPXSubband {
+  //----- computed
+  Guint x0, y0, x1, y1;		// bounds
+  Guint nXCBs, nYCBs;		// number of code-blocks in the x and y
+				//   directions
+
+  //----- tag trees
+  Guint maxTTLevel;		// max tag tree level
+  JPXTagTreeNode *inclusion;	// inclusion tag tree for each subband
+  JPXTagTreeNode *zeroBitPlane;	// zero-bit plane tag tree for each
+				//   subband
+
+  //----- children
+  JPXCodeBlock *cbs;		// the code-blocks (len = nXCBs * nYCBs)
+};
+
+//------------------------------------------------------------------------
+
+struct JPXPrecinct {
+  //----- computed
+  Guint x0, y0, x1, y1;		// bounds of the precinct
+
+  //----- children
+  JPXSubband *subbands;		// the subbands
+};
+
+//------------------------------------------------------------------------
+
+struct JPXResLevel {
+  //----- from the COD and COC segments (main and tile)
+  Guint precinctWidth;		// log2(precinct width)
+  Guint precinctHeight;		// log2(precinct height)
+
+  //----- computed
+  Guint x0, y0, x1, y1;		// bounds of the tile-comp (for this res level)
+  Guint bx0[3], by0[3],		// subband bounds
+        bx1[3], by1[3];
+
+  //---- children
+  JPXPrecinct *precincts;	// the precincts
+};
+
+//------------------------------------------------------------------------
+
+struct JPXTileComp {
+  //----- from the SIZ segment
+  GBool sgned;			// 1 for signed, 0 for unsigned
+  Guint prec;			// precision, in bits
+  Guint hSep;			// horizontal separation of samples
+  Guint vSep;			// vertical separation of samples
+
+  //----- from the COD and COC segments (main and tile)
+  Guint style;			// coding style parameter (Scod / Scoc)
+  Guint nDecompLevels;		// number of decomposition levels
+  Guint codeBlockW;		// log2(code-block width)
+  Guint codeBlockH;		// log2(code-block height)
+  Guint codeBlockStyle;		// code-block style
+  Guint transform;		// wavelet transformation
+
+  //----- from the QCD and QCC segments (main and tile)
+  Guint quantStyle;		// quantization style
+  Guint *quantSteps;		// quantization step size for each subband
+  Guint nQuantSteps;		// number of entries in quantSteps
+
+  //----- computed
+  Guint x0, y0, x1, y1;		// bounds of the tile-comp, in ref coords
+  Guint cbW;			// code-block width
+  Guint cbH;			// code-block height
+
+  //----- image data
+  int *data;			// the decoded image data
+  int *buf;			// intermediate buffer for the inverse
+				//   transform
+
+  //----- children
+  JPXResLevel *resLevels;	// the resolution levels
+				//   (len = nDecompLevels + 1)
+};
+
+//------------------------------------------------------------------------
+
+struct JPXTile {
+  //----- from the COD segments (main and tile)
+  Guint progOrder;		// progression order
+  Guint nLayers;		// number of layers
+  Guint multiComp;		// multiple component transformation
+
+  //----- computed
+  Guint x0, y0, x1, y1;		// bounds of the tile, in ref coords
+  Guint maxNDecompLevels;	// max number of decomposition levels used
+				//   in any component in this tile
+
+  //----- progression order loop counters
+  Guint comp;			//   component
+  Guint res;			//   resolution level
+  Guint precinct;		//   precinct
+  Guint layer;			//   layer
+
+  //----- children
+  JPXTileComp *tileComps;	// the tile-components (len = JPXImage.nComps)
+};
+
+//------------------------------------------------------------------------
+
+struct JPXImage {
+  //----- from the SIZ segment
+  Guint xSize, ySize;		// size of reference grid
+  Guint xOffset, yOffset;	// image offset
+  Guint xTileSize, yTileSize;	// size of tiles
+  Guint xTileOffset,		// offset of first tile
+        yTileOffset;
+  Guint nComps;			// number of components
+
+  //----- computed
+  Guint nXTiles;		// number of tiles in x direction
+  Guint nYTiles;		// number of tiles in y direction
+
+  //----- children
+  JPXTile *tiles;		// the tiles (len = nXTiles * nYTiles)
+};
+
+//------------------------------------------------------------------------
+
+class JPXStream: public FilterStream {
+public:
+
+  JPXStream(Stream *strA);
+  virtual ~JPXStream();
+  virtual StreamKind getKind() { return strJPX; }
+  virtual void reset();
+  virtual void close();
+  virtual int getChar();
+  virtual int lookChar();
+  virtual GString *getPSFilter(int psLevel, char *indent);
+  virtual GBool isBinary(GBool last = gTrue);
+  virtual void getImageParams(int *bitsPerComponent,
+			      StreamColorSpaceMode *csMode);
+
+private:
+
+  void fillReadBuf();
+  void getImageParams2(int *bitsPerComponent, StreamColorSpaceMode *csMode);
+  GBool readBoxes();
+  GBool readColorSpecBox(Guint dataLen);
+  GBool readCodestream(Guint len);
+  GBool readTilePart();
+  GBool readTilePartData(Guint tileIdx,
+			 Guint tilePartLen, GBool tilePartToEOC);
+  GBool readCodeBlockData(JPXTileComp *tileComp,
+			  JPXResLevel *resLevel,
+			  JPXPrecinct *precinct,
+			  JPXSubband *subband,
+			  Guint res, Guint sb,
+			  JPXCodeBlock *cb);
+  void inverseTransform(JPXTileComp *tileComp);
+  void inverseTransformLevel(JPXTileComp *tileComp,
+			     Guint r, JPXResLevel *resLevel,
+			     Guint nx0, Guint ny0,
+			     Guint nx1, Guint ny1);
+  void inverseTransform1D(JPXTileComp *tileComp,
+			  int *data, Guint stride,
+			  Guint i0, Guint i1);
+  GBool inverseMultiCompAndDC(JPXTile *tile);
+  GBool readBoxHdr(Guint *boxType, Guint *boxLen, Guint *dataLen);
+  int readMarkerHdr(int *segType, Guint *segLen);
+  GBool readUByte(Guint *x);
+  GBool readByte(int *x);
+  GBool readUWord(Guint *x);
+  GBool readULong(Guint *x);
+  GBool readNBytes(int nBytes, GBool signd, int *x);
+  GBool readBits(int nBits, Guint *x);
+  void startBitBuf(Guint byteCountA);
+  Guint finishBitBuf();
+
+  Guint nComps;			// number of components
+  Guint *bpc;			// bits per component, for each component
+  Guint width, height;		// image size
+  GBool haveImgHdr;		// set if a JP2/JPX image header has been
+				//   found
+  JPXColorSpec cs;		// color specification
+  GBool haveCS;			// set if a color spec has been found
+  JPXPalette palette;		// the palette
+  GBool havePalette;		// set if a palette has been found
+  JPXCompMap compMap;		// the component mapping
+  GBool haveCompMap;		// set if a component mapping has been found
+  JPXChannelDefn channelDefn;	// channel definition
+  GBool haveChannelDefn;	// set if a channel defn has been found
+
+  JPXImage img;			// JPEG2000 decoder data
+  Guint bitBuf;			// buffer for bit reads
+  int bitBufLen;		// number of bits in bitBuf
+  GBool bitBufSkip;		// true if next bit should be skipped
+				//   (for bit stuffing)
+  Guint byteCount;		// number of available bytes left
+
+  Guint curX, curY, curComp;	// current position for lookChar/getChar
+  Guint readBuf;		// read buffer
+  Guint readBufLen;		// number of valid bits in readBuf
+};
+
+#endif

data/pdf2json-0.52-source/xpdf/Lexer.cc

@@ -0,0 +1,485 @@
+//========================================================================
+//
+// Lexer.cc
+//
+// Copyright 1996-2003 Glyph & Cog, LLC
+//
+//========================================================================
+
+#include <aconf.h>
+
+#ifdef USE_GCC_PRAGMAS
+#pragma implementation
+#endif
+
+#include <stdlib.h>
+#include <stddef.h>
+#include <string.h>
+#include <ctype.h>
+#include "Lexer.h"
+#include "Error.h"
+
+//------------------------------------------------------------------------
+
+// A '1' in this array means the character is white space.  A '1' or
+// '2' means the character ends a name or command.
+static char specialChars[256] = {
+  1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 1, 1, 0, 0,   // 0x
+  0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,   // 1x
+  1, 0, 0, 0, 0, 2, 0, 0, 2, 2, 0, 0, 0, 0, 0, 2,   // 2x
+  0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 0,   // 3x
+  0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,   // 4x
+  0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 0, 0,   // 5x
+  0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,   // 6x
+  0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 0, 0,   // 7x
+  0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,   // 8x
+  0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,   // 9x
+  0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,   // ax
+  0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,   // bx
+  0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,   // cx
+  0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,   // dx
+  0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,   // ex
+  0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0    // fx
+};
+
+//------------------------------------------------------------------------
+// Lexer
+//------------------------------------------------------------------------
+
+Lexer::Lexer(XRef *xref, Stream *str) {
+  Object obj;
+
+  curStr.initStream(str);
+  streams = new Array(xref);
+  streams->add(curStr.copy(&obj));
+  strPtr = 0;
+  freeArray = gTrue;
+  curStr.streamReset();
+}
+
+Lexer::Lexer(XRef *xref, Object *obj) {
+  Object obj2;
+
+  if (obj->isStream()) {
+    streams = new Array(xref);
+    freeArray = gTrue;
+    streams->add(obj->copy(&obj2));
+  } else {
+    streams = obj->getArray();
+    freeArray = gFalse;
+  }
+  strPtr = 0;
+  if (streams->getLength() > 0) {
+    streams->get(strPtr, &curStr);
+    curStr.streamReset();
+  }
+}
+
+Lexer::~Lexer() {
+  if (!curStr.isNone()) {
+    curStr.streamClose();
+    curStr.free();
+  }
+  if (freeArray) {
+    delete streams;
+  }
+}
+
+int Lexer::getChar() {
+  int c;
+
+  c = EOF;
+  while (!curStr.isNone() && (c = curStr.streamGetChar()) == EOF) {
+    curStr.streamClose();
+    curStr.free();
+    ++strPtr;
+    if (strPtr < streams->getLength()) {
+      streams->get(strPtr, &curStr);
+      curStr.streamReset();
+    }
+  }
+  return c;
+}
+
+int Lexer::lookChar() {
+  if (curStr.isNone()) {
+    return EOF;
+  }
+  return curStr.streamLookChar();
+}
+
+Object *Lexer::getObj(Object *obj) {
+  char *p;
+  int c, c2;
+  GBool comment, neg, done;
+  int numParen;
+  int xi;
+  double xf, scale;
+  GString *s;
+  int n, m;
+
+  // skip whitespace and comments
+  comment = gFalse;
+  while (1) {
+    if ((c = getChar()) == EOF) {
+      return obj->initEOF();
+    }
+    if (comment) {
+      if (c == '\r' || c == '\n')
+	comment = gFalse;
+    } else if (c == '%') {
+      comment = gTrue;
+    } else if (specialChars[c] != 1) {
+      break;
+    }
+  }
+
+  // start reading token
+  switch (c) {
+
+  // number
+  case '0': case '1': case '2': case '3': case '4':
+  case '5': case '6': case '7': case '8': case '9':
+  case '-': case '.':
+    neg = gFalse;
+    xi = 0;
+    if (c == '-') {
+      neg = gTrue;
+    } else if (c == '.') {
+      goto doReal;
+    } else {
+      xi = c - '0';
+    }
+    while (1) {
+      c = lookChar();
+      if (isdigit(c)) {
+	getChar();
+	xi = xi * 10 + (c - '0');
+      } else if (c == '.') {
+	getChar();
+	goto doReal;
+      } else {
+	break;
+      }
+    }
+    if (neg)
+      xi = -xi;
+    obj->initInt(xi);
+    break;
+  doReal:
+    xf = xi;
+    scale = 0.1;
+    while (1) {
+      c = lookChar();
+      if (c == '-') {
+	// ignore minus signs in the middle of numbers to match
+	// Adobe's behavior
+	error(getPos(), "Badly formatted number");
+	getChar();
+	continue;
+      }
+      if (!isdigit(c)) {
+	break;
+      }
+      getChar();
+      xf = xf + scale * (c - '0');
+      scale *= 0.1;
+    }
+    if (neg)
+      xf = -xf;
+    obj->initReal(xf);
+    break;
+
+  // string
+  case '(':
+    p = tokBuf;
+    n = 0;
+    numParen = 1;
+    done = gFalse;
+    s = NULL;
+    do {
+      c2 = EOF;
+      switch (c = getChar()) {
+
+      case EOF:
+#if 0
+      // This breaks some PDF files, e.g., ones from Photoshop.
+      case '\r':
+      case '\n':
+#endif
+	error(getPos(), "Unterminated string");
+	done = gTrue;
+	break;
+
+      case '(':
+	++numParen;
+	c2 = c;
+	break;
+
+      case ')':
+	if (--numParen == 0) {
+	  done = gTrue;
+	} else {
+	  c2 = c;
+	}
+	break;
+
+      case '\\':
+	switch (c = getChar()) {
+	case 'n':
+	  c2 = '\n';
+	  break;
+	case 'r':
+	  c2 = '\r';
+	  break;
+	case 't':
+	  c2 = '\t';
+	  break;
+	case 'b':
+	  c2 = '\b';
+	  break;
+	case 'f':
+	  c2 = '\f';
+	  break;
+	case '\\':
+	case '(':
+	case ')':
+	  c2 = c;
+	  break;
+	case '0': case '1': case '2': case '3':
+	case '4': case '5': case '6': case '7':
+	  c2 = c - '0';
+	  c = lookChar();
+	  if (c >= '0' && c <= '7') {
+	    getChar();
+	    c2 = (c2 << 3) + (c - '0');
+	    c = lookChar();
+	    if (c >= '0' && c <= '7') {
+	      getChar();
+	      c2 = (c2 << 3) + (c - '0');
+	    }
+	  }
+	  break;
+	case '\r':
+	  c = lookChar();
+	  if (c == '\n') {
+	    getChar();
+	  }
+	  break;
+	case '\n':
+	  break;
+	case EOF:
+	  error(getPos(), "Unterminated string");
+	  done = gTrue;
+	  break;
+	default:
+	  c2 = c;
+	  break;
+	}
+	break;
+
+      default:
+	c2 = c;
+	break;
+      }
+
+      if (c2 != EOF) {
+	if (n == tokBufSize) {
+	  if (!s)
+	    s = new GString(tokBuf, tokBufSize);
+	  else
+	    s->append(tokBuf, tokBufSize);
+	  p = tokBuf;
+	  n = 0;
+	}
+	*p++ = (char)c2;
+	++n;
+      }
+    } while (!done);
+    if (!s)
+      s = new GString(tokBuf, n);
+    else
+      s->append(tokBuf, n);
+    obj->initString(s);
+    break;
+
+  // name
+  case '/':
+    p = tokBuf;
+    n = 0;
+    while ((c = lookChar()) != EOF && !specialChars[c]) {
+      getChar();
+      if (c == '#') {
+	c2 = lookChar();
+	if (c2 >= '0' && c2 <= '9') {
+	  c = c2 - '0';
+	} else if (c2 >= 'A' && c2 <= 'F') {
+	  c = c2 - 'A' + 10;
+	} else if (c2 >= 'a' && c2 <= 'f') {
+	  c = c2 - 'a' + 10;
+	} else {
+	  goto notEscChar;
+	}
+	getChar();
+	c <<= 4;
+	c2 = getChar();
+	if (c2 >= '0' && c2 <= '9') {
+	  c += c2 - '0';
+	} else if (c2 >= 'A' && c2 <= 'F') {
+	  c += c2 - 'A' + 10;
+	} else if (c2 >= 'a' && c2 <= 'f') {
+	  c += c2 - 'a' + 10;
+	} else {
+	  error(getPos(), "Illegal digit in hex char in name");
+	}
+      }
+     notEscChar:
+      if (++n == tokBufSize) {
+	error(getPos(), "Name token too long");
+	break;
+      }
+      *p++ = c;
+    }
+    *p = '\0';
+    obj->initName(tokBuf);
+    break;
+
+  // array punctuation
+  case '[':
+  case ']':
+    tokBuf[0] = c;
+    tokBuf[1] = '\0';
+    obj->initCmd(tokBuf);
+    break;
+
+  // hex string or dict punctuation
+  case '<':
+    c = lookChar();
+
+    // dict punctuation
+    if (c == '<') {
+      getChar();
+      tokBuf[0] = tokBuf[1] = '<';
+      tokBuf[2] = '\0';
+      obj->initCmd(tokBuf);
+
+    // hex string
+    } else {
+      p = tokBuf;
+      m = n = 0;
+      c2 = 0;
+      s = NULL;
+      while (1) {
+	c = getChar();
+	if (c == '>') {
+	  break;
+	} else if (c == EOF) {
+	  error(getPos(), "Unterminated hex string");
+	  break;
+	} else if (specialChars[c] != 1) {
+	  c2 = c2 << 4;
+	  if (c >= '0' && c <= '9')
+	    c2 += c - '0';
+	  else if (c >= 'A' && c <= 'F')
+	    c2 += c - 'A' + 10;
+	  else if (c >= 'a' && c <= 'f')
+	    c2 += c - 'a' + 10;
+	  else
+	    error(getPos(), "Illegal character <%02x> in hex string", c);
+	  if (++m == 2) {
+	    if (n == tokBufSize) {
+	      if (!s)
+		s = new GString(tokBuf, tokBufSize);
+	      else
+		s->append(tokBuf, tokBufSize);
+	      p = tokBuf;
+	      n = 0;
+	    }
+	    *p++ = (char)c2;
+	    ++n;
+	    c2 = 0;
+	    m = 0;
+	  }
+	}
+      }
+      if (!s)
+	s = new GString(tokBuf, n);
+      else
+	s->append(tokBuf, n);
+      if (m == 1)
+	s->append((char)(c2 << 4));
+      obj->initString(s);
+    }
+    break;
+
+  // dict punctuation
+  case '>':
+    c = lookChar();
+    if (c == '>') {
+      getChar();
+      tokBuf[0] = tokBuf[1] = '>';
+      tokBuf[2] = '\0';
+      obj->initCmd(tokBuf);
+    } else {
+      error(getPos(), "Illegal character '>'");
+      obj->initError();
+    }
+    break;
+
+  // error
+  case ')':
+  case '{':
+  case '}':
+    error(getPos(), "Illegal character '%c'", c);
+    obj->initError();
+    break;
+
+  // command
+  default:
+    p = tokBuf;
+    *p++ = c;
+    n = 1;
+    while ((c = lookChar()) != EOF && !specialChars[c]) {
+      getChar();
+      if (++n == tokBufSize) {
+	error(getPos(), "Command token too long");
+	break;
+      }
+      *p++ = c;
+    }
+    *p = '\0';
+    if (tokBuf[0] == 't' && !strcmp(tokBuf, "true")) {
+      obj->initBool(gTrue);
+    } else if (tokBuf[0] == 'f' && !strcmp(tokBuf, "false")) {
+      obj->initBool(gFalse);
+    } else if (tokBuf[0] == 'n' && !strcmp(tokBuf, "null")) {
+      obj->initNull();
+    } else {
+      obj->initCmd(tokBuf);
+    }
+    break;
+  }
+
+  return obj;
+}
+
+void Lexer::skipToNextLine() {
+  int c;
+
+  while (1) {
+    c = getChar();
+    if (c == EOF || c == '\n') {
+      return;
+    }
+    if (c == '\r') {
+      if ((c = lookChar()) == '\n') {
+	getChar();
+      }
+      return;
+    }
+  }
+}
+
+GBool Lexer::isSpace(int c) {
+  return c >= 0 && c <= 0xff && specialChars[c] == 1;
+}