charguess 1.0

data/ext/libcharguess/cpp/SBCSGroupProber.cpp

@@ -0,0 +1,244 @@
+/*
+	libcharguess	-	Guess the encoding/charset of a string
+    Copyright (C) 2003  Stephane Corbe <noubi@users.sourceforge.net>
+	Based on Mozilla sources
+
+    This library is free software; you can redistribute it and/or
+    modify it under the terms of the GNU Lesser General Public
+    License as published by the Free Software Foundation; either
+    version 2.1 of the License, or (at your option) any later version.
+
+    This library is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+    Lesser General Public License for more details.
+
+    You should have received a copy of the GNU Lesser General Public
+    License along with this library; if not, write to the Free Software
+    Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+*/
+
+#include "SBCharsetProber.h"
+#include "SBCSGroupProber.h"
+
+
+nsSBCSGroupProber::nsSBCSGroupProber()
+{
+  mProbers[0] = new nsSingleByteCharSetProber(&Win1251Model);
+  mProbers[1] = new nsSingleByteCharSetProber(&Koi8rModel);
+  mProbers[2] = new nsSingleByteCharSetProber(&Latin5Model);
+  mProbers[3] = new nsSingleByteCharSetProber(&MacCyrillicModel);
+  mProbers[4] = new nsSingleByteCharSetProber(&Ibm866Model);
+  mProbers[5] = new nsSingleByteCharSetProber(&Ibm855Model);
+  mProbers[6] = new nsSingleByteCharSetProber(&Latin7Model);
+  mProbers[7] = new nsSingleByteCharSetProber(&Win1253Model);
+  mProbers[8] = new nsSingleByteCharSetProber(&Latin5BulgarianModel);
+  mProbers[9] = new nsSingleByteCharSetProber(&Win1251BulgarianModel);
+
+  // disable latin2 before latin1 is available, otherwise all latin1 
+  // will be detected as latin2 because of their similarity.
+  //mProbers[10] = new nsSingleByteCharSetProber(&Latin2HungarianModel);
+  //mProbers[11] = new nsSingleByteCharSetProber(&Win1250HungarianModel);
+
+  Reset();
+}
+
+nsSBCSGroupProber::~nsSBCSGroupProber()
+{
+  for (PRUint32 i = 0; i < NUM_OF_SBCS_PROBERS; i++)
+  {
+    delete mProbers[i];
+  }
+}
+
+
+const char* nsSBCSGroupProber::GetCharSetName()
+{
+  //if we have no answer yet
+  if (mBestGuess == -1)
+  {
+    GetConfidence();
+    //no charset seems positive
+    if (mBestGuess == -1)
+      //we will use default.
+      mBestGuess = 0;
+  }
+  return mProbers[mBestGuess]->GetCharSetName();
+}
+
+void  nsSBCSGroupProber::Reset(void)
+{
+  for (PRUint32 i = 0; i < NUM_OF_SBCS_PROBERS; i++)
+  {
+    mProbers[i]->Reset();
+    mIsActive[i] = PR_TRUE;
+  }
+  mBestGuess = -1;
+  mState = eDetecting;
+}
+
+//This filter apply to all scripts that does not use latin letters (english letter)
+PRBool nsSBCSGroupProber::FilterWithoutEnglishLetters(const char* aBuf, PRUint32 aLen, char** newBuf, PRUint32& newLen)
+{
+  //do filtering to reduce load to probers
+  char *newptr;
+  char *prevPtr, *curPtr;
+  
+  PRBool meetMSB = PR_FALSE;   
+  newptr = *newBuf = (char*)PR_MALLOC(aLen);
+  if (!newptr)
+    return PR_FALSE;
+
+  for (curPtr = prevPtr = (char*)aBuf; curPtr < aBuf+aLen; curPtr++)
+  {
+    if (*curPtr & 0x80)
+      meetMSB = PR_TRUE;
+    else if (*curPtr < 'A' || (*curPtr > 'Z' && *curPtr < 'a') || *curPtr > 'z') 
+    {
+      //current char is a symbol, most likely a punctuation. we treat it as segment delimiter
+      if (meetMSB && curPtr > prevPtr) 
+      //this segment contains more than single symbol, and it has upper ascii, we need to keep it
+      {
+        while (prevPtr < curPtr) *newptr++ = *prevPtr++;  
+        prevPtr++;
+        *newptr++ = ' ';
+        meetMSB = PR_FALSE;
+      }
+      else //ignore current segment. (either because it is just a symbol or just a english word
+        prevPtr = curPtr+1;
+    }
+  }
+
+  newLen = newptr - *newBuf;
+
+  return PR_TRUE;
+}
+
+#ifdef  NO_ENGLISH_CONTAMINATION 
+//This filter apply to all scripts that does use latin letters (english letter)
+PRBool nsSBCSGroupProber::FilterWithEnglishLetters(const char* aBuf, PRUint32 aLen, char** newBuf, PRUint32& newLen)
+{
+  //do filtering to reduce load to probers
+  char *newptr;
+  char *prevPtr, *curPtr;
+  PRBool isInTag = PR_FALSE;
+
+  newptr = *newBuf = (char*)PR_MALLOC(aLen);
+  if (!newptr)
+    return PR_FALSE;
+
+  for (curPtr = prevPtr = (char*)aBuf; curPtr < aBuf+aLen; curPtr++)
+  {
+		if (*curPtr == '>')
+			isInTag = PR_FALSE;
+    else if (*curPtr == '<')
+      isInTag = PR_TRUE;
+
+    if (!(*curPtr & 0x80) &&
+        (*curPtr < 'A' || (*curPtr > 'Z' && *curPtr < 'a') || *curPtr > 'z') )
+    {
+      if (curPtr > prevPtr && !isInTag) //current segment contains more than just a symbol 
+                                        // and it is not inside a tag, keep it
+      {
+        while (prevPtr < curPtr) *newptr++ = *prevPtr++;  
+        prevPtr++;
+        *newptr++ = ' ';
+      }
+      else
+        prevPtr = curPtr+1;
+    }
+  }
+
+  newLen = newptr - *newBuf;
+
+  return PR_TRUE;
+}
+#endif //NO_ENGLISH_CONTAMINATION
+
+nsProbingState nsSBCSGroupProber::HandleData(const char* aBuf, PRUint32 aLen)
+{
+  nsProbingState st;
+  PRUint32 i;
+  char *newBuf1;
+  PRUint32 newLen1;
+
+  //apply filter to original buffer, and we got new buffer back
+  //depend on what script it is, we will feed them the new buffer 
+  //we got after applying proper filter
+  FilterWithoutEnglishLetters(aBuf, aLen, &newBuf1, newLen1);
+
+  for (i = 0; i < NUM_OF_SBCS_PROBERS; i++)
+  {
+     if (!mIsActive[i])
+       continue;
+     st = mProbers[i]->HandleData(newBuf1, newLen1);
+     if (st == eFoundIt)
+     {
+       mBestGuess = i;
+       mState = eFoundIt;
+       break;
+     }
+     else if (st == eNotMe)
+     {
+       mIsActive[i] = PR_FALSE;
+       mActiveNum--;
+       if (mActiveNum <= 0)
+       {
+         mState = eNotMe;
+         break;
+       }
+     }
+  }
+
+  PR_FREEIF(newBuf1);
+
+  return mState;
+}
+
+float nsSBCSGroupProber::GetConfidence(void)
+{
+  PRUint32 i;
+  float bestConf = 0.0, cf;
+
+  switch (mState)
+  {
+  case eFoundIt:
+    return (float)0.99; //sure yes
+  case eNotMe:
+    return (float)0.01;  //sure no
+  default:
+    for (i = 0; i < NUM_OF_SBCS_PROBERS; i++)
+    {
+      if (!mIsActive[i])
+        continue;
+      cf = mProbers[i]->GetConfidence();
+      if (bestConf < cf)
+      {
+        bestConf = cf;
+        mBestGuess = i;
+      }
+    }
+  }
+  return bestConf;
+}
+
+#ifdef DEBUG_chardet
+void 
+nsSBCSGroupProber::DumpStatus()
+{
+  PRUint32 i;
+  float cf;
+  
+  cf = GetConfidence();
+  printf("SBCS Group Prober --------begin status \r\n");
+  for (i = 0; i < NUM_OF_SBCS_PROBERS; i++)
+  {
+    if (!mIsActive[i])
+      printf("[%s] is inactive(ie. cofidence is too low).\r\n", mProbers[i]->GetCharSetName(), i);
+    else
+      mProbers[i]->DumpStatus();
+  }
+  printf("SBCS Group found best match [%s] confidence %f.\r\n", 
+        mProbers[mBestGuess]->GetCharSetName(), cf);
+}
+#endif

data/ext/libcharguess/cpp/SBCSGroupProber.h

@@ -0,0 +1,54 @@
+/*
+	libcharguess	-	Guess the encoding/charset of a string
+    Copyright (C) 2003  Stephane Corbe <noubi@users.sourceforge.net>
+	Based on Mozilla sources
+
+    This library is free software; you can redistribute it and/or
+    modify it under the terms of the GNU Lesser General Public
+    License as published by the Free Software Foundation; either
+    version 2.1 of the License, or (at your option) any later version.
+
+    This library is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+    Lesser General Public License for more details.
+
+    You should have received a copy of the GNU Lesser General Public
+    License along with this library; if not, write to the Free Software
+    Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+*/
+
+#ifndef nsSBCSGroupProber_h__
+#define nsSBCSGroupProber_h__
+
+
+#define NUM_OF_SBCS_PROBERS    10
+
+class nsSingleByteCharSetProber;
+class nsSBCSGroupProber: public nsCharSetProber {
+public:
+  nsSBCSGroupProber();
+  virtual ~nsSBCSGroupProber();
+  nsProbingState HandleData(const char* aBuf, PRUint32 aLen);
+  PRBool FilterWithoutEnglishLetters(const char* aBuf, PRUint32 aLen, char** newBuf, PRUint32& newLen);
+  PRBool FilterWithEnglishLetters(const char* aBuf, PRUint32 aLen, char** newBuf, PRUint32& newLen);
+  const char* GetCharSetName();
+  nsProbingState GetState(void) {return mState;};
+  void      Reset(void);
+  float     GetConfidence(void);
+  void      SetOpion() {};
+
+#ifdef DEBUG_chardet
+  void  DumpStatus();
+#endif
+
+protected:
+  nsProbingState mState;
+  nsSingleByteCharSetProber* mProbers[NUM_OF_SBCS_PROBERS];
+  PRBool          mIsActive[NUM_OF_SBCS_PROBERS];
+  PRInt32 mBestGuess;
+  PRUint32 mActiveNum;
+};
+
+#endif /* nsSBCSGroupProber_h__ */
+

data/ext/libcharguess/cpp/SBCharsetProber.cpp

@@ -0,0 +1,100 @@
+/*
+	libcharguess	-	Guess the encoding/charset of a string
+    Copyright (C) 2003  Stephane Corbe <noubi@users.sourceforge.net>
+	Based on Mozilla sources
+
+    This library is free software; you can redistribute it and/or
+    modify it under the terms of the GNU Lesser General Public
+    License as published by the Free Software Foundation; either
+    version 2.1 of the License, or (at your option) any later version.
+
+    This library is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+    Lesser General Public License for more details.
+
+    You should have received a copy of the GNU Lesser General Public
+    License along with this library; if not, write to the Free Software
+    Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+*/
+
+#include <stdio.h>
+#include "SBCharsetProber.h"
+
+nsProbingState nsSingleByteCharSetProber::HandleData(const char* aBuf, PRUint32 aLen)
+{
+  unsigned char order;
+
+  for (PRUint32 i = 0; i < aLen; i++)
+  {
+    order = mModel->charToOrderMap[(unsigned char)aBuf[i]];
+
+    if (order < SYMBOL_CAT_ORDER)
+      mTotalChar++;
+    if (order < SAMPLE_SIZE)
+    {
+        mFreqChar++;
+
+      if (mLastOrder < SAMPLE_SIZE)
+      {
+        mTotalSeqs++;
+        ++(mSeqCounters[mModel->precedenceMatrix[mLastOrder*SAMPLE_SIZE+order]]);
+      }
+    }
+    mLastOrder = order;
+  }
+
+  if (mState == eDetecting)
+    if (mTotalSeqs > SB_ENOUGH_REL_THRESHOLD)
+    {
+      float cf = GetConfidence();
+      if (cf > POSITIVE_SHORTCUT_THRESHOLD)
+        mState = eFoundIt;
+      else if (cf < NEGATIVE_SHORTCUT_THRESHOLD)
+        mState = eNotMe;
+    }
+
+  return mState;
+}
+
+void  nsSingleByteCharSetProber::Reset(void)
+{
+  mState = eDetecting;
+  mLastOrder = 255;
+  for (PRUint32 i = 0; i < NUMBER_OF_SEQ_CAT; i++)
+    mSeqCounters[i] = 0;
+  mTotalSeqs = 0;
+  mTotalChar = 0;
+  mFreqChar = 0;
+}
+
+//#define NEGATIVE_APPROACH 1
+
+float nsSingleByteCharSetProber::GetConfidence(void)
+{
+#ifdef NEGATIVE_APPROACH
+  if (mTotalSeqs > 0)
+    if (mTotalSeqs > mSeqCounters[NEGATIVE_CAT]*10 )
+      return ((float)(mTotalSeqs - mSeqCounters[NEGATIVE_CAT]*10))/mTotalSeqs * mFreqChar / mTotalChar;
+  return (float)0.01;
+#else  //POSITIVE_APPROACH
+  float r;
+
+  if (mTotalSeqs > 0) {
+    r = ((float)1.0) * mSeqCounters[POSITIVE_CAT] / mTotalSeqs / mModel->mTypicalPositiveRatio;
+    r = r*mFreqChar/mTotalChar;
+    if (r >= (float)1.00)
+      r = (float)0.99;
+    return r;
+  }
+  return (float)0.01;
+#endif
+}
+
+#ifdef DEBUG_chardet
+void 
+nsSingleByteCharSetProber::DumpStatus()
+{
+  printf("[%s] prober has confidence %f\r\n", GetCharSetName(), GetConfidence());
+}
+#endif

data/ext/libcharguess/cpp/SBCharsetProber.h

@@ -0,0 +1,89 @@
+/*
+	libcharguess	-	Guess the encoding/charset of a string
+    Copyright (C) 2003  Stephane Corbe <noubi@users.sourceforge.net>
+	Based on Mozilla sources
+
+    This library is free software; you can redistribute it and/or
+    modify it under the terms of the GNU Lesser General Public
+    License as published by the Free Software Foundation; either
+    version 2.1 of the License, or (at your option) any later version.
+
+    This library is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+    Lesser General Public License for more details.
+
+    You should have received a copy of the GNU Lesser General Public
+    License along with this library; if not, write to the Free Software
+    Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+*/
+
+#ifndef nsSingleByteCharSetProber_h__
+#define nsSingleByteCharSetProber_h__
+
+#include "charsetProber.h"
+
+#define SAMPLE_SIZE 64
+#define SB_ENOUGH_REL_THRESHOLD  1024
+#define POSITIVE_SHORTCUT_THRESHOLD  (float)0.95
+#define NEGATIVE_SHORTCUT_THRESHOLD  (float)0.05
+#define SYMBOL_CAT_ORDER  250
+#define NUMBER_OF_SEQ_CAT 4
+#define POSITIVE_CAT   (NUMBER_OF_SEQ_CAT-1)
+#define NEGATIVE_CAT   0
+
+typedef struct
+{
+  unsigned char *charToOrderMap;     //[256] table use to find a char's order
+  char *precedenceMatrix;           //[SAMPLE_SIZE][SAMPLE_SIZE]; table to find a 2-char sequence's frequency
+  float  mTypicalPositiveRatio;     // = freqSeqs / totalSeqs 
+  PRBool keepEnglishLetter;         //it says if this script contains latin letters
+  const char* charsetName;
+} SequenceModel;
+
+
+class nsSingleByteCharSetProber : public nsCharSetProber{
+public:
+  nsSingleByteCharSetProber(SequenceModel *model){mModel = model; Reset();};
+  const char* GetCharSetName() {return mModel->charsetName;};
+  nsProbingState HandleData(const char* aBuf, PRUint32 aLen);
+  nsProbingState GetState(void) {return mState;};
+  void      Reset(void);
+  float     GetConfidence(void);
+  void      SetOpion() {};
+  PRBool KeepEnglishLetters() {return mModel->keepEnglishLetter;};
+
+#ifdef DEBUG_chardet
+  void  DumpStatus();
+#endif
+
+protected:
+  nsProbingState mState;
+  SequenceModel *mModel;
+
+  //char order of last character
+  unsigned char mLastOrder;
+
+  PRUint32 mTotalSeqs;
+  PRUint32 mSeqCounters[NUMBER_OF_SEQ_CAT];
+
+  PRUint32 mTotalChar;
+  //characters that fall in our sampling range
+  PRUint32 mFreqChar;
+};
+
+
+extern SequenceModel Koi8rModel;
+extern SequenceModel Win1251Model;
+extern SequenceModel Latin5Model;
+extern SequenceModel MacCyrillicModel;
+extern SequenceModel Ibm866Model;
+extern SequenceModel Ibm855Model;
+extern SequenceModel Latin7Model;
+extern SequenceModel Win1253Model;
+extern SequenceModel Latin5BulgarianModel;
+extern SequenceModel Win1251BulgarianModel;
+extern SequenceModel Latin2HungarianModel;
+extern SequenceModel Win1250HungarianModel;
+
+#endif /* nsSingleByteCharSetProber_h__ */

data/ext/libcharguess/cpp/SJISProber.cpp

@@ -0,0 +1,86 @@
+/*
+	libcharguess	-	Guess the encoding/charset of a string
+    Copyright (C) 2003  Stephane Corbe <noubi@users.sourceforge.net>
+	Based on Mozilla sources
+
+    This library is free software; you can redistribute it and/or
+    modify it under the terms of the GNU Lesser General Public
+    License as published by the Free Software Foundation; either
+    version 2.1 of the License, or (at your option) any later version.
+
+    This library is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+    Lesser General Public License for more details.
+
+    You should have received a copy of the GNU Lesser General Public
+    License along with this library; if not, write to the Free Software
+    Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+*/
+
+// for S-JIS encoding, obeserve characteristic:
+// 1, kana character (or hankaku?) often have hight frequency of appereance
+// 2, kana character often exist in group
+// 3, certain combination of kana is never used in japanese language
+
+#include "SJISProber.h"
+
+void  nsSJISProber::Reset(void)
+{
+  mCodingSM->Reset(); 
+  mState = eDetecting;
+  mContextAnalyser.Reset();
+  mDistributionAnalyser.Reset();
+}
+
+nsProbingState nsSJISProber::HandleData(const char* aBuf, PRUint32 aLen)
+{
+  nsSMState codingState;
+
+  for (PRUint32 i = 0; i < aLen; i++)
+  {
+    codingState = mCodingSM->NextState(aBuf[i]);
+    if (codingState == eError)
+    {
+      mState = eNotMe;
+      break;
+    }
+    if (codingState == eItsMe)
+    {
+      mState = eFoundIt;
+      break;
+    }
+    if (codingState == eStart)
+    {
+      PRUint32 charLen = mCodingSM->GetCurrentCharLen();
+      if (i == 0)
+      {
+        mLastChar[1] = aBuf[0];
+        mContextAnalyser.HandleOneChar(mLastChar+2-charLen, charLen);
+        mDistributionAnalyser.HandleOneChar(mLastChar, charLen);
+      }
+      else
+      {
+        mContextAnalyser.HandleOneChar(aBuf+i+1-charLen, charLen);
+        mDistributionAnalyser.HandleOneChar(aBuf+i-1, charLen);
+      }
+    }
+  }
+
+  mLastChar[0] = aBuf[aLen-1];
+
+  if (mState == eDetecting)
+    if (mContextAnalyser.GotEnoughData() && GetConfidence() > SHORTCUT_THRESHOLD)
+      mState = eFoundIt;
+
+  return mState;
+}
+
+float nsSJISProber::GetConfidence(void)
+{
+  float contxtCf = mContextAnalyser.GetConfidence();
+  float distribCf = mDistributionAnalyser.GetConfidence();
+
+  return (contxtCf > distribCf ? contxtCf : distribCf);
+}
+

data/ext/libcharguess/cpp/SJISProber.h

@@ -0,0 +1,60 @@
+/*
+	libcharguess	-	Guess the encoding/charset of a string
+    Copyright (C) 2003  Stephane Corbe <noubi@users.sourceforge.net>
+	Based on Mozilla sources
+
+    This library is free software; you can redistribute it and/or
+    modify it under the terms of the GNU Lesser General Public
+    License as published by the Free Software Foundation; either
+    version 2.1 of the License, or (at your option) any later version.
+
+    This library is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+    Lesser General Public License for more details.
+
+    You should have received a copy of the GNU Lesser General Public
+    License along with this library; if not, write to the Free Software
+    Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+*/
+
+// for S-JIS encoding, obeserve characteristic:
+// 1, kana character (or hankaku?) often have hight frequency of appereance
+// 2, kana character often exist in group
+// 3, certain combination of kana is never used in japanese language
+
+#ifndef nsSJISProber_h__
+#define nsSJISProber_h__
+
+#include "charsetProber.h"
+#include "codingStateMachine.h"
+#include "jpCntx.h"
+#include "charDistribution.h"
+
+
+class nsSJISProber: public nsCharSetProber {
+public:
+  nsSJISProber(void){mCodingSM = new nsCodingStateMachine(&SJISSMModel);
+                      Reset();};
+  virtual ~nsSJISProber(void){delete mCodingSM;};
+  nsProbingState HandleData(const char* aBuf, PRUint32 aLen);
+  const char* GetCharSetName() {return "Shift_JIS";};
+  nsProbingState GetState(void) {return mState;};
+  void      Reset(void);
+  float     GetConfidence(void);
+  void      SetOpion() {};
+
+protected:
+  nsCodingStateMachine* mCodingSM;
+  nsProbingState mState;
+
+  SJISContextAnalysis mContextAnalyser;
+  SJISDistributionAnalysis mDistributionAnalyser;
+
+  char mLastChar[2];
+
+};
+
+
+#endif /* nsSJISProber_h__ */
+

data/ext/libcharguess/cpp/UTF8Prober.cpp

@@ -0,0 +1,75 @@
+/*
+	libcharguess	-	Guess the encoding/charset of a string
+    Copyright (C) 2003  Stephane Corbe <noubi@users.sourceforge.net>
+	Based on Mozilla sources
+
+    This library is free software; you can redistribute it and/or
+    modify it under the terms of the GNU Lesser General Public
+    License as published by the Free Software Foundation; either
+    version 2.1 of the License, or (at your option) any later version.
+
+    This library is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+    Lesser General Public License for more details.
+
+    You should have received a copy of the GNU Lesser General Public
+    License along with this library; if not, write to the Free Software
+    Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+*/
+
+#include "UTF8Prober.h"
+
+void  nsUTF8Prober::Reset(void)
+{
+  mCodingSM->Reset(); 
+  mNumOfMBChar = 0;
+  mState = eDetecting;
+}
+
+nsProbingState nsUTF8Prober::HandleData(const char* aBuf, PRUint32 aLen)
+{
+  nsSMState codingState;
+
+  for (PRUint32 i = 0; i < aLen; i++)
+  {
+    codingState = mCodingSM->NextState(aBuf[i]);
+    if (codingState == eError)
+    {
+      mState = eNotMe;
+      break;
+    }
+    if (codingState == eItsMe)
+    {
+      mState = eFoundIt;
+      break;
+    }
+    if (codingState == eStart)
+    {
+      if (mCodingSM->GetCurrentCharLen() >= 2)
+        mNumOfMBChar++;
+    }
+  }
+
+  if (mState == eDetecting)
+    if (GetConfidence() > SHORTCUT_THRESHOLD)
+      mState = eFoundIt;
+  return mState;
+}
+
+#define ONE_CHAR_PROB   (float)0.50
+
+float nsUTF8Prober::GetConfidence(void)
+{
+  float unlike = (float)0.99;
+
+  if (mNumOfMBChar < 6)
+  {
+    for (PRUint32 i = 0; i < mNumOfMBChar; i++)
+      unlike *= ONE_CHAR_PROB;
+    return (float)1.0 - unlike;
+  }
+  else
+    return (float)0.99;
+}
+