cchardet 0.2.0

data/ext/uchardet/src/nsBig5Prober.h

@@ -0,0 +1,75 @@
+/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* ***** BEGIN LICENSE BLOCK *****
+ * Version: MPL 1.1/GPL 2.0/LGPL 2.1
+ *
+ * The contents of this file are subject to the Mozilla Public License Version
+ * 1.1 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * http://www.mozilla.org/MPL/
+ *
+ * Software distributed under the License is distributed on an "AS IS" basis,
+ * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
+ * for the specific language governing rights and limitations under the
+ * License.
+ *
+ * The Original Code is mozilla.org code.
+ *
+ * The Initial Developer of the Original Code is
+ * Netscape Communications Corporation.
+ * Portions created by the Initial Developer are Copyright (C) 1998
+ * the Initial Developer. All Rights Reserved.
+ *
+ * Contributor(s):
+ *
+ * Alternatively, the contents of this file may be used under the terms of
+ * either the GNU General Public License Version 2 or later (the "GPL"), or
+ * the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
+ * in which case the provisions of the GPL or the LGPL are applicable instead
+ * of those above. If you wish to allow use of your version of this file only
+ * under the terms of either the GPL or the LGPL, and not to allow others to
+ * use your version of this file under the terms of the MPL, indicate your
+ * decision by deleting the provisions above and replace them with the notice
+ * and other provisions required by the GPL or the LGPL. If you do not delete
+ * the provisions above, a recipient may use your version of this file under
+ * the terms of any one of the MPL, the GPL or the LGPL.
+ *
+ * ***** END LICENSE BLOCK ***** */
+
+#ifndef nsBig5Prober_h__
+#define nsBig5Prober_h__
+
+#include "nsCharSetProber.h"
+#include "nsCodingStateMachine.h"
+#include "CharDistribution.h"
+
+class nsBig5Prober: public nsCharSetProber {
+public:
+  nsBig5Prober(PRBool aIsPreferredLanguage)
+    :mIsPreferredLanguage(aIsPreferredLanguage) 
+  {mCodingSM = new nsCodingStateMachine(&Big5SMModel); 
+    Reset();}
+  virtual ~nsBig5Prober(void){delete mCodingSM;}
+  nsProbingState HandleData(const char* aBuf, PRUint32 aLen);
+  const char* GetCharSetName() {return "BIG5";}
+  const char* GetLanguage() {return "zh";}
+  nsProbingState GetState(void) {return mState;}
+  void      Reset(void);
+  float     GetConfidence(void);
+  void      SetOpion() {}
+
+protected:
+  void      GetDistribution(PRUint32 aCharLen, const char* aStr);
+  
+  nsCodingStateMachine* mCodingSM;
+  nsProbingState mState;
+
+  //Big5ContextAnalysis mContextAnalyser;
+  Big5DistributionAnalysis mDistributionAnalyser;
+  char mLastChar[2];
+  PRBool mIsPreferredLanguage;
+
+};
+
+
+#endif /* nsBig5Prober_h__ */
+

data/ext/uchardet/src/nsCharSetProber.cpp

@@ -0,0 +1,125 @@
+/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* ***** BEGIN LICENSE BLOCK *****
+ * Version: MPL 1.1/GPL 2.0/LGPL 2.1
+ *
+ * The contents of this file are subject to the Mozilla Public License Version
+ * 1.1 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * http://www.mozilla.org/MPL/
+ *
+ * Software distributed under the License is distributed on an "AS IS" basis,
+ * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
+ * for the specific language governing rights and limitations under the
+ * License.
+ *
+ * The Original Code is Mozilla Universal charset detector code.
+ *
+ * The Initial Developer of the Original Code is
+ * Netscape Communications Corporation.
+ * Portions created by the Initial Developer are Copyright (C) 2001
+ * the Initial Developer. All Rights Reserved.
+ *
+ * Contributor(s):
+ *          Shy Shalom <shooshX@gmail.com>
+ *
+ * Alternatively, the contents of this file may be used under the terms of
+ * either the GNU General Public License Version 2 or later (the "GPL"), or
+ * the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
+ * in which case the provisions of the GPL or the LGPL are applicable instead
+ * of those above. If you wish to allow use of your version of this file only
+ * under the terms of either the GPL or the LGPL, and not to allow others to
+ * use your version of this file under the terms of the MPL, indicate your
+ * decision by deleting the provisions above and replace them with the notice
+ * and other provisions required by the GPL or the LGPL. If you do not delete
+ * the provisions above, a recipient may use your version of this file under
+ * the terms of any one of the MPL, the GPL or the LGPL.
+ *
+ * ***** END LICENSE BLOCK ***** */
+ 
+#include "nsCharSetProber.h"
+#include "prmem.h"
+
+//This filter applies to all scripts which do not use English characters
+PRBool nsCharSetProber::FilterWithoutEnglishLetters(const char* aBuf, PRUint32 aLen, char** newBuf, PRUint32& newLen)
+{
+  char *newptr;
+  char *prevPtr, *curPtr;
+  
+  PRBool meetMSB = PR_FALSE;   
+  newptr = *newBuf = (char*)PR_Malloc(aLen);
+  if (!newptr)
+    return PR_FALSE;
+
+  for (curPtr = prevPtr = (char*)aBuf; curPtr < aBuf+aLen; curPtr++)
+  {
+    if (*curPtr & 0x80)
+    {
+      meetMSB = PR_TRUE;
+    }
+    else if (*curPtr < 'A' || (*curPtr > 'Z' && *curPtr < 'a') || *curPtr > 'z') 
+    {
+      //current char is a symbol, most likely a punctuation. we treat it as segment delimiter
+      if (meetMSB && curPtr > prevPtr) 
+      //this segment contains more than single symbol, and it has upper ASCII, we need to keep it
+      {
+        while (prevPtr < curPtr) *newptr++ = *prevPtr++;  
+        prevPtr++;
+        *newptr++ = ' ';
+        meetMSB = PR_FALSE;
+      }
+      else //ignore current segment. (either because it is just a symbol or just an English word)
+        prevPtr = curPtr+1;
+    }
+  }
+  if (meetMSB && curPtr > prevPtr) 
+    while (prevPtr < curPtr) *newptr++ = *prevPtr++;  
+
+  newLen = (PRUint32) (newptr - *newBuf);
+
+  return PR_TRUE;
+}
+
+//This filter applies to all scripts which contain both English characters and upper ASCII characters.
+PRBool nsCharSetProber::FilterWithEnglishLetters(const char* aBuf, PRUint32 aLen, char** newBuf, PRUint32& newLen)
+{
+  //do filtering to reduce load to probers
+  char *newptr;
+  char *prevPtr, *curPtr;
+  PRBool isInTag = PR_FALSE;
+
+  newptr = *newBuf = (char*)PR_Malloc(aLen);
+  if (!newptr)
+    return PR_FALSE;
+
+  for (curPtr = prevPtr = (char*)aBuf; curPtr < aBuf+aLen; curPtr++)
+  {
+    if (*curPtr == '>')
+      isInTag = PR_FALSE;
+    else if (*curPtr == '<')
+      isInTag = PR_TRUE;
+
+    if (!(*curPtr & 0x80) &&
+        (*curPtr < 'A' || (*curPtr > 'Z' && *curPtr < 'a') || *curPtr > 'z') )
+    {
+      if (curPtr > prevPtr && !isInTag) // Current segment contains more than just a symbol 
+                                        // and it is not inside a tag, keep it.
+      {
+        while (prevPtr < curPtr) *newptr++ = *prevPtr++;  
+        prevPtr++;
+        *newptr++ = ' ';
+      }
+      else
+        prevPtr = curPtr+1;
+    }
+  }
+
+  // If the current segment contains more than just a symbol 
+  // and it is not inside a tag then keep it.
+  if (!isInTag)
+    while (prevPtr < curPtr)
+      *newptr++ = *prevPtr++;  
+
+  newLen = (PRUint32) (newptr - *newBuf);
+
+  return PR_TRUE;
+}

data/ext/uchardet/src/nsCharSetProber.h

@@ -0,0 +1,77 @@
+/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* ***** BEGIN LICENSE BLOCK *****
+ * Version: MPL 1.1/GPL 2.0/LGPL 2.1
+ *
+ * The contents of this file are subject to the Mozilla Public License Version
+ * 1.1 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * http://www.mozilla.org/MPL/
+ *
+ * Software distributed under the License is distributed on an "AS IS" basis,
+ * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
+ * for the specific language governing rights and limitations under the
+ * License.
+ *
+ * The Original Code is Mozilla Universal charset detector code.
+ *
+ * The Initial Developer of the Original Code is
+ * Netscape Communications Corporation.
+ * Portions created by the Initial Developer are Copyright (C) 2001
+ * the Initial Developer. All Rights Reserved.
+ *
+ * Contributor(s):
+ *          Shy Shalom <shooshX@gmail.com>
+ *
+ * Alternatively, the contents of this file may be used under the terms of
+ * either the GNU General Public License Version 2 or later (the "GPL"), or
+ * the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
+ * in which case the provisions of the GPL or the LGPL are applicable instead
+ * of those above. If you wish to allow use of your version of this file only
+ * under the terms of either the GPL or the LGPL, and not to allow others to
+ * use your version of this file under the terms of the MPL, indicate your
+ * decision by deleting the provisions above and replace them with the notice
+ * and other provisions required by the GPL or the LGPL. If you do not delete
+ * the provisions above, a recipient may use your version of this file under
+ * the terms of any one of the MPL, the GPL or the LGPL.
+ *
+ * ***** END LICENSE BLOCK ***** */
+#ifndef nsCharSetProber_h__
+#define nsCharSetProber_h__
+
+#include "nscore.h"
+
+//#define DEBUG_chardet // Uncomment this for debug dump.
+
+typedef enum {
+  eDetecting = 0,   //We are still detecting, no sure answer yet, but caller can ask for confidence.
+  eFoundIt = 1,     //That's a positive answer
+  eNotMe = 2        //Negative answer
+} nsProbingState;
+
+#define SHORTCUT_THRESHOLD      (float)0.95
+
+class nsCharSetProber {
+public:
+  virtual ~nsCharSetProber() {}
+  virtual const char* GetCharSetName() = 0;
+  virtual const char* GetLanguage() = 0;
+  virtual nsProbingState HandleData(const char* aBuf, PRUint32 aLen) = 0;
+  virtual nsProbingState GetState(void) = 0;
+  virtual void      Reset(void)  = 0;
+  virtual float     GetConfidence(void) = 0;
+  virtual void      SetOpion() = 0;
+
+#ifdef DEBUG_chardet
+  virtual void  DumpStatus() {};
+#endif
+
+  // Helper functions used in the Latin1 and Group probers.
+  // both functions Allocate a new buffer for newBuf. This buffer should be 
+  // freed by the caller using PR_FREEIF.
+  // Both functions return PR_FALSE in case of memory allocation failure.
+  static PRBool FilterWithoutEnglishLetters(const char* aBuf, PRUint32 aLen, char** newBuf, PRUint32& newLen);
+  static PRBool FilterWithEnglishLetters(const char* aBuf, PRUint32 aLen, char** newBuf, PRUint32& newLen);
+
+};
+
+#endif /* nsCharSetProber_h__ */

data/ext/uchardet/src/nsCodingStateMachine.h

@@ -0,0 +1,105 @@
+/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* ***** BEGIN LICENSE BLOCK *****
+ * Version: MPL 1.1/GPL 2.0/LGPL 2.1
+ *
+ * The contents of this file are subject to the Mozilla Public License Version
+ * 1.1 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * http://www.mozilla.org/MPL/
+ *
+ * Software distributed under the License is distributed on an "AS IS" basis,
+ * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
+ * for the specific language governing rights and limitations under the
+ * License.
+ *
+ * The Original Code is mozilla.org code.
+ *
+ * The Initial Developer of the Original Code is
+ * Netscape Communications Corporation.
+ * Portions created by the Initial Developer are Copyright (C) 1998
+ * the Initial Developer. All Rights Reserved.
+ *
+ * Contributor(s):
+ *
+ * Alternatively, the contents of this file may be used under the terms of
+ * either the GNU General Public License Version 2 or later (the "GPL"), or
+ * the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
+ * in which case the provisions of the GPL or the LGPL are applicable instead
+ * of those above. If you wish to allow use of your version of this file only
+ * under the terms of either the GPL or the LGPL, and not to allow others to
+ * use your version of this file under the terms of the MPL, indicate your
+ * decision by deleting the provisions above and replace them with the notice
+ * and other provisions required by the GPL or the LGPL. If you do not delete
+ * the provisions above, a recipient may use your version of this file under
+ * the terms of any one of the MPL, the GPL or the LGPL.
+ *
+ * ***** END LICENSE BLOCK ***** */
+#ifndef nsCodingStateMachine_h__
+#define nsCodingStateMachine_h__
+
+#include "nsPkgInt.h"
+
+/* Apart from these 3 generic states, machine states are specific to
+ * each charset prober.
+ */
+#define eStart 0
+#define eError 1
+#define eItsMe 2
+
+#define GETCLASS(c) GETFROMPCK(((unsigned char)(c)), mModel->classTable)
+
+//state machine model
+typedef struct 
+{
+  nsPkgInt classTable;
+  PRUint32 classFactor;
+  nsPkgInt stateTable;
+  const PRUint32* charLenTable;
+  const char* name;
+} SMModel;
+
+class nsCodingStateMachine {
+public:
+  nsCodingStateMachine(const SMModel* sm) : mModel(sm) { mCurrentState = eStart; }
+  PRUint32 NextState(char c){
+    //for each byte we get its class , if it is first byte, we also get byte length
+    PRUint32 byteCls = GETCLASS(c);
+    if (mCurrentState == eStart)
+    { 
+      mCurrentBytePos = 0; 
+      mCurrentCharLen = mModel->charLenTable[byteCls];
+    }
+    //from byte's class and stateTable, we get its next state
+    mCurrentState = GETFROMPCK(mCurrentState * mModel->classFactor + byteCls,
+                               mModel->stateTable);
+    mCurrentBytePos++;
+    return mCurrentState;
+  }
+  PRUint32  GetCurrentCharLen(void) {return mCurrentCharLen;}
+  void      Reset(void) {mCurrentState = eStart;}
+  const char * GetCodingStateMachine() {return mModel->name;}
+
+protected:
+  PRUint32 mCurrentState;
+  PRUint32 mCurrentCharLen;
+  PRUint32 mCurrentBytePos;
+
+  const SMModel *mModel;
+};
+
+extern const SMModel UTF8SMModel;
+extern const SMModel Big5SMModel;
+extern const SMModel EUCJPSMModel;
+extern const SMModel EUCKRSMModel;
+extern const SMModel EUCTWSMModel;
+extern const SMModel GB18030SMModel;
+extern const SMModel SJISSMModel;
+
+
+extern const SMModel HZSMModel;
+extern const SMModel ISO2022CNSMModel;
+extern const SMModel ISO2022JPSMModel;
+extern const SMModel ISO2022KRSMModel;
+
+#endif /* nsCodingStateMachine_h__ */
+

data/ext/uchardet/src/nsEUCJPProber.cpp

@@ -0,0 +1,99 @@
+/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* ***** BEGIN LICENSE BLOCK *****
+ * Version: MPL 1.1/GPL 2.0/LGPL 2.1
+ *
+ * The contents of this file are subject to the Mozilla Public License Version
+ * 1.1 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * http://www.mozilla.org/MPL/
+ *
+ * Software distributed under the License is distributed on an "AS IS" basis,
+ * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
+ * for the specific language governing rights and limitations under the
+ * License.
+ *
+ * The Original Code is mozilla.org code.
+ *
+ * The Initial Developer of the Original Code is
+ * Netscape Communications Corporation.
+ * Portions created by the Initial Developer are Copyright (C) 1998
+ * the Initial Developer. All Rights Reserved.
+ *
+ * Contributor(s):
+ *
+ * Alternatively, the contents of this file may be used under the terms of
+ * either the GNU General Public License Version 2 or later (the "GPL"), or
+ * the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
+ * in which case the provisions of the GPL or the LGPL are applicable instead
+ * of those above. If you wish to allow use of your version of this file only
+ * under the terms of either the GPL or the LGPL, and not to allow others to
+ * use your version of this file under the terms of the MPL, indicate your
+ * decision by deleting the provisions above and replace them with the notice
+ * and other provisions required by the GPL or the LGPL. If you do not delete
+ * the provisions above, a recipient may use your version of this file under
+ * the terms of any one of the MPL, the GPL or the LGPL.
+ *
+ * ***** END LICENSE BLOCK ***** */
+
+// for japanese encoding, obeserve characteristic:
+// 1, kana character (or hankaku?) often have hight frequency of appereance
+// 2, kana character often exist in group
+// 3, certain combination of kana is never used in japanese language
+
+#include "nsEUCJPProber.h"
+
+void  nsEUCJPProber::Reset(void)
+{
+  mCodingSM->Reset(); 
+  mState = eDetecting;
+  mContextAnalyser.Reset(mIsPreferredLanguage);
+  mDistributionAnalyser.Reset(mIsPreferredLanguage);
+}
+
+nsProbingState nsEUCJPProber::HandleData(const char* aBuf, PRUint32 aLen)
+{
+  PRUint32 codingState;
+
+  for (PRUint32 i = 0; i < aLen; i++)
+  {
+    codingState = mCodingSM->NextState(aBuf[i]);
+    if (codingState == eItsMe)
+    {
+      mState = eFoundIt;
+      break;
+    }
+    if (codingState == eStart)
+    {
+      PRUint32 charLen = mCodingSM->GetCurrentCharLen();
+
+      if (i == 0)
+      {
+        mLastChar[1] = aBuf[0];
+        mContextAnalyser.HandleOneChar(mLastChar, charLen);
+        mDistributionAnalyser.HandleOneChar(mLastChar, charLen);
+      }
+      else
+      {
+        mContextAnalyser.HandleOneChar(aBuf+i-1, charLen);
+        mDistributionAnalyser.HandleOneChar(aBuf+i-1, charLen);
+      }
+    }
+  }
+
+  mLastChar[0] = aBuf[aLen-1];
+
+  if (mState == eDetecting)
+    if (mContextAnalyser.GotEnoughData() && GetConfidence() > SHORTCUT_THRESHOLD)
+      mState = eFoundIt;
+
+  return mState;
+}
+
+float nsEUCJPProber::GetConfidence(void)
+{
+  float contxtCf = mContextAnalyser.GetConfidence();
+  float distribCf = mDistributionAnalyser.GetConfidence();
+
+  return (contxtCf > distribCf ? contxtCf : distribCf);
+}
+