cchardet 0.2.0

data/ext/uchardet/src/nsPkgInt.h

@@ -0,0 +1,89 @@
+/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* ***** BEGIN LICENSE BLOCK *****
+ * Version: MPL 1.1/GPL 2.0/LGPL 2.1
+ *
+ * The contents of this file are subject to the Mozilla Public License Version
+ * 1.1 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * http://www.mozilla.org/MPL/
+ *
+ * Software distributed under the License is distributed on an "AS IS" basis,
+ * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
+ * for the specific language governing rights and limitations under the
+ * License.
+ *
+ * The Original Code is mozilla.org code.
+ *
+ * The Initial Developer of the Original Code is
+ * Netscape Communications Corporation.
+ * Portions created by the Initial Developer are Copyright (C) 1998
+ * the Initial Developer. All Rights Reserved.
+ *
+ * Contributor(s):
+ *
+ * Alternatively, the contents of this file may be used under the terms of
+ * either the GNU General Public License Version 2 or later (the "GPL"), or
+ * the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
+ * in which case the provisions of the GPL or the LGPL are applicable instead
+ * of those above. If you wish to allow use of your version of this file only
+ * under the terms of either the GPL or the LGPL, and not to allow others to
+ * use your version of this file under the terms of the MPL, indicate your
+ * decision by deleting the provisions above and replace them with the notice
+ * and other provisions required by the GPL or the LGPL. If you do not delete
+ * the provisions above, a recipient may use your version of this file under
+ * the terms of any one of the MPL, the GPL or the LGPL.
+ *
+ * ***** END LICENSE BLOCK ***** */
+
+#ifndef nsPkgInt_h__
+#define nsPkgInt_h__
+#include "nscore.h"
+
+typedef enum {
+  eIdxSft4bits  = 3,
+  eIdxSft8bits  = 2,
+  eIdxSft16bits = 1
+} nsIdxSft; 
+
+typedef enum {
+  eSftMsk4bits  = 7,
+  eSftMsk8bits  = 3,
+  eSftMsk16bits = 1
+} nsSftMsk; 
+
+typedef enum {
+  eBitSft4bits  = 2,
+  eBitSft8bits  = 3,
+  eBitSft16bits = 4
+} nsBitSft; 
+
+typedef enum {
+  eUnitMsk4bits  = 0x0000000FL,
+  eUnitMsk8bits  = 0x000000FFL,
+  eUnitMsk16bits = 0x0000FFFFL
+} nsUnitMsk; 
+
+typedef struct nsPkgInt {
+  nsIdxSft  idxsft;
+  nsSftMsk  sftmsk;
+  nsBitSft  bitsft;
+  nsUnitMsk unitmsk;
+  const PRUint32* const data;
+} nsPkgInt;
+
+
+#define PCK16BITS(a,b)            ((PRUint32)(((b) << 16) | (a)))
+
+#define PCK8BITS(a,b,c,d)         PCK16BITS( ((PRUint32)(((b) << 8) | (a))),  \
+                                             ((PRUint32)(((d) << 8) | (c))))
+
+#define PCK4BITS(a,b,c,d,e,f,g,h) PCK8BITS(  ((PRUint32)(((b) << 4) | (a))), \
+                                             ((PRUint32)(((d) << 4) | (c))), \
+                                             ((PRUint32)(((f) << 4) | (e))), \
+                                             ((PRUint32)(((h) << 4) | (g))) )
+
+#define GETFROMPCK(i, c) \
+ (((((c).data)[(i)>>(c).idxsft])>>(((i)&(c).sftmsk)<<(c).bitsft))&(c).unitmsk)
+
+#endif /* nsPkgInt_h__ */
+

data/ext/uchardet/src/nsSBCSGroupProber.cpp

@@ -0,0 +1,343 @@
+/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* ***** BEGIN LICENSE BLOCK *****
+ * Version: MPL 1.1/GPL 2.0/LGPL 2.1
+ *
+ * The contents of this file are subject to the Mozilla Public License Version
+ * 1.1 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * http://www.mozilla.org/MPL/
+ *
+ * Software distributed under the License is distributed on an "AS IS" basis,
+ * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
+ * for the specific language governing rights and limitations under the
+ * License.
+ *
+ * The Original Code is Mozilla Universal charset detector code.
+ *
+ * The Initial Developer of the Original Code is
+ * Netscape Communications Corporation.
+ * Portions created by the Initial Developer are Copyright (C) 2001
+ * the Initial Developer. All Rights Reserved.
+ *
+ * Contributor(s):
+ *          Shy Shalom <shooshX@gmail.com>
+ *
+ * Alternatively, the contents of this file may be used under the terms of
+ * either the GNU General Public License Version 2 or later (the "GPL"), or
+ * the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
+ * in which case the provisions of the GPL or the LGPL are applicable instead
+ * of those above. If you wish to allow use of your version of this file only
+ * under the terms of either the GPL or the LGPL, and not to allow others to
+ * use your version of this file under the terms of the MPL, indicate your
+ * decision by deleting the provisions above and replace them with the notice
+ * and other provisions required by the GPL or the LGPL. If you do not delete
+ * the provisions above, a recipient may use your version of this file under
+ * the terms of any one of the MPL, the GPL or the LGPL.
+ *
+ * ***** END LICENSE BLOCK ***** */
+
+#include <stdio.h>
+#include "prmem.h"
+
+#include "nsSBCharSetProber.h"
+#include "nsSBCSGroupProber.h"
+
+#include "nsHebrewProber.h"
+
+nsSBCSGroupProber::nsSBCSGroupProber()
+{
+  mProbers[0] = new nsSingleByteCharSetProber(&Win1251RussianModel);
+  mProbers[1] = new nsSingleByteCharSetProber(&Koi8rRussianModel);
+  mProbers[2] = new nsSingleByteCharSetProber(&Latin5RussianModel);
+  mProbers[3] = new nsSingleByteCharSetProber(&MacCyrillicRussianModel);
+  mProbers[4] = new nsSingleByteCharSetProber(&Ibm866RussianModel);
+  mProbers[5] = new nsSingleByteCharSetProber(&Ibm855RussianModel);
+
+  mProbers[6] = new nsSingleByteCharSetProber(&Iso_8859_7GreekModel);
+  mProbers[7] = new nsSingleByteCharSetProber(&Windows_1253GreekModel);
+
+  mProbers[8] = new nsSingleByteCharSetProber(&Latin5BulgarianModel);
+  mProbers[9] = new nsSingleByteCharSetProber(&Win1251BulgarianModel);
+
+  nsHebrewProber *hebprober = new nsHebrewProber();
+  // Notice: Any change in these indexes - 10,11,12 must be reflected
+  // in the code below as well.
+  mProbers[10] = hebprober;
+  mProbers[11] = new nsSingleByteCharSetProber(&Win1255Model, PR_FALSE, hebprober); // Logical Hebrew
+  mProbers[12] = new nsSingleByteCharSetProber(&Win1255Model, PR_TRUE, hebprober); // Visual Hebrew
+  // Tell the Hebrew prober about the logical and visual probers
+  if (mProbers[10] && mProbers[11] && mProbers[12]) // all are not null
+  {
+    hebprober->SetModelProbers(mProbers[11], mProbers[12]);
+  }
+  else // One or more is null. avoid any Hebrew probing, null them all
+  {
+    for (PRUint32 i = 10; i <= 12; ++i)
+    {
+      delete mProbers[i];
+      mProbers[i] = 0;
+    }
+  }
+
+  mProbers[13] = new nsSingleByteCharSetProber(&Tis_620ThaiModel);
+  mProbers[14] = new nsSingleByteCharSetProber(&Iso_8859_11ThaiModel);
+
+  mProbers[15] = new nsSingleByteCharSetProber(&Iso_8859_1FrenchModel);
+  mProbers[16] = new nsSingleByteCharSetProber(&Iso_8859_15FrenchModel);
+  mProbers[17] = new nsSingleByteCharSetProber(&Windows_1252FrenchModel);
+
+  mProbers[18] = new nsSingleByteCharSetProber(&Iso_8859_1SpanishModel);
+  mProbers[19] = new nsSingleByteCharSetProber(&Iso_8859_15SpanishModel);
+  mProbers[20] = new nsSingleByteCharSetProber(&Windows_1252SpanishModel);
+
+  mProbers[21] = new nsSingleByteCharSetProber(&Iso_8859_2HungarianModel);
+  mProbers[22] = new nsSingleByteCharSetProber(&Windows_1250HungarianModel);
+
+  mProbers[23] = new nsSingleByteCharSetProber(&Iso_8859_1GermanModel);
+  mProbers[24] = new nsSingleByteCharSetProber(&Windows_1252GermanModel);
+
+  mProbers[25] = new nsSingleByteCharSetProber(&Iso_8859_3EsperantoModel);
+
+  mProbers[26] = new nsSingleByteCharSetProber(&Iso_8859_3TurkishModel);
+  mProbers[27] = new nsSingleByteCharSetProber(&Iso_8859_9TurkishModel);
+
+  mProbers[28] = new nsSingleByteCharSetProber(&Iso_8859_6ArabicModel);
+  mProbers[29] = new nsSingleByteCharSetProber(&Windows_1256ArabicModel);
+
+  mProbers[30] = new nsSingleByteCharSetProber(&VisciiVietnameseModel);
+  mProbers[31] = new nsSingleByteCharSetProber(&Windows_1258VietnameseModel);
+
+  mProbers[32] = new nsSingleByteCharSetProber(&Iso_8859_15DanishModel);
+  mProbers[33] = new nsSingleByteCharSetProber(&Iso_8859_1DanishModel);
+  mProbers[34] = new nsSingleByteCharSetProber(&Windows_1252DanishModel);
+
+  mProbers[35] = new nsSingleByteCharSetProber(&Iso_8859_13LithuanianModel);
+  mProbers[36] = new nsSingleByteCharSetProber(&Iso_8859_10LithuanianModel);
+  mProbers[37] = new nsSingleByteCharSetProber(&Iso_8859_4LithuanianModel);
+
+  mProbers[38] = new nsSingleByteCharSetProber(&Iso_8859_13LatvianModel);
+  mProbers[39] = new nsSingleByteCharSetProber(&Iso_8859_10LatvianModel);
+  mProbers[40] = new nsSingleByteCharSetProber(&Iso_8859_4LatvianModel);
+
+  mProbers[41] = new nsSingleByteCharSetProber(&Iso_8859_1PortugueseModel);
+  mProbers[42] = new nsSingleByteCharSetProber(&Iso_8859_9PortugueseModel);
+  mProbers[43] = new nsSingleByteCharSetProber(&Iso_8859_15PortugueseModel);
+  mProbers[44] = new nsSingleByteCharSetProber(&Windows_1252PortugueseModel);
+
+  mProbers[45] = new nsSingleByteCharSetProber(&Iso_8859_3MalteseModel);
+
+  mProbers[46] = new nsSingleByteCharSetProber(&Windows_1250CzechModel);
+  mProbers[47] = new nsSingleByteCharSetProber(&Iso_8859_2CzechModel);
+  mProbers[48] = new nsSingleByteCharSetProber(&Mac_CentraleuropeCzechModel);
+  mProbers[49] = new nsSingleByteCharSetProber(&Ibm852CzechModel);
+
+  mProbers[50] = new nsSingleByteCharSetProber(&Windows_1250SlovakModel);
+  mProbers[51] = new nsSingleByteCharSetProber(&Iso_8859_2SlovakModel);
+  mProbers[52] = new nsSingleByteCharSetProber(&Mac_CentraleuropeSlovakModel);
+  mProbers[53] = new nsSingleByteCharSetProber(&Ibm852SlovakModel);
+
+  mProbers[54] = new nsSingleByteCharSetProber(&Windows_1250PolishModel);
+  mProbers[55] = new nsSingleByteCharSetProber(&Iso_8859_2PolishModel);
+  mProbers[56] = new nsSingleByteCharSetProber(&Iso_8859_13PolishModel);
+  mProbers[57] = new nsSingleByteCharSetProber(&Iso_8859_16PolishModel);
+  mProbers[58] = new nsSingleByteCharSetProber(&Mac_CentraleuropePolishModel);
+  mProbers[59] = new nsSingleByteCharSetProber(&Ibm852PolishModel);
+
+  mProbers[60] = new nsSingleByteCharSetProber(&Iso_8859_1FinnishModel);
+  mProbers[61] = new nsSingleByteCharSetProber(&Iso_8859_4FinnishModel);
+  mProbers[62] = new nsSingleByteCharSetProber(&Iso_8859_9FinnishModel);
+  mProbers[63] = new nsSingleByteCharSetProber(&Iso_8859_13FinnishModel);
+  mProbers[64] = new nsSingleByteCharSetProber(&Iso_8859_15FinnishModel);
+  mProbers[65] = new nsSingleByteCharSetProber(&Windows_1252FinnishModel);
+
+  mProbers[66] = new nsSingleByteCharSetProber(&Iso_8859_1ItalianModel);
+  mProbers[67] = new nsSingleByteCharSetProber(&Iso_8859_3ItalianModel);
+  mProbers[68] = new nsSingleByteCharSetProber(&Iso_8859_9ItalianModel);
+  mProbers[69] = new nsSingleByteCharSetProber(&Iso_8859_15ItalianModel);
+  mProbers[70] = new nsSingleByteCharSetProber(&Windows_1252ItalianModel);
+
+  mProbers[71] = new nsSingleByteCharSetProber(&Windows_1250CroatianModel);
+  mProbers[72] = new nsSingleByteCharSetProber(&Iso_8859_2CroatianModel);
+  mProbers[73] = new nsSingleByteCharSetProber(&Iso_8859_13CroatianModel);
+  mProbers[74] = new nsSingleByteCharSetProber(&Iso_8859_16CroatianModel);
+  mProbers[75] = new nsSingleByteCharSetProber(&Mac_CentraleuropeCroatianModel);
+  mProbers[76] = new nsSingleByteCharSetProber(&Ibm852CroatianModel);
+
+  mProbers[77] = new nsSingleByteCharSetProber(&Windows_1252EstonianModel);
+  mProbers[78] = new nsSingleByteCharSetProber(&Windows_1257EstonianModel);
+  mProbers[79] = new nsSingleByteCharSetProber(&Iso_8859_4EstonianModel);
+  mProbers[80] = new nsSingleByteCharSetProber(&Iso_8859_13EstonianModel);
+  mProbers[81] = new nsSingleByteCharSetProber(&Iso_8859_15EstonianModel);
+
+  mProbers[82] = new nsSingleByteCharSetProber(&Iso_8859_1IrishModel);
+  mProbers[83] = new nsSingleByteCharSetProber(&Iso_8859_9IrishModel);
+  mProbers[84] = new nsSingleByteCharSetProber(&Iso_8859_15IrishModel);
+  mProbers[85] = new nsSingleByteCharSetProber(&Windows_1252IrishModel);
+
+  mProbers[86] = new nsSingleByteCharSetProber(&Windows_1250RomanianModel);
+  mProbers[87] = new nsSingleByteCharSetProber(&Iso_8859_2RomanianModel);
+  mProbers[88] = new nsSingleByteCharSetProber(&Iso_8859_16RomanianModel);
+  mProbers[89] = new nsSingleByteCharSetProber(&Ibm852RomanianModel);
+
+  mProbers[90] = new nsSingleByteCharSetProber(&Windows_1250SloveneModel);
+  mProbers[91] = new nsSingleByteCharSetProber(&Iso_8859_2SloveneModel);
+  mProbers[92] = new nsSingleByteCharSetProber(&Iso_8859_16SloveneModel);
+  mProbers[93] = new nsSingleByteCharSetProber(&Mac_CentraleuropeSloveneModel);
+  mProbers[94] = new nsSingleByteCharSetProber(&Ibm852SloveneModel);
+
+  mProbers[95] = new nsSingleByteCharSetProber(&Iso_8859_1SwedishModel);
+  mProbers[96] = new nsSingleByteCharSetProber(&Iso_8859_4SwedishModel);
+  mProbers[97] = new nsSingleByteCharSetProber(&Iso_8859_9SwedishModel);
+  mProbers[98] = new nsSingleByteCharSetProber(&Iso_8859_15SwedishModel);
+  mProbers[99] = new nsSingleByteCharSetProber(&Windows_1252SwedishModel);
+
+  Reset();
+}
+
+nsSBCSGroupProber::~nsSBCSGroupProber()
+{
+  for (PRUint32 i = 0; i < NUM_OF_SBCS_PROBERS; i++)
+  {
+    delete mProbers[i];
+  }
+}
+
+
+const char* nsSBCSGroupProber::GetCharSetName()
+{
+  //if we have no answer yet
+  if (mBestGuess == -1)
+  {
+    GetConfidence();
+    //no charset seems positive
+    if (mBestGuess == -1)
+      //we will use default.
+      mBestGuess = 0;
+  }
+  return mProbers[mBestGuess]->GetCharSetName();
+}
+
+const char* nsSBCSGroupProber::GetLanguage()
+{
+  if (mBestGuess == -1)
+  {
+    GetConfidence();
+    if (mBestGuess == -1)
+      mBestGuess = 0;
+  }
+  return mProbers[mBestGuess]->GetLanguage();
+}
+
+void  nsSBCSGroupProber::Reset(void)
+{
+  mActiveNum = 0;
+  for (PRUint32 i = 0; i < NUM_OF_SBCS_PROBERS; i++)
+  {
+    if (mProbers[i]) // not null
+    {
+      mProbers[i]->Reset();
+      mIsActive[i] = PR_TRUE;
+      ++mActiveNum;
+    }
+    else
+      mIsActive[i] = PR_FALSE;
+  }
+  mBestGuess = -1;
+  mState = eDetecting;
+}
+
+
+nsProbingState nsSBCSGroupProber::HandleData(const char* aBuf, PRUint32 aLen)
+{
+  nsProbingState st;
+  PRUint32 i;
+  char *newBuf1 = 0;
+  PRUint32 newLen1 = 0;
+
+  //apply filter to original buffer, and we got new buffer back
+  //depend on what script it is, we will feed them the new buffer 
+  //we got after applying proper filter
+  //this is done without any consideration to KeepEnglishLetters
+  //of each prober since as of now, there are no probers here which
+  //recognize languages with English characters.
+  if (!FilterWithoutEnglishLetters(aBuf, aLen, &newBuf1, newLen1))
+    goto done;
+  
+  if (newLen1 == 0)
+    goto done; // Nothing to see here, move on.
+
+  for (i = 0; i < NUM_OF_SBCS_PROBERS; i++)
+  {
+     if (!mIsActive[i])
+       continue;
+     st = mProbers[i]->HandleData(newBuf1, newLen1);
+     if (st == eFoundIt)
+     {
+       mBestGuess = i;
+       mState = eFoundIt;
+       break;
+     }
+     else if (st == eNotMe)
+     {
+       mIsActive[i] = PR_FALSE;
+       mActiveNum--;
+       if (mActiveNum <= 0)
+       {
+         mState = eNotMe;
+         break;
+       }
+     }
+  }
+
+done:
+  PR_FREEIF(newBuf1);
+
+  return mState;
+}
+
+float nsSBCSGroupProber::GetConfidence(void)
+{
+  PRUint32 i;
+  float bestConf = 0.0, cf;
+
+  switch (mState)
+  {
+  case eFoundIt:
+    return (float)0.99; //sure yes
+  case eNotMe:
+    return (float)0.01;  //sure no
+  default:
+    for (i = 0; i < NUM_OF_SBCS_PROBERS; i++)
+    {
+      if (!mIsActive[i])
+        continue;
+      cf = mProbers[i]->GetConfidence();
+      if (bestConf < cf)
+      {
+        bestConf = cf;
+        mBestGuess = i;
+      }
+    }
+  }
+  return bestConf;
+}
+
+#ifdef DEBUG_chardet
+void nsSBCSGroupProber::DumpStatus()
+{
+  PRUint32 i;
+  float cf;
+  
+  cf = GetConfidence();
+  printf(" SBCS Group Prober --------begin status \r\n");
+  for (i = 0; i < NUM_OF_SBCS_PROBERS; i++)
+  {
+    if (!mIsActive[i])
+      printf("  inactive: [%s] (i.e. confidence is too low).\r\n", mProbers[i]->GetCharSetName());
+    else
+      mProbers[i]->DumpStatus();
+  }
+  printf(" SBCS Group found best match [%s] confidence %f.\r\n",  
+         mProbers[mBestGuess]->GetCharSetName(), cf);
+}
+#endif

data/ext/uchardet/src/nsSBCSGroupProber.h

@@ -0,0 +1,71 @@
+/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* ***** BEGIN LICENSE BLOCK *****
+ * Version: MPL 1.1/GPL 2.0/LGPL 2.1
+ *
+ * The contents of this file are subject to the Mozilla Public License Version
+ * 1.1 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * http://www.mozilla.org/MPL/
+ *
+ * Software distributed under the License is distributed on an "AS IS" basis,
+ * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
+ * for the specific language governing rights and limitations under the
+ * License.
+ *
+ * The Original Code is Mozilla Universal charset detector code.
+ *
+ * The Initial Developer of the Original Code is
+ * Netscape Communications Corporation.
+ * Portions created by the Initial Developer are Copyright (C) 2001
+ * the Initial Developer. All Rights Reserved.
+ *
+ * Contributor(s):
+ *          Shy Shalom <shooshX@gmail.com>
+ *
+ * Alternatively, the contents of this file may be used under the terms of
+ * either the GNU General Public License Version 2 or later (the "GPL"), or
+ * the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
+ * in which case the provisions of the GPL or the LGPL are applicable instead
+ * of those above. If you wish to allow use of your version of this file only
+ * under the terms of either the GPL or the LGPL, and not to allow others to
+ * use your version of this file under the terms of the MPL, indicate your
+ * decision by deleting the provisions above and replace them with the notice
+ * and other provisions required by the GPL or the LGPL. If you do not delete
+ * the provisions above, a recipient may use your version of this file under
+ * the terms of any one of the MPL, the GPL or the LGPL.
+ *
+ * ***** END LICENSE BLOCK ***** */
+
+#ifndef nsSBCSGroupProber_h__
+#define nsSBCSGroupProber_h__
+
+
+#define NUM_OF_SBCS_PROBERS 100
+
+class nsCharSetProber;
+class nsSBCSGroupProber: public nsCharSetProber {
+public:
+  nsSBCSGroupProber();
+  virtual ~nsSBCSGroupProber();
+  nsProbingState HandleData(const char* aBuf, PRUint32 aLen);
+  const char* GetCharSetName();
+  const char* GetLanguage();
+  nsProbingState GetState(void) {return mState;}
+  void      Reset(void);
+  float     GetConfidence(void);
+  void      SetOpion() {}
+
+#ifdef DEBUG_chardet
+  void  DumpStatus();
+#endif
+
+protected:
+  nsProbingState mState;
+  nsCharSetProber* mProbers[NUM_OF_SBCS_PROBERS];
+  PRBool          mIsActive[NUM_OF_SBCS_PROBERS];
+  PRInt32 mBestGuess;
+  PRUint32 mActiveNum;
+};
+
+#endif /* nsSBCSGroupProber_h__ */
+