cchardet 0.2.0

data/ext/uchardet/src/CMakeLists.txt

@@ -0,0 +1,160 @@
+set(
+	UCHARDET_HEADERS
+	uchardet.h
+)
+
+set(
+	UCHARDET_SOURCES
+	CharDistribution.cpp
+	JpCntx.cpp
+	LangModels/LangArabicModel.cpp
+	LangModels/LangBulgarianModel.cpp
+    LangModels/LangCroatianModel.cpp
+    LangModels/LangCzechModel.cpp
+	LangModels/LangEsperantoModel.cpp
+	LangModels/LangEstonianModel.cpp
+	LangModels/LangFinnishModel.cpp
+	LangModels/LangFrenchModel.cpp
+	LangModels/LangDanishModel.cpp
+	LangModels/LangGermanModel.cpp
+	LangModels/LangGreekModel.cpp
+	LangModels/LangHungarianModel.cpp
+	LangModels/LangHebrewModel.cpp
+	LangModels/LangIrishModel.cpp
+	LangModels/LangItalianModel.cpp
+    LangModels/LangLithuanianModel.cpp
+    LangModels/LangLatvianModel.cpp
+    LangModels/LangMalteseModel.cpp
+    LangModels/LangPolishModel.cpp
+    LangModels/LangPortugueseModel.cpp
+	LangModels/LangRomanianModel.cpp
+	LangModels/LangRussianModel.cpp
+	LangModels/LangSlovakModel.cpp
+    LangModels/LangSloveneModel.cpp
+    LangModels/LangSwedishModel.cpp
+	LangModels/LangSpanishModel.cpp
+	LangModels/LangThaiModel.cpp
+	LangModels/LangTurkishModel.cpp
+	LangModels/LangVietnameseModel.cpp
+	nsHebrewProber.cpp
+	nsCharSetProber.cpp
+	nsBig5Prober.cpp
+	nsEUCJPProber.cpp
+	nsEUCKRProber.cpp
+	nsEUCTWProber.cpp
+	nsEscCharsetProber.cpp
+	nsEscSM.cpp
+	nsGB2312Prober.cpp
+	nsMBCSGroupProber.cpp
+	nsMBCSSM.cpp
+	nsSBCSGroupProber.cpp
+	nsSBCharSetProber.cpp
+	nsSJISProber.cpp
+	nsUTF8Prober.cpp
+	nsLatin1Prober.cpp
+	nsUniversalDetector.cpp
+	uchardet.cpp
+)
+
+set (UCHARDET_LIBRARY libuchardet)
+set (UCHARDET_LIBRARY libuchardet PARENT_SCOPE)
+
+if (BUILD_STATIC AND BUILD_SHARED_LIBS)
+	set (UCHARDET_STATIC_LIBRARY libuchardet_static)
+endif ()
+
+add_definitions(
+	-DVERSION="${UCHARDET_VERSION}"
+	-Wall
+)
+
+if (CMAKE_BUILD_TYPE MATCHES Debug)
+	add_definitions(
+		-O0
+		-g3
+	)
+endif (CMAKE_BUILD_TYPE MATCHES Debug)
+
+add_library(
+	${UCHARDET_LIBRARY}
+	${UCHARDET_SOURCES}
+)
+target_compile_definitions("${UCHARDET_LIBRARY}" PRIVATE BUILDING_UCHARDET)
+if(BUILD_SHARED_LIBS)
+	target_compile_definitions("${UCHARDET_LIBRARY}" PUBLIC UCHARDET_SHARED)
+endif()
+
+if (UCHARDET_STATIC_LIBRARY)
+	add_library(
+		${UCHARDET_STATIC_LIBRARY}
+		STATIC
+		${UCHARDET_SOURCES}
+	)
+	target_compile_definitions("${UCHARDET_STATIC_LIBRARY}" PRIVATE BUILDING_UCHARDET)
+endif (UCHARDET_STATIC_LIBRARY)
+
+set_target_properties(
+	${UCHARDET_LIBRARY}
+	PROPERTIES
+		LINKER_LANGUAGE
+			CXX
+		OUTPUT_NAME
+			${PACKAGE_NAME}
+		VERSION
+			${UCHARDET_VERSION}
+		SOVERSION
+			${UCHARDET_VERSION_MAJOR}
+)
+
+if (UCHARDET_STATIC_LIBRARY)
+	set_target_properties(
+		${UCHARDET_STATIC_LIBRARY}
+		PROPERTIES
+			LINKER_LANGUAGE
+				CXX
+			OUTPUT_NAME
+				${PACKAGE_NAME}
+	)
+endif (UCHARDET_STATIC_LIBRARY)
+
+if (NOT WIN32)
+	install(
+		TARGETS
+			${UCHARDET_LIBRARY}
+		LIBRARY DESTINATION
+			${CMAKE_INSTALL_LIBDIR}
+		ARCHIVE DESTINATION
+			${CMAKE_INSTALL_LIBDIR}
+	)
+else (NOT WIN32)
+	install(
+		TARGETS
+			${UCHARDET_LIBRARY}
+		RUNTIME DESTINATION
+			${CMAKE_INSTALL_BINDIR}
+		ARCHIVE DESTINATION
+			${CMAKE_INSTALL_LIBDIR}
+	)
+endif (NOT WIN32)
+
+if (UCHARDET_STATIC_LIBRARY)
+	install(
+		TARGETS
+			${UCHARDET_STATIC_LIBRARY}
+		ARCHIVE DESTINATION
+			${CMAKE_INSTALL_LIBDIR}
+	)
+endif (UCHARDET_STATIC_LIBRARY)
+
+install(
+	FILES
+		${UCHARDET_HEADERS}
+	DESTINATION
+		${CMAKE_INSTALL_INCLUDEDIR}/${PACKAGE_NAME}
+)
+
+include(symbols.cmake)
+
+if (BUILD_BINARY)
+	add_subdirectory(tools)
+endif (BUILD_BINARY)

data/ext/uchardet/src/CharDistribution.cpp

@@ -0,0 +1,109 @@
+/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* ***** BEGIN LICENSE BLOCK *****
+ * Version: MPL 1.1/GPL 2.0/LGPL 2.1
+ *
+ * The contents of this file are subject to the Mozilla Public License Version
+ * 1.1 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * http://www.mozilla.org/MPL/
+ *
+ * Software distributed under the License is distributed on an "AS IS" basis,
+ * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
+ * for the specific language governing rights and limitations under the
+ * License.
+ *
+ * The Original Code is Mozilla Communicator client code.
+ *
+ * The Initial Developer of the Original Code is
+ * Netscape Communications Corporation.
+ * Portions created by the Initial Developer are Copyright (C) 1998
+ * the Initial Developer. All Rights Reserved.
+ *
+ * Contributor(s):
+ *
+ * Alternatively, the contents of this file may be used under the terms of
+ * either the GNU General Public License Version 2 or later (the "GPL"), or
+ * the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
+ * in which case the provisions of the GPL or the LGPL are applicable instead
+ * of those above. If you wish to allow use of your version of this file only
+ * under the terms of either the GPL or the LGPL, and not to allow others to
+ * use your version of this file under the terms of the MPL, indicate your
+ * decision by deleting the provisions above and replace them with the notice
+ * and other provisions required by the GPL or the LGPL. If you do not delete
+ * the provisions above, a recipient may use your version of this file under
+ * the terms of any one of the MPL, the GPL or the LGPL.
+ *
+ * ***** END LICENSE BLOCK ***** */
+
+#include "CharDistribution.h"
+
+#include "JISFreq.tab"
+#include "Big5Freq.tab"
+#include "EUCKRFreq.tab"
+#include "EUCTWFreq.tab"
+#include "GB2312Freq.tab"
+
+#define SURE_YES 0.99f
+#define SURE_NO  0.01f
+
+//return confidence base on received data
+float CharDistributionAnalysis::GetConfidence(void)
+{ 
+  //if we didn't receive any character in our consideration range, or the
+  // number of frequent characters is below the minimum threshold, return
+  // negative answer
+  if (mTotalChars <= 0 || mFreqChars <= mDataThreshold)
+    return SURE_NO;
+
+  if (mTotalChars != mFreqChars) {
+    float r = mFreqChars / ((mTotalChars - mFreqChars) * mTypicalDistributionRatio);
+
+    if (r < SURE_YES)
+      return r;
+  }
+  //normalize confidence, (we don't want to be 100% sure)
+  return SURE_YES;
+}
+
+EUCTWDistributionAnalysis::EUCTWDistributionAnalysis()
+{
+  mCharToFreqOrder = EUCTWCharToFreqOrder;
+  mTableSize = EUCTW_TABLE_SIZE;
+  mTypicalDistributionRatio = EUCTW_TYPICAL_DISTRIBUTION_RATIO;
+}
+
+EUCKRDistributionAnalysis::EUCKRDistributionAnalysis()
+{
+  mCharToFreqOrder = EUCKRCharToFreqOrder;
+  mTableSize = EUCKR_TABLE_SIZE;
+  mTypicalDistributionRatio = EUCKR_TYPICAL_DISTRIBUTION_RATIO;
+}
+
+GB2312DistributionAnalysis::GB2312DistributionAnalysis()
+{
+  mCharToFreqOrder = GB2312CharToFreqOrder;
+  mTableSize = GB2312_TABLE_SIZE;
+  mTypicalDistributionRatio = GB2312_TYPICAL_DISTRIBUTION_RATIO;
+}
+
+Big5DistributionAnalysis::Big5DistributionAnalysis()
+{
+  mCharToFreqOrder = Big5CharToFreqOrder;
+  mTableSize = BIG5_TABLE_SIZE;
+  mTypicalDistributionRatio = BIG5_TYPICAL_DISTRIBUTION_RATIO;
+}
+
+SJISDistributionAnalysis::SJISDistributionAnalysis()
+{
+  mCharToFreqOrder = JISCharToFreqOrder;
+  mTableSize = JIS_TABLE_SIZE;
+  mTypicalDistributionRatio = JIS_TYPICAL_DISTRIBUTION_RATIO;
+}
+
+EUCJPDistributionAnalysis::EUCJPDistributionAnalysis()
+{
+  mCharToFreqOrder = JISCharToFreqOrder;
+  mTableSize = JIS_TABLE_SIZE;
+  mTypicalDistributionRatio = JIS_TYPICAL_DISTRIBUTION_RATIO;
+}
+

data/ext/uchardet/src/CharDistribution.h

@@ -0,0 +1,242 @@
+/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* ***** BEGIN LICENSE BLOCK *****
+ * Version: MPL 1.1/GPL 2.0/LGPL 2.1
+ *
+ * The contents of this file are subject to the Mozilla Public License Version
+ * 1.1 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * http://www.mozilla.org/MPL/
+ *
+ * Software distributed under the License is distributed on an "AS IS" basis,
+ * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
+ * for the specific language governing rights and limitations under the
+ * License.
+ *
+ * The Original Code is Mozilla Communicator client code.
+ *
+ * The Initial Developer of the Original Code is
+ * Netscape Communications Corporation.
+ * Portions created by the Initial Developer are Copyright (C) 1998
+ * the Initial Developer. All Rights Reserved.
+ *
+ * Contributor(s):
+ *
+ * Alternatively, the contents of this file may be used under the terms of
+ * either the GNU General Public License Version 2 or later (the "GPL"), or
+ * the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
+ * in which case the provisions of the GPL or the LGPL are applicable instead
+ * of those above. If you wish to allow use of your version of this file only
+ * under the terms of either the GPL or the LGPL, and not to allow others to
+ * use your version of this file under the terms of the MPL, indicate your
+ * decision by deleting the provisions above and replace them with the notice
+ * and other provisions required by the GPL or the LGPL. If you do not delete
+ * the provisions above, a recipient may use your version of this file under
+ * the terms of any one of the MPL, the GPL or the LGPL.
+ *
+ * ***** END LICENSE BLOCK ***** */
+
+#ifndef CharDistribution_h__
+#define CharDistribution_h__
+
+#include "nscore.h"
+
+#define ENOUGH_DATA_THRESHOLD 1024
+ 
+#define MINIMUM_DATA_THRESHOLD  4
+
+class CharDistributionAnalysis
+{
+public:
+  CharDistributionAnalysis() {Reset(PR_FALSE);}
+
+  //feed a block of data and do distribution analysis
+  void HandleData(const char* aBuf, PRUint32 aLen) {}
+  
+  //Feed a character with known length
+  void HandleOneChar(const char* aStr, PRUint32 aCharLen)
+  {
+    PRInt32 order;
+
+    //we only care about 2-bytes character in our distribution analysis
+    order = (aCharLen == 2) ? GetOrder(aStr) : -1;
+
+    if (order >= 0)
+    {
+      mTotalChars++;
+      //order is valid
+      if ((PRUint32)order < mTableSize)
+      {
+        if (512 > mCharToFreqOrder[order])
+          mFreqChars++;
+      }
+    }
+  }
+
+  //return confidence base on existing data
+  float GetConfidence(void);
+
+  //Reset analyser, clear any state 
+  void      Reset(PRBool aIsPreferredLanguage) 
+  {
+    mDone = PR_FALSE;
+    mTotalChars = 0;
+    mFreqChars = 0;
+    mDataThreshold = aIsPreferredLanguage ? 0 : MINIMUM_DATA_THRESHOLD;
+  }
+
+  //This function is for future extension. Caller can use this function to control
+  //analyser's behavior
+  void      SetOpion(){}
+
+  //It is not necessary to receive all data to draw conclusion. For charset detection,
+  // certain amount of data is enough
+  PRBool GotEnoughData() {return mTotalChars > ENOUGH_DATA_THRESHOLD;}
+
+protected:
+  //we do not handle character base on its original encoding string, but 
+  //convert this encoding string to a number, here called order.
+  //This allow multiple encoding of a language to share one frequency table 
+  virtual PRInt32 GetOrder(const char* str) {return -1;}
+  
+  //If this flag is set to PR_TRUE, detection is done and conclusion has been made
+  PRBool   mDone;
+
+  //The number of characters whose frequency order is less than 512
+  PRUint32 mFreqChars;
+
+  //Total character encounted.
+  PRUint32 mTotalChars;
+
+  //Number of hi-byte characters needed to trigger detection
+  PRUint32 mDataThreshold;
+
+  //Mapping table to get frequency order from char order (get from GetOrder())
+  const PRInt16  *mCharToFreqOrder;
+
+  //Size of above table
+  PRUint32 mTableSize;
+
+  //This is a constant value varies from language to language, it is used in 
+  //calculating confidence. See my paper for further detail.
+  float    mTypicalDistributionRatio;
+};
+
+
+class EUCTWDistributionAnalysis: public CharDistributionAnalysis
+{
+public:
+  EUCTWDistributionAnalysis();
+protected:
+
+  //for EUC-TW encoding, we are interested
+  //  first  byte range: 0xc4 -- 0xfe
+  //  second byte range: 0xa1 -- 0xfe
+  //no validation needed here. State machine has done that
+  PRInt32 GetOrder(const char* str) 
+  { if ((unsigned char)*str >= (unsigned char)0xc4)  
+      return 94*((unsigned char)str[0]-(unsigned char)0xc4) + (unsigned char)str[1] - (unsigned char)0xa1;
+    else
+      return -1;
+  }
+};
+
+
+class EUCKRDistributionAnalysis : public CharDistributionAnalysis
+{
+public:
+  EUCKRDistributionAnalysis();
+protected:
+  //for euc-KR encoding, we are interested 
+  //  first  byte range: 0xb0 -- 0xfe
+  //  second byte range: 0xa1 -- 0xfe
+  //no validation needed here. State machine has done that
+  PRInt32 GetOrder(const char* str) 
+  { if ((unsigned char)*str >= (unsigned char)0xb0)  
+      return 94*((unsigned char)str[0]-(unsigned char)0xb0) + (unsigned char)str[1] - (unsigned char)0xa1;
+    else
+      return -1;
+  }
+};
+
+class GB2312DistributionAnalysis : public CharDistributionAnalysis
+{
+public:
+  GB2312DistributionAnalysis();
+protected:
+  //for GB2312 encoding, we are interested 
+  //  first  byte range: 0xb0 -- 0xfe
+  //  second byte range: 0xa1 -- 0xfe
+  //no validation needed here. State machine has done that
+  PRInt32 GetOrder(const char* str) 
+  { if ((unsigned char)*str >= (unsigned char)0xb0 && (unsigned char)str[1] >= (unsigned char)0xa1)  
+      return 94*((unsigned char)str[0]-(unsigned char)0xb0) + (unsigned char)str[1] - (unsigned char)0xa1;
+    else
+      return -1;
+  }
+};
+
+
+class Big5DistributionAnalysis : public CharDistributionAnalysis
+{
+public:
+  Big5DistributionAnalysis();
+protected:
+  //for big5 encoding, we are interested 
+  //  first  byte range: 0xa4 -- 0xfe
+  //  second byte range: 0x40 -- 0x7e , 0xa1 -- 0xfe
+  //no validation needed here. State machine has done that
+  PRInt32 GetOrder(const char* str) 
+  { if ((unsigned char)*str >= (unsigned char)0xa4)  
+      if ((unsigned char)str[1] >= (unsigned char)0xa1)
+        return 157*((unsigned char)str[0]-(unsigned char)0xa4) + (unsigned char)str[1] - (unsigned char)0xa1 +63;
+      else
+        return 157*((unsigned char)str[0]-(unsigned char)0xa4) + (unsigned char)str[1] - (unsigned char)0x40;
+    else
+      return -1;
+  }
+};
+
+class SJISDistributionAnalysis : public CharDistributionAnalysis
+{
+public:
+  SJISDistributionAnalysis();
+protected:
+  //for sjis encoding, we are interested 
+  //  first  byte range: 0x81 -- 0x9f , 0xe0 -- 0xfe
+  //  second byte range: 0x40 -- 0x7e,  0x81 -- oxfe
+  //no validation needed here. State machine has done that
+  PRInt32 GetOrder(const char* str) 
+  { 
+    PRInt32 order;
+    if ((unsigned char)*str >= (unsigned char)0x81 && (unsigned char)*str <= (unsigned char)0x9f)  
+      order = 188 * ((unsigned char)str[0]-(unsigned char)0x81);
+    else if ((unsigned char)*str >= (unsigned char)0xe0 && (unsigned char)*str <= (unsigned char)0xef)  
+      order = 188 * ((unsigned char)str[0]-(unsigned char)0xe0 + 31);
+    else
+      return -1;
+    order += (unsigned char)*(str+1) - 0x40;
+    if ((unsigned char)str[1] > (unsigned char)0x7f)
+      order--;
+    return order;
+  }
+};
+
+class EUCJPDistributionAnalysis : public CharDistributionAnalysis
+{
+public:
+  EUCJPDistributionAnalysis();
+protected:
+  //for euc-JP encoding, we are interested 
+  //  first  byte range: 0xa0 -- 0xfe
+  //  second byte range: 0xa1 -- 0xfe
+  //no validation needed here. State machine has done that
+  PRInt32 GetOrder(const char* str) 
+  { if ((unsigned char)*str >= (unsigned char)0xa0)  
+      return 94*((unsigned char)str[0]-(unsigned char)0xa1) + (unsigned char)str[1] - (unsigned char)0xa1;
+    else
+      return -1;
+  }
+};
+
+#endif //CharDistribution_h__
+