charguess 1.0

data/ext/libcharguess/cpp/charsetProber.h

@@ -0,0 +1,50 @@
+/*
+	libcharguess	-	Guess the encoding/charset of a string
+    Copyright (C) 2003  Stephane Corbe <noubi@users.sourceforge.net>
+	Based on Mozilla sources
+
+    This library is free software; you can redistribute it and/or
+    modify it under the terms of the GNU Lesser General Public
+    License as published by the Free Software Foundation; either
+    version 2.1 of the License, or (at your option) any later version.
+
+    This library is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+    Lesser General Public License for more details.
+
+    You should have received a copy of the GNU Lesser General Public
+    License along with this library; if not, write to the Free Software
+    Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+*/
+
+#ifndef nsCharSetProber_h__
+#define nsCharSetProber_h__
+
+#include "types.h"
+
+typedef enum {
+  eDetecting = 0,   //we are still detecting, no sure answer yet, but caller can ask for confidence.
+  eFoundIt = 1,     //That's a positive answer
+  eNotMe = 2        //negative answer
+} nsProbingState;
+
+#define SHORTCUT_THRESHOLD      (float)0.95
+
+class nsCharSetProber {
+public:
+  virtual ~nsCharSetProber() {};
+  virtual const char* GetCharSetName() = 0;
+  virtual nsProbingState HandleData(const char* aBuf, PRUint32 aLen) = 0;
+  virtual nsProbingState GetState(void) = 0;
+  virtual void      Reset(void)  = 0;
+  virtual float     GetConfidence(void) = 0;
+  virtual void      SetOpion() = 0;
+
+#ifdef DEBUG_chardet
+  virtual void  DumpStatus() {};
+#endif
+};
+
+#endif /* nsCharSetProber_h__ */
+

data/ext/libcharguess/cpp/codingStateMachine.h

@@ -0,0 +1,92 @@
+/*
+	libcharguess	-	Guess the encoding/charset of a string
+    Copyright (C) 2003  Stephane Corbe <noubi@users.sourceforge.net>
+	Based on Mozilla sources
+
+    This library is free software; you can redistribute it and/or
+    modify it under the terms of the GNU Lesser General Public
+    License as published by the Free Software Foundation; either
+    version 2.1 of the License, or (at your option) any later version.
+
+    This library is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+    Lesser General Public License for more details.
+
+    You should have received a copy of the GNU Lesser General Public
+    License along with this library; if not, write to the Free Software
+    Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+*/
+
+#ifndef nsCodingStateMachine_h__
+#define nsCodingStateMachine_h__
+
+#include "pkgInt.h"
+
+typedef enum {
+   eStart = 0,
+   eError = 1,
+   eItsMe = 2 
+} nsSMState;
+
+#define GETCLASS(c) GETFROMPCK(((unsigned char)(c)), mModel->classTable)
+
+//state machine model
+typedef struct 
+{
+  nsPkgInt classTable;
+  PRUint32 classFactor;
+  nsPkgInt stateTable;
+  const PRUint32* charLenTable;
+  const char* name;
+} SMModel;
+
+class nsCodingStateMachine {
+public:
+  nsCodingStateMachine(SMModel* sm){
+          mCurrentState = eStart;
+          mModel = sm;
+        };
+  nsSMState NextState(char c){
+    //for each byte we get its class , if it is first byte, we also get byte length
+    PRUint32 byteCls = GETCLASS(c);
+    if (mCurrentState == eStart)
+    { 
+      mCurrentBytePos = 0; 
+      mCurrentCharLen = mModel->charLenTable[byteCls];
+    }
+    //from byte's class and stateTable, we get its next state
+    mCurrentState=(nsSMState)GETFROMPCK(mCurrentState*(mModel->classFactor)+byteCls,
+                                       mModel->stateTable);
+    mCurrentBytePos++;
+    return mCurrentState;
+  };
+  PRUint32  GetCurrentCharLen(void) {return mCurrentCharLen;};
+  void      Reset(void) {mCurrentState = eStart;};
+  const char * GetCodingStateMachine() {return mModel->name;};
+
+protected:
+  nsSMState mCurrentState;
+  PRUint32 mCurrentCharLen;
+  PRUint32 mCurrentBytePos;
+
+  SMModel *mModel;
+};
+
+extern SMModel UTF8SMModel;
+extern SMModel Big5SMModel;
+extern SMModel EUCJPSMModel;
+extern SMModel EUCKRSMModel;
+extern SMModel EUCTWSMModel;
+extern SMModel GB18030SMModel;
+extern SMModel SJISSMModel;
+extern SMModel UCS2BESMModel;
+
+
+extern SMModel HZSMModel;
+extern SMModel ISO2022CNSMModel;
+extern SMModel ISO2022JPSMModel;
+extern SMModel ISO2022KRSMModel;
+
+#endif /* nsCodingStateMachine_h__ */
+

data/ext/libcharguess/cpp/config.h

@@ -0,0 +1,36 @@
+/* config.h.  Generated by configure.  */
+/* config.h.in.  Generated from configure.in by autoheader.  */
+
+/* Name of package */
+#define PACKAGE "libcharguess"
+
+/* Define to the address where bug reports for this package should be sent. */
+#define PACKAGE_BUGREPORT ""
+
+/* Define to the full name of this package. */
+#define PACKAGE_NAME ""
+
+/* Define which directory should be used for package source. */
+#define PACKAGE_SOURCE_DIR "/Users/hydrus/charguess/ext/libcharguess/cpp"
+
+/* Define to the full name and version of this package. */
+#define PACKAGE_STRING ""
+
+/* Define to the one symbol short name of this package. */
+#define PACKAGE_TARNAME ""
+
+/* Define to the version of this package. */
+#define PACKAGE_VERSION ""
+
+/* Define to 1 if you have the ANSI C header files. */
+#define STDC_HEADERS 1
+
+/* Version number of package */
+#define VERSION "1.0b"
+
+/* Define to empty if `const' does not conform to ANSI C. */
+/* #undef const */
+
+/* Define as `__inline' if that's what the C compiler calls it, or to nothing
+   if it is not supported. */
+/* #undef inline */

data/ext/libcharguess/cpp/config.h.in

@@ -0,0 +1,35 @@
+/* config.h.in.  Generated from configure.in by autoheader.  */
+
+/* Name of package */
+#undef PACKAGE
+
+/* Define to the address where bug reports for this package should be sent. */
+#undef PACKAGE_BUGREPORT
+
+/* Define to the full name of this package. */
+#undef PACKAGE_NAME
+
+/* Define which directory should be used for package source. */
+#undef PACKAGE_SOURCE_DIR
+
+/* Define to the full name and version of this package. */
+#undef PACKAGE_STRING
+
+/* Define to the one symbol short name of this package. */
+#undef PACKAGE_TARNAME
+
+/* Define to the version of this package. */
+#undef PACKAGE_VERSION
+
+/* Define to 1 if you have the ANSI C header files. */
+#undef STDC_HEADERS
+
+/* Version number of package */
+#undef VERSION
+
+/* Define to empty if `const' does not conform to ANSI C. */
+#undef const
+
+/* Define as `__inline' if that's what the C compiler calls it, or to nothing
+   if it is not supported. */
+#undef inline