cchardet 0.2.0

data/ext/uchardet/src/nsUniversalDetector.h

@@ -0,0 +1,91 @@
+/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* ***** BEGIN LICENSE BLOCK *****
+ * Version: MPL 1.1/GPL 2.0/LGPL 2.1
+ *
+ * The contents of this file are subject to the Mozilla Public License Version
+ * 1.1 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * http://www.mozilla.org/MPL/
+ *
+ * Software distributed under the License is distributed on an "AS IS" basis,
+ * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
+ * for the specific language governing rights and limitations under the
+ * License.
+ *
+ * The Original Code is Mozilla Communicator client code.
+ *
+ * The Initial Developer of the Original Code is
+ * Netscape Communications Corporation.
+ * Portions created by the Initial Developer are Copyright (C) 1998
+ * the Initial Developer. All Rights Reserved.
+ *
+ * Contributor(s):
+ *
+ * Alternatively, the contents of this file may be used under the terms of
+ * either the GNU General Public License Version 2 or later (the "GPL"), or
+ * the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
+ * in which case the provisions of the GPL or the LGPL are applicable instead
+ * of those above. If you wish to allow use of your version of this file only
+ * under the terms of either the GPL or the LGPL, and not to allow others to
+ * use your version of this file under the terms of the MPL, indicate your
+ * decision by deleting the provisions above and replace them with the notice
+ * and other provisions required by the GPL or the LGPL. If you do not delete
+ * the provisions above, a recipient may use your version of this file under
+ * the terms of any one of the MPL, the GPL or the LGPL.
+ *
+ * ***** END LICENSE BLOCK ***** */
+
+#ifndef nsUniversalDetector_h__
+#define nsUniversalDetector_h__
+
+class nsCharSetProber;
+
+#define NUM_OF_CHARSET_PROBERS  3
+
+typedef enum {
+  ePureAscii = 0,
+  eEscAscii  = 1,
+  eHighbyte  = 2
+} nsInputState;
+
+#define NS_FILTER_CHINESE_SIMPLIFIED  0x01
+#define NS_FILTER_CHINESE_TRADITIONAL 0x02
+#define NS_FILTER_JAPANESE            0x04
+#define NS_FILTER_KOREAN              0x08
+#define NS_FILTER_NON_CJK             0x10
+#define NS_FILTER_ALL                 0x1F
+#define NS_FILTER_CHINESE (NS_FILTER_CHINESE_SIMPLIFIED | \
+                           NS_FILTER_CHINESE_TRADITIONAL)
+#define NS_FILTER_CJK (NS_FILTER_CHINESE_SIMPLIFIED | \
+                       NS_FILTER_CHINESE_TRADITIONAL | \
+                       NS_FILTER_JAPANESE | \
+                       NS_FILTER_KOREAN)
+
+class nsUniversalDetector {
+public:
+   nsUniversalDetector(PRUint32 aLanguageFilter);
+   virtual ~nsUniversalDetector();
+   virtual nsresult HandleData(const char* aBuf, PRUint32 aLen);
+   virtual void DataEnd(void);
+
+protected:
+   virtual void Report(const char *encoding,
+                       const char *language,
+                       float       confidence) = 0;
+   virtual void Reset();
+   nsInputState  mInputState;
+   PRBool  mNbspFound;
+   PRBool  mDone;
+   PRBool  mInTag;
+   PRBool  mStart;
+   PRBool  mGotData;
+   char    mLastChar;
+   const char *  mDetectedCharset;
+   PRInt32 mBestGuess;
+   PRUint32 mLanguageFilter;
+
+   nsCharSetProber  *mCharSetProbers[NUM_OF_CHARSET_PROBERS];
+   nsCharSetProber  *mEscCharSetProber;
+};
+
+#endif

data/ext/uchardet/src/nscore.h

@@ -0,0 +1,59 @@
+/* ***** BEGIN LICENSE BLOCK *****
+ * Version: MPL 1.1/GPL 2.0/LGPL 2.1
+ *
+ * The contents of this file are subject to the Mozilla Public License Version
+ * 1.1 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * http://www.mozilla.org/MPL/
+ *
+ * Software distributed under the License is distributed on an "AS IS" basis,
+ * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
+ * for the specific language governing rights and limitations under the
+ * License.
+ *
+ * The Original Code is Mozilla Universal charset detector code.
+ *
+ * The Initial Developer of the Original Code is
+ * Netscape Communications Corporation.
+ * Portions created by the Initial Developer are Copyright (C) 2001
+ * the Initial Developer. All Rights Reserved.
+ *
+ * Contributor(s):
+ *          Kohei TAKETA <k-tak@void.in>
+ *
+ * Alternatively, the contents of this file may be used under the terms of
+ * either the GNU General Public License Version 2 or later (the "GPL"), or
+ * the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
+ * in which case the provisions of the GPL or the LGPL are applicable instead
+ * of those above. If you wish to allow use of your version of this file only
+ * under the terms of either the GPL or the LGPL, and not to allow others to
+ * use your version of this file under the terms of the MPL, indicate your
+ * decision by deleting the provisions above and replace them with the notice
+ * and other provisions required by the GPL or the LGPL. If you do not delete
+ * the provisions above, a recipient may use your version of this file under
+ * the terms of any one of the MPL, the GPL or the LGPL.
+ *
+ * ***** END LICENSE BLOCK ***** */
+#ifndef nsDummyCore_h__
+#define nsDummyCore_h__
+
+typedef bool PRBool;
+typedef int PRInt32;
+typedef unsigned int PRUint32;
+typedef short PRInt16;
+typedef unsigned short PRUint16;
+typedef signed char PRInt8;
+typedef unsigned char PRUint8;
+
+#define PR_FALSE false
+#define PR_TRUE true
+#define nsnull 0
+
+
+enum nsresult
+{
+    NS_OK,
+    NS_ERROR_OUT_OF_MEMORY
+};
+
+#endif

data/ext/uchardet/src/prmem.h

@@ -0,0 +1,49 @@
+/* ***** BEGIN LICENSE BLOCK *****
+ * Version: MPL 1.1/GPL 2.0/LGPL 2.1
+ *
+ * The contents of this file are subject to the Mozilla Public License Version
+ * 1.1 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * http://www.mozilla.org/MPL/
+ *
+ * Software distributed under the License is distributed on an "AS IS" basis,
+ * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
+ * for the specific language governing rights and limitations under the
+ * License.
+ *
+ * The Original Code is Mozilla Universal charset detector code.
+ *
+ * The Initial Developer of the Original Code is
+ * Netscape Communications Corporation.
+ * Portions created by the Initial Developer are Copyright (C) 2001
+ * the Initial Developer. All Rights Reserved.
+ *
+ * Contributor(s):
+ *          Kohei TAKETA <k-tak@void.in>
+ *
+ * Alternatively, the contents of this file may be used under the terms of
+ * either the GNU General Public License Version 2 or later (the "GPL"), or
+ * the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
+ * in which case the provisions of the GPL or the LGPL are applicable instead
+ * of those above. If you wish to allow use of your version of this file only
+ * under the terms of either the GPL or the LGPL, and not to allow others to
+ * use your version of this file under the terms of the MPL, indicate your
+ * decision by deleting the provisions above and replace them with the notice
+ * and other provisions required by the GPL or the LGPL. If you do not delete
+ * the provisions above, a recipient may use your version of this file under
+ * the terms of any one of the MPL, the GPL or the LGPL.
+ *
+ * ***** END LICENSE BLOCK ***** */
+#ifndef nsDummyPrmem_h__
+#define nsDummyPrmem_h__
+
+#include <stdlib.h>
+
+inline void* PR_Malloc(size_t len)
+{
+    return malloc(len);
+}
+
+#define PR_FREEIF(p) if (p) free(p)
+
+#endif

data/ext/uchardet/src/symbols.cmake

@@ -0,0 +1,41 @@
+set(
+	UCHARDET_SYMBOLS
+	uchardet_new
+	uchardet_delete
+	uchardet_handle_data
+	uchardet_data_end
+	uchardet_reset
+	uchardet_get_charset
+    uchardet_get_candidates
+    uchardet_get_encoding
+    uchardet_get_confidence
+    uchardet_get_language
+    uchardet_set_default_weight
+    uchardet_weigh_language
+)
+
+set (LINK_FLAGS "")
+
+if (APPLE)
+	# Create a symbols_list file for the Darwin linker.
+	string(REPLACE ";" "\n_" _symbols "${UCHARDET_SYMBOLS}")
+	set(_symbols_list "${CMAKE_CURRENT_BINARY_DIR}/symbols.list")
+	file(WRITE ${_symbols_list} "_${_symbols}\n")
+
+	set(LINK_FLAGS
+		"${LINK_FLAGS} -Wl,-exported_symbols_list,'${_symbols_list}'")
+elseif (CMAKE_CXX_COMPILER_ID STREQUAL GNU)
+	# Create a version script for the GNU ld.
+	set(_symbols "{ global: ${UCHARDET_SYMBOLS}; local: *; };")
+	set(_version_script "${CMAKE_CURRENT_BINARY_DIR}/version.script")
+	file(WRITE ${_version_script} "${_symbols}\n")
+
+	set(LINK_FLAGS "${LINK_FLAGS} -Wl,--version-script,\"${_version_script}\"")
+endif (APPLE)
+
+set_target_properties(
+	${UCHARDET_LIBRARY}
+	PROPERTIES
+		LINK_FLAGS
+			"${LINK_FLAGS}"
+)

data/ext/uchardet/src/tools/CMakeLists.txt

@@ -0,0 +1,23 @@
+set(
+	UCHARDET_SOURCES
+	uchardet.cpp
+)
+
+set(UCHARDET_BINARY uchardet)
+
+add_executable(
+	${UCHARDET_BINARY}
+	${UCHARDET_SOURCES}
+)
+
+target_link_libraries(
+	${UCHARDET_BINARY}
+	${UCHARDET_LIBRARY}
+)
+
+install(
+	TARGETS
+		${UCHARDET_BINARY}
+	RUNTIME DESTINATION
+		${CMAKE_INSTALL_BINDIR}
+)

data/ext/uchardet/src/tools/uchardet.cpp

@@ -0,0 +1,254 @@
+/* ***** BEGIN LICENSE BLOCK *****
+ * Version: MPL 1.1/GPL 2.0/LGPL 2.1
+ *
+ * The contents of this file are subject to the Mozilla Public License Version
+ * 1.1 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * http://www.mozilla.org/MPL/
+ *
+ * Software distributed under the License is distributed on an "AS IS" basis,
+ * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
+ * for the specific language governing rights and limitations under the
+ * License.
+ *
+ * The Original Code is Mozilla Universal charset detector code.
+ *
+ * The Initial Developer of the Original Code is
+ * Netscape Communications Corporation.
+ * Portions created by the Initial Developer are Copyright (C) 2001
+ * the Initial Developer. All Rights Reserved.
+ *
+ * Contributor(s):
+ *          BYVoid <byvoid.kcp@gmail.com>
+ *
+ * Alternatively, the contents of this file may be used under the terms of
+ * either the GNU General Public License Version 2 or later (the "GPL"), or
+ * the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
+ * in which case the provisions of the GPL or the LGPL are applicable instead
+ * of those above. If you wish to allow use of your version of this file only
+ * under the terms of either the GPL or the LGPL, and not to allow others to
+ * use your version of this file under the terms of the MPL, indicate your
+ * decision by deleting the provisions above and replace them with the notice
+ * and other provisions required by the GPL or the LGPL. If you do not delete
+ * the provisions above, a recipient may use your version of this file under
+ * the terms of any one of the MPL, the GPL or the LGPL.
+ *
+ * ***** END LICENSE BLOCK ***** */
+#include "../uchardet.h"
+#include <cstdio>
+#include <cstring>
+#include <cstdlib>
+#include <getopt.h>
+#include <iostream>
+#include <stdio.h>
+#include <string.h>
+
+#ifndef VERSION
+#define VERSION "Unknown"
+#endif
+#define BUFFER_SIZE 65536
+
+char buffer[BUFFER_SIZE];
+
+void detect(uchardet_t  handle,
+            FILE       *fp,
+            bool        verbose)
+{
+    while (!feof(fp))
+    {
+        size_t len = fread(buffer, 1, BUFFER_SIZE, fp);
+        int retval = uchardet_handle_data(handle, buffer, len);
+        if (retval != 0)
+        {
+            fprintf(stderr, "Handle data error.\n");
+            exit(1);
+        }
+    }
+    uchardet_data_end(handle);
+
+    if (verbose)
+    {
+        size_t candidates = uchardet_get_candidates(handle);
+        size_t i;
+
+        printf("\n");
+        for (i = 0; i < candidates; i++)
+        {
+            if (uchardet_get_language(handle, i))
+                printf("\t%s / %s (%f)\n",
+                        uchardet_get_encoding(handle, i),
+                        uchardet_get_language(handle, i),
+                        uchardet_get_confidence(handle, i));
+            else
+                printf("\t%s (%f)\n",
+                        uchardet_get_encoding(handle, i),
+                        uchardet_get_confidence(handle, i));
+        }
+    }
+    else
+    {
+        const char * charset = uchardet_get_encoding(handle, 0);
+        if (*charset)
+            printf("%s\n", charset);
+        else
+            printf("unknown\n");
+    }
+
+    uchardet_reset(handle);
+}
+
+void show_version()
+{
+    printf("\n");
+    printf("uchardet Command Line Tool\n");
+    printf("Version %s\n", VERSION);
+    printf("\n");
+    printf("Authors: %s\n", "BYVoid, Jehan");
+    printf("Bug Report: %s\n", "https://gitlab.freedesktop.org/uchardet/uchardet/-/issues");
+    printf("\n");
+}
+
+void show_usage()
+{
+    show_version();
+    printf("Usage:\n");
+    printf(" uchardet [Options] [File]...\n");
+    printf("\n");
+    printf("Options:\n");
+    printf(" -v, --version         Print version and build information.\n");
+    printf(" -h, --help            Print this help.\n");
+    printf(" -V, --verbose         Show all candidates and their confidence value.\n");
+    printf(" -w, --weight          Tweak language weights.\n");
+    printf("\n");
+}
+
+int main(int argc, char ** argv)
+{
+    uchardet_t handle;
+    static struct option longopts[] =
+    {
+        { "version", no_argument, NULL, 'v' },
+        { "help", no_argument, NULL, 'h' },
+        { "verbose", no_argument, NULL, 'V' },
+        { "weight", required_argument, NULL, 'w' },
+        { 0, 0, 0, 0 },
+    };
+    bool end_options        = false;
+    bool ignore_next_option = false;
+    bool verbose            = false;
+
+    static int oc;
+
+    handle = uchardet_new();
+    while((oc = getopt_long(argc, argv, "vhVw:", longopts, NULL)) != -1)
+    {
+        switch (oc)
+        {
+        case 'v':
+            show_version();
+            uchardet_delete(handle);
+            return 0;
+        case 'h':
+            show_usage();
+            uchardet_delete(handle);
+            return 0;
+        case 'V':
+            verbose = true;
+            break;
+        case 'w':
+            {
+                char *lang_weight;
+                char *saveptr;
+                char *comma;
+
+                lang_weight = strtok_r (optarg, ",", &saveptr);
+                do
+                {
+                    comma = strchr (lang_weight, ':');
+                    if (! comma)
+                    {
+                        printf("-w format is lang1:weight1,lang2:weight2...\n");
+                        uchardet_delete(handle);
+                        return 1;
+                    }
+                    *comma = '\0';
+                    uchardet_weigh_language(handle, lang_weight, strtof (comma + 1, NULL));
+                }
+                while ((lang_weight = strtok_r (NULL, ",", &saveptr)));
+            }
+            break;
+        case '?':
+            printf("Please use %s --help.\n", argv[0]);
+            uchardet_delete(handle);
+            return 1;
+        }
+    }
+
+    FILE * f = stdin;
+    int error_seen = 0;
+    if (argc < 2 ||
+        (argc == 2 && strcmp(argv[1], "--") == 0))
+    {
+        // No file arg, use stdin by default
+        detect(handle, f, verbose);
+    }
+    for (int i = 1; i < argc; i++)
+    {
+        const char *filename = argv[i];
+
+        if (ignore_next_option)
+        {
+            ignore_next_option = false;
+            continue;
+        }
+        else if (! end_options && strcmp(filename, "--") == 0)
+        {
+            end_options = true;
+            continue;
+        }
+
+        if (! end_options)
+        {
+            if (strcmp(filename, "-V") == 0 ||
+                strcmp(filename, "--verbose") == 0)
+            {
+                continue;
+            }
+            else if (strcmp(filename, "-w") == 0 ||
+                     strcmp(filename, "--weight") == 0)
+            {
+                ignore_next_option = true;
+                continue;
+            }
+            else if (*filename == '-' &&
+                     (*(filename + 1) == 'w' ||
+                      (*(filename + 1) == '-' && *(filename + 2) == 'w')))
+            {
+                /* Some ugly trick to recognize -wlang:weight as well as
+                 * --weight=lang:weight patterns.
+                 * Obviously assuming that we have no other long option
+                 * starting with 'w'. If we end up having one, this
+                 * should be updated.
+                 */
+                continue;
+            }
+        }
+
+        f = fopen(filename, "r");
+        if (f == NULL)
+        {
+            perror(filename);
+            error_seen = 1;
+            continue;
+        }
+        if (argc > 2)
+        {
+            printf("%s: ", filename);
+        }
+        detect(handle, f, verbose);
+    }
+
+    uchardet_delete(handle);
+
+    return error_seen;
+}