compact_enc_det 0.1.0

data/ext/compact_enc_det/compact_enc_det/compact_enc_det/compact_enc_det.h

@@ -0,0 +1,83 @@
+// Copyright 2016 Google Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+////////////////////////////////////////////////////////////////////////////////
+
+#ifndef COMPACT_ENC_DET_COMPACT_ENC_DET_H_
+#define COMPACT_ENC_DET_COMPACT_ENC_DET_H_
+
+#include "util/encodings/encodings.h"  // for Encoding
+#include "util/languages/languages.h"  // for Language
+
+#include <string.h>
+
+namespace CompactEncDet {
+  // We may want different statistics, depending on whether the text being
+  // identfied is from the web, from email, etc.  This is currently ignored,
+  // except WEB_CORPUS enables ignoring chars inside tags.
+  enum TextCorpusType {
+    WEB_CORPUS,
+    XML_CORPUS,
+    QUERY_CORPUS,       // Use this for vanilla plaintext
+    EMAIL_CORPUS,
+    NUM_CORPA,          // always last
+  };
+
+  // Scan raw bytes and detect most likely encoding
+  // Design goals:
+  //   Skip over big initial stretches of seven-bit ASCII bytes very quickly
+  //   Thread safe
+  //   Works equally well on
+  //    50-byte queries,
+  //    5000-byte email and
+  //    50000-byte web pages
+  // Length 0 input returns ASCII (aka ISO-8859-1 or Latin1)
+  //
+  // Inputs: text and text_length
+  //  web page's url (preferred) or just
+  //    top-level domain name (e.g. "com") or NULL as a hint
+  //  web page's HTTPheader charset= string (e.g. "Latin1") or NULL as a hint
+  //  web page's <meta> tag charset= string (e.g. "utf-8") or NULL as a hint
+  //  an Encoding or UNKNOWN_ENCODING as a hint
+  //  a Language or UNKNOWN_LANGUAGE as a hint
+  //  corpus type from the list above. Currently ignored; may select
+  //    different probability tables in the future
+  //  ignore_7bit if true says to NOT return the pure seven-bit encodings
+  //    ISO-2022-JP (aka JIS), ISO-2022-CN, ISO-2022-KR, HZ, and UTF-7.
+  //    This may save a little scoring time on pure printable ASCII input text
+  // Outputs: bytes_consumed says how much of text_length was actually examined
+  //  is_reliable set true if the returned encoding is at least 2**10 time more
+  //  probable then the second-best encoding
+  // Return value: the most likely encoding for the input text
+  //
+  // Setting ignore_7bit_mail_encodings effectively turns off detection of
+  // UTF-7, HZ, and ISO-2022-xx. It is recommended that this flag be true
+  // when corpus_type is QUERY_CORPUS.
+  Encoding DetectEncoding(
+      const char* text, int text_length, const char* url_hint,
+      const char* http_charset_hint, const char* meta_charset_hint,
+      const int encoding_hint,
+      const Language language_hint,  // User interface lang
+      const TextCorpusType corpus_type, bool ignore_7bit_mail_encodings,
+      int* bytes_consumed, bool* is_reliable);
+
+  // Support functions for unit test program
+  int BackmapEncodingToRankedEncoding(Encoding enc);
+  Encoding TopEncodingOfLangHint(const char* name);
+  Encoding TopEncodingOfTLDHint(const char* name);
+  Encoding TopEncodingOfCharsetHint(const char* name);
+  const char* Version(void);
+}      // End namespace CompactEncDet
+
+#endif  // COMPACT_ENC_DET_COMPACT_ENC_DET_H_

data/ext/compact_enc_det/compact_enc_det/compact_enc_det/compact_enc_det_fuzz_test.cc

@@ -0,0 +1,54 @@
+// Copyright 2016 Google Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+////////////////////////////////////////////////////////////////////////////////
+
+#include <stddef.h>
+#include <stdlib.h>
+#include <memory>
+
+#include "compact_enc_det/compact_enc_det.h"
+#include "util/encodings/encodings.h"
+#include "util/languages/languages.h"
+#include "util/port.h"
+#include "gtest/gtest.h"
+
+namespace {
+
+class CompactEncDetFuzzTest : public testing::Test {};
+
+TEST_F(CompactEncDetFuzzTest, TestRandom) {
+  for (size_t i = 0; i < 16384; ++i) {
+    unsigned int seed = i;
+    srand(seed);
+    size_t length = static_cast<size_t>(rand()) % 1024;
+    std::unique_ptr<char[]> text(new char[length]);
+
+    for (size_t j = 0; j < length; ++j) text[j] = rand();
+
+    int bytes_consumed;
+    bool is_reliable;
+
+    CompactEncDet::DetectEncoding(text.get(), length, nullptr,  // URL hint
+                                  nullptr,                      // HTTP hint
+                                  nullptr,                      // Meta hint
+                                  UNKNOWN_ENCODING,
+                                  UNKNOWN_LANGUAGE,
+                                  CompactEncDet::WEB_CORPUS,
+                                  false,  // Include 7-bit encodings?
+                                  &bytes_consumed, &is_reliable);
+  }
+}
+
+}  // namespace