compact_enc_det 0.1.0

data/ext/compact_enc_det/compact_enc_det/compact_enc_det/compact_enc_det_hint_code.cc

@@ -0,0 +1,169 @@
+// Copyright 2016 Google Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+////////////////////////////////////////////////////////////////////////////////
+
+#include "compact_enc_det/compact_enc_det_hint_code.h"
+
+#include <ctype.h>                      // for isalpha
+#include <string.h>                     // for NULL, memchr, strlen, etc
+
+#include "util/basictypes.h"            // for uint8, uint32
+#include "util/string_util.h"
+
+// Upper to lower, keep digits, everything else to minus '-' (2d)
+static const char kCharsetToLowerTbl[256] = {
+  0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d, 0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,
+  0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d, 0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,
+  0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d, 0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,
+  0x30,0x31,0x32,0x33,0x34,0x35,0x36,0x37, 0x38,0x39,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,
+
+  0x2d,0x61,0x62,0x63,0x64,0x65,0x66,0x67, 0x68,0x69,0x6a,0x6b,0x6c,0x6d,0x6e,0x6f,
+  0x70,0x71,0x72,0x73,0x74,0x75,0x76,0x77, 0x78,0x79,0x7a,0x2d,0x2d,0x2d,0x2d,0x2d,
+  0x2d,0x61,0x62,0x63,0x64,0x65,0x66,0x67, 0x68,0x69,0x6a,0x6b,0x6c,0x6d,0x6e,0x6f,
+  0x70,0x71,0x72,0x73,0x74,0x75,0x76,0x77, 0x78,0x79,0x7a,0x2d,0x2d,0x2d,0x2d,0x2d,
+
+  0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d, 0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,
+  0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d, 0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,
+  0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d, 0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,
+  0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d, 0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,
+
+  0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d, 0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,
+  0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d, 0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,
+  0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d, 0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,
+  0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d, 0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,
+};
+
+
+static const char kIsAlpha[256] = {
+  0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+  0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+  0,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,0,0,0,0,0,
+  0,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,0,0,0,0,0,
+
+  0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+  0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+  0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+  0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+};
+
+static const char kIsDigit[256] = {
+  0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+  0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 1,1,1,1,1,1,1,1, 1,1,0,0,0,0,0,0,
+  0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+  0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+
+  0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+  0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+  0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+  0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+};
+
+static const char* kFakeEncodingName[] = {
+  "FakeEnc100", "FakeEnc101", "FakeEnc102", "FakeEnc103", "FakeEnc104",
+  "FakeEnc105", "FakeEnc106", "FakeEnc107", "FakeEnc108", "FakeEnc109",
+  "FakeEnc110", "FakeEnc111", "FakeEnc112", "FakeEnc113", "FakeEnc114",
+  "FakeEnc115", "FakeEnc116", "FakeEnc117", "FakeEnc118", "FakeEnc119",
+};
+static const char* kFakeEncodingName2[] = {
+  "FakeEnc_0", "FakeEnc_1", "FakeEnc_2", "FakeEnc_3", "FakeEnc_4",
+};
+
+// Return name for extended encoding
+const char* MyEncodingName(Encoding enc) {
+  if (enc < 0) {
+    return "~";
+  }
+  if (enc == ISO_8859_1) {
+    return "Latin1";   // I can't stand "ASCII" for this
+  }
+  if (enc < NUM_ENCODINGS) {
+    return EncodingName(enc);
+  }
+  // allow fake names, for exploration
+  if ((NUM_ENCODINGS <= enc) && (enc < (NUM_ENCODINGS + 4))) {
+    return kFakeEncodingName2[enc - NUM_ENCODINGS];
+  }
+  if ((100 <= enc) && (enc < 120)) {
+    return kFakeEncodingName[enc - 100];
+  }
+  return "~";
+}
+
+
+// Normalize ASCII string to first 4 alphabetic chars and last 4 digit chars
+// Letters are forced to lowercase ASCII
+// Used to normalize charset= values
+string MakeChar44(const string& str) {
+  string res("________");     // eight underscores
+  int l_ptr = 0;
+  int d_ptr = 0;
+  for (uint32 i = 0; i < str.size(); ++i) {
+    uint8 uc = static_cast<uint8>(str[i]);
+    if (kIsAlpha[uc]) {
+      if (l_ptr < 4) {                  // Else ignore
+        res[l_ptr] = kCharsetToLowerTbl[uc];
+        l_ptr++;
+      }
+    } else if (kIsDigit[uc]) {
+      if (d_ptr < 4) {
+        res[4 + d_ptr] = kCharsetToLowerTbl[uc];
+      } else {
+        // Keep last 4 digits by shifting left
+        res[4] = res[5];
+        res[5] = res[6];
+        res[6] = res[7];
+        res[7] = kCharsetToLowerTbl[uc];
+      }
+      d_ptr++;
+    }   // If neither letter nor digit, drop entirely
+  }
+  return res;
+}
+
+// Normalize ASCII string to first 8 alphabetic/digit chars
+// Letters are forced to lowercase ASCII
+// Used to normalize TLD values
+string MakeChar4(const string& str) {
+  string res("____");     // four underscores
+  int l_ptr = 0;
+  for (uint32 i = 0; i < str.size(); ++i) {
+    uint8 uc = static_cast<uint8>(str[i]);
+    if (kIsAlpha[uc] | kIsDigit[uc]) {
+      if (l_ptr < 4) {                  // Else ignore
+        res[l_ptr] = kCharsetToLowerTbl[uc];
+        l_ptr++;
+      }
+    }
+  }
+  return res;
+}
+
+// Normalize ASCII string to first 8 alphabetic/digit chars
+// Letters are forced to lowercase ASCII
+// Used to normalize TLD values
+string MakeChar8(const string& str) {
+  string res("________");     // eight dots
+  int l_ptr = 0;
+  for (uint32 i = 0; i < str.size(); ++i) {
+    uint8 uc = static_cast<uint8>(str[i]);
+    if (kIsAlpha[uc] | kIsDigit[uc]) {
+      if (l_ptr < 8) {                  // Else ignore
+        res[l_ptr] = kCharsetToLowerTbl[uc];
+        l_ptr++;
+      }
+    }
+  }
+  return res;
+}

data/ext/compact_enc_det/compact_enc_det/compact_enc_det/compact_enc_det_hint_code.h

@@ -0,0 +1,45 @@
+// Copyright 2016 Google Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+////////////////////////////////////////////////////////////////////////////////
+
+#ifndef COMPACT_ENC_DET_COMPACT_ENC_DET_HINT_CODE_H_
+#define COMPACT_ENC_DET_COMPACT_ENC_DET_HINT_CODE_H_
+
+#include <string>                        // for string
+
+#include "util/basictypes.h"             // for uint32
+#include "util/encodings/encodings.h"    // for Encoding
+
+using std::string;
+
+// Return name for extended encoding
+const char* MyEncodingName(Encoding enc);
+
+// Normalize ASCII string to first 4 alphabetic chars and last 4 digit chars
+// Letters are forced to lowercase ASCII
+// Used to normalize charset= values
+string MakeChar44(const string& str);
+
+// Normalize ASCII string to first 4 alphabetic/digit chars
+// Letters are forced to lowercase ASCII
+// Used to normalize TLD values
+string MakeChar4(const string& str);
+
+// Normalize ASCII string to first 8 alphabetic/digit chars
+// Letters are forced to lowercase ASCII
+// Used to normalize other values
+string MakeChar8(const string& str);
+
+#endif  // COMPACT_ENC_DET_COMPACT_ENC_DET_HINT_CODE_H_