language_detection 0.0.1

data/ext/cld/encodings/compact_lang_det/win/cld_utf8statetable.h

@@ -0,0 +1,141 @@
+// Copyright (c) 2006-2009 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef ENCODINGS_COMPACT_LANG_DET_WIN_CLD_UTF8STATETABLE_H_
+#define ENCODINGS_COMPACT_LANG_DET_WIN_CLD_UTF8STATETABLE_H_
+
+#if !defined(CLD_WINDOWS)
+
+#include "util/utf8/utf8statetable.h"
+
+#else
+
+#include "encodings/compact_lang_det/win/cld_basictypes.h"
+
+// These four-byte entries compactly encode how many bytes 0..255 to delete
+// in making a string replacement, how many bytes to add 0..255, and the offset
+// 0..64k-1 of the replacement string in remap_string.
+struct RemapEntry {
+  uint8 delete_bytes;
+  uint8 add_bytes;
+  uint16 bytes_offset;
+};
+
+// Exit type codes for state tables. All but the first get stuffed into
+// signed one-byte entries. The first is only generated by executable code.
+// To distinguish from next-state entries, these must be contiguous and
+// all <= kExitNone
+typedef enum {
+  kExitDstSpaceFull = 239,
+  kExitIllegalStructure,  // 240
+  kExitOK,                // 241
+  kExitReject,            // ...
+  kExitReplace1,
+  kExitReplace2,
+  kExitReplace3,
+  kExitReplace21,
+  kExitReplace31,
+  kExitReplace32,
+  kExitReplaceOffset1,
+  kExitReplaceOffset2,
+  kExitReplace1S0,
+  kExitSpecial,
+  kExitDoAgain,
+  kExitRejectAlt,
+  kExitNone               // 255
+} ExitReason;
+
+typedef enum {
+  kExitDstSpaceFull_2 = -32769,
+  kExitIllegalStructure_2,  // -32768
+  kExitOK_2,                // -32767
+  kExitReject_2,            // ...
+  kExitReplace1_2,
+  kExitReplace2_2,
+  kExitReplace3_2,
+  kExitReplace21_2,
+  kExitReplace31_2,
+  kExitReplace32_2,
+  kExitReplaceOffset1_2,
+  kExitReplaceOffset2_2,
+  kExitReplace1S0_2,
+  kExitSpecial_2,
+  kExitDoAgain_2,
+  kExitRejectAlt_2,
+  kExitNone_2               // -32753
+} ExitReason_2;
+
+// This struct represents one entire state table. The three initialized byte
+// areas are state_table, remap_base, and remap_string. state0 and state0_size
+// give the byte offset and length within state_table of the initial state --
+// table lookups are expected to start and end in this state, but for
+// truncated UTF-8 strings, may end in a different state. These allow a quick
+// test for that condition. entry_shift is 8 for tables subscripted by a full
+// byte value and 6 for space-optimized tables subscripted by only six
+// significant bits in UTF-8 continuation bytes.
+typedef struct {
+  const uint32 state0;
+  const uint32 state0_size;
+  const uint32 total_size;
+  const int max_expand;
+  const int entry_shift;
+  const int bytes_per_entry;
+  const uint32 losub;
+  const uint32 hiadd;
+  const uint8* state_table;
+  const RemapEntry* remap_base;
+  const uint8* remap_string;
+  const uint8* fast_state;
+} UTF8StateMachineObj;
+
+// Near-duplicate declaration for tables with two-byte entries
+typedef struct {
+  const uint32 state0;
+  const uint32 state0_size;
+  const uint32 total_size;
+  const int max_expand;
+  const int entry_shift;
+  const int bytes_per_entry;
+  const uint32 losub;
+  const uint32 hiadd;
+  const signed short* state_table;
+  const RemapEntry* remap_base;
+  const uint8* remap_string;
+  const uint8* fast_state;
+} UTF8StateMachineObj_2;
+
+
+typedef UTF8StateMachineObj UTF8PropObj;
+typedef UTF8StateMachineObj UTF8ScanObj;
+typedef UTF8StateMachineObj_2 UTF8PropObj_2;
+
+
+// Look up property of one UTF-8 character and advance over it
+// Return 0 if input length is zero
+// Return 0 and advance one byte if input is ill-formed
+uint8 UTF8GenericProperty(const UTF8PropObj* st,
+                          const uint8** src,
+                          int* srclen);
+
+// BigOneByte versions are needed for tables > 240 states, but most
+// won't need the TwoByte versions.
+
+// Look up property of one UTF-8 character and advance over it
+// Return 0 if input length is zero
+// Return 0 and advance one byte if input is ill-formed
+uint8 UTF8GenericPropertyBigOneByte(const UTF8PropObj* st,
+                          const uint8** src,
+                          int* srclen);
+
+// Scan a UTF-8 stringpiece based on a state table.
+// Always scan complete UTF-8 characters
+// Set number of bytes scanned. Return reason for exiting
+int UTF8GenericScan(const UTF8ScanObj* st,
+                    const uint8* str,
+                    const int len,
+                    int* bytes_consumed);
+
+#endif
+
+#endif  // ENCODINGS_COMPACT_LANG_DET_WIN_CLD_UTF8STATETABLE_H_

data/ext/cld/encodings/compact_lang_det/win/cld_utf8utils.h

@@ -0,0 +1,22 @@
+// Copyright (c) 2006-2009 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef ENCODINGS_COMPACT_LANG_DET_WIN_CLD_UTF8UTILS_H_
+#define ENCODINGS_COMPACT_LANG_DET_WIN_CLD_UTF8UTILS_H_
+
+#include "encodings/compact_lang_det/win/cld_utf8statetable.h"
+
+namespace cld {
+
+// Scan a UTF-8 stringpiece based on a state table.
+// Always scan complete UTF-8 characters
+// Set number of bytes scanned. Return reason for exiting
+int UTF8GenericScan(const UTF8ScanObj* st,
+                    const char* src,
+                    int len,
+                    int* bytes_consumed);
+
+}  // namespace cld
+
+#endif  // ENCODINGS_COMPACT_LANG_DET_WIN_CLD_UTF8UTILS_H_

data/ext/cld/encodings/compact_lang_det/win/cld_utf8utils_google3.cc

@@ -0,0 +1,18 @@
+// Copyright 2009 Google Inc.  All Rights Reserved.
+// Author: alekseys@google.com (Aleksey Shlyapnikov)
+
+// This code is not actually used, it was copied here for the reference only.
+// See cld_htmlutils_windows.cc for Windows version of this code.
+
+#include "cld/encodings/compact_lang_det/win/cld_utf8utils.h"
+
+#include "cld/util/utf8/utf8statetable.h"
+
+namespace cld {
+
+int UTF8GenericScan(const UTF8ScanObj* st, const char* src, int len,
+                    int* bytes_consumed) {
+  return ::UTF8GenericScan(st, StringPiece(src, len), bytes_consumed);
+}
+
+}  // namespace cld

data/ext/cld/encodings/compact_lang_det/win/cld_utf8utils_windows.cc

@@ -0,0 +1,17 @@
+// Copyright (c) 2006-2009 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "encodings/compact_lang_det/win/cld_utf8utils.h"
+
+#include "encodings/compact_lang_det/win/cld_utf8statetable.h"
+
+namespace cld {
+
+int UTF8GenericScan(const UTF8ScanObj* st, const char* src, int len,
+                    int* bytes_consumed) {
+  return ::UTF8GenericScan(st, reinterpret_cast<const uint8*>(src), len,
+                           bytes_consumed);
+}
+
+}  // namespace cld

data/ext/cld/encodings/compact_lang_det/win/normalizedunicodetext.cc

@@ -0,0 +1,172 @@
+// Copyright (c) 2006-2009 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "encodings/compact_lang_det/win/normalizedunicodetext.h"
+
+#include <tchar.h>
+#include <windows.h>
+#include <winnls.h>
+
+#include "encodings/compact_lang_det/win/cld_scopedptr.h"
+
+
+namespace {
+
+// Function prototypes copied from MSDN.
+typedef BOOL (WINAPI *IsNormalizedStringFunction)(NORM_FORM NormForm,
+                                                  LPCWSTR lpSrcString,
+                                                  int cwSrcLength);
+typedef int (WINAPI *NormalizeStringFunction)(NORM_FORM NormForm,
+                                              LPCWSTR lpSrcString,
+                                              int cwSrcLength,
+                                              LPWSTR lpDstString,
+                                              int cwDstLength);
+
+// A class to provide an access to Normaliz.dll functions.
+// New normalization API implemented in Normaliz.dll is available starting
+// from Windows XP SP2, that's why we have to bind to it dynamically.
+class NormalizationAPI {
+ public:
+  // Creates fully initialized NormalizationAPI object.
+  // Loads DLL and binds all referenced functions.
+  NormalizationAPI()
+      : library_(_T("Normaliz.dll")) {
+    if (library_.IsValid()) {
+      is_normalized_string_.Bind(library_.handle(), "IsNormalizedString");
+      normalize_string_.Bind(library_.handle(), "NormalizeString");
+    }
+  }
+
+  // Proxy functions for the ones loaded from DLL.
+  BOOL IsNormalizedString(NORM_FORM NormForm, LPCWSTR lpSrcString,
+                          int cwSrcLength) {
+    if (!is_normalized_string_.IsValid())
+      return FALSE;
+    return is_normalized_string_.function()(NormForm, lpSrcString, cwSrcLength);
+  }
+  int NormalizeString(NORM_FORM NormForm, LPCWSTR lpSrcString, int cwSrcLength,
+                      LPWSTR lpDstString, int cwDstLength) {
+    if (!normalize_string_.IsValid()) {
+      ::SetLastError(ERROR_INVALID_FUNCTION);
+      return 0;
+    }
+    return normalize_string_.function()(NormForm, lpSrcString, cwSrcLength,
+                                        lpDstString, cwDstLength);
+  }
+
+  // Returns true if all functions were bound successfully.
+  // This implies that library_ itself was loaded successfully.
+  bool IsValid() const {
+    return is_normalized_string_.IsValid() && normalize_string_.IsValid();
+  }
+
+ private:
+  // Holds a handle to loaded Normaliz.dll.
+  ScopedLibrary library_;
+  // Pointers to the functions loaded from Normaliz.dll.
+  FunctionFromDll<IsNormalizedStringFunction> is_normalized_string_;
+  FunctionFromDll<NormalizeStringFunction> normalize_string_;
+
+  DISALLOW_COPY_AND_ASSIGN(NormalizationAPI);
+};
+
+static NormalizationAPI normalization_api;
+
+}  // namespace
+
+
+// NormalizedUnicodeText
+
+NormalizedUnicodeText::NormalizedUnicodeText()
+    : normalized_text_(NULL) {
+}
+
+
+DWORD NormalizedUnicodeText::Normalize(NORM_FORM normalization_form,
+                                       const WCHAR* text) {
+  DWORD result = 0;
+  normalized_text_ = TryToNormalizeText(normalization_form, text, &result);
+  return result;
+}
+
+
+const WCHAR* NormalizedUnicodeText::TryToNormalizeText(
+    NORM_FORM normalization_form, const WCHAR* text, DWORD *error_code) {
+  if (!text) {
+    text_.reset();
+    return text;
+  }
+  _ASSERT(NULL != error_code);
+  if (!error_code)
+    return text;
+
+  if (!normalization_api.IsValid()) {
+    // Fall back to the previous version of normalization API.
+    int folded_text_size = ::FoldStringW(MAP_PRECOMPOSED, text, -1, NULL, 0);
+    if (!folded_text_size) {
+      *error_code = ::GetLastError();
+      return text;
+    }
+
+    text_.reset(new WCHAR[folded_text_size]);
+    if (!text_.get()) {
+      *error_code = ERROR_OUTOFMEMORY;
+      return text;
+    }
+
+    int folding_result =
+        ::FoldStringW(MAP_PRECOMPOSED, text, -1, text_.get(), folded_text_size);
+    if (!folding_result) {
+      *error_code = ::GetLastError();
+      text_.reset();
+      return text;
+    }
+
+    return text_.get();
+  }
+
+  // No need to allocate anything when text is already normalized.
+  if (normalization_api.IsNormalizedString(normalization_form, text, -1))
+    return text;
+
+  // Get the first approximation for the buffer size required to store
+  // normalized text.
+  int normalized_text_size_guess =
+      normalization_api.NormalizeString(normalization_form, text, -1, NULL, 0);
+
+  while (normalized_text_size_guess > 0) {
+    text_.reset(new WCHAR[normalized_text_size_guess]);
+    if (!text_.get()) {
+      *error_code = ERROR_OUTOFMEMORY;
+      break;
+    }
+
+    int normalized_text_size =
+        normalization_api.NormalizeString(normalization_form, text, -1,
+                                          text_.get(),
+                                          normalized_text_size_guess);
+
+    if (normalized_text_size > 0) {
+      // Text was successfully converted.
+      return text_.get();
+    }
+
+    if (ERROR_INSUFFICIENT_BUFFER != ::GetLastError()) {
+      *error_code = ::GetLastError();
+      // Text cannot be normalized, use the original.
+      // By the way, ERROR_SUCCESS is a puzzling case.
+      // MSDN says 'The action completed successfully but yielded no results'.
+      // Does this mean that output buffer was not changed?
+      // Anyway, just in case, also return the original text.
+      break;
+    }
+
+    // Try again with the corrected buffer size.
+    normalized_text_size_guess = -normalized_text_size;
+  }
+
+  // Use the original text in case of any problem with normalization.
+  text_.reset();
+  return text;
+}

data/ext/cld/encodings/compact_lang_det/win/normalizedunicodetext.h

@@ -0,0 +1,67 @@
+// Copyright (c) 2006-2009 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef ENCODINGS_COMPACT_LANG_DET_WIN_NORMALIZEDUNICODETEXT_H_
+#define ENCODINGS_COMPACT_LANG_DET_WIN_NORMALIZEDUNICODETEXT_H_
+
+#include <tchar.h>
+#include <windows.h>
+
+#include "encodings/compact_lang_det/win/cld_scopedptr.h"
+
+
+#if (WINVER < 0x0600)
+// Copied from winnls.h, we're not using the latest SDK yet.
+typedef enum _NORM_FORM {
+  NormalizationOther  = 0,
+  NormalizationC = 0x1,
+  NormalizationD = 0x2,
+  NormalizationKC = 0x5,
+  NormalizationKD = 0x6
+} NORM_FORM;
+#endif
+
+
+// Gives you back a normalized version of the input text.  Normalization is
+// performed to the specified form.
+// Instance lifetime should be within the lifetime span of the 'text'.
+class NormalizedUnicodeText {
+ public:
+  // Creates an empty instance of NormalizedUnicodeText.
+  NormalizedUnicodeText();
+
+  // Creates a fully initialized instance of NormalizedUnicodeText.
+  // [in] normalization_form - normalization rule set (see MSDN for details).
+  // [in] text - zero-terminated UTF-16 encoded string.
+  // Returns 0 in case of success, Win32 error code in case of failure.
+  //     In case of failure, get() returns the original text.
+  DWORD Normalize(NORM_FORM normalization_form, const WCHAR* text);
+
+  // Returns pointer to the normalized text.
+  const WCHAR* get() const { return normalized_text_; }
+
+ private:
+  // Normalizes 'text' by the 'normalization_form' rules.
+  // [in] normalization_form - normalization rule set (see MSDN for details).
+  // [in] text - zero-terminated UTF-16 encoded string.
+  // [out] error_code - Win32 error code.
+  const WCHAR* TryToNormalizeText(NORM_FORM normalization_form,
+                                  const WCHAR* text, DWORD *error_code);
+
+  // Pointer to the normalized text.
+  const WCHAR* normalized_text_;
+  // When the source text is already normalized by the requested normalization
+  // form, text_ is not used and normalized_text_ just points to the source
+  // text. When the source text requres normalization, text_ contains normalized
+  // version of the source text and normalized_text_ points to this buffer.
+  // Since CLD requires NormalizationC form and the overwhelming majority of all
+  // texts in the Internet is already normalized to this form, it's expected
+  // that this class will not introduce any runtime memory overhead.
+  scoped_array<WCHAR> text_;
+
+  DISALLOW_COPY_AND_ASSIGN(NormalizedUnicodeText);
+};
+
+
+#endif  // ENCODINGS_COMPACT_LANG_DET_WIN_NORMALIZEDUNICODETEXT_H_

data/ext/cld/encodings/internal/encodings.cc

@@ -0,0 +1,12 @@
+// Copyright (c) 2006-2009 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "encodings/public/encodings.h"
+
+
+// We do not use it, just to please a compiler and minimize ported
+// code changes.
+const char * EncodingName(const Encoding enc) {
+  return "";
+}