language_detection 0.0.1

data/ext/cld/encodings/compact_lang_det/letterscript_enum.cc

@@ -0,0 +1,117 @@
+// Copyright (c) 2006-2009 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "encodings/compact_lang_det/letterscript_enum.h"
+
+#include "encodings/compact_lang_det/win/cld_logging.h"
+
+static const char* kUnicodeLScriptNames[ULScript_NUM_SCRIPTS] = {
+  "Common",
+  "Latin",
+  "Greek",
+  "Cyrillic",
+  "Armenian",
+  "Hebrew",
+  "Arabic",
+  "Syriac",
+  "Thaana",
+  "Devanagari",
+  "Bengali",
+  "Gurmukhi",
+  "Gujarati",
+  "Oriya",
+  "Tamil",
+  "Telugu",
+  "Kannada",
+  "Malayalam",
+  "Sinhala",
+  "Thai",
+  "Lao",
+  "Tibetan",
+  "Myanmar",
+  "Georgian",
+  "HanCJK",
+  "Ethiopic",
+  "Cherokee",
+  "Canadian_Aboriginal",
+  "Ogham",
+  "Runic",
+  "Khmer",
+  "Mongolian",
+  "Yi",
+  "Old_Italic",
+  "Gothic",
+  "Deseret",
+  "Inherited",
+  "Tagalog",
+  "Hanunoo",
+  "Buhid",
+  "Tagbanwa",
+  "Limbu",
+  "Tai_Le",
+  "Linear_B",
+  "Ugaritic",
+  "Shavian",
+  "Osmanya",
+  "Cypriot",
+  "Buginese",
+  "Coptic",
+  "New_Tai_Lue",
+  "Glagolitic",
+  "Tifinagh",
+  "Syloti_Nagri",
+  "Old_Persian",
+  "Kharoshthi",
+  "Balinese",
+  "Cuneiform",
+  "Phoenician",
+  "Phags_Pa",
+  "Nko",
+
+  // Unicode 5.1 beta
+  "Sundanese",
+  "Lepcha",
+  "Ol_Chiki",
+  "Vai",
+  "Saurashtra",
+  "Kayah_Li",
+  "Rejang",
+  "Lycian",
+  "Carian",
+  "Lydian",
+  "Cham",
+};
+
+
+// Unicode 5.1 beta script names from
+// http://www.unicode.org/Public/5.1.0/diffs/5.0.0-5.1.0.all.2.diffs
+// NOTE: 'Vai ' => "Vaii" to make four letters, not three
+// see http://unicode.org/iso15924/iso15924-codes.html
+const char* const kLScriptName4[ULScript_NUM_SCRIPTS] = {
+  "Zyyy", "Latn", "Grek", "Cyrl",   "Armn", "Hebr", "Arab", "Syrc",
+  "Thaa", "Deva", "Beng", "Guru",   "Gujr", "Orya", "Taml", "Telu",
+  "Knda", "Mlym", "Sinh", "Thai",   "Laoo", "Tibt", "Mymr", "Geor",
+  "Hani", "Ethi", "Cher", "Cans",   "Ogam", "Runr", "Khmr", "Mong",
+
+  "Yiii", "Ital", "Goth", "Dsrt",   "Zzzz", "Tglg", "Hano", "Buhd",
+  "Tagb", "Limb", "Tale", "Linb",   "Ugar", "Shaw", "Osma", "Cprt",
+  "Bugi", "Copt", "Talu", "Glag",   "Tfng", "Sylo", "Xpeo", "Khar",
+  "Bali", "Xsux", "Phnx", "Phag",   "Nkoo",
+
+  // Unicode 5.1 beta
+  "Sund", "Lepc", "Olck", "Vaii", "Saur", "Kali", "Rjng", "Lyci",
+  "Cari", "Lydi", "Cham",
+};
+
+
+const char* UnicodeLScriptName(const UnicodeLScript ls) {
+  CHECK(ls >= 0 && ls < ULScript_NUM_SCRIPTS);
+  return kUnicodeLScriptNames[ls];
+}
+
+
+const char* UnicodeLScriptCode(const UnicodeLScript ls) {
+  CHECK(ls >= 0 && ls < ULScript_NUM_SCRIPTS);
+  return kLScriptName4[ls];
+}

data/ext/cld/encodings/compact_lang_det/letterscript_enum.h

@@ -0,0 +1,99 @@
+// Copyright (c) 2006-2009 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef ENCODINGS_COMPACT_LANG_DET_LETTERSCRIPT_ENUM_H__
+#define ENCODINGS_COMPACT_LANG_DET_LETTERSCRIPT_ENUM_H__
+
+enum UnicodeLScript {
+  ULScript_Common,
+  ULScript_Latin,
+  ULScript_Greek,
+  ULScript_Cyrillic,
+  ULScript_Armenian,
+  ULScript_Hebrew,
+  ULScript_Arabic,
+  ULScript_Syriac,
+  ULScript_Thaana,
+  ULScript_Devanagari,
+  ULScript_Bengali,
+  ULScript_Gurmukhi,
+  ULScript_Gujarati,
+  ULScript_Oriya,
+  ULScript_Tamil,
+  ULScript_Telugu,
+  ULScript_Kannada,
+  ULScript_Malayalam,
+  ULScript_Sinhala,
+  ULScript_Thai,
+  ULScript_Lao,
+  ULScript_Tibetan,
+  ULScript_Myanmar,
+  ULScript_Georgian,
+  ULScript_HanCJK,
+  ULScript_Ethiopic,
+  ULScript_Cherokee,
+  ULScript_Canadian_Aboriginal,
+  ULScript_Ogham,
+  ULScript_Runic,
+  ULScript_Khmer,
+  ULScript_Mongolian,
+  ULScript_Yi,
+  ULScript_Old_Italic,
+  ULScript_Gothic,
+  ULScript_Deseret,
+  ULScript_Inherited,
+  ULScript_Tagalog,
+  ULScript_Hanunoo,
+  ULScript_Buhid,
+  ULScript_Tagbanwa,
+  ULScript_Limbu,
+  ULScript_Tai_Le,
+  ULScript_Linear_B,
+  ULScript_Ugaritic,
+  ULScript_Shavian,
+  ULScript_Osmanya,
+  ULScript_Cypriot,
+  ULScript_Buginese,
+  ULScript_Coptic,
+  ULScript_New_Tai_Lue,
+  ULScript_Glagolitic,
+  ULScript_Tifinagh,
+  ULScript_Syloti_Nagri,
+  ULScript_Old_Persian,
+  ULScript_Kharoshthi,
+  ULScript_Balinese,
+  ULScript_Cuneiform,
+  ULScript_Phoenician,
+  ULScript_Phags_Pa,
+  ULScript_Nko,
+
+  // Unicode 5.1
+  ULScript_Sundanese,
+  ULScript_Lepcha,
+  ULScript_Ol_Chiki,
+  ULScript_Vai,
+  ULScript_Saurashtra,
+  ULScript_Kayah_Li,
+  ULScript_Rejang,
+  ULScript_Lycian,
+  ULScript_Carian,
+  ULScript_Lydian,
+  ULScript_Cham,
+  ULScript_NUM_SCRIPTS
+};
+
+
+static const UnicodeLScript UNKNOWN_LSCRIPT = ULScript_Common;
+
+
+// Return the name corresponding to the script ls, e.g. "Latin".
+// It is a fatal error if ls is not a valid UnicodeLScript.
+const char* UnicodeLScriptName(const UnicodeLScript ls);
+
+
+// Return the 4-letter code corresponding to the script ls, e.g. "Latn".
+// It is a fatal error if ls is not a valid UnicodeLScript.
+const char* UnicodeLScriptCode(const UnicodeLScript ls);
+
+#endif  // ENCODINGS_COMPACT_LANG_DET_LETTERSCRIPT_ENUM_H__

data/ext/cld/encodings/compact_lang_det/subsetsequence.cc

@@ -0,0 +1,259 @@
+// Copyright (c) 2006-2009 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+// Remember a subset of a sequence of values, using a modest amount of memory
+
+/***
+  Design:
+  Accumulate in powers of three, using 3-way median to collapse entries.
+  At any given time, there is one most-dense (highest power of 3) range of
+  entries and a series of less-dense ranges that hold 0..2 entries each. There
+  is a bounded-size storage array of S cells for all the entries.
+
+  The overflow detect is set up so that a new higher power of 3, K+1, is
+  triggered precisely when range K has 3n entries and all ranges < K have
+  zero entries.
+
+  In general, think of the range sizes as a multi-digit base 3 number, except
+  the highest digit may exceed 2:
+
+  3**6  3**5  3**4  3**3  3**2  3**1  3**0  K=2
+    0     0     0     0   3n-1    2     2   unused:1
+
+  There are a total of 3n-1 + 2 + 2 entries in use. Assume a size limit S at
+  one more than that, and we add a new 3**0 entry and "carry" by performing
+  medians on any group of 3 elements:
+
+  3**6  3**5  3**4  3**3  3**2  3**1  3**0       K=2
+    0     0     0     0   3n-1    2     3        unused:0
+    0     0     0     0   3n-1    3     0 carry  unused:2
+    0     0     0     0    3n     0     0 carry  unused:4
+
+  To accumulate 2 entries at all levels < K and 3 just before the first carry at
+  level 0, we need 2*K + 1 unused cells after doing all carries, or five cells
+  in this case. Since we only have 4 cells in the example above, we need to
+  make room by starting a new power of three:
+
+  3**6  3**5  3**4  3**3  3**2  3**1  3**0
+    0     0     0     0    3n     0     0        K=2 unused:4
+    0     0     0     n     0     0     0        K=3 unused:2n+4
+
+  In the code below, we don't worry about overflow from the topmost place.
+
+
+***/
+
+#include "encodings/compact_lang_det/subsetsequence.h"
+#include <stdio.h>
+
+#include "encodings/compact_lang_det/win/cld_logging.h"
+
+void DumpInts(const char* label, const int* v, int n) {
+  printf("%s ", label);
+  for (int i = 0; i < n; ++i) {
+    printf("%d ", v[i]);
+  }
+  printf("\n");
+}
+
+void DumpUint8s(const char* label, const uint8* v, int n) {
+  printf("%s ", label);
+  for (int i = 0; i < n; ++i) {
+    printf("%d ", v[i]);
+  }
+  printf("\n");
+}
+
+// Return median of seq_[sub] .. seq_[sub+2], favoring middle element
+uint8 SubsetSequence::Median3(int sub) {
+  if (seq_[sub] == seq_[sub + 1]) {
+    return seq_[sub];
+  }
+  if (seq_[sub] == seq_[sub + 2]) {
+    return seq_[sub];
+  }
+  return seq_[sub + 1];
+}
+
+void SubsetSequence::Init() {
+  // printf("Init\n");
+
+  k_ = 0;
+  count_[0] = 0;
+  next_e_ = 0;
+  seq_[0] = 0;    // Default value if no calls to Add
+
+  // Want largest <= kMaxSeq_ that allows reserve and makes count_[k_] = 0 mod 3
+  int reserve = (2 * k_ + 1);
+  level_limit_e_ = kMaxSeq_ - reserve;
+  level_limit_e_ = (level_limit_e_ / 3) * 3;  // Round down to multiple of 3
+  limit_e_ = level_limit_e_;
+}
+
+// Compress level k by 3x, creating level k+1
+void SubsetSequence::NewLevel() {
+  // printf("NewLevel 3 ** %d\n", k_ + 1);
+  //DumpUint8s("count[k]", count_, k_ + 1);
+  //DumpUint8s("seq[next]", seq_, next_e_);
+
+  // Incoming level must be an exact multiple of three in size
+  CHECK((count_[k_] % 3) == 0);
+  int k_size = count_[k_];
+  int new_size = k_size / 3;
+
+  // Compress down by 3x, via median
+  for (int j = 0; j < new_size; ++j) {
+    seq_[j] = Median3(j * 3);
+  }
+
+  // Update counts
+  count_[k_] = 0;
+  // Else Overflow -- just continue with 3x dense Level K
+  if (k_ < (kMaxLevel_ - 1)) {++k_;}
+  count_[k_] = new_size;
+
+  // Update limits
+  next_e_ = new_size;
+  limit_e_ = next_e_ + 3;
+
+  // Want largest <= kMaxSeq_ that allows reserve and makes count_[k_] = 0 mod 3
+  int reserve = (2 * k_ + 1);
+  level_limit_e_ = kMaxSeq_ - reserve;
+  level_limit_e_ = (level_limit_e_ / 3) * 3;  // Round down to multiple of 3
+                                              //
+  //DumpUint8s("after: count[k]", count_, k_ + 1);
+  //DumpUint8s("after: seq[next]", seq_, next_e_);
+}
+
+void SubsetSequence::DoCarries() {
+  CHECK(count_[k_] > 3);    // We depend on count_[k_] being > 3 to stop while
+  // Make room by carrying
+
+  //DumpUint8s("DoCarries count[k]", count_, k_ + 1);
+  //DumpUint8s("DoCarries seq[next]", seq_, next_e_);
+
+  int i = 0;
+  while (count_[i] == 3) {
+    next_e_ -= 3;
+    seq_[next_e_] = Median3(next_e_);
+    ++next_e_;
+    count_[i] = 0;
+    ++count_[i + 1];
+    ++i;
+  }
+  limit_e_ = next_e_ + 3;
+
+  //DumpUint8s("after: DoCarries count[k]", count_, k_ + 1);
+  //DumpUint8s("after: DoCarries seq[next]", seq_, next_e_);
+
+  // If we just fully carried into level K,
+  // Make sure there is now enough room, else start level K + 1
+  if (i >= k_) {
+    CHECK(count_[k_] == next_e_);
+    if (next_e_ >= level_limit_e_) {
+      NewLevel();
+    }
+  }
+}
+
+void SubsetSequence::Add(uint8 e) {
+  // Add an entry then carry as needed
+  seq_[next_e_] = e;
+  ++next_e_;
+  ++count_[0];
+
+  if (next_e_ >= limit_e_) {
+    DoCarries();
+  }
+}
+
+
+// Collapse tail end by simple median across disparate-weight values,
+// dropping or duplicating last value if need be.
+// This routine is idempotent.
+void SubsetSequence::Flush() {
+  // printf("Flush %d\n", count_[k_]);
+  int start_tail = count_[k_];
+  int size_tail = next_e_ - start_tail;
+  if ((size_tail % 3) == 2) {
+    seq_[next_e_] = seq_[next_e_ - 1];      // Duplicate last value
+    ++size_tail;
+  }
+
+  // Compress tail down by 3x, via median
+  int new_size = size_tail / 3;             // May delete last value
+  for (int j = 0; j < new_size; ++j) {
+    seq_[start_tail + j] = Median3(start_tail + j * 3);
+  }
+
+  next_e_ = start_tail + new_size;
+  count_[k_] = next_e_;
+}
+
+
+// Extract representative pattern of exactly N values into dst[0..n-1]
+// This routine may be called multiple times, but it may downsample as a
+// side effect, causing subsequent calls with larger N to get poor answers.
+void SubsetSequence::Extract(int to_n, uint8* dst) {
+  // Collapse partial-carries in tail
+  Flush();
+
+  // Just use Bresenham to resample
+  int from_n = next_e_;
+  if (to_n >= from_n) {
+    // Up-sample from_n => to_n
+    int err = to_n - 1;          // bias toward no overshoot
+    int j = 0;
+    for (int i = 0; i < to_n; ++i) {
+      dst[i] = seq_[j];
+      err -= from_n;
+      if (err < 0) {
+        ++j;
+        err += to_n;
+      }
+    }
+  } else {
+    // Get to the point that the number of samples is <= 3 * to_n
+    while (next_e_ > (to_n * 3)) {
+      // Compress down by 3x, via median
+      // printf("Extract, median %d / 3\n", next_e_);
+      if ((next_e_ % 3) == 2) {
+        seq_[next_e_] = seq_[next_e_ - 1];    // Duplicate last value
+        ++next_e_;
+      }
+      int new_size = next_e_ / 3;             // May delete last value
+      for (int j = 0; j < new_size; ++j) {
+        seq_[j] = Median3(j * 3);
+      }
+      next_e_ = new_size;
+      count_[k_] = next_e_;
+    }
+    from_n = next_e_;
+
+    if (to_n == from_n) {
+      // Copy verbatim
+      for (int i = 0; i < to_n; ++i) {
+        dst[i] = seq_[i];
+      }
+      return;
+    }
+
+    // Down-sample from_n => to_n, using medians
+    int err = 0;          // Bias to immediate median sample
+    int j = 0;
+    for (int i = 0; i < from_n; ++i) {
+      err -= to_n;
+      if (err < 0) {
+        if (i <= (next_e_ - 2)) {
+          dst[j] = Median3(i);
+        } else {
+          dst[j] = seq_[i];
+        }
+        ++j;
+        err += from_n;
+      }
+    }
+  }
+
+}

data/ext/cld/encodings/compact_lang_det/subsetsequence.h

@@ -0,0 +1,44 @@
+// Copyright (c) 2006-2009 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+// Remember a subset of a sequence of values, using a modest amount of memory
+
+#ifndef ENCODINGS_COMPACT_LANG_DET_SUBSETSEQUENCE_H_
+#define ENCODINGS_COMPACT_LANG_DET_SUBSETSEQUENCE_H_
+
+#include "encodings/compact_lang_det/win/cld_basictypes.h"
+#include "encodings/compact_lang_det/win/cld_google.h"
+
+
+class SubsetSequence {
+ public:
+   void Init();
+   void Add(uint8 e);
+   void Extract(int n, uint8* dst);
+   SubsetSequence() {Init();}
+   ~SubsetSequence() {};
+
+ private:
+   uint8 Median3(int sub);
+   void NewLevel();
+   void DoCarries();
+   void Flush();
+
+   static const int kMaxLevel_ = 16;    // 3**16 ~=  43M (3**20 ~= 3.4B)
+   static const int kMaxSeq_ = 128;
+
+   int k_;
+   int next_e_;
+   int limit_e_;
+   int level_limit_e_;
+   uint8 seq_[kMaxSeq_];
+   uint8 count_[kMaxLevel_ + 1];        // +1 allows graceful overflow
+
+   DISALLOW_EVIL_CONSTRUCTORS(SubsetSequence);
+
+   // Require enough room to end up with 40 entries plus carrying space
+   COMPILE_ASSERT(kMaxSeq_ >= (kMaxLevel_ * 2 + 40), kMaxSeq__is_too_small);
+};
+
+#endif  // ENCODINGS_COMPACT_LANG_DET_SUBSETSEQUENCE_H_

data/ext/cld/encodings/compact_lang_det/subsetsequence_unittest.cc

@@ -0,0 +1,99 @@
+// Copyright 2008 Google Inc. All Rights Reserved.
+// Author: dsites@google.com (Dick Sites)
+/*
+#include "testing/base/public/gunit.h"
+#include "testing/lib/strings/overrun_sensitive_memory_block.h"
+#include "cld/encodings/compact_lang_det/subsetsequence.h"
+
+// This always passes. It is just scaffolidng to exercise the subsequence
+// facility, which is likely to get abandoned soon. dsites 2008.11.17
+//
+TEST(SubsetSequence, foo) {
+  uint8 dst[120];
+
+  // Create 120-element vector
+  printf("Creating %d items:\n", 120);
+  SubsetSequence ss;
+  for (int i = 0; i < 120; ++i) {
+    ss.Add(i);
+  }
+
+  // Extract various lengths
+  for (int n = 120; n >= 0; --n) {
+    ss.Extract(n, dst);
+    printf("[%d] ", n);
+    for (int i = 0; i < n; ++i) {
+      printf("%d ", dst[i]);
+    }
+    printf("\n");
+  }
+
+  printf("\n");
+  printf("\n");
+
+  // Create 120-element vector of 7 items each
+  printf("Creating %d items:\n", 120);
+  ss.Init();
+  for (int i = 0; i < 120; ++i) {
+    ss.Add(i / 7);
+  }
+
+  // Extract various lengths
+  for (int n = 120; n >= 0; --n) {
+    ss.Extract(n, dst);
+    printf("[%d] ", n);
+    for (int i = 0; i < n; ++i) {
+      printf("%d ", dst[i]);
+    }
+    printf("\n");
+  }
+
+  printf("\n");
+  printf("\n");
+
+
+  // Create 400 element vector of patterns
+  int nn1 = 400;
+  int divisor = (nn1 + 239) / 240;  // Max inserted value = 240
+  printf("Creating %d items:\n", nn1);
+  ss.Init();
+  for (int i = 0; i < nn1; ++i) {
+    ss.Add(i / divisor);
+  }
+
+  // Extract 12-item summary lengths
+  int n1 = 12;
+  ss.Extract(n1, dst);
+  printf("[%d] ", n1);
+  for (int i = 0; i < n1; ++i) {
+    printf("%d ", dst[i]);
+  }
+  printf("\n");
+
+  printf("\n");
+  printf("\n");
+
+  // Create 10**n element vector of patterns
+  int pow_10 = 1;
+  for (int nn = 0; nn < 9; ++nn) {
+    printf("Creating %d items:\n", pow_10);
+    int divisor = (pow_10 + 239) / 240;  // Max inserted value = 240
+    ss.Init();
+    for (int i = 0; i < pow_10; ++i) {
+      ss.Add(i / divisor);
+    }
+
+    // Extract 12-item summary lengths
+    int n = 12;
+    ss.Extract(n, dst);
+    printf("[%d] ", n);
+    for (int i = 0; i < n; ++i) {
+      printf("%d ", dst[i]);
+    }
+    printf("\n");
+
+    pow_10 *= 10;
+  }
+
+}
+*/