cld-fixed 0.7.1

data/ext/cld/encodings/compact_lang_det/subsetsequence.h

@@ -0,0 +1,44 @@
+// Copyright (c) 2006-2009 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+// Remember a subset of a sequence of values, using a modest amount of memory
+
+#ifndef ENCODINGS_COMPACT_LANG_DET_SUBSETSEQUENCE_H_
+#define ENCODINGS_COMPACT_LANG_DET_SUBSETSEQUENCE_H_
+
+#include "encodings/compact_lang_det/win/cld_basictypes.h"
+#include "encodings/compact_lang_det/win/cld_google.h"
+
+
+class SubsetSequence {
+ public:
+   void Init();
+   void Add(uint8 e);
+   void Extract(int n, uint8* dst);
+   SubsetSequence() {Init();}
+   ~SubsetSequence() {};
+
+ private:
+   uint8 Median3(int sub);
+   void NewLevel();
+   void DoCarries();
+   void Flush();
+
+   static const int kMaxLevel_ = 16;    // 3**16 ~=  43M (3**20 ~= 3.4B)
+   static const int kMaxSeq_ = 128;
+
+   int k_;
+   int next_e_;
+   int limit_e_;
+   int level_limit_e_;
+   uint8 seq_[kMaxSeq_];
+   uint8 count_[kMaxLevel_ + 1];        // +1 allows graceful overflow
+
+   DISALLOW_EVIL_CONSTRUCTORS(SubsetSequence);
+
+   // Require enough room to end up with 40 entries plus carrying space
+   COMPILE_ASSERT(kMaxSeq_ >= (kMaxLevel_ * 2 + 40), kMaxSeq__is_too_small);
+};
+
+#endif  // ENCODINGS_COMPACT_LANG_DET_SUBSETSEQUENCE_H_

data/ext/cld/encodings/compact_lang_det/subsetsequence_unittest.cc

@@ -0,0 +1,99 @@
+// Copyright 2008 Google Inc. All Rights Reserved.
+// Author: dsites@google.com (Dick Sites)
+/*
+#include "testing/base/public/gunit.h"
+#include "testing/lib/strings/overrun_sensitive_memory_block.h"
+#include "cld/encodings/compact_lang_det/subsetsequence.h"
+
+// This always passes. It is just scaffolidng to exercise the subsequence
+// facility, which is likely to get abandoned soon. dsites 2008.11.17
+//
+TEST(SubsetSequence, foo) {
+  uint8 dst[120];
+
+  // Create 120-element vector
+  printf("Creating %d items:\n", 120);
+  SubsetSequence ss;
+  for (int i = 0; i < 120; ++i) {
+    ss.Add(i);
+  }
+
+  // Extract various lengths
+  for (int n = 120; n >= 0; --n) {
+    ss.Extract(n, dst);
+    printf("[%d] ", n);
+    for (int i = 0; i < n; ++i) {
+      printf("%d ", dst[i]);
+    }
+    printf("\n");
+  }
+
+  printf("\n");
+  printf("\n");
+
+  // Create 120-element vector of 7 items each
+  printf("Creating %d items:\n", 120);
+  ss.Init();
+  for (int i = 0; i < 120; ++i) {
+    ss.Add(i / 7);
+  }
+
+  // Extract various lengths
+  for (int n = 120; n >= 0; --n) {
+    ss.Extract(n, dst);
+    printf("[%d] ", n);
+    for (int i = 0; i < n; ++i) {
+      printf("%d ", dst[i]);
+    }
+    printf("\n");
+  }
+
+  printf("\n");
+  printf("\n");
+
+
+  // Create 400 element vector of patterns
+  int nn1 = 400;
+  int divisor = (nn1 + 239) / 240;  // Max inserted value = 240
+  printf("Creating %d items:\n", nn1);
+  ss.Init();
+  for (int i = 0; i < nn1; ++i) {
+    ss.Add(i / divisor);
+  }
+
+  // Extract 12-item summary lengths
+  int n1 = 12;
+  ss.Extract(n1, dst);
+  printf("[%d] ", n1);
+  for (int i = 0; i < n1; ++i) {
+    printf("%d ", dst[i]);
+  }
+  printf("\n");
+
+  printf("\n");
+  printf("\n");
+
+  // Create 10**n element vector of patterns
+  int pow_10 = 1;
+  for (int nn = 0; nn < 9; ++nn) {
+    printf("Creating %d items:\n", pow_10);
+    int divisor = (pow_10 + 239) / 240;  // Max inserted value = 240
+    ss.Init();
+    for (int i = 0; i < pow_10; ++i) {
+      ss.Add(i / divisor);
+    }
+
+    // Extract 12-item summary lengths
+    int n = 12;
+    ss.Extract(n, dst);
+    printf("[%d] ", n);
+    for (int i = 0; i < n; ++i) {
+      printf("%d ", dst[i]);
+    }
+    printf("\n");
+
+    pow_10 *= 10;
+  }
+
+}
+*/

data/ext/cld/encodings/compact_lang_det/tote.cc

@@ -0,0 +1,299 @@
+// Copyright (c) 2006-2009 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "encodings/compact_lang_det/tote.h"
+#include <string.h>     // memset
+
+#include "encodings/compact_lang_det/win/cld_logging.h"
+
+
+// Take a set of <key, value> pairs and tote them up.
+// After explicitly sorting, retrieve top key, value pairs
+Tote::Tote() {
+  gram_count_ = 0;
+  incr_count_ = 0;
+  byte_count_ = 0;
+  memset(key_, 0, sizeof(key_));
+  // No need to initialize values
+}
+
+Tote::~Tote() {
+}
+
+void Tote::Reinit() {
+  gram_count_ = 0;
+  incr_count_ = 0;
+  byte_count_ = 0;
+  memset(key_, 0, sizeof(key_));
+  // No need to initialize values
+}
+
+// Increment count of quadgrams/trigrams/unigrams scored
+void Tote::AddGram() {
+  ++gram_count_;
+}
+
+// Three-way associative, guaranteeing that the largest two counts are always
+// in the data structure. kMaxSize must be a multiple of 3, and is tied to the
+// subscript calculations here, which are for 8 sets of 3-way associative
+// buckets. The subscripts for set N are [N], [N+8], and [N+16] used in a
+// slightly-weird way: The initial probe point is [N] or [N+8], whichever
+// is specified by key mod 16. In most cases (nearly *all* cases except Latin
+// script), this entry matches and we update/return. The second probe is
+// the other of [N] and [N+8]. The third probe is only used as a fallback to
+// these two, and is there only for the rare case that there are three or more
+// languages with Language enum values equal mod 8, contending within the same
+// bucket. This can only happen in Latin and (rarely) Cyrillic scripts, because
+// the other scripts have fewer than 17 languages total.
+// If you change kMaxSize, change the constants 7/8/15/16 below
+void Tote::Add(uint8 ikey, int idelta) {
+  DCHECK(ikey != 0);
+  ++incr_count_;
+
+  // Look for existing entry
+  int sub0 = ikey & 15;
+  if (key_[sub0] == ikey) {
+    value_[sub0] += idelta;
+    return;
+  }
+  int sub1 = sub0 ^ 8;
+  if (key_[sub1] == ikey) {
+    value_[sub1] += idelta;
+    return;
+  }
+  int sub2 = (ikey & 7) + 16;
+  if (key_[sub2] == ikey) {
+    value_[sub2] += idelta;
+    return;
+  }
+
+  // Allocate new entry
+  int alloc = -1;
+  if (key_[sub0] == 0) {
+    alloc = sub0;
+  } else if (key_[sub1] == 0) {
+    alloc = sub1;
+  } else if (key_[sub2] == 0) {
+    alloc = sub2;
+  } else {
+    // All choices allocated, need to replace smallest one
+    alloc = sub0;
+    if (value_[sub1] < value_[alloc]) {alloc = sub1;}
+    if (value_[sub2] < value_[alloc]) {alloc = sub2;}
+  }
+  key_[alloc] = ikey;
+  value_[alloc] = idelta;
+  return;
+}
+
+// Return current top key
+int Tote::CurrentTopKey() {
+  int top_key = 0;
+  int top_value = -1;
+  for (int sub = 0; sub < kMaxSize_; ++sub) {
+    if (key_[sub] == 0) {continue;}
+    if (top_value < value_[sub]) {
+      top_value = value_[sub];
+      top_key = key_[sub];
+    }
+  }
+  return top_key;
+}
+
+
+// Sort first n entries by decreasing order of value
+// If key==0 other fields are not valid, treat value as -1
+void Tote::Sort(int n) {
+  // This is n**2, but n is small
+  for (int sub = 0; sub < n; ++sub) {
+    if (key_[sub] == 0) {value_[sub] = -1;}
+
+    // Bubble sort key[sub] and entry[sub]
+    for (int sub2 = sub + 1; sub2 < kMaxSize_; ++sub2) {
+      if (key_[sub2] == 0) {value_[sub2] = -1;}
+      if (value_[sub] < value_[sub2]) {
+        // swap
+        uint8 tmpk = key_[sub];
+        key_[sub] = key_[sub2];
+        key_[sub2] = tmpk;
+        int tmpv = value_[sub];
+        value_[sub] = value_[sub2];
+        value_[sub2] = tmpv;
+      }
+    }
+  }
+}
+
+void Tote::Dump(FILE* f) {
+  for (int sub = 0; sub < kMaxSize_; ++sub) {
+    if (key_[sub] > 0) {
+      fprintf(f, "[%2d] %3d %8d\n", sub, key_[sub], value_[sub]);
+    }
+  }
+  fprintf(f, "%d %d %d\n", gram_count_, incr_count_, byte_count_);
+}
+
+
+
+
+// Take a set of <key, value> pairs and tote them up.
+// After explicitly sorting, retrieve top key, value pairs
+ToteWithReliability::ToteWithReliability() {
+  // No need to initialize score_ or value_
+  incr_count_ = 0;
+  sorted_ = 0;
+  memset(closepair_, 0, sizeof(closepair_));
+  memset(key_, 0, sizeof(key_));
+}
+
+ToteWithReliability::~ToteWithReliability() {
+}
+
+void ToteWithReliability::Reinit() {
+  // No need to initialize score_ or value_
+  incr_count_ = 0;
+  sorted_ = 0;
+  memset(closepair_, 0, sizeof(closepair_));
+  memset(key_, 0, sizeof(key_));
+  ////ss_.Init();
+}
+
+// Weight reliability by ibytes
+// Also see three-way associative comments above for Tote
+void ToteWithReliability::Add(uint8 ikey, int ibytes,
+                              int score, int ireliability) {
+  DCHECK(ikey != 0);
+  CHECK(sorted_ == 0);
+  ++incr_count_;
+
+  // Look for existing entry
+  int sub0 = ikey & 15;
+  if (key_[sub0] == ikey) {
+    value_[sub0] += ibytes;
+    score_[sub0] += score;
+    reliability_[sub0] += ireliability * ibytes;
+    return;
+  }
+  int sub1 = sub0 ^ 8;
+  if (key_[sub1] == ikey) {
+    value_[sub1] += ibytes;
+    score_[sub1] += score;
+    reliability_[sub1] += ireliability * ibytes;
+    return;
+  }
+  int sub2 = (ikey & 7) + 16;
+  if (key_[sub2] == ikey) {
+    value_[sub2] += ibytes;
+    score_[sub2] += score;
+    reliability_[sub2] += ireliability * ibytes;
+    return;
+  }
+
+  // Allocate new entry
+  int alloc = -1;
+  if (key_[sub0] == 0) {
+    alloc = sub0;
+  } else if (key_[sub1] == 0) {
+    alloc = sub1;
+  } else if (key_[sub2] == 0) {
+    alloc = sub2;
+  } else {
+    // All choices allocated, need to replace smallest one
+    alloc = sub0;
+    if (value_[sub1] < value_[alloc]) {alloc = sub1;}
+    if (value_[sub2] < value_[alloc]) {alloc = sub2;}
+  }
+  key_[alloc] = ikey;
+  value_[alloc] = ibytes;
+  score_[alloc] = score;
+  reliability_[alloc] = ireliability * ibytes;
+  return;
+}
+
+// Find subscript of a given packed language, or -1
+int ToteWithReliability::Find(uint8 ikey) {
+  DCHECK(ikey != 0);
+
+  if (sorted_) {
+    // Linear search if sorted
+    for (int sub = 0; sub < kMaxSize_; ++sub) {
+      if (key_[sub] == ikey) {return sub;}
+    }
+    return -1;
+  }
+
+  // Look for existing entry
+  int sub0 = ikey & 15;
+  if (key_[sub0] == ikey) {
+    return sub0;
+  }
+  int sub1 = sub0 ^ 8;
+  if (key_[sub1] == ikey) {
+    return sub1;
+  }
+  int sub2 = (ikey & 7) + 16;
+  if (key_[sub2] == ikey) {
+    return sub2;
+  }
+
+  return -1;
+}
+
+// Return current top key
+int ToteWithReliability::CurrentTopKey() {
+  int top_key = 0;
+  int top_value = -1;
+  for (int sub = 0; sub < kMaxSize_; ++sub) {
+    if (key_[sub] == 0) {continue;}
+    if (top_value < value_[sub]) {
+      top_value = value_[sub];
+      top_key = key_[sub];
+    }
+  }
+  return top_key;
+}
+
+
+// Sort first n entries by decreasing order of value
+// If key==0 other fields are not valid, treat value as -1
+void ToteWithReliability::Sort(int n) {
+  // This is n**2, but n is small
+  for (int sub = 0; sub < n; ++sub) {
+    if (key_[sub] == 0) {value_[sub] = -1;}
+
+    // Bubble sort key[sub] and entry[sub]
+    for (int sub2 = sub + 1; sub2 < kMaxSize_; ++sub2) {
+      if (key_[sub2] == 0) {value_[sub2] = -1;}
+      if (value_[sub] < value_[sub2]) {
+        // swap
+        uint8 tmpk = key_[sub];
+        key_[sub] = key_[sub2];
+        key_[sub2] = tmpk;
+
+        int tmpv = value_[sub];
+        value_[sub] = value_[sub2];
+        value_[sub2] = tmpv;
+
+        double tmps = score_[sub];
+        score_[sub] = score_[sub2];
+        score_[sub2] = tmps;
+
+        int tmpr = reliability_[sub];
+        reliability_[sub] = reliability_[sub2];
+        reliability_[sub2] = tmpr;
+      }
+    }
+  }
+  sorted_ = 1;
+}
+
+void ToteWithReliability::Dump(FILE* f) {
+  for (int sub = 0; sub < kMaxSize_; ++sub) {
+    if (key_[sub] > 0) {
+      fprintf(f, "[%2d] %3d %6d %5d %4d\n",
+              sub, key_[sub], value_[sub], score_[sub], reliability_[sub]);
+    }
+  }
+  fprintf(f, "  %d#\n", incr_count_);
+}

data/ext/cld/encodings/compact_lang_det/tote.h

@@ -0,0 +1,89 @@
+// Copyright (c) 2006-2009 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef ENCODINGS_COMPACT_LANG_DET_TOTE_H_
+#define ENCODINGS_COMPACT_LANG_DET_TOTE_H_
+
+#include <stdio.h>
+#include "encodings/compact_lang_det/win/cld_basictypes.h"
+
+// Take a set of <key, value> pairs and tote them up.
+// After explicitly sorting, retrieve top key, value pairs
+class Tote {
+ public:
+  Tote();
+  ~Tote();
+  void Reinit();
+  void AddGram();
+  void Add(uint8 ikey, int idelta);
+  void AddBytes(int ibytes) {byte_count_ += ibytes;}
+  int CurrentTopKey();
+  void Sort(int n);
+  void Dump(FILE* f);
+  uint16 GetGramCount() const {return gram_count_;}
+  uint16 GetIncrCount() const {return incr_count_;}
+  int GetByteCount() const {return byte_count_;}
+  int MaxSize() const {return kMaxSize_;}
+  uint8 Key(int i) const {return key_[i];}
+  int Value(int i) const {return value_[i];}
+  void SetGramCount(uint16 v) {gram_count_ = v;}
+  void SetIncrCount(uint16 v) {incr_count_ = v;}
+  void SetKey(int i, int v) {key_[i] = v;}
+  void SetValue(int i, int v) {value_[i] = v;}
+
+ private:
+  static const int kMaxSize_ = 24;
+  uint16 gram_count_;       // Number of quadgrams/etc. scored
+  uint16 incr_count_;       // Number of Add calls (1-3 per gram)
+  int byte_count_;          // Bytes of text scored
+  // Align at multiple of 8 bytes
+  uint8 key_[kMaxSize_];    // Lang unassigned = 0, valid = 1..255
+  int value_[kMaxSize_];    // Probability score sum
+};
+
+
+// Take a set of <key, value, reliability> triples and tote them up.
+// After explicitly sorting, retrieve top key, value, reliability triples
+class ToteWithReliability {
+ public:
+  ToteWithReliability();
+  ~ToteWithReliability();
+  void Reinit();
+  void Add(uint8 ikey, int ibytes, int score, int ireliability);
+  int Find(uint8 ikey);
+  void AddClosePair(int subscr, int val) {closepair_[subscr] += val;}
+  int CurrentTopKey();
+  void Sort(int n);
+  void Dump(FILE* f);
+
+  ////void AddSeq(uint8 ikey) {ss_.Add(ikey);}
+  ////void ExtractSeq(int n, uint8* dst) {ss_.Extract(n, dst);}
+
+  int GetIncrCount() const {return incr_count_;}
+  int GetClosePair(int subscr) const {return closepair_[subscr];}
+  int MaxSize() const {return kMaxSize_;}
+  uint8 Key(int i) const {return key_[i];}
+  int Value(int i) const {return value_[i];}
+  int Score(int i) const {return score_[i];}
+  int Reliability(int i) const {return reliability_[i];}
+  void SetKey(int i, int v) {key_[i] = v;}
+  void SetValue(int i, int v) {value_[i] = v;}
+  void SetScore(int i, int v) {score_[i] = v;}
+  void SetReliability(int i, int v) {reliability_[i] = v;}
+
+ private:
+  static const int kMaxSize_ = 24;
+  static const int kMaxClosePairSize_ = 8;
+  int incr_count_;         // Number of Add calls
+  int sorted_;             // Contents have been sorted, cannot Add
+  // Align at multiple of 8 bytes
+  int closepair_[kMaxClosePairSize_];
+  uint8 key_[kMaxSize_];    // Lang unassigned = 0, valid = 1..255
+  int value_[kMaxSize_];    // Bytecount this lang
+  int score_[kMaxSize_];    // Probability score sum
+  int reliability_[kMaxSize_];  // Percentage 0..100
+  ////SubsetSequence ss_;
+};
+
+#endif  // ENCODINGS_COMPACT_LANG_DET_TOTE_H_