cld-fixed 0.7.1

data/ext/cld/encodings/compact_lang_det/#getonescriptspan.h#

@@ -0,0 +1,131 @@
+// Copyright (c) 2006-2009 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef ENCODINGS_COMPACT_LANG_DET_GETONESCRIPTSPAN_H_
+#define ENCODINGS_COMPACT_LANG_DET_GETONESCRIPTSPAN_H_
+
+#include "encodings/compact_lang_det/letterscript_enum.h"
+#include "encodings/compact_lang_det/compact_lang_det_impl.h"
+
+namespace getone {
+  static const int kMaxScriptBuffer = 4096;
+  static const int kMaxScriptLowerBuffer = (kMaxScriptBuffer * 3) / 2;
+  static const int kMaxScriptBytes = kMaxScriptBuffer- 8;   // Leave some room
+  static const int kMaxAnswerBuffer = 256;
+
+  typedef enum UnicodeLScript ULScript;
+
+  typedef struct {
+    char* text;             // Pointer to the span, somewhere
+    int text_bytes;         // Number of bytes of text in the span
+    int offset;             // Offset of start of span in original input buffer
+    ULScript script;        // Script of all the letters in this span
+    Language lang;          // Language identified for this span
+    bool truncated;         // true if buffer filled up before a
+                            // different script or EOF was found
+  } LangSpan;
+
+
+  static inline bool IsContinuationByte(char c) {
+    return static_cast<signed char>(c) < -64;
+  }
+
+  // Gets lscript number for letters; always returns
+  //   0 (common script) for non-letters
+  int GetUTF8LetterScriptNum(const char* src);
+
+
+  // Update src pointer to point to next quadgram, +2..+5
+  // Looks at src[0..4]
+  const char* AdvanceQuad(const char* src);
+}     // end namespace getone
+
+
+
+
+
+
+class ScriptScanner {
+ public:
+  ScriptScanner(const char* buffer, int buffer_length, bool is_plain_text);
+  ~ScriptScanner();
+
+  // Copy next run of same-script non-tag letters to buffer [NUL terminated]
+  bool GetOneScriptSpan(getone::LangSpan* span);
+
+  // Force Latin and Cyrillic scripts to be lowercase
+  void LowerScriptSpan(getone::LangSpan* span);
+
+  // Copy next run of same-script non-tag letters to buffer [NUL terminated]
+  // Force Latin and Cyrillic scripts to be lowercase
+  bool GetOneScriptSpanLower(getone::LangSpan* span);
+
+ private:
+  int SkipToFrontOfSpan(const char* src, int len, int* script);
+
+  const char* start_byte_;
+  const char* next_byte_;
+  const char* next_byte_limit_;
+  int byte_length_;
+  bool is_plain_text_;
+  char* script_buffer_;           // Holds text with expanded entities
+  char* script_buffer_lower_;     // Holds lowercased text
+};
+
+
+class LangScanner {
+ public:
+  LangScanner(const CompactLangDetImpl::LangDetObj* langdetobj,
+              getone::LangSpan* spn, int smoothwidth, int smoothcandidates,
+              int maxlangs, int minlangspan);
+  ~LangScanner();
+
+
+  int script() {return script_;}
+
+  // Use new text
+  // Keep smoothing state if same script, otherwise reinit smoothing
+  void NewText(getone::LangSpan* spn);
+
+  bool GetOneShortLangSpanBoot(getone::LangSpan* span);  // Just for bootstrapping
+  bool GetOneLangSpanBoot(getone::LangSpan* span);       // Just for bootstrapping
+
+  // The real ones
+  bool GetOneShortLangSpan(const CompactLangDetImpl::LangDetObj* langdetobj,
+                           getone::LangSpan* span);
+  bool GetOneLangSpan(const CompactLangDetImpl::LangDetObj* langdetobj,
+                      getone::LangSpan* span);
+
+  // Increases language bias by delta
+  void SetLanguageBias(const CompactLangDetImpl::LangDetObj* langdetobj,
+                       Language key, int delta);
+
+  // For debugging output
+  int next_answer_;
+  char answer_buffer_[getone::kMaxAnswerBuffer];
+  char answer_buffer2_[getone::kMaxAnswerBuffer];
+  char answer_buffer3_[getone::kMaxAnswerBuffer];
+  char answer_buffer4_[getone::kMaxAnswerBuffer];
+
+ private:
+  const char* start_byte_;
+  const char* next_byte_limit_;
+  const char* next_byte_;
+  const char* onelangspan_begin_;
+  int byte_length_;
+  int script_;
+  Language spanlang_;
+  int smoothwidth_;
+  int smoothwidth_2_;
+  int smoothcandidates_;
+  int maxlangs_;
+  int minlangspan_;
+  int rb_size_;
+  int next_rb_;
+  int rb_mask_;
+  uint32* rb_;
+  int* offset_rb_;
+};
+
+#endif  // ENCODINGS_COMPACT_LANG_DET_GETONESCRIPTSPAN_H_

data/ext/cld/encodings/compact_lang_det/#tote.cc#

@@ -0,0 +1,299 @@
+// Copyright (c) 2006-2009 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "encodings/compact_lang_det/tote.h"
+#include <string.h>     // memset
+
+#include "encodings/compact_lang_det/win/cld_logging.h"
+
+
+// Take a set of <key, value> pairs and tote them up.
+// After explicitly sorting, retrieve top key, value pairs
+Tote::Tote() {
+  gram_count_ = 0;
+  incr_count_ = 0;
+  byte_count_ = 0;
+  memset(key_, 0, sizeof(key_));
+  // No need to initialize values
+}
+
+Tote::~Tote() {
+}
+
+void Tote::Reinit() {
+  gram_count_ = 0;
+  incr_count_ = 0;
+  byte_count_ = 0;
+  memset(key_, 0, sizeof(key_));
+  // No need to initialize values
+}
+
+// Increment count of quadgrams/trigrams/unigrams scored
+void Tote::AddGram() {
+  ++gram_count_;
+}
+
+// Three-way associative, guaranteeing that the largest two counts are always
+// in the data structure. kMaxSize must be a multiple of 3, and is tied to the
+// subscript calculations here, which are for 8 sets of 3-way associative
+// buckets. The subscripts for set N are [N], [N+8], and [N+16] used in a
+// slightly-weird way: The initial probe point is [N] or [N+8], whichever
+// is specified by key mod 16. In most cases (nearly *all* cases except Latin
+// script), this entry matches and we update/return. The second probe is
+// the other of [N] and [N+8]. The third probe is only used as a fallback to
+// these two, and is there only for the rare case that there are three or more
+// languages with Language enum values equal mod 8, contending within the same
+// bucket. This can only happen in Latin and (rarely) Cyrillic scripts, because
+// the other scripts have fewer than 17 languages total.
+// If you change kMaxSize, change the constants 7/8/15/16 below
+void Tote::Add(uint8 ikey, int idelta) {
+  DCHECK(ikey != 0);
+  ++incr_count_;
+
+  // Look for existing entry
+  int sub0 = ikey & 15;
+  if (key_[sub0] == ikey) {
+    value_[sub0] += idelta;
+    return;
+  }
+  int sub1 = sub0 ^ 8;
+  if (key_[sub1] == ikey) {
+    value_[sub1] += idelta;
+    return;
+  }
+  int sub2 = (ikey & 7) + 16;
+  if (key_[sub2] == ikey) {
+    value_[sub2] += idelta;
+    return;
+  }
+
+  // Allocate new entry
+  int alloc = -1;
+  if (key_[sub0] == 0) {
+    alloc = sub0;
+  } else if (key_[sub1] == 0) {
+    alloc = sub1;
+  } else if (key_[sub2] == 0) {
+    alloc = sub2;
+  } else {
+    // All choices allocated, need to replace smallest one
+    alloc = sub0;
+    if (value_[sub1] < value_[alloc]) {alloc = sub1;}
+    if (value_[sub2] < value_[alloc]) {alloc = sub2;}
+  }
+  key_[alloc] = ikey;
+  value_[alloc] = idelta;
+  return;
+}
+
+// Return current top key
+int Tote::CurrentTopKey() {
+  int top_key = 0;
+  int top_value = -1;
+  for (int sub = 0; sub < kMaxSize_; ++sub) {
+    if (key_[sub] == 0) {continue;}
+    if (top_value < value_[sub]) {
+      top_value = value_[sub];
+      top_key = key_[sub];
+    }
+  }
+  return top_key;
+}
+
+
+// Sort first n entries by decreasing order of value
+// If key==0 other fields are not valid, treat value as -1
+void Tote::Sort(int n) {
+  // This is n**2, but n is small
+  for (int sub = 0; sub < n; ++sub) {
+    if (key_[sub] == 0) {value_[sub] = -1;}
+
+    // Bubble sort key[sub] and entry[sub]
+    for (int sub2 = sub + 1; sub2 < kMaxSize_; ++sub2) {
+      if (key_[sub2] == 0) {value_[sub2] = -1;}
+      if (value_[sub] < value_[sub2]) {
+        // swap
+        uint8 tmpk = key_[sub];
+        key_[sub] = key_[sub2];
+        key_[sub2] = tmpk;
+        int tmpv = value_[sub];
+        value_[sub] = value_[sub2];
+        value_[sub2] = tmpv;
+      }
+    }
+  }
+}
+
+void Tote::Dump(FILE* f) {
+  for (int sub = 0; sub < kMaxSize_; ++sub) {
+    if (key_[sub] > 0) {
+      fprintf(f, "[%2d] %3d %8d\n", sub, key_[sub], value_[sub]);
+    }
+  }
+  fprintf(f, "%d %d %d\n", gram_count_, incr_count_, byte_count_);
+}
+
+
+
+
+// Take a set of <key, value> pairs and tote them up.
+// After explicitly sorting, retrieve top key, value pairs
+ToteWithReliability::ToteWithReliability() {
+  // No need to initialize score_ or value_
+  incr_count_ = 0;
+  sorted_ = 0;
+  memset(closepair_, 0, sizeof(closepair_));
+  memset(key_, 0, sizeof(key_));
+}
+
+ToteWithReliability::~ToteWithReliability() {
+}
+
+void ToteWithReliability::Reinit() {
+  // No need to initialize score_ or value_
+  incr_count_ = 0;
+  sorted_ = 0;
+  memset(closepair_, 0, sizeof(closepair_));
+  memset(key_, 0, sizeof(key_));
+  ////ss_.Init();
+}
+
+// Weight reliability by ibytes
+// Also see three-way associative comments above for Tote
+void ToteWithReliability::Add(uint8 ikey, int ibytes,
+                              int score, int ireliability) {
+  DCHECK(ikey != 0);
+  CHECK(sorted_ == 0);
+  ++incr_count_;
+
+  // Look for existing entry
+  int sub0 = ikey & 15;
+  if (key_[sub0] == ikey) {
+    value_[sub0] += ibytes;
+    score_[sub0] += score;
+    reliability_[sub0] += ireliability * ibytes;
+    return;
+  }
+  int sub1 = sub0 ^ 8;
+  if (key_[sub1] == ikey) {
+    value_[sub1] += ibytes;
+    score_[sub1] += score;
+    reliability_[sub1] += ireliability * ibytes;
+    return;
+  }
+  int sub2 = (ikey & 7) + 16;
+  if (key_[sub2] == ikey) {
+    value_[sub2] += ibytes;
+    score_[sub2] += score;
+    reliability_[sub2] += ireliability * ibytes;
+    return;
+  }
+
+  // Allocate new entry
+  int alloc = -1;
+  if (key_[sub0] == 0) {
+    alloc = sub0;
+  } else if (key_[sub1] == 0) {
+    alloc = sub1;
+  } else if (key_[sub2] == 0) {
+    alloc = sub2;
+  } else {
+    // All choices allocated, need to replace smallest one
+    alloc = sub0;
+    if (value_[sub1] < value_[alloc]) {alloc = sub1;}
+    if (value_[sub2] < value_[alloc]) {alloc = sub2;}
+  }
+  key_[alloc] = ikey;
+  value_[alloc] = ibytes;
+  score_[alloc] = score;
+  reliability_[alloc] = ireliability * ibytes;
+  return;
+}
+
+// Find subscript of a given packed language, or -1
+int ToteWithReliability::Find(uint8 ikey) {
+  DCHECK(ikey != 0);
+
+  if (sorted_) {
+    // Linear search if sorted
+    for (int sub = 0; sub < kMaxSize_; ++sub) {
+      if (key_[sub] == ikey) {return sub;}
+    }
+    return -1;
+  }
+
+  // Look for existing entry
+  int sub0 = ikey & 15;
+  if (key_[sub0] == ikey) {
+    return sub0;
+  }
+  int sub1 = sub0 ^ 8;
+  if (key_[sub1] == ikey) {
+    return sub1;
+  }
+  int sub2 = (ikey & 7) + 16;
+  if (key_[sub2] == ikey) {
+    return sub2;
+  }
+
+  return -1;
+}
+
+// Return current top key
+int ToteWithReliability::CurrentTopKey() {
+  int top_key = 0;
+  int top_value = -1;
+  for (int sub = 0; sub < kMaxSize_; ++sub) {
+    if (key_[sub] == 0) {continue;}
+    if (top_value < value_[sub]) {
+      top_value = value_[sub];
+      top_key = key_[sub];
+    }
+  }
+  return top_key;
+}
+
+
+// Sort first n entries by decreasing order of value
+// If key==0 other fields are not valid, treat value as -1
+void ToteWithReliability::Sort(int n) {
+  // This is n**2, but n is small
+  for (int sub = 0; sub < n; ++sub) {
+    if (key_[sub] == 0) {value_[sub] = -1;}
+
+    // Bubble sort key[sub] and entry[sub]
+    for (int sub2 = sub + 1; sub2 < kMaxSize_; ++sub2) {
+      if (key_[sub2] == 0) {value_[sub2] = -1;}
+      if (value_[sub] < value_[sub2]) {
+        // swap
+        uint8 tmpk = key_[sub];
+        key_[sub] = key_[sub2];
+        key_[sub2] = tmpk;
+
+        int tmpv = value_[sub];
+        value_[sub] = value_[sub2];
+        value_[sub2] = tmpv;
+
+        double tmps = score_[sub];
+        score_[sub] = score_[sub2];
+        score_[sub2] = tmps;
+
+        int tmpr = reliability_[sub];
+        reliability_[sub] = reliability_[sub2];
+        reliability_[sub2] = tmpr;
+      }
+    }
+  }
+  sorted_ = 1;
+}
+
+void ToteWithReliability::Dump(FILE* f) {
+  for (int sub = 0; sub < kMaxSize_; ++sub) {
+    if (key_[sub] > 0) {
+      fprintf(f, "[%2d] %3d %6d %5d %4d\n",
+              sub, key_[sub], value_[sub], score_[sub], reliability_[sub]);
+    }
+  }
+  fprintf(f, "  %d#\n", incr_count_);
+}

data/ext/cld/encodings/compact_lang_det/#tote.h#

@@ -0,0 +1,89 @@
+// Copyright (c) 2006-2009 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef ENCODINGS_COMPACT_LANG_DET_TOTE_H_
+#define ENCODINGS_COMPACT_LANG_DET_TOTE_H_
+
+#include <stdio.h>
+#include "encodings/compact_lang_det/win/cld_basictypes.h"
+
+// Take a set of <key, value> pairs and tote them up.
+// After explicitly sorting, retrieve top key, value pairs
+class Tote {
+ public:
+  Tote();
+  ~Tote();
+  void Reinit();
+  void AddGram();
+  void Add(uint8 ikey, int idelta);
+  void AddBytes(int ibytes) {byte_count_ += ibytes;}
+  int CurrentTopKey();
+  void Sort(int n);
+  void Dump(FILE* f);
+  uint16 GetGramCount() const {return gram_count_;}
+  uint16 GetIncrCount() const {return incr_count_;}
+  int GetByteCount() const {return byte_count_;}
+  int MaxSize() const {return kMaxSize_;}
+  uint8 Key(int i) const {return key_[i];}
+  int Value(int i) const {return value_[i];}
+  void SetGramCount(uint16 v) {gram_count_ = v;}
+  void SetIncrCount(uint16 v) {incr_count_ = v;}
+  void SetKey(int i, int v) {key_[i] = v;}
+  void SetValue(int i, int v) {value_[i] = v;}
+
+ private:
+  static const int kMaxSize_ = 24;
+  uint16 gram_count_;       // Number of quadgrams/etc. scored
+  uint16 incr_count_;       // Number of Add calls (1-3 per gram)
+  int byte_count_;          // Bytes of text scored
+  // Align at multiple of 8 bytes
+  uint8 key_[kMaxSize_];    // Lang unassigned = 0, valid = 1..255
+  int value_[kMaxSize_];    // Probability score sum
+};
+
+
+// Take a set of <key, value, reliability> triples and tote them up.
+// After explicitly sorting, retrieve top key, value, reliability triples
+class ToteWithReliability {
+ public:
+  ToteWithReliability();
+  ~ToteWithReliability();
+  void Reinit();
+  void Add(uint8 ikey, int ibytes, int score, int ireliability);
+  int Find(uint8 ikey);
+  void AddClosePair(int subscr, int val) {closepair_[subscr] += val;}
+  int CurrentTopKey();
+  void Sort(int n);
+  void Dump(FILE* f);
+
+  ////void AddSeq(uint8 ikey) {ss_.Add(ikey);}
+  ////void ExtractSeq(int n, uint8* dst) {ss_.Extract(n, dst);}
+
+  int GetIncrCount() const {return incr_count_;}
+  int GetClosePair(int subscr) const {return closepair_[subscr];}
+  int MaxSize() const {return kMaxSize_;}
+  uint8 Key(int i) const {return key_[i];}
+  int Value(int i) const {return value_[i];}
+  int Score(int i) const {return score_[i];}
+  int Reliability(int i) const {return reliability_[i];}
+  void SetKey(int i, int v) {key_[i] = v;}
+  void SetValue(int i, int v) {value_[i] = v;}
+  void SetScore(int i, int v) {score_[i] = v;}
+  void SetReliability(int i, int v) {reliability_[i] = v;}
+
+ private:
+  static const int kMaxSize_ = 24;
+  static const int kMaxClosePairSize_ = 8;
+  int incr_count_;         // Number of Add calls
+  int sorted_;             // Contents have been sorted, cannot Add
+  // Align at multiple of 8 bytes
+  int closepair_[kMaxClosePairSize_];
+  uint8 key_[kMaxSize_];    // Lang unassigned = 0, valid = 1..255
+  int value_[kMaxSize_];    // Bytecount this lang
+  int score_[kMaxSize_];    // Probability score sum
+  int reliability_[kMaxSize_];  // Percentage 0..100
+  ////SubsetSequence ss_;
+};
+
+#endif  // ENCODINGS_COMPACT_LANG_DET_TOTE_H_