cld 0.1.0

data/base/vlog_is_on.h

@@ -0,0 +1,143 @@
+// Copyright (c) 2006-2009 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+// Defines the VLOG_IS_ON macro that controls the variable-verbosity
+// conditional logging.
+//
+// It's used by VLOG and VLOG_IF in logging.h
+// and by RAW_VLOG in raw_logging.h to trigger the logging.
+//
+// It can also be used directly e.g. like this:
+//   if (VLOG_IS_ON(2)) {
+//     // do some logging preparation and logging
+//     // that can't be accomplished e.g. via just VLOG(2) << ...;
+//   }
+//
+// The truth value that VLOG_IS_ON(level) returns is determined by
+// the three verbosity level flags:
+//   --v=<n>  Gives the default maximal active V-logging level;
+//            0 is the default.
+//            Normally positive values are used for V-logging levels.
+//   --vmodule=<str>  Gives the per-module maximal V-logging levels to override
+//                    the value given by --v.
+//                    E.g. "my_module=2,foo*=3" would change the logging level
+//                    for all code in source files "my_module.*" and "foo*.*"
+//                    ("-inl" suffixes are also disregarded for this matching).
+//   --silent_init  When true has the effect of increasing
+//                  the argument of VLOG_IS_ON by 1,
+//                  thus suppressing one more level of verbose logging.
+//
+// SetVLOGLevel helper function is provided to do limited dynamic control over
+// V-logging by overriding the per-module settings given via --vmodule flag.
+//
+// CAVEAT: --vmodule functionality is not available in non gcc compilers.
+//
+
+#ifndef BASE_VLOG_IS_ON_H_
+#define BASE_VLOG_IS_ON_H_
+
+#include "base/atomicops.h"
+#include "base/basictypes.h"
+#include "base/port.h"
+#include "base/commandlineflags.h"
+#include "base/log_severity.h"
+
+DECLARE_int32(v);  // in vlog_is_on.cc
+DECLARE_bool(silent_init);  // in google.cc
+
+#if defined(__GNUC__)
+// We pack an int16 verbosity level and an int16 epoch into an
+// Atomic32 at every VLOG_IS_ON() call site.  The level determines
+// whether the site should log, and the epoch determines whether the
+// site is stale and should be reinitialized.  A verbosity level of
+// kUseFlag (-1) indicates that the value of FLAGS_v should be used as
+// the verbosity level.  When the site is (re)initialized, a verbosity
+// level for the current source file is retrieved from an internal
+// list.  This list is mutated through calls to SetVLOGLevel() and
+// mutations to the --vmodule flag.  New log sites are initialized
+// with a stale epoch and a verbosity level of kUseFlag.
+//
+// TODO(llansing): Investigate using GCC's __builtin_constant_p() to
+// generate less code at call sites where verbositylevel is known to
+// be a compile-time constant.
+#define VLOG_IS_ON(verboselevel)                                               \
+  ({ static Atomic32 site__ = ::base::internal::kDefaultSite;                  \
+     ::base::internal::VLogEnabled(&site__, (verboselevel), __FILE__); })
+#else
+// GNU extensions not available, so we do not support --vmodule.
+// Dynamic value of FLAGS_v always controls the logging level.
+//
+// TODO(llansing): Investigate supporting --vmodule on other platforms.
+#define VLOG_IS_ON(verboselevel)                                               \
+  (FLAGS_v >= (verboselevel) + FLAGS_silent_init)
+#endif
+
+// Set VLOG(_IS_ON) level for module_pattern to log_level.
+// This lets us dynamically control what is normally set by the --vmodule flag.
+// Returns the level that previously applied to module_pattern.
+// NOTE: To change the log level for VLOG(_IS_ON) sites
+//       that have already executed after/during InitGoogle,
+//       one needs to supply the exact --vmodule pattern that applied to them.
+//       (If no --vmodule pattern applied to them
+//       the value of FLAGS_v will continue to control them.)
+int SetVLOGLevel(const char* module_pattern, int log_level);
+
+// Private implementation details.  No user-serviceable parts inside.
+namespace base {
+namespace internal {
+
+// Each log site determines whether its log level is up to date by
+// comparing its epoch to this global epoch.  Whenever the program's
+// vmodule configuration changes (ex: SetVLOGLevel is called), the
+// global epoch is advanced, invalidating all site epochs.
+extern Atomic32 vlog_epoch;
+
+// A log level of kUseFlag means "read the logging level from FLAGS_v."
+const int kUseFlag = -1;
+
+// Log sites use FLAGS_v by default, and have an initial epoch of 0.
+const Atomic32 kDefaultSite = kUseFlag << 16;
+
+// The global epoch is the least significant half of an Atomic32, and
+// may only be accessed through atomic operations.
+inline Atomic32 GlobalEpoch() {
+  return Acquire_Load(&vlog_epoch) & 0x0000FFFF;
+}
+
+// The least significant half of a site is the epoch.
+inline int SiteEpoch(Atomic32 site) { return site & 0x0000FFFF; }
+
+// The most significant half of a site is the logging level.
+inline int SiteLevel(Atomic32 site) { return site >> 16; }
+
+// Construct a logging site from a logging level and epoch.
+inline Atomic32 Site(int level, int epoch) {
+  return ((level & 0x0000FFFF) << 16) | (epoch & 0x0000FFFF);
+}
+
+// Attempt to initialize or reinitialize a VLOG site.  Returns the
+// level of the log site, regardless of whether the attempt succeeds
+// or fails.
+//   site: The address of the log site's state.
+//   fname: The filename of the current source file.
+int InitVLOG(Atomic32* site, const char* fname);
+
+// Determine whether verbose logging should occur at a given log site.
+//
+// TODO(llansing): Find a way to eliminate FLAGS_silent_init from this
+// function while preserving the silent initialization behavior.  The
+// common-case code path shouldn't pay for silent initialization.
+inline bool VLogEnabled(Atomic32* site, int32 level, const char* const file) {
+  const Atomic32 site_copy = Acquire_Load(site);
+  const int32 site_level =
+      PREDICT_TRUE(SiteEpoch(site_copy) == GlobalEpoch()) ?
+      SiteLevel(site_copy) : InitVLOG(site, file);
+  return (site_level == kUseFlag ? FLAGS_v : site_level) >=
+      (level + FLAGS_silent_init);
+}
+
+}  // namespace internal
+}  // namespace base
+
+#endif  // BASE_VLOG_IS_ON_H_

data/build.sh

@@ -0,0 +1,48 @@
+#!/bin/bash
+
+CFLAGS="-fPIC -I. -O2 -DCLD_WINDOWS"
+LDFLAGS=-L.
+CC=g++
+AR=ar
+
+rm -f *.o
+rm -f libcld.a
+
+SOURCES="encodings/compact_lang_det/cldutil.cc \
+        encodings/compact_lang_det/cldutil_dbg_empty.cc \
+        encodings/compact_lang_det/compact_lang_det.cc \
+        encodings/compact_lang_det/compact_lang_det_impl.cc \
+        encodings/compact_lang_det/ext_lang_enc.cc \
+        encodings/compact_lang_det/getonescriptspan.cc \
+        encodings/compact_lang_det/letterscript_enum.cc \
+        encodings/compact_lang_det/tote.cc \
+        encodings/compact_lang_det/generated/cld_generated_score_quadchrome_0406.cc \
+        encodings/compact_lang_det/generated/compact_lang_det_generated_cjkbis_0.cc \
+        encodings/compact_lang_det/generated/compact_lang_det_generated_ctjkvz.cc \
+        encodings/compact_lang_det/generated/compact_lang_det_generated_deltaoctachrome.cc \
+        encodings/compact_lang_det/generated/compact_lang_det_generated_quadschrome.cc \
+        encodings/compact_lang_det/win/cld_htmlutils_windows.cc \
+        encodings/compact_lang_det/win/cld_unilib_windows.cc \
+        encodings/compact_lang_det/win/cld_utf8statetable.cc \
+        encodings/compact_lang_det/win/cld_utf8utils_windows.cc \
+        encodings/internal/encodings.cc \
+        languages/internal/languages.cc \
+        thunk.cc"
+
+        #encodings/compact_lang_det/win/cld_unicodetext.cc \
+
+echo
+echo "Compile..."
+$CC -c $CFLAGS $SOURCES
+
+echo
+echo "Make libcld.a"
+$AR rcs libcld.a *.o
+
+echo
+#$CC -DCLD_WINDOWS -I. -L. -o example example.cc -lcld -lstdc++
+$CC -DCLD_WINDOWS -I. -L. -o cld.so -lcld -lstdc++ *.o
+
+
+echo
+echo "Done!"

data/build.win.cmd

@@ -0,0 +1,28 @@
+REM "c:\Program Files\Microsoft Visual Studio 8\vc\vcvarsall.bat"
+
+set CFLAGS=/nologo /I. /O2 /DCLD_WINDOWS /DWIN32 /EHsc
+set LDFLAGS=-L.
+set CC=cl.exe
+set AR=lib.exe
+
+del *.obj
+del libcld.lib
+
+set SOURCES=encodings/compact_lang_det/cldutil.cc encodings/compact_lang_det/cldutil_dbg_empty.cc encodings/compact_lang_det/compact_lang_det.cc encodings/compact_lang_det/compact_lang_det_impl.cc encodings/compact_lang_det/ext_lang_enc.cc encodings/compact_lang_det/getonescriptspan.cc encodings/compact_lang_det/letterscript_enum.cc encodings/compact_lang_det/tote.cc encodings/compact_lang_det/generated/cld_generated_score_quadchrome_0406.cc encodings/compact_lang_det/generated/compact_lang_det_generated_cjkbis_0.cc encodings/compact_lang_det/generated/compact_lang_det_generated_ctjkvz.cc encodings/compact_lang_det/generated/compact_lang_det_generated_deltaoctachrome.cc encodings/compact_lang_det/generated/compact_lang_det_generated_quadschrome.cc encodings/compact_lang_det/win/cld_htmlutils_windows.cc encodings/compact_lang_det/win/cld_unilib_windows.cc encodings/compact_lang_det/win/cld_utf8statetable.cc encodings/compact_lang_det/win/cld_utf8utils_windows.cc encodings/internal/encodings.cc languages/internal/languages.cc
+
+REM  encodings/compact_lang_det/win/cld_unicodetext.cc \
+
+echo ""
+echo "Compile..."
+%CC% /c %CFLAGS% %SOURCES%
+
+echo ""
+echo "Make libcld"
+%AR% *.obj -OUT:libcld.lib
+
+echo ""
+echo "Compile example.cc"
+%CC% %CFLAGS% %LFLAGS% example.cc libcld.lib
+
+echo
+echo "Done!"

data/cld.gemspec

@@ -0,0 +1,30 @@
+# -*- encoding: utf-8 -*-
+
+Gem::Specification.new do |s|
+  s.name = %q{cld}
+  s.version = "0.1.0"
+
+  s.required_rubygems_version = Gem::Requirement.new(">= 1.2") if s.respond_to? :required_rubygems_version=
+  s.authors = [%q{Jason Toy}]
+  s.date = %q{2011-11-03}
+  s.description = %q{Compact Language Detection from chrome}
+  s.email = %q{jtoy@jtoy.net}
+  s.extensions = [%q{ext/cld/extconf.rb}]
+  s.extra_rdoc_files = [%q{LICENSE}, %q{README.rdoc}, %q{ext/cld/extconf.rb}, %q{lib/cld.rb}]
+  s.files = [%q{LICENSE}, %q{README.rdoc}, %q{Rakefile}, %q{base/basictypes.h}, %q{base/build_config.h}, %q{base/casts.h}, %q{base/commandlineflags.h}, %q{base/crash.h}, %q{base/dynamic_annotations.h}, %q{base/global_strip_options.h}, %q{base/log_severity.h}, %q{base/logging.h}, %q{base/macros.h}, %q{base/port.h}, %q{base/scoped_ptr.h}, %q{base/stl_decl.h}, %q{base/stl_decl_msvc.h}, %q{base/string_util.h}, %q{base/strtoint.h}, %q{base/template_util.h}, %q{base/type_traits.h}, %q{base/vlog_is_on.h}, %q{build.sh}, %q{build.win.cmd}, %q{cld.gemspec}, %q{cld_encodings.h}, %q{encodings/compact_lang_det/#cldutil.cc#}, %q{encodings/compact_lang_det/#cldutil.h#}, %q{encodings/compact_lang_det/#compact_lang_det_impl.h#}, %q{encodings/compact_lang_det/#ext_lang_enc.cc#}, %q{encodings/compact_lang_det/#ext_lang_enc.h#}, %q{encodings/compact_lang_det/#getonescriptspan.cc#}, %q{encodings/compact_lang_det/#getonescriptspan.h#}, %q{encodings/compact_lang_det/#tote.cc#}, %q{encodings/compact_lang_det/#tote.h#}, %q{encodings/compact_lang_det/cldutil.cc}, %q{encodings/compact_lang_det/cldutil.h}, %q{encodings/compact_lang_det/cldutil_dbg.h}, %q{encodings/compact_lang_det/cldutil_dbg_empty.cc}, %q{encodings/compact_lang_det/compact_lang_det.cc}, %q{encodings/compact_lang_det/compact_lang_det.h}, %q{encodings/compact_lang_det/compact_lang_det_impl.cc}, %q{encodings/compact_lang_det/compact_lang_det_impl.h}, %q{encodings/compact_lang_det/compact_lang_det_unittest_small.cc}, %q{encodings/compact_lang_det/compile.cmd}, %q{encodings/compact_lang_det/ext_lang_enc.cc}, %q{encodings/compact_lang_det/ext_lang_enc.h}, %q{encodings/compact_lang_det/generated/cld_generated_score_deltaoctachrome_0406.cc}, %q{encodings/compact_lang_det/generated/cld_generated_score_quadchrome_0406.cc}, %q{encodings/compact_lang_det/generated/compact_lang_det_generated_cjkbis_0.cc}, %q{encodings/compact_lang_det/generated/compact_lang_det_generated_ctjkvz.cc}, %q{encodings/compact_lang_det/generated/compact_lang_det_generated_ctjkvz_0.cc}, %q{encodings/compact_lang_det/generated/compact_lang_det_generated_deltaoctachrome.cc}, %q{encodings/compact_lang_det/generated/compact_lang_det_generated_longwords8_0.cc}, %q{encodings/compact_lang_det/generated/compact_lang_det_generated_meanscore.h}, %q{encodings/compact_lang_det/generated/compact_lang_det_generated_quads_0.cc}, %q{encodings/compact_lang_det/generated/compact_lang_det_generated_quadschrome.cc}, %q{encodings/compact_lang_det/getonescriptspan.cc}, %q{encodings/compact_lang_det/getonescriptspan.h}, %q{encodings/compact_lang_det/letterscript_enum.cc}, %q{encodings/compact_lang_det/letterscript_enum.h}, %q{encodings/compact_lang_det/subsetsequence.cc}, %q{encodings/compact_lang_det/subsetsequence.h}, %q{encodings/compact_lang_det/subsetsequence_unittest.cc}, %q{encodings/compact_lang_det/tote.cc}, %q{encodings/compact_lang_det/tote.h}, %q{encodings/compact_lang_det/unittest_data.h}, %q{encodings/compact_lang_det/utf8propjustletter.h}, %q{encodings/compact_lang_det/utf8propletterscriptnum.h}, %q{encodings/compact_lang_det/utf8scannotjustletterspecial.h}, %q{encodings/compact_lang_det/win/#cld_unilib_windows.cc#}, %q{encodings/compact_lang_det/win/cld_basictypes.h}, %q{encodings/compact_lang_det/win/cld_commandlineflags.h}, %q{encodings/compact_lang_det/win/cld_google.h}, %q{encodings/compact_lang_det/win/cld_htmlutils.h}, %q{encodings/compact_lang_det/win/cld_htmlutils_google3.cc}, %q{encodings/compact_lang_det/win/cld_htmlutils_windows.cc}, %q{encodings/compact_lang_det/win/cld_logging.h}, %q{encodings/compact_lang_det/win/cld_macros.h}, %q{encodings/compact_lang_det/win/cld_strtoint.h}, %q{encodings/compact_lang_det/win/cld_unicodetext.cc}, %q{encodings/compact_lang_det/win/cld_unicodetext.h}, %q{encodings/compact_lang_det/win/cld_unilib.h}, %q{encodings/compact_lang_det/win/cld_unilib_google3.cc}, %q{encodings/compact_lang_det/win/cld_unilib_windows.cc}, %q{encodings/compact_lang_det/win/cld_utf.h}, %q{encodings/compact_lang_det/win/cld_utf8statetable.cc}, %q{encodings/compact_lang_det/win/cld_utf8statetable.h}, %q{encodings/compact_lang_det/win/cld_utf8utils.h}, %q{encodings/compact_lang_det/win/cld_utf8utils_google3.cc}, %q{encodings/compact_lang_det/win/cld_utf8utils_windows.cc}, %q{encodings/compact_lang_det/win/normalizedunicodetext.cc}, %q{encodings/compact_lang_det/win/normalizedunicodetext.h}, %q{encodings/internal/encodings.cc}, %q{encodings/lang_enc.h}, %q{encodings/proto/encodings.pb.h}, %q{encodings/public/encodings.h}, %q{ext/cld/extconf.rb}, %q{languages/internal/#languages.cc#}, %q{languages/internal/languages.cc}, %q{languages/proto/languages.pb.h}, %q{languages/public/languages.h}, %q{lib/cld.rb}, %q{test/test.rb}, %q{thunk.cc}, %q{Manifest}]
+  s.homepage = %q{http://github.com/jtoy/cld}
+  s.rdoc_options = [%q{--line-numbers}, %q{--inline-source}, %q{--title}, %q{Cld}, %q{--main}, %q{README.rdoc}]
+  s.require_paths = [%q{lib}, %q{ext}]
+  s.rubyforge_project = %q{cld}
+  s.rubygems_version = %q{1.8.6.1}
+  s.summary = %q{Compact Language Detection from chrome}
+
+  if s.respond_to? :specification_version then
+    s.specification_version = 3
+
+    if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
+    else
+    end
+  else
+  end
+end

data/cld_encodings.h

@@ -0,0 +1,95 @@
+// Copyright (c) 2006-2009 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+// MKM: I copied this from the PHP port
+// (https://github.com/lstrojny/php-cld/blob/master/cld_encodings.h);
+// it just copies the encodings out of encodings.pb.h and
+// gives them string matching string constants
+
+#include "encodings/proto/encodings.pb.h"
+
+struct cld_encoding {
+  const char *name;
+  Encoding encoding;
+};
+
+const cld_encoding cld_encoding_info[] = {
+{"ISO_8859_1", ISO_8859_1},
+{"ISO_8859_2", ISO_8859_2},
+{"ISO_8859_3", ISO_8859_3},
+{"ISO_8859_4", ISO_8859_4},
+{"ISO_8859_5", ISO_8859_5},
+{"ISO_8859_6", ISO_8859_6},
+{"ISO_8859_7", ISO_8859_7},
+{"ISO_8859_8", ISO_8859_8},
+{"ISO_8859_9", ISO_8859_9},
+{"ISO_8859_10", ISO_8859_10},
+{"JAPANESE_EUC_JP", JAPANESE_EUC_JP},
+{"JAPANESE_SHIFT_JIS", JAPANESE_SHIFT_JIS},
+{"JAPANESE_JIS", JAPANESE_JIS},
+{"CHINESE_BIG5", CHINESE_BIG5},
+{"CHINESE_GB", CHINESE_GB},
+{"CHINESE_EUC_CN", CHINESE_EUC_CN},
+{"KOREAN_EUC_KR", KOREAN_EUC_KR},
+{"UNICODE", UNICODE},
+{"CHINESE_EUC_DEC", CHINESE_EUC_DEC},
+{"CHINESE_CNS", CHINESE_CNS},
+{"CHINESE_BIG5_CP950", CHINESE_BIG5_CP950},
+{"JAPANESE_CP932", JAPANESE_CP932},
+{"UTF8", UTF8},
+{"UNKNOWN_ENCODING", UNKNOWN_ENCODING},
+{"ASCII_7BIT", ASCII_7BIT},
+{"RUSSIAN_KOI8_R", RUSSIAN_KOI8_R},
+{"RUSSIAN_CP1251", RUSSIAN_CP1251},
+{"MSFT_CP1252", MSFT_CP1252},
+{"RUSSIAN_KOI8_RU", RUSSIAN_KOI8_RU},
+{"MSFT_CP1250", MSFT_CP1250},
+{"ISO_8859_15", ISO_8859_15},
+{"MSFT_CP1254", MSFT_CP1254},
+{"MSFT_CP1257", MSFT_CP1257},
+{"ISO_8859_11", ISO_8859_11},
+{"MSFT_CP874", MSFT_CP874},
+{"MSFT_CP1256", MSFT_CP1256},
+{"MSFT_CP1255", MSFT_CP1255},
+{"ISO_8859_8_I", ISO_8859_8_I},
+{"HEBREW_VISUAL", HEBREW_VISUAL},
+{"CZECH_CP852", CZECH_CP852},
+{"CZECH_CSN_369103", CZECH_CSN_369103},
+{"MSFT_CP1253", MSFT_CP1253},
+{"RUSSIAN_CP866", RUSSIAN_CP866},
+{"ISO_8859_13", ISO_8859_13},
+{"ISO_2022_KR", ISO_2022_KR},
+{"GBK", GBK},
+{"GB18030", GB18030},
+{"BIG5_HKSCS", BIG5_HKSCS},
+{"ISO_2022_CN", ISO_2022_CN},
+{"TSCII", TSCII},
+{"TAMIL_MONO", TAMIL_MONO},
+{"TAMIL_BI", TAMIL_BI},
+{"JAGRAN", JAGRAN},
+{"MACINTOSH_ROMAN", MACINTOSH_ROMAN},
+{"UTF7", UTF7},
+{"BHASKAR", BHASKAR},
+{"HTCHANAKYA", HTCHANAKYA},
+{"UTF16BE", UTF16BE},
+{"UTF16LE", UTF16LE},
+{"UTF32BE", UTF32BE},
+{"UTF32LE", UTF32LE},
+{"BINARYENC", BINARYENC},
+{"HZ_GB_2312", HZ_GB_2312},
+{"UTF8UTF8", UTF8UTF8},
+{"TAM_ELANGO", TAM_ELANGO},
+{"TAM_LTTMBARANI", TAM_LTTMBARANI},
+{"TAM_SHREE", TAM_SHREE},
+{"TAM_TBOOMIS", TAM_TBOOMIS},
+{"TAM_TMNEWS", TAM_TMNEWS},
+{"TAM_WEBTAMIL", TAM_WEBTAMIL},
+{"KDDI_SHIFT_JIS", KDDI_SHIFT_JIS},
+{"DOCOMO_SHIFT_JIS", DOCOMO_SHIFT_JIS},
+{"SOFTBANK_SHIFT_JIS", SOFTBANK_SHIFT_JIS},
+{"KDDI_ISO_2022_JP", KDDI_ISO_2022_JP},
+{"SOFTBANK_ISO_2022_JP", SOFTBANK_ISO_2022_JP},
+};
+
+COMPILE_ASSERT(arraysize(cld_encoding_info) == NUM_ENCODINGS, cld_encoding_info_length_is_wrong);

data/encodings/compact_lang_det/#cldutil.cc#

@@ -0,0 +1,905 @@
+// Copyright (c) 2009 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include <string>
+#include "encodings/compact_lang_det/cldutil.h"
+#include "encodings/compact_lang_det/cldutil_dbg.h"
+#include "encodings/compact_lang_det/generated/compact_lang_det_generated_meanscore.h"
+#include "encodings/compact_lang_det/utf8propletterscriptnum.h"
+#include "encodings/compact_lang_det/win/cld_commandlineflags.h"
+#include "encodings/compact_lang_det/win/cld_logging.h"
+#include "encodings/compact_lang_det/win/cld_unilib.h"
+#include "encodings/compact_lang_det/win/cld_utf.h"
+#include "encodings/compact_lang_det/win/cld_utf8statetable.h"
+
+// Runtime routines for hashing, looking up, and scoring
+// unigrams (CJK), bigrams (CJK), quadgrams, and octagrams.
+// Unigrams and bigrams are for CJK languages only, including simplified/
+// traditional Chinese, Japanese, Korean, Vietnamese Han characters, and
+// Zhuang Han characters. Surrounding spaces are not considered.
+// Quadgrams and octagrams for for non-CJK and include two bits indicating
+// preceding and trailing spaces (word boundaries).
+
+
+// Indicator bits for leading/trailing space around quad/octagram
+// NOTE: 4444 bits are chosen to flip constant bits in hash of four chars of
+// 1-, 2-, or 3-bytes each.
+static const uint32 kPreSpaceIndicator =  0x00004444;
+static const uint32 kPostSpaceIndicator = 0x44440000;
+
+// Little-endian masks for 0..24 bytes picked up as uint32's
+static const uint32 kWordMask0[4] = {
+  0xFFFFFFFF, 0x000000FF, 0x0000FFFF, 0x00FFFFFF
+};
+
+static const int kMinCJKUTF8CharBytes = 3;
+
+static const int kMinGramCount = 3;
+static const int kMaxGramCount = 16;
+
+
+
+
+// Routines to access a hash table of <key:wordhash, value:probs> pairs
+// Buckets have 4-byte wordhash for sizes < 32K buckets, but only
+// 2-byte wordhash for sizes >= 32K buckets, with other wordhash bits used as
+// bucket subscript.
+// Probs is a packed: three languages plus a subscript for probability table
+// Buckets have all the keys together, then all the values.Key array never
+// crosses a cache-line boundary, so no-match case takes exactly one cache miss.
+// Match case may sometimes take an additional cache miss on value access.
+//
+// Other possibilites include 5 or 10 6-byte entries plus pad to make 32 or 64
+// byte buckets with single cache miss.
+// Or 2-byte key and 6-byte value, allowing 5 languages instead  of three.
+//------------------------------------------------------------------------------
+
+
+//------------------------------------------------------------------------------
+// Hashing groups of 1/2/4/8 letters, perhaps with spaces or underscores
+//------------------------------------------------------------------------------
+
+// Design principles for these hash functions
+// - Few operations
+// - Handle 1-, 2-, and 3-byte UTF-8 scripts, ignoring intermixing except in
+//   Latin script expect 1- and 2-byte mixtures.
+// - Last byte of each character has about 5 bits of information
+// - Spread good bits around so they can interact in at least two ways
+//   with other characters
+// - Use add for additional mixing thorugh carries
+
+// CJK Three-byte bigram
+//   ....dddd..cccccc..bbbbbb....aaaa
+//   ..................ffffff..eeeeee
+// make
+//   ....dddd..cccccc..bbbbbb....aaaa
+//   000....dddd..cccccc..bbbbbb....a
+//   ..................ffffff..eeeeee
+//   ffffff..eeeeee000000000000000000
+//
+// CJK Four-byte bigram
+//   ..dddddd..cccccc....bbbb....aaaa
+//   ..hhhhhh..gggggg....ffff....eeee
+// make
+//   ..dddddd..cccccc....bbbb....aaaa
+//   000..dddddd..cccccc....bbbb....a
+//   ..hhhhhh..gggggg....ffff....eeee
+//   ..ffff....eeee000000000000000000
+
+// BIGRAM
+// Pick up 1..8 bytes and hash them via mask/shift/add. NO pre/post
+// OVERSHOOTS up to 3 bytes
+// For runtime use of tables
+uint32 cld::BiHashV25(const char* word_ptr, int bytecount) {
+  if (bytecount == 0) {
+    return 0;
+  }
+  const uint32* word_ptr32 = reinterpret_cast<const uint32*>(word_ptr);
+  uint32 word0, word1;
+  if (bytecount <= 4) {
+    word0 = word_ptr32[0] & kWordMask0[bytecount & 3];
+    word0 = word0 ^ (word0 >> 3);
+    return word0;
+  }
+  // Else do 8 bytes
+  word0 = word_ptr32[0];
+  word0 = word0 ^ (word0 >> 3);
+  word1 = word_ptr32[1] & kWordMask0[bytecount & 3];
+  word1 = word1 ^ (word1 << 18);
+  return word0 + word1;
+}
+
+//
+// Ascii-7 One-byte chars
+//   ...ddddd...ccccc...bbbbb...aaaaa
+// make
+//   ...ddddd...ccccc...bbbbb...aaaaa
+//   000...ddddd...ccccc...bbbbb...aa
+//
+// Latin 1- and 2-byte chars
+//   ...ddddd...ccccc...bbbbb...aaaaa
+//   ...................fffff...eeeee
+// make
+//   ...ddddd...ccccc...bbbbb...aaaaa
+//   000...ddddd...ccccc...bbbbb...aa
+//   ...................fffff...eeeee
+//   ...............fffff...eeeee0000
+//
+// Non-CJK Two-byte chars
+//   ...ddddd...........bbbbb........
+//   ...hhhhh...........fffff........
+// make
+//   ...ddddd...........bbbbb........
+//   000...ddddd...........bbbbb.....
+//   ...hhhhh...........fffff........
+//   hhhh...........fffff........0000
+//
+// Non-CJK Three-byte chars
+//   ...........ccccc................
+//   ...................fffff........
+//   ...lllll...................iiiii
+// make
+//   ...........ccccc................
+//   000...........ccccc.............
+//   ...................fffff........
+//   ...............fffff........0000
+//   ...lllll...................iiiii
+//   .lllll...................iiiii00
+//
+
+// QUADGRAM
+// Pick up 1..12 bytes plus pre/post space and hash them via mask/shift/add
+// OVERSHOOTS up to 3 bytes
+// For runtime use of tables
+uint32 QuadHashV25Mix(const char* word_ptr, int bytecount, uint32 prepost) {
+  const uint32* word_ptr32 = reinterpret_cast<const uint32*>(word_ptr);
+  uint32 word0, word1, word2;
+  if (bytecount <= 4) {
+    word0 = word_ptr32[0] & kWordMask0[bytecount & 3];
+    word0 = word0 ^ (word0 >> 3);
+    return word0 ^ prepost;
+  } else if (bytecount <= 8) {
+    word0 = word_ptr32[0];
+    word0 = word0 ^ (word0 >> 3);
+    word1 = word_ptr32[1] & kWordMask0[bytecount & 3];
+    word1 = word1 ^ (word1 << 4);
+    return (word0 ^ prepost) + word1;
+  }
+  // else do 12 bytes
+  word0 = word_ptr32[0];
+  word0 = word0 ^ (word0 >> 3);
+  word1 = word_ptr32[1];
+  word1 = word1 ^ (word1 << 4);
+  word2 = word_ptr32[2] & kWordMask0[bytecount & 3];
+  word2 = word2 ^ (word2 << 2);
+  return (word0 ^ prepost) + word1 + word2;
+}
+
+
+// QUADGRAM wrapper with surrounding spaces
+// Pick up 1..12 bytes plus pre/post space and hash them via mask/shift/add
+// UNDERSHOOTS 1 byte, OVERSHOOTS up to 3 bytes
+// For runtime use of tables
+uint32 cld::QuadHashV25(const char* word_ptr, int bytecount) {
+  if (bytecount == 0) {
+    return 0;
+  }
+  uint32 prepost = 0;
+  if (word_ptr[-1] == ' ') {prepost |= kPreSpaceIndicator;}
+  if (word_ptr[bytecount] == ' ') {prepost |= kPostSpaceIndicator;}
+  return QuadHashV25Mix(word_ptr, bytecount, prepost);
+}
+
+// QUADGRAM wrapper with surrounding underscores (offline use)
+// Pick up 1..12 bytes plus pre/post '_' and hash them via mask/shift/add
+// OVERSHOOTS up to 3 bytes
+// For offline construction of tables
+uint32 cld::QuadHashV25Underscore(const char* word_ptr, int bytecount) {
+  if (bytecount == 0) {
+    return 0;
+  }
+  const char* local_word_ptr = word_ptr;
+  int local_bytecount = bytecount;
+  uint32 prepost = 0;
+  if (local_word_ptr[0] == '_') {
+    prepost |= kPreSpaceIndicator;
+    ++local_word_ptr;
+    --local_bytecount;
+  }
+  if (local_word_ptr[local_bytecount - 1] == '_') {
+    prepost |= kPostSpaceIndicator;
+    --local_bytecount;
+  }
+  return QuadHashV25Mix(local_word_ptr, local_bytecount, prepost);
+}
+
+
+// OCTAGRAM
+// Pick up 1..24 bytes plus pre/post space and hash them via mask/shift/add
+// UNDERSHOOTS 1 byte, OVERSHOOTS up to 3 bytes
+//
+// The low 32 bits follow the pattern from above, tuned to different scripts
+// The high 8 bits are a simple sum of all bytes, shifted by 0/1/2/3 bits each
+// For runtime use of tables V3
+uint64 OctaHash40Mix(const char* word_ptr, int bytecount, uint64 prepost) {
+  const uint32* word_ptr32 = reinterpret_cast<const uint32*>(word_ptr);
+  uint64 word0;
+  uint64 word1;
+  uint64 sum;
+
+  if (word_ptr[-1] == ' ') {prepost |= kPreSpaceIndicator;}
+  if (word_ptr[bytecount] == ' ') {prepost |= kPostSpaceIndicator;}
+  switch ((bytecount - 1) >> 2) {
+  case 0:       // 1..4 bytes
+    word0 = word_ptr32[0] & kWordMask0[bytecount & 3];
+    sum = word0;
+    word0 = word0 ^ (word0 >> 3);
+    break;
+  case 1:       // 5..8 bytes
+    word0 = word_ptr32[0];
+    sum = word0;
+    word0 = word0 ^ (word0 >> 3);
+    word1 = word_ptr32[1] & kWordMask0[bytecount & 3];
+    sum += word1;
+    word1 = word1 ^ (word1 << 4);
+    word0 += word1;
+    break;
+  case 2:       // 9..12 bytes
+    word0 = word_ptr32[0];
+    sum = word0;
+    word0 = word0 ^ (word0 >> 3);
+    word1 = word_ptr32[1];
+    sum += word1;
+    word1 = word1 ^ (word1 << 4);
+    word0 += word1;
+    word1 = word_ptr32[2] & kWordMask0[bytecount & 3];
+    sum += word1;
+    word1 = word1 ^ (word1 << 2);
+    word0 += word1;
+    break;
+  case 3:       // 13..16 bytes
+    word0 = word_ptr32[0];
+    sum = word0;
+    word0 = word0 ^ (word0 >> 3);
+    word1 = word_ptr32[1];
+    sum += word1;
+    word1 = word1 ^ (word1 << 4);
+    word0 += word1;
+    word1 = word_ptr32[2];
+    sum += word1;
+    word1 = word1 ^ (word1 << 2);
+    word0 += word1;
+    word1 = word_ptr32[3] & kWordMask0[bytecount & 3];
+    sum += word1;
+    word1 = word1 ^ (word1 >> 8);
+    word0 += word1;
+    break;
+  case 4:       // 17..20 bytes
+    word0 = word_ptr32[0];
+    sum = word0;
+    word0 = word0 ^ (word0 >> 3);
+    word1 = word_ptr32[1];
+    sum += word1;
+    word1 = word1 ^ (word1 << 4);
+    word0 += word1;
+    word1 = word_ptr32[2];
+    sum += word1;
+    word1 = word1 ^ (word1 << 2);
+    word0 += word1;
+    word1 = word_ptr32[3];
+    sum += word1;
+    word1 = word1 ^ (word1 >> 8);
+    word0 += word1;
+    word1 = word_ptr32[4] & kWordMask0[bytecount & 3];
+    sum += word1;
+    word1 = word1 ^ (word1 >> 4);
+    word0 += word1;
+    break;
+  default:      // 21..24 bytes and higher (ignores beyond 24)
+    word0 = word_ptr32[0];
+    sum = word0;
+    word0 = word0 ^ (word0 >> 3);
+    word1 = word_ptr32[1];
+    sum += word1;
+    word1 = word1 ^ (word1 << 4);
+    word0 += word1;
+    word1 = word_ptr32[2];
+    sum += word1;
+    word1 = word1 ^ (word1 << 2);
+    word0 += word1;
+    word1 = word_ptr32[3];
+    sum += word1;
+    word1 = word1 ^ (word1 >> 8);
+    word0 += word1;
+    word1 = word_ptr32[4];
+    sum += word1;
+    word1 = word1 ^ (word1 >> 4);
+    word0 += word1;
+    word1 = word_ptr32[5] & kWordMask0[bytecount & 3];
+    sum += word1;
+    word1 = word1 ^ (word1 >> 6);
+    word0 += word1;
+    break;
+  }
+
+  sum += (sum >> 17);             // extra 1-bit shift for bytes 2 & 3
+  sum += (sum >> 9);              // extra 1-bit shift for bytes 1 & 3
+  sum = (sum & 0xff) << 32;
+  return (word0 ^ prepost) + sum;
+}
+
+// OCTAGRAM wrapper with surrounding spaces
+// Pick up 1..24 bytes plus pre/post space and hash them via mask/shift/add
+// UNDERSHOOTS 1 byte, OVERSHOOTS up to 3 bytes
+//
+// The low 32 bits follow the pattern from above, tuned to different scripts
+// The high 8 bits are a simple sum of all bytes, shifted by 0/1/2/3 bits each
+// For runtime use of tables V3
+uint64 cld::OctaHash40(const char* word_ptr, int bytecount) {
+  if (bytecount == 0) {
+    return 0;
+  }
+  uint64 prepost = 0;
+  if (word_ptr[-1] == ' ') {prepost |= kPreSpaceIndicator;}
+  if (word_ptr[bytecount] == ' ') {prepost |= kPostSpaceIndicator;}
+  return OctaHash40Mix(word_ptr, bytecount, prepost);
+}
+
+
+// OCTAGRAM wrapper with surrounding underscores (offline use)
+// Pick up 1..24 bytes plus pre/post space and hash them via mask/shift/add
+// UNDERSHOOTS 1 byte, OVERSHOOTS up to 3 bytes
+//
+// The low 32 bits follow the pattern from above, tuned to different scripts
+// The high 8 bits are a simple sum of all bytes, shifted by 0/1/2/3 bits each
+// For offline construction of tables
+uint64 cld::OctaHash40underscore(const char* word_ptr, int bytecount) {
+  if (bytecount == 0) {
+    return 0;
+  }
+  const char* local_word_ptr = word_ptr;
+  int local_bytecount = bytecount;
+  uint64 prepost = 0;
+  if (local_word_ptr[0] == '_') {
+    prepost |= kPreSpaceIndicator;
+    ++local_word_ptr;
+    --local_bytecount;
+  }
+  if (local_word_ptr[local_bytecount - 1] == '_') {
+    prepost |= kPostSpaceIndicator;
+    --local_bytecount;
+  }
+  return OctaHash40Mix(local_word_ptr, local_bytecount, prepost);
+}
+
+
+
+
+//------------------------------------------------------------------------------
+// Scoring single groups of letters
+//------------------------------------------------------------------------------
+
+// UNIGRAM score one => tote
+// Input: 1-byte entry of subscript into unigram probs, plus
+//  an accumulator tote.
+// Output: running sums in tote updated
+void cld::ProcessProbV25UniTote(int propval, Tote* tote) {
+  tote->AddGram();
+  const UnigramProbArray* pa = &kTargetCTJKVZProbs[propval];
+  if (pa->probs[0] > 0) {tote->Add(cld::PackLanguage(CHINESE), pa->probs[0]);}
+  if (pa->probs[1] > 0) {tote->Add(cld::PackLanguage(CHINESE_T), pa->probs[1]);}
+  if (pa->probs[2] > 0) {tote->Add(cld::PackLanguage(JAPANESE), pa->probs[2]);}
+  if (pa->probs[3] > 0) {tote->Add(cld::PackLanguage(KOREAN), pa->probs[3]);}
+  if (pa->probs[4] > 0) {tote->Add(cld::PackLanguage(VIETNAMESE), pa->probs[4]);}
+  if (pa->probs[5] > 0) {tote->Add(cld::PackLanguage(ZHUANG), pa->probs[5]);}
+}
+
+// BIGRAM, QUADGRAM, OCTAGRAM score one => tote
+// Input: 4-byte entry of 3 language numbers and one probability subscript, plus
+//  an accumulator tote. (language 0 means unused entry)
+// Output: running sums in tote updated
+void cld::ProcessProbV25Tote(uint32 probs, Tote* tote) {
+  tote->AddGram();
+  uint8 prob123 = (probs >> 0) & 0xff;
+  const uint8* prob123_entry = cld::LgProb2TblEntry(prob123);
+
+  uint8 top1 = (probs >> 8) & 0xff;
+  if (top1 > 0) {tote->Add(top1, cld::LgProb3(prob123_entry, 0));}
+  uint8 top2 = (probs >> 16) & 0xff;
+  if (top2 > 0) {tote->Add(top2, cld::LgProb3(prob123_entry, 1));}
+  uint8 top3 = (probs >> 24) & 0xff;
+  if (top3 > 0) {tote->Add(top3, cld::LgProb3(prob123_entry, 2));}
+}
+
+
+//------------------------------------------------------------------------------
+// Routines to accumulate probabilities
+//------------------------------------------------------------------------------
+
+
+// UNIGRAM, using UTF-8 property table, advancing by 1/2/4/8 chars
+// Caller supplies table, such as compact_lang_det_generated_ctjkvz_b1_obj
+// Score up to n unigrams, returning number of bytes consumed
+// Updates tote_grams
+int cld::DoUniScoreV3(const UTF8PropObj* unigram_obj,
+                      const char* isrc, int srclen, int advance_by,
+                      int* tote_grams, int gram_limit, Tote* chunk_tote) {
+  const char* src = isrc;
+  if (FLAGS_dbgscore) {DbgScoreInit(src, srclen);}
+
+  // Property-based CJK unigram lookup
+  if (src[0] == ' ') {++src; --srclen;}
+
+  const uint8* usrc = reinterpret_cast<const uint8*>(src);
+  int usrclen = srclen;
+
+  while (usrclen > 0) {
+    int len = kAdvanceOneChar[usrc[0]];
+    // Look up property of one UTF-8 character and advance over it
+    // Return 0 if input length is zero
+    // Return 0 and advance one byte if input is ill-formed
+
+    int propval = UTF8GenericPropertyBigOneByte(unigram_obj, &usrc, &usrclen);
+
+    if (FLAGS_dbglookup) {
+      DbgUniTermToStderr(propval, usrc, len);
+    }
+
+    if (propval > 0) {
+      ProcessProbV25UniTote(propval, chunk_tote);
+      ++(*tote_grams);
+      if (FLAGS_dbgscore) {DbgScoreRecordUni((const char*)usrc, propval, len);}
+    }
+
+    // Advance by 1/2/4/8 characters (half of quad advance)
+    if (advance_by == 2) {
+      // Already advanced by 1
+    } else if (advance_by == 4) {
+      // Advance by 2 chars total, if not at end
+      if (UTFmax <= usrclen) {
+        int n = kAdvanceOneChar[*usrc]; usrc += n; usrclen -= n;
+      }
+    } else if (advance_by == 8) {
+      // Advance by 4 chars total, if not at end
+      if ((UTFmax * 3) <= usrclen) {
+        int n = kAdvanceOneChar[*usrc]; usrc += n; usrclen -= n;
+        n = kAdvanceOneChar[*usrc]; usrc += n; usrclen -= n;
+        n = kAdvanceOneChar[*usrc]; usrc += n; usrclen -= n;
+      }
+    } else {
+      // Advance by 8 chars total, if not at end
+      if ((UTFmax * 7) <= usrclen) {
+        int n = kAdvanceOneChar[*usrc]; usrc += n; usrclen -= n;
+        n = kAdvanceOneChar[*usrc]; usrc += n; usrclen -= n;
+        n = kAdvanceOneChar[*usrc]; usrc += n; usrclen -= n;
+        n = kAdvanceOneChar[*usrc]; usrc += n; usrclen -= n;
+        n = kAdvanceOneChar[*usrc]; usrc += n; usrclen -= n;
+        n = kAdvanceOneChar[*usrc]; usrc += n; usrclen -= n;
+        n = kAdvanceOneChar[*usrc]; usrc += n; usrclen -= n;
+      }
+    }
+    DCHECK(usrclen >= 0);
+
+    if (*tote_grams >= gram_limit) {
+      break;
+    }
+  }
+  if (FLAGS_dbgscore) {
+    // With advance_by>2, we consume more input to get the same number of quads
+    int len = src - isrc;
+    DbgScoreTop(src, (len * 2) / advance_by, chunk_tote);
+    DbgScoreFlush();
+  }
+
+  int consumed2 = reinterpret_cast<const char*>(usrc) - isrc;
+  return consumed2;
+}
+
+
+// BIGRAM, using hash table, always advancing by 1 char
+// Caller supplies table, such as &kCjkBiTable_obj or &kGibberishTable_obj
+// Score all bigrams in isrc, using languages that have bigrams (CJK)
+// Return number of bigrams that hit in the hash table
+int cld::DoBigramScoreV3(const cld::CLDTableSummary* bigram_obj,
+                         const char* isrc, int srclen, Tote* chunk_tote) {
+  int hit_count = 0;
+  const char* src = isrc;
+
+  // Hashtable-based CJK bigram lookup
+  const uint8* usrc = reinterpret_cast<const uint8*>(src);
+  const uint8* usrclimit1 = usrc + srclen - UTFmax;
+  if (FLAGS_dbgscore) {
+    fprintf(stderr, "  " );
+  }
+
+  while (usrc < usrclimit1) {
+    int len = kAdvanceOneChar[usrc[0]];
+    int len2 = kAdvanceOneChar[usrc[len]] + len;
+
+    if ((kMinCJKUTF8CharBytes * 2) <= len2) {      // Two CJK chars possible
+      // Lookup and score this bigram
+      // Always ignore pre/post spaces
+      uint32 bihash = BiHashV25(reinterpret_cast<const char*>(usrc), len2);
+      uint32 probs = QuadHashV3Lookup4(bigram_obj, bihash);
+      // Now go indirect on the subscript
+      probs = bigram_obj->kCLDTableInd[probs &
+        ~bigram_obj->kCLDTableKeyMask];
+
+      // Process the bigram
+      if (FLAGS_dbglookup) {
+        const char* ssrc = reinterpret_cast<const char*>(usrc);
+        DbgBiTermToStderr(bihash, probs, ssrc, len2);
+        DbgScoreRecord(NULL, probs, len2);
+      } else if (FLAGS_dbgscore && (probs != 0)) {
+        const char* ssrc = reinterpret_cast<const char*>(usrc);
+        DbgScoreRecord(NULL, probs, len2);
+        string temp(ssrc, len2);
+        fprintf(stderr, "%s ", temp.c_str());
+      }
+
+      if (probs != 0) {
+        ProcessProbV25Tote(probs, chunk_tote);
+        ++hit_count;
+      }
+    }
+    usrc += len;  // Advance by one char
+  }
+
+  if (FLAGS_dbgscore) {
+    fprintf(stderr, "[%d bigrams scored]\n", hit_count);
+    DbgScoreState();
+  }
+  return hit_count;
+}
+
+
+
+// QUADGRAM, using hash table, advancing by 2/4/8/16 chars
+// Caller supplies table, such as &kQuadTable_obj or &kGibberishTable_obj
+// Score up to n quadgrams, returning number of bytes consumed
+// Updates tote_grams
+int cld::DoQuadScoreV3(const cld::CLDTableSummary* quadgram_obj,
+                       const char* isrc, int srclen, int advance_by,
+                       int* tote_grams, int gram_limit, Tote* chunk_tote) {
+  const char* src = isrc;
+  const char* srclimit = src + srclen;
+  // Limit is end, which has extra 20 20 20 00 past len
+  const char* srclimit7 = src + srclen - (UTFmax * 7);
+  const char* srclimit15 = src + srclen - (UTFmax * 15);
+
+  if (FLAGS_dbgscore) {DbgScoreInit(src, srclen);}
+
+  // Run a little cache of last hits to catch overly-repetitive "text"
+  int next_prior = 0;
+  uint32 prior_quads[2] = {0, 0};
+
+  // Visit all quadgrams
+  if (src[0] == ' ') {++src;}
+  while (src < srclimit) {
+    // Find one quadgram
+    const char* src_end = src;
+    src_end += kAdvanceOneCharButSpace[(uint8)src_end[0]];
+    src_end += kAdvanceOneCharButSpace[(uint8)src_end[0]];
+    const char* src_mid = src_end;
+    src_end += kAdvanceOneCharButSpace[(uint8)src_end[0]];
+    src_end += kAdvanceOneCharButSpace[(uint8)src_end[0]];
+    int len = src_end - src;
+
+    // Lookup and score this quadgram
+    uint32 quadhash = QuadHashV25(src, len);
+    uint32 probs = QuadHashV3Lookup4(quadgram_obj, quadhash);
+    // Now go indirect on the subscript
+    probs = quadgram_obj->kCLDTableInd[probs &
+      ~quadgram_obj->kCLDTableKeyMask];
+
+    // Process the quadgram
+    if (FLAGS_dbglookup) {
+      DbgQuadTermToStderr(quadhash, probs, src, len);
+    }
+    if (probs != 0) {
+      // Filter out recent repeats. If this works out, use in the other lookups
+      if ((quadhash != prior_quads[0]) && (quadhash != prior_quads[1])) {
+        prior_quads[next_prior] = quadhash;
+        next_prior = (next_prior + 1) & 1;
+        ProcessProbV25Tote(probs, chunk_tote);
+        ++(*tote_grams);
+        if (FLAGS_dbgscore) {DbgScoreRecord(src, probs, len);}
+      }
+    }
+
+    // Advance all the way past word if at end-of-word
+    if (src_end[0] == ' ') {
+      src_mid = src_end;
+    }
+
+    // Advance by 2/4/8/16 characters
+    if (advance_by == 2) {
+      src = src_mid;
+    } else if (advance_by == 4) {
+      src = src_end;
+    } else if (advance_by == 8) {
+      // Advance by 8 chars total (4 more), if not at end
+      if (src < srclimit7) {
+        src_end += kAdvanceOneChar[(uint8)src_end[0]];
+        src_end += kAdvanceOneChar[(uint8)src_end[0]];
+        src_end += kAdvanceOneChar[(uint8)src_end[0]];
+        src_end += kAdvanceOneChar[(uint8)src_end[0]];
+      }
+      src = src_end;
+    } else {
+      // Advance by 16 chars total (12 more), if not at end
+      if (src < srclimit15) {
+        // Advance by ~16 chars by adding 3 * current bytelen
+        int fourcharlen = src_end - src;
+        src = src_end + (3 * fourcharlen);
+        // Advance a bit more if mid-character
+        src += kAdvanceOneCharSpaceVowel[(uint8)src[0]];
+        src += kAdvanceOneCharSpaceVowel[(uint8)src[0]];
+      } else {
+        src = src_end;
+      }
+    }
+    DCHECK(src < srclimit);
+    src += kAdvanceOneCharSpaceVowel[(uint8)src[0]];
+
+    if (*tote_grams >= gram_limit) {
+      break;
+    }
+  }
+
+  if (FLAGS_dbgscore) {
+    // With advance_by>2, we consume more input to get the same number of quads
+    int len = src - isrc;
+    DbgScoreTop(src, (len * 2) / advance_by, chunk_tote);
+    DbgScoreFlush();
+  }
+
+  int consumed = src - isrc;
+
+  // If advancing by more than 2, src may have overshot srclimit
+  if (consumed > srclen) {
+    consumed = srclen;
+  }
+
+  return consumed;
+}
+
+
+// OCTAGRAM, using hash table, always advancing by 1 word
+// Caller supplies table, such as &kLongWord8Table_obj
+// Score all words in isrc, using languages that have quadgrams
+// We don't normally use this routine except on the first quadgram run,
+// but it can be used to resolve unreliable pages.
+// This routine does not have an optimized advance_by
+// SOON: Uses indirect language/probability longword
+//
+// Return number of words that hit in the hash table
+int cld::DoOctaScoreV3(const cld::CLDTableSummary* octagram_obj,
+                       const char* isrc, int srclen, Tote* chunk_tote) {
+  int hit_count = 0;
+  const char* src = isrc;
+  const char* srclimit = src + srclen + 1;
+  // Limit is end+1, to include extra space char (0x20) off the end
+  //
+  // Score all words truncated to 8 characters
+  int charcount = 0;
+  // Skip any initial space
+  if (src[0] == ' ') {++src;}
+  const char* word_ptr = src;
+  const char* word_end = word_ptr;
+  if (FLAGS_dbgscore) {
+    fprintf(stderr, "  " );
+  }
+  while (src < srclimit) {
+    // Terminate previous word or continue current word
+    if (src[0] == ' ') {
+      int bytecount = word_end - word_ptr;
+      if (bytecount == 0)
+        break;
+      // Lookup and score this word
+      uint64 wordhash40 = OctaHash40(word_ptr, bytecount);
+      uint32 probs = OctaHashV3Lookup4(octagram_obj, wordhash40);
+      // Now go indirect on the subscript
+      probs = octagram_obj->kCLDTableInd[probs &
+        ~octagram_obj->kCLDTableKeyMask];
+
+      // // Lookup and score this word
+      // uint32 wordhash = QuadHashV25(word_ptr, bytecount);
+      // uint32 probs = WordHashLookup4(wordhash, kLongWord8Table,
+      //                                kLongWord8TableSize);
+      //
+      if (FLAGS_dbglookup) {
+        DbgWordTermToStderr(wordhash40, probs, word_ptr, bytecount);
+        DbgScoreRecord(NULL, probs, bytecount);
+      } else if (FLAGS_dbgscore && (probs != 0)) {
+        DbgScoreRecord(NULL, probs, bytecount);
+        string temp(word_ptr, bytecount);
+        fprintf(stderr, "%s ", temp.c_str());
+      }
+
+      if (probs != 0) {
+        ProcessProbV25Tote(probs, chunk_tote);
+        ++hit_count;
+      }
+      charcount = 0;
+      word_ptr = src + 1;   // Over the space
+      word_end = word_ptr;
+    } else {
+      ++charcount;
+    }
+
+    // Advance to next char
+    src += cld_UniLib::OneCharLen(src);
+    if (charcount <= 8) {
+      word_end = src;
+    }
+  }
+
+  if (FLAGS_dbgscore) {
+    fprintf(stderr, "[%d words scored]\n", hit_count);
+    DbgScoreState();
+  }
+  return hit_count;
+}
+
+
+
+//------------------------------------------------------------------------------
+// Reliability calculations, for single language and between languages
+//------------------------------------------------------------------------------
+
+// Return reliablity of result 0..100 for top two scores
+// delta==0 is 0% reliable, delta==fully_reliable_thresh is 100% reliable
+// (on a scale where +1 is a factor of  2 ** 1.6 = 3.02)
+// Threshold is uni/quadgram increment count, bounded above and below.
+//
+// Requiring a factor of 3 improvement (e.g. +1 log base 3)
+// for each scored quadgram is too stringent, so I've backed this off to a
+// factor of 2 (e.g. +5/8 log base 3).
+//
+// I also somewhat lowered the Min/MaxGramCount limits above
+//
+// Added: if fewer than 8 quads/unis, max reliability is 12*n percent
+//
+int cld::ReliabilityDelta(int value1, int value2, int gramcount) {
+  int max_reliability_percent = 100;
+  if (gramcount < 8) {
+    max_reliability_percent = 12 * gramcount;
+  }
+  int fully_reliable_thresh = (gramcount * 5) >> 3;     // see note above
+  if (fully_reliable_thresh < kMinGramCount) {          // Fully = 3..16
+    fully_reliable_thresh = kMinGramCount;
+  } else if (fully_reliable_thresh > kMaxGramCount) {
+    fully_reliable_thresh = kMaxGramCount;
+  }
+
+  int delta = value1 - value2;
+  if (delta >= fully_reliable_thresh) {return max_reliability_percent;}
+  if (delta <= 0) {return 0;}
+  return cld::minint(max_reliability_percent,
+                     (100 * delta) / fully_reliable_thresh);
+}
+
+// Return reliablity of result 0..100 for top score vs. mainsteam score
+// Values are score per 1024 bytes of input
+// ratio = max(top/mainstream, mainstream/top)
+// ratio > 4.0 is 0% reliable, <= 2.0 is 100% reliable
+// Change: short-text word scoring can give unusually good results.
+//  Let top exceed mainstream by 4x at 50% reliable
+int cld::ReliabilityMainstream(int topscore, int len, int mean_score) {
+  if (mean_score == 0) {return 100;}    // No reliability data available yet
+  if (topscore == 0) {return 0;}        // zero score = unreliable
+  if (len == 0) {return 0;}             // zero len = unreliable
+  int top_kb = (topscore << 10) / len;
+  double ratio;
+  double ratio_cutoff;
+  if (top_kb > mean_score) {
+    ratio = (1.0 * top_kb) / mean_score;
+    ratio_cutoff = 5.0;                 // ramp down from 100% to 0%: 3.0-5.0
+  } else {
+    ratio = (1.0 * mean_score) / top_kb;
+    ratio_cutoff = 4.0;                 // ramp down from 100% to 0%: 2.0-4.0
+  }
+  if (ratio <= ratio_cutoff - 2.0) {return 100;}
+  if (ratio > ratio_cutoff) {return 0;}
+
+  int iratio = static_cast<int>(100 * (ratio_cutoff - ratio) / 2.0);
+  return iratio;
+}
+
+// Calculate ratio of score per 1KB vs. expected score per 1KB
+double cld::GetNormalizedScore(Language lang, UnicodeLScript lscript,
+                          int bytes, int score) {
+  // Average training-data score for this language-script combo, per 1KB
+  int expected_score = kMeanScore[lang * 4 + LScript4(lscript)];
+  if (lscript == ULScript_Common) {
+    // We don't know the script (only happens with second-chance score)
+    // Look for first non-zero mean value
+    for (int i = 0; i < 3; ++i) {
+      if (kMeanScore[lang * 4 + i] > 0) {
+        expected_score = kMeanScore[lang * 4 + i];
+      }
+    }
+  }
+  if (expected_score < 100) {
+      expected_score = 1000;
+  }
+
+  // Our score per 1KB
+  double our_score = (score << 10) / (bytes ? bytes : 1);  // Avoid zdiv
+  double ratio = our_score / expected_score;
+
+  // Just the raw count normalized as though each language has mean=1000;
+  ratio = (score * 1000.0) /  expected_score;
+  return ratio;
+}
+
+// Calculate reliablity of len bytes of script lscript with chunk_tote
+int cld::GetReliability(int len, UnicodeLScript lscript,
+                   const Tote* chunk_tote) {
+  Language cur_lang = UnpackLanguage(chunk_tote->Key(0));
+  // Average score for this language-script combo
+  int mean_score = kMeanScore[cur_lang * 4 + LScript4(lscript)];
+  if (lscript == ULScript_Common) {
+    // We don't know the script (only happens with second-chance score)
+    // Look for first non-zero mean value
+    for (int i = 0; i < 3; ++i) {
+      if (kMeanScore[cur_lang * 4 + i] > 0) {
+        mean_score = kMeanScore[cur_lang * 4 + i];
+      }
+    }
+  }
+  int reliability_delta = ReliabilityDelta(chunk_tote->Value(0),
+                                           chunk_tote->Value(1),
+                                           chunk_tote->GetGramCount());
+
+  int reliability_main = ReliabilityMainstream(chunk_tote->Value(0),
+                                               len,
+                                               mean_score);
+
+  int reliability_min = minint(reliability_delta, reliability_main);
+
+
+  if (FLAGS_dbgreli) {
+    char temp1[4];
+    char temp2[4];
+    cld::DbgLangName3(UnpackLanguage(chunk_tote->Key(0)), temp1);
+    if (temp1[2] == ' ') {temp1[2] = '\0';}
+    cld::DbgLangName3(UnpackLanguage(chunk_tote->Key(1)), temp2);
+    if (temp2[2] == ' ') {temp2[2] = '\0';}
+    int srclen = len;
+    fprintf(stderr, "CALC GetReliability gram=%d incr=%d srclen=%d,  %s=%d %s=%d "
+                   "top/KB=%d mean/KB=%d del=%d%% reli=%d%%   "
+                   "lang/lscript %d %d\n",
+           chunk_tote->GetGramCount(),
+           chunk_tote->GetIncrCount(),
+           srclen,
+           temp1, chunk_tote->Value(0),
+           temp2, chunk_tote->Value(1),
+           (chunk_tote->Value(0) << 10) / (srclen ? srclen : 1),
+           mean_score,
+           reliability_delta,
+           reliability_main,
+           cur_lang, lscript);
+  }
+
+  return reliability_min;
+}
+
+
+//------------------------------------------------------------------------------
+// Miscellaneous
+//------------------------------------------------------------------------------
+
+// Demote all languages except Top40 and plus_one
+// Do this just before sorting chunk_tote results
+void cld::DemoteNotTop40(Tote* chunk_tote, int packed_plus_one) {
+  for (int sub = 0; sub < chunk_tote->MaxSize(); ++sub) {
+    if (chunk_tote->Key(sub) == 0) continue;
+    if (chunk_tote->Key(sub) == packed_plus_one) continue;
+    if (kIsPackedTop40[chunk_tote->Key(sub)]) continue;
+    // Quarter the score of others
+    chunk_tote->SetValue(sub, chunk_tote->Value(sub) >> 2);
+  }
+}