cld 0.1.0

data/LICENSE

@@ -0,0 +1,27 @@
+// Copyright (c) 2010 The Chromium Authors. All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//    * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//    * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//    * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

data/Manifest

@@ -0,0 +1,106 @@
+LICENSE
+README.rdoc
+Rakefile
+base/basictypes.h
+base/build_config.h
+base/casts.h
+base/commandlineflags.h
+base/crash.h
+base/dynamic_annotations.h
+base/global_strip_options.h
+base/log_severity.h
+base/logging.h
+base/macros.h
+base/port.h
+base/scoped_ptr.h
+base/stl_decl.h
+base/stl_decl_msvc.h
+base/string_util.h
+base/strtoint.h
+base/template_util.h
+base/type_traits.h
+base/vlog_is_on.h
+build.sh
+build.win.cmd
+cld.gemspec
+cld_encodings.h
+encodings/compact_lang_det/#cldutil.cc#
+encodings/compact_lang_det/#cldutil.h#
+encodings/compact_lang_det/#compact_lang_det_impl.h#
+encodings/compact_lang_det/#ext_lang_enc.cc#
+encodings/compact_lang_det/#ext_lang_enc.h#
+encodings/compact_lang_det/#getonescriptspan.cc#
+encodings/compact_lang_det/#getonescriptspan.h#
+encodings/compact_lang_det/#tote.cc#
+encodings/compact_lang_det/#tote.h#
+encodings/compact_lang_det/cldutil.cc
+encodings/compact_lang_det/cldutil.h
+encodings/compact_lang_det/cldutil_dbg.h
+encodings/compact_lang_det/cldutil_dbg_empty.cc
+encodings/compact_lang_det/compact_lang_det.cc
+encodings/compact_lang_det/compact_lang_det.h
+encodings/compact_lang_det/compact_lang_det_impl.cc
+encodings/compact_lang_det/compact_lang_det_impl.h
+encodings/compact_lang_det/compact_lang_det_unittest_small.cc
+encodings/compact_lang_det/compile.cmd
+encodings/compact_lang_det/ext_lang_enc.cc
+encodings/compact_lang_det/ext_lang_enc.h
+encodings/compact_lang_det/generated/cld_generated_score_deltaoctachrome_0406.cc
+encodings/compact_lang_det/generated/cld_generated_score_quadchrome_0406.cc
+encodings/compact_lang_det/generated/compact_lang_det_generated_cjkbis_0.cc
+encodings/compact_lang_det/generated/compact_lang_det_generated_ctjkvz.cc
+encodings/compact_lang_det/generated/compact_lang_det_generated_ctjkvz_0.cc
+encodings/compact_lang_det/generated/compact_lang_det_generated_deltaoctachrome.cc
+encodings/compact_lang_det/generated/compact_lang_det_generated_longwords8_0.cc
+encodings/compact_lang_det/generated/compact_lang_det_generated_meanscore.h
+encodings/compact_lang_det/generated/compact_lang_det_generated_quads_0.cc
+encodings/compact_lang_det/generated/compact_lang_det_generated_quadschrome.cc
+encodings/compact_lang_det/getonescriptspan.cc
+encodings/compact_lang_det/getonescriptspan.h
+encodings/compact_lang_det/letterscript_enum.cc
+encodings/compact_lang_det/letterscript_enum.h
+encodings/compact_lang_det/subsetsequence.cc
+encodings/compact_lang_det/subsetsequence.h
+encodings/compact_lang_det/subsetsequence_unittest.cc
+encodings/compact_lang_det/tote.cc
+encodings/compact_lang_det/tote.h
+encodings/compact_lang_det/unittest_data.h
+encodings/compact_lang_det/utf8propjustletter.h
+encodings/compact_lang_det/utf8propletterscriptnum.h
+encodings/compact_lang_det/utf8scannotjustletterspecial.h
+encodings/compact_lang_det/win/#cld_unilib_windows.cc#
+encodings/compact_lang_det/win/cld_basictypes.h
+encodings/compact_lang_det/win/cld_commandlineflags.h
+encodings/compact_lang_det/win/cld_google.h
+encodings/compact_lang_det/win/cld_htmlutils.h
+encodings/compact_lang_det/win/cld_htmlutils_google3.cc
+encodings/compact_lang_det/win/cld_htmlutils_windows.cc
+encodings/compact_lang_det/win/cld_logging.h
+encodings/compact_lang_det/win/cld_macros.h
+encodings/compact_lang_det/win/cld_strtoint.h
+encodings/compact_lang_det/win/cld_unicodetext.cc
+encodings/compact_lang_det/win/cld_unicodetext.h
+encodings/compact_lang_det/win/cld_unilib.h
+encodings/compact_lang_det/win/cld_unilib_google3.cc
+encodings/compact_lang_det/win/cld_unilib_windows.cc
+encodings/compact_lang_det/win/cld_utf.h
+encodings/compact_lang_det/win/cld_utf8statetable.cc
+encodings/compact_lang_det/win/cld_utf8statetable.h
+encodings/compact_lang_det/win/cld_utf8utils.h
+encodings/compact_lang_det/win/cld_utf8utils_google3.cc
+encodings/compact_lang_det/win/cld_utf8utils_windows.cc
+encodings/compact_lang_det/win/normalizedunicodetext.cc
+encodings/compact_lang_det/win/normalizedunicodetext.h
+encodings/internal/encodings.cc
+encodings/lang_enc.h
+encodings/proto/encodings.pb.h
+encodings/public/encodings.h
+ext/cld/extconf.rb
+languages/internal/#languages.cc#
+languages/internal/languages.cc
+languages/proto/languages.pb.h
+languages/public/languages.h
+lib/cld.rb
+test/test.rb
+thunk.cc
+Manifest

data/README.rdoc

@@ -0,0 +1,173 @@
+This is a wrapper of the Compact Language Detection library from Chrome.
+To use :
+require "cld"
+language = CLD.detect_language("piece of text")
+is_english = CLD.english?("我不是英文")
+
+
+detect_language returns a unique integer representing each language, here are the languages:
+ENGLISH       = 0,
+DANISH        = 1,
+DUTCH         = 2,
+FINNISH       = 3,
+FRENCH        = 4,
+GERMAN        = 5,
+HEBREW        = 6,
+ITALIAN       = 7,
+JAPANESE      = 8,
+KOREAN        = 9,
+NORWEGIAN     = 10,
+POLISH        = 11,
+PORTUGUESE    = 12,
+RUSSIAN       = 13,
+SPANISH       = 14,
+SWEDISH       = 15,
+CHINESE       = 16,
+CZECH         = 17,
+GREEK         = 18,
+ICELANDIC     = 19,
+LATVIAN       = 20,
+LITHUANIAN    = 21,
+ROMANIAN      = 22,
+HUNGARIAN     = 23,
+ESTONIAN      = 24,
+TG_UNKNOWN_LANGUAGE   = 25,
+UNKNOWN_LANGUAGE      = 26,
+BULGARIAN     = 27,
+CROATIAN      = 28,
+SERBIAN       = 29,
+IRISH         = 30,    
+GALICIAN      = 31,
+TAGALOG       = 32,  
+TURKISH       = 33,
+UKRAINIAN     = 34,
+HINDI         = 35,
+MACEDONIAN    = 36,
+BENGALI       = 37,
+INDONESIAN    = 38,
+LATIN         = 39, 
+MALAY         = 40,
+MALAYALAM     = 41,
+WELSH         = 42, 
+NEPALI        = 43,
+TELUGU        = 44,
+ALBANIAN      = 45,
+TAMIL         = 46,
+BELARUSIAN    = 47,
+JAVANESE      = 48,  
+OCCITAN       = 49, 
+URDU          = 50,
+BIHARI        = 51,
+GUJARATI      = 52,
+THAI          = 53,
+ARABIC        = 54,
+CATALAN       = 55,
+ESPERANTO     = 56,
+BASQUE        = 57,
+INTERLINGUA   = 58, 
+KANNADA       = 59,
+PUNJABI       = 60,
+SCOTS_GAELIC  = 61, 
+SWAHILI       = 62,
+SLOVENIAN     = 63,
+MARATHI       = 64,
+MALTESE       = 65,
+VIETNAMESE    = 66,
+FRISIAN       = 67, 
+SLOVAK        = 68,
+CHINESE_T     = 69, 
+FAROESE       = 70,
+SUNDANESE     = 71,
+UZBEK         = 72,
+AMHARIC       = 73,
+AZERBAIJANI   = 74,
+GEORGIAN      = 75,
+TIGRINYA      = 76, 
+PERSIAN       = 77,
+BOSNIAN       = 78,  
+SINHALESE     = 79,
+NORWEGIAN_N   = 80,
+PORTUGUESE_P  = 81,
+PORTUGUESE_B  = 82,
+XHOSA         = 83,  
+ZULU          = 84, 
+GUARANI       = 85,
+SESOTHO       = 86,  
+TURKMEN       = 87,
+KYRGYZ        = 88,
+BRETON        = 89,
+TWI           = 90, 
+YIDDISH       = 91, 
+SERBO_CROATIAN= 92,
+SOMALI        = 93,
+UIGHUR        = 94,
+KURDISH       = 95,
+MONGOLIAN     = 96,
+ARMENIAN      = 97,
+LAOTHIAN      = 98,
+SINDHI        = 99,
+RHAETO_ROMANCE= 100, 
+AFRIKAANS     = 101,
+LUXEMBOURGISH = 102, 
+BURMESE       = 103,
+KHMER         = 104,
+TIBETAN       = 105,
+DHIVEHI       = 106,  
+CHEROKEE      = 107,
+SYRIAC        = 108, 
+LIMBU         = 109,
+ORIYA         = 110,
+ASSAMESE      = 111,  
+CORSICAN      = 112,  
+INTERLINGUE   = 113,  
+KAZAKH        = 114,
+LINGALA       = 115,   
+MOLDAVIAN     = 116,    
+PASHTO        = 117,
+QUECHUA       = 118,  
+SHONA         = 119, 
+TAJIK         = 120,
+TATAR         = 121,  
+TONGA         = 122, 
+YORUBA        = 123,    
+CREOLES_AND_PIDGINS_ENGLISH_BASED       = 124,  
+CREOLES_AND_PIDGINS_FRENCH_BASED        = 125,
+CREOLES_AND_PIDGINS_PORTUGUESE_BASED    = 126, 
+CREOLES_AND_PIDGINS_OTHER               = 127, 
+MAORI         = 128,     
+WOLOF         = 129,    
+ABKHAZIAN     = 130,   
+AFAR          = 131,     
+AYMARA        = 132,    
+BASHKIR       = 133,    
+BISLAMA       = 134,   
+DZONGKHA      = 135, 
+FIJIAN        = 136,
+GREENLANDIC   = 137, 
+HAUSA         = 138, 
+HAITIAN_CREOLE= 139, 
+INUPIAK       = 140,   
+INUKTITUT     = 141,
+KASHMIRI      = 142, 
+KINYARWANDA   = 143,   
+MALAGASY      = 144, 
+NAURU         = 145, 
+OROMO         = 146, 
+RUNDI         = 147, 
+SAMOAN        = 148, 
+SANGO         = 149, 
+SANSKRIT      = 150,
+SISWANT       = 151, 
+TSONGA        = 152, 
+TSWANA        = 153,  
+VOLAPUK       = 154, 
+ZHUANG        = 155, 
+KHASI         = 156,   
+SCOTS         = 157,  
+GANDA         = 158,  
+MANX          = 159, 
+MONTENEGRIN   = 160, 
+NUM_LANGUAGES = 161,    
+
+Thanks to Mike McCandless for finding this code and writing a python version
+Thanks to the Chrome Authors.

data/Rakefile

@@ -0,0 +1,15 @@
+
+require 'rubygems'
+require 'rake'
+require 'echoe'
+
+Echoe.new('cld', '0.1.0') do |p|
+  p.description     = "Compact Language Detection from chrome"
+  p.url             = "http://github.com/jtoy/cld"
+  p.author          = "Jason Toy"
+  p.email           = "jtoy@jtoy.net"
+  p.ignore_pattern  = ["tmp/*", "script/*"]
+  p.development_dependencies = []
+end
+
+#Dir["#{File.dirname(__FILE__)}/tasks/*.rake"].sort.each { |ext| load ext }

data/base/basictypes.h

@@ -0,0 +1,348 @@
+// Copyright (c) 2010 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef BASE_BASICTYPES_H_
+#define BASE_BASICTYPES_H_
+
+#include <limits.h>         // So we can set the bounds of our types
+#include <stddef.h>         // For size_t
+#include <string.h>         // for memcpy
+
+#include "base/port.h"    // Types that only need exist on certain systems
+
+#ifndef COMPILER_MSVC
+// stdint.h is part of C99 but MSVC doesn't have it.
+#include <stdint.h>         // For intptr_t.
+#endif
+
+typedef signed char         schar;
+typedef signed char         int8;
+typedef short               int16;
+// TODO(mbelshe) Remove these type guards.  These are
+//               temporary to avoid conflicts with npapi.h.
+#ifndef _INT32
+#define _INT32
+typedef int                 int32;
+#endif
+
+// The NSPR system headers define 64-bit as |long| when possible.  In order to
+// not have typedef mismatches, we do the same on LP64.
+#if __LP64__
+typedef long                int64;
+#else
+typedef long long           int64;
+#endif
+
+// NOTE: unsigned types are DANGEROUS in loops and other arithmetical
+// places.  Use the signed types unless your variable represents a bit
+// pattern (eg a hash value) or you really need the extra bit.  Do NOT
+// use 'unsigned' to express "this value should always be positive";
+// use assertions for this.
+
+typedef unsigned char      uint8;
+typedef unsigned short     uint16;
+// TODO(mbelshe) Remove these type guards.  These are
+//               temporary to avoid conflicts with npapi.h.
+#ifndef _UINT32
+#define _UINT32
+typedef unsigned int       uint32;
+#endif
+
+// See the comment above about NSPR and 64-bit.
+#if __LP64__
+typedef unsigned long uint64;
+#else
+typedef unsigned long long uint64;
+#endif
+
+// A type to represent a Unicode code-point value. As of Unicode 4.0,
+// such values require up to 21 bits.
+// (For type-checking on pointers, make this explicitly signed,
+// and it should always be the signed version of whatever int32 is.)
+typedef signed int         char32;
+
+const uint8  kuint8max  = (( uint8) 0xFF);
+const uint16 kuint16max = ((uint16) 0xFFFF);
+const uint32 kuint32max = ((uint32) 0xFFFFFFFF);
+const uint64 kuint64max = ((uint64) GG_LONGLONG(0xFFFFFFFFFFFFFFFF));
+const  int8  kint8min   = ((  int8) 0x80);
+const  int8  kint8max   = ((  int8) 0x7F);
+const  int16 kint16min  = (( int16) 0x8000);
+const  int16 kint16max  = (( int16) 0x7FFF);
+const  int32 kint32min  = (( int32) 0x80000000);
+const  int32 kint32max  = (( int32) 0x7FFFFFFF);
+const  int64 kint64min  = (( int64) GG_LONGLONG(0x8000000000000000));
+const  int64 kint64max  = (( int64) GG_LONGLONG(0x7FFFFFFFFFFFFFFF));
+
+// A macro to disallow the copy constructor and operator= functions
+// This should be used in the private: declarations for a class
+#define DISALLOW_COPY_AND_ASSIGN(TypeName) \
+  TypeName(const TypeName&);               \
+  void operator=(const TypeName&)
+
+// An older, deprecated, politically incorrect name for the above.
+#define DISALLOW_EVIL_CONSTRUCTORS(TypeName) DISALLOW_COPY_AND_ASSIGN(TypeName)
+
+// A macro to disallow all the implicit constructors, namely the
+// default constructor, copy constructor and operator= functions.
+//
+// This should be used in the private: declarations for a class
+// that wants to prevent anyone from instantiating it. This is
+// especially useful for classes containing only static methods.
+#define DISALLOW_IMPLICIT_CONSTRUCTORS(TypeName) \
+  TypeName();                                    \
+  DISALLOW_COPY_AND_ASSIGN(TypeName)
+
+// The arraysize(arr) macro returns the # of elements in an array arr.
+// The expression is a compile-time constant, and therefore can be
+// used in defining new arrays, for example.  If you use arraysize on
+// a pointer by mistake, you will get a compile-time error.
+//
+// One caveat is that arraysize() doesn't accept any array of an
+// anonymous type or a type defined inside a function.  In these rare
+// cases, you have to use the unsafe ARRAYSIZE_UNSAFE() macro below.  This is
+// due to a limitation in C++'s template system.  The limitation might
+// eventually be removed, but it hasn't happened yet.
+
+// This template function declaration is used in defining arraysize.
+// Note that the function doesn't need an implementation, as we only
+// use its type.
+template <typename T, size_t N>
+char (&ArraySizeHelper(T (&array)[N]))[N];
+
+// That gcc wants both of these prototypes seems mysterious. VC, for
+// its part, can't decide which to use (another mystery). Matching of
+// template overloads: the final frontier.
+#ifndef _MSC_VER
+template <typename T, size_t N>
+char (&ArraySizeHelper(const T (&array)[N]))[N];
+#endif
+
+#define arraysize(array) (sizeof(ArraySizeHelper(array)))
+
+// ARRAYSIZE_UNSAFE performs essentially the same calculation as arraysize,
+// but can be used on anonymous types or types defined inside
+// functions.  It's less safe than arraysize as it accepts some
+// (although not all) pointers.  Therefore, you should use arraysize
+// whenever possible.
+//
+// The expression ARRAYSIZE_UNSAFE(a) is a compile-time constant of type
+// size_t.
+//
+// ARRAYSIZE_UNSAFE catches a few type errors.  If you see a compiler error
+//
+//   "warning: division by zero in ..."
+//
+// when using ARRAYSIZE_UNSAFE, you are (wrongfully) giving it a pointer.
+// You should only use ARRAYSIZE_UNSAFE on statically allocated arrays.
+//
+// The following comments are on the implementation details, and can
+// be ignored by the users.
+//
+// ARRAYSIZE_UNSAFE(arr) works by inspecting sizeof(arr) (the # of bytes in
+// the array) and sizeof(*(arr)) (the # of bytes in one array
+// element).  If the former is divisible by the latter, perhaps arr is
+// indeed an array, in which case the division result is the # of
+// elements in the array.  Otherwise, arr cannot possibly be an array,
+// and we generate a compiler error to prevent the code from
+// compiling.
+//
+// Since the size of bool is implementation-defined, we need to cast
+// !(sizeof(a) & sizeof(*(a))) to size_t in order to ensure the final
+// result has type size_t.
+//
+// This macro is not perfect as it wrongfully accepts certain
+// pointers, namely where the pointer size is divisible by the pointee
+// size.  Since all our code has to go through a 32-bit compiler,
+// where a pointer is 4 bytes, this means all pointers to a type whose
+// size is 3 or greater than 4 will be (righteously) rejected.
+
+#define ARRAYSIZE_UNSAFE(a) \
+  ((sizeof(a) / sizeof(*(a))) / \
+   static_cast<size_t>(!(sizeof(a) % sizeof(*(a)))))
+
+
+// Use implicit_cast as a safe version of static_cast or const_cast
+// for upcasting in the type hierarchy (i.e. casting a pointer to Foo
+// to a pointer to SuperclassOfFoo or casting a pointer to Foo to
+// a const pointer to Foo).
+// When you use implicit_cast, the compiler checks that the cast is safe.
+// Such explicit implicit_casts are necessary in surprisingly many
+// situations where C++ demands an exact type match instead of an
+// argument type convertable to a target type.
+//
+// The From type can be inferred, so the preferred syntax for using
+// implicit_cast is the same as for static_cast etc.:
+//
+//   implicit_cast<ToType>(expr)
+//
+// implicit_cast would have been part of the C++ standard library,
+// but the proposal was submitted too late.  It will probably make
+// its way into the language in the future.
+template<typename To, typename From>
+inline To implicit_cast(From const &f) {
+  return f;
+}
+
+// The COMPILE_ASSERT macro can be used to verify that a compile time
+// expression is true. For example, you could use it to verify the
+// size of a static array:
+//
+//   COMPILE_ASSERT(ARRAYSIZE_UNSAFE(content_type_names) == CONTENT_NUM_TYPES,
+//                  content_type_names_incorrect_size);
+//
+// or to make sure a struct is smaller than a certain size:
+//
+//   COMPILE_ASSERT(sizeof(foo) < 128, foo_too_large);
+//
+// The second argument to the macro is the name of the variable. If
+// the expression is false, most compilers will issue a warning/error
+// containing the name of the variable.
+
+template <bool>
+struct CompileAssert {
+};
+
+#undef COMPILE_ASSERT
+#define COMPILE_ASSERT(expr, msg) \
+  typedef CompileAssert<(bool(expr))> msg[bool(expr) ? 1 : -1]
+
+// Implementation details of COMPILE_ASSERT:
+//
+// - COMPILE_ASSERT works by defining an array type that has -1
+//   elements (and thus is invalid) when the expression is false.
+//
+// - The simpler definition
+//
+//     #define COMPILE_ASSERT(expr, msg) typedef char msg[(expr) ? 1 : -1]
+//
+//   does not work, as gcc supports variable-length arrays whose sizes
+//   are determined at run-time (this is gcc's extension and not part
+//   of the C++ standard).  As a result, gcc fails to reject the
+//   following code with the simple definition:
+//
+//     int foo;
+//     COMPILE_ASSERT(foo, msg); // not supposed to compile as foo is
+//                               // not a compile-time constant.
+//
+// - By using the type CompileAssert<(bool(expr))>, we ensures that
+//   expr is a compile-time constant.  (Template arguments must be
+//   determined at compile-time.)
+//
+// - The outter parentheses in CompileAssert<(bool(expr))> are necessary
+//   to work around a bug in gcc 3.4.4 and 4.0.1.  If we had written
+//
+//     CompileAssert<bool(expr)>
+//
+//   instead, these compilers will refuse to compile
+//
+//     COMPILE_ASSERT(5 > 0, some_message);
+//
+//   (They seem to think the ">" in "5 > 0" marks the end of the
+//   template argument list.)
+//
+// - The array size is (bool(expr) ? 1 : -1), instead of simply
+//
+//     ((expr) ? 1 : -1).
+//
+//   This is to avoid running into a bug in MS VC 7.1, which
+//   causes ((0.0) ? 1 : -1) to incorrectly evaluate to 1.
+
+
+// MetatagId refers to metatag-id that we assign to
+// each metatag <name, value> pair..
+typedef uint32 MetatagId;
+
+// Argument type used in interfaces that can optionally take ownership
+// of a passed in argument.  If TAKE_OWNERSHIP is passed, the called
+// object takes ownership of the argument.  Otherwise it does not.
+enum Ownership {
+  DO_NOT_TAKE_OWNERSHIP,
+  TAKE_OWNERSHIP
+};
+
+// bit_cast<Dest,Source> is a template function that implements the
+// equivalent of "*reinterpret_cast<Dest*>(&source)".  We need this in
+// very low-level functions like the protobuf library and fast math
+// support.
+//
+//   float f = 3.14159265358979;
+//   int i = bit_cast<int32>(f);
+//   // i = 0x40490fdb
+//
+// The classical address-casting method is:
+//
+//   // WRONG
+//   float f = 3.14159265358979;            // WRONG
+//   int i = * reinterpret_cast<int*>(&f);  // WRONG
+//
+// The address-casting method actually produces undefined behavior
+// according to ISO C++ specification section 3.10 -15 -.  Roughly, this
+// section says: if an object in memory has one type, and a program
+// accesses it with a different type, then the result is undefined
+// behavior for most values of "different type".
+//
+// This is true for any cast syntax, either *(int*)&f or
+// *reinterpret_cast<int*>(&f).  And it is particularly true for
+// conversions betweeen integral lvalues and floating-point lvalues.
+//
+// The purpose of 3.10 -15- is to allow optimizing compilers to assume
+// that expressions with different types refer to different memory.  gcc
+// 4.0.1 has an optimizer that takes advantage of this.  So a
+// non-conforming program quietly produces wildly incorrect output.
+//
+// The problem is not the use of reinterpret_cast.  The problem is type
+// punning: holding an object in memory of one type and reading its bits
+// back using a different type.
+//
+// The C++ standard is more subtle and complex than this, but that
+// is the basic idea.
+//
+// Anyways ...
+//
+// bit_cast<> calls memcpy() which is blessed by the standard,
+// especially by the example in section 3.9 .  Also, of course,
+// bit_cast<> wraps up the nasty logic in one place.
+//
+// Fortunately memcpy() is very fast.  In optimized mode, with a
+// constant size, gcc 2.95.3, gcc 4.0.1, and msvc 7.1 produce inline
+// code with the minimal amount of data movement.  On a 32-bit system,
+// memcpy(d,s,4) compiles to one load and one store, and memcpy(d,s,8)
+// compiles to two loads and two stores.
+//
+// I tested this code with gcc 2.95.3, gcc 4.0.1, icc 8.1, and msvc 7.1.
+//
+// WARNING: if Dest or Source is a non-POD type, the result of the memcpy
+// is likely to surprise you.
+
+template <class Dest, class Source>
+inline Dest bit_cast(const Source& source) {
+  // Compile time assertion: sizeof(Dest) == sizeof(Source)
+  // A compile error here means your Dest and Source have different sizes.
+  typedef char VerifySizesAreEqual [sizeof(Dest) == sizeof(Source) ? 1 : -1];
+
+  Dest dest;
+  memcpy(&dest, &source, sizeof(dest));
+  return dest;
+}
+
+// The following enum should be used only as a constructor argument to indicate
+// that the variable has static storage class, and that the constructor should
+// do nothing to its state.  It indicates to the reader that it is legal to
+// declare a static instance of the class, provided the constructor is given
+// the base::LINKER_INITIALIZED argument.  Normally, it is unsafe to declare a
+// static variable that has a constructor or a destructor because invocation
+// order is undefined.  However, IF the type can be initialized by filling with
+// zeroes (which the loader does for static variables), AND the destructor also
+// does nothing to the storage, AND there are no virtual methods, then a
+// constructor declared as
+//       explicit MyClass(base::LinkerInitialized x) {}
+// and invoked as
+//       static MyClass my_variable_name(base::LINKER_INITIALIZED);
+namespace base {
+enum LinkerInitialized { LINKER_INITIALIZED };
+}  // base
+
+
+#endif  // BASE_BASICTYPES_H_