cld3 3.1.0

data/ext/cld3/offsetmap.h

@@ -0,0 +1,168 @@
+// Copyright 2013 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//
+// Author: dsites@google.com (Dick Sites)
+//
+
+#ifndef SCRIPT_SPAN_OFFSETMAP_H_
+#define SCRIPT_SPAN_OFFSETMAP_H_
+
+#include <string>                       // for string
+
+#include "integral_types.h"             // for uint32
+
+// ***************************** OffsetMap **************************
+//
+// An OffsetMap object is a container for a mapping from offsets in one text
+// buffer A' to offsets in another text buffer A. It is most useful when A' is
+// built from A via substitutions that occasionally do not preserve byte length.
+//
+// A series of operators are used to build the correspondence map, then
+// calls can be made to map an offset in A' to an offset in A, or vice versa.
+// The map starts with offset 0 in A corresponding to offset 0 in A'.
+// The mapping is then built sequentially, adding on byte ranges that are
+// identical in A and A', byte ranges that are inserted in A', and byte ranges
+// that are deleted from A. All bytes beyond those specified when building the
+// map are assumed to correspond, i.e. a Copy(infinity) is assumed at the
+// end of the map.
+//
+// The internal data structure records positions at which bytes are added or
+// deleted. Using the map is O(1) when increasing the A' or A offset
+// monotonically, and O(n) when accessing random offsets, where n is the
+// number of differences.
+//
+
+namespace chrome_lang_id {
+namespace CLD2 {
+
+class OffsetMap {
+ public:
+  // Constructor, destructor
+  OffsetMap();
+  ~OffsetMap();
+
+  // Clear the map
+  void Clear();
+
+  // Add to  mapping from A to A', specifying how many next bytes correspond
+  // in A and A'
+  void Copy(int bytes);
+
+  // Add to mapping from A to A', specifying how many next bytes are
+  // inserted in A' while not advancing in A at all
+  void Insert(int bytes);
+
+  // Add to mapping from A to A', specifying how many next bytes are
+  // deleted from A while not advancing in A' at all
+  void Delete(int bytes);
+
+  // [Finish building map,] Re-position to offset 0
+  // This call is optional; MapForward and MapBack finish building the map
+  // if necessary
+  void Reset();
+
+  // Map an offset in A' to the corresponding offset in A
+  int MapBack(int aprimeoffset);
+
+  // Map an offset in A to the corresponding offset in A'
+  int MapForward(int aoffset);
+
+  // h = ComposeOffsetMap(g, f), where f is a map from A to A', g is
+  // from A' to A'' and h is from A to A''.
+  //
+  // Note that g->MoveForward(f->MoveForward(aoffset)) always equals
+  // to h->MoveForward(aoffset), while
+  // f->MoveBack(g->MoveBack(aprimeprimeoffset)) doesn't always equals
+  // to h->MoveBack(aprimeprimeoffset). This happens when deletion in
+  // f and insertion in g are at the same place.  For example,
+  //
+  // A    1   2   3   4
+  //      ^   |  ^    ^
+  //      |   | /     |  f
+  //      v   vv      v
+  // A'   1'  2'      3'
+  //      ^   ^^      ^
+  //      |   | \     |  g
+  //      v   |  v    v
+  // A''  1'' 2'' 3'' 4''
+  //
+  // results in:
+  //
+  // A    1   2   3   4
+  //      ^   ^\  ^   ^
+  //      |   | \ |   |  h
+  //      v   |  vv   v
+  // A''  1'' 2'' 3'' 4''
+  //
+  // 2'' is mapped 3 in the former figure, while 2'' is mapped to 2 in
+  // the latter figure.
+  static void ComposeOffsetMap(OffsetMap* g, OffsetMap* f, OffsetMap* h);
+
+  // For testing only -- force a mapping
+  void StuffIt(const std::string& diffs, int max_aoffset, int max_aprimeoffset);
+
+ private:
+  enum MapOp {PREFIX_OP, COPY_OP, INSERT_OP, DELETE_OP};
+
+  void Flush();
+  void FlushAll();
+  void MaybeFlushAll();
+  void Emit(MapOp op, int len);
+
+  void SetLeft();
+  void SetRight();
+
+  // Back up over previous range, 1..5 bytes
+  // Return subscript at the beginning of that. Pins at 0
+  int Backup(int sub);
+
+  // Parse next range, 1..5 bytes
+  // Return subscript just off the end of that
+  int ParseNext(int sub, MapOp* op, int* length);
+
+  // Parse previous range, 1..5 bytes
+  // Return current subscript
+  int ParsePrevious(int sub, MapOp* op, int* length);
+
+  bool MoveRight();     // Returns true if OK
+  bool MoveLeft();      // Returns true if OK
+
+  // Copies insert operations from source to dest. Returns true if no
+  // other operations are found.
+  static bool CopyInserts(OffsetMap* source, OffsetMap* dest);
+
+  // Copies delete operations from source to dest. Returns true if no other
+  // operations are found.
+  static bool CopyDeletes(OffsetMap* source, OffsetMap* dest);
+
+  std::string diffs_;
+  MapOp pending_op_;
+  uint32 pending_length_;
+
+  // Offsets in the ranges below correspond to each other, with A' = A + diff
+  int next_diff_sub_;
+  int current_lo_aoffset_;
+  int current_hi_aoffset_;
+  int current_lo_aprimeoffset_;
+  int current_hi_aprimeoffset_;
+  int current_diff_;
+  int max_aoffset_;
+  int max_aprimeoffset_;
+};
+
+}  // namespace CLD2
+}  // namespace chrome_lang_id
+
+#endif  // SCRIPT_SPAN_OFFSETMAP_H_

data/ext/cld3/port.h

@@ -0,0 +1,143 @@
+// Copyright 2013 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//
+// These are weird things we need to do to get this compiling on
+// random systems [subset].
+
+#ifndef SCRIPT_SPAN_PORT_H_
+#define SCRIPT_SPAN_PORT_H_
+
+#include <string.h>         // for memcpy()
+
+#include "integral_types.h"
+
+namespace chrome_lang_id {
+namespace CLD2 {
+
+// Portable handling of unaligned loads, stores, and copies.
+// On some platforms, like ARM, the copy functions can be more efficient
+// then a load and a store.
+
+#if defined(ARCH_PIII) || defined(ARCH_ATHLON) || defined(ARCH_K8) || defined(_ARCH_PPC)
+
+// x86 and x86-64 can perform unaligned loads/stores directly;
+// modern PowerPC hardware can also do unaligned integer loads and stores;
+// but note: the FPU still sends unaligned loads and stores to a trap handler!
+
+#define UNALIGNED_LOAD16(_p) (*reinterpret_cast<const uint16 *>(_p))
+#define UNALIGNED_LOAD32(_p) (*reinterpret_cast<const uint32 *>(_p))
+#define UNALIGNED_LOAD64(_p) (*reinterpret_cast<const uint64 *>(_p))
+
+#define UNALIGNED_STORE16(_p, _val) (*reinterpret_cast<uint16 *>(_p) = (_val))
+#define UNALIGNED_STORE32(_p, _val) (*reinterpret_cast<uint32 *>(_p) = (_val))
+#define UNALIGNED_STORE64(_p, _val) (*reinterpret_cast<uint64 *>(_p) = (_val))
+
+#elif defined(__arm__) && \
+      !defined(__ARM_ARCH_5__) && \
+      !defined(__ARM_ARCH_5T__) && \
+      !defined(__ARM_ARCH_5TE__) && \
+      !defined(__ARM_ARCH_5TEJ__) && \
+      !defined(__ARM_ARCH_6__) && \
+      !defined(__ARM_ARCH_6J__) && \
+      !defined(__ARM_ARCH_6K__) && \
+      !defined(__ARM_ARCH_6Z__) && \
+      !defined(__ARM_ARCH_6ZK__) && \
+      !defined(__ARM_ARCH_6T2__) && \
+      !defined(__ARM_ARCH_7__) && \
+      !defined(__ARM_ARCH_7A__) && \
+      !defined(__ARM_ARCH_7M__) && \
+      !defined(__ARM_ARCH_7R__) && \
+      !defined(__ARM_ARCH_8__) && \
+      !defined(__ARM_ARCH_8A__)
+
+// ARMv7 and newer support native unaligned accesses, but only of 16-bit
+// and 32-bit values (not 64-bit); older versions either raise a fatal signal,
+// do an unaligned read and rotate the words around a bit, or do the reads very
+// slowly (trip through kernel mode). There's no simple #define that says just
+// “ARMv7 or higher”, so we have to filter away all ARMv5 and ARMv6
+// sub-architectures. Newer gcc (>= 4.6) set an __ARM_FEATURE_ALIGNED #define,
+// so in time, maybe we can move on to that.
+//
+// Note that even if a chipset supports unaligned access, it might not be
+// enabled in any given system, e.g.:
+// http://infocenter.arm.com/help/index.jsp?topic=/com.arm.doc.dui0491c/CIHCGCFD.html
+// Therefore, it's generally just not safe to allow unaligned access on any ARM
+// variant.
+//
+// This is a mess, but there's not much we can do about it.
+
+#define UNALIGNED_LOAD16(_p) (*reinterpret_cast<const uint16 *>(_p))
+#define UNALIGNED_LOAD32(_p) (*reinterpret_cast<const uint32 *>(_p))
+
+#define UNALIGNED_STORE16(_p, _val) (*reinterpret_cast<uint16 *>(_p) = (_val))
+#define UNALIGNED_STORE32(_p, _val) (*reinterpret_cast<uint32 *>(_p) = (_val))
+
+// TODO(sesse): NEON supports unaligned 64-bit loads and stores.
+// See if that would be more efficient on platforms supporting it,
+// at least for copies.
+
+inline uint64 UNALIGNED_LOAD64(const void *p) {
+  uint64 t;
+  memcpy(&t, p, sizeof t);
+  return t;
+}
+
+inline void UNALIGNED_STORE64(void *p, uint64 v) {
+  memcpy(p, &v, sizeof v);
+}
+
+#else
+
+#define NEED_ALIGNED_LOADS
+
+// These functions are provided for architectures that don't support
+// unaligned loads and stores.
+
+inline uint16 UNALIGNED_LOAD16(const void *p) {
+  uint16 t;
+  memcpy(&t, p, sizeof t);
+  return t;
+}
+
+inline uint32 UNALIGNED_LOAD32(const void *p) {
+  uint32 t;
+  memcpy(&t, p, sizeof t);
+  return t;
+}
+
+inline uint64 UNALIGNED_LOAD64(const void *p) {
+  uint64 t;
+  memcpy(&t, p, sizeof t);
+  return t;
+}
+
+inline void UNALIGNED_STORE16(void *p, uint16 v) {
+  memcpy(p, &v, sizeof v);
+}
+
+inline void UNALIGNED_STORE32(void *p, uint32 v) {
+  memcpy(p, &v, sizeof v);
+}
+
+inline void UNALIGNED_STORE64(void *p, uint64 v) {
+  memcpy(p, &v, sizeof v);
+}
+
+#endif
+
+}       // End namespace CLD2
+}       // End namespace chrome_lang_id
+
+#endif  // SCRIPT_SPAN_PORT_H_

data/ext/cld3/registry.cc

@@ -0,0 +1,28 @@
+/* Copyright 2016 Google Inc. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "registry.h"
+
+namespace chrome_lang_id {
+
+// Global list of all component registries.
+RegistryMetadata *global_registry_list = NULL;
+
+void RegistryMetadata::Register(RegistryMetadata *registry) {
+  registry->set_link(global_registry_list);
+  global_registry_list = registry;
+}
+
+}  // namespace chrome_lang_id

data/ext/cld3/registry.h

@@ -0,0 +1,242 @@
+/* Copyright 2016 Google Inc. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+// Registry for component registration. These classes can be used for creating
+// registries of components conforming to the same interface. This is useful for
+// making a component-based architecture where the specific implementation
+// classes can be selected at runtime. There is support for both class-based and
+// instance based registries.
+//
+// Example:
+//  function.h:
+//
+//   class Function : public RegisterableInstance<Function> {
+//    public:
+//     virtual double Evaluate(double x) = 0;
+//   };
+//
+//   #define REGISTER_FUNCTION(type, component)
+//     REGISTER_INSTANCE_COMPONENT(Function, type, component);
+//
+//  function.cc:
+//
+//   REGISTER_INSTANCE_REGISTRY("function", Function);
+//
+//   class Cos : public Function {
+//    public:
+//     double Evaluate(double x) { return cos(x); }
+//   };
+//
+//   class Exp : public Function {
+//    public:
+//     double Evaluate(double x) { return exp(x); }
+//   };
+//
+//   REGISTER_FUNCTION("cos", Cos);
+//   REGISTER_FUNCTION("exp", Exp);
+//
+//   Function *f = Function::Lookup("cos");
+//   double result = f->Evaluate(arg);
+
+#ifndef REGISTRY_H_
+#define REGISTRY_H_
+
+#include <string.h>
+
+#include <string>
+
+#include "base.h"
+
+namespace chrome_lang_id {
+
+// Component metadata with information about name, class, and code location.
+class ComponentMetadata {
+ public:
+  ComponentMetadata(const char *name, const char *class_name, const char *file,
+                    int line)
+      : name_(name),
+        class_name_(class_name),
+        file_(file),
+        line_(line),
+        link_(NULL) {}
+
+  // Getters.
+  const char *name() const { return name_; }
+  const char *class_name() const { return class_name_; }
+  const char *file() const { return file_; }
+  int line() const { return line_; }
+
+  // Metadata objects can be linked in a list.
+  ComponentMetadata *link() const { return link_; }
+  void set_link(ComponentMetadata *link) { link_ = link; }
+
+ private:
+  // Component name.
+  const char *name_;
+
+  // Name of class for component.
+  const char *class_name_;
+
+  // Code file and location where the component was registered.
+  const char *file_;
+  int line_;
+
+  // Link to next metadata object in list.
+  ComponentMetadata *link_;
+};
+
+// The master registry contains all registered component registries. A registry
+// is not registered in the master registry until the first component of that
+// type is registered.
+class RegistryMetadata : public ComponentMetadata {
+ public:
+  RegistryMetadata(const char *name, const char *class_name, const char *file,
+                   int line)
+      : ComponentMetadata(name, class_name, file, line) {}
+
+  // Registers a component registry in the master registry.
+  static void Register(RegistryMetadata *registry);
+};
+
+// Registry for components. An object can be registered with a type name in the
+// registry. The named instances in the registry can be returned using the
+// Lookup() method. The components in the registry are put into a linked list
+// of components. It is important that the component registry can be statically
+// initialized in order not to depend on initialization order.
+template <class T>
+struct ComponentRegistry {
+  typedef ComponentRegistry<T> Self;
+
+  // Component registration class.
+  class Registrar : public ComponentMetadata {
+   public:
+    // Registers new component by linking itself into the component list of
+    // the registry.
+    Registrar(Self *registry, const char *type, const char *class_name,
+              const char *file, int line, T *object)
+        : ComponentMetadata(type, class_name, file, line), object_(object) {
+      // Register registry in master registry if this is the first registered
+      // component of this type.
+      if (registry->components == NULL) {
+        RegistryMetadata::Register(
+            new RegistryMetadata(registry->name, registry->class_name,
+                                 registry->file, registry->line));
+      }
+
+      // Register component in registry.
+      set_link(registry->components);
+      registry->components = this;
+    }
+
+    // Returns component type.
+    const char *type() const { return name(); }
+
+    // Returns component object.
+    T *object() const { return object_; }
+
+    // Returns the next component in the component list.
+    Registrar *next() const { return static_cast<Registrar *>(link()); }
+
+   private:
+    // Component object.
+    T *object_;
+  };
+
+  // Finds registrar for named component in registry.
+  const Registrar *GetComponent(const char *type) const {
+    Registrar *r = components;
+    while (r != NULL && strcmp(type, r->type()) != 0) r = r->next();
+    CLD3_DCHECK(r != nullptr);
+
+    return r;
+  }
+
+  // Finds a named component in the registry.
+  T *Lookup(const char *type) const { return GetComponent(type)->object(); }
+  T *Lookup(const string &type) const { return Lookup(type.c_str()); }
+
+  // Textual description of the kind of components in the registry.
+  const char *name;
+
+  // Base class name of component type.
+  const char *class_name;
+
+  // File and line where the registry is defined.
+  const char *file;
+  int line;
+
+  // Linked list of registered components.
+  Registrar *components;
+};
+
+// Base class for registerable class-based components.
+template <class T>
+class RegisterableClass {
+ public:
+  // Factory function type.
+  typedef T *(Factory)();
+
+  // Registry type.
+  typedef ComponentRegistry<Factory> Registry;
+
+  // Should be called before any call to Create() or registry(), i.e., before
+  // using the registration mechanism to register and or instantiate subclasses
+  // of T.
+  static void CreateRegistry(
+      const char *name,
+      const char *class_name,
+      const char *file,
+      int line) {
+    registry_ = new Registry();
+    registry_->name = name;
+    registry_->class_name = class_name;
+    registry_->file = file;
+    registry_->line = line;
+    registry_->components = nullptr;
+  }
+
+  // Should be called when one is done using the registration mechanism for
+  // class T.
+  static void DeleteRegistry() {
+    delete registry_;
+    registry_ = nullptr;
+  }
+
+  // Creates a new component instance.
+  static T *Create(const string &type) { return registry()->Lookup(type)(); }
+
+  // Returns registry for class.
+  static Registry *registry() { return registry_; }
+
+ private:
+  // Registry for class.
+  static Registry *registry_;
+};
+
+// Base class for registerable instance-based components.
+template <class T>
+class RegisterableInstance {
+ public:
+  // Registry type.
+  typedef ComponentRegistry<T> Registry;
+
+ private:
+  // Registry for class.
+  static Registry registry_;
+};
+
+}  // namespace chrome_lang_id
+
+#endif  // REGISTRY_H_