cld3 3.1.0

data/ext/cld3/fml_parser.cc

@@ -0,0 +1,308 @@
+/* Copyright 2016 Google Inc. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "fml_parser.h"
+
+#include <ctype.h>
+#include <string>
+
+#include "base.h"
+#include "utils.h"
+
+namespace chrome_lang_id {
+
+namespace {
+
+inline bool IsValidCharAtStartOfIdentifier(char c) {
+  return isalpha(c) || (c == '_') || (c == '/');
+}
+
+// Returns true iff character c can appear inside an identifier.
+inline bool IsValidCharInsideIdentifier(char c) {
+  return isalnum(c) || (c == '_') || (c == '-') || (c == '/');
+}
+
+// Returns true iff character c can appear at the beginning of a number.
+inline bool IsValidCharAtStartOfNumber(char c) {
+  return isdigit(c) || (c == '+') || (c == '-');
+}
+
+// Returns true iff character c can appear inside a number.
+inline bool IsValidCharInsideNumber(char c) { return isdigit(c) || (c == '.'); }
+
+}  // namespace
+
+FMLParser::FMLParser() {}
+FMLParser::~FMLParser() {}
+
+void FMLParser::Initialize(const string &source) {
+  // Initialize parser state.
+  source_ = source;
+  current_ = source_.begin();
+  item_start_ = line_start_ = current_;
+  line_number_ = item_line_number_ = 1;
+
+  // Read first input item.
+  NextItem();
+}
+
+void FMLParser::Next() {
+  // Move to the next input character. If we are at a line break update line
+  // number and line start position.
+  if (CurrentChar() == '\n') {
+    ++line_number_;
+    ++current_;
+    line_start_ = current_;
+  } else {
+    ++current_;
+  }
+}
+
+void FMLParser::NextItem() {
+  // Skip white space and comments.
+  while (!eos()) {
+    if (CurrentChar() == '#') {
+      // Skip comment.
+      while (!eos() && CurrentChar() != '\n') Next();
+    } else if (isspace(CurrentChar())) {
+      // Skip whitespace.
+      while (!eos() && isspace(CurrentChar())) Next();
+    } else {
+      break;
+    }
+  }
+
+  // Record start position for next item.
+  item_start_ = current_;
+  item_line_number_ = line_number_;
+
+  // Check for end of input.
+  if (eos()) {
+    item_type_ = END;
+    return;
+  }
+
+  // Parse number.
+  if (IsValidCharAtStartOfNumber(CurrentChar())) {
+    string::iterator start = current_;
+    Next();
+    while (!eos() && IsValidCharInsideNumber(CurrentChar())) Next();
+    item_text_.assign(start, current_);
+    item_type_ = NUMBER;
+    return;
+  }
+
+  // Parse string.
+  if (CurrentChar() == '"') {
+    Next();
+    string::iterator start = current_;
+    while (CurrentChar() != '"') {
+      CLD3_DCHECK(!eos());
+      Next();
+    }
+    item_text_.assign(start, current_);
+    item_type_ = STRING;
+    Next();
+    return;
+  }
+
+  // Parse identifier name.
+  if (IsValidCharAtStartOfIdentifier(CurrentChar())) {
+    string::iterator start = current_;
+    while (!eos() && IsValidCharInsideIdentifier(CurrentChar())) {
+      Next();
+    }
+    item_text_.assign(start, current_);
+    item_type_ = NAME;
+    return;
+  }
+
+  // Single character item.
+  item_type_ = CurrentChar();
+  Next();
+}
+
+void FMLParser::Parse(const string &source,
+                      FeatureExtractorDescriptor *result) {
+  // Initialize parser.
+  Initialize(source);
+
+  while (item_type_ != END) {
+    // Parse either a parameter name or a feature.
+    CLD3_DCHECK(item_type_ == NAME);
+    string name = item_text_;
+    NextItem();
+
+    // Feature expected.
+    CLD3_DCHECK(static_cast<char>(item_type_) != '=');
+
+    // Parse feature.
+    FeatureFunctionDescriptor *descriptor = result->add_feature();
+    descriptor->set_type(name);
+    ParseFeature(descriptor);
+  }
+}
+
+void FMLParser::ParseFeature(FeatureFunctionDescriptor *result) {
+  // Parse argument and parameters.
+  if (item_type_ == '(') {
+    NextItem();
+    ParseParameter(result);
+    while (item_type_ == ',') {
+      NextItem();
+      ParseParameter(result);
+    }
+
+    CLD3_DCHECK(item_type_ == ')');
+    NextItem();
+  }
+
+  // Parse feature name.
+  if (item_type_ == ':') {
+    NextItem();
+
+    // Feature name expected.
+    CLD3_DCHECK((item_type_ == NAME) || (item_type_ == STRING));
+    string name = item_text_;
+    NextItem();
+
+    // Set feature name.
+    result->set_name(name);
+  }
+
+  // Parse sub-features.
+  if (item_type_ == '.') {
+    // Parse dotted sub-feature.
+    NextItem();
+    CLD3_DCHECK(item_type_ == NAME);
+    string type = item_text_;
+    NextItem();
+
+    // Parse sub-feature.
+    FeatureFunctionDescriptor *subfeature = result->add_feature();
+    subfeature->set_type(type);
+    ParseFeature(subfeature);
+  } else if (item_type_ == '{') {
+    // Parse sub-feature block.
+    NextItem();
+    while (item_type_ != '}') {
+      CLD3_DCHECK(item_type_ == NAME);
+      string type = item_text_;
+      NextItem();
+
+      // Parse sub-feature.
+      FeatureFunctionDescriptor *subfeature = result->add_feature();
+      subfeature->set_type(type);
+      ParseFeature(subfeature);
+    }
+    NextItem();
+  }
+}
+
+void FMLParser::ParseParameter(FeatureFunctionDescriptor *result) {
+  CLD3_DCHECK((item_type_ == NUMBER) || (item_type_ == NAME));
+  if (item_type_ == NUMBER) {
+    int argument = utils::ParseUsing<int>(item_text_, utils::ParseInt32);
+    NextItem();
+
+    // Set default argument for feature.
+    result->set_argument(argument);
+  } else {  // item_type_ == NAME
+    string name = item_text_;
+    NextItem();
+    CLD3_DCHECK(item_type_ == '=');
+    NextItem();
+
+    // Parameter value expected.
+    CLD3_DCHECK(item_type_ < END);
+    string value = item_text_;
+    NextItem();
+
+    // Add parameter to feature.
+    Parameter *parameter;
+    parameter = result->add_parameter();
+    parameter->set_name(name);
+    parameter->set_value(value);
+  }
+}
+
+void ToFMLFunction(const FeatureFunctionDescriptor &function, string *output) {
+  output->append(function.type());
+  if (function.argument() != 0 || function.parameter_size() > 0) {
+    output->append("(");
+    bool first = true;
+    if (function.argument() != 0) {
+      output->append(Int64ToString(function.argument()));
+      first = false;
+    }
+    for (int i = 0; i < function.parameter_size(); ++i) {
+      if (!first) output->append(",");
+      output->append(function.parameter(i).name());
+      output->append("=");
+      output->append("\"");
+      output->append(function.parameter(i).value());
+      output->append("\"");
+      first = false;
+    }
+    output->append(")");
+  }
+}
+
+void ToFML(const FeatureFunctionDescriptor &function, string *output) {
+  ToFMLFunction(function, output);
+  if (function.feature_size() == 1) {
+    output->append(".");
+    ToFML(function.feature(0), output);
+  } else if (function.feature_size() > 1) {
+    output->append(" { ");
+    for (int i = 0; i < function.feature_size(); ++i) {
+      if (i > 0) output->append(" ");
+      ToFML(function.feature(i), output);
+    }
+    output->append(" } ");
+  }
+}
+
+void ToFML(const FeatureExtractorDescriptor &extractor, string *output) {
+  for (int i = 0; i < extractor.feature_size(); ++i) {
+    ToFML(extractor.feature(i), output);
+    output->append("\n");
+  }
+}
+
+string AsFML(const FeatureFunctionDescriptor &function) {
+  string str;
+  ToFML(function, &str);
+  return str;
+}
+
+string AsFML(const FeatureExtractorDescriptor &extractor) {
+  string str;
+  ToFML(extractor, &str);
+  return str;
+}
+
+void StripFML(string *fml_string) {
+  auto it = fml_string->begin();
+  while (it != fml_string->end()) {
+    if (*it == '"') {
+      it = fml_string->erase(it);
+    } else {
+      ++it;
+    }
+  }
+}
+
+}  // namespace chrome_lang_id

data/ext/cld3/fml_parser.h

@@ -0,0 +1,123 @@
+/* Copyright 2016 Google Inc. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+// Feature modeling language (fml) parser.
+//
+// BNF grammar for fml:
+//
+// <feature model> ::= { <feature extractor> }
+//
+// <feature extractor> ::= <extractor spec> |
+//                         <extractor spec> '.' <feature extractor> |
+//                         <extractor spec> '{' { <feature extractor> } '}'
+//
+// <extractor spec> ::= <extractor type>
+//                      [ '(' <parameter list> ')' ]
+//                      [ ':' <extractor name> ]
+//
+// <parameter list> = ( <parameter> | <argument> ) { ',' <parameter> }
+//
+// <parameter> ::= <parameter name> '=' <parameter value>
+//
+// <extractor type> ::= NAME
+// <extractor name> ::= NAME | STRING
+// <argument> ::= NUMBER
+// <parameter name> ::= NAME
+// <parameter value> ::= NUMBER | STRING | NAME
+
+#ifndef FML_PARSER_H_
+#define FML_PARSER_H_
+
+#include <string>
+
+#include "base.h"
+#include "cld_3/protos/feature_extractor.pb.h"
+
+namespace chrome_lang_id {
+
+class FMLParser {
+ public:
+  // Parses fml specification into feature extractor descriptor.
+  void Parse(const string &source, FeatureExtractorDescriptor *result);
+
+  FMLParser();
+  ~FMLParser();
+
+ private:
+  // Initializes the parser with the source text.
+  void Initialize(const string &source);
+
+  // Moves to the next input character.
+  void Next();
+
+  // Moves to the next input item.
+  void NextItem();
+
+  // Parses a feature descriptor.
+  void ParseFeature(FeatureFunctionDescriptor *result);
+
+  // Parses a parameter specification.
+  void ParseParameter(FeatureFunctionDescriptor *result);
+
+  // Returns true if end of source input has been reached.
+  bool eos() const { return current_ == source_.end(); }
+
+  // Returns current character.  Other methods should access the current
+  // character through this method (instead of using *current_ directly): this
+  // method performs extra safety checks.
+  char CurrentChar() const {
+    // CLD3_DCHECK that we are reading from inside the string.
+    CLD3_DCHECK(current_ >= source_.begin());
+    CLD3_DCHECK(current_ < source_.end());
+    return *current_;
+  }
+
+  // Item types.
+  enum ItemTypes {
+    END = 0,
+    NAME = -1,
+    NUMBER = -2,
+    STRING = -3,
+  };
+
+  // Source text.
+  string source_;
+
+  // Current input position.
+  string::iterator current_;
+
+  // Line number for current input position.
+  int line_number_;
+
+  // Start position for current item.
+  string::iterator item_start_;
+
+  // Start position for current line.
+  string::iterator line_start_;
+
+  // Line number for current item.
+  int item_line_number_;
+
+  // Item type for current item. If this is positive it is interpreted as a
+  // character. If it is negative it is interpreted as an item type.
+  int item_type_;
+
+  // Text for current item.
+  string item_text_;
+};
+
+}  // namespace chrome_lang_id
+
+#endif  // FML_PARSER_H_

data/ext/cld3/generated_entities.cc

@@ -0,0 +1,296 @@
+// Copyright 2013 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// generated_entities.cc
+// Machine generated. Do Not Edit.
+//
+// Declarations for HTML entities recognized by CLD2
+//
+#include "generated_ulscript.h"  // for CharIntPair
+
+namespace chrome_lang_id {
+namespace CLD2 {
+
+// Alphabetical order for binary search
+extern const int kNameToEntitySize = 265;
+extern const CharIntPair kNameToEntity[kNameToEntitySize] = {
+  {"AElig",   198},
+  {"AMP",      38},
+  {"Aacute",  193},
+  {"Acirc",   194},
+  {"Agrave",  192},
+  {"Alpha",   913},
+  {"Aring",   197},
+  {"Atilde",  195},
+  {"Auml",    196},
+  {"Beta",    914},
+  {"Ccaron",  268},
+  {"Ccedil",  199},
+  {"Chi",     935},
+  {"Dagger", 8225},
+  {"Delta",   916},
+  {"ETH",     208},
+  {"Eacute",  201},
+  {"Ecaron",  282},
+  {"Ecirc",   202},
+  {"Egrave",  200},
+  {"Epsilon",  917},
+  {"Eta",     919},
+  {"Euml",    203},
+  {"GT",       62},
+  {"Gamma",   915},
+  {"Iacute",  205},
+  {"Icirc",   206},
+  {"Igrave",  204},
+  {"Iota",    921},
+  {"Iuml",    207},
+  {"Kappa",   922},
+  {"LT",       60},
+  {"Lambda",  923},
+  {"Mu",      924},
+  {"Ntilde",  209},
+  {"Nu",      925},
+  {"OElig",   338},
+  {"Oacute",  211},
+  {"Ocirc",   212},
+  {"Ograve",  210},
+  {"Omega",   937},
+  {"Omicron",  927},
+  {"Oslash",  216},
+  {"Otilde",  213},
+  {"Ouml",    214},
+  {"Phi",     934},
+  {"Pi",      928},
+  {"Prime",  8243},
+  {"Psi",     936},
+  {"QUOT",     34},
+  {"Rcaron",  344},
+  {"Rho",     929},
+  {"Scaron",  352},
+  {"Sigma",   931},
+  {"THORN",   222},
+  {"Tau",     932},
+  {"Theta",   920},
+  {"Uacute",  218},
+  {"Ucirc",   219},
+  {"Ugrave",  217},
+  {"Upsilon",  933},
+  {"Uuml",    220},
+  {"Xi",      926},
+  {"Yacute",  221},
+  {"Yuml",    376},
+  {"Zeta",    918},
+  {"aacute",  225},
+  {"acirc",   226},
+  {"acute",   180},
+  {"aelig",   230},
+  {"agrave",  224},
+  {"alefsym", 8501},
+  {"alpha",   945},
+  {"amp",      38},
+  {"and",    8743},
+  {"ang",    8736},
+  {"apos",     39},
+  {"aring",   229},
+  {"asymp",  8776},
+  {"atilde",  227},
+  {"auml",    228},
+  {"bdquo",  8222},
+  {"beta",    946},
+  {"brvbar",  166},
+  {"bull",   8226},
+  {"cap",    8745},
+  {"ccaron",  269},
+  {"ccedil",  231},
+  {"cedil",   184},
+  {"cent",    162},
+  {"chi",     967},
+  {"circ",    710},
+  {"clubs",  9827},
+  {"cong",   8773},
+  {"copy",    169},
+  {"crarr",  8629},
+  {"cup",    8746},
+  {"curren",  164},
+  {"dArr",   8659},
+  {"dagger", 8224},
+  {"darr",   8595},
+  {"deg",     176},
+  {"delta",   948},
+  {"diams",  9830},
+  {"divide",  247},
+  {"eacute",  233},
+  {"ecaron",  283},
+  {"ecirc",   234},
+  {"egrave",  232},
+  {"emdash", 8212},
+  {"empty",  8709},
+  {"emsp",   8195},
+  {"endash", 8211},
+  {"ensp",   8194},
+  {"epsilon",  949},
+  {"equiv",  8801},
+  {"eta",     951},
+  {"eth",     240},
+  {"euml",    235},
+  {"euro",   8364},
+  {"exist",  8707},
+  {"fnof",    402},
+  {"forall", 8704},
+  {"frac12",  189},
+  {"frac14",  188},
+  {"frac34",  190},
+  {"frasl",  8260},
+  {"gamma",   947},
+  {"ge",     8805},
+  {"gt",       62},
+  {"hArr",   8660},
+  {"harr",   8596},
+  {"hearts", 9829},
+  {"hellip", 8230},
+  {"iacute",  237},
+  {"icirc",   238},
+  {"iexcl",   161},
+  {"igrave",  236},
+  {"image",  8465},
+  {"infin",  8734},
+  {"int",    8747},
+  {"iota",    953},
+  {"iquest",  191},
+  {"isin",   8712},
+  {"iuml",    239},
+  {"kappa",   954},
+  {"lArr",   8656},
+  {"lambda",  955},
+  {"lang",   9001},
+  {"laquo",   171},
+  {"larr",   8592},
+  {"lceil",  8968},
+  {"ldquo",  8220},
+  {"le",     8804},
+  {"lfloor", 8970},
+  {"lowast", 8727},
+  {"loz",    9674},
+  {"lrm",    8206},
+  {"lsaquo", 8249},
+  {"lsquo",  8216},
+  {"lt",       60},
+  {"macr",    175},
+  {"mdash",  8212},
+  {"micro",   181},
+  {"middot",  183},
+  {"minus",  8722},
+  {"mu",      956},
+  {"nabla",  8711},
+  {"nbsp",    160},
+  {"ndash",  8211},
+  {"ne",     8800},
+  {"ni",     8715},
+  {"not",     172},
+  {"notin",  8713},
+  {"nsub",   8836},
+  {"ntilde",  241},
+  {"nu",      957},
+  {"oacute",  243},
+  {"ocirc",   244},
+  {"oelig",   339},
+  {"ograve",  242},
+  {"oline",  8254},
+  {"omega",   969},
+  {"omicron",  959},
+  {"oplus",  8853},
+  {"or",     8744},
+  {"ordf",    170},
+  {"ordm",    186},
+  {"oslash",  248},
+  {"otilde",  245},
+  {"otimes", 8855},
+  {"ouml",    246},
+  {"para",    182},
+  {"part",   8706},
+  {"permil", 8240},
+  {"perp",   8869},
+  {"phi",     966},
+  {"pi",      960},
+  {"piv",     982},
+  {"plusmn",  177},
+  {"pound",   163},
+  {"prime",  8242},
+  {"prod",   8719},
+  {"prop",   8733},
+  {"psi",     968},
+  {"quot",     34},
+  {"rArr",   8658},
+  {"radic",  8730},
+  {"rang",   9002},
+  {"raquo",   187},
+  {"rarr",   8594},
+  {"rcaron",  345},
+  {"rceil",  8969},
+  {"rdquo",  8221},
+  {"real",   8476},
+  {"reg",     174},
+  {"rfloor", 8971},
+  {"rho",     961},
+  {"rlm",    8207},
+  {"rsaquo", 8250},
+  {"rsquo",  8217},
+  {"sbquo",  8218},
+  {"scaron",  353},
+  {"sdot",   8901},
+  {"sect",    167},
+  {"shy",     173},
+  {"sigma",   963},
+  {"sigmaf",  962},
+  {"sim",    8764},
+  {"spades", 9824},
+  {"sub",    8834},
+  {"sube",   8838},
+  {"sum",    8721},
+  {"sup",    8835},
+  {"sup1",    185},
+  {"sup2",    178},
+  {"sup3",    179},
+  {"supe",   8839},
+  {"szlig",   223},
+  {"tau",     964},
+  {"there4", 8756},
+  {"theta",   952},
+  {"thetasym",  977},
+  {"thinsp", 8201},
+  {"thorn",   254},
+  {"tilde",   732},
+  {"times",   215},
+  {"trade",  8482},
+  {"uArr",   8657},
+  {"uacute",  250},
+  {"uarr",   8593},
+  {"ucirc",   251},
+  {"ugrave",  249},
+  {"uml",     168},
+  {"upsih",   978},
+  {"upsilon",  965},
+  {"uuml",    252},
+  {"weierp", 8472},
+  {"xi",      958},
+  {"yacute",  253},
+  {"yen",     165},
+  {"yuml",    255},
+  {"zeta",    950},
+  {"zwj",    8205},
+  {"zwnj",   8204},
+};
+
+}  // namespace CLD2
+}  // namespace chrome_lang_id