libv8 6.7.288.46.1 → 7.3.492.27.0beta1

data/vendor/depot_tools/gclient_eval.py

@@ -3,15 +3,29 @@
 # found in the LICENSE file.
 
 import ast
-import cStringIO
 import collections
 import logging
+import sys
 import tokenize
 
 import gclient_utils
 
 from third_party import schema
 
+if sys.version_info.major == 2:
+  # We use cStringIO.StringIO because it is equivalent to Py3's io.StringIO.
+  from cStringIO import StringIO
+else:
+  from io import StringIO
+
+
+# TODO(crbug.com/953884): Remove this when python3 migration is done.
+try:
+  basestring
+except NameError:
+  # pylint: disable=redefined-builtin
+  basestring = str
+
 
 class _NodeDict(collections.MutableMapping):
   """Dict-like type that also stores information on AST nodes and tokens."""
@@ -20,7 +34,10 @@ class _NodeDict(collections.MutableMapping):
     self.tokens = tokens
 
   def __str__(self):
-    return str({k: v[0] for k, v in self.data.iteritems()})
+    return str({k: v[0] for k, v in self.data.items()})
+
+  def __repr__(self):
+    return self.__str__()
 
   def __getitem__(self, key):
     return self.data[key][0]
@@ -69,141 +86,146 @@ def _NodeDictSchema(dict_schema):
 
 # See https://github.com/keleshev/schema for docs how to configure schema.
 _GCLIENT_DEPS_SCHEMA = _NodeDictSchema({
-    schema.Optional(basestring): schema.Or(
-        None,
-        basestring,
-        _NodeDictSchema({
-            # Repo and revision to check out under the path
-            # (same as if no dict was used).
-            'url': schema.Or(None, basestring),
-
-            # Optional condition string. The dep will only be processed
-            # if the condition evaluates to True.
-            schema.Optional('condition'): basestring,
-
-            schema.Optional('dep_type', default='git'): basestring,
+    schema.Optional(basestring):
+        schema.Or(
+            None,
+            basestring,
+            _NodeDictSchema({
+                # Repo and revision to check out under the path
+                # (same as if no dict was used).
+                'url': schema.Or(None, basestring),
+
+                # Optional condition string. The dep will only be processed
+                # if the condition evaluates to True.
+                schema.Optional('condition'): basestring,
+                schema.Optional('dep_type', default='git'): basestring,
+            }),
+            # CIPD package.
+            _NodeDictSchema({
+                'packages': [
+                    _NodeDictSchema({
+                        'package': basestring,
+                        'version': basestring,
+                    })
+                ],
+                schema.Optional('condition'): basestring,
+                schema.Optional('dep_type', default='cipd'): basestring,
+            }),
+        ),
+})
+
+_GCLIENT_HOOKS_SCHEMA = [
+    _NodeDictSchema({
+        # Hook action: list of command-line arguments to invoke.
+        'action': [basestring],
+
+        # Name of the hook. Doesn't affect operation.
+        schema.Optional('name'): basestring,
+
+        # Hook pattern (regex). Originally intended to limit some hooks to run
+        # only when files matching the pattern have changed. In practice, with
+        # git, gclient runs all the hooks regardless of this field.
+        schema.Optional('pattern'): basestring,
+
+        # Working directory where to execute the hook.
+        schema.Optional('cwd'): basestring,
+
+        # Optional condition string. The hook will only be run
+        # if the condition evaluates to True.
+        schema.Optional('condition'): basestring,
+    })
+]
+
+_GCLIENT_SCHEMA = schema.Schema(
+    _NodeDictSchema({
+        # List of host names from which dependencies are allowed (whitelist).
+        # NOTE: when not present, all hosts are allowed.
+        # NOTE: scoped to current DEPS file, not recursive.
+        schema.Optional('allowed_hosts'): [schema.Optional(basestring)],
+
+        # Mapping from paths to repo and revision to check out under that path.
+        # Applying this mapping to the on-disk checkout is the main purpose
+        # of gclient, and also why the config file is called DEPS.
+        #
+        # The following functions are allowed:
+        #
+        #   Var(): allows variable substitution (either from 'vars' dict below,
+        #          or command-line override)
+        schema.Optional('deps'): _GCLIENT_DEPS_SCHEMA,
+
+        # Similar to 'deps' (see above) - also keyed by OS (e.g. 'linux').
+        # Also see 'target_os'.
+        schema.Optional('deps_os'): _NodeDictSchema({
+            schema.Optional(basestring): _GCLIENT_DEPS_SCHEMA,
         }),
-        # CIPD package.
-        _NodeDictSchema({
-            'packages': [
-                _NodeDictSchema({
-                    'package': basestring,
 
-                    'version': basestring,
-                })
-            ],
+        # Dependency to get gclient_gn_args* settings from. This allows these
+        # values to be set in a recursedeps file, rather than requiring that
+        # they exist in the top-level solution.
+        schema.Optional('gclient_gn_args_from'): basestring,
+
+        # Path to GN args file to write selected variables.
+        schema.Optional('gclient_gn_args_file'): basestring,
 
-            schema.Optional('condition'): basestring,
+        # Subset of variables to write to the GN args file (see above).
+        schema.Optional('gclient_gn_args'): [schema.Optional(basestring)],
 
-            schema.Optional('dep_type', default='cipd'): basestring,
+        # Hooks executed after gclient sync (unless suppressed), or explicitly
+        # on gclient hooks. See _GCLIENT_HOOKS_SCHEMA for details.
+        # Also see 'pre_deps_hooks'.
+        schema.Optional('hooks'): _GCLIENT_HOOKS_SCHEMA,
+
+        # Similar to 'hooks', also keyed by OS.
+        schema.Optional('hooks_os'): _NodeDictSchema({
+            schema.Optional(basestring): _GCLIENT_HOOKS_SCHEMA
         }),
-    ),
-})
 
-_GCLIENT_HOOKS_SCHEMA = [_NodeDictSchema({
-    # Hook action: list of command-line arguments to invoke.
-    'action': [basestring],
-
-    # Name of the hook. Doesn't affect operation.
-    schema.Optional('name'): basestring,
-
-    # Hook pattern (regex). Originally intended to limit some hooks to run
-    # only when files matching the pattern have changed. In practice, with git,
-    # gclient runs all the hooks regardless of this field.
-    schema.Optional('pattern'): basestring,
-
-    # Working directory where to execute the hook.
-    schema.Optional('cwd'): basestring,
-
-    # Optional condition string. The hook will only be run
-    # if the condition evaluates to True.
-    schema.Optional('condition'): basestring,
-})]
-
-_GCLIENT_SCHEMA = schema.Schema(_NodeDictSchema({
-    # List of host names from which dependencies are allowed (whitelist).
-    # NOTE: when not present, all hosts are allowed.
-    # NOTE: scoped to current DEPS file, not recursive.
-    schema.Optional('allowed_hosts'): [schema.Optional(basestring)],
-
-    # Mapping from paths to repo and revision to check out under that path.
-    # Applying this mapping to the on-disk checkout is the main purpose
-    # of gclient, and also why the config file is called DEPS.
-    #
-    # The following functions are allowed:
-    #
-    #   Var(): allows variable substitution (either from 'vars' dict below,
-    #          or command-line override)
-    schema.Optional('deps'): _GCLIENT_DEPS_SCHEMA,
-
-    # Similar to 'deps' (see above) - also keyed by OS (e.g. 'linux').
-    # Also see 'target_os'.
-    schema.Optional('deps_os'): _NodeDictSchema({
-        schema.Optional(basestring): _GCLIENT_DEPS_SCHEMA,
-    }),
-
-    # Dependency to get gclient_gn_args* settings from. This allows these values
-    # to be set in a recursedeps file, rather than requiring that they exist in
-    # the top-level solution.
-    schema.Optional('gclient_gn_args_from'): basestring,
-
-    # Path to GN args file to write selected variables.
-    schema.Optional('gclient_gn_args_file'): basestring,
-
-    # Subset of variables to write to the GN args file (see above).
-    schema.Optional('gclient_gn_args'): [schema.Optional(basestring)],
-
-    # Hooks executed after gclient sync (unless suppressed), or explicitly
-    # on gclient hooks. See _GCLIENT_HOOKS_SCHEMA for details.
-    # Also see 'pre_deps_hooks'.
-    schema.Optional('hooks'): _GCLIENT_HOOKS_SCHEMA,
-
-    # Similar to 'hooks', also keyed by OS.
-    schema.Optional('hooks_os'): _NodeDictSchema({
-        schema.Optional(basestring): _GCLIENT_HOOKS_SCHEMA
-    }),
-
-    # Rules which #includes are allowed in the directory.
-    # Also see 'skip_child_includes' and 'specific_include_rules'.
-    schema.Optional('include_rules'): [schema.Optional(basestring)],
-
-    # Hooks executed before processing DEPS. See 'hooks' for more details.
-    schema.Optional('pre_deps_hooks'): _GCLIENT_HOOKS_SCHEMA,
-
-    # Recursion limit for nested DEPS.
-    schema.Optional('recursion'): int,
-
-    # Whitelists deps for which recursion should be enabled.
-    schema.Optional('recursedeps'): [
-        schema.Optional(schema.Or(
-            basestring,
-            (basestring, basestring),
-            [basestring, basestring]
-        )),
-    ],
+        # Rules which #includes are allowed in the directory.
+        # Also see 'skip_child_includes' and 'specific_include_rules'.
+        schema.Optional('include_rules'): [schema.Optional(basestring)],
+
+        # Hooks executed before processing DEPS. See 'hooks' for more details.
+        schema.Optional('pre_deps_hooks'): _GCLIENT_HOOKS_SCHEMA,
 
-    # Blacklists directories for checking 'include_rules'.
-    schema.Optional('skip_child_includes'): [schema.Optional(basestring)],
+        # Recursion limit for nested DEPS.
+        schema.Optional('recursion'): int,
 
-    # Mapping from paths to include rules specific for that path.
-    # See 'include_rules' for more details.
-    schema.Optional('specific_include_rules'): _NodeDictSchema({
-        schema.Optional(basestring): [basestring]
-    }),
+        # Whitelists deps for which recursion should be enabled.
+        schema.Optional('recursedeps'): [
+            schema.Optional(schema.Or(
+                basestring,
+                (basestring, basestring),
+                [basestring, basestring]
+            )),
+        ],
 
-    # List of additional OS names to consider when selecting dependencies
-    # from deps_os.
-    schema.Optional('target_os'): [schema.Optional(basestring)],
+        # Blacklists directories for checking 'include_rules'.
+        schema.Optional('skip_child_includes'): [schema.Optional(basestring)],
 
-    # For recursed-upon sub-dependencies, check out their own dependencies
-    # relative to the paren't path, rather than relative to the .gclient file.
-    schema.Optional('use_relative_paths'): bool,
+        # Mapping from paths to include rules specific for that path.
+        # See 'include_rules' for more details.
+        schema.Optional('specific_include_rules'): _NodeDictSchema({
+            schema.Optional(basestring): [basestring]
+        }),
 
-    # Variables that can be referenced using Var() - see 'deps'.
-    schema.Optional('vars'): _NodeDictSchema({
-        schema.Optional(basestring): schema.Or(basestring, bool),
-    }),
-}))
+        # List of additional OS names to consider when selecting dependencies
+        # from deps_os.
+        schema.Optional('target_os'): [schema.Optional(basestring)],
+
+        # For recursed-upon sub-dependencies, check out their own dependencies
+        # relative to the parent's path, rather than relative to the .gclient
+        # file.
+        schema.Optional('use_relative_paths'): bool,
+
+        # For recursed-upon sub-dependencies, run their hooks relative to the
+        # parent's path instead of relative to the .gclient file.
+        schema.Optional('use_relative_hooks'): bool,
+
+        # Variables that can be referenced using Var() - see 'deps'.
+        schema.Optional('vars'): _NodeDictSchema({
+            schema.Optional(basestring): schema.Or(basestring, bool),
+        }),
+    }))
 
 
 def _gclient_eval(node_or_string, filename='<unknown>', vars_dict=None):
@@ -239,12 +261,16 @@ def _gclient_eval(node_or_string, filename='<unknown>', vars_dict=None):
             'invalid name %r (file %r, line %s)' % (
                 node.id, filename, getattr(node, 'lineno', '<unknown>')))
       return _allowed_names[node.id]
+    elif not sys.version_info[:2] < (3, 4) and isinstance(
+        node, ast.NameConstant):  # Since Python 3.4
+      return node.value
     elif isinstance(node, ast.Call):
       if not isinstance(node.func, ast.Name) or node.func.id != 'Var':
         raise ValueError(
             'Var is the only allowed function (file %r, line %s)' % (
                 filename, getattr(node, 'lineno', '<unknown>')))
-      if node.keywords or node.starargs or node.kwargs or len(node.args) != 1:
+      if node.keywords or getattr(node, 'starargs', None) or getattr(
+          node, 'kwargs', None) or len(node.args) != 1:
         raise ValueError(
             'Var takes exactly one argument (file %r, line %s)' % (
                 filename, getattr(node, 'lineno', '<unknown>')))
@@ -273,7 +299,7 @@ def _gclient_eval(node_or_string, filename='<unknown>', vars_dict=None):
   return _convert(node_or_string)
 
 
-def Exec(content, filename='<unknown>', vars_override=None):
+def Exec(content, filename='<unknown>', vars_override=None, builtin_vars=None):
   """Safely execs a set of assignments."""
   def _validate_statement(node, local_scope):
     if not isinstance(node, ast.Assign):
@@ -314,11 +340,19 @@ def Exec(content, filename='<unknown>', vars_override=None):
     _validate_statement(statement, statements)
     statements[statement.targets[0].id] = statement.value
 
+  # The tokenized representation needs to end with a newline token, otherwise
+  # untokenization will trigger an assert later on.
+  # In Python 2.7 on Windows we need to ensure the input ends with a newline
+  # for a newline token to be generated.
+  # In other cases a newline token is always generated during tokenization so
+  # this has no effect.
+  # TODO: Remove this workaround after migrating to Python 3.
+  content += '\n'
   tokens = {
-      token[2]: list(token)
-      for token in tokenize.generate_tokens(
-          cStringIO.StringIO(content).readline)
+      token[2]: list(token) for token in tokenize.generate_tokens(
+          StringIO(content).readline)
   }
+
   local_scope = _NodeDict({}, tokens)
 
   # Process vars first, so we can expand variables in the rest of the DEPS file.
@@ -330,20 +364,22 @@ def Exec(content, filename='<unknown>', vars_override=None):
     # Update the parsed vars with the overrides, but only if they are already
     # present (overrides do not introduce new variables).
     vars_dict.update(value)
-    if vars_override:
-      vars_dict.update({
-        k: v
-        for k, v in vars_override.iteritems()
-        if k in vars_dict})
 
-  for name, node in statements.iteritems():
+  if builtin_vars:
+    vars_dict.update(builtin_vars)
+
+  if vars_override:
+    vars_dict.update({k: v for k, v in vars_override.items() if k in vars_dict})
+
+  for name, node in statements.items():
     value = _gclient_eval(node, filename, vars_dict)
     local_scope.SetNode(name, value, node)
 
   return _GCLIENT_SCHEMA.validate(local_scope)
 
 
-def ExecLegacy(content, filename='<unknown>', vars_override=None):
+def ExecLegacy(content, filename='<unknown>', vars_override=None,
+               builtin_vars=None):
   """Executes a DEPS file |content| using exec."""
   local_scope = {}
   global_scope = {'Var': lambda var_name: '{%s}' % var_name}
@@ -354,26 +390,21 @@ def ExecLegacy(content, filename='<unknown>', vars_override=None):
   # as "exec a in b, c" (See https://bugs.python.org/issue21591).
   eval(compile(content, filename, 'exec'), global_scope, local_scope)
 
-  if 'vars' not in local_scope:
-    return local_scope
-
   vars_dict = {}
-  vars_dict.update(local_scope['vars'])
+  vars_dict.update(local_scope.get('vars', {}))
+  if builtin_vars:
+    vars_dict.update(builtin_vars)
   if vars_override:
-    vars_dict.update({
-        k: v
-        for k, v in vars_override.iteritems()
-        if k in vars_dict
-    })
+    vars_dict.update({k: v for k, v in vars_override.items() if k in vars_dict})
+
+  if not vars_dict:
+    return local_scope
 
   def _DeepFormat(node):
     if isinstance(node, basestring):
       return node.format(**vars_dict)
     elif isinstance(node, dict):
-      return {
-          k.format(**vars_dict): _DeepFormat(v)
-          for k, v in node.iteritems()
-      }
+      return {k.format(**vars_dict): _DeepFormat(v) for k, v in node.items()}
     elif isinstance(node, list):
       return [_DeepFormat(elem) for elem in node]
     elif isinstance(node, tuple):
@@ -449,7 +480,8 @@ def UpdateCondition(info_dict, op, new_condition):
     del info_dict['condition']
 
 
-def Parse(content, validate_syntax, filename, vars_override=None):
+def Parse(content, validate_syntax, filename, vars_override=None,
+          builtin_vars=None):
   """Parses DEPS strings.
 
   Executes the Python-like string stored in content, resulting in a Python
@@ -464,15 +496,17 @@ def Parse(content, validate_syntax, filename, vars_override=None):
       of the content, e.g. '<string>', '<unknown>'.
     vars_override: dict, optional. A dictionary with overrides for the variables
       defined by the DEPS file.
+    builtin_vars: dict, optional. A dictionary with variables that are provided
+      by default.
 
   Returns:
     A Python dict with the parsed contents of the DEPS file, as specified by the
     schema above.
   """
   if validate_syntax:
-    result = Exec(content, filename, vars_override)
+    result = Exec(content, filename, vars_override, builtin_vars)
   else:
-    result = ExecLegacy(content, filename, vars_override)
+    result = ExecLegacy(content, filename, vars_override, builtin_vars)
 
   vars_dict = result.get('vars', {})
   if 'deps' in result:
@@ -597,12 +631,18 @@ def EvaluateCondition(condition, variables, referenced_variables=None):
 
 def RenderDEPSFile(gclient_dict):
   contents = sorted(gclient_dict.tokens.values(), key=lambda token: token[2])
-  return tokenize.untokenize(contents)
+  # The last token is a newline, which we ensure in Exec() for compatibility.
+  # However tests pass in inputs not ending with a newline and expect the same
+  # back, so for backwards compatibility need to remove that newline character.
+  # TODO: Fix tests to expect the newline
+  return tokenize.untokenize(contents)[:-1]
 
 
 def _UpdateAstString(tokens, node, value):
   position = node.lineno, node.col_offset
-  quote_char = tokens[position][1][0]
+  quote_char = ''
+  if isinstance(node, ast.Str):
+    quote_char = tokens[position][1][0]
   tokens[position][1] = quote_char + value + quote_char
   node.s = value
 
@@ -660,8 +700,7 @@ def AddVar(gclient_dict, var_name, value):
   vars_node.values.insert(0, value_node)
 
   # Update the tokens.
-  var_tokens = list(tokenize.generate_tokens(
-      cStringIO.StringIO(var_content).readline))
+  var_tokens = list(tokenize.generate_tokens(StringIO(var_content).readline))
   var_tokens = {
       token[2]: list(token)
       # Ignore the tokens corresponding to braces and new lines.
@@ -696,6 +735,15 @@ def SetVar(gclient_dict, var_name, value):
   gclient_dict['vars'].SetNode(var_name, value, node)
 
 
+def _GetVarName(node):
+  if isinstance(node, ast.Call):
+    return node.args[0].s
+  elif node.s.endswith('}'):
+    last_brace = node.s.rfind('{')
+    return node.s[last_brace+1:-1]
+  return None
+
+
 def SetCIPD(gclient_dict, dep_name, package_name, new_version):
   if not isinstance(gclient_dict, _NodeDict) or gclient_dict.tokens is None:
     raise ValueError(
@@ -725,20 +773,20 @@ def SetCIPD(gclient_dict, dep_name, package_name, new_version):
         "The deps entry for %s:%s has no formatting information." %
         (dep_name, package_name))
 
-  new_version = 'version:' + new_version
-  _UpdateAstString(tokens, node, new_version)
-  packages[0].SetNode('version', new_version, node)
+  if not isinstance(node, ast.Call) and not isinstance(node, ast.Str):
+    raise ValueError(
+        "Unsupported dependency revision format. Please file a bug to the "
+        "Infra>SDK component in crbug.com")
 
+  var_name = _GetVarName(node)
+  if var_name is not None:
+    SetVar(gclient_dict, var_name, new_version)
+  else:
+    _UpdateAstString(tokens, node, new_version)
+    packages[0].SetNode('version', new_version, node)
 
-def SetRevision(gclient_dict, dep_name, new_revision):
-  def _GetVarName(node):
-    if isinstance(node, ast.Call):
-      return node.args[0].s
-    elif node.s.endswith('}'):
-      last_brace = node.s.rfind('{')
-      return node.s[last_brace+1:-1]
-    return None
 
+def SetRevision(gclient_dict, dep_name, new_revision):
   def _UpdateRevision(dep_dict, dep_key, new_revision):
     dep_node = dep_dict.GetNode(dep_key)
     if dep_node is None:
@@ -751,7 +799,8 @@ def SetRevision(gclient_dict, dep_name, new_revision):
 
     if not isinstance(node, ast.Call) and not isinstance(node, ast.Str):
       raise ValueError(
-          "Unsupported dependency revision format. Please file a bug.")
+          "Unsupported dependency revision format. Please file a bug to the "
+          "Infra>SDK component in crbug.com")
 
     var_name = _GetVarName(node)
     if var_name is not None:
@@ -808,7 +857,7 @@ def GetCIPD(gclient_dict, dep_name, package_name):
         "There must be exactly one package with the given name (%s), "
         "%s were found." % (package_name, len(packages)))
 
-  return packages[0]['version'][len('version:'):]
+  return packages[0]['version']
 
 
 def GetRevision(gclient_dict, dep_name):