libv8 7.3.492.27.1 → 8.4.255.0

data/vendor/depot_tools/gclient_utils.py

@@ -17,12 +17,6 @@ import operator
 import os
 import pipes
 import platform
-
-try:
-  import Queue as queue
-except ImportError:  # For Py3 compatibility
-  import queue
-
 import re
 import stat
 import subprocess
@@ -30,18 +24,18 @@ import sys
 import tempfile
 import threading
 import time
-
-try:
-  import urlparse
-except ImportError:  # For Py3 compatibility
-  import urllib.parse as urlparse
-
 import subprocess2
 
 if sys.version_info.major == 2:
   from cStringIO import StringIO
+  import collections as collections_abc
+  import Queue as queue
+  import urlparse
 else:
+  from collections import abc as collections_abc
   from io import StringIO
+  import queue
+  import urllib.parse as urlparse
 
 
 RETRY_MAX = 3
@@ -55,7 +49,7 @@ _WARNINGS = []
 # These repos are known to cause OOM errors on 32-bit platforms, due the the
 # very large objects they contain.  It is not safe to use threaded index-pack
 # when cloning/fetching them.
-THREADED_INDEX_PACK_BLACKLIST = [
+THREADED_INDEX_PACK_BLOCKLIST = [
   'https://chromium.googlesource.com/chromium/reference_builds/chrome_win.git'
 ]
 
@@ -164,20 +158,31 @@ class PrintableObject(object):
     return output
 
 
-def FileRead(filename, mode='rU'):
+def AskForData(message):
+  # Use this so that it can be mocked in tests on Python 2 and 3.
+  try:
+    if sys.version_info.major == 2:
+      return raw_input(message)
+    return input(message)
+  except KeyboardInterrupt:
+    # Hide the exception.
+    sys.exit(1)
+
+
+def FileRead(filename, mode='rbU'):
+  # Always decodes output to a Unicode string.
+  # On Python 3 newlines are converted to '\n' by default and 'U' is deprecated.
+  if mode == 'rbU' and sys.version_info.major == 3:
+    mode = 'rb'
   with open(filename, mode=mode) as f:
-    # codecs.open() has different behavior than open() on python 2.6 so use
-    # open() and decode manually.
     s = f.read()
-    try:
-      return s.decode('utf-8')
-    # AttributeError is for Py3 compatibility
-    except (UnicodeDecodeError, AttributeError):
-      return s
+    if isinstance(s, bytes):
+      return s.decode('utf-8', 'replace')
+    return s
 
 
-def FileWrite(filename, content, mode='w'):
-  with codecs.open(filename, mode=mode, encoding='utf-8') as f:
+def FileWrite(filename, content, mode='w', encoding='utf-8'):
+  with codecs.open(filename, mode=mode, encoding=encoding) as f:
     f.write(content)
 
 
@@ -191,6 +196,35 @@ def temporary_directory(**kwargs):
       rmtree(tdir)
 
 
+@contextlib.contextmanager
+def temporary_file():
+  """Creates a temporary file.
+
+  On Windows, a file must be closed before it can be opened again. This function
+  allows to write something like:
+
+    with gclient_utils.temporary_file() as tmp:
+      gclient_utils.FileWrite(tmp, foo)
+      useful_stuff(tmp)
+
+  Instead of something like:
+
+    with tempfile.NamedTemporaryFile(delete=False) as tmp:
+      tmp.write(foo)
+      tmp.close()
+      try:
+        useful_stuff(tmp)
+      finally:
+        os.remove(tmp.name)
+  """
+  handle, name = tempfile.mkstemp()
+  os.close(handle)
+  try:
+    yield name
+  finally:
+    os.remove(name)
+
+
 def safe_rename(old, new):
   """Renames a file reliably.
 
@@ -215,7 +249,7 @@ def safe_rename(old, new):
 
 
 def rm_file_or_tree(path):
-  if os.path.isfile(path):
+  if os.path.isfile(path) or os.path.islink(path):
     os.remove(path)
   else:
     rmtree(path)
@@ -290,7 +324,7 @@ def rmtree(path):
 def safe_makedirs(tree):
   """Creates the directory in a safe manner.
 
-  Because multiple threads can create these directories concurently, trap the
+  Because multiple threads can create these directories concurrently, trap the
   exception and pass on.
   """
   count = 0
@@ -312,38 +346,6 @@ def CommandToStr(args):
   return ' '.join(pipes.quote(arg) for arg in args)
 
 
-def CheckCallAndFilterAndHeader(args, always=False, header=None, **kwargs):
-  """Adds 'header' support to CheckCallAndFilter.
-
-  If |always| is True, a message indicating what is being done
-  is printed to stdout all the time even if not output is generated. Otherwise
-  the message header is printed only if the call generated any ouput.
-  """
-  stdout = kwargs.setdefault('stdout', sys.stdout)
-  if header is None:
-    # The automatically generated header only prepends newline if always is
-    # false: always is usually set to false if there's an external progress
-    # display, and it's better not to clobber it in that case.
-    header = "%s________ running '%s' in '%s'\n" % (
-                 '' if always else '\n',
-                 ' '.join(args), kwargs.get('cwd', '.'))
-
-  if always:
-    stdout.write(header)
-  else:
-    filter_fn = kwargs.get('filter_fn')
-    def filter_msg(line):
-      if line is None:
-        stdout.write(header)
-      elif filter_fn:
-        filter_fn(line)
-    kwargs['filter_fn'] = filter_msg
-    kwargs['call_filter_on_first_line'] = True
-  # Obviously.
-  kwargs.setdefault('print_stdout', True)
-  return CheckCallAndFilter(args, **kwargs)
-
-
 class Wrapper(object):
   """Wraps an object, acting as a transparent proxy for all properties by
   default.
@@ -355,22 +357,6 @@ class Wrapper(object):
     return getattr(self._wrapped, name)
 
 
-class WriteToStdout(Wrapper):
-  """Creates a file object clone to also print to sys.stdout."""
-  def __init__(self, wrapped):
-    super(WriteToStdout, self).__init__(wrapped)
-    if not hasattr(self, 'lock'):
-      self.lock = threading.Lock()
-
-  def write(self, out, *args, **kwargs):
-    self._wrapped.write(out, *args, **kwargs)
-    self.lock.acquire()
-    try:
-      sys.stdout.write(out, *args, **kwargs)
-    finally:
-      self.lock.release()
-
-
 class AutoFlush(Wrapper):
   """Creates a file object clone to automatically flush after N seconds."""
   def __init__(self, wrapped, delay):
@@ -408,16 +394,21 @@ class Annotated(Wrapper):
       self.lock = threading.Lock()
     self.__output_buffers = {}
     self.__include_zero = include_zero
+    self._wrapped_write = getattr(self._wrapped, 'buffer', self._wrapped).write
 
   @property
   def annotated(self):
     return self
 
   def write(self, out):
+    # Store as bytes to ensure Unicode characters get output correctly.
+    if not isinstance(out, bytes):
+      out = out.encode('utf-8')
+
     index = getattr(threading.currentThread(), 'index', 0)
     if not index and not self.__include_zero:
       # Unindexed threads aren't buffered.
-      return self._wrapped.write(out)
+      return self._wrapped_write(out)
 
     self.lock.acquire()
     try:
@@ -425,7 +416,7 @@ class Annotated(Wrapper):
       # Strings are immutable, requiring to keep a lock for the whole dictionary
       # otherwise. Using an array is faster than using a dummy object.
       if not index in self.__output_buffers:
-        obj = self.__output_buffers[index] = ['']
+        obj = self.__output_buffers[index] = [b'']
       else:
         obj = self.__output_buffers[index]
     finally:
@@ -433,10 +424,19 @@ class Annotated(Wrapper):
 
     # Continue lockless.
     obj[0] += out
-    while '\n' in obj[0]:
-      line, remaining = obj[0].split('\n', 1)
-      if line:
-        self._wrapped.write('%d>%s\n' % (index, line))
+    while True:
+      cr_loc = obj[0].find(b'\r')
+      lf_loc = obj[0].find(b'\n')
+      if cr_loc == lf_loc == -1:
+        break
+      elif cr_loc == -1 or (lf_loc >= 0 and lf_loc < cr_loc):
+        line, remaining = obj[0].split(b'\n', 1)
+        if line:
+          self._wrapped_write(b'%d>%s\n' % (index, line))
+      elif lf_loc == -1 or (cr_loc >= 0 and cr_loc < lf_loc):
+        line, remaining = obj[0].split(b'\r', 1)
+        if line:
+          self._wrapped_write(b'%d>%s\r' % (index, line))
       obj[0] = remaining
 
   def flush(self):
@@ -455,10 +455,10 @@ class Annotated(Wrapper):
     finally:
       self.lock.release()
 
-    # Don't keep the lock while writting. Will append \n when it shouldn't.
+    # Don't keep the lock while writing. Will append \n when it shouldn't.
     for orphan in orphans:
       if orphan[1]:
-        self._wrapped.write('%d>%s\n' % (orphan[0], orphan[1]))
+        self._wrapped_write(b'%d>%s\n' % (orphan[0], orphan[1]))
     return self._wrapped.flush()
 
 
@@ -473,7 +473,7 @@ def MakeFileAutoFlush(fileobj, delay=10):
 def MakeFileAnnotated(fileobj, include_zero=False):
   if getattr(fileobj, 'annotated', None):
     return fileobj
-  return Annotated(fileobj)
+  return Annotated(fileobj, include_zero)
 
 
 GCLIENT_CHILDREN = []
@@ -526,9 +526,9 @@ class GClientChildren(object):
           print('  ', zombie.pid, file=sys.stderr)
 
 
-def CheckCallAndFilter(args, stdout=None, filter_fn=None,
-                       print_stdout=None, call_filter_on_first_line=False,
-                       retry=False, **kwargs):
+def CheckCallAndFilter(args, print_stdout=False, filter_fn=None,
+                       show_header=False, always_show_header=False, retry=False,
+                       **kwargs):
   """Runs a command and calls back a filter function if needed.
 
   Accepts all subprocess2.Popen() parameters plus:
@@ -536,28 +536,69 @@ def CheckCallAndFilter(args, stdout=None, filter_fn=None,
     filter_fn: A function taking a single string argument called with each line
                of the subprocess2's output. Each line has the trailing newline
                character trimmed.
-    stdout: Can be any bufferable output.
+    show_header: Whether to display a header before the command output.
+    always_show_header: Show header even when the command produced no output.
     retry: If the process exits non-zero, sleep for a brief interval and try
            again, up to RETRY_MAX times.
 
   stderr is always redirected to stdout.
+
+  Returns the output of the command as a binary string.
   """
-  assert print_stdout or filter_fn
-  stdout = stdout or sys.stdout
-  output = io.BytesIO()
-  filter_fn = filter_fn or (lambda x: None)
+  def show_header_if_necessary(needs_header, attempt):
+    """Show the header at most once."""
+    if not needs_header[0]:
+      return
+
+    needs_header[0] = False
+    # Automatically generated header. We only prepend a newline if
+    # always_show_header is false, since it usually indicates there's an
+    # external progress display, and it's better not to clobber it in that case.
+    header = '' if always_show_header else '\n'
+    header += '________ running \'%s\' in \'%s\'' % (
+                  ' '.join(args), kwargs.get('cwd', '.'))
+    if attempt:
+      header += ' attempt %s / %s' % (attempt + 1, RETRY_MAX + 1)
+    header += '\n'
+
+    if print_stdout:
+      stdout_write = getattr(sys.stdout, 'buffer', sys.stdout).write
+      stdout_write(header.encode())
+    if filter_fn:
+      filter_fn(header)
+
+  def filter_line(command_output, line_start):
+    """Extract the last line from command output and filter it."""
+    if not filter_fn or line_start is None:
+      return
+    command_output.seek(line_start)
+    filter_fn(command_output.read().decode('utf-8'))
+
+  # Initialize stdout writer if needed. On Python 3, sys.stdout does not accept
+  # byte inputs and sys.stdout.buffer must be used instead.
+  if print_stdout:
+    sys.stdout.flush()
+    stdout_write = getattr(sys.stdout, 'buffer', sys.stdout).write
+  else:
+    stdout_write = lambda _: None
 
   sleep_interval = RETRY_INITIAL_SLEEP
   run_cwd = kwargs.get('cwd', os.getcwd())
-  for _ in range(RETRY_MAX + 1):
+  for attempt in range(RETRY_MAX + 1):
     kid = subprocess2.Popen(
         args, bufsize=0, stdout=subprocess2.PIPE, stderr=subprocess2.STDOUT,
         **kwargs)
 
     GClientChildren.add(kid)
 
-    # Do a flush of stdout before we begin reading from the subprocess2's stdout
-    stdout.flush()
+    # Store the output of the command regardless of the value of print_stdout or
+    # filter_fn.
+    command_output = io.BytesIO()
+
+    # Passed as a list for "by ref" semantics.
+    needs_header = [show_header]
+    if always_show_header:
+      show_header_if_necessary(needs_header, attempt)
 
     # Also, we need to forward stdout to prevent weird re-ordering of output.
     # This has to be done on a per byte basis to make sure it is not buffered:
@@ -565,26 +606,31 @@ def CheckCallAndFilter(args, stdout=None, filter_fn=None,
     # input, no end-of-line character is output after the prompt and it would
     # not show up.
     try:
-      in_byte = kid.stdout.read(1)
-      if in_byte:
-        if call_filter_on_first_line:
-          filter_fn(None)
-        in_line = b''
-        while in_byte:
-          output.write(in_byte)
-          if print_stdout:
-            stdout.write(in_byte)
-          if in_byte not in ['\r', '\n']:
-            in_line += in_byte
-          else:
-            filter_fn(in_line)
-            in_line = b''
-          in_byte = kid.stdout.read(1)
-        # Flush the rest of buffered output. This is only an issue with
-        # stdout/stderr not ending with a \n.
-        if len(in_line):
-          filter_fn(in_line)
+      line_start = None
+      while True:
+        in_byte = kid.stdout.read(1)
+        is_newline = in_byte in (b'\n', b'\r')
+        if not in_byte:
+          break
+
+        show_header_if_necessary(needs_header, attempt)
+
+        if is_newline:
+          filter_line(command_output, line_start)
+          line_start = None
+        elif line_start is None:
+          line_start = command_output.tell()
+
+        stdout_write(in_byte)
+        command_output.write(in_byte)
+
+      # Flush the rest of buffered output.
+      sys.stdout.flush()
+      if line_start is not None:
+        filter_line(command_output, line_start)
+
       rv = kid.wait()
+      kid.stdout.close()
 
       # Don't put this in a 'finally,' since the child may still run if we get
       # an exception.
@@ -595,13 +641,16 @@ def CheckCallAndFilter(args, stdout=None, filter_fn=None,
       raise
 
     if rv == 0:
-      return output.getvalue()
+      return command_output.getvalue()
+
     if not retry:
       break
+
     print("WARNING: subprocess '%s' in %s failed; will retry after a short "
           'nap...' % (' '.join('"%s"' % x for x in args), run_cwd))
     time.sleep(sleep_interval)
     sleep_interval *= 2
+
   raise subprocess2.CalledProcessError(
       rv, args, kwargs.get('cwd', None), None, None)
 
@@ -624,6 +673,7 @@ class GitFilter(object):
         The line will be skipped if predicate(line) returns False.
       out_fh: File handle to write output to.
     """
+    self.first_line = True
     self.last_time = 0
     self.time_throttle = time_throttle
     self.predicate = predicate
@@ -632,20 +682,22 @@ class GitFilter(object):
 
   def __call__(self, line):
     # git uses an escape sequence to clear the line; elide it.
-    esc = line.find(chr(0o33).encode())
+    esc = line.find(chr(0o33))
     if esc > -1:
       line = line[:esc]
     if self.predicate and not self.predicate(line):
       return
     now = time.time()
-    match = self.PERCENT_RE.match(line.decode())
+    match = self.PERCENT_RE.match(line)
     if match:
       if match.group(1) != self.progress_prefix:
         self.progress_prefix = match.group(1)
       elif now - self.last_time < self.time_throttle:
         return
     self.last_time = now
-    self.out_fh.write('[%s] ' % Elapsed())
+    if not self.first_line:
+      self.out_fh.write('[%s] ' % Elapsed())
+    self.first_line = False
     print(line, file=self.out_fh)
 
 
@@ -687,7 +739,8 @@ def GetGClientRootAndEntries(path=None):
     return None
   config_path = os.path.join(root, config_file)
   env = {}
-  execfile(config_path, env)
+  with open(config_path) as config:
+    exec(config.read(), env)
   config_dir = os.path.dirname(config_path)
   return config_dir, env['entries']
 
@@ -900,7 +953,7 @@ class ExecutionQueue(object):
     running = self.running
     self.running = []
     for t in running:
-      if t.isAlive():
+      if t.is_alive():
         self.running.append(t)
       else:
         t.join()
@@ -1140,7 +1193,7 @@ def DefaultIndexPackConfig(url=''):
   performance."""
   cache_limit = DefaultDeltaBaseCacheLimit()
   result = ['-c', 'core.deltaBaseCacheLimit=%s' % cache_limit]
-  if url in THREADED_INDEX_PACK_BLACKLIST:
+  if url in THREADED_INDEX_PACK_BLOCKLIST:
     result.extend(['-c', 'pack.threads=1'])
   return result
 
@@ -1151,7 +1204,7 @@ def FindExecutable(executable):
 
   for path_folder in path_folders:
     target = os.path.join(path_folder, executable)
-    # Just incase we have some ~/blah paths.
+    # Just in case we have some ~/blah paths.
     target = os.path.abspath(os.path.expanduser(target))
     if os.path.isfile(target) and os.access(target, os.X_OK):
       return target
@@ -1175,7 +1228,7 @@ def freeze(obj):
 
   Will raise TypeError if you pass an object which is not hashable.
   """
-  if isinstance(obj, collections.Mapping):
+  if isinstance(obj, collections_abc.Mapping):
     return FrozenDict((freeze(k), freeze(v)) for k, v in obj.items())
   elif isinstance(obj, (list, tuple)):
     return tuple(freeze(i) for i in obj)
@@ -1186,7 +1239,7 @@ def freeze(obj):
     return obj
 
 
-class FrozenDict(collections.Mapping):
+class FrozenDict(collections_abc.Mapping):
   """An immutable OrderedDict.
 
   Modified From: http://stackoverflow.com/a/2704866
@@ -1200,13 +1253,13 @@ class FrozenDict(collections.Mapping):
         operator.xor, (hash(i) for i in enumerate(self._d.items())), 0)
 
   def __eq__(self, other):
-    if not isinstance(other, collections.Mapping):
+    if not isinstance(other, collections_abc.Mapping):
       return NotImplemented
     if self is other:
       return True
     if len(self) != len(other):
       return False
-    for k, v in self.iteritems():
+    for k, v in self.items():
       if k not in other or other[k] != v:
         return False
     return True