libv8 7.3.492.27.0 → 8.4.255.0

data/vendor/depot_tools/git-cache

@@ -3,4 +3,4 @@
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
-. $(type -P python_runner.sh)
+. "$(type -P python_runner.sh)"

data/vendor/depot_tools/git-cl

@@ -3,4 +3,4 @@
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
-. $(type -P python_runner.sh)
+. "$(type -P python_runner.sh)"

data/vendor/depot_tools/git-drover

@@ -3,4 +3,4 @@
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
-. $(type -P python_runner.sh)
+. "$(type -P python_runner.sh)"

data/vendor/depot_tools/git-find-releases

@@ -3,4 +3,4 @@
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
-. $(type -P python_runner.sh)
+. "$(type -P python_runner.sh)"

data/vendor/depot_tools/git-footers

@@ -3,4 +3,4 @@
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
-. $(type -P python_runner.sh)
+. "$(type -P python_runner.sh)"

data/vendor/depot_tools/git-freeze

@@ -5,4 +5,4 @@
 
 SCRIPT=git_freezer.py
 set -- freeze "$@"
-. $(type -P python_runner.sh)
+. "$(type -P python_runner.sh)"

data/vendor/depot_tools/git-hyper-blame

@@ -3,4 +3,4 @@
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
-. $(type -P python_runner.sh)
+. "$(type -P python_runner.sh)"

data/vendor/depot_tools/git-map

@@ -3,4 +3,4 @@
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
-. $(type -P python_runner.sh) | less -R
+. "$(type -P python_runner.sh)"

data/vendor/depot_tools/git-map-branches

@@ -3,4 +3,4 @@
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
-. $(type -P python_runner.sh)
+. "$(type -P python_runner.sh)"

data/vendor/depot_tools/git-mark-merge-base

@@ -3,4 +3,4 @@
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
-. $(type -P python_runner.sh)
+. "$(type -P python_runner.sh)"

data/vendor/depot_tools/git-nav-downstream

@@ -3,4 +3,4 @@
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
-. $(type -P python_runner.sh)
+. "$(type -P python_runner.sh)"

data/vendor/depot_tools/git-new-branch

@@ -3,4 +3,4 @@
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
-. $(type -P python_runner.sh)
+. "$(type -P python_runner.sh)"

data/vendor/depot_tools/git-number

@@ -3,4 +3,4 @@
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
-. $(type -P python_runner.sh)
+. "$(type -P python_runner.sh)"

data/vendor/depot_tools/git-rebase-update

@@ -3,4 +3,4 @@
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
-. $(type -P python_runner.sh)
+. "$(type -P python_runner.sh)"

data/vendor/depot_tools/git-rename-branch

@@ -3,4 +3,4 @@
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
-. $(type -P python_runner.sh)
+. "$(type -P python_runner.sh)"

data/vendor/depot_tools/git-reparent-branch

@@ -3,4 +3,4 @@
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
-. $(type -P python_runner.sh)
+. "$(type -P python_runner.sh)"

data/vendor/depot_tools/git-retry

@@ -5,4 +5,4 @@
 
 SCRIPT=git_retry.py
 set -- retry "$@"
-. $(type -P python_runner.sh)
+. "$(type -P python_runner.sh)"

data/vendor/depot_tools/git-squash-branch

@@ -3,4 +3,4 @@
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
-. $(type -P python_runner.sh)
+. "$(type -P python_runner.sh)"

data/vendor/depot_tools/git-thaw

@@ -10,4 +10,4 @@
 
 SCRIPT=git_freezer.py
 set -- thaw "$@"
-. $(type -P python_runner.sh)
+. "$(type -P python_runner.sh)"

data/vendor/depot_tools/git-upstream-diff

@@ -6,4 +6,4 @@
 # git_upstream_diff.py -- Provide the diff between the current branch and its
 # upstream.
 
-. $(type -P python_runner.sh)
+. "$(type -P python_runner.sh)"

data/vendor/depot_tools/git_cache.py

@@ -13,21 +13,20 @@ import logging
 import optparse
 import os
 import re
+import subprocess
+import sys
 import tempfile
 import threading
 import time
-import subprocess
-import sys
 
 try:
   import urlparse
 except ImportError:  # For Py3 compatibility
   import urllib.parse as urlparse
 
-import zipfile
-
 from download_from_google_storage import Gsutil
 import gclient_utils
+import lockfile
 import subcommand
 
 # Analogous to gc.autopacklimit git config.
@@ -42,9 +41,6 @@ except NameError:
   class WinErr(Exception):
     pass
 
-class LockError(Exception):
-  pass
-
 class ClobberNeeded(Exception):
   pass
 
@@ -71,7 +67,7 @@ def exponential_backoff_retry(fn, excs=(Exception,), name=None, count=10,
   Returns: The return value of the successful fn.
   """
   printerr = printerr or logging.warning
-  for i in xrange(count):
+  for i in range(count):
     try:
       return fn()
     except excs as e:
@@ -84,116 +80,6 @@ def exponential_backoff_retry(fn, excs=(Exception,), name=None, count=10,
       sleep_time *= 2
 
 
-class Lockfile(object):
-  """Class to represent a cross-platform process-specific lockfile."""
-
-  def __init__(self, path, timeout=0):
-    self.path = os.path.abspath(path)
-    self.timeout = timeout
-    self.lockfile = self.path + ".lock"
-    self.pid = os.getpid()
-
-  def _read_pid(self):
-    """Read the pid stored in the lockfile.
-
-    Note: This method is potentially racy. By the time it returns the lockfile
-    may have been unlocked, removed, or stolen by some other process.
-    """
-    try:
-      with open(self.lockfile, 'r') as f:
-        pid = int(f.readline().strip())
-    except (IOError, ValueError):
-      pid = None
-    return pid
-
-  def _make_lockfile(self):
-    """Safely creates a lockfile containing the current pid."""
-    open_flags = (os.O_CREAT | os.O_EXCL | os.O_WRONLY)
-    fd = os.open(self.lockfile, open_flags, 0o644)
-    f = os.fdopen(fd, 'w')
-    print(self.pid, file=f)
-    f.close()
-
-  def _remove_lockfile(self):
-    """Delete the lockfile. Complains (implicitly) if it doesn't exist.
-
-    See gclient_utils.py:rmtree docstring for more explanation on the
-    windows case.
-    """
-    if sys.platform == 'win32':
-      lockfile = os.path.normcase(self.lockfile)
-
-      def delete():
-        exitcode = subprocess.call(['cmd.exe', '/c',
-                                    'del', '/f', '/q', lockfile])
-        if exitcode != 0:
-          raise LockError('Failed to remove lock: %s' % (lockfile,))
-      exponential_backoff_retry(
-          delete,
-          excs=(LockError,),
-          name='del [%s]' % (lockfile,))
-    else:
-      os.remove(self.lockfile)
-
-  def lock(self):
-    """Acquire the lock.
-
-    This will block with a deadline of self.timeout seconds.
-    """
-    elapsed = 0
-    while True:
-      try:
-        self._make_lockfile()
-        return
-      except OSError as e:
-        if elapsed < self.timeout:
-          sleep_time = max(10, min(3, self.timeout - elapsed))
-          logging.info('Could not create git cache lockfile; '
-                       'will retry after sleep(%d).', sleep_time);
-          elapsed += sleep_time
-          time.sleep(sleep_time)
-          continue
-        if e.errno == errno.EEXIST:
-          raise LockError("%s is already locked" % self.path)
-        else:
-          raise LockError("Failed to create %s (err %s)" % (self.path, e.errno))
-
-  def unlock(self):
-    """Release the lock."""
-    try:
-      if not self.is_locked():
-        raise LockError("%s is not locked" % self.path)
-      if not self.i_am_locking():
-        raise LockError("%s is locked, but not by me" % self.path)
-      self._remove_lockfile()
-    except WinErr:
-      # Windows is unreliable when it comes to file locking.  YMMV.
-      pass
-
-  def break_lock(self):
-    """Remove the lock, even if it was created by someone else."""
-    try:
-      self._remove_lockfile()
-      return True
-    except OSError as exc:
-      if exc.errno == errno.ENOENT:
-        return False
-      else:
-        raise
-
-  def is_locked(self):
-    """Test if the file is locked by anyone.
-
-    Note: This method is potentially racy. By the time it returns the lockfile
-    may have been unlocked, removed, or stolen by some other process.
-    """
-    return os.path.exists(self.lockfile)
-
-  def i_am_locking(self):
-    """Test if the file is locked by this process."""
-    return self.is_locked() and self.pid == self._read_pid()
-
-
 class Mirror(object):
 
   git_exe = 'git.bat' if sys.platform.startswith('win') else 'git'
@@ -251,13 +137,13 @@ class Mirror(object):
     u = urlparse.urlparse(self.url)
     if u.netloc == 'chromium.googlesource.com':
       return 'chromium-git-cache'
-    # TODO(tandrii): delete once LUCI migration is completed.
-    # Only public hosts will be supported going forward.
-    elif u.netloc == 'chrome-internal.googlesource.com':
-      return 'chrome-git-cache'
     # Not recognized.
     return None
 
+  @property
+  def _gs_path(self):
+    return 'gs://%s/v2/%s' % (self.bootstrap_bucket, self.basedir)
+
   @classmethod
   def FromPath(cls, path):
     return cls(cls.CacheDirToUrl(path))
@@ -265,6 +151,11 @@ class Mirror(object):
   @staticmethod
   def UrlToCacheDir(url):
     """Convert a git url to a normalized form for the cache dir path."""
+    if os.path.isdir(url):
+      # Ignore the drive letter in Windows
+      url = os.path.splitdrive(url)[1]
+      return url.replace('-', '--').replace(os.sep, '-')
+
     parsed = urlparse.urlparse(url)
     norm_url = parsed.netloc + parsed.path
     if norm_url.endswith('.git'):
@@ -294,7 +185,7 @@ class Mirror(object):
           cachepath = subprocess.check_output(
               [cls.git_exe, 'config'] +
               cls._GIT_CONFIG_LOCATION +
-              ['cache.cachepath']).strip()
+              ['cache.cachepath']).decode('utf-8', 'ignore').strip()
         except subprocess.CalledProcessError:
           cachepath = os.environ.get('GIT_CACHE_PATH', cls.UNSET_CACHEPATH)
         setattr(cls, 'cachepath', cachepath)
@@ -305,6 +196,23 @@ class Mirror(object):
                            '$GIT_CACHE_PATH is set.')
       return ret
 
+  @staticmethod
+  def _GetMostRecentCacheDirectory(ls_out_set):
+    ready_file_pattern = re.compile(r'.*/(\d+).ready$')
+    ready_dirs = []
+
+    for name in ls_out_set:
+      m = ready_file_pattern.match(name)
+      # Given <path>/<number>.ready,
+      # we are interested in <path>/<number> directory
+      if m and (name[:-len('.ready')] + '/') in ls_out_set:
+        ready_dirs.append((int(m.group(1)), name[:-len('.ready')]))
+
+    if not ready_dirs:
+      return None
+
+    return max(ready_dirs)[1]
+
   def Rename(self, src, dst):
     # This is somehow racy on Windows.
     # Catching OSError because WindowsError isn't portable and
@@ -372,87 +280,42 @@ class Mirror(object):
     """
     if not self.bootstrap_bucket:
       return False
-    python_fallback = (
-        (sys.platform.startswith('win') and
-          not gclient_utils.FindExecutable('7z')) or
-        (not gclient_utils.FindExecutable('unzip')) or
-        ('ZIP64_SUPPORT' not in subprocess.check_output(["unzip", "-v"]))
-    )
-
-    gs_folder = 'gs://%s/%s' % (self.bootstrap_bucket, self.basedir)
+
     gsutil = Gsutil(self.gsutil_exe, boto_path=None)
-    # Get the most recent version of the zipfile.
-    _, ls_out, ls_err = gsutil.check_call('ls', gs_folder)
-
-    def compare_filenames(a, b):
-      # |a| and |b| look like gs://.../.../9999.zip. They both have the same
-      # gs://bootstrap_bucket/basedir/ prefix because they come from the same
-      # `gsutil ls`.
-      # This function only compares the numeral parts before .zip.
-      regex_pattern = r'/(\d+)\.zip$'
-      match_a = re.search(regex_pattern, a)
-      match_b = re.search(regex_pattern, b)
-      if (match_a is not None) and (match_b is not None):
-        num_a = int(match_a.group(1))
-        num_b = int(match_b.group(1))
-        return cmp(num_a, num_b)
-      # If it doesn't match the format, fallback to string comparison.
-      return cmp(a, b)
-
-    ls_out_sorted = sorted(ls_out.splitlines(), cmp=compare_filenames)
-    if not ls_out_sorted:
-      # This repo is not on Google Storage.
+
+    # Get the most recent version of the directory.
+    # This is determined from the most recent version of a .ready file.
+    # The .ready file is only uploaded when an entire directory has been
+    # uploaded to GS.
+    _, ls_out, ls_err = gsutil.check_call('ls', self._gs_path)
+    ls_out_set = set(ls_out.strip().splitlines())
+    latest_dir = self._GetMostRecentCacheDirectory(ls_out_set)
+
+    if not latest_dir:
       self.print('No bootstrap file for %s found in %s, stderr:\n  %s' %
                  (self.mirror_path, self.bootstrap_bucket,
-                  '  '.join((ls_err or '').splitlines(True))))
+                '  '.join((ls_err or '').splitlines(True))))
       return False
-    latest_checkout = ls_out_sorted[-1]
 
-    # Download zip file to a temporary directory.
     try:
+      # create new temporary directory locally
       tempdir = tempfile.mkdtemp(prefix='_cache_tmp', dir=self.GetCachePath())
-      self.print('Downloading %s' % latest_checkout)
+      self.RunGit(['init', '--bare'], cwd=tempdir)
+      self.print('Downloading files in %s/* into %s.' %
+                 (latest_dir, tempdir))
       with self.print_duration_of('download'):
-        code = gsutil.call('cp', latest_checkout, tempdir)
+        code = gsutil.call('-m', 'cp', '-r', latest_dir + "/*",
+                           tempdir)
       if code:
         return False
-      filename = os.path.join(tempdir, latest_checkout.split('/')[-1])
-
-      # Unpack the file with 7z on Windows, unzip on linux, or fallback.
-      with self.print_duration_of('unzip'):
-        if not python_fallback:
-          if sys.platform.startswith('win'):
-            cmd = ['7z', 'x', '-o%s' % directory, '-tzip', filename]
-          else:
-            cmd = ['unzip', filename, '-d', directory]
-          retcode = subprocess.call(cmd)
-        else:
-          try:
-            with zipfile.ZipFile(filename, 'r') as f:
-              f.printdir()
-              f.extractall(directory)
-          except Exception as e:
-            self.print('Encountered error: %s' % str(e), file=sys.stderr)
-            retcode = 1
-          else:
-            retcode = 0
-    finally:
-      # Clean up the downloaded zipfile.
-      #
-      # This is somehow racy on Windows.
-      # Catching OSError because WindowsError isn't portable and
-      # pylint complains.
-      exponential_backoff_retry(
-          lambda: gclient_utils.rm_file_or_tree(tempdir),
-          excs=(OSError,),
-          name='rmtree [%s]' % (tempdir,),
-          printerr=self.print)
-
-    if retcode:
-      self.print(
-          'Extracting bootstrap zipfile %s failed.\n'
-          'Resuming normal operations.' % filename)
+    except Exception as e:
+      self.print('Encountered error: %s' % str(e), file=sys.stderr)
+      gclient_utils.rmtree(tempdir)
       return False
+    # delete the old directory
+    if os.path.exists(directory):
+      gclient_utils.rmtree(directory)
+    self.Rename(tempdir, directory)
     return True
 
   def contains_revision(self, revision):
@@ -493,7 +356,7 @@ class Mirror(object):
     try:
       config_fetchspecs = subprocess.check_output(
           [self.git_exe, 'config', '--get-all', 'remote.origin.fetch'],
-          cwd=self.mirror_path)
+          cwd=self.mirror_path).decode('utf-8', 'ignore')
       for fetchspec in config_fetchspecs.splitlines():
         self.fetch_specs.add(self.parse_fetch_spec(fetchspec))
     except subprocess.CalledProcessError:
@@ -502,61 +365,77 @@ class Mirror(object):
                    '%s and "git cache fetch" again.'
                    % os.path.join(self.mirror_path, 'config'))
 
-  def _ensure_bootstrapped(self, depth, bootstrap, force=False):
-    tempdir = None
+  def _ensure_bootstrapped(
+      self, depth, bootstrap, reset_fetch_config, force=False):
     pack_dir = os.path.join(self.mirror_path, 'objects', 'pack')
     pack_files = []
-
     if os.path.isdir(pack_dir):
       pack_files = [f for f in os.listdir(pack_dir) if f.endswith('.pack')]
-      self.print('%s has %d .pack files, re-bootstrapping if >%d' %
-                 (self.mirror_path, len(pack_files), GC_AUTOPACKLIMIT))
+      self.print('%s has %d .pack files, re-bootstrapping if >%d or ==0' %
+                (self.mirror_path, len(pack_files), GC_AUTOPACKLIMIT))
 
     should_bootstrap = (force or
                         not self.exists() or
-                        len(pack_files) > GC_AUTOPACKLIMIT)
-    if should_bootstrap:
-      if self.exists():
-        # Re-bootstrapping an existing mirror; preserve existing fetch spec.
-        self._preserve_fetchspec()
-      tempdir = tempfile.mkdtemp(
-          prefix='_cache_tmp', suffix=self.basedir, dir=self.GetCachePath())
-      bootstrapped = not depth and bootstrap and self.bootstrap_repo(tempdir)
-      if bootstrapped:
-        # Bootstrap succeeded; delete previous cache, if any.
+                        len(pack_files) > GC_AUTOPACKLIMIT or
+                        len(pack_files) == 0)
+
+    if not should_bootstrap:
+      if depth and os.path.exists(os.path.join(self.mirror_path, 'shallow')):
+        logging.warn(
+            'Shallow fetch requested, but repo cache already exists.')
+      return
+
+    if not self.exists():
+      if os.path.exists(self.mirror_path):
+        # If the mirror path exists but self.exists() returns false, we're
+        # in an unexpected state. Nuke the previous mirror directory and
+        # start fresh.
         gclient_utils.rmtree(self.mirror_path)
-      elif not self.exists() or not self.supported_project():
-        # Bootstrap failed due to either
-        # 1. No previous cache
-        # 2. Project doesn't have a bootstrap zip file
+      os.mkdir(self.mirror_path)
+    elif not reset_fetch_config:
+      # Re-bootstrapping an existing mirror; preserve existing fetch spec.
+      self._preserve_fetchspec()
+
+    bootstrapped = (not depth and bootstrap and
+                    self.bootstrap_repo(self.mirror_path))
+
+    if not bootstrapped:
+      if not self.exists() or not self.supported_project():
+        # Bootstrap failed due to:
+        # 1. No previous cache.
+        # 2. Project doesn't have a bootstrap folder.
         # Start with a bare git dir.
-        self.RunGit(['init', '--bare'], cwd=tempdir)
+        self.RunGit(['init', '--bare'], cwd=self.mirror_path)
       else:
         # Bootstrap failed, previous cache exists; warn and continue.
         logging.warn(
             'Git cache has a lot of pack files (%d). Tried to re-bootstrap '
             'but failed. Continuing with non-optimized repository.'
             % len(pack_files))
-        gclient_utils.rmtree(tempdir)
-        tempdir = None
-    else:
-      if depth and os.path.exists(os.path.join(self.mirror_path, 'shallow')):
-        logging.warn(
-            'Shallow fetch requested, but repo cache already exists.')
-    return tempdir
 
-  def _fetch(self, rundir, verbose, depth, reset_fetch_config):
+  def _fetch(self,
+             rundir,
+             verbose,
+             depth,
+             no_fetch_tags,
+             reset_fetch_config,
+             prune=True):
     self.config(rundir, reset_fetch_config)
-    v = []
-    d = []
+
+    fetch_cmd = ['fetch']
     if verbose:
-      v = ['-v', '--progress']
+      fetch_cmd.extend(['-v', '--progress'])
     if depth:
-      d = ['--depth', str(depth)]
-    fetch_cmd = ['fetch'] + v + d + ['origin']
+      fetch_cmd.extend(['--depth', str(depth)])
+    if no_fetch_tags:
+      fetch_cmd.append('--no-tags')
+    if prune:
+      fetch_cmd.append('--prune')
+    fetch_cmd.append('origin')
+
     fetch_specs = subprocess.check_output(
         [self.git_exe, 'config', '--get-all', 'remote.origin.fetch'],
-        cwd=rundir).strip().splitlines()
+        cwd=rundir).decode('utf-8', 'ignore').strip().splitlines()
     for spec in fetch_specs:
       try:
         self.print('Fetching %s' % spec)
@@ -567,61 +446,110 @@ class Mirror(object):
           raise ClobberNeeded()  # Corrupted cache.
         logging.warn('Fetch of %s failed' % spec)
 
-  def populate(self, depth=None, shallow=False, bootstrap=False,
-               verbose=False, ignore_lock=False, lock_timeout=0,
+  def populate(self,
+               depth=None,
+               no_fetch_tags=False,
+               shallow=False,
+               bootstrap=False,
+               verbose=False,
+               lock_timeout=0,
                reset_fetch_config=False):
     assert self.GetCachePath()
     if shallow and not depth:
       depth = 10000
     gclient_utils.safe_makedirs(self.GetCachePath())
 
-    lockfile = Lockfile(self.mirror_path, lock_timeout)
-    if not ignore_lock:
-      lockfile.lock()
+    with lockfile.lock(self.mirror_path, lock_timeout):
+      try:
+        self._ensure_bootstrapped(depth, bootstrap, reset_fetch_config)
+        self._fetch(self.mirror_path, verbose, depth, no_fetch_tags,
+                    reset_fetch_config)
+      except ClobberNeeded:
+        # This is a major failure, we need to clean and force a bootstrap.
+        gclient_utils.rmtree(self.mirror_path)
+        self.print(GIT_CACHE_CORRUPT_MESSAGE)
+        self._ensure_bootstrapped(depth,
+                                  bootstrap,
+                                  reset_fetch_config,
+                                  force=True)
+        self._fetch(self.mirror_path, verbose, depth, no_fetch_tags,
+                    reset_fetch_config)
+
+  def update_bootstrap(self, prune=False, gc_aggressive=False):
+    # The folder is <git number>
+    gen_number = subprocess.check_output(
+        [self.git_exe, 'number', 'master'],
+        cwd=self.mirror_path).decode('utf-8', 'ignore').strip()
+    gsutil = Gsutil(path=self.gsutil_exe, boto_path=None)
 
-    tempdir = None
+    src_name = self.mirror_path
+    dest_prefix = '%s/%s' % (self._gs_path, gen_number)
+
+    # ls_out lists contents in the format: gs://blah/blah/123...
+    _, ls_out, _ = gsutil.check_call('ls', self._gs_path)
+
+    # Check to see if folder already exists in gs
+    ls_out_set = set(ls_out.strip().splitlines())
+    if (dest_prefix + '/' in ls_out_set and
+        dest_prefix + '.ready' in ls_out_set):
+      print('Cache %s already exists.' % dest_prefix)
+      return
+
+    # Run Garbage Collect to compress packfile.
+    gc_args = ['gc', '--prune=all']
+    if gc_aggressive:
+      # The default "gc --aggressive" is often too aggressive for some machines,
+      # since it attempts to create as many threads as there are CPU cores,
+      # while not limiting per-thread memory usage, which puts too much pressure
+      # on RAM on high-core machines, causing them to thrash. Using lower-level
+      # commands gives more control over those settings.
+
+      # This might not be strictly necessary, but it's fast and is normally run
+      # by 'gc --aggressive', so it shouldn't hurt.
+      self.RunGit(['reflog', 'expire', '--all'])
+
+      # These are the default repack settings for 'gc --aggressive'.
+      gc_args = ['repack', '-d', '-l', '-f', '--depth=50', '--window=250', '-A',
+                 '--unpack-unreachable=all']
+      # A 1G memory limit seems to provide comparable pack results as the
+      # default, even for our largest repos, while preventing runaway memory (at
+      # least on current Chromium builders which have about 4G RAM per core).
+      gc_args.append('--window-memory=1g')
+      # NOTE: It might also be possible to avoid thrashing with a larger window
+      # (e.g. "--window-memory=2g") by limiting the number of threads created
+      # (e.g. "--threads=[cores/2]"). Some limited testing didn't show much
+      # difference in outcomes on our current repos, but it might be worth
+      # trying if the repos grow much larger and the packs don't seem to be
+      # getting compressed enough.
+    self.RunGit(gc_args)
+
+    gsutil.call('-m', 'cp', '-r', src_name, dest_prefix)
+
+    # Create .ready file and upload
+    _, ready_file_name =  tempfile.mkstemp(suffix='.ready')
     try:
-      tempdir = self._ensure_bootstrapped(depth, bootstrap)
-      rundir = tempdir or self.mirror_path
-      self._fetch(rundir, verbose, depth, reset_fetch_config)
-    except ClobberNeeded:
-      # This is a major failure, we need to clean and force a bootstrap.
-      gclient_utils.rmtree(rundir)
-      self.print(GIT_CACHE_CORRUPT_MESSAGE)
-      tempdir = self._ensure_bootstrapped(depth, bootstrap, force=True)
-      assert tempdir
-      self._fetch(tempdir, verbose, depth, reset_fetch_config)
+      gsutil.call('cp', ready_file_name, '%s.ready' % (dest_prefix))
     finally:
-      if tempdir:
-        if os.path.exists(self.mirror_path):
-          gclient_utils.rmtree(self.mirror_path)
-        self.Rename(tempdir, self.mirror_path)
-      if not ignore_lock:
-        lockfile.unlock()
-
-  def update_bootstrap(self, prune=False):
-    # The files are named <git number>.zip
-    gen_number = subprocess.check_output(
-        [self.git_exe, 'number', 'master'], cwd=self.mirror_path).strip()
-    # Run Garbage Collect to compress packfile.
-    self.RunGit(['gc', '--prune=all'])
-    # Creating a temp file and then deleting it ensures we can use this name.
-    _, tmp_zipfile = tempfile.mkstemp(suffix='.zip')
-    os.remove(tmp_zipfile)
-    subprocess.call(['zip', '-r', tmp_zipfile, '.'], cwd=self.mirror_path)
-    gsutil = Gsutil(path=self.gsutil_exe, boto_path=None)
-    gs_folder = 'gs://%s/%s' % (self.bootstrap_bucket, self.basedir)
-    dest_name = '%s/%s.zip' % (gs_folder, gen_number)
-    gsutil.call('cp', tmp_zipfile, dest_name)
-    os.remove(tmp_zipfile)
+      os.remove(ready_file_name)
+
+    # remove all other directory/.ready files in the same gs_path
+    # except for the directory/.ready file previously created
+    # which can be used for bootstrapping while the current one is
+    # being uploaded
+    if not prune:
+      return
+    prev_dest_prefix = self._GetMostRecentCacheDirectory(ls_out_set)
+    if not prev_dest_prefix:
+      return
+    for path in ls_out_set:
+      if (path == prev_dest_prefix + '/' or
+          path == prev_dest_prefix + '.ready'):
+        continue
+      if path.endswith('.ready'):
+        gsutil.call('rm', path)
+        continue
+      gsutil.call('-m', 'rm', '-r', path)
 
-    # Remove all other files in the same directory.
-    if prune:
-      _, ls_out, _ = gsutil.check_call('ls', gs_folder)
-      for filename in ls_out.splitlines():
-        if filename == dest_name:
-          continue
-        gsutil.call('rm', filename)
 
   @staticmethod
   def DeleteTmpPackFiles(path):
@@ -638,45 +566,6 @@ class Mirror(object):
       except OSError:
         logging.warn('Unable to delete temporary pack file %s' % f)
 
-  @classmethod
-  def BreakLocks(cls, path):
-    did_unlock = False
-    lf = Lockfile(path)
-    if lf.break_lock():
-      did_unlock = True
-    # Look for lock files that might have been left behind by an interrupted
-    # git process.
-    lf = os.path.join(path, 'config.lock')
-    if os.path.exists(lf):
-      os.remove(lf)
-      did_unlock = True
-    cls.DeleteTmpPackFiles(path)
-    return did_unlock
-
-  def unlock(self):
-    return self.BreakLocks(self.mirror_path)
-
-  @classmethod
-  def UnlockAll(cls):
-    cachepath = cls.GetCachePath()
-    if not cachepath:
-      return
-    dirlist = os.listdir(cachepath)
-    repo_dirs = set([os.path.join(cachepath, path) for path in dirlist
-                     if os.path.isdir(os.path.join(cachepath, path))])
-    for dirent in dirlist:
-      if dirent.startswith('_cache_tmp') or dirent.startswith('tmp'):
-        gclient_utils.rm_file_or_tree(os.path.join(cachepath, dirent))
-      elif (dirent.endswith('.lock') and
-          os.path.isfile(os.path.join(cachepath, dirent))):
-        repo_dirs.add(os.path.join(cachepath, dirent[:-5]))
-
-    unlocked_repos = []
-    for repo_dir in repo_dirs:
-      if cls.BreakLocks(repo_dir):
-        unlocked_repos.append(repo_dir)
-
-    return unlocked_repos
 
 @subcommand.usage('[url of repo to check for caching]')
 def CMDexists(parser, args):
@@ -700,19 +589,27 @@ def CMDupdate_bootstrap(parser, args):
     print('Sorry, update bootstrap will not work on Windows.', file=sys.stderr)
     return 1
 
+  parser.add_option('--skip-populate', action='store_true',
+                    help='Skips "populate" step if mirror already exists.')
+  parser.add_option('--gc-aggressive', action='store_true',
+                    help='Run aggressive repacking of the repo.')
   parser.add_option('--prune', action='store_true',
-                    help='Prune all other cached zipballs of the same repo.')
+                    help='Prune all other cached bundles of the same repo.')
 
-  # First, we need to ensure the cache is populated.
   populate_args = args[:]
-  populate_args.append('--no-bootstrap')
-  CMDpopulate(parser, populate_args)
-
-  # Get the repo directory.
   options, args = parser.parse_args(args)
   url = args[0]
   mirror = Mirror(url)
-  mirror.update_bootstrap(options.prune)
+  if not options.skip_populate or not mirror.exists():
+    CMDpopulate(parser, populate_args)
+  else:
+    print('Skipped populate step.')
+
+  # Get the repo directory.
+  _, args2 = parser.parse_args(args)
+  url = args2[0]
+  mirror = Mirror(url)
+  mirror.update_bootstrap(options.prune, options.gc_aggressive)
   return 0
 
 
@@ -721,6 +618,11 @@ def CMDpopulate(parser, args):
   """Ensure that the cache has all up-to-date objects for the given repo."""
   parser.add_option('--depth', type='int',
                     help='Only cache DEPTH commits of history')
+  parser.add_option(
+      '--no-fetch-tags',
+      action='store_true',
+      help=('Don\'t fetch tags from the server. This can speed up '
+            'fetch considerably when there are many tags.'))
   parser.add_option('--shallow', '-s', action='store_true',
                     help='Only cache 10000 commits of history')
   parser.add_option('--ref', action='append',
@@ -728,23 +630,31 @@ def CMDpopulate(parser, args):
   parser.add_option('--no_bootstrap', '--no-bootstrap',
                     action='store_true',
                     help='Don\'t bootstrap from Google Storage')
-  parser.add_option('--ignore_locks', '--ignore-locks',
+  parser.add_option('--ignore_locks',
+                    '--ignore-locks',
                     action='store_true',
-                    help='Don\'t try to lock repository')
+                    help='NOOP. This flag will be removed in the future.')
+  parser.add_option('--break-locks',
+                    action='store_true',
+                    help='Break any existing lock instead of just ignoring it')
   parser.add_option('--reset-fetch-config', action='store_true', default=False,
                     help='Reset the fetch config before populating the cache.')
 
   options, args = parser.parse_args(args)
   if not len(args) == 1:
     parser.error('git cache populate only takes exactly one repo url.')
+  if options.ignore_locks:
+    print('ignore_locks is no longer used. Please remove its usage.')
+  if options.break_locks:
+    print('break_locks is no longer used. Please remove its usage.')
   url = args[0]
 
   mirror = Mirror(url, refs=options.ref)
   kwargs = {
+      'no_fetch_tags': options.no_fetch_tags,
       'verbose': options.verbose,
       'shallow': options.shallow,
       'bootstrap': not options.no_bootstrap,
-      'ignore_lock': options.ignore_locks,
       'lock_timeout': options.timeout,
       'reset_fetch_config': options.reset_fetch_config,
   }
@@ -760,6 +670,11 @@ def CMDfetch(parser, args):
   parser.add_option('--no_bootstrap', '--no-bootstrap',
                     action='store_true',
                     help='Don\'t (re)bootstrap from Google Storage')
+  parser.add_option(
+      '--no-fetch-tags',
+      action='store_true',
+      help=('Don\'t fetch tags from the server. This can speed up '
+            'fetch considerably when there are many tags.'))
   options, args = parser.parse_args(args)
 
   # Figure out which remotes to fetch.  This mimics the behavior of regular
@@ -769,16 +684,18 @@ def CMDfetch(parser, args):
   remotes = []
   if options.all:
     assert not args, 'fatal: fetch --all does not take a repository argument'
-    remotes = subprocess.check_output([Mirror.git_exe, 'remote']).splitlines()
+    remotes = subprocess.check_output([Mirror.git_exe, 'remote'])
+    remotes = remotes.decode('utf-8', 'ignore').splitlines()
   elif args:
     remotes = args
   else:
     current_branch = subprocess.check_output(
-        [Mirror.git_exe, 'rev-parse', '--abbrev-ref', 'HEAD']).strip()
+        [Mirror.git_exe, 'rev-parse', '--abbrev-ref', 'HEAD'])
+    current_branch = current_branch.decode('utf-8', 'ignore').strip()
     if current_branch != 'HEAD':
       upstream = subprocess.check_output(
-          [Mirror.git_exe, 'config', 'branch.%s.remote' % current_branch]
-      ).strip()
+          [Mirror.git_exe, 'config', 'branch.%s.remote' % current_branch])
+      upstream = upstream.decode('utf-8', 'ignore').strip()
       if upstream and upstream != '.':
         remotes = [upstream]
   if not remotes:
@@ -786,57 +703,35 @@ def CMDfetch(parser, args):
 
   cachepath = Mirror.GetCachePath()
   git_dir = os.path.abspath(subprocess.check_output(
-      [Mirror.git_exe, 'rev-parse', '--git-dir']))
+      [Mirror.git_exe, 'rev-parse', '--git-dir']).decode('utf-8', 'ignore'))
   git_dir = os.path.abspath(git_dir)
   if git_dir.startswith(cachepath):
     mirror = Mirror.FromPath(git_dir)
     mirror.populate(
-        bootstrap=not options.no_bootstrap, lock_timeout=options.timeout)
+        bootstrap=not options.no_bootstrap,
+        no_fetch_tags=options.no_fetch_tags,
+        lock_timeout=options.timeout)
     return 0
   for remote in remotes:
     remote_url = subprocess.check_output(
-        [Mirror.git_exe, 'config', 'remote.%s.url' % remote]).strip()
+        [Mirror.git_exe, 'config', 'remote.%s.url' % remote])
+    remote_url = remote_url.decode('utf-8', 'ignore').strip()
     if remote_url.startswith(cachepath):
       mirror = Mirror.FromPath(remote_url)
       mirror.print = lambda *args: None
       print('Updating git cache...')
       mirror.populate(
-          bootstrap=not options.no_bootstrap, lock_timeout=options.timeout)
+          bootstrap=not options.no_bootstrap,
+          no_fetch_tags=options.no_fetch_tags,
+          lock_timeout=options.timeout)
     subprocess.check_call([Mirror.git_exe, 'fetch', remote])
   return 0
 
 
-@subcommand.usage('[url of repo to unlock, or -a|--all]')
+@subcommand.usage('do not use - it is a noop.')
 def CMDunlock(parser, args):
-  """Unlock one or all repos if their lock files are still around."""
-  parser.add_option('--force', '-f', action='store_true',
-                    help='Actually perform the action')
-  parser.add_option('--all', '-a', action='store_true',
-                    help='Unlock all repository caches')
-  options, args = parser.parse_args(args)
-  if len(args) > 1 or (len(args) == 0 and not options.all):
-    parser.error('git cache unlock takes exactly one repo url, or --all')
-
-  if not options.force:
-    cachepath = Mirror.GetCachePath()
-    lockfiles = [os.path.join(cachepath, path)
-                 for path in os.listdir(cachepath)
-                 if path.endswith('.lock') and os.path.isfile(path)]
-    parser.error('git cache unlock requires -f|--force to do anything. '
-                 'Refusing to unlock the following repo caches: '
-                 ', '.join(lockfiles))
-
-  unlocked_repos = []
-  if options.all:
-    unlocked_repos.extend(Mirror.UnlockAll())
-  else:
-    m = Mirror(args[0])
-    if m.unlock():
-      unlocked_repos.append(m.mirror_path)
-
-  if unlocked_repos:
-    logging.info('Broke locks on these caches:\n  %s' % '\n  '.join(
-        unlocked_repos))
+  """This command does nothing."""
+  print('This command does nothing and will be removed in the future.')
 
 
 class OptionParser(optparse.OptionParser):