libv8 6.7.288.46.1 → 7.3.492.27.0beta1

data/vendor/depot_tools/recipes/recipe_modules/git_cl/api.py

@@ -17,7 +17,7 @@ class GitClApi(recipe_api.RecipeApi):
     my_loc = self.c.repo_location if self.c else None
     with self.m.context(cwd=self.m.context.cwd or my_loc):
       return self.m.step(
-          name, [self.package_repo_resource('git_cl.py'), subcmd] + args,
+          name, [self.repo_resource('git_cl.py'), subcmd] + args,
           **kwargs)
 
   def get_description(self, patch_url=None, codereview=None, **kwargs):

data/vendor/depot_tools/recipes/recipe_modules/git_cl/examples/full.expected/basic.json

@@ -1,7 +1,7 @@
 [
   {
     "cmd": [
-      "RECIPE_PACKAGE_REPO[depot_tools]/git_cl.py",
+      "RECIPE_REPO[depot_tools]/git_cl.py",
       "upload",
       "--message-file",
       "Do the thing foobar\nNow with emoji: \ud83d\ude04"
@@ -10,7 +10,7 @@
   },
   {
     "cmd": [
-      "RECIPE_PACKAGE_REPO[depot_tools]/git_cl.py",
+      "RECIPE_REPO[depot_tools]/git_cl.py",
       "issue"
     ],
     "name": "git_cl issue",
@@ -18,7 +18,7 @@
   },
   {
     "cmd": [
-      "RECIPE_PACKAGE_REPO[depot_tools]/git_cl.py",
+      "RECIPE_REPO[depot_tools]/git_cl.py",
       "description",
       "-d",
       "--rietveld",
@@ -29,7 +29,7 @@
   },
   {
     "cmd": [
-      "RECIPE_PACKAGE_REPO[depot_tools]/git_cl.py",
+      "RECIPE_REPO[depot_tools]/git_cl.py",
       "description",
       "-n",
       "-",
@@ -49,11 +49,11 @@
   },
   {
     "cmd": [
-      "RECIPE_PACKAGE_REPO[depot_tools]/git_cl.py",
+      "RECIPE_REPO[depot_tools]/git_cl.py",
       "description",
       "-d"
     ],
-    "cwd": "[TMP_BASE]/fakerepo_tmp_1",
+    "cwd": "[CLEANUP]/fakerepo_tmp_1",
     "name": "git_cl description",
     "stdout": "/path/to/tmp/"
   },
@@ -66,23 +66,23 @@
   },
   {
     "cmd": [
-      "RECIPE_PACKAGE_REPO[depot_tools]/git_cl.py",
+      "RECIPE_REPO[depot_tools]/git_cl.py",
       "description",
       "-n",
       "-"
     ],
-    "cwd": "[TMP_BASE]/fakerepo_tmp_1",
+    "cwd": "[CLEANUP]/fakerepo_tmp_1",
     "name": "git_cl set description (2)",
     "stdin": "new description woo",
     "stdout": "/path/to/tmp/"
   },
   {
     "cmd": [
-      "RECIPE_PACKAGE_REPO[depot_tools]/git_cl.py",
+      "RECIPE_REPO[depot_tools]/git_cl.py",
       "description",
       "-d"
     ],
-    "cwd": "[TMP_BASE]/fakerepo_tmp_1",
+    "cwd": "[CLEANUP]/fakerepo_tmp_1",
     "name": "git_cl description (2)",
     "stdout": "/path/to/tmp/"
   },
@@ -94,8 +94,7 @@
     "name": "echo (3)"
   },
   {
-    "name": "$result",
-    "recipe_result": null,
-    "status_code": 0
+    "jsonResult": null,
+    "name": "$result"
   }
 ]

data/vendor/depot_tools/recipes/recipe_modules/gitiles/__init__.py

@@ -1,7 +1,12 @@
+# Copyright 2018 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
 DEPS = [
     'recipe_engine/json',
     'recipe_engine/path',
     'recipe_engine/python',
     'recipe_engine/raw_io',
+    'recipe_engine/step',
     'recipe_engine/url',
 ]

data/vendor/depot_tools/recipes/recipe_modules/gitiles/api.py

@@ -1,8 +1,9 @@
-# Copyright 2014 The Chromium Authors. All rights reserved.
+# Copyright 2018 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 import base64
+import urlparse
 
 from recipe_engine import recipe_api
 
@@ -11,21 +12,33 @@ class Gitiles(recipe_api.RecipeApi):
   """Module for polling a git repository using the Gitiles web interface."""
 
   def _fetch(self, url, step_name, fmt, attempts=None, add_json_log=True,
-             log_limit=None, log_start=None, **kwargs):
+             log_limit=None, log_start=None, extract_to=None, **kwargs):
     """Fetches information from Gitiles.
 
     Arguments:
+      fmt (str): one of ('text', 'json', 'archive'). Instructs the underlying
+        gitiles_client tool how to process the HTTP response.
+          * text - implies the response is base64 encoded
+          * json - implies the response is JSON
+          * archive - implies the response is a compressed tarball; requires
+            `extract_to`.
+      extract_to (Path): When fmt=='archive', instructs gitiles_client to
+        extract the archive to this non-existant folder.
       log_limit: for log URLs, limit number of results. None implies 1 page,
         as returned by Gitiles.
       log_start: for log URLs, the start cursor for paging.
       add_json_log: if True, will spill out json into log.
     """
-    assert fmt in ('json', 'text')
+    assert fmt in ('json', 'text', 'archive')
+
     args = [
         '--json-file', self.m.json.output(add_json_log=add_json_log),
         '--url', url,
         '--format', fmt,
     ]
+    if fmt == 'archive':
+      assert extract_to is not None, 'archive format requires extract_to'
+      args.extend(['--extract-to', extract_to])
     if attempts:
       args.extend(['--attempts', attempts])
     if log_limit is not None:
@@ -37,9 +50,8 @@ class Gitiles(recipe_api.RecipeApi):
       args.extend([
           '--accept-statuses',
           ','.join([str(s) for s in accept_statuses])])
-    a = self.m.python(
+    return self.m.python(
         step_name, self.resource('gerrit_client.py'), args, **kwargs)
-    return a
 
   def refs(self, url, step_name='refs', attempts=None):
     """Returns a list of refs in the remote repository."""
@@ -140,3 +152,106 @@ class Gitiles(recipe_api.RecipeApi):
     if step_result.json.output['value'] is None:
       return None
     return base64.b64decode(step_result.json.output['value'])
+
+  def download_archive(self, repository_url, destination,
+                       revision='refs/heads/master'):
+    """Downloads an archive of the repo and extracts it to `destination`.
+
+    If the gitiles server attempts to provide a tarball with paths which escape
+    `destination`, this function will extract all valid files and then
+    raise StepFailure with an attribute `StepFailure.gitiles_skipped_files`
+    containing the names of the files that were skipped.
+
+    Args:
+      repository_url (str): Full URL to the repository
+      destination (Path): Local path to extract the archive to. Must not exist
+        prior to this call.
+      revision (str): The ref or revision in the repo to download. Defaults to
+        'refs/heads/master'.
+    """
+    step_name = 'download %s @ %s' % (repository_url, revision)
+    fetch_url = self.m.url.join(repository_url, '+archive/%s.tgz' % (revision,))
+    step_result = self._fetch(
+      fetch_url,
+      step_name,
+      fmt='archive',
+      add_json_log=False,
+      extract_to=destination,
+      step_test_data=lambda: self.m.json.test_api.output({
+        'extracted': {
+          'filecount': 1337,
+          'bytes': 7192345,
+        },
+      })
+    )
+    self.m.path.mock_add_paths(destination)
+    j = step_result.json.output
+    if j['extracted']['filecount']:
+      stat = j['extracted']
+      step_result.presentation.step_text += (
+        '<br/>extracted %s files - %.02f MB' % (
+          stat['filecount'], stat['bytes'] / (1000.0**2)))
+    if j.get('skipped', {}).get('filecount'):
+      stat = j['skipped']
+      step_result.presentation.step_text += (
+        '<br/>SKIPPED %s files - %.02f MB' % (
+          stat['filecount'], stat['bytes'] / (1000.0**2)))
+      step_result.presentation.logs['skipped files'] = stat['names']
+      step_result.presentation.status = self.m.step.FAILURE
+      ex = self.m.step.StepFailure(step_name)
+      ex.gitiles_skipped_files = stat['names']
+      raise ex
+
+  def parse_repo_url(self, repo_url):
+    """Returns (host, project) pair.
+
+    Returns (None, None) if repo_url is not recognized.
+    """
+    return parse_repo_url(repo_url)
+
+  def unparse_repo_url(self, host, project):
+    """Generates a Gitiles repo URL. See also parse_repo_url."""
+    return unparse_repo_url(host, project)
+
+  def canonicalize_repo_url(self, repo_url):
+    """Returns a canonical form of repo_url. If not recognized, returns as is.
+    """
+    if repo_url:
+      host, project = parse_repo_url(repo_url)
+      if host and project:
+        repo_url = unparse_repo_url(host, project)
+    return repo_url
+
+
+def parse_http_host_and_path(url):
+  # Copied from https://chromium.googlesource.com/infra/luci/recipes-py/+/809e57935211b3fcb802f74a7844d4f36eff6b87/recipe_modules/buildbucket/util.py
+  parsed = urlparse.urlparse(url)
+  if not parsed.scheme:
+    parsed = urlparse.urlparse('https://' + url)
+  if (parsed.scheme in ('http', 'https') and
+      not parsed.params and
+      not parsed.query and
+      not parsed.fragment):
+    return parsed.netloc, parsed.path
+  return None, None
+
+
+def parse_repo_url(repo_url):
+  """Returns (host, project) pair.
+
+  Returns (None, None) if repo_url is not recognized.
+  """
+  # Adapted from https://chromium.googlesource.com/infra/luci/recipes-py/+/809e57935211b3fcb802f74a7844d4f36eff6b87/recipe_modules/buildbucket/util.py
+  host, project = parse_http_host_and_path(repo_url)
+  if not host or not project or '+' in project.split('/'):
+    return None, None
+  project = project.strip('/')
+  if project.startswith('a/'):
+    project = project[len('a/'):]
+  if project.endswith('.git'):
+    project = project[:-len('.git')]
+  return host, project
+
+
+def unparse_repo_url(host, project):
+  return 'https://%s/%s' % (host, project)

data/vendor/depot_tools/recipes/recipe_modules/gitiles/examples/full.expected/basic.json

@@ -548,8 +548,50 @@
     "name": "fetch master:NONEXISTENT"
   },
   {
-    "name": "$result",
-    "recipe_result": null,
-    "status_code": 0
+    "cmd": [
+      "python",
+      "-u",
+      "RECIPE_MODULE[depot_tools::gitiles]/resources/gerrit_client.py",
+      "--json-file",
+      "/path/to/tmp/json",
+      "--url",
+      "https://chromium.googlesource.com/chromium/src/+archive/refs/heads/master.tgz",
+      "--format",
+      "archive",
+      "--extract-to",
+      "[START_DIR]/archive"
+    ],
+    "name": "download https://chromium.googlesource.com/chromium/src @ refs/heads/master",
+    "~followup_annotations": [
+      "@@@STEP_TEXT@<br/>extracted 1337 files - 7.19 MB@@@"
+    ]
+  },
+  {
+    "cmd": [
+      "python",
+      "-u",
+      "RECIPE_MODULE[depot_tools::gitiles]/resources/gerrit_client.py",
+      "--json-file",
+      "/path/to/tmp/json",
+      "--url",
+      "https://chromium.googlesource.com/chromium/src/+archive/refs/heads/master.tgz",
+      "--format",
+      "archive",
+      "--extract-to",
+      "[START_DIR]/archive2"
+    ],
+    "name": "download https://chromium.googlesource.com/chromium/src @ refs/heads/master (2)",
+    "~followup_annotations": [
+      "@@@STEP_TEXT@<br/>extracted 10 files - 0.01 MB<br/>SKIPPED 4 files - 7.19 MB@@@",
+      "@@@STEP_LOG_LINE@skipped files@/root@@@",
+      "@@@STEP_LOG_LINE@skipped files@../relative@@@",
+      "@@@STEP_LOG_LINE@skipped files@sneaky/../../relative@@@",
+      "@@@STEP_LOG_END@skipped files@@@",
+      "@@@STEP_FAILURE@@@"
+    ]
+  },
+  {
+    "jsonResult": null,
+    "name": "$result"
   }
 ]

data/vendor/depot_tools/recipes/recipe_modules/gitiles/examples/full.py

@@ -5,6 +5,8 @@
 DEPS = [
     'gitiles',
     'recipe_engine/json',
+    'recipe_engine/step',
+    'recipe_engine/path',
     'recipe_engine/properties',
 ]
 
@@ -22,6 +24,14 @@ def RunSteps(api):
   data = api.gitiles.download_file(url, 'NONEXISTENT', attempts=1,
                                    accept_statuses=[404])
 
+  api.gitiles.download_archive(url, api.path['start_dir'].join('archive'))
+
+  try:
+    api.gitiles.download_archive(url, api.path['start_dir'].join('archive2'))
+    assert False  # pragma: no cover
+  except api.step.StepFailure as ex:
+    assert '/root' in ex.gitiles_skipped_files
+
 
 def GenTests(api):
   yield (
@@ -65,4 +75,19 @@ def GenTests(api):
           'fetch master:NONEXISTENT',
           api.json.output({'value': None})
       )
+      + api.step_data(
+          ('download https://chromium.googlesource.com/chromium/src @ '
+           'refs/heads/master (2)'),
+        api.json.output({
+          'extracted': {
+            'filecount': 10,
+            'bytes': 14925,
+          },
+          'skipped': {
+            'filecount': 4,
+            'bytes': 7192345,
+            'names': ['/root', '../relative', 'sneaky/../../relative'],
+          },
+        })
+      )
   )

data/vendor/depot_tools/recipes/recipe_modules/gitiles/resources/gerrit_client.py

@@ -15,6 +15,7 @@ import json
 import logging
 import os
 import sys
+import tarfile
 import time
 import urllib
 import urlparse
@@ -100,6 +101,16 @@ def main(arguments):
   parser = create_argparser()
   args = parser.parse_args(arguments)
 
+  if args.extract_to and args.format != "archive":
+    parser.error('--extract-to requires --format=archive')
+  if not args.extract_to and args.format == "archive":
+    parser.error('--format=archive requires --extract-to')
+
+  if args.extract_to:
+    # make sure it is absolute and ends with '/'
+    args.extract_to = os.path.join(os.path.abspath(args.extract_to), '')
+    os.makedirs(args.extract_to)
+
   parsed_url = urlparse.urlparse(args.url)
   if not parsed_url.scheme.startswith('http'):
     parser.error('Invalid URI scheme (expected http or https): %s' % args.url)
@@ -125,11 +136,49 @@ def main(arguments):
   elif args.format == 'text':
     # Text fetching will pack the text into structured JSON.
     def handler(conn):
-      result = ReadHttpResponse(conn, **kwargs).read()
       # Wrap in a structured JSON for export to recipe module.
       return {
-          'value': result or None,
+        'value': ReadHttpResponse(conn, **kwargs).read() or None,
       }
+  elif args.format == 'archive':
+    # Archive fetching hooks result to tarfile extraction. This implementation
+    # is able to do a streaming extraction operation without having to buffer
+    # the entire tarfile.
+    def handler(conn):
+      ret = {
+        'extracted': {
+          'filecount': 0,
+          'bytes': 0,
+        },
+        'skipped': {
+          'filecount': 0,
+          'bytes': 0,
+          'names': [],
+        }
+      }
+      fileobj = ReadHttpResponse(conn, **kwargs)
+      with tarfile.open(mode='r|*', fileobj=fileobj) as tf:
+        # monkeypatch the TarFile object to allow printing messages and
+        # collecting stats for each extracted file. extractall makes a single
+        # linear pass over the tarfile, which is compatible with
+        # ReadHttpResponse; other naive implementations (such as `getmembers`)
+        # do random access over the file and would require buffering the whole
+        # thing (!!).
+        em = tf._extract_member
+        def _extract_member(tarinfo, targetpath):
+          if not os.path.abspath(targetpath).startswith(args.extract_to):
+            print 'Skipping %s' % (tarinfo.name,)
+            ret['skipped']['filecount'] += 1
+            ret['skipped']['bytes'] += tarinfo.size
+            ret['skipped']['names'].append(tarinfo.name)
+            return
+          print 'Extracting %s' % (tarinfo.name,)
+          ret['extracted']['filecount'] += 1
+          ret['extracted']['bytes'] += tarinfo.size
+          return em(tarinfo, targetpath)
+        tf._extract_member = _extract_member
+        tf.extractall(args.extract_to)
+      return ret
 
   if args.log_start:
     query_params['s'] = args.log_start
@@ -158,10 +207,13 @@ def main(arguments):
 def create_argparser():
   parser = argparse.ArgumentParser()
   parser.add_argument(
-      '-j', '--json-file', required=True,
+      '-j', '--json-file',
       help='Path to json file for output.')
   parser.add_argument(
-      '-f', '--format', required=True, choices=('json', 'text'))
+      '--extract-to',
+      help='Local path to extract archive url. Must not exist.')
+  parser.add_argument(
+      '-f', '--format', required=True, choices=('json', 'text', 'archive'))
   parser.add_argument(
       '-u', '--url', required=True,
       help='Url of gitiles. For example, '