libv8 6.7.288.46.1 → 7.3.492.27.0beta1

data/vendor/depot_tools/ninja

@@ -12,7 +12,7 @@ cat <<-EOF
 No prebuilt ninja binary was found for this system.
 Try building your own binary by doing:
   cd ~
-  git clone https://github.com/martine/ninja.git -b v1.8.2
+  git clone https://github.com/ninja-build/ninja.git -b v1.8.2
   cd ninja && ./configure.py --bootstrap
 Then add ~/ninja/ to your PATH.
 EOF

data/vendor/depot_tools/ninjalog.README.md

@@ -0,0 +1,64 @@
+# Ninja build log collection
+
+[TOC]
+
+## Overview
+
+When chromium developers use autoninja for their build,
+
+e.g.
+```
+$ autoninja -C out/Release chrome
+```
+
+autoninja uploads ninja's build log for Google employees but we don't collect
+logs from external contributors.
+
+We use [this page](https://chromium-build-stats-staging.appspot.com/should-upload)
+to decide whether an autoninja user is a Googler or not. Only people with access
+to Google's internal network can see 'Success'.
+
+Before uploading logs, autoninja shows a message 10 times to warn users that we
+will collect build logs.
+
+autoninja users can also opt in or out by using the following commands:
+
+* `ninjalog_uploader_wrapper.py opt-in`
+* `ninjalog_uploader_wrapper.py opt-out`
+
+## What type of data are collected?
+
+The collected build log contains
+
+* output file of build tasks (e.g. chrome, obj/url/url/url_util.o)
+* hash of build command
+* start and end time of build tasks
+
+See [manual of ninja](https://ninja-build.org/manual.html#ref_log) for more
+details.
+
+In addition to ninja's build log, we send the following data for further
+analysis:
+
+* OS (e.g. Win, Mac or Linux)
+* number of cpu cores of building machine
+* build targets (e.g. chrome, browser_tests)
+* parallelism passed by -j flag
+* following build configs
+  * host\_os, host\_cpu
+  * target\_os, target\_cpu
+  * symbol\_level
+  * use\_goma
+  * is\_debug
+  * is\_component\_build
+
+ We don't collect personally identifiable information
+(e.g. username, ip address).
+
+## Why ninja log is collected? / How the collected logs are used?
+
+We (goma team) collect build logs to find slow build tasks that harm developer's
+productivity. Based on collected stats, we find the place/build tasks where we
+need to focus on. Also we use collected stats to track chrome build performance
+on developer's machine. We'll use this stats to measure how much we can/can't
+improve build performance on developer's machine.

data/vendor/depot_tools/ninjalog_uploader.py

@@ -0,0 +1,232 @@
+#!/usr/bin/env python
+# Copyright 2018 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""
+This is script to upload ninja_log from googler.
+
+Server side implementation is in
+https://cs.chromium.org/chromium/infra/go/src/infra/appengine/chromium_build_stats/
+
+Uploaded ninjalog is stored in BigQuery table having following schema.
+https://cs.chromium.org/chromium/infra/go/src/infra/appengine/chromium_build_stats/ninjaproto/ninjalog.proto
+
+The log will be used to analyze user side build performance.
+"""
+
+import argparse
+import cStringIO
+import gzip
+import json
+import logging
+import multiprocessing
+import os
+import platform
+import socket
+import subprocess
+import sys
+import time
+
+
+ROOT_DIR = os.path.dirname(os.path.abspath(__file__))
+sys.path.insert(0, os.path.join(ROOT_DIR, 'third_party'))
+
+import httplib2
+
+# These build configs affect build performance a lot.
+# TODO(tikuta): Add 'blink_symbol_level', 'closure_compile' and
+#               'use_jumbo_build'.
+WHITELISTED_CONFIGS = (
+    'symbol_level', 'use_goma', 'is_debug', 'is_component_build', 'enable_nacl',
+    'host_os', 'host_cpu', 'target_os', 'target_cpu'
+)
+
+def IsGoogler(server):
+    """Check whether this script run inside corp network."""
+    try:
+        h = httplib2.Http()
+        _, content = h.request('https://'+server+'/should-upload', 'GET')
+        return content == 'Success'
+    except httplib2.HttpLib2Error:
+        return False
+
+def ParseGNArgs(gn_args):
+    """Parse gn_args as json and return config dictionary."""
+    configs = json.loads(gn_args)
+    build_configs = {}
+
+    for config in configs:
+        key = config["name"]
+        if key not in WHITELISTED_CONFIGS:
+            continue
+        if 'current' in config:
+            build_configs[key] = config['current']['value']
+        else:
+            build_configs[key] = config['default']['value']
+
+    return build_configs
+
+def GetBuildTargetFromCommandLine(cmdline):
+    """Get build targets from commandline."""
+
+    # Skip argv0.
+    idx = 1
+
+    # Skipping all args that involve these flags, and taking all remaining args
+    # as targets.
+    onearg_flags = ('-C', '-f', '-j', '-k', '-l', '-d', '-t', '-w')
+    zeroarg_flags = ('--version', '-n', '-v')
+
+    targets = []
+
+    while idx < len(cmdline):
+        if cmdline[idx] in onearg_flags:
+            idx += 2
+            continue
+
+        if (cmdline[idx][:2] in onearg_flags or
+            cmdline[idx] in zeroarg_flags):
+            idx += 1
+            continue
+
+        targets.append(cmdline[idx])
+        idx += 1
+
+    return targets
+
+def GetJflag(cmdline):
+    """Parse cmdline to get flag value for -j"""
+
+    for i in range(len(cmdline)):
+        if (cmdline[i] == '-j' and i + 1 < len(cmdline) and
+            cmdline[i+1].isdigit()):
+            return int(cmdline[i+1])
+
+        if (cmdline[i].startswith('-j') and
+            cmdline[i][len('-j'):].isdigit()):
+            return int(cmdline[i][len('-j'):])
+
+
+def GetMetadata(cmdline, ninjalog):
+    """Get metadata for uploaded ninjalog.
+
+    Returned metadata has schema defined in
+    https://cs.chromium.org?q="type+Metadata+struct+%7B"+file:%5Einfra/go/src/infra/appengine/chromium_build_stats/ninjalog/
+
+    TODO(tikuta): Collect GOMA_* env var.
+    """
+
+    build_dir = os.path.dirname(ninjalog)
+
+    build_configs = {}
+
+    try:
+        args = ['gn', 'args', build_dir, '--list', '--short', '--json']
+        if sys.platform == 'win32':
+            # gn in PATH is bat file in windows environment (except cygwin).
+            args = ['cmd', '/c'] + args
+
+        gn_args = subprocess.check_output(args)
+        build_configs = ParseGNArgs(gn_args)
+    except subprocess.CalledProcessError as e:
+        logging.error("Failed to call gn %s", e)
+        build_configs = {}
+
+    # Stringify config.
+    for k in build_configs:
+        build_configs[k] = str(build_configs[k])
+
+    metadata = {
+        'platform': platform.system(),
+        'cpu_core': multiprocessing.cpu_count(),
+        'build_configs': build_configs,
+        'targets': GetBuildTargetFromCommandLine(cmdline),
+    }
+
+    jflag = GetJflag(cmdline)
+    if jflag is not None:
+        metadata['jobs'] = jflag
+
+    return metadata
+
+def GetNinjalog(cmdline):
+    """GetNinjalog returns the path to ninjalog from cmdline."""
+    # ninjalog is in current working directory by default.
+    ninjalog_dir = '.'
+
+    i = 0
+    while i < len(cmdline):
+        cmd = cmdline[i]
+        i += 1
+        if cmd == '-C' and i < len(cmdline):
+            ninjalog_dir = cmdline[i]
+            i += 1
+            continue
+
+        if cmd.startswith('-C') and len(cmd) > len('-C'):
+            ninjalog_dir = cmd[len('-C'):]
+
+    return os.path.join(ninjalog_dir, '.ninja_log')
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--server',
+                        default='chromium-build-stats.appspot.com',
+                        help='server to upload ninjalog file.')
+    parser.add_argument('--ninjalog', help='ninjalog file to upload.')
+    parser.add_argument('--verbose', action='store_true',
+                        help='Enable verbose logging.')
+    parser.add_argument('--cmdline', required=True, nargs=argparse.REMAINDER,
+                        help='command line args passed to ninja.')
+
+    args = parser.parse_args()
+
+    if args.verbose:
+        logging.basicConfig(level=logging.INFO)
+    else:
+        # Disable logging.
+        logging.disable(logging.CRITICAL)
+
+    if not IsGoogler(args.server):
+        return 0
+
+
+    ninjalog = args.ninjalog or GetNinjalog(args.cmdline)
+    if not os.path.isfile(ninjalog):
+        logging.warn("ninjalog is not found in %s", ninjalog)
+        return 1
+
+    # We assume that each ninja invocation interval takes at least 2 seconds.
+    # This is not to have duplicate entry in server when current build is no-op.
+    if os.stat(ninjalog).st_mtime < time.time() - 2:
+        logging.info("ninjalog is not updated recently %s", ninjalog)
+        return 0
+
+    output = cStringIO.StringIO()
+
+    with open(ninjalog) as f:
+        with gzip.GzipFile(fileobj=output, mode='wb') as g:
+            g.write(f.read())
+            g.write('# end of ninja log\n')
+
+            metadata = GetMetadata(args.cmdline, ninjalog)
+            logging.info('send metadata: %s', json.dumps(metadata))
+            g.write(json.dumps(metadata))
+
+    h = httplib2.Http()
+    resp_headers, content = h.request(
+        'https://'+args.server+'/upload_ninja_log/', 'POST',
+        body=output.getvalue(), headers={'Content-Encoding': 'gzip'})
+
+    if resp_headers.status != 200:
+        logging.warn("unexpected status code for response: %s",
+                     resp_headers.status)
+        return 1
+
+    logging.info('response header: %s', resp_headers)
+    logging.info('response content: %s', content)
+    return 0
+
+if __name__ == '__main__':
+    sys.exit(main())

data/vendor/depot_tools/ninjalog_uploader_wrapper.py

@@ -0,0 +1,116 @@
+#!/usr/bin/env python
+# Copyright 2018 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import os
+import subprocess
+import json
+import sys
+
+import ninjalog_uploader
+
+THIS_DIR = os.path.dirname(__file__)
+UPLOADER = os.path.join(THIS_DIR, 'ninjalog_uploader.py')
+CONFIG = os.path.join(THIS_DIR, 'ninjalog.cfg')
+VERSION = 2
+
+
+def LoadConfig():
+    if os.path.isfile(CONFIG):
+        with open(CONFIG, 'rb') as f:
+            config = json.load(f)
+            if config['version'] == VERSION:
+                config['countdown'] = max(0, config['countdown'] - 1)
+                return config
+
+    return {
+        'is-googler': ninjalog_uploader.IsGoogler(
+            'chromium-build-stats.appspot.com'),
+        'countdown': 10,
+        'version': VERSION,
+    }
+
+
+def SaveConfig(config):
+    with open(CONFIG, 'wb') as f:
+        json.dump(config, f)
+
+
+def ShowMessage(countdown):
+    whitelisted = '\n'.join(['  * %s' % config for config in
+                             ninjalog_uploader.WHITELISTED_CONFIGS])
+    print """
+Your ninjalog will be uploaded to build stats server. The uploaded log will be
+used to analyze user side build performance.
+
+The following information will be uploaded with ninjalog.
+* OS (e.g. Win, Mac or Linux)
+* number of cpu cores of building machine
+* build targets (e.g. chrome, browser_tests)
+* parallelism passed by -j flag
+* following build configs
+%s
+
+Uploading ninjalog will be started after you run autoninja another %d time.
+
+If you don't want to upload ninjalog, please run following command.
+$ %s opt-out
+
+If you want to allow upload ninjalog from next autoninja run, please run the
+following command.
+$ %s opt-in
+
+If you have questions about this, please send mail to infra-dev@chromium.org
+
+You can find a more detailed explanation in
+%s
+
+""" % (whitelisted, countdown, __file__, __file__,
+       os.path.abspath(os.path.join(THIS_DIR, "ninjalog.README.md")))
+
+
+def main():
+    config = LoadConfig()
+
+    if len(sys.argv) == 2 and sys.argv[1] == 'opt-in':
+        config['opt-in'] = True
+        config['countdown'] = 0
+        SaveConfig(config)
+        print('ninjalog upload is opted in.')
+        return 0
+
+    if len(sys.argv) == 2 and sys.argv[1] == 'opt-out':
+        config['opt-in'] = False
+        SaveConfig(config)
+        print('ninjalog upload is opted out.')
+        return 0
+
+    if 'opt-in' in config and not config['opt-in']:
+        # Upload is opted out.
+        return 0
+
+    if not config.get("is-googler", False):
+        # Not googler.
+        return 0
+
+    if config.get("countdown", 0) > 0:
+        # Need to show message.
+        ShowMessage(config["countdown"])
+        # Only save config if something has meaningfully changed.
+        SaveConfig(config)
+        return 0
+
+    if len(sys.argv) == 1:
+        # dry-run for debugging.
+        print("upload ninjalog dry-run")
+        return 0
+
+    # Run upload script without wait.
+    devnull = open(os.devnull, "w")
+    subprocess.Popen([sys.executable, UPLOADER] + sys.argv[1:],
+                     stdout=devnull, stderr=devnull)
+
+
+if __name__ == '__main__':
+    sys.exit(main())

data/vendor/depot_tools/owners.py

@@ -11,19 +11,22 @@ so you may need approval from both an OWNER and a reviewer in many cases.
 
 The syntax of the OWNERS file is, roughly:
 
-lines     := (\s* line? \s* comment? \s* "\n")*
+lines      := (\s* line? \s* comment? \s* "\n")*
 
-line      := directive
-          | "per-file" \s+ glob \s* "=" \s* directive
+line       := directive
+           | "per-file" \s+ glob \s* "=" \s* directive
 
-directive := "set noparent"
-          |  "file:" glob
-          |  email_address
-          |  "*"
+directive  := "set noparent"
+           |  "file:" owner_file
+           |  email_address
+           |  "*"
 
-glob      := [a-zA-Z0-9_-*?]+
+glob       := [a-zA-Z0-9_-*?]+
 
-comment   := "#" [^"\n"]*
+comment    := "#" [^"\n"]*
+
+owner_file := "OWNERS"
+           |  [^"\n"]* "_OWNERS"
 
 Email addresses must follow the foo@bar.com short form (exact syntax given
 in BASIC_EMAIL_REGEXP, below). Filename globs follow the simple unix
@@ -48,7 +51,8 @@ glob.
 If the "file:" directive is used, the referred to OWNERS file will be parsed and
 considered when determining the valid set of OWNERS. If the filename starts with
 "//" it is relative to the root of the repository, otherwise it is relative to
-the current file
+the current file. The referred to file *must* be named OWNERS or end in a suffix
+of _OWNERS.
 
 Examples for all of these combinations can be found in tests/owners_unittest.py.
 """
@@ -145,6 +149,12 @@ class Database(object):
     # File with global status lines for owners.
     self._status_file = None
 
+  def _file_affects_ownership(self, path):
+    """Returns true if the path refers to a file that could affect ownership."""
+    filename = self.os_path.split(path)[-1]
+    return filename == 'OWNERS' or filename.endswith('_OWNERS')
+
+
   def reviewers_for(self, files, author):
     """Returns a suggested set of reviewers that will cover the files.
 
@@ -356,7 +366,7 @@ class Database(object):
     if directive == 'set noparent':
       self._stop_looking.add(owned_paths)
     elif directive.startswith('file:'):
-      include_file = self._resolve_include(directive[5:], owners_path)
+      include_file = self._resolve_include(directive[5:], owners_path, lineno)
       if not include_file:
         raise SyntaxErrorInOwnersFile(owners_path, lineno,
             ('%s does not refer to an existing file.' % directive[5:]))
@@ -376,7 +386,7 @@ class Database(object):
           ('"%s" is not a "set noparent", file include, "*", '
            'or an email address.' % (directive,)))
 
-  def _resolve_include(self, path, start):
+  def _resolve_include(self, path, start, lineno):
     if path.startswith('//'):
       include_path = path[2:]
     else:
@@ -388,6 +398,13 @@ class Database(object):
       return include_path
 
     owners_path = self.os_path.join(self.root, include_path)
+    # Paths included via "file:" must end in OWNERS or _OWNERS. Files that can
+    # affect ownership have a different set of ownership rules, so that users
+    # cannot self-approve changes adding themselves to an OWNERS file.
+    if not self._file_affects_ownership(owners_path):
+      raise SyntaxErrorInOwnersFile(start, lineno, 'file: include must specify '
+                                    'a file named OWNERS or ending in _OWNERS')
+
     if not self.os_path.exists(owners_path):
       return None
 
@@ -416,7 +433,7 @@ class Database(object):
         owners.add(line)
         continue
       if line.startswith('file:'):
-        sub_include_file = self._resolve_include(line[5:], include_file)
+        sub_include_file = self._resolve_include(line[5:], include_file, lineno)
         sub_owners = self._read_just_the_owners(sub_include_file)
         owners.update(sub_owners)
         continue