libv8 6.7.288.46.1 → 7.3.492.27.0beta1

data/vendor/depot_tools/metrics_utils.py

@@ -0,0 +1,303 @@
+#!/usr/bin/env python
+# Copyright (c) 2018 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+from __future__ import print_function
+
+import re
+import scm
+import subprocess2
+import sys
+
+try:
+  import urlparse
+except ImportError:  # For Py3 compatibility
+  import urllib.parse as urlparse
+
+
+# Current version of metrics recording.
+# When we add new metrics, the version number will be increased, we display the
+# user what has changed, and ask the user to agree again.
+CURRENT_VERSION = 1
+
+APP_URL = 'https://cit-cli-metrics.appspot.com'
+
+def get_notice_countdown_header(countdown):
+  if countdown == 0:
+    yield '     METRICS COLLECTION IS TAKING PLACE'
+  else:
+    yield '  METRICS COLLECTION WILL START IN %d EXECUTIONS' % countdown
+
+def get_notice_version_change_header():
+  yield '       WE ARE COLLECTING ADDITIONAL METRICS'
+  yield ''
+  yield ' Please review the changes and opt-in again.'
+
+def get_notice_footer():
+  yield 'To suppress this message opt in or out using:'
+  yield '$ gclient metrics [--opt-in] [--opt-out]'
+  yield 'For more information please see metrics.README.md'
+  yield 'in your depot_tools checkout or visit'
+  yield 'https://goo.gl/yNpRDV.'
+
+def get_change_notice(version):
+  if version == 0:
+    pass # No changes for version 0
+  elif version == 1:
+    yield 'We want to collect the Git version.'
+    yield 'We want to collect information about the HTTP'
+    yield 'requests that depot_tools makes, and the git and'
+    yield 'cipd commands it executes.'
+    yield ''
+    yield 'We only collect known strings to make sure we'
+    yield 'don\'t record PII.'
+
+
+KNOWN_PROJECT_URLS = {
+  'https://chrome-internal.googlesource.com/chrome/ios_internal',
+  'https://chrome-internal.googlesource.com/infra/infra_internal',
+  'https://chromium.googlesource.com/breakpad/breakpad',
+  'https://chromium.googlesource.com/chromium/src',
+  'https://chromium.googlesource.com/chromium/tools/depot_tools',
+  'https://chromium.googlesource.com/crashpad/crashpad',
+  'https://chromium.googlesource.com/external/gyp',
+  'https://chromium.googlesource.com/external/naclports',
+  'https://chromium.googlesource.com/infra/goma/client',
+  'https://chromium.googlesource.com/infra/infra',
+  'https://chromium.googlesource.com/native_client/',
+  'https://chromium.googlesource.com/syzygy',
+  'https://chromium.googlesource.com/v8/v8',
+  'https://dart.googlesource.com/sdk',
+  'https://pdfium.googlesource.com/pdfium',
+  'https://skia.googlesource.com/buildbot',
+  'https://skia.googlesource.com/skia',
+  'https://webrtc.googlesource.com/src',
+}
+
+KNOWN_HTTP_HOSTS = {
+  'chrome-internal-review.googlesource.com',
+  'chromium-review.googlesource.com',
+  'dart-review.googlesource.com',
+  'eu1-mirror-chromium-review.googlesource.com',
+  'pdfium-review.googlesource.com',
+  'skia-review.googlesource.com',
+  'us1-mirror-chromium-review.googlesource.com',
+  'us2-mirror-chromium-review.googlesource.com',
+  'us3-mirror-chromium-review.googlesource.com',
+  'webrtc-review.googlesource.com',
+}
+
+KNOWN_HTTP_METHODS = {
+  'DELETE',
+  'GET',
+  'PATCH',
+  'POST',
+  'PUT',
+}
+
+KNOWN_HTTP_PATHS = {
+  'accounts':
+      re.compile(r'(/a)?/accounts/.*'),
+  'changes':
+      re.compile(r'(/a)?/changes/([^/]+)?$'),
+  'changes/abandon':
+      re.compile(r'(/a)?/changes/.*/abandon'),
+  'changes/comments':
+      re.compile(r'(/a)?/changes/.*/comments'),
+  'changes/detail':
+      re.compile(r'(/a)?/changes/.*/detail'),
+  'changes/edit':
+      re.compile(r'(/a)?/changes/.*/edit'),
+  'changes/message':
+      re.compile(r'(/a)?/changes/.*/message'),
+  'changes/restore':
+      re.compile(r'(/a)?/changes/.*/restore'),
+  'changes/reviewers':
+      re.compile(r'(/a)?/changes/.*/reviewers/.*'),
+  'changes/revisions/commit':
+      re.compile(r'(/a)?/changes/.*/revisions/.*/commit'),
+  'changes/revisions/review':
+      re.compile(r'(/a)?/changes/.*/revisions/.*/review'),
+  'changes/submit':
+      re.compile(r'(/a)?/changes/.*/submit'),
+  'projects/branches':
+      re.compile(r'(/a)?/projects/.*/branches/.*'),
+}
+
+KNOWN_HTTP_ARGS = {
+  'ALL_REVISIONS',
+  'CURRENT_COMMIT',
+  'CURRENT_REVISION',
+  'DETAILED_ACCOUNTS',
+  'LABELS',
+}
+
+GIT_VERSION_RE = re.compile(
+  r'git version (\d)\.(\d{0,2})\.(\d{0,2})'
+)
+
+KNOWN_SUBCOMMAND_ARGS = {
+  'cc',
+  'hashtag',
+  'l=Auto-Submit+1',
+  'l=Code-Review+1',
+  'l=Code-Review+2',
+  'l=Commit-Queue+1',
+  'l=Commit-Queue+2',
+  'label',
+  'm',
+  'notify=ALL',
+  'notify=NONE',
+  'private',
+  'r',
+  'ready',
+  'topic',
+  'wip'
+}
+
+
+def get_python_version():
+  """Return the python version in the major.minor.micro format."""
+  return '{v.major}.{v.minor}.{v.micro}'.format(v=sys.version_info)
+
+
+def get_git_version():
+  """Return the Git version in the major.minor.micro format."""
+  p = subprocess2.Popen(
+      ['git', '--version'],
+      stdout=subprocess2.PIPE, stderr=subprocess2.PIPE)
+  stdout, _ = p.communicate()
+  match = GIT_VERSION_RE.match(stdout)
+  if not match:
+    return None
+  return '%s.%s.%s' % match.groups()
+
+
+def return_code_from_exception(exception):
+  """Returns the exit code that would result of raising the exception."""
+  if exception is None:
+    return 0
+  if isinstance(exception[1], SystemExit):
+    return exception[1].code
+  return 1
+
+
+def seconds_to_weeks(duration):
+  """Transform a |duration| from seconds to weeks approximately.
+
+  Drops the lowest 19 bits of the integer representation, which ammounts to
+  about 6 days.
+  """
+  return int(duration) >> 19
+
+
+def extract_known_subcommand_args(args):
+  """Extract the known arguments from the passed list of args."""
+  known_args = []
+  for arg in args:
+    if arg in KNOWN_SUBCOMMAND_ARGS:
+      known_args.append(arg)
+    else:
+      arg = arg.split('=')[0]
+      if arg in KNOWN_SUBCOMMAND_ARGS:
+        known_args.append(arg)
+  return sorted(known_args)
+
+
+def extract_http_metrics(request_uri, method, status, response_time):
+  """Extract metrics from the request URI.
+
+  Extracts the host, path, and arguments from the request URI, and returns them
+  along with the method, status and response time.
+
+  The host, method, path and arguments must be in the KNOWN_HTTP_* constants
+  defined above.
+
+  Arguments are the values of the o= url parameter. In Gerrit, additional fields
+  can be obtained by adding o parameters, each option requires more database
+  lookups and slows down the query response time to the client, so we make an
+  effort to collect them.
+
+  The regex defined in KNOWN_HTTP_PATH_RES are checked against the path, and
+  those that match will be returned.
+  """
+  http_metrics = {
+    'status': status,
+    'response_time': response_time,
+  }
+
+  if method in KNOWN_HTTP_METHODS:
+    http_metrics['method'] = method
+
+  parsed_url = urlparse.urlparse(request_uri)
+
+  if parsed_url.netloc in KNOWN_HTTP_HOSTS:
+    http_metrics['host'] = parsed_url.netloc
+
+  for name, path_re in KNOWN_HTTP_PATHS.iteritems():
+    if path_re.match(parsed_url.path):
+      http_metrics['path'] = name
+      break
+
+  parsed_query = urlparse.parse_qs(parsed_url.query)
+
+  # Collect o-parameters from the request.
+  args = [
+    arg for arg in parsed_query.get('o', [])
+    if arg in KNOWN_HTTP_ARGS
+  ]
+  if args:
+    http_metrics['arguments'] = args
+
+  return http_metrics
+
+
+def get_repo_timestamp(path_to_repo):
+  """Get an approximate timestamp for the upstream of |path_to_repo|.
+
+  Returns the top two bits of the timestamp of the HEAD for the upstream of the
+  branch path_to_repo is checked out at.
+  """
+  # Get the upstream for the current branch. If we're not in a branch, fallback
+  # to HEAD.
+  try:
+    upstream = scm.GIT.GetUpstreamBranch(path_to_repo) or 'HEAD'
+  except subprocess2.CalledProcessError:
+    upstream = 'HEAD'
+
+  # Get the timestamp of the HEAD for the upstream of the current branch.
+  p = subprocess2.Popen(
+      ['git', '-C', path_to_repo, 'log', '-n1', upstream, '--format=%at'],
+      stdout=subprocess2.PIPE, stderr=subprocess2.PIPE)
+  stdout, _ = p.communicate()
+
+  # If there was an error, give up.
+  if p.returncode != 0:
+    return None
+
+  # Get the age of the checkout in weeks.
+  return seconds_to_weeks(stdout.strip())
+
+def print_boxed_text(out, min_width, lines):
+  [EW, NS, SE, SW, NE, NW] = list('=|++++')
+  width = max(min_width, max(len(line) for line in lines))
+  out(SE + EW * (width + 2) + SW + '\n')
+  for line in lines:
+    out('%s %-*s %s\n' % (NS, width, line, NS))
+  out(NE + EW * (width + 2) + NW + '\n')
+
+def print_notice(countdown):
+  """Print a notice to let the user know the status of metrics collection."""
+  lines = list(get_notice_countdown_header(countdown))
+  lines.append('')
+  lines += list(get_notice_footer())
+  print_boxed_text(sys.stderr.write, 49, lines)
+
+def print_version_change(config_version):
+  """Print a notice to let the user know we are collecting more metrics."""
+  lines = list(get_notice_version_change_header())
+  for version in xrange(config_version + 1, CURRENT_VERSION + 1):
+    lines.append('')
+    lines += list(get_change_notice(version))
+  print_boxed_text(sys.stderr.write, 49, lines)

data/vendor/depot_tools/my_activity.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env vpython
 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
@@ -21,6 +21,17 @@ Example:
 # check those details to determine if there was activity in the given period.
 # This means that query time scales mostly with (today() - begin).
 
+# [VPYTHON:BEGIN]
+# wheel: <
+#   name: "infra/python/wheels/python-dateutil-py2_py3"
+#   version: "version:2.7.3"
+# >
+# wheel: <
+#   name: "infra/python/wheels/six-py2_py3"
+#   version: "version:1.10.0"
+# >
+# [VPYTHON:END]
+
 import collections
 import contextlib
 from datetime import datetime
@@ -96,9 +107,7 @@ rietveld_instances = [
 
 gerrit_instances = [
   {
-    'url': 'chromium-review.googlesource.com',
-    'shorturl': 'crrev.com/c',
-    'short_url_protocol': 'https',
+    'url': 'android-review.googlesource.com',
   },
   {
     'url': 'chrome-internal-review.googlesource.com',
@@ -106,11 +115,16 @@ gerrit_instances = [
     'short_url_protocol': 'https',
   },
   {
-    'url': 'android-review.googlesource.com',
+    'url': 'chromium-review.googlesource.com',
+    'shorturl': 'crrev.com/c',
+    'short_url_protocol': 'https',
   },
   {
     'url': 'pdfium-review.googlesource.com',
   },
+  {
+    'url': 'skia-review.googlesource.com',
+  },
 ]
 
 monorail_projects = {
@@ -269,7 +283,7 @@ class MyActivity(object):
 
     return issues
 
-  def extract_bug_number_from_description(self, issue):
+  def extract_bug_numbers_from_description(self, issue):
     description = None
 
     if 'description' in issue:
@@ -293,7 +307,7 @@ class MyActivity(object):
         # Add default chromium: prefix if none specified.
         bugs = [bug if ':' in bug else 'chromium:%s' % bug for bug in bugs]
 
-    return bugs
+    return sorted(set(bugs))
 
   def process_rietveld_issue(self, remote, instance, issue):
     ret = {}
@@ -336,7 +350,7 @@ class MyActivity(object):
     ret['created'] = datetime_from_rietveld(issue['created'])
     ret['replies'] = self.process_rietveld_replies(issue['messages'])
 
-    ret['bugs'] = self.extract_bug_number_from_description(issue)
+    ret['bugs'] = self.extract_bug_numbers_from_description(issue)
     ret['landed_days_ago'] = issue['landed_days_ago']
 
     return ret
@@ -369,16 +383,22 @@ class MyActivity(object):
 
   def gerrit_search(self, instance, owner=None, reviewer=None):
     max_age = datetime.today() - self.modified_after
-    max_age = max_age.days * 24 * 3600 + max_age.seconds
-    user_filter = 'owner:%s' % owner if owner else 'reviewer:%s' % reviewer
-    filters = ['-age:%ss' % max_age, user_filter]
+    filters = ['-age:%ss' % (max_age.days * 24 * 3600 + max_age.seconds)]
+    if owner:
+      assert not reviewer
+      filters.append('owner:%s' % owner)
+    else:
+      filters.extend(('-owner:%s' % reviewer, 'reviewer:%s' % reviewer))
+    # TODO(cjhopman): Should abandoned changes be filtered out when
+    # merged_only is not enabled?
+    if self.options.merged_only:
+      filters.append('status:merged')
 
     issues = self.gerrit_changes_over_rest(instance, filters)
     self.show_progress()
     issues = [self.process_gerrit_issue(instance, issue)
               for issue in issues]
 
-    # TODO(cjhopman): should we filter abandoned changes?
     issues = filter(self.filter_issue, issues)
     issues = sorted(issues, key=lambda i: i['modified'], reverse=True)
 
@@ -400,7 +420,7 @@ class MyActivity(object):
     ret['review_url'] = '%s://%s/%s' % (protocol, url, issue['_number'])
 
     ret['header'] = issue['subject']
-    ret['owner'] = issue['owner']['email']
+    ret['owner'] = issue['owner'].get('email', '')
     ret['author'] = ret['owner']
     ret['created'] = datetime_from_gerrit(issue['created'])
     ret['modified'] = datetime_from_gerrit(issue['updated'])
@@ -410,7 +430,7 @@ class MyActivity(object):
       ret['replies'] = []
     ret['reviewers'] = set(r['author'] for r in ret['replies'])
     ret['reviewers'].discard(ret['author'])
-    ret['bugs'] = self.extract_bug_number_from_description(issue)
+    ret['bugs'] = self.extract_bug_numbers_from_description(issue)
     return ret
 
   @staticmethod
@@ -430,7 +450,10 @@ class MyActivity(object):
     auth_config = auth.extract_auth_config_from_options(self.options)
     authenticator = auth.get_authenticator_for_host(
         'bugs.chromium.org', auth_config)
-    return authenticator.authorize(httplib2.Http())
+    # Manually use a long timeout (10m); for some users who have a
+    # long history on the issue tracker, whatever the default timeout
+    # is is reached.
+    return authenticator.authorize(httplib2.Http(timeout=600))
 
   def filter_modified_monorail_issue(self, issue):
     """Precisely checks if an issue has been modified in the time range.
@@ -515,6 +538,7 @@ class MyActivity(object):
 
   def monorail_issue_search(self, project):
     epoch = datetime.utcfromtimestamp(0)
+    # TODO(tandrii): support non-chromium email, too.
     user_str = '%s@chromium.org' % self.user
 
     issues = self.monorail_query_issues(project, {
@@ -524,6 +548,13 @@ class MyActivity(object):
       'updatedMin': '%d' % (self.modified_after - epoch).total_seconds(),
     })
 
+    if self.options.completed_issues:
+      return [
+          issue for issue in issues
+          if (self.match(issue['owner']) and
+              issue['status'].lower() in ('verified', 'fixed'))
+      ]
+
     return [
         issue for issue in issues
         if issue['author'] == user_str or issue['owner'] == user_str]
@@ -564,6 +595,8 @@ class MyActivity(object):
                        change['header'],
                        change['review_url'],
                        change['author'],
+                       change['created'],
+                       change['modified'],
                        optional_values)
 
   def print_issue(self, issue):
@@ -578,6 +611,8 @@ class MyActivity(object):
                        issue['header'],
                        issue['url'],
                        issue['author'],
+                       issue['created'],
+                       issue['modified'],
                        optional_values)
 
   def print_review(self, review):
@@ -600,11 +635,13 @@ class MyActivity(object):
                        review['header'],
                        review['review_url'],
                        review['author'],
+                       review['created'],
+                       review['modified'],
                        optional_values)
 
   @staticmethod
   def print_generic(default_fmt, specific_fmt,
-                    title, url, author,
+                    title, url, author, created, modified,
                     optional_values=None):
     output_format = specific_fmt if specific_fmt is not None else default_fmt
     output_format = unicode(output_format)
@@ -612,6 +649,8 @@ class MyActivity(object):
         'title': title,
         'url': url,
         'author': author,
+        'created': created,
+        'modified': modified,
     }
     if optional_values is not None:
       values.update(optional_values)
@@ -669,7 +708,7 @@ class MyActivity(object):
     if self.changes:
       self.print_heading('Changes')
       for change in self.changes:
-          self.print_change(change)
+        self.print_change(change)
 
   def print_access_errors(self):
     if self.access_errors:
@@ -688,7 +727,6 @@ class MyActivity(object):
           gerrit_instances)
       rietveld_reviews = itertools.chain.from_iterable(rietveld_reviews.get())
       gerrit_reviews = itertools.chain.from_iterable(gerrit_reviews.get())
-      gerrit_reviews = [r for r in gerrit_reviews if r['owner'] != self.user]
       self.reviews = list(rietveld_reviews) + list(gerrit_reviews)
 
   def print_reviews(self):
@@ -763,9 +801,12 @@ class MyActivity(object):
     for issue_uid in issues:
       if changes_by_issue_uid[issue_uid] or not skip_empty_own:
         self.print_issue(issues[issue_uid])
-      for change in changes_by_issue_uid[issue_uid]:
-        print '',  # this prints one space due to comma, but no newline
-        self.print_change(change)
+      if changes_by_issue_uid[issue_uid]:
+        print
+        for change in changes_by_issue_uid[issue_uid]:
+          print '   ',  # this prints one space due to comma, but no newline
+          self.print_change(change)
+        print
 
     # Changes referencing others' issues.
     for issue_uid in ref_issues:
@@ -824,7 +865,8 @@ def main():
   parser = optparse.OptionParser(description=sys.modules[__name__].__doc__)
   parser.add_option(
       '-u', '--user', metavar='<email>',
-      default=os.environ.get('USER'),
+      # Look for USER and USERNAME (Windows) environment variables.
+      default=os.environ.get('USER', os.environ.get('USERNAME')),
       help='Filter on user, default=%default')
   parser.add_option(
       '-b', '--begin', metavar='<date>',
@@ -896,9 +938,10 @@ def main():
                               'individually for each activity type. The format '
                               'is defined as documented for '
                               'string.format(...). The variables available for '
-                              'all activity types are url, title and author. '
-                              'Format options for specific activity types will '
-                              'override the generic format.')
+                              'all activity types are url, title, author, '
+                              'created and modified. Format options for '
+                              'specific activity types will override the '
+                              'generic format.')
   output_format_group.add_option(
       '-f', '--output-format', metavar='<format>',
       default=u'{url} {title}',
@@ -949,6 +992,19 @@ def main():
       const=logging.ERROR,
       help='Suppress non-error messages.'
   )
+  parser.add_option(
+      '-M', '--merged-only',
+      action='store_true',
+      dest='merged_only',
+      default=False,
+      help='Shows only changes that have been merged.')
+  parser.add_option(
+      '-C', '--completed-issues',
+      action='store_true',
+      dest='completed_issues',
+      default=False,
+      help='Shows only monorail issues that have completed (Fixed|Verified) '
+           'by the user.')
   parser.add_option(
       '-o', '--output', metavar='<file>',
       help='Where to output the results. By default prints to stdout.')
@@ -962,7 +1018,7 @@ def main():
   if args:
     parser.error('Args unsupported')
   if not options.user:
-    parser.error('USER is not set, please use -u')
+    parser.error('USER/USERNAME is not set, please use -u')
   options.user = username(options.user)
 
   logging.basicConfig(level=options.verbosity)
@@ -992,11 +1048,17 @@ def main():
     else:
       end = datetime.today()
   options.begin, options.end = begin, end
+  if begin >= end:
+    # The queries fail in peculiar ways when the begin date is in the future.
+    # Give a descriptive error message instead.
+    logging.error('Start date (%s) is the same or later than end date (%s)' %
+                  (begin, end))
+    return 1
 
   if options.markdown:
-    options.output_format = ' * [{title}]({url})'
-    options.output_format_heading = '### {heading} ###'
-    options.output_format_no_url = ' * {title}'
+    options.output_format_heading = '### {heading}\n'
+    options.output_format = '  * [{title}]({url})'
+    options.output_format_no_url = '  * {title}'
   logging.info('Searching for activity by %s', options.user)
   logging.info('Using range %s to %s', options.begin, options.end)