libv8 6.3.292.48.1 → 6.7.288.46.0

data/vendor/depot_tools/man/src/make_docs.sh

@@ -120,6 +120,12 @@ a
 .
 -1
 j
+/XMLTO_EXTRA
+a
+ --skip-validation
+.
+-1
+j
 /^\$(MAN_HTML):
 a
  asciidoc-override.css

data/vendor/depot_tools/my_activity.py

@@ -14,9 +14,6 @@ Example:
   - my_activity.py -jd to output stats for the week to json with deltas data.
 """
 
-# TODO(vadimsh): This script knows too much about ClientLogin and cookies. It
-# will stop to work on ~20 Apr 2015.
-
 # These services typically only provide a created time and a last modified time
 # for each item for general queries. This is not enough to determine if there
 # was activity in a given time period. So, we first query for all things created
@@ -24,11 +21,15 @@ Example:
 # check those details to determine if there was activity in the given period.
 # This means that query time scales mostly with (today() - begin).
 
+import collections
+import contextlib
 from datetime import datetime
 from datetime import timedelta
 from functools import partial
+import itertools
 import json
 import logging
+from multiprocessing.pool import ThreadPool
 import optparse
 import os
 import subprocess
@@ -112,27 +113,23 @@ gerrit_instances = [
   },
 ]
 
-google_code_projects = [
-  {
-    'name': 'chromium',
+monorail_projects = {
+  'chromium': {
     'shorturl': 'crbug.com',
     'short_url_protocol': 'https',
   },
-  {
-    'name': 'google-breakpad',
-  },
-  {
-    'name': 'gyp',
-  },
-  {
-    'name': 'skia',
-  },
-  {
-    'name': 'pdfium',
+  'google-breakpad': {},
+  'gyp': {},
+  'skia': {},
+  'pdfium': {
     'shorturl': 'crbug.com/pdfium',
     'short_url_protocol': 'https',
   },
-]
+  'v8': {
+    'shorturl': 'crbug.com/v8',
+    'short_url_protocol': 'https',
+  },
+}
 
 def username(email):
   """Keeps the username of an email address."""
@@ -183,8 +180,8 @@ def datetime_from_rietveld(date_string):
     return datetime.strptime(date_string, '%Y-%m-%d %H:%M:%S')
 
 
-def datetime_from_google_code(date_string):
-  return datetime.strptime(date_string, '%Y-%m-%dT%H:%M:%S.%fZ')
+def datetime_from_monorail(date_string):
+  return datetime.strptime(date_string, '%Y-%m-%dT%H:%M:%S')
 
 
 class MyActivity(object):
@@ -196,8 +193,15 @@ class MyActivity(object):
     self.changes = []
     self.reviews = []
     self.issues = []
+    self.referenced_issues = []
     self.check_cookies()
     self.google_code_auth_token = None
+    self.access_errors = set()
+
+  def show_progress(self, how='.'):
+    if sys.stdout.isatty():
+      sys.stdout.write(how)
+      sys.stdout.flush()
 
   # Check the codereview cookie jar to determine which Rietveld instances to
   # authenticate to.
@@ -247,6 +251,7 @@ class MyActivity(object):
         reviewer=reviewer_email,
         modified_after=query_modified_after,
         with_messages=True)
+    self.show_progress()
 
     issues = filter(
         lambda i: (datetime_from_rietveld(i['created']) < self.modified_before),
@@ -279,11 +284,14 @@ class MyActivity(object):
     if description:
       # Handle both "Bug: 99999" and "BUG=99999" bug notations
       # Multiple bugs can be noted on a single line or in multiple ones.
-      matches = re.findall(r'BUG[=:]\s?(((\d+)(,\s?)?)+)', description,
-                           flags=re.IGNORECASE)
+      matches = re.findall(
+          r'BUG[=:]\s?((((?:[a-zA-Z0-9-]+:)?\d+)(,\s?)?)+)', description,
+          flags=re.IGNORECASE)
       if matches:
         for match in matches:
           bugs.extend(match[0].replace(' ', '').split(','))
+        # Add default chromium: prefix if none specified.
+        bugs = [bug if ':' in bug else 'chromium:%s' % bug for bug in bugs]
 
     return bugs
 
@@ -293,6 +301,7 @@ class MyActivity(object):
       patchset_props = remote.get_patchset_properties(
           issue['issue'],
           issue['patchsets'][-1])
+      self.show_progress()
       ret['delta'] = '+%d,-%d' % (
           sum(f['num_added'] for f in patchset_props['files'].itervalues()),
           sum(f['num_removed'] for f in patchset_props['files'].itervalues()))
@@ -327,7 +336,7 @@ class MyActivity(object):
     ret['created'] = datetime_from_rietveld(issue['created'])
     ret['replies'] = self.process_rietveld_replies(issue['messages'])
 
-    ret['bug'] = self.extract_bug_number_from_description(issue)
+    ret['bugs'] = self.extract_bug_number_from_description(issue)
     ret['landed_days_ago'] = issue['landed_days_ago']
 
     return ret
@@ -343,8 +352,7 @@ class MyActivity(object):
       ret.append(r)
     return ret
 
-  @staticmethod
-  def gerrit_changes_over_rest(instance, filters):
+  def gerrit_changes_over_rest(self, instance, filters):
     # Convert the "key:value" filter to a list of (key, value) pairs.
     req = list(f.split(':', 1) for f in filters)
     try:
@@ -354,7 +362,9 @@ class MyActivity(object):
           o_params=['MESSAGES', 'LABELS', 'DETAILED_ACCOUNTS',
                     'CURRENT_REVISION', 'CURRENT_COMMIT']))
     except gerrit_util.GerritError, e:
-      logging.error('Looking up %r: %s', instance['url'], e)
+      error_message = 'Looking up %r: %s' % (instance['url'], e)
+      if error_message not in self.access_errors:
+        self.access_errors.add(error_message)
       return []
 
   def gerrit_search(self, instance, owner=None, reviewer=None):
@@ -364,6 +374,7 @@ class MyActivity(object):
     filters = ['-age:%ss' % max_age, user_filter]
 
     issues = self.gerrit_changes_over_rest(instance, filters)
+    self.show_progress()
     issues = [self.process_gerrit_issue(instance, issue)
               for issue in issues]
 
@@ -399,7 +410,7 @@ class MyActivity(object):
       ret['replies'] = []
     ret['reviewers'] = set(r['author'] for r in ret['replies'])
     ret['reviewers'].discard(ret['author'])
-    ret['bug'] = self.extract_bug_number_from_description(issue)
+    ret['bugs'] = self.extract_bug_number_from_description(issue)
     return ret
 
   @staticmethod
@@ -415,64 +426,114 @@ class MyActivity(object):
       })
     return ret
 
-  def project_hosting_issue_search(self, instance):
+  def monorail_get_auth_http(self):
     auth_config = auth.extract_auth_config_from_options(self.options)
     authenticator = auth.get_authenticator_for_host(
         'bugs.chromium.org', auth_config)
-    http = authenticator.authorize(httplib2.Http())
+    return authenticator.authorize(httplib2.Http())
+
+  def filter_modified_monorail_issue(self, issue):
+    """Precisely checks if an issue has been modified in the time range.
+
+    This fetches all issue comments to check if the issue has been modified in
+    the time range specified by user. This is needed because monorail only
+    allows filtering by last updated and published dates, which is not
+    sufficient to tell whether a given issue has been modified at some specific
+    time range. Any update to the issue is a reported as comment on Monorail.
+
+    Args:
+      issue: Issue dict as returned by monorail_query_issues method. In
+          particular, must have a key 'uid' formatted as 'project:issue_id'.
+
+    Returns:
+      Passed issue if modified, None otherwise.
+    """
+    http = self.monorail_get_auth_http()
+    project, issue_id = issue['uid'].split(':')
     url = ('https://monorail-prod.appspot.com/_ah/api/monorail/v1/projects'
-           '/%s/issues') % instance['name']
-    epoch = datetime.utcfromtimestamp(0)
-    user_str = '%s@chromium.org' % self.user
+           '/%s/issues/%s/comments?maxResults=10000') % (project, issue_id)
+    _, body = http.request(url)
+    self.show_progress()
+    content = json.loads(body)
+    if not content:
+      logging.error('Unable to parse %s response from monorail.', project)
+      return issue
 
-    query_data = urllib.urlencode({
-      'maxResults': 10000,
-      'q': user_str,
-      'publishedMax': '%d' % (self.modified_before - epoch).total_seconds(),
-      'updatedMin': '%d' % (self.modified_after - epoch).total_seconds(),
-    })
+    for item in content.get('items', []):
+      comment_published = datetime_from_monorail(item['published'])
+      if self.filter_modified(comment_published):
+        return issue
+
+    return None
+
+  def monorail_query_issues(self, project, query):
+    http = self.monorail_get_auth_http()
+    url = ('https://monorail-prod.appspot.com/_ah/api/monorail/v1/projects'
+           '/%s/issues') % project
+    query_data = urllib.urlencode(query)
     url = url + '?' + query_data
     _, body = http.request(url)
+    self.show_progress()
     content = json.loads(body)
     if not content:
-      logging.error('Unable to parse %s response from projecthosting.',
-          instance['name'])
+      logging.error('Unable to parse %s response from monorail.', project)
       return []
 
     issues = []
-    if 'items' in content:
-      items = content['items']
-      for item in items:
-        if instance.get('shorturl'):
-          protocol = instance.get('short_url_protocol', 'http')
-          item_url = '%s://%s/%d' % (protocol, instance['shorturl'], item['id'])
-        else:
-          item_url = 'https://bugs.chromium.org/p/%s/issues/detail?id=%d' % (
-              instance['name'], item['id'])
-        issue = {
-          'header': item['title'],
-          'created': dateutil.parser.parse(item['published']),
-          'modified': dateutil.parser.parse(item['updated']),
-          'author': item['author']['name'],
-          'url': item_url,
-          'comments': [],
-          'status': item['status'],
-          'labels': [],
-          'components': []
-        }
-        if 'owner' in item:
-          issue['owner'] = item['owner']['name']
-        else:
-          issue['owner'] = 'None'
-        if issue['owner'] == user_str or issue['author'] == user_str:
-          issues.append(issue)
-        if 'labels' in item:
-          issue['labels'] = item['labels']
-        if 'components' in item:
-          issue['components'] = item['components']
+    project_config = monorail_projects.get(project, {})
+    for item in content.get('items', []):
+      if project_config.get('shorturl'):
+        protocol = project_config.get('short_url_protocol', 'http')
+        item_url = '%s://%s/%d' % (
+            protocol, project_config['shorturl'], item['id'])
+      else:
+        item_url = 'https://bugs.chromium.org/p/%s/issues/detail?id=%d' % (
+            project, item['id'])
+      issue = {
+        'uid': '%s:%s' % (project, item['id']),
+        'header': item['title'],
+        'created': datetime_from_monorail(item['published']),
+        'modified': datetime_from_monorail(item['updated']),
+        'author': item['author']['name'],
+        'url': item_url,
+        'comments': [],
+        'status': item['status'],
+        'labels': [],
+        'components': []
+      }
+      if 'owner' in item:
+        issue['owner'] = item['owner']['name']
+      else:
+        issue['owner'] = 'None'
+      if 'labels' in item:
+        issue['labels'] = item['labels']
+      if 'components' in item:
+        issue['components'] = item['components']
+      issues.append(issue)
 
     return issues
 
+  def monorail_issue_search(self, project):
+    epoch = datetime.utcfromtimestamp(0)
+    user_str = '%s@chromium.org' % self.user
+
+    issues = self.monorail_query_issues(project, {
+      'maxResults': 10000,
+      'q': user_str,
+      'publishedMax': '%d' % (self.modified_before - epoch).total_seconds(),
+      'updatedMin': '%d' % (self.modified_after - epoch).total_seconds(),
+    })
+
+    return [
+        issue for issue in issues
+        if issue['author'] == user_str or issue['owner'] == user_str]
+
+  def monorail_get_issues(self, project, issue_ids):
+    return self.monorail_query_issues(project, {
+      'maxResults': 10000,
+      'q': 'id:%s' % ','.join(issue_ids)
+    })
+
   def print_heading(self, heading):
     print
     print self.options.output_format_heading.format(heading=heading)
@@ -528,8 +589,12 @@ class MyActivity(object):
     optional_values = {
         'created': review['created'].date().isoformat(),
         'modified': review['modified'].date().isoformat(),
+        'status': review['status'],
         'activity': activity,
     }
+    if self.options.deltas:
+      optional_values['delta'] = review['delta']
+
     self.print_generic(self.options.output_format,
                        self.options.output_format_reviews,
                        review['header'],
@@ -588,26 +653,43 @@ class MyActivity(object):
     pass
 
   def get_changes(self):
-    for instance in rietveld_instances:
-      self.changes += self.rietveld_search(instance, owner=self.user)
-
-    for instance in gerrit_instances:
-      self.changes += self.gerrit_search(instance, owner=self.user)
+    num_instances = len(rietveld_instances) + len(gerrit_instances)
+    with contextlib.closing(ThreadPool(num_instances)) as pool:
+      rietveld_changes = pool.map_async(
+          lambda instance: self.rietveld_search(instance, owner=self.user),
+          rietveld_instances)
+      gerrit_changes = pool.map_async(
+          lambda instance: self.gerrit_search(instance, owner=self.user),
+          gerrit_instances)
+      rietveld_changes = itertools.chain.from_iterable(rietveld_changes.get())
+      gerrit_changes = itertools.chain.from_iterable(gerrit_changes.get())
+      self.changes = list(rietveld_changes) + list(gerrit_changes)
 
   def print_changes(self):
     if self.changes:
       self.print_heading('Changes')
       for change in self.changes:
-        self.print_change(change)
+          self.print_change(change)
 
-  def get_reviews(self):
-    for instance in rietveld_instances:
-      self.reviews += self.rietveld_search(instance, reviewer=self.user)
+  def print_access_errors(self):
+    if self.access_errors:
+      logging.error('Access Errors:')
+      for error in self.access_errors:
+        logging.error(error.rstrip())
 
-    for instance in gerrit_instances:
-      reviews = self.gerrit_search(instance, reviewer=self.user)
-      reviews = filter(lambda r: not username(r['owner']) == self.user, reviews)
-      self.reviews += reviews
+  def get_reviews(self):
+    num_instances = len(rietveld_instances) + len(gerrit_instances)
+    with contextlib.closing(ThreadPool(num_instances)) as pool:
+      rietveld_reviews = pool.map_async(
+          lambda instance: self.rietveld_search(instance, reviewer=self.user),
+          rietveld_instances)
+      gerrit_reviews = pool.map_async(
+          lambda instance: self.gerrit_search(instance, reviewer=self.user),
+          gerrit_instances)
+      rietveld_reviews = itertools.chain.from_iterable(rietveld_reviews.get())
+      gerrit_reviews = itertools.chain.from_iterable(gerrit_reviews.get())
+      gerrit_reviews = [r for r in gerrit_reviews if r['owner'] != self.user]
+      self.reviews = list(rietveld_reviews) + list(gerrit_reviews)
 
   def print_reviews(self):
     if self.reviews:
@@ -616,8 +698,38 @@ class MyActivity(object):
         self.print_review(review)
 
   def get_issues(self):
-    for project in google_code_projects:
-      self.issues += self.project_hosting_issue_search(project)
+    with contextlib.closing(ThreadPool(len(monorail_projects))) as pool:
+      monorail_issues = pool.map(
+          self.monorail_issue_search, monorail_projects.keys())
+      monorail_issues = list(itertools.chain.from_iterable(monorail_issues))
+
+    if not monorail_issues:
+      return
+
+    with contextlib.closing(ThreadPool(len(monorail_issues))) as pool:
+      filtered_issues = pool.map(
+          self.filter_modified_monorail_issue, monorail_issues)
+      self.issues = [issue for issue in filtered_issues if issue]
+
+  def get_referenced_issues(self):
+    if not self.issues:
+      self.get_issues()
+
+    if not self.changes:
+      self.get_changes()
+
+    referenced_issue_uids = set(itertools.chain.from_iterable(
+      change['bugs'] for change in self.changes))
+    fetched_issue_uids = set(issue['uid'] for issue in self.issues)
+    missing_issue_uids = referenced_issue_uids - fetched_issue_uids
+
+    missing_issues_by_project = collections.defaultdict(list)
+    for issue_uid in missing_issue_uids:
+      project, issue_id = issue_uid.split(':')
+      missing_issues_by_project[project].append(issue_id)
+
+    for project, issue_ids in missing_issues_by_project.iteritems():
+      self.referenced_issues += self.monorail_get_issues(project, issue_ids)
 
   def print_issues(self):
     if self.issues:
@@ -625,6 +737,51 @@ class MyActivity(object):
       for issue in self.issues:
         self.print_issue(issue)
 
+  def print_changes_by_issue(self, skip_empty_own):
+    if not self.issues or not self.changes:
+      return
+
+    self.print_heading('Changes by referenced issue(s)')
+    issues = {issue['uid']: issue for issue in self.issues}
+    ref_issues = {issue['uid']: issue for issue in self.referenced_issues}
+    changes_by_issue_uid = collections.defaultdict(list)
+    changes_by_ref_issue_uid = collections.defaultdict(list)
+    changes_without_issue = []
+    for change in self.changes:
+      added = False
+      for issue_uid in change['bugs']:
+        if issue_uid in issues:
+          changes_by_issue_uid[issue_uid].append(change)
+          added = True
+        if issue_uid in ref_issues:
+          changes_by_ref_issue_uid[issue_uid].append(change)
+          added = True
+      if not added:
+        changes_without_issue.append(change)
+
+    # Changes referencing own issues.
+    for issue_uid in issues:
+      if changes_by_issue_uid[issue_uid] or not skip_empty_own:
+        self.print_issue(issues[issue_uid])
+      for change in changes_by_issue_uid[issue_uid]:
+        print '',  # this prints one space due to comma, but no newline
+        self.print_change(change)
+
+    # Changes referencing others' issues.
+    for issue_uid in ref_issues:
+      assert changes_by_ref_issue_uid[issue_uid]
+      self.print_issue(ref_issues[issue_uid])
+      for change in changes_by_ref_issue_uid[issue_uid]:
+        print '',  # this prints one space due to comma, but no newline
+        self.print_change(change)
+
+    # Changes referencing no issues.
+    if changes_without_issue:
+      print self.options.output_format_no_url.format(title='Other changes')
+      for change in changes_without_issue:
+        print '',  # this prints one space due to comma, but no newline
+        self.print_change(change)
+
   def print_activity(self):
     self.print_changes()
     self.print_reviews()
@@ -697,7 +854,19 @@ def main():
   parser.add_option(
       '-d', '--deltas',
       action='store_true',
-      help='Fetch deltas for changes (slow).')
+      help='Fetch deltas for changes.')
+  parser.add_option(
+      '--no-referenced-issues',
+      action='store_true',
+      help='Do not fetch issues referenced by owned changes. Useful in '
+           'combination with --changes-by-issue when you only want to list '
+           'issues that have also been modified in the same time period.')
+  parser.add_option(
+      '--skip-own-issues-without-changes',
+      action='store_true',
+      help='Skips listing own issues without changes when showing changes '
+           'grouped by referenced issue(s). See --changes-by-issue for more '
+           'details.')
 
   activity_types_group = optparse.OptionGroup(parser, 'Activity Types',
                                'By default, all activity will be looked up and '
@@ -715,6 +884,9 @@ def main():
       '-r', '--reviews',
       action='store_true',
       help='Show reviews.')
+  activity_types_group.add_option(
+      '--changes-by-issue', action='store_true',
+      help='Show changes grouped by referenced issue(s).')
   parser.add_option_group(activity_types_group)
 
   output_format_group = optparse.OptionGroup(parser, 'Output Format',
@@ -749,6 +921,9 @@ def main():
       '--output-format-heading', metavar='<format>',
       default=u'{heading}:',
       help='Specifies the format to use when printing headings.')
+  output_format_group.add_option(
+      '--output-format-no-url', default='{title}',
+      help='Specifies the format to use when printing activity without url.')
   output_format_group.add_option(
       '-m', '--markdown', action='store_true',
       help='Use markdown-friendly output (overrides --output-format '
@@ -821,19 +996,22 @@ def main():
   if options.markdown:
     options.output_format = ' * [{title}]({url})'
     options.output_format_heading = '### {heading} ###'
+    options.output_format_no_url = ' * {title}'
   logging.info('Searching for activity by %s', options.user)
   logging.info('Using range %s to %s', options.begin, options.end)
 
   my_activity = MyActivity(options)
+  my_activity.show_progress('Loading data')
 
-  if not (options.changes or options.reviews or options.issues):
+  if not (options.changes or options.reviews or options.issues or
+          options.changes_by_issue):
     options.changes = True
     options.issues = True
     options.reviews = True
 
   # First do any required authentication so none of the user interaction has to
   # wait for actual work.
-  if options.changes:
+  if options.changes or options.changes_by_issue:
     my_activity.auth_for_changes()
   if options.reviews:
     my_activity.auth_for_reviews()
@@ -841,15 +1019,21 @@ def main():
   logging.info('Looking up activity.....')
 
   try:
-    if options.changes:
+    if options.changes or options.changes_by_issue:
       my_activity.get_changes()
     if options.reviews:
       my_activity.get_reviews()
-    if options.issues:
+    if options.issues or options.changes_by_issue:
       my_activity.get_issues()
+    if not options.no_referenced_issues:
+      my_activity.get_referenced_issues()
   except auth.AuthenticationError as e:
     logging.error('auth.AuthenticationError: %s', e)
 
+  my_activity.show_progress('\n')
+
+  my_activity.print_access_errors()
+
   output_file = None
   try:
     if options.output:
@@ -862,9 +1046,15 @@ def main():
     if options.json:
       my_activity.dump_json()
     else:
-      my_activity.print_changes()
-      my_activity.print_reviews()
-      my_activity.print_issues()
+      if options.changes:
+        my_activity.print_changes()
+      if options.reviews:
+        my_activity.print_reviews()
+      if options.issues:
+        my_activity.print_issues()
+      if options.changes_by_issue:
+        my_activity.print_changes_by_issue(
+            options.skip_own_issues_without_changes)
   finally:
     if output_file:
       logging.info('Done printing to file.')