libv8 7.3.492.27.1 → 8.4.255.0

data/vendor/depot_tools/presubmit_canned_checks.py

@@ -4,37 +4,44 @@
 
 """Generic presubmit checks that can be reused by other presubmit checks."""
 
+from __future__ import print_function
+
 import os as _os
 _HERE = _os.path.dirname(_os.path.abspath(__file__))
 
+# These filters will be disabled if callers do not explicitly supply a
+# (possibly-empty) list.  Ideally over time this list could be driven to zero.
+# TODO(pkasting): If some of these look like "should never enable", move them
+# to OFF_UNLESS_MANUALLY_ENABLED_LINT_FILTERS.
+#
 # Justifications for each filter:
 #
-# - build/include       : Too many; fix in the future.
-# - build/include_order : Not happening; #ifdefed includes.
-# - build/namespace     : I'm surprised by how often we violate this rule.
-# - readability/casting : Mistakes a whole bunch of function pointer.
+# - build/include       : Too many; fix in the future
+#                         TODO(pkasting): Try enabling subcategories
+# - build/include_order : Not happening; #ifdefed includes
+# - build/namespaces    : TODO(pkasting): Try re-enabling
+# - readability/casting : Mistakes a whole bunch of function pointers
 # - runtime/int         : Can be fixed long term; volume of errors too high
-# - runtime/virtual     : Broken now, but can be fixed in the future?
-# - whitespace/braces   : We have a lot of explicit scoping in chrome code.
-DEFAULT_LINT_FILTERS = [
+# - whitespace/braces   : We have a lot of explicit scoping in chrome code
+OFF_BY_DEFAULT_LINT_FILTERS = [
   '-build/include',
   '-build/include_order',
-  '-build/namespace',
+  '-build/namespaces',
   '-readability/casting',
   '-runtime/int',
-  '-runtime/virtual',
   '-whitespace/braces',
 ]
 
-# These filters will always be removed, even if the caller specifies a filter
-# set, as they are problematic or broken in some way.
+# These filters will be disabled unless callers explicitly enable them, because
+# they are undesirable in some way.
 #
 # Justifications for each filter:
-# - build/c++11         : Rvalue ref checks are unreliable (false positives),
-#                         include file and feature blacklists are
-#                         google3-specific.
-BLACKLIST_LINT_FILTERS = [
+# - build/c++11         : Include file and feature blocklists are
+#                         google3-specific
+# - runtime/references  : No longer banned by Google style guide
+OFF_UNLESS_MANUALLY_ENABLED_LINT_FILTERS = [
   '-build/c++11',
+  '-runtime/references',
 ]
 
 ### Description checks
@@ -47,6 +54,22 @@ def CheckChangeHasBugField(input_api, output_api):
     return [output_api.PresubmitNotifyResult(
         'If this change has an associated bug, add Bug: [bug number].')]
 
+def CheckChangeHasNoUnwantedTags(input_api, output_api):
+  UNWANTED_TAGS = {
+      'FIXED': {
+          'why': 'is not supported',
+          'instead': 'Use "Fixed:" instead.'
+      },
+      # TODO: BUG, ISSUE
+  }
+
+  errors = []
+  for tag, desc in UNWANTED_TAGS.items():
+    if tag in input_api.change.tags:
+      subs = tag, desc['why'], desc.get('instead', '')
+      errors.append(('%s= %s. %s' % subs).rstrip())
+
+  return [output_api.PresubmitError('\n'.join(errors))] if errors else []
 
 def CheckDoNotSubmitInDescription(input_api, output_api):
   """Checks that the user didn't add 'DO NOT ''SUBMIT' to the CL description.
@@ -122,8 +145,14 @@ def CheckDoNotSubmitInFiles(input_api, output_api):
   # We want to check every text file, not just source files.
   file_filter = lambda x : x
   keyword = 'DO NOT ''SUBMIT'
-  errors = _FindNewViolationsOfRule(lambda _, line : keyword not in line,
-                                    input_api, file_filter)
+  def DoNotSubmitRule(extension, line):
+    try:
+      return keyword not in line
+    # Fallback to True for non-text content
+    except UnicodeDecodeError:
+      return True
+
+  errors = _FindNewViolationsOfRule(DoNotSubmitRule, input_api, file_filter)
   text = '\n'.join('Found %s in %s' % (keyword, loc) for loc in errors)
   if text:
     return [output_api.PresubmitError(text)]
@@ -141,9 +170,11 @@ def CheckChangeLintsClean(input_api, output_api, source_file_filter=None,
   # pylint: disable=protected-access
   cpplint._cpplint_state.ResetErrorCounts()
 
-  lint_filters = lint_filters or DEFAULT_LINT_FILTERS
-  lint_filters.extend(BLACKLIST_LINT_FILTERS)
-  cpplint._SetFilters(','.join(lint_filters))
+  filters = OFF_UNLESS_MANUALLY_ENABLED_LINT_FILTERS[:]
+  if lint_filters is None:
+    lint_filters = OFF_BY_DEFAULT_LINT_FILTERS
+  filters.extend(lint_filters)
+  cpplint._SetFilters(','.join(filters))
 
   # We currently are more strict with normal code than unit tests; 4 and 5 are
   # the verbosity level that would normally be passed to cpplint.py through
@@ -227,7 +258,7 @@ def CheckGenderNeutral(input_api, output_api, source_file_filter=None):
   submitted.
   """
   gendered_re = input_api.re.compile(
-      '(^|\s|\(|\[)([Hh]e|[Hh]is|[Hh]ers?|[Hh]im|[Ss]he|[Gg]uys?)\\b')
+      r'(^|\s|\(|\[)([Hh]e|[Hh]is|[Hh]ers?|[Hh]im|[Ss]he|[Gg]uys?)\\b')
 
   errors = []
   for f in input_api.AffectedFiles(include_deletes=False,
@@ -328,7 +359,7 @@ def CheckChangeHasNoTabs(input_api, output_api, source_file_filter=None):
   """Checks that there are no tab characters in any of the text files to be
   submitted.
   """
-  # In addition to the filter, make sure that makefiles are blacklisted.
+  # In addition to the filter, make sure that makefiles are skipped.
   if not source_file_filter:
     # It's the default filter.
     source_file_filter = input_api.FilterSourceFile
@@ -392,7 +423,9 @@ def CheckLongLines(input_api, output_api, maxlen, source_file_filter=None):
   JAVA_FILE_EXTS = ('java',)
   JAVA_EXCEPTIONS = ('import ', 'package ')
   JS_FILE_EXTS = ('js',)
-  JS_EXCEPTIONS = ("GEN('#include",)
+  JS_EXCEPTIONS = ("GEN('#include", 'import ')
+  TS_FILE_EXTS = ('ts',)
+  TS_EXCEPTIONS = ('import ')
   OBJC_FILE_EXTS = ('h', 'm', 'mm')
   OBJC_EXCEPTIONS = ('#define', '#endif', '#if', '#import', '#include',
                      '#pragma')
@@ -404,6 +437,7 @@ def CheckLongLines(input_api, output_api, maxlen, source_file_filter=None):
     (HTML_FILE_EXTS, HTML_EXCEPTIONS),
     (JAVA_FILE_EXTS, JAVA_EXCEPTIONS),
     (JS_FILE_EXTS, JS_EXCEPTIONS),
+    (TS_FILE_EXTS, TS_EXCEPTIONS),
     (OBJC_FILE_EXTS, OBJC_EXCEPTIONS),
     (PY_FILE_EXTS, PY_EXCEPTIONS),
   ]
@@ -552,7 +586,7 @@ def CheckTreeIsOpen(input_api, output_api,
     return []
   try:
     if json_url:
-      connection = input_api.urllib2.urlopen(json_url)
+      connection = input_api.urllib_request.urlopen(json_url)
       status = input_api.json.loads(connection.read())
       connection.close()
       if not status['can_commit_freely']:
@@ -561,7 +595,7 @@ def CheckTreeIsOpen(input_api, output_api,
         return [output_api.PresubmitError(short_text, long_text=long_text)]
     else:
       # TODO(bradnelson): drop this once all users are gone.
-      connection = input_api.urllib2.urlopen(url)
+      connection = input_api.urllib_request.urlopen(url)
       status = connection.read()
       connection.close()
       if input_api.re.match(closed, status):
@@ -574,12 +608,17 @@ def CheckTreeIsOpen(input_api, output_api,
   return []
 
 def GetUnitTestsInDirectory(
-    input_api, output_api, directory, whitelist=None, blacklist=None, env=None):
+    input_api, output_api, directory, files_to_check=None, files_to_skip=None,
+    env=None, run_on_python2=True, run_on_python3=True, whitelist=None,
+    blacklist=None, allowlist=None, blocklist=None):
   """Lists all files in a directory and runs them. Doesn't recurse.
 
-  It's mainly a wrapper for RunUnitTests. Use whitelist and blacklist to filter
+  It's mainly a wrapper for RunUnitTests. Use allowlist and blocklist to filter
   tests accordingly.
   """
+  # TODO(https://crbug.com/1098560): Add warnings before removing bc.
+  files_to_check = files_to_check or allowlist or whitelist
+  files_to_skip = files_to_skip or blocklist or blacklist
   unit_tests = []
   test_path = input_api.os_path.abspath(
       input_api.os_path.join(input_api.PresubmitLocalPath(), directory))
@@ -593,9 +632,9 @@ def GetUnitTestsInDirectory(
     fullpath = input_api.os_path.join(test_path, filename)
     if not input_api.os_path.isfile(fullpath):
       continue
-    if whitelist and not check(filename, whitelist):
+    if files_to_check and not check(filename, files_to_check):
       continue
-    if blacklist and check(filename, blacklist):
+    if files_to_skip and check(filename, files_to_skip):
       continue
     unit_tests.append(input_api.os_path.join(directory, filename))
     to_run += 1
@@ -604,17 +643,27 @@ def GetUnitTestsInDirectory(
   if not to_run:
     return [
         output_api.PresubmitPromptWarning(
-          'Out of %d files, found none that matched w=%r, b=%r in directory %s'
-          % (found, whitelist, blacklist, directory))
+          'Out of %d files, found none that matched c=%r, s=%r in directory %s'
+          % (found, files_to_check, files_to_skip, directory))
     ]
-  return GetUnitTests(input_api, output_api, unit_tests, env)
+  return GetUnitTests(
+      input_api, output_api, unit_tests, env, run_on_python2, run_on_python3)
 
 
-def GetUnitTests(input_api, output_api, unit_tests, env=None):
+def GetUnitTests(
+    input_api, output_api, unit_tests, env=None, run_on_python2=True,
+    run_on_python3=True):
   """Runs all unit tests in a directory.
 
   On Windows, sys.executable is used for unit tests ending with ".py".
   """
+  assert run_on_python3 or run_on_python2, (
+      'At least one of "run_on_python2" or "run_on_python3" must be set.')
+  def has_py3_shebang(test):
+    with open(test) as f:
+      maybe_shebang = f.readline()
+    return maybe_shebang.startswith('#!') and 'python3' in maybe_shebang
+
   # We don't want to hinder users from uploading incomplete patches.
   if input_api.is_committing:
     message_type = output_api.PresubmitError
@@ -629,24 +678,47 @@ def GetUnitTests(input_api, output_api, unit_tests, env=None):
     kwargs = {'cwd': input_api.PresubmitLocalPath()}
     if env:
       kwargs['env'] = env
-    results.append(input_api.Command(
-        name=unit_test,
-        cmd=cmd,
-        kwargs=kwargs,
-        message=message_type))
+    if not unit_test.endswith('.py'):
+      results.append(input_api.Command(
+          name=unit_test,
+          cmd=cmd,
+          kwargs=kwargs,
+          message=message_type))
+    else:
+      if has_py3_shebang(unit_test) and run_on_python3:
+        results.append(input_api.Command(
+            name=unit_test,
+            cmd=cmd,
+            kwargs=kwargs,
+            message=message_type,
+            python3=True))
+      if run_on_python2:
+        results.append(input_api.Command(
+            name=unit_test,
+            cmd=cmd,
+            kwargs=kwargs,
+            message=message_type))
   return results
 
 
 def GetUnitTestsRecursively(input_api, output_api, directory,
-                            whitelist, blacklist):
+                            files_to_check=None, files_to_skip=None,
+                            run_on_python2=True, run_on_python3=True,
+                            whitelist=None, blacklist=None, allowlist=None,
+                            blocklist=None):
   """Gets all files in the directory tree (git repo) that match the whitelist.
 
   Restricts itself to only find files within the Change's source repo, not
   dependencies.
   """
+  files_to_check = files_to_check or allowlist or whitelist
+  files_to_skip = files_to_skip or blocklist or blacklist
+  assert files_to_check is not None
+  assert files_to_skip is not None
+
   def check(filename):
-    return (any(input_api.re.match(f, filename) for f in whitelist) and
-            not any(input_api.re.match(f, filename) for f in blacklist))
+    return (any(input_api.re.match(f, filename) for f in files_to_check) and
+            not any(input_api.re.match(f, filename) for f in files_to_skip))
 
   tests = []
 
@@ -660,11 +732,13 @@ def GetUnitTestsRecursively(input_api, output_api, directory,
   if not to_run:
     return [
         output_api.PresubmitPromptWarning(
-          'Out of %d files, found none that matched w=%r, b=%r in directory %s'
-          % (found, whitelist, blacklist, directory))
+          'Out of %d files, found none that matched c=%r, s=%r in directory %s'
+          % (found, files_to_check, files_to_skip, directory))
     ]
 
-  return GetUnitTests(input_api, output_api, tests)
+  return GetUnitTests(input_api, output_api, tests,
+                      run_on_python2=run_on_python2,
+                      run_on_python3=run_on_python3)
 
 
 def GetPythonUnitTests(input_api, output_api, unit_tests):
@@ -699,6 +773,9 @@ def GetPythonUnitTests(input_api, output_api, unit_tests):
       backpath = [
           '.', input_api.os_path.pathsep.join(['..'] * (cwd.count('/') + 1))
         ]
+      # We convert to str, since on Windows on Python 2 only strings are allowed
+      # as environment variables, but literals are unicode since we're importing
+      # unicode_literals from __future__.
       if env.get('PYTHONPATH'):
         backpath.append(env.get('PYTHONPATH'))
       env['PYTHONPATH'] = input_api.os_path.pathsep.join((backpath))
@@ -740,7 +817,7 @@ def RunPythonUnitTests(input_api, *args, **kwargs):
       GetPythonUnitTests(input_api, *args, **kwargs), False)
 
 
-def _FetchAllFiles(input_api, white_list, black_list):
+def _FetchAllFiles(input_api, files_to_check, files_to_skip):
   """Hack to fetch all files."""
   # We cannot use AffectedFiles here because we want to test every python
   # file on each single python change. It's because a change in a python file
@@ -759,26 +836,30 @@ def _FetchAllFiles(input_api, white_list, black_list):
   path_len = len(input_api.PresubmitLocalPath())
   for dirpath, dirnames, filenames in input_api.os_walk(
       input_api.PresubmitLocalPath()):
-    # Passes dirnames in black list to speed up search.
+    # Passes dirnames in block list to speed up search.
     for item in dirnames[:]:
       filepath = input_api.os_path.join(dirpath, item)[path_len + 1:]
-      if Find(filepath, black_list):
+      if Find(filepath, files_to_skip):
         dirnames.remove(item)
     for item in filenames:
       filepath = input_api.os_path.join(dirpath, item)[path_len + 1:]
-      if Find(filepath, white_list) and not Find(filepath, black_list):
+      if Find(filepath, files_to_check) and not Find(filepath, files_to_skip):
         files.append(filepath)
   return files
 
 
-def GetPylint(input_api, output_api, white_list=None, black_list=None,
-              disabled_warnings=None, extra_paths_list=None, pylintrc=None):
+def GetPylint(input_api, output_api, files_to_check=None, files_to_skip=None,
+              disabled_warnings=None, extra_paths_list=None, pylintrc=None,
+              white_list=None, black_list=None, allow_list=None,
+              block_list=None):
   """Run pylint on python files.
 
-  The default white_list enforces looking only at *.py files.
+  The default files_to_check enforces looking only at *.py files.
   """
-  white_list = tuple(white_list or ('.*\.py$',))
-  black_list = tuple(black_list or input_api.DEFAULT_BLACK_LIST)
+  files_to_check = tuple(files_to_check or allow_list or white_list or
+                         (r'.*\.py$',))
+  files_to_skip = tuple(files_to_skip or block_list or black_list or
+                    input_api.DEFAULT_FILES_TO_SKIP)
   extra_paths_list = extra_paths_list or []
 
   if input_api.is_committing:
@@ -802,7 +883,7 @@ def GetPylint(input_api, output_api, white_list=None, black_list=None,
         input_api.PresubmitLocalPath(), input_api.change.RepositoryRoot()), '')
     return input_api.re.escape(prefix) + regex
   src_filter = lambda x: input_api.FilterSourceFile(
-      x, map(rel_path, white_list), map(rel_path, black_list))
+      x, map(rel_path, files_to_check), map(rel_path, files_to_skip))
   if not input_api.AffectedSourceFiles(src_filter):
     input_api.logging.info('Skipping pylint: no matching changes.')
     return []
@@ -815,7 +896,7 @@ def GetPylint(input_api, output_api, white_list=None, black_list=None,
   if disabled_warnings:
     extra_args.extend(['-d', ','.join(disabled_warnings)])
 
-  files = _FetchAllFiles(input_api, white_list, black_list)
+  files = _FetchAllFiles(input_api, files_to_check, files_to_skip)
   if not files:
     return []
   files.sort()
@@ -823,8 +904,7 @@ def GetPylint(input_api, output_api, white_list=None, black_list=None,
   input_api.logging.info('Running pylint on %d files', len(files))
   input_api.logging.debug('Running pylint on: %s', files)
   env = input_api.environ.copy()
-  env['PYTHONPATH'] = input_api.os_path.pathsep.join(
-    extra_paths_list).encode('utf8')
+  env['PYTHONPATH'] = input_api.os_path.pathsep.join(extra_paths_list)
   env.pop('VPYTHON_CLEAR_PYTHONPATH', None)
   input_api.logging.debug('  with extra PYTHONPATH: %r', extra_paths_list)
 
@@ -832,9 +912,20 @@ def GetPylint(input_api, output_api, white_list=None, black_list=None,
     # Windows needs help running python files so we explicitly specify
     # the interpreter to use. It also has limitations on the size of
     # the command-line, so we pass arguments via a pipe.
-    cmd = [input_api.python_executable,
-           input_api.os_path.join(_HERE, 'third_party', 'pylint.py'),
-           '--args-on-stdin']
+    tool = input_api.os_path.join(_HERE, 'pylint')
+    kwargs = {'env': env}
+    if input_api.platform == 'win32':
+      # On Windows, scripts on the current directory take precedence over PATH.
+      # When `pylint.bat` calls `vpython`, it will execute the `vpython` of the
+      # depot_tools under test instead of the one in the bot.
+      # As a workaround, we run the tests from the parent directory instead.
+      cwd = input_api.change.RepositoryRoot()
+      if input_api.os_path.basename(cwd) == 'depot_tools':
+        kwargs['cwd'] = input_api.os_path.dirname(cwd)
+        flist = [input_api.os_path.join('depot_tools', f) for f in flist]
+      tool += '.bat'
+
+    cmd = [tool, '--args-on-stdin']
     if len(flist) == 1:
       description = flist[0]
     else:
@@ -848,10 +939,12 @@ def GetPylint(input_api, output_api, white_list=None, black_list=None,
       args.append('--jobs=%s' % input_api.cpu_count)
       description += ' on %d cores' % input_api.cpu_count
 
+    kwargs['stdin'] = '\n'.join(args + flist)
+
     return input_api.Command(
         name='Pylint (%s)' % description,
         cmd=cmd,
-        kwargs={'env': env, 'stdin': '\n'.join(args + flist)},
+        kwargs=kwargs,
         message=error_type)
 
   # Always run pylint and pass it all the py files at once.
@@ -891,7 +984,7 @@ def RunPylint(input_api, *args, **kwargs):
 def CheckBuildbotPendingBuilds(input_api, output_api, url, max_pendings,
     ignored):
   try:
-    connection = input_api.urllib2.urlopen(url)
+    connection = input_api.urllib_request.urlopen(url)
     raw_data = connection.read()
     connection.close()
   except IOError:
@@ -904,7 +997,7 @@ def CheckBuildbotPendingBuilds(input_api, output_api, url, max_pendings,
                                              'looking up buildbot status')]
 
   out = []
-  for (builder_name, builder) in data.iteritems():
+  for (builder_name, builder) in data.items():
     if builder_name in ignored:
       continue
     if builder.get('state', '') == 'offline':
@@ -942,7 +1035,27 @@ def CheckOwnersFormat(input_api, output_api):
 def CheckOwners(input_api, output_api, source_file_filter=None):
   affected_files = set([f.LocalPath() for f in
       input_api.change.AffectedFiles(file_filter=source_file_filter)])
-  affects_owners = any('OWNERS' in name for name in affected_files)
+  owners_db = input_api.owners_db
+  owners_db.override_files = input_api.change.OriginalOwnersFiles()
+  owner_email, reviewers = GetCodereviewOwnerAndReviewers(
+      input_api,
+      owners_db.email_regexp,
+      approval_needed=input_api.is_committing)
+
+  owner_email = owner_email or input_api.change.author_email
+
+  finder = input_api.owners_finder(
+      affected_files,
+      input_api.change.RepositoryRoot(),
+      owner_email,
+      reviewers,
+      fopen=open,
+      os_path=input_api.os_path,
+      email_postfix='',
+      disable_color=True,
+      override_files=input_api.change.OriginalOwnersFiles())
+  missing_files = finder.unreviewed_files
+  affects_owners = any('OWNERS' in name for name in missing_files)
 
   if input_api.is_committing:
     if input_api.tbr and not affects_owners:
@@ -963,29 +1076,12 @@ def CheckOwners(input_api, output_api, source_file_filter=None):
     needed = 'OWNER reviewers'
     output_fn = output_api.PresubmitNotifyResult
 
-  owners_db = input_api.owners_db
-  owners_db.override_files = input_api.change.OriginalOwnersFiles()
-  owner_email, reviewers = GetCodereviewOwnerAndReviewers(
-      input_api,
-      owners_db.email_regexp,
-      approval_needed=input_api.is_committing)
-
-  owner_email = owner_email or input_api.change.author_email
-
-  finder = input_api.owners_finder(
-      affected_files, input_api.change.RepositoryRoot(),
-      owner_email, reviewers, fopen=file, os_path=input_api.os_path,
-      email_postfix='', disable_color=True,
-      override_files=input_api.change.OriginalOwnersFiles())
-  missing_files = finder.unreviewed_files
-
   if missing_files:
     output_list = [
         output_fn('Missing %s for these files:\n    %s' %
                   (needed, '\n    '.join(sorted(missing_files))))]
     if input_api.tbr and affects_owners:
-      output_list.append(output_fn('The CL affects an OWNERS file, so TBR will '
-                                   'be ignored.'))
+      output_list.append(output_fn('TBR for OWNERS files are ignored.'))
     if not input_api.is_committing:
       suggested_owners = owners_db.reviewers_for(missing_files, owner_email)
       owners_with_comments = []
@@ -1078,7 +1174,7 @@ def PanProjectChecks(input_api, output_api,
   # 2006-20xx string used on the oldest files. 2006-20xx is deprecated, but
   # tolerated on old files.
   current_year = int(input_api.time.strftime('%Y'))
-  allowed_years = (str(s) for s in reversed(xrange(2006, current_year + 1)))
+  allowed_years = (str(s) for s in reversed(range(2006, current_year + 1)))
   years_re = '(' + '|'.join(allowed_years) + '|2006-2008|2006-2009|2006-2010)'
 
   # The (c) is deprecated, but tolerate it until it's removed from all files.
@@ -1094,24 +1190,27 @@ def PanProjectChecks(input_api, output_api,
   }
 
   results = []
-  # This code loads the default black list (e.g. third_party, experimental, etc)
-  # and add our black list (breakpad, skia and v8 are still not following
+  # This code loads the default skip list (e.g. third_party, experimental, etc)
+  # and add our skip list (breakpad, skia and v8 are still not following
   # google style and are not really living this repository).
   # See presubmit_support.py InputApi.FilterSourceFile for the (simple) usage.
-  black_list = input_api.DEFAULT_BLACK_LIST + excluded_paths
-  white_list = input_api.DEFAULT_WHITE_LIST + text_files
-  sources = lambda x: input_api.FilterSourceFile(x, black_list=black_list)
+  files_to_skip = input_api.DEFAULT_FILES_TO_SKIP + excluded_paths
+  files_to_check = input_api.DEFAULT_FILES_TO_CHECK + text_files
+  sources = lambda x: input_api.FilterSourceFile(x, files_to_skip=files_to_skip)
   text_files = lambda x: input_api.FilterSourceFile(
-      x, black_list=black_list, white_list=white_list)
+      x, files_to_skip=files_to_skip, files_to_check=files_to_check)
 
   snapshot_memory = []
   def snapshot(msg):
     """Measures & prints performance warning if a rule is running slow."""
-    dt2 = input_api.time.clock()
+    if input_api.sys.version_info.major == 2:
+      dt2 = input_api.time.clock()
+    else:
+      dt2 = input_api.time.process_time()
     if snapshot_memory:
       delta_ms = int(1000*(dt2 - snapshot_memory[0]))
       if delta_ms > 500:
-        print "  %s took a long time: %dms" % (snapshot_memory[1], delta_ms)
+        print("  %s took a long time: %dms" % (snapshot_memory[1], delta_ms))
     snapshot_memory[:] = (dt2, msg)
 
   snapshot("checking owners files format")
@@ -1148,6 +1247,10 @@ def PanProjectChecks(input_api, output_api,
     snapshot("checking do not submit in files")
     results.extend(input_api.canned_checks.CheckDoNotSubmitInFiles(
         input_api, output_api))
+    if input_api.change.scm == 'git':
+      snapshot("checking for commit objects in tree")
+      results.extend(input_api.canned_checks.CheckForCommitObjects(
+          input_api, output_api))
   snapshot("done")
   return results
 
@@ -1155,6 +1258,7 @@ def PanProjectChecks(input_api, output_api,
 def CheckPatchFormatted(input_api,
                         output_api,
                         bypass_warnings=True,
+                        check_clang_format=True,
                         check_js=False,
                         check_python=None,
                         result_factory=None):
@@ -1162,6 +1266,9 @@ def CheckPatchFormatted(input_api,
   import git_cl
 
   display_args = []
+  if not check_clang_format:
+    display_args.append('--no-clang-format')
+
   if check_js:
     display_args.append('--js')
 
@@ -1278,7 +1385,7 @@ def CheckCIPDPackages(input_api, output_api, platforms, packages):
   manifest = []
   for p in platforms:
     manifest.append('$VerifiedPlatform %s' % (p,))
-  for k, v in packages.iteritems():
+  for k, v in packages.items():
     manifest.append('%s %s' % (k, v))
   return CheckCIPDManifest(input_api, output_api, content='\n'.join(manifest))
 
@@ -1306,6 +1413,57 @@ def CheckCIPDClientDigests(input_api, output_api, client_version_file):
       output_api.PresubmitError)
 
 
+def CheckForCommitObjects(input_api, output_api):
+  """Validates that there are no commit objects in the repository.
+
+  Commit objects are put into the git tree typically by submodule tooling.
+  Because we use gclient to handle external repository references instead,
+  we want to avoid this. Having commit objects in the tree can confuse git
+  tooling in some scenarios into thinking that the tree is dirty (e.g. the
+  presence of a DEPS subrepo at a location where a commit object is stored
+  in the tree).
+
+  Args:
+    input_api: Bag of input related interfaces.
+    output_api: Bag of output related interfaces.
+
+  Returns:
+    A presubmit error if a commit object is present in the tree.
+  """
+
+  def parse_tree_entry(ent):
+    """Splits a tree entry into components
+
+    Args:
+      ent: a tree entry in the form "filemode type hash\tname"
+
+    Returns:
+      The tree entry split into component parts
+    """
+    tabparts = ent.split('\t', 1)
+    spaceparts = tabparts[0].split(' ', 2)
+    return (spaceparts[0], spaceparts[1], spaceparts[2], tabparts[1])
+
+  full_tree = input_api.subprocess.check_output(
+          ['git', 'ls-tree', '-r', '--full-tree', 'HEAD'],
+          cwd=input_api.PresubmitLocalPath()
+        )
+  tree_entries = full_tree.split('\n')
+  tree_entries = filter(lambda x: len(x) > 0, tree_entries)
+  tree_entries = map(parse_tree_entry, tree_entries)
+  bad_tree_entries = filter(lambda x: x[1] == 'commit', tree_entries)
+  bad_tree_entries = map(lambda x: x[3], bad_tree_entries)
+  if len(bad_tree_entries) > 0:
+    return [output_api.PresubmitError(
+      'Commit objects present within tree.\n'
+      'This may be due to submodule-related interactions; the presence of a\n'
+      'commit object in the tree may lead to odd situations where files are\n'
+      'inconsistently checked-out. Remove these commit entries and validate\n'
+      'your changeset again:\n',
+      bad_tree_entries)]
+  return []
+
+
 def CheckVPythonSpec(input_api, output_api, file_filter=None):
   """Validates any changed .vpython files with vpython verification tool.
 
@@ -1340,7 +1498,6 @@ def CheckChangedLUCIConfigs(input_api, output_api):
   import base64
   import json
   import logging
-  import urllib2
 
   import auth
   import git_cl
@@ -1369,30 +1526,28 @@ def CheckChangedLUCIConfigs(input_api, output_api):
 
   # authentication
   try:
-    authenticator = auth.get_authenticator_for_host(
-        LUCI_CONFIG_HOST_NAME, auth.make_auth_config())
-    acc_tkn = authenticator.get_access_token()
-  except auth.AuthenticationError as e:
+    acc_tkn = auth.Authenticator().get_access_token()
+  except auth.LoginRequiredError as e:
     return [output_api.PresubmitError(
         'Error in authenticating user.', long_text=str(e))]
 
   def request(endpoint, body=None):
     api_url = ('https://%s/_ah/api/config/v1/%s'
                % (LUCI_CONFIG_HOST_NAME, endpoint))
-    req = urllib2.Request(api_url)
+    req = input_api.urllib_request.Request(api_url)
     req.add_header('Authorization', 'Bearer %s' % acc_tkn.token)
     if body is not None:
       req.add_header('Content-Type', 'application/json')
-      req.add_data(json.dumps(body))
-    return json.load(urllib2.urlopen(req))
+      req.data = json.dumps(body).encode('utf-8')
+    return json.load(input_api.urllib_request.urlopen(req))
 
   try:
     config_sets = request('config-sets').get('config_sets')
-  except urllib2.HTTPError as e:
+  except input_api.urllib_error.HTTPError as e:
     return [output_api.PresubmitError(
         'Config set request to luci-config failed', long_text=str(e))]
   if not config_sets:
-    return [output_api.PresubmitWarning('No config_sets were returned')]
+    return [output_api.PresubmitPromptWarning('No config_sets were returned')]
   loc_pref = '%s/+/%s/' % (remote_host_url, remote_branch)
   logging.debug('Derived location prefix: %s', loc_pref)
   dir_to_config_set = {
@@ -1401,25 +1556,40 @@ def CheckChangedLUCIConfigs(input_api, output_api):
     if cs['location'].startswith(loc_pref) or
     ('%s/' % cs['location']) == loc_pref
   }
+  if not dir_to_config_set:
+    warning_long_text_lines = [
+        'No config_set found for %s.' % loc_pref,
+        'Found the following:',
+    ]
+    for loc in sorted(cs['location'] for cs in config_sets):
+      warning_long_text_lines.append('  %s' % loc)
+    warning_long_text_lines.append('')
+    warning_long_text_lines.append(
+        'If the requested location is internal,'
+        ' the requester may not have access.')
+
+    return [output_api.PresubmitPromptWarning(
+        warning_long_text_lines[0],
+        long_text='\n'.join(warning_long_text_lines))]
   cs_to_files = collections.defaultdict(list)
   for f in input_api.AffectedFiles(include_deletes=False):
     # windows
     file_path = f.LocalPath().replace(_os.sep, '/')
     logging.debug('Affected file path: %s', file_path)
-    for dr, cs in dir_to_config_set.iteritems():
+    for dr, cs in dir_to_config_set.items():
       if dr == '/' or file_path.startswith(dr):
         cs_to_files[cs].append({
           'path': file_path[len(dr):] if dr != '/' else file_path,
           'content': base64.b64encode(
-              '\n'.join(f.NewContents()).encode('utf-8'))
+              '\n'.join(f.NewContents()).encode('utf-8')).decode('utf-8')
         })
   outputs = []
-  for cs, f in cs_to_files.iteritems():
+  for cs, f in cs_to_files.items():
     try:
       # TODO(myjang): parallelize
       res = request(
           'validate-config', body={'config_set': cs, 'files': f})
-    except urllib2.HTTPError as e:
+    except input_api.urllib_error.HTTPError as e:
       return [output_api.PresubmitError(
           'Validation request to luci-config failed', long_text=str(e))]
     for msg in res.get('messages', []):
@@ -1466,3 +1636,20 @@ def CheckLucicfgGenOutput(input_api, output_api, entry_script):
         },
         output_api.PresubmitError)
   ]
+
+def CheckJsonParses(input_api, output_api):
+  """Verifies that all JSON files at least parse as valid JSON."""
+  import json
+  affected_files = input_api.AffectedFiles(
+      include_deletes=False,
+      file_filter=lambda x: x.LocalPath().endswith('.json'))
+  warnings = []
+  for f in affected_files:
+    with open(f.AbsoluteLocalPath()) as j:
+      try:
+        json.load(j)
+      except ValueError:
+        # Just a warning for now, in case people are using JSON5 somewhere.
+        warnings.append(output_api.PresubmitPromptWarning(
+            '%s does not appear to be valid JSON.' % f.LocalPath()))
+  return warnings