tree-sitter 0.0.1 → 0.1.0

data/ext/tree-sitter/tree-sitter/externals/gyp/buildbot/buildbot_run.py

@@ -7,6 +7,7 @@
 """Argument-less script to select what to run on the buildbots."""
 
 
+import filecmp
 import os
 import shutil
 import subprocess
@@ -30,7 +31,8 @@ OUT_DIR = os.path.join(TRUNK_DIR, 'out')
 
 def CallSubProcess(*args, **kwargs):
   """Wrapper around subprocess.call which treats errors as build exceptions."""
-  retcode = subprocess.call(*args, **kwargs)
+  with open(os.devnull) as devnull_fd:
+    retcode = subprocess.call(stdin=devnull_fd, *args, **kwargs)
   if retcode != 0:
     print '@@@STEP_EXCEPTION@@@'
     sys.exit(1)
@@ -49,10 +51,6 @@ def PrepareCmake():
 
   print '@@@BUILD_STEP Initialize CMake checkout@@@'
   os.mkdir(CMAKE_DIR)
-  CallSubProcess(['git', 'config', '--global', 'user.name', 'trybot'])
-  CallSubProcess(['git', 'config', '--global',
-                  'user.email', 'chrome-bot@google.com'])
-  CallSubProcess(['git', 'config', '--global', 'color.ui', 'false'])
 
   print '@@@BUILD_STEP Sync CMake@@@'
   CallSubProcess(
@@ -73,41 +71,96 @@ def PrepareCmake():
   CallSubProcess( ['make', 'cmake'], cwd=CMAKE_DIR)
 
 
+_ANDROID_SETUP = 'source build/envsetup.sh && lunch full-eng'
+
+
 def PrepareAndroidTree():
   """Prepare an Android tree to run 'android' format tests."""
   if os.environ['BUILDBOT_CLOBBER'] == '1':
     print '@@@BUILD_STEP Clobber Android checkout@@@'
     shutil.rmtree(ANDROID_DIR)
 
-  # The release of Android we use is static, so there's no need to do anything
-  # if the directory already exists.
-  if os.path.isdir(ANDROID_DIR):
+  # (Re)create the directory so that the following steps will succeed.
+  if not os.path.isdir(ANDROID_DIR):
+    os.mkdir(ANDROID_DIR)
+
+  # We use a manifest from the gyp project listing pinned revisions of AOSP to
+  # use, to ensure that we test against a stable target. This needs to be
+  # updated to pick up new build system changes sometimes, so we must test if
+  # it has changed.
+  manifest_filename = 'aosp_manifest.xml'
+  gyp_manifest = os.path.join(BUILDBOT_DIR, manifest_filename)
+  android_manifest = os.path.join(ANDROID_DIR, '.repo', 'manifests',
+                                  manifest_filename)
+  manifest_is_current = (os.path.isfile(android_manifest) and
+                         filecmp.cmp(gyp_manifest, android_manifest))
+  if not manifest_is_current:
+    # It's safe to repeat these steps, so just do them again to make sure we are
+    # in a good state.
+    print '@@@BUILD_STEP Initialize Android checkout@@@'
+    CallSubProcess(
+        ['repo', 'init',
+         '-u', 'https://android.googlesource.com/platform/manifest',
+         '-b', 'master',
+         '-g', 'all,-notdefault,-device,-darwin,-mips,-x86'],
+        cwd=ANDROID_DIR)
+    shutil.copy(gyp_manifest, android_manifest)
+
+    print '@@@BUILD_STEP Sync Android@@@'
+    CallSubProcess(['repo', 'sync', '-j4', '-m', manifest_filename],
+                   cwd=ANDROID_DIR)
+
+  # If we already built the system image successfully and didn't sync to a new
+  # version of the source, skip running the build again as it's expensive even
+  # when there's nothing to do.
+  system_img = os.path.join(ANDROID_DIR, 'out', 'target', 'product', 'generic',
+                            'system.img')
+  if manifest_is_current and os.path.isfile(system_img):
     return
 
-  print '@@@BUILD_STEP Initialize Android checkout@@@'
-  os.mkdir(ANDROID_DIR)
-  CallSubProcess(['git', 'config', '--global', 'user.name', 'trybot'])
-  CallSubProcess(['git', 'config', '--global',
-                  'user.email', 'chrome-bot@google.com'])
-  CallSubProcess(['git', 'config', '--global', 'color.ui', 'false'])
+  print '@@@BUILD_STEP Build Android@@@'
   CallSubProcess(
-      ['repo', 'init',
-       '-u', 'https://android.googlesource.com/platform/manifest',
-       '-b', 'android-4.2.1_r1',
-       '-g', 'all,-notdefault,-device,-darwin,-mips,-x86'],
+      ['/bin/bash',
+       '-c', '%s && make -j4' % _ANDROID_SETUP],
       cwd=ANDROID_DIR)
 
-  print '@@@BUILD_STEP Sync Android@@@'
-  CallSubProcess(['repo', 'sync', '-j4'], cwd=ANDROID_DIR)
 
-  print '@@@BUILD_STEP Build Android@@@'
+def StartAndroidEmulator():
+  """Start an android emulator from the built android tree."""
+  print '@@@BUILD_STEP Start Android emulator@@@'
+
+  CallSubProcess(['/bin/bash', '-c',
+      '%s && adb kill-server ' % _ANDROID_SETUP],
+      cwd=ANDROID_DIR)
+
+  # If taskset is available, use it to force adbd to run only on one core, as,
+  # sadly, it improves its reliability (see crbug.com/268450).
+  adbd_wrapper = ''
+  with open(os.devnull, 'w') as devnull_fd:
+    if subprocess.call(['which', 'taskset'], stdout=devnull_fd) == 0:
+      adbd_wrapper = 'taskset -c 0'
+  CallSubProcess(['/bin/bash', '-c',
+      '%s && %s adb start-server ' % (_ANDROID_SETUP, adbd_wrapper)],
+      cwd=ANDROID_DIR)
+
+  subprocess.Popen(
+      ['/bin/bash', '-c',
+       '%s && emulator -no-window' % _ANDROID_SETUP],
+      cwd=ANDROID_DIR)
   CallSubProcess(
-      ['/bin/bash',
-       '-c', 'source build/envsetup.sh && lunch full-eng && make -j4'],
+      ['/bin/bash', '-c',
+       '%s && adb wait-for-device' % _ANDROID_SETUP],
       cwd=ANDROID_DIR)
 
 
-def GypTestFormat(title, format=None, msvs_version=None):
+def StopAndroidEmulator():
+  """Stop all android emulators."""
+  print '@@@BUILD_STEP Stop Android emulator@@@'
+  # If this fails, it's because there is no emulator running.
+  subprocess.call(['pkill', 'emulator.*'])
+
+
+def GypTestFormat(title, format=None, msvs_version=None, tests=[]):
   """Run the gyp tests for a given format, emitting annotator tags.
 
   See annotator docs at:
@@ -131,14 +184,13 @@ def GypTestFormat(title, format=None, msvs_version=None):
        '--passed',
        '--format', format,
        '--path', CMAKE_BIN_DIR,
-       '--chdir', 'trunk'])
+       '--chdir', 'trunk'] + tests)
   if format == 'android':
     # gyptest needs the environment setup from envsetup/lunch in order to build
     # using the 'android' backend, so this is done in a single shell.
     retcode = subprocess.call(
         ['/bin/bash',
-         '-c', 'source build/envsetup.sh && lunch full-eng && cd %s && %s'
-         % (ROOT_DIR, command)],
+         '-c', '%s && cd %s && %s' % (_ANDROID_SETUP, ROOT_DIR, command)],
         cwd=ANDROID_DIR, env=env)
   else:
     retcode = subprocess.call(command, cwd=ROOT_DIR, env=env, shell=True)
@@ -160,7 +212,11 @@ def GypBuild():
   # The Android gyp bot runs on linux so this must be tested first.
   if os.environ['BUILDBOT_BUILDERNAME'] == 'gyp-android':
     PrepareAndroidTree()
-    retcode += GypTestFormat('android')
+    StartAndroidEmulator()
+    try:
+      retcode += GypTestFormat('android')
+    finally:
+      StopAndroidEmulator()
   elif sys.platform.startswith('linux'):
     retcode += GypTestFormat('ninja')
     retcode += GypTestFormat('make')
@@ -173,8 +229,13 @@ def GypBuild():
   elif sys.platform == 'win32':
     retcode += GypTestFormat('ninja')
     if os.environ['BUILDBOT_BUILDERNAME'] == 'gyp-win64':
-      retcode += GypTestFormat('msvs-2010', format='msvs', msvs_version='2010')
-      retcode += GypTestFormat('msvs-2012', format='msvs', msvs_version='2012')
+      retcode += GypTestFormat('msvs-ninja-2013', format='msvs-ninja',
+                               msvs_version='2013',
+                               tests=[
+                                  r'test\generator-output\gyptest-actions.py',
+                                  r'test\generator-output\gyptest-relocate.py',
+                                  r'test\generator-output\gyptest-rules.py'])
+      retcode += GypTestFormat('msvs-2013', format='msvs', msvs_version='2013')
   else:
     raise Exception('Unknown platform')
   if retcode:

data/ext/tree-sitter/tree-sitter/externals/gyp/gyp

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/bin/sh
 # Copyright 2013 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.

data/ext/tree-sitter/tree-sitter/externals/gyp/gyptest.py

@@ -13,7 +13,7 @@ import optparse
 import subprocess
 import sys
 
-class CommandRunner:
+class CommandRunner(object):
   """
   Executor class for commands, including "commands" implemented by
   Python functions.
@@ -117,7 +117,7 @@ class CommandRunner:
     return self.execute(command, stdout, stderr)
 
 
-class Unbuffered:
+class Unbuffered(object):
   def __init__(self, fp):
     self.fp = fp
   def write(self, arg):
@@ -224,7 +224,7 @@ def main(argv=None):
       'win32':    ['msvs', 'ninja'],
       'linux2':   ['make', 'ninja'],
       'linux3':   ['make', 'ninja'],
-      'darwin':   ['make', 'ninja', 'xcode'],
+      'darwin':   ['make', 'ninja', 'xcode', 'xcode-ninja'],
     }[sys.platform]
 
   for format in format_list:

data/ext/tree-sitter/tree-sitter/externals/gyp/pylib/gyp/MSVSNew.py

@@ -172,7 +172,7 @@ class MSVSProject(MSVSSolutionEntry):
 #------------------------------------------------------------------------------
 
 
-class MSVSSolution:
+class MSVSSolution(object):
   """Visual Studio solution."""
 
   def __init__(self, path, version, entries=None, variants=None,

data/ext/tree-sitter/tree-sitter/externals/gyp/pylib/gyp/MSVSSettings.py

@@ -2,7 +2,7 @@
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
-"""Code to validate and convert settings of the Microsoft build tools.
+r"""Code to validate and convert settings of the Microsoft build tools.
 
 This file contains code to validate and convert settings of the Microsoft
 build tools.  The function ConvertToMSBuildSettings(), ValidateMSVSSettings(),
@@ -417,11 +417,11 @@ def ConvertVCMacrosToMSBuild(s):
   if '$' in s:
     replace_map = {
         '$(ConfigurationName)': '$(Configuration)',
-        '$(InputDir)': '%(RootDir)%(Directory)',
+        '$(InputDir)': '%(RelativeDir)',
         '$(InputExt)': '%(Extension)',
         '$(InputFileName)': '%(Filename)%(Extension)',
         '$(InputName)': '%(Filename)',
-        '$(InputPath)': '%(FullPath)',
+        '$(InputPath)': '%(Identity)',
         '$(ParentName)': '$(ProjectFileName)',
         '$(PlatformName)': '$(Platform)',
         '$(SafeInputName)': '%(Filename)',
@@ -602,7 +602,8 @@ _Same(_compile, 'BrowseInformation',
 _Same(_compile, 'CallingConvention',
       _Enumeration(['Cdecl',  # /Gd
                     'FastCall',  # /Gr
-                    'StdCall']))  # /Gz
+                    'StdCall',  # /Gz
+                    'VectorCall']))  # /Gv
 _Same(_compile, 'CompileAs',
       _Enumeration(['Default',
                     'CompileAsC',  # /TC
@@ -616,7 +617,12 @@ _Same(_compile, 'DebugInformationFormat',
 _Same(_compile, 'EnableEnhancedInstructionSet',
       _Enumeration(['NotSet',
                     'StreamingSIMDExtensions',  # /arch:SSE
-                    'StreamingSIMDExtensions2']))  # /arch:SSE2
+                    'StreamingSIMDExtensions2',  # /arch:SSE2
+                    'AdvancedVectorExtensions',  # /arch:AVX (vs2012+)
+                    'NoExtensions',  # /arch:IA32 (vs2012+)
+                    # This one only exists in the new msbuild format.
+                    'AdvancedVectorExtensions2',  # /arch:AVX2 (vs2013r2+)
+                    ]))
 _Same(_compile, 'ErrorReporting',
       _Enumeration(['None',  # /errorReport:none
                     'Prompt',  # /errorReport:prompt

data/ext/tree-sitter/tree-sitter/externals/gyp/pylib/gyp/MSVSSettings_test.py

@@ -267,7 +267,7 @@ class TestSequenceFunctions(unittest.TestCase):
         'Warning: for VCCLCompilerTool/BrowseInformation, '
         "invalid literal for int() with base 10: 'fdkslj'",
         'Warning: for VCCLCompilerTool/CallingConvention, '
-        'index value (-1) not in expected range [0, 3)',
+        'index value (-1) not in expected range [0, 4)',
         'Warning: for VCCLCompilerTool/DebugInformationFormat, '
         'converted value for 2 not specified.',
         'Warning: unrecognized setting VCCLCompilerTool/Enableprefast',

data/ext/tree-sitter/tree-sitter/externals/gyp/pylib/gyp/MSVSUtil.py

@@ -8,10 +8,12 @@ import copy
 import os
 
 
-_TARGET_TYPE_EXT = {
-  'executable': '.exe',
-  'loadable_module': '.dll',
-  'shared_library': '.dll',
+# A dictionary mapping supported target types to extensions.
+TARGET_TYPE_EXT = {
+  'executable': 'exe',
+  'loadable_module': 'dll',
+  'shared_library': 'dll',
+  'static_library': 'lib',
 }
 
 
@@ -109,15 +111,16 @@ def ShardTargets(target_list, target_dicts):
       new_target_dicts[t] = target_dicts[t]
   # Shard dependencies.
   for t in new_target_dicts:
-    dependencies = copy.copy(new_target_dicts[t].get('dependencies', []))
-    new_dependencies = []
-    for d in dependencies:
-      if d in targets_to_shard:
-        for i in range(targets_to_shard[d]):
-          new_dependencies.append(_ShardName(d, i))
-      else:
-        new_dependencies.append(d)
-    new_target_dicts[t]['dependencies'] = new_dependencies
+    for deptype in ('dependencies', 'dependencies_original'):
+      dependencies = copy.copy(new_target_dicts[t].get(deptype, []))
+      new_dependencies = []
+      for d in dependencies:
+        if d in targets_to_shard:
+          for i in range(targets_to_shard[d]):
+            new_dependencies.append(_ShardName(d, i))
+        else:
+          new_dependencies.append(d)
+      new_target_dicts[t][deptype] = new_dependencies
 
   return (new_target_list, new_target_dicts)
 
@@ -156,7 +159,7 @@ def _GetPdbPath(target_dict, config_name, vars):
 
 
   pdb_base = target_dict.get('product_name', target_dict['target_name'])
-  pdb_base = '%s%s.pdb' % (pdb_base, _TARGET_TYPE_EXT[target_dict['type']])
+  pdb_base = '%s.%s.pdb' % (pdb_base, TARGET_TYPE_EXT[target_dict['type']])
   pdb_path = vars['PRODUCT_DIR'] + '/' + pdb_base
 
   return pdb_path
@@ -264,4 +267,4 @@ def InsertLargePdbShims(target_list, target_dicts, vars):
     # Update the original target to depend on the shim target.
     target_dict.setdefault('dependencies', []).append(full_shim_target_name)
 
-  return (target_list, target_dicts)
+  return (target_list, target_dicts)

data/ext/tree-sitter/tree-sitter/externals/gyp/pylib/gyp/MSVSVersion.py

@@ -138,7 +138,7 @@ def _RegistryQueryBase(sysdir, key, value):
 
 
 def _RegistryQuery(key, value=None):
-  """Use reg.exe to read a particular key through _RegistryQueryBase.
+  r"""Use reg.exe to read a particular key through _RegistryQueryBase.
 
   First tries to launch from %WinDir%\Sysnative to avoid WoW64 redirection. If
   that fails, it falls back to System32.  Sysnative is available on Vista and
@@ -165,8 +165,33 @@ def _RegistryQuery(key, value=None):
   return text
 
 
+def _RegistryGetValueUsingWinReg(key, value):
+  """Use the _winreg module to obtain the value of a registry key.
+
+  Args:
+    key: The registry key.
+    value: The particular registry value to read.
+  Return:
+    contents of the registry key's value, or None on failure.  Throws
+    ImportError if _winreg is unavailable.
+  """
+  import _winreg
+  try:
+    root, subkey = key.split('\\', 1)
+    assert root == 'HKLM'  # Only need HKLM for now.
+    with _winreg.OpenKey(_winreg.HKEY_LOCAL_MACHINE, subkey) as hkey:
+      return _winreg.QueryValueEx(hkey, value)[0]
+  except WindowsError:
+    return None
+
+
 def _RegistryGetValue(key, value):
-  """Use reg.exe to obtain the value of a registry key.
+  """Use _winreg or reg.exe to obtain the value of a registry key.
+
+  Using _winreg is preferable because it solves an issue on some corporate
+  environments where access to reg.exe is locked down. However, we still need
+  to fallback to reg.exe for the case where the _winreg module is not available
+  (for example in cygwin python).
 
   Args:
     key: The registry key.
@@ -174,6 +199,12 @@ def _RegistryGetValue(key, value):
   Return:
     contents of the registry key's value, or None on failure.
   """
+  try:
+    return _RegistryGetValueUsingWinReg(key, value)
+  except ImportError:
+    pass
+
+  # Fallback to reg.exe if we fail to import _winreg.
   text = _RegistryQuery(key, value)
   if not text:
     return None
@@ -184,19 +215,6 @@ def _RegistryGetValue(key, value):
   return match.group(1)
 
 
-def _RegistryKeyExists(key):
-  """Use reg.exe to see if a key exists.
-
-  Args:
-    key: The registry key to check.
-  Return:
-    True if the key exists
-  """
-  if not _RegistryQuery(key):
-    return False
-  return True
-
-
 def _CreateVersion(name, path, sdk_based=False):
   """Sets up MSVS project generation.
 
@@ -207,6 +225,15 @@ def _CreateVersion(name, path, sdk_based=False):
   if path:
     path = os.path.normpath(path)
   versions = {
+      '2015': VisualStudioVersion('2015',
+                                  'Visual Studio 2015',
+                                  solution_version='12.00',
+                                  project_version='14.0',
+                                  flat_sln=False,
+                                  uses_vcxproj=True,
+                                  path=path,
+                                  sdk_based=sdk_based,
+                                  default_toolset='v140'),
       '2013': VisualStudioVersion('2013',
                                   'Visual Studio 2013',
                                   solution_version='13.00',
@@ -316,7 +343,8 @@ def _DetectVisualStudioVersions(versions_to_check, force_express):
       2008(e) - Visual Studio 2008 (9)
       2010(e) - Visual Studio 2010 (10)
       2012(e) - Visual Studio 2012 (11)
-      2013(e) - Visual Studio 2013 (11)
+      2013(e) - Visual Studio 2013 (12)
+      2015    - Visual Studio 2015 (14)
     Where (e) is e for express editions of MSVS and blank otherwise.
   """
   version_to_year = {
@@ -325,6 +353,7 @@ def _DetectVisualStudioVersions(versions_to_check, force_express):
       '10.0': '2010',
       '11.0': '2012',
       '12.0': '2013',
+      '14.0': '2015',
   }
   versions = []
   for version in versions_to_check:
@@ -361,13 +390,14 @@ def _DetectVisualStudioVersions(versions_to_check, force_express):
       if not path:
         continue
       path = _ConvertToCygpath(path)
-      versions.append(_CreateVersion(version_to_year[version] + 'e',
-          os.path.join(path, '..'), sdk_based=True))
+      if version != '14.0':  # There is no Express edition for 2015.
+        versions.append(_CreateVersion(version_to_year[version] + 'e',
+            os.path.join(path, '..'), sdk_based=True))
 
   return versions
 
 
-def SelectVisualStudioVersion(version='auto'):
+def SelectVisualStudioVersion(version='auto', allow_fallback=True):
   """Select which version of Visual Studio projects to generate.
 
   Arguments:
@@ -379,7 +409,7 @@ def SelectVisualStudioVersion(version='auto'):
   if version == 'auto':
     version = os.environ.get('GYP_MSVS_VERSION', 'auto')
   version_map = {
-    'auto': ('10.0', '12.0', '9.0', '8.0', '11.0'),
+    'auto': ('14.0', '12.0', '10.0', '9.0', '8.0', '11.0'),
     '2005': ('8.0',),
     '2005e': ('8.0',),
     '2008': ('9.0',),
@@ -390,6 +420,7 @@ def SelectVisualStudioVersion(version='auto'):
     '2012e': ('11.0',),
     '2013': ('12.0',),
     '2013e': ('12.0',),
+    '2015': ('14.0',),
   }
   override_path = os.environ.get('GYP_MSVS_OVERRIDE_PATH')
   if override_path:
@@ -401,6 +432,8 @@ def SelectVisualStudioVersion(version='auto'):
   version = str(version)
   versions = _DetectVisualStudioVersions(version_map[version], 'e' in version)
   if not versions:
+    if not allow_fallback:
+      raise ValueError('Could not locate Visual Studio installation.')
     if version == 'auto':
       # Default to 2005 if we couldn't find anything
       return _CreateVersion('2005', None)