tree-sitter 0.0.1 → 0.1.0

data/ext/tree-sitter/tree-sitter/externals/gyp/test/lib/TestGyp.py

@@ -7,14 +7,17 @@ TestGyp.py:  a testing framework for GYP integration tests.
 """
 
 import collections
+from contextlib import contextmanager
 import itertools
 import os
+import random
 import re
 import shutil
 import stat
 import subprocess
 import sys
 import tempfile
+import time
 
 import TestCmd
 import TestCommon
@@ -24,9 +27,10 @@ __all__.extend([
   'TestGyp',
 ])
 
+
 def remove_debug_line_numbers(contents):
   """Function to remove the line numbers from the debug output
-  of gyp and thus remove the exremem fragility of the stdout
+  of gyp and thus reduce the extreme fragility of the stdout
   comparison tests.
   """
   lines = contents.splitlines()
@@ -37,12 +41,26 @@ def remove_debug_line_numbers(contents):
   lines = [len(l) > 3 and ":".join(l[3:]) or l for l in lines]
   return "\n".join(lines)
 
+
 def match_modulo_line_numbers(contents_a, contents_b):
   """File contents matcher that ignores line numbers."""
   contents_a = remove_debug_line_numbers(contents_a)
   contents_b = remove_debug_line_numbers(contents_b)
   return TestCommon.match_exact(contents_a, contents_b)
 
+
+@contextmanager
+def LocalEnv(local_env):
+  """Context manager to provide a local OS environment."""
+  old_env = os.environ.copy()
+  os.environ.update(local_env)
+  try:
+    yield
+  finally:
+    os.environ.clear()
+    os.environ.update(old_env)
+
+
 class TestGypBase(TestCommon.TestCommon):
   """
   Class for controlling end-to-end tests of gyp generators.
@@ -62,6 +80,7 @@ class TestGypBase(TestCommon.TestCommon):
   configuration and to run executables generated by those builds.
   """
 
+  formats = []
   build_tool = None
   build_tool_list = []
 
@@ -97,6 +116,8 @@ class TestGypBase(TestCommon.TestCommon):
     self.gyp = os.path.abspath(gyp)
     self.no_parallel = False
 
+    self.formats = [self.format]
+
     self.initialize_build_tool()
 
     kw.setdefault('match', TestCommon.match_exact)
@@ -114,10 +135,11 @@ class TestGypBase(TestCommon.TestCommon):
 
     super(TestGypBase, self).__init__(*args, **kw)
 
+    real_format = self.format.split('-')[-1]
     excluded_formats = set([f for f in formats if f[0] == '!'])
     included_formats = set(formats) - excluded_formats
-    if ('!'+self.format in excluded_formats or
-        included_formats and self.format not in included_formats):
+    if ('!'+real_format in excluded_formats or
+        included_formats and real_format not in included_formats):
       msg = 'Invalid test for %r format; skipping test.\n'
       self.skip_test(msg % self.format)
 
@@ -131,6 +153,9 @@ class TestGypBase(TestCommon.TestCommon):
     # Clear $GYP_DEFINES for the same reason.
     if 'GYP_DEFINES' in os.environ:
       del os.environ['GYP_DEFINES']
+    # Override the user's language settings, which could
+    # otherwise make the output vary from what is expected.
+    os.environ['LC_ALL'] = 'C'
 
   def built_file_must_exist(self, name, type=None, **kw):
     """
@@ -256,10 +281,18 @@ class TestGypBase(TestCommon.TestCommon):
 
     # TODO:  --depth=. works around Chromium-specific tree climbing.
     depth = kw.pop('depth', '.')
-    run_args = ['--depth='+depth, '--format='+self.format, gyp_file]
+    run_args = ['--depth='+depth]
+    run_args.extend(['--format='+f for f in self.formats]);
+    run_args.append(gyp_file)
     if self.no_parallel:
       run_args += ['--no-parallel']
+    # TODO: if extra_args contains a '--build' flag
+    # we really want that to only apply to the last format (self.format).
     run_args.extend(self.extra_args)
+    # Default xcode_ninja_target_pattern to ^.*$ to fix xcode-ninja tests
+    xcode_ninja_target_pattern = kw.pop('xcode_ninja_target_pattern', '.*')
+    run_args.extend(
+      ['-G', 'xcode_ninja_target_pattern=%s' % xcode_ninja_target_pattern])
     run_args.extend(args)
     return self.run(program=self.gyp, arguments=run_args, **kw)
 
@@ -382,10 +415,6 @@ class TestGypAndroid(TestGypBase):
   """
   Subclass for testing the GYP Android makefile generator. Note that
   build/envsetup.sh and lunch must have been run before running tests.
-
-  TODO: This is currently an incomplete implementation. We do not support
-  run_built_executable(), so we pass only tests which do not use this. As a
-  result, support for host targets is not properly tested.
   """
   format = 'android'
 
@@ -428,6 +457,20 @@ class TestGypAndroid(TestGypBase):
           os.remove(os.path.join(out_dir, x, d))
 
     super(TestGypAndroid, self).__init__(*args, **kw)
+    self._adb_path = os.path.join(os.environ['ANDROID_HOST_OUT'], 'bin', 'adb')
+    self._device_serial = None
+    adb_devices_out = self._call_adb(['devices'])
+    devices = [l.split()[0] for l in adb_devices_out.splitlines()[1:-1]
+               if l.split()[1] == 'device']
+    if len(devices) == 0:
+      self._device_serial = None
+    else:
+      if len(devices) > 1:
+        self._device_serial = random.choice(devices)
+      else:
+        self._device_serial = devices[0]
+      self._call_adb(['root'])
+    self._to_install = set()
 
   def target_name(self, target):
     if target == self.ALL:
@@ -438,6 +481,8 @@ class TestGypAndroid(TestGypBase):
       return self.ALL
     return target
 
+  _INSTALLABLE_PREFIX = 'Install: '
+
   def build(self, gyp_file, target=None, **kw):
     """
     Runs a build using the Android makefiles generated from the specified
@@ -452,6 +497,11 @@ class TestGypAndroid(TestGypBase):
     makefile = os.path.join(self.workdir, chdir, 'GypAndroid.mk')
     os.environ['ONE_SHOT_MAKEFILE'] = makefile
     result = self.run(program=self.build_tool, **kw)
+    for l in self.stdout().splitlines():
+      if l.startswith(TestGypAndroid._INSTALLABLE_PREFIX):
+        self._to_install.add(os.path.abspath(os.path.join(
+            os.environ['ANDROID_BUILD_TOP'],
+            l[len(TestGypAndroid._INSTALLABLE_PREFIX):])))
     del os.environ['ONE_SHOT_MAKEFILE']
     return result
 
@@ -474,15 +524,10 @@ class TestGypAndroid(TestGypBase):
     """
     # Built files are in $ANDROID_PRODUCT_OUT. This requires copying logic from
     # the Android build system.
-    if type == None:
+    if type == None or type == self.EXECUTABLE:
       return os.path.join(os.environ['ANDROID_PRODUCT_OUT'], 'obj', 'GYP',
                           'shared_intermediates', name)
     subdir = kw.get('subdir')
-    if type == self.EXECUTABLE:
-      # We don't install executables
-      group = 'EXECUTABLES'
-      module_name = self.android_module(group, name, subdir)
-      return os.path.join(self.intermediates_dir(group, module_name), name)
     if type == self.STATIC_LIB:
       group = 'STATIC_LIBRARIES'
       module_name = self.android_module(group, name, subdir)
@@ -495,23 +540,124 @@ class TestGypAndroid(TestGypBase):
                           '%s.so' % module_name)
     assert False, 'Unhandled type'
 
+  def _adb_failure(self, command, msg, stdout, stderr):
+    """ Reports a failed adb command and fails the containing test.
+
+    Args:
+      command: The adb command that failed.
+      msg: The error description.
+      stdout: The standard output.
+      stderr: The standard error.
+    """
+    print '%s failed%s' % (' '.join(command), ': %s' % msg if msg else '')
+    print self.banner('STDOUT ')
+    stdout.seek(0)
+    print stdout.read()
+    print self.banner('STDERR ')
+    stderr.seek(0)
+    print stderr.read()
+    self.fail_test()
+
+  def _call_adb(self, command, timeout=15, retry=3):
+    """ Calls the provided adb command.
+
+    If the command fails, the test fails.
+
+    Args:
+      command: The adb command to call.
+    Returns:
+      The command's output.
+    """
+    with tempfile.TemporaryFile(bufsize=0) as adb_out:
+      with tempfile.TemporaryFile(bufsize=0) as adb_err:
+        adb_command = [self._adb_path]
+        if self._device_serial:
+          adb_command += ['-s', self._device_serial]
+        is_shell = (command[0] == 'shell')
+        if is_shell:
+          command = [command[0], '%s; echo "\n$?";' % ' '.join(command[1:])]
+        adb_command += command
+
+        for attempt in xrange(1, retry + 1):
+          adb_out.seek(0)
+          adb_out.truncate(0)
+          adb_err.seek(0)
+          adb_err.truncate(0)
+          proc = subprocess.Popen(adb_command, stdout=adb_out, stderr=adb_err)
+          deadline = time.time() + timeout
+          timed_out = False
+          while proc.poll() is None and not timed_out:
+            time.sleep(1)
+            timed_out = time.time() > deadline
+          if timed_out:
+            print 'Timeout for command %s (attempt %d of %s)' % (
+                adb_command, attempt, retry)
+            try:
+              proc.kill()
+            except:
+              pass
+          else:
+            break
+
+        if proc.returncode != 0:  # returncode is None in the case of a timeout.
+          self._adb_failure(
+              adb_command, 'retcode=%s' % proc.returncode, adb_out, adb_err)
+          return
+
+        adb_out.seek(0)
+        output = adb_out.read()
+        if is_shell:
+          output = output.splitlines(True)
+          try:
+            output[-2] = output[-2].rstrip('\r\n')
+            output, rc = (''.join(output[:-1]), int(output[-1]))
+          except ValueError:
+            self._adb_failure(adb_command, 'unexpected output format',
+                              adb_out, adb_err)
+          if rc != 0:
+            self._adb_failure(adb_command, 'exited with %d' % rc, adb_out,
+                              adb_err)
+        return output
+
   def run_built_executable(self, name, *args, **kw):
     """
     Runs an executable program built from a gyp-generated configuration.
-
-    This is not correctly implemented for Android. For now, we simply check
-    that the executable file exists.
     """
-    # Running executables requires a device. Even if we build for target x86,
-    # the binary is not built with the correct toolchain options to actually
-    # run on the host.
-
-    # Copied from TestCommon.run()
     match = kw.pop('match', self.match)
-    status = None
-    if os.path.exists(self.built_file_path(name)):
-      status = 1
-    self._complete(None, None, None, None, status, match)
+
+    executable_file = self.built_file_path(name, type=self.EXECUTABLE, **kw)
+    if executable_file not in self._to_install:
+      self.fail_test()
+
+    if not self._device_serial:
+      self.skip_test(message='No devices attached.\n')
+
+    storage = self._call_adb(['shell', 'echo', '$ANDROID_DATA']).strip()
+    if not len(storage):
+      self.fail_test()
+
+    installed = set()
+    try:
+      for i in self._to_install:
+        a = os.path.abspath(
+            os.path.join(os.environ['ANDROID_BUILD_TOP'], i))
+        dest = '%s/%s' % (storage, os.path.basename(a))
+        self._call_adb(['push', os.path.abspath(a), dest])
+        installed.add(dest)
+        if i == executable_file:
+          device_executable = dest
+          self._call_adb(['shell', 'chmod', '755', device_executable])
+
+      out = self._call_adb(
+          ['shell', 'LD_LIBRARY_PATH=$LD_LIBRARY_PATH:%s' % storage,
+           device_executable],
+          timeout=60,
+          retry=1)
+      out = out.replace('\r\n', '\n')
+      self._complete(out, kw.pop('stdout', None), None, None, None, match)
+    finally:
+      if len(installed):
+        self._call_adb(['shell', 'rm'] + list(installed))
 
   def match_single_line(self, lines = None, expected_line = None):
     """
@@ -731,6 +877,53 @@ def ConvertToCygpath(path):
   return path
 
 
+def FindMSBuildInstallation(msvs_version = 'auto'):
+  """Returns path to MSBuild for msvs_version or latest available.
+
+  Looks in the registry to find install location of MSBuild.
+  MSBuild before v4.0 will not build c++ projects, so only use newer versions.
+  """
+  import TestWin
+  registry = TestWin.Registry()
+
+  msvs_to_msbuild = {
+      '2013': r'12.0',
+      '2012': r'4.0',  # Really v4.0.30319 which comes with .NET 4.5.
+      '2010': r'4.0'}
+
+  msbuild_basekey = r'HKLM\SOFTWARE\Microsoft\MSBuild\ToolsVersions'
+  if not registry.KeyExists(msbuild_basekey):
+    print 'Error: could not find MSBuild base registry entry'
+    return None
+
+  msbuild_version = None
+  if msvs_version in msvs_to_msbuild:
+    msbuild_test_version = msvs_to_msbuild[msvs_version]
+    if registry.KeyExists(msbuild_basekey + '\\' + msbuild_test_version):
+      msbuild_version = msbuild_test_version
+    else:
+      print ('Warning: Environment variable GYP_MSVS_VERSION specifies "%s" '
+             'but corresponding MSBuild "%s" was not found.' %
+             (msvs_version, msbuild_version))
+  if not msbuild_version:
+    for msvs_version in sorted(msvs_to_msbuild, reverse=True):
+      msbuild_test_version = msvs_to_msbuild[msvs_version]
+      if registry.KeyExists(msbuild_basekey + '\\' + msbuild_test_version):
+        msbuild_version = msbuild_test_version
+        break
+  if not msbuild_version:
+    print 'Error: could not find MSBuild registry entry'
+    return None
+
+  msbuild_path = registry.GetValue(msbuild_basekey + '\\' + msbuild_version,
+                                   'MSBuildToolsPath')
+  if not msbuild_path:
+    print 'Error: could not get MSBuild registry entry value'
+    return None
+
+  return os.path.join(msbuild_path, 'MSBuild.exe')
+
+
 def FindVisualStudioInstallation():
   """Returns appropriate values for .build_tool and .uses_msbuild fields
   of TestGypBase for Visual Studio.
@@ -756,39 +949,28 @@ def FindVisualStudioInstallation():
     msvs_version = flag.split('=')[-1]
   msvs_version = os.environ.get('GYP_MSVS_VERSION', msvs_version)
 
-  build_tool = None
   if msvs_version in possible_paths:
     # Check that the path to the specified GYP_MSVS_VERSION exists.
     path = possible_paths[msvs_version]
     for r in possible_roots:
-      bt = os.path.join(r, path)
-      if os.path.exists(bt):
-        build_tool = bt
+      build_tool = os.path.join(r, path)
+      if os.path.exists(build_tool):
         uses_msbuild = msvs_version >= '2010'
-        return build_tool, uses_msbuild
+        msbuild_path = FindMSBuildInstallation(msvs_version)
+        return build_tool, uses_msbuild, msbuild_path
     else:
       print ('Warning: Environment variable GYP_MSVS_VERSION specifies "%s" '
               'but corresponding "%s" was not found.' % (msvs_version, path))
-  if build_tool:
-    # We found 'devenv' on the path, use that and try to guess the version.
-    for version, path in possible_paths.iteritems():
-      if build_tool.find(path) >= 0:
-        uses_msbuild = version >= '2010'
-        return build_tool, uses_msbuild
-    else:
-      # If not, assume not MSBuild.
-      uses_msbuild = False
-    return build_tool, uses_msbuild
   # Neither GYP_MSVS_VERSION nor the path help us out.  Iterate through
   # the choices looking for a match.
   for version in sorted(possible_paths, reverse=True):
     path = possible_paths[version]
     for r in possible_roots:
-      bt = os.path.join(r, path)
-      if os.path.exists(bt):
-        build_tool = bt
+      build_tool = os.path.join(r, path)
+      if os.path.exists(build_tool):
         uses_msbuild = msvs_version >= '2010'
-        return build_tool, uses_msbuild
+        msbuild_path = FindMSBuildInstallation(msvs_version)
+        return build_tool, uses_msbuild, msbuild_path
   print 'Error: could not find devenv'
   sys.exit(1)
 
@@ -806,7 +988,8 @@ class TestGypOnMSToolchain(TestGypBase):
   def initialize_build_tool(self):
     super(TestGypOnMSToolchain, self).initialize_build_tool()
     if sys.platform in ('win32', 'cygwin'):
-      self.devenv_path, self.uses_msbuild = FindVisualStudioInstallation()
+      build_tools = FindVisualStudioInstallation()
+      self.devenv_path, self.uses_msbuild, self.msbuild_path = build_tools
       self.vsvars_path = TestGypOnMSToolchain._ComputeVsvarsPath(
           self.devenv_path)
 
@@ -986,6 +1169,56 @@ class TestGypMSVS(TestGypOnMSToolchain):
     return self.workpath(*result)
 
 
+class TestGypMSVSNinja(TestGypNinja):
+  """
+  Subclass for testing the GYP Visual Studio Ninja generator.
+  """
+  format = 'msvs-ninja'
+
+  def initialize_build_tool(self):
+    super(TestGypMSVSNinja, self).initialize_build_tool()
+    # When using '--build', make sure ninja is first in the format list.
+    self.formats.insert(0, 'ninja')
+
+  def build(self, gyp_file, target=None, rebuild=False, clean=False, **kw):
+    """
+    Runs a Visual Studio build using the configuration generated
+    from the specified gyp_file.
+    """
+    arguments = kw.get('arguments', [])[:]
+    if target in (None, self.ALL, self.DEFAULT):
+      # Note: the Visual Studio generator doesn't add an explicit 'all' target.
+      # This will build each project. This will work if projects are hermetic,
+      # but may fail if they are not (a project may run more than once).
+      # It would be nice to supply an all.metaproj for MSBuild.
+      arguments.extend([gyp_file.replace('.gyp', '.sln')])
+    else:
+      # MSBuild documentation claims that one can specify a sln but then build a
+      # project target like 'msbuild a.sln /t:proj:target' but this format only
+      # supports 'Clean', 'Rebuild', and 'Publish' (with none meaning Default).
+      # This limitation is due to the .sln -> .sln.metaproj conversion.
+      # The ':' is not special, 'proj:target' is a target in the metaproj.
+      arguments.extend([target+'.vcxproj'])
+
+    if clean:
+      build = 'Clean'
+    elif rebuild:
+      build = 'Rebuild'
+    else:
+      build = 'Build'
+    arguments.extend(['/target:'+build])
+    configuration = self.configuration_buildname()
+    config = configuration.split('|')
+    arguments.extend(['/property:Configuration='+config[0]])
+    if len(config) > 1:
+      arguments.extend(['/property:Platform='+config[1]])
+    arguments.extend(['/property:BuildInParallel=false'])
+    arguments.extend(['/verbosity:minimal'])
+
+    kw['arguments'] = arguments
+    return self.run(program=self.msbuild_path, **kw)
+
+
 class TestGypXcode(TestGypBase):
   """
   Subclass for testing the GYP Xcode generator.
@@ -1096,14 +1329,84 @@ class TestGypXcode(TestGypBase):
     return self.workpath(*result)
 
 
+class TestGypXcodeNinja(TestGypXcode):
+  """
+  Subclass for testing the GYP Xcode Ninja generator.
+  """
+  format = 'xcode-ninja'
+
+  def initialize_build_tool(self):
+    super(TestGypXcodeNinja, self).initialize_build_tool()
+    # When using '--build', make sure ninja is first in the format list.
+    self.formats.insert(0, 'ninja')
+
+  def build(self, gyp_file, target=None, **kw):
+    """
+    Runs an xcodebuild using the .xcodeproj generated from the specified
+    gyp_file.
+    """
+    build_config = self.configuration
+    if build_config and build_config.endswith(('-iphoneos',
+                                               '-iphonesimulator')):
+      build_config, sdk = self.configuration.split('-')
+      kw['arguments'] = kw.get('arguments', []) + ['-sdk', sdk]
+
+    with self._build_configuration(build_config):
+      return super(TestGypXcodeNinja, self).build(
+        gyp_file.replace('.gyp', '.ninja.gyp'), target, **kw)
+
+  @contextmanager
+  def _build_configuration(self, build_config):
+    config = self.configuration
+    self.configuration = build_config
+    try:
+      yield
+    finally:
+      self.configuration = config
+
+  def built_file_path(self, name, type=None, **kw):
+    result = []
+    chdir = kw.get('chdir')
+    if chdir:
+      result.append(chdir)
+    result.append('out')
+    result.append(self.configuration_dirname())
+    subdir = kw.get('subdir')
+    if subdir and type != self.SHARED_LIB:
+      result.append(subdir)
+    result.append(self.built_file_basename(name, type, **kw))
+    return self.workpath(*result)
+
+  def up_to_date(self, gyp_file, target=None, **kw):
+    result = self.build(gyp_file, target, **kw)
+    if not result:
+      stdout = self.stdout()
+      if 'ninja: no work to do' not in stdout:
+        self.report_not_up_to_date()
+        self.fail_test()
+    return result
+
+  def run_built_executable(self, name, *args, **kw):
+    """
+    Runs an executable built by xcodebuild + ninja.
+    """
+    configuration = self.configuration_dirname()
+    os.environ['DYLD_LIBRARY_PATH'] = os.path.join('out', configuration)
+    # Enclosing the name in a list avoids prepending the original dir.
+    program = [self.built_file_path(name, type=self.EXECUTABLE, **kw)]
+    return self.run(program=program, *args, **kw)
+
+
 format_class_list = [
   TestGypGypd,
   TestGypAndroid,
   TestGypCMake,
   TestGypMake,
   TestGypMSVS,
+  TestGypMSVSNinja,
   TestGypNinja,
   TestGypXcode,
+  TestGypXcodeNinja,
 ]
 
 def TestGyp(*args, **kw):