libv8 3.11.8.17 → 3.16.14.0

data/vendor/v8/tools/testrunner/server/constants.py

@@ -0,0 +1,51 @@
+# Copyright 2012 the V8 project authors. All rights reserved.
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+#       notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+#       copyright notice, this list of conditions and the following
+#       disclaimer in the documentation and/or other materials provided
+#       with the distribution.
+#     * Neither the name of Google Inc. nor the names of its
+#       contributors may be used to endorse or promote products derived
+#       from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+
+CLIENT_PORT = 9991  # Port for the local client to connect to.
+PEER_PORT = 9992  # Port for peers on the network to connect to.
+PRESENCE_PORT = 9993  # Port for presence daemon.
+STATUS_PORT = 9994  # Port for network requests not related to workpackets.
+
+END_OF_STREAM = "end of dtest stream"  # Marker for end of network requests.
+SIZE_T = 4  # Number of bytes used for network request size header.
+
+# Messages understood by the local request handler.
+ADD_TRUSTED = "add trusted"
+INFORM_DURATION = "inform about duration"
+REQUEST_PEERS = "get peers"
+UNRESPONSIVE_PEER = "unresponsive peer"
+REQUEST_PUBKEY_FINGERPRINT = "get pubkey fingerprint"
+REQUEST_STATUS = "get status"
+UPDATE_PERF = "update performance"
+
+# Messages understood by the status request handler.
+LIST_TRUSTED_PUBKEYS = "list trusted pubkeys"
+GET_SIGNED_PUBKEY = "pass on signed pubkey"
+NOTIFY_NEW_TRUSTED = "new trusted peer"
+TRUST_YOU_NOW = "trust you now"
+DO_YOU_TRUST = "do you trust"

data/vendor/v8/tools/testrunner/server/daemon.py

@@ -0,0 +1,147 @@
+#!/usr/bin/env python
+
+# This code has been written by Sander Marechal and published at:
+# http://www.jejik.com/articles/2007/02/a_simple_unix_linux_daemon_in_python/
+# where the author has placed it in the public domain (see comment #6 at
+# http://www.jejik.com/articles/2007/02/a_simple_unix_linux_daemon_in_python/#c6
+# ).
+# Some minor modifications have been made by the V8 authors. The work remains
+# in the public domain.
+
+import atexit
+import os
+from signal import SIGTERM
+from signal import SIGINT
+import sys
+import time
+
+
+class Daemon(object):
+  """
+  A generic daemon class.
+
+  Usage: subclass the Daemon class and override the run() method
+  """
+  def __init__(self, pidfile, stdin='/dev/null',
+               stdout='/dev/null', stderr='/dev/null'):
+    self.stdin = stdin
+    self.stdout = stdout
+    self.stderr = stderr
+    self.pidfile = pidfile
+
+  def daemonize(self):
+    """
+    do the UNIX double-fork magic, see Stevens' "Advanced
+    Programming in the UNIX Environment" for details (ISBN 0201563177)
+    http://www.erlenstar.demon.co.uk/unix/faq_2.html#SEC16
+    """
+    try:
+      pid = os.fork()
+      if pid > 0:
+        # exit first parent
+        sys.exit(0)
+    except OSError, e:
+      sys.stderr.write("fork #1 failed: %d (%s)\n" % (e.errno, e.strerror))
+      sys.exit(1)
+
+    # decouple from parent environment
+    os.chdir("/")
+    os.setsid()
+    os.umask(0)
+
+    # do second fork
+    try:
+      pid = os.fork()
+      if pid > 0:
+        # exit from second parent
+        sys.exit(0)
+    except OSError, e:
+      sys.stderr.write("fork #2 failed: %d (%s)\n" % (e.errno, e.strerror))
+      sys.exit(1)
+
+    # redirect standard file descriptors
+    sys.stdout.flush()
+    sys.stderr.flush()
+    si = file(self.stdin, 'r')
+    so = file(self.stdout, 'a+')
+    se = file(self.stderr, 'a+', 0)
+    # TODO: (debug) re-enable this!
+    #os.dup2(si.fileno(), sys.stdin.fileno())
+    #os.dup2(so.fileno(), sys.stdout.fileno())
+    #os.dup2(se.fileno(), sys.stderr.fileno())
+
+    # write pidfile
+    atexit.register(self.delpid)
+    pid = str(os.getpid())
+    file(self.pidfile, 'w+').write("%s\n" % pid)
+
+  def delpid(self):
+    os.remove(self.pidfile)
+
+  def start(self):
+    """
+    Start the daemon
+    """
+    # Check for a pidfile to see if the daemon already runs
+    try:
+      pf = file(self.pidfile, 'r')
+      pid = int(pf.read().strip())
+      pf.close()
+    except IOError:
+      pid = None
+
+    if pid:
+      message = "pidfile %s already exist. Daemon already running?\n"
+      sys.stderr.write(message % self.pidfile)
+      sys.exit(1)
+
+    # Start the daemon
+    self.daemonize()
+    self.run()
+
+  def stop(self):
+    """
+    Stop the daemon
+    """
+    # Get the pid from the pidfile
+    try:
+      pf = file(self.pidfile, 'r')
+      pid = int(pf.read().strip())
+      pf.close()
+    except IOError:
+      pid = None
+
+    if not pid:
+      message = "pidfile %s does not exist. Daemon not running?\n"
+      sys.stderr.write(message % self.pidfile)
+      return # not an error in a restart
+
+    # Try killing the daemon process
+    try:
+      # Give the process a one-second chance to exit gracefully.
+      os.kill(pid, SIGINT)
+      time.sleep(1)
+      while 1:
+        os.kill(pid, SIGTERM)
+        time.sleep(0.1)
+    except OSError, err:
+      err = str(err)
+      if err.find("No such process") > 0:
+        if os.path.exists(self.pidfile):
+          os.remove(self.pidfile)
+      else:
+        print str(err)
+        sys.exit(1)
+
+  def restart(self):
+    """
+    Restart the daemon
+    """
+    self.stop()
+    self.start()
+
+  def run(self):
+    """
+    You should override this method when you subclass Daemon. It will be
+    called after the process has been daemonized by start() or restart().
+    """

data/vendor/v8/tools/testrunner/server/local_handler.py

@@ -0,0 +1,119 @@
+# Copyright 2012 the V8 project authors. All rights reserved.
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+#       notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+#       copyright notice, this list of conditions and the following
+#       disclaimer in the documentation and/or other materials provided
+#       with the distribution.
+#     * Neither the name of Google Inc. nor the names of its
+#       contributors may be used to endorse or promote products derived
+#       from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+
+import socket
+import SocketServer
+import StringIO
+
+from . import compression
+from . import constants
+
+
+def LocalQuery(query):
+  sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+  code = sock.connect_ex(("localhost", constants.CLIENT_PORT))
+  if code != 0: return None
+  compression.Send(query, sock)
+  compression.Send(constants.END_OF_STREAM, sock)
+  rec = compression.Receiver(sock)
+  data = None
+  while not rec.IsDone():
+    data = rec.Current()
+    assert data[0] == query[0]
+    data = data[1]
+    rec.Advance()
+  sock.close()
+  return data
+
+
+class LocalHandler(SocketServer.BaseRequestHandler):
+  def handle(self):
+    rec = compression.Receiver(self.request)
+    while not rec.IsDone():
+      data = rec.Current()
+      action = data[0]
+
+      if action == constants.REQUEST_PEERS:
+        with self.server.daemon.peer_list_lock:
+          response = [ p.Pack() for p in self.server.daemon.peers
+                       if p.trusting_me ]
+        compression.Send([action, response], self.request)
+
+      elif action == constants.UNRESPONSIVE_PEER:
+        self.server.daemon.DeletePeer(data[1])
+
+      elif action == constants.REQUEST_PUBKEY_FINGERPRINT:
+        compression.Send([action, self.server.daemon.pubkey_fingerprint],
+                         self.request)
+
+      elif action == constants.REQUEST_STATUS:
+        compression.Send([action, self._GetStatusMessage()], self.request)
+
+      elif action == constants.ADD_TRUSTED:
+        fingerprint = self.server.daemon.CopyToTrusted(data[1])
+        compression.Send([action, fingerprint], self.request)
+
+      elif action == constants.INFORM_DURATION:
+        test_key = data[1]
+        test_duration = data[2]
+        arch = data[3]
+        mode = data[4]
+        self.server.daemon.AddPerfData(test_key, test_duration, arch, mode)
+
+      elif action == constants.UPDATE_PERF:
+        address = data[1]
+        perf = data[2]
+        self.server.daemon.UpdatePeerPerformance(data[1], data[2])
+
+      rec.Advance()
+    compression.Send(constants.END_OF_STREAM, self.request)
+
+  def _GetStatusMessage(self):
+    sio = StringIO.StringIO()
+    sio.write("Peers:\n")
+    with self.server.daemon.peer_list_lock:
+      for p in self.server.daemon.peers:
+        sio.write("%s\n" % p)
+    sio.write("My own jobs: %d, relative performance: %.2f\n" %
+              (self.server.daemon.jobs, self.server.daemon.relative_perf))
+    # Low-priority TODO: Return more information. Ideas:
+    #   - currently running anything,
+    #   - time since last job,
+    #   - time since last repository fetch
+    #   - number of workpackets/testcases handled since startup
+    #   - slowest test(s)
+    result = sio.getvalue()
+    sio.close()
+    return result
+
+
+class LocalSocketServer(SocketServer.ThreadingMixIn, SocketServer.TCPServer):
+  def __init__(self, daemon):
+    SocketServer.TCPServer.__init__(self, ("localhost", constants.CLIENT_PORT),
+                                    LocalHandler)
+    self.daemon = daemon

data/vendor/v8/tools/testrunner/server/main.py

@@ -0,0 +1,245 @@
+# Copyright 2012 the V8 project authors. All rights reserved.
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+#       notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+#       copyright notice, this list of conditions and the following
+#       disclaimer in the documentation and/or other materials provided
+#       with the distribution.
+#     * Neither the name of Google Inc. nor the names of its
+#       contributors may be used to endorse or promote products derived
+#       from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+
+import multiprocessing
+import os
+import shutil
+import subprocess
+import threading
+import time
+
+from . import daemon
+from . import local_handler
+from . import presence_handler
+from . import signatures
+from . import status_handler
+from . import work_handler
+from ..network import perfdata
+
+
+class Server(daemon.Daemon):
+
+  def __init__(self, pidfile, root, stdin="/dev/null",
+               stdout="/dev/null", stderr="/dev/null"):
+    super(Server, self).__init__(pidfile, stdin, stdout, stderr)
+    self.root = root
+    self.local_handler = None
+    self.local_handler_thread = None
+    self.work_handler = None
+    self.work_handler_thread = None
+    self.status_handler = None
+    self.status_handler_thread = None
+    self.presence_daemon = None
+    self.presence_daemon_thread = None
+    self.peers = []
+    self.jobs = multiprocessing.cpu_count()
+    self.peer_list_lock = threading.Lock()
+    self.perf_data_lock = None
+    self.presence_daemon_lock = None
+    self.datadir = os.path.join(self.root, "data")
+    pubkey_fingerprint_filename = os.path.join(self.datadir, "mypubkey")
+    with open(pubkey_fingerprint_filename) as f:
+      self.pubkey_fingerprint = f.read().strip()
+    self.relative_perf_filename = os.path.join(self.datadir, "myperf")
+    if os.path.exists(self.relative_perf_filename):
+      with open(self.relative_perf_filename) as f:
+        try:
+          self.relative_perf = float(f.read())
+        except:
+          self.relative_perf = 1.0
+    else:
+      self.relative_perf = 1.0
+
+  def run(self):
+    os.nice(20)
+    self.ip = presence_handler.GetOwnIP()
+    self.perf_data_manager = perfdata.PerfDataManager(self.datadir)
+    self.perf_data_lock = threading.Lock()
+
+    self.local_handler = local_handler.LocalSocketServer(self)
+    self.local_handler_thread = threading.Thread(
+        target=self.local_handler.serve_forever)
+    self.local_handler_thread.start()
+
+    self.work_handler = work_handler.WorkSocketServer(self)
+    self.work_handler_thread = threading.Thread(
+        target=self.work_handler.serve_forever)
+    self.work_handler_thread.start()
+
+    self.status_handler = status_handler.StatusSocketServer(self)
+    self.status_handler_thread = threading.Thread(
+        target=self.status_handler.serve_forever)
+    self.status_handler_thread.start()
+
+    self.presence_daemon = presence_handler.PresenceDaemon(self)
+    self.presence_daemon_thread = threading.Thread(
+        target=self.presence_daemon.serve_forever)
+    self.presence_daemon_thread.start()
+
+    self.presence_daemon.FindPeers()
+    time.sleep(0.5)  # Give those peers some time to reply.
+
+    with self.peer_list_lock:
+      for p in self.peers:
+        if p.address == self.ip: continue
+        status_handler.RequestTrustedPubkeys(p, self)
+
+    while True:
+      try:
+        self.PeriodicTasks()
+        time.sleep(60)
+      except Exception, e:
+        print("MAIN LOOP EXCEPTION: %s" % e)
+        self.Shutdown()
+        break
+      except KeyboardInterrupt:
+        self.Shutdown()
+        break
+
+  def Shutdown(self):
+    with open(self.relative_perf_filename, "w") as f:
+      f.write("%s" % self.relative_perf)
+    self.presence_daemon.shutdown()
+    self.presence_daemon.server_close()
+    self.local_handler.shutdown()
+    self.local_handler.server_close()
+    self.work_handler.shutdown()
+    self.work_handler.server_close()
+    self.status_handler.shutdown()
+    self.status_handler.server_close()
+
+  def PeriodicTasks(self):
+    # If we know peers we don't trust, see if someone else trusts them.
+    with self.peer_list_lock:
+      for p in self.peers:
+        if p.trusted: continue
+        if self.IsTrusted(p.pubkey):
+          p.trusted = True
+          status_handler.ITrustYouNow(p)
+          continue
+        for p2 in self.peers:
+          if not p2.trusted: continue
+          status_handler.TryTransitiveTrust(p2, p.pubkey, self)
+    # TODO: Ping for more peers waiting to be discovered.
+    # TODO: Update the checkout (if currently idle).
+
+  def AddPeer(self, peer):
+    with self.peer_list_lock:
+      for p in self.peers:
+        if p.address == peer.address:
+          return
+      self.peers.append(peer)
+    if peer.trusted:
+      status_handler.ITrustYouNow(peer)
+
+  def DeletePeer(self, peer_address):
+    with self.peer_list_lock:
+      for i in xrange(len(self.peers)):
+        if self.peers[i].address == peer_address:
+          del self.peers[i]
+          return
+
+  def MarkPeerAsTrusting(self, peer_address):
+    with self.peer_list_lock:
+      for p in self.peers:
+        if p.address == peer_address:
+          p.trusting_me = True
+          break
+
+  def UpdatePeerPerformance(self, peer_address, performance):
+    with self.peer_list_lock:
+      for p in self.peers:
+        if p.address == peer_address:
+          p.relative_performance = performance
+
+  def CopyToTrusted(self, pubkey_filename):
+    with open(pubkey_filename, "r") as f:
+      lines = f.readlines()
+      fingerprint = lines[-1].strip()
+    target_filename = self._PubkeyFilename(fingerprint)
+    shutil.copy(pubkey_filename, target_filename)
+    with self.peer_list_lock:
+      for peer in self.peers:
+        if peer.address == self.ip: continue
+        if peer.pubkey == fingerprint:
+          status_handler.ITrustYouNow(peer)
+        else:
+          result = self.SignTrusted(fingerprint)
+          status_handler.NotifyNewTrusted(peer, result)
+    return fingerprint
+
+  def _PubkeyFilename(self, pubkey_fingerprint):
+    return os.path.join(self.root, "trusted", "%s.pem" % pubkey_fingerprint)
+
+  def IsTrusted(self, pubkey_fingerprint):
+    return os.path.exists(self._PubkeyFilename(pubkey_fingerprint))
+
+  def ListTrusted(self):
+    path = os.path.join(self.root, "trusted")
+    if not os.path.exists(path): return []
+    return [ f[:-4] for f in os.listdir(path) if f.endswith(".pem") ]
+
+  def SignTrusted(self, pubkey_fingerprint):
+    if not self.IsTrusted(pubkey_fingerprint):
+      return []
+    filename = self._PubkeyFilename(pubkey_fingerprint)
+    result = signatures.ReadFileAndSignature(filename)  # Format: [key, sig].
+    return [pubkey_fingerprint, result[0], result[1], self.pubkey_fingerprint]
+
+  def AcceptNewTrusted(self, data):
+    # The format of |data| matches the return value of |SignTrusted()|.
+    if not data: return
+    fingerprint = data[0]
+    pubkey = data[1]
+    signature = data[2]
+    signer = data[3]
+    if not self.IsTrusted(signer):
+      return
+    if self.IsTrusted(fingerprint):
+      return  # Already trust this guy.
+    filename = self._PubkeyFilename(fingerprint)
+    signer_pubkeyfile = self._PubkeyFilename(signer)
+    if not signatures.VerifySignature(filename, pubkey, signature,
+                                      signer_pubkeyfile):
+      return
+    return  # Nothing more to do.
+
+  def AddPerfData(self, test_key, duration, arch, mode):
+    data_store = self.perf_data_manager.GetStore(arch, mode)
+    data_store.RawUpdatePerfData(str(test_key), duration)
+
+  def CompareOwnPerf(self, test, arch, mode):
+    data_store = self.perf_data_manager.GetStore(arch, mode)
+    observed = data_store.FetchPerfData(test)
+    if not observed: return
+    own_perf_estimate = observed / test.duration
+    with self.perf_data_lock:
+      kLearnRateLimiter = 9999
+      self.relative_perf *= kLearnRateLimiter
+      self.relative_perf += own_perf_estimate
+      self.relative_perf /= (kLearnRateLimiter + 1)