sauce 1.0.2 → 2.0.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (56) hide show
  1. data/.document +5 -0
  2. data/.gitignore +30 -0
  3. data/Gemfile +16 -0
  4. data/README.markdown +39 -145
  5. data/Rakefile +46 -20
  6. data/bin/sauce +72 -61
  7. data/gemfiles/rails2.gemfile +10 -0
  8. data/gemfiles/rails2.gemfile.lock +77 -0
  9. data/gemfiles/rails3.gemfile +9 -0
  10. data/gemfiles/rails3.gemfile.lock +137 -0
  11. data/lib/generators/sauce/install/install_generator.rb +1 -2
  12. data/lib/sauce.rb +0 -22
  13. data/lib/sauce/capybara.rb +70 -32
  14. data/lib/sauce/capybara/cucumber.rb +121 -0
  15. data/lib/sauce/config.rb +57 -13
  16. data/lib/sauce/connect.rb +22 -11
  17. data/lib/sauce/integrations.rb +27 -69
  18. data/lib/sauce/jasmine.rb +35 -0
  19. data/lib/sauce/jasmine/rake.rb +47 -0
  20. data/lib/sauce/jasmine/runner.rb +4 -0
  21. data/lib/sauce/job.rb +10 -6
  22. data/lib/sauce/raketasks.rb +0 -21
  23. data/lib/sauce/selenium.rb +9 -18
  24. data/lib/sauce/utilities.rb +0 -17
  25. data/sauce.gemspec +8 -60
  26. data/spec/integration/connect_integration_spec.rb +84 -0
  27. data/spec/sauce/capybara/cucumber_spec.rb +156 -0
  28. data/spec/sauce/capybara/spec_helper.rb +42 -0
  29. data/spec/sauce/capybara_spec.rb +121 -0
  30. data/spec/sauce/config_spec.rb +239 -0
  31. data/spec/sauce/jasmine_spec.rb +49 -0
  32. data/spec/sauce/selenium_spec.rb +57 -0
  33. data/spec/spec_helper.rb +4 -0
  34. data/support/Sauce-Connect.jar +0 -0
  35. data/test/test_integrations.rb +202 -0
  36. data/test/test_testcase.rb +13 -0
  37. metadata +170 -171
  38. data/examples/helper.rb +0 -16
  39. data/examples/other_spec.rb +0 -7
  40. data/examples/saucelabs_spec.rb +0 -12
  41. data/examples/test_saucelabs.rb +0 -13
  42. data/examples/test_saucelabs2.rb +0 -9
  43. data/support/sauce_connect +0 -938
  44. data/support/selenium-server.jar +0 -0
  45. data/support/simplejson/LICENSE.txt +0 -19
  46. data/support/simplejson/__init__.py +0 -437
  47. data/support/simplejson/decoder.py +0 -421
  48. data/support/simplejson/encoder.py +0 -501
  49. data/support/simplejson/ordered_dict.py +0 -119
  50. data/support/simplejson/scanner.py +0 -77
  51. data/support/simplejson/tool.py +0 -39
  52. data/test/test_config.rb +0 -112
  53. data/test/test_connect.rb +0 -45
  54. data/test/test_job.rb +0 -13
  55. data/test/test_selenium.rb +0 -50
  56. data/test/test_selenium2.rb +0 -9
data/examples/helper.rb DELETED
@@ -1,16 +0,0 @@
1
- require 'rubygems'
2
-
3
- $LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
4
- $LOAD_PATH.unshift(File.dirname(__FILE__))
5
- require 'sauce'
6
-
7
- Sauce.config do |config|
8
- config.browsers = [
9
- ["Linux", "firefox", "3.6."],
10
- #["Windows 2003", "safariproxy", "5."]
11
- ]
12
- config.browser_url = "http://saucelabs.com"
13
-
14
- #config.application_host = "localhost"
15
- #config.application_port = "4444"
16
- end
@@ -1,7 +0,0 @@
1
- require File.join(File.dirname(__FILE__), "helper")
2
-
3
- describe "The login form", :type => :selenium do
4
- it "exists" do
5
- selenium.open "/"
6
- end
7
- end
@@ -1,12 +0,0 @@
1
- require File.join(File.dirname(__FILE__), "helper")
2
-
3
- describe "The Sauce website", :type => :selenium do
4
- it "works" do
5
- selenium.open "/"
6
- page.is_text_present("Sauce Labs").should be_true
7
- end
8
-
9
- it "has a pricing page" do
10
- selenium.open "/pricing"
11
- end
12
- end
@@ -1,13 +0,0 @@
1
- require File.join(File.dirname(__FILE__), "helper")
2
-
3
- class TestSaucelabs < Sauce::TestCase
4
- def test_basic_functionality
5
- selenium.open "/"
6
- assert page.is_text_present("Sauce Labs")
7
- end
8
-
9
- def test_pricing_page
10
- selenium.open "/pricing"
11
- assert page.is_text_present("$")
12
- end
13
- end
@@ -1,9 +0,0 @@
1
- require File.join(File.dirname(__FILE__), "helper")
2
-
3
- class TestSaucelabs2 < Sauce::TestCase
4
- def test_basic_functionality_two
5
- selenium.open "/"
6
- assert page.is_text_present("Sauce Labs")
7
- end
8
- end
9
-
@@ -1,938 +0,0 @@
1
- #!/usr/bin/env python
2
- # encoding: utf-8
3
- from __future__ import with_statement
4
-
5
- # TODO:
6
- # * Move to REST API v1
7
- # * windows: SSH link healthcheck (PuTTY session file hack?)
8
- # * Daemonizing
9
- # * issue: windows: no os.fork()
10
- # * issue: unix: null file descriptors causes Expect script to fail
11
- # * Renew tunnel lease (backend not implemented)
12
- # * Check tunnel machine ports are open (backend not implemented)
13
-
14
- import os
15
- import sys
16
- import re
17
- import optparse
18
- import logging
19
- import logging.handlers
20
- import signal
21
- import atexit
22
- import httplib
23
- import urllib2
24
- import subprocess
25
- import socket
26
- import time
27
- import platform
28
- import tempfile
29
- import string
30
- from base64 import b64encode
31
- from collections import defaultdict
32
- from contextlib import closing
33
- from functools import wraps
34
-
35
- try:
36
- import json
37
- except ImportError:
38
- import simplejson as json # Python 2.5 dependency
39
-
40
- NAME = "sauce_connect"
41
- RELEASE = 25
42
- DISPLAY_VERSION = "%s release %s" % (NAME, RELEASE)
43
- PRODUCT_NAME = u"Sauce Connect"
44
- VERSIONS_URL = "http://saucelabs.com/versions.json"
45
-
46
- RETRY_PROVISION_MAX = 4
47
- RETRY_BOOT_MAX = 4
48
- RETRY_REST_WAIT = 5
49
- RETRY_REST_MAX = 6
50
- REST_POLL_WAIT = 3
51
- RETRY_SSH_MAX = 4
52
- HEALTH_CHECK_INTERVAL = 15
53
- HEALTH_CHECK_FAIL = 5 * 60 # no good check after this amount of time == fail
54
- LATENCY_LOG = 150 # log when making connections takes this many ms
55
- LATENCY_WARNING = 350 # warn when making connections takes this many ms
56
- SIGNALS_RECV_MAX = 4 # used with --allow-unclean-exit
57
-
58
- is_windows = platform.system().lower() == "windows"
59
- is_openbsd = platform.system().lower() == "openbsd"
60
- logger = logging.getLogger(NAME)
61
-
62
-
63
- class DeleteRequest(urllib2.Request):
64
-
65
- def get_method(self):
66
- return "DELETE"
67
-
68
-
69
- class HTTPResponseError(Exception):
70
-
71
- def __init__(self, msg):
72
- self.msg = msg
73
-
74
- def __str__(self):
75
- return "HTTP server responded with '%s' (expected 'OK')" % self.msg
76
-
77
-
78
- class TunnelMachineError(Exception):
79
- pass
80
-
81
-
82
- class TunnelMachineProvisionError(TunnelMachineError):
83
- pass
84
-
85
-
86
- class TunnelMachineBootError(TunnelMachineError):
87
- pass
88
-
89
-
90
- class TunnelMachine(object):
91
-
92
- _host_search = re.compile("//([^/]+)").search
93
-
94
- def __init__(self, rest_url, user, password, domains, ssh_port, metadata=None):
95
- self.user = user
96
- self.password = password
97
- self.domains = set(domains)
98
- self.ssh_port = ssh_port
99
- self.metadata = metadata or dict()
100
-
101
- self.reverse_ssh = None
102
- self.is_shutdown = False
103
- self.base_url = "%(rest_url)s/%(user)s/tunnels" % locals()
104
- self.rest_host = self._host_search(rest_url).group(1)
105
- self.basic_auth_header = {"Authorization": "Basic %s"
106
- % b64encode("%s:%s" % (user, password))}
107
-
108
- self._set_urlopen(user, password)
109
-
110
- for attempt in xrange(1, RETRY_PROVISION_MAX):
111
- try:
112
- self._provision_tunnel()
113
- break
114
- except TunnelMachineProvisionError, e:
115
- logger.warning(e)
116
- if attempt == RETRY_PROVISION_MAX:
117
- raise TunnelMachineError(
118
- "!! Could not provision tunnel host. Please contact "
119
- "help@saucelabs.com.")
120
-
121
- def _set_urlopen(self, user, password):
122
- # always send Basic Auth header (HTTPBasicAuthHandler was unreliable)
123
- opener = urllib2.build_opener()
124
- opener.addheaders = self.basic_auth_header.items()
125
- self.urlopen = opener.open
126
-
127
- # decorator
128
- def _retry_rest_api(f):
129
- @wraps(f)
130
- def wrapper(*args, **kwargs):
131
- previous_failed = False
132
- for attempt in xrange(1, RETRY_REST_MAX + 1):
133
- try:
134
- result = f(*args, **kwargs)
135
- if previous_failed:
136
- logger.info("Connection succeeded")
137
- return result
138
- except (HTTPResponseError,
139
- urllib2.URLError, httplib.HTTPException,
140
- socket.gaierror, socket.error), e:
141
- logger.warning("Problem connecting to Sauce Labs REST API "
142
- "(%s)", str(e))
143
- if attempt == RETRY_REST_MAX:
144
- raise TunnelMachineError(
145
- "Could not reach Sauce Labs REST API after %d "
146
- "tries. Is your network down or firewalled?"
147
- % attempt)
148
- previous_failed = True
149
- logger.debug("Retrying in %ds", RETRY_REST_WAIT)
150
- time.sleep(RETRY_REST_WAIT)
151
- except Exception, e:
152
- raise TunnelMachineError(
153
- "An error occurred while contacting Sauce Labs REST "
154
- "API (%s). Please contact help@saucelabs.com." % str(e))
155
- return wrapper
156
-
157
- @_retry_rest_api
158
- def _get_doc(self, url_or_req):
159
- with closing(self.urlopen(url_or_req)) as resp:
160
- if resp.msg != "OK":
161
- raise HTTPResponseError(resp.msg)
162
- return json.loads(resp.read())
163
-
164
- def _provision_tunnel(self):
165
- # Shutdown any tunnel using a requested domain
166
- kill_list = set()
167
- for doc in self._get_doc(self.base_url):
168
- if not doc.get('DomainNames'):
169
- continue
170
- if set(doc['DomainNames']) & self.domains:
171
- kill_list.add(doc['id'])
172
- if kill_list:
173
- logger.info(
174
- "Shutting down other tunnel hosts using requested domains")
175
- for tunnel_id in kill_list:
176
- for attempt in xrange(1, 4): # try a few times, then bail
177
- logger.debug(
178
- "Shutting down old tunnel host: %s" % tunnel_id)
179
- url = "%s/%s" % (self.base_url, tunnel_id)
180
- doc = self._get_doc(DeleteRequest(url=url))
181
- if not doc.get('ok'):
182
- logger.warning("Old tunnel host failed to shutdown?")
183
- continue
184
- doc = self._get_doc(url)
185
- while doc.get('Status') not in ["halting", "terminated"]:
186
- logger.debug(
187
- "Waiting for old tunnel host to start halting")
188
- time.sleep(REST_POLL_WAIT)
189
- doc = self._get_doc(url)
190
- break
191
-
192
- # Request a tunnel machine
193
- headers = {"Content-Type": "application/json"}
194
- data = json.dumps(dict(DomainNames=list(self.domains),
195
- Metadata=self.metadata,
196
- SSHPort=self.ssh_port))
197
- req = urllib2.Request(url=self.base_url, headers=headers, data=data)
198
- doc = self._get_doc(req)
199
- if doc.get('error'):
200
- raise TunnelMachineProvisionError(doc['error'])
201
- for key in ['ok', 'id']:
202
- if not doc.get(key):
203
- raise TunnelMachineProvisionError(
204
- "Document for provisioned tunnel host is missing the key "
205
- "or value for '%s'" % key)
206
- self.id = doc['id']
207
- self.url = "%s/%s" % (self.base_url, self.id)
208
- logger.info("Tunnel host is provisioned (%s)" % self.id)
209
-
210
- def ready_wait(self):
211
- """Wait for the machine to reach the 'running' state."""
212
- previous_status = None
213
- while True:
214
- doc = self._get_doc(self.url)
215
- status = doc.get('Status')
216
- if status == "running":
217
- break
218
- if status in ["halting", "terminated"]:
219
- raise TunnelMachineBootError("Tunnel host was shutdown")
220
- if status != previous_status:
221
- logger.info("Tunnel host is %s .." % status)
222
- previous_status = status
223
- time.sleep(REST_POLL_WAIT)
224
- self.host = doc['Host']
225
- logger.info("Tunnel host is running at %s" % self.host)
226
-
227
- def shutdown(self):
228
- if self.is_shutdown:
229
- return
230
-
231
- if self.reverse_ssh:
232
- self.reverse_ssh.stop()
233
-
234
- logger.info("Shutting down tunnel host (please wait)")
235
- logger.debug("Tunnel host ID: %s" % self.id)
236
-
237
- try:
238
- doc = self._get_doc(DeleteRequest(url=self.url))
239
- except TunnelMachineError, e:
240
- logger.warning("Unable to shut down tunnel host")
241
- logger.debug("Shut down failed because: %s", str(e))
242
- self.is_shutdown = True # fuhgeddaboudit
243
- return
244
- assert doc.get('ok')
245
-
246
- previous_status = None
247
- while True:
248
- doc = self._get_doc(self.url)
249
- status = doc.get('Status')
250
- if status == "terminated":
251
- break
252
- if status != previous_status:
253
- logger.info("Tunnel host is %s .." % status)
254
- previous_status = status
255
- time.sleep(REST_POLL_WAIT)
256
- logger.info("Tunnel host is shutdown")
257
- self.is_shutdown = True
258
-
259
- # Make us usable with contextlib.closing
260
- close = shutdown
261
-
262
- def check_running(self):
263
- doc = self._get_doc(self.url)
264
- if doc.get('Status') == "running":
265
- return
266
- raise TunnelMachineError(
267
- "The tunnel host is no longer running. It may have been shutdown "
268
- "via the website or by another Sauce Connect script requesting these "
269
- "domains: %s" % list(self.domains))
270
-
271
-
272
- class HealthCheckFail(Exception):
273
- pass
274
-
275
-
276
- class HealthChecker(object):
277
-
278
- latency_log = LATENCY_LOG
279
-
280
- def __init__(self, host, ports, fail_msg=None):
281
- """fail_msg can include '%(host)s' and '%(port)d'"""
282
- self.host = host
283
- self.fail_msg = fail_msg
284
- if not self.fail_msg:
285
- self.fail_msg = ("!! Your tests will fail while your network "
286
- "can not get to %(host)s:%(port)d.")
287
- self.ports = frozenset(int(p) for p in ports)
288
- self.last_tcp_connect = defaultdict(time.time)
289
- self.last_tcp_ping = defaultdict(lambda: None)
290
-
291
- def _tcp_ping(self, port):
292
- with closing(socket.socket()) as sock:
293
- start_time = time.time()
294
- try:
295
- sock.connect((self.host, port))
296
- return int(1000 * (time.time() - start_time))
297
- except (socket.gaierror, socket.error), e:
298
- logger.warning("Could not connect to %s:%s (%s)",
299
- self.host, port, str(e))
300
-
301
- def check(self):
302
- now = time.time()
303
- for port in self.ports:
304
- ping_time = self._tcp_ping(port)
305
- if ping_time is not None:
306
- # TCP connection succeeded
307
- self.last_tcp_connect[port] = now
308
- result = (self.host, port, ping_time)
309
-
310
- if ping_time >= self.latency_log:
311
- logger.debug("Connected to %s:%s in in %dms" % result)
312
-
313
- if ping_time >= LATENCY_WARNING:
314
- if (self.last_tcp_ping[port] is None
315
- or self.last_tcp_ping[port] < LATENCY_WARNING):
316
- logger.warn("High latency to %s:%s (took %dms to "
317
- "connect); tests may run slowly" % result)
318
-
319
- if (ping_time < (LATENCY_WARNING / 2)
320
- and self.last_tcp_ping[port]
321
- and self.last_tcp_ping[port] >= LATENCY_WARNING):
322
- logger.info("Latency to %s:%s has lowered (took %dms to "
323
- "connect)" % result)
324
-
325
- if self.last_tcp_ping[port] is None:
326
- logger.info("Succesfully connected to %s:%s in %dms" % result)
327
-
328
- self.last_tcp_ping[port] = ping_time
329
- continue
330
-
331
- # TCP connection failed
332
- self.last_tcp_ping[port] = ping_time
333
- logger.warning(self.fail_msg % dict(host=self.host, port=port))
334
- if now - self.last_tcp_connect[port] > HEALTH_CHECK_FAIL:
335
- raise HealthCheckFail(
336
- "Could not connect to %s:%s for over %s seconds"
337
- % (self.host, port, HEALTH_CHECK_FAIL))
338
-
339
-
340
- class ReverseSSHError(Exception):
341
- pass
342
-
343
-
344
- class ReverseSSH(object):
345
-
346
- def __init__(self, tunnel, host, ports, tunnel_ports, ssh_port,
347
- use_ssh_config=False, debug=False):
348
- self.tunnel = tunnel
349
- self.host = host
350
- self.ports = ports
351
- self.tunnel_ports = tunnel_ports
352
- self.use_ssh_config = use_ssh_config
353
- self.ssh_port = ssh_port
354
- self.debug = debug
355
-
356
- self.proc = None
357
- self.readyfile = None
358
- self.stdout_f = None
359
- self.stderr_f = None
360
-
361
- if self.debug:
362
- logger.debug("ReverseSSH debugging is on.")
363
-
364
- def _check_dot_ssh_files(self):
365
- if not os.environ.get('HOME'):
366
- logger.debug("No HOME env, skipping .ssh file checks")
367
- return
368
-
369
- ssh_config_file = os.path.join(os.environ['HOME'], ".ssh", "config")
370
- if os.path.exists(ssh_config_file):
371
- logger.debug("Found %s" % ssh_config_file)
372
- if self.use_ssh_config:
373
- logger.warn("Using local SSH config")
374
-
375
- @property
376
- def _dash_Rs(self):
377
- dash_Rs = ""
378
- for port, tunnel_port in zip(self.ports, self.tunnel_ports):
379
- dash_Rs += "-R 0.0.0.0:%s:%s:%s " % (tunnel_port, self.host, port)
380
- return dash_Rs
381
-
382
- def get_plink_command(self):
383
- """Return the Windows SSH command."""
384
- verbosity = "-v" if self.debug else ""
385
- return ("plink\plink %s -P %s -l %s -pw %s -N %s %s"
386
- % (verbosity, self.ssh_port, self.tunnel.user, self.tunnel.password,
387
- self._dash_Rs, self.tunnel.host))
388
-
389
- def get_expect_script(self):
390
- """Return the Unix SSH command."""
391
- wait = "wait"
392
- if is_openbsd: # using 'wait;' hangs the script on OpenBSD
393
- wait = "wait -nowait;sleep 1" # hack
394
-
395
- verbosity = "-v" if self.debug else "-q"
396
- config_file = "" if self.use_ssh_config else "-F /dev/null"
397
- host_ip = socket.gethostbyname(self.tunnel.host)
398
- script = (
399
- "spawn ssh %s %s -p %s -l %s -o ServerAliveInterval=%s -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -N %s %s;"
400
- % (verbosity, config_file, self.ssh_port, self.tunnel.user,
401
- HEALTH_CHECK_INTERVAL, self._dash_Rs, self.tunnel.host) +
402
- "expect *password:;send -- %s\\r;" % self.tunnel.password +
403
- "expect -timeout -1 timeout")
404
- return script
405
-
406
- def _start_reverse_ssh(self, readyfile=None):
407
- self._check_dot_ssh_files()
408
- logger.info("Starting SSH process ..")
409
- if is_windows:
410
- cmd = "echo 'n' | %s" % self.get_plink_command()
411
- else:
412
- cmd = 'exec expect -c "%s"' % self.get_expect_script()
413
-
414
- # start ssh process
415
- if self.debug:
416
- self.stdout_f = tempfile.TemporaryFile()
417
- else:
418
- self.stdout_f = open(os.devnull)
419
- self.stderr_f = tempfile.TemporaryFile()
420
- self.proc = subprocess.Popen(
421
- cmd, shell=True, stdout=self.stdout_f, stderr=self.stderr_f)
422
- self.tunnel.reverse_ssh = self # BUG: circular ref
423
- time.sleep(3) # HACK: some startup time
424
-
425
- # ssh process is running
426
- announced_running = False
427
-
428
- # setup recurring healthchecks
429
- forwarded_health = HealthChecker(self.host, self.ports)
430
- tunnel_health = HealthChecker(host=self.tunnel.host, ports=[self.ssh_port],
431
- fail_msg="!! Your tests may fail because your network can not get "
432
- "to the tunnel host (%s:%d)." % (self.tunnel.host, self.ssh_port))
433
-
434
- start_time = int(time.time())
435
- while self.proc.poll() is None:
436
- now = int(time.time())
437
- if not announced_running:
438
- # guarantee we health check on first iteration
439
- now = start_time
440
- if (now - start_time) % HEALTH_CHECK_INTERVAL == 0:
441
- self.tunnel.check_running()
442
- try:
443
- forwarded_health.check()
444
- tunnel_health.check()
445
- except HealthCheckFail, e:
446
- raise ReverseSSHError(e)
447
- if not announced_running:
448
- logger.info("SSH is running. You may start your tests.")
449
- if readyfile:
450
- self.readyfile = readyfile
451
- f = open(readyfile, 'w')
452
- f.close()
453
- announced_running = True
454
- time.sleep(1)
455
-
456
- # ssh process has exited
457
- self._log_output()
458
- if self.proc.returncode != 0:
459
- logger.warning("SSH process exited with error code %d",
460
- self.proc.returncode)
461
- else:
462
- logger.info("SSH process exited (maybe due to network problems)")
463
-
464
- return self.proc.returncode
465
-
466
- def _log_output(self):
467
- if not self.stderr_f.closed:
468
- self.stderr_f.seek(0)
469
- reverse_ssh_stderr = self.stderr_f.read().strip()
470
- self.stderr_f.close()
471
-
472
- if reverse_ssh_stderr:
473
- logger.debug("ReverseSSH stderr was:\n%s\n" % reverse_ssh_stderr)
474
-
475
- if not self.stdout_f.closed:
476
- self.stdout_f.seek(0)
477
- reverse_ssh_stdout = self.stdout_f.read().strip()
478
- self.stdout_f.close()
479
-
480
- if self.debug:
481
- logger.debug("ReverseSSH stdout was:\n%s\n" % reverse_ssh_stdout)
482
-
483
- def _rm_readyfile(self):
484
- if self.readyfile and os.path.exists(self.readyfile):
485
- try:
486
- os.remove(self.readyfile)
487
- except OSError, e:
488
- logger.error("Couldn't remove %s: %s", self.readyfile, str(e))
489
-
490
- def stop(self):
491
- self._rm_readyfile()
492
- if not self.proc or self.proc.poll() is not None: # not running, done
493
- return
494
-
495
- if not is_windows: # windows no have kill()
496
- try:
497
- os.kill(self.proc.pid, signal.SIGHUP)
498
- logger.debug("Sent SIGHUP to PID %d", self.proc.pid)
499
- except OSError:
500
- pass
501
- self._log_output()
502
-
503
- def run(self, readyfile=None):
504
- clean_exit = False
505
- for attempt in xrange(1, RETRY_SSH_MAX + 1):
506
- # returncode 0 will happen due to ServerAlive checks failing.
507
- # this may result in a listening port forwarding nowhere, so
508
- # don't bother restarting the SSH connection.
509
- # TODO: revisit if server uses OpenSSH instead of Twisted SSH
510
- if self._start_reverse_ssh(readyfile) == 0:
511
- clean_exit = True
512
- break
513
- if attempt < RETRY_SSH_MAX:
514
- logger.debug("Will restart SSH in 3 seconds")
515
- time.sleep(3) # wait a bit for old connections to close
516
- self._rm_readyfile()
517
- if not clean_exit:
518
- raise ReverseSSHError(
519
- "SSH process errored %d times (bad network?)" % attempt)
520
-
521
-
522
- def peace_out(tunnel=None, returncode=0, atexit=False):
523
- """Shutdown the tunnel and raise SystemExit."""
524
- if tunnel:
525
- tunnel.shutdown()
526
- if not atexit:
527
- logger.info("\ Exiting /")
528
- raise SystemExit(returncode)
529
- else:
530
- logger.debug("-- fin --")
531
-
532
-
533
- def setup_signal_handler(tunnel, options):
534
- signal_count = defaultdict(int)
535
- signal_name = {}
536
-
537
- def sig_handler(signum, frame):
538
- if options.allow_unclean_exit:
539
- signal_count[signum] += 1
540
- if signal_count[signum] > SIGNALS_RECV_MAX:
541
- logger.info(
542
- "Received %s too many times (%d). Making unclean "
543
- "exit now!", signal_name[signum], signal_count[signum])
544
- raise SystemExit(1)
545
- logger.info("Received signal %s", signal_name[signum])
546
- peace_out(tunnel) # exits
547
-
548
- # TODO: ?? remove SIGTERM when we implement tunnel leases
549
- if is_windows:
550
- supported_signals = ["SIGABRT", "SIGBREAK", "SIGINT", "SIGTERM"]
551
- else:
552
- supported_signals = ["SIGHUP", "SIGINT", "SIGQUIT", "SIGTERM"]
553
- for sig in supported_signals:
554
- signum = getattr(signal, sig)
555
- signal_name[signum] = sig
556
- signal.signal(signum, sig_handler)
557
-
558
-
559
- def check_version():
560
- failed_msg = "Skipping version check"
561
- logger.debug("Checking version")
562
- try:
563
- with closing(urllib2.urlopen(VERSIONS_URL)) as resp:
564
- assert resp.msg == "OK", "Got HTTP response %s" % resp.msg
565
- version_doc = json.loads(resp.read())
566
- except (urllib2.URLError, AssertionError, ValueError), e:
567
- logger.debug("Could not check version: %s", str(e))
568
- logger.info(failed_msg)
569
- return
570
- try:
571
- version = version_doc[PRODUCT_NAME][u'version']
572
- download_url = version_doc[PRODUCT_NAME][u'download_url']
573
- except KeyError, e:
574
- logger.debug("Bad version doc, missing key: %s", str(e))
575
- logger.info(failed_msg)
576
- return
577
-
578
- try:
579
- latest = int(version.partition("-")[2].strip(string.ascii_letters))
580
- except (IndexError, ValueError), e:
581
- logger.debug("Couldn't parse release number: %s", str(e))
582
- logger.info(failed_msg)
583
- return
584
- if RELEASE < latest:
585
- msgs = ["** This version of %s is outdated." % PRODUCT_NAME,
586
- "** Please update with %s" % download_url]
587
- for update_msg in msgs:
588
- logger.warning(update_msg)
589
- for update_msg in msgs:
590
- sys.stderr.write("%s\n" % update_msg)
591
- time.sleep(15)
592
-
593
-
594
- def setup_logging(logfile=None, quiet=False):
595
- logger.setLevel(logging.DEBUG)
596
-
597
- if not quiet:
598
- stdout = logging.StreamHandler(sys.stdout)
599
- stdout.setLevel(logging.INFO)
600
- stdout.setFormatter(logging.Formatter("%(asctime)s - %(message)s"))
601
- logger.addHandler(stdout)
602
-
603
- if logfile:
604
- if not quiet:
605
- print "* Debug messages will be sent to %s" % logfile
606
- fileout = logging.handlers.RotatingFileHandler(
607
- filename=logfile, maxBytes=128 * 1024, backupCount=8)
608
- fileout.setLevel(logging.DEBUG)
609
- fileout.setFormatter(logging.Formatter(
610
- "%(asctime)s - %(name)s:%(lineno)d - %(levelname)s - %(message)s"))
611
- logger.addHandler(fileout)
612
-
613
-
614
- def check_domains(domains):
615
- """Display error and exit script if any requested domains are invalid."""
616
-
617
- for dom in domains:
618
- # no URLs
619
- if '/' in dom:
620
- sys.stderr.write(
621
- "Error: Domain contains illegal character '/' in it.\n")
622
- print " Did you use a URL instead of just the domain?\n"
623
- print "Examples: -d example.com -d '*.example.com' -d another.site"
624
- print
625
- raise SystemExit(1)
626
-
627
- # no numerical addresses
628
- if all(map(lambda c: c.isdigit() or c == '.', dom)):
629
- sys.stderr.write("Error: Domain must be a hostname not an IP\n")
630
- print
631
- print "Examples: -d example.com -d '*.example.com' -d another.site"
632
- print
633
- raise SystemExit(1)
634
-
635
- # need a dot and 2 char TLD
636
- # NOTE: if this restriction is relaxed, still check for "localhost"
637
- if '.' not in dom or len(dom.rpartition('.')[2]) < 2:
638
- sys.stderr.write(
639
- "Error: Domain requires a TLD of 2 characters or more\n")
640
- print
641
- print "Example: -d example.tld -d '*.example.tld' -d another.tld"
642
- print
643
- raise SystemExit(1)
644
-
645
- # *.com will break uploading to S3
646
- if dom == "*.com":
647
- sys.stderr.write(
648
- "Error: Matching *.com will break videos and logs. Use a hostname.\n")
649
- print
650
- print "Example: -d example.com -d *.example.com"
651
- print
652
- raise SystemExit(1)
653
-
654
-
655
- def get_options():
656
- usage = """
657
- Usage: %(name)s -u <user> -k <api_key> -s <webserver> -d <domain> [options]
658
-
659
- Examples:
660
- Have tests for example.com go to a staging server on your intranet:
661
- %(name)s -u user -k 123-abc -s staging.local -d example.com
662
-
663
- Have HTTP and HTTPS traffic for *.example.com go to the staging server:
664
- %(name)s -u user -k 123-abc -s staging.local -p 80 -p 443 \\
665
- -d example.com -d *.example.com
666
-
667
- Have tests for example.com go to your local machine on port 5000:
668
- %(name)s -u user -k 123-abc -s 127.0.0.1 -t 80 -p 5000 -d example.com
669
-
670
- Performance tip:
671
- It is highly recommended you run this script on the same machine as your
672
- test server (i.e., you would use "-s 127.0.0.1" or "-s localhost"). Using
673
- a remote server introduces higher latency (slower web requests) and is
674
- another failure point.
675
- """ % dict(name=NAME)
676
-
677
- usage = usage.strip()
678
- logfile = "%s.log" % NAME
679
-
680
- op = optparse.OptionParser(usage=usage, version=DISPLAY_VERSION)
681
- op.add_option("-u", "--user", "--username",
682
- help="Your Sauce Labs account name.")
683
- op.add_option("-k", "--api-key",
684
- help="On your account at https://saucelabs.com/account")
685
- op.add_option("-s", "--host", default="localhost",
686
- help="Host to forward requests to. [%default]")
687
- op.add_option("-p", "--port", metavar="PORT",
688
- action="append", dest="ports", default=[],
689
- help="Forward to this port on HOST. Can be specified "
690
- "multiple times. [80]")
691
- op.add_option("-d", "--domain", action="append", dest="domains",
692
- help="Repeat for each domain you want to forward requests for. "
693
- "Example: -d example.test -d '*.example.test'")
694
- op.add_option("-q", "--quiet", action="store_true", default=False,
695
- help="Minimize standard output (see %s)" % logfile)
696
-
697
- og = optparse.OptionGroup(op, "Advanced options")
698
- og.add_option("-t", "--tunnel-port", metavar="TUNNEL_PORT",
699
- action="append", dest="tunnel_ports", default=[],
700
- help="The port your tests expect to hit when they run."
701
- " By default, we use the same ports as the HOST."
702
- " If you know for sure _all_ your tests use something like"
703
- " http://site.test:8080/ then set this 8080.")
704
- og.add_option("--logfile", default=logfile,
705
- help="Path of the logfile to write to. [%default]")
706
- og.add_option("--readyfile",
707
- help="Path of the file to drop when the tunnel is ready "
708
- "for tests to run. By default, no file is dropped.")
709
- og.add_option("--use-ssh-config", action="store_true", default=False,
710
- help="Use the local SSH config. WARNING: Turning this on "
711
- "may break the script!")
712
- og.add_option("--rest-url", default="https://saucelabs.com/rest",
713
- help=optparse.SUPPRESS_HELP)
714
- og.add_option("--allow-unclean-exit", action="store_true", default=False,
715
- help=optparse.SUPPRESS_HELP)
716
- og.add_option("--ssh-port", default=22, type="int",
717
- help=optparse.SUPPRESS_HELP)
718
- op.add_option_group(og)
719
-
720
- og = optparse.OptionGroup(op, "Script debugging options")
721
- og.add_option("--debug-ssh", action="store_true", default=False,
722
- help="Log SSH output.")
723
- og.add_option("--latency-log", type=int, default=LATENCY_LOG,
724
- help="Threshold for logging latency (ms) [%default]")
725
- op.add_option_group(og)
726
-
727
- (options, args) = op.parse_args()
728
-
729
- # check ports are numbers
730
- try:
731
- map(int, options.ports)
732
- map(int, options.tunnel_ports)
733
- except ValueError:
734
- sys.stderr.write("Error: Ports must be integers\n\n")
735
- print "Help with options -t and -p:"
736
- print " All ports must be integers. You used:"
737
- if options.ports:
738
- print " -p", " -p ".join(options.ports)
739
- if options.tunnel_ports:
740
- print " -t", " -t ".join(options.tunnel_ports)
741
- raise SystemExit(1)
742
-
743
- # default to 80 and default to matching host ports with tunnel ports
744
- if not options.ports and not options.tunnel_ports:
745
- options.ports = ["80"]
746
- if options.ports and not options.tunnel_ports:
747
- options.tunnel_ports = options.ports[:]
748
-
749
- if len(options.ports) != len(options.tunnel_ports):
750
- sys.stderr.write("Error: Options -t and -p need to be paired\n\n")
751
- print "Help with options -t and -p:"
752
- print " When forwarding multiple ports, you must pair the tunnel port"
753
- print " to forward with the host port to forward to."
754
- print ""
755
- print "Example option usage:"
756
- print " To have your test's requests to 80 and 443 go to your test"
757
- print " server on ports 5000 and 5001: -t 80 -p 5000 -t 443 -p 5001"
758
- raise SystemExit(1)
759
-
760
- # check for required options without defaults
761
- for opt in ["user", "api_key", "host", "domains"]:
762
- if not hasattr(options, opt) or not getattr(options, opt):
763
- sys.stderr.write("Error: Missing required argument(s)\n\n")
764
- op.print_help()
765
- raise SystemExit(1)
766
-
767
- check_domains(options.domains)
768
- return options
769
-
770
-
771
- class MissingDependenciesError(Exception):
772
-
773
- deb_pkg = dict(ssh="openssh-client", expect="expect")
774
-
775
- def __init__(self, dependency, included=False, extra_msg=None):
776
- self.dependency = dependency
777
- self.included = included
778
- self.extra_msg = extra_msg
779
-
780
- def __str__(self):
781
- msg = ("%s\n\n" % self.extra_msg) if self.extra_msg else ""
782
- msg += "You are missing '%s'." % self.dependency
783
- if self.included:
784
- return (msg + " This should have come with the zip\n"
785
- "you downloaded. If you need assistance, please "
786
- "contact help@saucelabs.com.")
787
-
788
- msg += " Please install it or contact\nhelp@saucelabs.com for help."
789
- try:
790
- linux_distro = platform.linux_distribution
791
- except AttributeError: # Python 2.5
792
- linux_distro = platform.dist
793
- if linux_distro()[0].lower() in ['ubuntu', 'debian']:
794
- if self.dependency in self.deb_pkg:
795
- msg += ("\n\nTo install: sudo aptitude install %s"
796
- % self.deb_pkg[self.dependency])
797
- return msg
798
-
799
-
800
- def check_dependencies():
801
- if is_windows:
802
- if not os.path.exists("plink\plink.exe"):
803
- raise MissingDependenciesError("plink\plink.exe", included=True)
804
- return
805
-
806
- def check(command):
807
- # on unix
808
- with tempfile.TemporaryFile() as output:
809
- try:
810
- subprocess.check_call(command, shell=True, stdout=output,
811
- stderr=subprocess.STDOUT)
812
- except subprocess.CalledProcessError:
813
- dependency = command.split(" ")[0]
814
- raise MissingDependenciesError(dependency)
815
- output.seek(0)
816
- return output.read().strip()
817
-
818
- version = {}
819
- version['expect'] = check("expect -v")
820
-
821
- version['ssh'] = check("ssh -V")
822
- if not version['ssh'].startswith("OpenSSH"):
823
- msg = "You have '%s' installed,\nbut %s only supports OpenSSH." % (
824
- version['ssh'], PRODUCT_NAME)
825
- raise MissingDependenciesError("OpenSSH", extra_msg=msg)
826
-
827
- return version
828
-
829
-
830
- def _get_loggable_options(options):
831
- ops = dict(options.__dict__)
832
- del ops['api_key'] # no need to log the API key
833
- return ops
834
-
835
-
836
- def run(options, dependency_versions=None):
837
- if not options.quiet:
838
- print ".---------------------------------------------------."
839
- print "| Have questions or need help with Sauce Connect? |"
840
- print "| Contact us: http://saucelabs.com/forums |"
841
- print "-----------------------------------------------------"
842
- logger.info("/ Starting \\")
843
- logger.info('Please wait for "You may start your tests" to start your tests.')
844
- logger.info("%s" % DISPLAY_VERSION)
845
- check_version()
846
-
847
- metadata = dict(ScriptName=NAME,
848
- ScriptRelease=RELEASE,
849
- Platform=platform.platform(),
850
- PythonVersion=platform.python_version(),
851
- OwnerHost=options.host,
852
- OwnerPorts=options.ports,
853
- Ports=options.tunnel_ports, )
854
- if dependency_versions:
855
- metadata['DependencyVersions'] = dependency_versions
856
-
857
- logger.debug("System is %s hours off UTC" %
858
- (- (time.timezone, time.altzone)[time.daylight] / 3600.))
859
- logger.debug("options: %s" % _get_loggable_options(options))
860
- logger.debug("metadata: %s" % metadata)
861
-
862
- logger.info("Forwarding: %s:%s -> %s:%s", options.domains,
863
- options.tunnel_ports, options.host, options.ports)
864
-
865
- # Setup HealthChecker latency and make initial check of forwarded ports
866
- HealthChecker.latency_log = options.latency_log
867
- fail_msg = ("!! Are you sure this machine can get to your web server on "
868
- "host '%(host)s' listening on port %(port)d? Your tests will "
869
- "fail while the server is unreachable.")
870
- HealthChecker(options.host, options.ports, fail_msg=fail_msg).check()
871
-
872
- for attempt in xrange(1, RETRY_BOOT_MAX + 1):
873
- try:
874
- tunnel = TunnelMachine(options.rest_url, options.user,
875
- options.api_key, options.domains,
876
- options.ssh_port, metadata)
877
- except TunnelMachineError, e:
878
- logger.error(e)
879
- peace_out(returncode=1) # exits
880
- setup_signal_handler(tunnel, options)
881
- atexit.register(peace_out, tunnel, atexit=True)
882
- try:
883
- tunnel.ready_wait()
884
- break
885
- except TunnelMachineError, e:
886
- logger.warning(e)
887
- if attempt < RETRY_BOOT_MAX:
888
- logger.info("Requesting new tunnel")
889
- continue
890
- logger.error("!! Could not get tunnel host")
891
- logger.info("** Please contact help@saucelabs.com")
892
- peace_out(tunnel, returncode=1) # exits
893
-
894
- ssh = ReverseSSH(tunnel=tunnel, host=options.host,
895
- ports=options.ports, tunnel_ports=options.tunnel_ports,
896
- ssh_port=options.ssh_port,
897
- use_ssh_config=options.use_ssh_config,
898
- debug=options.debug_ssh)
899
- try:
900
- ssh.run(options.readyfile)
901
- except (ReverseSSHError, TunnelMachineError), e:
902
- logger.error(e)
903
- peace_out(tunnel) # exits
904
-
905
-
906
- def main():
907
- if map(int, platform.python_version_tuple ()) < [2, 5]:
908
- print "%s requires Python 2.5 (2006) or newer." % PRODUCT_NAME
909
- raise SystemExit(1)
910
-
911
- try:
912
- dependency_versions = check_dependencies()
913
- except MissingDependenciesError, e:
914
- print "\n== Missing requirements ==\n"
915
- print e
916
- raise SystemExit(1)
917
-
918
- options = get_options()
919
- setup_logging(options.logfile, options.quiet)
920
-
921
- try:
922
- run(options, dependency_versions)
923
- except Exception, e:
924
- logger.exception("Unhandled exception: %s", str(e))
925
- msg = "*** Please send this error to help@saucelabs.com. ***"
926
- logger.critical(msg)
927
- sys.stderr.write("\noptions: %s\n\n%s\n"
928
- % (_get_loggable_options(options), msg))
929
-
930
-
931
- if __name__ == '__main__':
932
- try:
933
- main()
934
- except Exception, e:
935
- msg = "*** Please send this error to help@saucelabs.com. ***"
936
- msg = "*" * len(msg) + "\n%s\n" % msg + "*" * len(msg)
937
- sys.stderr.write("\n%s\n\n" % msg)
938
- raise