sauce 1.0.2 → 2.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (56) hide show
  1. data/.document +5 -0
  2. data/.gitignore +30 -0
  3. data/Gemfile +16 -0
  4. data/README.markdown +39 -145
  5. data/Rakefile +46 -20
  6. data/bin/sauce +72 -61
  7. data/gemfiles/rails2.gemfile +10 -0
  8. data/gemfiles/rails2.gemfile.lock +77 -0
  9. data/gemfiles/rails3.gemfile +9 -0
  10. data/gemfiles/rails3.gemfile.lock +137 -0
  11. data/lib/generators/sauce/install/install_generator.rb +1 -2
  12. data/lib/sauce.rb +0 -22
  13. data/lib/sauce/capybara.rb +70 -32
  14. data/lib/sauce/capybara/cucumber.rb +121 -0
  15. data/lib/sauce/config.rb +57 -13
  16. data/lib/sauce/connect.rb +22 -11
  17. data/lib/sauce/integrations.rb +27 -69
  18. data/lib/sauce/jasmine.rb +35 -0
  19. data/lib/sauce/jasmine/rake.rb +47 -0
  20. data/lib/sauce/jasmine/runner.rb +4 -0
  21. data/lib/sauce/job.rb +10 -6
  22. data/lib/sauce/raketasks.rb +0 -21
  23. data/lib/sauce/selenium.rb +9 -18
  24. data/lib/sauce/utilities.rb +0 -17
  25. data/sauce.gemspec +8 -60
  26. data/spec/integration/connect_integration_spec.rb +84 -0
  27. data/spec/sauce/capybara/cucumber_spec.rb +156 -0
  28. data/spec/sauce/capybara/spec_helper.rb +42 -0
  29. data/spec/sauce/capybara_spec.rb +121 -0
  30. data/spec/sauce/config_spec.rb +239 -0
  31. data/spec/sauce/jasmine_spec.rb +49 -0
  32. data/spec/sauce/selenium_spec.rb +57 -0
  33. data/spec/spec_helper.rb +4 -0
  34. data/support/Sauce-Connect.jar +0 -0
  35. data/test/test_integrations.rb +202 -0
  36. data/test/test_testcase.rb +13 -0
  37. metadata +170 -171
  38. data/examples/helper.rb +0 -16
  39. data/examples/other_spec.rb +0 -7
  40. data/examples/saucelabs_spec.rb +0 -12
  41. data/examples/test_saucelabs.rb +0 -13
  42. data/examples/test_saucelabs2.rb +0 -9
  43. data/support/sauce_connect +0 -938
  44. data/support/selenium-server.jar +0 -0
  45. data/support/simplejson/LICENSE.txt +0 -19
  46. data/support/simplejson/__init__.py +0 -437
  47. data/support/simplejson/decoder.py +0 -421
  48. data/support/simplejson/encoder.py +0 -501
  49. data/support/simplejson/ordered_dict.py +0 -119
  50. data/support/simplejson/scanner.py +0 -77
  51. data/support/simplejson/tool.py +0 -39
  52. data/test/test_config.rb +0 -112
  53. data/test/test_connect.rb +0 -45
  54. data/test/test_job.rb +0 -13
  55. data/test/test_selenium.rb +0 -50
  56. data/test/test_selenium2.rb +0 -9
data/examples/helper.rb DELETED
@@ -1,16 +0,0 @@
1
- require 'rubygems'
2
-
3
- $LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
4
- $LOAD_PATH.unshift(File.dirname(__FILE__))
5
- require 'sauce'
6
-
7
- Sauce.config do |config|
8
- config.browsers = [
9
- ["Linux", "firefox", "3.6."],
10
- #["Windows 2003", "safariproxy", "5."]
11
- ]
12
- config.browser_url = "http://saucelabs.com"
13
-
14
- #config.application_host = "localhost"
15
- #config.application_port = "4444"
16
- end
@@ -1,7 +0,0 @@
1
- require File.join(File.dirname(__FILE__), "helper")
2
-
3
- describe "The login form", :type => :selenium do
4
- it "exists" do
5
- selenium.open "/"
6
- end
7
- end
@@ -1,12 +0,0 @@
1
- require File.join(File.dirname(__FILE__), "helper")
2
-
3
- describe "The Sauce website", :type => :selenium do
4
- it "works" do
5
- selenium.open "/"
6
- page.is_text_present("Sauce Labs").should be_true
7
- end
8
-
9
- it "has a pricing page" do
10
- selenium.open "/pricing"
11
- end
12
- end
@@ -1,13 +0,0 @@
1
- require File.join(File.dirname(__FILE__), "helper")
2
-
3
- class TestSaucelabs < Sauce::TestCase
4
- def test_basic_functionality
5
- selenium.open "/"
6
- assert page.is_text_present("Sauce Labs")
7
- end
8
-
9
- def test_pricing_page
10
- selenium.open "/pricing"
11
- assert page.is_text_present("$")
12
- end
13
- end
@@ -1,9 +0,0 @@
1
- require File.join(File.dirname(__FILE__), "helper")
2
-
3
- class TestSaucelabs2 < Sauce::TestCase
4
- def test_basic_functionality_two
5
- selenium.open "/"
6
- assert page.is_text_present("Sauce Labs")
7
- end
8
- end
9
-
@@ -1,938 +0,0 @@
1
- #!/usr/bin/env python
2
- # encoding: utf-8
3
- from __future__ import with_statement
4
-
5
- # TODO:
6
- # * Move to REST API v1
7
- # * windows: SSH link healthcheck (PuTTY session file hack?)
8
- # * Daemonizing
9
- # * issue: windows: no os.fork()
10
- # * issue: unix: null file descriptors causes Expect script to fail
11
- # * Renew tunnel lease (backend not implemented)
12
- # * Check tunnel machine ports are open (backend not implemented)
13
-
14
- import os
15
- import sys
16
- import re
17
- import optparse
18
- import logging
19
- import logging.handlers
20
- import signal
21
- import atexit
22
- import httplib
23
- import urllib2
24
- import subprocess
25
- import socket
26
- import time
27
- import platform
28
- import tempfile
29
- import string
30
- from base64 import b64encode
31
- from collections import defaultdict
32
- from contextlib import closing
33
- from functools import wraps
34
-
35
- try:
36
- import json
37
- except ImportError:
38
- import simplejson as json # Python 2.5 dependency
39
-
40
- NAME = "sauce_connect"
41
- RELEASE = 25
42
- DISPLAY_VERSION = "%s release %s" % (NAME, RELEASE)
43
- PRODUCT_NAME = u"Sauce Connect"
44
- VERSIONS_URL = "http://saucelabs.com/versions.json"
45
-
46
- RETRY_PROVISION_MAX = 4
47
- RETRY_BOOT_MAX = 4
48
- RETRY_REST_WAIT = 5
49
- RETRY_REST_MAX = 6
50
- REST_POLL_WAIT = 3
51
- RETRY_SSH_MAX = 4
52
- HEALTH_CHECK_INTERVAL = 15
53
- HEALTH_CHECK_FAIL = 5 * 60 # no good check after this amount of time == fail
54
- LATENCY_LOG = 150 # log when making connections takes this many ms
55
- LATENCY_WARNING = 350 # warn when making connections takes this many ms
56
- SIGNALS_RECV_MAX = 4 # used with --allow-unclean-exit
57
-
58
- is_windows = platform.system().lower() == "windows"
59
- is_openbsd = platform.system().lower() == "openbsd"
60
- logger = logging.getLogger(NAME)
61
-
62
-
63
- class DeleteRequest(urllib2.Request):
64
-
65
- def get_method(self):
66
- return "DELETE"
67
-
68
-
69
- class HTTPResponseError(Exception):
70
-
71
- def __init__(self, msg):
72
- self.msg = msg
73
-
74
- def __str__(self):
75
- return "HTTP server responded with '%s' (expected 'OK')" % self.msg
76
-
77
-
78
- class TunnelMachineError(Exception):
79
- pass
80
-
81
-
82
- class TunnelMachineProvisionError(TunnelMachineError):
83
- pass
84
-
85
-
86
- class TunnelMachineBootError(TunnelMachineError):
87
- pass
88
-
89
-
90
- class TunnelMachine(object):
91
-
92
- _host_search = re.compile("//([^/]+)").search
93
-
94
- def __init__(self, rest_url, user, password, domains, ssh_port, metadata=None):
95
- self.user = user
96
- self.password = password
97
- self.domains = set(domains)
98
- self.ssh_port = ssh_port
99
- self.metadata = metadata or dict()
100
-
101
- self.reverse_ssh = None
102
- self.is_shutdown = False
103
- self.base_url = "%(rest_url)s/%(user)s/tunnels" % locals()
104
- self.rest_host = self._host_search(rest_url).group(1)
105
- self.basic_auth_header = {"Authorization": "Basic %s"
106
- % b64encode("%s:%s" % (user, password))}
107
-
108
- self._set_urlopen(user, password)
109
-
110
- for attempt in xrange(1, RETRY_PROVISION_MAX):
111
- try:
112
- self._provision_tunnel()
113
- break
114
- except TunnelMachineProvisionError, e:
115
- logger.warning(e)
116
- if attempt == RETRY_PROVISION_MAX:
117
- raise TunnelMachineError(
118
- "!! Could not provision tunnel host. Please contact "
119
- "help@saucelabs.com.")
120
-
121
- def _set_urlopen(self, user, password):
122
- # always send Basic Auth header (HTTPBasicAuthHandler was unreliable)
123
- opener = urllib2.build_opener()
124
- opener.addheaders = self.basic_auth_header.items()
125
- self.urlopen = opener.open
126
-
127
- # decorator
128
- def _retry_rest_api(f):
129
- @wraps(f)
130
- def wrapper(*args, **kwargs):
131
- previous_failed = False
132
- for attempt in xrange(1, RETRY_REST_MAX + 1):
133
- try:
134
- result = f(*args, **kwargs)
135
- if previous_failed:
136
- logger.info("Connection succeeded")
137
- return result
138
- except (HTTPResponseError,
139
- urllib2.URLError, httplib.HTTPException,
140
- socket.gaierror, socket.error), e:
141
- logger.warning("Problem connecting to Sauce Labs REST API "
142
- "(%s)", str(e))
143
- if attempt == RETRY_REST_MAX:
144
- raise TunnelMachineError(
145
- "Could not reach Sauce Labs REST API after %d "
146
- "tries. Is your network down or firewalled?"
147
- % attempt)
148
- previous_failed = True
149
- logger.debug("Retrying in %ds", RETRY_REST_WAIT)
150
- time.sleep(RETRY_REST_WAIT)
151
- except Exception, e:
152
- raise TunnelMachineError(
153
- "An error occurred while contacting Sauce Labs REST "
154
- "API (%s). Please contact help@saucelabs.com." % str(e))
155
- return wrapper
156
-
157
- @_retry_rest_api
158
- def _get_doc(self, url_or_req):
159
- with closing(self.urlopen(url_or_req)) as resp:
160
- if resp.msg != "OK":
161
- raise HTTPResponseError(resp.msg)
162
- return json.loads(resp.read())
163
-
164
- def _provision_tunnel(self):
165
- # Shutdown any tunnel using a requested domain
166
- kill_list = set()
167
- for doc in self._get_doc(self.base_url):
168
- if not doc.get('DomainNames'):
169
- continue
170
- if set(doc['DomainNames']) & self.domains:
171
- kill_list.add(doc['id'])
172
- if kill_list:
173
- logger.info(
174
- "Shutting down other tunnel hosts using requested domains")
175
- for tunnel_id in kill_list:
176
- for attempt in xrange(1, 4): # try a few times, then bail
177
- logger.debug(
178
- "Shutting down old tunnel host: %s" % tunnel_id)
179
- url = "%s/%s" % (self.base_url, tunnel_id)
180
- doc = self._get_doc(DeleteRequest(url=url))
181
- if not doc.get('ok'):
182
- logger.warning("Old tunnel host failed to shutdown?")
183
- continue
184
- doc = self._get_doc(url)
185
- while doc.get('Status') not in ["halting", "terminated"]:
186
- logger.debug(
187
- "Waiting for old tunnel host to start halting")
188
- time.sleep(REST_POLL_WAIT)
189
- doc = self._get_doc(url)
190
- break
191
-
192
- # Request a tunnel machine
193
- headers = {"Content-Type": "application/json"}
194
- data = json.dumps(dict(DomainNames=list(self.domains),
195
- Metadata=self.metadata,
196
- SSHPort=self.ssh_port))
197
- req = urllib2.Request(url=self.base_url, headers=headers, data=data)
198
- doc = self._get_doc(req)
199
- if doc.get('error'):
200
- raise TunnelMachineProvisionError(doc['error'])
201
- for key in ['ok', 'id']:
202
- if not doc.get(key):
203
- raise TunnelMachineProvisionError(
204
- "Document for provisioned tunnel host is missing the key "
205
- "or value for '%s'" % key)
206
- self.id = doc['id']
207
- self.url = "%s/%s" % (self.base_url, self.id)
208
- logger.info("Tunnel host is provisioned (%s)" % self.id)
209
-
210
- def ready_wait(self):
211
- """Wait for the machine to reach the 'running' state."""
212
- previous_status = None
213
- while True:
214
- doc = self._get_doc(self.url)
215
- status = doc.get('Status')
216
- if status == "running":
217
- break
218
- if status in ["halting", "terminated"]:
219
- raise TunnelMachineBootError("Tunnel host was shutdown")
220
- if status != previous_status:
221
- logger.info("Tunnel host is %s .." % status)
222
- previous_status = status
223
- time.sleep(REST_POLL_WAIT)
224
- self.host = doc['Host']
225
- logger.info("Tunnel host is running at %s" % self.host)
226
-
227
- def shutdown(self):
228
- if self.is_shutdown:
229
- return
230
-
231
- if self.reverse_ssh:
232
- self.reverse_ssh.stop()
233
-
234
- logger.info("Shutting down tunnel host (please wait)")
235
- logger.debug("Tunnel host ID: %s" % self.id)
236
-
237
- try:
238
- doc = self._get_doc(DeleteRequest(url=self.url))
239
- except TunnelMachineError, e:
240
- logger.warning("Unable to shut down tunnel host")
241
- logger.debug("Shut down failed because: %s", str(e))
242
- self.is_shutdown = True # fuhgeddaboudit
243
- return
244
- assert doc.get('ok')
245
-
246
- previous_status = None
247
- while True:
248
- doc = self._get_doc(self.url)
249
- status = doc.get('Status')
250
- if status == "terminated":
251
- break
252
- if status != previous_status:
253
- logger.info("Tunnel host is %s .." % status)
254
- previous_status = status
255
- time.sleep(REST_POLL_WAIT)
256
- logger.info("Tunnel host is shutdown")
257
- self.is_shutdown = True
258
-
259
- # Make us usable with contextlib.closing
260
- close = shutdown
261
-
262
- def check_running(self):
263
- doc = self._get_doc(self.url)
264
- if doc.get('Status') == "running":
265
- return
266
- raise TunnelMachineError(
267
- "The tunnel host is no longer running. It may have been shutdown "
268
- "via the website or by another Sauce Connect script requesting these "
269
- "domains: %s" % list(self.domains))
270
-
271
-
272
- class HealthCheckFail(Exception):
273
- pass
274
-
275
-
276
- class HealthChecker(object):
277
-
278
- latency_log = LATENCY_LOG
279
-
280
- def __init__(self, host, ports, fail_msg=None):
281
- """fail_msg can include '%(host)s' and '%(port)d'"""
282
- self.host = host
283
- self.fail_msg = fail_msg
284
- if not self.fail_msg:
285
- self.fail_msg = ("!! Your tests will fail while your network "
286
- "can not get to %(host)s:%(port)d.")
287
- self.ports = frozenset(int(p) for p in ports)
288
- self.last_tcp_connect = defaultdict(time.time)
289
- self.last_tcp_ping = defaultdict(lambda: None)
290
-
291
- def _tcp_ping(self, port):
292
- with closing(socket.socket()) as sock:
293
- start_time = time.time()
294
- try:
295
- sock.connect((self.host, port))
296
- return int(1000 * (time.time() - start_time))
297
- except (socket.gaierror, socket.error), e:
298
- logger.warning("Could not connect to %s:%s (%s)",
299
- self.host, port, str(e))
300
-
301
- def check(self):
302
- now = time.time()
303
- for port in self.ports:
304
- ping_time = self._tcp_ping(port)
305
- if ping_time is not None:
306
- # TCP connection succeeded
307
- self.last_tcp_connect[port] = now
308
- result = (self.host, port, ping_time)
309
-
310
- if ping_time >= self.latency_log:
311
- logger.debug("Connected to %s:%s in in %dms" % result)
312
-
313
- if ping_time >= LATENCY_WARNING:
314
- if (self.last_tcp_ping[port] is None
315
- or self.last_tcp_ping[port] < LATENCY_WARNING):
316
- logger.warn("High latency to %s:%s (took %dms to "
317
- "connect); tests may run slowly" % result)
318
-
319
- if (ping_time < (LATENCY_WARNING / 2)
320
- and self.last_tcp_ping[port]
321
- and self.last_tcp_ping[port] >= LATENCY_WARNING):
322
- logger.info("Latency to %s:%s has lowered (took %dms to "
323
- "connect)" % result)
324
-
325
- if self.last_tcp_ping[port] is None:
326
- logger.info("Succesfully connected to %s:%s in %dms" % result)
327
-
328
- self.last_tcp_ping[port] = ping_time
329
- continue
330
-
331
- # TCP connection failed
332
- self.last_tcp_ping[port] = ping_time
333
- logger.warning(self.fail_msg % dict(host=self.host, port=port))
334
- if now - self.last_tcp_connect[port] > HEALTH_CHECK_FAIL:
335
- raise HealthCheckFail(
336
- "Could not connect to %s:%s for over %s seconds"
337
- % (self.host, port, HEALTH_CHECK_FAIL))
338
-
339
-
340
- class ReverseSSHError(Exception):
341
- pass
342
-
343
-
344
- class ReverseSSH(object):
345
-
346
- def __init__(self, tunnel, host, ports, tunnel_ports, ssh_port,
347
- use_ssh_config=False, debug=False):
348
- self.tunnel = tunnel
349
- self.host = host
350
- self.ports = ports
351
- self.tunnel_ports = tunnel_ports
352
- self.use_ssh_config = use_ssh_config
353
- self.ssh_port = ssh_port
354
- self.debug = debug
355
-
356
- self.proc = None
357
- self.readyfile = None
358
- self.stdout_f = None
359
- self.stderr_f = None
360
-
361
- if self.debug:
362
- logger.debug("ReverseSSH debugging is on.")
363
-
364
- def _check_dot_ssh_files(self):
365
- if not os.environ.get('HOME'):
366
- logger.debug("No HOME env, skipping .ssh file checks")
367
- return
368
-
369
- ssh_config_file = os.path.join(os.environ['HOME'], ".ssh", "config")
370
- if os.path.exists(ssh_config_file):
371
- logger.debug("Found %s" % ssh_config_file)
372
- if self.use_ssh_config:
373
- logger.warn("Using local SSH config")
374
-
375
- @property
376
- def _dash_Rs(self):
377
- dash_Rs = ""
378
- for port, tunnel_port in zip(self.ports, self.tunnel_ports):
379
- dash_Rs += "-R 0.0.0.0:%s:%s:%s " % (tunnel_port, self.host, port)
380
- return dash_Rs
381
-
382
- def get_plink_command(self):
383
- """Return the Windows SSH command."""
384
- verbosity = "-v" if self.debug else ""
385
- return ("plink\plink %s -P %s -l %s -pw %s -N %s %s"
386
- % (verbosity, self.ssh_port, self.tunnel.user, self.tunnel.password,
387
- self._dash_Rs, self.tunnel.host))
388
-
389
- def get_expect_script(self):
390
- """Return the Unix SSH command."""
391
- wait = "wait"
392
- if is_openbsd: # using 'wait;' hangs the script on OpenBSD
393
- wait = "wait -nowait;sleep 1" # hack
394
-
395
- verbosity = "-v" if self.debug else "-q"
396
- config_file = "" if self.use_ssh_config else "-F /dev/null"
397
- host_ip = socket.gethostbyname(self.tunnel.host)
398
- script = (
399
- "spawn ssh %s %s -p %s -l %s -o ServerAliveInterval=%s -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -N %s %s;"
400
- % (verbosity, config_file, self.ssh_port, self.tunnel.user,
401
- HEALTH_CHECK_INTERVAL, self._dash_Rs, self.tunnel.host) +
402
- "expect *password:;send -- %s\\r;" % self.tunnel.password +
403
- "expect -timeout -1 timeout")
404
- return script
405
-
406
- def _start_reverse_ssh(self, readyfile=None):
407
- self._check_dot_ssh_files()
408
- logger.info("Starting SSH process ..")
409
- if is_windows:
410
- cmd = "echo 'n' | %s" % self.get_plink_command()
411
- else:
412
- cmd = 'exec expect -c "%s"' % self.get_expect_script()
413
-
414
- # start ssh process
415
- if self.debug:
416
- self.stdout_f = tempfile.TemporaryFile()
417
- else:
418
- self.stdout_f = open(os.devnull)
419
- self.stderr_f = tempfile.TemporaryFile()
420
- self.proc = subprocess.Popen(
421
- cmd, shell=True, stdout=self.stdout_f, stderr=self.stderr_f)
422
- self.tunnel.reverse_ssh = self # BUG: circular ref
423
- time.sleep(3) # HACK: some startup time
424
-
425
- # ssh process is running
426
- announced_running = False
427
-
428
- # setup recurring healthchecks
429
- forwarded_health = HealthChecker(self.host, self.ports)
430
- tunnel_health = HealthChecker(host=self.tunnel.host, ports=[self.ssh_port],
431
- fail_msg="!! Your tests may fail because your network can not get "
432
- "to the tunnel host (%s:%d)." % (self.tunnel.host, self.ssh_port))
433
-
434
- start_time = int(time.time())
435
- while self.proc.poll() is None:
436
- now = int(time.time())
437
- if not announced_running:
438
- # guarantee we health check on first iteration
439
- now = start_time
440
- if (now - start_time) % HEALTH_CHECK_INTERVAL == 0:
441
- self.tunnel.check_running()
442
- try:
443
- forwarded_health.check()
444
- tunnel_health.check()
445
- except HealthCheckFail, e:
446
- raise ReverseSSHError(e)
447
- if not announced_running:
448
- logger.info("SSH is running. You may start your tests.")
449
- if readyfile:
450
- self.readyfile = readyfile
451
- f = open(readyfile, 'w')
452
- f.close()
453
- announced_running = True
454
- time.sleep(1)
455
-
456
- # ssh process has exited
457
- self._log_output()
458
- if self.proc.returncode != 0:
459
- logger.warning("SSH process exited with error code %d",
460
- self.proc.returncode)
461
- else:
462
- logger.info("SSH process exited (maybe due to network problems)")
463
-
464
- return self.proc.returncode
465
-
466
- def _log_output(self):
467
- if not self.stderr_f.closed:
468
- self.stderr_f.seek(0)
469
- reverse_ssh_stderr = self.stderr_f.read().strip()
470
- self.stderr_f.close()
471
-
472
- if reverse_ssh_stderr:
473
- logger.debug("ReverseSSH stderr was:\n%s\n" % reverse_ssh_stderr)
474
-
475
- if not self.stdout_f.closed:
476
- self.stdout_f.seek(0)
477
- reverse_ssh_stdout = self.stdout_f.read().strip()
478
- self.stdout_f.close()
479
-
480
- if self.debug:
481
- logger.debug("ReverseSSH stdout was:\n%s\n" % reverse_ssh_stdout)
482
-
483
- def _rm_readyfile(self):
484
- if self.readyfile and os.path.exists(self.readyfile):
485
- try:
486
- os.remove(self.readyfile)
487
- except OSError, e:
488
- logger.error("Couldn't remove %s: %s", self.readyfile, str(e))
489
-
490
- def stop(self):
491
- self._rm_readyfile()
492
- if not self.proc or self.proc.poll() is not None: # not running, done
493
- return
494
-
495
- if not is_windows: # windows no have kill()
496
- try:
497
- os.kill(self.proc.pid, signal.SIGHUP)
498
- logger.debug("Sent SIGHUP to PID %d", self.proc.pid)
499
- except OSError:
500
- pass
501
- self._log_output()
502
-
503
- def run(self, readyfile=None):
504
- clean_exit = False
505
- for attempt in xrange(1, RETRY_SSH_MAX + 1):
506
- # returncode 0 will happen due to ServerAlive checks failing.
507
- # this may result in a listening port forwarding nowhere, so
508
- # don't bother restarting the SSH connection.
509
- # TODO: revisit if server uses OpenSSH instead of Twisted SSH
510
- if self._start_reverse_ssh(readyfile) == 0:
511
- clean_exit = True
512
- break
513
- if attempt < RETRY_SSH_MAX:
514
- logger.debug("Will restart SSH in 3 seconds")
515
- time.sleep(3) # wait a bit for old connections to close
516
- self._rm_readyfile()
517
- if not clean_exit:
518
- raise ReverseSSHError(
519
- "SSH process errored %d times (bad network?)" % attempt)
520
-
521
-
522
- def peace_out(tunnel=None, returncode=0, atexit=False):
523
- """Shutdown the tunnel and raise SystemExit."""
524
- if tunnel:
525
- tunnel.shutdown()
526
- if not atexit:
527
- logger.info("\ Exiting /")
528
- raise SystemExit(returncode)
529
- else:
530
- logger.debug("-- fin --")
531
-
532
-
533
- def setup_signal_handler(tunnel, options):
534
- signal_count = defaultdict(int)
535
- signal_name = {}
536
-
537
- def sig_handler(signum, frame):
538
- if options.allow_unclean_exit:
539
- signal_count[signum] += 1
540
- if signal_count[signum] > SIGNALS_RECV_MAX:
541
- logger.info(
542
- "Received %s too many times (%d). Making unclean "
543
- "exit now!", signal_name[signum], signal_count[signum])
544
- raise SystemExit(1)
545
- logger.info("Received signal %s", signal_name[signum])
546
- peace_out(tunnel) # exits
547
-
548
- # TODO: ?? remove SIGTERM when we implement tunnel leases
549
- if is_windows:
550
- supported_signals = ["SIGABRT", "SIGBREAK", "SIGINT", "SIGTERM"]
551
- else:
552
- supported_signals = ["SIGHUP", "SIGINT", "SIGQUIT", "SIGTERM"]
553
- for sig in supported_signals:
554
- signum = getattr(signal, sig)
555
- signal_name[signum] = sig
556
- signal.signal(signum, sig_handler)
557
-
558
-
559
- def check_version():
560
- failed_msg = "Skipping version check"
561
- logger.debug("Checking version")
562
- try:
563
- with closing(urllib2.urlopen(VERSIONS_URL)) as resp:
564
- assert resp.msg == "OK", "Got HTTP response %s" % resp.msg
565
- version_doc = json.loads(resp.read())
566
- except (urllib2.URLError, AssertionError, ValueError), e:
567
- logger.debug("Could not check version: %s", str(e))
568
- logger.info(failed_msg)
569
- return
570
- try:
571
- version = version_doc[PRODUCT_NAME][u'version']
572
- download_url = version_doc[PRODUCT_NAME][u'download_url']
573
- except KeyError, e:
574
- logger.debug("Bad version doc, missing key: %s", str(e))
575
- logger.info(failed_msg)
576
- return
577
-
578
- try:
579
- latest = int(version.partition("-")[2].strip(string.ascii_letters))
580
- except (IndexError, ValueError), e:
581
- logger.debug("Couldn't parse release number: %s", str(e))
582
- logger.info(failed_msg)
583
- return
584
- if RELEASE < latest:
585
- msgs = ["** This version of %s is outdated." % PRODUCT_NAME,
586
- "** Please update with %s" % download_url]
587
- for update_msg in msgs:
588
- logger.warning(update_msg)
589
- for update_msg in msgs:
590
- sys.stderr.write("%s\n" % update_msg)
591
- time.sleep(15)
592
-
593
-
594
- def setup_logging(logfile=None, quiet=False):
595
- logger.setLevel(logging.DEBUG)
596
-
597
- if not quiet:
598
- stdout = logging.StreamHandler(sys.stdout)
599
- stdout.setLevel(logging.INFO)
600
- stdout.setFormatter(logging.Formatter("%(asctime)s - %(message)s"))
601
- logger.addHandler(stdout)
602
-
603
- if logfile:
604
- if not quiet:
605
- print "* Debug messages will be sent to %s" % logfile
606
- fileout = logging.handlers.RotatingFileHandler(
607
- filename=logfile, maxBytes=128 * 1024, backupCount=8)
608
- fileout.setLevel(logging.DEBUG)
609
- fileout.setFormatter(logging.Formatter(
610
- "%(asctime)s - %(name)s:%(lineno)d - %(levelname)s - %(message)s"))
611
- logger.addHandler(fileout)
612
-
613
-
614
- def check_domains(domains):
615
- """Display error and exit script if any requested domains are invalid."""
616
-
617
- for dom in domains:
618
- # no URLs
619
- if '/' in dom:
620
- sys.stderr.write(
621
- "Error: Domain contains illegal character '/' in it.\n")
622
- print " Did you use a URL instead of just the domain?\n"
623
- print "Examples: -d example.com -d '*.example.com' -d another.site"
624
- print
625
- raise SystemExit(1)
626
-
627
- # no numerical addresses
628
- if all(map(lambda c: c.isdigit() or c == '.', dom)):
629
- sys.stderr.write("Error: Domain must be a hostname not an IP\n")
630
- print
631
- print "Examples: -d example.com -d '*.example.com' -d another.site"
632
- print
633
- raise SystemExit(1)
634
-
635
- # need a dot and 2 char TLD
636
- # NOTE: if this restriction is relaxed, still check for "localhost"
637
- if '.' not in dom or len(dom.rpartition('.')[2]) < 2:
638
- sys.stderr.write(
639
- "Error: Domain requires a TLD of 2 characters or more\n")
640
- print
641
- print "Example: -d example.tld -d '*.example.tld' -d another.tld"
642
- print
643
- raise SystemExit(1)
644
-
645
- # *.com will break uploading to S3
646
- if dom == "*.com":
647
- sys.stderr.write(
648
- "Error: Matching *.com will break videos and logs. Use a hostname.\n")
649
- print
650
- print "Example: -d example.com -d *.example.com"
651
- print
652
- raise SystemExit(1)
653
-
654
-
655
- def get_options():
656
- usage = """
657
- Usage: %(name)s -u <user> -k <api_key> -s <webserver> -d <domain> [options]
658
-
659
- Examples:
660
- Have tests for example.com go to a staging server on your intranet:
661
- %(name)s -u user -k 123-abc -s staging.local -d example.com
662
-
663
- Have HTTP and HTTPS traffic for *.example.com go to the staging server:
664
- %(name)s -u user -k 123-abc -s staging.local -p 80 -p 443 \\
665
- -d example.com -d *.example.com
666
-
667
- Have tests for example.com go to your local machine on port 5000:
668
- %(name)s -u user -k 123-abc -s 127.0.0.1 -t 80 -p 5000 -d example.com
669
-
670
- Performance tip:
671
- It is highly recommended you run this script on the same machine as your
672
- test server (i.e., you would use "-s 127.0.0.1" or "-s localhost"). Using
673
- a remote server introduces higher latency (slower web requests) and is
674
- another failure point.
675
- """ % dict(name=NAME)
676
-
677
- usage = usage.strip()
678
- logfile = "%s.log" % NAME
679
-
680
- op = optparse.OptionParser(usage=usage, version=DISPLAY_VERSION)
681
- op.add_option("-u", "--user", "--username",
682
- help="Your Sauce Labs account name.")
683
- op.add_option("-k", "--api-key",
684
- help="On your account at https://saucelabs.com/account")
685
- op.add_option("-s", "--host", default="localhost",
686
- help="Host to forward requests to. [%default]")
687
- op.add_option("-p", "--port", metavar="PORT",
688
- action="append", dest="ports", default=[],
689
- help="Forward to this port on HOST. Can be specified "
690
- "multiple times. [80]")
691
- op.add_option("-d", "--domain", action="append", dest="domains",
692
- help="Repeat for each domain you want to forward requests for. "
693
- "Example: -d example.test -d '*.example.test'")
694
- op.add_option("-q", "--quiet", action="store_true", default=False,
695
- help="Minimize standard output (see %s)" % logfile)
696
-
697
- og = optparse.OptionGroup(op, "Advanced options")
698
- og.add_option("-t", "--tunnel-port", metavar="TUNNEL_PORT",
699
- action="append", dest="tunnel_ports", default=[],
700
- help="The port your tests expect to hit when they run."
701
- " By default, we use the same ports as the HOST."
702
- " If you know for sure _all_ your tests use something like"
703
- " http://site.test:8080/ then set this 8080.")
704
- og.add_option("--logfile", default=logfile,
705
- help="Path of the logfile to write to. [%default]")
706
- og.add_option("--readyfile",
707
- help="Path of the file to drop when the tunnel is ready "
708
- "for tests to run. By default, no file is dropped.")
709
- og.add_option("--use-ssh-config", action="store_true", default=False,
710
- help="Use the local SSH config. WARNING: Turning this on "
711
- "may break the script!")
712
- og.add_option("--rest-url", default="https://saucelabs.com/rest",
713
- help=optparse.SUPPRESS_HELP)
714
- og.add_option("--allow-unclean-exit", action="store_true", default=False,
715
- help=optparse.SUPPRESS_HELP)
716
- og.add_option("--ssh-port", default=22, type="int",
717
- help=optparse.SUPPRESS_HELP)
718
- op.add_option_group(og)
719
-
720
- og = optparse.OptionGroup(op, "Script debugging options")
721
- og.add_option("--debug-ssh", action="store_true", default=False,
722
- help="Log SSH output.")
723
- og.add_option("--latency-log", type=int, default=LATENCY_LOG,
724
- help="Threshold for logging latency (ms) [%default]")
725
- op.add_option_group(og)
726
-
727
- (options, args) = op.parse_args()
728
-
729
- # check ports are numbers
730
- try:
731
- map(int, options.ports)
732
- map(int, options.tunnel_ports)
733
- except ValueError:
734
- sys.stderr.write("Error: Ports must be integers\n\n")
735
- print "Help with options -t and -p:"
736
- print " All ports must be integers. You used:"
737
- if options.ports:
738
- print " -p", " -p ".join(options.ports)
739
- if options.tunnel_ports:
740
- print " -t", " -t ".join(options.tunnel_ports)
741
- raise SystemExit(1)
742
-
743
- # default to 80 and default to matching host ports with tunnel ports
744
- if not options.ports and not options.tunnel_ports:
745
- options.ports = ["80"]
746
- if options.ports and not options.tunnel_ports:
747
- options.tunnel_ports = options.ports[:]
748
-
749
- if len(options.ports) != len(options.tunnel_ports):
750
- sys.stderr.write("Error: Options -t and -p need to be paired\n\n")
751
- print "Help with options -t and -p:"
752
- print " When forwarding multiple ports, you must pair the tunnel port"
753
- print " to forward with the host port to forward to."
754
- print ""
755
- print "Example option usage:"
756
- print " To have your test's requests to 80 and 443 go to your test"
757
- print " server on ports 5000 and 5001: -t 80 -p 5000 -t 443 -p 5001"
758
- raise SystemExit(1)
759
-
760
- # check for required options without defaults
761
- for opt in ["user", "api_key", "host", "domains"]:
762
- if not hasattr(options, opt) or not getattr(options, opt):
763
- sys.stderr.write("Error: Missing required argument(s)\n\n")
764
- op.print_help()
765
- raise SystemExit(1)
766
-
767
- check_domains(options.domains)
768
- return options
769
-
770
-
771
- class MissingDependenciesError(Exception):
772
-
773
- deb_pkg = dict(ssh="openssh-client", expect="expect")
774
-
775
- def __init__(self, dependency, included=False, extra_msg=None):
776
- self.dependency = dependency
777
- self.included = included
778
- self.extra_msg = extra_msg
779
-
780
- def __str__(self):
781
- msg = ("%s\n\n" % self.extra_msg) if self.extra_msg else ""
782
- msg += "You are missing '%s'." % self.dependency
783
- if self.included:
784
- return (msg + " This should have come with the zip\n"
785
- "you downloaded. If you need assistance, please "
786
- "contact help@saucelabs.com.")
787
-
788
- msg += " Please install it or contact\nhelp@saucelabs.com for help."
789
- try:
790
- linux_distro = platform.linux_distribution
791
- except AttributeError: # Python 2.5
792
- linux_distro = platform.dist
793
- if linux_distro()[0].lower() in ['ubuntu', 'debian']:
794
- if self.dependency in self.deb_pkg:
795
- msg += ("\n\nTo install: sudo aptitude install %s"
796
- % self.deb_pkg[self.dependency])
797
- return msg
798
-
799
-
800
- def check_dependencies():
801
- if is_windows:
802
- if not os.path.exists("plink\plink.exe"):
803
- raise MissingDependenciesError("plink\plink.exe", included=True)
804
- return
805
-
806
- def check(command):
807
- # on unix
808
- with tempfile.TemporaryFile() as output:
809
- try:
810
- subprocess.check_call(command, shell=True, stdout=output,
811
- stderr=subprocess.STDOUT)
812
- except subprocess.CalledProcessError:
813
- dependency = command.split(" ")[0]
814
- raise MissingDependenciesError(dependency)
815
- output.seek(0)
816
- return output.read().strip()
817
-
818
- version = {}
819
- version['expect'] = check("expect -v")
820
-
821
- version['ssh'] = check("ssh -V")
822
- if not version['ssh'].startswith("OpenSSH"):
823
- msg = "You have '%s' installed,\nbut %s only supports OpenSSH." % (
824
- version['ssh'], PRODUCT_NAME)
825
- raise MissingDependenciesError("OpenSSH", extra_msg=msg)
826
-
827
- return version
828
-
829
-
830
- def _get_loggable_options(options):
831
- ops = dict(options.__dict__)
832
- del ops['api_key'] # no need to log the API key
833
- return ops
834
-
835
-
836
- def run(options, dependency_versions=None):
837
- if not options.quiet:
838
- print ".---------------------------------------------------."
839
- print "| Have questions or need help with Sauce Connect? |"
840
- print "| Contact us: http://saucelabs.com/forums |"
841
- print "-----------------------------------------------------"
842
- logger.info("/ Starting \\")
843
- logger.info('Please wait for "You may start your tests" to start your tests.')
844
- logger.info("%s" % DISPLAY_VERSION)
845
- check_version()
846
-
847
- metadata = dict(ScriptName=NAME,
848
- ScriptRelease=RELEASE,
849
- Platform=platform.platform(),
850
- PythonVersion=platform.python_version(),
851
- OwnerHost=options.host,
852
- OwnerPorts=options.ports,
853
- Ports=options.tunnel_ports, )
854
- if dependency_versions:
855
- metadata['DependencyVersions'] = dependency_versions
856
-
857
- logger.debug("System is %s hours off UTC" %
858
- (- (time.timezone, time.altzone)[time.daylight] / 3600.))
859
- logger.debug("options: %s" % _get_loggable_options(options))
860
- logger.debug("metadata: %s" % metadata)
861
-
862
- logger.info("Forwarding: %s:%s -> %s:%s", options.domains,
863
- options.tunnel_ports, options.host, options.ports)
864
-
865
- # Setup HealthChecker latency and make initial check of forwarded ports
866
- HealthChecker.latency_log = options.latency_log
867
- fail_msg = ("!! Are you sure this machine can get to your web server on "
868
- "host '%(host)s' listening on port %(port)d? Your tests will "
869
- "fail while the server is unreachable.")
870
- HealthChecker(options.host, options.ports, fail_msg=fail_msg).check()
871
-
872
- for attempt in xrange(1, RETRY_BOOT_MAX + 1):
873
- try:
874
- tunnel = TunnelMachine(options.rest_url, options.user,
875
- options.api_key, options.domains,
876
- options.ssh_port, metadata)
877
- except TunnelMachineError, e:
878
- logger.error(e)
879
- peace_out(returncode=1) # exits
880
- setup_signal_handler(tunnel, options)
881
- atexit.register(peace_out, tunnel, atexit=True)
882
- try:
883
- tunnel.ready_wait()
884
- break
885
- except TunnelMachineError, e:
886
- logger.warning(e)
887
- if attempt < RETRY_BOOT_MAX:
888
- logger.info("Requesting new tunnel")
889
- continue
890
- logger.error("!! Could not get tunnel host")
891
- logger.info("** Please contact help@saucelabs.com")
892
- peace_out(tunnel, returncode=1) # exits
893
-
894
- ssh = ReverseSSH(tunnel=tunnel, host=options.host,
895
- ports=options.ports, tunnel_ports=options.tunnel_ports,
896
- ssh_port=options.ssh_port,
897
- use_ssh_config=options.use_ssh_config,
898
- debug=options.debug_ssh)
899
- try:
900
- ssh.run(options.readyfile)
901
- except (ReverseSSHError, TunnelMachineError), e:
902
- logger.error(e)
903
- peace_out(tunnel) # exits
904
-
905
-
906
- def main():
907
- if map(int, platform.python_version_tuple ()) < [2, 5]:
908
- print "%s requires Python 2.5 (2006) or newer." % PRODUCT_NAME
909
- raise SystemExit(1)
910
-
911
- try:
912
- dependency_versions = check_dependencies()
913
- except MissingDependenciesError, e:
914
- print "\n== Missing requirements ==\n"
915
- print e
916
- raise SystemExit(1)
917
-
918
- options = get_options()
919
- setup_logging(options.logfile, options.quiet)
920
-
921
- try:
922
- run(options, dependency_versions)
923
- except Exception, e:
924
- logger.exception("Unhandled exception: %s", str(e))
925
- msg = "*** Please send this error to help@saucelabs.com. ***"
926
- logger.critical(msg)
927
- sys.stderr.write("\noptions: %s\n\n%s\n"
928
- % (_get_loggable_options(options), msg))
929
-
930
-
931
- if __name__ == '__main__':
932
- try:
933
- main()
934
- except Exception, e:
935
- msg = "*** Please send this error to help@saucelabs.com. ***"
936
- msg = "*" * len(msg) + "\n%s\n" % msg + "*" * len(msg)
937
- sys.stderr.write("\n%s\n\n" % msg)
938
- raise