gowalk-cicd 1.0.90 → 1.0.92
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CLAUDE.md +7 -2
- package/README.md +7 -0
- package/action/.daemux-version +1 -1
- package/android-action/.daemux-version +1 -1
- package/android-action/scripts/flutter_download.py +36 -8
- package/android-action/scripts/flutter_manifest.py +5 -1
- package/android-action/scripts/play_preflight.py +2 -2
- package/android-action/scripts/play_preflight_transport.py +33 -3
- package/android-action/scripts/play_proxy_test_support.py +82 -0
- package/android-action/scripts/test_flutter_download_budget.py +42 -0
- package/android-action/scripts/test_flutter_transfer_progress.py +81 -0
- package/android-action/scripts/test_play_connect_retry.py +115 -0
- package/android-action/scripts/test_play_connect_runtime.py +62 -0
- package/backend-action/.daemux-version +1 -1
- package/package.json +1 -1
package/CLAUDE.md
CHANGED
|
@@ -156,11 +156,13 @@ node /path/to/gowalk-cicd/bin/cli.mjs
|
|
|
156
156
|
consumer autoupdate changes the action directory. The installer delivers this development
|
|
157
157
|
tooling for native iOS repositories too; it does not require an Android app.
|
|
158
158
|
- Flutter manifest/archive GET recovery is scoped to the two pinned setup invocations through `BASH_ENV`.
|
|
159
|
-
It retains the account proxy and allows four attempts within 60 seconds for manifests or
|
|
159
|
+
It retains the account proxy and allows four attempts within 60 seconds for manifests or 20 minutes
|
|
160
160
|
for archives, with an 8 GiB archive bound. Interrupted archives resume private bytes with HTTP Range; an origin ignoring
|
|
161
161
|
Range permits a bounded full restart. Only complete JSON or an archive matching the selected official
|
|
162
162
|
manifest's SHA-256 enters action outputs. Authentication/certificate refusals remain terminal.
|
|
163
163
|
`flutter_download_failed` emits fixed transport codes and retained-byte counts, never curl stderr.
|
|
164
|
+
Archive progress emits retained byte counts every minute and safe results after each attempt;
|
|
165
|
+
the transfer deadline leaves room for a failure receipt before a 30-minute validation job ends.
|
|
164
166
|
- Web Release selects bundled renderer resources with `--no-web-resources-cdn` inside
|
|
165
167
|
the existing proxied build command. This is not a font or integration offline guarantee;
|
|
166
168
|
preserve app-authored font declarations, bootstrap configuration and other build inputs.
|
|
@@ -280,7 +282,10 @@ node /path/to/gowalk-cicd/bin/cli.mjs
|
|
|
280
282
|
symbols and uploads them before the store bundle, using the configured proxy.
|
|
281
283
|
- Play readiness and AAB/APK version inventory share one prebuild edit. Retry OAuth
|
|
282
284
|
transport, inventory reads and cleanup of that known edit; never replay an ambiguous
|
|
283
|
-
`edits.insert`.
|
|
285
|
+
`edits.insert`. Edit creation retries at most three times only for typed connection-timeout
|
|
286
|
+
or typed connection failure / CONNECT 502,503,504 evidence that no provider request was sent. Bare proxy errors,
|
|
287
|
+
TLS verification failures, provider responses and lost origin responses do not authorize a retry.
|
|
288
|
+
Only an explicit package-404 allows first-release defaults; failed
|
|
284
289
|
inventory is not an empty account. The action exposes `play-api-ready` so current
|
|
285
290
|
workflows skip the redundant postbuild check. Error annotations carry safe
|
|
286
291
|
phase/category evidence and any cleanup edit ID without raw provider exceptions.
|
package/README.md
CHANGED
|
@@ -299,6 +299,13 @@ the situation and whose **message** is one line of JSON carrying a versioned
|
|
|
299
299
|
| `apple_build_upload_failed` (error) | `gowalk-cicd/apple-build-upload-failed.v2` | `category`, `stage`, `receipt_available`, receipt identifiers (`cfBundleVersion`, `upload_id`, `file_id`, `state`, `provider_status`, `provider_code`, `provider_pointer`), `action` |
|
|
300
300
|
| `flutter_download_failed` (error) | `gowalk-cicd/flutter-download-failed.v1` | `phase`, `curl_exit`, `http_status`, `proxy_status`, `attempts`, `retained_bytes` |
|
|
301
301
|
|
|
302
|
+
SDK transfers also emit `flutter_download_progress` notices with schema
|
|
303
|
+
`gowalk-cicd/flutter-download-progress.v1`. Each carries `phase`, `event`, `elapsed_ms`
|
|
304
|
+
and `retained_bytes`; `attempt_finished` adds `attempt`, `curl_exit`, `http_status`
|
|
305
|
+
and `proxy_status`. In-progress notices arrive each minute; elapsed time belongs to
|
|
306
|
+
the current attempt. Archives have a 20-minute total transfer budget so a failure
|
|
307
|
+
receipt precedes a 30-minute validation job's deadline. Progress is not a success receipt.
|
|
308
|
+
|
|
302
309
|
Add a field by bumping the schema version; never change the meaning of an
|
|
303
310
|
existing one.
|
|
304
311
|
|
package/action/.daemux-version
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
1.0.
|
|
1
|
+
1.0.92
|
|
@@ -1 +1 @@
|
|
|
1
|
-
1.0.
|
|
1
|
+
1.0.92
|
|
@@ -7,8 +7,10 @@ import re
|
|
|
7
7
|
import shutil
|
|
8
8
|
import subprocess
|
|
9
9
|
import sys
|
|
10
|
+
import time
|
|
10
11
|
|
|
11
12
|
MAX_ARCHIVE_BYTES = 8 * 1024 * 1024 * 1024
|
|
13
|
+
PROGRESS_SECONDS = 60
|
|
12
14
|
|
|
13
15
|
|
|
14
16
|
def archive_digest() -> str:
|
|
@@ -20,19 +22,45 @@ def archive_digest() -> str:
|
|
|
20
22
|
|
|
21
23
|
def transfer(url: str, destination: Path, kind: str, remaining: float, env: dict) -> tuple[int, int, int]:
|
|
22
24
|
resume = kind == "archive" and destination.stat().st_size > 0
|
|
25
|
+
budget = min(20 if kind == "manifest" else 600, remaining)
|
|
23
26
|
command = ["curl", "--disable", "--silent", "--show-error", "--fail", "--retry", "0",
|
|
24
|
-
"--connect-timeout", "15", "--max-time", str(
|
|
27
|
+
"--connect-timeout", "15", "--max-time", str(budget),
|
|
25
28
|
"--max-filesize", str(16777216 if kind == "manifest" else MAX_ARCHIVE_BYTES),
|
|
26
29
|
"--output", str(destination), "--write-out", "%{http_code} %{http_connect}"]
|
|
27
30
|
if resume:
|
|
28
31
|
command += ["--continue-at", "-"]
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
32
|
+
with subprocess.Popen([*command, url], stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env) as process:
|
|
33
|
+
try:
|
|
34
|
+
return wait_transfer(process, destination, kind, budget)
|
|
35
|
+
finally:
|
|
36
|
+
if process.poll() is None:
|
|
37
|
+
process.kill()
|
|
38
|
+
process.communicate()
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def wait_transfer(process, destination: Path, kind: str, budget: float) -> tuple[int, int, int]:
|
|
42
|
+
started = time.monotonic()
|
|
43
|
+
while True:
|
|
44
|
+
left = budget - (time.monotonic() - started)
|
|
45
|
+
if left <= 0:
|
|
46
|
+
return 28, 0, 0
|
|
47
|
+
try:
|
|
48
|
+
output, _private_error = process.communicate(timeout=min(PROGRESS_SECONDS, left))
|
|
49
|
+
status = output.split()
|
|
50
|
+
http, proxy = (map(int, status) if len(status) == 2
|
|
51
|
+
and all(part.isdigit() for part in status) else (0, 0))
|
|
52
|
+
return process.returncode, http, proxy
|
|
53
|
+
except subprocess.TimeoutExpired:
|
|
54
|
+
if kind == "archive":
|
|
55
|
+
progress(destination, started, "transferring")
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def progress(destination: Path, started: float, event: str, **fields) -> None:
|
|
59
|
+
note = {"schema": "gowalk-cicd/flutter-download-progress.v1", "phase": "archive", "event": event,
|
|
60
|
+
"elapsed_ms": round((time.monotonic() - started) * 1000),
|
|
61
|
+
"retained_bytes": destination.stat().st_size, **fields}
|
|
62
|
+
print("::notice title=flutter_download_progress::" + json.dumps(note, separators=(",", ":")),
|
|
63
|
+
file=sys.stderr, flush=True)
|
|
36
64
|
|
|
37
65
|
|
|
38
66
|
def publish(destination: Path, kind: str) -> None:
|
|
@@ -22,7 +22,7 @@ def download(arguments: list[str], directory: Path) -> int:
|
|
|
22
22
|
if kind == "archive":
|
|
23
23
|
flutter_download.archive_digest()
|
|
24
24
|
child_env = {**os.environ, "NO_PROXY": "", "no_proxy": ""}
|
|
25
|
-
deadline, code, http, proxy_http = time.monotonic() + (60 if kind == "manifest" else
|
|
25
|
+
deadline, code, http, proxy_http = time.monotonic() + (60 if kind == "manifest" else 1200), 1, 0, 0
|
|
26
26
|
destination = directory / "download"
|
|
27
27
|
destination.touch(mode=0o600, exist_ok=False)
|
|
28
28
|
destination.chmod(0o600)
|
|
@@ -32,7 +32,11 @@ def download(arguments: list[str], directory: Path) -> int:
|
|
|
32
32
|
if remaining <= 0:
|
|
33
33
|
break
|
|
34
34
|
attempted = attempt
|
|
35
|
+
started = time.monotonic()
|
|
35
36
|
code, http, proxy_http = flutter_download.transfer(arguments[-1], destination, kind, remaining, child_env)
|
|
37
|
+
if kind == "archive":
|
|
38
|
+
flutter_download.progress(destination, started, "attempt_finished", attempt=attempt,
|
|
39
|
+
curl_exit=code, http_status=http, proxy_status=proxy_http)
|
|
36
40
|
if code == 0 and http in ({200, 206} if kind == "archive" else {200}):
|
|
37
41
|
flutter_download.publish(destination, kind)
|
|
38
42
|
if attempt > 1:
|
|
@@ -55,9 +55,9 @@ def opened_edit(client, package: str, account: Path):
|
|
|
55
55
|
call("oauth_refresh", lambda: credentials.refresh(request), retry=True)
|
|
56
56
|
headers = {"Authorization": f"Bearer {credentials.token}"}
|
|
57
57
|
url = f"{BASE_URL}/applications/{package}/edits"
|
|
58
|
-
#
|
|
58
|
+
# Retry only a proven failed connection; an ambiguous insert is never replayed.
|
|
59
59
|
response = call("edit_create", lambda: client.post(url, headers=headers, json={}, timeout=30),
|
|
60
|
-
accepted=(200, 404))
|
|
60
|
+
accepted=(200, 404), retry_connect=True)
|
|
61
61
|
if response.status_code == 404:
|
|
62
62
|
print(
|
|
63
63
|
"::warning::Google Play package is not API-ready. The app session must upload the "
|
|
@@ -2,15 +2,45 @@
|
|
|
2
2
|
from __future__ import annotations
|
|
3
3
|
|
|
4
4
|
import json
|
|
5
|
+
import re
|
|
5
6
|
import time
|
|
6
7
|
|
|
7
8
|
from google.auth.exceptions import RefreshError, TransportError
|
|
8
9
|
import requests
|
|
10
|
+
from urllib3 import exceptions as urllib_errors
|
|
9
11
|
|
|
10
12
|
RETRY_STATUSES = {429, 500, 502, 503, 504}
|
|
11
13
|
TRANSPORT_ERRORS = (requests.RequestException, TransportError)
|
|
12
14
|
|
|
13
15
|
|
|
16
|
+
def connection_not_sent(error: Exception) -> bool:
|
|
17
|
+
"""Retry only typed connection failure or a transient HTTP CONNECT refusal.
|
|
18
|
+
|
|
19
|
+
Requests explicitly declares ConnectTimeout safe to retry. Other failures must
|
|
20
|
+
carry urllib3's structured reason. ProxyError alone is ambiguous: it can wrap
|
|
21
|
+
a dropped origin response. A plain OSError with http.client's exact tunnel
|
|
22
|
+
status framing identifies CONNECT 502/503/504 before any origin TLS/POST.
|
|
23
|
+
"""
|
|
24
|
+
if isinstance(error, requests.exceptions.ConnectTimeout):
|
|
25
|
+
return error.response is None
|
|
26
|
+
if not isinstance(error, requests.exceptions.ConnectionError) or error.response is not None:
|
|
27
|
+
return False
|
|
28
|
+
if len(error.args) != 1:
|
|
29
|
+
return False
|
|
30
|
+
reason = error.args[0]
|
|
31
|
+
if isinstance(reason, urllib_errors.MaxRetryError):
|
|
32
|
+
reason = reason.reason
|
|
33
|
+
if isinstance(reason, urllib_errors.ConnectTimeoutError):
|
|
34
|
+
return True
|
|
35
|
+
if not isinstance(reason, urllib_errors.ProxyError):
|
|
36
|
+
return False
|
|
37
|
+
original = reason.original_error
|
|
38
|
+
if isinstance(original, urllib_errors.ConnectTimeoutError):
|
|
39
|
+
return True
|
|
40
|
+
return (type(original) is OSError and original.errno is None
|
|
41
|
+
and re.fullmatch(r"Tunnel connection failed: (?:502|503|504)(?: [^\r\n]*)?", str(original)) is not None)
|
|
42
|
+
|
|
43
|
+
|
|
14
44
|
def transport_code(error: Exception) -> str:
|
|
15
45
|
pending, seen = [error], set()
|
|
16
46
|
while pending and len(seen) < 12:
|
|
@@ -45,13 +75,13 @@ def fail(phase: str, code: str, attempt: int, *, status: int | None = None,
|
|
|
45
75
|
|
|
46
76
|
|
|
47
77
|
def call(phase: str, operation, *, retry: bool = False, edit_id: str | None = None,
|
|
48
|
-
accepted: tuple[int, ...] | None = None):
|
|
49
|
-
limit = 3 if retry else 1
|
|
78
|
+
accepted: tuple[int, ...] | None = None, retry_connect: bool = False):
|
|
79
|
+
limit = 3 if retry or retry_connect else 1
|
|
50
80
|
for attempt in range(1, limit + 1):
|
|
51
81
|
try:
|
|
52
82
|
result = operation()
|
|
53
83
|
except TRANSPORT_ERRORS as error:
|
|
54
|
-
if attempt == limit:
|
|
84
|
+
if attempt == limit or (not retry and not connection_not_sent(error)):
|
|
55
85
|
fail(phase, transport_code(error), attempt, edit_id=edit_id)
|
|
56
86
|
except RefreshError:
|
|
57
87
|
fail(phase, "oauth_refresh_failed", attempt)
|
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
"""An owned TLS origin inside a loopback CONNECT proxy; no external traffic."""
|
|
2
|
+
from contextlib import contextmanager
|
|
3
|
+
from datetime import datetime, timedelta, timezone
|
|
4
|
+
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
import ssl
|
|
7
|
+
import tempfile
|
|
8
|
+
import threading
|
|
9
|
+
|
|
10
|
+
from cryptography import x509
|
|
11
|
+
from cryptography.hazmat.primitives import hashes, serialization
|
|
12
|
+
from cryptography.hazmat.primitives.asymmetric import rsa
|
|
13
|
+
from cryptography.x509.oid import NameOID
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def tls_material(root):
|
|
17
|
+
key = rsa.generate_private_key(public_exponent=65537, key_size=2048)
|
|
18
|
+
name = x509.Name([x509.NameAttribute(NameOID.COMMON_NAME, "play.test")])
|
|
19
|
+
now = datetime.now(timezone.utc)
|
|
20
|
+
cert = (x509.CertificateBuilder().subject_name(name).issuer_name(name).public_key(key.public_key())
|
|
21
|
+
.serial_number(x509.random_serial_number()).not_valid_before(now - timedelta(days=1))
|
|
22
|
+
.not_valid_after(now + timedelta(days=1))
|
|
23
|
+
.add_extension(x509.SubjectAlternativeName([x509.DNSName("play.test")]), critical=False)
|
|
24
|
+
.add_extension(x509.BasicConstraints(ca=True, path_length=None), critical=True)
|
|
25
|
+
.sign(key, hashes.SHA256()))
|
|
26
|
+
path = Path(root)
|
|
27
|
+
certificate, private = path / "certificate.pem", path / "private.pem"
|
|
28
|
+
certificate.write_bytes(cert.public_bytes(serialization.Encoding.PEM))
|
|
29
|
+
private.write_bytes(key.private_bytes(serialization.Encoding.PEM, serialization.PrivateFormat.PKCS8,
|
|
30
|
+
serialization.NoEncryption()))
|
|
31
|
+
private.chmod(0o600)
|
|
32
|
+
context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER)
|
|
33
|
+
context.load_cert_chain(certificate, private)
|
|
34
|
+
return context, str(certificate)
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
class Proxy(BaseHTTPRequestHandler):
|
|
38
|
+
def do_CONNECT(self):
|
|
39
|
+
self.server.connections.append(self.path)
|
|
40
|
+
self.server.authenticated.append(bool(self.headers.get("Proxy-Authorization")))
|
|
41
|
+
if len(self.server.connections) <= self.server.reject:
|
|
42
|
+
self.send_error(502, "private proxy rejection")
|
|
43
|
+
return
|
|
44
|
+
self.send_response(200)
|
|
45
|
+
self.end_headers()
|
|
46
|
+
try:
|
|
47
|
+
with self.server.context.wrap_socket(self.connection, server_side=True) as stream:
|
|
48
|
+
stream.settimeout(3)
|
|
49
|
+
request = b""
|
|
50
|
+
while b"\r\n\r\n" not in request:
|
|
51
|
+
data = stream.recv(4096)
|
|
52
|
+
if not data:
|
|
53
|
+
return
|
|
54
|
+
request += data
|
|
55
|
+
self.server.requests.append(request.split(b"\r\n", 1)[0])
|
|
56
|
+
if self.server.drop_response:
|
|
57
|
+
return
|
|
58
|
+
body = b'{"id":"edit-1"}'
|
|
59
|
+
response = b"HTTP/1.1 200 OK\r\nConnection: close\r\nContent-Length: " + str(len(body)).encode()
|
|
60
|
+
stream.sendall(response + b"\r\nContent-Type: application/json\r\n\r\n" + body)
|
|
61
|
+
except ssl.SSLError:
|
|
62
|
+
pass
|
|
63
|
+
|
|
64
|
+
def log_message(self, *args):
|
|
65
|
+
pass
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
@contextmanager
|
|
69
|
+
def running(*, reject=0, drop_response=False):
|
|
70
|
+
with tempfile.TemporaryDirectory(prefix="play-connect-test-") as root:
|
|
71
|
+
server = ThreadingHTTPServer(("127.0.0.1", 0), Proxy)
|
|
72
|
+
server.context, certificate = tls_material(root)
|
|
73
|
+
server.connections, server.requests, server.authenticated = [], [], []
|
|
74
|
+
server.reject, server.drop_response = reject, drop_response
|
|
75
|
+
thread = threading.Thread(target=server.serve_forever, daemon=True)
|
|
76
|
+
thread.start()
|
|
77
|
+
try:
|
|
78
|
+
yield server, certificate
|
|
79
|
+
finally:
|
|
80
|
+
server.shutdown()
|
|
81
|
+
server.server_close()
|
|
82
|
+
thread.join(timeout=2)
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
"""Archive failure remains observable before the consumer's 30-minute validation job expires."""
|
|
2
|
+
import contextlib
|
|
3
|
+
import io
|
|
4
|
+
import json
|
|
5
|
+
import os
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
import tempfile
|
|
8
|
+
import unittest
|
|
9
|
+
from unittest.mock import patch
|
|
10
|
+
|
|
11
|
+
import flutter_manifest
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
class DownloadBudgetTests(unittest.TestCase):
|
|
15
|
+
def test_failed_archive_reports_before_the_outer_job_cancels_it(self):
|
|
16
|
+
elapsed = [0.0]
|
|
17
|
+
|
|
18
|
+
def transfer(_url, destination, _kind, remaining, _env):
|
|
19
|
+
elapsed[0] += min(600, remaining)
|
|
20
|
+
with destination.open("ab") as output:
|
|
21
|
+
output.write(b"partial")
|
|
22
|
+
return 28, 200, 200
|
|
23
|
+
|
|
24
|
+
with tempfile.TemporaryDirectory() as directory, contextlib.redirect_stderr(io.StringIO()) as error:
|
|
25
|
+
with patch.dict(os.environ, HTTPS_PROXY="http://fixture-user:fixture-secret@localhost:1",
|
|
26
|
+
GOWALK_FLUTTER_ARCHIVE_SHA256="a" * 64):
|
|
27
|
+
with patch.object(flutter_manifest.time, "monotonic", side_effect=lambda: elapsed[0]), \
|
|
28
|
+
patch.object(flutter_manifest.time, "sleep", side_effect=lambda seconds: None), \
|
|
29
|
+
patch.object(flutter_manifest.flutter_download, "transfer", side_effect=transfer):
|
|
30
|
+
result = flutter_manifest.download(["--connect-timeout", "15", "--retry", "5",
|
|
31
|
+
"https://fixture.invalid/flutter_infra_release/releases/stable/linux/sdk.tar.xz"],
|
|
32
|
+
Path(directory))
|
|
33
|
+
self.assertEqual(result, 28)
|
|
34
|
+
self.assertLess(elapsed[0], 1800 - 60, "A failure receipt must precede the enclosing job deadline")
|
|
35
|
+
receipt = next(line.split("::", 2)[2] for line in error.getvalue().splitlines()
|
|
36
|
+
if line.startswith("::error title=flutter_download_failed::"))
|
|
37
|
+
self.assertGreater(json.loads(receipt)["retained_bytes"], 0)
|
|
38
|
+
self.assertNotIn("fixture-secret", error.getvalue())
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
if __name__ == "__main__":
|
|
42
|
+
unittest.main()
|
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
"""Private child transfers expose byte progress and terminate within their own deadline."""
|
|
2
|
+
import contextlib
|
|
3
|
+
import io
|
|
4
|
+
import json
|
|
5
|
+
import os
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
import subprocess
|
|
8
|
+
import sys
|
|
9
|
+
import tempfile
|
|
10
|
+
import time
|
|
11
|
+
import unittest
|
|
12
|
+
from unittest.mock import patch
|
|
13
|
+
|
|
14
|
+
import flutter_download
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
class TransferProgressTests(unittest.TestCase):
|
|
18
|
+
def invoke(self, script, remaining):
|
|
19
|
+
children = []
|
|
20
|
+
popen = subprocess.Popen
|
|
21
|
+
with tempfile.TemporaryDirectory() as directory, contextlib.redirect_stderr(io.StringIO()) as error:
|
|
22
|
+
destination = Path(directory) / "archive"
|
|
23
|
+
destination.touch()
|
|
24
|
+
|
|
25
|
+
def child(_command, **kwargs):
|
|
26
|
+
process = popen([sys.executable, "-c", script, str(destination)], **kwargs)
|
|
27
|
+
children.append(process)
|
|
28
|
+
return process
|
|
29
|
+
|
|
30
|
+
with patch.object(flutter_download.subprocess, "Popen", side_effect=child), \
|
|
31
|
+
patch.object(flutter_download, "PROGRESS_SECONDS", .03):
|
|
32
|
+
result = flutter_download.transfer("https://fixture.invalid/sdk.tar.xz", destination,
|
|
33
|
+
"archive", remaining, dict(os.environ))
|
|
34
|
+
data = destination.read_bytes()
|
|
35
|
+
self.assertTrue(all(process.poll() is not None for process in children))
|
|
36
|
+
self.assertNotIn("fixture-secret", error.getvalue())
|
|
37
|
+
notices = [json.loads(line.split("::", 2)[2]) for line in error.getvalue().splitlines()]
|
|
38
|
+
self.assertTrue(notices)
|
|
39
|
+
self.assertTrue(any(note["retained_bytes"] > 0 for note in notices))
|
|
40
|
+
self.assertTrue(all(note["event"] == "transferring" for note in notices))
|
|
41
|
+
return result, data
|
|
42
|
+
|
|
43
|
+
def test_progress_does_not_publish_private_child_stderr_or_partial_archive(self):
|
|
44
|
+
script = ("import pathlib,sys,time; p=pathlib.Path(sys.argv[1]); p.write_bytes(b'partial'); "
|
|
45
|
+
"sys.stderr.write('fixture-secret'); time.sleep(.2); p.write_bytes(b'complete'); "
|
|
46
|
+
"print('206 200',end='')")
|
|
47
|
+
result, data = self.invoke(script, 2)
|
|
48
|
+
self.assertEqual((result, data), ((0, 206, 200), b"complete"))
|
|
49
|
+
|
|
50
|
+
def test_deadline_kills_only_the_owned_transfer_and_preserves_resume_bytes(self):
|
|
51
|
+
script = ("import pathlib,sys,time; pathlib.Path(sys.argv[1]).write_bytes(b'partial'); "
|
|
52
|
+
"sys.stderr.write('fixture-secret'); time.sleep(5)")
|
|
53
|
+
started = time.monotonic()
|
|
54
|
+
result, data = self.invoke(script, .2)
|
|
55
|
+
self.assertLess(time.monotonic() - started, 2)
|
|
56
|
+
self.assertEqual((result, data), ((28, 0, 0), b"partial"))
|
|
57
|
+
|
|
58
|
+
def test_progress_failure_does_not_leave_the_owned_child_running(self):
|
|
59
|
+
popen, children = subprocess.Popen, []
|
|
60
|
+
|
|
61
|
+
def child(_command, **kwargs):
|
|
62
|
+
process = popen([sys.executable, "-c", "import time; time.sleep(5)"], **kwargs)
|
|
63
|
+
children.append(process)
|
|
64
|
+
return process
|
|
65
|
+
|
|
66
|
+
started = time.monotonic()
|
|
67
|
+
with tempfile.TemporaryDirectory() as directory:
|
|
68
|
+
destination = Path(directory) / "archive"
|
|
69
|
+
destination.touch()
|
|
70
|
+
with patch.object(flutter_download.subprocess, "Popen", side_effect=child), \
|
|
71
|
+
patch.object(flutter_download, "PROGRESS_SECONDS", .03), \
|
|
72
|
+
patch.object(flutter_download, "progress", side_effect=OSError("closed output")):
|
|
73
|
+
with self.assertRaises(OSError):
|
|
74
|
+
flutter_download.transfer("https://fixture.invalid/sdk.tar.xz", destination,
|
|
75
|
+
"archive", 2, dict(os.environ))
|
|
76
|
+
self.assertLess(time.monotonic() - started, 2)
|
|
77
|
+
self.assertTrue(all(process.poll() is not None for process in children))
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
if __name__ == "__main__":
|
|
81
|
+
unittest.main()
|
|
@@ -0,0 +1,115 @@
|
|
|
1
|
+
"""Edit inserts retry only transport evidence that excludes an origin request."""
|
|
2
|
+
import io
|
|
3
|
+
from pathlib import Path
|
|
4
|
+
import ssl
|
|
5
|
+
import sys
|
|
6
|
+
import unittest
|
|
7
|
+
from unittest import mock
|
|
8
|
+
|
|
9
|
+
import requests
|
|
10
|
+
from urllib3 import exceptions as errors
|
|
11
|
+
|
|
12
|
+
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
|
13
|
+
import play_preflight_transport as transport
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def proxy_failure(cause=None):
|
|
17
|
+
cause = cause or OSError("Tunnel connection failed: 502 private proxy refusal")
|
|
18
|
+
reason = errors.ProxyError("private proxy", cause)
|
|
19
|
+
return requests.exceptions.ProxyError(errors.MaxRetryError(None, "private URL", reason))
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
class ConnectRetryTests(unittest.TestCase):
|
|
23
|
+
def setUp(self):
|
|
24
|
+
self.sleep = self.enterContext(mock.patch.object(transport.time, "sleep"))
|
|
25
|
+
self.output = self.enterContext(mock.patch("sys.stdout", new_callable=io.StringIO))
|
|
26
|
+
|
|
27
|
+
def test_proven_connection_failures_recover_with_one_successful_insert(self):
|
|
28
|
+
connection = errors.NewConnectionError(None, "private connect failure")
|
|
29
|
+
direct = requests.exceptions.ConnectionError(errors.MaxRetryError(None, "private URL", connection))
|
|
30
|
+
for error in (requests.exceptions.ConnectTimeout("private"), proxy_failure(),
|
|
31
|
+
proxy_failure(connection), direct):
|
|
32
|
+
with self.subTest(kind=type(error).__name__):
|
|
33
|
+
response = mock.Mock(status_code=200)
|
|
34
|
+
operation = mock.Mock(side_effect=[error, response])
|
|
35
|
+
self.assertIs(transport.call("edit_create", operation, retry_connect=True, accepted=(200,)), response)
|
|
36
|
+
self.assertEqual(operation.call_count, 2)
|
|
37
|
+
self.assertNotIn("private", self.output.getvalue())
|
|
38
|
+
|
|
39
|
+
def test_ambiguous_errors_never_authorize_an_insert_retry(self):
|
|
40
|
+
for error in (requests.exceptions.ReadTimeout("private"), requests.exceptions.ProxyError("private"),
|
|
41
|
+
requests.exceptions.ConnectionError(errors.ProtocolError("private")),
|
|
42
|
+
proxy_failure(OSError("unknown private refusal")),
|
|
43
|
+
proxy_failure(OSError("Tunnel connection failed: 407 private authentication")),
|
|
44
|
+
proxy_failure(errors.ProtocolError("Tunnel connection failed: 502 private"))):
|
|
45
|
+
with self.subTest(kind=type(error).__name__):
|
|
46
|
+
operation = mock.Mock(side_effect=error)
|
|
47
|
+
with self.assertRaises(SystemExit):
|
|
48
|
+
transport.call("edit_create", operation, retry_connect=True)
|
|
49
|
+
operation.assert_called_once()
|
|
50
|
+
self.sleep.assert_not_called()
|
|
51
|
+
|
|
52
|
+
def test_proxy_wrapped_tls_failure_remains_terminal(self):
|
|
53
|
+
error = proxy_failure(errors.SSLError(ssl.SSLCertVerificationError("private certificate")))
|
|
54
|
+
operation = mock.Mock(side_effect=error)
|
|
55
|
+
with self.assertRaises(SystemExit):
|
|
56
|
+
transport.call("edit_create", operation, retry_connect=True)
|
|
57
|
+
operation.assert_called_once()
|
|
58
|
+
self.sleep.assert_not_called()
|
|
59
|
+
self.assertNotIn("private", self.output.getvalue())
|
|
60
|
+
|
|
61
|
+
def test_origin_response_never_uses_connection_retry_permission(self):
|
|
62
|
+
operation = mock.Mock(return_value=mock.Mock(status_code=503))
|
|
63
|
+
with self.assertRaises(SystemExit) as caught:
|
|
64
|
+
transport.call("edit_create", operation, retry_connect=True, accepted=(200, 404))
|
|
65
|
+
operation.assert_called_once()
|
|
66
|
+
self.sleep.assert_not_called()
|
|
67
|
+
self.assertIn('"status": 503', str(caught.exception))
|
|
68
|
+
|
|
69
|
+
def test_lost_response_after_a_connection_retry_stops_at_that_attempt(self):
|
|
70
|
+
operation = mock.Mock(side_effect=[proxy_failure(), requests.exceptions.ReadTimeout("private")])
|
|
71
|
+
with self.assertRaises(SystemExit) as caught:
|
|
72
|
+
transport.call("edit_create", operation, retry_connect=True)
|
|
73
|
+
self.assertEqual(operation.call_count, 2)
|
|
74
|
+
self.assertIn('"attempts": 2', str(caught.exception))
|
|
75
|
+
self.sleep.assert_called_once_with(1)
|
|
76
|
+
|
|
77
|
+
def test_failed_connections_are_bounded_and_keep_safe_evidence(self):
|
|
78
|
+
operation = mock.Mock(side_effect=proxy_failure())
|
|
79
|
+
with self.assertRaises(SystemExit) as caught:
|
|
80
|
+
transport.call("edit_create", operation, retry_connect=True)
|
|
81
|
+
self.assertEqual(operation.call_count, 3)
|
|
82
|
+
self.assertEqual(self.sleep.call_args_list, [mock.call(1), mock.call(2)])
|
|
83
|
+
self.assertIn('"attempts": 3', str(caught.exception))
|
|
84
|
+
self.assertIn('"code": "proxy_connection_failed"', str(caught.exception))
|
|
85
|
+
self.assertNotIn("private", str(caught.exception) + self.output.getvalue())
|
|
86
|
+
|
|
87
|
+
def test_other_writes_have_no_implicit_retry_permission(self):
|
|
88
|
+
operation = mock.Mock(side_effect=proxy_failure())
|
|
89
|
+
with self.assertRaises(SystemExit):
|
|
90
|
+
transport.call("another_write", operation)
|
|
91
|
+
operation.assert_called_once()
|
|
92
|
+
self.sleep.assert_not_called()
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
class PreflightConnectWiringTests(unittest.TestCase):
|
|
96
|
+
def test_real_preflight_recovers_connection_before_one_known_edit_cleanup(self):
|
|
97
|
+
import play_preflight as preflight
|
|
98
|
+
|
|
99
|
+
client = mock.Mock()
|
|
100
|
+
client.post.side_effect = [proxy_failure(), mock.Mock(status_code=200, text='{"id":"known-edit"}')]
|
|
101
|
+
client.delete.return_value = mock.Mock(status_code=204)
|
|
102
|
+
credentials = mock.Mock(token="private token")
|
|
103
|
+
with mock.patch.object(preflight.service_account.Credentials, "from_service_account_file",
|
|
104
|
+
return_value=credentials), mock.patch.object(preflight, "write_ready") as ready, \
|
|
105
|
+
mock.patch.object(transport.time, "sleep"), mock.patch("sys.stdout", new_callable=io.StringIO):
|
|
106
|
+
preflight.check_ready(client, "com.example.app", Path("account.json"))
|
|
107
|
+
self.assertEqual(client.post.call_count, 2)
|
|
108
|
+
self.assertEqual(client.post.call_args_list[0], client.post.call_args_list[1])
|
|
109
|
+
client.delete.assert_called_once()
|
|
110
|
+
self.assertTrue(client.delete.call_args.args[0].endswith("/known-edit"))
|
|
111
|
+
ready.assert_called_once_with(True)
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
if __name__ == "__main__":
|
|
115
|
+
unittest.main()
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
"""Real Requests traffic proves bounded proxy retry and no replay after an origin POST."""
|
|
2
|
+
from contextlib import redirect_stdout
|
|
3
|
+
import io
|
|
4
|
+
import os
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
import sys
|
|
7
|
+
import unittest
|
|
8
|
+
from unittest import mock
|
|
9
|
+
|
|
10
|
+
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
|
11
|
+
import play_preflight_transport as transport
|
|
12
|
+
import play_store_proxy
|
|
13
|
+
from play_proxy_test_support import running
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
class ConnectRuntimeTests(unittest.TestCase):
|
|
17
|
+
def test_two_failed_tunnels_then_one_origin_post_on_the_same_assigned_proxy(self):
|
|
18
|
+
with running(reject=2) as (server, certificate):
|
|
19
|
+
env = {"GOOGLE_STORE_PROXY_URL": f"http://user:private-password@127.0.0.1:{server.server_port}",
|
|
20
|
+
"HTTPS_PROXY": "http://127.0.0.1:1", "NO_PROXY": "*"}
|
|
21
|
+
output = io.StringIO()
|
|
22
|
+
with mock.patch.dict(os.environ, env), play_store_proxy.session() as client, \
|
|
23
|
+
mock.patch.object(transport.time, "sleep"), redirect_stdout(output):
|
|
24
|
+
client.verify = certificate
|
|
25
|
+
result = transport.call("edit_create", lambda: client.post("https://play.test/edits", timeout=3),
|
|
26
|
+
retry_connect=True, accepted=(200,))
|
|
27
|
+
self.assertEqual(result.json(), {"id": "edit-1"})
|
|
28
|
+
self.assertEqual(server.connections, ["play.test:443"] * 3)
|
|
29
|
+
self.assertEqual(server.requests, [b"POST /edits HTTP/1.1"])
|
|
30
|
+
self.assertEqual(server.authenticated, [True] * 3)
|
|
31
|
+
self.assertNotIn("private", output.getvalue())
|
|
32
|
+
|
|
33
|
+
def test_dropped_response_after_received_post_is_never_replayed(self):
|
|
34
|
+
with running(drop_response=True) as (server, certificate):
|
|
35
|
+
env = {"GOOGLE_STORE_PROXY_URL": f"http://127.0.0.1:{server.server_port}"}
|
|
36
|
+
output = io.StringIO()
|
|
37
|
+
with mock.patch.dict(os.environ, env), play_store_proxy.session() as client, \
|
|
38
|
+
mock.patch.object(transport.time, "sleep") as sleep, redirect_stdout(output):
|
|
39
|
+
client.verify = certificate
|
|
40
|
+
with self.assertRaises(SystemExit) as caught:
|
|
41
|
+
transport.call("edit_create", lambda: client.post("https://play.test/edits", timeout=3),
|
|
42
|
+
retry_connect=True, accepted=(200,))
|
|
43
|
+
self.assertEqual(server.connections, ["play.test:443"])
|
|
44
|
+
self.assertEqual(server.requests, [b"POST /edits HTTP/1.1"])
|
|
45
|
+
self.assertIn('"attempts": 1', str(caught.exception))
|
|
46
|
+
sleep.assert_not_called()
|
|
47
|
+
|
|
48
|
+
def test_untrusted_tls_certificate_stops_before_origin_post_without_retry(self):
|
|
49
|
+
with running() as (server, _certificate):
|
|
50
|
+
env = {"GOOGLE_STORE_PROXY_URL": f"http://127.0.0.1:{server.server_port}"}
|
|
51
|
+
with mock.patch.dict(os.environ, env), play_store_proxy.session() as client, \
|
|
52
|
+
mock.patch.object(transport.time, "sleep") as sleep, redirect_stdout(io.StringIO()):
|
|
53
|
+
with self.assertRaises(SystemExit):
|
|
54
|
+
transport.call("edit_create", lambda: client.post("https://play.test/edits", timeout=3),
|
|
55
|
+
retry_connect=True, accepted=(200,))
|
|
56
|
+
self.assertEqual(server.connections, ["play.test:443"])
|
|
57
|
+
self.assertEqual(server.requests, [])
|
|
58
|
+
sleep.assert_not_called()
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
if __name__ == "__main__":
|
|
62
|
+
unittest.main()
|
|
@@ -1 +1 @@
|
|
|
1
|
-
1.0.
|
|
1
|
+
1.0.92
|