@sitar_fiercer4c/skills 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +5 -0
- package/README.md +75 -0
- package/bin/install.js +45 -0
- package/package.json +29 -0
- package/skills/architecture-walkthrough/SKILL.md +223 -0
- package/skills/architecture-walkthrough/references/sections.md +29 -0
- package/skills/architecture-walkthrough/scripts/check_structure.py +200 -0
- package/skills/autotest-webapp-ui/SKILL.md +58 -0
- package/skills/backend-code-review/SKILL.md +386 -0
- package/skills/backend-code-review/references/report-format.md +333 -0
- package/skills/backend-code-review/scripts/list_routes.py +269 -0
- package/skills/backend-code-review/scripts/sweep.py +550 -0
- package/skills/backend-code-review/scripts/verify_citations.py +201 -0
- package/skills/be-brief/SKILL.md +18 -0
- package/skills/clarke-list-excel/SKILL.md +51 -0
- package/skills/clarke-list-excel/references/output-schema.md +125 -0
- package/skills/clarke-list-excel/scripts/clarke_common.py +251 -0
- package/skills/clarke-list-excel/scripts/clarke_extract.py +487 -0
- package/skills/clarke-list-excel/scripts/load_clarke.py +322 -0
- package/skills/clarke-list-excel/scripts/run_all.py +63 -0
- package/skills/datalab-api/SKILL.md +163 -0
- package/skills/datalab-api/references/parameters-and-payload.md +121 -0
- package/skills/datalab-api/references/table-selection.md +35 -0
- package/skills/datalab-api/scripts/datalab_tables.py +365 -0
- package/skills/find-test-seam/SKILL.md +41 -0
- package/skills/frontend-code-review/SKILL.md +247 -0
- package/skills/frontend-code-review-2/SKILL.md +192 -0
- package/skills/frontend-code-review-2/scripts/fetch_pr_comments.py +65 -0
- package/skills/frontend-code-review-2/scripts/render_report.py +139 -0
- package/skills/murtaza-breif/SKILL.md +143 -0
- package/skills/murtaza-breif/scripts/save_brief.py +128 -0
- package/skills/pdf-to-json/SKILL.md +42 -0
- package/skills/pdf-to-json/references/output-schema.md +168 -0
- package/skills/pdf-to-json/scripts/extract_figures.py +319 -0
- package/skills/pdf-to-json/scripts/load_mongo.py +287 -0
- package/skills/pdf-to-json/scripts/pdf_extract.py +1313 -0
- package/skills/record-api-traffic/SKILL.md +434 -0
- package/skills/record-api-traffic/references/reading-recordings.md +224 -0
- package/skills/record-api-traffic/scripts/check-schema.mjs +184 -0
- package/skills/record-api-traffic/scripts/dump-quotation.mjs +67 -0
- package/skills/record-api-traffic/scripts/dump-source-excel.mjs +75 -0
- package/skills/record-api-traffic/scripts/lib/repo.mjs +109 -0
- package/skills/record-api-traffic/scripts/preflight.py +528 -0
- package/skills/record-api-traffic/scripts/record-api-traffic.py +720 -0
- package/skills/refac-wrt-business-goal/SKILL.md +305 -0
- package/skills/refac-wrt-business-goal/references/critic.md +170 -0
- package/skills/system-resource-triage/SKILL.md +180 -0
- package/skills/system-resource-triage/scripts/reap.sh +116 -0
- package/skills/system-resource-triage/scripts/triage.sh +111 -0
- package/skills/using-git-worktrees/SKILL.md +167 -0
|
@@ -0,0 +1,720 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""Record every HTTP call a Node/Next app makes, on all four legs.
|
|
3
|
+
|
|
4
|
+
Run it with the repo path, open the link it prints, use the app, Ctrl-C.
|
|
5
|
+
Each leg gets an aligned table to scan and a JSONL file with full
|
|
6
|
+
request/response bodies.
|
|
7
|
+
|
|
8
|
+
Both servers are recorded the same way: a mitmdump reverse proxy takes the
|
|
9
|
+
port the browser already calls, and the server moves one port up behind it,
|
|
10
|
+
so nothing on the VS Code side and no config file in the repo has to change.
|
|
11
|
+
|
|
12
|
+
browser -> :4100 recorder -> :4101 frontend frontend-inbound.*
|
|
13
|
+
browser -> :7100 recorder -> :7101 backend backend-inbound.*
|
|
14
|
+
|
|
15
|
+
Each server also gets a mitmdump in regular (forward) mode recording the
|
|
16
|
+
calls it makes itself, from Node rather than the browser:
|
|
17
|
+
|
|
18
|
+
frontend -> :4102 recorder -> the backend, and anything external
|
|
19
|
+
backend -> :7102 recorder -> Microsoft Graph, Datalab
|
|
20
|
+
|
|
21
|
+
--no-frontend drops the frontend legs, --no-outbound drops both forward
|
|
22
|
+
proxies, --frontend-all keeps the static-asset noise the frontend leg
|
|
23
|
+
discards by default.
|
|
24
|
+
|
|
25
|
+
Nothing is written inside the repo. The backend's scratch working directory
|
|
26
|
+
is a temp dir, and recordings land in this skill's own workspace.
|
|
27
|
+
|
|
28
|
+
This file plays two roles. Run directly, it orchestrates. Loaded by
|
|
29
|
+
mitmdump via -s, it is the recording addon.
|
|
30
|
+
"""
|
|
31
|
+
|
|
32
|
+
import argparse
|
|
33
|
+
import json
|
|
34
|
+
import os
|
|
35
|
+
import re
|
|
36
|
+
import shutil
|
|
37
|
+
import signal
|
|
38
|
+
import socket
|
|
39
|
+
import subprocess
|
|
40
|
+
import sys
|
|
41
|
+
import tempfile
|
|
42
|
+
import threading
|
|
43
|
+
import time
|
|
44
|
+
import urllib.error
|
|
45
|
+
import urllib.request
|
|
46
|
+
from collections import deque
|
|
47
|
+
from datetime import datetime
|
|
48
|
+
from pathlib import Path
|
|
49
|
+
|
|
50
|
+
SKILL = Path(__file__).resolve().parent.parent
|
|
51
|
+
WORKSPACE = SKILL.parent / "record-api-traffic-workspace" / "recordings"
|
|
52
|
+
|
|
53
|
+
PROXY_PORT = 7100
|
|
54
|
+
BACKEND_PORT = 7101
|
|
55
|
+
OUTBOUND_PORT = 7102
|
|
56
|
+
FRONTEND_PROXY_PORT = 4100
|
|
57
|
+
FRONTEND_PORT = 4101
|
|
58
|
+
FRONTEND_OUTBOUND_PORT = 4102
|
|
59
|
+
|
|
60
|
+
SCRATCH_PREFIX = "record-api-traffic-"
|
|
61
|
+
|
|
62
|
+
# Every child is started with this in its environment, so a leftover process
|
|
63
|
+
# from a crashed run can be told apart from the app's own dev server even
|
|
64
|
+
# though the frontend has to run with its real working directory.
|
|
65
|
+
CHILD_MARKER = "RECORD_API_TRAFFIC_RUN"
|
|
66
|
+
|
|
67
|
+
# Build output and dev-server plumbing on the frontend leg. One page load in
|
|
68
|
+
# `next dev` pulls dozens of chunks, each recorded in full would bury the
|
|
69
|
+
# handful of rows that carry meaning -- navigations, RSC payloads, server
|
|
70
|
+
# actions and any route handler the frontend serves itself.
|
|
71
|
+
FRONTEND_NOISE = r"^/(?:_next/static/|_next/image|_next/webpack-hmr|__nextjs|favicon\.ico$)"
|
|
72
|
+
|
|
73
|
+
MITM_CA = Path.home() / ".mitmproxy" / "mitmproxy-ca-cert.pem"
|
|
74
|
+
|
|
75
|
+
MAX_BODY = 1024 * 1024
|
|
76
|
+
|
|
77
|
+
BINARY_TYPES = (
|
|
78
|
+
"image/",
|
|
79
|
+
"audio/",
|
|
80
|
+
"video/",
|
|
81
|
+
"font/",
|
|
82
|
+
"application/pdf",
|
|
83
|
+
"application/zip",
|
|
84
|
+
"application/octet-stream",
|
|
85
|
+
"application/vnd.openxmlformats",
|
|
86
|
+
"application/vnd.ms-excel",
|
|
87
|
+
)
|
|
88
|
+
|
|
89
|
+
ROW_FMT = "{ts:<12} {id:<4} {ms:>6} {method:<6} {status:<6} {path:<44} {req:>9} {resp:>9}"
|
|
90
|
+
HEADER = ROW_FMT.format(
|
|
91
|
+
ts="TIME", id="ID", ms="MS", method="METHOD", status="STATUS",
|
|
92
|
+
path="PATH", req="REQ", resp="RESP",
|
|
93
|
+
)
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def human_size(n):
|
|
97
|
+
if n is None:
|
|
98
|
+
return "-"
|
|
99
|
+
if n == 0:
|
|
100
|
+
return "-"
|
|
101
|
+
if n < 1024:
|
|
102
|
+
return f"{n} B"
|
|
103
|
+
if n < 1024 * 1024:
|
|
104
|
+
return f"{n / 1024:.1f} KB"
|
|
105
|
+
return f"{n / (1024 * 1024):.1f} MB"
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
def workspace_slug(repo):
|
|
109
|
+
"""Last two path segments, so two clones of the same repo stay apart."""
|
|
110
|
+
parts = [p for p in repo.parts if p not in ("/", "")][-2:]
|
|
111
|
+
return re.sub(r"[^a-z0-9]+", "-", "-".join(parts).lower()).strip("-")
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
# --------------------------------------------------------------------------
|
|
115
|
+
# Recorder — active only inside the mitmdump child process.
|
|
116
|
+
# --------------------------------------------------------------------------
|
|
117
|
+
|
|
118
|
+
class Recorder:
|
|
119
|
+
def __init__(self):
|
|
120
|
+
self.jsonl_path = os.environ["MITM_RECORD_JSONL"]
|
|
121
|
+
self.log_path = os.environ["MITM_RECORD_LOG"]
|
|
122
|
+
self.show_host = bool(os.environ.get("MITM_RECORD_SHOW_HOST"))
|
|
123
|
+
skip = os.environ.get("MITM_RECORD_SKIP")
|
|
124
|
+
self.skip = re.compile(skip) if skip else None
|
|
125
|
+
self.jsonl = None
|
|
126
|
+
self.log = None
|
|
127
|
+
self.counter = 0
|
|
128
|
+
|
|
129
|
+
def running(self):
|
|
130
|
+
self.jsonl = open(self.jsonl_path, "a", encoding="utf-8")
|
|
131
|
+
self.log = open(self.log_path, "a", encoding="utf-8")
|
|
132
|
+
self.log.write(HEADER + "\n")
|
|
133
|
+
self.log.flush()
|
|
134
|
+
|
|
135
|
+
def done(self):
|
|
136
|
+
for handle in (self.jsonl, self.log):
|
|
137
|
+
if handle:
|
|
138
|
+
handle.close()
|
|
139
|
+
|
|
140
|
+
def response(self, flow):
|
|
141
|
+
self._record(flow, None)
|
|
142
|
+
|
|
143
|
+
def error(self, flow):
|
|
144
|
+
if flow.response is None:
|
|
145
|
+
self._record(flow, str(flow.error) if flow.error else "connection error")
|
|
146
|
+
|
|
147
|
+
def _record(self, flow, error):
|
|
148
|
+
req = flow.request
|
|
149
|
+
# Before the counter moves, so a dropped call leaves no gap and the
|
|
150
|
+
# row ID stays equal to the line number in the JSONL.
|
|
151
|
+
if self.skip and self.skip.search(req.path.split("?")[0]):
|
|
152
|
+
return
|
|
153
|
+
|
|
154
|
+
self.counter += 1
|
|
155
|
+
rec_id = f"{self.counter:04d}"
|
|
156
|
+
|
|
157
|
+
resp = flow.response
|
|
158
|
+
|
|
159
|
+
started = req.timestamp_start
|
|
160
|
+
ended = (resp.timestamp_end if resp and resp.timestamp_end else time.time())
|
|
161
|
+
duration_ms = int((ended - started) * 1000)
|
|
162
|
+
|
|
163
|
+
req_body, req_size = self._body(req)
|
|
164
|
+
if resp is not None:
|
|
165
|
+
resp_body, resp_size = self._body(resp)
|
|
166
|
+
else:
|
|
167
|
+
resp_body, resp_size = None, 0
|
|
168
|
+
|
|
169
|
+
record = {
|
|
170
|
+
"id": rec_id,
|
|
171
|
+
"ts": datetime.fromtimestamp(ended).astimezone().isoformat(),
|
|
172
|
+
"started_at": datetime.fromtimestamp(started).astimezone().isoformat(),
|
|
173
|
+
"duration_ms": duration_ms,
|
|
174
|
+
"method": req.method,
|
|
175
|
+
"path": req.path.split("?")[0],
|
|
176
|
+
"host": req.pretty_host,
|
|
177
|
+
"url": req.pretty_url,
|
|
178
|
+
"query": dict(req.query),
|
|
179
|
+
"client_ip": flow.client_conn.peername[0] if flow.client_conn.peername else None,
|
|
180
|
+
"request": {
|
|
181
|
+
"headers": dict(req.headers),
|
|
182
|
+
"body": req_body["value"],
|
|
183
|
+
"body_encoding": req_body["encoding"],
|
|
184
|
+
"body_size": req_size,
|
|
185
|
+
"truncated": req_body["truncated"],
|
|
186
|
+
},
|
|
187
|
+
}
|
|
188
|
+
if resp is not None:
|
|
189
|
+
record["response"] = {
|
|
190
|
+
"status": resp.status_code,
|
|
191
|
+
"headers": dict(resp.headers),
|
|
192
|
+
"body": resp_body["value"],
|
|
193
|
+
"body_encoding": resp_body["encoding"],
|
|
194
|
+
"body_size": resp_size,
|
|
195
|
+
"truncated": resp_body["truncated"],
|
|
196
|
+
}
|
|
197
|
+
if error:
|
|
198
|
+
record["error"] = error
|
|
199
|
+
|
|
200
|
+
self.jsonl.write(json.dumps(record, ensure_ascii=False, default=str) + "\n")
|
|
201
|
+
self.jsonl.flush()
|
|
202
|
+
|
|
203
|
+
path = req.path
|
|
204
|
+
if self.show_host:
|
|
205
|
+
# With the port: a forward-proxy leg spans hosts, and on the
|
|
206
|
+
# frontend's it also spans loopback ports, where :7100 (the
|
|
207
|
+
# backend) and :4101 (the frontend calling itself) are otherwise
|
|
208
|
+
# the same "127.0.0.1".
|
|
209
|
+
host = req.pretty_host
|
|
210
|
+
if req.port != (443 if req.scheme == "https" else 80):
|
|
211
|
+
host = f"{host}:{req.port}"
|
|
212
|
+
path = f"{host}{req.path}"
|
|
213
|
+
if len(path) > 44:
|
|
214
|
+
path = path[:43] + "…"
|
|
215
|
+
self.log.write(ROW_FMT.format(
|
|
216
|
+
ts=datetime.fromtimestamp(ended).strftime("%H:%M:%S.%f")[:-3],
|
|
217
|
+
id=rec_id,
|
|
218
|
+
ms=duration_ms,
|
|
219
|
+
method=req.method,
|
|
220
|
+
status=str(resp.status_code) if resp is not None else "ERR",
|
|
221
|
+
path=path,
|
|
222
|
+
req=human_size(req_size),
|
|
223
|
+
resp=human_size(resp_size) if resp is not None else "-",
|
|
224
|
+
) + "\n")
|
|
225
|
+
self.log.flush()
|
|
226
|
+
|
|
227
|
+
def _body(self, message):
|
|
228
|
+
"""Return ({value, encoding, truncated}, size) for a request or response."""
|
|
229
|
+
empty = {"value": None, "encoding": None, "truncated": False}
|
|
230
|
+
try:
|
|
231
|
+
content = message.get_content(strict=False)
|
|
232
|
+
except Exception:
|
|
233
|
+
content = message.raw_content
|
|
234
|
+
if not content:
|
|
235
|
+
return empty, 0
|
|
236
|
+
|
|
237
|
+
size = len(content)
|
|
238
|
+
ctype = message.headers.get("content-type", "").lower()
|
|
239
|
+
|
|
240
|
+
if any(ctype.startswith(t) for t in BINARY_TYPES):
|
|
241
|
+
return {
|
|
242
|
+
"value": {"binary": True, "content_type": ctype or None, "size": size},
|
|
243
|
+
"encoding": "binary",
|
|
244
|
+
"truncated": False,
|
|
245
|
+
}, size
|
|
246
|
+
|
|
247
|
+
truncated = size > MAX_BODY
|
|
248
|
+
text = content[:MAX_BODY].decode("utf-8", errors="replace")
|
|
249
|
+
if truncated:
|
|
250
|
+
return {"value": text, "encoding": "text", "truncated": True}, size
|
|
251
|
+
|
|
252
|
+
try:
|
|
253
|
+
return {"value": json.loads(text), "encoding": "json", "truncated": False}, size
|
|
254
|
+
except (ValueError, TypeError):
|
|
255
|
+
return {"value": text, "encoding": "text", "truncated": False}, size
|
|
256
|
+
|
|
257
|
+
|
|
258
|
+
if os.environ.get("MITM_RECORD_JSONL"):
|
|
259
|
+
addons = [Recorder()]
|
|
260
|
+
|
|
261
|
+
|
|
262
|
+
# --------------------------------------------------------------------------
|
|
263
|
+
# Launcher
|
|
264
|
+
# --------------------------------------------------------------------------
|
|
265
|
+
|
|
266
|
+
class Child:
|
|
267
|
+
def __init__(self, name, argv, cwd, env=None):
|
|
268
|
+
self.name = name
|
|
269
|
+
self.tail = deque(maxlen=40)
|
|
270
|
+
env = dict(env or os.environ)
|
|
271
|
+
env[CHILD_MARKER] = str(os.getpid())
|
|
272
|
+
self.proc = subprocess.Popen(
|
|
273
|
+
argv,
|
|
274
|
+
cwd=str(cwd),
|
|
275
|
+
env=env,
|
|
276
|
+
stdout=subprocess.PIPE,
|
|
277
|
+
stderr=subprocess.STDOUT,
|
|
278
|
+
text=True,
|
|
279
|
+
bufsize=1,
|
|
280
|
+
start_new_session=True,
|
|
281
|
+
)
|
|
282
|
+
threading.Thread(target=self._pump, daemon=True).start()
|
|
283
|
+
|
|
284
|
+
def _pump(self):
|
|
285
|
+
for line in self.proc.stdout:
|
|
286
|
+
line = line.rstrip()
|
|
287
|
+
self.tail.append(line)
|
|
288
|
+
print(f"[{self.name}] {line}", flush=True)
|
|
289
|
+
|
|
290
|
+
def alive(self):
|
|
291
|
+
return self.proc.poll() is None
|
|
292
|
+
|
|
293
|
+
def stop(self):
|
|
294
|
+
if self.proc.poll() is not None:
|
|
295
|
+
return
|
|
296
|
+
try:
|
|
297
|
+
pgid = os.getpgid(self.proc.pid)
|
|
298
|
+
except ProcessLookupError:
|
|
299
|
+
return
|
|
300
|
+
for sig in (signal.SIGTERM, signal.SIGKILL):
|
|
301
|
+
try:
|
|
302
|
+
os.killpg(pgid, sig)
|
|
303
|
+
except ProcessLookupError:
|
|
304
|
+
return
|
|
305
|
+
deadline = time.time() + 10
|
|
306
|
+
while time.time() < deadline:
|
|
307
|
+
if self.proc.poll() is not None:
|
|
308
|
+
return
|
|
309
|
+
time.sleep(0.2)
|
|
310
|
+
|
|
311
|
+
|
|
312
|
+
children = []
|
|
313
|
+
scratch = None
|
|
314
|
+
cleaned = False
|
|
315
|
+
|
|
316
|
+
|
|
317
|
+
def port_in_use(port):
|
|
318
|
+
with socket.socket() as s:
|
|
319
|
+
s.settimeout(0.5)
|
|
320
|
+
return s.connect_ex(("127.0.0.1", port)) == 0
|
|
321
|
+
|
|
322
|
+
|
|
323
|
+
def port_holder(port):
|
|
324
|
+
try:
|
|
325
|
+
out = subprocess.run(
|
|
326
|
+
["ss", "-ltnp", f"sport = :{port}"],
|
|
327
|
+
capture_output=True, text=True, timeout=5,
|
|
328
|
+
).stdout
|
|
329
|
+
match = re.search(r'users:\(\("([^"]+)",pid=(\d+)', out)
|
|
330
|
+
if match:
|
|
331
|
+
return f"{match.group(1)} (pid {match.group(2)})"
|
|
332
|
+
except Exception:
|
|
333
|
+
pass
|
|
334
|
+
return "unknown process"
|
|
335
|
+
|
|
336
|
+
|
|
337
|
+
def wait_for_http(url, timeout, child=None):
|
|
338
|
+
deadline = time.time() + timeout
|
|
339
|
+
while time.time() < deadline:
|
|
340
|
+
if child and not child.alive():
|
|
341
|
+
return False
|
|
342
|
+
try:
|
|
343
|
+
with urllib.request.urlopen(url, timeout=3) as r:
|
|
344
|
+
if r.status < 500:
|
|
345
|
+
return True
|
|
346
|
+
except urllib.error.HTTPError as e:
|
|
347
|
+
if e.code < 500:
|
|
348
|
+
return True
|
|
349
|
+
except Exception:
|
|
350
|
+
pass
|
|
351
|
+
time.sleep(0.5)
|
|
352
|
+
return False
|
|
353
|
+
|
|
354
|
+
|
|
355
|
+
def wait_for_port(port, timeout, child=None):
|
|
356
|
+
deadline = time.time() + timeout
|
|
357
|
+
while time.time() < deadline:
|
|
358
|
+
if child and not child.alive():
|
|
359
|
+
return False
|
|
360
|
+
if port_in_use(port):
|
|
361
|
+
return True
|
|
362
|
+
time.sleep(0.5)
|
|
363
|
+
return False
|
|
364
|
+
|
|
365
|
+
|
|
366
|
+
def die(message, child=None):
|
|
367
|
+
print(f"\n ERROR: {message}\n", file=sys.stderr)
|
|
368
|
+
if child and child.tail:
|
|
369
|
+
print(f" last output from {child.name}:", file=sys.stderr)
|
|
370
|
+
for line in child.tail:
|
|
371
|
+
print(f" {line}", file=sys.stderr)
|
|
372
|
+
print(file=sys.stderr)
|
|
373
|
+
cleanup()
|
|
374
|
+
sys.exit(1)
|
|
375
|
+
|
|
376
|
+
|
|
377
|
+
def cleanup(*_):
|
|
378
|
+
global cleaned
|
|
379
|
+
if cleaned:
|
|
380
|
+
return
|
|
381
|
+
cleaned = True
|
|
382
|
+
print("\n Shutting down...", flush=True)
|
|
383
|
+
for child in reversed(children):
|
|
384
|
+
child.stop()
|
|
385
|
+
if scratch:
|
|
386
|
+
shutil.rmtree(scratch, ignore_errors=True)
|
|
387
|
+
|
|
388
|
+
|
|
389
|
+
def preflight(repo, outbound, record_frontend):
|
|
390
|
+
backend, frontend = repo / "Backend", repo / "Frontend"
|
|
391
|
+
if not shutil.which("mitmdump"):
|
|
392
|
+
die("mitmdump not found on PATH. Install with: pipx install mitmproxy")
|
|
393
|
+
for path in (backend / ".env", frontend / ".env.local",
|
|
394
|
+
backend / "node_modules", frontend / "node_modules"):
|
|
395
|
+
if not path.exists():
|
|
396
|
+
die(f"missing {path}")
|
|
397
|
+
ports = [(PROXY_PORT, "backend proxy"), (BACKEND_PORT, "backend"),
|
|
398
|
+
(FRONTEND_PROXY_PORT, "frontend proxy" if record_frontend else "frontend")]
|
|
399
|
+
if outbound:
|
|
400
|
+
ports.append((OUTBOUND_PORT, "backend outbound proxy"))
|
|
401
|
+
if record_frontend:
|
|
402
|
+
ports.append((FRONTEND_PORT, "frontend"))
|
|
403
|
+
if outbound:
|
|
404
|
+
ports.append((FRONTEND_OUTBOUND_PORT, "frontend outbound proxy"))
|
|
405
|
+
for port, what in ports:
|
|
406
|
+
if port_in_use(port):
|
|
407
|
+
die(f"port {port} ({what}) is already in use by {port_holder(port)}. "
|
|
408
|
+
"Stop it and re-run.")
|
|
409
|
+
if outbound and not MITM_CA.exists():
|
|
410
|
+
die(f"missing {MITM_CA}. mitmproxy writes it on its first run; start "
|
|
411
|
+
"mitmdump once, or pass --no-outbound.")
|
|
412
|
+
|
|
413
|
+
|
|
414
|
+
def dev_script(frontend):
|
|
415
|
+
try:
|
|
416
|
+
pkg = json.loads((frontend / "package.json").read_text())
|
|
417
|
+
except (OSError, ValueError):
|
|
418
|
+
return ""
|
|
419
|
+
return (pkg.get("scripts") or {}).get("dev", "")
|
|
420
|
+
|
|
421
|
+
|
|
422
|
+
def frontend_argv(frontend, port):
|
|
423
|
+
"""How to start the dev server on `port`.
|
|
424
|
+
|
|
425
|
+
A Next dev script normally pins the port itself (`next dev -H 0.0.0.0
|
|
426
|
+
-p 4100`), and a pinned CLI flag beats the PORT env var -- commander
|
|
427
|
+
only consults `.env("PORT")` when the flag is absent. Appending a second
|
|
428
|
+
`-p` moves it anyway: both argument parsers Next has shipped, `arg` and
|
|
429
|
+
`commander`, take the last occurrence of a non-variadic flag.
|
|
430
|
+
|
|
431
|
+
PORT is still exported alongside, for a dev script that leaves the port
|
|
432
|
+
unpinned and for anything downstream that reads it.
|
|
433
|
+
"""
|
|
434
|
+
script = dev_script(frontend)
|
|
435
|
+
if "next" in script:
|
|
436
|
+
return ["npm", "run", "dev", "--", "-p", str(port)]
|
|
437
|
+
return ["npm", "run", "dev"]
|
|
438
|
+
|
|
439
|
+
|
|
440
|
+
def make_scratch(backend):
|
|
441
|
+
"""A temp working directory holding a copy of .env with PORT overridden.
|
|
442
|
+
|
|
443
|
+
The backend's app.js calls dotenv.config({ override: true }), so a shell
|
|
444
|
+
env var loses to .env. dotenv resolves its file from the working
|
|
445
|
+
directory, so moving the port means moving the cwd -- and it lives outside
|
|
446
|
+
the repo so that recording never writes there.
|
|
447
|
+
"""
|
|
448
|
+
path = Path(tempfile.mkdtemp(prefix=SCRATCH_PREFIX))
|
|
449
|
+
path.chmod(0o700)
|
|
450
|
+
lines = (backend / ".env").read_text().splitlines()
|
|
451
|
+
replaced = False
|
|
452
|
+
for i, line in enumerate(lines):
|
|
453
|
+
if re.match(r"^\s*PORT\s*=", line):
|
|
454
|
+
lines[i] = f"PORT={BACKEND_PORT}"
|
|
455
|
+
replaced = True
|
|
456
|
+
if not replaced:
|
|
457
|
+
lines.append(f"PORT={BACKEND_PORT}")
|
|
458
|
+
target = path / ".env"
|
|
459
|
+
target.write_text("\n".join(lines) + "\n")
|
|
460
|
+
target.chmod(0o600)
|
|
461
|
+
return path
|
|
462
|
+
|
|
463
|
+
|
|
464
|
+
class Leg:
|
|
465
|
+
"""One recorded direction: a label and the pair of files it writes."""
|
|
466
|
+
|
|
467
|
+
def __init__(self, run_dir, stem, title):
|
|
468
|
+
self.stem = stem
|
|
469
|
+
self.title = title
|
|
470
|
+
self.jsonl = run_dir / f"{stem}.jsonl"
|
|
471
|
+
self.log = run_dir / f"{stem}.log"
|
|
472
|
+
|
|
473
|
+
def summarise(self):
|
|
474
|
+
total = failed = 0
|
|
475
|
+
try:
|
|
476
|
+
with open(self.jsonl, encoding="utf-8") as f:
|
|
477
|
+
for line in f:
|
|
478
|
+
if not line.strip():
|
|
479
|
+
continue
|
|
480
|
+
total += 1
|
|
481
|
+
try:
|
|
482
|
+
rec = json.loads(line)
|
|
483
|
+
except ValueError:
|
|
484
|
+
continue
|
|
485
|
+
status = rec.get("response", {}).get("status")
|
|
486
|
+
if status is None or status >= 400:
|
|
487
|
+
failed += 1
|
|
488
|
+
except FileNotFoundError:
|
|
489
|
+
pass
|
|
490
|
+
print(f"\n {self.title}: {total} calls ({failed} failed).")
|
|
491
|
+
print(f" Table : {self.log}")
|
|
492
|
+
print(f" Details : {self.jsonl}")
|
|
493
|
+
|
|
494
|
+
|
|
495
|
+
def start_proxy(name, leg, listen_port, mode, cwd,
|
|
496
|
+
listen_host="127.0.0.1", show_host=False, skip=None):
|
|
497
|
+
"""Start one recording mitmdump and register it for teardown."""
|
|
498
|
+
env = os.environ.copy()
|
|
499
|
+
env["MITM_RECORD_JSONL"] = str(leg.jsonl)
|
|
500
|
+
env["MITM_RECORD_LOG"] = str(leg.log)
|
|
501
|
+
if show_host:
|
|
502
|
+
env["MITM_RECORD_SHOW_HOST"] = "1"
|
|
503
|
+
if skip:
|
|
504
|
+
env["MITM_RECORD_SKIP"] = skip
|
|
505
|
+
argv = [
|
|
506
|
+
"mitmdump",
|
|
507
|
+
"--mode", mode,
|
|
508
|
+
"--listen-host", listen_host,
|
|
509
|
+
"--listen-port", str(listen_port),
|
|
510
|
+
"-s", str(Path(__file__).resolve()),
|
|
511
|
+
"-q",
|
|
512
|
+
]
|
|
513
|
+
if mode.startswith("reverse"):
|
|
514
|
+
argv += ["--set", "keep_host_header=true"]
|
|
515
|
+
child = Child(name, argv, cwd, env)
|
|
516
|
+
children.append(child)
|
|
517
|
+
if not wait_for_port(listen_port, 30, child):
|
|
518
|
+
die(f"{name} recorder did not listen on :{listen_port}", child)
|
|
519
|
+
return child
|
|
520
|
+
|
|
521
|
+
|
|
522
|
+
def proxy_env(env, port):
|
|
523
|
+
"""Point a Node process's outbound HTTP at one of our forward proxies.
|
|
524
|
+
|
|
525
|
+
Node's global fetch honours the proxy variables only with
|
|
526
|
+
NODE_USE_ENV_PROXY (Node 24+); NODE_EXTRA_CA_CERTS is what makes it
|
|
527
|
+
trust mitmproxy's certificate for the HTTPS calls it intercepts.
|
|
528
|
+
"""
|
|
529
|
+
env["NODE_USE_ENV_PROXY"] = "1"
|
|
530
|
+
env["HTTP_PROXY"] = f"http://127.0.0.1:{port}"
|
|
531
|
+
env["HTTPS_PROXY"] = f"http://127.0.0.1:{port}"
|
|
532
|
+
env["NODE_EXTRA_CA_CERTS"] = str(MITM_CA)
|
|
533
|
+
return env
|
|
534
|
+
|
|
535
|
+
|
|
536
|
+
def main():
|
|
537
|
+
global scratch
|
|
538
|
+
|
|
539
|
+
parser = argparse.ArgumentParser(description=__doc__.split("\n")[0])
|
|
540
|
+
parser.add_argument("repo", help="path to the app repo")
|
|
541
|
+
parser.add_argument("--no-outbound", action="store_true",
|
|
542
|
+
help="skip both forward proxies (:7102 Graph/Datalab, "
|
|
543
|
+
":4102 the frontend's own server-side calls)")
|
|
544
|
+
parser.add_argument("--no-frontend", action="store_true",
|
|
545
|
+
help="leave the dev server on :4100 unrecorded, as it "
|
|
546
|
+
"was before the frontend legs existed")
|
|
547
|
+
parser.add_argument("--frontend-all", action="store_true",
|
|
548
|
+
help="keep /_next/static and other dev-server noise in "
|
|
549
|
+
"the frontend recording instead of dropping it")
|
|
550
|
+
args = parser.parse_args()
|
|
551
|
+
|
|
552
|
+
repo = Path(args.repo).expanduser().resolve()
|
|
553
|
+
if not repo.exists():
|
|
554
|
+
print(f"repo not found: {repo}", file=sys.stderr)
|
|
555
|
+
return 2
|
|
556
|
+
backend, frontend = repo / "Backend", repo / "Frontend"
|
|
557
|
+
|
|
558
|
+
outbound = not args.no_outbound
|
|
559
|
+
record_frontend = not args.no_frontend
|
|
560
|
+
# Unrecorded, the dev server stays on the port the browser already calls.
|
|
561
|
+
frontend_port = FRONTEND_PORT if record_frontend else FRONTEND_PROXY_PORT
|
|
562
|
+
preflight(repo, outbound, record_frontend)
|
|
563
|
+
|
|
564
|
+
stamp = datetime.now().strftime("%Y%m%d-%H%M%S")
|
|
565
|
+
run_dir = WORKSPACE / workspace_slug(repo) / f"run-{stamp}"
|
|
566
|
+
run_dir.mkdir(parents=True, exist_ok=True)
|
|
567
|
+
(run_dir / "run.json").write_text(json.dumps({
|
|
568
|
+
"repo": str(repo), "started_at": datetime.now().astimezone().isoformat(),
|
|
569
|
+
"outbound": outbound,
|
|
570
|
+
"frontend": record_frontend,
|
|
571
|
+
"frontend_all": bool(args.frontend_all),
|
|
572
|
+
}, indent=2) + "\n")
|
|
573
|
+
|
|
574
|
+
be_in = Leg(run_dir, "backend-inbound", "into backend (browser, frontend)")
|
|
575
|
+
be_out = Leg(run_dir, "backend-outbound", "out of backend (Graph, Datalab)")
|
|
576
|
+
fe_in = Leg(run_dir, "frontend-inbound", "into frontend (browser)")
|
|
577
|
+
fe_out = Leg(run_dir, "frontend-outbound", "out of frontend (backend, external)")
|
|
578
|
+
|
|
579
|
+
def on_sigterm(*_):
|
|
580
|
+
raise KeyboardInterrupt
|
|
581
|
+
|
|
582
|
+
signal.signal(signal.SIGTERM, on_sigterm)
|
|
583
|
+
|
|
584
|
+
scratch = make_scratch(backend)
|
|
585
|
+
|
|
586
|
+
backend_env = os.environ.copy()
|
|
587
|
+
if outbound:
|
|
588
|
+
print(f" Starting backend outbound recorder on :{OUTBOUND_PORT} ...", flush=True)
|
|
589
|
+
start_proxy("be-outbound", be_out, OUTBOUND_PORT, "regular", scratch, show_host=True)
|
|
590
|
+
backend_env = proxy_env(backend_env, OUTBOUND_PORT)
|
|
591
|
+
backend_env["NO_PROXY"] = "127.0.0.1,localhost"
|
|
592
|
+
|
|
593
|
+
print(f" Starting backend on :{BACKEND_PORT} ...", flush=True)
|
|
594
|
+
backend_child = Child("backend", ["node", str(backend / "src" / "server.js")],
|
|
595
|
+
scratch, backend_env)
|
|
596
|
+
children.append(backend_child)
|
|
597
|
+
if not wait_for_http(f"http://127.0.0.1:{BACKEND_PORT}/health", 90, backend_child):
|
|
598
|
+
die(f"backend did not become healthy on :{BACKEND_PORT}", backend_child)
|
|
599
|
+
|
|
600
|
+
print(f" Starting backend recorder on :{PROXY_PORT} ...", flush=True)
|
|
601
|
+
proxy = start_proxy("be-proxy", be_in, PROXY_PORT,
|
|
602
|
+
f"reverse:http://127.0.0.1:{BACKEND_PORT}", scratch,
|
|
603
|
+
listen_host="0.0.0.0")
|
|
604
|
+
if not wait_for_http(f"http://127.0.0.1:{PROXY_PORT}/health", 30, proxy):
|
|
605
|
+
die(f"recorder did not proxy :{PROXY_PORT} -> :{BACKEND_PORT}", proxy)
|
|
606
|
+
|
|
607
|
+
frontend_env = os.environ.copy()
|
|
608
|
+
frontend_env["PORT"] = str(frontend_port)
|
|
609
|
+
if record_frontend:
|
|
610
|
+
# Bound before the dev server starts, so the port the browser calls is
|
|
611
|
+
# never briefly free during the first compile.
|
|
612
|
+
print(f" Starting frontend recorder on :{FRONTEND_PROXY_PORT} ...", flush=True)
|
|
613
|
+
start_proxy("fe-proxy", fe_in, FRONTEND_PROXY_PORT,
|
|
614
|
+
f"reverse:http://127.0.0.1:{FRONTEND_PORT}", scratch,
|
|
615
|
+
listen_host="0.0.0.0",
|
|
616
|
+
skip=None if args.frontend_all else FRONTEND_NOISE)
|
|
617
|
+
if outbound:
|
|
618
|
+
print(f" Starting frontend outbound recorder on :{FRONTEND_OUTBOUND_PORT} ...",
|
|
619
|
+
flush=True)
|
|
620
|
+
start_proxy("fe-outbound", fe_out, FRONTEND_OUTBOUND_PORT, "regular",
|
|
621
|
+
scratch, show_host=True)
|
|
622
|
+
frontend_env = proxy_env(frontend_env, FRONTEND_OUTBOUND_PORT)
|
|
623
|
+
# Only the frontend's own two ports are excluded, not all of
|
|
624
|
+
# loopback: a server component calling the backend on :7100 is
|
|
625
|
+
# exactly what this leg exists to catch. Entries carry a port
|
|
626
|
+
# because undici matches NO_PROXY on host and port both.
|
|
627
|
+
frontend_env["NO_PROXY"] = ",".join(
|
|
628
|
+
f"{host}:{port}"
|
|
629
|
+
for host in ("127.0.0.1", "localhost")
|
|
630
|
+
for port in (FRONTEND_PROXY_PORT, FRONTEND_PORT)
|
|
631
|
+
)
|
|
632
|
+
|
|
633
|
+
print(f" Starting frontend on :{frontend_port} ...", flush=True)
|
|
634
|
+
frontend_child = Child("frontend", frontend_argv(frontend, frontend_port),
|
|
635
|
+
frontend, frontend_env)
|
|
636
|
+
children.append(frontend_child)
|
|
637
|
+
if not wait_for_port(frontend_port, 120, frontend_child):
|
|
638
|
+
die(f"frontend did not listen on :{frontend_port}", frontend_child)
|
|
639
|
+
|
|
640
|
+
print(" Warming up the first page compile (can take a minute) ...", flush=True)
|
|
641
|
+
# Straight at the dev server first. It binds its port well before it can
|
|
642
|
+
# answer, and a request landing in that gap fails -- through the recorder
|
|
643
|
+
# that failure is a row, so every frontend recording would open with a
|
|
644
|
+
# spurious ERR.
|
|
645
|
+
if not wait_for_http(f"http://127.0.0.1:{frontend_port}/", 180, frontend_child):
|
|
646
|
+
die(f"frontend never served a page on :{frontend_port}", frontend_child)
|
|
647
|
+
# Then once through the recorder, which proves the browser-facing chain
|
|
648
|
+
# before the banner claims it works.
|
|
649
|
+
if record_frontend and not wait_for_http(
|
|
650
|
+
f"http://127.0.0.1:{FRONTEND_PROXY_PORT}/", 30, frontend_child):
|
|
651
|
+
die(f"recorder did not proxy :{FRONTEND_PROXY_PORT} -> :{FRONTEND_PORT}",
|
|
652
|
+
frontend_child)
|
|
653
|
+
|
|
654
|
+
legs = [be_in]
|
|
655
|
+
if outbound:
|
|
656
|
+
legs.append(be_out)
|
|
657
|
+
if record_frontend:
|
|
658
|
+
legs.append(fe_in)
|
|
659
|
+
if outbound:
|
|
660
|
+
legs.append(fe_out)
|
|
661
|
+
|
|
662
|
+
width = max(len(leg.title) for leg in legs)
|
|
663
|
+
leg_lines = "\n".join(
|
|
664
|
+
f" {leg.title:<{width}} : {leg.log}" for leg in legs)
|
|
665
|
+
disabled = []
|
|
666
|
+
if not record_frontend:
|
|
667
|
+
disabled.append("frontend legs off (--no-frontend)")
|
|
668
|
+
if not outbound:
|
|
669
|
+
disabled.append("outbound legs off (--no-outbound)")
|
|
670
|
+
if record_frontend and not args.frontend_all:
|
|
671
|
+
disabled.append("frontend static assets dropped (--frontend-all keeps them)")
|
|
672
|
+
|
|
673
|
+
print(f"""
|
|
674
|
+
Recording active on {len(legs)} leg(s).
|
|
675
|
+
|
|
676
|
+
Open: http://localhost:{FRONTEND_PROXY_PORT}
|
|
677
|
+
|
|
678
|
+
VS Code must be forwarding: {FRONTEND_PROXY_PORT} (app) {PROXY_PORT} (api)
|
|
679
|
+
Both now front a recorder; both were already forwarded before this script
|
|
680
|
+
ran, so there is nothing new to add.
|
|
681
|
+
|
|
682
|
+
Repo : {repo}
|
|
683
|
+
Run dir : {run_dir}
|
|
684
|
+
|
|
685
|
+
{leg_lines}
|
|
686
|
+
|
|
687
|
+
Each table has a .jsonl beside it holding the full bodies.
|
|
688
|
+
{(" " + "; ".join(disabled) + chr(10)) if disabled else ""}
|
|
689
|
+
Files contain live tokens and passwords. Do not share or commit.
|
|
690
|
+
Live view: tail -f {be_in.log}
|
|
691
|
+
Ctrl-C to stop.
|
|
692
|
+
""", flush=True)
|
|
693
|
+
|
|
694
|
+
exit_code = 0
|
|
695
|
+
try:
|
|
696
|
+
while True:
|
|
697
|
+
for child in children:
|
|
698
|
+
if not child.alive():
|
|
699
|
+
print(f"\n {child.name} exited unexpectedly.", file=sys.stderr)
|
|
700
|
+
exit_code = 1
|
|
701
|
+
raise KeyboardInterrupt
|
|
702
|
+
time.sleep(1)
|
|
703
|
+
except KeyboardInterrupt:
|
|
704
|
+
pass
|
|
705
|
+
|
|
706
|
+
cleanup()
|
|
707
|
+
for leg in legs:
|
|
708
|
+
leg.summarise()
|
|
709
|
+
print("\n These files contain live tokens and passwords. "
|
|
710
|
+
"Do not share or commit.\n")
|
|
711
|
+
return exit_code
|
|
712
|
+
|
|
713
|
+
|
|
714
|
+
if __name__ == "__main__":
|
|
715
|
+
try:
|
|
716
|
+
sys.exit(main())
|
|
717
|
+
except KeyboardInterrupt:
|
|
718
|
+
sys.exit(130)
|
|
719
|
+
finally:
|
|
720
|
+
cleanup()
|