@sitar_fiercer4c/skills 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (50) hide show
  1. package/LICENSE +5 -0
  2. package/README.md +75 -0
  3. package/bin/install.js +45 -0
  4. package/package.json +29 -0
  5. package/skills/architecture-walkthrough/SKILL.md +223 -0
  6. package/skills/architecture-walkthrough/references/sections.md +29 -0
  7. package/skills/architecture-walkthrough/scripts/check_structure.py +200 -0
  8. package/skills/autotest-webapp-ui/SKILL.md +58 -0
  9. package/skills/backend-code-review/SKILL.md +386 -0
  10. package/skills/backend-code-review/references/report-format.md +333 -0
  11. package/skills/backend-code-review/scripts/list_routes.py +269 -0
  12. package/skills/backend-code-review/scripts/sweep.py +550 -0
  13. package/skills/backend-code-review/scripts/verify_citations.py +201 -0
  14. package/skills/be-brief/SKILL.md +18 -0
  15. package/skills/clarke-list-excel/SKILL.md +51 -0
  16. package/skills/clarke-list-excel/references/output-schema.md +125 -0
  17. package/skills/clarke-list-excel/scripts/clarke_common.py +251 -0
  18. package/skills/clarke-list-excel/scripts/clarke_extract.py +487 -0
  19. package/skills/clarke-list-excel/scripts/load_clarke.py +322 -0
  20. package/skills/clarke-list-excel/scripts/run_all.py +63 -0
  21. package/skills/datalab-api/SKILL.md +163 -0
  22. package/skills/datalab-api/references/parameters-and-payload.md +121 -0
  23. package/skills/datalab-api/references/table-selection.md +35 -0
  24. package/skills/datalab-api/scripts/datalab_tables.py +365 -0
  25. package/skills/find-test-seam/SKILL.md +41 -0
  26. package/skills/frontend-code-review/SKILL.md +247 -0
  27. package/skills/frontend-code-review-2/SKILL.md +192 -0
  28. package/skills/frontend-code-review-2/scripts/fetch_pr_comments.py +65 -0
  29. package/skills/frontend-code-review-2/scripts/render_report.py +139 -0
  30. package/skills/murtaza-breif/SKILL.md +143 -0
  31. package/skills/murtaza-breif/scripts/save_brief.py +128 -0
  32. package/skills/pdf-to-json/SKILL.md +42 -0
  33. package/skills/pdf-to-json/references/output-schema.md +168 -0
  34. package/skills/pdf-to-json/scripts/extract_figures.py +319 -0
  35. package/skills/pdf-to-json/scripts/load_mongo.py +287 -0
  36. package/skills/pdf-to-json/scripts/pdf_extract.py +1313 -0
  37. package/skills/record-api-traffic/SKILL.md +434 -0
  38. package/skills/record-api-traffic/references/reading-recordings.md +224 -0
  39. package/skills/record-api-traffic/scripts/check-schema.mjs +184 -0
  40. package/skills/record-api-traffic/scripts/dump-quotation.mjs +67 -0
  41. package/skills/record-api-traffic/scripts/dump-source-excel.mjs +75 -0
  42. package/skills/record-api-traffic/scripts/lib/repo.mjs +109 -0
  43. package/skills/record-api-traffic/scripts/preflight.py +528 -0
  44. package/skills/record-api-traffic/scripts/record-api-traffic.py +720 -0
  45. package/skills/refac-wrt-business-goal/SKILL.md +305 -0
  46. package/skills/refac-wrt-business-goal/references/critic.md +170 -0
  47. package/skills/system-resource-triage/SKILL.md +180 -0
  48. package/skills/system-resource-triage/scripts/reap.sh +116 -0
  49. package/skills/system-resource-triage/scripts/triage.sh +111 -0
  50. package/skills/using-git-worktrees/SKILL.md +167 -0
@@ -0,0 +1,720 @@
1
+ #!/usr/bin/env python3
2
+ """Record every HTTP call a Node/Next app makes, on all four legs.
3
+
4
+ Run it with the repo path, open the link it prints, use the app, Ctrl-C.
5
+ Each leg gets an aligned table to scan and a JSONL file with full
6
+ request/response bodies.
7
+
8
+ Both servers are recorded the same way: a mitmdump reverse proxy takes the
9
+ port the browser already calls, and the server moves one port up behind it,
10
+ so nothing on the VS Code side and no config file in the repo has to change.
11
+
12
+ browser -> :4100 recorder -> :4101 frontend frontend-inbound.*
13
+ browser -> :7100 recorder -> :7101 backend backend-inbound.*
14
+
15
+ Each server also gets a mitmdump in regular (forward) mode recording the
16
+ calls it makes itself, from Node rather than the browser:
17
+
18
+ frontend -> :4102 recorder -> the backend, and anything external
19
+ backend -> :7102 recorder -> Microsoft Graph, Datalab
20
+
21
+ --no-frontend drops the frontend legs, --no-outbound drops both forward
22
+ proxies, --frontend-all keeps the static-asset noise the frontend leg
23
+ discards by default.
24
+
25
+ Nothing is written inside the repo. The backend's scratch working directory
26
+ is a temp dir, and recordings land in this skill's own workspace.
27
+
28
+ This file plays two roles. Run directly, it orchestrates. Loaded by
29
+ mitmdump via -s, it is the recording addon.
30
+ """
31
+
32
+ import argparse
33
+ import json
34
+ import os
35
+ import re
36
+ import shutil
37
+ import signal
38
+ import socket
39
+ import subprocess
40
+ import sys
41
+ import tempfile
42
+ import threading
43
+ import time
44
+ import urllib.error
45
+ import urllib.request
46
+ from collections import deque
47
+ from datetime import datetime
48
+ from pathlib import Path
49
+
50
+ SKILL = Path(__file__).resolve().parent.parent
51
+ WORKSPACE = SKILL.parent / "record-api-traffic-workspace" / "recordings"
52
+
53
+ PROXY_PORT = 7100
54
+ BACKEND_PORT = 7101
55
+ OUTBOUND_PORT = 7102
56
+ FRONTEND_PROXY_PORT = 4100
57
+ FRONTEND_PORT = 4101
58
+ FRONTEND_OUTBOUND_PORT = 4102
59
+
60
+ SCRATCH_PREFIX = "record-api-traffic-"
61
+
62
+ # Every child is started with this in its environment, so a leftover process
63
+ # from a crashed run can be told apart from the app's own dev server even
64
+ # though the frontend has to run with its real working directory.
65
+ CHILD_MARKER = "RECORD_API_TRAFFIC_RUN"
66
+
67
+ # Build output and dev-server plumbing on the frontend leg. One page load in
68
+ # `next dev` pulls dozens of chunks, each recorded in full would bury the
69
+ # handful of rows that carry meaning -- navigations, RSC payloads, server
70
+ # actions and any route handler the frontend serves itself.
71
+ FRONTEND_NOISE = r"^/(?:_next/static/|_next/image|_next/webpack-hmr|__nextjs|favicon\.ico$)"
72
+
73
+ MITM_CA = Path.home() / ".mitmproxy" / "mitmproxy-ca-cert.pem"
74
+
75
+ MAX_BODY = 1024 * 1024
76
+
77
+ BINARY_TYPES = (
78
+ "image/",
79
+ "audio/",
80
+ "video/",
81
+ "font/",
82
+ "application/pdf",
83
+ "application/zip",
84
+ "application/octet-stream",
85
+ "application/vnd.openxmlformats",
86
+ "application/vnd.ms-excel",
87
+ )
88
+
89
+ ROW_FMT = "{ts:<12} {id:<4} {ms:>6} {method:<6} {status:<6} {path:<44} {req:>9} {resp:>9}"
90
+ HEADER = ROW_FMT.format(
91
+ ts="TIME", id="ID", ms="MS", method="METHOD", status="STATUS",
92
+ path="PATH", req="REQ", resp="RESP",
93
+ )
94
+
95
+
96
+ def human_size(n):
97
+ if n is None:
98
+ return "-"
99
+ if n == 0:
100
+ return "-"
101
+ if n < 1024:
102
+ return f"{n} B"
103
+ if n < 1024 * 1024:
104
+ return f"{n / 1024:.1f} KB"
105
+ return f"{n / (1024 * 1024):.1f} MB"
106
+
107
+
108
+ def workspace_slug(repo):
109
+ """Last two path segments, so two clones of the same repo stay apart."""
110
+ parts = [p for p in repo.parts if p not in ("/", "")][-2:]
111
+ return re.sub(r"[^a-z0-9]+", "-", "-".join(parts).lower()).strip("-")
112
+
113
+
114
+ # --------------------------------------------------------------------------
115
+ # Recorder — active only inside the mitmdump child process.
116
+ # --------------------------------------------------------------------------
117
+
118
+ class Recorder:
119
+ def __init__(self):
120
+ self.jsonl_path = os.environ["MITM_RECORD_JSONL"]
121
+ self.log_path = os.environ["MITM_RECORD_LOG"]
122
+ self.show_host = bool(os.environ.get("MITM_RECORD_SHOW_HOST"))
123
+ skip = os.environ.get("MITM_RECORD_SKIP")
124
+ self.skip = re.compile(skip) if skip else None
125
+ self.jsonl = None
126
+ self.log = None
127
+ self.counter = 0
128
+
129
+ def running(self):
130
+ self.jsonl = open(self.jsonl_path, "a", encoding="utf-8")
131
+ self.log = open(self.log_path, "a", encoding="utf-8")
132
+ self.log.write(HEADER + "\n")
133
+ self.log.flush()
134
+
135
+ def done(self):
136
+ for handle in (self.jsonl, self.log):
137
+ if handle:
138
+ handle.close()
139
+
140
+ def response(self, flow):
141
+ self._record(flow, None)
142
+
143
+ def error(self, flow):
144
+ if flow.response is None:
145
+ self._record(flow, str(flow.error) if flow.error else "connection error")
146
+
147
+ def _record(self, flow, error):
148
+ req = flow.request
149
+ # Before the counter moves, so a dropped call leaves no gap and the
150
+ # row ID stays equal to the line number in the JSONL.
151
+ if self.skip and self.skip.search(req.path.split("?")[0]):
152
+ return
153
+
154
+ self.counter += 1
155
+ rec_id = f"{self.counter:04d}"
156
+
157
+ resp = flow.response
158
+
159
+ started = req.timestamp_start
160
+ ended = (resp.timestamp_end if resp and resp.timestamp_end else time.time())
161
+ duration_ms = int((ended - started) * 1000)
162
+
163
+ req_body, req_size = self._body(req)
164
+ if resp is not None:
165
+ resp_body, resp_size = self._body(resp)
166
+ else:
167
+ resp_body, resp_size = None, 0
168
+
169
+ record = {
170
+ "id": rec_id,
171
+ "ts": datetime.fromtimestamp(ended).astimezone().isoformat(),
172
+ "started_at": datetime.fromtimestamp(started).astimezone().isoformat(),
173
+ "duration_ms": duration_ms,
174
+ "method": req.method,
175
+ "path": req.path.split("?")[0],
176
+ "host": req.pretty_host,
177
+ "url": req.pretty_url,
178
+ "query": dict(req.query),
179
+ "client_ip": flow.client_conn.peername[0] if flow.client_conn.peername else None,
180
+ "request": {
181
+ "headers": dict(req.headers),
182
+ "body": req_body["value"],
183
+ "body_encoding": req_body["encoding"],
184
+ "body_size": req_size,
185
+ "truncated": req_body["truncated"],
186
+ },
187
+ }
188
+ if resp is not None:
189
+ record["response"] = {
190
+ "status": resp.status_code,
191
+ "headers": dict(resp.headers),
192
+ "body": resp_body["value"],
193
+ "body_encoding": resp_body["encoding"],
194
+ "body_size": resp_size,
195
+ "truncated": resp_body["truncated"],
196
+ }
197
+ if error:
198
+ record["error"] = error
199
+
200
+ self.jsonl.write(json.dumps(record, ensure_ascii=False, default=str) + "\n")
201
+ self.jsonl.flush()
202
+
203
+ path = req.path
204
+ if self.show_host:
205
+ # With the port: a forward-proxy leg spans hosts, and on the
206
+ # frontend's it also spans loopback ports, where :7100 (the
207
+ # backend) and :4101 (the frontend calling itself) are otherwise
208
+ # the same "127.0.0.1".
209
+ host = req.pretty_host
210
+ if req.port != (443 if req.scheme == "https" else 80):
211
+ host = f"{host}:{req.port}"
212
+ path = f"{host}{req.path}"
213
+ if len(path) > 44:
214
+ path = path[:43] + "…"
215
+ self.log.write(ROW_FMT.format(
216
+ ts=datetime.fromtimestamp(ended).strftime("%H:%M:%S.%f")[:-3],
217
+ id=rec_id,
218
+ ms=duration_ms,
219
+ method=req.method,
220
+ status=str(resp.status_code) if resp is not None else "ERR",
221
+ path=path,
222
+ req=human_size(req_size),
223
+ resp=human_size(resp_size) if resp is not None else "-",
224
+ ) + "\n")
225
+ self.log.flush()
226
+
227
+ def _body(self, message):
228
+ """Return ({value, encoding, truncated}, size) for a request or response."""
229
+ empty = {"value": None, "encoding": None, "truncated": False}
230
+ try:
231
+ content = message.get_content(strict=False)
232
+ except Exception:
233
+ content = message.raw_content
234
+ if not content:
235
+ return empty, 0
236
+
237
+ size = len(content)
238
+ ctype = message.headers.get("content-type", "").lower()
239
+
240
+ if any(ctype.startswith(t) for t in BINARY_TYPES):
241
+ return {
242
+ "value": {"binary": True, "content_type": ctype or None, "size": size},
243
+ "encoding": "binary",
244
+ "truncated": False,
245
+ }, size
246
+
247
+ truncated = size > MAX_BODY
248
+ text = content[:MAX_BODY].decode("utf-8", errors="replace")
249
+ if truncated:
250
+ return {"value": text, "encoding": "text", "truncated": True}, size
251
+
252
+ try:
253
+ return {"value": json.loads(text), "encoding": "json", "truncated": False}, size
254
+ except (ValueError, TypeError):
255
+ return {"value": text, "encoding": "text", "truncated": False}, size
256
+
257
+
258
+ if os.environ.get("MITM_RECORD_JSONL"):
259
+ addons = [Recorder()]
260
+
261
+
262
+ # --------------------------------------------------------------------------
263
+ # Launcher
264
+ # --------------------------------------------------------------------------
265
+
266
+ class Child:
267
+ def __init__(self, name, argv, cwd, env=None):
268
+ self.name = name
269
+ self.tail = deque(maxlen=40)
270
+ env = dict(env or os.environ)
271
+ env[CHILD_MARKER] = str(os.getpid())
272
+ self.proc = subprocess.Popen(
273
+ argv,
274
+ cwd=str(cwd),
275
+ env=env,
276
+ stdout=subprocess.PIPE,
277
+ stderr=subprocess.STDOUT,
278
+ text=True,
279
+ bufsize=1,
280
+ start_new_session=True,
281
+ )
282
+ threading.Thread(target=self._pump, daemon=True).start()
283
+
284
+ def _pump(self):
285
+ for line in self.proc.stdout:
286
+ line = line.rstrip()
287
+ self.tail.append(line)
288
+ print(f"[{self.name}] {line}", flush=True)
289
+
290
+ def alive(self):
291
+ return self.proc.poll() is None
292
+
293
+ def stop(self):
294
+ if self.proc.poll() is not None:
295
+ return
296
+ try:
297
+ pgid = os.getpgid(self.proc.pid)
298
+ except ProcessLookupError:
299
+ return
300
+ for sig in (signal.SIGTERM, signal.SIGKILL):
301
+ try:
302
+ os.killpg(pgid, sig)
303
+ except ProcessLookupError:
304
+ return
305
+ deadline = time.time() + 10
306
+ while time.time() < deadline:
307
+ if self.proc.poll() is not None:
308
+ return
309
+ time.sleep(0.2)
310
+
311
+
312
+ children = []
313
+ scratch = None
314
+ cleaned = False
315
+
316
+
317
+ def port_in_use(port):
318
+ with socket.socket() as s:
319
+ s.settimeout(0.5)
320
+ return s.connect_ex(("127.0.0.1", port)) == 0
321
+
322
+
323
+ def port_holder(port):
324
+ try:
325
+ out = subprocess.run(
326
+ ["ss", "-ltnp", f"sport = :{port}"],
327
+ capture_output=True, text=True, timeout=5,
328
+ ).stdout
329
+ match = re.search(r'users:\(\("([^"]+)",pid=(\d+)', out)
330
+ if match:
331
+ return f"{match.group(1)} (pid {match.group(2)})"
332
+ except Exception:
333
+ pass
334
+ return "unknown process"
335
+
336
+
337
+ def wait_for_http(url, timeout, child=None):
338
+ deadline = time.time() + timeout
339
+ while time.time() < deadline:
340
+ if child and not child.alive():
341
+ return False
342
+ try:
343
+ with urllib.request.urlopen(url, timeout=3) as r:
344
+ if r.status < 500:
345
+ return True
346
+ except urllib.error.HTTPError as e:
347
+ if e.code < 500:
348
+ return True
349
+ except Exception:
350
+ pass
351
+ time.sleep(0.5)
352
+ return False
353
+
354
+
355
+ def wait_for_port(port, timeout, child=None):
356
+ deadline = time.time() + timeout
357
+ while time.time() < deadline:
358
+ if child and not child.alive():
359
+ return False
360
+ if port_in_use(port):
361
+ return True
362
+ time.sleep(0.5)
363
+ return False
364
+
365
+
366
+ def die(message, child=None):
367
+ print(f"\n ERROR: {message}\n", file=sys.stderr)
368
+ if child and child.tail:
369
+ print(f" last output from {child.name}:", file=sys.stderr)
370
+ for line in child.tail:
371
+ print(f" {line}", file=sys.stderr)
372
+ print(file=sys.stderr)
373
+ cleanup()
374
+ sys.exit(1)
375
+
376
+
377
+ def cleanup(*_):
378
+ global cleaned
379
+ if cleaned:
380
+ return
381
+ cleaned = True
382
+ print("\n Shutting down...", flush=True)
383
+ for child in reversed(children):
384
+ child.stop()
385
+ if scratch:
386
+ shutil.rmtree(scratch, ignore_errors=True)
387
+
388
+
389
+ def preflight(repo, outbound, record_frontend):
390
+ backend, frontend = repo / "Backend", repo / "Frontend"
391
+ if not shutil.which("mitmdump"):
392
+ die("mitmdump not found on PATH. Install with: pipx install mitmproxy")
393
+ for path in (backend / ".env", frontend / ".env.local",
394
+ backend / "node_modules", frontend / "node_modules"):
395
+ if not path.exists():
396
+ die(f"missing {path}")
397
+ ports = [(PROXY_PORT, "backend proxy"), (BACKEND_PORT, "backend"),
398
+ (FRONTEND_PROXY_PORT, "frontend proxy" if record_frontend else "frontend")]
399
+ if outbound:
400
+ ports.append((OUTBOUND_PORT, "backend outbound proxy"))
401
+ if record_frontend:
402
+ ports.append((FRONTEND_PORT, "frontend"))
403
+ if outbound:
404
+ ports.append((FRONTEND_OUTBOUND_PORT, "frontend outbound proxy"))
405
+ for port, what in ports:
406
+ if port_in_use(port):
407
+ die(f"port {port} ({what}) is already in use by {port_holder(port)}. "
408
+ "Stop it and re-run.")
409
+ if outbound and not MITM_CA.exists():
410
+ die(f"missing {MITM_CA}. mitmproxy writes it on its first run; start "
411
+ "mitmdump once, or pass --no-outbound.")
412
+
413
+
414
+ def dev_script(frontend):
415
+ try:
416
+ pkg = json.loads((frontend / "package.json").read_text())
417
+ except (OSError, ValueError):
418
+ return ""
419
+ return (pkg.get("scripts") or {}).get("dev", "")
420
+
421
+
422
+ def frontend_argv(frontend, port):
423
+ """How to start the dev server on `port`.
424
+
425
+ A Next dev script normally pins the port itself (`next dev -H 0.0.0.0
426
+ -p 4100`), and a pinned CLI flag beats the PORT env var -- commander
427
+ only consults `.env("PORT")` when the flag is absent. Appending a second
428
+ `-p` moves it anyway: both argument parsers Next has shipped, `arg` and
429
+ `commander`, take the last occurrence of a non-variadic flag.
430
+
431
+ PORT is still exported alongside, for a dev script that leaves the port
432
+ unpinned and for anything downstream that reads it.
433
+ """
434
+ script = dev_script(frontend)
435
+ if "next" in script:
436
+ return ["npm", "run", "dev", "--", "-p", str(port)]
437
+ return ["npm", "run", "dev"]
438
+
439
+
440
+ def make_scratch(backend):
441
+ """A temp working directory holding a copy of .env with PORT overridden.
442
+
443
+ The backend's app.js calls dotenv.config({ override: true }), so a shell
444
+ env var loses to .env. dotenv resolves its file from the working
445
+ directory, so moving the port means moving the cwd -- and it lives outside
446
+ the repo so that recording never writes there.
447
+ """
448
+ path = Path(tempfile.mkdtemp(prefix=SCRATCH_PREFIX))
449
+ path.chmod(0o700)
450
+ lines = (backend / ".env").read_text().splitlines()
451
+ replaced = False
452
+ for i, line in enumerate(lines):
453
+ if re.match(r"^\s*PORT\s*=", line):
454
+ lines[i] = f"PORT={BACKEND_PORT}"
455
+ replaced = True
456
+ if not replaced:
457
+ lines.append(f"PORT={BACKEND_PORT}")
458
+ target = path / ".env"
459
+ target.write_text("\n".join(lines) + "\n")
460
+ target.chmod(0o600)
461
+ return path
462
+
463
+
464
+ class Leg:
465
+ """One recorded direction: a label and the pair of files it writes."""
466
+
467
+ def __init__(self, run_dir, stem, title):
468
+ self.stem = stem
469
+ self.title = title
470
+ self.jsonl = run_dir / f"{stem}.jsonl"
471
+ self.log = run_dir / f"{stem}.log"
472
+
473
+ def summarise(self):
474
+ total = failed = 0
475
+ try:
476
+ with open(self.jsonl, encoding="utf-8") as f:
477
+ for line in f:
478
+ if not line.strip():
479
+ continue
480
+ total += 1
481
+ try:
482
+ rec = json.loads(line)
483
+ except ValueError:
484
+ continue
485
+ status = rec.get("response", {}).get("status")
486
+ if status is None or status >= 400:
487
+ failed += 1
488
+ except FileNotFoundError:
489
+ pass
490
+ print(f"\n {self.title}: {total} calls ({failed} failed).")
491
+ print(f" Table : {self.log}")
492
+ print(f" Details : {self.jsonl}")
493
+
494
+
495
+ def start_proxy(name, leg, listen_port, mode, cwd,
496
+ listen_host="127.0.0.1", show_host=False, skip=None):
497
+ """Start one recording mitmdump and register it for teardown."""
498
+ env = os.environ.copy()
499
+ env["MITM_RECORD_JSONL"] = str(leg.jsonl)
500
+ env["MITM_RECORD_LOG"] = str(leg.log)
501
+ if show_host:
502
+ env["MITM_RECORD_SHOW_HOST"] = "1"
503
+ if skip:
504
+ env["MITM_RECORD_SKIP"] = skip
505
+ argv = [
506
+ "mitmdump",
507
+ "--mode", mode,
508
+ "--listen-host", listen_host,
509
+ "--listen-port", str(listen_port),
510
+ "-s", str(Path(__file__).resolve()),
511
+ "-q",
512
+ ]
513
+ if mode.startswith("reverse"):
514
+ argv += ["--set", "keep_host_header=true"]
515
+ child = Child(name, argv, cwd, env)
516
+ children.append(child)
517
+ if not wait_for_port(listen_port, 30, child):
518
+ die(f"{name} recorder did not listen on :{listen_port}", child)
519
+ return child
520
+
521
+
522
+ def proxy_env(env, port):
523
+ """Point a Node process's outbound HTTP at one of our forward proxies.
524
+
525
+ Node's global fetch honours the proxy variables only with
526
+ NODE_USE_ENV_PROXY (Node 24+); NODE_EXTRA_CA_CERTS is what makes it
527
+ trust mitmproxy's certificate for the HTTPS calls it intercepts.
528
+ """
529
+ env["NODE_USE_ENV_PROXY"] = "1"
530
+ env["HTTP_PROXY"] = f"http://127.0.0.1:{port}"
531
+ env["HTTPS_PROXY"] = f"http://127.0.0.1:{port}"
532
+ env["NODE_EXTRA_CA_CERTS"] = str(MITM_CA)
533
+ return env
534
+
535
+
536
+ def main():
537
+ global scratch
538
+
539
+ parser = argparse.ArgumentParser(description=__doc__.split("\n")[0])
540
+ parser.add_argument("repo", help="path to the app repo")
541
+ parser.add_argument("--no-outbound", action="store_true",
542
+ help="skip both forward proxies (:7102 Graph/Datalab, "
543
+ ":4102 the frontend's own server-side calls)")
544
+ parser.add_argument("--no-frontend", action="store_true",
545
+ help="leave the dev server on :4100 unrecorded, as it "
546
+ "was before the frontend legs existed")
547
+ parser.add_argument("--frontend-all", action="store_true",
548
+ help="keep /_next/static and other dev-server noise in "
549
+ "the frontend recording instead of dropping it")
550
+ args = parser.parse_args()
551
+
552
+ repo = Path(args.repo).expanduser().resolve()
553
+ if not repo.exists():
554
+ print(f"repo not found: {repo}", file=sys.stderr)
555
+ return 2
556
+ backend, frontend = repo / "Backend", repo / "Frontend"
557
+
558
+ outbound = not args.no_outbound
559
+ record_frontend = not args.no_frontend
560
+ # Unrecorded, the dev server stays on the port the browser already calls.
561
+ frontend_port = FRONTEND_PORT if record_frontend else FRONTEND_PROXY_PORT
562
+ preflight(repo, outbound, record_frontend)
563
+
564
+ stamp = datetime.now().strftime("%Y%m%d-%H%M%S")
565
+ run_dir = WORKSPACE / workspace_slug(repo) / f"run-{stamp}"
566
+ run_dir.mkdir(parents=True, exist_ok=True)
567
+ (run_dir / "run.json").write_text(json.dumps({
568
+ "repo": str(repo), "started_at": datetime.now().astimezone().isoformat(),
569
+ "outbound": outbound,
570
+ "frontend": record_frontend,
571
+ "frontend_all": bool(args.frontend_all),
572
+ }, indent=2) + "\n")
573
+
574
+ be_in = Leg(run_dir, "backend-inbound", "into backend (browser, frontend)")
575
+ be_out = Leg(run_dir, "backend-outbound", "out of backend (Graph, Datalab)")
576
+ fe_in = Leg(run_dir, "frontend-inbound", "into frontend (browser)")
577
+ fe_out = Leg(run_dir, "frontend-outbound", "out of frontend (backend, external)")
578
+
579
+ def on_sigterm(*_):
580
+ raise KeyboardInterrupt
581
+
582
+ signal.signal(signal.SIGTERM, on_sigterm)
583
+
584
+ scratch = make_scratch(backend)
585
+
586
+ backend_env = os.environ.copy()
587
+ if outbound:
588
+ print(f" Starting backend outbound recorder on :{OUTBOUND_PORT} ...", flush=True)
589
+ start_proxy("be-outbound", be_out, OUTBOUND_PORT, "regular", scratch, show_host=True)
590
+ backend_env = proxy_env(backend_env, OUTBOUND_PORT)
591
+ backend_env["NO_PROXY"] = "127.0.0.1,localhost"
592
+
593
+ print(f" Starting backend on :{BACKEND_PORT} ...", flush=True)
594
+ backend_child = Child("backend", ["node", str(backend / "src" / "server.js")],
595
+ scratch, backend_env)
596
+ children.append(backend_child)
597
+ if not wait_for_http(f"http://127.0.0.1:{BACKEND_PORT}/health", 90, backend_child):
598
+ die(f"backend did not become healthy on :{BACKEND_PORT}", backend_child)
599
+
600
+ print(f" Starting backend recorder on :{PROXY_PORT} ...", flush=True)
601
+ proxy = start_proxy("be-proxy", be_in, PROXY_PORT,
602
+ f"reverse:http://127.0.0.1:{BACKEND_PORT}", scratch,
603
+ listen_host="0.0.0.0")
604
+ if not wait_for_http(f"http://127.0.0.1:{PROXY_PORT}/health", 30, proxy):
605
+ die(f"recorder did not proxy :{PROXY_PORT} -> :{BACKEND_PORT}", proxy)
606
+
607
+ frontend_env = os.environ.copy()
608
+ frontend_env["PORT"] = str(frontend_port)
609
+ if record_frontend:
610
+ # Bound before the dev server starts, so the port the browser calls is
611
+ # never briefly free during the first compile.
612
+ print(f" Starting frontend recorder on :{FRONTEND_PROXY_PORT} ...", flush=True)
613
+ start_proxy("fe-proxy", fe_in, FRONTEND_PROXY_PORT,
614
+ f"reverse:http://127.0.0.1:{FRONTEND_PORT}", scratch,
615
+ listen_host="0.0.0.0",
616
+ skip=None if args.frontend_all else FRONTEND_NOISE)
617
+ if outbound:
618
+ print(f" Starting frontend outbound recorder on :{FRONTEND_OUTBOUND_PORT} ...",
619
+ flush=True)
620
+ start_proxy("fe-outbound", fe_out, FRONTEND_OUTBOUND_PORT, "regular",
621
+ scratch, show_host=True)
622
+ frontend_env = proxy_env(frontend_env, FRONTEND_OUTBOUND_PORT)
623
+ # Only the frontend's own two ports are excluded, not all of
624
+ # loopback: a server component calling the backend on :7100 is
625
+ # exactly what this leg exists to catch. Entries carry a port
626
+ # because undici matches NO_PROXY on host and port both.
627
+ frontend_env["NO_PROXY"] = ",".join(
628
+ f"{host}:{port}"
629
+ for host in ("127.0.0.1", "localhost")
630
+ for port in (FRONTEND_PROXY_PORT, FRONTEND_PORT)
631
+ )
632
+
633
+ print(f" Starting frontend on :{frontend_port} ...", flush=True)
634
+ frontend_child = Child("frontend", frontend_argv(frontend, frontend_port),
635
+ frontend, frontend_env)
636
+ children.append(frontend_child)
637
+ if not wait_for_port(frontend_port, 120, frontend_child):
638
+ die(f"frontend did not listen on :{frontend_port}", frontend_child)
639
+
640
+ print(" Warming up the first page compile (can take a minute) ...", flush=True)
641
+ # Straight at the dev server first. It binds its port well before it can
642
+ # answer, and a request landing in that gap fails -- through the recorder
643
+ # that failure is a row, so every frontend recording would open with a
644
+ # spurious ERR.
645
+ if not wait_for_http(f"http://127.0.0.1:{frontend_port}/", 180, frontend_child):
646
+ die(f"frontend never served a page on :{frontend_port}", frontend_child)
647
+ # Then once through the recorder, which proves the browser-facing chain
648
+ # before the banner claims it works.
649
+ if record_frontend and not wait_for_http(
650
+ f"http://127.0.0.1:{FRONTEND_PROXY_PORT}/", 30, frontend_child):
651
+ die(f"recorder did not proxy :{FRONTEND_PROXY_PORT} -> :{FRONTEND_PORT}",
652
+ frontend_child)
653
+
654
+ legs = [be_in]
655
+ if outbound:
656
+ legs.append(be_out)
657
+ if record_frontend:
658
+ legs.append(fe_in)
659
+ if outbound:
660
+ legs.append(fe_out)
661
+
662
+ width = max(len(leg.title) for leg in legs)
663
+ leg_lines = "\n".join(
664
+ f" {leg.title:<{width}} : {leg.log}" for leg in legs)
665
+ disabled = []
666
+ if not record_frontend:
667
+ disabled.append("frontend legs off (--no-frontend)")
668
+ if not outbound:
669
+ disabled.append("outbound legs off (--no-outbound)")
670
+ if record_frontend and not args.frontend_all:
671
+ disabled.append("frontend static assets dropped (--frontend-all keeps them)")
672
+
673
+ print(f"""
674
+ Recording active on {len(legs)} leg(s).
675
+
676
+ Open: http://localhost:{FRONTEND_PROXY_PORT}
677
+
678
+ VS Code must be forwarding: {FRONTEND_PROXY_PORT} (app) {PROXY_PORT} (api)
679
+ Both now front a recorder; both were already forwarded before this script
680
+ ran, so there is nothing new to add.
681
+
682
+ Repo : {repo}
683
+ Run dir : {run_dir}
684
+
685
+ {leg_lines}
686
+
687
+ Each table has a .jsonl beside it holding the full bodies.
688
+ {(" " + "; ".join(disabled) + chr(10)) if disabled else ""}
689
+ Files contain live tokens and passwords. Do not share or commit.
690
+ Live view: tail -f {be_in.log}
691
+ Ctrl-C to stop.
692
+ """, flush=True)
693
+
694
+ exit_code = 0
695
+ try:
696
+ while True:
697
+ for child in children:
698
+ if not child.alive():
699
+ print(f"\n {child.name} exited unexpectedly.", file=sys.stderr)
700
+ exit_code = 1
701
+ raise KeyboardInterrupt
702
+ time.sleep(1)
703
+ except KeyboardInterrupt:
704
+ pass
705
+
706
+ cleanup()
707
+ for leg in legs:
708
+ leg.summarise()
709
+ print("\n These files contain live tokens and passwords. "
710
+ "Do not share or commit.\n")
711
+ return exit_code
712
+
713
+
714
+ if __name__ == "__main__":
715
+ try:
716
+ sys.exit(main())
717
+ except KeyboardInterrupt:
718
+ sys.exit(130)
719
+ finally:
720
+ cleanup()