@capagents/forgeqe 2.2.1 → 2.2.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/HOW_TO_USE.md +235 -51
- package/README.md +2 -2
- package/THIRD_PARTY_NOTICES.md +2 -1
- package/dist/forgeqe.js +1145 -951
- package/dist/template.html +36 -36
- package/docs/ARCHITECTURE.md +6 -2
- package/docs/CONTAINERS.md +80 -4
- package/docs/user-guide.html +1509 -143
- package/forgeqe.example.yaml +3 -1
- package/llms.example.json +2 -4
- package/package.json +5 -2
- package/prompts/README.md +1 -0
- package/prompts/brain.md +3 -1
- package/prompts/test-plan.md +61 -0
- package/py/forgeqe_engine/browser.py +106 -1
- package/py/forgeqe_engine/cli.py +57 -0
- package/py/forgeqe_engine/copilot_gateway.py +369 -0
- package/py/forgeqe_engine/dynamic.py +149 -0
- package/py/forgeqe_engine/recordsession.py +908 -0
- package/py/forgeqe_engine/service.py +29 -0
- package/py/forgeqe_engine/webrecord.py +589 -0
- package/py/forgeqe_engine/webrecord_page.py +530 -0
- package/reporters/playwright.mjs +83 -0
|
@@ -0,0 +1,369 @@
|
|
|
1
|
+
"""
|
|
2
|
+
GitHub Copilot as an OpenAI-style chat endpoint on this machine, through GitHub's Copilot SDK.
|
|
3
|
+
|
|
4
|
+
Copilot has no model API for other tools: the SDK runs Copilot "sessions" on the user's seat. The browser engine and
|
|
5
|
+
the desktop agent speak OpenAI chat completions, so this gateway turns each request into one SDK session with no
|
|
6
|
+
tools of its own (the system prompt replaced, screenshots attached, a JSON schema passed on) and answers in the
|
|
7
|
+
OpenAI shape.
|
|
8
|
+
|
|
9
|
+
python -m forgeqe_engine copilot-gateway
|
|
10
|
+
env COPILOT_GITHUB_TOKEN the GitHub token (fine-grained with Copilot Requests, or the GitHub CLI's)
|
|
11
|
+
FORGEQE_COPILOT_GATEWAY_KEY the one-time key every request must carry as "Bearer <key>"
|
|
12
|
+
stdout one line: {"port": N, "login": "<account>"} once it listens, or {"error": "..."}
|
|
13
|
+
stdin end of input stops it, so it never outlives ForgeQE
|
|
14
|
+
|
|
15
|
+
It listens on 127.0.0.1 only, compares the key in constant time and refuses requests a web page sends.
|
|
16
|
+
"""
|
|
17
|
+
from __future__ import annotations
|
|
18
|
+
|
|
19
|
+
import asyncio
|
|
20
|
+
import hmac
|
|
21
|
+
import json
|
|
22
|
+
import os
|
|
23
|
+
import sys
|
|
24
|
+
import tempfile
|
|
25
|
+
import time
|
|
26
|
+
import uuid
|
|
27
|
+
from typing import Any, Dict, List, Optional, TextIO, Tuple
|
|
28
|
+
|
|
29
|
+
TOKEN_ENV = "COPILOT_GITHUB_TOKEN"
|
|
30
|
+
KEY_ENV = "FORGEQE_COPILOT_GATEWAY_KEY"
|
|
31
|
+
|
|
32
|
+
MAX_BODY = 32 * 1024 * 1024
|
|
33
|
+
# A screenshot step can take minutes on a slow model; past this the request fails rather than hangs.
|
|
34
|
+
REQUEST_TIMEOUT_S = 300.0
|
|
35
|
+
LOCAL_ORIGINS = ("http://127.0.0.1", "http://localhost", "https://127.0.0.1", "https://localhost")
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
class GatewayError(Exception):
|
|
39
|
+
def __init__(self, status: int, message: str):
|
|
40
|
+
super().__init__(message)
|
|
41
|
+
self.status = status
|
|
42
|
+
self.message = message
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
# ---------------------------------------------------------------------------------------------------------------------
|
|
46
|
+
# OpenAI chat completions → one Copilot session
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def _parts(content: Any) -> List[Dict[str, Any]]:
|
|
50
|
+
if isinstance(content, str):
|
|
51
|
+
return [{"type": "text", "text": content}]
|
|
52
|
+
if isinstance(content, list):
|
|
53
|
+
return [p for p in content if isinstance(p, dict)]
|
|
54
|
+
return []
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def _data_url(url: str) -> Optional[Tuple[str, str]]:
|
|
58
|
+
"""data:image/png;base64,AAAA → ("image/png", "AAAA"); None for anything else (remote images are not fetched)."""
|
|
59
|
+
if not url.startswith("data:") or ";base64," not in url:
|
|
60
|
+
return None
|
|
61
|
+
head, data = url[5:].split(";base64,", 1)
|
|
62
|
+
mime = head.split(";", 1)[0].strip().lower()
|
|
63
|
+
if not mime.startswith("image/") or not data:
|
|
64
|
+
return None
|
|
65
|
+
return mime, data
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def translate(body: Dict[str, Any]) -> Dict[str, Any]:
|
|
69
|
+
"""The session a chat completions request becomes: {model, system, prompt, attachments, schema, json_only}."""
|
|
70
|
+
if not isinstance(body, dict):
|
|
71
|
+
raise GatewayError(400, "The request body must be a JSON object.")
|
|
72
|
+
model = str(body.get("model") or "").strip()
|
|
73
|
+
if not model:
|
|
74
|
+
raise GatewayError(400, "The request names no model.")
|
|
75
|
+
if body.get("stream"):
|
|
76
|
+
raise GatewayError(400, "Streaming is not supported by ForgeQE's Copilot gateway.")
|
|
77
|
+
if body.get("tools") or body.get("functions"):
|
|
78
|
+
raise GatewayError(400, "Tool calls are not supported by ForgeQE's Copilot gateway yet.")
|
|
79
|
+
messages = body.get("messages")
|
|
80
|
+
if not isinstance(messages, list) or not messages:
|
|
81
|
+
raise GatewayError(400, "The request has no messages.")
|
|
82
|
+
|
|
83
|
+
system: List[str] = []
|
|
84
|
+
turns: List[Tuple[str, List[Dict[str, Any]]]] = []
|
|
85
|
+
for m in messages:
|
|
86
|
+
if not isinstance(m, dict):
|
|
87
|
+
continue
|
|
88
|
+
role = str(m.get("role") or "user")
|
|
89
|
+
parts = _parts(m.get("content"))
|
|
90
|
+
if role in ("system", "developer"):
|
|
91
|
+
system.extend(str(p.get("text") or "") for p in parts if p.get("type") == "text")
|
|
92
|
+
else:
|
|
93
|
+
turns.append((role, parts))
|
|
94
|
+
if not turns:
|
|
95
|
+
raise GatewayError(400, "The request has no user message.")
|
|
96
|
+
|
|
97
|
+
# Only the latest message's images go to the model; the browser agent sends the current screenshot there.
|
|
98
|
+
last_with_images = max((i for i, (_, parts) in enumerate(turns) if any(p.get("type") == "image_url" for p in parts)), default=-1)
|
|
99
|
+
attachments: List[Dict[str, Any]] = []
|
|
100
|
+
lines: List[str] = []
|
|
101
|
+
for i, (role, parts) in enumerate(turns):
|
|
102
|
+
texts: List[str] = []
|
|
103
|
+
for p in parts:
|
|
104
|
+
if p.get("type") == "text":
|
|
105
|
+
texts.append(str(p.get("text") or ""))
|
|
106
|
+
elif p.get("type") == "image_url":
|
|
107
|
+
url = p.get("image_url", {}).get("url") if isinstance(p.get("image_url"), dict) else p.get("image_url")
|
|
108
|
+
img = _data_url(str(url or ""))
|
|
109
|
+
if img and i == last_with_images:
|
|
110
|
+
attachments.append({"type": "blob", "mimeType": img[0], "data": img[1], "displayName": f"screenshot-{len(attachments) + 1}"})
|
|
111
|
+
texts.append(f"[image attached: screenshot-{len(attachments)}]")
|
|
112
|
+
else:
|
|
113
|
+
texts.append("[earlier image omitted]")
|
|
114
|
+
text = "\n".join(t for t in texts if t)
|
|
115
|
+
lines.append(text if len(turns) == 1 else f"<{role}>\n{text}\n</{role}>")
|
|
116
|
+
prompt = "\n\n".join(lines)
|
|
117
|
+
if len(turns) > 1:
|
|
118
|
+
prompt = "The conversation so far, oldest first. Reply as the assistant to the last message.\n\n" + prompt
|
|
119
|
+
|
|
120
|
+
schema = None
|
|
121
|
+
json_only = False
|
|
122
|
+
fmt = body.get("response_format")
|
|
123
|
+
if isinstance(fmt, dict):
|
|
124
|
+
if fmt.get("type") == "json_schema" and isinstance(fmt.get("json_schema"), dict):
|
|
125
|
+
schema = fmt["json_schema"].get("schema")
|
|
126
|
+
if not isinstance(schema, dict):
|
|
127
|
+
raise GatewayError(400, "response_format.json_schema has no schema.")
|
|
128
|
+
elif fmt.get("type") == "json_object":
|
|
129
|
+
json_only = True
|
|
130
|
+
prompt += "\n\nReply with a single JSON object and nothing else."
|
|
131
|
+
return {"model": model, "system": "\n\n".join(s for s in system if s), "prompt": prompt, "attachments": attachments, "schema": schema, "json_only": json_only}
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
def completion(model: str, content: str, usage: Dict[str, int]) -> Dict[str, Any]:
|
|
135
|
+
"""An OpenAI chat.completion object."""
|
|
136
|
+
prompt_tokens = int(usage.get("input", 0))
|
|
137
|
+
completion_tokens = int(usage.get("output", 0))
|
|
138
|
+
out: Dict[str, Any] = {
|
|
139
|
+
"id": f"chatcmpl-{uuid.uuid4().hex}",
|
|
140
|
+
"object": "chat.completion",
|
|
141
|
+
"created": int(time.time()),
|
|
142
|
+
"model": model,
|
|
143
|
+
"choices": [{"index": 0, "message": {"role": "assistant", "content": content}, "finish_reason": "stop"}],
|
|
144
|
+
"usage": {"prompt_tokens": prompt_tokens, "completion_tokens": completion_tokens, "total_tokens": prompt_tokens + completion_tokens},
|
|
145
|
+
}
|
|
146
|
+
if usage.get("cached"):
|
|
147
|
+
out["usage"]["prompt_tokens_details"] = {"cached_tokens": int(usage["cached"])}
|
|
148
|
+
return out
|
|
149
|
+
|
|
150
|
+
|
|
151
|
+
def strip_fences(text: str) -> str:
|
|
152
|
+
"""```json … ``` → the JSON inside, for models that wrap structured replies in a code fence."""
|
|
153
|
+
t = text.strip()
|
|
154
|
+
if t.startswith("```") and t.endswith("```"):
|
|
155
|
+
t = t[3:-3]
|
|
156
|
+
first, _, rest = t.partition("\n")
|
|
157
|
+
return (rest if first.strip().isalpha() or not first.strip() else t).strip()
|
|
158
|
+
return text
|
|
159
|
+
|
|
160
|
+
|
|
161
|
+
# ---------------------------------------------------------------------------------------------------------------------
|
|
162
|
+
# Requests from this machine only
|
|
163
|
+
|
|
164
|
+
|
|
165
|
+
def authorized(headers: Dict[str, str], key: str) -> bool:
|
|
166
|
+
expected = f"Bearer {key}".encode()
|
|
167
|
+
got = headers.get("authorization", "").encode()
|
|
168
|
+
return bool(key) and hmac.compare_digest(got, expected)
|
|
169
|
+
|
|
170
|
+
|
|
171
|
+
def local_origin(headers: Dict[str, str]) -> bool:
|
|
172
|
+
"""True when no browser page sent the request: no Origin, or one on this machine."""
|
|
173
|
+
origin = headers.get("origin")
|
|
174
|
+
if origin is None:
|
|
175
|
+
return True
|
|
176
|
+
o = origin.strip().lower()
|
|
177
|
+
return any(o == base or o.startswith(base + ":") for base in LOCAL_ORIGINS)
|
|
178
|
+
|
|
179
|
+
|
|
180
|
+
# ---------------------------------------------------------------------------------------------------------------------
|
|
181
|
+
# The Copilot SDK
|
|
182
|
+
|
|
183
|
+
|
|
184
|
+
class Copilot:
|
|
185
|
+
def __init__(self, token: str):
|
|
186
|
+
from copilot import CopilotClient
|
|
187
|
+
|
|
188
|
+
# The SDK keeps its state under this folder, apart from the user's own Copilot CLI.
|
|
189
|
+
home = os.path.join(os.environ.get("FORGEQE_HOME") or os.path.join(os.path.expanduser("~"), ".forgeqe"), "copilot")
|
|
190
|
+
os.makedirs(home, exist_ok=True)
|
|
191
|
+
# An empty working folder: the session reads no project files and no custom instructions.
|
|
192
|
+
self.workdir = tempfile.mkdtemp(prefix="forgeqe-copilot-")
|
|
193
|
+
self.client = CopilotClient(github_token=token, use_logged_in_user=False, base_directory=home, working_directory=self.workdir, log_level="error")
|
|
194
|
+
|
|
195
|
+
async def start(self) -> None:
|
|
196
|
+
await self.client.start()
|
|
197
|
+
status = await self.client.get_auth_status()
|
|
198
|
+
self.login = status.login or ""
|
|
199
|
+
if not status.isAuthenticated:
|
|
200
|
+
raise GatewayError(401, "GitHub Copilot did not accept the token: use a fine-grained token with the Copilot Requests permission, or the GitHub CLI's sign-in, for an account with a Copilot seat.")
|
|
201
|
+
|
|
202
|
+
async def stop(self) -> None:
|
|
203
|
+
try:
|
|
204
|
+
await self.client.stop()
|
|
205
|
+
except Exception:
|
|
206
|
+
pass
|
|
207
|
+
|
|
208
|
+
async def models(self) -> List[Dict[str, Any]]:
|
|
209
|
+
out = []
|
|
210
|
+
for m in await self.client.list_models():
|
|
211
|
+
supports = getattr(getattr(m, "capabilities", None), "supports", None)
|
|
212
|
+
out.append({"id": m.id, "object": "model", "owned_by": "github-copilot", "vision": bool(getattr(supports, "vision", False))})
|
|
213
|
+
return out
|
|
214
|
+
|
|
215
|
+
async def complete(self, req: Dict[str, Any]) -> Dict[str, Any]:
|
|
216
|
+
from copilot.generated.rpc import PermissionDecisionReject
|
|
217
|
+
from copilot.session_events import AssistantMessageData, AssistantUsageData, SessionErrorData
|
|
218
|
+
|
|
219
|
+
usage = {"input": 0, "output": 0, "cached": 0}
|
|
220
|
+
errors: List[str] = []
|
|
221
|
+
|
|
222
|
+
def on_event(event: Any) -> None:
|
|
223
|
+
data = getattr(event, "data", None)
|
|
224
|
+
if isinstance(data, AssistantUsageData):
|
|
225
|
+
usage["input"] += data.input_tokens or 0
|
|
226
|
+
usage["output"] += data.output_tokens or 0
|
|
227
|
+
usage["cached"] += data.cache_read_tokens or 0
|
|
228
|
+
elif isinstance(data, SessionErrorData):
|
|
229
|
+
errors.append(data.message or data.error_type or "error")
|
|
230
|
+
|
|
231
|
+
session = await self.client.create_session(
|
|
232
|
+
model=req["model"],
|
|
233
|
+
# The caller's prompt is the whole prompt; without one the session still gets no tools.
|
|
234
|
+
system_message={"mode": "replace", "content": req["system"] or "You are a helpful assistant."},
|
|
235
|
+
available_tools=[],
|
|
236
|
+
on_permission_request=lambda *_: PermissionDecisionReject(),
|
|
237
|
+
working_directory=self.workdir,
|
|
238
|
+
skip_custom_instructions=True,
|
|
239
|
+
enable_config_discovery=False,
|
|
240
|
+
enable_skills=False,
|
|
241
|
+
enable_session_store=False,
|
|
242
|
+
on_event=on_event,
|
|
243
|
+
)
|
|
244
|
+
try:
|
|
245
|
+
reply = await session.send_and_wait(req["prompt"], attachments=req["attachments"] or None, response_schema=req["schema"], timeout=REQUEST_TIMEOUT_S)
|
|
246
|
+
except TimeoutError:
|
|
247
|
+
raise GatewayError(504, f"GitHub Copilot did not answer within {int(REQUEST_TIMEOUT_S)} seconds.")
|
|
248
|
+
finally:
|
|
249
|
+
try:
|
|
250
|
+
await session.disconnect()
|
|
251
|
+
except Exception:
|
|
252
|
+
pass
|
|
253
|
+
if errors:
|
|
254
|
+
msg = errors[-1]
|
|
255
|
+
raise GatewayError(401 if "auth" in msg.lower() or "401" in msg else 502, f"GitHub Copilot: {msg}")
|
|
256
|
+
data = getattr(reply, "data", None)
|
|
257
|
+
if not isinstance(data, AssistantMessageData):
|
|
258
|
+
raise GatewayError(502, "GitHub Copilot sent no reply.")
|
|
259
|
+
content = data.content or ""
|
|
260
|
+
if req["schema"] is not None or req["json_only"]:
|
|
261
|
+
content = strip_fences(content)
|
|
262
|
+
return completion(req["model"], content, usage)
|
|
263
|
+
|
|
264
|
+
|
|
265
|
+
# ---------------------------------------------------------------------------------------------------------------------
|
|
266
|
+
# A small HTTP/1.1 server: one request per connection
|
|
267
|
+
|
|
268
|
+
|
|
269
|
+
def _response(status: int, payload: Dict[str, Any]) -> bytes:
|
|
270
|
+
body = json.dumps(payload).encode()
|
|
271
|
+
reason = {200: "OK", 400: "Bad Request", 401: "Unauthorized", 403: "Forbidden", 404: "Not Found", 413: "Payload Too Large", 502: "Bad Gateway", 504: "Gateway Timeout"}.get(status, "Error")
|
|
272
|
+
head = f"HTTP/1.1 {status} {reason}\r\nContent-Type: application/json\r\nContent-Length: {len(body)}\r\nConnection: close\r\n\r\n"
|
|
273
|
+
return head.encode() + body
|
|
274
|
+
|
|
275
|
+
|
|
276
|
+
def _error(status: int, message: str) -> bytes:
|
|
277
|
+
return _response(status, {"error": {"message": message, "type": "forgeqe_copilot_gateway"}})
|
|
278
|
+
|
|
279
|
+
|
|
280
|
+
async def handle(reader: asyncio.StreamReader, writer: asyncio.StreamWriter, copilot: Any, key: str) -> None:
|
|
281
|
+
try:
|
|
282
|
+
try:
|
|
283
|
+
head = await reader.readuntil(b"\r\n\r\n")
|
|
284
|
+
except (asyncio.IncompleteReadError, asyncio.LimitOverrunError):
|
|
285
|
+
return
|
|
286
|
+
lines = head.decode("latin-1").split("\r\n")
|
|
287
|
+
method, path = (lines[0].split(" ") + ["", ""])[:2]
|
|
288
|
+
headers = {}
|
|
289
|
+
for line in lines[1:]:
|
|
290
|
+
if ":" in line:
|
|
291
|
+
k, v = line.split(":", 1)
|
|
292
|
+
headers[k.strip().lower()] = v.strip()
|
|
293
|
+
if not local_origin(headers):
|
|
294
|
+
writer.write(_error(403, "Requests from web pages are refused."))
|
|
295
|
+
return
|
|
296
|
+
if not authorized(headers, key):
|
|
297
|
+
writer.write(_error(401, "Unauthorized"))
|
|
298
|
+
return
|
|
299
|
+
path = path.split("?", 1)[0].rstrip("/")
|
|
300
|
+
if path.startswith("/v1/"):
|
|
301
|
+
path = path[3:]
|
|
302
|
+
if method == "GET" and path == "/models":
|
|
303
|
+
writer.write(_response(200, {"object": "list", "data": await copilot.models()}))
|
|
304
|
+
return
|
|
305
|
+
if method != "POST" or path != "/chat/completions":
|
|
306
|
+
writer.write(_error(404, f"{method} {path} is not served here."))
|
|
307
|
+
return
|
|
308
|
+
try:
|
|
309
|
+
length = int(headers.get("content-length", "0"))
|
|
310
|
+
except ValueError:
|
|
311
|
+
length = -1
|
|
312
|
+
if length < 0 or length > MAX_BODY:
|
|
313
|
+
writer.write(_error(413, "The request body is too large."))
|
|
314
|
+
return
|
|
315
|
+
raw = await reader.readexactly(length)
|
|
316
|
+
try:
|
|
317
|
+
body = json.loads(raw.decode("utf-8"))
|
|
318
|
+
except ValueError:
|
|
319
|
+
writer.write(_error(400, "The request body is not JSON."))
|
|
320
|
+
return
|
|
321
|
+
writer.write(_response(200, await copilot.complete(translate(body))))
|
|
322
|
+
except GatewayError as e:
|
|
323
|
+
writer.write(_error(e.status, e.message))
|
|
324
|
+
except Exception as e: # The SDK's own errors: report them rather than drop the connection.
|
|
325
|
+
writer.write(_error(502, f"GitHub Copilot: {type(e).__name__}: {e}"[:2000]))
|
|
326
|
+
finally:
|
|
327
|
+
try:
|
|
328
|
+
await writer.drain()
|
|
329
|
+
writer.close()
|
|
330
|
+
except Exception:
|
|
331
|
+
pass
|
|
332
|
+
|
|
333
|
+
|
|
334
|
+
async def serve(out: TextIO, copilot: Any = None) -> int:
|
|
335
|
+
key = os.environ.get(KEY_ENV, "")
|
|
336
|
+
token = os.environ.get(TOKEN_ENV, "").strip()
|
|
337
|
+
if len(key) < 32:
|
|
338
|
+
out.write(json.dumps({"error": f"{KEY_ENV} must hold the gateway's one-time key."}) + "\n")
|
|
339
|
+
return 2
|
|
340
|
+
if copilot is None:
|
|
341
|
+
if not token:
|
|
342
|
+
out.write(json.dumps({"error": "No GitHub token for Copilot: run forgeqe auth login copilot."}) + "\n")
|
|
343
|
+
return 1
|
|
344
|
+
if token.startswith("ghp_"):
|
|
345
|
+
out.write(json.dumps({"error": "GitHub Copilot does not take classic tokens (ghp_): use a fine-grained token with the Copilot Requests permission."}) + "\n")
|
|
346
|
+
return 1
|
|
347
|
+
try:
|
|
348
|
+
copilot = Copilot(token)
|
|
349
|
+
await copilot.start()
|
|
350
|
+
except GatewayError as e:
|
|
351
|
+
out.write(json.dumps({"error": e.message}) + "\n")
|
|
352
|
+
return 1
|
|
353
|
+
except Exception as e:
|
|
354
|
+
out.write(json.dumps({"error": f"GitHub Copilot could not start: {type(e).__name__}: {e}"[:1500]}) + "\n")
|
|
355
|
+
return 1
|
|
356
|
+
server = await asyncio.start_server(lambda r, w: handle(r, w, copilot, key), host="127.0.0.1", port=0, limit=64 * 1024)
|
|
357
|
+
port = server.sockets[0].getsockname()[1]
|
|
358
|
+
out.write(json.dumps({"port": port, "login": getattr(copilot, "login", "")}) + "\n")
|
|
359
|
+
out.flush()
|
|
360
|
+
loop = asyncio.get_running_loop()
|
|
361
|
+
# Stop when ForgeQE closes stdin (or exits).
|
|
362
|
+
await loop.run_in_executor(None, sys.stdin.read)
|
|
363
|
+
server.close()
|
|
364
|
+
await copilot.stop()
|
|
365
|
+
return 0
|
|
366
|
+
|
|
367
|
+
|
|
368
|
+
def main(out: TextIO) -> int:
|
|
369
|
+
return asyncio.run(serve(out))
|
|
@@ -0,0 +1,149 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Data that changes from one run to the next, in what a recording keeps: prices, dates, times, ids and order numbers,
|
|
3
|
+
long numbers and counts in a sentence. A test that spells this run's value out fails on the next run, so the recorder
|
|
4
|
+
keeps the parts that hold and makes the rest a pattern, a variable, or a value made fresh on each run.
|
|
5
|
+
|
|
6
|
+
- ``parts(text)``: the spans of ``text`` that change, with their kind.
|
|
7
|
+
- ``pattern_of(text)``: a regular expression (JavaScript and Python alike) that the text matches on any run: the stable
|
|
8
|
+
parts as they are, each changing part by its kind.
|
|
9
|
+
- ``stable_part(text)``: the longest part that holds, for a locator that finds the element by it.
|
|
10
|
+
- ``relative_date(text, today)``: a typed date as days from today, for ``{{$date:+N:FORMAT}}``.
|
|
11
|
+
|
|
12
|
+
A small number on its own ("1" in a cart badge, "3" items) is not changing data: it is usually what the check is about.
|
|
13
|
+
"""
|
|
14
|
+
from __future__ import annotations
|
|
15
|
+
|
|
16
|
+
import re
|
|
17
|
+
from datetime import date, datetime, timedelta
|
|
18
|
+
from typing import List, Optional, Tuple
|
|
19
|
+
|
|
20
|
+
_MONTHS = "jan|feb|mar|apr|may|jun|jul|aug|sep|sept|oct|nov|dec|january|february|march|april|june|july|august|september|october|november|december"
|
|
21
|
+
|
|
22
|
+
# (kind, what finds it, the pattern a test asserts instead). Order matters: the first kind to claim a span keeps it.
|
|
23
|
+
_KINDS: List[Tuple[str, "re.Pattern[str]", str]] = [
|
|
24
|
+
("uuid", re.compile(r"\b[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\b", re.I), r"[0-9a-fA-F-]{36}"),
|
|
25
|
+
("date", re.compile(r"\b\d{4}-\d{2}-\d{2}(?:[T ]\d{2}:\d{2}(?::\d{2})?(?:\.\d+)?(?:Z|[+-]\d{2}:?\d{2})?)?\b"), r"\d{4}-\d{2}-\d{2}(?:[T ][\d:.]+(?:Z|[+-][\d:]+)?)?"),
|
|
26
|
+
("date", re.compile(r"\b\d{1,2}[/.]\d{1,2}[/.]\d{2,4}\b"), r"\d{1,2}[/.]\d{1,2}[/.]\d{2,4}"),
|
|
27
|
+
("date", re.compile(rf"\b(?:\d{{1,2}}\s+(?:{_MONTHS})\.?,?\s+\d{{4}}|(?:{_MONTHS})\.?\s+\d{{1,2}},?\s+\d{{4}})\b", re.I), r"[\w.]+\s+[\w.]+,?\s+\d{4}"),
|
|
28
|
+
("time", re.compile(r"\b\d{1,2}:\d{2}(?::\d{2})?\s?(?:[ap]\.?m\.?)?(?![\d:])", re.I), r"\d{1,2}:\d{2}(?::\d{2})?\s?(?:[aApP]\.?[mM]\.?)?"),
|
|
29
|
+
("money", re.compile(r"(?:[$€£¥₹]\s?|\b(?:USD|EUR|GBP|INR|JPY|AUD|CAD)\s?)\d[\d,]*(?:\.\d{1,2})?\b|\b\d[\d,]*(?:\.\d{1,2})?\s?(?:USD|EUR|GBP|INR|JPY|AUD|CAD|€|£)", re.I), r"(?:[$€£¥₹]\s?|[A-Z]{3}\s?)?\d[\d,]*(?:\.\d{1,2})?(?:\s?(?:[A-Z]{3}|€|£))?"),
|
|
30
|
+
("id", re.compile(r"(?<![\w-])(?=[A-Z0-9-]*\d)(?=[A-Z0-9-]*[A-Z])[A-Z0-9]{1,6}-[A-Z0-9]{2,}(?:-[A-Z0-9]+)*(?![\w-])"), r"[A-Z0-9]+(?:-[A-Z0-9]+)+"),
|
|
31
|
+
("id", re.compile(r"#\d{3,}\b"), r"#\d+"),
|
|
32
|
+
("id", re.compile(r"\b(?=[A-Za-z0-9]*\d)(?=[A-Za-z0-9]*[A-Za-z])[A-Za-z0-9]{10,}\b"), r"[A-Za-z0-9]{10,}"),
|
|
33
|
+
("number", re.compile(r"\b\d{1,3}(?:,\d{3})+(?:\.\d+)?\b|\b\d{4,}(?:\.\d+)?\b|\b\d+\.\d+\b"), r"\d[\d,]*(?:\.\d+)?"),
|
|
34
|
+
]
|
|
35
|
+
# A count in a sentence: "2,463 properties found", "12 results", "3 items in your cart".
|
|
36
|
+
_COUNT = re.compile(r"\b\d[\d,]*(?=\s+(?:results?|items?|products?|properties|matches|hits|records|rows|orders|reviews|comments|people|days|hours|minutes|seconds|left|available|found|in stock)\b)", re.I)
|
|
37
|
+
_COUNT_PATTERN = r"\d[\d,]*"
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def _spans(text: str) -> List[Tuple[int, int, str, str]]:
|
|
41
|
+
"""(start, end, kind, the pattern of the matcher that found it), in order."""
|
|
42
|
+
t = text or ""
|
|
43
|
+
found: List[Tuple[int, int, str, str]] = []
|
|
44
|
+
|
|
45
|
+
def free(a: int, b: int) -> bool:
|
|
46
|
+
return all(b <= s or a >= e for s, e, _, _ in found)
|
|
47
|
+
|
|
48
|
+
for kind, rx, pat in _KINDS:
|
|
49
|
+
for m in rx.finditer(t):
|
|
50
|
+
if free(m.start(), m.end()):
|
|
51
|
+
found.append((m.start(), m.end(), kind, pat))
|
|
52
|
+
for m in _COUNT.finditer(t):
|
|
53
|
+
if free(m.start(), m.end()):
|
|
54
|
+
found.append((m.start(), m.end(), "count", _COUNT_PATTERN))
|
|
55
|
+
return sorted(found)
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def parts(text: str) -> List[Tuple[int, int, str]]:
|
|
59
|
+
"""The spans of `text` that change from run to run, in order, as (start, end, kind)."""
|
|
60
|
+
return [(s, e, k) for s, e, k, _ in _spans(text)]
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def is_dynamic(text: str) -> bool:
|
|
64
|
+
return bool(parts(text))
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def _token_pattern(kind: str, value: str, matched: str) -> str:
|
|
68
|
+
if kind == "date" and re.fullmatch(r"\d{4}-\d{2}-\d{2}", value):
|
|
69
|
+
return r"\d{4}-\d{2}-\d{2}"
|
|
70
|
+
if kind == "money":
|
|
71
|
+
# Keep the currency the run showed; the amount changes.
|
|
72
|
+
cur = re.match(r"^([$€£¥₹]|[A-Za-z]{3})\s?", value)
|
|
73
|
+
amount = r"\d[\d,]*(?:\.\d{1,2})?"
|
|
74
|
+
if cur:
|
|
75
|
+
return re.escape(cur.group(1)) + r"\s?" + amount
|
|
76
|
+
tail = re.search(r"\s?([A-Za-z]{3}|€|£)$", value)
|
|
77
|
+
return amount + (r"\s?" + re.escape(tail.group(1)) if tail else "")
|
|
78
|
+
return matched
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def pattern_of(text: str, anchored: bool = False) -> Optional[str]:
|
|
82
|
+
"""A regular expression the text matches on any run, or None when nothing in it changes."""
|
|
83
|
+
t = " ".join((text or "").split())
|
|
84
|
+
spans = _spans(t)
|
|
85
|
+
if not spans:
|
|
86
|
+
return None
|
|
87
|
+
out, at = [], 0
|
|
88
|
+
for s, e, kind, matched in spans:
|
|
89
|
+
out.append(re.escape(t[at:s]).replace("\\ ", r"\s+"))
|
|
90
|
+
out.append(_token_pattern(kind, t[s:e], matched))
|
|
91
|
+
at = e
|
|
92
|
+
out.append(re.escape(t[at:]).replace("\\ ", r"\s+"))
|
|
93
|
+
body = "".join(out)
|
|
94
|
+
return f"^{body}$" if anchored else body
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def stable_part(text: str, min_len: int = 3) -> str:
|
|
98
|
+
"""The longest stretch of `text` with nothing that changes in it, trimmed; "" when none is long enough."""
|
|
99
|
+
t = " ".join((text or "").split())
|
|
100
|
+
cuts, at = [], 0
|
|
101
|
+
for s, e, _ in parts(t):
|
|
102
|
+
cuts.append(t[at:s])
|
|
103
|
+
at = e
|
|
104
|
+
cuts.append(t[at:])
|
|
105
|
+
best = max((c.strip(" ,.:;|/-–·") for c in cuts), key=len, default="")
|
|
106
|
+
return best if len(best) >= min_len and re.search(r"[A-Za-z]", best) else ""
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
def values(text: str) -> List[str]:
|
|
110
|
+
"""The changing values in `text`, as shown ("$14.15", "A-1042")."""
|
|
111
|
+
t = " ".join((text or "").split())
|
|
112
|
+
return [t[s:e] for s, e, _ in parts(t)]
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
_DATE_FORMATS = (("%Y-%m-%d", r"\d{4}-\d{2}-\d{2}"), ("%d/%m/%Y", r"\d{2}/\d{2}/\d{4}"), ("%m/%d/%Y", r"\d{2}/\d{2}/\d{4}"), ("%d.%m.%Y", r"\d{2}\.\d{2}\.\d{4}"))
|
|
116
|
+
MAX_DATE_DAYS = 3660
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
def relative_date(text: str, today: Optional[date] = None) -> Optional[str]:
|
|
120
|
+
"""
|
|
121
|
+
A typed date, as ``{{$date:+N:FORMAT}}`` (N days from today) when it is the whole value and reads one way only;
|
|
122
|
+
None otherwise. 03/04/2026 could be either day-first or month-first: it is left as typed.
|
|
123
|
+
"""
|
|
124
|
+
t = (text or "").strip()
|
|
125
|
+
today = today or date.today()
|
|
126
|
+
hits = []
|
|
127
|
+
for fmt, rx in _DATE_FORMATS:
|
|
128
|
+
if not re.fullmatch(rx, t):
|
|
129
|
+
continue
|
|
130
|
+
try:
|
|
131
|
+
hits.append((fmt, datetime.strptime(t, fmt).date()))
|
|
132
|
+
except ValueError:
|
|
133
|
+
continue
|
|
134
|
+
if len({d for _, d in hits}) != 1:
|
|
135
|
+
return None
|
|
136
|
+
fmt, d = hits[0]
|
|
137
|
+
days = (d - today).days
|
|
138
|
+
if abs(days) > MAX_DATE_DAYS:
|
|
139
|
+
return None
|
|
140
|
+
return "{{$date:%s%d:%s}}" % ("+" if days >= 0 else "-", abs(days), fmt)
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
def date_from(template: str, today: Optional[date] = None) -> str:
|
|
144
|
+
"""The date a ``{{$date:+N:FORMAT}}`` stands for today (for tests of this module and the replay's own)."""
|
|
145
|
+
m = re.fullmatch(r"\{\{\$date:([+-])(\d+):([^}]+)\}\}", template)
|
|
146
|
+
if not m:
|
|
147
|
+
return template
|
|
148
|
+
d = (today or date.today()) + timedelta(days=int(m.group(2)) * (1 if m.group(1) == "+" else -1))
|
|
149
|
+
return d.strftime(m.group(3))
|