dev-memory-cli 0.22.1 → 0.23.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +255 -291
- package/bin/dev-memory.js +125 -2
- package/hooks/README.md +181 -47
- package/hooks/codex-hooks.json +1 -1
- package/hooks/hooks.json +1 -1
- package/lib/dev_memory_capture.py +152 -11
- package/lib/dev_memory_common.py +68 -19
- package/lib/dev_memory_graduate.py +1 -1
- package/lib/dev_memory_session_scan.py +1565 -0
- package/lib/dev_memory_setup.py +8 -3
- package/lib/dev_memory_summary.py +25 -16
- package/lib/maintenance/archive.md +84 -0
- package/lib/maintenance/tidy.md +103 -0
- package/lib/ui-app.html +128 -1
- package/lib/ui-server.js +67 -1
- package/package.json +2 -1
- package/scripts/hooks/_common.py +48 -12
- package/scripts/hooks/stop.py +6 -0
- package/suite-manifest.json +4 -4
|
@@ -0,0 +1,1565 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
|
|
3
|
+
import argparse
|
|
4
|
+
import datetime as dt
|
|
5
|
+
import hashlib
|
|
6
|
+
import json
|
|
7
|
+
import os
|
|
8
|
+
import plistlib
|
|
9
|
+
import re
|
|
10
|
+
import shlex
|
|
11
|
+
import shutil
|
|
12
|
+
import subprocess
|
|
13
|
+
import sys
|
|
14
|
+
import tempfile
|
|
15
|
+
import time
|
|
16
|
+
from pathlib import Path
|
|
17
|
+
|
|
18
|
+
from dev_memory_common import get_branch_paths, list_repos_in_workspace, now_iso
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
SCHEMA_VERSION = 1
|
|
22
|
+
PACKAGE_ROOT = Path(__file__).resolve().parents[1]
|
|
23
|
+
CONFIG_PATH = Path(os.environ.get("DEV_MEMORY_CONFIG_PATH", "~/.dev-memory/config.json")).expanduser()
|
|
24
|
+
DEV_MEMORY_HOME = Path(os.environ.get("DEV_MEMORY_HOME", "~/.dev-memory")).expanduser()
|
|
25
|
+
SCAN_ROOT = Path(os.environ.get("DEV_MEMORY_SCAN_ROOT", DEV_MEMORY_HOME / "jobs" / "session-scan")).expanduser()
|
|
26
|
+
CODEX_HOME = Path(os.environ.get("CODEX_HOME", "~/.codex")).expanduser()
|
|
27
|
+
PLIST_PATH = Path(os.environ.get(
|
|
28
|
+
"DEV_MEMORY_SCAN_PLIST",
|
|
29
|
+
"~/Library/LaunchAgents/com.dev-memory.session-scan.plist",
|
|
30
|
+
)).expanduser()
|
|
31
|
+
INTERNAL_MARKER = "DEV_MEMORY_INTERNAL_SESSION_SUMMARY_V1"
|
|
32
|
+
MAINTENANCE_MARKER = "DEV_MEMORY_INTERNAL_MAINTENANCE_AGENT_V1"
|
|
33
|
+
INTERNAL_MARKERS = (INTERNAL_MARKER, MAINTENANCE_MARKER)
|
|
34
|
+
SUMMARY_MUTATION_FIELDS = (
|
|
35
|
+
"file_map",
|
|
36
|
+
"decisions",
|
|
37
|
+
"risks",
|
|
38
|
+
"glossary",
|
|
39
|
+
"shared_decisions",
|
|
40
|
+
"shared_context",
|
|
41
|
+
"shared_sources",
|
|
42
|
+
"upserts",
|
|
43
|
+
"appends",
|
|
44
|
+
"rewrites",
|
|
45
|
+
"deletes",
|
|
46
|
+
)
|
|
47
|
+
SUMMARY_PLACEHOLDER_WORDS = {
|
|
48
|
+
"decision",
|
|
49
|
+
"summary",
|
|
50
|
+
"reason",
|
|
51
|
+
"impact",
|
|
52
|
+
"risk",
|
|
53
|
+
"mitigation",
|
|
54
|
+
"term",
|
|
55
|
+
"definition",
|
|
56
|
+
"name",
|
|
57
|
+
"url",
|
|
58
|
+
"note",
|
|
59
|
+
"label",
|
|
60
|
+
"path",
|
|
61
|
+
"content",
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
DEFAULT_EXECUTORS = {
|
|
66
|
+
"coco": {
|
|
67
|
+
"enabled": True,
|
|
68
|
+
"command": "coco",
|
|
69
|
+
"model": None,
|
|
70
|
+
"profile": None,
|
|
71
|
+
"extra_args": [],
|
|
72
|
+
"env": {},
|
|
73
|
+
},
|
|
74
|
+
"codex": {
|
|
75
|
+
"enabled": True,
|
|
76
|
+
"command": "codex",
|
|
77
|
+
"model": None,
|
|
78
|
+
"profile": None,
|
|
79
|
+
"extra_args": [],
|
|
80
|
+
"env": {},
|
|
81
|
+
},
|
|
82
|
+
"claude": {
|
|
83
|
+
"enabled": True,
|
|
84
|
+
"command": "claude",
|
|
85
|
+
"model": None,
|
|
86
|
+
"profile": None,
|
|
87
|
+
"extra_args": [],
|
|
88
|
+
"env": {},
|
|
89
|
+
},
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
def _read_json(path, default=None):
|
|
94
|
+
try:
|
|
95
|
+
value = json.loads(Path(path).read_text(encoding="utf-8"))
|
|
96
|
+
return value
|
|
97
|
+
except (OSError, json.JSONDecodeError):
|
|
98
|
+
return default
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
def _atomic_json(path, payload):
|
|
102
|
+
path = Path(path)
|
|
103
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
104
|
+
tmp = path.with_name(f".{path.name}.tmp.{os.getpid()}.{time.time_ns()}")
|
|
105
|
+
tmp.write_text(json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
|
106
|
+
os.replace(tmp, path)
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
def _append_jsonl(path, payload):
|
|
110
|
+
path = Path(path)
|
|
111
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
112
|
+
with path.open("a", encoding="utf-8") as stream:
|
|
113
|
+
stream.write(json.dumps(payload, ensure_ascii=False, separators=(",", ":")) + "\n")
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
def _deep_merge(base, override):
|
|
117
|
+
out = dict(base)
|
|
118
|
+
for key, value in (override or {}).items():
|
|
119
|
+
if isinstance(value, dict) and isinstance(out.get(key), dict):
|
|
120
|
+
out[key] = _deep_merge(out[key], value)
|
|
121
|
+
else:
|
|
122
|
+
out[key] = value
|
|
123
|
+
return out
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
def default_scan_config():
|
|
127
|
+
return {
|
|
128
|
+
"executor": "auto",
|
|
129
|
+
"order": ["coco", "codex", "claude"],
|
|
130
|
+
"executors": json.loads(json.dumps(DEFAULT_EXECUTORS)),
|
|
131
|
+
"schedule_times": ["03:00", "13:00"],
|
|
132
|
+
"skip_when_computer_active": True,
|
|
133
|
+
"active_within_minutes": 10,
|
|
134
|
+
"activity_check_fail_closed": True,
|
|
135
|
+
"chunk_chars": 60000,
|
|
136
|
+
"idle_minutes": 60,
|
|
137
|
+
"first_lookback_days": 3,
|
|
138
|
+
"max_attempts": 2,
|
|
139
|
+
"invocation_timeout_seconds": 360,
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
def load_config():
|
|
144
|
+
root = _read_json(CONFIG_PATH, {})
|
|
145
|
+
root = root if isinstance(root, dict) else {}
|
|
146
|
+
section = root.get("session_scan")
|
|
147
|
+
return _deep_merge(default_scan_config(), section if isinstance(section, dict) else {})
|
|
148
|
+
|
|
149
|
+
|
|
150
|
+
def save_scan_config(section):
|
|
151
|
+
root = _read_json(CONFIG_PATH, {})
|
|
152
|
+
root = root if isinstance(root, dict) else {}
|
|
153
|
+
root["session_scan"] = section
|
|
154
|
+
_atomic_json(CONFIG_PATH, root)
|
|
155
|
+
|
|
156
|
+
|
|
157
|
+
def validate_config(config):
|
|
158
|
+
errors = []
|
|
159
|
+
warnings = []
|
|
160
|
+
selected = config.get("executor", "auto")
|
|
161
|
+
executors = config.get("executors")
|
|
162
|
+
if not isinstance(executors, dict) or not executors:
|
|
163
|
+
errors.append("session_scan.executors must be a non-empty object")
|
|
164
|
+
executors = {}
|
|
165
|
+
if selected != "auto" and selected not in executors:
|
|
166
|
+
errors.append(f"executor '{selected}' has no preset")
|
|
167
|
+
order = config.get("order")
|
|
168
|
+
if not isinstance(order, list) or not order:
|
|
169
|
+
errors.append("session_scan.order must be a non-empty list")
|
|
170
|
+
for name, preset in executors.items():
|
|
171
|
+
if not isinstance(preset, dict):
|
|
172
|
+
errors.append(f"executor '{name}' must be an object")
|
|
173
|
+
continue
|
|
174
|
+
command = preset.get("command")
|
|
175
|
+
if not isinstance(command, str) or not command.strip():
|
|
176
|
+
errors.append(f"executor '{name}' requires command")
|
|
177
|
+
if name == "codex" and "--ephemeral" in (preset.get("extra_args") or []):
|
|
178
|
+
warnings.append("codex.extra_args does not need --ephemeral; the scanner enforces it")
|
|
179
|
+
try:
|
|
180
|
+
if int(config.get("chunk_chars", 0)) < 1000:
|
|
181
|
+
errors.append("chunk_chars must be at least 1000")
|
|
182
|
+
except (TypeError, ValueError):
|
|
183
|
+
errors.append("chunk_chars must be an integer")
|
|
184
|
+
schedules = config.get("schedule_times")
|
|
185
|
+
if not isinstance(schedules, list) or not schedules:
|
|
186
|
+
errors.append("schedule_times must be a non-empty list")
|
|
187
|
+
else:
|
|
188
|
+
for value in schedules:
|
|
189
|
+
try:
|
|
190
|
+
_parse_schedule_time(value)
|
|
191
|
+
except ValueError as exc:
|
|
192
|
+
errors.append(str(exc))
|
|
193
|
+
try:
|
|
194
|
+
if int(config.get("active_within_minutes", 0)) < 1:
|
|
195
|
+
errors.append("active_within_minutes must be at least 1")
|
|
196
|
+
except (TypeError, ValueError):
|
|
197
|
+
errors.append("active_within_minutes must be an integer")
|
|
198
|
+
try:
|
|
199
|
+
if int(config.get("invocation_timeout_seconds", 0)) < 30:
|
|
200
|
+
errors.append("invocation_timeout_seconds must be at least 30")
|
|
201
|
+
except (TypeError, ValueError):
|
|
202
|
+
errors.append("invocation_timeout_seconds must be an integer")
|
|
203
|
+
return {"valid": not errors, "errors": errors, "warnings": warnings}
|
|
204
|
+
|
|
205
|
+
|
|
206
|
+
def _parse_schedule_time(value):
|
|
207
|
+
match = re.fullmatch(r"(\d{1,2}):(\d{2})", str(value or "").strip())
|
|
208
|
+
if not match:
|
|
209
|
+
raise ValueError(f"invalid schedule time '{value}', expected HH:MM")
|
|
210
|
+
hour, minute = int(match.group(1)), int(match.group(2))
|
|
211
|
+
if not 0 <= hour <= 23 or not 0 <= minute <= 59:
|
|
212
|
+
raise ValueError(f"invalid schedule time '{value}', expected HH:MM")
|
|
213
|
+
return hour, minute
|
|
214
|
+
|
|
215
|
+
|
|
216
|
+
def _calendar_intervals(config):
|
|
217
|
+
unique = sorted({_parse_schedule_time(value) for value in config.get("schedule_times", [])})
|
|
218
|
+
return [{"Hour": hour, "Minute": minute} for hour, minute in unique]
|
|
219
|
+
|
|
220
|
+
|
|
221
|
+
def _mac_idle_seconds():
|
|
222
|
+
if sys.platform != "darwin":
|
|
223
|
+
return None
|
|
224
|
+
result = subprocess.run(
|
|
225
|
+
["ioreg", "-c", "IOHIDSystem", "-d", "4"],
|
|
226
|
+
capture_output=True,
|
|
227
|
+
text=True,
|
|
228
|
+
check=False,
|
|
229
|
+
)
|
|
230
|
+
if result.returncode != 0:
|
|
231
|
+
return None
|
|
232
|
+
match = re.search(r'"HIDIdleTime"\s*=\s*(\d+)', result.stdout or "")
|
|
233
|
+
if not match:
|
|
234
|
+
return None
|
|
235
|
+
return int(match.group(1)) / 1_000_000_000
|
|
236
|
+
|
|
237
|
+
|
|
238
|
+
def computer_activity(config):
|
|
239
|
+
threshold_seconds = int(config.get("active_within_minutes", 10)) * 60
|
|
240
|
+
idle_seconds = _mac_idle_seconds()
|
|
241
|
+
if idle_seconds is None:
|
|
242
|
+
return {
|
|
243
|
+
"status": "unknown",
|
|
244
|
+
"idle_seconds": None,
|
|
245
|
+
"threshold_seconds": threshold_seconds,
|
|
246
|
+
"skip": bool(config.get("activity_check_fail_closed", True)),
|
|
247
|
+
}
|
|
248
|
+
return {
|
|
249
|
+
"status": "active" if idle_seconds < threshold_seconds else "idle",
|
|
250
|
+
"idle_seconds": round(idle_seconds, 3),
|
|
251
|
+
"threshold_seconds": threshold_seconds,
|
|
252
|
+
"skip": idle_seconds < threshold_seconds,
|
|
253
|
+
}
|
|
254
|
+
|
|
255
|
+
|
|
256
|
+
def choose_executor(config):
|
|
257
|
+
executors = config.get("executors", {})
|
|
258
|
+
selected = config.get("executor", "auto")
|
|
259
|
+
names = config.get("order", []) if selected == "auto" else [selected]
|
|
260
|
+
for name in names:
|
|
261
|
+
preset = executors.get(name)
|
|
262
|
+
if not isinstance(preset, dict) or preset.get("enabled") is False:
|
|
263
|
+
continue
|
|
264
|
+
command = preset.get("command", name)
|
|
265
|
+
executable = shlex.split(command)[0] if command else ""
|
|
266
|
+
if executable and (Path(executable).exists() or shutil.which(executable)):
|
|
267
|
+
return name, preset
|
|
268
|
+
raise RuntimeError(f"no available session-scan executor in {names}")
|
|
269
|
+
|
|
270
|
+
|
|
271
|
+
def _content_text(content):
|
|
272
|
+
if isinstance(content, str):
|
|
273
|
+
return content.strip()
|
|
274
|
+
if not isinstance(content, list):
|
|
275
|
+
return ""
|
|
276
|
+
parts = []
|
|
277
|
+
for item in content:
|
|
278
|
+
if not isinstance(item, dict):
|
|
279
|
+
continue
|
|
280
|
+
if item.get("type") in {"tool_use", "tool_result", "function_call", "function_call_output"}:
|
|
281
|
+
continue
|
|
282
|
+
text = item.get("text")
|
|
283
|
+
if isinstance(text, str) and text.strip():
|
|
284
|
+
parts.append(text.strip())
|
|
285
|
+
return "\n".join(parts)
|
|
286
|
+
|
|
287
|
+
|
|
288
|
+
def _semantic_message(obj):
|
|
289
|
+
if obj.get("type") != "response_item":
|
|
290
|
+
return None
|
|
291
|
+
payload = obj.get("payload")
|
|
292
|
+
if not isinstance(payload, dict) or payload.get("type") != "message":
|
|
293
|
+
return None
|
|
294
|
+
role = payload.get("role")
|
|
295
|
+
if role not in {"user", "assistant"}:
|
|
296
|
+
return None
|
|
297
|
+
text = _content_text(payload.get("content"))
|
|
298
|
+
if not text:
|
|
299
|
+
return None
|
|
300
|
+
return {"role": role, "text": text, "timestamp": obj.get("timestamp")}
|
|
301
|
+
|
|
302
|
+
|
|
303
|
+
def _usage_dict(value):
|
|
304
|
+
if not isinstance(value, dict):
|
|
305
|
+
return None
|
|
306
|
+
aliases = {
|
|
307
|
+
"input_tokens": ("input_tokens", "inputTokens"),
|
|
308
|
+
"cached_input_tokens": ("cached_input_tokens", "cachedInputTokens", "cache_read_input_tokens"),
|
|
309
|
+
"output_tokens": ("output_tokens", "outputTokens"),
|
|
310
|
+
"reasoning_output_tokens": ("reasoning_output_tokens", "reasoningOutputTokens"),
|
|
311
|
+
"total_tokens": ("total_tokens", "totalTokens"),
|
|
312
|
+
}
|
|
313
|
+
out = {}
|
|
314
|
+
for target, keys in aliases.items():
|
|
315
|
+
for key in keys:
|
|
316
|
+
if isinstance(value.get(key), (int, float)):
|
|
317
|
+
out[target] = int(value[key])
|
|
318
|
+
break
|
|
319
|
+
if out and "total_tokens" not in out:
|
|
320
|
+
out["total_tokens"] = out.get("input_tokens", 0) + out.get("output_tokens", 0)
|
|
321
|
+
return out or None
|
|
322
|
+
|
|
323
|
+
|
|
324
|
+
def parse_codex_session(path, since_offset=0):
|
|
325
|
+
path = Path(path)
|
|
326
|
+
meta = {}
|
|
327
|
+
messages = []
|
|
328
|
+
total_usage = None
|
|
329
|
+
internal_marker = False
|
|
330
|
+
end_offset = since_offset
|
|
331
|
+
with path.open("rb") as stream:
|
|
332
|
+
if since_offset:
|
|
333
|
+
while True:
|
|
334
|
+
raw = stream.readline()
|
|
335
|
+
if not raw:
|
|
336
|
+
break
|
|
337
|
+
try:
|
|
338
|
+
obj = json.loads(raw.decode("utf-8"))
|
|
339
|
+
except (UnicodeDecodeError, json.JSONDecodeError):
|
|
340
|
+
continue
|
|
341
|
+
if obj.get("type") == "session_meta" and isinstance(obj.get("payload"), dict):
|
|
342
|
+
meta = obj["payload"]
|
|
343
|
+
break
|
|
344
|
+
stream.seek(since_offset)
|
|
345
|
+
while True:
|
|
346
|
+
start = stream.tell()
|
|
347
|
+
raw = stream.readline()
|
|
348
|
+
if not raw:
|
|
349
|
+
break
|
|
350
|
+
end = stream.tell()
|
|
351
|
+
end_offset = end
|
|
352
|
+
try:
|
|
353
|
+
obj = json.loads(raw.decode("utf-8"))
|
|
354
|
+
except (UnicodeDecodeError, json.JSONDecodeError):
|
|
355
|
+
continue
|
|
356
|
+
if obj.get("type") == "session_meta" and isinstance(obj.get("payload"), dict):
|
|
357
|
+
meta = obj["payload"]
|
|
358
|
+
payload = obj.get("payload") if isinstance(obj.get("payload"), dict) else {}
|
|
359
|
+
if obj.get("type") == "event_msg" and payload.get("type") == "token_count":
|
|
360
|
+
info = payload.get("info") if isinstance(payload.get("info"), dict) else {}
|
|
361
|
+
usage = _usage_dict(info.get("total_token_usage"))
|
|
362
|
+
if usage:
|
|
363
|
+
total_usage = usage
|
|
364
|
+
message = _semantic_message(obj)
|
|
365
|
+
if message and any(marker in message["text"] for marker in INTERNAL_MARKERS):
|
|
366
|
+
internal_marker = True
|
|
367
|
+
if message and end > since_offset:
|
|
368
|
+
message.update({"start_offset": start, "end_offset": end})
|
|
369
|
+
messages.append(message)
|
|
370
|
+
stat = path.stat()
|
|
371
|
+
return {
|
|
372
|
+
"path": str(path),
|
|
373
|
+
"size": stat.st_size,
|
|
374
|
+
"mtime": dt.datetime.fromtimestamp(stat.st_mtime, dt.timezone.utc).isoformat(),
|
|
375
|
+
"end_offset": end_offset,
|
|
376
|
+
"meta": meta,
|
|
377
|
+
"messages": messages,
|
|
378
|
+
"session_usage": total_usage,
|
|
379
|
+
"internal_marker": internal_marker,
|
|
380
|
+
}
|
|
381
|
+
|
|
382
|
+
|
|
383
|
+
def _codex_session_meta(path):
|
|
384
|
+
with Path(path).open("rb") as stream:
|
|
385
|
+
for raw in stream:
|
|
386
|
+
try:
|
|
387
|
+
obj = json.loads(raw.decode("utf-8"))
|
|
388
|
+
except (UnicodeDecodeError, json.JSONDecodeError):
|
|
389
|
+
continue
|
|
390
|
+
if obj.get("type") == "session_meta" and isinstance(obj.get("payload"), dict):
|
|
391
|
+
return obj["payload"]
|
|
392
|
+
return {}
|
|
393
|
+
|
|
394
|
+
|
|
395
|
+
def _session_files(lookback_days=None, cutoff_epoch=None):
|
|
396
|
+
roots = [CODEX_HOME / "sessions", CODEX_HOME / "archived_sessions"]
|
|
397
|
+
cutoff = cutoff_epoch
|
|
398
|
+
if cutoff is None and lookback_days is not None:
|
|
399
|
+
cutoff = time.time() - (lookback_days * 86400)
|
|
400
|
+
found = []
|
|
401
|
+
for root in roots:
|
|
402
|
+
if not root.exists():
|
|
403
|
+
continue
|
|
404
|
+
for path in root.rglob("*.jsonl"):
|
|
405
|
+
try:
|
|
406
|
+
if cutoff is not None and path.stat().st_mtime < cutoff:
|
|
407
|
+
continue
|
|
408
|
+
except OSError:
|
|
409
|
+
continue
|
|
410
|
+
found.append(path)
|
|
411
|
+
return sorted(found, key=lambda p: (p.stat().st_mtime, str(p)))
|
|
412
|
+
|
|
413
|
+
|
|
414
|
+
def _state_path(session_id):
|
|
415
|
+
safe = re.sub(r"[^0-9A-Za-z_.-]", "_", session_id)
|
|
416
|
+
return SCAN_ROOT / "state" / f"{safe}.json"
|
|
417
|
+
|
|
418
|
+
|
|
419
|
+
def _session_audit_path(session_id):
|
|
420
|
+
safe = re.sub(r"[^0-9A-Za-z_.-]", "_", session_id)
|
|
421
|
+
return SCAN_ROOT / "sessions" / f"{safe}.json"
|
|
422
|
+
|
|
423
|
+
|
|
424
|
+
def _workspace_primary(cwd):
|
|
425
|
+
for name in (".dev-memory-workspace.json", ".dev-assets-workspace.json"):
|
|
426
|
+
data = _read_json(Path(cwd) / name, {})
|
|
427
|
+
if isinstance(data, dict) and data.get("primary_repo"):
|
|
428
|
+
return data["primary_repo"]
|
|
429
|
+
return None
|
|
430
|
+
|
|
431
|
+
|
|
432
|
+
def resolve_target(cwd):
|
|
433
|
+
if not cwd:
|
|
434
|
+
return None, "missing_cwd"
|
|
435
|
+
root = Path(cwd).expanduser()
|
|
436
|
+
if not root.exists() or not root.is_dir():
|
|
437
|
+
return None, "cwd_not_found"
|
|
438
|
+
repos = list_repos_in_workspace(str(root))
|
|
439
|
+
if repos:
|
|
440
|
+
primary = _workspace_primary(root)
|
|
441
|
+
if primary:
|
|
442
|
+
root = next((repo for repo in repos if repo.name == primary), None)
|
|
443
|
+
if root is None:
|
|
444
|
+
return None, "workspace_primary_not_found"
|
|
445
|
+
elif len(repos) == 1:
|
|
446
|
+
root = repos[0]
|
|
447
|
+
else:
|
|
448
|
+
return None, "workspace_primary_required"
|
|
449
|
+
try:
|
|
450
|
+
repo_root, branch, branch_key, storage_root, repo_key, repo_dir, branch_dir = get_branch_paths(str(root))
|
|
451
|
+
except Exception as exc:
|
|
452
|
+
return None, f"repo_resolution_failed:{exc}"
|
|
453
|
+
if not branch_dir.exists():
|
|
454
|
+
return None, "memory_not_initialized"
|
|
455
|
+
return {
|
|
456
|
+
"repo_root": str(repo_root),
|
|
457
|
+
"repo_key": repo_key,
|
|
458
|
+
"repo_dir": str(repo_dir),
|
|
459
|
+
"branch": branch,
|
|
460
|
+
"branch_key": branch_key,
|
|
461
|
+
"branch_dir": str(branch_dir),
|
|
462
|
+
"storage_root": str(storage_root),
|
|
463
|
+
}, None
|
|
464
|
+
|
|
465
|
+
|
|
466
|
+
def _chunk_messages(messages, max_chars):
|
|
467
|
+
expanded = []
|
|
468
|
+
for message in messages:
|
|
469
|
+
text = message["text"]
|
|
470
|
+
if len(text) <= max_chars:
|
|
471
|
+
expanded.append(message)
|
|
472
|
+
continue
|
|
473
|
+
segment_count = (len(text) + max_chars - 1) // max_chars
|
|
474
|
+
for index in range(segment_count):
|
|
475
|
+
expanded.append({
|
|
476
|
+
**message,
|
|
477
|
+
"text": text[index * max_chars:(index + 1) * max_chars],
|
|
478
|
+
"segment_index": index + 1,
|
|
479
|
+
"segment_count": segment_count,
|
|
480
|
+
})
|
|
481
|
+
chunks = []
|
|
482
|
+
current = []
|
|
483
|
+
current_chars = 0
|
|
484
|
+
for message in expanded:
|
|
485
|
+
text = message["text"]
|
|
486
|
+
if current and current_chars + len(text) > max_chars:
|
|
487
|
+
chunks.append(current)
|
|
488
|
+
current = []
|
|
489
|
+
current_chars = 0
|
|
490
|
+
current.append(message)
|
|
491
|
+
current_chars += len(text)
|
|
492
|
+
if current_chars >= max_chars:
|
|
493
|
+
chunks.append(current)
|
|
494
|
+
current = []
|
|
495
|
+
current_chars = 0
|
|
496
|
+
if current:
|
|
497
|
+
chunks.append(current)
|
|
498
|
+
return chunks
|
|
499
|
+
|
|
500
|
+
|
|
501
|
+
def _existing_memory(target):
|
|
502
|
+
paths = [
|
|
503
|
+
Path(target["branch_dir"]) / name
|
|
504
|
+
for name in ("overview.md", "decisions.md", "risks.md", "glossary.md")
|
|
505
|
+
] + [
|
|
506
|
+
Path(target["repo_dir"]) / "repo" / name
|
|
507
|
+
for name in ("decisions.md", "glossary.md")
|
|
508
|
+
]
|
|
509
|
+
return [
|
|
510
|
+
{"path": str(path), "content": path.read_text(encoding="utf-8")}
|
|
511
|
+
for path in paths if path.exists()
|
|
512
|
+
]
|
|
513
|
+
|
|
514
|
+
|
|
515
|
+
def _partial_prompt(target, chunk, index, total):
|
|
516
|
+
material = [{"role": item["role"], "text": item["text"]} for item in chunk]
|
|
517
|
+
return f"""{INTERNAL_MARKER}
|
|
518
|
+
你是 dev-memory 的后台会话总结器。下面是仓库 {target['repo_key']} 分支 {target['branch']} 的第 {index}/{total} 个连续会话分块。
|
|
519
|
+
完整阅读本分块,不要忽略前部内容。只提炼在未来开发会话中仍有价值的决策、约束、风险、术语、命令、外部入口和功能文件定位;不要记录聊天流水账、普通进展或可从 Git 直接恢复的历史。
|
|
520
|
+
只输出 JSON 对象,允许字段:decisions、risks、glossary、file_map、shared_decisions、shared_context、shared_sources、skip_reason。字段内容沿用 summary-output 语义。
|
|
521
|
+
MESSAGES_JSON:
|
|
522
|
+
{json.dumps(material, ensure_ascii=False)}
|
|
523
|
+
"""
|
|
524
|
+
|
|
525
|
+
|
|
526
|
+
def _single_prompt(target, chunk):
|
|
527
|
+
payload = {
|
|
528
|
+
"existing_memory": _existing_memory(target),
|
|
529
|
+
"messages": [{"role": item["role"], "text": item["text"]} for item in chunk],
|
|
530
|
+
}
|
|
531
|
+
return f"""{INTERNAL_MARKER}
|
|
532
|
+
你是 dev-memory 的后台会话总结器。阅读完整会话与 existing_memory,直接生成最终 summary-output,不要先输出中间摘要。
|
|
533
|
+
只保留未来开发会话中仍有价值、且 existing_memory 尚未覆盖的决策、约束、风险、术语、命令、外部入口和功能文件定位;不要记录聊天流水账、普通进展或可从 Git 直接恢复的历史。
|
|
534
|
+
旧结论失效时使用 rewrites/deletes,不要追加矛盾条目。如果没有任何需要写入、改写或删除的内容,必须返回非空 skip_reason;禁止只返回 title 或空对象。
|
|
535
|
+
字段 schema:decisions/shared_decisions 是对象数组,每项使用 summary(完整结论)、可选 reason、可选 impact;risks/glossary/shared_context/shared_sources 是完整自然语言字符串数组;file_map 每项为 {{"label":"功能说明","paths":["真实相对路径"]}};upserts/appends 每项必须有 kind 和 content;rewrites 每项必须有 id/content/reason;deletes 每项必须有 id/reason。
|
|
536
|
+
所有值必须来自会话事实并可独立理解。禁止输出 decision/summary/reason/impact/risk/mitigation/term/definition/name/url/note/path/content 等 schema 占位词,禁止留空 summary。
|
|
537
|
+
只输出一个 JSON 对象,不要 markdown fence。允许字段:title、file_map、decisions、risks、glossary、shared_decisions、shared_context、shared_sources、upserts、appends、rewrites、deletes、skip_reason。
|
|
538
|
+
INPUT_JSON:
|
|
539
|
+
{json.dumps(payload, ensure_ascii=False)}
|
|
540
|
+
"""
|
|
541
|
+
|
|
542
|
+
|
|
543
|
+
def _final_prompt(target, partials):
|
|
544
|
+
payload = {"existing_memory": _existing_memory(target), "partial_summaries": partials}
|
|
545
|
+
return f"""{INTERNAL_MARKER}
|
|
546
|
+
你是 dev-memory 的后台会话总结器。把所有 partial_summaries 与 existing_memory 合并为一个最终 summary-output。
|
|
547
|
+
旧结论失效时使用 rewrites/deletes,不要追加矛盾条目;重复内容跳过。不要写当前进展、下一步或提交历史。
|
|
548
|
+
如果没有任何需要写入、改写或删除的内容,必须返回非空 skip_reason;禁止只返回 title 或空对象。
|
|
549
|
+
decisions/shared_decisions 每项必须有非空 summary;risks/glossary/shared_context/shared_sources 必须是完整自然语言字符串;file_map 每项必须有 label 和真实 paths。禁止输出 schema 占位词或空 summary。
|
|
550
|
+
只输出一个 JSON 对象,不要 markdown fence。允许字段:title、file_map、decisions、risks、glossary、shared_decisions、shared_context、shared_sources、upserts、appends、rewrites、deletes、skip_reason。
|
|
551
|
+
INPUT_JSON:
|
|
552
|
+
{json.dumps(payload, ensure_ascii=False)}
|
|
553
|
+
"""
|
|
554
|
+
|
|
555
|
+
|
|
556
|
+
def _summary_payload_meta(payload):
|
|
557
|
+
payload = payload if isinstance(payload, dict) else {}
|
|
558
|
+
serialized = json.dumps(payload, ensure_ascii=False, sort_keys=True, separators=(",", ":"))
|
|
559
|
+
field_counts = {}
|
|
560
|
+
for key in SUMMARY_MUTATION_FIELDS:
|
|
561
|
+
value = payload.get(key)
|
|
562
|
+
if isinstance(value, list):
|
|
563
|
+
field_counts[key] = len(value)
|
|
564
|
+
elif value:
|
|
565
|
+
field_counts[key] = 1
|
|
566
|
+
skip_reason = payload.get("skip_reason")
|
|
567
|
+
skip_reason = skip_reason.strip() if isinstance(skip_reason, str) else None
|
|
568
|
+
return {
|
|
569
|
+
"keys": sorted(payload),
|
|
570
|
+
"field_counts": field_counts,
|
|
571
|
+
"mutation_count": sum(field_counts.values()),
|
|
572
|
+
"skip_reason": skip_reason or None,
|
|
573
|
+
"chars": len(serialized),
|
|
574
|
+
"sha256": hashlib.sha256(serialized.encode("utf-8")).hexdigest(),
|
|
575
|
+
}
|
|
576
|
+
|
|
577
|
+
|
|
578
|
+
def _looks_like_placeholder_text(value):
|
|
579
|
+
if not isinstance(value, str):
|
|
580
|
+
return False
|
|
581
|
+
words = []
|
|
582
|
+
for line in value.splitlines():
|
|
583
|
+
normalized = re.sub(r"^[\s>*+-]+", "", line).strip().strip("::").lower()
|
|
584
|
+
if normalized:
|
|
585
|
+
words.append(normalized)
|
|
586
|
+
return bool(words) and all(word in SUMMARY_PLACEHOLDER_WORDS for word in words)
|
|
587
|
+
|
|
588
|
+
|
|
589
|
+
def _summary_payload_validation_errors(payload):
|
|
590
|
+
errors = []
|
|
591
|
+
if not isinstance(payload, dict):
|
|
592
|
+
return ["summary output must be an object"]
|
|
593
|
+
for field in ("decisions", "shared_decisions"):
|
|
594
|
+
for index, item in enumerate(payload.get(field) or []):
|
|
595
|
+
if isinstance(item, str):
|
|
596
|
+
summary = item.strip()
|
|
597
|
+
elif isinstance(item, dict):
|
|
598
|
+
summary = str(item.get("summary") or item.get("decision") or "").strip()
|
|
599
|
+
else:
|
|
600
|
+
summary = ""
|
|
601
|
+
if not summary:
|
|
602
|
+
errors.append(f"{field}[{index}] requires a non-empty summary")
|
|
603
|
+
elif _looks_like_placeholder_text(summary):
|
|
604
|
+
errors.append(f"{field}[{index}] contains schema placeholder text")
|
|
605
|
+
for field in ("risks", "glossary", "shared_context", "shared_sources"):
|
|
606
|
+
for index, item in enumerate(payload.get(field) or []):
|
|
607
|
+
if not isinstance(item, str) or not item.strip():
|
|
608
|
+
errors.append(f"{field}[{index}] must be a non-empty string")
|
|
609
|
+
elif _looks_like_placeholder_text(item):
|
|
610
|
+
errors.append(f"{field}[{index}] contains schema placeholder text")
|
|
611
|
+
for index, item in enumerate(payload.get("file_map") or []):
|
|
612
|
+
paths = item.get("paths") if isinstance(item, dict) else None
|
|
613
|
+
if not isinstance(item, dict) or not str(item.get("label") or "").strip() or not paths:
|
|
614
|
+
errors.append(f"file_map[{index}] requires label and paths")
|
|
615
|
+
return errors
|
|
616
|
+
|
|
617
|
+
|
|
618
|
+
def _semantic_action_count(apply_result):
|
|
619
|
+
actions = apply_result.get("actions") if isinstance(apply_result, dict) else []
|
|
620
|
+
return sum(
|
|
621
|
+
1
|
|
622
|
+
for action in (actions or [])
|
|
623
|
+
if isinstance(action, dict) and not str(action.get("op") or "").startswith("prune-")
|
|
624
|
+
)
|
|
625
|
+
|
|
626
|
+
|
|
627
|
+
def _executor_args(name, preset, prompt):
|
|
628
|
+
command = shlex.split(preset.get("command", name))
|
|
629
|
+
model = preset.get("model")
|
|
630
|
+
profile = preset.get("profile")
|
|
631
|
+
extra = [str(item) for item in (preset.get("extra_args") or [])]
|
|
632
|
+
if name == "coco":
|
|
633
|
+
args = command + ["-p", "--yolo", "--output-format", "json"]
|
|
634
|
+
if model:
|
|
635
|
+
args += ["-c", f"model={model}"]
|
|
636
|
+
if profile:
|
|
637
|
+
args += ["-c", f"profile={profile}"]
|
|
638
|
+
args += extra + [prompt]
|
|
639
|
+
elif name == "codex":
|
|
640
|
+
args = command + [
|
|
641
|
+
"exec", "--ephemeral", "--json", "--ignore-rules",
|
|
642
|
+
"--skip-git-repo-check", "--sandbox", "danger-full-access",
|
|
643
|
+
]
|
|
644
|
+
if model:
|
|
645
|
+
args += ["--model", str(model)]
|
|
646
|
+
if profile:
|
|
647
|
+
args += ["--profile", str(profile)]
|
|
648
|
+
args += extra + [prompt]
|
|
649
|
+
elif name == "claude":
|
|
650
|
+
args = command + ["-p", "--permission-mode", "bypassPermissions", "--output-format", "json", "--no-session-persistence"]
|
|
651
|
+
if model:
|
|
652
|
+
args += ["--model", str(model)]
|
|
653
|
+
args += extra + [prompt]
|
|
654
|
+
else:
|
|
655
|
+
args = command + extra
|
|
656
|
+
args = [part.replace("{prompt}", prompt) for part in args]
|
|
657
|
+
if not any("{prompt}" in part for part in command + extra):
|
|
658
|
+
args.append(prompt)
|
|
659
|
+
return args
|
|
660
|
+
|
|
661
|
+
|
|
662
|
+
def _find_json_object(text):
|
|
663
|
+
text = (text or "").strip()
|
|
664
|
+
try:
|
|
665
|
+
value = json.loads(text)
|
|
666
|
+
if isinstance(value, dict):
|
|
667
|
+
return value
|
|
668
|
+
except json.JSONDecodeError:
|
|
669
|
+
pass
|
|
670
|
+
decoder = json.JSONDecoder()
|
|
671
|
+
for match in re.finditer(r"\{", text):
|
|
672
|
+
try:
|
|
673
|
+
value, _ = decoder.raw_decode(text[match.start():])
|
|
674
|
+
if isinstance(value, dict):
|
|
675
|
+
return value
|
|
676
|
+
except json.JSONDecodeError:
|
|
677
|
+
continue
|
|
678
|
+
raise ValueError("executor output contains no JSON object")
|
|
679
|
+
|
|
680
|
+
|
|
681
|
+
def _walk_dicts(value):
|
|
682
|
+
if isinstance(value, dict):
|
|
683
|
+
yield value
|
|
684
|
+
for child in value.values():
|
|
685
|
+
yield from _walk_dicts(child)
|
|
686
|
+
elif isinstance(value, list):
|
|
687
|
+
for child in value:
|
|
688
|
+
yield from _walk_dicts(child)
|
|
689
|
+
|
|
690
|
+
|
|
691
|
+
def _parse_executor_output(name, stdout, stderr):
|
|
692
|
+
records = []
|
|
693
|
+
for line in stdout.splitlines():
|
|
694
|
+
try:
|
|
695
|
+
records.append(json.loads(line))
|
|
696
|
+
except json.JSONDecodeError:
|
|
697
|
+
continue
|
|
698
|
+
if not records:
|
|
699
|
+
try:
|
|
700
|
+
records = [json.loads(stdout)]
|
|
701
|
+
except json.JSONDecodeError:
|
|
702
|
+
records = []
|
|
703
|
+
usage = None
|
|
704
|
+
session_id = None
|
|
705
|
+
candidates = []
|
|
706
|
+
for record in records:
|
|
707
|
+
for item in _walk_dicts(record):
|
|
708
|
+
for key in ("usage", "token_usage", "total_token_usage"):
|
|
709
|
+
parsed_usage = _usage_dict(item.get(key))
|
|
710
|
+
if parsed_usage:
|
|
711
|
+
usage = parsed_usage
|
|
712
|
+
session_id = session_id or item.get("thread_id") or item.get("session_id")
|
|
713
|
+
for key in ("result", "text", "output_text"):
|
|
714
|
+
if isinstance(item.get(key), str):
|
|
715
|
+
candidates.append(item[key])
|
|
716
|
+
if item.get("type") == "agent_message" and isinstance(item.get("text"), str):
|
|
717
|
+
candidates.append(item["text"])
|
|
718
|
+
last_error = None
|
|
719
|
+
for candidate in reversed(candidates):
|
|
720
|
+
try:
|
|
721
|
+
return _find_json_object(candidate), usage, session_id
|
|
722
|
+
except ValueError as exc:
|
|
723
|
+
last_error = exc
|
|
724
|
+
for candidate in (stdout, stderr):
|
|
725
|
+
try:
|
|
726
|
+
value = json.loads(candidate)
|
|
727
|
+
if isinstance(value, dict):
|
|
728
|
+
for key in ("result", "text", "output_text"):
|
|
729
|
+
if isinstance(value.get(key), str):
|
|
730
|
+
return _find_json_object(value[key]), usage, session_id
|
|
731
|
+
return value, usage, session_id
|
|
732
|
+
except json.JSONDecodeError:
|
|
733
|
+
try:
|
|
734
|
+
return _find_json_object(candidate), usage, session_id
|
|
735
|
+
except ValueError as exc:
|
|
736
|
+
last_error = exc
|
|
737
|
+
raise last_error or ValueError(f"unable to parse {name} output")
|
|
738
|
+
|
|
739
|
+
|
|
740
|
+
def run_executor(name, preset, prompt, cwd, run_id, invocation):
|
|
741
|
+
args = _executor_args(name, preset, prompt)
|
|
742
|
+
env = dict(os.environ)
|
|
743
|
+
env.update({str(k): str(v) for k, v in (preset.get("env") or {}).items()})
|
|
744
|
+
started = time.time()
|
|
745
|
+
timeout_seconds = int(preset.get("_timeout_seconds", 360))
|
|
746
|
+
record = {
|
|
747
|
+
"invocation": invocation,
|
|
748
|
+
"stage": "final" if ":final" in invocation else "partial",
|
|
749
|
+
"executor": name,
|
|
750
|
+
"model": preset.get("model"),
|
|
751
|
+
"profile": preset.get("profile"),
|
|
752
|
+
"prompt_chars": len(prompt),
|
|
753
|
+
"started_at": dt.datetime.fromtimestamp(started, dt.timezone.utc).isoformat(),
|
|
754
|
+
"duration_ms": 0,
|
|
755
|
+
"returncode": None,
|
|
756
|
+
"usage_status": "unavailable",
|
|
757
|
+
}
|
|
758
|
+
try:
|
|
759
|
+
result = subprocess.run(
|
|
760
|
+
args,
|
|
761
|
+
cwd=cwd,
|
|
762
|
+
env=env,
|
|
763
|
+
capture_output=True,
|
|
764
|
+
text=True,
|
|
765
|
+
check=False,
|
|
766
|
+
timeout=timeout_seconds,
|
|
767
|
+
)
|
|
768
|
+
except subprocess.TimeoutExpired:
|
|
769
|
+
record.update({
|
|
770
|
+
"duration_ms": int((time.time() - started) * 1000),
|
|
771
|
+
"returncode": None,
|
|
772
|
+
"timed_out": True,
|
|
773
|
+
"timeout_seconds": timeout_seconds,
|
|
774
|
+
"error": f"executor timed out after {timeout_seconds} seconds",
|
|
775
|
+
})
|
|
776
|
+
return None, record
|
|
777
|
+
record["duration_ms"] = int((time.time() - started) * 1000)
|
|
778
|
+
record["returncode"] = result.returncode
|
|
779
|
+
if result.returncode != 0:
|
|
780
|
+
record["error"] = (result.stderr or result.stdout or "executor failed")[-4000:]
|
|
781
|
+
return None, record
|
|
782
|
+
try:
|
|
783
|
+
payload, usage, session_id = _parse_executor_output(name, result.stdout, result.stderr)
|
|
784
|
+
record["output"] = _summary_payload_meta(payload)
|
|
785
|
+
record["usage"] = usage
|
|
786
|
+
record["usage_status"] = "reported" if usage else "unavailable"
|
|
787
|
+
record["internal_session_id"] = session_id
|
|
788
|
+
if session_id:
|
|
789
|
+
_append_jsonl(SCAN_ROOT / "internal-sessions.jsonl", {
|
|
790
|
+
"at": now_iso(), "run_id": run_id, "executor": name, "session_id": session_id,
|
|
791
|
+
})
|
|
792
|
+
return payload, record
|
|
793
|
+
except Exception as exc:
|
|
794
|
+
record["error"] = str(exc)
|
|
795
|
+
return None, record
|
|
796
|
+
|
|
797
|
+
|
|
798
|
+
def run_executor_with_retries(name, preset, prompt, cwd, run_id, invocation, max_attempts):
|
|
799
|
+
records = []
|
|
800
|
+
payload = None
|
|
801
|
+
for attempt in range(1, max(1, max_attempts) + 1):
|
|
802
|
+
attempt_prompt = prompt
|
|
803
|
+
if attempt > 1:
|
|
804
|
+
attempt_prompt += "\n上一次调用失败。请重新阅读输入,只输出一个合法 JSON 对象。"
|
|
805
|
+
payload, record = run_executor(
|
|
806
|
+
name, preset, attempt_prompt, cwd, run_id, f"{invocation}:attempt:{attempt}"
|
|
807
|
+
)
|
|
808
|
+
record["attempt"] = attempt
|
|
809
|
+
records.append(record)
|
|
810
|
+
if payload is not None:
|
|
811
|
+
break
|
|
812
|
+
if record.get("timed_out"):
|
|
813
|
+
break
|
|
814
|
+
return payload, records
|
|
815
|
+
|
|
816
|
+
|
|
817
|
+
def _apply_summary(target, payload):
|
|
818
|
+
result = subprocess.run(
|
|
819
|
+
[sys.executable, str(PACKAGE_ROOT / "lib" / "dev_memory_capture.py"), "apply-summary-output", "--repo", target["repo_root"], "--json", json.dumps(payload, ensure_ascii=False)],
|
|
820
|
+
cwd=target["repo_root"], capture_output=True, text=True, check=False,
|
|
821
|
+
)
|
|
822
|
+
if result.returncode not in (0, 2):
|
|
823
|
+
raise RuntimeError((result.stderr or result.stdout or "apply-summary-output failed").strip())
|
|
824
|
+
try:
|
|
825
|
+
return json.loads(result.stdout)
|
|
826
|
+
except json.JSONDecodeError:
|
|
827
|
+
return {"raw": result.stdout.strip()}
|
|
828
|
+
|
|
829
|
+
|
|
830
|
+
def _sum_usage(invocations):
|
|
831
|
+
total = {}
|
|
832
|
+
unavailable = 0
|
|
833
|
+
for item in invocations:
|
|
834
|
+
usage = item.get("usage")
|
|
835
|
+
if not isinstance(usage, dict):
|
|
836
|
+
unavailable += 1
|
|
837
|
+
continue
|
|
838
|
+
for key, value in usage.items():
|
|
839
|
+
if isinstance(value, int):
|
|
840
|
+
total[key] = total.get(key, 0) + value
|
|
841
|
+
return total or None, unavailable
|
|
842
|
+
|
|
843
|
+
|
|
844
|
+
def _internal_ids():
|
|
845
|
+
ids = set()
|
|
846
|
+
path = SCAN_ROOT / "internal-sessions.jsonl"
|
|
847
|
+
try:
|
|
848
|
+
for line in path.read_text(encoding="utf-8").splitlines():
|
|
849
|
+
value = json.loads(line)
|
|
850
|
+
if value.get("session_id"):
|
|
851
|
+
ids.add(value["session_id"])
|
|
852
|
+
except (OSError, json.JSONDecodeError):
|
|
853
|
+
pass
|
|
854
|
+
return ids
|
|
855
|
+
|
|
856
|
+
|
|
857
|
+
def discover(config, since=None):
|
|
858
|
+
origin = _read_json(SCAN_ROOT / "scan-origin.json", {}) or {}
|
|
859
|
+
first_run = not origin
|
|
860
|
+
lookback = config.get("first_lookback_days", 3) if first_run and since is None else None
|
|
861
|
+
cutoff_epoch = origin.get("initial_cutoff_epoch") if origin and since is None else None
|
|
862
|
+
if since:
|
|
863
|
+
parsed = dt.datetime.fromisoformat(since.replace("Z", "+00:00"))
|
|
864
|
+
cutoff_epoch = parsed.astimezone(dt.timezone.utc).timestamp()
|
|
865
|
+
lookback = None
|
|
866
|
+
internal_ids = _internal_ids()
|
|
867
|
+
sessions = []
|
|
868
|
+
idle_seconds = int(config.get("idle_minutes", 60)) * 60
|
|
869
|
+
source_by_session = {}
|
|
870
|
+
for path in _session_files(lookback, cutoff_epoch=cutoff_epoch):
|
|
871
|
+
meta = _codex_session_meta(path)
|
|
872
|
+
session_id = meta.get("session_id") or meta.get("id") or path.stem
|
|
873
|
+
previous = source_by_session.get(session_id)
|
|
874
|
+
if previous is None or (path.stat().st_size, path.stat().st_mtime) > (
|
|
875
|
+
previous[0].stat().st_size, previous[0].stat().st_mtime
|
|
876
|
+
):
|
|
877
|
+
source_by_session[session_id] = (path, meta)
|
|
878
|
+
for session_id, (path, meta) in sorted(
|
|
879
|
+
source_by_session.items(), key=lambda item: (item[1][0].stat().st_mtime, str(item[1][0]))
|
|
880
|
+
):
|
|
881
|
+
stat = path.stat()
|
|
882
|
+
state = _read_json(_state_path(session_id), {}) or {}
|
|
883
|
+
cursor = int(state.get("processed_offset", 0))
|
|
884
|
+
if cursor and stat.st_size <= cursor:
|
|
885
|
+
parsed = {
|
|
886
|
+
"path": str(path),
|
|
887
|
+
"size": stat.st_size,
|
|
888
|
+
"mtime": dt.datetime.fromtimestamp(stat.st_mtime, dt.timezone.utc).isoformat(),
|
|
889
|
+
"end_offset": cursor,
|
|
890
|
+
"meta": meta,
|
|
891
|
+
"messages": [],
|
|
892
|
+
"session_usage": state.get("session_usage"),
|
|
893
|
+
"internal_marker": bool(state.get("internal_marker")),
|
|
894
|
+
}
|
|
895
|
+
else:
|
|
896
|
+
parsed = parse_codex_session(path, cursor)
|
|
897
|
+
reason = None
|
|
898
|
+
if session_id in internal_ids or str(session_id).startswith("dev-memory-summary-") or parsed["internal_marker"]:
|
|
899
|
+
reason = "excluded_internal"
|
|
900
|
+
elif meta.get("thread_source") == "automation":
|
|
901
|
+
reason = "excluded_automation"
|
|
902
|
+
elif time.time() - path.stat().st_mtime < idle_seconds:
|
|
903
|
+
reason = "not_idle"
|
|
904
|
+
elif parsed["size"] <= cursor:
|
|
905
|
+
reason = "unchanged"
|
|
906
|
+
target, target_error = resolve_target(meta.get("cwd"))
|
|
907
|
+
if not reason and target_error:
|
|
908
|
+
reason = target_error
|
|
909
|
+
sessions.append({
|
|
910
|
+
**parsed,
|
|
911
|
+
"session_id": session_id,
|
|
912
|
+
"cursor_before": cursor,
|
|
913
|
+
"new_bytes": max(0, parsed["size"] - cursor),
|
|
914
|
+
"target": target,
|
|
915
|
+
"status": "candidate" if not reason else "skipped",
|
|
916
|
+
"reason": reason,
|
|
917
|
+
})
|
|
918
|
+
return sessions
|
|
919
|
+
|
|
920
|
+
|
|
921
|
+
def _persist_scan_run(run, event):
|
|
922
|
+
run_id = run["run_id"]
|
|
923
|
+
_atomic_json(SCAN_ROOT / "runs" / f"{run_id}.json", run)
|
|
924
|
+
_atomic_json(SCAN_ROOT / "last-run.json", run)
|
|
925
|
+
_append_jsonl(SCAN_ROOT / "events.jsonl", {"at": now_iso(), "event": event, "run_id": run_id})
|
|
926
|
+
|
|
927
|
+
|
|
928
|
+
def _skipped_activity_run(run_id, started, activity, *, dry_run=False):
|
|
929
|
+
status = "skipped_active" if activity["status"] == "active" else "skipped_activity_unknown"
|
|
930
|
+
return {
|
|
931
|
+
"schema_version": SCHEMA_VERSION,
|
|
932
|
+
"run_id": run_id,
|
|
933
|
+
"status": status,
|
|
934
|
+
"skip_reason": activity["status"],
|
|
935
|
+
"started_at": dt.datetime.fromtimestamp(started, dt.timezone.utc).isoformat(),
|
|
936
|
+
"finished_at": now_iso(),
|
|
937
|
+
"duration_ms": int((time.time() - started) * 1000),
|
|
938
|
+
"scheduled": True,
|
|
939
|
+
"dry_run": bool(dry_run),
|
|
940
|
+
"activity": activity,
|
|
941
|
+
"executor": None,
|
|
942
|
+
"model": None,
|
|
943
|
+
"profile": None,
|
|
944
|
+
"session_count": 0,
|
|
945
|
+
"candidate_count": 0,
|
|
946
|
+
"done_count": 0,
|
|
947
|
+
"summary_skipped_count": 0,
|
|
948
|
+
"failed_count": 0,
|
|
949
|
+
"skipped_count": 0,
|
|
950
|
+
"discovery_skipped_count": 0,
|
|
951
|
+
"raw_bytes": 0,
|
|
952
|
+
"new_bytes": 0,
|
|
953
|
+
"observed_new_bytes": 0,
|
|
954
|
+
"eligible_new_bytes": 0,
|
|
955
|
+
"skipped_new_bytes": 0,
|
|
956
|
+
"semantic_messages": 0,
|
|
957
|
+
"semantic_chars": 0,
|
|
958
|
+
"summary_usage": None,
|
|
959
|
+
"usage_unavailable_invocations": 0,
|
|
960
|
+
"invocations": [],
|
|
961
|
+
"sessions": [],
|
|
962
|
+
}
|
|
963
|
+
|
|
964
|
+
|
|
965
|
+
def _advance_session_state(session):
|
|
966
|
+
state_path = _state_path(session["session_id"])
|
|
967
|
+
existing = _read_json(state_path, {}) or {}
|
|
968
|
+
existing_offset = int(existing.get("processed_offset", 0))
|
|
969
|
+
if existing_offset >= session["end_offset"]:
|
|
970
|
+
return existing_offset
|
|
971
|
+
_atomic_json(state_path, {
|
|
972
|
+
"schema_version": SCHEMA_VERSION,
|
|
973
|
+
"session_id": session["session_id"],
|
|
974
|
+
"path": session["path"],
|
|
975
|
+
"processed_offset": session["end_offset"],
|
|
976
|
+
"raw_size": session["size"],
|
|
977
|
+
"sha256": hashlib.sha256(Path(session["path"]).read_bytes()).hexdigest(),
|
|
978
|
+
"session_usage": session["session_usage"],
|
|
979
|
+
"internal_marker": session["internal_marker"],
|
|
980
|
+
"repo_key": session["target"]["repo_key"],
|
|
981
|
+
"branch": session["target"]["branch"],
|
|
982
|
+
"updated_at": now_iso(),
|
|
983
|
+
})
|
|
984
|
+
return session["end_offset"]
|
|
985
|
+
|
|
986
|
+
|
|
987
|
+
def _execute_sessions(config, args, sessions, run_id, started, *, activity=None, run_kind="scan", replay_source=None):
|
|
988
|
+
candidates = [item for item in sessions if item["status"] == "candidate" and item["messages"]]
|
|
989
|
+
executor_name = None
|
|
990
|
+
preset = None
|
|
991
|
+
if candidates and not args.dry_run:
|
|
992
|
+
executor_override = getattr(args, "executor", None)
|
|
993
|
+
executor_config = {**config, "executor": executor_override} if executor_override else config
|
|
994
|
+
executor_name, preset = choose_executor(executor_config)
|
|
995
|
+
preset = dict(preset)
|
|
996
|
+
preset["_timeout_seconds"] = int(config.get("invocation_timeout_seconds", 360))
|
|
997
|
+
invocations = []
|
|
998
|
+
results = []
|
|
999
|
+
for session in sessions:
|
|
1000
|
+
audit = {
|
|
1001
|
+
"schema_version": SCHEMA_VERSION,
|
|
1002
|
+
"session_id": session["session_id"],
|
|
1003
|
+
"path": session["path"],
|
|
1004
|
+
"originator": session["meta"].get("originator"),
|
|
1005
|
+
"source": session["meta"].get("source"),
|
|
1006
|
+
"thread_source": session["meta"].get("thread_source"),
|
|
1007
|
+
"cwd": session["meta"].get("cwd"),
|
|
1008
|
+
"raw_size": session["size"],
|
|
1009
|
+
"cursor_before": session["cursor_before"],
|
|
1010
|
+
"new_bytes": session["new_bytes"],
|
|
1011
|
+
"semantic_messages": len(session["messages"]),
|
|
1012
|
+
"semantic_chars": sum(len(item["text"]) for item in session["messages"]),
|
|
1013
|
+
"session_usage": session["session_usage"],
|
|
1014
|
+
"repo_key": (session["target"] or {}).get("repo_key"),
|
|
1015
|
+
"branch": (session["target"] or {}).get("branch"),
|
|
1016
|
+
"status": session["status"],
|
|
1017
|
+
"reason": session["reason"],
|
|
1018
|
+
"last_scanned_at": now_iso(),
|
|
1019
|
+
}
|
|
1020
|
+
if session.get("replay_source"):
|
|
1021
|
+
audit["replay_source"] = session["replay_source"]
|
|
1022
|
+
if session not in candidates:
|
|
1023
|
+
if audit["status"] == "candidate":
|
|
1024
|
+
audit["status"] = "skipped"
|
|
1025
|
+
audit["reason"] = "no_semantic_messages"
|
|
1026
|
+
results.append(audit)
|
|
1027
|
+
_atomic_json(_session_audit_path(session["session_id"]), audit)
|
|
1028
|
+
continue
|
|
1029
|
+
chunks = _chunk_messages(session["messages"], int(config.get("chunk_chars", 60000)))
|
|
1030
|
+
audit["chunk_count"] = len(chunks)
|
|
1031
|
+
audit["chunks"] = [
|
|
1032
|
+
{
|
|
1033
|
+
"index": index,
|
|
1034
|
+
"start_offset": chunk[0]["start_offset"],
|
|
1035
|
+
"end_offset": chunk[-1]["end_offset"],
|
|
1036
|
+
"messages": len(chunk),
|
|
1037
|
+
"chars": sum(len(item["text"]) for item in chunk),
|
|
1038
|
+
"sha256": hashlib.sha256("\n".join(item["text"] for item in chunk).encode()).hexdigest(),
|
|
1039
|
+
}
|
|
1040
|
+
for index, chunk in enumerate(chunks, 1)
|
|
1041
|
+
]
|
|
1042
|
+
if args.dry_run:
|
|
1043
|
+
audit["status"] = "dry_run"
|
|
1044
|
+
results.append(audit)
|
|
1045
|
+
_atomic_json(_session_audit_path(session["session_id"]), audit)
|
|
1046
|
+
continue
|
|
1047
|
+
failed = None
|
|
1048
|
+
invocation_start = len(invocations)
|
|
1049
|
+
final_payload = None
|
|
1050
|
+
if len(chunks) == 1:
|
|
1051
|
+
final_payload, attempt_records = run_executor_with_retries(
|
|
1052
|
+
executor_name, preset, _single_prompt(session["target"], chunks[0]),
|
|
1053
|
+
session["target"]["repo_root"], run_id, f"{session['session_id']}:final",
|
|
1054
|
+
int(config.get("max_attempts", 2)),
|
|
1055
|
+
)
|
|
1056
|
+
invocations.extend(attempt_records)
|
|
1057
|
+
if final_payload is None:
|
|
1058
|
+
failed = attempt_records[-1].get("error", "final summary failed")
|
|
1059
|
+
else:
|
|
1060
|
+
partials = []
|
|
1061
|
+
for index, chunk in enumerate(chunks, 1):
|
|
1062
|
+
payload, attempt_records = run_executor_with_retries(
|
|
1063
|
+
executor_name, preset, _partial_prompt(session["target"], chunk, index, len(chunks)),
|
|
1064
|
+
session["target"]["repo_root"], run_id, f"{session['session_id']}:chunk:{index}",
|
|
1065
|
+
int(config.get("max_attempts", 2)),
|
|
1066
|
+
)
|
|
1067
|
+
invocations.extend(attempt_records)
|
|
1068
|
+
if payload is None:
|
|
1069
|
+
failed = attempt_records[-1].get("error", "chunk summary failed")
|
|
1070
|
+
break
|
|
1071
|
+
partials.append(payload)
|
|
1072
|
+
if not failed:
|
|
1073
|
+
final_payload, attempt_records = run_executor_with_retries(
|
|
1074
|
+
executor_name, preset, _final_prompt(session["target"], partials),
|
|
1075
|
+
session["target"]["repo_root"], run_id, f"{session['session_id']}:final",
|
|
1076
|
+
int(config.get("max_attempts", 2)),
|
|
1077
|
+
)
|
|
1078
|
+
invocations.extend(attempt_records)
|
|
1079
|
+
if final_payload is None:
|
|
1080
|
+
failed = attempt_records[-1].get("error", "final summary failed")
|
|
1081
|
+
if not failed:
|
|
1082
|
+
summary_output = _summary_payload_meta(final_payload)
|
|
1083
|
+
validation_errors = _summary_payload_validation_errors(final_payload)
|
|
1084
|
+
if validation_errors:
|
|
1085
|
+
summary_output["validation_errors"] = validation_errors
|
|
1086
|
+
audit["summary_output"] = summary_output
|
|
1087
|
+
if validation_errors:
|
|
1088
|
+
failed = "invalid final summary output: " + "; ".join(validation_errors)
|
|
1089
|
+
elif not summary_output["mutation_count"] and not summary_output["skip_reason"]:
|
|
1090
|
+
failed = "final summary output has no memory mutations or skip_reason"
|
|
1091
|
+
if not failed and audit["summary_output"]["mutation_count"]:
|
|
1092
|
+
try:
|
|
1093
|
+
audit["apply_result"] = _apply_summary(session["target"], final_payload)
|
|
1094
|
+
audit["semantic_action_count"] = _semantic_action_count(audit["apply_result"])
|
|
1095
|
+
if not audit["semantic_action_count"]:
|
|
1096
|
+
if audit["summary_output"]["skip_reason"]:
|
|
1097
|
+
audit["status"] = "skipped_summary"
|
|
1098
|
+
else:
|
|
1099
|
+
failed = "summary output produced no semantic memory actions or skip_reason"
|
|
1100
|
+
else:
|
|
1101
|
+
audit["status"] = "done"
|
|
1102
|
+
except Exception as exc:
|
|
1103
|
+
failed = str(exc)
|
|
1104
|
+
elif not failed:
|
|
1105
|
+
audit["status"] = "skipped_summary"
|
|
1106
|
+
audit["semantic_action_count"] = 0
|
|
1107
|
+
audit["apply_result"] = {
|
|
1108
|
+
"mode": "not-applied",
|
|
1109
|
+
"touched_targets": [],
|
|
1110
|
+
"actions": [],
|
|
1111
|
+
"skip_reason": audit["summary_output"]["skip_reason"],
|
|
1112
|
+
}
|
|
1113
|
+
if not failed and audit["status"] in {"done", "skipped_summary"}:
|
|
1114
|
+
audit["cursor_after"] = session["end_offset"]
|
|
1115
|
+
audit["state_offset_after"] = _advance_session_state(session)
|
|
1116
|
+
if failed:
|
|
1117
|
+
audit["status"] = "failed"
|
|
1118
|
+
audit["error"] = failed
|
|
1119
|
+
audit["summary_usage"], audit["usage_unavailable_invocations"] = _sum_usage(invocations[invocation_start:])
|
|
1120
|
+
audit["executor"] = executor_name
|
|
1121
|
+
audit["model"] = preset.get("model")
|
|
1122
|
+
results.append(audit)
|
|
1123
|
+
_atomic_json(_session_audit_path(session["session_id"]), audit)
|
|
1124
|
+
usage, unavailable = _sum_usage(invocations)
|
|
1125
|
+
run = {
|
|
1126
|
+
"schema_version": SCHEMA_VERSION,
|
|
1127
|
+
"run_id": run_id,
|
|
1128
|
+
"run_kind": run_kind,
|
|
1129
|
+
"started_at": dt.datetime.fromtimestamp(started, dt.timezone.utc).isoformat(),
|
|
1130
|
+
"finished_at": now_iso(),
|
|
1131
|
+
"duration_ms": int((time.time() - started) * 1000),
|
|
1132
|
+
"dry_run": bool(args.dry_run),
|
|
1133
|
+
"scheduled": bool(getattr(args, "scheduled", False)),
|
|
1134
|
+
"activity": activity,
|
|
1135
|
+
"executor": executor_name,
|
|
1136
|
+
"model": preset.get("model") if preset else None,
|
|
1137
|
+
"profile": preset.get("profile") if preset else None,
|
|
1138
|
+
"session_count": len(results),
|
|
1139
|
+
"candidate_count": len(candidates),
|
|
1140
|
+
"done_count": sum(item["status"] == "done" for item in results),
|
|
1141
|
+
"summary_skipped_count": sum(item["status"] == "skipped_summary" for item in results),
|
|
1142
|
+
"failed_count": sum(item["status"] == "failed" for item in results),
|
|
1143
|
+
"skipped_count": sum(item["status"] in {"skipped", "skipped_summary"} for item in results),
|
|
1144
|
+
"discovery_skipped_count": sum(item["status"] == "skipped" for item in results),
|
|
1145
|
+
"raw_bytes": sum(item["raw_size"] for item in results),
|
|
1146
|
+
"new_bytes": sum(item["new_bytes"] for item in results),
|
|
1147
|
+
"observed_new_bytes": sum(item["new_bytes"] for item in results),
|
|
1148
|
+
"eligible_new_bytes": sum(item["new_bytes"] for item in results if item["status"] != "skipped"),
|
|
1149
|
+
"skipped_new_bytes": sum(item["new_bytes"] for item in results if item["status"] == "skipped"),
|
|
1150
|
+
"semantic_messages": sum(item["semantic_messages"] for item in results),
|
|
1151
|
+
"semantic_chars": sum(item["semantic_chars"] for item in results),
|
|
1152
|
+
"summary_usage": usage,
|
|
1153
|
+
"usage_unavailable_invocations": unavailable,
|
|
1154
|
+
"invocations": invocations,
|
|
1155
|
+
"sessions": results,
|
|
1156
|
+
}
|
|
1157
|
+
if replay_source:
|
|
1158
|
+
run["replay_source"] = replay_source
|
|
1159
|
+
_persist_scan_run(run, "scan_completed")
|
|
1160
|
+
print(json.dumps(run, ensure_ascii=False, indent=2) if args.json else _format_run(run))
|
|
1161
|
+
return 1 if run["failed_count"] else 0
|
|
1162
|
+
|
|
1163
|
+
|
|
1164
|
+
def run_scan(args):
|
|
1165
|
+
config = load_config()
|
|
1166
|
+
validation = validate_config(config)
|
|
1167
|
+
if not validation["valid"]:
|
|
1168
|
+
raise RuntimeError("; ".join(validation["errors"]))
|
|
1169
|
+
run_id = dt.datetime.now().strftime("%Y%m%dT%H%M%S") + f"-{os.getpid()}"
|
|
1170
|
+
started = time.time()
|
|
1171
|
+
activity = None
|
|
1172
|
+
if getattr(args, "scheduled", False) and config.get("skip_when_computer_active", True):
|
|
1173
|
+
activity = computer_activity(config)
|
|
1174
|
+
if activity["skip"]:
|
|
1175
|
+
run = _skipped_activity_run(run_id, started, activity, dry_run=args.dry_run)
|
|
1176
|
+
_persist_scan_run(run, run["status"])
|
|
1177
|
+
print(json.dumps(run, ensure_ascii=False, indent=2) if args.json else _format_run(run))
|
|
1178
|
+
return 0
|
|
1179
|
+
origin_path = SCAN_ROOT / "scan-origin.json"
|
|
1180
|
+
if not origin_path.exists():
|
|
1181
|
+
lookback_days = int(config.get("first_lookback_days", 3))
|
|
1182
|
+
_atomic_json(origin_path, {
|
|
1183
|
+
"created_at": now_iso(),
|
|
1184
|
+
"initial_cutoff_epoch": time.time() - lookback_days * 86400,
|
|
1185
|
+
"first_lookback_days": lookback_days,
|
|
1186
|
+
})
|
|
1187
|
+
sessions = discover(config, args.since)
|
|
1188
|
+
return _execute_sessions(config, args, sessions, run_id, started, activity=activity)
|
|
1189
|
+
|
|
1190
|
+
|
|
1191
|
+
def _format_tokens(usage):
|
|
1192
|
+
return str((usage or {}).get("total_tokens", "unavailable"))
|
|
1193
|
+
|
|
1194
|
+
|
|
1195
|
+
def _format_run(run):
|
|
1196
|
+
if str(run.get("status", "")).startswith("skipped_"):
|
|
1197
|
+
activity = run.get("activity") or {}
|
|
1198
|
+
return (
|
|
1199
|
+
f"run {run['run_id']}: {run['status']}; "
|
|
1200
|
+
f"idle_seconds={activity.get('idle_seconds')} threshold={activity.get('threshold_seconds')}"
|
|
1201
|
+
)
|
|
1202
|
+
return (
|
|
1203
|
+
f"run {run['run_id']}: {run['done_count']} done, "
|
|
1204
|
+
f"{run.get('summary_skipped_count', 0)} summary-skipped, {run['failed_count']} failed, "
|
|
1205
|
+
f"{run.get('discovery_skipped_count', run['skipped_count'])} discovery-skipped; "
|
|
1206
|
+
f"{run.get('eligible_new_bytes', run['new_bytes'])} eligible / "
|
|
1207
|
+
f"{run.get('observed_new_bytes', run['new_bytes'])} observed new bytes; "
|
|
1208
|
+
f"summary tokens {_format_tokens(run.get('summary_usage'))}"
|
|
1209
|
+
)
|
|
1210
|
+
|
|
1211
|
+
|
|
1212
|
+
def _runs():
|
|
1213
|
+
return [value for path in sorted((SCAN_ROOT / "runs").glob("*.json"), reverse=True) if isinstance((value := _read_json(path)), dict)]
|
|
1214
|
+
|
|
1215
|
+
|
|
1216
|
+
def _covered_bytes(intervals):
|
|
1217
|
+
total = 0
|
|
1218
|
+
end = None
|
|
1219
|
+
for start, stop in sorted(intervals):
|
|
1220
|
+
if stop <= start:
|
|
1221
|
+
continue
|
|
1222
|
+
if end is None or start > end:
|
|
1223
|
+
total += stop - start
|
|
1224
|
+
end = stop
|
|
1225
|
+
elif stop > end:
|
|
1226
|
+
total += stop - end
|
|
1227
|
+
end = stop
|
|
1228
|
+
return total
|
|
1229
|
+
|
|
1230
|
+
|
|
1231
|
+
def command_stats(args):
|
|
1232
|
+
runs = _runs()
|
|
1233
|
+
if args.since:
|
|
1234
|
+
runs = [item for item in runs if (item.get("started_at") or "") >= args.since]
|
|
1235
|
+
repos = {}
|
|
1236
|
+
total_usage = {}
|
|
1237
|
+
unavailable = 0
|
|
1238
|
+
for run in runs:
|
|
1239
|
+
usage = run.get("summary_usage") or {}
|
|
1240
|
+
for key, value in usage.items():
|
|
1241
|
+
if isinstance(value, int):
|
|
1242
|
+
total_usage[key] = total_usage.get(key, 0) + value
|
|
1243
|
+
unavailable += int(run.get("usage_unavailable_invocations", 0))
|
|
1244
|
+
for session in run.get("sessions", []):
|
|
1245
|
+
key = session.get("repo_key")
|
|
1246
|
+
if not key or (args.repo and key != args.repo):
|
|
1247
|
+
continue
|
|
1248
|
+
item = repos.setdefault(key, {
|
|
1249
|
+
"repo_key": key,
|
|
1250
|
+
"scan_count": 0,
|
|
1251
|
+
"sessions": set(),
|
|
1252
|
+
"intervals": {},
|
|
1253
|
+
"raw_bytes": 0,
|
|
1254
|
+
"new_bytes": 0,
|
|
1255
|
+
"observed_new_bytes": 0,
|
|
1256
|
+
"eligible_new_bytes": 0,
|
|
1257
|
+
"skipped_new_bytes": 0,
|
|
1258
|
+
"summary_tokens": 0,
|
|
1259
|
+
"done_count": 0,
|
|
1260
|
+
"summary_skipped_count": 0,
|
|
1261
|
+
"failed_count": 0,
|
|
1262
|
+
"empty_apply_count": 0,
|
|
1263
|
+
})
|
|
1264
|
+
item["scan_count"] += 1
|
|
1265
|
+
session_id = session.get("session_id")
|
|
1266
|
+
item["sessions"].add(session_id)
|
|
1267
|
+
item["raw_bytes"] += int(session.get("raw_size", 0))
|
|
1268
|
+
observed = int(session.get("new_bytes", 0))
|
|
1269
|
+
item["new_bytes"] += observed
|
|
1270
|
+
item["observed_new_bytes"] += observed
|
|
1271
|
+
if session.get("status") == "skipped":
|
|
1272
|
+
item["skipped_new_bytes"] += observed
|
|
1273
|
+
else:
|
|
1274
|
+
item["eligible_new_bytes"] += observed
|
|
1275
|
+
start = int(session.get("cursor_before", 0))
|
|
1276
|
+
stop = min(int(session.get("raw_size", start + observed)), start + observed)
|
|
1277
|
+
item["intervals"].setdefault(session_id, []).append((start, stop))
|
|
1278
|
+
item["summary_tokens"] += int((session.get("summary_usage") or {}).get("total_tokens", 0))
|
|
1279
|
+
status = session.get("status")
|
|
1280
|
+
item["done_count"] += int(status == "done")
|
|
1281
|
+
item["summary_skipped_count"] += int(status == "skipped_summary")
|
|
1282
|
+
item["failed_count"] += int(status == "failed")
|
|
1283
|
+
item["empty_apply_count"] += int(
|
|
1284
|
+
status == "done"
|
|
1285
|
+
and not (session.get("apply_result") or {}).get("touched_targets")
|
|
1286
|
+
)
|
|
1287
|
+
repo_rows = []
|
|
1288
|
+
for item in repos.values():
|
|
1289
|
+
item["session_count"] = len(item.pop("sessions"))
|
|
1290
|
+
item["unique_new_bytes"] = sum(_covered_bytes(value) for value in item.pop("intervals").values())
|
|
1291
|
+
repo_rows.append(item)
|
|
1292
|
+
payload = {"run_count": len(runs), "summary_usage": total_usage or None, "usage_unavailable_invocations": unavailable, "repos": sorted(repo_rows, key=lambda x: x["repo_key"])}
|
|
1293
|
+
print(json.dumps(payload, ensure_ascii=False, indent=2))
|
|
1294
|
+
|
|
1295
|
+
|
|
1296
|
+
def command_history(args):
|
|
1297
|
+
runs = _runs()
|
|
1298
|
+
if args.repo:
|
|
1299
|
+
runs = [run for run in runs if any(item.get("repo_key") == args.repo for item in run.get("sessions", []))]
|
|
1300
|
+
runs = runs[:args.limit]
|
|
1301
|
+
print(json.dumps(runs, ensure_ascii=False, indent=2) if args.json else "\n".join(_format_run(run) for run in runs))
|
|
1302
|
+
|
|
1303
|
+
|
|
1304
|
+
def command_show(args):
|
|
1305
|
+
path = SCAN_ROOT / "runs" / f"{args.run_id}.json"
|
|
1306
|
+
value = _read_json(path)
|
|
1307
|
+
if not isinstance(value, dict):
|
|
1308
|
+
raise RuntimeError(f"run not found: {args.run_id}")
|
|
1309
|
+
print(json.dumps(value, ensure_ascii=False, indent=2))
|
|
1310
|
+
|
|
1311
|
+
|
|
1312
|
+
def _replay_session(source_run_id, audit):
|
|
1313
|
+
session_id = audit.get("session_id")
|
|
1314
|
+
path = Path(audit.get("path") or "").expanduser()
|
|
1315
|
+
if not path.is_file():
|
|
1316
|
+
matches = [
|
|
1317
|
+
candidate
|
|
1318
|
+
for root in (CODEX_HOME / "sessions", CODEX_HOME / "archived_sessions")
|
|
1319
|
+
if root.exists()
|
|
1320
|
+
for candidate in root.rglob(f"*{session_id}*.jsonl")
|
|
1321
|
+
]
|
|
1322
|
+
if not matches:
|
|
1323
|
+
raise RuntimeError(f"session transcript not found: {session_id}")
|
|
1324
|
+
path = max(matches, key=lambda item: (item.stat().st_size, item.stat().st_mtime))
|
|
1325
|
+
cursor_before = int(audit.get("cursor_before", 0))
|
|
1326
|
+
cursor_after = int(audit.get("cursor_after") or audit.get("raw_size") or 0)
|
|
1327
|
+
parsed = parse_codex_session(path, cursor_before)
|
|
1328
|
+
if cursor_after <= cursor_before or cursor_after > parsed["size"]:
|
|
1329
|
+
raise RuntimeError(
|
|
1330
|
+
f"invalid replay cursor for {session_id}: {cursor_before}..{cursor_after}, size={parsed['size']}"
|
|
1331
|
+
)
|
|
1332
|
+
parsed["messages"] = [
|
|
1333
|
+
item for item in parsed["messages"]
|
|
1334
|
+
if item["start_offset"] >= cursor_before and item["end_offset"] <= cursor_after
|
|
1335
|
+
]
|
|
1336
|
+
parsed["size"] = cursor_after
|
|
1337
|
+
parsed["end_offset"] = cursor_after
|
|
1338
|
+
parsed["session_usage"] = audit.get("session_usage") or parsed.get("session_usage")
|
|
1339
|
+
meta = dict(parsed.get("meta") or {})
|
|
1340
|
+
if audit.get("cwd"):
|
|
1341
|
+
meta["cwd"] = audit["cwd"]
|
|
1342
|
+
parsed["meta"] = meta
|
|
1343
|
+
target, target_error = resolve_target(meta.get("cwd"))
|
|
1344
|
+
return {
|
|
1345
|
+
**parsed,
|
|
1346
|
+
"session_id": session_id,
|
|
1347
|
+
"cursor_before": cursor_before,
|
|
1348
|
+
"new_bytes": cursor_after - cursor_before,
|
|
1349
|
+
"target": target,
|
|
1350
|
+
"status": "candidate" if not target_error else "skipped",
|
|
1351
|
+
"reason": target_error,
|
|
1352
|
+
"replay_source": {
|
|
1353
|
+
"run_id": source_run_id,
|
|
1354
|
+
"previous_status": audit.get("status"),
|
|
1355
|
+
"previous_apply_empty": not bool((audit.get("apply_result") or {}).get("touched_targets")),
|
|
1356
|
+
},
|
|
1357
|
+
}
|
|
1358
|
+
|
|
1359
|
+
|
|
1360
|
+
def command_replay(args):
|
|
1361
|
+
config = load_config()
|
|
1362
|
+
validation = validate_config(config)
|
|
1363
|
+
if not validation["valid"]:
|
|
1364
|
+
raise RuntimeError("; ".join(validation["errors"]))
|
|
1365
|
+
source_path = SCAN_ROOT / "runs" / f"{args.run_id}.json"
|
|
1366
|
+
source_run = _read_json(source_path)
|
|
1367
|
+
if not isinstance(source_run, dict):
|
|
1368
|
+
raise RuntimeError(f"run not found: {args.run_id}")
|
|
1369
|
+
requested = list(dict.fromkeys(args.session_id))
|
|
1370
|
+
audits = {item.get("session_id"): item for item in source_run.get("sessions", [])}
|
|
1371
|
+
missing = [session_id for session_id in requested if session_id not in audits]
|
|
1372
|
+
if missing:
|
|
1373
|
+
raise RuntimeError(f"sessions not found in run {args.run_id}: {', '.join(missing)}")
|
|
1374
|
+
sessions = [_replay_session(args.run_id, audits[session_id]) for session_id in requested]
|
|
1375
|
+
run_id = dt.datetime.now().strftime("%Y%m%dT%H%M%S") + f"-replay-{os.getpid()}"
|
|
1376
|
+
return _execute_sessions(
|
|
1377
|
+
config,
|
|
1378
|
+
args,
|
|
1379
|
+
sessions,
|
|
1380
|
+
run_id,
|
|
1381
|
+
time.time(),
|
|
1382
|
+
run_kind="replay",
|
|
1383
|
+
replay_source={"run_id": args.run_id, "session_ids": requested},
|
|
1384
|
+
)
|
|
1385
|
+
|
|
1386
|
+
|
|
1387
|
+
def _cli_path():
|
|
1388
|
+
explicit = os.environ.get("DEV_MEMORY_CLI_PATH", "").strip()
|
|
1389
|
+
if explicit:
|
|
1390
|
+
return str(Path(explicit).expanduser().resolve())
|
|
1391
|
+
installed = shutil.which("dev-memory-cli")
|
|
1392
|
+
return installed or str(PACKAGE_ROOT / "bin" / "dev-memory.js")
|
|
1393
|
+
|
|
1394
|
+
|
|
1395
|
+
def command_install(_args):
|
|
1396
|
+
config = load_config()
|
|
1397
|
+
save_scan_config(config)
|
|
1398
|
+
logs = SCAN_ROOT / "logs"
|
|
1399
|
+
logs.mkdir(parents=True, exist_ok=True)
|
|
1400
|
+
plist = {
|
|
1401
|
+
"Label": "com.dev-memory.session-scan",
|
|
1402
|
+
"ProgramArguments": [_cli_path(), "session-scan", "run", "--scheduled"],
|
|
1403
|
+
"StartCalendarInterval": _calendar_intervals(config),
|
|
1404
|
+
"RunAtLoad": False,
|
|
1405
|
+
"StandardOutPath": str(logs / "launchd.stdout.log"),
|
|
1406
|
+
"StandardErrorPath": str(logs / "launchd.stderr.log"),
|
|
1407
|
+
"EnvironmentVariables": {"PATH": os.environ.get("PATH", "/usr/local/bin:/usr/bin:/bin")},
|
|
1408
|
+
}
|
|
1409
|
+
PLIST_PATH.parent.mkdir(parents=True, exist_ok=True)
|
|
1410
|
+
with PLIST_PATH.open("wb") as stream:
|
|
1411
|
+
plistlib.dump(plist, stream)
|
|
1412
|
+
subprocess.run(["launchctl", "unload", str(PLIST_PATH)], capture_output=True, check=False)
|
|
1413
|
+
result = subprocess.run(["launchctl", "load", str(PLIST_PATH)], capture_output=True, text=True, check=False)
|
|
1414
|
+
if result.returncode != 0:
|
|
1415
|
+
raise RuntimeError(result.stderr.strip() or "launchctl load failed")
|
|
1416
|
+
print(json.dumps({
|
|
1417
|
+
"installed": True,
|
|
1418
|
+
"plist": str(PLIST_PATH),
|
|
1419
|
+
"schedule_times": config["schedule_times"],
|
|
1420
|
+
"skip_when_computer_active": config["skip_when_computer_active"],
|
|
1421
|
+
"active_within_minutes": config["active_within_minutes"],
|
|
1422
|
+
"logs": str(logs),
|
|
1423
|
+
}, ensure_ascii=False, indent=2))
|
|
1424
|
+
|
|
1425
|
+
|
|
1426
|
+
def command_uninstall(_args):
|
|
1427
|
+
subprocess.run(["launchctl", "unload", str(PLIST_PATH)], capture_output=True, check=False)
|
|
1428
|
+
PLIST_PATH.unlink(missing_ok=True)
|
|
1429
|
+
print(json.dumps({"installed": False, "plist": str(PLIST_PATH)}, ensure_ascii=False, indent=2))
|
|
1430
|
+
|
|
1431
|
+
|
|
1432
|
+
def command_status(_args):
|
|
1433
|
+
config = load_config()
|
|
1434
|
+
validation = validate_config(config)
|
|
1435
|
+
try:
|
|
1436
|
+
executor, preset = choose_executor(config)
|
|
1437
|
+
except RuntimeError:
|
|
1438
|
+
executor, preset = None, None
|
|
1439
|
+
print(json.dumps({
|
|
1440
|
+
"installed": PLIST_PATH.exists(),
|
|
1441
|
+
"plist": str(PLIST_PATH),
|
|
1442
|
+
"scan_root": str(SCAN_ROOT),
|
|
1443
|
+
"codex_sessions": str(CODEX_HOME / "sessions"),
|
|
1444
|
+
"executor": executor,
|
|
1445
|
+
"model": preset.get("model") if preset else None,
|
|
1446
|
+
"schedule_times": config.get("schedule_times"),
|
|
1447
|
+
"skip_when_computer_active": config.get("skip_when_computer_active"),
|
|
1448
|
+
"active_within_minutes": config.get("active_within_minutes"),
|
|
1449
|
+
"config": validation,
|
|
1450
|
+
"last_run": _read_json(SCAN_ROOT / "last-run.json"),
|
|
1451
|
+
}, ensure_ascii=False, indent=2))
|
|
1452
|
+
|
|
1453
|
+
|
|
1454
|
+
def command_config(args):
|
|
1455
|
+
config = load_config()
|
|
1456
|
+
if args.config_command == "show":
|
|
1457
|
+
print(json.dumps(config, ensure_ascii=False, indent=2))
|
|
1458
|
+
return
|
|
1459
|
+
if args.config_command == "validate":
|
|
1460
|
+
result = validate_config(config)
|
|
1461
|
+
print(json.dumps(result, ensure_ascii=False, indent=2))
|
|
1462
|
+
if not result["valid"]:
|
|
1463
|
+
raise SystemExit(1)
|
|
1464
|
+
return
|
|
1465
|
+
if args.config_command == "set-executor":
|
|
1466
|
+
if args.executor != "auto" and args.executor not in config["executors"]:
|
|
1467
|
+
raise RuntimeError(f"unknown executor preset: {args.executor}")
|
|
1468
|
+
config["executor"] = args.executor
|
|
1469
|
+
elif args.config_command in {"set-model", "set-profile"}:
|
|
1470
|
+
if args.executor not in config["executors"]:
|
|
1471
|
+
raise RuntimeError(f"unknown executor preset: {args.executor}")
|
|
1472
|
+
field = "model" if args.config_command == "set-model" else "profile"
|
|
1473
|
+
config["executors"][args.executor][field] = args.value
|
|
1474
|
+
elif args.config_command == "set-schedule":
|
|
1475
|
+
for value in args.times:
|
|
1476
|
+
_parse_schedule_time(value)
|
|
1477
|
+
config["schedule_times"] = args.times
|
|
1478
|
+
elif args.config_command == "set-active-minutes":
|
|
1479
|
+
if args.minutes < 1:
|
|
1480
|
+
raise RuntimeError("active minutes must be at least 1")
|
|
1481
|
+
config["active_within_minutes"] = args.minutes
|
|
1482
|
+
elif args.config_command == "set-timeout":
|
|
1483
|
+
if args.seconds < 30:
|
|
1484
|
+
raise RuntimeError("timeout seconds must be at least 30")
|
|
1485
|
+
config["invocation_timeout_seconds"] = args.seconds
|
|
1486
|
+
elif args.config_command == "set-active-check":
|
|
1487
|
+
config["skip_when_computer_active"] = args.enabled == "on"
|
|
1488
|
+
save_scan_config(config)
|
|
1489
|
+
if args.config_command == "set-schedule" and PLIST_PATH.exists():
|
|
1490
|
+
command_install(args)
|
|
1491
|
+
return
|
|
1492
|
+
print(json.dumps(config, ensure_ascii=False, indent=2))
|
|
1493
|
+
|
|
1494
|
+
|
|
1495
|
+
def build_parser():
|
|
1496
|
+
parser = argparse.ArgumentParser(description="Scan Codex sessions into dev-memory")
|
|
1497
|
+
sub = parser.add_subparsers(dest="command", required=True)
|
|
1498
|
+
run = sub.add_parser("run")
|
|
1499
|
+
run.add_argument("--since")
|
|
1500
|
+
run.add_argument("--dry-run", action="store_true")
|
|
1501
|
+
run.add_argument("--json", action="store_true")
|
|
1502
|
+
run.add_argument("--scheduled", action="store_true", help="Apply computer activity guard before scanning")
|
|
1503
|
+
run.add_argument("--executor", help="Override the configured executor for this run only")
|
|
1504
|
+
sub.add_parser("install")
|
|
1505
|
+
sub.add_parser("status")
|
|
1506
|
+
stats = sub.add_parser("stats")
|
|
1507
|
+
stats.add_argument("--repo")
|
|
1508
|
+
stats.add_argument("--since")
|
|
1509
|
+
stats.add_argument("--json", action="store_true")
|
|
1510
|
+
history = sub.add_parser("history")
|
|
1511
|
+
history.add_argument("--repo")
|
|
1512
|
+
history.add_argument("--limit", type=int, default=20)
|
|
1513
|
+
history.add_argument("--json", action="store_true")
|
|
1514
|
+
show = sub.add_parser("show")
|
|
1515
|
+
show.add_argument("run_id")
|
|
1516
|
+
replay = sub.add_parser("replay")
|
|
1517
|
+
replay.add_argument("--run-id", required=True)
|
|
1518
|
+
replay.add_argument("--session-id", action="append", required=True)
|
|
1519
|
+
replay.add_argument("--dry-run", action="store_true")
|
|
1520
|
+
replay.add_argument("--json", action="store_true")
|
|
1521
|
+
replay.add_argument("--executor", help="Override the configured executor for this replay only")
|
|
1522
|
+
sub.add_parser("uninstall")
|
|
1523
|
+
config = sub.add_parser("config")
|
|
1524
|
+
config_sub = config.add_subparsers(dest="config_command", required=True)
|
|
1525
|
+
config_sub.add_parser("show")
|
|
1526
|
+
config_sub.add_parser("validate")
|
|
1527
|
+
set_executor = config_sub.add_parser("set-executor")
|
|
1528
|
+
set_executor.add_argument("executor")
|
|
1529
|
+
for command in ("set-model", "set-profile"):
|
|
1530
|
+
item = config_sub.add_parser(command)
|
|
1531
|
+
item.add_argument("executor")
|
|
1532
|
+
item.add_argument("value")
|
|
1533
|
+
set_schedule = config_sub.add_parser("set-schedule")
|
|
1534
|
+
set_schedule.add_argument("times", nargs="+")
|
|
1535
|
+
set_active_minutes = config_sub.add_parser("set-active-minutes")
|
|
1536
|
+
set_active_minutes.add_argument("minutes", type=int)
|
|
1537
|
+
set_timeout = config_sub.add_parser("set-timeout")
|
|
1538
|
+
set_timeout.add_argument("seconds", type=int)
|
|
1539
|
+
set_active_check = config_sub.add_parser("set-active-check")
|
|
1540
|
+
set_active_check.add_argument("enabled", choices=("on", "off"))
|
|
1541
|
+
return parser
|
|
1542
|
+
|
|
1543
|
+
|
|
1544
|
+
def main():
|
|
1545
|
+
args = build_parser().parse_args()
|
|
1546
|
+
handlers = {
|
|
1547
|
+
"run": run_scan,
|
|
1548
|
+
"install": command_install,
|
|
1549
|
+
"status": command_status,
|
|
1550
|
+
"stats": command_stats,
|
|
1551
|
+
"history": command_history,
|
|
1552
|
+
"show": command_show,
|
|
1553
|
+
"replay": command_replay,
|
|
1554
|
+
"uninstall": command_uninstall,
|
|
1555
|
+
"config": command_config,
|
|
1556
|
+
}
|
|
1557
|
+
try:
|
|
1558
|
+
return handlers[args.command](args) or 0
|
|
1559
|
+
except RuntimeError as exc:
|
|
1560
|
+
print(f"ERROR: {exc}", file=sys.stderr)
|
|
1561
|
+
return 1
|
|
1562
|
+
|
|
1563
|
+
|
|
1564
|
+
if __name__ == "__main__":
|
|
1565
|
+
raise SystemExit(main())
|