dev-memory-cli 0.22.1 → 0.23.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,1565 @@
1
+ #!/usr/bin/env python3
2
+
3
+ import argparse
4
+ import datetime as dt
5
+ import hashlib
6
+ import json
7
+ import os
8
+ import plistlib
9
+ import re
10
+ import shlex
11
+ import shutil
12
+ import subprocess
13
+ import sys
14
+ import tempfile
15
+ import time
16
+ from pathlib import Path
17
+
18
+ from dev_memory_common import get_branch_paths, list_repos_in_workspace, now_iso
19
+
20
+
21
+ SCHEMA_VERSION = 1
22
+ PACKAGE_ROOT = Path(__file__).resolve().parents[1]
23
+ CONFIG_PATH = Path(os.environ.get("DEV_MEMORY_CONFIG_PATH", "~/.dev-memory/config.json")).expanduser()
24
+ DEV_MEMORY_HOME = Path(os.environ.get("DEV_MEMORY_HOME", "~/.dev-memory")).expanduser()
25
+ SCAN_ROOT = Path(os.environ.get("DEV_MEMORY_SCAN_ROOT", DEV_MEMORY_HOME / "jobs" / "session-scan")).expanduser()
26
+ CODEX_HOME = Path(os.environ.get("CODEX_HOME", "~/.codex")).expanduser()
27
+ PLIST_PATH = Path(os.environ.get(
28
+ "DEV_MEMORY_SCAN_PLIST",
29
+ "~/Library/LaunchAgents/com.dev-memory.session-scan.plist",
30
+ )).expanduser()
31
+ INTERNAL_MARKER = "DEV_MEMORY_INTERNAL_SESSION_SUMMARY_V1"
32
+ MAINTENANCE_MARKER = "DEV_MEMORY_INTERNAL_MAINTENANCE_AGENT_V1"
33
+ INTERNAL_MARKERS = (INTERNAL_MARKER, MAINTENANCE_MARKER)
34
+ SUMMARY_MUTATION_FIELDS = (
35
+ "file_map",
36
+ "decisions",
37
+ "risks",
38
+ "glossary",
39
+ "shared_decisions",
40
+ "shared_context",
41
+ "shared_sources",
42
+ "upserts",
43
+ "appends",
44
+ "rewrites",
45
+ "deletes",
46
+ )
47
+ SUMMARY_PLACEHOLDER_WORDS = {
48
+ "decision",
49
+ "summary",
50
+ "reason",
51
+ "impact",
52
+ "risk",
53
+ "mitigation",
54
+ "term",
55
+ "definition",
56
+ "name",
57
+ "url",
58
+ "note",
59
+ "label",
60
+ "path",
61
+ "content",
62
+ }
63
+
64
+
65
+ DEFAULT_EXECUTORS = {
66
+ "coco": {
67
+ "enabled": True,
68
+ "command": "coco",
69
+ "model": None,
70
+ "profile": None,
71
+ "extra_args": [],
72
+ "env": {},
73
+ },
74
+ "codex": {
75
+ "enabled": True,
76
+ "command": "codex",
77
+ "model": None,
78
+ "profile": None,
79
+ "extra_args": [],
80
+ "env": {},
81
+ },
82
+ "claude": {
83
+ "enabled": True,
84
+ "command": "claude",
85
+ "model": None,
86
+ "profile": None,
87
+ "extra_args": [],
88
+ "env": {},
89
+ },
90
+ }
91
+
92
+
93
+ def _read_json(path, default=None):
94
+ try:
95
+ value = json.loads(Path(path).read_text(encoding="utf-8"))
96
+ return value
97
+ except (OSError, json.JSONDecodeError):
98
+ return default
99
+
100
+
101
+ def _atomic_json(path, payload):
102
+ path = Path(path)
103
+ path.parent.mkdir(parents=True, exist_ok=True)
104
+ tmp = path.with_name(f".{path.name}.tmp.{os.getpid()}.{time.time_ns()}")
105
+ tmp.write_text(json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
106
+ os.replace(tmp, path)
107
+
108
+
109
+ def _append_jsonl(path, payload):
110
+ path = Path(path)
111
+ path.parent.mkdir(parents=True, exist_ok=True)
112
+ with path.open("a", encoding="utf-8") as stream:
113
+ stream.write(json.dumps(payload, ensure_ascii=False, separators=(",", ":")) + "\n")
114
+
115
+
116
+ def _deep_merge(base, override):
117
+ out = dict(base)
118
+ for key, value in (override or {}).items():
119
+ if isinstance(value, dict) and isinstance(out.get(key), dict):
120
+ out[key] = _deep_merge(out[key], value)
121
+ else:
122
+ out[key] = value
123
+ return out
124
+
125
+
126
+ def default_scan_config():
127
+ return {
128
+ "executor": "auto",
129
+ "order": ["coco", "codex", "claude"],
130
+ "executors": json.loads(json.dumps(DEFAULT_EXECUTORS)),
131
+ "schedule_times": ["03:00", "13:00"],
132
+ "skip_when_computer_active": True,
133
+ "active_within_minutes": 10,
134
+ "activity_check_fail_closed": True,
135
+ "chunk_chars": 60000,
136
+ "idle_minutes": 60,
137
+ "first_lookback_days": 3,
138
+ "max_attempts": 2,
139
+ "invocation_timeout_seconds": 360,
140
+ }
141
+
142
+
143
+ def load_config():
144
+ root = _read_json(CONFIG_PATH, {})
145
+ root = root if isinstance(root, dict) else {}
146
+ section = root.get("session_scan")
147
+ return _deep_merge(default_scan_config(), section if isinstance(section, dict) else {})
148
+
149
+
150
+ def save_scan_config(section):
151
+ root = _read_json(CONFIG_PATH, {})
152
+ root = root if isinstance(root, dict) else {}
153
+ root["session_scan"] = section
154
+ _atomic_json(CONFIG_PATH, root)
155
+
156
+
157
+ def validate_config(config):
158
+ errors = []
159
+ warnings = []
160
+ selected = config.get("executor", "auto")
161
+ executors = config.get("executors")
162
+ if not isinstance(executors, dict) or not executors:
163
+ errors.append("session_scan.executors must be a non-empty object")
164
+ executors = {}
165
+ if selected != "auto" and selected not in executors:
166
+ errors.append(f"executor '{selected}' has no preset")
167
+ order = config.get("order")
168
+ if not isinstance(order, list) or not order:
169
+ errors.append("session_scan.order must be a non-empty list")
170
+ for name, preset in executors.items():
171
+ if not isinstance(preset, dict):
172
+ errors.append(f"executor '{name}' must be an object")
173
+ continue
174
+ command = preset.get("command")
175
+ if not isinstance(command, str) or not command.strip():
176
+ errors.append(f"executor '{name}' requires command")
177
+ if name == "codex" and "--ephemeral" in (preset.get("extra_args") or []):
178
+ warnings.append("codex.extra_args does not need --ephemeral; the scanner enforces it")
179
+ try:
180
+ if int(config.get("chunk_chars", 0)) < 1000:
181
+ errors.append("chunk_chars must be at least 1000")
182
+ except (TypeError, ValueError):
183
+ errors.append("chunk_chars must be an integer")
184
+ schedules = config.get("schedule_times")
185
+ if not isinstance(schedules, list) or not schedules:
186
+ errors.append("schedule_times must be a non-empty list")
187
+ else:
188
+ for value in schedules:
189
+ try:
190
+ _parse_schedule_time(value)
191
+ except ValueError as exc:
192
+ errors.append(str(exc))
193
+ try:
194
+ if int(config.get("active_within_minutes", 0)) < 1:
195
+ errors.append("active_within_minutes must be at least 1")
196
+ except (TypeError, ValueError):
197
+ errors.append("active_within_minutes must be an integer")
198
+ try:
199
+ if int(config.get("invocation_timeout_seconds", 0)) < 30:
200
+ errors.append("invocation_timeout_seconds must be at least 30")
201
+ except (TypeError, ValueError):
202
+ errors.append("invocation_timeout_seconds must be an integer")
203
+ return {"valid": not errors, "errors": errors, "warnings": warnings}
204
+
205
+
206
+ def _parse_schedule_time(value):
207
+ match = re.fullmatch(r"(\d{1,2}):(\d{2})", str(value or "").strip())
208
+ if not match:
209
+ raise ValueError(f"invalid schedule time '{value}', expected HH:MM")
210
+ hour, minute = int(match.group(1)), int(match.group(2))
211
+ if not 0 <= hour <= 23 or not 0 <= minute <= 59:
212
+ raise ValueError(f"invalid schedule time '{value}', expected HH:MM")
213
+ return hour, minute
214
+
215
+
216
+ def _calendar_intervals(config):
217
+ unique = sorted({_parse_schedule_time(value) for value in config.get("schedule_times", [])})
218
+ return [{"Hour": hour, "Minute": minute} for hour, minute in unique]
219
+
220
+
221
+ def _mac_idle_seconds():
222
+ if sys.platform != "darwin":
223
+ return None
224
+ result = subprocess.run(
225
+ ["ioreg", "-c", "IOHIDSystem", "-d", "4"],
226
+ capture_output=True,
227
+ text=True,
228
+ check=False,
229
+ )
230
+ if result.returncode != 0:
231
+ return None
232
+ match = re.search(r'"HIDIdleTime"\s*=\s*(\d+)', result.stdout or "")
233
+ if not match:
234
+ return None
235
+ return int(match.group(1)) / 1_000_000_000
236
+
237
+
238
+ def computer_activity(config):
239
+ threshold_seconds = int(config.get("active_within_minutes", 10)) * 60
240
+ idle_seconds = _mac_idle_seconds()
241
+ if idle_seconds is None:
242
+ return {
243
+ "status": "unknown",
244
+ "idle_seconds": None,
245
+ "threshold_seconds": threshold_seconds,
246
+ "skip": bool(config.get("activity_check_fail_closed", True)),
247
+ }
248
+ return {
249
+ "status": "active" if idle_seconds < threshold_seconds else "idle",
250
+ "idle_seconds": round(idle_seconds, 3),
251
+ "threshold_seconds": threshold_seconds,
252
+ "skip": idle_seconds < threshold_seconds,
253
+ }
254
+
255
+
256
+ def choose_executor(config):
257
+ executors = config.get("executors", {})
258
+ selected = config.get("executor", "auto")
259
+ names = config.get("order", []) if selected == "auto" else [selected]
260
+ for name in names:
261
+ preset = executors.get(name)
262
+ if not isinstance(preset, dict) or preset.get("enabled") is False:
263
+ continue
264
+ command = preset.get("command", name)
265
+ executable = shlex.split(command)[0] if command else ""
266
+ if executable and (Path(executable).exists() or shutil.which(executable)):
267
+ return name, preset
268
+ raise RuntimeError(f"no available session-scan executor in {names}")
269
+
270
+
271
+ def _content_text(content):
272
+ if isinstance(content, str):
273
+ return content.strip()
274
+ if not isinstance(content, list):
275
+ return ""
276
+ parts = []
277
+ for item in content:
278
+ if not isinstance(item, dict):
279
+ continue
280
+ if item.get("type") in {"tool_use", "tool_result", "function_call", "function_call_output"}:
281
+ continue
282
+ text = item.get("text")
283
+ if isinstance(text, str) and text.strip():
284
+ parts.append(text.strip())
285
+ return "\n".join(parts)
286
+
287
+
288
+ def _semantic_message(obj):
289
+ if obj.get("type") != "response_item":
290
+ return None
291
+ payload = obj.get("payload")
292
+ if not isinstance(payload, dict) or payload.get("type") != "message":
293
+ return None
294
+ role = payload.get("role")
295
+ if role not in {"user", "assistant"}:
296
+ return None
297
+ text = _content_text(payload.get("content"))
298
+ if not text:
299
+ return None
300
+ return {"role": role, "text": text, "timestamp": obj.get("timestamp")}
301
+
302
+
303
+ def _usage_dict(value):
304
+ if not isinstance(value, dict):
305
+ return None
306
+ aliases = {
307
+ "input_tokens": ("input_tokens", "inputTokens"),
308
+ "cached_input_tokens": ("cached_input_tokens", "cachedInputTokens", "cache_read_input_tokens"),
309
+ "output_tokens": ("output_tokens", "outputTokens"),
310
+ "reasoning_output_tokens": ("reasoning_output_tokens", "reasoningOutputTokens"),
311
+ "total_tokens": ("total_tokens", "totalTokens"),
312
+ }
313
+ out = {}
314
+ for target, keys in aliases.items():
315
+ for key in keys:
316
+ if isinstance(value.get(key), (int, float)):
317
+ out[target] = int(value[key])
318
+ break
319
+ if out and "total_tokens" not in out:
320
+ out["total_tokens"] = out.get("input_tokens", 0) + out.get("output_tokens", 0)
321
+ return out or None
322
+
323
+
324
+ def parse_codex_session(path, since_offset=0):
325
+ path = Path(path)
326
+ meta = {}
327
+ messages = []
328
+ total_usage = None
329
+ internal_marker = False
330
+ end_offset = since_offset
331
+ with path.open("rb") as stream:
332
+ if since_offset:
333
+ while True:
334
+ raw = stream.readline()
335
+ if not raw:
336
+ break
337
+ try:
338
+ obj = json.loads(raw.decode("utf-8"))
339
+ except (UnicodeDecodeError, json.JSONDecodeError):
340
+ continue
341
+ if obj.get("type") == "session_meta" and isinstance(obj.get("payload"), dict):
342
+ meta = obj["payload"]
343
+ break
344
+ stream.seek(since_offset)
345
+ while True:
346
+ start = stream.tell()
347
+ raw = stream.readline()
348
+ if not raw:
349
+ break
350
+ end = stream.tell()
351
+ end_offset = end
352
+ try:
353
+ obj = json.loads(raw.decode("utf-8"))
354
+ except (UnicodeDecodeError, json.JSONDecodeError):
355
+ continue
356
+ if obj.get("type") == "session_meta" and isinstance(obj.get("payload"), dict):
357
+ meta = obj["payload"]
358
+ payload = obj.get("payload") if isinstance(obj.get("payload"), dict) else {}
359
+ if obj.get("type") == "event_msg" and payload.get("type") == "token_count":
360
+ info = payload.get("info") if isinstance(payload.get("info"), dict) else {}
361
+ usage = _usage_dict(info.get("total_token_usage"))
362
+ if usage:
363
+ total_usage = usage
364
+ message = _semantic_message(obj)
365
+ if message and any(marker in message["text"] for marker in INTERNAL_MARKERS):
366
+ internal_marker = True
367
+ if message and end > since_offset:
368
+ message.update({"start_offset": start, "end_offset": end})
369
+ messages.append(message)
370
+ stat = path.stat()
371
+ return {
372
+ "path": str(path),
373
+ "size": stat.st_size,
374
+ "mtime": dt.datetime.fromtimestamp(stat.st_mtime, dt.timezone.utc).isoformat(),
375
+ "end_offset": end_offset,
376
+ "meta": meta,
377
+ "messages": messages,
378
+ "session_usage": total_usage,
379
+ "internal_marker": internal_marker,
380
+ }
381
+
382
+
383
+ def _codex_session_meta(path):
384
+ with Path(path).open("rb") as stream:
385
+ for raw in stream:
386
+ try:
387
+ obj = json.loads(raw.decode("utf-8"))
388
+ except (UnicodeDecodeError, json.JSONDecodeError):
389
+ continue
390
+ if obj.get("type") == "session_meta" and isinstance(obj.get("payload"), dict):
391
+ return obj["payload"]
392
+ return {}
393
+
394
+
395
+ def _session_files(lookback_days=None, cutoff_epoch=None):
396
+ roots = [CODEX_HOME / "sessions", CODEX_HOME / "archived_sessions"]
397
+ cutoff = cutoff_epoch
398
+ if cutoff is None and lookback_days is not None:
399
+ cutoff = time.time() - (lookback_days * 86400)
400
+ found = []
401
+ for root in roots:
402
+ if not root.exists():
403
+ continue
404
+ for path in root.rglob("*.jsonl"):
405
+ try:
406
+ if cutoff is not None and path.stat().st_mtime < cutoff:
407
+ continue
408
+ except OSError:
409
+ continue
410
+ found.append(path)
411
+ return sorted(found, key=lambda p: (p.stat().st_mtime, str(p)))
412
+
413
+
414
+ def _state_path(session_id):
415
+ safe = re.sub(r"[^0-9A-Za-z_.-]", "_", session_id)
416
+ return SCAN_ROOT / "state" / f"{safe}.json"
417
+
418
+
419
+ def _session_audit_path(session_id):
420
+ safe = re.sub(r"[^0-9A-Za-z_.-]", "_", session_id)
421
+ return SCAN_ROOT / "sessions" / f"{safe}.json"
422
+
423
+
424
+ def _workspace_primary(cwd):
425
+ for name in (".dev-memory-workspace.json", ".dev-assets-workspace.json"):
426
+ data = _read_json(Path(cwd) / name, {})
427
+ if isinstance(data, dict) and data.get("primary_repo"):
428
+ return data["primary_repo"]
429
+ return None
430
+
431
+
432
+ def resolve_target(cwd):
433
+ if not cwd:
434
+ return None, "missing_cwd"
435
+ root = Path(cwd).expanduser()
436
+ if not root.exists() or not root.is_dir():
437
+ return None, "cwd_not_found"
438
+ repos = list_repos_in_workspace(str(root))
439
+ if repos:
440
+ primary = _workspace_primary(root)
441
+ if primary:
442
+ root = next((repo for repo in repos if repo.name == primary), None)
443
+ if root is None:
444
+ return None, "workspace_primary_not_found"
445
+ elif len(repos) == 1:
446
+ root = repos[0]
447
+ else:
448
+ return None, "workspace_primary_required"
449
+ try:
450
+ repo_root, branch, branch_key, storage_root, repo_key, repo_dir, branch_dir = get_branch_paths(str(root))
451
+ except Exception as exc:
452
+ return None, f"repo_resolution_failed:{exc}"
453
+ if not branch_dir.exists():
454
+ return None, "memory_not_initialized"
455
+ return {
456
+ "repo_root": str(repo_root),
457
+ "repo_key": repo_key,
458
+ "repo_dir": str(repo_dir),
459
+ "branch": branch,
460
+ "branch_key": branch_key,
461
+ "branch_dir": str(branch_dir),
462
+ "storage_root": str(storage_root),
463
+ }, None
464
+
465
+
466
+ def _chunk_messages(messages, max_chars):
467
+ expanded = []
468
+ for message in messages:
469
+ text = message["text"]
470
+ if len(text) <= max_chars:
471
+ expanded.append(message)
472
+ continue
473
+ segment_count = (len(text) + max_chars - 1) // max_chars
474
+ for index in range(segment_count):
475
+ expanded.append({
476
+ **message,
477
+ "text": text[index * max_chars:(index + 1) * max_chars],
478
+ "segment_index": index + 1,
479
+ "segment_count": segment_count,
480
+ })
481
+ chunks = []
482
+ current = []
483
+ current_chars = 0
484
+ for message in expanded:
485
+ text = message["text"]
486
+ if current and current_chars + len(text) > max_chars:
487
+ chunks.append(current)
488
+ current = []
489
+ current_chars = 0
490
+ current.append(message)
491
+ current_chars += len(text)
492
+ if current_chars >= max_chars:
493
+ chunks.append(current)
494
+ current = []
495
+ current_chars = 0
496
+ if current:
497
+ chunks.append(current)
498
+ return chunks
499
+
500
+
501
+ def _existing_memory(target):
502
+ paths = [
503
+ Path(target["branch_dir"]) / name
504
+ for name in ("overview.md", "decisions.md", "risks.md", "glossary.md")
505
+ ] + [
506
+ Path(target["repo_dir"]) / "repo" / name
507
+ for name in ("decisions.md", "glossary.md")
508
+ ]
509
+ return [
510
+ {"path": str(path), "content": path.read_text(encoding="utf-8")}
511
+ for path in paths if path.exists()
512
+ ]
513
+
514
+
515
+ def _partial_prompt(target, chunk, index, total):
516
+ material = [{"role": item["role"], "text": item["text"]} for item in chunk]
517
+ return f"""{INTERNAL_MARKER}
518
+ 你是 dev-memory 的后台会话总结器。下面是仓库 {target['repo_key']} 分支 {target['branch']} 的第 {index}/{total} 个连续会话分块。
519
+ 完整阅读本分块,不要忽略前部内容。只提炼在未来开发会话中仍有价值的决策、约束、风险、术语、命令、外部入口和功能文件定位;不要记录聊天流水账、普通进展或可从 Git 直接恢复的历史。
520
+ 只输出 JSON 对象,允许字段:decisions、risks、glossary、file_map、shared_decisions、shared_context、shared_sources、skip_reason。字段内容沿用 summary-output 语义。
521
+ MESSAGES_JSON:
522
+ {json.dumps(material, ensure_ascii=False)}
523
+ """
524
+
525
+
526
+ def _single_prompt(target, chunk):
527
+ payload = {
528
+ "existing_memory": _existing_memory(target),
529
+ "messages": [{"role": item["role"], "text": item["text"]} for item in chunk],
530
+ }
531
+ return f"""{INTERNAL_MARKER}
532
+ 你是 dev-memory 的后台会话总结器。阅读完整会话与 existing_memory,直接生成最终 summary-output,不要先输出中间摘要。
533
+ 只保留未来开发会话中仍有价值、且 existing_memory 尚未覆盖的决策、约束、风险、术语、命令、外部入口和功能文件定位;不要记录聊天流水账、普通进展或可从 Git 直接恢复的历史。
534
+ 旧结论失效时使用 rewrites/deletes,不要追加矛盾条目。如果没有任何需要写入、改写或删除的内容,必须返回非空 skip_reason;禁止只返回 title 或空对象。
535
+ 字段 schema:decisions/shared_decisions 是对象数组,每项使用 summary(完整结论)、可选 reason、可选 impact;risks/glossary/shared_context/shared_sources 是完整自然语言字符串数组;file_map 每项为 {{"label":"功能说明","paths":["真实相对路径"]}};upserts/appends 每项必须有 kind 和 content;rewrites 每项必须有 id/content/reason;deletes 每项必须有 id/reason。
536
+ 所有值必须来自会话事实并可独立理解。禁止输出 decision/summary/reason/impact/risk/mitigation/term/definition/name/url/note/path/content 等 schema 占位词,禁止留空 summary。
537
+ 只输出一个 JSON 对象,不要 markdown fence。允许字段:title、file_map、decisions、risks、glossary、shared_decisions、shared_context、shared_sources、upserts、appends、rewrites、deletes、skip_reason。
538
+ INPUT_JSON:
539
+ {json.dumps(payload, ensure_ascii=False)}
540
+ """
541
+
542
+
543
+ def _final_prompt(target, partials):
544
+ payload = {"existing_memory": _existing_memory(target), "partial_summaries": partials}
545
+ return f"""{INTERNAL_MARKER}
546
+ 你是 dev-memory 的后台会话总结器。把所有 partial_summaries 与 existing_memory 合并为一个最终 summary-output。
547
+ 旧结论失效时使用 rewrites/deletes,不要追加矛盾条目;重复内容跳过。不要写当前进展、下一步或提交历史。
548
+ 如果没有任何需要写入、改写或删除的内容,必须返回非空 skip_reason;禁止只返回 title 或空对象。
549
+ decisions/shared_decisions 每项必须有非空 summary;risks/glossary/shared_context/shared_sources 必须是完整自然语言字符串;file_map 每项必须有 label 和真实 paths。禁止输出 schema 占位词或空 summary。
550
+ 只输出一个 JSON 对象,不要 markdown fence。允许字段:title、file_map、decisions、risks、glossary、shared_decisions、shared_context、shared_sources、upserts、appends、rewrites、deletes、skip_reason。
551
+ INPUT_JSON:
552
+ {json.dumps(payload, ensure_ascii=False)}
553
+ """
554
+
555
+
556
+ def _summary_payload_meta(payload):
557
+ payload = payload if isinstance(payload, dict) else {}
558
+ serialized = json.dumps(payload, ensure_ascii=False, sort_keys=True, separators=(",", ":"))
559
+ field_counts = {}
560
+ for key in SUMMARY_MUTATION_FIELDS:
561
+ value = payload.get(key)
562
+ if isinstance(value, list):
563
+ field_counts[key] = len(value)
564
+ elif value:
565
+ field_counts[key] = 1
566
+ skip_reason = payload.get("skip_reason")
567
+ skip_reason = skip_reason.strip() if isinstance(skip_reason, str) else None
568
+ return {
569
+ "keys": sorted(payload),
570
+ "field_counts": field_counts,
571
+ "mutation_count": sum(field_counts.values()),
572
+ "skip_reason": skip_reason or None,
573
+ "chars": len(serialized),
574
+ "sha256": hashlib.sha256(serialized.encode("utf-8")).hexdigest(),
575
+ }
576
+
577
+
578
+ def _looks_like_placeholder_text(value):
579
+ if not isinstance(value, str):
580
+ return False
581
+ words = []
582
+ for line in value.splitlines():
583
+ normalized = re.sub(r"^[\s>*+-]+", "", line).strip().strip("::").lower()
584
+ if normalized:
585
+ words.append(normalized)
586
+ return bool(words) and all(word in SUMMARY_PLACEHOLDER_WORDS for word in words)
587
+
588
+
589
+ def _summary_payload_validation_errors(payload):
590
+ errors = []
591
+ if not isinstance(payload, dict):
592
+ return ["summary output must be an object"]
593
+ for field in ("decisions", "shared_decisions"):
594
+ for index, item in enumerate(payload.get(field) or []):
595
+ if isinstance(item, str):
596
+ summary = item.strip()
597
+ elif isinstance(item, dict):
598
+ summary = str(item.get("summary") or item.get("decision") or "").strip()
599
+ else:
600
+ summary = ""
601
+ if not summary:
602
+ errors.append(f"{field}[{index}] requires a non-empty summary")
603
+ elif _looks_like_placeholder_text(summary):
604
+ errors.append(f"{field}[{index}] contains schema placeholder text")
605
+ for field in ("risks", "glossary", "shared_context", "shared_sources"):
606
+ for index, item in enumerate(payload.get(field) or []):
607
+ if not isinstance(item, str) or not item.strip():
608
+ errors.append(f"{field}[{index}] must be a non-empty string")
609
+ elif _looks_like_placeholder_text(item):
610
+ errors.append(f"{field}[{index}] contains schema placeholder text")
611
+ for index, item in enumerate(payload.get("file_map") or []):
612
+ paths = item.get("paths") if isinstance(item, dict) else None
613
+ if not isinstance(item, dict) or not str(item.get("label") or "").strip() or not paths:
614
+ errors.append(f"file_map[{index}] requires label and paths")
615
+ return errors
616
+
617
+
618
+ def _semantic_action_count(apply_result):
619
+ actions = apply_result.get("actions") if isinstance(apply_result, dict) else []
620
+ return sum(
621
+ 1
622
+ for action in (actions or [])
623
+ if isinstance(action, dict) and not str(action.get("op") or "").startswith("prune-")
624
+ )
625
+
626
+
627
+ def _executor_args(name, preset, prompt):
628
+ command = shlex.split(preset.get("command", name))
629
+ model = preset.get("model")
630
+ profile = preset.get("profile")
631
+ extra = [str(item) for item in (preset.get("extra_args") or [])]
632
+ if name == "coco":
633
+ args = command + ["-p", "--yolo", "--output-format", "json"]
634
+ if model:
635
+ args += ["-c", f"model={model}"]
636
+ if profile:
637
+ args += ["-c", f"profile={profile}"]
638
+ args += extra + [prompt]
639
+ elif name == "codex":
640
+ args = command + [
641
+ "exec", "--ephemeral", "--json", "--ignore-rules",
642
+ "--skip-git-repo-check", "--sandbox", "danger-full-access",
643
+ ]
644
+ if model:
645
+ args += ["--model", str(model)]
646
+ if profile:
647
+ args += ["--profile", str(profile)]
648
+ args += extra + [prompt]
649
+ elif name == "claude":
650
+ args = command + ["-p", "--permission-mode", "bypassPermissions", "--output-format", "json", "--no-session-persistence"]
651
+ if model:
652
+ args += ["--model", str(model)]
653
+ args += extra + [prompt]
654
+ else:
655
+ args = command + extra
656
+ args = [part.replace("{prompt}", prompt) for part in args]
657
+ if not any("{prompt}" in part for part in command + extra):
658
+ args.append(prompt)
659
+ return args
660
+
661
+
662
+ def _find_json_object(text):
663
+ text = (text or "").strip()
664
+ try:
665
+ value = json.loads(text)
666
+ if isinstance(value, dict):
667
+ return value
668
+ except json.JSONDecodeError:
669
+ pass
670
+ decoder = json.JSONDecoder()
671
+ for match in re.finditer(r"\{", text):
672
+ try:
673
+ value, _ = decoder.raw_decode(text[match.start():])
674
+ if isinstance(value, dict):
675
+ return value
676
+ except json.JSONDecodeError:
677
+ continue
678
+ raise ValueError("executor output contains no JSON object")
679
+
680
+
681
+ def _walk_dicts(value):
682
+ if isinstance(value, dict):
683
+ yield value
684
+ for child in value.values():
685
+ yield from _walk_dicts(child)
686
+ elif isinstance(value, list):
687
+ for child in value:
688
+ yield from _walk_dicts(child)
689
+
690
+
691
+ def _parse_executor_output(name, stdout, stderr):
692
+ records = []
693
+ for line in stdout.splitlines():
694
+ try:
695
+ records.append(json.loads(line))
696
+ except json.JSONDecodeError:
697
+ continue
698
+ if not records:
699
+ try:
700
+ records = [json.loads(stdout)]
701
+ except json.JSONDecodeError:
702
+ records = []
703
+ usage = None
704
+ session_id = None
705
+ candidates = []
706
+ for record in records:
707
+ for item in _walk_dicts(record):
708
+ for key in ("usage", "token_usage", "total_token_usage"):
709
+ parsed_usage = _usage_dict(item.get(key))
710
+ if parsed_usage:
711
+ usage = parsed_usage
712
+ session_id = session_id or item.get("thread_id") or item.get("session_id")
713
+ for key in ("result", "text", "output_text"):
714
+ if isinstance(item.get(key), str):
715
+ candidates.append(item[key])
716
+ if item.get("type") == "agent_message" and isinstance(item.get("text"), str):
717
+ candidates.append(item["text"])
718
+ last_error = None
719
+ for candidate in reversed(candidates):
720
+ try:
721
+ return _find_json_object(candidate), usage, session_id
722
+ except ValueError as exc:
723
+ last_error = exc
724
+ for candidate in (stdout, stderr):
725
+ try:
726
+ value = json.loads(candidate)
727
+ if isinstance(value, dict):
728
+ for key in ("result", "text", "output_text"):
729
+ if isinstance(value.get(key), str):
730
+ return _find_json_object(value[key]), usage, session_id
731
+ return value, usage, session_id
732
+ except json.JSONDecodeError:
733
+ try:
734
+ return _find_json_object(candidate), usage, session_id
735
+ except ValueError as exc:
736
+ last_error = exc
737
+ raise last_error or ValueError(f"unable to parse {name} output")
738
+
739
+
740
+ def run_executor(name, preset, prompt, cwd, run_id, invocation):
741
+ args = _executor_args(name, preset, prompt)
742
+ env = dict(os.environ)
743
+ env.update({str(k): str(v) for k, v in (preset.get("env") or {}).items()})
744
+ started = time.time()
745
+ timeout_seconds = int(preset.get("_timeout_seconds", 360))
746
+ record = {
747
+ "invocation": invocation,
748
+ "stage": "final" if ":final" in invocation else "partial",
749
+ "executor": name,
750
+ "model": preset.get("model"),
751
+ "profile": preset.get("profile"),
752
+ "prompt_chars": len(prompt),
753
+ "started_at": dt.datetime.fromtimestamp(started, dt.timezone.utc).isoformat(),
754
+ "duration_ms": 0,
755
+ "returncode": None,
756
+ "usage_status": "unavailable",
757
+ }
758
+ try:
759
+ result = subprocess.run(
760
+ args,
761
+ cwd=cwd,
762
+ env=env,
763
+ capture_output=True,
764
+ text=True,
765
+ check=False,
766
+ timeout=timeout_seconds,
767
+ )
768
+ except subprocess.TimeoutExpired:
769
+ record.update({
770
+ "duration_ms": int((time.time() - started) * 1000),
771
+ "returncode": None,
772
+ "timed_out": True,
773
+ "timeout_seconds": timeout_seconds,
774
+ "error": f"executor timed out after {timeout_seconds} seconds",
775
+ })
776
+ return None, record
777
+ record["duration_ms"] = int((time.time() - started) * 1000)
778
+ record["returncode"] = result.returncode
779
+ if result.returncode != 0:
780
+ record["error"] = (result.stderr or result.stdout or "executor failed")[-4000:]
781
+ return None, record
782
+ try:
783
+ payload, usage, session_id = _parse_executor_output(name, result.stdout, result.stderr)
784
+ record["output"] = _summary_payload_meta(payload)
785
+ record["usage"] = usage
786
+ record["usage_status"] = "reported" if usage else "unavailable"
787
+ record["internal_session_id"] = session_id
788
+ if session_id:
789
+ _append_jsonl(SCAN_ROOT / "internal-sessions.jsonl", {
790
+ "at": now_iso(), "run_id": run_id, "executor": name, "session_id": session_id,
791
+ })
792
+ return payload, record
793
+ except Exception as exc:
794
+ record["error"] = str(exc)
795
+ return None, record
796
+
797
+
798
+ def run_executor_with_retries(name, preset, prompt, cwd, run_id, invocation, max_attempts):
799
+ records = []
800
+ payload = None
801
+ for attempt in range(1, max(1, max_attempts) + 1):
802
+ attempt_prompt = prompt
803
+ if attempt > 1:
804
+ attempt_prompt += "\n上一次调用失败。请重新阅读输入,只输出一个合法 JSON 对象。"
805
+ payload, record = run_executor(
806
+ name, preset, attempt_prompt, cwd, run_id, f"{invocation}:attempt:{attempt}"
807
+ )
808
+ record["attempt"] = attempt
809
+ records.append(record)
810
+ if payload is not None:
811
+ break
812
+ if record.get("timed_out"):
813
+ break
814
+ return payload, records
815
+
816
+
817
+ def _apply_summary(target, payload):
818
+ result = subprocess.run(
819
+ [sys.executable, str(PACKAGE_ROOT / "lib" / "dev_memory_capture.py"), "apply-summary-output", "--repo", target["repo_root"], "--json", json.dumps(payload, ensure_ascii=False)],
820
+ cwd=target["repo_root"], capture_output=True, text=True, check=False,
821
+ )
822
+ if result.returncode not in (0, 2):
823
+ raise RuntimeError((result.stderr or result.stdout or "apply-summary-output failed").strip())
824
+ try:
825
+ return json.loads(result.stdout)
826
+ except json.JSONDecodeError:
827
+ return {"raw": result.stdout.strip()}
828
+
829
+
830
+ def _sum_usage(invocations):
831
+ total = {}
832
+ unavailable = 0
833
+ for item in invocations:
834
+ usage = item.get("usage")
835
+ if not isinstance(usage, dict):
836
+ unavailable += 1
837
+ continue
838
+ for key, value in usage.items():
839
+ if isinstance(value, int):
840
+ total[key] = total.get(key, 0) + value
841
+ return total or None, unavailable
842
+
843
+
844
+ def _internal_ids():
845
+ ids = set()
846
+ path = SCAN_ROOT / "internal-sessions.jsonl"
847
+ try:
848
+ for line in path.read_text(encoding="utf-8").splitlines():
849
+ value = json.loads(line)
850
+ if value.get("session_id"):
851
+ ids.add(value["session_id"])
852
+ except (OSError, json.JSONDecodeError):
853
+ pass
854
+ return ids
855
+
856
+
857
+ def discover(config, since=None):
858
+ origin = _read_json(SCAN_ROOT / "scan-origin.json", {}) or {}
859
+ first_run = not origin
860
+ lookback = config.get("first_lookback_days", 3) if first_run and since is None else None
861
+ cutoff_epoch = origin.get("initial_cutoff_epoch") if origin and since is None else None
862
+ if since:
863
+ parsed = dt.datetime.fromisoformat(since.replace("Z", "+00:00"))
864
+ cutoff_epoch = parsed.astimezone(dt.timezone.utc).timestamp()
865
+ lookback = None
866
+ internal_ids = _internal_ids()
867
+ sessions = []
868
+ idle_seconds = int(config.get("idle_minutes", 60)) * 60
869
+ source_by_session = {}
870
+ for path in _session_files(lookback, cutoff_epoch=cutoff_epoch):
871
+ meta = _codex_session_meta(path)
872
+ session_id = meta.get("session_id") or meta.get("id") or path.stem
873
+ previous = source_by_session.get(session_id)
874
+ if previous is None or (path.stat().st_size, path.stat().st_mtime) > (
875
+ previous[0].stat().st_size, previous[0].stat().st_mtime
876
+ ):
877
+ source_by_session[session_id] = (path, meta)
878
+ for session_id, (path, meta) in sorted(
879
+ source_by_session.items(), key=lambda item: (item[1][0].stat().st_mtime, str(item[1][0]))
880
+ ):
881
+ stat = path.stat()
882
+ state = _read_json(_state_path(session_id), {}) or {}
883
+ cursor = int(state.get("processed_offset", 0))
884
+ if cursor and stat.st_size <= cursor:
885
+ parsed = {
886
+ "path": str(path),
887
+ "size": stat.st_size,
888
+ "mtime": dt.datetime.fromtimestamp(stat.st_mtime, dt.timezone.utc).isoformat(),
889
+ "end_offset": cursor,
890
+ "meta": meta,
891
+ "messages": [],
892
+ "session_usage": state.get("session_usage"),
893
+ "internal_marker": bool(state.get("internal_marker")),
894
+ }
895
+ else:
896
+ parsed = parse_codex_session(path, cursor)
897
+ reason = None
898
+ if session_id in internal_ids or str(session_id).startswith("dev-memory-summary-") or parsed["internal_marker"]:
899
+ reason = "excluded_internal"
900
+ elif meta.get("thread_source") == "automation":
901
+ reason = "excluded_automation"
902
+ elif time.time() - path.stat().st_mtime < idle_seconds:
903
+ reason = "not_idle"
904
+ elif parsed["size"] <= cursor:
905
+ reason = "unchanged"
906
+ target, target_error = resolve_target(meta.get("cwd"))
907
+ if not reason and target_error:
908
+ reason = target_error
909
+ sessions.append({
910
+ **parsed,
911
+ "session_id": session_id,
912
+ "cursor_before": cursor,
913
+ "new_bytes": max(0, parsed["size"] - cursor),
914
+ "target": target,
915
+ "status": "candidate" if not reason else "skipped",
916
+ "reason": reason,
917
+ })
918
+ return sessions
919
+
920
+
921
+ def _persist_scan_run(run, event):
922
+ run_id = run["run_id"]
923
+ _atomic_json(SCAN_ROOT / "runs" / f"{run_id}.json", run)
924
+ _atomic_json(SCAN_ROOT / "last-run.json", run)
925
+ _append_jsonl(SCAN_ROOT / "events.jsonl", {"at": now_iso(), "event": event, "run_id": run_id})
926
+
927
+
928
+ def _skipped_activity_run(run_id, started, activity, *, dry_run=False):
929
+ status = "skipped_active" if activity["status"] == "active" else "skipped_activity_unknown"
930
+ return {
931
+ "schema_version": SCHEMA_VERSION,
932
+ "run_id": run_id,
933
+ "status": status,
934
+ "skip_reason": activity["status"],
935
+ "started_at": dt.datetime.fromtimestamp(started, dt.timezone.utc).isoformat(),
936
+ "finished_at": now_iso(),
937
+ "duration_ms": int((time.time() - started) * 1000),
938
+ "scheduled": True,
939
+ "dry_run": bool(dry_run),
940
+ "activity": activity,
941
+ "executor": None,
942
+ "model": None,
943
+ "profile": None,
944
+ "session_count": 0,
945
+ "candidate_count": 0,
946
+ "done_count": 0,
947
+ "summary_skipped_count": 0,
948
+ "failed_count": 0,
949
+ "skipped_count": 0,
950
+ "discovery_skipped_count": 0,
951
+ "raw_bytes": 0,
952
+ "new_bytes": 0,
953
+ "observed_new_bytes": 0,
954
+ "eligible_new_bytes": 0,
955
+ "skipped_new_bytes": 0,
956
+ "semantic_messages": 0,
957
+ "semantic_chars": 0,
958
+ "summary_usage": None,
959
+ "usage_unavailable_invocations": 0,
960
+ "invocations": [],
961
+ "sessions": [],
962
+ }
963
+
964
+
965
+ def _advance_session_state(session):
966
+ state_path = _state_path(session["session_id"])
967
+ existing = _read_json(state_path, {}) or {}
968
+ existing_offset = int(existing.get("processed_offset", 0))
969
+ if existing_offset >= session["end_offset"]:
970
+ return existing_offset
971
+ _atomic_json(state_path, {
972
+ "schema_version": SCHEMA_VERSION,
973
+ "session_id": session["session_id"],
974
+ "path": session["path"],
975
+ "processed_offset": session["end_offset"],
976
+ "raw_size": session["size"],
977
+ "sha256": hashlib.sha256(Path(session["path"]).read_bytes()).hexdigest(),
978
+ "session_usage": session["session_usage"],
979
+ "internal_marker": session["internal_marker"],
980
+ "repo_key": session["target"]["repo_key"],
981
+ "branch": session["target"]["branch"],
982
+ "updated_at": now_iso(),
983
+ })
984
+ return session["end_offset"]
985
+
986
+
987
+ def _execute_sessions(config, args, sessions, run_id, started, *, activity=None, run_kind="scan", replay_source=None):
988
+ candidates = [item for item in sessions if item["status"] == "candidate" and item["messages"]]
989
+ executor_name = None
990
+ preset = None
991
+ if candidates and not args.dry_run:
992
+ executor_override = getattr(args, "executor", None)
993
+ executor_config = {**config, "executor": executor_override} if executor_override else config
994
+ executor_name, preset = choose_executor(executor_config)
995
+ preset = dict(preset)
996
+ preset["_timeout_seconds"] = int(config.get("invocation_timeout_seconds", 360))
997
+ invocations = []
998
+ results = []
999
+ for session in sessions:
1000
+ audit = {
1001
+ "schema_version": SCHEMA_VERSION,
1002
+ "session_id": session["session_id"],
1003
+ "path": session["path"],
1004
+ "originator": session["meta"].get("originator"),
1005
+ "source": session["meta"].get("source"),
1006
+ "thread_source": session["meta"].get("thread_source"),
1007
+ "cwd": session["meta"].get("cwd"),
1008
+ "raw_size": session["size"],
1009
+ "cursor_before": session["cursor_before"],
1010
+ "new_bytes": session["new_bytes"],
1011
+ "semantic_messages": len(session["messages"]),
1012
+ "semantic_chars": sum(len(item["text"]) for item in session["messages"]),
1013
+ "session_usage": session["session_usage"],
1014
+ "repo_key": (session["target"] or {}).get("repo_key"),
1015
+ "branch": (session["target"] or {}).get("branch"),
1016
+ "status": session["status"],
1017
+ "reason": session["reason"],
1018
+ "last_scanned_at": now_iso(),
1019
+ }
1020
+ if session.get("replay_source"):
1021
+ audit["replay_source"] = session["replay_source"]
1022
+ if session not in candidates:
1023
+ if audit["status"] == "candidate":
1024
+ audit["status"] = "skipped"
1025
+ audit["reason"] = "no_semantic_messages"
1026
+ results.append(audit)
1027
+ _atomic_json(_session_audit_path(session["session_id"]), audit)
1028
+ continue
1029
+ chunks = _chunk_messages(session["messages"], int(config.get("chunk_chars", 60000)))
1030
+ audit["chunk_count"] = len(chunks)
1031
+ audit["chunks"] = [
1032
+ {
1033
+ "index": index,
1034
+ "start_offset": chunk[0]["start_offset"],
1035
+ "end_offset": chunk[-1]["end_offset"],
1036
+ "messages": len(chunk),
1037
+ "chars": sum(len(item["text"]) for item in chunk),
1038
+ "sha256": hashlib.sha256("\n".join(item["text"] for item in chunk).encode()).hexdigest(),
1039
+ }
1040
+ for index, chunk in enumerate(chunks, 1)
1041
+ ]
1042
+ if args.dry_run:
1043
+ audit["status"] = "dry_run"
1044
+ results.append(audit)
1045
+ _atomic_json(_session_audit_path(session["session_id"]), audit)
1046
+ continue
1047
+ failed = None
1048
+ invocation_start = len(invocations)
1049
+ final_payload = None
1050
+ if len(chunks) == 1:
1051
+ final_payload, attempt_records = run_executor_with_retries(
1052
+ executor_name, preset, _single_prompt(session["target"], chunks[0]),
1053
+ session["target"]["repo_root"], run_id, f"{session['session_id']}:final",
1054
+ int(config.get("max_attempts", 2)),
1055
+ )
1056
+ invocations.extend(attempt_records)
1057
+ if final_payload is None:
1058
+ failed = attempt_records[-1].get("error", "final summary failed")
1059
+ else:
1060
+ partials = []
1061
+ for index, chunk in enumerate(chunks, 1):
1062
+ payload, attempt_records = run_executor_with_retries(
1063
+ executor_name, preset, _partial_prompt(session["target"], chunk, index, len(chunks)),
1064
+ session["target"]["repo_root"], run_id, f"{session['session_id']}:chunk:{index}",
1065
+ int(config.get("max_attempts", 2)),
1066
+ )
1067
+ invocations.extend(attempt_records)
1068
+ if payload is None:
1069
+ failed = attempt_records[-1].get("error", "chunk summary failed")
1070
+ break
1071
+ partials.append(payload)
1072
+ if not failed:
1073
+ final_payload, attempt_records = run_executor_with_retries(
1074
+ executor_name, preset, _final_prompt(session["target"], partials),
1075
+ session["target"]["repo_root"], run_id, f"{session['session_id']}:final",
1076
+ int(config.get("max_attempts", 2)),
1077
+ )
1078
+ invocations.extend(attempt_records)
1079
+ if final_payload is None:
1080
+ failed = attempt_records[-1].get("error", "final summary failed")
1081
+ if not failed:
1082
+ summary_output = _summary_payload_meta(final_payload)
1083
+ validation_errors = _summary_payload_validation_errors(final_payload)
1084
+ if validation_errors:
1085
+ summary_output["validation_errors"] = validation_errors
1086
+ audit["summary_output"] = summary_output
1087
+ if validation_errors:
1088
+ failed = "invalid final summary output: " + "; ".join(validation_errors)
1089
+ elif not summary_output["mutation_count"] and not summary_output["skip_reason"]:
1090
+ failed = "final summary output has no memory mutations or skip_reason"
1091
+ if not failed and audit["summary_output"]["mutation_count"]:
1092
+ try:
1093
+ audit["apply_result"] = _apply_summary(session["target"], final_payload)
1094
+ audit["semantic_action_count"] = _semantic_action_count(audit["apply_result"])
1095
+ if not audit["semantic_action_count"]:
1096
+ if audit["summary_output"]["skip_reason"]:
1097
+ audit["status"] = "skipped_summary"
1098
+ else:
1099
+ failed = "summary output produced no semantic memory actions or skip_reason"
1100
+ else:
1101
+ audit["status"] = "done"
1102
+ except Exception as exc:
1103
+ failed = str(exc)
1104
+ elif not failed:
1105
+ audit["status"] = "skipped_summary"
1106
+ audit["semantic_action_count"] = 0
1107
+ audit["apply_result"] = {
1108
+ "mode": "not-applied",
1109
+ "touched_targets": [],
1110
+ "actions": [],
1111
+ "skip_reason": audit["summary_output"]["skip_reason"],
1112
+ }
1113
+ if not failed and audit["status"] in {"done", "skipped_summary"}:
1114
+ audit["cursor_after"] = session["end_offset"]
1115
+ audit["state_offset_after"] = _advance_session_state(session)
1116
+ if failed:
1117
+ audit["status"] = "failed"
1118
+ audit["error"] = failed
1119
+ audit["summary_usage"], audit["usage_unavailable_invocations"] = _sum_usage(invocations[invocation_start:])
1120
+ audit["executor"] = executor_name
1121
+ audit["model"] = preset.get("model")
1122
+ results.append(audit)
1123
+ _atomic_json(_session_audit_path(session["session_id"]), audit)
1124
+ usage, unavailable = _sum_usage(invocations)
1125
+ run = {
1126
+ "schema_version": SCHEMA_VERSION,
1127
+ "run_id": run_id,
1128
+ "run_kind": run_kind,
1129
+ "started_at": dt.datetime.fromtimestamp(started, dt.timezone.utc).isoformat(),
1130
+ "finished_at": now_iso(),
1131
+ "duration_ms": int((time.time() - started) * 1000),
1132
+ "dry_run": bool(args.dry_run),
1133
+ "scheduled": bool(getattr(args, "scheduled", False)),
1134
+ "activity": activity,
1135
+ "executor": executor_name,
1136
+ "model": preset.get("model") if preset else None,
1137
+ "profile": preset.get("profile") if preset else None,
1138
+ "session_count": len(results),
1139
+ "candidate_count": len(candidates),
1140
+ "done_count": sum(item["status"] == "done" for item in results),
1141
+ "summary_skipped_count": sum(item["status"] == "skipped_summary" for item in results),
1142
+ "failed_count": sum(item["status"] == "failed" for item in results),
1143
+ "skipped_count": sum(item["status"] in {"skipped", "skipped_summary"} for item in results),
1144
+ "discovery_skipped_count": sum(item["status"] == "skipped" for item in results),
1145
+ "raw_bytes": sum(item["raw_size"] for item in results),
1146
+ "new_bytes": sum(item["new_bytes"] for item in results),
1147
+ "observed_new_bytes": sum(item["new_bytes"] for item in results),
1148
+ "eligible_new_bytes": sum(item["new_bytes"] for item in results if item["status"] != "skipped"),
1149
+ "skipped_new_bytes": sum(item["new_bytes"] for item in results if item["status"] == "skipped"),
1150
+ "semantic_messages": sum(item["semantic_messages"] for item in results),
1151
+ "semantic_chars": sum(item["semantic_chars"] for item in results),
1152
+ "summary_usage": usage,
1153
+ "usage_unavailable_invocations": unavailable,
1154
+ "invocations": invocations,
1155
+ "sessions": results,
1156
+ }
1157
+ if replay_source:
1158
+ run["replay_source"] = replay_source
1159
+ _persist_scan_run(run, "scan_completed")
1160
+ print(json.dumps(run, ensure_ascii=False, indent=2) if args.json else _format_run(run))
1161
+ return 1 if run["failed_count"] else 0
1162
+
1163
+
1164
+ def run_scan(args):
1165
+ config = load_config()
1166
+ validation = validate_config(config)
1167
+ if not validation["valid"]:
1168
+ raise RuntimeError("; ".join(validation["errors"]))
1169
+ run_id = dt.datetime.now().strftime("%Y%m%dT%H%M%S") + f"-{os.getpid()}"
1170
+ started = time.time()
1171
+ activity = None
1172
+ if getattr(args, "scheduled", False) and config.get("skip_when_computer_active", True):
1173
+ activity = computer_activity(config)
1174
+ if activity["skip"]:
1175
+ run = _skipped_activity_run(run_id, started, activity, dry_run=args.dry_run)
1176
+ _persist_scan_run(run, run["status"])
1177
+ print(json.dumps(run, ensure_ascii=False, indent=2) if args.json else _format_run(run))
1178
+ return 0
1179
+ origin_path = SCAN_ROOT / "scan-origin.json"
1180
+ if not origin_path.exists():
1181
+ lookback_days = int(config.get("first_lookback_days", 3))
1182
+ _atomic_json(origin_path, {
1183
+ "created_at": now_iso(),
1184
+ "initial_cutoff_epoch": time.time() - lookback_days * 86400,
1185
+ "first_lookback_days": lookback_days,
1186
+ })
1187
+ sessions = discover(config, args.since)
1188
+ return _execute_sessions(config, args, sessions, run_id, started, activity=activity)
1189
+
1190
+
1191
+ def _format_tokens(usage):
1192
+ return str((usage or {}).get("total_tokens", "unavailable"))
1193
+
1194
+
1195
+ def _format_run(run):
1196
+ if str(run.get("status", "")).startswith("skipped_"):
1197
+ activity = run.get("activity") or {}
1198
+ return (
1199
+ f"run {run['run_id']}: {run['status']}; "
1200
+ f"idle_seconds={activity.get('idle_seconds')} threshold={activity.get('threshold_seconds')}"
1201
+ )
1202
+ return (
1203
+ f"run {run['run_id']}: {run['done_count']} done, "
1204
+ f"{run.get('summary_skipped_count', 0)} summary-skipped, {run['failed_count']} failed, "
1205
+ f"{run.get('discovery_skipped_count', run['skipped_count'])} discovery-skipped; "
1206
+ f"{run.get('eligible_new_bytes', run['new_bytes'])} eligible / "
1207
+ f"{run.get('observed_new_bytes', run['new_bytes'])} observed new bytes; "
1208
+ f"summary tokens {_format_tokens(run.get('summary_usage'))}"
1209
+ )
1210
+
1211
+
1212
+ def _runs():
1213
+ return [value for path in sorted((SCAN_ROOT / "runs").glob("*.json"), reverse=True) if isinstance((value := _read_json(path)), dict)]
1214
+
1215
+
1216
+ def _covered_bytes(intervals):
1217
+ total = 0
1218
+ end = None
1219
+ for start, stop in sorted(intervals):
1220
+ if stop <= start:
1221
+ continue
1222
+ if end is None or start > end:
1223
+ total += stop - start
1224
+ end = stop
1225
+ elif stop > end:
1226
+ total += stop - end
1227
+ end = stop
1228
+ return total
1229
+
1230
+
1231
+ def command_stats(args):
1232
+ runs = _runs()
1233
+ if args.since:
1234
+ runs = [item for item in runs if (item.get("started_at") or "") >= args.since]
1235
+ repos = {}
1236
+ total_usage = {}
1237
+ unavailable = 0
1238
+ for run in runs:
1239
+ usage = run.get("summary_usage") or {}
1240
+ for key, value in usage.items():
1241
+ if isinstance(value, int):
1242
+ total_usage[key] = total_usage.get(key, 0) + value
1243
+ unavailable += int(run.get("usage_unavailable_invocations", 0))
1244
+ for session in run.get("sessions", []):
1245
+ key = session.get("repo_key")
1246
+ if not key or (args.repo and key != args.repo):
1247
+ continue
1248
+ item = repos.setdefault(key, {
1249
+ "repo_key": key,
1250
+ "scan_count": 0,
1251
+ "sessions": set(),
1252
+ "intervals": {},
1253
+ "raw_bytes": 0,
1254
+ "new_bytes": 0,
1255
+ "observed_new_bytes": 0,
1256
+ "eligible_new_bytes": 0,
1257
+ "skipped_new_bytes": 0,
1258
+ "summary_tokens": 0,
1259
+ "done_count": 0,
1260
+ "summary_skipped_count": 0,
1261
+ "failed_count": 0,
1262
+ "empty_apply_count": 0,
1263
+ })
1264
+ item["scan_count"] += 1
1265
+ session_id = session.get("session_id")
1266
+ item["sessions"].add(session_id)
1267
+ item["raw_bytes"] += int(session.get("raw_size", 0))
1268
+ observed = int(session.get("new_bytes", 0))
1269
+ item["new_bytes"] += observed
1270
+ item["observed_new_bytes"] += observed
1271
+ if session.get("status") == "skipped":
1272
+ item["skipped_new_bytes"] += observed
1273
+ else:
1274
+ item["eligible_new_bytes"] += observed
1275
+ start = int(session.get("cursor_before", 0))
1276
+ stop = min(int(session.get("raw_size", start + observed)), start + observed)
1277
+ item["intervals"].setdefault(session_id, []).append((start, stop))
1278
+ item["summary_tokens"] += int((session.get("summary_usage") or {}).get("total_tokens", 0))
1279
+ status = session.get("status")
1280
+ item["done_count"] += int(status == "done")
1281
+ item["summary_skipped_count"] += int(status == "skipped_summary")
1282
+ item["failed_count"] += int(status == "failed")
1283
+ item["empty_apply_count"] += int(
1284
+ status == "done"
1285
+ and not (session.get("apply_result") or {}).get("touched_targets")
1286
+ )
1287
+ repo_rows = []
1288
+ for item in repos.values():
1289
+ item["session_count"] = len(item.pop("sessions"))
1290
+ item["unique_new_bytes"] = sum(_covered_bytes(value) for value in item.pop("intervals").values())
1291
+ repo_rows.append(item)
1292
+ payload = {"run_count": len(runs), "summary_usage": total_usage or None, "usage_unavailable_invocations": unavailable, "repos": sorted(repo_rows, key=lambda x: x["repo_key"])}
1293
+ print(json.dumps(payload, ensure_ascii=False, indent=2))
1294
+
1295
+
1296
+ def command_history(args):
1297
+ runs = _runs()
1298
+ if args.repo:
1299
+ runs = [run for run in runs if any(item.get("repo_key") == args.repo for item in run.get("sessions", []))]
1300
+ runs = runs[:args.limit]
1301
+ print(json.dumps(runs, ensure_ascii=False, indent=2) if args.json else "\n".join(_format_run(run) for run in runs))
1302
+
1303
+
1304
+ def command_show(args):
1305
+ path = SCAN_ROOT / "runs" / f"{args.run_id}.json"
1306
+ value = _read_json(path)
1307
+ if not isinstance(value, dict):
1308
+ raise RuntimeError(f"run not found: {args.run_id}")
1309
+ print(json.dumps(value, ensure_ascii=False, indent=2))
1310
+
1311
+
1312
+ def _replay_session(source_run_id, audit):
1313
+ session_id = audit.get("session_id")
1314
+ path = Path(audit.get("path") or "").expanduser()
1315
+ if not path.is_file():
1316
+ matches = [
1317
+ candidate
1318
+ for root in (CODEX_HOME / "sessions", CODEX_HOME / "archived_sessions")
1319
+ if root.exists()
1320
+ for candidate in root.rglob(f"*{session_id}*.jsonl")
1321
+ ]
1322
+ if not matches:
1323
+ raise RuntimeError(f"session transcript not found: {session_id}")
1324
+ path = max(matches, key=lambda item: (item.stat().st_size, item.stat().st_mtime))
1325
+ cursor_before = int(audit.get("cursor_before", 0))
1326
+ cursor_after = int(audit.get("cursor_after") or audit.get("raw_size") or 0)
1327
+ parsed = parse_codex_session(path, cursor_before)
1328
+ if cursor_after <= cursor_before or cursor_after > parsed["size"]:
1329
+ raise RuntimeError(
1330
+ f"invalid replay cursor for {session_id}: {cursor_before}..{cursor_after}, size={parsed['size']}"
1331
+ )
1332
+ parsed["messages"] = [
1333
+ item for item in parsed["messages"]
1334
+ if item["start_offset"] >= cursor_before and item["end_offset"] <= cursor_after
1335
+ ]
1336
+ parsed["size"] = cursor_after
1337
+ parsed["end_offset"] = cursor_after
1338
+ parsed["session_usage"] = audit.get("session_usage") or parsed.get("session_usage")
1339
+ meta = dict(parsed.get("meta") or {})
1340
+ if audit.get("cwd"):
1341
+ meta["cwd"] = audit["cwd"]
1342
+ parsed["meta"] = meta
1343
+ target, target_error = resolve_target(meta.get("cwd"))
1344
+ return {
1345
+ **parsed,
1346
+ "session_id": session_id,
1347
+ "cursor_before": cursor_before,
1348
+ "new_bytes": cursor_after - cursor_before,
1349
+ "target": target,
1350
+ "status": "candidate" if not target_error else "skipped",
1351
+ "reason": target_error,
1352
+ "replay_source": {
1353
+ "run_id": source_run_id,
1354
+ "previous_status": audit.get("status"),
1355
+ "previous_apply_empty": not bool((audit.get("apply_result") or {}).get("touched_targets")),
1356
+ },
1357
+ }
1358
+
1359
+
1360
+ def command_replay(args):
1361
+ config = load_config()
1362
+ validation = validate_config(config)
1363
+ if not validation["valid"]:
1364
+ raise RuntimeError("; ".join(validation["errors"]))
1365
+ source_path = SCAN_ROOT / "runs" / f"{args.run_id}.json"
1366
+ source_run = _read_json(source_path)
1367
+ if not isinstance(source_run, dict):
1368
+ raise RuntimeError(f"run not found: {args.run_id}")
1369
+ requested = list(dict.fromkeys(args.session_id))
1370
+ audits = {item.get("session_id"): item for item in source_run.get("sessions", [])}
1371
+ missing = [session_id for session_id in requested if session_id not in audits]
1372
+ if missing:
1373
+ raise RuntimeError(f"sessions not found in run {args.run_id}: {', '.join(missing)}")
1374
+ sessions = [_replay_session(args.run_id, audits[session_id]) for session_id in requested]
1375
+ run_id = dt.datetime.now().strftime("%Y%m%dT%H%M%S") + f"-replay-{os.getpid()}"
1376
+ return _execute_sessions(
1377
+ config,
1378
+ args,
1379
+ sessions,
1380
+ run_id,
1381
+ time.time(),
1382
+ run_kind="replay",
1383
+ replay_source={"run_id": args.run_id, "session_ids": requested},
1384
+ )
1385
+
1386
+
1387
+ def _cli_path():
1388
+ explicit = os.environ.get("DEV_MEMORY_CLI_PATH", "").strip()
1389
+ if explicit:
1390
+ return str(Path(explicit).expanduser().resolve())
1391
+ installed = shutil.which("dev-memory-cli")
1392
+ return installed or str(PACKAGE_ROOT / "bin" / "dev-memory.js")
1393
+
1394
+
1395
+ def command_install(_args):
1396
+ config = load_config()
1397
+ save_scan_config(config)
1398
+ logs = SCAN_ROOT / "logs"
1399
+ logs.mkdir(parents=True, exist_ok=True)
1400
+ plist = {
1401
+ "Label": "com.dev-memory.session-scan",
1402
+ "ProgramArguments": [_cli_path(), "session-scan", "run", "--scheduled"],
1403
+ "StartCalendarInterval": _calendar_intervals(config),
1404
+ "RunAtLoad": False,
1405
+ "StandardOutPath": str(logs / "launchd.stdout.log"),
1406
+ "StandardErrorPath": str(logs / "launchd.stderr.log"),
1407
+ "EnvironmentVariables": {"PATH": os.environ.get("PATH", "/usr/local/bin:/usr/bin:/bin")},
1408
+ }
1409
+ PLIST_PATH.parent.mkdir(parents=True, exist_ok=True)
1410
+ with PLIST_PATH.open("wb") as stream:
1411
+ plistlib.dump(plist, stream)
1412
+ subprocess.run(["launchctl", "unload", str(PLIST_PATH)], capture_output=True, check=False)
1413
+ result = subprocess.run(["launchctl", "load", str(PLIST_PATH)], capture_output=True, text=True, check=False)
1414
+ if result.returncode != 0:
1415
+ raise RuntimeError(result.stderr.strip() or "launchctl load failed")
1416
+ print(json.dumps({
1417
+ "installed": True,
1418
+ "plist": str(PLIST_PATH),
1419
+ "schedule_times": config["schedule_times"],
1420
+ "skip_when_computer_active": config["skip_when_computer_active"],
1421
+ "active_within_minutes": config["active_within_minutes"],
1422
+ "logs": str(logs),
1423
+ }, ensure_ascii=False, indent=2))
1424
+
1425
+
1426
+ def command_uninstall(_args):
1427
+ subprocess.run(["launchctl", "unload", str(PLIST_PATH)], capture_output=True, check=False)
1428
+ PLIST_PATH.unlink(missing_ok=True)
1429
+ print(json.dumps({"installed": False, "plist": str(PLIST_PATH)}, ensure_ascii=False, indent=2))
1430
+
1431
+
1432
+ def command_status(_args):
1433
+ config = load_config()
1434
+ validation = validate_config(config)
1435
+ try:
1436
+ executor, preset = choose_executor(config)
1437
+ except RuntimeError:
1438
+ executor, preset = None, None
1439
+ print(json.dumps({
1440
+ "installed": PLIST_PATH.exists(),
1441
+ "plist": str(PLIST_PATH),
1442
+ "scan_root": str(SCAN_ROOT),
1443
+ "codex_sessions": str(CODEX_HOME / "sessions"),
1444
+ "executor": executor,
1445
+ "model": preset.get("model") if preset else None,
1446
+ "schedule_times": config.get("schedule_times"),
1447
+ "skip_when_computer_active": config.get("skip_when_computer_active"),
1448
+ "active_within_minutes": config.get("active_within_minutes"),
1449
+ "config": validation,
1450
+ "last_run": _read_json(SCAN_ROOT / "last-run.json"),
1451
+ }, ensure_ascii=False, indent=2))
1452
+
1453
+
1454
+ def command_config(args):
1455
+ config = load_config()
1456
+ if args.config_command == "show":
1457
+ print(json.dumps(config, ensure_ascii=False, indent=2))
1458
+ return
1459
+ if args.config_command == "validate":
1460
+ result = validate_config(config)
1461
+ print(json.dumps(result, ensure_ascii=False, indent=2))
1462
+ if not result["valid"]:
1463
+ raise SystemExit(1)
1464
+ return
1465
+ if args.config_command == "set-executor":
1466
+ if args.executor != "auto" and args.executor not in config["executors"]:
1467
+ raise RuntimeError(f"unknown executor preset: {args.executor}")
1468
+ config["executor"] = args.executor
1469
+ elif args.config_command in {"set-model", "set-profile"}:
1470
+ if args.executor not in config["executors"]:
1471
+ raise RuntimeError(f"unknown executor preset: {args.executor}")
1472
+ field = "model" if args.config_command == "set-model" else "profile"
1473
+ config["executors"][args.executor][field] = args.value
1474
+ elif args.config_command == "set-schedule":
1475
+ for value in args.times:
1476
+ _parse_schedule_time(value)
1477
+ config["schedule_times"] = args.times
1478
+ elif args.config_command == "set-active-minutes":
1479
+ if args.minutes < 1:
1480
+ raise RuntimeError("active minutes must be at least 1")
1481
+ config["active_within_minutes"] = args.minutes
1482
+ elif args.config_command == "set-timeout":
1483
+ if args.seconds < 30:
1484
+ raise RuntimeError("timeout seconds must be at least 30")
1485
+ config["invocation_timeout_seconds"] = args.seconds
1486
+ elif args.config_command == "set-active-check":
1487
+ config["skip_when_computer_active"] = args.enabled == "on"
1488
+ save_scan_config(config)
1489
+ if args.config_command == "set-schedule" and PLIST_PATH.exists():
1490
+ command_install(args)
1491
+ return
1492
+ print(json.dumps(config, ensure_ascii=False, indent=2))
1493
+
1494
+
1495
+ def build_parser():
1496
+ parser = argparse.ArgumentParser(description="Scan Codex sessions into dev-memory")
1497
+ sub = parser.add_subparsers(dest="command", required=True)
1498
+ run = sub.add_parser("run")
1499
+ run.add_argument("--since")
1500
+ run.add_argument("--dry-run", action="store_true")
1501
+ run.add_argument("--json", action="store_true")
1502
+ run.add_argument("--scheduled", action="store_true", help="Apply computer activity guard before scanning")
1503
+ run.add_argument("--executor", help="Override the configured executor for this run only")
1504
+ sub.add_parser("install")
1505
+ sub.add_parser("status")
1506
+ stats = sub.add_parser("stats")
1507
+ stats.add_argument("--repo")
1508
+ stats.add_argument("--since")
1509
+ stats.add_argument("--json", action="store_true")
1510
+ history = sub.add_parser("history")
1511
+ history.add_argument("--repo")
1512
+ history.add_argument("--limit", type=int, default=20)
1513
+ history.add_argument("--json", action="store_true")
1514
+ show = sub.add_parser("show")
1515
+ show.add_argument("run_id")
1516
+ replay = sub.add_parser("replay")
1517
+ replay.add_argument("--run-id", required=True)
1518
+ replay.add_argument("--session-id", action="append", required=True)
1519
+ replay.add_argument("--dry-run", action="store_true")
1520
+ replay.add_argument("--json", action="store_true")
1521
+ replay.add_argument("--executor", help="Override the configured executor for this replay only")
1522
+ sub.add_parser("uninstall")
1523
+ config = sub.add_parser("config")
1524
+ config_sub = config.add_subparsers(dest="config_command", required=True)
1525
+ config_sub.add_parser("show")
1526
+ config_sub.add_parser("validate")
1527
+ set_executor = config_sub.add_parser("set-executor")
1528
+ set_executor.add_argument("executor")
1529
+ for command in ("set-model", "set-profile"):
1530
+ item = config_sub.add_parser(command)
1531
+ item.add_argument("executor")
1532
+ item.add_argument("value")
1533
+ set_schedule = config_sub.add_parser("set-schedule")
1534
+ set_schedule.add_argument("times", nargs="+")
1535
+ set_active_minutes = config_sub.add_parser("set-active-minutes")
1536
+ set_active_minutes.add_argument("minutes", type=int)
1537
+ set_timeout = config_sub.add_parser("set-timeout")
1538
+ set_timeout.add_argument("seconds", type=int)
1539
+ set_active_check = config_sub.add_parser("set-active-check")
1540
+ set_active_check.add_argument("enabled", choices=("on", "off"))
1541
+ return parser
1542
+
1543
+
1544
+ def main():
1545
+ args = build_parser().parse_args()
1546
+ handlers = {
1547
+ "run": run_scan,
1548
+ "install": command_install,
1549
+ "status": command_status,
1550
+ "stats": command_stats,
1551
+ "history": command_history,
1552
+ "show": command_show,
1553
+ "replay": command_replay,
1554
+ "uninstall": command_uninstall,
1555
+ "config": command_config,
1556
+ }
1557
+ try:
1558
+ return handlers[args.command](args) or 0
1559
+ except RuntimeError as exc:
1560
+ print(f"ERROR: {exc}", file=sys.stderr)
1561
+ return 1
1562
+
1563
+
1564
+ if __name__ == "__main__":
1565
+ raise SystemExit(main())