pakem-cli 0.0.0-stage → 1.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (76) hide show
  1. package/CHANGELOG.md +280 -0
  2. package/LICENSE +21 -0
  3. package/README-ID.md +666 -0
  4. package/README.md +665 -2
  5. package/SKILL.md +264 -0
  6. package/adapters/git-worktree/v1/README.md +17 -0
  7. package/adapters/identity/v1/README.md +26 -0
  8. package/adapters/mcp/v1/README.md +56 -0
  9. package/adapters/storage-sqlite/v1/README.md +5 -0
  10. package/assets/banner.svg +275 -0
  11. package/assets/divider.svg +82 -0
  12. package/assets/footer.svg +174 -0
  13. package/assets/templates/BUG-HISTORY.md +18 -0
  14. package/assets/templates/CONTEXT.md +26 -0
  15. package/assets/templates/DECISIONS.md +8 -0
  16. package/assets/templates/MODULE-MAP.md +14 -0
  17. package/assets/templates/POLICY.json +15 -0
  18. package/assets/templates/PROJECT.md +42 -0
  19. package/conformance/v1/README.md +15 -0
  20. package/coordination/v1/README.md +14 -0
  21. package/coordination/v1/board.schema.json +59 -0
  22. package/coordination/v1/event.schema.json +26 -0
  23. package/coordination/v1/status.schema.json +49 -0
  24. package/evals/README.md +63 -0
  25. package/evals/evals.json +276 -0
  26. package/identity/v1/README.md +14 -0
  27. package/identity/v1/agent-identity.schema.json +95 -0
  28. package/identity/v1/identity-registry.schema.json +29 -0
  29. package/identity/v1/signed-payload.schema.json +42 -0
  30. package/intelligence/v1/README.md +19 -0
  31. package/intelligence/v1/dependency-graph.schema.json +82 -0
  32. package/intelligence/v1/impact-analysis.schema.json +51 -0
  33. package/intelligence/v1/reconciliation.schema.json +156 -0
  34. package/npm/bin/pakem.js +272 -0
  35. package/npm/check-version.js +26 -0
  36. package/package.json +58 -4
  37. package/policy/v1/README.md +15 -0
  38. package/policy/v1/policy.schema.json +20 -0
  39. package/protocol/v1/README.md +68 -0
  40. package/protocol/v1/capabilities.schema.json +17 -0
  41. package/protocol/v1/event.schema.json +114 -0
  42. package/protocol/v1/session.schema.json +51 -0
  43. package/references/artifacts.md +96 -0
  44. package/references/conventions.md +83 -0
  45. package/references/reviews.md +44 -0
  46. package/runtime/v1/README.md +21 -0
  47. package/runtime/v1/approval.schema.json +100 -0
  48. package/runtime/v1/checkpoint.schema.json +17 -0
  49. package/runtime/v1/hook-decision.schema.json +34 -0
  50. package/runtime/v1/hook-payload.schema.json +12 -0
  51. package/runtime/v1/task-evidence.schema.json +38 -0
  52. package/runtime/v1/work-report.schema.json +123 -0
  53. package/runtime/v1/workflow.schema.json +21 -0
  54. package/scripts/__pycache__/pakem.cpython-312.pyc +0 -0
  55. package/scripts/pakem.py +4911 -0
  56. package/scripts/pakem_core/__init__.py +3 -0
  57. package/scripts/pakem_core/__pycache__/__init__.cpython-312.pyc +0 -0
  58. package/scripts/pakem_core/__pycache__/coordination.cpython-312.pyc +0 -0
  59. package/scripts/pakem_core/__pycache__/git_worktree.cpython-312.pyc +0 -0
  60. package/scripts/pakem_core/__pycache__/identity.cpython-312.pyc +0 -0
  61. package/scripts/pakem_core/__pycache__/intelligence.cpython-312.pyc +0 -0
  62. package/scripts/pakem_core/__pycache__/mcp_server.cpython-312.pyc +0 -0
  63. package/scripts/pakem_core/__pycache__/protocol.cpython-312.pyc +0 -0
  64. package/scripts/pakem_core/__pycache__/reconcile.cpython-312.pyc +0 -0
  65. package/scripts/pakem_core/__pycache__/runtime.cpython-312.pyc +0 -0
  66. package/scripts/pakem_core/coordination.py +977 -0
  67. package/scripts/pakem_core/git_worktree.py +265 -0
  68. package/scripts/pakem_core/identity.py +494 -0
  69. package/scripts/pakem_core/intelligence.py +684 -0
  70. package/scripts/pakem_core/mcp_server.py +308 -0
  71. package/scripts/pakem_core/protocol.py +61 -0
  72. package/scripts/pakem_core/reconcile.py +119 -0
  73. package/scripts/pakem_core/runtime.py +457 -0
  74. package/scripts/run_evals.py +289 -0
  75. package/storage/v1/README.md +14 -0
  76. package/storage/v1/team-storage.schema.json +27 -0
@@ -0,0 +1,289 @@
1
+ #!/usr/bin/env python3
2
+ """Jalankan evals Pakem: satu sesi agent per prompt, lalu nilai assertion-nya.
3
+
4
+ Runner ini tidak tahu apa-apa soal model. Ia menyiapkan direktori bersih, memberi
5
+ prompt ke perintah agent lewat stdin, merekam transkrip, lalu menyerahkan penilaian
6
+ ke perintah judge. Tanpa judge, transkrip tetap direkam tapi assertion tidak dinilai.
7
+
8
+ Contoh:
9
+
10
+ python3 scripts/run_evals.py --list
11
+ python3 scripts/run_evals.py --validate
12
+
13
+ python3 scripts/run_evals.py --agent-cmd "hermes run --skill ." \\
14
+ --judge-cmd "python3 judge.py" --out evals-results.json
15
+
16
+ Kontrak agent-cmd: membaca prompt dari stdin, bekerja di cwd (direktori sandbox),
17
+ boleh menulis apa pun. exit code dicatat, bukan dinilai.
18
+
19
+ Kontrak judge-cmd: menerima satu objek JSON di stdin (lihat build_judge_payload),
20
+ menulis satu objek JSON di stdout:
21
+
22
+ {"results": [{"assertion": "...", "pass": true, "why": "..."}]}
23
+
24
+ Setiap assertion pada eval harus muncul tepat sekali. Yang tidak dinilai dihitung gagal.
25
+ """
26
+ from __future__ import annotations
27
+
28
+ import argparse
29
+ import json
30
+ import shlex
31
+ import subprocess
32
+ import sys
33
+ import tempfile
34
+ import time
35
+ from pathlib import Path
36
+
37
+ HERE = Path(__file__).resolve().parent
38
+ EVALS_FILE = HERE.parent / "evals" / "evals.json"
39
+ MAX_TRANSCRIPT = 20_000
40
+ MAX_LISTED_FILES = 200
41
+
42
+
43
+ def load_evals(path: Path = EVALS_FILE) -> dict:
44
+ return json.loads(path.read_text(encoding="utf-8"))
45
+
46
+
47
+ def validate(doc: dict) -> list[str]:
48
+ """Cek struktural. Mengembalikan daftar masalah; kosong berarti sehat."""
49
+ problems = []
50
+ if not isinstance(doc, dict):
51
+ return ["evals.json harus berupa object"]
52
+ evals = doc.get("evals")
53
+ if not isinstance(evals, list) or not evals:
54
+ return ["evals harus berupa array yang tidak kosong"]
55
+ seen = set()
56
+ for index, item in enumerate(evals):
57
+ label = f"eval[{index}]"
58
+ if not isinstance(item, dict):
59
+ problems.append(f"{label}: bukan object")
60
+ continue
61
+ for key in ("id", "prompt", "expected_output", "assertions"):
62
+ if key not in item:
63
+ problems.append(f"{label}: '{key}' tidak ada")
64
+ eval_id = item.get("id")
65
+ if eval_id in seen:
66
+ problems.append(f"{label}: id duplikat {eval_id!r}")
67
+ seen.add(eval_id)
68
+ if not str(item.get("prompt", "")).strip():
69
+ problems.append(f"{label}: prompt kosong")
70
+ assertions = item.get("assertions")
71
+ if not isinstance(assertions, list) or not assertions:
72
+ problems.append(f"{label}: assertions harus array tidak kosong")
73
+ elif any(not str(a).strip() for a in assertions):
74
+ problems.append(f"{label}: ada assertion kosong")
75
+ return problems
76
+
77
+
78
+ def snapshot(directory: Path) -> list[dict]:
79
+ """Daftar file di sandbox, supaya judge bisa melihat hasil kerja agent."""
80
+ files = []
81
+ for path in sorted(directory.rglob("*")):
82
+ if path.is_file() and len(files) < MAX_LISTED_FILES:
83
+ try:
84
+ size = path.stat().st_size
85
+ except OSError:
86
+ continue
87
+ files.append({"path": path.relative_to(directory).as_posix(), "bytes": size})
88
+ return files
89
+
90
+
91
+ def run_agent(command: str, prompt: str, cwd: Path, timeout: int) -> dict:
92
+ started = time.time()
93
+ try:
94
+ result = subprocess.run(shlex.split(command), input=prompt, cwd=cwd, text=True,
95
+ capture_output=True, timeout=timeout, check=False)
96
+ return {"exit_code": result.returncode,
97
+ "stdout": result.stdout or "",
98
+ "stderr": result.stderr or "",
99
+ "seconds": round(time.time() - started, 1),
100
+ "timed_out": False}
101
+ except subprocess.TimeoutExpired as exc:
102
+ return {"exit_code": None,
103
+ "stdout": (exc.stdout or "") if isinstance(exc.stdout, str) else "",
104
+ "stderr": (exc.stderr or "") if isinstance(exc.stderr, str) else "",
105
+ "seconds": round(time.time() - started, 1),
106
+ "timed_out": True}
107
+ except OSError as exc:
108
+ return {"exit_code": None, "stdout": "", "stderr": f"agent tidak bisa dijalankan: {exc}",
109
+ "seconds": 0.0, "timed_out": False}
110
+
111
+
112
+ def build_judge_payload(item: dict, run: dict, files: list[dict]) -> dict:
113
+ transcript = (run["stdout"] + "\n" + run["stderr"]).strip()
114
+ return {
115
+ "eval_id": item.get("id"),
116
+ "prompt": item.get("prompt"),
117
+ "expected_output": item.get("expected_output"),
118
+ "assertions": item.get("assertions", []),
119
+ "exit_code": run["exit_code"],
120
+ "timed_out": run["timed_out"],
121
+ "transcript": transcript[:MAX_TRANSCRIPT],
122
+ "transcript_truncated": len(transcript) > MAX_TRANSCRIPT,
123
+ "files": files,
124
+ }
125
+
126
+
127
+ def judge(command: str, payload: dict, timeout: int) -> tuple[list[dict], str]:
128
+ """Kembalikan (results, error). error kosong berarti judge berhasil."""
129
+ try:
130
+ result = subprocess.run(shlex.split(command), input=json.dumps(payload), text=True,
131
+ capture_output=True, timeout=timeout, check=False)
132
+ except subprocess.TimeoutExpired:
133
+ return [], "judge timeout"
134
+ except OSError as exc:
135
+ return [], f"judge tidak bisa dijalankan: {exc}"
136
+ if result.returncode != 0:
137
+ return [], f"judge exit {result.returncode}: {(result.stderr or '').strip()[:300]}"
138
+ try:
139
+ data = json.loads(result.stdout)
140
+ except json.JSONDecodeError:
141
+ return [], "output judge bukan JSON"
142
+ results = data.get("results")
143
+ if not isinstance(results, list):
144
+ return [], "judge harus mengembalikan {\"results\": [...]}"
145
+ clean = []
146
+ for entry in results:
147
+ if not isinstance(entry, dict) or "assertion" not in entry:
148
+ continue
149
+ clean.append({"assertion": str(entry["assertion"]),
150
+ "pass": bool(entry.get("pass")),
151
+ "why": str(entry.get("why", ""))[:300]})
152
+ return clean, ""
153
+
154
+
155
+ def grade(item: dict, results: list[dict], error: str) -> list[dict]:
156
+ """Pastikan setiap assertion dinilai; yang hilang dihitung gagal."""
157
+ wanted = [str(a) for a in item.get("assertions", [])]
158
+ by_text = {r["assertion"]: r for r in results}
159
+ graded = []
160
+ for assertion in wanted:
161
+ if assertion in by_text:
162
+ graded.append({"assertion": assertion, "pass": by_text[assertion]["pass"],
163
+ "why": by_text[assertion]["why"]})
164
+ else:
165
+ graded.append({"assertion": assertion, "pass": False,
166
+ "why": error or "judge tidak menilai assertion ini"})
167
+ return graded
168
+
169
+
170
+ def run_one(item: dict, args, workdir: Path) -> dict:
171
+ sandbox = Path(tempfile.mkdtemp(prefix=f"pakem-eval-{item.get('id')}-", dir=workdir))
172
+ run = run_agent(args.agent_cmd, str(item.get("prompt", "")), sandbox, args.timeout)
173
+ files = snapshot(sandbox)
174
+ report = {"eval_id": item.get("id"), "sandbox": str(sandbox), "exit_code": run["exit_code"],
175
+ "timed_out": run["timed_out"], "seconds": run["seconds"],
176
+ "files_written": len(files), "judged": bool(args.judge_cmd),
177
+ "stderr_head": run["stderr"].strip()[:300]}
178
+ if args.judge_cmd:
179
+ results, error = judge(args.judge_cmd, build_judge_payload(item, run, files), args.judge_timeout)
180
+ graded = grade(item, results, error)
181
+ report["results"] = graded
182
+ report["passed"] = sum(1 for g in graded if g["pass"])
183
+ report["total"] = len(graded)
184
+ report["ok"] = report["passed"] == report["total"]
185
+ if error:
186
+ report["judge_error"] = error
187
+ else:
188
+ report["ok"] = None
189
+ report["note"] = "tanpa --judge-cmd: transkrip direkam, assertion tidak dinilai"
190
+ if args.keep or not args.judge_cmd:
191
+ for name, text in (("stdout.txt", run["stdout"]), ("stderr.txt", run["stderr"]),
192
+ ("report.json", json.dumps({**report, "files": files}, ensure_ascii=False, indent=2))):
193
+ (sandbox / name).write_text(text, encoding="utf-8")
194
+ else:
195
+ (sandbox.parent / f"{sandbox.name}.report.json").write_text(
196
+ json.dumps({**report, "files": files}, ensure_ascii=False, indent=2), encoding="utf-8")
197
+ return report
198
+
199
+
200
+ def select(doc: dict, wanted: list[str]) -> list[dict]:
201
+ evals = doc["evals"]
202
+ if not wanted:
203
+ return evals
204
+ chosen = []
205
+ for token in wanted:
206
+ for item in evals:
207
+ if str(item.get("id")) == str(token):
208
+ chosen.append(item)
209
+ break
210
+ else:
211
+ raise SystemExit(f"eval {token} tidak ditemukan")
212
+ return chosen
213
+
214
+
215
+ def main(argv=None) -> int:
216
+ parser = argparse.ArgumentParser(prog="run_evals", description="Jalankan evals Pakem.")
217
+ parser.add_argument("--file", default=str(EVALS_FILE))
218
+ parser.add_argument("--list", action="store_true", help="tampilkan eval yang tersedia")
219
+ parser.add_argument("--validate", action="store_true", help="cek struktur evals.json")
220
+ parser.add_argument("--eval", dest="only", action="append", default=[], help="id eval; dapat diulang")
221
+ parser.add_argument("--agent-cmd", help="perintah agent; prompt dikirim lewat stdin")
222
+ parser.add_argument("--judge-cmd", help="perintah penilai; menerima JSON di stdin")
223
+ parser.add_argument("--timeout", type=int, default=600, help="batas detik per eval (default 600)")
224
+ parser.add_argument("--judge-timeout", type=int, default=180)
225
+ parser.add_argument("--workdir", help="direktori induk untuk sandbox (default: temp)")
226
+ parser.add_argument("--keep", action="store_true", help="simpan sandbox beserta transkrip")
227
+ parser.add_argument("--out", help="tulis laporan JSON ke path ini")
228
+ parser.add_argument("--json", action="store_true", help="cetak laporan JSON ke stdout")
229
+ args = parser.parse_args(argv)
230
+
231
+ path = Path(args.file).expanduser()
232
+ if not path.is_file():
233
+ print(f"run_evals: {path} tidak ada", file=sys.stderr)
234
+ return 2
235
+ doc = load_evals(path)
236
+
237
+ if args.validate or not (args.list or args.agent_cmd):
238
+ problems = validate(doc)
239
+ if problems:
240
+ for problem in problems:
241
+ print(f"[FAIL] {problem}")
242
+ return 1
243
+ print(f"evals: struktur sehat ({len(doc['evals'])} eval, "
244
+ f"{sum(len(e['assertions']) for e in doc['evals'])} assertion)")
245
+ if not (args.list or args.agent_cmd):
246
+ print("Jalankan dengan --agent-cmd untuk mengeksekusi, atau --list untuk melihat daftar.")
247
+ return 0
248
+
249
+ if args.list:
250
+ for item in doc["evals"]:
251
+ prompt = " ".join(str(item["prompt"]).split())
252
+ print(f"{str(item['id']):>3} {prompt[:88]}")
253
+ return 0
254
+
255
+ workdir = Path(args.workdir).expanduser() if args.workdir else Path(tempfile.mkdtemp(prefix="pakem-evals-"))
256
+ workdir.mkdir(parents=True, exist_ok=True)
257
+ reports = [run_one(item, args, workdir) for item in select(doc, args.only)]
258
+
259
+ if args.json:
260
+ print(json.dumps({"workdir": str(workdir), "reports": reports}, ensure_ascii=False, indent=2))
261
+ else:
262
+ for report in reports:
263
+ if report["judged"]:
264
+ mark = "PASS" if report["ok"] else "FAIL"
265
+ print(f"[{mark}] eval {report['eval_id']}: {report['passed']}/{report['total']} assertion "
266
+ f"(exit {report['exit_code']}, {report['seconds']}s)")
267
+ for result in report["results"]:
268
+ if not result["pass"]:
269
+ print(f" x {result['assertion']} — {result['why']}")
270
+ else:
271
+ print(f"[REKAM] eval {report['eval_id']}: exit {report['exit_code']}, "
272
+ f"{report['files_written']} file, {report['seconds']}s — assertion tidak dinilai")
273
+ if report["timed_out"]:
274
+ print(" ! agent timeout")
275
+ elif report["exit_code"] not in (0, None) and report.get("stderr_head"):
276
+ print(f" ! agent exit {report['exit_code']}: {report['stderr_head']}")
277
+ print(f"\nSandbox: {workdir}")
278
+
279
+ if args.out:
280
+ Path(args.out).write_text(json.dumps({"workdir": str(workdir), "reports": reports},
281
+ ensure_ascii=False, indent=2), encoding="utf-8")
282
+
283
+ if not args.judge_cmd:
284
+ return 0
285
+ return 0 if all(report["ok"] for report in reports) else 1
286
+
287
+
288
+ if __name__ == "__main__":
289
+ sys.exit(main())
@@ -0,0 +1,14 @@
1
+ # Pakem Team Storage v1
2
+
3
+ The default backend remains `jsonl` for compatibility with older projects. Projects can explicitly migrate to the local SQLite backend:
4
+
5
+ ```bash
6
+ pakem team storage info
7
+ pakem team storage migrate --to sqlite
8
+ ```
9
+
10
+ SQLite stores a serialized team board and append-only protocol event records in `.pakem/team/team.sqlite3`. Each state mutation updates the board and inserts its corresponding event in one SQLite transaction. WAL mode, `synchronous=FULL`, and a busy timeout are enabled.
11
+
12
+ Migration runs under the existing project-local coordination lock. Original `board.json` and `events.jsonl` files are retained as backups; after migration, SQLite is authoritative and the old files are not updated. Back up the whole `.pakem/team/` directory before migration.
13
+
14
+ **Scope:** this backend is for a local project directory or a filesystem with reliable SQLite locking. It is not a distributed database, does not provide multi-host consensus, and should not be placed on filesystems that do not support SQLite locking/WAL semantics. For agents on separate hosts, deploy a future authenticated server-side coordination backend.
@@ -0,0 +1,27 @@
1
+ {
2
+ "$schema": "https://json-schema.org/draft/2020-12/schema",
3
+ "$id": "urn:pakem:storage:v1:team-storage",
4
+ "title": "Pakem Team Storage Configuration v1",
5
+ "type": "object",
6
+ "required": [
7
+ "schemaVersion",
8
+ "backend",
9
+ "backupFilesRetained"
10
+ ],
11
+ "properties": {
12
+ "schemaVersion": {
13
+ "const": 1
14
+ },
15
+ "backend": {
16
+ "const": "sqlite"
17
+ },
18
+ "migratedAt": {
19
+ "type": "string",
20
+ "format": "date-time"
21
+ },
22
+ "backupFilesRetained": {
23
+ "const": true
24
+ }
25
+ },
26
+ "additionalProperties": false
27
+ }