@suifei/dsh-ai-team 0.0.0-stage → 0.3.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (84) hide show
  1. package/CHANGELOG.md +104 -0
  2. package/LICENSE +21 -0
  3. package/README.md +226 -2
  4. package/client.js +2901 -0
  5. package/cordis.patch.yml +31 -0
  6. package/icon.svg +27 -0
  7. package/index.js +893 -0
  8. package/lib/domain/errors.js +36 -0
  9. package/lib/domain/ids.js +26 -0
  10. package/lib/domain/inbox.js +87 -0
  11. package/lib/domain/member-config.js +168 -0
  12. package/lib/domain/member-profile.js +53 -0
  13. package/lib/domain/policy.js +102 -0
  14. package/lib/domain/task.js +309 -0
  15. package/lib/domain/thread.js +54 -0
  16. package/lib/host/fault-adapter.js +50 -0
  17. package/lib/host/member-tools.js +225 -0
  18. package/lib/host/permission-guard.js +28 -0
  19. package/lib/host/project-registry.js +68 -0
  20. package/lib/host/project-runtime.js +73 -0
  21. package/lib/host/service.js +810 -0
  22. package/lib/host/views.js +213 -0
  23. package/lib/host/waker.js +177 -0
  24. package/lib/ledger/checkpoint.js +395 -0
  25. package/lib/ledger/events.js +398 -0
  26. package/lib/ledger/fold.js +991 -0
  27. package/lib/ledger/writer.js +983 -0
  28. package/lib/runtime/failure-policy.js +113 -0
  29. package/lib/runtime/host-agent.js +101 -0
  30. package/lib/runtime/member-runtime.js +286 -0
  31. package/lib/runtime/skill-catalog.js +27 -0
  32. package/lib/runtime/skill-loader.js +26 -0
  33. package/lib/runtime/skill-provider.js +121 -0
  34. package/lib/vendor/skillopt/LICENSE +21 -0
  35. package/lib/vendor/skillopt/requirements.txt +31 -0
  36. package/lib/vendor/skillopt/scripts/sleep.py +119 -0
  37. package/lib/vendor/skillopt/skillopt_sleep/__init__.py +20 -0
  38. package/lib/vendor/skillopt/skillopt_sleep/__main__.py +942 -0
  39. package/lib/vendor/skillopt/skillopt_sleep/adapters/__init__.py +1 -0
  40. package/lib/vendor/skillopt/skillopt_sleep/adapters/superpowers.py +1379 -0
  41. package/lib/vendor/skillopt/skillopt_sleep/backend.py +2844 -0
  42. package/lib/vendor/skillopt/skillopt_sleep/budget.py +75 -0
  43. package/lib/vendor/skillopt/skillopt_sleep/config.py +257 -0
  44. package/lib/vendor/skillopt/skillopt_sleep/consolidate.py +558 -0
  45. package/lib/vendor/skillopt/skillopt_sleep/cycle.py +1270 -0
  46. package/lib/vendor/skillopt/skillopt_sleep/dream.py +173 -0
  47. package/lib/vendor/skillopt/skillopt_sleep/evalkit.py +718 -0
  48. package/lib/vendor/skillopt/skillopt_sleep/evidence.py +143 -0
  49. package/lib/vendor/skillopt/skillopt_sleep/experiments/__init__.py +1 -0
  50. package/lib/vendor/skillopt/skillopt_sleep/experiments/gbrain_bench.py +119 -0
  51. package/lib/vendor/skillopt/skillopt_sleep/experiments/personas.py +86 -0
  52. package/lib/vendor/skillopt/skillopt_sleep/experiments/report.py +132 -0
  53. package/lib/vendor/skillopt/skillopt_sleep/experiments/run_experiment.py +178 -0
  54. package/lib/vendor/skillopt/skillopt_sleep/experiments/run_gbrain.py +209 -0
  55. package/lib/vendor/skillopt/skillopt_sleep/experiments/run_transfer.py +155 -0
  56. package/lib/vendor/skillopt/skillopt_sleep/experiments/sweep.py +164 -0
  57. package/lib/vendor/skillopt/skillopt_sleep/gate.py +50 -0
  58. package/lib/vendor/skillopt/skillopt_sleep/handoff_backend.py +173 -0
  59. package/lib/vendor/skillopt/skillopt_sleep/harvest.py +691 -0
  60. package/lib/vendor/skillopt/skillopt_sleep/harvest_codex.py +260 -0
  61. package/lib/vendor/skillopt/skillopt_sleep/harvest_copilot.py +443 -0
  62. package/lib/vendor/skillopt/skillopt_sleep/harvest_copilot_cli.py +244 -0
  63. package/lib/vendor/skillopt/skillopt_sleep/harvest_cursor.py +316 -0
  64. package/lib/vendor/skillopt/skillopt_sleep/harvest_opencode.py +364 -0
  65. package/lib/vendor/skillopt/skillopt_sleep/harvest_pi.py +311 -0
  66. package/lib/vendor/skillopt/skillopt_sleep/harvest_sources.py +135 -0
  67. package/lib/vendor/skillopt/skillopt_sleep/judges.py +344 -0
  68. package/lib/vendor/skillopt/skillopt_sleep/llm_miner.py +187 -0
  69. package/lib/vendor/skillopt/skillopt_sleep/memory.py +166 -0
  70. package/lib/vendor/skillopt/skillopt_sleep/mine.py +427 -0
  71. package/lib/vendor/skillopt/skillopt_sleep/multi_skill.py +190 -0
  72. package/lib/vendor/skillopt/skillopt_sleep/prompts.py +250 -0
  73. package/lib/vendor/skillopt/skillopt_sleep/replay.py +211 -0
  74. package/lib/vendor/skillopt/skillopt_sleep/rollout.py +184 -0
  75. package/lib/vendor/skillopt/skillopt_sleep/scheduler.py +286 -0
  76. package/lib/vendor/skillopt/skillopt_sleep/skill_resolver.py +244 -0
  77. package/lib/vendor/skillopt/skillopt_sleep/slow_update.py +141 -0
  78. package/lib/vendor/skillopt/skillopt_sleep/staging.py +3285 -0
  79. package/lib/vendor/skillopt/skillopt_sleep/state.py +283 -0
  80. package/lib/vendor/skillopt/skillopt_sleep/tasks_file.py +100 -0
  81. package/lib/vendor/skillopt/skillopt_sleep/types.py +227 -0
  82. package/lib/vendor/skillopt/src/index.js +394 -0
  83. package/package.json +62 -4
  84. package/skills/skillopt-sleep/SKILL.md +121 -0
@@ -0,0 +1,135 @@
1
+ """Source selection for SkillOpt-Sleep transcript harvesting."""
2
+ from __future__ import annotations
3
+
4
+ from typing import Optional
5
+
6
+ from skillopt_sleep.harvest import harvest
7
+ from skillopt_sleep.harvest_codex import harvest_codex
8
+ from skillopt_sleep.harvest_copilot import harvest_copilot
9
+ from skillopt_sleep.harvest_copilot_cli import harvest_copilot_cli
10
+ from skillopt_sleep.harvest_cursor import harvest_cursor
11
+ from skillopt_sleep.harvest_opencode import harvest_opencode
12
+ from skillopt_sleep.harvest_pi import harvest_pi
13
+ from skillopt_sleep.types import SessionDigest
14
+
15
+ SUPPORTED_SOURCES = (
16
+ "claude",
17
+ "codex",
18
+ "copilot",
19
+ "copilot_cli",
20
+ "cursor",
21
+ "pi",
22
+ "opencode",
23
+ "auto",
24
+ )
25
+
26
+
27
+ def normalize_transcript_source(value: object) -> str:
28
+ """Canonical name of a configured ``transcript_source``.
29
+
30
+ Unset or empty means ``claude``; otherwise case and surrounding
31
+ whitespace are ignored. The harvester, its per-source cursor and the
32
+ tasks document must all use this one value: a cursor keyed by the raw
33
+ spelling (``<project>::Codex``) is not the one the harvested source
34
+ owns, and skips its older sessions (#312).
35
+ """
36
+ supported = ", ".join(SUPPORTED_SOURCES)
37
+ if value is None:
38
+ return "claude"
39
+ if not isinstance(value, str):
40
+ raise ValueError(
41
+ f"transcript_source must be a string, not {type(value).__name__} "
42
+ f"{value!r}. Supported sources: {supported}"
43
+ )
44
+ source = (value or "claude").strip().lower()
45
+ if source not in SUPPORTED_SOURCES:
46
+ raise ValueError(
47
+ f"Unknown transcript source '{value}'. Supported sources: {supported}"
48
+ )
49
+ return source
50
+
51
+
52
+ def harvest_for_config(
53
+ cfg,
54
+ *,
55
+ since_iso: Optional[str] = None,
56
+ limit: int = 0,
57
+ stats: Optional[dict] = None,
58
+ ) -> list[SessionDigest]:
59
+ """Harvest with the configured source.
60
+
61
+ ``stats`` is filled in place by the Claude source (see
62
+ `skillopt_sleep.harvest.harvest`); the other sources have no repeat
63
+ filter to report yet and leave it untouched.
64
+ """
65
+ source = normalize_transcript_source(cfg.get("transcript_source", "claude"))
66
+ scope = cfg.get("projects", "invoked")
67
+ invoked_project = cfg.get("invoked_project", "")
68
+
69
+ if source == "codex":
70
+ return harvest_codex(
71
+ cfg.codex_archived_sessions_dir,
72
+ scope=scope,
73
+ invoked_project=invoked_project,
74
+ since_iso=since_iso,
75
+ limit=limit,
76
+ )
77
+ if source == "copilot":
78
+ return harvest_copilot(
79
+ cfg.vscode_workspace_storage,
80
+ scope=scope,
81
+ invoked_project=invoked_project,
82
+ since_iso=since_iso,
83
+ limit=limit,
84
+ )
85
+ if source == "copilot_cli":
86
+ return harvest_copilot_cli(
87
+ cfg.copilot_cli_session_store,
88
+ scope=scope,
89
+ invoked_project=invoked_project,
90
+ since_iso=since_iso,
91
+ limit=limit,
92
+ )
93
+ if source == "cursor":
94
+ return harvest_cursor(
95
+ cfg.cursor_projects_dir,
96
+ scope=scope,
97
+ invoked_project=invoked_project,
98
+ since_iso=since_iso,
99
+ limit=limit,
100
+ )
101
+ if source == "pi":
102
+ return harvest_pi(
103
+ cfg.pi_sessions_dir,
104
+ scope=scope,
105
+ invoked_project=invoked_project,
106
+ since_iso=since_iso,
107
+ limit=limit,
108
+ )
109
+ if source == "opencode":
110
+ return harvest_opencode(
111
+ cfg.opencode_db_path,
112
+ scope=scope,
113
+ invoked_project=invoked_project,
114
+ since_iso=since_iso,
115
+ limit=limit,
116
+ )
117
+ if source == "auto":
118
+ codex_digests = harvest_codex(
119
+ cfg.codex_archived_sessions_dir,
120
+ scope=scope,
121
+ invoked_project=invoked_project,
122
+ since_iso=since_iso,
123
+ limit=limit,
124
+ )
125
+ if codex_digests:
126
+ return codex_digests
127
+
128
+ return harvest(
129
+ cfg.transcripts_dir,
130
+ scope=scope,
131
+ invoked_project=invoked_project,
132
+ since_iso=since_iso,
133
+ limit=limit,
134
+ stats=stats,
135
+ )
@@ -0,0 +1,344 @@
1
+ """SkillOpt-Sleep — rule-based judges (gbrain-evals compatible).
2
+
3
+ Implements the programmatic check operators used by gbrain-evals'
4
+ skillopt-v1 benchmark so we can score skill outputs locally, with NO judge
5
+ API call:
6
+
7
+ * section_present <name> — legacy strict heading/bold/label check
8
+ * section_contains <name> — an ATX markdown heading contains literal <name>
9
+ * regex <pattern> — the pattern matches the response
10
+ * max_chars <n> — response length <= n
11
+ * min_chars <n> — response length >= n
12
+ * contains <text> — substring present (case-insensitive)
13
+ * not_contains <text> — substring absent (case-insensitive)
14
+ * no_refusal — the response is not a bare refusal/abstention
15
+ * tool_called <name> — a tool with <name> was invoked (needs a tool loop;
16
+ in single-shot replay we approximate via an
17
+ explicit "TOOL_CALL: <name>" marker the agent emits)
18
+
19
+ Ops divide into two families, and the distinction is load-bearing:
20
+
21
+ * *shape* ops (section_present, section_contains, max_chars, min_chars)
22
+ constrain how an answer is formatted. They are trivially satisfiable by an
23
+ optimizer — adding a heading scores 1.0 without the answer improving at all.
24
+ * *outcome* ops (contains, not_contains, no_refusal, regex, tool_called)
25
+ constrain what the answer actually does.
26
+
27
+ A judge built only from shape ops is a formatting checker, not a grader; see
28
+ :func:`is_shape_only`. Callers should prefer an outcome-based judge (rubric or
29
+ outcome ops) so the gate cannot be won by reformatting.
30
+
31
+ A task whose judge is {"kind": "rule", "checks": [...]} passes (hard=1.0) iff
32
+ ALL checks pass; soft = fraction of checks passed. This mirrors gbrain's
33
+ all-checks-must-pass rule scoring and gives the gate a smooth signal.
34
+ """
35
+ from __future__ import annotations
36
+
37
+ import re
38
+ from typing import Any, Dict, List, Tuple
39
+
40
+
41
+ def _section_present(response: str, name: str) -> bool:
42
+ # a markdown heading line (#, ##, ...) or bold line that contains `name`
43
+ pat = re.compile(
44
+ r"(?im)^\s{0,3}(#{1,6}\s*.*%s|\*\*.*%s.*\*\*\s*:?)\s*$" % (re.escape(name), re.escape(name))
45
+ )
46
+ if pat.search(response or ""):
47
+ return True
48
+ # also accept "Name:" style label at line start
49
+ label = re.compile(r"(?im)^\s*%s\s*:" % re.escape(name))
50
+ return bool(label.search(response or ""))
51
+
52
+
53
+ def _section_contains(response: str, name: str) -> bool:
54
+ """Whether an ATX Markdown heading contains ``name`` literally.
55
+
56
+ Unlike the legacy ``section_present`` check, this opt-in operator permits
57
+ text before and after the requested name, such as numbering, translations,
58
+ or subtitles. Only ATX heading lines with one to six markers and up to three
59
+ leading spaces are inspected, so body text and other Markdown constructs do
60
+ not satisfy the check. ``casefold`` gives literal, Unicode-aware,
61
+ case-insensitive matching without interpreting regex metacharacters.
62
+ """
63
+ needle = name.casefold()
64
+ heading = re.compile(r"(?m)^ {0,3}#{1,6}(?:[ \t]+|$)([^\n]*)$")
65
+ return any(needle in match.group(1).casefold() for match in heading.finditer(response or ""))
66
+
67
+
68
+ _REFUSAL_PREFIXES = (
69
+ "cannot complete",
70
+ "i cannot",
71
+ "i can't",
72
+ "i'm unable",
73
+ "i am unable",
74
+ "unable to complete",
75
+ "sorry, i can't",
76
+ "sorry, i cannot",
77
+ "no can do",
78
+ )
79
+
80
+
81
+ def _is_refusal(response: str) -> bool:
82
+ """Detect a bare refusal: an abstention with no substantive work reported.
83
+
84
+ A refusal that still explains what was searched and what is missing is a
85
+ useful answer, so only short responses whose opening is an abstention count.
86
+ """
87
+ text = (response or "").strip()
88
+ if not text:
89
+ return True
90
+ # Strip leading markdown markers -- blockquote (>), list bullets (-, *),
91
+ # numbered items (1. / 1)), emphasis and headings -- BEFORE bounding the
92
+ # head, so a refusal cannot hide behind >160 marker characters.
93
+ content = re.sub(r"^(?:[>\-*_#\s]|\d+[.)])+", "", text)
94
+ head = content[:160].lower()
95
+ if not any(head.startswith(p) for p in _REFUSAL_PREFIXES):
96
+ return False
97
+ # A long response that opens with an abstention still did the work of
98
+ # explaining why; only terse dead-ends are refusals.
99
+ return len(content) < 600
100
+
101
+
102
+ def _check(op: str, arg: Any, response: str,
103
+ tools_called: List[str]) -> Tuple[bool, str]:
104
+ """Evaluate one check.
105
+
106
+ Returns ``(passed, problem)``. ``problem`` is non-empty only when the check
107
+ itself is malformed (e.g. an unparseable regex) rather than simply unmet —
108
+ the two need opposite fixes, so they must not look alike in the rationale.
109
+ """
110
+ r = response or ""
111
+ if op == "section_present":
112
+ return _section_present(r, str(arg)), ""
113
+ if op == "section_contains":
114
+ return _section_contains(r, str(arg)), ""
115
+ if op == "regex":
116
+ try:
117
+ return bool(re.search(str(arg), r)), ""
118
+ except re.error as exc:
119
+ # A malformed pattern can never match, so it would fail every
120
+ # rollout forever and read exactly like a model that never
121
+ # complies. Surface it instead of hiding it behind a False.
122
+ return False, f"invalid regex ({exc})"
123
+ if op == "max_chars":
124
+ return len(r) <= int(arg), ""
125
+ if op == "min_chars":
126
+ return len(r) >= int(arg), ""
127
+ if op == "contains":
128
+ return str(arg).lower() in r.lower(), ""
129
+ if op == "not_contains":
130
+ return str(arg).lower() not in r.lower(), ""
131
+ if op == "no_refusal":
132
+ return not _is_refusal(r), ""
133
+ if op == "tool_called":
134
+ name = str(arg).lower()
135
+ if any(name == t.lower() for t in tools_called):
136
+ return True, ""
137
+ # single-shot approximation: the agent emits an explicit marker
138
+ return bool(re.search(r"(?i)\btool_call\s*:\s*%s\b" % re.escape(name), r)), ""
139
+ # unknown op: do not block
140
+ return True, ""
141
+
142
+
143
+ KNOWN_OPS = frozenset({
144
+ "section_present", "section_contains", "regex", "max_chars", "min_chars",
145
+ "contains", "not_contains", "no_refusal", "tool_called",
146
+ })
147
+
148
+ # Ops that only constrain formatting. An optimizer satisfies these by editing
149
+ # the output template, which is why a judge made solely of them is gameable.
150
+ SHAPE_OPS = frozenset({
151
+ "section_present", "section_contains", "max_chars", "min_chars",
152
+ })
153
+
154
+
155
+ def is_shape_only(judge: Any) -> bool:
156
+ """True when every check in ``judge`` merely constrains formatting.
157
+
158
+ Such a judge cannot distinguish a better answer from a reformatted one, so
159
+ callers should prefer outcome grading (a rubric) instead of trusting it.
160
+ """
161
+ if not isinstance(judge, dict):
162
+ return False
163
+ checks = judge.get("checks", []) or []
164
+ if not isinstance(checks, list) or not checks:
165
+ return False
166
+ ops = [c.get("op") for c in checks if isinstance(c, dict)]
167
+ if not ops or len(ops) != len(checks):
168
+ return False
169
+ return all(op in SHAPE_OPS for op in ops)
170
+
171
+
172
+ def char_bound(arg: Any) -> int:
173
+ """Parse a ``max_chars``/``min_chars`` argument, or raise ``ValueError``.
174
+
175
+ Shared by :func:`validate_checks` and the LLM miner so the accepted shapes
176
+ cannot drift apart: a miner that emits a bound the validator later rejects
177
+ produces a tasks file that fails its own validation. ``bool`` is refused
178
+ (it is an ``int`` subclass, so ``int(True)`` would silently become 1) and a
179
+ non-integral float is refused rather than truncated.
180
+ """
181
+ if isinstance(arg, bool):
182
+ raise ValueError("bool is not a char bound")
183
+ if isinstance(arg, int):
184
+ return arg
185
+ if isinstance(arg, float):
186
+ if not arg.is_integer():
187
+ raise ValueError("non-integral float is not a char bound")
188
+ return int(arg) # may raise OverflowError for inf/nan
189
+ if isinstance(arg, str) and re.fullmatch(r"[+-]?\d+", arg.strip()):
190
+ return int(arg.strip())
191
+ raise ValueError("not a char bound")
192
+
193
+
194
+ def validate_checks(judge: Any) -> Tuple[List[str], List[str]]:
195
+ """Return ``(errors, warnings)`` for a rule judge's checks.
196
+
197
+ An *error* means the check can never behave as written — a regex that does
198
+ not compile always scores 0.0, which is indistinguishable from a model that
199
+ never complies. A *warning* means the check is accepted but toothless, e.g.
200
+ an unknown op, which :func:`_check` deliberately lets pass.
201
+ """
202
+ errors: List[str] = []
203
+ warnings: List[str] = []
204
+ if judge is not None and not isinstance(judge, dict):
205
+ return [f"judge must be an object, got {type(judge).__name__}"], warnings
206
+ checks = (judge or {}).get("checks", []) or []
207
+ if not isinstance(checks, list):
208
+ return [f"judge 'checks' must be an array, got {type(checks).__name__}"], warnings
209
+ for i, c in enumerate(checks):
210
+ if not isinstance(c, dict):
211
+ errors.append(f"check #{i} is not an object")
212
+ continue
213
+ op = c.get("op", "")
214
+ arg = c.get("arg")
215
+ if not isinstance(op, str):
216
+ errors.append(
217
+ f"check #{i} op must be a string, got {type(op).__name__}"
218
+ )
219
+ continue
220
+ if op in {
221
+ "regex", "section_present", "section_contains", "contains",
222
+ "tool_called", "not_contains",
223
+ } and (
224
+ arg is None or not str(arg).strip()
225
+ ):
226
+ errors.append(f"check #{i} {op} needs a non-empty arg")
227
+ continue
228
+ if op == "regex":
229
+ try:
230
+ re.compile(str(arg))
231
+ except re.error as exc:
232
+ errors.append(f"check #{i} regex does not compile ({exc}): {arg!r}")
233
+ elif op in {"max_chars", "min_chars"}:
234
+ try:
235
+ bound = char_bound(arg)
236
+ except (OverflowError, TypeError, ValueError):
237
+ errors.append(f"check #{i} {op} needs an integer arg, got {arg!r}")
238
+ else:
239
+ if bound < 0:
240
+ errors.append(f"check #{i} {op} cannot be negative, got {bound}")
241
+ elif op == "min_chars" and bound == 0:
242
+ warnings.append(f"check #{i} min_chars=0 always passes")
243
+ elif op not in KNOWN_OPS:
244
+ warnings.append(f"check #{i} has unknown op {op!r} — it always passes")
245
+ if not errors and is_shape_only(judge):
246
+ warnings.append(
247
+ "judge is shape-only (formatting checks); it can be satisfied by "
248
+ "reformatting rather than by a better answer"
249
+ )
250
+ return errors, warnings
251
+
252
+
253
+ def _semantic_failure(check: Dict[str, Any], problem: str = "") -> str:
254
+ """Describe an unmet check without exposing its implementation syntax.
255
+
256
+ Rule-judge details are evidence, not instructions for the optimizer. In
257
+ particular, copying a regex into reflection lets an evolving skill learn
258
+ the verifier rather than the intended behavior. An operator-supplied
259
+ plain-language description is the strongest signal; conservative built-in
260
+ descriptions are used for legacy checks that do not carry one.
261
+ """
262
+ if problem:
263
+ return (
264
+ "The evaluator configuration is invalid; do not change the skill "
265
+ "based on this result."
266
+ )
267
+
268
+ op = str(check.get("op", ""))
269
+ arg = check.get("arg")
270
+ description = check.get("description")
271
+ if isinstance(description, str) and description.strip():
272
+ cleaned = description.strip()
273
+ # A generated or hand-authored description may accidentally copy the
274
+ # regex it is meant to explain. Fail closed instead of laundering the
275
+ # verifier implementation into the optimizer channel.
276
+ if op != "regex" or not isinstance(arg, str) or arg not in cleaned:
277
+ return cleaned
278
+
279
+ if op == "max_chars":
280
+ return f"Keep the entire response at or below {arg} characters."
281
+ if op == "min_chars":
282
+ return f"Provide at least {arg} characters of substantive response."
283
+ if op == "section_present":
284
+ return f"Include a section or heading titled {arg!r}."
285
+ if op == "section_contains":
286
+ return f"Include {arg!r} in an ATX markdown heading."
287
+ if op == "contains":
288
+ return f"The response must include the required concept or phrase {arg!r}."
289
+ if op == "not_contains":
290
+ return f"The response included prohibited content related to {arg!r}."
291
+ if op == "no_refusal":
292
+ return "Complete the task instead of refusing it."
293
+ if op == "tool_called":
294
+ return f"Actually call the {arg!r} tool while completing the task."
295
+ if op == "regex":
296
+ return "The response did not satisfy a private content or format requirement."
297
+ return "The response did not satisfy one of the task requirements."
298
+
299
+
300
+ def score_rule_judge_with_feedback(
301
+ judge: Dict[str, Any],
302
+ response: str,
303
+ tools_called: List[str] | None = None,
304
+ ) -> Tuple[float, float, str, str]:
305
+ """Return scores, audit rationale, and optimizer-safe feedback."""
306
+ checks = (judge or {}).get("checks", []) or []
307
+ if not checks:
308
+ return (
309
+ 0.0,
310
+ 0.0,
311
+ "no checks",
312
+ "The evaluator has no checks; do not change the skill based on this result.",
313
+ )
314
+ tools_called = tools_called or []
315
+ passed = 0
316
+ failed_desc: List[str] = []
317
+ semantic_failures: List[str] = []
318
+ for c in checks:
319
+ ok, problem = _check(c.get("op", ""), c.get("arg"), response, tools_called)
320
+ if ok:
321
+ passed += 1
322
+ else:
323
+ desc = f"{c.get('op')}={c.get('arg')}"
324
+ if problem:
325
+ desc += f" [{problem}]"
326
+ failed_desc.append(desc)
327
+ semantic_failures.append(_semantic_failure(c, problem))
328
+ soft = passed / len(checks)
329
+ hard = 1.0 if passed == len(checks) else 0.0
330
+ rationale = "all checks passed" if hard else "failed: " + ", ".join(failed_desc)
331
+ feedback = " ".join(dict.fromkeys(semantic_failures))
332
+ return hard, soft, rationale, feedback
333
+
334
+
335
+ def score_rule_judge(
336
+ judge: Dict[str, Any],
337
+ response: str,
338
+ tools_called: List[str] | None = None,
339
+ ) -> Tuple[float, float, str]:
340
+ """Return the backward-compatible (hard, soft, rationale) tuple."""
341
+ hard, soft, rationale, _feedback = score_rule_judge_with_feedback(
342
+ judge, response, tools_called
343
+ )
344
+ return hard, soft, rationale
@@ -0,0 +1,187 @@
1
+ """SkillOpt-Sleep — LLM-backed task miner.
2
+
3
+ The heuristic miner (mine.py) produces TaskRecords without a checkable
4
+ reference, so real harvested transcripts can't show measurable lift. This
5
+ module uses an optimizer backend to turn session digests into TaskRecords
6
+ WITH a checkable rubric judge — the missing piece for real-data improvement.
7
+
8
+ For each recurring intent it extracts:
9
+ * a clean, generalized `intent` (the reusable task, stripped of one-off specifics)
10
+ * a `rubric` (what a good answer must satisfy). A rubric, when present, is
11
+ always stored as the reference and scored by the backend's judge() -- it
12
+ resists the reward-hacking that literal/format checks invite. Programmatic
13
+ `contains`/`regex`/`section_present`/`section_contains`/`tool_called` checks
14
+ are kept only as a fallback reference for the intents where the miner
15
+ supplies no rubric.
16
+ * a preference signal (was the user satisfied?) to weight failures
17
+
18
+ It is deliberately conservative: it only emits a task when it can name a
19
+ concrete, checkable success criterion, so the gate has real signal. Tasks it
20
+ can't make checkable are dropped (logged), not faked.
21
+ """
22
+ from __future__ import annotations
23
+
24
+ from typing import Any, Callable, Dict, List
25
+
26
+ from skillopt_sleep import prompts as prompt_registry
27
+ from skillopt_sleep.backend import Backend, _extract_json
28
+ from skillopt_sleep.judges import char_bound
29
+ from skillopt_sleep.mine import session_skill_hint
30
+ from skillopt_sleep.types import SessionDigest, TaskRecord
31
+
32
+
33
+ def _digest_to_prompt(d: SessionDigest) -> str:
34
+ # Template lives in the central prompt registry (skillopt_sleep.prompts)
35
+ # so the dashboard can display and override it live.
36
+ prompts = "\n".join(f" - {p[:240]}" for p in d.user_prompts[:6]) or " (none)"
37
+ final = (d.assistant_finals[-1][:400] if d.assistant_finals else "(none)")
38
+ return prompt_registry.render("miner", {
39
+ "__PROJECT__": d.project or "(unknown)",
40
+ "__PROMPTS__": prompts,
41
+ "__FINAL__": final,
42
+ "__FEEDBACK__": ", ".join(d.feedback_signals[:6]) or "(none)",
43
+ })
44
+
45
+
46
+ def _mk_task(d: SessionDigest, obj: Dict[str, Any], idx: int) -> TaskRecord | None:
47
+ intent = str(obj.get("intent", "")).strip()
48
+ if len(intent) < 8:
49
+ return None
50
+ checks = obj.get("checks") or []
51
+ # A non-string rubric (e.g. a JSON null) must not become the literal "None"
52
+ # and outrank the checks; only a real string counts as a rubric.
53
+ rubric_raw = obj.get("rubric")
54
+ rubric = rubric_raw.strip() if isinstance(rubric_raw, str) else ""
55
+ satisfied = bool(obj.get("satisfied", False))
56
+
57
+ # Keep only well-formed checks: the scorer runs these verbatim during
58
+ # replay, so a max_chars/min_chars with a non-integer arg (or an arg-less
59
+ # op that needs one) would crash or fail forever. Drop those here, and keep
60
+ # the accepted shapes aligned with validate_checks() so a mined tasks file
61
+ # never fails validation later (it rejects bools and negative bounds).
62
+ _needs_str_arg = {
63
+ "section_present", "section_contains", "regex", "contains",
64
+ "not_contains", "tool_called",
65
+ }
66
+ # Trimming a regex would change what it matches (leading/trailing spaces are
67
+ # significant in a pattern), so only substring/tool/heading args are stripped.
68
+ _strip_arg = _needs_str_arg - {"regex"}
69
+ clean_checks = []
70
+ for c in checks:
71
+ if not isinstance(c, dict):
72
+ continue
73
+ op = c.get("op")
74
+ arg = c.get("arg")
75
+ description_raw = c.get("description")
76
+ description = (
77
+ description_raw.strip()
78
+ if isinstance(description_raw, str) and description_raw.strip()
79
+ else ""
80
+ )
81
+
82
+ def _with_description(check: Dict[str, Any]) -> Dict[str, Any]:
83
+ if description:
84
+ check["description"] = description
85
+ return check
86
+
87
+ if op in _needs_str_arg:
88
+ # Store the stripped value: stray whitespace would otherwise become
89
+ # part of the required substring / tool name.
90
+ if isinstance(arg, str) and arg.strip():
91
+ clean_checks.append(_with_description(
92
+ {"op": op, "arg": arg.strip() if op in _strip_arg else arg}
93
+ ))
94
+ elif op in {"max_chars", "min_chars"}:
95
+ # Shared parser with validate_checks() so the two cannot drift:
96
+ # rejects bools, non-integral floats and inf/nan (OverflowError).
97
+ try:
98
+ bound = char_bound(arg)
99
+ except (OverflowError, TypeError, ValueError):
100
+ continue
101
+ if bound < 0:
102
+ continue
103
+ clean_checks.append(_with_description({"op": op, "arg": bound}))
104
+ elif op == "no_refusal":
105
+ clean_checks.append(_with_description({"op": op, "arg": None}))
106
+
107
+ import hashlib
108
+ tid = "llm_" + hashlib.sha256((d.project + intent).encode()).hexdigest()[:12]
109
+
110
+ judge = {"kind": "rule", "checks": clean_checks}
111
+ # The optimizer edits skill text that is prepended to the model's context,
112
+ # so ANY check for a literal string in the response can be satisfied by
113
+ # instructing the model to emit that string -- observed twice in practice,
114
+ # first with section_present and then with contains. Only semantic grading
115
+ # resists that, so a rubric always wins when the miner supplies one.
116
+ # Rule judges remain for imported gbrain-style benchmarks, which carry
117
+ # checks but no rubric.
118
+ if rubric:
119
+ return TaskRecord(
120
+ id=tid, project=d.project, intent=intent,
121
+ reference_kind="rubric", reference=rubric,
122
+ outcome="success" if satisfied else "fail",
123
+ tags=["mined:llm"], source_sessions=[d.session_id],
124
+ skill_hint=session_skill_hint(d),
125
+ )
126
+ if clean_checks:
127
+ return TaskRecord(
128
+ id=tid, project=d.project, intent=intent,
129
+ reference_kind="rule", judge=judge,
130
+ outcome="success" if satisfied else "fail",
131
+ tags=["mined:llm"], source_sessions=[d.session_id],
132
+ skill_hint=session_skill_hint(d),
133
+ )
134
+ return None # not checkable -> drop
135
+
136
+
137
+ def make_llm_miner(
138
+ backend: Backend,
139
+ *,
140
+ max_sessions: int = 20,
141
+ max_tasks: int = 40,
142
+ ) -> Callable[[List[SessionDigest]], List[TaskRecord]]:
143
+ """Return an llm_miner(digests) -> list[TaskRecord] bound to a backend."""
144
+
145
+ def _miner(digests: List[SessionDigest]) -> List[TaskRecord]:
146
+ ev = getattr(backend, "evidence", None)
147
+ out: List[TaskRecord] = []
148
+ for d in digests[:max_sessions]:
149
+ if not d.user_prompts:
150
+ continue
151
+ prompt = _digest_to_prompt(d)
152
+ raw = backend._call(prompt, max_tokens=800) # type: ignore[attr-defined]
153
+ arr = _extract_json(raw, "array")
154
+ candidates = arr if isinstance(arr, list) else []
155
+ made: List[TaskRecord] = []
156
+ dropped = 0
157
+ full = False
158
+ for i, obj in enumerate(candidates[:3]):
159
+ if isinstance(obj, dict):
160
+ t = _mk_task(d, obj, i)
161
+ if t is not None:
162
+ made.append(t)
163
+ out.append(t)
164
+ else:
165
+ dropped += 1 # not checkable -> dropped, and now logged
166
+ if len(out) >= max_tasks:
167
+ full = True
168
+ break
169
+ if ev is not None:
170
+ # The transcript->task link of the evidentiary chain: what this
171
+ # session was, exactly what the miner was asked, exactly what it
172
+ # replied, and which TaskRecords (with checks) came out of it.
173
+ ev.log("mine", "miner_exchange", session_id=d.session_id,
174
+ project=d.project, prompt=prompt, raw_reply=raw,
175
+ parse_ok=isinstance(arr, list), n_candidates=len(candidates),
176
+ n_tasks=len(made), n_dropped_uncheckable=dropped)
177
+ for t in made:
178
+ ev.log("mine", "task_mined", task_id=t.id,
179
+ session_id=d.session_id, intent=t.intent,
180
+ reference_kind=t.reference_kind,
181
+ checks=(t.judge or {}).get("checks", []),
182
+ rubric=t.reference, outcome=t.outcome)
183
+ if full:
184
+ return out
185
+ return out
186
+
187
+ return _miner