@suifei/dsh-ai-team 0.0.0-stage → 0.3.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +98 -0
- package/LICENSE +21 -0
- package/README.md +38 -2
- package/client.js +2901 -0
- package/cordis.patch.yml +31 -0
- package/icon.svg +27 -0
- package/index.js +893 -0
- package/lib/domain/errors.js +36 -0
- package/lib/domain/ids.js +26 -0
- package/lib/domain/inbox.js +87 -0
- package/lib/domain/member-config.js +168 -0
- package/lib/domain/member-profile.js +53 -0
- package/lib/domain/policy.js +102 -0
- package/lib/domain/task.js +309 -0
- package/lib/domain/thread.js +54 -0
- package/lib/host/fault-adapter.js +50 -0
- package/lib/host/member-tools.js +225 -0
- package/lib/host/permission-guard.js +28 -0
- package/lib/host/project-registry.js +68 -0
- package/lib/host/project-runtime.js +73 -0
- package/lib/host/service.js +810 -0
- package/lib/host/views.js +213 -0
- package/lib/host/waker.js +177 -0
- package/lib/ledger/checkpoint.js +395 -0
- package/lib/ledger/events.js +398 -0
- package/lib/ledger/fold.js +991 -0
- package/lib/ledger/writer.js +983 -0
- package/lib/runtime/failure-policy.js +113 -0
- package/lib/runtime/host-agent.js +101 -0
- package/lib/runtime/member-runtime.js +286 -0
- package/lib/runtime/skill-catalog.js +27 -0
- package/lib/runtime/skill-loader.js +26 -0
- package/lib/runtime/skill-provider.js +121 -0
- package/lib/vendor/skillopt/LICENSE +21 -0
- package/lib/vendor/skillopt/requirements.txt +31 -0
- package/lib/vendor/skillopt/scripts/sleep.py +119 -0
- package/lib/vendor/skillopt/skillopt_sleep/__init__.py +20 -0
- package/lib/vendor/skillopt/skillopt_sleep/__main__.py +942 -0
- package/lib/vendor/skillopt/skillopt_sleep/adapters/__init__.py +1 -0
- package/lib/vendor/skillopt/skillopt_sleep/adapters/superpowers.py +1379 -0
- package/lib/vendor/skillopt/skillopt_sleep/backend.py +2844 -0
- package/lib/vendor/skillopt/skillopt_sleep/budget.py +75 -0
- package/lib/vendor/skillopt/skillopt_sleep/config.py +257 -0
- package/lib/vendor/skillopt/skillopt_sleep/consolidate.py +558 -0
- package/lib/vendor/skillopt/skillopt_sleep/cycle.py +1270 -0
- package/lib/vendor/skillopt/skillopt_sleep/dream.py +173 -0
- package/lib/vendor/skillopt/skillopt_sleep/evalkit.py +718 -0
- package/lib/vendor/skillopt/skillopt_sleep/evidence.py +143 -0
- package/lib/vendor/skillopt/skillopt_sleep/experiments/__init__.py +1 -0
- package/lib/vendor/skillopt/skillopt_sleep/experiments/gbrain_bench.py +119 -0
- package/lib/vendor/skillopt/skillopt_sleep/experiments/personas.py +86 -0
- package/lib/vendor/skillopt/skillopt_sleep/experiments/report.py +132 -0
- package/lib/vendor/skillopt/skillopt_sleep/experiments/run_experiment.py +178 -0
- package/lib/vendor/skillopt/skillopt_sleep/experiments/run_gbrain.py +209 -0
- package/lib/vendor/skillopt/skillopt_sleep/experiments/run_transfer.py +155 -0
- package/lib/vendor/skillopt/skillopt_sleep/experiments/sweep.py +164 -0
- package/lib/vendor/skillopt/skillopt_sleep/gate.py +50 -0
- package/lib/vendor/skillopt/skillopt_sleep/handoff_backend.py +173 -0
- package/lib/vendor/skillopt/skillopt_sleep/harvest.py +691 -0
- package/lib/vendor/skillopt/skillopt_sleep/harvest_codex.py +260 -0
- package/lib/vendor/skillopt/skillopt_sleep/harvest_copilot.py +443 -0
- package/lib/vendor/skillopt/skillopt_sleep/harvest_copilot_cli.py +244 -0
- package/lib/vendor/skillopt/skillopt_sleep/harvest_cursor.py +316 -0
- package/lib/vendor/skillopt/skillopt_sleep/harvest_opencode.py +364 -0
- package/lib/vendor/skillopt/skillopt_sleep/harvest_pi.py +311 -0
- package/lib/vendor/skillopt/skillopt_sleep/harvest_sources.py +135 -0
- package/lib/vendor/skillopt/skillopt_sleep/judges.py +344 -0
- package/lib/vendor/skillopt/skillopt_sleep/llm_miner.py +187 -0
- package/lib/vendor/skillopt/skillopt_sleep/memory.py +166 -0
- package/lib/vendor/skillopt/skillopt_sleep/mine.py +427 -0
- package/lib/vendor/skillopt/skillopt_sleep/multi_skill.py +190 -0
- package/lib/vendor/skillopt/skillopt_sleep/prompts.py +250 -0
- package/lib/vendor/skillopt/skillopt_sleep/replay.py +211 -0
- package/lib/vendor/skillopt/skillopt_sleep/rollout.py +184 -0
- package/lib/vendor/skillopt/skillopt_sleep/scheduler.py +286 -0
- package/lib/vendor/skillopt/skillopt_sleep/skill_resolver.py +244 -0
- package/lib/vendor/skillopt/skillopt_sleep/slow_update.py +141 -0
- package/lib/vendor/skillopt/skillopt_sleep/staging.py +3285 -0
- package/lib/vendor/skillopt/skillopt_sleep/state.py +283 -0
- package/lib/vendor/skillopt/skillopt_sleep/tasks_file.py +100 -0
- package/lib/vendor/skillopt/skillopt_sleep/types.py +227 -0
- package/lib/vendor/skillopt/src/index.js +394 -0
- package/package.json +62 -4
- package/skills/skillopt-sleep/SKILL.md +121 -0
|
@@ -0,0 +1,135 @@
|
|
|
1
|
+
"""Source selection for SkillOpt-Sleep transcript harvesting."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
from typing import Optional
|
|
5
|
+
|
|
6
|
+
from skillopt_sleep.harvest import harvest
|
|
7
|
+
from skillopt_sleep.harvest_codex import harvest_codex
|
|
8
|
+
from skillopt_sleep.harvest_copilot import harvest_copilot
|
|
9
|
+
from skillopt_sleep.harvest_copilot_cli import harvest_copilot_cli
|
|
10
|
+
from skillopt_sleep.harvest_cursor import harvest_cursor
|
|
11
|
+
from skillopt_sleep.harvest_opencode import harvest_opencode
|
|
12
|
+
from skillopt_sleep.harvest_pi import harvest_pi
|
|
13
|
+
from skillopt_sleep.types import SessionDigest
|
|
14
|
+
|
|
15
|
+
SUPPORTED_SOURCES = (
|
|
16
|
+
"claude",
|
|
17
|
+
"codex",
|
|
18
|
+
"copilot",
|
|
19
|
+
"copilot_cli",
|
|
20
|
+
"cursor",
|
|
21
|
+
"pi",
|
|
22
|
+
"opencode",
|
|
23
|
+
"auto",
|
|
24
|
+
)
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def normalize_transcript_source(value: object) -> str:
|
|
28
|
+
"""Canonical name of a configured ``transcript_source``.
|
|
29
|
+
|
|
30
|
+
Unset or empty means ``claude``; otherwise case and surrounding
|
|
31
|
+
whitespace are ignored. The harvester, its per-source cursor and the
|
|
32
|
+
tasks document must all use this one value: a cursor keyed by the raw
|
|
33
|
+
spelling (``<project>::Codex``) is not the one the harvested source
|
|
34
|
+
owns, and skips its older sessions (#312).
|
|
35
|
+
"""
|
|
36
|
+
supported = ", ".join(SUPPORTED_SOURCES)
|
|
37
|
+
if value is None:
|
|
38
|
+
return "claude"
|
|
39
|
+
if not isinstance(value, str):
|
|
40
|
+
raise ValueError(
|
|
41
|
+
f"transcript_source must be a string, not {type(value).__name__} "
|
|
42
|
+
f"{value!r}. Supported sources: {supported}"
|
|
43
|
+
)
|
|
44
|
+
source = (value or "claude").strip().lower()
|
|
45
|
+
if source not in SUPPORTED_SOURCES:
|
|
46
|
+
raise ValueError(
|
|
47
|
+
f"Unknown transcript source '{value}'. Supported sources: {supported}"
|
|
48
|
+
)
|
|
49
|
+
return source
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def harvest_for_config(
|
|
53
|
+
cfg,
|
|
54
|
+
*,
|
|
55
|
+
since_iso: Optional[str] = None,
|
|
56
|
+
limit: int = 0,
|
|
57
|
+
stats: Optional[dict] = None,
|
|
58
|
+
) -> list[SessionDigest]:
|
|
59
|
+
"""Harvest with the configured source.
|
|
60
|
+
|
|
61
|
+
``stats`` is filled in place by the Claude source (see
|
|
62
|
+
`skillopt_sleep.harvest.harvest`); the other sources have no repeat
|
|
63
|
+
filter to report yet and leave it untouched.
|
|
64
|
+
"""
|
|
65
|
+
source = normalize_transcript_source(cfg.get("transcript_source", "claude"))
|
|
66
|
+
scope = cfg.get("projects", "invoked")
|
|
67
|
+
invoked_project = cfg.get("invoked_project", "")
|
|
68
|
+
|
|
69
|
+
if source == "codex":
|
|
70
|
+
return harvest_codex(
|
|
71
|
+
cfg.codex_archived_sessions_dir,
|
|
72
|
+
scope=scope,
|
|
73
|
+
invoked_project=invoked_project,
|
|
74
|
+
since_iso=since_iso,
|
|
75
|
+
limit=limit,
|
|
76
|
+
)
|
|
77
|
+
if source == "copilot":
|
|
78
|
+
return harvest_copilot(
|
|
79
|
+
cfg.vscode_workspace_storage,
|
|
80
|
+
scope=scope,
|
|
81
|
+
invoked_project=invoked_project,
|
|
82
|
+
since_iso=since_iso,
|
|
83
|
+
limit=limit,
|
|
84
|
+
)
|
|
85
|
+
if source == "copilot_cli":
|
|
86
|
+
return harvest_copilot_cli(
|
|
87
|
+
cfg.copilot_cli_session_store,
|
|
88
|
+
scope=scope,
|
|
89
|
+
invoked_project=invoked_project,
|
|
90
|
+
since_iso=since_iso,
|
|
91
|
+
limit=limit,
|
|
92
|
+
)
|
|
93
|
+
if source == "cursor":
|
|
94
|
+
return harvest_cursor(
|
|
95
|
+
cfg.cursor_projects_dir,
|
|
96
|
+
scope=scope,
|
|
97
|
+
invoked_project=invoked_project,
|
|
98
|
+
since_iso=since_iso,
|
|
99
|
+
limit=limit,
|
|
100
|
+
)
|
|
101
|
+
if source == "pi":
|
|
102
|
+
return harvest_pi(
|
|
103
|
+
cfg.pi_sessions_dir,
|
|
104
|
+
scope=scope,
|
|
105
|
+
invoked_project=invoked_project,
|
|
106
|
+
since_iso=since_iso,
|
|
107
|
+
limit=limit,
|
|
108
|
+
)
|
|
109
|
+
if source == "opencode":
|
|
110
|
+
return harvest_opencode(
|
|
111
|
+
cfg.opencode_db_path,
|
|
112
|
+
scope=scope,
|
|
113
|
+
invoked_project=invoked_project,
|
|
114
|
+
since_iso=since_iso,
|
|
115
|
+
limit=limit,
|
|
116
|
+
)
|
|
117
|
+
if source == "auto":
|
|
118
|
+
codex_digests = harvest_codex(
|
|
119
|
+
cfg.codex_archived_sessions_dir,
|
|
120
|
+
scope=scope,
|
|
121
|
+
invoked_project=invoked_project,
|
|
122
|
+
since_iso=since_iso,
|
|
123
|
+
limit=limit,
|
|
124
|
+
)
|
|
125
|
+
if codex_digests:
|
|
126
|
+
return codex_digests
|
|
127
|
+
|
|
128
|
+
return harvest(
|
|
129
|
+
cfg.transcripts_dir,
|
|
130
|
+
scope=scope,
|
|
131
|
+
invoked_project=invoked_project,
|
|
132
|
+
since_iso=since_iso,
|
|
133
|
+
limit=limit,
|
|
134
|
+
stats=stats,
|
|
135
|
+
)
|
|
@@ -0,0 +1,344 @@
|
|
|
1
|
+
"""SkillOpt-Sleep — rule-based judges (gbrain-evals compatible).
|
|
2
|
+
|
|
3
|
+
Implements the programmatic check operators used by gbrain-evals'
|
|
4
|
+
skillopt-v1 benchmark so we can score skill outputs locally, with NO judge
|
|
5
|
+
API call:
|
|
6
|
+
|
|
7
|
+
* section_present <name> — legacy strict heading/bold/label check
|
|
8
|
+
* section_contains <name> — an ATX markdown heading contains literal <name>
|
|
9
|
+
* regex <pattern> — the pattern matches the response
|
|
10
|
+
* max_chars <n> — response length <= n
|
|
11
|
+
* min_chars <n> — response length >= n
|
|
12
|
+
* contains <text> — substring present (case-insensitive)
|
|
13
|
+
* not_contains <text> — substring absent (case-insensitive)
|
|
14
|
+
* no_refusal — the response is not a bare refusal/abstention
|
|
15
|
+
* tool_called <name> — a tool with <name> was invoked (needs a tool loop;
|
|
16
|
+
in single-shot replay we approximate via an
|
|
17
|
+
explicit "TOOL_CALL: <name>" marker the agent emits)
|
|
18
|
+
|
|
19
|
+
Ops divide into two families, and the distinction is load-bearing:
|
|
20
|
+
|
|
21
|
+
* *shape* ops (section_present, section_contains, max_chars, min_chars)
|
|
22
|
+
constrain how an answer is formatted. They are trivially satisfiable by an
|
|
23
|
+
optimizer — adding a heading scores 1.0 without the answer improving at all.
|
|
24
|
+
* *outcome* ops (contains, not_contains, no_refusal, regex, tool_called)
|
|
25
|
+
constrain what the answer actually does.
|
|
26
|
+
|
|
27
|
+
A judge built only from shape ops is a formatting checker, not a grader; see
|
|
28
|
+
:func:`is_shape_only`. Callers should prefer an outcome-based judge (rubric or
|
|
29
|
+
outcome ops) so the gate cannot be won by reformatting.
|
|
30
|
+
|
|
31
|
+
A task whose judge is {"kind": "rule", "checks": [...]} passes (hard=1.0) iff
|
|
32
|
+
ALL checks pass; soft = fraction of checks passed. This mirrors gbrain's
|
|
33
|
+
all-checks-must-pass rule scoring and gives the gate a smooth signal.
|
|
34
|
+
"""
|
|
35
|
+
from __future__ import annotations
|
|
36
|
+
|
|
37
|
+
import re
|
|
38
|
+
from typing import Any, Dict, List, Tuple
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def _section_present(response: str, name: str) -> bool:
|
|
42
|
+
# a markdown heading line (#, ##, ...) or bold line that contains `name`
|
|
43
|
+
pat = re.compile(
|
|
44
|
+
r"(?im)^\s{0,3}(#{1,6}\s*.*%s|\*\*.*%s.*\*\*\s*:?)\s*$" % (re.escape(name), re.escape(name))
|
|
45
|
+
)
|
|
46
|
+
if pat.search(response or ""):
|
|
47
|
+
return True
|
|
48
|
+
# also accept "Name:" style label at line start
|
|
49
|
+
label = re.compile(r"(?im)^\s*%s\s*:" % re.escape(name))
|
|
50
|
+
return bool(label.search(response or ""))
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def _section_contains(response: str, name: str) -> bool:
|
|
54
|
+
"""Whether an ATX Markdown heading contains ``name`` literally.
|
|
55
|
+
|
|
56
|
+
Unlike the legacy ``section_present`` check, this opt-in operator permits
|
|
57
|
+
text before and after the requested name, such as numbering, translations,
|
|
58
|
+
or subtitles. Only ATX heading lines with one to six markers and up to three
|
|
59
|
+
leading spaces are inspected, so body text and other Markdown constructs do
|
|
60
|
+
not satisfy the check. ``casefold`` gives literal, Unicode-aware,
|
|
61
|
+
case-insensitive matching without interpreting regex metacharacters.
|
|
62
|
+
"""
|
|
63
|
+
needle = name.casefold()
|
|
64
|
+
heading = re.compile(r"(?m)^ {0,3}#{1,6}(?:[ \t]+|$)([^\n]*)$")
|
|
65
|
+
return any(needle in match.group(1).casefold() for match in heading.finditer(response or ""))
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
_REFUSAL_PREFIXES = (
|
|
69
|
+
"cannot complete",
|
|
70
|
+
"i cannot",
|
|
71
|
+
"i can't",
|
|
72
|
+
"i'm unable",
|
|
73
|
+
"i am unable",
|
|
74
|
+
"unable to complete",
|
|
75
|
+
"sorry, i can't",
|
|
76
|
+
"sorry, i cannot",
|
|
77
|
+
"no can do",
|
|
78
|
+
)
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def _is_refusal(response: str) -> bool:
|
|
82
|
+
"""Detect a bare refusal: an abstention with no substantive work reported.
|
|
83
|
+
|
|
84
|
+
A refusal that still explains what was searched and what is missing is a
|
|
85
|
+
useful answer, so only short responses whose opening is an abstention count.
|
|
86
|
+
"""
|
|
87
|
+
text = (response or "").strip()
|
|
88
|
+
if not text:
|
|
89
|
+
return True
|
|
90
|
+
# Strip leading markdown markers -- blockquote (>), list bullets (-, *),
|
|
91
|
+
# numbered items (1. / 1)), emphasis and headings -- BEFORE bounding the
|
|
92
|
+
# head, so a refusal cannot hide behind >160 marker characters.
|
|
93
|
+
content = re.sub(r"^(?:[>\-*_#\s]|\d+[.)])+", "", text)
|
|
94
|
+
head = content[:160].lower()
|
|
95
|
+
if not any(head.startswith(p) for p in _REFUSAL_PREFIXES):
|
|
96
|
+
return False
|
|
97
|
+
# A long response that opens with an abstention still did the work of
|
|
98
|
+
# explaining why; only terse dead-ends are refusals.
|
|
99
|
+
return len(content) < 600
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
def _check(op: str, arg: Any, response: str,
|
|
103
|
+
tools_called: List[str]) -> Tuple[bool, str]:
|
|
104
|
+
"""Evaluate one check.
|
|
105
|
+
|
|
106
|
+
Returns ``(passed, problem)``. ``problem`` is non-empty only when the check
|
|
107
|
+
itself is malformed (e.g. an unparseable regex) rather than simply unmet —
|
|
108
|
+
the two need opposite fixes, so they must not look alike in the rationale.
|
|
109
|
+
"""
|
|
110
|
+
r = response or ""
|
|
111
|
+
if op == "section_present":
|
|
112
|
+
return _section_present(r, str(arg)), ""
|
|
113
|
+
if op == "section_contains":
|
|
114
|
+
return _section_contains(r, str(arg)), ""
|
|
115
|
+
if op == "regex":
|
|
116
|
+
try:
|
|
117
|
+
return bool(re.search(str(arg), r)), ""
|
|
118
|
+
except re.error as exc:
|
|
119
|
+
# A malformed pattern can never match, so it would fail every
|
|
120
|
+
# rollout forever and read exactly like a model that never
|
|
121
|
+
# complies. Surface it instead of hiding it behind a False.
|
|
122
|
+
return False, f"invalid regex ({exc})"
|
|
123
|
+
if op == "max_chars":
|
|
124
|
+
return len(r) <= int(arg), ""
|
|
125
|
+
if op == "min_chars":
|
|
126
|
+
return len(r) >= int(arg), ""
|
|
127
|
+
if op == "contains":
|
|
128
|
+
return str(arg).lower() in r.lower(), ""
|
|
129
|
+
if op == "not_contains":
|
|
130
|
+
return str(arg).lower() not in r.lower(), ""
|
|
131
|
+
if op == "no_refusal":
|
|
132
|
+
return not _is_refusal(r), ""
|
|
133
|
+
if op == "tool_called":
|
|
134
|
+
name = str(arg).lower()
|
|
135
|
+
if any(name == t.lower() for t in tools_called):
|
|
136
|
+
return True, ""
|
|
137
|
+
# single-shot approximation: the agent emits an explicit marker
|
|
138
|
+
return bool(re.search(r"(?i)\btool_call\s*:\s*%s\b" % re.escape(name), r)), ""
|
|
139
|
+
# unknown op: do not block
|
|
140
|
+
return True, ""
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
KNOWN_OPS = frozenset({
|
|
144
|
+
"section_present", "section_contains", "regex", "max_chars", "min_chars",
|
|
145
|
+
"contains", "not_contains", "no_refusal", "tool_called",
|
|
146
|
+
})
|
|
147
|
+
|
|
148
|
+
# Ops that only constrain formatting. An optimizer satisfies these by editing
|
|
149
|
+
# the output template, which is why a judge made solely of them is gameable.
|
|
150
|
+
SHAPE_OPS = frozenset({
|
|
151
|
+
"section_present", "section_contains", "max_chars", "min_chars",
|
|
152
|
+
})
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
def is_shape_only(judge: Any) -> bool:
|
|
156
|
+
"""True when every check in ``judge`` merely constrains formatting.
|
|
157
|
+
|
|
158
|
+
Such a judge cannot distinguish a better answer from a reformatted one, so
|
|
159
|
+
callers should prefer outcome grading (a rubric) instead of trusting it.
|
|
160
|
+
"""
|
|
161
|
+
if not isinstance(judge, dict):
|
|
162
|
+
return False
|
|
163
|
+
checks = judge.get("checks", []) or []
|
|
164
|
+
if not isinstance(checks, list) or not checks:
|
|
165
|
+
return False
|
|
166
|
+
ops = [c.get("op") for c in checks if isinstance(c, dict)]
|
|
167
|
+
if not ops or len(ops) != len(checks):
|
|
168
|
+
return False
|
|
169
|
+
return all(op in SHAPE_OPS for op in ops)
|
|
170
|
+
|
|
171
|
+
|
|
172
|
+
def char_bound(arg: Any) -> int:
|
|
173
|
+
"""Parse a ``max_chars``/``min_chars`` argument, or raise ``ValueError``.
|
|
174
|
+
|
|
175
|
+
Shared by :func:`validate_checks` and the LLM miner so the accepted shapes
|
|
176
|
+
cannot drift apart: a miner that emits a bound the validator later rejects
|
|
177
|
+
produces a tasks file that fails its own validation. ``bool`` is refused
|
|
178
|
+
(it is an ``int`` subclass, so ``int(True)`` would silently become 1) and a
|
|
179
|
+
non-integral float is refused rather than truncated.
|
|
180
|
+
"""
|
|
181
|
+
if isinstance(arg, bool):
|
|
182
|
+
raise ValueError("bool is not a char bound")
|
|
183
|
+
if isinstance(arg, int):
|
|
184
|
+
return arg
|
|
185
|
+
if isinstance(arg, float):
|
|
186
|
+
if not arg.is_integer():
|
|
187
|
+
raise ValueError("non-integral float is not a char bound")
|
|
188
|
+
return int(arg) # may raise OverflowError for inf/nan
|
|
189
|
+
if isinstance(arg, str) and re.fullmatch(r"[+-]?\d+", arg.strip()):
|
|
190
|
+
return int(arg.strip())
|
|
191
|
+
raise ValueError("not a char bound")
|
|
192
|
+
|
|
193
|
+
|
|
194
|
+
def validate_checks(judge: Any) -> Tuple[List[str], List[str]]:
|
|
195
|
+
"""Return ``(errors, warnings)`` for a rule judge's checks.
|
|
196
|
+
|
|
197
|
+
An *error* means the check can never behave as written — a regex that does
|
|
198
|
+
not compile always scores 0.0, which is indistinguishable from a model that
|
|
199
|
+
never complies. A *warning* means the check is accepted but toothless, e.g.
|
|
200
|
+
an unknown op, which :func:`_check` deliberately lets pass.
|
|
201
|
+
"""
|
|
202
|
+
errors: List[str] = []
|
|
203
|
+
warnings: List[str] = []
|
|
204
|
+
if judge is not None and not isinstance(judge, dict):
|
|
205
|
+
return [f"judge must be an object, got {type(judge).__name__}"], warnings
|
|
206
|
+
checks = (judge or {}).get("checks", []) or []
|
|
207
|
+
if not isinstance(checks, list):
|
|
208
|
+
return [f"judge 'checks' must be an array, got {type(checks).__name__}"], warnings
|
|
209
|
+
for i, c in enumerate(checks):
|
|
210
|
+
if not isinstance(c, dict):
|
|
211
|
+
errors.append(f"check #{i} is not an object")
|
|
212
|
+
continue
|
|
213
|
+
op = c.get("op", "")
|
|
214
|
+
arg = c.get("arg")
|
|
215
|
+
if not isinstance(op, str):
|
|
216
|
+
errors.append(
|
|
217
|
+
f"check #{i} op must be a string, got {type(op).__name__}"
|
|
218
|
+
)
|
|
219
|
+
continue
|
|
220
|
+
if op in {
|
|
221
|
+
"regex", "section_present", "section_contains", "contains",
|
|
222
|
+
"tool_called", "not_contains",
|
|
223
|
+
} and (
|
|
224
|
+
arg is None or not str(arg).strip()
|
|
225
|
+
):
|
|
226
|
+
errors.append(f"check #{i} {op} needs a non-empty arg")
|
|
227
|
+
continue
|
|
228
|
+
if op == "regex":
|
|
229
|
+
try:
|
|
230
|
+
re.compile(str(arg))
|
|
231
|
+
except re.error as exc:
|
|
232
|
+
errors.append(f"check #{i} regex does not compile ({exc}): {arg!r}")
|
|
233
|
+
elif op in {"max_chars", "min_chars"}:
|
|
234
|
+
try:
|
|
235
|
+
bound = char_bound(arg)
|
|
236
|
+
except (OverflowError, TypeError, ValueError):
|
|
237
|
+
errors.append(f"check #{i} {op} needs an integer arg, got {arg!r}")
|
|
238
|
+
else:
|
|
239
|
+
if bound < 0:
|
|
240
|
+
errors.append(f"check #{i} {op} cannot be negative, got {bound}")
|
|
241
|
+
elif op == "min_chars" and bound == 0:
|
|
242
|
+
warnings.append(f"check #{i} min_chars=0 always passes")
|
|
243
|
+
elif op not in KNOWN_OPS:
|
|
244
|
+
warnings.append(f"check #{i} has unknown op {op!r} — it always passes")
|
|
245
|
+
if not errors and is_shape_only(judge):
|
|
246
|
+
warnings.append(
|
|
247
|
+
"judge is shape-only (formatting checks); it can be satisfied by "
|
|
248
|
+
"reformatting rather than by a better answer"
|
|
249
|
+
)
|
|
250
|
+
return errors, warnings
|
|
251
|
+
|
|
252
|
+
|
|
253
|
+
def _semantic_failure(check: Dict[str, Any], problem: str = "") -> str:
|
|
254
|
+
"""Describe an unmet check without exposing its implementation syntax.
|
|
255
|
+
|
|
256
|
+
Rule-judge details are evidence, not instructions for the optimizer. In
|
|
257
|
+
particular, copying a regex into reflection lets an evolving skill learn
|
|
258
|
+
the verifier rather than the intended behavior. An operator-supplied
|
|
259
|
+
plain-language description is the strongest signal; conservative built-in
|
|
260
|
+
descriptions are used for legacy checks that do not carry one.
|
|
261
|
+
"""
|
|
262
|
+
if problem:
|
|
263
|
+
return (
|
|
264
|
+
"The evaluator configuration is invalid; do not change the skill "
|
|
265
|
+
"based on this result."
|
|
266
|
+
)
|
|
267
|
+
|
|
268
|
+
op = str(check.get("op", ""))
|
|
269
|
+
arg = check.get("arg")
|
|
270
|
+
description = check.get("description")
|
|
271
|
+
if isinstance(description, str) and description.strip():
|
|
272
|
+
cleaned = description.strip()
|
|
273
|
+
# A generated or hand-authored description may accidentally copy the
|
|
274
|
+
# regex it is meant to explain. Fail closed instead of laundering the
|
|
275
|
+
# verifier implementation into the optimizer channel.
|
|
276
|
+
if op != "regex" or not isinstance(arg, str) or arg not in cleaned:
|
|
277
|
+
return cleaned
|
|
278
|
+
|
|
279
|
+
if op == "max_chars":
|
|
280
|
+
return f"Keep the entire response at or below {arg} characters."
|
|
281
|
+
if op == "min_chars":
|
|
282
|
+
return f"Provide at least {arg} characters of substantive response."
|
|
283
|
+
if op == "section_present":
|
|
284
|
+
return f"Include a section or heading titled {arg!r}."
|
|
285
|
+
if op == "section_contains":
|
|
286
|
+
return f"Include {arg!r} in an ATX markdown heading."
|
|
287
|
+
if op == "contains":
|
|
288
|
+
return f"The response must include the required concept or phrase {arg!r}."
|
|
289
|
+
if op == "not_contains":
|
|
290
|
+
return f"The response included prohibited content related to {arg!r}."
|
|
291
|
+
if op == "no_refusal":
|
|
292
|
+
return "Complete the task instead of refusing it."
|
|
293
|
+
if op == "tool_called":
|
|
294
|
+
return f"Actually call the {arg!r} tool while completing the task."
|
|
295
|
+
if op == "regex":
|
|
296
|
+
return "The response did not satisfy a private content or format requirement."
|
|
297
|
+
return "The response did not satisfy one of the task requirements."
|
|
298
|
+
|
|
299
|
+
|
|
300
|
+
def score_rule_judge_with_feedback(
|
|
301
|
+
judge: Dict[str, Any],
|
|
302
|
+
response: str,
|
|
303
|
+
tools_called: List[str] | None = None,
|
|
304
|
+
) -> Tuple[float, float, str, str]:
|
|
305
|
+
"""Return scores, audit rationale, and optimizer-safe feedback."""
|
|
306
|
+
checks = (judge or {}).get("checks", []) or []
|
|
307
|
+
if not checks:
|
|
308
|
+
return (
|
|
309
|
+
0.0,
|
|
310
|
+
0.0,
|
|
311
|
+
"no checks",
|
|
312
|
+
"The evaluator has no checks; do not change the skill based on this result.",
|
|
313
|
+
)
|
|
314
|
+
tools_called = tools_called or []
|
|
315
|
+
passed = 0
|
|
316
|
+
failed_desc: List[str] = []
|
|
317
|
+
semantic_failures: List[str] = []
|
|
318
|
+
for c in checks:
|
|
319
|
+
ok, problem = _check(c.get("op", ""), c.get("arg"), response, tools_called)
|
|
320
|
+
if ok:
|
|
321
|
+
passed += 1
|
|
322
|
+
else:
|
|
323
|
+
desc = f"{c.get('op')}={c.get('arg')}"
|
|
324
|
+
if problem:
|
|
325
|
+
desc += f" [{problem}]"
|
|
326
|
+
failed_desc.append(desc)
|
|
327
|
+
semantic_failures.append(_semantic_failure(c, problem))
|
|
328
|
+
soft = passed / len(checks)
|
|
329
|
+
hard = 1.0 if passed == len(checks) else 0.0
|
|
330
|
+
rationale = "all checks passed" if hard else "failed: " + ", ".join(failed_desc)
|
|
331
|
+
feedback = " ".join(dict.fromkeys(semantic_failures))
|
|
332
|
+
return hard, soft, rationale, feedback
|
|
333
|
+
|
|
334
|
+
|
|
335
|
+
def score_rule_judge(
|
|
336
|
+
judge: Dict[str, Any],
|
|
337
|
+
response: str,
|
|
338
|
+
tools_called: List[str] | None = None,
|
|
339
|
+
) -> Tuple[float, float, str]:
|
|
340
|
+
"""Return the backward-compatible (hard, soft, rationale) tuple."""
|
|
341
|
+
hard, soft, rationale, _feedback = score_rule_judge_with_feedback(
|
|
342
|
+
judge, response, tools_called
|
|
343
|
+
)
|
|
344
|
+
return hard, soft, rationale
|
|
@@ -0,0 +1,187 @@
|
|
|
1
|
+
"""SkillOpt-Sleep — LLM-backed task miner.
|
|
2
|
+
|
|
3
|
+
The heuristic miner (mine.py) produces TaskRecords without a checkable
|
|
4
|
+
reference, so real harvested transcripts can't show measurable lift. This
|
|
5
|
+
module uses an optimizer backend to turn session digests into TaskRecords
|
|
6
|
+
WITH a checkable rubric judge — the missing piece for real-data improvement.
|
|
7
|
+
|
|
8
|
+
For each recurring intent it extracts:
|
|
9
|
+
* a clean, generalized `intent` (the reusable task, stripped of one-off specifics)
|
|
10
|
+
* a `rubric` (what a good answer must satisfy). A rubric, when present, is
|
|
11
|
+
always stored as the reference and scored by the backend's judge() -- it
|
|
12
|
+
resists the reward-hacking that literal/format checks invite. Programmatic
|
|
13
|
+
`contains`/`regex`/`section_present`/`section_contains`/`tool_called` checks
|
|
14
|
+
are kept only as a fallback reference for the intents where the miner
|
|
15
|
+
supplies no rubric.
|
|
16
|
+
* a preference signal (was the user satisfied?) to weight failures
|
|
17
|
+
|
|
18
|
+
It is deliberately conservative: it only emits a task when it can name a
|
|
19
|
+
concrete, checkable success criterion, so the gate has real signal. Tasks it
|
|
20
|
+
can't make checkable are dropped (logged), not faked.
|
|
21
|
+
"""
|
|
22
|
+
from __future__ import annotations
|
|
23
|
+
|
|
24
|
+
from typing import Any, Callable, Dict, List
|
|
25
|
+
|
|
26
|
+
from skillopt_sleep import prompts as prompt_registry
|
|
27
|
+
from skillopt_sleep.backend import Backend, _extract_json
|
|
28
|
+
from skillopt_sleep.judges import char_bound
|
|
29
|
+
from skillopt_sleep.mine import session_skill_hint
|
|
30
|
+
from skillopt_sleep.types import SessionDigest, TaskRecord
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def _digest_to_prompt(d: SessionDigest) -> str:
|
|
34
|
+
# Template lives in the central prompt registry (skillopt_sleep.prompts)
|
|
35
|
+
# so the dashboard can display and override it live.
|
|
36
|
+
prompts = "\n".join(f" - {p[:240]}" for p in d.user_prompts[:6]) or " (none)"
|
|
37
|
+
final = (d.assistant_finals[-1][:400] if d.assistant_finals else "(none)")
|
|
38
|
+
return prompt_registry.render("miner", {
|
|
39
|
+
"__PROJECT__": d.project or "(unknown)",
|
|
40
|
+
"__PROMPTS__": prompts,
|
|
41
|
+
"__FINAL__": final,
|
|
42
|
+
"__FEEDBACK__": ", ".join(d.feedback_signals[:6]) or "(none)",
|
|
43
|
+
})
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def _mk_task(d: SessionDigest, obj: Dict[str, Any], idx: int) -> TaskRecord | None:
|
|
47
|
+
intent = str(obj.get("intent", "")).strip()
|
|
48
|
+
if len(intent) < 8:
|
|
49
|
+
return None
|
|
50
|
+
checks = obj.get("checks") or []
|
|
51
|
+
# A non-string rubric (e.g. a JSON null) must not become the literal "None"
|
|
52
|
+
# and outrank the checks; only a real string counts as a rubric.
|
|
53
|
+
rubric_raw = obj.get("rubric")
|
|
54
|
+
rubric = rubric_raw.strip() if isinstance(rubric_raw, str) else ""
|
|
55
|
+
satisfied = bool(obj.get("satisfied", False))
|
|
56
|
+
|
|
57
|
+
# Keep only well-formed checks: the scorer runs these verbatim during
|
|
58
|
+
# replay, so a max_chars/min_chars with a non-integer arg (or an arg-less
|
|
59
|
+
# op that needs one) would crash or fail forever. Drop those here, and keep
|
|
60
|
+
# the accepted shapes aligned with validate_checks() so a mined tasks file
|
|
61
|
+
# never fails validation later (it rejects bools and negative bounds).
|
|
62
|
+
_needs_str_arg = {
|
|
63
|
+
"section_present", "section_contains", "regex", "contains",
|
|
64
|
+
"not_contains", "tool_called",
|
|
65
|
+
}
|
|
66
|
+
# Trimming a regex would change what it matches (leading/trailing spaces are
|
|
67
|
+
# significant in a pattern), so only substring/tool/heading args are stripped.
|
|
68
|
+
_strip_arg = _needs_str_arg - {"regex"}
|
|
69
|
+
clean_checks = []
|
|
70
|
+
for c in checks:
|
|
71
|
+
if not isinstance(c, dict):
|
|
72
|
+
continue
|
|
73
|
+
op = c.get("op")
|
|
74
|
+
arg = c.get("arg")
|
|
75
|
+
description_raw = c.get("description")
|
|
76
|
+
description = (
|
|
77
|
+
description_raw.strip()
|
|
78
|
+
if isinstance(description_raw, str) and description_raw.strip()
|
|
79
|
+
else ""
|
|
80
|
+
)
|
|
81
|
+
|
|
82
|
+
def _with_description(check: Dict[str, Any]) -> Dict[str, Any]:
|
|
83
|
+
if description:
|
|
84
|
+
check["description"] = description
|
|
85
|
+
return check
|
|
86
|
+
|
|
87
|
+
if op in _needs_str_arg:
|
|
88
|
+
# Store the stripped value: stray whitespace would otherwise become
|
|
89
|
+
# part of the required substring / tool name.
|
|
90
|
+
if isinstance(arg, str) and arg.strip():
|
|
91
|
+
clean_checks.append(_with_description(
|
|
92
|
+
{"op": op, "arg": arg.strip() if op in _strip_arg else arg}
|
|
93
|
+
))
|
|
94
|
+
elif op in {"max_chars", "min_chars"}:
|
|
95
|
+
# Shared parser with validate_checks() so the two cannot drift:
|
|
96
|
+
# rejects bools, non-integral floats and inf/nan (OverflowError).
|
|
97
|
+
try:
|
|
98
|
+
bound = char_bound(arg)
|
|
99
|
+
except (OverflowError, TypeError, ValueError):
|
|
100
|
+
continue
|
|
101
|
+
if bound < 0:
|
|
102
|
+
continue
|
|
103
|
+
clean_checks.append(_with_description({"op": op, "arg": bound}))
|
|
104
|
+
elif op == "no_refusal":
|
|
105
|
+
clean_checks.append(_with_description({"op": op, "arg": None}))
|
|
106
|
+
|
|
107
|
+
import hashlib
|
|
108
|
+
tid = "llm_" + hashlib.sha256((d.project + intent).encode()).hexdigest()[:12]
|
|
109
|
+
|
|
110
|
+
judge = {"kind": "rule", "checks": clean_checks}
|
|
111
|
+
# The optimizer edits skill text that is prepended to the model's context,
|
|
112
|
+
# so ANY check for a literal string in the response can be satisfied by
|
|
113
|
+
# instructing the model to emit that string -- observed twice in practice,
|
|
114
|
+
# first with section_present and then with contains. Only semantic grading
|
|
115
|
+
# resists that, so a rubric always wins when the miner supplies one.
|
|
116
|
+
# Rule judges remain for imported gbrain-style benchmarks, which carry
|
|
117
|
+
# checks but no rubric.
|
|
118
|
+
if rubric:
|
|
119
|
+
return TaskRecord(
|
|
120
|
+
id=tid, project=d.project, intent=intent,
|
|
121
|
+
reference_kind="rubric", reference=rubric,
|
|
122
|
+
outcome="success" if satisfied else "fail",
|
|
123
|
+
tags=["mined:llm"], source_sessions=[d.session_id],
|
|
124
|
+
skill_hint=session_skill_hint(d),
|
|
125
|
+
)
|
|
126
|
+
if clean_checks:
|
|
127
|
+
return TaskRecord(
|
|
128
|
+
id=tid, project=d.project, intent=intent,
|
|
129
|
+
reference_kind="rule", judge=judge,
|
|
130
|
+
outcome="success" if satisfied else "fail",
|
|
131
|
+
tags=["mined:llm"], source_sessions=[d.session_id],
|
|
132
|
+
skill_hint=session_skill_hint(d),
|
|
133
|
+
)
|
|
134
|
+
return None # not checkable -> drop
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
def make_llm_miner(
|
|
138
|
+
backend: Backend,
|
|
139
|
+
*,
|
|
140
|
+
max_sessions: int = 20,
|
|
141
|
+
max_tasks: int = 40,
|
|
142
|
+
) -> Callable[[List[SessionDigest]], List[TaskRecord]]:
|
|
143
|
+
"""Return an llm_miner(digests) -> list[TaskRecord] bound to a backend."""
|
|
144
|
+
|
|
145
|
+
def _miner(digests: List[SessionDigest]) -> List[TaskRecord]:
|
|
146
|
+
ev = getattr(backend, "evidence", None)
|
|
147
|
+
out: List[TaskRecord] = []
|
|
148
|
+
for d in digests[:max_sessions]:
|
|
149
|
+
if not d.user_prompts:
|
|
150
|
+
continue
|
|
151
|
+
prompt = _digest_to_prompt(d)
|
|
152
|
+
raw = backend._call(prompt, max_tokens=800) # type: ignore[attr-defined]
|
|
153
|
+
arr = _extract_json(raw, "array")
|
|
154
|
+
candidates = arr if isinstance(arr, list) else []
|
|
155
|
+
made: List[TaskRecord] = []
|
|
156
|
+
dropped = 0
|
|
157
|
+
full = False
|
|
158
|
+
for i, obj in enumerate(candidates[:3]):
|
|
159
|
+
if isinstance(obj, dict):
|
|
160
|
+
t = _mk_task(d, obj, i)
|
|
161
|
+
if t is not None:
|
|
162
|
+
made.append(t)
|
|
163
|
+
out.append(t)
|
|
164
|
+
else:
|
|
165
|
+
dropped += 1 # not checkable -> dropped, and now logged
|
|
166
|
+
if len(out) >= max_tasks:
|
|
167
|
+
full = True
|
|
168
|
+
break
|
|
169
|
+
if ev is not None:
|
|
170
|
+
# The transcript->task link of the evidentiary chain: what this
|
|
171
|
+
# session was, exactly what the miner was asked, exactly what it
|
|
172
|
+
# replied, and which TaskRecords (with checks) came out of it.
|
|
173
|
+
ev.log("mine", "miner_exchange", session_id=d.session_id,
|
|
174
|
+
project=d.project, prompt=prompt, raw_reply=raw,
|
|
175
|
+
parse_ok=isinstance(arr, list), n_candidates=len(candidates),
|
|
176
|
+
n_tasks=len(made), n_dropped_uncheckable=dropped)
|
|
177
|
+
for t in made:
|
|
178
|
+
ev.log("mine", "task_mined", task_id=t.id,
|
|
179
|
+
session_id=d.session_id, intent=t.intent,
|
|
180
|
+
reference_kind=t.reference_kind,
|
|
181
|
+
checks=(t.judge or {}).get("checks", []),
|
|
182
|
+
rubric=t.reference, outcome=t.outcome)
|
|
183
|
+
if full:
|
|
184
|
+
return out
|
|
185
|
+
return out
|
|
186
|
+
|
|
187
|
+
return _miner
|