dsh-biocore 0.3.3 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/index.js +30 -1
- package/lib/modules/standby.js +61 -0
- package/lib/modules/state.js +22 -3
- package/package.json +1 -1
- package/python/biocore_memory/core/config.py +7 -3
- package/python/biocore_memory/engine.py +91 -6
- package/python/biocore_memory/recall/embedding.py +0 -51
- package/python/biocore_memory/recall/feedback.py +34 -7
- package/python/biocore_memory/recall/fusion.py +5 -1
- package/python/biocore_memory/scoring.py +203 -0
- package/python/biocore_memory/space.py +355 -0
- package/python/biocore_memory/store/dedupe.py +1 -0
- package/python/biocore_memory/store/tiering.py +14 -3
- package/python/biocore_model/affective_labor.py +14 -1
- package/python/biocore_model/appraisal.py +172 -3
- package/python/biocore_model/bio_tick.py +43 -13
- package/python/biocore_model/emotion_geometry.py +9 -0
- package/python/biocore_model/envy.py +21 -0
- package/python/biocore_model/feedback_memory.py +53 -0
- package/python/biocore_model/needs_events.py +32 -11
- package/python/biocore_model/regulation_strategies.py +5 -0
- package/python/biocore_model/repair_after_boundary.py +1 -0
- package/python/biocore_model/self_esteem.py +14 -2
- package/python/biocore_model/self_intent.py +6 -4
- package/python/biocore_model/tests/test_autonomy_self.py +153 -153
- package/python/biocore_model/tests/test_cache_series.py +137 -0
- package/python/biocore_model/tests/test_core_state.py +240 -240
- package/python/biocore_model/tests/test_extensions.py +167 -167
- package/python/biocore_model/tests/test_field_arbiter.py +444 -0
- package/python/biocore_model/tests/test_governance.py +181 -181
- package/python/biocore_model/tests/test_lifecycle.py +153 -0
- package/python/biocore_model/tests/test_memory_inject.py +211 -211
- package/python/biocore_model/tests/test_retrieval_plus.py +143 -0
- package/python/biocore_model/tests/test_signals.py +106 -106
- package/python/biocore_model/tests/test_standby.py +199 -0
- package/python/biocore_runtime/__init__.py +3 -0
- package/python/biocore_runtime/arbiter.py +307 -0
- package/python/biocore_runtime/batch_b.py +724 -0
- package/python/biocore_runtime/batch_c.py +186 -0
- package/python/biocore_runtime/batch_d.py +659 -0
- package/python/biocore_runtime/batch_e.py +328 -0
- package/python/biocore_runtime/batch_registry.py +446 -0
- package/python/biocore_runtime/cache_stats.py +161 -0
- package/python/biocore_runtime/core_state.py +106 -3
- package/python/biocore_runtime/dimensions.py +419 -0
- package/python/biocore_runtime/expression.py +261 -0
- package/python/biocore_runtime/field.py +440 -0
- package/python/biocore_runtime/lifecycle.py +233 -0
- package/python/biocore_runtime/perception.py +1045 -0
- package/python/biocore_runtime/pipeline.py +4458 -2974
- package/python/biocore_runtime/priority.py +210 -0
- package/python/biocore_runtime/retrieval_plus.py +352 -0
- package/python/biocore_runtime/standby.py +581 -472
- package/python/biocore_runtime/timescales.py +263 -0
- package/python/data/signals.json +3266 -206
- package/ui/panel-layout.js +32 -29
- package/ui/panel.css +563 -974
- package/ui/panel.html +427 -401
- package/ui/panel.js +543 -450
|
@@ -0,0 +1,446 @@
|
|
|
1
|
+
# -*- coding: utf-8 -*-
|
|
2
|
+
"""Batch integration: one registry for b / c / d / e, all ON, per-module monitor.
|
|
3
|
+
|
|
4
|
+
The four batch hooks (biocore_runtime.batch_b / batch_c / batch_d / batch_e)
|
|
5
|
+
have different return shapes -- b/d/e return {"modules", "activity",
|
|
6
|
+
"field_deltas", "state_deltas", "skipped"}, the older c returns a flat bundle
|
|
7
|
+
with dotted "deltas". This module normalises both, merges the safe parts into
|
|
8
|
+
the session state, hands the six-field proposals to the arbiter, and keeps one
|
|
9
|
+
monitor row per module:
|
|
10
|
+
|
|
11
|
+
<dataDir>/batch_monitor.json
|
|
12
|
+
{"schema": 1, "ts": ..., "batches": {"b": {"calls": n, ...}},
|
|
13
|
+
"modules": {"b.entity_resolve": {"calls": n, "changed": n,
|
|
14
|
+
"last_effect": 0.0, "errored": n,
|
|
15
|
+
"last_ts": 0.0}}}
|
|
16
|
+
|
|
17
|
+
No IO besides that one file; never raises into the turn.
|
|
18
|
+
"""
|
|
19
|
+
from __future__ import annotations
|
|
20
|
+
|
|
21
|
+
import json
|
|
22
|
+
import math
|
|
23
|
+
import time
|
|
24
|
+
from pathlib import Path
|
|
25
|
+
|
|
26
|
+
try: # one resolution strategy for every field (never a cross-source sum)
|
|
27
|
+
from . import arbiter as _arbiter
|
|
28
|
+
except Exception: # pragma: no cover
|
|
29
|
+
try:
|
|
30
|
+
from biocore_runtime import arbiter as _arbiter
|
|
31
|
+
except Exception:
|
|
32
|
+
_arbiter = None
|
|
33
|
+
|
|
34
|
+
MONITOR_FILE = "batch_monitor.json"
|
|
35
|
+
SCHEMA = 1
|
|
36
|
+
MODULE_LIMIT = 320
|
|
37
|
+
|
|
38
|
+
BATCHES = ("b", "c", "d", "e")
|
|
39
|
+
|
|
40
|
+
# State blocks a batch may write top-level. Everything else is namespaced under
|
|
41
|
+
# state["batches"][<batch>] so a batch can never clobber the body/core fields.
|
|
42
|
+
SAFE_KEYS = ("entity_graph", "graph_stats", "memory_tiers", "dual_scores",
|
|
43
|
+
"memory_scores", "memory_strength", "co_recall", "policy_q",
|
|
44
|
+
"predictor_forecast", "predictor_history", "prediction",
|
|
45
|
+
"migrate_report", "identity_read", "autonomy", "autonomy_acc",
|
|
46
|
+
"expression", "state_tag", "topic_control", "reply", "silence",
|
|
47
|
+
"feedback", "diagnostics", "diagnostic_row", "memory_layers",
|
|
48
|
+
"graph", "co_recall_pairs", "reconstruction", "narrative")
|
|
49
|
+
|
|
50
|
+
FIELD_NAMES = ("v", "a", "stress", "social", "trust", "closeness")
|
|
51
|
+
CAPS = {"v": 0.10, "a": 0.10, "stress": 0.08, "social": 0.08,
|
|
52
|
+
"trust": 0.02, "closeness": 0.02}
|
|
53
|
+
|
|
54
|
+
# Which read slice belongs to which module. A module without a slice has no
|
|
55
|
+
# observable view of its own this turn (it reports zero change, not "changed").
|
|
56
|
+
READ_OWNERS = {
|
|
57
|
+
"b": {"identity": ("bio_identity",), "graph": ("graph_recall",),
|
|
58
|
+
"entities": ("entity_resolve",), "memory_tiers": ("layered_memory",),
|
|
59
|
+
"dual": ("memory_dual",), "forgetting": ("memory_forgetting",),
|
|
60
|
+
"store": ("memory_store",), "policy": ("policy_learning",),
|
|
61
|
+
"forecast": ("predictor",), "scores": ("scoring",),
|
|
62
|
+
"graph_stats": ("entity_graph",), "strength": ("feedback_memory",),
|
|
63
|
+
"co_recall": ("co_recall",), "migrate": ("migrate_store",)},
|
|
64
|
+
"d": {"autonomy": ("autonomy_check", "autonomy_loop", "autonomy_model",
|
|
65
|
+
"autonomy_params", "autonomy_reason", "autonomy_mode"),
|
|
66
|
+
"expression": ("expr_intent", "expr_tone", "granularity",
|
|
67
|
+
"exit_or_silence", "reply_limit", "reply_split",
|
|
68
|
+
"expr_state", "prompt_rules", "expr_guard",
|
|
69
|
+
"state_tag", "topic_control", "expr_template"),
|
|
70
|
+
"feedback": ("feedback_flow", "feedback_guard", "feedback_llm",
|
|
71
|
+
"feedback_user"),
|
|
72
|
+
"schedule": ("scheduler", "voice_bank", "proactive_context")},
|
|
73
|
+
"e": {"diagnostics": ("diag_trace", "delta_budget", "schema", "bio_const"),
|
|
74
|
+
"config": ("core_commands", "provider_fallback", "stages"),
|
|
75
|
+
"monitoring": ("source_monitoring", "mind_wander", "noise_bank"),
|
|
76
|
+
"probe": ("learning_probe", "extreme_scenarios")},
|
|
77
|
+
}
|
|
78
|
+
VIEW_LIMIT = 24
|
|
79
|
+
EFFECT_LIMIT = 8
|
|
80
|
+
|
|
81
|
+
# Reasons that are normal skips, not failures.
|
|
82
|
+
BENIGN = {"module_missing", "platform_isolation", "disabled", "duplicate",
|
|
83
|
+
"no_entries", "no_tag", "no_sample", "no_pairs", "no_choice",
|
|
84
|
+
"no_context", "no_action", "no_store", "no_forecast",
|
|
85
|
+
"nothing_to_resolve", "bad_phase", "unavailable", "no_snapshot",
|
|
86
|
+
"convert_failed", "no_deltas", "dry_run", "no_change"}
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def _num(value, default=0.0):
|
|
90
|
+
try:
|
|
91
|
+
number = float(value)
|
|
92
|
+
except Exception:
|
|
93
|
+
return default
|
|
94
|
+
if not math.isfinite(number):
|
|
95
|
+
return default
|
|
96
|
+
return number
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def _root(store):
|
|
100
|
+
if isinstance(store, dict):
|
|
101
|
+
raw = store.get("root") or store.get("store") or store.get("path")
|
|
102
|
+
else:
|
|
103
|
+
raw = store
|
|
104
|
+
text = str(raw or "").strip()
|
|
105
|
+
return Path(text) if text else None
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
def _data_dir(store):
|
|
109
|
+
root = _root(store)
|
|
110
|
+
return (root.parent if root.name == "memory" else root) if root else None
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
def _read(path):
|
|
114
|
+
try:
|
|
115
|
+
if path is None or not path.exists():
|
|
116
|
+
return {}
|
|
117
|
+
doc = json.loads(path.read_text(encoding="utf-8"))
|
|
118
|
+
return doc if isinstance(doc, dict) else {}
|
|
119
|
+
except Exception:
|
|
120
|
+
return {}
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
def _write(path, doc):
|
|
124
|
+
try:
|
|
125
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
126
|
+
tmp = path.with_name(path.name + ".tmp")
|
|
127
|
+
tmp.write_text(json.dumps(doc, ensure_ascii=False, indent=2),
|
|
128
|
+
encoding="utf-8")
|
|
129
|
+
tmp.replace(path)
|
|
130
|
+
return True
|
|
131
|
+
except Exception:
|
|
132
|
+
return False
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
def path(store):
|
|
136
|
+
root = _data_dir(store)
|
|
137
|
+
return (root / MONITOR_FILE) if root is not None else None
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
# ---------------------------------------------------------------- normalise
|
|
141
|
+
def modules_of(batch_name, output):
|
|
142
|
+
"""The module names one batch claims (declared list or c-style fields)."""
|
|
143
|
+
if not isinstance(output, dict):
|
|
144
|
+
return ()
|
|
145
|
+
declared = output.get("modules")
|
|
146
|
+
if isinstance(declared, dict):
|
|
147
|
+
return tuple(declared.keys())
|
|
148
|
+
fields = output.get("fields")
|
|
149
|
+
if isinstance(fields, list):
|
|
150
|
+
return tuple(str(item) for item in fields)
|
|
151
|
+
return ()
|
|
152
|
+
|
|
153
|
+
|
|
154
|
+
def normalize(batch_name, output):
|
|
155
|
+
"""Both shapes -> {enabled, modules, activity, field_deltas,
|
|
156
|
+
state_deltas, skipped, errored, raw_keys}."""
|
|
157
|
+
name = str(batch_name or "")
|
|
158
|
+
if not isinstance(output, dict):
|
|
159
|
+
return {"enabled": False, "reason": "bad_output", "modules": {},
|
|
160
|
+
"activity": {}, "field_deltas": {}, "state_deltas": {},
|
|
161
|
+
"skipped": [], "errored": 0, "raw_keys": []}
|
|
162
|
+
declared = output.get("modules")
|
|
163
|
+
if isinstance(declared, dict): # b / d / e
|
|
164
|
+
modules = {str(key): bool(value) for key, value in declared.items()}
|
|
165
|
+
activity = output.get("activity")
|
|
166
|
+
activity = ({str(key): bool(value) for key, value in activity.items()}
|
|
167
|
+
if isinstance(activity, dict) else dict(modules))
|
|
168
|
+
field_deltas = {key: _num(value) for key, value in
|
|
169
|
+
(output.get("field_deltas") or {}).items()
|
|
170
|
+
if key in FIELD_NAMES}
|
|
171
|
+
state_deltas = dict(output.get("state_deltas") or {})
|
|
172
|
+
skipped = [dict(row) for row in (output.get("skipped") or [])
|
|
173
|
+
if isinstance(row, dict)]
|
|
174
|
+
enabled = bool(output.get("enabled", True))
|
|
175
|
+
else: # batch_c style: flat bundle + dotted deltas
|
|
176
|
+
fields = [str(item) for item in (output.get("fields") or [])]
|
|
177
|
+
modules = {field: bool(output.get("ok", True)) for field in fields}
|
|
178
|
+
activity = dict(modules)
|
|
179
|
+
field_deltas = {}
|
|
180
|
+
for key, value in (output.get("deltas") or {}).items():
|
|
181
|
+
head = str(key).split(".")[0]
|
|
182
|
+
if head in FIELD_NAMES:
|
|
183
|
+
field_deltas[head] = _num(value)
|
|
184
|
+
state_deltas = {"bundle": output}
|
|
185
|
+
skipped = []
|
|
186
|
+
enabled = bool(output.get("ok", True))
|
|
187
|
+
errored = 0
|
|
188
|
+
for row in skipped:
|
|
189
|
+
reason = str(row.get("reason") or "")
|
|
190
|
+
if reason and reason not in BENIGN:
|
|
191
|
+
errored += 1
|
|
192
|
+
reads = output.get("reads")
|
|
193
|
+
if not isinstance(reads, dict):
|
|
194
|
+
reads = output if not isinstance(declared, dict) else {}
|
|
195
|
+
return {"enabled": enabled, "reason": str(output.get("reason") or "ok"),
|
|
196
|
+
"modules": modules, "activity": activity,
|
|
197
|
+
"field_deltas": field_deltas, "state_deltas": state_deltas,
|
|
198
|
+
"skipped": skipped, "errored": errored,
|
|
199
|
+
"reads": reads if isinstance(reads, dict) else {},
|
|
200
|
+
"raw_keys": sorted(str(key) for key in output.keys())}
|
|
201
|
+
|
|
202
|
+
|
|
203
|
+
# ---------------------------------------------------------------- monitor
|
|
204
|
+
def read_monitor(store):
|
|
205
|
+
doc = _read(path(store))
|
|
206
|
+
modules = doc.get("modules") if isinstance(doc.get("modules"), dict) else {}
|
|
207
|
+
batches = doc.get("batches") if isinstance(doc.get("batches"), dict) else {}
|
|
208
|
+
return {"schema": SCHEMA, "ts": _num(doc.get("ts"), 0.0),
|
|
209
|
+
"batches": {str(key): dict(value) for key, value in batches.items()
|
|
210
|
+
if isinstance(value, dict)},
|
|
211
|
+
"modules": {str(key): dict(value) for key, value in modules.items()
|
|
212
|
+
if isinstance(value, dict)},
|
|
213
|
+
"totals": _totals_of(doc)}
|
|
214
|
+
|
|
215
|
+
|
|
216
|
+
def flatten(value, prefix="", out=None, limit=VIEW_LIMIT):
|
|
217
|
+
"""Numeric channels of one module view: nested containers -> dotted keys."""
|
|
218
|
+
out = {} if out is None else out
|
|
219
|
+
if len(out) >= limit:
|
|
220
|
+
return out
|
|
221
|
+
if isinstance(value, dict):
|
|
222
|
+
for key, item in value.items():
|
|
223
|
+
if len(out) >= limit:
|
|
224
|
+
break
|
|
225
|
+
label = "%s.%s" % (prefix, key) if prefix else str(key)
|
|
226
|
+
flatten(item, label, out, limit)
|
|
227
|
+
elif isinstance(value, (list, tuple)):
|
|
228
|
+
for index, item in enumerate(list(value)[:4]):
|
|
229
|
+
flatten(item, "%s[%d]" % (prefix, index), out, limit)
|
|
230
|
+
elif isinstance(value, bool):
|
|
231
|
+
out[prefix] = 1.0 if value else 0.0
|
|
232
|
+
else:
|
|
233
|
+
number = _num(value, None)
|
|
234
|
+
if number is not None:
|
|
235
|
+
out[prefix] = round(number, 6)
|
|
236
|
+
return out
|
|
237
|
+
|
|
238
|
+
|
|
239
|
+
def module_view(batch, module, row):
|
|
240
|
+
"""The read slice one module owns (its own output, not the batch total)."""
|
|
241
|
+
row = row if isinstance(row, dict) else {}
|
|
242
|
+
reads = row.get("reads") if isinstance(row.get("reads"), dict) else {}
|
|
243
|
+
view = {}
|
|
244
|
+
for key, modules in (READ_OWNERS.get(str(batch)) or {}).items():
|
|
245
|
+
if str(module) not in modules:
|
|
246
|
+
continue
|
|
247
|
+
value = reads.get(key)
|
|
248
|
+
if value is None:
|
|
249
|
+
continue
|
|
250
|
+
view.update(flatten(value, str(key), view))
|
|
251
|
+
if len(view) >= VIEW_LIMIT:
|
|
252
|
+
break
|
|
253
|
+
if str(batch) == "c":
|
|
254
|
+
bundle = (row.get("state_deltas") or {}).get("bundle")
|
|
255
|
+
bundle = bundle if isinstance(bundle, dict) else {}
|
|
256
|
+
if module in bundle:
|
|
257
|
+
view.update(flatten(bundle.get(module), str(module), view))
|
|
258
|
+
return dict(list(view.items())[:VIEW_LIMIT])
|
|
259
|
+
|
|
260
|
+
|
|
261
|
+
def diff_view(before, after, eps=1e-6, limit=EFFECT_LIMIT):
|
|
262
|
+
"""{channel: delta} for the channels this module itself moved."""
|
|
263
|
+
old = before if isinstance(before, dict) else {}
|
|
264
|
+
new = after if isinstance(after, dict) else {}
|
|
265
|
+
changed = {}
|
|
266
|
+
for key in set(old) | set(new):
|
|
267
|
+
delta = round(_num(new.get(key), 0.0) - _num(old.get(key), 0.0), 6)
|
|
268
|
+
if abs(delta) > eps:
|
|
269
|
+
changed[key] = delta
|
|
270
|
+
if len(changed) > limit:
|
|
271
|
+
ordered = sorted(changed.items(), key=lambda item: -abs(item[1]))
|
|
272
|
+
changed = dict(ordered[:limit])
|
|
273
|
+
return changed
|
|
274
|
+
|
|
275
|
+
|
|
276
|
+
def _totals_of(doc):
|
|
277
|
+
totals = doc.get("totals") if isinstance(doc.get("totals"), dict) else {}
|
|
278
|
+
return {"calls": int(_num(totals.get("calls"), 0)),
|
|
279
|
+
"changed": int(_num(totals.get("changed"), 0)),
|
|
280
|
+
"zero_change": int(_num(totals.get("zero_change"), 0)),
|
|
281
|
+
"errored": int(_num(totals.get("errored"), 0)),
|
|
282
|
+
"active": int(_num(totals.get("active"), 0))}
|
|
283
|
+
|
|
284
|
+
|
|
285
|
+
def _bump_monitor(store, snapshots, now, changed_batches):
|
|
286
|
+
"""One monitor row per module: calls / changed / zero_change / effect."""
|
|
287
|
+
doc = read_monitor(store)
|
|
288
|
+
stamp = round(_num(now, time.time()), 3)
|
|
289
|
+
doc["ts"] = stamp
|
|
290
|
+
totals = _totals_of(doc)
|
|
291
|
+
errors_this_tick = 0
|
|
292
|
+
for name, row in snapshots.items():
|
|
293
|
+
batch_row = doc["batches"].setdefault(
|
|
294
|
+
name, {"calls": 0, "changed": 0, "zero_change": 0, "errored": 0})
|
|
295
|
+
batch_row["calls"] = int(_num(batch_row.get("calls"), 0)) + 1
|
|
296
|
+
batch_row["last_ts"] = stamp
|
|
297
|
+
batch_row.setdefault("zero_change", 0)
|
|
298
|
+
activity = row.get("activity") if isinstance(row.get("activity"), dict) else {}
|
|
299
|
+
errors = int(_num(row.get("errored"), 0))
|
|
300
|
+
if errors:
|
|
301
|
+
errors_this_tick += errors
|
|
302
|
+
batch_row["errored"] = int(_num(batch_row.get("errored"), 0)) + errors
|
|
303
|
+
for module in (row.get("modules") or {}):
|
|
304
|
+
if not bool(activity.get(module)):
|
|
305
|
+
continue # declared but idle this phase: not a call, no row
|
|
306
|
+
key = "%s.%s" % (name, module)
|
|
307
|
+
slot = doc["modules"].setdefault(
|
|
308
|
+
key, {"calls": 0, "changed": 0, "zero_change": 0,
|
|
309
|
+
"last_effect": {}, "errored": 0})
|
|
310
|
+
slot["calls"] = int(_num(slot.get("calls"), 0)) + 1
|
|
311
|
+
slot.setdefault("zero_change", 0)
|
|
312
|
+
slot["last_ts"] = stamp
|
|
313
|
+
view = module_view(name, module, row)
|
|
314
|
+
before = slot.get("view") if isinstance(slot.get("view"), dict) else None
|
|
315
|
+
effect = diff_view(before or {}, view)
|
|
316
|
+
totals["calls"] += 1
|
|
317
|
+
if before is None:
|
|
318
|
+
# first snapshot: nothing to compare yet, so not a change
|
|
319
|
+
slot["zero_change"] += 1
|
|
320
|
+
totals["zero_change"] += 1
|
|
321
|
+
batch_row["zero_change"] += 1
|
|
322
|
+
elif effect:
|
|
323
|
+
slot["changed"] = int(_num(slot.get("changed"), 0)) + 1
|
|
324
|
+
slot["last_effect"] = effect
|
|
325
|
+
totals["changed"] += 1
|
|
326
|
+
batch_row["changed"] = int(_num(batch_row.get("changed"), 0)) + 1
|
|
327
|
+
else:
|
|
328
|
+
slot["zero_change"] += 1
|
|
329
|
+
totals["zero_change"] += 1
|
|
330
|
+
batch_row["zero_change"] += 1
|
|
331
|
+
if view:
|
|
332
|
+
slot["view"] = view
|
|
333
|
+
if name in changed_batches and effect:
|
|
334
|
+
slot["last_ts"] = stamp
|
|
335
|
+
if errors:
|
|
336
|
+
slot["errored"] = int(_num(slot.get("errored"), 0)) + errors
|
|
337
|
+
totals["errored"] += errors_this_tick
|
|
338
|
+
totals["active"] = len(doc["modules"])
|
|
339
|
+
doc["totals"] = totals
|
|
340
|
+
if len(doc["modules"]) > MODULE_LIMIT:
|
|
341
|
+
ordered = sorted(doc["modules"].items(),
|
|
342
|
+
key=lambda item: -_num(item[1].get("last_ts"), 0.0))
|
|
343
|
+
doc["modules"] = dict(ordered[:MODULE_LIMIT])
|
|
344
|
+
_write(path(store), doc)
|
|
345
|
+
return doc
|
|
346
|
+
|
|
347
|
+
|
|
348
|
+
# ---------------------------------------------------------------- tick
|
|
349
|
+
def resolve_field(proposals, strategy=None):
|
|
350
|
+
"""(delta, strategy) for one field: the four batches are NOT summed.
|
|
351
|
+
|
|
352
|
+
Each batch keeps its own proposal row; the field value is chosen by the
|
|
353
|
+
arbiter's strategy (weighted median by default) over those proposals.
|
|
354
|
+
"""
|
|
355
|
+
rows = [(_num(row.get("delta"), 0.0), 1.0) for row in proposals or []
|
|
356
|
+
if isinstance(row, dict)]
|
|
357
|
+
if not rows:
|
|
358
|
+
return 0.0, "none"
|
|
359
|
+
if _arbiter is not None:
|
|
360
|
+
return _arbiter.resolve(rows, strategy)
|
|
361
|
+
best = max(rows, key=lambda item: abs(item[0]))
|
|
362
|
+
return best[0], "max_magnitude"
|
|
363
|
+
|
|
364
|
+
|
|
365
|
+
def tick(store, session_id, phase, payload=None, state=None, now=None,
|
|
366
|
+
switches=None, hooks=None):
|
|
367
|
+
"""Run every enabled batch once; returns normalized rows + per-field deltas.
|
|
368
|
+
|
|
369
|
+
A field may be named by more than one batch: every proposal is kept in
|
|
370
|
+
field_proposals{field: [{batch, delta}]} and the value comes from
|
|
371
|
+
resolve_field(), never from adding the batches together.
|
|
372
|
+
"""
|
|
373
|
+
data = payload if isinstance(payload, dict) else {}
|
|
374
|
+
session_state = state if isinstance(state, dict) else {}
|
|
375
|
+
stamp = round(_num(now if now is not None else data.get("now"),
|
|
376
|
+
time.time()), 3)
|
|
377
|
+
hooks = hooks if isinstance(hooks, dict) else {}
|
|
378
|
+
table = switches if isinstance(switches, dict) else {}
|
|
379
|
+
rows = {}
|
|
380
|
+
field_proposals = {}
|
|
381
|
+
field_deltas = {}
|
|
382
|
+
resolved = {}
|
|
383
|
+
state_deltas = {}
|
|
384
|
+
snapshots = {}
|
|
385
|
+
changed_batches = set()
|
|
386
|
+
skipped_batches = []
|
|
387
|
+
for name in BATCHES:
|
|
388
|
+
hook = hooks.get(name)
|
|
389
|
+
if hook is None:
|
|
390
|
+
skipped_batches.append({"batch": name, "reason": "unavailable"})
|
|
391
|
+
continue
|
|
392
|
+
if table and table.get("batch_" + name) is False:
|
|
393
|
+
skipped_batches.append({"batch": name, "reason": "disabled"})
|
|
394
|
+
continue
|
|
395
|
+
try:
|
|
396
|
+
output = hook.apply_turn(session_state, phase, data)
|
|
397
|
+
except Exception as error: # a broken batch never breaks a turn
|
|
398
|
+
rows[name] = {"enabled": False, "reason": type(error).__name__,
|
|
399
|
+
"modules": {}, "activity": {}, "field_deltas": {},
|
|
400
|
+
"state_deltas": {}, "skipped": [], "errored": 1,
|
|
401
|
+
"raw_keys": []}
|
|
402
|
+
snapshots[name] = rows[name]
|
|
403
|
+
continue
|
|
404
|
+
row = normalize(name, output)
|
|
405
|
+
rows[name] = row
|
|
406
|
+
snapshots[name] = row
|
|
407
|
+
if row["field_deltas"] or row["state_deltas"]:
|
|
408
|
+
changed_batches.add(name)
|
|
409
|
+
for key, value in row["field_deltas"].items():
|
|
410
|
+
if key in FIELD_NAMES:
|
|
411
|
+
field_proposals.setdefault(key, []).append(
|
|
412
|
+
{"batch": name, "delta": round(_num(value), 6)})
|
|
413
|
+
for key, value in row["state_deltas"].items():
|
|
414
|
+
if key in SAFE_KEYS:
|
|
415
|
+
state_deltas[key] = value
|
|
416
|
+
else:
|
|
417
|
+
bucket = state_deltas.setdefault("batches", {})
|
|
418
|
+
if isinstance(bucket, dict):
|
|
419
|
+
bucket.setdefault(name, {})[str(key)] = value
|
|
420
|
+
for key, proposals in field_proposals.items():
|
|
421
|
+
chosen, strategy = resolve_field(proposals)
|
|
422
|
+
capped = round(max(-CAPS[key], min(CAPS[key], chosen)), 6)
|
|
423
|
+
field_deltas[key] = capped
|
|
424
|
+
resolved[key] = {"strategy": strategy, "delta": round(chosen, 6),
|
|
425
|
+
"capped": capped, "proposals": proposals}
|
|
426
|
+
monitor = _bump_monitor(store, snapshots, stamp, changed_batches)
|
|
427
|
+
return {"enabled": True, "phase": str(phase), "ts": stamp,
|
|
428
|
+
"batches": rows, "field_deltas": field_deltas,
|
|
429
|
+
"field_proposals": field_proposals, "resolved": resolved,
|
|
430
|
+
"state_deltas": state_deltas, "skipped": skipped_batches,
|
|
431
|
+
"monitor": {"batches": monitor.get("batches", {}),
|
|
432
|
+
"modules": monitor.get("modules", {}),
|
|
433
|
+
"totals": _totals_of(monitor)},
|
|
434
|
+
"counts": {"batches": len(rows),
|
|
435
|
+
"modules": sum(len(row.get("modules") or {})
|
|
436
|
+
for row in rows.values()),
|
|
437
|
+
"changed": len(changed_batches)}}
|
|
438
|
+
|
|
439
|
+
|
|
440
|
+
def summary(store):
|
|
441
|
+
"""Read-only monitor projection for /state."""
|
|
442
|
+
doc = read_monitor(store)
|
|
443
|
+
totals = _totals_of(doc)
|
|
444
|
+
totals["active"] = len(doc["modules"])
|
|
445
|
+
return {"ts": doc["ts"], "batches": doc["batches"],
|
|
446
|
+
"modules": doc["modules"], "totals": totals}
|
|
@@ -0,0 +1,161 @@
|
|
|
1
|
+
# -*- coding: utf-8 -*-
|
|
2
|
+
"""Cache/usage accounting in the core (contract migration 1).
|
|
3
|
+
|
|
4
|
+
Usage events live in <dataDir>/usage_events.jsonl; the core is the only writer.
|
|
5
|
+
The adapter may still read the legacy <dataDir>/cache_stats.json for a week as
|
|
6
|
+
a read-only fallback, but it never writes or recomputes hit_rate.
|
|
7
|
+
|
|
8
|
+
hit_rate = cache_read / (cache_read + cache_write), falling back to
|
|
9
|
+
cache_read / prompt_tokens when no write tokens were reported; clamped to
|
|
10
|
+
[0, 1]. The default window is 7 days and every event is bounded.
|
|
11
|
+
"""
|
|
12
|
+
from __future__ import annotations
|
|
13
|
+
|
|
14
|
+
import json
|
|
15
|
+
import time
|
|
16
|
+
from pathlib import Path
|
|
17
|
+
|
|
18
|
+
USAGE_FILE = "usage_events.jsonl"
|
|
19
|
+
LEGACY_FILE = "cache_stats.json"
|
|
20
|
+
SCHEMA = 1
|
|
21
|
+
DEFAULT_WINDOW_HOURS = 24 * 7
|
|
22
|
+
MAX_EVENTS = 5000
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def _root(store):
|
|
26
|
+
if isinstance(store, dict):
|
|
27
|
+
raw = store.get("root") or store.get("store") or store.get("path")
|
|
28
|
+
else:
|
|
29
|
+
raw = store
|
|
30
|
+
text = str(raw or "").strip()
|
|
31
|
+
return Path(text) if text else None
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def _data_dir(store):
|
|
35
|
+
root = _root(store)
|
|
36
|
+
return (root.parent if root.name == "memory" else root) if root else None
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def _num(value, default=0.0):
|
|
40
|
+
try:
|
|
41
|
+
number = float(value)
|
|
42
|
+
except Exception:
|
|
43
|
+
return default
|
|
44
|
+
return number if number == number else default
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def usage_path(store):
|
|
48
|
+
data_dir = _data_dir(store)
|
|
49
|
+
return (data_dir / USAGE_FILE) if data_dir is not None else None
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def legacy_path(store):
|
|
53
|
+
data_dir = _data_dir(store)
|
|
54
|
+
return (data_dir / LEGACY_FILE) if data_dir is not None else None
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def _read_events(store):
|
|
58
|
+
path = usage_path(store)
|
|
59
|
+
rows = []
|
|
60
|
+
try:
|
|
61
|
+
if path is None or not path.exists():
|
|
62
|
+
return rows
|
|
63
|
+
for line in path.read_text(encoding="utf-8").splitlines():
|
|
64
|
+
line = line.strip()
|
|
65
|
+
if not line:
|
|
66
|
+
continue
|
|
67
|
+
try:
|
|
68
|
+
item = json.loads(line)
|
|
69
|
+
except Exception:
|
|
70
|
+
continue
|
|
71
|
+
if isinstance(item, dict):
|
|
72
|
+
rows.append(item)
|
|
73
|
+
except Exception:
|
|
74
|
+
return []
|
|
75
|
+
return rows
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def record_event(store, event, now=None):
|
|
79
|
+
"""Append one usage event; bounded to MAX_EVENTS."""
|
|
80
|
+
path = usage_path(store)
|
|
81
|
+
if path is None:
|
|
82
|
+
return None
|
|
83
|
+
stamp = round(float(now or event.get("ts") or time.time()), 3)
|
|
84
|
+
row = {
|
|
85
|
+
"ts": stamp,
|
|
86
|
+
"prompt_tokens": max(0, int(_num(event.get("prompt_tokens")))),
|
|
87
|
+
"output_tokens": max(0, int(_num(event.get("output_tokens")))),
|
|
88
|
+
"cache_read": max(0, int(_num(event.get("cache_read")))),
|
|
89
|
+
"cache_write": max(0, int(_num(event.get("cache_write")))),
|
|
90
|
+
}
|
|
91
|
+
try:
|
|
92
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
93
|
+
rows = _read_events(store)[-(MAX_EVENTS - 1):] + [row]
|
|
94
|
+
tmp = path.with_name(path.name + ".tmp")
|
|
95
|
+
tmp.write_text("\n".join(json.dumps(item, ensure_ascii=False)
|
|
96
|
+
for item in rows) + "\n", encoding="utf-8")
|
|
97
|
+
tmp.replace(path)
|
|
98
|
+
except Exception:
|
|
99
|
+
return None
|
|
100
|
+
return row
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
def _legacy(store):
|
|
104
|
+
path = legacy_path(store)
|
|
105
|
+
try:
|
|
106
|
+
if path is None or not path.exists():
|
|
107
|
+
return None
|
|
108
|
+
doc = json.loads(path.read_text(encoding="utf-8"))
|
|
109
|
+
except Exception:
|
|
110
|
+
return None
|
|
111
|
+
if not isinstance(doc, dict):
|
|
112
|
+
return None
|
|
113
|
+
return {
|
|
114
|
+
"turns": max(0, int(_num(doc.get("turns")))),
|
|
115
|
+
"prompt_tokens": max(0, int(_num(doc.get("prompt_tokens")))),
|
|
116
|
+
"output_tokens": max(0, int(_num(doc.get("output_tokens")))),
|
|
117
|
+
"cache_read": max(0, int(_num(doc.get("cache_read")))),
|
|
118
|
+
"cache_write": max(0, int(_num(doc.get("cache_write")))),
|
|
119
|
+
"hit_rate": round(min(1.0, max(0.0, _num(doc.get("hit_rate")))), 4),
|
|
120
|
+
"updated_at": round(_num(doc.get("updated_at")), 3),
|
|
121
|
+
"source": "legacy_file",
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
def compute(store, now=None, window_hours=DEFAULT_WINDOW_HOURS):
|
|
126
|
+
"""Aggregate usage events in the window; legacy file only if no events."""
|
|
127
|
+
stamp = float(now or time.time())
|
|
128
|
+
window = max(1.0, _num(window_hours, DEFAULT_WINDOW_HOURS))
|
|
129
|
+
cutoff = stamp - window * 3600.0
|
|
130
|
+
events = [row for row in _read_events(store)
|
|
131
|
+
if _num(row.get("ts"), 0.0) >= cutoff]
|
|
132
|
+
if not events:
|
|
133
|
+
legacy = _legacy(store)
|
|
134
|
+
if legacy is not None:
|
|
135
|
+
legacy["window_hours"] = round(window, 2)
|
|
136
|
+
return legacy
|
|
137
|
+
return {"turns": 0, "prompt_tokens": 0, "output_tokens": 0,
|
|
138
|
+
"cache_read": 0, "cache_write": 0, "hit_rate": 0.0,
|
|
139
|
+
"updated_at": 0.0, "window_hours": round(window, 2),
|
|
140
|
+
"source": "empty"}
|
|
141
|
+
prompt = sum(max(0, int(_num(row.get("prompt_tokens")))) for row in events)
|
|
142
|
+
output = sum(max(0, int(_num(row.get("output_tokens")))) for row in events)
|
|
143
|
+
cache_read = sum(max(0, int(_num(row.get("cache_read")))) for row in events)
|
|
144
|
+
cache_write = sum(max(0, int(_num(row.get("cache_write")))) for row in events)
|
|
145
|
+
denom = cache_read + cache_write
|
|
146
|
+
if prompt > 0:
|
|
147
|
+
hit_rate = cache_read / float(prompt)
|
|
148
|
+
elif denom > 0:
|
|
149
|
+
hit_rate = cache_read / float(denom)
|
|
150
|
+
else:
|
|
151
|
+
hit_rate = 0.0
|
|
152
|
+
hit_rate = round(min(1.0, max(0.0, hit_rate)), 4)
|
|
153
|
+
return {"turns": len(events), "prompt_tokens": prompt,
|
|
154
|
+
"output_tokens": output, "cache_read": cache_read,
|
|
155
|
+
"cache_write": cache_write, "hit_rate": hit_rate,
|
|
156
|
+
"updated_at": round(_num(events[-1].get("ts")), 3),
|
|
157
|
+
"window_hours": round(window, 2), "source": "usage_events"}
|
|
158
|
+
|
|
159
|
+
|
|
160
|
+
def summary(store, now=None):
|
|
161
|
+
return compute(store, now=now, window_hours=DEFAULT_WINDOW_HOURS)
|