dsh-biocore 0.3.3 → 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (59) hide show
  1. package/lib/index.js +30 -1
  2. package/lib/modules/standby.js +61 -0
  3. package/lib/modules/state.js +22 -3
  4. package/package.json +1 -1
  5. package/python/biocore_memory/core/config.py +7 -3
  6. package/python/biocore_memory/engine.py +91 -6
  7. package/python/biocore_memory/recall/embedding.py +0 -51
  8. package/python/biocore_memory/recall/feedback.py +34 -7
  9. package/python/biocore_memory/recall/fusion.py +5 -1
  10. package/python/biocore_memory/scoring.py +203 -0
  11. package/python/biocore_memory/space.py +355 -0
  12. package/python/biocore_memory/store/dedupe.py +1 -0
  13. package/python/biocore_memory/store/tiering.py +14 -3
  14. package/python/biocore_model/affective_labor.py +14 -1
  15. package/python/biocore_model/appraisal.py +172 -3
  16. package/python/biocore_model/bio_tick.py +43 -13
  17. package/python/biocore_model/emotion_geometry.py +9 -0
  18. package/python/biocore_model/envy.py +21 -0
  19. package/python/biocore_model/feedback_memory.py +53 -0
  20. package/python/biocore_model/needs_events.py +32 -11
  21. package/python/biocore_model/regulation_strategies.py +5 -0
  22. package/python/biocore_model/repair_after_boundary.py +1 -0
  23. package/python/biocore_model/self_esteem.py +14 -2
  24. package/python/biocore_model/self_intent.py +6 -4
  25. package/python/biocore_model/tests/test_autonomy_self.py +153 -153
  26. package/python/biocore_model/tests/test_cache_series.py +137 -0
  27. package/python/biocore_model/tests/test_core_state.py +240 -240
  28. package/python/biocore_model/tests/test_extensions.py +167 -167
  29. package/python/biocore_model/tests/test_field_arbiter.py +444 -0
  30. package/python/biocore_model/tests/test_governance.py +181 -181
  31. package/python/biocore_model/tests/test_lifecycle.py +153 -0
  32. package/python/biocore_model/tests/test_memory_inject.py +211 -211
  33. package/python/biocore_model/tests/test_retrieval_plus.py +143 -0
  34. package/python/biocore_model/tests/test_signals.py +106 -106
  35. package/python/biocore_model/tests/test_standby.py +199 -0
  36. package/python/biocore_runtime/__init__.py +3 -0
  37. package/python/biocore_runtime/arbiter.py +307 -0
  38. package/python/biocore_runtime/batch_b.py +724 -0
  39. package/python/biocore_runtime/batch_c.py +186 -0
  40. package/python/biocore_runtime/batch_d.py +659 -0
  41. package/python/biocore_runtime/batch_e.py +328 -0
  42. package/python/biocore_runtime/batch_registry.py +446 -0
  43. package/python/biocore_runtime/cache_stats.py +161 -0
  44. package/python/biocore_runtime/core_state.py +106 -3
  45. package/python/biocore_runtime/dimensions.py +419 -0
  46. package/python/biocore_runtime/expression.py +261 -0
  47. package/python/biocore_runtime/field.py +440 -0
  48. package/python/biocore_runtime/lifecycle.py +233 -0
  49. package/python/biocore_runtime/perception.py +1045 -0
  50. package/python/biocore_runtime/pipeline.py +4458 -2974
  51. package/python/biocore_runtime/priority.py +210 -0
  52. package/python/biocore_runtime/retrieval_plus.py +352 -0
  53. package/python/biocore_runtime/standby.py +581 -472
  54. package/python/biocore_runtime/timescales.py +263 -0
  55. package/python/data/signals.json +3266 -206
  56. package/ui/panel-layout.js +32 -29
  57. package/ui/panel.css +563 -974
  58. package/ui/panel.html +427 -401
  59. package/ui/panel.js +543 -450
@@ -0,0 +1,446 @@
1
+ # -*- coding: utf-8 -*-
2
+ """Batch integration: one registry for b / c / d / e, all ON, per-module monitor.
3
+
4
+ The four batch hooks (biocore_runtime.batch_b / batch_c / batch_d / batch_e)
5
+ have different return shapes -- b/d/e return {"modules", "activity",
6
+ "field_deltas", "state_deltas", "skipped"}, the older c returns a flat bundle
7
+ with dotted "deltas". This module normalises both, merges the safe parts into
8
+ the session state, hands the six-field proposals to the arbiter, and keeps one
9
+ monitor row per module:
10
+
11
+ <dataDir>/batch_monitor.json
12
+ {"schema": 1, "ts": ..., "batches": {"b": {"calls": n, ...}},
13
+ "modules": {"b.entity_resolve": {"calls": n, "changed": n,
14
+ "last_effect": 0.0, "errored": n,
15
+ "last_ts": 0.0}}}
16
+
17
+ No IO besides that one file; never raises into the turn.
18
+ """
19
+ from __future__ import annotations
20
+
21
+ import json
22
+ import math
23
+ import time
24
+ from pathlib import Path
25
+
26
+ try: # one resolution strategy for every field (never a cross-source sum)
27
+ from . import arbiter as _arbiter
28
+ except Exception: # pragma: no cover
29
+ try:
30
+ from biocore_runtime import arbiter as _arbiter
31
+ except Exception:
32
+ _arbiter = None
33
+
34
+ MONITOR_FILE = "batch_monitor.json"
35
+ SCHEMA = 1
36
+ MODULE_LIMIT = 320
37
+
38
+ BATCHES = ("b", "c", "d", "e")
39
+
40
+ # State blocks a batch may write top-level. Everything else is namespaced under
41
+ # state["batches"][<batch>] so a batch can never clobber the body/core fields.
42
+ SAFE_KEYS = ("entity_graph", "graph_stats", "memory_tiers", "dual_scores",
43
+ "memory_scores", "memory_strength", "co_recall", "policy_q",
44
+ "predictor_forecast", "predictor_history", "prediction",
45
+ "migrate_report", "identity_read", "autonomy", "autonomy_acc",
46
+ "expression", "state_tag", "topic_control", "reply", "silence",
47
+ "feedback", "diagnostics", "diagnostic_row", "memory_layers",
48
+ "graph", "co_recall_pairs", "reconstruction", "narrative")
49
+
50
+ FIELD_NAMES = ("v", "a", "stress", "social", "trust", "closeness")
51
+ CAPS = {"v": 0.10, "a": 0.10, "stress": 0.08, "social": 0.08,
52
+ "trust": 0.02, "closeness": 0.02}
53
+
54
+ # Which read slice belongs to which module. A module without a slice has no
55
+ # observable view of its own this turn (it reports zero change, not "changed").
56
+ READ_OWNERS = {
57
+ "b": {"identity": ("bio_identity",), "graph": ("graph_recall",),
58
+ "entities": ("entity_resolve",), "memory_tiers": ("layered_memory",),
59
+ "dual": ("memory_dual",), "forgetting": ("memory_forgetting",),
60
+ "store": ("memory_store",), "policy": ("policy_learning",),
61
+ "forecast": ("predictor",), "scores": ("scoring",),
62
+ "graph_stats": ("entity_graph",), "strength": ("feedback_memory",),
63
+ "co_recall": ("co_recall",), "migrate": ("migrate_store",)},
64
+ "d": {"autonomy": ("autonomy_check", "autonomy_loop", "autonomy_model",
65
+ "autonomy_params", "autonomy_reason", "autonomy_mode"),
66
+ "expression": ("expr_intent", "expr_tone", "granularity",
67
+ "exit_or_silence", "reply_limit", "reply_split",
68
+ "expr_state", "prompt_rules", "expr_guard",
69
+ "state_tag", "topic_control", "expr_template"),
70
+ "feedback": ("feedback_flow", "feedback_guard", "feedback_llm",
71
+ "feedback_user"),
72
+ "schedule": ("scheduler", "voice_bank", "proactive_context")},
73
+ "e": {"diagnostics": ("diag_trace", "delta_budget", "schema", "bio_const"),
74
+ "config": ("core_commands", "provider_fallback", "stages"),
75
+ "monitoring": ("source_monitoring", "mind_wander", "noise_bank"),
76
+ "probe": ("learning_probe", "extreme_scenarios")},
77
+ }
78
+ VIEW_LIMIT = 24
79
+ EFFECT_LIMIT = 8
80
+
81
+ # Reasons that are normal skips, not failures.
82
+ BENIGN = {"module_missing", "platform_isolation", "disabled", "duplicate",
83
+ "no_entries", "no_tag", "no_sample", "no_pairs", "no_choice",
84
+ "no_context", "no_action", "no_store", "no_forecast",
85
+ "nothing_to_resolve", "bad_phase", "unavailable", "no_snapshot",
86
+ "convert_failed", "no_deltas", "dry_run", "no_change"}
87
+
88
+
89
+ def _num(value, default=0.0):
90
+ try:
91
+ number = float(value)
92
+ except Exception:
93
+ return default
94
+ if not math.isfinite(number):
95
+ return default
96
+ return number
97
+
98
+
99
+ def _root(store):
100
+ if isinstance(store, dict):
101
+ raw = store.get("root") or store.get("store") or store.get("path")
102
+ else:
103
+ raw = store
104
+ text = str(raw or "").strip()
105
+ return Path(text) if text else None
106
+
107
+
108
+ def _data_dir(store):
109
+ root = _root(store)
110
+ return (root.parent if root.name == "memory" else root) if root else None
111
+
112
+
113
+ def _read(path):
114
+ try:
115
+ if path is None or not path.exists():
116
+ return {}
117
+ doc = json.loads(path.read_text(encoding="utf-8"))
118
+ return doc if isinstance(doc, dict) else {}
119
+ except Exception:
120
+ return {}
121
+
122
+
123
+ def _write(path, doc):
124
+ try:
125
+ path.parent.mkdir(parents=True, exist_ok=True)
126
+ tmp = path.with_name(path.name + ".tmp")
127
+ tmp.write_text(json.dumps(doc, ensure_ascii=False, indent=2),
128
+ encoding="utf-8")
129
+ tmp.replace(path)
130
+ return True
131
+ except Exception:
132
+ return False
133
+
134
+
135
+ def path(store):
136
+ root = _data_dir(store)
137
+ return (root / MONITOR_FILE) if root is not None else None
138
+
139
+
140
+ # ---------------------------------------------------------------- normalise
141
+ def modules_of(batch_name, output):
142
+ """The module names one batch claims (declared list or c-style fields)."""
143
+ if not isinstance(output, dict):
144
+ return ()
145
+ declared = output.get("modules")
146
+ if isinstance(declared, dict):
147
+ return tuple(declared.keys())
148
+ fields = output.get("fields")
149
+ if isinstance(fields, list):
150
+ return tuple(str(item) for item in fields)
151
+ return ()
152
+
153
+
154
+ def normalize(batch_name, output):
155
+ """Both shapes -> {enabled, modules, activity, field_deltas,
156
+ state_deltas, skipped, errored, raw_keys}."""
157
+ name = str(batch_name or "")
158
+ if not isinstance(output, dict):
159
+ return {"enabled": False, "reason": "bad_output", "modules": {},
160
+ "activity": {}, "field_deltas": {}, "state_deltas": {},
161
+ "skipped": [], "errored": 0, "raw_keys": []}
162
+ declared = output.get("modules")
163
+ if isinstance(declared, dict): # b / d / e
164
+ modules = {str(key): bool(value) for key, value in declared.items()}
165
+ activity = output.get("activity")
166
+ activity = ({str(key): bool(value) for key, value in activity.items()}
167
+ if isinstance(activity, dict) else dict(modules))
168
+ field_deltas = {key: _num(value) for key, value in
169
+ (output.get("field_deltas") or {}).items()
170
+ if key in FIELD_NAMES}
171
+ state_deltas = dict(output.get("state_deltas") or {})
172
+ skipped = [dict(row) for row in (output.get("skipped") or [])
173
+ if isinstance(row, dict)]
174
+ enabled = bool(output.get("enabled", True))
175
+ else: # batch_c style: flat bundle + dotted deltas
176
+ fields = [str(item) for item in (output.get("fields") or [])]
177
+ modules = {field: bool(output.get("ok", True)) for field in fields}
178
+ activity = dict(modules)
179
+ field_deltas = {}
180
+ for key, value in (output.get("deltas") or {}).items():
181
+ head = str(key).split(".")[0]
182
+ if head in FIELD_NAMES:
183
+ field_deltas[head] = _num(value)
184
+ state_deltas = {"bundle": output}
185
+ skipped = []
186
+ enabled = bool(output.get("ok", True))
187
+ errored = 0
188
+ for row in skipped:
189
+ reason = str(row.get("reason") or "")
190
+ if reason and reason not in BENIGN:
191
+ errored += 1
192
+ reads = output.get("reads")
193
+ if not isinstance(reads, dict):
194
+ reads = output if not isinstance(declared, dict) else {}
195
+ return {"enabled": enabled, "reason": str(output.get("reason") or "ok"),
196
+ "modules": modules, "activity": activity,
197
+ "field_deltas": field_deltas, "state_deltas": state_deltas,
198
+ "skipped": skipped, "errored": errored,
199
+ "reads": reads if isinstance(reads, dict) else {},
200
+ "raw_keys": sorted(str(key) for key in output.keys())}
201
+
202
+
203
+ # ---------------------------------------------------------------- monitor
204
+ def read_monitor(store):
205
+ doc = _read(path(store))
206
+ modules = doc.get("modules") if isinstance(doc.get("modules"), dict) else {}
207
+ batches = doc.get("batches") if isinstance(doc.get("batches"), dict) else {}
208
+ return {"schema": SCHEMA, "ts": _num(doc.get("ts"), 0.0),
209
+ "batches": {str(key): dict(value) for key, value in batches.items()
210
+ if isinstance(value, dict)},
211
+ "modules": {str(key): dict(value) for key, value in modules.items()
212
+ if isinstance(value, dict)},
213
+ "totals": _totals_of(doc)}
214
+
215
+
216
+ def flatten(value, prefix="", out=None, limit=VIEW_LIMIT):
217
+ """Numeric channels of one module view: nested containers -> dotted keys."""
218
+ out = {} if out is None else out
219
+ if len(out) >= limit:
220
+ return out
221
+ if isinstance(value, dict):
222
+ for key, item in value.items():
223
+ if len(out) >= limit:
224
+ break
225
+ label = "%s.%s" % (prefix, key) if prefix else str(key)
226
+ flatten(item, label, out, limit)
227
+ elif isinstance(value, (list, tuple)):
228
+ for index, item in enumerate(list(value)[:4]):
229
+ flatten(item, "%s[%d]" % (prefix, index), out, limit)
230
+ elif isinstance(value, bool):
231
+ out[prefix] = 1.0 if value else 0.0
232
+ else:
233
+ number = _num(value, None)
234
+ if number is not None:
235
+ out[prefix] = round(number, 6)
236
+ return out
237
+
238
+
239
+ def module_view(batch, module, row):
240
+ """The read slice one module owns (its own output, not the batch total)."""
241
+ row = row if isinstance(row, dict) else {}
242
+ reads = row.get("reads") if isinstance(row.get("reads"), dict) else {}
243
+ view = {}
244
+ for key, modules in (READ_OWNERS.get(str(batch)) or {}).items():
245
+ if str(module) not in modules:
246
+ continue
247
+ value = reads.get(key)
248
+ if value is None:
249
+ continue
250
+ view.update(flatten(value, str(key), view))
251
+ if len(view) >= VIEW_LIMIT:
252
+ break
253
+ if str(batch) == "c":
254
+ bundle = (row.get("state_deltas") or {}).get("bundle")
255
+ bundle = bundle if isinstance(bundle, dict) else {}
256
+ if module in bundle:
257
+ view.update(flatten(bundle.get(module), str(module), view))
258
+ return dict(list(view.items())[:VIEW_LIMIT])
259
+
260
+
261
+ def diff_view(before, after, eps=1e-6, limit=EFFECT_LIMIT):
262
+ """{channel: delta} for the channels this module itself moved."""
263
+ old = before if isinstance(before, dict) else {}
264
+ new = after if isinstance(after, dict) else {}
265
+ changed = {}
266
+ for key in set(old) | set(new):
267
+ delta = round(_num(new.get(key), 0.0) - _num(old.get(key), 0.0), 6)
268
+ if abs(delta) > eps:
269
+ changed[key] = delta
270
+ if len(changed) > limit:
271
+ ordered = sorted(changed.items(), key=lambda item: -abs(item[1]))
272
+ changed = dict(ordered[:limit])
273
+ return changed
274
+
275
+
276
+ def _totals_of(doc):
277
+ totals = doc.get("totals") if isinstance(doc.get("totals"), dict) else {}
278
+ return {"calls": int(_num(totals.get("calls"), 0)),
279
+ "changed": int(_num(totals.get("changed"), 0)),
280
+ "zero_change": int(_num(totals.get("zero_change"), 0)),
281
+ "errored": int(_num(totals.get("errored"), 0)),
282
+ "active": int(_num(totals.get("active"), 0))}
283
+
284
+
285
+ def _bump_monitor(store, snapshots, now, changed_batches):
286
+ """One monitor row per module: calls / changed / zero_change / effect."""
287
+ doc = read_monitor(store)
288
+ stamp = round(_num(now, time.time()), 3)
289
+ doc["ts"] = stamp
290
+ totals = _totals_of(doc)
291
+ errors_this_tick = 0
292
+ for name, row in snapshots.items():
293
+ batch_row = doc["batches"].setdefault(
294
+ name, {"calls": 0, "changed": 0, "zero_change": 0, "errored": 0})
295
+ batch_row["calls"] = int(_num(batch_row.get("calls"), 0)) + 1
296
+ batch_row["last_ts"] = stamp
297
+ batch_row.setdefault("zero_change", 0)
298
+ activity = row.get("activity") if isinstance(row.get("activity"), dict) else {}
299
+ errors = int(_num(row.get("errored"), 0))
300
+ if errors:
301
+ errors_this_tick += errors
302
+ batch_row["errored"] = int(_num(batch_row.get("errored"), 0)) + errors
303
+ for module in (row.get("modules") or {}):
304
+ if not bool(activity.get(module)):
305
+ continue # declared but idle this phase: not a call, no row
306
+ key = "%s.%s" % (name, module)
307
+ slot = doc["modules"].setdefault(
308
+ key, {"calls": 0, "changed": 0, "zero_change": 0,
309
+ "last_effect": {}, "errored": 0})
310
+ slot["calls"] = int(_num(slot.get("calls"), 0)) + 1
311
+ slot.setdefault("zero_change", 0)
312
+ slot["last_ts"] = stamp
313
+ view = module_view(name, module, row)
314
+ before = slot.get("view") if isinstance(slot.get("view"), dict) else None
315
+ effect = diff_view(before or {}, view)
316
+ totals["calls"] += 1
317
+ if before is None:
318
+ # first snapshot: nothing to compare yet, so not a change
319
+ slot["zero_change"] += 1
320
+ totals["zero_change"] += 1
321
+ batch_row["zero_change"] += 1
322
+ elif effect:
323
+ slot["changed"] = int(_num(slot.get("changed"), 0)) + 1
324
+ slot["last_effect"] = effect
325
+ totals["changed"] += 1
326
+ batch_row["changed"] = int(_num(batch_row.get("changed"), 0)) + 1
327
+ else:
328
+ slot["zero_change"] += 1
329
+ totals["zero_change"] += 1
330
+ batch_row["zero_change"] += 1
331
+ if view:
332
+ slot["view"] = view
333
+ if name in changed_batches and effect:
334
+ slot["last_ts"] = stamp
335
+ if errors:
336
+ slot["errored"] = int(_num(slot.get("errored"), 0)) + errors
337
+ totals["errored"] += errors_this_tick
338
+ totals["active"] = len(doc["modules"])
339
+ doc["totals"] = totals
340
+ if len(doc["modules"]) > MODULE_LIMIT:
341
+ ordered = sorted(doc["modules"].items(),
342
+ key=lambda item: -_num(item[1].get("last_ts"), 0.0))
343
+ doc["modules"] = dict(ordered[:MODULE_LIMIT])
344
+ _write(path(store), doc)
345
+ return doc
346
+
347
+
348
+ # ---------------------------------------------------------------- tick
349
+ def resolve_field(proposals, strategy=None):
350
+ """(delta, strategy) for one field: the four batches are NOT summed.
351
+
352
+ Each batch keeps its own proposal row; the field value is chosen by the
353
+ arbiter's strategy (weighted median by default) over those proposals.
354
+ """
355
+ rows = [(_num(row.get("delta"), 0.0), 1.0) for row in proposals or []
356
+ if isinstance(row, dict)]
357
+ if not rows:
358
+ return 0.0, "none"
359
+ if _arbiter is not None:
360
+ return _arbiter.resolve(rows, strategy)
361
+ best = max(rows, key=lambda item: abs(item[0]))
362
+ return best[0], "max_magnitude"
363
+
364
+
365
+ def tick(store, session_id, phase, payload=None, state=None, now=None,
366
+ switches=None, hooks=None):
367
+ """Run every enabled batch once; returns normalized rows + per-field deltas.
368
+
369
+ A field may be named by more than one batch: every proposal is kept in
370
+ field_proposals{field: [{batch, delta}]} and the value comes from
371
+ resolve_field(), never from adding the batches together.
372
+ """
373
+ data = payload if isinstance(payload, dict) else {}
374
+ session_state = state if isinstance(state, dict) else {}
375
+ stamp = round(_num(now if now is not None else data.get("now"),
376
+ time.time()), 3)
377
+ hooks = hooks if isinstance(hooks, dict) else {}
378
+ table = switches if isinstance(switches, dict) else {}
379
+ rows = {}
380
+ field_proposals = {}
381
+ field_deltas = {}
382
+ resolved = {}
383
+ state_deltas = {}
384
+ snapshots = {}
385
+ changed_batches = set()
386
+ skipped_batches = []
387
+ for name in BATCHES:
388
+ hook = hooks.get(name)
389
+ if hook is None:
390
+ skipped_batches.append({"batch": name, "reason": "unavailable"})
391
+ continue
392
+ if table and table.get("batch_" + name) is False:
393
+ skipped_batches.append({"batch": name, "reason": "disabled"})
394
+ continue
395
+ try:
396
+ output = hook.apply_turn(session_state, phase, data)
397
+ except Exception as error: # a broken batch never breaks a turn
398
+ rows[name] = {"enabled": False, "reason": type(error).__name__,
399
+ "modules": {}, "activity": {}, "field_deltas": {},
400
+ "state_deltas": {}, "skipped": [], "errored": 1,
401
+ "raw_keys": []}
402
+ snapshots[name] = rows[name]
403
+ continue
404
+ row = normalize(name, output)
405
+ rows[name] = row
406
+ snapshots[name] = row
407
+ if row["field_deltas"] or row["state_deltas"]:
408
+ changed_batches.add(name)
409
+ for key, value in row["field_deltas"].items():
410
+ if key in FIELD_NAMES:
411
+ field_proposals.setdefault(key, []).append(
412
+ {"batch": name, "delta": round(_num(value), 6)})
413
+ for key, value in row["state_deltas"].items():
414
+ if key in SAFE_KEYS:
415
+ state_deltas[key] = value
416
+ else:
417
+ bucket = state_deltas.setdefault("batches", {})
418
+ if isinstance(bucket, dict):
419
+ bucket.setdefault(name, {})[str(key)] = value
420
+ for key, proposals in field_proposals.items():
421
+ chosen, strategy = resolve_field(proposals)
422
+ capped = round(max(-CAPS[key], min(CAPS[key], chosen)), 6)
423
+ field_deltas[key] = capped
424
+ resolved[key] = {"strategy": strategy, "delta": round(chosen, 6),
425
+ "capped": capped, "proposals": proposals}
426
+ monitor = _bump_monitor(store, snapshots, stamp, changed_batches)
427
+ return {"enabled": True, "phase": str(phase), "ts": stamp,
428
+ "batches": rows, "field_deltas": field_deltas,
429
+ "field_proposals": field_proposals, "resolved": resolved,
430
+ "state_deltas": state_deltas, "skipped": skipped_batches,
431
+ "monitor": {"batches": monitor.get("batches", {}),
432
+ "modules": monitor.get("modules", {}),
433
+ "totals": _totals_of(monitor)},
434
+ "counts": {"batches": len(rows),
435
+ "modules": sum(len(row.get("modules") or {})
436
+ for row in rows.values()),
437
+ "changed": len(changed_batches)}}
438
+
439
+
440
+ def summary(store):
441
+ """Read-only monitor projection for /state."""
442
+ doc = read_monitor(store)
443
+ totals = _totals_of(doc)
444
+ totals["active"] = len(doc["modules"])
445
+ return {"ts": doc["ts"], "batches": doc["batches"],
446
+ "modules": doc["modules"], "totals": totals}
@@ -0,0 +1,161 @@
1
+ # -*- coding: utf-8 -*-
2
+ """Cache/usage accounting in the core (contract migration 1).
3
+
4
+ Usage events live in <dataDir>/usage_events.jsonl; the core is the only writer.
5
+ The adapter may still read the legacy <dataDir>/cache_stats.json for a week as
6
+ a read-only fallback, but it never writes or recomputes hit_rate.
7
+
8
+ hit_rate = cache_read / (cache_read + cache_write), falling back to
9
+ cache_read / prompt_tokens when no write tokens were reported; clamped to
10
+ [0, 1]. The default window is 7 days and every event is bounded.
11
+ """
12
+ from __future__ import annotations
13
+
14
+ import json
15
+ import time
16
+ from pathlib import Path
17
+
18
+ USAGE_FILE = "usage_events.jsonl"
19
+ LEGACY_FILE = "cache_stats.json"
20
+ SCHEMA = 1
21
+ DEFAULT_WINDOW_HOURS = 24 * 7
22
+ MAX_EVENTS = 5000
23
+
24
+
25
+ def _root(store):
26
+ if isinstance(store, dict):
27
+ raw = store.get("root") or store.get("store") or store.get("path")
28
+ else:
29
+ raw = store
30
+ text = str(raw or "").strip()
31
+ return Path(text) if text else None
32
+
33
+
34
+ def _data_dir(store):
35
+ root = _root(store)
36
+ return (root.parent if root.name == "memory" else root) if root else None
37
+
38
+
39
+ def _num(value, default=0.0):
40
+ try:
41
+ number = float(value)
42
+ except Exception:
43
+ return default
44
+ return number if number == number else default
45
+
46
+
47
+ def usage_path(store):
48
+ data_dir = _data_dir(store)
49
+ return (data_dir / USAGE_FILE) if data_dir is not None else None
50
+
51
+
52
+ def legacy_path(store):
53
+ data_dir = _data_dir(store)
54
+ return (data_dir / LEGACY_FILE) if data_dir is not None else None
55
+
56
+
57
+ def _read_events(store):
58
+ path = usage_path(store)
59
+ rows = []
60
+ try:
61
+ if path is None or not path.exists():
62
+ return rows
63
+ for line in path.read_text(encoding="utf-8").splitlines():
64
+ line = line.strip()
65
+ if not line:
66
+ continue
67
+ try:
68
+ item = json.loads(line)
69
+ except Exception:
70
+ continue
71
+ if isinstance(item, dict):
72
+ rows.append(item)
73
+ except Exception:
74
+ return []
75
+ return rows
76
+
77
+
78
+ def record_event(store, event, now=None):
79
+ """Append one usage event; bounded to MAX_EVENTS."""
80
+ path = usage_path(store)
81
+ if path is None:
82
+ return None
83
+ stamp = round(float(now or event.get("ts") or time.time()), 3)
84
+ row = {
85
+ "ts": stamp,
86
+ "prompt_tokens": max(0, int(_num(event.get("prompt_tokens")))),
87
+ "output_tokens": max(0, int(_num(event.get("output_tokens")))),
88
+ "cache_read": max(0, int(_num(event.get("cache_read")))),
89
+ "cache_write": max(0, int(_num(event.get("cache_write")))),
90
+ }
91
+ try:
92
+ path.parent.mkdir(parents=True, exist_ok=True)
93
+ rows = _read_events(store)[-(MAX_EVENTS - 1):] + [row]
94
+ tmp = path.with_name(path.name + ".tmp")
95
+ tmp.write_text("\n".join(json.dumps(item, ensure_ascii=False)
96
+ for item in rows) + "\n", encoding="utf-8")
97
+ tmp.replace(path)
98
+ except Exception:
99
+ return None
100
+ return row
101
+
102
+
103
+ def _legacy(store):
104
+ path = legacy_path(store)
105
+ try:
106
+ if path is None or not path.exists():
107
+ return None
108
+ doc = json.loads(path.read_text(encoding="utf-8"))
109
+ except Exception:
110
+ return None
111
+ if not isinstance(doc, dict):
112
+ return None
113
+ return {
114
+ "turns": max(0, int(_num(doc.get("turns")))),
115
+ "prompt_tokens": max(0, int(_num(doc.get("prompt_tokens")))),
116
+ "output_tokens": max(0, int(_num(doc.get("output_tokens")))),
117
+ "cache_read": max(0, int(_num(doc.get("cache_read")))),
118
+ "cache_write": max(0, int(_num(doc.get("cache_write")))),
119
+ "hit_rate": round(min(1.0, max(0.0, _num(doc.get("hit_rate")))), 4),
120
+ "updated_at": round(_num(doc.get("updated_at")), 3),
121
+ "source": "legacy_file",
122
+ }
123
+
124
+
125
+ def compute(store, now=None, window_hours=DEFAULT_WINDOW_HOURS):
126
+ """Aggregate usage events in the window; legacy file only if no events."""
127
+ stamp = float(now or time.time())
128
+ window = max(1.0, _num(window_hours, DEFAULT_WINDOW_HOURS))
129
+ cutoff = stamp - window * 3600.0
130
+ events = [row for row in _read_events(store)
131
+ if _num(row.get("ts"), 0.0) >= cutoff]
132
+ if not events:
133
+ legacy = _legacy(store)
134
+ if legacy is not None:
135
+ legacy["window_hours"] = round(window, 2)
136
+ return legacy
137
+ return {"turns": 0, "prompt_tokens": 0, "output_tokens": 0,
138
+ "cache_read": 0, "cache_write": 0, "hit_rate": 0.0,
139
+ "updated_at": 0.0, "window_hours": round(window, 2),
140
+ "source": "empty"}
141
+ prompt = sum(max(0, int(_num(row.get("prompt_tokens")))) for row in events)
142
+ output = sum(max(0, int(_num(row.get("output_tokens")))) for row in events)
143
+ cache_read = sum(max(0, int(_num(row.get("cache_read")))) for row in events)
144
+ cache_write = sum(max(0, int(_num(row.get("cache_write")))) for row in events)
145
+ denom = cache_read + cache_write
146
+ if prompt > 0:
147
+ hit_rate = cache_read / float(prompt)
148
+ elif denom > 0:
149
+ hit_rate = cache_read / float(denom)
150
+ else:
151
+ hit_rate = 0.0
152
+ hit_rate = round(min(1.0, max(0.0, hit_rate)), 4)
153
+ return {"turns": len(events), "prompt_tokens": prompt,
154
+ "output_tokens": output, "cache_read": cache_read,
155
+ "cache_write": cache_write, "hit_rate": hit_rate,
156
+ "updated_at": round(_num(events[-1].get("ts")), 3),
157
+ "window_hours": round(window, 2), "source": "usage_events"}
158
+
159
+
160
+ def summary(store, now=None):
161
+ return compute(store, now=now, window_hours=DEFAULT_WINDOW_HOURS)