nemoris 0.1.0 → 0.1.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (248) hide show
  1. package/.env.example +49 -49
  2. package/LICENSE +21 -21
  3. package/README.md +209 -209
  4. package/SECURITY.md +59 -119
  5. package/bin/nemoris +46 -46
  6. package/config/agents/agent.toml.example +28 -28
  7. package/config/agents/content.toml +23 -0
  8. package/config/agents/default.toml +22 -22
  9. package/config/agents/heartbeat.toml +35 -0
  10. package/config/agents/iris.toml +23 -0
  11. package/config/agents/lab.toml +23 -0
  12. package/config/agents/main.toml +45 -0
  13. package/config/agents/nemo.toml +21 -0
  14. package/config/agents/ops.toml +38 -0
  15. package/config/agents/orchestrator.toml +18 -18
  16. package/config/agents/revenue.toml +23 -0
  17. package/config/agents/testyboo.toml +19 -0
  18. package/config/delivery.toml +73 -73
  19. package/config/embeddings.toml +5 -5
  20. package/config/identity/content-purpose.md +11 -0
  21. package/config/identity/content-soul.md +45 -0
  22. package/config/identity/default-purpose.md +1 -1
  23. package/config/identity/default-soul.md +3 -3
  24. package/config/identity/heartbeat-purpose.md +9 -0
  25. package/config/identity/heartbeat-soul.md +16 -0
  26. package/config/identity/iris-purpose.md +17 -0
  27. package/config/identity/iris-soul.md +68 -0
  28. package/config/identity/lab-purpose.md +10 -0
  29. package/config/identity/lab-soul.md +38 -0
  30. package/config/identity/main-purpose.md +17 -0
  31. package/config/identity/main-soul.md +66 -0
  32. package/config/identity/main-user.md +22 -0
  33. package/config/identity/ops-purpose.md +9 -0
  34. package/config/identity/ops-soul.md +16 -0
  35. package/config/identity/orchestrator-purpose.md +1 -1
  36. package/config/identity/orchestrator-soul.md +1 -1
  37. package/config/identity/revenue-purpose.md +9 -0
  38. package/config/identity/revenue-soul.md +41 -0
  39. package/config/identity/testyboo-purpose.md +13 -0
  40. package/config/identity/testyboo-soul.md +20 -0
  41. package/config/improvement-targets.toml +15 -15
  42. package/config/jobs/heartbeat-check.toml +30 -30
  43. package/config/jobs/memory-rollup.toml +46 -46
  44. package/config/jobs/workspace-health.toml +63 -63
  45. package/config/mcp.toml +16 -16
  46. package/config/output-contracts.toml +17 -17
  47. package/config/peers.toml +32 -32
  48. package/config/peers.toml.example +32 -32
  49. package/config/policies/memory-default.toml +10 -10
  50. package/config/policies/memory-heartbeat.toml +5 -5
  51. package/config/policies/memory-ops.toml +10 -10
  52. package/config/policies/tools-heartbeat-minimal.toml +8 -8
  53. package/config/policies/tools-interactive-safe.toml +8 -8
  54. package/config/policies/tools-ops-bounded.toml +8 -8
  55. package/config/policies/tools-orchestrator.toml +7 -7
  56. package/config/providers/anthropic.toml +15 -15
  57. package/config/providers/ollama.toml +5 -5
  58. package/config/providers/openai-codex.toml +9 -9
  59. package/config/providers/openrouter.toml +5 -5
  60. package/config/router.toml +22 -22
  61. package/config/runtime.toml +114 -114
  62. package/config/skills/self-improvement.toml +15 -15
  63. package/config/skills/telegram-onboarding-spec.md +240 -240
  64. package/config/skills/workspace-monitor.toml +15 -15
  65. package/config/task-router.toml +42 -42
  66. package/install.sh +50 -50
  67. package/package.json +91 -90
  68. package/src/auth/auth-profiles.js +169 -169
  69. package/src/auth/openai-codex-oauth.js +285 -285
  70. package/src/battle.js +449 -449
  71. package/src/cli/help.js +265 -265
  72. package/src/cli/output-filter.js +49 -49
  73. package/src/cli/runtime-control.js +704 -704
  74. package/src/cli-main.js +2763 -2763
  75. package/src/cli.js +78 -78
  76. package/src/config/loader.js +332 -332
  77. package/src/config/schema-validator.js +214 -214
  78. package/src/config/toml-lite.js +8 -8
  79. package/src/daemon/action-handlers.js +71 -71
  80. package/src/daemon/healing-tick.js +87 -87
  81. package/src/daemon/health-probes.js +90 -90
  82. package/src/daemon/notifier.js +57 -57
  83. package/src/daemon/nurse.js +218 -218
  84. package/src/daemon/repair-log.js +106 -106
  85. package/src/daemon/rule-staging.js +90 -90
  86. package/src/daemon/rules.js +29 -29
  87. package/src/daemon/telegram-commands.js +54 -54
  88. package/src/daemon/updater.js +85 -85
  89. package/src/jobs/job-runner.js +78 -78
  90. package/src/mcp/consumer.js +129 -129
  91. package/src/memory/active-recall.js +171 -171
  92. package/src/memory/backend-manager.js +97 -97
  93. package/src/memory/backends/file-backend.js +38 -38
  94. package/src/memory/backends/qmd-backend.js +219 -219
  95. package/src/memory/embedding-guards.js +24 -24
  96. package/src/memory/embedding-index.js +118 -118
  97. package/src/memory/embedding-service.js +179 -179
  98. package/src/memory/file-index.js +177 -177
  99. package/src/memory/memory-signature.js +5 -5
  100. package/src/memory/memory-store.js +648 -648
  101. package/src/memory/retrieval-planner.js +66 -66
  102. package/src/memory/scoring.js +145 -145
  103. package/src/memory/simhash.js +78 -78
  104. package/src/memory/sqlite-active-store.js +824 -824
  105. package/src/memory/write-policy.js +36 -36
  106. package/src/onboarding/aliases.js +33 -33
  107. package/src/onboarding/auth/api-key.js +224 -224
  108. package/src/onboarding/auth/ollama-detect.js +42 -42
  109. package/src/onboarding/clack-prompter.js +77 -77
  110. package/src/onboarding/doctor.js +530 -530
  111. package/src/onboarding/lock.js +42 -42
  112. package/src/onboarding/model-catalog.js +344 -344
  113. package/src/onboarding/phases/auth.js +576 -589
  114. package/src/onboarding/phases/build.js +130 -130
  115. package/src/onboarding/phases/choose.js +82 -82
  116. package/src/onboarding/phases/detect.js +98 -98
  117. package/src/onboarding/phases/hatch.js +216 -216
  118. package/src/onboarding/phases/identity.js +79 -79
  119. package/src/onboarding/phases/ollama.js +345 -345
  120. package/src/onboarding/phases/scaffold.js +99 -99
  121. package/src/onboarding/phases/telegram.js +377 -377
  122. package/src/onboarding/phases/validate.js +204 -204
  123. package/src/onboarding/phases/verify.js +206 -206
  124. package/src/onboarding/platform.js +482 -482
  125. package/src/onboarding/status-bar.js +95 -95
  126. package/src/onboarding/templates.js +794 -794
  127. package/src/onboarding/toml-writer.js +38 -38
  128. package/src/onboarding/tui.js +250 -250
  129. package/src/onboarding/uninstall.js +153 -153
  130. package/src/onboarding/wizard.js +516 -499
  131. package/src/providers/anthropic.js +168 -168
  132. package/src/providers/base.js +247 -247
  133. package/src/providers/circuit-breaker.js +136 -136
  134. package/src/providers/ollama.js +163 -163
  135. package/src/providers/openai-codex.js +149 -149
  136. package/src/providers/openrouter.js +136 -136
  137. package/src/providers/registry.js +36 -36
  138. package/src/providers/router.js +16 -16
  139. package/src/runtime/bootstrap-cache.js +47 -47
  140. package/src/runtime/capabilities-prompt.js +25 -25
  141. package/src/runtime/completion-ping.js +99 -99
  142. package/src/runtime/config-validator.js +121 -121
  143. package/src/runtime/context-ledger.js +360 -360
  144. package/src/runtime/cutover-readiness.js +42 -42
  145. package/src/runtime/daemon.js +729 -729
  146. package/src/runtime/delivery-ack.js +195 -195
  147. package/src/runtime/delivery-adapters/local-file.js +41 -41
  148. package/src/runtime/delivery-adapters/openclaw-cli.js +94 -94
  149. package/src/runtime/delivery-adapters/openclaw-peer.js +98 -98
  150. package/src/runtime/delivery-adapters/shadow.js +13 -13
  151. package/src/runtime/delivery-adapters/standalone-http.js +98 -98
  152. package/src/runtime/delivery-adapters/telegram.js +104 -104
  153. package/src/runtime/delivery-adapters/tui.js +128 -128
  154. package/src/runtime/delivery-manager.js +807 -807
  155. package/src/runtime/delivery-store.js +168 -168
  156. package/src/runtime/dependency-health.js +118 -118
  157. package/src/runtime/envelope.js +114 -114
  158. package/src/runtime/evaluation.js +1089 -1089
  159. package/src/runtime/exec-approvals.js +216 -216
  160. package/src/runtime/executor.js +500 -500
  161. package/src/runtime/failure-ping.js +67 -67
  162. package/src/runtime/flows.js +83 -83
  163. package/src/runtime/guards.js +45 -45
  164. package/src/runtime/handoff.js +51 -51
  165. package/src/runtime/identity-cache.js +28 -28
  166. package/src/runtime/improvement-engine.js +109 -109
  167. package/src/runtime/improvement-harness.js +581 -581
  168. package/src/runtime/input-sanitiser.js +72 -72
  169. package/src/runtime/interaction-contract.js +347 -347
  170. package/src/runtime/lane-readiness.js +226 -226
  171. package/src/runtime/migration.js +323 -323
  172. package/src/runtime/model-resolution.js +78 -78
  173. package/src/runtime/network.js +64 -64
  174. package/src/runtime/notification-store.js +97 -97
  175. package/src/runtime/notifier.js +256 -256
  176. package/src/runtime/orchestrator.js +53 -53
  177. package/src/runtime/orphan-reaper.js +41 -41
  178. package/src/runtime/output-contract-schema.js +139 -139
  179. package/src/runtime/output-contract-validator.js +439 -439
  180. package/src/runtime/peer-readiness.js +69 -69
  181. package/src/runtime/peer-registry.js +133 -133
  182. package/src/runtime/pilot-status.js +108 -108
  183. package/src/runtime/prompt-builder.js +261 -261
  184. package/src/runtime/provider-attempt.js +582 -582
  185. package/src/runtime/report-fallback.js +71 -71
  186. package/src/runtime/result-normalizer.js +183 -183
  187. package/src/runtime/retention.js +74 -74
  188. package/src/runtime/review.js +244 -244
  189. package/src/runtime/route-job.js +15 -15
  190. package/src/runtime/run-store.js +38 -38
  191. package/src/runtime/schedule.js +88 -88
  192. package/src/runtime/scheduler-state.js +434 -434
  193. package/src/runtime/scheduler.js +656 -656
  194. package/src/runtime/session-compactor.js +182 -182
  195. package/src/runtime/session-search.js +155 -155
  196. package/src/runtime/slack-inbound.js +249 -249
  197. package/src/runtime/ssrf.js +102 -102
  198. package/src/runtime/status-aggregator.js +330 -330
  199. package/src/runtime/task-contract.js +140 -140
  200. package/src/runtime/task-packet.js +107 -107
  201. package/src/runtime/task-router.js +140 -140
  202. package/src/runtime/telegram-inbound.js +1565 -1565
  203. package/src/runtime/token-counter.js +134 -134
  204. package/src/runtime/token-estimator.js +59 -59
  205. package/src/runtime/tool-loop.js +200 -200
  206. package/src/runtime/transport-server.js +311 -311
  207. package/src/runtime/tui-server.js +411 -411
  208. package/src/runtime/ulid.js +44 -44
  209. package/src/security/ssrf-check.js +197 -197
  210. package/src/setup.js +369 -369
  211. package/src/shadow/bridge.js +303 -303
  212. package/src/skills/loader.js +84 -84
  213. package/src/tools/catalog.json +49 -49
  214. package/src/tools/cli-delegate.js +44 -44
  215. package/src/tools/mcp-client.js +106 -106
  216. package/src/tools/micro/cancel-task.js +6 -6
  217. package/src/tools/micro/complete-task.js +6 -6
  218. package/src/tools/micro/fail-task.js +6 -6
  219. package/src/tools/micro/http-fetch.js +74 -74
  220. package/src/tools/micro/index.js +36 -36
  221. package/src/tools/micro/lcm-recall.js +60 -60
  222. package/src/tools/micro/list-dir.js +17 -17
  223. package/src/tools/micro/list-skills.js +46 -46
  224. package/src/tools/micro/load-skill.js +38 -38
  225. package/src/tools/micro/memory-search.js +45 -45
  226. package/src/tools/micro/read-file.js +11 -11
  227. package/src/tools/micro/session-search.js +54 -54
  228. package/src/tools/micro/shell-exec.js +43 -43
  229. package/src/tools/micro/trigger-job.js +79 -79
  230. package/src/tools/micro/web-search.js +58 -58
  231. package/src/tools/micro/workspace-paths.js +39 -39
  232. package/src/tools/micro/write-file.js +14 -14
  233. package/src/tools/micro/write-memory.js +41 -41
  234. package/src/tools/registry.js +348 -348
  235. package/src/tools/tool-result-contract.js +36 -36
  236. package/src/tui/chat.js +835 -835
  237. package/src/tui/renderer.js +175 -175
  238. package/src/tui/socket-client.js +217 -217
  239. package/src/utils/canonical-json.js +29 -29
  240. package/src/utils/compaction.js +30 -30
  241. package/src/utils/env-loader.js +5 -5
  242. package/src/utils/errors.js +80 -80
  243. package/src/utils/fs.js +101 -101
  244. package/src/utils/ids.js +5 -5
  245. package/src/utils/model-context-limits.js +30 -30
  246. package/src/utils/token-budget.js +74 -74
  247. package/src/utils/usage-cost.js +25 -25
  248. package/src/utils/usage-metrics.js +14 -14
@@ -1,360 +1,360 @@
1
- /**
2
- * Context Ledger — event-sourced session store (Phase 1).
3
- *
4
- * Append-only event log with sha256 dedup, snapshots, and outbox
5
- * for exactly-once side effects. SQLite + WAL mode.
6
- */
7
-
8
- import { createHash, randomUUID } from "node:crypto";
9
- import { DatabaseSync } from "node:sqlite";
10
- import { canonicalJson } from "../utils/canonical-json.js";
11
-
12
- const VALID_KINDS = new Set([
13
- "message_in", "message_out", "tool_call", "tool_result",
14
- "memory_write", "state_patch", "compaction",
15
- ]);
16
-
17
- export class ContextLedger {
18
- #snapshotEveryNEvents;
19
- #lastSnapshotEventId;
20
- #sessionStates;
21
-
22
- constructor({ dbPath, snapshotEveryNEvents = 10 }) {
23
- this.dbPath = dbPath;
24
- this.db = null;
25
- this.#snapshotEveryNEvents = snapshotEveryNEvents;
26
- this.#lastSnapshotEventId = new Map();
27
- this.#sessionStates = new Map();
28
- }
29
-
30
- open() {
31
- if (this.db) return;
32
- this.db = new DatabaseSync(this.dbPath, { timeout: 5000 });
33
- this.db.exec("pragma journal_mode = WAL;");
34
- this.db.exec("pragma foreign_keys = ON;");
35
- this.db.exec("pragma synchronous = NORMAL;");
36
- this.db.exec("pragma busy_timeout = 5000;");
37
- this._createTables();
38
- }
39
-
40
- _createTables() {
41
- this.db.exec(`
42
- create table if not exists context_events (
43
- id text primary key,
44
- session_id text not null,
45
- ts integer not null,
46
- kind text not null check (kind in (
47
- 'message_in','message_out','tool_call','tool_result','memory_write','state_patch','compaction'
48
- )),
49
- payload_json text not null,
50
- token_cost_in integer,
51
- token_cost_out integer,
52
- hash text not null,
53
- parent_event_id text,
54
- foreign key(parent_event_id) references context_events(id)
55
- );
56
-
57
- create index if not exists idx_context_events_session_ts
58
- on context_events(session_id, ts);
59
-
60
- create index if not exists idx_context_events_session_id
61
- on context_events(session_id, id);
62
-
63
- create unique index if not exists uq_context_events_session_hash
64
- on context_events(session_id, hash);
65
- `);
66
-
67
- this.db.exec(`
68
- create table if not exists context_snapshots (
69
- session_id text primary key,
70
- at_event_id text not null,
71
- ts integer not null,
72
- state_json text not null,
73
- summary_compact text,
74
- working_set_ids_json text not null,
75
- foreign key(at_event_id) references context_events(id)
76
- );
77
-
78
- create index if not exists idx_context_snapshots_ts
79
- on context_snapshots(ts);
80
- `);
81
-
82
- this.db.exec(`
83
- create table if not exists context_outbox (
84
- id text primary key,
85
- session_id text not null,
86
- ts integer not null,
87
- kind text not null,
88
- payload_json text not null,
89
- dedupe_key text not null,
90
- status text not null check (status in ('pending','processing','done','failed')),
91
- locked_at integer,
92
- lock_owner text,
93
- attempt_count integer not null default 0,
94
- last_error text,
95
- done_at integer
96
- );
97
-
98
- create unique index if not exists uq_context_outbox_dedupe
99
- on context_outbox(session_id, dedupe_key);
100
-
101
- create index if not exists idx_context_outbox_status_ts
102
- on context_outbox(status, ts);
103
- `);
104
-
105
- this.db.exec(`
106
- create table if not exists prompt_builds (
107
- id text primary key,
108
- session_id text not null,
109
- ts integer not null,
110
- token_budget integer not null,
111
- chosen_item_ids_json text not null,
112
- total_tokens_est integer not null,
113
- prefix_hash text not null,
114
- notes text
115
- );
116
-
117
- create index if not exists idx_prompt_builds_session_ts
118
- on prompt_builds(session_id, ts);
119
- `);
120
-
121
- this.db.exec(`
122
- create table if not exists context_summaries (
123
- id integer primary key autoincrement,
124
- session_id text not null,
125
- depth integer not null default 0,
126
- source_event_ids text not null default '[]',
127
- summary_text text not null,
128
- token_count integer not null default 0,
129
- created_at text not null default (datetime('now'))
130
- );
131
-
132
- create index if not exists idx_context_summaries_session_depth
133
- on context_summaries(session_id, depth);
134
- `);
135
- }
136
-
137
- // --- Reducer ---
138
-
139
- #reduceEvent(payload, currentState) {
140
- const p = typeof payload === "string" ? JSON.parse(payload) : payload;
141
- if (p && typeof p === "object" && !Array.isArray(p)) {
142
- return { ...currentState, ...p };
143
- }
144
- return currentState;
145
- }
146
-
147
- // --- Event log ---
148
-
149
- _computeHash(kind, payloadJson) {
150
- const payload = kind + "\n" + canonicalJson(payloadJson);
151
- return createHash("sha256").update(payload).digest("hex");
152
- }
153
-
154
- appendEvent({ session_id, kind, payload_json, ts, token_cost_in, token_cost_out, parent_event_id }) {
155
- if (!VALID_KINDS.has(kind)) {
156
- throw new Error(`Invalid event kind: ${kind}`);
157
- }
158
-
159
- const id = randomUUID();
160
- const eventTs = ts ?? Date.now();
161
- const hash = this._computeHash(kind, payload_json);
162
- const payloadStr = typeof payload_json === "string" ? payload_json : JSON.stringify(payload_json);
163
-
164
- // Dedup: if same session+hash exists, return existing event
165
- const existing = this.db.prepare(
166
- "select id, session_id, ts, kind, payload_json, token_cost_in, token_cost_out, hash, parent_event_id from context_events where session_id = ? and hash = ?"
167
- ).get(session_id, hash);
168
-
169
- if (existing) {
170
- return existing;
171
- }
172
-
173
- const { lastInsertRowid } = this.db.prepare(`
174
- insert into context_events (id, session_id, ts, kind, payload_json, token_cost_in, token_cost_out, hash, parent_event_id)
175
- values (?, ?, ?, ?, ?, ?, ?, ?, ?)
176
- `).run(id, session_id, eventTs, kind, payloadStr, token_cost_in ?? null, token_cost_out ?? null, hash, parent_event_id ?? null);
177
-
178
- // Bug B: maintain reducer-derived working state
179
- if (!this.#sessionStates.has(session_id)) {
180
- const snap = this.db.prepare(
181
- "select state_json from context_snapshots where session_id = ?"
182
- ).get(session_id);
183
- this.#sessionStates.set(session_id, snap ? JSON.parse(snap.state_json) : {});
184
- }
185
- const newState = this.#reduceEvent(payload_json, this.#sessionStates.get(session_id));
186
- this.#sessionStates.set(session_id, newState);
187
-
188
- // Bug C: lazy-init snapshot cadence from DB on first access
189
- if (!this.#lastSnapshotEventId.has(session_id)) {
190
- const snapRow = this.db.prepare(
191
- "select at_event_id from context_snapshots where session_id = ? order by ts desc limit 1"
192
- ).get(session_id);
193
- if (snapRow?.at_event_id) {
194
- const eventRow = this.db.prepare(
195
- "select rowid from context_events where id = ?"
196
- ).get(snapRow.at_event_id);
197
- this.#lastSnapshotEventId.set(session_id, eventRow?.rowid ?? null);
198
- } else {
199
- this.#lastSnapshotEventId.set(session_id, null);
200
- }
201
- }
202
-
203
- const lastRowid = this.#lastSnapshotEventId.get(session_id) ?? 0;
204
- const { cnt } = this.db.prepare(
205
- "select count(*) as cnt from context_events where session_id = ? and rowid > ?"
206
- ).get(session_id, lastRowid);
207
- if (cnt >= this.#snapshotEveryNEvents) {
208
- // Bug B: snapshot stores reducer-derived state, not raw payload
209
- this.writeSnapshot({ session_id, at_event_id: id, state_json: JSON.stringify(newState), summary_compact: null, working_set_ids_json: "[]" });
210
- this.#lastSnapshotEventId.set(session_id, lastInsertRowid);
211
- }
212
-
213
- return { id, session_id, ts: eventTs, kind, payload_json: payloadStr, token_cost_in: token_cost_in ?? null, token_cost_out: token_cost_out ?? null, hash, parent_event_id: parent_event_id ?? null };
214
- }
215
-
216
- getEvents(sessionId, opts = {}) {
217
- const { afterEventId } = opts;
218
-
219
- if (afterEventId) {
220
- // Bug A: use rowid as tiebreaker so same-ms events are never skipped
221
- const ref = this.db.prepare("select ts, rowid from context_events where id = ?").get(afterEventId);
222
- if (!ref) return [];
223
- return this.db.prepare(
224
- "select id, session_id, ts, kind, payload_json, token_cost_in, token_cost_out, hash, parent_event_id from context_events where session_id = ? and (ts > ? or (ts = ? and rowid > ?)) order by ts asc, rowid asc"
225
- ).all(sessionId, ref.ts, ref.ts, ref.rowid);
226
- }
227
-
228
- return this.db.prepare(
229
- "select id, session_id, ts, kind, payload_json, token_cost_in, token_cost_out, hash, parent_event_id from context_events where session_id = ? order by ts asc, rowid asc"
230
- ).all(sessionId);
231
- }
232
-
233
- // --- Summaries ---
234
-
235
- saveContextSummary({ session_id, depth, source_event_ids, summary_text, token_count }) {
236
- const idsStr = Array.isArray(source_event_ids) ? JSON.stringify(source_event_ids) : source_event_ids;
237
- return this.db.prepare(`
238
- insert into context_summaries (session_id, depth, source_event_ids, summary_text, token_count)
239
- values (?, ?, ?, ?, ?)
240
- `).run(session_id, depth, idsStr, summary_text, token_count);
241
- }
242
-
243
- getContextSummaries({ session_id, depth }) {
244
- if (depth !== undefined) {
245
- return this.db.prepare(
246
- "select id, session_id, depth, source_event_ids, summary_text, token_count, created_at from context_summaries where session_id = ? and depth = ? order by created_at desc"
247
- ).all(session_id, depth);
248
- }
249
- return this.db.prepare(
250
- "select id, session_id, depth, source_event_ids, summary_text, token_count, created_at from context_summaries where session_id = ? order by created_at desc"
251
- ).all(session_id);
252
- }
253
-
254
- deleteContextSummaries({ session_id, depth }) {
255
- this.db.prepare(
256
- "delete from context_summaries where session_id = ? and depth = ?"
257
- ).run(session_id, depth);
258
- }
259
-
260
- // --- Snapshots ---
261
-
262
- writeSnapshot({ session_id, at_event_id, state_json, summary_compact, working_set_ids_json, ts }) {
263
- const snapTs = ts ?? Date.now();
264
- const stateStr = typeof state_json === "string" ? state_json : JSON.stringify(state_json);
265
-
266
- this.db.prepare(`
267
- insert into context_snapshots (session_id, at_event_id, ts, state_json, summary_compact, working_set_ids_json)
268
- values (?, ?, ?, ?, ?, ?)
269
- on conflict(session_id) do update set
270
- at_event_id = excluded.at_event_id,
271
- ts = excluded.ts,
272
- state_json = excluded.state_json,
273
- summary_compact = excluded.summary_compact,
274
- working_set_ids_json = excluded.working_set_ids_json
275
- `).run(session_id, at_event_id, snapTs, stateStr, summary_compact ?? null, working_set_ids_json);
276
- }
277
-
278
- getLatestSnapshot(sessionId) {
279
- return this.db.prepare(
280
- "select session_id, at_event_id, ts, state_json, summary_compact, working_set_ids_json from context_snapshots where session_id = ?"
281
- ).get(sessionId) || null;
282
- }
283
-
284
- // --- Load state (snapshot + replay) ---
285
-
286
- loadState(sessionId) {
287
- const snapshot = this.getLatestSnapshot(sessionId);
288
- if (snapshot) {
289
- const events = this.getEvents(sessionId, { afterEventId: snapshot.at_event_id });
290
- return { snapshot, events };
291
- }
292
- return { snapshot: null, events: this.getEvents(sessionId) };
293
- }
294
-
295
- // --- Outbox ---
296
-
297
- enqueueOutbox({ session_id, kind, payload_json, dedupe_key }) {
298
- const id = randomUUID();
299
- const ts = Date.now();
300
- const payloadStr = typeof payload_json === "string" ? payload_json : JSON.stringify(payload_json);
301
-
302
- this.db.prepare(`
303
- insert or ignore into context_outbox (id, session_id, ts, kind, payload_json, dedupe_key, status, attempt_count)
304
- values (?, ?, ?, ?, ?, ?, 'pending', 0)
305
- `).run(id, session_id, ts, kind, payloadStr, dedupe_key);
306
- }
307
-
308
- claimOutboxBatch({ limit = 10, lockTtlMs = 2 * 60 * 1000 } = {}) {
309
- const now = Date.now();
310
- const staleThreshold = now - lockTtlMs;
311
-
312
- // Claim pending items + stale processing items (lock expired)
313
- this.db.exec("begin immediate");
314
- try {
315
- const rows = this.db.prepare(`
316
- select id, session_id, ts, kind, payload_json, dedupe_key, status, locked_at, lock_owner, attempt_count, last_error, done_at
317
- from context_outbox
318
- where status = 'pending'
319
- or (status = 'processing' and locked_at < ?)
320
- order by ts asc
321
- limit ?
322
- `).all(staleThreshold, limit);
323
-
324
- if (rows.length > 0) {
325
- const ids = rows.map((r) => r.id);
326
- const placeholders = ids.map(() => "?").join(", ");
327
- this.db.prepare(
328
- `update context_outbox set status = 'processing', locked_at = ? where id in (${placeholders})`
329
- ).run(now, ...ids);
330
- }
331
-
332
- this.db.exec("commit");
333
-
334
- return rows.map((r) => ({ ...r, status: "processing", locked_at: now }));
335
- } catch (err) {
336
- try { this.db.exec("rollback"); } catch {}
337
- throw err;
338
- }
339
- }
340
-
341
- markOutboxDone(id) {
342
- this.db.prepare(
343
- "update context_outbox set status = 'done', done_at = ? where id = ?"
344
- ).run(Date.now(), id);
345
- }
346
-
347
- markOutboxFailed(id, error) {
348
- this.db.prepare(
349
- "update context_outbox set status = 'pending', attempt_count = attempt_count + 1, last_error = ?, locked_at = null where id = ?"
350
- ).run(error, id);
351
- }
352
-
353
- // --- Lifecycle ---
354
-
355
- close() {
356
- if (!this.db) return;
357
- this.db.close();
358
- this.db = null;
359
- }
360
- }
1
+ /**
2
+ * Context Ledger — event-sourced session store (Phase 1).
3
+ *
4
+ * Append-only event log with sha256 dedup, snapshots, and outbox
5
+ * for exactly-once side effects. SQLite + WAL mode.
6
+ */
7
+
8
+ import { createHash, randomUUID } from "node:crypto";
9
+ import { DatabaseSync } from "node:sqlite";
10
+ import { canonicalJson } from "../utils/canonical-json.js";
11
+
12
+ const VALID_KINDS = new Set([
13
+ "message_in", "message_out", "tool_call", "tool_result",
14
+ "memory_write", "state_patch", "compaction",
15
+ ]);
16
+
17
+ export class ContextLedger {
18
+ #snapshotEveryNEvents;
19
+ #lastSnapshotEventId;
20
+ #sessionStates;
21
+
22
+ constructor({ dbPath, snapshotEveryNEvents = 10 }) {
23
+ this.dbPath = dbPath;
24
+ this.db = null;
25
+ this.#snapshotEveryNEvents = snapshotEveryNEvents;
26
+ this.#lastSnapshotEventId = new Map();
27
+ this.#sessionStates = new Map();
28
+ }
29
+
30
+ open() {
31
+ if (this.db) return;
32
+ this.db = new DatabaseSync(this.dbPath, { timeout: 5000 });
33
+ this.db.exec("pragma journal_mode = WAL;");
34
+ this.db.exec("pragma foreign_keys = ON;");
35
+ this.db.exec("pragma synchronous = NORMAL;");
36
+ this.db.exec("pragma busy_timeout = 5000;");
37
+ this._createTables();
38
+ }
39
+
40
+ _createTables() {
41
+ this.db.exec(`
42
+ create table if not exists context_events (
43
+ id text primary key,
44
+ session_id text not null,
45
+ ts integer not null,
46
+ kind text not null check (kind in (
47
+ 'message_in','message_out','tool_call','tool_result','memory_write','state_patch','compaction'
48
+ )),
49
+ payload_json text not null,
50
+ token_cost_in integer,
51
+ token_cost_out integer,
52
+ hash text not null,
53
+ parent_event_id text,
54
+ foreign key(parent_event_id) references context_events(id)
55
+ );
56
+
57
+ create index if not exists idx_context_events_session_ts
58
+ on context_events(session_id, ts);
59
+
60
+ create index if not exists idx_context_events_session_id
61
+ on context_events(session_id, id);
62
+
63
+ create unique index if not exists uq_context_events_session_hash
64
+ on context_events(session_id, hash);
65
+ `);
66
+
67
+ this.db.exec(`
68
+ create table if not exists context_snapshots (
69
+ session_id text primary key,
70
+ at_event_id text not null,
71
+ ts integer not null,
72
+ state_json text not null,
73
+ summary_compact text,
74
+ working_set_ids_json text not null,
75
+ foreign key(at_event_id) references context_events(id)
76
+ );
77
+
78
+ create index if not exists idx_context_snapshots_ts
79
+ on context_snapshots(ts);
80
+ `);
81
+
82
+ this.db.exec(`
83
+ create table if not exists context_outbox (
84
+ id text primary key,
85
+ session_id text not null,
86
+ ts integer not null,
87
+ kind text not null,
88
+ payload_json text not null,
89
+ dedupe_key text not null,
90
+ status text not null check (status in ('pending','processing','done','failed')),
91
+ locked_at integer,
92
+ lock_owner text,
93
+ attempt_count integer not null default 0,
94
+ last_error text,
95
+ done_at integer
96
+ );
97
+
98
+ create unique index if not exists uq_context_outbox_dedupe
99
+ on context_outbox(session_id, dedupe_key);
100
+
101
+ create index if not exists idx_context_outbox_status_ts
102
+ on context_outbox(status, ts);
103
+ `);
104
+
105
+ this.db.exec(`
106
+ create table if not exists prompt_builds (
107
+ id text primary key,
108
+ session_id text not null,
109
+ ts integer not null,
110
+ token_budget integer not null,
111
+ chosen_item_ids_json text not null,
112
+ total_tokens_est integer not null,
113
+ prefix_hash text not null,
114
+ notes text
115
+ );
116
+
117
+ create index if not exists idx_prompt_builds_session_ts
118
+ on prompt_builds(session_id, ts);
119
+ `);
120
+
121
+ this.db.exec(`
122
+ create table if not exists context_summaries (
123
+ id integer primary key autoincrement,
124
+ session_id text not null,
125
+ depth integer not null default 0,
126
+ source_event_ids text not null default '[]',
127
+ summary_text text not null,
128
+ token_count integer not null default 0,
129
+ created_at text not null default (datetime('now'))
130
+ );
131
+
132
+ create index if not exists idx_context_summaries_session_depth
133
+ on context_summaries(session_id, depth);
134
+ `);
135
+ }
136
+
137
+ // --- Reducer ---
138
+
139
+ #reduceEvent(payload, currentState) {
140
+ const p = typeof payload === "string" ? JSON.parse(payload) : payload;
141
+ if (p && typeof p === "object" && !Array.isArray(p)) {
142
+ return { ...currentState, ...p };
143
+ }
144
+ return currentState;
145
+ }
146
+
147
+ // --- Event log ---
148
+
149
+ _computeHash(kind, payloadJson) {
150
+ const payload = kind + "\n" + canonicalJson(payloadJson);
151
+ return createHash("sha256").update(payload).digest("hex");
152
+ }
153
+
154
+ appendEvent({ session_id, kind, payload_json, ts, token_cost_in, token_cost_out, parent_event_id }) {
155
+ if (!VALID_KINDS.has(kind)) {
156
+ throw new Error(`Invalid event kind: ${kind}`);
157
+ }
158
+
159
+ const id = randomUUID();
160
+ const eventTs = ts ?? Date.now();
161
+ const hash = this._computeHash(kind, payload_json);
162
+ const payloadStr = typeof payload_json === "string" ? payload_json : JSON.stringify(payload_json);
163
+
164
+ // Dedup: if same session+hash exists, return existing event
165
+ const existing = this.db.prepare(
166
+ "select id, session_id, ts, kind, payload_json, token_cost_in, token_cost_out, hash, parent_event_id from context_events where session_id = ? and hash = ?"
167
+ ).get(session_id, hash);
168
+
169
+ if (existing) {
170
+ return existing;
171
+ }
172
+
173
+ const { lastInsertRowid } = this.db.prepare(`
174
+ insert into context_events (id, session_id, ts, kind, payload_json, token_cost_in, token_cost_out, hash, parent_event_id)
175
+ values (?, ?, ?, ?, ?, ?, ?, ?, ?)
176
+ `).run(id, session_id, eventTs, kind, payloadStr, token_cost_in ?? null, token_cost_out ?? null, hash, parent_event_id ?? null);
177
+
178
+ // Bug B: maintain reducer-derived working state
179
+ if (!this.#sessionStates.has(session_id)) {
180
+ const snap = this.db.prepare(
181
+ "select state_json from context_snapshots where session_id = ?"
182
+ ).get(session_id);
183
+ this.#sessionStates.set(session_id, snap ? JSON.parse(snap.state_json) : {});
184
+ }
185
+ const newState = this.#reduceEvent(payload_json, this.#sessionStates.get(session_id));
186
+ this.#sessionStates.set(session_id, newState);
187
+
188
+ // Bug C: lazy-init snapshot cadence from DB on first access
189
+ if (!this.#lastSnapshotEventId.has(session_id)) {
190
+ const snapRow = this.db.prepare(
191
+ "select at_event_id from context_snapshots where session_id = ? order by ts desc limit 1"
192
+ ).get(session_id);
193
+ if (snapRow?.at_event_id) {
194
+ const eventRow = this.db.prepare(
195
+ "select rowid from context_events where id = ?"
196
+ ).get(snapRow.at_event_id);
197
+ this.#lastSnapshotEventId.set(session_id, eventRow?.rowid ?? null);
198
+ } else {
199
+ this.#lastSnapshotEventId.set(session_id, null);
200
+ }
201
+ }
202
+
203
+ const lastRowid = this.#lastSnapshotEventId.get(session_id) ?? 0;
204
+ const { cnt } = this.db.prepare(
205
+ "select count(*) as cnt from context_events where session_id = ? and rowid > ?"
206
+ ).get(session_id, lastRowid);
207
+ if (cnt >= this.#snapshotEveryNEvents) {
208
+ // Bug B: snapshot stores reducer-derived state, not raw payload
209
+ this.writeSnapshot({ session_id, at_event_id: id, state_json: JSON.stringify(newState), summary_compact: null, working_set_ids_json: "[]" });
210
+ this.#lastSnapshotEventId.set(session_id, lastInsertRowid);
211
+ }
212
+
213
+ return { id, session_id, ts: eventTs, kind, payload_json: payloadStr, token_cost_in: token_cost_in ?? null, token_cost_out: token_cost_out ?? null, hash, parent_event_id: parent_event_id ?? null };
214
+ }
215
+
216
+ getEvents(sessionId, opts = {}) {
217
+ const { afterEventId } = opts;
218
+
219
+ if (afterEventId) {
220
+ // Bug A: use rowid as tiebreaker so same-ms events are never skipped
221
+ const ref = this.db.prepare("select ts, rowid from context_events where id = ?").get(afterEventId);
222
+ if (!ref) return [];
223
+ return this.db.prepare(
224
+ "select id, session_id, ts, kind, payload_json, token_cost_in, token_cost_out, hash, parent_event_id from context_events where session_id = ? and (ts > ? or (ts = ? and rowid > ?)) order by ts asc, rowid asc"
225
+ ).all(sessionId, ref.ts, ref.ts, ref.rowid);
226
+ }
227
+
228
+ return this.db.prepare(
229
+ "select id, session_id, ts, kind, payload_json, token_cost_in, token_cost_out, hash, parent_event_id from context_events where session_id = ? order by ts asc, rowid asc"
230
+ ).all(sessionId);
231
+ }
232
+
233
+ // --- Summaries ---
234
+
235
+ saveContextSummary({ session_id, depth, source_event_ids, summary_text, token_count }) {
236
+ const idsStr = Array.isArray(source_event_ids) ? JSON.stringify(source_event_ids) : source_event_ids;
237
+ return this.db.prepare(`
238
+ insert into context_summaries (session_id, depth, source_event_ids, summary_text, token_count)
239
+ values (?, ?, ?, ?, ?)
240
+ `).run(session_id, depth, idsStr, summary_text, token_count);
241
+ }
242
+
243
+ getContextSummaries({ session_id, depth }) {
244
+ if (depth !== undefined) {
245
+ return this.db.prepare(
246
+ "select id, session_id, depth, source_event_ids, summary_text, token_count, created_at from context_summaries where session_id = ? and depth = ? order by created_at desc"
247
+ ).all(session_id, depth);
248
+ }
249
+ return this.db.prepare(
250
+ "select id, session_id, depth, source_event_ids, summary_text, token_count, created_at from context_summaries where session_id = ? order by created_at desc"
251
+ ).all(session_id);
252
+ }
253
+
254
+ deleteContextSummaries({ session_id, depth }) {
255
+ this.db.prepare(
256
+ "delete from context_summaries where session_id = ? and depth = ?"
257
+ ).run(session_id, depth);
258
+ }
259
+
260
+ // --- Snapshots ---
261
+
262
+ writeSnapshot({ session_id, at_event_id, state_json, summary_compact, working_set_ids_json, ts }) {
263
+ const snapTs = ts ?? Date.now();
264
+ const stateStr = typeof state_json === "string" ? state_json : JSON.stringify(state_json);
265
+
266
+ this.db.prepare(`
267
+ insert into context_snapshots (session_id, at_event_id, ts, state_json, summary_compact, working_set_ids_json)
268
+ values (?, ?, ?, ?, ?, ?)
269
+ on conflict(session_id) do update set
270
+ at_event_id = excluded.at_event_id,
271
+ ts = excluded.ts,
272
+ state_json = excluded.state_json,
273
+ summary_compact = excluded.summary_compact,
274
+ working_set_ids_json = excluded.working_set_ids_json
275
+ `).run(session_id, at_event_id, snapTs, stateStr, summary_compact ?? null, working_set_ids_json);
276
+ }
277
+
278
+ getLatestSnapshot(sessionId) {
279
+ return this.db.prepare(
280
+ "select session_id, at_event_id, ts, state_json, summary_compact, working_set_ids_json from context_snapshots where session_id = ?"
281
+ ).get(sessionId) || null;
282
+ }
283
+
284
+ // --- Load state (snapshot + replay) ---
285
+
286
+ loadState(sessionId) {
287
+ const snapshot = this.getLatestSnapshot(sessionId);
288
+ if (snapshot) {
289
+ const events = this.getEvents(sessionId, { afterEventId: snapshot.at_event_id });
290
+ return { snapshot, events };
291
+ }
292
+ return { snapshot: null, events: this.getEvents(sessionId) };
293
+ }
294
+
295
+ // --- Outbox ---
296
+
297
+ enqueueOutbox({ session_id, kind, payload_json, dedupe_key }) {
298
+ const id = randomUUID();
299
+ const ts = Date.now();
300
+ const payloadStr = typeof payload_json === "string" ? payload_json : JSON.stringify(payload_json);
301
+
302
+ this.db.prepare(`
303
+ insert or ignore into context_outbox (id, session_id, ts, kind, payload_json, dedupe_key, status, attempt_count)
304
+ values (?, ?, ?, ?, ?, ?, 'pending', 0)
305
+ `).run(id, session_id, ts, kind, payloadStr, dedupe_key);
306
+ }
307
+
308
+ claimOutboxBatch({ limit = 10, lockTtlMs = 2 * 60 * 1000 } = {}) {
309
+ const now = Date.now();
310
+ const staleThreshold = now - lockTtlMs;
311
+
312
+ // Claim pending items + stale processing items (lock expired)
313
+ this.db.exec("begin immediate");
314
+ try {
315
+ const rows = this.db.prepare(`
316
+ select id, session_id, ts, kind, payload_json, dedupe_key, status, locked_at, lock_owner, attempt_count, last_error, done_at
317
+ from context_outbox
318
+ where status = 'pending'
319
+ or (status = 'processing' and locked_at < ?)
320
+ order by ts asc
321
+ limit ?
322
+ `).all(staleThreshold, limit);
323
+
324
+ if (rows.length > 0) {
325
+ const ids = rows.map((r) => r.id);
326
+ const placeholders = ids.map(() => "?").join(", ");
327
+ this.db.prepare(
328
+ `update context_outbox set status = 'processing', locked_at = ? where id in (${placeholders})`
329
+ ).run(now, ...ids);
330
+ }
331
+
332
+ this.db.exec("commit");
333
+
334
+ return rows.map((r) => ({ ...r, status: "processing", locked_at: now }));
335
+ } catch (err) {
336
+ try { this.db.exec("rollback"); } catch {}
337
+ throw err;
338
+ }
339
+ }
340
+
341
+ markOutboxDone(id) {
342
+ this.db.prepare(
343
+ "update context_outbox set status = 'done', done_at = ? where id = ?"
344
+ ).run(Date.now(), id);
345
+ }
346
+
347
+ markOutboxFailed(id, error) {
348
+ this.db.prepare(
349
+ "update context_outbox set status = 'pending', attempt_count = attempt_count + 1, last_error = ?, locked_at = null where id = ?"
350
+ ).run(error, id);
351
+ }
352
+
353
+ // --- Lifecycle ---
354
+
355
+ close() {
356
+ if (!this.db) return;
357
+ this.db.close();
358
+ this.db = null;
359
+ }
360
+ }