@sigrun/memory-honcho 0.0.0-stage → 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +64 -0
- package/LICENSE +21 -0
- package/README.md +103 -2
- package/dist/capture.d.ts +17 -0
- package/dist/capture.js +61 -0
- package/dist/chains.d.ts +18 -0
- package/dist/chains.js +33 -0
- package/dist/config.d.ts +28 -0
- package/dist/config.js +43 -0
- package/dist/delivery.d.ts +58 -0
- package/dist/delivery.js +264 -0
- package/dist/fork-rules.d.ts +32 -0
- package/dist/fork-rules.js +42 -0
- package/dist/honcho.d.ts +65 -0
- package/dist/honcho.js +127 -0
- package/dist/index.d.ts +42 -0
- package/dist/index.js +345 -0
- package/dist/representations.d.ts +42 -0
- package/dist/representations.js +81 -0
- package/dist/status.d.ts +57 -0
- package/dist/status.js +89 -0
- package/dist/stores.d.ts +83 -0
- package/dist/stores.js +244 -0
- package/dist/sweep.d.ts +60 -0
- package/dist/sweep.js +198 -0
- package/dist/tools.d.ts +41 -0
- package/dist/tools.js +157 -0
- package/dist/types.d.ts +139 -0
- package/dist/types.js +10 -0
- package/package.json +56 -4
package/dist/stores.d.ts
ADDED
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The extension's filesystem stores, rooted at `host.stores.paths`:
|
|
3
|
+
*
|
|
4
|
+
* - `data/outbox/capture.jsonl` — the durable outbox (the only durable copy
|
|
5
|
+
* of not-yet-delivered conversation content, so it takes the backed-up
|
|
6
|
+
* bucket), one JSON record per line in append order.
|
|
7
|
+
* - `state/last-delivery.json` — the persistent newest-delivery marker the
|
|
8
|
+
* status route reads (the sweeper drops delivered records, so the marker
|
|
9
|
+
* is the only surviving source of `lastDeliveryAt` in steady state).
|
|
10
|
+
* - `state/ledger/<conversationId>.json` — per-conversation bookkeeping
|
|
11
|
+
* (session mapping, delivered-batch window, fork anchors).
|
|
12
|
+
* - `state/forks/pending/<childId>.json` — pending-fork records.
|
|
13
|
+
* - `cache/representations/<conversationId>.json` — the representation
|
|
14
|
+
* snapshots the prompt section renders.
|
|
15
|
+
*
|
|
16
|
+
* `withOutbox` is a single MODULE-LEVEL mutex serializing every outbox
|
|
17
|
+
* read-modify-write cycle across ALL conversations and ALL ExtensionStores
|
|
18
|
+
* instances (the spec's serialization requirement: the sweeper's rewrite
|
|
19
|
+
* must not lose a concurrent capture append). In-process only — a restart
|
|
20
|
+
* loses in-flight work exactly as the outbox posture already accepts.
|
|
21
|
+
*
|
|
22
|
+
* All writes are atomic (temp file + rename in the same directory). The
|
|
23
|
+
* `*Sync` reads use `readFileSync` because the prompt-section render is
|
|
24
|
+
* synchronous (the spec'd cache-read posture).
|
|
25
|
+
*/
|
|
26
|
+
import type { HonchoSettings } from "./config.js";
|
|
27
|
+
import type { LastDeliveryMarker, Ledger, OutboxRecord, PendingForkRecord, RepresentationSnapshot } from "./types.js";
|
|
28
|
+
/** The three store roots, exactly `host.stores.paths`' shape. */
|
|
29
|
+
export interface StorePaths {
|
|
30
|
+
readonly data: string;
|
|
31
|
+
readonly state: string;
|
|
32
|
+
readonly cache: string;
|
|
33
|
+
}
|
|
34
|
+
export declare class ExtensionStores {
|
|
35
|
+
#private;
|
|
36
|
+
constructor(paths: StorePaths, settings: HonchoSettings);
|
|
37
|
+
/**
|
|
38
|
+
* Run `work` holding the outbox mutex. The caller's promise settles with
|
|
39
|
+
* the work's own result (rejection included); the internal tail is
|
|
40
|
+
* self-catching so one failing caller cannot poison the queue.
|
|
41
|
+
* NOT re-entrant: do not call `withOutbox` from within `work`.
|
|
42
|
+
*/
|
|
43
|
+
withOutbox<T>(work: () => Promise<T>): Promise<T>;
|
|
44
|
+
readLedger(conversationId: number): Promise<Ledger | undefined>;
|
|
45
|
+
/** Every ledger on disk (status/observability reads; missing dir → []). */
|
|
46
|
+
readLedgers(): Promise<Ledger[]>;
|
|
47
|
+
readLedgerSync(conversationId: number): Ledger | undefined;
|
|
48
|
+
writeLedger(conversationId: number, ledger: Ledger): Promise<void>;
|
|
49
|
+
/**
|
|
50
|
+
* Read every outbox record in file order. For anything other than a
|
|
51
|
+
* display-only read, call inside `withOutbox` (see the module comment).
|
|
52
|
+
*/
|
|
53
|
+
readOutbox(): Promise<OutboxRecord[]>;
|
|
54
|
+
/** Append one record (one JSONL line). Call inside `withOutbox`. */
|
|
55
|
+
appendOutboxRecord(record: OutboxRecord): Promise<void>;
|
|
56
|
+
/** Replace the outbox (atomic). Call inside `withOutbox`. */
|
|
57
|
+
rewriteOutbox(records: OutboxRecord[]): Promise<void>;
|
|
58
|
+
readPendingFork(childId: number): Promise<PendingForkRecord | undefined>;
|
|
59
|
+
/**
|
|
60
|
+
* Every pending-fork record on disk, resolved ones included (they are
|
|
61
|
+
* kept stamped so a late duplicate establish never re-clones; callers
|
|
62
|
+
* wanting the still-pending count filter on `resolvedAt`).
|
|
63
|
+
*/
|
|
64
|
+
readPendingForks(): Promise<PendingForkRecord[]>;
|
|
65
|
+
writePendingFork(record: PendingForkRecord): Promise<void>;
|
|
66
|
+
/**
|
|
67
|
+
* Stamp `resolvedAt` on the record (kept on disk, so a late duplicate
|
|
68
|
+
* establish never re-clones). Returns the resolved record, or `undefined`
|
|
69
|
+
* when no pending fork exists for the child.
|
|
70
|
+
*/
|
|
71
|
+
resolvePendingFork(childId: number): Promise<PendingForkRecord | undefined>;
|
|
72
|
+
readRepresentationSync(conversationId: number): RepresentationSnapshot | undefined;
|
|
73
|
+
writeRepresentation(conversationId: number, snapshot: RepresentationSnapshot): Promise<void>;
|
|
74
|
+
/**
|
|
75
|
+
* `state/last-delivery.json` — the persistent "newest delivery" marker.
|
|
76
|
+
* The delivery engine writes it when it flips a record (inside the same
|
|
77
|
+
* outbox critical section), because the sweeper drops delivered records
|
|
78
|
+
* every 15 minutes — the status route's retained-records derivation dies
|
|
79
|
+
* with them.
|
|
80
|
+
*/
|
|
81
|
+
readLastDelivery(): Promise<LastDeliveryMarker | undefined>;
|
|
82
|
+
writeLastDelivery(marker: LastDeliveryMarker): Promise<void>;
|
|
83
|
+
}
|
package/dist/stores.js
ADDED
|
@@ -0,0 +1,244 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The extension's filesystem stores, rooted at `host.stores.paths`:
|
|
3
|
+
*
|
|
4
|
+
* - `data/outbox/capture.jsonl` — the durable outbox (the only durable copy
|
|
5
|
+
* of not-yet-delivered conversation content, so it takes the backed-up
|
|
6
|
+
* bucket), one JSON record per line in append order.
|
|
7
|
+
* - `state/last-delivery.json` — the persistent newest-delivery marker the
|
|
8
|
+
* status route reads (the sweeper drops delivered records, so the marker
|
|
9
|
+
* is the only surviving source of `lastDeliveryAt` in steady state).
|
|
10
|
+
* - `state/ledger/<conversationId>.json` — per-conversation bookkeeping
|
|
11
|
+
* (session mapping, delivered-batch window, fork anchors).
|
|
12
|
+
* - `state/forks/pending/<childId>.json` — pending-fork records.
|
|
13
|
+
* - `cache/representations/<conversationId>.json` — the representation
|
|
14
|
+
* snapshots the prompt section renders.
|
|
15
|
+
*
|
|
16
|
+
* `withOutbox` is a single MODULE-LEVEL mutex serializing every outbox
|
|
17
|
+
* read-modify-write cycle across ALL conversations and ALL ExtensionStores
|
|
18
|
+
* instances (the spec's serialization requirement: the sweeper's rewrite
|
|
19
|
+
* must not lose a concurrent capture append). In-process only — a restart
|
|
20
|
+
* loses in-flight work exactly as the outbox posture already accepts.
|
|
21
|
+
*
|
|
22
|
+
* All writes are atomic (temp file + rename in the same directory). The
|
|
23
|
+
* `*Sync` reads use `readFileSync` because the prompt-section render is
|
|
24
|
+
* synchronous (the spec'd cache-read posture).
|
|
25
|
+
*/
|
|
26
|
+
import { randomBytes } from "node:crypto";
|
|
27
|
+
import { readFileSync } from "node:fs";
|
|
28
|
+
import { mkdir, open, readdir, readFile, rename, writeFile } from "node:fs/promises";
|
|
29
|
+
import { basename, dirname, join } from "node:path";
|
|
30
|
+
/**
|
|
31
|
+
* The module-level outbox tail: every `withOutbox` caller queues behind the
|
|
32
|
+
* previous one, so a rewrite can never lose a concurrent append. The tail
|
|
33
|
+
* never rejects (a failing caller settles for ITS caller, the next queued
|
|
34
|
+
* work still runs).
|
|
35
|
+
*/
|
|
36
|
+
let outboxTail = Promise.resolve();
|
|
37
|
+
export class ExtensionStores {
|
|
38
|
+
#paths;
|
|
39
|
+
#anchorWindow;
|
|
40
|
+
#deliveredBatchWindow;
|
|
41
|
+
constructor(paths, settings) {
|
|
42
|
+
this.#paths = paths;
|
|
43
|
+
this.#anchorWindow = settings.anchorWindow;
|
|
44
|
+
this.#deliveredBatchWindow = settings.deliveredBatchWindow;
|
|
45
|
+
}
|
|
46
|
+
/**
|
|
47
|
+
* Run `work` holding the outbox mutex. The caller's promise settles with
|
|
48
|
+
* the work's own result (rejection included); the internal tail is
|
|
49
|
+
* self-catching so one failing caller cannot poison the queue.
|
|
50
|
+
* NOT re-entrant: do not call `withOutbox` from within `work`.
|
|
51
|
+
*/
|
|
52
|
+
async withOutbox(work) {
|
|
53
|
+
const previous = outboxTail;
|
|
54
|
+
let release;
|
|
55
|
+
outboxTail = new Promise((resolve) => {
|
|
56
|
+
release = resolve;
|
|
57
|
+
});
|
|
58
|
+
await previous;
|
|
59
|
+
try {
|
|
60
|
+
return await work();
|
|
61
|
+
}
|
|
62
|
+
finally {
|
|
63
|
+
release();
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
// -- ledger (state) -------------------------------------------------------
|
|
67
|
+
async readLedger(conversationId) {
|
|
68
|
+
return readJson(this.#ledgerPath(conversationId));
|
|
69
|
+
}
|
|
70
|
+
/** Every ledger on disk (status/observability reads; missing dir → []). */
|
|
71
|
+
async readLedgers() {
|
|
72
|
+
return readJsonDir(join(this.#paths.state, "ledger"));
|
|
73
|
+
}
|
|
74
|
+
readLedgerSync(conversationId) {
|
|
75
|
+
return readJsonSync(this.#ledgerPath(conversationId));
|
|
76
|
+
}
|
|
77
|
+
async writeLedger(conversationId, ledger) {
|
|
78
|
+
// Window bounds fold oldest-first on write (spec §ledger).
|
|
79
|
+
const bounded = {
|
|
80
|
+
...ledger,
|
|
81
|
+
deliveredBatchIds: ledger.deliveredBatchIds.slice(-this.#deliveredBatchWindow),
|
|
82
|
+
anchors: ledger.anchors.slice(-this.#anchorWindow),
|
|
83
|
+
};
|
|
84
|
+
await writeJsonAtomic(this.#ledgerPath(conversationId), bounded);
|
|
85
|
+
}
|
|
86
|
+
// -- outbox (data) --------------------------------------------------------
|
|
87
|
+
/**
|
|
88
|
+
* Read every outbox record in file order. For anything other than a
|
|
89
|
+
* display-only read, call inside `withOutbox` (see the module comment).
|
|
90
|
+
*/
|
|
91
|
+
async readOutbox() {
|
|
92
|
+
let raw;
|
|
93
|
+
try {
|
|
94
|
+
raw = await readFile(this.#outboxPath(), "utf8");
|
|
95
|
+
}
|
|
96
|
+
catch (error) {
|
|
97
|
+
if (isNotFound(error))
|
|
98
|
+
return [];
|
|
99
|
+
throw error;
|
|
100
|
+
}
|
|
101
|
+
return raw
|
|
102
|
+
.split("\n")
|
|
103
|
+
.filter((line) => line !== "")
|
|
104
|
+
.map((line) => JSON.parse(line));
|
|
105
|
+
}
|
|
106
|
+
/** Append one record (one JSONL line). Call inside `withOutbox`. */
|
|
107
|
+
async appendOutboxRecord(record) {
|
|
108
|
+
const path = this.#outboxPath();
|
|
109
|
+
await mkdir(dirname(path), { recursive: true });
|
|
110
|
+
const file = await open(path, "a");
|
|
111
|
+
try {
|
|
112
|
+
await file.appendFile(JSON.stringify(record) + "\n", "utf8");
|
|
113
|
+
}
|
|
114
|
+
finally {
|
|
115
|
+
await file.close();
|
|
116
|
+
}
|
|
117
|
+
}
|
|
118
|
+
/** Replace the outbox (atomic). Call inside `withOutbox`. */
|
|
119
|
+
async rewriteOutbox(records) {
|
|
120
|
+
await writeTextAtomic(this.#outboxPath(), records.map((record) => JSON.stringify(record)).join("\n") +
|
|
121
|
+
(records.length === 0 ? "" : "\n"));
|
|
122
|
+
}
|
|
123
|
+
// -- pending forks (state) ------------------------------------------------
|
|
124
|
+
async readPendingFork(childId) {
|
|
125
|
+
return readJson(this.#pendingForkPath(childId));
|
|
126
|
+
}
|
|
127
|
+
/**
|
|
128
|
+
* Every pending-fork record on disk, resolved ones included (they are
|
|
129
|
+
* kept stamped so a late duplicate establish never re-clones; callers
|
|
130
|
+
* wanting the still-pending count filter on `resolvedAt`).
|
|
131
|
+
*/
|
|
132
|
+
async readPendingForks() {
|
|
133
|
+
return readJsonDir(join(this.#paths.state, "forks", "pending"));
|
|
134
|
+
}
|
|
135
|
+
async writePendingFork(record) {
|
|
136
|
+
await writeJsonAtomic(this.#pendingForkPath(record.childId), record);
|
|
137
|
+
}
|
|
138
|
+
/**
|
|
139
|
+
* Stamp `resolvedAt` on the record (kept on disk, so a late duplicate
|
|
140
|
+
* establish never re-clones). Returns the resolved record, or `undefined`
|
|
141
|
+
* when no pending fork exists for the child.
|
|
142
|
+
*/
|
|
143
|
+
async resolvePendingFork(childId) {
|
|
144
|
+
const existing = await this.readPendingFork(childId);
|
|
145
|
+
if (existing === undefined)
|
|
146
|
+
return undefined;
|
|
147
|
+
const resolved = { ...existing, resolvedAt: Date.now() };
|
|
148
|
+
await this.writePendingFork(resolved);
|
|
149
|
+
return resolved;
|
|
150
|
+
}
|
|
151
|
+
// -- representations (cache) ----------------------------------------------
|
|
152
|
+
readRepresentationSync(conversationId) {
|
|
153
|
+
return readJsonSync(this.#representationPath(conversationId));
|
|
154
|
+
}
|
|
155
|
+
async writeRepresentation(conversationId, snapshot) {
|
|
156
|
+
await writeJsonAtomic(this.#representationPath(conversationId), snapshot);
|
|
157
|
+
}
|
|
158
|
+
// -- last-delivery marker (state) -------------------------------------------
|
|
159
|
+
/**
|
|
160
|
+
* `state/last-delivery.json` — the persistent "newest delivery" marker.
|
|
161
|
+
* The delivery engine writes it when it flips a record (inside the same
|
|
162
|
+
* outbox critical section), because the sweeper drops delivered records
|
|
163
|
+
* every 15 minutes — the status route's retained-records derivation dies
|
|
164
|
+
* with them.
|
|
165
|
+
*/
|
|
166
|
+
async readLastDelivery() {
|
|
167
|
+
return readJson(this.#lastDeliveryPath());
|
|
168
|
+
}
|
|
169
|
+
async writeLastDelivery(marker) {
|
|
170
|
+
await writeJsonAtomic(this.#lastDeliveryPath(), marker);
|
|
171
|
+
}
|
|
172
|
+
// -- paths ------------------------------------------------------------------
|
|
173
|
+
#ledgerPath(conversationId) {
|
|
174
|
+
return join(this.#paths.state, "ledger", `${conversationId}.json`);
|
|
175
|
+
}
|
|
176
|
+
#outboxPath() {
|
|
177
|
+
return join(this.#paths.data, "outbox", "capture.jsonl");
|
|
178
|
+
}
|
|
179
|
+
#pendingForkPath(childId) {
|
|
180
|
+
return join(this.#paths.state, "forks", "pending", `${childId}.json`);
|
|
181
|
+
}
|
|
182
|
+
#representationPath(conversationId) {
|
|
183
|
+
return join(this.#paths.cache, "representations", `${conversationId}.json`);
|
|
184
|
+
}
|
|
185
|
+
#lastDeliveryPath() {
|
|
186
|
+
return join(this.#paths.state, "last-delivery.json");
|
|
187
|
+
}
|
|
188
|
+
}
|
|
189
|
+
function isNotFound(error) {
|
|
190
|
+
return (typeof error === "object" &&
|
|
191
|
+
error !== null &&
|
|
192
|
+
"code" in error &&
|
|
193
|
+
error.code === "ENOENT");
|
|
194
|
+
}
|
|
195
|
+
async function readJson(path) {
|
|
196
|
+
try {
|
|
197
|
+
return JSON.parse(await readFile(path, "utf8"));
|
|
198
|
+
}
|
|
199
|
+
catch (error) {
|
|
200
|
+
if (isNotFound(error))
|
|
201
|
+
return undefined;
|
|
202
|
+
throw error;
|
|
203
|
+
}
|
|
204
|
+
}
|
|
205
|
+
/** Parse every `*.json` file in a directory; a missing directory is empty. */
|
|
206
|
+
async function readJsonDir(dir) {
|
|
207
|
+
let names;
|
|
208
|
+
try {
|
|
209
|
+
names = await readdir(dir);
|
|
210
|
+
}
|
|
211
|
+
catch (error) {
|
|
212
|
+
if (isNotFound(error))
|
|
213
|
+
return [];
|
|
214
|
+
throw error;
|
|
215
|
+
}
|
|
216
|
+
const values = [];
|
|
217
|
+
for (const name of names) {
|
|
218
|
+
if (!name.endsWith(".json"))
|
|
219
|
+
continue;
|
|
220
|
+
values.push(JSON.parse(await readFile(join(dir, name), "utf8")));
|
|
221
|
+
}
|
|
222
|
+
return values;
|
|
223
|
+
}
|
|
224
|
+
function readJsonSync(path) {
|
|
225
|
+
try {
|
|
226
|
+
return JSON.parse(readFileSync(path, "utf8"));
|
|
227
|
+
}
|
|
228
|
+
catch (error) {
|
|
229
|
+
if (isNotFound(error))
|
|
230
|
+
return undefined;
|
|
231
|
+
throw error;
|
|
232
|
+
}
|
|
233
|
+
}
|
|
234
|
+
/** Atomic JSON write: temp file in the same directory, then rename. */
|
|
235
|
+
async function writeJsonAtomic(path, value) {
|
|
236
|
+
await writeTextAtomic(path, JSON.stringify(value));
|
|
237
|
+
}
|
|
238
|
+
async function writeTextAtomic(path, content) {
|
|
239
|
+
const dir = dirname(path);
|
|
240
|
+
await mkdir(dir, { recursive: true });
|
|
241
|
+
const temp = join(dir, `.${basename(path)}.${process.pid}.${randomBytes(6).toString("hex")}.tmp`);
|
|
242
|
+
await writeFile(temp, content, "utf8");
|
|
243
|
+
await rename(temp, path);
|
|
244
|
+
}
|
package/dist/sweep.d.ts
ADDED
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The outbox sweeper task (`honcho-outbox-sweep`), fired by the
|
|
3
|
+
* `outbox-sweep` cron job every 15 minutes.
|
|
4
|
+
*
|
|
5
|
+
* Body: group the outbox by conversation in file order; ESTABLISH every
|
|
6
|
+
* planned conversation (unresolved pending-forks first — a cloned child's
|
|
7
|
+
* establishment must land before its own deliveries), so the spec's
|
|
8
|
+
* recovery lane works: a pending record whose conversation has no ledger
|
|
9
|
+
* and no in-memory session mark (the missing-key / Honcho-down-at-capture
|
|
10
|
+
* state) gets the get-or-create it needs, because `establish` is
|
|
11
|
+
* idempotent (one ledger read when the ledger exists, no network). Then
|
|
12
|
+
* `deliverPending` each conversation; then compact inside `withOutbox`
|
|
13
|
+
* (drop delivered records — including the skipped-but-lingering kind whose
|
|
14
|
+
* batchId already sits in the ledger's delivered window — and pending
|
|
15
|
+
* records older than `outboxRetentionDays`, warning per retention drop;
|
|
16
|
+
* rewrite). Idempotent under re-fire: a sweep with a fully-delivered
|
|
17
|
+
* outbox touches nothing but the terminal commit.
|
|
18
|
+
*
|
|
19
|
+
* Failure posture: per-conversation try/catch + `console.error` +
|
|
20
|
+
* continue. ONE poisoned conversation can never fault the task, block the
|
|
21
|
+
* rest of the file, or skip compaction — a phase rejection faults the
|
|
22
|
+
* task, and a faulted sweep leaves retention drops and later-conversation
|
|
23
|
+
* deliveries undone forever.
|
|
24
|
+
*
|
|
25
|
+
* Lock discipline: the engine calls happen OUTSIDE the outbox mutex
|
|
26
|
+
* (`deliverPending` owns its own `withOutbox`; the mutex is not
|
|
27
|
+
* re-entrant). The planning read runs inside `withOutbox` (a read-only
|
|
28
|
+
* hold; nothing here calls the engine inside it). The compaction pass
|
|
29
|
+
* re-reads inside its own `withOutbox`, so records appended between the
|
|
30
|
+
* planning read and the rewrite are re-examined, never clobbered.
|
|
31
|
+
* Settings are resolved FRESH per sweep (the stores freeze
|
|
32
|
+
* `anchorWindow`/`deliveredBatchWindow` at first use, but retention must
|
|
33
|
+
* follow live settings).
|
|
34
|
+
*
|
|
35
|
+
* The real pi-durable `TaskDefinition` shape: `{name, version, initial,
|
|
36
|
+
* phases, abort}` — there is NO `run` member. A phase must commit durable
|
|
37
|
+
* progress before returning (returning without a commit faults the task),
|
|
38
|
+
* so `phases.sweep` ends in the terminal-completed commit and `abort`
|
|
39
|
+
* commits terminal-failed.
|
|
40
|
+
*/
|
|
41
|
+
import type { HonchoSettings } from "./config.js";
|
|
42
|
+
import type { DeliveryEngine } from "./delivery.js";
|
|
43
|
+
import type { ExtensionStores } from "./stores.js";
|
|
44
|
+
/** The registered task kind; the cron job's action names it. */
|
|
45
|
+
export declare const SWEEP_TASK_NAME = "honcho-outbox-sweep";
|
|
46
|
+
/** The sweep's dependencies — the factory wires the real ones; tests inject fakes. */
|
|
47
|
+
export interface SweepTaskDeps {
|
|
48
|
+
readonly engine: () => DeliveryEngine | Promise<DeliveryEngine>;
|
|
49
|
+
readonly stores: () => ExtensionStores | Promise<ExtensionStores>;
|
|
50
|
+
readonly settings: () => HonchoSettings | Promise<HonchoSettings>;
|
|
51
|
+
}
|
|
52
|
+
/** The sweep body, shared by the task phase and directly unit-testable. */
|
|
53
|
+
export declare function runSweep(deps: SweepTaskDeps, signal?: AbortSignal): Promise<void>;
|
|
54
|
+
/** The sweep is one phase; the checkpoint never carries data. */
|
|
55
|
+
type SweepCheckpoint = {
|
|
56
|
+
phase: "sweep";
|
|
57
|
+
};
|
|
58
|
+
/** The task object the extension registers (`extension.tasks`). */
|
|
59
|
+
export declare function createSweepTask(deps: SweepTaskDeps): import("@earendil-works/pi-durable").Task<unknown, SweepCheckpoint, unknown, object>;
|
|
60
|
+
export {};
|
package/dist/sweep.js
ADDED
|
@@ -0,0 +1,198 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The outbox sweeper task (`honcho-outbox-sweep`), fired by the
|
|
3
|
+
* `outbox-sweep` cron job every 15 minutes.
|
|
4
|
+
*
|
|
5
|
+
* Body: group the outbox by conversation in file order; ESTABLISH every
|
|
6
|
+
* planned conversation (unresolved pending-forks first — a cloned child's
|
|
7
|
+
* establishment must land before its own deliveries), so the spec's
|
|
8
|
+
* recovery lane works: a pending record whose conversation has no ledger
|
|
9
|
+
* and no in-memory session mark (the missing-key / Honcho-down-at-capture
|
|
10
|
+
* state) gets the get-or-create it needs, because `establish` is
|
|
11
|
+
* idempotent (one ledger read when the ledger exists, no network). Then
|
|
12
|
+
* `deliverPending` each conversation; then compact inside `withOutbox`
|
|
13
|
+
* (drop delivered records — including the skipped-but-lingering kind whose
|
|
14
|
+
* batchId already sits in the ledger's delivered window — and pending
|
|
15
|
+
* records older than `outboxRetentionDays`, warning per retention drop;
|
|
16
|
+
* rewrite). Idempotent under re-fire: a sweep with a fully-delivered
|
|
17
|
+
* outbox touches nothing but the terminal commit.
|
|
18
|
+
*
|
|
19
|
+
* Failure posture: per-conversation try/catch + `console.error` +
|
|
20
|
+
* continue. ONE poisoned conversation can never fault the task, block the
|
|
21
|
+
* rest of the file, or skip compaction — a phase rejection faults the
|
|
22
|
+
* task, and a faulted sweep leaves retention drops and later-conversation
|
|
23
|
+
* deliveries undone forever.
|
|
24
|
+
*
|
|
25
|
+
* Lock discipline: the engine calls happen OUTSIDE the outbox mutex
|
|
26
|
+
* (`deliverPending` owns its own `withOutbox`; the mutex is not
|
|
27
|
+
* re-entrant). The planning read runs inside `withOutbox` (a read-only
|
|
28
|
+
* hold; nothing here calls the engine inside it). The compaction pass
|
|
29
|
+
* re-reads inside its own `withOutbox`, so records appended between the
|
|
30
|
+
* planning read and the rewrite are re-examined, never clobbered.
|
|
31
|
+
* Settings are resolved FRESH per sweep (the stores freeze
|
|
32
|
+
* `anchorWindow`/`deliveredBatchWindow` at first use, but retention must
|
|
33
|
+
* follow live settings).
|
|
34
|
+
*
|
|
35
|
+
* The real pi-durable `TaskDefinition` shape: `{name, version, initial,
|
|
36
|
+
* phases, abort}` — there is NO `run` member. A phase must commit durable
|
|
37
|
+
* progress before returning (returning without a commit faults the task),
|
|
38
|
+
* so `phases.sweep` ends in the terminal-completed commit and `abort`
|
|
39
|
+
* commits terminal-failed.
|
|
40
|
+
*/
|
|
41
|
+
import { defineTask } from "@earendil-works/pi-durable";
|
|
42
|
+
/** The registered task kind; the cron job's action names it. */
|
|
43
|
+
export const SWEEP_TASK_NAME = "honcho-outbox-sweep";
|
|
44
|
+
const DAY_MS = 86_400_000;
|
|
45
|
+
/**
|
|
46
|
+
* The planning read, INSIDE the outbox mutex (read-only; it never calls
|
|
47
|
+
* the engine, so the non-reentrancy rule is never tripped): which
|
|
48
|
+
* conversations have outbox records, and which of their pending-fork
|
|
49
|
+
* records are still unresolved.
|
|
50
|
+
*/
|
|
51
|
+
async function plan(stores) {
|
|
52
|
+
return stores.withOutbox(async () => {
|
|
53
|
+
const records = await stores.readOutbox();
|
|
54
|
+
const conversationIds = [];
|
|
55
|
+
const seen = new Set();
|
|
56
|
+
const ownerByConversation = new Map();
|
|
57
|
+
for (const record of records) {
|
|
58
|
+
if (!seen.has(record.conversationId)) {
|
|
59
|
+
seen.add(record.conversationId);
|
|
60
|
+
conversationIds.push(record.conversationId);
|
|
61
|
+
ownerByConversation.set(record.conversationId, record.owner);
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
const unresolvedForks = [];
|
|
65
|
+
for (const conversationId of conversationIds) {
|
|
66
|
+
const fork = await stores.readPendingFork(conversationId);
|
|
67
|
+
if (fork !== undefined && fork.resolvedAt === undefined) {
|
|
68
|
+
unresolvedForks.push(conversationId);
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
return { conversationIds, ownerByConversation, unresolvedForks };
|
|
72
|
+
});
|
|
73
|
+
}
|
|
74
|
+
/**
|
|
75
|
+
* Compact: drop delivered records (a re-read inside the mutex catches
|
|
76
|
+
* records the engine flipped after the planning read, and the ledger's
|
|
77
|
+
* delivered window catches the skipped-but-lingering kind — a pending
|
|
78
|
+
* record whose batchId is already delivered, dropped SILENTLY: the ledger
|
|
79
|
+
* already counts the batch, so this is housekeeping, not data loss), and
|
|
80
|
+
* pending records older than the retention cutoff (warned, per record).
|
|
81
|
+
* Records appended between the planning read and here are re-examined by
|
|
82
|
+
* the same rules. The rewrite is skipped when nothing was dropped. The
|
|
83
|
+
* counts are logged for observability.
|
|
84
|
+
*/
|
|
85
|
+
async function compact(stores, retentionDays) {
|
|
86
|
+
const cutoff = Date.now() - retentionDays * DAY_MS;
|
|
87
|
+
const counts = await stores.withOutbox(async () => {
|
|
88
|
+
const records = await stores.readOutbox();
|
|
89
|
+
let droppedDelivered = 0;
|
|
90
|
+
let droppedExpired = 0;
|
|
91
|
+
const deliveredWindows = new Map();
|
|
92
|
+
const kept = [];
|
|
93
|
+
for (const record of records) {
|
|
94
|
+
if (record.status === "delivered") {
|
|
95
|
+
droppedDelivered += 1;
|
|
96
|
+
continue;
|
|
97
|
+
}
|
|
98
|
+
let delivered = deliveredWindows.get(record.conversationId);
|
|
99
|
+
if (delivered === undefined) {
|
|
100
|
+
const ledger = await stores.readLedger(record.conversationId);
|
|
101
|
+
delivered = new Set(ledger?.deliveredBatchIds ?? []);
|
|
102
|
+
deliveredWindows.set(record.conversationId, delivered);
|
|
103
|
+
}
|
|
104
|
+
if (delivered.has(record.batchId)) {
|
|
105
|
+
// Skipped-but-lingering: the ledger already counts the batch as
|
|
106
|
+
// delivered but the record never flipped (the accepted crash
|
|
107
|
+
// window) — drop it silently so every later sweep stops
|
|
108
|
+
// revisiting it.
|
|
109
|
+
droppedDelivered += 1;
|
|
110
|
+
continue;
|
|
111
|
+
}
|
|
112
|
+
if (record.createdAt < cutoff) {
|
|
113
|
+
droppedExpired += 1;
|
|
114
|
+
console.warn(`memory-honcho: dropping over-retention outbox record ${record.batchId} (conversation ${record.conversationId}, created ${new Date(record.createdAt).toISOString()}, older than ${retentionDays}d retention)`);
|
|
115
|
+
continue;
|
|
116
|
+
}
|
|
117
|
+
kept.push(record);
|
|
118
|
+
}
|
|
119
|
+
if (droppedDelivered + droppedExpired > 0) {
|
|
120
|
+
await stores.rewriteOutbox(kept);
|
|
121
|
+
}
|
|
122
|
+
return { delivered: droppedDelivered, expired: droppedExpired };
|
|
123
|
+
});
|
|
124
|
+
if (counts.delivered + counts.expired > 0) {
|
|
125
|
+
console.info(`memory-honcho: outbox compaction dropped ${counts.delivered} delivered, ${counts.expired} expired`);
|
|
126
|
+
}
|
|
127
|
+
}
|
|
128
|
+
/** The sweep body, shared by the task phase and directly unit-testable. */
|
|
129
|
+
export async function runSweep(deps, signal) {
|
|
130
|
+
const stores = await deps.stores();
|
|
131
|
+
const settings = await deps.settings();
|
|
132
|
+
const engine = await deps.engine();
|
|
133
|
+
const { conversationIds, ownerByConversation, unresolvedForks } = await plan(stores);
|
|
134
|
+
// Establish EVERY planned conversation — idempotent (one ledger read
|
|
135
|
+
// when the ledger exists, no network), and for a ledger-less
|
|
136
|
+
// conversation the get-or-create the recovery lane needs. Unresolved
|
|
137
|
+
// forks go first (a clone decision reads the parent's ledger, which
|
|
138
|
+
// this very sweep may have just advanced). One failure logs and moves
|
|
139
|
+
// on: a poisoned conversation can never block the rest.
|
|
140
|
+
const establishOrder = [
|
|
141
|
+
...unresolvedForks,
|
|
142
|
+
...conversationIds.filter((id) => !unresolvedForks.includes(id)),
|
|
143
|
+
];
|
|
144
|
+
for (const conversationId of establishOrder) {
|
|
145
|
+
if (signal?.aborted === true)
|
|
146
|
+
break;
|
|
147
|
+
const owner = ownerByConversation.get(conversationId) ?? "user";
|
|
148
|
+
try {
|
|
149
|
+
await engine.establish(conversationId, owner);
|
|
150
|
+
}
|
|
151
|
+
catch (error) {
|
|
152
|
+
console.error(`memory-honcho: sweep establish failed for conversation ${conversationId}`, error);
|
|
153
|
+
}
|
|
154
|
+
}
|
|
155
|
+
for (const conversationId of conversationIds) {
|
|
156
|
+
if (signal?.aborted === true)
|
|
157
|
+
break;
|
|
158
|
+
const owner = ownerByConversation.get(conversationId) ?? "user";
|
|
159
|
+
try {
|
|
160
|
+
await engine.deliverPending(conversationId, owner);
|
|
161
|
+
}
|
|
162
|
+
catch (error) {
|
|
163
|
+
console.error(`memory-honcho: sweep delivery failed for conversation ${conversationId}`, error);
|
|
164
|
+
}
|
|
165
|
+
}
|
|
166
|
+
// Compaction runs even when conversations above failed: retention and
|
|
167
|
+
// delivered-record drops must not be hostage to a poisoned entry.
|
|
168
|
+
await compact(stores, settings.outboxRetentionDays);
|
|
169
|
+
}
|
|
170
|
+
/** The task object the extension registers (`extension.tasks`). */
|
|
171
|
+
export function createSweepTask(deps) {
|
|
172
|
+
return defineTask({
|
|
173
|
+
name: SWEEP_TASK_NAME,
|
|
174
|
+
version: 1,
|
|
175
|
+
initial: () => ({ phase: "sweep" }),
|
|
176
|
+
phases: {
|
|
177
|
+
sweep: async (_task, runtime, context) => {
|
|
178
|
+
await runSweep(deps, runtime.signal);
|
|
179
|
+
// REQUIRED: a phase returning without durable progress faults the
|
|
180
|
+
// task. The sweep's bookkeeping lives in the outbox files; the
|
|
181
|
+
// task's own terminal outcome is the durable progress.
|
|
182
|
+
await runtime.commit(() => ({
|
|
183
|
+
status: "terminal",
|
|
184
|
+
outcome: { status: "completed", result: undefined },
|
|
185
|
+
}), context);
|
|
186
|
+
},
|
|
187
|
+
},
|
|
188
|
+
abort: async (_task, runtime, context) => {
|
|
189
|
+
await runtime.commit(() => ({
|
|
190
|
+
status: "terminal",
|
|
191
|
+
outcome: {
|
|
192
|
+
status: "failed",
|
|
193
|
+
error: { message: "aborted" },
|
|
194
|
+
},
|
|
195
|
+
}), context);
|
|
196
|
+
},
|
|
197
|
+
});
|
|
198
|
+
}
|
package/dist/tools.d.ts
ADDED
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The three recall tools: `honcho_search`, `honcho_chat`, `honcho_remember`.
|
|
3
|
+
*
|
|
4
|
+
* All three:
|
|
5
|
+
* - resolve the calling conversation's owner from `host.conversations.meta`
|
|
6
|
+
* (memoized in-process per conversation; ownerless conversations → `"user"`);
|
|
7
|
+
* - await `engine.establish(conversationId, owner)` first (the spec's second
|
|
8
|
+
* clone trigger — a rebuilt client re-establishes from the ledger);
|
|
9
|
+
* - convert Honcho failures to `memory-honcho: Honcho unavailable (<message>)`.
|
|
10
|
+
*
|
|
11
|
+
* `honcho_remember` passes NO `sessionId` in v1: a remembered fact is a
|
|
12
|
+
* user-level conclusion, matching the cross-conversation recall ruling.
|
|
13
|
+
* The README documents the visibility caveat.
|
|
14
|
+
*/
|
|
15
|
+
import { type ToolRegistration } from "@earendil-works/pi-durable";
|
|
16
|
+
import type { ConversationId } from "@earendil-works/pi-durable";
|
|
17
|
+
import type { HonchoSettings } from "./config.js";
|
|
18
|
+
import type { DeliveryEngine } from "./delivery.js";
|
|
19
|
+
import type { HonchoClient } from "./honcho.js";
|
|
20
|
+
/** Structural slice of `host.conversations.meta` the tools read. */
|
|
21
|
+
interface MetaReader {
|
|
22
|
+
meta(id: ConversationId): Promise<{
|
|
23
|
+
owner?: string;
|
|
24
|
+
}>;
|
|
25
|
+
}
|
|
26
|
+
/**
|
|
27
|
+
* Test seam: the factory wires the real accessors; tests inject fakes.
|
|
28
|
+
* The accessors are functions (not the values themselves) because the
|
|
29
|
+
* engine/client are lazily constructed and rebuilt on settings/key change.
|
|
30
|
+
*/
|
|
31
|
+
export interface ToolsDeps {
|
|
32
|
+
readonly host: {
|
|
33
|
+
conversations: MetaReader;
|
|
34
|
+
};
|
|
35
|
+
readonly client: () => HonchoClient | undefined;
|
|
36
|
+
readonly engine: () => Promise<DeliveryEngine>;
|
|
37
|
+
readonly settings: () => HonchoSettings | undefined;
|
|
38
|
+
}
|
|
39
|
+
/** Build the three recall tools. */
|
|
40
|
+
export declare function createTools(deps: ToolsDeps): readonly ToolRegistration[];
|
|
41
|
+
export {};
|