@oharness/engine 0.5.0 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent/agent.d.ts +63 -0
- package/dist/agent/agent.js +257 -7
- package/dist/agent/agent.js.map +1 -1
- package/dist/agent/memory-backup.d.ts +38 -0
- package/dist/agent/memory-backup.js +126 -0
- package/dist/agent/memory-backup.js.map +1 -0
- package/dist/agent/memory-consolidate.d.ts +47 -0
- package/dist/agent/memory-consolidate.js +118 -0
- package/dist/agent/memory-consolidate.js.map +1 -0
- package/dist/agent/memory-errors.d.ts +73 -0
- package/dist/agent/memory-errors.js +74 -0
- package/dist/agent/memory-errors.js.map +1 -0
- package/dist/agent/memory-extract.d.ts +36 -0
- package/dist/agent/memory-extract.js +169 -0
- package/dist/agent/memory-extract.js.map +1 -0
- package/dist/agent/memory-limits.d.ts +115 -0
- package/dist/agent/memory-limits.js +109 -0
- package/dist/agent/memory-limits.js.map +1 -0
- package/dist/agent/memory-links.d.ts +21 -0
- package/dist/agent/memory-links.js +45 -0
- package/dist/agent/memory-links.js.map +1 -0
- package/dist/agent/memory-merge.d.ts +34 -0
- package/dist/agent/memory-merge.js +152 -0
- package/dist/agent/memory-merge.js.map +1 -0
- package/dist/agent/memory-provider.d.ts +73 -0
- package/dist/agent/memory-provider.js +2 -0
- package/dist/agent/memory-provider.js.map +1 -0
- package/dist/agent/memory-recall.d.ts +72 -0
- package/dist/agent/memory-recall.js +192 -0
- package/dist/agent/memory-recall.js.map +1 -0
- package/dist/agent/memory-redact.d.ts +15 -0
- package/dist/agent/memory-redact.js +48 -0
- package/dist/agent/memory-redact.js.map +1 -0
- package/dist/agent/memory-review.d.ts +47 -0
- package/dist/agent/memory-review.js +127 -0
- package/dist/agent/memory-review.js.map +1 -0
- package/dist/agent/memory-usage.d.ts +24 -0
- package/dist/agent/memory-usage.js +68 -0
- package/dist/agent/memory-usage.js.map +1 -0
- package/dist/agent/memory.d.ts +137 -14
- package/dist/agent/memory.js +431 -130
- package/dist/agent/memory.js.map +1 -1
- package/dist/agent/permissions.d.ts +10 -2
- package/dist/agent/permissions.js +22 -3
- package/dist/agent/permissions.js.map +1 -1
- package/dist/agent/session.d.ts +8 -1
- package/dist/agent/session.js +9 -0
- package/dist/agent/session.js.map +1 -1
- package/dist/agent/store.d.ts +3 -0
- package/dist/agent/store.js.map +1 -1
- package/dist/agent/titles.js +2 -2
- package/dist/config.d.ts +7 -1
- package/dist/config.js +27 -0
- package/dist/config.js.map +1 -1
- package/dist/core/audit.d.ts +19 -0
- package/dist/core/types.d.ts +6 -0
- package/dist/core/types.js +12 -0
- package/dist/core/types.js.map +1 -1
- package/dist/harness.d.ts +7 -1
- package/dist/harness.js +33 -2
- package/dist/harness.js.map +1 -1
- package/dist/providers/builtin.d.ts +20 -0
- package/dist/providers/builtin.js +82 -0
- package/dist/providers/builtin.js.map +1 -1
- package/dist/providers/catalog.js +30 -3
- package/dist/providers/catalog.js.map +1 -1
- package/dist/providers/index.d.ts +2 -2
- package/dist/providers/index.js +1 -1
- package/dist/providers/index.js.map +1 -1
- package/dist/providers/piai.js +17 -8
- package/dist/providers/piai.js.map +1 -1
- package/dist/providers/registry.d.ts +5 -0
- package/dist/providers/registry.js +10 -2
- package/dist/providers/registry.js.map +1 -1
- package/dist/providers/types.d.ts +31 -0
- package/dist/providers/types.js +36 -0
- package/dist/providers/types.js.map +1 -1
- package/dist/runtime-plugins/builtin/memory-coordinator.d.ts +79 -0
- package/dist/runtime-plugins/builtin/memory-coordinator.js +169 -0
- package/dist/runtime-plugins/builtin/memory-coordinator.js.map +1 -0
- package/dist/runtime-plugins/builtin/memory-migrate.d.ts +35 -0
- package/dist/runtime-plugins/builtin/memory-migrate.js +141 -0
- package/dist/runtime-plugins/builtin/memory-migrate.js.map +1 -0
- package/dist/runtime-plugins/builtin/memory.d.ts +144 -5
- package/dist/runtime-plugins/builtin/memory.js +1016 -112
- package/dist/runtime-plugins/builtin/memory.js.map +1 -1
- package/dist/runtime-plugins/builtin/studio.js +19 -4
- package/dist/runtime-plugins/builtin/studio.js.map +1 -1
- package/dist/runtime-plugins/cloud-connector.js +4 -1
- package/dist/runtime-plugins/cloud-connector.js.map +1 -1
- package/dist/runtime-plugins/extensions.d.ts +9 -0
- package/dist/runtime-plugins/extensions.js +15 -0
- package/dist/runtime-plugins/extensions.js.map +1 -1
- package/dist/runtime-plugins/host.js +24 -2
- package/dist/runtime-plugins/host.js.map +1 -1
- package/dist/runtime-plugins/types.d.ts +6 -0
- package/dist/tools/builtin/ask.d.ts +9 -0
- package/dist/tools/builtin/ask.js +71 -0
- package/dist/tools/builtin/ask.js.map +1 -0
- package/dist/tools/builtin/index.d.ts +1 -0
- package/dist/tools/builtin/index.js +3 -0
- package/dist/tools/builtin/index.js.map +1 -1
- package/dist/tools/builtin/memory.d.ts +5 -5
- package/dist/tools/builtin/memory.js +151 -40
- package/dist/tools/builtin/memory.js.map +1 -1
- package/dist/tools/index.d.ts +2 -0
- package/dist/tools/index.js +1 -0
- package/dist/tools/index.js.map +1 -1
- package/dist/tools/questions.d.ts +89 -0
- package/dist/tools/questions.js +97 -0
- package/dist/tools/questions.js.map +1 -0
- package/dist/tools/types.d.ts +7 -0
- package/dist/tools/types.js.map +1 -1
- package/package.json +2 -2
|
@@ -1,58 +1,126 @@
|
|
|
1
1
|
import path from "node:path";
|
|
2
|
+
import { existsSync } from "node:fs";
|
|
2
3
|
import { createHash } from "node:crypto";
|
|
3
|
-
import {
|
|
4
|
+
import { AUTO_KINDS, reviewNotes } from "../../agent/memory-review.js";
|
|
5
|
+
import { CONSOLIDATE_PROMPT, draftByHand, mergePrompt, parseMergeReply, planMerge } from "../../agent/memory-consolidate.js";
|
|
6
|
+
import { rm } from "node:fs/promises";
|
|
7
|
+
import { MemoryStore, loadScoped, memoryIndexSection, memoryNote, noteHash, recallFromStores, storeFor, validateNote, validMemoryId } from "../../agent/memory.js";
|
|
8
|
+
import { DEFAULT_MEMORY_POLICY, MEMORY_SAFETY, consolidationEnabled, isMemoryType } from "../../agent/memory-limits.js";
|
|
9
|
+
import { rankNotes } from "../../agent/memory-recall.js";
|
|
10
|
+
import { findSecrets } from "../../agent/memory-redact.js";
|
|
11
|
+
import { MemoryUsage } from "../../agent/memory-usage.js";
|
|
12
|
+
import { backupNote, listBackups, restoreBackup } from "../../agent/memory-backup.js";
|
|
13
|
+
import { mergeNotes } from "../../agent/memory-merge.js";
|
|
14
|
+
import { MemoryError, memoryFailure } from "../../agent/memory-errors.js";
|
|
15
|
+
import { userPath } from "../../paths.js";
|
|
16
|
+
import { shortId } from "../../util/id.js";
|
|
17
|
+
import { resolvePlaceholders } from "../../agent/memory-extract.js";
|
|
4
18
|
import { createMemoryTools } from "../../tools/builtin/memory.js";
|
|
5
19
|
import { readPluginFile, readUserFile, updateRuntimePluginConfig, writeUserFile } from "../../config-files.js";
|
|
6
20
|
import { projectId } from "../../storage/project.js";
|
|
7
21
|
import { resolveSyncUrl } from "../../sync/settings.js";
|
|
22
|
+
import { MemorySyncCoordinator } from "./memory-coordinator.js";
|
|
8
23
|
const digest = (x) => createHash("sha256").update(x).digest("hex");
|
|
9
24
|
const object = (x) => !!x && typeof x === "object" && !Array.isArray(x);
|
|
10
|
-
|
|
25
|
+
const HASH = /^[a-f0-9]{64}$/;
|
|
26
|
+
/** A refusal from the sync service: what to tell a person, and the server's own code. */
|
|
27
|
+
class SyncRequestError extends MemoryError {
|
|
28
|
+
status;
|
|
29
|
+
serverCode;
|
|
30
|
+
retryAfterMs;
|
|
31
|
+
constructor(reason, status, serverCode, retryAfterMs) {
|
|
32
|
+
super(reason.code, reason.params);
|
|
33
|
+
this.status = status;
|
|
34
|
+
if (serverCode)
|
|
35
|
+
this.serverCode = serverCode;
|
|
36
|
+
if (retryAfterMs !== undefined)
|
|
37
|
+
this.retryAfterMs = retryAfterMs;
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
/** Retry-After as seconds or an HTTP date, bounded to something a background loop should wait. */
|
|
41
|
+
function retryAfterMs(value) {
|
|
42
|
+
const clamp = (ms) => Math.min(10 * 60_000, Math.max(1_000, ms));
|
|
43
|
+
if (value) {
|
|
44
|
+
const seconds = Number(value);
|
|
45
|
+
if (Number.isFinite(seconds) && seconds >= 0)
|
|
46
|
+
return clamp(seconds * 1000);
|
|
47
|
+
const date = Date.parse(value);
|
|
48
|
+
if (!Number.isNaN(date))
|
|
49
|
+
return clamp(date - Date.now());
|
|
50
|
+
}
|
|
51
|
+
return 60_000;
|
|
52
|
+
}
|
|
53
|
+
function planLimitError(quota) {
|
|
54
|
+
return quota?.maxNotes && quota.maxNoteBytes
|
|
55
|
+
? new MemoryError("planLimit", { notes: quota.maxNotes, kb: Math.max(1, Math.round(quota.maxNoteBytes / 1024)) })
|
|
56
|
+
: new MemoryError("planLimitPlain");
|
|
57
|
+
}
|
|
58
|
+
/** One scope's local notes and explicit per-device/account sync consent. */
|
|
11
59
|
export class MemoryService {
|
|
12
60
|
store;
|
|
61
|
+
scope;
|
|
13
62
|
options;
|
|
14
63
|
target;
|
|
64
|
+
coordinator;
|
|
65
|
+
syncTarget;
|
|
66
|
+
detach;
|
|
15
67
|
controller;
|
|
16
|
-
timer;
|
|
17
68
|
running;
|
|
18
69
|
closed = false;
|
|
19
70
|
disabling = false;
|
|
20
71
|
conflicts = [];
|
|
21
72
|
error;
|
|
22
73
|
lastSync;
|
|
74
|
+
lastPull;
|
|
75
|
+
lastChanged = false;
|
|
23
76
|
binding;
|
|
24
77
|
settingsRevision = 0;
|
|
25
78
|
retryAt = 0;
|
|
26
79
|
snapshotCache;
|
|
80
|
+
quota;
|
|
81
|
+
/** The server predates `have`, so every snapshot carries every note. */
|
|
82
|
+
legacyServer = false;
|
|
27
83
|
constructor(options) {
|
|
28
84
|
this.options = options;
|
|
29
|
-
this.
|
|
85
|
+
this.scope = options.scope ?? "project";
|
|
86
|
+
const policy = options.policy ?? DEFAULT_MEMORY_POLICY;
|
|
87
|
+
this.store = new MemoryStore(options.directory, {
|
|
88
|
+
maxNotes: policy.maxNotes, maxNoteBytes: policy.maxNoteBytes, scope: this.scope,
|
|
89
|
+
...(options.onWarning ? { onWarning: options.onWarning } : {}),
|
|
90
|
+
});
|
|
30
91
|
this.target = digest(path.resolve(options.directory));
|
|
92
|
+
this.coordinator = options.coordinator ?? MemorySyncCoordinator.shared;
|
|
93
|
+
this.syncTarget = {
|
|
94
|
+
scope: this.scope,
|
|
95
|
+
eligible: () => this.eligible(),
|
|
96
|
+
syncOnce: async () => { await this.sync(); return { changed: this.lastChanged }; },
|
|
97
|
+
};
|
|
31
98
|
}
|
|
99
|
+
get isClosed() { return this.closed; }
|
|
32
100
|
consent() {
|
|
33
101
|
const plugin = readPluginFile()?.runtimePlugins?.find((p) => p.id === "memory");
|
|
34
102
|
if (plugin?.enabled === false)
|
|
35
103
|
return { userId: "", origin: "", enabled: false };
|
|
36
104
|
const config = plugin?.config;
|
|
37
105
|
if (config !== undefined && (!object(config) || config.syncTargets !== undefined && !object(config.syncTargets)))
|
|
38
|
-
throw new
|
|
106
|
+
throw new MemoryError("configInvalid");
|
|
39
107
|
const value = object(config) && object(config.syncTargets) ? config.syncTargets[this.target] : undefined;
|
|
40
108
|
if (value === undefined)
|
|
41
109
|
return undefined;
|
|
42
110
|
if (!object(value) || typeof value.enabled !== "boolean" || typeof value.userId !== "string" || typeof value.origin !== "string")
|
|
43
|
-
throw new
|
|
111
|
+
throw new MemoryError("configInvalid");
|
|
44
112
|
return value;
|
|
45
113
|
}
|
|
46
114
|
stateFile(identity) { return "memory-sync/" + digest(identity.origin + ":" + identity.userId + ":" + this.target) + ".json"; }
|
|
47
115
|
async allowed() {
|
|
48
116
|
if (this.closed || this.disabling)
|
|
49
|
-
throw new
|
|
117
|
+
throw new MemoryError("stopped");
|
|
50
118
|
const consent = this.consent();
|
|
51
119
|
if (consent?.enabled === false)
|
|
52
|
-
throw new
|
|
120
|
+
throw new MemoryError("disabledHere");
|
|
53
121
|
const identity = await this.options.identity();
|
|
54
122
|
if (!identity || consent && (identity.userId !== consent.userId || identity.origin !== consent.origin))
|
|
55
|
-
throw new
|
|
123
|
+
throw new MemoryError("accountChanged");
|
|
56
124
|
if (!consent) {
|
|
57
125
|
await updateRuntimePluginConfig("memory", (config) => {
|
|
58
126
|
const targets = object(config.syncTargets) ? config.syncTargets : {};
|
|
@@ -63,39 +131,71 @@ export class MemoryService {
|
|
|
63
131
|
});
|
|
64
132
|
const pinned = this.consent();
|
|
65
133
|
if (!pinned?.enabled || pinned.userId !== identity.userId || pinned.origin !== identity.origin)
|
|
66
|
-
throw new
|
|
134
|
+
throw new MemoryError("settingsChanged");
|
|
67
135
|
}
|
|
68
136
|
if (this.closed || this.disabling || this.consent()?.enabled === false)
|
|
69
|
-
throw new
|
|
137
|
+
throw new MemoryError("stopped");
|
|
70
138
|
return identity;
|
|
71
139
|
}
|
|
140
|
+
async eligible() {
|
|
141
|
+
// Conflicts do not stop background sync: they hold back only their own notes.
|
|
142
|
+
if (this.closed || this.disabling || Date.now() < this.retryAt)
|
|
143
|
+
return false;
|
|
144
|
+
try {
|
|
145
|
+
if (this.consent()?.enabled === false)
|
|
146
|
+
return false;
|
|
147
|
+
}
|
|
148
|
+
catch {
|
|
149
|
+
return false;
|
|
150
|
+
}
|
|
151
|
+
return !!(await this.options.identity());
|
|
152
|
+
}
|
|
153
|
+
/**
|
|
154
|
+
* Hand this directory to the coordinator. It is checked straight away, then
|
|
155
|
+
* as the coordinator decides; a local write brings the next check forward.
|
|
156
|
+
* Calling it again does nothing.
|
|
157
|
+
*/
|
|
72
158
|
start() {
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
159
|
+
if (this.detach || this.closed)
|
|
160
|
+
return;
|
|
161
|
+
const pollMs = Math.max(1, this.options.pollMs ?? 60_000);
|
|
162
|
+
const unregister = this.coordinator.add(this.syncTarget, {
|
|
163
|
+
baseMs: pollMs,
|
|
164
|
+
maxMs: Math.max(pollMs, this.options.maxPollMs ?? pollMs * 5),
|
|
165
|
+
debounceMs: this.options.debounceMs ?? Math.min(3_000, pollMs),
|
|
166
|
+
// A user's own notes matter in every project, so they are always kept current.
|
|
167
|
+
activeWindowMs: this.scope === "user" ? Infinity : this.options.activeWindowMs ?? 30 * 60_000,
|
|
168
|
+
});
|
|
169
|
+
const unsubscribe = this.store.onChange((change) => {
|
|
170
|
+
if (change.origin === "local")
|
|
171
|
+
this.coordinator.touch(this.syncTarget, "local-change");
|
|
172
|
+
});
|
|
173
|
+
this.detach = () => { unregister(); unsubscribe(); };
|
|
174
|
+
}
|
|
175
|
+
/** These notes are in use: check the cloud soon, if it has been a while. */
|
|
176
|
+
activity() {
|
|
177
|
+
if (this.detach)
|
|
178
|
+
this.coordinator.touch(this.syncTarget, "activity");
|
|
83
179
|
}
|
|
84
180
|
async status() {
|
|
85
181
|
const identity = await this.options.identity(), consent = this.consent();
|
|
86
182
|
const matching = !!identity && (!consent?.userId || identity.userId === consent.userId && identity.origin === consent.origin);
|
|
87
183
|
const persisted = identity && matching ? readUserFile(this.stateFile(identity)) : undefined;
|
|
88
|
-
let entries = [],
|
|
184
|
+
let entries = [], omitted = 0, localFailure;
|
|
89
185
|
try {
|
|
90
|
-
|
|
186
|
+
const snapshot = await this.store.snapshot();
|
|
187
|
+
entries = snapshot.entries.map((entry) => ({ ...memoryNote(entry), hash: noteHash(entry) }));
|
|
188
|
+
omitted = snapshot.omitted;
|
|
91
189
|
}
|
|
92
190
|
catch {
|
|
93
|
-
|
|
191
|
+
localFailure = memoryFailure(new MemoryError("localUnreadable"));
|
|
94
192
|
}
|
|
95
|
-
|
|
96
|
-
|
|
193
|
+
const lastPull = this.lastPull ?? persisted?.lastPull;
|
|
194
|
+
return { scope: this.scope, signedIn: !!identity, enabled: consent?.enabled !== false, ready: matching && consent?.enabled !== false,
|
|
195
|
+
syncing: !!this.running, lastSync: persisted?.lastSync ?? this.lastSync, error: (localFailure ?? this.error)?.message, failure: localFailure ?? this.error,
|
|
97
196
|
accountChanged: !!identity && !!consent?.userId && !matching, conflicts: this.conflicts,
|
|
98
|
-
location: this.store.location, entries }
|
|
197
|
+
location: this.store.location, entries, omitted, ...(this.quota ? { quota: this.quota } : {}),
|
|
198
|
+
...(lastPull ? { lastPull } : {}) };
|
|
99
199
|
}
|
|
100
200
|
async enable(enabled) {
|
|
101
201
|
const revision = ++this.settingsRevision;
|
|
@@ -117,18 +217,18 @@ export class MemoryService {
|
|
|
117
217
|
}
|
|
118
218
|
const identity = await this.options.identity();
|
|
119
219
|
if (!identity)
|
|
120
|
-
throw new
|
|
220
|
+
throw new MemoryError("signInFirst");
|
|
121
221
|
const previous = this.consent();
|
|
122
222
|
if (previous?.userId && (previous.userId !== identity.userId || previous.origin !== identity.origin)) {
|
|
123
|
-
throw new
|
|
223
|
+
throw new MemoryError("boundElsewhere");
|
|
124
224
|
}
|
|
125
225
|
// Explicit opt-in verifies identity remotely, but uploads no content yet.
|
|
126
226
|
const verified = await this.request("identity", {}, identity, new AbortController().signal, false);
|
|
127
227
|
if (verified.principal !== identity.userId)
|
|
128
|
-
throw new
|
|
228
|
+
throw new MemoryError("verifyFailed");
|
|
129
229
|
await updateRuntimePluginConfig("memory", (config) => {
|
|
130
230
|
if (this.closed || revision !== this.settingsRevision)
|
|
131
|
-
throw new
|
|
231
|
+
throw new MemoryError("settingsChanged");
|
|
132
232
|
return { ...config, syncTargets: {
|
|
133
233
|
...(object(config.syncTargets) ? config.syncTargets : {}), [this.target]: { ...identity, enabled: true },
|
|
134
234
|
} };
|
|
@@ -136,130 +236,521 @@ export class MemoryService {
|
|
|
136
236
|
this.disabling = false;
|
|
137
237
|
this.error = undefined;
|
|
138
238
|
this.retryAt = 0;
|
|
239
|
+
this.activity();
|
|
139
240
|
}
|
|
140
241
|
sync() {
|
|
141
242
|
if (this.running)
|
|
142
243
|
return this.running;
|
|
143
|
-
const job = this.perform().catch((error) => {
|
|
144
|
-
this.error = error
|
|
145
|
-
this.retryAt = Date.now() + 60000;
|
|
244
|
+
const job = this.perform().then((changed) => { this.lastChanged = changed; }).catch((error) => {
|
|
245
|
+
this.error = memoryFailure(error);
|
|
246
|
+
this.retryAt = Date.now() + (error instanceof SyncRequestError && error.retryAfterMs ? error.retryAfterMs : 60000);
|
|
146
247
|
throw error;
|
|
147
248
|
}).finally(() => { this.running = undefined; this.controller = undefined; });
|
|
148
249
|
this.running = job;
|
|
149
250
|
return job;
|
|
150
251
|
}
|
|
252
|
+
/**
|
|
253
|
+
* One reconciliation. Resolves with whether anything went up or came down.
|
|
254
|
+
*
|
|
255
|
+
* A note both sides changed since the last sync is merged field by field
|
|
256
|
+
* when the changes do not touch (memory-merge.ts), and held back as a
|
|
257
|
+
* conflict only when they do. A held note stops nothing else: every other
|
|
258
|
+
* note still goes up and comes down, and the held one keeps its baseline
|
|
259
|
+
* until someone chooses.
|
|
260
|
+
*/
|
|
151
261
|
async perform() {
|
|
152
262
|
const controller = this.controller = new AbortController();
|
|
153
263
|
const identity = await this.allowed();
|
|
154
264
|
controller.signal.throwIfAborted();
|
|
155
265
|
this.binding = this.stateFile(identity);
|
|
156
|
-
const
|
|
266
|
+
const baseDir = this.baseDir(identity);
|
|
267
|
+
const where = this.scope === "user" ? { scope: "user" } : { project: await projectId(this.options.cwd) };
|
|
157
268
|
const state = readUserFile(this.binding);
|
|
158
269
|
const base = state?.base ?? {};
|
|
159
270
|
const local = new Map((await this.store.load()).map((entry) => [entry.id, memoryNote(entry)]));
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
271
|
+
// What this machine already holds. The server leaves those bodies out of
|
|
272
|
+
// its reply, so a sync where little changed downloads little.
|
|
273
|
+
const have = {};
|
|
274
|
+
for (const [id, note] of local)
|
|
275
|
+
have[id] = noteHash(note);
|
|
276
|
+
let remote = await this.snapshot(where, have, identity, controller.signal, true);
|
|
277
|
+
// A cached manifest carries only the bodies the reply it came from carried.
|
|
278
|
+
// When one that is needed now is missing, ask again in full.
|
|
279
|
+
if (remote.entries.some((entry) => entry.hash !== null && !entry.note && entry.hash !== (have[entry.id] ?? null))) {
|
|
280
|
+
remote = await this.snapshot(where, have, identity, controller.signal, false);
|
|
165
281
|
}
|
|
166
|
-
this.validateSnapshot(remote);
|
|
167
|
-
this.snapshotCache = remote;
|
|
168
282
|
const cloud = new Map(remote.entries.map((entry) => [entry.id, entry]));
|
|
169
283
|
const changes = [], nextBase = { ...base };
|
|
170
|
-
|
|
284
|
+
const conflicts = [];
|
|
285
|
+
const held = new Set();
|
|
286
|
+
const pull = { at: Date.now(), created: [], updated: [], deleted: [], merged: [] };
|
|
287
|
+
const merged = pull.merged;
|
|
288
|
+
const ref = (note) => ({ id: note.id, summary: note.summary });
|
|
289
|
+
const maxBytes = Math.min(this.store.maxNoteBytes, this.quota?.maxNoteBytes ?? Number.MAX_SAFE_INTEGER);
|
|
171
290
|
for (const id of new Set([...Object.keys(base), ...local.keys(), ...cloud.keys()])) {
|
|
172
291
|
const l = local.get(id), r = cloud.get(id), lh = noteHash(l), rh = r?.hash ?? null, old = base[id] ?? null;
|
|
173
292
|
if (lh === rh) {
|
|
174
|
-
|
|
293
|
+
if (rh === null)
|
|
294
|
+
delete nextBase[id];
|
|
295
|
+
else
|
|
296
|
+
nextBase[id] = rh;
|
|
297
|
+
continue;
|
|
298
|
+
}
|
|
299
|
+
// One side changed: it goes up here, or comes down below.
|
|
300
|
+
if (lh === old || rh === old) {
|
|
301
|
+
if (lh !== old)
|
|
302
|
+
changes.push({ id, hash: lh, ...(l ? { note: l } : {}) });
|
|
175
303
|
continue;
|
|
176
304
|
}
|
|
177
|
-
|
|
178
|
-
|
|
305
|
+
// Both sides changed since the last sync.
|
|
306
|
+
const theirs = r?.note;
|
|
307
|
+
if (!l || !theirs) {
|
|
308
|
+
// A deletion against an edit keeps the edit. Deleting again is one
|
|
309
|
+
// click; an edit lost to a deletion is gone. A local edit goes up; a
|
|
310
|
+
// cloud edit comes down with the pull below.
|
|
311
|
+
if (l) {
|
|
312
|
+
changes.push({ id, hash: lh, note: l });
|
|
313
|
+
merged.push(ref(l));
|
|
314
|
+
}
|
|
315
|
+
continue;
|
|
179
316
|
}
|
|
180
|
-
|
|
181
|
-
|
|
317
|
+
const lastSynced = this.readBase(baseDir, id, old);
|
|
318
|
+
let outcome = mergeNotes(lastSynced, l, theirs);
|
|
319
|
+
if ("note" in outcome) {
|
|
320
|
+
// A merge the service would refuse to store is not a merge.
|
|
321
|
+
try {
|
|
322
|
+
validateNote(outcome.note, maxBytes);
|
|
323
|
+
}
|
|
324
|
+
catch {
|
|
325
|
+
outcome = { conflict: ["body"] };
|
|
326
|
+
}
|
|
327
|
+
}
|
|
328
|
+
if (!("note" in outcome)) {
|
|
329
|
+
conflicts.push({ id, local: l, cloud: theirs, base: lastSynced ?? null, localHash: lh, cloudHash: rh, fields: outcome.conflict });
|
|
330
|
+
held.add(id);
|
|
331
|
+
continue;
|
|
332
|
+
}
|
|
333
|
+
const note = outcome.note, mh = noteHash(note);
|
|
334
|
+
await this.allowed();
|
|
335
|
+
controller.signal.throwIfAborted();
|
|
336
|
+
if (mh !== lh) {
|
|
337
|
+
await backupNote(this.store, l, "auto-merge");
|
|
338
|
+
await this.store.apply(id, note, lh, "sync");
|
|
339
|
+
local.set(id, note);
|
|
340
|
+
have[id] = mh;
|
|
341
|
+
}
|
|
342
|
+
if (mh !== rh) {
|
|
343
|
+
await backupNote(this.store, theirs, "auto-merge");
|
|
344
|
+
changes.push({ id, hash: mh, note });
|
|
345
|
+
}
|
|
346
|
+
merged.push(ref(note));
|
|
182
347
|
}
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
348
|
+
for (const batch of this.batches(changes)) {
|
|
349
|
+
for (const change of batch) {
|
|
350
|
+
if (change.hash)
|
|
351
|
+
have[change.id] = change.hash;
|
|
352
|
+
else
|
|
353
|
+
delete have[change.id];
|
|
354
|
+
}
|
|
355
|
+
remote = await this.commit(where, remote.etag, batch, have, identity, controller.signal);
|
|
189
356
|
}
|
|
190
357
|
// No writes/merges after a disable or account switch observed during the request.
|
|
191
358
|
await this.allowed();
|
|
192
359
|
controller.signal.throwIfAborted();
|
|
360
|
+
const listed = new Set();
|
|
193
361
|
for (const r of remote.entries) {
|
|
194
|
-
|
|
362
|
+
listed.add(r.id);
|
|
363
|
+
if (held.has(r.id))
|
|
364
|
+
continue;
|
|
365
|
+
const current = local.get(r.id);
|
|
366
|
+
const expected = noteHash(current);
|
|
195
367
|
if (expected !== r.hash) {
|
|
368
|
+
if (r.hash !== null && !r.note)
|
|
369
|
+
throw new MemoryError("invalidVersion");
|
|
196
370
|
await this.allowed();
|
|
197
371
|
controller.signal.throwIfAborted();
|
|
198
|
-
|
|
372
|
+
// Kept before an automatic pull replaces or removes it. Nobody looked
|
|
373
|
+
// at the change, and the cloud's old revisions have no way back.
|
|
374
|
+
if (current)
|
|
375
|
+
await backupNote(this.store, current, r.note ? "sync-update" : "sync-delete");
|
|
376
|
+
await this.store.apply(r.id, r.note, expected, "sync");
|
|
377
|
+
if (r.note) {
|
|
378
|
+
(current ? pull.updated : pull.created).push(ref(r.note));
|
|
379
|
+
local.set(r.id, r.note);
|
|
380
|
+
}
|
|
381
|
+
else if (current) {
|
|
382
|
+
pull.deleted.push(ref(current));
|
|
383
|
+
local.delete(r.id);
|
|
384
|
+
}
|
|
385
|
+
}
|
|
386
|
+
if (r.hash === null)
|
|
387
|
+
delete nextBase[r.id];
|
|
388
|
+
else
|
|
389
|
+
nextBase[r.id] = r.hash;
|
|
390
|
+
}
|
|
391
|
+
// The server drops tombstones once they are old enough. A note this machine
|
|
392
|
+
// last saw in sync, unchanged here, that the manifest no longer lists at
|
|
393
|
+
// all was deleted elsewhere long enough ago for its tombstone to be gone.
|
|
394
|
+
for (const [id, old] of Object.entries(base)) {
|
|
395
|
+
if (listed.has(id) || held.has(id))
|
|
396
|
+
continue;
|
|
397
|
+
const current = local.get(id);
|
|
398
|
+
if (old !== null && current && noteHash(current) === old) {
|
|
399
|
+
await this.allowed();
|
|
400
|
+
controller.signal.throwIfAborted();
|
|
401
|
+
await backupNote(this.store, current, "sync-delete");
|
|
402
|
+
await this.store.apply(id, undefined, old, "sync");
|
|
403
|
+
pull.deleted.push(ref(current));
|
|
404
|
+
local.delete(id);
|
|
405
|
+
delete nextBase[id];
|
|
406
|
+
continue;
|
|
199
407
|
}
|
|
200
|
-
|
|
408
|
+
if (!local.has(id))
|
|
409
|
+
delete nextBase[id];
|
|
201
410
|
}
|
|
202
411
|
await this.allowed();
|
|
203
412
|
controller.signal.throwIfAborted();
|
|
413
|
+
// A baseline directory written before copies were kept is filled in full, once.
|
|
414
|
+
await this.writeBases(baseDir, state?.baseContent === true ? base : {}, nextBase, local, remote);
|
|
415
|
+
const pulled = pull.created.length + pull.updated.length + pull.deleted.length + merged.length;
|
|
416
|
+
if (pulled)
|
|
417
|
+
this.lastPull = pull;
|
|
418
|
+
this.conflicts = conflicts;
|
|
204
419
|
this.lastSync = Date.now();
|
|
205
|
-
this.error = undefined;
|
|
206
420
|
this.retryAt = 0;
|
|
207
|
-
|
|
421
|
+
this.error = conflicts.length ? memoryFailure(new MemoryError("conflictsPending", { count: conflicts.length })) : undefined;
|
|
422
|
+
const lastPull = this.lastPull ?? state?.lastPull;
|
|
423
|
+
await writeUserFile(this.binding, { base: nextBase, lastSync: this.lastSync, baseContent: true, ...(lastPull ? { lastPull } : {}) });
|
|
424
|
+
return changes.length > 0 || pulled > 0;
|
|
425
|
+
}
|
|
426
|
+
/** Where copies of notes at their baseline are kept, beside the baseline file. */
|
|
427
|
+
baseDir(identity) { return this.stateFile(identity).replace(/\.json$/, "-base"); }
|
|
428
|
+
/** The last synced version of a note, only when it is exactly the one the baseline records. */
|
|
429
|
+
readBase(dir, id, hash) {
|
|
430
|
+
if (hash === null || !validMemoryId(id))
|
|
431
|
+
return undefined;
|
|
432
|
+
try {
|
|
433
|
+
const note = readUserFile(`${dir}/${id}.json`);
|
|
434
|
+
if (!note)
|
|
435
|
+
return undefined;
|
|
436
|
+
validateNote(note, MEMORY_SAFETY.maxNoteBytesCeiling);
|
|
437
|
+
return noteHash(note) === hash ? note : undefined;
|
|
438
|
+
}
|
|
439
|
+
catch {
|
|
440
|
+
return undefined;
|
|
441
|
+
}
|
|
442
|
+
}
|
|
443
|
+
async writeBaseNote(dir, id, note) {
|
|
444
|
+
if (!validMemoryId(id))
|
|
445
|
+
return;
|
|
446
|
+
if (note)
|
|
447
|
+
await writeUserFile(`${dir}/${id}.json`, note);
|
|
448
|
+
else
|
|
449
|
+
await rm(userPath(`${dir}/${id}.json`), { force: true });
|
|
450
|
+
}
|
|
451
|
+
/** Keep the baseline copies in step with the baseline hashes, writing only what moved. */
|
|
452
|
+
async writeBases(dir, before, after, local, remote) {
|
|
453
|
+
const fromCloud = new Map();
|
|
454
|
+
for (const entry of remote.entries)
|
|
455
|
+
if (entry.note)
|
|
456
|
+
fromCloud.set(entry.id, entry.note);
|
|
457
|
+
for (const id of new Set([...Object.keys(before), ...Object.keys(after)])) {
|
|
458
|
+
const hash = after[id] ?? null;
|
|
459
|
+
if (hash === null) {
|
|
460
|
+
if ((before[id] ?? null) !== null)
|
|
461
|
+
await this.writeBaseNote(dir, id, undefined);
|
|
462
|
+
continue;
|
|
463
|
+
}
|
|
464
|
+
if (before[id] === hash)
|
|
465
|
+
continue;
|
|
466
|
+
// A copy that does not match its hash is worse than none, so an unknown one is removed.
|
|
467
|
+
await this.writeBaseNote(dir, id, [local.get(id), fromCloud.get(id)].find((note) => noteHash(note) === hash));
|
|
468
|
+
}
|
|
469
|
+
}
|
|
470
|
+
/**
|
|
471
|
+
* Write a batch cloud first, for changes made without anyone reviewing each
|
|
472
|
+
* note (consolidation).
|
|
473
|
+
*
|
|
474
|
+
* Every note in the batch must be fully in sync, the same here, in the cloud
|
|
475
|
+
* and in the baseline, or the batch is refused before anything is written.
|
|
476
|
+
* The commit carries the manifest version those checks read, so a change
|
|
477
|
+
* from another device in between is refused by the server rather than
|
|
478
|
+
* becoming a conflict. Only once the cloud has accepted the batch is it
|
|
479
|
+
* written here. It needs the network; there is no offline fallback.
|
|
480
|
+
*/
|
|
481
|
+
async commitFirst(edits, reason) {
|
|
482
|
+
if (!edits.length)
|
|
483
|
+
return;
|
|
484
|
+
await this.running?.catch(() => { });
|
|
485
|
+
if (this.running)
|
|
486
|
+
throw new MemoryError("waitForSync");
|
|
487
|
+
const job = (async () => {
|
|
488
|
+
const controller = this.controller = new AbortController();
|
|
489
|
+
const identity = await this.allowed();
|
|
490
|
+
this.binding = this.stateFile(identity);
|
|
491
|
+
const baseDir = this.baseDir(identity);
|
|
492
|
+
const where = this.scope === "user" ? { scope: "user" } : { project: await projectId(this.options.cwd) };
|
|
493
|
+
const base = readUserFile(this.binding)?.base ?? {};
|
|
494
|
+
const local = new Map((await this.store.load()).map((entry) => [entry.id, memoryNote(entry)]));
|
|
495
|
+
const have = {};
|
|
496
|
+
for (const [id, note] of local)
|
|
497
|
+
have[id] = noteHash(note);
|
|
498
|
+
const remote = await this.snapshot(where, have, identity, controller.signal, false);
|
|
499
|
+
const cloud = new Map(remote.entries.map((entry) => [entry.id, entry.hash]));
|
|
500
|
+
const maxBytes = Math.min(this.store.maxNoteBytes, this.quota?.maxNoteBytes ?? Number.MAX_SAFE_INTEGER);
|
|
501
|
+
const seen = new Set();
|
|
502
|
+
for (const edit of edits) {
|
|
503
|
+
if (!validMemoryId(edit.id) || seen.has(edit.id) || (edit.note && edit.note.id !== edit.id))
|
|
504
|
+
throw new MemoryError("invalidBatch");
|
|
505
|
+
seen.add(edit.id);
|
|
506
|
+
const lh = noteHash(local.get(edit.id));
|
|
507
|
+
if (lh !== edit.expectedHash || (cloud.get(edit.id) ?? null) !== lh || (base[edit.id] ?? null) !== lh)
|
|
508
|
+
throw new MemoryError("stale");
|
|
509
|
+
if (edit.note)
|
|
510
|
+
validateNote(edit.note, maxBytes);
|
|
511
|
+
}
|
|
512
|
+
const changes = edits.map((edit) => ({ id: edit.id, hash: noteHash(edit.note), ...(edit.note ? { note: edit.note } : {}) }));
|
|
513
|
+
if (this.batches(changes).length > 1)
|
|
514
|
+
throw new MemoryError("batchTooLarge");
|
|
515
|
+
for (const change of changes) {
|
|
516
|
+
if (change.hash)
|
|
517
|
+
have[change.id] = change.hash;
|
|
518
|
+
else
|
|
519
|
+
delete have[change.id];
|
|
520
|
+
}
|
|
521
|
+
try {
|
|
522
|
+
await this.commit(where, remote.etag, changes, have, identity, controller.signal);
|
|
523
|
+
}
|
|
524
|
+
catch (error) {
|
|
525
|
+
if (error instanceof SyncRequestError && error.status === 409)
|
|
526
|
+
throw new MemoryError("stale");
|
|
527
|
+
throw error;
|
|
528
|
+
}
|
|
529
|
+
await this.allowed();
|
|
530
|
+
controller.signal.throwIfAborted();
|
|
531
|
+
for (const edit of edits) {
|
|
532
|
+
const current = local.get(edit.id);
|
|
533
|
+
if (current)
|
|
534
|
+
await backupNote(this.store, current, reason);
|
|
535
|
+
await this.store.apply(edit.id, edit.note, edit.expectedHash, "sync");
|
|
536
|
+
await this.writeBaseNote(baseDir, edit.id, edit.note);
|
|
537
|
+
}
|
|
538
|
+
const latest = readUserFile(this.binding);
|
|
539
|
+
const nextBase = { ...latest?.base };
|
|
540
|
+
for (const edit of edits) {
|
|
541
|
+
const hash = noteHash(edit.note);
|
|
542
|
+
if (hash === null)
|
|
543
|
+
delete nextBase[edit.id];
|
|
544
|
+
else
|
|
545
|
+
nextBase[edit.id] = hash;
|
|
546
|
+
}
|
|
547
|
+
await writeUserFile(this.binding, { ...latest, base: nextBase });
|
|
548
|
+
})().finally(() => { this.running = undefined; this.controller = undefined; });
|
|
549
|
+
this.running = job;
|
|
550
|
+
return job;
|
|
551
|
+
}
|
|
552
|
+
/**
|
|
553
|
+
* Whether changes here should go through the cloud first: sync is on and
|
|
554
|
+
* bound to the account signed in now. Otherwise they are ordinary local
|
|
555
|
+
* edits that a later sync uploads.
|
|
556
|
+
*/
|
|
557
|
+
async syncReady() {
|
|
558
|
+
if (this.closed || this.disabling)
|
|
559
|
+
return false;
|
|
560
|
+
let consent;
|
|
561
|
+
try {
|
|
562
|
+
consent = this.consent();
|
|
563
|
+
}
|
|
564
|
+
catch {
|
|
565
|
+
return false;
|
|
566
|
+
}
|
|
567
|
+
if (!consent?.enabled || !consent.userId)
|
|
568
|
+
return false;
|
|
569
|
+
const identity = await this.options.identity();
|
|
570
|
+
return !!identity && identity.userId === consent.userId && identity.origin === consent.origin;
|
|
571
|
+
}
|
|
572
|
+
/** The plan the sync service last announced for this account, if any. */
|
|
573
|
+
get tier() { return this.quota?.tier; }
|
|
574
|
+
/** Largest note body this machine and the sync service will both accept. */
|
|
575
|
+
noteBytesLimit() {
|
|
576
|
+
return Math.min(this.store.maxNoteBytes, this.quota?.maxNoteBytes ?? Number.MAX_SAFE_INTEGER);
|
|
577
|
+
}
|
|
578
|
+
/** Changes grouped to fit the server's announced change quota and a request size its body limit accepts. */
|
|
579
|
+
batches(changes) {
|
|
580
|
+
const maxChanges = Math.max(1, this.quota?.maxChanges ?? MEMORY_SAFETY.syncCommitBatchChanges);
|
|
581
|
+
const batches = [];
|
|
582
|
+
let current = [], bytes = 0;
|
|
583
|
+
for (const change of changes) {
|
|
584
|
+
const size = Buffer.byteLength(JSON.stringify(change));
|
|
585
|
+
if (current.length && (current.length >= maxChanges || bytes + size > MEMORY_SAFETY.syncCommitBatchBytes)) {
|
|
586
|
+
batches.push(current);
|
|
587
|
+
current = [];
|
|
588
|
+
bytes = 0;
|
|
589
|
+
}
|
|
590
|
+
current.push(change);
|
|
591
|
+
bytes += size;
|
|
592
|
+
}
|
|
593
|
+
if (current.length)
|
|
594
|
+
batches.push(current);
|
|
595
|
+
return batches;
|
|
596
|
+
}
|
|
597
|
+
async snapshot(where, have, identity, signal, useCache) {
|
|
598
|
+
const knownEtag = useCache ? this.snapshotCache?.etag : undefined;
|
|
599
|
+
let remote = await this.call("snapshot", { ...where, ...(knownEtag ? { knownEtag } : {}) }, have, identity, signal);
|
|
600
|
+
if (remote.unchanged) {
|
|
601
|
+
if (!this.snapshotCache || remote.etag !== this.snapshotCache.etag)
|
|
602
|
+
throw new MemoryError("invalidVersion");
|
|
603
|
+
if (object(remote.quota))
|
|
604
|
+
this.readQuota(remote.quota);
|
|
605
|
+
remote = this.snapshotCache;
|
|
606
|
+
}
|
|
607
|
+
this.validateSnapshot(remote);
|
|
608
|
+
this.snapshotCache = remote;
|
|
609
|
+
return remote;
|
|
208
610
|
}
|
|
209
|
-
async
|
|
611
|
+
async commit(where, etag, changes, have, identity, signal) {
|
|
612
|
+
const remote = await this.call("commit", { ...where, etag, changes }, have, identity, signal);
|
|
613
|
+
this.validateSnapshot(remote);
|
|
614
|
+
this.snapshotCache = remote;
|
|
615
|
+
return remote;
|
|
616
|
+
}
|
|
617
|
+
/** A request carrying `have`, retried without it once against a server that predates the field. */
|
|
618
|
+
async call(action, input, have, identity, signal) {
|
|
619
|
+
if (!this.legacyServer) {
|
|
620
|
+
try {
|
|
621
|
+
return await this.request(action, { ...input, have }, identity, signal);
|
|
622
|
+
}
|
|
623
|
+
catch (error) {
|
|
624
|
+
if (!(error instanceof SyncRequestError) || error.serverCode !== "unknown_field")
|
|
625
|
+
throw error;
|
|
626
|
+
if (this.scope === "user")
|
|
627
|
+
throw new MemoryError("serverTooOld");
|
|
628
|
+
this.legacyServer = true;
|
|
629
|
+
}
|
|
630
|
+
}
|
|
631
|
+
return this.request(action, input, identity, signal);
|
|
632
|
+
}
|
|
633
|
+
/**
|
|
634
|
+
* Settle a conflict. Whatever is set aside is backed up first, whichever side
|
|
635
|
+
* it came from, and the baseline moves to the cloud's version so the next
|
|
636
|
+
* sync uploads the outcome.
|
|
637
|
+
*
|
|
638
|
+
* `local` keeps this machine's version. `cloud` takes the cloud's. `both`
|
|
639
|
+
* keeps this machine's version under the note's id and the cloud's as a new
|
|
640
|
+
* note beside it. `manual` saves `merged`, written by a person from all three.
|
|
641
|
+
*/
|
|
642
|
+
async resolve(id, choice, merged) {
|
|
210
643
|
if (this.running)
|
|
211
|
-
throw new
|
|
644
|
+
throw new MemoryError("waitForSync");
|
|
212
645
|
const identity = await this.allowed();
|
|
213
646
|
if (this.stateFile(identity) !== this.binding)
|
|
214
|
-
throw new
|
|
647
|
+
throw new MemoryError("accountSwitched");
|
|
215
648
|
const conflict = this.conflicts.find((entry) => entry.id === id);
|
|
216
|
-
if (!conflict || !["local", "cloud"].includes(choice))
|
|
217
|
-
throw new
|
|
649
|
+
if (!conflict || !["local", "cloud", "both", "manual"].includes(choice))
|
|
650
|
+
throw new MemoryError("invalidChoice");
|
|
218
651
|
const current = (await this.store.load()).find((entry) => entry.id === id);
|
|
219
652
|
if (noteHash(current) !== conflict.localHash)
|
|
220
|
-
throw new
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
653
|
+
throw new MemoryError("localChanged");
|
|
654
|
+
const mine = current ? memoryNote(current) : undefined;
|
|
655
|
+
const maxBytes = Math.min(this.store.maxNoteBytes, this.quota?.maxNoteBytes ?? Number.MAX_SAFE_INTEGER);
|
|
656
|
+
if (choice === "local") {
|
|
657
|
+
// The cloud side is usually what another device just wrote. Keeping this
|
|
658
|
+
// machine's version must not make that unrecoverable.
|
|
659
|
+
if (conflict.cloud)
|
|
660
|
+
await backupNote(this.store, conflict.cloud, "conflict-cloud");
|
|
661
|
+
}
|
|
662
|
+
else if (choice === "cloud") {
|
|
663
|
+
if (mine)
|
|
664
|
+
await backupNote(this.store, mine, "conflict");
|
|
665
|
+
await this.store.apply(id, conflict.cloud ?? undefined, conflict.localHash, "sync");
|
|
666
|
+
}
|
|
667
|
+
else if (choice === "both") {
|
|
668
|
+
if (!conflict.cloud || !mine)
|
|
669
|
+
throw new MemoryError("oneSideDeleted");
|
|
670
|
+
const copyId = `${id.slice(0, 120)}-${shortId(3)}`;
|
|
671
|
+
await this.store.apply(copyId, { ...conflict.cloud, id: copyId, updatedAt: Date.now() }, null, "local");
|
|
672
|
+
}
|
|
673
|
+
else {
|
|
674
|
+
const basis = mine ?? conflict.cloud;
|
|
675
|
+
if (!basis || !object(merged))
|
|
676
|
+
throw new MemoryError("mergedMissing");
|
|
677
|
+
const note = {
|
|
678
|
+
...basis, id,
|
|
679
|
+
summary: typeof merged.summary === "string" ? merged.summary.trim() : basis.summary,
|
|
680
|
+
body: typeof merged.body === "string" ? merged.body.trim() : basis.body,
|
|
681
|
+
tags: Array.isArray(merged.tags) ? merged.tags.filter((tag) => typeof tag === "string") : basis.tags,
|
|
682
|
+
createdAt: basis.createdAt, updatedAt: Date.now(),
|
|
683
|
+
};
|
|
684
|
+
if (isMemoryType(merged.type))
|
|
685
|
+
note.type = merged.type;
|
|
686
|
+
else if (merged.type === "" || merged.type === null)
|
|
687
|
+
delete note.type;
|
|
688
|
+
if (findSecrets(`${note.summary}\n${note.body}`).length)
|
|
689
|
+
throw new MemoryError("secret");
|
|
690
|
+
validateNote(note, maxBytes);
|
|
691
|
+
if (mine)
|
|
692
|
+
await backupNote(this.store, mine, "conflict");
|
|
693
|
+
if (conflict.cloud)
|
|
694
|
+
await backupNote(this.store, conflict.cloud, "conflict-cloud");
|
|
695
|
+
await this.store.apply(id, note, conflict.localHash, "local");
|
|
226
696
|
}
|
|
227
697
|
const file = this.stateFile(identity), state = readUserFile(file);
|
|
228
|
-
await writeUserFile(file, { base: { ...state?.base, [id]: conflict.cloudHash } });
|
|
698
|
+
await writeUserFile(file, { ...state, base: { ...state?.base, [id]: conflict.cloudHash } });
|
|
699
|
+
await this.writeBaseNote(this.baseDir(identity), id, conflict.cloud ?? undefined);
|
|
229
700
|
this.conflicts = this.conflicts.filter((entry) => entry.id !== id);
|
|
230
|
-
this.error = this.conflicts.length ? "
|
|
701
|
+
this.error = this.conflicts.length ? memoryFailure(new MemoryError("conflictsRemain")) : undefined;
|
|
702
|
+
// What was chosen, if not the cloud's version, is a change the cloud has not seen.
|
|
703
|
+
if (this.detach)
|
|
704
|
+
this.coordinator.touch(this.syncTarget, "local-change");
|
|
705
|
+
}
|
|
706
|
+
readQuota(raw) {
|
|
707
|
+
const quota = {};
|
|
708
|
+
for (const key of ["maxNotes", "maxChanges", "maxNoteBytes", "tombstoneDays"]) {
|
|
709
|
+
const n = raw[key];
|
|
710
|
+
if (typeof n === "number" && Number.isSafeInteger(n) && n > 0)
|
|
711
|
+
quota[key] = n;
|
|
712
|
+
}
|
|
713
|
+
if (raw.tier === "free" || raw.tier === "paid")
|
|
714
|
+
quota.tier = raw.tier;
|
|
715
|
+
this.quota = quota;
|
|
231
716
|
}
|
|
232
717
|
validateSnapshot(value) {
|
|
233
|
-
if (!object(value) || !Array.isArray(value.entries) || value.entries.length >
|
|
234
|
-
throw new
|
|
718
|
+
if (!object(value) || !Array.isArray(value.entries) || value.entries.length > MEMORY_SAFETY.maxNotesCeiling * 4 || !(value.etag === null || typeof value.etag === "string"))
|
|
719
|
+
throw new MemoryError("invalidResponse");
|
|
235
720
|
const seen = new Set();
|
|
236
721
|
for (const entry of value.entries) {
|
|
237
722
|
if (!object(entry) || !validMemoryId(entry.id) || seen.has(entry.id))
|
|
238
|
-
throw new
|
|
723
|
+
throw new MemoryError("invalidResponse");
|
|
239
724
|
seen.add(entry.id);
|
|
725
|
+
if (!(entry.hash === null || typeof entry.hash === "string" && HASH.test(entry.hash)))
|
|
726
|
+
throw new MemoryError("invalidResponse");
|
|
240
727
|
if (entry.note) {
|
|
241
|
-
validateNote(entry.note);
|
|
728
|
+
validateNote(entry.note, MEMORY_SAFETY.maxNoteBytesCeiling);
|
|
242
729
|
if (entry.note.id !== entry.id)
|
|
243
|
-
throw new
|
|
730
|
+
throw new MemoryError("invalidResponse");
|
|
731
|
+
if (noteHash(entry.note) !== entry.hash)
|
|
732
|
+
throw new MemoryError("invalidResponse");
|
|
244
733
|
}
|
|
245
|
-
|
|
246
|
-
throw new Error("记忆校验失败。");
|
|
734
|
+
// A live entry without a body is one this machine said it already has.
|
|
247
735
|
}
|
|
736
|
+
if (object(value.quota))
|
|
737
|
+
this.readQuota(value.quota);
|
|
248
738
|
}
|
|
249
739
|
async request(action, input, identity, signal, gated = true) {
|
|
250
740
|
if (gated)
|
|
251
741
|
await this.allowed();
|
|
252
742
|
const current = await this.options.identity();
|
|
253
743
|
if (!current || current.userId !== identity.userId || current.origin !== identity.origin)
|
|
254
|
-
throw new
|
|
744
|
+
throw new MemoryError("accountSwitched");
|
|
255
745
|
const token = await this.options.token();
|
|
256
746
|
if (!token)
|
|
257
|
-
throw new
|
|
747
|
+
throw new MemoryError("signInAgain");
|
|
258
748
|
if (gated)
|
|
259
749
|
await this.allowed();
|
|
260
750
|
signal.throwIfAborted();
|
|
261
751
|
let response;
|
|
262
752
|
const endpoint = resolveSyncUrl({ url: this.options.syncUrl?.() ?? resolveSyncUrl() });
|
|
753
|
+
this.coordinator.noteRequest();
|
|
263
754
|
try {
|
|
264
755
|
response = await (this.options.fetch ?? fetch)(endpoint + "/v1/memory/" + action, {
|
|
265
756
|
method: "POST", redirect: "error", headers: { "content-type": "application/json", authorization: "Bearer " + token },
|
|
@@ -268,15 +759,38 @@ export class MemoryService {
|
|
|
268
759
|
});
|
|
269
760
|
}
|
|
270
761
|
catch {
|
|
271
|
-
throw new
|
|
762
|
+
throw new MemoryError("requestFailed");
|
|
272
763
|
}
|
|
273
764
|
if (!response.ok) {
|
|
274
|
-
|
|
275
|
-
|
|
765
|
+
// The error body is a short code chosen by the server, read to tell an
|
|
766
|
+
// old server rejecting a new field, or a plan limit, from other refusals.
|
|
767
|
+
let code;
|
|
768
|
+
try {
|
|
769
|
+
const text = await response.text();
|
|
770
|
+
if (text.length < 4096) {
|
|
771
|
+
const parsed = JSON.parse(text);
|
|
772
|
+
if (typeof parsed?.code === "string")
|
|
773
|
+
code = parsed.code;
|
|
774
|
+
}
|
|
775
|
+
}
|
|
776
|
+
catch { /* not JSON */ }
|
|
777
|
+
let wait;
|
|
778
|
+
if (response.status === 429) {
|
|
779
|
+
wait = retryAfterMs(response.headers.get("retry-after"));
|
|
780
|
+
// The limit is the account's, so every directory waits, not only this one.
|
|
781
|
+
this.coordinator.pause(Date.now() + wait);
|
|
782
|
+
}
|
|
783
|
+
const reason = response.status === 409 ? new MemoryError("cloudChanged")
|
|
784
|
+
: response.status === 429 ? new MemoryError("rateLimited")
|
|
785
|
+
: code === "memory_plan_limit" ? planLimitError(this.quota)
|
|
786
|
+
: response.status === 503 ? new MemoryError("serverNotConfigured")
|
|
787
|
+
: response.status === 413 ? new MemoryError("quotaExceeded")
|
|
788
|
+
: new MemoryError("rejected");
|
|
789
|
+
throw new SyncRequestError(reason, response.status, code, wait);
|
|
276
790
|
}
|
|
277
791
|
const reader = response.body?.getReader();
|
|
278
792
|
if (!reader)
|
|
279
|
-
throw new
|
|
793
|
+
throw new MemoryError("invalidResponse");
|
|
280
794
|
const chunks = [];
|
|
281
795
|
let length = 0;
|
|
282
796
|
try {
|
|
@@ -285,9 +799,9 @@ export class MemoryService {
|
|
|
285
799
|
if (done)
|
|
286
800
|
break;
|
|
287
801
|
length += value.length;
|
|
288
|
-
if (length >
|
|
802
|
+
if (length > MEMORY_SAFETY.syncResponseBytes) {
|
|
289
803
|
await reader.cancel();
|
|
290
|
-
throw new
|
|
804
|
+
throw new MemoryError("responseTooLarge");
|
|
291
805
|
}
|
|
292
806
|
chunks.push(value);
|
|
293
807
|
}
|
|
@@ -297,63 +811,453 @@ export class MemoryService {
|
|
|
297
811
|
}
|
|
298
812
|
const result = JSON.parse(Buffer.concat(chunks).toString("utf8"));
|
|
299
813
|
if (result.principal !== identity.userId)
|
|
300
|
-
throw new
|
|
814
|
+
throw new MemoryError("principalMismatch");
|
|
301
815
|
return result;
|
|
302
816
|
}
|
|
303
|
-
async close() {
|
|
817
|
+
async close() {
|
|
818
|
+
this.closed = true;
|
|
819
|
+
this.detach?.();
|
|
820
|
+
this.detach = undefined;
|
|
821
|
+
this.controller?.abort();
|
|
822
|
+
await this.running?.catch(() => { });
|
|
823
|
+
}
|
|
824
|
+
}
|
|
825
|
+
/**
|
|
826
|
+
* One service per notes directory in a process.
|
|
827
|
+
*
|
|
828
|
+
* A local web server builds a harness per project, and every one of them has
|
|
829
|
+
* the same user-level notes. Separate services for one directory would each
|
|
830
|
+
* reconcile against the same baseline file and the same cloud manifest, and
|
|
831
|
+
* trip over each other's commits. The first harness to ask supplies the
|
|
832
|
+
* options; the service stops when the last one lets go.
|
|
833
|
+
*/
|
|
834
|
+
const sharedServices = new Map();
|
|
835
|
+
export function acquireMemoryService(options) {
|
|
836
|
+
const key = `${options.scope ?? "project"}${path.resolve(options.directory)}`;
|
|
837
|
+
let held = sharedServices.get(key);
|
|
838
|
+
if (!held || held.service.isClosed) {
|
|
839
|
+
held = { service: new MemoryService(options), refs: 0 };
|
|
840
|
+
sharedServices.set(key, held);
|
|
841
|
+
}
|
|
842
|
+
const lease = held;
|
|
843
|
+
lease.refs++;
|
|
844
|
+
let released = false;
|
|
845
|
+
return {
|
|
846
|
+
service: lease.service,
|
|
847
|
+
async release() {
|
|
848
|
+
if (released)
|
|
849
|
+
return;
|
|
850
|
+
released = true;
|
|
851
|
+
if (--lease.refs > 0)
|
|
852
|
+
return;
|
|
853
|
+
if (sharedServices.get(key) === lease)
|
|
854
|
+
sharedServices.delete(key);
|
|
855
|
+
await lease.service.close();
|
|
856
|
+
},
|
|
857
|
+
};
|
|
858
|
+
}
|
|
859
|
+
/** Recall counts per directory, shared for the same reason services are: one file, one writer. */
|
|
860
|
+
const sharedUsage = new Map();
|
|
861
|
+
function usageFor(location) {
|
|
862
|
+
const key = path.resolve(location);
|
|
863
|
+
let usage = sharedUsage.get(key);
|
|
864
|
+
if (!usage) {
|
|
865
|
+
usage = new MemoryUsage(location);
|
|
866
|
+
sharedUsage.set(key, usage);
|
|
867
|
+
}
|
|
868
|
+
return usage;
|
|
869
|
+
}
|
|
870
|
+
/**
|
|
871
|
+
* Write what an extraction pass proposed. New notes first, so every
|
|
872
|
+
* `[[new:N]]` placeholder has an id to become; then the notes that used a
|
|
873
|
+
* placeholder are rewritten with it.
|
|
874
|
+
*/
|
|
875
|
+
export async function commitExtracted(stores, notes) {
|
|
876
|
+
const created = [], updated = [];
|
|
877
|
+
const summaries = [];
|
|
878
|
+
const ids = new Map();
|
|
879
|
+
const written = [];
|
|
880
|
+
for (const note of notes) {
|
|
881
|
+
if (findSecrets(`${note.summary}\n${note.body}`).length)
|
|
882
|
+
continue;
|
|
883
|
+
let store;
|
|
884
|
+
try {
|
|
885
|
+
store = storeFor(stores, note.scope);
|
|
886
|
+
}
|
|
887
|
+
catch {
|
|
888
|
+
store = stores.project;
|
|
889
|
+
}
|
|
890
|
+
try {
|
|
891
|
+
if (note.action === "update") {
|
|
892
|
+
const current = note.id ? (await store.load()).find((entry) => entry.id === note.id) : undefined;
|
|
893
|
+
if (!current)
|
|
894
|
+
continue;
|
|
895
|
+
// An extractor's rewrite replaces a note nobody reviewed, so the version
|
|
896
|
+
// it replaces is kept where a deleted note would be.
|
|
897
|
+
await backupNote(store, memoryNote(current), "extract-update");
|
|
898
|
+
}
|
|
899
|
+
const entry = await store.remember({
|
|
900
|
+
summary: note.summary, body: note.body, tags: note.tags,
|
|
901
|
+
...(note.type ? { type: note.type } : {}),
|
|
902
|
+
...(note.action === "update" && note.id ? { id: note.id } : {}),
|
|
903
|
+
});
|
|
904
|
+
(note.action === "update" ? updated : created).push(entry.id);
|
|
905
|
+
summaries.push({ id: entry.id, summary: entry.summary, scope: store.scope, action: note.action === "update" ? "updated" : "created" });
|
|
906
|
+
if (note.ref !== undefined)
|
|
907
|
+
ids.set(note.ref, entry.id);
|
|
908
|
+
written.push({ store, id: entry.id, note });
|
|
909
|
+
}
|
|
910
|
+
catch { /* one bad proposal does not stop the rest */ }
|
|
911
|
+
}
|
|
912
|
+
for (const item of written) {
|
|
913
|
+
if (!/\[\[new:\d+\]\]/.test(item.note.body))
|
|
914
|
+
continue;
|
|
915
|
+
await item.store.remember({
|
|
916
|
+
id: item.id, summary: item.note.summary, body: resolvePlaceholders(item.note.body, ids), tags: item.note.tags,
|
|
917
|
+
...(item.note.type ? { type: item.note.type } : {}),
|
|
918
|
+
}).catch(() => { });
|
|
919
|
+
}
|
|
920
|
+
return { created, updated, notes: summaries };
|
|
304
921
|
}
|
|
305
922
|
export function createMemoryPlugin(options, onReady) {
|
|
306
|
-
return { id: "memory", version: "1.
|
|
923
|
+
return { id: "memory", version: "1.2.0", apiVersion: 1, description: "Local Markdown memory with recall, account Cloud sync and a persistent off switch",
|
|
307
924
|
parseConfig(raw) { if (raw !== undefined && !object(raw))
|
|
308
925
|
throw new Error("Invalid memory config"); return {}; },
|
|
309
926
|
setup(ctx) {
|
|
310
|
-
const
|
|
311
|
-
|
|
312
|
-
const
|
|
927
|
+
const policy = options.policy ?? DEFAULT_MEMORY_POLICY;
|
|
928
|
+
const warn = options.onWarning ?? ((message) => ctx.logger.warn(message));
|
|
929
|
+
const projectLease = acquireMemoryService({ ...options, scope: "project", policy, onWarning: warn });
|
|
930
|
+
const userLease = options.userDirectory
|
|
931
|
+
? acquireMemoryService({ ...options, directory: options.userDirectory, scope: "user", policy, onWarning: warn })
|
|
932
|
+
: undefined;
|
|
933
|
+
const project = projectLease.service, user = userLease?.service;
|
|
934
|
+
onReady(project, user);
|
|
935
|
+
const services = [project, ...(user ? [user] : [])];
|
|
936
|
+
const stores = { project: project.store, ...(user ? { user: user.store } : {}) };
|
|
937
|
+
const usage = { project: usageFor(project.store.location), ...(user ? { user: usageFor(user.store.location) } : {}) };
|
|
938
|
+
let closing;
|
|
939
|
+
const closeAll = () => closing ??= (async () => {
|
|
940
|
+
await Promise.all([projectLease.release(), userLease?.release()]);
|
|
941
|
+
await Promise.all([usage.project.flush(), usage.user?.flush()]);
|
|
942
|
+
})();
|
|
943
|
+
const stop = () => { void closeAll(); };
|
|
313
944
|
ctx.signal.addEventListener("abort", stop, { once: true });
|
|
314
|
-
ctx.onDispose(async () => { ctx.signal.removeEventListener("abort", stop); await
|
|
315
|
-
for (const tool of createMemoryTools(
|
|
945
|
+
ctx.onDispose(async () => { ctx.signal.removeEventListener("abort", stop); await closeAll(); });
|
|
946
|
+
for (const tool of createMemoryTools(stores))
|
|
316
947
|
ctx.tools.register(tool);
|
|
317
948
|
ctx.prompts.register(async () => {
|
|
318
949
|
try {
|
|
319
|
-
return await
|
|
950
|
+
return memoryIndexSection((await loadScoped(stores)).notes, policy.indexChars, { showScope: !!user });
|
|
320
951
|
}
|
|
321
952
|
catch {
|
|
322
953
|
ctx.logger.warn("Memory prompt unavailable; check local note files");
|
|
323
954
|
return undefined;
|
|
324
955
|
}
|
|
325
956
|
});
|
|
326
|
-
|
|
957
|
+
const inUse = () => { for (const one of services)
|
|
958
|
+
one.activity(); };
|
|
959
|
+
const provider = {
|
|
960
|
+
recall: (input) => {
|
|
961
|
+
// A message is being sent in this project: its notes should be current.
|
|
962
|
+
inUse();
|
|
963
|
+
return recallFromStores(stores, input, policy.recall, usage);
|
|
964
|
+
},
|
|
965
|
+
extraction: {
|
|
966
|
+
policy: () => policy.autoExtract,
|
|
967
|
+
async candidates(text, limit) {
|
|
968
|
+
const { notes } = await loadScoped(stores);
|
|
969
|
+
const chosen = new Map(rankNotes(notes, text.slice(-8_000)).slice(0, limit).map((entry) => [entry.note.id, entry.note]));
|
|
970
|
+
// Few matches still leaves the extractor needing to see what
|
|
971
|
+
// exists, or it re-creates notes it could have updated.
|
|
972
|
+
for (const note of [...notes].sort((a, b) => (b.updatedAt ?? b.createdAt) - (a.updatedAt ?? a.createdAt))) {
|
|
973
|
+
if (chosen.size >= limit)
|
|
974
|
+
break;
|
|
975
|
+
chosen.set(note.id, note);
|
|
976
|
+
}
|
|
977
|
+
// The most relevant notes come with their bodies, within a budget,
|
|
978
|
+
// and only those may be updated; the rest are there to link to.
|
|
979
|
+
let editable = 0, budget = MEMORY_SAFETY.extractBodyBudgetChars;
|
|
980
|
+
return [...chosen.values()].map((note) => {
|
|
981
|
+
const shown = editable < MEMORY_SAFETY.extractEditableNotes &&
|
|
982
|
+
note.body.length <= MEMORY_SAFETY.extractEditableBodyChars && note.body.length <= budget;
|
|
983
|
+
if (shown) {
|
|
984
|
+
editable++;
|
|
985
|
+
budget -= note.body.length;
|
|
986
|
+
}
|
|
987
|
+
return { id: note.id, summary: note.summary, scope: note.scope, ...(note.type ? { type: note.type } : {}), ...(shown ? { body: note.body } : {}) };
|
|
988
|
+
});
|
|
989
|
+
},
|
|
990
|
+
commit: (notes) => commitExtracted(stores, notes),
|
|
991
|
+
},
|
|
992
|
+
};
|
|
993
|
+
ctx.memory.register(provider);
|
|
994
|
+
// --- consolidation -------------------------------------------------
|
|
995
|
+
const reviewFile = `memory-review/${digest(`${project.store.location}\0${user?.store.location ?? ""}`)}.json`;
|
|
996
|
+
const readReview = () => { try {
|
|
997
|
+
return readUserFile(reviewFile) ?? {};
|
|
998
|
+
}
|
|
999
|
+
catch {
|
|
1000
|
+
return {};
|
|
1001
|
+
} };
|
|
1002
|
+
const serviceFor = (scope) => (scope === "user" && user ? user : project);
|
|
1003
|
+
const lifetime = new AbortController();
|
|
1004
|
+
let proposal;
|
|
1005
|
+
let autoTimer;
|
|
1006
|
+
const allNotes = async () => (await loadScoped(stores)).notes.map((note) => ({ ...memoryNote(note.entry), scope: note.scope }));
|
|
1007
|
+
const review = async () => {
|
|
1008
|
+
const notes = (await allNotes()).map((note) => ({ ...note, hash: noteHash(note) }));
|
|
1009
|
+
return reviewNotes(notes, {
|
|
1010
|
+
dismissed: new Set(readReview().dismissed ?? []),
|
|
1011
|
+
unusedDays: policy.consolidate.unusedDays,
|
|
1012
|
+
usage: (scope, id) => usage[scope]?.get(id),
|
|
1013
|
+
pathExists: (relative) => existsSync(path.resolve(options.cwd, relative)),
|
|
1014
|
+
});
|
|
1015
|
+
};
|
|
1016
|
+
/** The notes of a merge, in the order given, each at its current hash. */
|
|
1017
|
+
const groupOf = async (scope, ids) => {
|
|
1018
|
+
const all = await allNotes();
|
|
1019
|
+
const group = ids.map((id) => all.find((note) => note.id === id && note.scope === scope));
|
|
1020
|
+
if (group.length < 2 || group.length > 6 || group.some((note) => !note) || new Set(ids).size !== ids.length) {
|
|
1021
|
+
throw new MemoryError("mergeGone");
|
|
1022
|
+
}
|
|
1023
|
+
return group;
|
|
1024
|
+
};
|
|
1025
|
+
const propose = async (scope, ids, useModel) => {
|
|
1026
|
+
const group = await groupOf(scope, ids);
|
|
1027
|
+
let draft, source = "manual";
|
|
1028
|
+
if (useModel && options.generate) {
|
|
1029
|
+
const reply = await options.generate(CONSOLIDATE_PROMPT, mergePrompt(group), lifetime.signal);
|
|
1030
|
+
draft = parseMergeReply(reply);
|
|
1031
|
+
if (!draft)
|
|
1032
|
+
throw new MemoryError("modelNoDraft");
|
|
1033
|
+
source = "model";
|
|
1034
|
+
}
|
|
1035
|
+
return {
|
|
1036
|
+
scope, ids: group.map((note) => note.id), source,
|
|
1037
|
+
hashes: Object.fromEntries(group.map((note) => [note.id, noteHash(note)])),
|
|
1038
|
+
draft: draft ?? draftByHand(group),
|
|
1039
|
+
};
|
|
1040
|
+
};
|
|
1041
|
+
/**
|
|
1042
|
+
* Apply a merge. With cloud sync on, the scope is synced first and the
|
|
1043
|
+
* merge is committed to the cloud before it is written here, so another
|
|
1044
|
+
* device's edit in between refuses the merge rather than conflicting
|
|
1045
|
+
* with it. Links from the other scope are relinked as ordinary edits.
|
|
1046
|
+
*/
|
|
1047
|
+
const applyMerge = async (scope, ids, draft, expected) => {
|
|
1048
|
+
if (findSecrets(`${draft.summary}\n${draft.body}`).length)
|
|
1049
|
+
throw new MemoryError("mergedSecret");
|
|
1050
|
+
const service = serviceFor(scope);
|
|
1051
|
+
const cloudFirst = await service.syncReady();
|
|
1052
|
+
if (cloudFirst)
|
|
1053
|
+
await service.sync();
|
|
1054
|
+
const group = await groupOf(scope, ids);
|
|
1055
|
+
if (group.some((note) => noteHash(note) !== expected[note.id]))
|
|
1056
|
+
throw new MemoryError("mergeStale");
|
|
1057
|
+
const edits = planMerge({ all: await allNotes(), scope, ids, draft, hash: noteHash });
|
|
1058
|
+
for (const edit of edits)
|
|
1059
|
+
if (edit.note)
|
|
1060
|
+
validateNote(edit.note, serviceFor(edit.scope).noteBytesLimit());
|
|
1061
|
+
const own = edits.filter((edit) => edit.scope === scope);
|
|
1062
|
+
if (cloudFirst) {
|
|
1063
|
+
await service.commitFirst(own.map(({ id, note, expectedHash }) => ({ id, expectedHash, ...(note ? { note } : {}) })), "consolidate");
|
|
1064
|
+
}
|
|
1065
|
+
for (const edit of cloudFirst ? edits.filter((one) => one.scope !== scope) : edits) {
|
|
1066
|
+
const store = serviceFor(edit.scope).store;
|
|
1067
|
+
const current = (await store.load()).find((entry) => entry.id === edit.id);
|
|
1068
|
+
if (noteHash(current) !== edit.expectedHash)
|
|
1069
|
+
throw new MemoryError("mergeStale");
|
|
1070
|
+
if (current)
|
|
1071
|
+
await backupNote(store, memoryNote(current), "consolidate");
|
|
1072
|
+
await store.apply(edit.id, edit.note, edit.expectedHash, "local");
|
|
1073
|
+
}
|
|
1074
|
+
};
|
|
1075
|
+
const autoOn = () => consolidationEnabled(policy.consolidate.auto, services.map((one) => one.tier));
|
|
1076
|
+
/** The background pass: duplicates only, merged by the model, within a cooldown. */
|
|
1077
|
+
const autoConsolidate = async () => {
|
|
1078
|
+
const settings = policy.consolidate;
|
|
1079
|
+
if (!autoOn() || !options.generate || settings.maxMerges === 0 || lifetime.signal.aborted)
|
|
1080
|
+
return;
|
|
1081
|
+
const state = readReview();
|
|
1082
|
+
if (Date.now() - (state.lastAuto ?? 0) < settings.cooldownHours * 3_600_000)
|
|
1083
|
+
return;
|
|
1084
|
+
// Recorded before the pass, so a pass that crashes waits out the cooldown too.
|
|
1085
|
+
await writeUserFile(reviewFile, { ...state, lastAuto: Date.now() });
|
|
1086
|
+
const merged = [];
|
|
1087
|
+
for (const finding of (await review()).filter((one) => AUTO_KINDS.has(one.kind)).slice(0, settings.maxMerges)) {
|
|
1088
|
+
if (lifetime.signal.aborted)
|
|
1089
|
+
break;
|
|
1090
|
+
try {
|
|
1091
|
+
const next = await propose(finding.scope, finding.ids, true);
|
|
1092
|
+
await applyMerge(finding.scope, next.ids, next.draft, next.hashes);
|
|
1093
|
+
merged.push({ scope: finding.scope, id: next.ids[0], summary: next.draft.summary, count: next.ids.length });
|
|
1094
|
+
}
|
|
1095
|
+
catch (error) {
|
|
1096
|
+
ctx.logger.warn("memory consolidation skipped a group", { error: error instanceof Error ? error.message : String(error) });
|
|
1097
|
+
}
|
|
1098
|
+
}
|
|
1099
|
+
if (merged.length)
|
|
1100
|
+
await writeUserFile(reviewFile, { ...readReview(), autoResult: { at: Date.now(), merged } });
|
|
1101
|
+
};
|
|
1102
|
+
const status = async () => {
|
|
1103
|
+
inUse();
|
|
1104
|
+
const [projectStatus, userStatus] = await Promise.all([project.status(), user?.status()]);
|
|
1105
|
+
const scoped = userStatus ? [projectStatus, userStatus] : [projectStatus];
|
|
1106
|
+
let links;
|
|
1107
|
+
try {
|
|
1108
|
+
links = (await loadScoped(stores)).links;
|
|
1109
|
+
}
|
|
1110
|
+
catch {
|
|
1111
|
+
links = undefined;
|
|
1112
|
+
}
|
|
1113
|
+
const decorate = (entries, scope) => entries.map((entry) => ({
|
|
1114
|
+
...entry, scope,
|
|
1115
|
+
links: { out: links?.outgoing.get(entry.id) ?? [], in: links?.incoming.get(entry.id) ?? [], dangling: links?.dangling.get(entry.id) ?? [] },
|
|
1116
|
+
}));
|
|
1117
|
+
const synced = scoped.map((one) => one.lastSync);
|
|
1118
|
+
const backups = await listBackups(50).catch(() => []);
|
|
1119
|
+
const findings = await review().catch(() => []);
|
|
1120
|
+
const reviewState = readReview();
|
|
1121
|
+
return {
|
|
1122
|
+
review: findings.slice(0, 100),
|
|
1123
|
+
reviewTotal: findings.length,
|
|
1124
|
+
...(proposal ? { proposal } : {}),
|
|
1125
|
+
consolidate: {
|
|
1126
|
+
auto: autoOn() && !!options.generate, modelAvailable: !!options.generate,
|
|
1127
|
+
...(reviewState.autoResult ? { lastAuto: reviewState.autoResult } : {}),
|
|
1128
|
+
},
|
|
1129
|
+
...projectStatus,
|
|
1130
|
+
// Both scopes, not the project alone: the page's one switch and one
|
|
1131
|
+
// "last synced" line speak for every note it lists.
|
|
1132
|
+
enabled: scoped.every((one) => one.enabled),
|
|
1133
|
+
ready: scoped.every((one) => one.ready),
|
|
1134
|
+
accountChanged: scoped.some((one) => one.accountChanged),
|
|
1135
|
+
syncing: scoped.some((one) => one.syncing),
|
|
1136
|
+
lastSync: synced.every((at) => typeof at === "number") ? Math.min(...synced) : undefined,
|
|
1137
|
+
error: scoped.find((one) => one.error)?.error,
|
|
1138
|
+
errors: scoped.filter((one) => one.failure).map((one) => ({ scope: one.scope, ...one.failure })),
|
|
1139
|
+
recent: scoped.filter((one) => one.lastPull).map((one) => ({ scope: one.scope, ...one.lastPull })),
|
|
1140
|
+
conflicts: [
|
|
1141
|
+
...projectStatus.conflicts.map((conflict) => ({ ...conflict, scope: "project" })),
|
|
1142
|
+
...(userStatus?.conflicts ?? []).map((conflict) => ({ ...conflict, scope: "user" })),
|
|
1143
|
+
],
|
|
1144
|
+
locations: { project: projectStatus.location, ...(userStatus ? { user: userStatus.location } : {}) },
|
|
1145
|
+
entries: [...decorate(projectStatus.entries, "project"), ...(userStatus ? decorate(userStatus.entries, "user") : [])],
|
|
1146
|
+
omitted: projectStatus.omitted + (userStatus?.omitted ?? 0),
|
|
1147
|
+
limits: { maxNotes: policy.maxNotes },
|
|
1148
|
+
quota: projectStatus.quota ?? userStatus?.quota,
|
|
1149
|
+
userAvailable: !!user,
|
|
1150
|
+
backups: backups.map((backup) => ({
|
|
1151
|
+
file: backup.file, backedUpAt: backup.backedUpAt, id: backup.note.id, summary: backup.note.summary,
|
|
1152
|
+
...(backup.reason ? { reason: backup.reason } : {}), ...(backup.scope ? { scope: backup.scope } : {}),
|
|
1153
|
+
})),
|
|
1154
|
+
};
|
|
1155
|
+
};
|
|
1156
|
+
ctx.pages.register({ id: "memory", title: "Memory", renderer: "memory", read: status,
|
|
327
1157
|
async act(input) {
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
await
|
|
332
|
-
|
|
333
|
-
|
|
1158
|
+
const scope = input.scope === "user" && user ? "user" : "project";
|
|
1159
|
+
const service = scope === "user" ? user : project;
|
|
1160
|
+
if (input.action === "sync") {
|
|
1161
|
+
const results = await Promise.allSettled(services.map((one) => one.sync()));
|
|
1162
|
+
const failed = results.find((result) => result.status === "rejected");
|
|
1163
|
+
if (failed)
|
|
1164
|
+
throw failed.reason;
|
|
1165
|
+
}
|
|
1166
|
+
else if (input.action === "enable" && typeof input.enabled === "boolean") {
|
|
1167
|
+
// One switch for both scopes: a person turning cloud sync off means
|
|
1168
|
+
// all of their notes, not the half the page happened to show first.
|
|
1169
|
+
for (const one of services)
|
|
1170
|
+
await one.enable(input.enabled);
|
|
1171
|
+
}
|
|
1172
|
+
else if (input.action === "resolve" && typeof input.id === "string" && typeof input.choice === "string") {
|
|
1173
|
+
await service.resolve(input.id, input.choice, object(input.note) ? input.note : undefined);
|
|
1174
|
+
}
|
|
334
1175
|
else if (input.action === "save" && object(input.note)) {
|
|
1176
|
+
const note = input.note;
|
|
1177
|
+
const secrets = findSecrets(`${String(note.summary ?? "")}\n${String(note.body ?? "")}`);
|
|
1178
|
+
if (secrets.length)
|
|
1179
|
+
throw new MemoryError("secretFound", { kinds: secrets.join(", ") });
|
|
1180
|
+
const type = isMemoryType(note.type) ? note.type : undefined;
|
|
1181
|
+
const clearType = note.type === "" || note.type === null;
|
|
335
1182
|
if (input.id) {
|
|
336
1183
|
const id = String(input.id), current = (await service.store.load()).find((entry) => entry.id === id);
|
|
337
1184
|
if (!current || input.expectedHash !== noteHash(current))
|
|
338
|
-
throw new
|
|
339
|
-
|
|
1185
|
+
throw new MemoryError("editStale");
|
|
1186
|
+
const next = {
|
|
1187
|
+
...memoryNote(current),
|
|
1188
|
+
...(typeof note.summary === "string" ? { summary: note.summary } : {}),
|
|
1189
|
+
...(typeof note.body === "string" ? { body: note.body } : {}),
|
|
1190
|
+
...(Array.isArray(note.tags) ? { tags: note.tags } : {}),
|
|
1191
|
+
id, createdAt: current.createdAt, updatedAt: Date.now(),
|
|
1192
|
+
};
|
|
1193
|
+
if (type)
|
|
1194
|
+
next.type = type;
|
|
1195
|
+
else if (clearType)
|
|
1196
|
+
delete next.type;
|
|
1197
|
+
await service.store.apply(id, next, noteHash(current));
|
|
1198
|
+
}
|
|
1199
|
+
else {
|
|
1200
|
+
await service.store.remember({
|
|
1201
|
+
summary: String(note.summary ?? ""), body: String(note.body ?? ""),
|
|
1202
|
+
...(Array.isArray(note.tags) ? { tags: note.tags } : {}),
|
|
1203
|
+
...(type ? { type } : {}),
|
|
1204
|
+
});
|
|
340
1205
|
}
|
|
341
|
-
else
|
|
342
|
-
await service.store.remember(input.note);
|
|
343
1206
|
}
|
|
344
1207
|
else if (input.action === "delete" && typeof input.id === "string") {
|
|
345
1208
|
const entry = (await service.store.load()).find((one) => one.id === input.id);
|
|
346
1209
|
if (!entry || input.expectedHash !== noteHash(entry))
|
|
347
|
-
throw new
|
|
348
|
-
await
|
|
1210
|
+
throw new MemoryError("deleteStale");
|
|
1211
|
+
await backupNote(service.store, memoryNote(entry), "delete");
|
|
349
1212
|
await service.store.apply(entry.id, undefined, noteHash(entry));
|
|
350
1213
|
}
|
|
1214
|
+
else if (input.action === "restore" && typeof input.file === "string") {
|
|
1215
|
+
await restoreBackup(stores, input.file);
|
|
1216
|
+
}
|
|
1217
|
+
else if (input.action === "dismiss" && typeof input.key === "string" && /^[a-z-]+:[a-f0-9]{24}$/.test(input.key)) {
|
|
1218
|
+
const state = readReview();
|
|
1219
|
+
const dismissed = [...new Set([...(state.dismissed ?? []), input.key])].slice(-2000);
|
|
1220
|
+
await writeUserFile(reviewFile, { ...state, dismissed });
|
|
1221
|
+
}
|
|
1222
|
+
else if (input.action === "propose" && Array.isArray(input.ids) && input.ids.every((id) => typeof id === "string")) {
|
|
1223
|
+
proposal = await propose(scope, input.ids, input.model === true);
|
|
1224
|
+
}
|
|
1225
|
+
else if (input.action === "cancelMerge") {
|
|
1226
|
+
proposal = undefined;
|
|
1227
|
+
}
|
|
1228
|
+
else if (input.action === "merge" && proposal && object(input.note)) {
|
|
1229
|
+
const note = input.note;
|
|
1230
|
+
const draft = {
|
|
1231
|
+
summary: typeof note.summary === "string" ? note.summary.replace(/[\r\n]+/g, " ").trim() : "",
|
|
1232
|
+
body: typeof note.body === "string" ? note.body.trim() : "",
|
|
1233
|
+
tags: Array.isArray(note.tags) ? note.tags.filter((tag) => typeof tag === "string") : [],
|
|
1234
|
+
...(isMemoryType(note.type) ? { type: note.type } : {}),
|
|
1235
|
+
};
|
|
1236
|
+
if (!draft.summary || !draft.body)
|
|
1237
|
+
throw new MemoryError("mergeEmpty");
|
|
1238
|
+
await applyMerge(proposal.scope, proposal.ids, draft, proposal.hashes);
|
|
1239
|
+
proposal = undefined;
|
|
1240
|
+
}
|
|
351
1241
|
else
|
|
352
|
-
throw new
|
|
353
|
-
return
|
|
1242
|
+
throw new MemoryError("unknownAction");
|
|
1243
|
+
return status();
|
|
354
1244
|
},
|
|
355
1245
|
});
|
|
356
|
-
ctx.onStart(() =>
|
|
1246
|
+
ctx.onStart(() => {
|
|
1247
|
+
for (const one of services)
|
|
1248
|
+
one.start();
|
|
1249
|
+
// Not at startup itself: a pass syncs and calls a model, and the first
|
|
1250
|
+
// minutes of a session are when the person is waiting on both. By then
|
|
1251
|
+
// the first sync has also said whether the account is a subscriber.
|
|
1252
|
+
if (policy.consolidate.auto !== false && options.generate) {
|
|
1253
|
+
autoTimer = setTimeout(() => {
|
|
1254
|
+
void autoConsolidate().catch((error) => ctx.logger.warn("memory consolidation failed", { error: error instanceof Error ? error.message : String(error) }));
|
|
1255
|
+
}, 2 * 60_000);
|
|
1256
|
+
autoTimer.unref?.();
|
|
1257
|
+
}
|
|
1258
|
+
});
|
|
1259
|
+
ctx.onDispose(() => { lifetime.abort(); if (autoTimer)
|
|
1260
|
+
clearTimeout(autoTimer); });
|
|
357
1261
|
},
|
|
358
1262
|
};
|
|
359
1263
|
}
|