agent-recall-mcp 3.5.2 → 3.5.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (219) hide show
  1. package/README.md +254 -143
  2. package/dist/build-server.d.ts +28 -0
  3. package/dist/build-server.d.ts.map +1 -0
  4. package/dist/build-server.js +55 -0
  5. package/dist/build-server.js.map +1 -0
  6. package/dist/http/activity.d.ts +60 -0
  7. package/dist/http/activity.d.ts.map +1 -0
  8. package/dist/http/activity.js +254 -0
  9. package/dist/http/activity.js.map +1 -0
  10. package/dist/http/alerts.d.ts +84 -0
  11. package/dist/http/alerts.d.ts.map +1 -0
  12. package/dist/http/alerts.js +213 -0
  13. package/dist/http/alerts.js.map +1 -0
  14. package/dist/http/cli.d.ts +3 -0
  15. package/dist/http/cli.d.ts.map +1 -0
  16. package/dist/http/cli.js +515 -0
  17. package/dist/http/cli.js.map +1 -0
  18. package/dist/http/export-tar.d.ts +23 -0
  19. package/dist/http/export-tar.d.ts.map +1 -0
  20. package/dist/http/export-tar.js +264 -0
  21. package/dist/http/export-tar.js.map +1 -0
  22. package/dist/http/federation.d.ts +45 -0
  23. package/dist/http/federation.d.ts.map +1 -0
  24. package/dist/http/federation.js +185 -0
  25. package/dist/http/federation.js.map +1 -0
  26. package/dist/http/film/film.css +80 -0
  27. package/dist/http/film/film.js +446 -0
  28. package/dist/http/film/human.jpg +0 -0
  29. package/dist/http/film/m0.jpg +0 -0
  30. package/dist/http/film/m2.jpg +0 -0
  31. package/dist/http/film/m3.jpg +0 -0
  32. package/dist/http/film/wrong.jpg +0 -0
  33. package/dist/http/film.d.ts +16 -0
  34. package/dist/http/film.d.ts.map +1 -0
  35. package/dist/http/film.js +119 -0
  36. package/dist/http/film.js.map +1 -0
  37. package/dist/http/home/cabinet.svg +1 -0
  38. package/dist/http/home/drawer-project.svg +1 -0
  39. package/dist/http/home/drawer-rules.svg +1 -0
  40. package/dist/http/home/drawer-you.svg +1 -0
  41. package/dist/http/home/fig1.svg +242 -0
  42. package/dist/http/home/fig2-1.svg +61 -0
  43. package/dist/http/home/fig2-2.svg +92 -0
  44. package/dist/http/home/fig3.svg +1 -0
  45. package/dist/http/home/home.css +509 -0
  46. package/dist/http/home/home.html +517 -0
  47. package/dist/http/home/icons.svg +15 -0
  48. package/dist/http/home.d.ts +23 -0
  49. package/dist/http/home.d.ts.map +1 -0
  50. package/dist/http/home.js +295 -0
  51. package/dist/http/home.js.map +1 -0
  52. package/dist/http/identities.d.ts +29 -0
  53. package/dist/http/identities.d.ts.map +1 -0
  54. package/dist/http/identities.js +91 -0
  55. package/dist/http/identities.js.map +1 -0
  56. package/dist/http/import-memory.d.ts +161 -0
  57. package/dist/http/import-memory.d.ts.map +1 -0
  58. package/dist/http/import-memory.js +718 -0
  59. package/dist/http/import-memory.js.map +1 -0
  60. package/dist/http/jsonfile.d.ts +6 -0
  61. package/dist/http/jsonfile.d.ts.map +1 -0
  62. package/dist/http/jsonfile.js +60 -0
  63. package/dist/http/jsonfile.js.map +1 -0
  64. package/dist/http/legal/PRIVACY.md +70 -0
  65. package/dist/http/legal.d.ts +10 -0
  66. package/dist/http/legal.d.ts.map +1 -0
  67. package/dist/http/legal.js +35 -0
  68. package/dist/http/legal.js.map +1 -0
  69. package/dist/http/limits.d.ts +59 -0
  70. package/dist/http/limits.d.ts.map +1 -0
  71. package/dist/http/limits.js +163 -0
  72. package/dist/http/limits.js.map +1 -0
  73. package/dist/http/markdown-lite.d.ts +2 -0
  74. package/dist/http/markdown-lite.d.ts.map +1 -0
  75. package/dist/http/markdown-lite.js +86 -0
  76. package/dist/http/markdown-lite.js.map +1 -0
  77. package/dist/http/multipart.d.ts +43 -0
  78. package/dist/http/multipart.d.ts.map +1 -0
  79. package/dist/http/multipart.js +152 -0
  80. package/dist/http/multipart.js.map +1 -0
  81. package/dist/http/oauth-app.d.ts +75 -0
  82. package/dist/http/oauth-app.d.ts.map +1 -0
  83. package/dist/http/oauth-app.js +259 -0
  84. package/dist/http/oauth-app.js.map +1 -0
  85. package/dist/http/oauth-config.d.ts +46 -0
  86. package/dist/http/oauth-config.d.ts.map +1 -0
  87. package/dist/http/oauth-config.js +122 -0
  88. package/dist/http/oauth-config.js.map +1 -0
  89. package/dist/http/oauth-pages.d.ts +40 -0
  90. package/dist/http/oauth-pages.d.ts.map +1 -0
  91. package/dist/http/oauth-pages.js +97 -0
  92. package/dist/http/oauth-pages.js.map +1 -0
  93. package/dist/http/oauth-provider.d.ts +124 -0
  94. package/dist/http/oauth-provider.d.ts.map +1 -0
  95. package/dist/http/oauth-provider.js +486 -0
  96. package/dist/http/oauth-provider.js.map +1 -0
  97. package/dist/http/oauth-store.d.ts +128 -0
  98. package/dist/http/oauth-store.d.ts.map +1 -0
  99. package/dist/http/oauth-store.js +317 -0
  100. package/dist/http/oauth-store.js.map +1 -0
  101. package/dist/http/redirect-policy.d.ts +16 -0
  102. package/dist/http/redirect-policy.d.ts.map +1 -0
  103. package/dist/http/redirect-policy.js +83 -0
  104. package/dist/http/redirect-policy.js.map +1 -0
  105. package/dist/http/review-pages.d.ts +39 -0
  106. package/dist/http/review-pages.d.ts.map +1 -0
  107. package/dist/http/review-pages.js +78 -0
  108. package/dist/http/review-pages.js.map +1 -0
  109. package/dist/http/review.d.ts +36 -0
  110. package/dist/http/review.d.ts.map +1 -0
  111. package/dist/http/review.js +220 -0
  112. package/dist/http/review.js.map +1 -0
  113. package/dist/http/serve.d.ts +58 -0
  114. package/dist/http/serve.d.ts.map +1 -0
  115. package/dist/http/serve.js +324 -0
  116. package/dist/http/serve.js.map +1 -0
  117. package/dist/http/site-pages.d.ts +110 -0
  118. package/dist/http/site-pages.d.ts.map +1 -0
  119. package/dist/http/site-pages.js +630 -0
  120. package/dist/http/site-pages.js.map +1 -0
  121. package/dist/http/site-theme.d.ts +13 -0
  122. package/dist/http/site-theme.d.ts.map +1 -0
  123. package/dist/http/site-theme.js +22 -0
  124. package/dist/http/site-theme.js.map +1 -0
  125. package/dist/http/site.d.ts +52 -0
  126. package/dist/http/site.d.ts.map +1 -0
  127. package/dist/http/site.js +353 -0
  128. package/dist/http/site.js.map +1 -0
  129. package/dist/http/stats.d.ts +139 -0
  130. package/dist/http/stats.d.ts.map +1 -0
  131. package/dist/http/stats.js +599 -0
  132. package/dist/http/stats.js.map +1 -0
  133. package/dist/http/tar-import.d.ts +69 -0
  134. package/dist/http/tar-import.d.ts.map +1 -0
  135. package/dist/http/tar-import.js +234 -0
  136. package/dist/http/tar-import.js.map +1 -0
  137. package/dist/http/tenant-paths.d.ts +14 -0
  138. package/dist/http/tenant-paths.d.ts.map +1 -0
  139. package/dist/http/tenant-paths.js +42 -0
  140. package/dist/http/tenant-paths.js.map +1 -0
  141. package/dist/http/tokens.d.ts +32 -0
  142. package/dist/http/tokens.d.ts.map +1 -0
  143. package/dist/http/tokens.js +137 -0
  144. package/dist/http/tokens.js.map +1 -0
  145. package/dist/http/trial.d.ts +55 -0
  146. package/dist/http/trial.d.ts.map +1 -0
  147. package/dist/http/trial.js +218 -0
  148. package/dist/http/trial.js.map +1 -0
  149. package/dist/http/web-session.d.ts +108 -0
  150. package/dist/http/web-session.d.ts.map +1 -0
  151. package/dist/http/web-session.js +267 -0
  152. package/dist/http/web-session.js.map +1 -0
  153. package/dist/index.js +12 -65
  154. package/dist/index.js.map +1 -1
  155. package/dist/lib/ambient-capture.d.ts.map +1 -1
  156. package/dist/lib/ambient-capture.js +4 -1
  157. package/dist/lib/ambient-capture.js.map +1 -1
  158. package/dist/lib/bookkeeping.d.ts +26 -0
  159. package/dist/lib/bookkeeping.d.ts.map +1 -0
  160. package/dist/lib/bookkeeping.js +31 -0
  161. package/dist/lib/bookkeeping.js.map +1 -0
  162. package/dist/lib/hosted-redact.d.ts +17 -0
  163. package/dist/lib/hosted-redact.d.ts.map +1 -0
  164. package/dist/lib/hosted-redact.js +57 -0
  165. package/dist/lib/hosted-redact.js.map +1 -0
  166. package/dist/lib/lifecycle-exit.d.ts +5 -13
  167. package/dist/lib/lifecycle-exit.d.ts.map +1 -1
  168. package/dist/lib/lifecycle-exit.js +38 -16
  169. package/dist/lib/lifecycle-exit.js.map +1 -1
  170. package/dist/server.d.ts +3 -2
  171. package/dist/server.d.ts.map +1 -1
  172. package/dist/server.js +6 -3
  173. package/dist/server.js.map +1 -1
  174. package/dist/tools/annotations.d.ts +65 -0
  175. package/dist/tools/annotations.d.ts.map +1 -0
  176. package/dist/tools/annotations.js +4 -0
  177. package/dist/tools/annotations.js.map +1 -0
  178. package/dist/tools/check-action.d.ts.map +1 -1
  179. package/dist/tools/check-action.js +2 -0
  180. package/dist/tools/check-action.js.map +1 -1
  181. package/dist/tools/check.d.ts.map +1 -1
  182. package/dist/tools/check.js +6 -2
  183. package/dist/tools/check.js.map +1 -1
  184. package/dist/tools/digest.d.ts.map +1 -1
  185. package/dist/tools/digest.js +2 -0
  186. package/dist/tools/digest.js.map +1 -1
  187. package/dist/tools/pipeline-close.d.ts.map +1 -1
  188. package/dist/tools/pipeline-close.js +2 -0
  189. package/dist/tools/pipeline-close.js.map +1 -1
  190. package/dist/tools/pipeline-current.d.ts.map +1 -1
  191. package/dist/tools/pipeline-current.js +2 -0
  192. package/dist/tools/pipeline-current.js.map +1 -1
  193. package/dist/tools/pipeline-list.d.ts.map +1 -1
  194. package/dist/tools/pipeline-list.js +2 -0
  195. package/dist/tools/pipeline-list.js.map +1 -1
  196. package/dist/tools/pipeline-open.d.ts.map +1 -1
  197. package/dist/tools/pipeline-open.js +2 -0
  198. package/dist/tools/pipeline-open.js.map +1 -1
  199. package/dist/tools/pipeline-show.d.ts.map +1 -1
  200. package/dist/tools/pipeline-show.js +2 -0
  201. package/dist/tools/pipeline-show.js.map +1 -1
  202. package/dist/tools/recall.d.ts.map +1 -1
  203. package/dist/tools/recall.js +4 -1
  204. package/dist/tools/recall.js.map +1 -1
  205. package/dist/tools/register-rule.d.ts.map +1 -1
  206. package/dist/tools/register-rule.js +2 -0
  207. package/dist/tools/register-rule.js.map +1 -1
  208. package/dist/tools/remember.d.ts.map +1 -1
  209. package/dist/tools/remember.js +10 -1
  210. package/dist/tools/remember.js.map +1 -1
  211. package/dist/tools/session-end.d.ts +15 -0
  212. package/dist/tools/session-end.d.ts.map +1 -1
  213. package/dist/tools/session-end.js +13 -1
  214. package/dist/tools/session-end.js.map +1 -1
  215. package/dist/tools/session-start.d.ts +6 -1
  216. package/dist/tools/session-start.d.ts.map +1 -1
  217. package/dist/tools/session-start.js +24 -212
  218. package/dist/tools/session-start.js.map +1 -1
  219. package/package.json +11 -8
@@ -0,0 +1,718 @@
1
+ /**
2
+ * import-memory.ts — "Import memory": bring a local AgentRecall store (a .tar) into the
3
+ * signed-in user's hosted account. The mirror of "Download my memory".
4
+ *
5
+ * Why this is careful: an imported correction becomes an always-loaded HARD RULE for every
6
+ * agent of the account, and the archive is input from outside. So:
7
+ * - only the owner's browser session reaches this (site.ts); an agent token never does;
8
+ * - the archive is parsed as a STREAM (multipart.ts → tar-import.ts) and written to a
9
+ * staging directory (DATA/_server/import-staging/<uid>-<random>/), never into the tenant;
10
+ * - only what a memory is gets staged (tar-import.ts: allowlist, path rules, limits); the
11
+ * size/entry/quota limits are enforced while streaming and abort at once;
12
+ * - nothing reaches the tenant until the whole archive has been read successfully and its
13
+ * end marker seen. Any error, abort or disconnect removes the staging directory (finally);
14
+ * - commit "add": each file is hard-linked (or copied, exclusive) into the tenant only if
15
+ * the destination does not exist, with every parent re-checked (lstat + realpath);
16
+ * - commit "replace": the current tenant directory is renamed into DATA/_server/trash/
17
+ * (kept 7 days, then swept) and the staging directory is renamed into its place.
18
+ */
19
+ import * as fs from "node:fs";
20
+ import * as path from "node:path";
21
+ import { randomBytes } from "node:crypto";
22
+ import { decodeTextFile } from "agent-recall-core";
23
+ import { dirSize } from "./limits.js";
24
+ import { tenantRoot } from "./tenant-paths.js";
25
+ import { UID_RE } from "./tokens.js";
26
+ import { MultipartParser, MultipartError } from "./multipart.js";
27
+ import { TarReader, TarError, classifyPath } from "./tar-import.js";
28
+ export const DEFAULT_IMPORT_LIMITS = {
29
+ maxFileBytes: 8 * 1024 * 1024,
30
+ maxEntries: 50_000,
31
+ idleMs: 30_000,
32
+ totalMs: 15 * 60_000,
33
+ minBytesPerSec: 16 * 1024,
34
+ windowMs: 10_000,
35
+ };
36
+ /** No import may take in more than this, even with no quota configured. */
37
+ export const HARD_IMPORT_CAP = 1024 ** 3;
38
+ /** Bytes of tar headers/padding/skipped entries allowed on top of what is accepted. */
39
+ export const RAW_SLACK = 64 * 1024 * 1024;
40
+ const DISK_BLOCK = 4096;
41
+ /** New directories an import may create (each costs a block too): bounds inode amplification. */
42
+ const MAX_DIRS = 2000;
43
+ export const TRASH_TTL_MS = 7 * 24 * 3600 * 1000;
44
+ const STAGING_MAX_AGE_MS = 3600 * 1000;
45
+ export const IMPORTS_PER_HOUR = 5;
46
+ const NOFOLLOW = fs.constants.O_NOFOLLOW ?? 0;
47
+ /** A refusal the user should see: status, short title and a plain sentence. */
48
+ export class ImportAbort extends Error {
49
+ status;
50
+ title;
51
+ retryAfter;
52
+ constructor(status, title, message, retryAfter) {
53
+ super(message);
54
+ this.status = status;
55
+ this.title = title;
56
+ this.retryAfter = retryAfter;
57
+ }
58
+ }
59
+ const dirs = (dataDir) => ({
60
+ staging: path.join(path.resolve(dataDir), "_server", "import-staging"),
61
+ trash: path.join(path.resolve(dataDir), "_server", "trash"),
62
+ });
63
+ const inside = (real, rootReal) => real === rootReal || real.startsWith(rootReal + path.sep);
64
+ const code = (e) => e?.code;
65
+ const printable = (s) => s.replace(/[^\x20-\x7e]/g, "?").slice(0, 80);
66
+ /** What a file costs on disk: whole blocks, and an empty file still takes one (an inode is never free). */
67
+ const blocks = (n) => Math.max(1, Math.ceil(n / DISK_BLOCK)) * DISK_BLOCK;
68
+ // ---- one import per user, a few per hour -----------------------------------------
69
+ export class ImportGate {
70
+ now;
71
+ perHour;
72
+ maxConcurrent;
73
+ active = new Set();
74
+ ips = new Set();
75
+ /** At most this many imports run at once across all accounts (each holds a staging tree and bookkeeping in memory). */
76
+ static MAX_CONCURRENT = 4;
77
+ times = new Map();
78
+ constructor(now, perHour = IMPORTS_PER_HOUR, maxConcurrent = ImportGate.MAX_CONCURRENT) {
79
+ this.now = now;
80
+ this.perHour = perHour;
81
+ this.maxConcurrent = maxConcurrent;
82
+ }
83
+ /**
84
+ * null = granted (call release() later); otherwise why not:
85
+ * busy = this account already has one running; rate = too many this hour; ip = another import from
86
+ * this address is running; full = the server's global slots are all taken (not the caller's fault).
87
+ */
88
+ tryAcquire(uid, ip) {
89
+ if (this.active.has(uid))
90
+ return "busy";
91
+ const t = this.now();
92
+ const recent = (this.times.get(uid) ?? []).filter((x) => t - x < 3600_000);
93
+ if (recent.length >= this.perHour) {
94
+ this.times.set(uid, recent);
95
+ return "rate";
96
+ }
97
+ if (this.ips.has(ip))
98
+ return "ip";
99
+ if (this.active.size >= this.maxConcurrent)
100
+ return "full";
101
+ recent.push(t);
102
+ if (this.times.size > 5000) {
103
+ for (const [k, v] of this.times)
104
+ if (v.every((x) => t - x >= 3600_000))
105
+ this.times.delete(k);
106
+ }
107
+ this.times.set(uid, recent);
108
+ this.active.add(uid);
109
+ this.ips.add(ip);
110
+ return null;
111
+ }
112
+ release(uid, ip) { this.active.delete(uid); this.ips.delete(ip); }
113
+ isBusy(uid) { return this.active.has(uid); }
114
+ }
115
+ // ---- trash and stale staging -------------------------------------------------------
116
+ const TRASH_NAME_RE = /-(\d{4})-(\d{2})-(\d{2})T(\d{2})-(\d{2})-(\d{2})-(\d{3})Z(?:-[a-z]+)?$/;
117
+ const isoName = (ms) => new Date(ms).toISOString().replace(/[:.]/g, "-");
118
+ /** Deletes trash entries whose name-embedded timestamp is older than the TTL. Entries it cannot date are left alone. Returns how many were removed. */
119
+ export async function sweepTrash(dataDir, nowMs, ttlMs = TRASH_TTL_MS) {
120
+ const { trash } = dirs(dataDir);
121
+ let names;
122
+ try {
123
+ names = await fs.promises.readdir(trash);
124
+ }
125
+ catch {
126
+ return 0;
127
+ }
128
+ let removed = 0;
129
+ for (const n of names) {
130
+ const m = TRASH_NAME_RE.exec(n);
131
+ if (!m)
132
+ continue;
133
+ const at = Date.UTC(+m[1], +m[2] - 1, +m[3], +m[4], +m[5], +m[6], +m[7]);
134
+ if (!(nowMs - at > ttlMs))
135
+ continue;
136
+ try {
137
+ await fs.promises.rm(path.join(trash, n), { recursive: true, force: true });
138
+ removed++;
139
+ }
140
+ catch { /* retried next sweep */ }
141
+ }
142
+ return removed;
143
+ }
144
+ /** Removes staging directories a crashed process left behind (older than an hour). */
145
+ export async function sweepStaging(dataDir, nowMs, maxAgeMs = STAGING_MAX_AGE_MS) {
146
+ const { staging } = dirs(dataDir);
147
+ let names;
148
+ try {
149
+ names = await fs.promises.readdir(staging);
150
+ }
151
+ catch {
152
+ return 0;
153
+ }
154
+ let removed = 0;
155
+ for (const n of names) {
156
+ const p = path.join(staging, n);
157
+ try {
158
+ const st = await fs.promises.lstat(p);
159
+ if (nowMs - st.mtimeMs <= maxAgeMs)
160
+ continue;
161
+ await fs.promises.rm(p, { recursive: true, force: true });
162
+ removed++;
163
+ }
164
+ catch { /* next time */ }
165
+ }
166
+ return removed;
167
+ }
168
+ const TRASH_ENTRY_RE = /^(.+)-(\d{4})-(\d{2})-(\d{2})T(\d{2})-(\d{2})-(\d{2})-(\d{3})Z(-late)?$/;
169
+ /** A replace dies between its two renames within seconds; an older trash entry is a deliberate one (never resurrect it). */
170
+ export const ORPHAN_MAX_AGE_MS = 3600_000;
171
+ /**
172
+ * If a replace was cut short (the process died between moving the tenant to trash and moving the new tree
173
+ * in), the tenant is missing (or an agent request re-created it empty) while its data sits in trash. Put the
174
+ * newest recent trash entry back. Run at server start for every account, and before an import for its account.
175
+ */
176
+ export async function restoreOrphans(dataDir, nowMs, log, only) {
177
+ const { trash } = dirs(dataDir);
178
+ let names;
179
+ try {
180
+ names = await fs.promises.readdir(trash);
181
+ }
182
+ catch {
183
+ return 0;
184
+ }
185
+ const best = new Map();
186
+ for (const n of names) {
187
+ const m = TRASH_ENTRY_RE.exec(n);
188
+ if (!m || !UID_RE.test(m[1]) || (only !== undefined && m[1] !== only))
189
+ continue;
190
+ const at = Date.UTC(+m[2], +m[3] - 1, +m[4], +m[5], +m[6], +m[7], +m[8]);
191
+ if (nowMs - at > ORPHAN_MAX_AGE_MS || at - nowMs > ORPHAN_MAX_AGE_MS)
192
+ continue;
193
+ const cur = best.get(m[1]);
194
+ const late = m[9] !== undefined;
195
+ if (!cur || at > cur.at || (at === cur.at && cur.late && !late))
196
+ best.set(m[1], { name: n, at, late });
197
+ }
198
+ let restored = 0;
199
+ for (const [uid, b] of best) {
200
+ const root = tenantRoot(dataDir, uid);
201
+ try {
202
+ let missing = false;
203
+ try {
204
+ const st = await fs.promises.lstat(root);
205
+ missing = st.isDirectory() && (await fs.promises.readdir(root)).length === 0;
206
+ if (missing)
207
+ await fs.promises.rmdir(root);
208
+ }
209
+ catch (err) {
210
+ if (code(err) === "ENOENT")
211
+ missing = true;
212
+ else
213
+ throw err;
214
+ }
215
+ if (!missing)
216
+ continue;
217
+ await fs.promises.mkdir(path.dirname(root), { recursive: true, mode: 0o700 });
218
+ await fs.promises.rename(path.join(trash, b.name), root);
219
+ restored++;
220
+ log(`[connect] import recovery: restored the memory of uid=${uid} from trash (${b.name}); a replace had been interrupted`);
221
+ }
222
+ catch (err) {
223
+ log(`[connect] import recovery failed uid=${uid} code=${code(err) ?? "-"}`);
224
+ }
225
+ }
226
+ return restored;
227
+ }
228
+ export class ImportSession {
229
+ o;
230
+ report;
231
+ stagingDir = "";
232
+ stagingReal = "";
233
+ tar;
234
+ raw = 0;
235
+ rawCap;
236
+ cost = 0;
237
+ dirCount = 0;
238
+ layout = "unknown";
239
+ wrapper;
240
+ canon = new Map();
241
+ staged = [];
242
+ projects = new Set();
243
+ cur = null;
244
+ constructor(o) {
245
+ this.o = o;
246
+ this.report = { mode: o.mode, imported: 0, skippedExisting: 0, refused: { path: 0, type: 0, size: 0, special: 0, duplicate: 0 }, rulesSkipped: 0, projects: [], alwaysFollow: 0, bytes: 0, samples: [] };
247
+ this.rawCap = o.budgetBytes + RAW_SLACK;
248
+ this.tar = new TarReader({
249
+ entry: (e) => this.onEntry(e),
250
+ data: (c) => this.onData(c),
251
+ fileEnd: () => this.onFileEnd(),
252
+ }, { maxEntries: o.limits.maxEntries });
253
+ }
254
+ async begin() {
255
+ const { staging } = dirs(this.o.dataDir);
256
+ await fs.promises.mkdir(staging, { recursive: true, mode: 0o700 });
257
+ this.stagingDir = path.join(staging, `${this.o.uid}-${randomBytes(8).toString("hex")}`);
258
+ await fs.promises.mkdir(this.stagingDir, { mode: 0o700 });
259
+ this.stagingReal = await fs.promises.realpath(this.stagingDir);
260
+ }
261
+ get stagingPath() { return this.stagingDir; }
262
+ get rawBytes() { return this.raw; }
263
+ /** Feeds archive bytes. Throws ImportAbort / TarError / fs errors; the caller must cleanup(). */
264
+ async feed(chunk) {
265
+ this.raw += chunk.length;
266
+ if (this.raw > this.rawCap)
267
+ throw new ImportAbort(413, "Archive too large", "This archive is larger than your memory can take. Pack only the memory files with the command on the connect page, or free some space first.");
268
+ await this.tar.write(chunk);
269
+ }
270
+ /** The upload is complete. */
271
+ endOfArchive() {
272
+ this.tar.end();
273
+ if (this.cur)
274
+ throw new TarError("the archive ended inside a file");
275
+ }
276
+ refuse(kind, name) {
277
+ this.report.refused[kind]++;
278
+ if (this.report.samples.length < 8)
279
+ this.report.samples.push(printable(name));
280
+ }
281
+ async onEntry(e) {
282
+ if (e.kind === "special") {
283
+ this.refuse("special", e.name);
284
+ return false;
285
+ }
286
+ const v = classifyPath(e.name, e.kind === "dir", this.layout === "wrapped" ? this.wrapper : undefined);
287
+ if (!v.ok) {
288
+ this.refuse(v.why, e.name);
289
+ return false;
290
+ }
291
+ // One archive, one layout: every entry wrapped in the download's top directory, or none.
292
+ const wrapped = v.wrapperSeen !== undefined;
293
+ if (this.layout === "unknown") {
294
+ this.layout = wrapped ? "wrapped" : "plain";
295
+ this.wrapper = v.wrapperSeen;
296
+ }
297
+ else if (this.layout === "plain" && wrapped) {
298
+ this.refuse("path", e.name);
299
+ return false;
300
+ }
301
+ if (e.kind === "dir")
302
+ return false;
303
+ const segs = v.segs;
304
+ if (v.rule && !this.o.rules) {
305
+ this.report.rulesSkipped++;
306
+ return false;
307
+ }
308
+ // Case-fold and file/directory collisions (the archive must also unpack on a case-insensitive disk).
309
+ const folded = segs.map((s) => s.normalize("NFKC").toLowerCase());
310
+ for (let i = 0; i < segs.length; i++) {
311
+ const prev = this.canon.get(folded.slice(0, i + 1).join("/"));
312
+ const last = i === segs.length - 1;
313
+ if (prev && (prev.actual !== segs.slice(0, i + 1).join("/") || last || prev.isFile)) {
314
+ this.refuse("duplicate", e.name);
315
+ return false;
316
+ }
317
+ }
318
+ if (e.size > this.o.limits.maxFileBytes) {
319
+ this.refuse("size", e.name);
320
+ return false;
321
+ }
322
+ let newDirs = 0;
323
+ for (let i = 0; i < segs.length - 1; i++)
324
+ if (!this.canon.has(folded.slice(0, i + 1).join("/")))
325
+ newDirs++;
326
+ if (this.dirCount + newDirs > MAX_DIRS)
327
+ throw new ImportAbort(413, "Too many folders", "This archive would create too many folders, so nothing was imported.");
328
+ const cost = blocks(e.size) + newDirs * DISK_BLOCK;
329
+ if (this.cost + cost > this.o.budgetBytes) {
330
+ throw new ImportAbort(413, "Not enough space", "This memory does not fit in your storage quota, so nothing was imported. Free some space, or use \"replace\" if you want the archive to take the place of what is there.");
331
+ }
332
+ const dest = path.join(this.stagingDir, ...segs);
333
+ await this.o.hooks?.beforeFile?.(dest);
334
+ const parent = path.dirname(dest);
335
+ await fs.promises.mkdir(parent, { recursive: true, mode: 0o700 });
336
+ await this.checkStagingParent(parent);
337
+ await this.o.hooks?.beforeOpen?.(dest);
338
+ let fh;
339
+ try {
340
+ fh = await fs.promises.open(dest, fs.constants.O_WRONLY | fs.constants.O_CREAT | fs.constants.O_EXCL | NOFOLLOW, 0o600);
341
+ }
342
+ catch (err) {
343
+ if (code(err) === "EEXIST" || code(err) === "EISDIR" || code(err) === "ENOTDIR") {
344
+ this.refuse("duplicate", e.name);
345
+ return false;
346
+ }
347
+ throw err;
348
+ }
349
+ try {
350
+ // Opened through a parent that was swapped for a link? Then the file landed elsewhere: take it back and stop.
351
+ const real = await fs.promises.realpath(dest);
352
+ if (!inside(real, this.stagingReal)) {
353
+ await fh.close().catch(() => undefined);
354
+ await fs.promises.unlink(real).catch(() => undefined);
355
+ throw new ImportAbort(400, "Refused", "The archive could not be unpacked safely, so nothing was imported.");
356
+ }
357
+ }
358
+ catch (err) {
359
+ await fh.close().catch(() => undefined);
360
+ throw err;
361
+ }
362
+ for (let i = 0; i < segs.length; i++)
363
+ this.canon.set(folded.slice(0, i + 1).join("/"), { actual: segs.slice(0, i + 1).join("/"), isFile: i === segs.length - 1 });
364
+ this.cost += cost;
365
+ this.dirCount += newDirs;
366
+ this.cur = { fh, size: e.size, left: e.size };
367
+ this.staged.push(segs.join("/"));
368
+ if (segs[0] === "projects")
369
+ this.projects.add(segs[1]);
370
+ return true;
371
+ }
372
+ async checkStagingParent(parent) {
373
+ let ok = false;
374
+ try {
375
+ ok = (await fs.promises.lstat(parent)).isDirectory() && inside(await fs.promises.realpath(parent), this.stagingReal);
376
+ }
377
+ catch { /* not ok */ }
378
+ if (!ok)
379
+ throw new ImportAbort(400, "Refused", "The archive could not be unpacked safely, so nothing was imported.");
380
+ }
381
+ async onData(chunk) {
382
+ const c = this.cur;
383
+ if (!c)
384
+ return;
385
+ let off = 0;
386
+ while (off < chunk.length) {
387
+ const { bytesWritten } = await c.fh.write(chunk, off, chunk.length - off);
388
+ off += bytesWritten;
389
+ }
390
+ c.left -= chunk.length;
391
+ }
392
+ async onFileEnd() {
393
+ const c = this.cur;
394
+ this.cur = null;
395
+ if (!c)
396
+ return;
397
+ await c.fh.close();
398
+ this.report.imported++;
399
+ this.report.bytes += c.size;
400
+ }
401
+ /** Remove the staging directory and any open file. Safe to call twice; never throws. */
402
+ async cleanup() {
403
+ const c = this.cur;
404
+ this.cur = null;
405
+ await c?.fh.close().catch(() => undefined);
406
+ if (this.stagingDir)
407
+ await fs.promises.rm(this.stagingDir, { recursive: true, force: true }).catch(() => undefined);
408
+ }
409
+ // ---- commit ------------------------------------------------------------------------
410
+ async commit() {
411
+ if (this.staged.length === 0) {
412
+ throw new ImportAbort(422, "Nothing to import", "No file in this archive is part of an AgentRecall memory (projects/<name>/… and the awareness, identity and insight files), so nothing was changed.");
413
+ }
414
+ this.report.projects = [...this.projects].sort();
415
+ if (this.o.mode === "replace")
416
+ await this.commitReplace();
417
+ else
418
+ await this.commitAdd();
419
+ this.report.alwaysFollow = await countAlwaysFollow(tenantRoot(this.o.dataDir, this.o.uid));
420
+ return this.report;
421
+ }
422
+ async commitReplace() {
423
+ const root = tenantRoot(this.o.dataDir, this.o.uid);
424
+ const { trash } = dirs(this.o.dataDir);
425
+ await fs.promises.mkdir(trash, { recursive: true, mode: 0o700 });
426
+ await fs.promises.mkdir(path.dirname(root), { recursive: true, mode: 0o700 });
427
+ const stamp = isoName(this.o.now());
428
+ const old = path.join(trash, `${this.o.uid}-${stamp}`);
429
+ let moved = false;
430
+ try {
431
+ await fs.promises.rename(root, old);
432
+ moved = true;
433
+ }
434
+ catch (err) {
435
+ if (code(err) !== "ENOENT")
436
+ throw err; // no memory yet: nothing to keep
437
+ }
438
+ try {
439
+ await fs.promises.rename(this.stagingDir, root);
440
+ }
441
+ catch (err) {
442
+ // An agent request may have recreated the (empty) tenant directory in between: set it aside too and retry once.
443
+ if (code(err) === "ENOTEMPTY" || code(err) === "EEXIST" || code(err) === "EPERM") {
444
+ try {
445
+ await fs.promises.rename(root, path.join(trash, `${this.o.uid}-${stamp}-late`));
446
+ await fs.promises.rename(this.stagingDir, root);
447
+ this.stagingDir = "";
448
+ return;
449
+ }
450
+ catch { /* fall through to restore */ }
451
+ }
452
+ if (moved) {
453
+ try {
454
+ await fs.promises.rename(old, root);
455
+ }
456
+ catch {
457
+ throw new ImportAbort(500, "Import failed", "The import failed and your previous memory could not be put back automatically. It is safe in the server's trash for 7 days: ask the server operator to restore it.");
458
+ }
459
+ }
460
+ throw err;
461
+ }
462
+ this.stagingDir = ""; // it is the tenant now: cleanup() must not remove it
463
+ // Keep only the most recent replaced memory per account in trash (the trash is not charged to the quota).
464
+ const mine = new RegExp(`^${this.o.uid}-\\d{4}-\\d{2}-\\d{2}T\\d{2}-\\d{2}-\\d{2}-\\d{3}Z(?:-late)?$`);
465
+ try {
466
+ for (const n of await fs.promises.readdir(trash)) {
467
+ if (mine.test(n) && !n.startsWith(`${this.o.uid}-${stamp}`))
468
+ await fs.promises.rm(path.join(trash, n), { recursive: true, force: true }).catch(() => undefined);
469
+ }
470
+ }
471
+ catch { /* the sweep will get to it */ }
472
+ await fs.promises.chmod(root, 0o700).catch(() => undefined);
473
+ }
474
+ async commitAdd() {
475
+ const root = tenantRoot(this.o.dataDir, this.o.uid);
476
+ await fs.promises.mkdir(root, { recursive: true, mode: 0o700 });
477
+ if (!(await fs.promises.lstat(root)).isDirectory())
478
+ throw new ImportAbort(500, "Could not import", "Your memory directory could not be written. Nothing was changed.");
479
+ const rootReal = await fs.promises.realpath(root);
480
+ let imported = 0;
481
+ for (const rel of [...this.staged].sort()) {
482
+ const src = path.join(this.stagingDir, ...rel.split("/"));
483
+ const segs = rel.split("/");
484
+ let cur = root;
485
+ let ok = true;
486
+ for (const s of segs.slice(0, -1)) {
487
+ cur = path.join(cur, s);
488
+ try {
489
+ await fs.promises.mkdir(cur, { mode: 0o700 });
490
+ }
491
+ catch (err) {
492
+ if (code(err) !== "EEXIST")
493
+ throw err;
494
+ }
495
+ // A real directory only, and still inside the tenant (a link planted here is never followed).
496
+ let good = false;
497
+ try {
498
+ good = (await fs.promises.lstat(cur)).isDirectory() && inside(await fs.promises.realpath(cur), rootReal);
499
+ }
500
+ catch { /* no */ }
501
+ if (!good) {
502
+ ok = false;
503
+ break;
504
+ }
505
+ }
506
+ if (!ok) {
507
+ this.report.skippedExisting++;
508
+ continue;
509
+ }
510
+ const dest = path.join(root, ...segs);
511
+ await this.o.hooks?.beforeCommitFile?.(dest);
512
+ let placed = false;
513
+ try {
514
+ await fs.promises.link(src, dest); // fails with EEXIST instead of overwriting
515
+ placed = true;
516
+ }
517
+ catch (err) {
518
+ if (code(err) === "EEXIST") {
519
+ this.report.skippedExisting++;
520
+ continue;
521
+ }
522
+ try {
523
+ await fs.promises.copyFile(src, dest, fs.constants.COPYFILE_EXCL);
524
+ placed = true;
525
+ }
526
+ catch (err2) {
527
+ if (code(err2) === "EEXIST") {
528
+ this.report.skippedExisting++;
529
+ continue;
530
+ }
531
+ throw err2;
532
+ }
533
+ }
534
+ if (placed) {
535
+ let real = "";
536
+ try {
537
+ real = await fs.promises.realpath(dest);
538
+ }
539
+ catch { /* vanished */ }
540
+ if (real === "" || !inside(real, rootReal)) {
541
+ if (real !== "")
542
+ await fs.promises.unlink(real).catch(() => undefined);
543
+ throw new ImportAbort(400, "Refused", "Your memory directory changed while importing, so the import was stopped. Files already added are kept; nothing was overwritten.");
544
+ }
545
+ imported++;
546
+ }
547
+ }
548
+ this.report.imported = imported;
549
+ }
550
+ }
551
+ /** Active always-follow (p0) corrections in a tenant: projects/<slug>/corrections/*.json, not `_`-prefixed. Bounded. */
552
+ export async function countAlwaysFollow(root) {
553
+ let n = 0;
554
+ let scanned = 0;
555
+ let projects;
556
+ try {
557
+ projects = await fs.promises.readdir(path.join(root, "projects"));
558
+ }
559
+ catch {
560
+ return 0;
561
+ }
562
+ for (const p of projects) {
563
+ const dir = path.join(root, "projects", p, "corrections");
564
+ let files;
565
+ try {
566
+ files = await fs.promises.readdir(dir);
567
+ }
568
+ catch {
569
+ continue;
570
+ }
571
+ for (const f of files) {
572
+ if (!f.endsWith(".json") || f.startsWith("_"))
573
+ continue;
574
+ if (++scanned > 5000)
575
+ return n;
576
+ try {
577
+ const file = path.join(dir, f);
578
+ const st = await fs.promises.lstat(file);
579
+ if (!st.isFile() || st.size > 1024 * 1024)
580
+ continue;
581
+ const rec = JSON.parse(decodeTextFile(await fs.promises.readFile(file)));
582
+ if (rec.severity === "p0" && rec.active !== false)
583
+ n++;
584
+ }
585
+ catch { /* unreadable or not a record */ }
586
+ }
587
+ }
588
+ return n;
589
+ }
590
+ const FIELDS = new Set(["csrf", "mode", "confirm", "rules_seen", "rules"]);
591
+ const MAX_FIELD = 256;
592
+ /**
593
+ * Reads the multipart body of POST /connect/import, stages and commits it.
594
+ * Rejects with ImportAbort (show the user) or another error (log it; say "failed").
595
+ * Always removes its staging directory and releases the per-user lock.
596
+ */
597
+ export async function receiveImport(req, o) {
598
+ const fields = {};
599
+ let session = null;
600
+ let acquired = false;
601
+ let field = null;
602
+ let sawArchive = false;
603
+ let archiveDone = false;
604
+ let monitor;
605
+ const parser = new MultipartParser(o.boundary, {
606
+ async start(part) {
607
+ if (archiveDone || (sawArchive))
608
+ throw new ImportAbort(400, "Not valid", "The form had unexpected extra parts.");
609
+ if (part.name === "archive") {
610
+ if (part.filename === undefined)
611
+ throw new ImportAbort(400, "Not valid", "The archive must be sent as a file.");
612
+ const verdict = o.check(fields); // CSRF, mode, confirmation: before a byte is staged
613
+ sawArchive = true;
614
+ const denied = o.gate.tryAcquire(o.uid, o.ip);
615
+ if (denied === "busy")
616
+ throw new ImportAbort(409, "Already importing", "An import is already running for your account. Wait for it to finish, then try again.");
617
+ if (denied === "ip")
618
+ throw new ImportAbort(409, "Already importing", "Another import from your network address is running. Wait for it to finish, then try again.");
619
+ if (denied === "full")
620
+ throw new ImportAbort(503, "Server busy", "The server is busy importing for other users, try again in a minute.", 60);
621
+ if (denied === "rate")
622
+ throw new ImportAbort(429, "Slow down", `At most ${IMPORTS_PER_HOUR} imports per hour are allowed. Please try again later.`);
623
+ acquired = true;
624
+ await restoreOrphans(o.dataDir, o.now(), o.log, o.uid); // a replace that died between its two renames
625
+ const root = tenantRoot(o.dataDir, o.uid);
626
+ const used = verdict.mode === "add" && fs.existsSync(root) ? await dirSize(root) : 0;
627
+ const quota = o.quotaBytes > 0 ? o.quotaBytes : Infinity;
628
+ const budget = Math.min(quota - used, HARD_IMPORT_CAP);
629
+ if (!(budget > 0))
630
+ throw new ImportAbort(413, "Storage full", "Your storage is full, so nothing was imported. Free some space first, or use \"replace\".");
631
+ session = new ImportSession({ dataDir: o.dataDir, uid: o.uid, mode: verdict.mode, rules: verdict.rules, budgetBytes: budget, limits: o.limits, hooks: o.hooks, now: o.now });
632
+ await session.begin();
633
+ // From here on the archive must keep flowing: a trickle would hold a slot for everyone else.
634
+ const s = session;
635
+ let last = 0;
636
+ monitor = setInterval(() => {
637
+ const got = s.rawBytes - last;
638
+ last = s.rawBytes;
639
+ if (got < (o.limits.minBytesPerSec * o.limits.windowMs) / 1000) {
640
+ req.destroy(new ImportAbort(408, "Upload too slow", "The upload is too slow (it must average at least 16 KiB per second), so nothing was imported."));
641
+ }
642
+ }, o.limits.windowMs);
643
+ return;
644
+ }
645
+ if (!FIELDS.has(part.name) || part.filename !== undefined || part.name in fields)
646
+ throw new ImportAbort(400, "Not valid", "The form had unexpected fields.");
647
+ field = { name: part.name, chunks: [], len: 0 };
648
+ },
649
+ async data(chunk) {
650
+ if (field) {
651
+ field.len += chunk.length;
652
+ if (field.len > MAX_FIELD)
653
+ throw new ImportAbort(400, "Not valid", "A form field was too long.");
654
+ field.chunks.push(Buffer.from(chunk));
655
+ }
656
+ else if (session) {
657
+ await session.feed(chunk);
658
+ }
659
+ },
660
+ async end() {
661
+ if (field) {
662
+ fields[field.name] = Buffer.concat(field.chunks).toString("utf8");
663
+ field = null;
664
+ return;
665
+ }
666
+ if (session) {
667
+ if (monitor)
668
+ clearInterval(monitor);
669
+ session.endOfArchive();
670
+ archiveDone = true;
671
+ }
672
+ },
673
+ });
674
+ let idle;
675
+ const arm = () => {
676
+ if (idle)
677
+ clearTimeout(idle);
678
+ idle = setTimeout(() => req.destroy(new ImportAbort(408, "Upload stalled", "The upload stopped sending data, so nothing was imported.")), o.limits.idleMs);
679
+ };
680
+ const total = setTimeout(() => req.destroy(new ImportAbort(408, "Upload too slow", "The upload took too long, so nothing was imported.")), o.limits.totalMs);
681
+ try {
682
+ arm();
683
+ for await (const chunk of req.iterator({ destroyOnReturn: false })) {
684
+ if (idle)
685
+ clearTimeout(idle);
686
+ await parser.write(chunk);
687
+ arm();
688
+ }
689
+ // The upload is over: the stall and total-time limits must not cut the commit short.
690
+ if (idle)
691
+ clearTimeout(idle);
692
+ clearTimeout(total);
693
+ parser.finish();
694
+ if (!session || !archiveDone)
695
+ throw new ImportAbort(400, "No file", "No archive was received. Choose the .tar file and try again.");
696
+ return await session.commit();
697
+ }
698
+ catch (err) {
699
+ if (err instanceof MultipartError || err instanceof TarError)
700
+ throw new ImportAbort(400, "Not a valid archive", `${err.message}. Nothing was imported.`);
701
+ throw err;
702
+ }
703
+ finally {
704
+ if (idle)
705
+ clearTimeout(idle);
706
+ if (monitor)
707
+ clearInterval(monitor);
708
+ clearTimeout(total);
709
+ try {
710
+ await session?.cleanup();
711
+ }
712
+ finally {
713
+ if (acquired)
714
+ o.gate.release(o.uid, o.ip);
715
+ }
716
+ }
717
+ }
718
+ //# sourceMappingURL=import-memory.js.map