meta-sovereign 0.49.4 → 0.49.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (51) hide show
  1. package/js/src/cli/index.js +2 -1
  2. package/js/src/server/audience-sync.js +230 -0
  3. package/js/src/server/background-jobs.js +91 -0
  4. package/js/src/server/backup-job.js +91 -0
  5. package/js/src/server/chat-index.js +13 -3
  6. package/js/src/server/index.js +14 -8
  7. package/js/src/server/routes-deleted.js +108 -0
  8. package/js/src/server/routes-messages.js +5 -1
  9. package/js/src/server/routes-mutating.js +2 -1
  10. package/js/src/sources/archive/context.js +99 -0
  11. package/js/src/sources/archive/legacy.js +64 -0
  12. package/js/src/sources/archive/media-store.js +60 -0
  13. package/js/src/sources/archive/meta-thread.js +278 -0
  14. package/js/src/sources/archive/text.js +207 -0
  15. package/js/src/sources/archive/x-dms.js +196 -0
  16. package/js/src/sources/archive/zip-file.js +200 -0
  17. package/js/src/sources/auto-import/files.js +146 -43
  18. package/js/src/sources/auto-import/manager.js +32 -5
  19. package/js/src/sources/auto-import/recipes.js +6 -29
  20. package/js/src/sources/facebook.js +32 -23
  21. package/js/src/sources/link.js +7 -0
  22. package/js/src/sources/linkedin.js +68 -17
  23. package/js/src/sources/peopleperhour.js +5 -21
  24. package/js/src/sources/soft-cache.js +25 -0
  25. package/js/src/sources/telegram-account-live.js +11 -4
  26. package/js/src/sources/telegram-account.js +23 -3
  27. package/js/src/sources/telegram-actions.js +6 -2
  28. package/js/src/sources/telegram-catchup.js +50 -13
  29. package/js/src/sources/telegram-chat-removal.js +5 -1
  30. package/js/src/sources/telegram-ephemeral.js +5 -1
  31. package/js/src/sources/upwork.js +5 -21
  32. package/js/src/sources/vk.js +31 -4
  33. package/js/src/sources/x.js +9 -14
  34. package/js/src/storage/backup.js +21 -2
  35. package/js/src/storage/index.js +2 -0
  36. package/js/src/storage/migrations/0005-archive-ids.js +162 -0
  37. package/js/src/storage/migrations/0006-tombstones.js +86 -0
  38. package/js/src/storage/migrations/index.js +11 -2
  39. package/js/src/storage/soft-delete.js +31 -5
  40. package/js/src/web/app.css +55 -0
  41. package/js/src/web/automations-state.js +21 -0
  42. package/js/src/web/automations-view.js +46 -0
  43. package/js/src/web/chat-deleted.js +241 -0
  44. package/js/src/web/chat-view.js +7 -0
  45. package/js/src/web/dom.js +18 -0
  46. package/js/src/web/locales/en-chat-account.js +23 -0
  47. package/js/src/web/locales/hi-chat-account.js +21 -0
  48. package/js/src/web/locales/ru-chat-account.js +23 -0
  49. package/js/src/web/locales/zh-chat-account.js +21 -0
  50. package/js/src/web/tutorial.js +25 -5
  51. package/package.json +5 -1
@@ -36,6 +36,7 @@ import {
36
36
  openDualStore,
37
37
  bulkPurge,
38
38
  isTombstone,
39
+ deletedAtOf,
39
40
  createVault,
40
41
  } from '../storage/index.js';
41
42
  import {
@@ -702,7 +703,7 @@ const purgeTombstonesCmd = async (args, log) => {
702
703
  if (idPrefix && !link.id?.startsWith(idPrefix)) {
703
704
  return false;
704
705
  }
705
- if (olderThan && (link.deleted?.at ?? '') > olderThan) {
706
+ if (olderThan && (deletedAtOf(link) ?? '') > olderThan) {
706
707
  return false;
707
708
  }
708
709
  return true;
@@ -0,0 +1,230 @@
1
+ /**
2
+ * Audience membership sync (R-X13; wired by R-Z7).
3
+ *
4
+ * An audience such as `friends:vk & chat:vk:2000000001` only finds people
5
+ * when their `member:` links are in the store. This reads, through each
6
+ * network's official API and the user's own token, the friends, chat members
7
+ * and community members that the saved audiences and the automations name,
8
+ * and stores them (`syncMembership`). It only reads; nothing is sent.
9
+ *
10
+ * POST /api/audience/sync { source? } -> { ok, sources: [{ source, ok,
11
+ * groups, members, error? }] } now, for one network or every one
12
+ * the audiences name
13
+ *
14
+ * It also runs by itself once a day (`MS_AUDIENCE_SYNC_HOURS`, 0 = off,
15
+ * default 24) while the server is up. A network without a token, or one
16
+ * that cannot list a kind of group (Telegram's Bot API has no friends and
17
+ * lists only a chat's administrators), reports that and the rest goes on.
18
+ */
19
+
20
+ import { clearInterval, setInterval } from 'node:timers';
21
+ import { fail, json, readBody } from './util.js';
22
+ import { isTrustedMutation } from './routes-auto-import.js';
23
+ import { safeMessage } from './redact.js';
24
+ import { toAudienceSpec } from '../crm/people.js';
25
+ import {
26
+ getSource,
27
+ liveOptionsFrom,
28
+ syncMembership,
29
+ } from '../sources/index.js';
30
+
31
+ const HOUR_MS = 3_600_000;
32
+ const SOURCE_NAME = /^[a-z][a-z0-9-]{0,30}$/;
33
+
34
+ // The groups a spec names: `friends:vk`, `chat:vk:<id>`, `community:vk:<id>`,
35
+ // through saved audiences too (each followed once).
36
+ const collect = (spec, byId, found, seen = new Set()) => {
37
+ if (!spec || typeof spec !== 'object') {
38
+ return;
39
+ }
40
+ if (Array.isArray(spec.of)) {
41
+ spec.of.forEach((child) => collect(child, byId, found, seen));
42
+ } else if (spec.kind === 'friends' && spec.source) {
43
+ found.set(`${spec.source}\0friends`, [spec.source, { kind: 'friends' }]);
44
+ } else if (['chat', 'community'].includes(spec.kind) && spec.source) {
45
+ const id = String(spec[spec.kind]);
46
+ found.set(`${spec.source}\0${spec.kind}\0${id}`, [
47
+ spec.source,
48
+ { kind: spec.kind, id },
49
+ ]);
50
+ } else if (spec.kind === 'saved' && !seen.has(spec.id)) {
51
+ seen.add(spec.id);
52
+ const link = byId.get(`audience:${spec.id}`);
53
+ safeSpec(link?.spec ?? link?.text, (inner) =>
54
+ collect(inner, byId, found, seen)
55
+ );
56
+ }
57
+ };
58
+
59
+ const safeSpec = (value, use) => {
60
+ try {
61
+ use(toAudienceSpec(value));
62
+ } catch {
63
+ // An audience that does not parse names no group.
64
+ }
65
+ };
66
+
67
+ /** `Map<source, group[]>` named by the saved audiences and automations. */
68
+ export const audienceGroups = (links) => {
69
+ const byId = new Map(links.map((l) => [l?.id, l]));
70
+ const found = new Map();
71
+ for (const link of links) {
72
+ const id = String(link?.id ?? '');
73
+ if (id.startsWith('audience:')) {
74
+ safeSpec(link.spec ?? link.text, (s) => collect(s, byId, found));
75
+ } else if (id.startsWith('automation:') && link.audience) {
76
+ safeSpec(link.audience, (s) => collect(s, byId, found));
77
+ }
78
+ }
79
+ const out = new Map();
80
+ for (const [source, group] of found.values()) {
81
+ out.set(source, [...(out.get(source) ?? []), group]);
82
+ }
83
+ return out;
84
+ };
85
+
86
+ // The network's live API and the user's saved options for it, or an error.
87
+ const liveFor = async (store, source, resolve) => {
88
+ try {
89
+ return {
90
+ live: resolve(source).live,
91
+ options: await liveOptionsFrom(store, source),
92
+ };
93
+ } catch (error) {
94
+ return { error: safeMessage(error) };
95
+ }
96
+ };
97
+
98
+ // One group at a time, so a group the network refuses does not stop the rest.
99
+ const syncSource = async (store, source, groups, resolve, now) => {
100
+ const result = { source, ok: true, groups: 0, members: 0 };
101
+ const prepared = await liveFor(store, source, resolve);
102
+ if (prepared.error) {
103
+ return { ...result, ok: false, error: prepared.error };
104
+ }
105
+ const { live, options } = prepared;
106
+ if (!options.token) {
107
+ return { ...result, ok: false, error: 'no token saved for this network' };
108
+ }
109
+ const errors = [];
110
+ for (const group of groups) {
111
+ try {
112
+ const done = await syncMembership({
113
+ source,
114
+ live,
115
+ groups: [group],
116
+ put: (link) => store.put(link),
117
+ now,
118
+ options,
119
+ });
120
+ result.groups += done.groups.length;
121
+ result.members += done.members;
122
+ } catch (error) {
123
+ // The token itself never appears in what is reported.
124
+ errors.push(safeMessage(error).split(options.token).join('***'));
125
+ }
126
+ }
127
+ return errors.length > 0
128
+ ? { ...result, ok: false, error: errors.join('; ').slice(0, 300) }
129
+ : result;
130
+ };
131
+
132
+ /** Sync every network the audiences name (or just `only`); never throws. */
133
+ export const syncAudiences = async ({
134
+ store,
135
+ resolve = getSource,
136
+ only = null,
137
+ now = Date.now(),
138
+ }) => {
139
+ const wanted = audienceGroups(await store.query());
140
+ const sources = [];
141
+ for (const [source, groups] of wanted) {
142
+ if (!only || only === source) {
143
+ sources.push(await syncSource(store, source, groups, resolve, now));
144
+ }
145
+ }
146
+ return { ok: sources.every((s) => s.ok), sources };
147
+ };
148
+
149
+ const DEFAULT_HOURS = 24;
150
+
151
+ // `MS_AUDIENCE_SYNC_HOURS`: hours between runs, `0` = off, unset = daily.
152
+ // Nonsense is ignored for the default with one log line.
153
+ const audienceHours = (env, log) => {
154
+ const raw = env?.MS_AUDIENCE_SYNC_HOURS;
155
+ if (raw === undefined || raw === null || raw === '') {
156
+ return DEFAULT_HOURS;
157
+ }
158
+ const n = Number(raw);
159
+ if (Number.isFinite(n) && n >= 0) {
160
+ return n;
161
+ }
162
+ log?.({
163
+ level: 'warn',
164
+ event: 'audience_sync_setting_ignored',
165
+ name: 'MS_AUDIENCE_SYNC_HOURS',
166
+ });
167
+ return DEFAULT_HOURS;
168
+ };
169
+
170
+ /** Once a day, once the port is ours; `{ stopAudienceSync }` for the context. */
171
+ export const audienceSyncFor = ({
172
+ store,
173
+ log,
174
+ later,
175
+ resolve = getSource,
176
+ env = process.env,
177
+ }) => {
178
+ const hours = audienceHours(env, log);
179
+ let timer = null;
180
+ if (hours > 0) {
181
+ later.push(() => {
182
+ timer = setInterval(
183
+ () =>
184
+ syncAudiences({ store, resolve })
185
+ .then((done) =>
186
+ log?.({
187
+ level: 'info',
188
+ event: 'audience_sync',
189
+ sources: done.sources.length,
190
+ ok: done.ok,
191
+ })
192
+ )
193
+ .catch((error) =>
194
+ log?.({
195
+ level: 'warn',
196
+ event: 'audience_sync_failed',
197
+ error: safeMessage(error),
198
+ })
199
+ ),
200
+ hours * HOUR_MS
201
+ );
202
+ timer.unref?.();
203
+ });
204
+ }
205
+ return { stopAudienceSync: () => clearInterval(timer) };
206
+ };
207
+
208
+ export const handleAudienceSyncRoute = async (store, req, res, p, ctx) => {
209
+ if (p !== '/api/audience/sync' || req.method !== 'POST') {
210
+ return false;
211
+ }
212
+ if (!isTrustedMutation(req, ctx.allowOrigins)) {
213
+ return (
214
+ fail(res, 403, 'forbidden', 'must come from the meta-sovereign app') ??
215
+ true
216
+ );
217
+ }
218
+ const body = await readBody(req);
219
+ if (body.source !== undefined && !SOURCE_NAME.test(String(body.source))) {
220
+ return (
221
+ fail(res, 400, 'bad-request', 'source must be a network name') ?? true
222
+ );
223
+ }
224
+ const done = await syncAudiences({
225
+ store,
226
+ resolve: ctx.audienceSources ?? getSource,
227
+ only: body.source ?? null,
228
+ });
229
+ return json(res, 200, done) ?? true;
230
+ };
@@ -0,0 +1,91 @@
1
+ /**
2
+ * The jobs that run by themselves while the server is up, started once the
3
+ * port is really ours (R-Z7; a second `serve` on a busy port must not run
4
+ * them on the same store):
5
+ *
6
+ * automatic backup ./backup-job.js daily, keep 7 (R-A4)
7
+ * audience membership sync ./audience-sync.js daily (R-X13)
8
+ * 24-hour cache purge here hourly (R-S12, R-T12)
9
+ *
10
+ * `backgroundJobs(...)` returns the part of the context holding `stopJobs`.
11
+ */
12
+ import {
13
+ clearInterval,
14
+ clearTimeout,
15
+ setInterval,
16
+ setTimeout,
17
+ } from 'node:timers';
18
+ import { backupJobFor } from './backup-job.js';
19
+ import { audienceSyncFor } from './audience-sync.js';
20
+ import { safeMessage } from './redact.js';
21
+ import { softCacheRetention as upworkRetention } from '../sources/upwork.js';
22
+ import { softCacheRetention as pphRetention } from '../sources/peopleperhour.js';
23
+
24
+ const HOUR_MS = 3_600_000;
25
+ // The first purge, a minute after start, so a start-up is not slowed by a
26
+ // scan of the store; `MS_CACHE_PURGE_DELAY_MS` changes it (tests).
27
+ const FIRST_PURGE_MS = 60_000;
28
+
29
+ /**
30
+ * Upwork and PeoplePerHour content pulled live from their APIs is deleted
31
+ * 24 hours after it was pulled; what the user imported from an export stays.
32
+ */
33
+ export const purgeSoftCaches = async (store, now = Date.now()) => {
34
+ const upwork = await upworkRetention(store, { now });
35
+ const peoplePerHour = await pphRetention(store, { now });
36
+ return { upwork: upwork.purged, peopleperhour: peoplePerHour.purged };
37
+ };
38
+
39
+ const cachePurgeFor = ({ store, log, later }) => {
40
+ let timer = null;
41
+ let first = null;
42
+ const run = () =>
43
+ purgeSoftCaches(store)
44
+ .then((purged) =>
45
+ purged.upwork + purged.peopleperhour > 0
46
+ ? log?.({ level: 'info', event: 'soft_cache_purged', ...purged })
47
+ : null
48
+ )
49
+ .catch((error) =>
50
+ log?.({
51
+ level: 'warn',
52
+ event: 'soft_cache_purge_failed',
53
+ error: safeMessage(error),
54
+ })
55
+ );
56
+ later.push(() => {
57
+ const delay = Number(process.env.MS_CACHE_PURGE_DELAY_MS ?? FIRST_PURGE_MS);
58
+ first = setTimeout(run, delay);
59
+ timer = setInterval(run, HOUR_MS);
60
+ first.unref?.();
61
+ timer.unref?.();
62
+ });
63
+ return {
64
+ stopCachePurge: () => {
65
+ clearTimeout(first);
66
+ clearInterval(timer);
67
+ },
68
+ };
69
+ };
70
+
71
+ export const backgroundJobs = ({ ctx, store, log, later, backups }) => {
72
+ const backup = backupJobFor({ ctx, store, log, later, backups });
73
+ const audience = audienceSyncFor({
74
+ store,
75
+ log,
76
+ later,
77
+ resolve: ctx.audienceSources,
78
+ });
79
+ const purge = cachePurgeFor({ store, log, later });
80
+ return {
81
+ stopJobs: () => {
82
+ backup.stopBackups();
83
+ audience.stopAudienceSync();
84
+ purge.stopCachePurge();
85
+ // The other timers and pollers the server started with the port.
86
+ ctx.autoImport?.close();
87
+ ctx.whatsappSignIn?.stop().catch(() => {});
88
+ ctx.stopPulls?.();
89
+ },
90
+ };
91
+ };
@@ -0,0 +1,91 @@
1
+ /**
2
+ * The automatic backup, started once the port is ours (R-A4, R-Z7).
3
+ *
4
+ * OFF by default (the streaming backups of R-Z16 switch it on).
5
+ * `MS_BACKUP_INTERVAL_HOURS` (or the `backups.intervalHours` option) sets
6
+ * the interval in hours, `0` or unset means off; `MS_BACKUP_KEEP` (or
7
+ * `backups.keep`) sets how many archives are kept (default 7, `0` keeps
8
+ * all). A value that is not a number >= 0 is ignored for the default and
9
+ * logged once.
10
+ * The archive is encrypted, and holds the secrets, only when the server has
11
+ * a passphrase (`MS_SECRET_PASSPHRASE`); otherwise it leaves them out.
12
+ * The first run is one interval after start, never at start. A nearly full
13
+ * disk skips the run (R-Z14) and the log says so.
14
+ */
15
+ import { createBackupScheduler } from '../storage/backup.js';
16
+
17
+ const HOUR_MS = 3_600_000;
18
+ export const DEFAULT_INTERVAL_HOURS = 0;
19
+ export const DEFAULT_KEEP = 7;
20
+
21
+ // A number >= 0, or the fallback; `bad` collects the names that were wrong.
22
+ const number = (value, fallback, name, bad) => {
23
+ if (value === undefined || value === null || value === '') {
24
+ return fallback;
25
+ }
26
+ const n = Number(value);
27
+ if (Number.isFinite(n) && n >= 0) {
28
+ return n;
29
+ }
30
+ bad.push(name);
31
+ return fallback;
32
+ };
33
+
34
+ /**
35
+ * `{ intervalMs, keep, invalid }` from options and the environment; an
36
+ * interval of 0 is off; `invalid` names the settings that were ignored.
37
+ */
38
+ export const backupSettings = (options = {}, env = process.env) => {
39
+ const invalid = [];
40
+ const hours = number(
41
+ options.intervalHours ?? env.MS_BACKUP_INTERVAL_HOURS,
42
+ DEFAULT_INTERVAL_HOURS,
43
+ 'MS_BACKUP_INTERVAL_HOURS',
44
+ invalid
45
+ );
46
+ const keep = number(
47
+ options.keep ?? env.MS_BACKUP_KEEP,
48
+ DEFAULT_KEEP,
49
+ 'MS_BACKUP_KEEP',
50
+ invalid
51
+ );
52
+ return {
53
+ intervalMs: Math.round(hours * HOUR_MS),
54
+ keep: Math.floor(keep),
55
+ invalid,
56
+ };
57
+ };
58
+
59
+ /** The context part: `{ stopBackups }`; the timer starts with `later`. */
60
+ export const backupJobFor = ({ ctx, store, log, later, backups }) => {
61
+ const { archiveDir, secretPassphrase: passphrase } = ctx;
62
+ const measureFree = ctx.diskFree;
63
+ const { intervalMs, keep, invalid } = backupSettings(backups);
64
+ if (invalid.length > 0) {
65
+ log?.({ level: 'warn', event: 'backup_setting_ignored', names: invalid });
66
+ }
67
+ let scheduler = null;
68
+ if (archiveDir && intervalMs > 0) {
69
+ later.push(() => {
70
+ scheduler = createBackupScheduler({
71
+ store,
72
+ archiveDir,
73
+ intervalMs,
74
+ keep,
75
+ passphrase,
76
+ measureFree,
77
+ onEvent: (event) => log?.(event),
78
+ onBackup: () => log?.({ level: 'info', event: 'backup_done' }),
79
+ // Never the text of a secret: only the message the backup raised.
80
+ onError: (error) =>
81
+ log?.({
82
+ level: 'warn',
83
+ event: 'backup_failed',
84
+ code: error.code ?? null,
85
+ error: String(error.message).slice(0, 200),
86
+ }),
87
+ });
88
+ });
89
+ }
90
+ return { stopBackups: () => scheduler?.stop() };
91
+ };
@@ -20,6 +20,7 @@
20
20
  */
21
21
 
22
22
  import { chatKey, networkMessageId } from '../web/chat-model.js';
23
+ import { isTombstone } from '../storage/soft-delete.js';
23
24
  import { splitTopicKey, topicOfMessage } from '../web/forum-topics.js';
24
25
 
25
26
  const isMessage = (link) =>
@@ -115,13 +116,14 @@ export const createChatIndex = (store) => {
115
116
  const chatOf = (key) => {
116
117
  let chat = chats.get(key);
117
118
  if (!chat) {
118
- chat = { entries: new Map(), sorted: null, head: null };
119
+ chat = { entries: new Map(), sorted: null, head: null, gone: new Set() };
119
120
  chats.set(key, chat);
120
121
  }
121
122
  return chat;
122
123
  };
123
124
 
124
125
  const remove = (chat, id) => {
126
+ chat.gone.delete(id);
125
127
  if (!chat.entries.delete(id)) {
126
128
  return;
127
129
  }
@@ -135,10 +137,13 @@ export const createChatIndex = (store) => {
135
137
  const add = (link) => {
136
138
  const chat = chatOf(chatKey(link.source, link.chat));
137
139
  const known = chat.entries.has(link.id);
138
- if (link.deleted) {
140
+ if (isTombstone(link)) {
139
141
  remove(chat, link.id);
142
+ // Kept apart for the "deleted messages" view (R-Z7).
143
+ chat.gone.add(link.id);
140
144
  return 'update';
141
145
  }
146
+ chat.gone.delete(link.id);
142
147
  const entry = entryOf(link);
143
148
  chat.entries.set(link.id, entry);
144
149
  chat.sorted = null;
@@ -178,7 +183,7 @@ export const createChatIndex = (store) => {
178
183
 
179
184
  const linksOf = async (entries) =>
180
185
  (await Promise.all(entries.map((e) => store.get(e.id)))).filter(
181
- (link) => link && !link.deleted
186
+ (link) => link && !isTombstone(link)
182
187
  );
183
188
 
184
189
  const sortedOf = (chat) => {
@@ -242,6 +247,11 @@ export const createChatIndex = (store) => {
242
247
  ...(range.at || range.around ? { target } : {}),
243
248
  };
244
249
  },
250
+ /** The ids of a chat's deleted (tombstoned) messages (R-Z7). */
251
+ async deleted(key) {
252
+ await ready;
253
+ return [...(chats.get(key)?.gone ?? [])];
254
+ },
245
255
  /** How many messages of a chat the store holds. */
246
256
  count: (key) => chats.get(key)?.entries.size ?? 0,
247
257
  close: () => unsubscribe(),
@@ -81,6 +81,9 @@ import {
81
81
  import { createDefaultSender } from '../sources/browser-sender.js';
82
82
  import { createAutopilot } from '../automation/autopilot.js';
83
83
  import { recoverOutbox } from '../automation/outbox.js';
84
+ import { backgroundJobs } from './background-jobs.js';
85
+ import { handleDeletedRoutes } from './routes-deleted.js';
86
+ import { handleAudienceSyncRoute } from './audience-sync.js';
84
87
  import { setTypingFor } from '../sources/live-links.js';
85
88
  import { createReplyService } from '../replies/reply-service.js';
86
89
  import { createChatIndex } from './chat-index.js';
@@ -583,6 +586,8 @@ const routeOn = async (store, req, res, ctx, url, p) => {
583
586
  () => handleTelegramRoutes(req, res, p, ctx),
584
587
  () => handleMessageRoutes(store, req, res, p, url, ctx),
585
588
  () => handleEmailGets(store, req, res, p, url, ctx),
589
+ () => handleDeletedRoutes(store, req, res, p, url, ctx),
590
+ () => handleAudienceSyncRoute(store, req, res, p, ctx),
586
591
  () => handleChatRoutes(req, res, p, url, ctx),
587
592
  () => ctx.learned?.(store, req, res, p, url, ctx) ?? false,
588
593
  () => handleDerivedRoutes(store, req, res, p, url, ctx),
@@ -771,16 +776,16 @@ export const startServer = async ({
771
776
  emailLiveFactory = null,
772
777
  emailPoll,
773
778
  cv = {},
774
- // `{ self?, clink? }` for the reply-graph autocomplete (/api/suggest).
779
+ // replies: `{ self?, clink? }` for /api/suggest; automations: `{ send?,
780
+ // setTyping?, resolveAudience?, self?, scheduler?, tickMs? }`.
775
781
  replies,
776
- // `{ send?, setTyping?, resolveAudience?, self?, scheduler?, tickMs? }`.
777
782
  automations,
778
783
  greetings,
779
784
  allowOrigins,
780
785
  autoImportLauncher,
781
- // Tests inject a fake `createSignIns()` (no real browsers are read).
786
+ // Tests inject a fake `createSignIns()` and a fake MTProto (no real browsers
787
+ // or network are used).
782
788
  autoImportSignIns,
783
- // Tests inject a fake MTProto (no real GramJS / network for Telegram login).
784
789
  telegramMtproto,
785
790
  telegramCreds,
786
791
  // A `createTelegramAccount()` (or `false` for none); see telegramAccountFor.
@@ -796,9 +801,11 @@ export const startServer = async ({
796
801
  localUi,
797
802
  // HTTPS on the same port for Safari (./local-tls.js, R-Y64); `serve` sets it.
798
803
  tls,
799
- // Tests: a fake `(dir) => bytes|null` for the free disk space (R-Z14).
804
+ // diskFree: a test fake `(dir) => bytes|null` (R-Z14); backups:
805
+ // `{ intervalHours?, keep? }` for the automatic backup (R-Z7).
800
806
  diskFree,
801
807
  pulls,
808
+ backups,
802
809
  } = {}) => {
803
810
  const { store, resolvedArchiveDir, flush, storeHealth } = await initStore({
804
811
  providedStore,
@@ -858,6 +865,7 @@ export const startServer = async ({
858
865
  chatIndex,
859
866
  ...pullContext(store, log, pulls, later),
860
867
  };
868
+ Object.assign(ctx, backgroundJobs({ ctx, store, log, later, backups }));
861
869
  const server = createHttpServer(store, ctx, log);
862
870
  const sockets = trackSockets(server);
863
871
  const bus = enableHandlers
@@ -904,9 +912,7 @@ export const startServer = async ({
904
912
  ctx.emailPoller.stop();
905
913
  ctx.replies.close().catch(() => {});
906
914
  ctx.closeAutomations().catch(() => {});
907
- ctx.autoImport?.close();
908
- ctx.whatsappSignIn?.stop().catch(() => {});
909
- ctx.stopPulls?.();
915
+ ctx.stopJobs();
910
916
  ctx.chatIndex.close();
911
917
  stopAll(telegramAccount, ctx.telegramAccounts, ctx.telegramExtras);
912
918
  wsHandle?.close();
@@ -0,0 +1,108 @@
1
+ /**
2
+ * The deleted messages of a chat: kept, never lost, and the user's to
3
+ * bring back or to remove (R-K1, R-K2, R-K4; wired to a page by R-Z7).
4
+ * These are local actions only: nothing is sent to any network.
5
+ *
6
+ * GET /api/chat/deleted?key=<chat key> -> { messages } the chat's
7
+ * tombstoned messages, oldest
8
+ * first (at most 500)
9
+ * POST /api/chat/deleted/recover { ids } -> { ok, recovered } the
10
+ * messages show in the chat
11
+ * again
12
+ * POST /api/chat/deleted/purge { ids, confirm: true }
13
+ * -> { ok, purged } erased from
14
+ * this device for good
15
+ *
16
+ * Only a message that is already deleted can be recovered or purged; a live
17
+ * one is never touched, whatever ids a request names.
18
+ */
19
+
20
+ import { fail as failWith, json, readBody } from './util.js';
21
+ import { isTrustedMutation } from './routes-auto-import.js';
22
+ import { isTombstone, unmarkDeleted } from '../storage/soft-delete.js';
23
+ import { byTime } from '../web/chat-model.js';
24
+
25
+ const fail = (res, status, code, error) =>
26
+ failWith(res, status, code, error) ?? true;
27
+
28
+ const PREFIX = '/api/chat/deleted';
29
+ const MAX_LIST = 500;
30
+ const MAX_IDS = 500;
31
+
32
+ const idsOf = (body) =>
33
+ Array.isArray(body.ids) &&
34
+ body.ids.length > 0 &&
35
+ body.ids.length <= MAX_IDS &&
36
+ body.ids.every((id) => typeof id === 'string' && id.startsWith('msg:'))
37
+ ? body.ids
38
+ : null;
39
+
40
+ // The tombstoned message links among `ids`.
41
+ const deletedAmong = async (store, ids) => {
42
+ const found = [];
43
+ for (const id of ids) {
44
+ const link = await store.get(id);
45
+ if (link && isTombstone(link)) {
46
+ found.push(link);
47
+ }
48
+ }
49
+ return found;
50
+ };
51
+
52
+ const list = async (store, res, url, ctx) => {
53
+ const key = url.searchParams.get('key');
54
+ if (!key || !ctx.chatIndex) {
55
+ return fail(res, 400, 'bad-request', 'key is required');
56
+ }
57
+ const ids = await ctx.chatIndex.deleted(key);
58
+ const links = await deletedAmong(store, ids.slice(0, MAX_LIST));
59
+ return json(res, 200, { messages: links.sort(byTime) }) ?? true;
60
+ };
61
+
62
+ const recover = async (store, res, ids) => {
63
+ const links = await deletedAmong(store, ids);
64
+ for (const link of links) {
65
+ await store.put(unmarkDeleted(link));
66
+ }
67
+ return (
68
+ json(res, 200, { ok: true, recovered: links.map((l) => l.id) }) ?? true
69
+ );
70
+ };
71
+
72
+ const purge = async (store, res, ids) => {
73
+ const purged = [];
74
+ for (const link of await deletedAmong(store, ids)) {
75
+ if (await store.delete(link.id)) {
76
+ purged.push(link.id);
77
+ }
78
+ }
79
+ return json(res, 200, { ok: true, purged }) ?? true;
80
+ };
81
+
82
+ export const handleDeletedRoutes = async (store, req, res, p, url, ctx) => {
83
+ if (!p.startsWith(PREFIX)) {
84
+ return false;
85
+ }
86
+ if (p === PREFIX && req.method === 'GET') {
87
+ return list(store, res, url, ctx);
88
+ }
89
+ const action = p.slice(PREFIX.length + 1);
90
+ if (req.method !== 'POST' || !['recover', 'purge'].includes(action)) {
91
+ return false;
92
+ }
93
+ if (!isTrustedMutation(req, ctx.allowOrigins)) {
94
+ return fail(res, 403, 'forbidden', 'must come from the meta-sovereign app');
95
+ }
96
+ const body = await readBody(req, { allowEmpty: false });
97
+ const ids = idsOf(body);
98
+ if (!ids) {
99
+ return fail(res, 400, 'bad-request', 'ids: message ids are needed');
100
+ }
101
+ if (action === 'purge') {
102
+ if (body.confirm !== true) {
103
+ return fail(res, 400, 'confirm-required', 'purging needs confirm: true');
104
+ }
105
+ return purge(store, res, ids);
106
+ }
107
+ return recover(store, res, ids);
108
+ };
@@ -24,6 +24,8 @@ import { json, readBody } from './util.js';
24
24
  import { isTrustedMutation } from './routes-auto-import.js';
25
25
  import { previewFields } from '../sources/telegram-link-preview.js';
26
26
 
27
+ import { markDeleted as tombstone } from '../storage/soft-delete.js';
28
+
27
29
  const PREFIX = '/api/message/';
28
30
  const MAX_IDS = 100;
29
31
 
@@ -102,7 +104,9 @@ const ACTIONS = {
102
104
  }),
103
105
  after: async (store, body, links) => {
104
106
  for (const link of links) {
105
- await store.put({ ...link, deleted: true, deletedAt: now() });
107
+ await store.put(
108
+ tombstone(link, { by: 'user', reason: 'deleted-in-app', at: now() })
109
+ );
106
110
  }
107
111
  },
108
112
  },