meta-sovereign 0.49.4 → 0.49.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/js/src/cli/index.js +2 -1
- package/js/src/server/audience-sync.js +230 -0
- package/js/src/server/background-jobs.js +91 -0
- package/js/src/server/backup-job.js +91 -0
- package/js/src/server/chat-index.js +13 -3
- package/js/src/server/index.js +14 -8
- package/js/src/server/routes-deleted.js +108 -0
- package/js/src/server/routes-messages.js +5 -1
- package/js/src/server/routes-mutating.js +2 -1
- package/js/src/sources/archive/context.js +99 -0
- package/js/src/sources/archive/legacy.js +64 -0
- package/js/src/sources/archive/media-store.js +60 -0
- package/js/src/sources/archive/meta-thread.js +278 -0
- package/js/src/sources/archive/text.js +207 -0
- package/js/src/sources/archive/x-dms.js +196 -0
- package/js/src/sources/archive/zip-file.js +200 -0
- package/js/src/sources/auto-import/files.js +146 -43
- package/js/src/sources/auto-import/manager.js +32 -5
- package/js/src/sources/auto-import/recipes.js +6 -29
- package/js/src/sources/facebook.js +32 -23
- package/js/src/sources/link.js +7 -0
- package/js/src/sources/linkedin.js +68 -17
- package/js/src/sources/peopleperhour.js +5 -21
- package/js/src/sources/soft-cache.js +25 -0
- package/js/src/sources/telegram-account-live.js +11 -4
- package/js/src/sources/telegram-account.js +23 -3
- package/js/src/sources/telegram-actions.js +6 -2
- package/js/src/sources/telegram-catchup.js +50 -13
- package/js/src/sources/telegram-chat-removal.js +5 -1
- package/js/src/sources/telegram-ephemeral.js +5 -1
- package/js/src/sources/upwork.js +5 -21
- package/js/src/sources/vk.js +31 -4
- package/js/src/sources/x.js +9 -14
- package/js/src/storage/backup.js +21 -2
- package/js/src/storage/index.js +2 -0
- package/js/src/storage/migrations/0005-archive-ids.js +162 -0
- package/js/src/storage/migrations/0006-tombstones.js +86 -0
- package/js/src/storage/migrations/index.js +11 -2
- package/js/src/storage/soft-delete.js +31 -5
- package/js/src/web/app.css +55 -0
- package/js/src/web/automations-state.js +21 -0
- package/js/src/web/automations-view.js +46 -0
- package/js/src/web/chat-deleted.js +241 -0
- package/js/src/web/chat-view.js +7 -0
- package/js/src/web/dom.js +18 -0
- package/js/src/web/locales/en-chat-account.js +23 -0
- package/js/src/web/locales/hi-chat-account.js +21 -0
- package/js/src/web/locales/ru-chat-account.js +23 -0
- package/js/src/web/locales/zh-chat-account.js +21 -0
- package/js/src/web/tutorial.js +25 -5
- package/package.json +5 -1
package/js/src/cli/index.js
CHANGED
|
@@ -36,6 +36,7 @@ import {
|
|
|
36
36
|
openDualStore,
|
|
37
37
|
bulkPurge,
|
|
38
38
|
isTombstone,
|
|
39
|
+
deletedAtOf,
|
|
39
40
|
createVault,
|
|
40
41
|
} from '../storage/index.js';
|
|
41
42
|
import {
|
|
@@ -702,7 +703,7 @@ const purgeTombstonesCmd = async (args, log) => {
|
|
|
702
703
|
if (idPrefix && !link.id?.startsWith(idPrefix)) {
|
|
703
704
|
return false;
|
|
704
705
|
}
|
|
705
|
-
if (olderThan && (link
|
|
706
|
+
if (olderThan && (deletedAtOf(link) ?? '') > olderThan) {
|
|
706
707
|
return false;
|
|
707
708
|
}
|
|
708
709
|
return true;
|
|
@@ -0,0 +1,230 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Audience membership sync (R-X13; wired by R-Z7).
|
|
3
|
+
*
|
|
4
|
+
* An audience such as `friends:vk & chat:vk:2000000001` only finds people
|
|
5
|
+
* when their `member:` links are in the store. This reads, through each
|
|
6
|
+
* network's official API and the user's own token, the friends, chat members
|
|
7
|
+
* and community members that the saved audiences and the automations name,
|
|
8
|
+
* and stores them (`syncMembership`). It only reads; nothing is sent.
|
|
9
|
+
*
|
|
10
|
+
* POST /api/audience/sync { source? } -> { ok, sources: [{ source, ok,
|
|
11
|
+
* groups, members, error? }] } now, for one network or every one
|
|
12
|
+
* the audiences name
|
|
13
|
+
*
|
|
14
|
+
* It also runs by itself once a day (`MS_AUDIENCE_SYNC_HOURS`, 0 = off,
|
|
15
|
+
* default 24) while the server is up. A network without a token, or one
|
|
16
|
+
* that cannot list a kind of group (Telegram's Bot API has no friends and
|
|
17
|
+
* lists only a chat's administrators), reports that and the rest goes on.
|
|
18
|
+
*/
|
|
19
|
+
|
|
20
|
+
import { clearInterval, setInterval } from 'node:timers';
|
|
21
|
+
import { fail, json, readBody } from './util.js';
|
|
22
|
+
import { isTrustedMutation } from './routes-auto-import.js';
|
|
23
|
+
import { safeMessage } from './redact.js';
|
|
24
|
+
import { toAudienceSpec } from '../crm/people.js';
|
|
25
|
+
import {
|
|
26
|
+
getSource,
|
|
27
|
+
liveOptionsFrom,
|
|
28
|
+
syncMembership,
|
|
29
|
+
} from '../sources/index.js';
|
|
30
|
+
|
|
31
|
+
const HOUR_MS = 3_600_000;
|
|
32
|
+
const SOURCE_NAME = /^[a-z][a-z0-9-]{0,30}$/;
|
|
33
|
+
|
|
34
|
+
// The groups a spec names: `friends:vk`, `chat:vk:<id>`, `community:vk:<id>`,
|
|
35
|
+
// through saved audiences too (each followed once).
|
|
36
|
+
const collect = (spec, byId, found, seen = new Set()) => {
|
|
37
|
+
if (!spec || typeof spec !== 'object') {
|
|
38
|
+
return;
|
|
39
|
+
}
|
|
40
|
+
if (Array.isArray(spec.of)) {
|
|
41
|
+
spec.of.forEach((child) => collect(child, byId, found, seen));
|
|
42
|
+
} else if (spec.kind === 'friends' && spec.source) {
|
|
43
|
+
found.set(`${spec.source}\0friends`, [spec.source, { kind: 'friends' }]);
|
|
44
|
+
} else if (['chat', 'community'].includes(spec.kind) && spec.source) {
|
|
45
|
+
const id = String(spec[spec.kind]);
|
|
46
|
+
found.set(`${spec.source}\0${spec.kind}\0${id}`, [
|
|
47
|
+
spec.source,
|
|
48
|
+
{ kind: spec.kind, id },
|
|
49
|
+
]);
|
|
50
|
+
} else if (spec.kind === 'saved' && !seen.has(spec.id)) {
|
|
51
|
+
seen.add(spec.id);
|
|
52
|
+
const link = byId.get(`audience:${spec.id}`);
|
|
53
|
+
safeSpec(link?.spec ?? link?.text, (inner) =>
|
|
54
|
+
collect(inner, byId, found, seen)
|
|
55
|
+
);
|
|
56
|
+
}
|
|
57
|
+
};
|
|
58
|
+
|
|
59
|
+
const safeSpec = (value, use) => {
|
|
60
|
+
try {
|
|
61
|
+
use(toAudienceSpec(value));
|
|
62
|
+
} catch {
|
|
63
|
+
// An audience that does not parse names no group.
|
|
64
|
+
}
|
|
65
|
+
};
|
|
66
|
+
|
|
67
|
+
/** `Map<source, group[]>` named by the saved audiences and automations. */
|
|
68
|
+
export const audienceGroups = (links) => {
|
|
69
|
+
const byId = new Map(links.map((l) => [l?.id, l]));
|
|
70
|
+
const found = new Map();
|
|
71
|
+
for (const link of links) {
|
|
72
|
+
const id = String(link?.id ?? '');
|
|
73
|
+
if (id.startsWith('audience:')) {
|
|
74
|
+
safeSpec(link.spec ?? link.text, (s) => collect(s, byId, found));
|
|
75
|
+
} else if (id.startsWith('automation:') && link.audience) {
|
|
76
|
+
safeSpec(link.audience, (s) => collect(s, byId, found));
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
const out = new Map();
|
|
80
|
+
for (const [source, group] of found.values()) {
|
|
81
|
+
out.set(source, [...(out.get(source) ?? []), group]);
|
|
82
|
+
}
|
|
83
|
+
return out;
|
|
84
|
+
};
|
|
85
|
+
|
|
86
|
+
// The network's live API and the user's saved options for it, or an error.
|
|
87
|
+
const liveFor = async (store, source, resolve) => {
|
|
88
|
+
try {
|
|
89
|
+
return {
|
|
90
|
+
live: resolve(source).live,
|
|
91
|
+
options: await liveOptionsFrom(store, source),
|
|
92
|
+
};
|
|
93
|
+
} catch (error) {
|
|
94
|
+
return { error: safeMessage(error) };
|
|
95
|
+
}
|
|
96
|
+
};
|
|
97
|
+
|
|
98
|
+
// One group at a time, so a group the network refuses does not stop the rest.
|
|
99
|
+
const syncSource = async (store, source, groups, resolve, now) => {
|
|
100
|
+
const result = { source, ok: true, groups: 0, members: 0 };
|
|
101
|
+
const prepared = await liveFor(store, source, resolve);
|
|
102
|
+
if (prepared.error) {
|
|
103
|
+
return { ...result, ok: false, error: prepared.error };
|
|
104
|
+
}
|
|
105
|
+
const { live, options } = prepared;
|
|
106
|
+
if (!options.token) {
|
|
107
|
+
return { ...result, ok: false, error: 'no token saved for this network' };
|
|
108
|
+
}
|
|
109
|
+
const errors = [];
|
|
110
|
+
for (const group of groups) {
|
|
111
|
+
try {
|
|
112
|
+
const done = await syncMembership({
|
|
113
|
+
source,
|
|
114
|
+
live,
|
|
115
|
+
groups: [group],
|
|
116
|
+
put: (link) => store.put(link),
|
|
117
|
+
now,
|
|
118
|
+
options,
|
|
119
|
+
});
|
|
120
|
+
result.groups += done.groups.length;
|
|
121
|
+
result.members += done.members;
|
|
122
|
+
} catch (error) {
|
|
123
|
+
// The token itself never appears in what is reported.
|
|
124
|
+
errors.push(safeMessage(error).split(options.token).join('***'));
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
return errors.length > 0
|
|
128
|
+
? { ...result, ok: false, error: errors.join('; ').slice(0, 300) }
|
|
129
|
+
: result;
|
|
130
|
+
};
|
|
131
|
+
|
|
132
|
+
/** Sync every network the audiences name (or just `only`); never throws. */
|
|
133
|
+
export const syncAudiences = async ({
|
|
134
|
+
store,
|
|
135
|
+
resolve = getSource,
|
|
136
|
+
only = null,
|
|
137
|
+
now = Date.now(),
|
|
138
|
+
}) => {
|
|
139
|
+
const wanted = audienceGroups(await store.query());
|
|
140
|
+
const sources = [];
|
|
141
|
+
for (const [source, groups] of wanted) {
|
|
142
|
+
if (!only || only === source) {
|
|
143
|
+
sources.push(await syncSource(store, source, groups, resolve, now));
|
|
144
|
+
}
|
|
145
|
+
}
|
|
146
|
+
return { ok: sources.every((s) => s.ok), sources };
|
|
147
|
+
};
|
|
148
|
+
|
|
149
|
+
const DEFAULT_HOURS = 24;
|
|
150
|
+
|
|
151
|
+
// `MS_AUDIENCE_SYNC_HOURS`: hours between runs, `0` = off, unset = daily.
|
|
152
|
+
// Nonsense is ignored for the default with one log line.
|
|
153
|
+
const audienceHours = (env, log) => {
|
|
154
|
+
const raw = env?.MS_AUDIENCE_SYNC_HOURS;
|
|
155
|
+
if (raw === undefined || raw === null || raw === '') {
|
|
156
|
+
return DEFAULT_HOURS;
|
|
157
|
+
}
|
|
158
|
+
const n = Number(raw);
|
|
159
|
+
if (Number.isFinite(n) && n >= 0) {
|
|
160
|
+
return n;
|
|
161
|
+
}
|
|
162
|
+
log?.({
|
|
163
|
+
level: 'warn',
|
|
164
|
+
event: 'audience_sync_setting_ignored',
|
|
165
|
+
name: 'MS_AUDIENCE_SYNC_HOURS',
|
|
166
|
+
});
|
|
167
|
+
return DEFAULT_HOURS;
|
|
168
|
+
};
|
|
169
|
+
|
|
170
|
+
/** Once a day, once the port is ours; `{ stopAudienceSync }` for the context. */
|
|
171
|
+
export const audienceSyncFor = ({
|
|
172
|
+
store,
|
|
173
|
+
log,
|
|
174
|
+
later,
|
|
175
|
+
resolve = getSource,
|
|
176
|
+
env = process.env,
|
|
177
|
+
}) => {
|
|
178
|
+
const hours = audienceHours(env, log);
|
|
179
|
+
let timer = null;
|
|
180
|
+
if (hours > 0) {
|
|
181
|
+
later.push(() => {
|
|
182
|
+
timer = setInterval(
|
|
183
|
+
() =>
|
|
184
|
+
syncAudiences({ store, resolve })
|
|
185
|
+
.then((done) =>
|
|
186
|
+
log?.({
|
|
187
|
+
level: 'info',
|
|
188
|
+
event: 'audience_sync',
|
|
189
|
+
sources: done.sources.length,
|
|
190
|
+
ok: done.ok,
|
|
191
|
+
})
|
|
192
|
+
)
|
|
193
|
+
.catch((error) =>
|
|
194
|
+
log?.({
|
|
195
|
+
level: 'warn',
|
|
196
|
+
event: 'audience_sync_failed',
|
|
197
|
+
error: safeMessage(error),
|
|
198
|
+
})
|
|
199
|
+
),
|
|
200
|
+
hours * HOUR_MS
|
|
201
|
+
);
|
|
202
|
+
timer.unref?.();
|
|
203
|
+
});
|
|
204
|
+
}
|
|
205
|
+
return { stopAudienceSync: () => clearInterval(timer) };
|
|
206
|
+
};
|
|
207
|
+
|
|
208
|
+
export const handleAudienceSyncRoute = async (store, req, res, p, ctx) => {
|
|
209
|
+
if (p !== '/api/audience/sync' || req.method !== 'POST') {
|
|
210
|
+
return false;
|
|
211
|
+
}
|
|
212
|
+
if (!isTrustedMutation(req, ctx.allowOrigins)) {
|
|
213
|
+
return (
|
|
214
|
+
fail(res, 403, 'forbidden', 'must come from the meta-sovereign app') ??
|
|
215
|
+
true
|
|
216
|
+
);
|
|
217
|
+
}
|
|
218
|
+
const body = await readBody(req);
|
|
219
|
+
if (body.source !== undefined && !SOURCE_NAME.test(String(body.source))) {
|
|
220
|
+
return (
|
|
221
|
+
fail(res, 400, 'bad-request', 'source must be a network name') ?? true
|
|
222
|
+
);
|
|
223
|
+
}
|
|
224
|
+
const done = await syncAudiences({
|
|
225
|
+
store,
|
|
226
|
+
resolve: ctx.audienceSources ?? getSource,
|
|
227
|
+
only: body.source ?? null,
|
|
228
|
+
});
|
|
229
|
+
return json(res, 200, done) ?? true;
|
|
230
|
+
};
|
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The jobs that run by themselves while the server is up, started once the
|
|
3
|
+
* port is really ours (R-Z7; a second `serve` on a busy port must not run
|
|
4
|
+
* them on the same store):
|
|
5
|
+
*
|
|
6
|
+
* automatic backup ./backup-job.js daily, keep 7 (R-A4)
|
|
7
|
+
* audience membership sync ./audience-sync.js daily (R-X13)
|
|
8
|
+
* 24-hour cache purge here hourly (R-S12, R-T12)
|
|
9
|
+
*
|
|
10
|
+
* `backgroundJobs(...)` returns the part of the context holding `stopJobs`.
|
|
11
|
+
*/
|
|
12
|
+
import {
|
|
13
|
+
clearInterval,
|
|
14
|
+
clearTimeout,
|
|
15
|
+
setInterval,
|
|
16
|
+
setTimeout,
|
|
17
|
+
} from 'node:timers';
|
|
18
|
+
import { backupJobFor } from './backup-job.js';
|
|
19
|
+
import { audienceSyncFor } from './audience-sync.js';
|
|
20
|
+
import { safeMessage } from './redact.js';
|
|
21
|
+
import { softCacheRetention as upworkRetention } from '../sources/upwork.js';
|
|
22
|
+
import { softCacheRetention as pphRetention } from '../sources/peopleperhour.js';
|
|
23
|
+
|
|
24
|
+
const HOUR_MS = 3_600_000;
|
|
25
|
+
// The first purge, a minute after start, so a start-up is not slowed by a
|
|
26
|
+
// scan of the store; `MS_CACHE_PURGE_DELAY_MS` changes it (tests).
|
|
27
|
+
const FIRST_PURGE_MS = 60_000;
|
|
28
|
+
|
|
29
|
+
/**
|
|
30
|
+
* Upwork and PeoplePerHour content pulled live from their APIs is deleted
|
|
31
|
+
* 24 hours after it was pulled; what the user imported from an export stays.
|
|
32
|
+
*/
|
|
33
|
+
export const purgeSoftCaches = async (store, now = Date.now()) => {
|
|
34
|
+
const upwork = await upworkRetention(store, { now });
|
|
35
|
+
const peoplePerHour = await pphRetention(store, { now });
|
|
36
|
+
return { upwork: upwork.purged, peopleperhour: peoplePerHour.purged };
|
|
37
|
+
};
|
|
38
|
+
|
|
39
|
+
const cachePurgeFor = ({ store, log, later }) => {
|
|
40
|
+
let timer = null;
|
|
41
|
+
let first = null;
|
|
42
|
+
const run = () =>
|
|
43
|
+
purgeSoftCaches(store)
|
|
44
|
+
.then((purged) =>
|
|
45
|
+
purged.upwork + purged.peopleperhour > 0
|
|
46
|
+
? log?.({ level: 'info', event: 'soft_cache_purged', ...purged })
|
|
47
|
+
: null
|
|
48
|
+
)
|
|
49
|
+
.catch((error) =>
|
|
50
|
+
log?.({
|
|
51
|
+
level: 'warn',
|
|
52
|
+
event: 'soft_cache_purge_failed',
|
|
53
|
+
error: safeMessage(error),
|
|
54
|
+
})
|
|
55
|
+
);
|
|
56
|
+
later.push(() => {
|
|
57
|
+
const delay = Number(process.env.MS_CACHE_PURGE_DELAY_MS ?? FIRST_PURGE_MS);
|
|
58
|
+
first = setTimeout(run, delay);
|
|
59
|
+
timer = setInterval(run, HOUR_MS);
|
|
60
|
+
first.unref?.();
|
|
61
|
+
timer.unref?.();
|
|
62
|
+
});
|
|
63
|
+
return {
|
|
64
|
+
stopCachePurge: () => {
|
|
65
|
+
clearTimeout(first);
|
|
66
|
+
clearInterval(timer);
|
|
67
|
+
},
|
|
68
|
+
};
|
|
69
|
+
};
|
|
70
|
+
|
|
71
|
+
export const backgroundJobs = ({ ctx, store, log, later, backups }) => {
|
|
72
|
+
const backup = backupJobFor({ ctx, store, log, later, backups });
|
|
73
|
+
const audience = audienceSyncFor({
|
|
74
|
+
store,
|
|
75
|
+
log,
|
|
76
|
+
later,
|
|
77
|
+
resolve: ctx.audienceSources,
|
|
78
|
+
});
|
|
79
|
+
const purge = cachePurgeFor({ store, log, later });
|
|
80
|
+
return {
|
|
81
|
+
stopJobs: () => {
|
|
82
|
+
backup.stopBackups();
|
|
83
|
+
audience.stopAudienceSync();
|
|
84
|
+
purge.stopCachePurge();
|
|
85
|
+
// The other timers and pollers the server started with the port.
|
|
86
|
+
ctx.autoImport?.close();
|
|
87
|
+
ctx.whatsappSignIn?.stop().catch(() => {});
|
|
88
|
+
ctx.stopPulls?.();
|
|
89
|
+
},
|
|
90
|
+
};
|
|
91
|
+
};
|
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The automatic backup, started once the port is ours (R-A4, R-Z7).
|
|
3
|
+
*
|
|
4
|
+
* OFF by default (the streaming backups of R-Z16 switch it on).
|
|
5
|
+
* `MS_BACKUP_INTERVAL_HOURS` (or the `backups.intervalHours` option) sets
|
|
6
|
+
* the interval in hours, `0` or unset means off; `MS_BACKUP_KEEP` (or
|
|
7
|
+
* `backups.keep`) sets how many archives are kept (default 7, `0` keeps
|
|
8
|
+
* all). A value that is not a number >= 0 is ignored for the default and
|
|
9
|
+
* logged once.
|
|
10
|
+
* The archive is encrypted, and holds the secrets, only when the server has
|
|
11
|
+
* a passphrase (`MS_SECRET_PASSPHRASE`); otherwise it leaves them out.
|
|
12
|
+
* The first run is one interval after start, never at start. A nearly full
|
|
13
|
+
* disk skips the run (R-Z14) and the log says so.
|
|
14
|
+
*/
|
|
15
|
+
import { createBackupScheduler } from '../storage/backup.js';
|
|
16
|
+
|
|
17
|
+
const HOUR_MS = 3_600_000;
|
|
18
|
+
export const DEFAULT_INTERVAL_HOURS = 0;
|
|
19
|
+
export const DEFAULT_KEEP = 7;
|
|
20
|
+
|
|
21
|
+
// A number >= 0, or the fallback; `bad` collects the names that were wrong.
|
|
22
|
+
const number = (value, fallback, name, bad) => {
|
|
23
|
+
if (value === undefined || value === null || value === '') {
|
|
24
|
+
return fallback;
|
|
25
|
+
}
|
|
26
|
+
const n = Number(value);
|
|
27
|
+
if (Number.isFinite(n) && n >= 0) {
|
|
28
|
+
return n;
|
|
29
|
+
}
|
|
30
|
+
bad.push(name);
|
|
31
|
+
return fallback;
|
|
32
|
+
};
|
|
33
|
+
|
|
34
|
+
/**
|
|
35
|
+
* `{ intervalMs, keep, invalid }` from options and the environment; an
|
|
36
|
+
* interval of 0 is off; `invalid` names the settings that were ignored.
|
|
37
|
+
*/
|
|
38
|
+
export const backupSettings = (options = {}, env = process.env) => {
|
|
39
|
+
const invalid = [];
|
|
40
|
+
const hours = number(
|
|
41
|
+
options.intervalHours ?? env.MS_BACKUP_INTERVAL_HOURS,
|
|
42
|
+
DEFAULT_INTERVAL_HOURS,
|
|
43
|
+
'MS_BACKUP_INTERVAL_HOURS',
|
|
44
|
+
invalid
|
|
45
|
+
);
|
|
46
|
+
const keep = number(
|
|
47
|
+
options.keep ?? env.MS_BACKUP_KEEP,
|
|
48
|
+
DEFAULT_KEEP,
|
|
49
|
+
'MS_BACKUP_KEEP',
|
|
50
|
+
invalid
|
|
51
|
+
);
|
|
52
|
+
return {
|
|
53
|
+
intervalMs: Math.round(hours * HOUR_MS),
|
|
54
|
+
keep: Math.floor(keep),
|
|
55
|
+
invalid,
|
|
56
|
+
};
|
|
57
|
+
};
|
|
58
|
+
|
|
59
|
+
/** The context part: `{ stopBackups }`; the timer starts with `later`. */
|
|
60
|
+
export const backupJobFor = ({ ctx, store, log, later, backups }) => {
|
|
61
|
+
const { archiveDir, secretPassphrase: passphrase } = ctx;
|
|
62
|
+
const measureFree = ctx.diskFree;
|
|
63
|
+
const { intervalMs, keep, invalid } = backupSettings(backups);
|
|
64
|
+
if (invalid.length > 0) {
|
|
65
|
+
log?.({ level: 'warn', event: 'backup_setting_ignored', names: invalid });
|
|
66
|
+
}
|
|
67
|
+
let scheduler = null;
|
|
68
|
+
if (archiveDir && intervalMs > 0) {
|
|
69
|
+
later.push(() => {
|
|
70
|
+
scheduler = createBackupScheduler({
|
|
71
|
+
store,
|
|
72
|
+
archiveDir,
|
|
73
|
+
intervalMs,
|
|
74
|
+
keep,
|
|
75
|
+
passphrase,
|
|
76
|
+
measureFree,
|
|
77
|
+
onEvent: (event) => log?.(event),
|
|
78
|
+
onBackup: () => log?.({ level: 'info', event: 'backup_done' }),
|
|
79
|
+
// Never the text of a secret: only the message the backup raised.
|
|
80
|
+
onError: (error) =>
|
|
81
|
+
log?.({
|
|
82
|
+
level: 'warn',
|
|
83
|
+
event: 'backup_failed',
|
|
84
|
+
code: error.code ?? null,
|
|
85
|
+
error: String(error.message).slice(0, 200),
|
|
86
|
+
}),
|
|
87
|
+
});
|
|
88
|
+
});
|
|
89
|
+
}
|
|
90
|
+
return { stopBackups: () => scheduler?.stop() };
|
|
91
|
+
};
|
|
@@ -20,6 +20,7 @@
|
|
|
20
20
|
*/
|
|
21
21
|
|
|
22
22
|
import { chatKey, networkMessageId } from '../web/chat-model.js';
|
|
23
|
+
import { isTombstone } from '../storage/soft-delete.js';
|
|
23
24
|
import { splitTopicKey, topicOfMessage } from '../web/forum-topics.js';
|
|
24
25
|
|
|
25
26
|
const isMessage = (link) =>
|
|
@@ -115,13 +116,14 @@ export const createChatIndex = (store) => {
|
|
|
115
116
|
const chatOf = (key) => {
|
|
116
117
|
let chat = chats.get(key);
|
|
117
118
|
if (!chat) {
|
|
118
|
-
chat = { entries: new Map(), sorted: null, head: null };
|
|
119
|
+
chat = { entries: new Map(), sorted: null, head: null, gone: new Set() };
|
|
119
120
|
chats.set(key, chat);
|
|
120
121
|
}
|
|
121
122
|
return chat;
|
|
122
123
|
};
|
|
123
124
|
|
|
124
125
|
const remove = (chat, id) => {
|
|
126
|
+
chat.gone.delete(id);
|
|
125
127
|
if (!chat.entries.delete(id)) {
|
|
126
128
|
return;
|
|
127
129
|
}
|
|
@@ -135,10 +137,13 @@ export const createChatIndex = (store) => {
|
|
|
135
137
|
const add = (link) => {
|
|
136
138
|
const chat = chatOf(chatKey(link.source, link.chat));
|
|
137
139
|
const known = chat.entries.has(link.id);
|
|
138
|
-
if (link
|
|
140
|
+
if (isTombstone(link)) {
|
|
139
141
|
remove(chat, link.id);
|
|
142
|
+
// Kept apart for the "deleted messages" view (R-Z7).
|
|
143
|
+
chat.gone.add(link.id);
|
|
140
144
|
return 'update';
|
|
141
145
|
}
|
|
146
|
+
chat.gone.delete(link.id);
|
|
142
147
|
const entry = entryOf(link);
|
|
143
148
|
chat.entries.set(link.id, entry);
|
|
144
149
|
chat.sorted = null;
|
|
@@ -178,7 +183,7 @@ export const createChatIndex = (store) => {
|
|
|
178
183
|
|
|
179
184
|
const linksOf = async (entries) =>
|
|
180
185
|
(await Promise.all(entries.map((e) => store.get(e.id)))).filter(
|
|
181
|
-
(link) => link && !link
|
|
186
|
+
(link) => link && !isTombstone(link)
|
|
182
187
|
);
|
|
183
188
|
|
|
184
189
|
const sortedOf = (chat) => {
|
|
@@ -242,6 +247,11 @@ export const createChatIndex = (store) => {
|
|
|
242
247
|
...(range.at || range.around ? { target } : {}),
|
|
243
248
|
};
|
|
244
249
|
},
|
|
250
|
+
/** The ids of a chat's deleted (tombstoned) messages (R-Z7). */
|
|
251
|
+
async deleted(key) {
|
|
252
|
+
await ready;
|
|
253
|
+
return [...(chats.get(key)?.gone ?? [])];
|
|
254
|
+
},
|
|
245
255
|
/** How many messages of a chat the store holds. */
|
|
246
256
|
count: (key) => chats.get(key)?.entries.size ?? 0,
|
|
247
257
|
close: () => unsubscribe(),
|
package/js/src/server/index.js
CHANGED
|
@@ -81,6 +81,9 @@ import {
|
|
|
81
81
|
import { createDefaultSender } from '../sources/browser-sender.js';
|
|
82
82
|
import { createAutopilot } from '../automation/autopilot.js';
|
|
83
83
|
import { recoverOutbox } from '../automation/outbox.js';
|
|
84
|
+
import { backgroundJobs } from './background-jobs.js';
|
|
85
|
+
import { handleDeletedRoutes } from './routes-deleted.js';
|
|
86
|
+
import { handleAudienceSyncRoute } from './audience-sync.js';
|
|
84
87
|
import { setTypingFor } from '../sources/live-links.js';
|
|
85
88
|
import { createReplyService } from '../replies/reply-service.js';
|
|
86
89
|
import { createChatIndex } from './chat-index.js';
|
|
@@ -583,6 +586,8 @@ const routeOn = async (store, req, res, ctx, url, p) => {
|
|
|
583
586
|
() => handleTelegramRoutes(req, res, p, ctx),
|
|
584
587
|
() => handleMessageRoutes(store, req, res, p, url, ctx),
|
|
585
588
|
() => handleEmailGets(store, req, res, p, url, ctx),
|
|
589
|
+
() => handleDeletedRoutes(store, req, res, p, url, ctx),
|
|
590
|
+
() => handleAudienceSyncRoute(store, req, res, p, ctx),
|
|
586
591
|
() => handleChatRoutes(req, res, p, url, ctx),
|
|
587
592
|
() => ctx.learned?.(store, req, res, p, url, ctx) ?? false,
|
|
588
593
|
() => handleDerivedRoutes(store, req, res, p, url, ctx),
|
|
@@ -771,16 +776,16 @@ export const startServer = async ({
|
|
|
771
776
|
emailLiveFactory = null,
|
|
772
777
|
emailPoll,
|
|
773
778
|
cv = {},
|
|
774
|
-
// `{ self?, clink? }` for
|
|
779
|
+
// replies: `{ self?, clink? }` for /api/suggest; automations: `{ send?,
|
|
780
|
+
// setTyping?, resolveAudience?, self?, scheduler?, tickMs? }`.
|
|
775
781
|
replies,
|
|
776
|
-
// `{ send?, setTyping?, resolveAudience?, self?, scheduler?, tickMs? }`.
|
|
777
782
|
automations,
|
|
778
783
|
greetings,
|
|
779
784
|
allowOrigins,
|
|
780
785
|
autoImportLauncher,
|
|
781
|
-
// Tests inject a fake `createSignIns()` (no real browsers
|
|
786
|
+
// Tests inject a fake `createSignIns()` and a fake MTProto (no real browsers
|
|
787
|
+
// or network are used).
|
|
782
788
|
autoImportSignIns,
|
|
783
|
-
// Tests inject a fake MTProto (no real GramJS / network for Telegram login).
|
|
784
789
|
telegramMtproto,
|
|
785
790
|
telegramCreds,
|
|
786
791
|
// A `createTelegramAccount()` (or `false` for none); see telegramAccountFor.
|
|
@@ -796,9 +801,11 @@ export const startServer = async ({
|
|
|
796
801
|
localUi,
|
|
797
802
|
// HTTPS on the same port for Safari (./local-tls.js, R-Y64); `serve` sets it.
|
|
798
803
|
tls,
|
|
799
|
-
//
|
|
804
|
+
// diskFree: a test fake `(dir) => bytes|null` (R-Z14); backups:
|
|
805
|
+
// `{ intervalHours?, keep? }` for the automatic backup (R-Z7).
|
|
800
806
|
diskFree,
|
|
801
807
|
pulls,
|
|
808
|
+
backups,
|
|
802
809
|
} = {}) => {
|
|
803
810
|
const { store, resolvedArchiveDir, flush, storeHealth } = await initStore({
|
|
804
811
|
providedStore,
|
|
@@ -858,6 +865,7 @@ export const startServer = async ({
|
|
|
858
865
|
chatIndex,
|
|
859
866
|
...pullContext(store, log, pulls, later),
|
|
860
867
|
};
|
|
868
|
+
Object.assign(ctx, backgroundJobs({ ctx, store, log, later, backups }));
|
|
861
869
|
const server = createHttpServer(store, ctx, log);
|
|
862
870
|
const sockets = trackSockets(server);
|
|
863
871
|
const bus = enableHandlers
|
|
@@ -904,9 +912,7 @@ export const startServer = async ({
|
|
|
904
912
|
ctx.emailPoller.stop();
|
|
905
913
|
ctx.replies.close().catch(() => {});
|
|
906
914
|
ctx.closeAutomations().catch(() => {});
|
|
907
|
-
ctx.
|
|
908
|
-
ctx.whatsappSignIn?.stop().catch(() => {});
|
|
909
|
-
ctx.stopPulls?.();
|
|
915
|
+
ctx.stopJobs();
|
|
910
916
|
ctx.chatIndex.close();
|
|
911
917
|
stopAll(telegramAccount, ctx.telegramAccounts, ctx.telegramExtras);
|
|
912
918
|
wsHandle?.close();
|
|
@@ -0,0 +1,108 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The deleted messages of a chat: kept, never lost, and the user's to
|
|
3
|
+
* bring back or to remove (R-K1, R-K2, R-K4; wired to a page by R-Z7).
|
|
4
|
+
* These are local actions only: nothing is sent to any network.
|
|
5
|
+
*
|
|
6
|
+
* GET /api/chat/deleted?key=<chat key> -> { messages } the chat's
|
|
7
|
+
* tombstoned messages, oldest
|
|
8
|
+
* first (at most 500)
|
|
9
|
+
* POST /api/chat/deleted/recover { ids } -> { ok, recovered } the
|
|
10
|
+
* messages show in the chat
|
|
11
|
+
* again
|
|
12
|
+
* POST /api/chat/deleted/purge { ids, confirm: true }
|
|
13
|
+
* -> { ok, purged } erased from
|
|
14
|
+
* this device for good
|
|
15
|
+
*
|
|
16
|
+
* Only a message that is already deleted can be recovered or purged; a live
|
|
17
|
+
* one is never touched, whatever ids a request names.
|
|
18
|
+
*/
|
|
19
|
+
|
|
20
|
+
import { fail as failWith, json, readBody } from './util.js';
|
|
21
|
+
import { isTrustedMutation } from './routes-auto-import.js';
|
|
22
|
+
import { isTombstone, unmarkDeleted } from '../storage/soft-delete.js';
|
|
23
|
+
import { byTime } from '../web/chat-model.js';
|
|
24
|
+
|
|
25
|
+
const fail = (res, status, code, error) =>
|
|
26
|
+
failWith(res, status, code, error) ?? true;
|
|
27
|
+
|
|
28
|
+
const PREFIX = '/api/chat/deleted';
|
|
29
|
+
const MAX_LIST = 500;
|
|
30
|
+
const MAX_IDS = 500;
|
|
31
|
+
|
|
32
|
+
const idsOf = (body) =>
|
|
33
|
+
Array.isArray(body.ids) &&
|
|
34
|
+
body.ids.length > 0 &&
|
|
35
|
+
body.ids.length <= MAX_IDS &&
|
|
36
|
+
body.ids.every((id) => typeof id === 'string' && id.startsWith('msg:'))
|
|
37
|
+
? body.ids
|
|
38
|
+
: null;
|
|
39
|
+
|
|
40
|
+
// The tombstoned message links among `ids`.
|
|
41
|
+
const deletedAmong = async (store, ids) => {
|
|
42
|
+
const found = [];
|
|
43
|
+
for (const id of ids) {
|
|
44
|
+
const link = await store.get(id);
|
|
45
|
+
if (link && isTombstone(link)) {
|
|
46
|
+
found.push(link);
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
return found;
|
|
50
|
+
};
|
|
51
|
+
|
|
52
|
+
const list = async (store, res, url, ctx) => {
|
|
53
|
+
const key = url.searchParams.get('key');
|
|
54
|
+
if (!key || !ctx.chatIndex) {
|
|
55
|
+
return fail(res, 400, 'bad-request', 'key is required');
|
|
56
|
+
}
|
|
57
|
+
const ids = await ctx.chatIndex.deleted(key);
|
|
58
|
+
const links = await deletedAmong(store, ids.slice(0, MAX_LIST));
|
|
59
|
+
return json(res, 200, { messages: links.sort(byTime) }) ?? true;
|
|
60
|
+
};
|
|
61
|
+
|
|
62
|
+
const recover = async (store, res, ids) => {
|
|
63
|
+
const links = await deletedAmong(store, ids);
|
|
64
|
+
for (const link of links) {
|
|
65
|
+
await store.put(unmarkDeleted(link));
|
|
66
|
+
}
|
|
67
|
+
return (
|
|
68
|
+
json(res, 200, { ok: true, recovered: links.map((l) => l.id) }) ?? true
|
|
69
|
+
);
|
|
70
|
+
};
|
|
71
|
+
|
|
72
|
+
const purge = async (store, res, ids) => {
|
|
73
|
+
const purged = [];
|
|
74
|
+
for (const link of await deletedAmong(store, ids)) {
|
|
75
|
+
if (await store.delete(link.id)) {
|
|
76
|
+
purged.push(link.id);
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
return json(res, 200, { ok: true, purged }) ?? true;
|
|
80
|
+
};
|
|
81
|
+
|
|
82
|
+
export const handleDeletedRoutes = async (store, req, res, p, url, ctx) => {
|
|
83
|
+
if (!p.startsWith(PREFIX)) {
|
|
84
|
+
return false;
|
|
85
|
+
}
|
|
86
|
+
if (p === PREFIX && req.method === 'GET') {
|
|
87
|
+
return list(store, res, url, ctx);
|
|
88
|
+
}
|
|
89
|
+
const action = p.slice(PREFIX.length + 1);
|
|
90
|
+
if (req.method !== 'POST' || !['recover', 'purge'].includes(action)) {
|
|
91
|
+
return false;
|
|
92
|
+
}
|
|
93
|
+
if (!isTrustedMutation(req, ctx.allowOrigins)) {
|
|
94
|
+
return fail(res, 403, 'forbidden', 'must come from the meta-sovereign app');
|
|
95
|
+
}
|
|
96
|
+
const body = await readBody(req, { allowEmpty: false });
|
|
97
|
+
const ids = idsOf(body);
|
|
98
|
+
if (!ids) {
|
|
99
|
+
return fail(res, 400, 'bad-request', 'ids: message ids are needed');
|
|
100
|
+
}
|
|
101
|
+
if (action === 'purge') {
|
|
102
|
+
if (body.confirm !== true) {
|
|
103
|
+
return fail(res, 400, 'confirm-required', 'purging needs confirm: true');
|
|
104
|
+
}
|
|
105
|
+
return purge(store, res, ids);
|
|
106
|
+
}
|
|
107
|
+
return recover(store, res, ids);
|
|
108
|
+
};
|
|
@@ -24,6 +24,8 @@ import { json, readBody } from './util.js';
|
|
|
24
24
|
import { isTrustedMutation } from './routes-auto-import.js';
|
|
25
25
|
import { previewFields } from '../sources/telegram-link-preview.js';
|
|
26
26
|
|
|
27
|
+
import { markDeleted as tombstone } from '../storage/soft-delete.js';
|
|
28
|
+
|
|
27
29
|
const PREFIX = '/api/message/';
|
|
28
30
|
const MAX_IDS = 100;
|
|
29
31
|
|
|
@@ -102,7 +104,9 @@ const ACTIONS = {
|
|
|
102
104
|
}),
|
|
103
105
|
after: async (store, body, links) => {
|
|
104
106
|
for (const link of links) {
|
|
105
|
-
await store.put(
|
|
107
|
+
await store.put(
|
|
108
|
+
tombstone(link, { by: 'user', reason: 'deleted-in-app', at: now() })
|
|
109
|
+
);
|
|
106
110
|
}
|
|
107
111
|
},
|
|
108
112
|
},
|