@wirecat/cli-core 0.396.0 → 0.398.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +74 -1
- package/README.md +13 -9
- package/dist/background/daily-archive-timer.d.ts +40 -0
- package/dist/background/daily-archive-timer.js +181 -0
- package/dist/background/index.d.ts +2 -0
- package/dist/background/index.js +1 -0
- package/dist/cli/bot/admins.js +2 -2
- package/dist/cli/bot/command.js +2 -2
- package/dist/cli/bot/setup.js +3 -3
- package/dist/cli/commands-command.js +1 -1
- package/dist/cli/config-command.js +1 -1
- package/dist/cli/guide.js +0 -1
- package/dist/cli/help/command-help.d.ts +9 -2
- package/dist/cli/help/command-help.js +22 -4
- package/dist/cli/help/first-screen.js +3 -3
- package/dist/cli/messenger/account-command.js +3 -1
- package/dist/cli/messenger/account-sessions-command.js +3 -1
- package/dist/cli/messenger/admin-commands.js +1 -0
- package/dist/cli/messenger/admin-contacts-command.js +9 -3
- package/dist/cli/messenger/archive-commands.js +5 -3
- package/dist/cli/messenger/attachments-command.js +17 -7
- package/dist/cli/messenger/backfill-command.js +7 -3
- package/dist/cli/messenger/catch-up-answers.d.ts +2 -0
- package/dist/cli/messenger/catch-up-answers.js +5 -1
- package/dist/cli/messenger/chats-command.js +3 -1
- package/dist/cli/messenger/chats-members-command.js +4 -0
- package/dist/cli/messenger/chats-stats-command.js +1 -0
- package/dist/cli/messenger/chats-tracking-command.js +4 -1
- package/dist/cli/messenger/complete-command.js +6 -2
- package/dist/cli/messenger/contacts-command.js +11 -4
- package/dist/cli/messenger/context.d.ts +5 -0
- package/dist/cli/messenger/conversations-command.js +18 -1
- package/dist/cli/messenger/counters-command.js +1 -0
- package/dist/cli/messenger/doctor-command.js +2 -2
- package/dist/cli/messenger/download-command.js +1 -0
- package/dist/cli/messenger/flood-command.js +2 -1
- package/dist/cli/messenger/guard-commands.js +5 -4
- package/dist/cli/messenger/inbox.js +2 -0
- package/dist/cli/messenger/lacks.d.ts +7 -0
- package/dist/cli/messenger/lacks.js +18 -1
- package/dist/cli/messenger/mcp-command.js +6 -2
- package/dist/cli/messenger/media-archive-command.d.ts +3 -0
- package/dist/cli/messenger/media-archive-command.js +72 -0
- package/dist/cli/messenger/media-archive-move-command.d.ts +3 -0
- package/dist/cli/messenger/media-archive-move-command.js +39 -0
- package/dist/cli/messenger/media-archive-timer-command.d.ts +3 -0
- package/dist/cli/messenger/media-archive-timer-command.js +70 -0
- package/dist/cli/messenger/messages-command.js +2 -1
- package/dist/cli/messenger/messages-comments-command.js +1 -0
- package/dist/cli/messenger/messages-delete-command.js +1 -1
- package/dist/cli/messenger/messages-search-command.js +1 -0
- package/dist/cli/messenger/messages-stats-command.js +1 -0
- package/dist/cli/messenger/models-command.js +8 -2
- package/dist/cli/messenger/private-people-command.js +12 -10
- package/dist/cli/messenger/rankings-command.js +2 -0
- package/dist/cli/messenger/reactions-command.js +8 -2
- package/dist/cli/messenger/replies-command.js +1 -1
- package/dist/cli/messenger/retention-command.js +1 -0
- package/dist/cli/messenger/review.js +8 -2
- package/dist/cli/messenger/schedule-command.js +3 -3
- package/dist/cli/messenger/search-command.js +7 -3
- package/dist/cli/messenger/searches-command.js +24 -7
- package/dist/cli/messenger/serve-command.js +5 -1
- package/dist/cli/messenger/serve-relay.d.ts +1 -0
- package/dist/cli/messenger/server-command.js +17 -3
- package/dist/cli/messenger/store-maintenance-command.d.ts +1 -1
- package/dist/cli/messenger/store-maintenance-command.js +10 -3
- package/dist/cli/messenger/tags-command.js +6 -2
- package/dist/cli/messenger/tasks-command.js +10 -7
- package/dist/cli/messenger/transcribe-command.js +1 -0
- package/dist/cli/messenger/watch-command.js +6 -1
- package/dist/cli/runs/command.js +2 -3
- package/dist/cli/skill-command.js +1 -1
- package/dist/domain/models.d.ts +2 -0
- package/dist/entities/contact-read.d.ts +119 -0
- package/dist/entities/contact-read.js +50 -0
- package/dist/entities/index.d.ts +120 -1
- package/dist/entities/index.js +5 -1
- package/dist/meetings/mcp/index.d.ts +15 -15
- package/dist/parity/catalogue.js +1 -1
- package/dist/parity/manifest.d.ts +3 -1
- package/dist/parity/manifest.js +2 -0
- package/dist/parity/wording.js +2 -0
- package/dist/sends/permissions.js +7 -0
- package/dist/services/deps.d.ts +1 -1
- package/dist/services/deps.js +1 -1
- package/dist/services/inbox.js +5 -5
- package/dist/services/index.d.ts +5 -0
- package/dist/services/index.js +2 -0
- package/dist/services/media-archive-files.d.ts +10 -0
- package/dist/services/media-archive-files.js +110 -0
- package/dist/services/media-archive-locks.d.ts +6 -0
- package/dist/services/media-archive-locks.js +71 -0
- package/dist/services/media-archive-manifest.d.ts +29 -0
- package/dist/services/media-archive-manifest.js +116 -0
- package/dist/services/media-archive-move.d.ts +21 -0
- package/dist/services/media-archive-move.js +163 -0
- package/dist/services/media-archive.d.ts +34 -0
- package/dist/services/media-archive.js +165 -0
- package/dist/services/search-kind.js +1 -1
- package/package.json +2 -2
- package/parity.json +116 -193
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
import { mkdir, realpath } from "node:fs/promises";
|
|
2
|
+
import { join, resolve } from "node:path";
|
|
3
|
+
import { CliError } from "@wirecat/cli-base";
|
|
4
|
+
import { acquireLock } from "../files/index.js";
|
|
5
|
+
import { missingMediaFile, readMediaManifests } from "./media-archive-manifest.js";
|
|
6
|
+
const archiveLocations = async (initial, signal) => {
|
|
7
|
+
const roots = new Set();
|
|
8
|
+
const queued = new Set(initial.map((directory) => resolve(directory)));
|
|
9
|
+
const pending = [...queued];
|
|
10
|
+
for (let at = 0; at < pending.length; at++) {
|
|
11
|
+
signal?.throwIfAborted();
|
|
12
|
+
let root;
|
|
13
|
+
try {
|
|
14
|
+
root = await realpath(pending[at]);
|
|
15
|
+
}
|
|
16
|
+
catch (error) {
|
|
17
|
+
if (missingMediaFile(error))
|
|
18
|
+
continue;
|
|
19
|
+
throw error;
|
|
20
|
+
}
|
|
21
|
+
if (roots.has(root))
|
|
22
|
+
continue;
|
|
23
|
+
if (roots.size >= 64)
|
|
24
|
+
throw new CliError("validation_error", "too many linked media archive directories");
|
|
25
|
+
roots.add(root);
|
|
26
|
+
for (const entry of await readMediaManifests(root, signal))
|
|
27
|
+
if (entry.relocatedTo) {
|
|
28
|
+
const location = resolve(entry.relocatedTo);
|
|
29
|
+
if (!queued.has(location)) {
|
|
30
|
+
queued.add(location);
|
|
31
|
+
pending.push(location);
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
return [...roots].sort();
|
|
36
|
+
};
|
|
37
|
+
/** Acquire in canonical path order, then recheck the graph before trusting relocation records. */
|
|
38
|
+
export const withMediaArchiveLocks = async (directory, signal, run, options = {}) => {
|
|
39
|
+
signal?.throwIfAborted();
|
|
40
|
+
if (options.create)
|
|
41
|
+
await mkdir(directory, { recursive: true, mode: 0o700 });
|
|
42
|
+
const root = await realpath(directory);
|
|
43
|
+
if (options.createAdditional)
|
|
44
|
+
for (const other of options.additional ?? [])
|
|
45
|
+
await mkdir(other, { recursive: true, mode: 0o700 });
|
|
46
|
+
const initial = [root, ...(options.additional ?? [])];
|
|
47
|
+
for (let attempt = 0; attempt < 3; attempt++) {
|
|
48
|
+
const locations = await archiveLocations(initial, signal);
|
|
49
|
+
const leases = [];
|
|
50
|
+
try {
|
|
51
|
+
for (const location of locations)
|
|
52
|
+
leases.push(await acquireLock(join(location, ".media-writers"), { signal }));
|
|
53
|
+
const current = await archiveLocations(initial, signal);
|
|
54
|
+
if (current.some((location) => !locations.includes(location)))
|
|
55
|
+
continue;
|
|
56
|
+
const beforeWrite = async () => {
|
|
57
|
+
signal?.throwIfAborted();
|
|
58
|
+
for (const lease of leases)
|
|
59
|
+
if (!(await lease.held()))
|
|
60
|
+
throw new CliError("validation_error", "media archive writer lost its lock");
|
|
61
|
+
};
|
|
62
|
+
await beforeWrite();
|
|
63
|
+
return await run(root, beforeWrite);
|
|
64
|
+
}
|
|
65
|
+
finally {
|
|
66
|
+
for (const lease of leases.reverse())
|
|
67
|
+
await lease.release();
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
throw new CliError("validation_error", "archive locations changed during locking; run the command again");
|
|
71
|
+
};
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
export interface MediaIdentity {
|
|
2
|
+
provider: string;
|
|
3
|
+
account: string;
|
|
4
|
+
chat: string;
|
|
5
|
+
message: string;
|
|
6
|
+
position: number;
|
|
7
|
+
}
|
|
8
|
+
export interface MediaArchiveEntry extends MediaIdentity {
|
|
9
|
+
version: 1;
|
|
10
|
+
kind: string;
|
|
11
|
+
path: string;
|
|
12
|
+
size: number;
|
|
13
|
+
sha256: string;
|
|
14
|
+
messageTime?: string;
|
|
15
|
+
mime?: string;
|
|
16
|
+
relocatedTo?: string;
|
|
17
|
+
}
|
|
18
|
+
export declare const mediaKey: (one: MediaIdentity) => string;
|
|
19
|
+
export declare const mediaManifestName: (one: MediaIdentity) => string;
|
|
20
|
+
export declare const missingMediaFile: (error: unknown) => boolean;
|
|
21
|
+
export declare const parseMediaEntry: (value: unknown) => MediaArchiveEntry;
|
|
22
|
+
export declare const mediaFingerprint: (path: string, signal?: AbortSignal) => Promise<{
|
|
23
|
+
size: number;
|
|
24
|
+
sha256: string;
|
|
25
|
+
}>;
|
|
26
|
+
export declare const readMediaManifests: (root: string, signal?: AbortSignal) => Promise<MediaArchiveEntry[]>;
|
|
27
|
+
export declare const verifyMediaEntry: (root: string, one: MediaArchiveEntry, signal?: AbortSignal) => Promise<boolean>;
|
|
28
|
+
export declare const sameMediaCopy: (a: MediaArchiveEntry, b: MediaArchiveEntry) => boolean;
|
|
29
|
+
export declare const commitMediaEntry: (root: string, one: MediaArchiveEntry, beforePublish: () => Promise<void>) => Promise<void>;
|
|
@@ -0,0 +1,116 @@
|
|
|
1
|
+
import { createHash } from "node:crypto";
|
|
2
|
+
import { readdir } from "node:fs/promises";
|
|
3
|
+
import { isAbsolute, join, resolve } from "node:path";
|
|
4
|
+
import { CliError } from "@wirecat/cli-base";
|
|
5
|
+
import { openOrdinaryFile, readBoundedUtf8File, writeFileAtomic } from "../files/index.js";
|
|
6
|
+
export const mediaKey = (one) => createHash("sha256")
|
|
7
|
+
.update(JSON.stringify([one.provider, one.account, one.chat, one.message, one.position]))
|
|
8
|
+
.digest("hex");
|
|
9
|
+
export const mediaManifestName = (one) => `.media-archive-${mediaKey(one)}.json`;
|
|
10
|
+
export const missingMediaFile = (error) => typeof error === "object" && error !== null && "code" in error && error.code === "ENOENT";
|
|
11
|
+
export const parseMediaEntry = (value) => {
|
|
12
|
+
const one = value;
|
|
13
|
+
if (!one ||
|
|
14
|
+
typeof one !== "object" ||
|
|
15
|
+
Array.isArray(one) ||
|
|
16
|
+
one.version !== 1 ||
|
|
17
|
+
Object.keys(one).some((key) => ![
|
|
18
|
+
"account",
|
|
19
|
+
"chat",
|
|
20
|
+
"kind",
|
|
21
|
+
"message",
|
|
22
|
+
"path",
|
|
23
|
+
"position",
|
|
24
|
+
"provider",
|
|
25
|
+
"sha256",
|
|
26
|
+
"size",
|
|
27
|
+
"version",
|
|
28
|
+
"messageTime",
|
|
29
|
+
"mime",
|
|
30
|
+
"relocatedTo",
|
|
31
|
+
].includes(key)) ||
|
|
32
|
+
(one.messageTime !== undefined &&
|
|
33
|
+
(typeof one.messageTime !== "string" ||
|
|
34
|
+
!Number.isFinite(Date.parse(one.messageTime)) ||
|
|
35
|
+
new Date(one.messageTime).toISOString() !== one.messageTime)) ||
|
|
36
|
+
(one.mime !== undefined &&
|
|
37
|
+
(typeof one.mime !== "string" || one.mime.length > 127 || !/^[a-z0-9.+-]+\/[a-z0-9.+-]+$/.test(one.mime))) ||
|
|
38
|
+
(one.relocatedTo !== undefined &&
|
|
39
|
+
(typeof one.relocatedTo !== "string" || !isAbsolute(one.relocatedTo) || /[\p{Cc}]/u.test(one.relocatedTo))) ||
|
|
40
|
+
[one.provider, one.account, one.chat, one.message, one.kind].some((text) => typeof text !== "string" || !text.trim() || text.length > 512 || /[\0\r\n]/.test(text)) ||
|
|
41
|
+
!Number.isSafeInteger(one.position) ||
|
|
42
|
+
one.position < 0 ||
|
|
43
|
+
!Number.isSafeInteger(one.size) ||
|
|
44
|
+
one.size < 0 ||
|
|
45
|
+
typeof one.sha256 !== "string" ||
|
|
46
|
+
!/^[a-f0-9]{64}$/.test(one.sha256) ||
|
|
47
|
+
one.path !== `${mediaKey(one)}.bin`)
|
|
48
|
+
throw new CliError("validation_error", "invalid media archive manifest");
|
|
49
|
+
return { ...one };
|
|
50
|
+
};
|
|
51
|
+
export const mediaFingerprint = async (path, signal) => {
|
|
52
|
+
signal?.throwIfAborted();
|
|
53
|
+
const file = await openOrdinaryFile(path);
|
|
54
|
+
try {
|
|
55
|
+
const hash = createHash("sha256");
|
|
56
|
+
let size = 0;
|
|
57
|
+
for await (const chunk of file.createReadStream({ autoClose: false, ...(signal ? { signal } : {}) })) {
|
|
58
|
+
signal?.throwIfAborted();
|
|
59
|
+
hash.update(chunk);
|
|
60
|
+
size += chunk.length;
|
|
61
|
+
}
|
|
62
|
+
return { size, sha256: hash.digest("hex") };
|
|
63
|
+
}
|
|
64
|
+
finally {
|
|
65
|
+
await file.close();
|
|
66
|
+
}
|
|
67
|
+
};
|
|
68
|
+
export const readMediaManifests = async (root, signal) => {
|
|
69
|
+
const entries = [];
|
|
70
|
+
for (const name of await readdir(root)) {
|
|
71
|
+
signal?.throwIfAborted();
|
|
72
|
+
if (!name.startsWith(".media-archive-") || !name.endsWith(".json"))
|
|
73
|
+
continue;
|
|
74
|
+
let one;
|
|
75
|
+
try {
|
|
76
|
+
one = parseMediaEntry(JSON.parse(await readBoundedUtf8File(join(root, name), { maxBytes: 16384 })));
|
|
77
|
+
}
|
|
78
|
+
catch {
|
|
79
|
+
throw new CliError("validation_error", "invalid media archive manifest");
|
|
80
|
+
}
|
|
81
|
+
if (name !== mediaManifestName(one))
|
|
82
|
+
throw new CliError("validation_error", "media manifest identity mismatch");
|
|
83
|
+
entries.push(one);
|
|
84
|
+
}
|
|
85
|
+
return entries;
|
|
86
|
+
};
|
|
87
|
+
export const verifyMediaEntry = async (root, one, signal) => {
|
|
88
|
+
let location = resolve(root);
|
|
89
|
+
let entry = one;
|
|
90
|
+
const visited = new Set();
|
|
91
|
+
while (entry.relocatedTo !== undefined) {
|
|
92
|
+
signal?.throwIfAborted();
|
|
93
|
+
if (visited.has(location) || visited.size >= 64)
|
|
94
|
+
throw new CliError("validation_error", "media archive relocation cycle or excessive depth");
|
|
95
|
+
visited.add(location);
|
|
96
|
+
location = resolve(entry.relocatedTo);
|
|
97
|
+
const target = parseMediaEntry(JSON.parse(await readBoundedUtf8File(join(location, mediaManifestName(one)), { maxBytes: 16384 })));
|
|
98
|
+
if (!sameMediaCopy(one, target))
|
|
99
|
+
return false;
|
|
100
|
+
entry = target;
|
|
101
|
+
}
|
|
102
|
+
if (visited.has(location))
|
|
103
|
+
return false;
|
|
104
|
+
const actual = await mediaFingerprint(join(location, one.path), signal);
|
|
105
|
+
return actual.size === one.size && actual.sha256 === one.sha256;
|
|
106
|
+
};
|
|
107
|
+
export const sameMediaCopy = (a, b) => mediaKey(a) === mediaKey(b) &&
|
|
108
|
+
a.size === b.size &&
|
|
109
|
+
a.sha256 === b.sha256 &&
|
|
110
|
+
a.kind === b.kind &&
|
|
111
|
+
a.messageTime === b.messageTime &&
|
|
112
|
+
a.mime === b.mime;
|
|
113
|
+
export const commitMediaEntry = (root, one, beforePublish) => writeFileAtomic(join(root, mediaManifestName(one)), JSON.stringify(parseMediaEntry(one)), {
|
|
114
|
+
overwrite: false,
|
|
115
|
+
beforePublish,
|
|
116
|
+
});
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
export interface MoveMediaArchiveOptions {
|
|
2
|
+
directory: string;
|
|
3
|
+
archiveDirectory: string;
|
|
4
|
+
beforeTime: string;
|
|
5
|
+
dryRun?: boolean;
|
|
6
|
+
signal?: AbortSignal;
|
|
7
|
+
}
|
|
8
|
+
/** The destination is durable before a source relocation record is published, then the verified local copy is removed. */
|
|
9
|
+
export declare const moveMediaArchive: (options: MoveMediaArchiveOptions) => Promise<{
|
|
10
|
+
directory: string;
|
|
11
|
+
archiveDirectory: string;
|
|
12
|
+
beforeTime: string;
|
|
13
|
+
selected: number;
|
|
14
|
+
selectedBytes: number;
|
|
15
|
+
moved: number;
|
|
16
|
+
movedBytes: number;
|
|
17
|
+
alreadyArchived: number;
|
|
18
|
+
undated: number;
|
|
19
|
+
complete: boolean;
|
|
20
|
+
dryRun: boolean;
|
|
21
|
+
}>;
|
|
@@ -0,0 +1,163 @@
|
|
|
1
|
+
import { realpath, unlink } from "node:fs/promises";
|
|
2
|
+
import { basename, dirname, join, resolve } from "node:path";
|
|
3
|
+
import { CliError } from "@wirecat/cli-base";
|
|
4
|
+
import { openOrdinaryFile, sameOrdinaryFile, syncDirectory, writeFileAtomic } from "../files/index.js";
|
|
5
|
+
import { validateMediaSince } from "./media-archive.js";
|
|
6
|
+
import { archiveMediaFile } from "./media-archive-files.js";
|
|
7
|
+
import { withMediaArchiveLocks } from "./media-archive-locks.js";
|
|
8
|
+
import { mediaFingerprint, mediaKey, mediaManifestName, missingMediaFile, parseMediaEntry, readMediaManifests, sameMediaCopy, verifyMediaEntry, } from "./media-archive-manifest.js";
|
|
9
|
+
const canonicalDestination = async (directory) => {
|
|
10
|
+
const path = resolve(directory);
|
|
11
|
+
try {
|
|
12
|
+
return await realpath(path);
|
|
13
|
+
}
|
|
14
|
+
catch (error) {
|
|
15
|
+
if (!missingMediaFile(error) || dirname(path) === path)
|
|
16
|
+
throw error;
|
|
17
|
+
return join(await canonicalDestination(dirname(path)), basename(path));
|
|
18
|
+
}
|
|
19
|
+
};
|
|
20
|
+
const image = (entry) => ["photo", "image"].includes(entry.kind) || entry.mime?.startsWith("image/");
|
|
21
|
+
const removeLocalCopy = async (root, entry, beforeWrite, signal) => {
|
|
22
|
+
const path = join(root, entry.path);
|
|
23
|
+
let file;
|
|
24
|
+
try {
|
|
25
|
+
file = await openOrdinaryFile(path);
|
|
26
|
+
}
|
|
27
|
+
catch (error) {
|
|
28
|
+
if (missingMediaFile(error))
|
|
29
|
+
return;
|
|
30
|
+
throw error;
|
|
31
|
+
}
|
|
32
|
+
try {
|
|
33
|
+
const identity = await file.stat({ bigint: true });
|
|
34
|
+
const proof = await mediaFingerprint(path, signal);
|
|
35
|
+
if (proof.size !== entry.size || proof.sha256 !== entry.sha256 || !(await sameOrdinaryFile(path, identity)))
|
|
36
|
+
throw new CliError("validation_error", "local image changed during rotation; it was preserved");
|
|
37
|
+
if (!(await verifyMediaEntry(root, entry, signal)))
|
|
38
|
+
throw new CliError("validation_error", "rotated image checksum mismatch; the local copy was preserved");
|
|
39
|
+
await beforeWrite();
|
|
40
|
+
if (!(await sameOrdinaryFile(path, identity)))
|
|
41
|
+
throw new CliError("validation_error", "local image was replaced; it was preserved");
|
|
42
|
+
await unlink(path);
|
|
43
|
+
await syncDirectory(root);
|
|
44
|
+
}
|
|
45
|
+
finally {
|
|
46
|
+
await file.close();
|
|
47
|
+
}
|
|
48
|
+
};
|
|
49
|
+
/** The destination is durable before a source relocation record is published, then the verified local copy is removed. */
|
|
50
|
+
export const moveMediaArchive = async (options) => {
|
|
51
|
+
const cutoff = validateMediaSince(options.beforeTime, "--before-time");
|
|
52
|
+
if (cutoff === undefined)
|
|
53
|
+
throw new CliError("validation_error", "--before-time is required");
|
|
54
|
+
const target = await canonicalDestination(options.archiveDirectory);
|
|
55
|
+
const source = await realpath(options.directory);
|
|
56
|
+
if (source === target)
|
|
57
|
+
throw new CliError("validation_error", "choose a different archive destination");
|
|
58
|
+
return withMediaArchiveLocks(source, options.signal, async (root, beforeWrite) => {
|
|
59
|
+
const entries = await readMediaManifests(root, options.signal);
|
|
60
|
+
if (!entries.length)
|
|
61
|
+
throw new CliError("not_found", "no media archive manifests in this directory");
|
|
62
|
+
const targetEntries = await readMediaManifests(target, options.signal).catch((error) => {
|
|
63
|
+
if (options.dryRun && missingMediaFile(error))
|
|
64
|
+
return [];
|
|
65
|
+
throw error;
|
|
66
|
+
});
|
|
67
|
+
for (const [folder, manifests] of [
|
|
68
|
+
[root, entries],
|
|
69
|
+
[target, targetEntries],
|
|
70
|
+
])
|
|
71
|
+
for (const entry of manifests) {
|
|
72
|
+
let valid = false;
|
|
73
|
+
try {
|
|
74
|
+
valid = await verifyMediaEntry(folder, entry, options.signal);
|
|
75
|
+
}
|
|
76
|
+
catch {
|
|
77
|
+
options.signal?.throwIfAborted();
|
|
78
|
+
}
|
|
79
|
+
if (!valid)
|
|
80
|
+
throw new CliError("validation_error", "archive checksum mismatch or missing file; no images were moved");
|
|
81
|
+
}
|
|
82
|
+
const known = new Map(targetEntries.map((entry) => [mediaKey(entry), entry]));
|
|
83
|
+
let selected = 0, moved = 0, movedBytes = 0, selectedBytes = 0, undated = 0, alreadyArchived = 0;
|
|
84
|
+
const candidates = [];
|
|
85
|
+
for (const entry of entries) {
|
|
86
|
+
if (!image(entry))
|
|
87
|
+
continue;
|
|
88
|
+
if (!entry.messageTime) {
|
|
89
|
+
undated++;
|
|
90
|
+
continue;
|
|
91
|
+
}
|
|
92
|
+
if (Date.parse(entry.messageTime) >= cutoff)
|
|
93
|
+
continue;
|
|
94
|
+
if (entry.relocatedTo) {
|
|
95
|
+
alreadyArchived++;
|
|
96
|
+
if (!options.dryRun)
|
|
97
|
+
await removeLocalCopy(root, entry, beforeWrite, options.signal);
|
|
98
|
+
continue;
|
|
99
|
+
}
|
|
100
|
+
const existing = known.get(mediaKey(entry));
|
|
101
|
+
if (existing?.relocatedTo)
|
|
102
|
+
throw new CliError("validation_error", "destination image is already relocated; choose its current archive directory");
|
|
103
|
+
if (existing && !sameMediaCopy(entry, existing))
|
|
104
|
+
throw new CliError("validation_error", "destination has a different copy of this image; choose a new destination");
|
|
105
|
+
candidates.push(entry);
|
|
106
|
+
selected++;
|
|
107
|
+
selectedBytes += entry.size;
|
|
108
|
+
}
|
|
109
|
+
if (!options.dryRun)
|
|
110
|
+
for (const entry of candidates) {
|
|
111
|
+
await beforeWrite();
|
|
112
|
+
if (!known.has(mediaKey(entry))) {
|
|
113
|
+
const file = await openOrdinaryFile(join(root, entry.path));
|
|
114
|
+
try {
|
|
115
|
+
const identity = {
|
|
116
|
+
provider: entry.provider,
|
|
117
|
+
account: entry.account,
|
|
118
|
+
chat: entry.chat,
|
|
119
|
+
message: entry.message,
|
|
120
|
+
position: entry.position,
|
|
121
|
+
};
|
|
122
|
+
const copied = await archiveMediaFile(target, identity, entry.kind, async () => ({
|
|
123
|
+
kind: entry.kind,
|
|
124
|
+
size: entry.size,
|
|
125
|
+
bytes: () => file.createReadStream({ autoClose: false, ...(options.signal ? { signal: options.signal } : {}) }),
|
|
126
|
+
}), beforeWrite, options.signal, {
|
|
127
|
+
messageTime: entry.messageTime,
|
|
128
|
+
mime: entry.mime,
|
|
129
|
+
expected: { size: entry.size, sha256: entry.sha256 },
|
|
130
|
+
});
|
|
131
|
+
known.set(mediaKey(entry), copied);
|
|
132
|
+
}
|
|
133
|
+
finally {
|
|
134
|
+
await file.close();
|
|
135
|
+
}
|
|
136
|
+
}
|
|
137
|
+
const receipt = parseMediaEntry({ ...entry, relocatedTo: target });
|
|
138
|
+
if (!(await verifyMediaEntry(root, receipt, options.signal)))
|
|
139
|
+
throw new CliError("validation_error", "archive destination changed before rotation; the original was preserved");
|
|
140
|
+
await beforeWrite();
|
|
141
|
+
await writeFileAtomic(join(root, mediaManifestName(entry)), JSON.stringify(receipt), {
|
|
142
|
+
beforePublish: beforeWrite,
|
|
143
|
+
signal: options.signal,
|
|
144
|
+
});
|
|
145
|
+
await removeLocalCopy(root, receipt, beforeWrite, options.signal);
|
|
146
|
+
moved++;
|
|
147
|
+
movedBytes += entry.size;
|
|
148
|
+
}
|
|
149
|
+
return {
|
|
150
|
+
directory: root,
|
|
151
|
+
archiveDirectory: target,
|
|
152
|
+
beforeTime: new Date(cutoff).toISOString(),
|
|
153
|
+
selected,
|
|
154
|
+
selectedBytes,
|
|
155
|
+
moved,
|
|
156
|
+
movedBytes,
|
|
157
|
+
alreadyArchived,
|
|
158
|
+
undated,
|
|
159
|
+
complete: undated === 0,
|
|
160
|
+
dryRun: options.dryRun === true,
|
|
161
|
+
};
|
|
162
|
+
}, { additional: [target], createAdditional: !options.dryRun });
|
|
163
|
+
};
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
import type { ChatsService } from "./chats.js";
|
|
2
|
+
import { type MediaArchiveEntry } from "./media-archive-manifest.js";
|
|
3
|
+
import type { MessagesService } from "./messages.js";
|
|
4
|
+
export interface MediaArchivePorts {
|
|
5
|
+
chats: Pick<ChatsService, "list" | "account" | "show">;
|
|
6
|
+
messages: Pick<MessagesService, "list" | "download">;
|
|
7
|
+
}
|
|
8
|
+
export interface MediaArchiveOptions {
|
|
9
|
+
directory: string;
|
|
10
|
+
chats?: readonly string[];
|
|
11
|
+
sinceTime?: string;
|
|
12
|
+
orderByTime?: boolean;
|
|
13
|
+
pageSize?: number;
|
|
14
|
+
pauseMs?: number;
|
|
15
|
+
signal?: AbortSignal;
|
|
16
|
+
onSaved?: (entry: MediaArchiveEntry) => void;
|
|
17
|
+
}
|
|
18
|
+
export declare const validateMediaSince: (value?: string, flag?: string) => number | undefined;
|
|
19
|
+
export declare const checkMediaArchive: (directory: string, signal?: AbortSignal) => Promise<{
|
|
20
|
+
valid: boolean;
|
|
21
|
+
checked: number;
|
|
22
|
+
files: {
|
|
23
|
+
path: string;
|
|
24
|
+
valid: boolean;
|
|
25
|
+
archiveDirectory?: string;
|
|
26
|
+
}[];
|
|
27
|
+
}>;
|
|
28
|
+
export declare const archiveMedia: (ports: MediaArchivePorts, options: MediaArchiveOptions) => Promise<{
|
|
29
|
+
directory: string;
|
|
30
|
+
chats: number;
|
|
31
|
+
saved: number;
|
|
32
|
+
existing: number;
|
|
33
|
+
complete: boolean;
|
|
34
|
+
}>;
|
|
@@ -0,0 +1,165 @@
|
|
|
1
|
+
import { setTimeout as sleep } from "node:timers/promises";
|
|
2
|
+
import { CliError } from "@wirecat/cli-base";
|
|
3
|
+
import { matchDownloads, NOT_FILES } from "../domain/attachments.js";
|
|
4
|
+
import { archiveMediaFile } from "./media-archive-files.js";
|
|
5
|
+
import { withMediaArchiveLocks } from "./media-archive-locks.js";
|
|
6
|
+
import { mediaKey, readMediaManifests, verifyMediaEntry } from "./media-archive-manifest.js";
|
|
7
|
+
export const validateMediaSince = (value, flag = "--since-time") => {
|
|
8
|
+
if (value === undefined)
|
|
9
|
+
return undefined;
|
|
10
|
+
if (!/^\d{4}-\d\d-\d\dT\d\d:\d\d:\d\d(?:\.\d{1,3})?(?:Z|[+-]\d\d:\d\d)$/.test(value) ||
|
|
11
|
+
!Number.isFinite(Date.parse(value)))
|
|
12
|
+
throw new CliError("validation_error", `${flag} needs an ISO timestamp with a time zone`);
|
|
13
|
+
const date = value.slice(0, 10);
|
|
14
|
+
if (new Date(`${date}T00:00:00Z`).toISOString().slice(0, 10) !== date)
|
|
15
|
+
throw new CliError("validation_error", `${flag} has an invalid calendar date`);
|
|
16
|
+
return Date.parse(value);
|
|
17
|
+
};
|
|
18
|
+
export const checkMediaArchive = (directory, signal) => withMediaArchiveLocks(directory, signal, async (root) => {
|
|
19
|
+
const entries = await readMediaManifests(root, signal);
|
|
20
|
+
if (!entries.length)
|
|
21
|
+
throw new CliError("not_found", "no media archive manifests in this directory");
|
|
22
|
+
const files = [];
|
|
23
|
+
for (const entry of entries) {
|
|
24
|
+
let valid = false;
|
|
25
|
+
try {
|
|
26
|
+
valid = await verifyMediaEntry(root, entry, signal);
|
|
27
|
+
}
|
|
28
|
+
catch {
|
|
29
|
+
signal?.throwIfAborted();
|
|
30
|
+
}
|
|
31
|
+
files.push({ path: entry.path, valid, ...(entry.relocatedTo ? { archiveDirectory: entry.relocatedTo } : {}) });
|
|
32
|
+
}
|
|
33
|
+
return { valid: files.every((one) => one.valid), checked: files.length, files };
|
|
34
|
+
});
|
|
35
|
+
const saveMessage = async (root, account, message, ports, known, beforeWrite, options) => {
|
|
36
|
+
let download;
|
|
37
|
+
let saved = 0;
|
|
38
|
+
let existing = 0;
|
|
39
|
+
for (const [position, attachment] of message.attachments.entries()) {
|
|
40
|
+
if (NOT_FILES.has(attachment.kind))
|
|
41
|
+
continue;
|
|
42
|
+
await beforeWrite();
|
|
43
|
+
const identity = { ...account, chat: message.chatId, message: message.id, position };
|
|
44
|
+
const key = mediaKey(identity);
|
|
45
|
+
if (known.has(key)) {
|
|
46
|
+
existing++;
|
|
47
|
+
continue;
|
|
48
|
+
}
|
|
49
|
+
const entry = await archiveMediaFile(root, identity, attachment.kind, async () => {
|
|
50
|
+
download ??= await ports.messages.download(message.chatId, message.id);
|
|
51
|
+
const positions = matchDownloads(message.attachments.map((one, index) => ({
|
|
52
|
+
position: index,
|
|
53
|
+
kind: one.kind,
|
|
54
|
+
name: one.name ?? null,
|
|
55
|
+
})), download.files);
|
|
56
|
+
const index = positions.indexOf(position);
|
|
57
|
+
const file = download.files[index];
|
|
58
|
+
if (!file)
|
|
59
|
+
throw new CliError("invalid_response", "the messenger did not return an identifiable attachment");
|
|
60
|
+
return file;
|
|
61
|
+
}, beforeWrite, options.signal, { messageTime: new Date(message.timestamp).toISOString(), ...(attachment.mime ? { mime: attachment.mime } : {}) });
|
|
62
|
+
known.set(key, entry);
|
|
63
|
+
saved++;
|
|
64
|
+
options.onSaved?.(entry);
|
|
65
|
+
}
|
|
66
|
+
return { saved, existing };
|
|
67
|
+
};
|
|
68
|
+
export const archiveMedia = async (ports, options) => {
|
|
69
|
+
const since = validateMediaSince(options.sinceTime);
|
|
70
|
+
const pageSize = options.pageSize ?? 100;
|
|
71
|
+
const pauseMs = options.pauseMs ?? 1000;
|
|
72
|
+
if (!Number.isSafeInteger(pageSize) ||
|
|
73
|
+
pageSize < 1 ||
|
|
74
|
+
pageSize > 1000 ||
|
|
75
|
+
!Number.isSafeInteger(pauseMs) ||
|
|
76
|
+
pauseMs < 0)
|
|
77
|
+
throw new CliError("validation_error", "invalid media archive page size or pause");
|
|
78
|
+
return withMediaArchiveLocks(options.directory, options.signal, async (root, beforeWrite) => {
|
|
79
|
+
const known = new Map();
|
|
80
|
+
for (const entry of await readMediaManifests(root, options.signal)) {
|
|
81
|
+
let valid = false;
|
|
82
|
+
try {
|
|
83
|
+
valid = await verifyMediaEntry(root, entry, options.signal);
|
|
84
|
+
}
|
|
85
|
+
catch {
|
|
86
|
+
options.signal?.throwIfAborted();
|
|
87
|
+
}
|
|
88
|
+
if (!valid)
|
|
89
|
+
throw new CliError("validation_error", "archive checksum mismatch or missing file; choose a new destination");
|
|
90
|
+
known.set(mediaKey(entry), entry);
|
|
91
|
+
}
|
|
92
|
+
const account = await ports.chats.account();
|
|
93
|
+
let saved = 0, existing = 0, chats = 0;
|
|
94
|
+
const walk = async (chat) => {
|
|
95
|
+
const seen = new Set();
|
|
96
|
+
let before;
|
|
97
|
+
for (;;) {
|
|
98
|
+
await beforeWrite();
|
|
99
|
+
const page = await ports.messages.list(chat, { limit: pageSize, ...(before ? { before } : {}) });
|
|
100
|
+
let fresh = 0;
|
|
101
|
+
let reachedSince = false;
|
|
102
|
+
let oldest;
|
|
103
|
+
for (const message of page.items) {
|
|
104
|
+
const timestamp = Date.parse(message.timestamp);
|
|
105
|
+
if (!Number.isFinite(timestamp))
|
|
106
|
+
throw new CliError("invalid_response", "media history has an invalid timestamp");
|
|
107
|
+
if (!oldest || timestamp < Date.parse(oldest.timestamp))
|
|
108
|
+
oldest = message;
|
|
109
|
+
if (since !== undefined && timestamp < since) {
|
|
110
|
+
reachedSince = true;
|
|
111
|
+
continue;
|
|
112
|
+
}
|
|
113
|
+
if (seen.has(message.id))
|
|
114
|
+
continue;
|
|
115
|
+
seen.add(message.id);
|
|
116
|
+
fresh++;
|
|
117
|
+
const one = await saveMessage(root, account, message, ports, known, beforeWrite, options);
|
|
118
|
+
saved += one.saved;
|
|
119
|
+
existing += one.existing;
|
|
120
|
+
}
|
|
121
|
+
if (!page.hasMore || reachedSince)
|
|
122
|
+
break;
|
|
123
|
+
if (!oldest || !fresh)
|
|
124
|
+
throw new CliError("invalid_response", "media history pagination did not advance; archive is incomplete");
|
|
125
|
+
before = options.orderByTime ? new Date(Date.parse(oldest.timestamp) + 1).toISOString() : page.items[0]?.id;
|
|
126
|
+
if (pauseMs)
|
|
127
|
+
await sleep(pauseMs, undefined, { signal: options.signal });
|
|
128
|
+
}
|
|
129
|
+
chats++;
|
|
130
|
+
};
|
|
131
|
+
if (options.chats?.length) {
|
|
132
|
+
const ids = new Set();
|
|
133
|
+
for (const chat of options.chats) {
|
|
134
|
+
const found = await ports.chats.show(chat);
|
|
135
|
+
if (!ids.has(found.id)) {
|
|
136
|
+
ids.add(found.id);
|
|
137
|
+
await walk(found.id);
|
|
138
|
+
}
|
|
139
|
+
}
|
|
140
|
+
}
|
|
141
|
+
else {
|
|
142
|
+
const seen = new Set();
|
|
143
|
+
let offset = 0;
|
|
144
|
+
for (;;) {
|
|
145
|
+
await beforeWrite();
|
|
146
|
+
const page = await ports.chats.list({ offset, limit: 100 });
|
|
147
|
+
let fresh = 0;
|
|
148
|
+
for (const chat of page.items)
|
|
149
|
+
if (!seen.has(chat.id)) {
|
|
150
|
+
seen.add(chat.id);
|
|
151
|
+
fresh++;
|
|
152
|
+
await walk(chat.id);
|
|
153
|
+
}
|
|
154
|
+
if (!page.hasMore)
|
|
155
|
+
break;
|
|
156
|
+
if (!fresh)
|
|
157
|
+
throw new CliError("invalid_response", "chat pagination did not advance; archive is incomplete");
|
|
158
|
+
offset += page.items.length;
|
|
159
|
+
if (pauseMs)
|
|
160
|
+
await sleep(pauseMs, undefined, { signal: options.signal });
|
|
161
|
+
}
|
|
162
|
+
}
|
|
163
|
+
return { directory: root, chats, saved, existing, complete: true };
|
|
164
|
+
}, { create: true });
|
|
165
|
+
};
|
|
@@ -50,7 +50,7 @@ export const accountsOfKind = (kind, chosen, held, explicit, command = undefined
|
|
|
50
50
|
throw new CliError("validation_error", `search mail reads mail only — \`${command === undefined ? "" : `${command} `}search messages\` searches ${explicit}`);
|
|
51
51
|
const mail = (listed ? chosen : held).filter(({ provider }) => kindOf(provider) === "mail");
|
|
52
52
|
if (mail.length === 0)
|
|
53
|
-
throw new CliError("not_found", "no mail in the store yet — `memo sync --mail --since-time <time>` imports it", {
|
|
53
|
+
throw new CliError("not_found", "no mail in the store yet — `memo sync --only mail --since-time <time>` imports it", {
|
|
54
54
|
reason: NO_MAIL,
|
|
55
55
|
});
|
|
56
56
|
const address = mailAccount?.trim().toLowerCase();
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@wirecat/cli-core",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.398.0",
|
|
4
4
|
"description": "The messenger-neutral half of a messaging CLI: domain model, message locators, rendering, name resolution, the SQLite seam, the command skeleton and shared read commands, the send guard, run records and the message store",
|
|
5
5
|
"license": "Apache-2.0",
|
|
6
6
|
"author": "Viacheslav Ptsarev",
|
|
@@ -165,7 +165,7 @@
|
|
|
165
165
|
"@anthropic-ai/sdk": "0.131.0",
|
|
166
166
|
"@bomb.sh/tab": "0.0.22",
|
|
167
167
|
"@huggingface/tokenizers": "0.2.0",
|
|
168
|
-
"@modelcontextprotocol/server": "2.
|
|
168
|
+
"@modelcontextprotocol/server": "2.3.0",
|
|
169
169
|
"@resvg/resvg-js": "2.6.2",
|
|
170
170
|
"@valibot/to-json-schema": "1.8.0",
|
|
171
171
|
"@wirecat/cli-core-onnx": "1.0.1",
|