@wirecat/cli-core 0.396.0 → 0.397.0

package/dist/services/media-archive-move.d.ts

@@ -0,0 +1,21 @@
+export interface MoveMediaArchiveOptions {
+    directory: string;
+    archiveDirectory: string;
+    beforeTime: string;
+    dryRun?: boolean;
+    signal?: AbortSignal;
+}
+/** The destination is durable before a source relocation record is published, then the verified local copy is removed. */
+export declare const moveMediaArchive: (options: MoveMediaArchiveOptions) => Promise<{
+    directory: string;
+    archiveDirectory: string;
+    beforeTime: string;
+    selected: number;
+    selectedBytes: number;
+    moved: number;
+    movedBytes: number;
+    alreadyArchived: number;
+    undated: number;
+    complete: boolean;
+    dryRun: boolean;
+}>;

package/dist/services/media-archive-move.js

@@ -0,0 +1,163 @@
+import { realpath, unlink } from "node:fs/promises";
+import { basename, dirname, join, resolve } from "node:path";
+import { CliError } from "@wirecat/cli-base";
+import { openOrdinaryFile, sameOrdinaryFile, syncDirectory, writeFileAtomic } from "../files/index.js";
+import { validateMediaSince } from "./media-archive.js";
+import { archiveMediaFile } from "./media-archive-files.js";
+import { withMediaArchiveLocks } from "./media-archive-locks.js";
+import { mediaFingerprint, mediaKey, mediaManifestName, missingMediaFile, parseMediaEntry, readMediaManifests, sameMediaCopy, verifyMediaEntry, } from "./media-archive-manifest.js";
+const canonicalDestination = async (directory) => {
+    const path = resolve(directory);
+    try {
+        return await realpath(path);
+    }
+    catch (error) {
+        if (!missingMediaFile(error) || dirname(path) === path)
+            throw error;
+        return join(await canonicalDestination(dirname(path)), basename(path));
+    }
+};
+const image = (entry) => ["photo", "image"].includes(entry.kind) || entry.mime?.startsWith("image/");
+const removeLocalCopy = async (root, entry, beforeWrite, signal) => {
+    const path = join(root, entry.path);
+    let file;
+    try {
+        file = await openOrdinaryFile(path);
+    }
+    catch (error) {
+        if (missingMediaFile(error))
+            return;
+        throw error;
+    }
+    try {
+        const identity = await file.stat({ bigint: true });
+        const proof = await mediaFingerprint(path, signal);
+        if (proof.size !== entry.size || proof.sha256 !== entry.sha256 || !(await sameOrdinaryFile(path, identity)))
+            throw new CliError("validation_error", "local image changed during rotation; it was preserved");
+        if (!(await verifyMediaEntry(root, entry, signal)))
+            throw new CliError("validation_error", "rotated image checksum mismatch; the local copy was preserved");
+        await beforeWrite();
+        if (!(await sameOrdinaryFile(path, identity)))
+            throw new CliError("validation_error", "local image was replaced; it was preserved");
+        await unlink(path);
+        await syncDirectory(root);
+    }
+    finally {
+        await file.close();
+    }
+};
+/** The destination is durable before a source relocation record is published, then the verified local copy is removed. */
+export const moveMediaArchive = async (options) => {
+    const cutoff = validateMediaSince(options.beforeTime, "--before-time");
+    if (cutoff === undefined)
+        throw new CliError("validation_error", "--before-time is required");
+    const target = await canonicalDestination(options.archiveDirectory);
+    const source = await realpath(options.directory);
+    if (source === target)
+        throw new CliError("validation_error", "choose a different archive destination");
+    return withMediaArchiveLocks(source, options.signal, async (root, beforeWrite) => {
+        const entries = await readMediaManifests(root, options.signal);
+        if (!entries.length)
+            throw new CliError("not_found", "no media archive manifests in this directory");
+        const targetEntries = await readMediaManifests(target, options.signal).catch((error) => {
+            if (options.dryRun && missingMediaFile(error))
+                return [];
+            throw error;
+        });
+        for (const [folder, manifests] of [
+            [root, entries],
+            [target, targetEntries],
+        ])
+            for (const entry of manifests) {
+                let valid = false;
+                try {
+                    valid = await verifyMediaEntry(folder, entry, options.signal);
+                }
+                catch {
+                    options.signal?.throwIfAborted();
+                }
+                if (!valid)
+                    throw new CliError("validation_error", "archive checksum mismatch or missing file; no images were moved");
+            }
+        const known = new Map(targetEntries.map((entry) => [mediaKey(entry), entry]));
+        let selected = 0, moved = 0, movedBytes = 0, selectedBytes = 0, undated = 0, alreadyArchived = 0;
+        const candidates = [];
+        for (const entry of entries) {
+            if (!image(entry))
+                continue;
+            if (!entry.messageTime) {
+                undated++;
+                continue;
+            }
+            if (Date.parse(entry.messageTime) >= cutoff)
+                continue;
+            if (entry.relocatedTo) {
+                alreadyArchived++;
+                if (!options.dryRun)
+                    await removeLocalCopy(root, entry, beforeWrite, options.signal);
+                continue;
+            }
+            const existing = known.get(mediaKey(entry));
+            if (existing?.relocatedTo)
+                throw new CliError("validation_error", "destination image is already relocated; choose its current archive directory");
+            if (existing && !sameMediaCopy(entry, existing))
+                throw new CliError("validation_error", "destination has a different copy of this image; choose a new destination");
+            candidates.push(entry);
+            selected++;
+            selectedBytes += entry.size;
+        }
+        if (!options.dryRun)
+            for (const entry of candidates) {
+                await beforeWrite();
+                if (!known.has(mediaKey(entry))) {
+                    const file = await openOrdinaryFile(join(root, entry.path));
+                    try {
+                        const identity = {
+                            provider: entry.provider,
+                            account: entry.account,
+                            chat: entry.chat,
+                            message: entry.message,
+                            position: entry.position,
+                        };
+                        const copied = await archiveMediaFile(target, identity, entry.kind, async () => ({
+                            kind: entry.kind,
+                            size: entry.size,
+                            bytes: () => file.createReadStream({ autoClose: false, ...(options.signal ? { signal: options.signal } : {}) }),
+                        }), beforeWrite, options.signal, {
+                            messageTime: entry.messageTime,
+                            mime: entry.mime,
+                            expected: { size: entry.size, sha256: entry.sha256 },
+                        });
+                        known.set(mediaKey(entry), copied);
+                    }
+                    finally {
+                        await file.close();
+                    }
+                }
+                const receipt = parseMediaEntry({ ...entry, relocatedTo: target });
+                if (!(await verifyMediaEntry(root, receipt, options.signal)))
+                    throw new CliError("validation_error", "archive destination changed before rotation; the original was preserved");
+                await beforeWrite();
+                await writeFileAtomic(join(root, mediaManifestName(entry)), JSON.stringify(receipt), {
+                    beforePublish: beforeWrite,
+                    signal: options.signal,
+                });
+                await removeLocalCopy(root, receipt, beforeWrite, options.signal);
+                moved++;
+                movedBytes += entry.size;
+            }
+        return {
+            directory: root,
+            archiveDirectory: target,
+            beforeTime: new Date(cutoff).toISOString(),
+            selected,
+            selectedBytes,
+            moved,
+            movedBytes,
+            alreadyArchived,
+            undated,
+            complete: undated === 0,
+            dryRun: options.dryRun === true,
+        };
+    }, { additional: [target], createAdditional: !options.dryRun });
+};

package/dist/services/media-archive.d.ts

@@ -0,0 +1,34 @@
+import type { ChatsService } from "./chats.js";
+import { type MediaArchiveEntry } from "./media-archive-manifest.js";
+import type { MessagesService } from "./messages.js";
+export interface MediaArchivePorts {
+    chats: Pick<ChatsService, "list" | "account" | "show">;
+    messages: Pick<MessagesService, "list" | "download">;
+}
+export interface MediaArchiveOptions {
+    directory: string;
+    chats?: readonly string[];
+    sinceTime?: string;
+    orderByTime?: boolean;
+    pageSize?: number;
+    pauseMs?: number;
+    signal?: AbortSignal;
+    onSaved?: (entry: MediaArchiveEntry) => void;
+}
+export declare const validateMediaSince: (value?: string, flag?: string) => number | undefined;
+export declare const checkMediaArchive: (directory: string, signal?: AbortSignal) => Promise<{
+    valid: boolean;
+    checked: number;
+    files: {
+        path: string;
+        valid: boolean;
+        archiveDirectory?: string;
+    }[];
+}>;
+export declare const archiveMedia: (ports: MediaArchivePorts, options: MediaArchiveOptions) => Promise<{
+    directory: string;
+    chats: number;
+    saved: number;
+    existing: number;
+    complete: boolean;
+}>;

package/dist/services/media-archive.js

@@ -0,0 +1,165 @@
+import { setTimeout as sleep } from "node:timers/promises";
+import { CliError } from "@wirecat/cli-base";
+import { matchDownloads, NOT_FILES } from "../domain/attachments.js";
+import { archiveMediaFile } from "./media-archive-files.js";
+import { withMediaArchiveLocks } from "./media-archive-locks.js";
+import { mediaKey, readMediaManifests, verifyMediaEntry } from "./media-archive-manifest.js";
+export const validateMediaSince = (value, flag = "--since-time") => {
+    if (value === undefined)
+        return undefined;
+    if (!/^\d{4}-\d\d-\d\dT\d\d:\d\d:\d\d(?:\.\d{1,3})?(?:Z|[+-]\d\d:\d\d)$/.test(value) ||
+        !Number.isFinite(Date.parse(value)))
+        throw new CliError("validation_error", `${flag} needs an ISO timestamp with a time zone`);
+    const date = value.slice(0, 10);
+    if (new Date(`${date}T00:00:00Z`).toISOString().slice(0, 10) !== date)
+        throw new CliError("validation_error", `${flag} has an invalid calendar date`);
+    return Date.parse(value);
+};
+export const checkMediaArchive = (directory, signal) => withMediaArchiveLocks(directory, signal, async (root) => {
+    const entries = await readMediaManifests(root, signal);
+    if (!entries.length)
+        throw new CliError("not_found", "no media archive manifests in this directory");
+    const files = [];
+    for (const entry of entries) {
+        let valid = false;
+        try {
+            valid = await verifyMediaEntry(root, entry, signal);
+        }
+        catch {
+            signal?.throwIfAborted();
+        }
+        files.push({ path: entry.path, valid, ...(entry.relocatedTo ? { archiveDirectory: entry.relocatedTo } : {}) });
+    }
+    return { valid: files.every((one) => one.valid), checked: files.length, files };
+});
+const saveMessage = async (root, account, message, ports, known, beforeWrite, options) => {
+    let download;
+    let saved = 0;
+    let existing = 0;
+    for (const [position, attachment] of message.attachments.entries()) {
+        if (NOT_FILES.has(attachment.kind))
+            continue;
+        await beforeWrite();
+        const identity = { ...account, chat: message.chatId, message: message.id, position };
+        const key = mediaKey(identity);
+        if (known.has(key)) {
+            existing++;
+            continue;
+        }
+        const entry = await archiveMediaFile(root, identity, attachment.kind, async () => {
+            download ??= await ports.messages.download(message.chatId, message.id);
+            const positions = matchDownloads(message.attachments.map((one, index) => ({
+                position: index,
+                kind: one.kind,
+                name: one.name ?? null,
+            })), download.files);
+            const index = positions.indexOf(position);
+            const file = download.files[index];
+            if (!file)
+                throw new CliError("invalid_response", "the messenger did not return an identifiable attachment");
+            return file;
+        }, beforeWrite, options.signal, { messageTime: new Date(message.timestamp).toISOString(), ...(attachment.mime ? { mime: attachment.mime } : {}) });
+        known.set(key, entry);
+        saved++;
+        options.onSaved?.(entry);
+    }
+    return { saved, existing };
+};
+export const archiveMedia = async (ports, options) => {
+    const since = validateMediaSince(options.sinceTime);
+    const pageSize = options.pageSize ?? 100;
+    const pauseMs = options.pauseMs ?? 1000;
+    if (!Number.isSafeInteger(pageSize) ||
+        pageSize < 1 ||
+        pageSize > 1000 ||
+        !Number.isSafeInteger(pauseMs) ||
+        pauseMs < 0)
+        throw new CliError("validation_error", "invalid media archive page size or pause");
+    return withMediaArchiveLocks(options.directory, options.signal, async (root, beforeWrite) => {
+        const known = new Map();
+        for (const entry of await readMediaManifests(root, options.signal)) {
+            let valid = false;
+            try {
+                valid = await verifyMediaEntry(root, entry, options.signal);
+            }
+            catch {
+                options.signal?.throwIfAborted();
+            }
+            if (!valid)
+                throw new CliError("validation_error", "archive checksum mismatch or missing file; choose a new destination");
+            known.set(mediaKey(entry), entry);
+        }
+        const account = await ports.chats.account();
+        let saved = 0, existing = 0, chats = 0;
+        const walk = async (chat) => {
+            const seen = new Set();
+            let before;
+            for (;;) {
+                await beforeWrite();
+                const page = await ports.messages.list(chat, { limit: pageSize, ...(before ? { before } : {}) });
+                let fresh = 0;
+                let reachedSince = false;
+                let oldest;
+                for (const message of page.items) {
+                    const timestamp = Date.parse(message.timestamp);
+                    if (!Number.isFinite(timestamp))
+                        throw new CliError("invalid_response", "media history has an invalid timestamp");
+                    if (!oldest || timestamp < Date.parse(oldest.timestamp))
+                        oldest = message;
+                    if (since !== undefined && timestamp < since) {
+                        reachedSince = true;
+                        continue;
+                    }
+                    if (seen.has(message.id))
+                        continue;
+                    seen.add(message.id);
+                    fresh++;
+                    const one = await saveMessage(root, account, message, ports, known, beforeWrite, options);
+                    saved += one.saved;
+                    existing += one.existing;
+                }
+                if (!page.hasMore || reachedSince)
+                    break;
+                if (!oldest || !fresh)
+                    throw new CliError("invalid_response", "media history pagination did not advance; archive is incomplete");
+                before = options.orderByTime ? new Date(Date.parse(oldest.timestamp) + 1).toISOString() : page.items[0]?.id;
+                if (pauseMs)
+                    await sleep(pauseMs, undefined, { signal: options.signal });
+            }
+            chats++;
+        };
+        if (options.chats?.length) {
+            const ids = new Set();
+            for (const chat of options.chats) {
+                const found = await ports.chats.show(chat);
+                if (!ids.has(found.id)) {
+                    ids.add(found.id);
+                    await walk(found.id);
+                }
+            }
+        }
+        else {
+            const seen = new Set();
+            let offset = 0;
+            for (;;) {
+                await beforeWrite();
+                const page = await ports.chats.list({ offset, limit: 100 });
+                let fresh = 0;
+                for (const chat of page.items)
+                    if (!seen.has(chat.id)) {
+                        seen.add(chat.id);
+                        fresh++;
+                        await walk(chat.id);
+                    }
+                if (!page.hasMore)
+                    break;
+                if (!fresh)
+                    throw new CliError("invalid_response", "chat pagination did not advance; archive is incomplete");
+                offset += page.items.length;
+                if (pauseMs)
+                    await sleep(pauseMs, undefined, { signal: options.signal });
+            }
+        }
+        return { directory: root, chats, saved, existing, complete: true };
+    }, { create: true });
+};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@wirecat/cli-core",
-  "version": "0.396.0",
+  "version": "0.397.0",
   "description": "The messenger-neutral half of a messaging CLI: domain model, message locators, rendering, name resolution, the SQLite seam, the command skeleton and shared read commands, the send guard, run records and the message store",
   "license": "Apache-2.0",
   "author": "Viacheslav Ptsarev",

package/parity.json

@@ -121,6 +121,10 @@
       "value": "<n>",
       "meaning": "how many messages before it"
     },
+    "--before-time": {
+      "value": "<time>",
+      "meaning": "read what came before this time"
+    },
     "--block": {
       "value": "<value>",
       "meaning": "Set to `true` if user should be blocked in chat."
@@ -1287,12 +1291,26 @@
         "--description": "all",
         "--first-name": "all",
         "--last-name": "all",
-        "--photo": "all"
+        "--photo": {
+          "in": "all",
+          "wording": "WhatsApp accepts JPEG (.jpg or .jpeg) profile photos; help names each messenger's accepted image formats (WC-CMD-25)."
+        }
       }
     },
     "attachments": {
       "in": "all"
     },
+    "attachments check": {
+      "in": "all"
+    },
+    "attachments download": {
+      "in": "all",
+      "options": {
+        "--chat": "all",
+        "--pause": "all",
+        "--since-time": "all"
+      }
+    },
     "attachments extract": {
       "in": "all",
       "options": {
@@ -1318,6 +1336,30 @@
         }
       }
     },
+    "attachments jobs": {
+      "in": "all"
+    },
+    "attachments jobs install": {
+      "in": "all",
+      "options": {
+        "--at": "all",
+        "--chat": "all",
+        "--since-time": "all",
+        "--timezone": "all"
+      }
+    },
+    "attachments jobs start": {
+      "in": "all"
+    },
+    "attachments jobs status": {
+      "in": "all"
+    },
+    "attachments jobs stop": {
+      "in": "all"
+    },
+    "attachments jobs uninstall": {
+      "in": "all"
+    },
     "attachments list": {
       "in": "all",
       "options": {
@@ -1332,6 +1374,13 @@
         }
       }
     },
+    "attachments move": {
+      "in": "all",
+      "options": {
+        "--before-time": "all",
+        "--dry-run": "all"
+      }
+    },
     "attachments show": {
       "in": "all",
       "options": {