@kin-tio/cli 0.6.0

package/dist/src/services/wecom-sync.js

@@ -0,0 +1,316 @@
+import { normalizeWecomMessage } from '../domain/wecom-message.js';
+import { WecomApiError } from './wecom-api.js';
+const MAX_SYNC_PAGES = 100;
+const CALLBACK_TOKEN_LIFETIME_MS = 9 * 60 * 1_000;
+const RETRY_BASE_MS = 250;
+const RETRY_MAX_MS = 30_000;
+const TOKENLESS_RETRY_MS = 5 * 60_000;
+const PROVIDER_ERROR_RETRY_MS = 5 * 60_000;
+const MAX_CONCURRENT_SYNC_REQUESTS = 4;
+function jitteredRetry(baseMs, key) {
+    let hash = 0;
+    for (const character of key) {
+        hash = ((hash * 31) + (character.codePointAt(0) || 0)) >>> 0;
+    }
+    return baseMs + Math.floor(baseMs * (hash % 21) / 100);
+}
+export class WecomSync {
+    apiClient;
+    store;
+    processor;
+    logger;
+    onDeferredReady;
+    workers = new Map();
+    pending = new Map();
+    liveRequested = new Set();
+    preemptedConversations = new Map();
+    callbackTokens = new Map();
+    retries = new Map();
+    syncWaiters = [];
+    activeSyncRequests = 0;
+    accepting = true;
+    consuming = false;
+    constructor({ apiClient, store, processor, logger = console, startPaused = false, onDeferredReady = () => { }, }) {
+        this.apiClient = apiClient;
+        this.store = store;
+        this.processor = processor;
+        this.logger = logger;
+        this.onDeferredReady = onDeferredReady;
+        this.consuming = !startPaused;
+    }
+    enqueue({ callbackToken, openKfId, }) {
+        if (!this.accepting)
+            return false;
+        this.store.registerSyncAccountKey(openKfId);
+        this.callbackTokens.set(openKfId, {
+            value: callbackToken,
+            expiresAt: Date.now() + CALLBACK_TOKEN_LIFETIME_MS,
+        });
+        void this.#requestSync(openKfId, false, 0);
+        return true;
+    }
+    catchUp() {
+        if (!this.accepting)
+            return Promise.resolve();
+        if (this.consuming) {
+            throw new Error('startup catch-up requires paused consumption');
+        }
+        const openKfIds = this.store.listSyncAccountKeys();
+        if (openKfIds.length) {
+            this.logger.info?.(`[wecom] startup catch-up accounts=${openKfIds.length}`);
+        }
+        return Promise.all(openKfIds.map((openKfId) => this.#requestSync(openKfId, true, 0))).then(() => undefined);
+    }
+    #requestSync(openKfId, deferred, attempt) {
+        const current = this.pending.get(openKfId);
+        this.pending.set(openKfId, {
+            live: Boolean(current?.live || !deferred),
+            deferred: Boolean(current?.deferred || deferred),
+            attempt: current ? Math.min(current.attempt, attempt) : attempt,
+        });
+        if (!deferred)
+            this.liveRequested.add(openKfId);
+        this.#cancelRetry(openKfId);
+        return this.#startWorker(openKfId);
+    }
+    #startWorker(openKfId) {
+        const current = this.workers.get(openKfId);
+        if (current)
+            return current;
+        const worker = this.#runWorker(openKfId).catch((error) => {
+            this.logger.error?.(`[wecom] sync worker failed: ${error instanceof Error ? error.message : String(error)}`);
+        }).finally(() => {
+            if (this.workers.get(openKfId) !== worker)
+                return;
+            this.workers.delete(openKfId);
+            if (this.accepting
+                && this.pending.has(openKfId)
+                && !this.retries.has(openKfId))
+                void this.#startWorker(openKfId);
+        });
+        this.workers.set(openKfId, worker);
+        return worker;
+    }
+    async #runWorker(openKfId) {
+        while (this.accepting) {
+            const request = this.pending.get(openKfId);
+            if (!request)
+                return;
+            this.pending.delete(openKfId);
+            if (request.live)
+                this.liveRequested.delete(openKfId);
+            const deferred = !request.live && request.deferred;
+            try {
+                const completed = await this.#drain(this.#callbackToken(openKfId), openKfId, deferred);
+                if (deferred && completed)
+                    this.onDeferredReady();
+            }
+            catch (error) {
+                if (!this.accepting)
+                    return;
+                const current = this.pending.get(openKfId);
+                this.pending.set(openKfId, {
+                    live: Boolean(current?.live || request.live),
+                    deferred: Boolean(current?.deferred || request.deferred),
+                    attempt: current
+                        ? Math.min(current.attempt, request.attempt + 1)
+                        : request.attempt + 1,
+                });
+                this.#scheduleRetry(openKfId, error);
+                return;
+            }
+        }
+    }
+    #callbackToken(openKfId) {
+        const current = this.callbackTokens.get(openKfId);
+        if (!current)
+            return '';
+        if (current.expiresAt > Date.now())
+            return current.value;
+        this.callbackTokens.delete(openKfId);
+        return '';
+    }
+    #scheduleRetry(openKfId, error) {
+        this.#cancelRetry(openKfId);
+        const attempt = this.pending.get(openKfId)?.attempt || 1;
+        const hasCallbackToken = Boolean(this.#callbackToken(openKfId));
+        const providerRejected = error instanceof WecomApiError
+            && error.code !== undefined;
+        const retryBase = !hasCallbackToken
+            ? TOKENLESS_RETRY_MS
+            : providerRejected
+                ? PROVIDER_ERROR_RETRY_MS
+                : Math.min(RETRY_BASE_MS * (2 ** Math.min(Math.max(0, attempt - 1), 16)), RETRY_MAX_MS);
+        const retryMs = retryBase >= PROVIDER_ERROR_RETRY_MS
+            ? jitteredRetry(retryBase, `${openKfId}:${attempt}`)
+            : retryBase;
+        this.logger.error?.(`[wecom] sync failed: ${error instanceof Error ? error.message : String(error)}; retry_ms=${retryMs}`);
+        const timer = setTimeout(() => {
+            if (this.retries.get(openKfId)?.timer !== timer)
+                return;
+            this.retries.delete(openKfId);
+            if (this.accepting)
+                void this.#startWorker(openKfId);
+        }, retryMs);
+        timer.unref?.();
+        this.retries.set(openKfId, { timer });
+    }
+    #cancelRetry(openKfId) {
+        const retry = this.retries.get(openKfId);
+        if (!retry)
+            return;
+        clearTimeout(retry.timer);
+        this.retries.delete(openKfId);
+    }
+    #cancelRetries() {
+        for (const { timer } of this.retries.values()) {
+            clearTimeout(timer);
+        }
+        this.retries.clear();
+    }
+    #acquireSyncSlot() {
+        if (!this.accepting)
+            return Promise.resolve(false);
+        if (this.activeSyncRequests < MAX_CONCURRENT_SYNC_REQUESTS) {
+            this.activeSyncRequests += 1;
+            return Promise.resolve(true);
+        }
+        return new Promise((resolve) => this.syncWaiters.push(resolve));
+    }
+    #releaseSyncSlot() {
+        this.activeSyncRequests -= 1;
+        while (this.syncWaiters.length) {
+            const waiter = this.syncWaiters.shift();
+            if (!waiter)
+                continue;
+            if (!this.accepting) {
+                waiter(false);
+                continue;
+            }
+            this.activeSyncRequests += 1;
+            waiter(true);
+            return;
+        }
+    }
+    #cancelSyncWaiters() {
+        for (const waiter of this.syncWaiters.splice(0))
+            waiter(false);
+    }
+    async #drain(callbackToken, openKfId, deferred) {
+        let cursor = this.store.getCursor(openKfId);
+        const liveConversationPages = [];
+        let liveDrainCompleted = false;
+        try {
+            for (let page = 0; page < MAX_SYNC_PAGES; page += 1) {
+                const acquired = await this.#acquireSyncSlot();
+                if (!acquired)
+                    return false;
+                let result;
+                try {
+                    if (!this.accepting)
+                        return false;
+                    result = await this.apiClient.syncMessages({
+                        cursor,
+                        callbackToken,
+                        openKfId,
+                    });
+                }
+                finally {
+                    this.#releaseSyncSlot();
+                }
+                if (!this.accepting)
+                    return false;
+                const nextCursor = String(result.next_cursor || cursor);
+                if (result.has_more === 1 && nextCursor === cursor) {
+                    throw new Error('sync_msg returned has_more=1 without a new cursor');
+                }
+                const messages = result.msg_list.map((raw, index) => normalizeWecomMessage(raw, openKfId, { cursor, index }));
+                const ingested = this.store.ingestSyncPage({
+                    accountKey: openKfId,
+                    expectedCursor: cursor,
+                    nextCursor,
+                    messages,
+                    deferred,
+                });
+                cursor = ingested.cursor;
+                const interruptedByLive = deferred && this.liveRequested.has(openKfId);
+                if (interruptedByLive) {
+                    const pending = this.preemptedConversations.get(openKfId) || new Set();
+                    for (const message of messages) {
+                        pending.add(`wechat_kf\0${message.conversation.accountKey}\0${message.conversation.peerId}`);
+                    }
+                    this.preemptedConversations.set(openKfId, pending);
+                    return false;
+                }
+                if (!deferred) {
+                    const liveConversations = new Set(messages.map((message) => `wechat_kf\0${message.conversation.accountKey}\0${message.conversation.peerId}`));
+                    liveConversationPages.push([...liveConversations]);
+                }
+                if (result.has_more !== 1) {
+                    liveDrainCompleted = true;
+                    return true;
+                }
+            }
+            throw new Error(`sync_msg exceeded ${MAX_SYNC_PAGES} pages`);
+        }
+        finally {
+            if (this.accepting
+                && !deferred
+                && (liveDrainCompleted || liveConversationPages.some((page) => page.length > 0))) {
+                this.#enqueueLiveConversations(openKfId, liveConversationPages);
+            }
+        }
+    }
+    #enqueueLiveConversations(openKfId, pages) {
+        const latestFirst = new Set();
+        for (const page of [...pages].reverse()) {
+            for (const conversation of page)
+                latestFirst.add(conversation);
+        }
+        const conversations = latestFirst.size
+            ? latestFirst
+            : this.preemptedConversations.get(openKfId) || new Set();
+        this.preemptedConversations.delete(openKfId);
+        if (!this.consuming)
+            return;
+        const seen = new Set();
+        for (const conversation of conversations) {
+            const [channel = '', accountKey = '', peerId = ''] = conversation.split('\0');
+            if (channel !== 'wechat_kf')
+                continue;
+            for (const record of this.store.promoteDeferredConversation({
+                channel,
+                accountKey,
+                peerId,
+            })) {
+                if (seen.has(record.messageKey))
+                    continue;
+                seen.add(record.messageKey);
+                void this.processor.enqueue(record.messageKey).catch((error) => {
+                    this.logger.error?.(`[wecom] live enqueue failed: ${error instanceof Error ? error.message : String(error)}`);
+                });
+            }
+        }
+    }
+    async waitForIdle() {
+        while (this.workers.size) {
+            await Promise.allSettled([...this.workers.values()]);
+        }
+    }
+    startConsuming() {
+        this.consuming = true;
+    }
+    stopAccepting() {
+        if (!this.accepting)
+            return;
+        this.accepting = false;
+        this.callbackTokens.clear();
+        this.pending.clear();
+        this.#cancelRetries();
+        this.#cancelSyncWaiters();
+    }
+    async close() {
+        this.stopAccepting();
+        await this.waitForIdle();
+    }
+}

package/dist/src/state/persistence.js

@@ -0,0 +1,124 @@
+import fs from 'node:fs';
+import path from 'node:path';
+import { DatabaseSync } from 'node:sqlite';
+import { IlinkLoginStore } from '../ilink/login-store.js';
+import { IlinkSqliteStore } from '../ilink/sqlite-store.js';
+import { ensurePrivateDirectory } from '../lib/private-directory.js';
+import { secureSqliteFiles, SqliteStore, } from './sqlite-store.js';
+export class StatePersistenceUnclosedError extends AggregateError {
+    constructor(errors, message) {
+        super(errors, message);
+        this.name = 'StatePersistenceUnclosedError';
+    }
+}
+export class StatePersistence {
+    #database;
+    core;
+    #closed = false;
+    get closed() {
+        return this.#closed;
+    }
+    constructor(options) {
+        if (!options.filePath)
+            throw new Error('SQLite filePath is required');
+        if (options.journalMode !== undefined &&
+            !['WAL', 'DELETE'].includes(options.journalMode)) {
+            throw new Error(`Unsupported SQLite journal mode: ${options.journalMode}`);
+        }
+        const filePath = path.resolve(options.filePath);
+        ensurePrivateDirectory(path.dirname(filePath));
+        this.#database = new DatabaseSync(filePath);
+        try {
+            this.core = new SqliteStore({
+                filePath,
+                ...(options.clock ? { clock: options.clock } : {}),
+                ...(options.journalMode ? { journalMode: options.journalMode } : {}),
+            }, {
+                database: this.#database,
+            });
+        }
+        catch (error) {
+            try {
+                this.#database.close();
+            }
+            catch (closeError) {
+                throw new StatePersistenceUnclosedError([error, closeError], 'SQLite initialization and cleanup both failed');
+            }
+            throw error;
+        }
+    }
+    static hasActiveWriter(filePath) {
+        try {
+            fs.statSync(filePath);
+        }
+        catch (error) {
+            if (error.code === 'ENOENT')
+                return false;
+            throw error;
+        }
+        let database;
+        try {
+            database = new DatabaseSync(filePath);
+            database.exec('PRAGMA busy_timeout = 0');
+            database.exec('BEGIN IMMEDIATE');
+            database.exec('ROLLBACK');
+            return false;
+        }
+        catch (error) {
+            if (error instanceof Error &&
+                'code' in error &&
+                String(error.code) === 'ERR_SQLITE_ERROR' &&
+                /busy|locked/iu.test(error.message)) {
+                return true;
+            }
+            throw error;
+        }
+        finally {
+            database?.close();
+        }
+    }
+    createIlinkStore(options = {}) {
+        this.#assertOpen();
+        return new IlinkSqliteStore({
+            database: this.#database,
+            inbox: this.core,
+            ...(options.clock ? { clock: options.clock } : {}),
+        });
+    }
+    createIlinkLoginStore(options) {
+        this.#assertOpen();
+        return new IlinkLoginStore({
+            store: this.core,
+            database: this.#database,
+            secretBox: options.secretBox,
+            ...(options.clock ? { clock: options.clock } : {}),
+        });
+    }
+    close() {
+        if (this.#closed)
+            return;
+        let secureError;
+        try {
+            secureSqliteFiles(this.core.filePath);
+        }
+        catch (error) {
+            secureError = error;
+        }
+        try {
+            this.#database.close();
+        }
+        catch (closeError) {
+            if (secureError !== undefined) {
+                throw new StatePersistenceUnclosedError([secureError, closeError], 'SQLite file hardening and close both failed');
+            }
+            throw closeError;
+        }
+        this.#closed = true;
+        if (secureError !== undefined)
+            throw secureError;
+    }
+    #assertOpen() {
+        if (this.#closed)
+            throw new Error('State persistence is closed');
+    }
+}