@rikaidev/yomi 0.5.0 → 0.6.0

package/dist/ambient/ingestion-adapter.js

@@ -0,0 +1,203 @@
+import { sanitizeMessagePreview } from '../line/core/message-preview.js';
+import { resolveConversationNames, resolveSenderNames } from '../mcp/names.js';
+import { computeEventId } from './journal/crypto.js';
+function timestamp(message) {
+    return Number(message?.deliveredTime || message?.createdTime || 0);
+}
+function mediaType(contentType) {
+    const map = {
+        1: 'image',
+        2: 'video',
+        3: 'audio',
+        7: 'sticker',
+        14: 'file',
+    };
+    return map[contentType] ?? 'text';
+}
+function mentionMids(metadata) {
+    try {
+        const value = typeof metadata?.MENTION === 'string'
+            ? JSON.parse(metadata.MENTION)
+            : metadata?.MENTION;
+        const mentions = value?.MENTIONEES ?? value?.MENTIONS ?? [];
+        return mentions
+            .map((item) => item?.M ?? item?.mid)
+            .filter((mid) => typeof mid === 'string');
+    }
+    catch {
+        return [];
+    }
+}
+export class LineIngestionAdapter {
+    service;
+    constructor(service) {
+        this.service = service;
+    }
+    async collectRecentEvents(options) {
+        const windowStart = options.windowStart;
+        const maxChats = options.maxChats ?? 40;
+        const maxMessagesPerChat = options.maxMessagesPerChat ?? 100;
+        const boxesResult = await this.service.client.getAllMessageBoxes({
+            messageBoxCountLimit: 200,
+            lastMessagesPerMessageBoxCount: 1,
+            withUnreadCount: false,
+        });
+        const boxes = (boxesResult.messageBoxes ?? [])
+            .filter((box) => {
+            const last = box.lastMessages?.[0];
+            return (Number(last?.deliveredTime ||
+                box.lastDeliveredMessageId?.deliveredTime ||
+                0) >= windowStart);
+        })
+            .slice(0, 80);
+        if ((boxesResult.messageBoxes ?? []).length === 0) {
+            throw new Error('LINE returned no message boxes for ambient observation');
+        }
+        const chatIds = boxes.map((box) => String(box.id)).filter(Boolean);
+        const resolvedChatNames = await resolveConversationNames(this.service, chatIds);
+        const chatNames = new Map([...resolvedChatNames].map(([id, name]) => [id, name ?? 'LINE 對話']));
+        const events = [];
+        const officialChatIds = new Set();
+        const userChatIds = chatIds.filter((id) => id.startsWith('u'));
+        if (userChatIds.length > 0) {
+            try {
+                const contacts = await this.service.client.getContacts(userChatIds);
+                for (const contact of contacts ?? []) {
+                    if (contact?.mid && contact?.isOfficial) {
+                        officialChatIds.add(String(contact.mid));
+                    }
+                }
+            }
+            catch { }
+        }
+        const observableChatIds = chatIds
+            .filter((chatId) => !officialChatIds.has(chatId))
+            .slice(0, maxChats);
+        let coverageLimited = false;
+        for (const chatId of observableChatIds) {
+            const recent = await this.service.getRecentMessages(chatId, maxMessagesPerChat);
+            if (recent.length >= maxMessagesPerChat &&
+                Math.min(...recent.map((message) => timestamp(message))) >
+                    windowStart) {
+                coverageLimited = true;
+            }
+            const messages = recent.filter((message) => timestamp(message) >= windowStart);
+            const mids = [
+                ...new Set(messages
+                    .flatMap((message) => [
+                    message.from,
+                    ...mentionMids(message.contentMetadata),
+                ])
+                    .filter(Boolean)),
+            ];
+            const names = await resolveSenderNames(this.service, mids);
+            for (const message of messages) {
+                const type = mediaType(Number(message.contentType ?? 0));
+                const text = sanitizeMessagePreview(message.text) || null;
+                const mNames = mentionMids(message.contentMetadata)
+                    .map((mid) => names.get(mid))
+                    .filter((name) => Boolean(name));
+                const isSelf = message.from === this.service.profile?.mid;
+                const mentionsSelf = mentionMids(message.contentMetadata).includes(String(this.service.profile?.mid ?? ''));
+                const msgId = String(message.id);
+                events.push({
+                    eventId: computeEventId(chatId, msgId),
+                    chatId,
+                    messageId: msgId,
+                    opRevision: message.opRevision ? Number(message.opRevision) : null,
+                    senderMid: message.from ? String(message.from) : null,
+                    senderName: names.get(message.from) ?? null,
+                    sentAt: timestamp(message),
+                    contentType: type,
+                    text,
+                    metadata: {
+                        mentionNames: mNames,
+                        mentionMids: mentionMids(message.contentMetadata),
+                        fileName: message.fileName ? String(message.fileName) : undefined,
+                        stickerId: type === 'sticker' && message.contentMetadata?.STKID
+                            ? String(message.contentMetadata.STKID)
+                            : undefined,
+                        sourceUrl: type === 'sticker' && message.contentMetadata?.STKID
+                            ? `https://stickershop.line-scdn.net/stickershop/v1/sticker/${message.contentMetadata.STKID}/android/sticker.png`
+                            : null,
+                    },
+                    isSelf,
+                    mentionsSelf,
+                    ocrEligible: true,
+                });
+            }
+        }
+        try {
+            const joinedSquareChats = await this.service.client.getJoinedSquareChats(40);
+            const hydratedSquareChats = await Promise.all(joinedSquareChats.map(async (chat) => {
+                if (chat.lastMessage)
+                    return chat;
+                try {
+                    const status = await this.service.client.getSquareChatStatus(chat.id);
+                    return { ...chat, lastMessage: status?.[3]?.[1] ?? null };
+                }
+                catch {
+                    return chat;
+                }
+            }));
+            const activeSquareChats = hydratedSquareChats.filter((chat) => {
+                const last = chat.lastMessage;
+                return Number(last?.[6] || last?.[5] || 0) >= windowStart;
+            });
+            if (activeSquareChats.length > 20)
+                coverageLimited = true;
+            for (const chat of activeSquareChats.slice(0, 20)) {
+                const recent = await this.service.client.getSquareChatMessages(chat.id, maxMessagesPerChat);
+                if (recent.length >= maxMessagesPerChat &&
+                    Math.min(...recent.map((message) => Number(message.createdTime ?? 0))) > windowStart) {
+                    coverageLimited = true;
+                }
+                const messages = recent.filter((message) => Number(message.createdTime ?? 0) >= windowStart);
+                chatNames.set(chat.id, chat.name ?? 'LINE 社群');
+                const senderNames = new Map(messages
+                    .filter((m) => m?.fromMid && m?.fromName)
+                    .map((m) => [String(m.fromMid), String(m.fromName)]));
+                for (const message of messages) {
+                    const type = mediaType(Number(message.contentType ?? 0));
+                    const mentions = mentionMids(message.contentMetadata);
+                    const msgId = String(message.id);
+                    events.push({
+                        eventId: computeEventId(chat.id, msgId),
+                        chatId: chat.id,
+                        messageId: msgId,
+                        opRevision: null,
+                        senderMid: message.fromMid ? String(message.fromMid) : null,
+                        senderName: message.fromName ? String(message.fromName) : null,
+                        sentAt: Number(message.createdTime ?? 0),
+                        contentType: type,
+                        text: sanitizeMessagePreview(message.text) || null,
+                        metadata: {
+                            fileName: message.fileName ? String(message.fileName) : undefined,
+                            mentionMids: mentions,
+                            mentionNames: mentions
+                                .map((mid) => senderNames.get(mid))
+                                .filter((name) => Boolean(name)),
+                            sourceUrl: type === 'sticker' && message.contentMetadata?.STKID
+                                ? `https://stickershop.line-scdn.net/stickershop/v1/sticker/${message.contentMetadata.STKID}/android/sticker.png`
+                                : null,
+                        },
+                        isSelf: message.isSelf === true,
+                        mentionsSelf: Boolean(chat.memberMid && mentions.includes(String(chat.memberMid))),
+                        ocrEligible: true,
+                    });
+                }
+            }
+        }
+        catch {
+            // SquareService discovery is optional.
+        }
+        return {
+            events,
+            chatNames,
+            coverageLimited,
+            chatsScanned: new Set(events.map((e) => e.chatId)).size,
+            messagesScanned: events.length,
+            officialChatsSkipped: officialChatIds.size,
+        };
+    }
+}

package/dist/ambient/journal/crypto.js

@@ -0,0 +1,58 @@
+import { createCipheriv, createDecipheriv, createHash, hkdf, randomBytes, } from 'node:crypto';
+const IV_LENGTH = 12;
+const TAG_LENGTH = 16;
+const KEY_LENGTH = 32;
+/**
+ * Derive an encryption key from a seed or context string using HKDF-SHA256.
+ */
+export async function deriveJournalKey(secretContext) {
+    return new Promise((resolve, reject) => {
+        hkdf('sha256', Buffer.from(secretContext, 'utf8'), Buffer.from('YomiJournalSalt', 'utf8'), 'LineActivityEventJournalKey', KEY_LENGTH, (err, key) => {
+            if (err)
+                reject(err);
+            else
+                resolve(Buffer.from(key));
+        });
+    });
+}
+/**
+ * Encrypt a plaintext buffer with AES-256-GCM authenticated encryption.
+ * Output format: [12-byte IV][16-byte Auth Tag][Ciphertext]
+ */
+export function encryptPayload(plaintext, key) {
+    if (key.length !== KEY_LENGTH) {
+        throw new Error(`Invalid journal encryption key length: expected ${KEY_LENGTH} bytes, got ${key.length}`);
+    }
+    const input = typeof plaintext === 'string' ? Buffer.from(plaintext, 'utf8') : plaintext;
+    const iv = randomBytes(IV_LENGTH);
+    const cipher = createCipheriv('aes-256-gcm', key, iv);
+    const ciphertext = Buffer.concat([cipher.update(input), cipher.final()]);
+    const tag = cipher.getAuthTag();
+    return Buffer.concat([iv, tag, ciphertext]);
+}
+/**
+ * Decrypt an AES-256-GCM envelope produced by encryptPayload.
+ */
+export function decryptPayload(envelope, key) {
+    if (key.length !== KEY_LENGTH) {
+        throw new Error(`Invalid journal encryption key length: expected ${KEY_LENGTH} bytes, got ${key.length}`);
+    }
+    if (envelope.length < IV_LENGTH + TAG_LENGTH) {
+        throw new Error('Encrypted envelope is truncated');
+    }
+    const iv = envelope.subarray(0, IV_LENGTH);
+    const tag = envelope.subarray(IV_LENGTH, IV_LENGTH + TAG_LENGTH);
+    const ciphertext = envelope.subarray(IV_LENGTH + TAG_LENGTH);
+    const decipher = createDecipheriv('aes-256-gcm', key, iv);
+    decipher.setAuthTag(tag);
+    return Buffer.concat([decipher.update(ciphertext), decipher.final()]);
+}
+export function hashIdentifier(id) {
+    return createHash('sha256').update(id).digest('hex').slice(0, 16);
+}
+export function computeEventId(chatId, messageId) {
+    return createHash('sha256')
+        .update(`${chatId}\0${messageId}`)
+        .digest('hex')
+        .slice(0, 24);
+}

package/dist/ambient/journal/journal-store.js

@@ -0,0 +1,284 @@
+import { createHash } from 'node:crypto';
+import { mkdir, readdir, readFile, rename, rm, writeFile, } from 'node:fs/promises';
+import { join } from 'node:path';
+import { yomiDataPath } from '../../util/data-dir.js';
+import { computeEventId, decryptPayload, deriveJournalKey, encryptPayload, hashIdentifier, } from './crypto.js';
+const DEFAULT_MAX_EVENTS = 500;
+const DEFAULT_MAX_SPAN_MS = 24 * 60 * 60 * 1000; // 24 hours
+export class LineActivityJournalStore {
+    baseDir;
+    encryptionKey = null;
+    maxEventsPerSegment;
+    maxSegmentSpanMs;
+    writeQueue = Promise.resolve();
+    constructor(options = {}) {
+        this.baseDir = options.baseDir ?? yomiDataPath('journal');
+        this.encryptionKey = options.encryptionKey ?? null;
+        this.maxEventsPerSegment = options.maxEventsPerSegment ?? DEFAULT_MAX_EVENTS;
+        this.maxSegmentSpanMs = options.maxSegmentSpanMs ?? DEFAULT_MAX_SPAN_MS;
+    }
+    async getKey() {
+        if (!this.encryptionKey) {
+            this.encryptionKey = await deriveJournalKey(this.baseDir);
+        }
+        return this.encryptionKey;
+    }
+    indexPath() {
+        return join(this.baseDir, 'segments-index.json');
+    }
+    segmentPath(fileName) {
+        return join(this.baseDir, fileName);
+    }
+    async readIndex() {
+        try {
+            const raw = await readFile(this.indexPath(), 'utf8');
+            const parsed = JSON.parse(raw);
+            if (parsed.schemaVersion === 1 && Array.isArray(parsed.segments)) {
+                return parsed;
+            }
+        }
+        catch { }
+        return {
+            schemaVersion: 1,
+            segments: [],
+            lastCompactedAt: 0,
+        };
+    }
+    async writeIndex(index) {
+        await mkdir(this.baseDir, { recursive: true });
+        const temp = `${this.indexPath()}.tmp-${process.pid}-${Date.now()}`;
+        await writeFile(temp, JSON.stringify(index, null, 2), { mode: 0o600 });
+        await rename(temp, this.indexPath());
+    }
+    async readSegmentEvents(meta) {
+        const key = await this.getKey();
+        const filePath = this.segmentPath(meta.cipherFileName);
+        try {
+            const cipher = await readFile(filePath);
+            const decrypted = decryptPayload(cipher, key);
+            const lines = decrypted
+                .toString('utf8')
+                .split('\n')
+                .filter((line) => line.trim().length > 0);
+            return lines.map((line) => JSON.parse(line));
+        }
+        catch {
+            return [];
+        }
+    }
+    async writeSegmentEvents(segmentId, events, completeness = 'open') {
+        const key = await this.getKey();
+        await mkdir(this.baseDir, { recursive: true });
+        const fileName = `segment-${segmentId}.enc`;
+        const filePath = this.segmentPath(fileName);
+        const serialized = `${events.map((event) => JSON.stringify(event)).join('\n')}\n`;
+        const encrypted = encryptPayload(Buffer.from(serialized, 'utf8'), key);
+        const temp = `${filePath}.tmp-${process.pid}-${Date.now()}`;
+        await writeFile(temp, encrypted, { mode: 0o600 });
+        await rename(temp, filePath);
+        const timestamps = events.map((e) => e.sentAt);
+        const chatHashes = [...new Set(events.map((e) => hashIdentifier(e.chatId)))];
+        const digest = createHash('sha256').update(encrypted).digest('hex');
+        return {
+            segmentId,
+            cipherFileName: fileName,
+            minSentAt: timestamps.length ? Math.min(...timestamps) : 0,
+            maxSentAt: timestamps.length ? Math.max(...timestamps) : 0,
+            eventCount: events.length,
+            chatIdHashes: chatHashes,
+            completeness,
+            sha256Digest: digest,
+            byteLength: encrypted.byteLength,
+        };
+    }
+    /**
+     * Append events into the journal. Deduplicates by eventId and updates active segment.
+     */
+    async append(incoming) {
+        if (incoming.length === 0)
+            return;
+        const run = async () => {
+            const index = await this.readIndex();
+            let openSegmentMeta = index.segments.find((s) => s.completeness === 'open');
+            let openEvents = [];
+            if (openSegmentMeta) {
+                openEvents = await this.readSegmentEvents(openSegmentMeta);
+            }
+            else {
+                const newSegmentId = `seg-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`;
+                openSegmentMeta = {
+                    segmentId: newSegmentId,
+                    cipherFileName: `segment-${newSegmentId}.enc`,
+                    minSentAt: 0,
+                    maxSentAt: 0,
+                    eventCount: 0,
+                    chatIdHashes: [],
+                    completeness: 'open',
+                    sha256Digest: '',
+                    byteLength: 0,
+                };
+            }
+            const existingIds = new Set(openEvents.map((e) => e.eventId));
+            const toAdd = incoming
+                .map((event) => ({
+                ...event,
+                eventId: event.eventId || computeEventId(event.chatId, event.messageId),
+            }))
+                .filter((event) => !existingIds.has(event.eventId));
+            if (toAdd.length === 0)
+                return;
+            openEvents.push(...toAdd);
+            openEvents.sort((a, b) => a.sentAt - b.sentAt);
+            const span = openEvents.length > 1
+                ? openEvents[openEvents.length - 1].sentAt - openEvents[0].sentAt
+                : 0;
+            const shouldSeal = openEvents.length >= this.maxEventsPerSegment ||
+                span >= this.maxSegmentSpanMs;
+            const updatedMeta = await this.writeSegmentEvents(openSegmentMeta.segmentId, openEvents, shouldSeal ? 'sealed' : 'open');
+            const segmentIndex = index.segments.findIndex((s) => s.segmentId === openSegmentMeta.segmentId);
+            if (segmentIndex >= 0) {
+                index.segments[segmentIndex] = updatedMeta;
+            }
+            else {
+                index.segments.push(updatedMeta);
+            }
+            await this.writeIndex(index);
+        };
+        this.writeQueue = this.writeQueue.then(run, run);
+        await this.writeQueue;
+    }
+    /**
+     * Query events matching time range, chat IDs, or limits.
+     */
+    async query(query = {}) {
+        await this.writeQueue;
+        const index = await this.readIndex();
+        const targetChatHashes = query.chatId
+            ? [hashIdentifier(query.chatId)]
+            : query.chatIds?.map(hashIdentifier);
+        // Filter relevant segments based on metadata
+        const candidateSegments = index.segments.filter((meta) => {
+            if (meta.eventCount === 0)
+                return false;
+            if (query.since && meta.maxSentAt < query.since)
+                return false;
+            if (query.until && meta.minSentAt > query.until)
+                return false;
+            if (targetChatHashes && targetChatHashes.length > 0) {
+                const hasMatchingChat = targetChatHashes.some((h) => meta.chatIdHashes.includes(h));
+                if (!hasMatchingChat)
+                    return false;
+            }
+            return true;
+        });
+        const matchingEvents = [];
+        const targetChatIds = query.chatId
+            ? new Set([query.chatId])
+            : query.chatIds
+                ? new Set(query.chatIds)
+                : null;
+        for (const segment of candidateSegments) {
+            const events = await this.readSegmentEvents(segment);
+            for (const event of events) {
+                if (query.since && event.sentAt < query.since)
+                    continue;
+                if (query.until && event.sentAt > query.until)
+                    continue;
+                if (targetChatIds && !targetChatIds.has(event.chatId))
+                    continue;
+                matchingEvents.push(event);
+            }
+        }
+        matchingEvents.sort((a, b) => query.reverse ? b.sentAt - a.sentAt : a.sentAt - b.sentAt);
+        if (query.limit && query.limit > 0) {
+            return matchingEvents.slice(0, query.limit);
+        }
+        return matchingEvents;
+    }
+    /**
+     * Immediately purge all events associated with a specific chatId from all segments.
+     */
+    async purgeChat(chatId) {
+        let purgedCount = 0;
+        const run = async () => {
+            const index = await this.readIndex();
+            const targetHash = hashIdentifier(chatId);
+            const updatedSegments = [];
+            for (const meta of index.segments) {
+                if (!meta.chatIdHashes.includes(targetHash)) {
+                    updatedSegments.push(meta);
+                    continue;
+                }
+                const events = await this.readSegmentEvents(meta);
+                const filtered = events.filter((e) => e.chatId !== chatId);
+                purgedCount += events.length - filtered.length;
+                if (filtered.length === 0) {
+                    // Remove segment completely
+                    try {
+                        await rm(this.segmentPath(meta.cipherFileName), { force: true });
+                    }
+                    catch { }
+                }
+                else {
+                    // Rewrite segment without purged events
+                    const newMeta = await this.writeSegmentEvents(meta.segmentId, filtered, meta.completeness);
+                    updatedSegments.push(newMeta);
+                }
+            }
+            index.segments = updatedSegments;
+            await this.writeIndex(index);
+        };
+        this.writeQueue = this.writeQueue.then(run, run);
+        await this.writeQueue;
+        return purgedCount;
+    }
+    /**
+     * Delete all encrypted segments and reset journal index.
+     */
+    async purgeAll() {
+        const run = async () => {
+            try {
+                const files = await readdir(this.baseDir);
+                for (const file of files) {
+                    await rm(join(this.baseDir, file), { recursive: true, force: true });
+                }
+            }
+            catch { }
+        };
+        this.writeQueue = this.writeQueue.then(run, run);
+        await this.writeQueue;
+    }
+    /**
+     * Compact journal by pruning segments older than retention period.
+     */
+    async compact(retentionMs = 14 * 24 * 60 * 60 * 1000, now = Date.now()) {
+        let removedSegmentsCount = 0;
+        const cutoff = now - retentionMs;
+        const run = async () => {
+            const index = await this.readIndex();
+            const keepSegments = [];
+            for (const meta of index.segments) {
+                if (meta.completeness === 'sealed' && meta.maxSentAt < cutoff) {
+                    try {
+                        await rm(this.segmentPath(meta.cipherFileName), { force: true });
+                        removedSegmentsCount += 1;
+                    }
+                    catch { }
+                }
+                else {
+                    keepSegments.push(meta);
+                }
+            }
+            index.segments = keepSegments;
+            index.lastCompactedAt = now;
+            await this.writeIndex(index);
+        };
+        this.writeQueue = this.writeQueue.then(run, run);
+        await this.writeQueue;
+        return removedSegmentsCount;
+    }
+    async getIndex() {
+        await this.writeQueue;
+        return this.readIndex();
+    }
+}

package/dist/ambient/journal/types.js

@@ -0,0 +1,7 @@
+/**
+ * Types for the encrypted append-only LineActivityEvent journal.
+ *
+ * LineActivityEvent is the immutable unit of observation recorded from LINE sync
+ * operations, TalkService message boxes, and SquareService channels.
+ */
+export {};

package/dist/ambient/summaries/retriever.js

@@ -0,0 +1,55 @@
+import { buildEpisodeSummaries, buildRollupSummary } from './summary-engine.js';
+export class CoarseToFineRetriever {
+    journalStore;
+    constructor(journalStore) {
+        this.journalStore = journalStore;
+    }
+    /**
+     * Coarse step: Scan 6-hour Rollup windows and return summary of activity.
+     */
+    async retrieveRollups(since, until, windowSpanMs = 6 * 60 * 60 * 1000) {
+        const rawEvents = await this.journalStore.query({ since, until });
+        const episodes = buildEpisodeSummaries(rawEvents);
+        const rollups = [];
+        let currentStart = since;
+        while (currentStart < until) {
+            const currentEnd = Math.min(until, currentStart + windowSpanMs);
+            rollups.push(buildRollupSummary(currentStart, currentEnd, episodes));
+            currentStart += windowSpanMs;
+        }
+        return rollups;
+    }
+    /**
+     * Medium step: Find candidate episodes matching query criteria.
+     */
+    async retrieveEpisodes(query = {}) {
+        const rawEvents = await this.journalStore.query({
+            since: query.since,
+            until: query.until,
+            chatId: query.chatId,
+        });
+        const episodes = buildEpisodeSummaries(rawEvents);
+        if (query.onlyActionable) {
+            return episodes.filter((ep) => ep.hasMentionsSelf ||
+                ep.responseState === 'waiting' ||
+                ep.candidateCount > 0);
+        }
+        return episodes;
+    }
+    /**
+     * Fine step: Given an EpisodeSummary, load exact raw LineActivityEvent records
+     * from the encrypted journal.
+     */
+    async retrieveConnectedEvidence(episode) {
+        const events = await this.journalStore.query({
+            chatId: episode.chatId,
+            since: episode.minSentAt,
+            until: episode.maxSentAt,
+        });
+        const filtered = events.filter((e) => episode.messageIds.includes(e.messageId));
+        return {
+            episode,
+            events: filtered,
+        };
+    }
+}