@rikaidev/yomi 0.2.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (119) hide show
  1. package/LICENSE +21 -0
  2. package/NOTICE +81 -0
  3. package/PRIVACY.md +39 -0
  4. package/README.md +659 -0
  5. package/dist/auth/credential-store.js +395 -0
  6. package/dist/auth/keychain.js +256 -0
  7. package/dist/cli/login.js +142 -0
  8. package/dist/line/auth/protocol/e2ee-bootstrap.js +87 -0
  9. package/dist/line/auth/protocol/index.js +3 -0
  10. package/dist/line/auth/protocol/login-metadata.js +159 -0
  11. package/dist/line/auth/protocol/pwless-login-flow.js +83 -0
  12. package/dist/line/auth/pwless/index.js +163 -0
  13. package/dist/line/auth/pwless/steps.js +213 -0
  14. package/dist/line/client/auth-service/client.js +240 -0
  15. package/dist/line/client/e2ee-service/client.js +69 -0
  16. package/dist/line/client/e2ee-service/requests.js +92 -0
  17. package/dist/line/client/e2ee-service/shared-key-log.js +24 -0
  18. package/dist/line/client/e2ee-service/shared-key-payload.js +35 -0
  19. package/dist/line/client/index.js +213 -0
  20. package/dist/line/client/obs-media-client.js +140 -0
  21. package/dist/line/client/parsers.js +69 -0
  22. package/dist/line/client/shop-service/client.js +63 -0
  23. package/dist/line/client/sticker-meta.js +46 -0
  24. package/dist/line/client/sync-service/client.js +317 -0
  25. package/dist/line/client/talk-service/chat-query.js +122 -0
  26. package/dist/line/client/talk-service/command-client.js +242 -0
  27. package/dist/line/client/talk-service/contact-query.js +77 -0
  28. package/dist/line/client/talk-service/message-box-query.js +137 -0
  29. package/dist/line/client/talk-service/message-id-query.js +241 -0
  30. package/dist/line/client/talk-service/query-client.js +219 -0
  31. package/dist/line/client/talk-service/requests.js +383 -0
  32. package/dist/line/client/transport.js +138 -0
  33. package/dist/line/core/auth-session-logout.js +55 -0
  34. package/dist/line/core/auth-session-runtime.js +231 -0
  35. package/dist/line/core/auth-session-service.js +85 -0
  36. package/dist/line/core/chat-runtime-service.js +343 -0
  37. package/dist/line/core/client-runtime.js +17 -0
  38. package/dist/line/core/config.js +67 -0
  39. package/dist/line/core/constants.js +56 -0
  40. package/dist/line/core/contact-query-service.js +20 -0
  41. package/dist/line/core/directory-runtime-service.js +83 -0
  42. package/dist/line/core/e2ee/bootstrap/keychain.js +41 -0
  43. package/dist/line/core/e2ee/crypto/crypto-primitives.js +98 -0
  44. package/dist/line/core/e2ee/index.js +6 -0
  45. package/dist/line/core/e2ee/keys/decrypt.js +110 -0
  46. package/dist/line/core/e2ee/keys/encrypt.js +117 -0
  47. package/dist/line/core/e2ee/keys/group-key.js +260 -0
  48. package/dist/line/core/e2ee/keys/key-manager.js +231 -0
  49. package/dist/line/core/e2ee/keys/key-payload.js +130 -0
  50. package/dist/line/core/e2ee/keys/key-types.js +1 -0
  51. package/dist/line/core/e2ee/keys/message-crypto.js +146 -0
  52. package/dist/line/core/e2ee/keys/message-decrypt.js +390 -0
  53. package/dist/line/core/e2ee/keys/peer-key.js +191 -0
  54. package/dist/line/core/e2ee/recovery/bootstrap-keys.js +161 -0
  55. package/dist/line/core/e2ee/types.js +4 -0
  56. package/dist/line/core/group-query-service.js +20 -0
  57. package/dist/line/core/media-decrypt.js +78 -0
  58. package/dist/line/core/media-encrypt.js +97 -0
  59. package/dist/line/core/mention.js +89 -0
  60. package/dist/line/core/message-command-service.js +454 -0
  61. package/dist/line/core/message-query-service.js +161 -0
  62. package/dist/line/core/service.js +134 -0
  63. package/dist/line/core/session-state/index.js +243 -0
  64. package/dist/line/core/session-state/persist.js +192 -0
  65. package/dist/line/core/store.js +8 -0
  66. package/dist/line/core/thrift/binary/binary-reader.js +151 -0
  67. package/dist/line/core/thrift/binary/binary-writer.js +157 -0
  68. package/dist/line/core/thrift/binary/type-codec-dsl.js +42 -0
  69. package/dist/line/core/thrift/compact/compact-reader.js +180 -0
  70. package/dist/line/core/thrift/compact/compact-writer.js +161 -0
  71. package/dist/line/core/thrift/compact/type-codec-dsl.js +66 -0
  72. package/dist/line/core/thrift/fields/builders.js +124 -0
  73. package/dist/line/core/thrift/index.js +12 -0
  74. package/dist/line/core/thrift/message/decoder.js +55 -0
  75. package/dist/line/core/thrift/message/encoder.js +25 -0
  76. package/dist/line/core/thrift/primitives/varint.js +64 -0
  77. package/dist/line/core/thrift/struct/reader.js +172 -0
  78. package/dist/line/core/thrift/struct/writer.js +189 -0
  79. package/dist/line/core/thrift/types.js +23 -0
  80. package/dist/line/core/video-thumbnail.js +104 -0
  81. package/dist/mcp/handlers/contacts.js +166 -0
  82. package/dist/mcp/handlers/conversations.js +193 -0
  83. package/dist/mcp/handlers/groups.js +123 -0
  84. package/dist/mcp/handlers/index.js +15 -0
  85. package/dist/mcp/handlers/login.js +473 -0
  86. package/dist/mcp/handlers/media.js +69 -0
  87. package/dist/mcp/handlers/message-actions.js +62 -0
  88. package/dist/mcp/handlers/scope.js +120 -0
  89. package/dist/mcp/handlers/search.js +196 -0
  90. package/dist/mcp/handlers/send.js +523 -0
  91. package/dist/mcp/handlers/shared.js +84 -0
  92. package/dist/mcp/handlers/stickers.js +87 -0
  93. package/dist/mcp/media.js +228 -0
  94. package/dist/mcp/names.js +133 -0
  95. package/dist/mcp/phi-guard.js +105 -0
  96. package/dist/mcp/policy.js +34 -0
  97. package/dist/mcp/server.js +301 -0
  98. package/dist/mcp/tools.js +761 -0
  99. package/dist/mcp/ui/capability.js +26 -0
  100. package/dist/mcp/ui/login-app-protocol.js +118 -0
  101. package/dist/mcp/ui/login-app-styles.js +142 -0
  102. package/dist/mcp/ui/login-app-view.js +361 -0
  103. package/dist/mcp/ui/login-app.js +72 -0
  104. package/dist/mcp/ui/login-structured-content.js +1 -0
  105. package/dist/mcp/ui/resource.js +35 -0
  106. package/dist/mcp/ui/tools-with-ui.js +40 -0
  107. package/dist/search/bigram.js +86 -0
  108. package/dist/search/capture.js +140 -0
  109. package/dist/search/collector.js +176 -0
  110. package/dist/search/default-embedder.js +101 -0
  111. package/dist/search/embedder.js +13 -0
  112. package/dist/search/scope.js +142 -0
  113. package/dist/search/sqlite.js +141 -0
  114. package/dist/search/store.js +469 -0
  115. package/dist/util/data-dir.js +55 -0
  116. package/dist/util/log.js +86 -0
  117. package/dist/version.js +7 -0
  118. package/package.json +63 -0
  119. package/run.mjs +71 -0
@@ -0,0 +1,72 @@
1
+ /**
2
+ * KNOWN CEILING — read before touching anything that depends on this view
3
+ * actually rendering: this implementation follows the 2026-01-26 MCP Apps
4
+ * spec (specification/2026-01-26/apps.mdx) and is spec-correct per
5
+ * ext-apps#615 (renders fine under MCP Inspector). In production, the host
6
+ * fetches this resource successfully and then NEVER completes the
7
+ * `ui/initialize` handshake — no iframe is created, no response to this
8
+ * view's `ui/initialize` request ever arrives. This is an open, unfixed,
9
+ * upstream host bug, not something fixable from the server side:
10
+ * anthropics/claude-ai-mcp#165 (stdio transport: resources/read succeeds,
11
+ * tool calls succeed, `ui/initialize` hangs, no iframe) and #236 (Claude
12
+ * Cowork with third-party inference renders plain text, no iframe, despite
13
+ * identical protocol negotiation). Measured on the operator's real machine:
14
+ * `resources.list` reports `supportsUi=true`, `resources.read` succeeds —
15
+ * and still nothing renders.
16
+ *
17
+ * Consequence for every other file in ./ui and ../handlers-login.ts: THIS
18
+ * VIEW HAS NEVER BEEN OBSERVED TO RENDER on any deployed client. Nothing in
19
+ * this codebase may put it on the critical path of any flow — the plain
20
+ * tool-result text the model sees must always be fully actionable on its
21
+ * own, with or without this view. Do not remove the view; it costs nothing,
22
+ * is spec-correct, and will start working the moment the host does.
23
+ *
24
+ * MCP Apps UI for the `login` tool — served as the `ui://yomi/login`
25
+ * resource (see ../server.ts). Self-contained document: no external
26
+ * requests, no CDN, no fonts. The composed document performs the MCP Apps
27
+ * lifecycle handshake (`ui/initialize` -> `ui/notifications/initialized`,
28
+ * see ./login-app-protocol.ts) as soon as it loads, then renders off
29
+ * `ui/notifications/tool-result` — and it is no longer strictly read-only:
30
+ * a submitted login form still calls `tools/call login` itself (a
31
+ * HUMAN-initiated action). It no longer calls `login_complete` on its own —
32
+ * that is the model's job, always (see ./login-app-view.ts).
33
+ *
34
+ * Composed from three sibling modules, split out purely to keep every
35
+ * individual file under the project's 500-scc-line module cap as this view
36
+ * grew from a read-only display into a form-driving client:
37
+ * - ./login-app-styles.ts — the `<style>` block.
38
+ * - ./login-app-protocol.ts — JSON-RPC transport + lifecycle handshake.
39
+ * - ./login-app-view.ts — stage rendering + tool-call orchestration.
40
+ *
41
+ * The PIN is remote data — rendered with `textContent`, never `innerHTML`;
42
+ * same for any error message. All timing numbers (the 3-minute PIN code
43
+ * lifetime, the client's poll-ceiling minutes) are read from
44
+ * `structuredContent` at render time — none are written into this
45
+ * HTML/JS source, so there is exactly one place (../handlers-login.ts,
46
+ * sourced from ../../line/auth/pwless/index.ts) that owns each number.
47
+ */
48
+ import { YOMI_VERSION } from '../../version.js';
49
+ import { buildLoginAppProtocolJs } from './login-app-protocol.js';
50
+ import { LOGIN_APP_CSS } from './login-app-styles.js';
51
+ import { LOGIN_APP_VIEW_JS } from './login-app-view.js';
52
+ export const LOGIN_APP_HTML = `<!DOCTYPE html>
53
+ <html lang="en">
54
+ <head>
55
+ <meta charset="utf-8">
56
+ <title>LINE Login</title>
57
+ <style>
58
+ ${LOGIN_APP_CSS}
59
+ </style>
60
+ </head>
61
+ <body>
62
+ <main id="app"></main>
63
+ <script>
64
+ (function () {
65
+ 'use strict';
66
+ ${buildLoginAppProtocolJs(YOMI_VERSION)}
67
+ ${LOGIN_APP_VIEW_JS}
68
+ })();
69
+ </script>
70
+ </body>
71
+ </html>
72
+ `;
@@ -0,0 +1 @@
1
+ export {};
@@ -0,0 +1,35 @@
1
+ /**
2
+ * The single MCP Apps UI resource this server exposes: the login view.
3
+ * Registered by ../server.ts's `resources/list` and `resources/read`
4
+ * handlers, gated on client capability (see ./capability.ts) so a client
5
+ * without MCP Apps support sees an empty resource list, unchanged from
6
+ * before this feature existed.
7
+ *
8
+ * The `csp` key within `_meta.ui` is deliberately omitted: the login view
9
+ * makes no outbound connections and loads nothing external, so the host's
10
+ * most restrictive default (no external connections at all) is exactly right.
11
+ * The `_meta.ui` block itself (containing `prefersBorder`) is present in both
12
+ * the `resources/list` entry and the `resources/read` contents, matching the
13
+ * MCP Apps spec.
14
+ */
15
+ import { MCP_APPS_MIME_TYPE } from './capability.js';
16
+ import { LOGIN_APP_HTML } from './login-app.js';
17
+ /** The `ui://` resource URI for the login view. */
18
+ export const LOGIN_UI_RESOURCE_URI = 'ui://yomi/login';
19
+ /** UI metadata shared by both resources/list and resources/read entries. */
20
+ const LOGIN_UI_META = { ui: { prefersBorder: true } };
21
+ /** `resources/list` entry for the login view. */
22
+ export const LOGIN_UI_RESOURCE_LISTING = {
23
+ uri: LOGIN_UI_RESOURCE_URI,
24
+ name: 'login_view',
25
+ description: 'Interactive LINE login view — shows the PIN or device-approval step with a live countdown.',
26
+ mimeType: MCP_APPS_MIME_TYPE,
27
+ _meta: LOGIN_UI_META,
28
+ };
29
+ /** `resources/read` contents entry for the login view. */
30
+ export const LOGIN_UI_RESOURCE_CONTENTS = {
31
+ uri: LOGIN_UI_RESOURCE_URI,
32
+ mimeType: MCP_APPS_MIME_TYPE,
33
+ text: LOGIN_APP_HTML,
34
+ _meta: LOGIN_UI_META,
35
+ };
@@ -0,0 +1,40 @@
1
+ /**
2
+ * Attaches the MCP Apps `_meta.ui` linkage to the `login` tool definition,
3
+ * but only for clients that negotiated MCP Apps support (see
4
+ * ./capability.ts) — on a client without it, the tool list served must stay
5
+ * byte-for-byte what it always was. Kept separate from ../tools.ts (a flat,
6
+ * static schema registry) because this step is per-connection, not static.
7
+ */
8
+ import { LOGIN_UI_RESOURCE_URI } from './resource.js';
9
+ /**
10
+ * Return `tools` as-is for a client without MCP Apps support, or a copy with
11
+ * the `login` entry augmented with `_meta.ui.resourceUri` for one that has
12
+ * it. Never mutates the input array/objects. Generic over the caller's
13
+ * exact tool type (rather than a loosened `Record<string, unknown>`) so the
14
+ * SDK's stricter `Tool[]` return type still checks at the call site in
15
+ * ../server.ts.
16
+ *
17
+ * @param tools - The static tool list (../tools.ts's TOOLS).
18
+ * @param supportsMcpApps - Result of ./capability.ts's `supportsMcpApps`.
19
+ * @returns The tool list to serve for this client.
20
+ */
21
+ export function toolsForClient(tools, supportsMcpApps) {
22
+ if (!supportsMcpApps) {
23
+ return tools;
24
+ }
25
+ return tools.map((tool) => {
26
+ if (tool.name !== 'login') {
27
+ return tool;
28
+ }
29
+ return {
30
+ ...tool,
31
+ _meta: {
32
+ ...(typeof tool._meta === 'object' && tool._meta ? tool._meta : {}),
33
+ ui: {
34
+ resourceUri: LOGIN_UI_RESOURCE_URI,
35
+ visibility: ['model', 'app'],
36
+ },
37
+ },
38
+ };
39
+ });
40
+ }
@@ -0,0 +1,86 @@
1
+ /**
2
+ * CJK-aware bigram preprocessing for FTS5 indexing/querying.
3
+ *
4
+ * The FTS5 `unicode61` tokenizer has no Chinese/Japanese/Korean word
5
+ * segmentation: a whole CJK run collapses into a single opaque token, so a
6
+ * substring like "會議" inside "會議通知" never matches. Yomi has no ICU
7
+ * tokenizer available and takes on no new dependency to get one, so instead
8
+ * both the indexed text and the search query are rewritten into overlapping
9
+ * character bigrams for CJK runs before ever reaching FTS5. Overlapping
10
+ * bigrams turn "會議通知" into "會議 議通 通知" — searching for "會議"
11
+ * (itself a single bigram once transformed) becomes a plain token match.
12
+ * This covers the common case of 2+ character CJK words; single-character
13
+ * CJK terms still match via the emitted unigram.
14
+ *
15
+ * Non-CJK runs (Latin, digits, punctuation) are left untouched and keep
16
+ * relying on unicode61's normal whitespace/punctuation tokenization.
17
+ */
18
+ /**
19
+ * CJK ranges covered: CJK Unified Ideographs + Extension A, CJK
20
+ * Compatibility Ideographs, Hiragana/Katakana, and Hangul Syllables.
21
+ */
22
+ const CJK_PATTERN = /[㐀-鿿豈-﫿぀-ヿ가-퟿]/;
23
+ /**
24
+ * Determine whether a single character falls in a CJK range that needs
25
+ * bigram splitting rather than unicode61's default tokenization.
26
+ *
27
+ * @param ch - A single character to classify.
28
+ * @returns True when the character is CJK (ideograph, kana, or Hangul).
29
+ */
30
+ function isCjkChar(ch) {
31
+ return CJK_PATTERN.test(ch);
32
+ }
33
+ /**
34
+ * Convert a run of CJK characters into space-separated overlapping
35
+ * bigrams (or the single character itself, for a length-1 run).
36
+ *
37
+ * @param run - A maximal run of CJK characters.
38
+ * @returns Space-separated bigrams (or the lone character).
39
+ */
40
+ function bigramsForRun(run) {
41
+ const chars = Array.from(run);
42
+ if (chars.length <= 1) {
43
+ return chars.join('');
44
+ }
45
+ const grams = [];
46
+ for (let i = 0; i < chars.length - 1; i++) {
47
+ grams.push(chars[i] + chars[i + 1]);
48
+ }
49
+ return grams.join(' ');
50
+ }
51
+ /**
52
+ * Rewrite raw text into its FTS5-searchable form: CJK runs become
53
+ * space-separated overlapping bigrams, non-CJK runs pass through
54
+ * unchanged. Apply this identically at index time (to the stored
55
+ * `search_text`) and at query time (to the user's search string) so the
56
+ * same substring produces matching tokens on both sides.
57
+ *
58
+ * @param raw - Raw display text (message body or user query).
59
+ * @returns Bigram-preprocessed text suitable for an FTS5 MATCH/index.
60
+ */
61
+ export function toSearchText(raw) {
62
+ if (!raw) {
63
+ return '';
64
+ }
65
+ const chars = Array.from(raw);
66
+ const pieces = [];
67
+ let runStart = 0;
68
+ let runIsCjk = chars.length > 0 ? isCjkChar(chars[0]) : false;
69
+ const flush = (end) => {
70
+ if (end <= runStart) {
71
+ return;
72
+ }
73
+ const run = chars.slice(runStart, end).join('');
74
+ pieces.push(runIsCjk ? bigramsForRun(run) : run);
75
+ };
76
+ for (let i = 1; i < chars.length; i++) {
77
+ const cjk = isCjkChar(chars[i]);
78
+ if (cjk !== runIsCjk) {
79
+ flush(i);
80
+ runStart = i;
81
+ runIsCjk = cjk;
82
+ }
83
+ }
84
+ flush(chars.length);
85
+ return pieces.join(' ').replace(/\s+/g, ' ').trim();
86
+ }
@@ -0,0 +1,140 @@
1
+ /**
2
+ * Yomi continuous capture — the in-process, silent, denylist-gated sync loop.
3
+ *
4
+ * Unlike collect_messages (explicit, one-shot, caller-invoked), this runs for
5
+ * the life of the MCP process: LINE's stdio MCP server is spawned once by the
6
+ * client and lives as long as that client, so a background loop here keeps the
7
+ * local search index current while the user is actually using the app.
8
+ *
9
+ * Two mechanisms:
10
+ * - Startup catch-up: one collectMessages() over all conversations, to fill
11
+ * the gap for anything that arrived while the process was down. Idempotent
12
+ * (upsert), and denylist-gated inside collectMessages.
13
+ * - Live loop: LINE's startPolling long-polls SYNC4; on each new message we
14
+ * index just the affected chat via collectMessages({ chatIds }).
15
+ *
16
+ * This is SILENT: SYNC4 sync never sends a read receipt, so capture never marks
17
+ * anything read. Excluded (denylisted) chats are dropped inside collectMessages,
18
+ * so they are never fetched or indexed. Sending/read-receipts are never
19
+ * triggered here.
20
+ */
21
+ import { createCliLogger } from '../util/log.js';
22
+ import { collectMessages } from './collector.js';
23
+ import { SqliteUnavailableError } from './sqlite.js';
24
+ import { saveCaptureState } from './store.js';
25
+ const log = createCliLogger('Yomi');
26
+ /** Milliseconds to batch new-message signals before indexing affected chats. */
27
+ const FLUSH_DEBOUNCE_MS = 2000;
28
+ /** Recent messages to (re)index per affected chat on a live flush. */
29
+ const LIVE_PER_CHAT = 30;
30
+ /**
31
+ * Resolve the chat MID a parsed LINE message belongs to. Group/room messages
32
+ * are keyed by their `to`; 1:1 (toType 0) messages are keyed by the peer (the
33
+ * party that is not the authenticated user).
34
+ *
35
+ * @param message - Parsed LINE message from the poll loop.
36
+ * @param myMid - Authenticated user's MID, or null.
37
+ * @returns Chat MID, or null when it cannot be resolved.
38
+ */
39
+ function resolveChatId(message, myMid) {
40
+ if (!message) {
41
+ return null;
42
+ }
43
+ if (message.toType === 0) {
44
+ return message.from === myMid ? message.to : message.from;
45
+ }
46
+ return message.to ?? null;
47
+ }
48
+ /**
49
+ * Start continuous capture for a resumed LINE session: run a one-time startup
50
+ * catch-up, then keep the local index current from the live poll loop. Silent
51
+ * and denylist-gated. Fire-and-forget: safe to call without awaiting.
52
+ *
53
+ * @param service - Resumed LineProtocolService (must have a live client).
54
+ * @param embedder - Embedder used to keep semantic vectors current.
55
+ * @returns Promise resolving once startup catch-up completes and polling starts.
56
+ */
57
+ export async function startCapture(service, embedder) {
58
+ try {
59
+ const summary = await collectMessages(service, { embedder });
60
+ log.info('capture.catchup', {
61
+ chatsScanned: summary.chatsScanned,
62
+ messagesIndexed: summary.messagesIndexed,
63
+ });
64
+ }
65
+ catch (error) {
66
+ // A runtime with no SQLite is not a hiccup this loop recovers from — the
67
+ // index will stay empty for the life of the process and every later flush
68
+ // fails the same way. Log it once, at error, and stop: a WARN here let a
69
+ // Node 20 install poll silently for hours while capturing nothing.
70
+ if (error instanceof SqliteUnavailableError) {
71
+ log.error('capture.unavailable', {
72
+ action: 'capture_disabled_for_this_process',
73
+ error: error.message,
74
+ });
75
+ return;
76
+ }
77
+ log.warn('capture.catchup_failed', {
78
+ error: error?.message ?? String(error),
79
+ });
80
+ }
81
+ const pending = new Set();
82
+ let flushTimer = null;
83
+ const flush = async () => {
84
+ flushTimer = null;
85
+ const chatIds = [...pending];
86
+ pending.clear();
87
+ if (chatIds.length === 0) {
88
+ return;
89
+ }
90
+ try {
91
+ await collectMessages(service, {
92
+ chatIds,
93
+ perChat: LIVE_PER_CHAT,
94
+ embedder,
95
+ });
96
+ }
97
+ catch (error) {
98
+ log.warn('capture.index_failed', {
99
+ chats: chatIds.length,
100
+ error: error?.message ?? String(error),
101
+ });
102
+ }
103
+ persistCursor(service);
104
+ };
105
+ service.client.on('message', (message) => {
106
+ const chatId = resolveChatId(message, service.profile?.mid ?? null);
107
+ if (chatId) {
108
+ pending.add(chatId);
109
+ }
110
+ if (!flushTimer) {
111
+ flushTimer = setTimeout(() => {
112
+ void flush();
113
+ }, FLUSH_DEBOUNCE_MS);
114
+ }
115
+ });
116
+ log.info('capture.started', {});
117
+ service.client.startPolling().catch((error) => {
118
+ log.error('capture.polling_stopped', {
119
+ error: error?.message ?? String(error),
120
+ });
121
+ });
122
+ }
123
+ /**
124
+ * Persist the current LINE sync cursor for observability and future resume.
125
+ *
126
+ * @param service - Resumed LineProtocolService.
127
+ */
128
+ function persistCursor(service) {
129
+ try {
130
+ saveCaptureState({
131
+ revision: Number(service.client.revision ?? 0),
132
+ globalRevision: Number(service.client.globalRevision ?? 0),
133
+ individualRevision: Number(service.client.individualRevision ?? 0),
134
+ updatedAt: Date.now(),
135
+ });
136
+ }
137
+ catch {
138
+ // Cursor persistence is best-effort observability; never let it break capture.
139
+ }
140
+ }
@@ -0,0 +1,176 @@
1
+ /**
2
+ * Yomi cross-conversation message collector.
3
+ *
4
+ * LINE only exposes per-conversation reads; there is no "search all my
5
+ * chats" primitive server-side. This module is the explicit, caller-
6
+ * invoked bulk fetch that pulls recent messages from one or more LINE
7
+ * conversations and upserts them into the local FTS5 search index (see
8
+ * ./store.ts) so search_messages has something to query.
9
+ *
10
+ * This is the ONLY place in Yomi allowed to bulk-fetch across chats, and
11
+ * only because it is explicit: it runs exactly once per collect_messages
12
+ * tool call, never on a timer, never in the background. The pure-query MCP
13
+ * tools (list_conversations, get_chat_messages, ...) never call this and
14
+ * remain unaffected.
15
+ *
16
+ * Reuses the existing read+decrypt path (LineProtocolService.
17
+ * getRecentMessages, which already runs messages through E2EE decrypt) —
18
+ * message fetching and decryption are not reimplemented here.
19
+ *
20
+ * Also batch-embeds each chat's collected messages (when an Embedder is
21
+ * supplied) so semantic search has vectors to rank over. The embedder is
22
+ * always caller-injected — never imported directly here — so the MCP
23
+ * server can pass its DefaultEmbedder while a host app (e.g. inboxd) can
24
+ * pass its own local model instead. Embedding failure (model unavailable,
25
+ * offline first run, etc.) is caught per-chat and never aborts collection:
26
+ * messages + bigram search_text are always stored regardless of whether
27
+ * embedding succeeded, so keyword search keeps working either way.
28
+ */
29
+ import { resolveConversationNames, resolveSenderNames } from '../mcp/names.js';
30
+ import { createCliLogger } from '../util/log.js';
31
+ import { getExcludedChatIds } from './scope.js';
32
+ import { getMessagesMissingEmbedding, upsertEmbeddings, upsertMessages, } from './store.js';
33
+ const log = createCliLogger('Yomi');
34
+ const EMBED_BATCH = 64;
35
+ const DEFAULT_PER_CHAT = 100;
36
+ const DEFAULT_CHAT_LIMIT = 200;
37
+ /**
38
+ * Resolve the target chat ids for a collection run, along with a
39
+ * best-effort display name for each.
40
+ *
41
+ * @param service - Resumed LineProtocolService.
42
+ * @param chatIds - Caller-supplied chat ids; when omitted, all conversations are used.
43
+ * @returns Map of chat MID to resolved name (null when unresolvable).
44
+ */
45
+ async function resolveTargetChats(service, chatIds) {
46
+ if (chatIds && chatIds.length > 0) {
47
+ return resolveConversationNames(service, chatIds);
48
+ }
49
+ const result = await service.client.getMessageBoxes({
50
+ lastMessagesPerMessageBoxCount: 1,
51
+ messageBoxCountLimit: DEFAULT_CHAT_LIMIT,
52
+ withUnreadCount: false,
53
+ });
54
+ const boxes = result.messageBoxes || [];
55
+ const ids = boxes.map((box) => box.id).filter(Boolean);
56
+ return resolveConversationNames(service, ids);
57
+ }
58
+ /**
59
+ * Collect recent messages from one or more LINE conversations and upsert
60
+ * them into the local search index. Explicit operation — runs only when a
61
+ * caller invokes the `collect_messages` MCP tool.
62
+ *
63
+ * Messages without decrypted/plaintext text (media-only, or E2EE-locked)
64
+ * are skipped rather than indexed as empty rows — nothing fabricated.
65
+ * Chats on the exclusion denylist (see ../search/scope.ts) are dropped
66
+ * before the fetch loop, so they are never fetched from LINE or indexed.
67
+ *
68
+ * @param service - Resumed LineProtocolService.
69
+ * @param options - Collection options.
70
+ * @returns Summary of chats scanned and messages indexed.
71
+ */
72
+ export async function collectMessages(service, options = {}) {
73
+ const perChat = options.perChat ?? DEFAULT_PER_CHAT;
74
+ const chatNames = await resolveTargetChats(service, options.chatIds);
75
+ // Scoping is a denylist: excluded chats are removed here, before the
76
+ // fetch loop below, so they are never fetched from LINE or indexed —
77
+ // not just filtered out of the results afterward. See ../search/scope.ts.
78
+ const excluded = getExcludedChatIds();
79
+ for (const chatId of excluded) {
80
+ chatNames.delete(chatId);
81
+ }
82
+ let messagesIndexed = 0;
83
+ let messagesEmbedded = 0;
84
+ for (const [chatId, chatName] of chatNames) {
85
+ const messages = await service.getRecentMessages(chatId, perChat);
86
+ const withText = messages.filter((message) => typeof message.text === 'string' && message.text.length > 0);
87
+ if (withText.length === 0) {
88
+ continue;
89
+ }
90
+ const senderNames = await resolveSenderNames(service, withText.map((message) => message.from).filter(Boolean));
91
+ const rows = withText.map((message) => ({
92
+ chatId,
93
+ chatName: chatName ?? null,
94
+ messageId: String(message.id),
95
+ fromMid: message.from ?? null,
96
+ fromName: senderNames.get(message.from) ?? null,
97
+ text: message.text,
98
+ createdTime: message.createdTime ?? message.deliveredTime ?? null,
99
+ }));
100
+ messagesIndexed += upsertMessages(rows);
101
+ messagesEmbedded += await embedAndStore(rows, options.embedder);
102
+ }
103
+ // Self-heal embedding coverage: embed anything already in the index that
104
+ // still lacks a vector for this model — legacy keyword-only rows, or rows
105
+ // from a run where the model wasn't loaded. This is why re-running
106
+ // collect_messages repairs a partially-embedded index without re-fetching
107
+ // from LINE. The rows just embedded above are no longer "missing", so they
108
+ // are not embedded twice.
109
+ if (options.embedder) {
110
+ messagesEmbedded += await backfillMissingEmbeddings(options.embedder);
111
+ }
112
+ return { chatsScanned: chatNames.size, messagesIndexed, messagesEmbedded };
113
+ }
114
+ /**
115
+ * Embed every indexed message that still lacks a vector for the given
116
+ * model, in batches, from stored text alone (no LINE round-trip). Best-
117
+ * effort like embedAndStore: if the model fails mid-sweep it stops and
118
+ * returns what it managed, leaving the rest keyword-only rather than
119
+ * aborting. Idempotent — a fully-embedded index makes this a no-op.
120
+ *
121
+ * @param embedder - Injected embedder to embed missing rows with.
122
+ * @returns Number of messages newly embedded and stored.
123
+ */
124
+ export async function backfillMissingEmbeddings(embedder) {
125
+ const missing = getMessagesMissingEmbedding(embedder.modelLabel);
126
+ if (missing.length === 0) {
127
+ return 0;
128
+ }
129
+ let embedded = 0;
130
+ for (let i = 0; i < missing.length; i += EMBED_BATCH) {
131
+ const batch = missing.slice(i, i + EMBED_BATCH);
132
+ try {
133
+ const vectors = await embedder.embed(batch.map((row) => row.text));
134
+ embedded += upsertEmbeddings(batch.map((row, j) => ({
135
+ messageId: row.messageId,
136
+ vector: vectors[j],
137
+ })), embedder.modelLabel);
138
+ }
139
+ catch (error) {
140
+ log.warn('backfill.failed', {
141
+ error: error?.message ?? String(error),
142
+ remaining: missing.length - embedded,
143
+ });
144
+ break;
145
+ }
146
+ }
147
+ return embedded;
148
+ }
149
+ /**
150
+ * Batch-embed a chat's collected messages and store the resulting vectors,
151
+ * when an Embedder was supplied. Embedding is best-effort: any failure
152
+ * (model unavailable, offline, OOM, etc.) is caught and logged, never
153
+ * thrown — messages are already durably stored via upsertMessages by the
154
+ * time this runs, so a failed embed only means that batch stays
155
+ * keyword-only, not that collection itself failed.
156
+ *
157
+ * @param rows - Messages already upserted this batch.
158
+ * @param embedder - Injected embedder, or undefined to skip embedding.
159
+ * @returns Number of messages successfully embedded and stored.
160
+ */
161
+ async function embedAndStore(rows, embedder) {
162
+ if (!embedder || rows.length === 0) {
163
+ return 0;
164
+ }
165
+ try {
166
+ const vectors = await embedder.embed(rows.map((row) => row.text));
167
+ return upsertEmbeddings(rows.map((row, i) => ({ messageId: row.messageId, vector: vectors[i] })), embedder.modelLabel);
168
+ }
169
+ catch (error) {
170
+ log.warn('embed.failed', {
171
+ error: error?.message ?? String(error),
172
+ count: rows.length,
173
+ });
174
+ return 0;
175
+ }
176
+ }
@@ -0,0 +1,101 @@
1
+ /**
2
+ * Yomi's default semantic-search embedder — a local, offline ONNX model
3
+ * via transformers.js (@huggingface/transformers), used when no host app
4
+ * injects its own Embedder (see ./embedder.ts).
5
+ *
6
+ * Model: Xenova/bge-small-zh-v1.5 (BAAI general embedding, small, Chinese-
7
+ * primary but multilingual-capable; 512-dim). Chosen because Yomi's data
8
+ * is LINE messages, which for this project are predominantly Chinese.
9
+ *
10
+ * bun compatibility note: transformers.js picks a Node-flavored backend
11
+ * under bun (it detects a Node-like `process`), which only accepts device
12
+ * "cpu" | "coreml" | "webgpu" — "wasm" throws "Unsupported device". "cpu"
13
+ * routes through onnxruntime-node, and it works under bun WITHOUT running
14
+ * onnxruntime-node's postinstall script (bun blocks postinstalls by
15
+ * default; the packaged binary still loads fine here) — verified: model
16
+ * loads, embeds Chinese text, returns 512-dim unit-normalized vectors.
17
+ * If bge-small-zh-v1.5 ever fails to load in some environment, fall back to
18
+ * Xenova/multilingual-e5-small (384-dim, e5 prefix convention: query
19
+ * embeds get `query: `, document embeds get `passage: `).
20
+ *
21
+ * Retrieval convention: bge models are asymmetric — the QUERY gets an
22
+ * instruction prefix, stored documents do not. Using the model's official
23
+ * recommended instruction for retrieval tasks.
24
+ */
25
+ const MODEL_ID = 'Xenova/bge-small-zh-v1.5';
26
+ const BGE_QUERY_INSTRUCTION = '为这个句子生成表示以用于检索相关文章:';
27
+ let pipelinePromise = null;
28
+ /**
29
+ * Lazily load (and memoize) the transformers.js feature-extraction
30
+ * pipeline for MODEL_ID. First call downloads/caches the ONNX model;
31
+ * subsequent calls reuse the same pipeline instance.
32
+ *
33
+ * @returns Promise resolving to the feature-extraction pipeline.
34
+ */
35
+ function getPipeline() {
36
+ if (!pipelinePromise) {
37
+ pipelinePromise = import('@huggingface/transformers')
38
+ .then(({ pipeline }) => pipeline('feature-extraction', MODEL_ID, {
39
+ device: 'cpu',
40
+ }))
41
+ .catch((error) => {
42
+ // Reset the memo on failure so a later call can retry (e.g. the
43
+ // first attempt failed offline, network comes back later) instead
44
+ // of permanently caching a rejected promise.
45
+ pipelinePromise = null;
46
+ throw error;
47
+ });
48
+ }
49
+ return pipelinePromise;
50
+ }
51
+ /**
52
+ * Split a pipeline output tensor (flat data + dims [batch, hidden]) into
53
+ * one number[] vector per input row.
54
+ *
55
+ * @param output - Raw pipeline output.
56
+ * @returns One vector array per row.
57
+ */
58
+ function splitRows(output) {
59
+ const [rows, hidden] = output.dims;
60
+ const data = output.data;
61
+ const vectors = [];
62
+ for (let i = 0; i < rows; i++) {
63
+ vectors.push(Array.from(data.slice(i * hidden, (i + 1) * hidden)));
64
+ }
65
+ return vectors;
66
+ }
67
+ /**
68
+ * Default Yomi embedder backed by transformers.js/ONNX, running fully
69
+ * local and offline after the first model download. Model load happens on
70
+ * first embed() or embedQuery() call, not at construction, so importing
71
+ * this module never triggers a download by itself.
72
+ */
73
+ export class DefaultEmbedder {
74
+ modelLabel = MODEL_ID;
75
+ async embed(texts) {
76
+ if (texts.length === 0) {
77
+ return [];
78
+ }
79
+ const extractor = await getPipeline();
80
+ const output = await extractor(texts, { pooling: 'mean', normalize: true });
81
+ return splitRows(output);
82
+ }
83
+ async embedQuery(query) {
84
+ const [vector] = await this.embed([`${BGE_QUERY_INSTRUCTION}${query}`]);
85
+ return vector;
86
+ }
87
+ }
88
+ let defaultEmbedder = null;
89
+ /**
90
+ * Get the process-wide singleton DefaultEmbedder instance. Constructing it
91
+ * is cheap (no model load); the underlying pipeline only loads lazily on
92
+ * first use.
93
+ *
94
+ * @returns Shared DefaultEmbedder instance.
95
+ */
96
+ export function getDefaultEmbedder() {
97
+ if (!defaultEmbedder) {
98
+ defaultEmbedder = new DefaultEmbedder();
99
+ }
100
+ return defaultEmbedder;
101
+ }
@@ -0,0 +1,13 @@
1
+ /**
2
+ * Semantic-search embedder seam for Yomi's search index.
3
+ *
4
+ * This interface is the injection point between the local search index
5
+ * (store.ts, collector.ts) and whatever produces dense vectors. Yomi ships
6
+ * a default implementation (./default-embedder.ts, transformers.js/ONNX) so
7
+ * the MCP server works standalone, but nothing in store.ts/collector.ts
8
+ * imports that default directly — they only depend on this interface, so a
9
+ * host app (e.g. inboxd) can inject its own embedder (its local MLX
10
+ * `generateEmbedding`) instead. See ./default-embedder.ts's doc comment for
11
+ * the exact adapter shape.
12
+ */
13
+ export {};