@rikaidev/yomi 0.2.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/NOTICE +81 -0
- package/PRIVACY.md +39 -0
- package/README.md +659 -0
- package/dist/auth/credential-store.js +395 -0
- package/dist/auth/keychain.js +256 -0
- package/dist/cli/login.js +142 -0
- package/dist/line/auth/protocol/e2ee-bootstrap.js +87 -0
- package/dist/line/auth/protocol/index.js +3 -0
- package/dist/line/auth/protocol/login-metadata.js +159 -0
- package/dist/line/auth/protocol/pwless-login-flow.js +83 -0
- package/dist/line/auth/pwless/index.js +163 -0
- package/dist/line/auth/pwless/steps.js +213 -0
- package/dist/line/client/auth-service/client.js +240 -0
- package/dist/line/client/e2ee-service/client.js +69 -0
- package/dist/line/client/e2ee-service/requests.js +92 -0
- package/dist/line/client/e2ee-service/shared-key-log.js +24 -0
- package/dist/line/client/e2ee-service/shared-key-payload.js +35 -0
- package/dist/line/client/index.js +213 -0
- package/dist/line/client/obs-media-client.js +140 -0
- package/dist/line/client/parsers.js +69 -0
- package/dist/line/client/shop-service/client.js +63 -0
- package/dist/line/client/sticker-meta.js +46 -0
- package/dist/line/client/sync-service/client.js +317 -0
- package/dist/line/client/talk-service/chat-query.js +122 -0
- package/dist/line/client/talk-service/command-client.js +242 -0
- package/dist/line/client/talk-service/contact-query.js +77 -0
- package/dist/line/client/talk-service/message-box-query.js +137 -0
- package/dist/line/client/talk-service/message-id-query.js +241 -0
- package/dist/line/client/talk-service/query-client.js +219 -0
- package/dist/line/client/talk-service/requests.js +383 -0
- package/dist/line/client/transport.js +138 -0
- package/dist/line/core/auth-session-logout.js +55 -0
- package/dist/line/core/auth-session-runtime.js +231 -0
- package/dist/line/core/auth-session-service.js +85 -0
- package/dist/line/core/chat-runtime-service.js +343 -0
- package/dist/line/core/client-runtime.js +17 -0
- package/dist/line/core/config.js +67 -0
- package/dist/line/core/constants.js +56 -0
- package/dist/line/core/contact-query-service.js +20 -0
- package/dist/line/core/directory-runtime-service.js +83 -0
- package/dist/line/core/e2ee/bootstrap/keychain.js +41 -0
- package/dist/line/core/e2ee/crypto/crypto-primitives.js +98 -0
- package/dist/line/core/e2ee/index.js +6 -0
- package/dist/line/core/e2ee/keys/decrypt.js +110 -0
- package/dist/line/core/e2ee/keys/encrypt.js +117 -0
- package/dist/line/core/e2ee/keys/group-key.js +260 -0
- package/dist/line/core/e2ee/keys/key-manager.js +231 -0
- package/dist/line/core/e2ee/keys/key-payload.js +130 -0
- package/dist/line/core/e2ee/keys/key-types.js +1 -0
- package/dist/line/core/e2ee/keys/message-crypto.js +146 -0
- package/dist/line/core/e2ee/keys/message-decrypt.js +390 -0
- package/dist/line/core/e2ee/keys/peer-key.js +191 -0
- package/dist/line/core/e2ee/recovery/bootstrap-keys.js +161 -0
- package/dist/line/core/e2ee/types.js +4 -0
- package/dist/line/core/group-query-service.js +20 -0
- package/dist/line/core/media-decrypt.js +78 -0
- package/dist/line/core/media-encrypt.js +97 -0
- package/dist/line/core/mention.js +89 -0
- package/dist/line/core/message-command-service.js +454 -0
- package/dist/line/core/message-query-service.js +161 -0
- package/dist/line/core/service.js +134 -0
- package/dist/line/core/session-state/index.js +243 -0
- package/dist/line/core/session-state/persist.js +192 -0
- package/dist/line/core/store.js +8 -0
- package/dist/line/core/thrift/binary/binary-reader.js +151 -0
- package/dist/line/core/thrift/binary/binary-writer.js +157 -0
- package/dist/line/core/thrift/binary/type-codec-dsl.js +42 -0
- package/dist/line/core/thrift/compact/compact-reader.js +180 -0
- package/dist/line/core/thrift/compact/compact-writer.js +161 -0
- package/dist/line/core/thrift/compact/type-codec-dsl.js +66 -0
- package/dist/line/core/thrift/fields/builders.js +124 -0
- package/dist/line/core/thrift/index.js +12 -0
- package/dist/line/core/thrift/message/decoder.js +55 -0
- package/dist/line/core/thrift/message/encoder.js +25 -0
- package/dist/line/core/thrift/primitives/varint.js +64 -0
- package/dist/line/core/thrift/struct/reader.js +172 -0
- package/dist/line/core/thrift/struct/writer.js +189 -0
- package/dist/line/core/thrift/types.js +23 -0
- package/dist/line/core/video-thumbnail.js +104 -0
- package/dist/mcp/handlers/contacts.js +166 -0
- package/dist/mcp/handlers/conversations.js +193 -0
- package/dist/mcp/handlers/groups.js +123 -0
- package/dist/mcp/handlers/index.js +15 -0
- package/dist/mcp/handlers/login.js +473 -0
- package/dist/mcp/handlers/media.js +69 -0
- package/dist/mcp/handlers/message-actions.js +62 -0
- package/dist/mcp/handlers/scope.js +120 -0
- package/dist/mcp/handlers/search.js +196 -0
- package/dist/mcp/handlers/send.js +523 -0
- package/dist/mcp/handlers/shared.js +84 -0
- package/dist/mcp/handlers/stickers.js +87 -0
- package/dist/mcp/media.js +228 -0
- package/dist/mcp/names.js +133 -0
- package/dist/mcp/phi-guard.js +105 -0
- package/dist/mcp/policy.js +34 -0
- package/dist/mcp/server.js +301 -0
- package/dist/mcp/tools.js +761 -0
- package/dist/mcp/ui/capability.js +26 -0
- package/dist/mcp/ui/login-app-protocol.js +118 -0
- package/dist/mcp/ui/login-app-styles.js +142 -0
- package/dist/mcp/ui/login-app-view.js +361 -0
- package/dist/mcp/ui/login-app.js +72 -0
- package/dist/mcp/ui/login-structured-content.js +1 -0
- package/dist/mcp/ui/resource.js +35 -0
- package/dist/mcp/ui/tools-with-ui.js +40 -0
- package/dist/search/bigram.js +86 -0
- package/dist/search/capture.js +140 -0
- package/dist/search/collector.js +176 -0
- package/dist/search/default-embedder.js +101 -0
- package/dist/search/embedder.js +13 -0
- package/dist/search/scope.js +142 -0
- package/dist/search/sqlite.js +141 -0
- package/dist/search/store.js +469 -0
- package/dist/util/data-dir.js +55 -0
- package/dist/util/log.js +86 -0
- package/dist/version.js +7 -0
- package/package.json +63 -0
- package/run.mjs +71 -0
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* KNOWN CEILING — read before touching anything that depends on this view
|
|
3
|
+
* actually rendering: this implementation follows the 2026-01-26 MCP Apps
|
|
4
|
+
* spec (specification/2026-01-26/apps.mdx) and is spec-correct per
|
|
5
|
+
* ext-apps#615 (renders fine under MCP Inspector). In production, the host
|
|
6
|
+
* fetches this resource successfully and then NEVER completes the
|
|
7
|
+
* `ui/initialize` handshake — no iframe is created, no response to this
|
|
8
|
+
* view's `ui/initialize` request ever arrives. This is an open, unfixed,
|
|
9
|
+
* upstream host bug, not something fixable from the server side:
|
|
10
|
+
* anthropics/claude-ai-mcp#165 (stdio transport: resources/read succeeds,
|
|
11
|
+
* tool calls succeed, `ui/initialize` hangs, no iframe) and #236 (Claude
|
|
12
|
+
* Cowork with third-party inference renders plain text, no iframe, despite
|
|
13
|
+
* identical protocol negotiation). Measured on the operator's real machine:
|
|
14
|
+
* `resources.list` reports `supportsUi=true`, `resources.read` succeeds —
|
|
15
|
+
* and still nothing renders.
|
|
16
|
+
*
|
|
17
|
+
* Consequence for every other file in ./ui and ../handlers-login.ts: THIS
|
|
18
|
+
* VIEW HAS NEVER BEEN OBSERVED TO RENDER on any deployed client. Nothing in
|
|
19
|
+
* this codebase may put it on the critical path of any flow — the plain
|
|
20
|
+
* tool-result text the model sees must always be fully actionable on its
|
|
21
|
+
* own, with or without this view. Do not remove the view; it costs nothing,
|
|
22
|
+
* is spec-correct, and will start working the moment the host does.
|
|
23
|
+
*
|
|
24
|
+
* MCP Apps UI for the `login` tool — served as the `ui://yomi/login`
|
|
25
|
+
* resource (see ../server.ts). Self-contained document: no external
|
|
26
|
+
* requests, no CDN, no fonts. The composed document performs the MCP Apps
|
|
27
|
+
* lifecycle handshake (`ui/initialize` -> `ui/notifications/initialized`,
|
|
28
|
+
* see ./login-app-protocol.ts) as soon as it loads, then renders off
|
|
29
|
+
* `ui/notifications/tool-result` — and it is no longer strictly read-only:
|
|
30
|
+
* a submitted login form still calls `tools/call login` itself (a
|
|
31
|
+
* HUMAN-initiated action). It no longer calls `login_complete` on its own —
|
|
32
|
+
* that is the model's job, always (see ./login-app-view.ts).
|
|
33
|
+
*
|
|
34
|
+
* Composed from three sibling modules, split out purely to keep every
|
|
35
|
+
* individual file under the project's 500-scc-line module cap as this view
|
|
36
|
+
* grew from a read-only display into a form-driving client:
|
|
37
|
+
* - ./login-app-styles.ts — the `<style>` block.
|
|
38
|
+
* - ./login-app-protocol.ts — JSON-RPC transport + lifecycle handshake.
|
|
39
|
+
* - ./login-app-view.ts — stage rendering + tool-call orchestration.
|
|
40
|
+
*
|
|
41
|
+
* The PIN is remote data — rendered with `textContent`, never `innerHTML`;
|
|
42
|
+
* same for any error message. All timing numbers (the 3-minute PIN code
|
|
43
|
+
* lifetime, the client's poll-ceiling minutes) are read from
|
|
44
|
+
* `structuredContent` at render time — none are written into this
|
|
45
|
+
* HTML/JS source, so there is exactly one place (../handlers-login.ts,
|
|
46
|
+
* sourced from ../../line/auth/pwless/index.ts) that owns each number.
|
|
47
|
+
*/
|
|
48
|
+
import { YOMI_VERSION } from '../../version.js';
|
|
49
|
+
import { buildLoginAppProtocolJs } from './login-app-protocol.js';
|
|
50
|
+
import { LOGIN_APP_CSS } from './login-app-styles.js';
|
|
51
|
+
import { LOGIN_APP_VIEW_JS } from './login-app-view.js';
|
|
52
|
+
export const LOGIN_APP_HTML = `<!DOCTYPE html>
|
|
53
|
+
<html lang="en">
|
|
54
|
+
<head>
|
|
55
|
+
<meta charset="utf-8">
|
|
56
|
+
<title>LINE Login</title>
|
|
57
|
+
<style>
|
|
58
|
+
${LOGIN_APP_CSS}
|
|
59
|
+
</style>
|
|
60
|
+
</head>
|
|
61
|
+
<body>
|
|
62
|
+
<main id="app"></main>
|
|
63
|
+
<script>
|
|
64
|
+
(function () {
|
|
65
|
+
'use strict';
|
|
66
|
+
${buildLoginAppProtocolJs(YOMI_VERSION)}
|
|
67
|
+
${LOGIN_APP_VIEW_JS}
|
|
68
|
+
})();
|
|
69
|
+
</script>
|
|
70
|
+
</body>
|
|
71
|
+
</html>
|
|
72
|
+
`;
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The single MCP Apps UI resource this server exposes: the login view.
|
|
3
|
+
* Registered by ../server.ts's `resources/list` and `resources/read`
|
|
4
|
+
* handlers, gated on client capability (see ./capability.ts) so a client
|
|
5
|
+
* without MCP Apps support sees an empty resource list, unchanged from
|
|
6
|
+
* before this feature existed.
|
|
7
|
+
*
|
|
8
|
+
* The `csp` key within `_meta.ui` is deliberately omitted: the login view
|
|
9
|
+
* makes no outbound connections and loads nothing external, so the host's
|
|
10
|
+
* most restrictive default (no external connections at all) is exactly right.
|
|
11
|
+
* The `_meta.ui` block itself (containing `prefersBorder`) is present in both
|
|
12
|
+
* the `resources/list` entry and the `resources/read` contents, matching the
|
|
13
|
+
* MCP Apps spec.
|
|
14
|
+
*/
|
|
15
|
+
import { MCP_APPS_MIME_TYPE } from './capability.js';
|
|
16
|
+
import { LOGIN_APP_HTML } from './login-app.js';
|
|
17
|
+
/** The `ui://` resource URI for the login view. */
|
|
18
|
+
export const LOGIN_UI_RESOURCE_URI = 'ui://yomi/login';
|
|
19
|
+
/** UI metadata shared by both resources/list and resources/read entries. */
|
|
20
|
+
const LOGIN_UI_META = { ui: { prefersBorder: true } };
|
|
21
|
+
/** `resources/list` entry for the login view. */
|
|
22
|
+
export const LOGIN_UI_RESOURCE_LISTING = {
|
|
23
|
+
uri: LOGIN_UI_RESOURCE_URI,
|
|
24
|
+
name: 'login_view',
|
|
25
|
+
description: 'Interactive LINE login view — shows the PIN or device-approval step with a live countdown.',
|
|
26
|
+
mimeType: MCP_APPS_MIME_TYPE,
|
|
27
|
+
_meta: LOGIN_UI_META,
|
|
28
|
+
};
|
|
29
|
+
/** `resources/read` contents entry for the login view. */
|
|
30
|
+
export const LOGIN_UI_RESOURCE_CONTENTS = {
|
|
31
|
+
uri: LOGIN_UI_RESOURCE_URI,
|
|
32
|
+
mimeType: MCP_APPS_MIME_TYPE,
|
|
33
|
+
text: LOGIN_APP_HTML,
|
|
34
|
+
_meta: LOGIN_UI_META,
|
|
35
|
+
};
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Attaches the MCP Apps `_meta.ui` linkage to the `login` tool definition,
|
|
3
|
+
* but only for clients that negotiated MCP Apps support (see
|
|
4
|
+
* ./capability.ts) — on a client without it, the tool list served must stay
|
|
5
|
+
* byte-for-byte what it always was. Kept separate from ../tools.ts (a flat,
|
|
6
|
+
* static schema registry) because this step is per-connection, not static.
|
|
7
|
+
*/
|
|
8
|
+
import { LOGIN_UI_RESOURCE_URI } from './resource.js';
|
|
9
|
+
/**
|
|
10
|
+
* Return `tools` as-is for a client without MCP Apps support, or a copy with
|
|
11
|
+
* the `login` entry augmented with `_meta.ui.resourceUri` for one that has
|
|
12
|
+
* it. Never mutates the input array/objects. Generic over the caller's
|
|
13
|
+
* exact tool type (rather than a loosened `Record<string, unknown>`) so the
|
|
14
|
+
* SDK's stricter `Tool[]` return type still checks at the call site in
|
|
15
|
+
* ../server.ts.
|
|
16
|
+
*
|
|
17
|
+
* @param tools - The static tool list (../tools.ts's TOOLS).
|
|
18
|
+
* @param supportsMcpApps - Result of ./capability.ts's `supportsMcpApps`.
|
|
19
|
+
* @returns The tool list to serve for this client.
|
|
20
|
+
*/
|
|
21
|
+
export function toolsForClient(tools, supportsMcpApps) {
|
|
22
|
+
if (!supportsMcpApps) {
|
|
23
|
+
return tools;
|
|
24
|
+
}
|
|
25
|
+
return tools.map((tool) => {
|
|
26
|
+
if (tool.name !== 'login') {
|
|
27
|
+
return tool;
|
|
28
|
+
}
|
|
29
|
+
return {
|
|
30
|
+
...tool,
|
|
31
|
+
_meta: {
|
|
32
|
+
...(typeof tool._meta === 'object' && tool._meta ? tool._meta : {}),
|
|
33
|
+
ui: {
|
|
34
|
+
resourceUri: LOGIN_UI_RESOURCE_URI,
|
|
35
|
+
visibility: ['model', 'app'],
|
|
36
|
+
},
|
|
37
|
+
},
|
|
38
|
+
};
|
|
39
|
+
});
|
|
40
|
+
}
|
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* CJK-aware bigram preprocessing for FTS5 indexing/querying.
|
|
3
|
+
*
|
|
4
|
+
* The FTS5 `unicode61` tokenizer has no Chinese/Japanese/Korean word
|
|
5
|
+
* segmentation: a whole CJK run collapses into a single opaque token, so a
|
|
6
|
+
* substring like "會議" inside "會議通知" never matches. Yomi has no ICU
|
|
7
|
+
* tokenizer available and takes on no new dependency to get one, so instead
|
|
8
|
+
* both the indexed text and the search query are rewritten into overlapping
|
|
9
|
+
* character bigrams for CJK runs before ever reaching FTS5. Overlapping
|
|
10
|
+
* bigrams turn "會議通知" into "會議 議通 通知" — searching for "會議"
|
|
11
|
+
* (itself a single bigram once transformed) becomes a plain token match.
|
|
12
|
+
* This covers the common case of 2+ character CJK words; single-character
|
|
13
|
+
* CJK terms still match via the emitted unigram.
|
|
14
|
+
*
|
|
15
|
+
* Non-CJK runs (Latin, digits, punctuation) are left untouched and keep
|
|
16
|
+
* relying on unicode61's normal whitespace/punctuation tokenization.
|
|
17
|
+
*/
|
|
18
|
+
/**
|
|
19
|
+
* CJK ranges covered: CJK Unified Ideographs + Extension A, CJK
|
|
20
|
+
* Compatibility Ideographs, Hiragana/Katakana, and Hangul Syllables.
|
|
21
|
+
*/
|
|
22
|
+
const CJK_PATTERN = /[㐀-鿿豈--ヿ가-]/;
|
|
23
|
+
/**
|
|
24
|
+
* Determine whether a single character falls in a CJK range that needs
|
|
25
|
+
* bigram splitting rather than unicode61's default tokenization.
|
|
26
|
+
*
|
|
27
|
+
* @param ch - A single character to classify.
|
|
28
|
+
* @returns True when the character is CJK (ideograph, kana, or Hangul).
|
|
29
|
+
*/
|
|
30
|
+
function isCjkChar(ch) {
|
|
31
|
+
return CJK_PATTERN.test(ch);
|
|
32
|
+
}
|
|
33
|
+
/**
|
|
34
|
+
* Convert a run of CJK characters into space-separated overlapping
|
|
35
|
+
* bigrams (or the single character itself, for a length-1 run).
|
|
36
|
+
*
|
|
37
|
+
* @param run - A maximal run of CJK characters.
|
|
38
|
+
* @returns Space-separated bigrams (or the lone character).
|
|
39
|
+
*/
|
|
40
|
+
function bigramsForRun(run) {
|
|
41
|
+
const chars = Array.from(run);
|
|
42
|
+
if (chars.length <= 1) {
|
|
43
|
+
return chars.join('');
|
|
44
|
+
}
|
|
45
|
+
const grams = [];
|
|
46
|
+
for (let i = 0; i < chars.length - 1; i++) {
|
|
47
|
+
grams.push(chars[i] + chars[i + 1]);
|
|
48
|
+
}
|
|
49
|
+
return grams.join(' ');
|
|
50
|
+
}
|
|
51
|
+
/**
|
|
52
|
+
* Rewrite raw text into its FTS5-searchable form: CJK runs become
|
|
53
|
+
* space-separated overlapping bigrams, non-CJK runs pass through
|
|
54
|
+
* unchanged. Apply this identically at index time (to the stored
|
|
55
|
+
* `search_text`) and at query time (to the user's search string) so the
|
|
56
|
+
* same substring produces matching tokens on both sides.
|
|
57
|
+
*
|
|
58
|
+
* @param raw - Raw display text (message body or user query).
|
|
59
|
+
* @returns Bigram-preprocessed text suitable for an FTS5 MATCH/index.
|
|
60
|
+
*/
|
|
61
|
+
export function toSearchText(raw) {
|
|
62
|
+
if (!raw) {
|
|
63
|
+
return '';
|
|
64
|
+
}
|
|
65
|
+
const chars = Array.from(raw);
|
|
66
|
+
const pieces = [];
|
|
67
|
+
let runStart = 0;
|
|
68
|
+
let runIsCjk = chars.length > 0 ? isCjkChar(chars[0]) : false;
|
|
69
|
+
const flush = (end) => {
|
|
70
|
+
if (end <= runStart) {
|
|
71
|
+
return;
|
|
72
|
+
}
|
|
73
|
+
const run = chars.slice(runStart, end).join('');
|
|
74
|
+
pieces.push(runIsCjk ? bigramsForRun(run) : run);
|
|
75
|
+
};
|
|
76
|
+
for (let i = 1; i < chars.length; i++) {
|
|
77
|
+
const cjk = isCjkChar(chars[i]);
|
|
78
|
+
if (cjk !== runIsCjk) {
|
|
79
|
+
flush(i);
|
|
80
|
+
runStart = i;
|
|
81
|
+
runIsCjk = cjk;
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
flush(chars.length);
|
|
85
|
+
return pieces.join(' ').replace(/\s+/g, ' ').trim();
|
|
86
|
+
}
|
|
@@ -0,0 +1,140 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Yomi continuous capture — the in-process, silent, denylist-gated sync loop.
|
|
3
|
+
*
|
|
4
|
+
* Unlike collect_messages (explicit, one-shot, caller-invoked), this runs for
|
|
5
|
+
* the life of the MCP process: LINE's stdio MCP server is spawned once by the
|
|
6
|
+
* client and lives as long as that client, so a background loop here keeps the
|
|
7
|
+
* local search index current while the user is actually using the app.
|
|
8
|
+
*
|
|
9
|
+
* Two mechanisms:
|
|
10
|
+
* - Startup catch-up: one collectMessages() over all conversations, to fill
|
|
11
|
+
* the gap for anything that arrived while the process was down. Idempotent
|
|
12
|
+
* (upsert), and denylist-gated inside collectMessages.
|
|
13
|
+
* - Live loop: LINE's startPolling long-polls SYNC4; on each new message we
|
|
14
|
+
* index just the affected chat via collectMessages({ chatIds }).
|
|
15
|
+
*
|
|
16
|
+
* This is SILENT: SYNC4 sync never sends a read receipt, so capture never marks
|
|
17
|
+
* anything read. Excluded (denylisted) chats are dropped inside collectMessages,
|
|
18
|
+
* so they are never fetched or indexed. Sending/read-receipts are never
|
|
19
|
+
* triggered here.
|
|
20
|
+
*/
|
|
21
|
+
import { createCliLogger } from '../util/log.js';
|
|
22
|
+
import { collectMessages } from './collector.js';
|
|
23
|
+
import { SqliteUnavailableError } from './sqlite.js';
|
|
24
|
+
import { saveCaptureState } from './store.js';
|
|
25
|
+
const log = createCliLogger('Yomi');
|
|
26
|
+
/** Milliseconds to batch new-message signals before indexing affected chats. */
|
|
27
|
+
const FLUSH_DEBOUNCE_MS = 2000;
|
|
28
|
+
/** Recent messages to (re)index per affected chat on a live flush. */
|
|
29
|
+
const LIVE_PER_CHAT = 30;
|
|
30
|
+
/**
|
|
31
|
+
* Resolve the chat MID a parsed LINE message belongs to. Group/room messages
|
|
32
|
+
* are keyed by their `to`; 1:1 (toType 0) messages are keyed by the peer (the
|
|
33
|
+
* party that is not the authenticated user).
|
|
34
|
+
*
|
|
35
|
+
* @param message - Parsed LINE message from the poll loop.
|
|
36
|
+
* @param myMid - Authenticated user's MID, or null.
|
|
37
|
+
* @returns Chat MID, or null when it cannot be resolved.
|
|
38
|
+
*/
|
|
39
|
+
function resolveChatId(message, myMid) {
|
|
40
|
+
if (!message) {
|
|
41
|
+
return null;
|
|
42
|
+
}
|
|
43
|
+
if (message.toType === 0) {
|
|
44
|
+
return message.from === myMid ? message.to : message.from;
|
|
45
|
+
}
|
|
46
|
+
return message.to ?? null;
|
|
47
|
+
}
|
|
48
|
+
/**
|
|
49
|
+
* Start continuous capture for a resumed LINE session: run a one-time startup
|
|
50
|
+
* catch-up, then keep the local index current from the live poll loop. Silent
|
|
51
|
+
* and denylist-gated. Fire-and-forget: safe to call without awaiting.
|
|
52
|
+
*
|
|
53
|
+
* @param service - Resumed LineProtocolService (must have a live client).
|
|
54
|
+
* @param embedder - Embedder used to keep semantic vectors current.
|
|
55
|
+
* @returns Promise resolving once startup catch-up completes and polling starts.
|
|
56
|
+
*/
|
|
57
|
+
export async function startCapture(service, embedder) {
|
|
58
|
+
try {
|
|
59
|
+
const summary = await collectMessages(service, { embedder });
|
|
60
|
+
log.info('capture.catchup', {
|
|
61
|
+
chatsScanned: summary.chatsScanned,
|
|
62
|
+
messagesIndexed: summary.messagesIndexed,
|
|
63
|
+
});
|
|
64
|
+
}
|
|
65
|
+
catch (error) {
|
|
66
|
+
// A runtime with no SQLite is not a hiccup this loop recovers from — the
|
|
67
|
+
// index will stay empty for the life of the process and every later flush
|
|
68
|
+
// fails the same way. Log it once, at error, and stop: a WARN here let a
|
|
69
|
+
// Node 20 install poll silently for hours while capturing nothing.
|
|
70
|
+
if (error instanceof SqliteUnavailableError) {
|
|
71
|
+
log.error('capture.unavailable', {
|
|
72
|
+
action: 'capture_disabled_for_this_process',
|
|
73
|
+
error: error.message,
|
|
74
|
+
});
|
|
75
|
+
return;
|
|
76
|
+
}
|
|
77
|
+
log.warn('capture.catchup_failed', {
|
|
78
|
+
error: error?.message ?? String(error),
|
|
79
|
+
});
|
|
80
|
+
}
|
|
81
|
+
const pending = new Set();
|
|
82
|
+
let flushTimer = null;
|
|
83
|
+
const flush = async () => {
|
|
84
|
+
flushTimer = null;
|
|
85
|
+
const chatIds = [...pending];
|
|
86
|
+
pending.clear();
|
|
87
|
+
if (chatIds.length === 0) {
|
|
88
|
+
return;
|
|
89
|
+
}
|
|
90
|
+
try {
|
|
91
|
+
await collectMessages(service, {
|
|
92
|
+
chatIds,
|
|
93
|
+
perChat: LIVE_PER_CHAT,
|
|
94
|
+
embedder,
|
|
95
|
+
});
|
|
96
|
+
}
|
|
97
|
+
catch (error) {
|
|
98
|
+
log.warn('capture.index_failed', {
|
|
99
|
+
chats: chatIds.length,
|
|
100
|
+
error: error?.message ?? String(error),
|
|
101
|
+
});
|
|
102
|
+
}
|
|
103
|
+
persistCursor(service);
|
|
104
|
+
};
|
|
105
|
+
service.client.on('message', (message) => {
|
|
106
|
+
const chatId = resolveChatId(message, service.profile?.mid ?? null);
|
|
107
|
+
if (chatId) {
|
|
108
|
+
pending.add(chatId);
|
|
109
|
+
}
|
|
110
|
+
if (!flushTimer) {
|
|
111
|
+
flushTimer = setTimeout(() => {
|
|
112
|
+
void flush();
|
|
113
|
+
}, FLUSH_DEBOUNCE_MS);
|
|
114
|
+
}
|
|
115
|
+
});
|
|
116
|
+
log.info('capture.started', {});
|
|
117
|
+
service.client.startPolling().catch((error) => {
|
|
118
|
+
log.error('capture.polling_stopped', {
|
|
119
|
+
error: error?.message ?? String(error),
|
|
120
|
+
});
|
|
121
|
+
});
|
|
122
|
+
}
|
|
123
|
+
/**
|
|
124
|
+
* Persist the current LINE sync cursor for observability and future resume.
|
|
125
|
+
*
|
|
126
|
+
* @param service - Resumed LineProtocolService.
|
|
127
|
+
*/
|
|
128
|
+
function persistCursor(service) {
|
|
129
|
+
try {
|
|
130
|
+
saveCaptureState({
|
|
131
|
+
revision: Number(service.client.revision ?? 0),
|
|
132
|
+
globalRevision: Number(service.client.globalRevision ?? 0),
|
|
133
|
+
individualRevision: Number(service.client.individualRevision ?? 0),
|
|
134
|
+
updatedAt: Date.now(),
|
|
135
|
+
});
|
|
136
|
+
}
|
|
137
|
+
catch {
|
|
138
|
+
// Cursor persistence is best-effort observability; never let it break capture.
|
|
139
|
+
}
|
|
140
|
+
}
|
|
@@ -0,0 +1,176 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Yomi cross-conversation message collector.
|
|
3
|
+
*
|
|
4
|
+
* LINE only exposes per-conversation reads; there is no "search all my
|
|
5
|
+
* chats" primitive server-side. This module is the explicit, caller-
|
|
6
|
+
* invoked bulk fetch that pulls recent messages from one or more LINE
|
|
7
|
+
* conversations and upserts them into the local FTS5 search index (see
|
|
8
|
+
* ./store.ts) so search_messages has something to query.
|
|
9
|
+
*
|
|
10
|
+
* This is the ONLY place in Yomi allowed to bulk-fetch across chats, and
|
|
11
|
+
* only because it is explicit: it runs exactly once per collect_messages
|
|
12
|
+
* tool call, never on a timer, never in the background. The pure-query MCP
|
|
13
|
+
* tools (list_conversations, get_chat_messages, ...) never call this and
|
|
14
|
+
* remain unaffected.
|
|
15
|
+
*
|
|
16
|
+
* Reuses the existing read+decrypt path (LineProtocolService.
|
|
17
|
+
* getRecentMessages, which already runs messages through E2EE decrypt) —
|
|
18
|
+
* message fetching and decryption are not reimplemented here.
|
|
19
|
+
*
|
|
20
|
+
* Also batch-embeds each chat's collected messages (when an Embedder is
|
|
21
|
+
* supplied) so semantic search has vectors to rank over. The embedder is
|
|
22
|
+
* always caller-injected — never imported directly here — so the MCP
|
|
23
|
+
* server can pass its DefaultEmbedder while a host app (e.g. inboxd) can
|
|
24
|
+
* pass its own local model instead. Embedding failure (model unavailable,
|
|
25
|
+
* offline first run, etc.) is caught per-chat and never aborts collection:
|
|
26
|
+
* messages + bigram search_text are always stored regardless of whether
|
|
27
|
+
* embedding succeeded, so keyword search keeps working either way.
|
|
28
|
+
*/
|
|
29
|
+
import { resolveConversationNames, resolveSenderNames } from '../mcp/names.js';
|
|
30
|
+
import { createCliLogger } from '../util/log.js';
|
|
31
|
+
import { getExcludedChatIds } from './scope.js';
|
|
32
|
+
import { getMessagesMissingEmbedding, upsertEmbeddings, upsertMessages, } from './store.js';
|
|
33
|
+
const log = createCliLogger('Yomi');
|
|
34
|
+
const EMBED_BATCH = 64;
|
|
35
|
+
const DEFAULT_PER_CHAT = 100;
|
|
36
|
+
const DEFAULT_CHAT_LIMIT = 200;
|
|
37
|
+
/**
|
|
38
|
+
* Resolve the target chat ids for a collection run, along with a
|
|
39
|
+
* best-effort display name for each.
|
|
40
|
+
*
|
|
41
|
+
* @param service - Resumed LineProtocolService.
|
|
42
|
+
* @param chatIds - Caller-supplied chat ids; when omitted, all conversations are used.
|
|
43
|
+
* @returns Map of chat MID to resolved name (null when unresolvable).
|
|
44
|
+
*/
|
|
45
|
+
async function resolveTargetChats(service, chatIds) {
|
|
46
|
+
if (chatIds && chatIds.length > 0) {
|
|
47
|
+
return resolveConversationNames(service, chatIds);
|
|
48
|
+
}
|
|
49
|
+
const result = await service.client.getMessageBoxes({
|
|
50
|
+
lastMessagesPerMessageBoxCount: 1,
|
|
51
|
+
messageBoxCountLimit: DEFAULT_CHAT_LIMIT,
|
|
52
|
+
withUnreadCount: false,
|
|
53
|
+
});
|
|
54
|
+
const boxes = result.messageBoxes || [];
|
|
55
|
+
const ids = boxes.map((box) => box.id).filter(Boolean);
|
|
56
|
+
return resolveConversationNames(service, ids);
|
|
57
|
+
}
|
|
58
|
+
/**
|
|
59
|
+
* Collect recent messages from one or more LINE conversations and upsert
|
|
60
|
+
* them into the local search index. Explicit operation — runs only when a
|
|
61
|
+
* caller invokes the `collect_messages` MCP tool.
|
|
62
|
+
*
|
|
63
|
+
* Messages without decrypted/plaintext text (media-only, or E2EE-locked)
|
|
64
|
+
* are skipped rather than indexed as empty rows — nothing fabricated.
|
|
65
|
+
* Chats on the exclusion denylist (see ../search/scope.ts) are dropped
|
|
66
|
+
* before the fetch loop, so they are never fetched from LINE or indexed.
|
|
67
|
+
*
|
|
68
|
+
* @param service - Resumed LineProtocolService.
|
|
69
|
+
* @param options - Collection options.
|
|
70
|
+
* @returns Summary of chats scanned and messages indexed.
|
|
71
|
+
*/
|
|
72
|
+
export async function collectMessages(service, options = {}) {
|
|
73
|
+
const perChat = options.perChat ?? DEFAULT_PER_CHAT;
|
|
74
|
+
const chatNames = await resolveTargetChats(service, options.chatIds);
|
|
75
|
+
// Scoping is a denylist: excluded chats are removed here, before the
|
|
76
|
+
// fetch loop below, so they are never fetched from LINE or indexed —
|
|
77
|
+
// not just filtered out of the results afterward. See ../search/scope.ts.
|
|
78
|
+
const excluded = getExcludedChatIds();
|
|
79
|
+
for (const chatId of excluded) {
|
|
80
|
+
chatNames.delete(chatId);
|
|
81
|
+
}
|
|
82
|
+
let messagesIndexed = 0;
|
|
83
|
+
let messagesEmbedded = 0;
|
|
84
|
+
for (const [chatId, chatName] of chatNames) {
|
|
85
|
+
const messages = await service.getRecentMessages(chatId, perChat);
|
|
86
|
+
const withText = messages.filter((message) => typeof message.text === 'string' && message.text.length > 0);
|
|
87
|
+
if (withText.length === 0) {
|
|
88
|
+
continue;
|
|
89
|
+
}
|
|
90
|
+
const senderNames = await resolveSenderNames(service, withText.map((message) => message.from).filter(Boolean));
|
|
91
|
+
const rows = withText.map((message) => ({
|
|
92
|
+
chatId,
|
|
93
|
+
chatName: chatName ?? null,
|
|
94
|
+
messageId: String(message.id),
|
|
95
|
+
fromMid: message.from ?? null,
|
|
96
|
+
fromName: senderNames.get(message.from) ?? null,
|
|
97
|
+
text: message.text,
|
|
98
|
+
createdTime: message.createdTime ?? message.deliveredTime ?? null,
|
|
99
|
+
}));
|
|
100
|
+
messagesIndexed += upsertMessages(rows);
|
|
101
|
+
messagesEmbedded += await embedAndStore(rows, options.embedder);
|
|
102
|
+
}
|
|
103
|
+
// Self-heal embedding coverage: embed anything already in the index that
|
|
104
|
+
// still lacks a vector for this model — legacy keyword-only rows, or rows
|
|
105
|
+
// from a run where the model wasn't loaded. This is why re-running
|
|
106
|
+
// collect_messages repairs a partially-embedded index without re-fetching
|
|
107
|
+
// from LINE. The rows just embedded above are no longer "missing", so they
|
|
108
|
+
// are not embedded twice.
|
|
109
|
+
if (options.embedder) {
|
|
110
|
+
messagesEmbedded += await backfillMissingEmbeddings(options.embedder);
|
|
111
|
+
}
|
|
112
|
+
return { chatsScanned: chatNames.size, messagesIndexed, messagesEmbedded };
|
|
113
|
+
}
|
|
114
|
+
/**
|
|
115
|
+
* Embed every indexed message that still lacks a vector for the given
|
|
116
|
+
* model, in batches, from stored text alone (no LINE round-trip). Best-
|
|
117
|
+
* effort like embedAndStore: if the model fails mid-sweep it stops and
|
|
118
|
+
* returns what it managed, leaving the rest keyword-only rather than
|
|
119
|
+
* aborting. Idempotent — a fully-embedded index makes this a no-op.
|
|
120
|
+
*
|
|
121
|
+
* @param embedder - Injected embedder to embed missing rows with.
|
|
122
|
+
* @returns Number of messages newly embedded and stored.
|
|
123
|
+
*/
|
|
124
|
+
export async function backfillMissingEmbeddings(embedder) {
|
|
125
|
+
const missing = getMessagesMissingEmbedding(embedder.modelLabel);
|
|
126
|
+
if (missing.length === 0) {
|
|
127
|
+
return 0;
|
|
128
|
+
}
|
|
129
|
+
let embedded = 0;
|
|
130
|
+
for (let i = 0; i < missing.length; i += EMBED_BATCH) {
|
|
131
|
+
const batch = missing.slice(i, i + EMBED_BATCH);
|
|
132
|
+
try {
|
|
133
|
+
const vectors = await embedder.embed(batch.map((row) => row.text));
|
|
134
|
+
embedded += upsertEmbeddings(batch.map((row, j) => ({
|
|
135
|
+
messageId: row.messageId,
|
|
136
|
+
vector: vectors[j],
|
|
137
|
+
})), embedder.modelLabel);
|
|
138
|
+
}
|
|
139
|
+
catch (error) {
|
|
140
|
+
log.warn('backfill.failed', {
|
|
141
|
+
error: error?.message ?? String(error),
|
|
142
|
+
remaining: missing.length - embedded,
|
|
143
|
+
});
|
|
144
|
+
break;
|
|
145
|
+
}
|
|
146
|
+
}
|
|
147
|
+
return embedded;
|
|
148
|
+
}
|
|
149
|
+
/**
|
|
150
|
+
* Batch-embed a chat's collected messages and store the resulting vectors,
|
|
151
|
+
* when an Embedder was supplied. Embedding is best-effort: any failure
|
|
152
|
+
* (model unavailable, offline, OOM, etc.) is caught and logged, never
|
|
153
|
+
* thrown — messages are already durably stored via upsertMessages by the
|
|
154
|
+
* time this runs, so a failed embed only means that batch stays
|
|
155
|
+
* keyword-only, not that collection itself failed.
|
|
156
|
+
*
|
|
157
|
+
* @param rows - Messages already upserted this batch.
|
|
158
|
+
* @param embedder - Injected embedder, or undefined to skip embedding.
|
|
159
|
+
* @returns Number of messages successfully embedded and stored.
|
|
160
|
+
*/
|
|
161
|
+
async function embedAndStore(rows, embedder) {
|
|
162
|
+
if (!embedder || rows.length === 0) {
|
|
163
|
+
return 0;
|
|
164
|
+
}
|
|
165
|
+
try {
|
|
166
|
+
const vectors = await embedder.embed(rows.map((row) => row.text));
|
|
167
|
+
return upsertEmbeddings(rows.map((row, i) => ({ messageId: row.messageId, vector: vectors[i] })), embedder.modelLabel);
|
|
168
|
+
}
|
|
169
|
+
catch (error) {
|
|
170
|
+
log.warn('embed.failed', {
|
|
171
|
+
error: error?.message ?? String(error),
|
|
172
|
+
count: rows.length,
|
|
173
|
+
});
|
|
174
|
+
return 0;
|
|
175
|
+
}
|
|
176
|
+
}
|
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Yomi's default semantic-search embedder — a local, offline ONNX model
|
|
3
|
+
* via transformers.js (@huggingface/transformers), used when no host app
|
|
4
|
+
* injects its own Embedder (see ./embedder.ts).
|
|
5
|
+
*
|
|
6
|
+
* Model: Xenova/bge-small-zh-v1.5 (BAAI general embedding, small, Chinese-
|
|
7
|
+
* primary but multilingual-capable; 512-dim). Chosen because Yomi's data
|
|
8
|
+
* is LINE messages, which for this project are predominantly Chinese.
|
|
9
|
+
*
|
|
10
|
+
* bun compatibility note: transformers.js picks a Node-flavored backend
|
|
11
|
+
* under bun (it detects a Node-like `process`), which only accepts device
|
|
12
|
+
* "cpu" | "coreml" | "webgpu" — "wasm" throws "Unsupported device". "cpu"
|
|
13
|
+
* routes through onnxruntime-node, and it works under bun WITHOUT running
|
|
14
|
+
* onnxruntime-node's postinstall script (bun blocks postinstalls by
|
|
15
|
+
* default; the packaged binary still loads fine here) — verified: model
|
|
16
|
+
* loads, embeds Chinese text, returns 512-dim unit-normalized vectors.
|
|
17
|
+
* If bge-small-zh-v1.5 ever fails to load in some environment, fall back to
|
|
18
|
+
* Xenova/multilingual-e5-small (384-dim, e5 prefix convention: query
|
|
19
|
+
* embeds get `query: `, document embeds get `passage: `).
|
|
20
|
+
*
|
|
21
|
+
* Retrieval convention: bge models are asymmetric — the QUERY gets an
|
|
22
|
+
* instruction prefix, stored documents do not. Using the model's official
|
|
23
|
+
* recommended instruction for retrieval tasks.
|
|
24
|
+
*/
|
|
25
|
+
const MODEL_ID = 'Xenova/bge-small-zh-v1.5';
|
|
26
|
+
const BGE_QUERY_INSTRUCTION = '为这个句子生成表示以用于检索相关文章:';
|
|
27
|
+
let pipelinePromise = null;
|
|
28
|
+
/**
|
|
29
|
+
* Lazily load (and memoize) the transformers.js feature-extraction
|
|
30
|
+
* pipeline for MODEL_ID. First call downloads/caches the ONNX model;
|
|
31
|
+
* subsequent calls reuse the same pipeline instance.
|
|
32
|
+
*
|
|
33
|
+
* @returns Promise resolving to the feature-extraction pipeline.
|
|
34
|
+
*/
|
|
35
|
+
function getPipeline() {
|
|
36
|
+
if (!pipelinePromise) {
|
|
37
|
+
pipelinePromise = import('@huggingface/transformers')
|
|
38
|
+
.then(({ pipeline }) => pipeline('feature-extraction', MODEL_ID, {
|
|
39
|
+
device: 'cpu',
|
|
40
|
+
}))
|
|
41
|
+
.catch((error) => {
|
|
42
|
+
// Reset the memo on failure so a later call can retry (e.g. the
|
|
43
|
+
// first attempt failed offline, network comes back later) instead
|
|
44
|
+
// of permanently caching a rejected promise.
|
|
45
|
+
pipelinePromise = null;
|
|
46
|
+
throw error;
|
|
47
|
+
});
|
|
48
|
+
}
|
|
49
|
+
return pipelinePromise;
|
|
50
|
+
}
|
|
51
|
+
/**
|
|
52
|
+
* Split a pipeline output tensor (flat data + dims [batch, hidden]) into
|
|
53
|
+
* one number[] vector per input row.
|
|
54
|
+
*
|
|
55
|
+
* @param output - Raw pipeline output.
|
|
56
|
+
* @returns One vector array per row.
|
|
57
|
+
*/
|
|
58
|
+
function splitRows(output) {
|
|
59
|
+
const [rows, hidden] = output.dims;
|
|
60
|
+
const data = output.data;
|
|
61
|
+
const vectors = [];
|
|
62
|
+
for (let i = 0; i < rows; i++) {
|
|
63
|
+
vectors.push(Array.from(data.slice(i * hidden, (i + 1) * hidden)));
|
|
64
|
+
}
|
|
65
|
+
return vectors;
|
|
66
|
+
}
|
|
67
|
+
/**
|
|
68
|
+
* Default Yomi embedder backed by transformers.js/ONNX, running fully
|
|
69
|
+
* local and offline after the first model download. Model load happens on
|
|
70
|
+
* first embed() or embedQuery() call, not at construction, so importing
|
|
71
|
+
* this module never triggers a download by itself.
|
|
72
|
+
*/
|
|
73
|
+
export class DefaultEmbedder {
|
|
74
|
+
modelLabel = MODEL_ID;
|
|
75
|
+
async embed(texts) {
|
|
76
|
+
if (texts.length === 0) {
|
|
77
|
+
return [];
|
|
78
|
+
}
|
|
79
|
+
const extractor = await getPipeline();
|
|
80
|
+
const output = await extractor(texts, { pooling: 'mean', normalize: true });
|
|
81
|
+
return splitRows(output);
|
|
82
|
+
}
|
|
83
|
+
async embedQuery(query) {
|
|
84
|
+
const [vector] = await this.embed([`${BGE_QUERY_INSTRUCTION}${query}`]);
|
|
85
|
+
return vector;
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
let defaultEmbedder = null;
|
|
89
|
+
/**
|
|
90
|
+
* Get the process-wide singleton DefaultEmbedder instance. Constructing it
|
|
91
|
+
* is cheap (no model load); the underlying pipeline only loads lazily on
|
|
92
|
+
* first use.
|
|
93
|
+
*
|
|
94
|
+
* @returns Shared DefaultEmbedder instance.
|
|
95
|
+
*/
|
|
96
|
+
export function getDefaultEmbedder() {
|
|
97
|
+
if (!defaultEmbedder) {
|
|
98
|
+
defaultEmbedder = new DefaultEmbedder();
|
|
99
|
+
}
|
|
100
|
+
return defaultEmbedder;
|
|
101
|
+
}
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Semantic-search embedder seam for Yomi's search index.
|
|
3
|
+
*
|
|
4
|
+
* This interface is the injection point between the local search index
|
|
5
|
+
* (store.ts, collector.ts) and whatever produces dense vectors. Yomi ships
|
|
6
|
+
* a default implementation (./default-embedder.ts, transformers.js/ONNX) so
|
|
7
|
+
* the MCP server works standalone, but nothing in store.ts/collector.ts
|
|
8
|
+
* imports that default directly — they only depend on this interface, so a
|
|
9
|
+
* host app (e.g. inboxd) can inject its own embedder (its local MLX
|
|
10
|
+
* `generateEmbedding`) instead. See ./default-embedder.ts's doc comment for
|
|
11
|
+
* the exact adapter shape.
|
|
12
|
+
*/
|
|
13
|
+
export {};
|