@rikaidev/yomi 0.2.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/NOTICE +81 -0
- package/PRIVACY.md +39 -0
- package/README.md +659 -0
- package/dist/auth/credential-store.js +395 -0
- package/dist/auth/keychain.js +256 -0
- package/dist/cli/login.js +142 -0
- package/dist/line/auth/protocol/e2ee-bootstrap.js +87 -0
- package/dist/line/auth/protocol/index.js +3 -0
- package/dist/line/auth/protocol/login-metadata.js +159 -0
- package/dist/line/auth/protocol/pwless-login-flow.js +83 -0
- package/dist/line/auth/pwless/index.js +163 -0
- package/dist/line/auth/pwless/steps.js +213 -0
- package/dist/line/client/auth-service/client.js +240 -0
- package/dist/line/client/e2ee-service/client.js +69 -0
- package/dist/line/client/e2ee-service/requests.js +92 -0
- package/dist/line/client/e2ee-service/shared-key-log.js +24 -0
- package/dist/line/client/e2ee-service/shared-key-payload.js +35 -0
- package/dist/line/client/index.js +213 -0
- package/dist/line/client/obs-media-client.js +140 -0
- package/dist/line/client/parsers.js +69 -0
- package/dist/line/client/shop-service/client.js +63 -0
- package/dist/line/client/sticker-meta.js +46 -0
- package/dist/line/client/sync-service/client.js +317 -0
- package/dist/line/client/talk-service/chat-query.js +122 -0
- package/dist/line/client/talk-service/command-client.js +242 -0
- package/dist/line/client/talk-service/contact-query.js +77 -0
- package/dist/line/client/talk-service/message-box-query.js +137 -0
- package/dist/line/client/talk-service/message-id-query.js +241 -0
- package/dist/line/client/talk-service/query-client.js +219 -0
- package/dist/line/client/talk-service/requests.js +383 -0
- package/dist/line/client/transport.js +138 -0
- package/dist/line/core/auth-session-logout.js +55 -0
- package/dist/line/core/auth-session-runtime.js +231 -0
- package/dist/line/core/auth-session-service.js +85 -0
- package/dist/line/core/chat-runtime-service.js +343 -0
- package/dist/line/core/client-runtime.js +17 -0
- package/dist/line/core/config.js +67 -0
- package/dist/line/core/constants.js +56 -0
- package/dist/line/core/contact-query-service.js +20 -0
- package/dist/line/core/directory-runtime-service.js +83 -0
- package/dist/line/core/e2ee/bootstrap/keychain.js +41 -0
- package/dist/line/core/e2ee/crypto/crypto-primitives.js +98 -0
- package/dist/line/core/e2ee/index.js +6 -0
- package/dist/line/core/e2ee/keys/decrypt.js +110 -0
- package/dist/line/core/e2ee/keys/encrypt.js +117 -0
- package/dist/line/core/e2ee/keys/group-key.js +260 -0
- package/dist/line/core/e2ee/keys/key-manager.js +231 -0
- package/dist/line/core/e2ee/keys/key-payload.js +130 -0
- package/dist/line/core/e2ee/keys/key-types.js +1 -0
- package/dist/line/core/e2ee/keys/message-crypto.js +146 -0
- package/dist/line/core/e2ee/keys/message-decrypt.js +390 -0
- package/dist/line/core/e2ee/keys/peer-key.js +191 -0
- package/dist/line/core/e2ee/recovery/bootstrap-keys.js +161 -0
- package/dist/line/core/e2ee/types.js +4 -0
- package/dist/line/core/group-query-service.js +20 -0
- package/dist/line/core/media-decrypt.js +78 -0
- package/dist/line/core/media-encrypt.js +97 -0
- package/dist/line/core/mention.js +89 -0
- package/dist/line/core/message-command-service.js +454 -0
- package/dist/line/core/message-query-service.js +161 -0
- package/dist/line/core/service.js +134 -0
- package/dist/line/core/session-state/index.js +243 -0
- package/dist/line/core/session-state/persist.js +192 -0
- package/dist/line/core/store.js +8 -0
- package/dist/line/core/thrift/binary/binary-reader.js +151 -0
- package/dist/line/core/thrift/binary/binary-writer.js +157 -0
- package/dist/line/core/thrift/binary/type-codec-dsl.js +42 -0
- package/dist/line/core/thrift/compact/compact-reader.js +180 -0
- package/dist/line/core/thrift/compact/compact-writer.js +161 -0
- package/dist/line/core/thrift/compact/type-codec-dsl.js +66 -0
- package/dist/line/core/thrift/fields/builders.js +124 -0
- package/dist/line/core/thrift/index.js +12 -0
- package/dist/line/core/thrift/message/decoder.js +55 -0
- package/dist/line/core/thrift/message/encoder.js +25 -0
- package/dist/line/core/thrift/primitives/varint.js +64 -0
- package/dist/line/core/thrift/struct/reader.js +172 -0
- package/dist/line/core/thrift/struct/writer.js +189 -0
- package/dist/line/core/thrift/types.js +23 -0
- package/dist/line/core/video-thumbnail.js +104 -0
- package/dist/mcp/handlers/contacts.js +166 -0
- package/dist/mcp/handlers/conversations.js +193 -0
- package/dist/mcp/handlers/groups.js +123 -0
- package/dist/mcp/handlers/index.js +15 -0
- package/dist/mcp/handlers/login.js +473 -0
- package/dist/mcp/handlers/media.js +69 -0
- package/dist/mcp/handlers/message-actions.js +62 -0
- package/dist/mcp/handlers/scope.js +120 -0
- package/dist/mcp/handlers/search.js +196 -0
- package/dist/mcp/handlers/send.js +523 -0
- package/dist/mcp/handlers/shared.js +84 -0
- package/dist/mcp/handlers/stickers.js +87 -0
- package/dist/mcp/media.js +228 -0
- package/dist/mcp/names.js +133 -0
- package/dist/mcp/phi-guard.js +105 -0
- package/dist/mcp/policy.js +34 -0
- package/dist/mcp/server.js +301 -0
- package/dist/mcp/tools.js +761 -0
- package/dist/mcp/ui/capability.js +26 -0
- package/dist/mcp/ui/login-app-protocol.js +118 -0
- package/dist/mcp/ui/login-app-styles.js +142 -0
- package/dist/mcp/ui/login-app-view.js +361 -0
- package/dist/mcp/ui/login-app.js +72 -0
- package/dist/mcp/ui/login-structured-content.js +1 -0
- package/dist/mcp/ui/resource.js +35 -0
- package/dist/mcp/ui/tools-with-ui.js +40 -0
- package/dist/search/bigram.js +86 -0
- package/dist/search/capture.js +140 -0
- package/dist/search/collector.js +176 -0
- package/dist/search/default-embedder.js +101 -0
- package/dist/search/embedder.js +13 -0
- package/dist/search/scope.js +142 -0
- package/dist/search/sqlite.js +141 -0
- package/dist/search/store.js +469 -0
- package/dist/util/data-dir.js +55 -0
- package/dist/util/log.js +86 -0
- package/dist/version.js +7 -0
- package/package.json +63 -0
- package/run.mjs +71 -0
|
@@ -0,0 +1,469 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Yomi local cross-conversation search index — a node:sqlite FTS5 store
|
|
3
|
+
* over collected LINE messages.
|
|
4
|
+
*
|
|
5
|
+
* This is a SEPARATE concern from the pure-query MCP tools in src/mcp/.
|
|
6
|
+
* The index is only ever written to by collect_messages (see
|
|
7
|
+
* ../search/collector.ts), an explicit caller-invoked operation — never by
|
|
8
|
+
* list_conversations/get_chat_messages/etc, which stay pure-query and never
|
|
9
|
+
* touch this file. Search is keyword-only (FTS5 bm25 ranking); no
|
|
10
|
+
* relationship graph, no affinity scoring, no interaction metrics live
|
|
11
|
+
* here.
|
|
12
|
+
*
|
|
13
|
+
* Keyword search has zero external dependencies (Node.js built-in
|
|
14
|
+
* `node:sqlite`, which bundles FTS5). Semantic search is an additional,
|
|
15
|
+
* optional tier on top: each message's text can also be embedded into a
|
|
16
|
+
* dense vector (stored in the `embeddings` table, see getDb()) and ranked
|
|
17
|
+
* by brute-force cosine similarity in JS — no native vector extension, this
|
|
18
|
+
* is personal-scale (thousands of rows, not millions). The embedder itself
|
|
19
|
+
* is always caller-injected (see ../search/embedder.ts); this file never
|
|
20
|
+
* imports a concrete embedder implementation.
|
|
21
|
+
*/
|
|
22
|
+
import { chmodSync, existsSync, mkdirSync } from 'node:fs';
|
|
23
|
+
import { dirname } from 'node:path';
|
|
24
|
+
import { yomiDataPath } from '../util/data-dir.js';
|
|
25
|
+
import { toSearchText } from './bigram.js';
|
|
26
|
+
import { openDatabase } from './sqlite.js';
|
|
27
|
+
/**
|
|
28
|
+
* Narrow files to owner-only (0600), skipping any that do not exist.
|
|
29
|
+
*
|
|
30
|
+
* Best-effort: a chmod failure must never take the index down, since the data
|
|
31
|
+
* is still usable and the process may not own the file (e.g. a shared path an
|
|
32
|
+
* operator deliberately set up).
|
|
33
|
+
*
|
|
34
|
+
* @param paths - Files to restrict.
|
|
35
|
+
*/
|
|
36
|
+
function restrictToOwner(...paths) {
|
|
37
|
+
for (const path of paths) {
|
|
38
|
+
try {
|
|
39
|
+
if (existsSync(path)) {
|
|
40
|
+
chmodSync(path, 0o600);
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
catch {
|
|
44
|
+
// Not ours to tighten, or a platform without POSIX modes.
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
// A per-user data directory (see ../util/data-dir.ts), NOT process.cwd() and
|
|
49
|
+
// NOT the install directory. This used to resolve relative to the module, which
|
|
50
|
+
// put the index inside the installed package — and `npx @rikaidev/yomi` unpacks
|
|
51
|
+
// under ~/.npm/_npx/<hash>/, which npm deletes on any cache refresh. The index
|
|
52
|
+
// went with it. YOMI_INDEX_DB_PATH still overrides for tests/custom setups.
|
|
53
|
+
const DB_PATH = process.env.YOMI_INDEX_DB_PATH ?? yomiDataPath('search-index.db');
|
|
54
|
+
let db = null;
|
|
55
|
+
/**
|
|
56
|
+
* Open (creating if needed) the Yomi search index database and ensure its
|
|
57
|
+
* schema exists. Memoized: subsequent calls reuse the same connection.
|
|
58
|
+
*
|
|
59
|
+
* Schema: `messages` is the authoritative row store (unique on messageId
|
|
60
|
+
* for dedup). `text` holds the raw message body for display; `search_text`
|
|
61
|
+
* holds the bigram-preprocessed form (see ./bigram.ts) that actually gets
|
|
62
|
+
* indexed — the default unicode61 FTS5 tokenizer has no CJK word
|
|
63
|
+
* segmentation, so a whole Chinese/Japanese/Korean run would otherwise
|
|
64
|
+
* collapse into one opaque token and never match a substring search.
|
|
65
|
+
* `messages_fts` is an external-content FTS5 index over `messages.search_text`,
|
|
66
|
+
* kept in sync by INSERT/UPDATE/DELETE triggers so plain upserts into
|
|
67
|
+
* `messages` automatically maintain the search index.
|
|
68
|
+
*
|
|
69
|
+
* @returns Open node:sqlite database handle.
|
|
70
|
+
*/
|
|
71
|
+
export function getDb() {
|
|
72
|
+
if (db) {
|
|
73
|
+
return db;
|
|
74
|
+
}
|
|
75
|
+
const dir = dirname(DB_PATH);
|
|
76
|
+
if (dir && dir !== '.' && !existsSync(dir)) {
|
|
77
|
+
mkdirSync(dir, { recursive: true, mode: 0o700 });
|
|
78
|
+
}
|
|
79
|
+
const handle = openDatabase(DB_PATH);
|
|
80
|
+
handle.exec('PRAGMA journal_mode = WAL;');
|
|
81
|
+
// This file is every captured message in plaintext — the single most
|
|
82
|
+
// sensitive artifact Yomi produces. It was being created with the default
|
|
83
|
+
// 0644, readable by every other account on the machine. WAL mode means the
|
|
84
|
+
// -wal and -shm siblings hold message data too, so they get the same
|
|
85
|
+
// treatment. Applied on every open, not just creation, so databases that
|
|
86
|
+
// already exist with the old permissions are repaired rather than left
|
|
87
|
+
// exposed for the lifetime of the install.
|
|
88
|
+
restrictToOwner(DB_PATH, `${DB_PATH}-wal`, `${DB_PATH}-shm`);
|
|
89
|
+
handle.exec(`
|
|
90
|
+
CREATE TABLE IF NOT EXISTS messages (
|
|
91
|
+
id INTEGER PRIMARY KEY,
|
|
92
|
+
messageId TEXT UNIQUE NOT NULL,
|
|
93
|
+
chatId TEXT NOT NULL,
|
|
94
|
+
chatName TEXT,
|
|
95
|
+
fromMid TEXT,
|
|
96
|
+
fromName TEXT,
|
|
97
|
+
text TEXT NOT NULL,
|
|
98
|
+
search_text TEXT NOT NULL,
|
|
99
|
+
createdTime INTEGER
|
|
100
|
+
);
|
|
101
|
+
`);
|
|
102
|
+
handle.exec(`
|
|
103
|
+
CREATE VIRTUAL TABLE IF NOT EXISTS messages_fts USING fts5(
|
|
104
|
+
search_text,
|
|
105
|
+
content='messages',
|
|
106
|
+
content_rowid='id'
|
|
107
|
+
);
|
|
108
|
+
`);
|
|
109
|
+
handle.exec(`
|
|
110
|
+
CREATE TRIGGER IF NOT EXISTS messages_ai AFTER INSERT ON messages BEGIN
|
|
111
|
+
INSERT INTO messages_fts(rowid, search_text) VALUES (new.id, new.search_text);
|
|
112
|
+
END;
|
|
113
|
+
`);
|
|
114
|
+
handle.exec(`
|
|
115
|
+
CREATE TRIGGER IF NOT EXISTS messages_ad AFTER DELETE ON messages BEGIN
|
|
116
|
+
INSERT INTO messages_fts(messages_fts, rowid, search_text) VALUES('delete', old.id, old.search_text);
|
|
117
|
+
END;
|
|
118
|
+
`);
|
|
119
|
+
handle.exec(`
|
|
120
|
+
CREATE TRIGGER IF NOT EXISTS messages_au AFTER UPDATE ON messages BEGIN
|
|
121
|
+
INSERT INTO messages_fts(messages_fts, rowid, search_text) VALUES('delete', old.id, old.search_text);
|
|
122
|
+
INSERT INTO messages_fts(rowid, search_text) VALUES (new.id, new.search_text);
|
|
123
|
+
END;
|
|
124
|
+
`);
|
|
125
|
+
// Semantic search vectors, kept separate from `messages` and keyed by
|
|
126
|
+
// (messageId, model): a Float32 vector BLOB per (message, embedding
|
|
127
|
+
// model) pair. Keying on model means switching embedders never mixes
|
|
128
|
+
// incompatible vectors/dimensionality — old-model rows just go unused
|
|
129
|
+
// until re-embedded, they are never compared against new-model vectors.
|
|
130
|
+
handle.exec(`
|
|
131
|
+
CREATE TABLE IF NOT EXISTS embeddings (
|
|
132
|
+
messageId TEXT NOT NULL,
|
|
133
|
+
model TEXT NOT NULL,
|
|
134
|
+
dim INTEGER NOT NULL,
|
|
135
|
+
vector BLOB NOT NULL,
|
|
136
|
+
PRIMARY KEY (messageId, model)
|
|
137
|
+
);
|
|
138
|
+
`);
|
|
139
|
+
// Observability/resume cursor for the continuous capture loop (see
|
|
140
|
+
// ../search/capture.ts): a single-row table recording the LINE sync
|
|
141
|
+
// revision counters as of the last successful capture flush. Capture
|
|
142
|
+
// itself is idempotent (collectMessages upserts), so this is not load-
|
|
143
|
+
// bearing for correctness — it exists so a future resume path or
|
|
144
|
+
// diagnostics can see where capture last got to.
|
|
145
|
+
handle.exec(`
|
|
146
|
+
CREATE TABLE IF NOT EXISTS capture_state (
|
|
147
|
+
id INTEGER PRIMARY KEY CHECK (id = 1),
|
|
148
|
+
revision INTEGER,
|
|
149
|
+
globalRevision INTEGER,
|
|
150
|
+
individualRevision INTEGER,
|
|
151
|
+
updatedAt INTEGER
|
|
152
|
+
);
|
|
153
|
+
`);
|
|
154
|
+
db = handle;
|
|
155
|
+
return handle;
|
|
156
|
+
}
|
|
157
|
+
/**
|
|
158
|
+
* Upsert a batch of collected messages into the search index, deduping on
|
|
159
|
+
* messageId (LINE's stable per-message identifier). Existing rows are
|
|
160
|
+
* updated in place; the messages_ai/messages_au triggers keep the FTS5
|
|
161
|
+
* index in sync automatically, so callers never touch messages_fts
|
|
162
|
+
* directly.
|
|
163
|
+
*
|
|
164
|
+
* @param rows - Messages to index.
|
|
165
|
+
* @returns Number of rows upserted.
|
|
166
|
+
*/
|
|
167
|
+
export function upsertMessages(rows) {
|
|
168
|
+
if (rows.length === 0) {
|
|
169
|
+
return 0;
|
|
170
|
+
}
|
|
171
|
+
const handle = getDb();
|
|
172
|
+
const stmt = handle.prepare(`
|
|
173
|
+
INSERT INTO messages (messageId, chatId, chatName, fromMid, fromName, text, search_text, createdTime)
|
|
174
|
+
VALUES ($messageId, $chatId, $chatName, $fromMid, $fromName, $text, $searchText, $createdTime)
|
|
175
|
+
ON CONFLICT(messageId) DO UPDATE SET
|
|
176
|
+
chatId = excluded.chatId,
|
|
177
|
+
chatName = excluded.chatName,
|
|
178
|
+
fromMid = excluded.fromMid,
|
|
179
|
+
fromName = excluded.fromName,
|
|
180
|
+
text = excluded.text,
|
|
181
|
+
search_text = excluded.search_text,
|
|
182
|
+
createdTime = excluded.createdTime;
|
|
183
|
+
`);
|
|
184
|
+
const upsertAll = handle.transaction((batch) => {
|
|
185
|
+
for (const row of batch) {
|
|
186
|
+
stmt.run({
|
|
187
|
+
$messageId: row.messageId,
|
|
188
|
+
$chatId: row.chatId,
|
|
189
|
+
$chatName: row.chatName,
|
|
190
|
+
$fromMid: row.fromMid,
|
|
191
|
+
$fromName: row.fromName,
|
|
192
|
+
$text: row.text,
|
|
193
|
+
$searchText: toSearchText(row.text),
|
|
194
|
+
$createdTime: row.createdTime,
|
|
195
|
+
});
|
|
196
|
+
}
|
|
197
|
+
});
|
|
198
|
+
upsertAll(rows);
|
|
199
|
+
return rows.length;
|
|
200
|
+
}
|
|
201
|
+
/**
|
|
202
|
+
* Count messages currently in the search index. Used to give an honest
|
|
203
|
+
* "index is empty, run collect_messages first" answer instead of silently
|
|
204
|
+
* returning zero results as if the search itself found nothing.
|
|
205
|
+
*
|
|
206
|
+
* @returns Total indexed message count.
|
|
207
|
+
*/
|
|
208
|
+
export function getIndexedMessageCount() {
|
|
209
|
+
const handle = getDb();
|
|
210
|
+
const row = handle
|
|
211
|
+
.prepare('SELECT COUNT(*) as count FROM messages;')
|
|
212
|
+
.get();
|
|
213
|
+
return row?.count ?? 0;
|
|
214
|
+
}
|
|
215
|
+
/**
|
|
216
|
+
* Build a safe FTS5 MATCH string from a raw user query: bigram-preprocess
|
|
217
|
+
* it with the same toSearchText used at index time (so a CJK substring
|
|
218
|
+
* lines up with the bigrams stored in search_text), then wrap each
|
|
219
|
+
* resulting token in double quotes and join with spaces. Quoting every
|
|
220
|
+
* token neutralizes FTS5 query-syntax characters (AND, OR, NOT, prefix
|
|
221
|
+
* wildcard, column filters, minus-exclusion, etc.) that could otherwise
|
|
222
|
+
* appear in arbitrary user input and break or hijack
|
|
223
|
+
* the query; space-separated quoted tokens form an implicit AND, which is
|
|
224
|
+
* exactly the "all bigrams present" substring semantics this feature needs.
|
|
225
|
+
*
|
|
226
|
+
* @param query - Raw user search string.
|
|
227
|
+
* @returns Quoted-token MATCH string, or '' when the query has no tokens.
|
|
228
|
+
*/
|
|
229
|
+
function buildMatchQuery(query) {
|
|
230
|
+
const searchText = toSearchText(query);
|
|
231
|
+
if (!searchText) {
|
|
232
|
+
return '';
|
|
233
|
+
}
|
|
234
|
+
return searchText
|
|
235
|
+
.split(' ')
|
|
236
|
+
.filter(Boolean)
|
|
237
|
+
.map((token) => `"${token.replace(/"/g, '""')}"`)
|
|
238
|
+
.join(' ');
|
|
239
|
+
}
|
|
240
|
+
/**
|
|
241
|
+
* Full-text search the collected message index, ranked by FTS5 bm25
|
|
242
|
+
* relevance (best match first). Searches ONLY already-collected messages —
|
|
243
|
+
* callers should check getIndexedMessageCount() first to distinguish "no
|
|
244
|
+
* matches" from "nothing indexed yet".
|
|
245
|
+
*
|
|
246
|
+
* The query is bigram-preprocessed with the same toSearchText applied at
|
|
247
|
+
* index time, so CJK substrings (no native word boundaries) match the same
|
|
248
|
+
* way Latin words do. When preprocessing yields no tokens at all (e.g. a
|
|
249
|
+
* punctuation-only query), this falls back to a plain LIKE substring scan
|
|
250
|
+
* over the raw `messages.text` rather than returning an empty result.
|
|
251
|
+
*
|
|
252
|
+
* @param query - Search query. Plain keywords (Latin or CJK) work.
|
|
253
|
+
* @param limit - Maximum number of results (default 20).
|
|
254
|
+
* @returns Matching messages, best relevance first.
|
|
255
|
+
*/
|
|
256
|
+
export function searchMessages(query, limit = 20) {
|
|
257
|
+
const handle = getDb();
|
|
258
|
+
const matchQuery = buildMatchQuery(query);
|
|
259
|
+
if (!matchQuery) {
|
|
260
|
+
const rows = handle
|
|
261
|
+
.prepare(`
|
|
262
|
+
SELECT chatId, chatName, messageId, fromName, text, createdTime
|
|
263
|
+
FROM messages
|
|
264
|
+
WHERE text LIKE '%' || $query || '%'
|
|
265
|
+
ORDER BY id DESC
|
|
266
|
+
LIMIT $limit;
|
|
267
|
+
`)
|
|
268
|
+
.all({ $query: query, $limit: limit });
|
|
269
|
+
return rows;
|
|
270
|
+
}
|
|
271
|
+
const rows = handle
|
|
272
|
+
.prepare(`
|
|
273
|
+
SELECT m.chatId as chatId, m.chatName as chatName, m.messageId as messageId,
|
|
274
|
+
m.fromName as fromName, m.text as text, m.createdTime as createdTime
|
|
275
|
+
FROM messages_fts f
|
|
276
|
+
JOIN messages m ON m.id = f.rowid
|
|
277
|
+
WHERE messages_fts MATCH $query
|
|
278
|
+
ORDER BY bm25(messages_fts)
|
|
279
|
+
LIMIT $limit;
|
|
280
|
+
`)
|
|
281
|
+
.all({ $query: matchQuery, $limit: limit });
|
|
282
|
+
return rows;
|
|
283
|
+
}
|
|
284
|
+
/**
|
|
285
|
+
* Serialize a unit-normalized embedding vector to a compact BLOB (raw
|
|
286
|
+
* Float32 bytes) for storage.
|
|
287
|
+
*
|
|
288
|
+
* @param vector - Embedding vector.
|
|
289
|
+
* @returns Buffer suitable for a node:sqlite BLOB parameter.
|
|
290
|
+
*/
|
|
291
|
+
function encodeVector(vector) {
|
|
292
|
+
return new Uint8Array(Float32Array.from(vector).buffer);
|
|
293
|
+
}
|
|
294
|
+
/**
|
|
295
|
+
* Deserialize a stored BLOB back into a Float32Array vector.
|
|
296
|
+
*
|
|
297
|
+
* @param blob - Raw BLOB bytes read back from the `embeddings` table.
|
|
298
|
+
* @returns Decoded Float32Array vector.
|
|
299
|
+
*/
|
|
300
|
+
function decodeVector(blob) {
|
|
301
|
+
return new Float32Array(blob.buffer, blob.byteOffset, blob.byteLength / Float32Array.BYTES_PER_ELEMENT);
|
|
302
|
+
}
|
|
303
|
+
/**
|
|
304
|
+
* Upsert a batch of message embeddings for one embedder model, keyed on
|
|
305
|
+
* (messageId, model). Called by the collector after upsertMessages, with
|
|
306
|
+
* whatever Embedder the caller injected — this function has no opinion on
|
|
307
|
+
* which model produced the vectors, it only records the label alongside
|
|
308
|
+
* them so semanticSearch can compare like with like.
|
|
309
|
+
*
|
|
310
|
+
* @param rows - messageId + vector pairs to store.
|
|
311
|
+
* @param model - Embedder.modelLabel that produced these vectors.
|
|
312
|
+
* @returns Number of rows upserted.
|
|
313
|
+
*/
|
|
314
|
+
export function upsertEmbeddings(rows, model) {
|
|
315
|
+
if (rows.length === 0) {
|
|
316
|
+
return 0;
|
|
317
|
+
}
|
|
318
|
+
const handle = getDb();
|
|
319
|
+
const stmt = handle.prepare(`
|
|
320
|
+
INSERT INTO embeddings (messageId, model, dim, vector)
|
|
321
|
+
VALUES ($messageId, $model, $dim, $vector)
|
|
322
|
+
ON CONFLICT(messageId, model) DO UPDATE SET
|
|
323
|
+
dim = excluded.dim,
|
|
324
|
+
vector = excluded.vector;
|
|
325
|
+
`);
|
|
326
|
+
const upsertAll = handle.transaction((batch) => {
|
|
327
|
+
for (const row of batch) {
|
|
328
|
+
stmt.run({
|
|
329
|
+
$messageId: row.messageId,
|
|
330
|
+
$model: model,
|
|
331
|
+
$dim: row.vector.length,
|
|
332
|
+
$vector: encodeVector(row.vector),
|
|
333
|
+
});
|
|
334
|
+
}
|
|
335
|
+
});
|
|
336
|
+
upsertAll(rows);
|
|
337
|
+
return rows.length;
|
|
338
|
+
}
|
|
339
|
+
/**
|
|
340
|
+
* Count vectors currently stored for a given embedder model. Used to
|
|
341
|
+
* decide whether semantic search has anything to rank over for the
|
|
342
|
+
* currently-configured embedder, distinct from getIndexedMessageCount()
|
|
343
|
+
* (which counts raw messages regardless of embedding status).
|
|
344
|
+
*
|
|
345
|
+
* @param model - Embedder.modelLabel to count vectors for.
|
|
346
|
+
* @returns Number of stored vectors for that model.
|
|
347
|
+
*/
|
|
348
|
+
export function getEmbeddingCount(model) {
|
|
349
|
+
const handle = getDb();
|
|
350
|
+
const row = handle
|
|
351
|
+
.prepare('SELECT COUNT(*) as count FROM embeddings WHERE model = $model;')
|
|
352
|
+
.get({ $model: model });
|
|
353
|
+
return row?.count ?? 0;
|
|
354
|
+
}
|
|
355
|
+
/**
|
|
356
|
+
* List indexed messages that have no embedding for the given model — the
|
|
357
|
+
* text still needed to embed them. This is what makes embedding coverage
|
|
358
|
+
* self-healing: a message can enter the index keyword-only (collected
|
|
359
|
+
* before an embedder was wired, or during a run where the model failed to
|
|
360
|
+
* load) and later be embedded from its stored text alone, with no need to
|
|
361
|
+
* re-fetch it from LINE. Empty-text rows are excluded (nothing to embed).
|
|
362
|
+
*
|
|
363
|
+
* @param model - Embedder.modelLabel to check coverage for.
|
|
364
|
+
* @returns messageId + text for every indexed message missing a vector for that model.
|
|
365
|
+
*/
|
|
366
|
+
export function getMessagesMissingEmbedding(model) {
|
|
367
|
+
const handle = getDb();
|
|
368
|
+
return handle
|
|
369
|
+
.prepare(`
|
|
370
|
+
SELECT m.messageId as messageId, m.text as text
|
|
371
|
+
FROM messages m
|
|
372
|
+
LEFT JOIN embeddings e ON e.messageId = m.messageId AND e.model = $model
|
|
373
|
+
WHERE e.messageId IS NULL AND length(m.text) > 0;
|
|
374
|
+
`)
|
|
375
|
+
.all({ $model: model });
|
|
376
|
+
}
|
|
377
|
+
/**
|
|
378
|
+
* Cosine similarity between two equal-length vectors. Both stored and
|
|
379
|
+
* query vectors are unit-normalized at embed time, so this reduces to a
|
|
380
|
+
* plain dot product — kept as an explicit cosine for clarity and so the
|
|
381
|
+
* function stays correct even if that normalization invariant ever
|
|
382
|
+
* changes.
|
|
383
|
+
*
|
|
384
|
+
* @param a - First vector.
|
|
385
|
+
* @param b - Second vector.
|
|
386
|
+
* @returns Cosine similarity in [-1, 1].
|
|
387
|
+
*/
|
|
388
|
+
function cosineSimilarity(a, b) {
|
|
389
|
+
let dot = 0;
|
|
390
|
+
let normA = 0;
|
|
391
|
+
let normB = 0;
|
|
392
|
+
for (let i = 0; i < b.length; i++) {
|
|
393
|
+
dot += a[i] * b[i];
|
|
394
|
+
normA += a[i] * a[i];
|
|
395
|
+
normB += b[i] * b[i];
|
|
396
|
+
}
|
|
397
|
+
const denom = Math.sqrt(normA) * Math.sqrt(normB);
|
|
398
|
+
return denom === 0 ? 0 : dot / denom;
|
|
399
|
+
}
|
|
400
|
+
/**
|
|
401
|
+
* Semantic search: embed the query with the caller-injected Embedder, then
|
|
402
|
+
* brute-force cosine-similarity-rank all stored vectors for that model.
|
|
403
|
+
* Personal-scale (thousands of rows) — no vector index, just a JS scan.
|
|
404
|
+
*
|
|
405
|
+
* Callers should check getEmbeddingCount(embedder.modelLabel) > 0 first
|
|
406
|
+
* (or catch embed() throwing) to distinguish "no semantic index yet" from
|
|
407
|
+
* "genuinely no matches", and fall back to searchMessages() accordingly —
|
|
408
|
+
* this function itself does not fall back, so tiering stays a caller
|
|
409
|
+
* decision (see ../mcp/search-handlers.ts).
|
|
410
|
+
*
|
|
411
|
+
* @param query - Raw user search query.
|
|
412
|
+
* @param limit - Maximum number of results (default 20).
|
|
413
|
+
* @param embedder - Injected embedder (default: DefaultEmbedder, or a host-supplied one).
|
|
414
|
+
* @returns Matching messages, best cosine similarity first.
|
|
415
|
+
*/
|
|
416
|
+
export async function semanticSearch(query, limit, embedder) {
|
|
417
|
+
const queryVector = await embedder.embedQuery(query);
|
|
418
|
+
const handle = getDb();
|
|
419
|
+
const rows = handle
|
|
420
|
+
.prepare(`
|
|
421
|
+
SELECT m.chatId as chatId, m.chatName as chatName, m.messageId as messageId,
|
|
422
|
+
m.fromName as fromName, m.text as text, m.createdTime as createdTime,
|
|
423
|
+
e.vector as vector
|
|
424
|
+
FROM embeddings e
|
|
425
|
+
JOIN messages m ON m.messageId = e.messageId
|
|
426
|
+
WHERE e.model = $model;
|
|
427
|
+
`)
|
|
428
|
+
.all({ $model: embedder.modelLabel });
|
|
429
|
+
const scored = rows.map((row) => {
|
|
430
|
+
const { vector, ...result } = row;
|
|
431
|
+
return {
|
|
432
|
+
...result,
|
|
433
|
+
score: cosineSimilarity(decodeVector(vector), queryVector),
|
|
434
|
+
};
|
|
435
|
+
});
|
|
436
|
+
scored.sort((a, b) => b.score - a.score);
|
|
437
|
+
return scored.slice(0, limit);
|
|
438
|
+
}
|
|
439
|
+
/**
|
|
440
|
+
* Read the persisted capture cursor, if any has been saved yet.
|
|
441
|
+
*
|
|
442
|
+
* @returns The persisted capture state, or null before capture has ever flushed.
|
|
443
|
+
*/
|
|
444
|
+
export function getCaptureState() {
|
|
445
|
+
const row = getDb()
|
|
446
|
+
.prepare('SELECT revision, globalRevision, individualRevision, updatedAt FROM capture_state WHERE id = 1')
|
|
447
|
+
.get();
|
|
448
|
+
return row
|
|
449
|
+
? {
|
|
450
|
+
revision: row.revision ?? 0,
|
|
451
|
+
globalRevision: row.globalRevision ?? 0,
|
|
452
|
+
individualRevision: row.individualRevision ?? 0,
|
|
453
|
+
updatedAt: row.updatedAt ?? 0,
|
|
454
|
+
}
|
|
455
|
+
: null;
|
|
456
|
+
}
|
|
457
|
+
/**
|
|
458
|
+
* Persist the current capture cursor, overwriting the single stored row.
|
|
459
|
+
*
|
|
460
|
+
* @param state - Capture cursor to persist.
|
|
461
|
+
*/
|
|
462
|
+
export function saveCaptureState(state) {
|
|
463
|
+
getDb().exec('INSERT INTO capture_state (id, revision, globalRevision, individualRevision, updatedAt) VALUES (1, ?, ?, ?, ?) ON CONFLICT(id) DO UPDATE SET revision = excluded.revision, globalRevision = excluded.globalRevision, individualRevision = excluded.individualRevision, updatedAt = excluded.updatedAt', [
|
|
464
|
+
state.revision,
|
|
465
|
+
state.globalRevision,
|
|
466
|
+
state.individualRevision,
|
|
467
|
+
state.updatedAt,
|
|
468
|
+
]);
|
|
469
|
+
}
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Where Yomi keeps its on-disk state (message index, E2EE key cache, and the
|
|
3
|
+
* credential file on platforms without a keychain).
|
|
4
|
+
*
|
|
5
|
+
* This has to avoid two tempting-but-wrong answers, and the project shipped
|
|
6
|
+
* both of them at once:
|
|
7
|
+
*
|
|
8
|
+
* - `process.cwd()` — an MCP client spawns the server with whatever working
|
|
9
|
+
* directory it likes, and Claude Desktop uses `/`. The E2EE cache was
|
|
10
|
+
* cwd-relative, so it tried to write `/data/line-e2ee-cache.json`, failed,
|
|
11
|
+
* and had the failure swallowed. Every peer key was refetched on every run
|
|
12
|
+
* and no group-key epoch was ever persisted.
|
|
13
|
+
*
|
|
14
|
+
* - the install directory — `npx @rikaidev/yomi` unpacks under
|
|
15
|
+
* `~/.npm/_npx/<hash>/`, which npm deletes on any cache refresh. The index
|
|
16
|
+
* was resolved relative to the module, so it lived there: one
|
|
17
|
+
* `npx cache clean` would take the whole message index with it. Group-key
|
|
18
|
+
* epochs are the sharp edge, since LINE only ever returns the latest one.
|
|
19
|
+
*
|
|
20
|
+
* A stable per-user directory is the only location that survives both. Honour
|
|
21
|
+
* the platform convention so it lands where a user would look for it, and let
|
|
22
|
+
* YOMI_DATA_DIR override for tests and custom setups.
|
|
23
|
+
*/
|
|
24
|
+
import { homedir } from 'node:os';
|
|
25
|
+
import { join } from 'node:path';
|
|
26
|
+
/**
|
|
27
|
+
* Resolve Yomi's data directory. Never cwd-relative, never install-relative.
|
|
28
|
+
*
|
|
29
|
+
* @returns Absolute path to the directory Yomi's state lives in.
|
|
30
|
+
*/
|
|
31
|
+
export function resolveYomiDataDir() {
|
|
32
|
+
const override = process.env.YOMI_DATA_DIR;
|
|
33
|
+
if (override) {
|
|
34
|
+
return override;
|
|
35
|
+
}
|
|
36
|
+
const home = homedir();
|
|
37
|
+
if (process.platform === 'darwin') {
|
|
38
|
+
return join(home, 'Library', 'Application Support', 'yomi');
|
|
39
|
+
}
|
|
40
|
+
if (process.platform === 'win32') {
|
|
41
|
+
const appData = process.env.APPDATA ?? join(home, 'AppData', 'Roaming');
|
|
42
|
+
return join(appData, 'yomi');
|
|
43
|
+
}
|
|
44
|
+
const xdg = process.env.XDG_DATA_HOME ?? join(home, '.local', 'share');
|
|
45
|
+
return join(xdg, 'yomi');
|
|
46
|
+
}
|
|
47
|
+
/**
|
|
48
|
+
* Resolve one file inside Yomi's data directory.
|
|
49
|
+
*
|
|
50
|
+
* @param name - File name, e.g. `search-index.db`.
|
|
51
|
+
* @returns Absolute path to that file.
|
|
52
|
+
*/
|
|
53
|
+
export function yomiDataPath(name) {
|
|
54
|
+
return join(resolveYomiDataDir(), name);
|
|
55
|
+
}
|
package/dist/util/log.js
ADDED
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Yomi CLI logger — minimal structured logger.
|
|
3
|
+
*
|
|
4
|
+
* Mirrors the call shape used throughout the vendored LINE protocol core
|
|
5
|
+
* (`logger.info(event, context)` / `.warn` / `.error` / `.debug` / `.child`)
|
|
6
|
+
* so no call sites needed to change during extraction.
|
|
7
|
+
*
|
|
8
|
+
* Output uses `[TAG]` prefixes — no emoji, per project convention.
|
|
9
|
+
*/
|
|
10
|
+
/**
|
|
11
|
+
* Format a primitive value for key=value log output.
|
|
12
|
+
*
|
|
13
|
+
* @param value - Primitive log value.
|
|
14
|
+
* @returns String representation suitable for log output.
|
|
15
|
+
*/
|
|
16
|
+
function formatValue(value) {
|
|
17
|
+
if (value === null)
|
|
18
|
+
return 'null';
|
|
19
|
+
if (value === undefined)
|
|
20
|
+
return 'undefined';
|
|
21
|
+
if (typeof value === 'string')
|
|
22
|
+
return JSON.stringify(value);
|
|
23
|
+
if (typeof value === 'bigint')
|
|
24
|
+
return value.toString();
|
|
25
|
+
return String(value);
|
|
26
|
+
}
|
|
27
|
+
/**
|
|
28
|
+
* Format structured context into sorted key=value segments.
|
|
29
|
+
*
|
|
30
|
+
* @param context - Optional structured log context.
|
|
31
|
+
* @returns Formatted suffix beginning with a leading space when non-empty.
|
|
32
|
+
*/
|
|
33
|
+
function formatContext(context) {
|
|
34
|
+
if (!context)
|
|
35
|
+
return '';
|
|
36
|
+
const entries = Object.entries(context)
|
|
37
|
+
.filter(([, value]) => value !== undefined)
|
|
38
|
+
.sort(([a], [b]) => a.localeCompare(b))
|
|
39
|
+
.map(([key, value]) => `${key}=${formatValue(value)}`);
|
|
40
|
+
return entries.length > 0 ? ` ${entries.join(' ')}` : '';
|
|
41
|
+
}
|
|
42
|
+
/**
|
|
43
|
+
* Emit one log line to stderr, prefixed with the scope tag.
|
|
44
|
+
*
|
|
45
|
+
* stderr is used (not stdout) because Yomi's MCP server speaks JSON-RPC
|
|
46
|
+
* over stdout for the stdio transport — any stray stdout write would
|
|
47
|
+
* corrupt the protocol stream.
|
|
48
|
+
*
|
|
49
|
+
* @param method - Console method to use.
|
|
50
|
+
* @param scope - Logger scope, rendered as `[SCOPE]`.
|
|
51
|
+
* @param level - Log level name.
|
|
52
|
+
* @param event - Event identifier.
|
|
53
|
+
* @param context - Optional structured context.
|
|
54
|
+
* @param options - Optional rendering options such as indentation.
|
|
55
|
+
*/
|
|
56
|
+
function emit(_method, scope, level, event, context, options = {}) {
|
|
57
|
+
const indent = options.indent ?? '';
|
|
58
|
+
const line = `${indent}[${scope}] ${level.toUpperCase()} ${event}${formatContext(context)}`;
|
|
59
|
+
console.error(line);
|
|
60
|
+
}
|
|
61
|
+
/**
|
|
62
|
+
* Create a scope-bound CLI logger.
|
|
63
|
+
*
|
|
64
|
+
* @param scope - Scope label rendered inside square brackets, e.g. "LINE".
|
|
65
|
+
* @param options - Optional rendering options such as indentation.
|
|
66
|
+
* @returns Scope-bound logger.
|
|
67
|
+
*/
|
|
68
|
+
export function createCliLogger(scope, options = {}) {
|
|
69
|
+
return {
|
|
70
|
+
info(event, context) {
|
|
71
|
+
emit('log', scope, 'info', event, context, options);
|
|
72
|
+
},
|
|
73
|
+
warn(event, context) {
|
|
74
|
+
emit('warn', scope, 'warn', event, context, options);
|
|
75
|
+
},
|
|
76
|
+
error(event, context) {
|
|
77
|
+
emit('error', scope, 'error', event, context, options);
|
|
78
|
+
},
|
|
79
|
+
debug(event, context) {
|
|
80
|
+
emit('log', scope, 'debug', event, context, options);
|
|
81
|
+
},
|
|
82
|
+
child(nextOptions = {}) {
|
|
83
|
+
return createCliLogger(scope, { ...options, ...nextOptions });
|
|
84
|
+
},
|
|
85
|
+
};
|
|
86
|
+
}
|
package/dist/version.js
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Yomi's version, as a build-time constant so it survives `bun build
|
|
3
|
+
* --compile` (a runtime read of package.json fails inside bun's compiled
|
|
4
|
+
* virtual filesystem). MUST stay equal to the "version" field in
|
|
5
|
+
* package.json — the test in version.test.ts enforces that.
|
|
6
|
+
*/
|
|
7
|
+
export const YOMI_VERSION = '0.2.6';
|
package/package.json
ADDED
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "@rikaidev/yomi",
|
|
3
|
+
"engines": {
|
|
4
|
+
"node": ">=22.13.0 <23.0.0 || >=23.4.0"
|
|
5
|
+
},
|
|
6
|
+
"version": "0.2.6",
|
|
7
|
+
"description": "Yomi (読み) — reads your LINE (decrypting Letter-Sealing E2EE) and sends on your behalf, via a stdio MCP server, so an agent can catch up on your LINE for you.",
|
|
8
|
+
"type": "module",
|
|
9
|
+
"license": "MIT",
|
|
10
|
+
"homepage": "https://github.com/RikaiDev/yomi#readme",
|
|
11
|
+
"repository": {
|
|
12
|
+
"type": "git",
|
|
13
|
+
"url": "git+https://github.com/RikaiDev/yomi.git"
|
|
14
|
+
},
|
|
15
|
+
"bugs": {
|
|
16
|
+
"url": "https://github.com/RikaiDev/yomi/issues"
|
|
17
|
+
},
|
|
18
|
+
"keywords": [
|
|
19
|
+
"line",
|
|
20
|
+
"mcp",
|
|
21
|
+
"model-context-protocol",
|
|
22
|
+
"e2ee",
|
|
23
|
+
"messaging",
|
|
24
|
+
"chat"
|
|
25
|
+
],
|
|
26
|
+
"bin": {
|
|
27
|
+
"yomi": "run.mjs"
|
|
28
|
+
},
|
|
29
|
+
"files": [
|
|
30
|
+
"run.mjs",
|
|
31
|
+
"dist",
|
|
32
|
+
"README.md",
|
|
33
|
+
"NOTICE",
|
|
34
|
+
"LICENSE",
|
|
35
|
+
"PRIVACY.md"
|
|
36
|
+
],
|
|
37
|
+
"publishConfig": {
|
|
38
|
+
"access": "public"
|
|
39
|
+
},
|
|
40
|
+
"scripts": {
|
|
41
|
+
"build": "tsc",
|
|
42
|
+
"typecheck": "tsc --noEmit",
|
|
43
|
+
"start": "node run.mjs",
|
|
44
|
+
"lint": "biome check src/",
|
|
45
|
+
"lint:fix": "biome check --write src/",
|
|
46
|
+
"format": "biome format --write src/",
|
|
47
|
+
"test": "bun test",
|
|
48
|
+
"version": "node scripts/sync-version.mjs && git add src/version.ts",
|
|
49
|
+
"prepublishOnly": "npm run build"
|
|
50
|
+
},
|
|
51
|
+
"dependencies": {
|
|
52
|
+
"@huggingface/transformers": "^4.2.0",
|
|
53
|
+
"@modelcontextprotocol/sdk": "^1.12.0",
|
|
54
|
+
"curve25519-js": "^0.0.4",
|
|
55
|
+
"tweetnacl": "^1.0.3"
|
|
56
|
+
},
|
|
57
|
+
"devDependencies": {
|
|
58
|
+
"@biomejs/biome": "^2.5.3",
|
|
59
|
+
"@types/bun": "^1.3.14",
|
|
60
|
+
"@types/node": "^22.10.0",
|
|
61
|
+
"typescript": "^7.0.0"
|
|
62
|
+
}
|
|
63
|
+
}
|