@loicngr/kobo 1.11.39 → 1.11.41
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +101 -32
- package/CHANGELOG.md +8 -0
- package/README.md +6 -4
- package/dist/mcp-server/kobo-tasks-handlers.js +16 -9
- package/dist/mcp-server/kobo-tasks-server.js +13 -2
- package/dist/mcp-server/workspace-dialogue-client.js +28 -0
- package/dist/server/db/mcp-message-schema.js +16 -0
- package/dist/server/db/migrations.js +4 -0
- package/dist/server/db/schema.js +4 -0
- package/dist/server/db/search-schema.js +53 -0
- package/dist/server/index.js +17 -86
- package/dist/server/middleware/network-auth-middleware.js +5 -1
- package/dist/server/routes/changelog.js +4 -49
- package/dist/server/routes/git.js +2 -2
- package/dist/server/routes/images.js +65 -5
- package/dist/server/routes/mcp.js +47 -0
- package/dist/server/routes/pull-requests.js +1 -1
- package/dist/server/routes/search.js +13 -4
- package/dist/server/routes/settings.js +5 -0
- package/dist/server/routes/workspaces.js +250 -248
- package/dist/server/services/agent/engines/claude-code/engine.js +31 -33
- package/dist/server/services/agent/engines/codex/engine.js +37 -8
- package/dist/server/services/agent/engines/codex/event-mapper.js +3 -1
- package/dist/server/services/agent/engines/codex/options-builder.js +4 -3
- package/dist/server/services/agent/engines/codex/server-requests.js +8 -1
- package/dist/server/services/agent/orchestrator.js +20 -3
- package/dist/server/services/attachment-service.js +122 -0
- package/dist/server/services/change-source-branch-service.js +10 -42
- package/dist/server/services/chat-receipt-service.js +33 -0
- package/dist/server/services/cron-service.js +27 -1
- package/dist/server/services/image-service.js +16 -7
- package/dist/server/services/mcp-client-context.js +12 -0
- package/dist/server/services/mcp-connection-info-service.js +63 -0
- package/dist/server/services/mcp-message-delivery-service.js +59 -0
- package/dist/server/services/mcp-message-request-service.js +64 -0
- package/dist/server/services/pr-checkout-service.js +10 -10
- package/dist/server/services/pr-watcher-service.js +14 -9
- package/dist/server/services/search/indexer.js +232 -0
- package/dist/server/services/search/worker.js +76 -0
- package/dist/server/services/search-service.js +104 -110
- package/dist/server/services/update-check-service.js +119 -0
- package/dist/server/services/websocket-service.js +121 -35
- package/dist/server/services/workspace-dialogue-service.js +128 -0
- package/dist/server/services/workspace-message-service.js +58 -0
- package/dist/server/services/workspace-permission-policy-service.js +33 -3
- package/dist/server/services/worktree-restore-service.js +2 -2
- package/dist/server/services/worktree-service.js +197 -124
- package/dist/server/utils/bounded-process.js +137 -0
- package/dist/server/utils/git-ops.js +76 -128
- package/dist/server/utils/git-repo-lock.js +27 -13
- package/dist/shared/attachments.js +58 -0
- package/dist/shared/workspace-dialogue-tools.js +89 -0
- package/dist/shared/workspace-message-types.js +19 -0
- package/package.json +2 -1
- package/src/client/dist/pwa/assets/{ActivityFeed-Chsw1Mh7.css → ActivityFeed-BYR5x7aa.css} +1 -1
- package/src/client/dist/pwa/assets/ActivityFeed-DTIe6O9o.js +8 -0
- package/src/client/dist/pwa/assets/{ChangelogPage-D8563008.js → ChangelogPage-BWeu0YeI.js} +1 -1
- package/src/client/dist/pwa/assets/{ComparisonPanel-Bd0X08sg.js → ComparisonPanel-o_EHlTwV.js} +2 -2
- package/src/client/dist/pwa/assets/CreatePage-CGi88k6z.css +1 -0
- package/src/client/dist/pwa/assets/CreatePage-DbY5gBv8.js +2 -0
- package/src/client/dist/pwa/assets/DashboardPage-CIo_gesD.js +1 -0
- package/src/client/dist/pwa/assets/{DiffViewer-DnR79ycb.css → DiffViewer-DEPOa3D7.css} +1 -1
- package/src/client/dist/pwa/assets/DiffViewer-DEcf48KI.js +8 -0
- package/src/client/dist/pwa/assets/{DocumentsPanel-BJsJ0fNs.js → DocumentsPanel-D9vcB91W.js} +1 -1
- package/src/client/dist/pwa/assets/{DrawerToggleButton-BVLO5K5d.js → DrawerToggleButton-Ca0_CxSl.js} +1 -1
- package/src/client/dist/pwa/assets/GitPanel-DCmB2R5i.js +3 -0
- package/src/client/dist/pwa/assets/{HealthPage-RFZS7Lao.js → HealthPage-_q3vy_xn.js} +1 -1
- package/src/client/dist/pwa/assets/MainLayout-Duwnq3cc.js +2 -0
- package/src/client/dist/pwa/assets/{QBtnDropdown-X8oNU5Mb.js → QBtnDropdown-BeiZIHVA.js} +1 -1
- package/src/client/dist/pwa/assets/{QDrawer-Cij17skX.js → QDrawer-Fj3M9QGD.js} +1 -1
- package/src/client/dist/pwa/assets/QSelect-DtSUuK8t.js +1 -0
- package/src/client/dist/pwa/assets/{QTooltip-CwcCGSWq.js → QTooltip-DwkX8eQ6.js} +1 -1
- package/src/client/dist/pwa/assets/SchedulePanel-BAA4Aajn.css +1 -0
- package/src/client/dist/pwa/assets/SchedulePanel-BlZDd7TK.js +1 -0
- package/src/client/dist/pwa/assets/SearchPage-BGc1oQSM.css +1 -0
- package/src/client/dist/pwa/assets/SearchPage-BGi1yRxU.js +1 -0
- package/src/client/dist/pwa/assets/SettingsPage-DnO-ImWE.js +9 -0
- package/src/client/dist/pwa/assets/SettingsPage-sVlsJQ0b.css +1 -0
- package/src/client/dist/pwa/assets/{SplitWorkspacePage-D7lGFRKs.js → SplitWorkspacePage-vz6jN1DM.js} +1 -1
- package/src/client/dist/pwa/assets/{TerminalPanel-DI8DHb9e.js → TerminalPanel-IdPIGp2_.js} +9 -9
- package/src/client/dist/pwa/assets/{TouchPan-B3nrkCY8.js → TouchPan-DTJqxVp9.js} +1 -1
- package/src/client/dist/pwa/assets/{TourReplayButton-DU1UShMY.js → TourReplayButton-Bj2GGyhU.js} +1 -1
- package/src/client/dist/pwa/assets/{WorkspaceAttentionLabels-C-yN5IX8.js → WorkspaceAttentionLabels-BBaGZlwv.js} +1 -1
- package/src/client/dist/pwa/assets/WorkspacePage-DCg9zSxz.css +1 -0
- package/src/client/dist/pwa/assets/WorkspacePage-Dy0spsfi.js +4 -0
- package/src/client/dist/pwa/assets/{browser-f9MHQjfi.js → browser-BFj5wwbS.js} +1 -1
- package/src/client/dist/pwa/assets/{build-path-tree-BWBzP8os.js → build-path-tree-C-HSiySk.js} +1 -1
- package/src/client/dist/pwa/assets/de-BdLe7apJ.js +17 -0
- package/src/client/dist/pwa/assets/{documents-C2cLnEBQ.js → documents-C2Xnjh9Z.js} +1 -1
- package/src/client/dist/pwa/assets/{editor.api-sw146MWN.js → editor.api-BbKfnNmv.js} +1 -1
- package/src/client/dist/pwa/assets/{efforts-C5ZZgCH4.js → efforts-BM7fdaaL.js} +1 -1
- package/src/client/dist/pwa/assets/engineFeatures-b74YRODT.js +1 -0
- package/src/client/dist/pwa/assets/es-C1LM-3aq.js +17 -0
- package/src/client/dist/pwa/assets/expand-template-DiDWiIkx.js +1 -0
- package/src/client/dist/pwa/assets/{format-BwWP6Aay.js → format-CnbUxAmw.js} +1 -1
- package/src/client/dist/pwa/assets/{formatters-DyWtO-nR.js → formatters-B2uUIbAY.js} +1 -1
- package/src/client/dist/pwa/assets/fr-KCK6r5ga.js +17 -0
- package/src/client/dist/pwa/assets/{freemarker2-DZ5Hm8UH.js → freemarker2-Cu1vGY0x.js} +1 -1
- package/src/client/dist/pwa/assets/{handlebars-BRUGpTA7.js → handlebars-CETuqLyz.js} +1 -1
- package/src/client/dist/pwa/assets/{html-DNOayNNs.js → html-BTNpr39P.js} +1 -1
- package/src/client/dist/pwa/assets/i18n-BkUirOzl.js +1 -0
- package/src/client/dist/pwa/assets/index-mvh5w1cM.js +20 -0
- package/src/client/dist/pwa/assets/it-BM6t0IGU.js +17 -0
- package/src/client/dist/pwa/assets/{javascript-C0tDQU3z.js → javascript-C7Ncb6V7.js} +1 -1
- package/src/client/dist/pwa/assets/{liquid-CN46O6OF.js → liquid-D0hUdg4x.js} +1 -1
- package/src/client/dist/pwa/assets/{mdx-DM-lrr8Q.js → mdx-BK5H1Xst.js} +1 -1
- package/src/client/dist/pwa/assets/{monaco-contributions-CkN7y-8c.js → monaco-contributions-BBW6JNgz.js} +1 -1
- package/src/client/dist/pwa/assets/{monaco.contribution-2BcAOLjc.js → monaco.contribution-1LZrAlY4.js} +2 -2
- package/src/client/dist/pwa/assets/network-auth-CVGS__6a.js +1 -0
- package/src/client/dist/pwa/assets/{notify-theme-a91Tcr6o.js → notify-theme-DIKLxr8F.js} +1 -1
- package/src/client/dist/pwa/assets/{pending-diff-open-DOtELZfB.js → pending-diff-open-B02zuH1j.js} +1 -1
- package/src/client/dist/pwa/assets/{permissionModes-C-9Crj-c.js → permissionModes-C3otN1Vk.js} +1 -1
- package/src/client/dist/pwa/assets/{project-color-BRe4oyNx.js → project-color-BUcW_eZA.js} +1 -1
- package/src/client/dist/pwa/assets/{python-BRpJG6sZ.js → python-DEeUVvnp.js} +1 -1
- package/src/client/dist/pwa/assets/{razor-CehCmvHl.js → razor-CG6cL8ya.js} +1 -1
- package/src/client/dist/pwa/assets/{render-chat-markdown-DlbQLZ02.js → render-chat-markdown-Ct_T5TPN.js} +1 -1
- package/src/client/dist/pwa/assets/{typescript-Dt_8mVuj.js → typescript-CSQAg5UC.js} +1 -1
- package/src/client/dist/pwa/assets/use-search-index-status-DtLzygcy.js +1 -0
- package/src/client/dist/pwa/assets/use-tours-_6HwJ_Rp.js +2 -0
- package/src/client/dist/pwa/assets/{workspace-templates-GCObcjO6.js → workspace-templates-B8WWsWEI.js} +1 -1
- package/src/client/dist/pwa/assets/{xml-_gOwdzB4.js → xml-BSB4-DXR.js} +1 -1
- package/src/client/dist/pwa/assets/{yaml-BxeBqpGf.js → yaml-CSGKph85.js} +1 -1
- package/src/client/dist/pwa/index.html +1 -1
- package/src/client/dist/pwa/sw.js +1 -1
- package/src/mcp-server/README.md +169 -9
- package/src/mcp-server/kobo-tasks-handlers.ts +14 -11
- package/src/mcp-server/kobo-tasks-server.ts +19 -2
- package/src/mcp-server/workspace-dialogue-client.ts +34 -0
- package/src/client/dist/pwa/assets/ActivityFeed-BzzvcOmq.js +0 -8
- package/src/client/dist/pwa/assets/CreatePage-A3pVradx.css +0 -1
- package/src/client/dist/pwa/assets/CreatePage-Bon8qIka.js +0 -2
- package/src/client/dist/pwa/assets/DashboardPage-DYxdxEul.js +0 -1
- package/src/client/dist/pwa/assets/DiffViewer-BpsLNxnj.js +0 -8
- package/src/client/dist/pwa/assets/GitPanel-Cal_wzC0.js +0 -3
- package/src/client/dist/pwa/assets/MainLayout-vvyiRcql.js +0 -2
- package/src/client/dist/pwa/assets/QSelect-xd-AlGo_.js +0 -1
- package/src/client/dist/pwa/assets/SchedulePanel-BunlzsOY.css +0 -1
- package/src/client/dist/pwa/assets/SchedulePanel-CXRUsz6G.js +0 -1
- package/src/client/dist/pwa/assets/SearchPage-CVgRa025.css +0 -1
- package/src/client/dist/pwa/assets/SearchPage-DopbgLxh.js +0 -1
- package/src/client/dist/pwa/assets/SettingsPage-Do9d0Pov.css +0 -1
- package/src/client/dist/pwa/assets/SettingsPage-Jx6TnOzR.js +0 -9
- package/src/client/dist/pwa/assets/WorkspacePage-BAC6u9OP.css +0 -1
- package/src/client/dist/pwa/assets/WorkspacePage-DKRpdpPg.js +0 -4
- package/src/client/dist/pwa/assets/de-QkDLrcGq.js +0 -17
- package/src/client/dist/pwa/assets/engineFeatures-CAVMDawq.js +0 -1
- package/src/client/dist/pwa/assets/es-BgQFKlo8.js +0 -17
- package/src/client/dist/pwa/assets/expand-template-B6xAlAaM.js +0 -1
- package/src/client/dist/pwa/assets/fr-DAdLWjNM.js +0 -17
- package/src/client/dist/pwa/assets/i18n-CBez_vip.js +0 -1
- package/src/client/dist/pwa/assets/index-VL3fI9a3.js +0 -20
- package/src/client/dist/pwa/assets/it-Bumaewp1.js +0 -17
- package/src/client/dist/pwa/assets/network-auth-KYS4snQH.js +0 -1
- package/src/client/dist/pwa/assets/use-tours-CSVM_54K.js +0 -2
|
@@ -0,0 +1,232 @@
|
|
|
1
|
+
export const normalizeSearchText = (text) => text.normalize('NFC').toLowerCase();
|
|
2
|
+
const graphemes = new Intl.Segmenter(undefined, { granularity: 'grapheme' });
|
|
3
|
+
function readable(row) {
|
|
4
|
+
let p;
|
|
5
|
+
try {
|
|
6
|
+
p = JSON.parse(row.payload);
|
|
7
|
+
}
|
|
8
|
+
catch {
|
|
9
|
+
return null;
|
|
10
|
+
}
|
|
11
|
+
if (!p || typeof p !== 'object')
|
|
12
|
+
return null;
|
|
13
|
+
let text;
|
|
14
|
+
let messageId = row.id;
|
|
15
|
+
if (row.type === 'user:message' && typeof p.content === 'string')
|
|
16
|
+
text = p.content;
|
|
17
|
+
if (row.type === 'agent:event' && p.kind === 'message:text' && typeof p.text === 'string') {
|
|
18
|
+
text = p.text;
|
|
19
|
+
if (typeof p.messageId === 'string')
|
|
20
|
+
messageId = p.messageId;
|
|
21
|
+
}
|
|
22
|
+
if (row.type === 'agent:output') {
|
|
23
|
+
const content = p.message?.content;
|
|
24
|
+
if (Array.isArray(content))
|
|
25
|
+
text = content.flatMap((b) => (b?.type === 'text' && typeof b.text === 'string' ? [b.text] : [])).join('\n');
|
|
26
|
+
}
|
|
27
|
+
return text === undefined
|
|
28
|
+
? null
|
|
29
|
+
: { text, key: JSON.stringify([row.workspace_id, row.session_id, row.type, messageId]) };
|
|
30
|
+
}
|
|
31
|
+
/** Map lowercased UTF-16 offsets back to display text (e.g. İ expands to i + dot). */
|
|
32
|
+
function snippet(text, match, needleLength) {
|
|
33
|
+
let normalizedOffset = 0;
|
|
34
|
+
let displayOffset = 0;
|
|
35
|
+
let startOffset = 0;
|
|
36
|
+
let endOffset = text.length;
|
|
37
|
+
for (const character of text) {
|
|
38
|
+
if (normalizedOffset <= match)
|
|
39
|
+
startOffset = displayOffset;
|
|
40
|
+
normalizedOffset += character.toLowerCase().length;
|
|
41
|
+
displayOffset += character.length;
|
|
42
|
+
if (normalizedOffset >= match + needleLength) {
|
|
43
|
+
endOffset = displayOffset;
|
|
44
|
+
break;
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
const start = Math.max(0, startOffset - 100);
|
|
48
|
+
const end = Math.min(text.length, endOffset + 100);
|
|
49
|
+
return `${start ? '…' : ''}${text.slice(start, end).trim()}${end < text.length ? '…' : ''}`;
|
|
50
|
+
}
|
|
51
|
+
/** Runs on a worker connection. Every batch is atomic and restartable. */
|
|
52
|
+
export class SearchIndexer {
|
|
53
|
+
db;
|
|
54
|
+
constructor(db) {
|
|
55
|
+
this.db = db;
|
|
56
|
+
}
|
|
57
|
+
state() {
|
|
58
|
+
return this.db.prepare('SELECT * FROM search_index_state WHERE id = 1').get();
|
|
59
|
+
}
|
|
60
|
+
status() {
|
|
61
|
+
const state = this.state();
|
|
62
|
+
const pending = this.db.prepare('SELECT 1 FROM search_changes LIMIT 1').get();
|
|
63
|
+
return {
|
|
64
|
+
state: state.complete && !pending ? 'ready' : 'building',
|
|
65
|
+
processed: state.processed,
|
|
66
|
+
total: Math.max(state.processed, state.total, 0),
|
|
67
|
+
};
|
|
68
|
+
}
|
|
69
|
+
rebuild(keys) {
|
|
70
|
+
const fragments = this.db.prepare('SELECT f.* FROM search_fragments f JOIN ws_events e ON e.id = f.event_id WHERE f.message_key = ? ORDER BY e.rowid');
|
|
71
|
+
const write = this.db.prepare(`INSERT INTO search_messages(message_key, workspace_id, session_id, event_id, type, created_at, text, normalized)
|
|
72
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT(message_key) DO UPDATE SET
|
|
73
|
+
workspace_id=excluded.workspace_id, session_id=excluded.session_id, event_id=excluded.event_id,
|
|
74
|
+
type=excluded.type, created_at=excluded.created_at, text=excluded.text, normalized=excluded.normalized`);
|
|
75
|
+
for (const key of keys) {
|
|
76
|
+
const rows = fragments.all(key);
|
|
77
|
+
const first = rows[0];
|
|
78
|
+
if (!first) {
|
|
79
|
+
this.db.prepare('DELETE FROM search_messages WHERE message_key = ?').run(key);
|
|
80
|
+
continue;
|
|
81
|
+
}
|
|
82
|
+
const text = rows
|
|
83
|
+
.map((row) => row.text)
|
|
84
|
+
.join('')
|
|
85
|
+
.normalize('NFC');
|
|
86
|
+
write.run(key, first.workspace_id, first.session_id, first.event_id, first.type, first.created_at, text, normalizeSearchText(text));
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
consume(row, eventId, order, keys) {
|
|
90
|
+
const old = this.db
|
|
91
|
+
.prepare('SELECT message_key, event_order FROM search_fragments WHERE event_id = ?')
|
|
92
|
+
.get(eventId);
|
|
93
|
+
if (old)
|
|
94
|
+
keys.add(old.message_key);
|
|
95
|
+
const value = row ? readable(row) : null;
|
|
96
|
+
if (!value || !row) {
|
|
97
|
+
this.db.prepare('DELETE FROM search_fragments WHERE event_id = ?').run(eventId);
|
|
98
|
+
return;
|
|
99
|
+
}
|
|
100
|
+
keys.add(value.key);
|
|
101
|
+
this.db
|
|
102
|
+
.prepare(`INSERT INTO search_fragments(event_id, message_key, workspace_id, session_id, type, created_at, event_order, text)
|
|
103
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT(event_id) DO UPDATE SET
|
|
104
|
+
message_key=excluded.message_key, workspace_id=excluded.workspace_id, session_id=excluded.session_id,
|
|
105
|
+
type=excluded.type, created_at=excluded.created_at, text=excluded.text`)
|
|
106
|
+
.run(row.id, value.key, row.workspace_id, row.session_id, row.type, row.created_at, old?.event_order ?? order, value.text);
|
|
107
|
+
}
|
|
108
|
+
tick(batchSize = 500) {
|
|
109
|
+
// Acquire the writer reservation before reading: a deferred WAL snapshot
|
|
110
|
+
// cannot be upgraded after another connection commits (SQLITE_BUSY_SNAPSHOT).
|
|
111
|
+
this.db
|
|
112
|
+
.transaction(() => {
|
|
113
|
+
const state = this.state();
|
|
114
|
+
if (state.total < 0)
|
|
115
|
+
this.db.prepare('UPDATE search_index_state SET total = (SELECT count(*) FROM ws_events) WHERE id = 1').run();
|
|
116
|
+
const keys = new Set();
|
|
117
|
+
const changeBudget = state.complete ? batchSize : Math.max(1, Math.floor(batchSize / 2));
|
|
118
|
+
const changes = this.db.prepare('SELECT * FROM search_changes ORDER BY id LIMIT ?').all(changeBudget);
|
|
119
|
+
for (const change of changes) {
|
|
120
|
+
const row = this.db.prepare('SELECT rowid AS rid, * FROM ws_events WHERE id = ?').get(change.event_id);
|
|
121
|
+
this.consume(row, change.event_id, change.operation === 'insert' ? state.base_order + change.id : (row?.rid ?? 0), keys);
|
|
122
|
+
}
|
|
123
|
+
if (changes.length)
|
|
124
|
+
this.db.prepare('DELETE FROM search_changes WHERE id <= ?').run(changes.at(-1).id);
|
|
125
|
+
const backfillBudget = batchSize - changes.length;
|
|
126
|
+
if (!state.complete && backfillBudget > 0) {
|
|
127
|
+
const rows = this.db
|
|
128
|
+
.prepare('SELECT rowid AS rid, * FROM ws_events WHERE id > ? ORDER BY id LIMIT ?')
|
|
129
|
+
.all(state.cursor, backfillBudget);
|
|
130
|
+
for (const row of rows)
|
|
131
|
+
this.consume(row, row.id, row.rid, keys);
|
|
132
|
+
this.db
|
|
133
|
+
.prepare('UPDATE search_index_state SET cursor = ?, processed = processed + ?, complete = ? WHERE id = 1')
|
|
134
|
+
.run(rows.at(-1)?.id ?? state.cursor, rows.length, rows.length < backfillBudget ? 1 : 0);
|
|
135
|
+
}
|
|
136
|
+
this.rebuild(keys);
|
|
137
|
+
})
|
|
138
|
+
.immediate();
|
|
139
|
+
return this.status();
|
|
140
|
+
}
|
|
141
|
+
/** Drain deletions ahead of queries, including those behind an insertion backlog. */
|
|
142
|
+
removeDeleted() {
|
|
143
|
+
this.db
|
|
144
|
+
.transaction(() => {
|
|
145
|
+
const rows = this.db
|
|
146
|
+
.prepare(`SELECT DISTINCT f.event_id, f.message_key FROM search_changes c
|
|
147
|
+
JOIN search_fragments f ON f.event_id = c.event_id
|
|
148
|
+
WHERE NOT EXISTS (SELECT 1 FROM ws_events e WHERE e.id = c.event_id)`)
|
|
149
|
+
.all();
|
|
150
|
+
const keys = new Set();
|
|
151
|
+
for (const row of rows) {
|
|
152
|
+
keys.add(row.message_key);
|
|
153
|
+
this.db.prepare('DELETE FROM search_fragments WHERE event_id = ?').run(row.event_id);
|
|
154
|
+
}
|
|
155
|
+
this.rebuild(keys);
|
|
156
|
+
})
|
|
157
|
+
.immediate();
|
|
158
|
+
}
|
|
159
|
+
/** Resolve a normalized match to the original event used by history deep links. */
|
|
160
|
+
matchEventId(messageKey, match, fallback) {
|
|
161
|
+
const fragments = this.db
|
|
162
|
+
.prepare(`SELECT f.event_id, f.text FROM search_fragments f JOIN ws_events e ON e.id = f.event_id
|
|
163
|
+
WHERE f.message_key = ? ORDER BY e.rowid`)
|
|
164
|
+
.all(messageKey);
|
|
165
|
+
if (fragments.length < 2)
|
|
166
|
+
return fragments[0]?.event_id ?? fallback;
|
|
167
|
+
// Normalize whole graphemes: NFC can compose characters across fragment
|
|
168
|
+
// boundaries, and lowercasing can expand them (İ → i + combining dot).
|
|
169
|
+
const source = fragments.map((fragment) => fragment.text).join('');
|
|
170
|
+
let normalizedEnd = 0;
|
|
171
|
+
let sourceOffset = 0;
|
|
172
|
+
for (const { segment, index } of graphemes.segment(source)) {
|
|
173
|
+
normalizedEnd += normalizeSearchText(segment).length;
|
|
174
|
+
if (normalizedEnd > match) {
|
|
175
|
+
sourceOffset = index;
|
|
176
|
+
break;
|
|
177
|
+
}
|
|
178
|
+
}
|
|
179
|
+
let fragmentEnd = 0;
|
|
180
|
+
for (const fragment of fragments) {
|
|
181
|
+
fragmentEnd += fragment.text.length;
|
|
182
|
+
if (fragmentEnd > sourceOffset)
|
|
183
|
+
return fragment.event_id;
|
|
184
|
+
}
|
|
185
|
+
return fallback;
|
|
186
|
+
}
|
|
187
|
+
async search(query, options = {}, cancelled = () => false) {
|
|
188
|
+
this.removeDeleted();
|
|
189
|
+
const needle = normalizeSearchText(query.trim());
|
|
190
|
+
if (!needle)
|
|
191
|
+
return [];
|
|
192
|
+
const trigram = [...needle].length >= 3 && !needle.includes('\0');
|
|
193
|
+
const clauses = [options.includeArchived ? '1' : 'w.archived_at IS NULL'];
|
|
194
|
+
const args = [];
|
|
195
|
+
if (trigram) {
|
|
196
|
+
clauses.push('search_messages_fts MATCH ?');
|
|
197
|
+
args.push(`"${needle.replaceAll('"', '""')}"`);
|
|
198
|
+
}
|
|
199
|
+
if (options.workspaceId) {
|
|
200
|
+
clauses.push('m.workspace_id = ?');
|
|
201
|
+
args.push(options.workspaceId);
|
|
202
|
+
}
|
|
203
|
+
const statement = this.db.prepare(`SELECT m.*, w.name AS workspace_name, w.archived_at
|
|
204
|
+
FROM search_messages m ${trigram ? 'JOIN search_messages_fts ON search_messages_fts.rowid = m.id' : ''}
|
|
205
|
+
JOIN workspaces w ON w.id = m.workspace_id WHERE ${clauses.join(' AND ')} ORDER BY m.created_at DESC, m.event_id DESC`);
|
|
206
|
+
const results = [];
|
|
207
|
+
let scanned = 0;
|
|
208
|
+
const limit = Math.min(200, Math.max(1, options.limit ?? 50));
|
|
209
|
+
for (const row of statement.iterate(...args)) {
|
|
210
|
+
if (cancelled())
|
|
211
|
+
throw new Error('Search cancelled');
|
|
212
|
+
const index = row.normalized.indexOf(needle);
|
|
213
|
+
if (index >= 0) {
|
|
214
|
+
results.push({
|
|
215
|
+
eventId: row.type === 'agent:event' ? this.matchEventId(row.message_key, index, row.event_id) : row.event_id,
|
|
216
|
+
sessionId: row.session_id,
|
|
217
|
+
workspaceId: row.workspace_id,
|
|
218
|
+
workspaceName: row.workspace_name,
|
|
219
|
+
archived: row.archived_at !== null,
|
|
220
|
+
type: row.type,
|
|
221
|
+
timestamp: row.created_at,
|
|
222
|
+
snippet: snippet(row.text, index, needle.length),
|
|
223
|
+
});
|
|
224
|
+
if (results.length >= limit)
|
|
225
|
+
break;
|
|
226
|
+
}
|
|
227
|
+
if (++scanned % 500 === 0)
|
|
228
|
+
await new Promise((resolve) => setImmediate(resolve));
|
|
229
|
+
}
|
|
230
|
+
return results;
|
|
231
|
+
}
|
|
232
|
+
}
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
import { parentPort, workerData } from 'node:worker_threads';
|
|
2
|
+
import Database from 'better-sqlite3';
|
|
3
|
+
import { SearchIndexer } from './indexer.js';
|
|
4
|
+
const port = parentPort;
|
|
5
|
+
const db = new Database(workerData.dbPath);
|
|
6
|
+
db.pragma('journal_mode=WAL');
|
|
7
|
+
db.pragma('foreign_keys=ON');
|
|
8
|
+
db.pragma('busy_timeout=5000');
|
|
9
|
+
const indexer = new SearchIndexer(db);
|
|
10
|
+
const cancelled = new Set();
|
|
11
|
+
const requests = new Set();
|
|
12
|
+
let queue = Promise.resolve();
|
|
13
|
+
let stopped = false;
|
|
14
|
+
let timer;
|
|
15
|
+
function report() {
|
|
16
|
+
port.postMessage({ status: indexer.status() });
|
|
17
|
+
}
|
|
18
|
+
function pump() {
|
|
19
|
+
if (stopped)
|
|
20
|
+
return;
|
|
21
|
+
queue = queue
|
|
22
|
+
.then(() => {
|
|
23
|
+
if (stopped)
|
|
24
|
+
return;
|
|
25
|
+
indexer.tick();
|
|
26
|
+
report();
|
|
27
|
+
})
|
|
28
|
+
.catch((err) => {
|
|
29
|
+
// A writer may outlast busy_timeout. The rolled-back batch remains queued
|
|
30
|
+
// in SQLite; retry it on the next pump instead of poisoning this worker.
|
|
31
|
+
if (err?.code === 'SQLITE_BUSY' || err?.code?.startsWith('SQLITE_BUSY_'))
|
|
32
|
+
return;
|
|
33
|
+
stopped = true;
|
|
34
|
+
port.postMessage({ status: { state: 'error', processed: 0, total: 0, error: String(err) } });
|
|
35
|
+
})
|
|
36
|
+
.finally(() => {
|
|
37
|
+
if (!stopped)
|
|
38
|
+
timer = setTimeout(pump, indexer.status().state === 'building' ? 10 : 200);
|
|
39
|
+
});
|
|
40
|
+
}
|
|
41
|
+
port.on('message', (message) => {
|
|
42
|
+
if (message.cancel) {
|
|
43
|
+
if (requests.has(message.id))
|
|
44
|
+
cancelled.add(message.id);
|
|
45
|
+
return;
|
|
46
|
+
}
|
|
47
|
+
if (message.stop) {
|
|
48
|
+
stopped = true;
|
|
49
|
+
clearTimeout(timer);
|
|
50
|
+
void queue.finally(() => {
|
|
51
|
+
db.close();
|
|
52
|
+
port.close();
|
|
53
|
+
});
|
|
54
|
+
return;
|
|
55
|
+
}
|
|
56
|
+
requests.add(message.id);
|
|
57
|
+
queue = queue
|
|
58
|
+
.then(async () => {
|
|
59
|
+
if (stopped)
|
|
60
|
+
throw new Error('Search index unavailable');
|
|
61
|
+
if (cancelled.has(message.id))
|
|
62
|
+
return;
|
|
63
|
+
indexer.tick();
|
|
64
|
+
const results = await indexer.search(message.query ?? '', message.options, () => cancelled.has(message.id));
|
|
65
|
+
report();
|
|
66
|
+
port.postMessage({ id: message.id, results });
|
|
67
|
+
})
|
|
68
|
+
.catch((err) => {
|
|
69
|
+
port.postMessage({ id: message.id, error: err instanceof Error ? err.message : String(err) });
|
|
70
|
+
})
|
|
71
|
+
.finally(() => {
|
|
72
|
+
cancelled.delete(message.id);
|
|
73
|
+
requests.delete(message.id);
|
|
74
|
+
});
|
|
75
|
+
});
|
|
76
|
+
pump();
|
|
@@ -1,117 +1,111 @@
|
|
|
1
|
+
import { Worker } from 'node:worker_threads';
|
|
1
2
|
import { getDb } from '../db/index.js';
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
const
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
3
|
+
let active;
|
|
4
|
+
let requestId = 0;
|
|
5
|
+
/** Start lazily for service consumers, eagerly at server boot. Never opens another home directory. */
|
|
6
|
+
export function startSearchIndex() {
|
|
7
|
+
const dbPath = getDb().name;
|
|
8
|
+
if (active?.dbPath === dbPath)
|
|
9
|
+
return;
|
|
10
|
+
if (active)
|
|
11
|
+
void stopSearchIndex();
|
|
12
|
+
const development = import.meta.url.endsWith('.ts');
|
|
13
|
+
const entry = new URL(`./search/worker.${development ? 'ts' : 'js'}`, import.meta.url);
|
|
14
|
+
const worker = development
|
|
15
|
+
? new Worker(`const { workerData } = require('node:worker_threads'); import('tsx/esm/api').then(({ tsImport }) => tsImport(workerData.entry, workerData.parent));`, {
|
|
16
|
+
eval: true,
|
|
17
|
+
workerData: { dbPath, entry: entry.href, parent: import.meta.url },
|
|
18
|
+
})
|
|
19
|
+
: new Worker(entry, { workerData: { dbPath } });
|
|
20
|
+
const state = {
|
|
21
|
+
worker,
|
|
22
|
+
dbPath,
|
|
23
|
+
pending: new Map(),
|
|
24
|
+
status: { state: 'building', processed: 0, total: 0 },
|
|
25
|
+
};
|
|
26
|
+
active = state;
|
|
27
|
+
worker.unref();
|
|
28
|
+
const fail = (error) => {
|
|
29
|
+
state.status = { ...state.status, state: 'error', error: error.message };
|
|
30
|
+
for (const pending of state.pending.values()) {
|
|
31
|
+
pending.cleanup();
|
|
32
|
+
pending.reject(error);
|
|
30
33
|
}
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
34
|
+
state.pending.clear();
|
|
35
|
+
};
|
|
36
|
+
worker.on('error', fail);
|
|
37
|
+
worker.on('exit', (code) => {
|
|
38
|
+
if (active === state)
|
|
39
|
+
fail(new Error(`Search worker stopped (${code})`));
|
|
40
|
+
});
|
|
41
|
+
worker.on('message', (message) => {
|
|
42
|
+
if (message.status)
|
|
43
|
+
state.status = message.status;
|
|
44
|
+
if (message.id !== undefined) {
|
|
45
|
+
const pending = state.pending.get(message.id);
|
|
46
|
+
if (!pending)
|
|
47
|
+
return;
|
|
48
|
+
state.pending.delete(message.id);
|
|
49
|
+
pending.cleanup();
|
|
50
|
+
if (message.error)
|
|
51
|
+
pending.reject(new Error(message.error));
|
|
52
|
+
else
|
|
53
|
+
pending.resolve(message.results ?? []);
|
|
54
|
+
}
|
|
55
|
+
});
|
|
36
56
|
}
|
|
37
|
-
function
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
const prefix = start > 0 ? '…' : '';
|
|
41
|
-
const suffix = end < text.length ? '…' : '';
|
|
42
|
-
return `${prefix}${text.slice(start, end).trim()}${suffix}`;
|
|
57
|
+
export function getSearchIndexStatus() {
|
|
58
|
+
startSearchIndex();
|
|
59
|
+
return { ...active.status };
|
|
43
60
|
}
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
*
|
|
47
|
-
* - Trimmed empty queries return `[]` without hitting the database.
|
|
48
|
-
* - SQLite does the first filter via `LIKE '%q%'` on the raw payload JSON.
|
|
49
|
-
* Results are then post-filtered in JS against the **readable** text
|
|
50
|
-
* (extracted from the JSON) to avoid false positives like matches inside
|
|
51
|
-
* field names or schema strings.
|
|
52
|
-
* - Snippets are 100 chars of context on either side of the first match.
|
|
53
|
-
*/
|
|
54
|
-
export function searchEvents(query, options = {}) {
|
|
55
|
-
const trimmed = query.trim();
|
|
56
|
-
if (trimmed.length === 0)
|
|
61
|
+
export async function searchEvents(query, options = {}, signal) {
|
|
62
|
+
if (!query.trim())
|
|
57
63
|
return [];
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
const
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
const
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
continue;
|
|
92
|
-
const idx = text.toLowerCase().indexOf(needle);
|
|
93
|
-
if (idx < 0)
|
|
94
|
-
continue;
|
|
95
|
-
results.push({
|
|
96
|
-
eventId: row.id,
|
|
97
|
-
sessionId: row.session_id,
|
|
98
|
-
workspaceId: row.workspace_id,
|
|
99
|
-
workspaceName: row.workspace_name,
|
|
100
|
-
archived: row.archived_at !== null,
|
|
101
|
-
type: row.type,
|
|
102
|
-
timestamp: row.created_at,
|
|
103
|
-
snippet: buildSnippet(text, idx, trimmed),
|
|
104
|
-
});
|
|
105
|
-
if (results.length >= limit)
|
|
106
|
-
break;
|
|
107
|
-
}
|
|
108
|
-
if (rows.length < batchSize)
|
|
109
|
-
break;
|
|
110
|
-
const last = rows.at(-1);
|
|
111
|
-
if (!last)
|
|
112
|
-
break;
|
|
113
|
-
cursorCreatedAt = last.created_at;
|
|
114
|
-
cursorId = last.id;
|
|
64
|
+
if (signal?.aborted)
|
|
65
|
+
throw new Error('Search cancelled');
|
|
66
|
+
startSearchIndex();
|
|
67
|
+
const state = active;
|
|
68
|
+
if (state.status.state === 'error')
|
|
69
|
+
throw new Error(state.status.error ?? 'Search index unavailable');
|
|
70
|
+
const id = ++requestId;
|
|
71
|
+
return new Promise((resolve, reject) => {
|
|
72
|
+
const cancel = (message) => {
|
|
73
|
+
state.pending.delete(id);
|
|
74
|
+
cleanup();
|
|
75
|
+
state.worker.postMessage({ id, cancel: true });
|
|
76
|
+
reject(new Error(message));
|
|
77
|
+
};
|
|
78
|
+
const aborted = () => cancel('Search cancelled');
|
|
79
|
+
const timeout = setTimeout(() => cancel('Search timed out'), 10_000);
|
|
80
|
+
const cleanup = () => {
|
|
81
|
+
clearTimeout(timeout);
|
|
82
|
+
signal?.removeEventListener('abort', aborted);
|
|
83
|
+
};
|
|
84
|
+
state.pending.set(id, { resolve, reject, cleanup });
|
|
85
|
+
signal?.addEventListener('abort', aborted, { once: true });
|
|
86
|
+
state.worker.postMessage({ id, query, options });
|
|
87
|
+
});
|
|
88
|
+
}
|
|
89
|
+
export async function stopSearchIndex() {
|
|
90
|
+
const state = active;
|
|
91
|
+
if (!state)
|
|
92
|
+
return;
|
|
93
|
+
active = undefined;
|
|
94
|
+
for (const pending of state.pending.values()) {
|
|
95
|
+
pending.cleanup();
|
|
96
|
+
pending.reject(new Error('Search index stopped'));
|
|
115
97
|
}
|
|
116
|
-
|
|
98
|
+
state.pending.clear();
|
|
99
|
+
if (state.worker.threadId === -1)
|
|
100
|
+
return;
|
|
101
|
+
await new Promise((resolve) => {
|
|
102
|
+
const timeout = setTimeout(() => {
|
|
103
|
+
void state.worker.terminate();
|
|
104
|
+
}, 5_000);
|
|
105
|
+
state.worker.once('exit', () => {
|
|
106
|
+
clearTimeout(timeout);
|
|
107
|
+
resolve();
|
|
108
|
+
});
|
|
109
|
+
state.worker.postMessage({ stop: true });
|
|
110
|
+
});
|
|
117
111
|
}
|
|
@@ -0,0 +1,119 @@
|
|
|
1
|
+
import { getPackageVersion } from '../utils/paths.js';
|
|
2
|
+
import { broadcastAll } from './websocket-service.js';
|
|
3
|
+
export const UPDATE_CHECK_INTERVAL_MS = 20 * 60 * 1000;
|
|
4
|
+
const REGISTRY_URL = 'https://registry.npmjs.org/@loicngr/kobo/latest';
|
|
5
|
+
const STABLE_VERSION = /^(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)$/;
|
|
6
|
+
let latestVersion = null;
|
|
7
|
+
let lastAttempt = null;
|
|
8
|
+
let lastSuccessAt = null;
|
|
9
|
+
let checkStatus = 'unknown';
|
|
10
|
+
let inFlight = null;
|
|
11
|
+
let controller = null;
|
|
12
|
+
let timer = null;
|
|
13
|
+
let running = false;
|
|
14
|
+
let generation = 0;
|
|
15
|
+
let stopped = false;
|
|
16
|
+
export function getUpdateCheckSnapshot() {
|
|
17
|
+
return {
|
|
18
|
+
currentVersion: getPackageVersion(),
|
|
19
|
+
latestVersion,
|
|
20
|
+
lastAttemptAt: lastAttempt === null ? null : new Date(lastAttempt).toISOString(),
|
|
21
|
+
lastSuccessAt,
|
|
22
|
+
checkStatus,
|
|
23
|
+
};
|
|
24
|
+
}
|
|
25
|
+
/** Shared by HTTP loads and the single process-wide poller; failures are cached too. */
|
|
26
|
+
export function refreshUpdateCheck() {
|
|
27
|
+
if (inFlight)
|
|
28
|
+
return inFlight;
|
|
29
|
+
if (stopped || (lastAttempt !== null && Date.now() - lastAttempt < UPDATE_CHECK_INTERVAL_MS)) {
|
|
30
|
+
return Promise.resolve(getUpdateCheckSnapshot());
|
|
31
|
+
}
|
|
32
|
+
lastAttempt = Date.now();
|
|
33
|
+
const owner = generation;
|
|
34
|
+
const abort = new AbortController();
|
|
35
|
+
controller = abort;
|
|
36
|
+
const timeout = setTimeout(() => abort.abort(), 5000);
|
|
37
|
+
timeout.unref?.();
|
|
38
|
+
const aborted = new Promise((_, reject) => {
|
|
39
|
+
abort.signal.addEventListener('abort', () => reject(new Error('Update lookup cancelled')), { once: true });
|
|
40
|
+
});
|
|
41
|
+
const lookup = (async () => {
|
|
42
|
+
const res = await fetch(REGISTRY_URL, { signal: abort.signal });
|
|
43
|
+
if (!res.ok)
|
|
44
|
+
throw new Error('Registry lookup failed');
|
|
45
|
+
const body = await res.json();
|
|
46
|
+
if (!body ||
|
|
47
|
+
typeof body !== 'object' ||
|
|
48
|
+
!('version' in body) ||
|
|
49
|
+
typeof body.version !== 'string' ||
|
|
50
|
+
!STABLE_VERSION.test(body.version)) {
|
|
51
|
+
throw new Error('Invalid registry version');
|
|
52
|
+
}
|
|
53
|
+
return body.version;
|
|
54
|
+
})();
|
|
55
|
+
inFlight = (async () => {
|
|
56
|
+
try {
|
|
57
|
+
const version = await Promise.race([lookup, aborted]);
|
|
58
|
+
if (owner !== generation)
|
|
59
|
+
return getUpdateCheckSnapshot();
|
|
60
|
+
latestVersion = version;
|
|
61
|
+
lastSuccessAt = new Date().toISOString();
|
|
62
|
+
checkStatus = 'success';
|
|
63
|
+
}
|
|
64
|
+
catch {
|
|
65
|
+
if (owner !== generation)
|
|
66
|
+
return getUpdateCheckSnapshot();
|
|
67
|
+
checkStatus = 'failed';
|
|
68
|
+
}
|
|
69
|
+
finally {
|
|
70
|
+
clearTimeout(timeout);
|
|
71
|
+
if (owner === generation) {
|
|
72
|
+
inFlight = null;
|
|
73
|
+
controller = null;
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
const snapshot = getUpdateCheckSnapshot();
|
|
77
|
+
broadcastAll('kobo:update-checked', snapshot);
|
|
78
|
+
return snapshot;
|
|
79
|
+
})();
|
|
80
|
+
return inFlight;
|
|
81
|
+
}
|
|
82
|
+
function scheduleNextCheck(owner) {
|
|
83
|
+
if (!running || owner !== generation)
|
|
84
|
+
return;
|
|
85
|
+
const delay = lastAttempt === null ? 0 : Math.max(0, UPDATE_CHECK_INTERVAL_MS - (Date.now() - lastAttempt));
|
|
86
|
+
timer = setTimeout(() => {
|
|
87
|
+
timer = null;
|
|
88
|
+
void refreshUpdateCheck().finally(() => scheduleNextCheck(owner));
|
|
89
|
+
}, delay);
|
|
90
|
+
timer.unref?.();
|
|
91
|
+
}
|
|
92
|
+
export function startUpdateChecker() {
|
|
93
|
+
if (running)
|
|
94
|
+
return;
|
|
95
|
+
running = true;
|
|
96
|
+
stopped = false;
|
|
97
|
+
const owner = generation;
|
|
98
|
+
void refreshUpdateCheck().finally(() => scheduleNextCheck(owner));
|
|
99
|
+
}
|
|
100
|
+
export function stopUpdateChecker() {
|
|
101
|
+
running = false;
|
|
102
|
+
if (timer)
|
|
103
|
+
clearTimeout(timer);
|
|
104
|
+
timer = null;
|
|
105
|
+
stopped = true;
|
|
106
|
+
generation++;
|
|
107
|
+
controller?.abort();
|
|
108
|
+
controller = null;
|
|
109
|
+
inFlight = null;
|
|
110
|
+
}
|
|
111
|
+
/** @internal test-only — clear the scheduler and registry cache. */
|
|
112
|
+
export function _clearLatestVersionCache() {
|
|
113
|
+
stopUpdateChecker();
|
|
114
|
+
stopped = false;
|
|
115
|
+
latestVersion = null;
|
|
116
|
+
lastAttempt = null;
|
|
117
|
+
lastSuccessAt = null;
|
|
118
|
+
checkStatus = 'unknown';
|
|
119
|
+
}
|