@coffer-org/server 2.6.0 → 2.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/auth-api.js +17 -3
- package/dist/auth-store.js +7 -1
- package/dist/background-scheduler.js +20 -6
- package/dist/collection-io.js +5 -2
- package/dist/embed-openai.js +2 -1
- package/dist/embeddings.d.ts +4 -1
- package/dist/embeddings.js +17 -8
- package/dist/entity-schema.d.ts +1 -0
- package/dist/entity-schema.js +28 -4
- package/dist/extend-io.js +6 -1
- package/dist/extend-table.js +9 -4
- package/dist/file-fields.d.ts +2 -0
- package/dist/file-fields.js +25 -6
- package/dist/index.js +16 -2
- package/dist/mcp-http.test-helpers.js +4 -1
- package/dist/mcp-local.js +3 -1
- package/dist/mcp-tools.d.ts +15 -2
- package/dist/mcp-tools.js +83 -11
- package/dist/msg-log.d.ts +2 -0
- package/dist/msg-log.js +21 -3
- package/dist/mutate.d.ts +3 -1
- package/dist/mutate.js +6 -4
- package/dist/oauth-api.js +25 -10
- package/dist/plugin-i18n.d.ts +6 -0
- package/dist/plugin-i18n.js +38 -0
- package/dist/rag-search.d.ts +40 -0
- package/dist/rag-search.js +94 -0
- package/dist/records-api.d.ts +4 -0
- package/dist/records-api.js +53 -4
- package/dist/search-index.js +2 -8
- package/dist/temporal.d.ts +5 -2
- package/dist/temporal.js +8 -6
- package/dist/thread-state.d.ts +8 -0
- package/dist/thread-state.js +39 -0
- package/dist/thread-store.js +3 -1
- package/dist/uploads.js +1 -1
- package/package.json +2 -2
package/dist/mcp-tools.js
CHANGED
|
@@ -10,7 +10,8 @@ import { countTargetsFor, recordCounts } from "./counts.js";
|
|
|
10
10
|
import { describeCondition } from '@coffer-org/sdk/condition';
|
|
11
11
|
import { getEm } from "./db.js";
|
|
12
12
|
import { getPluginSettings } from "./plugin-runtime.js";
|
|
13
|
-
import {
|
|
13
|
+
import { configuredPublicUrl } from "./public-url.js";
|
|
14
|
+
import { hybridSearch, buildSearchShelves } from "./rag-search.js";
|
|
14
15
|
import { embedOne, embeddingFailureMessage } from "./embed-openai.js";
|
|
15
16
|
import { getLogger } from "./log.js";
|
|
16
17
|
import { writePluginSettings, listSettings } from "./settings-write.js";
|
|
@@ -21,7 +22,14 @@ export function formatHits(hits) {
|
|
|
21
22
|
if (hits.length === 0)
|
|
22
23
|
return 'No matching records.';
|
|
23
24
|
return hits
|
|
24
|
-
.map((h) =>
|
|
25
|
+
.map((h) => {
|
|
26
|
+
const legs = [];
|
|
27
|
+
if (h.distance != null)
|
|
28
|
+
legs.push(`vector ${h.distance.toFixed(3)} #${h.vectorRank}`);
|
|
29
|
+
if (h.textRank != null)
|
|
30
|
+
legs.push(`text #${h.textRank}`);
|
|
31
|
+
return `[${h.shelfKey}/${h.recordId}] (${legs.join(' | ')})\n${h.snippet}`;
|
|
32
|
+
})
|
|
25
33
|
.join('\n\n');
|
|
26
34
|
}
|
|
27
35
|
const ok = (data) => ({
|
|
@@ -31,7 +39,7 @@ const fail = (text) => ({ content: [{ type: 'text', text }], isError: true });
|
|
|
31
39
|
export async function resolveRagDeps() {
|
|
32
40
|
const db = (await getPluginSettings('claude-agent'));
|
|
33
41
|
const enabled = db['rag_enabled'] !== false;
|
|
34
|
-
const embeddingApiKey =
|
|
42
|
+
const embeddingApiKey = process.env.OPENAI_API_KEY ?? db['openai_api_key'] ?? '';
|
|
35
43
|
if (!enabled || !embeddingApiKey)
|
|
36
44
|
return null;
|
|
37
45
|
return { embeddingApiKey };
|
|
@@ -64,11 +72,16 @@ export async function collectMcpTools(opts = {}) {
|
|
|
64
72
|
role: 'member',
|
|
65
73
|
handler: async () => {
|
|
66
74
|
const { token, expiresInSec } = mintUploadTicket(actor);
|
|
75
|
+
const base = await configuredPublicUrl();
|
|
76
|
+
const uploadUrl = base ? `${base}/api/upload` : '/api/upload';
|
|
67
77
|
return ok({
|
|
68
78
|
token,
|
|
69
|
-
upload_url:
|
|
79
|
+
upload_url: uploadUrl,
|
|
70
80
|
expires_in: expiresInSec,
|
|
71
|
-
how_to:
|
|
81
|
+
how_to: `curl -H "Authorization: Bearer <token>" -F "file=@<path>" ${uploadUrl} → {"name":"<name>"}. Then set a file field to {"name":"<name>"}. mime/size are filled in by the server — do not send your own.` +
|
|
82
|
+
(base
|
|
83
|
+
? ''
|
|
84
|
+
: " The server has no public URL configured (core settings publicUrl / PUBLIC_URL env), so the path is relative — resolve it against this MCP server's own origin."),
|
|
72
85
|
});
|
|
73
86
|
},
|
|
74
87
|
});
|
|
@@ -132,18 +145,55 @@ export async function collectMcpTools(opts = {}) {
|
|
|
132
145
|
server: 'rag',
|
|
133
146
|
bareName: 'search_records',
|
|
134
147
|
httpName: 'search_records',
|
|
135
|
-
description: "
|
|
136
|
-
|
|
148
|
+
description: "Hybrid search over the user's coffer records: semantic similarity plus exact-token matching. " +
|
|
149
|
+
'Returns the most relevant records as library/shelf/id refs with a text snippet. Narrow it with `library` ' +
|
|
150
|
+
'(and optionally `shelf`) when you already know where the answer lives. An empty result means nothing ' +
|
|
151
|
+
'relevant is stored — say so instead of answering from a weak match. If the first call returns nothing ' +
|
|
152
|
+
'useful, retry with a rephrasing, or with the exact literal string (a model number, a name, an error code): ' +
|
|
153
|
+
'the exact-token leg finds those where a paraphrase cannot.',
|
|
154
|
+
inputSchema: {
|
|
155
|
+
query: z.string(),
|
|
156
|
+
k: z.number().int().positive().optional(),
|
|
157
|
+
library: z.string().optional(),
|
|
158
|
+
shelf: z.string().optional(),
|
|
159
|
+
},
|
|
137
160
|
scope: 'rag',
|
|
138
161
|
role: 'member',
|
|
139
162
|
handler: async (args) => {
|
|
163
|
+
const library = args.library;
|
|
164
|
+
const shelf = args.shelf;
|
|
165
|
+
if (shelf && !library) {
|
|
166
|
+
return fail('Error: `shelf` needs `library` (a shelf name is only unique inside its library).');
|
|
167
|
+
}
|
|
168
|
+
if (library) {
|
|
169
|
+
const known = buildSearchShelves({ library, shelf });
|
|
170
|
+
if (known.length === 0) {
|
|
171
|
+
const all = buildSearchShelves();
|
|
172
|
+
const valid = shelf
|
|
173
|
+
? all.filter((s) => s.key.startsWith(`${library}/`)).map((s) => s.key)
|
|
174
|
+
: [...new Set(all.map((s) => s.key.split('/')[0]))];
|
|
175
|
+
return fail(`Error: unknown ${shelf ? 'shelf' : 'library'}. Valid: ${valid.join(', ')}`);
|
|
176
|
+
}
|
|
177
|
+
}
|
|
178
|
+
let vector = null;
|
|
179
|
+
try {
|
|
180
|
+
vector = (await embedOne(args.query, embeddingApiKey)).vector;
|
|
181
|
+
}
|
|
182
|
+
catch (e) {
|
|
183
|
+
log.warn(`embedding failed, text-only search — ${embeddingFailureMessage(e)}`);
|
|
184
|
+
}
|
|
140
185
|
try {
|
|
141
|
-
const
|
|
142
|
-
|
|
186
|
+
const hits = await hybridSearch({
|
|
187
|
+
query: args.query,
|
|
188
|
+
vector,
|
|
189
|
+
k: args.k ?? DEFAULT_RAG_TOP_K,
|
|
190
|
+
library,
|
|
191
|
+
shelf,
|
|
192
|
+
});
|
|
143
193
|
return { content: [{ type: 'text', text: formatHits(hits) }] };
|
|
144
194
|
}
|
|
145
195
|
catch (e) {
|
|
146
|
-
return fail(`
|
|
196
|
+
return fail(`Search unavailable: ${e.message} Use the regular coffer tools (list_records/get_record) instead.`);
|
|
147
197
|
}
|
|
148
198
|
},
|
|
149
199
|
});
|
|
@@ -214,6 +264,22 @@ export async function collectPluginInstructions(hooks = pluginHooks, emFactory =
|
|
|
214
264
|
}
|
|
215
265
|
return out;
|
|
216
266
|
}
|
|
267
|
+
export function collectSingleShelves(reg) {
|
|
268
|
+
let registry = reg;
|
|
269
|
+
if (!registry) {
|
|
270
|
+
try {
|
|
271
|
+
registry = getActiveRegistry();
|
|
272
|
+
}
|
|
273
|
+
catch {
|
|
274
|
+
return [];
|
|
275
|
+
}
|
|
276
|
+
}
|
|
277
|
+
if (!registry)
|
|
278
|
+
return [];
|
|
279
|
+
return registry.shelves
|
|
280
|
+
.filter((s) => s.single && s.claude)
|
|
281
|
+
.map((s) => ({ library: s.library, shelf: s.shelf, claude: s.claude }));
|
|
282
|
+
}
|
|
217
283
|
export function collectLibraryPurposes(reg) {
|
|
218
284
|
let registry = reg;
|
|
219
285
|
if (!registry) {
|
|
@@ -259,6 +325,11 @@ export async function buildDomainSections() {
|
|
|
259
325
|
overview =
|
|
260
326
|
'## Libraries (what each holds / when to use it — pick the right one before searching)\n\n' + blocks.join('\n\n');
|
|
261
327
|
}
|
|
328
|
+
const singles = collectSingleShelves();
|
|
329
|
+
const singleSection = singles.length
|
|
330
|
+
? '## Single-record shelves (one document each — read the record, do not search the shelf)\n\n' +
|
|
331
|
+
singles.map((s) => `- ${s.library}/${s.shelf}: ${s.claude.replace(/\s*\n\s*/g, ' ')}`).join('\n')
|
|
332
|
+
: null;
|
|
262
333
|
const dataModel = '## Data model\n' +
|
|
263
334
|
'Library (top-level area) → shelf (a kind of record, e.g. things/item) → record (addressed library/shelf/id) → fields. ' +
|
|
264
335
|
'Some field values are JSON (e.g. quantity {"value":2000,"unit":"ml"}); some are relations (hold another record\'s id); ' +
|
|
@@ -273,6 +344,7 @@ export async function buildDomainSections() {
|
|
|
273
344
|
return [
|
|
274
345
|
dataModel,
|
|
275
346
|
...(overview ? [overview] : []),
|
|
347
|
+
...(singleSection ? [singleSection] : []),
|
|
276
348
|
...(site ? [`## web\n${site}`] : []),
|
|
277
349
|
...rules,
|
|
278
350
|
];
|
|
@@ -280,7 +352,7 @@ export async function buildDomainSections() {
|
|
|
280
352
|
export function buildMcpInstructions(sections) {
|
|
281
353
|
const base = [
|
|
282
354
|
"Coffer is the user's personal database, organized into libraries (kitchen, people, finance, health, devices, home, garden, documents, travel, and more), each holding typed records.",
|
|
283
|
-
'Use the tools to read and write this data: call list_libraries to see what exists, describe_shelf before create_record/update_record, then list_records / get_record to read. When a search_records tool is available, use it for
|
|
355
|
+
'Use the tools to read and write this data: call list_libraries to see what exists, describe_shelf before create_record/update_record, then list_records / get_record to read. When a search_records tool is available, use it for lookup — it searches by meaning and by exact token at once, and takes an optional library/shelf to narrow it. State where each fact came from by citing its record as [library/shelf/id]: an answer assembled from search results without citations cannot be checked against the data.',
|
|
284
356
|
'The per-library notes below name the shelves and the rules — not every field. For exact field names, types, and which are required, call describe_shelf(library, shelf) rather than guessing from the notes.',
|
|
285
357
|
'Record field values may be JSON-encoded (e.g. quantity {"value":2000,"unit":"ml"}) — parse them.',
|
|
286
358
|
'Action safety: read a complete record before editing or deleting it. update_record must be the smallest patch and must preserve unmentioned fields. delete_record moves the record to reversible trash; never blank fields to simulate deletion. For suspected duplicates, read both complete records, compare fields/collections/extends/attachments, identify the less complete record, and offer a field-by-field merge before moving anything to trash. Use list_trash/restore_record for recovery; purge_record is irreversible and requires explicit confirmation.',
|
package/dist/msg-log.d.ts
CHANGED
package/dist/msg-log.js
CHANGED
|
@@ -1,11 +1,29 @@
|
|
|
1
1
|
import { getEm } from "./db.js";
|
|
2
2
|
export async function logMessage(row) {
|
|
3
|
-
await getEm()
|
|
4
|
-
|
|
3
|
+
await getEm()
|
|
4
|
+
.fork()
|
|
5
|
+
.getConnection()
|
|
6
|
+
.execute(`INSERT INTO _orch_msg_log (ts, connector, chat_id, user_id, role, text, tokens_in, tokens_out, ms, agent_id, preset_id)
|
|
7
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, [
|
|
8
|
+
new Date().toISOString(),
|
|
9
|
+
row.connector,
|
|
10
|
+
row.chatId,
|
|
11
|
+
row.userId,
|
|
12
|
+
row.role,
|
|
13
|
+
row.text,
|
|
14
|
+
row.tokensIn,
|
|
15
|
+
row.tokensOut,
|
|
16
|
+
row.ms,
|
|
17
|
+
row.agentId,
|
|
18
|
+
row.presetId,
|
|
19
|
+
]);
|
|
5
20
|
}
|
|
6
21
|
export async function listRecentMessageMetrics(limit = 25) {
|
|
7
22
|
const bounded = Math.max(1, Math.min(100, Math.trunc(limit)));
|
|
8
|
-
const rows = (await getEm()
|
|
23
|
+
const rows = (await getEm()
|
|
24
|
+
.fork()
|
|
25
|
+
.getConnection()
|
|
26
|
+
.execute(`SELECT id, ts, connector, role, tokens_in, tokens_out, ms
|
|
9
27
|
FROM _orch_msg_log
|
|
10
28
|
ORDER BY id DESC
|
|
11
29
|
LIMIT ?`, [bounded]));
|
package/dist/mutate.d.ts
CHANGED
|
@@ -1,7 +1,9 @@
|
|
|
1
1
|
import type { EntityManager } from '@mikro-orm/core';
|
|
2
2
|
import type { ShelfDef } from '@coffer-org/sdk/shelf';
|
|
3
|
+
import type { LayoutEl } from '@coffer-org/sdk/fields';
|
|
3
4
|
import { z } from 'zod';
|
|
4
|
-
export declare function
|
|
5
|
+
export declare function encodeJsonAt(fields: LayoutEl[], data: Record<string, unknown>): Record<string, unknown>;
|
|
6
|
+
export declare const encodeJson: (m: ShelfDef, data: Record<string, unknown>) => Record<string, unknown>;
|
|
5
7
|
export interface MutateCtx {
|
|
6
8
|
actor: string;
|
|
7
9
|
}
|
package/dist/mutate.js
CHANGED
|
@@ -11,15 +11,16 @@ import { splitCollections, writeCollections, deleteCollections, flattenEmbedded,
|
|
|
11
11
|
function nowIso() {
|
|
12
12
|
return new Date().toISOString();
|
|
13
13
|
}
|
|
14
|
-
export function
|
|
14
|
+
export function encodeJsonAt(fields, data) {
|
|
15
15
|
const out = { ...data };
|
|
16
|
-
for (const [k, f] of fieldEntries(
|
|
16
|
+
for (const [k, f] of fieldEntries(fields)) {
|
|
17
17
|
if (isJsonStored(f) && out[k] !== null && out[k] !== undefined && typeof out[k] === 'object') {
|
|
18
18
|
out[k] = JSON.stringify(out[k]);
|
|
19
19
|
}
|
|
20
20
|
}
|
|
21
21
|
return out;
|
|
22
22
|
}
|
|
23
|
+
export const encodeJson = (m, data) => encodeJsonAt(m.fields, data);
|
|
23
24
|
export class ValidationError extends Error {
|
|
24
25
|
issues;
|
|
25
26
|
constructor(issues) {
|
|
@@ -119,10 +120,11 @@ export async function updateRecord(m, entityName, id, input, ctx, afterBase) {
|
|
|
119
120
|
const existingFlat = decodeTemporal(m, serialize(found));
|
|
120
121
|
const existing = nestEmbedded(m, existingFlat);
|
|
121
122
|
const ts = nowIso();
|
|
122
|
-
const
|
|
123
|
-
const fileIssues = normalizeFileFields(m,
|
|
123
|
+
const patchData = parsed.data;
|
|
124
|
+
const fileIssues = normalizeFileFields(m, patchData);
|
|
124
125
|
if (fileIssues.length)
|
|
125
126
|
throw new ValidationError(fileIssues);
|
|
127
|
+
const { base, collections } = splitCollections(m, { ...patchData });
|
|
126
128
|
const merged = { ...existing, ...base };
|
|
127
129
|
const reqIssues = [];
|
|
128
130
|
for (const [key, f] of fieldEntries(m.fields)) {
|
package/dist/oauth-api.js
CHANGED
|
@@ -3,10 +3,7 @@ import { resolveRequestUser, startPasswordSession } from "./auth-api.js";
|
|
|
3
3
|
import { baseUrl, mcpResource } from "./public-url.js";
|
|
4
4
|
import { getLogger } from "./log.js";
|
|
5
5
|
const log = getLogger('oauth');
|
|
6
|
-
const DEFAULT_REDIRECT_ALLOW = [
|
|
7
|
-
'https://claude.ai/api/mcp/auth_callback',
|
|
8
|
-
'https://claude.com/api/mcp/auth_callback',
|
|
9
|
-
];
|
|
6
|
+
const DEFAULT_REDIRECT_ALLOW = ['https://claude.ai/api/mcp/auth_callback', 'https://claude.com/api/mcp/auth_callback'];
|
|
10
7
|
function redirectAllowlist() {
|
|
11
8
|
const extra = (process.env['OAUTH_REDIRECT_ALLOW'] ?? '')
|
|
12
9
|
.split(',')
|
|
@@ -147,7 +144,10 @@ export function registerOAuthApi(app) {
|
|
|
147
144
|
.code(400)
|
|
148
145
|
.send({ error: 'invalid_redirect_uri', error_description: `redirect_uri not allowed: ${bad}` });
|
|
149
146
|
}
|
|
150
|
-
const client = await registerClient({
|
|
147
|
+
const client = await registerClient({
|
|
148
|
+
clientName: body.client_name?.slice(0, 200) || 'MCP client',
|
|
149
|
+
redirectUris: uris,
|
|
150
|
+
});
|
|
151
151
|
log.info(`registered client ${client.clientId} (${client.clientName})`);
|
|
152
152
|
return reply.code(201).send({
|
|
153
153
|
client_id: client.clientId,
|
|
@@ -160,14 +160,23 @@ export function registerOAuthApi(app) {
|
|
|
160
160
|
});
|
|
161
161
|
async function validateAuthz(p) {
|
|
162
162
|
if (!p.client_id || !p.redirect_uri) {
|
|
163
|
-
return {
|
|
163
|
+
return {
|
|
164
|
+
ok: false,
|
|
165
|
+
html: page('Invalid request', '<h1>Invalid request</h1><p>Missing client_id or redirect_uri.</p>'),
|
|
166
|
+
};
|
|
164
167
|
}
|
|
165
168
|
const client = await findClient(p.client_id);
|
|
166
169
|
if (!client) {
|
|
167
|
-
return {
|
|
170
|
+
return {
|
|
171
|
+
ok: false,
|
|
172
|
+
html: page('Unknown client', '<h1>Unknown client</h1><p>This application is not registered.</p>'),
|
|
173
|
+
};
|
|
168
174
|
}
|
|
169
175
|
if (!client.redirectUris.includes(p.redirect_uri)) {
|
|
170
|
-
return {
|
|
176
|
+
return {
|
|
177
|
+
ok: false,
|
|
178
|
+
html: page('Invalid redirect', '<h1>Invalid redirect</h1><p>redirect_uri does not match this client.</p>'),
|
|
179
|
+
};
|
|
171
180
|
}
|
|
172
181
|
return { ok: true, clientName: client.clientName };
|
|
173
182
|
}
|
|
@@ -197,12 +206,18 @@ export function registerOAuthApi(app) {
|
|
|
197
206
|
return reply.code(429).type('text/html').send(page('Slow down', '<h1>Too many attempts</h1>'));
|
|
198
207
|
const user = await startPasswordSession(reply, String(body['login'] ?? ''), String(body['password'] ?? ''));
|
|
199
208
|
if (!user)
|
|
200
|
-
return reply
|
|
209
|
+
return reply
|
|
210
|
+
.code(401)
|
|
211
|
+
.type('text/html')
|
|
212
|
+
.send(loginForm(p, check.clientName, 'Wrong login or password.'));
|
|
201
213
|
return reply.type('text/html').send(consentForm(p, check.clientName, user));
|
|
202
214
|
}
|
|
203
215
|
const user = await resolveRequestUser(req);
|
|
204
216
|
if (!user)
|
|
205
|
-
return reply
|
|
217
|
+
return reply
|
|
218
|
+
.code(401)
|
|
219
|
+
.type('text/html')
|
|
220
|
+
.send(loginForm(p, check.clientName, 'Your session expired. Sign in again.'));
|
|
206
221
|
if (body['action'] !== 'approve')
|
|
207
222
|
return redirectWithError(reply, p, 'access_denied');
|
|
208
223
|
if (!p.code_challenge)
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
import { readFile } from 'node:fs/promises';
|
|
2
|
+
const cache = new Map();
|
|
3
|
+
async function systemLanguage(fallback) {
|
|
4
|
+
try {
|
|
5
|
+
const { getPluginSettings } = await import("./plugin-runtime.js");
|
|
6
|
+
return (await getPluginSettings('core'))['language'] || fallback;
|
|
7
|
+
}
|
|
8
|
+
catch {
|
|
9
|
+
return fallback;
|
|
10
|
+
}
|
|
11
|
+
}
|
|
12
|
+
export async function loadPluginI18n(localesDir, fallbackLang = 'uk') {
|
|
13
|
+
const lang = await systemLanguage(fallbackLang);
|
|
14
|
+
const url = new URL(`${lang}.json`, localesDir);
|
|
15
|
+
let dict = cache.get(url.href);
|
|
16
|
+
if (!dict) {
|
|
17
|
+
try {
|
|
18
|
+
dict = JSON.parse(await readFile(url, 'utf8'));
|
|
19
|
+
}
|
|
20
|
+
catch {
|
|
21
|
+
dict = {};
|
|
22
|
+
}
|
|
23
|
+
cache.set(url.href, dict);
|
|
24
|
+
}
|
|
25
|
+
const loaded = dict;
|
|
26
|
+
return {
|
|
27
|
+
lang,
|
|
28
|
+
t(key, fallback) {
|
|
29
|
+
const value = key
|
|
30
|
+
.split('.')
|
|
31
|
+
.reduce((node, part) => (node == null ? undefined : node[part]), loaded);
|
|
32
|
+
return typeof value === 'string' ? value : (fallback ?? key);
|
|
33
|
+
},
|
|
34
|
+
};
|
|
35
|
+
}
|
|
36
|
+
export function clearPluginI18nCache() {
|
|
37
|
+
cache.clear();
|
|
38
|
+
}
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
import { type EmbeddingHit } from './embeddings.ts';
|
|
2
|
+
import { type GlobalSearchHit, type SearchShelf } from './global-search.ts';
|
|
3
|
+
export declare const RRF_K = 60;
|
|
4
|
+
export declare const MAX_VECTOR_DISTANCE = 0.85;
|
|
5
|
+
export declare const SEARCH_POOL = 50;
|
|
6
|
+
export interface FusedEntry {
|
|
7
|
+
score: number;
|
|
8
|
+
ranks: (number | null)[];
|
|
9
|
+
}
|
|
10
|
+
export declare function fuseRrf<T>(lists: T[][], keyOf: (item: T) => string, k?: number): Map<string, FusedEntry>;
|
|
11
|
+
export interface RagHit {
|
|
12
|
+
shelfKey: string;
|
|
13
|
+
recordId: number;
|
|
14
|
+
snippet: string;
|
|
15
|
+
distance: number | null;
|
|
16
|
+
vectorRank: number | null;
|
|
17
|
+
textRank: number | null;
|
|
18
|
+
score: number;
|
|
19
|
+
}
|
|
20
|
+
export interface HybridOpts {
|
|
21
|
+
query: string;
|
|
22
|
+
vector: number[] | null;
|
|
23
|
+
k: number;
|
|
24
|
+
library?: string;
|
|
25
|
+
shelf?: string;
|
|
26
|
+
maxDistance?: number;
|
|
27
|
+
rrfK?: number;
|
|
28
|
+
}
|
|
29
|
+
export interface HybridDeps {
|
|
30
|
+
vectorSearch?: (vec: number[], k: number, o?: {
|
|
31
|
+
shelfKeys?: Set<string>;
|
|
32
|
+
}) => Promise<EmbeddingHit[]>;
|
|
33
|
+
textSearch?: (shelves: SearchShelf[], q: string, k: number) => Promise<GlobalSearchHit[]>;
|
|
34
|
+
shelves?: SearchShelf[];
|
|
35
|
+
}
|
|
36
|
+
export declare function buildSearchShelves(filter?: {
|
|
37
|
+
library?: string;
|
|
38
|
+
shelf?: string;
|
|
39
|
+
}): SearchShelf[];
|
|
40
|
+
export declare function hybridSearch(opts: HybridOpts, deps?: HybridDeps): Promise<RagHit[]>;
|
|
@@ -0,0 +1,94 @@
|
|
|
1
|
+
import { searchEmbeddings } from "./embeddings.js";
|
|
2
|
+
import { globalSearch } from "./global-search.js";
|
|
3
|
+
import { getActiveRegistry } from "./registry-context.js";
|
|
4
|
+
import { shelfTableName } from "./entity-schema.js";
|
|
5
|
+
import { getLogger } from "./log.js";
|
|
6
|
+
const log = getLogger('rag-search');
|
|
7
|
+
export const RRF_K = 60;
|
|
8
|
+
export const MAX_VECTOR_DISTANCE = 0.85;
|
|
9
|
+
export const SEARCH_POOL = 50;
|
|
10
|
+
export function fuseRrf(lists, keyOf, k = RRF_K) {
|
|
11
|
+
const out = new Map();
|
|
12
|
+
lists.forEach((list, listIndex) => {
|
|
13
|
+
list.forEach((item, i) => {
|
|
14
|
+
const key = keyOf(item);
|
|
15
|
+
let entry = out.get(key);
|
|
16
|
+
if (!entry) {
|
|
17
|
+
entry = { score: 0, ranks: lists.map(() => null) };
|
|
18
|
+
out.set(key, entry);
|
|
19
|
+
}
|
|
20
|
+
if (entry.ranks[listIndex] !== null)
|
|
21
|
+
return;
|
|
22
|
+
entry.ranks[listIndex] = i + 1;
|
|
23
|
+
entry.score += 1 / (k + i + 1);
|
|
24
|
+
});
|
|
25
|
+
});
|
|
26
|
+
return out;
|
|
27
|
+
}
|
|
28
|
+
export function buildSearchShelves(filter = {}) {
|
|
29
|
+
return getActiveRegistry()
|
|
30
|
+
.shelves.filter((m) => (filter.library ? m.library === filter.library : true))
|
|
31
|
+
.filter((m) => (filter.shelf ? m.shelf === filter.shelf : true))
|
|
32
|
+
.map((m) => ({
|
|
33
|
+
key: `${m.library}/${m.shelf}`,
|
|
34
|
+
table: shelfTableName(m.library, m.shelf),
|
|
35
|
+
def: m,
|
|
36
|
+
}));
|
|
37
|
+
}
|
|
38
|
+
const hitKey = (shelfKey, recordId) => `${shelfKey}/${recordId}`;
|
|
39
|
+
async function leg(name, run) {
|
|
40
|
+
try {
|
|
41
|
+
return await run();
|
|
42
|
+
}
|
|
43
|
+
catch (e) {
|
|
44
|
+
log.warn(`${name} leg failed, degrading — ${e.message}`);
|
|
45
|
+
return [];
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
export async function hybridSearch(opts, deps = {}) {
|
|
49
|
+
const vectorSearch = deps.vectorSearch ?? searchEmbeddings;
|
|
50
|
+
const textSearch = deps.textSearch ?? globalSearch;
|
|
51
|
+
const narrowed = Boolean(opts.library || opts.shelf);
|
|
52
|
+
const shelves = deps.shelves
|
|
53
|
+
? deps.shelves
|
|
54
|
+
.filter((s) => (opts.library ? s.key.startsWith(`${opts.library}/`) : true))
|
|
55
|
+
.filter((s) => (opts.shelf ? s.key.endsWith(`/${opts.shelf}`) : true))
|
|
56
|
+
: buildSearchShelves({ library: opts.library, shelf: opts.shelf });
|
|
57
|
+
const shelfKeys = narrowed ? new Set(shelves.map((s) => s.key)) : undefined;
|
|
58
|
+
const maxDistance = opts.maxDistance ?? MAX_VECTOR_DISTANCE;
|
|
59
|
+
const [vectorHits, textHits] = await Promise.all([
|
|
60
|
+
opts.vector
|
|
61
|
+
? leg('vector', () => vectorSearch(opts.vector, SEARCH_POOL, { shelfKeys }))
|
|
62
|
+
: Promise.resolve([]),
|
|
63
|
+
leg('text', () => textSearch(shelves, opts.query, SEARCH_POOL)),
|
|
64
|
+
]);
|
|
65
|
+
const near = vectorHits.filter((h) => h.distance <= maxDistance);
|
|
66
|
+
const keyedText = textHits.map((h) => ({
|
|
67
|
+
shelfKey: `${h.library}/${h.shelf}`,
|
|
68
|
+
recordId: Number(h.id),
|
|
69
|
+
snippet: h.snippet,
|
|
70
|
+
}));
|
|
71
|
+
const fused = fuseRrf([near, keyedText], (h) => hitKey(h.shelfKey, h.recordId), opts.rrfK);
|
|
72
|
+
const snippets = new Map();
|
|
73
|
+
for (const h of keyedText)
|
|
74
|
+
snippets.set(hitKey(h.shelfKey, h.recordId), h.snippet);
|
|
75
|
+
for (const h of near)
|
|
76
|
+
snippets.set(hitKey(h.shelfKey, h.recordId), h.snippet);
|
|
77
|
+
const distances = new Map(near.map((h) => [hitKey(h.shelfKey, h.recordId), h.distance]));
|
|
78
|
+
return [...fused.entries()]
|
|
79
|
+
.map(([key, entry]) => {
|
|
80
|
+
const slash = key.lastIndexOf('/');
|
|
81
|
+
const shelfKey = key.slice(0, slash);
|
|
82
|
+
return {
|
|
83
|
+
shelfKey,
|
|
84
|
+
recordId: Number(key.slice(slash + 1)),
|
|
85
|
+
snippet: snippets.get(key) ?? '',
|
|
86
|
+
distance: distances.get(key) ?? null,
|
|
87
|
+
vectorRank: entry.ranks[0] ?? null,
|
|
88
|
+
textRank: entry.ranks[1] ?? null,
|
|
89
|
+
score: entry.score,
|
|
90
|
+
};
|
|
91
|
+
})
|
|
92
|
+
.sort((a, b) => b.score - a.score || hitKey(a.shelfKey, a.recordId).localeCompare(hitKey(b.shelfKey, b.recordId)))
|
|
93
|
+
.slice(0, opts.k);
|
|
94
|
+
}
|
package/dist/records-api.d.ts
CHANGED
|
@@ -1,6 +1,9 @@
|
|
|
1
1
|
import type { ShelfDef } from '@coffer-org/sdk/shelf';
|
|
2
2
|
export declare class UnknownShelfError extends Error {
|
|
3
3
|
}
|
|
4
|
+
export declare class SingleShelfError extends Error {
|
|
5
|
+
constructor(message: string);
|
|
6
|
+
}
|
|
4
7
|
export type RecordListQuery = {
|
|
5
8
|
q?: string;
|
|
6
9
|
id?: string;
|
|
@@ -33,6 +36,7 @@ export declare function recordGet(library: string, shelf: string, id: number, op
|
|
|
33
36
|
includeDeleted?: boolean;
|
|
34
37
|
}): Promise<Record<string, unknown> | null>;
|
|
35
38
|
export declare function recordCreate(library: string, shelf: string, body: unknown, actor?: string): Promise<Record<string, unknown>>;
|
|
39
|
+
export declare function ensureSingle(library: string, shelf: string): Promise<Record<string, unknown>>;
|
|
36
40
|
export declare function recordUpdate(library: string, shelf: string, id: number, body: unknown, actor?: string): Promise<Record<string, unknown>>;
|
|
37
41
|
export declare function recordDelete(library: string, shelf: string, id: number, actor?: string): Promise<void>;
|
|
38
42
|
export declare function recordRestore(library: string, shelf: string, id: number, actor?: string): Promise<void>;
|
package/dist/records-api.js
CHANGED
|
@@ -11,6 +11,12 @@ import { createRecord, updateRecord, getRecord, deleteRecord, restoreRecord, pur
|
|
|
11
11
|
import { deleteRecordActivity } from "./record-activity.js";
|
|
12
12
|
export class UnknownShelfError extends Error {
|
|
13
13
|
}
|
|
14
|
+
export class SingleShelfError extends Error {
|
|
15
|
+
constructor(message) {
|
|
16
|
+
super(message);
|
|
17
|
+
this.name = 'SingleShelfError';
|
|
18
|
+
}
|
|
19
|
+
}
|
|
14
20
|
export const MAX_PAGE = 500;
|
|
15
21
|
function pickCols(row, cols) {
|
|
16
22
|
const out = {};
|
|
@@ -110,9 +116,11 @@ export async function recordCount(library, shelf, query = {}, opts = {}) {
|
|
|
110
116
|
where.deleted_at = { $ne: null };
|
|
111
117
|
if (m.standalone === false && Object.keys(filterParams).length === 0)
|
|
112
118
|
return 0;
|
|
113
|
-
return getEm()
|
|
119
|
+
return getEm()
|
|
120
|
+
.fork()
|
|
121
|
+
.count(ename, where);
|
|
114
122
|
}
|
|
115
|
-
|
|
123
|
+
async function listRecords(library, shelf, query = {}, opts = {}) {
|
|
116
124
|
const { view = 'full', extends: withExt = true, deleted = 'active' } = opts;
|
|
117
125
|
const { m, ename } = resolve(library, shelf);
|
|
118
126
|
const { q, ...filterParams } = query;
|
|
@@ -147,6 +155,12 @@ export async function recordList(library, shelf, query = {}, opts = {}) {
|
|
|
147
155
|
return decoded;
|
|
148
156
|
return withExtendsMany(decoded, library, shelf);
|
|
149
157
|
}
|
|
158
|
+
export async function recordList(library, shelf, query = {}, opts = {}) {
|
|
159
|
+
const { m } = resolve(library, shelf);
|
|
160
|
+
if (m.single)
|
|
161
|
+
await ensureSingle(library, shelf);
|
|
162
|
+
return listRecords(library, shelf, query, opts);
|
|
163
|
+
}
|
|
150
164
|
export function isPagedRequest(limit, offset) {
|
|
151
165
|
return [limit, offset].some((v) => v !== undefined && v.trim() !== '' && Number.isFinite(Number(v)));
|
|
152
166
|
}
|
|
@@ -179,7 +193,7 @@ export async function recordGet(library, shelf, id, opts = {}) {
|
|
|
179
193
|
return null;
|
|
180
194
|
return withExtends(row, library, shelf);
|
|
181
195
|
}
|
|
182
|
-
|
|
196
|
+
async function createOne(library, shelf, body, actor = 'gui') {
|
|
183
197
|
const { m, ename } = resolve(library, shelf);
|
|
184
198
|
const { base, extData } = splitBody(body);
|
|
185
199
|
validateExtends(library, shelf, extData);
|
|
@@ -189,6 +203,37 @@ export async function recordCreate(library, shelf, body, actor = 'gui') {
|
|
|
189
203
|
});
|
|
190
204
|
return withExtends(row, library, shelf);
|
|
191
205
|
}
|
|
206
|
+
export async function recordCreate(library, shelf, body, actor = 'gui') {
|
|
207
|
+
const { m } = resolve(library, shelf);
|
|
208
|
+
if (m.single) {
|
|
209
|
+
await ensureSingle(library, shelf);
|
|
210
|
+
throw new SingleShelfError(`single_shelf ${library}/${shelf}: the record already exists — update it instead`);
|
|
211
|
+
}
|
|
212
|
+
return createOne(library, shelf, body, actor);
|
|
213
|
+
}
|
|
214
|
+
const singleInFlight = new Map();
|
|
215
|
+
export async function ensureSingle(library, shelf) {
|
|
216
|
+
const { m } = resolve(library, shelf);
|
|
217
|
+
if (!m.single)
|
|
218
|
+
throw new Error(`not a single shelf: ${library}/${shelf}`);
|
|
219
|
+
const key = `${library}/${shelf}`;
|
|
220
|
+
const pending = singleInFlight.get(key);
|
|
221
|
+
if (pending)
|
|
222
|
+
return pending;
|
|
223
|
+
const task = (async () => {
|
|
224
|
+
const rows = await listRecords(library, shelf, {}, { limit: 1, orderBy: { id: 'ASC' } });
|
|
225
|
+
if (rows[0])
|
|
226
|
+
return rows[0];
|
|
227
|
+
return createOne(library, shelf, {}, 'system');
|
|
228
|
+
})();
|
|
229
|
+
singleInFlight.set(key, task);
|
|
230
|
+
try {
|
|
231
|
+
return await task;
|
|
232
|
+
}
|
|
233
|
+
finally {
|
|
234
|
+
singleInFlight.delete(key);
|
|
235
|
+
}
|
|
236
|
+
}
|
|
192
237
|
export async function recordUpdate(library, shelf, id, body, actor = 'gui') {
|
|
193
238
|
const { m, ename } = resolve(library, shelf);
|
|
194
239
|
const { base, extData } = splitBody(body);
|
|
@@ -201,6 +246,8 @@ export async function recordUpdate(library, shelf, id, body, actor = 'gui') {
|
|
|
201
246
|
}
|
|
202
247
|
export async function recordDelete(library, shelf, id, actor = 'gui') {
|
|
203
248
|
const { m, ename } = resolve(library, shelf);
|
|
249
|
+
if (m.single)
|
|
250
|
+
throw new SingleShelfError(`single_shelf ${library}/${shelf}: the record cannot be deleted — clear its fields instead`);
|
|
204
251
|
await deleteRecord(m, ename, id, { actor });
|
|
205
252
|
}
|
|
206
253
|
export async function recordRestore(library, shelf, id, actor = 'gui') {
|
|
@@ -209,6 +256,8 @@ export async function recordRestore(library, shelf, id, actor = 'gui') {
|
|
|
209
256
|
}
|
|
210
257
|
export async function recordPurge(library, shelf, id, actor = 'gui') {
|
|
211
258
|
const { m, ename } = resolve(library, shelf);
|
|
259
|
+
if (m.single)
|
|
260
|
+
throw new SingleShelfError(`single_shelf ${library}/${shelf}: the record cannot be deleted — clear its fields instead`);
|
|
212
261
|
await purgeRecord(m, ename, id, { actor }, (tx, recordId) => deleteExtends(tx, library, shelf, recordId));
|
|
213
262
|
await deleteRecordActivity(library, shelf, id);
|
|
214
263
|
}
|
|
@@ -216,7 +265,7 @@ export async function recordTrashList() {
|
|
|
216
265
|
const shelves = getActiveRegistry().shelves.filter((s) => s.standalone !== false);
|
|
217
266
|
const out = [];
|
|
218
267
|
for (const m of shelves) {
|
|
219
|
-
const rows = await
|
|
268
|
+
const rows = await listRecords(m.library, m.shelf, {}, { view: 'full', extends: true, deleted: 'deleted' });
|
|
220
269
|
for (const record of rows)
|
|
221
270
|
out.push({ library: m.library, shelf: m.shelf, label: m.label, record });
|
|
222
271
|
}
|
package/dist/search-index.js
CHANGED
|
@@ -57,11 +57,7 @@ export async function upsertSearchRow(shelf, recordId, folded) {
|
|
|
57
57
|
await conn.execute('DELETE FROM "_search" WHERE shelf = ? AND record_id = ?', [shelf, recordId]);
|
|
58
58
|
if (!folded)
|
|
59
59
|
return;
|
|
60
|
-
await conn.execute('INSERT INTO "_search" (shelf, record_id, folded) VALUES (?, ?, ?)', [
|
|
61
|
-
shelf,
|
|
62
|
-
recordId,
|
|
63
|
-
folded,
|
|
64
|
-
]);
|
|
60
|
+
await conn.execute('INSERT INTO "_search" (shelf, record_id, folded) VALUES (?, ?, ?)', [shelf, recordId, folded]);
|
|
65
61
|
}
|
|
66
62
|
export async function deleteSearchRow(shelf, recordId) {
|
|
67
63
|
if (!ftsAvailable())
|
|
@@ -81,9 +77,7 @@ export async function ftsCandidates(tokens) {
|
|
|
81
77
|
const rows = (await getEm()
|
|
82
78
|
.fork()
|
|
83
79
|
.getConnection()
|
|
84
|
-
.execute('SELECT shelf, record_id FROM "_search" WHERE "_search" MATCH ?', [
|
|
85
|
-
buildFtsMatchQuery(tokens),
|
|
86
|
-
]));
|
|
80
|
+
.execute('SELECT shelf, record_id FROM "_search" WHERE "_search" MATCH ?', [buildFtsMatchQuery(tokens)]));
|
|
87
81
|
for (const r of rows) {
|
|
88
82
|
const list = out.get(r.shelf);
|
|
89
83
|
if (list)
|