obsidian-mcp-server 3.7.1 → 3.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +28 -13
- package/CLAUDE.md +28 -13
- package/Dockerfile +7 -0
- package/README.md +43 -99
- package/changelog/3.7.x/3.7.2.md +34 -0
- package/changelog/3.8.x/3.8.0.md +38 -0
- package/dist/config/server-config.d.ts +1 -0
- package/dist/config/server-config.d.ts.map +1 -1
- package/dist/config/server-config.js +24 -15
- package/dist/config/server-config.js.map +1 -1
- package/dist/index.js +6 -1
- package/dist/index.js.map +1 -1
- package/dist/instructions.d.ts.map +1 -1
- package/dist/instructions.js +21 -7
- package/dist/instructions.js.map +1 -1
- package/dist/mcp-server/resources/definitions/index.d.ts +2 -2
- package/dist/mcp-server/resources/definitions/obsidian-tags.resource.d.ts +3 -2
- package/dist/mcp-server/resources/definitions/obsidian-tags.resource.d.ts.map +1 -1
- package/dist/mcp-server/resources/definitions/obsidian-tags.resource.js +6 -5
- package/dist/mcp-server/resources/definitions/obsidian-tags.resource.js.map +1 -1
- package/dist/mcp-server/resources/definitions/obsidian-vault-note.resource.d.ts +2 -2
- package/dist/mcp-server/resources/definitions/obsidian-vault-note.resource.js +2 -2
- package/dist/mcp-server/resources/definitions/obsidian-vault-note.resource.js.map +1 -1
- package/dist/mcp-server/tools/definitions/_shared/markdown.d.ts +14 -0
- package/dist/mcp-server/tools/definitions/_shared/markdown.d.ts.map +1 -0
- package/dist/mcp-server/tools/definitions/_shared/markdown.js +17 -0
- package/dist/mcp-server/tools/definitions/_shared/markdown.js.map +1 -0
- package/dist/mcp-server/tools/definitions/_shared/suggest-paths.d.ts +8 -1
- package/dist/mcp-server/tools/definitions/_shared/suggest-paths.d.ts.map +1 -1
- package/dist/mcp-server/tools/definitions/_shared/suggest-paths.js +17 -6
- package/dist/mcp-server/tools/definitions/_shared/suggest-paths.js.map +1 -1
- package/dist/mcp-server/tools/definitions/index.d.ts +18 -18
- package/dist/mcp-server/tools/definitions/obsidian-append-to-note.tool.d.ts +2 -2
- package/dist/mcp-server/tools/definitions/obsidian-append-to-note.tool.d.ts.map +1 -1
- package/dist/mcp-server/tools/definitions/obsidian-append-to-note.tool.js +5 -4
- package/dist/mcp-server/tools/definitions/obsidian-append-to-note.tool.js.map +1 -1
- package/dist/mcp-server/tools/definitions/obsidian-delete-note.tool.d.ts +7 -6
- package/dist/mcp-server/tools/definitions/obsidian-delete-note.tool.d.ts.map +1 -1
- package/dist/mcp-server/tools/definitions/obsidian-delete-note.tool.js +9 -8
- package/dist/mcp-server/tools/definitions/obsidian-delete-note.tool.js.map +1 -1
- package/dist/mcp-server/tools/definitions/obsidian-execute-command.tool.d.ts.map +1 -1
- package/dist/mcp-server/tools/definitions/obsidian-execute-command.tool.js +2 -1
- package/dist/mcp-server/tools/definitions/obsidian-execute-command.tool.js.map +1 -1
- package/dist/mcp-server/tools/definitions/obsidian-get-note.tool.d.ts +2 -2
- package/dist/mcp-server/tools/definitions/obsidian-get-note.tool.d.ts.map +1 -1
- package/dist/mcp-server/tools/definitions/obsidian-get-note.tool.js +38 -28
- package/dist/mcp-server/tools/definitions/obsidian-get-note.tool.js.map +1 -1
- package/dist/mcp-server/tools/definitions/obsidian-list-commands.tool.d.ts.map +1 -1
- package/dist/mcp-server/tools/definitions/obsidian-list-commands.tool.js +7 -4
- package/dist/mcp-server/tools/definitions/obsidian-list-commands.tool.js.map +1 -1
- package/dist/mcp-server/tools/definitions/obsidian-list-notes.tool.d.ts +6 -6
- package/dist/mcp-server/tools/definitions/obsidian-list-notes.tool.d.ts.map +1 -1
- package/dist/mcp-server/tools/definitions/obsidian-list-notes.tool.js +15 -14
- package/dist/mcp-server/tools/definitions/obsidian-list-notes.tool.js.map +1 -1
- package/dist/mcp-server/tools/definitions/obsidian-list-tags.tool.d.ts +4 -4
- package/dist/mcp-server/tools/definitions/obsidian-list-tags.tool.d.ts.map +1 -1
- package/dist/mcp-server/tools/definitions/obsidian-list-tags.tool.js +9 -8
- package/dist/mcp-server/tools/definitions/obsidian-list-tags.tool.js.map +1 -1
- package/dist/mcp-server/tools/definitions/obsidian-manage-frontmatter.tool.d.ts +2 -2
- package/dist/mcp-server/tools/definitions/obsidian-manage-frontmatter.tool.d.ts.map +1 -1
- package/dist/mcp-server/tools/definitions/obsidian-manage-frontmatter.tool.js +9 -7
- package/dist/mcp-server/tools/definitions/obsidian-manage-frontmatter.tool.js.map +1 -1
- package/dist/mcp-server/tools/definitions/obsidian-manage-tags.tool.d.ts +2 -2
- package/dist/mcp-server/tools/definitions/obsidian-manage-tags.tool.d.ts.map +1 -1
- package/dist/mcp-server/tools/definitions/obsidian-manage-tags.tool.js +7 -5
- package/dist/mcp-server/tools/definitions/obsidian-manage-tags.tool.js.map +1 -1
- package/dist/mcp-server/tools/definitions/obsidian-open-in-ui.tool.d.ts +2 -2
- package/dist/mcp-server/tools/definitions/obsidian-open-in-ui.tool.d.ts.map +1 -1
- package/dist/mcp-server/tools/definitions/obsidian-open-in-ui.tool.js +4 -3
- package/dist/mcp-server/tools/definitions/obsidian-open-in-ui.tool.js.map +1 -1
- package/dist/mcp-server/tools/definitions/obsidian-patch-note.tool.d.ts +2 -2
- package/dist/mcp-server/tools/definitions/obsidian-patch-note.tool.d.ts.map +1 -1
- package/dist/mcp-server/tools/definitions/obsidian-patch-note.tool.js +5 -4
- package/dist/mcp-server/tools/definitions/obsidian-patch-note.tool.js.map +1 -1
- package/dist/mcp-server/tools/definitions/obsidian-replace-in-note.tool.d.ts +2 -2
- package/dist/mcp-server/tools/definitions/obsidian-replace-in-note.tool.d.ts.map +1 -1
- package/dist/mcp-server/tools/definitions/obsidian-replace-in-note.tool.js +10 -4
- package/dist/mcp-server/tools/definitions/obsidian-replace-in-note.tool.js.map +1 -1
- package/dist/mcp-server/tools/definitions/obsidian-search-notes.tool.d.ts +46 -12
- package/dist/mcp-server/tools/definitions/obsidian-search-notes.tool.d.ts.map +1 -1
- package/dist/mcp-server/tools/definitions/obsidian-search-notes.tool.js +93 -43
- package/dist/mcp-server/tools/definitions/obsidian-search-notes.tool.js.map +1 -1
- package/dist/mcp-server/tools/definitions/obsidian-write-note.tool.d.ts +2 -2
- package/dist/mcp-server/tools/definitions/obsidian-write-note.tool.d.ts.map +1 -1
- package/dist/mcp-server/tools/definitions/obsidian-write-note.tool.js +5 -4
- package/dist/mcp-server/tools/definitions/obsidian-write-note.tool.js.map +1 -1
- package/dist/services/obsidian/jsonlogic-evaluator.d.ts +33 -8
- package/dist/services/obsidian/jsonlogic-evaluator.d.ts.map +1 -1
- package/dist/services/obsidian/jsonlogic-evaluator.js +171 -32
- package/dist/services/obsidian/jsonlogic-evaluator.js.map +1 -1
- package/dist/services/obsidian/jsonlogic-worker.d.ts +6 -5
- package/dist/services/obsidian/jsonlogic-worker.d.ts.map +1 -1
- package/dist/services/obsidian/jsonlogic-worker.js +7 -5
- package/dist/services/obsidian/jsonlogic-worker.js.map +1 -1
- package/dist/services/obsidian/obsidian-service.d.ts +38 -22
- package/dist/services/obsidian/obsidian-service.d.ts.map +1 -1
- package/dist/services/obsidian/obsidian-service.js +293 -99
- package/dist/services/obsidian/obsidian-service.js.map +1 -1
- package/dist/services/obsidian/patch-instruction.d.ts +4 -1
- package/dist/services/obsidian/patch-instruction.d.ts.map +1 -1
- package/dist/services/obsidian/patch-instruction.js +5 -2
- package/dist/services/obsidian/patch-instruction.js.map +1 -1
- package/dist/services/obsidian/path-policy.d.ts +67 -23
- package/dist/services/obsidian/path-policy.d.ts.map +1 -1
- package/dist/services/obsidian/path-policy.js +183 -80
- package/dist/services/obsidian/path-policy.js.map +1 -1
- package/dist/services/obsidian/section-extractor.d.ts +6 -1
- package/dist/services/obsidian/section-extractor.d.ts.map +1 -1
- package/dist/services/obsidian/section-extractor.js +106 -11
- package/dist/services/obsidian/section-extractor.js.map +1 -1
- package/dist/services/obsidian/simple-search.d.ts +62 -0
- package/dist/services/obsidian/simple-search.d.ts.map +1 -0
- package/dist/services/obsidian/simple-search.js +264 -0
- package/dist/services/obsidian/simple-search.js.map +1 -0
- package/manifest.json +10 -2
- package/package.json +8 -7
- package/server.json +19 -7
|
@@ -5,21 +5,68 @@
|
|
|
5
5
|
* @module services/obsidian/obsidian-service
|
|
6
6
|
*/
|
|
7
7
|
import { configurationError, conflict, forbidden, JsonRpcErrorCode, McpError, notFound, serviceUnavailable, unauthorized, validationError, } from '@cyanheads/mcp-ts-core/errors';
|
|
8
|
-
import { httpStatusToErrorCode, withRetry } from '@cyanheads/mcp-ts-core/utils';
|
|
8
|
+
import { createPacer, httpStatusToErrorCode, withRetry } from '@cyanheads/mcp-ts-core/utils';
|
|
9
9
|
import { Agent, fetch as undiciFetch } from 'undici';
|
|
10
10
|
import { getServerConfig } from '../../config/server-config.js';
|
|
11
|
-
import { assertTreeDepth, createJsonLogicEvaluation, readsContent } from './jsonlogic-evaluator.js';
|
|
11
|
+
import { assertNoLinkFields, assertTreeDepth, createJsonLogicEvaluation, readsContent, } from './jsonlogic-evaluator.js';
|
|
12
12
|
import { canonicalContent, flattenDocumentMap, flattenHeadingTree, hasIntegerKey, inNoteOrder, isIntegerKey, patchFormatFor, relativeHeadingLevels, v1HeadingBody, v1PatchHeaders, v2Instruction, } from './patch-instruction.js';
|
|
13
|
-
import { PathPolicy } from './path-policy.js';
|
|
13
|
+
import { caseFoldedPattern, PathPolicy } from './path-policy.js';
|
|
14
14
|
import { atxHeadingMarkers, blockKinds, isIsolatedBlockId, listHeadingPaths, sectionBody, sectionLevel, } from './section-extractor.js';
|
|
15
|
+
import { isHighSurrogate, isLowSurrogate, SimpleSearch, simpleSearchTokens, } from './simple-search.js';
|
|
15
16
|
/**
|
|
16
|
-
* Most note-body bytes (by `stat.size`) a scoped JSONLogic search
|
|
17
|
-
* evaluates at once
|
|
18
|
-
*
|
|
19
|
-
*
|
|
20
|
-
* readable vault sits in memory.
|
|
17
|
+
* Most note-body bytes (by `stat.size`) a scoped JSONLogic search holds and
|
|
18
|
+
* evaluates at once, so a broad scope never holds every readable body. Each
|
|
19
|
+
* body is its own `GET /vault/{path}`, so the batch size sets memory, not the
|
|
20
|
+
* number of upstream reads.
|
|
21
21
|
*/
|
|
22
22
|
const CONTENT_BATCH_BYTES = 32 * 1024 * 1024;
|
|
23
|
+
/**
|
|
24
|
+
* Most note-body reads in flight at once, across every search the service
|
|
25
|
+
* runs, so concurrent searches share the bound rather than each taking it.
|
|
26
|
+
* The plugin serves every read from Obsidian's own event loop, so each request
|
|
27
|
+
* in flight queues there; 32 keeps a whole-vault read well inside
|
|
28
|
+
* {@link SCOPED_SEARCH_DEADLINE_MS} without queuing more than that.
|
|
29
|
+
*/
|
|
30
|
+
const BODY_READ_CONCURRENCY = 32;
|
|
31
|
+
/**
|
|
32
|
+
* Wall-clock ceiling on one scoped text or JSONLogic search: its upstream
|
|
33
|
+
* search, every body read with its retries, and the matching or evaluation.
|
|
34
|
+
* One deadline for the whole search, not one per request, so a search over a
|
|
35
|
+
* broad scope answers or fails inside the 60 s an MCP client waits for a
|
|
36
|
+
* response by default.
|
|
37
|
+
*/
|
|
38
|
+
const SCOPED_SEARCH_DEADLINE_MS = 45_000;
|
|
39
|
+
/**
|
|
40
|
+
* Most characters the plugin's `/search/simple/` response may take for the
|
|
41
|
+
* hits of a scoped text search: V8's longest string (2^29 − 24 on 64-bit),
|
|
42
|
+
* where the plugin's `res.json` fails serializing it. `SimpleSearch` counts
|
|
43
|
+
* that response as the plugin writes it, each match's offsets, escaped window,
|
|
44
|
+
* and framing with every hit's filename and score, so a scoped search fails
|
|
45
|
+
* with `context_length_too_large` where the plugin would fail over the
|
|
46
|
+
* readable notes alone, and never holds more matches than such a response.
|
|
47
|
+
*/
|
|
48
|
+
const SCOPED_TEXT_RESPONSE_CHARS = 2 ** 29 - 24;
|
|
49
|
+
/**
|
|
50
|
+
* Run a scoped search under {@link SCOPED_SEARCH_DEADLINE_MS}: `run` gets a
|
|
51
|
+
* signal that aborts with `ctx.signal` or at the deadline, and a failure
|
|
52
|
+
* caused by the deadline alone is replaced by `tooCostly(cause)`.
|
|
53
|
+
*/
|
|
54
|
+
async function withScopedDeadline(ctx, tooCostly, run) {
|
|
55
|
+
// A timer rather than `AbortSignal.timeout`, which can fail on a realm mismatch under Bun's stdio transport.
|
|
56
|
+
const deadline = new AbortController();
|
|
57
|
+
const timer = setTimeout(() => deadline.abort(new DOMException('Scoped search deadline', 'TimeoutError')), SCOPED_SEARCH_DEADLINE_MS);
|
|
58
|
+
try {
|
|
59
|
+
return await run(AbortSignal.any([ctx.signal, deadline.signal]));
|
|
60
|
+
}
|
|
61
|
+
catch (err) {
|
|
62
|
+
if (!deadline.signal.aborted || ctx.signal.aborted)
|
|
63
|
+
throw err;
|
|
64
|
+
throw tooCostly(err);
|
|
65
|
+
}
|
|
66
|
+
finally {
|
|
67
|
+
clearTimeout(timer);
|
|
68
|
+
}
|
|
69
|
+
}
|
|
23
70
|
/** Split `notes`, in order, into runs whose bodies total at most {@link CONTENT_BATCH_BYTES}; a larger note gets a run of its own. */
|
|
24
71
|
function contentBatches(notes) {
|
|
25
72
|
const batches = [];
|
|
@@ -152,6 +199,15 @@ export class ObsidianService {
|
|
|
152
199
|
/** Stored scoped JSONLogic walks by id, least recently read first (`pageJsonLogic`). */
|
|
153
200
|
#walks = new Map();
|
|
154
201
|
#walkChars = 0;
|
|
202
|
+
/**
|
|
203
|
+
* The queue every note-body read of a scoped text or JSONLogic search waits
|
|
204
|
+
* in, so {@link BODY_READ_CONCURRENCY} bounds the reads in flight across
|
|
205
|
+
* concurrent searches rather than per search.
|
|
206
|
+
*/
|
|
207
|
+
#bodyReads = createPacer({
|
|
208
|
+
name: 'obsidian-note-bodies',
|
|
209
|
+
maxConcurrent: BODY_READ_CONCURRENCY,
|
|
210
|
+
});
|
|
155
211
|
/**
|
|
156
212
|
* @param config - Validated server config (api key, base URL, TLS, timeouts).
|
|
157
213
|
* @param fetchImpl - Optional fetch override for tests. Defaults to undici's
|
|
@@ -183,14 +239,16 @@ export class ObsidianService {
|
|
|
183
239
|
}
|
|
184
240
|
/**
|
|
185
241
|
* Close the undici dispatcher, releasing its keep-alive sockets to the Local
|
|
186
|
-
* REST API and Omnisearch
|
|
187
|
-
* in-flight requests to
|
|
242
|
+
* REST API and Omnisearch, after rejecting the note-body reads still queued.
|
|
243
|
+
* Wired through `createApp({ teardown })`; waits for in-flight requests to
|
|
244
|
+
* settle rather than aborting them.
|
|
188
245
|
*
|
|
189
246
|
* Bun resolves `undici` to a built-in shim whose `Agent` is an inert object
|
|
190
247
|
* with no `close` — Bun's `fetch` pools sockets itself and ignores the
|
|
191
248
|
* dispatcher — so there is nothing to release there.
|
|
192
249
|
*/
|
|
193
250
|
async close() {
|
|
251
|
+
this.#bodyReads.dispose();
|
|
194
252
|
const dispatcher = this.#dispatcher;
|
|
195
253
|
await dispatcher.close?.();
|
|
196
254
|
}
|
|
@@ -400,7 +458,7 @@ export class ObsidianService {
|
|
|
400
458
|
let url = '/vault/';
|
|
401
459
|
let normalized = '';
|
|
402
460
|
if (dirPath) {
|
|
403
|
-
normalized = dirPath
|
|
461
|
+
normalized = trimSlashes(dirPath);
|
|
404
462
|
if (normalized)
|
|
405
463
|
url = `/vault/${encodeVaultPath(normalized)}/`;
|
|
406
464
|
}
|
|
@@ -423,9 +481,10 @@ export class ObsidianService {
|
|
|
423
481
|
/**
|
|
424
482
|
* Vault tags with usage counts, hierarchical parents included. Unscoped, the
|
|
425
483
|
* plugin's `/tags/` with its occurrence counts. That payload carries no
|
|
426
|
-
* per-note attribution, so under OBSIDIAN_READ_PATHS
|
|
427
|
-
* the readable notes' contexts instead (one upstream
|
|
428
|
-
* each one counts the readable notes carrying it or a
|
|
484
|
+
* per-note attribution, so under OBSIDIAN_READ_PATHS or OBSIDIAN_DENY_PATHS
|
|
485
|
+
* the tags are read from the readable notes' contexts instead (one upstream
|
|
486
|
+
* search, no bodies), and each one counts the readable notes carrying it or a
|
|
487
|
+
* tag nested under it.
|
|
429
488
|
*/
|
|
430
489
|
async listTags(ctx) {
|
|
431
490
|
if (this.#policy.restrictsReads) {
|
|
@@ -446,13 +505,18 @@ export class ObsidianService {
|
|
|
446
505
|
* (`contextRelativeSpan`), then a multi-token query's spans are merged into
|
|
447
506
|
* match locations (`mergeIntoLocations`).
|
|
448
507
|
*
|
|
449
|
-
* The upstream
|
|
450
|
-
*
|
|
451
|
-
*
|
|
452
|
-
*
|
|
453
|
-
*
|
|
508
|
+
* The upstream serializes every match in the vault, its context window and
|
|
509
|
+
* about 150 characters of offsets and framing, into one response, so a
|
|
510
|
+
* broad query exhausts V8's string capacity, even at a small
|
|
511
|
+
* `contextLength`, and comes back as an opaque 500 — caught
|
|
512
|
+
* here and re-thrown typed, the way `searchOmnisearch` re-throws its own
|
|
513
|
+
* reachability failure. Under OBSIDIAN_READ_PATHS or OBSIDIAN_DENY_PATHS
|
|
514
|
+
* that would make the outcome depend on notes outside the scope, so the
|
|
515
|
+
* search runs here instead (`#searchTextScoped`).
|
|
454
516
|
*/
|
|
455
517
|
async searchText(ctx, query, contextLength = 100) {
|
|
518
|
+
if (this.#policy.restrictsReads)
|
|
519
|
+
return await this.#searchTextScoped(ctx, query, contextLength);
|
|
456
520
|
const params = new URLSearchParams({ query, contextLength: String(contextLength) });
|
|
457
521
|
let res;
|
|
458
522
|
try {
|
|
@@ -467,58 +531,142 @@ export class ObsidianService {
|
|
|
467
531
|
* and through it the upstream body `#throwForStatus` attached — on the
|
|
468
532
|
* server side for the log record.
|
|
469
533
|
*/
|
|
470
|
-
throw validationError(`The Local REST API ran out of string capacity
|
|
534
|
+
throw validationError(`The Local REST API ran out of string capacity serializing the matches for this search (contextLength ${contextLength}). Each match costs its context window plus about 150 characters of offsets and framing, so the total tracks the number of matches as well as contextLength.`, {
|
|
471
535
|
reason: 'context_length_too_large',
|
|
472
536
|
contextLength,
|
|
473
537
|
}, { cause: err });
|
|
474
538
|
}
|
|
475
539
|
const raw = (await res.json());
|
|
476
|
-
const tokens =
|
|
477
|
-
// Upstream
|
|
478
|
-
//
|
|
479
|
-
//
|
|
540
|
+
const tokens = simpleSearchTokens(query);
|
|
541
|
+
// Upstream's `score` orders the hits but is not a relevance measure — drop
|
|
542
|
+
// it on the way out so consumers don't mistake it for one. Omnisearch is
|
|
543
|
+
// the source of real BM25 ranking.
|
|
480
544
|
return raw.map((h) => ({
|
|
481
545
|
filename: h.filename,
|
|
482
|
-
matches: mergeIntoLocations(h.matches.map((m) => contextRelativeSpan(m, h.filename, contextLength, tokens)), tokens.length, contextLength)
|
|
546
|
+
matches: mergeIntoLocations(h.matches.map((m) => contextRelativeSpan(m, h.filename, contextLength, tokens)), tokens.length, contextLength),
|
|
483
547
|
}));
|
|
484
548
|
}
|
|
549
|
+
/**
|
|
550
|
+
* Text search under a read scope, with no `/search/simple/` request: one
|
|
551
|
+
* `/search/` guarded by the read scope and, per token, a case-folded
|
|
552
|
+
* `regexp` on `path` or `content` (`#textCandidates`) names the readable
|
|
553
|
+
* notes that may hold every token, their bodies are read one GET each
|
|
554
|
+
* (`#bodyBatches`), and `SimpleSearch` matches them the way the plugin
|
|
555
|
+
* does, in the plugin's order. `context_length_too_large` then counts the
|
|
556
|
+
* plugin's response over the readable notes alone, against
|
|
557
|
+
* `SCOPED_TEXT_RESPONSE_CHARS`. Every request shares `ctx.signal` and one
|
|
558
|
+
* deadline, `SCOPED_SEARCH_DEADLINE_MS`; past it the search fails with
|
|
559
|
+
* `text_search_too_costly`.
|
|
560
|
+
*/
|
|
561
|
+
async #searchTextScoped(ctx, query, contextLength) {
|
|
562
|
+
const search = new SimpleSearch(query, contextLength, SCOPED_TEXT_RESPONSE_CHARS);
|
|
563
|
+
await withScopedDeadline(ctx, (cause) => validationError(`The scoped text search ran past ${SCOPED_SEARCH_DEADLINE_MS / 1000} s reading and matching the readable notes.`, { reason: 'text_search_too_costly' }, { cause }), async (signal) => {
|
|
564
|
+
const notes = await this.#textCandidates(ctx, search.tokens, signal);
|
|
565
|
+
// With no token every note matches with no span, and no body can change that.
|
|
566
|
+
if (search.tokens.length === 0)
|
|
567
|
+
for (const note of notes)
|
|
568
|
+
search.add(note.path, '');
|
|
569
|
+
else {
|
|
570
|
+
for await (const batch of this.#bodyBatches(ctx, notes, signal)) {
|
|
571
|
+
for (const note of batch)
|
|
572
|
+
search.add(note.path, note.content);
|
|
573
|
+
}
|
|
574
|
+
}
|
|
575
|
+
});
|
|
576
|
+
return search.hits().map((h) => ({
|
|
577
|
+
filename: h.filename,
|
|
578
|
+
matches: mergeIntoLocations(h.matches.map(({ context, contextStart, match }) => ({
|
|
579
|
+
context,
|
|
580
|
+
match: {
|
|
581
|
+
start: match.start,
|
|
582
|
+
end: match.end,
|
|
583
|
+
contextStart,
|
|
584
|
+
contextEnd: contextStart + match.end - match.start,
|
|
585
|
+
},
|
|
586
|
+
subject: match.source === 'filename' ? 'filename' : 'body',
|
|
587
|
+
})), search.tokens.length, contextLength),
|
|
588
|
+
}));
|
|
589
|
+
}
|
|
590
|
+
/**
|
|
591
|
+
* The readable notes that may hold every token of a scoped text search, in
|
|
592
|
+
* vault order, from one upstream search: the read-scope guard
|
|
593
|
+
* (`PathPolicy.readScopePattern`) and, per token, a case-folded `regexp`
|
|
594
|
+
* (`caseFoldedPattern`) on `path` or `content`, the two places the plugin's
|
|
595
|
+
* search looks. A token with a character the folding cannot express, or a
|
|
596
|
+
* line break that could span the basename and the body the plugin joins,
|
|
597
|
+
* sends no `regexp`, so the notes returned always include every readable
|
|
598
|
+
* match; `filterReadable` decides what is kept. With no `regexp` at all the
|
|
599
|
+
* tree never names `content`, so the plugin reads no bodies.
|
|
600
|
+
*/
|
|
601
|
+
async #textCandidates(ctx, tokens, signal) {
|
|
602
|
+
const clauses = [{ regexp: [this.#policy.readScopePattern(), { var: 'path' }] }];
|
|
603
|
+
for (const token of tokens) {
|
|
604
|
+
const pattern = token.includes('\n') ? undefined : caseFoldedPattern(token);
|
|
605
|
+
if (pattern === undefined)
|
|
606
|
+
continue;
|
|
607
|
+
clauses.push({
|
|
608
|
+
or: [{ regexp: [pattern, { var: 'path' }] }, { regexp: [pattern, { var: 'content' }] }],
|
|
609
|
+
});
|
|
610
|
+
}
|
|
611
|
+
const guard = { if: [{ and: clauses }, { var: 'stat' }, null] };
|
|
612
|
+
const res = await this.#postJsonLogic(ctx, guard, signal);
|
|
613
|
+
const hits = (await res.json());
|
|
614
|
+
return this.#policy
|
|
615
|
+
.filterReadable(hits)
|
|
616
|
+
.map(({ filename, result }) => ({ path: filename, stat: result }));
|
|
617
|
+
}
|
|
485
618
|
/**
|
|
486
619
|
* JSONLogic search. Unscoped, the tree goes to the plugin unchanged. Under
|
|
487
|
-
* OBSIDIAN_READ_PATHS
|
|
488
|
-
*
|
|
620
|
+
* OBSIDIAN_READ_PATHS or OBSIDIAN_DENY_PATHS (`restrictsReads`) the plugin
|
|
621
|
+
* would evaluate it against every note in the vault, denied ones included,
|
|
622
|
+
* so the tree stays here instead: the readable notes' contexts
|
|
489
623
|
* are fetched and the tree is evaluated over those alone
|
|
490
624
|
* (`createJsonLogicEvaluation`), the way the plugin evaluates it. No hit,
|
|
491
|
-
* result, or error then reads a note outside the read scope
|
|
492
|
-
*
|
|
493
|
-
*
|
|
494
|
-
*
|
|
625
|
+
* result, or error then reads a note outside the read scope. The contexts
|
|
626
|
+
* carry no `links`, `backlinks`, or `unresolvedLinks`, since which note a
|
|
627
|
+
* link resolves to depends on notes outside the scope; a tree that names
|
|
628
|
+
* one fails before any request (`assertNoLinkFields`).
|
|
495
629
|
*
|
|
496
630
|
* The plugin hands a tree note bodies only when its JSON holds the string
|
|
497
|
-
* `"content"` (`readsContent`); such a tree gets them here too,
|
|
498
|
-
*
|
|
499
|
-
* scope never holds every readable body at once.
|
|
631
|
+
* `"content"` (`readsContent`); such a tree gets them here too, one read
|
|
632
|
+
* per readable note, evaluated in batches of at most `CONTENT_BATCH_BYTES`
|
|
633
|
+
* so a broad scope never holds every readable body at once. Every request
|
|
634
|
+
* the search makes shares `ctx.signal` and one deadline,
|
|
635
|
+
* `SCOPED_SEARCH_DEADLINE_MS`; past it the search fails with
|
|
636
|
+
* `logic_too_costly`.
|
|
500
637
|
*/
|
|
501
638
|
async searchJsonLogic(ctx, logic) {
|
|
502
|
-
if (this.#policy.restrictsReads) {
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
const notes = await this.#readableNoteContexts(ctx);
|
|
506
|
-
if (!readsContent(logic))
|
|
507
|
-
return evaluation.evaluate(notes);
|
|
508
|
-
const hits = [];
|
|
509
|
-
for (const batch of contentBatches(notes)) {
|
|
510
|
-
const bodies = await this.#noteBodies(ctx, batch);
|
|
511
|
-
const withBodies = batch.map((note) => ({ ...note, content: bodies.get(note.path) ?? '' }));
|
|
512
|
-
hits.push(...(await evaluation.evaluate(withBodies)));
|
|
513
|
-
}
|
|
514
|
-
return hits;
|
|
639
|
+
if (!this.#policy.restrictsReads) {
|
|
640
|
+
const res = await this.#postJsonLogic(ctx, logic);
|
|
641
|
+
return (await res.json());
|
|
515
642
|
}
|
|
516
|
-
|
|
517
|
-
|
|
643
|
+
assertTreeDepth(logic);
|
|
644
|
+
assertNoLinkFields(logic);
|
|
645
|
+
return await withScopedDeadline(ctx, (cause) => validationError(`The scoped JSONLogic search ran past ${SCOPED_SEARCH_DEADLINE_MS / 1000} s reading and evaluating the readable notes.`, {
|
|
646
|
+
reason: 'logic_too_costly',
|
|
647
|
+
recovery: {
|
|
648
|
+
hint: 'A tree that names `content` reads the body of every readable note. Test `path`, `tags`, `frontmatter`, or `stat` instead where they can answer, or ask the operator to narrow OBSIDIAN_READ_PATHS or extend OBSIDIAN_DENY_PATHS.',
|
|
649
|
+
},
|
|
650
|
+
}, { cause }), async (signal) => {
|
|
651
|
+
const evaluation = createJsonLogicEvaluation(logic, { signal });
|
|
652
|
+
try {
|
|
653
|
+
const notes = await this.#readableNoteContexts(ctx, signal);
|
|
654
|
+
if (!readsContent(logic))
|
|
655
|
+
return await evaluation.evaluate(notes);
|
|
656
|
+
const hits = [];
|
|
657
|
+
for await (const batch of this.#bodyBatches(ctx, notes, signal)) {
|
|
658
|
+
hits.push(...(await evaluation.evaluate(batch)));
|
|
659
|
+
}
|
|
660
|
+
return hits;
|
|
661
|
+
}
|
|
662
|
+
finally {
|
|
663
|
+
await evaluation.close();
|
|
664
|
+
}
|
|
665
|
+
});
|
|
518
666
|
}
|
|
519
667
|
/**
|
|
520
668
|
* A JSONLogic search for a caller paging through its hits `pageSize` at a
|
|
521
|
-
* time. Unscoped, it is `searchJsonLogic`. Under
|
|
669
|
+
* time. Unscoped, it is `searchJsonLogic`. Under a read scope a call
|
|
522
670
|
* without `walk` evaluates fresh and, when the hits run past one page,
|
|
523
671
|
* stores the full ordered hit list under a new walk id, returned as `walk`;
|
|
524
672
|
* a call naming that id with the same tree is served from the stored list
|
|
@@ -568,11 +716,12 @@ export class ObsidianService {
|
|
|
568
716
|
return id;
|
|
569
717
|
}
|
|
570
718
|
/** One upstream JSONLogic search: `tree` posted to `/search/`. */
|
|
571
|
-
#postJsonLogic(ctx, tree) {
|
|
719
|
+
#postJsonLogic(ctx, tree, signal = ctx.signal) {
|
|
572
720
|
return this.#request(ctx, '/search/', {
|
|
573
721
|
method: 'POST',
|
|
574
722
|
headers: { 'Content-Type': JSONLOGIC_CT },
|
|
575
723
|
body: JSON.stringify(tree),
|
|
724
|
+
signal,
|
|
576
725
|
});
|
|
577
726
|
}
|
|
578
727
|
/**
|
|
@@ -580,14 +729,14 @@ export class ObsidianService {
|
|
|
580
729
|
* leaves them, in vault order, from one upstream search whose tree is the
|
|
581
730
|
* server's own: a `regexp` guard on the path (`PathPolicy.readScopePattern`)
|
|
582
731
|
* narrows what the plugin returns, and `filterReadable` decides what is
|
|
583
|
-
* kept. Each context carries only the fields the plugin documents,
|
|
584
|
-
* fields
|
|
732
|
+
* kept. Each context carries only the fields the plugin documents, less the
|
|
733
|
+
* link fields (`RawNoteContextHit`).
|
|
585
734
|
*/
|
|
586
|
-
async #readableNoteContexts(ctx) {
|
|
735
|
+
async #readableNoteContexts(ctx, signal = ctx.signal) {
|
|
587
736
|
const guard = {
|
|
588
737
|
if: [{ regexp: [this.#policy.readScopePattern(), { var: 'path' }] }, { var: '' }, null],
|
|
589
738
|
};
|
|
590
|
-
const res = await this.#postJsonLogic(ctx, guard);
|
|
739
|
+
const res = await this.#postJsonLogic(ctx, guard, signal);
|
|
591
740
|
const hits = (await res.json());
|
|
592
741
|
return this.#policy.filterReadable(hits).map(({ filename, result }) => ({
|
|
593
742
|
tags: result.tags,
|
|
@@ -595,27 +744,60 @@ export class ObsidianService {
|
|
|
595
744
|
stat: result.stat,
|
|
596
745
|
path: filename,
|
|
597
746
|
content: result.content,
|
|
598
|
-
links: result.links && this.#policy.keepReadable(result.links),
|
|
599
|
-
backlinks: result.backlinks && this.#policy.keepReadable(result.backlinks),
|
|
600
|
-
unresolvedLinks: result.unresolvedLinks && this.#policy.keepReadable(result.unresolvedLinks),
|
|
601
747
|
}));
|
|
602
748
|
}
|
|
603
749
|
/**
|
|
604
|
-
*
|
|
605
|
-
*
|
|
606
|
-
*
|
|
750
|
+
* The bodies of `notes` — readable notes, already gated — in their order,
|
|
751
|
+
* as runs whose `stat.size` totals at most `CONTENT_BATCH_BYTES` (a larger
|
|
752
|
+
* note gets a run of its own). Each body is one `GET /vault/{path}`, which
|
|
753
|
+
* reads that file alone, where a JSONLogic search naming `content` would
|
|
754
|
+
* read every vault note's body. Reads wait in the service's queue, at most
|
|
755
|
+
* `BODY_READ_CONCURRENCY` in flight. A run is read only when the caller
|
|
756
|
+
* asks for the next one, so one run is held at a time. A note gone by the
|
|
757
|
+
* time it is read has an empty body, as the plugin's search gives it; any
|
|
758
|
+
* other failure stops the run's remaining reads, waits for the ones in
|
|
759
|
+
* flight to settle, and is thrown. Every request carries `signal`.
|
|
607
760
|
*/
|
|
608
|
-
async
|
|
609
|
-
const
|
|
610
|
-
|
|
611
|
-
|
|
612
|
-
|
|
613
|
-
|
|
614
|
-
|
|
615
|
-
|
|
616
|
-
|
|
761
|
+
async *#bodyBatches(ctx, notes, signal) {
|
|
762
|
+
for (const run of contentBatches(notes)) {
|
|
763
|
+
const stop = new AbortController();
|
|
764
|
+
const reads = AbortSignal.any([signal, stop.signal]);
|
|
765
|
+
const queue = run.entries();
|
|
766
|
+
const withBodies = [];
|
|
767
|
+
let failure;
|
|
768
|
+
const lane = async () => {
|
|
769
|
+
while (failure === undefined) {
|
|
770
|
+
const next = queue.next();
|
|
771
|
+
if (next.done)
|
|
772
|
+
return;
|
|
773
|
+
const [index, note] = next.value;
|
|
774
|
+
try {
|
|
775
|
+
const content = await this.#bodyReads.run((granted) => this.#noteBody(ctx, note.path, granted), { signal: reads });
|
|
776
|
+
withBodies[index] = { ...note, content };
|
|
777
|
+
}
|
|
778
|
+
catch (error) {
|
|
779
|
+
failure ??= { error };
|
|
780
|
+
stop.abort();
|
|
781
|
+
}
|
|
782
|
+
}
|
|
783
|
+
};
|
|
784
|
+
await Promise.all(Array.from({ length: Math.min(BODY_READ_CONCURRENCY, run.length) }, lane));
|
|
785
|
+
if (failure)
|
|
786
|
+
throw failure.error;
|
|
787
|
+
yield withBodies;
|
|
617
788
|
}
|
|
618
|
-
|
|
789
|
+
}
|
|
790
|
+
/** One note's body, read raw as UTF-8 the way the plugin's search reads it; `""` once the note is gone. */
|
|
791
|
+
async #noteBody(ctx, path, signal) {
|
|
792
|
+
const res = await this.#request(ctx, `/vault/${encodeVaultPath(path)}`, {
|
|
793
|
+
method: 'GET',
|
|
794
|
+
headers: { Accept: 'text/markdown' },
|
|
795
|
+
noteRead: true,
|
|
796
|
+
missingOk: true,
|
|
797
|
+
signal,
|
|
798
|
+
});
|
|
799
|
+
const body = await res.text();
|
|
800
|
+
return res.status === 404 ? '' : body;
|
|
619
801
|
}
|
|
620
802
|
/**
|
|
621
803
|
* One-shot startup probe for the Omnisearch plugin's HTTP endpoint. Returns
|
|
@@ -1075,19 +1257,20 @@ export class ObsidianService {
|
|
|
1075
1257
|
* on the spot, while an unreachable plugin comes back as a transient
|
|
1076
1258
|
* `ServiceUnavailable` and keeps its retries (issues #133 and #136). A
|
|
1077
1259
|
* rejection that is neither passes through untouched, and so does anything
|
|
1078
|
-
* thrown after the
|
|
1260
|
+
* thrown after the request's signal aborted — `ctx.signal`, or the signal of
|
|
1261
|
+
* the search it belongs to — since a cancellation is not a network fault.
|
|
1079
1262
|
*/
|
|
1080
|
-
async #send(ctx, url, init) {
|
|
1263
|
+
async #send(ctx, url, init, signal = ctx.signal) {
|
|
1081
1264
|
try {
|
|
1082
1265
|
return await this.#fetch(url, {
|
|
1083
1266
|
...init,
|
|
1084
1267
|
dispatcher: this.#dispatcher,
|
|
1085
1268
|
...this.#tlsInit(url),
|
|
1086
|
-
signal
|
|
1269
|
+
signal,
|
|
1087
1270
|
});
|
|
1088
1271
|
}
|
|
1089
1272
|
catch (err) {
|
|
1090
|
-
if (
|
|
1273
|
+
if (signal.aborted)
|
|
1091
1274
|
throw err;
|
|
1092
1275
|
throw classifyFetchRejection(err) ?? err;
|
|
1093
1276
|
}
|
|
@@ -1098,12 +1281,15 @@ export class ObsidianService {
|
|
|
1098
1281
|
...(init.headers ?? {}),
|
|
1099
1282
|
Authorization: `Bearer ${this.#config.apiKey}`,
|
|
1100
1283
|
};
|
|
1284
|
+
const signal = init.signal ?? ctx.signal;
|
|
1101
1285
|
const exec = async () => {
|
|
1102
1286
|
const res = await this.#send(ctx, url, {
|
|
1103
1287
|
method: init.method,
|
|
1104
1288
|
headers,
|
|
1105
1289
|
...(init.body !== undefined ? { body: init.body } : {}),
|
|
1106
|
-
});
|
|
1290
|
+
}, signal);
|
|
1291
|
+
if (res.status === 404 && init.missingOk)
|
|
1292
|
+
return res;
|
|
1107
1293
|
if (!res.ok) {
|
|
1108
1294
|
await this.#throwForStatus(res, pathAndQuery, ctx, {
|
|
1109
1295
|
method: init.method,
|
|
@@ -1133,7 +1319,7 @@ export class ObsidianService {
|
|
|
1133
1319
|
},
|
|
1134
1320
|
baseDelayMs: 200,
|
|
1135
1321
|
maxRetries: 3,
|
|
1136
|
-
signal
|
|
1322
|
+
signal,
|
|
1137
1323
|
});
|
|
1138
1324
|
}
|
|
1139
1325
|
/**
|
|
@@ -1478,14 +1664,29 @@ function countTagsPerNote(notes) {
|
|
|
1478
1664
|
}
|
|
1479
1665
|
return Array.from(counts, ([name, count]) => ({ name, count }));
|
|
1480
1666
|
}
|
|
1667
|
+
/**
|
|
1668
|
+
* `path` less its leading and trailing `/` runs, as `/^\/+|\/+$/g` leaves it,
|
|
1669
|
+
* found by scanning in from each end: the regex retries at every `/` of a run
|
|
1670
|
+
* inside the path, in time quadratic in the run.
|
|
1671
|
+
*/
|
|
1672
|
+
export function trimSlashes(path) {
|
|
1673
|
+
let start = 0;
|
|
1674
|
+
let end = path.length;
|
|
1675
|
+
while (start < end && path[start] === '/')
|
|
1676
|
+
start++;
|
|
1677
|
+
while (end > start && path[end - 1] === '/')
|
|
1678
|
+
end--;
|
|
1679
|
+
return path.slice(start, end);
|
|
1680
|
+
}
|
|
1481
1681
|
/**
|
|
1482
1682
|
* Encode a vault-relative path for the URL. Splits on `/` and `\` (so
|
|
1483
1683
|
* Windows-style separators are honored), URL-encodes each segment, and
|
|
1484
1684
|
* rejoins with `/` since the Local REST API plugin expects forward slashes.
|
|
1485
1685
|
*
|
|
1486
1686
|
* Rejects `.` and `..` segments here rather than relying on the upstream Local
|
|
1487
|
-
* REST API plugin to normalize them — `
|
|
1488
|
-
*
|
|
1687
|
+
* REST API plugin to normalize them — with `OBSIDIAN_READ_PATHS` unset,
|
|
1688
|
+
* `PathPolicy` allows every path no deny prefix covers, and `..` is
|
|
1689
|
+
* unreserved per RFC 3986 so
|
|
1489
1690
|
* `encodeURIComponent` leaves it intact. This is the single chokepoint before
|
|
1490
1691
|
* URL construction, so guard vault escape here. Backslash is treated as a
|
|
1491
1692
|
* separator so `..\..\etc` traverses identically to `../../etc` and can't
|
|
@@ -1512,15 +1713,6 @@ export function encodeVaultPath(path) {
|
|
|
1512
1713
|
}
|
|
1513
1714
|
return segments.map((seg) => encodeURIComponent(seg)).join('/');
|
|
1514
1715
|
}
|
|
1515
|
-
/**
|
|
1516
|
-
* The query as Obsidian's `prepareSimpleSearch` reads it: split on whitespace,
|
|
1517
|
-
* case-folded, each distinct token required. Quotes are ordinary characters.
|
|
1518
|
-
*/
|
|
1519
|
-
function queryTokens(query) {
|
|
1520
|
-
return [...new Set(query.toLowerCase().split(/\s+/).filter(Boolean))];
|
|
1521
|
-
}
|
|
1522
|
-
const isHighSurrogate = (cu) => cu >= 0xd800 && cu <= 0xdbff;
|
|
1523
|
-
const isLowSurrogate = (cu) => cu >= 0xdc00 && cu <= 0xdfff;
|
|
1524
1716
|
/**
|
|
1525
1717
|
* Locate the match span inside the `context` window the upstream ships with
|
|
1526
1718
|
* it, and name the subject its `start`/`end` index.
|
|
@@ -1600,9 +1792,10 @@ function bodyWindowOffset(m, contextLength, reproduces) {
|
|
|
1600
1792
|
return !reproduces(base) && reproduces(base + 1) ? base + 1 : base;
|
|
1601
1793
|
}
|
|
1602
1794
|
/**
|
|
1603
|
-
* Merge a hit's spans into match locations
|
|
1604
|
-
*
|
|
1605
|
-
* of a phrase arrives as its own span with a
|
|
1795
|
+
* Merge a hit's spans into the match locations `TextSearchHit` carries.
|
|
1796
|
+
* `prepareSimpleSearch` coalesces only token occurrences that overlap or
|
|
1797
|
+
* touch, so each word of a phrase arrives as its own span with a
|
|
1798
|
+
* near-identical window.
|
|
1606
1799
|
*
|
|
1607
1800
|
* Walking the spans in upstream order, a location takes the next span when all
|
|
1608
1801
|
* of these hold:
|
|
@@ -1624,7 +1817,7 @@ function bodyWindowOffset(m, contextLength, reproduces) {
|
|
|
1624
1817
|
*/
|
|
1625
1818
|
function mergeIntoLocations(spans, distinctTokens, contextLength) {
|
|
1626
1819
|
if (distinctTokens < 2)
|
|
1627
|
-
return spans;
|
|
1820
|
+
return spans.map(({ context, match }) => ({ context, match }));
|
|
1628
1821
|
const locations = [];
|
|
1629
1822
|
let current;
|
|
1630
1823
|
for (const span of spans) {
|
|
@@ -1643,7 +1836,7 @@ function mergeIntoLocations(spans, distinctTokens, contextLength) {
|
|
|
1643
1836
|
}
|
|
1644
1837
|
if (current)
|
|
1645
1838
|
locations.push(current.location);
|
|
1646
|
-
return locations;
|
|
1839
|
+
return locations.map(({ context, match }) => ({ context, match }));
|
|
1647
1840
|
}
|
|
1648
1841
|
function matchedText(span) {
|
|
1649
1842
|
return span.context.slice(span.match.contextStart, span.match.contextEnd).toLowerCase();
|
|
@@ -1932,12 +2125,13 @@ function parseContentLength(res, url) {
|
|
|
1932
2125
|
/**
|
|
1933
2126
|
* Resolve the Omnisearch URL. Override wins. Otherwise: take the host from
|
|
1934
2127
|
* `OBSIDIAN_BASE_URL`, force `http:` (Omnisearch is HTTP-only), swap port
|
|
1935
|
-
* `27123/27124` → `51361`. `127.0.0.1` is mapped to `localhost`
|
|
1936
|
-
* Omnisearch
|
|
1937
|
-
*
|
|
1938
|
-
*
|
|
1939
|
-
*
|
|
1940
|
-
*
|
|
2128
|
+
* `27123/27124` → `51361`. `127.0.0.1` is mapped to `localhost` because
|
|
2129
|
+
* Omnisearch (1.31.0) listens on host `localhost` by default, which binds the
|
|
2130
|
+
* one address `localhost` resolves to first — `::1` on a macOS host, so a
|
|
2131
|
+
* literal `127.0.0.1` would be refused there — and requesting `localhost`
|
|
2132
|
+
* reaches whichever address that is. Falls back to `http://localhost:51361`
|
|
2133
|
+
* on any URL parse failure (config validation catches malformed `baseUrl`, so
|
|
2134
|
+
* this is belt-and-suspenders).
|
|
1941
2135
|
*/
|
|
1942
2136
|
function deriveOmnisearchUrl(config) {
|
|
1943
2137
|
if (config.omnisearchUrl)
|