@jacklika/dsh-tool-memory-filesystem 0.1.3-alpha.1 → 0.1.3-alpha.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +12 -11
- package/README.zh.md +12 -11
- package/lib/index.js +464 -65
- package/lib/search.js +150 -0
- package/lib/types/index.d.ts +47 -7
- package/lib/types/search.d.ts +41 -0
- package/lib/types/types.d.ts +11 -0
- package/package.json +5 -4
package/README.md
CHANGED
|
@@ -45,27 +45,27 @@ A relative `vaultRoot` resolves against the calling session's workspace.
|
|
|
45
45
|
| Field | Type | Default | Description |
|
|
46
46
|
|---|---|---|---|
|
|
47
47
|
| `vaultRoot` | `string` | `''` → `<session cwd>/.dsh/memory/` | Vault root. Empty selects the per-workspace memory directory; a relative path resolves against the session workspace. |
|
|
48
|
-
| `extensions` | `string[]` | `['.md']` | File extensions treated as notes. |
|
|
48
|
+
| `extensions` | `string[]` | `['.md']` | File extensions treated as notes. Must list at least one extension; an empty list fails plugin load. |
|
|
49
49
|
| `maxLinkDepth` | `number` | `1` | Maximum `[[link]]` hops `wiki_read` resolves. |
|
|
50
50
|
| `maxSearchResults` | `number` | `20` | Maximum `wiki_search` hits. |
|
|
51
51
|
| `indexHiddenDirs` | `boolean` | `false` | Index directories whose names start with `.` (`.git` and `node_modules` stay excluded). Enable when `vaultRoot` points at the workspace root so `.dsh/memory/` notes are included. |
|
|
52
52
|
|
|
53
53
|
### Tools
|
|
54
54
|
|
|
55
|
-
- `wiki_read(id)` — read one note by vault-relative path and return its frontmatter, body, links, linked notes,
|
|
56
|
-
- `wiki_search(query)` — keyword search across note titles, ids, and bodies;
|
|
57
|
-
- `wiki_write(id, content, mode?, baseVersion?)` — create or append to a note. Append mode preserves frontmatter and adds a timestamp header. Passing a `version` from `wiki_read` as `baseVersion` makes the write fail loudly when another writer changed the note in between.
|
|
55
|
+
- `wiki_read(id)` — read one note by vault-relative path and return its frontmatter, body, links, linked notes, a `version` content fingerprint, the file's `mtime` as a `+08:00` timestamp, and a `modifiedExternally` flag that is `true` when the file changed on disk since this plugin last observed it (e.g. an Obsidian edit). The plugin keeps a bounded 256-entry LRU of observed mtimes keyed by absolute path; its own `wiki_write` re-records the mtime so plugin writes never flag. Failures come back as structured errors that name the vault-relative id (`cannot read note "x.md": no such note`) and never embed host paths.
|
|
56
|
+
- `wiki_search(query)` — keyword search across note titles, ids, and bodies; query terms are OR-matched and ranked by field-weighted BM25-style scoring (title/id hits weigh most, rare terms weigh more) plus a verbatim-phrase bonus and a link-graph boost; when `wiki_semantic_search` is mounted the two rankings fuse via reciprocal rank fusion. Returns a `{ hits, total, truncated }` page: each hit carries the note id, title, score, and backlinks; `total` counts every match *before* the `maxSearchResults` cap; `truncated` is `true` when that cap cut the list, so a caller can tell a complete result set from a partial one instead of assuming the vault holds nothing else. Entries that cannot be read (deleted mid-scan, permission denied) are skipped and logged rather than failing the call.
|
|
57
|
+
- `wiki_write(id, content, mode?, baseVersion?)` — create or append to a note. Append mode preserves frontmatter and adds a timestamp header; both write modes normalize `created`/`updated` frontmatter values to `+08:00` second precision (unparseable values pass through). Passing a `version` from `wiki_read` as `baseVersion` makes the write fail loudly when another writer changed the note in between. The result is `{ id, mode, bytes, version }`, where `version` fingerprints the bytes just published — feed it straight back as `baseVersion` on the next write to chain optimistic-concurrency checks without a `wiki_read` round-trip. Writing through an in-vault symlink updates the linked note and keeps the link (a broken link is refused), and atomic replacement carries the note's existing permission bits to the new file.
|
|
58
58
|
|
|
59
59
|
### Security
|
|
60
60
|
|
|
61
|
-
All paths are resolved under the vault root for that call; a path that escapes the vault is rejected. The plugin does not use `ctx.fs`, so the configured filesystem sandbox policy does not apply; vault access is governed by OS permissions and this containment check.
|
|
61
|
+
All paths are resolved under the vault root for that call; a path that escapes the vault is rejected. In-vault symlinks are followed to their real target, which is re-checked for containment, so a symlinked note is appended through rather than replaced; a broken symlink is refused. Error messages stay vault-relative and never embed host paths. The plugin does not use `ctx.fs`, so the configured filesystem sandbox policy does not apply; vault access is governed by OS permissions and this containment check.
|
|
62
62
|
|
|
63
63
|
-----
|
|
64
64
|
|
|
65
65
|
<a id="understand-the-implementation"></a>
|
|
66
66
|
## Understand the implementation
|
|
67
67
|
|
|
68
|
-
This package is a single Cordis function plugin with no runtime service. It registers three typed tools on `ctx.tools`. The vault root is resolved per tool call: the explicit `vaultRoot` config when set (relative paths anchor at the session workspace), otherwise `<session cwd>/.dsh/memory/` so each workspace owns its notes. Each tool reads Markdown files directly through `node:fs/promises` and stays inside the resolved root via `path.resolve` + prefix checking. YAML frontmatter is parsed with `js-yaml
|
|
68
|
+
This package is a single Cordis function plugin with no runtime service. It registers three typed tools on `ctx.tools`. The vault root is resolved per tool call: the explicit `vaultRoot` config when set (relative paths anchor at the session workspace), otherwise `<session cwd>/.dsh/memory/` so each workspace owns its notes. Each tool reads Markdown files directly through `node:fs/promises` and stays inside the resolved root via `path.resolve` + prefix checking plus a realpath containment re-check. YAML frontmatter is parsed with `js-yaml` (a leading UTF-8 BOM before the opening `---` is tolerated, and duplicate keys resolve last-one-wins instead of dropping the block); `[[link|alias]]` references extract the target before the pipe; `[[...]]` inside inline code spans or fenced code blocks is documentation, not a link, and is ignored. Link targets resolve Obsidian-style against the indexed note ids: an exact vault-relative path first, then a path-suffix match (`[[notes/foo]]` finds `shared/notes/foo.md`), then a basename match anywhere in the vault — each tier tolerates the target carrying or omitting the extension. Ambiguous matches pick the fewest path segments, then the lowest id in code-point order, so resolution is deterministic across platforms. Search builds a transient index from the vault contents and ranks it in layers: OR-matched field-weighted scoring (id/title ×3, backlinks ×1, body ×1, with IDF so rare terms weigh more and BM25 saturation plus length normalization so long notes do not dominate), a verbatim-phrase bonus, and a graph boost for notes linked from strong hits. When `wiki_semantic_search` is registered, its ranking is fused in via reciprocal rank fusion; semantic failures fall back to the lexical ranking. The transient index skips entries that cannot be read (each logged as a warning) so one bad file cannot fail a scan. `wiki_write` publishes atomically: the body is staged in a sibling temp file and `rename`d over the target, so concurrent readers never observe a partially written note; the staged file inherits the target's permission bits and is removed again if staging or rename fails.
|
|
69
69
|
|
|
70
70
|
-----
|
|
71
71
|
|
|
@@ -76,29 +76,29 @@ This package is a single Cordis function plugin with no runtime service. It regi
|
|
|
76
76
|
|
|
77
77
|
#### What the model sees
|
|
78
78
|
|
|
79
|
-
The model sees the generated
|
|
79
|
+
The model sees the generated `wiki_read`, `wiki_search`, and `wiki_write` schemas. Their descriptions tell the model that notes are Markdown files with YAML frontmatter and Obsidian-style `[[link]]` references, and that `wiki_read` follows links up to the configured depth.
|
|
80
80
|
|
|
81
81
|
##### Verbatim description for `wiki_read`
|
|
82
82
|
|
|
83
83
|
```markdown
|
|
84
|
-
Read one Markdown note from the wiki vault, optionally following Obsidian-style [[link]] references up to the configured depth. Returns the note id, frontmatter, body, and
|
|
84
|
+
Read one Markdown note from the wiki vault, optionally following Obsidian-style [[link]] references up to the configured depth. Returns the note id, frontmatter, body, linked notes, mtime, and a modifiedExternally flag that is true when the file changed on disk since this tool last observed it.
|
|
85
85
|
```
|
|
86
86
|
|
|
87
87
|
##### Verbatim description for `wiki_search`
|
|
88
88
|
|
|
89
89
|
```markdown
|
|
90
|
-
Search the wiki vault by note title or body keyword. Returns
|
|
90
|
+
Search the wiki vault by note title or body keyword. Terms are OR-matched; results are ranked by field-weighted relevance (title/id hits outrank body hits, rare terms weigh more, notes linked from strong hits get a boost, and results may be fused with semantic search when available). Returns { hits, total, truncated }: each hit carries the note id, title, score, and backlinks, total counts every match before the result cap, and truncated is true when that cap cut the list. Use this before asking the user which note to read.
|
|
91
91
|
```
|
|
92
92
|
|
|
93
93
|
##### Verbatim description for `wiki_write`
|
|
94
94
|
|
|
95
95
|
```markdown
|
|
96
|
-
Create a new note or append to an existing note in the wiki vault. The path is relative to the vault root. When appending, the new content is inserted at the end of the body after a timestamp header.
|
|
96
|
+
Create a new note or append to an existing note in the wiki vault. The path is relative to the vault root. When appending, the new content is inserted at the end of the body after a timestamp header. Returns { id, mode, bytes, version }, where version fingerprints the written content and can be passed as baseVersion on the next write without reading the note again.
|
|
97
97
|
```
|
|
98
98
|
|
|
99
99
|
#### Token effect
|
|
100
100
|
|
|
101
|
-
`wiki_read` returns the full body of the requested note plus every linked note reachable within `maxLinkDepth`. Long notes or dense link graphs can add many tokens to the next request. `wiki_search` returns a bounded list of result metadata only.
|
|
101
|
+
`wiki_read` returns the full body of the requested note plus every linked note reachable within `maxLinkDepth`. Long notes or dense link graphs can add many tokens to the next request. `wiki_search` returns a bounded list of result metadata only; its `total` reports matches beyond the cap without returning them.
|
|
102
102
|
|
|
103
103
|
#### KV Cache effect
|
|
104
104
|
|
|
@@ -112,6 +112,7 @@ Independent. The plugin only supplies tool results; it does not change the reque
|
|
|
112
112
|
- **No vector search** — search is keyword-only. A separate `tool-memory-vector` package could add embedding-based retrieval without changing this package.
|
|
113
113
|
- **No embedded image or binary support** — notes are treated as UTF-8 text. Attachments should remain in the attachment seam.
|
|
114
114
|
- **No built-in write coordination** — writes are atomic (temp file + `rename`), so readers never see partial files, but simultaneous `wiki_write` calls to the same note can still lose updates. Cooperative callers can pass `baseVersion` to turn a lost update into a loud conflict; mount `@jacklika/dsh-memory-queue` for in-process serialization, optionally with a heartbeat-refreshed cross-process lock.
|
|
115
|
+
- **Unreadable entries are skipped, not surfaced as data** — a note or directory the process cannot read is omitted from read, search, and linked-note results and reported only through a logger warning; the tool result itself does not list what was skipped.
|
|
115
116
|
|
|
116
117
|
<a id="dev-note"></a>
|
|
117
118
|
### Dev Note
|
package/README.zh.md
CHANGED
|
@@ -45,27 +45,27 @@ kind: "package-reference"
|
|
|
45
45
|
| 字段 | 类型 | 默认值 | 说明 |
|
|
46
46
|
|---|---|---|---|
|
|
47
47
|
| `vaultRoot` | `string` | `''` → `<session cwd>/.dsh/memory/` | 仓库根目录。留空表示使用按工作区隔离的记忆目录;相对路径相对于会话工作区解析。 |
|
|
48
|
-
| `extensions` | `string[]` | `['.md']` |
|
|
48
|
+
| `extensions` | `string[]` | `['.md']` | 视为笔记的文件扩展名。至少需列出一个扩展名;空列表会导致插件加载失败。 |
|
|
49
49
|
| `maxLinkDepth` | `number` | `1` | `wiki_read` 解析 `[[link]]` 的最大深度。 |
|
|
50
50
|
| `maxSearchResults` | `number` | `20` | `wiki_search` 返回的最大结果数。 |
|
|
51
51
|
| `indexHiddenDirs` | `boolean` | `false` | 是否索引以 `.` 开头的目录(`.git` 与 `node_modules` 始终排除)。当 `vaultRoot` 指向工作区根时开启,以纳入 `.dsh/memory/` 中的笔记。 |
|
|
52
52
|
|
|
53
53
|
### 工具
|
|
54
54
|
|
|
55
|
-
- `wiki_read(id)` — 按仓库相对路径读取笔记,返回 frontmatter
|
|
56
|
-
- `wiki_search(query)` — 按标题、id
|
|
57
|
-
- `wiki_write(id, content, mode?, baseVersion?)` — 创建或追加笔记。追加模式保留 frontmatter
|
|
55
|
+
- `wiki_read(id)` — 按仓库相对路径读取笔记,返回 frontmatter、正文、链接、已解析的链接笔记、内容指纹 `version`、`+08:00` 格式的文件 `mtime`,以及 `modifiedExternally` 标志(文件自本插件上次观察以来在磁盘上被改动时为 `true`,例如 Obsidian 编辑)。插件按绝对路径维护一个 256 条上限的 LRU mtime 表;自身的 `wiki_write` 会刷新记录,因此插件写入永不误报。失败会以结构化错误返回,错误信息只含仓库相对 id(如 `cannot read note "x.md": no such note`),绝不嵌入宿主绝对路径。
|
|
56
|
+
- `wiki_search(query)` — 按标题、id、正文关键词搜索;查询词采用 OR 匹配,并按字段加权的 BM25 式评分排序(标题/id 命中权重最高,稀有词权重更高),另有整段短语加分与被强命中笔记链接的图增强;挂载 `wiki_semantic_search` 时两路排序经 RRF 融合。返回 `{ hits, total, truncated }` 分页结构:每条命中含笔记 id、标题、分数与反向链接;`total` 是 `maxSearchResults` 截断**之前**的全部命中数;`truncated` 为 `true` 表示结果被该上限截断,调用方因此能区分“结果完整”与“还有更多”,而不会误以为仓库里只有这些。无法读取的条目(扫描途中被删除、权限不足)会被跳过并记入日志,而不是让整次调用失败。
|
|
57
|
+
- `wiki_write(id, content, mode?, baseVersion?)` — 创建或追加笔记。追加模式保留 frontmatter 并添加时间戳标题;两种写入模式都会把 `created`/`updated` frontmatter 值归一化为 `+08:00` 秒级精度(不可解析的值原样保留)。把 `wiki_read` 返回的 `version` 作为 `baseVersion` 传入时,若笔记在读取后被其他写入方改动,写入会显式失败。返回值为 `{ id, mode, bytes, version }`,其中 `version` 就是刚发布内容的指纹——可直接作为下一次写入的 `baseVersion`,无需再 `wiki_read` 一次即可串联乐观并发校验。通过仓库内 symlink 写入会更新链接指向的笔记并保留链接本身(损坏的链接会被拒绝);原子替换会把笔记原有的权限位带到新文件。
|
|
58
58
|
|
|
59
59
|
### 安全
|
|
60
60
|
|
|
61
|
-
|
|
61
|
+
所有路径都必须在该次调用解析出的仓库根目录下;越界路径会被拒绝。仓库内的 symlink 会先解析到真实目标并重新做包含校验,因此追加写入落在链接目标上而非替换链接;损坏的 symlink 会被拒绝。错误信息保持仓库相对路径,不嵌入宿主绝对路径。该插件不通过 `ctx.fs` 访问文件,因此不继承文件沙箱策略;访问权限由 OS 文件权限和本插件的越界检查共同控制。
|
|
62
62
|
|
|
63
63
|
-----
|
|
64
64
|
|
|
65
65
|
<a id="understand-the-implementation"></a>
|
|
66
66
|
## 实现说明
|
|
67
67
|
|
|
68
|
-
本包是一个没有运行时服务的 Cordis 函数插件,只在 `ctx.tools` 上注册三个类型化工具。仓库根目录在每次工具调用时解析:优先使用显式 `vaultRoot`(相对路径锚定会话工作区),否则使用 `<session cwd>/.dsh/memory/`,使每个工作区拥有自己的笔记。每个工具都通过 `node:fs/promises` 直接读取 Markdown 文件,并借助 `path.resolve`
|
|
68
|
+
本包是一个没有运行时服务的 Cordis 函数插件,只在 `ctx.tools` 上注册三个类型化工具。仓库根目录在每次工具调用时解析:优先使用显式 `vaultRoot`(相对路径锚定会话工作区),否则使用 `<session cwd>/.dsh/memory/`,使每个工作区拥有自己的笔记。每个工具都通过 `node:fs/promises` 直接读取 Markdown 文件,并借助 `path.resolve` 词法前缀检查加 realpath 包含复检保持在解析出的根目录内。YAML frontmatter 使用 `js-yaml` 解析(开头 `---` 前带 UTF-8 BOM 亦可识别;frontmatter 出现重复键时按“后者胜出”处理,而不是丢弃整个块);`[[link|alias]]` 形式的引用会提取管道符前的目标;出现在行内代码或围栏代码块中的 `[[...]]` 是语法示例而非链接,会被忽略。链接目标按 Obsidian 语义对已索引的笔记 id 解析:先精确匹配仓库相对路径,再做路径后缀匹配(`[[notes/foo]]` 可命中 `shared/notes/foo.md`),最后匹配仓库内任意位置的 basename——每一级都容忍目标带或不带扩展名。多重命中时优先路径段最少者,仍并列则取码点序最小的 id,因此解析结果跨平台确定。搜索时从仓库内容构建临时索引(无法读取的条目会被跳过并记入日志,单个坏文件不会使整次扫描失败)并分层排序:OR 匹配的字段加权评分(id/标题 ×3、反向链接 ×1、正文 ×1,含 IDF 使稀有词权重更高、BM25 饱和与长度归一化避免长笔记占优)、整段短语加分,以及被强命中笔记链接的图增强。挂载 `wiki_semantic_search` 时其排序经倒数排名融合并入;语义检索失败时回退到纯词法排序。`wiki_write` 以原子方式发布:正文先写入同级临时文件,再 `rename` 覆盖目标,因此并发读者永远不会读到写了一半的笔记;临时文件继承目标笔记的权限位,且写入或 rename 失败时会被清理。
|
|
69
69
|
|
|
70
70
|
-----
|
|
71
71
|
|
|
@@ -76,29 +76,29 @@ kind: "package-reference"
|
|
|
76
76
|
|
|
77
77
|
#### 模型看到的内容
|
|
78
78
|
|
|
79
|
-
模型看到的是生成的
|
|
79
|
+
模型看到的是生成的 `wiki_read`、`wiki_search` 和 `wiki_write` schema。工具描述会告知模型:笔记是带 YAML frontmatter 和 Obsidian 风格 `[[link]]` 链接的 Markdown 文件,`wiki_read` 会按配置深度跟随链接。
|
|
80
80
|
|
|
81
81
|
##### `wiki_read` 的完整描述
|
|
82
82
|
|
|
83
83
|
```markdown
|
|
84
|
-
Read one Markdown note from the wiki vault, optionally following Obsidian-style [[link]] references up to the configured depth. Returns the note id, frontmatter, body, and
|
|
84
|
+
Read one Markdown note from the wiki vault, optionally following Obsidian-style [[link]] references up to the configured depth. Returns the note id, frontmatter, body, linked notes, mtime, and a modifiedExternally flag that is true when the file changed on disk since this tool last observed it.
|
|
85
85
|
```
|
|
86
86
|
|
|
87
87
|
##### `wiki_search` 的完整描述
|
|
88
88
|
|
|
89
89
|
```markdown
|
|
90
|
-
Search the wiki vault by note title or body keyword. Returns
|
|
90
|
+
Search the wiki vault by note title or body keyword. Terms are OR-matched; results are ranked by field-weighted relevance (title/id hits outrank body hits, rare terms weigh more, notes linked from strong hits get a boost, and results may be fused with semantic search when available). Returns { hits, total, truncated }: each hit carries the note id, title, score, and backlinks, total counts every match before the result cap, and truncated is true when that cap cut the list. Use this before asking the user which note to read.
|
|
91
91
|
```
|
|
92
92
|
|
|
93
93
|
##### `wiki_write` 的完整描述
|
|
94
94
|
|
|
95
95
|
```markdown
|
|
96
|
-
Create a new note or append to an existing note in the wiki vault. The path is relative to the vault root. When appending, the new content is inserted at the end of the body after a timestamp header.
|
|
96
|
+
Create a new note or append to an existing note in the wiki vault. The path is relative to the vault root. When appending, the new content is inserted at the end of the body after a timestamp header. Returns { id, mode, bytes, version }, where version fingerprints the written content and can be passed as baseVersion on the next write without reading the note again.
|
|
97
97
|
```
|
|
98
98
|
|
|
99
99
|
#### Token 影响
|
|
100
100
|
|
|
101
|
-
`wiki_read` 返回请求笔记的完整正文,以及 `maxLinkDepth` 范围内所有可达链接笔记的正文。长笔记或密集链接图可能显著增加下次请求的 token 数。`wiki_search`
|
|
101
|
+
`wiki_read` 返回请求笔记的完整正文,以及 `maxLinkDepth` 范围内所有可达链接笔记的正文。长笔记或密集链接图可能显著增加下次请求的 token 数。`wiki_search` 只返回有限的结果元数据;其 `total` 会告知被上限截断而未返回的命中数量。
|
|
102
102
|
|
|
103
103
|
#### KV Cache 影响
|
|
104
104
|
|
|
@@ -112,6 +112,7 @@ Create a new note or append to an existing note in the wiki vault. The path is r
|
|
|
112
112
|
- **无向量搜索** — 仅支持关键词搜索。未来可新增 `tool-memory-vector` 包提供基于 embedding 的检索,而无需修改本包。
|
|
113
113
|
- **不支持图片或二进制附件** — 笔记按 UTF-8 文本处理。附件应继续使用 attachment 体系。
|
|
114
114
|
- **无内置写协调** —— 写入是原子的(临时文件 + `rename`),读者不会看到半写文件,但对同一笔记的并发 `wiki_write` 仍可能丢更新。协作调用方可传 `baseVersion` 把丢更新变成显式冲突;挂载 `@jacklika/dsh-memory-queue` 可获得进程内串行化,并可选用心跳刷新的跨进程锁。
|
|
115
|
+
- **无法读取的条目被跳过而非作为数据暴露** —— 进程读不到的笔记或目录会从读取、搜索与链接解析结果中消失,仅通过 logger 警告报告;工具返回值本身不会列出被跳过的条目。
|
|
115
116
|
|
|
116
117
|
<a id="dev-note"></a>
|
|
117
118
|
### 开发备注
|
package/lib/index.js
CHANGED
|
@@ -8,11 +8,19 @@
|
|
|
8
8
|
* @module @jacklika/dsh-tool-memory-filesystem
|
|
9
9
|
*/
|
|
10
10
|
import { createHash, randomUUID } from 'node:crypto';
|
|
11
|
-
import { readdir, readFile, realpath, rename, rm, stat, writeFile
|
|
11
|
+
import { chmod, lstat, mkdir, readdir, readFile, realpath, rename, rm, stat, writeFile } from 'node:fs/promises';
|
|
12
12
|
import { dirname, isAbsolute, join, relative, resolve, sep } from 'node:path';
|
|
13
13
|
import z from '@deepseek-ai/schemastery';
|
|
14
14
|
import { defineTool } from '@deepseek-ai/dsh-tools';
|
|
15
|
+
import { formatBeijingTime } from '@jacklika/dsh-memory-time';
|
|
15
16
|
import yaml from 'js-yaml';
|
|
17
|
+
import { codeUnitCompare, fuseRankings, rankNotes } from "./search.js";
|
|
18
|
+
/**
|
|
19
|
+
* Locale-independent note-id ordering, re-exported so sibling packages that
|
|
20
|
+
* truncate or tie-break id lists sort by the same rule instead of a copy that
|
|
21
|
+
* can drift.
|
|
22
|
+
*/
|
|
23
|
+
export { codeUnitCompare };
|
|
16
24
|
/**
|
|
17
25
|
* Compute a vault-relative note id from an absolute path. The result always
|
|
18
26
|
* uses POSIX separators so note ids are stable across Windows, macOS, and Linux.
|
|
@@ -56,11 +64,14 @@ export function resolveMemoryVaultRoot(vaultRoot, exec) {
|
|
|
56
64
|
* @returns the absolute, contained path.
|
|
57
65
|
*/
|
|
58
66
|
export function containedPath(root, candidate) {
|
|
67
|
+
if (candidate.includes('\0')) {
|
|
68
|
+
throw new Error('tool-memory-filesystem: note id must not contain a NUL byte');
|
|
69
|
+
}
|
|
59
70
|
const absoluteRoot = resolve(root);
|
|
60
71
|
const absolute = resolve(absoluteRoot, candidate);
|
|
61
72
|
const remainder = relative(absoluteRoot, absolute);
|
|
62
73
|
if (remainder === '..' || remainder.startsWith(`..${sep}`) || isAbsolute(remainder)) {
|
|
63
|
-
throw new Error(`tool-memory-filesystem: path ${candidate} is outside vault root
|
|
74
|
+
throw new Error(`tool-memory-filesystem: path ${JSON.stringify(candidate)} is outside the vault root`);
|
|
64
75
|
}
|
|
65
76
|
return absolute;
|
|
66
77
|
}
|
|
@@ -81,7 +92,7 @@ export async function containedPathReal(root, candidate) {
|
|
|
81
92
|
const canonicalAncestor = await realpath(ancestor);
|
|
82
93
|
const remainder = relative(canonicalRoot, canonicalAncestor);
|
|
83
94
|
if (remainder === '..' || remainder.startsWith(`..${sep}`) || isAbsolute(remainder)) {
|
|
84
|
-
throw new Error(`tool-memory-filesystem: path ${candidate} resolves outside vault root
|
|
95
|
+
throw new Error(`tool-memory-filesystem: path ${JSON.stringify(candidate)} resolves outside the vault root`);
|
|
85
96
|
}
|
|
86
97
|
return absolute;
|
|
87
98
|
}
|
|
@@ -97,25 +108,35 @@ export async function containedPathReal(root, candidate) {
|
|
|
97
108
|
}
|
|
98
109
|
/**
|
|
99
110
|
* Extract YAML frontmatter and body from Markdown text. Only the leading
|
|
100
|
-
* `---\n...\n---\n` form is recognized.
|
|
111
|
+
* `---\n...\n---\n` form is recognized, optionally behind a UTF-8 BOM.
|
|
112
|
+
* Duplicate keys resolve last-one-wins rather than dropping the block.
|
|
101
113
|
* @param text - raw file contents.
|
|
102
114
|
* @returns frontmatter map and body.
|
|
103
115
|
*/
|
|
104
116
|
export function splitFrontmatter(text) {
|
|
105
|
-
|
|
117
|
+
// A UTF-8 BOM is invisible but pushes the opening fence off the start of the
|
|
118
|
+
// string, which would silently drop the whole frontmatter block.
|
|
119
|
+
const source = text.startsWith('\uFEFF') ? text.slice(1) : text;
|
|
120
|
+
const match = /^---\r?\n([\s\S]*?)\r?\n---\r?\n([\s\S]*)$/.exec(source);
|
|
106
121
|
if (match === null) {
|
|
107
122
|
return { frontmatter: {}, body: text };
|
|
108
123
|
}
|
|
109
124
|
const frontmatterText = match[1] ?? '';
|
|
110
125
|
const bodyText = match[2] ?? '';
|
|
111
126
|
try {
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
127
|
+
// `json: true` makes duplicate keys last-one-wins instead of throwing;
|
|
128
|
+
// throwing falls into the catch below and drops every field.
|
|
129
|
+
const parsed = yaml.load(frontmatterText, { json: true });
|
|
130
|
+
const frontmatter = parsed !== null && typeof parsed === 'object' && !Array.isArray(parsed)
|
|
131
|
+
? parsed
|
|
132
|
+
: {};
|
|
133
|
+
// js-yaml coerces unquoted timestamps to Date; vault convention stores them
|
|
134
|
+
// as Asia/Shanghai +08:00 strings, so convert back before they reach callers.
|
|
135
|
+
for (const [key, value] of Object.entries(frontmatter)) {
|
|
136
|
+
if (value instanceof Date)
|
|
137
|
+
frontmatter[key] = formatBeijingTime(value);
|
|
138
|
+
}
|
|
139
|
+
return { frontmatter, body: bodyText };
|
|
119
140
|
}
|
|
120
141
|
catch {
|
|
121
142
|
return { frontmatter: {}, body: text };
|
|
@@ -130,6 +151,68 @@ export function splitFrontmatter(text) {
|
|
|
130
151
|
export function noteVersion(text) {
|
|
131
152
|
return createHash('sha1').update(text, 'utf8').digest('hex');
|
|
132
153
|
}
|
|
154
|
+
/**
|
|
155
|
+
* Last `mtimeMs` observed per note path, keyed by absolute path so identical
|
|
156
|
+
* note ids in different vaults never collide. Bounded at 256 entries with
|
|
157
|
+
* LRU eviction: an unbounded map would grow with every note ever read in a
|
|
158
|
+
* long-running process, while a fixed cap keeps the newest observations —
|
|
159
|
+
* the only ones a `modifiedExternally` comparison can still act on.
|
|
160
|
+
*/
|
|
161
|
+
const observedMtimes = new Map();
|
|
162
|
+
const OBSERVED_MTIME_LIMIT = 256;
|
|
163
|
+
/**
|
|
164
|
+
* Record the freshest observed `mtimeMs` for a path, evicting the oldest
|
|
165
|
+
* entry when the cap is reached.
|
|
166
|
+
*/
|
|
167
|
+
function recordMtime(path, mtimeMs) {
|
|
168
|
+
observedMtimes.delete(path);
|
|
169
|
+
observedMtimes.set(path, mtimeMs);
|
|
170
|
+
if (observedMtimes.size > OBSERVED_MTIME_LIMIT) {
|
|
171
|
+
const oldest = observedMtimes.keys().next().value;
|
|
172
|
+
if (oldest !== undefined)
|
|
173
|
+
observedMtimes.delete(oldest);
|
|
174
|
+
}
|
|
175
|
+
}
|
|
176
|
+
/**
|
|
177
|
+
* Whether the note changed on disk since this plugin last observed it.
|
|
178
|
+
* The first observation records and reports `false`; a successful
|
|
179
|
+
* `wiki_write` re-records so the plugin's own writes never flag.
|
|
180
|
+
*/
|
|
181
|
+
function detectExternalChange(path, mtimeMs) {
|
|
182
|
+
const seen = observedMtimes.get(path);
|
|
183
|
+
recordMtime(path, mtimeMs);
|
|
184
|
+
return seen !== undefined && seen !== mtimeMs;
|
|
185
|
+
}
|
|
186
|
+
/**
|
|
187
|
+
* Normalize `created`/`updated` frontmatter timestamps to the vault's
|
|
188
|
+
* `+08:00` second-precision convention. Parseable values in any other form
|
|
189
|
+
* (`Z`, `+00:00`, millisecond precision) are rewritten; unparseable values
|
|
190
|
+
* pass through untouched so a non-date string is never corrupted.
|
|
191
|
+
*/
|
|
192
|
+
export function normalizeNoteTimestamps(frontmatter) {
|
|
193
|
+
for (const key of ['created', 'updated']) {
|
|
194
|
+
const value = frontmatter[key];
|
|
195
|
+
if (typeof value !== 'string')
|
|
196
|
+
continue;
|
|
197
|
+
const parsed = new Date(value);
|
|
198
|
+
if (Number.isNaN(parsed.getTime()))
|
|
199
|
+
continue;
|
|
200
|
+
const normalized = formatBeijingTime(parsed);
|
|
201
|
+
if (normalized !== value)
|
|
202
|
+
frontmatter[key] = normalized;
|
|
203
|
+
}
|
|
204
|
+
}
|
|
205
|
+
/**
|
|
206
|
+
* Normalize timestamps inside a complete note text. Returns the text
|
|
207
|
+
* unchanged when there is no frontmatter to rewrite.
|
|
208
|
+
*/
|
|
209
|
+
function normalizeTextTimestamps(text) {
|
|
210
|
+
const { frontmatter, body } = splitFrontmatter(text);
|
|
211
|
+
if (Object.keys(frontmatter).length === 0)
|
|
212
|
+
return text;
|
|
213
|
+
normalizeNoteTimestamps(frontmatter);
|
|
214
|
+
return `---\n${yaml.dump(frontmatter).trim()}\n---\n${body}`;
|
|
215
|
+
}
|
|
133
216
|
/**
|
|
134
217
|
* Find all Obsidian-style `[[link]]` references in note text. Aliases of the
|
|
135
218
|
* form `[[link|alias]]` return the link target only.
|
|
@@ -140,12 +223,66 @@ export function extractLinks(text) {
|
|
|
140
223
|
const links = [];
|
|
141
224
|
const pattern = /\[\[([^|\]\r\n]+)(?:\|[^\]]*)?\]\]/g;
|
|
142
225
|
let match;
|
|
143
|
-
while ((match = pattern.exec(text)) !== null) {
|
|
226
|
+
while ((match = pattern.exec(stripCode(text))) !== null) {
|
|
144
227
|
if (match[1] !== undefined)
|
|
145
228
|
links.push(match[1].trim());
|
|
146
229
|
}
|
|
147
230
|
return [...new Set(links)];
|
|
148
231
|
}
|
|
232
|
+
/**
|
|
233
|
+
* Remove fenced code blocks and inline code spans so literal `[[...]]` inside
|
|
234
|
+
* them is not treated as a link. Fences follow CommonMark: a closing fence
|
|
235
|
+
* must use the same marker character and be at least as long as the opener —
|
|
236
|
+
* a rule a regex backreference cannot express, so fences are scanned line by
|
|
237
|
+
* line. Inline spans treat backtick runs as maximal units: a run closes only
|
|
238
|
+
* at a later run of exactly equal length; an unmatched run is literal text.
|
|
239
|
+
* @param text - note body.
|
|
240
|
+
* @returns the text with all code sections dropped.
|
|
241
|
+
*/
|
|
242
|
+
function stripCode(text) {
|
|
243
|
+
const kept = [];
|
|
244
|
+
let fenceChar = '';
|
|
245
|
+
let fenceLen = 0;
|
|
246
|
+
for (const line of text.split('\n')) {
|
|
247
|
+
if (fenceChar === '') {
|
|
248
|
+
const open = /^( {0,3})(`{3,}|~{3,})/.exec(line);
|
|
249
|
+
if (open?.[2] !== undefined) {
|
|
250
|
+
fenceChar = open[2][0] ?? '';
|
|
251
|
+
fenceLen = open[2].length;
|
|
252
|
+
continue;
|
|
253
|
+
}
|
|
254
|
+
kept.push(line);
|
|
255
|
+
}
|
|
256
|
+
else if (new RegExp(`^ {0,3}${fenceChar === '`' ? '`' : '~'}{${fenceLen},}[ \t]*$`).test(line)) {
|
|
257
|
+
fenceChar = '';
|
|
258
|
+
}
|
|
259
|
+
}
|
|
260
|
+
const body = kept.join('\n');
|
|
261
|
+
let out = '';
|
|
262
|
+
let i = 0;
|
|
263
|
+
while (i < body.length) {
|
|
264
|
+
if (body[i] !== '`') {
|
|
265
|
+
out += body[i];
|
|
266
|
+
i++;
|
|
267
|
+
continue;
|
|
268
|
+
}
|
|
269
|
+
let j = i;
|
|
270
|
+
while (body[j] === '`')
|
|
271
|
+
j++;
|
|
272
|
+
const run = body.slice(i, j);
|
|
273
|
+
let close = body.indexOf(run, j);
|
|
274
|
+
while (close !== -1 && (body[close - 1] === '`' || body[close + run.length] === '`')) {
|
|
275
|
+
close = body.indexOf(run, close + 1);
|
|
276
|
+
}
|
|
277
|
+
if (close === -1) {
|
|
278
|
+
out += run;
|
|
279
|
+
i = j;
|
|
280
|
+
continue;
|
|
281
|
+
}
|
|
282
|
+
i = close + run.length;
|
|
283
|
+
}
|
|
284
|
+
return out;
|
|
285
|
+
}
|
|
149
286
|
/**
|
|
150
287
|
* Normalize a configured file extension to a dot-prefixed form.
|
|
151
288
|
* @param ext - extension string.
|
|
@@ -155,17 +292,86 @@ export function dottedExtension(ext) {
|
|
|
155
292
|
return ext.startsWith('.') ? ext : `.${ext}`;
|
|
156
293
|
}
|
|
157
294
|
/**
|
|
158
|
-
*
|
|
159
|
-
*
|
|
295
|
+
* Build a resolver over the known note ids, matching Obsidian semantics: an
|
|
296
|
+
* exact vault-relative path first, then a path-suffix match, then a bare
|
|
297
|
+
* basename match — each tier tolerates the target carrying or omitting the
|
|
298
|
+
* extension. Ambiguity within a tier resolves to the id with the fewest path
|
|
299
|
+
* segments, ties by code-unit order, so results are identical across hosts.
|
|
300
|
+
* @param ids - vault-relative note ids (e.g. `shared/notes/foo.md`).
|
|
301
|
+
* @returns resolver function for `[[...]]` targets.
|
|
302
|
+
*/
|
|
303
|
+
export function createLinkResolver(ids) {
|
|
304
|
+
const entries = ids.map(id => {
|
|
305
|
+
const stem = id.replace(/\.[^./]+$/, '');
|
|
306
|
+
return {
|
|
307
|
+
id,
|
|
308
|
+
stem,
|
|
309
|
+
basenameId: id.slice(id.lastIndexOf('/') + 1),
|
|
310
|
+
basenameStem: stem.slice(stem.lastIndexOf('/') + 1),
|
|
311
|
+
segments: id.split('/').length,
|
|
312
|
+
};
|
|
313
|
+
});
|
|
314
|
+
// O(1) lookup tables keyed by every segment-boundary suffix of each id and
|
|
315
|
+
// stem: an exact hit is the full-string key, a path-suffix hit is a proper
|
|
316
|
+
// suffix, and a basename hit is the last segment.
|
|
317
|
+
const exact = new Map();
|
|
318
|
+
const suffix = new Map();
|
|
319
|
+
const basename = new Map();
|
|
320
|
+
const add = (table, key, entry) => {
|
|
321
|
+
const list = table.get(key);
|
|
322
|
+
if (list === undefined)
|
|
323
|
+
table.set(key, [entry]);
|
|
324
|
+
else
|
|
325
|
+
list.push(entry);
|
|
326
|
+
};
|
|
327
|
+
for (const entry of entries) {
|
|
328
|
+
add(basename, entry.basenameId, entry);
|
|
329
|
+
add(basename, entry.basenameStem, entry);
|
|
330
|
+
for (const s of [entry.id, entry.stem]) {
|
|
331
|
+
let boundary = -1;
|
|
332
|
+
while ((boundary = s.indexOf('/', boundary + 1)) !== -1) {
|
|
333
|
+
add(suffix, s.slice(boundary + 1), entry);
|
|
334
|
+
}
|
|
335
|
+
add(exact, s, entry);
|
|
336
|
+
}
|
|
337
|
+
}
|
|
338
|
+
const pick = (matches) => {
|
|
339
|
+
if (matches === undefined || matches.length === 0)
|
|
340
|
+
return undefined;
|
|
341
|
+
matches.sort((a, b) => a.segments - b.segments || codeUnitCompare(a.id, b.id));
|
|
342
|
+
return matches[0]?.id;
|
|
343
|
+
};
|
|
344
|
+
return (target) => {
|
|
345
|
+
const t = target.replace(/\\/g, '/');
|
|
346
|
+
const hit = pick(exact.get(t)) ?? pick(suffix.get(t)) ?? pick(basename.get(t));
|
|
347
|
+
return hit;
|
|
348
|
+
};
|
|
349
|
+
}
|
|
350
|
+
/**
|
|
351
|
+
* Resolve a link target to an existing note file inside the vault. When
|
|
352
|
+
* `resolve` is provided the target is first looked up in the note-id set with
|
|
353
|
+
* Obsidian semantics (exact path, then path suffix, then basename); otherwise
|
|
354
|
+
* the target is probed as a vault-relative path and extensions are tried in
|
|
355
|
+
* config order. Every result still passes the realpath containment check.
|
|
160
356
|
* @param root - vault root.
|
|
161
357
|
* @param extensions - note extensions.
|
|
162
358
|
* @param target - link target from `[[...]]`.
|
|
359
|
+
* @param resolve - optional resolver built by {@link createLinkResolver}.
|
|
163
360
|
* @returns the resolved absolute path, or `undefined` if no file exists.
|
|
164
361
|
*/
|
|
165
|
-
export async function resolveLinkTarget(root, extensions, target) {
|
|
362
|
+
export async function resolveLinkTarget(root, extensions, target, resolve) {
|
|
363
|
+
if (resolve !== undefined) {
|
|
364
|
+
const id = resolve(target);
|
|
365
|
+
if (id !== undefined)
|
|
366
|
+
return containedPathReal(root, id);
|
|
367
|
+
}
|
|
166
368
|
const candidates = extensions.map(ext => `${target}${dottedExtension(ext)}`);
|
|
167
369
|
candidates.push(target);
|
|
168
370
|
for (const candidate of candidates) {
|
|
371
|
+
// A NUL byte can never name a real note; skip it instead of aborting the
|
|
372
|
+
// whole read over one malformed link.
|
|
373
|
+
if (candidate.includes('\0'))
|
|
374
|
+
continue;
|
|
169
375
|
try {
|
|
170
376
|
const path = await containedPathReal(root, candidate);
|
|
171
377
|
const info = await stat(path);
|
|
@@ -186,13 +392,32 @@ export async function resolveLinkTarget(root, extensions, target) {
|
|
|
186
392
|
* @param root - vault root.
|
|
187
393
|
* @param extensions - note extensions.
|
|
188
394
|
* @param indexHiddenDirs - descend into remaining dot-directories.
|
|
395
|
+
* @param onSkip - optional callback for entries skipped because they could not
|
|
396
|
+
* be read; one unreadable directory must not fail the whole vault scan.
|
|
189
397
|
* @returns absolute paths of every note file.
|
|
190
398
|
*/
|
|
191
|
-
export async function listNotePaths(root, extensions, indexHiddenDirs = false) {
|
|
399
|
+
export async function listNotePaths(root, extensions, indexHiddenDirs = false, onSkip) {
|
|
192
400
|
const results = [];
|
|
193
401
|
const exclude = new Set(['.git', 'node_modules', '.obsidian']);
|
|
402
|
+
try {
|
|
403
|
+
if (!(await stat(root)).isDirectory())
|
|
404
|
+
return results;
|
|
405
|
+
}
|
|
406
|
+
catch (err) {
|
|
407
|
+
if (err.code === 'ENOENT')
|
|
408
|
+
return results;
|
|
409
|
+
throw err;
|
|
410
|
+
}
|
|
194
411
|
async function walk(dir) {
|
|
195
|
-
|
|
412
|
+
let entries;
|
|
413
|
+
try {
|
|
414
|
+
entries = await readdir(dir, { withFileTypes: true });
|
|
415
|
+
}
|
|
416
|
+
catch (error) {
|
|
417
|
+
// An unreadable directory hides only its own subtree.
|
|
418
|
+
onSkip?.(dir, error);
|
|
419
|
+
return;
|
|
420
|
+
}
|
|
196
421
|
for (const entry of entries) {
|
|
197
422
|
if (!indexHiddenDirs && entry.name.startsWith('.') && entry.name !== '.')
|
|
198
423
|
continue;
|
|
@@ -217,9 +442,11 @@ export async function listNotePaths(root, extensions, indexHiddenDirs = false) {
|
|
|
217
442
|
* @param maxLinkDepth - how many link hops to resolve.
|
|
218
443
|
* @param absolutePath - absolute path of the note.
|
|
219
444
|
* @param visited - set of already-visited absolute paths to prevent cycles.
|
|
445
|
+
* @param linkResolver - resolver over the vault's note ids; built once per
|
|
446
|
+
* call tree so bare `[[filename]]` links resolve without rescanning.
|
|
220
447
|
* @returns the parsed note with linked notes attached.
|
|
221
448
|
*/
|
|
222
|
-
export async function readNote(root, extensions, maxLinkDepth, absolutePath, visited = new Set()) {
|
|
449
|
+
export async function readNote(root, extensions, maxLinkDepth, absolutePath, visited = new Set(), linkResolver) {
|
|
223
450
|
if (visited.has(absolutePath)) {
|
|
224
451
|
return {
|
|
225
452
|
path: absolutePath,
|
|
@@ -239,9 +466,9 @@ export async function readNote(root, extensions, maxLinkDepth, absolutePath, vis
|
|
|
239
466
|
const linkedNotes = [];
|
|
240
467
|
if (maxLinkDepth > 0) {
|
|
241
468
|
for (const link of links) {
|
|
242
|
-
const target = await resolveLinkTarget(root, extensions, link);
|
|
469
|
+
const target = await resolveLinkTarget(root, extensions, link, linkResolver);
|
|
243
470
|
if (target !== undefined) {
|
|
244
|
-
const child = await readNote(root, extensions, maxLinkDepth - 1, target, nextVisited);
|
|
471
|
+
const child = await readNote(root, extensions, maxLinkDepth - 1, target, nextVisited, linkResolver);
|
|
245
472
|
linkedNotes.push(child);
|
|
246
473
|
}
|
|
247
474
|
}
|
|
@@ -263,13 +490,22 @@ export async function readNote(root, extensions, maxLinkDepth, absolutePath, vis
|
|
|
263
490
|
* @param root - vault root.
|
|
264
491
|
* @param extensions - note extensions.
|
|
265
492
|
* @param indexHiddenDirs - descend into dot-directories besides the fixed exclusions.
|
|
493
|
+
* @param onSkip - optional callback for notes skipped because they could not
|
|
494
|
+
* be read; one unreadable note must not fail search for the whole vault.
|
|
266
495
|
* @returns a map from note id to search result.
|
|
267
496
|
*/
|
|
268
|
-
export async function buildIndex(root, extensions, indexHiddenDirs = false) {
|
|
269
|
-
const paths = await listNotePaths(root, extensions, indexHiddenDirs);
|
|
497
|
+
export async function buildIndex(root, extensions, indexHiddenDirs = false, onSkip) {
|
|
498
|
+
const paths = await listNotePaths(root, extensions, indexHiddenDirs, onSkip);
|
|
270
499
|
const notes = [];
|
|
271
500
|
for (const path of paths) {
|
|
272
|
-
|
|
501
|
+
let text;
|
|
502
|
+
try {
|
|
503
|
+
text = await readFile(path, 'utf8');
|
|
504
|
+
}
|
|
505
|
+
catch (error) {
|
|
506
|
+
onSkip?.(path, error);
|
|
507
|
+
continue;
|
|
508
|
+
}
|
|
273
509
|
const { frontmatter, body } = splitFrontmatter(text);
|
|
274
510
|
notes.push({
|
|
275
511
|
path,
|
|
@@ -282,18 +518,19 @@ export async function buildIndex(root, extensions, indexHiddenDirs = false) {
|
|
|
282
518
|
}
|
|
283
519
|
const index = new Map();
|
|
284
520
|
for (const note of notes) {
|
|
521
|
+
const explicitTitle = typeof note.frontmatter.title === 'string' ? note.frontmatter.title.trim() : '';
|
|
285
522
|
const headingMatch = /^#\s+(.+)$/m.exec(note.body);
|
|
286
|
-
const title =
|
|
287
|
-
?
|
|
288
|
-
:
|
|
523
|
+
const title = explicitTitle !== ''
|
|
524
|
+
? explicitTitle
|
|
525
|
+
: (headingMatch !== null && headingMatch[1] !== undefined
|
|
526
|
+
? headingMatch[1].trim()
|
|
527
|
+
: note.id.replace(/\.[^.]+$/, ''));
|
|
289
528
|
index.set(note.id, { id: note.id, title, backlinks: [], body: note.body });
|
|
290
529
|
}
|
|
530
|
+
const linkResolver = createLinkResolver([...index.keys()]);
|
|
291
531
|
for (const note of notes) {
|
|
292
532
|
for (const link of note.links) {
|
|
293
|
-
const targetId =
|
|
294
|
-
const base = id.replace(/\.[^.]+$/, '');
|
|
295
|
-
return base === link || id === link;
|
|
296
|
-
});
|
|
533
|
+
const targetId = linkResolver(link);
|
|
297
534
|
if (targetId !== undefined) {
|
|
298
535
|
const entry = index.get(targetId);
|
|
299
536
|
if (entry !== undefined && !entry.backlinks.includes(note.id)) {
|
|
@@ -313,8 +550,19 @@ export async function buildIndex(root, extensions, indexHiddenDirs = false) {
|
|
|
313
550
|
*/
|
|
314
551
|
async function writeAtomic(absolutePath, contents) {
|
|
315
552
|
const tmpPath = `${absolutePath}.tmp-${process.pid}-${randomUUID()}`;
|
|
553
|
+
// rename replaces the inode, so carry an existing target's permission bits
|
|
554
|
+
// onto the staged file; otherwise a read-only note silently becomes writable.
|
|
555
|
+
let mode;
|
|
316
556
|
try {
|
|
317
|
-
await
|
|
557
|
+
mode = (await stat(absolutePath)).mode & 0o7777;
|
|
558
|
+
}
|
|
559
|
+
catch {
|
|
560
|
+
// New note: keep the default creation mode.
|
|
561
|
+
}
|
|
562
|
+
try {
|
|
563
|
+
await writeFile(tmpPath, contents, { encoding: 'utf8', ...(mode === undefined ? {} : { mode }) });
|
|
564
|
+
if (mode !== undefined)
|
|
565
|
+
await chmod(tmpPath, mode);
|
|
318
566
|
await rename(tmpPath, absolutePath);
|
|
319
567
|
}
|
|
320
568
|
catch (error) {
|
|
@@ -322,6 +570,66 @@ async function writeAtomic(absolutePath, contents) {
|
|
|
322
570
|
throw error;
|
|
323
571
|
}
|
|
324
572
|
}
|
|
573
|
+
/**
|
|
574
|
+
* Turn a filesystem failure into a structured, vault-relative tool error. Raw
|
|
575
|
+
* Node errors carry absolute host paths and no plugin context; callers see
|
|
576
|
+
* `tool-memory-filesystem: cannot read note "x.md": permission denied` instead.
|
|
577
|
+
*/
|
|
578
|
+
function noteIoError(action, id, error) {
|
|
579
|
+
return new Error(`tool-memory-filesystem: cannot ${action} note ${JSON.stringify(id)}: ${describeIoError(error)}`);
|
|
580
|
+
}
|
|
581
|
+
/** Short, host-path-free description of one filesystem error. */
|
|
582
|
+
function describeIoError(error) {
|
|
583
|
+
const code = error?.code;
|
|
584
|
+
if (code === 'ENOENT')
|
|
585
|
+
return 'no such note';
|
|
586
|
+
if (code === 'EACCES' || code === 'EPERM')
|
|
587
|
+
return 'permission denied';
|
|
588
|
+
if (code === 'EISDIR')
|
|
589
|
+
return 'path is a directory';
|
|
590
|
+
if (typeof code === 'string' && code !== '')
|
|
591
|
+
return code;
|
|
592
|
+
return error instanceof Error ? error.message : String(error);
|
|
593
|
+
}
|
|
594
|
+
/**
|
|
595
|
+
* Follow a symlink at the final path component so a write publishes through
|
|
596
|
+
* the link instead of replacing it: the sibling-temp + rename sequence targets
|
|
597
|
+
* the directory entry itself, which would otherwise turn an alias note into a
|
|
598
|
+
* regular file and leave the linked note untouched.
|
|
599
|
+
*/
|
|
600
|
+
async function resolveSymlinkWriteTarget(root, absolutePath, id) {
|
|
601
|
+
const info = await lstat(absolutePath).catch((error) => {
|
|
602
|
+
if (error.code === 'ENOENT')
|
|
603
|
+
return undefined;
|
|
604
|
+
throw noteIoError('write', id, error);
|
|
605
|
+
});
|
|
606
|
+
if (info === undefined || !info.isSymbolicLink())
|
|
607
|
+
return absolutePath;
|
|
608
|
+
const canonical = await realpath(absolutePath).catch(() => {
|
|
609
|
+
// A broken link cannot be written through; replacing it would silently
|
|
610
|
+
// discard the alias, so refuse instead.
|
|
611
|
+
throw new Error(`tool-memory-filesystem: cannot write through broken symbolic link ${JSON.stringify(id)}`);
|
|
612
|
+
});
|
|
613
|
+
const canonicalRoot = await realpath(resolve(root));
|
|
614
|
+
const remainder = relative(canonicalRoot, canonical);
|
|
615
|
+
if (remainder === '..' || remainder.startsWith(`..${sep}`) || isAbsolute(remainder)) {
|
|
616
|
+
throw new Error(`tool-memory-filesystem: path ${JSON.stringify(id)} resolves outside the vault root`);
|
|
617
|
+
}
|
|
618
|
+
return canonical;
|
|
619
|
+
}
|
|
620
|
+
/** Build the skip callback that reports unreadable entries to the Loader logger. */
|
|
621
|
+
function skipLogger(ctx, root) {
|
|
622
|
+
return (path, error) => {
|
|
623
|
+
const code = error.code ?? 'unknown';
|
|
624
|
+
ctx.logger?.warn(`tool-memory-filesystem: skipped unreadable entry ${vaultRelativeId(root, path)} (${code})`);
|
|
625
|
+
};
|
|
626
|
+
}
|
|
627
|
+
/** Validate that at least one note extension is configured. */
|
|
628
|
+
function assertExtensions(extensions) {
|
|
629
|
+
if (extensions.length === 0) {
|
|
630
|
+
throw new Error('tool-memory-filesystem: extensions must list at least one file extension');
|
|
631
|
+
}
|
|
632
|
+
}
|
|
325
633
|
/** Validate a positive-integer config bound. */
|
|
326
634
|
function assertPositiveInteger(name, value) {
|
|
327
635
|
if (!Number.isInteger(value) || value < 1) {
|
|
@@ -343,10 +651,11 @@ export function apply(ctx, config) {
|
|
|
343
651
|
const resolved = config;
|
|
344
652
|
assertPositiveInteger('maxLinkDepth', resolved.maxLinkDepth);
|
|
345
653
|
assertPositiveInteger('maxSearchResults', resolved.maxSearchResults);
|
|
654
|
+
assertExtensions(resolved.extensions);
|
|
346
655
|
const vaultRootFor = (exec) => resolveMemoryVaultRoot(resolved.vaultRoot, exec);
|
|
347
656
|
ctx.effect(() => ctx.tools.register(defineTool({
|
|
348
657
|
name: 'wiki_read',
|
|
349
|
-
description: 'Read one Markdown note from the wiki vault, optionally following Obsidian-style [[link]] references up to the configured depth. Returns the note id, frontmatter, body, and
|
|
658
|
+
description: 'Read one Markdown note from the wiki vault, optionally following Obsidian-style [[link]] references up to the configured depth. Returns the note id, frontmatter, body, linked notes, mtime, and a modifiedExternally flag that is true when the file changed on disk since this tool last observed it.',
|
|
350
659
|
parameters: {
|
|
351
660
|
id: {
|
|
352
661
|
type: 'string',
|
|
@@ -361,13 +670,21 @@ export function apply(ctx, config) {
|
|
|
361
670
|
async execute(args, exec) {
|
|
362
671
|
const vaultRoot = vaultRootFor(exec);
|
|
363
672
|
const absolutePath = await containedPathReal(vaultRoot, args.id);
|
|
364
|
-
const
|
|
365
|
-
|
|
673
|
+
const linkResolver = createLinkResolver((await listNotePaths(vaultRoot, resolved.extensions, resolved.indexHiddenDirs, skipLogger(ctx, vaultRoot)))
|
|
674
|
+
.map(path => vaultRelativeId(vaultRoot, path)));
|
|
675
|
+
const note = await readNote(vaultRoot, resolved.extensions, resolved.maxLinkDepth, absolutePath, new Set(), linkResolver)
|
|
676
|
+
.catch(error => { throw noteIoError('read', args.id, error); });
|
|
677
|
+
const info = await stat(absolutePath).catch(error => { throw noteIoError('read', args.id, error); });
|
|
678
|
+
return {
|
|
679
|
+
...note,
|
|
680
|
+
mtime: formatBeijingTime(info.mtime),
|
|
681
|
+
modifiedExternally: detectExternalChange(absolutePath, info.mtimeMs),
|
|
682
|
+
};
|
|
366
683
|
},
|
|
367
684
|
})));
|
|
368
685
|
ctx.effect(() => ctx.tools.register(defineTool({
|
|
369
686
|
name: 'wiki_search',
|
|
370
|
-
description: 'Search the wiki vault by note title or body keyword. Returns
|
|
687
|
+
description: 'Search the wiki vault by note title or body keyword. Terms are OR-matched; results are ranked by field-weighted relevance (title/id hits outrank body hits, rare terms weigh more, notes linked from strong hits get a boost, and results may be fused with semantic search when available). Returns { hits, total, truncated }: each hit carries the note id, title, score, and backlinks, total counts every match before the result cap, and truncated is true when that cap cut the list. Use this before asking the user which note to read.',
|
|
371
688
|
parameters: {
|
|
372
689
|
query: {
|
|
373
690
|
type: 'string',
|
|
@@ -377,37 +694,73 @@ export function apply(ctx, config) {
|
|
|
377
694
|
},
|
|
378
695
|
output: {
|
|
379
696
|
schema: {
|
|
380
|
-
type: '
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
697
|
+
type: 'object',
|
|
698
|
+
additionalProperties: false,
|
|
699
|
+
properties: {
|
|
700
|
+
hits: {
|
|
701
|
+
type: 'array',
|
|
702
|
+
items: {
|
|
703
|
+
type: 'object',
|
|
704
|
+
additionalProperties: false,
|
|
705
|
+
properties: {
|
|
706
|
+
id: { type: 'string' },
|
|
707
|
+
title: { type: 'string' },
|
|
708
|
+
score: { type: 'number' },
|
|
709
|
+
backlinks: { type: 'array', items: { type: 'string' } },
|
|
710
|
+
},
|
|
711
|
+
},
|
|
388
712
|
},
|
|
713
|
+
total: { type: 'integer' },
|
|
714
|
+
truncated: { type: 'boolean' },
|
|
389
715
|
},
|
|
390
716
|
},
|
|
391
717
|
render: (_args, value) => [{ type: 'text', text: JSON.stringify(value, null, 2) }],
|
|
392
718
|
},
|
|
393
719
|
async execute(args, exec) {
|
|
394
720
|
const vaultRoot = vaultRootFor(exec);
|
|
395
|
-
const index = await buildIndex(vaultRoot, resolved.extensions, resolved.indexHiddenDirs);
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
721
|
+
const index = await buildIndex(vaultRoot, resolved.extensions, resolved.indexHiddenDirs, skipLogger(ctx, vaultRoot));
|
|
722
|
+
let hits = rankNotes(index, args.query);
|
|
723
|
+
// Optional semantic layer: when `wiki_semantic_search` is mounted, fuse
|
|
724
|
+
// both rankings via reciprocal rank fusion. Only the dispatch is
|
|
725
|
+
// fail-soft — an unavailable or erroring semantic layer degrades to the
|
|
726
|
+
// lexical ranking; parse/fusion errors propagate so real bugs surface
|
|
727
|
+
// instead of silently weakening results.
|
|
728
|
+
if (ctx.tools.get('wiki_semantic_search') !== undefined) {
|
|
729
|
+
let semantic;
|
|
730
|
+
try {
|
|
731
|
+
semantic = await ctx.tools.execute({
|
|
732
|
+
callId: `${exec.callId}:wiki_semantic_search:${randomUUID()}`,
|
|
733
|
+
rootCallId: exec.rootCallId,
|
|
734
|
+
name: 'wiki_semantic_search',
|
|
735
|
+
arguments: { query: args.query },
|
|
736
|
+
parent: exec.token,
|
|
737
|
+
signal: exec.signal,
|
|
738
|
+
...(exec.agent !== undefined ? { agent: exec.agent } : {}),
|
|
739
|
+
});
|
|
740
|
+
}
|
|
741
|
+
catch (error) {
|
|
742
|
+
ctx.logger?.warn('tool-memory-filesystem: semantic fusion skipped: ' + String(error));
|
|
743
|
+
}
|
|
744
|
+
if (semantic !== undefined && !semantic.isError) {
|
|
745
|
+
const semanticHits = JSON.parse(semantic.content
|
|
746
|
+
.map(block => {
|
|
747
|
+
const maybe = block;
|
|
748
|
+
return maybe.type === 'text' && typeof maybe.text === 'string' ? maybe.text : '';
|
|
749
|
+
})
|
|
750
|
+
.join('') || '[]');
|
|
751
|
+
hits = fuseRankings(hits, semanticHits.map(hit => hit.id), index);
|
|
752
|
+
}
|
|
753
|
+
else if (semantic !== undefined) {
|
|
754
|
+
ctx.logger?.warn('tool-memory-filesystem: semantic fusion skipped: wiki_semantic_search returned an error');
|
|
402
755
|
}
|
|
403
756
|
}
|
|
404
|
-
hits.
|
|
405
|
-
return hits.
|
|
757
|
+
const capped = hits.slice(0, resolved.maxSearchResults);
|
|
758
|
+
return { hits: capped, total: hits.length, truncated: capped.length < hits.length };
|
|
406
759
|
},
|
|
407
760
|
})));
|
|
408
761
|
ctx.effect(() => ctx.tools.register(defineTool({
|
|
409
762
|
name: 'wiki_write',
|
|
410
|
-
description: 'Create a new note or append to an existing note in the wiki vault. The path is relative to the vault root. When appending, the new content is inserted at the end of the body after a timestamp header.',
|
|
763
|
+
description: 'Create a new note or append to an existing note in the wiki vault. The path is relative to the vault root. When appending, the new content is inserted at the end of the body after a timestamp header. Returns { id, mode, bytes, version }, where version fingerprints the written content and can be passed as baseVersion on the next write without reading the note again.',
|
|
411
764
|
parameters: {
|
|
412
765
|
id: {
|
|
413
766
|
type: 'string',
|
|
@@ -426,7 +779,7 @@ export function apply(ctx, config) {
|
|
|
426
779
|
},
|
|
427
780
|
baseVersion: {
|
|
428
781
|
type: 'string',
|
|
429
|
-
description: 'Optional version returned by wiki_read. When provided, the write fails if the note changed since that
|
|
782
|
+
description: 'Optional version returned by wiki_read or by an earlier wiki_write. When provided, the write fails if the note changed since that version.',
|
|
430
783
|
},
|
|
431
784
|
},
|
|
432
785
|
output: {
|
|
@@ -437,6 +790,10 @@ export function apply(ctx, config) {
|
|
|
437
790
|
id: { type: 'string' },
|
|
438
791
|
mode: { type: 'string' },
|
|
439
792
|
bytes: { type: 'integer' },
|
|
793
|
+
version: { type: 'string' },
|
|
794
|
+
created: { type: 'string' },
|
|
795
|
+
updated: { type: 'string' },
|
|
796
|
+
committed: { type: 'boolean' },
|
|
440
797
|
},
|
|
441
798
|
},
|
|
442
799
|
render: (_args, value) => [{ type: 'text', text: JSON.stringify(value, null, 2) }],
|
|
@@ -450,34 +807,76 @@ export function apply(ctx, config) {
|
|
|
450
807
|
if (!hasExtension) {
|
|
451
808
|
throw new Error(`tool-memory-filesystem: note id must end with one of ${resolved.extensions.join(', ')}`);
|
|
452
809
|
}
|
|
453
|
-
await
|
|
810
|
+
const writeTarget = await resolveSymlinkWriteTarget(vaultRoot, absolutePath, args.id);
|
|
811
|
+
try {
|
|
812
|
+
await mkdir(dirname(writeTarget), { recursive: true });
|
|
813
|
+
}
|
|
814
|
+
catch (error) {
|
|
815
|
+
throw noteIoError('write', args.id, error);
|
|
816
|
+
}
|
|
454
817
|
await containedPathReal(vaultRoot, args.id);
|
|
455
818
|
exec.signal.throwIfAborted();
|
|
456
819
|
let existing;
|
|
457
820
|
try {
|
|
458
|
-
existing = await readFile(
|
|
821
|
+
existing = await readFile(writeTarget, 'utf8');
|
|
459
822
|
}
|
|
460
|
-
catch {
|
|
461
|
-
// file
|
|
823
|
+
catch (error) {
|
|
824
|
+
// Only a missing file means a first write; any other failure (a
|
|
825
|
+
// directory in the way, an unreadable note) must not silently fall
|
|
826
|
+
// back to treating the note as new and replacing it.
|
|
827
|
+
if (error.code !== 'ENOENT')
|
|
828
|
+
throw noteIoError('write', args.id, error);
|
|
462
829
|
}
|
|
463
830
|
if (args.baseVersion !== undefined && noteVersion(existing ?? '') !== args.baseVersion) {
|
|
464
831
|
throw new Error(`tool-memory-filesystem: note ${args.id} changed since it was read; re-read it before writing`);
|
|
465
832
|
}
|
|
466
833
|
let finalBody;
|
|
834
|
+
let frontmatter = {};
|
|
467
835
|
if (mode === 'overwrite') {
|
|
468
|
-
finalBody = args.content;
|
|
836
|
+
finalBody = normalizeTextTimestamps(args.content);
|
|
837
|
+
frontmatter = splitFrontmatter(finalBody).frontmatter;
|
|
469
838
|
}
|
|
470
839
|
else {
|
|
471
|
-
const
|
|
840
|
+
const split = splitFrontmatter(existing ?? '');
|
|
841
|
+
frontmatter = split.frontmatter;
|
|
842
|
+
const body = split.body;
|
|
843
|
+
normalizeNoteTimestamps(frontmatter);
|
|
844
|
+
const timestamp = formatBeijingTime(new Date());
|
|
845
|
+
// Track note lifetime the way memory_capture does: `created` on the
|
|
846
|
+
// first write, `updated` on every append thereafter.
|
|
847
|
+
if (existing === undefined) {
|
|
848
|
+
frontmatter.created ??= timestamp;
|
|
849
|
+
}
|
|
850
|
+
else {
|
|
851
|
+
frontmatter.updated = timestamp;
|
|
852
|
+
}
|
|
472
853
|
const frontmatterText = Object.keys(frontmatter).length > 0
|
|
473
854
|
? `---\n${yaml.dump(frontmatter).trim()}\n---\n\n`
|
|
474
855
|
: '';
|
|
475
|
-
|
|
476
|
-
|
|
856
|
+
// Content that opens with its own heading supplies the section
|
|
857
|
+
// header, so a bare `## <time>` above it would duplicate it.
|
|
858
|
+
const header = args.content.trimStart().startsWith('#') ? '' : `## ${timestamp}\n\n`;
|
|
859
|
+
const base = `${frontmatterText}${body}`.replace(/\s+$/, '');
|
|
860
|
+
finalBody = base === '' ? `${header}${args.content}\n` : `${base}\n\n${header}${args.content}\n`;
|
|
477
861
|
}
|
|
478
862
|
exec.signal.throwIfAborted();
|
|
479
|
-
await writeAtomic(
|
|
480
|
-
|
|
863
|
+
await writeAtomic(writeTarget, finalBody)
|
|
864
|
+
.catch(error => { throw noteIoError('write', args.id, error); });
|
|
865
|
+
const info = await stat(writeTarget);
|
|
866
|
+
recordMtime(absolutePath, info.mtimeMs);
|
|
867
|
+
const result = {
|
|
868
|
+
id: vaultRelativeId(vaultRoot, absolutePath),
|
|
869
|
+
mode,
|
|
870
|
+
bytes: Buffer.byteLength(finalBody, 'utf8'),
|
|
871
|
+
// `writeAtomic` publishes `finalBody` verbatim, so this is the same
|
|
872
|
+
// fingerprint `wiki_read` would report — without a second read.
|
|
873
|
+
version: noteVersion(finalBody),
|
|
874
|
+
};
|
|
875
|
+
if (typeof frontmatter.created === 'string' && frontmatter.created !== '')
|
|
876
|
+
result.created = frontmatter.created;
|
|
877
|
+
if (typeof frontmatter.updated === 'string' && frontmatter.updated !== '')
|
|
878
|
+
result.updated = frontmatter.updated;
|
|
879
|
+
return result;
|
|
481
880
|
},
|
|
482
881
|
})));
|
|
483
882
|
}
|
package/lib/search.js
ADDED
|
@@ -0,0 +1,150 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Multi-layer lexical ranking for `wiki_search`: field-weighted BM25-style
|
|
3
|
+
* scoring over note id/title/backlinks/body, a verbatim-phrase bonus, and a
|
|
4
|
+
* link-graph boost for notes referenced by strong lexical hits. Substring
|
|
5
|
+
* matching (not tokenization) keeps CJK text working without a segmenter.
|
|
6
|
+
* @module @jacklika/dsh-tool-memory-filesystem/search
|
|
7
|
+
*/
|
|
8
|
+
const K1 = 1.2;
|
|
9
|
+
const B = 0.75;
|
|
10
|
+
const TITLE_WEIGHT = 3;
|
|
11
|
+
const BODY_WEIGHT = 1;
|
|
12
|
+
const BACKLINK_WEIGHT = 1;
|
|
13
|
+
const GRAPH_BOOST = 0.15;
|
|
14
|
+
const RRF_K = 60;
|
|
15
|
+
function countOccurrences(haystack, needle) {
|
|
16
|
+
if (needle === '')
|
|
17
|
+
return 0;
|
|
18
|
+
let count = 0;
|
|
19
|
+
let from = 0;
|
|
20
|
+
while ((from = haystack.indexOf(needle, from)) !== -1) {
|
|
21
|
+
count += 1;
|
|
22
|
+
from += needle.length;
|
|
23
|
+
}
|
|
24
|
+
return count;
|
|
25
|
+
}
|
|
26
|
+
function toHit(note, score) {
|
|
27
|
+
return {
|
|
28
|
+
id: note.id,
|
|
29
|
+
title: note.title,
|
|
30
|
+
score: Math.round(score * 10000) / 10000,
|
|
31
|
+
backlinks: note.backlinks,
|
|
32
|
+
};
|
|
33
|
+
}
|
|
34
|
+
/**
|
|
35
|
+
* Rank every note in `index` against `query`. Any note containing at least one
|
|
36
|
+
* query term (OR semantics) is a candidate; scores follow BM25 saturation with
|
|
37
|
+
* field weights (`id`/`title` ×3, backlinks ×1, body ×1), a phrase bonus for a
|
|
38
|
+
* verbatim multi-term match, and a graph boost for notes linked from a scored
|
|
39
|
+
* hit.
|
|
40
|
+
* @param index - note index built by {@link buildIndex}.
|
|
41
|
+
* @param query - raw query text.
|
|
42
|
+
* @returns hits sorted by descending score.
|
|
43
|
+
*/
|
|
44
|
+
export function rankNotes(index, query) {
|
|
45
|
+
const lowered = query.toLowerCase();
|
|
46
|
+
const terms = [...new Set(lowered.split(/\s+/).filter(Boolean))];
|
|
47
|
+
if (terms.length === 0)
|
|
48
|
+
return [];
|
|
49
|
+
const prepared = [...index.values()].map(note => ({
|
|
50
|
+
note,
|
|
51
|
+
titleField: `${note.id} ${note.title}`.toLowerCase(),
|
|
52
|
+
backlinkField: note.backlinks.join(' ').toLowerCase(),
|
|
53
|
+
bodyField: note.body.toLowerCase(),
|
|
54
|
+
docLen: Math.max(1, note.body.length),
|
|
55
|
+
}));
|
|
56
|
+
const docCount = prepared.length;
|
|
57
|
+
const avgdl = prepared.reduce((sum, p) => sum + p.docLen, 0) / Math.max(1, docCount);
|
|
58
|
+
const idf = new Map();
|
|
59
|
+
for (const term of terms) {
|
|
60
|
+
const df = prepared.filter(p => p.titleField.includes(term) || p.backlinkField.includes(term) || p.bodyField.includes(term)).length;
|
|
61
|
+
idf.set(term, Math.log(1 + (docCount - df + 0.5) / (df + 0.5)));
|
|
62
|
+
}
|
|
63
|
+
const sumIdf = [...idf.values()].reduce((a, b) => a + b, 0);
|
|
64
|
+
const scores = new Map();
|
|
65
|
+
const phrase = lowered.trim();
|
|
66
|
+
for (const p of prepared) {
|
|
67
|
+
let score = 0;
|
|
68
|
+
for (const term of terms) {
|
|
69
|
+
const tf = TITLE_WEIGHT * countOccurrences(p.titleField, term) +
|
|
70
|
+
BACKLINK_WEIGHT * countOccurrences(p.backlinkField, term) +
|
|
71
|
+
BODY_WEIGHT * countOccurrences(p.bodyField, term);
|
|
72
|
+
if (tf === 0)
|
|
73
|
+
continue;
|
|
74
|
+
const termIdf = idf.get(term) ?? 0;
|
|
75
|
+
score += termIdf * (tf * (K1 + 1)) / (tf + K1 * (1 - B + (B * p.docLen) / avgdl));
|
|
76
|
+
}
|
|
77
|
+
if (terms.length > 1) {
|
|
78
|
+
if (p.titleField.includes(phrase))
|
|
79
|
+
score += sumIdf;
|
|
80
|
+
else if (p.bodyField.includes(phrase))
|
|
81
|
+
score += 0.5 * sumIdf;
|
|
82
|
+
}
|
|
83
|
+
if (score > 0)
|
|
84
|
+
scores.set(p.note.id, score);
|
|
85
|
+
}
|
|
86
|
+
// Graph boost: a note linked from a scored hit gains a fraction of that
|
|
87
|
+
// hit's score (the strongest link wins). `backlinks` lists the sources, so
|
|
88
|
+
// hit H links to T exactly when H.id ∈ T.backlinks.
|
|
89
|
+
const linkedBoost = new Map();
|
|
90
|
+
for (const p of prepared) {
|
|
91
|
+
for (const sourceId of p.note.backlinks) {
|
|
92
|
+
const sourceScore = scores.get(sourceId);
|
|
93
|
+
if (sourceScore !== undefined) {
|
|
94
|
+
linkedBoost.set(p.note.id, Math.max(linkedBoost.get(p.note.id) ?? 0, GRAPH_BOOST * sourceScore));
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
const hits = [];
|
|
99
|
+
for (const p of prepared) {
|
|
100
|
+
const score = (scores.get(p.note.id) ?? 0) + (linkedBoost.get(p.note.id) ?? 0);
|
|
101
|
+
if (score > 0)
|
|
102
|
+
hits.push(toHit(p.note, score));
|
|
103
|
+
}
|
|
104
|
+
hits.sort((a, b) => b.score - a.score || codeUnitCompare(a.id, b.id));
|
|
105
|
+
return hits;
|
|
106
|
+
}
|
|
107
|
+
/**
|
|
108
|
+
* Locale-independent code-unit ordering — `localeCompare` without a locale
|
|
109
|
+
* follows the host runtime locale, which makes tied-score ordering differ
|
|
110
|
+
* across machines.
|
|
111
|
+
*/
|
|
112
|
+
export function codeUnitCompare(a, b) {
|
|
113
|
+
return a < b ? -1 : a > b ? 1 : 0;
|
|
114
|
+
}
|
|
115
|
+
/**
|
|
116
|
+
* Fuse the lexical ranking with a semantic-search ranking using reciprocal
|
|
117
|
+
* rank fusion (k = 60). Semantic-only notes are surfaced with their index
|
|
118
|
+
* metadata so the output shape stays uniform.
|
|
119
|
+
* @param lexical - hits from {@link rankNotes}.
|
|
120
|
+
* @param semanticIds - note ids ordered by descending semantic similarity.
|
|
121
|
+
* @param index - the note index for metadata lookup.
|
|
122
|
+
* @returns fused hits sorted by descending RRF score.
|
|
123
|
+
*/
|
|
124
|
+
export function fuseRankings(lexical, semanticIds, index) {
|
|
125
|
+
const fused = new Map();
|
|
126
|
+
const contribute = (id, rank) => {
|
|
127
|
+
const existing = fused.get(id);
|
|
128
|
+
if (existing !== undefined) {
|
|
129
|
+
existing.score += 1 / (RRF_K + rank + 1);
|
|
130
|
+
return;
|
|
131
|
+
}
|
|
132
|
+
const note = index.get(id);
|
|
133
|
+
if (note === undefined)
|
|
134
|
+
return; // never fabricate a hit for an id outside the vault index
|
|
135
|
+
fused.set(id, {
|
|
136
|
+
id,
|
|
137
|
+
title: note.title,
|
|
138
|
+
score: 1 / (RRF_K + rank + 1),
|
|
139
|
+
backlinks: note.backlinks,
|
|
140
|
+
});
|
|
141
|
+
};
|
|
142
|
+
lexical.forEach((hit, rank) => contribute(hit.id, rank));
|
|
143
|
+
semanticIds.forEach((id, rank) => contribute(id, rank));
|
|
144
|
+
const hits = [...fused.values()];
|
|
145
|
+
for (const hit of hits)
|
|
146
|
+
hit.score = Math.round(hit.score * 10000) / 10000;
|
|
147
|
+
hits.sort((a, b) => b.score - a.score || codeUnitCompare(a.id, b.id));
|
|
148
|
+
return hits;
|
|
149
|
+
}
|
|
150
|
+
//# sourceMappingURL=search.js.map
|
package/lib/types/index.d.ts
CHANGED
|
@@ -10,8 +10,15 @@
|
|
|
10
10
|
import type { Context } from '@deepseek-ai/cordis';
|
|
11
11
|
import z from '@deepseek-ai/schemastery';
|
|
12
12
|
import { type ToolRunContext } from '@deepseek-ai/dsh-tools';
|
|
13
|
+
import { codeUnitCompare } from './search.ts';
|
|
13
14
|
import type { IndexedNote, Note } from './types.ts';
|
|
14
15
|
export type * from './types.ts';
|
|
16
|
+
/**
|
|
17
|
+
* Locale-independent note-id ordering, re-exported so sibling packages that
|
|
18
|
+
* truncate or tie-break id lists sort by the same rule instead of a copy that
|
|
19
|
+
* can drift.
|
|
20
|
+
*/
|
|
21
|
+
export { codeUnitCompare };
|
|
15
22
|
/**
|
|
16
23
|
* Compute a vault-relative note id from an absolute path. The result always
|
|
17
24
|
* uses POSIX separators so note ids are stable across Windows, macOS, and Linux.
|
|
@@ -73,7 +80,8 @@ export declare function containedPath(root: string, candidate: string): string;
|
|
|
73
80
|
export declare function containedPathReal(root: string, candidate: string): Promise<string>;
|
|
74
81
|
/**
|
|
75
82
|
* Extract YAML frontmatter and body from Markdown text. Only the leading
|
|
76
|
-
* `---\n...\n---\n` form is recognized.
|
|
83
|
+
* `---\n...\n---\n` form is recognized, optionally behind a UTF-8 BOM.
|
|
84
|
+
* Duplicate keys resolve last-one-wins rather than dropping the block.
|
|
77
85
|
* @param text - raw file contents.
|
|
78
86
|
* @returns frontmatter map and body.
|
|
79
87
|
*/
|
|
@@ -88,6 +96,13 @@ export declare function splitFrontmatter(text: string): {
|
|
|
88
96
|
* @returns content hash stable across processes.
|
|
89
97
|
*/
|
|
90
98
|
export declare function noteVersion(text: string): string;
|
|
99
|
+
/**
|
|
100
|
+
* Normalize `created`/`updated` frontmatter timestamps to the vault's
|
|
101
|
+
* `+08:00` second-precision convention. Parseable values in any other form
|
|
102
|
+
* (`Z`, `+00:00`, millisecond precision) are rewritten; unparseable values
|
|
103
|
+
* pass through untouched so a non-date string is never corrupted.
|
|
104
|
+
*/
|
|
105
|
+
export declare function normalizeNoteTimestamps(frontmatter: Record<string, unknown>): void;
|
|
91
106
|
/**
|
|
92
107
|
* Find all Obsidian-style `[[link]]` references in note text. Aliases of the
|
|
93
108
|
* form `[[link|alias]]` return the link target only.
|
|
@@ -102,14 +117,33 @@ export declare function extractLinks(text: string): string[];
|
|
|
102
117
|
*/
|
|
103
118
|
export declare function dottedExtension(ext: string): string;
|
|
104
119
|
/**
|
|
105
|
-
*
|
|
106
|
-
*
|
|
120
|
+
* A link resolver maps an Obsidian `[[target]]` to a vault-relative note id,
|
|
121
|
+
* or `undefined` when nothing matches.
|
|
122
|
+
*/
|
|
123
|
+
export type LinkResolver = (target: string) => string | undefined;
|
|
124
|
+
/**
|
|
125
|
+
* Build a resolver over the known note ids, matching Obsidian semantics: an
|
|
126
|
+
* exact vault-relative path first, then a path-suffix match, then a bare
|
|
127
|
+
* basename match — each tier tolerates the target carrying or omitting the
|
|
128
|
+
* extension. Ambiguity within a tier resolves to the id with the fewest path
|
|
129
|
+
* segments, ties by code-unit order, so results are identical across hosts.
|
|
130
|
+
* @param ids - vault-relative note ids (e.g. `shared/notes/foo.md`).
|
|
131
|
+
* @returns resolver function for `[[...]]` targets.
|
|
132
|
+
*/
|
|
133
|
+
export declare function createLinkResolver(ids: string[]): LinkResolver;
|
|
134
|
+
/**
|
|
135
|
+
* Resolve a link target to an existing note file inside the vault. When
|
|
136
|
+
* `resolve` is provided the target is first looked up in the note-id set with
|
|
137
|
+
* Obsidian semantics (exact path, then path suffix, then basename); otherwise
|
|
138
|
+
* the target is probed as a vault-relative path and extensions are tried in
|
|
139
|
+
* config order. Every result still passes the realpath containment check.
|
|
107
140
|
* @param root - vault root.
|
|
108
141
|
* @param extensions - note extensions.
|
|
109
142
|
* @param target - link target from `[[...]]`.
|
|
143
|
+
* @param resolve - optional resolver built by {@link createLinkResolver}.
|
|
110
144
|
* @returns the resolved absolute path, or `undefined` if no file exists.
|
|
111
145
|
*/
|
|
112
|
-
export declare function resolveLinkTarget(root: string, extensions: string[], target: string): Promise<string | undefined>;
|
|
146
|
+
export declare function resolveLinkTarget(root: string, extensions: string[], target: string, resolve?: LinkResolver): Promise<string | undefined>;
|
|
113
147
|
/**
|
|
114
148
|
* Recursively discover note files under the vault root. `.git`, `.obsidian`,
|
|
115
149
|
* and `node_modules` are always excluded; other dot-directories are skipped
|
|
@@ -117,9 +151,11 @@ export declare function resolveLinkTarget(root: string, extensions: string[], ta
|
|
|
117
151
|
* @param root - vault root.
|
|
118
152
|
* @param extensions - note extensions.
|
|
119
153
|
* @param indexHiddenDirs - descend into remaining dot-directories.
|
|
154
|
+
* @param onSkip - optional callback for entries skipped because they could not
|
|
155
|
+
* be read; one unreadable directory must not fail the whole vault scan.
|
|
120
156
|
* @returns absolute paths of every note file.
|
|
121
157
|
*/
|
|
122
|
-
export declare function listNotePaths(root: string, extensions: string[], indexHiddenDirs?: boolean): Promise<string[]>;
|
|
158
|
+
export declare function listNotePaths(root: string, extensions: string[], indexHiddenDirs?: boolean, onSkip?: (path: string, error: unknown) => void): Promise<string[]>;
|
|
123
159
|
/** Wire view of one linked note, truncated to avoid deep recursion types. */
|
|
124
160
|
export interface LinkedNote {
|
|
125
161
|
id: string;
|
|
@@ -137,9 +173,11 @@ export interface LinkedNote {
|
|
|
137
173
|
* @param maxLinkDepth - how many link hops to resolve.
|
|
138
174
|
* @param absolutePath - absolute path of the note.
|
|
139
175
|
* @param visited - set of already-visited absolute paths to prevent cycles.
|
|
176
|
+
* @param linkResolver - resolver over the vault's note ids; built once per
|
|
177
|
+
* call tree so bare `[[filename]]` links resolve without rescanning.
|
|
140
178
|
* @returns the parsed note with linked notes attached.
|
|
141
179
|
*/
|
|
142
|
-
export declare function readNote(root: string, extensions: string[], maxLinkDepth: number, absolutePath: string, visited?: ReadonlySet<string
|
|
180
|
+
export declare function readNote(root: string, extensions: string[], maxLinkDepth: number, absolutePath: string, visited?: ReadonlySet<string>, linkResolver?: LinkResolver): Promise<Note & {
|
|
143
181
|
linkedNotes: LinkedNote[];
|
|
144
182
|
}>;
|
|
145
183
|
/**
|
|
@@ -149,9 +187,11 @@ export declare function readNote(root: string, extensions: string[], maxLinkDept
|
|
|
149
187
|
* @param root - vault root.
|
|
150
188
|
* @param extensions - note extensions.
|
|
151
189
|
* @param indexHiddenDirs - descend into dot-directories besides the fixed exclusions.
|
|
190
|
+
* @param onSkip - optional callback for notes skipped because they could not
|
|
191
|
+
* be read; one unreadable note must not fail search for the whole vault.
|
|
152
192
|
* @returns a map from note id to search result.
|
|
153
193
|
*/
|
|
154
|
-
export declare function buildIndex(root: string, extensions: string[], indexHiddenDirs?: boolean): Promise<Map<string, IndexedNote>>;
|
|
194
|
+
export declare function buildIndex(root: string, extensions: string[], indexHiddenDirs?: boolean, onSkip?: (path: string, error: unknown) => void): Promise<Map<string, IndexedNote>>;
|
|
155
195
|
/**
|
|
156
196
|
* Register the `wiki_read`, `wiki_search`, and `wiki_write` tools on
|
|
157
197
|
* `ctx.tools`.
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Multi-layer lexical ranking for `wiki_search`: field-weighted BM25-style
|
|
3
|
+
* scoring over note id/title/backlinks/body, a verbatim-phrase bonus, and a
|
|
4
|
+
* link-graph boost for notes referenced by strong lexical hits. Substring
|
|
5
|
+
* matching (not tokenization) keeps CJK text working without a segmenter.
|
|
6
|
+
* @module @jacklika/dsh-tool-memory-filesystem/search
|
|
7
|
+
*/
|
|
8
|
+
import type { IndexedNote, SearchResult } from './types.ts';
|
|
9
|
+
/** A search hit carrying its fused relevance score. */
|
|
10
|
+
export interface RankedHit extends SearchResult {
|
|
11
|
+
/** Relevance score: BM25 sum (+phrase/graph bonuses), or an RRF score when semantic fusion ran. */
|
|
12
|
+
score: number;
|
|
13
|
+
}
|
|
14
|
+
/**
|
|
15
|
+
* Rank every note in `index` against `query`. Any note containing at least one
|
|
16
|
+
* query term (OR semantics) is a candidate; scores follow BM25 saturation with
|
|
17
|
+
* field weights (`id`/`title` ×3, backlinks ×1, body ×1), a phrase bonus for a
|
|
18
|
+
* verbatim multi-term match, and a graph boost for notes linked from a scored
|
|
19
|
+
* hit.
|
|
20
|
+
* @param index - note index built by {@link buildIndex}.
|
|
21
|
+
* @param query - raw query text.
|
|
22
|
+
* @returns hits sorted by descending score.
|
|
23
|
+
*/
|
|
24
|
+
export declare function rankNotes(index: Map<string, IndexedNote>, query: string): RankedHit[];
|
|
25
|
+
/**
|
|
26
|
+
* Locale-independent code-unit ordering — `localeCompare` without a locale
|
|
27
|
+
* follows the host runtime locale, which makes tied-score ordering differ
|
|
28
|
+
* across machines.
|
|
29
|
+
*/
|
|
30
|
+
export declare function codeUnitCompare(a: string, b: string): number;
|
|
31
|
+
/**
|
|
32
|
+
* Fuse the lexical ranking with a semantic-search ranking using reciprocal
|
|
33
|
+
* rank fusion (k = 60). Semantic-only notes are surfaced with their index
|
|
34
|
+
* metadata so the output shape stays uniform.
|
|
35
|
+
* @param lexical - hits from {@link rankNotes}.
|
|
36
|
+
* @param semanticIds - note ids ordered by descending semantic similarity.
|
|
37
|
+
* @param index - the note index for metadata lookup.
|
|
38
|
+
* @returns fused hits sorted by descending RRF score.
|
|
39
|
+
*/
|
|
40
|
+
export declare function fuseRankings(lexical: RankedHit[], semanticIds: string[], index: Map<string, IndexedNote>): RankedHit[];
|
|
41
|
+
//# sourceMappingURL=search.d.ts.map
|
package/lib/types/types.d.ts
CHANGED
|
@@ -20,6 +20,17 @@ export interface Note {
|
|
|
20
20
|
* note in between.
|
|
21
21
|
*/
|
|
22
22
|
version: string;
|
|
23
|
+
/**
|
|
24
|
+
* File modification time rendered as a `+08:00` timestamp. Only set on the
|
|
25
|
+
* top-level note returned by `wiki_read`.
|
|
26
|
+
*/
|
|
27
|
+
mtime?: string;
|
|
28
|
+
/**
|
|
29
|
+
* True when the file's `mtime` changed since this plugin last observed it —
|
|
30
|
+
* evidence that an external writer (e.g. Obsidian) touched the note. Only
|
|
31
|
+
* set on the top-level note returned by `wiki_read`.
|
|
32
|
+
*/
|
|
33
|
+
modifiedExternally?: boolean;
|
|
23
34
|
}
|
|
24
35
|
/** Result of a search across the vault. */
|
|
25
36
|
export interface SearchResult {
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@jacklika/dsh-tool-memory-filesystem",
|
|
3
3
|
"description": "Model-facing wiki/memory tools over a local Markdown vault: read, search, and write notes with Obsidian-style links and YAML frontmatter.",
|
|
4
|
-
"version": "0.1.3-alpha.
|
|
4
|
+
"version": "0.1.3-alpha.11",
|
|
5
5
|
"publishConfig": {
|
|
6
6
|
"access": "public"
|
|
7
7
|
},
|
|
@@ -32,11 +32,12 @@
|
|
|
32
32
|
"license": "MIT",
|
|
33
33
|
"dependencies": {
|
|
34
34
|
"@deepseek-ai/schemastery": "3.18.4",
|
|
35
|
-
"js-yaml": "^4.2.0"
|
|
35
|
+
"js-yaml": "^4.2.0",
|
|
36
|
+
"@jacklika/dsh-memory-time": "^0.1.0-alpha.0"
|
|
36
37
|
},
|
|
37
38
|
"peerDependencies": {
|
|
38
39
|
"@deepseek-ai/cordis": "4.0.4",
|
|
39
|
-
"@deepseek-ai/dsh-tools": ">=0.1.7-rc.1 <0.3.0",
|
|
40
|
-
"@deepseek-ai/dsh-util-values": ">=0.1.7-rc.1 <0.3.0"
|
|
40
|
+
"@deepseek-ai/dsh-tools": ">=0.1.7-rc.1 <0.3.0-0 || >=0.2.0-rc.1 <0.3.0-0",
|
|
41
|
+
"@deepseek-ai/dsh-util-values": ">=0.1.7-rc.1 <0.3.0-0 || >=0.2.0-rc.1 <0.3.0-0"
|
|
41
42
|
}
|
|
42
43
|
}
|