namidb-vault 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +226 -0
- package/SKILL.md +105 -0
- package/namidb-vault.config.example.json +6 -0
- package/package.json +44 -0
- package/sync.mjs +933 -0
package/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 NamiDB, Inc.
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
package/README.md
ADDED
|
@@ -0,0 +1,226 @@
|
|
|
1
|
+
# namidb-vault — sync a markdown vault to NamiDB Cloud
|
|
2
|
+
|
|
3
|
+
A self-contained AI-tool skill that pushes your **local** Obsidian/markdown
|
|
4
|
+
vault to your **NamiDB Cloud** namespace as a real graph, then lets your AI
|
|
5
|
+
tool query it back over MCP. The vault on disk stays the source of truth; the
|
|
6
|
+
cloud graph is a rebuildable index.
|
|
7
|
+
|
|
8
|
+
Why this beats a plain graph view: because the index lives in a real graph
|
|
9
|
+
database with semantic vector search, you get backlinks, multi-hop neighbor
|
|
10
|
+
traversal, orphan detection, shared-tag clustering, and (with a server-side
|
|
11
|
+
embedder) semantic "find related notes" — all as queries, from any AI tool.
|
|
12
|
+
|
|
13
|
+
## Install & run
|
|
14
|
+
|
|
15
|
+
Zero-install with `npx` (recommended) — works in any terminal and inside
|
|
16
|
+
Claude Code / Cursor / Codex:
|
|
17
|
+
|
|
18
|
+
```bash
|
|
19
|
+
# Get an API key in your NamiDB dashboard (API keys) and export it.
|
|
20
|
+
# NEVER paste the key into chat — keep it in the environment.
|
|
21
|
+
export NAMIDB_API_KEY='nk_live_...'
|
|
22
|
+
|
|
23
|
+
npx namidb-vault@latest \
|
|
24
|
+
--vault ./my-notes \
|
|
25
|
+
--api-url https://api.namidb.com \
|
|
26
|
+
--namespace my-vault
|
|
27
|
+
```
|
|
28
|
+
|
|
29
|
+
- `--dry-run` — parse + diff, print the plan, write nothing.
|
|
30
|
+
- `--no-prune` — additive push; never delete cloud notes whose files are gone.
|
|
31
|
+
- `--watch` — live re-sync on file changes (debounced).
|
|
32
|
+
|
|
33
|
+
**As a Claude Code skill** (so the agent runs it for you, proactively): drop
|
|
34
|
+
this folder into `~/.claude/skills/namidb-vault/` and Claude Code auto-loads
|
|
35
|
+
it — then just say *"sync my vault to namidb"*. See `SKILL.md`.
|
|
36
|
+
|
|
37
|
+
## What lands in the graph
|
|
38
|
+
|
|
39
|
+
Faithful to the NamiDB markdown mapping:
|
|
40
|
+
|
|
41
|
+
| Vault thing | Graph thing |
|
|
42
|
+
| --- | --- |
|
|
43
|
+
| every `.md` / `.markdown` file | a `Note` node (`title`, `path`, `body`, `key`, `body_hash`, + your frontmatter) |
|
|
44
|
+
| `[[wikilink]]`, `[text](note.md)` | `LINKS_TO` edge |
|
|
45
|
+
| `![[embed]]` | `EMBEDS` edge |
|
|
46
|
+
| `#tag`, frontmatter `tags:` | shared `:Tag` node + `:TAGGED` edge |
|
|
47
|
+
| nested tag `area/db` | ancestor `:Tag` + `:SUBTAG_OF` edge |
|
|
48
|
+
| a link to a note that doesn't exist | placeholder `:Note` (`placeholder: true`) |
|
|
49
|
+
|
|
50
|
+
Name resolution matches the engine: links resolve by **normalized basename**
|
|
51
|
+
with Latin diacritic folding, so `[[User Role]]`, `[[user-role]]` and
|
|
52
|
+
`user_role.md` collapse to one note, and `[[Matías]]` resolves to `matias.md`.
|
|
53
|
+
Links inside fenced/inline code are ignored.
|
|
54
|
+
|
|
55
|
+
**Embeddings are computed server-side.** This skill pushes note *text* only —
|
|
56
|
+
it never sends vectors and never reads or transmits an embedder API key. After
|
|
57
|
+
each successful sync it **auto-triggers a server-side embed** of the changed
|
|
58
|
+
notes (`POST /v1/embed`), so `vault_search` stays current with no extra step.
|
|
59
|
+
That trigger is best-effort: a missing embedder or an embed error never fails
|
|
60
|
+
the sync (the next sync re-triggers it). Semantic search lights up once you
|
|
61
|
+
configure an embedder on the namespace (in the dashboard → namespace detail →
|
|
62
|
+
**Semantic search**); links/tags/graph lenses work without one. When no embedder
|
|
63
|
+
is configured the sync prints `semantic search off (no embedder configured)`.
|
|
64
|
+
|
|
65
|
+
## Requirements
|
|
66
|
+
|
|
67
|
+
- Node.js >= 18 (uses built-in `fetch`, `crypto`, `fs/promises` — **zero npm deps**).
|
|
68
|
+
- A NamiDB project API key with write access to the target namespace.
|
|
69
|
+
|
|
70
|
+
## Config
|
|
71
|
+
|
|
72
|
+
Routing config lives in `namidb-vault.config.json` at your vault/repo root
|
|
73
|
+
(copy `namidb-vault.config.example.json`). It holds only non-secret values:
|
|
74
|
+
|
|
75
|
+
```json
|
|
76
|
+
{
|
|
77
|
+
"vaultDir": "./vault",
|
|
78
|
+
"apiUrl": "https://api.namidb.com",
|
|
79
|
+
"namespace": "my-vault"
|
|
80
|
+
}
|
|
81
|
+
```
|
|
82
|
+
|
|
83
|
+
The **API key is never stored in the file**. Export it as an environment
|
|
84
|
+
variable:
|
|
85
|
+
|
|
86
|
+
```bash
|
|
87
|
+
export NAMIDB_API_KEY='your-project-api-key'
|
|
88
|
+
```
|
|
89
|
+
|
|
90
|
+
The script reads `NAMIDB_API_URL`, `NAMIDB_API_KEY`, and `NAMIDB_NAMESPACE`
|
|
91
|
+
from the environment; CLI flags (`--api-url`, `--namespace`) override them.
|
|
92
|
+
|
|
93
|
+
## Run it
|
|
94
|
+
|
|
95
|
+
```bash
|
|
96
|
+
# Dry run — parse + diff, print the plan, write nothing
|
|
97
|
+
node sync.mjs --vault ./vault --dry-run
|
|
98
|
+
|
|
99
|
+
# Incremental sync (default): upsert notes, prune notes deleted from disk
|
|
100
|
+
node sync.mjs --vault ./vault --api-url https://api.namidb.com --namespace my-vault
|
|
101
|
+
|
|
102
|
+
# Additive only — never delete cloud notes
|
|
103
|
+
node sync.mjs --vault ./vault --no-prune
|
|
104
|
+
```
|
|
105
|
+
|
|
106
|
+
Ids are stable per note path (`uuid5` of the relative path), so re-running
|
|
107
|
+
upserts in place — sync is idempotent. A default sync **prunes**: cloud notes
|
|
108
|
+
whose files are gone from disk are `DETACH DELETE`d, and orphaned placeholder
|
|
109
|
+
stubs + unused tags are cleaned up, so the graph stays a faithful mirror.
|
|
110
|
+
Point `--vault` at the right directory before the first run against a
|
|
111
|
+
populated namespace; use `--no-prune` for a purely additive push.
|
|
112
|
+
|
|
113
|
+
## Live folder sync (`--watch`)
|
|
114
|
+
|
|
115
|
+
```bash
|
|
116
|
+
node sync.mjs --vault ./vault --watch
|
|
117
|
+
```
|
|
118
|
+
|
|
119
|
+
Watches the vault recursively and re-syncs on any `.md` change, debounced
|
|
120
|
+
(`--watch-debounce`, default 1500ms) and serialized so overlapping edits
|
|
121
|
+
coalesce into one pass. Each pass is the same incremental + prune sync, so the
|
|
122
|
+
graph tracks the folder live. Leave it running in a terminal while you write.
|
|
123
|
+
|
|
124
|
+
## Install in your AI tool
|
|
125
|
+
|
|
126
|
+
All four tools run the **same** `sync.mjs`. Drop this folder somewhere stable
|
|
127
|
+
(e.g. `~/.namidb/skills/namidb-vault/`) and reference it.
|
|
128
|
+
|
|
129
|
+
### Claude Code
|
|
130
|
+
|
|
131
|
+
This directory **is** a Claude Code skill — it ships `SKILL.md`. Install it:
|
|
132
|
+
|
|
133
|
+
```bash
|
|
134
|
+
mkdir -p ~/.claude/skills
|
|
135
|
+
cp -R ./namidb-vault ~/.claude/skills/
|
|
136
|
+
```
|
|
137
|
+
|
|
138
|
+
Then just ask: "sync my vault" / "push my notes to NamiDB". Claude reads
|
|
139
|
+
`SKILL.md`, finds your config, and runs the sync. (Set `NAMIDB_API_KEY` in
|
|
140
|
+
your shell first.)
|
|
141
|
+
|
|
142
|
+
### Cursor
|
|
143
|
+
|
|
144
|
+
Add a project rule that points the agent at the script. In
|
|
145
|
+
`.cursor/rules/namidb-vault.md`:
|
|
146
|
+
|
|
147
|
+
```md
|
|
148
|
+
When I ask to sync / push / index my vault, run:
|
|
149
|
+
node ~/.namidb/skills/namidb-vault/sync.mjs --vault <vaultDir> \
|
|
150
|
+
--api-url <apiUrl> --namespace <namespace>
|
|
151
|
+
The API key is in $NAMIDB_API_KEY — never print it or ask me to paste it.
|
|
152
|
+
Read namidb-vault.config.json for vaultDir/apiUrl/namespace.
|
|
153
|
+
```
|
|
154
|
+
|
|
155
|
+
### Codex CLI
|
|
156
|
+
|
|
157
|
+
Add to your `AGENTS.md` (or run directly):
|
|
158
|
+
|
|
159
|
+
```md
|
|
160
|
+
## Vault sync
|
|
161
|
+
To sync the markdown vault to NamiDB Cloud:
|
|
162
|
+
node ~/.namidb/skills/namidb-vault/sync.mjs --vault ./vault \
|
|
163
|
+
--api-url https://api.namidb.com --namespace my-vault
|
|
164
|
+
Key: $NAMIDB_API_KEY (env only, never echoed).
|
|
165
|
+
```
|
|
166
|
+
|
|
167
|
+
### Kimi (and other agent CLIs)
|
|
168
|
+
|
|
169
|
+
Any agent that can run a shell command works the same way — there is nothing
|
|
170
|
+
tool-specific in the script:
|
|
171
|
+
|
|
172
|
+
```bash
|
|
173
|
+
NAMIDB_API_KEY=… node ~/.namidb/skills/namidb-vault/sync.mjs \
|
|
174
|
+
--vault ./vault --api-url https://api.namidb.com --namespace my-vault
|
|
175
|
+
```
|
|
176
|
+
|
|
177
|
+
## Query the graph back (hosted MCP)
|
|
178
|
+
|
|
179
|
+
NamiDB Cloud hosts an MCP server in the gateway, authed by the **same project
|
|
180
|
+
API key**. Point your AI tool's MCP config at it to query the notes you just
|
|
181
|
+
synced — no extra service to run. Example MCP client config:
|
|
182
|
+
|
|
183
|
+
```json
|
|
184
|
+
{
|
|
185
|
+
"mcpServers": {
|
|
186
|
+
"namidb": {
|
|
187
|
+
"type": "http",
|
|
188
|
+
"url": "https://api.namidb.com/v1/mcp",
|
|
189
|
+
"headers": {
|
|
190
|
+
"Authorization": "Bearer ${NAMIDB_API_KEY}",
|
|
191
|
+
"X-NamiDB-Namespace": "my-vault"
|
|
192
|
+
}
|
|
193
|
+
}
|
|
194
|
+
}
|
|
195
|
+
}
|
|
196
|
+
```
|
|
197
|
+
|
|
198
|
+
Exposed tools (read-only over your namespace): `vault_search` (semantic; needs
|
|
199
|
+
a configured embedder), `backlinks`, `neighbors`, `orphans`, `shared_tags`,
|
|
200
|
+
and `get_note`, plus a raw `cypher` escape hatch. The same lenses the Obsidian
|
|
201
|
+
graph view can draw but cannot query:
|
|
202
|
+
|
|
203
|
+
```cypher
|
|
204
|
+
-- backlinks of a note (links + embeds)
|
|
205
|
+
MATCH (src:Note)-[:LINKS_TO|:EMBEDS]->(:Note {path: $path}) RETURN DISTINCT src
|
|
206
|
+
|
|
207
|
+
-- orphans: nothing references them and they reference nothing
|
|
208
|
+
MATCH (n:Note) WHERE NOT EXISTS((n)-[:LINKS_TO|:EMBEDS]-()) RETURN n
|
|
209
|
+
|
|
210
|
+
-- notes that share a tag
|
|
211
|
+
MATCH (:Note {path: $path})-[:TAGGED]->(:Tag)<-[:TAGGED]-(o:Note) RETURN DISTINCT o
|
|
212
|
+
```
|
|
213
|
+
|
|
214
|
+
## Notes & limits
|
|
215
|
+
|
|
216
|
+
- The frontmatter parser handles the shapes vaults actually use (scalars,
|
|
217
|
+
flow/block lists, one-level maps, last-wins duplicate keys). Deeply nested
|
|
218
|
+
YAML is JSON-stringified onto the note property (the `:Tag`/`LINKS_TO`
|
|
219
|
+
relationships carry the structure that matters for queries).
|
|
220
|
+
- List/map frontmatter values are stored as a JSON string on the `Note` (the
|
|
221
|
+
bulk endpoint rejects nested objects); scalars pass through typed.
|
|
222
|
+
- `body_hash` is a stable per-note change marker (blake2b512 over the file
|
|
223
|
+
bytes). The server-side embedder compares it to decide which notes to
|
|
224
|
+
re-embed, so only changed notes are re-embedded. This skill always pushes the
|
|
225
|
+
full current vault (cheap: a vault is small, and bulk upsert is one commit
|
|
226
|
+
per chunk).
|
package/SKILL.md
ADDED
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: namidb-vault
|
|
3
|
+
description: >-
|
|
4
|
+
Sync a local Obsidian/markdown vault to a NamiDB Cloud namespace as a
|
|
5
|
+
queryable graph (Note nodes; LINKS_TO/EMBEDS/TAGGED/SUBTAG_OF edges).
|
|
6
|
+
Use this whenever the user asks to "sync my vault", "push my notes",
|
|
7
|
+
"index my markdown", "rebuild the note graph", or after they add,
|
|
8
|
+
edit, rename, or delete `.md` files and want the cloud graph to match.
|
|
9
|
+
The vault on disk stays the source of truth; the cloud graph is a
|
|
10
|
+
rebuildable index that powers backlinks, neighbor traversal, orphan
|
|
11
|
+
detection, shared-tag queries, and (when an embedder is configured
|
|
12
|
+
server-side) semantic search over the same notes.
|
|
13
|
+
version: 1.0.0
|
|
14
|
+
allowed-tools: Bash(node:*), Read, Glob
|
|
15
|
+
---
|
|
16
|
+
|
|
17
|
+
# NamiDB vault sync
|
|
18
|
+
|
|
19
|
+
Push a local markdown vault to a NamiDB Cloud namespace. The cloud graph
|
|
20
|
+
mirrors the vault: every `.md` file is a `Note`, every `[[wikilink]]` /
|
|
21
|
+
`[text](note.md)` is a `LINKS_TO` edge, every `![[embed]]` is an `EMBEDS`
|
|
22
|
+
edge, and `#tags` + frontmatter `tags` become shared `:Tag` nodes joined by
|
|
23
|
+
`:TAGGED` (nested tags add a `:SUBTAG_OF` tree). Embeddings are computed
|
|
24
|
+
server-side, so this skill never sends vectors and never touches an embedder
|
|
25
|
+
key — it pushes note text only, then **auto-triggers a server-side embed** of
|
|
26
|
+
the changed notes after a successful sync. Semantic search (`vault_search`)
|
|
27
|
+
lights up only once an embedder is configured for the namespace in the
|
|
28
|
+
dashboard; if none is, the sync still succeeds and reports that semantic search
|
|
29
|
+
is off.
|
|
30
|
+
|
|
31
|
+
## When to run this
|
|
32
|
+
|
|
33
|
+
Run a sync when the user:
|
|
34
|
+
|
|
35
|
+
- asks to "sync", "push", "upload", "index", or "rebuild" their vault / notes,
|
|
36
|
+
- has just created, edited, renamed, moved, or deleted `.md` files and wants
|
|
37
|
+
the graph to reflect disk,
|
|
38
|
+
- asks a graph/semantic question about their notes and the graph looks stale
|
|
39
|
+
(when in doubt, sync first — it is incremental and idempotent).
|
|
40
|
+
|
|
41
|
+
Do NOT run it on every file save unless the user explicitly asks for live
|
|
42
|
+
sync; for that, point them at `--watch` (see the README) rather than looping
|
|
43
|
+
the tool yourself.
|
|
44
|
+
|
|
45
|
+
## How to run it
|
|
46
|
+
|
|
47
|
+
1. Find the config. Look for `namidb-vault.config.json` at the vault root (or
|
|
48
|
+
the repo root). If present, read it for `vaultDir`, `apiUrl`, and
|
|
49
|
+
`namespace`. The API key is **only** ever read from the `NAMIDB_API_KEY`
|
|
50
|
+
environment variable — never from the config file, never from chat, and
|
|
51
|
+
never echoed back.
|
|
52
|
+
2. Resolve config -> env. The script reads `NAMIDB_API_URL`,
|
|
53
|
+
`NAMIDB_API_KEY`, and `NAMIDB_NAMESPACE` from the environment; CLI flags
|
|
54
|
+
override them. Prefer flags sourced from the config file for URL +
|
|
55
|
+
namespace, and leave the key in the environment.
|
|
56
|
+
3. Run the sync from the directory that holds this skill's `sync.mjs`:
|
|
57
|
+
|
|
58
|
+
```bash
|
|
59
|
+
node sync.mjs --vault "<vaultDir>" --api-url "<apiUrl>" --namespace "<ns>"
|
|
60
|
+
```
|
|
61
|
+
|
|
62
|
+
The key is taken from `NAMIDB_API_KEY` in the environment. If it is unset,
|
|
63
|
+
tell the user to export it (give them the variable name, not a value) and
|
|
64
|
+
stop — do not ask them to paste the key into the conversation.
|
|
65
|
+
4. Report the printed summary (notes added/modified/unchanged/deleted, edges,
|
|
66
|
+
tags, placeholders). After a successful sync the script auto-triggers a
|
|
67
|
+
server-side embed and prints either `embedded N, skipped M` or, when no
|
|
68
|
+
embedder is configured, `semantic search off (no embedder configured)` —
|
|
69
|
+
relay that line too. On a non-zero exit, surface the error message; the
|
|
70
|
+
script prints the failing note path / chunk, not secrets.
|
|
71
|
+
|
|
72
|
+
## Hard rules
|
|
73
|
+
|
|
74
|
+
- NEVER print, log, echo, or store `NAMIDB_API_KEY` (or any value that looks
|
|
75
|
+
like an API key). If the user pastes a key into chat, treat it as
|
|
76
|
+
compromised: tell them to rotate it and set it as an env var instead.
|
|
77
|
+
- NEVER hardcode the API URL, namespace, or bucket — always read them from
|
|
78
|
+
config/flags/env.
|
|
79
|
+
- The vault is the source of truth. This skill only writes to the cloud
|
|
80
|
+
graph; it never edits `.md` files.
|
|
81
|
+
- A `--prune` sync (the default) deletes cloud notes whose files are gone
|
|
82
|
+
from disk. If the user points it at the wrong/empty directory it will
|
|
83
|
+
prune the namespace; confirm the `--vault` path before a first run against
|
|
84
|
+
a populated namespace, and offer `--no-prune` for an additive push.
|
|
85
|
+
|
|
86
|
+
## Quick reference
|
|
87
|
+
|
|
88
|
+
```bash
|
|
89
|
+
# Dry run — parse + diff, print the plan, write nothing
|
|
90
|
+
node sync.mjs --vault ./vault --dry-run
|
|
91
|
+
|
|
92
|
+
# Incremental sync (default): upsert changed notes, prune deleted ones
|
|
93
|
+
node sync.mjs --vault ./vault
|
|
94
|
+
|
|
95
|
+
# Additive only — never delete cloud notes
|
|
96
|
+
node sync.mjs --vault ./vault --no-prune
|
|
97
|
+
|
|
98
|
+
# Live sync — re-sync on file changes (debounced)
|
|
99
|
+
node sync.mjs --vault ./vault --watch
|
|
100
|
+
```
|
|
101
|
+
|
|
102
|
+
After a sync, query the graph through the hosted NamiDB MCP server (see
|
|
103
|
+
`README.md` -> "Query the graph back"): `vault_search`, `backlinks`,
|
|
104
|
+
`neighbors`, `orphans`, `shared_tags`, `get_note`, and `cypher` are exposed as
|
|
105
|
+
MCP tools authed by the same project API key.
|
package/package.json
ADDED
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "namidb-vault",
|
|
3
|
+
"version": "1.0.0",
|
|
4
|
+
"description": "Sync a local Obsidian/markdown vault to NamiDB Cloud as a queryable graph for AI coding agents (Claude Code, Cursor, Codex). Note nodes + LINKS_TO/EMBEDS/TAGGED edges; backlinks, n-hop, orphans, shared-tags and semantic search over MCP. Zero-dependency.",
|
|
5
|
+
"type": "module",
|
|
6
|
+
"bin": {
|
|
7
|
+
"namidb-vault": "sync.mjs"
|
|
8
|
+
},
|
|
9
|
+
"files": [
|
|
10
|
+
"sync.mjs",
|
|
11
|
+
"SKILL.md",
|
|
12
|
+
"README.md",
|
|
13
|
+
"namidb-vault.config.example.json",
|
|
14
|
+
"LICENSE"
|
|
15
|
+
],
|
|
16
|
+
"engines": {
|
|
17
|
+
"node": ">=18"
|
|
18
|
+
},
|
|
19
|
+
"keywords": [
|
|
20
|
+
"namidb",
|
|
21
|
+
"obsidian",
|
|
22
|
+
"markdown",
|
|
23
|
+
"vault",
|
|
24
|
+
"knowledge-graph",
|
|
25
|
+
"graph-database",
|
|
26
|
+
"mcp",
|
|
27
|
+
"model-context-protocol",
|
|
28
|
+
"claude-code",
|
|
29
|
+
"cursor",
|
|
30
|
+
"codex",
|
|
31
|
+
"semantic-search",
|
|
32
|
+
"rag",
|
|
33
|
+
"notes"
|
|
34
|
+
],
|
|
35
|
+
"homepage": "https://namidb.com",
|
|
36
|
+
"bugs": {
|
|
37
|
+
"url": "https://namidb.com/contact"
|
|
38
|
+
},
|
|
39
|
+
"license": "MIT",
|
|
40
|
+
"author": "NamiDB, Inc.",
|
|
41
|
+
"publishConfig": {
|
|
42
|
+
"access": "public"
|
|
43
|
+
}
|
|
44
|
+
}
|
package/sync.mjs
ADDED
|
@@ -0,0 +1,933 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
/**
|
|
3
|
+
* namidb vault sync — push a local Obsidian/markdown vault to a NamiDB Cloud
|
|
4
|
+
* namespace as a queryable graph. Zero dependencies (Node >= 18 built-ins).
|
|
5
|
+
*
|
|
6
|
+
* The cloud graph mirrors the vault, faithful to the engine's namidb-markdown
|
|
7
|
+
* mapping:
|
|
8
|
+
* - every `.md`/`.markdown` file -> a `Note` node
|
|
9
|
+
* props: title, path, body, key, body_hash, + typed frontmatter
|
|
10
|
+
* - `[[wikilink]]` / `[text](note.md)` -> `LINKS_TO` edge
|
|
11
|
+
* - `![[embed]]` -> `EMBEDS` edge
|
|
12
|
+
* - `#tag` + frontmatter `tags` -> shared `:Tag` node + `:TAGGED` edge
|
|
13
|
+
* - nested tag `area/db` -> ancestor `:Tag` + `:SUBTAG_OF` edge
|
|
14
|
+
* - dangling reference -> placeholder `:Note` (placeholder=true)
|
|
15
|
+
*
|
|
16
|
+
* Node identity is STABLE per note path: uuid5(DNS, "namidb-note:" + relPath).
|
|
17
|
+
* Tags/placeholders get uuid5("namidb-tag:" + name) / uuid5("namidb-note-key:"
|
|
18
|
+
* + key). These ids are owned by THIS skill (the engine loader is not on this
|
|
19
|
+
* path); re-running upserts in place. Link/embed resolution is by normalized
|
|
20
|
+
* basename key (diacritic-folded), exactly like the engine, so a `[[User Role]]`
|
|
21
|
+
* resolves to `user-role.md` regardless of folder.
|
|
22
|
+
*
|
|
23
|
+
* Embeddings are computed SERVER-SIDE: this script pushes note TEXT only. It
|
|
24
|
+
* never sends vectors and never reads or transmits an embedder API key. After a
|
|
25
|
+
* successful push (+ prune), it triggers the gateway to (re-)embed the changed
|
|
26
|
+
* Notes via POST /v1/embed (best-effort; a missing embedder or embed error never
|
|
27
|
+
* fails the sync). Semantic search needs an embedder configured in the dashboard.
|
|
28
|
+
*
|
|
29
|
+
* Config (env; flags override):
|
|
30
|
+
* NAMIDB_API_URL e.g. https://api.namidb.com (--api-url)
|
|
31
|
+
* NAMIDB_API_KEY the project API key, Bearer (env only; never a flag)
|
|
32
|
+
* NAMIDB_NAMESPACE the target namespace name (--namespace)
|
|
33
|
+
*
|
|
34
|
+
* Usage:
|
|
35
|
+
* export NAMIDB_API_KEY='nk_live_...'
|
|
36
|
+
* node sync.mjs --vault ./vault --api-url https://api.namidb.com --namespace my-vault
|
|
37
|
+
* node sync.mjs --vault ./vault --dry-run
|
|
38
|
+
* node sync.mjs --vault ./vault --no-prune
|
|
39
|
+
* node sync.mjs --vault ./vault --watch
|
|
40
|
+
*/
|
|
41
|
+
import { createHash } from "node:crypto";
|
|
42
|
+
import { readFile, readdir, stat } from "node:fs/promises";
|
|
43
|
+
import { watch } from "node:fs";
|
|
44
|
+
import path from "node:path";
|
|
45
|
+
import { performance } from "node:perf_hooks";
|
|
46
|
+
|
|
47
|
+
// ─── CLI / config ────────────────────────────────────────────────────────
|
|
48
|
+
const args = parseArgs(process.argv.slice(2));
|
|
49
|
+
if (args.help || args.h) {
|
|
50
|
+
printUsage();
|
|
51
|
+
process.exit(0);
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
const VAULT = path.resolve(args.vault || process.env.NAMIDB_VAULT || ".");
|
|
55
|
+
const API_URL = String(
|
|
56
|
+
args["api-url"] || process.env.NAMIDB_API_URL || "",
|
|
57
|
+
).replace(/\/+$/, "");
|
|
58
|
+
const NAMESPACE = String(args.namespace || process.env.NAMIDB_NAMESPACE || "");
|
|
59
|
+
const API_KEY = process.env.NAMIDB_API_KEY || "";
|
|
60
|
+
const CHUNK = Math.max(50, Math.min(20_000, Number(args.chunk ?? 2000)));
|
|
61
|
+
const DRY_RUN = Boolean(args["dry-run"]);
|
|
62
|
+
const PRUNE = args["no-prune"] ? false : true;
|
|
63
|
+
const WATCH = Boolean(args.watch);
|
|
64
|
+
const REQ_TIMEOUT_MS = Math.max(5000, Number(args.timeout ?? 120_000));
|
|
65
|
+
const DEBOUNCE_MS = Math.max(200, Number(args["watch-debounce"] ?? 1500));
|
|
66
|
+
|
|
67
|
+
if (!API_URL || !NAMESPACE) {
|
|
68
|
+
fail(
|
|
69
|
+
"missing config: set --api-url/NAMIDB_API_URL and --namespace/NAMIDB_NAMESPACE\n" +
|
|
70
|
+
" (the API key is read from the NAMIDB_API_KEY environment variable)",
|
|
71
|
+
);
|
|
72
|
+
}
|
|
73
|
+
if (!DRY_RUN && !API_KEY) {
|
|
74
|
+
fail(
|
|
75
|
+
"NAMIDB_API_KEY is not set in the environment (do not pass it as a flag or paste it in chat)",
|
|
76
|
+
);
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
// ─── Graph constants (must match the engine's namidb-markdown shapes) ─────
|
|
80
|
+
const NOTE_LABEL = "Note";
|
|
81
|
+
const TAG_LABEL = "Tag";
|
|
82
|
+
const LINKS_TO = "LINKS_TO";
|
|
83
|
+
const EMBEDS = "EMBEDS";
|
|
84
|
+
const TAGGED = "TAGGED";
|
|
85
|
+
const SUBTAG_OF = "SUBTAG_OF";
|
|
86
|
+
// Engine-owned property names an author may not override (stem-derived `key`,
|
|
87
|
+
// byte-derived `body_hash`) plus the engine-reserved set. Dropped from
|
|
88
|
+
// frontmatter so they cannot shadow a managed column. The server-side embed
|
|
89
|
+
// change-detection compares against `body_hash`, so this skill writes a stable
|
|
90
|
+
// hash under that exact name; every Note carries one so the server re-embeds
|
|
91
|
+
// only the notes whose `body_hash` changed.
|
|
92
|
+
const RESERVED_PROPS = new Set([
|
|
93
|
+
"key",
|
|
94
|
+
"body_hash",
|
|
95
|
+
"node_id",
|
|
96
|
+
"tombstone",
|
|
97
|
+
"lsn",
|
|
98
|
+
]);
|
|
99
|
+
function isReservedProp(name) {
|
|
100
|
+
return (
|
|
101
|
+
RESERVED_PROPS.has(name) ||
|
|
102
|
+
name.startsWith("__") ||
|
|
103
|
+
name.startsWith("prop_")
|
|
104
|
+
);
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
// ─── Stable ids: RFC-4122 v5 (SHA-1) over the DNS namespace ───────────────
|
|
108
|
+
const NS_DNS = "6ba7b810-9dad-11d1-80b4-00c04fd430c8";
|
|
109
|
+
function uuid5(name) {
|
|
110
|
+
const nsBytes = Buffer.from(NS_DNS.replace(/-/g, ""), "hex");
|
|
111
|
+
const hash = createHash("sha1")
|
|
112
|
+
.update(nsBytes)
|
|
113
|
+
.update(Buffer.from(name, "utf8"))
|
|
114
|
+
.digest();
|
|
115
|
+
const b = Buffer.from(hash.subarray(0, 16));
|
|
116
|
+
b[6] = (b[6] & 0x0f) | 0x50; // version 5
|
|
117
|
+
b[8] = (b[8] & 0x3f) | 0x80; // RFC-4122 variant
|
|
118
|
+
const h = b.toString("hex");
|
|
119
|
+
return `${h.slice(0, 8)}-${h.slice(8, 12)}-${h.slice(12, 16)}-${h.slice(
|
|
120
|
+
16,
|
|
121
|
+
20,
|
|
122
|
+
)}-${h.slice(20, 32)}`;
|
|
123
|
+
}
|
|
124
|
+
const noteId = (relPath) => uuid5(`namidb-note:${relPath}`);
|
|
125
|
+
const noteKeyId = (key) => uuid5(`namidb-note-key:${key}`); // placeholders resolve here
|
|
126
|
+
const tagId = (name) => uuid5(`namidb-tag:${name}`);
|
|
127
|
+
|
|
128
|
+
// ─── Key normalization + diacritic fold (mirror crates/.../src/id.rs) ─────
|
|
129
|
+
const FOLD = {
|
|
130
|
+
À: "A", Á: "A", Â: "A", Ã: "A", Ä: "A", Å: "A",
|
|
131
|
+
à: "a", á: "a", â: "a", ã: "a", ä: "a", å: "a",
|
|
132
|
+
Ç: "C", ç: "c",
|
|
133
|
+
È: "E", É: "E", Ê: "E", Ë: "E", è: "e", é: "e", ê: "e", ë: "e",
|
|
134
|
+
Ì: "I", Í: "I", Î: "I", Ï: "I", ì: "i", í: "i", î: "i", ï: "i",
|
|
135
|
+
Ñ: "N", ñ: "n",
|
|
136
|
+
Ò: "O", Ó: "O", Ô: "O", Õ: "O", Ö: "O", ò: "o", ó: "o", ô: "o", õ: "o", ö: "o",
|
|
137
|
+
Ù: "U", Ú: "U", Û: "U", Ü: "U", ù: "u", ú: "u", û: "u", ü: "u",
|
|
138
|
+
Ý: "Y", ý: "y", ÿ: "y",
|
|
139
|
+
};
|
|
140
|
+
function normalizeKey(name) {
|
|
141
|
+
let out = "";
|
|
142
|
+
let prevSep = false;
|
|
143
|
+
for (const ch of name.trim()) {
|
|
144
|
+
if (ch === "-" || ch === "_" || /\s/u.test(ch)) {
|
|
145
|
+
if (out.length && !prevSep) {
|
|
146
|
+
out += "-";
|
|
147
|
+
prevSep = true;
|
|
148
|
+
}
|
|
149
|
+
} else {
|
|
150
|
+
out += (FOLD[ch] ?? ch).toLowerCase();
|
|
151
|
+
prevSep = false;
|
|
152
|
+
}
|
|
153
|
+
}
|
|
154
|
+
return out.replace(/-+$/u, "");
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
// ─── Code-fence + inline-code masking (so links inside code don't count) ──
|
|
158
|
+
// Replace fenced blocks (``` / ~~~) and inline `code` spans with spaces of the
|
|
159
|
+
// same length, preserving offsets like the engine's pulldown-cmark masker.
|
|
160
|
+
function maskCode(body) {
|
|
161
|
+
const lines = body.split("\n");
|
|
162
|
+
let fence = null; // current fence marker char-run, or null
|
|
163
|
+
const out = [];
|
|
164
|
+
for (const line of lines) {
|
|
165
|
+
const fm = line.match(/^\s*(```+|~~~+)/);
|
|
166
|
+
if (fence) {
|
|
167
|
+
out.push(" ".repeat(line.length));
|
|
168
|
+
if (fm && line.trim().startsWith(fence[0]) && line.trim().length >= fence.length)
|
|
169
|
+
fence = null;
|
|
170
|
+
continue;
|
|
171
|
+
}
|
|
172
|
+
if (fm) {
|
|
173
|
+
fence = fm[1];
|
|
174
|
+
out.push(" ".repeat(line.length));
|
|
175
|
+
continue;
|
|
176
|
+
}
|
|
177
|
+
// Inline code spans: blank out backtick-delimited runs.
|
|
178
|
+
out.push(line.replace(/`+[^`]*`+/g, (m) => " ".repeat(m.length)));
|
|
179
|
+
}
|
|
180
|
+
return out.join("\n");
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
// ─── Frontmatter (minimal YAML: scalars, flow/block lists, one-level maps) ─
|
|
184
|
+
function splitFrontmatter(raw) {
|
|
185
|
+
const m = raw.match(/^---\r?\n([\s\S]*?)\r?\n---\r?\n?/);
|
|
186
|
+
if (!m) return [null, raw];
|
|
187
|
+
return [m[1], raw.slice(m[0].length)];
|
|
188
|
+
}
|
|
189
|
+
function parseScalar(s) {
|
|
190
|
+
const t = s.trim();
|
|
191
|
+
if (t === "") return null;
|
|
192
|
+
if (
|
|
193
|
+
(t.startsWith('"') && t.endsWith('"')) ||
|
|
194
|
+
(t.startsWith("'") && t.endsWith("'"))
|
|
195
|
+
)
|
|
196
|
+
return t.slice(1, -1);
|
|
197
|
+
if (t === "true") return true;
|
|
198
|
+
if (t === "false") return false;
|
|
199
|
+
if (t === "null" || t === "~") return null;
|
|
200
|
+
if (/^-?\d+$/.test(t)) return Number.parseInt(t, 10);
|
|
201
|
+
if (/^-?\d*\.\d+$/.test(t)) return Number.parseFloat(t);
|
|
202
|
+
return t;
|
|
203
|
+
}
|
|
204
|
+
function parseFlowList(s) {
|
|
205
|
+
return s
|
|
206
|
+
.slice(1, -1)
|
|
207
|
+
.split(",")
|
|
208
|
+
.map((x) => parseScalar(x))
|
|
209
|
+
.filter((x) => x !== null || x === 0 || x === false);
|
|
210
|
+
}
|
|
211
|
+
// Best-effort YAML for the shapes vaults actually use. Top-level keys only;
|
|
212
|
+
// `key:` then either an inline scalar/flow-list or a block list of `- item`.
|
|
213
|
+
function parseFrontmatter(yaml) {
|
|
214
|
+
if (yaml == null) return {};
|
|
215
|
+
const props = {};
|
|
216
|
+
const lines = yaml.split(/\r?\n/);
|
|
217
|
+
for (let i = 0; i < lines.length; i++) {
|
|
218
|
+
const line = lines[i];
|
|
219
|
+
if (!line.trim() || line.trim().startsWith("#")) continue;
|
|
220
|
+
if (/^\s/.test(line)) continue; // belongs to a block value handled below
|
|
221
|
+
const cm = line.match(/^([^:#][^:]*):(.*)$/);
|
|
222
|
+
if (!cm) continue;
|
|
223
|
+
const key = cm[1].trim();
|
|
224
|
+
const rest = cm[2].trim();
|
|
225
|
+
if (!key) continue;
|
|
226
|
+
let value;
|
|
227
|
+
if (rest === "") {
|
|
228
|
+
// Block list: consume following `- item` / indented lines.
|
|
229
|
+
const items = [];
|
|
230
|
+
let isList = false;
|
|
231
|
+
while (i + 1 < lines.length && /^\s+\S/.test(lines[i + 1])) {
|
|
232
|
+
const next = lines[++i].trim();
|
|
233
|
+
if (next.startsWith("- ")) {
|
|
234
|
+
isList = true;
|
|
235
|
+
items.push(parseScalar(next.slice(2)));
|
|
236
|
+
}
|
|
237
|
+
}
|
|
238
|
+
value = isList ? items : null;
|
|
239
|
+
} else if (rest.startsWith("[") && rest.endsWith("]")) {
|
|
240
|
+
value = parseFlowList(rest);
|
|
241
|
+
} else {
|
|
242
|
+
value = parseScalar(rest);
|
|
243
|
+
}
|
|
244
|
+
if (value === null && rest !== "") continue;
|
|
245
|
+
if (value === null && rest === "") continue;
|
|
246
|
+
props[key] = value; // last-wins on duplicate top-level keys, like Obsidian
|
|
247
|
+
}
|
|
248
|
+
return props;
|
|
249
|
+
}
|
|
250
|
+
|
|
251
|
+
// ─── Wikilink / embed / markdown-link / tag extraction ───────────────────
|
|
252
|
+
const WIKILINK_RE = /(!?)\[\[([^[\]\r\n]+?)\]\]/g;
|
|
253
|
+
const MD_LINK_RE = /(!?)\[[^\]]*\]\(([^)\s]+)(?:\s+"[^"]*")?\)/g;
|
|
254
|
+
const TAG_RE = /(?:^|\s)#([\p{L}\p{N}_/-]*[\p{L}_/-][\p{L}\p{N}_/-]*)/gu;
|
|
255
|
+
|
|
256
|
+
function linkTargetKey(inner) {
|
|
257
|
+
const beforeAlias = inner.split("|")[0];
|
|
258
|
+
const beforeAnchor = beforeAlias.split("#")[0].trim();
|
|
259
|
+
if (!beforeAnchor) return null;
|
|
260
|
+
const base = beforeAnchor.split("/").pop();
|
|
261
|
+
const k = normalizeKey(base);
|
|
262
|
+
return k || null;
|
|
263
|
+
}
|
|
264
|
+
function classifyWikilinks(masked) {
|
|
265
|
+
const links = [];
|
|
266
|
+
const embeds = [];
|
|
267
|
+
const sl = new Set();
|
|
268
|
+
const se = new Set();
|
|
269
|
+
for (const m of masked.matchAll(WIKILINK_RE)) {
|
|
270
|
+
const k = linkTargetKey(m[2]);
|
|
271
|
+
if (!k) continue;
|
|
272
|
+
if (m[1] === "!") {
|
|
273
|
+
if (!se.has(k)) {
|
|
274
|
+
se.add(k);
|
|
275
|
+
embeds.push(k);
|
|
276
|
+
}
|
|
277
|
+
} else if (!sl.has(k)) {
|
|
278
|
+
sl.add(k);
|
|
279
|
+
links.push(k);
|
|
280
|
+
}
|
|
281
|
+
}
|
|
282
|
+
return { links, embeds };
|
|
283
|
+
}
|
|
284
|
+
function mdLinkTargetKey(dest) {
|
|
285
|
+
const d = dest.trim();
|
|
286
|
+
if (!d || d.startsWith("#") || d.startsWith("//") || d.includes("://"))
|
|
287
|
+
return null;
|
|
288
|
+
const colon = d.indexOf(":");
|
|
289
|
+
if (colon > 0) {
|
|
290
|
+
const scheme = d.slice(0, colon);
|
|
291
|
+
if (/^[A-Za-z][A-Za-z0-9+.-]*$/.test(scheme)) return null; // mailto:, tel:, ...
|
|
292
|
+
}
|
|
293
|
+
const noFrag = d.split(/[#?]/)[0];
|
|
294
|
+
const base = noFrag.split(/[/\\]/).pop();
|
|
295
|
+
const decoded = decodeURIComponentSafe(base);
|
|
296
|
+
const lower = decoded.toLowerCase();
|
|
297
|
+
let stem;
|
|
298
|
+
if (lower.endsWith(".md")) stem = decoded.slice(0, -3);
|
|
299
|
+
else if (lower.endsWith(".markdown")) stem = decoded.slice(0, -9);
|
|
300
|
+
else return null;
|
|
301
|
+
if (/[:/\\#?]/.test(stem)) return null;
|
|
302
|
+
const k = normalizeKey(stem);
|
|
303
|
+
return k || null;
|
|
304
|
+
}
|
|
305
|
+
function decodeURIComponentSafe(s) {
|
|
306
|
+
try {
|
|
307
|
+
return decodeURIComponent(s);
|
|
308
|
+
} catch {
|
|
309
|
+
return s;
|
|
310
|
+
}
|
|
311
|
+
}
|
|
312
|
+
function extractMarkdownLinks(masked) {
|
|
313
|
+
const out = [];
|
|
314
|
+
const seen = new Set();
|
|
315
|
+
for (const m of masked.matchAll(MD_LINK_RE)) {
|
|
316
|
+
if (m[1] === "!") continue; // image embed, not a note link
|
|
317
|
+
const k = mdLinkTargetKey(m[2]);
|
|
318
|
+
if (k && !seen.has(k)) {
|
|
319
|
+
seen.add(k);
|
|
320
|
+
out.push(k);
|
|
321
|
+
}
|
|
322
|
+
}
|
|
323
|
+
return out;
|
|
324
|
+
}
|
|
325
|
+
function extractTags(masked) {
|
|
326
|
+
const out = [];
|
|
327
|
+
const seen = new Set();
|
|
328
|
+
for (const m of masked.matchAll(TAG_RE)) {
|
|
329
|
+
if (!seen.has(m[1])) {
|
|
330
|
+
seen.add(m[1]);
|
|
331
|
+
out.push(m[1]);
|
|
332
|
+
}
|
|
333
|
+
}
|
|
334
|
+
return out;
|
|
335
|
+
}
|
|
336
|
+
// A frontmatter value that IS a single whole `[[wikilink]]` (a link field).
|
|
337
|
+
function frontmatterLinkTarget(value) {
|
|
338
|
+
if (typeof value !== "string") return null;
|
|
339
|
+
const t = value.trim();
|
|
340
|
+
if (!t.startsWith("[[") || !t.endsWith("]]")) return null;
|
|
341
|
+
const inner = t.slice(2, -2);
|
|
342
|
+
if (inner.includes("[[") || inner.includes("]]") || /[\r\n]/.test(inner))
|
|
343
|
+
return null;
|
|
344
|
+
return linkTargetKey(inner);
|
|
345
|
+
}
|
|
346
|
+
|
|
347
|
+
// ─── Parse one note ──────────────────────────────────────────────────────
|
|
348
|
+
function noteTitle(relPath) {
|
|
349
|
+
const base = relPath.split("/").pop();
|
|
350
|
+
return base.replace(/\.(md|markdown)$/i, "");
|
|
351
|
+
}
|
|
352
|
+
function parseNote(relPath, raw) {
|
|
353
|
+
const [yaml, body] = splitFrontmatter(raw);
|
|
354
|
+
const fm = parseFrontmatter(yaml);
|
|
355
|
+
|
|
356
|
+
const title = noteTitle(relPath);
|
|
357
|
+
const key = normalizeKey(title);
|
|
358
|
+
const masked = maskCode(body);
|
|
359
|
+
|
|
360
|
+
// Properties: typed frontmatter (reserved dropped) + engine-owned fields.
|
|
361
|
+
const properties = {};
|
|
362
|
+
for (const [k, v] of Object.entries(fm)) {
|
|
363
|
+
if (isReservedProp(k)) continue;
|
|
364
|
+
properties[k] = v;
|
|
365
|
+
}
|
|
366
|
+
// `title` defers to a STRING frontmatter title; else the file stem.
|
|
367
|
+
if (typeof properties.title !== "string") properties.title = title;
|
|
368
|
+
properties.path = relPath;
|
|
369
|
+
properties.body = body;
|
|
370
|
+
properties.key = key;
|
|
371
|
+
// Stable per-note change marker. The server-side embed change-detection
|
|
372
|
+
// copies whatever value is in `body_hash`, so the PROPERTY NAME must match;
|
|
373
|
+
// the algorithm only needs to be stable (Node stdlib blake2b512 over the raw
|
|
374
|
+
// file bytes is fine). A changed hash is what tells the server to re-embed.
|
|
375
|
+
properties.body_hash = createHash("blake2b512")
|
|
376
|
+
.update(raw, "utf8")
|
|
377
|
+
.digest("hex");
|
|
378
|
+
|
|
379
|
+
// Merge inline #tags into the `tags` property (frontmatter first, dedup).
|
|
380
|
+
const inlineTags = extractTags(masked);
|
|
381
|
+
let tagList = [];
|
|
382
|
+
if (Array.isArray(properties.tags))
|
|
383
|
+
tagList = properties.tags.filter((t) => typeof t === "string");
|
|
384
|
+
else if (typeof properties.tags === "string") tagList = [properties.tags];
|
|
385
|
+
if (inlineTags.length) {
|
|
386
|
+
const present = new Set(tagList);
|
|
387
|
+
for (const t of inlineTags)
|
|
388
|
+
if (!present.has(t)) {
|
|
389
|
+
present.add(t);
|
|
390
|
+
tagList.push(t);
|
|
391
|
+
}
|
|
392
|
+
if (
|
|
393
|
+
properties.tags === undefined ||
|
|
394
|
+
Array.isArray(properties.tags) ||
|
|
395
|
+
typeof properties.tags === "string"
|
|
396
|
+
)
|
|
397
|
+
properties.tags = tagList.slice();
|
|
398
|
+
}
|
|
399
|
+
const tags = dedup(tagList);
|
|
400
|
+
|
|
401
|
+
// Links: body wikilinks (doc order) + markdown links, then whole-value
|
|
402
|
+
// frontmatter wikilinks; embeds kept separate.
|
|
403
|
+
const { links: bodyLinks, embeds } = classifyWikilinks(masked);
|
|
404
|
+
const links = dedup([...bodyLinks, ...extractMarkdownLinks(masked)]);
|
|
405
|
+
for (const [name, value] of Object.entries(fm)) {
|
|
406
|
+
if (["tags", "aliases", "title", "path", "body"].includes(name)) continue;
|
|
407
|
+
const vals = Array.isArray(value) ? value : [value];
|
|
408
|
+
for (const v of vals) {
|
|
409
|
+
const k = frontmatterLinkTarget(v);
|
|
410
|
+
if (k && !links.includes(k)) links.push(k);
|
|
411
|
+
}
|
|
412
|
+
}
|
|
413
|
+
|
|
414
|
+
// Aliases (frontmatter `aliases`), normalized like link targets.
|
|
415
|
+
const aliases = [];
|
|
416
|
+
const aliasSeen = new Set();
|
|
417
|
+
const aliasSrc = Array.isArray(fm.aliases)
|
|
418
|
+
? fm.aliases
|
|
419
|
+
: fm.aliases != null
|
|
420
|
+
? [fm.aliases]
|
|
421
|
+
: [];
|
|
422
|
+
for (const a of aliasSrc) {
|
|
423
|
+
if (typeof a !== "string") continue;
|
|
424
|
+
const k = normalizeKey(a);
|
|
425
|
+
if (k && !aliasSeen.has(k)) {
|
|
426
|
+
aliasSeen.add(k);
|
|
427
|
+
aliases.push(k);
|
|
428
|
+
}
|
|
429
|
+
}
|
|
430
|
+
|
|
431
|
+
return {
|
|
432
|
+
id: noteId(relPath),
|
|
433
|
+
key,
|
|
434
|
+
title,
|
|
435
|
+
relPath,
|
|
436
|
+
properties,
|
|
437
|
+
links,
|
|
438
|
+
embeds,
|
|
439
|
+
tags,
|
|
440
|
+
aliases,
|
|
441
|
+
};
|
|
442
|
+
}
|
|
443
|
+
|
|
444
|
+
function dedup(arr) {
|
|
445
|
+
const seen = new Set();
|
|
446
|
+
const out = [];
|
|
447
|
+
for (const x of arr)
|
|
448
|
+
if (!seen.has(x)) {
|
|
449
|
+
seen.add(x);
|
|
450
|
+
out.push(x);
|
|
451
|
+
}
|
|
452
|
+
return out;
|
|
453
|
+
}
|
|
454
|
+
|
|
455
|
+
// ─── Walk the vault ──────────────────────────────────────────────────────
|
|
456
|
+
async function walkVault(root) {
|
|
457
|
+
const files = [];
|
|
458
|
+
async function rec(dir) {
|
|
459
|
+
let entries;
|
|
460
|
+
try {
|
|
461
|
+
entries = await readdir(dir, { withFileTypes: true });
|
|
462
|
+
} catch (e) {
|
|
463
|
+
throw new Error(`cannot read ${dir}: ${e.message}`);
|
|
464
|
+
}
|
|
465
|
+
for (const ent of entries) {
|
|
466
|
+
const full = path.join(dir, ent.name);
|
|
467
|
+
if (ent.isDirectory()) {
|
|
468
|
+
if (ent.name.startsWith(".") || ent.name === "_templates") continue;
|
|
469
|
+
await rec(full);
|
|
470
|
+
} else if (/\.(md|markdown)$/i.test(ent.name)) {
|
|
471
|
+
const rel = path.relative(root, full).split(path.sep).join("/");
|
|
472
|
+
files.push(rel);
|
|
473
|
+
}
|
|
474
|
+
}
|
|
475
|
+
}
|
|
476
|
+
await rec(root);
|
|
477
|
+
files.sort();
|
|
478
|
+
return files;
|
|
479
|
+
}
|
|
480
|
+
|
|
481
|
+
// ─── Build the desired graph (notes + resolved edges + tags + placeholders) ─
|
|
482
|
+
function buildGraph(notes) {
|
|
483
|
+
const known = new Set(notes.map((n) => n.key));
|
|
484
|
+
// key -> note id (first note in path order wins a duplicate basename key).
|
|
485
|
+
const keyToId = new Map();
|
|
486
|
+
for (const n of notes) if (!keyToId.has(n.key)) keyToId.set(n.key, n.id);
|
|
487
|
+
// Alias map: first note (path order) to declare an alias wins; a real note
|
|
488
|
+
// key always shadows an alias.
|
|
489
|
+
const aliasMap = new Map();
|
|
490
|
+
for (const n of notes)
|
|
491
|
+
for (const a of n.aliases)
|
|
492
|
+
if (!known.has(a) && !aliasMap.has(a)) aliasMap.set(a, n.id);
|
|
493
|
+
|
|
494
|
+
const linkEdges = [];
|
|
495
|
+
const embedEdges = [];
|
|
496
|
+
const placeholders = new Map(); // key -> placeholder node id
|
|
497
|
+
|
|
498
|
+
const pushEdge = (arr, srcId, target) => {
|
|
499
|
+
let dst = null;
|
|
500
|
+
if (known.has(target)) dst = keyToId.get(target);
|
|
501
|
+
else if (aliasMap.has(target)) dst = aliasMap.get(target);
|
|
502
|
+
if (dst != null) {
|
|
503
|
+
arr.push([srcId, dst]);
|
|
504
|
+
} else {
|
|
505
|
+
// Dangling -> placeholder Note, keyed by the normalized target so a real
|
|
506
|
+
// note created later upserts over the stub.
|
|
507
|
+
const pid = noteKeyId(target);
|
|
508
|
+
if (!placeholders.has(target)) placeholders.set(target, pid);
|
|
509
|
+
arr.push([srcId, pid]);
|
|
510
|
+
}
|
|
511
|
+
};
|
|
512
|
+
for (const n of notes) {
|
|
513
|
+
for (const t of n.links) pushEdge(linkEdges, n.id, t);
|
|
514
|
+
for (const t of n.embeds) pushEdge(embedEdges, n.id, t);
|
|
515
|
+
}
|
|
516
|
+
dedupPairs(linkEdges);
|
|
517
|
+
dedupPairs(embedEdges);
|
|
518
|
+
|
|
519
|
+
// Tags + nested SUBTAG_OF tree.
|
|
520
|
+
const tagNodes = new Map(); // id -> name
|
|
521
|
+
const tagged = [];
|
|
522
|
+
const subtags = new Set(); // "childIdparentId"
|
|
523
|
+
for (const n of notes) {
|
|
524
|
+
for (const tag of n.tags) {
|
|
525
|
+
const tid = tagId(tag);
|
|
526
|
+
tagNodes.set(tid, tag);
|
|
527
|
+
tagged.push([n.id, tid]);
|
|
528
|
+
let child = tag;
|
|
529
|
+
let slash;
|
|
530
|
+
while ((slash = child.lastIndexOf("/")) !== -1) {
|
|
531
|
+
const parent = child.slice(0, slash);
|
|
532
|
+
if (!parent) break;
|
|
533
|
+
const cid = tagId(child);
|
|
534
|
+
const pid = tagId(parent);
|
|
535
|
+
tagNodes.set(pid, parent);
|
|
536
|
+
subtags.add(`${cid}${pid}`);
|
|
537
|
+
child = parent;
|
|
538
|
+
}
|
|
539
|
+
}
|
|
540
|
+
}
|
|
541
|
+
return {
|
|
542
|
+
notes,
|
|
543
|
+
linkEdges,
|
|
544
|
+
embedEdges,
|
|
545
|
+
placeholders,
|
|
546
|
+
tagNodes,
|
|
547
|
+
tagged,
|
|
548
|
+
subtags,
|
|
549
|
+
};
|
|
550
|
+
}
|
|
551
|
+
function dedupPairs(pairs) {
|
|
552
|
+
const seen = new Set();
|
|
553
|
+
for (let i = pairs.length - 1; i >= 0; i--) {
|
|
554
|
+
const k = pairs[i][0] + "" + pairs[i][1];
|
|
555
|
+
if (seen.has(k)) pairs.splice(i, 1);
|
|
556
|
+
else seen.add(k);
|
|
557
|
+
}
|
|
558
|
+
pairs.reverse(); // restore first-seen order after reverse-walk
|
|
559
|
+
}
|
|
560
|
+
|
|
561
|
+
// ─── Property encoding for /v1/bulk ──────────────────────────────────────
|
|
562
|
+
// Scalars (null/bool/number/string) pass through. Homogeneous numeric arrays
|
|
563
|
+
// become vectors (we never emit those here). Lists of strings (tags/aliases)
|
|
564
|
+
// and one-level maps are JSON-stringified where the engine rejects nesting.
|
|
565
|
+
function encodeProps(properties) {
|
|
566
|
+
const out = {};
|
|
567
|
+
for (const [k, v] of Object.entries(properties)) {
|
|
568
|
+
if (
|
|
569
|
+
v === null ||
|
|
570
|
+
typeof v === "boolean" ||
|
|
571
|
+
typeof v === "number" ||
|
|
572
|
+
typeof v === "string"
|
|
573
|
+
) {
|
|
574
|
+
out[k] = v;
|
|
575
|
+
} else if (Array.isArray(v) && v.every((x) => typeof x === "number")) {
|
|
576
|
+
out[k] = v; // homogeneous numeric array -> vector (not used by notes)
|
|
577
|
+
} else {
|
|
578
|
+
// Lists/maps: the bulk endpoint rejects nested objects, so flatten to a
|
|
579
|
+
// JSON string. The graph relationships (:Tag, LINKS_TO) carry structure;
|
|
580
|
+
// this is just a faithful display copy of the frontmatter.
|
|
581
|
+
out[k] = JSON.stringify(v);
|
|
582
|
+
}
|
|
583
|
+
}
|
|
584
|
+
return out;
|
|
585
|
+
}
|
|
586
|
+
|
|
587
|
+
// ─── HTTP ────────────────────────────────────────────────────────────────
|
|
588
|
+
async function postBulk(payload) {
|
|
589
|
+
const ctrl = new AbortController();
|
|
590
|
+
const timer = setTimeout(() => ctrl.abort(), REQ_TIMEOUT_MS);
|
|
591
|
+
try {
|
|
592
|
+
const res = await fetch(`${API_URL}/v1/bulk`, {
|
|
593
|
+
method: "POST",
|
|
594
|
+
headers: {
|
|
595
|
+
Authorization: `Bearer ${API_KEY}`,
|
|
596
|
+
"X-NamiDB-Namespace": NAMESPACE,
|
|
597
|
+
"Content-Type": "application/json",
|
|
598
|
+
},
|
|
599
|
+
body: JSON.stringify(payload),
|
|
600
|
+
signal: ctrl.signal,
|
|
601
|
+
});
|
|
602
|
+
if (!res.ok)
|
|
603
|
+
throw new Error(
|
|
604
|
+
`bulk HTTP ${res.status}: ${(await res.text().catch(() => "")).slice(
|
|
605
|
+
0,
|
|
606
|
+
300,
|
|
607
|
+
)}`,
|
|
608
|
+
);
|
|
609
|
+
return res.json();
|
|
610
|
+
} finally {
|
|
611
|
+
clearTimeout(timer);
|
|
612
|
+
}
|
|
613
|
+
}
|
|
614
|
+
async function postCypher(query, params = {}) {
|
|
615
|
+
const ctrl = new AbortController();
|
|
616
|
+
const timer = setTimeout(() => ctrl.abort(), REQ_TIMEOUT_MS);
|
|
617
|
+
try {
|
|
618
|
+
const res = await fetch(`${API_URL}/v1/cypher`, {
|
|
619
|
+
method: "POST",
|
|
620
|
+
headers: {
|
|
621
|
+
Authorization: `Bearer ${API_KEY}`,
|
|
622
|
+
"X-NamiDB-Namespace": NAMESPACE,
|
|
623
|
+
"Content-Type": "application/json",
|
|
624
|
+
},
|
|
625
|
+
body: JSON.stringify({ query, params }),
|
|
626
|
+
signal: ctrl.signal,
|
|
627
|
+
});
|
|
628
|
+
if (!res.ok)
|
|
629
|
+
throw new Error(
|
|
630
|
+
`cypher HTTP ${res.status}: ${(await res.text().catch(() => "")).slice(
|
|
631
|
+
0,
|
|
632
|
+
300,
|
|
633
|
+
)}`,
|
|
634
|
+
);
|
|
635
|
+
return res.json();
|
|
636
|
+
} finally {
|
|
637
|
+
clearTimeout(timer);
|
|
638
|
+
}
|
|
639
|
+
}
|
|
640
|
+
// Trigger the gateway to (re-)embed this namespace's changed Notes. Authed the
|
|
641
|
+
// same way as /v1/bulk and /v1/cypher (Bearer + X-NamiDB-Namespace); the body
|
|
642
|
+
// is empty JSON. Returns the parsed { embedded, skipped, status } envelope —
|
|
643
|
+
// status "no_embedder" is a 200 (not an error) when no embedder is configured,
|
|
644
|
+
// so a missing embedder never fails a sync. Never sends or reads an embedder
|
|
645
|
+
// key; the gateway holds the encrypted key control-plane-side.
|
|
646
|
+
async function postEmbed() {
|
|
647
|
+
const ctrl = new AbortController();
|
|
648
|
+
const timer = setTimeout(() => ctrl.abort(), REQ_TIMEOUT_MS);
|
|
649
|
+
try {
|
|
650
|
+
const res = await fetch(`${API_URL}/v1/embed`, {
|
|
651
|
+
method: "POST",
|
|
652
|
+
headers: {
|
|
653
|
+
Authorization: `Bearer ${API_KEY}`,
|
|
654
|
+
"X-NamiDB-Namespace": NAMESPACE,
|
|
655
|
+
"Content-Type": "application/json",
|
|
656
|
+
},
|
|
657
|
+
body: "{}",
|
|
658
|
+
signal: ctrl.signal,
|
|
659
|
+
});
|
|
660
|
+
if (!res.ok)
|
|
661
|
+
throw new Error(
|
|
662
|
+
`embed HTTP ${res.status}: ${(await res.text().catch(() => "")).slice(
|
|
663
|
+
0,
|
|
664
|
+
300,
|
|
665
|
+
)}`,
|
|
666
|
+
);
|
|
667
|
+
return res.json();
|
|
668
|
+
} finally {
|
|
669
|
+
clearTimeout(timer);
|
|
670
|
+
}
|
|
671
|
+
}
|
|
672
|
+
// Best-effort embed trigger: log a one-line status, never throw. A failed embed
|
|
673
|
+
// must not fail an otherwise-successful sync — the vault is already pushed and
|
|
674
|
+
// the server re-embeds idempotently on the next sync.
|
|
675
|
+
async function triggerEmbed() {
|
|
676
|
+
try {
|
|
677
|
+
const r = await postEmbed();
|
|
678
|
+
if (r && r.status === "no_embedder") {
|
|
679
|
+
console.log(
|
|
680
|
+
"semantic search off (no embedder configured) — configure one in the dashboard to enable vault_search",
|
|
681
|
+
);
|
|
682
|
+
} else {
|
|
683
|
+
const embedded = Number(r?.embedded ?? 0);
|
|
684
|
+
const skipped = Number(r?.skipped ?? 0);
|
|
685
|
+
console.log(`embedded ${embedded}, skipped ${skipped}`);
|
|
686
|
+
}
|
|
687
|
+
} catch (e) {
|
|
688
|
+
console.warn(`embed trigger skipped (sync still succeeded): ${e.message}`);
|
|
689
|
+
}
|
|
690
|
+
}
|
|
691
|
+
|
|
692
|
+
// ─── Bulk push in chunks ─────────────────────────────────────────────────
|
|
693
|
+
async function pushNodes(nodes) {
|
|
694
|
+
let written = 0;
|
|
695
|
+
for (let i = 0; i < nodes.length; i += CHUNK) {
|
|
696
|
+
const slice = nodes.slice(i, i + CHUNK);
|
|
697
|
+
const r = await postBulk({ nodes: slice, edges: [] });
|
|
698
|
+
written += r.nodes_written ?? 0;
|
|
699
|
+
}
|
|
700
|
+
return written;
|
|
701
|
+
}
|
|
702
|
+
async function pushEdges(edges) {
|
|
703
|
+
let written = 0;
|
|
704
|
+
for (let i = 0; i < edges.length; i += CHUNK) {
|
|
705
|
+
const slice = edges.slice(i, i + CHUNK);
|
|
706
|
+
const r = await postBulk({ nodes: [], edges: slice });
|
|
707
|
+
written += r.edges_written ?? 0;
|
|
708
|
+
}
|
|
709
|
+
return written;
|
|
710
|
+
}
|
|
711
|
+
|
|
712
|
+
// ─── Diff-prune: delete cloud notes/tags removed locally ─────────────────
|
|
713
|
+
async function pruneRemoved(graph) {
|
|
714
|
+
// Current real-note paths in the cloud (placeholders have no path).
|
|
715
|
+
const resp = await postCypher(
|
|
716
|
+
`MATCH (n:${NOTE_LABEL}) WHERE n.path IS NOT NULL RETURN n.path AS path`,
|
|
717
|
+
);
|
|
718
|
+
const cloudPaths = new Set(
|
|
719
|
+
(resp.rows || []).map((r) => r.path).filter(Boolean),
|
|
720
|
+
);
|
|
721
|
+
const localPaths = new Set(graph.notes.map((n) => n.relPath));
|
|
722
|
+
const removed = [...cloudPaths].filter((p) => !localPaths.has(p));
|
|
723
|
+
let deleted = 0;
|
|
724
|
+
for (const p of removed) {
|
|
725
|
+
await postCypher(`MATCH (n:${NOTE_LABEL} {path: $path}) DETACH DELETE n`, {
|
|
726
|
+
path: p,
|
|
727
|
+
});
|
|
728
|
+
deleted++;
|
|
729
|
+
}
|
|
730
|
+
// Drop placeholder stubs that nothing references any more, and now-orphaned
|
|
731
|
+
// tags (no :TAGGED edge left). Cheap idempotent cleanup so the graph stays a
|
|
732
|
+
// faithful index after deletions.
|
|
733
|
+
await postCypher(
|
|
734
|
+
`MATCH (n:${NOTE_LABEL}) WHERE n.placeholder = true AND NOT EXISTS((n)<-[:${LINKS_TO}|:${EMBEDS}]-()) DETACH DELETE n`,
|
|
735
|
+
);
|
|
736
|
+
await postCypher(
|
|
737
|
+
`MATCH (t:${TAG_LABEL}) WHERE NOT EXISTS((t)<-[:${TAGGED}]-()) AND NOT EXISTS((t)<-[:${SUBTAG_OF}]-()) DETACH DELETE t`,
|
|
738
|
+
);
|
|
739
|
+
return deleted;
|
|
740
|
+
}
|
|
741
|
+
|
|
742
|
+
// ─── One sync pass ───────────────────────────────────────────────────────
|
|
743
|
+
async function syncOnce() {
|
|
744
|
+
const t0 = performance.now();
|
|
745
|
+
const relPaths = await walkVault(VAULT);
|
|
746
|
+
const notes = [];
|
|
747
|
+
for (const rel of relPaths) {
|
|
748
|
+
const raw = await readFile(path.join(VAULT, rel), "utf8");
|
|
749
|
+
notes.push(parseNote(rel, raw));
|
|
750
|
+
}
|
|
751
|
+
const graph = buildGraph(notes);
|
|
752
|
+
|
|
753
|
+
// Assemble bulk payloads.
|
|
754
|
+
const noteNodes = graph.notes.map((n) => ({
|
|
755
|
+
id: n.id,
|
|
756
|
+
label: NOTE_LABEL,
|
|
757
|
+
properties: encodeProps(n.properties),
|
|
758
|
+
}));
|
|
759
|
+
const placeholderNodes = [...graph.placeholders.entries()].map(
|
|
760
|
+
([key, id]) => ({
|
|
761
|
+
id,
|
|
762
|
+
label: NOTE_LABEL,
|
|
763
|
+
properties: { key, title: key, placeholder: true },
|
|
764
|
+
}),
|
|
765
|
+
);
|
|
766
|
+
const tagNodeList = [...graph.tagNodes.entries()].map(([id, name]) => ({
|
|
767
|
+
id,
|
|
768
|
+
label: TAG_LABEL,
|
|
769
|
+
properties: { name },
|
|
770
|
+
}));
|
|
771
|
+
const linkEdges = graph.linkEdges.map(([src, dst]) => ({
|
|
772
|
+
type: LINKS_TO,
|
|
773
|
+
src,
|
|
774
|
+
dst,
|
|
775
|
+
properties: {},
|
|
776
|
+
}));
|
|
777
|
+
const embedEdges = graph.embedEdges.map(([src, dst]) => ({
|
|
778
|
+
type: EMBEDS,
|
|
779
|
+
src,
|
|
780
|
+
dst,
|
|
781
|
+
properties: {},
|
|
782
|
+
}));
|
|
783
|
+
const taggedEdges = graph.tagged.map(([src, dst]) => ({
|
|
784
|
+
type: TAGGED,
|
|
785
|
+
src,
|
|
786
|
+
dst,
|
|
787
|
+
properties: {},
|
|
788
|
+
}));
|
|
789
|
+
const subtagEdges = [...graph.subtags].map((s) => {
|
|
790
|
+
const [child, parent] = s.split("");
|
|
791
|
+
return { type: SUBTAG_OF, src: child, dst: parent, properties: {} };
|
|
792
|
+
});
|
|
793
|
+
|
|
794
|
+
const plan = {
|
|
795
|
+
notes: noteNodes.length,
|
|
796
|
+
placeholders: placeholderNodes.length,
|
|
797
|
+
tags: tagNodeList.length,
|
|
798
|
+
links: linkEdges.length,
|
|
799
|
+
embeds: embedEdges.length,
|
|
800
|
+
tagged: taggedEdges.length,
|
|
801
|
+
subtags: subtagEdges.length,
|
|
802
|
+
};
|
|
803
|
+
|
|
804
|
+
if (DRY_RUN) {
|
|
805
|
+
console.log(`[dry-run] vault=${VAULT} ns=${NAMESPACE}`);
|
|
806
|
+
console.log(`[dry-run] plan: ${JSON.stringify(plan)}`);
|
|
807
|
+
console.log(`[dry-run] no writes performed.`);
|
|
808
|
+
return plan;
|
|
809
|
+
}
|
|
810
|
+
|
|
811
|
+
// Order: nodes first (notes, placeholders, tags), then edges. The bulk
|
|
812
|
+
// endpoint does not require endpoints in-batch, but creating nodes first
|
|
813
|
+
// keeps a partial failure from leaving dangling edges.
|
|
814
|
+
const nodesWritten = await pushNodes([
|
|
815
|
+
...noteNodes,
|
|
816
|
+
...placeholderNodes,
|
|
817
|
+
...tagNodeList,
|
|
818
|
+
]);
|
|
819
|
+
const edgesWritten = await pushEdges([
|
|
820
|
+
...linkEdges,
|
|
821
|
+
...embedEdges,
|
|
822
|
+
...taggedEdges,
|
|
823
|
+
...subtagEdges,
|
|
824
|
+
]);
|
|
825
|
+
|
|
826
|
+
let deleted = 0;
|
|
827
|
+
if (PRUNE) deleted = await pruneRemoved(graph);
|
|
828
|
+
|
|
829
|
+
const secs = ((performance.now() - t0) / 1000).toFixed(1);
|
|
830
|
+
console.log(
|
|
831
|
+
`synced ${plan.notes} notes (${nodesWritten} nodes, ${edgesWritten} edges, ` +
|
|
832
|
+
`${deleted} deleted) to ns=${NAMESPACE} in ${secs}s` +
|
|
833
|
+
(PRUNE ? "" : " [additive, no prune]"),
|
|
834
|
+
);
|
|
835
|
+
|
|
836
|
+
// Sync succeeded: trigger the server-side embed of changed Notes so semantic
|
|
837
|
+
// search stays current. Best-effort — never fails the sync (see triggerEmbed).
|
|
838
|
+
await triggerEmbed();
|
|
839
|
+
|
|
840
|
+
return { ...plan, deleted, nodesWritten, edgesWritten };
|
|
841
|
+
}
|
|
842
|
+
|
|
843
|
+
// ─── Watch mode ──────────────────────────────────────────────────────────
|
|
844
|
+
async function runWatch() {
|
|
845
|
+
console.log(
|
|
846
|
+
`watching ${VAULT} for changes (debounce ${DEBOUNCE_MS}ms)... Ctrl-C to stop`,
|
|
847
|
+
);
|
|
848
|
+
await syncOnce().catch((e) => console.error(`sync error: ${e.message}`));
|
|
849
|
+
let timer = null;
|
|
850
|
+
let running = false;
|
|
851
|
+
let again = false;
|
|
852
|
+
const trigger = () => {
|
|
853
|
+
if (timer) clearTimeout(timer);
|
|
854
|
+
timer = setTimeout(async () => {
|
|
855
|
+
if (running) {
|
|
856
|
+
again = true;
|
|
857
|
+
return;
|
|
858
|
+
}
|
|
859
|
+
running = true;
|
|
860
|
+
try {
|
|
861
|
+
await syncOnce();
|
|
862
|
+
} catch (e) {
|
|
863
|
+
console.error(`sync error: ${e.message}`);
|
|
864
|
+
} finally {
|
|
865
|
+
running = false;
|
|
866
|
+
if (again) {
|
|
867
|
+
again = false;
|
|
868
|
+
trigger();
|
|
869
|
+
}
|
|
870
|
+
}
|
|
871
|
+
}, DEBOUNCE_MS);
|
|
872
|
+
};
|
|
873
|
+
watch(VAULT, { recursive: true }, (_evt, name) => {
|
|
874
|
+
if (name && /\.(md|markdown)$/i.test(name)) trigger();
|
|
875
|
+
});
|
|
876
|
+
}
|
|
877
|
+
|
|
878
|
+
// ─── Entry ───────────────────────────────────────────────────────────────
|
|
879
|
+
try {
|
|
880
|
+
await stat(VAULT);
|
|
881
|
+
} catch {
|
|
882
|
+
fail(`vault directory not found: ${VAULT}`);
|
|
883
|
+
}
|
|
884
|
+
if (WATCH) {
|
|
885
|
+
await runWatch();
|
|
886
|
+
} else {
|
|
887
|
+
try {
|
|
888
|
+
await syncOnce();
|
|
889
|
+
} catch (e) {
|
|
890
|
+
fail(e.message);
|
|
891
|
+
}
|
|
892
|
+
}
|
|
893
|
+
|
|
894
|
+
// ─── helpers ─────────────────────────────────────────────────────────────
|
|
895
|
+
function parseArgs(argv) {
|
|
896
|
+
const out = {};
|
|
897
|
+
for (let i = 0; i < argv.length; i++) {
|
|
898
|
+
const a = argv[i];
|
|
899
|
+
if (a.startsWith("--")) {
|
|
900
|
+
const key = a.slice(2);
|
|
901
|
+
const next = argv[i + 1];
|
|
902
|
+
if (next === undefined || next.startsWith("--")) out[key] = true;
|
|
903
|
+
else {
|
|
904
|
+
out[key] = next;
|
|
905
|
+
i++;
|
|
906
|
+
}
|
|
907
|
+
} else if (a.startsWith("-")) {
|
|
908
|
+
out[a.slice(1)] = true;
|
|
909
|
+
}
|
|
910
|
+
}
|
|
911
|
+
return out;
|
|
912
|
+
}
|
|
913
|
+
function fail(msg) {
|
|
914
|
+
console.error(`error: ${msg}`);
|
|
915
|
+
process.exit(1);
|
|
916
|
+
}
|
|
917
|
+
function printUsage() {
|
|
918
|
+
console.log(
|
|
919
|
+
`namidb vault sync\n\n` +
|
|
920
|
+
`usage: node sync.mjs --vault DIR [--api-url URL] [--namespace NS] [options]\n\n` +
|
|
921
|
+
`config (env; flags override):\n` +
|
|
922
|
+
` NAMIDB_API_URL gateway base URL (--api-url)\n` +
|
|
923
|
+
` NAMIDB_API_KEY project API key, Bearer (env only)\n` +
|
|
924
|
+
` NAMIDB_NAMESPACE target namespace (--namespace)\n\n` +
|
|
925
|
+
`options:\n` +
|
|
926
|
+
` --vault DIR vault root (default: $NAMIDB_VAULT or .)\n` +
|
|
927
|
+
` --dry-run parse + plan, write nothing\n` +
|
|
928
|
+
` --no-prune additive only (never delete cloud notes)\n` +
|
|
929
|
+
` --watch re-sync on .md changes (debounced)\n` +
|
|
930
|
+
` --chunk N rows per bulk request (default 2000)\n` +
|
|
931
|
+
` --timeout MS per-request timeout (default 120000)\n`,
|
|
932
|
+
);
|
|
933
|
+
}
|