@geml/logseq-sync 2.0.6 → 2.0.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +14 -6
- package/core/src/bridge.mjs +8 -8
- package/core/src/mapping.mjs +59 -2
- package/docs/how-it-works.svg +42 -42
- package/package.json +1 -1
- package/watcher/bin/live-roundtrip.mjs +131 -131
package/README.md
CHANGED
|
@@ -238,9 +238,20 @@ grep -rl "old-tag" <vault-dir>/pages | xargs sed -i 's/old-tag/new-tag/g'
|
|
|
238
238
|
logseq-sync restore <vault-dir> --yes # or let --two-way pick it up
|
|
239
239
|
```
|
|
240
240
|
|
|
241
|
-
A
|
|
242
|
-
|
|
243
|
-
|
|
241
|
+
A check after a bulk edit is cheap insurance — it names a mangled block, and a
|
|
242
|
+
reference that now goes nowhere, before the import carries either into the
|
|
243
|
+
graph:
|
|
244
|
+
|
|
245
|
+
```sh
|
|
246
|
+
geml check <vault-dir>/pages/foo.geml --root <vault-dir>
|
|
247
|
+
```
|
|
248
|
+
|
|
249
|
+
`--root` is what lets a reference into another page resolve: block refs are
|
|
250
|
+
translated on the way out, so `[[<uuid>]]` in the graph becomes GEML's checked
|
|
251
|
+
`[[#uuid]]` (same page) or `[[../pages/other.geml#uuid]]` (another one), and
|
|
252
|
+
the translation reverses exactly on the way back. (Today's output also draws a
|
|
253
|
+
`unknown attribute level` warning per block — noise, not a problem: `level=N`
|
|
254
|
+
carries the outline depth and no error is implied.)
|
|
244
255
|
|
|
245
256
|
## Honesty corner
|
|
246
257
|
|
|
@@ -348,9 +359,6 @@ are this package's own.
|
|
|
348
359
|
|
|
349
360
|
## Next
|
|
350
361
|
|
|
351
|
-
- **Reference translation**: block refs in titles are literally `[[<uuid>]]`,
|
|
352
|
-
one character away from GEML's checked `[[#uuid]]` — translating them lets
|
|
353
|
-
`geml check` catch broken block refs, the actual headline of the proposal.
|
|
354
362
|
- Property readability: scalar `:build/properties` as GEML attributes instead
|
|
355
363
|
of the `.block-meta` EDN ride-along (NAME rules permitting).
|
|
356
364
|
- **Write-back**: wiring `syncDiskToEdn` to the CLI so the vault is
|
package/core/src/bridge.mjs
CHANGED
|
@@ -1,8 +1,8 @@
|
|
|
1
|
-
// The contract between the two halves of Sync Vault with GEML. The in-app plugin
|
|
2
|
-
// writes SIGNAL_FILE through logseq.FileStorage; the watcher reacts to it and
|
|
3
|
-
// writes STATUS_FILE back beside it. Both land in the plugin's storage
|
|
4
|
-
// directory (<dotdir>/storages/<plugin-id>/) — the one disk location both
|
|
5
|
-
// sides can reach. These names ARE the protocol: change them only together.
|
|
6
|
-
|
|
7
|
-
export const SIGNAL_FILE = "geml-sync-dirty.json";
|
|
8
|
-
export const STATUS_FILE = "geml-sync-status.json";
|
|
1
|
+
// The contract between the two halves of Sync Vault with GEML. The in-app plugin
|
|
2
|
+
// writes SIGNAL_FILE through logseq.FileStorage; the watcher reacts to it and
|
|
3
|
+
// writes STATUS_FILE back beside it. Both land in the plugin's storage
|
|
4
|
+
// directory (<dotdir>/storages/<plugin-id>/) — the one disk location both
|
|
5
|
+
// sides can reach. These names ARE the protocol: change them only together.
|
|
6
|
+
|
|
7
|
+
export const SIGNAL_FILE = "geml-sync-dirty.json";
|
|
8
|
+
export const STATUS_FILE = "geml-sync-status.json";
|
package/core/src/mapping.mjs
CHANGED
|
@@ -70,6 +70,53 @@ function gemlBlock(type, attrs, body) {
|
|
|
70
70
|
|
|
71
71
|
// Returns Map<relativePath, gemlText>. Page order is preserved by a numeric
|
|
72
72
|
// filename prefix: :pages-and-blocks is a vector, and order is content.
|
|
73
|
+
// A Logseq block reference, as the DB export writes it: `[[<uuid>]]` inside a
|
|
74
|
+
// block's title. A PAGE reference looks identical apart from its target
|
|
75
|
+
// (`[[Some Page]]`), so the uuid shape is the whole discriminator — matching
|
|
76
|
+
// anything looser would rewrite people's page links.
|
|
77
|
+
const REF_BARE = /\[\[([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12})\]\]/g;
|
|
78
|
+
// The GEML form, on the way back: `[[#uuid]]` or `[[path/to/doc.geml#uuid]]`.
|
|
79
|
+
const REF_GEML = /\[\[([^\[\]]*?)#([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12})\]\]/g;
|
|
80
|
+
|
|
81
|
+
/** POSIX-relative path from one vault file to another, as GEML resolves it. */
|
|
82
|
+
function relFromTo(fromPath, toPath) {
|
|
83
|
+
const from = fromPath.split("/").slice(0, -1);
|
|
84
|
+
const to = toPath.split("/");
|
|
85
|
+
let i = 0;
|
|
86
|
+
while (i < from.length && i < to.length - 1 && from[i] === to[i]) i++;
|
|
87
|
+
return [...from.slice(i).map(() => ".."), ...to.slice(i)].join("/");
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
/**
|
|
91
|
+
* Turn Logseq's unchecked `[[uuid]]` into GEML's checked reference — the whole
|
|
92
|
+
* point of the exercise: `geml check` then reports a reference that goes
|
|
93
|
+
* nowhere instead of shrugging at it.
|
|
94
|
+
*
|
|
95
|
+
* A target in the same file becomes `[[#uuid]]`, one in another file
|
|
96
|
+
* `[[<relative path>#uuid]]`. A uuid the export never wrote also becomes
|
|
97
|
+
* `[[#uuid]]`, which `check` calls unresolved — because within this vault it
|
|
98
|
+
* IS: `@logseq/cli` 0.4.3 does not export journal pages, so a ref into one
|
|
99
|
+
* genuinely leads nowhere here, and saying so is the promise being kept, not
|
|
100
|
+
* broken. Translation is exactly reversible, which is what keeps the round
|
|
101
|
+
* trip an identity.
|
|
102
|
+
*/
|
|
103
|
+
export function translateRefsOut(files, uuidPath) {
|
|
104
|
+
for (const [path, text] of files) {
|
|
105
|
+
const next = text.replace(REF_BARE, (_m, uuid) => {
|
|
106
|
+
const target = uuidPath.get(uuid.toLowerCase());
|
|
107
|
+
if (!target || target === path) return `[[#${uuid}]]`;
|
|
108
|
+
return `[[${relFromTo(path, target)}#${uuid}]]`;
|
|
109
|
+
});
|
|
110
|
+
if (next !== text) files.set(path, next);
|
|
111
|
+
}
|
|
112
|
+
return files;
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
/** The inverse: any `[[…#uuid]]` back to the `[[uuid]]` Logseq stores. */
|
|
116
|
+
export function translateRefsIn(text) {
|
|
117
|
+
return text.replace(REF_GEML, (_m, _prefix, uuid) => `[[${uuid}]]`);
|
|
118
|
+
}
|
|
119
|
+
|
|
73
120
|
export function ednToGemlFiles(ednText) {
|
|
74
121
|
const top = parseEDNString(ednText);
|
|
75
122
|
const files = new Map();
|
|
@@ -87,6 +134,9 @@ export function ednToGemlFiles(ednText) {
|
|
|
87
134
|
files.set("ontology.geml", onto);
|
|
88
135
|
|
|
89
136
|
const order = [];
|
|
137
|
+
// uuid → the file that will hold that block, filled during the walk and used
|
|
138
|
+
// once every file exists: a reference can point at a page written later.
|
|
139
|
+
const uuidPath = new Map();
|
|
90
140
|
pages.forEach((entry) => {
|
|
91
141
|
const page = mapGet(entry, "page") ?? { map: [] };
|
|
92
142
|
const blocksVal = mapGet(entry, "blocks");
|
|
@@ -132,6 +182,7 @@ export function ednToGemlFiles(ednText) {
|
|
|
132
182
|
// The uuid stays inside the meta EDN too — losslessness never depends
|
|
133
183
|
// on the id attribute; `{#uuid}` is the ADDRESS.
|
|
134
184
|
const u = uuidOf(mapGet(b, "block/uuid"));
|
|
185
|
+
if (u) uuidPath.set(u.toLowerCase(), path);
|
|
135
186
|
const id = u ? `#${u} ` : "";
|
|
136
187
|
out += gemlBlock("text", `${id}level=${level}`, typeof btitle === "string" ? btitle : edn(btitle ?? null));
|
|
137
188
|
if (mapSize(meta) > 0) out += gemlBlock("code", ".block-meta lang=edn", edn(meta));
|
|
@@ -149,7 +200,8 @@ export function ednToGemlFiles(ednText) {
|
|
|
149
200
|
'=== meta\ntitle = "Logseq graph index"\n===\n\n' +
|
|
150
201
|
gemlBlock("data", "#page-order", JSON.stringify(order, null, 1)));
|
|
151
202
|
|
|
152
|
-
|
|
203
|
+
// Last, because a reference needs to know where every block ended up.
|
|
204
|
+
return translateRefsOut(files, uuidPath);
|
|
153
205
|
}
|
|
154
206
|
|
|
155
207
|
// --- import: GEML files → EDN ------------------------------------------------
|
|
@@ -163,8 +215,13 @@ export function ednToGemlFiles(ednText) {
|
|
|
163
215
|
// raw bytes. The bytes come from `sliceUnit` over the block's span, exactly the
|
|
164
216
|
// route `geml get` takes. Blocks arrive in document order from both, so the
|
|
165
217
|
// two sequences zip.
|
|
166
|
-
export function gemlFilesToEdn(
|
|
218
|
+
export function gemlFilesToEdn(filesIn, lib) {
|
|
167
219
|
const { parse, addressedUnits, sliceUnit } = lib;
|
|
220
|
+
// Checked references go back to the `[[uuid]]` Logseq stores, before any
|
|
221
|
+
// parsing: the graph is the other side of the translation, not a party to it.
|
|
222
|
+
// Vaults written before the translation existed hold bare uuids already, and
|
|
223
|
+
// this leaves those alone — the same import handles both.
|
|
224
|
+
const files = new Map([...filesIn].map(([path, text]) => [path, translateRefsIn(text)]));
|
|
168
225
|
const blocksOf = (text) => {
|
|
169
226
|
const nodes = parse(text).children.filter((c) => c.kind === "block");
|
|
170
227
|
const units = [...addressedUnits(text)].map((a) => a.unit).filter((u) => u.kind === "block");
|
package/docs/how-it-works.svg
CHANGED
|
@@ -1,42 +1,42 @@
|
|
|
1
|
-
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 760 250" font-family="ui-sans-serif,system-ui,Segoe UI,Helvetica,Arial,sans-serif">
|
|
2
|
-
<defs>
|
|
3
|
-
<marker id="arr" viewBox="0 0 10 10" refX="9" refY="5" markerWidth="7" markerHeight="7" orient="auto-start-reverse">
|
|
4
|
-
<path d="M0,0 L10,5 L0,10 z" fill="#8b949e"/>
|
|
5
|
-
</marker>
|
|
6
|
-
</defs>
|
|
7
|
-
<rect x="0" y="0" width="760" height="250" rx="12" fill="#f6f8fa" stroke="#d0d7de"/>
|
|
8
|
-
|
|
9
|
-
<!-- Logseq app box -->
|
|
10
|
-
<rect x="24" y="36" width="220" height="130" rx="10" fill="#ffffff" stroke="#a475f9" stroke-width="1.5"/>
|
|
11
|
-
<text x="134" y="62" text-anchor="middle" font-size="15" font-weight="700" fill="#57606a">Logseq 2.0 (DB graph)</text>
|
|
12
|
-
<rect x="44" y="80" width="180" height="66" rx="8" fill="#f3ecff" stroke="#a475f9"/>
|
|
13
|
-
<text x="134" y="104" text-anchor="middle" font-size="13" font-weight="600" fill="#3b2a63">Sync Vault with GEML plugin</text>
|
|
14
|
-
<text x="134" y="124" text-anchor="middle" font-size="11.5" fill="#57606a">DB.onChanged → debounce</text>
|
|
15
|
-
<text x="134" y="139" text-anchor="middle" font-size="11.5" fill="#57606a">status in toolbar ⇄</text>
|
|
16
|
-
|
|
17
|
-
<!-- signal file -->
|
|
18
|
-
<rect x="286" y="70" width="160" height="42" rx="8" fill="#fff8c5" stroke="#d4a72c"/>
|
|
19
|
-
<text x="366" y="88" text-anchor="middle" font-size="12" font-weight="600" fill="#57606a">dirty-marker file</text>
|
|
20
|
-
<text x="366" y="103" text-anchor="middle" font-size="10.5" fill="#7d8590">…/storages/logseq-plugin-sync-vault-with-geml/</text>
|
|
21
|
-
|
|
22
|
-
<!-- status file -->
|
|
23
|
-
<rect x="286" y="128" width="160" height="34" rx="8" fill="#ddf4ff" stroke="#54aeff"/>
|
|
24
|
-
<text x="366" y="149" text-anchor="middle" font-size="12" font-weight="600" fill="#57606a">geml-sync-status.json</text>
|
|
25
|
-
|
|
26
|
-
<!-- watcher box -->
|
|
27
|
-
<rect x="488" y="36" width="248" height="130" rx="10" fill="#ffffff" stroke="#2da44e" stroke-width="1.5"/>
|
|
28
|
-
<text x="612" y="62" text-anchor="middle" font-size="15" font-weight="700" fill="#57606a">logseq-sync watcher (CLI)</text>
|
|
29
|
-
<text x="612" y="86" text-anchor="middle" font-size="11.5" fill="#57606a">export via official @logseq/cli</text>
|
|
30
|
-
<text x="612" y="103" text-anchor="middle" font-size="11.5" fill="#57606a">writes only files that changed</text>
|
|
31
|
-
<text x="612" y="120" text-anchor="middle" font-size="11.5" fill="#57606a">git commit scoped to the vault</text>
|
|
32
|
-
<text x="612" y="145" text-anchor="middle" font-size="11.5" font-weight="600" fill="#2da44e">your-vault/pages/*.geml · git</text>
|
|
33
|
-
|
|
34
|
-
<!-- arrows -->
|
|
35
|
-
<line x1="244" y1="91" x2="284" y2="91" stroke="#8b949e" stroke-width="1.6" marker-end="url(#arr)"/>
|
|
36
|
-
<line x1="446" y1="91" x2="486" y2="91" stroke="#8b949e" stroke-width="1.6" marker-end="url(#arr)"/>
|
|
37
|
-
<line x1="486" y1="145" x2="446" y2="145" stroke="#8b949e" stroke-width="1.6" marker-end="url(#arr)"/>
|
|
38
|
-
<line x1="286" y1="145" x2="246" y2="145" stroke="#8b949e" stroke-width="1.6" marker-end="url(#arr)"/>
|
|
39
|
-
|
|
40
|
-
<text x="380" y="196" text-anchor="middle" font-size="12.5" fill="#57606a">The plugin is a doorbell, not the mover: the sandbox has no filesystem and no git,</text>
|
|
41
|
-
<text x="380" y="214" text-anchor="middle" font-size="12.5" fill="#57606a">so everything with side effects lives in the watcher — auditable, scoped, outside the app.</text>
|
|
42
|
-
</svg>
|
|
1
|
+
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 760 250" font-family="ui-sans-serif,system-ui,Segoe UI,Helvetica,Arial,sans-serif">
|
|
2
|
+
<defs>
|
|
3
|
+
<marker id="arr" viewBox="0 0 10 10" refX="9" refY="5" markerWidth="7" markerHeight="7" orient="auto-start-reverse">
|
|
4
|
+
<path d="M0,0 L10,5 L0,10 z" fill="#8b949e"/>
|
|
5
|
+
</marker>
|
|
6
|
+
</defs>
|
|
7
|
+
<rect x="0" y="0" width="760" height="250" rx="12" fill="#f6f8fa" stroke="#d0d7de"/>
|
|
8
|
+
|
|
9
|
+
<!-- Logseq app box -->
|
|
10
|
+
<rect x="24" y="36" width="220" height="130" rx="10" fill="#ffffff" stroke="#a475f9" stroke-width="1.5"/>
|
|
11
|
+
<text x="134" y="62" text-anchor="middle" font-size="15" font-weight="700" fill="#57606a">Logseq 2.0 (DB graph)</text>
|
|
12
|
+
<rect x="44" y="80" width="180" height="66" rx="8" fill="#f3ecff" stroke="#a475f9"/>
|
|
13
|
+
<text x="134" y="104" text-anchor="middle" font-size="13" font-weight="600" fill="#3b2a63">Sync Vault with GEML plugin</text>
|
|
14
|
+
<text x="134" y="124" text-anchor="middle" font-size="11.5" fill="#57606a">DB.onChanged → debounce</text>
|
|
15
|
+
<text x="134" y="139" text-anchor="middle" font-size="11.5" fill="#57606a">status in toolbar ⇄</text>
|
|
16
|
+
|
|
17
|
+
<!-- signal file -->
|
|
18
|
+
<rect x="286" y="70" width="160" height="42" rx="8" fill="#fff8c5" stroke="#d4a72c"/>
|
|
19
|
+
<text x="366" y="88" text-anchor="middle" font-size="12" font-weight="600" fill="#57606a">dirty-marker file</text>
|
|
20
|
+
<text x="366" y="103" text-anchor="middle" font-size="10.5" fill="#7d8590">…/storages/logseq-plugin-sync-vault-with-geml/</text>
|
|
21
|
+
|
|
22
|
+
<!-- status file -->
|
|
23
|
+
<rect x="286" y="128" width="160" height="34" rx="8" fill="#ddf4ff" stroke="#54aeff"/>
|
|
24
|
+
<text x="366" y="149" text-anchor="middle" font-size="12" font-weight="600" fill="#57606a">geml-sync-status.json</text>
|
|
25
|
+
|
|
26
|
+
<!-- watcher box -->
|
|
27
|
+
<rect x="488" y="36" width="248" height="130" rx="10" fill="#ffffff" stroke="#2da44e" stroke-width="1.5"/>
|
|
28
|
+
<text x="612" y="62" text-anchor="middle" font-size="15" font-weight="700" fill="#57606a">logseq-sync watcher (CLI)</text>
|
|
29
|
+
<text x="612" y="86" text-anchor="middle" font-size="11.5" fill="#57606a">export via official @logseq/cli</text>
|
|
30
|
+
<text x="612" y="103" text-anchor="middle" font-size="11.5" fill="#57606a">writes only files that changed</text>
|
|
31
|
+
<text x="612" y="120" text-anchor="middle" font-size="11.5" fill="#57606a">git commit scoped to the vault</text>
|
|
32
|
+
<text x="612" y="145" text-anchor="middle" font-size="11.5" font-weight="600" fill="#2da44e">your-vault/pages/*.geml · git</text>
|
|
33
|
+
|
|
34
|
+
<!-- arrows -->
|
|
35
|
+
<line x1="244" y1="91" x2="284" y2="91" stroke="#8b949e" stroke-width="1.6" marker-end="url(#arr)"/>
|
|
36
|
+
<line x1="446" y1="91" x2="486" y2="91" stroke="#8b949e" stroke-width="1.6" marker-end="url(#arr)"/>
|
|
37
|
+
<line x1="486" y1="145" x2="446" y2="145" stroke="#8b949e" stroke-width="1.6" marker-end="url(#arr)"/>
|
|
38
|
+
<line x1="286" y1="145" x2="246" y2="145" stroke="#8b949e" stroke-width="1.6" marker-end="url(#arr)"/>
|
|
39
|
+
|
|
40
|
+
<text x="380" y="196" text-anchor="middle" font-size="12.5" fill="#57606a">The plugin is a doorbell, not the mover: the sandbox has no filesystem and no git,</text>
|
|
41
|
+
<text x="380" y="214" text-anchor="middle" font-size="12.5" fill="#57606a">so everything with side effects lives in the watcher — auditable, scoped, outside the app.</text>
|
|
42
|
+
</svg>
|
package/package.json
CHANGED
|
@@ -1,131 +1,131 @@
|
|
|
1
|
-
// The live half of the spike: run the round trip against a REAL DB graph via
|
|
2
|
-
// the official @logseq/cli, with Logseq's own `validate` as the judge.
|
|
3
|
-
//
|
|
4
|
-
// node bin/live-roundtrip.mjs <graph-name> [--edit]
|
|
5
|
-
//
|
|
6
|
-
// Stages (each printed, each gated):
|
|
7
|
-
// 1. `logseq export-edn -g <graph>` → out/export-1.edn
|
|
8
|
-
// 2. ednToGemlFiles → out/geml/**.geml
|
|
9
|
-
// 3. `geml check --root out/geml` on every doc (zero errors required)
|
|
10
|
-
// 4. gemlFilesToEdn → out/import.edn
|
|
11
|
-
// 5. STRUCTURAL identity export-1 ⇄ import.edn (EDN semantics) — the offline
|
|
12
|
-
// criterion, now on real data
|
|
13
|
-
// 6. with --edit: `geml set` the first uuid block, rebuild import.edn, and
|
|
14
|
-
// `logseq import-edn` it back, then `logseq validate` — the semantics
|
|
15
|
-
// probe: does re-import by uuid update in place, or append?
|
|
16
|
-
//
|
|
17
|
-
// Import-back is opt-in (--edit) because import semantics on a whole-graph
|
|
18
|
-
// re-import are exactly what this stage exists to LEARN — it may merge, it may
|
|
19
|
-
// duplicate id-less blocks. The script never touches a graph unless told to.
|
|
20
|
-
import { execFileSync } from "node:child_process";
|
|
21
|
-
import { mkdirSync, writeFileSync, readFileSync, rmSync, readdirSync } from "node:fs";
|
|
22
|
-
import { dirname, join, resolve } from "node:path";
|
|
23
|
-
import { fileURLToPath } from "node:url";
|
|
24
|
-
import { parseEDNString } from "edn-data";
|
|
25
|
-
import { ednToGemlFiles, gemlFilesToEdn } from "../../core/src/mapping.mjs";
|
|
26
|
-
import { parse, addressedUnits, sliceUnit } from "../../../../geml-parser/dist/geml.js";
|
|
27
|
-
|
|
28
|
-
const here = dirname(fileURLToPath(import.meta.url));
|
|
29
|
-
const out = join(here, "..", "out");
|
|
30
|
-
const GEML = resolve(here, "..", "..", "..", "geml-parser", "dist", "geml.js");
|
|
31
|
-
const lib = { parse, addressedUnits, sliceUnit };
|
|
32
|
-
|
|
33
|
-
const [graph, ...flags] = process.argv.slice(2);
|
|
34
|
-
if (!graph) { console.error("usage: node bin/live-roundtrip.mjs <graph-name> [--edit]"); process.exit(2); }
|
|
35
|
-
const doEdit = flags.includes("--edit");
|
|
36
|
-
|
|
37
|
-
// The CLI is driven through npx so nothing here depends on a global install;
|
|
38
|
-
// LOGSEQ_CLI_DIR points at a directory whose node_modules has @logseq/cli.
|
|
39
|
-
const cliCwd = process.env.LOGSEQ_CLI_DIR ?? join(here, "..");
|
|
40
|
-
const logseq = (...args) =>
|
|
41
|
-
execFileSync("npx", ["-y", "@logseq/cli", ...args], { cwd: cliCwd, encoding: "utf8", shell: true, maxBuffer: 1 << 28 });
|
|
42
|
-
const geml = (...args) =>
|
|
43
|
-
execFileSync(process.execPath, [GEML, ...args], { encoding: "utf8", maxBuffer: 1 << 28 });
|
|
44
|
-
|
|
45
|
-
// EDN-semantics canonical form (same as the test suite).
|
|
46
|
-
function canon(v) {
|
|
47
|
-
if (Array.isArray(v)) return v.map(canon);
|
|
48
|
-
if (v !== null && typeof v === "object") {
|
|
49
|
-
if (Array.isArray(v.map)) {
|
|
50
|
-
const entries = v.map.map(([k, val]) => [canon(k), canon(val)]);
|
|
51
|
-
entries.sort((a, b) => (JSON.stringify(a[0]) < JSON.stringify(b[0]) ? -1 : 1));
|
|
52
|
-
return { map: entries };
|
|
53
|
-
}
|
|
54
|
-
if (Array.isArray(v.set)) {
|
|
55
|
-
const items = v.set.map(canon);
|
|
56
|
-
items.sort((a, b) => (JSON.stringify(a) < JSON.stringify(b) ? -1 : 1));
|
|
57
|
-
return { set: items };
|
|
58
|
-
}
|
|
59
|
-
const o = {};
|
|
60
|
-
for (const k of Object.keys(v).sort()) o[k] = canon(v[k]);
|
|
61
|
-
return o;
|
|
62
|
-
}
|
|
63
|
-
return v;
|
|
64
|
-
}
|
|
65
|
-
const same = (a, b) => JSON.stringify(canon(parseEDNString(a))) === JSON.stringify(canon(parseEDNString(b)));
|
|
66
|
-
|
|
67
|
-
rmSync(out, { recursive: true, force: true });
|
|
68
|
-
mkdirSync(join(out, "geml"), { recursive: true });
|
|
69
|
-
|
|
70
|
-
console.log(`1. export-edn from graph "${graph}"`);
|
|
71
|
-
logseq("export-edn", "-g", graph, "-f", join(out, "export-1.edn"));
|
|
72
|
-
const edn1 = readFileSync(join(out, "export-1.edn"), "utf8");
|
|
73
|
-
console.log(` ${edn1.length} bytes of EDN`);
|
|
74
|
-
|
|
75
|
-
console.log("2. EDN -> GEML");
|
|
76
|
-
const files = ednToGemlFiles(edn1);
|
|
77
|
-
for (const [rel, text] of files) {
|
|
78
|
-
mkdirSync(dirname(join(out, "geml", rel)), { recursive: true });
|
|
79
|
-
writeFileSync(join(out, "geml", rel), text);
|
|
80
|
-
}
|
|
81
|
-
console.log(` ${files.size} documents`);
|
|
82
|
-
|
|
83
|
-
console.log("3. geml check on every document");
|
|
84
|
-
let dirty = 0;
|
|
85
|
-
for (const rel of files.keys()) {
|
|
86
|
-
try { geml("check", "--root", join(out, "geml"), join(out, "geml", rel)); }
|
|
87
|
-
catch (e) { dirty++; console.error(` FAIL ${rel}\n${e.stdout ?? ""}${e.stderr ?? ""}`); }
|
|
88
|
-
}
|
|
89
|
-
if (dirty) { console.error(` ${dirty} document(s) not clean — stopping`); process.exit(1); }
|
|
90
|
-
console.log(" all clean");
|
|
91
|
-
|
|
92
|
-
console.log("4. GEML -> EDN");
|
|
93
|
-
const files2 = new Map();
|
|
94
|
-
for (const rel of files.keys()) files2.set(rel, readFileSync(join(out, "geml", rel), "utf8"));
|
|
95
|
-
const edn2 = gemlFilesToEdn(files2, lib);
|
|
96
|
-
writeFileSync(join(out, "import.edn"), edn2);
|
|
97
|
-
|
|
98
|
-
console.log("5. structural identity, on the real graph");
|
|
99
|
-
if (!same(edn1, edn2)) { console.error(" NOT identical — diff out/export-1.edn against out/import.edn"); process.exit(1); }
|
|
100
|
-
console.log(" identical (EDN semantics)");
|
|
101
|
-
|
|
102
|
-
if (!doEdit) { console.log("\nround trip holds. Re-run with --edit to probe import-back semantics."); process.exit(0); }
|
|
103
|
-
|
|
104
|
-
console.log("6. edit one uuid block via `geml set`, import back, validate");
|
|
105
|
-
const withUuid = [...files.keys()].map((rel) => {
|
|
106
|
-
const text = files2.get(rel);
|
|
107
|
-
const unit = [...addressedUnits(text)].map((a) => a.unit).find((u) => u.kind === "block" && u.id && /^[0-9a-f-]{36}$/.test(u.id));
|
|
108
|
-
return unit ? { rel, unit } : null;
|
|
109
|
-
}).find(Boolean);
|
|
110
|
-
if (!withUuid) { console.log(" no uuid-bearing block in this graph (nothing referenced) — skipping the edit probe"); process.exit(0); }
|
|
111
|
-
|
|
112
|
-
const target = join(out, "geml", withUuid.rel);
|
|
113
|
-
// Keep the block's own head line (type, id, level) — the edit is to the BODY.
|
|
114
|
-
const src = files2.get(withUuid.rel);
|
|
115
|
-
const head = sliceUnit(src, withUuid.unit.span, "head").trimEnd();
|
|
116
|
-
const body = sliceUnit(src, withUuid.unit.span, "body").trimEnd();
|
|
117
|
-
const fence = head.match(/^=+/)[0];
|
|
118
|
-
writeFileSync(join(out, "edit.txt"), `${head}\n${body} — edited by geml\n${fence}\n`);
|
|
119
|
-
geml("set", target, `#${withUuid.unit.id}`, "--in", join(out, "edit.txt"), "--root", join(out, "geml"));
|
|
120
|
-
console.log(` edited #${withUuid.unit.id} in ${withUuid.rel}`);
|
|
121
|
-
|
|
122
|
-
const files3 = new Map();
|
|
123
|
-
for (const rel of files.keys()) files3.set(rel, readFileSync(join(out, "geml", rel), "utf8"));
|
|
124
|
-
writeFileSync(join(out, "import-edited.edn"), gemlFilesToEdn(files3, lib));
|
|
125
|
-
logseq("import-edn", "-g", graph, "-f", join(out, "import-edited.edn"));
|
|
126
|
-
console.log(" imported");
|
|
127
|
-
console.log(logseq("validate", "-g", graph).trim());
|
|
128
|
-
|
|
129
|
-
console.log("7. export again — inspect out/export-2.edn to judge merge semantics");
|
|
130
|
-
logseq("export-edn", "-g", graph, "-f", join(out, "export-2.edn"));
|
|
131
|
-
console.log("done — compare out/export-1.edn / out/export-2.edn");
|
|
1
|
+
// The live half of the spike: run the round trip against a REAL DB graph via
|
|
2
|
+
// the official @logseq/cli, with Logseq's own `validate` as the judge.
|
|
3
|
+
//
|
|
4
|
+
// node bin/live-roundtrip.mjs <graph-name> [--edit]
|
|
5
|
+
//
|
|
6
|
+
// Stages (each printed, each gated):
|
|
7
|
+
// 1. `logseq export-edn -g <graph>` → out/export-1.edn
|
|
8
|
+
// 2. ednToGemlFiles → out/geml/**.geml
|
|
9
|
+
// 3. `geml check --root out/geml` on every doc (zero errors required)
|
|
10
|
+
// 4. gemlFilesToEdn → out/import.edn
|
|
11
|
+
// 5. STRUCTURAL identity export-1 ⇄ import.edn (EDN semantics) — the offline
|
|
12
|
+
// criterion, now on real data
|
|
13
|
+
// 6. with --edit: `geml set` the first uuid block, rebuild import.edn, and
|
|
14
|
+
// `logseq import-edn` it back, then `logseq validate` — the semantics
|
|
15
|
+
// probe: does re-import by uuid update in place, or append?
|
|
16
|
+
//
|
|
17
|
+
// Import-back is opt-in (--edit) because import semantics on a whole-graph
|
|
18
|
+
// re-import are exactly what this stage exists to LEARN — it may merge, it may
|
|
19
|
+
// duplicate id-less blocks. The script never touches a graph unless told to.
|
|
20
|
+
import { execFileSync } from "node:child_process";
|
|
21
|
+
import { mkdirSync, writeFileSync, readFileSync, rmSync, readdirSync } from "node:fs";
|
|
22
|
+
import { dirname, join, resolve } from "node:path";
|
|
23
|
+
import { fileURLToPath } from "node:url";
|
|
24
|
+
import { parseEDNString } from "edn-data";
|
|
25
|
+
import { ednToGemlFiles, gemlFilesToEdn } from "../../core/src/mapping.mjs";
|
|
26
|
+
import { parse, addressedUnits, sliceUnit } from "../../../../geml-parser/dist/geml.js";
|
|
27
|
+
|
|
28
|
+
const here = dirname(fileURLToPath(import.meta.url));
|
|
29
|
+
const out = join(here, "..", "out");
|
|
30
|
+
const GEML = resolve(here, "..", "..", "..", "geml-parser", "dist", "geml.js");
|
|
31
|
+
const lib = { parse, addressedUnits, sliceUnit };
|
|
32
|
+
|
|
33
|
+
const [graph, ...flags] = process.argv.slice(2);
|
|
34
|
+
if (!graph) { console.error("usage: node bin/live-roundtrip.mjs <graph-name> [--edit]"); process.exit(2); }
|
|
35
|
+
const doEdit = flags.includes("--edit");
|
|
36
|
+
|
|
37
|
+
// The CLI is driven through npx so nothing here depends on a global install;
|
|
38
|
+
// LOGSEQ_CLI_DIR points at a directory whose node_modules has @logseq/cli.
|
|
39
|
+
const cliCwd = process.env.LOGSEQ_CLI_DIR ?? join(here, "..");
|
|
40
|
+
const logseq = (...args) =>
|
|
41
|
+
execFileSync("npx", ["-y", "@logseq/cli", ...args], { cwd: cliCwd, encoding: "utf8", shell: true, maxBuffer: 1 << 28 });
|
|
42
|
+
const geml = (...args) =>
|
|
43
|
+
execFileSync(process.execPath, [GEML, ...args], { encoding: "utf8", maxBuffer: 1 << 28 });
|
|
44
|
+
|
|
45
|
+
// EDN-semantics canonical form (same as the test suite).
|
|
46
|
+
function canon(v) {
|
|
47
|
+
if (Array.isArray(v)) return v.map(canon);
|
|
48
|
+
if (v !== null && typeof v === "object") {
|
|
49
|
+
if (Array.isArray(v.map)) {
|
|
50
|
+
const entries = v.map.map(([k, val]) => [canon(k), canon(val)]);
|
|
51
|
+
entries.sort((a, b) => (JSON.stringify(a[0]) < JSON.stringify(b[0]) ? -1 : 1));
|
|
52
|
+
return { map: entries };
|
|
53
|
+
}
|
|
54
|
+
if (Array.isArray(v.set)) {
|
|
55
|
+
const items = v.set.map(canon);
|
|
56
|
+
items.sort((a, b) => (JSON.stringify(a) < JSON.stringify(b) ? -1 : 1));
|
|
57
|
+
return { set: items };
|
|
58
|
+
}
|
|
59
|
+
const o = {};
|
|
60
|
+
for (const k of Object.keys(v).sort()) o[k] = canon(v[k]);
|
|
61
|
+
return o;
|
|
62
|
+
}
|
|
63
|
+
return v;
|
|
64
|
+
}
|
|
65
|
+
const same = (a, b) => JSON.stringify(canon(parseEDNString(a))) === JSON.stringify(canon(parseEDNString(b)));
|
|
66
|
+
|
|
67
|
+
rmSync(out, { recursive: true, force: true });
|
|
68
|
+
mkdirSync(join(out, "geml"), { recursive: true });
|
|
69
|
+
|
|
70
|
+
console.log(`1. export-edn from graph "${graph}"`);
|
|
71
|
+
logseq("export-edn", "-g", graph, "-f", join(out, "export-1.edn"));
|
|
72
|
+
const edn1 = readFileSync(join(out, "export-1.edn"), "utf8");
|
|
73
|
+
console.log(` ${edn1.length} bytes of EDN`);
|
|
74
|
+
|
|
75
|
+
console.log("2. EDN -> GEML");
|
|
76
|
+
const files = ednToGemlFiles(edn1);
|
|
77
|
+
for (const [rel, text] of files) {
|
|
78
|
+
mkdirSync(dirname(join(out, "geml", rel)), { recursive: true });
|
|
79
|
+
writeFileSync(join(out, "geml", rel), text);
|
|
80
|
+
}
|
|
81
|
+
console.log(` ${files.size} documents`);
|
|
82
|
+
|
|
83
|
+
console.log("3. geml check on every document");
|
|
84
|
+
let dirty = 0;
|
|
85
|
+
for (const rel of files.keys()) {
|
|
86
|
+
try { geml("check", "--root", join(out, "geml"), join(out, "geml", rel)); }
|
|
87
|
+
catch (e) { dirty++; console.error(` FAIL ${rel}\n${e.stdout ?? ""}${e.stderr ?? ""}`); }
|
|
88
|
+
}
|
|
89
|
+
if (dirty) { console.error(` ${dirty} document(s) not clean — stopping`); process.exit(1); }
|
|
90
|
+
console.log(" all clean");
|
|
91
|
+
|
|
92
|
+
console.log("4. GEML -> EDN");
|
|
93
|
+
const files2 = new Map();
|
|
94
|
+
for (const rel of files.keys()) files2.set(rel, readFileSync(join(out, "geml", rel), "utf8"));
|
|
95
|
+
const edn2 = gemlFilesToEdn(files2, lib);
|
|
96
|
+
writeFileSync(join(out, "import.edn"), edn2);
|
|
97
|
+
|
|
98
|
+
console.log("5. structural identity, on the real graph");
|
|
99
|
+
if (!same(edn1, edn2)) { console.error(" NOT identical — diff out/export-1.edn against out/import.edn"); process.exit(1); }
|
|
100
|
+
console.log(" identical (EDN semantics)");
|
|
101
|
+
|
|
102
|
+
if (!doEdit) { console.log("\nround trip holds. Re-run with --edit to probe import-back semantics."); process.exit(0); }
|
|
103
|
+
|
|
104
|
+
console.log("6. edit one uuid block via `geml set`, import back, validate");
|
|
105
|
+
const withUuid = [...files.keys()].map((rel) => {
|
|
106
|
+
const text = files2.get(rel);
|
|
107
|
+
const unit = [...addressedUnits(text)].map((a) => a.unit).find((u) => u.kind === "block" && u.id && /^[0-9a-f-]{36}$/.test(u.id));
|
|
108
|
+
return unit ? { rel, unit } : null;
|
|
109
|
+
}).find(Boolean);
|
|
110
|
+
if (!withUuid) { console.log(" no uuid-bearing block in this graph (nothing referenced) — skipping the edit probe"); process.exit(0); }
|
|
111
|
+
|
|
112
|
+
const target = join(out, "geml", withUuid.rel);
|
|
113
|
+
// Keep the block's own head line (type, id, level) — the edit is to the BODY.
|
|
114
|
+
const src = files2.get(withUuid.rel);
|
|
115
|
+
const head = sliceUnit(src, withUuid.unit.span, "head").trimEnd();
|
|
116
|
+
const body = sliceUnit(src, withUuid.unit.span, "body").trimEnd();
|
|
117
|
+
const fence = head.match(/^=+/)[0];
|
|
118
|
+
writeFileSync(join(out, "edit.txt"), `${head}\n${body} — edited by geml\n${fence}\n`);
|
|
119
|
+
geml("set", target, `#${withUuid.unit.id}`, "--in", join(out, "edit.txt"), "--root", join(out, "geml"));
|
|
120
|
+
console.log(` edited #${withUuid.unit.id} in ${withUuid.rel}`);
|
|
121
|
+
|
|
122
|
+
const files3 = new Map();
|
|
123
|
+
for (const rel of files.keys()) files3.set(rel, readFileSync(join(out, "geml", rel), "utf8"));
|
|
124
|
+
writeFileSync(join(out, "import-edited.edn"), gemlFilesToEdn(files3, lib));
|
|
125
|
+
logseq("import-edn", "-g", graph, "-f", join(out, "import-edited.edn"));
|
|
126
|
+
console.log(" imported");
|
|
127
|
+
console.log(logseq("validate", "-g", graph).trim());
|
|
128
|
+
|
|
129
|
+
console.log("7. export again — inspect out/export-2.edn to judge merge semantics");
|
|
130
|
+
logseq("export-edn", "-g", graph, "-f", join(out, "export-2.edn"));
|
|
131
|
+
console.log("done — compare out/export-1.edn / out/export-2.edn");
|