@mnemosyne_os/affine-reader 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/NOTICE.md +42 -0
- package/README.md +89 -0
- package/dist/cli.d.ts +14 -0
- package/dist/cli.js +79 -0
- package/dist/exportMarkdown.d.ts +46 -0
- package/dist/exportMarkdown.js +104 -0
- package/dist/index.d.ts +17 -0
- package/dist/index.js +34 -0
- package/dist/locate.d.ts +24 -0
- package/dist/locate.js +91 -0
- package/dist/nodeSqlite.d.ts +20 -0
- package/dist/nodeSqlite.js +50 -0
- package/dist/read.d.ts +45 -0
- package/dist/read.js +197 -0
- package/dist/stage.d.ts +27 -0
- package/dist/stage.js +54 -0
- package/dist/types.d.ts +72 -0
- package/dist/types.js +10 -0
- package/dist/vendor/affine/blocksuite-types.d.ts +25 -0
- package/dist/vendor/affine/blocksuite-types.js +14 -0
- package/dist/vendor/affine/delta-to-md/delta-converters.d.ts +39 -0
- package/dist/vendor/affine/delta-to-md/delta-converters.js +81 -0
- package/dist/vendor/affine/delta-to-md/delta-to-md.d.ts +1 -0
- package/dist/vendor/affine/delta-to-md/delta-to-md.js +132 -0
- package/dist/vendor/affine/delta-to-md/index.d.ts +2 -0
- package/dist/vendor/affine/delta-to-md/index.js +7 -0
- package/dist/vendor/affine/delta-to-md/utils/node.d.ts +13 -0
- package/dist/vendor/affine/delta-to-md/utils/node.js +59 -0
- package/dist/vendor/affine/delta-to-md/utils/url.d.ts +1 -0
- package/dist/vendor/affine/delta-to-md/utils/url.js +8 -0
- package/dist/vendor/affine/parser.d.ts +5 -0
- package/dist/vendor/affine/parser.js +386 -0
- package/dist/vendor/affine/types.d.ts +100 -0
- package/dist/vendor/affine/types.js +2 -0
- package/package.json +64 -0
package/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2024–2030 Tony Trochet — Mnemosyne OS
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
package/NOTICE.md
ADDED
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
# Third-party code vendored in this package
|
|
2
|
+
|
|
3
|
+
## AFFiNE — `src/vendor/affine/`
|
|
4
|
+
|
|
5
|
+
Copyright (c) 2022-present TOEVERYTHING PTE. LTD. and its affiliates.
|
|
6
|
+
Licensed under the **MIT License**.
|
|
7
|
+
|
|
8
|
+
| | |
|
|
9
|
+
|---|---|
|
|
10
|
+
| Upstream | https://github.com/toeverything/AFFiNE |
|
|
11
|
+
| Path | `packages/common/reader/src/doc-parser/` |
|
|
12
|
+
| Fetched | 2026-09-06, from the repository's default branch |
|
|
13
|
+
| Last upstream commit touching that path | `0c7b20dc18759dc63adbd93df491eba556baa6fe` (2026-08-10) |
|
|
14
|
+
|
|
15
|
+
AFFiNE's repository is under a **mixed** licence: `packages/backend` and
|
|
16
|
+
`packages/common/native` carry a separate one, and everything else — including
|
|
17
|
+
`packages/common/reader` — is MIT. The vendored files come only from the MIT part.
|
|
18
|
+
|
|
19
|
+
The upstream package (`@affine/reader`) is marked `private: true` and is not
|
|
20
|
+
published to npm, which is why the source is vendored rather than depended upon.
|
|
21
|
+
|
|
22
|
+
### What was changed
|
|
23
|
+
|
|
24
|
+
Two lines, both **type-only** and therefore erased at compilation, so runtime
|
|
25
|
+
behaviour is byte-for-byte the upstream one:
|
|
26
|
+
|
|
27
|
+
| File | Before | After |
|
|
28
|
+
|---|---|---|
|
|
29
|
+
| `parser.ts` | `import type { ColumnDataType } from '@blocksuite/affine/model'` | `from './blocksuite-types'` |
|
|
30
|
+
| `types.ts` | `import { type CellDataType } from '@blocksuite/affine/model'` | `from './blocksuite-types'` |
|
|
31
|
+
|
|
32
|
+
`src/vendor/affine/blocksuite-types.ts` is ours, not AFFiNE's. It declares the two
|
|
33
|
+
shapes locally so this package does not need `@blocksuite/affine` (MPL-2.0) on its
|
|
34
|
+
resolution path merely to typecheck. Every changed line is marked
|
|
35
|
+
`// MODIFIED (Mnemosyne OS)` in place.
|
|
36
|
+
|
|
37
|
+
Nothing else was edited. `delta-to-md/` is byte-identical to upstream.
|
|
38
|
+
|
|
39
|
+
## Not vendored
|
|
40
|
+
|
|
41
|
+
**BlockSuite** (MPL-2.0) is not included, not linked, and not required at runtime.
|
|
42
|
+
Rendering a document needs only `yjs` (MIT) and the files above.
|
package/README.md
ADDED
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
# @mnemosyne_os/affine-reader
|
|
2
|
+
|
|
3
|
+
Read a local [AFFiNE](https://github.com/toeverything/AFFiNE) workspace and render its
|
|
4
|
+
documents to Markdown — without running AFFiNE, without BlockSuite, without a server.
|
|
5
|
+
|
|
6
|
+
**Read-only, in every direction.** Two processes writing one CRDT corrupt it, so this
|
|
7
|
+
package copies the database before it reads and never writes back.
|
|
8
|
+
|
|
9
|
+
## Install
|
|
10
|
+
|
|
11
|
+
```bash
|
|
12
|
+
npm install @mnemosyne_os/affine-reader
|
|
13
|
+
```
|
|
14
|
+
|
|
15
|
+
Runtime dependency: `yjs`. That is the whole list.
|
|
16
|
+
|
|
17
|
+
## Use
|
|
18
|
+
|
|
19
|
+
```ts
|
|
20
|
+
import {
|
|
21
|
+
affineDataDir,
|
|
22
|
+
exportWorkspace,
|
|
23
|
+
findWorkspaces,
|
|
24
|
+
openNodeSqlite,
|
|
25
|
+
stageDatabase,
|
|
26
|
+
} from '@mnemosyne_os/affine-reader';
|
|
27
|
+
import { mkdtempSync } from 'node:fs';
|
|
28
|
+
import { tmpdir } from 'node:os';
|
|
29
|
+
import { join } from 'node:path';
|
|
30
|
+
|
|
31
|
+
for (const ref of findWorkspaces()) {
|
|
32
|
+
const staged = stageDatabase(ref.dbPath, mkdtempSync(join(tmpdir(), 'affine-')));
|
|
33
|
+
const db = openNodeSqlite(staged.path);
|
|
34
|
+
try {
|
|
35
|
+
const result = exportWorkspace(db, `./out/${ref.id}`);
|
|
36
|
+
console.log(result.files, result.skipped);
|
|
37
|
+
} finally {
|
|
38
|
+
db.close();
|
|
39
|
+
staged.dispose();
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
```
|
|
43
|
+
|
|
44
|
+
`findWorkspaces()` returning `[]` and `affineDataDir()` returning `null` are **different
|
|
45
|
+
answers** — nothing to read versus AFFiNE not installed. Both are reported.
|
|
46
|
+
|
|
47
|
+
### Bring your own SQLite
|
|
48
|
+
|
|
49
|
+
`openNodeSqlite` uses Node's built-in `node:sqlite`, which needs **Node ≥ 22.5**.
|
|
50
|
+
Electron 31 ships Node 20 and has none, so a host passes its own adapter:
|
|
51
|
+
|
|
52
|
+
```ts
|
|
53
|
+
import Database from 'better-sqlite3';
|
|
54
|
+
import { readWorkspace } from '@mnemosyne_os/affine-reader';
|
|
55
|
+
|
|
56
|
+
const raw = new Database(stagedPath);
|
|
57
|
+
readWorkspace({ prepare: (sql) => raw.prepare(sql), close: () => raw.close() });
|
|
58
|
+
```
|
|
59
|
+
|
|
60
|
+
## What it reads
|
|
61
|
+
|
|
62
|
+
| | |
|
|
63
|
+
|---|---|
|
|
64
|
+
| Location | `<app-data>/AFFiNE/<workspaces\|userspaces>/<peer>/<id>/storage.db` |
|
|
65
|
+
| Schema | nbstore `v2` (`snapshots` + `updates` + `blobs`), and the legacy `v1` shape |
|
|
66
|
+
| Content | Yjs history replayed into a document, rendered by AFFiNE's own MIT parser |
|
|
67
|
+
|
|
68
|
+
Headings, lists, checkboxes, bold/italic, links, code, tables, database views, LaTeX and
|
|
69
|
+
images all survive the trip. Images come out as blob files with links that point at them.
|
|
70
|
+
|
|
71
|
+
## Two things that will bite you
|
|
72
|
+
|
|
73
|
+
**The content may live in the `-wal`, not in `storage.db`.** Measured on one running
|
|
74
|
+
0.27.4 install: at 14:28 `storage.db` was 4 KB next to a 1.8 MB `-wal`; at 14:46, same
|
|
75
|
+
app still running, the journal had been checkpointed away and `storage.db` was 1.14 MB
|
|
76
|
+
with no sidecar at all. A reader that copies only `storage.db` sees an **empty
|
|
77
|
+
workspace** in the first state and cannot tell. `stageDatabase()` copies the sidecars
|
|
78
|
+
when they exist and reports which ones it took.
|
|
79
|
+
|
|
80
|
+
**A snapshot alone is stale.** A document typed one minute earlier decoded to **2
|
|
81
|
+
characters** from its snapshot row and **323** once its 16 `updates` rows were replayed.
|
|
82
|
+
Long-settled documents give the same answer either way — so a test written on those
|
|
83
|
+
passes while the reader loses everything the user just wrote. Every load here replays
|
|
84
|
+
the updates.
|
|
85
|
+
|
|
86
|
+
## Licence
|
|
87
|
+
|
|
88
|
+
MIT. Includes MIT-licensed source vendored from AFFiNE — see [NOTICE.md](./NOTICE.md).
|
|
89
|
+
BlockSuite (MPL-2.0) is **not** included and is not required at runtime.
|
package/dist/cli.d.ts
ADDED
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
/**
|
|
3
|
+
* affine-export — write every local AFFiNE workspace out as Markdown.
|
|
4
|
+
*
|
|
5
|
+
* npx @mnemosyne_os/affine-reader ./out
|
|
6
|
+
*
|
|
7
|
+
* Read-only: the database is copied (with its journal, where the content may
|
|
8
|
+
* actually live) before a byte is read, and nothing is ever written back into
|
|
9
|
+
* AFFiNE's directory.
|
|
10
|
+
*
|
|
11
|
+
* ⚠️ Needs Node >= 22.5 for `node:sqlite`. It says so rather than failing with
|
|
12
|
+
* an opaque module error.
|
|
13
|
+
*/
|
|
14
|
+
export declare function runCli(argv?: string[]): number;
|
package/dist/cli.js
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
"use strict";
|
|
3
|
+
/**
|
|
4
|
+
* affine-export — write every local AFFiNE workspace out as Markdown.
|
|
5
|
+
*
|
|
6
|
+
* npx @mnemosyne_os/affine-reader ./out
|
|
7
|
+
*
|
|
8
|
+
* Read-only: the database is copied (with its journal, where the content may
|
|
9
|
+
* actually live) before a byte is read, and nothing is ever written back into
|
|
10
|
+
* AFFiNE's directory.
|
|
11
|
+
*
|
|
12
|
+
* ⚠️ Needs Node >= 22.5 for `node:sqlite`. It says so rather than failing with
|
|
13
|
+
* an opaque module error.
|
|
14
|
+
*/
|
|
15
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
16
|
+
exports.runCli = runCli;
|
|
17
|
+
const node_fs_1 = require("node:fs");
|
|
18
|
+
const node_os_1 = require("node:os");
|
|
19
|
+
const node_path_1 = require("node:path");
|
|
20
|
+
const exportMarkdown_1 = require("./exportMarkdown");
|
|
21
|
+
const locate_1 = require("./locate");
|
|
22
|
+
const nodeSqlite_1 = require("./nodeSqlite");
|
|
23
|
+
const stage_1 = require("./stage");
|
|
24
|
+
function runCli(argv = process.argv.slice(2)) {
|
|
25
|
+
const outDir = argv[0];
|
|
26
|
+
if (!outDir || outDir === '--help' || outDir === '-h') {
|
|
27
|
+
console.log('usage: affine-export <outDir>');
|
|
28
|
+
console.log(' Writes one Markdown file per AFFiNE document, plus its images.');
|
|
29
|
+
return outDir ? 0 : 2;
|
|
30
|
+
}
|
|
31
|
+
if (!(0, nodeSqlite_1.hasNodeSqlite)()) {
|
|
32
|
+
console.error(`This tool needs Node >= 22.5 (for node:sqlite); this is ${process.version}.`);
|
|
33
|
+
return 1;
|
|
34
|
+
}
|
|
35
|
+
const dataDir = (0, locate_1.affineDataDir)();
|
|
36
|
+
const workspaces = (0, locate_1.findWorkspaces)();
|
|
37
|
+
if (workspaces.length === 0) {
|
|
38
|
+
// 🎭 Two different answers, and the person needs to know which one they got.
|
|
39
|
+
console.error(dataDir
|
|
40
|
+
? `AFFiNE is installed (${dataDir}) but holds no workspace.`
|
|
41
|
+
: 'AFFiNE does not appear to be installed for this user.');
|
|
42
|
+
return 1;
|
|
43
|
+
}
|
|
44
|
+
let failures = 0;
|
|
45
|
+
for (const ref of workspaces) {
|
|
46
|
+
const staged = (0, stage_1.stageDatabase)(ref.dbPath, (0, node_fs_1.mkdtempSync)((0, node_path_1.join)((0, node_os_1.tmpdir)(), 'affine-read-')));
|
|
47
|
+
try {
|
|
48
|
+
const db = (0, nodeSqlite_1.openNodeSqlite)(staged.path);
|
|
49
|
+
try {
|
|
50
|
+
const result = (0, exportMarkdown_1.exportWorkspace)(db, (0, node_path_1.join)(outDir, (0, exportMarkdown_1.workspaceFolderName)(ref)));
|
|
51
|
+
console.log(`\n${ref.kind} ${ref.id} — peer ${ref.peer}, schema ${result.schema}, ` +
|
|
52
|
+
`journal ${staged.sidecars.join('+') || 'none'}`);
|
|
53
|
+
for (const doc of result.docs)
|
|
54
|
+
console.log(` ${String(doc.markdown.length).padStart(7)} chars ` +
|
|
55
|
+
`${doc.hadSnapshot ? 'snapshot' : 'no snapshot'}+${doc.updatesReplayed} ` +
|
|
56
|
+
`${doc.title || '(untitled)'}`);
|
|
57
|
+
// Never hidden: a document left out and a document that does not exist
|
|
58
|
+
// look identical in a count.
|
|
59
|
+
for (const skip of result.skipped) {
|
|
60
|
+
failures++;
|
|
61
|
+
console.log(` SKIPPED ${skip.id} — ${skip.reason}`);
|
|
62
|
+
}
|
|
63
|
+
console.log(` → ${result.files.length} file(s), ${result.blobsWritten} image(s)` +
|
|
64
|
+
(result.trashed ? `, ${result.trashed} left in AFFiNE's trash` : ''));
|
|
65
|
+
}
|
|
66
|
+
finally {
|
|
67
|
+
db.close();
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
finally {
|
|
71
|
+
staged.dispose();
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
return failures > 0 ? 1 : 0;
|
|
75
|
+
}
|
|
76
|
+
// `require.main === module` rather than a bare call: importing this file from a
|
|
77
|
+
// test must not run the tool against the developer's own AFFiNE install.
|
|
78
|
+
if (require.main === module)
|
|
79
|
+
process.exit(runCli());
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Writing a workspace out as Markdown + blob files, ready for a watched folder.
|
|
3
|
+
*/
|
|
4
|
+
import type { ReadOptions, SqliteDatabase, WorkspaceContent } from './types';
|
|
5
|
+
export interface ExportResult extends WorkspaceContent {
|
|
6
|
+
outDir: string;
|
|
7
|
+
files: {
|
|
8
|
+
docId: string;
|
|
9
|
+
path: string;
|
|
10
|
+
}[];
|
|
11
|
+
blobsWritten: number;
|
|
12
|
+
}
|
|
13
|
+
export interface ExportOptions extends ReadOptions {
|
|
14
|
+
/** Also write the blob bytes next to the docs. Default true. */
|
|
15
|
+
writeBlobs?: boolean;
|
|
16
|
+
}
|
|
17
|
+
/** Filesystem-safe stem. Accents are folded, never dropped into an empty name. */
|
|
18
|
+
export declare function slugify(input: string, fallback: string): string;
|
|
19
|
+
/**
|
|
20
|
+
* A file name that is stable across runs and unique within a workspace.
|
|
21
|
+
*
|
|
22
|
+
* The doc id is part of the name on purpose: a watched folder is re-exported over
|
|
23
|
+
* and over, and keying the file on the TITLE would leave an orphan copy behind
|
|
24
|
+
* every time someone renames a doc — the folder would slowly fill with duplicates
|
|
25
|
+
* of the same document under its old names.
|
|
26
|
+
*/
|
|
27
|
+
export declare function fileNameFor(title: string, docId: string): string;
|
|
28
|
+
/**
|
|
29
|
+
* The folder one workspace is exported into, relative to the destination.
|
|
30
|
+
*
|
|
31
|
+
* 🚨 The peer is part of the name, not decoration. A workspace synced from a
|
|
32
|
+
* server and a local one can carry the SAME id, and a name built from the id
|
|
33
|
+
* alone makes the second export silently overwrite the first. Both callers —
|
|
34
|
+
* the CLI and the app's IPC handler — go through this one function, so the two
|
|
35
|
+
* doors cannot resolve the same workspace to two different directories.
|
|
36
|
+
*/
|
|
37
|
+
export declare function workspaceFolderName(ref: {
|
|
38
|
+
id: string;
|
|
39
|
+
peer: string;
|
|
40
|
+
}): string;
|
|
41
|
+
/**
|
|
42
|
+
* Read `db` and write one Markdown file per doc into `outDir`.
|
|
43
|
+
*
|
|
44
|
+
* The caller owns `outDir`. Nothing is ever written back into AFFiNE's directory.
|
|
45
|
+
*/
|
|
46
|
+
export declare function exportWorkspace(db: SqliteDatabase, outDir: string, options?: ExportOptions): ExportResult;
|
|
@@ -0,0 +1,104 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* Writing a workspace out as Markdown + blob files, ready for a watched folder.
|
|
4
|
+
*/
|
|
5
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
+
exports.slugify = slugify;
|
|
7
|
+
exports.fileNameFor = fileNameFor;
|
|
8
|
+
exports.workspaceFolderName = workspaceFolderName;
|
|
9
|
+
exports.exportWorkspace = exportWorkspace;
|
|
10
|
+
const node_fs_1 = require("node:fs");
|
|
11
|
+
const node_path_1 = require("node:path");
|
|
12
|
+
const read_1 = require("./read");
|
|
13
|
+
/** Filesystem-safe stem. Accents are folded, never dropped into an empty name. */
|
|
14
|
+
function slugify(input, fallback) {
|
|
15
|
+
const cleaned = input
|
|
16
|
+
.normalize('NFD')
|
|
17
|
+
.replace(/[̀-ͯ]/g, '')
|
|
18
|
+
.replace(/[^a-zA-Z0-9]+/g, '-')
|
|
19
|
+
.replace(/^-+|-+$/g, '')
|
|
20
|
+
.slice(0, 60);
|
|
21
|
+
return cleaned || fallback;
|
|
22
|
+
}
|
|
23
|
+
/**
|
|
24
|
+
* A file name that is stable across runs and unique within a workspace.
|
|
25
|
+
*
|
|
26
|
+
* The doc id is part of the name on purpose: a watched folder is re-exported over
|
|
27
|
+
* and over, and keying the file on the TITLE would leave an orphan copy behind
|
|
28
|
+
* every time someone renames a doc — the folder would slowly fill with duplicates
|
|
29
|
+
* of the same document under its old names.
|
|
30
|
+
*/
|
|
31
|
+
function fileNameFor(title, docId) {
|
|
32
|
+
return `${slugify(title, 'doc')}--${slugify(docId, 'id')}.md`;
|
|
33
|
+
}
|
|
34
|
+
/**
|
|
35
|
+
* The folder one workspace is exported into, relative to the destination.
|
|
36
|
+
*
|
|
37
|
+
* 🚨 The peer is part of the name, not decoration. A workspace synced from a
|
|
38
|
+
* server and a local one can carry the SAME id, and a name built from the id
|
|
39
|
+
* alone makes the second export silently overwrite the first. Both callers —
|
|
40
|
+
* the CLI and the app's IPC handler — go through this one function, so the two
|
|
41
|
+
* doors cannot resolve the same workspace to two different directories.
|
|
42
|
+
*/
|
|
43
|
+
function workspaceFolderName(ref) {
|
|
44
|
+
return `affine-${slugify(ref.peer, 'peer')}-${slugify(ref.id, 'id')}`;
|
|
45
|
+
}
|
|
46
|
+
function blobFileName(key, mime) {
|
|
47
|
+
const ext = (mime.split('/')[1] ?? 'bin').replace('+xml', '').replace(/[^a-z0-9]/gi, '');
|
|
48
|
+
return `${slugify(key, 'blob')}.${ext || 'bin'}`;
|
|
49
|
+
}
|
|
50
|
+
function frontMatter(fields) {
|
|
51
|
+
const lines = Object.entries(fields).map(([key, value]) => typeof value === 'string' ? `${key}: ${JSON.stringify(value)}` : `${key}: ${value}`);
|
|
52
|
+
return ['---', ...lines, '---', ''].join('\n');
|
|
53
|
+
}
|
|
54
|
+
/**
|
|
55
|
+
* Read `db` and write one Markdown file per doc into `outDir`.
|
|
56
|
+
*
|
|
57
|
+
* The caller owns `outDir`. Nothing is ever written back into AFFiNE's directory.
|
|
58
|
+
*/
|
|
59
|
+
function exportWorkspace(db, outDir, options = {}) {
|
|
60
|
+
(0, node_fs_1.mkdirSync)(outDir, { recursive: true });
|
|
61
|
+
// Blob file names are decided BEFORE the docs are rendered, so an image link and
|
|
62
|
+
// the file it points at come from the SAME map. Computing them separately is how
|
|
63
|
+
// an export ends up with links that look right and resolve to nothing.
|
|
64
|
+
const schema = (0, read_1.detectSchema)(db);
|
|
65
|
+
const blobNames = new Map();
|
|
66
|
+
for (const blob of (0, read_1.readBlobIndex)(db, schema))
|
|
67
|
+
blobNames.set(blob.key, blobFileName(blob.key, blob.mime));
|
|
68
|
+
const rendered = (0, read_1.readWorkspace)(db, {
|
|
69
|
+
...options,
|
|
70
|
+
blobUrl: (key) => {
|
|
71
|
+
const name = blobNames.get(key);
|
|
72
|
+
return name ? `blobs/${name}` : (options.blobUrl?.(key) ?? null);
|
|
73
|
+
},
|
|
74
|
+
});
|
|
75
|
+
let blobsWritten = 0;
|
|
76
|
+
if (options.writeBlobs !== false && rendered.blobs.length > 0) {
|
|
77
|
+
const blobDir = (0, node_path_1.join)(outDir, 'blobs');
|
|
78
|
+
(0, node_fs_1.mkdirSync)(blobDir, { recursive: true });
|
|
79
|
+
(0, read_1.forEachBlob)(db, rendered.schema, (blob) => {
|
|
80
|
+
const name = blobNames.get(blob.key) ?? blobFileName(blob.key, blob.mime);
|
|
81
|
+
(0, node_fs_1.writeFileSync)((0, node_path_1.join)(blobDir, name), blob.data);
|
|
82
|
+
blobsWritten++;
|
|
83
|
+
});
|
|
84
|
+
}
|
|
85
|
+
const files = [];
|
|
86
|
+
for (const doc of rendered.docs) {
|
|
87
|
+
const path = (0, node_path_1.join)(outDir, fileNameFor(doc.title, doc.id));
|
|
88
|
+
// 🪤 No timestamp here, deliberately. A `read_at` written on every export
|
|
89
|
+
// changes the bytes of every file on every run, so a watched folder would
|
|
90
|
+
// see the whole workspace as modified and re-ingest (and re-embed) all of
|
|
91
|
+
// it each time — churn and cost for documents nobody touched. What changed
|
|
92
|
+
// belongs in the log and the result, not in content that is hashed.
|
|
93
|
+
const body = frontMatter({
|
|
94
|
+
source: 'affine',
|
|
95
|
+
workspace: rendered.workspaceId,
|
|
96
|
+
doc_id: doc.id,
|
|
97
|
+
title: doc.title,
|
|
98
|
+
});
|
|
99
|
+
const heading = doc.title ? `# ${doc.title}\n\n` : '';
|
|
100
|
+
(0, node_fs_1.writeFileSync)(path, `${body}${heading}${doc.markdown}`, 'utf8');
|
|
101
|
+
files.push({ docId: doc.id, path });
|
|
102
|
+
}
|
|
103
|
+
return { ...rendered, outDir, files, blobsWritten };
|
|
104
|
+
}
|
package/dist/index.d.ts
ADDED
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @mnemosyne_os/affine-reader — read a local AFFiNE workspace, without AFFiNE.
|
|
3
|
+
*
|
|
4
|
+
* Reads the SQLite database AFFiNE keeps on disk, replays its Yjs history and
|
|
5
|
+
* renders each document to Markdown using AFFiNE's own MIT parser (vendored, see
|
|
6
|
+
* NOTICE.md). No BlockSuite at runtime, no native module, no server.
|
|
7
|
+
*
|
|
8
|
+
* ⛔ READ ONLY, in every direction. Two processes writing one CRDT corrupt it.
|
|
9
|
+
*/
|
|
10
|
+
export { affineDataDir, findWorkspaces } from './locate';
|
|
11
|
+
export { SIDECAR_SUFFIXES, stageDatabase } from './stage';
|
|
12
|
+
export type { StagedDatabase } from './stage';
|
|
13
|
+
export { detectSchema, forEachBlob, loadDoc, readBlobIndex, readDocIndex, readWorkspace, readWorkspaceId, } from './read';
|
|
14
|
+
export { exportWorkspace, fileNameFor, slugify, workspaceFolderName } from './exportMarkdown';
|
|
15
|
+
export type { ExportOptions, ExportResult } from './exportMarkdown';
|
|
16
|
+
export { hasNodeSqlite, openNodeSqlite } from './nodeSqlite';
|
|
17
|
+
export type { AffineDoc, AffineSchema, BlobRef, ReadOptions, SkippedDoc, SqliteDatabase, SqliteOpener, SqliteStatement, WorkspaceContent, WorkspaceRef, } from './types';
|
package/dist/index.js
ADDED
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* @mnemosyne_os/affine-reader — read a local AFFiNE workspace, without AFFiNE.
|
|
4
|
+
*
|
|
5
|
+
* Reads the SQLite database AFFiNE keeps on disk, replays its Yjs history and
|
|
6
|
+
* renders each document to Markdown using AFFiNE's own MIT parser (vendored, see
|
|
7
|
+
* NOTICE.md). No BlockSuite at runtime, no native module, no server.
|
|
8
|
+
*
|
|
9
|
+
* ⛔ READ ONLY, in every direction. Two processes writing one CRDT corrupt it.
|
|
10
|
+
*/
|
|
11
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
12
|
+
exports.openNodeSqlite = exports.hasNodeSqlite = exports.workspaceFolderName = exports.slugify = exports.fileNameFor = exports.exportWorkspace = exports.readWorkspaceId = exports.readWorkspace = exports.readDocIndex = exports.readBlobIndex = exports.loadDoc = exports.forEachBlob = exports.detectSchema = exports.stageDatabase = exports.SIDECAR_SUFFIXES = exports.findWorkspaces = exports.affineDataDir = void 0;
|
|
13
|
+
var locate_1 = require("./locate");
|
|
14
|
+
Object.defineProperty(exports, "affineDataDir", { enumerable: true, get: function () { return locate_1.affineDataDir; } });
|
|
15
|
+
Object.defineProperty(exports, "findWorkspaces", { enumerable: true, get: function () { return locate_1.findWorkspaces; } });
|
|
16
|
+
var stage_1 = require("./stage");
|
|
17
|
+
Object.defineProperty(exports, "SIDECAR_SUFFIXES", { enumerable: true, get: function () { return stage_1.SIDECAR_SUFFIXES; } });
|
|
18
|
+
Object.defineProperty(exports, "stageDatabase", { enumerable: true, get: function () { return stage_1.stageDatabase; } });
|
|
19
|
+
var read_1 = require("./read");
|
|
20
|
+
Object.defineProperty(exports, "detectSchema", { enumerable: true, get: function () { return read_1.detectSchema; } });
|
|
21
|
+
Object.defineProperty(exports, "forEachBlob", { enumerable: true, get: function () { return read_1.forEachBlob; } });
|
|
22
|
+
Object.defineProperty(exports, "loadDoc", { enumerable: true, get: function () { return read_1.loadDoc; } });
|
|
23
|
+
Object.defineProperty(exports, "readBlobIndex", { enumerable: true, get: function () { return read_1.readBlobIndex; } });
|
|
24
|
+
Object.defineProperty(exports, "readDocIndex", { enumerable: true, get: function () { return read_1.readDocIndex; } });
|
|
25
|
+
Object.defineProperty(exports, "readWorkspace", { enumerable: true, get: function () { return read_1.readWorkspace; } });
|
|
26
|
+
Object.defineProperty(exports, "readWorkspaceId", { enumerable: true, get: function () { return read_1.readWorkspaceId; } });
|
|
27
|
+
var exportMarkdown_1 = require("./exportMarkdown");
|
|
28
|
+
Object.defineProperty(exports, "exportWorkspace", { enumerable: true, get: function () { return exportMarkdown_1.exportWorkspace; } });
|
|
29
|
+
Object.defineProperty(exports, "fileNameFor", { enumerable: true, get: function () { return exportMarkdown_1.fileNameFor; } });
|
|
30
|
+
Object.defineProperty(exports, "slugify", { enumerable: true, get: function () { return exportMarkdown_1.slugify; } });
|
|
31
|
+
Object.defineProperty(exports, "workspaceFolderName", { enumerable: true, get: function () { return exportMarkdown_1.workspaceFolderName; } });
|
|
32
|
+
var nodeSqlite_1 = require("./nodeSqlite");
|
|
33
|
+
Object.defineProperty(exports, "hasNodeSqlite", { enumerable: true, get: function () { return nodeSqlite_1.hasNodeSqlite; } });
|
|
34
|
+
Object.defineProperty(exports, "openNodeSqlite", { enumerable: true, get: function () { return nodeSqlite_1.openNodeSqlite; } });
|
package/dist/locate.d.ts
ADDED
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Finding AFFiNE workspaces on disk.
|
|
3
|
+
*
|
|
4
|
+
* Layout, from AFFiNE's own comment in
|
|
5
|
+
* `packages/frontend/apps/electron/src/helper/dialog/dialog.ts`, verified against a
|
|
6
|
+
* real 0.27.4 install on Windows 2026-09-06:
|
|
7
|
+
*
|
|
8
|
+
* <app-data>/AFFiNE/<workspaces|userspaces>/<peer>/<workspace-id>/storage.db (v2)
|
|
9
|
+
* <app-data>/AFFiNE/<workspaces|userspaces>/<workspace-id>/storage.db (v1, legacy)
|
|
10
|
+
*
|
|
11
|
+
* ⚠️ The v1 branch is written from their source, NOT measured — no v1 install was
|
|
12
|
+
* available. `layout` says which one a ref came from so a caller can tell them apart.
|
|
13
|
+
*/
|
|
14
|
+
import type { WorkspaceRef } from './types';
|
|
15
|
+
/** Root of AFFiNE's application data, or null when the app was never installed here. */
|
|
16
|
+
export declare function affineDataDir(env?: NodeJS.ProcessEnv, platform?: NodeJS.Platform): string | null;
|
|
17
|
+
/**
|
|
18
|
+
* Every workspace reachable on this machine. Never throws for a missing AFFiNE —
|
|
19
|
+
* an empty list means "nothing found", and `affineDataDir()` says whether AFFiNE
|
|
20
|
+
* is installed at all. The two are different answers and callers need both.
|
|
21
|
+
*/
|
|
22
|
+
export declare function findWorkspaces(opts?: {
|
|
23
|
+
dataDir?: string | null;
|
|
24
|
+
}): WorkspaceRef[];
|
package/dist/locate.js
ADDED
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* Finding AFFiNE workspaces on disk.
|
|
4
|
+
*
|
|
5
|
+
* Layout, from AFFiNE's own comment in
|
|
6
|
+
* `packages/frontend/apps/electron/src/helper/dialog/dialog.ts`, verified against a
|
|
7
|
+
* real 0.27.4 install on Windows 2026-09-06:
|
|
8
|
+
*
|
|
9
|
+
* <app-data>/AFFiNE/<workspaces|userspaces>/<peer>/<workspace-id>/storage.db (v2)
|
|
10
|
+
* <app-data>/AFFiNE/<workspaces|userspaces>/<workspace-id>/storage.db (v1, legacy)
|
|
11
|
+
*
|
|
12
|
+
* ⚠️ The v1 branch is written from their source, NOT measured — no v1 install was
|
|
13
|
+
* available. `layout` says which one a ref came from so a caller can tell them apart.
|
|
14
|
+
*/
|
|
15
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
16
|
+
exports.affineDataDir = affineDataDir;
|
|
17
|
+
exports.findWorkspaces = findWorkspaces;
|
|
18
|
+
const node_fs_1 = require("node:fs");
|
|
19
|
+
const node_path_1 = require("node:path");
|
|
20
|
+
const SPACE_DIRS = ['workspaces', 'userspaces'];
|
|
21
|
+
/** Root of AFFiNE's application data, or null when the app was never installed here. */
|
|
22
|
+
function affineDataDir(env = process.env, platform = process.platform) {
|
|
23
|
+
const home = env.HOME ?? env.USERPROFILE ?? '';
|
|
24
|
+
const base = platform === 'win32'
|
|
25
|
+
? env.APPDATA
|
|
26
|
+
: platform === 'darwin'
|
|
27
|
+
? home && (0, node_path_1.join)(home, 'Library', 'Application Support')
|
|
28
|
+
: (env.XDG_CONFIG_HOME ?? (home && (0, node_path_1.join)(home, '.config')));
|
|
29
|
+
if (!base)
|
|
30
|
+
return null;
|
|
31
|
+
const dir = (0, node_path_1.join)(base, 'AFFiNE');
|
|
32
|
+
return (0, node_fs_1.existsSync)(dir) ? dir : null;
|
|
33
|
+
}
|
|
34
|
+
function isDir(path) {
|
|
35
|
+
try {
|
|
36
|
+
return (0, node_fs_1.statSync)(path).isDirectory();
|
|
37
|
+
}
|
|
38
|
+
catch {
|
|
39
|
+
// A racing uninstall, or a permission we do not have. Not a directory we can
|
|
40
|
+
// walk, which is all the caller needs — and the failure is not swallowed for
|
|
41
|
+
// any decision: an unreadable entry simply yields no workspace.
|
|
42
|
+
return false;
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
/**
|
|
46
|
+
* Every workspace reachable on this machine. Never throws for a missing AFFiNE —
|
|
47
|
+
* an empty list means "nothing found", and `affineDataDir()` says whether AFFiNE
|
|
48
|
+
* is installed at all. The two are different answers and callers need both.
|
|
49
|
+
*/
|
|
50
|
+
function findWorkspaces(opts = {}) {
|
|
51
|
+
const dataDir = opts.dataDir === undefined ? affineDataDir() : opts.dataDir;
|
|
52
|
+
if (!dataDir || !isDir(dataDir))
|
|
53
|
+
return [];
|
|
54
|
+
const found = [];
|
|
55
|
+
for (const kindDir of SPACE_DIRS) {
|
|
56
|
+
const kind = kindDir === 'workspaces' ? 'workspace' : 'userspace';
|
|
57
|
+
const root = (0, node_path_1.join)(dataDir, kindDir);
|
|
58
|
+
if (!isDir(root))
|
|
59
|
+
continue;
|
|
60
|
+
for (const entry of safeReaddir(root)) {
|
|
61
|
+
const entryPath = (0, node_path_1.join)(root, entry);
|
|
62
|
+
if (!isDir(entryPath))
|
|
63
|
+
continue;
|
|
64
|
+
// v1 put storage.db directly under the workspace id; v2 inserted a <peer> level.
|
|
65
|
+
const v1Db = (0, node_path_1.join)(entryPath, 'storage.db');
|
|
66
|
+
if ((0, node_fs_1.existsSync)(v1Db)) {
|
|
67
|
+
found.push({ id: entry, peer: 'local', kind, dbPath: v1Db, layout: 'v1' });
|
|
68
|
+
continue;
|
|
69
|
+
}
|
|
70
|
+
for (const workspaceId of safeReaddir(entryPath)) {
|
|
71
|
+
const dbPath = (0, node_path_1.join)(entryPath, workspaceId, 'storage.db');
|
|
72
|
+
if ((0, node_fs_1.existsSync)(dbPath))
|
|
73
|
+
found.push({ id: workspaceId, peer: entry, kind, dbPath, layout: 'v2' });
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
return found;
|
|
78
|
+
}
|
|
79
|
+
function safeReaddir(path) {
|
|
80
|
+
try {
|
|
81
|
+
return (0, node_fs_1.readdirSync)(path);
|
|
82
|
+
}
|
|
83
|
+
catch (error) {
|
|
84
|
+
// ⚠️ An unreadable directory is NOT an empty one, and the caller cannot tell
|
|
85
|
+
// the two apart from a list. It is at least said out loud rather than
|
|
86
|
+
// skipped in silence (rule 7) — a permission error here surfaces to whoever
|
|
87
|
+
// reads the log as "we could not look", not as "there is nothing there".
|
|
88
|
+
console.warn(`[affine-reader] could not read ${path}: ${error instanceof Error ? error.message : String(error)}`);
|
|
89
|
+
return [];
|
|
90
|
+
}
|
|
91
|
+
}
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* A read-only SQLite adapter backed by Node's built-in `node:sqlite`.
|
|
3
|
+
*
|
|
4
|
+
* ⚠️ `node:sqlite` landed in Node 22.5. Electron 31 ships Node 20, so the app CANNOT
|
|
5
|
+
* use this one — it passes an adapter over its own better-sqlite3. This exists for
|
|
6
|
+
* scripts and tests, and it says so when the runtime cannot provide it instead of
|
|
7
|
+
* failing with an opaque module error.
|
|
8
|
+
*/
|
|
9
|
+
import type { SqliteOpener } from './types';
|
|
10
|
+
/** True when this runtime can provide `node:sqlite`. */
|
|
11
|
+
export declare function hasNodeSqlite(): boolean;
|
|
12
|
+
/**
|
|
13
|
+
* Opens a STAGED COPY. Never point this at a live AFFiNE file.
|
|
14
|
+
*
|
|
15
|
+
* 🪤 Deliberately NOT opened with `readOnly: true`: recovering a WAL needs to touch
|
|
16
|
+
* the -shm, and a read-only handle on a database with a journal fails instead of
|
|
17
|
+
* reading it. Safety here comes from `stageDatabase()` — the file is our own copy —
|
|
18
|
+
* not from a flag that would break the very case the copy exists for.
|
|
19
|
+
*/
|
|
20
|
+
export declare const openNodeSqlite: SqliteOpener;
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* A read-only SQLite adapter backed by Node's built-in `node:sqlite`.
|
|
4
|
+
*
|
|
5
|
+
* ⚠️ `node:sqlite` landed in Node 22.5. Electron 31 ships Node 20, so the app CANNOT
|
|
6
|
+
* use this one — it passes an adapter over its own better-sqlite3. This exists for
|
|
7
|
+
* scripts and tests, and it says so when the runtime cannot provide it instead of
|
|
8
|
+
* failing with an opaque module error.
|
|
9
|
+
*/
|
|
10
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
11
|
+
exports.openNodeSqlite = void 0;
|
|
12
|
+
exports.hasNodeSqlite = hasNodeSqlite;
|
|
13
|
+
const node_module_1 = require("node:module");
|
|
14
|
+
/** True when this runtime can provide `node:sqlite`. */
|
|
15
|
+
function hasNodeSqlite() {
|
|
16
|
+
try {
|
|
17
|
+
(0, node_module_1.createRequire)(__filename)('node:sqlite');
|
|
18
|
+
return true;
|
|
19
|
+
}
|
|
20
|
+
catch {
|
|
21
|
+
// Not an error to report: on Node 20 / Electron this is simply the answer,
|
|
22
|
+
// and the caller is expected to bring its own adapter.
|
|
23
|
+
return false;
|
|
24
|
+
}
|
|
25
|
+
}
|
|
26
|
+
/**
|
|
27
|
+
* Opens a STAGED COPY. Never point this at a live AFFiNE file.
|
|
28
|
+
*
|
|
29
|
+
* 🪤 Deliberately NOT opened with `readOnly: true`: recovering a WAL needs to touch
|
|
30
|
+
* the -shm, and a read-only handle on a database with a journal fails instead of
|
|
31
|
+
* reading it. Safety here comes from `stageDatabase()` — the file is our own copy —
|
|
32
|
+
* not from a flag that would break the very case the copy exists for.
|
|
33
|
+
*/
|
|
34
|
+
const openNodeSqlite = (path) => {
|
|
35
|
+
let mod;
|
|
36
|
+
try {
|
|
37
|
+
mod = (0, node_module_1.createRequire)(__filename)('node:sqlite');
|
|
38
|
+
}
|
|
39
|
+
catch (error) {
|
|
40
|
+
throw new Error('node:sqlite is unavailable in this runtime (it needs Node >= 22.5; Electron 31 ships Node 20). ' +
|
|
41
|
+
'Pass your own SqliteOpener instead. Cause: ' +
|
|
42
|
+
(error instanceof Error ? error.message : String(error)));
|
|
43
|
+
}
|
|
44
|
+
const db = new mod.DatabaseSync(path);
|
|
45
|
+
return {
|
|
46
|
+
prepare: (sql) => db.prepare(sql),
|
|
47
|
+
close: () => db.close(),
|
|
48
|
+
};
|
|
49
|
+
};
|
|
50
|
+
exports.openNodeSqlite = openNodeSqlite;
|