@mnemosyne_os/affine-reader 0.1.0

package/dist/read.d.ts

@@ -0,0 +1,45 @@
+/**
+ * Turning an AFFiNE workspace database into documents.
+ *
+ * The pipeline, measured end to end against AFFiNE 0.27.4 on 2026-09-06:
+ *   storage.db (+ -wal) -> snapshot + updates -> Y.Doc -> AFFiNE's own parser -> Markdown
+ *
+ * 🚨 A snapshot ALONE is stale. A doc typed one minute earlier decoded to 2
+ * characters from its snapshot and 323 once its 16 `updates` rows were replayed.
+ * The onboarding docs, whose snapshots are current, give the SAME answer either
+ * way — so a test written on them passes while the reader loses everything the
+ * person just wrote. Every load here replays the updates.
+ */
+import * as Y from 'yjs';
+import type { AffineSchema, BlobRef, ReadOptions, SqliteDatabase, WorkspaceContent } from './types';
+/** Reads the schema the file actually has rather than trusting a directory shape. */
+export declare function detectSchema(db: SqliteDatabase): AffineSchema;
+/** The workspace id, which is ALSO the id of the root doc holding the doc index. */
+export declare function readWorkspaceId(db: SqliteDatabase, schema: AffineSchema): string | null;
+interface LoadResult {
+    doc: Y.Doc;
+    hadSnapshot: boolean;
+    updatesReplayed: number;
+}
+/** Snapshot first, then every update in order. Both halves, always. */
+export declare function loadDoc(db: SqliteDatabase, schema: AffineSchema, docId: string | null): LoadResult;
+/**
+ * The doc index, straight out of the root doc's `meta.pages`.
+ * (This is AFFiNE's `readAllDocsFromRootDoc`, which is pure yjs.)
+ */
+export declare function readDocIndex(rootDoc: Y.Doc): {
+    id: string;
+    title: string;
+    trash: boolean;
+}[];
+/** Blob metadata only — the bytes are streamed separately, a workspace can hold GBs. */
+export declare function readBlobIndex(db: SqliteDatabase, schema: AffineSchema): BlobRef[];
+/** Streams blob bytes one row at a time so a large workspace never lands in memory at once. */
+export declare function forEachBlob(db: SqliteDatabase, schema: AffineSchema, visit: (blob: {
+    key: string;
+    mime: string;
+    data: Uint8Array;
+}) => void): void;
+/** Every doc of one workspace, rendered. Docs that fail are NAMED, never dropped. */
+export declare function readWorkspace(db: SqliteDatabase, options?: ReadOptions): WorkspaceContent;
+export {};

package/dist/read.js

@@ -0,0 +1,197 @@
+"use strict";
+/**
+ * Turning an AFFiNE workspace database into documents.
+ *
+ * The pipeline, measured end to end against AFFiNE 0.27.4 on 2026-09-06:
+ *   storage.db (+ -wal) -> snapshot + updates -> Y.Doc -> AFFiNE's own parser -> Markdown
+ *
+ * 🚨 A snapshot ALONE is stale. A doc typed one minute earlier decoded to 2
+ * characters from its snapshot and 323 once its 16 `updates` rows were replayed.
+ * The onboarding docs, whose snapshots are current, give the SAME answer either
+ * way — so a test written on them passes while the reader loses everything the
+ * person just wrote. Every load here replays the updates.
+ */
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.detectSchema = detectSchema;
+exports.readWorkspaceId = readWorkspaceId;
+exports.loadDoc = loadDoc;
+exports.readDocIndex = readDocIndex;
+exports.readBlobIndex = readBlobIndex;
+exports.forEachBlob = forEachBlob;
+exports.readWorkspace = readWorkspace;
+const Y = __importStar(require("yjs"));
+const parser_1 = require("./vendor/affine/parser");
+/** Reads the schema the file actually has rather than trusting a directory shape. */
+function detectSchema(db) {
+    const tables = new Set(db
+        .prepare("SELECT name FROM sqlite_master WHERE type = 'table'")
+        .all()
+        .map((row) => String(row.name)));
+    if (!tables.has('updates'))
+        throw new Error('Not an AFFiNE workspace database: no `updates` table');
+    return tables.has('snapshots') ? 'v2' : 'v1';
+}
+/** The workspace id, which is ALSO the id of the root doc holding the doc index. */
+function readWorkspaceId(db, schema) {
+    if (schema === 'v2') {
+        const row = db.prepare('SELECT space_id FROM meta').get();
+        return row && typeof row.space_id === 'string' ? row.space_id : null;
+    }
+    // v1 has no `meta`. The root doc is the one whose updates carry no doc_id.
+    return null;
+}
+function toBytes(value, what) {
+    if (value instanceof Uint8Array)
+        return value;
+    if (value instanceof ArrayBuffer)
+        return new Uint8Array(value);
+    throw new Error(`${what}: expected a BLOB, got ${typeof value}`);
+}
+/** Snapshot first, then every update in order. Both halves, always. */
+function loadDoc(db, schema, docId) {
+    const doc = new Y.Doc();
+    let hadSnapshot = false;
+    if (schema === 'v2' && docId !== null) {
+        const snap = db.prepare('SELECT data FROM snapshots WHERE doc_id = ?').get(docId);
+        if (snap) {
+            Y.applyUpdate(doc, toBytes(snap.data, `snapshot of ${docId}`));
+            hadSnapshot = true;
+        }
+    }
+    const rows = schema === 'v2'
+        ? db.prepare('SELECT data FROM updates WHERE doc_id = ? ORDER BY created_at').all(docId)
+        : docId === null
+            ? db.prepare('SELECT data FROM updates WHERE doc_id IS NULL ORDER BY id').all()
+            : db
+                .prepare('SELECT data FROM updates WHERE doc_id = ? ORDER BY id')
+                .all(docId);
+    for (const row of rows)
+        Y.applyUpdate(doc, toBytes(row.data, `update of ${docId ?? 'root'}`));
+    return { doc, hadSnapshot, updatesReplayed: rows.length };
+}
+/**
+ * The doc index, straight out of the root doc's `meta.pages`.
+ * (This is AFFiNE's `readAllDocsFromRootDoc`, which is pure yjs.)
+ */
+function readDocIndex(rootDoc) {
+    const pages = rootDoc.getMap('meta').get('pages');
+    if (!(pages instanceof Y.Array))
+        return [];
+    const out = [];
+    for (const page of pages) {
+        if (!(page instanceof Y.Map))
+            continue;
+        const id = page.get('id');
+        if (typeof id !== 'string')
+            continue;
+        const title = page.get('title');
+        out.push({
+            id,
+            title: typeof title === 'string' ? title : '',
+            trash: page.get('trash') === true,
+        });
+    }
+    return out;
+}
+/** Blob metadata only — the bytes are streamed separately, a workspace can hold GBs. */
+function readBlobIndex(db, schema) {
+    const rows = schema === 'v2'
+        ? db.prepare('SELECT key, mime, size FROM blobs WHERE deleted_at IS NULL').all()
+        : db.prepare('SELECT key, length(data) AS size FROM blobs').all();
+    return rows.map((row) => ({
+        key: String(row.key),
+        // v1 stored no mime. An unknown type is unknown — never guessed into image/png.
+        mime: typeof row.mime === 'string' && row.mime ? row.mime : 'application/octet-stream',
+        size: typeof row.size === 'number' ? row.size : null,
+    }));
+}
+/** Streams blob bytes one row at a time so a large workspace never lands in memory at once. */
+function forEachBlob(db, schema, visit) {
+    const rows = schema === 'v2'
+        ? db.prepare('SELECT key, mime, data FROM blobs WHERE deleted_at IS NULL').all()
+        : db.prepare('SELECT key, data FROM blobs').all();
+    for (const row of rows) {
+        visit({
+            key: String(row.key),
+            mime: typeof row.mime === 'string' && row.mime ? row.mime : 'application/octet-stream',
+            data: toBytes(row.data, `blob ${String(row.key)}`),
+        });
+    }
+}
+/** Every doc of one workspace, rendered. Docs that fail are NAMED, never dropped. */
+function readWorkspace(db, options = {}) {
+    const schema = detectSchema(db);
+    const workspaceId = readWorkspaceId(db, schema) ?? '';
+    const root = loadDoc(db, schema, schema === 'v2' ? workspaceId : null).doc;
+    const index = readDocIndex(root);
+    const docs = [];
+    const skipped = [];
+    let trashed = 0;
+    for (const entry of index) {
+        if (entry.trash) {
+            trashed++;
+            if (!options.includeTrash)
+                continue;
+        }
+        try {
+            const loaded = loadDoc(db, schema, entry.id);
+            const ctx = {
+                workspaceId,
+                doc: loaded.doc,
+                buildBlobUrl: (key) => options.blobUrl?.(key) ?? `affine-blob:${key}`,
+                buildDocUrl: (docId) => options.docUrl?.(docId) ?? `affine-doc:${docId}`,
+            };
+            const parsed = (0, parser_1.parsePageDoc)(ctx);
+            docs.push({
+                id: entry.id,
+                // The index title is what AFFiNE shows in its sidebar; the parsed one comes
+                // from the page block. They can differ mid-edit — prefer the page block and
+                // fall back, rather than rendering a doc with no name at all.
+                title: parsed.title || entry.title,
+                markdown: parsed.md,
+                updatesReplayed: loaded.updatesReplayed,
+                hadSnapshot: loaded.hadSnapshot,
+            });
+        }
+        catch (error) {
+            skipped.push({
+                id: entry.id,
+                reason: error instanceof Error ? error.message : String(error),
+            });
+        }
+    }
+    return { workspaceId, schema, docs, skipped, blobs: readBlobIndex(db, schema), trashed };
+}

package/dist/stage.d.ts

@@ -0,0 +1,27 @@
+/**
+ * Taking a safe copy of a LIVE AFFiNE database before reading it.
+ *
+ * 🚨 The content is in the -wal, not in storage.db. Measured on a real 0.27.4
+ * install 2026-09-06: `storage.db` 4 KB, `storage.db-wal` 1.8 MB. A reader that
+ * copies storage.db alone opens an EMPTY workspace and has no way to notice —
+ * it does not error, it just returns nothing.
+ *
+ * ⛔ Nothing here ever writes into AFFiNE's directory. Opening the live file
+ * read-write could checkpoint or corrupt a database another process owns.
+ */
+/** The journal files that must travel with storage.db. Order matters: -wal first. */
+export declare const SIDECAR_SUFFIXES: readonly ["-wal", "-shm"];
+export interface StagedDatabase {
+    /** Path of the copy — this is what to open. */
+    path: string;
+    /** Which sidecars were actually present and copied. */
+    sidecars: string[];
+    /** Removes the copy. Safe to call twice. */
+    dispose(): void;
+}
+/**
+ * Copy `dbPath` and its journal sidecars into `stagingDir`.
+ *
+ * `stagingDir` must be a directory this process owns — it is REMOVED by dispose().
+ */
+export declare function stageDatabase(dbPath: string, stagingDir: string): StagedDatabase;

package/dist/stage.js

@@ -0,0 +1,54 @@
+"use strict";
+/**
+ * Taking a safe copy of a LIVE AFFiNE database before reading it.
+ *
+ * 🚨 The content is in the -wal, not in storage.db. Measured on a real 0.27.4
+ * install 2026-09-06: `storage.db` 4 KB, `storage.db-wal` 1.8 MB. A reader that
+ * copies storage.db alone opens an EMPTY workspace and has no way to notice —
+ * it does not error, it just returns nothing.
+ *
+ * ⛔ Nothing here ever writes into AFFiNE's directory. Opening the live file
+ * read-write could checkpoint or corrupt a database another process owns.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.SIDECAR_SUFFIXES = void 0;
+exports.stageDatabase = stageDatabase;
+const node_fs_1 = require("node:fs");
+const node_path_1 = require("node:path");
+/** The journal files that must travel with storage.db. Order matters: -wal first. */
+exports.SIDECAR_SUFFIXES = ['-wal', '-shm'];
+/**
+ * Copy `dbPath` and its journal sidecars into `stagingDir`.
+ *
+ * `stagingDir` must be a directory this process owns — it is REMOVED by dispose().
+ */
+function stageDatabase(dbPath, stagingDir) {
+    if (!(0, node_fs_1.existsSync)(dbPath))
+        throw new Error(`AFFiNE database not found: ${dbPath}`);
+    // ⛔ A floor under the delete. dispose() removes this directory recursively, so
+    // it must be one we created and own. Handed an existing folder with anything in
+    // it, this would be a public API that erases whatever it was pointed at.
+    if ((0, node_fs_1.existsSync)(stagingDir) && (0, node_fs_1.readdirSync)(stagingDir).length > 0)
+        throw new Error(`Staging directory is not empty, refusing to use it: ${stagingDir}`);
+    (0, node_fs_1.mkdirSync)(stagingDir, { recursive: true });
+    const target = (0, node_path_1.join)(stagingDir, 'storage.db');
+    (0, node_fs_1.copyFileSync)(dbPath, target);
+    const sidecars = [];
+    for (const suffix of exports.SIDECAR_SUFFIXES) {
+        if ((0, node_fs_1.existsSync)(dbPath + suffix)) {
+            (0, node_fs_1.copyFileSync)(dbPath + suffix, target + suffix);
+            sidecars.push(suffix);
+        }
+    }
+    let disposed = false;
+    return {
+        path: target,
+        sidecars,
+        dispose() {
+            if (disposed)
+                return;
+            disposed = true;
+            (0, node_fs_1.rmSync)(stagingDir, { recursive: true, force: true });
+        },
+    };
+}

package/dist/types.d.ts

@@ -0,0 +1,72 @@
+/**
+ * Public types for @mnemosyne_os/affine-reader.
+ *
+ * The package brings NO SQLite driver of its own. Electron 31 ships Node 20, which
+ * has no `node:sqlite`, while a plain CLI on Node >= 22.5 does — so the driver is
+ * injected and the package stays honest in both. `openNodeSqlite()` is provided for
+ * the CLI case; the Electron host passes its own better-sqlite3 adapter.
+ */
+export interface SqliteStatement {
+    all(...params: unknown[]): Record<string, unknown>[];
+    get(...params: unknown[]): Record<string, unknown> | undefined;
+}
+export interface SqliteDatabase {
+    prepare(sql: string): SqliteStatement;
+    close(): void;
+}
+/** MUST open the file READ-ONLY. Two writers on one CRDT corrupt it. */
+export type SqliteOpener = (path: string) => SqliteDatabase;
+/** Which on-disk layout a workspace uses. `v1` is the pre-nbstore shape. */
+export type AffineSchema = 'v1' | 'v2';
+export interface WorkspaceRef {
+    /** Workspace id as AFFiNE names the directory. */
+    id: string;
+    /**
+     * `local` for a workspace that never left the machine; otherwise the escaped
+     * server name — a synced workspace keeps the SAME schema, only the folder differs.
+     */
+    peer: string;
+    kind: 'workspace' | 'userspace';
+    /** Absolute path to `storage.db`. */
+    dbPath: string;
+    layout: AffineSchema;
+}
+export interface AffineDoc {
+    id: string;
+    /** Title as the ROOT doc records it. May be empty — that is a real AFFiNE state. */
+    title: string;
+    markdown: string;
+    /** How many `updates` rows were replayed on top of the snapshot. */
+    updatesReplayed: number;
+    /** False when the doc has no snapshot row yet — normal for a freshly created doc. */
+    hadSnapshot: boolean;
+}
+export interface SkippedDoc {
+    id: string;
+    /** Why this doc is not in `docs`. Never dropped silently. */
+    reason: string;
+}
+export interface BlobRef {
+    key: string;
+    mime: string;
+    /** Size AFFiNE recorded. `null` when the column held something unreadable. */
+    size: number | null;
+}
+export interface WorkspaceContent {
+    workspaceId: string;
+    schema: AffineSchema;
+    docs: AffineDoc[];
+    /** Docs found in the index but not rendered, each with its reason. */
+    skipped: SkippedDoc[];
+    blobs: BlobRef[];
+    /** Docs in the trash are counted, never rendered. */
+    trashed: number;
+}
+export interface ReadOptions {
+    /** Include docs AFFiNE has moved to the trash. Default false. */
+    includeTrash?: boolean;
+    /** How an image block's blob key becomes a link. Return null for "cannot point at it". */
+    blobUrl?: (key: string) => string | null;
+    /** How a link to another AFFiNE doc is rendered. */
+    docUrl?: (docId: string) => string;
+}

package/dist/types.js

@@ -0,0 +1,10 @@
+"use strict";
+/**
+ * Public types for @mnemosyne_os/affine-reader.
+ *
+ * The package brings NO SQLite driver of its own. Electron 31 ships Node 20, which
+ * has no `node:sqlite`, while a plain CLI on Node >= 22.5 does — so the driver is
+ * injected and the package stays honest in both. `openNodeSqlite()` is provided for
+ * the CLI case; the Electron host passes its own better-sqlite3 adapter.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });

package/dist/vendor/affine/blocksuite-types.d.ts

@@ -0,0 +1,25 @@
+/**
+ * Local stand-ins for the two types the vendored parser imported from
+ * `@blocksuite/affine/model`.
+ *
+ * Both were `import type` — erased at compilation, never present at runtime —
+ * so replacing them changes NO behaviour. They are declared here only so this
+ * package does not need `@blocksuite/affine` (MPL-2.0) on its resolution path
+ * just to typecheck. See ../../..//NOTICE.md.
+ *
+ * Shapes derive from how `parser.ts` actually reads them (a database block's
+ * `prop:columns` and `prop:cells`), not from BlockSuite's full model.
+ */
+/** One column of an `affine:database` block. */
+export interface ColumnDataType {
+    id: string;
+    /** 'title' | 'select' | 'multi-select' | … — free-form, the parser switches on it. */
+    type: string;
+    name: string;
+    data: Record<string, unknown>;
+}
+/** One cell of an `affine:database` block. */
+export interface CellDataType {
+    columnId?: string;
+    value?: unknown;
+}

package/dist/vendor/affine/blocksuite-types.js

@@ -0,0 +1,14 @@
+"use strict";
+/**
+ * Local stand-ins for the two types the vendored parser imported from
+ * `@blocksuite/affine/model`.
+ *
+ * Both were `import type` — erased at compilation, never present at runtime —
+ * so replacing them changes NO behaviour. They are declared here only so this
+ * package does not need `@blocksuite/affine` (MPL-2.0) on its resolution path
+ * just to typecheck. See ../../..//NOTICE.md.
+ *
+ * Shapes derive from how `parser.ts` actually reads them (a database block's
+ * `prop:columns` and `prop:cells`), not from BlockSuite's full model.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });

package/dist/vendor/affine/delta-to-md/delta-converters.d.ts

@@ -0,0 +1,39 @@
+import { Node } from './utils/node';
+export interface InlineReference {
+    type: 'LinkedPage';
+    pageId: string;
+    title?: string;
+    params?: {
+        mode: 'doc' | 'edgeless';
+    };
+}
+export interface ConverterOptions {
+    convertInlineReferenceLink?: (reference: InlineReference) => {
+        title: string;
+        link: string;
+    };
+}
+export declare function getConverters(opts?: ConverterOptions): {
+    embed: {
+        image: (src: any) => void;
+        thematic_break: () => void;
+    };
+    inline: {
+        italic: () => string[];
+        bold: () => string[];
+        link: (url: any) => string[];
+        reference: (reference: InlineReference) => string[];
+        strike: () => string[];
+        code: () => string[];
+    };
+    block: {
+        header: ({ header }: {
+            header: any;
+        }) => void;
+        blockquote: () => void;
+        list: {
+            group: () => Node;
+            line: (attrs: any, group: any) => void;
+        };
+    };
+};

package/dist/vendor/affine/delta-to-md/delta-converters.js

@@ -0,0 +1,81 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.getConverters = getConverters;
+// oxlint-disable
+// @ts-nocheck
+const node_1 = require("./utils/node");
+const url_1 = require("./utils/url");
+const defaultConvertInlineReferenceLink = (reference) => {
+    return {
+        title: reference.title || '',
+        link: [reference.type, reference.pageId, reference.params?.mode]
+            .filter(Boolean)
+            .join(':'),
+    };
+};
+function getConverters(opts = {}) {
+    const { convertInlineReferenceLink = defaultConvertInlineReferenceLink } = opts;
+    return {
+        embed: {
+            image: function (src) {
+                this.append('![](' + (0, url_1.encodeLink)(src) + ')');
+            },
+            // Not a default Quill feature, converts custom divider embed blot added when
+            // creating quill editor instance.
+            // See https://quilljs.com/guides/cloning-medium-with-parchment/#dividers
+            thematic_break: function () {
+                this.open = '\n---\n' + this.open;
+            },
+        },
+        inline: {
+            italic: function () {
+                return ['_', '_'];
+            },
+            bold: function () {
+                return ['**', '**'];
+            },
+            link: function (url) {
+                return ['[', '](' + url + ')'];
+            },
+            reference: function (reference) {
+                const { title, link } = convertInlineReferenceLink(reference);
+                return ['[', `${title}](${link})`];
+            },
+            strike: function () {
+                return ['~~', '~~'];
+            },
+            code: function () {
+                return ['`', '`'];
+            },
+        },
+        block: {
+            header: function ({ header }) {
+                this.open = '#'.repeat(header) + ' ' + this.open;
+            },
+            blockquote: function () {
+                this.open = '> ' + this.open;
+            },
+            list: {
+                group: function () {
+                    return new node_1.Node(['', '\n']);
+                },
+                line: function (attrs, group) {
+                    if (attrs.list === 'bullet') {
+                        this.open = '- ' + this.open;
+                    }
+                    else if (attrs.list === 'checked') {
+                        this.open = '- [x] ' + this.open;
+                    }
+                    else if (attrs.list === 'unchecked') {
+                        this.open = '- [ ] ' + this.open;
+                    }
+                    else if (attrs.list === 'ordered') {
+                        group.count = group.count || 0;
+                        var count = ++group.count;
+                        this.open = count + '. ' + this.open;
+                    }
+                },
+            },
+        },
+    };
+}

package/dist/vendor/affine/delta-to-md/delta-to-md.d.ts

@@ -0,0 +1 @@
+export declare const deltaToMd: (delta: any, converters: any) => string;