@moxn/kb-migrate 0.4.41 → 0.6.0

package/dist/client.js

@@ -2,12 +2,40 @@
  * API client for Moxn KB
  */
 import * as fs from 'fs/promises';
-import { formatApiError } from './api-error.js';
+import { formatApiError, readJson } from './api-error.js';
+import { moxnFetch } from './http.js';
+import { sectionsToGrammarMarkdown, buildImportFrontMatter, } from './blocks-to-grammar.js';
+/**
+ * The warning (if any) for an import whose `--default-permission` /
+ * `--ai-access` did not take effect: an existing document is never
+ * re-permissioned by a re-import, and a server that predates the fields
+ * creates documents with its defaults without echoing `permissions`.
+ */
+export function importPermissionWarning(requested, result) {
+    const asked = [
+        requested.defaultPermission &&
+            `--default-permission ${requested.defaultPermission}`,
+        requested.aiAccess && `--ai-access ${requested.aiAccess}`,
+    ].filter(Boolean);
+    if (asked.length === 0 || result.outcome === 'skipped')
+        return undefined;
+    if (result.outcome === 'updated') {
+        return `${asked.join(', ')} not applied: this updated an existing document, whose permissions are left as they were (change them in the app)`;
+    }
+    if (!result.permissions) {
+        return `${asked.join(', ')}: the server did not apply them (it predates importer permissions) — the document was created with the server defaults`;
+    }
+    return undefined;
+}
 export class MoxnClient {
     apiUrl;
     apiKey;
     defaultPermission;
     aiAccess;
+    /** Every request goes through {@link moxnFetch} (bypass-header aware). */
+    fetch(url, init = {}) {
+        return moxnFetch(url, init);
+    }
     constructor(options) {
         this.apiUrl = options.apiUrl.replace(/\/$/, '');
         this.apiKey = options.apiKey;
@@ -21,6 +49,17 @@ export class MoxnClient {
     /**
      * Migrate a single document
      */
+    /**
+     * Migrate one extracted document via `import_markdown` (Phase B of the
+     * write-surface tightening — the REST block-input path is retired).
+     *
+     * Flow: upload/re-host every media block (local files, base64 payloads,
+     * remote URLs) → serialize the sections to grammar markdown → prepend a
+     * name/description front-matter → UPSERT via `import_markdown`
+     * (create-or-replace_document; `onConflict: 'skip'` skips an existing
+     * path). Idempotent: a byte-identical re-import is reported as a no-op
+     * update by the server.
+     */
     async migrateDocument(doc, basePath, onConflict, dryRun) {
         const startTime = Date.now();
         const documentPath = this.buildPath(basePath, doc.relativePath);
@@ -34,91 +73,37 @@ export class MoxnClient {
             };
         }
         try {
-            // Process content blocks (convert file paths to base64)
+            // Upload/re-host media (local paths, base64, remote URLs) → storage keys.
             const processedSections = await this.processSections(doc.sections);
-            // Try to create the document
-            const createResult = await this.createDocument({
+            const { markdown: body, dropped } = sectionsToGrammarMarkdown(processedSections);
+            if (dropped > 0) {
+                console.error(`  ! ${dropped} media block(s) had no uploadable payload and were dropped: ${doc.sourcePath}`);
+            }
+            const markdown = buildImportFrontMatter(doc.name, doc.description) + body;
+            const result = await this.importMarkdown({
+                markdown,
                 path: documentPath,
-                name: doc.name,
-                description: doc.description,
-                defaultPermission: this.defaultPermission,
-                aiAccess: this.aiAccess,
-                sections: processedSections,
+                onConflict,
             });
             return {
                 sourcePath: doc.sourcePath,
-                documentPath,
-                status: 'created',
-                documentId: createResult.id,
-                branchId: createResult.branchId,
-                sectionsCount: createResult.sections.length,
-                sectionIds: createResult.sections.map((s) => s.id),
+                documentPath: result.path,
+                status: result.outcome === 'created'
+                    ? 'created'
+                    : result.outcome === 'updated'
+                        ? 'updated'
+                        : 'skipped',
+                documentId: result.id,
+                branchId: result.branchId,
+                sectionsCount: result.sectionIds?.length,
+                sectionIds: result.sectionIds,
                 references: doc.references,
                 sourcePageId: doc.metadata?.notionPageId,
+                ...this.permissionWarningFor(result, doc.sourcePath),
                 duration: Date.now() - startTime,
             };
         }
         catch (error) {
-            // Check for conflict (409)
-            if (this.isConflictError(error)) {
-                if (onConflict === 'skip') {
-                    return {
-                        sourcePath: doc.sourcePath,
-                        documentPath,
-                        status: 'skipped',
-                        documentId: error.documentId,
-                        branchId: error.branchId,
-                        duration: Date.now() - startTime,
-                    };
-                }
-                // Update existing document
-                try {
-                    const processedSections = await this.processSections(doc.sections);
-                    const updateResult = await this.updateDocument(error.documentId, {
-                        name: doc.name,
-                        description: doc.description,
-                        sections: processedSections,
-                    });
-                    return {
-                        sourcePath: doc.sourcePath,
-                        documentPath,
-                        status: 'updated',
-                        documentId: updateResult.id,
-                        branchId: updateResult.branchId,
-                        sectionsCount: updateResult.sections.length,
-                        sectionIds: updateResult.sections.map((s) => s.id),
-                        references: doc.references,
-                        sourcePageId: doc.metadata?.notionPageId,
-                        duration: Date.now() - startTime,
-                    };
-                }
-                catch (updateError) {
-                    // Re-importing BYTE-IDENTICAL content makes the server reject the
-                    // empty commit ("No changes to commit"). For an importer that's a
-                    // NO-OP, not a failure — report a successful (no-op) update, matching
-                    // the grammar path's idempotent re-import. (Audit finding B.)
-                    const msg = updateError instanceof Error ? updateError.message : '';
-                    if (msg.includes('No changes to commit')) {
-                        return {
-                            sourcePath: doc.sourcePath,
-                            documentPath,
-                            status: 'updated',
-                            documentId: error.documentId,
-                            branchId: error.branchId,
-                            sourcePageId: doc.metadata?.notionPageId,
-                            duration: Date.now() - startTime,
-                        };
-                    }
-                    return {
-                        sourcePath: doc.sourcePath,
-                        documentPath,
-                        status: 'failed',
-                        documentId: error.documentId,
-                        error: updateError instanceof Error ? updateError.message : 'Update failed',
-                        duration: Date.now() - startTime,
-                    };
-                }
-            }
             return {
                 sourcePath: doc.sourcePath,
                 documentPath,
@@ -128,6 +113,17 @@ export class MoxnClient {
             };
         }
     }
+    /**
+     * {@link importPermissionWarning} for this client's requested permissions,
+     * logged once and returned as a spreadable `{ warning }`.
+     */
+    permissionWarningFor(result, sourcePath) {
+        const warning = importPermissionWarning({ defaultPermission: this.defaultPermission, aiAccess: this.aiAccess }, result);
+        if (!warning)
+            return {};
+        console.warn(`  ⚠ ${warning}: ${sourcePath}`);
+        return { warning };
+    }
     // ──────────────────────────────────────────────
     // Export methods
     // ──────────────────────────────────────────────
@@ -155,14 +151,14 @@ export class MoxnClient {
                 params.set('modifiedAfter', dateFilter.modifiedAfter);
             if (dateFilter?.modifiedBefore)
                 params.set('modifiedBefore', dateFilter.modifiedBefore);
-            const response = await fetch(`${this.apiUrl}/api/v1/kb/documents?${params}`, {
+            const response = await this.fetch(`${this.apiUrl}/api/v1/kb/documents?${params}`, {
                 headers: { 'x-api-key': this.apiKey },
             });
             if (!response.ok) {
                 const error = await response.text();
                 throw new Error(`Failed to list documents: ${response.status} ${error}`);
             }
-            const data = await response.json();
+            const data = await readJson(response);
             allDocs.push(...data.documents);
             if (!data.pagination.hasMore)
                 break;
@@ -174,14 +170,14 @@ export class MoxnClient {
      * Get full document detail with sections and content.
      */
     async getDocument(documentId) {
-        const response = await fetch(`${this.apiUrl}/api/v1/kb/documents/${documentId}`, {
+        const response = await this.fetch(`${this.apiUrl}/api/v1/kb/documents/${documentId}`, {
             headers: { 'x-api-key': this.apiKey },
         });
         if (!response.ok) {
             const error = await response.text();
             throw new Error(`Failed to get document ${documentId}: ${response.status} ${error}`);
         }
-        return response.json();
+        return readJson(response);
     }
     /**
      * Get a document's GRAMMAR markdown (front-matter + body + `:::image/:::csv/
@@ -193,7 +189,7 @@ export class MoxnClient {
      * address sections stably.
      */
     async getDocumentMarkdown(documentId, opts) {
-        const response = await fetch(`${this.apiUrl}/api/v1/kb/export`, {
+        const response = await this.fetch(`${this.apiUrl}/api/v1/kb/export`, {
             method: 'POST',
             headers: {
                 'Content-Type': 'application/json',
@@ -209,9 +205,38 @@ export class MoxnClient {
             const body = await response.text();
             throw new Error(`Failed to get document markdown ${documentId}: ${response.status} ${body}`);
         }
-        const data = await response.json();
+        const data = await readJson(response);
         return (data.result?.document ?? null);
     }
+    /**
+     * Read one document through the `read` tool (`/api/v1/kb/tools`) — the same
+     * serialization agents and the CLI see. The export uses it for the kinds the
+     * grammar export has no body for (report / file / slides). Returns the single
+     * read item: a document shape carrying `kind`, or an `{ error }` item.
+     */
+    async readDocument(documentId) {
+        const response = await this.fetch(`${this.apiUrl}/api/v1/kb/tools`, {
+            method: 'POST',
+            headers: {
+                'Content-Type': 'application/json',
+                'x-api-key': this.apiKey,
+            },
+            body: JSON.stringify({
+                tool: 'read',
+                params: { items: [{ type: 'document', documentId }] },
+            }),
+        });
+        if (!response.ok) {
+            const body = await response.json().catch(() => ({}));
+            throw new Error(formatApiError(response.status, body));
+        }
+        const data = await readJson(response);
+        const item = data.result?.items?.[0];
+        if (!item) {
+            throw new Error(`read returned no item for document ${documentId}`);
+        }
+        return item;
+    }
     /**
      * Build a `storageKey → signedDownloadUrl` map for a document's media, by
      * reusing the `get_document_content` export action (the blocks read, which
@@ -221,7 +246,7 @@ export class MoxnClient {
      * Returns an empty map when the document has no resolvable media.
      */
     async getDocumentMediaUrls(documentId) {
-        const response = await fetch(`${this.apiUrl}/api/v1/kb/export`, {
+        const response = await this.fetch(`${this.apiUrl}/api/v1/kb/export`, {
             method: 'POST',
             headers: {
                 'Content-Type': 'application/json',
@@ -233,7 +258,7 @@ export class MoxnClient {
             const body = await response.text();
             throw new Error(`Failed to get document content ${documentId}: ${response.status} ${body}`);
         }
-        const data = await response.json();
+        const data = await readJson(response);
         const doc = data.result?.document;
         const map = new Map();
         if (!doc)
@@ -258,7 +283,7 @@ export class MoxnClient {
      *   onConflict 'update' → replace_document (full-set PUT)
      */
     async importMarkdown(input) {
-        const response = await fetch(`${this.apiUrl}/api/v1/kb/import`, {
+        const response = await this.fetch(`${this.apiUrl}/api/v1/kb/import`, {
             method: 'POST',
             headers: {
                 'Content-Type': 'application/json',
@@ -269,20 +294,25 @@ export class MoxnClient {
                 markdown: input.markdown,
                 path: input.path,
                 onConflict: input.onConflict,
+                // Applied by the server to a document this import CREATES.
+                ...(this.defaultPermission
+                    ? { defaultPermission: this.defaultPermission }
+                    : {}),
+                ...(this.aiAccess ? { aiAccess: this.aiAccess } : {}),
             }),
         });
         if (!response.ok) {
             const body = await response.json().catch(() => ({}));
             throw new Error(formatApiError(response.status, body));
         }
-        const data = await response.json();
+        const data = await readJson(response);
         return data.result;
     }
     /**
      * Download a file from a URL to a local path.
      */
     async downloadFile(url, destPath) {
-        const response = await fetch(url);
+        const response = await this.fetch(url);
         if (!response.ok) {
             throw new Error(`Failed to download ${url}: ${response.status}`);
         }
@@ -358,6 +388,45 @@ export class MoxnClient {
                     filename: block.filename,
                 };
             }
+            // Base64 payloads (extractors that inline-downloaded media) → upload.
+            if ((block.blockType === 'image' ||
+                block.blockType === 'document' ||
+                block.blockType === 'csv' ||
+                block.blockType === 'file') &&
+                block.type === 'base64' &&
+                block.base64) {
+                const data = Buffer.from(block.base64, 'base64');
+                const filename = ('filename' in block ? block.filename : undefined) || 'media';
+                const { key } = await this.uploadFile(data, block.mediaType || 'application/octet-stream', filename);
+                return { ...block, type: 'storage', key, base64: undefined };
+            }
+            // Remote URLs (e.g. expiring Notion signed URLs) → download + re-host,
+            // matching the durability the server-side blocksToTipTap re-hosting
+            // used to provide. A failed download keeps the URL ref (the grammar
+            // carries external URLs) rather than failing the document.
+            if ((block.blockType === 'image' ||
+                block.blockType === 'document' ||
+                block.blockType === 'csv' ||
+                block.blockType === 'file') &&
+                block.type === 'url' &&
+                block.url) {
+                try {
+                    const response = await this.fetch(block.url);
+                    if (!response.ok)
+                        throw new Error(`HTTP ${response.status}`);
+                    const data = Buffer.from(await response.arrayBuffer());
+                    const filename = ('filename' in block ? block.filename : undefined) ||
+                        new URL(block.url).pathname.split('/').pop() ||
+                        'media';
+                    const { key } = await this.uploadFile(data, block.mediaType || 'application/octet-stream', filename);
+                    return { ...block, type: 'storage', key, url: undefined };
+                }
+                catch (err) {
+                    const msg = err instanceof Error ? err.message : String(err);
+                    console.error(`  ! media re-host failed, keeping external URL ref: ${block.url}: ${msg}`);
+                    return block;
+                }
+            }
             return block;
         }));
     }
@@ -368,7 +437,7 @@ export class MoxnClient {
         // 1. Get presigned upload URL
         const { key, uploadUrl } = await this.getUploadUrl(mimeType, filename);
         // 2. PUT file to presigned URL
-        const response = await fetch(uploadUrl, {
+        const response = await this.fetch(uploadUrl, {
             method: 'PUT',
             headers: { 'Content-Type': mimeType },
             body: new Uint8Array(data),
@@ -379,7 +448,7 @@ export class MoxnClient {
         return { key };
     }
     async getUploadUrl(type, filename) {
-        const response = await fetch(`${this.apiUrl}/api/v1/kb/upload`, {
+        const response = await this.fetch(`${this.apiUrl}/api/v1/kb/upload`, {
             method: 'POST',
             headers: {
                 'Content-Type': 'application/json',
@@ -391,45 +460,9 @@ export class MoxnClient {
             const body = await response.text();
             throw new Error(`Upload URL request failed: ${response.status} ${body}`);
         }
-        const data = await response.json();
+        const data = await readJson(response);
         return { key: data.key, uploadUrl: data.uploadUrl };
     }
-    async createDocument(request) {
-        const response = await fetch(`${this.apiUrl}/api/v1/kb/documents`, {
-            method: 'POST',
-            headers: {
-                'Content-Type': 'application/json',
-                'x-api-key': this.apiKey,
-            },
-            body: JSON.stringify(request),
-        });
-        if (!response.ok) {
-            const body = await response.json().catch(() => ({}));
-            if (response.status === 409 && body.documentId) {
-                const error = new Error(body.error || 'Document already exists');
-                error.documentId = body.documentId;
-                error.branchId = body.branchId;
-                throw error;
-            }
-            throw new Error(formatApiError(response.status, body));
-        }
-        return response.json();
-    }
-    async updateDocument(documentId, request) {
-        const response = await fetch(`${this.apiUrl}/api/v1/kb/documents/${documentId}`, {
-            method: 'PUT',
-            headers: {
-                'Content-Type': 'application/json',
-                'x-api-key': this.apiKey,
-            },
-            body: JSON.stringify(request),
-        });
-        if (!response.ok) {
-            const body = await response.json().catch(() => ({}));
-            throw new Error(formatApiError(response.status, body));
-        }
-        return response.json();
-    }
     // ──────────────────────────────────────────────
     // Database & tag methods (for Notion import)
     // ──────────────────────────────────────────────
@@ -437,7 +470,7 @@ export class MoxnClient {
      * Create a KB database.
      */
     async createDatabase(input) {
-        const response = await fetch(`${this.apiUrl}/api/v1/kb/databases`, {
+        const response = await this.fetch(`${this.apiUrl}/api/v1/kb/databases`, {
             method: 'POST',
             headers: {
                 'Content-Type': 'application/json',
@@ -449,13 +482,13 @@ export class MoxnClient {
             const body = await response.json().catch(() => ({}));
             throw new Error(body.error || `Failed to create database: ${response.status}`);
         }
-        return response.json();
+        return readJson(response);
     }
     /**
      * Add a column to a KB database.
      */
     async addDatabaseColumn(databaseId, input) {
-        const response = await fetch(`${this.apiUrl}/api/v1/kb/databases/${databaseId}/columns`, {
+        const response = await this.fetch(`${this.apiUrl}/api/v1/kb/databases/${databaseId}/columns`, {
             method: 'POST',
             headers: {
                 'Content-Type': 'application/json',
@@ -467,13 +500,13 @@ export class MoxnClient {
             const body = await response.json().catch(() => ({}));
             throw new Error(body.error || `Failed to add column: ${response.status}`);
         }
-        return response.json();
+        return readJson(response);
     }
     /**
      * Add a document to a KB database.
      */
     async addDocumentToDatabase(databaseId, documentId) {
-        const response = await fetch(`${this.apiUrl}/api/v1/kb/databases/${databaseId}/documents`, {
+        const response = await this.fetch(`${this.apiUrl}/api/v1/kb/databases/${databaseId}/documents`, {
             method: 'POST',
             headers: {
                 'Content-Type': 'application/json',
@@ -493,7 +526,7 @@ export class MoxnClient {
      * Create a tag (with automatic ancestor creation).
      */
     async createTag(input) {
-        const response = await fetch(`${this.apiUrl}/api/v1/kb/tags`, {
+        const response = await this.fetch(`${this.apiUrl}/api/v1/kb/tags`, {
             method: 'POST',
             headers: {
                 'Content-Type': 'application/json',
@@ -513,13 +546,13 @@ export class MoxnClient {
             }
             throw new Error(body.error || `Failed to create tag: ${response.status}`);
         }
-        return response.json();
+        return readJson(response);
     }
     /**
      * Assign a tag to a document.
      */
     async assignTag(documentId, tagId, branchId) {
-        const response = await fetch(`${this.apiUrl}/api/v1/kb/documents/${documentId}/tags`, {
+        const response = await this.fetch(`${this.apiUrl}/api/v1/kb/documents/${documentId}/tags`, {
             method: 'POST',
             headers: {
                 'Content-Type': 'application/json',
@@ -537,7 +570,7 @@ export class MoxnClient {
      * Returns { created, skipped, errors }.
      */
     async createReferences(documentId, references) {
-        const response = await fetch(`${this.apiUrl}/api/v1/kb/documents/${documentId}/references`, {
+        const response = await this.fetch(`${this.apiUrl}/api/v1/kb/documents/${documentId}/references`, {
             method: 'POST',
             headers: {
                 'Content-Type': 'application/json',
@@ -549,51 +582,51 @@ export class MoxnClient {
             const body = await response.json().catch(() => ({}));
             throw new Error(body.error || `Failed to create references: ${response.status}`);
         }
-        return response.json();
+        return readJson(response);
     }
     /**
      * List all KB databases for the tenant.
      */
     async listDatabases() {
-        const response = await fetch(`${this.apiUrl}/api/v1/kb/databases`, {
+        const response = await this.fetch(`${this.apiUrl}/api/v1/kb/databases`, {
             headers: { 'x-api-key': this.apiKey },
         });
         if (!response.ok) {
             const body = await response.json().catch(() => ({}));
             throw new Error(body.error || `Failed to list databases: ${response.status}`);
         }
-        const data = await response.json();
+        const data = await readJson(response);
         return data.databases;
     }
     /**
      * Get fully resolved database with columns, options, and document property values.
      */
     async resolveDatabase(databaseId) {
-        const response = await fetch(`${this.apiUrl}/api/v1/kb/databases/${databaseId}/resolve`, {
+        const response = await this.fetch(`${this.apiUrl}/api/v1/kb/databases/${databaseId}/resolve`, {
             headers: { 'x-api-key': this.apiKey },
         });
         if (!response.ok) {
             const body = await response.json().catch(() => ({}));
             throw new Error(body.error || `Failed to resolve database: ${response.status}`);
         }
-        return response.json();
+        return readJson(response);
     }
     /**
      * Get Notion database mapping for a KB database.
      */
     async getNotionDatabaseMapping(kbDatabaseId) {
-        const response = await fetch(`${this.apiUrl}/api/v1/kb/notion-mappings/databases/by-kb-database/${kbDatabaseId}`, { headers: { 'x-api-key': this.apiKey } });
+        const response = await this.fetch(`${this.apiUrl}/api/v1/kb/notion-mappings/databases/by-kb-database/${kbDatabaseId}`, { headers: { 'x-api-key': this.apiKey } });
         if (!response.ok) {
             const body = await response.json().catch(() => ({}));
             throw new Error(body.error || `Failed to get database mapping: ${response.status}`);
         }
-        return response.json();
+        return readJson(response);
     }
     /**
      * Create or upsert a Notion database mapping.
      */
     async createNotionDatabaseMapping(input) {
-        const response = await fetch(`${this.apiUrl}/api/v1/kb/notion-mappings/databases`, {
+        const response = await this.fetch(`${this.apiUrl}/api/v1/kb/notion-mappings/databases`, {
             method: 'POST',
             headers: {
                 'Content-Type': 'application/json',
@@ -605,13 +638,13 @@ export class MoxnClient {
             const body = await response.json().catch(() => ({}));
             throw new Error(body.error || `Failed to create database mapping: ${response.status}`);
         }
-        return response.json();
+        return readJson(response);
     }
     /**
      * Set a scalar property value on a document in a database.
      */
     async setPropertyValue(databaseId, documentId, columnName, value) {
-        const response = await fetch(`${this.apiUrl}/api/v1/kb/databases/${databaseId}/properties`, {
+        const response = await this.fetch(`${this.apiUrl}/api/v1/kb/databases/${databaseId}/properties`, {
             method: 'POST',
             headers: {
                 'Content-Type': 'application/json',
@@ -631,7 +664,7 @@ export class MoxnClient {
      * Create or upsert a Notion page → KB document mapping.
      */
     async createNotionPageMapping(input) {
-        const response = await fetch(`${this.apiUrl}/api/v1/kb/notion-mappings`, {
+        const response = await this.fetch(`${this.apiUrl}/api/v1/kb/notion-mappings`, {
             method: 'POST',
             headers: {
                 'Content-Type': 'application/json',
@@ -649,14 +682,14 @@ export class MoxnClient {
      * Returns a Map of notionDatabaseId → kbDatabaseId.
      */
     async getAllDatabaseMappings() {
-        const response = await fetch(`${this.apiUrl}/api/v1/kb/notion-mappings/databases`, {
+        const response = await this.fetch(`${this.apiUrl}/api/v1/kb/notion-mappings/databases`, {
             headers: { 'x-api-key': this.apiKey },
         });
         if (!response.ok) {
             const body = await response.json().catch(() => ({}));
             throw new Error(body.error || `Failed to get database mappings: ${response.status}`);
         }
-        const data = await response.json();
+        const data = await readJson(response);
         const map = new Map();
         for (const mapping of data.mappings) {
             map.set(mapping.notionDatabaseId, mapping.kbDatabaseId);
@@ -670,11 +703,4 @@ export class MoxnClient {
         const databases = await this.listDatabases();
         return databases.find((db) => db.id === databaseId) ?? null;
     }
-    isConflictError(error) {
-        return (error instanceof Error &&
-            'documentId' in error &&
-            'branchId' in error &&
-            typeof error.documentId === 'string' &&
-            typeof error.branchId === 'string');
-    }
 }