@moxn/kb-migrate 0.4.41 → 0.6.0

package/dist/api-error.d.ts

@@ -12,3 +12,10 @@ export declare function formatApiError(status: number, body: {
     details?: unknown;
     [key: string]: unknown;
 } | null | undefined): string;
+/**
+ * Parse a response body that should be JSON. When it isn't — typically the
+ * Vercel Auth wall's HTML page on a protected deployment — say what arrived
+ * (status, content-type) instead of surfacing a raw `JSON.parse` error like
+ * `Unexpected token '<', "<!DOCTYPE "... is not valid JSON`.
+ */
+export declare function readJson<T = any>(response: Response): Promise<T>;

package/dist/api-error.js

@@ -17,3 +17,24 @@ export function formatApiError(status, body) {
             : '';
     return detail ? `API error ${status}: ${detail}` : `API error: ${status}`;
 }
+/**
+ * Parse a response body that should be JSON. When it isn't — typically the
+ * Vercel Auth wall's HTML page on a protected deployment — say what arrived
+ * (status, content-type) instead of surfacing a raw `JSON.parse` error like
+ * `Unexpected token '<', "<!DOCTYPE "... is not valid JSON`.
+ */
+export async function readJson(response) {
+    try {
+        return (await response.json());
+    }
+    catch (cause) {
+        if (!(cause instanceof SyntaxError))
+            throw cause;
+        const contentType = response.headers?.get?.('content-type') ?? 'no content-type';
+        const where = response.url ? ` from ${response.url}` : '';
+        const hint = /html/i.test(contentType)
+            ? ' — an HTML page usually means the Vercel protection wall: set MOXN_VERCEL_BYPASS, with MOXN_BASE_URL on this origin'
+            : '';
+        throw new Error(`Expected JSON${where} but got a non-JSON response (HTTP ${response.status}, ${contentType})${hint}`);
+    }
+}

package/dist/blocks-to-grammar.d.ts

@@ -0,0 +1,51 @@
+/**
+ * ContentBlock[] → grammar markdown serializer (Phase B of the write-surface
+ * tightening: the REST block-input path is retired; every import lands via
+ * `import_markdown`, the same grammar the export emits and the app's codec
+ * round-trips).
+ *
+ * The extractors (Notion / OneNote) keep producing ContentBlock sections —
+ * that stays a useful internal representation. This module is the WRITE
+ * BOUNDARY: it assumes media blocks have already been uploaded/re-hosted to
+ * storage keys (see `MoxnClient.processSections`, which now also re-hosts
+ * `url`/`base64` media) and reduces the sections to the `## H2` + `:::`
+ * directive grammar.
+ *
+ * Directive attr serialization REPLICATES the app codec
+ * (`src/lib/kb/grammar/attrs.ts` — kb-migrate is a separate package and
+ * cannot import it, same posture as grammar-media.ts): string → double-quoted
+ * with `&`→`&`, `"`→`"`; number → bare; string[] → `[a,b]` with
+ * `\` escaping for `\`, `,`, `]`.
+ */
+import type { ContentBlock } from './types.js';
+type AttrValue = string | number | string[] | null | undefined;
+/** Mirror of the app codec's `serializeAttrs` (src/lib/kb/grammar/attrs.ts). */
+export declare function serializeAttrs(obj: Record<string, AttrValue>): string;
+/**
+ * One ContentBlock → grammar chunk (markdown text or a `:::` directive line).
+ * Returns null for a block that cannot be expressed (e.g. base64 media that
+ * was not uploaded — the caller is expected to have re-hosted those first).
+ */
+export declare function blockToGrammar(block: ContentBlock): string | null;
+export interface SectionsToGrammarResult {
+    markdown: string;
+    /** Media blocks that could not be expressed (no storage key / URL). */
+    dropped: number;
+}
+/**
+ * Sections → grammar markdown BODY (no front-matter — the caller prepends
+ * it). Each section becomes an `## <name>` chunk; blocks join with blank
+ * lines, matching the H2-split the server applies on the way back in.
+ */
+export declare function sectionsToGrammarMarkdown(sections: Array<{
+    name: string;
+    content: ContentBlock[];
+}>): SectionsToGrammarResult;
+/**
+ * Minimal YAML front-matter for an import: `name` (+ optional description).
+ * `import_markdown` HONORS the front-matter (replace_document semantics on
+ * update — the import source is the source of truth for metadata).
+ * Values are JSON-quoted, which is valid YAML for scalar strings.
+ */
+export declare function buildImportFrontMatter(name: string, description?: string | null): string;
+export {};

package/dist/blocks-to-grammar.js

@@ -0,0 +1,133 @@
+/**
+ * ContentBlock[] → grammar markdown serializer (Phase B of the write-surface
+ * tightening: the REST block-input path is retired; every import lands via
+ * `import_markdown`, the same grammar the export emits and the app's codec
+ * round-trips).
+ *
+ * The extractors (Notion / OneNote) keep producing ContentBlock sections —
+ * that stays a useful internal representation. This module is the WRITE
+ * BOUNDARY: it assumes media blocks have already been uploaded/re-hosted to
+ * storage keys (see `MoxnClient.processSections`, which now also re-hosts
+ * `url`/`base64` media) and reduces the sections to the `## H2` + `:::`
+ * directive grammar.
+ *
+ * Directive attr serialization REPLICATES the app codec
+ * (`src/lib/kb/grammar/attrs.ts` — kb-migrate is a separate package and
+ * cannot import it, same posture as grammar-media.ts): string → double-quoted
+ * with `&`→`&`, `"`→`"`; number → bare; string[] → `[a,b]` with
+ * `\` escaping for `\`, `,`, `]`.
+ */
+function escapeArrayElement(v) {
+    return v.replace(/\\/g, '\\\\').replace(/,/g, '\\,').replace(/]/g, '\\]');
+}
+/** Mirror of the app codec's `serializeAttrs` (src/lib/kb/grammar/attrs.ts). */
+export function serializeAttrs(obj) {
+    const parts = [];
+    for (const [key, value] of Object.entries(obj)) {
+        if (value === null || value === undefined)
+            continue;
+        if (Array.isArray(value)) {
+            parts.push(`${key}=[${value.map(escapeArrayElement).join(',')}]`);
+        }
+        else if (typeof value === 'number') {
+            parts.push(`${key}=${value}`);
+        }
+        else {
+            const escaped = value.replace(/&/g, '&').replace(/"/g, '"');
+            parts.push(`${key}="${escaped}"`);
+        }
+    }
+    return parts.join(' ');
+}
+/** The `ref` a media block serializes: storage key first, else its URL. */
+function blockRef(block) {
+    if (block.type === 'storage' && block.key)
+        return block.key;
+    if (block.type === 'url' && block.url)
+        return block.url;
+    return null;
+}
+/**
+ * One ContentBlock → grammar chunk (markdown text or a `:::` directive line).
+ * Returns null for a block that cannot be expressed (e.g. base64 media that
+ * was not uploaded — the caller is expected to have re-hosted those first).
+ */
+export function blockToGrammar(block) {
+    switch (block.blockType) {
+        case 'text':
+            return block.text;
+        case 'image': {
+            const ref = blockRef(block);
+            if (!ref)
+                return null;
+            return `:::image{${serializeAttrs({
+                ref,
+                alt: block.alt,
+                mime: block.mediaType,
+            })}}`;
+        }
+        case 'document':
+        case 'file': {
+            const ref = blockRef(block);
+            if (!ref)
+                return null;
+            return `:::file{${serializeAttrs({
+                ref,
+                filename: 'filename' in block ? block.filename : undefined,
+                mime: block.mediaType,
+            })}}`;
+        }
+        case 'csv': {
+            const ref = blockRef(block);
+            if (!ref)
+                return null;
+            return `:::csv{${serializeAttrs({
+                ref,
+                filename: block.filename,
+                mime: 'text/csv',
+                headers: block.headers,
+                rowCount: block.rowCount,
+            })}}`;
+        }
+        case 'database_embed':
+            return `:::db{${serializeAttrs({ ref: block.databaseId })}}`;
+        default:
+            return null;
+    }
+}
+/**
+ * Sections → grammar markdown BODY (no front-matter — the caller prepends
+ * it). Each section becomes an `## <name>` chunk; blocks join with blank
+ * lines, matching the H2-split the server applies on the way back in.
+ */
+export function sectionsToGrammarMarkdown(sections) {
+    let dropped = 0;
+    const chunks = [];
+    for (const section of sections) {
+        const parts = [`## ${section.name}`];
+        for (const block of section.content) {
+            const chunk = blockToGrammar(block);
+            if (chunk === null) {
+                dropped++;
+                continue;
+            }
+            if (chunk.trim().length > 0)
+                parts.push(chunk.trim());
+        }
+        chunks.push(parts.join('\n\n'));
+    }
+    return { markdown: chunks.join('\n\n'), dropped };
+}
+/**
+ * Minimal YAML front-matter for an import: `name` (+ optional description).
+ * `import_markdown` HONORS the front-matter (replace_document semantics on
+ * update — the import source is the source of truth for metadata).
+ * Values are JSON-quoted, which is valid YAML for scalar strings.
+ */
+export function buildImportFrontMatter(name, description) {
+    const lines = [`name: ${JSON.stringify(name)}`];
+    if (description !== undefined && description !== null && description !== '') {
+        lines.push(`description: ${JSON.stringify(description)}`);
+    }
+    return `---\n${lines.join('\n')}\n---\n\n`;
+}

package/dist/client-http.test.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/client-http.test.js

@@ -0,0 +1,89 @@
+/**
+ * kb-migrate's HTTP client against a Vercel-protected deployment (staging).
+ *
+ * `export-local` against https://staging.moxn.dev failed with
+ * `Unexpected token '<', "<!DOCTYPE "... is not valid JSON`: kb-migrate never
+ * sent the protection-bypass header, and the auth wall's HTML page reached a
+ * bare `response.json()`. The client now attaches the SAME
+ * `vercelBypassHeaders` kb-cli uses (@moxn/auth — env-gated, scoped to
+ * MOXN_BASE_URL's origin, so signed storage URLs never see the secret), and a
+ * non-JSON body is reported as what it is.
+ */
+import { describe, it, expect, vi, afterEach, beforeEach } from 'vitest';
+import * as fs from 'fs';
+import * as os from 'os';
+import * as path from 'path';
+import { MoxnClient } from './client.js';
+const STAGING = 'https://staging.moxn.dev';
+const HEADER = 'x-vercel-protection-bypass';
+function jsonResponse(body) {
+    return new Response(JSON.stringify(body), {
+        status: 200,
+        headers: { 'content-type': 'application/json' },
+    });
+}
+function headersOf(init) {
+    return new Headers(init?.headers).has(HEADER)
+        ? { [HEADER]: new Headers(init?.headers).get(HEADER) }
+        : {};
+}
+describe('MoxnClient — Vercel protection bypass', () => {
+    const saved = { ...process.env };
+    let fetchSpy;
+    beforeEach(() => {
+        fetchSpy = vi.spyOn(globalThis, 'fetch');
+    });
+    afterEach(() => {
+        vi.restoreAllMocks();
+        process.env = { ...saved };
+    });
+    function client() {
+        return new MoxnClient({
+            apiUrl: STAGING,
+            apiKey: 'k',
+            basePath: '/',
+            onConflict: 'skip',
+            dryRun: false,
+        });
+    }
+    it('attaches the bypass header to requests for the MOXN_BASE_URL origin only', async () => {
+        process.env.MOXN_VERCEL_BYPASS = 'bypass-secret';
+        process.env.MOXN_BASE_URL = STAGING;
+        fetchSpy.mockImplementation(async (url) => String(url).startsWith(STAGING)
+            ? jsonResponse({ documents: [], pagination: { hasMore: false } })
+            : new Response('blob', { status: 200 }));
+        await client().listDocuments('/qa');
+        const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'kb-bypass-'));
+        await client().downloadFile('https://storage.example/signed/f.png', path.join(dir, 'f.png'));
+        const [apiCall, storageCall] = fetchSpy.mock.calls;
+        expect(String(apiCall[0])).toContain(`${STAGING}/api/v1/kb/documents`);
+        expect(headersOf(apiCall[1])).toEqual({ [HEADER]: 'bypass-secret' });
+        // The API key still rides alongside.
+        expect(new Headers(apiCall[1].headers).get('x-api-key')).toBe('k');
+        // A different origin (signed storage URL) never receives the secret.
+        expect(headersOf(storageCall[1])).toEqual({});
+        fs.rmSync(dir, { recursive: true, force: true });
+    });
+    it('sends no bypass header when MOXN_VERCEL_BYPASS is unset', async () => {
+        delete process.env.MOXN_VERCEL_BYPASS;
+        process.env.MOXN_BASE_URL = STAGING;
+        fetchSpy.mockResolvedValue(jsonResponse({ documents: [], pagination: { hasMore: false } }));
+        await client().listDocuments('/qa');
+        expect(headersOf(fetchSpy.mock.calls[0][1])).toEqual({});
+    });
+    it('reports a non-JSON (auth-wall HTML) body by status + content-type, not a raw JSON.parse error', async () => {
+        fetchSpy.mockResolvedValue(new Response('<!DOCTYPE html><html>Vercel Authentication</html>', {
+            status: 200,
+            headers: { 'content-type': 'text/html; charset=utf-8' },
+        }));
+        const err = await client()
+            .listDocuments('/qa')
+            .then(() => null, (e) => e);
+        expect(err).toBeInstanceOf(Error);
+        expect(err.message).not.toMatch(/Unexpected token/);
+        expect(err.message).toMatch(/non-JSON/i);
+        expect(err.message).toContain('200');
+        expect(err.message).toContain('text/html');
+        expect(err.message).toMatch(/MOXN_VERCEL_BYPASS/);
+    });
+});

package/dist/client.d.ts

@@ -32,17 +32,86 @@ export interface ImportMarkdownResult {
      * lenient fallback recovered the name. Carries the js-yaml error message.
      */
     frontMatterError?: string;
+    /** Section anchor ids in POSITION ORDER after the upsert (for cross-refs). */
+    sectionIds?: string[];
+    /** The branch the upsert landed on (for branch-scoped follow-ups like tags). */
+    branchId?: string;
+    /**
+     * The permissions a CREATED document got. Absent on update/skip (an import
+     * never re-permissions an existing document) and from servers that predate
+     * the importer permission fields.
+     */
+    permissions?: {
+        defaultPermission: 'edit' | 'read' | 'none';
+        aiAccess: 'edit' | 'read' | 'none';
+    };
+}
+/**
+ * The warning (if any) for an import whose `--default-permission` /
+ * `--ai-access` did not take effect: an existing document is never
+ * re-permissioned by a re-import, and a server that predates the fields
+ * creates documents with its defaults without echoing `permissions`.
+ */
+export declare function importPermissionWarning(requested: {
+    defaultPermission?: string;
+    aiAccess?: string;
+}, result: Pick<ImportMarkdownResult, 'outcome' | 'permissions'>): string | undefined;
+/**
+ * One item of a `read` tool response, narrowed to the fields the export uses.
+ * `kind` is absent on an error item and on older md reads.
+ */
+export interface ReadDocumentItem {
+    type: string;
+    documentId: string;
+    error?: string;
+    kind?: string;
+    /** report: the HTML; slides: the deck-marker HTML. */
+    content?: Array<{
+        type: string;
+        text?: string;
+    }>;
+    /** kind='file' only. */
+    file?: {
+        filename: string;
+        download: {
+            url: string;
+        };
+    };
+    /** kind='slides' only. */
+    deck?: {
+        themeCss?: string | null;
+    };
 }
 export declare class MoxnClient {
     private apiUrl;
     private apiKey;
     private defaultPermission?;
     private aiAccess?;
+    /** Every request goes through {@link moxnFetch} (bypass-header aware). */
+    private fetch;
     constructor(options: MigrationOptions | ExportOptions);
     /**
      * Migrate a single document
      */
+    /**
+     * Migrate one extracted document via `import_markdown` (Phase B of the
+     * write-surface tightening — the REST block-input path is retired).
+     *
+     * Flow: upload/re-host every media block (local files, base64 payloads,
+     * remote URLs) → serialize the sections to grammar markdown → prepend a
+     * name/description front-matter → UPSERT via `import_markdown`
+     * (create-or-replace_document; `onConflict: 'skip'` skips an existing
+     * path). Idempotent: a byte-identical re-import is reported as a no-op
+     * update by the server.
+     */
     migrateDocument(doc: ExtractedDocument, basePath: string, onConflict: 'skip' | 'update', dryRun: boolean): Promise<MigrationResult>;
+    /**
+     * {@link importPermissionWarning} for this client's requested permissions,
+     * logged once and returned as a spreadable `{ warning }`.
+     */
+    permissionWarningFor(result: Pick<ImportMarkdownResult, 'outcome' | 'permissions'>, sourcePath: string): {
+        warning?: string;
+    };
     /**
      * List all documents, optionally filtered by path prefix.
      * Handles pagination automatically.
@@ -64,6 +133,13 @@ export declare class MoxnClient {
     getDocumentMarkdown(documentId: string, opts?: {
         forEdit?: boolean;
     }): Promise<DocumentMarkdownResponse | null>;
+    /**
+     * Read one document through the `read` tool (`/api/v1/kb/tools`) — the same
+     * serialization agents and the CLI see. The export uses it for the kinds the
+     * grammar export has no body for (report / file / slides). Returns the single
+     * read item: a document shape carrying `kind`, or an `{ error }` item.
+     */
+    readDocument(documentId: string): Promise<ReadDocumentItem>;
     /**
      * Build a `storageKey → signedDownloadUrl` map for a document's media, by
      * reusing the `get_document_content` export action (the blocks read, which
@@ -102,8 +178,6 @@ export declare class MoxnClient {
         key: string;
     }>;
     private getUploadUrl;
-    private createDocument;
-    private updateDocument;
     /**
      * Create a KB database.
      */
@@ -263,5 +337,4 @@ export declare class MoxnClient {
         id: string;
         name: string;
     } | null>;
-    private isConflictError;
 }