@moxn/kb-migrate 0.4.21 → 0.4.23

package/dist/sources/notion.d.ts

@@ -9,7 +9,8 @@
  */
 import type { ExtractedDocument } from '../types.js';
 import { MigrationSource, type SourceConfig } from './base.js';
-import type { NotionPage } from './notion-api.js';
+import type { NotionPage, NotionDatabase } from './notion-api.js';
+import { type PagePathMap } from './notion-blocks.js';
 import { type ParsedDatabaseSchema } from './notion-databases.js';
 export interface NotionSourceConfig extends SourceConfig {
     token: string;
@@ -18,6 +19,16 @@ export interface NotionSourceConfig extends SourceConfig {
     /** Date filter for source pages */
     dateFilter?: import('../date-filter.js').DateFilter;
 }
+export interface PageTreeNode {
+    page: NotionPage;
+    title: string;
+    slug: string;
+    children: PageTreeNode[];
+    kbPath: string;
+    isDatabaseEntry: boolean;
+    isDatabaseNode?: boolean;
+    parentDatabaseId?: string;
+}
 /** Info yielded to the migration runner for database import (post-document pass). */
 export interface NotionDatabaseImportInfo {
     notionDatabaseId: string;
@@ -62,14 +73,36 @@ export declare class NotionSource extends MigrationSource<NotionSourceConfig> {
     extract(): AsyncGenerator<ExtractedDocument, void, unknown>;
     /** Clean up temp files after migration completes. */
     cleanup(): Promise<void>;
-    private buildPageTree;
-    private getParentPageId;
+    private buildPageTreeInternal;
     private processDatabases;
     private getDatabaseParentId;
     private extractPage;
     private extractDatabaseEntry;
     private downloadSectionMedia;
 }
+/** Options for building the page tree. */
+export interface BuildPageTreeOptions {
+    rootPageId?: string;
+    maxDepth?: number;
+    databases?: NotionDatabase[];
+}
+/** Result of building the page tree. */
+export interface PageTreeResult {
+    tree: PageTreeNode[];
+    allPages: PageTreeNode[];
+    pagePathMap: PagePathMap;
+    databaseEntryPageIds: Set<string>;
+}
+/**
+ * Build a hierarchical page tree from a flat list of Notion pages.
+ *
+ * Computes kbPath for each page based on parent-child relationships,
+ * deduplicates sibling slugs, and identifies database entries.
+ *
+ * Exported so the import-orchestrator can build hierarchical paths
+ * without instantiating a full NotionSource.
+ */
+export declare function buildPageTree(pages: NotionPage[], options?: BuildPageTreeOptions): PageTreeResult;
 /**
  * Slugify a page title for use as a KB path segment.
  * Lowercase, spaces→hyphens, strip special chars, ltree-compatible.

package/dist/sources/notion.js

@@ -84,12 +84,12 @@ export class NotionSource extends MigrationSource {
         console.log('Discovering databases...');
         const allNotionDatabases = await this.client.searchDatabases();
         console.log(`  Found ${allNotionDatabases.length} databases.`);
-        // 4. Build page tree
-        this.buildPageTree(allNotionPages);
+        // 4. Build page tree (pass databases so they become tree nodes for nesting entries)
+        this.buildPageTreeInternal(allNotionPages, allNotionDatabases);
         // 5. Process databases — identify entries and build schemas
         await this.processDatabases(allNotionDatabases);
-        // 6. Count and validate
-        this._documentCount = this.allPages.length;
+        // 6. Count and validate (exclude synthetic database container nodes)
+        this._documentCount = this.allPages.filter((n) => !n.isDatabaseNode).length;
         // Add database-only entries (pages that are in databases but not in page tree)
         for (const dbInfo of this.databases) {
             for (const entry of dbInfo.entries) {
@@ -141,6 +141,9 @@ export class NotionSource extends MigrationSource {
             // unless they also appear in the page tree (child_page)
             if (node.isDatabaseEntry)
                 continue;
+            // Skip synthetic database container nodes — they're for nesting, not documents
+            if (node.isDatabaseNode)
+                continue;
             const doc = await this.extractPage(node);
             if (doc)
                 yield doc;
@@ -169,123 +172,18 @@ export class NotionSource extends MigrationSource {
         await this.mediaDownloader.cleanup();
     }
     // ============================================
-    // Tree building
+    // Tree building (delegates to standalone function)
     // ============================================
-    buildPageTree(pages) {
-        // Build lookup maps
-        const pageById = new Map();
-        const childrenByParent = new Map();
-        for (const page of pages) {
-            const nid = normalizeId(page.id);
-            pageById.set(nid, page);
-            const parentId = this.getParentPageId(page);
-            if (parentId) {
-                const npid = normalizeId(parentId);
-                if (!childrenByParent.has(npid)) {
-                    childrenByParent.set(npid, []);
-                }
-                childrenByParent.get(npid).push(page);
-            }
-        }
-        // Identify database entries
-        for (const page of pages) {
-            if (page.parent.type === 'database_id' && page.parent.database_id) {
-                this.databaseEntryPageIds.add(normalizeId(page.id));
-            }
-        }
-        // Find root pages
-        const rootPages = [];
-        if (this.config.rootPageId) {
-            // Subtree mode: start from specified page
-            const rootId = normalizeId(this.config.rootPageId);
-            const rootPage = pageById.get(rootId);
-            if (rootPage) {
-                rootPages.push(rootPage);
-            }
-            else {
-                // Root page itself wasn't in search results — search for its children
-                const children = childrenByParent.get(rootId) ?? [];
-                rootPages.push(...children);
-            }
-        }
-        else {
-            // Full workspace mode: pages with workspace parent or no parent in our set
-            for (const page of pages) {
-                if (page.parent.type === 'workspace') {
-                    rootPages.push(page);
-                }
-                else {
-                    const parentId = this.getParentPageId(page);
-                    if (parentId && !pageById.has(normalizeId(parentId))) {
-                        // Parent is not in our page set — treat as root
-                        if (page.parent.type !== 'database_id') {
-                            rootPages.push(page);
-                        }
-                    }
-                }
-            }
-        }
-        // Build tree recursively
-        const buildNode = (page, parentPath, depth, siblingSlugCounts) => {
-            if (this.config.maxDepth !== undefined && depth > this.config.maxDepth) {
-                return null;
-            }
-            const title = getPageTitle(page);
-            let slug = slugify(title);
-            // Deduplicate sibling slugs
-            const existing = siblingSlugCounts.get(slug) ?? 0;
-            siblingSlugCounts.set(slug, existing + 1);
-            if (existing > 0) {
-                slug = `${slug}-${existing + 1}`;
-            }
-            const kbPath = parentPath ? `${parentPath}/${slug}` : slug;
-            const nid = normalizeId(page.id);
-            const isDatabaseEntry = this.databaseEntryPageIds.has(nid);
-            const node = {
-                page,
-                title,
-                slug,
-                kbPath,
-                isDatabaseEntry,
-                parentDatabaseId: page.parent.type === 'database_id' ? page.parent.database_id : undefined,
-                children: [],
-            };
-            // Register in path map
-            this.pagePathMap.set(nid, kbPath);
-            // Process children
-            const childPages = childrenByParent.get(nid) ?? [];
-            const childSlugCounts = new Map();
-            for (const childPage of childPages) {
-                const childNode = buildNode(childPage, kbPath, depth + 1, childSlugCounts);
-                if (childNode) {
-                    node.children.push(childNode);
-                }
-            }
-            return node;
-        };
-        // Build roots
-        const rootSlugCounts = new Map();
-        for (const rootPage of rootPages) {
-            const node = buildNode(rootPage, '', 0, rootSlugCounts);
-            if (node) {
-                this.pageTree.push(node);
-            }
-        }
-        // Flatten tree to depth-first list
-        const flatten = (nodes) => {
-            for (const node of nodes) {
-                this.allPages.push(node);
-                flatten(node.children);
-            }
-        };
-        flatten(this.pageTree);
-    }
-    getParentPageId(page) {
-        if (page.parent.type === 'page_id')
-            return page.parent.page_id ?? null;
-        if (page.parent.type === 'block_id')
-            return page.parent.block_id ?? null;
-        return null;
+    buildPageTreeInternal(pages, databases) {
+        const result = buildPageTree(pages, {
+            rootPageId: this.config.rootPageId,
+            maxDepth: this.config.maxDepth,
+            databases,
+        });
+        this.pageTree = result.tree;
+        this.allPages = result.allPages;
+        this.pagePathMap = result.pagePathMap;
+        this.databaseEntryPageIds = result.databaseEntryPageIds;
     }
     // ============================================
     // Database processing
@@ -421,6 +319,183 @@ export class NotionSource extends MigrationSource {
         return result;
     }
 }
+/**
+ * Build a hierarchical page tree from a flat list of Notion pages.
+ *
+ * Computes kbPath for each page based on parent-child relationships,
+ * deduplicates sibling slugs, and identifies database entries.
+ *
+ * Exported so the import-orchestrator can build hierarchical paths
+ * without instantiating a full NotionSource.
+ */
+export function buildPageTree(pages, options) {
+    const pagePathMap = new Map();
+    const databaseEntryPageIds = new Set();
+    const tree = [];
+    const allPages = [];
+    // Build lookup maps
+    const pageById = new Map();
+    const childrenByParent = new Map();
+    const syntheticDatabaseIds = new Set();
+    for (const page of pages) {
+        const nid = normalizeId(page.id);
+        pageById.set(nid, page);
+        const parentId = getParentPageId(page);
+        if (parentId) {
+            const npid = normalizeId(parentId);
+            if (!childrenByParent.has(npid)) {
+                childrenByParent.set(npid, []);
+            }
+            childrenByParent.get(npid).push(page);
+        }
+    }
+    // Inject synthetic page nodes for databases so entries nest under them
+    if (options?.databases?.length) {
+        for (const db of options.databases) {
+            const dbNid = normalizeId(db.id);
+            // Skip if a real page already has this ID
+            if (pageById.has(dbNid))
+                continue;
+            // Create synthetic NotionPage wrapper
+            const syntheticPage = {
+                id: db.id,
+                object: 'page',
+                parent: db.parent,
+                created_time: db.created_time,
+                last_edited_time: db.last_edited_time,
+                created_by: { id: '', object: 'user' },
+                last_edited_by: { id: '', object: 'user' },
+                url: '',
+                properties: {
+                    title: { id: 'title', type: 'title', title: db.title },
+                },
+            };
+            pageById.set(dbNid, syntheticPage);
+            // Add to childrenByParent
+            const parentId = getParentPageId(syntheticPage);
+            if (parentId) {
+                const npid = normalizeId(parentId);
+                if (!childrenByParent.has(npid))
+                    childrenByParent.set(npid, []);
+                childrenByParent.get(npid).push(syntheticPage);
+            }
+            syntheticDatabaseIds.add(dbNid);
+        }
+    }
+    // Identify database entries
+    for (const page of pages) {
+        if (page.parent.type === 'database_id' && page.parent.database_id) {
+            databaseEntryPageIds.add(normalizeId(page.id));
+        }
+    }
+    // Find root pages
+    const rootPages = [];
+    if (options?.rootPageId) {
+        const rootId = normalizeId(options.rootPageId);
+        const rootPage = pageById.get(rootId);
+        if (rootPage) {
+            rootPages.push(rootPage);
+        }
+        else {
+            const children = childrenByParent.get(rootId) ?? [];
+            rootPages.push(...children);
+        }
+    }
+    else {
+        for (const page of pages) {
+            if (page.parent.type === 'workspace') {
+                rootPages.push(page);
+            }
+            else {
+                const parentId = getParentPageId(page);
+                if (parentId && !pageById.has(normalizeId(parentId))) {
+                    if (page.parent.type !== 'database_id') {
+                        rootPages.push(page);
+                    }
+                }
+            }
+        }
+        // Also add synthetic database nodes whose parent is workspace or not in the tree
+        for (const dbNid of syntheticDatabaseIds) {
+            const synth = pageById.get(dbNid);
+            if (synth.parent.type === 'workspace') {
+                rootPages.push(synth);
+            }
+            else {
+                const parentId = getParentPageId(synth);
+                if (parentId && !pageById.has(normalizeId(parentId))) {
+                    rootPages.push(synth);
+                }
+            }
+        }
+    }
+    // Build tree recursively
+    const buildNode = (page, parentPath, depth, siblingSlugCounts) => {
+        if (options?.maxDepth !== undefined && depth > options.maxDepth) {
+            return null;
+        }
+        const title = getPageTitle(page);
+        let slug = slugify(title);
+        // Deduplicate sibling slugs
+        const existing = siblingSlugCounts.get(slug) ?? 0;
+        siblingSlugCounts.set(slug, existing + 1);
+        if (existing > 0) {
+            slug = `${slug}-${existing + 1}`;
+        }
+        const kbPath = parentPath ? `${parentPath}/${slug}` : slug;
+        const nid = normalizeId(page.id);
+        const isDatabaseEntry = databaseEntryPageIds.has(nid);
+        const node = {
+            page,
+            title,
+            slug,
+            kbPath,
+            isDatabaseEntry,
+            isDatabaseNode: syntheticDatabaseIds.has(nid) || undefined,
+            parentDatabaseId: page.parent.type === 'database_id' ? page.parent.database_id : undefined,
+            children: [],
+        };
+        // Register in path map
+        pagePathMap.set(nid, kbPath);
+        // Process children
+        const childPages = childrenByParent.get(nid) ?? [];
+        const childSlugCounts = new Map();
+        for (const childPage of childPages) {
+            const childNode = buildNode(childPage, kbPath, depth + 1, childSlugCounts);
+            if (childNode) {
+                node.children.push(childNode);
+            }
+        }
+        return node;
+    };
+    // Build roots
+    const rootSlugCounts = new Map();
+    for (const rootPage of rootPages) {
+        const node = buildNode(rootPage, '', 0, rootSlugCounts);
+        if (node) {
+            tree.push(node);
+        }
+    }
+    // Flatten tree to depth-first list
+    const flatten = (nodes) => {
+        for (const node of nodes) {
+            allPages.push(node);
+            flatten(node.children);
+        }
+    };
+    flatten(tree);
+    return { tree, allPages, pagePathMap, databaseEntryPageIds };
+}
+/** Extract the parent page ID from a Notion page's parent field. */
+function getParentPageId(page) {
+    if (page.parent.type === 'page_id')
+        return page.parent.page_id ?? null;
+    if (page.parent.type === 'database_id')
+        return page.parent.database_id ?? null;
+    if (page.parent.type === 'block_id')
+        return page.parent.block_id ?? null;
+    return null;
+}
 // ============================================
 // Path helpers
 // ============================================

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@moxn/kb-migrate",
-  "version": "0.4.21",
+  "version": "0.4.23",
   "description": "Migration tool for importing documents into Moxn Knowledge Base from local files, Notion, Google Docs, and more",
   "type": "module",
   "main": "dist/index.js",