gatsby-source-notion-churnotion 1.2.1 → 1.2.3

package/README.md

@@ -62,6 +62,15 @@ If you're considering Notion as your CMS for Gatsby, this plugin could be a grea
   - Better error handling
   - Improved type safety
 
+## Features
+
+- **Recursive Database Fetching**: Connect with a Notion database and fetch all entries, including nested entries.
+- **Type Safe**: All data types are converted to TypeScript types.
+- **Powerful TypeScript Implementation**: Uses efficient, battle-tested TypeScript code for processing Notion API requests in parallel.
+- **Customizable**: Allows for custom filtering and properties selection.
+
+> **Note**: The plugin currently uses a pure TypeScript implementation for parallel processing of Notion API requests. A Rust-based implementation was initially planned for high-performance use cases but has been put on hold due to cross-platform compilation issues.
+
 ## Install
 
 ```shell

package/dist/rust-bindings.d.ts

@@ -1,6 +1,6 @@
 /**
- * This file provides TypeScript bindings to the Rust implementation
- * of parallel Notion API processing.
+ * This file provides TypeScript bindings to the Notion API processing.
+ * Currently using the TypeScript implementation as the default.
  */
 import { Reporter } from "gatsby";
 /**
@@ -17,9 +17,8 @@ export interface RustNotionServiceOptions {
     enableCaching?: boolean;
 }
 /**
- * A wrapper class that provides access to the Rust implementation
- * of parallel processing for Notion API, with a fallback to the
- * TypeScript implementation if the Rust module is not available.
+ * A wrapper class that provides access to the TypeScript implementation
+ * for parallel processing of Notion API requests.
  */
 export declare class RustNotionService {
     private reporter;
@@ -27,14 +26,13 @@ export declare class RustNotionService {
     private parallelLimit;
     private enableCaching;
     private cache;
-    private rustInstance;
     constructor(options: RustNotionServiceOptions);
     /**
      * Get blocks for a single page, with recursive fetching of child blocks
      */
     getPageBlocks(pageId: string): Promise<NotionBlock[]>;
     /**
-     * Get blocks for multiple pages in parallel, using Rust implementation if available
+     * Get blocks for multiple pages in parallel
      */
     getMultiplePagesBlocks(pageIds: string[]): Promise<{
         [id: string]: NotionBlock[];

package/dist/rust-bindings.js

@@ -34,19 +34,9 @@ var __importStar = (this && this.__importStar) || (function () {
 })();
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.RustNotionService = void 0;
-// Use dynamic import with try-catch to handle module loading
-let notionParallel = null;
-try {
-    // When the compiled Rust library exists, load it
-    notionParallel = require("../rust/notion-parallel/index.node");
-}
-catch (error) {
-    console.warn("Rust bindings for notion-parallel not found. Using TypeScript fallback implementation.");
-}
 /**
- * A wrapper class that provides access to the Rust implementation
- * of parallel processing for Notion API, with a fallback to the
- * TypeScript implementation if the Rust module is not available.
+ * A wrapper class that provides access to the TypeScript implementation
+ * for parallel processing of Notion API requests.
  */
 class RustNotionService {
     reporter;
@@ -54,24 +44,13 @@ class RustNotionService {
     parallelLimit;
     enableCaching;
     cache;
-    rustInstance = null;
     constructor(options) {
         this.reporter = options.reporter;
         this.notionApiKey = options.notionApiKey;
         this.parallelLimit = options.parallelLimit || 5;
         this.enableCaching = options.enableCaching !== false;
         this.cache = new Map();
-        // Initialize Rust instance if available
-        if (notionParallel) {
-            try {
-                this.rustInstance = new notionParallel.NotionParallel(this.notionApiKey, this.parallelLimit);
-                this.reporter.info(`[RUST] Initialized Rust-based parallel processing with limit: ${this.parallelLimit}`);
-            }
-            catch (error) {
-                this.reporter.warn(`[RUST] Failed to initialize Rust implementation: ${error}`);
-                this.rustInstance = null;
-            }
-        }
+        this.reporter.info(`[NOTION] Initialized TypeScript-based parallel processing with limit: ${this.parallelLimit}`);
     }
     /**
      * Get blocks for a single page, with recursive fetching of child blocks
@@ -82,8 +61,6 @@ class RustNotionService {
             this.reporter.info(`[CACHE] Using cached page blocks for ${pageId}`);
             return this.cache.get(cacheKey);
         }
-        // Currently we don't have a direct Rust counterpart for a single page
-        // So we use the multiple pages method with a single ID
         const results = await this.getMultiplePagesBlocks([pageId]);
         const blocks = results[pageId] || [];
         if (this.enableCaching) {
@@ -92,24 +69,11 @@ class RustNotionService {
         return blocks;
     }
     /**
-     * Get blocks for multiple pages in parallel, using Rust implementation if available
+     * Get blocks for multiple pages in parallel
      */
     async getMultiplePagesBlocks(pageIds) {
         this.reporter.info(`[NOTION] Fetching blocks for ${pageIds.length} pages in parallel (limit: ${this.parallelLimit})`);
-        // If Rust implementation is available and initialized, use it
-        if (this.rustInstance) {
-            try {
-                this.reporter.info(`[RUST] Using Rust implementation for parallel processing`);
-                const results = await this.rustInstance.getMultiplePagesBlocks(pageIds);
-                return results;
-            }
-            catch (error) {
-                this.reporter.warn(`[RUST] Error using Rust implementation, falling back to TypeScript: ${error}`);
-                // Fall back to TypeScript implementation
-            }
-        }
-        // Fallback to TypeScript implementation
-        // Import the TypeScript implementation dynamically
+        // Import the TypeScript implementation
         const { NotionService } = await Promise.resolve().then(() => __importStar(require("./api/service/notionService")));
         // Create a NotionService instance with the same configuration
         const tsService = new NotionService({
@@ -125,10 +89,6 @@ class RustNotionService {
     setParallelLimit(limit) {
         this.reporter.info(`[NOTION] Updated parallel request limit to ${limit}`);
         this.parallelLimit = limit;
-        // Update the Rust instance if available
-        if (this.rustInstance) {
-            this.rustInstance.setParallelLimit(limit);
-        }
     }
 }
 exports.RustNotionService = RustNotionService;

package/dist/util/blocks/blockProcessor.d.ts

@@ -1,6 +1,7 @@
 import { BaseContentBlock } from "notion-types";
 import { Actions, GatsbyCache, Reporter } from "gatsby";
 import { CustomImageBlock } from "../../types";
+import { TocEntry } from "../tocHelper";
 export interface BlockProcessorContext {
     actions: Actions;
     getCache: (this: void, id: string) => GatsbyCache;
@@ -12,11 +13,7 @@ export interface ProcessBlockResult {
     thumbnail?: string | null;
     plainText?: string;
     updatedBlock?: BaseContentBlock;
-    tableOfContents?: {
-        type: string;
-        hash: string;
-        title: string;
-    }[];
+    tableOfContents?: TocEntry[];
 }
 export declare abstract class BlockProcessor {
     protected context: BlockProcessorContext;

package/dist/util/blocks/blockProcessorRegistry.js

@@ -5,6 +5,7 @@ const textBlockProcessor_1 = require("./textBlockProcessor");
 const imageBlockProcessor_1 = require("./imageBlockProcessor");
 const mediaBlockProcessor_1 = require("./mediaBlockProcessor");
 const structureBlockProcessor_1 = require("./structureBlockProcessor");
+const tableOfContentsBlockProcessor_1 = require("./tableOfContentsBlockProcessor");
 class BlockProcessorRegistry {
     processors = [];
     context;
@@ -17,6 +18,7 @@ class BlockProcessorRegistry {
         this.registerProcessor(new textBlockProcessor_1.TextBlockProcessor(this.context));
         this.registerProcessor(new imageBlockProcessor_1.ImageBlockProcessor(this.context));
         this.registerProcessor(new mediaBlockProcessor_1.MediaBlockProcessor(this.context));
+        this.registerProcessor(new tableOfContentsBlockProcessor_1.TableOfContentsBlockProcessor(this.context));
         this.registerProcessor(new structureBlockProcessor_1.StructureBlockProcessor(this.context));
     }
     registerProcessor(processor) {

package/dist/util/blocks/index.d.ts

@@ -4,3 +4,4 @@ export * from "./imageBlockProcessor";
 export * from "./mediaBlockProcessor";
 export * from "./structureBlockProcessor";
 export * from "./textBlockProcessor";
+export * from "./tableOfContentsBlockProcessor";

package/dist/util/blocks/index.js

@@ -20,3 +20,4 @@ __exportStar(require("./imageBlockProcessor"), exports);
 __exportStar(require("./mediaBlockProcessor"), exports);
 __exportStar(require("./structureBlockProcessor"), exports);
 __exportStar(require("./textBlockProcessor"), exports);
+__exportStar(require("./tableOfContentsBlockProcessor"), exports);

package/dist/util/blocks/structureBlockProcessor.js

@@ -11,7 +11,6 @@ class StructureBlockProcessor extends blockProcessor_1.BlockProcessor {
             "table_row",
             "divider",
             "breadcrumb",
-            "table_of_contents",
             "equation",
             "synced_block",
             "template",
@@ -44,9 +43,6 @@ class StructureBlockProcessor extends blockProcessor_1.BlockProcessor {
             case "breadcrumb":
                 reporter.info(`Processing breadcrumb block`);
                 break;
-            case "table_of_contents":
-                reporter.info(`Processing table_of_contents block`);
-                break;
             case "equation":
                 reporter.info(`Processing equation block: ${JSON.stringify(block.equation?.expression)}`);
                 break;

package/dist/util/blocks/tableOfContentsBlockProcessor.d.ts

@@ -0,0 +1,6 @@
+import { BaseContentBlock } from "notion-types";
+import { BlockProcessor, ProcessBlockResult } from "./blockProcessor";
+export declare class TableOfContentsBlockProcessor extends BlockProcessor {
+    canProcess(block: BaseContentBlock): boolean;
+    process(block: BaseContentBlock): Promise<ProcessBlockResult>;
+}

package/dist/util/blocks/tableOfContentsBlockProcessor.js

@@ -0,0 +1,23 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.TableOfContentsBlockProcessor = void 0;
+const blockProcessor_1 = require("./blockProcessor");
+class TableOfContentsBlockProcessor extends blockProcessor_1.BlockProcessor {
+    canProcess(block) {
+        return block.type === "table_of_contents";
+    }
+    async process(block) {
+        const { reporter } = this.context;
+        reporter.info(`Processing table_of_contents block with id: ${block.id}`);
+        // Table of contents blocks don't have any content by themselves
+        // They are used as placeholders, and the actual TOC data is generated separately
+        // and passed through the GraphQL schema
+        return {
+            plainText: "",
+            updatedBlock: block,
+            // We don't return tableOfContents from here because that's handled
+            // at a higher level in the processor.ts file
+        };
+    }
+}
+exports.TableOfContentsBlockProcessor = TableOfContentsBlockProcessor;

package/dist/util/processor.d.ts

@@ -1,7 +1,4 @@
 import { Actions, GatsbyCache, Reporter } from "gatsby";
 import { BaseContentBlock } from "notion-types";
-export declare const processor: (blocks: BaseContentBlock[], actions: Actions, getCache: (this: void, id: string) => GatsbyCache, createNodeId: (this: void, input: string) => string, reporter: Reporter, cache: GatsbyCache) => Promise<[string | null, {
-    type: string;
-    hash: string;
-    title: string;
-}[], BaseContentBlock[], string]>;
+import { TocEntry } from "./tocHelper";
+export declare const processor: (blocks: BaseContentBlock[], actions: Actions, getCache: (this: void, id: string) => GatsbyCache, createNodeId: (this: void, input: string) => string, reporter: Reporter, cache: GatsbyCache) => Promise<[string | null, TocEntry[], BaseContentBlock[], string]>;

package/dist/util/processor.js

@@ -3,6 +3,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.processor = void 0;
 const metadataProcessor_1 = require("./metadataProcessor");
 const blocks_1 = require("./blocks");
+const tocHelper_1 = require("./tocHelper");
 const processor = async (blocks, actions, getCache, createNodeId, reporter, cache) => {
     const { thumbnail, tableOfContents, updatedBlocks, rawText } = await processBlocksForContent(blocks, actions, getCache, createNodeId, reporter, cache);
     await (0, metadataProcessor_1.processMetadata)(blocks, actions, createNodeId, reporter, cache);
@@ -42,6 +43,8 @@ const processBlocksForContent = async (blocks, actions, getCache, createNodeId,
         }
         return result;
     }));
+    // 목차 최적화
+    const processedToc = (0, tocHelper_1.optimizeTocArray)(tableOfContents, reporter);
     // 업데이트된 블록 적용
     processResults.forEach((result, index) => {
         if (result.updatedBlock) {
@@ -51,5 +54,5 @@ const processBlocksForContent = async (blocks, actions, getCache, createNodeId,
             updatedBlocks[index] = blocks[index];
         }
     });
-    return { thumbnail, tableOfContents, updatedBlocks, rawText };
+    return { thumbnail, tableOfContents: processedToc, updatedBlocks, rawText };
 };

package/dist/util/tableOfContent.d.ts

@@ -1,6 +1,3 @@
 import { BaseContentBlock } from "notion-types";
-export declare const processTableOfContents: (block: BaseContentBlock, tableOfContents: {
-    type: string;
-    hash: string;
-    title: string;
-}[]) => Promise<void>;
+import { TocEntry } from "./tocHelper";
+export declare const processTableOfContents: (block: BaseContentBlock, tableOfContents: TocEntry[]) => Promise<void>;

package/dist/util/tableOfContent.js

@@ -2,21 +2,35 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.processTableOfContents = void 0;
 const crypto_1 = require("crypto");
+// Hash 생성 시 사용할 캐시
+const hashCache = new Map();
 const processTableOfContents = async (block, tableOfContents) => {
     if (["heading_1", "heading_2", "heading_3"].includes(block.type) &&
         block[block.type]?.rich_text?.length > 0) {
         const plainText = block[block.type]?.rich_text?.[0]?.plain_text || "";
-        const hash = `link-${plainText
-            .replace(/[^a-zA-Z0-9가-힣\s-_]/g, "")
-            .trim()
-            .replace(/\s+/g, "-")
-            .toLowerCase()}-${(0, crypto_1.randomUUID)().substring(0, 4)}`;
+        // 이미 처리된 플레인텍스트인지 확인
+        let hash;
+        if (hashCache.has(plainText)) {
+            hash = hashCache.get(plainText);
+        }
+        else {
+            hash = `link-${plainText
+                .replace(/[^a-zA-Z0-9가-힣\s-_]/g, "")
+                .trim()
+                .replace(/\s+/g, "-")
+                .toLowerCase()}-${(0, crypto_1.randomUUID)().substring(0, 4)}`;
+            hashCache.set(plainText, hash);
+        }
         block.hash = hash;
-        tableOfContents.push({
-            type: block.type,
-            hash,
-            title: plainText,
-        });
+        // 중복 체크 - 동일한 hash가 이미 존재하는지 확인
+        const existingTocIndex = tableOfContents.findIndex((toc) => toc.hash === hash);
+        if (existingTocIndex === -1) {
+            tableOfContents.push({
+                type: block.type,
+                hash,
+                title: plainText,
+            });
+        }
     }
 };
 exports.processTableOfContents = processTableOfContents;

package/dist/util/tocHelper.d.ts

@@ -0,0 +1,18 @@
+import { Reporter } from "gatsby";
+/**
+ * TocEntry interface representing a table of contents entry
+ */
+export interface TocEntry {
+    type: string;
+    hash: string;
+    title: string;
+}
+/**
+ * Utility function to efficiently handle large table of contents arrays
+ * by removing duplicates and optionally limiting size
+ */
+export declare const optimizeTocArray: (tocEntries: TocEntry[], reporter: Reporter, options?: {
+    maxSize?: number;
+    warnThreshold?: number;
+    removeDuplicates?: boolean;
+}) => TocEntry[];

package/dist/util/tocHelper.js

@@ -0,0 +1,43 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.optimizeTocArray = void 0;
+/**
+ * Utility function to efficiently handle large table of contents arrays
+ * by removing duplicates and optionally limiting size
+ */
+const optimizeTocArray = (tocEntries, reporter, options = {}) => {
+    const { maxSize = 1000, // Maximum entries to include
+    warnThreshold = 300, // Threshold to issue warning
+    removeDuplicates = true, // Whether to remove duplicates
+     } = options;
+    if (!tocEntries || tocEntries.length === 0) {
+        return [];
+    }
+    // Track memory usage for large TOCs
+    if (tocEntries.length > warnThreshold) {
+        reporter.warn(`Large table of contents detected (${tocEntries.length} items). This might affect performance.`);
+    }
+    // Remove duplicates if requested
+    let processedToc = tocEntries;
+    if (removeDuplicates) {
+        const startTime = Date.now();
+        const uniqueMap = new Map();
+        // Use a map for faster duplicate removal - O(n) instead of O(n²)
+        for (const entry of tocEntries) {
+            uniqueMap.set(entry.hash, entry);
+        }
+        processedToc = Array.from(uniqueMap.values());
+        const removedCount = tocEntries.length - processedToc.length;
+        const processTime = Date.now() - startTime;
+        if (removedCount > 0) {
+            reporter.info(`Removed ${removedCount} duplicate TOC entries in ${processTime}ms.`);
+        }
+    }
+    // Limit size if necessary
+    if (processedToc.length > maxSize) {
+        reporter.warn(`Table of contents exceeds maximum size (${processedToc.length} > ${maxSize}). Truncating.`);
+        processedToc = processedToc.slice(0, maxSize);
+    }
+    return processedToc;
+};
+exports.optimizeTocArray = optimizeTocArray;

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "gatsby-source-notion-churnotion",
   "description": "Gatsby plugin that can connect with One Notion Database RECURSIVELY using official API",
-  "version": "1.2.1",
+  "version": "1.2.3",
   "skipLibCheck": true,
   "license": "0BSD",
   "main": "./dist/gatsby-node.js",
@@ -11,8 +11,7 @@
   ],
   "scripts": {
     "clean": "del-cli dist",
-    "build": "npm run build:rust && tsc",
-    "build:rust": "cd rust/notion-parallel && (cargo build --release || cargo build --release --target x86_64-pc-windows-gnu || echo 'Rust build failed, using TypeScript fallback') && cd ../..",
+    "build": "tsc",
     "develop": "tsc --watch",
     "test": "jest",
     "prepare": "npm run clean && npm run build"