notion-to-md 4.0.0-alpha.2 → 4.0.0-alpha.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/build/core/block-fetcher/index.d.ts +1 -1
- package/build/core/block-fetcher/index.d.ts.map +1 -1
- package/build/core/block-fetcher/index.js.map +1 -1
- package/build/index.d.ts +1 -1
- package/build/index.d.ts.map +1 -1
- package/build/index.js.map +1 -1
- package/build/plugins/exporter/DefaultExporter/index.d.ts +40 -0
- package/build/plugins/exporter/DefaultExporter/index.d.ts.map +1 -0
- package/build/plugins/exporter/DefaultExporter/index.js +90 -0
- package/build/plugins/exporter/DefaultExporter/index.js.map +1 -0
- package/build/plugins/exporter/index.d.ts +1 -39
- package/build/plugins/exporter/index.d.ts.map +1 -1
- package/build/plugins/exporter/index.js +14 -88
- package/build/plugins/exporter/index.js.map +1 -1
- package/build/plugins/renderer/index.d.ts +2 -0
- package/build/plugins/renderer/index.d.ts.map +1 -0
- package/build/plugins/renderer/index.js +18 -0
- package/build/plugins/renderer/index.js.map +1 -0
- package/build/utils/page-ref-builder/index.d.ts +1 -1
- package/build/utils/page-ref-builder/index.d.ts.map +1 -1
- package/build/utils/page-ref-builder/index.js.map +1 -1
- package/package.json +16 -5
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/core/block-fetcher/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/core/block-fetcher/index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAC/C,OAAO,EAML,qBAAqB,EACrB,kBAAkB,EAClB,SAAS,EACV,MAAM,aAAa,CAAC;AAGrB,MAAM,WAAW,kBAAkB;IACjC,mBAAmB,CAAC,EAAE,OAAO,CAAC;IAC9B,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAC9B,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAC3B,kBAAkB,CAAC,EAAE,OAAO,CAAC;CAC9B;AAQD,qBAAa,YAAa,YAAW,kBAAkB;IAmBnD,OAAO,CAAC,MAAM;IACd,OAAO,CAAC,MAAM;IAnBhB,IAAI,CAAC,EAAE,kBAAkB,CAAC;IAE1B,OAAO,CAAC,KAAK,CAAmB;IAChC,OAAO,CAAC,MAAM,CAAsD;IACpE,OAAO,CAAC,cAAc,CAAqB;IAC3C,OAAO,CAAC,cAAc,CAAC,CAAuB;IAC9C,OAAO,CAAC,YAAY,CAA8B;IAClD,OAAO,CAAC,WAAW,CAAc;IACjC,OAAO,CAAC,WAAW,CAAyC;IAC5D,OAAO,CAAC,aAAa,CAAyC;IAE9D,OAAO,CAAC,eAAe,CAGrB;gBAGA,MAAM,EAAE,MAAM,EACN,MAAM,EAAE,MAAM,EACd,MAAM,GAAE,kBAKf;IAOG,OAAO,CAAC,IAAI,EAAE,SAAS,GAAG,OAAO,CAAC,SAAS,CAAC;IAW5C,SAAS,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,qBAAqB,CAAC;IA6C/D,OAAO,CAAC,OAAO;YAOD,WAAW;YAuEX,gBAAgB;YA0BhB,kBAAkB;YA6BlB,gBAAgB;YAuBhB,mBAAmB;IAUjC,OAAO,CAAC,WAAW;IAInB,OAAO,CAAC,cAAc;CAmCvB"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/core/block-fetcher/index.ts"],"names":[],"mappings":";;;AAWA,+CAAkE;AAiBlE,MAAa,YAAY;IAiBvB,YACE,MAAc,EACN,MAAc,EACd,SAA6B;QACnC,mBAAmB,EAAE,KAAK;QAC1B,aAAa,EAAE,KAAK;QACpB,oBAAoB,EAAE,CAAC;QACvB,SAAS,EAAE,CAAC;KACb;;QANO,WAAM,GAAN,MAAM,CAAQ;QACd,WAAM,GAAN,MAAM,CAKb;QAtBK,UAAK,GAAgB,EAAE,CAAC;QACxB,WAAM,GAAG,IAAI,GAAG,EAA2C,CAAC;QAC5D,mBAAc,GAAG,IAAI,GAAG,EAAU,CAAC;QAEnC,iBAAY,GAA2B,EAAE,CAAC;QAC1C,gBAAW,GAAW,EAAE,CAAC;QACzB,gBAAW,GAAsC,EAAE,CAAC;QACpD,kBAAa,GAAsC,EAAE,CAAC;QAEtD,oBAAe,GAAG;YACxB,QAAQ,EAAE,CAAC;YACX,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE;SACtB,CAAC;QAYA,MAAM,UAAU,GAAG,cAAc,CAAC;QAClC,IAAI,CAAC,MAAM,CAAC,oBAAoB,GAAG,MAAA,MAAM,CAAC,oBAAoB,mCAAI,CAAC,CAAC;QACpE,IAAI,CAAC,MAAM,CAAC,SAAS,GAAG,MAAA,MAAM,CAAC,SAAS,mCAAI,CAAC,CAAC;IAChD,CAAC;IAED,KAAK,CAAC,OAAO,CAAC,IAAe;QAC3B,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAEpD,MAAM,WAAW,GAAc;YAC7B,GAAG,IAAI;YACP,SAAS;SACV,CAAC;QAEF,OAAO,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC;IAClE,CAAC;IAED,KAAK,CAAC,SAAS,CAAC,MAAc;QAC5B,IAAI,CAAC,WAAW,GAAG,MAAM,CAAC;QAC1B,IAAI,CAAC,KAAK,GAAG,EAAE,CAAC;QAChB,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;QACpB,IAAI,CAAC,cAAc,CAAC,KAAK,EAAE,CAAC;QAC5B,IAAI,CAAC,cAAc,GAAG,SAAS,CAAC;QAChC,IAAI,CAAC,YAAY,GAAG,EAAE,CAAC;QAEvB,yCAAyC;QACzC,IAAI,CAAC,OAAO,CAAC,EAAE,IAAI,EAAE,cAAc,EAAE,EAAE,EAAE,MAAM,EAAE,CAAC,CAAC;QAEnD,IAAI,IAAI,CAAC,MAAM,CAAC,mBAAmB,EAAE;YACnC,IAAI,CAAC,OAAO,CAAC,EAAE,IAAI,EAAE,kBAAkB,EAAE,EAAE,EAAE,MAAM,EAAE,CAAC,CAAC;SACxD;QAED,IAAI,IAAI,CAAC,MAAM,CAAC,aAAa,EAAE;YAC7B,IAAI,CAAC,OAAO,CAAC,EAAE,IAAI,EAAE,gBAAgB,EAAE,EAAE,EAAE,MAAM,EAAE,CAAC,CAAC;SACtD;QAED,4BAA4B;QAC5B,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE;YAC5B,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,EAAE,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;YAC1D,MAAM,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;SAChE;QAED,MAAM,UAAU,GAAkB;YAChC,UAAU,EAAE,IAAI,CAAC,cAAc,IAAI,EAAE;YACrC,QAAQ,EAAE,IAAI,CAAC,YAAY;YAC3B,MAAM,EAAE,IAAI,CAAC,cAAc,CAAC,MAAM,CAAC;SACpC,CAAC;QAEF,sDAAsD;QACtD,IAAI,IAAI,CAAC,MAAM,CAAC,gBAAgB,IAAI,IAAI,CAAC,MAAM,CAAC,kBAAkB,EAAE;YAClE,OAAO;gBACL,GAAG,UAAU;gBACb,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,gBAAgB,IAAI,EAAE,WAAW,EAAE,IAAI,CAAC,WAAW,EAAE,CAAC;gBACtE,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,kBAAkB,IAAI;oBACpC,aAAa,EAAE,IAAI,CAAC,aAAa;iBAClC,CAAC;aACH,CAAC;SACH;QAED,OAAO,UAAU,CAAC;IACpB,CAAC;IAEO,OAAO,CAAC,IAAe;QAC7B,MAAM,MAAM,GAAG,GAAG,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,EAAE,EAAE,CAAC;QACzC,IAAI,CAAC,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE;YACpC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;SACvB;IACH,CAAC;IAEO,KAAK,CAAC,WAAW,CAAC,IAAe;QACvC,MAAM,MAAM,GAAG,GAAG,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,EAAE,EAAE,CAAC;QACzC,IAAI,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,MAAM,CAAC;YAAE,OAAO;QAE5C,QAAQ,IAAI,CAAC,IAAI,EAAE;YACjB,KAAK,cAAc,CAAC,CAAC;gBACnB,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,kBAAkB,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;gBAEtD,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE;oBAC1B,MAAM,WAAW,GAAG;wBAClB,GAAG,KAAK;wBACR,QAAQ,EAAE,EAAE;wBACZ,QAAQ,EAAE,EAAE;qBACb,CAAC;oBACF,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,EAAE,WAAW,CAAC,CAAC;oBAEvC,uDAAuD;oBACvD,wDAAwD;oBACxD,4CAA4C;oBAC5C,IAAI,IAAI,CAAC,MAAM,CAAC,gBAAgB,IAAI,IAAA,qBAAY,EAAC,WAAW,CAAC,EAAE;wBAC7D,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;qBACpC;oBACD,IAAI,IAAI,CAAC,MAAM,CAAC,kBAAkB,IAAI,IAAA,uBAAc,EAAC,WAAW,CAAC,EAAE;wBACjE,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;qBACtC;oBAED,kDAAkD;oBAClD,IAAI,cAAc,IAAI,KAAK,IAAI,KAAK,CAAC,YAAY,EAAE;wBACjD,IAAI,CAAC,OAAO,CAAC;4BACX,IAAI,EAAE,cAAc;4BACpB,EAAE,EAAE,KAAK,CAAC,EAAE;4BACZ,QAAQ,EAAE,IAAI,CAAC,EAAE;yBAClB,CAAC,CAAC;qBACJ;oBAED,mDAAmD;oBACnD,gEAAgE;oBAChE,IAAI,IAAI,CAAC,MAAM,CAAC,aAAa,EAAE;wBAC7B,IAAI,CAAC,OAAO,CAAC;4BACX,IAAI,EAAE,gBAAgB;4BACtB,EAAE,EAAE,KAAK,CAAC,EAAE;yBACb,CAAC,CAAC;qBACJ;iBACF;gBACD,MAAM;aACP;YAED,KAAK,gBAAgB,CAAC,CAAC;gBACrB,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;gBACtD,IAAI,IAAI,CAAC,EAAE,KAAK,IAAI,CAAC,WAAW,EAAE;oBAChC,sBAAsB;oBACtB,IAAI,CAAC,YAAY,GAAG,QAAQ,CAAC;iBAC9B;qBAAM;oBACL,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;oBACvC,IAAI,KAAK,EAAE;wBACT,KAAK,CAAC,QAAQ,GAAG,QAAQ,CAAC;qBAC3B;iBACF;gBACD,MAAM;aACP;YAED,KAAK,kBAAkB,CAAC,CAAC;gBACvB,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,mBAAmB,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;gBAC3D,IAAI,CAAC,cAAc,GAAG,UAAU,CAAC;gBACjC,MAAM;aACP;SACF;QAED,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;IAClC,CAAC;IAEO,KAAK,CAAC,gBAAgB,CAAI,OAAyB;QACzD,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACvB,MAAM,UAAU,GAAG,IAAI,CAAC;QAExB,+BAA+B;QAC/B,IAAI,GAAG,GAAG,IAAI,CAAC,eAAe,CAAC,SAAS,IAAI,UAAU,EAAE;YACtD,IAAI,CAAC,eAAe,GAAG;gBACrB,QAAQ,EAAE,CAAC;gBACX,SAAS,EAAE,GAAG;aACf,CAAC;SACH;QAED,mCAAmC;QACnC,IAAI,IAAI,CAAC,eAAe,CAAC,QAAQ,IAAI,IAAI,CAAC,MAAM,CAAC,oBAAqB,EAAE;YACtE,MAAM,QAAQ,GAAG,UAAU,GAAG,CAAC,GAAG,GAAG,IAAI,CAAC,eAAe,CAAC,SAAS,CAAC,CAAC;YACrE,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC,CAAC;YAC9D,IAAI,CAAC,eAAe,GAAG;gBACrB,QAAQ,EAAE,CAAC;gBACX,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE;aACtB,CAAC;SACH;QAED,IAAI,CAAC,eAAe,CAAC,QAAQ,EAAE,CAAC;QAChC,OAAO,OAAO,EAAE,CAAC;IACnB,CAAC;IAEO,KAAK,CAAC,kBAAkB,CAC9B,OAAe;;QAEf,IAAI,SAAS,GAAqC,EAAE,CAAC;QACrD,IAAI,OAAO,GAAG,IAAI,CAAC;QACnB,IAAI,MAA0B,CAAC;QAE/B,OAAO,OAAO,EAAE;YACd,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,GAAG,EAAE,CAChD,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC;gBAC/B,QAAQ,EAAE,OAAO;gBACjB,YAAY,EAAE,MAAM;aACrB,CAAC,CACH,CAAC;YAEF,gCAAgC;YAChC,6DAA6D;YAC7D,MAAM,MAAM,GAAG,QAAQ,CAAC,OAAO,CAAC,MAAM,CACpC,CAAC,KAAK,EAAE,EAAE,CAAC,MAAM,IAAI,KAAK,IAAI,KAAK,CAAC,IAAI,KAAK,aAAa,CACvB,CAAC;YAEtC,SAAS,GAAG,CAAC,GAAG,SAAS,EAAE,GAAG,MAAM,CAAC,CAAC;YACtC,OAAO,GAAG,QAAQ,CAAC,QAAQ,CAAC;YAC5B,MAAM,GAAG,MAAA,QAAQ,CAAC,WAAW,mCAAI,SAAS,CAAC;SAC5C;QAED,OAAO,SAAS,CAAC;IACnB,CAAC;IAEO,KAAK,CAAC,gBAAgB,CAC5B,OAAe;;QAEf,IAAI,WAAW,GAA2B,EAAE,CAAC;QAC7C,IAAI,OAAO,GAAG,IAAI,CAAC;QACnB,IAAI,MAA0B,CAAC;QAE/B,OAAO,OAAO,EAAE;YACd,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,GAAG,EAAE,CAChD,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC;gBACxB,QAAQ,EAAE,OAAO;gBACjB,YAAY,EAAE,MAAM;aACrB,CAAC,CACH,CAAC;YAEF,WAAW,GAAG,CAAC,GAAG,WAAW,EAAE,GAAG,QAAQ,CAAC,OAAO,CAAC,CAAC;YACpD,OAAO,GAAG,QAAQ,CAAC,QAAQ,CAAC;YAC5B,MAAM,GAAG,MAAA,QAAQ,CAAC,WAAW,mCAAI,SAAS,CAAC;SAC5C;QAED,OAAO,WAAW,CAAC;IACrB,CAAC;IAEO,KAAK,CAAC,mBAAmB,CAC/B,MAAc;QAEd,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,GAAG,EAAE,CAChD,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,CAChD,CAAC;QAEF,OAAO,YAAY,IAAI,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE,CAAC;IAC7D,CAAC;IAEO,WAAW,CAAC,EAAU;QAC5B,OAAO,EAAE,CAAC,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;IAC9B,CAAC;IAEO,cAAc,CAAC,MAAc;;QACnC,MAAM,WAAW,GAAG,IAAI,GAAG,EAA6C,CAAC;QAEzE,KAAK,MAAM,CAAC,EAAE,EAAE,KAAK,CAAC,IAAI,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,EAAE;YAC/C,MAAM,QAAQ;YACZ,aAAa;YACb,CAAA,MAAA,KAAK,CAAC,MAAM,0CAAE,IAAI,MAAK,UAAU;gBAC/B,CAAC,CAAC,aAAa;oBACb,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,MAAM,CAAC,QAAQ,CAAC;gBACzC,CAAC,CAAC,aAAa;oBACb,CAAA,MAAA,KAAK,CAAC,MAAM,0CAAE,IAAI,MAAK,SAAS;wBAChC,CAAC,CAAC,aAAa;4BACb,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,MAAM,CAAC,OAAO,CAAC;wBACxC,CAAC,CAAC,SAAS,CAAC;YAElB,IAAI,QAAQ,EAAE;gBACZ,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE;oBAC9B,WAAW,CAAC,GAAG,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;iBAC/B;gBACD,WAAW,CAAC,GAAG,CAAC,QAAQ,CAAE,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;aACxC;SACF;QAED,MAAM,aAAa,GAAG,CACpB,QAAgB,EACkB,EAAE;YACpC,MAAM,QAAQ,GAAG,WAAW,CAAC,GAAG,CAAC,IAAI,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC,IAAI,EAAE,CAAC;YACnE,OAAO,QAAQ,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE;gBAC5B,KAAK,CAAC,QAAQ,GAAG,aAAa,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;gBACzC,OAAO,KAAK,CAAC;YACf,CAAC,CAAC,CAAC;QACL,CAAC,CAAC;QAEF,OAAO,aAAa,CAAC,MAAM,CAAC,CAAC;IAC/B,CAAC;CACF;AArSD,oCAqSC","sourcesContent":["import { Client } from '@notionhq/client';\nimport {\n ListBlockChildrenResponseResult,\n PageObjectProperties,\n CommentResponseResults,\n FetcherOutput,\n ListBlockChildrenResponseResults,\n ExtendedFetcherOutput,\n ProcessorChainNode,\n ChainData,\n} from '../../types';\nimport { isMediaBlock, isPageRefBlock } from '../../utils/notion';\n\nexport interface BlockFetcherConfig {\n fetchPageProperties?: boolean;\n fetchComments?: boolean;\n maxRequestsPerSecond?: number;\n batchSize?: number;\n trackMediaBlocks?: boolean;\n trackPageRefBlocks?: boolean;\n}\n\ninterface QueueTask {\n type: 'fetch_properties' | 'fetch_comments' | 'fetch_blocks';\n id: string;\n parentId?: string;\n}\n\nexport class BlockFetcher implements ProcessorChainNode {\n next?: ProcessorChainNode;\n\n private queue: QueueTask[] = [];\n private blocks = new Map<string, ListBlockChildrenResponseResult>();\n private processedTasks = new Set<string>();\n private pageProperties?: PageObjectProperties;\n private rootComments: CommentResponseResults = [];\n private rootBlockId: string = '';\n private mediaBlocks: ListBlockChildrenResponseResult[] = [];\n private pageRefBlocks: ListBlockChildrenResponseResult[] = [];\n\n private rateLimitWindow = {\n requests: 0,\n startTime: Date.now(),\n };\n\n constructor(\n pageId: string,\n private client: Client,\n private config: BlockFetcherConfig = {\n fetchPageProperties: false,\n fetchComments: false,\n maxRequestsPerSecond: 3,\n batchSize: 3,\n },\n ) {\n const moduleType = 'BlockFetcher';\n this.config.maxRequestsPerSecond = config.maxRequestsPerSecond ?? 3;\n this.config.batchSize = config.batchSize ?? 3;\n }\n\n async process(data: ChainData): Promise<ChainData> {\n const blockTree = await this.getBlocks(data.pageId);\n\n const updatedData: ChainData = {\n ...data,\n blockTree,\n };\n\n return this.next ? this.next.process(updatedData) : updatedData;\n }\n\n async getBlocks(pageId: string): Promise<ExtendedFetcherOutput> {\n this.rootBlockId = pageId;\n this.queue = [];\n this.blocks.clear();\n this.processedTasks.clear();\n this.pageProperties = undefined;\n this.rootComments = [];\n\n // Initialize queue with root level tasks\n this.addTask({ type: 'fetch_blocks', id: pageId });\n\n if (this.config.fetchPageProperties) {\n this.addTask({ type: 'fetch_properties', id: pageId });\n }\n\n if (this.config.fetchComments) {\n this.addTask({ type: 'fetch_comments', id: pageId });\n }\n\n // Process queue until empty\n while (this.queue.length > 0) {\n const batch = this.queue.splice(0, this.config.batchSize);\n await Promise.all(batch.map((task) => this.processTask(task)));\n }\n\n const baseOutput: FetcherOutput = {\n properties: this.pageProperties || {},\n comments: this.rootComments,\n blocks: this.buildBlockTree(pageId),\n };\n\n // Only include special arrays if tracking was enabled\n if (this.config.trackMediaBlocks || this.config.trackPageRefBlocks) {\n return {\n ...baseOutput,\n ...(this.config.trackMediaBlocks && { mediaBlocks: this.mediaBlocks }),\n ...(this.config.trackPageRefBlocks && {\n pageRefBlocks: this.pageRefBlocks,\n }),\n };\n }\n\n return baseOutput;\n }\n\n private addTask(task: QueueTask): void {\n const taskId = `${task.type}-${task.id}`;\n if (!this.processedTasks.has(taskId)) {\n this.queue.push(task);\n }\n }\n\n private async processTask(task: QueueTask): Promise<void> {\n const taskId = `${task.type}-${task.id}`;\n if (this.processedTasks.has(taskId)) return;\n\n switch (task.type) {\n case 'fetch_blocks': {\n const blocks = await this.fetchBlockChildren(task.id);\n\n for (const block of blocks) {\n const storedBlock = {\n ...block,\n children: [],\n comments: [],\n };\n this.blocks.set(block.id, storedBlock);\n\n // Track special blocks if enabled (storing references)\n // these will be used by the next stage saving us from a\n // lot of recursive calls to idenitfy blocks\n if (this.config.trackMediaBlocks && isMediaBlock(storedBlock)) {\n this.mediaBlocks.push(storedBlock);\n }\n if (this.config.trackPageRefBlocks && isPageRefBlock(storedBlock)) {\n this.pageRefBlocks.push(storedBlock);\n }\n\n // If block has children, queue task to fetch them\n if ('has_children' in block && block.has_children) {\n this.addTask({\n type: 'fetch_blocks',\n id: block.id,\n parentId: task.id,\n });\n }\n\n // Queue comment fetching for each block if enabled\n // extra api call for comment fetching, no way to know otherwise\n if (this.config.fetchComments) {\n this.addTask({\n type: 'fetch_comments',\n id: block.id,\n });\n }\n }\n break;\n }\n\n case 'fetch_comments': {\n const comments = await this.fetchAllComments(task.id);\n if (task.id === this.rootBlockId) {\n // page level comments\n this.rootComments = comments;\n } else {\n const block = this.blocks.get(task.id);\n if (block) {\n block.comments = comments;\n }\n }\n break;\n }\n\n case 'fetch_properties': {\n const properties = await this.fetchPageProperties(task.id);\n this.pageProperties = properties;\n break;\n }\n }\n\n this.processedTasks.add(taskId);\n }\n\n private async rateLimitRequest<T>(request: () => Promise<T>): Promise<T> {\n const now = Date.now();\n const windowSize = 1000;\n\n // Reset window if it's expired\n if (now - this.rateLimitWindow.startTime >= windowSize) {\n this.rateLimitWindow = {\n requests: 0,\n startTime: now,\n };\n }\n\n // Wait if we've hit the rate limit\n if (this.rateLimitWindow.requests >= this.config.maxRequestsPerSecond!) {\n const waitTime = windowSize - (now - this.rateLimitWindow.startTime);\n await new Promise((resolve) => setTimeout(resolve, waitTime));\n this.rateLimitWindow = {\n requests: 0,\n startTime: Date.now(),\n };\n }\n\n this.rateLimitWindow.requests++;\n return request();\n }\n\n private async fetchBlockChildren(\n blockId: string,\n ): Promise<ListBlockChildrenResponseResults> {\n let allBlocks: ListBlockChildrenResponseResults = [];\n let hasMore = true;\n let cursor: string | undefined;\n\n while (hasMore) {\n const response = await this.rateLimitRequest(() =>\n this.client.blocks.children.list({\n block_id: blockId,\n start_cursor: cursor,\n }),\n );\n\n // Filter out unsupported blocks\n // should we leave it here or let user handle it in renderer?\n const blocks = response.results.filter(\n (block) => 'type' in block && block.type !== 'unsupported',\n ) as ListBlockChildrenResponseResults;\n\n allBlocks = [...allBlocks, ...blocks];\n hasMore = response.has_more;\n cursor = response.next_cursor ?? undefined;\n }\n\n return allBlocks;\n }\n\n private async fetchAllComments(\n blockId: string,\n ): Promise<CommentResponseResults> {\n let allComments: CommentResponseResults = [];\n let hasMore = true;\n let cursor: string | undefined;\n\n while (hasMore) {\n const response = await this.rateLimitRequest(() =>\n this.client.comments.list({\n block_id: blockId,\n start_cursor: cursor,\n }),\n );\n\n allComments = [...allComments, ...response.results];\n hasMore = response.has_more;\n cursor = response.next_cursor ?? undefined;\n }\n\n return allComments;\n }\n\n private async fetchPageProperties(\n pageId: string,\n ): Promise<PageObjectProperties> {\n const response = await this.rateLimitRequest(() =>\n this.client.pages.retrieve({ page_id: pageId }),\n );\n\n return 'properties' in response ? response.properties : {};\n }\n\n private normalizeId(id: string): string {\n return id.replace(/-/g, '');\n }\n\n private buildBlockTree(rootId: string): ListBlockChildrenResponseResults {\n const childrenMap = new Map<string, ListBlockChildrenResponseResult[]>();\n\n for (const [id, block] of this.blocks.entries()) {\n const parentId =\n // @ts-ignore\n block.parent?.type === 'block_id'\n ? // @ts-ignore\n this.normalizeId(block.parent.block_id)\n : // @ts-ignore\n block.parent?.type === 'page_id'\n ? // @ts-ignore\n this.normalizeId(block.parent.page_id)\n : undefined;\n\n if (parentId) {\n if (!childrenMap.has(parentId)) {\n childrenMap.set(parentId, []);\n }\n childrenMap.get(parentId)!.push(block);\n }\n }\n\n const buildChildren = (\n parentId: string,\n ): ListBlockChildrenResponseResults => {\n const children = childrenMap.get(this.normalizeId(parentId)) || [];\n return children.map((block) => {\n block.children = buildChildren(block.id);\n return block;\n });\n };\n\n return buildChildren(rootId);\n }\n}\n"]}
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/core/block-fetcher/index.ts"],"names":[],"mappings":";;;AAWA,+CAAkE;AAiBlE,MAAa,YAAY;IAiBvB,YACE,MAAc,EACN,MAAc,EACd,SAA6B;QACnC,mBAAmB,EAAE,KAAK;QAC1B,aAAa,EAAE,KAAK;QACpB,oBAAoB,EAAE,CAAC;QACvB,SAAS,EAAE,CAAC;KACb;;QANO,WAAM,GAAN,MAAM,CAAQ;QACd,WAAM,GAAN,MAAM,CAKb;QAtBK,UAAK,GAAgB,EAAE,CAAC;QACxB,WAAM,GAAG,IAAI,GAAG,EAA2C,CAAC;QAC5D,mBAAc,GAAG,IAAI,GAAG,EAAU,CAAC;QAEnC,iBAAY,GAA2B,EAAE,CAAC;QAC1C,gBAAW,GAAW,EAAE,CAAC;QACzB,gBAAW,GAAsC,EAAE,CAAC;QACpD,kBAAa,GAAsC,EAAE,CAAC;QAEtD,oBAAe,GAAG;YACxB,QAAQ,EAAE,CAAC;YACX,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE;SACtB,CAAC;QAYA,MAAM,UAAU,GAAG,cAAc,CAAC;QAClC,IAAI,CAAC,MAAM,CAAC,oBAAoB,GAAG,MAAA,MAAM,CAAC,oBAAoB,mCAAI,CAAC,CAAC;QACpE,IAAI,CAAC,MAAM,CAAC,SAAS,GAAG,MAAA,MAAM,CAAC,SAAS,mCAAI,CAAC,CAAC;IAChD,CAAC;IAED,KAAK,CAAC,OAAO,CAAC,IAAe;QAC3B,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAEpD,MAAM,WAAW,GAAc;YAC7B,GAAG,IAAI;YACP,SAAS;SACV,CAAC;QAEF,OAAO,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC;IAClE,CAAC;IAED,KAAK,CAAC,SAAS,CAAC,MAAc;QAC5B,IAAI,CAAC,WAAW,GAAG,MAAM,CAAC;QAC1B,IAAI,CAAC,KAAK,GAAG,EAAE,CAAC;QAChB,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;QACpB,IAAI,CAAC,cAAc,CAAC,KAAK,EAAE,CAAC;QAC5B,IAAI,CAAC,cAAc,GAAG,SAAS,CAAC;QAChC,IAAI,CAAC,YAAY,GAAG,EAAE,CAAC;QAEvB,yCAAyC;QACzC,IAAI,CAAC,OAAO,CAAC,EAAE,IAAI,EAAE,cAAc,EAAE,EAAE,EAAE,MAAM,EAAE,CAAC,CAAC;QAEnD,IAAI,IAAI,CAAC,MAAM,CAAC,mBAAmB,EAAE;YACnC,IAAI,CAAC,OAAO,CAAC,EAAE,IAAI,EAAE,kBAAkB,EAAE,EAAE,EAAE,MAAM,EAAE,CAAC,CAAC;SACxD;QAED,IAAI,IAAI,CAAC,MAAM,CAAC,aAAa,EAAE;YAC7B,IAAI,CAAC,OAAO,CAAC,EAAE,IAAI,EAAE,gBAAgB,EAAE,EAAE,EAAE,MAAM,EAAE,CAAC,CAAC;SACtD;QAED,4BAA4B;QAC5B,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE;YAC5B,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,EAAE,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;YAC1D,MAAM,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;SAChE;QAED,MAAM,UAAU,GAAkB;YAChC,UAAU,EAAE,IAAI,CAAC,cAAc,IAAI,EAAE;YACrC,QAAQ,EAAE,IAAI,CAAC,YAAY;YAC3B,MAAM,EAAE,IAAI,CAAC,cAAc,CAAC,MAAM,CAAC;SACpC,CAAC;QAEF,sDAAsD;QACtD,IAAI,IAAI,CAAC,MAAM,CAAC,gBAAgB,IAAI,IAAI,CAAC,MAAM,CAAC,kBAAkB,EAAE;YAClE,OAAO;gBACL,GAAG,UAAU;gBACb,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,gBAAgB,IAAI,EAAE,WAAW,EAAE,IAAI,CAAC,WAAW,EAAE,CAAC;gBACtE,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,kBAAkB,IAAI;oBACpC,aAAa,EAAE,IAAI,CAAC,aAAa;iBAClC,CAAC;aACH,CAAC;SACH;QAED,OAAO,UAAU,CAAC;IACpB,CAAC;IAEO,OAAO,CAAC,IAAe;QAC7B,MAAM,MAAM,GAAG,GAAG,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,EAAE,EAAE,CAAC;QACzC,IAAI,CAAC,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE;YACpC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;SACvB;IACH,CAAC;IAEO,KAAK,CAAC,WAAW,CAAC,IAAe;QACvC,MAAM,MAAM,GAAG,GAAG,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,EAAE,EAAE,CAAC;QACzC,IAAI,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,MAAM,CAAC;YAAE,OAAO;QAE5C,QAAQ,IAAI,CAAC,IAAI,EAAE;YACjB,KAAK,cAAc,CAAC,CAAC;gBACnB,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,kBAAkB,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;gBAEtD,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE;oBAC1B,MAAM,WAAW,GAAG;wBAClB,GAAG,KAAK;wBACR,QAAQ,EAAE,EAAE;wBACZ,QAAQ,EAAE,EAAE;qBACb,CAAC;oBACF,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,EAAE,WAAW,CAAC,CAAC;oBAEvC,uDAAuD;oBACvD,wDAAwD;oBACxD,4CAA4C;oBAC5C,IAAI,IAAI,CAAC,MAAM,CAAC,gBAAgB,IAAI,IAAA,qBAAY,EAAC,WAAW,CAAC,EAAE;wBAC7D,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;qBACpC;oBACD,IAAI,IAAI,CAAC,MAAM,CAAC,kBAAkB,IAAI,IAAA,uBAAc,EAAC,WAAW,CAAC,EAAE;wBACjE,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;qBACtC;oBAED,kDAAkD;oBAClD,IAAI,cAAc,IAAI,KAAK,IAAI,KAAK,CAAC,YAAY,EAAE;wBACjD,IAAI,CAAC,OAAO,CAAC;4BACX,IAAI,EAAE,cAAc;4BACpB,EAAE,EAAE,KAAK,CAAC,EAAE;4BACZ,QAAQ,EAAE,IAAI,CAAC,EAAE;yBAClB,CAAC,CAAC;qBACJ;oBAED,mDAAmD;oBACnD,gEAAgE;oBAChE,IAAI,IAAI,CAAC,MAAM,CAAC,aAAa,EAAE;wBAC7B,IAAI,CAAC,OAAO,CAAC;4BACX,IAAI,EAAE,gBAAgB;4BACtB,EAAE,EAAE,KAAK,CAAC,EAAE;yBACb,CAAC,CAAC;qBACJ;iBACF;gBACD,MAAM;aACP;YAED,KAAK,gBAAgB,CAAC,CAAC;gBACrB,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;gBACtD,IAAI,IAAI,CAAC,EAAE,KAAK,IAAI,CAAC,WAAW,EAAE;oBAChC,sBAAsB;oBACtB,IAAI,CAAC,YAAY,GAAG,QAAQ,CAAC;iBAC9B;qBAAM;oBACL,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;oBACvC,IAAI,KAAK,EAAE;wBACT,KAAK,CAAC,QAAQ,GAAG,QAAQ,CAAC;qBAC3B;iBACF;gBACD,MAAM;aACP;YAED,KAAK,kBAAkB,CAAC,CAAC;gBACvB,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,mBAAmB,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;gBAC3D,IAAI,CAAC,cAAc,GAAG,UAAU,CAAC;gBACjC,MAAM;aACP;SACF;QAED,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;IAClC,CAAC;IAEO,KAAK,CAAC,gBAAgB,CAAI,OAAyB;QACzD,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACvB,MAAM,UAAU,GAAG,IAAI,CAAC;QAExB,+BAA+B;QAC/B,IAAI,GAAG,GAAG,IAAI,CAAC,eAAe,CAAC,SAAS,IAAI,UAAU,EAAE;YACtD,IAAI,CAAC,eAAe,GAAG;gBACrB,QAAQ,EAAE,CAAC;gBACX,SAAS,EAAE,GAAG;aACf,CAAC;SACH;QAED,mCAAmC;QACnC,IAAI,IAAI,CAAC,eAAe,CAAC,QAAQ,IAAI,IAAI,CAAC,MAAM,CAAC,oBAAqB,EAAE;YACtE,MAAM,QAAQ,GAAG,UAAU,GAAG,CAAC,GAAG,GAAG,IAAI,CAAC,eAAe,CAAC,SAAS,CAAC,CAAC;YACrE,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC,CAAC;YAC9D,IAAI,CAAC,eAAe,GAAG;gBACrB,QAAQ,EAAE,CAAC;gBACX,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE;aACtB,CAAC;SACH;QAED,IAAI,CAAC,eAAe,CAAC,QAAQ,EAAE,CAAC;QAChC,OAAO,OAAO,EAAE,CAAC;IACnB,CAAC;IAEO,KAAK,CAAC,kBAAkB,CAC9B,OAAe;;QAEf,IAAI,SAAS,GAAqC,EAAE,CAAC;QACrD,IAAI,OAAO,GAAG,IAAI,CAAC;QACnB,IAAI,MAA0B,CAAC;QAE/B,OAAO,OAAO,EAAE;YACd,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,GAAG,EAAE,CAChD,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC;gBAC/B,QAAQ,EAAE,OAAO;gBACjB,YAAY,EAAE,MAAM;aACrB,CAAC,CACH,CAAC;YAEF,gCAAgC;YAChC,6DAA6D;YAC7D,MAAM,MAAM,GAAG,QAAQ,CAAC,OAAO,CAAC,MAAM,CACpC,CAAC,KAAK,EAAE,EAAE,CAAC,MAAM,IAAI,KAAK,IAAI,KAAK,CAAC,IAAI,KAAK,aAAa,CACvB,CAAC;YAEtC,SAAS,GAAG,CAAC,GAAG,SAAS,EAAE,GAAG,MAAM,CAAC,CAAC;YACtC,OAAO,GAAG,QAAQ,CAAC,QAAQ,CAAC;YAC5B,MAAM,GAAG,MAAA,QAAQ,CAAC,WAAW,mCAAI,SAAS,CAAC;SAC5C;QAED,OAAO,SAAS,CAAC;IACnB,CAAC;IAEO,KAAK,CAAC,gBAAgB,CAC5B,OAAe;;QAEf,IAAI,WAAW,GAA2B,EAAE,CAAC;QAC7C,IAAI,OAAO,GAAG,IAAI,CAAC;QACnB,IAAI,MAA0B,CAAC;QAE/B,OAAO,OAAO,EAAE;YACd,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,GAAG,EAAE,CAChD,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC;gBACxB,QAAQ,EAAE,OAAO;gBACjB,YAAY,EAAE,MAAM;aACrB,CAAC,CACH,CAAC;YAEF,WAAW,GAAG,CAAC,GAAG,WAAW,EAAE,GAAG,QAAQ,CAAC,OAAO,CAAC,CAAC;YACpD,OAAO,GAAG,QAAQ,CAAC,QAAQ,CAAC;YAC5B,MAAM,GAAG,MAAA,QAAQ,CAAC,WAAW,mCAAI,SAAS,CAAC;SAC5C;QAED,OAAO,WAAW,CAAC;IACrB,CAAC;IAEO,KAAK,CAAC,mBAAmB,CAC/B,MAAc;QAEd,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,GAAG,EAAE,CAChD,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,CAChD,CAAC;QAEF,OAAO,YAAY,IAAI,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE,CAAC;IAC7D,CAAC;IAEO,WAAW,CAAC,EAAU;QAC5B,OAAO,EAAE,CAAC,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;IAC9B,CAAC;IAEO,cAAc,CAAC,MAAc;;QACnC,MAAM,WAAW,GAAG,IAAI,GAAG,EAA6C,CAAC;QAEzE,KAAK,MAAM,CAAC,EAAE,EAAE,KAAK,CAAC,IAAI,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,EAAE;YAC/C,MAAM,QAAQ;YACZ,aAAa;YACb,CAAA,MAAA,KAAK,CAAC,MAAM,0CAAE,IAAI,MAAK,UAAU;gBAC/B,CAAC,CAAC,aAAa;oBACb,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,MAAM,CAAC,QAAQ,CAAC;gBACzC,CAAC,CAAC,aAAa;oBACb,CAAA,MAAA,KAAK,CAAC,MAAM,0CAAE,IAAI,MAAK,SAAS;wBAChC,CAAC,CAAC,aAAa;4BACb,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,MAAM,CAAC,OAAO,CAAC;wBACxC,CAAC,CAAC,SAAS,CAAC;YAElB,IAAI,QAAQ,EAAE;gBACZ,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE;oBAC9B,WAAW,CAAC,GAAG,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;iBAC/B;gBACD,WAAW,CAAC,GAAG,CAAC,QAAQ,CAAE,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;aACxC;SACF;QAED,MAAM,aAAa,GAAG,CACpB,QAAgB,EACkB,EAAE;YACpC,MAAM,QAAQ,GAAG,WAAW,CAAC,GAAG,CAAC,IAAI,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC,IAAI,EAAE,CAAC;YACnE,OAAO,QAAQ,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE;gBAC5B,KAAK,CAAC,QAAQ,GAAG,aAAa,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;gBACzC,OAAO,KAAK,CAAC;YACf,CAAC,CAAC,CAAC;QACL,CAAC,CAAC;QAEF,OAAO,aAAa,CAAC,MAAM,CAAC,CAAC;IAC/B,CAAC;CACF;AArSD,oCAqSC","sourcesContent":["import type { Client } from '@notionhq/client';\nimport {\n ListBlockChildrenResponseResult,\n PageObjectProperties,\n CommentResponseResults,\n FetcherOutput,\n ListBlockChildrenResponseResults,\n ExtendedFetcherOutput,\n ProcessorChainNode,\n ChainData,\n} from '../../types';\nimport { isMediaBlock, isPageRefBlock } from '../../utils/notion';\n\nexport interface BlockFetcherConfig {\n fetchPageProperties?: boolean;\n fetchComments?: boolean;\n maxRequestsPerSecond?: number;\n batchSize?: number;\n trackMediaBlocks?: boolean;\n trackPageRefBlocks?: boolean;\n}\n\ninterface QueueTask {\n type: 'fetch_properties' | 'fetch_comments' | 'fetch_blocks';\n id: string;\n parentId?: string;\n}\n\nexport class BlockFetcher implements ProcessorChainNode {\n next?: ProcessorChainNode;\n\n private queue: QueueTask[] = [];\n private blocks = new Map<string, ListBlockChildrenResponseResult>();\n private processedTasks = new Set<string>();\n private pageProperties?: PageObjectProperties;\n private rootComments: CommentResponseResults = [];\n private rootBlockId: string = '';\n private mediaBlocks: ListBlockChildrenResponseResult[] = [];\n private pageRefBlocks: ListBlockChildrenResponseResult[] = [];\n\n private rateLimitWindow = {\n requests: 0,\n startTime: Date.now(),\n };\n\n constructor(\n pageId: string,\n private client: Client,\n private config: BlockFetcherConfig = {\n fetchPageProperties: false,\n fetchComments: false,\n maxRequestsPerSecond: 3,\n batchSize: 3,\n },\n ) {\n const moduleType = 'BlockFetcher';\n this.config.maxRequestsPerSecond = config.maxRequestsPerSecond ?? 3;\n this.config.batchSize = config.batchSize ?? 3;\n }\n\n async process(data: ChainData): Promise<ChainData> {\n const blockTree = await this.getBlocks(data.pageId);\n\n const updatedData: ChainData = {\n ...data,\n blockTree,\n };\n\n return this.next ? this.next.process(updatedData) : updatedData;\n }\n\n async getBlocks(pageId: string): Promise<ExtendedFetcherOutput> {\n this.rootBlockId = pageId;\n this.queue = [];\n this.blocks.clear();\n this.processedTasks.clear();\n this.pageProperties = undefined;\n this.rootComments = [];\n\n // Initialize queue with root level tasks\n this.addTask({ type: 'fetch_blocks', id: pageId });\n\n if (this.config.fetchPageProperties) {\n this.addTask({ type: 'fetch_properties', id: pageId });\n }\n\n if (this.config.fetchComments) {\n this.addTask({ type: 'fetch_comments', id: pageId });\n }\n\n // Process queue until empty\n while (this.queue.length > 0) {\n const batch = this.queue.splice(0, this.config.batchSize);\n await Promise.all(batch.map((task) => this.processTask(task)));\n }\n\n const baseOutput: FetcherOutput = {\n properties: this.pageProperties || {},\n comments: this.rootComments,\n blocks: this.buildBlockTree(pageId),\n };\n\n // Only include special arrays if tracking was enabled\n if (this.config.trackMediaBlocks || this.config.trackPageRefBlocks) {\n return {\n ...baseOutput,\n ...(this.config.trackMediaBlocks && { mediaBlocks: this.mediaBlocks }),\n ...(this.config.trackPageRefBlocks && {\n pageRefBlocks: this.pageRefBlocks,\n }),\n };\n }\n\n return baseOutput;\n }\n\n private addTask(task: QueueTask): void {\n const taskId = `${task.type}-${task.id}`;\n if (!this.processedTasks.has(taskId)) {\n this.queue.push(task);\n }\n }\n\n private async processTask(task: QueueTask): Promise<void> {\n const taskId = `${task.type}-${task.id}`;\n if (this.processedTasks.has(taskId)) return;\n\n switch (task.type) {\n case 'fetch_blocks': {\n const blocks = await this.fetchBlockChildren(task.id);\n\n for (const block of blocks) {\n const storedBlock = {\n ...block,\n children: [],\n comments: [],\n };\n this.blocks.set(block.id, storedBlock);\n\n // Track special blocks if enabled (storing references)\n // these will be used by the next stage saving us from a\n // lot of recursive calls to idenitfy blocks\n if (this.config.trackMediaBlocks && isMediaBlock(storedBlock)) {\n this.mediaBlocks.push(storedBlock);\n }\n if (this.config.trackPageRefBlocks && isPageRefBlock(storedBlock)) {\n this.pageRefBlocks.push(storedBlock);\n }\n\n // If block has children, queue task to fetch them\n if ('has_children' in block && block.has_children) {\n this.addTask({\n type: 'fetch_blocks',\n id: block.id,\n parentId: task.id,\n });\n }\n\n // Queue comment fetching for each block if enabled\n // extra api call for comment fetching, no way to know otherwise\n if (this.config.fetchComments) {\n this.addTask({\n type: 'fetch_comments',\n id: block.id,\n });\n }\n }\n break;\n }\n\n case 'fetch_comments': {\n const comments = await this.fetchAllComments(task.id);\n if (task.id === this.rootBlockId) {\n // page level comments\n this.rootComments = comments;\n } else {\n const block = this.blocks.get(task.id);\n if (block) {\n block.comments = comments;\n }\n }\n break;\n }\n\n case 'fetch_properties': {\n const properties = await this.fetchPageProperties(task.id);\n this.pageProperties = properties;\n break;\n }\n }\n\n this.processedTasks.add(taskId);\n }\n\n private async rateLimitRequest<T>(request: () => Promise<T>): Promise<T> {\n const now = Date.now();\n const windowSize = 1000;\n\n // Reset window if it's expired\n if (now - this.rateLimitWindow.startTime >= windowSize) {\n this.rateLimitWindow = {\n requests: 0,\n startTime: now,\n };\n }\n\n // Wait if we've hit the rate limit\n if (this.rateLimitWindow.requests >= this.config.maxRequestsPerSecond!) {\n const waitTime = windowSize - (now - this.rateLimitWindow.startTime);\n await new Promise((resolve) => setTimeout(resolve, waitTime));\n this.rateLimitWindow = {\n requests: 0,\n startTime: Date.now(),\n };\n }\n\n this.rateLimitWindow.requests++;\n return request();\n }\n\n private async fetchBlockChildren(\n blockId: string,\n ): Promise<ListBlockChildrenResponseResults> {\n let allBlocks: ListBlockChildrenResponseResults = [];\n let hasMore = true;\n let cursor: string | undefined;\n\n while (hasMore) {\n const response = await this.rateLimitRequest(() =>\n this.client.blocks.children.list({\n block_id: blockId,\n start_cursor: cursor,\n }),\n );\n\n // Filter out unsupported blocks\n // should we leave it here or let user handle it in renderer?\n const blocks = response.results.filter(\n (block) => 'type' in block && block.type !== 'unsupported',\n ) as ListBlockChildrenResponseResults;\n\n allBlocks = [...allBlocks, ...blocks];\n hasMore = response.has_more;\n cursor = response.next_cursor ?? undefined;\n }\n\n return allBlocks;\n }\n\n private async fetchAllComments(\n blockId: string,\n ): Promise<CommentResponseResults> {\n let allComments: CommentResponseResults = [];\n let hasMore = true;\n let cursor: string | undefined;\n\n while (hasMore) {\n const response = await this.rateLimitRequest(() =>\n this.client.comments.list({\n block_id: blockId,\n start_cursor: cursor,\n }),\n );\n\n allComments = [...allComments, ...response.results];\n hasMore = response.has_more;\n cursor = response.next_cursor ?? undefined;\n }\n\n return allComments;\n }\n\n private async fetchPageProperties(\n pageId: string,\n ): Promise<PageObjectProperties> {\n const response = await this.rateLimitRequest(() =>\n this.client.pages.retrieve({ page_id: pageId }),\n );\n\n return 'properties' in response ? response.properties : {};\n }\n\n private normalizeId(id: string): string {\n return id.replace(/-/g, '');\n }\n\n private buildBlockTree(rootId: string): ListBlockChildrenResponseResults {\n const childrenMap = new Map<string, ListBlockChildrenResponseResult[]>();\n\n for (const [id, block] of this.blocks.entries()) {\n const parentId =\n // @ts-ignore\n block.parent?.type === 'block_id'\n ? // @ts-ignore\n this.normalizeId(block.parent.block_id)\n : // @ts-ignore\n block.parent?.type === 'page_id'\n ? // @ts-ignore\n this.normalizeId(block.parent.page_id)\n : undefined;\n\n if (parentId) {\n if (!childrenMap.has(parentId)) {\n childrenMap.set(parentId, []);\n }\n childrenMap.get(parentId)!.push(block);\n }\n }\n\n const buildChildren = (\n parentId: string,\n ): ListBlockChildrenResponseResults => {\n const children = childrenMap.get(this.normalizeId(parentId)) || [];\n return children.map((block) => {\n block.children = buildChildren(block.id);\n return block;\n });\n };\n\n return buildChildren(rootId);\n }\n}\n"]}
|
package/build/index.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { Client } from '@notionhq/client';
|
|
1
|
+
import type { Client } from '@notionhq/client';
|
|
2
2
|
import { BlockFetcherConfig } from './core/block-fetcher';
|
|
3
3
|
import { PageRefConfig } from './core/page-ref-handler';
|
|
4
4
|
import { NotionExporter, DownloadStrategyConfig, UploadStrategyConfig } from './types';
|
package/build/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAC/C,OAAO,EAAgB,kBAAkB,EAAE,MAAM,sBAAsB,CAAC;AAKxE,OAAO,EAAE,aAAa,EAAwB,MAAM,yBAAyB,CAAC;AAC9E,OAAO,EACL,cAAc,EAKd,sBAAsB,EACtB,oBAAoB,EAErB,MAAM,SAAS,CAAC;AAKjB,OAAO,EAAE,kBAAkB,EAAE,MAAM,iBAAiB,CAAC;AAkBrD;;;GAGG;AACH,qBAAa,eAAe;IAId,OAAO,CAAC,YAAY;IAHhC,OAAO,CAAC,MAAM,CAA6B;IAC3C,OAAO,CAAC,cAAc,CAAC,CAAqB;gBAExB,YAAY,EAAE,MAAM;IAgBxC;;;OAGG;IACH,gBAAgB,CAAC,MAAM,EAAE,OAAO,CAAC,kBAAkB,CAAC,GAAG,IAAI;IAa3D;;;OAGG;IACH,eAAe,CAAC,MAAM,EAAE,sBAAsB,GAAG,IAAI;IAYrD;;;OAGG;IACH,gBAAgB,CAAC,MAAM,EAAE,oBAAoB,GAAG,IAAI;IAYpD;;;OAGG;IACH,kBAAkB,CAAC,MAAM,CAAC,EAAE,aAAa,GAAG,IAAI;IAShD;;;OAGG;IACH,YAAY,CAAC,QAAQ,EAAE,kBAAkB,GAAG,IAAI;IAKhD;;;OAGG;IACH,YAAY,CACV,QAAQ,EAAE,cAAc,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC,GACzD,IAAI;IAUP;;OAEG;IACG,OAAO,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IA6B5C;;;OAGG;YACW,wBAAwB;IAiFtC;;OAEG;IACH,OAAO,CAAC,mBAAmB;IAc3B;;OAEG;IACH,OAAO,CAAC,WAAW;CAIpB"}
|
package/build/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";;;AACA,wDAAwE;AACxE,8CAA2C;AAC3C,wDAAoD;AACpD,uEAA4E;AAC5E,mEAAwE;AACxE,8DAA8E;AAC9E,mCASiB;AACjB,+DAGkC;AAElC,gDAAqD;AAiBrD;;;GAGG;AACH,MAAa,eAAe;IAI1B,YAAoB,YAAoB;QAApB,iBAAY,GAAZ,YAAY,CAAQ;QAHhC,WAAM,GAA0B,EAAE,CAAC;QAIzC,OAAO,CAAC,KAAK,CAAC,2DAA2D,CAAC,CAAC;QAC3E,IAAI,CAAC,MAAM,CAAC,kBAAkB,GAAG;YAC/B,mBAAmB,EAAE,IAAI;YACzB,aAAa,EAAE,KAAK;YACpB,oBAAoB,EAAE,CAAC;YACvB,SAAS,EAAE,CAAC;YACZ,gBAAgB,EAAE,KAAK;YACvB,kBAAkB,EAAE,KAAK;SAC1B,CAAC;QACF,OAAO,CAAC,KAAK,CACX,iDAAiD,EACjD,IAAI,CAAC,MAAM,CAAC,kBAAkB,CAC/B,CAAC;IACJ,CAAC;IAED;;;OAGG;IACH,gBAAgB,CAAC,MAAmC;QAClD,OAAO,CAAC,KAAK,CAAC,mDAAmD,EAAE,MAAM,CAAC,CAAC;QAC3E,IAAI,CAAC,MAAM,CAAC,kBAAkB,GAAG;YAC/B,GAAG,IAAI,CAAC,MAAM,CAAC,kBAAkB;YACjC,GAAG,MAAM;SACV,CAAC;QACF,OAAO,CAAC,KAAK,CACX,iDAAiD,EACjD,IAAI,CAAC,MAAM,CAAC,kBAAkB,CAC/B,CAAC;QACF,OAAO,IAAI,CAAC;IACd,CAAC;IAED;;;OAGG;IACH,eAAe,CAAC,MAA8B;QAC5C,OAAO,CAAC,KAAK,CACX,6DAA6D,EAC7D,MAAM,CACP,CAAC;QACF,IAAI,CAAC,MAAM,CAAC,WAAW,GAAG;YACxB,IAAI,EAAE,yBAAiB,CAAC,QAAQ;YAChC,MAAM;SACP,CAAC;QACF,OAAO,IAAI,CAAC;IACd,CAAC;IAED;;;OAGG;IACH,gBAAgB,CAAC,MAA4B;QAC3C,OAAO,CAAC,KAAK,CACX,2DAA2D,EAC3D,MAAM,CACP,CAAC;QACF,IAAI,CAAC,MAAM,CAAC,WAAW,GAAG;YACxB,IAAI,EAAE,yBAAiB,CAAC,MAAM;YAC9B,MAAM;SACP,CAAC;QACF,OAAO,IAAI,CAAC;IACd,CAAC;IAED;;;OAGG;IACH,kBAAkB,CAAC,MAAsB;QACvC,OAAO,CAAC,KAAK,CACX,6DAA6D,EAC7D,MAAM,IAAI,EAAE,CACb,CAAC;QACF,IAAI,CAAC,MAAM,CAAC,aAAa,GAAG,MAAM,IAAI,EAAE,CAAC;QACzC,OAAO,IAAI,CAAC;IACd,CAAC;IAED;;;OAGG;IACH,YAAY,CAAC,QAA4B;QACvC,IAAI,CAAC,MAAM,CAAC,QAAQ,GAAG,QAAQ,CAAC;QAChC,OAAO,IAAI,CAAC;IACd,CAAC;IAED;;;OAGG;IACH,YAAY,CACV,QAA0D;QAE1D,OAAO,CAAC,KAAK,CAAC,yCAAyC,CAAC,CAAC;QACzD,IAAI,CAAC,MAAM,CAAC,SAAS,GAAG,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC;QACxE,OAAO,CAAC,KAAK,CACX,mDAAmD,EACnD,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,MAAM,CAC7B,CAAC;QACF,OAAO,IAAI,CAAC;IACd,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,OAAO,CAAC,MAAc;QAC1B,OAAO,CAAC,KAAK,CAAC,iDAAiD,EAAE,MAAM,CAAC,CAAC;QACzE,IAAI;YACF,qDAAqD;YACrD,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE;gBACxB,OAAO,CAAC,KAAK,CAAC,gDAAgD,CAAC,CAAC;gBAChE,MAAM,IAAI,CAAC,wBAAwB,CAAC,MAAM,CAAC,CAAC;aAC7C;YAED,+CAA+C;YAC/C,OAAO,CAAC,KAAK,CAAC,+CAA+C,CAAC,CAAC;YAC/D,MAAM,SAAS,GAAc;gBAC3B,MAAM;gBACN,SAAS,EAAE,EAA2B;gBACtC,OAAO,EAAE,EAAE;aACZ,CAAC;YAEF,4BAA4B;YAC5B,OAAO,CAAC,KAAK,CAAC,8CAA8C,CAAC,CAAC;YAC9D,MAAM,IAAI,CAAC,cAAe,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;YAC9C,OAAO,CAAC,KAAK,CACX,2DAA2D,CAC5D,CAAC;SACH;QAAC,OAAO,KAAK,EAAE;YACd,OAAO,CAAC,KAAK,CAAC,4CAA4C,EAAE,KAAK,CAAC,CAAC;YACnE,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC;SACzB;IACH,CAAC;IAED;;;OAGG;IACK,KAAK,CAAC,wBAAwB,CAAC,MAAc;;QACnD,OAAO,CAAC,KAAK,CACX,0DAA0D,EAC1D,MAAM,CACP,CAAC;QAEF,mDAAmD;QACnD,IAAI,CAAC,MAAM,CAAC,kBAAkB,GAAG;YAC/B,GAAG,IAAI,CAAC,MAAM,CAAC,kBAAkB;YACjC,gBAAgB,EAAE,IAAI,CAAC,MAAM,CAAC,WAAW;gBACvC,CAAC,CAAC,IAAI;gBACN,CAAC,CAAC,MAAA,IAAI,CAAC,MAAM,CAAC,kBAAkB,0CAAE,gBAAgB;YACpD,kBAAkB,EAAE,IAAI,CAAC,MAAM,CAAC,aAAa;gBAC3C,CAAC,CAAC,IAAI;gBACN,CAAC,CAAC,MAAA,IAAI,CAAC,MAAM,CAAC,kBAAkB,0CAAE,kBAAkB;SACvD,CAAC;QACF,OAAO,CAAC,KAAK,CACX,wDAAwD,EACxD,IAAI,CAAC,MAAM,CAAC,kBAAkB,CAC/B,CAAC;QAEF,IAAI,IAAI,GAAG,IAAI,4BAAY,CACzB,MAAM,EACN,IAAI,CAAC,YAAY,EACjB,IAAI,CAAC,MAAM,CAAC,kBAAkB,CAC/B,CAAC;QACF,IAAI,OAAO,GAAuB,IAAI,CAAC;QAEvC,qDAAqD;QACrD,IAAI,IAAI,CAAC,MAAM,CAAC,WAAW,EAAE;YAC3B,OAAO,CAAC,KAAK,CAAC,gDAAgD,CAAC,CAAC;YAChE,MAAM,QAAQ,GAAG,IAAI,CAAC,mBAAmB,CAAC,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC;YACnE,MAAM,oBAAoB,GAAG,IAAI,uCAAoB,EAAE,CAAC;YACxD,MAAM,oBAAoB,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;YAE9C,MAAM,YAAY,GAAG,IAAI,4BAAY,CACnC,MAAM,EACN,EAAE,QAAQ,EAAE,EACZ,oBAAoB,CACrB,CAAC;YACF,OAAO,CAAC,IAAI,GAAG,YAAY,CAAC;YAC5B,OAAO,GAAG,YAAY,CAAC;SACxB;QAED,yCAAyC;QACzC,IAAI,IAAI,CAAC,MAAM,CAAC,aAAa,EAAE;YAC7B,OAAO,CAAC,KAAK,CAAC,wDAAwD,CAAC,CAAC;YACxE,MAAM,4BAA4B,GAAG,IAAI,+CAA4B,EAAE,CAAC;YACxE,MAAM,4BAA4B,CAAC,UAAU,EAAE,CAAC;YAEhD,MAAM,cAAc,GAAG,IAAI,uCAAoB,CAC7C,MAAM,EACN,IAAI,CAAC,MAAM,CAAC,aAAa,EACzB,4BAA4B,CAC7B,CAAC;YACF,OAAO,CAAC,IAAI,GAAG,cAAc,CAAC;YAC9B,OAAO,GAAG,cAAc,CAAC;SAC1B;QAED,2DAA2D;QAC3D,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,EAAE;YACzB,OAAO,CAAC,KAAK,CAAC,mDAAmD,CAAC,CAAC;YACnE,IAAI,CAAC,MAAM,CAAC,QAAQ,GAAG,IAAI,iBAAW,EAAE,CAAC,CAAC,sBAAsB;SACjE;QAED,OAAO,CAAC,KAAK,CAAC,4CAA4C,CAAC,CAAC;QAC5D,OAAO,CAAC,IAAI,GAAG,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC;QACpC,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC;QAE/B,gDAAgD;QAChD,IAAI,MAAA,IAAI,CAAC,MAAM,CAAC,SAAS,0CAAE,MAAM,EAAE;YACjC,OAAO,CAAC,KAAK,CAAC,4CAA4C,CAAC,CAAC;YAC5D,MAAM,YAAY,GAAG,IAAI,mBAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;YACzD,OAAO,CAAC,IAAI,GAAG,YAAY,CAAC;YAC5B,6DAA6D;SAC9D;QAED,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC;QAC3B,OAAO,CAAC,KAAK,CAAC,2DAA2D,CAAC,CAAC;IAC7E,CAAC;IAED;;OAEG;IACK,mBAAmB,CACzB,MAA4C;QAE5C,OAAO,CAAC,KAAK,CACX,oDAAoD,EACpD,MAAM,aAAN,MAAM,uBAAN,MAAM,CAAE,IAAI,CACb,CAAC;QACF,IAAI,CAAC,MAAM;YAAE,MAAM,IAAI,KAAK,CAAC,6CAA6C,CAAC,CAAC;QAE5E,OAAO,MAAM,CAAC,IAAI,KAAK,yBAAiB,CAAC,QAAQ;YAC/C,CAAC,CAAC,IAAI,2BAAgB,CAAC,MAAM,CAAC,MAAgC,CAAC;YAC/D,CAAC,CAAC,IAAI,uBAAc,CAAC,MAAM,CAAC,MAA8B,CAAC,CAAC;IAChE,CAAC;IAED;;OAEG;IACK,WAAW,CAAC,KAAc;QAChC,OAAO,CAAC,KAAK,CAAC,mCAAmC,EAAE,KAAK,CAAC,CAAC;QAC1D,MAAM,KAAK,CAAC;IACd,CAAC;CACF;AAxPD,0CAwPC","sourcesContent":["import { Client } from '@notionhq/client';\nimport { BlockFetcher, BlockFetcherConfig } from './core/block-fetcher';\nimport { Exporter } from './core/exporter';\nimport { MediaHandler } from './core/media-handler';\nimport { DownloadStrategy } from './core/media-handler/strategies/download';\nimport { UploadStrategy } from './core/media-handler/strategies/upload';\nimport { PageRefConfig, PageReferenceHandler } from './core/page-ref-handler';\nimport {\n NotionExporter,\n ProcessorChainNode,\n ChainData,\n ExtendedFetcherOutput,\n MediaStrategy,\n DownloadStrategyConfig,\n UploadStrategyConfig,\n MediaStrategyType,\n} from './types';\nimport {\n MediaManifestManager,\n PageReferenceManifestManager,\n} from './utils/manifest-manager';\nimport { BaseRendererPlugin } from './core/renderer';\nimport { MDXRenderer } from './plugins/renderer/mdx';\n\n/**\n * Configuration interface for NotionConverter that is built up\n * through the builder pattern methods\n */\ninterface NotionConverterConfig {\n mediaConfig?: {\n type: MediaStrategyType;\n config: DownloadStrategyConfig | UploadStrategyConfig;\n };\n pageRefConfig?: PageRefConfig;\n renderer?: BaseRendererPlugin;\n exporters?: Array<NotionExporter<any>>;\n blockFetcherConfig?: BlockFetcherConfig;\n}\n\n/**\n * Main class that orchestrates the conversion process using a chain of processors.\n * Uses builder pattern for configuration and manages the complete pipeline.\n */\nexport class NotionConverter {\n private config: NotionConverterConfig = {};\n private processorChain?: ProcessorChainNode; // points to head of the processor chain\n\n constructor(private notionClient: Client) {\n console.debug('[NotionConverter] Initializing with default configuration');\n this.config.blockFetcherConfig = {\n fetchPageProperties: true,\n fetchComments: false,\n maxRequestsPerSecond: 3, // Notion's default rate limit\n batchSize: 3,\n trackMediaBlocks: false,\n trackPageRefBlocks: false,\n };\n console.debug(\n '[NotionConverter] Default block fetcher config:',\n this.config.blockFetcherConfig,\n );\n }\n\n /**\n * Configures the block fetcher behavior.\n * Controls how blocks are fetched from Notion and what additional data is tracked.\n */\n configureFetcher(config: Partial<BlockFetcherConfig>): this {\n console.debug('[NotionConverter] Configuring block fetcher with:', config);\n this.config.blockFetcherConfig = {\n ...this.config.blockFetcherConfig,\n ...config,\n };\n console.debug(\n '[NotionConverter] Updated block fetcher config:',\n this.config.blockFetcherConfig,\n );\n return this;\n }\n\n /**\n * Configures the download strategy for media handling.\n * Files will be downloaded to local filesystem.\n */\n downloadMediaTo(config: DownloadStrategyConfig): this {\n console.debug(\n '[NotionConverter] Configuring download media strategy with:',\n config,\n );\n this.config.mediaConfig = {\n type: MediaStrategyType.DOWNLOAD,\n config,\n };\n return this;\n }\n\n /**\n * Configures the upload strategy for media handling.\n * Files will be uploaded to external storage.\n */\n uploadMediaUsing(config: UploadStrategyConfig): this {\n console.debug(\n '[NotionConverter] Configuring upload media strategy with:',\n config,\n );\n this.config.mediaConfig = {\n type: MediaStrategyType.UPLOAD,\n config,\n };\n return this;\n }\n\n /**\n * Configures page reference handling for transforming Notion URLs\n * into public-facing URLs.\n */\n withPageReferences(config?: PageRefConfig): this {\n console.debug(\n '[NotionConverter] Configuring page reference handling with:',\n config || {},\n );\n this.config.pageRefConfig = config || {};\n return this;\n }\n\n /**\n * Sets the renderer to use for converting blocks to the target format.\n * If not set, default markdown renderer is used.\n */\n withRenderer(renderer: BaseRendererPlugin): this {\n this.config.renderer = renderer;\n return this;\n }\n\n /**\n * Configures one or more exporters to handle the final output.\n * Exporters receive the complete ChainData for maximum flexibility.\n */\n withExporter(\n exporter: NotionExporter<any> | Array<NotionExporter<any>>,\n ): this {\n console.debug('[NotionConverter] Configuring exporters');\n this.config.exporters = Array.isArray(exporter) ? exporter : [exporter];\n console.debug(\n '[NotionConverter] Number of configured exporters:',\n this.config.exporters.length,\n );\n return this;\n }\n\n /**\n * Main conversion method that processes a Notion page through the chain.\n */\n async convert(pageId: string): Promise<void> {\n console.debug('[NotionConverter] Starting conversion for page:', pageId);\n try {\n // Initialize the processor chain if not already done\n if (!this.processorChain) {\n console.debug('[NotionConverter] Initializing processor chain');\n await this.initializeProcessorChain(pageId);\n }\n\n // Start the processing chain with initial data\n console.debug('[NotionConverter] Creating initial chain data');\n const chainData: ChainData = {\n pageId,\n blockTree: {} as ExtendedFetcherOutput,\n content: '',\n };\n\n // Process through the chain\n console.debug('[NotionConverter] Beginning chain processing');\n await this.processorChain!.process(chainData);\n console.debug(\n '[NotionConverter] Chain processing completed successfully',\n );\n } catch (error) {\n console.debug('[NotionConverter] Error during conversion:', error);\n this.handleError(error);\n }\n }\n\n /**\n * Initializes and links together all the processors based on configuration.\n * Sets up the complete processing chain in the correct order.\n */\n private async initializeProcessorChain(pageId: string): Promise<void> {\n console.debug(\n '[NotionConverter] Initializing processor chain for page:',\n pageId,\n );\n\n // Start with BlockFetcher as the head of our chain\n this.config.blockFetcherConfig = {\n ...this.config.blockFetcherConfig,\n trackMediaBlocks: this.config.mediaConfig\n ? true\n : this.config.blockFetcherConfig?.trackMediaBlocks,\n trackPageRefBlocks: this.config.pageRefConfig\n ? true\n : this.config.blockFetcherConfig?.trackPageRefBlocks,\n };\n console.debug(\n '[NotionConverter] Creating BlockFetcher with config \\n',\n this.config.blockFetcherConfig,\n );\n\n let head = new BlockFetcher(\n pageId,\n this.notionClient,\n this.config.blockFetcherConfig,\n );\n let current: ProcessorChainNode = head;\n\n // Add MediaHandler if media processing is configured\n if (this.config.mediaConfig) {\n console.debug('[NotionConverter] Adding MediaHandler to chain');\n const strategy = this.createMediaStrategy(this.config.mediaConfig);\n const mediaManifestManager = new MediaManifestManager();\n await mediaManifestManager.initialize(pageId);\n\n const mediaHandler = new MediaHandler(\n pageId,\n { strategy },\n mediaManifestManager,\n );\n current.next = mediaHandler;\n current = mediaHandler;\n }\n\n // Add PageReferenceHandler if configured\n if (this.config.pageRefConfig) {\n console.debug('[NotionConverter] Adding PageReferenceHandler to chain');\n const pageReferenceManifestManager = new PageReferenceManifestManager();\n await pageReferenceManifestManager.initialize();\n\n const pageRefHandler = new PageReferenceHandler(\n pageId,\n this.config.pageRefConfig,\n pageReferenceManifestManager,\n );\n current.next = pageRefHandler;\n current = pageRefHandler;\n }\n\n // Add renderer node if configured else fallback to default\n if (!this.config.renderer) {\n console.debug('[NotionConverter] Using default markdown renderer');\n this.config.renderer = new MDXRenderer(); // without frontmatter\n }\n\n console.debug('[NotionConverter] Adding renderer to chain');\n current.next = this.config.renderer;\n current = this.config.renderer;\n\n // Add exporter node if exporters are configured\n if (this.config.exporters?.length) {\n console.debug('[NotionConverter] Adding Exporter to chain');\n const exporterNode = new Exporter(this.config.exporters);\n current.next = exporterNode;\n // No need to update current since exporter is last as of now\n }\n\n this.processorChain = head;\n console.debug('[NotionConverter] Processor chain initialization complete');\n }\n\n /**\n * Creates appropriate media strategy based on configuration\n */\n private createMediaStrategy(\n config: NotionConverterConfig['mediaConfig'],\n ): MediaStrategy {\n console.debug(\n '[NotionConverter] Creating media strategy of type:',\n config?.type,\n );\n if (!config) throw new Error('Media config is required to create strategy');\n\n return config.type === MediaStrategyType.DOWNLOAD\n ? new DownloadStrategy(config.config as DownloadStrategyConfig)\n : new UploadStrategy(config.config as UploadStrategyConfig);\n }\n\n /**\n * Handles errors that occur during the conversion process\n */\n private handleError(error: unknown): never {\n console.debug('[NotionConverter] Handling error:', error);\n throw error;\n }\n}\n"]}
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";;;AACA,wDAAwE;AACxE,8CAA2C;AAC3C,wDAAoD;AACpD,uEAA4E;AAC5E,mEAAwE;AACxE,8DAA8E;AAC9E,mCASiB;AACjB,+DAGkC;AAElC,gDAAqD;AAiBrD;;;GAGG;AACH,MAAa,eAAe;IAI1B,YAAoB,YAAoB;QAApB,iBAAY,GAAZ,YAAY,CAAQ;QAHhC,WAAM,GAA0B,EAAE,CAAC;QAIzC,OAAO,CAAC,KAAK,CAAC,2DAA2D,CAAC,CAAC;QAC3E,IAAI,CAAC,MAAM,CAAC,kBAAkB,GAAG;YAC/B,mBAAmB,EAAE,IAAI;YACzB,aAAa,EAAE,KAAK;YACpB,oBAAoB,EAAE,CAAC;YACvB,SAAS,EAAE,CAAC;YACZ,gBAAgB,EAAE,KAAK;YACvB,kBAAkB,EAAE,KAAK;SAC1B,CAAC;QACF,OAAO,CAAC,KAAK,CACX,iDAAiD,EACjD,IAAI,CAAC,MAAM,CAAC,kBAAkB,CAC/B,CAAC;IACJ,CAAC;IAED;;;OAGG;IACH,gBAAgB,CAAC,MAAmC;QAClD,OAAO,CAAC,KAAK,CAAC,mDAAmD,EAAE,MAAM,CAAC,CAAC;QAC3E,IAAI,CAAC,MAAM,CAAC,kBAAkB,GAAG;YAC/B,GAAG,IAAI,CAAC,MAAM,CAAC,kBAAkB;YACjC,GAAG,MAAM;SACV,CAAC;QACF,OAAO,CAAC,KAAK,CACX,iDAAiD,EACjD,IAAI,CAAC,MAAM,CAAC,kBAAkB,CAC/B,CAAC;QACF,OAAO,IAAI,CAAC;IACd,CAAC;IAED;;;OAGG;IACH,eAAe,CAAC,MAA8B;QAC5C,OAAO,CAAC,KAAK,CACX,6DAA6D,EAC7D,MAAM,CACP,CAAC;QACF,IAAI,CAAC,MAAM,CAAC,WAAW,GAAG;YACxB,IAAI,EAAE,yBAAiB,CAAC,QAAQ;YAChC,MAAM;SACP,CAAC;QACF,OAAO,IAAI,CAAC;IACd,CAAC;IAED;;;OAGG;IACH,gBAAgB,CAAC,MAA4B;QAC3C,OAAO,CAAC,KAAK,CACX,2DAA2D,EAC3D,MAAM,CACP,CAAC;QACF,IAAI,CAAC,MAAM,CAAC,WAAW,GAAG;YACxB,IAAI,EAAE,yBAAiB,CAAC,MAAM;YAC9B,MAAM;SACP,CAAC;QACF,OAAO,IAAI,CAAC;IACd,CAAC;IAED;;;OAGG;IACH,kBAAkB,CAAC,MAAsB;QACvC,OAAO,CAAC,KAAK,CACX,6DAA6D,EAC7D,MAAM,IAAI,EAAE,CACb,CAAC;QACF,IAAI,CAAC,MAAM,CAAC,aAAa,GAAG,MAAM,IAAI,EAAE,CAAC;QACzC,OAAO,IAAI,CAAC;IACd,CAAC;IAED;;;OAGG;IACH,YAAY,CAAC,QAA4B;QACvC,IAAI,CAAC,MAAM,CAAC,QAAQ,GAAG,QAAQ,CAAC;QAChC,OAAO,IAAI,CAAC;IACd,CAAC;IAED;;;OAGG;IACH,YAAY,CACV,QAA0D;QAE1D,OAAO,CAAC,KAAK,CAAC,yCAAyC,CAAC,CAAC;QACzD,IAAI,CAAC,MAAM,CAAC,SAAS,GAAG,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC;QACxE,OAAO,CAAC,KAAK,CACX,mDAAmD,EACnD,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,MAAM,CAC7B,CAAC;QACF,OAAO,IAAI,CAAC;IACd,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,OAAO,CAAC,MAAc;QAC1B,OAAO,CAAC,KAAK,CAAC,iDAAiD,EAAE,MAAM,CAAC,CAAC;QACzE,IAAI;YACF,qDAAqD;YACrD,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE;gBACxB,OAAO,CAAC,KAAK,CAAC,gDAAgD,CAAC,CAAC;gBAChE,MAAM,IAAI,CAAC,wBAAwB,CAAC,MAAM,CAAC,CAAC;aAC7C;YAED,+CAA+C;YAC/C,OAAO,CAAC,KAAK,CAAC,+CAA+C,CAAC,CAAC;YAC/D,MAAM,SAAS,GAAc;gBAC3B,MAAM;gBACN,SAAS,EAAE,EAA2B;gBACtC,OAAO,EAAE,EAAE;aACZ,CAAC;YAEF,4BAA4B;YAC5B,OAAO,CAAC,KAAK,CAAC,8CAA8C,CAAC,CAAC;YAC9D,MAAM,IAAI,CAAC,cAAe,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;YAC9C,OAAO,CAAC,KAAK,CACX,2DAA2D,CAC5D,CAAC;SACH;QAAC,OAAO,KAAK,EAAE;YACd,OAAO,CAAC,KAAK,CAAC,4CAA4C,EAAE,KAAK,CAAC,CAAC;YACnE,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC;SACzB;IACH,CAAC;IAED;;;OAGG;IACK,KAAK,CAAC,wBAAwB,CAAC,MAAc;;QACnD,OAAO,CAAC,KAAK,CACX,0DAA0D,EAC1D,MAAM,CACP,CAAC;QAEF,mDAAmD;QACnD,IAAI,CAAC,MAAM,CAAC,kBAAkB,GAAG;YAC/B,GAAG,IAAI,CAAC,MAAM,CAAC,kBAAkB;YACjC,gBAAgB,EAAE,IAAI,CAAC,MAAM,CAAC,WAAW;gBACvC,CAAC,CAAC,IAAI;gBACN,CAAC,CAAC,MAAA,IAAI,CAAC,MAAM,CAAC,kBAAkB,0CAAE,gBAAgB;YACpD,kBAAkB,EAAE,IAAI,CAAC,MAAM,CAAC,aAAa;gBAC3C,CAAC,CAAC,IAAI;gBACN,CAAC,CAAC,MAAA,IAAI,CAAC,MAAM,CAAC,kBAAkB,0CAAE,kBAAkB;SACvD,CAAC;QACF,OAAO,CAAC,KAAK,CACX,wDAAwD,EACxD,IAAI,CAAC,MAAM,CAAC,kBAAkB,CAC/B,CAAC;QAEF,IAAI,IAAI,GAAG,IAAI,4BAAY,CACzB,MAAM,EACN,IAAI,CAAC,YAAY,EACjB,IAAI,CAAC,MAAM,CAAC,kBAAkB,CAC/B,CAAC;QACF,IAAI,OAAO,GAAuB,IAAI,CAAC;QAEvC,qDAAqD;QACrD,IAAI,IAAI,CAAC,MAAM,CAAC,WAAW,EAAE;YAC3B,OAAO,CAAC,KAAK,CAAC,gDAAgD,CAAC,CAAC;YAChE,MAAM,QAAQ,GAAG,IAAI,CAAC,mBAAmB,CAAC,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC;YACnE,MAAM,oBAAoB,GAAG,IAAI,uCAAoB,EAAE,CAAC;YACxD,MAAM,oBAAoB,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;YAE9C,MAAM,YAAY,GAAG,IAAI,4BAAY,CACnC,MAAM,EACN,EAAE,QAAQ,EAAE,EACZ,oBAAoB,CACrB,CAAC;YACF,OAAO,CAAC,IAAI,GAAG,YAAY,CAAC;YAC5B,OAAO,GAAG,YAAY,CAAC;SACxB;QAED,yCAAyC;QACzC,IAAI,IAAI,CAAC,MAAM,CAAC,aAAa,EAAE;YAC7B,OAAO,CAAC,KAAK,CAAC,wDAAwD,CAAC,CAAC;YACxE,MAAM,4BAA4B,GAAG,IAAI,+CAA4B,EAAE,CAAC;YACxE,MAAM,4BAA4B,CAAC,UAAU,EAAE,CAAC;YAEhD,MAAM,cAAc,GAAG,IAAI,uCAAoB,CAC7C,MAAM,EACN,IAAI,CAAC,MAAM,CAAC,aAAa,EACzB,4BAA4B,CAC7B,CAAC;YACF,OAAO,CAAC,IAAI,GAAG,cAAc,CAAC;YAC9B,OAAO,GAAG,cAAc,CAAC;SAC1B;QAED,2DAA2D;QAC3D,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,EAAE;YACzB,OAAO,CAAC,KAAK,CAAC,mDAAmD,CAAC,CAAC;YACnE,IAAI,CAAC,MAAM,CAAC,QAAQ,GAAG,IAAI,iBAAW,EAAE,CAAC,CAAC,sBAAsB;SACjE;QAED,OAAO,CAAC,KAAK,CAAC,4CAA4C,CAAC,CAAC;QAC5D,OAAO,CAAC,IAAI,GAAG,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC;QACpC,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC;QAE/B,gDAAgD;QAChD,IAAI,MAAA,IAAI,CAAC,MAAM,CAAC,SAAS,0CAAE,MAAM,EAAE;YACjC,OAAO,CAAC,KAAK,CAAC,4CAA4C,CAAC,CAAC;YAC5D,MAAM,YAAY,GAAG,IAAI,mBAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;YACzD,OAAO,CAAC,IAAI,GAAG,YAAY,CAAC;YAC5B,6DAA6D;SAC9D;QAED,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC;QAC3B,OAAO,CAAC,KAAK,CAAC,2DAA2D,CAAC,CAAC;IAC7E,CAAC;IAED;;OAEG;IACK,mBAAmB,CACzB,MAA4C;QAE5C,OAAO,CAAC,KAAK,CACX,oDAAoD,EACpD,MAAM,aAAN,MAAM,uBAAN,MAAM,CAAE,IAAI,CACb,CAAC;QACF,IAAI,CAAC,MAAM;YAAE,MAAM,IAAI,KAAK,CAAC,6CAA6C,CAAC,CAAC;QAE5E,OAAO,MAAM,CAAC,IAAI,KAAK,yBAAiB,CAAC,QAAQ;YAC/C,CAAC,CAAC,IAAI,2BAAgB,CAAC,MAAM,CAAC,MAAgC,CAAC;YAC/D,CAAC,CAAC,IAAI,uBAAc,CAAC,MAAM,CAAC,MAA8B,CAAC,CAAC;IAChE,CAAC;IAED;;OAEG;IACK,WAAW,CAAC,KAAc;QAChC,OAAO,CAAC,KAAK,CAAC,mCAAmC,EAAE,KAAK,CAAC,CAAC;QAC1D,MAAM,KAAK,CAAC;IACd,CAAC;CACF;AAxPD,0CAwPC","sourcesContent":["import type { Client } from '@notionhq/client';\nimport { BlockFetcher, BlockFetcherConfig } from './core/block-fetcher';\nimport { Exporter } from './core/exporter';\nimport { MediaHandler } from './core/media-handler';\nimport { DownloadStrategy } from './core/media-handler/strategies/download';\nimport { UploadStrategy } from './core/media-handler/strategies/upload';\nimport { PageRefConfig, PageReferenceHandler } from './core/page-ref-handler';\nimport {\n NotionExporter,\n ProcessorChainNode,\n ChainData,\n ExtendedFetcherOutput,\n MediaStrategy,\n DownloadStrategyConfig,\n UploadStrategyConfig,\n MediaStrategyType,\n} from './types';\nimport {\n MediaManifestManager,\n PageReferenceManifestManager,\n} from './utils/manifest-manager';\nimport { BaseRendererPlugin } from './core/renderer';\nimport { MDXRenderer } from './plugins/renderer/mdx';\n\n/**\n * Configuration interface for NotionConverter that is built up\n * through the builder pattern methods\n */\ninterface NotionConverterConfig {\n mediaConfig?: {\n type: MediaStrategyType;\n config: DownloadStrategyConfig | UploadStrategyConfig;\n };\n pageRefConfig?: PageRefConfig;\n renderer?: BaseRendererPlugin;\n exporters?: Array<NotionExporter<any>>;\n blockFetcherConfig?: BlockFetcherConfig;\n}\n\n/**\n * Main class that orchestrates the conversion process using a chain of processors.\n * Uses builder pattern for configuration and manages the complete pipeline.\n */\nexport class NotionConverter {\n private config: NotionConverterConfig = {};\n private processorChain?: ProcessorChainNode; // points to head of the processor chain\n\n constructor(private notionClient: Client) {\n console.debug('[NotionConverter] Initializing with default configuration');\n this.config.blockFetcherConfig = {\n fetchPageProperties: true,\n fetchComments: false,\n maxRequestsPerSecond: 3, // Notion's default rate limit\n batchSize: 3,\n trackMediaBlocks: false,\n trackPageRefBlocks: false,\n };\n console.debug(\n '[NotionConverter] Default block fetcher config:',\n this.config.blockFetcherConfig,\n );\n }\n\n /**\n * Configures the block fetcher behavior.\n * Controls how blocks are fetched from Notion and what additional data is tracked.\n */\n configureFetcher(config: Partial<BlockFetcherConfig>): this {\n console.debug('[NotionConverter] Configuring block fetcher with:', config);\n this.config.blockFetcherConfig = {\n ...this.config.blockFetcherConfig,\n ...config,\n };\n console.debug(\n '[NotionConverter] Updated block fetcher config:',\n this.config.blockFetcherConfig,\n );\n return this;\n }\n\n /**\n * Configures the download strategy for media handling.\n * Files will be downloaded to local filesystem.\n */\n downloadMediaTo(config: DownloadStrategyConfig): this {\n console.debug(\n '[NotionConverter] Configuring download media strategy with:',\n config,\n );\n this.config.mediaConfig = {\n type: MediaStrategyType.DOWNLOAD,\n config,\n };\n return this;\n }\n\n /**\n * Configures the upload strategy for media handling.\n * Files will be uploaded to external storage.\n */\n uploadMediaUsing(config: UploadStrategyConfig): this {\n console.debug(\n '[NotionConverter] Configuring upload media strategy with:',\n config,\n );\n this.config.mediaConfig = {\n type: MediaStrategyType.UPLOAD,\n config,\n };\n return this;\n }\n\n /**\n * Configures page reference handling for transforming Notion URLs\n * into public-facing URLs.\n */\n withPageReferences(config?: PageRefConfig): this {\n console.debug(\n '[NotionConverter] Configuring page reference handling with:',\n config || {},\n );\n this.config.pageRefConfig = config || {};\n return this;\n }\n\n /**\n * Sets the renderer to use for converting blocks to the target format.\n * If not set, default markdown renderer is used.\n */\n withRenderer(renderer: BaseRendererPlugin): this {\n this.config.renderer = renderer;\n return this;\n }\n\n /**\n * Configures one or more exporters to handle the final output.\n * Exporters receive the complete ChainData for maximum flexibility.\n */\n withExporter(\n exporter: NotionExporter<any> | Array<NotionExporter<any>>,\n ): this {\n console.debug('[NotionConverter] Configuring exporters');\n this.config.exporters = Array.isArray(exporter) ? exporter : [exporter];\n console.debug(\n '[NotionConverter] Number of configured exporters:',\n this.config.exporters.length,\n );\n return this;\n }\n\n /**\n * Main conversion method that processes a Notion page through the chain.\n */\n async convert(pageId: string): Promise<void> {\n console.debug('[NotionConverter] Starting conversion for page:', pageId);\n try {\n // Initialize the processor chain if not already done\n if (!this.processorChain) {\n console.debug('[NotionConverter] Initializing processor chain');\n await this.initializeProcessorChain(pageId);\n }\n\n // Start the processing chain with initial data\n console.debug('[NotionConverter] Creating initial chain data');\n const chainData: ChainData = {\n pageId,\n blockTree: {} as ExtendedFetcherOutput,\n content: '',\n };\n\n // Process through the chain\n console.debug('[NotionConverter] Beginning chain processing');\n await this.processorChain!.process(chainData);\n console.debug(\n '[NotionConverter] Chain processing completed successfully',\n );\n } catch (error) {\n console.debug('[NotionConverter] Error during conversion:', error);\n this.handleError(error);\n }\n }\n\n /**\n * Initializes and links together all the processors based on configuration.\n * Sets up the complete processing chain in the correct order.\n */\n private async initializeProcessorChain(pageId: string): Promise<void> {\n console.debug(\n '[NotionConverter] Initializing processor chain for page:',\n pageId,\n );\n\n // Start with BlockFetcher as the head of our chain\n this.config.blockFetcherConfig = {\n ...this.config.blockFetcherConfig,\n trackMediaBlocks: this.config.mediaConfig\n ? true\n : this.config.blockFetcherConfig?.trackMediaBlocks,\n trackPageRefBlocks: this.config.pageRefConfig\n ? true\n : this.config.blockFetcherConfig?.trackPageRefBlocks,\n };\n console.debug(\n '[NotionConverter] Creating BlockFetcher with config \\n',\n this.config.blockFetcherConfig,\n );\n\n let head = new BlockFetcher(\n pageId,\n this.notionClient,\n this.config.blockFetcherConfig,\n );\n let current: ProcessorChainNode = head;\n\n // Add MediaHandler if media processing is configured\n if (this.config.mediaConfig) {\n console.debug('[NotionConverter] Adding MediaHandler to chain');\n const strategy = this.createMediaStrategy(this.config.mediaConfig);\n const mediaManifestManager = new MediaManifestManager();\n await mediaManifestManager.initialize(pageId);\n\n const mediaHandler = new MediaHandler(\n pageId,\n { strategy },\n mediaManifestManager,\n );\n current.next = mediaHandler;\n current = mediaHandler;\n }\n\n // Add PageReferenceHandler if configured\n if (this.config.pageRefConfig) {\n console.debug('[NotionConverter] Adding PageReferenceHandler to chain');\n const pageReferenceManifestManager = new PageReferenceManifestManager();\n await pageReferenceManifestManager.initialize();\n\n const pageRefHandler = new PageReferenceHandler(\n pageId,\n this.config.pageRefConfig,\n pageReferenceManifestManager,\n );\n current.next = pageRefHandler;\n current = pageRefHandler;\n }\n\n // Add renderer node if configured else fallback to default\n if (!this.config.renderer) {\n console.debug('[NotionConverter] Using default markdown renderer');\n this.config.renderer = new MDXRenderer(); // without frontmatter\n }\n\n console.debug('[NotionConverter] Adding renderer to chain');\n current.next = this.config.renderer;\n current = this.config.renderer;\n\n // Add exporter node if exporters are configured\n if (this.config.exporters?.length) {\n console.debug('[NotionConverter] Adding Exporter to chain');\n const exporterNode = new Exporter(this.config.exporters);\n current.next = exporterNode;\n // No need to update current since exporter is last as of now\n }\n\n this.processorChain = head;\n console.debug('[NotionConverter] Processor chain initialization complete');\n }\n\n /**\n * Creates appropriate media strategy based on configuration\n */\n private createMediaStrategy(\n config: NotionConverterConfig['mediaConfig'],\n ): MediaStrategy {\n console.debug(\n '[NotionConverter] Creating media strategy of type:',\n config?.type,\n );\n if (!config) throw new Error('Media config is required to create strategy');\n\n return config.type === MediaStrategyType.DOWNLOAD\n ? new DownloadStrategy(config.config as DownloadStrategyConfig)\n : new UploadStrategy(config.config as UploadStrategyConfig);\n }\n\n /**\n * Handles errors that occur during the conversion process\n */\n private handleError(error: unknown): never {\n console.debug('[NotionConverter] Handling error:', error);\n throw error;\n }\n}\n"]}
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
import { NotionExporter, ChainData } from '../../../types';
|
|
2
|
+
/**
|
|
3
|
+
* Configuration for the FileSystemExporter
|
|
4
|
+
*/
|
|
5
|
+
export interface DefaultExporterConfig {
|
|
6
|
+
outputType: 'file' | 'stdout' | 'buffer';
|
|
7
|
+
outputPath?: string;
|
|
8
|
+
buffer?: {
|
|
9
|
+
[key: string]: string;
|
|
10
|
+
};
|
|
11
|
+
}
|
|
12
|
+
/**
|
|
13
|
+
* Default exporter that handles saving content to the filesystem,
|
|
14
|
+
* printing to stdout, or storing in a buffer
|
|
15
|
+
*/
|
|
16
|
+
export declare class DefaultExporter implements NotionExporter<DefaultExporterConfig> {
|
|
17
|
+
private config;
|
|
18
|
+
constructor(config: DefaultExporterConfig);
|
|
19
|
+
/**
|
|
20
|
+
* Export the conversion result according to configuration
|
|
21
|
+
*/
|
|
22
|
+
export(data: ChainData): Promise<void>;
|
|
23
|
+
/**
|
|
24
|
+
* Export content to a file
|
|
25
|
+
*/
|
|
26
|
+
private exportToFile;
|
|
27
|
+
/**
|
|
28
|
+
* Export content to stdout
|
|
29
|
+
*/
|
|
30
|
+
private exportToStdout;
|
|
31
|
+
/**
|
|
32
|
+
* Export content to a buffer object
|
|
33
|
+
*/
|
|
34
|
+
private exportToBuffer;
|
|
35
|
+
/**
|
|
36
|
+
* Validate the provided configuration
|
|
37
|
+
*/
|
|
38
|
+
private validateConfig;
|
|
39
|
+
}
|
|
40
|
+
//# sourceMappingURL=index.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/plugins/exporter/DefaultExporter/index.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,cAAc,EAAE,SAAS,EAAiB,MAAM,gBAAgB,CAAC;AAE1E;;GAEG;AACH,MAAM,WAAW,qBAAqB;IAEpC,UAAU,EAAE,MAAM,GAAG,QAAQ,GAAG,QAAQ,CAAC;IAGzC,UAAU,CAAC,EAAE,MAAM,CAAC;IAGpB,MAAM,CAAC,EAAE;QAAE,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAAA;KAAE,CAAC;CACpC;AAED;;;GAGG;AACH,qBAAa,eAAgB,YAAW,cAAc,CAAC,qBAAqB,CAAC;IAC/D,OAAO,CAAC,MAAM;gBAAN,MAAM,EAAE,qBAAqB;IAIjD;;OAEG;IACU,MAAM,CAAC,IAAI,EAAE,SAAS,GAAG,OAAO,CAAC,IAAI,CAAC;IA2BnD;;OAEG;YACW,YAAY;IAoB1B;;OAEG;IACH,OAAO,CAAC,cAAc;IAItB;;OAEG;IACH,OAAO,CAAC,cAAc;IAQtB;;OAEG;IACH,OAAO,CAAC,cAAc;CAavB"}
|
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
+
};
|
|
5
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
+
exports.DefaultExporter = void 0;
|
|
7
|
+
const promises_1 = __importDefault(require("fs/promises"));
|
|
8
|
+
const path_1 = __importDefault(require("path"));
|
|
9
|
+
const types_1 = require("../../../types");
|
|
10
|
+
/**
|
|
11
|
+
* Default exporter that handles saving content to the filesystem,
|
|
12
|
+
* printing to stdout, or storing in a buffer
|
|
13
|
+
*/
|
|
14
|
+
class DefaultExporter {
|
|
15
|
+
constructor(config) {
|
|
16
|
+
this.config = config;
|
|
17
|
+
this.validateConfig();
|
|
18
|
+
}
|
|
19
|
+
/**
|
|
20
|
+
* Export the conversion result according to configuration
|
|
21
|
+
*/
|
|
22
|
+
async export(data) {
|
|
23
|
+
try {
|
|
24
|
+
const { pageId, content } = data;
|
|
25
|
+
switch (this.config.outputType) {
|
|
26
|
+
case 'file':
|
|
27
|
+
await this.exportToFile(pageId, content);
|
|
28
|
+
break;
|
|
29
|
+
case 'stdout':
|
|
30
|
+
this.exportToStdout(content);
|
|
31
|
+
break;
|
|
32
|
+
case 'buffer':
|
|
33
|
+
this.exportToBuffer(pageId, content);
|
|
34
|
+
break;
|
|
35
|
+
}
|
|
36
|
+
}
|
|
37
|
+
catch (error) {
|
|
38
|
+
throw new types_1.ExporterError(`Export failed: ${error instanceof Error ? error.message : String(error)}`, data.pageId, 'export', error);
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
/**
|
|
42
|
+
* Export content to a file
|
|
43
|
+
*/
|
|
44
|
+
async exportToFile(pageId, content) {
|
|
45
|
+
if (!this.config.outputPath) {
|
|
46
|
+
throw new Error('outputPath is required for file output type');
|
|
47
|
+
}
|
|
48
|
+
// Get the directory path from the full file path
|
|
49
|
+
const dirPath = path_1.default.dirname(this.config.outputPath);
|
|
50
|
+
// Create all parent directories if they don't exist
|
|
51
|
+
await promises_1.default.mkdir(dirPath, { recursive: true });
|
|
52
|
+
try {
|
|
53
|
+
await promises_1.default.writeFile(this.config.outputPath, content, 'utf-8');
|
|
54
|
+
}
|
|
55
|
+
catch (error) {
|
|
56
|
+
throw new Error(`Failed to write file: ${error instanceof Error ? error.message : String(error)}`);
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
/**
|
|
60
|
+
* Export content to stdout
|
|
61
|
+
*/
|
|
62
|
+
exportToStdout(content) {
|
|
63
|
+
console.log(content);
|
|
64
|
+
}
|
|
65
|
+
/**
|
|
66
|
+
* Export content to a buffer object
|
|
67
|
+
*/
|
|
68
|
+
exportToBuffer(pageId, content) {
|
|
69
|
+
if (!this.config.buffer) {
|
|
70
|
+
throw new Error('buffer is required for buffer output type');
|
|
71
|
+
}
|
|
72
|
+
this.config.buffer[pageId] = content;
|
|
73
|
+
}
|
|
74
|
+
/**
|
|
75
|
+
* Validate the provided configuration
|
|
76
|
+
*/
|
|
77
|
+
validateConfig() {
|
|
78
|
+
if (!this.config.outputType) {
|
|
79
|
+
throw new Error('outputType is required');
|
|
80
|
+
}
|
|
81
|
+
if (this.config.outputType === 'file' && !this.config.outputPath) {
|
|
82
|
+
throw new Error('outputPath is required for file output type');
|
|
83
|
+
}
|
|
84
|
+
if (this.config.outputType === 'buffer' && !this.config.buffer) {
|
|
85
|
+
throw new Error('buffer is required for buffer output type');
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
exports.DefaultExporter = DefaultExporter;
|
|
90
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../../src/plugins/exporter/DefaultExporter/index.ts"],"names":[],"mappings":";;;;;;AAAA,2DAA6B;AAC7B,gDAAwB;AACxB,0CAA0E;AAgB1E;;;GAGG;AACH,MAAa,eAAe;IAC1B,YAAoB,MAA6B;QAA7B,WAAM,GAAN,MAAM,CAAuB;QAC/C,IAAI,CAAC,cAAc,EAAE,CAAC;IACxB,CAAC;IAED;;OAEG;IACI,KAAK,CAAC,MAAM,CAAC,IAAe;QACjC,IAAI;YACF,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,GAAG,IAAI,CAAC;YAEjC,QAAQ,IAAI,CAAC,MAAM,CAAC,UAAU,EAAE;gBAC9B,KAAK,MAAM;oBACT,MAAM,IAAI,CAAC,YAAY,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;oBACzC,MAAM;gBAER,KAAK,QAAQ;oBACX,IAAI,CAAC,cAAc,CAAC,OAAO,CAAC,CAAC;oBAC7B,MAAM;gBAER,KAAK,QAAQ;oBACX,IAAI,CAAC,cAAc,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;oBACrC,MAAM;aACT;SACF;QAAC,OAAO,KAAK,EAAE;YACd,MAAM,IAAI,qBAAa,CACrB,kBAAkB,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,EAC1E,IAAI,CAAC,MAAM,EACX,QAAQ,EACR,KAAK,CACN,CAAC;SACH;IACH,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,YAAY,CAAC,MAAc,EAAE,OAAe;QACxD,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,UAAU,EAAE;YAC3B,MAAM,IAAI,KAAK,CAAC,6CAA6C,CAAC,CAAC;SAChE;QAED,iDAAiD;QACjD,MAAM,OAAO,GAAG,cAAI,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;QAErD,oDAAoD;QACpD,MAAM,kBAAE,CAAC,KAAK,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAE7C,IAAI;YACF,MAAM,kBAAE,CAAC,SAAS,CAAC,IAAI,CAAC,MAAM,CAAC,UAAU,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC;SAC9D;QAAC,OAAO,KAAK,EAAE;YACd,MAAM,IAAI,KAAK,CACb,yBAAyB,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAClF,CAAC;SACH;IACH,CAAC;IAED;;OAEG;IACK,cAAc,CAAC,OAAe;QACpC,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;IACvB,CAAC;IAED;;OAEG;IACK,cAAc,CAAC,MAAc,EAAE,OAAe;QACpD,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE;YACvB,MAAM,IAAI,KAAK,CAAC,2CAA2C,CAAC,CAAC;SAC9D;QAED,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,OAAO,CAAC;IACvC,CAAC;IAED;;OAEG;IACK,cAAc;QACpB,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,UAAU,EAAE;YAC3B,MAAM,IAAI,KAAK,CAAC,wBAAwB,CAAC,CAAC;SAC3C;QAED,IAAI,IAAI,CAAC,MAAM,CAAC,UAAU,KAAK,MAAM,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,UAAU,EAAE;YAChE,MAAM,IAAI,KAAK,CAAC,6CAA6C,CAAC,CAAC;SAChE;QAED,IAAI,IAAI,CAAC,MAAM,CAAC,UAAU,KAAK,QAAQ,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE;YAC9D,MAAM,IAAI,KAAK,CAAC,2CAA2C,CAAC,CAAC;SAC9D;IACH,CAAC;CACF;AA5FD,0CA4FC","sourcesContent":["import fs from 'fs/promises';\nimport path from 'path';\nimport { NotionExporter, ChainData, ExporterError } from '../../../types';\n\n/**\n * Configuration for the FileSystemExporter\n */\nexport interface DefaultExporterConfig {\n // Output type: 'file', 'stdout', or 'buffer'\n outputType: 'file' | 'stdout' | 'buffer';\n\n // For 'file' type: path where files will be written, directories will be created if they don't exist\n outputPath?: string;\n\n // For 'buffer' type: reference to store the content\n buffer?: { [key: string]: string };\n}\n\n/**\n * Default exporter that handles saving content to the filesystem,\n * printing to stdout, or storing in a buffer\n */\nexport class DefaultExporter implements NotionExporter<DefaultExporterConfig> {\n constructor(private config: DefaultExporterConfig) {\n this.validateConfig();\n }\n\n /**\n * Export the conversion result according to configuration\n */\n public async export(data: ChainData): Promise<void> {\n try {\n const { pageId, content } = data;\n\n switch (this.config.outputType) {\n case 'file':\n await this.exportToFile(pageId, content);\n break;\n\n case 'stdout':\n this.exportToStdout(content);\n break;\n\n case 'buffer':\n this.exportToBuffer(pageId, content);\n break;\n }\n } catch (error) {\n throw new ExporterError(\n `Export failed: ${error instanceof Error ? error.message : String(error)}`,\n data.pageId,\n 'export',\n error,\n );\n }\n }\n\n /**\n * Export content to a file\n */\n private async exportToFile(pageId: string, content: string): Promise<void> {\n if (!this.config.outputPath) {\n throw new Error('outputPath is required for file output type');\n }\n\n // Get the directory path from the full file path\n const dirPath = path.dirname(this.config.outputPath);\n\n // Create all parent directories if they don't exist\n await fs.mkdir(dirPath, { recursive: true });\n\n try {\n await fs.writeFile(this.config.outputPath, content, 'utf-8');\n } catch (error) {\n throw new Error(\n `Failed to write file: ${error instanceof Error ? error.message : String(error)}`,\n );\n }\n }\n\n /**\n * Export content to stdout\n */\n private exportToStdout(content: string): void {\n console.log(content);\n }\n\n /**\n * Export content to a buffer object\n */\n private exportToBuffer(pageId: string, content: string): void {\n if (!this.config.buffer) {\n throw new Error('buffer is required for buffer output type');\n }\n\n this.config.buffer[pageId] = content;\n }\n\n /**\n * Validate the provided configuration\n */\n private validateConfig(): void {\n if (!this.config.outputType) {\n throw new Error('outputType is required');\n }\n\n if (this.config.outputType === 'file' && !this.config.outputPath) {\n throw new Error('outputPath is required for file output type');\n }\n\n if (this.config.outputType === 'buffer' && !this.config.buffer) {\n throw new Error('buffer is required for buffer output type');\n }\n }\n}\n"]}
|
|
@@ -1,40 +1,2 @@
|
|
|
1
|
-
|
|
2
|
-
/**
|
|
3
|
-
* Configuration for the FileSystemExporter
|
|
4
|
-
*/
|
|
5
|
-
export interface DefaultExporterConfig {
|
|
6
|
-
outputType: 'file' | 'stdout' | 'buffer';
|
|
7
|
-
outputPath?: string;
|
|
8
|
-
buffer?: {
|
|
9
|
-
[key: string]: string;
|
|
10
|
-
};
|
|
11
|
-
}
|
|
12
|
-
/**
|
|
13
|
-
* Default exporter that handles saving content to the filesystem,
|
|
14
|
-
* printing to stdout, or storing in a buffer
|
|
15
|
-
*/
|
|
16
|
-
export declare class DefaultExporter implements NotionExporter<DefaultExporterConfig> {
|
|
17
|
-
private config;
|
|
18
|
-
constructor(config: DefaultExporterConfig);
|
|
19
|
-
/**
|
|
20
|
-
* Export the conversion result according to configuration
|
|
21
|
-
*/
|
|
22
|
-
export(data: ChainData): Promise<void>;
|
|
23
|
-
/**
|
|
24
|
-
* Export content to a file
|
|
25
|
-
*/
|
|
26
|
-
private exportToFile;
|
|
27
|
-
/**
|
|
28
|
-
* Export content to stdout
|
|
29
|
-
*/
|
|
30
|
-
private exportToStdout;
|
|
31
|
-
/**
|
|
32
|
-
* Export content to a buffer object
|
|
33
|
-
*/
|
|
34
|
-
private exportToBuffer;
|
|
35
|
-
/**
|
|
36
|
-
* Validate the provided configuration
|
|
37
|
-
*/
|
|
38
|
-
private validateConfig;
|
|
39
|
-
}
|
|
1
|
+
export * from './DefaultExporter';
|
|
40
2
|
//# sourceMappingURL=index.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/plugins/exporter/index.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/plugins/exporter/index.ts"],"names":[],"mappings":"AAAA,cAAc,mBAAmB,CAAC"}
|
|
@@ -1,92 +1,18 @@
|
|
|
1
1
|
"use strict";
|
|
2
|
-
var
|
|
3
|
-
|
|
2
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
+
if (k2 === undefined) k2 = k;
|
|
4
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
+
}
|
|
8
|
+
Object.defineProperty(o, k2, desc);
|
|
9
|
+
}) : (function(o, m, k, k2) {
|
|
10
|
+
if (k2 === undefined) k2 = k;
|
|
11
|
+
o[k2] = m[k];
|
|
12
|
+
}));
|
|
13
|
+
var __exportStar = (this && this.__exportStar) || function(m, exports) {
|
|
14
|
+
for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
|
|
4
15
|
};
|
|
5
16
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
-
|
|
7
|
-
const promises_1 = __importDefault(require("fs/promises"));
|
|
8
|
-
const path_1 = __importDefault(require("path"));
|
|
9
|
-
const types_1 = require("../../types");
|
|
10
|
-
/**
|
|
11
|
-
* Default exporter that handles saving content to the filesystem,
|
|
12
|
-
* printing to stdout, or storing in a buffer
|
|
13
|
-
*/
|
|
14
|
-
class DefaultExporter {
|
|
15
|
-
constructor(config) {
|
|
16
|
-
this.config = config;
|
|
17
|
-
this.validateConfig();
|
|
18
|
-
}
|
|
19
|
-
/**
|
|
20
|
-
* Export the conversion result according to configuration
|
|
21
|
-
*/
|
|
22
|
-
async export(data) {
|
|
23
|
-
try {
|
|
24
|
-
const { pageId, content } = data;
|
|
25
|
-
switch (this.config.outputType) {
|
|
26
|
-
case 'file':
|
|
27
|
-
await this.exportToFile(pageId, content);
|
|
28
|
-
break;
|
|
29
|
-
case 'stdout':
|
|
30
|
-
this.exportToStdout(content);
|
|
31
|
-
break;
|
|
32
|
-
case 'buffer':
|
|
33
|
-
this.exportToBuffer(pageId, content);
|
|
34
|
-
break;
|
|
35
|
-
}
|
|
36
|
-
}
|
|
37
|
-
catch (error) {
|
|
38
|
-
throw new types_1.ExporterError(`Export failed: ${error instanceof Error ? error.message : String(error)}`, data.pageId, 'export', error);
|
|
39
|
-
}
|
|
40
|
-
}
|
|
41
|
-
/**
|
|
42
|
-
* Export content to a file
|
|
43
|
-
*/
|
|
44
|
-
async exportToFile(pageId, content) {
|
|
45
|
-
if (!this.config.outputPath) {
|
|
46
|
-
throw new Error('outputPath is required for file output type');
|
|
47
|
-
}
|
|
48
|
-
// Get the directory path from the full file path
|
|
49
|
-
const dirPath = path_1.default.dirname(this.config.outputPath);
|
|
50
|
-
// Create all parent directories if they don't exist
|
|
51
|
-
await promises_1.default.mkdir(dirPath, { recursive: true });
|
|
52
|
-
// Write the file
|
|
53
|
-
await promises_1.default.writeFile(this.config.outputPath, content, 'utf-8');
|
|
54
|
-
try {
|
|
55
|
-
await promises_1.default.writeFile(this.config.outputPath, content, 'utf-8');
|
|
56
|
-
}
|
|
57
|
-
catch (error) {
|
|
58
|
-
throw new Error(`Failed to write file: ${error instanceof Error ? error.message : String(error)}`);
|
|
59
|
-
}
|
|
60
|
-
}
|
|
61
|
-
/**
|
|
62
|
-
* Export content to stdout
|
|
63
|
-
*/
|
|
64
|
-
exportToStdout(content) {
|
|
65
|
-
console.log(content);
|
|
66
|
-
}
|
|
67
|
-
/**
|
|
68
|
-
* Export content to a buffer object
|
|
69
|
-
*/
|
|
70
|
-
exportToBuffer(pageId, content) {
|
|
71
|
-
if (!this.config.buffer) {
|
|
72
|
-
throw new Error('buffer is required for buffer output type');
|
|
73
|
-
}
|
|
74
|
-
this.config.buffer[pageId] = content;
|
|
75
|
-
}
|
|
76
|
-
/**
|
|
77
|
-
* Validate the provided configuration
|
|
78
|
-
*/
|
|
79
|
-
validateConfig() {
|
|
80
|
-
if (!this.config.outputType) {
|
|
81
|
-
throw new Error('outputType is required');
|
|
82
|
-
}
|
|
83
|
-
if (this.config.outputType === 'file' && !this.config.outputPath) {
|
|
84
|
-
throw new Error('outputPath is required for file output type');
|
|
85
|
-
}
|
|
86
|
-
if (this.config.outputType === 'buffer' && !this.config.buffer) {
|
|
87
|
-
throw new Error('buffer is required for buffer output type');
|
|
88
|
-
}
|
|
89
|
-
}
|
|
90
|
-
}
|
|
91
|
-
exports.DefaultExporter = DefaultExporter;
|
|
17
|
+
__exportStar(require("./DefaultExporter"), exports);
|
|
92
18
|
//# sourceMappingURL=index.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/plugins/exporter/index.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/plugins/exporter/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;AAAA,oDAAkC","sourcesContent":["export * from './DefaultExporter';\n"]}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/plugins/renderer/index.ts"],"names":[],"mappings":"AAAA,cAAc,OAAO,CAAC"}
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
+
if (k2 === undefined) k2 = k;
|
|
4
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
+
}
|
|
8
|
+
Object.defineProperty(o, k2, desc);
|
|
9
|
+
}) : (function(o, m, k, k2) {
|
|
10
|
+
if (k2 === undefined) k2 = k;
|
|
11
|
+
o[k2] = m[k];
|
|
12
|
+
}));
|
|
13
|
+
var __exportStar = (this && this.__exportStar) || function(m, exports) {
|
|
14
|
+
for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
|
|
15
|
+
};
|
|
16
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
17
|
+
__exportStar(require("./mdx"), exports);
|
|
18
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/plugins/renderer/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;AAAA,wCAAsB","sourcesContent":["export * from './mdx';\n"]}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/utils/page-ref-builder/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/utils/page-ref-builder/index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAC/C,OAAO,EAAE,4BAA4B,EAAE,MAAM,8BAA8B,CAAC;AAI5E,UAAU,oBAAoB;IAC5B,qBAAqB,EAAE,MAAM,CAAC;IAC9B,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,qBAAa,4BAA4B;IAMrC,OAAO,CAAC,MAAM;IACd,OAAO,CAAC,MAAM;IANhB,OAAO,CAAC,cAAc,CAA0B;IAChD,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAS;IACrC,OAAO,CAAC,eAAe,CAA+B;gBAG5C,MAAM,EAAE,MAAM,EACd,MAAM,EAAE,oBAAoB;IAMhC,KAAK,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;YA6B5B,eAAe;YAiCf,uBAAuB;YAgCvB,mBAAmB;YA4BnB,mBAAmB;IAwBjC,kBAAkB,IAAI,4BAA4B;CAGnD"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/utils/page-ref-builder/index.ts"],"names":[],"mappings":";;;AACA,mEAA4E;AAC5E,uCAAqD;AACrD,8CAA8D;AAQ9D,MAAa,4BAA4B;IAKvC,YACU,MAAc,EACd,MAA4B;QAD5B,WAAM,GAAN,MAAM,CAAQ;QACd,WAAM,GAAN,MAAM,CAAsB;QAN9B,mBAAc,GAAgB,IAAI,GAAG,EAAE,CAAC;QAQ9C,IAAI,CAAC,WAAW,GAAG,MAAM,CAAC,WAAW,IAAI,CAAC,CAAC;QAC3C,IAAI,CAAC,eAAe,GAAG,IAAI,+CAA4B,EAAE,CAAC;IAC5D,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,MAAc;QACxB,OAAO,CAAC,IAAI,CAAC,qCAAqC,MAAM,EAAE,CAAC,CAAC;QAC5D,IAAI;YACF,MAAM,IAAI,CAAC,eAAe,CAAC,UAAU,EAAE,CAAC;YAExC,IAAI;gBACF,8BAA8B;gBAC9B,MAAM,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,QAAQ,CAAC,EAAE,WAAW,EAAE,MAAM,EAAE,CAAC,CAAC;gBAC9D,OAAO,CAAC,IAAI,CAAC,uBAAuB,MAAM,EAAE,CAAC,CAAC;gBAC9C,MAAM,IAAI,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC;aACpC;YAAC,OAAO,KAAK,EAAE;gBACd,+BAA+B;gBAC/B,OAAO,CAAC,IAAI,CAAC,mBAAmB,MAAM,gBAAgB,CAAC,CAAC;gBACxD,MAAM,IAAI,CAAC,uBAAuB,CAAC,MAAM,CAAC,CAAC;aAC5C;YAED,MAAM,IAAI,CAAC,eAAe,CAAC,IAAI,EAAE,CAAC;YAClC,OAAO,CAAC,IAAI,CAAC,uCAAuC,CAAC,CAAC;SACvD;QAAC,OAAO,KAAK,EAAE;YACd,OAAO,CAAC,KAAK,CACX,iBAAiB,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAC1E,CAAC;YACF,MAAM,IAAI,kCAAyB,CACjC,0BAA0B,EAC1B,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,CAC3C,CAAC;SACH;IACH,CAAC;IAEO,KAAK,CAAC,eAAe,CAAC,UAAkB;;QAC9C,IAAI,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,UAAU,CAAC,EAAE;YACvC,OAAO,CAAC,KAAK,CAAC,+BAA+B,UAAU,EAAE,CAAC,CAAC;YAC3D,OAAO;SACR;QAED,OAAO,CAAC,KAAK,CAAC,qBAAqB,UAAU,EAAE,CAAC,CAAC;QACjD,IAAI,MAA0B,CAAC;QAE/B,IAAI;YACF,GAAG;gBACD,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC;oBACjD,WAAW,EAAE,UAAU;oBACvB,YAAY,EAAE,MAAM;iBACrB,CAAC,CAAC;gBAEH,MAAM,OAAO,CAAC,GAAG,CACf,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE;oBAClC,MAAM,IAAI,CAAC,mBAAmB,CAAC,IAAI,CAAC,CAAC;gBACvC,CAAC,CAAC,CACH,CAAC;gBAEF,MAAM,GAAG,MAAA,QAAQ,CAAC,WAAW,mCAAI,SAAS,CAAC;aAC5C,QAAQ,MAAM,EAAE;YAEjB,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;SACrC;QAAC,OAAO,KAAK,EAAE;YACd,OAAO,CAAC,KAAK,CACX,6BAA6B,UAAU,KAAK,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CACrG,CAAC;SACH;IACH,CAAC;IAEO,KAAK,CAAC,uBAAuB,CAAC,MAAc;;QAClD,IAAI,MAA0B,CAAC;QAE/B,IAAI;YACF,GAAG;gBACD,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC;oBACtD,QAAQ,EAAE,MAAM;oBAChB,YAAY,EAAE,MAAM;iBACrB,CAAC,CAAC;gBAEH,MAAM,cAAc,GAAG,QAAQ,CAAC,OAAO,CAAC,MAAM;gBAC5C,aAAa;gBACb,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,KAAK,gBAAgB,CAC3C,CAAC;gBAEF,OAAO,CAAC,KAAK,CACX,SAAS,cAAc,CAAC,MAAM,sBAAsB,MAAM,EAAE,CAC7D,CAAC;gBAEF,KAAK,MAAM,KAAK,IAAI,cAAc,EAAE;oBAClC,MAAM,IAAI,CAAC,eAAe,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;iBACtC;gBAED,MAAM,GAAG,MAAA,QAAQ,CAAC,WAAW,mCAAI,SAAS,CAAC;aAC5C,QAAQ,MAAM,EAAE;SAClB;QAAC,OAAO,KAAK,EAAE;YACd,OAAO,CAAC,IAAI,CACV,wBAAwB,MAAM,mBAAmB,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAC1G,CAAC;SACH;IACH,CAAC;IAEO,KAAK,CAAC,mBAAmB,CAAC,IAAS;QACzC,IAAI,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE;YACpC,OAAO;SACR;QAED,OAAO,CAAC,KAAK,CAAC,4BAA4B,IAAI,CAAC,EAAE,EAAE,CAAC,CAAC;QACrD,IAAI;YACF,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,mBAAmB,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;YAC5D,IAAI,CAAC,GAAG,EAAE;gBACR,OAAO,CAAC,KAAK,CAAC,yBAAyB,IAAI,CAAC,EAAE,EAAE,CAAC,CAAC;gBAClD,OAAO;aACR;YAED,MAAM,IAAI,CAAC,eAAe,CAAC,WAAW,CAAC,IAAI,CAAC,EAAE,EAAE;gBAC9C,GAAG;gBACH,MAAM,EAAE,8BAAsB,CAAC,QAAQ;gBACvC,WAAW,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;aACtC,CAAC,CAAC;YAEH,OAAO,CAAC,KAAK,CAAC,6BAA6B,IAAI,CAAC,EAAE,aAAa,GAAG,EAAE,CAAC,CAAC;YACtE,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;SAClC;QAAC,OAAO,KAAK,EAAE;YACd,OAAO,CAAC,IAAI,CACV,0BAA0B,IAAI,CAAC,EAAE,KAAK,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAC/F,CAAC;SACH;IACH,CAAC;IAEO,KAAK,CAAC,mBAAmB,CAC/B,UAA+B;;QAE/B,MAAM,WAAW,GAAG,UAAU,CAAC,IAAI,CAAC,MAAM,CAAC,qBAAqB,CAAC,CAAC;QAClE,IAAI,CAAC,WAAW;YAAE,OAAO,IAAI,CAAC;QAE9B,IAAI,GAAG,GAAkB,IAAI,CAAC;QAE9B,IAAI,KAAK,IAAI,WAAW,EAAE;YACxB,GAAG,GAAG,WAAW,CAAC,GAAG,CAAC;SACvB;aAAM,IACL,WAAW,IAAI,WAAW;YAC1B,KAAK,CAAC,OAAO,CAAC,WAAW,CAAC,SAAS,CAAC;YACpC,WAAW,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,EAChC;YACA,GAAG,GAAG,MAAA,WAAW,CAAC,SAAS,CAAC,CAAC,CAAC,0CAAE,UAAU,CAAC;SAC5C;QAED,IAAI,CAAC,CAAA,GAAG,aAAH,GAAG,uBAAH,GAAG,CAAE,UAAU,CAAC,MAAM,CAAC,CAAA,IAAI,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE;YACnD,GAAG,GAAG,GAAG,IAAI,CAAC,MAAM,CAAC,OAAO,GAAG,GAAG,EAAE,CAAC;SACtC;QACD,OAAO,GAAG,CAAC;IACb,CAAC;IAED,kBAAkB;QAChB,OAAO,IAAI,CAAC,eAAe,CAAC;IAC9B,CAAC;CACF;AAlKD,oEAkKC","sourcesContent":["import { Client } from '@notionhq/client';\nimport { PageReferenceManifestManager } from '../../utils/manifest-manager';\nimport { PageReferenceEntryType } from '../../types';\nimport { PageReferenceHandlerError } from '../../core/errors';\n\ninterface PageRefBuilderConfig {\n urlPropertyNameNotion: string;\n concurrency?: number;\n baseUrl?: string;\n}\n\nexport class PageReferenceManifestBuilder {\n private processedItems: Set<string> = new Set();\n private readonly concurrency: number;\n private manifestManager: PageReferenceManifestManager;\n\n constructor(\n private client: Client,\n private config: PageRefBuilderConfig,\n ) {\n this.concurrency = config.concurrency || 3;\n this.manifestManager = new PageReferenceManifestManager();\n }\n\n async build(rootId: string): Promise<void> {\n console.info(`Starting manifest build from root ${rootId}`);\n try {\n await this.manifestManager.initialize();\n\n try {\n // Try database endpoint first\n await this.client.databases.retrieve({ database_id: rootId });\n console.info(`Processing database ${rootId}`);\n await this.processDatabase(rootId);\n } catch (error) {\n // If not database, try as page\n console.info(`Processing page ${rootId} for databases`);\n await this.findAndProcessDatabases(rootId);\n }\n\n await this.manifestManager.save();\n console.info('Manifest build completed successfully');\n } catch (error) {\n console.error(\n `Build failed: ${error instanceof Error ? error.message : String(error)}`,\n );\n throw new PageReferenceHandlerError(\n 'Failed to build manifest',\n error instanceof Error ? error : undefined,\n );\n }\n }\n\n private async processDatabase(databaseId: string): Promise<void> {\n if (this.processedItems.has(databaseId)) {\n console.debug(`Skipping processed database ${databaseId}`);\n return;\n }\n\n console.debug(`Querying database ${databaseId}`);\n let cursor: string | undefined;\n\n try {\n do {\n const response = await this.client.databases.query({\n database_id: databaseId,\n start_cursor: cursor,\n });\n\n await Promise.all(\n response.results.map(async (page) => {\n await this.processDatabasePage(page);\n }),\n );\n\n cursor = response.next_cursor ?? undefined;\n } while (cursor);\n\n this.processedItems.add(databaseId);\n } catch (error) {\n console.error(\n `Database query failed for ${databaseId}: ${error instanceof Error ? error.message : String(error)}`,\n );\n }\n }\n\n private async findAndProcessDatabases(pageId: string): Promise<void> {\n let cursor: string | undefined;\n\n try {\n do {\n const response = await this.client.blocks.children.list({\n block_id: pageId,\n start_cursor: cursor,\n });\n\n const databaseBlocks = response.results.filter(\n // @ts-ignore\n (block) => block.type === 'child_database',\n );\n\n console.debug(\n `Found ${databaseBlocks.length} databases in page ${pageId}`,\n );\n\n for (const block of databaseBlocks) {\n await this.processDatabase(block.id);\n }\n\n cursor = response.next_cursor ?? undefined;\n } while (cursor);\n } catch (error) {\n console.warn(\n `Failed scanning page ${pageId} for databases: ${error instanceof Error ? error.message : String(error)}`,\n );\n }\n }\n\n private async processDatabasePage(page: any): Promise<void> {\n if (this.processedItems.has(page.id)) {\n return;\n }\n\n console.debug(`Processing database page ${page.id}`);\n try {\n const url = await this.extractPublishedUrl(page.properties);\n if (!url) {\n console.debug(`No valid URL for page ${page.id}`);\n return;\n }\n\n await this.manifestManager.updateEntry(page.id, {\n url,\n source: PageReferenceEntryType.PROPERTY,\n lastUpdated: new Date().toISOString(),\n });\n\n console.debug(`Updated manifest for page ${page.id} with URL ${url}`);\n this.processedItems.add(page.id);\n } catch (error) {\n console.warn(\n `Failed processing page ${page.id}: ${error instanceof Error ? error.message : String(error)}`,\n );\n }\n }\n\n private async extractPublishedUrl(\n properties: Record<string, any>,\n ): Promise<string | null> {\n const urlProperty = properties[this.config.urlPropertyNameNotion];\n if (!urlProperty) return null;\n\n let url: string | null = null;\n\n if ('url' in urlProperty) {\n url = urlProperty.url;\n } else if (\n 'rich_text' in urlProperty &&\n Array.isArray(urlProperty.rich_text) &&\n urlProperty.rich_text.length > 0\n ) {\n url = urlProperty.rich_text[0]?.plain_text;\n }\n\n if (!url?.startsWith('http') && this.config.baseUrl) {\n url = `${this.config.baseUrl}${url}`;\n }\n return url;\n }\n\n getManifestManager(): PageReferenceManifestManager {\n return this.manifestManager;\n }\n}\n"]}
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/utils/page-ref-builder/index.ts"],"names":[],"mappings":";;;AACA,mEAA4E;AAC5E,uCAAqD;AACrD,8CAA8D;AAQ9D,MAAa,4BAA4B;IAKvC,YACU,MAAc,EACd,MAA4B;QAD5B,WAAM,GAAN,MAAM,CAAQ;QACd,WAAM,GAAN,MAAM,CAAsB;QAN9B,mBAAc,GAAgB,IAAI,GAAG,EAAE,CAAC;QAQ9C,IAAI,CAAC,WAAW,GAAG,MAAM,CAAC,WAAW,IAAI,CAAC,CAAC;QAC3C,IAAI,CAAC,eAAe,GAAG,IAAI,+CAA4B,EAAE,CAAC;IAC5D,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,MAAc;QACxB,OAAO,CAAC,IAAI,CAAC,qCAAqC,MAAM,EAAE,CAAC,CAAC;QAC5D,IAAI;YACF,MAAM,IAAI,CAAC,eAAe,CAAC,UAAU,EAAE,CAAC;YAExC,IAAI;gBACF,8BAA8B;gBAC9B,MAAM,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,QAAQ,CAAC,EAAE,WAAW,EAAE,MAAM,EAAE,CAAC,CAAC;gBAC9D,OAAO,CAAC,IAAI,CAAC,uBAAuB,MAAM,EAAE,CAAC,CAAC;gBAC9C,MAAM,IAAI,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC;aACpC;YAAC,OAAO,KAAK,EAAE;gBACd,+BAA+B;gBAC/B,OAAO,CAAC,IAAI,CAAC,mBAAmB,MAAM,gBAAgB,CAAC,CAAC;gBACxD,MAAM,IAAI,CAAC,uBAAuB,CAAC,MAAM,CAAC,CAAC;aAC5C;YAED,MAAM,IAAI,CAAC,eAAe,CAAC,IAAI,EAAE,CAAC;YAClC,OAAO,CAAC,IAAI,CAAC,uCAAuC,CAAC,CAAC;SACvD;QAAC,OAAO,KAAK,EAAE;YACd,OAAO,CAAC,KAAK,CACX,iBAAiB,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAC1E,CAAC;YACF,MAAM,IAAI,kCAAyB,CACjC,0BAA0B,EAC1B,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,CAC3C,CAAC;SACH;IACH,CAAC;IAEO,KAAK,CAAC,eAAe,CAAC,UAAkB;;QAC9C,IAAI,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,UAAU,CAAC,EAAE;YACvC,OAAO,CAAC,KAAK,CAAC,+BAA+B,UAAU,EAAE,CAAC,CAAC;YAC3D,OAAO;SACR;QAED,OAAO,CAAC,KAAK,CAAC,qBAAqB,UAAU,EAAE,CAAC,CAAC;QACjD,IAAI,MAA0B,CAAC;QAE/B,IAAI;YACF,GAAG;gBACD,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC;oBACjD,WAAW,EAAE,UAAU;oBACvB,YAAY,EAAE,MAAM;iBACrB,CAAC,CAAC;gBAEH,MAAM,OAAO,CAAC,GAAG,CACf,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE;oBAClC,MAAM,IAAI,CAAC,mBAAmB,CAAC,IAAI,CAAC,CAAC;gBACvC,CAAC,CAAC,CACH,CAAC;gBAEF,MAAM,GAAG,MAAA,QAAQ,CAAC,WAAW,mCAAI,SAAS,CAAC;aAC5C,QAAQ,MAAM,EAAE;YAEjB,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;SACrC;QAAC,OAAO,KAAK,EAAE;YACd,OAAO,CAAC,KAAK,CACX,6BAA6B,UAAU,KAAK,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CACrG,CAAC;SACH;IACH,CAAC;IAEO,KAAK,CAAC,uBAAuB,CAAC,MAAc;;QAClD,IAAI,MAA0B,CAAC;QAE/B,IAAI;YACF,GAAG;gBACD,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC;oBACtD,QAAQ,EAAE,MAAM;oBAChB,YAAY,EAAE,MAAM;iBACrB,CAAC,CAAC;gBAEH,MAAM,cAAc,GAAG,QAAQ,CAAC,OAAO,CAAC,MAAM;gBAC5C,aAAa;gBACb,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,KAAK,gBAAgB,CAC3C,CAAC;gBAEF,OAAO,CAAC,KAAK,CACX,SAAS,cAAc,CAAC,MAAM,sBAAsB,MAAM,EAAE,CAC7D,CAAC;gBAEF,KAAK,MAAM,KAAK,IAAI,cAAc,EAAE;oBAClC,MAAM,IAAI,CAAC,eAAe,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;iBACtC;gBAED,MAAM,GAAG,MAAA,QAAQ,CAAC,WAAW,mCAAI,SAAS,CAAC;aAC5C,QAAQ,MAAM,EAAE;SAClB;QAAC,OAAO,KAAK,EAAE;YACd,OAAO,CAAC,IAAI,CACV,wBAAwB,MAAM,mBAAmB,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAC1G,CAAC;SACH;IACH,CAAC;IAEO,KAAK,CAAC,mBAAmB,CAAC,IAAS;QACzC,IAAI,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE;YACpC,OAAO;SACR;QAED,OAAO,CAAC,KAAK,CAAC,4BAA4B,IAAI,CAAC,EAAE,EAAE,CAAC,CAAC;QACrD,IAAI;YACF,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,mBAAmB,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;YAC5D,IAAI,CAAC,GAAG,EAAE;gBACR,OAAO,CAAC,KAAK,CAAC,yBAAyB,IAAI,CAAC,EAAE,EAAE,CAAC,CAAC;gBAClD,OAAO;aACR;YAED,MAAM,IAAI,CAAC,eAAe,CAAC,WAAW,CAAC,IAAI,CAAC,EAAE,EAAE;gBAC9C,GAAG;gBACH,MAAM,EAAE,8BAAsB,CAAC,QAAQ;gBACvC,WAAW,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;aACtC,CAAC,CAAC;YAEH,OAAO,CAAC,KAAK,CAAC,6BAA6B,IAAI,CAAC,EAAE,aAAa,GAAG,EAAE,CAAC,CAAC;YACtE,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;SAClC;QAAC,OAAO,KAAK,EAAE;YACd,OAAO,CAAC,IAAI,CACV,0BAA0B,IAAI,CAAC,EAAE,KAAK,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAC/F,CAAC;SACH;IACH,CAAC;IAEO,KAAK,CAAC,mBAAmB,CAC/B,UAA+B;;QAE/B,MAAM,WAAW,GAAG,UAAU,CAAC,IAAI,CAAC,MAAM,CAAC,qBAAqB,CAAC,CAAC;QAClE,IAAI,CAAC,WAAW;YAAE,OAAO,IAAI,CAAC;QAE9B,IAAI,GAAG,GAAkB,IAAI,CAAC;QAE9B,IAAI,KAAK,IAAI,WAAW,EAAE;YACxB,GAAG,GAAG,WAAW,CAAC,GAAG,CAAC;SACvB;aAAM,IACL,WAAW,IAAI,WAAW;YAC1B,KAAK,CAAC,OAAO,CAAC,WAAW,CAAC,SAAS,CAAC;YACpC,WAAW,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,EAChC;YACA,GAAG,GAAG,MAAA,WAAW,CAAC,SAAS,CAAC,CAAC,CAAC,0CAAE,UAAU,CAAC;SAC5C;QAED,IAAI,CAAC,CAAA,GAAG,aAAH,GAAG,uBAAH,GAAG,CAAE,UAAU,CAAC,MAAM,CAAC,CAAA,IAAI,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE;YACnD,GAAG,GAAG,GAAG,IAAI,CAAC,MAAM,CAAC,OAAO,GAAG,GAAG,EAAE,CAAC;SACtC;QACD,OAAO,GAAG,CAAC;IACb,CAAC;IAED,kBAAkB;QAChB,OAAO,IAAI,CAAC,eAAe,CAAC;IAC9B,CAAC;CACF;AAlKD,oEAkKC","sourcesContent":["import type { Client } from '@notionhq/client';\nimport { PageReferenceManifestManager } from '../../utils/manifest-manager';\nimport { PageReferenceEntryType } from '../../types';\nimport { PageReferenceHandlerError } from '../../core/errors';\n\ninterface PageRefBuilderConfig {\n urlPropertyNameNotion: string;\n concurrency?: number;\n baseUrl?: string;\n}\n\nexport class PageReferenceManifestBuilder {\n private processedItems: Set<string> = new Set();\n private readonly concurrency: number;\n private manifestManager: PageReferenceManifestManager;\n\n constructor(\n private client: Client,\n private config: PageRefBuilderConfig,\n ) {\n this.concurrency = config.concurrency || 3;\n this.manifestManager = new PageReferenceManifestManager();\n }\n\n async build(rootId: string): Promise<void> {\n console.info(`Starting manifest build from root ${rootId}`);\n try {\n await this.manifestManager.initialize();\n\n try {\n // Try database endpoint first\n await this.client.databases.retrieve({ database_id: rootId });\n console.info(`Processing database ${rootId}`);\n await this.processDatabase(rootId);\n } catch (error) {\n // If not database, try as page\n console.info(`Processing page ${rootId} for databases`);\n await this.findAndProcessDatabases(rootId);\n }\n\n await this.manifestManager.save();\n console.info('Manifest build completed successfully');\n } catch (error) {\n console.error(\n `Build failed: ${error instanceof Error ? error.message : String(error)}`,\n );\n throw new PageReferenceHandlerError(\n 'Failed to build manifest',\n error instanceof Error ? error : undefined,\n );\n }\n }\n\n private async processDatabase(databaseId: string): Promise<void> {\n if (this.processedItems.has(databaseId)) {\n console.debug(`Skipping processed database ${databaseId}`);\n return;\n }\n\n console.debug(`Querying database ${databaseId}`);\n let cursor: string | undefined;\n\n try {\n do {\n const response = await this.client.databases.query({\n database_id: databaseId,\n start_cursor: cursor,\n });\n\n await Promise.all(\n response.results.map(async (page) => {\n await this.processDatabasePage(page);\n }),\n );\n\n cursor = response.next_cursor ?? undefined;\n } while (cursor);\n\n this.processedItems.add(databaseId);\n } catch (error) {\n console.error(\n `Database query failed for ${databaseId}: ${error instanceof Error ? error.message : String(error)}`,\n );\n }\n }\n\n private async findAndProcessDatabases(pageId: string): Promise<void> {\n let cursor: string | undefined;\n\n try {\n do {\n const response = await this.client.blocks.children.list({\n block_id: pageId,\n start_cursor: cursor,\n });\n\n const databaseBlocks = response.results.filter(\n // @ts-ignore\n (block) => block.type === 'child_database',\n );\n\n console.debug(\n `Found ${databaseBlocks.length} databases in page ${pageId}`,\n );\n\n for (const block of databaseBlocks) {\n await this.processDatabase(block.id);\n }\n\n cursor = response.next_cursor ?? undefined;\n } while (cursor);\n } catch (error) {\n console.warn(\n `Failed scanning page ${pageId} for databases: ${error instanceof Error ? error.message : String(error)}`,\n );\n }\n }\n\n private async processDatabasePage(page: any): Promise<void> {\n if (this.processedItems.has(page.id)) {\n return;\n }\n\n console.debug(`Processing database page ${page.id}`);\n try {\n const url = await this.extractPublishedUrl(page.properties);\n if (!url) {\n console.debug(`No valid URL for page ${page.id}`);\n return;\n }\n\n await this.manifestManager.updateEntry(page.id, {\n url,\n source: PageReferenceEntryType.PROPERTY,\n lastUpdated: new Date().toISOString(),\n });\n\n console.debug(`Updated manifest for page ${page.id} with URL ${url}`);\n this.processedItems.add(page.id);\n } catch (error) {\n console.warn(\n `Failed processing page ${page.id}: ${error instanceof Error ? error.message : String(error)}`,\n );\n }\n }\n\n private async extractPublishedUrl(\n properties: Record<string, any>,\n ): Promise<string | null> {\n const urlProperty = properties[this.config.urlPropertyNameNotion];\n if (!urlProperty) return null;\n\n let url: string | null = null;\n\n if ('url' in urlProperty) {\n url = urlProperty.url;\n } else if (\n 'rich_text' in urlProperty &&\n Array.isArray(urlProperty.rich_text) &&\n urlProperty.rich_text.length > 0\n ) {\n url = urlProperty.rich_text[0]?.plain_text;\n }\n\n if (!url?.startsWith('http') && this.config.baseUrl) {\n url = `${this.config.baseUrl}${url}`;\n }\n return url;\n }\n\n getManifestManager(): PageReferenceManifestManager {\n return this.manifestManager;\n }\n}\n"]}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "notion-to-md",
|
|
3
|
-
"version": "4.0.0-alpha.
|
|
3
|
+
"version": "4.0.0-alpha.3",
|
|
4
4
|
"engines": {
|
|
5
5
|
"node": ">=12"
|
|
6
6
|
},
|
|
@@ -16,6 +16,13 @@
|
|
|
16
16
|
"lint": "eslint \"src/**/*.ts\"",
|
|
17
17
|
"format": "prettier --write ."
|
|
18
18
|
},
|
|
19
|
+
"peerDependencies": {
|
|
20
|
+
"@notionhq/client": "^2.0.0"
|
|
21
|
+
},
|
|
22
|
+
"exports": {
|
|
23
|
+
".": "./build/index.js",
|
|
24
|
+
"./*": "./build/*/index.js"
|
|
25
|
+
},
|
|
19
26
|
"lint-staged": {
|
|
20
27
|
"*.ts": [
|
|
21
28
|
"eslint --fix",
|
|
@@ -29,9 +36,13 @@
|
|
|
29
36
|
"notion",
|
|
30
37
|
"notion-api",
|
|
31
38
|
"notion-to-md",
|
|
32
|
-
"notion2md"
|
|
39
|
+
"notion2md",
|
|
40
|
+
"notion-export",
|
|
41
|
+
"notion-exporter",
|
|
42
|
+
"notion-convert",
|
|
43
|
+
"export"
|
|
33
44
|
],
|
|
34
|
-
"author": "Souvik Kar Mahapatra <souvikat001
|
|
45
|
+
"author": "Souvik Kar Mahapatra <souvikat001@gmail.com>",
|
|
35
46
|
"license": "ISC",
|
|
36
47
|
"files": [
|
|
37
48
|
"build/**"
|
|
@@ -44,8 +55,9 @@
|
|
|
44
55
|
"url": "https://github.com/souvikinator/notion-to-md/issues"
|
|
45
56
|
},
|
|
46
57
|
"homepage": "https://github.com/souvikinator/notion-to-md#readme",
|
|
47
|
-
"description": "
|
|
58
|
+
"description": "Convert notion pages, block and list of blocks to markdown, html, jsx, latex, pdf and literally anything and export anywhere, compatible with any content workflow",
|
|
48
59
|
"devDependencies": {
|
|
60
|
+
"@notionhq/client": "^2.0.0",
|
|
49
61
|
"@types/download": "^8.0.5",
|
|
50
62
|
"@types/jest": "^27.4.1",
|
|
51
63
|
"@types/mime": "^3.0.0",
|
|
@@ -63,7 +75,6 @@
|
|
|
63
75
|
"typescript": "^4.5.4"
|
|
64
76
|
},
|
|
65
77
|
"dependencies": {
|
|
66
|
-
"@notionhq/client": "^2.2.15",
|
|
67
78
|
"mime": "^3.0.0",
|
|
68
79
|
"node-fetch": "^2.7.0"
|
|
69
80
|
}
|