@moxn/kb-migrate 0.4.0 → 0.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/client.d.ts +1 -0
- package/dist/sources/notion-blocks.js +1 -1
- package/dist/sources/notion-media.d.ts +2 -2
- package/dist/sources/notion-media.js +43 -18
- package/dist/sources/notion.js +1 -1
- package/dist/types.d.ts +55 -9
- package/package.json +43 -1
package/dist/client.d.ts
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* API client for Moxn KB
|
|
3
3
|
*/
|
|
4
|
+
/// <reference types="node" resolution-mode="require"/>
|
|
4
5
|
import type { ExtractedDocument, MigrationOptions, MigrationResult, DocumentListItem, DocumentDetail, ExportOptions } from './types.js';
|
|
5
6
|
export declare class MoxnClient {
|
|
6
7
|
private apiUrl;
|
|
@@ -21,8 +21,8 @@ export declare class NotionMediaDownloader {
|
|
|
21
21
|
* "prod-files-secure" or "s3.us-west-2.amazonaws.com") are downloaded.
|
|
22
22
|
*/
|
|
23
23
|
processContentBlocks(blocks: ContentBlock[]): Promise<ContentBlock[]>;
|
|
24
|
-
/** Check if a content block has a Notion signed URL that needs downloading. */
|
|
25
|
-
private
|
|
24
|
+
/** Check if a content block has a Notion signed URL that needs downloading. Returns typed block or null. */
|
|
25
|
+
private asNotionSignedUrlBlock;
|
|
26
26
|
/** Download a single file and return the local path. */
|
|
27
27
|
private downloadFile;
|
|
28
28
|
/** Get file extension from URL or content block metadata. */
|
|
@@ -34,15 +34,39 @@ export class NotionMediaDownloader {
|
|
|
34
34
|
async processContentBlocks(blocks) {
|
|
35
35
|
const results = [];
|
|
36
36
|
for (const block of blocks) {
|
|
37
|
-
|
|
37
|
+
const urlBlock = this.asNotionSignedUrlBlock(block);
|
|
38
|
+
if (urlBlock) {
|
|
38
39
|
try {
|
|
39
|
-
const localPath = await this.downloadFile(
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
40
|
+
const localPath = await this.downloadFile(urlBlock.url, urlBlock);
|
|
41
|
+
if (urlBlock.blockType === 'image') {
|
|
42
|
+
results.push({
|
|
43
|
+
blockType: 'image',
|
|
44
|
+
type: 'file',
|
|
45
|
+
path: localPath,
|
|
46
|
+
mediaType: urlBlock.mediaType,
|
|
47
|
+
alt: urlBlock.alt,
|
|
48
|
+
});
|
|
49
|
+
}
|
|
50
|
+
else if (urlBlock.blockType === 'document') {
|
|
51
|
+
results.push({
|
|
52
|
+
blockType: 'document',
|
|
53
|
+
type: 'file',
|
|
54
|
+
path: localPath,
|
|
55
|
+
mediaType: urlBlock.mediaType,
|
|
56
|
+
filename: urlBlock.filename,
|
|
57
|
+
});
|
|
58
|
+
}
|
|
59
|
+
else {
|
|
60
|
+
results.push({
|
|
61
|
+
blockType: 'csv',
|
|
62
|
+
type: 'file',
|
|
63
|
+
path: localPath,
|
|
64
|
+
mediaType: urlBlock.mediaType,
|
|
65
|
+
filename: urlBlock.filename,
|
|
66
|
+
headers: urlBlock.headers,
|
|
67
|
+
rowCount: urlBlock.rowCount,
|
|
68
|
+
});
|
|
69
|
+
}
|
|
46
70
|
}
|
|
47
71
|
catch (error) {
|
|
48
72
|
console.warn(` Warning: Failed to download ${block.blockType}: ${error instanceof Error ? error.message : error}`);
|
|
@@ -56,15 +80,18 @@ export class NotionMediaDownloader {
|
|
|
56
80
|
}
|
|
57
81
|
return results;
|
|
58
82
|
}
|
|
59
|
-
/** Check if a content block has a Notion signed URL that needs downloading. */
|
|
60
|
-
|
|
83
|
+
/** Check if a content block has a Notion signed URL that needs downloading. Returns typed block or null. */
|
|
84
|
+
asNotionSignedUrlBlock(block) {
|
|
85
|
+
if (block.blockType === 'text')
|
|
86
|
+
return null;
|
|
61
87
|
if (block.type !== 'url' || !block.url)
|
|
62
|
-
return
|
|
88
|
+
return null;
|
|
63
89
|
const url = block.url;
|
|
64
|
-
|
|
90
|
+
const isNotionUrl = url.includes('secure.notion-static.com') ||
|
|
65
91
|
url.includes('prod-files-secure') ||
|
|
66
92
|
url.includes('s3.us-west-2.amazonaws.com') ||
|
|
67
|
-
url.includes('s3-us-west-2.amazonaws.com')
|
|
93
|
+
url.includes('s3-us-west-2.amazonaws.com');
|
|
94
|
+
return isNotionUrl ? block : null;
|
|
68
95
|
}
|
|
69
96
|
/** Download a single file and return the local path. */
|
|
70
97
|
async downloadFile(url, block) {
|
|
@@ -92,11 +119,9 @@ export class NotionMediaDownloader {
|
|
|
92
119
|
/** Get file extension from URL or content block metadata. */
|
|
93
120
|
getExtension(url, block) {
|
|
94
121
|
// Try from mediaType
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
return ext;
|
|
99
|
-
}
|
|
122
|
+
const ext = mimeToExtension(block.mediaType);
|
|
123
|
+
if (ext)
|
|
124
|
+
return ext;
|
|
100
125
|
// Try from URL path
|
|
101
126
|
try {
|
|
102
127
|
const pathname = new URL(url).pathname;
|
package/dist/sources/notion.js
CHANGED
|
@@ -12,7 +12,7 @@ import { NotionApiClient } from './notion-api.js';
|
|
|
12
12
|
import { blocksToSections, getPageTitle, normalizeId, } from './notion-blocks.js';
|
|
13
13
|
import { NotionMediaDownloader } from './notion-media.js';
|
|
14
14
|
import { parseDatabaseSchema, parseEntryValues, renderPropertiesSection, } from './notion-databases.js';
|
|
15
|
-
const MAX_DOCUMENT_COUNT =
|
|
15
|
+
const MAX_DOCUMENT_COUNT = 10000;
|
|
16
16
|
// ============================================
|
|
17
17
|
// Source
|
|
18
18
|
// ============================================
|
package/dist/types.d.ts
CHANGED
|
@@ -2,20 +2,66 @@
|
|
|
2
2
|
* Types for the KB migration tool
|
|
3
3
|
*/
|
|
4
4
|
/**
|
|
5
|
-
* A content block in a section
|
|
5
|
+
* A content block in a section.
|
|
6
|
+
*
|
|
7
|
+
* Discriminated union on `blockType`. The `type: 'file'` variants are internal
|
|
8
|
+
* to kb-migrate (local filesystem paths) — MoxnClient converts them to
|
|
9
|
+
* `type: 'storage'` before sending to the KB API.
|
|
6
10
|
*/
|
|
7
|
-
export
|
|
8
|
-
|
|
9
|
-
text
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
11
|
+
export type ContentBlock = TextBlock | ImageRemoteBlock | ImageFileBlock | DocumentRemoteBlock | DocumentFileBlock | CsvRemoteBlock | CsvFileBlock;
|
|
12
|
+
export interface TextBlock {
|
|
13
|
+
blockType: 'text';
|
|
14
|
+
text: string;
|
|
15
|
+
}
|
|
16
|
+
export interface ImageRemoteBlock {
|
|
17
|
+
blockType: 'image';
|
|
18
|
+
type: 'base64' | 'url' | 'storage';
|
|
19
|
+
mediaType: string;
|
|
14
20
|
base64?: string;
|
|
21
|
+
url?: string;
|
|
15
22
|
key?: string;
|
|
16
23
|
alt?: string;
|
|
24
|
+
}
|
|
25
|
+
export interface ImageFileBlock {
|
|
26
|
+
blockType: 'image';
|
|
27
|
+
type: 'file';
|
|
28
|
+
path: string;
|
|
29
|
+
mediaType: string;
|
|
30
|
+
alt?: string;
|
|
31
|
+
}
|
|
32
|
+
export interface DocumentRemoteBlock {
|
|
33
|
+
blockType: 'document';
|
|
34
|
+
type: 'base64' | 'url' | 'storage';
|
|
35
|
+
mediaType: string;
|
|
36
|
+
base64?: string;
|
|
37
|
+
url?: string;
|
|
38
|
+
key?: string;
|
|
39
|
+
filename?: string;
|
|
40
|
+
}
|
|
41
|
+
export interface DocumentFileBlock {
|
|
42
|
+
blockType: 'document';
|
|
43
|
+
type: 'file';
|
|
44
|
+
path: string;
|
|
45
|
+
mediaType: string;
|
|
46
|
+
filename?: string;
|
|
47
|
+
}
|
|
48
|
+
export interface CsvRemoteBlock {
|
|
49
|
+
blockType: 'csv';
|
|
50
|
+
type: 'base64' | 'url' | 'storage';
|
|
51
|
+
mediaType: string;
|
|
52
|
+
base64?: string;
|
|
53
|
+
url?: string;
|
|
54
|
+
key?: string;
|
|
55
|
+
filename?: string;
|
|
56
|
+
headers?: string[];
|
|
57
|
+
rowCount?: number;
|
|
58
|
+
}
|
|
59
|
+
export interface CsvFileBlock {
|
|
60
|
+
blockType: 'csv';
|
|
61
|
+
type: 'file';
|
|
62
|
+
path: string;
|
|
63
|
+
mediaType: string;
|
|
17
64
|
filename?: string;
|
|
18
|
-
/** CSV-specific metadata */
|
|
19
65
|
headers?: string[];
|
|
20
66
|
rowCount?: number;
|
|
21
67
|
}
|
package/package.json
CHANGED
|
@@ -1,10 +1,52 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@moxn/kb-migrate",
|
|
3
|
-
"version": "0.4.
|
|
3
|
+
"version": "0.4.1",
|
|
4
4
|
"description": "Migration tool for importing documents into Moxn Knowledge Base from local files, Notion, Google Docs, and more",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "dist/index.js",
|
|
7
7
|
"types": "dist/index.d.ts",
|
|
8
|
+
"exports": {
|
|
9
|
+
".": {
|
|
10
|
+
"types": "./dist/index.d.ts",
|
|
11
|
+
"default": "./dist/index.js"
|
|
12
|
+
},
|
|
13
|
+
"./sources/notion-api": {
|
|
14
|
+
"types": "./dist/sources/notion-api.d.ts",
|
|
15
|
+
"default": "./dist/sources/notion-api.js"
|
|
16
|
+
},
|
|
17
|
+
"./sources/notion-blocks": {
|
|
18
|
+
"types": "./dist/sources/notion-blocks.d.ts",
|
|
19
|
+
"default": "./dist/sources/notion-blocks.js"
|
|
20
|
+
},
|
|
21
|
+
"./sources/notion": {
|
|
22
|
+
"types": "./dist/sources/notion.d.ts",
|
|
23
|
+
"default": "./dist/sources/notion.js"
|
|
24
|
+
},
|
|
25
|
+
"./sources/notion-databases": {
|
|
26
|
+
"types": "./dist/sources/notion-databases.d.ts",
|
|
27
|
+
"default": "./dist/sources/notion-databases.js"
|
|
28
|
+
},
|
|
29
|
+
"./sources/notion-media": {
|
|
30
|
+
"types": "./dist/sources/notion-media.d.ts",
|
|
31
|
+
"default": "./dist/sources/notion-media.js"
|
|
32
|
+
},
|
|
33
|
+
"./sources/local": {
|
|
34
|
+
"types": "./dist/sources/local.d.ts",
|
|
35
|
+
"default": "./dist/sources/local.js"
|
|
36
|
+
},
|
|
37
|
+
"./sources/base": {
|
|
38
|
+
"types": "./dist/sources/base.d.ts",
|
|
39
|
+
"default": "./dist/sources/base.js"
|
|
40
|
+
},
|
|
41
|
+
"./types": {
|
|
42
|
+
"types": "./dist/types.d.ts",
|
|
43
|
+
"default": "./dist/types.js"
|
|
44
|
+
},
|
|
45
|
+
"./client": {
|
|
46
|
+
"types": "./dist/client.d.ts",
|
|
47
|
+
"default": "./dist/client.js"
|
|
48
|
+
}
|
|
49
|
+
},
|
|
8
50
|
"bin": {
|
|
9
51
|
"moxn-kb-migrate": "./bin/moxn-kb-migrate"
|
|
10
52
|
},
|